ActiveGo 1.8Package tsm1
Overview ▹
Index ▹
Constants
const (
// CompactionTempExtension is the extension used for temporary files created during compaction.
CompactionTempExtension = "tmp"
// TSMFileExtension is the extension used for TSM files.
TSMFileExtension = "tsm"
)
const (
// BlockFloat64 designates a block encodes float64 values.
BlockFloat64 = byte(0)
// BlockInteger designates a block encodes int64 values.
BlockInteger = byte(1)
// BlockBoolean designates a block encodes boolean values.
BlockBoolean = byte(2)
// BlockString designates a block encodes string values.
BlockString = byte(3)
)
const (
// DefaultSegmentSize of 10MB is the size at which segment files will be rolled over.
DefaultSegmentSize = 10 * 1024 * 1024
// WALFileExtension is the file extension we expect for wal segments.
WALFileExtension = "wal"
// WALFilePrefix is the prefix on all wal segment files.
WALFilePrefix = "_"
)
const (
// MagicNumber is written as the first 4 bytes of a data file to
// identify the file as a tsm1 formatted file
MagicNumber uint32 = 0x16D116D1
// Version indicates the version of the TSM file format.
Version byte = 1
)
Variables
var (
// ErrWALClosed is returned when attempting to write to a closed WAL file.
ErrWALClosed = fmt.Errorf("WAL closed")
// ErrWALCorrupt is returned when reading a corrupt WAL entry.
ErrWALCorrupt = fmt.Errorf("corrupted WAL entry")
)
var (
//ErrNoValues is returned when TSMWriter.WriteIndex is called and there are no values to write.
ErrNoValues = fmt.Errorf("no values written")
// ErrTSMClosed is returned when performing an operation against a closed TSM file.
ErrTSMClosed = fmt.Errorf("tsm file closed")
// ErrMaxKeyLengthExceeded is returned when attempting to write a key that is too long.
ErrMaxKeyLengthExceeded = fmt.Errorf("max key length exceeded")
// ErrMaxBlocksExceeded is returned when attempting to write a block past the allowed number.
ErrMaxBlocksExceeded = fmt.Errorf("max blocks exceeded")
)
ErrFileInUse is returned when attempting to remove or close a TSM file that is still being used.
var ErrFileInUse = fmt.Errorf("file still in use")
var (
// ErrSnapshotInProgress is returned if a snapshot is attempted while one is already running.
ErrSnapshotInProgress = fmt.Errorf("snapshot in progress")
)
func BlockCount ¶
func BlockCount(block []byte) int
BlockCount returns the number of timestamps encoded in block.
func BlockType ¶
func BlockType(block []byte) (byte, error)
BlockType returns the type of value encoded in a block or an error if the block type is unknown.
func CountTimestamps ¶
func CountTimestamps(b []byte) int
func DecodeBlock ¶
func DecodeBlock(block []byte, vals []Value) ([]Value, error)
DecodeBlock takes a byte slice and decodes it into values of the appropriate type based on the block.
func DecodeBooleanBlock ¶
func DecodeBooleanBlock(block []byte, a *[]BooleanValue) ([]BooleanValue, error)
DecodeBooleanBlock decodes the boolean block from the byte slice and appends the boolean values to a.
func DecodeFloatBlock ¶
func DecodeFloatBlock(block []byte, a *[]FloatValue) ([]FloatValue, error)
DecodeFloatBlock decodes the float block from the byte slice and appends the float values to a.
func DecodeIntegerBlock ¶
func DecodeIntegerBlock(block []byte, a *[]IntegerValue) ([]IntegerValue, error)
DecodeIntegerBlock decodes the integer block from the byte slice and appends the integer values to a.
func DecodeStringBlock ¶
func DecodeStringBlock(block []byte, a *[]StringValue) ([]StringValue, error)
DecodeStringBlock decodes the string block from the byte slice and appends the string values to a.
func ErrCacheMemorySizeLimitExceeded ¶
func ErrCacheMemorySizeLimitExceeded(n, limit uint64) error
ErrCacheMemorySizeLimitExceeded returns an error indicating an operation could not be completed due to exceeding the cache-max-memory-size setting.
func NewEngine ¶
func NewEngine(id uint64, path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine
NewEngine returns a new instance of Engine.
func NewIndirectIndex ¶
func NewIndirectIndex() *indirectIndex
NewIndirectIndex returns a new indirect index.
func NewMultiFieldCursor ¶
func NewMultiFieldCursor(fields []string, cursors []tsdb.Cursor, ascending bool) tsdb.Cursor
NewMultiFieldCursor returns an instance of Cursor that joins the results of cursors.
func ParseTSMFileName ¶
func ParseTSMFileName(name string) (int, int, error)
ParseTSMFileName parses the generation and sequence from a TSM file name.
func SeriesAndFieldFromCompositeKey ¶
func SeriesAndFieldFromCompositeKey(key []byte) ([]byte, string)
SeriesAndFieldFromCompositeKey returns the series key and the field key extracted from the composite key.
func SeriesFieldKey ¶
func SeriesFieldKey(seriesKey, field string) string
SeriesFieldKey combine a series key and field name for a unique string to be hashed to a numeric ID.
func ZigZagDecode ¶
func ZigZagDecode(v uint64) int64
ZigZagDecode converts a previously zigzag encoded uint64 back to a int64.
func ZigZagEncode ¶
func ZigZagEncode(x int64) uint64
ZigZagEncode converts a int64 to a uint64 by zig zagging negative and positive values across even and odd numbers. Eg. [0,-1,1,-2] becomes [0, 1, 2, 3].
type BitReader ¶
BitReader reads bits from an io.Reader.
type BitReader struct {
// contains filtered or unexported fields
}
func NewBitReader ¶
func NewBitReader(data []byte) *BitReader
NewBitReader returns a new instance of BitReader that reads from data.
func (*BitReader) CanReadBitFast ¶
func (r *BitReader) CanReadBitFast() bool
CanReadBitFast returns true if calling ReadBitFast() is allowed. Fast bit reads are allowed when at least 2 values are in the buffer. This is because it is not required to refilled the buffer and the caller can inline the calls.
func (*BitReader) ReadBit ¶
func (r *BitReader) ReadBit() (bool, error)
ReadBit returns the next bit from the underlying data.
func (*BitReader) ReadBitFast ¶
func (r *BitReader) ReadBitFast() bool
ReadBitFast is an optimized bit read. IMPORTANT: Only allowed if CanReadFastBit() is true!
func (*BitReader) ReadBits ¶
func (r *BitReader) ReadBits(nbits uint) (uint64, error)
ReadBits reads nbits from the underlying data into a uint64. nbits must be from 1 to 64, inclusive.
func (*BitReader) Reset ¶
func (r *BitReader) Reset(data []byte)
Reset sets the underlying reader on b and reinitializes.
type BlockIterator ¶
BlockIterator allows iterating over each block in a TSM file in order. It provides raw access to the block bytes without decoding them.
type BlockIterator struct {
// contains filtered or unexported fields
}
func (*BlockIterator) Next ¶
func (b *BlockIterator) Next() bool
Next returns true if there are more blocks to iterate through.
func (*BlockIterator) PeekNext ¶
func (b *BlockIterator) PeekNext() string
PeekNext returns the next key to be iterated or an empty string.
func (*BlockIterator) Read ¶
func (b *BlockIterator) Read() (key string, minTime int64, maxTime int64, typ byte, checksum uint32, buf []byte, err error)
Read reads information about the next block to be iterated.
type BooleanDecoder ¶
BooleanDecoder decodes a series of booleans from an in-memory buffer.
type BooleanDecoder struct {
// contains filtered or unexported fields
}
func (*BooleanDecoder) Error ¶
func (e *BooleanDecoder) Error() error
Error returns the error encountered during decoding, if one occurred.
func (*BooleanDecoder) Next ¶
func (e *BooleanDecoder) Next() bool
Next returns whether there are any bits remaining in the decoder. It returns false if there was an error decoding. The error is available on the Error method.
func (*BooleanDecoder) Read ¶
func (e *BooleanDecoder) Read() bool
Read returns the next bit from the decoder.
func (*BooleanDecoder) SetBytes ¶
func (e *BooleanDecoder) SetBytes(b []byte)
SetBytes initializes the decoder with a new set of bytes to read from. This must be called before calling any other methods.
type BooleanEncoder ¶
BooleanEncoder encodes a series of booleans to an in-memory buffer.
type BooleanEncoder struct {
// contains filtered or unexported fields
}
func NewBooleanEncoder ¶
func NewBooleanEncoder(sz int) BooleanEncoder
NewBooleanEncoder returns a new instance of BooleanEncoder.
func (*BooleanEncoder) Bytes ¶
func (e *BooleanEncoder) Bytes() ([]byte, error)
Bytes returns a new byte slice containing the encoded booleans from previous calls to Write.
func (*BooleanEncoder) Flush ¶
func (e *BooleanEncoder) Flush()
Flush is no-op
func (*BooleanEncoder) Reset ¶
func (e *BooleanEncoder) Reset()
Reset sets the encoder to its initial state.
func (*BooleanEncoder) Write ¶
func (e *BooleanEncoder) Write(b bool)
Write encodes b to the underlying buffer.
type BooleanValue ¶
BooleanValue represents a boolean value.
type BooleanValue struct {
// contains filtered or unexported fields
}
func (BooleanValue) Size ¶
func (b BooleanValue) Size() int
Size returns the number of bytes necessary to represent the value and its timestamp.
func (BooleanValue) String ¶
func (f BooleanValue) String() string
String returns the string representation of the value and its timestamp.
func (BooleanValue) UnixNano ¶
func (b BooleanValue) UnixNano() int64
UnixNano returns the timestamp of the value in nanoseconds since unix epoch.
func (BooleanValue) Value ¶
func (b BooleanValue) Value() interface{}
Value returns the underlying boolean value.
type BooleanValues ¶
BooleanValues represents a slice of Boolean values.
type BooleanValues []BooleanValue
func (BooleanValues) Deduplicate ¶
func (a BooleanValues) Deduplicate() BooleanValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.
func (BooleanValues) Encode ¶
func (a BooleanValues) Encode(buf []byte) ([]byte, error)
func (BooleanValues) Exclude ¶
func (a BooleanValues) Exclude(min, max int64) BooleanValues
Exclude returns the subset of values not in [min, max]
func (BooleanValues) Include ¶
func (a BooleanValues) Include(min, max int64) BooleanValues
Include returns the subset values between min and max inclusive.
func (BooleanValues) Len ¶
func (a BooleanValues) Len() int
Sort methods
func (BooleanValues) Less ¶
func (a BooleanValues) Less(i, j int) bool
func (BooleanValues) MaxTime ¶
func (a BooleanValues) MaxTime() int64
func (BooleanValues) Merge ¶
func (a BooleanValues) Merge(b BooleanValues) BooleanValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (BooleanValues) MinTime ¶
func (a BooleanValues) MinTime() int64
func (BooleanValues) Size ¶
func (a BooleanValues) Size() int
func (BooleanValues) Swap ¶
func (a BooleanValues) Swap(i, j int)
type Cache ¶
Cache maintains an in-memory store of Values for a set of keys.
type Cache struct {
// contains filtered or unexported fields
}
func NewCache ¶
func NewCache(maxSize uint64, path string) *Cache
NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory. Only used for engine caches, never for snapshots.
func (*Cache) ApplyEntryFn ¶
func (c *Cache) ApplyEntryFn(f func(key string, entry *entry) error) error
ApplyEntryFn applies the function f to each entry in the Cache. ApplyEntryFn calls f on each entry in turn, within the same goroutine. It is safe for use by multiple goroutines.
func (*Cache) ClearSnapshot ¶
func (c *Cache) ClearSnapshot(success bool)
ClearSnapshot removes the snapshot cache from the list of flushing caches and adjusts the size.
func (*Cache) Deduplicate ¶
func (c *Cache) Deduplicate()
Deduplicate sorts the snapshot before returning it. The compactor and any queries coming in while it writes will need the values sorted.
func (*Cache) Delete ¶
func (c *Cache) Delete(keys []string)
Delete removes all values for the given keys from the cache.
func (*Cache) DeleteRange ¶
func (c *Cache) DeleteRange(keys []string, min, max int64)
DeleteRange removes the values for all keys containing points with timestamps between between min and max from the cache.
TODO(edd): Lock usage could possibly be optimised if necessary.
func (*Cache) Keys ¶
func (c *Cache) Keys() []string
Keys returns a sorted slice of all keys under management by the cache.
func (*Cache) MaxSize ¶
func (c *Cache) MaxSize() uint64
MaxSize returns the maximum number of bytes the cache may consume.
func (*Cache) SetMaxSize ¶
func (c *Cache) SetMaxSize(size uint64)
SetMaxSize updates the memory limit of the cache.
func (*Cache) Size ¶
func (c *Cache) Size() uint64
Size returns the number of point-calcuated bytes the cache currently uses.
func (*Cache) Snapshot ¶
func (c *Cache) Snapshot() (*Cache, error)
Snapshot takes a snapshot of the current cache, adds it to the slice of caches that are being flushed, and resets the current cache with new values.
func (*Cache) Statistics ¶
func (c *Cache) Statistics(tags map[string]string) []models.Statistic
Statistics returns statistics for periodic monitoring.
func (*Cache) UpdateAge ¶
func (c *Cache) UpdateAge()
UpdateAge updates the age statistic based on the current time.
func (*Cache) UpdateCompactTime ¶
func (c *Cache) UpdateCompactTime(d time.Duration)
UpdateCompactTime updates WAL compaction time statistic based on d.
func (*Cache) Values ¶
func (c *Cache) Values(key string) Values
Values returns a copy of all values, deduped and sorted, for the given key.
func (*Cache) Write ¶
func (c *Cache) Write(key string, values []Value) error
Write writes the set of values for the key to the cache. This function is goroutine-safe. It returns an error if the cache will exceed its max size by adding the new values.
func (*Cache) WriteMulti ¶
func (c *Cache) WriteMulti(values map[string][]Value) error
WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe. It returns an error if the cache will exceeded its max size by adding the new values. The write attempts to write as many values as possible. If one key fails, the others can still succeed and an error will be returned.
type CacheLoader ¶
CacheLoader processes a set of WAL segment files, and loads a cache with the data contained within those files. Processing of the supplied files take place in the order they exist in the files slice.
type CacheLoader struct {
Logger zap.Logger
// contains filtered or unexported fields
}
func NewCacheLoader ¶
func NewCacheLoader(files []string) *CacheLoader
NewCacheLoader returns a new instance of a CacheLoader.
func (*CacheLoader) Load ¶
func (cl *CacheLoader) Load(cache *Cache) error
Load returns a cache loaded with the data contained within the segment files. If, during reading of a segment file, corruption is encountered, that segment file is truncated up to and including the last valid byte, and processing continues with the next segment file.
func (*CacheLoader) WithLogger ¶
func (cl *CacheLoader) WithLogger(log zap.Logger)
WithLogger sets the logger on the CacheLoader.
type CacheStatistics ¶
CacheStatistics hold statistics related to the cache.
type CacheStatistics struct {
MemSizeBytes int64
DiskSizeBytes int64
SnapshotCount int64
CacheAgeMs int64
CachedBytes int64
WALCompactionTimeMs int64
WriteOK int64
WriteErr int64
WriteDropped int64
}
type CompactionGroup ¶
CompactionGroup represents a list of files eligible to be compacted together.
type CompactionGroup []string
type CompactionPlanner ¶
CompactionPlanner determines what TSM files and WAL segments to include in a given compaction run.
type CompactionPlanner interface {
Plan(lastWrite time.Time) []CompactionGroup
PlanLevel(level int) []CompactionGroup
PlanOptimize() []CompactionGroup
}
type Compactor ¶
Compactor merges multiple TSM files into new files or writes a Cache into 1 or more TSM files.
type Compactor struct {
Dir string
Size int
FileStore interface {
NextGeneration() int
}
// contains filtered or unexported fields
}
func (*Compactor) Close ¶
func (c *Compactor) Close()
Close disables the Compactor.
func (*Compactor) CompactFast ¶
func (c *Compactor) CompactFast(tsmFiles []string) ([]string, error)
CompactFast writes multiple smaller TSM files into 1 or more larger files.
func (*Compactor) CompactFull ¶
func (c *Compactor) CompactFull(tsmFiles []string) ([]string, error)
CompactFull writes multiple smaller TSM files into 1 or more larger files.
func (*Compactor) DisableCompactions ¶
func (c *Compactor) DisableCompactions()
DisableSnapshots disables the compactor from performing compactions.
func (*Compactor) DisableSnapshots ¶
func (c *Compactor) DisableSnapshots()
DisableSnapshots disables the compactor from performing snapshots.
func (*Compactor) EnableCompactions ¶
func (c *Compactor) EnableCompactions()
EnableCompactions allows the compactor to perform compactions.
func (*Compactor) EnableSnapshots ¶
func (c *Compactor) EnableSnapshots()
EnableSnapshots allows the compactor to perform snapshots.
func (*Compactor) Open ¶
func (c *Compactor) Open()
Open initializes the Compactor.
func (*Compactor) WriteSnapshot ¶
func (c *Compactor) WriteSnapshot(cache *Cache) ([]string, error)
WriteSnapshot writes a Cache snapshot to one or more new TSM files.
type DefaultPlanner ¶
DefaultPlanner implements CompactionPlanner using a strategy to roll up multiple generations of TSM files into larger files in stages. It attempts to minimize the number of TSM files on disk while rolling up a bounder number of files.
type DefaultPlanner struct {
FileStore interface {
Stats() []FileStat
LastModified() time.Time
BlockCount(path string, idx int) int
}
// CompactFullWriteColdDuration specifies the length of time after
// which if no writes have been committed to the WAL, the engine will
// do a full compaction of the TSM files in this shard. This duration
// should always be greater than the CacheFlushWriteColdDuraion
CompactFullWriteColdDuration time.Duration
// contains filtered or unexported fields
}
func (*DefaultPlanner) Plan ¶
func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup
Plan returns a set of TSM files to rewrite for level 4 or higher. The planning returns multiple groups if possible to allow compactions to run concurrently.
func (*DefaultPlanner) PlanLevel ¶
func (c *DefaultPlanner) PlanLevel(level int) []CompactionGroup
PlanLevel returns a set of TSM files to rewrite for a specific level.
func (*DefaultPlanner) PlanOptimize ¶
func (c *DefaultPlanner) PlanOptimize() []CompactionGroup
PlanOptimize returns all TSM files if they are in different generations in order to optimize the index across TSM files. Each returned compaction group can be compacted concurrently.
type DeleteRangeWALEntry ¶
DeleteRangeWALEntry represents the deletion of multiple series.
type DeleteRangeWALEntry struct {
Keys []string
Min, Max int64
}
func (*DeleteRangeWALEntry) Encode ¶
func (w *DeleteRangeWALEntry) Encode(b []byte) ([]byte, error)
Encode converts the DeleteRangeWALEntry into a byte slice, appending to b.
func (*DeleteRangeWALEntry) MarshalBinary ¶
func (w *DeleteRangeWALEntry) MarshalBinary() ([]byte, error)
MarshalBinary returns a binary representation of the entry in a new byte slice.
func (*DeleteRangeWALEntry) Type ¶
func (w *DeleteRangeWALEntry) Type() WalEntryType
Type returns DeleteRangeWALEntryType.
func (*DeleteRangeWALEntry) UnmarshalBinary ¶
func (w *DeleteRangeWALEntry) UnmarshalBinary(b []byte) error
UnmarshalBinary deserializes the byte slice into w.
type DeleteWALEntry ¶
DeleteWALEntry represents the deletion of multiple series.
type DeleteWALEntry struct {
Keys []string
}
func (*DeleteWALEntry) Encode ¶
func (w *DeleteWALEntry) Encode(dst []byte) ([]byte, error)
Encode converts the DeleteWALEntry into a byte slice, appending to dst.
func (*DeleteWALEntry) MarshalBinary ¶
func (w *DeleteWALEntry) MarshalBinary() ([]byte, error)
MarshalBinary returns a binary representation of the entry in a new byte slice.
func (*DeleteWALEntry) Type ¶
func (w *DeleteWALEntry) Type() WalEntryType
Type returns DeleteWALEntryType.
func (*DeleteWALEntry) UnmarshalBinary ¶
func (w *DeleteWALEntry) UnmarshalBinary(b []byte) error
UnmarshalBinary deserializes the byte slice into w.
type EmptyValue ¶
EmptyValue is used when there is no appropriate other value.
type EmptyValue struct{}
func (EmptyValue) Size ¶
func (e EmptyValue) Size() int
Size returns 0.
func (EmptyValue) String ¶
func (e EmptyValue) String() string
String returns the empty string.
func (EmptyValue) UnixNano ¶
func (e EmptyValue) UnixNano() int64
UnixNano returns tsdb.EOF.
func (EmptyValue) Value ¶
func (e EmptyValue) Value() interface{}
Value returns nil.
type Engine ¶
Engine represents a storage engine with compressed blocks.
type Engine struct {
WAL *WAL
Cache *Cache
Compactor *Compactor
CompactionPlan CompactionPlanner
FileStore *FileStore
MaxPointsPerBlock int
// CacheFlushMemorySizeThreshold specifies the minimum size threshodl for
// the cache when the engine should write a snapshot to a TSM file
CacheFlushMemorySizeThreshold uint64
// CacheFlushWriteColdDuration specifies the length of time after which if
// no writes have been committed to the WAL, the engine will write
// a snapshot of the cache to a TSM file
CacheFlushWriteColdDuration time.Duration
// contains filtered or unexported fields
}
func (*Engine) Backup ¶
func (e *Engine) Backup(w io.Writer, basePath string, since time.Time) error
Backup writes a tar archive of any TSM files modified since the passed in time to the passed in writer. The basePath will be prepended to the names of the files in the archive. It will force a snapshot of the WAL first then perform the backup with a read lock against the file store. This means that new TSM files will not be able to be created in this shard while the backup is running. For shards that are still acively getting writes, this could cause the WAL to backup, increasing memory usage and evenutally rejecting writes.
func (*Engine) Close ¶
func (e *Engine) Close() error
Close closes the engine. Subsequent calls to Close are a nop.
func (*Engine) ContainsSeries ¶
func (e *Engine) ContainsSeries(keys []string) (map[string]bool, error)
ContainsSeries returns a map of keys indicating whether the key exists and has values or not.
func (*Engine) CreateIterator ¶
func (e *Engine) CreateIterator(measurement string, opt influxql.IteratorOptions) (influxql.Iterator, error)
CreateIterator returns an iterator for the measurement based on opt.
func (*Engine) CreateSnapshot ¶
func (e *Engine) CreateSnapshot() (string, error)
CreateSnapshot will create a temp directory that holds temporary hardlinks to the underylyng shard files.
func (*Engine) DeleteMeasurement ¶
func (e *Engine) DeleteMeasurement(name string, seriesKeys []string) error
DeleteMeasurement deletes a measurement and all related series.
func (*Engine) DeleteSeries ¶
func (e *Engine) DeleteSeries(seriesKeys []string) error
DeleteSeries removes all series keys from the engine.
func (*Engine) DeleteSeriesRange ¶
func (e *Engine) DeleteSeriesRange(seriesKeys []string, min, max int64) error
DeleteSeriesRange removes the values between min and max (inclusive) from all series.
func (*Engine) Format ¶
func (e *Engine) Format() tsdb.EngineFormat
Format returns the format type of this engine.
func (*Engine) Index ¶
func (e *Engine) Index() *tsdb.DatabaseIndex
Index returns the database index.
func (*Engine) KeyCursor ¶
func (e *Engine) KeyCursor(key string, t int64, ascending bool) *KeyCursor
KeyCursor returns a KeyCursor for the given key starting at time t.
func (*Engine) LastModified ¶
func (e *Engine) LastModified() time.Time
LastModified returns the time when this shard was last modified.
func (*Engine) LoadMetadataIndex ¶
func (e *Engine) LoadMetadataIndex(shardID uint64, index *tsdb.DatabaseIndex) error
LoadMetadataIndex loads the shard metadata into memory.
func (*Engine) MeasurementFields ¶
func (e *Engine) MeasurementFields(measurement string) *tsdb.MeasurementFields
MeasurementFields returns the measurement fields for a measurement.
func (*Engine) Open ¶
func (e *Engine) Open() error
Open opens and initializes the engine.
func (*Engine) Path ¶
func (e *Engine) Path() string
Path returns the path the engine was opened with.
func (*Engine) Restore ¶
func (e *Engine) Restore(r io.Reader, basePath string) error
Restore reads a tar archive generated by Backup(). Only files that match basePath will be copied into the directory. This obtains a write lock so no operations can be performed while restoring.
func (*Engine) SeriesCount ¶
func (e *Engine) SeriesCount() (n int, err error)
SeriesCount returns the number of series buckets on the shard.
func (*Engine) SetCompactionsEnabled ¶
func (e *Engine) SetCompactionsEnabled(enabled bool)
SetCompactionsEnabled enables compactions on the engine. When disabled all running compactions are aborted and new compactions stop running.
func (*Engine) SetEnabled ¶
func (e *Engine) SetEnabled(enabled bool)
SetEnabled sets whether the engine is enabled.
func (*Engine) ShouldCompactCache ¶
func (e *Engine) ShouldCompactCache(lastWriteTime time.Time) bool
ShouldCompactCache returns true if the Cache is over its flush threshold or if the passed in lastWriteTime is older than the write cold threshold.
func (*Engine) Statistics ¶
func (e *Engine) Statistics(tags map[string]string) []models.Statistic
Statistics returns statistics for periodic monitoring.
func (*Engine) WithLogger ¶
func (e *Engine) WithLogger(log zap.Logger)
WithLogger sets the logger for the engine.
func (*Engine) WritePoints ¶
func (e *Engine) WritePoints(points []models.Point) error
WritePoints writes metadata and point data into the engine. It returns an error if new points are added to an existing key.
func (*Engine) WriteSnapshot ¶
func (e *Engine) WriteSnapshot() error
WriteSnapshot will snapshot the cache and write a new TSM file with its contents, releasing the snapshot when done.
func (*Engine) WriteTo ¶
func (e *Engine) WriteTo(w io.Writer) (n int64, err error)
WriteTo is not implemented.
type EngineStatistics ¶
EngineStatistics maintains statistics for the engine.
type EngineStatistics struct {
CacheCompactions int64 // Counter of cache compactions that have ever run.
CacheCompactionsActive int64 // Gauge of cache compactions currently running.
CacheCompactionErrors int64 // Counter of cache compactions that have failed due to error.
CacheCompactionDuration int64 // Counter of number of wall nanoseconds spent in cache compactions.
TSMCompactions [3]int64 // Counter of TSM compactions (by level) that have ever run.
TSMCompactionsActive [3]int64 // Gauge of TSM compactions (by level) currently running.
TSMCompactionErrors [3]int64 // Counter of TSM compcations (by level) that have failed due to error.
TSMCompactionDuration [3]int64 // Counter of number of wall nanoseconds spent in TSM compactions (by level).
TSMOptimizeCompactions int64 // Counter of optimize compactions that have ever run.
TSMOptimizeCompactionsActive int64 // Gauge of optimize compactions currently running.
TSMOptimizeCompactionErrors int64 // Counter of optimize compactions that have failed due to error.
TSMOptimizeCompactionDuration int64 // Counter of number of wall nanoseconds spent in optimize compactions.
TSMFullCompactions int64 // Counter of full compactions that have ever run.
TSMFullCompactionsActive int64 // Gauge of full compactions currently running.
TSMFullCompactionErrors int64 // Counter of full compactions that have failed due to error.
TSMFullCompactionDuration int64 // Counter of number of wall nanoseconds spent in full compactions.
}
type FileStat ¶
FileStat holds information about a TSM file on disk.
type FileStat struct {
Path string
HasTombstone bool
Size uint32
LastModified int64
MinTime, MaxTime int64
MinKey, MaxKey string
}
func (FileStat) ContainsKey ¶
func (f FileStat) ContainsKey(key string) bool
ContainsKey returns true if the min and max keys of the file overlap the arguments min and max.
func (FileStat) OverlapsKeyRange ¶
func (f FileStat) OverlapsKeyRange(min, max string) bool
OverlapsKeyRange returns true if the min and max keys of the file overlap the arguments min and max.
func (FileStat) OverlapsTimeRange ¶
func (f FileStat) OverlapsTimeRange(min, max int64) bool
OverlapsTimeRange returns true if the time range of the file intersect min and max.
type FileStore ¶
FileStore is an abstraction around multiple TSM files.
type FileStore struct {
// contains filtered or unexported fields
}
func NewFileStore ¶
func NewFileStore(dir string) *FileStore
NewFileStore returns a new instance of FileStore based on the given directory.
func (*FileStore) Add ¶
func (f *FileStore) Add(files ...TSMFile)
Add adds the given files to the file store's list of files.
func (*FileStore) BlockCount ¶
func (f *FileStore) BlockCount(path string, idx int) int
BlockCount returns number of values stored in the block at location idx in the file at path. If path does not match any file in the store, 0 is returned. If idx is out of range for the number of blocks in the file, 0 is returned.
func (*FileStore) Close ¶
func (f *FileStore) Close() error
Close closes the file store.
func (*FileStore) Count ¶
func (f *FileStore) Count() int
Count returns the number of TSM files currently loaded.
func (*FileStore) CreateSnapshot ¶
func (f *FileStore) CreateSnapshot() (string, error)
CreateSnapshot creates hardlinks for all tsm and tombstone files in the path provided.
func (*FileStore) CurrentGeneration ¶
func (f *FileStore) CurrentGeneration() int
CurrentGeneration returns the current generation of the TSM files.
func (*FileStore) Delete ¶
func (f *FileStore) Delete(keys []string) error
Delete removes the keys from the set of keys available in this file.
func (*FileStore) DeleteRange ¶
func (f *FileStore) DeleteRange(keys []string, min, max int64) error
DeleteRange removes the values for keys between timestamps min and max.
func (*FileStore) Files ¶
func (f *FileStore) Files() []TSMFile
Files returns the slice of TSM files currently loaded.
func (*FileStore) KeyCursor ¶
func (f *FileStore) KeyCursor(key string, t int64, ascending bool) *KeyCursor
KeyCursor returns a KeyCursor for key and t across the files in the FileStore.
func (*FileStore) Keys ¶
func (f *FileStore) Keys() map[string]byte
Keys returns all keys and types for all files in the file store.
func (*FileStore) LastModified ¶
func (f *FileStore) LastModified() time.Time
LastModified returns the last time the file store was updated with new TSM files or a delete.
func (*FileStore) NextGeneration ¶
func (f *FileStore) NextGeneration() int
NextGeneration increments the max file ID and returns the new value.
func (*FileStore) Open ¶
func (f *FileStore) Open() error
Open loads all the TSM files in the configured directory.
func (*FileStore) Read ¶
func (f *FileStore) Read(key string, t int64) ([]Value, error)
Read returns the slice of values for the given key and the given timestamp, if any file matches those constraints.
func (*FileStore) Remove ¶
func (f *FileStore) Remove(paths ...string)
Remove removes the files with matching paths from the set of active files. It does not remove the paths from disk.
func (*FileStore) Replace ¶
func (f *FileStore) Replace(oldFiles, newFiles []string) error
Replace replaces oldFiles with newFiles.
func (*FileStore) Statistics ¶
func (f *FileStore) Statistics(tags map[string]string) []models.Statistic
Statistics returns statistics for periodic monitoring.
func (*FileStore) Stats ¶
func (f *FileStore) Stats() []FileStat
Stats returns the stats of the underlying files, preferring the cached version if it is still valid.
func (*FileStore) Type ¶
func (f *FileStore) Type(key string) (byte, error)
Type returns the type of values store at the block for key.
func (*FileStore) WalkKeys ¶
func (f *FileStore) WalkKeys(fn func(key []byte, typ byte) error) error
WalkKeys calls fn for every key in every TSM file known to the FileStore. If the key exists in multiple files, it will be invoked for each file.
func (*FileStore) WithLogger ¶
func (f *FileStore) WithLogger(log zap.Logger)
WithLogger sets the logger on the file store.
type FileStoreStatistics ¶
FileStoreStatistics keeps statistics about the file store.
type FileStoreStatistics struct {
DiskBytes int64
FileCount int64
}
type FloatDecoder ¶
FloatDecoder decodes a byte slice into multiple float64 values.
type FloatDecoder struct {
// contains filtered or unexported fields
}
func (*FloatDecoder) Error ¶
func (it *FloatDecoder) Error() error
Error returns the current decoding error.
func (*FloatDecoder) Next ¶
func (it *FloatDecoder) Next() bool
Next returns true if there are remaining values to read.
func (*FloatDecoder) SetBytes ¶
func (it *FloatDecoder) SetBytes(b []byte) error
SetBytes initializes the decoder with b. Must call before calling Next().
func (*FloatDecoder) Values ¶
func (it *FloatDecoder) Values() float64
Values returns the current float64 value.
type FloatEncoder ¶
FloatEncoder encodes multiple float64s into a byte slice.
type FloatEncoder struct {
// contains filtered or unexported fields
}
func NewFloatEncoder ¶
func NewFloatEncoder() *FloatEncoder
NewFloatEncoder returns a new FloatEncoder.
func (*FloatEncoder) Bytes ¶
func (s *FloatEncoder) Bytes() ([]byte, error)
Bytes returns a copy of the underlying byte buffer used in the encoder.
func (*FloatEncoder) Flush ¶
func (s *FloatEncoder) Flush()
Flush indicates there are no more values to encode.
func (*FloatEncoder) Reset ¶
func (s *FloatEncoder) Reset()
Reset sets the encoder back to its initial state.
func (*FloatEncoder) Write ¶
func (s *FloatEncoder) Write(v float64)
Write encodes v to the underlying buffer.
type FloatValue ¶
FloatValue represents a float64 value.
type FloatValue struct {
// contains filtered or unexported fields
}
func (FloatValue) Size ¶
func (f FloatValue) Size() int
Size returns the number of bytes necessary to represent the value and its timestamp.
func (FloatValue) String ¶
func (f FloatValue) String() string
String returns the string representation of the value and its timestamp.
func (FloatValue) UnixNano ¶
func (f FloatValue) UnixNano() int64
UnixNano returns the timestamp of the value.
func (FloatValue) Value ¶
func (f FloatValue) Value() interface{}
Value returns the underlying float64 value.
type FloatValues ¶
FloatValues represents a slice of Float values.
type FloatValues []FloatValue
func (FloatValues) Deduplicate ¶
func (a FloatValues) Deduplicate() FloatValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.
func (FloatValues) Encode ¶
func (a FloatValues) Encode(buf []byte) ([]byte, error)
func (FloatValues) Exclude ¶
func (a FloatValues) Exclude(min, max int64) FloatValues
Exclude returns the subset of values not in [min, max]
func (FloatValues) Include ¶
func (a FloatValues) Include(min, max int64) FloatValues
Include returns the subset values between min and max inclusive.
func (FloatValues) Len ¶
func (a FloatValues) Len() int
Sort methods
func (FloatValues) Less ¶
func (a FloatValues) Less(i, j int) bool
func (FloatValues) MaxTime ¶
func (a FloatValues) MaxTime() int64
func (FloatValues) Merge ¶
func (a FloatValues) Merge(b FloatValues) FloatValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (FloatValues) MinTime ¶
func (a FloatValues) MinTime() int64
func (FloatValues) Size ¶
func (a FloatValues) Size() int
func (FloatValues) Swap ¶
func (a FloatValues) Swap(i, j int)
type IndexEntry ¶
IndexEntry is the index information for a given block in a TSM file.
type IndexEntry struct {
// The min and max time of all points stored in the block.
MinTime, MaxTime int64
// The absolute position in the file where this block is located.
Offset int64
// The size in bytes of the block in the file.
Size uint32
}
func (*IndexEntry) AppendTo ¶
func (e *IndexEntry) AppendTo(b []byte) []byte
AppendTo writes a binary-encoded version of IndexEntry to b, allocating and returning a new slice, if necessary.
func (*IndexEntry) Contains ¶
func (e *IndexEntry) Contains(t int64) bool
Contains returns true if this IndexEntry may contain values for the given time. The min and max times are inclusive.
func (*IndexEntry) OverlapsTimeRange ¶
func (e *IndexEntry) OverlapsTimeRange(min, max int64) bool
OverlapsTimeRange returns true if the given time ranges are completely within the entry's time bounds.
func (*IndexEntry) String ¶
func (e *IndexEntry) String() string
String returns a string representation of the entry.
func (*IndexEntry) UnmarshalBinary ¶
func (e *IndexEntry) UnmarshalBinary(b []byte) error
UnmarshalBinary decodes an IndexEntry from a byte slice.
type IndexWriter ¶
IndexWriter writes a TSMIndex.
type IndexWriter interface {
// Add records a new block entry for a key in the index.
Add(key string, blockType byte, minTime, maxTime int64, offset int64, size uint32)
// Entries returns all index entries for a key.
Entries(key string) []IndexEntry
// Keys returns the unique set of keys in the index.
Keys() []string
// KeyCount returns the count of unique keys in the index.
KeyCount() int
// Size returns the size of a the current index in bytes.
Size() uint32
// MarshalBinary returns a byte slice encoded version of the index.
MarshalBinary() ([]byte, error)
// WriteTo writes the index contents to a writer.
WriteTo(w io.Writer) (int64, error)
}
func NewIndexWriter ¶
func NewIndexWriter() IndexWriter
NewIndexWriter returns a new IndexWriter.
type IntegerDecoder ¶
IntegerDecoder decodes a byte slice into int64s.
type IntegerDecoder struct {
// contains filtered or unexported fields
}
func (*IntegerDecoder) Error ¶
func (d *IntegerDecoder) Error() error
Error returns the last error encountered by the decoder.
func (*IntegerDecoder) Next ¶
func (d *IntegerDecoder) Next() bool
Next returns true if there are any values remaining to be decoded.
func (*IntegerDecoder) Read ¶
func (d *IntegerDecoder) Read() int64
Read returns the next value from the decoder.
func (*IntegerDecoder) SetBytes ¶
func (d *IntegerDecoder) SetBytes(b []byte)
SetBytes sets the underlying byte slice of the decoder.
type IntegerEncoder ¶
IntegerEncoder encodes int64s into byte slices.
type IntegerEncoder struct {
// contains filtered or unexported fields
}
func NewIntegerEncoder ¶
func NewIntegerEncoder(sz int) IntegerEncoder
NewIntegerEncoder returns a new integer encoder with an initial buffer of values sized at sz.
func (*IntegerEncoder) Bytes ¶
func (e *IntegerEncoder) Bytes() ([]byte, error)
Bytes returns a copy of the underlying buffer.
func (*IntegerEncoder) Flush ¶
func (e *IntegerEncoder) Flush()
Flush is no-op
func (*IntegerEncoder) Reset ¶
func (e *IntegerEncoder) Reset()
Reset sets the encoder back to its initial state.
func (*IntegerEncoder) Write ¶
func (e *IntegerEncoder) Write(v int64)
Write encodes v to the underlying buffers.
type IntegerValue ¶
FloatValue represents an int64 value.
type IntegerValue struct {
// contains filtered or unexported fields
}
func (IntegerValue) Size ¶
func (v IntegerValue) Size() int
Size returns the number of bytes necessary to represent the value and its timestamp.
func (IntegerValue) String ¶
func (f IntegerValue) String() string
String returns the string representation of the value and its timestamp.
func (IntegerValue) UnixNano ¶
func (v IntegerValue) UnixNano() int64
UnixNano returns the timestamp of the value.
func (IntegerValue) Value ¶
func (v IntegerValue) Value() interface{}
Value returns the underlying int64 value.
type IntegerValues ¶
IntegerValues represents a slice of Integer values.
type IntegerValues []IntegerValue
func (IntegerValues) Deduplicate ¶
func (a IntegerValues) Deduplicate() IntegerValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.
func (IntegerValues) Encode ¶
func (a IntegerValues) Encode(buf []byte) ([]byte, error)
func (IntegerValues) Exclude ¶
func (a IntegerValues) Exclude(min, max int64) IntegerValues
Exclude returns the subset of values not in [min, max]
func (IntegerValues) Include ¶
func (a IntegerValues) Include(min, max int64) IntegerValues
Include returns the subset values between min and max inclusive.
func (IntegerValues) Len ¶
func (a IntegerValues) Len() int
Sort methods
func (IntegerValues) Less ¶
func (a IntegerValues) Less(i, j int) bool
func (IntegerValues) MaxTime ¶
func (a IntegerValues) MaxTime() int64
func (IntegerValues) Merge ¶
func (a IntegerValues) Merge(b IntegerValues) IntegerValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (IntegerValues) MinTime ¶
func (a IntegerValues) MinTime() int64
func (IntegerValues) Size ¶
func (a IntegerValues) Size() int
func (IntegerValues) Swap ¶
func (a IntegerValues) Swap(i, j int)
type KeyCursor ¶
KeyCursor allows iteration through keys in a set of files within a FileStore.
type KeyCursor struct {
// contains filtered or unexported fields
}
func (*KeyCursor) Close ¶
func (c *KeyCursor) Close()
Close removes all references on the cursor.
func (*KeyCursor) Next ¶
func (c *KeyCursor) Next()
Next moves the cursor to the next position. Data should be read by the ReadBlock functions.
func (*KeyCursor) ReadBooleanBlock ¶
func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error)
ReadBooleanBlock reads the next block as a set of boolean values.
func (*KeyCursor) ReadFloatBlock ¶
func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error)
ReadFloatBlock reads the next block as a set of float values.
func (*KeyCursor) ReadIntegerBlock ¶
func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error)
ReadIntegerBlock reads the next block as a set of integer values.
func (*KeyCursor) ReadStringBlock ¶
func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error)
ReadStringBlock reads the next block as a set of string values.
type KeyIterator ¶
KeyIterator allows iteration over set of keys and values in sorted order.
type KeyIterator interface {
// Next returns true if there are any values remaining in the iterator.
Next() bool
// Read returns the key, time range, and raw data for the next block,
// or any error that occurred.
Read() (key string, minTime int64, maxTime int64, data []byte, err error)
// Close closes the iterator.
Close() error
}
func NewCacheKeyIterator ¶
func NewCacheKeyIterator(cache *Cache, size int) KeyIterator
NewCacheKeyIterator returns a new KeyIterator from a Cache.
func NewTSMKeyIterator ¶
func NewTSMKeyIterator(size int, fast bool, readers ...*TSMReader) (KeyIterator, error)
NewTSMKeyIterator returns a new TSM key iterator from readers. size indicates the maximum number of values to encode in a single block.
type SegmentInfo ¶
SegmentInfo represents metadata about a segment.
type SegmentInfo struct {
// contains filtered or unexported fields
}
type StringDecoder ¶
StringDecoder decodes a byte slice into strings.
type StringDecoder struct {
// contains filtered or unexported fields
}
func (*StringDecoder) Error ¶
func (e *StringDecoder) Error() error
Error returns the last error encountered by the decoder.
func (*StringDecoder) Next ¶
func (e *StringDecoder) Next() bool
Next returns true if there are any values remaining to be decoded.
func (*StringDecoder) Read ¶
func (e *StringDecoder) Read() string
Read returns the next value from the decoder.
func (*StringDecoder) SetBytes ¶
func (e *StringDecoder) SetBytes(b []byte) error
SetBytes initializes the decoder with bytes to read from. This must be called before calling any other method.
type StringEncoder ¶
StringEncoder encodes multiple strings into a byte slice.
type StringEncoder struct {
// contains filtered or unexported fields
}
func NewStringEncoder ¶
func NewStringEncoder(sz int) StringEncoder
NewStringEncoder returns a new StringEncoder with an initial buffer ready to hold sz bytes.
func (*StringEncoder) Bytes ¶
func (e *StringEncoder) Bytes() ([]byte, error)
Bytes returns a copy of the underlying buffer.
func (*StringEncoder) Flush ¶
func (e *StringEncoder) Flush()
Flush is no-op
func (*StringEncoder) Reset ¶
func (e *StringEncoder) Reset()
Reset sets the encoder back to its initial state.
func (*StringEncoder) Write ¶
func (e *StringEncoder) Write(s string)
Write encodes s to the underlying buffer.
type StringValue ¶
StringValue represents a string value.
type StringValue struct {
// contains filtered or unexported fields
}
func (StringValue) Size ¶
func (v StringValue) Size() int
Size returns the number of bytes necessary to represent the value and its timestamp.
func (StringValue) String ¶
func (f StringValue) String() string
String returns the string representation of the value and its timestamp.
func (StringValue) UnixNano ¶
func (v StringValue) UnixNano() int64
UnixNano returns the timestamp of the value.
func (StringValue) Value ¶
func (v StringValue) Value() interface{}
Value returns the underlying string value.
type StringValues ¶
StringValues represents a slice of String values.
type StringValues []StringValue
func (StringValues) Deduplicate ¶
func (a StringValues) Deduplicate() StringValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.
func (StringValues) Encode ¶
func (a StringValues) Encode(buf []byte) ([]byte, error)
func (StringValues) Exclude ¶
func (a StringValues) Exclude(min, max int64) StringValues
Exclude returns the subset of values not in [min, max]
func (StringValues) Include ¶
func (a StringValues) Include(min, max int64) StringValues
Include returns the subset values between min and max inclusive.
func (StringValues) Len ¶
func (a StringValues) Len() int
Sort methods
func (StringValues) Less ¶
func (a StringValues) Less(i, j int) bool
func (StringValues) MaxTime ¶
func (a StringValues) MaxTime() int64
func (StringValues) Merge ¶
func (a StringValues) Merge(b StringValues) StringValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (StringValues) MinTime ¶
func (a StringValues) MinTime() int64
func (StringValues) Size ¶
func (a StringValues) Size() int
func (StringValues) Swap ¶
func (a StringValues) Swap(i, j int)
type TSMFile ¶
TSMFile represents an on-disk TSM file.
type TSMFile interface {
// Path returns the underlying file path for the TSMFile. If the file
// has not be written or loaded from disk, the zero value is returned.
Path() string
// Read returns all the values in the block where time t resides.
Read(key string, t int64) ([]Value, error)
// ReadAt returns all the values in the block identified by entry.
ReadAt(entry *IndexEntry, values []Value) ([]Value, error)
ReadFloatBlockAt(entry *IndexEntry, values *[]FloatValue) ([]FloatValue, error)
ReadIntegerBlockAt(entry *IndexEntry, values *[]IntegerValue) ([]IntegerValue, error)
ReadStringBlockAt(entry *IndexEntry, values *[]StringValue) ([]StringValue, error)
ReadBooleanBlockAt(entry *IndexEntry, values *[]BooleanValue) ([]BooleanValue, error)
// Entries returns the index entries for all blocks for the given key.
Entries(key string) []IndexEntry
ReadEntries(key string, entries *[]IndexEntry)
// Returns true if the TSMFile may contain a value with the specified
// key and time.
ContainsValue(key string, t int64) bool
// Contains returns true if the file contains any values for the given
// key.
Contains(key string) bool
// TimeRange returns the min and max time across all keys in the file.
TimeRange() (int64, int64)
// TombstoneRange returns ranges of time that are deleted for the given key.
TombstoneRange(key string) []TimeRange
// KeyRange returns the min and max keys in the file.
KeyRange() (string, string)
// KeyCount returns the number of distinct keys in the file.
KeyCount() int
// KeyAt returns the key located at index position idx.
KeyAt(idx int) ([]byte, byte)
// Type returns the block type of the values stored for the key. Returns one of
// BlockFloat64, BlockInt64, BlockBoolean, BlockString. If key does not exist,
// an error is returned.
Type(key string) (byte, error)
// Delete removes the keys from the set of keys available in this file.
Delete(keys []string) error
// DeleteRange removes the values for keys between timestamps min and max.
DeleteRange(keys []string, min, max int64) error
// HasTombstones returns true if file contains values that have been deleted.
HasTombstones() bool
// TombstoneFiles returns the tombstone filestats if there are any tombstones
// written for this file.
TombstoneFiles() []FileStat
// Close closes the underlying file resources.
Close() error
// Size returns the size of the file on disk in bytes.
Size() uint32
// Rename renames the existing TSM file to a new name and replaces the mmap backing slice using the new
// file name. Index and Reader state are not re-initialized.
Rename(path string) error
// Remove deletes the file from the filesystem.
Remove() error
// InUse returns true if the file is currently in use by queries.
InUse() bool
// Ref records that this file is actively in use.
Ref()
// Unref records that this file is no longer in use.
Unref()
// Stats returns summary information about the TSM file.
Stats() FileStat
// BlockIterator returns an iterator pointing to the first block in the file and
// allows sequential iteration to each and every block.
BlockIterator() *BlockIterator
}
type TSMIndex ¶
TSMIndex represent the index section of a TSM file. The index records all blocks, their locations, sizes, min and max times.
type TSMIndex interface {
// Delete removes the given keys from the index.
Delete(keys []string)
// DeleteRange removes the given keys with data between minTime and maxTime from the index.
DeleteRange(keys []string, minTime, maxTime int64)
// Contains return true if the given key exists in the index.
Contains(key string) bool
// ContainsValue returns true if key and time might exist in this file. This function could
// return true even though the actual point does not exists. For example, the key may
// exist in this file, but not have a point exactly at time t.
ContainsValue(key string, timestamp int64) bool
// Entries returns all index entries for a key.
Entries(key string) []IndexEntry
// ReadEntries reads the index entries for key into entries.
ReadEntries(key string, entries *[]IndexEntry)
// Entry returns the index entry for the specified key and timestamp. If no entry
// matches the key and timestamp, nil is returned.
Entry(key string, timestamp int64) *IndexEntry
// Key returns the key in the index at the given position.
Key(index int) (string, byte, []IndexEntry)
// KeyAt returns the key in the index at the given position.
KeyAt(index int) ([]byte, byte)
// KeyCount returns the count of unique keys in the index.
KeyCount() int
// Size returns the size of the current index in bytes.
Size() uint32
// TimeRange returns the min and max time across all keys in the file.
TimeRange() (int64, int64)
// TombstoneRange returns ranges of time that are deleted for the given key.
TombstoneRange(key string) []TimeRange
// KeyRange returns the min and max keys in the file.
KeyRange() (string, string)
// Type returns the block type of the values stored for the key. Returns one of
// BlockFloat64, BlockInt64, BlockBool, BlockString. If key does not exist,
// an error is returned.
Type(key string) (byte, error)
// UnmarshalBinary populates an index from an encoded byte slice
// representation of an index.
UnmarshalBinary(b []byte) error
}
type TSMReader ¶
TSMReader is a reader for a TSM file.
type TSMReader struct {
// contains filtered or unexported fields
}
func NewTSMReader ¶
func NewTSMReader(f *os.File) (*TSMReader, error)
NewTSMReader returns a new TSMReader from the given file.
func (*TSMReader) BlockIterator ¶
func (t *TSMReader) BlockIterator() *BlockIterator
BlockIterator returns a BlockIterator for the underlying TSM file.
func (*TSMReader) Close ¶
func (t *TSMReader) Close() error
Close closes the TSMReader.
func (*TSMReader) Contains ¶
func (t *TSMReader) Contains(key string) bool
Contains returns whether the given key is present in the index.
func (*TSMReader) ContainsValue ¶
func (t *TSMReader) ContainsValue(key string, ts int64) bool
ContainsValue returns true if key and time might exists in this file. This function could return true even though the actual point does not exist. For example, the key may exist in this file, but not have a point exactly at time t.
func (*TSMReader) Delete ¶
func (t *TSMReader) Delete(keys []string) error
Delete deletes blocks indicated by keys.
func (*TSMReader) DeleteRange ¶
func (t *TSMReader) DeleteRange(keys []string, minTime, maxTime int64) error
DeleteRange removes the given points for keys between minTime and maxTime.
func (*TSMReader) Entries ¶
func (t *TSMReader) Entries(key string) []IndexEntry
Entries returns all index entries for key.
func (*TSMReader) HasTombstones ¶
func (t *TSMReader) HasTombstones() bool
HasTombstones return true if there are any tombstone entries recorded.
func (*TSMReader) InUse ¶
func (t *TSMReader) InUse() bool
InUse returns whether the TSMReader currently has any active references.
func (*TSMReader) IndexSize ¶
func (t *TSMReader) IndexSize() uint32
IndexSize returns the size of the index in bytes.
func (*TSMReader) Key ¶
func (t *TSMReader) Key(index int) (string, byte, []IndexEntry)
Key returns the key and the underlying entry at the numeric index.
func (*TSMReader) KeyAt ¶
func (t *TSMReader) KeyAt(idx int) ([]byte, byte)
KeyAt returns the key and key type at position idx in the index.
func (*TSMReader) KeyCount ¶
func (t *TSMReader) KeyCount() int
KeyCount returns the count of unique keys in the TSMReader.
func (*TSMReader) KeyRange ¶
func (t *TSMReader) KeyRange() (string, string)
KeyRange returns the min and max key across all keys in the file.
func (*TSMReader) LastModified ¶
func (t *TSMReader) LastModified() int64
LastModified returns the last time the underlying file was modified.
func (*TSMReader) Path ¶
func (t *TSMReader) Path() string
Path returns the path of the file the TSMReader was initialized with.
func (*TSMReader) Read ¶
func (t *TSMReader) Read(key string, timestamp int64) ([]Value, error)
Read returns the values corresponding to the block at the given key and timestamp.
func (*TSMReader) ReadAll ¶
func (t *TSMReader) ReadAll(key string) ([]Value, error)
ReadAll returns all values for a key in all blocks.
func (*TSMReader) ReadAt ¶
func (t *TSMReader) ReadAt(entry *IndexEntry, vals []Value) ([]Value, error)
ReadAt returns the values corresponding to the given index entry.
func (*TSMReader) ReadBooleanBlockAt ¶
func (t *TSMReader) ReadBooleanBlockAt(entry *IndexEntry, vals *[]BooleanValue) ([]BooleanValue, error)
ReadBooleanBlockAt returns the boolean values corresponding to the given index entry.
func (*TSMReader) ReadEntries ¶
func (t *TSMReader) ReadEntries(key string, entries *[]IndexEntry)
ReadEntries reads the index entries for key into entries.
func (*TSMReader) ReadFloatBlockAt ¶
func (t *TSMReader) ReadFloatBlockAt(entry *IndexEntry, vals *[]FloatValue) ([]FloatValue, error)
ReadFloatBlockAt returns the float values corresponding to the given index entry.
func (*TSMReader) ReadIntegerBlockAt ¶
func (t *TSMReader) ReadIntegerBlockAt(entry *IndexEntry, vals *[]IntegerValue) ([]IntegerValue, error)
ReadIntegerBlockAt returns the integer values corresponding to the given index entry.
func (*TSMReader) ReadStringBlockAt ¶
func (t *TSMReader) ReadStringBlockAt(entry *IndexEntry, vals *[]StringValue) ([]StringValue, error)
ReadStringBlockAt returns the string values corresponding to the given index entry.
func (*TSMReader) Ref ¶
func (t *TSMReader) Ref()
Ref records a usage of this TSMReader. If there are active references when the reader is closed or removed, the reader will remain open until there are no more references.
func (*TSMReader) Remove ¶
func (t *TSMReader) Remove() error
Remove removes any underlying files stored on disk for this reader.
func (*TSMReader) Rename ¶
func (t *TSMReader) Rename(path string) error
Rename renames the underlying file to the new path.
func (*TSMReader) Size ¶
func (t *TSMReader) Size() uint32
Size returns the size of the underlying file in bytes.
func (*TSMReader) Stats ¶
func (t *TSMReader) Stats() FileStat
Stats returns the FileStat for the TSMReader's underlying file.
func (*TSMReader) TimeRange ¶
func (t *TSMReader) TimeRange() (int64, int64)
TimeRange returns the min and max time across all keys in the file.
func (*TSMReader) TombstoneFiles ¶
func (t *TSMReader) TombstoneFiles() []FileStat
TombstoneFiles returns any tombstone files associated with this TSM file.
func (*TSMReader) TombstoneRange ¶
func (t *TSMReader) TombstoneRange(key string) []TimeRange
TombstoneRange returns ranges of time that are deleted for the given key.
func (*TSMReader) Type ¶
func (t *TSMReader) Type(key string) (byte, error)
Type returns the type of values stored at the given key.
func (*TSMReader) Unref ¶
func (t *TSMReader) Unref()
Unref removes a usage record of this TSMReader. If the Reader was closed by another goroutine while there were active references, the file will be closed and remove
type TSMWriter ¶
TSMWriter writes TSM formatted key and values.
type TSMWriter interface {
// Write writes a new block for key containing and values. Writes append
// blocks in the order that the Write function is called. The caller is
// responsible for ensuring keys and blocks are sorted appropriately.
// Values are encoded as a full block. The caller is responsible for
// ensuring a fixed number of values are encoded in each block as well as
// ensuring the Values are sorted. The first and last timestamp values are
// used as the minimum and maximum values for the index entry.
Write(key string, values Values) error
// WriteBlock writes a new block for key containing the bytes in block. WriteBlock appends
// blocks in the order that the WriteBlock function is called. The caller is
// responsible for ensuring keys and blocks are sorted appropriately, and that the
// block and index information is correct for the block. The minTime and maxTime
// timestamp values are used as the minimum and maximum values for the index entry.
WriteBlock(key string, minTime, maxTime int64, block []byte) error
// WriteIndex finishes the TSM write streams and writes the index.
WriteIndex() error
// Close closes any underlying file resources.
Close() error
// Size returns the current size in bytes of the file.
Size() uint32
}
func NewTSMWriter ¶
func NewTSMWriter(w io.Writer) (TSMWriter, error)
NewTSMWriter returns a new TSMWriter writing to w.
type TimeDecoder ¶
TimeDecoder decodes a byte slice into timestamps.
type TimeDecoder struct {
// contains filtered or unexported fields
}
func (*TimeDecoder) Error ¶
func (d *TimeDecoder) Error() error
Error returns the last error encountered by the decoder.
func (*TimeDecoder) Init ¶
func (d *TimeDecoder) Init(b []byte)
Init initializes the decoder with bytes to read from.
func (*TimeDecoder) Next ¶
func (d *TimeDecoder) Next() bool
Next returns true if there are any timestamps remaining to be decoded.
func (*TimeDecoder) Read ¶
func (d *TimeDecoder) Read() int64
Read returns the next timestamp from the decoder.
type TimeEncoder ¶
TimeEncoder encodes time.Time to byte slices.
type TimeEncoder interface {
Write(t int64)
Bytes() ([]byte, error)
Reset()
}
func NewTimeEncoder ¶
func NewTimeEncoder(sz int) TimeEncoder
NewTimeEncoder returns a TimeEncoder with an initial buffer ready to hold sz bytes.
type TimeRange ¶
TimeRange holds a min and max timestamp.
type TimeRange struct {
Min, Max int64
}
type Tombstone ¶
Tombstone represents an individual deletion.
type Tombstone struct {
// Key is the tombstoned series key.
Key string
// Min and Max are the min and max unix nanosecond time ranges of Key that are deleted. If
// the full range is deleted, both values are -1.
Min, Max int64
}
type Tombstoner ¶
Tombstoner records tombstones when entries are deleted.
type Tombstoner struct {
// Path is the location of the file to record tombstone. This should be the
// full path to a TSM file.
Path string
// contains filtered or unexported fields
}
func (*Tombstoner) Add ¶
func (t *Tombstoner) Add(keys []string) error
Add adds the all keys, across all timestamps, to the tombstone.
func (*Tombstoner) AddRange ¶
func (t *Tombstoner) AddRange(keys []string, min, max int64) error
AddRange adds all keys to the tombstone specifying only the data between min and max to be removed.
func (*Tombstoner) Delete ¶
func (t *Tombstoner) Delete() error
Delete removes all the tombstone files from disk.
func (*Tombstoner) HasTombstones ¶
func (t *Tombstoner) HasTombstones() bool
HasTombstones return true if there are any tombstone entries recorded.
func (*Tombstoner) ReadAll ¶
func (t *Tombstoner) ReadAll() ([]Tombstone, error)
ReadAll returns all the tombstones in the Tombstoner's directory.
func (*Tombstoner) TombstoneFiles ¶
func (t *Tombstoner) TombstoneFiles() []FileStat
TombstoneFiles returns any tombstone files associated with Tombstoner's TSM file.
func (*Tombstoner) Walk ¶
func (t *Tombstoner) Walk(fn func(t Tombstone) error) error
Walk calls fn for every Tombstone under the Tombstoner.
type Value ¶
Value represents a TSM-encoded value.
type Value interface {
// UnixNano returns the timestamp of the value in nanoseconds since unix epoch.
UnixNano() int64
// Value returns the underlying value.
Value() interface{}
// Size returns the number of bytes necessary to represent the value and its timestamp.
Size() int
// String returns the string representation of the value and its timestamp.
String() string
// contains filtered or unexported methods
}
func NewBooleanValue ¶
func NewBooleanValue(t int64, v bool) Value
NewBooleanValue returns a new boolean value.
func NewFloatValue ¶
func NewFloatValue(t int64, v float64) Value
NewFloatValue returns a new float value.
func NewIntegerValue ¶
func NewIntegerValue(t int64, v int64) Value
NewIntegerValue returns a new integer value.
func NewStringValue ¶
func NewStringValue(t int64, v string) Value
NewStringValue returns a new string value.
func NewValue ¶
func NewValue(t int64, value interface{}) Value
NewValue returns a new Value with the underlying type dependent on value.
type Values ¶
Values represents a slice of values.
type Values []Value
func (Values) Deduplicate ¶
func (a Values) Deduplicate() Values
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.
func (Values) Encode ¶
func (a Values) Encode(buf []byte) ([]byte, error)
Encode converts the values to a byte slice. If there are no values, this function panics.
func (Values) Exclude ¶
func (a Values) Exclude(min, max int64) Values
Exclude returns the subset of values not in [min, max]
func (Values) Include ¶
func (a Values) Include(min, max int64) Values
Include returns the subset values between min and max inclusive.
func (Values) InfluxQLType ¶
func (a Values) InfluxQLType() (influxql.DataType, error)
InfluxQLType returns the influxql.DataType the values map to.
func (Values) Len ¶
func (a Values) Len() int
Sort methods
func (Values) Less ¶
func (a Values) Less(i, j int) bool
func (Values) MaxTime ¶
func (a Values) MaxTime() int64
func (Values) Merge ¶
func (a Values) Merge(b Values) Values
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (Values) MinTime ¶
func (a Values) MinTime() int64
func (Values) Size ¶
func (a Values) Size() int
func (Values) Swap ¶
func (a Values) Swap(i, j int)
type WAL ¶
WAL represents the write-ahead log used for writing TSM files.
type WAL struct {
// SegmentSize is the file size at which a segment file will be rotated
SegmentSize int
// contains filtered or unexported fields
}
func NewWAL ¶
func NewWAL(path string) *WAL
NewWAL initializes a new WAL at the given directory.
func (*WAL) Close ¶
func (l *WAL) Close() error
Close will finish any flush that is currently in progress and close file handles.
func (*WAL) CloseSegment ¶
func (l *WAL) CloseSegment() error
CloseSegment closes the current segment if it is non-empty and opens a new one.
func (*WAL) ClosedSegments ¶
func (l *WAL) ClosedSegments() ([]string, error)
ClosedSegments returns a slice of the names of the closed segment files.
func (*WAL) Delete ¶
func (l *WAL) Delete(keys []string) (int, error)
Delete deletes the given keys, returning the segment ID for the operation.
func (*WAL) DeleteRange ¶
func (l *WAL) DeleteRange(keys []string, min, max int64) (int, error)
DeleteRange deletes the given keys within the given time range, returning the segment ID for the operation.
func (*WAL) LastWriteTime ¶
func (l *WAL) LastWriteTime() time.Time
LastWriteTime is the last time anything was written to the WAL.
func (*WAL) Open ¶
func (l *WAL) Open() error
Open opens and initializes the Log. Open can recover from previous unclosed shutdowns.
func (*WAL) Path ¶
func (l *WAL) Path() string
Path returns the directory the log was initialized with.
func (*WAL) Remove ¶
func (l *WAL) Remove(files []string) error
Remove deletes the given segment file paths from disk and cleans up any associated objects.
func (*WAL) Statistics ¶
func (l *WAL) Statistics(tags map[string]string) []models.Statistic
Statistics returns statistics for periodic monitoring.
func (*WAL) WithLogger ¶
func (l *WAL) WithLogger(log zap.Logger)
WithLogger sets the WAL's logger.
func (*WAL) WritePoints ¶
func (l *WAL) WritePoints(values map[string][]Value) (int, error)
WritePoints writes the given points to the WAL. It returns the WAL segment ID to which the points were written. If an error is returned the segment ID should be ignored.
type WALEntry ¶
WALEntry is record stored in each WAL segment. Each entry has a type and an opaque, type dependent byte slice data attribute.
type WALEntry interface {
Type() WalEntryType
Encode(dst []byte) ([]byte, error)
MarshalBinary() ([]byte, error)
UnmarshalBinary(b []byte) error
}
type WALSegmentReader ¶
WALSegmentReader reads WAL segments.
type WALSegmentReader struct {
// contains filtered or unexported fields
}
func NewWALSegmentReader ¶
func NewWALSegmentReader(r io.ReadCloser) *WALSegmentReader
NewWALSegmentReader returns a new WALSegmentReader reading from r.
func (*WALSegmentReader) Close ¶
func (r *WALSegmentReader) Close() error
Close closes the underlying io.Reader.
func (*WALSegmentReader) Count ¶
func (r *WALSegmentReader) Count() int64
Count returns the total number of bytes read successfully from the segment, as of the last call to Read(). The segment is guaranteed to be valid up to and including this number of bytes.
func (*WALSegmentReader) Error ¶
func (r *WALSegmentReader) Error() error
Error returns the last error encountered by the reader.
func (*WALSegmentReader) Next ¶
func (r *WALSegmentReader) Next() bool
Next indicates if there is a value to read.
func (*WALSegmentReader) Read ¶
func (r *WALSegmentReader) Read() (WALEntry, error)
Read returns the next entry in the reader.
type WALSegmentWriter ¶
WALSegmentWriter writes WAL segments.
type WALSegmentWriter struct {
// contains filtered or unexported fields
}
func NewWALSegmentWriter ¶
func NewWALSegmentWriter(w io.WriteCloser) *WALSegmentWriter
NewWALSegmentWriter returns a new WALSegmentWriter writing to w.
func (*WALSegmentWriter) Write ¶
func (w *WALSegmentWriter) Write(entryType WalEntryType, compressed []byte) error
Write writes entryType and the buffer containing compressed entry data.
type WALStatistics ¶
WALStatistics maintains statistics about the WAL.
type WALStatistics struct {
OldBytes int64
CurrentBytes int64
WriteOK int64
WriteErr int64
}
type WalEntryType ¶
WalEntryType is a byte written to a wal segment file that indicates what the following compressed block contains.
type WalEntryType byte
const (
// WriteWALEntryType indicates a write entry.
WriteWALEntryType WalEntryType = 0x01
// DeleteWALEntryType indicates a delete entry.
DeleteWALEntryType WalEntryType = 0x02
// DeleteRangeWALEntryType indicates a delete range entry.
DeleteRangeWALEntryType WalEntryType = 0x03
)
type WriteWALEntry ¶
WriteWALEntry represents a write of points.
type WriteWALEntry struct {
Values map[string][]Value
}
func (*WriteWALEntry) Encode ¶
func (w *WriteWALEntry) Encode(dst []byte) ([]byte, error)
Encode converts the WriteWALEntry into a byte stream using dst if it is large enough. If dst is too small, the slice will be grown to fit the encoded entry.
func (*WriteWALEntry) MarshalBinary ¶
func (w *WriteWALEntry) MarshalBinary() ([]byte, error)
MarshalBinary returns a binary representation of the entry in a new byte slice.
func (*WriteWALEntry) Type ¶
func (w *WriteWALEntry) Type() WalEntryType
Type returns WriteWALEntryType.
func (*WriteWALEntry) UnmarshalBinary ¶
func (w *WriteWALEntry) UnmarshalBinary(b []byte) error
UnmarshalBinary deserializes the byte slice into w.