func TestTSMReader_Type(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
typ, err := r.Type("cpu")
if err != nil {
fatal(t, "reading type", err)
}
if got, exp := typ, tsm1.BlockInt64; got != exp {
t.Fatalf("type mismatch: got %v, exp %v", got, exp)
}
}
func newFiles(values ...keyValues) ([]tsm1.TSMFile, error) {
var files []tsm1.TSMFile
for _, v := range values {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
return nil, err
}
if err := w.Write(v.key, v.values); err != nil {
return nil, err
}
if err := w.WriteIndex(); err != nil {
return nil, err
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
return nil, err
}
files = append(files, r)
}
return files, nil
}
func TestBlockIterator_Single(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
var count int
iter := r.BlockIterator()
for iter.Next() {
key, minTime, maxTime, buf, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error creating iterator: %v", err)
}
if got, exp := key, "cpu"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := minTime, time.Unix(0, 0); got != exp {
t.Fatalf("min time mismatch: got %v, exp %v", got, exp)
}
if got, exp := maxTime, time.Unix(0, 0); got != exp {
t.Fatalf("max time mismatch: got %v, exp %v", got, exp)
}
if len(buf) == 0 {
t.Fatalf("buf length = 0")
}
count++
}
if got, exp := count, len(values); got != exp {
t.Fatalf("value count mismatch: got %v, exp %v", got, exp)
}
}
// Tests that writing keys in reverse is able to read them back.
func TestTSMWriter_Write_SameKey(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"cpu", []tsm1.Value{
tsm1.NewValue(time.Unix(0, 0), 1.0),
tsm1.NewValue(time.Unix(1, 0), 2.0)},
},
{"cpu", []tsm1.Value{
tsm1.NewValue(time.Unix(2, 0), 3.0),
tsm1.NewValue(time.Unix(3, 0), 4.0)},
},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
values := append(data[0].values, data[1].values...)
readValues, err := r.ReadAll("cpu")
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp := len(values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
func TestBlockIterator_Sorted(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := map[string][]tsm1.Value{
"mem": []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))},
"cpu": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), float64(2))},
"disk": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), true)},
"load": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), "string")},
}
for k, v := range values {
if err := w.Write(k, v); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
var count int
iter := r.BlockIterator()
var lastKey string
for iter.Next() {
key, _, _, buf, err := iter.Read()
if key < lastKey {
t.Fatalf("keys not sorted: got %v, last %v", key, lastKey)
}
lastKey = key
if err != nil {
t.Fatalf("unexpected error creating iterator: %v", err)
}
if len(buf) == 0 {
t.Fatalf("buf length = 0")
}
count++
}
if got, exp := count, len(values); got != exp {
t.Fatalf("value count mismatch: got %v, exp %v", got, exp)
}
}
func TestTSMWriter_Write_Single(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error writing index: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if got, exp := len(b.Bytes()), 5; got < exp {
t.Fatalf("file size mismatch: got %v, exp %v", got, exp)
}
if got := binary.BigEndian.Uint32(b.Bytes()[0:4]); got != tsm1.MagicNumber {
t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
readValues, err := r.ReadAll("cpu")
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if len(readValues) != len(values) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), len(values))
}
for i, v := range values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
func TestCompacted_NotFull(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error writing index: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
iter := r.BlockIterator()
if !iter.Next() {
t.Fatalf("expected next, got false")
}
_, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error reading block: %v", err)
}
if got, exp := tsm1.BlockCount(block), 1; got != exp {
t.Fatalf("block count mismatch: got %v, exp %v", got, exp)
}
}
// Tests that calling Read returns all the values for block matching the key
// and timestamp
func TestTSMWriter_Read_Multiple(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"cpu", []tsm1.Value{
tsm1.NewValue(time.Unix(0, 0), 1.0),
tsm1.NewValue(time.Unix(1, 0), 2.0)},
},
{"cpu", []tsm1.Value{
tsm1.NewValue(time.Unix(2, 0), 3.0),
tsm1.NewValue(time.Unix(3, 0), 4.0)},
},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpeted error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpeted error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
for _, values := range data {
// Try the first timestamp
readValues, err := r.Read("cpu", values.values[0].Time())
if err != nil {
t.Fatalf("unexpeted error readin: %v", err)
}
if exp := len(values.values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range values.values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
// Try the last timestamp too
readValues, err = r.Read("cpu", values.values[1].Time())
if err != nil {
t.Fatalf("unexpeted error readin: %v", err)
}
if exp := len(values.values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range values.values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
}
func TestTSMWriter_WriteBlock_Multiple(t *testing.T) {
// Write a new TSM file
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
{"mem", []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), 2.0)}},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if got, exp := len(b.Bytes()), 5; got < exp {
t.Fatalf("file size mismatch: got %v, exp %v", got, exp)
}
if got := binary.BigEndian.Uint32(b.Bytes()[0:4]); got != tsm1.MagicNumber {
t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber)
}
// Create reader for that file
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
// Using the reader, write a new file using WriteBlocks
b.Reset()
w, err = tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
iter := r.BlockIterator()
for iter.Next() {
key, minTime, maxTime, b, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error reading block: %v", err)
}
if err := w.WriteBlock(key, minTime, maxTime, b); err != nil {
t.Fatalf("unexpected error writing block: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
// Now create a reader to verify the written blocks matches the originally
// written file using Write
r, err = tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
for _, d := range data {
readValues, err := r.ReadAll(d.key)
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp := len(d.values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range d.values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
}
// // Tests compacting a multiple wal segment into one tsm file
func TestCompactor_MultipleWALSegment(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
// First WAL segment
v1 := tsm1.NewValue(time.Unix(1, 0), float64(1))
v2 := tsm1.NewValue(time.Unix(1, 0), float64(1))
v3 := tsm1.NewValue(time.Unix(2, 0), float64(2))
points1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1, v3},
"cpu,host=B#!~#value": []tsm1.Value{v2},
}
entries := []tsm1.WALEntry{
&tsm1.WriteWALEntry{
Values: points1,
},
}
f1 := MustTempFile(dir)
defer f1.Close()
w := tsm1.NewWALSegmentWriter(f1)
for _, e := range entries {
if err := w.Write(e); err != nil {
t.Fatalf("unexpected error writing entry: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing writer: %v", err)
}
// Second WAL segment
v4 := tsm1.NewValue(time.Unix(2, 0), float64(2))
v5 := tsm1.NewValue(time.Unix(3, 0), float64(1))
v6 := tsm1.NewValue(time.Unix(4, 0), float64(1))
points2 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v5, v6},
"cpu,host=B#!~#value": []tsm1.Value{v4},
}
entries = []tsm1.WALEntry{
&tsm1.WriteWALEntry{
Values: points2,
},
}
f2 := MustTempFile(dir)
defer f2.Close()
w = tsm1.NewWALSegmentWriter(f2)
for _, e := range entries {
if err := w.Write(e); err != nil {
t.Fatalf("unexpected error writing entry: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing writer: %v", err)
}
compactor := &tsm1.Compactor{
Dir: dir,
}
files, err := compactor.Compact([]string{f1.Name(), f2.Name()})
if err != nil {
t.Fatalf("unexpected error compacting: %v", err)
}
if got, exp := len(files), 1; got != exp {
t.Fatalf("files length mismatch: got %v, exp %v", got, exp)
}
f, err := os.Open(files[0])
if err != nil {
t.Fatalf("unexpected error openting tsm: %v", err)
}
defer f.Close()
r, err := tsm1.NewTSMReader(f)
if err != nil {
t.Fatalf("unexpected error creating tsm reader: %v", err)
}
defer r.Close()
keys := r.Keys()
if got, exp := len(keys), 2; got != exp {
t.Fatalf("keys length mismatch: got %v, exp %v", got, exp)
}
var data = []struct {
key string
points []tsm1.Value
}{
{"cpu,host=A#!~#value", []tsm1.Value{v1, v3, v5, v6}},
{"cpu,host=B#!~#value", []tsm1.Value{v2, v4}},
}
for _, p := range data {
values, err := r.ReadAll(p.key)
if err != nil {
t.Fatalf("unexpected error reading: %v", err)
}
if got, exp := len(values), len(p.points); got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
for i, point := range p.points {
assertValueEqual(t, values[i], point)
}
}
}
func cmdDumpTsm1dev(opts *tsdmDumpOpts) {
var errors []error
f, err := os.Open(opts.path)
if err != nil {
println(err.Error())
os.Exit(1)
}
// Get the file size
stat, err := f.Stat()
if err != nil {
println(err.Error())
os.Exit(1)
}
b := make([]byte, 8)
r, err := tsm1.NewTSMReader(f)
if err != nil {
println("Error opening TSM files: ", err.Error())
}
defer r.Close()
minTime, maxTime := r.TimeRange()
keys := r.Keys()
blockStats := &blockStats{}
println("Summary:")
fmt.Printf(" File: %s\n", opts.path)
fmt.Printf(" Time Range: %s - %s\n",
minTime.UTC().Format(time.RFC3339Nano),
maxTime.UTC().Format(time.RFC3339Nano),
)
fmt.Printf(" Duration: %s ", maxTime.Sub(minTime))
fmt.Printf(" Series: %d ", len(keys))
fmt.Printf(" File Size: %d\n", stat.Size())
println()
tw := tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
fmt.Fprintln(tw, " "+strings.Join([]string{"Pos", "Min Time", "Max Time", "Ofs", "Size", "Key", "Field"}, "\t"))
var pos int
for _, key := range keys {
for _, e := range r.Entries(key) {
pos++
split := strings.Split(key, "#!~#")
// We dont' know know if we have fields so use an informative default
var measurement, field string = "UNKNOWN", "UNKNOWN"
// Possible corruption? Try to read as much as we can and point to the problem.
measurement = split[0]
field = split[1]
if opts.filterKey != "" && !strings.Contains(key, opts.filterKey) {
continue
}
fmt.Fprintln(tw, " "+strings.Join([]string{
strconv.FormatInt(int64(pos), 10),
e.MinTime.UTC().Format(time.RFC3339Nano),
e.MaxTime.UTC().Format(time.RFC3339Nano),
strconv.FormatInt(int64(e.Offset), 10),
strconv.FormatInt(int64(e.Size), 10),
measurement,
field,
}, "\t"))
}
}
if opts.dumpIndex {
println("Index:")
tw.Flush()
println()
}
tw = tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
fmt.Fprintln(tw, " "+strings.Join([]string{"Blk", "Chk", "Ofs", "Len", "Type", "Min Time", "Points", "Enc [T/V]", "Len [T/V]"}, "\t"))
// Starting at 5 because the magic number is 4 bytes + 1 byte version
i := int64(5)
var blockCount, pointCount, blockSize int64
indexSize := r.IndexSize()
// Start at the beginning and read every block
for _, key := range keys {
for _, e := range r.Entries(key) {
f.Seek(int64(e.Offset), 0)
f.Read(b[:4])
chksum := btou32(b)
buf := make([]byte, e.Size)
f.Read(buf)
blockSize += int64(len(buf)) + 4
startTime := time.Unix(0, int64(btou64(buf[:8])))
blockType := buf[8]
encoded := buf[9:]
var v []tsm1.Value
v, err := tsm1.DecodeBlock(buf, v)
if err != nil {
fmt.Printf("error: %v\n", err.Error())
os.Exit(1)
}
pointCount += int64(len(v))
// Length of the timestamp block
tsLen, j := binary.Uvarint(encoded)
// Unpack the timestamp bytes
ts := encoded[int(j) : int(j)+int(tsLen)]
// Unpack the value bytes
values := encoded[int(j)+int(tsLen):]
tsEncoding := timeEnc[int(ts[0]>>4)]
vEncoding := encDescs[int(blockType+1)][values[0]>>4]
typeDesc := blockTypes[blockType]
blockStats.inc(0, ts[0]>>4)
blockStats.inc(int(blockType+1), values[0]>>4)
blockStats.size(len(buf))
if opts.filterKey != "" && !strings.Contains(key, opts.filterKey) {
i += (4 + int64(e.Size))
blockCount++
continue
}
fmt.Fprintln(tw, " "+strings.Join([]string{
strconv.FormatInt(blockCount, 10),
strconv.FormatUint(uint64(chksum), 10),
strconv.FormatInt(i, 10),
strconv.FormatInt(int64(len(buf)), 10),
typeDesc,
startTime.UTC().Format(time.RFC3339Nano),
strconv.FormatInt(int64(len(v)), 10),
fmt.Sprintf("%s/%s", tsEncoding, vEncoding),
fmt.Sprintf("%d/%d", len(ts), len(values)),
}, "\t"))
i += (4 + int64(e.Size))
blockCount++
}
}
if opts.dumpBlocks {
println("Blocks:")
tw.Flush()
println()
}
fmt.Printf("Statistics\n")
fmt.Printf(" Blocks:\n")
fmt.Printf(" Total: %d Size: %d Min: %d Max: %d Avg: %d\n",
blockCount, blockSize, blockStats.min, blockStats.max, blockSize/blockCount)
fmt.Printf(" Index:\n")
fmt.Printf(" Total: %d Size: %d\n", blockCount, indexSize)
fmt.Printf(" Points:\n")
fmt.Printf(" Total: %d", pointCount)
println()
println(" Encoding:")
for i, counts := range blockStats.counts {
if len(counts) == 0 {
continue
}
fmt.Printf(" %s: ", strings.Title(fieldType[i]))
for j, v := range counts {
fmt.Printf("\t%s: %d (%d%%) ", encDescs[i][j], v, int(float64(v)/float64(blockCount)*100))
}
println()
}
fmt.Printf(" Compression:\n")
fmt.Printf(" Per block: %0.2f bytes/point\n", float64(blockSize)/float64(pointCount))
fmt.Printf(" Total: %0.2f bytes/point\n", float64(stat.Size())/float64(pointCount))
if len(errors) > 0 {
println()
fmt.Printf("Errors (%d):\n", len(errors))
for _, err := range errors {
fmt.Printf(" * %v\n", err)
}
println()
}
}