func TestEncoding_IntBlock_Basic(t *testing.T) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
values[i] = tsm1.NewValue(t, int64(i))
}
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
decodedValues, err = tsm1.DecodeBlock(b, decodedValues)
if err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if len(decodedValues) != len(values) {
t.Fatalf("unexpected results length:\n\tgot: %v\n\texp: %v\n", len(decodedValues), len(values))
}
for i := 0; i < len(decodedValues); i++ {
if decodedValues[i].UnixNano() != values[i].UnixNano() {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues[i].UnixNano(), values[i].UnixNano())
}
if decodedValues[i].Value() != values[i].Value() {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues[i].Value(), values[i].Value())
}
}
}
func TestEncoding_BooleanBlock_Basic(t *testing.T) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
v := true
if i%2 == 0 {
v = false
}
values[i] = tsm1.NewValue(t, v)
}
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
decodedValues, err = tsm1.DecodeBlock(b, decodedValues)
if err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if !reflect.DeepEqual(decodedValues, values) {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues, values)
}
}
// Tests that a single TSM file can be read and iterated over
func TestTSMKeyIterator_Chunked(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v0 := tsm1.NewValue(1, 1.1)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v0},
}
r1 := MustTSMReader(dir, 1, writes)
v1 := tsm1.NewValue(2, 2.1)
writes1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1},
}
r2 := MustTSMReader(dir, 2, writes1)
iter, err := tsm1.NewTSMKeyIterator(2, false, r1, r2)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
var chunk int
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 2; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
readValues = len(values) > 0
assertValueEqual(t, values[0], v0)
assertValueEqual(t, values[1], v1)
chunk++
}
if got, exp := chunk, 1; got != exp {
t.Fatalf("chunk count mismatch: got %v, exp %v", got, exp)
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
// Tests that duplicate point values are merged. There is only one case
// where this could happen and that is when a compaction completed and we replace
// the old TSM file with a new one and we crash just before deleting the old file.
// No data is lost but the same point time/value would exist in two files until
// compaction corrects it.
func TestTSMKeyIterator_Duplicate(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v1 := tsm1.NewValue(1, int64(1))
v2 := tsm1.NewValue(1, int64(2))
writes1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1},
}
r1 := MustTSMReader(dir, 1, writes1)
writes2 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v2},
}
r2 := MustTSMReader(dir, 2, writes2)
iter, err := tsm1.NewTSMKeyIterator(1, false, r1, r2)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
readValues = true
assertValueEqual(t, values[0], v2)
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
// Tests that a single TSM file can be read and iterated over
func TestTSMKeyIterator_Chunked(t *testing.T) {
t.Skip("fixme")
dir := MustTempDir()
defer os.RemoveAll(dir)
v0 := tsm1.NewValue(1, 1.1)
v1 := tsm1.NewValue(2, 2.1)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v0, v1},
}
r := MustTSMReader(dir, 1, writes)
iter, err := tsm1.NewTSMKeyIterator(1, false, r)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
var chunk int
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), len(writes); got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
for _, v := range values {
readValues = true
assertValueEqual(t, v, writes["cpu,host=A#!~#value"][chunk])
}
chunk++
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
func TestCacheKeyIterator_Chunked(t *testing.T) {
v0 := tsm1.NewValue(1, 1.0)
v1 := tsm1.NewValue(2, 2.0)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v0, v1},
}
c := tsm1.NewCache(0, "")
for k, v := range writes {
if err := c.Write(k, v); err != nil {
t.Fatalf("failed to write key foo to cache: %s", err.Error())
}
}
iter := tsm1.NewCacheKeyIterator(c, 1)
var readValues bool
var chunk int
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
for _, v := range values {
readValues = true
assertValueEqual(t, v, writes["cpu,host=A#!~#value"][chunk])
}
chunk++
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
func TestEncoding_FloatBlock_ZeroTime(t *testing.T) {
values := make([]tsm1.Value, 3)
for i := 0; i < 3; i++ {
values[i] = tsm1.NewValue(0, float64(i))
}
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
decodedValues, err = tsm1.DecodeBlock(b, decodedValues)
if err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if !reflect.DeepEqual(decodedValues, values) {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues, values)
}
}
func BenchmarkDecodeBlock_String_EqualSize(b *testing.B) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
values[i] = tsm1.NewValue(t, fmt.Sprintf("value %d", i))
}
bytes, err := tsm1.Values(values).Encode(nil)
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
decodedValues := make([]tsm1.Value, len(values))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err = tsm1.DecodeBlock(bytes, decodedValues)
if err != nil {
b.Fatalf("unexpected error decoding block: %v", err)
}
}
}
func BenchmarkDecodeBlock_Boolean_Empty(b *testing.B) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
values[i] = tsm1.NewValue(t, true)
}
bytes, err := tsm1.Values(values).Encode(nil)
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err = tsm1.DecodeBlock(bytes, decodedValues)
if err != nil {
b.Fatalf("unexpected error decoding block: %v", err)
}
}
}
func TestEncoding_FloatBlock_SimilarFloats(t *testing.T) {
values := make([]tsm1.Value, 5)
values[0] = tsm1.NewValue(1444238178437870000, 6.00065e+06)
values[1] = tsm1.NewValue(1444238185286830000, 6.000656e+06)
values[2] = tsm1.NewValue(1444238188441501000, 6.000657e+06)
values[3] = tsm1.NewValue(1444238195286811000, 6.000659e+06)
values[4] = tsm1.NewValue(1444238198439917000, 6.000661e+06)
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
decodedValues, err = tsm1.DecodeBlock(b, decodedValues)
if err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if !reflect.DeepEqual(decodedValues, values) {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues, values)
}
}
func TestEncoding_FloatBlock(t *testing.T) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
values[i] = tsm1.NewValue(t, float64(i))
}
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
decodedValues, err = tsm1.DecodeBlock(b, decodedValues)
if err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if !reflect.DeepEqual(decodedValues, values) {
t.Fatalf("unexpected results:\n\tgot: %s\n\texp: %s\n", spew.Sdump(decodedValues), spew.Sdump(values))
}
}
func (cmd *Command) dump() error {
var errors []error
f, err := os.Open(cmd.path)
if err != nil {
return err
}
// Get the file size
stat, err := f.Stat()
if err != nil {
return err
}
b := make([]byte, 8)
r, err := tsm1.NewTSMReader(f)
if err != nil {
return fmt.Errorf("Error opening TSM files: %s", err.Error())
}
defer r.Close()
minTime, maxTime := r.TimeRange()
keyCount := r.KeyCount()
blockStats := &blockStats{}
println("Summary:")
fmt.Printf(" File: %s\n", cmd.path)
fmt.Printf(" Time Range: %s - %s\n",
time.Unix(0, minTime).UTC().Format(time.RFC3339Nano),
time.Unix(0, maxTime).UTC().Format(time.RFC3339Nano),
)
fmt.Printf(" Duration: %s ", time.Unix(0, maxTime).Sub(time.Unix(0, minTime)))
fmt.Printf(" Series: %d ", keyCount)
fmt.Printf(" File Size: %d\n", stat.Size())
println()
tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
if cmd.dumpIndex {
println("Index:")
tw.Flush()
println()
fmt.Fprintln(tw, " "+strings.Join([]string{"Pos", "Min Time", "Max Time", "Ofs", "Size", "Key", "Field"}, "\t"))
var pos int
for i := 0; i < keyCount; i++ {
key, _ := r.KeyAt(i)
for _, e := range r.Entries(string(key)) {
pos++
split := strings.Split(string(key), "#!~#")
// We dont' know know if we have fields so use an informative default
var measurement, field string = "UNKNOWN", "UNKNOWN"
// Possible corruption? Try to read as much as we can and point to the problem.
measurement = split[0]
field = split[1]
if cmd.filterKey != "" && !strings.Contains(string(key), cmd.filterKey) {
continue
}
fmt.Fprintln(tw, " "+strings.Join([]string{
strconv.FormatInt(int64(pos), 10),
time.Unix(0, e.MinTime).UTC().Format(time.RFC3339Nano),
time.Unix(0, e.MaxTime).UTC().Format(time.RFC3339Nano),
strconv.FormatInt(int64(e.Offset), 10),
strconv.FormatInt(int64(e.Size), 10),
measurement,
field,
}, "\t"))
tw.Flush()
}
}
}
tw = tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
fmt.Fprintln(tw, " "+strings.Join([]string{"Blk", "Chk", "Ofs", "Len", "Type", "Min Time", "Points", "Enc [T/V]", "Len [T/V]"}, "\t"))
// Starting at 5 because the magic number is 4 bytes + 1 byte version
i := int64(5)
var blockCount, pointCount, blockSize int64
indexSize := r.IndexSize()
// Start at the beginning and read every block
for j := 0; j < keyCount; j++ {
key, _ := r.KeyAt(j)
for _, e := range r.Entries(string(key)) {
f.Seek(int64(e.Offset), 0)
f.Read(b[:4])
chksum := binary.BigEndian.Uint32(b[:4])
buf := make([]byte, e.Size-4)
f.Read(buf)
blockSize += int64(e.Size)
if cmd.filterKey != "" && !strings.Contains(string(key), cmd.filterKey) {
i += blockSize
blockCount++
continue
}
blockType := buf[0]
encoded := buf[1:]
var v []tsm1.Value
v, err := tsm1.DecodeBlock(buf, v)
if err != nil {
return err
}
startTime := time.Unix(0, v[0].UnixNano())
pointCount += int64(len(v))
// Length of the timestamp block
tsLen, j := binary.Uvarint(encoded)
// Unpack the timestamp bytes
ts := encoded[int(j) : int(j)+int(tsLen)]
// Unpack the value bytes
values := encoded[int(j)+int(tsLen):]
tsEncoding := timeEnc[int(ts[0]>>4)]
vEncoding := encDescs[int(blockType+1)][values[0]>>4]
typeDesc := blockTypes[blockType]
blockStats.inc(0, ts[0]>>4)
blockStats.inc(int(blockType+1), values[0]>>4)
blockStats.size(len(buf))
if cmd.dumpBlocks {
fmt.Fprintln(tw, " "+strings.Join([]string{
strconv.FormatInt(blockCount, 10),
strconv.FormatUint(uint64(chksum), 10),
strconv.FormatInt(i, 10),
strconv.FormatInt(int64(len(buf)), 10),
typeDesc,
startTime.UTC().Format(time.RFC3339Nano),
strconv.FormatInt(int64(len(v)), 10),
fmt.Sprintf("%s/%s", tsEncoding, vEncoding),
fmt.Sprintf("%d/%d", len(ts), len(values)),
}, "\t"))
}
i += blockSize
blockCount++
}
}
if cmd.dumpBlocks {
println("Blocks:")
tw.Flush()
println()
}
var blockSizeAvg int64
if blockCount > 0 {
blockSizeAvg = blockSize / blockCount
}
fmt.Printf("Statistics\n")
fmt.Printf(" Blocks:\n")
fmt.Printf(" Total: %d Size: %d Min: %d Max: %d Avg: %d\n",
blockCount, blockSize, blockStats.min, blockStats.max, blockSizeAvg)
fmt.Printf(" Index:\n")
fmt.Printf(" Total: %d Size: %d\n", blockCount, indexSize)
fmt.Printf(" Points:\n")
fmt.Printf(" Total: %d", pointCount)
println()
println(" Encoding:")
for i, counts := range blockStats.counts {
if len(counts) == 0 {
continue
}
fmt.Printf(" %s: ", strings.Title(fieldType[i]))
for j, v := range counts {
fmt.Printf("\t%s: %d (%d%%) ", encDescs[i][j], v, int(float64(v)/float64(blockCount)*100))
}
println()
}
fmt.Printf(" Compression:\n")
fmt.Printf(" Per block: %0.2f bytes/point\n", float64(blockSize)/float64(pointCount))
fmt.Printf(" Total: %0.2f bytes/point\n", float64(stat.Size())/float64(pointCount))
if len(errors) > 0 {
println()
fmt.Printf("Errors (%d):\n", len(errors))
for _, err := range errors {
fmt.Printf(" * %v\n", err)
}
println()
return fmt.Errorf("error count %d", len(errors))
}
return nil
}
// Tests that deleted keys are not seen during iteration with
// TSM files.
func TestTSMKeyIterator_MultipleKeysDeleted(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v1 := tsm1.NewValue(2, int64(1))
points1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1},
}
r1 := MustTSMReader(dir, 1, points1)
if e := r1.Delete([]string{"cpu,host=A#!~#value"}); nil != e {
t.Fatal(e)
}
v2 := tsm1.NewValue(1, float64(1))
v3 := tsm1.NewValue(1, float64(1))
points2 := map[string][]tsm1.Value{
"cpu,host=A#!~#count": []tsm1.Value{v2},
"cpu,host=B#!~#value": []tsm1.Value{v3},
}
r2 := MustTSMReader(dir, 2, points2)
r2.Delete([]string{"cpu,host=A#!~#count"})
iter, err := tsm1.NewTSMKeyIterator(1, false, r1, r2)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
var data = []struct {
key string
value tsm1.Value
}{
{"cpu,host=B#!~#value", v3},
}
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, data[0].key; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
readValues = true
assertValueEqual(t, values[0], data[0].value)
data = data[1:]
}
if !readValues {
t.Fatalf("failed to read any values")
}
}