func TestFileStore_Read(t *testing.T) {
fs := tsm1.NewFileStore("")
// Setup 3 files
data := []keyValues{
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), 2.0)}},
keyValues{"mem", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
}
files, err := newFiles(data...)
if err != nil {
t.Fatalf("unexpected error creating files: %v", err)
}
fs.Add(files...)
// Search for an entry that exists in the second file
values, err := fs.Read("cpu", time.Unix(1, 0))
if err != nil {
t.Fatalf("unexpected error reading values: %v", err)
}
exp := data[1]
if got, exp := len(values), len(exp.values); got != exp {
t.Fatalf("value length mismatch: got %v, exp %v", got, exp)
}
for i, v := range exp.values {
if got, exp := values[i].Value(), v.Value(); got != exp {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, got, exp)
}
}
}
func TestEngine_Write_Concurrent(t *testing.T) {
t.Skip("re-enable once tsm1 Write refactor is merged")
e := OpenDefaultEngine()
defer e.Engine.Close()
values1 := make(tsm1.Values, 1)
values1[0] = tsm1.NewValue(time.Unix(0, 0), float64(1))
pointsByKey1 := map[string]tsm1.Values{
"foo": values1,
}
values2 := make(tsm1.Values, 1)
values2[0] = tsm1.NewValue(time.Unix(10, 0), float64(1))
pointsByKey2 := map[string]tsm1.Values{
"foo": values2,
}
var wg sync.WaitGroup
for i := 0; i < 4; i++ {
wg.Add(1)
go func() {
e.Write(pointsByKey1, nil, nil)
wg.Done()
}()
wg.Add(1)
go func() {
e.Write(pointsByKey2, nil, nil)
wg.Done()
}()
}
wg.Wait()
}
func TestFileStore_SeekToDesc_End(t *testing.T) {
fs := tsm1.NewFileStore("")
// Setup 3 files
data := []keyValues{
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), 2.0)}},
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(2, 0), 3.0)}},
}
files, err := newFiles(data...)
if err != nil {
t.Fatalf("unexpected error creating files: %v", err)
}
fs.Add(files...)
c := fs.KeyCursor("cpu")
values, err := c.SeekTo(time.Unix(2, 0), false)
if err != nil {
t.Fatalf("unexpected error reading values: %v", err)
}
exp := data[2]
if got, exp := len(values), len(exp.values); got != exp {
t.Fatalf("value length mismatch: got %v, exp %v", got, exp)
}
for i, v := range exp.values {
if got, exp := values[i].Value(), v.Value(); got != exp {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, got, exp)
}
}
}
func TestFileStore_Open(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
// Create 3 TSM files...
data := []keyValues{
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), 2.0)}},
keyValues{"mem", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
}
_, err := newFileDir(dir, data...)
if err != nil {
fatal(t, "creating test files", err)
}
fs := tsm1.NewFileStore(dir)
if err := fs.Open(); err != nil {
fatal(t, "opening file store", err)
}
defer fs.Close()
if got, exp := fs.Count(), 3; got != exp {
t.Fatalf("file count mismatch: got %v, exp %v", got, exp)
}
if got, exp := fs.CurrentID(), 4; got != exp {
t.Fatalf("current ID mismatch: got %v, exp %v", got, exp)
}
}
func TestFileStore_Delete(t *testing.T) {
fs := tsm1.NewFileStore("")
// Setup 3 files
data := []keyValues{
keyValues{"cpu,host=server2!~#!value", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
keyValues{"cpu,host=server1!~#!value", []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), 2.0)}},
keyValues{"mem,host=server1!~#!value", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
}
files, err := newFiles(data...)
if err != nil {
t.Fatalf("unexpected error creating files: %v", err)
}
fs.Add(files...)
keys := fs.Keys()
if got, exp := len(keys), 3; got != exp {
t.Fatalf("key length mismatch: got %v, exp %v", got, exp)
}
if err := fs.Delete("cpu,host=server2!~#!value"); err != nil {
fatal(t, "deleting", err)
}
keys = fs.Keys()
if got, exp := len(keys), 2; got != exp {
t.Fatalf("key length mismatch: got %v, exp %v", got, exp)
}
}
func TestScrubValues(t *testing.T) {
dummy := Converter{
tracker: new(tracker),
}
epoch := time.Unix(0, 0)
simple := []tsm1.Value{tsm1.NewValue(epoch, 1.0)}
for _, tt := range []struct {
input, expected []tsm1.Value
}{
{
input: simple,
expected: simple,
}, {
input: []tsm1.Value{simple[0], tsm1.NewValue(epoch, math.NaN())},
expected: simple,
}, {
input: []tsm1.Value{simple[0], tsm1.NewValue(epoch, math.Inf(-1))},
expected: simple,
}, {
input: []tsm1.Value{simple[0], tsm1.NewValue(epoch, math.Inf(1)), tsm1.NewValue(epoch, math.NaN())},
expected: simple,
},
} {
out := dummy.scrubValues(tt.input)
if !reflect.DeepEqual(out, tt.expected) {
t.Errorf("Failed to scrub '%s': Got '%s', Expected '%s'", pretty(tt.input), pretty(out), pretty(tt.expected))
}
}
}
// Tests that deleted keys are not seen during iteration with
// TSM files.
func TestKeyIterator_TSM_MultipleKeysDeleted(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v1 := tsm1.NewValue(time.Unix(2, 0), int64(1))
points1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1},
}
r1 := MustTSMReader(dir, 1, points1)
r1.Delete([]string{"cpu,host=A#!~#value"})
v2 := tsm1.NewValue(time.Unix(1, 0), float64(1))
v3 := tsm1.NewValue(time.Unix(1, 0), float64(1))
points2 := map[string][]tsm1.Value{
"cpu,host=A#!~#count": []tsm1.Value{v2},
"cpu,host=B#!~#value": []tsm1.Value{v3},
}
r2 := MustTSMReader(dir, 1, points2)
r2.Delete([]string{"cpu,host=A#!~#count"})
iter, err := tsm1.NewTSMKeyIterator(r1, r2)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
var data = []struct {
key string
value tsm1.Value
}{
{"cpu,host=B#!~#value", v3},
}
for iter.Next() {
key, values, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
if got, exp := key, data[0].key; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
readValues = true
assertValueEqual(t, values[0], data[0].value)
data = data[1:]
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
// // Tests that a multiple WAL segment can be read and iterated over
func TestKeyIterator_WALSegment_Multiple(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v1 := tsm1.NewValue(time.Unix(1, 0), int64(1))
points1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1},
}
entries := []tsm1.WALEntry{
&tsm1.WriteWALEntry{
Values: points1,
},
}
r1 := MustWALSegment(dir, entries)
v2 := tsm1.NewValue(time.Unix(2, 0), int64(2))
points2 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v2},
}
entries = []tsm1.WALEntry{
&tsm1.WriteWALEntry{
Values: points2,
},
}
r2 := MustWALSegment(dir, entries)
iter, err := tsm1.NewWALKeyIterator(r1, r2)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
for iter.Next() {
key, values, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 2; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
readValues = true
assertValueEqual(t, values[0], v1)
assertValueEqual(t, values[1], v2)
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
func Test_scrubValuesFilterInf(t *testing.T) {
intV := tsm1.NewValue(time.Unix(0, 0), 1.0)
values := []tsm1.Value{intV, tsm1.NewValue(time.Unix(0, 0), math.Inf(-1))}
scrubbed := scrubValues(values)
if !reflect.DeepEqual([]tsm1.Value{intV}, scrubbed) {
t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", []tsm1.Value{intV}, scrubbed)
}
}
func TestWALWriter_WritePoints_Single(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
w := tsm1.NewWALSegmentWriter(f)
p1 := tsm1.NewValue(time.Unix(1, 0), 1.1)
p2 := tsm1.NewValue(time.Unix(1, 0), int64(1))
p3 := tsm1.NewValue(time.Unix(1, 0), true)
p4 := tsm1.NewValue(time.Unix(1, 0), "string")
values := map[string][]tsm1.Value{
"cpu,host=A#!~#float": []tsm1.Value{p1},
"cpu,host=A#!~#int": []tsm1.Value{p2},
"cpu,host=A#!~#bool": []tsm1.Value{p3},
"cpu,host=A#!~#string": []tsm1.Value{p4},
}
entry := &tsm1.WriteWALEntry{
Values: values,
}
if err := w.Write(mustMarshalEntry(entry)); err != nil {
fatal(t, "write points", err)
}
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
fatal(t, "seek", err)
}
r := tsm1.NewWALSegmentReader(f)
if !r.Next() {
t.Fatalf("expected next, got false")
}
we, err := r.Read()
if err != nil {
fatal(t, "read entry", err)
}
e, ok := we.(*tsm1.WriteWALEntry)
if !ok {
t.Fatalf("expected WriteWALEntry: got %#v", e)
}
for k, v := range e.Values {
for i, vv := range v {
if got, exp := vv.String(), values[k][i].String(); got != exp {
t.Fatalf("points mismatch: got %v, exp %v", got, exp)
}
}
}
if n := r.Count(); n != MustReadFileSize(f) {
t.Fatalf("wrong count of bytes read, got %d, exp %d", n, MustReadFileSize(f))
}
}
// Tests that writing keys in reverse is able to read them back.
func TestTSMWriter_Write_SameKey(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"cpu", []tsm1.Value{
tsm1.NewValue(time.Unix(0, 0), 1.0),
tsm1.NewValue(time.Unix(1, 0), 2.0)},
},
{"cpu", []tsm1.Value{
tsm1.NewValue(time.Unix(2, 0), 3.0),
tsm1.NewValue(time.Unix(3, 0), 4.0)},
},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
values := append(data[0].values, data[1].values...)
readValues, err := r.ReadAll("cpu")
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp := len(values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
func TestBlockIterator_Sorted(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := map[string][]tsm1.Value{
"mem": []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))},
"cpu": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), float64(2))},
"disk": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), true)},
"load": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), "string")},
}
for k, v := range values {
if err := w.Write(k, v); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
var count int
iter := r.BlockIterator()
var lastKey string
for iter.Next() {
key, _, _, buf, err := iter.Read()
if key < lastKey {
t.Fatalf("keys not sorted: got %v, last %v", key, lastKey)
}
lastKey = key
if err != nil {
t.Fatalf("unexpected error creating iterator: %v", err)
}
if len(buf) == 0 {
t.Fatalf("buf length = 0")
}
count++
}
if got, exp := count, len(values); got != exp {
t.Fatalf("value count mismatch: got %v, exp %v", got, exp)
}
}
func TestFileStore_SeekToAsc_Duplicate(t *testing.T) {
fs := tsm1.NewFileStore("")
// Setup 3 files
data := []keyValues{
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}},
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 2.0)}},
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(2, 0), 3.0)}},
keyValues{"cpu", []tsm1.Value{tsm1.NewValue(time.Unix(2, 0), 4.0)}},
}
files, err := newFiles(data...)
if err != nil {
t.Fatalf("unexpected error creating files: %v", err)
}
fs.Add(files...)
c := fs.KeyCursor("cpu")
// Search for an entry that exists in the second file
values, err := c.SeekTo(time.Unix(0, 0), true)
if err != nil {
t.Fatalf("unexpected error reading values: %v", err)
}
exp := data[1]
if got, exp := len(values), len(exp.values); got != exp {
t.Fatalf("value length mismatch: got %v, exp %v", got, exp)
}
for i, v := range exp.values {
if got, exp := values[i].Value(), v.Value(); got != exp {
t.Fatalf("read value mismatch(%d): got %v, exp %v", i, got, exp)
}
}
// Check that calling Next will dedupe points
values, err = c.Next(true)
if err != nil {
t.Fatalf("unexpected error reading values: %v", err)
}
exp = data[3]
if got, exp := len(values), len(exp.values); got != exp {
t.Fatalf("value length mismatch: got %v, exp %v", got, exp)
}
for i, v := range exp.values {
if got, exp := values[i].Value(), v.Value(); got != exp {
t.Fatalf("read value mismatch(%d): got %v, exp %v", i, got, exp)
}
}
}
// Tests that duplicate point values are merged. There is only one case
// where this could happen and that is when a compaction completed and we replace
// the old TSM file with a new one and we crash just before deleting the old file.
// No data is lost but the same point time/value would exist in two files until
// compaction corrects it.
func TestTSMKeyIterator_Duplicate(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v1 := tsm1.NewValue(time.Unix(1, 0), int64(1))
v2 := tsm1.NewValue(time.Unix(1, 0), int64(2))
writes1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1},
}
r1 := MustTSMReader(dir, 1, writes1)
writes2 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v2},
}
r2 := MustTSMReader(dir, 2, writes2)
iter, err := tsm1.NewTSMKeyIterator(1, false, r1, r2)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
readValues = true
assertValueEqual(t, values[0], v2)
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
// // Tests that merge iterator over a wal returns points order correctly.
func TestMergeIteragor_Single(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v1 := tsm1.NewValue(time.Unix(1, 0), float64(1))
v2 := tsm1.NewValue(time.Unix(2, 0), float64(2))
points := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1, v2},
}
entries := []tsm1.WALEntry{
&tsm1.WriteWALEntry{
Values: points,
},
}
r := MustWALSegment(dir, entries)
iter, err := tsm1.NewWALKeyIterator(r)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
// Read should return a chunk of 1 value
m := tsm1.NewMergeIterator(iter, 1)
var readValues bool
for _, p := range points {
if !m.Next() {
t.Fatalf("expected next, got false")
}
key, values, err := m.Read()
if err != nil {
t.Fatalf("unexpected error reading: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
readValues = true
assertValueEqual(t, values[0], p[0])
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
// Tests that a single TSM file can be read and iterated over
func TestTSMKeyIterator_Chunked(t *testing.T) {
t.Skip("fixme")
dir := MustTempDir()
defer os.RemoveAll(dir)
v0 := tsm1.NewValue(time.Unix(1, 0), 1.1)
v1 := tsm1.NewValue(time.Unix(2, 0), 2.1)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v0, v1},
}
r := MustTSMReader(dir, 1, writes)
iter, err := tsm1.NewTSMKeyIterator(1, false, r)
if err != nil {
t.Fatalf("unexpected error creating WALKeyIterator: %v", err)
}
var readValues bool
var chunk int
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), len(writes); got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
for _, v := range values {
readValues = true
assertValueEqual(t, v, writes["cpu,host=A#!~#value"][chunk])
}
chunk++
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
func TestCacheKeyIterator_Chunked(t *testing.T) {
v0 := tsm1.NewValue(time.Unix(1, 0).UTC(), 1.0)
v1 := tsm1.NewValue(time.Unix(2, 0).UTC(), 2.0)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v0, v1},
}
c := tsm1.NewCache(0)
for k, v := range writes {
if err := c.Write(k, v); err != nil {
t.Fatalf("failed to write key foo to cache: %s", err.Error())
}
}
iter := tsm1.NewCacheKeyIterator(c, 1)
var readValues bool
var chunk int
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
for _, v := range values {
readValues = true
assertValueEqual(t, v, writes["cpu,host=A#!~#value"][chunk])
}
chunk++
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
func TestEncoding_IntBlock_Negatives(t *testing.T) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
v := int64(i)
if i%2 == 0 {
v = -v
}
values[i] = tsm1.NewValue(t, int64(v))
}
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
if err := tsm1.DecodeBlock(b, &decodedValues); err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if !reflect.DeepEqual(decodedValues, values) {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues, values)
}
}
func BenchmarkDecodeBlock_String_TypeSpecific(b *testing.B) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
values[i] = tsm1.NewValue(t, fmt.Sprintf("value %d", i))
}
bytes, err := tsm1.Values(values).Encode(nil)
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
decodedValues := make([]*tsm1.StringValue, len(values))
for i := 0; i < len(decodedValues); i++ {
decodedValues[i] = &tsm1.StringValue{}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err = tsm1.DecodeStringBlock(bytes, decodedValues)
if err != nil {
b.Fatalf("unexpected error decoding block: %v", err)
}
}
}
func TestEncoding_IntBlock_Basic(t *testing.T) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
values[i] = tsm1.NewValue(t, int64(i))
}
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
decodedValues, err = tsm1.DecodeBlock(b, decodedValues)
if err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if len(decodedValues) != len(values) {
t.Fatalf("unexpected results length:\n\tgot: %v\n\texp: %v\n", len(decodedValues), len(values))
}
for i := 0; i < len(decodedValues); i++ {
if decodedValues[i].Time() != values[i].Time() {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues[i].Time(), values[i].Time())
}
if decodedValues[i].Value() != values[i].Value() {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues[i].Value(), values[i].Value())
}
}
}
// ReadAllCursor slurps all values from a cursor.
func ReadAllCursor(c tsdb.Cursor) tsm1.Values {
var values tsm1.Values
for k, v := c.Next(); k != tsdb.EOF; k, v = c.Next() {
values = append(values, tsm1.NewValue(time.Unix(0, k).UTC(), v))
}
return values
}
func TestEncoding_BoolBlock_Basic(t *testing.T) {
valueCount := 1000
times := getTimes(valueCount, 60, time.Second)
values := make([]tsm1.Value, len(times))
for i, t := range times {
v := true
if i%2 == 0 {
v = false
}
values[i] = tsm1.NewValue(t, v)
}
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var decodedValues []tsm1.Value
decodedValues, err = tsm1.DecodeBlock(b, decodedValues)
if err != nil {
t.Fatalf("unexpected error decoding block: %v", err)
}
if !reflect.DeepEqual(decodedValues, values) {
t.Fatalf("unexpected results:\n\tgot: %v\n\texp: %v\n", decodedValues, values)
}
}
func TestEncoding_Count(t *testing.T) {
tests := []struct {
value interface{}
blockType byte
}{
{value: float64(1.0), blockType: tsm1.BlockFloat64},
{value: int64(1), blockType: tsm1.BlockInt64},
{value: true, blockType: tsm1.BlockBool},
{value: "string", blockType: tsm1.BlockString},
}
for _, test := range tests {
var values []tsm1.Value
values = append(values, tsm1.NewValue(time.Unix(0, 0), test.value))
b, err := tsm1.Values(values).Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got, exp := tsm1.BlockCount(b), 1; got != exp {
t.Fatalf("block count mismatch: got %v, exp %v", got, exp)
}
}
}
func Test_scrubValuesNoFilter(t *testing.T) {
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}
scrubbed := scrubValues(values)
if !reflect.DeepEqual(values, scrubbed) {
t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, scrubbed)
}
}
func TestTSMReader_Type(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
typ, err := r.Type("cpu")
if err != nil {
fatal(t, "reading type", err)
}
if got, exp := typ, tsm1.BlockInt64; got != exp {
t.Fatalf("type mismatch: got %v, exp %v", got, exp)
}
}
// Series returns a set of per-series data.
func (a PointsSlice) Series() map[string]*Series {
m := map[string]*Series{}
for _, points := range a {
for _, p := range points {
pp := p.Encode()
// Create series if not exists.
s := m[string(pp.Key())]
if s == nil {
s = &Series{
Name: string(pp.Key()),
Fields: make(map[string]struct{}),
}
m[string(pp.Key())] = s
}
// Append point data.
s.Values = append(s.Values, tsm1.NewValue(p.Time, p.Fields["value"]))
// Add fields.
for k := range p.Fields {
s.Fields[k] = struct{}{}
}
}
}
// Deduplicate & sort items in each series.
for _, s := range m {
s.Values = DedupeValues(s.Values)
sort.Sort(s.Values)
}
return m
}
func TestEngine_DecodeAndCombine_NoNewValues(t *testing.T) {
var newValues tsm1.Values
e := OpenDefaultEngine()
defer e.Engine.Close()
values := make(tsm1.Values, 1)
values[0] = tsm1.NewValue(time.Unix(0, 0), float64(1))
block, err := values.Encode(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
remaining, encoded, err := e.DecodeAndCombine(newValues, block, nil, time.Unix(1, 0).UnixNano(), false)
if len(remaining) != 0 {
t.Fatalf("unexpected remaining values: exp %v, got %v", 0, len(remaining))
}
if len(encoded) != len(block) {
t.Fatalf("unexpected encoded block length: exp %v, got %v", len(block), len(encoded))
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTSMReader_MMAP_Tombstone(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
defer f.Close()
w, err := tsm1.NewTSMWriter(f)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.Write("mem", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error writing index: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
f, err = os.Open(f.Name())
if err != nil {
t.Fatalf("unexpected error open file: %v", err)
}
r, err := tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
if err := r.Delete([]string{"mem"}); err != nil {
t.Fatalf("unexpected error deleting: %v", err)
}
r, err = tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
defer r.Close()
if got, exp := len(r.Keys()), 1; got != exp {
t.Fatalf("key length mismatch: got %v, exp %v", got, exp)
}
}
func TestBlockIterator_Single(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
var count int
iter := r.BlockIterator()
for iter.Next() {
key, minTime, maxTime, buf, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error creating iterator: %v", err)
}
if got, exp := key, "cpu"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := minTime, time.Unix(0, 0); got != exp {
t.Fatalf("min time mismatch: got %v, exp %v", got, exp)
}
if got, exp := maxTime, time.Unix(0, 0); got != exp {
t.Fatalf("max time mismatch: got %v, exp %v", got, exp)
}
if len(buf) == 0 {
t.Fatalf("buf length = 0")
}
count++
}
if got, exp := count, len(values); got != exp {
t.Fatalf("value count mismatch: got %v, exp %v", got, exp)
}
}
func TestWALWriter_WritePoints_LargeBatch(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
w := tsm1.NewWALSegmentWriter(f)
var points []tsm1.Value
for i := 0; i < 100000; i++ {
points = append(points, tsm1.NewValue(time.Unix(int64(i), 0), int64(1)))
}
values := map[string][]tsm1.Value{
"cpu,host=A,server=01,foo=bar,tag=really-long#!~#float": points,
"mem,host=A,server=01,foo=bar,tag=really-long#!~#float": points,
}
entry := &tsm1.WriteWALEntry{
Values: values,
}
if err := w.Write(mustMarshalEntry(entry)); err != nil {
fatal(t, "write points", err)
}
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
fatal(t, "seek", err)
}
r := tsm1.NewWALSegmentReader(f)
if !r.Next() {
t.Fatalf("expected next, got false")
}
we, err := r.Read()
if err != nil {
fatal(t, "read entry", err)
}
e, ok := we.(*tsm1.WriteWALEntry)
if !ok {
t.Fatalf("expected WriteWALEntry: got %#v", e)
}
for k, v := range e.Values {
for i, vv := range v {
if got, exp := vv.String(), values[k][i].String(); got != exp {
t.Fatalf("points mismatch: got %v, exp %v", got, exp)
}
}
}
if n := r.Count(); n != MustReadFileSize(f) {
t.Fatalf("wrong count of bytes read, got %d, exp %d", n, MustReadFileSize(f))
}
}