// Ensures that a compaction will properly rollover to a new file when the
// max keys per blocks is exceeded
func TestCompactor_CompactFull_MaxKeys(t *testing.T) {
// This test creates a lot of data and causes timeout failures for these envs
if testing.Short() || os.Getenv("CI") != "" || os.Getenv("GORACE") != "" {
t.Skip("Skipping max keys compaction test")
}
dir := MustTempDir()
defer os.RemoveAll(dir)
// write two files where the first contains a single key with the maximum
// number of full blocks that can fit in a TSM file
f1, f1Name := MustTSMWriter(dir, 1)
values := make([]tsm1.Value, 1000)
for i := 0; i < 65535; i++ {
values = values[:0]
for j := 0; j < 1000; j++ {
values = append(values, tsm1.NewValue(int64(i*1000+j), int64(1)))
}
if err := f1.Write("cpu,host=A#!~#value", values); err != nil {
t.Fatalf("write tsm f1: %v", err)
}
}
if err := f1.WriteIndex(); err != nil {
t.Fatalf("write index f1: %v", err)
}
f1.Close()
// Write a new file with 1 block that when compacted would exceed the max
// blocks
lastTimeStamp := values[len(values)-1].UnixNano()
values = values[:0]
f2, f2Name := MustTSMWriter(dir, 2)
for j := lastTimeStamp; j < lastTimeStamp+1000; j++ {
values = append(values, tsm1.NewValue(int64(j), int64(1)))
}
if err := f2.Write("cpu,host=A#!~#value", values); err != nil {
t.Fatalf("write tsm f1: %v", err)
}
if err := f2.WriteIndex(); err != nil {
t.Fatalf("write index f2: %v", err)
}
f2.Close()
compactor := &tsm1.Compactor{
Dir: dir,
FileStore: &fakeFileStore{},
}
compactor.Open()
// Compact both files, should get 2 files back
files, err := compactor.CompactFull([]string{f1Name, f2Name})
if err != nil {
t.Fatalf("unexpected error writing snapshot: %v", err)
}
if got, exp := len(files), 2; got != exp {
t.Fatalf("files length mismatch: got %v, exp %v", got, exp)
}
expGen, expSeq, err := tsm1.ParseTSMFileName(f2Name)
if err != nil {
t.Fatalf("unexpected error parsing file name: %v", err)
}
expSeq = expSeq + 1
gotGen, gotSeq, err := tsm1.ParseTSMFileName(files[0])
if err != nil {
t.Fatalf("unexpected error parsing file name: %v", err)
}
if gotGen != expGen {
t.Fatalf("wrong generation for new file: got %v, exp %v", gotGen, expGen)
}
if gotSeq != expSeq {
t.Fatalf("wrong sequence for new file: got %v, exp %v", gotSeq, expSeq)
}
}
// Ensures that a compaction will properly merge multiple TSM files
func TestCompactor_CompactFull(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
// write 3 TSM files with different data and one new point
a1 := tsm1.NewValue(1, 1.1)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{a1},
}
f1 := MustWriteTSM(dir, 1, writes)
a2 := tsm1.NewValue(2, 1.2)
b1 := tsm1.NewValue(1, 2.1)
writes = map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{a2},
"cpu,host=B#!~#value": []tsm1.Value{b1},
}
f2 := MustWriteTSM(dir, 2, writes)
a3 := tsm1.NewValue(1, 1.3)
c1 := tsm1.NewValue(1, 3.1)
writes = map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{a3},
"cpu,host=C#!~#value": []tsm1.Value{c1},
}
f3 := MustWriteTSM(dir, 3, writes)
compactor := &tsm1.Compactor{
Dir: dir,
FileStore: &fakeFileStore{},
}
files, err := compactor.CompactFull([]string{f1, f2, f3})
if err == nil {
t.Fatalf("expected error writing snapshot: %v", err)
}
if len(files) > 0 {
t.Fatalf("no files should be compacted: got %v", len(files))
}
compactor.Open()
files, err = compactor.CompactFull([]string{f1, f2, f3})
if err != nil {
t.Fatalf("unexpected error writing snapshot: %v", err)
}
if got, exp := len(files), 1; got != exp {
t.Fatalf("files length mismatch: got %v, exp %v", got, exp)
}
expGen, expSeq, err := tsm1.ParseTSMFileName(f3)
if err != nil {
t.Fatalf("unexpected error parsing file name: %v", err)
}
expSeq = expSeq + 1
gotGen, gotSeq, err := tsm1.ParseTSMFileName(files[0])
if err != nil {
t.Fatalf("unexpected error parsing file name: %v", err)
}
if gotGen != expGen {
t.Fatalf("wrong generation for new file: got %v, exp %v", gotGen, expGen)
}
if gotSeq != expSeq {
t.Fatalf("wrong sequence for new file: got %v, exp %v", gotSeq, expSeq)
}
r := MustOpenTSMReader(files[0])
if got, exp := r.KeyCount(), 3; got != exp {
t.Fatalf("keys length mismatch: got %v, exp %v", got, exp)
}
var data = []struct {
key string
points []tsm1.Value
}{
{"cpu,host=A#!~#value", []tsm1.Value{a3, a2}},
{"cpu,host=B#!~#value", []tsm1.Value{b1}},
{"cpu,host=C#!~#value", []tsm1.Value{c1}},
}
for _, p := range data {
values, err := r.ReadAll(p.key)
if err != nil {
t.Fatalf("unexpected error reading: %v", err)
}
if got, exp := len(values), len(p.points); got != exp {
t.Fatalf("values length mismatch %s: got %v, exp %v", p.key, got, exp)
}
for i, point := range p.points {
assertValueEqual(t, values[i], point)
}
}
}
// Ensures that a full compaction will decode and combine blocks with
// multiple tombstoned ranges within the block e.g. (t1, t2, t3, t4)
// having t2 and t3 removed
func TestCompactor_CompactFull_TombstonedMultipleRanges(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
// write 3 TSM files with different data and one new point
a1 := tsm1.NewValue(1, 1.1)
a2 := tsm1.NewValue(2, 1.2)
a3 := tsm1.NewValue(3, 1.3)
a4 := tsm1.NewValue(4, 1.4)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{a1, a2, a3, a4},
}
f1 := MustWriteTSM(dir, 1, writes)
ts := tsm1.Tombstoner{
Path: f1,
}
// a1, a3 should remain after compaction
ts.AddRange([]string{"cpu,host=A#!~#value"}, 2, 2)
ts.AddRange([]string{"cpu,host=A#!~#value"}, 4, 4)
a5 := tsm1.NewValue(5, 1.5)
writes = map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{a5},
}
f2 := MustWriteTSM(dir, 2, writes)
a6 := tsm1.NewValue(6, 1.6)
writes = map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{a6},
}
f3 := MustWriteTSM(dir, 3, writes)
compactor := &tsm1.Compactor{
Dir: dir,
FileStore: &fakeFileStore{},
Size: 2,
}
compactor.Open()
files, err := compactor.CompactFull([]string{f1, f2, f3})
if err != nil {
t.Fatalf("unexpected error writing snapshot: %v", err)
}
if got, exp := len(files), 1; got != exp {
t.Fatalf("files length mismatch: got %v, exp %v", got, exp)
}
expGen, expSeq, err := tsm1.ParseTSMFileName(f3)
if err != nil {
t.Fatalf("unexpected error parsing file name: %v", err)
}
expSeq = expSeq + 1
gotGen, gotSeq, err := tsm1.ParseTSMFileName(files[0])
if err != nil {
t.Fatalf("unexpected error parsing file name: %v", err)
}
if gotGen != expGen {
t.Fatalf("wrong generation for new file: got %v, exp %v", gotGen, expGen)
}
if gotSeq != expSeq {
t.Fatalf("wrong sequence for new file: got %v, exp %v", gotSeq, expSeq)
}
r := MustOpenTSMReader(files[0])
if got, exp := r.KeyCount(), 1; got != exp {
t.Fatalf("keys length mismatch: got %v, exp %v", got, exp)
}
var data = []struct {
key string
points []tsm1.Value
}{
{"cpu,host=A#!~#value", []tsm1.Value{a1, a3, a5, a6}},
}
for _, p := range data {
values, err := r.ReadAll(p.key)
if err != nil {
t.Fatalf("unexpected error reading: %v", err)
}
if got, exp := len(values), len(p.points); got != exp {
t.Fatalf("values length mismatch %s: got %v, exp %v", p.key, got, exp)
}
for i, point := range p.points {
assertValueEqual(t, values[i], point)
}
}
if got, exp := len(r.Entries("cpu,host=A#!~#value")), 2; got != exp {
t.Fatalf("block count mismatch: got %v, exp %v", got, exp)
}
}