func TestCacheKeyIterator_Chunked(t *testing.T) {
v0 := tsm1.NewValue(1, 1.0)
v1 := tsm1.NewValue(2, 2.0)
writes := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v0, v1},
}
c := tsm1.NewCache(0, "")
for k, v := range writes {
if err := c.Write(k, v); err != nil {
t.Fatalf("failed to write key foo to cache: %s", err.Error())
}
}
iter := tsm1.NewCacheKeyIterator(c, 1)
var readValues bool
var chunk int
for iter.Next() {
key, _, _, block, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error read: %v", err)
}
values, err := tsm1.DecodeBlock(block, nil)
if err != nil {
t.Fatalf("unexpected error decode: %v", err)
}
if got, exp := key, "cpu,host=A#!~#value"; got != exp {
t.Fatalf("key mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(values), 1; got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
for _, v := range values {
readValues = true
assertValueEqual(t, v, writes["cpu,host=A#!~#value"][chunk])
}
chunk++
}
if !readValues {
t.Fatalf("failed to read any values")
}
}
func TestCacheRace(t *testing.T) {
values := make(tsm1.Values, 1000)
timestamps := make([]int64, len(values))
series := make([]string, 100)
for i := range timestamps {
timestamps[i] = int64(rand.Int63n(int64(len(values))))
}
for i := range values {
values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
}
for i := range series {
series[i] = fmt.Sprintf("series%d", i)
}
wg := sync.WaitGroup{}
c := tsm1.NewCache(1000000, "")
ch := make(chan struct{})
for _, s := range series {
for _, v := range values {
c.Write(s, tsm1.Values{v})
}
wg.Add(1)
go func(s string) {
defer wg.Done()
<-ch
c.Values(s)
}(s)
}
wg.Add(1)
go func() {
wg.Done()
<-ch
s, err := c.Snapshot()
if err == tsm1.ErrSnapshotInProgress {
return
}
if err != nil {
t.Fatalf("failed to snapshot cache: %v", err)
}
s.Deduplicate()
c.ClearSnapshot(true)
}()
close(ch)
wg.Wait()
}
func TestCheckConcurrentReadsAreSafe(t *testing.T) {
values := make(tsm1.Values, 1000)
timestamps := make([]int64, len(values))
series := make([]string, 100)
for i := range timestamps {
timestamps[i] = int64(rand.Int63n(int64(len(values))))
}
for i := range values {
values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
}
for i := range series {
series[i] = fmt.Sprintf("series%d", i)
}
wg := sync.WaitGroup{}
c := tsm1.NewCache(1000000, "")
ch := make(chan struct{})
for _, s := range series {
for _, v := range values {
c.Write(s, tsm1.Values{v})
}
wg.Add(3)
go func(s string) {
defer wg.Done()
<-ch
c.Values(s)
}(s)
go func(s string) {
defer wg.Done()
<-ch
c.Values(s)
}(s)
go func(s string) {
defer wg.Done()
<-ch
c.Values(s)
}(s)
}
close(ch)
wg.Wait()
}
// Tests compacting a Cache snapshot into a single TSM file
func TestCompactor_Snapshot(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
v1 := tsm1.NewValue(1, float64(1))
v2 := tsm1.NewValue(1, float64(1))
v3 := tsm1.NewValue(2, float64(2))
points1 := map[string][]tsm1.Value{
"cpu,host=A#!~#value": []tsm1.Value{v1},
"cpu,host=B#!~#value": []tsm1.Value{v2, v3},
}
c := tsm1.NewCache(0, "")
for k, v := range points1 {
if err := c.Write(k, v); err != nil {
t.Fatalf("failed to write key foo to cache: %s", err.Error())
}
}
compactor := &tsm1.Compactor{
Dir: dir,
FileStore: &fakeFileStore{},
}
files, err := compactor.WriteSnapshot(c)
if err != nil {
t.Fatalf("unexpected error writing snapshot: %v", err)
}
if got, exp := len(files), 1; got != exp {
t.Fatalf("files length mismatch: got %v, exp %v", got, exp)
}
r := MustOpenTSMReader(files[0])
keys := r.Keys()
if got, exp := len(keys), 2; got != exp {
t.Fatalf("keys length mismatch: got %v, exp %v", got, exp)
}
var data = []struct {
key string
points []tsm1.Value
}{
{"cpu,host=A#!~#value", []tsm1.Value{v1}},
{"cpu,host=B#!~#value", []tsm1.Value{v2, v3}},
}
for _, p := range data {
values, err := r.ReadAll(p.key)
if err != nil {
t.Fatalf("unexpected error reading: %v", err)
}
if got, exp := len(values), len(p.points); got != exp {
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
}
for i, point := range p.points {
assertValueEqual(t, values[i], point)
}
}
}
func TestCacheRace2Compacters(t *testing.T) {
values := make(tsm1.Values, 1000)
timestamps := make([]int64, len(values))
series := make([]string, 100)
for i := range timestamps {
timestamps[i] = int64(rand.Int63n(int64(len(values))))
}
for i := range values {
values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
}
for i := range series {
series[i] = fmt.Sprintf("series%d", i)
}
wg := sync.WaitGroup{}
c := tsm1.NewCache(1000000, "")
ch := make(chan struct{})
for _, s := range series {
for _, v := range values {
c.Write(s, tsm1.Values{v})
}
wg.Add(1)
go func(s string) {
defer wg.Done()
<-ch
c.Values(s)
}(s)
}
fileCounter := 0
mapFiles := map[int]bool{}
mu := sync.Mutex{}
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
wg.Done()
<-ch
s, err := c.Snapshot()
if err == tsm1.ErrSnapshotInProgress {
return
}
if err != nil {
t.Fatalf("failed to snapshot cache: %v", err)
}
mu.Lock()
mapFiles[fileCounter] = true
fileCounter++
myFiles := map[int]bool{}
for k, e := range mapFiles {
myFiles[k] = e
}
mu.Unlock()
s.Deduplicate()
c.ClearSnapshot(true)
mu.Lock()
defer mu.Unlock()
for k, _ := range myFiles {
if _, ok := mapFiles[k]; !ok {
t.Fatalf("something else deleted one of my files")
} else {
delete(mapFiles, k)
}
}
}()
}
close(ch)
wg.Wait()
}