func TestWALWriter_WritePoints_Single(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) p1 := tsm1.NewValue(time.Unix(1, 0), 1.1) p2 := tsm1.NewValue(time.Unix(1, 0), int64(1)) p3 := tsm1.NewValue(time.Unix(1, 0), true) p4 := tsm1.NewValue(time.Unix(1, 0), "string") values := map[string][]tsm1.Value{ "cpu,host=A#!~#float": []tsm1.Value{p1}, "cpu,host=A#!~#int": []tsm1.Value{p2}, "cpu,host=A#!~#bool": []tsm1.Value{p3}, "cpu,host=A#!~#string": []tsm1.Value{p4}, } entry := &tsm1.WriteWALEntry{ Values: values, } if err := w.Write(mustMarshalEntry(entry)); err != nil { fatal(t, "write points", err) } if _, err := f.Seek(0, os.SEEK_SET); err != nil { fatal(t, "seek", err) } r := tsm1.NewWALSegmentReader(f) if !r.Next() { t.Fatalf("expected next, got false") } we, err := r.Read() if err != nil { fatal(t, "read entry", err) } e, ok := we.(*tsm1.WriteWALEntry) if !ok { t.Fatalf("expected WriteWALEntry: got %#v", e) } for k, v := range e.Values { for i, vv := range v { if got, exp := vv.String(), values[k][i].String(); got != exp { t.Fatalf("points mismatch: got %v, exp %v", got, exp) } } } if n := r.Count(); n != MustReadFileSize(f) { t.Fatalf("wrong count of bytes read, got %d, exp %d", n, MustReadFileSize(f)) } }
func TestWALWriter_WritePoints_LargeBatch(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) var points []tsm1.Value for i := 0; i < 100000; i++ { points = append(points, tsm1.NewValue(time.Unix(int64(i), 0), int64(1))) } values := map[string][]tsm1.Value{ "cpu,host=A,server=01,foo=bar,tag=really-long#!~#float": points, "mem,host=A,server=01,foo=bar,tag=really-long#!~#float": points, } entry := &tsm1.WriteWALEntry{ Values: values, } if err := w.Write(mustMarshalEntry(entry)); err != nil { fatal(t, "write points", err) } if _, err := f.Seek(0, os.SEEK_SET); err != nil { fatal(t, "seek", err) } r := tsm1.NewWALSegmentReader(f) if !r.Next() { t.Fatalf("expected next, got false") } we, err := r.Read() if err != nil { fatal(t, "read entry", err) } e, ok := we.(*tsm1.WriteWALEntry) if !ok { t.Fatalf("expected WriteWALEntry: got %#v", e) } for k, v := range e.Values { for i, vv := range v { if got, exp := vv.String(), values[k][i].String(); got != exp { t.Fatalf("points mismatch: got %v, exp %v", got, exp) } } } if n := r.Count(); n != MustReadFileSize(f) { t.Fatalf("wrong count of bytes read, got %d, exp %d", n, MustReadFileSize(f)) } }
func TestWALWriter_Corrupt(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) corruption := []byte{1, 4, 0, 0, 0} p1 := tsm1.NewValue(time.Unix(1, 0), 1.1) values := map[string][]tsm1.Value{ "cpu,host=A#!~#float": []tsm1.Value{p1}, } entry := &tsm1.WriteWALEntry{ Values: values, } if err := w.Write(mustMarshalEntry(entry)); err != nil { fatal(t, "write points", err) } // Write some random bytes to the file to simulate corruption. if _, err := f.Write(corruption); err != nil { fatal(t, "corrupt WAL segment", err) } // Create the WAL segment reader. if _, err := f.Seek(0, os.SEEK_SET); err != nil { fatal(t, "seek", err) } r := tsm1.NewWALSegmentReader(f) // Try to decode two entries. if !r.Next() { t.Fatalf("expected next, got false") } if _, err := r.Read(); err != nil { fatal(t, "read entry", err) } if !r.Next() { t.Fatalf("expected next, got false") } if _, err := r.Read(); err == nil { fatal(t, "read entry did not return err", nil) } // Count should only return size of valid data. expCount := MustReadFileSize(f) - int64(len(corruption)) if n := r.Count(); n != expCount { t.Fatalf("wrong count of bytes read, got %d, exp %d", n, expCount) } }
func MustWALSegment(dir string, entries []tsm1.WALEntry) *tsm1.WALSegmentReader { f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) for _, e := range entries { if err := w.Write(mustMarshalEntry(e)); err != nil { panic(fmt.Sprintf("write WAL entry: %v", err)) } } if _, err := f.Seek(0, os.SEEK_SET); err != nil { panic(fmt.Sprintf("seek WAL: %v", err)) } return tsm1.NewWALSegmentReader(f) }
func TestWALWriter_WriteDelete_Single(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) entry := &tsm1.DeleteWALEntry{ Keys: []string{"cpu"}, } if err := w.Write(mustMarshalEntry(entry)); err != nil { fatal(t, "write points", err) } if _, err := f.Seek(0, os.SEEK_SET); err != nil { fatal(t, "seek", err) } r := tsm1.NewWALSegmentReader(f) if !r.Next() { t.Fatalf("expected next, got false") } we, err := r.Read() if err != nil { fatal(t, "read entry", err) } e, ok := we.(*tsm1.DeleteWALEntry) if !ok { t.Fatalf("expected WriteWALEntry: got %#v", e) } if got, exp := len(e.Keys), len(entry.Keys); got != exp { t.Fatalf("key length mismatch: got %v, exp %v", got, exp) } if got, exp := e.Keys[0], entry.Keys[0]; got != exp { t.Fatalf("key mismatch: got %v, exp %v", got, exp) } }
func BenchmarkWALSegmentReader(b *testing.B) { points := map[string][]tsm1.Value{} for i := 0; i < 5000; i++ { k := "cpu,host=A#!~#value" points[k] = append(points[k], tsm1.NewValue(time.Unix(int64(i), 0), 1.1)) } dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) write := &tsm1.WriteWALEntry{ Values: points, } for i := 0; i < 100; i++ { if err := w.Write(mustMarshalEntry(write)); err != nil { b.Fatalf("unexpected error writing entry: %v", err) } } r := tsm1.NewWALSegmentReader(f) b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() f.Seek(0, os.SEEK_SET) b.StartTimer() for r.Next() { _, err := r.Read() if err != nil { b.Fatalf("unexpected error reading entry: %v", err) } } } }
func TestWALWriter_WritePointsDelete_Multiple(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) p1 := tsm1.NewValue(time.Unix(1, 0), true) values := map[string][]tsm1.Value{ "cpu,host=A#!~#value": []tsm1.Value{p1}, } writeEntry := &tsm1.WriteWALEntry{ Values: values, } if err := w.Write(mustMarshalEntry(writeEntry)); err != nil { fatal(t, "write points", err) } // Write the delete entry deleteEntry := &tsm1.DeleteWALEntry{ Keys: []string{"cpu,host=A#!~value"}, } if err := w.Write(mustMarshalEntry(deleteEntry)); err != nil { fatal(t, "write points", err) } // Seek back to the beinning of the file for reading if _, err := f.Seek(0, os.SEEK_SET); err != nil { fatal(t, "seek", err) } r := tsm1.NewWALSegmentReader(f) // Read the write points first if !r.Next() { t.Fatalf("expected next, got false") } we, err := r.Read() if err != nil { fatal(t, "read entry", err) } e, ok := we.(*tsm1.WriteWALEntry) if !ok { t.Fatalf("expected WriteWALEntry: got %#v", e) } for k, v := range e.Values { if got, exp := len(v), len(values[k]); got != exp { t.Fatalf("values length mismatch: got %v, exp %v", got, exp) } for i, vv := range v { if got, exp := vv.String(), values[k][i].String(); got != exp { t.Fatalf("points mismatch: got %v, exp %v", got, exp) } } } // Read the delete second if !r.Next() { t.Fatalf("expected next, got false") } we, err = r.Read() if err != nil { fatal(t, "read entry", err) } de, ok := we.(*tsm1.DeleteWALEntry) if !ok { t.Fatalf("expected DeleteWALEntry: got %#v", e) } if got, exp := len(de.Keys), len(deleteEntry.Keys); got != exp { t.Fatalf("key length mismatch: got %v, exp %v", got, exp) } if got, exp := de.Keys[0], deleteEntry.Keys[0]; got != exp { t.Fatalf("key mismatch: got %v, exp %v", got, exp) } }
func TestWALWriter_WritePoints_Multiple(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w := tsm1.NewWALSegmentWriter(f) p1 := tsm1.NewValue(time.Unix(1, 0), int64(1)) p2 := tsm1.NewValue(time.Unix(1, 0), int64(2)) exp := []struct { key string values []tsm1.Value }{ {"cpu,host=A#!~#value", []tsm1.Value{p1}}, {"cpu,host=B#!~#value", []tsm1.Value{p2}}, } for _, v := range exp { entry := &tsm1.WriteWALEntry{ Values: map[string][]tsm1.Value{v.key: v.values}, } if err := w.Write(mustMarshalEntry(entry)); err != nil { fatal(t, "write points", err) } } // Seek back to the beinning of the file for reading if _, err := f.Seek(0, os.SEEK_SET); err != nil { fatal(t, "seek", err) } r := tsm1.NewWALSegmentReader(f) for _, ep := range exp { if !r.Next() { t.Fatalf("expected next, got false") } we, err := r.Read() if err != nil { fatal(t, "read entry", err) } e, ok := we.(*tsm1.WriteWALEntry) if !ok { t.Fatalf("expected WriteWALEntry: got %#v", e) } for k, v := range e.Values { if got, exp := k, ep.key; got != exp { t.Fatalf("key mismatch. got %v, exp %v", got, exp) } if got, exp := len(v), len(ep.values); got != exp { t.Fatalf("values length mismatch: got %v, exp %v", got, exp) } for i, vv := range v { if got, exp := vv.String(), ep.values[i].String(); got != exp { t.Fatalf("points mismatch: got %v, exp %v", got, exp) } } } } if n := r.Count(); n != MustReadFileSize(f) { t.Fatalf("wrong count of bytes read, got %d, exp %d", n, MustReadFileSize(f)) } }
func (cmd *Command) writeWALFiles(w io.WriteCloser, files []string, key string) error { fmt.Fprintln(w, "# writing wal data") // we need to make sure we write the same order that the wal received the data sort.Strings(files) var once sync.Once warn := func() { msg := fmt.Sprintf(`WARNING: detected deletes in wal file. Some series for %q may be brought back by replaying this data. To resolve, you can either let the shard snapshot prior to exporting the data or manually editing the exported file. `, key) fmt.Fprintln(cmd.Stderr, msg) } // use a function here to close the files in the defers and not let them accumulate in the loop write := func(f string) error { file, err := os.OpenFile(f, os.O_RDONLY, 0600) if err != nil { return fmt.Errorf("%v", err) } defer file.Close() reader := tsm1.NewWALSegmentReader(file) defer reader.Close() for reader.Next() { entry, err := reader.Read() if err != nil { n := reader.Count() fmt.Fprintf(os.Stderr, "file %s corrupt at position %d", file.Name(), n) break } switch t := entry.(type) { case *tsm1.DeleteWALEntry: once.Do(warn) continue case *tsm1.DeleteRangeWALEntry: once.Do(warn) continue case *tsm1.WriteWALEntry: var pairs string for key, values := range t.Values { measurement, field := tsm1.SeriesAndFieldFromCompositeKey([]byte(key)) for _, value := range values { if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) { continue } switch value.Value().(type) { case float64: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) case int64: pairs = field + "=" + fmt.Sprintf("%vi", value.Value()) case bool: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) case string: pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value()))) default: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) } fmt.Fprintln(w, string(measurement), pairs, value.UnixNano()) } } } } return nil } for _, f := range files { if err := write(f); err != nil { return err } } return nil }