func TestTSMReader_MMAP_Tombstone(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) defer f.Close() w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(0, 1.0)} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.Write("mem", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } f, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } if err := r.Delete([]string{"mem"}); err != nil { t.Fatalf("unexpected error deleting: %v", err) } r, err = tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() if got, exp := r.KeyCount(), 1; got != exp { t.Fatalf("key length mismatch: got %v, exp %v", got, exp) } }
func TestTSMReader_Type(t *testing.T) { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } typ, err := r.Type("cpu") if err != nil { fatal(t, "reading type", err) } if got, exp := typ, tsm1.BlockInteger; got != exp { t.Fatalf("type mismatch: got %v, exp %v", got, exp) } }
func TestTSMWriter_Write_Multiple(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } var data = []struct { key string values []tsm1.Value }{ {"cpu", []tsm1.Value{tsm1.NewValue(0, 1.0)}}, {"mem", []tsm1.Value{tsm1.NewValue(1, 2.0)}}, } for _, d := range data { if err := w.Write(d.key, d.values); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } fd, err := os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(fd) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() for _, d := range data { readValues, err := r.ReadAll(d.key) if err != nil { t.Fatalf("unexpected error readin: %v", err) } if exp := len(d.values); exp != len(readValues) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp) } for i, v := range d.values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } } }
func TestIndirectIndex_UnmarshalBinary_BlockCountOverflow(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) defer f.Close() w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } for i := 0; i < 3280; i++ { w.Write("cpu", []tsm1.Value{tsm1.NewValue(int64(i), float64(i))}) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } f, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() }
func newFiles(values ...keyValues) ([]tsm1.TSMFile, error) { var files []tsm1.TSMFile for _, v := range values { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { return nil, err } if err := w.Write(v.key, v.values); err != nil { return nil, err } if err := w.WriteIndex(); err != nil { return nil, err } if err := w.Close(); err != nil { return nil, err } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { return nil, err } files = append(files, r) } return files, nil }
func TestTSMReader_MMAP_TombstoneMultipleRanges(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) defer f.Close() w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } expValues := []tsm1.Value{ tsm1.NewValue(1, 1.0), tsm1.NewValue(2, 2.0), tsm1.NewValue(3, 3.0), tsm1.NewValue(4, 4.0), tsm1.NewValue(5, 5.0), } if err := w.Write("cpu", expValues); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } f, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() if err := r.DeleteRange([]string{"cpu"}, 2, 2); err != nil { t.Fatalf("unexpected error deleting: %v", err) } if err := r.DeleteRange([]string{"cpu"}, 4, 4); err != nil { t.Fatalf("unexpected error deleting: %v", err) } values, err := r.ReadAll("cpu") if err != nil { t.Fatalf("unexpected error reading all: %v", err) } if got, exp := len(values), 3; got != exp { t.Fatalf("values length mismatch: got %v, exp %v", got, exp) } }
// Tests that writing keys in reverse is able to read them back. func TestTSMWriter_Write_SameKey(t *testing.T) { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } var data = []struct { key string values []tsm1.Value }{ {"cpu", []tsm1.Value{ tsm1.NewValue(0, 1.0), tsm1.NewValue(1, 2.0)}, }, {"cpu", []tsm1.Value{ tsm1.NewValue(2, 3.0), tsm1.NewValue(3, 4.0)}, }, } for _, d := range data { if err := w.Write(d.key, d.values); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } values := append(data[0].values, data[1].values...) readValues, err := r.ReadAll("cpu") if err != nil { t.Fatalf("unexpected error readin: %v", err) } if exp := len(values); exp != len(readValues) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp) } for i, v := range values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } }
func TestBlockIterator_Single(t *testing.T) { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } var count int iter := r.BlockIterator() for iter.Next() { key, minTime, maxTime, buf, err := iter.Read() if err != nil { t.Fatalf("unexpected error creating iterator: %v", err) } if got, exp := key, "cpu"; got != exp { t.Fatalf("key mismatch: got %v, exp %v", got, exp) } if got, exp := minTime, time.Unix(0, 0); got != exp { t.Fatalf("min time mismatch: got %v, exp %v", got, exp) } if got, exp := maxTime, time.Unix(0, 0); got != exp { t.Fatalf("max time mismatch: got %v, exp %v", got, exp) } if len(buf) == 0 { t.Fatalf("buf length = 0") } count++ } if got, exp := count, len(values); got != exp { t.Fatalf("value count mismatch: got %v, exp %v", got, exp) } }
func MustOpenTSMReader(name string) *tsm1.TSMReader { f, err := os.Open(name) if err != nil { panic(fmt.Sprintf("open file: %v", err)) } r, err := tsm1.NewTSMReader(f) if err != nil { panic(fmt.Sprintf("new reader: %v", err)) } return r }
// Ensure that we return an error if we try to open a non-tsm file func TestTSMReader_VerifiesFileType(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) defer f.Close() // write some garbage f.Write([]byte{0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77}) _, err := tsm1.NewTSMReader(f) if err == nil { t.Fatal("expected error trying to open non-tsm file") } }
func TestCompacted_NotFull(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(0, 1.0)} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } fd, err := os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(fd) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } iter := r.BlockIterator() if !iter.Next() { t.Fatalf("expected next, got false") } _, _, _, _, block, err := iter.Read() if err != nil { t.Fatalf("unexpected error reading block: %v", err) } if got, exp := tsm1.BlockCount(block), 1; got != exp { t.Fatalf("block count mismatch: got %v, exp %v", got, exp) } }
func TestTSMWriter_Write_Single(t *testing.T) { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(0, 1.0)} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if got, exp := len(b.Bytes()), 5; got < exp { t.Fatalf("file size mismatch: got %v, exp %v", got, exp) } if got := binary.BigEndian.Uint32(b.Bytes()[0:4]); got != tsm1.MagicNumber { t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber) } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } readValues, err := r.ReadAll("cpu") if err != nil { t.Fatalf("unexpected error readin: %v", err) } if len(readValues) != len(values) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), len(values)) } for i, v := range values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } }
func newFiles(dir string, values ...keyValues) ([]tsm1.TSMFile, error) { var files []tsm1.TSMFile id := 1 for _, v := range values { f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { return nil, err } if err := w.Write(v.key, v.values); err != nil { return nil, err } if err := w.WriteIndex(); err != nil { return nil, err } if err := w.Close(); err != nil { return nil, err } newName := filepath.Join(filepath.Dir(f.Name()), tsmFileName(id)) if err := os.Rename(f.Name(), newName); err != nil { return nil, err } id++ fd, err := os.Open(newName) if err != nil { return nil, err } r, err := tsm1.NewTSMReader(fd) if err != nil { return nil, err } files = append(files, r) } return files, nil }
func TestTSMReader_Type(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(0, int64(1))} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } f, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error opening: %v", err) } r, err := tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } typ, err := r.Type("cpu") if err != nil { fatal(t, "reading type", err) } if got, exp := typ, tsm1.BlockInteger; got != exp { t.Fatalf("type mismatch: got %v, exp %v", got, exp) } }
func TestCompacted_NotFull(t *testing.T) { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } iter := r.BlockIterator() if !iter.Next() { t.Fatalf("expected next, got false") } _, _, _, block, err := iter.Read() if err != nil { t.Fatalf("unexpected error reading block: %v", err) } if got, exp := tsm1.BlockCount(block), 1; got != exp { t.Fatalf("block count mismatch: got %v, exp %v", got, exp) } }
func TestTSMReader_MMAP_TombstoneOutsideRange(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) defer f.Close() w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } cpuValues := []tsm1.Value{ tsm1.NewValue(1, 1.0), tsm1.NewValue(2, 2.0), tsm1.NewValue(3, 3.0), } if err := w.Write("cpu", cpuValues); err != nil { t.Fatalf("unexpected error writing: %v", err) } memValues := []tsm1.Value{ tsm1.NewValue(1, 1.0), tsm1.NewValue(2, 2.0), tsm1.NewValue(30, 3.0), } if err := w.Write("mem", memValues); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } f, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } if err := r.DeleteRange([]string{"cpu", "mem"}, 5, math.MaxInt64); err != nil { t.Fatalf("unexpected error deleting: %v", err) } defer r.Close() if got, exp := r.KeyCount(), 2; got != exp { t.Fatalf("key count mismatch: got %v, exp %v", got, exp) } if got, exp := len(r.TombstoneRange("cpu")), 0; got != exp { t.Fatalf("tombstone range mismatch: got %v, exp %v", got, exp) } values, err := r.ReadAll("cpu") if err != nil { t.Fatalf("unexpected error reading all: %v", err) } if got, exp := len(values), len(cpuValues); got != exp { t.Fatalf("values length mismatch: got %v, exp %v", got, exp) } if got, exp := len(r.TombstoneRange("mem")), 1; got != exp { t.Fatalf("tombstone range mismatch: got %v, exp %v", got, exp) } values, err = r.ReadAll("mem") if err != nil { t.Fatalf("unexpected error reading all: %v", err) } if got, exp := len(values), len(memValues[:2]); got != exp { t.Fatalf("values length mismatch: got %v, exp %v", got, exp) } }
func TestTSMWriter_WriteBlock_Multiple(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } var data = []struct { key string values []tsm1.Value }{ {"cpu", []tsm1.Value{tsm1.NewValue(0, 1.0)}}, {"mem", []tsm1.Value{tsm1.NewValue(1, 2.0)}}, } for _, d := range data { if err := w.Write(d.key, d.values); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } fd, err := os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } defer fd.Close() b, err := ioutil.ReadAll(fd) if err != nil { t.Fatalf("unexpected error read all: %v", err) } if got, exp := len(b), 5; got < exp { t.Fatalf("file size mismatch: got %v, exp %v", got, exp) } if got := binary.BigEndian.Uint32(b[0:4]); got != tsm1.MagicNumber { t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber) } if _, err := fd.Seek(0, os.SEEK_SET); err != nil { t.Fatalf("error seeking: %v", err) } // Create reader for that file r, err := tsm1.NewTSMReader(fd) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } f = MustTempFile(dir) w, err = tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } iter := r.BlockIterator() for iter.Next() { key, minTime, maxTime, _, b, err := iter.Read() if err != nil { t.Fatalf("unexpected error reading block: %v", err) } if err := w.WriteBlock(key, minTime, maxTime, b); err != nil { t.Fatalf("unexpected error writing block: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } fd, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } // Now create a reader to verify the written blocks matches the originally // written file using Write r, err = tsm1.NewTSMReader(fd) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() for _, d := range data { readValues, err := r.ReadAll(d.key) if err != nil { t.Fatalf("unexpected error readin: %v", err) } if exp := len(d.values); exp != len(readValues) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp) } for i, v := range d.values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } } }
func TestTSMWriter_Write_Single(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := []tsm1.Value{tsm1.NewValue(0, 1.0)} if err := w.Write("cpu", values); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } fd, err := os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } b, err := ioutil.ReadAll(fd) if err != nil { t.Fatalf("unexpected error reading: %v", err) } if got, exp := len(b), 5; got < exp { t.Fatalf("file size mismatch: got %v, exp %v", got, exp) } if got := binary.BigEndian.Uint32(b[0:4]); got != tsm1.MagicNumber { t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber) } if _, err := fd.Seek(0, os.SEEK_SET); err != nil { t.Fatalf("unexpected error seeking: %v", err) } r, err := tsm1.NewTSMReader(fd) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() readValues, err := r.ReadAll("cpu") if err != nil { t.Fatalf("unexpected error readin: %v", err) } if len(readValues) != len(values) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), len(values)) } for i, v := range values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } }
func TestTSMReader_MMAP_Stats(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) defer f.Close() w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values1 := []tsm1.Value{tsm1.NewValue(0, 1.0)} if err := w.Write("cpu", values1); err != nil { t.Fatalf("unexpected error writing: %v", err) } values2 := []tsm1.Value{tsm1.NewValue(1, 1.0)} if err := w.Write("mem", values2); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } f, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() stats := r.Stats() if got, exp := stats.MinKey, "cpu"; got != exp { t.Fatalf("min key mismatch: got %v, exp %v", got, exp) } if got, exp := stats.MaxKey, "mem"; got != exp { t.Fatalf("max key mismatch: got %v, exp %v", got, exp) } if got, exp := stats.MinTime, values1[0].UnixNano(); got != exp { t.Fatalf("min time mismatch: got %v, exp %v", got, exp) } if got, exp := stats.MaxTime, values2[0].UnixNano(); got != exp { t.Fatalf("max time mismatch: got %v, exp %v", got, exp) } if got, exp := r.KeyCount(), 2; got != exp { t.Fatalf("key length mismatch: got %v, exp %v", got, exp) } }
func cmdVerify(path string) { start := time.Now() dataPath := filepath.Join(path, "data") brokenBlocks := 0 totalBlocks := 0 // No need to do this in a loop ext := fmt.Sprintf(".%s", tsm1.TSMFileExtension) // Get all TSM files by walking through the data dir files := []string{} err := filepath.Walk(dataPath, func(path string, f os.FileInfo, err error) error { if err != nil { return err } if filepath.Ext(path) == ext { files = append(files, path) } return nil }) if err != nil { panic(err) } tw := tabwriter.NewWriter(os.Stdout, 16, 8, 0, '\t', 0) // Verify the checksums of every block in every file for _, f := range files { file, err := os.OpenFile(f, os.O_RDONLY, 0600) if err != nil { fmt.Printf("%v", err) os.Exit(1) } reader, err := tsm1.NewTSMReader(file) if err != nil { fmt.Printf("%v", err) os.Exit(1) } blockItr := reader.BlockIterator() brokenFileBlocks := 0 count := 0 for blockItr.Next() { totalBlocks++ key, _, _, checksum, buf, err := blockItr.Read() if err != nil { brokenBlocks++ fmt.Fprintf(tw, "%s: could not get checksum for key %v block %d due to error: %q\n", f, key, count, err) } else if expected := crc32.ChecksumIEEE(buf); checksum != expected { brokenBlocks++ fmt.Fprintf(tw, "%s: got %d but expected %d for key %v, block %d\n", f, checksum, expected, key, count) } count++ } if brokenFileBlocks == 0 { fmt.Fprintf(tw, "%s: healthy\n", f) } reader.Close() } fmt.Fprintf(tw, "Broken Blocks: %d / %d, in %vs\n", brokenBlocks, totalBlocks, time.Since(start).Seconds()) tw.Flush() }
func TestBlockIterator_Sorted(t *testing.T) { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := map[string][]tsm1.Value{ "mem": []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), int64(1))}, "cpu": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), float64(2))}, "disk": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), true)}, "load": []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), "string")}, } for k, v := range values { if err := w.Write(k, v); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } var count int iter := r.BlockIterator() var lastKey string for iter.Next() { key, _, _, buf, err := iter.Read() if key < lastKey { t.Fatalf("keys not sorted: got %v, last %v", key, lastKey) } lastKey = key if err != nil { t.Fatalf("unexpected error creating iterator: %v", err) } if len(buf) == 0 { t.Fatalf("buf length = 0") } count++ } if got, exp := count, len(values); got != exp { t.Fatalf("value count mismatch: got %v, exp %v", got, exp) } }
func TestBlockIterator_MultipleBlocks(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values1 := []tsm1.Value{tsm1.NewValue(0, int64(1))} if err := w.Write("cpu", values1); err != nil { t.Fatalf("unexpected error writing: %v", err) } values2 := []tsm1.Value{tsm1.NewValue(1, int64(2))} if err := w.Write("cpu", values2); err != nil { t.Fatalf("unexpected error writing: %v", err) } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } fd, err := os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error opening: %v", err) } r, err := tsm1.NewTSMReader(fd) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } var count int expData := []tsm1.Values{values1, values2} iter := r.BlockIterator() var i int for iter.Next() { key, minTime, maxTime, _, buf, err := iter.Read() if err != nil { t.Fatalf("unexpected error creating iterator: %v", err) } if got, exp := key, "cpu"; got != exp { t.Fatalf("key mismatch: got %v, exp %v", got, exp) } if got, exp := minTime, expData[i][0].UnixNano(); got != exp { t.Fatalf("min time mismatch: got %v, exp %v", got, exp) } if got, exp := maxTime, expData[i][0].UnixNano(); got != exp { t.Fatalf("max time mismatch: got %v, exp %v", got, exp) } if len(buf) == 0 { t.Fatalf("buf length = 0") } count++ i++ } if got, exp := count, 2; got != exp { t.Fatalf("value count mismatch: got %v, exp %v", got, exp) } }
func TestTSMReader_MMAP_ReadAll(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) defer f.Close() w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } var data = []struct { key string values []tsm1.Value }{ {"float", []tsm1.Value{ tsm1.NewValue(1, 1.0)}, }, {"int", []tsm1.Value{ tsm1.NewValue(1, int64(1))}, }, {"bool", []tsm1.Value{ tsm1.NewValue(1, true)}, }, {"string", []tsm1.Value{ tsm1.NewValue(1, "foo")}, }, } for _, d := range data { if err := w.Write(d.key, d.values); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error writing index: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } f, err = os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error open file: %v", err) } r, err := tsm1.NewTSMReader(f) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } defer r.Close() var count int for _, d := range data { readValues, err := r.ReadAll(d.key) if err != nil { t.Fatalf("unexpected error readin: %v", err) } if exp := len(d.values); exp != len(readValues) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp) } for i, v := range d.values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } count++ } if got, exp := count, len(data); got != exp { t.Fatalf("read values count mismatch: got %v, exp %v", got, exp) } }
func (c *cmdExport) writeFiles() error { // open our output file and create an output buffer var w io.WriteCloser w, err := os.Create(c.out) if err != nil { return err } defer w.Close() if c.compress { w = gzip.NewWriter(w) } // Write out all the DDL fmt.Fprintln(w, "# DDL") for key := range c.files { keys := strings.Split(key, string(byte(os.PathSeparator))) fmt.Fprintf(w, "CREATE DATABASE %s\n", keys[0]) fmt.Fprintf(w, "CREATE RETENTION POLICY %s ON %s DURATION inf REPLICATION 1\n", keys[1], keys[0]) } fmt.Fprintln(w, "# DML") for key, files := range c.files { keys := strings.Split(key, string(byte(os.PathSeparator))) fmt.Fprintf(w, "# CONTEXT-DATABASE:%s\n", keys[0]) fmt.Fprintf(w, "# CONTEXT-RETENTION-POLICY:%s\n", keys[1]) for _, f := range files { // use an anonymous function here to close the files in the defers and not let them // accumulate in the loop if err := func(f string) error { file, err := os.OpenFile(f, os.O_RDONLY, 0600) if err != nil { return fmt.Errorf("%v", err) } defer file.Close() reader, err := tsm1.NewTSMReader(file) if err != nil { log.Printf("unable to read %s, skipping\n", f) return nil } defer reader.Close() for i := 0; i < reader.KeyCount(); i++ { var pairs string key, typ := reader.KeyAt(i) values, _ := reader.ReadAll(key) measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key) for _, value := range values { switch typ { case tsm1.BlockFloat64: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) case tsm1.BlockInteger: pairs = field + "=" + fmt.Sprintf("%vi", value.Value()) case tsm1.BlockBoolean: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) case tsm1.BlockString: pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value()))) default: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) } fmt.Fprintln(w, measurement, pairs, value.UnixNano()) } } return nil }(f); err != nil { return err } } _ = key } return nil }
func cmdReport(opts *reportOpts) { start := time.Now() files, err := filepath.Glob(filepath.Join(opts.dir, fmt.Sprintf("*.%s", tsm1.TSMFileExtension))) if err != nil { fmt.Printf("%v\n", err) os.Exit(1) } var filtered []string if opts.pattern != "" { for _, f := range files { if strings.Contains(f, opts.pattern) { filtered = append(filtered, f) } } files = filtered } if len(files) == 0 { fmt.Printf("no tsm files at %v\n", opts.dir) os.Exit(1) } tw := tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0) fmt.Fprintln(tw, strings.Join([]string{"File", "Series", "Load Time"}, "\t")) totalSeries := hllpp.New() tagCardialities := map[string]*hllpp.HLLPP{} measCardinalities := map[string]*hllpp.HLLPP{} fieldCardinalities := map[string]*hllpp.HLLPP{} ordering := make([]chan struct{}, 0, len(files)) for range files { ordering = append(ordering, make(chan struct{})) } for _, f := range files { file, err := os.OpenFile(f, os.O_RDONLY, 0600) if err != nil { fmt.Fprintf(os.Stderr, "error: %s: %v. Skipping.\n", f, err) continue } loadStart := time.Now() reader, err := tsm1.NewTSMReader(file) if err != nil { fmt.Fprintf(os.Stderr, "error: %s: %v. Skipping.\n", file.Name(), err) continue } loadTime := time.Since(loadStart) seriesCount := reader.KeyCount() for i := 0; i < seriesCount; i++ { key, _ := reader.KeyAt(i) totalSeries.Add([]byte(key)) if opts.detailed { sep := strings.Index(key, "#!~#") seriesKey, field := key[:sep], key[sep+4:] measurement, tags, _ := models.ParseKey(seriesKey) measCount, ok := measCardinalities[measurement] if !ok { measCount = hllpp.New() measCardinalities[measurement] = measCount } measCount.Add([]byte(key)) fieldCount, ok := fieldCardinalities[measurement] if !ok { fieldCount = hllpp.New() fieldCardinalities[measurement] = fieldCount } fieldCount.Add([]byte(field)) for t, v := range tags { tagCount, ok := tagCardialities[t] if !ok { tagCount = hllpp.New() tagCardialities[t] = tagCount } tagCount.Add([]byte(v)) } } } reader.Close() fmt.Fprintln(tw, strings.Join([]string{ filepath.Base(file.Name()), strconv.FormatInt(int64(seriesCount), 10), loadTime.String(), }, "\t")) tw.Flush() } tw.Flush() println() fmt.Printf("Statistics\n") fmt.Printf(" Series:\n") fmt.Printf(" Total (est): %d\n", totalSeries.Count()) if opts.detailed { fmt.Printf(" Measurements (est):\n") for t, card := range measCardinalities { fmt.Printf(" %v: %d (%d%%)\n", t, card.Count(), int((float64(card.Count())/float64(totalSeries.Count()))*100)) } fmt.Printf(" Fields (est):\n") for t, card := range fieldCardinalities { fmt.Printf(" %v: %d\n", t, card.Count()) } fmt.Printf(" Tags (est):\n") for t, card := range tagCardialities { fmt.Printf(" %v: %d\n", t, card.Count()) } } fmt.Printf("Completed in %s\n", time.Since(start)) }
func TestBlockIterator_Sorted(t *testing.T) { dir := MustTempDir() defer os.RemoveAll(dir) f := MustTempFile(dir) w, err := tsm1.NewTSMWriter(f) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } values := map[string][]tsm1.Value{ "mem": []tsm1.Value{tsm1.NewValue(0, int64(1))}, "cpu": []tsm1.Value{tsm1.NewValue(1, float64(2))}, "disk": []tsm1.Value{tsm1.NewValue(1, true)}, "load": []tsm1.Value{tsm1.NewValue(1, "string")}, } for k, v := range values { if err := w.Write(k, v); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } fd, err := os.Open(f.Name()) if err != nil { t.Fatalf("unexpected error opening: %v", err) } r, err := tsm1.NewTSMReader(fd) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } var count int iter := r.BlockIterator() var lastKey string for iter.Next() { key, _, _, _, buf, err := iter.Read() if key < lastKey { t.Fatalf("keys not sorted: got %v, last %v", key, lastKey) } lastKey = key if err != nil { t.Fatalf("unexpected error creating iterator: %v", err) } if len(buf) == 0 { t.Fatalf("buf length = 0") } count++ } if got, exp := count, len(values); got != exp { t.Fatalf("value count mismatch: got %v, exp %v", got, exp) } }
// Tests that calling Read returns all the values for block matching the key // and timestamp func TestTSMWriter_Read_Multiple(t *testing.T) { var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } var data = []struct { key string values []tsm1.Value }{ {"cpu", []tsm1.Value{ tsm1.NewValue(0, 1.0), tsm1.NewValue(1, 2.0)}, }, {"cpu", []tsm1.Value{ tsm1.NewValue(2, 3.0), tsm1.NewValue(3, 4.0)}, }, } for _, d := range data { if err := w.Write(d.key, d.values); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } for _, values := range data { // Try the first timestamp readValues, err := r.Read("cpu", values.values[0].UnixNano()) if err != nil { t.Fatalf("unexpected error readin: %v", err) } if exp := len(values.values); exp != len(readValues) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp) } for i, v := range values.values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } // Try the last timestamp too readValues, err = r.Read("cpu", values.values[1].UnixNano()) if err != nil { t.Fatalf("unexpected error readin: %v", err) } if exp := len(values.values); exp != len(readValues) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp) } for i, v := range values.values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } } }
func TestTSMWriter_WriteBlock_Multiple(t *testing.T) { // Write a new TSM file var b bytes.Buffer w, err := tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } var data = []struct { key string values []tsm1.Value }{ {"cpu", []tsm1.Value{tsm1.NewValue(0, 1.0)}}, {"mem", []tsm1.Value{tsm1.NewValue(1, 2.0)}}, } for _, d := range data { if err := w.Write(d.key, d.values); err != nil { t.Fatalf("unexpected error writing: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if got, exp := len(b.Bytes()), 5; got < exp { t.Fatalf("file size mismatch: got %v, exp %v", got, exp) } if got := binary.BigEndian.Uint32(b.Bytes()[0:4]); got != tsm1.MagicNumber { t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber) } // Create reader for that file r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } // Using the reader, write a new file using WriteBlocks b.Reset() w, err = tsm1.NewTSMWriter(&b) if err != nil { t.Fatalf("unexpected error creating writer: %v", err) } iter := r.BlockIterator() for iter.Next() { key, minTime, maxTime, b, err := iter.Read() if err != nil { t.Fatalf("unexpected error reading block: %v", err) } if err := w.WriteBlock(key, minTime, maxTime, b); err != nil { t.Fatalf("unexpected error writing block: %v", err) } } if err := w.WriteIndex(); err != nil { t.Fatalf("unexpected error closing: %v", err) } if err := w.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } // Now create a reader to verify the written blocks matches the originally // written file using Write r, err = tsm1.NewTSMReader(bytes.NewReader(b.Bytes())) if err != nil { t.Fatalf("unexpected error created reader: %v", err) } for _, d := range data { readValues, err := r.ReadAll(d.key) if err != nil { t.Fatalf("unexpected error readin: %v", err) } if exp := len(d.values); exp != len(readValues) { t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp) } for i, v := range d.values { if v.Value() != readValues[i].Value() { t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value()) } } } }
func (cmd *Command) dump() error { var errors []error f, err := os.Open(cmd.path) if err != nil { return err } // Get the file size stat, err := f.Stat() if err != nil { return err } b := make([]byte, 8) r, err := tsm1.NewTSMReader(f) if err != nil { return fmt.Errorf("Error opening TSM files: %s", err.Error()) } defer r.Close() minTime, maxTime := r.TimeRange() keyCount := r.KeyCount() blockStats := &blockStats{} println("Summary:") fmt.Printf(" File: %s\n", cmd.path) fmt.Printf(" Time Range: %s - %s\n", time.Unix(0, minTime).UTC().Format(time.RFC3339Nano), time.Unix(0, maxTime).UTC().Format(time.RFC3339Nano), ) fmt.Printf(" Duration: %s ", time.Unix(0, maxTime).Sub(time.Unix(0, minTime))) fmt.Printf(" Series: %d ", keyCount) fmt.Printf(" File Size: %d\n", stat.Size()) println() tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0) if cmd.dumpIndex { println("Index:") tw.Flush() println() fmt.Fprintln(tw, " "+strings.Join([]string{"Pos", "Min Time", "Max Time", "Ofs", "Size", "Key", "Field"}, "\t")) var pos int for i := 0; i < keyCount; i++ { key, _ := r.KeyAt(i) for _, e := range r.Entries(string(key)) { pos++ split := strings.Split(string(key), "#!~#") // We dont' know know if we have fields so use an informative default var measurement, field string = "UNKNOWN", "UNKNOWN" // Possible corruption? Try to read as much as we can and point to the problem. measurement = split[0] field = split[1] if cmd.filterKey != "" && !strings.Contains(string(key), cmd.filterKey) { continue } fmt.Fprintln(tw, " "+strings.Join([]string{ strconv.FormatInt(int64(pos), 10), time.Unix(0, e.MinTime).UTC().Format(time.RFC3339Nano), time.Unix(0, e.MaxTime).UTC().Format(time.RFC3339Nano), strconv.FormatInt(int64(e.Offset), 10), strconv.FormatInt(int64(e.Size), 10), measurement, field, }, "\t")) tw.Flush() } } } tw = tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0) fmt.Fprintln(tw, " "+strings.Join([]string{"Blk", "Chk", "Ofs", "Len", "Type", "Min Time", "Points", "Enc [T/V]", "Len [T/V]"}, "\t")) // Starting at 5 because the magic number is 4 bytes + 1 byte version i := int64(5) var blockCount, pointCount, blockSize int64 indexSize := r.IndexSize() // Start at the beginning and read every block for j := 0; j < keyCount; j++ { key, _ := r.KeyAt(j) for _, e := range r.Entries(string(key)) { f.Seek(int64(e.Offset), 0) f.Read(b[:4]) chksum := binary.BigEndian.Uint32(b[:4]) buf := make([]byte, e.Size-4) f.Read(buf) blockSize += int64(e.Size) if cmd.filterKey != "" && !strings.Contains(string(key), cmd.filterKey) { i += blockSize blockCount++ continue } blockType := buf[0] encoded := buf[1:] var v []tsm1.Value v, err := tsm1.DecodeBlock(buf, v) if err != nil { return err } startTime := time.Unix(0, v[0].UnixNano()) pointCount += int64(len(v)) // Length of the timestamp block tsLen, j := binary.Uvarint(encoded) // Unpack the timestamp bytes ts := encoded[int(j) : int(j)+int(tsLen)] // Unpack the value bytes values := encoded[int(j)+int(tsLen):] tsEncoding := timeEnc[int(ts[0]>>4)] vEncoding := encDescs[int(blockType+1)][values[0]>>4] typeDesc := blockTypes[blockType] blockStats.inc(0, ts[0]>>4) blockStats.inc(int(blockType+1), values[0]>>4) blockStats.size(len(buf)) if cmd.dumpBlocks { fmt.Fprintln(tw, " "+strings.Join([]string{ strconv.FormatInt(blockCount, 10), strconv.FormatUint(uint64(chksum), 10), strconv.FormatInt(i, 10), strconv.FormatInt(int64(len(buf)), 10), typeDesc, startTime.UTC().Format(time.RFC3339Nano), strconv.FormatInt(int64(len(v)), 10), fmt.Sprintf("%s/%s", tsEncoding, vEncoding), fmt.Sprintf("%d/%d", len(ts), len(values)), }, "\t")) } i += blockSize blockCount++ } } if cmd.dumpBlocks { println("Blocks:") tw.Flush() println() } var blockSizeAvg int64 if blockCount > 0 { blockSizeAvg = blockSize / blockCount } fmt.Printf("Statistics\n") fmt.Printf(" Blocks:\n") fmt.Printf(" Total: %d Size: %d Min: %d Max: %d Avg: %d\n", blockCount, blockSize, blockStats.min, blockStats.max, blockSizeAvg) fmt.Printf(" Index:\n") fmt.Printf(" Total: %d Size: %d\n", blockCount, indexSize) fmt.Printf(" Points:\n") fmt.Printf(" Total: %d", pointCount) println() println(" Encoding:") for i, counts := range blockStats.counts { if len(counts) == 0 { continue } fmt.Printf(" %s: ", strings.Title(fieldType[i])) for j, v := range counts { fmt.Printf("\t%s: %d (%d%%) ", encDescs[i][j], v, int(float64(v)/float64(blockCount)*100)) } println() } fmt.Printf(" Compression:\n") fmt.Printf(" Per block: %0.2f bytes/point\n", float64(blockSize)/float64(pointCount)) fmt.Printf(" Total: %0.2f bytes/point\n", float64(stat.Size())/float64(pointCount)) if len(errors) > 0 { println() fmt.Printf("Errors (%d):\n", len(errors)) for _, err := range errors { fmt.Printf(" * %v\n", err) } println() return fmt.Errorf("error count %d", len(errors)) } return nil }
func (cmd *Command) writeTsmFiles(w io.WriteCloser, files []string) error { fmt.Fprintln(w, "# writing tsm data") // we need to make sure we write the same order that the files were written sort.Strings(files) // use a function here to close the files in the defers and not let them accumulate in the loop write := func(f string) error { file, err := os.OpenFile(f, os.O_RDONLY, 0600) if err != nil { return fmt.Errorf("%v", err) } defer file.Close() reader, err := tsm1.NewTSMReader(file) if err != nil { log.Printf("unable to read %s, skipping\n", f) return nil } defer reader.Close() if sgStart, sgEnd := reader.TimeRange(); sgStart > cmd.endTime || sgEnd < cmd.startTime { return nil } for i := 0; i < reader.KeyCount(); i++ { var pairs string key, typ := reader.KeyAt(i) values, _ := reader.ReadAll(string(key)) measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key) for _, value := range values { if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) { continue } switch typ { case tsm1.BlockFloat64: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) case tsm1.BlockInteger: pairs = field + "=" + fmt.Sprintf("%vi", value.Value()) case tsm1.BlockBoolean: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) case tsm1.BlockString: pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value()))) default: pairs = field + "=" + fmt.Sprintf("%v", value.Value()) } fmt.Fprintln(w, string(measurement), pairs, value.UnixNano()) } } return nil } for _, f := range files { if err := write(f); err != nil { return err } } return nil }