func TestTSMReader_MMAP_Tombstone(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
defer f.Close()
w, err := tsm1.NewTSMWriter(f)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.Write("mem", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error writing index: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
f, err = os.Open(f.Name())
if err != nil {
t.Fatalf("unexpected error open file: %v", err)
}
r, err := tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
if err := r.Delete([]string{"mem"}); err != nil {
t.Fatalf("unexpected error deleting: %v", err)
}
r, err = tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
defer r.Close()
if got, exp := len(r.Keys()), 1; got != exp {
t.Fatalf("key length mismatch: got %v, exp %v", got, exp)
}
}
func MustOpenTSMReader(name string) *tsm1.TSMReader {
f, err := os.Open(name)
if err != nil {
panic(fmt.Sprintf("open file: %v", err))
}
r, err := tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
panic(fmt.Sprintf("new reader: %v", err))
}
return r
}
// Ensure that we return an error if we try to open a non-tsm file
func TestTSMReader_VerifiesFileType(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
defer f.Close()
// write some garbage
f.Write([]byte{0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77})
_, err := tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err == nil {
t.Fatal("expected error trying to open non-tsm file")
}
}
func TestIndirectIndex_UnmarshalBinary_BlockCountOverflow(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
defer f.Close()
w, err := tsm1.NewTSMWriter(f)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
for i := 0; i < 3280; i++ {
w.Write("cpu", []tsm1.Value{tsm1.NewValue(time.Unix(int64(i), 0), float64(i))})
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
f, err = os.Open(f.Name())
if err != nil {
t.Fatalf("unexpected error open file: %v", err)
}
r, err := tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
defer r.Close()
}
func cmdDumpTsm1dev(opts *tsdmDumpOpts) {
var errors []error
f, err := os.Open(opts.path)
if err != nil {
println(err.Error())
os.Exit(1)
}
// Get the file size
stat, err := f.Stat()
if err != nil {
println(err.Error())
os.Exit(1)
}
b := make([]byte, 8)
r, err := tsm1.NewTSMReaderWithOptions(tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
println("Error opening TSM files: ", err.Error())
}
defer r.Close()
minTime, maxTime := r.TimeRange()
keys := r.Keys()
blockStats := &blockStats{}
println("Summary:")
fmt.Printf(" File: %s\n", opts.path)
fmt.Printf(" Time Range: %s - %s\n",
minTime.UTC().Format(time.RFC3339Nano),
maxTime.UTC().Format(time.RFC3339Nano),
)
fmt.Printf(" Duration: %s ", maxTime.Sub(minTime))
fmt.Printf(" Series: %d ", len(keys))
fmt.Printf(" File Size: %d\n", stat.Size())
println()
tw := tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
fmt.Fprintln(tw, " "+strings.Join([]string{"Pos", "Min Time", "Max Time", "Ofs", "Size", "Key", "Field"}, "\t"))
var pos int
for _, key := range keys {
for _, e := range r.Entries(key) {
pos++
split := strings.Split(key, "#!~#")
// We dont' know know if we have fields so use an informative default
var measurement, field string = "UNKNOWN", "UNKNOWN"
// Possible corruption? Try to read as much as we can and point to the problem.
measurement = split[0]
field = split[1]
if opts.filterKey != "" && !strings.Contains(key, opts.filterKey) {
continue
}
fmt.Fprintln(tw, " "+strings.Join([]string{
strconv.FormatInt(int64(pos), 10),
e.MinTime.UTC().Format(time.RFC3339Nano),
e.MaxTime.UTC().Format(time.RFC3339Nano),
strconv.FormatInt(int64(e.Offset), 10),
strconv.FormatInt(int64(e.Size), 10),
measurement,
field,
}, "\t"))
}
}
if opts.dumpIndex {
println("Index:")
tw.Flush()
println()
}
tw = tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
fmt.Fprintln(tw, " "+strings.Join([]string{"Blk", "Chk", "Ofs", "Len", "Type", "Min Time", "Points", "Enc [T/V]", "Len [T/V]"}, "\t"))
// Starting at 5 because the magic number is 4 bytes + 1 byte version
i := int64(5)
var blockCount, pointCount, blockSize int64
indexSize := r.IndexSize()
// Start at the beginning and read every block
for _, key := range keys {
for _, e := range r.Entries(key) {
f.Seek(int64(e.Offset), 0)
f.Read(b[:4])
chksum := btou32(b[:4])
buf := make([]byte, e.Size-4)
f.Read(buf)
blockSize += int64(e.Size)
blockType := buf[0]
encoded := buf[1:]
var v []tsm1.Value
v, err := tsm1.DecodeBlock(buf, v)
if err != nil {
fmt.Printf("error: %v\n", err.Error())
os.Exit(1)
}
startTime := v[0].Time()
pointCount += int64(len(v))
// Length of the timestamp block
tsLen, j := binary.Uvarint(encoded)
// Unpack the timestamp bytes
ts := encoded[int(j) : int(j)+int(tsLen)]
// Unpack the value bytes
values := encoded[int(j)+int(tsLen):]
tsEncoding := timeEnc[int(ts[0]>>4)]
vEncoding := encDescs[int(blockType+1)][values[0]>>4]
typeDesc := blockTypes[blockType]
blockStats.inc(0, ts[0]>>4)
blockStats.inc(int(blockType+1), values[0]>>4)
blockStats.size(len(buf))
if opts.filterKey != "" && !strings.Contains(key, opts.filterKey) {
i += blockSize
blockCount++
continue
}
fmt.Fprintln(tw, " "+strings.Join([]string{
strconv.FormatInt(blockCount, 10),
strconv.FormatUint(uint64(chksum), 10),
strconv.FormatInt(i, 10),
strconv.FormatInt(int64(len(buf)), 10),
typeDesc,
startTime.UTC().Format(time.RFC3339Nano),
strconv.FormatInt(int64(len(v)), 10),
fmt.Sprintf("%s/%s", tsEncoding, vEncoding),
fmt.Sprintf("%d/%d", len(ts), len(values)),
}, "\t"))
i += blockSize
blockCount++
}
}
if opts.dumpBlocks {
println("Blocks:")
tw.Flush()
println()
}
var blockSizeAvg int64
if blockCount > 0 {
blockSizeAvg = blockSize / blockCount
}
fmt.Printf("Statistics\n")
fmt.Printf(" Blocks:\n")
fmt.Printf(" Total: %d Size: %d Min: %d Max: %d Avg: %d\n",
blockCount, blockSize, blockStats.min, blockStats.max, blockSizeAvg)
fmt.Printf(" Index:\n")
fmt.Printf(" Total: %d Size: %d\n", blockCount, indexSize)
fmt.Printf(" Points:\n")
fmt.Printf(" Total: %d", pointCount)
println()
println(" Encoding:")
for i, counts := range blockStats.counts {
if len(counts) == 0 {
continue
}
fmt.Printf(" %s: ", strings.Title(fieldType[i]))
for j, v := range counts {
fmt.Printf("\t%s: %d (%d%%) ", encDescs[i][j], v, int(float64(v)/float64(blockCount)*100))
}
println()
}
fmt.Printf(" Compression:\n")
fmt.Printf(" Per block: %0.2f bytes/point\n", float64(blockSize)/float64(pointCount))
fmt.Printf(" Total: %0.2f bytes/point\n", float64(stat.Size())/float64(pointCount))
if len(errors) > 0 {
println()
fmt.Printf("Errors (%d):\n", len(errors))
for _, err := range errors {
fmt.Printf(" * %v\n", err)
}
println()
}
}
func TestTSMReader_File_Read(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
defer f.Close()
w, err := tsm1.NewTSMWriter(f)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"float", []tsm1.Value{
tsm1.NewValue(time.Unix(1, 0), 1.0)},
},
{"int", []tsm1.Value{
tsm1.NewValue(time.Unix(1, 0), int64(1))},
},
{"bool", []tsm1.Value{
tsm1.NewValue(time.Unix(1, 0), true)},
},
{"string", []tsm1.Value{
tsm1.NewValue(time.Unix(1, 0), "foo")},
},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error writing index: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
f, err = os.Open(f.Name())
if err != nil {
t.Fatalf("unexpected error open file: %v", err)
}
r, err := tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
Reader: f,
})
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
defer r.Close()
var count int
for _, d := range data {
readValues, err := r.Read(d.key, d.values[0].Time())
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp, got := len(d.values), len(readValues); exp != got {
t.Fatalf("read values length mismatch: exp %v, got %v", exp, len(readValues))
}
for i, v := range d.values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): exp %v, got %d", i, v.Value(), readValues[i].Value())
}
}
count++
}
if exp, got := count, len(data); exp != got {
t.Fatalf("read values count mismatch: exp %v, got %v", exp, got)
}
}
func TestTSMReader_MMAP_Stats(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
defer f.Close()
w, err := tsm1.NewTSMWriter(f)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values1 := []tsm1.Value{tsm1.NewValue(time.Unix(0, 0), 1.0)}
if err := w.Write("cpu", values1); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
values2 := []tsm1.Value{tsm1.NewValue(time.Unix(1, 0), 1.0)}
if err := w.Write("mem", values2); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error writing index: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
f, err = os.Open(f.Name())
if err != nil {
t.Fatalf("unexpected error open file: %v", err)
}
r, err := tsm1.NewTSMReaderWithOptions(
tsm1.TSMReaderOptions{
MMAPFile: f,
})
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
defer r.Close()
stats := r.Stats()
if got, exp := stats.MinKey, "cpu"; got != exp {
t.Fatalf("min key mismatch: got %v, exp %v", got, exp)
}
if got, exp := stats.MaxKey, "mem"; got != exp {
t.Fatalf("max key mismatch: got %v, exp %v", got, exp)
}
if got, exp := stats.MinTime, values1[0].Time(); got != exp {
t.Fatalf("min time mismatch: got %v, exp %v", got, exp)
}
if got, exp := stats.MaxTime, values2[0].Time(); got != exp {
t.Fatalf("max time mismatch: got %v, exp %v", got, exp)
}
if got, exp := len(r.Keys()), 2; got != exp {
t.Fatalf("key length mismatch: got %v, exp %v", got, exp)
}
}
func cmdVerify(path string) {
start := time.Now()
dataPath := filepath.Join(path, "data")
brokenBlocks := 0
totalBlocks := 0
// No need to do this in a loop
ext := fmt.Sprintf(".%s", tsm1.TSMFileExtension)
// Get all TSM files by walking through the data dir
files := []string{}
err := filepath.Walk(dataPath, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
if filepath.Ext(path) == ext {
files = append(files, path)
}
return nil
})
if err != nil {
panic(err)
}
tw := tabwriter.NewWriter(os.Stdout, 16, 8, 0, '\t', 0)
// Verify the checksums of every block in every file
for _, f := range files {
file, err := os.OpenFile(f, os.O_RDONLY, 0600)
if err != nil {
fmt.Printf("%v", err)
os.Exit(1)
}
reader, err := tsm1.NewTSMReaderWithOptions(tsm1.TSMReaderOptions{
MMAPFile: file,
})
if err != nil {
fmt.Printf("%v", err)
os.Exit(1)
}
blockItr := reader.BlockIterator()
brokenFileBlocks := 0
count := 0
for blockItr.Next() {
totalBlocks++
key, _, _, checksum, buf, err := blockItr.Read()
if err != nil {
brokenBlocks++
fmt.Fprintf(tw, "%s: could not get checksum for key %v block %d due to error: %q\n", f, key, count, err)
} else if expected := crc32.ChecksumIEEE(buf); checksum != expected {
brokenBlocks++
fmt.Fprintf(tw, "%s: got %d but expected %d for key %v, block %d\n", f, checksum, expected, key, count)
}
count++
}
if brokenFileBlocks == 0 {
fmt.Fprintf(tw, "%s: healthy\n", f)
}
}
fmt.Fprintf(tw, "Broken Blocks: %d / %d, in %vs\n", brokenBlocks, totalBlocks, time.Since(start).Seconds())
tw.Flush()
}