func newFileDir(dir string, values ...keyValues) ([]string, error) {
var files []string
id := 1
for _, v := range values {
f := MustTempFile(dir)
w, err := tsm1.NewTSMWriter(f)
if err != nil {
return nil, err
}
if err := w.Write(v.key, v.values); err != nil {
return nil, err
}
if err := w.WriteIndex(); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
newName := filepath.Join(filepath.Dir(f.Name()), tsmFileName(id))
if err := os.Rename(f.Name(), newName); err != nil {
return nil, err
}
id++
files = append(files, newName)
}
return files, nil
}
func newFiles(values ...keyValues) ([]tsm1.TSMFile, error) {
var files []tsm1.TSMFile
for _, v := range values {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
return nil, err
}
if err := w.Write(v.key, v.values); err != nil {
return nil, err
}
if err := w.WriteIndex(); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
return nil, err
}
files = append(files, r)
}
return files, nil
}
// Tests that writing keys in reverse is able to read them back.
func TestTSMWriter_Write_SameKey(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"cpu", []tsm1.Value{
tsm1.NewValue(0, 1.0),
tsm1.NewValue(1, 2.0)},
},
{"cpu", []tsm1.Value{
tsm1.NewValue(2, 3.0),
tsm1.NewValue(3, 4.0)},
},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
values := append(data[0].values, data[1].values...)
readValues, err := r.ReadAll("cpu")
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp := len(values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
func TestTSMWriter_Write_Empty(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error created writer: %v", err)
}
if err := w.WriteIndex(); err != tsm1.ErrNoValues {
t.Fatalf("unexpected error closing: %v", err)
}
if got, exp := len(b.Bytes()), 0; got < exp {
t.Fatalf("file size mismatch: got %v, exp %v", got, exp)
}
}
func TestTSMWriter_Write_Single(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
values := []tsm1.Value{tsm1.NewValue(0, 1.0)}
if err := w.Write("cpu", values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error writing index: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if got, exp := len(b.Bytes()), 5; got < exp {
t.Fatalf("file size mismatch: got %v, exp %v", got, exp)
}
if got := binary.BigEndian.Uint32(b.Bytes()[0:4]); got != tsm1.MagicNumber {
t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
readValues, err := r.ReadAll("cpu")
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if len(readValues) != len(values) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), len(values))
}
for i, v := range values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
// nextTSMWriter returns the next TSMWriter for the Converter.
func (c *Converter) nextTSMWriter() (tsm1.TSMWriter, error) {
c.sequence++
fileName := filepath.Join(c.path, fmt.Sprintf("%09d-%09d.%s", 1, c.sequence, tsm1.TSMFileExtension))
fd, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
return nil, err
}
// Create the writer for the new TSM file.
w, err := tsm1.NewTSMWriter(fd)
if err != nil {
return nil, err
}
c.stats.IncrTSMFileCount()
return w, nil
}
func TestTSMWriter_Write_MaxKey(t *testing.T) {
dir := MustTempDir()
defer os.RemoveAll(dir)
f := MustTempFile(dir)
defer f.Close()
w, err := tsm1.NewTSMWriter(f)
if err != nil {
t.Fatalf("unexpected error created writer: %v", err)
}
var key string
for i := 0; i < 100000; i++ {
key += "a"
}
if err := w.Write(key, []tsm1.Value{tsm1.NewValue(0, 1.0)}); err != tsm1.ErrMaxKeyLengthExceeded {
t.Fatalf("expected max key length error writing key: %v", err)
}
}
func MustWriteTSM(dir string, gen int, values map[string][]tsm1.Value) string {
f := MustTempFile(dir)
oldName := f.Name()
// Windows can't rename a file while it's open. Close first, rename and
// then re-open
if err := f.Close(); err != nil {
panic(fmt.Sprintf("close temp file: %v", err))
}
newName := filepath.Join(filepath.Dir(oldName), tsmFileName(gen))
if err := os.Rename(oldName, newName); err != nil {
panic(fmt.Sprintf("create tsm file: %v", err))
}
var err error
f, err = os.OpenFile(newName, os.O_RDWR, 0666)
if err != nil {
panic(fmt.Sprintf("open tsm files: %v", err))
}
w, err := tsm1.NewTSMWriter(f)
if err != nil {
panic(fmt.Sprintf("create TSM writer: %v", err))
}
for k, v := range values {
if err := w.Write(k, v); err != nil {
panic(fmt.Sprintf("write TSM value: %v", err))
}
}
if err := w.WriteIndex(); err != nil {
panic(fmt.Sprintf("write TSM index: %v", err))
}
if err := w.Close(); err != nil {
panic(fmt.Sprintf("write TSM close: %v", err))
}
return newName
}
func TestTSMWriter_WriteBlock_Multiple(t *testing.T) {
// Write a new TSM file
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"cpu", []tsm1.Value{tsm1.NewValue(0, 1.0)}},
{"mem", []tsm1.Value{tsm1.NewValue(1, 2.0)}},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if got, exp := len(b.Bytes()), 5; got < exp {
t.Fatalf("file size mismatch: got %v, exp %v", got, exp)
}
if got := binary.BigEndian.Uint32(b.Bytes()[0:4]); got != tsm1.MagicNumber {
t.Fatalf("magic number mismatch: got %v, exp %v", got, tsm1.MagicNumber)
}
// Create reader for that file
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
// Using the reader, write a new file using WriteBlocks
b.Reset()
w, err = tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
iter := r.BlockIterator()
for iter.Next() {
key, minTime, maxTime, b, err := iter.Read()
if err != nil {
t.Fatalf("unexpected error reading block: %v", err)
}
if err := w.WriteBlock(key, minTime, maxTime, b); err != nil {
t.Fatalf("unexpected error writing block: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
// Now create a reader to verify the written blocks matches the originally
// written file using Write
r, err = tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
for _, d := range data {
readValues, err := r.ReadAll(d.key)
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp := len(d.values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range d.values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
}
// Tests that calling Read returns all the values for block matching the key
// and timestamp
func TestTSMWriter_Read_Multiple(t *testing.T) {
var b bytes.Buffer
w, err := tsm1.NewTSMWriter(&b)
if err != nil {
t.Fatalf("unexpected error creating writer: %v", err)
}
var data = []struct {
key string
values []tsm1.Value
}{
{"cpu", []tsm1.Value{
tsm1.NewValue(0, 1.0),
tsm1.NewValue(1, 2.0)},
},
{"cpu", []tsm1.Value{
tsm1.NewValue(2, 3.0),
tsm1.NewValue(3, 4.0)},
},
}
for _, d := range data {
if err := w.Write(d.key, d.values); err != nil {
t.Fatalf("unexpected error writing: %v", err)
}
}
if err := w.WriteIndex(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error closing: %v", err)
}
r, err := tsm1.NewTSMReader(bytes.NewReader(b.Bytes()))
if err != nil {
t.Fatalf("unexpected error created reader: %v", err)
}
for _, values := range data {
// Try the first timestamp
readValues, err := r.Read("cpu", values.values[0].UnixNano())
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp := len(values.values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range values.values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
// Try the last timestamp too
readValues, err = r.Read("cpu", values.values[1].UnixNano())
if err != nil {
t.Fatalf("unexpected error readin: %v", err)
}
if exp := len(values.values); exp != len(readValues) {
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
}
for i, v := range values.values {
if v.Value() != readValues[i].Value() {
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
}
}
}
}