func appendField(b []byte, k string, v interface{}) []byte {
b = append(b, []byte(escape.String(k))...)
b = append(b, '=')
// check popular types first
switch v := v.(type) {
case float64:
b = strconv.AppendFloat(b, v, 'f', -1, 64)
case int64:
b = strconv.AppendInt(b, v, 10)
b = append(b, 'i')
case string:
b = append(b, '"')
b = append(b, []byte(EscapeStringField(v))...)
b = append(b, '"')
case bool:
b = strconv.AppendBool(b, v)
case int32:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
case int16:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
case int8:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
case int:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
case uint32:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
case uint16:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
case uint8:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
// TODO: 'uint' should be considered just as "dangerous" as a uint64,
// perhaps the value should be checked and capped at MaxInt64? We could
// then include uint64 as an accepted value
case uint:
b = strconv.AppendInt(b, int64(v), 10)
b = append(b, 'i')
case float32:
b = strconv.AppendFloat(b, float64(v), 'f', -1, 32)
case []byte:
b = append(b, v...)
case nil:
// skip
default:
// Can't determine the type, so convert to string
b = append(b, '"')
b = append(b, []byte(EscapeStringField(fmt.Sprintf("%v", v)))...)
b = append(b, '"')
}
return b
}
// MarshalBinary encodes all the fields to their proper type and returns the binary
// represenation
// NOTE: uint64 is specifically not supported due to potential overflow when we decode
// again later to an int64
func (p Fields) MarshalBinary() []byte {
b := []byte{}
keys := make([]string, len(p))
i := 0
for k := range p {
keys[i] = k
i++
}
sort.Strings(keys)
for _, k := range keys {
v := p[k]
b = append(b, []byte(escape.String(k))...)
b = append(b, '=')
switch t := v.(type) {
case int:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case int8:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case int16:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case int32:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case int64:
b = append(b, []byte(strconv.FormatInt(t, 10))...)
b = append(b, 'i')
case uint:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case uint8:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case uint16:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case uint32:
b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
b = append(b, 'i')
case float32:
val := []byte(strconv.FormatFloat(float64(t), 'f', -1, 32))
b = append(b, val...)
case float64:
val := []byte(strconv.FormatFloat(t, 'f', -1, 64))
b = append(b, val...)
case bool:
b = append(b, []byte(strconv.FormatBool(t))...)
case []byte:
b = append(b, t...)
case string:
b = append(b, '"')
b = append(b, []byte(escapeStringField(t))...)
b = append(b, '"')
case nil:
// skip
default:
// Can't determine the type, so convert to string
b = append(b, '"')
b = append(b, []byte(escapeStringField(fmt.Sprintf("%v", v)))...)
b = append(b, '"')
}
b = append(b, ',')
}
if len(b) > 0 {
return b[0 : len(b)-1]
}
return b
}
func (cmd *Command) writeWALFiles(w io.WriteCloser, files []string, key string) error {
fmt.Fprintln(w, "# writing wal data")
// we need to make sure we write the same order that the wal received the data
sort.Strings(files)
var once sync.Once
warn := func() {
msg := fmt.Sprintf(`WARNING: detected deletes in wal file.
Some series for %q may be brought back by replaying this data.
To resolve, you can either let the shard snapshot prior to exporting the data
or manually editing the exported file.
`, key)
fmt.Fprintln(cmd.Stderr, msg)
}
// use a function here to close the files in the defers and not let them accumulate in the loop
write := func(f string) error {
file, err := os.OpenFile(f, os.O_RDONLY, 0600)
if err != nil {
return fmt.Errorf("%v", err)
}
defer file.Close()
reader := tsm1.NewWALSegmentReader(file)
defer reader.Close()
for reader.Next() {
entry, err := reader.Read()
if err != nil {
n := reader.Count()
fmt.Fprintf(os.Stderr, "file %s corrupt at position %d", file.Name(), n)
break
}
switch t := entry.(type) {
case *tsm1.DeleteWALEntry:
once.Do(warn)
continue
case *tsm1.DeleteRangeWALEntry:
once.Do(warn)
continue
case *tsm1.WriteWALEntry:
var pairs string
for key, values := range t.Values {
measurement, field := tsm1.SeriesAndFieldFromCompositeKey([]byte(key))
// measurements are stored escaped, field names are not
field = escape.String(field)
for _, value := range values {
if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) {
continue
}
switch value.Value().(type) {
case float64:
pairs = field + "=" + fmt.Sprintf("%v", value.Value())
case int64:
pairs = field + "=" + fmt.Sprintf("%vi", value.Value())
case bool:
pairs = field + "=" + fmt.Sprintf("%v", value.Value())
case string:
pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value())))
default:
pairs = field + "=" + fmt.Sprintf("%v", value.Value())
}
fmt.Fprintln(w, string(measurement), pairs, value.UnixNano())
}
}
}
}
return nil
}
for _, f := range files {
if err := write(f); err != nil {
return err
}
}
return nil
}
func (cmd *Command) writeTsmFiles(w io.WriteCloser, files []string) error {
fmt.Fprintln(w, "# writing tsm data")
// we need to make sure we write the same order that the files were written
sort.Strings(files)
// use a function here to close the files in the defers and not let them accumulate in the loop
write := func(f string) error {
file, err := os.OpenFile(f, os.O_RDONLY, 0600)
if err != nil {
return fmt.Errorf("%v", err)
}
defer file.Close()
reader, err := tsm1.NewTSMReader(file)
if err != nil {
log.Printf("unable to read %s, skipping\n", f)
return nil
}
defer reader.Close()
if sgStart, sgEnd := reader.TimeRange(); sgStart > cmd.endTime || sgEnd < cmd.startTime {
return nil
}
for i := 0; i < reader.KeyCount(); i++ {
var pairs string
key, typ := reader.KeyAt(i)
values, _ := reader.ReadAll(string(key))
measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key)
// measurements are stored escaped, field names are not
field = escape.String(field)
for _, value := range values {
if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) {
continue
}
switch typ {
case tsm1.BlockFloat64:
pairs = field + "=" + fmt.Sprintf("%v", value.Value())
case tsm1.BlockInteger:
pairs = field + "=" + fmt.Sprintf("%vi", value.Value())
case tsm1.BlockBoolean:
pairs = field + "=" + fmt.Sprintf("%v", value.Value())
case tsm1.BlockString:
pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value())))
default:
pairs = field + "=" + fmt.Sprintf("%v", value.Value())
}
fmt.Fprintln(w, string(measurement), pairs, value.UnixNano())
}
}
return nil
}
for _, f := range files {
if err := write(f); err != nil {
return err
}
}
return nil
}
