Example #1
0
func appendField(b []byte, k string, v interface{}) []byte {
	b = append(b, []byte(escape.String(k))...)
	b = append(b, '=')

	// check popular types first
	switch v := v.(type) {
	case float64:
		b = strconv.AppendFloat(b, v, 'f', -1, 64)
	case int64:
		b = strconv.AppendInt(b, v, 10)
		b = append(b, 'i')
	case string:
		b = append(b, '"')
		b = append(b, []byte(EscapeStringField(v))...)
		b = append(b, '"')
	case bool:
		b = strconv.AppendBool(b, v)
	case int32:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	case int16:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	case int8:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	case int:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	case uint32:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	case uint16:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	case uint8:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	// TODO: 'uint' should be considered just as "dangerous" as a uint64,
	// perhaps the value should be checked and capped at MaxInt64? We could
	// then include uint64 as an accepted value
	case uint:
		b = strconv.AppendInt(b, int64(v), 10)
		b = append(b, 'i')
	case float32:
		b = strconv.AppendFloat(b, float64(v), 'f', -1, 32)
	case []byte:
		b = append(b, v...)
	case nil:
		// skip
	default:
		// Can't determine the type, so convert to string
		b = append(b, '"')
		b = append(b, []byte(EscapeStringField(fmt.Sprintf("%v", v)))...)
		b = append(b, '"')

	}

	return b
}
Example #2
0
// MarshalBinary encodes all the fields to their proper type and returns the binary
// represenation
// NOTE: uint64 is specifically not supported due to potential overflow when we decode
// again later to an int64
func (p Fields) MarshalBinary() []byte {
	b := []byte{}
	keys := make([]string, len(p))
	i := 0
	for k := range p {
		keys[i] = k
		i++
	}
	sort.Strings(keys)

	for _, k := range keys {
		v := p[k]
		b = append(b, []byte(escape.String(k))...)
		b = append(b, '=')
		switch t := v.(type) {
		case int:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case int8:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case int16:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case int32:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case int64:
			b = append(b, []byte(strconv.FormatInt(t, 10))...)
			b = append(b, 'i')
		case uint:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case uint8:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case uint16:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case uint32:
			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
			b = append(b, 'i')
		case float32:
			val := []byte(strconv.FormatFloat(float64(t), 'f', -1, 32))
			b = append(b, val...)
		case float64:
			val := []byte(strconv.FormatFloat(t, 'f', -1, 64))
			b = append(b, val...)
		case bool:
			b = append(b, []byte(strconv.FormatBool(t))...)
		case []byte:
			b = append(b, t...)
		case string:
			b = append(b, '"')
			b = append(b, []byte(escapeStringField(t))...)
			b = append(b, '"')
		case nil:
			// skip
		default:
			// Can't determine the type, so convert to string
			b = append(b, '"')
			b = append(b, []byte(escapeStringField(fmt.Sprintf("%v", v)))...)
			b = append(b, '"')

		}
		b = append(b, ',')
	}
	if len(b) > 0 {
		return b[0 : len(b)-1]
	}
	return b
}
Example #3
0
func (cmd *Command) writeWALFiles(w io.WriteCloser, files []string, key string) error {
	fmt.Fprintln(w, "# writing wal data")

	// we need to make sure we write the same order that the wal received the data
	sort.Strings(files)

	var once sync.Once
	warn := func() {
		msg := fmt.Sprintf(`WARNING: detected deletes in wal file.
		Some series for %q may be brought back by replaying this data.
		To resolve, you can either let the shard snapshot prior to exporting the data
		or manually editing the exported file.
		`, key)
		fmt.Fprintln(cmd.Stderr, msg)
	}

	// use a function here to close the files in the defers and not let them accumulate in the loop
	write := func(f string) error {
		file, err := os.OpenFile(f, os.O_RDONLY, 0600)
		if err != nil {
			return fmt.Errorf("%v", err)
		}
		defer file.Close()

		reader := tsm1.NewWALSegmentReader(file)
		defer reader.Close()
		for reader.Next() {
			entry, err := reader.Read()
			if err != nil {
				n := reader.Count()
				fmt.Fprintf(os.Stderr, "file %s corrupt at position %d", file.Name(), n)
				break
			}

			switch t := entry.(type) {
			case *tsm1.DeleteWALEntry:
				once.Do(warn)
				continue
			case *tsm1.DeleteRangeWALEntry:
				once.Do(warn)
				continue
			case *tsm1.WriteWALEntry:
				var pairs string

				for key, values := range t.Values {
					measurement, field := tsm1.SeriesAndFieldFromCompositeKey([]byte(key))
					// measurements are stored escaped, field names are not
					field = escape.String(field)

					for _, value := range values {
						if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) {
							continue
						}

						switch value.Value().(type) {
						case float64:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						case int64:
							pairs = field + "=" + fmt.Sprintf("%vi", value.Value())
						case bool:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						case string:
							pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value())))
						default:
							pairs = field + "=" + fmt.Sprintf("%v", value.Value())
						}
						fmt.Fprintln(w, string(measurement), pairs, value.UnixNano())
					}
				}
			}
		}
		return nil
	}

	for _, f := range files {
		if err := write(f); err != nil {
			return err
		}
	}

	return nil
}
Example #4
0
func (cmd *Command) writeTsmFiles(w io.WriteCloser, files []string) error {
	fmt.Fprintln(w, "# writing tsm data")

	// we need to make sure we write the same order that the files were written
	sort.Strings(files)

	// use a function here to close the files in the defers and not let them accumulate in the loop
	write := func(f string) error {
		file, err := os.OpenFile(f, os.O_RDONLY, 0600)
		if err != nil {
			return fmt.Errorf("%v", err)
		}
		defer file.Close()
		reader, err := tsm1.NewTSMReader(file)
		if err != nil {
			log.Printf("unable to read %s, skipping\n", f)
			return nil
		}
		defer reader.Close()

		if sgStart, sgEnd := reader.TimeRange(); sgStart > cmd.endTime || sgEnd < cmd.startTime {
			return nil
		}

		for i := 0; i < reader.KeyCount(); i++ {
			var pairs string
			key, typ := reader.KeyAt(i)
			values, _ := reader.ReadAll(string(key))
			measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key)
			// measurements are stored escaped, field names are not
			field = escape.String(field)

			for _, value := range values {
				if (value.UnixNano() < cmd.startTime) || (value.UnixNano() > cmd.endTime) {
					continue
				}

				switch typ {
				case tsm1.BlockFloat64:
					pairs = field + "=" + fmt.Sprintf("%v", value.Value())
				case tsm1.BlockInteger:
					pairs = field + "=" + fmt.Sprintf("%vi", value.Value())
				case tsm1.BlockBoolean:
					pairs = field + "=" + fmt.Sprintf("%v", value.Value())
				case tsm1.BlockString:
					pairs = field + "=" + fmt.Sprintf("%q", models.EscapeStringField(fmt.Sprintf("%s", value.Value())))
				default:
					pairs = field + "=" + fmt.Sprintf("%v", value.Value())
				}

				fmt.Fprintln(w, string(measurement), pairs, value.UnixNano())
			}
		}
		return nil
	}

	for _, f := range files {
		if err := write(f); err != nil {
			return err
		}
	}

	return nil
}