// EncodeFields converts a map of values with string keys to a byte slice of field
// IDs and values.
//
// If a field exists in the codec, but its type is different, an error is returned. If
// a field is not present in the codec, the system panics.
func (f *FieldCodec) EncodeFields(values map[string]interface{}) ([]byte, error) {
// Allocate byte slice
b := make([]byte, 0, 10)
for k, v := range values {
field := f.fieldsByName[k]
if field == nil {
panic(fmt.Sprintf("field does not exist for %s", k))
} else if influxql.InspectDataType(v) != field.Type {
return nil, fmt.Errorf("field \"%s\" is type %T, mapped as type %s", k, k, field.Type)
}
var buf []byte
switch field.Type {
case influxql.Number:
var value float64
// Convert integers to floats.
if intval, ok := v.(int); ok {
value = float64(intval)
} else {
value = v.(float64)
}
buf = make([]byte, 9)
binary.BigEndian.PutUint64(buf[1:9], math.Float64bits(value))
case influxql.Boolean:
value := v.(bool)
// Only 1 byte need for a boolean.
buf = make([]byte, 2)
if value {
buf[1] = byte(1)
}
case influxql.String:
value := v.(string)
if len(value) > maxStringLength {
value = value[:maxStringLength]
}
// Make a buffer for field ID (1 bytes), the string length (2 bytes), and the string.
buf = make([]byte, len(value)+3)
// Set the string length, then copy the string itself.
binary.BigEndian.PutUint16(buf[1:3], uint16(len(value)))
for i, c := range []byte(value) {
buf[i+3] = byte(c)
}
default:
panic(fmt.Sprintf("unsupported value type: %T", v))
}
// Always set the field ID as the leading byte.
buf[0] = field.ID
// Append temp buffer to the end.
b = append(b, buf...)
}
return b, nil
}
// Ensure a value's data type can be retrieved.
func TestInspectDataType(t *testing.T) {
for i, tt := range []struct {
v interface{}
typ influxql.DataType
}{
{float64(100), influxql.Float},
} {
if typ := influxql.InspectDataType(tt.v); tt.typ != typ {
t.Errorf("%d. %v (%s): unexpected type: %s", i, tt.v, tt.typ, typ)
continue
}
}
}
// validateSeriesAndFields checks which series and fields are new and whose metadata should be saved and indexed
func (s *Shard) validateSeriesAndFields(points []Point) ([]*SeriesCreate, []*FieldCreate, []string, error) {
var seriesToCreate []*SeriesCreate
var fieldsToCreate []*FieldCreate
var seriesToAddShardTo []string
// get the mutex for the in memory index, which is shared across shards
s.index.mu.RLock()
defer s.index.mu.RUnlock()
// get the shard mutex for locally defined fields
s.mu.RLock()
defer s.mu.RUnlock()
for _, p := range points {
// see if the series should be added to the index
if ss := s.index.series[string(p.Key())]; ss == nil {
series := NewSeries(string(p.Key()), p.Tags())
seriesToCreate = append(seriesToCreate, &SeriesCreate{p.Name(), series})
seriesToAddShardTo = append(seriesToAddShardTo, series.Key)
} else if !ss.shardIDs[s.id] {
// this is the first time this series is being written into this shard, persist it
seriesToCreate = append(seriesToCreate, &SeriesCreate{p.Name(), ss})
seriesToAddShardTo = append(seriesToAddShardTo, ss.Key)
}
// see if the field definitions need to be saved to the shard
mf := s.measurementFields[p.Name()]
if mf == nil {
for name, value := range p.Fields() {
fieldsToCreate = append(fieldsToCreate, &FieldCreate{p.Name(), &Field{Name: name, Type: influxql.InspectDataType(value)}})
}
continue // skip validation since all fields are new
}
// validate field types and encode data
for name, value := range p.Fields() {
if f := mf.Fields[name]; f != nil {
// Field present in shard metadata, make sure there is no type conflict.
if f.Type != influxql.InspectDataType(value) {
return nil, nil, nil, fmt.Errorf("field type conflict: input field \"%s\" on measurement \"%s\" is type %T, already exists as type %s", name, p.Name(), value, f.Type)
}
continue // Field is present, and it's of the same type. Nothing more to do.
}
fieldsToCreate = append(fieldsToCreate, &FieldCreate{p.Name(), &Field{Name: name, Type: influxql.InspectDataType(value)}})
}
}
return seriesToCreate, fieldsToCreate, seriesToAddShardTo, nil
}
// MeasurementFields returns a set of fields used across all points.
func (a PointsSlice) MeasurementFields() map[string]*tsdb.MeasurementFields {
mfs := map[string]*tsdb.MeasurementFields{}
for _, points := range a {
for _, p := range points {
pp := p.Encode()
// Create measurement field, if not exists.
mf := mfs[string(pp.Key())]
if mf == nil {
mf = &tsdb.MeasurementFields{Fields: make(map[string]*tsdb.Field)}
mfs[string(pp.Key())] = mf
}
// Add all fields on the point.
for name, value := range p.Fields {
mf.CreateFieldIfNotExists(name, influxql.InspectDataType(value), false)
}
}
}
return mfs
}