// addToIndexFromKey will pull the measurement name, series key, and field name from a composite key and add it to the
// database index and measurement fields
func (e *Engine) addToIndexFromKey(key string, fieldType influxql.DataType, index *tsdb.DatabaseIndex, measurementFields map[string]*tsdb.MeasurementFields) error {
seriesKey, field := seriesAndFieldFromCompositeKey(key)
measurement := tsdb.MeasurementFromSeriesKey(seriesKey)
m := index.CreateMeasurementIndexIfNotExists(measurement)
m.SetFieldName(field)
mf := measurementFields[measurement]
if mf == nil {
mf = &tsdb.MeasurementFields{
Fields: map[string]*tsdb.Field{},
}
measurementFields[measurement] = mf
}
if err := mf.CreateFieldIfNotExists(field, fieldType, false); err != nil {
return err
}
_, tags, err := models.ParseKey(seriesKey)
if err == nil {
return err
}
s := tsdb.NewSeries(seriesKey, tags)
s.InitializeShards()
index.CreateSeriesIndexIfNotExists(measurement, s)
return nil
}
// addToIndexFromKey will pull the measurement name, series key, and field name from a composite key and add it to the
// database index and measurement fields
func (e *Engine) addToIndexFromKey(shardID uint64, key []byte, fieldType influxql.DataType, index *tsdb.DatabaseIndex) error {
seriesKey, field := SeriesAndFieldFromCompositeKey(key)
measurement := tsdb.MeasurementFromSeriesKey(string(seriesKey))
m := index.CreateMeasurementIndexIfNotExists(measurement)
m.SetFieldName(field)
mf := e.measurementFields[measurement]
if mf == nil {
mf = tsdb.NewMeasurementFields()
e.measurementFields[measurement] = mf
}
if err := mf.CreateFieldIfNotExists(field, fieldType, false); err != nil {
return err
}
// Have we already indexed this series?
ss := index.SeriesBytes(seriesKey)
if ss != nil {
// Add this shard to the existing series
ss.AssignShard(shardID)
return nil
}
// ignore error because ParseKey returns "missing fields" and we don't have
// fields (in line protocol format) in the series key
_, tags, _ := models.ParseKey(seriesKey)
s := tsdb.NewSeries(string(seriesKey), tags)
index.CreateSeriesIndexIfNotExists(measurement, s)
s.AssignShard(shardID)
return nil
}
// LoadMetadataIndex loads the new series and fields files into memory and flushes them to the BoltDB index. This function
// should be called before making a call to Open()
func (l *Log) LoadMetadataIndex(index *tsdb.DatabaseIndex, measurementFields map[string]*tsdb.MeasurementFields) error {
metaFiles, err := l.metadataFiles()
if err != nil {
return err
}
measurementFieldsToSave := make(map[string]*tsdb.MeasurementFields)
var seriesToCreate []*tsdb.SeriesCreate
// read all the metafiles off disk
for _, fn := range metaFiles {
a, err := l.readMetadataFile(fn)
if err != nil {
return err
}
// loop through the seriesAndFields and add them to the index and the collection to be written to the index
for _, sf := range a {
for k, mf := range sf.Fields {
measurementFieldsToSave[k] = mf
m := index.CreateMeasurementIndexIfNotExists(string(k))
for name := range mf.Fields {
m.SetFieldName(name)
}
mf.Codec = tsdb.NewFieldCodec(mf.Fields)
measurementFields[m.Name] = mf
}
for _, sc := range sf.Series {
seriesToCreate = append(seriesToCreate, sc)
sc.Series.InitializeShards()
index.CreateSeriesIndexIfNotExists(tsdb.MeasurementFromSeriesKey(string(sc.Series.Key)), sc.Series)
}
}
}
if err := l.Index.WriteIndex(nil, measurementFieldsToSave, seriesToCreate); err != nil {
return err
}
// now remove all the old metafiles
for _, fn := range metaFiles {
if err := os.Remove(fn); err != nil {
return err
}
}
return nil
}