func chunkedWrite(shard *tsdb.Shard, points []models.Point) {
nPts := len(points)
chunkSz := 10000
start := 0
end := chunkSz
for {
if end > nPts {
end = nPts
}
if end-start == 0 {
break
}
shard.WritePoints(points[start:end])
start = end
end += chunkSz
}
}
// LoadMetadataIndex loads the shard metadata into memory.
func (e *Engine) LoadMetadataIndex(sh *tsdb.Shard, index *tsdb.DatabaseIndex) error {
// Save reference to index for iterator creation.
e.index = index
start := time.Now()
if err := e.FileStore.WalkKeys(func(key string, typ byte) error {
fieldType, err := tsmFieldTypeToInfluxQLDataType(typ)
if err != nil {
return err
}
if err := e.addToIndexFromKey(key, fieldType, index); err != nil {
return err
}
return nil
}); err != nil {
return err
}
// load metadata from the Cache
e.Cache.RLock() // shouldn't need the lock, but just to be safe
defer e.Cache.RUnlock()
for key, entry := range e.Cache.Store() {
fieldType, err := entry.values.InfluxQLType()
if err != nil {
e.logger.Printf("error getting the data type of values for key %s: %s", key, err.Error())
continue
}
if err := e.addToIndexFromKey(key, fieldType, index); err != nil {
return err
}
}
// sh may be nil in tests
if sh != nil {
e.logger.Printf("%s database index loaded in %s", sh.Path(), time.Now().Sub(start))
}
return nil
}