func CreateDataStore(cnf *conf.Config, writtenSpans chan *common.Span) (*dataStore, error) {
// Get the configuration.
clearStored := cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR)
dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
dirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)
var err error
lg := common.NewLogger("datastore", cnf)
store := &dataStore{lg: lg, shards: []*shard{}, WrittenSpans: writtenSpans}
// If we return an error, close the store.
defer func() {
if err != nil {
store.Close()
store = nil
}
}()
store.readOpts = levigo.NewReadOptions()
store.readOpts.SetFillCache(true)
store.writeOpts = levigo.NewWriteOptions()
store.writeOpts.SetSync(false)
// Open all shards
for idx := range dirs {
path := dirs[idx] + conf.PATH_SEP + "db"
var shd *shard
shd, err = CreateShard(store, cnf, path, clearStored)
if err != nil {
lg.Errorf("Error creating shard %s: %s\n", path, err.Error())
return nil, err
}
store.shards = append(store.shards, shd)
}
for idx := range store.shards {
shd := store.shards[idx]
shd.exited = make(chan bool, 1)
go shd.processIncoming()
}
return store, nil
}
// Create a new datastore loader.
// Initializes the loader, but does not load any leveldb instances.
func NewDataStoreLoader(cnf *conf.Config) *DataStoreLoader {
dld := &DataStoreLoader{
lg: common.NewLogger("datastore", cnf),
ClearStored: cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR),
}
dld.readOpts = levigo.NewReadOptions()
dld.readOpts.SetFillCache(true)
dld.readOpts.SetVerifyChecksums(false)
dld.writeOpts = levigo.NewWriteOptions()
dld.writeOpts.SetSync(false)
dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
rdirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)
// Filter out empty entries
dirs := make([]string, 0, len(rdirs))
for i := range rdirs {
if strings.TrimSpace(rdirs[i]) != "" {
dirs = append(dirs, rdirs[i])
}
}
dld.shards = make([]*ShardLoader, len(dirs))
for i := range dirs {
dld.shards[i] = &ShardLoader{
dld: dld,
path: dirs[i] + conf.PATH_SEP + "db",
}
}
dld.openOpts = levigo.NewOptions()
cacheSize := cnf.GetInt(conf.HTRACE_LEVELDB_CACHE_SIZE)
dld.openOpts.SetCache(levigo.NewLRUCache(cacheSize))
dld.openOpts.SetParanoidChecks(false)
writeBufferSize := cnf.GetInt(conf.HTRACE_LEVELDB_WRITE_BUFFER_SIZE)
if writeBufferSize > 0 {
dld.openOpts.SetWriteBufferSize(writeBufferSize)
}
maxFdPerShard := dld.calculateMaxOpenFilesPerShard()
if maxFdPerShard > 0 {
dld.openOpts.SetMaxOpenFiles(maxFdPerShard)
}
return dld
}