Example #1
0
func CreateDataStore(cnf *conf.Config, writtenSpans chan *common.Span) (*dataStore, error) {
	// Get the configuration.
	clearStored := cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR)
	dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
	dirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)

	var err error
	lg := common.NewLogger("datastore", cnf)
	store := &dataStore{lg: lg, shards: []*shard{}, WrittenSpans: writtenSpans}

	// If we return an error, close the store.
	defer func() {
		if err != nil {
			store.Close()
			store = nil
		}
	}()

	store.readOpts = levigo.NewReadOptions()
	store.readOpts.SetFillCache(true)
	store.writeOpts = levigo.NewWriteOptions()
	store.writeOpts.SetSync(false)

	// Open all shards
	for idx := range dirs {
		path := dirs[idx] + conf.PATH_SEP + "db"
		var shd *shard
		shd, err = CreateShard(store, cnf, path, clearStored)
		if err != nil {
			lg.Errorf("Error creating shard %s: %s\n", path, err.Error())
			return nil, err
		}
		store.shards = append(store.shards, shd)
	}
	for idx := range store.shards {
		shd := store.shards[idx]
		shd.exited = make(chan bool, 1)
		go shd.processIncoming()
	}
	return store, nil
}
Example #2
0
// Create a new datastore loader.
// Initializes the loader, but does not load any leveldb instances.
func NewDataStoreLoader(cnf *conf.Config) *DataStoreLoader {
	dld := &DataStoreLoader{
		lg:          common.NewLogger("datastore", cnf),
		ClearStored: cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR),
	}
	dld.readOpts = levigo.NewReadOptions()
	dld.readOpts.SetFillCache(true)
	dld.readOpts.SetVerifyChecksums(false)
	dld.writeOpts = levigo.NewWriteOptions()
	dld.writeOpts.SetSync(false)
	dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
	rdirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)
	// Filter out empty entries
	dirs := make([]string, 0, len(rdirs))
	for i := range rdirs {
		if strings.TrimSpace(rdirs[i]) != "" {
			dirs = append(dirs, rdirs[i])
		}
	}
	dld.shards = make([]*ShardLoader, len(dirs))
	for i := range dirs {
		dld.shards[i] = &ShardLoader{
			dld:  dld,
			path: dirs[i] + conf.PATH_SEP + "db",
		}
	}
	dld.openOpts = levigo.NewOptions()
	cacheSize := cnf.GetInt(conf.HTRACE_LEVELDB_CACHE_SIZE)
	dld.openOpts.SetCache(levigo.NewLRUCache(cacheSize))
	dld.openOpts.SetParanoidChecks(false)
	writeBufferSize := cnf.GetInt(conf.HTRACE_LEVELDB_WRITE_BUFFER_SIZE)
	if writeBufferSize > 0 {
		dld.openOpts.SetWriteBufferSize(writeBufferSize)
	}
	maxFdPerShard := dld.calculateMaxOpenFilesPerShard()
	if maxFdPerShard > 0 {
		dld.openOpts.SetMaxOpenFiles(maxFdPerShard)
	}
	return dld
}