func NewMetricsSink(cnf *conf.Config) *MetricsSink {
return &MetricsSink{
lg: common.NewLogger("metrics", cnf),
maxMtx: cnf.GetInt(conf.HTRACE_METRICS_MAX_ADDR_ENTRIES),
HostSpanMetrics: make(common.SpanMetricsMap),
wsLatencyCircBuf: NewCircBufU32(LATENCY_CIRC_BUF_SIZE),
}
}
func CreateHrpcServer(cnf *conf.Config, store *dataStore,
testHooks *hrpcTestHooks) (*HrpcServer, error) {
lg := common.NewLogger("hrpc", cnf)
numHandlers := cnf.GetInt(conf.HTRACE_NUM_HRPC_HANDLERS)
if numHandlers < 1 {
lg.Warnf("%s must be positive: using 1 handler.\n", conf.HTRACE_NUM_HRPC_HANDLERS)
numHandlers = 1
}
if numHandlers > MAX_HRPC_HANDLERS {
lg.Warnf("%s cannot be more than %d: using %d handlers\n",
conf.HTRACE_NUM_HRPC_HANDLERS, MAX_HRPC_HANDLERS, MAX_HRPC_HANDLERS)
numHandlers = MAX_HRPC_HANDLERS
}
hsv := &HrpcServer{
Server: rpc.NewServer(),
hand: &HrpcHandler{
lg: lg,
store: store,
},
cdcs: make(chan *HrpcServerCodec, numHandlers),
shutdown: make(chan interface{}),
ioTimeo: time.Millisecond *
time.Duration(cnf.GetInt64(conf.HTRACE_HRPC_IO_TIMEOUT_MS)),
testHooks: testHooks,
}
for i := 0; i < numHandlers; i++ {
hsv.cdcs <- &HrpcServerCodec{
lg: lg,
hsv: hsv,
msgpackHandle: codec.MsgpackHandle{
WriteExt: true,
},
}
}
var err error
hsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_HRPC_ADDRESS))
if err != nil {
return nil, err
}
hsv.Server.Register(hsv.hand)
hsv.exited.Add(1)
go hsv.run()
lg.Infof("Started HRPC server on %s with %d handler routines. "+
"ioTimeo=%s.\n", hsv.listener.Addr().String(), numHandlers,
hsv.ioTimeo.String())
return hsv, nil
}
func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataStore, error) {
dld := NewDataStoreLoader(cnf)
defer dld.Close()
err := dld.Load()
if err != nil {
dld.lg.Errorf("Error loading datastore: %s\n", err.Error())
return nil, err
}
store := &dataStore{
lg: dld.lg,
shards: make([]*shard, len(dld.shards)),
readOpts: dld.readOpts,
writeOpts: dld.writeOpts,
WrittenSpans: writtenSpans,
msink: NewMetricsSink(cnf),
hb: NewHeartbeater("DatastoreHeartbeater",
cnf.GetInt64(conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS), dld.lg),
rpr: NewReaper(cnf),
startMs: common.TimeToUnixMs(time.Now().UTC()),
}
spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE)
for shdIdx := range store.shards {
shd := &shard{
store: store,
ldb: dld.shards[shdIdx].ldb,
path: dld.shards[shdIdx].path,
incoming: make(chan []*IncomingSpan, spanBufferSize),
heartbeats: make(chan interface{}, 1),
}
shd.exited.Add(1)
go shd.processIncoming()
store.shards[shdIdx] = shd
store.hb.AddHeartbeatTarget(&HeartbeatTarget{
name: fmt.Sprintf("shard(%s)", shd.path),
targetChan: shd.heartbeats,
})
}
dld.DisownResources()
return store, nil
}
// Create a new datastore loader.
// Initializes the loader, but does not load any leveldb instances.
func NewDataStoreLoader(cnf *conf.Config) *DataStoreLoader {
dld := &DataStoreLoader{
lg: common.NewLogger("datastore", cnf),
ClearStored: cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR),
}
dld.readOpts = levigo.NewReadOptions()
dld.readOpts.SetFillCache(true)
dld.readOpts.SetVerifyChecksums(false)
dld.writeOpts = levigo.NewWriteOptions()
dld.writeOpts.SetSync(false)
dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
rdirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)
// Filter out empty entries
dirs := make([]string, 0, len(rdirs))
for i := range rdirs {
if strings.TrimSpace(rdirs[i]) != "" {
dirs = append(dirs, rdirs[i])
}
}
dld.shards = make([]*ShardLoader, len(dirs))
for i := range dirs {
dld.shards[i] = &ShardLoader{
dld: dld,
path: dirs[i] + conf.PATH_SEP + "db",
}
}
dld.openOpts = levigo.NewOptions()
cacheSize := cnf.GetInt(conf.HTRACE_LEVELDB_CACHE_SIZE)
dld.openOpts.SetCache(levigo.NewLRUCache(cacheSize))
dld.openOpts.SetParanoidChecks(false)
writeBufferSize := cnf.GetInt(conf.HTRACE_LEVELDB_WRITE_BUFFER_SIZE)
if writeBufferSize > 0 {
dld.openOpts.SetWriteBufferSize(writeBufferSize)
}
maxFdPerShard := dld.calculateMaxOpenFilesPerShard()
if maxFdPerShard > 0 {
dld.openOpts.SetMaxOpenFiles(maxFdPerShard)
}
return dld
}
func CreateShard(store *dataStore, cnf *conf.Config, path string,
clearStored bool) (*shard, error) {
lg := store.lg
if clearStored {
fi, err := os.Stat(path)
if err != nil && !os.IsNotExist(err) {
lg.Errorf("Failed to stat %s: %s\n", path, err.Error())
return nil, err
}
if fi != nil {
err = os.RemoveAll(path)
if err != nil {
lg.Errorf("Failed to clear existing datastore directory %s: %s\n",
path, err.Error())
return nil, err
}
lg.Infof("Cleared existing datastore directory %s\n", path)
}
}
err := os.MkdirAll(path, 0777)
if err != nil {
lg.Errorf("Failed to MkdirAll(%s): %s\n", path, err.Error())
return nil, err
}
var shd *shard
openOpts := levigo.NewOptions()
defer openOpts.Close()
newlyCreated := false
ldb, err := levigo.Open(path, openOpts)
if err == nil {
store.lg.Infof("LevelDB opened %s\n", path)
} else {
store.lg.Debugf("LevelDB failed to open %s: %s\n", path, err.Error())
openOpts.SetCreateIfMissing(true)
ldb, err = levigo.Open(path, openOpts)
if err != nil {
store.lg.Errorf("LevelDB failed to create %s: %s\n", path, err.Error())
return nil, err
}
store.lg.Infof("Created new LevelDB instance in %s\n", path)
newlyCreated = true
}
defer func() {
if shd == nil {
ldb.Close()
}
}()
lv, err := readLayoutVersion(store, ldb)
if err != nil {
store.lg.Errorf("Got error while reading datastore version for %s: %s\n",
path, err.Error())
return nil, err
}
if newlyCreated && (lv == UNKNOWN_LAYOUT_VERSION) {
err = writeDataStoreVersion(store, ldb, CURRENT_LAYOUT_VERSION)
if err != nil {
store.lg.Errorf("Got error while writing datastore version for %s: %s\n",
path, err.Error())
return nil, err
}
store.lg.Tracef("Wrote layout version %d to shard at %s.\n",
CURRENT_LAYOUT_VERSION, path)
} else if lv != CURRENT_LAYOUT_VERSION {
versionName := "unknown"
if lv != UNKNOWN_LAYOUT_VERSION {
versionName = fmt.Sprintf("%d", lv)
}
store.lg.Errorf("Can't read old datastore. Its layout version is %s, but this "+
"software is at layout version %d. Please set %s to clear the datastore "+
"on startup, or clear it manually.\n", versionName,
CURRENT_LAYOUT_VERSION, conf.HTRACE_DATA_STORE_CLEAR)
return nil, errors.New(fmt.Sprintf("Invalid layout version: got %s, expected %d.",
versionName, CURRENT_LAYOUT_VERSION))
} else {
store.lg.Tracef("Found layout version %d in %s.\n", lv, path)
}
spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE)
shd = &shard{store: store, ldb: ldb, path: path,
incoming: make(chan *common.Span, spanBufferSize)}
return shd, nil
}