func NewReaper(cnf *conf.Config) *Reaper {
rpr := &Reaper{
lg: common.NewLogger("reaper", cnf),
spanExpiryMs: cnf.GetInt64(conf.HTRACE_SPAN_EXPIRY_MS),
heartbeats: make(chan interface{}, 1),
}
if rpr.spanExpiryMs >= MAX_SPAN_EXPIRY_MS {
rpr.spanExpiryMs = MAX_SPAN_EXPIRY_MS
} else if rpr.spanExpiryMs <= 0 {
rpr.spanExpiryMs = MAX_SPAN_EXPIRY_MS
}
rpr.hb = NewHeartbeater("ReaperHeartbeater",
cnf.GetInt64(conf.HTRACE_REAPER_HEARTBEAT_PERIOD_MS), rpr.lg)
rpr.exited.Add(1)
go rpr.run()
rpr.hb.AddHeartbeatTarget(&HeartbeatTarget{
name: "reaper",
targetChan: rpr.heartbeats,
})
var when string
if rpr.spanExpiryMs >= MAX_SPAN_EXPIRY_MS {
when = "never"
} else {
when = "after " + time.Duration(rpr.spanExpiryMs).String()
}
rpr.lg.Infof("Initializing span reaper: span time out = %s.\n", when)
return rpr
}
func CreateRestServer(cnf *conf.Config, store *dataStore) (*RestServer, error) {
var err error
rsv := &RestServer{}
rsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_WEB_ADDRESS))
if err != nil {
return nil, err
}
var success bool
defer func() {
if !success {
rsv.Close()
}
}()
rsv.lg = common.NewLogger("rest", cnf)
r := mux.NewRouter().StrictSlash(false)
r.Handle("/server/info", &serverInfoHandler{lg: rsv.lg}).Methods("GET")
serverStatsH := &serverStatsHandler{dataStoreHandler: dataStoreHandler{
store: store, lg: rsv.lg}}
r.Handle("/server/stats", serverStatsH).Methods("GET")
writeSpansH := &writeSpansHandler{dataStoreHandler: dataStoreHandler{
store: store, lg: rsv.lg}}
r.Handle("/writeSpans", writeSpansH).Methods("POST")
queryH := &queryHandler{lg: rsv.lg, dataStoreHandler: dataStoreHandler{store: store}}
r.Handle("/query", queryH).Methods("GET")
span := r.PathPrefix("/span").Subrouter()
findSidH := &findSidHandler{dataStoreHandler: dataStoreHandler{store: store, lg: rsv.lg}}
span.Handle("/{id}", findSidH).Methods("GET")
findChildrenH := &findChildrenHandler{dataStoreHandler: dataStoreHandler{store: store,
lg: rsv.lg}}
span.Handle("/{id}/children", findChildrenH).Methods("GET")
// Default Handler. This will serve requests for static requests.
webdir := os.Getenv("HTRACED_WEB_DIR")
if webdir == "" {
webdir, err = filepath.Abs(filepath.Join(filepath.Dir(os.Args[0]), "..", "web"))
if err != nil {
return nil, err
}
}
rsv.lg.Infof("Serving static files from %s\n.", webdir)
r.PathPrefix("/").Handler(http.FileServer(http.Dir(webdir))).Methods("GET")
// Log an error message for unknown non-GET requests.
r.PathPrefix("/").Handler(&logErrorHandler{lg: rsv.lg})
go http.Serve(rsv.listener, r)
rsv.lg.Infof("Started REST server on %s...\n", rsv.listener.Addr().String())
success = true
return rsv, nil
}
func NewMetricsSink(cnf *conf.Config) *MetricsSink {
return &MetricsSink{
lg: common.NewLogger("metrics", cnf),
maxMtx: cnf.GetInt(conf.HTRACE_METRICS_MAX_ADDR_ENTRIES),
HostSpanMetrics: make(common.SpanMetricsMap),
wsLatencyCircBuf: NewCircBufU32(LATENCY_CIRC_BUF_SIZE),
}
}
// A golang client for htraced.
// TODO: fancier APIs for streaming spans in the background, optimize TCP stuff
func NewClient(cnf *conf.Config, testHooks *TestHooks) (*Client, error) {
hcl := Client{testHooks: testHooks}
hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS)
if testHooks != nil && testHooks.HrpcDisabled {
hcl.hrpcAddr = ""
} else {
hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS)
}
return &hcl, nil
}
func CreateHrpcServer(cnf *conf.Config, store *dataStore) (*HrpcServer, error) {
lg := common.NewLogger("hrpc", cnf)
hsv := &HrpcServer{
Server: rpc.NewServer(),
hand: &HrpcHandler{
lg: lg,
store: store,
},
}
var err error
hsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_HRPC_ADDRESS))
if err != nil {
return nil, err
}
hsv.Server.Register(hsv.hand)
go hsv.run()
lg.Infof("Started HRPC server on %s...\n", hsv.listener.Addr().String())
return hsv, nil
}
func CreateDataStore(cnf *conf.Config, writtenSpans chan *common.Span) (*dataStore, error) {
// Get the configuration.
clearStored := cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR)
dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
dirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)
var err error
lg := common.NewLogger("datastore", cnf)
store := &dataStore{lg: lg, shards: []*shard{}, WrittenSpans: writtenSpans}
// If we return an error, close the store.
defer func() {
if err != nil {
store.Close()
store = nil
}
}()
store.readOpts = levigo.NewReadOptions()
store.readOpts.SetFillCache(true)
store.writeOpts = levigo.NewWriteOptions()
store.writeOpts.SetSync(false)
// Open all shards
for idx := range dirs {
path := dirs[idx] + conf.PATH_SEP + "db"
var shd *shard
shd, err = CreateShard(store, cnf, path, clearStored)
if err != nil {
lg.Errorf("Error creating shard %s: %s\n", path, err.Error())
return nil, err
}
store.shards = append(store.shards, shd)
}
for idx := range store.shards {
shd := store.shards[idx]
shd.exited = make(chan bool, 1)
go shd.processIncoming()
}
return store, nil
}
func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataStore, error) {
dld := NewDataStoreLoader(cnf)
defer dld.Close()
err := dld.Load()
if err != nil {
dld.lg.Errorf("Error loading datastore: %s\n", err.Error())
return nil, err
}
store := &dataStore{
lg: dld.lg,
shards: make([]*shard, len(dld.shards)),
readOpts: dld.readOpts,
writeOpts: dld.writeOpts,
WrittenSpans: writtenSpans,
msink: NewMetricsSink(cnf),
hb: NewHeartbeater("DatastoreHeartbeater",
cnf.GetInt64(conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS), dld.lg),
rpr: NewReaper(cnf),
startMs: common.TimeToUnixMs(time.Now().UTC()),
}
spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE)
for shdIdx := range store.shards {
shd := &shard{
store: store,
ldb: dld.shards[shdIdx].ldb,
path: dld.shards[shdIdx].path,
incoming: make(chan []*IncomingSpan, spanBufferSize),
heartbeats: make(chan interface{}, 1),
}
shd.exited.Add(1)
go shd.processIncoming()
store.shards[shdIdx] = shd
store.hb.AddHeartbeatTarget(&HeartbeatTarget{
name: fmt.Sprintf("shard(%s)", shd.path),
targetChan: shd.heartbeats,
})
}
dld.DisownResources()
return store, nil
}
// Create a new datastore loader.
// Initializes the loader, but does not load any leveldb instances.
func NewDataStoreLoader(cnf *conf.Config) *DataStoreLoader {
dld := &DataStoreLoader{
lg: common.NewLogger("datastore", cnf),
ClearStored: cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR),
}
dld.readOpts = levigo.NewReadOptions()
dld.readOpts.SetFillCache(true)
dld.readOpts.SetVerifyChecksums(false)
dld.writeOpts = levigo.NewWriteOptions()
dld.writeOpts.SetSync(false)
dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES)
rdirs := strings.Split(dirsStr, conf.PATH_LIST_SEP)
// Filter out empty entries
dirs := make([]string, 0, len(rdirs))
for i := range rdirs {
if strings.TrimSpace(rdirs[i]) != "" {
dirs = append(dirs, rdirs[i])
}
}
dld.shards = make([]*ShardLoader, len(dirs))
for i := range dirs {
dld.shards[i] = &ShardLoader{
dld: dld,
path: dirs[i] + conf.PATH_SEP + "db",
}
}
dld.openOpts = levigo.NewOptions()
cacheSize := cnf.GetInt(conf.HTRACE_LEVELDB_CACHE_SIZE)
dld.openOpts.SetCache(levigo.NewLRUCache(cacheSize))
dld.openOpts.SetParanoidChecks(false)
writeBufferSize := cnf.GetInt(conf.HTRACE_LEVELDB_WRITE_BUFFER_SIZE)
if writeBufferSize > 0 {
dld.openOpts.SetWriteBufferSize(writeBufferSize)
}
maxFdPerShard := dld.calculateMaxOpenFilesPerShard()
if maxFdPerShard > 0 {
dld.openOpts.SetMaxOpenFiles(maxFdPerShard)
}
return dld
}
func CreateHrpcServer(cnf *conf.Config, store *dataStore,
testHooks *hrpcTestHooks) (*HrpcServer, error) {
lg := common.NewLogger("hrpc", cnf)
numHandlers := cnf.GetInt(conf.HTRACE_NUM_HRPC_HANDLERS)
if numHandlers < 1 {
lg.Warnf("%s must be positive: using 1 handler.\n", conf.HTRACE_NUM_HRPC_HANDLERS)
numHandlers = 1
}
if numHandlers > MAX_HRPC_HANDLERS {
lg.Warnf("%s cannot be more than %d: using %d handlers\n",
conf.HTRACE_NUM_HRPC_HANDLERS, MAX_HRPC_HANDLERS, MAX_HRPC_HANDLERS)
numHandlers = MAX_HRPC_HANDLERS
}
hsv := &HrpcServer{
Server: rpc.NewServer(),
hand: &HrpcHandler{
lg: lg,
store: store,
},
cdcs: make(chan *HrpcServerCodec, numHandlers),
shutdown: make(chan interface{}),
ioTimeo: time.Millisecond *
time.Duration(cnf.GetInt64(conf.HTRACE_HRPC_IO_TIMEOUT_MS)),
testHooks: testHooks,
}
for i := 0; i < numHandlers; i++ {
hsv.cdcs <- &HrpcServerCodec{
lg: lg,
hsv: hsv,
msgpackHandle: codec.MsgpackHandle{
WriteExt: true,
},
}
}
var err error
hsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_HRPC_ADDRESS))
if err != nil {
return nil, err
}
hsv.Server.Register(hsv.hand)
hsv.exited.Add(1)
go hsv.run()
lg.Infof("Started HRPC server on %s with %d handler routines. "+
"ioTimeo=%s.\n", hsv.listener.Addr().String(), numHandlers,
hsv.ioTimeo.String())
return hsv, nil
}
func NewClient(cnf *conf.Config) (*Client, error) {
hcl := Client{}
hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS)
hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS)
return &hcl, nil
}
func parseConf(faculty string, cnf *conf.Config) (string, Level) {
facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH
var facultyLogPath string
if cnf.Contains(facultyLogPathKey) {
facultyLogPath = cnf.Get(facultyLogPathKey)
} else {
facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH)
}
facultyLogLevelKey := faculty + "." + conf.HTRACE_LOG_LEVEL
var facultyLogLevelStr string
if cnf.Contains(facultyLogLevelKey) {
facultyLogLevelStr = cnf.Get(facultyLogLevelKey)
} else {
facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL)
}
level, err := LevelFromString(facultyLogLevelStr)
if err != nil {
fmt.Fprintf(os.Stderr, "Error configuring log level: %s. Using TRACE.\n")
level = TRACE
}
return facultyLogPath, level
}
func CreateShard(store *dataStore, cnf *conf.Config, path string,
clearStored bool) (*shard, error) {
lg := store.lg
if clearStored {
fi, err := os.Stat(path)
if err != nil && !os.IsNotExist(err) {
lg.Errorf("Failed to stat %s: %s\n", path, err.Error())
return nil, err
}
if fi != nil {
err = os.RemoveAll(path)
if err != nil {
lg.Errorf("Failed to clear existing datastore directory %s: %s\n",
path, err.Error())
return nil, err
}
lg.Infof("Cleared existing datastore directory %s\n", path)
}
}
err := os.MkdirAll(path, 0777)
if err != nil {
lg.Errorf("Failed to MkdirAll(%s): %s\n", path, err.Error())
return nil, err
}
var shd *shard
openOpts := levigo.NewOptions()
defer openOpts.Close()
newlyCreated := false
ldb, err := levigo.Open(path, openOpts)
if err == nil {
store.lg.Infof("LevelDB opened %s\n", path)
} else {
store.lg.Debugf("LevelDB failed to open %s: %s\n", path, err.Error())
openOpts.SetCreateIfMissing(true)
ldb, err = levigo.Open(path, openOpts)
if err != nil {
store.lg.Errorf("LevelDB failed to create %s: %s\n", path, err.Error())
return nil, err
}
store.lg.Infof("Created new LevelDB instance in %s\n", path)
newlyCreated = true
}
defer func() {
if shd == nil {
ldb.Close()
}
}()
lv, err := readLayoutVersion(store, ldb)
if err != nil {
store.lg.Errorf("Got error while reading datastore version for %s: %s\n",
path, err.Error())
return nil, err
}
if newlyCreated && (lv == UNKNOWN_LAYOUT_VERSION) {
err = writeDataStoreVersion(store, ldb, CURRENT_LAYOUT_VERSION)
if err != nil {
store.lg.Errorf("Got error while writing datastore version for %s: %s\n",
path, err.Error())
return nil, err
}
store.lg.Tracef("Wrote layout version %d to shard at %s.\n",
CURRENT_LAYOUT_VERSION, path)
} else if lv != CURRENT_LAYOUT_VERSION {
versionName := "unknown"
if lv != UNKNOWN_LAYOUT_VERSION {
versionName = fmt.Sprintf("%d", lv)
}
store.lg.Errorf("Can't read old datastore. Its layout version is %s, but this "+
"software is at layout version %d. Please set %s to clear the datastore "+
"on startup, or clear it manually.\n", versionName,
CURRENT_LAYOUT_VERSION, conf.HTRACE_DATA_STORE_CLEAR)
return nil, errors.New(fmt.Sprintf("Invalid layout version: got %s, expected %d.",
versionName, CURRENT_LAYOUT_VERSION))
} else {
store.lg.Tracef("Found layout version %d in %s.\n", lv, path)
}
spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE)
shd = &shard{store: store, ldb: ldb, path: path,
incoming: make(chan *common.Span, spanBufferSize)}
return shd, nil
}