func TestReapingOldSpans(t *testing.T) {
const NUM_TEST_SPANS = 20
testSpans := make([]*common.Span, NUM_TEST_SPANS)
rnd := rand.New(rand.NewSource(2))
now := common.TimeToUnixMs(time.Now().UTC())
for i := range testSpans {
testSpans[i] = test.NewRandomSpan(rnd, testSpans[0:i])
testSpans[i].Begin = now - int64(NUM_TEST_SPANS-1-i)
testSpans[i].Description = fmt.Sprintf("Span%02d", i)
}
htraceBld := &MiniHTracedBuilder{Name: "TestReapingOldSpans",
Cnf: map[string]string{
conf.HTRACE_SPAN_EXPIRY_MS: fmt.Sprintf("%d", 60*60*1000),
conf.HTRACE_REAPER_HEARTBEAT_PERIOD_MS: "1",
conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS: "1",
},
WrittenSpans: common.NewSemaphore(0),
DataDirs: make([]string, 2),
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create mini htraced cluster: %s\n", err.Error())
}
ing := ht.Store.NewSpanIngestor(ht.Store.lg, "127.0.0.1", "")
for spanIdx := range testSpans {
ing.IngestSpan(testSpans[spanIdx])
}
ing.Close(time.Now())
// Wait the spans to be created
ht.Store.WrittenSpans.Waits(NUM_TEST_SPANS)
// Set a reaper date that will remove all the spans except final one.
ht.Store.rpr.SetReaperDate(now)
common.WaitFor(5*time.Minute, time.Millisecond, func() bool {
for i := 0; i < NUM_TEST_SPANS-1; i++ {
span := ht.Store.FindSpan(testSpans[i].Id)
if span != nil {
ht.Store.lg.Debugf("Waiting for %s to be removed...\n",
testSpans[i].Description)
return false
}
}
span := ht.Store.FindSpan(testSpans[NUM_TEST_SPANS-1].Id)
if span == nil {
ht.Store.lg.Debugf("Did not expect %s to be removed\n",
testSpans[NUM_TEST_SPANS-1].Description)
return false
}
return true
})
defer ht.Close()
}
func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataStore, error) {
dld := NewDataStoreLoader(cnf)
defer dld.Close()
err := dld.Load()
if err != nil {
dld.lg.Errorf("Error loading datastore: %s\n", err.Error())
return nil, err
}
store := &dataStore{
lg: dld.lg,
shards: make([]*shard, len(dld.shards)),
readOpts: dld.readOpts,
writeOpts: dld.writeOpts,
WrittenSpans: writtenSpans,
msink: NewMetricsSink(cnf),
hb: NewHeartbeater("DatastoreHeartbeater",
cnf.GetInt64(conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS), dld.lg),
rpr: NewReaper(cnf),
startMs: common.TimeToUnixMs(time.Now().UTC()),
}
spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE)
for shdIdx := range store.shards {
shd := &shard{
store: store,
ldb: dld.shards[shdIdx].ldb,
path: dld.shards[shdIdx].path,
incoming: make(chan []*IncomingSpan, spanBufferSize),
heartbeats: make(chan interface{}, 1),
}
shd.exited.Add(1)
go shd.processIncoming()
store.shards[shdIdx] = shd
store.hb.AddHeartbeatTarget(&HeartbeatTarget{
name: fmt.Sprintf("shard(%s)", shd.path),
targetChan: shd.heartbeats,
})
}
dld.DisownResources()
return store, nil
}
func (rpr *Reaper) handleHeartbeat() {
// TODO: check dataStore fullness
now := common.TimeToUnixMs(time.Now().UTC())
d, updated := func() (int64, bool) {
rpr.lock.Lock()
defer rpr.lock.Unlock()
newReaperDate := now - rpr.spanExpiryMs
if newReaperDate > rpr.reaperDate {
rpr.reaperDate = newReaperDate
return rpr.reaperDate, true
} else {
return rpr.reaperDate, false
}
}()
if rpr.lg.DebugEnabled() {
if updated {
rpr.lg.Debugf("Updating UTC reaper date to %s.\n",
common.UnixMsToTime(d).Format(time.RFC3339))
} else {
rpr.lg.Debugf("Not updating previous reaperDate of %s.\n",
common.UnixMsToTime(d).Format(time.RFC3339))
}
}
}
func (store *dataStore) ServerStats() *common.ServerStats {
serverStats := common.ServerStats{
Dirs: make([]common.StorageDirectoryStats, len(store.shards)),
}
for shardIdx := range store.shards {
shard := store.shards[shardIdx]
serverStats.Dirs[shardIdx].Path = shard.path
r := levigo.Range{
Start: []byte{0},
Limit: []byte{0xff},
}
vals := shard.ldb.GetApproximateSizes([]levigo.Range{r})
serverStats.Dirs[shardIdx].ApproximateBytes = vals[0]
serverStats.Dirs[shardIdx].LevelDbStats =
shard.ldb.PropertyValue("leveldb.stats")
store.msink.lg.Debugf("levedb.stats for %s: %s\n",
shard.path, shard.ldb.PropertyValue("leveldb.stats"))
}
serverStats.LastStartMs = store.startMs
serverStats.CurMs = common.TimeToUnixMs(time.Now().UTC())
serverStats.ReapedSpans = atomic.LoadUint64(&store.rpr.ReapedSpans)
store.msink.PopulateServerStats(&serverStats)
return &serverStats
}