func testIngestedSpansMetricsImpl(t *testing.T, usePacked bool) {
htraceBld := &MiniHTracedBuilder{Name: "TestIngestedSpansMetrics",
DataDirs: make([]string, 2),
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), &htrace.TestHooks{
HrpcDisabled: !usePacked,
})
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
NUM_TEST_SPANS := 12
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
err = hcl.WriteSpans(allSpans)
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
for {
var stats *common.ServerStats
stats, err = hcl.GetServerStats()
if err != nil {
t.Fatalf("GetServerStats failed: %s\n", err.Error())
}
if stats.IngestedSpans == uint64(NUM_TEST_SPANS) {
break
}
time.Sleep(1 * time.Millisecond)
}
}
func TestClientGetServerConf(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestClientGetServerConf",
Cnf: map[string]string{
EXAMPLE_CONF_KEY: EXAMPLE_CONF_VALUE,
},
DataDirs: make([]string, 2)}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
defer hcl.Close()
serverCnf, err2 := hcl.GetServerConf()
if err2 != nil {
t.Fatalf("failed to call GetServerConf: %s", err2.Error())
}
if serverCnf[EXAMPLE_CONF_KEY] != EXAMPLE_CONF_VALUE {
t.Fatalf("unexpected value for %s: %s",
EXAMPLE_CONF_KEY, EXAMPLE_CONF_VALUE)
}
}
func TestClientGetServerDebugInfo(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestClientGetServerDebugInfo",
DataDirs: make([]string, 2)}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
defer hcl.Close()
debugInfo, err := hcl.GetServerDebugInfo()
if err != nil {
t.Fatalf("failed to call GetServerDebugInfo: %s", err.Error())
}
if debugInfo.StackTraces == "" {
t.Fatalf(`debugInfo.StackTraces == ""`)
}
if debugInfo.GCStats == "" {
t.Fatalf(`debugInfo.GCStats == ""`)
}
}
func doLoadSpans(hcl *htrace.Client, reader io.Reader) int {
dec := json.NewDecoder(reader)
spans := make([]*common.Span, 0, 32)
var err error
for {
var span common.Span
if err = dec.Decode(&span); err != nil {
if err == io.EOF {
break
}
fmt.Printf("Failed to decode JSON: %s\n", err.Error())
return EXIT_FAILURE
}
spans = append(spans, &span)
}
if *verbose {
fmt.Printf("Writing ")
prefix := ""
for i := range spans {
fmt.Printf("%s%s", prefix, spans[i].ToJson())
prefix = ", "
}
fmt.Printf("\n")
}
err = hcl.WriteSpans(&common.WriteSpansReq{
Spans: spans,
})
if err != nil {
fmt.Println(err.Error())
return EXIT_FAILURE
}
return EXIT_SUCCESS
}
// Print information retrieved from an htraced server via /server/info
func printServerInfo(hcl *htrace.Client) int {
info, err := hcl.GetServerInfo()
if err != nil {
fmt.Println(err.Error())
return EXIT_FAILURE
}
fmt.Printf("HTraced server version %s (%s)\n", info.ReleaseVersion, info.GitVersion)
return EXIT_SUCCESS
}
func TestDumpAll(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestDumpAll",
DataDirs: make([]string, 2)}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf())
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
NUM_TEST_SPANS := 100
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
sort.Sort(allSpans)
err = hcl.WriteSpans(&common.WriteSpansReq{
Spans: allSpans,
})
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
out := make(chan *common.Span, 50)
var dumpErr error
go func() {
dumpErr = hcl.DumpAll(3, out)
}()
var numSpans int
nextLogTime := time.Now().Add(time.Millisecond * 5)
for {
span, channelOpen := <-out
if !channelOpen {
break
}
common.ExpectSpansEqual(t, allSpans[numSpans], span)
numSpans++
if testing.Verbose() {
now := time.Now()
if !now.Before(nextLogTime) {
nextLogTime = now
nextLogTime = nextLogTime.Add(time.Millisecond * 5)
fmt.Printf("read back %d span(s)...\n", numSpans)
}
}
}
if numSpans != len(allSpans) {
t.Fatalf("expected to read %d spans... but only read %d\n",
len(allSpans), numSpans)
}
if dumpErr != nil {
t.Fatalf("got dump error %s\n", dumpErr.Error())
}
}
// Tests that HRPC I/O timeouts work.
func TestHrpcIoTimeout(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestHrpcIoTimeout",
DataDirs: make([]string, 2),
Cnf: map[string]string{
conf.HTRACE_NUM_HRPC_HANDLERS: fmt.Sprintf("%d", TEST_NUM_HRPC_HANDLERS),
conf.HTRACE_HRPC_IO_TIMEOUT_MS: "1",
},
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
finishClient := make(chan interface{})
defer func() {
// Close the finishClient channel, if it hasn't already been closed.
defer func() { recover() }()
close(finishClient)
}()
testHooks := &htrace.TestHooks{
HandleWriteRequestBody: func() {
<-finishClient
},
}
hcl, err = htrace.NewClient(ht.ClientConf(), testHooks)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
// Create some random trace spans.
allSpans := createRandomTestSpans(TEST_NUM_WRITESPANS)
var wg sync.WaitGroup
wg.Add(TEST_NUM_WRITESPANS)
for iter := 0; iter < TEST_NUM_WRITESPANS; iter++ {
go func(i int) {
defer wg.Done()
// Ignore the error return because there are internal retries in
// the client which will make this succeed eventually, usually.
// Keep in mind that we only block until we have seen
// TEST_NUM_WRITESPANS I/O errors in the HRPC server-- after that,
// we let requests through so that the test can exit cleanly.
hcl.WriteSpans(allSpans[i : i+1])
}(iter)
}
for {
if ht.Hsv.GetNumIoErrors() >= TEST_NUM_WRITESPANS {
break
}
time.Sleep(1000 * time.Nanosecond)
}
close(finishClient)
wg.Wait()
}
// Dump all spans from the htraced daemon.
func doDumpAll(hcl *htrace.Client, outPath string, lim int) error {
file, err := CreateOutputFile(outPath)
if err != nil {
return err
}
w := bufio.NewWriter(file)
defer func() {
if file != nil {
w.Flush()
file.Close()
}
}()
out := make(chan *common.Span, 50)
var dumpErr error
go func() {
dumpErr = hcl.DumpAll(lim, out)
}()
var numSpans int64
nextLogTime := time.Now().Add(time.Second * 5)
for {
span, channelOpen := <-out
if !channelOpen {
break
}
if err == nil {
_, err = fmt.Fprintf(w, "%s\n", span.ToJson())
}
if *verbose {
numSpans++
now := time.Now()
if !now.Before(nextLogTime) {
nextLogTime = now.Add(time.Second * 5)
fmt.Printf("received %d span(s)...\n", numSpans)
}
}
}
if err != nil {
return errors.New(fmt.Sprintf("Write error %s", err.Error()))
}
if dumpErr != nil {
return errors.New(fmt.Sprintf("Dump error %s", dumpErr.Error()))
}
err = w.Flush()
if err != nil {
return err
}
err = file.Close()
file = nil
if err != nil {
return err
}
return nil
}
// Tests that HRPC limits the number of simultaneous connections being processed.
func TestHrpcAdmissionsControl(t *testing.T) {
var wg sync.WaitGroup
wg.Add(TEST_NUM_WRITESPANS)
var numConcurrentHrpcCalls int32
testHooks := &hrpcTestHooks{
HandleAdmission: func() {
defer wg.Done()
n := atomic.AddInt32(&numConcurrentHrpcCalls, 1)
if n > TEST_NUM_HRPC_HANDLERS {
t.Fatalf("The number of concurrent HRPC calls went above "+
"%d: it's at %d\n", TEST_NUM_HRPC_HANDLERS, n)
}
time.Sleep(1 * time.Millisecond)
n = atomic.AddInt32(&numConcurrentHrpcCalls, -1)
if n >= TEST_NUM_HRPC_HANDLERS {
t.Fatalf("The number of concurrent HRPC calls went above "+
"%d: it was at %d\n", TEST_NUM_HRPC_HANDLERS, n+1)
}
},
}
htraceBld := &MiniHTracedBuilder{Name: "TestHrpcAdmissionsControl",
DataDirs: make([]string, 2),
Cnf: map[string]string{
conf.HTRACE_NUM_HRPC_HANDLERS: fmt.Sprintf("%d", TEST_NUM_HRPC_HANDLERS),
},
WrittenSpans: common.NewSemaphore(0),
HrpcTestHooks: testHooks,
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
// Create some random trace spans.
allSpans := createRandomTestSpans(TEST_NUM_WRITESPANS)
for iter := 0; iter < TEST_NUM_WRITESPANS; iter++ {
go func(i int) {
err = hcl.WriteSpans(allSpans[i : i+1])
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
}(iter)
}
wg.Wait()
ht.Store.WrittenSpans.Waits(int64(TEST_NUM_WRITESPANS))
}
// Print information retrieved from an htraced server via /server/debugInfo
func printServerDebugInfo(hcl *htrace.Client) int {
stats, err := hcl.GetServerDebugInfo()
if err != nil {
fmt.Println(err.Error())
return EXIT_FAILURE
}
fmt.Println("=== GOROUTINE STACKS ===")
fmt.Print(stats.StackTraces)
fmt.Println("=== END GOROUTINE STACKS ===")
fmt.Println("=== GC STATISTICS ===")
fmt.Print(stats.GCStats)
fmt.Println("=== END GC STATISTICS ===")
return EXIT_SUCCESS
}
// Print information retrieved from an htraced server via /server/info
func printServerStats(hcl *htrace.Client) int {
stats, err := hcl.GetServerStats()
if err != nil {
fmt.Println(err.Error())
return EXIT_FAILURE
}
w := new(tabwriter.Writer)
w.Init(os.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintf(w, "HTRACED SERVER STATS\n")
fmt.Fprintf(w, "Datastore Start\t%s\n",
common.UnixMsToTime(stats.LastStartMs).Format(time.RFC3339))
fmt.Fprintf(w, "Server Time\t%s\n",
common.UnixMsToTime(stats.CurMs).Format(time.RFC3339))
fmt.Fprintf(w, "Spans reaped\t%d\n", stats.ReapedSpans)
fmt.Fprintf(w, "Spans ingested\t%d\n", stats.IngestedSpans)
fmt.Fprintf(w, "Spans written\t%d\n", stats.WrittenSpans)
fmt.Fprintf(w, "Spans dropped by server\t%d\n", stats.ServerDroppedSpans)
dur := time.Millisecond * time.Duration(stats.AverageWriteSpansLatencyMs)
fmt.Fprintf(w, "Average WriteSpan Latency\t%s\n", dur.String())
dur = time.Millisecond * time.Duration(stats.MaxWriteSpansLatencyMs)
fmt.Fprintf(w, "Maximum WriteSpan Latency\t%s\n", dur.String())
fmt.Fprintf(w, "Number of leveldb directories\t%d\n", len(stats.Dirs))
w.Flush()
fmt.Println("")
for i := range stats.Dirs {
dir := stats.Dirs[i]
fmt.Printf("==== %s ===\n", dir.Path)
fmt.Printf("Approximate number of bytes: %d\n", dir.ApproximateBytes)
stats := strings.Replace(dir.LevelDbStats, "\\n", "\n", -1)
fmt.Printf("%s\n", stats)
}
w = new(tabwriter.Writer)
w.Init(os.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintf(w, "HOST SPAN METRICS\n")
mtxMap := stats.HostSpanMetrics
keys := make(sort.StringSlice, len(mtxMap))
i := 0
for k, _ := range mtxMap {
keys[i] = k
i++
}
sort.Sort(keys)
for k := range keys {
mtx := mtxMap[keys[k]]
fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\n",
keys[k], mtx.Written, mtx.ServerDropped)
}
w.Flush()
return EXIT_SUCCESS
}
// Find information about the children of a span.
func doFindChildren(hcl *htrace.Client, sid common.SpanId, lim int) int {
spanIds, err := hcl.FindChildren(sid, lim)
if err != nil {
fmt.Printf("%s\n", err.Error())
return EXIT_FAILURE
}
pbuf, err := json.MarshalIndent(spanIds, "", " ")
if err != nil {
fmt.Println("Error: error pretty-printing span IDs to JSON: %s", err.Error())
return 1
}
fmt.Printf("%s\n", string(pbuf))
return 0
}
// Print information retrieved from an htraced server via /server/info as JSON
func printServerStatsJson(hcl *htrace.Client) int {
stats, err := hcl.GetServerStats()
if err != nil {
fmt.Println(err.Error())
return EXIT_FAILURE
}
buf, err := json.MarshalIndent(stats, "", " ")
if err != nil {
fmt.Printf("Error marshalling server stats: %s", err.Error())
return EXIT_FAILURE
}
fmt.Printf("%s\n", string(buf))
return EXIT_SUCCESS
}
// Print information retrieved from an htraced server via /server/info
func printServerStats(hcl *htrace.Client) int {
stats, err := hcl.GetServerStats()
if err != nil {
fmt.Println(err.Error())
return EXIT_FAILURE
}
fmt.Printf("HTraced server stats:\n")
fmt.Printf("%d leveldb shards.\n", len(stats.Shards))
for i := range stats.Shards {
shard := stats.Shards[i]
fmt.Printf("==== %s ===\n", shard.Path)
fmt.Printf("Approximate number of spans: %d\n", shard.ApproxNumSpans)
stats := strings.Replace(shard.LevelDbStats, "\\n", "\n", -1)
fmt.Printf("%s\n", stats)
}
return EXIT_SUCCESS
}
func verifySuccessfulLoad(t *testing.T, allSpans common.SpanSlice,
dataDirs []string) {
htraceBld := &MiniHTracedBuilder{
Name: "TestReloadDataStore#verifySuccessfulLoad",
DataDirs: dataDirs,
KeepDataDirsOnClose: true,
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
defer hcl.Close()
for i := 0; i < len(allSpans); i++ {
span, err := hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
// Look up the spans we wrote.
var span *common.Span
for i := 0; i < len(allSpans); i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
}
// Print information about a trace span.
func doFindSpan(hcl *htrace.Client, sid common.SpanId) int {
span, err := hcl.FindSpan(sid)
if err != nil {
fmt.Println(err.Error())
return EXIT_FAILURE
}
if span == nil {
fmt.Printf("Span ID not found.\n")
return EXIT_FAILURE
}
pbuf, err := json.MarshalIndent(span, "", " ")
if err != nil {
fmt.Printf("Error: error pretty-printing span to JSON: %s\n", err.Error())
return EXIT_FAILURE
}
fmt.Printf("%s\n", string(pbuf))
return EXIT_SUCCESS
}
func TestClientGetServerVersion(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestClientGetServerVersion",
DataDirs: make([]string, 2)}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
defer hcl.Close()
_, err = hcl.GetServerVersion()
if err != nil {
t.Fatalf("failed to call GetServerVersion: %s", err.Error())
}
}
// Send a query.
func doQuery(hcl *htrace.Client, query *common.Query) error {
if *verbose {
qbytes, err := json.Marshal(*query)
if err != nil {
qbytes = []byte("marshaling error: " + err.Error())
}
fmt.Printf("Sending query: %s\n", string(qbytes))
}
spans, err := hcl.Query(query)
if err != nil {
return err
}
if *verbose {
fmt.Printf("%d results...\n", len(spans))
}
for i := range spans {
fmt.Printf("%s\n", spans[i].ToJson())
}
return nil
}
func TestReloadDataStore(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore",
Cnf: map[string]string{
conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS: "30000",
},
DataDirs: make([]string, 2),
KeepDataDirsOnClose: true,
WrittenSpans: common.NewSemaphore(0),
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
dataDirs := make([]string, len(ht.DataDirs))
copy(dataDirs, ht.DataDirs)
defer func() {
if ht != nil {
ht.Close()
}
for i := range dataDirs {
os.RemoveAll(dataDirs[i])
}
}()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
hcnf := ht.Cnf.Clone()
// Create some random trace spans.
NUM_TEST_SPANS := 5
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
err = hcl.WriteSpans(allSpans)
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
ht.Store.WrittenSpans.Waits(int64(NUM_TEST_SPANS))
// Look up the spans we wrote.
var span *common.Span
for i := 0; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
hcl.Close()
ht.Close()
ht = nil
// Verify that we can reload the datastore, even if we configure the data
// directories in a different order.
verifySuccessfulLoad(t, allSpans, []string{dataDirs[1], dataDirs[0]})
// If we try to reload the datastore with only one directory, it won't work
// (we need both).
verifyFailedLoad(t, []string{dataDirs[1]},
"The TotalShards field of all shards is 2, but we have 1 shards.")
// Test that we give an intelligent error message when 0 directories are
// configured.
verifyFailedLoad(t, []string{}, "No shard directories found.")
// Can't specify the same directory more than once... will get "lock
// already held by process"
verifyFailedLoad(t, []string{dataDirs[0], dataDirs[1], dataDirs[1]},
" already held by process.")
// Open the datastore and modify it to have the wrong DaemonId
dld := NewDataStoreLoader(hcnf)
defer func() {
if dld != nil {
dld.Close()
dld = nil
}
}()
dld.LoadShards()
sinfo, err := dld.shards[0].readShardInfo()
if err != nil {
t.Fatalf("error reading shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
newDaemonId := sinfo.DaemonId + 1
dld.lg.Infof("Read %s from shard %s. Changing daemonId to 0x%016x\n.",
asJson(sinfo), dld.shards[0].path, newDaemonId)
sinfo.DaemonId = newDaemonId
err = dld.shards[0].writeShardInfo(sinfo)
if err != nil {
t.Fatalf("error writing shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
dld.Close()
dld = nil
verifyFailedLoad(t, dataDirs, "DaemonId mismatch.")
// Open the datastore and modify it to have the wrong TotalShards
dld = NewDataStoreLoader(hcnf)
dld.LoadShards()
sinfo, err = dld.shards[0].readShardInfo()
if err != nil {
t.Fatalf("error reading shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
newDaemonId = sinfo.DaemonId - 1
dld.lg.Infof("Read %s from shard %s. Changing daemonId to 0x%016x, "+
"TotalShards to 3\n.",
asJson(sinfo), dld.shards[0].path, newDaemonId)
sinfo.DaemonId = newDaemonId
sinfo.TotalShards = 3
err = dld.shards[0].writeShardInfo(sinfo)
if err != nil {
t.Fatalf("error writing shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
dld.Close()
dld = nil
verifyFailedLoad(t, dataDirs, "TotalShards mismatch.")
// Open the datastore and modify it to have the wrong LayoutVersion
dld = NewDataStoreLoader(hcnf)
dld.LoadShards()
for shardIdx := range dld.shards {
sinfo, err = dld.shards[shardIdx].readShardInfo()
if err != nil {
t.Fatalf("error reading shard info for shard %s: %s\n",
dld.shards[shardIdx].path, err.Error())
}
dld.lg.Infof("Read %s from shard %s. Changing TotalShards to 2, "+
"LayoutVersion to 2\n", asJson(sinfo), dld.shards[shardIdx].path)
sinfo.TotalShards = 2
sinfo.LayoutVersion = 2
err = dld.shards[shardIdx].writeShardInfo(sinfo)
if err != nil {
t.Fatalf("error writing shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
}
dld.Close()
dld = nil
verifyFailedLoad(t, dataDirs, "The layout version of all shards is 2, "+
"but we only support")
// It should work with data.store.clear set.
htraceBld = &MiniHTracedBuilder{
Name: "TestReloadDataStore#clear",
DataDirs: dataDirs,
KeepDataDirsOnClose: true,
Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"},
}
ht, err = htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
}
func TestReloadDataStore(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore",
DataDirs: make([]string, 2), KeepDataDirsOnClose: true}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
dataDirs := make([]string, len(ht.DataDirs))
copy(dataDirs, ht.DataDirs)
defer func() {
if ht != nil {
ht.Close()
}
for i := range dataDirs {
os.RemoveAll(dataDirs[i])
}
}()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf())
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
// Create some random trace spans.
NUM_TEST_SPANS := 5
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
err = hcl.WriteSpans(&common.WriteSpansReq{
Spans: allSpans,
})
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
// Look up the spans we wrote.
var span *common.Span
for i := 0; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
ht.Close()
ht = nil
htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore2",
DataDirs: dataDirs, KeepDataDirsOnClose: true}
ht, err = htraceBld.Build()
if err != nil {
t.Fatalf("failed to re-create datastore: %s", err.Error())
}
hcl, err = htrace.NewClient(ht.ClientConf())
if err != nil {
t.Fatalf("failed to re-create client: %s", err.Error())
}
// Look up the spans we wrote earlier.
for i := 0; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
// Set an old datastore version number.
for i := range ht.Store.shards {
shard := ht.Store.shards[i]
writeDataStoreVersion(ht.Store, shard.ldb, CURRENT_LAYOUT_VERSION-1)
}
ht.Close()
ht = nil
htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore3",
DataDirs: dataDirs, KeepDataDirsOnClose: true}
ht, err = htraceBld.Build()
if err == nil {
t.Fatalf("expected the datastore to fail to load after setting an " +
"incorrect version.\n")
}
if !strings.Contains(err.Error(), "Invalid layout version") {
t.Fatal(`expected the loading error to contain "invalid layout version"` + "\n")
}
// It should work with data.store.clear set.
htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore4",
DataDirs: dataDirs, KeepDataDirsOnClose: true,
Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"}}
ht, err = htraceBld.Build()
if err != nil {
t.Fatalf("expected the datastore loading to succeed after setting an "+
"incorrect version. But it failed with error %s\n", err.Error())
}
}
func doWriteSpans(name string, N int, maxSpansPerRpc uint32, b *testing.B) {
htraceBld := &MiniHTracedBuilder{Name: "doWriteSpans",
Cnf: map[string]string{
conf.HTRACE_LOG_LEVEL: "INFO",
conf.HTRACE_NUM_HRPC_HANDLERS: "20",
},
WrittenSpans: common.NewSemaphore(int64(1 - N)),
}
ht, err := htraceBld.Build()
if err != nil {
panic(err)
}
defer ht.Close()
rnd := rand.New(rand.NewSource(1))
allSpans := make([]*common.Span, N)
for n := 0; n < N; n++ {
allSpans[n] = test.NewRandomSpan(rnd, allSpans[0:n])
}
// Determine how many calls to WriteSpans we should make. Each writeSpans
// message should be small enough so that it doesn't exceed the max RPC
// body length limit. TODO: a production-quality golang client would do
// this internally rather than needing us to do it here in the unit test.
bodyLen := (4 * common.MAX_HRPC_BODY_LENGTH) / 5
reqs := make([][]*common.Span, 0, 4)
curReq := -1
curReqLen := bodyLen
var curReqSpans uint32
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
var mbuf [8192]byte
buf := mbuf[:0]
enc := codec.NewEncoderBytes(&buf, mh)
for n := 0; n < N; n++ {
span := allSpans[n]
if (curReqSpans >= maxSpansPerRpc) ||
(curReqLen >= bodyLen) {
reqs = append(reqs, make([]*common.Span, 0, 16))
curReqLen = 0
curReq++
curReqSpans = 0
}
buf = mbuf[:0]
enc.ResetBytes(&buf)
err := enc.Encode(span)
if err != nil {
panic(fmt.Sprintf("Error encoding span %s: %s\n",
span.String(), err.Error()))
}
bufLen := len(buf)
if bufLen > (bodyLen / 5) {
panic(fmt.Sprintf("Span too long at %d bytes\n", bufLen))
}
curReqLen += bufLen
reqs[curReq] = append(reqs[curReq], span)
curReqSpans++
}
ht.Store.lg.Infof("num spans: %d. num WriteSpansReq calls: %d\n", N, len(reqs))
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
panic(fmt.Sprintf("failed to create client: %s", err.Error()))
}
defer hcl.Close()
// Reset the timer to avoid including the time required to create new
// random spans in the benchmark total.
if b != nil {
b.ResetTimer()
}
// Write many random spans.
for reqIdx := range reqs {
go func(i int) {
err = hcl.WriteSpans(reqs[i])
if err != nil {
panic(fmt.Sprintf("failed to send WriteSpans request %d: %s",
i, err.Error()))
}
}(reqIdx)
}
// Wait for all the spans to be written.
ht.Store.WrittenSpans.Wait()
}
func TestClientOperations(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestClientOperations",
DataDirs: make([]string, 2)}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf())
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
// Create some random trace spans.
NUM_TEST_SPANS := 30
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
// Write half of the spans to htraced via the client.
err = hcl.WriteSpans(&common.WriteSpansReq{
Spans: allSpans[0 : NUM_TEST_SPANS/2],
})
if err != nil {
t.Fatalf("WriteSpans(0:%d) failed: %s\n", NUM_TEST_SPANS/2,
err.Error())
}
// Look up the first half of the spans. They should be found.
var span *common.Span
for i := 0; i < NUM_TEST_SPANS/2; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
// Look up the second half of the spans. They should not be found.
for i := NUM_TEST_SPANS / 2; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
if span != nil {
t.Fatalf("Unexpectedly found a span we never write to "+
"the server: FindSpan(%d) succeeded\n", i)
}
}
// Test FindChildren
childSpan := allSpans[1]
parentId := childSpan.Parents[0]
var children []common.SpanId
children, err = hcl.FindChildren(parentId, 1)
if err != nil {
t.Fatalf("FindChildren(%s) failed: %s\n", parentId, err.Error())
}
if len(children) != 1 {
t.Fatalf("FindChildren(%s) returned an invalid number of "+
"children: expected %d, got %d\n", parentId, 1, len(children))
}
if !children[0].Equal(childSpan.Id) {
t.Fatalf("FindChildren(%s) returned an invalid child id: expected %s, "+
" got %s\n", parentId, childSpan.Id, children[0])
}
// Test FindChildren on a span that has no children
childlessSpan := allSpans[NUM_TEST_SPANS/2]
children, err = hcl.FindChildren(childlessSpan.Id, 10)
if err != nil {
t.Fatalf("FindChildren(%d) failed: %s\n", childlessSpan.Id, err.Error())
}
if len(children) != 0 {
t.Fatalf("FindChildren(%d) returned an invalid number of "+
"children: expected %d, got %d\n", childlessSpan.Id, 0, len(children))
}
// Test Query
var query common.Query
query = common.Query{Lim: 10}
spans, err := hcl.Query(&query)
if err != nil {
t.Fatalf("Query({lim: %d}) failed: %s\n", 10, err.Error())
}
if len(spans) != 10 {
t.Fatalf("Query({lim: %d}) returned an invalid number of "+
"children: expected %d, got %d\n", 10, 10, len(spans))
}
}
