func verifySuccessfulLoad(t *testing.T, allSpans common.SpanSlice, dataDirs []string) { htraceBld := &MiniHTracedBuilder{ Name: "TestReloadDataStore#verifySuccessfulLoad", DataDirs: dataDirs, KeepDataDirsOnClose: true, } ht, err := htraceBld.Build() if err != nil { t.Fatalf("failed to create datastore: %s", err.Error()) } defer ht.Close() var hcl *htrace.Client hcl, err = htrace.NewClient(ht.ClientConf(), nil) if err != nil { t.Fatalf("failed to create client: %s", err.Error()) } defer hcl.Close() for i := 0; i < len(allSpans); i++ { span, err := hcl.FindSpan(allSpans[i].Id) if err != nil { t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) } common.ExpectSpansEqual(t, allSpans[i], span) } // Look up the spans we wrote. var span *common.Span for i := 0; i < len(allSpans); i++ { span, err = hcl.FindSpan(allSpans[i].Id) if err != nil { t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) } common.ExpectSpansEqual(t, allSpans[i], span) } }
func TestDumpAll(t *testing.T) { htraceBld := &MiniHTracedBuilder{Name: "TestDumpAll", DataDirs: make([]string, 2), WrittenSpans: common.NewSemaphore(0), Cnf: map[string]string{ conf.HTRACE_LOG_LEVEL: "INFO", }, } ht, err := htraceBld.Build() if err != nil { t.Fatalf("failed to create datastore: %s", err.Error()) } defer ht.Close() var hcl *htrace.Client hcl, err = htrace.NewClient(ht.ClientConf(), nil) if err != nil { t.Fatalf("failed to create client: %s", err.Error()) } defer hcl.Close() NUM_TEST_SPANS := 100 allSpans := createRandomTestSpans(NUM_TEST_SPANS) sort.Sort(allSpans) err = hcl.WriteSpans(allSpans) if err != nil { t.Fatalf("WriteSpans failed: %s\n", err.Error()) } ht.Store.WrittenSpans.Waits(int64(NUM_TEST_SPANS)) out := make(chan *common.Span, NUM_TEST_SPANS) var dumpErr error go func() { dumpErr = hcl.DumpAll(3, out) }() var numSpans int nextLogTime := time.Now().Add(time.Millisecond * 5) for { span, channelOpen := <-out if !channelOpen { break } common.ExpectSpansEqual(t, allSpans[numSpans], span) numSpans++ if testing.Verbose() { now := time.Now() if !now.Before(nextLogTime) { nextLogTime = now nextLogTime = nextLogTime.Add(time.Millisecond * 5) fmt.Printf("read back %d span(s)...\n", numSpans) } } } if numSpans != len(allSpans) { t.Fatalf("expected to read %d spans... but only read %d\n", len(allSpans), numSpans) } if dumpErr != nil { t.Fatalf("got dump error %s\n", dumpErr.Error()) } }
func TestReadSpans(t *testing.T) { SPAN_TEST_STR := `{"a":"b9f2a1e07b6e4f16b0c2b27303b20e79",` + `"b":1424736225037,"e":1424736225901,"d":"ClientNamenodeProtocol#getFileInfo",` + `"r":"FsShell","p":["3afebdc0a13f4feb811cc5c0e42d30b1"]} {"a":"3afebdc0a13f4feb811cc5c0e42d30b1","b":1424736224969,` + `"e":1424736225960,"d":"getFileInfo","r":"FsShell","p":[],"n":{"path":"/"}} ` r := strings.NewReader(SPAN_TEST_STR) spans, err := readSpans(r) if err != nil { t.Fatalf("Failed to read spans from string via readSpans: %s\n", err.Error()) } SPAN_TEST_EXPECTED := common.SpanSlice{ &common.Span{ Id: common.TestId("b9f2a1e07b6e4f16b0c2b27303b20e79"), SpanData: common.SpanData{ Begin: 1424736225037, End: 1424736225901, Description: "ClientNamenodeProtocol#getFileInfo", TracerId: "FsShell", Parents: []common.SpanId{common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1")}, }, }, &common.Span{ Id: common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1"), SpanData: common.SpanData{ Begin: 1424736224969, End: 1424736225960, Description: "getFileInfo", TracerId: "FsShell", Parents: []common.SpanId{}, Info: common.TraceInfoMap{ "path": "/", }, }, }, } if len(spans) != len(SPAN_TEST_EXPECTED) { t.Fatalf("Expected %d spans, but got %d\n", len(SPAN_TEST_EXPECTED), len(spans)) } for i := range SPAN_TEST_EXPECTED { common.ExpectSpansEqual(t, spans[i], SPAN_TEST_EXPECTED[i]) } }
// Test creating a datastore and adding some spans. func TestDatastoreWriteAndRead(t *testing.T) { t.Parallel() htraceBld := &MiniHTracedBuilder{Name: "TestDatastoreWriteAndRead", Cnf: map[string]string{ conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS: "30000", }, WrittenSpans: common.NewSemaphore(0), } ht, err := htraceBld.Build() if err != nil { panic(err) } defer ht.Close() createSpans(SIMPLE_TEST_SPANS, ht.Store) span := ht.Store.FindSpan(common.TestId("00000000000000000000000000000001")) if span == nil { t.Fatal() } if !span.Id.Equal(common.TestId("00000000000000000000000000000001")) { t.Fatal() } common.ExpectSpansEqual(t, &SIMPLE_TEST_SPANS[0], span) children := ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 1) if len(children) != 1 { t.Fatalf("expected 1 child, but got %d\n", len(children)) } children = ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 2) if len(children) != 2 { t.Fatalf("expected 2 children, but got %d\n", len(children)) } sort.Sort(common.SpanIdSlice(children)) if !children[0].Equal(common.TestId("00000000000000000000000000000002")) { t.Fatal() } if !children[1].Equal(common.TestId("00000000000000000000000000000003")) { t.Fatal() } }
// Test creating a datastore and adding some spans. func TestDatastoreWriteAndRead(t *testing.T) { t.Parallel() htraceBld := &MiniHTracedBuilder{Name: "TestDatastoreWriteAndRead", WrittenSpans: make(chan *common.Span, 100)} ht, err := htraceBld.Build() if err != nil { panic(err) } defer ht.Close() createSpans(SIMPLE_TEST_SPANS, ht.Store) if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { t.Fatal() } span := ht.Store.FindSpan(common.TestId("00000000000000000000000000000001")) if span == nil { t.Fatal() } if !span.Id.Equal(common.TestId("00000000000000000000000000000001")) { t.Fatal() } common.ExpectSpansEqual(t, &SIMPLE_TEST_SPANS[0], span) children := ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 1) if len(children) != 1 { t.Fatalf("expected 1 child, but got %d\n", len(children)) } children = ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 2) if len(children) != 2 { t.Fatalf("expected 2 children, but got %d\n", len(children)) } sort.Sort(common.SpanIdSlice(children)) if !children[0].Equal(common.TestId("00000000000000000000000000000002")) { t.Fatal() } if !children[1].Equal(common.TestId("00000000000000000000000000000003")) { t.Fatal() } }
func TestReloadDataStore(t *testing.T) { htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore", Cnf: map[string]string{ conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS: "30000", }, DataDirs: make([]string, 2), KeepDataDirsOnClose: true, WrittenSpans: common.NewSemaphore(0), } ht, err := htraceBld.Build() if err != nil { t.Fatalf("failed to create datastore: %s", err.Error()) } dataDirs := make([]string, len(ht.DataDirs)) copy(dataDirs, ht.DataDirs) defer func() { if ht != nil { ht.Close() } for i := range dataDirs { os.RemoveAll(dataDirs[i]) } }() var hcl *htrace.Client hcl, err = htrace.NewClient(ht.ClientConf(), nil) if err != nil { t.Fatalf("failed to create client: %s", err.Error()) } hcnf := ht.Cnf.Clone() // Create some random trace spans. NUM_TEST_SPANS := 5 allSpans := createRandomTestSpans(NUM_TEST_SPANS) err = hcl.WriteSpans(allSpans) if err != nil { t.Fatalf("WriteSpans failed: %s\n", err.Error()) } ht.Store.WrittenSpans.Waits(int64(NUM_TEST_SPANS)) // Look up the spans we wrote. var span *common.Span for i := 0; i < NUM_TEST_SPANS; i++ { span, err = hcl.FindSpan(allSpans[i].Id) if err != nil { t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) } common.ExpectSpansEqual(t, allSpans[i], span) } hcl.Close() ht.Close() ht = nil // Verify that we can reload the datastore, even if we configure the data // directories in a different order. verifySuccessfulLoad(t, allSpans, []string{dataDirs[1], dataDirs[0]}) // If we try to reload the datastore with only one directory, it won't work // (we need both). verifyFailedLoad(t, []string{dataDirs[1]}, "The TotalShards field of all shards is 2, but we have 1 shards.") // Test that we give an intelligent error message when 0 directories are // configured. verifyFailedLoad(t, []string{}, "No shard directories found.") // Can't specify the same directory more than once... will get "lock // already held by process" verifyFailedLoad(t, []string{dataDirs[0], dataDirs[1], dataDirs[1]}, " already held by process.") // Open the datastore and modify it to have the wrong DaemonId dld := NewDataStoreLoader(hcnf) defer func() { if dld != nil { dld.Close() dld = nil } }() dld.LoadShards() sinfo, err := dld.shards[0].readShardInfo() if err != nil { t.Fatalf("error reading shard info for shard %s: %s\n", dld.shards[0].path, err.Error()) } newDaemonId := sinfo.DaemonId + 1 dld.lg.Infof("Read %s from shard %s. Changing daemonId to 0x%016x\n.", asJson(sinfo), dld.shards[0].path, newDaemonId) sinfo.DaemonId = newDaemonId err = dld.shards[0].writeShardInfo(sinfo) if err != nil { t.Fatalf("error writing shard info for shard %s: %s\n", dld.shards[0].path, err.Error()) } dld.Close() dld = nil verifyFailedLoad(t, dataDirs, "DaemonId mismatch.") // Open the datastore and modify it to have the wrong TotalShards dld = NewDataStoreLoader(hcnf) dld.LoadShards() sinfo, err = dld.shards[0].readShardInfo() if err != nil { t.Fatalf("error reading shard info for shard %s: %s\n", dld.shards[0].path, err.Error()) } newDaemonId = sinfo.DaemonId - 1 dld.lg.Infof("Read %s from shard %s. Changing daemonId to 0x%016x, "+ "TotalShards to 3\n.", asJson(sinfo), dld.shards[0].path, newDaemonId) sinfo.DaemonId = newDaemonId sinfo.TotalShards = 3 err = dld.shards[0].writeShardInfo(sinfo) if err != nil { t.Fatalf("error writing shard info for shard %s: %s\n", dld.shards[0].path, err.Error()) } dld.Close() dld = nil verifyFailedLoad(t, dataDirs, "TotalShards mismatch.") // Open the datastore and modify it to have the wrong LayoutVersion dld = NewDataStoreLoader(hcnf) dld.LoadShards() for shardIdx := range dld.shards { sinfo, err = dld.shards[shardIdx].readShardInfo() if err != nil { t.Fatalf("error reading shard info for shard %s: %s\n", dld.shards[shardIdx].path, err.Error()) } dld.lg.Infof("Read %s from shard %s. Changing TotalShards to 2, "+ "LayoutVersion to 2\n", asJson(sinfo), dld.shards[shardIdx].path) sinfo.TotalShards = 2 sinfo.LayoutVersion = 2 err = dld.shards[shardIdx].writeShardInfo(sinfo) if err != nil { t.Fatalf("error writing shard info for shard %s: %s\n", dld.shards[0].path, err.Error()) } } dld.Close() dld = nil verifyFailedLoad(t, dataDirs, "The layout version of all shards is 2, "+ "but we only support") // It should work with data.store.clear set. htraceBld = &MiniHTracedBuilder{ Name: "TestReloadDataStore#clear", DataDirs: dataDirs, KeepDataDirsOnClose: true, Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"}, } ht, err = htraceBld.Build() if err != nil { t.Fatalf("failed to create datastore: %s", err.Error()) } }
func TestReloadDataStore(t *testing.T) { htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore", DataDirs: make([]string, 2), KeepDataDirsOnClose: true} ht, err := htraceBld.Build() if err != nil { t.Fatalf("failed to create datastore: %s", err.Error()) } dataDirs := make([]string, len(ht.DataDirs)) copy(dataDirs, ht.DataDirs) defer func() { if ht != nil { ht.Close() } for i := range dataDirs { os.RemoveAll(dataDirs[i]) } }() var hcl *htrace.Client hcl, err = htrace.NewClient(ht.ClientConf()) if err != nil { t.Fatalf("failed to create client: %s", err.Error()) } // Create some random trace spans. NUM_TEST_SPANS := 5 allSpans := createRandomTestSpans(NUM_TEST_SPANS) err = hcl.WriteSpans(&common.WriteSpansReq{ Spans: allSpans, }) if err != nil { t.Fatalf("WriteSpans failed: %s\n", err.Error()) } // Look up the spans we wrote. var span *common.Span for i := 0; i < NUM_TEST_SPANS; i++ { span, err = hcl.FindSpan(allSpans[i].Id) if err != nil { t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) } common.ExpectSpansEqual(t, allSpans[i], span) } ht.Close() ht = nil htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore2", DataDirs: dataDirs, KeepDataDirsOnClose: true} ht, err = htraceBld.Build() if err != nil { t.Fatalf("failed to re-create datastore: %s", err.Error()) } hcl, err = htrace.NewClient(ht.ClientConf()) if err != nil { t.Fatalf("failed to re-create client: %s", err.Error()) } // Look up the spans we wrote earlier. for i := 0; i < NUM_TEST_SPANS; i++ { span, err = hcl.FindSpan(allSpans[i].Id) if err != nil { t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) } common.ExpectSpansEqual(t, allSpans[i], span) } // Set an old datastore version number. for i := range ht.Store.shards { shard := ht.Store.shards[i] writeDataStoreVersion(ht.Store, shard.ldb, CURRENT_LAYOUT_VERSION-1) } ht.Close() ht = nil htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore3", DataDirs: dataDirs, KeepDataDirsOnClose: true} ht, err = htraceBld.Build() if err == nil { t.Fatalf("expected the datastore to fail to load after setting an " + "incorrect version.\n") } if !strings.Contains(err.Error(), "Invalid layout version") { t.Fatal(`expected the loading error to contain "invalid layout version"` + "\n") } // It should work with data.store.clear set. htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore4", DataDirs: dataDirs, KeepDataDirsOnClose: true, Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"}} ht, err = htraceBld.Build() if err != nil { t.Fatalf("expected the datastore loading to succeed after setting an "+ "incorrect version. But it failed with error %s\n", err.Error()) } }
func TestClientOperations(t *testing.T) { htraceBld := &MiniHTracedBuilder{Name: "TestClientOperations", DataDirs: make([]string, 2)} ht, err := htraceBld.Build() if err != nil { t.Fatalf("failed to create datastore: %s", err.Error()) } defer ht.Close() var hcl *htrace.Client hcl, err = htrace.NewClient(ht.ClientConf()) if err != nil { t.Fatalf("failed to create client: %s", err.Error()) } // Create some random trace spans. NUM_TEST_SPANS := 30 allSpans := createRandomTestSpans(NUM_TEST_SPANS) // Write half of the spans to htraced via the client. err = hcl.WriteSpans(&common.WriteSpansReq{ Spans: allSpans[0 : NUM_TEST_SPANS/2], }) if err != nil { t.Fatalf("WriteSpans(0:%d) failed: %s\n", NUM_TEST_SPANS/2, err.Error()) } // Look up the first half of the spans. They should be found. var span *common.Span for i := 0; i < NUM_TEST_SPANS/2; i++ { span, err = hcl.FindSpan(allSpans[i].Id) if err != nil { t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) } common.ExpectSpansEqual(t, allSpans[i], span) } // Look up the second half of the spans. They should not be found. for i := NUM_TEST_SPANS / 2; i < NUM_TEST_SPANS; i++ { span, err = hcl.FindSpan(allSpans[i].Id) if err != nil { t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) } if span != nil { t.Fatalf("Unexpectedly found a span we never write to "+ "the server: FindSpan(%d) succeeded\n", i) } } // Test FindChildren childSpan := allSpans[1] parentId := childSpan.Parents[0] var children []common.SpanId children, err = hcl.FindChildren(parentId, 1) if err != nil { t.Fatalf("FindChildren(%s) failed: %s\n", parentId, err.Error()) } if len(children) != 1 { t.Fatalf("FindChildren(%s) returned an invalid number of "+ "children: expected %d, got %d\n", parentId, 1, len(children)) } if !children[0].Equal(childSpan.Id) { t.Fatalf("FindChildren(%s) returned an invalid child id: expected %s, "+ " got %s\n", parentId, childSpan.Id, children[0]) } // Test FindChildren on a span that has no children childlessSpan := allSpans[NUM_TEST_SPANS/2] children, err = hcl.FindChildren(childlessSpan.Id, 10) if err != nil { t.Fatalf("FindChildren(%d) failed: %s\n", childlessSpan.Id, err.Error()) } if len(children) != 0 { t.Fatalf("FindChildren(%d) returned an invalid number of "+ "children: expected %d, got %d\n", childlessSpan.Id, 0, len(children)) } // Test Query var query common.Query query = common.Query{Lim: 10} spans, err := hcl.Query(&query) if err != nil { t.Fatalf("Query({lim: %d}) failed: %s\n", 10, err.Error()) } if len(spans) != 10 { t.Fatalf("Query({lim: %d}) returned an invalid number of "+ "children: expected %d, got %d\n", 10, 10, len(spans)) } }