func verifySuccessfulLoad(t *testing.T, allSpans common.SpanSlice,
dataDirs []string) {
htraceBld := &MiniHTracedBuilder{
Name: "TestReloadDataStore#verifySuccessfulLoad",
DataDirs: dataDirs,
KeepDataDirsOnClose: true,
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
defer hcl.Close()
for i := 0; i < len(allSpans); i++ {
span, err := hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
// Look up the spans we wrote.
var span *common.Span
for i := 0; i < len(allSpans); i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
}
func TestDumpAll(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestDumpAll",
DataDirs: make([]string, 2),
WrittenSpans: common.NewSemaphore(0),
Cnf: map[string]string{
conf.HTRACE_LOG_LEVEL: "INFO",
},
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
defer hcl.Close()
NUM_TEST_SPANS := 100
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
sort.Sort(allSpans)
err = hcl.WriteSpans(allSpans)
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
ht.Store.WrittenSpans.Waits(int64(NUM_TEST_SPANS))
out := make(chan *common.Span, NUM_TEST_SPANS)
var dumpErr error
go func() {
dumpErr = hcl.DumpAll(3, out)
}()
var numSpans int
nextLogTime := time.Now().Add(time.Millisecond * 5)
for {
span, channelOpen := <-out
if !channelOpen {
break
}
common.ExpectSpansEqual(t, allSpans[numSpans], span)
numSpans++
if testing.Verbose() {
now := time.Now()
if !now.Before(nextLogTime) {
nextLogTime = now
nextLogTime = nextLogTime.Add(time.Millisecond * 5)
fmt.Printf("read back %d span(s)...\n", numSpans)
}
}
}
if numSpans != len(allSpans) {
t.Fatalf("expected to read %d spans... but only read %d\n",
len(allSpans), numSpans)
}
if dumpErr != nil {
t.Fatalf("got dump error %s\n", dumpErr.Error())
}
}
func TestReadSpans(t *testing.T) {
SPAN_TEST_STR := `{"a":"b9f2a1e07b6e4f16b0c2b27303b20e79",` +
`"b":1424736225037,"e":1424736225901,"d":"ClientNamenodeProtocol#getFileInfo",` +
`"r":"FsShell","p":["3afebdc0a13f4feb811cc5c0e42d30b1"]}
{"a":"3afebdc0a13f4feb811cc5c0e42d30b1","b":1424736224969,` +
`"e":1424736225960,"d":"getFileInfo","r":"FsShell","p":[],"n":{"path":"/"}}
`
r := strings.NewReader(SPAN_TEST_STR)
spans, err := readSpans(r)
if err != nil {
t.Fatalf("Failed to read spans from string via readSpans: %s\n", err.Error())
}
SPAN_TEST_EXPECTED := common.SpanSlice{
&common.Span{
Id: common.TestId("b9f2a1e07b6e4f16b0c2b27303b20e79"),
SpanData: common.SpanData{
Begin: 1424736225037,
End: 1424736225901,
Description: "ClientNamenodeProtocol#getFileInfo",
TracerId: "FsShell",
Parents: []common.SpanId{common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1")},
},
},
&common.Span{
Id: common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1"),
SpanData: common.SpanData{
Begin: 1424736224969,
End: 1424736225960,
Description: "getFileInfo",
TracerId: "FsShell",
Parents: []common.SpanId{},
Info: common.TraceInfoMap{
"path": "/",
},
},
},
}
if len(spans) != len(SPAN_TEST_EXPECTED) {
t.Fatalf("Expected %d spans, but got %d\n",
len(SPAN_TEST_EXPECTED), len(spans))
}
for i := range SPAN_TEST_EXPECTED {
common.ExpectSpansEqual(t, spans[i], SPAN_TEST_EXPECTED[i])
}
}
// Test creating a datastore and adding some spans.
func TestDatastoreWriteAndRead(t *testing.T) {
t.Parallel()
htraceBld := &MiniHTracedBuilder{Name: "TestDatastoreWriteAndRead",
Cnf: map[string]string{
conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS: "30000",
},
WrittenSpans: common.NewSemaphore(0),
}
ht, err := htraceBld.Build()
if err != nil {
panic(err)
}
defer ht.Close()
createSpans(SIMPLE_TEST_SPANS, ht.Store)
span := ht.Store.FindSpan(common.TestId("00000000000000000000000000000001"))
if span == nil {
t.Fatal()
}
if !span.Id.Equal(common.TestId("00000000000000000000000000000001")) {
t.Fatal()
}
common.ExpectSpansEqual(t, &SIMPLE_TEST_SPANS[0], span)
children := ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 1)
if len(children) != 1 {
t.Fatalf("expected 1 child, but got %d\n", len(children))
}
children = ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 2)
if len(children) != 2 {
t.Fatalf("expected 2 children, but got %d\n", len(children))
}
sort.Sort(common.SpanIdSlice(children))
if !children[0].Equal(common.TestId("00000000000000000000000000000002")) {
t.Fatal()
}
if !children[1].Equal(common.TestId("00000000000000000000000000000003")) {
t.Fatal()
}
}
// Test creating a datastore and adding some spans.
func TestDatastoreWriteAndRead(t *testing.T) {
t.Parallel()
htraceBld := &MiniHTracedBuilder{Name: "TestDatastoreWriteAndRead",
WrittenSpans: make(chan *common.Span, 100)}
ht, err := htraceBld.Build()
if err != nil {
panic(err)
}
defer ht.Close()
createSpans(SIMPLE_TEST_SPANS, ht.Store)
if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) {
t.Fatal()
}
span := ht.Store.FindSpan(common.TestId("00000000000000000000000000000001"))
if span == nil {
t.Fatal()
}
if !span.Id.Equal(common.TestId("00000000000000000000000000000001")) {
t.Fatal()
}
common.ExpectSpansEqual(t, &SIMPLE_TEST_SPANS[0], span)
children := ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 1)
if len(children) != 1 {
t.Fatalf("expected 1 child, but got %d\n", len(children))
}
children = ht.Store.FindChildren(common.TestId("00000000000000000000000000000001"), 2)
if len(children) != 2 {
t.Fatalf("expected 2 children, but got %d\n", len(children))
}
sort.Sort(common.SpanIdSlice(children))
if !children[0].Equal(common.TestId("00000000000000000000000000000002")) {
t.Fatal()
}
if !children[1].Equal(common.TestId("00000000000000000000000000000003")) {
t.Fatal()
}
}
func TestReloadDataStore(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore",
Cnf: map[string]string{
conf.HTRACE_DATASTORE_HEARTBEAT_PERIOD_MS: "30000",
},
DataDirs: make([]string, 2),
KeepDataDirsOnClose: true,
WrittenSpans: common.NewSemaphore(0),
}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
dataDirs := make([]string, len(ht.DataDirs))
copy(dataDirs, ht.DataDirs)
defer func() {
if ht != nil {
ht.Close()
}
for i := range dataDirs {
os.RemoveAll(dataDirs[i])
}
}()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
hcnf := ht.Cnf.Clone()
// Create some random trace spans.
NUM_TEST_SPANS := 5
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
err = hcl.WriteSpans(allSpans)
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
ht.Store.WrittenSpans.Waits(int64(NUM_TEST_SPANS))
// Look up the spans we wrote.
var span *common.Span
for i := 0; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
hcl.Close()
ht.Close()
ht = nil
// Verify that we can reload the datastore, even if we configure the data
// directories in a different order.
verifySuccessfulLoad(t, allSpans, []string{dataDirs[1], dataDirs[0]})
// If we try to reload the datastore with only one directory, it won't work
// (we need both).
verifyFailedLoad(t, []string{dataDirs[1]},
"The TotalShards field of all shards is 2, but we have 1 shards.")
// Test that we give an intelligent error message when 0 directories are
// configured.
verifyFailedLoad(t, []string{}, "No shard directories found.")
// Can't specify the same directory more than once... will get "lock
// already held by process"
verifyFailedLoad(t, []string{dataDirs[0], dataDirs[1], dataDirs[1]},
" already held by process.")
// Open the datastore and modify it to have the wrong DaemonId
dld := NewDataStoreLoader(hcnf)
defer func() {
if dld != nil {
dld.Close()
dld = nil
}
}()
dld.LoadShards()
sinfo, err := dld.shards[0].readShardInfo()
if err != nil {
t.Fatalf("error reading shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
newDaemonId := sinfo.DaemonId + 1
dld.lg.Infof("Read %s from shard %s. Changing daemonId to 0x%016x\n.",
asJson(sinfo), dld.shards[0].path, newDaemonId)
sinfo.DaemonId = newDaemonId
err = dld.shards[0].writeShardInfo(sinfo)
if err != nil {
t.Fatalf("error writing shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
dld.Close()
dld = nil
verifyFailedLoad(t, dataDirs, "DaemonId mismatch.")
// Open the datastore and modify it to have the wrong TotalShards
dld = NewDataStoreLoader(hcnf)
dld.LoadShards()
sinfo, err = dld.shards[0].readShardInfo()
if err != nil {
t.Fatalf("error reading shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
newDaemonId = sinfo.DaemonId - 1
dld.lg.Infof("Read %s from shard %s. Changing daemonId to 0x%016x, "+
"TotalShards to 3\n.",
asJson(sinfo), dld.shards[0].path, newDaemonId)
sinfo.DaemonId = newDaemonId
sinfo.TotalShards = 3
err = dld.shards[0].writeShardInfo(sinfo)
if err != nil {
t.Fatalf("error writing shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
dld.Close()
dld = nil
verifyFailedLoad(t, dataDirs, "TotalShards mismatch.")
// Open the datastore and modify it to have the wrong LayoutVersion
dld = NewDataStoreLoader(hcnf)
dld.LoadShards()
for shardIdx := range dld.shards {
sinfo, err = dld.shards[shardIdx].readShardInfo()
if err != nil {
t.Fatalf("error reading shard info for shard %s: %s\n",
dld.shards[shardIdx].path, err.Error())
}
dld.lg.Infof("Read %s from shard %s. Changing TotalShards to 2, "+
"LayoutVersion to 2\n", asJson(sinfo), dld.shards[shardIdx].path)
sinfo.TotalShards = 2
sinfo.LayoutVersion = 2
err = dld.shards[shardIdx].writeShardInfo(sinfo)
if err != nil {
t.Fatalf("error writing shard info for shard %s: %s\n",
dld.shards[0].path, err.Error())
}
}
dld.Close()
dld = nil
verifyFailedLoad(t, dataDirs, "The layout version of all shards is 2, "+
"but we only support")
// It should work with data.store.clear set.
htraceBld = &MiniHTracedBuilder{
Name: "TestReloadDataStore#clear",
DataDirs: dataDirs,
KeepDataDirsOnClose: true,
Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"},
}
ht, err = htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
}
func TestReloadDataStore(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore",
DataDirs: make([]string, 2), KeepDataDirsOnClose: true}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
dataDirs := make([]string, len(ht.DataDirs))
copy(dataDirs, ht.DataDirs)
defer func() {
if ht != nil {
ht.Close()
}
for i := range dataDirs {
os.RemoveAll(dataDirs[i])
}
}()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf())
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
// Create some random trace spans.
NUM_TEST_SPANS := 5
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
err = hcl.WriteSpans(&common.WriteSpansReq{
Spans: allSpans,
})
if err != nil {
t.Fatalf("WriteSpans failed: %s\n", err.Error())
}
// Look up the spans we wrote.
var span *common.Span
for i := 0; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
ht.Close()
ht = nil
htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore2",
DataDirs: dataDirs, KeepDataDirsOnClose: true}
ht, err = htraceBld.Build()
if err != nil {
t.Fatalf("failed to re-create datastore: %s", err.Error())
}
hcl, err = htrace.NewClient(ht.ClientConf())
if err != nil {
t.Fatalf("failed to re-create client: %s", err.Error())
}
// Look up the spans we wrote earlier.
for i := 0; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
// Set an old datastore version number.
for i := range ht.Store.shards {
shard := ht.Store.shards[i]
writeDataStoreVersion(ht.Store, shard.ldb, CURRENT_LAYOUT_VERSION-1)
}
ht.Close()
ht = nil
htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore3",
DataDirs: dataDirs, KeepDataDirsOnClose: true}
ht, err = htraceBld.Build()
if err == nil {
t.Fatalf("expected the datastore to fail to load after setting an " +
"incorrect version.\n")
}
if !strings.Contains(err.Error(), "Invalid layout version") {
t.Fatal(`expected the loading error to contain "invalid layout version"` + "\n")
}
// It should work with data.store.clear set.
htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore4",
DataDirs: dataDirs, KeepDataDirsOnClose: true,
Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"}}
ht, err = htraceBld.Build()
if err != nil {
t.Fatalf("expected the datastore loading to succeed after setting an "+
"incorrect version. But it failed with error %s\n", err.Error())
}
}
func TestClientOperations(t *testing.T) {
htraceBld := &MiniHTracedBuilder{Name: "TestClientOperations",
DataDirs: make([]string, 2)}
ht, err := htraceBld.Build()
if err != nil {
t.Fatalf("failed to create datastore: %s", err.Error())
}
defer ht.Close()
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf())
if err != nil {
t.Fatalf("failed to create client: %s", err.Error())
}
// Create some random trace spans.
NUM_TEST_SPANS := 30
allSpans := createRandomTestSpans(NUM_TEST_SPANS)
// Write half of the spans to htraced via the client.
err = hcl.WriteSpans(&common.WriteSpansReq{
Spans: allSpans[0 : NUM_TEST_SPANS/2],
})
if err != nil {
t.Fatalf("WriteSpans(0:%d) failed: %s\n", NUM_TEST_SPANS/2,
err.Error())
}
// Look up the first half of the spans. They should be found.
var span *common.Span
for i := 0; i < NUM_TEST_SPANS/2; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
common.ExpectSpansEqual(t, allSpans[i], span)
}
// Look up the second half of the spans. They should not be found.
for i := NUM_TEST_SPANS / 2; i < NUM_TEST_SPANS; i++ {
span, err = hcl.FindSpan(allSpans[i].Id)
if err != nil {
t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error())
}
if span != nil {
t.Fatalf("Unexpectedly found a span we never write to "+
"the server: FindSpan(%d) succeeded\n", i)
}
}
// Test FindChildren
childSpan := allSpans[1]
parentId := childSpan.Parents[0]
var children []common.SpanId
children, err = hcl.FindChildren(parentId, 1)
if err != nil {
t.Fatalf("FindChildren(%s) failed: %s\n", parentId, err.Error())
}
if len(children) != 1 {
t.Fatalf("FindChildren(%s) returned an invalid number of "+
"children: expected %d, got %d\n", parentId, 1, len(children))
}
if !children[0].Equal(childSpan.Id) {
t.Fatalf("FindChildren(%s) returned an invalid child id: expected %s, "+
" got %s\n", parentId, childSpan.Id, children[0])
}
// Test FindChildren on a span that has no children
childlessSpan := allSpans[NUM_TEST_SPANS/2]
children, err = hcl.FindChildren(childlessSpan.Id, 10)
if err != nil {
t.Fatalf("FindChildren(%d) failed: %s\n", childlessSpan.Id, err.Error())
}
if len(children) != 0 {
t.Fatalf("FindChildren(%d) returned an invalid number of "+
"children: expected %d, got %d\n", childlessSpan.Id, 0, len(children))
}
// Test Query
var query common.Query
query = common.Query{Lim: 10}
spans, err := hcl.Query(&query)
if err != nil {
t.Fatalf("Query({lim: %d}) failed: %s\n", 10, err.Error())
}
if len(spans) != 10 {
t.Fatalf("Query({lim: %d}) returned an invalid number of "+
"children: expected %d, got %d\n", 10, 10, len(spans))
}
}
