func dumpTileToJSON(store tiling.TileStore, nCommits int, nTraces int, fname string) {
tile, err := store.Get(0, -1)
if err != nil {
glog.Fatal("Could not read tile: " + err.Error())
}
newTile := tile
if (nCommits > 0) || (nTraces > 0) {
lastIdx := tile.LastCommitIndex()
if nCommits <= 0 {
nCommits = lastIdx + 1
}
if nTraces <= 0 {
nTraces = len(tile.Traces)
}
commitLen := util.MinInt(nCommits, lastIdx+1)
startCommit := lastIdx + 1 - commitLen
newTraces := map[string]tiling.Trace{}
for key, trace := range tile.Traces {
for i := startCommit; i <= lastIdx; i++ {
if !trace.IsMissing(i) {
newTraces[key] = trace
break
}
}
if len(newTraces) >= nTraces {
break
}
}
newCommits := tile.Commits[startCommit:]
newParamSet := map[string][]string{}
tiling.GetParamSet(newTraces, newParamSet)
newTile = &tiling.Tile{
Traces: newTraces,
ParamSet: newParamSet,
Commits: newCommits,
Scale: tile.Scale,
TileIndex: tile.TileIndex,
}
}
result, err := json.Marshal(newTile)
if err != nil {
glog.Fatalf("Could not marshal to JSON: %s", err)
}
err = ioutil.WriteFile(fname, result, 0644)
if err != nil {
glog.Fatalf("Could not write output file %s", err)
}
fmt.Printf("Commits included: %d\n", len(newTile.Commits))
fmt.Printf("Traces included: %d\n", len(newTile.Traces))
}
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, error) {
ctx := context.Background()
// Build the Tile.
tile := tiling.NewTile()
n := len(commitIDs)
tile.Commits = make([]*tiling.Commit, n, n)
// Populate the Tile's commits.
for i, cid := range commitIDs {
tile.Commits[i] = &tiling.Commit{
Hash: cid.ID,
CommitTime: cid.Timestamp.Unix(),
}
}
// tileMutex protects access to the Tile. Note that this only means the Tile,
// while writing values into a Trace that already exists and is the right
// size is Go routine safe.
var tileMutex sync.Mutex
errCh := make(chan error, len(commitIDs))
// Fill in the data for each commit in it's own Go routine.
var wg sync.WaitGroup
for i, cid := range commitIDs {
wg.Add(1)
go func(i int, cid *CommitID) {
defer wg.Done()
// Load the values for the commit.
getValuesRequest := &traceservice.GetValuesRequest{
Commitid: tsCommitID(cid),
}
getValuesResponse, err := ts.traceService.GetValues(ctx, getValuesRequest)
if err != nil {
errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
return
}
for _, pair := range getValuesResponse.Values {
if pair == nil {
glog.Errorf("Got a nil ValuePair in response: %s", err)
continue
}
tr, ok := tile.Traces[pair.Key]
if !ok {
tileMutex.Lock()
tile.Traces[pair.Key] = ts.traceBuilder(n)
tileMutex.Unlock()
tr = tile.Traces[pair.Key]
}
if err := tr.SetAt(i, pair.Value); err != nil {
errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
return
}
}
}(i, cid)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, fmt.Errorf("Failed to load trace data: %s", err)
}
default:
}
glog.Infof("Finished loading values. Starting to load Params.")
// Now load the params for the traces.
traceids := []string{}
for k, _ := range tile.Traces {
traceids = append(traceids, k)
}
tracechunks := [][]string{}
for len(traceids) > 0 {
if len(traceids) > CHUNK_SIZE {
tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
traceids = traceids[CHUNK_SIZE:]
} else {
tracechunks = append(tracechunks, traceids)
traceids = []string{}
}
}
errCh = make(chan error, len(tracechunks))
for _, chunk := range tracechunks {
wg.Add(1)
go func(chunk []string) {
defer wg.Done()
req := &traceservice.GetParamsRequest{
Traceids: chunk,
}
resp, err := ts.traceService.GetParams(ctx, req)
if err != nil {
errCh <- fmt.Errorf("Failed to load params: %s", err)
return
}
for _, param := range resp.Params {
dst := tile.Traces[param.Key].Params()
for k, v := range param.Params {
dst[k] = v
}
}
}(chunk)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, fmt.Errorf("Failed to load params: %s", err)
}
default:
}
// Rebuild the ParamSet.
glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
tiling.GetParamSet(tile.Traces, tile.ParamSet)
return tile, nil
}
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, []string, error) {
ts.clearMutex.RLock()
ts.clearMutex.RUnlock()
ctx := context.Background()
// Build the Tile.
tile := tiling.NewTile()
n := len(commitIDs)
tile.Commits = make([]*tiling.Commit, n, n)
hash := make([]string, n)
// Populate the Tile's commits.
for i, cid := range commitIDs {
tile.Commits[i] = &tiling.Commit{
Hash: cid.ID,
CommitTime: cid.Timestamp,
}
}
// tileMutex protects access to the Tile. Note that this only means the Tile,
// while writing values into a Trace that already exists and is the right
// size is Go routine safe.
var tileMutex sync.Mutex
errCh := make(chan error, len(commitIDs))
// Fill in the data for each commit in it's own Go routine.
var wg sync.WaitGroup
for i, cid := range commitIDs {
wg.Add(1)
go func(i int, cid *CommitID) {
defer wg.Done()
// Load the values for the commit.
getValuesRequest := &traceservice.GetValuesRequest{
Commitid: tsCommitID(cid),
}
getRawValues, err := ts.traceService.GetValuesRaw(ctx, getValuesRequest)
if err != nil {
errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
return
}
// Convert raw response into values.
ci, err := traceservice.NewCommitInfo(getRawValues.Value)
if err != nil {
errCh <- fmt.Errorf("Failed to convert values for %d %#v: %s", i, *cid, err)
return
}
// Now make sure we have all the traceids for the trace64ids in ci.
missingKeys64 := []uint64{}
ts.mutex.Lock()
for id64, _ := range ci.Values {
if _, ok := ts.id64Cache[id64]; !ok {
missingKeys64 = append(missingKeys64, id64)
}
}
ts.mutex.Unlock()
if len(missingKeys64) > 0 {
traceidsRequest := &traceservice.GetTraceIDsRequest{
Id: missingKeys64,
}
traceids, err := ts.traceService.GetTraceIDs(ctx, traceidsRequest)
if err != nil {
errCh <- fmt.Errorf("Failed to get traceids for trace64ids for %d %#v: %s", i, *cid, err)
return
}
ts.mutex.Lock()
for _, tid := range traceids.Ids {
ts.id64Cache[tid.Id64] = tid.Id
}
ts.mutex.Unlock()
}
ts.mutex.Lock()
for id64, rawValue := range ci.Values {
if rawValue == nil {
glog.Errorf("Got a nil rawValue in response: %s", err)
continue
}
traceid := ts.id64Cache[id64]
tileMutex.Lock()
tr, ok := tile.Traces[traceid]
if !ok || tr == nil {
tile.Traces[traceid] = ts.traceBuilder(n)
tr = tile.Traces[traceid]
}
tileMutex.Unlock()
if tr == nil {
glog.Errorf("Trace was still nil for key: %v", traceid)
continue
}
if err := tr.SetAt(i, rawValue); err != nil {
errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
return
}
}
// Fill in the commits hash.
hash[i] = getRawValues.Md5
ts.mutex.Unlock()
}(i, cid)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, nil, fmt.Errorf("Failed to load trace data: %s", err)
}
default:
}
glog.Infof("Finished loading values. Starting to load Params.")
// Now load the params for the traces.
traceids := []string{}
ts.mutex.Lock()
for k, _ := range tile.Traces {
// Only load params for traces not already in the cache.
if _, ok := ts.paramsCache[k]; !ok {
traceids = append(traceids, k)
}
}
ts.mutex.Unlock()
// Break the loading of params into chunks and make those requests concurrently.
// The params are just loaded into the paramsCache.
tracechunks := [][]string{}
for len(traceids) > 0 {
if len(traceids) > CHUNK_SIZE {
tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
traceids = traceids[CHUNK_SIZE:]
} else {
tracechunks = append(tracechunks, traceids)
traceids = []string{}
}
}
errCh = make(chan error, len(tracechunks))
for _, chunk := range tracechunks {
wg.Add(1)
go func(chunk []string) {
defer wg.Done()
req := &traceservice.GetParamsRequest{
Traceids: chunk,
}
resp, err := ts.traceService.GetParams(ctx, req)
if err != nil {
errCh <- fmt.Errorf("Failed to load params: %s", err)
return
}
for _, param := range resp.Params {
ts.mutex.Lock()
ts.paramsCache[param.Key] = param.Params
ts.mutex.Unlock()
}
}(chunk)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, nil, fmt.Errorf("Failed to load params: %s", err)
}
default:
}
// Add all params from the cache.
ts.mutex.Lock()
for k, tr := range tile.Traces {
p := tr.Params()
for pk, pv := range ts.paramsCache[k] {
p[pk] = pv
}
}
ts.mutex.Unlock()
// Rebuild the ParamSet.
glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
tiling.GetParamSet(tile.Traces, tile.ParamSet)
return tile, hash, nil
}
