func dumpTileToJSON(store tiling.TileStore, nCommits int, nTraces int, fname string) { tile, err := store.Get(0, -1) if err != nil { glog.Fatal("Could not read tile: " + err.Error()) } newTile := tile if (nCommits > 0) || (nTraces > 0) { lastIdx := tile.LastCommitIndex() if nCommits <= 0 { nCommits = lastIdx + 1 } if nTraces <= 0 { nTraces = len(tile.Traces) } commitLen := util.MinInt(nCommits, lastIdx+1) startCommit := lastIdx + 1 - commitLen newTraces := map[string]tiling.Trace{} for key, trace := range tile.Traces { for i := startCommit; i <= lastIdx; i++ { if !trace.IsMissing(i) { newTraces[key] = trace break } } if len(newTraces) >= nTraces { break } } newCommits := tile.Commits[startCommit:] newParamSet := map[string][]string{} tiling.GetParamSet(newTraces, newParamSet) newTile = &tiling.Tile{ Traces: newTraces, ParamSet: newParamSet, Commits: newCommits, Scale: tile.Scale, TileIndex: tile.TileIndex, } } result, err := json.Marshal(newTile) if err != nil { glog.Fatalf("Could not marshal to JSON: %s", err) } err = ioutil.WriteFile(fname, result, 0644) if err != nil { glog.Fatalf("Could not write output file %s", err) } fmt.Printf("Commits included: %d\n", len(newTile.Commits)) fmt.Printf("Traces included: %d\n", len(newTile.Traces)) }
// TileFromCommits implements DB.TileFromCommits(). func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, error) { ctx := context.Background() // Build the Tile. tile := tiling.NewTile() n := len(commitIDs) tile.Commits = make([]*tiling.Commit, n, n) // Populate the Tile's commits. for i, cid := range commitIDs { tile.Commits[i] = &tiling.Commit{ Hash: cid.ID, CommitTime: cid.Timestamp.Unix(), } } // tileMutex protects access to the Tile. Note that this only means the Tile, // while writing values into a Trace that already exists and is the right // size is Go routine safe. var tileMutex sync.Mutex errCh := make(chan error, len(commitIDs)) // Fill in the data for each commit in it's own Go routine. var wg sync.WaitGroup for i, cid := range commitIDs { wg.Add(1) go func(i int, cid *CommitID) { defer wg.Done() // Load the values for the commit. getValuesRequest := &traceservice.GetValuesRequest{ Commitid: tsCommitID(cid), } getValuesResponse, err := ts.traceService.GetValues(ctx, getValuesRequest) if err != nil { errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err) return } for _, pair := range getValuesResponse.Values { if pair == nil { glog.Errorf("Got a nil ValuePair in response: %s", err) continue } tr, ok := tile.Traces[pair.Key] if !ok { tileMutex.Lock() tile.Traces[pair.Key] = ts.traceBuilder(n) tileMutex.Unlock() tr = tile.Traces[pair.Key] } if err := tr.SetAt(i, pair.Value); err != nil { errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err) return } } }(i, cid) } wg.Wait() // See if any Go routine generated an error. select { case err, ok := <-errCh: if ok { return nil, fmt.Errorf("Failed to load trace data: %s", err) } default: } glog.Infof("Finished loading values. Starting to load Params.") // Now load the params for the traces. traceids := []string{} for k, _ := range tile.Traces { traceids = append(traceids, k) } tracechunks := [][]string{} for len(traceids) > 0 { if len(traceids) > CHUNK_SIZE { tracechunks = append(tracechunks, traceids[:CHUNK_SIZE]) traceids = traceids[CHUNK_SIZE:] } else { tracechunks = append(tracechunks, traceids) traceids = []string{} } } errCh = make(chan error, len(tracechunks)) for _, chunk := range tracechunks { wg.Add(1) go func(chunk []string) { defer wg.Done() req := &traceservice.GetParamsRequest{ Traceids: chunk, } resp, err := ts.traceService.GetParams(ctx, req) if err != nil { errCh <- fmt.Errorf("Failed to load params: %s", err) return } for _, param := range resp.Params { dst := tile.Traces[param.Key].Params() for k, v := range param.Params { dst[k] = v } } }(chunk) } wg.Wait() // See if any Go routine generated an error. select { case err, ok := <-errCh: if ok { return nil, fmt.Errorf("Failed to load params: %s", err) } default: } // Rebuild the ParamSet. glog.Infof("Finished loading params. Starting to rebuild ParamSet.") tiling.GetParamSet(tile.Traces, tile.ParamSet) return tile, nil }
// TileFromCommits implements DB.TileFromCommits(). func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, []string, error) { ts.clearMutex.RLock() ts.clearMutex.RUnlock() ctx := context.Background() // Build the Tile. tile := tiling.NewTile() n := len(commitIDs) tile.Commits = make([]*tiling.Commit, n, n) hash := make([]string, n) // Populate the Tile's commits. for i, cid := range commitIDs { tile.Commits[i] = &tiling.Commit{ Hash: cid.ID, CommitTime: cid.Timestamp, } } // tileMutex protects access to the Tile. Note that this only means the Tile, // while writing values into a Trace that already exists and is the right // size is Go routine safe. var tileMutex sync.Mutex errCh := make(chan error, len(commitIDs)) // Fill in the data for each commit in it's own Go routine. var wg sync.WaitGroup for i, cid := range commitIDs { wg.Add(1) go func(i int, cid *CommitID) { defer wg.Done() // Load the values for the commit. getValuesRequest := &traceservice.GetValuesRequest{ Commitid: tsCommitID(cid), } getRawValues, err := ts.traceService.GetValuesRaw(ctx, getValuesRequest) if err != nil { errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err) return } // Convert raw response into values. ci, err := traceservice.NewCommitInfo(getRawValues.Value) if err != nil { errCh <- fmt.Errorf("Failed to convert values for %d %#v: %s", i, *cid, err) return } // Now make sure we have all the traceids for the trace64ids in ci. missingKeys64 := []uint64{} ts.mutex.Lock() for id64, _ := range ci.Values { if _, ok := ts.id64Cache[id64]; !ok { missingKeys64 = append(missingKeys64, id64) } } ts.mutex.Unlock() if len(missingKeys64) > 0 { traceidsRequest := &traceservice.GetTraceIDsRequest{ Id: missingKeys64, } traceids, err := ts.traceService.GetTraceIDs(ctx, traceidsRequest) if err != nil { errCh <- fmt.Errorf("Failed to get traceids for trace64ids for %d %#v: %s", i, *cid, err) return } ts.mutex.Lock() for _, tid := range traceids.Ids { ts.id64Cache[tid.Id64] = tid.Id } ts.mutex.Unlock() } ts.mutex.Lock() for id64, rawValue := range ci.Values { if rawValue == nil { glog.Errorf("Got a nil rawValue in response: %s", err) continue } traceid := ts.id64Cache[id64] tileMutex.Lock() tr, ok := tile.Traces[traceid] if !ok || tr == nil { tile.Traces[traceid] = ts.traceBuilder(n) tr = tile.Traces[traceid] } tileMutex.Unlock() if tr == nil { glog.Errorf("Trace was still nil for key: %v", traceid) continue } if err := tr.SetAt(i, rawValue); err != nil { errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err) return } } // Fill in the commits hash. hash[i] = getRawValues.Md5 ts.mutex.Unlock() }(i, cid) } wg.Wait() // See if any Go routine generated an error. select { case err, ok := <-errCh: if ok { return nil, nil, fmt.Errorf("Failed to load trace data: %s", err) } default: } glog.Infof("Finished loading values. Starting to load Params.") // Now load the params for the traces. traceids := []string{} ts.mutex.Lock() for k, _ := range tile.Traces { // Only load params for traces not already in the cache. if _, ok := ts.paramsCache[k]; !ok { traceids = append(traceids, k) } } ts.mutex.Unlock() // Break the loading of params into chunks and make those requests concurrently. // The params are just loaded into the paramsCache. tracechunks := [][]string{} for len(traceids) > 0 { if len(traceids) > CHUNK_SIZE { tracechunks = append(tracechunks, traceids[:CHUNK_SIZE]) traceids = traceids[CHUNK_SIZE:] } else { tracechunks = append(tracechunks, traceids) traceids = []string{} } } errCh = make(chan error, len(tracechunks)) for _, chunk := range tracechunks { wg.Add(1) go func(chunk []string) { defer wg.Done() req := &traceservice.GetParamsRequest{ Traceids: chunk, } resp, err := ts.traceService.GetParams(ctx, req) if err != nil { errCh <- fmt.Errorf("Failed to load params: %s", err) return } for _, param := range resp.Params { ts.mutex.Lock() ts.paramsCache[param.Key] = param.Params ts.mutex.Unlock() } }(chunk) } wg.Wait() // See if any Go routine generated an error. select { case err, ok := <-errCh: if ok { return nil, nil, fmt.Errorf("Failed to load params: %s", err) } default: } // Add all params from the cache. ts.mutex.Lock() for k, tr := range tile.Traces { p := tr.Params() for pk, pv := range ts.paramsCache[k] { p[pk] = pv } } ts.mutex.Unlock() // Rebuild the ParamSet. glog.Infof("Finished loading params. Starting to rebuild ParamSet.") tiling.GetParamSet(tile.Traces, tile.ParamSet) return tile, hash, nil }