// Move changes the current Tile to the one that contains the given Git hash.
func (tt *TileTracker) Move(hash string) error {
if _, ok := tt.hashToNumber[hash]; !ok {
return fmt.Errorf("Commit does not exist in table: %s", hash)
}
hashNumber := tt.hashToNumber[hash]
tileNum := hashNumber / tiling.TILE_SIZE
if tileNum != tt.lastTileNum {
glog.Infof("Moving from tile %d to %d", tt.lastTileNum, tileNum)
if tt.lastTileNum != -1 {
if err := tt.tileStore.Put(0, tt.lastTileNum, tt.currentTile); err != nil {
return fmt.Errorf("TileTracker.Move() failed to flush old tile: %s", err)
}
}
tt.lastTileNum = tileNum
var err error
tt.currentTile, err = tt.tileStore.GetModifiable(0, tileNum)
if err != nil {
return fmt.Errorf("UpdateCommitInfo: Failed to get modifiable tile %d: %s", tileNum, err)
}
if tt.currentTile == nil {
tt.currentTile = tiling.NewTile()
tt.currentTile.Scale = 0
tt.currentTile.TileIndex = tileNum
}
}
return nil
}
// GetModifiable returns a tile from disk.
// This ensures the tile can be modified without affecting the cache.
// NOTE: Currently relies on getLastTile returning a new copy in all cases.
func (store *FileTileStore) GetModifiable(scale, index int) (*tiling.Tile, error) {
store.lock.Lock()
defer store.lock.Unlock()
// -1 means find the last tile for the given scale.
if index == -1 {
return store.getLastTile(scale)
}
filename, err := store.tileFilename(scale, index)
if err != nil {
return nil, fmt.Errorf("Unable to create a file name for the tile %d, %d: %s\n", scale, index, err)
}
_, err = os.Stat(filename)
// File probably isn't there, so return a new tile.
if err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("Tile %d,%d retrieval caused error : %s.", scale, index, err)
} else {
newTile := tiling.NewTile()
newTile.Scale = scale
newTile.TileIndex = index
return newTile, nil
}
}
t, err := openTile(filename)
if err != nil {
return nil, fmt.Errorf("Failed to retrieve tile %s: %s", filename, err)
}
return t, nil
}
// UpdateCommitInfo finds all the new commits since the last time we ran and
// adds them to the tiles, creating new tiles if necessary.
func (i *Ingester) UpdateCommitInfo(pull bool) error {
glog.Infof("Ingest %s: Starting UpdateCommitInfo", i.datasetName)
if err := i.git.Update(pull, false); err != nil {
return fmt.Errorf("Ingest %s: Failed git pull for during UpdateCommitInfo: %s", i.datasetName, err)
}
// Compute Git CL number for each Git hash.
allHashes := i.git.From(time.Time(BEGINNING_OF_TIME))
hashToNumber := map[string]int{}
for i, h := range allHashes {
hashToNumber[h] = i
}
i.hashToNumber = hashToNumber
// Find the time of the last Commit seen.
ts := time.Time(BEGINNING_OF_TIME)
lastTile, err := i.tileStore.Get(0, -1)
if err == nil && lastTile != nil {
ts = i.lastCommitTimeInTile(lastTile)
} else {
// Boundary condition; just started making Tiles and none exist.
newTile := tiling.NewTile()
newTile.Scale = 0
newTile.TileIndex = 0
if err := i.tileStore.Put(0, 0, newTile); err != nil {
return fmt.Errorf("Ingest %s: UpdateCommitInfo: Failed to write new tile: %s", i.datasetName, err)
}
}
glog.Infof("Ingest %s: UpdateCommitInfo: Last commit timestamp: %s", i.datasetName, ts)
// Find all the Git hashes that are new to us.
newHashes := i.git.From(ts)
glog.Infof("Ingest %s: len(newHashes): from %d", i.datasetName, len(newHashes))
// Add Commit info to the Tiles for each new hash.
tt := NewTileTracker(i.tileStore, i.hashToNumber)
for _, hash := range newHashes {
if err := tt.Move(hash); err != nil {
glog.Errorf("UpdateCommitInfo Move(%s) failed with: %s", hash, err)
continue
}
details, err := i.git.Details(hash, true)
if err != nil {
glog.Errorf("Failed to get details for hash: %s: %s", hash, err)
continue
}
tt.Tile().Commits[tt.Offset(hash)] = &tiling.Commit{
CommitTime: details.Timestamp.Unix(),
Hash: hash,
Author: details.Author,
}
}
glog.Infof("Ingest %s: Starting to flush tile.", i.datasetName)
tt.Flush()
glog.Infof("Ingest %s: Finished UpdateCommitInfo", i.datasetName)
return nil
}
func TestTallyBasic(t *testing.T) {
// Create a tile to test against.
tile := tiling.NewTile()
trace1 := types.NewGoldenTrace()
trace1.Values[0] = "aaa"
trace1.Values[1] = "aaa"
trace1.Values[2] = "bbb"
trace1.Params_[types.PRIMARY_KEY_FIELD] = "foo"
trace1.Params_["corpus"] = "gm"
tile.Traces["foo:x86"] = trace1
trace2 := types.NewGoldenTrace()
trace2.Values[0] = "ccc"
trace2.Values[1] = "aaa"
trace2.Params_[types.PRIMARY_KEY_FIELD] = "foo"
trace2.Params_["corpus"] = "image"
tile.Traces["foo:x86_64"] = trace2
// Test tallyTile with our Tile.
trace, test := tallyTile(tile)
if got, want := len(trace), 2; got != want {
t.Errorf("Wrong trace count: Got %v Want %v", got, want)
}
if got, want := (trace["foo:x86"])["aaa"], 2; got != want {
t.Errorf("Miscount: Got %v Want %v", got, want)
}
if got, want := len(test), 1; got != want {
t.Errorf("Wrong trace count: Got %v Want %v", got, want)
}
if got, want := (test["foo"])["aaa"], 3; got != want {
t.Errorf("Miscount: Got %v Want %v", got, want)
}
if got, want := (test["foo"])["bbb"], 1; got != want {
t.Errorf("Miscount: Got %v Want %v", got, want)
}
if got, want := (test["foo"])["ccc"], 1; got != want {
t.Errorf("Miscount: Got %v Want %v", got, want)
}
// Test tallyBy with our Tile.
ta := tallyBy(tile, trace, url.Values{"corpus": []string{"gm"}})
if got, want := len(ta), 2; got != want {
t.Errorf("Wrong trace count: Got %v Want %v", got, want)
}
if got, want := ta["aaa"], 2; got != want {
t.Errorf("Miscount: Got %v Want %v", got, want)
}
if got, want := ta["bbb"], 1; got != want {
t.Errorf("Miscount: Got %v Want %v", got, want)
}
}
// openTile opens the tile file passed in and returns the decoded contents.
func openTile(filename string) (*tiling.Tile, error) {
f, err := os.Open(filename)
if err != nil {
return nil, fmt.Errorf("Failed to open tile %s for reading: %s", filename, err)
}
defer util.Close(f)
t := tiling.NewTile()
dec := gob.NewDecoder(f)
if err := dec.Decode(t); err != nil {
return nil, fmt.Errorf("Failed to decode tile %s: %s", filename, err)
}
return t, nil
}
func TestTileTrim(t *testing.T) {
t1 := tiling.NewTile()
t1.Scale = 1
t1.TileIndex = 1
t1.Commits[len(t1.Commits)-2].Hash = "hash0"
t1.Commits[len(t1.Commits)-1].Hash = "hash1"
tr := NewPerfTrace()
tr.Values[0] = 0.5
tr.Values[1] = 0.6
tr.Values[2] = 0.7
t1.Traces["bar"] = tr
t2, err := t1.Trim(len(t1.Commits)-2, len(t1.Commits))
if err != nil {
t.Errorf("Failed to trim: %s", err)
}
if got, want := len(t2.Commits), 2; got != want {
t.Errorf("Trimmed tile length wrong: Got %v Want %v", got, want)
}
if got, want := len(t2.Traces["bar"].(*PerfTrace).Values), 2; got != want {
t.Errorf("Failed to trim traces: Got %v Want %v", got, want)
}
if got, want := t2.Commits[0].Hash, "hash0"; got != want {
t.Errorf("Failed to copy commit over: Got %v Want %v", got, want)
}
if got, want := t2.Commits[1].Hash, "hash1"; got != want {
t.Errorf("Failed to copy commit over: Got %v Want %v", got, want)
}
// Test error conditions.
t2, err = t1.Trim(1, 0)
if err == nil {
t.Errorf("Failed to raise error on Trim(1, 0).")
}
t2, err = t1.Trim(-1, 1)
if err == nil {
t.Errorf("Failed to raise error on Trim(-1, 1).")
}
t2, err = t1.Trim(-1, 1)
if err == nil {
t.Errorf("Failed to raise error on Trim(-1, 1).")
}
t2, err = t1.Trim(0, tiling.TILE_SIZE+1)
if err == nil {
t.Errorf("Failed to raise error on Trim(0, config.TILE_SIZE+1).")
}
}
func newTestContext() *Context {
tile := tiling.NewTile()
t1 := types.NewPerfTraceN(3)
t1.Params_["os"] = "Ubuntu12"
t1.Params_["config"] = "8888"
t1.Values[1] = 1.234
tile.Traces["t1"] = t1
t2 := types.NewPerfTraceN(3)
t2.Params_["os"] = "Ubuntu12"
t2.Params_["config"] = "gpu"
t2.Values[1] = 1.236
tile.Traces["t2"] = t2
return NewContext(tile)
}
// Mock the tilestore for GoldenTraces
func NewMockTileStore(t assert.TestingT, digests [][]string, params []map[string]string, commits []*tiling.Commit) tiling.TileStore {
// Build the tile from the digests, params and commits.
traces := map[string]tiling.Trace{}
for idx, traceDigests := range digests {
traces[TraceKey(params[idx])] = &types.GoldenTrace{
Params_: params[idx],
Values: traceDigests,
}
}
tile := tiling.NewTile()
tile.Traces = traces
tile.Commits = commits
return &MockTileStore{
t: t,
tile: tile,
}
}
func TestTrimTileFunc(t *testing.T) {
t1 := tiling.NewTile()
t1.Scale = 1
t1.TileIndex = 1
now := time.Now().Unix()
// Pretend this is a full tile by setting non-zero CommitTime's for all commits.
for i, _ := range t1.Commits {
t1.Commits[i].CommitTime = now
}
tr := types.NewPerfTrace()
tr.Values[len(t1.Commits)-1] = 0.7
t1.Traces["bar"] = tr
var err error
t1, err = trimTile(t1)
if err != nil {
t.Errorf("Failed to trim Tile: %s", err)
}
if got, want := len(t1.Commits), config.MAX_CLUSTER_COMMITS; got != want {
t.Errorf("Failed to trim Tile correctly: Got %v Want %v", got, want)
}
if got, want := t1.Traces["bar"].Len(), config.MAX_CLUSTER_COMMITS; got != want {
t.Errorf("Failed to trim Tile Values correctly: Got %v Want %v", got, want)
}
// Now test trimming a Tile with less than config.MAX_CLUSTER_COMMITS commits.
const N = 20
t1, err = t1.Trim(0, N)
if err != nil {
t.Errorf("Failed to trim Tile a second time: %s", err)
}
t1, err = trimTile(t1)
if got, want := len(t1.Commits), N; got != want {
t.Errorf("Failed to trim Tile correctly: Got %v Want %v", got, want)
}
if got, want := t1.Traces["bar"].Len(), N; got != want {
t.Errorf("Failed to trim Tile Values correctly: Got %v Want %v", got, want)
}
}
func TestParsing(t *testing.T) {
tile := tiling.NewTile()
offset := 1
dm := loadDMResults(t)
metricsProcessed := metrics.NewRegisteredCounter("testing.ingestion.processed", metrics.DefaultRegistry)
addResultToTile(dm, tile, offset, metricsProcessed)
if got, want := len(tile.Traces), 2; got != want {
t.Errorf("Wrong number of Traces: Got %v Want %v", got, want)
}
tr := tile.Traces["x86_64:565:Debug:HD7770:ShuttleA:varied_text_clipped_no_lcd:Win8"].(*types.GoldenTrace)
if got, want := tr.Values[1], "445aa63b2200baaba9b37fd5f80c0447"; got != want {
t.Errorf("Digest wrong: Got %v Want %v", got, want)
}
if got, want := len(tr.Params()), 9; got != want {
t.Errorf("Params wrong: Got %v Want %v", got, want)
}
if got, want := tr.Params()["ext"], "png"; got != want {
t.Errorf("Extension not injected:: Got %v Want %v", got, want)
}
if got, want := int64(2), metricsProcessed.Count(); got != want {
t.Errorf("Wrong number of points ingested: Got %v Want %v", got, want)
}
}
func TestMerge(t *testing.T) {
t1 := tiling.NewTile()
t1.Scale = 1
t1.TileIndex = 20
t1.Commits[1].Hash = "hash1"
t2 := tiling.NewTile()
t2.Scale = 1
t2.TileIndex = 21
t2.Commits[1].Hash = "hash33"
t2.Commits[2].Hash = "hash34"
t3 := tiling.NewTile()
t2.Scale = 1
t2.TileIndex = 22
t2.Commits[1].Hash = "hash43"
t2.Commits[2].Hash = "hash44"
// Create a Trace that exists in both tile1 and tile2.
tr := NewPerfTrace()
tr.Params_["p1"] = "v1"
tr.Params_["p2"] = "v2"
tr.Values[0] = 0.1
tr.Values[1] = 0.2
t1.Traces["foo"] = tr
tr = NewPerfTrace()
tr.Params_["p1"] = "v1"
tr.Params_["p2"] = "v2"
tr.Params_["p5"] = "5"
tr.Values[0] = 0.3
tr.Values[1] = 0.4
t2.Traces["foo"] = tr
// Add a trace that only appears in tile2.
tr = NewPerfTrace()
tr.Params_["p1"] = "v1"
tr.Params_["p3"] = "v3"
tr.Values[0] = 0.5
tr.Values[1] = 0.6
t2.Traces["bar"] = tr
// Merge the two tiles.
merged := tiling.Merge(tiling.Merge(t1, t2), t3)
if got, want := len(merged.Traces["foo"].(*PerfTrace).Values), 3*tiling.TILE_SIZE; got != want {
t.Errorf("Wrong config.TILE_SIZE: Got %v Want %v", got, want)
}
if got, want := merged.Scale, 1; got != want {
t.Errorf("Wrong scale: Got %v Want %v", got, want)
}
if got, want := merged.TileIndex, t1.TileIndex; got != want {
t.Errorf("TileIndex is wrong: Got %v Want %v", got, want)
}
if got, want := len(merged.Traces), 2; got != want {
t.Errorf("Number of traces: Got %v Want %v", got, want)
}
if got, want := len(merged.Traces["foo"].(*PerfTrace).Values), 3*tiling.TILE_SIZE; got != want {
t.Errorf("Number of values: Got %v Want %v", got, want)
}
if got, want := len(merged.ParamSet), 4; got != want {
t.Errorf("ParamSet length: Got %v Want %v", got, want)
}
if _, ok := merged.ParamSet["p5"]; !ok {
t.Errorf("Merged tile missing 'p5' param.")
}
// Test the "foo" trace.
tr = merged.Traces["foo"].(*PerfTrace)
testCases := []struct {
N int
V float64
}{
{127, 1e100},
{50, 0.3},
{51, 0.4},
{130, 1e100},
{0, 0.1},
{1, 0.2},
{2, 1e100},
}
for _, tc := range testCases {
if got, want := tr.Values[tc.N], tc.V; got != want {
t.Errorf("Error copying trace values: Got %v Want %v at %d", got, want, tc.N)
}
}
if got, want := tr.Params()["p1"], "v1"; got != want {
t.Errorf("Wrong params for trace: Got %v Want %v", got, want)
}
// Test the "bar" trace.
tr = merged.Traces["bar"].(*PerfTrace)
testCases = []struct {
N int
V float64
}{
{127, 1e100},
{50, 0.5},
{51, 0.6},
{130, 1e100},
}
for _, tc := range testCases {
if got, want := tr.Values[tc.N], tc.V; got != want {
t.Errorf("Error copying trace values: Got %v Want %v at %d", got, want, tc.N)
}
}
if got, want := tr.Params()["p3"], "v3"; got != want {
t.Errorf("Wrong params for trace: Got %v Want %v", got, want)
}
}
func TestAddBenchDataToTile(t *testing.T) {
// Load the sample data file as BenchData.
_, filename, _, _ := runtime.Caller(0)
r, err := os.Open(filepath.Join(filepath.Dir(filename), "testdata", "nano.json"))
if err != nil {
t.Fatal("Failed to open test file: ", err)
}
benchData, err := ParseBenchDataFromReader(r)
if err != nil {
t.Fatal("Failed to parse test file: ", err)
}
metricsProcessed := metrics.NewRegisteredCounter("testing.ingestion.processed", metrics.DefaultRegistry)
// Create an empty Tile.
tile := tiling.NewTile()
tile.Scale = 0
tile.TileIndex = 0
offset := 1
testcases := []struct {
key string
value float64
subResult string
}{
{
key: "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_discardable_640_480:gpu",
value: 0.1157132745098039,
subResult: "min_ms",
},
{
key: "x86:GTX660:ShuttleA:Ubuntu12:memory_usage_0_0:meta:max_rss_mb",
value: 858,
subResult: "max_rss_mb",
},
{
key: "x86:GTX660:ShuttleA:Ubuntu12:src_pipe_global_weak_symbol:memory:bytes",
value: 158,
subResult: "bytes",
},
{
key: "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_nonDiscardable_640_480:8888",
value: 2.855735,
subResult: "min_ms",
},
{
key: "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_nonDiscardable_640_480:8888:bytes",
value: 298888,
subResult: "bytes",
},
{
key: "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_nonDiscardable_640_480:8888:ops",
value: 3333,
subResult: "ops",
},
}
// Do everything twice to ensure that we are idempotent.
for i := 0; i < 2; i++ {
// Add the BenchData to the Tile.
addBenchDataToTile(benchData, tile, offset, metricsProcessed)
// Test that the Tile has the right data.
if got, want := len(tile.Traces), 13; got != want {
t.Errorf("Wrong number of traces: Got %d Want %d", got, want)
}
for _, tc := range testcases {
trace, ok := tile.Traces[tc.key]
if !ok {
t.Errorf("Missing expected key: %s", tc.key)
}
if got, want := trace.(*types.PerfTrace).Values[offset], tc.value; got != want {
t.Errorf("Wrong value in trace: Got %v Want %v", got, want)
}
}
trace := tile.Traces[testcases[0].key]
// Validate the traces Params.
expected := map[string]string{
"arch": "x86",
"gpu": "GTX660",
"model": "ShuttleA",
"os": "Ubuntu12",
"system": "UNIX",
"test": "DeferredSurfaceCopy_discardable_640_480",
"config": "gpu",
"GL_RENDERER": "GeForce GTX 660/PCIe/SSE2",
"GL_SHADING_LANGUAGE_VERSION": "4.40 NVIDIA via Cg compiler",
"GL_VENDOR": "NVIDIA Corporation",
"GL_VERSION": "4.4.0 NVIDIA 331.49",
"source_type": "bench",
"sub_result": "min_ms",
}
if got, want := len(trace.Params()), len(expected); got != want {
t.Errorf("Params wrong length: Got %v Want %v", got, want)
}
for k, v := range expected {
if got, want := trace.Params()[k], v; got != want {
t.Errorf("Wrong params: Got %v Want %v", got, want)
}
}
// Validate the Tiles ParamSet.
if got, want := len(tile.ParamSet), len(expected)+2; got != want {
t.Errorf("Wrong ParamSet length: Got %v Want %v", got, want)
}
for k, _ := range expected {
if _, ok := tile.ParamSet[k]; !ok {
t.Errorf("Missing from ParamSet: %s", k)
}
}
// The new symbol table size options values should also show up in the ParamSet.
for _, k := range []string{"path", "symbol"} {
if _, ok := tile.ParamSet[k]; !ok {
t.Errorf("Missing from ParamSet: %s", k)
}
}
if got, want := len(tile.ParamSet["source_type"]), 1; got != want {
t.Errorf("Wrong ParamSet for source_type: Got %v Want %v", got, want)
}
if got, want := tile.ParamSet["source_type"][0], "bench"; got != want {
t.Errorf("Wrong ParamSet value: Got %v Want %v", got, want)
}
}
if got, want := metricsProcessed.Count(), int64(26); got != want {
t.Errorf("Wrong number of points ingested: Got %v Want %v", got, want)
}
// Now update one of the params for a trace and reingest and confirm that the
// trace params get updated.
benchData.Options["system"] = "Linux"
addBenchDataToTile(benchData, tile, offset, metricsProcessed)
if got, want := "Linux", tile.Traces[testcases[0].key].Params()["system"]; got != want {
t.Errorf("Failed to update params: Got %v Want %v", got, want)
}
if got, want := metricsProcessed.Count(), int64(39); got != want {
t.Errorf("Wrong number of points ingested: Got %v Want %v", got, want)
}
}
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, error) {
ctx := context.Background()
// Build the Tile.
tile := tiling.NewTile()
n := len(commitIDs)
tile.Commits = make([]*tiling.Commit, n, n)
// Populate the Tile's commits.
for i, cid := range commitIDs {
tile.Commits[i] = &tiling.Commit{
Hash: cid.ID,
CommitTime: cid.Timestamp.Unix(),
}
}
// tileMutex protects access to the Tile. Note that this only means the Tile,
// while writing values into a Trace that already exists and is the right
// size is Go routine safe.
var tileMutex sync.Mutex
errCh := make(chan error, len(commitIDs))
// Fill in the data for each commit in it's own Go routine.
var wg sync.WaitGroup
for i, cid := range commitIDs {
wg.Add(1)
go func(i int, cid *CommitID) {
defer wg.Done()
// Load the values for the commit.
getValuesRequest := &traceservice.GetValuesRequest{
Commitid: tsCommitID(cid),
}
getValuesResponse, err := ts.traceService.GetValues(ctx, getValuesRequest)
if err != nil {
errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
return
}
for _, pair := range getValuesResponse.Values {
if pair == nil {
glog.Errorf("Got a nil ValuePair in response: %s", err)
continue
}
tr, ok := tile.Traces[pair.Key]
if !ok {
tileMutex.Lock()
tile.Traces[pair.Key] = ts.traceBuilder(n)
tileMutex.Unlock()
tr = tile.Traces[pair.Key]
}
if err := tr.SetAt(i, pair.Value); err != nil {
errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
return
}
}
}(i, cid)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, fmt.Errorf("Failed to load trace data: %s", err)
}
default:
}
glog.Infof("Finished loading values. Starting to load Params.")
// Now load the params for the traces.
traceids := []string{}
for k, _ := range tile.Traces {
traceids = append(traceids, k)
}
tracechunks := [][]string{}
for len(traceids) > 0 {
if len(traceids) > CHUNK_SIZE {
tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
traceids = traceids[CHUNK_SIZE:]
} else {
tracechunks = append(tracechunks, traceids)
traceids = []string{}
}
}
errCh = make(chan error, len(tracechunks))
for _, chunk := range tracechunks {
wg.Add(1)
go func(chunk []string) {
defer wg.Done()
req := &traceservice.GetParamsRequest{
Traceids: chunk,
}
resp, err := ts.traceService.GetParams(ctx, req)
if err != nil {
errCh <- fmt.Errorf("Failed to load params: %s", err)
return
}
for _, param := range resp.Params {
dst := tile.Traces[param.Key].Params()
for k, v := range param.Params {
dst[k] = v
}
}
}(chunk)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, fmt.Errorf("Failed to load params: %s", err)
}
default:
}
// Rebuild the ParamSet.
glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
tiling.GetParamSet(tile.Traces, tile.ParamSet)
return tile, nil
}
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, []string, error) {
ts.clearMutex.RLock()
ts.clearMutex.RUnlock()
ctx := context.Background()
// Build the Tile.
tile := tiling.NewTile()
n := len(commitIDs)
tile.Commits = make([]*tiling.Commit, n, n)
hash := make([]string, n)
// Populate the Tile's commits.
for i, cid := range commitIDs {
tile.Commits[i] = &tiling.Commit{
Hash: cid.ID,
CommitTime: cid.Timestamp,
}
}
// tileMutex protects access to the Tile. Note that this only means the Tile,
// while writing values into a Trace that already exists and is the right
// size is Go routine safe.
var tileMutex sync.Mutex
errCh := make(chan error, len(commitIDs))
// Fill in the data for each commit in it's own Go routine.
var wg sync.WaitGroup
for i, cid := range commitIDs {
wg.Add(1)
go func(i int, cid *CommitID) {
defer wg.Done()
// Load the values for the commit.
getValuesRequest := &traceservice.GetValuesRequest{
Commitid: tsCommitID(cid),
}
getRawValues, err := ts.traceService.GetValuesRaw(ctx, getValuesRequest)
if err != nil {
errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
return
}
// Convert raw response into values.
ci, err := traceservice.NewCommitInfo(getRawValues.Value)
if err != nil {
errCh <- fmt.Errorf("Failed to convert values for %d %#v: %s", i, *cid, err)
return
}
// Now make sure we have all the traceids for the trace64ids in ci.
missingKeys64 := []uint64{}
ts.mutex.Lock()
for id64, _ := range ci.Values {
if _, ok := ts.id64Cache[id64]; !ok {
missingKeys64 = append(missingKeys64, id64)
}
}
ts.mutex.Unlock()
if len(missingKeys64) > 0 {
traceidsRequest := &traceservice.GetTraceIDsRequest{
Id: missingKeys64,
}
traceids, err := ts.traceService.GetTraceIDs(ctx, traceidsRequest)
if err != nil {
errCh <- fmt.Errorf("Failed to get traceids for trace64ids for %d %#v: %s", i, *cid, err)
return
}
ts.mutex.Lock()
for _, tid := range traceids.Ids {
ts.id64Cache[tid.Id64] = tid.Id
}
ts.mutex.Unlock()
}
ts.mutex.Lock()
for id64, rawValue := range ci.Values {
if rawValue == nil {
glog.Errorf("Got a nil rawValue in response: %s", err)
continue
}
traceid := ts.id64Cache[id64]
tileMutex.Lock()
tr, ok := tile.Traces[traceid]
if !ok || tr == nil {
tile.Traces[traceid] = ts.traceBuilder(n)
tr = tile.Traces[traceid]
}
tileMutex.Unlock()
if tr == nil {
glog.Errorf("Trace was still nil for key: %v", traceid)
continue
}
if err := tr.SetAt(i, rawValue); err != nil {
errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
return
}
}
// Fill in the commits hash.
hash[i] = getRawValues.Md5
ts.mutex.Unlock()
}(i, cid)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, nil, fmt.Errorf("Failed to load trace data: %s", err)
}
default:
}
glog.Infof("Finished loading values. Starting to load Params.")
// Now load the params for the traces.
traceids := []string{}
ts.mutex.Lock()
for k, _ := range tile.Traces {
// Only load params for traces not already in the cache.
if _, ok := ts.paramsCache[k]; !ok {
traceids = append(traceids, k)
}
}
ts.mutex.Unlock()
// Break the loading of params into chunks and make those requests concurrently.
// The params are just loaded into the paramsCache.
tracechunks := [][]string{}
for len(traceids) > 0 {
if len(traceids) > CHUNK_SIZE {
tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
traceids = traceids[CHUNK_SIZE:]
} else {
tracechunks = append(tracechunks, traceids)
traceids = []string{}
}
}
errCh = make(chan error, len(tracechunks))
for _, chunk := range tracechunks {
wg.Add(1)
go func(chunk []string) {
defer wg.Done()
req := &traceservice.GetParamsRequest{
Traceids: chunk,
}
resp, err := ts.traceService.GetParams(ctx, req)
if err != nil {
errCh <- fmt.Errorf("Failed to load params: %s", err)
return
}
for _, param := range resp.Params {
ts.mutex.Lock()
ts.paramsCache[param.Key] = param.Params
ts.mutex.Unlock()
}
}(chunk)
}
wg.Wait()
// See if any Go routine generated an error.
select {
case err, ok := <-errCh:
if ok {
return nil, nil, fmt.Errorf("Failed to load params: %s", err)
}
default:
}
// Add all params from the cache.
ts.mutex.Lock()
for k, tr := range tile.Traces {
p := tr.Params()
for pk, pv := range ts.paramsCache[k] {
p[pk] = pv
}
}
ts.mutex.Unlock()
// Rebuild the ParamSet.
glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
tiling.GetParamSet(tile.Traces, tile.ParamSet)
return tile, hash, nil
}