コード例 #1
0
ファイル: ingester.go プロジェクト: saltmueller/skia-buildbot
// Move changes the current Tile to the one that contains the given Git hash.
func (tt *TileTracker) Move(hash string) error {
	if _, ok := tt.hashToNumber[hash]; !ok {
		return fmt.Errorf("Commit does not exist in table: %s", hash)
	}
	hashNumber := tt.hashToNumber[hash]
	tileNum := hashNumber / tiling.TILE_SIZE
	if tileNum != tt.lastTileNum {
		glog.Infof("Moving from tile %d to %d", tt.lastTileNum, tileNum)
		if tt.lastTileNum != -1 {
			if err := tt.tileStore.Put(0, tt.lastTileNum, tt.currentTile); err != nil {
				return fmt.Errorf("TileTracker.Move() failed to flush old tile: %s", err)
			}
		}
		tt.lastTileNum = tileNum
		var err error
		tt.currentTile, err = tt.tileStore.GetModifiable(0, tileNum)
		if err != nil {
			return fmt.Errorf("UpdateCommitInfo: Failed to get modifiable tile %d: %s", tileNum, err)
		}
		if tt.currentTile == nil {
			tt.currentTile = tiling.NewTile()
			tt.currentTile.Scale = 0
			tt.currentTile.TileIndex = tileNum
		}
	}
	return nil
}
コード例 #2
0
// GetModifiable returns a tile from disk.
// This ensures the tile can be modified without affecting the cache.
// NOTE: Currently relies on getLastTile returning a new copy in all cases.
func (store *FileTileStore) GetModifiable(scale, index int) (*tiling.Tile, error) {
	store.lock.Lock()
	defer store.lock.Unlock()
	// -1 means find the last tile for the given scale.
	if index == -1 {
		return store.getLastTile(scale)
	}
	filename, err := store.tileFilename(scale, index)
	if err != nil {
		return nil, fmt.Errorf("Unable to create a file name for the tile %d, %d: %s\n", scale, index, err)
	}
	_, err = os.Stat(filename)
	// File probably isn't there, so return a new tile.
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, fmt.Errorf("Tile %d,%d retrieval caused error : %s.", scale, index, err)
		} else {
			newTile := tiling.NewTile()
			newTile.Scale = scale
			newTile.TileIndex = index
			return newTile, nil
		}
	}
	t, err := openTile(filename)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve tile %s: %s", filename, err)
	}
	return t, nil
}
コード例 #3
0
ファイル: ingester.go プロジェクト: saltmueller/skia-buildbot
// UpdateCommitInfo finds all the new commits since the last time we ran and
// adds them to the tiles, creating new tiles if necessary.
func (i *Ingester) UpdateCommitInfo(pull bool) error {
	glog.Infof("Ingest %s: Starting UpdateCommitInfo", i.datasetName)
	if err := i.git.Update(pull, false); err != nil {
		return fmt.Errorf("Ingest %s: Failed git pull for during UpdateCommitInfo: %s", i.datasetName, err)
	}

	// Compute Git CL number for each Git hash.
	allHashes := i.git.From(time.Time(BEGINNING_OF_TIME))
	hashToNumber := map[string]int{}
	for i, h := range allHashes {
		hashToNumber[h] = i
	}
	i.hashToNumber = hashToNumber

	// Find the time of the last Commit seen.
	ts := time.Time(BEGINNING_OF_TIME)
	lastTile, err := i.tileStore.Get(0, -1)
	if err == nil && lastTile != nil {
		ts = i.lastCommitTimeInTile(lastTile)
	} else {
		// Boundary condition; just started making Tiles and none exist.
		newTile := tiling.NewTile()
		newTile.Scale = 0
		newTile.TileIndex = 0
		if err := i.tileStore.Put(0, 0, newTile); err != nil {
			return fmt.Errorf("Ingest %s: UpdateCommitInfo: Failed to write new tile: %s", i.datasetName, err)
		}
	}
	glog.Infof("Ingest %s: UpdateCommitInfo: Last commit timestamp: %s", i.datasetName, ts)

	// Find all the Git hashes that are new to us.
	newHashes := i.git.From(ts)

	glog.Infof("Ingest %s: len(newHashes): from %d", i.datasetName, len(newHashes))

	// Add Commit info to the Tiles for each new hash.
	tt := NewTileTracker(i.tileStore, i.hashToNumber)
	for _, hash := range newHashes {
		if err := tt.Move(hash); err != nil {
			glog.Errorf("UpdateCommitInfo Move(%s) failed with: %s", hash, err)
			continue
		}
		details, err := i.git.Details(hash, true)
		if err != nil {
			glog.Errorf("Failed to get details for hash: %s: %s", hash, err)
			continue
		}
		tt.Tile().Commits[tt.Offset(hash)] = &tiling.Commit{
			CommitTime: details.Timestamp.Unix(),
			Hash:       hash,
			Author:     details.Author,
		}
	}
	glog.Infof("Ingest %s: Starting to flush tile.", i.datasetName)
	tt.Flush()

	glog.Infof("Ingest %s: Finished UpdateCommitInfo", i.datasetName)
	return nil
}
コード例 #4
0
func TestTallyBasic(t *testing.T) {
	// Create a tile to test against.
	tile := tiling.NewTile()
	trace1 := types.NewGoldenTrace()
	trace1.Values[0] = "aaa"
	trace1.Values[1] = "aaa"
	trace1.Values[2] = "bbb"
	trace1.Params_[types.PRIMARY_KEY_FIELD] = "foo"
	trace1.Params_["corpus"] = "gm"
	tile.Traces["foo:x86"] = trace1

	trace2 := types.NewGoldenTrace()
	trace2.Values[0] = "ccc"
	trace2.Values[1] = "aaa"
	trace2.Params_[types.PRIMARY_KEY_FIELD] = "foo"
	trace2.Params_["corpus"] = "image"
	tile.Traces["foo:x86_64"] = trace2

	// Test tallyTile with our Tile.
	trace, test := tallyTile(tile)
	if got, want := len(trace), 2; got != want {
		t.Errorf("Wrong trace count: Got %v Want %v", got, want)
	}
	if got, want := (trace["foo:x86"])["aaa"], 2; got != want {
		t.Errorf("Miscount: Got %v Want %v", got, want)
	}

	if got, want := len(test), 1; got != want {
		t.Errorf("Wrong trace count: Got %v Want %v", got, want)
	}
	if got, want := (test["foo"])["aaa"], 3; got != want {
		t.Errorf("Miscount: Got %v Want %v", got, want)
	}
	if got, want := (test["foo"])["bbb"], 1; got != want {
		t.Errorf("Miscount: Got %v Want %v", got, want)
	}
	if got, want := (test["foo"])["ccc"], 1; got != want {
		t.Errorf("Miscount: Got %v Want %v", got, want)
	}

	// Test tallyBy with our Tile.
	ta := tallyBy(tile, trace, url.Values{"corpus": []string{"gm"}})
	if got, want := len(ta), 2; got != want {
		t.Errorf("Wrong trace count: Got %v Want %v", got, want)
	}
	if got, want := ta["aaa"], 2; got != want {
		t.Errorf("Miscount: Got %v Want %v", got, want)
	}
	if got, want := ta["bbb"], 1; got != want {
		t.Errorf("Miscount: Got %v Want %v", got, want)
	}
}
コード例 #5
0
// openTile opens the tile file passed in and returns the decoded contents.
func openTile(filename string) (*tiling.Tile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("Failed to open tile %s for reading: %s", filename, err)
	}
	defer util.Close(f)
	t := tiling.NewTile()
	dec := gob.NewDecoder(f)
	if err := dec.Decode(t); err != nil {
		return nil, fmt.Errorf("Failed to decode tile %s: %s", filename, err)
	}
	return t, nil
}
コード例 #6
0
func TestTileTrim(t *testing.T) {
	t1 := tiling.NewTile()
	t1.Scale = 1
	t1.TileIndex = 1
	t1.Commits[len(t1.Commits)-2].Hash = "hash0"
	t1.Commits[len(t1.Commits)-1].Hash = "hash1"

	tr := NewPerfTrace()
	tr.Values[0] = 0.5
	tr.Values[1] = 0.6
	tr.Values[2] = 0.7

	t1.Traces["bar"] = tr

	t2, err := t1.Trim(len(t1.Commits)-2, len(t1.Commits))
	if err != nil {
		t.Errorf("Failed to trim: %s", err)
	}
	if got, want := len(t2.Commits), 2; got != want {
		t.Errorf("Trimmed tile length wrong: Got %v Want %v", got, want)
	}
	if got, want := len(t2.Traces["bar"].(*PerfTrace).Values), 2; got != want {
		t.Errorf("Failed to trim traces: Got %v Want %v", got, want)
	}
	if got, want := t2.Commits[0].Hash, "hash0"; got != want {
		t.Errorf("Failed to copy commit over: Got %v Want %v", got, want)
	}
	if got, want := t2.Commits[1].Hash, "hash1"; got != want {
		t.Errorf("Failed to copy commit over: Got %v Want %v", got, want)
	}

	// Test error conditions.
	t2, err = t1.Trim(1, 0)
	if err == nil {
		t.Errorf("Failed to raise error on Trim(1, 0).")
	}
	t2, err = t1.Trim(-1, 1)
	if err == nil {
		t.Errorf("Failed to raise error on Trim(-1, 1).")
	}
	t2, err = t1.Trim(-1, 1)
	if err == nil {
		t.Errorf("Failed to raise error on Trim(-1, 1).")
	}
	t2, err = t1.Trim(0, tiling.TILE_SIZE+1)
	if err == nil {
		t.Errorf("Failed to raise error on Trim(0, config.TILE_SIZE+1).")
	}
}
コード例 #7
0
func newTestContext() *Context {
	tile := tiling.NewTile()
	t1 := types.NewPerfTraceN(3)
	t1.Params_["os"] = "Ubuntu12"
	t1.Params_["config"] = "8888"
	t1.Values[1] = 1.234
	tile.Traces["t1"] = t1

	t2 := types.NewPerfTraceN(3)
	t2.Params_["os"] = "Ubuntu12"
	t2.Params_["config"] = "gpu"
	t2.Values[1] = 1.236
	tile.Traces["t2"] = t2

	return NewContext(tile)
}
コード例 #8
0
ファイル: mocks.go プロジェクト: 1394/skia-buildbot
// Mock the tilestore for GoldenTraces
func NewMockTileStore(t assert.TestingT, digests [][]string, params []map[string]string, commits []*tiling.Commit) tiling.TileStore {
	// Build the tile from the digests, params and commits.
	traces := map[string]tiling.Trace{}

	for idx, traceDigests := range digests {
		traces[TraceKey(params[idx])] = &types.GoldenTrace{
			Params_: params[idx],
			Values:  traceDigests,
		}
	}

	tile := tiling.NewTile()
	tile.Traces = traces
	tile.Commits = commits

	return &MockTileStore{
		t:    t,
		tile: tile,
	}
}
コード例 #9
0
func TestTrimTileFunc(t *testing.T) {
	t1 := tiling.NewTile()
	t1.Scale = 1
	t1.TileIndex = 1
	now := time.Now().Unix()
	// Pretend this is a full tile by setting non-zero CommitTime's for all commits.
	for i, _ := range t1.Commits {
		t1.Commits[i].CommitTime = now
	}
	tr := types.NewPerfTrace()
	tr.Values[len(t1.Commits)-1] = 0.7
	t1.Traces["bar"] = tr

	var err error
	t1, err = trimTile(t1)
	if err != nil {
		t.Errorf("Failed to trim Tile: %s", err)
	}

	if got, want := len(t1.Commits), config.MAX_CLUSTER_COMMITS; got != want {
		t.Errorf("Failed to trim Tile correctly: Got %v Want %v", got, want)
	}
	if got, want := t1.Traces["bar"].Len(), config.MAX_CLUSTER_COMMITS; got != want {
		t.Errorf("Failed to trim Tile Values correctly: Got %v Want %v", got, want)
	}

	// Now test trimming a Tile with less than config.MAX_CLUSTER_COMMITS commits.
	const N = 20
	t1, err = t1.Trim(0, N)
	if err != nil {
		t.Errorf("Failed to trim Tile a second time: %s", err)
	}
	t1, err = trimTile(t1)
	if got, want := len(t1.Commits), N; got != want {
		t.Errorf("Failed to trim Tile correctly: Got %v Want %v", got, want)
	}
	if got, want := t1.Traces["bar"].Len(), N; got != want {
		t.Errorf("Failed to trim Tile Values correctly: Got %v Want %v", got, want)
	}
}
コード例 #10
0
func TestParsing(t *testing.T) {
	tile := tiling.NewTile()
	offset := 1
	dm := loadDMResults(t)

	metricsProcessed := metrics.NewRegisteredCounter("testing.ingestion.processed", metrics.DefaultRegistry)
	addResultToTile(dm, tile, offset, metricsProcessed)
	if got, want := len(tile.Traces), 2; got != want {
		t.Errorf("Wrong number of Traces: Got %v Want %v", got, want)
	}
	tr := tile.Traces["x86_64:565:Debug:HD7770:ShuttleA:varied_text_clipped_no_lcd:Win8"].(*types.GoldenTrace)
	if got, want := tr.Values[1], "445aa63b2200baaba9b37fd5f80c0447"; got != want {
		t.Errorf("Digest wrong: Got %v Want %v", got, want)
	}
	if got, want := len(tr.Params()), 9; got != want {
		t.Errorf("Params wrong: Got %v Want %v", got, want)
	}
	if got, want := tr.Params()["ext"], "png"; got != want {
		t.Errorf("Extension not injected:: Got %v Want %v", got, want)
	}
	if got, want := int64(2), metricsProcessed.Count(); got != want {
		t.Errorf("Wrong number of points ingested: Got %v Want %v", got, want)
	}
}
コード例 #11
0
func TestMerge(t *testing.T) {
	t1 := tiling.NewTile()
	t1.Scale = 1
	t1.TileIndex = 20
	t1.Commits[1].Hash = "hash1"

	t2 := tiling.NewTile()
	t2.Scale = 1
	t2.TileIndex = 21
	t2.Commits[1].Hash = "hash33"
	t2.Commits[2].Hash = "hash34"

	t3 := tiling.NewTile()
	t2.Scale = 1
	t2.TileIndex = 22
	t2.Commits[1].Hash = "hash43"
	t2.Commits[2].Hash = "hash44"

	// Create a Trace that exists in both tile1 and tile2.
	tr := NewPerfTrace()
	tr.Params_["p1"] = "v1"
	tr.Params_["p2"] = "v2"
	tr.Values[0] = 0.1
	tr.Values[1] = 0.2

	t1.Traces["foo"] = tr

	tr = NewPerfTrace()
	tr.Params_["p1"] = "v1"
	tr.Params_["p2"] = "v2"
	tr.Params_["p5"] = "5"
	tr.Values[0] = 0.3
	tr.Values[1] = 0.4

	t2.Traces["foo"] = tr

	// Add a trace that only appears in tile2.
	tr = NewPerfTrace()
	tr.Params_["p1"] = "v1"
	tr.Params_["p3"] = "v3"
	tr.Values[0] = 0.5
	tr.Values[1] = 0.6

	t2.Traces["bar"] = tr

	// Merge the two tiles.
	merged := tiling.Merge(tiling.Merge(t1, t2), t3)
	if got, want := len(merged.Traces["foo"].(*PerfTrace).Values), 3*tiling.TILE_SIZE; got != want {
		t.Errorf("Wrong config.TILE_SIZE: Got %v Want %v", got, want)
	}

	if got, want := merged.Scale, 1; got != want {
		t.Errorf("Wrong scale: Got %v Want %v", got, want)
	}
	if got, want := merged.TileIndex, t1.TileIndex; got != want {
		t.Errorf("TileIndex is wrong: Got %v Want %v", got, want)
	}
	if got, want := len(merged.Traces), 2; got != want {
		t.Errorf("Number of traces: Got %v Want %v", got, want)
	}
	if got, want := len(merged.Traces["foo"].(*PerfTrace).Values), 3*tiling.TILE_SIZE; got != want {
		t.Errorf("Number of values: Got %v Want %v", got, want)
	}
	if got, want := len(merged.ParamSet), 4; got != want {
		t.Errorf("ParamSet length: Got %v Want %v", got, want)
	}
	if _, ok := merged.ParamSet["p5"]; !ok {
		t.Errorf("Merged tile missing 'p5' param.")
	}

	// Test the "foo" trace.
	tr = merged.Traces["foo"].(*PerfTrace)
	testCases := []struct {
		N int
		V float64
	}{
		{127, 1e100},
		{50, 0.3},
		{51, 0.4},
		{130, 1e100},
		{0, 0.1},
		{1, 0.2},
		{2, 1e100},
	}
	for _, tc := range testCases {
		if got, want := tr.Values[tc.N], tc.V; got != want {
			t.Errorf("Error copying trace values: Got %v Want %v at %d", got, want, tc.N)
		}
	}
	if got, want := tr.Params()["p1"], "v1"; got != want {
		t.Errorf("Wrong params for trace: Got %v Want %v", got, want)
	}

	// Test the "bar" trace.
	tr = merged.Traces["bar"].(*PerfTrace)
	testCases = []struct {
		N int
		V float64
	}{
		{127, 1e100},
		{50, 0.5},
		{51, 0.6},
		{130, 1e100},
	}
	for _, tc := range testCases {
		if got, want := tr.Values[tc.N], tc.V; got != want {
			t.Errorf("Error copying trace values: Got %v Want %v at %d", got, want, tc.N)
		}
	}
	if got, want := tr.Params()["p3"], "v3"; got != want {
		t.Errorf("Wrong params for trace: Got %v Want %v", got, want)
	}
}
コード例 #12
0
func TestAddBenchDataToTile(t *testing.T) {
	// Load the sample data file as BenchData.
	_, filename, _, _ := runtime.Caller(0)
	r, err := os.Open(filepath.Join(filepath.Dir(filename), "testdata", "nano.json"))
	if err != nil {
		t.Fatal("Failed to open test file: ", err)
	}

	benchData, err := ParseBenchDataFromReader(r)
	if err != nil {
		t.Fatal("Failed to parse test file: ", err)
	}
	metricsProcessed := metrics.NewRegisteredCounter("testing.ingestion.processed", metrics.DefaultRegistry)

	// Create an empty Tile.
	tile := tiling.NewTile()
	tile.Scale = 0
	tile.TileIndex = 0

	offset := 1
	testcases := []struct {
		key       string
		value     float64
		subResult string
	}{
		{
			key:       "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_discardable_640_480:gpu",
			value:     0.1157132745098039,
			subResult: "min_ms",
		},
		{
			key:       "x86:GTX660:ShuttleA:Ubuntu12:memory_usage_0_0:meta:max_rss_mb",
			value:     858,
			subResult: "max_rss_mb",
		},
		{
			key:       "x86:GTX660:ShuttleA:Ubuntu12:src_pipe_global_weak_symbol:memory:bytes",
			value:     158,
			subResult: "bytes",
		},
		{
			key:       "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_nonDiscardable_640_480:8888",
			value:     2.855735,
			subResult: "min_ms",
		},
		{
			key:       "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_nonDiscardable_640_480:8888:bytes",
			value:     298888,
			subResult: "bytes",
		},
		{
			key:       "x86:GTX660:ShuttleA:Ubuntu12:DeferredSurfaceCopy_nonDiscardable_640_480:8888:ops",
			value:     3333,
			subResult: "ops",
		},
	}
	// Do everything twice to ensure that we are idempotent.
	for i := 0; i < 2; i++ {
		// Add the BenchData to the Tile.
		addBenchDataToTile(benchData, tile, offset, metricsProcessed)

		// Test that the Tile has the right data.
		if got, want := len(tile.Traces), 13; got != want {
			t.Errorf("Wrong number of traces: Got %d Want %d", got, want)
		}
		for _, tc := range testcases {
			trace, ok := tile.Traces[tc.key]
			if !ok {
				t.Errorf("Missing expected key: %s", tc.key)
			}
			if got, want := trace.(*types.PerfTrace).Values[offset], tc.value; got != want {
				t.Errorf("Wrong value in trace: Got %v Want %v", got, want)
			}
		}
		trace := tile.Traces[testcases[0].key]

		// Validate the traces Params.
		expected := map[string]string{
			"arch":                        "x86",
			"gpu":                         "GTX660",
			"model":                       "ShuttleA",
			"os":                          "Ubuntu12",
			"system":                      "UNIX",
			"test":                        "DeferredSurfaceCopy_discardable_640_480",
			"config":                      "gpu",
			"GL_RENDERER":                 "GeForce GTX 660/PCIe/SSE2",
			"GL_SHADING_LANGUAGE_VERSION": "4.40 NVIDIA via Cg compiler",
			"GL_VENDOR":                   "NVIDIA Corporation",
			"GL_VERSION":                  "4.4.0 NVIDIA 331.49",
			"source_type":                 "bench",
			"sub_result":                  "min_ms",
		}
		if got, want := len(trace.Params()), len(expected); got != want {
			t.Errorf("Params wrong length: Got %v Want %v", got, want)
		}
		for k, v := range expected {
			if got, want := trace.Params()[k], v; got != want {
				t.Errorf("Wrong params: Got %v Want %v", got, want)
			}
		}

		// Validate the Tiles ParamSet.
		if got, want := len(tile.ParamSet), len(expected)+2; got != want {
			t.Errorf("Wrong ParamSet length: Got %v Want %v", got, want)
		}
		for k, _ := range expected {
			if _, ok := tile.ParamSet[k]; !ok {
				t.Errorf("Missing from ParamSet: %s", k)
			}
		}
		// The new symbol table size options values should also show up in the ParamSet.
		for _, k := range []string{"path", "symbol"} {
			if _, ok := tile.ParamSet[k]; !ok {
				t.Errorf("Missing from ParamSet: %s", k)
			}
		}

		if got, want := len(tile.ParamSet["source_type"]), 1; got != want {
			t.Errorf("Wrong ParamSet for source_type: Got %v Want %v", got, want)
		}
		if got, want := tile.ParamSet["source_type"][0], "bench"; got != want {
			t.Errorf("Wrong ParamSet value: Got %v Want %v", got, want)
		}
	}

	if got, want := metricsProcessed.Count(), int64(26); got != want {
		t.Errorf("Wrong number of points ingested: Got %v Want %v", got, want)
	}
	// Now update one of the params for a trace and reingest and confirm that the
	// trace params get updated.

	benchData.Options["system"] = "Linux"
	addBenchDataToTile(benchData, tile, offset, metricsProcessed)
	if got, want := "Linux", tile.Traces[testcases[0].key].Params()["system"]; got != want {
		t.Errorf("Failed to update params: Got %v Want %v", got, want)
	}
	if got, want := metricsProcessed.Count(), int64(39); got != want {
		t.Errorf("Wrong number of points ingested: Got %v Want %v", got, want)
	}
}
コード例 #13
0
ファイル: db.go プロジェクト: 1394/skia-buildbot
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, error) {
	ctx := context.Background()

	// Build the Tile.
	tile := tiling.NewTile()
	n := len(commitIDs)
	tile.Commits = make([]*tiling.Commit, n, n)

	// Populate the Tile's commits.
	for i, cid := range commitIDs {
		tile.Commits[i] = &tiling.Commit{
			Hash:       cid.ID,
			CommitTime: cid.Timestamp.Unix(),
		}
	}

	// tileMutex protects access to the Tile. Note that this only means the Tile,
	// while writing values into a Trace that already exists and is the right
	// size is Go routine safe.
	var tileMutex sync.Mutex

	errCh := make(chan error, len(commitIDs))

	// Fill in the data for each commit in it's own Go routine.
	var wg sync.WaitGroup
	for i, cid := range commitIDs {
		wg.Add(1)
		go func(i int, cid *CommitID) {
			defer wg.Done()
			// Load the values for the commit.
			getValuesRequest := &traceservice.GetValuesRequest{
				Commitid: tsCommitID(cid),
			}
			getValuesResponse, err := ts.traceService.GetValues(ctx, getValuesRequest)
			if err != nil {
				errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
				return
			}
			for _, pair := range getValuesResponse.Values {
				if pair == nil {
					glog.Errorf("Got a nil ValuePair in response: %s", err)
					continue
				}
				tr, ok := tile.Traces[pair.Key]
				if !ok {
					tileMutex.Lock()
					tile.Traces[pair.Key] = ts.traceBuilder(n)
					tileMutex.Unlock()
					tr = tile.Traces[pair.Key]
				}
				if err := tr.SetAt(i, pair.Value); err != nil {
					errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
					return
				}
			}
		}(i, cid)
	}
	wg.Wait()

	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, fmt.Errorf("Failed to load trace data: %s", err)
		}
	default:
	}

	glog.Infof("Finished loading values. Starting to load Params.")

	// Now load the params for the traces.
	traceids := []string{}
	for k, _ := range tile.Traces {
		traceids = append(traceids, k)
	}

	tracechunks := [][]string{}
	for len(traceids) > 0 {
		if len(traceids) > CHUNK_SIZE {
			tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
			traceids = traceids[CHUNK_SIZE:]
		} else {
			tracechunks = append(tracechunks, traceids)
			traceids = []string{}
		}
	}

	errCh = make(chan error, len(tracechunks))
	for _, chunk := range tracechunks {
		wg.Add(1)
		go func(chunk []string) {
			defer wg.Done()
			req := &traceservice.GetParamsRequest{
				Traceids: chunk,
			}
			resp, err := ts.traceService.GetParams(ctx, req)
			if err != nil {
				errCh <- fmt.Errorf("Failed to load params: %s", err)
				return
			}
			for _, param := range resp.Params {
				dst := tile.Traces[param.Key].Params()
				for k, v := range param.Params {
					dst[k] = v
				}
			}
		}(chunk)
	}

	wg.Wait()
	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, fmt.Errorf("Failed to load params: %s", err)
		}
	default:
	}

	// Rebuild the ParamSet.
	glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
	tiling.GetParamSet(tile.Traces, tile.ParamSet)
	return tile, nil
}
コード例 #14
0
ファイル: db.go プロジェクト: saltmueller/skia-buildbot
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, []string, error) {
	ts.clearMutex.RLock()
	ts.clearMutex.RUnlock()
	ctx := context.Background()

	// Build the Tile.
	tile := tiling.NewTile()
	n := len(commitIDs)
	tile.Commits = make([]*tiling.Commit, n, n)
	hash := make([]string, n)

	// Populate the Tile's commits.
	for i, cid := range commitIDs {
		tile.Commits[i] = &tiling.Commit{
			Hash:       cid.ID,
			CommitTime: cid.Timestamp,
		}
	}

	// tileMutex protects access to the Tile. Note that this only means the Tile,
	// while writing values into a Trace that already exists and is the right
	// size is Go routine safe.
	var tileMutex sync.Mutex

	errCh := make(chan error, len(commitIDs))

	// Fill in the data for each commit in it's own Go routine.
	var wg sync.WaitGroup
	for i, cid := range commitIDs {
		wg.Add(1)
		go func(i int, cid *CommitID) {
			defer wg.Done()
			// Load the values for the commit.
			getValuesRequest := &traceservice.GetValuesRequest{
				Commitid: tsCommitID(cid),
			}
			getRawValues, err := ts.traceService.GetValuesRaw(ctx, getValuesRequest)
			if err != nil {
				errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
				return
			}
			// Convert raw response into values.
			ci, err := traceservice.NewCommitInfo(getRawValues.Value)
			if err != nil {
				errCh <- fmt.Errorf("Failed to convert values for %d %#v: %s", i, *cid, err)
				return
			}
			// Now make sure we have all the traceids for the trace64ids in ci.
			missingKeys64 := []uint64{}
			ts.mutex.Lock()
			for id64, _ := range ci.Values {
				if _, ok := ts.id64Cache[id64]; !ok {
					missingKeys64 = append(missingKeys64, id64)
				}
			}
			ts.mutex.Unlock()
			if len(missingKeys64) > 0 {
				traceidsRequest := &traceservice.GetTraceIDsRequest{
					Id: missingKeys64,
				}
				traceids, err := ts.traceService.GetTraceIDs(ctx, traceidsRequest)
				if err != nil {
					errCh <- fmt.Errorf("Failed to get traceids for trace64ids for %d %#v: %s", i, *cid, err)
					return
				}
				ts.mutex.Lock()
				for _, tid := range traceids.Ids {
					ts.id64Cache[tid.Id64] = tid.Id
				}
				ts.mutex.Unlock()
			}

			ts.mutex.Lock()
			for id64, rawValue := range ci.Values {
				if rawValue == nil {
					glog.Errorf("Got a nil rawValue in response: %s", err)
					continue
				}
				traceid := ts.id64Cache[id64]
				tileMutex.Lock()
				tr, ok := tile.Traces[traceid]
				if !ok || tr == nil {
					tile.Traces[traceid] = ts.traceBuilder(n)
					tr = tile.Traces[traceid]
				}
				tileMutex.Unlock()
				if tr == nil {
					glog.Errorf("Trace was still nil for key: %v", traceid)
					continue
				}
				if err := tr.SetAt(i, rawValue); err != nil {
					errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
					return
				}
			}
			// Fill in the commits hash.
			hash[i] = getRawValues.Md5
			ts.mutex.Unlock()
		}(i, cid)
	}
	wg.Wait()

	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, nil, fmt.Errorf("Failed to load trace data: %s", err)
		}
	default:
	}

	glog.Infof("Finished loading values. Starting to load Params.")

	// Now load the params for the traces.
	traceids := []string{}
	ts.mutex.Lock()
	for k, _ := range tile.Traces {
		// Only load params for traces not already in the cache.
		if _, ok := ts.paramsCache[k]; !ok {
			traceids = append(traceids, k)
		}
	}
	ts.mutex.Unlock()

	// Break the loading of params into chunks and make those requests concurrently.
	// The params are just loaded into the paramsCache.
	tracechunks := [][]string{}
	for len(traceids) > 0 {
		if len(traceids) > CHUNK_SIZE {
			tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
			traceids = traceids[CHUNK_SIZE:]
		} else {
			tracechunks = append(tracechunks, traceids)
			traceids = []string{}
		}
	}

	errCh = make(chan error, len(tracechunks))
	for _, chunk := range tracechunks {
		wg.Add(1)
		go func(chunk []string) {
			defer wg.Done()
			req := &traceservice.GetParamsRequest{
				Traceids: chunk,
			}
			resp, err := ts.traceService.GetParams(ctx, req)
			if err != nil {
				errCh <- fmt.Errorf("Failed to load params: %s", err)
				return
			}
			for _, param := range resp.Params {
				ts.mutex.Lock()
				ts.paramsCache[param.Key] = param.Params
				ts.mutex.Unlock()
			}
		}(chunk)
	}
	wg.Wait()

	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, nil, fmt.Errorf("Failed to load params: %s", err)
		}
	default:
	}

	// Add all params from the cache.
	ts.mutex.Lock()
	for k, tr := range tile.Traces {
		p := tr.Params()
		for pk, pv := range ts.paramsCache[k] {
			p[pk] = pv
		}
	}
	ts.mutex.Unlock()

	// Rebuild the ParamSet.
	glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
	tiling.GetParamSet(tile.Traces, tile.ParamSet)
	return tile, hash, nil
}