Exemple #1
0
func dumpTileToJSON(store tiling.TileStore, nCommits int, nTraces int, fname string) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	newTile := tile
	if (nCommits > 0) || (nTraces > 0) {
		lastIdx := tile.LastCommitIndex()
		if nCommits <= 0 {
			nCommits = lastIdx + 1
		}

		if nTraces <= 0 {
			nTraces = len(tile.Traces)
		}

		commitLen := util.MinInt(nCommits, lastIdx+1)
		startCommit := lastIdx + 1 - commitLen
		newTraces := map[string]tiling.Trace{}
		for key, trace := range tile.Traces {
			for i := startCommit; i <= lastIdx; i++ {
				if !trace.IsMissing(i) {
					newTraces[key] = trace
					break
				}
			}
			if len(newTraces) >= nTraces {
				break
			}
		}

		newCommits := tile.Commits[startCommit:]
		newParamSet := map[string][]string{}
		tiling.GetParamSet(newTraces, newParamSet)

		newTile = &tiling.Tile{
			Traces:    newTraces,
			ParamSet:  newParamSet,
			Commits:   newCommits,
			Scale:     tile.Scale,
			TileIndex: tile.TileIndex,
		}
	}

	result, err := json.Marshal(newTile)
	if err != nil {
		glog.Fatalf("Could not marshal to JSON: %s", err)
	}

	err = ioutil.WriteFile(fname, result, 0644)
	if err != nil {
		glog.Fatalf("Could not write output file %s", err)
	}

	fmt.Printf("Commits included: %d\n", len(newTile.Commits))
	fmt.Printf("Traces included:  %d\n", len(newTile.Traces))
}
Exemple #2
0
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, error) {
	ctx := context.Background()

	// Build the Tile.
	tile := tiling.NewTile()
	n := len(commitIDs)
	tile.Commits = make([]*tiling.Commit, n, n)

	// Populate the Tile's commits.
	for i, cid := range commitIDs {
		tile.Commits[i] = &tiling.Commit{
			Hash:       cid.ID,
			CommitTime: cid.Timestamp.Unix(),
		}
	}

	// tileMutex protects access to the Tile. Note that this only means the Tile,
	// while writing values into a Trace that already exists and is the right
	// size is Go routine safe.
	var tileMutex sync.Mutex

	errCh := make(chan error, len(commitIDs))

	// Fill in the data for each commit in it's own Go routine.
	var wg sync.WaitGroup
	for i, cid := range commitIDs {
		wg.Add(1)
		go func(i int, cid *CommitID) {
			defer wg.Done()
			// Load the values for the commit.
			getValuesRequest := &traceservice.GetValuesRequest{
				Commitid: tsCommitID(cid),
			}
			getValuesResponse, err := ts.traceService.GetValues(ctx, getValuesRequest)
			if err != nil {
				errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
				return
			}
			for _, pair := range getValuesResponse.Values {
				if pair == nil {
					glog.Errorf("Got a nil ValuePair in response: %s", err)
					continue
				}
				tr, ok := tile.Traces[pair.Key]
				if !ok {
					tileMutex.Lock()
					tile.Traces[pair.Key] = ts.traceBuilder(n)
					tileMutex.Unlock()
					tr = tile.Traces[pair.Key]
				}
				if err := tr.SetAt(i, pair.Value); err != nil {
					errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
					return
				}
			}
		}(i, cid)
	}
	wg.Wait()

	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, fmt.Errorf("Failed to load trace data: %s", err)
		}
	default:
	}

	glog.Infof("Finished loading values. Starting to load Params.")

	// Now load the params for the traces.
	traceids := []string{}
	for k, _ := range tile.Traces {
		traceids = append(traceids, k)
	}

	tracechunks := [][]string{}
	for len(traceids) > 0 {
		if len(traceids) > CHUNK_SIZE {
			tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
			traceids = traceids[CHUNK_SIZE:]
		} else {
			tracechunks = append(tracechunks, traceids)
			traceids = []string{}
		}
	}

	errCh = make(chan error, len(tracechunks))
	for _, chunk := range tracechunks {
		wg.Add(1)
		go func(chunk []string) {
			defer wg.Done()
			req := &traceservice.GetParamsRequest{
				Traceids: chunk,
			}
			resp, err := ts.traceService.GetParams(ctx, req)
			if err != nil {
				errCh <- fmt.Errorf("Failed to load params: %s", err)
				return
			}
			for _, param := range resp.Params {
				dst := tile.Traces[param.Key].Params()
				for k, v := range param.Params {
					dst[k] = v
				}
			}
		}(chunk)
	}

	wg.Wait()
	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, fmt.Errorf("Failed to load params: %s", err)
		}
	default:
	}

	// Rebuild the ParamSet.
	glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
	tiling.GetParamSet(tile.Traces, tile.ParamSet)
	return tile, nil
}
Exemple #3
0
// TileFromCommits implements DB.TileFromCommits().
func (ts *TsDB) TileFromCommits(commitIDs []*CommitID) (*tiling.Tile, []string, error) {
	ts.clearMutex.RLock()
	ts.clearMutex.RUnlock()
	ctx := context.Background()

	// Build the Tile.
	tile := tiling.NewTile()
	n := len(commitIDs)
	tile.Commits = make([]*tiling.Commit, n, n)
	hash := make([]string, n)

	// Populate the Tile's commits.
	for i, cid := range commitIDs {
		tile.Commits[i] = &tiling.Commit{
			Hash:       cid.ID,
			CommitTime: cid.Timestamp,
		}
	}

	// tileMutex protects access to the Tile. Note that this only means the Tile,
	// while writing values into a Trace that already exists and is the right
	// size is Go routine safe.
	var tileMutex sync.Mutex

	errCh := make(chan error, len(commitIDs))

	// Fill in the data for each commit in it's own Go routine.
	var wg sync.WaitGroup
	for i, cid := range commitIDs {
		wg.Add(1)
		go func(i int, cid *CommitID) {
			defer wg.Done()
			// Load the values for the commit.
			getValuesRequest := &traceservice.GetValuesRequest{
				Commitid: tsCommitID(cid),
			}
			getRawValues, err := ts.traceService.GetValuesRaw(ctx, getValuesRequest)
			if err != nil {
				errCh <- fmt.Errorf("Failed to get values for %d %#v: %s", i, *cid, err)
				return
			}
			// Convert raw response into values.
			ci, err := traceservice.NewCommitInfo(getRawValues.Value)
			if err != nil {
				errCh <- fmt.Errorf("Failed to convert values for %d %#v: %s", i, *cid, err)
				return
			}
			// Now make sure we have all the traceids for the trace64ids in ci.
			missingKeys64 := []uint64{}
			ts.mutex.Lock()
			for id64, _ := range ci.Values {
				if _, ok := ts.id64Cache[id64]; !ok {
					missingKeys64 = append(missingKeys64, id64)
				}
			}
			ts.mutex.Unlock()
			if len(missingKeys64) > 0 {
				traceidsRequest := &traceservice.GetTraceIDsRequest{
					Id: missingKeys64,
				}
				traceids, err := ts.traceService.GetTraceIDs(ctx, traceidsRequest)
				if err != nil {
					errCh <- fmt.Errorf("Failed to get traceids for trace64ids for %d %#v: %s", i, *cid, err)
					return
				}
				ts.mutex.Lock()
				for _, tid := range traceids.Ids {
					ts.id64Cache[tid.Id64] = tid.Id
				}
				ts.mutex.Unlock()
			}

			ts.mutex.Lock()
			for id64, rawValue := range ci.Values {
				if rawValue == nil {
					glog.Errorf("Got a nil rawValue in response: %s", err)
					continue
				}
				traceid := ts.id64Cache[id64]
				tileMutex.Lock()
				tr, ok := tile.Traces[traceid]
				if !ok || tr == nil {
					tile.Traces[traceid] = ts.traceBuilder(n)
					tr = tile.Traces[traceid]
				}
				tileMutex.Unlock()
				if tr == nil {
					glog.Errorf("Trace was still nil for key: %v", traceid)
					continue
				}
				if err := tr.SetAt(i, rawValue); err != nil {
					errCh <- fmt.Errorf("Unable to convert trace value %d %#v: %s", i, *cid, err)
					return
				}
			}
			// Fill in the commits hash.
			hash[i] = getRawValues.Md5
			ts.mutex.Unlock()
		}(i, cid)
	}
	wg.Wait()

	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, nil, fmt.Errorf("Failed to load trace data: %s", err)
		}
	default:
	}

	glog.Infof("Finished loading values. Starting to load Params.")

	// Now load the params for the traces.
	traceids := []string{}
	ts.mutex.Lock()
	for k, _ := range tile.Traces {
		// Only load params for traces not already in the cache.
		if _, ok := ts.paramsCache[k]; !ok {
			traceids = append(traceids, k)
		}
	}
	ts.mutex.Unlock()

	// Break the loading of params into chunks and make those requests concurrently.
	// The params are just loaded into the paramsCache.
	tracechunks := [][]string{}
	for len(traceids) > 0 {
		if len(traceids) > CHUNK_SIZE {
			tracechunks = append(tracechunks, traceids[:CHUNK_SIZE])
			traceids = traceids[CHUNK_SIZE:]
		} else {
			tracechunks = append(tracechunks, traceids)
			traceids = []string{}
		}
	}

	errCh = make(chan error, len(tracechunks))
	for _, chunk := range tracechunks {
		wg.Add(1)
		go func(chunk []string) {
			defer wg.Done()
			req := &traceservice.GetParamsRequest{
				Traceids: chunk,
			}
			resp, err := ts.traceService.GetParams(ctx, req)
			if err != nil {
				errCh <- fmt.Errorf("Failed to load params: %s", err)
				return
			}
			for _, param := range resp.Params {
				ts.mutex.Lock()
				ts.paramsCache[param.Key] = param.Params
				ts.mutex.Unlock()
			}
		}(chunk)
	}
	wg.Wait()

	// See if any Go routine generated an error.
	select {
	case err, ok := <-errCh:
		if ok {
			return nil, nil, fmt.Errorf("Failed to load params: %s", err)
		}
	default:
	}

	// Add all params from the cache.
	ts.mutex.Lock()
	for k, tr := range tile.Traces {
		p := tr.Params()
		for pk, pv := range ts.paramsCache[k] {
			p[pk] = pv
		}
	}
	ts.mutex.Unlock()

	// Rebuild the ParamSet.
	glog.Infof("Finished loading params. Starting to rebuild ParamSet.")
	tiling.GetParamSet(tile.Traces, tile.ParamSet)
	return tile, hash, nil
}