Esempio n. 1
0
// update syncs code in the relevant repositories.
func (r *repoManager) update() error {
	// Sync the projects.
	r.mtx.Lock()
	defer r.mtx.Unlock()

	// Create the chromium parent directory if needed.
	if _, err := os.Stat(r.chromiumParentDir); err != nil {
		if err := os.MkdirAll(r.chromiumParentDir, 0755); err != nil {
			return err
		}
	}

	if _, err := os.Stat(path.Join(r.chromiumDir, ".git")); err == nil {
		if err := r.cleanChromium(); err != nil {
			return err
		}
	}

	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumParentDir,
		Env:  []string{fmt.Sprintf("PATH=%s:%s", r.depot_tools, os.Getenv("PATH"))},
		Name: r.gclient,
		Args: []string{"config", REPO_CHROMIUM},
	}); err != nil {
		return err
	}
	if _, err := exec.RunCommand(&exec.Command{
		Dir:  r.chromiumParentDir,
		Env:  []string{fmt.Sprintf("PATH=%s:%s", r.depot_tools, os.Getenv("PATH"))},
		Name: r.gclient,
		Args: []string{"sync", "--nohooks"},
	}); err != nil {
		return err
	}

	// Create the child GitInfo if needed.
	if r.childRepo == nil {
		childRepo, err := gitinfo.NewGitInfo(r.childDir, false, true)
		if err != nil {
			return err
		}
		r.childRepo = childRepo
	}

	// Get the last roll revision.
	lastRollRev, err := r.getLastRollRev()
	if err != nil {
		return err
	}
	r.lastRollRev = lastRollRev

	// Record child HEAD
	childHead, err := r.childRepo.FullHash("origin/master")
	if err != nil {
		return err
	}
	r.childHead = childHead
	return nil
}
func TestIngestCommits(t *testing.T) {
	// Get a known Git repo with 34 commits in it setup.
	tr := util.NewTempRepo()
	defer tr.Cleanup()

	// Create a temporary place for a filetilestore.
	tileDir, err := ioutil.TempDir("", "skiaperf")
	if err != nil {
		t.Fatal("Failed to create testing Tile dir: ", err)
	}
	defer testutils.RemoveAll(t, tileDir)

	git, err := gitinfo.NewGitInfo(filepath.Join(tr.Dir, "testrepo"), false, false)
	if err != nil {
		glog.Fatalf("Failed loading Git info: %s\n", err)
	}

	// Construct an Ingestor and have it UpdateCommitInfo.
	i, err := ingester.NewIngester(git, tileDir, config.DATASET_NANO, NewNanoBenchIngester(), 1, time.Second, map[string]string{}, "", "")
	if err != nil {
		t.Fatal("Failed to create ingester:", err)
	}

	if err := i.UpdateCommitInfo(false); err != nil {
		t.Fatal("Failed to ingest commits:", err)
	}

	// Validate the generated Tiles.
	store := filetilestore.NewFileTileStore(tileDir, config.DATASET_NANO, 0)
	if !validator.ValidateDataset(store, false, false) {
		t.Error("Failed to validate the created Tiles:", err)
	}

	// Test TileTracker while were here.
	tt := ingester.NewTileTracker(store, i.HashToNumber())
	err = tt.Move("7a6fe813047d1a84107ef239e81f310f27861473")
	if err != nil {
		t.Fatal(err)
	}
	if got, want := tt.LastTileNum(), 2; got != want {
		t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want)
	}
	err = tt.Move("87709bc360f35de52c2f2bc2fc70962fb234db2d")
	if err != nil {
		t.Fatal(err)
	}
	if got, want := tt.LastTileNum(), 2; got != want {
		t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want)
	}
}
func TestNewTraceDBBuilder(t *testing.T) {
	defer cleanup()

	// First spin up a traceservice server that we wil talk to.
	server, err := traceservice.NewTraceServiceServer(FILENAME)
	if err != nil {
		t.Fatalf("Failed to initialize the tracestore server: %s", err)
	}

	// Start the server on an open port.
	lis, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("failed to listen: %v", err)
	}
	port := lis.Addr().String()
	s := grpc.NewServer()
	traceservice.RegisterTraceServiceServer(s, server)
	go func() {
		t.Fatalf("Failed while serving: %s", s.Serve(lis))
	}()

	// Get a Git repo.
	tr := util.NewTempRepo()
	defer tr.Cleanup()

	// Set up a connection to the server.
	conn, err := grpc.Dial(port, grpc.WithInsecure())
	if err != nil {
		t.Fatalf("did not connect: %v", err)
	}
	ts, err := db.NewTraceServiceDB(conn, types.PerfTraceBuilder)
	if err != nil {
		t.Fatalf("Failed to create tracedb.DB: %s", err)
	}
	defer util.Close(ts)

	git, err := gitinfo.NewGitInfo(filepath.Join(tr.Dir, "testrepo"), false, false)
	if err != nil {
		t.Fatal(err)
	}
	hashes := git.LastN(50)
	assert.Equal(t, 50, len(hashes))

	// Populate the tracedb with some data.
	commitID := &db.CommitID{
		Timestamp: git.Timestamp(hashes[1]).Unix(),
		ID:        hashes[1],
		Source:    "master",
	}

	entries := map[string]*db.Entry{
		"key:8888:android": &db.Entry{
			Params: map[string]string{
				"config":   "8888",
				"platform": "android",
				"type":     "skp",
			},
			Value: types.BytesFromFloat64(0.01),
		},
		"key:gpu:win8": &db.Entry{
			Params: map[string]string{
				"config":   "gpu",
				"platform": "win8",
				"type":     "skp",
			},
			Value: types.BytesFromFloat64(1.234),
		},
	}

	err = ts.Add(commitID, entries)
	if err != nil {
		t.Fatalf("Failed to add data to traceservice: %s", err)
	}

	evt := eventbus.New(nil)
	traceDB, err := db.NewTraceServiceDBFromAddress(port, types.PerfTraceBuilder)
	assert.Nil(t, err)

	builder, err := db.NewMasterTileBuilder(traceDB, git, 50, evt)
	if err != nil {
		t.Fatalf("Failed to construct TraceStore: %s", err)
	}
	tile := builder.GetTile()
	assert.Equal(t, 50, len(tile.Commits))
	assert.Equal(t, 2, len(tile.Traces))
	assert.Equal(t, commitID.ID, tile.Commits[1].Hash)
	assert.Equal(t, "Joe Gregorio ([email protected])", tile.Commits[1].Author)

	ptrace := tile.Traces["key:8888:android"].(*types.PerfTrace)
	assert.Equal(t, 0.01, ptrace.Values[1])
	assert.Equal(t, config.MISSING_DATA_SENTINEL, ptrace.Values[0])

	ptrace = tile.Traces["key:gpu:win8"].(*types.PerfTrace)
	assert.Equal(t, 1.234, ptrace.Values[1])
	assert.Equal(t, config.MISSING_DATA_SENTINEL, ptrace.Values[0])

	// Now add more data to the tracestore, trigger LoadTile() and check the results.
	commitID = &db.CommitID{
		Timestamp: git.Timestamp(hashes[2]).Unix(),
		ID:        hashes[2],
		Source:    "master",
	}

	entries = map[string]*db.Entry{
		"key:8888:android": &db.Entry{
			Params: map[string]string{
				"config":   "8888",
				"platform": "android",
				"type":     "skp",
			},
			Value: types.BytesFromFloat64(0.02),
		},
		"key:565:ubuntu": &db.Entry{
			Params: map[string]string{
				"config":   "565",
				"platform": "ubuntu",
				"type":     "skp",
			},
			Value: types.BytesFromFloat64(2.345),
		},
	}

	err = ts.Add(commitID, entries)
	if err != nil {
		t.Fatalf("Failed to add data to traceservice: %s", err)
	}

	// Access the LoadTile method without exposing it in the MasterBuilder interface.
	type hasLoadTile interface {
		LoadTile() error
	}
	masterBuilder := builder.(hasLoadTile)
	if err := masterBuilder.LoadTile(); err != nil {
		t.Fatalf("Failed to force load Tile: %s", err)
	}

	tile = builder.GetTile()
	assert.Equal(t, 50, len(tile.Commits))
	assert.Equal(t, 3, len(tile.Traces))
	assert.Equal(t, commitID.ID, tile.Commits[2].Hash)

	ptrace = tile.Traces["key:8888:android"].(*types.PerfTrace)
	assert.Equal(t, 0.01, ptrace.Values[1])
	assert.Equal(t, 0.02, ptrace.Values[2])
	assert.Equal(t, config.MISSING_DATA_SENTINEL, ptrace.Values[0])
	assert.Equal(t, config.MISSING_DATA_SENTINEL, ptrace.Values[3])

	ptrace = tile.Traces["key:gpu:win8"].(*types.PerfTrace)
	assert.Equal(t, 1.234, ptrace.Values[1])
	assert.Equal(t, config.MISSING_DATA_SENTINEL, ptrace.Values[0])

	ptrace = tile.Traces["key:565:ubuntu"].(*types.PerfTrace)
	assert.Equal(t, 2.345, ptrace.Values[2])
	assert.Equal(t, config.MISSING_DATA_SENTINEL, ptrace.Values[3])
}
Esempio n. 4
0
func main() {
	defer common.LogPanic()
	// Setup DB flags.
	dbConf := db.DBConfigFromFlags()

	common.InitWithMetricsCB("ingest", func() string {
		common.DecodeTomlFile(*configFilename, &config)
		return config.Common.GraphiteServer
	})

	// Initialize the database. We might not need the oauth dialog if it fails.
	if !config.Common.Local {
		if err := dbConf.GetPasswordFromMetadata(); err != nil {
			glog.Fatal(err)
		}
	}
	if err := dbConf.InitDB(); err != nil {
		glog.Fatal(err)
	}

	// Get a backoff transport.
	transport := util.NewBackOffTransport()

	// Determine the oauth scopes we are going to use. Storage is used by all
	// ingesters.
	scopes := []string{storage.CloudPlatformScope}
	for _, ingesterConfig := range config.Ingesters {
		if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
			scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
		}
	}

	// Initialize the oauth client that is used to access all scopes.
	var client *http.Client
	var err error
	if config.Common.Local {
		if config.Common.DoOAuth {
			client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile,
				config.Common.OAuthClientSecretFile,
				transport, scopes...)
			if err != nil {
				glog.Fatalf("Failed to auth: %s", err)
			}
		} else {
			client = nil
			// Add back service account access here when it's fixed.
		}
	} else {
		// Assume we are on a GCE instance.
		client = auth.GCEServiceAccountClient(transport)
	}

	// Initialize the ingester and gold ingester.
	ingester.Init(client)
	if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
		if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
			glog.Fatalf("Unable to initialize GoldIngester: %s", err)
		}
	}

	// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
	// database connection and they are all instantiated the same way instead of the
	// one-off approaches.
	if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
		dbConf := &database.DatabaseConfig{
			Host:           ingesterConf.DBHost,
			Port:           ingesterConf.DBPort,
			User:           database.USER_RW,
			Name:           ingesterConf.DBName,
			MigrationSteps: golddb.MigrationSteps(),
		}

		if !config.Common.Local {
			if err := dbConf.GetPasswordFromMetadata(); err != nil {
				glog.Fatal(err)
			}
		}
		vdb, err := dbConf.NewVersionedDB()
		if err != nil {
			glog.Fatalf("Unable to open db connection: %s", err)
		}
		goldtrybot.Init(vdb)
	}

	git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
	if err != nil {
		glog.Fatalf("Failed loading Git info: %s\n", err)
	}

	for dataset, ingesterConfig := range config.Ingesters {
		// Get duration equivalent to the number of days.
		minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)

		constructorName := ingesterConfig.ConstructorName
		if constructorName == "" {
			constructorName = dataset
		}

		constructor := ingester.Constructor(constructorName)
		resultIngester := constructor()

		glog.Infof("Process name: %s", dataset)
		startProcess := NewIngestionProcess(git,
			config.Common.TileDir,
			dataset,
			resultIngester,
			ingesterConfig.ExtraParams,
			ingesterConfig.RunEvery.Duration,
			ingesterConfig.NCommits,
			minDuration,
			ingesterConfig.StatusDir,
			ingesterConfig.MetricName)
		startProcess()
	}

	select {}
}