func Init() {
rand.Seed(time.Now().UnixNano())
if *resourcesDir == "" {
_, filename, _, _ := runtime.Caller(0)
*resourcesDir = filepath.Join(filepath.Dir(filename), "../..")
}
loadTemplates()
var err error
git, err = gitinfo.CloneOrUpdate(*gitRepoURL, *gitRepoDir, false)
if err != nil {
glog.Fatal(err)
}
evt := eventbus.New(nil)
tileStats = tilestats.New(evt)
// Connect to traceDB and create the builders.
db, err := tracedb.NewTraceServiceDBFromAddress(*traceservice, types.PerfTraceBuilder)
if err != nil {
glog.Fatalf("Failed to connect to tracedb: %s", err)
}
masterTileBuilder, err = tracedb.NewMasterTileBuilder(db, git, *tileSize, evt)
if err != nil {
glog.Fatalf("Failed to build trace/db.DB: %s", err)
}
branchTileBuilder = tracedb.NewBranchTileBuilder(db, git, rietveld.RIETVELD_SKIA_URL, evt)
}
func main() {
common.Init()
grpclog.Init()
// Set up a connection to the server.
conn, err := grpc.Dial(*address, grpc.WithInsecure())
if err != nil {
glog.Fatalf("did not connect: %v", err)
}
defer util.Close(conn)
// Build a TraceService client.
builder := ptypes.PerfTraceBuilder
if *gold {
builder = types.GoldenTraceBuilder
}
ts, err := db.NewTraceServiceDB(conn, builder)
if err != nil {
log.Fatalf("Failed to create db.DB: %s", err)
}
glog.Infof("Opened tracedb")
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
glog.Fatalf("Failed to open profiling file: %s", err)
}
if err := pprof.StartCPUProfile(f); err != nil {
glog.Fatalf("Failed to start profiling: %s", err)
}
defer pprof.StopCPUProfile()
_main(ts)
} else {
_main(ts)
}
}
func main() {
defer common.LogPanic()
common.InitWithMetrics("skia-ingestion", graphiteServer)
// Initialize oauth client and start the ingesters.
client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, storage.CloudPlatformScope)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
// Start the ingesters.
config, err := sharedconfig.ConfigFromTomlFile(*configFilename)
if err != nil {
glog.Fatalf("Unable to read config file %s. Got error: %s", *configFilename, err)
}
ingesters, err := ingestion.IngestersFromConfig(config, client)
if err != nil {
glog.Fatalf("Unable to instantiate ingesters: %s", err)
}
for _, oneIngester := range ingesters {
oneIngester.Start()
}
// Run the ingesters forever.
select {}
}
func main() {
// Set up flags.
dbConf := database.ConfigFromFlags(buildbot.PROD_DB_HOST, buildbot.PROD_DB_PORT, database.USER_ROOT, buildbot.PROD_DB_NAME, buildbot.MigrationSteps())
// Global init to initialize glog and parse arguments.
common.Init()
if err := dbConf.PromptForPassword(); err != nil {
glog.Fatal(err)
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatal(err)
}
// Get the current database version
maxDBVersion := vdb.MaxDBVersion()
glog.Infof("Latest database version: %d", maxDBVersion)
dbVersion, err := vdb.DBVersion()
if err != nil {
glog.Fatalf("Unable to retrieve database version. Error: %s", err)
}
glog.Infof("Current database version: %d", dbVersion)
if dbVersion < maxDBVersion {
glog.Infof("Migrating to version: %d", maxDBVersion)
err = vdb.Migrate(maxDBVersion)
if err != nil {
glog.Fatalf("Unable to retrieve database version. Error: %s", err)
}
}
glog.Infoln("Database migration finished.")
}
func startMetrics(appName, graphiteServer string) {
if graphiteServer == "" {
glog.Warningf("No metrics server specified.")
return
}
addr, err := net.ResolveTCPAddr("tcp", graphiteServer)
if err != nil {
glog.Fatalf("Unable to resolve metrics server address: %s", err)
}
// Get the hostname and create the app-prefix.
hostName, err := os.Hostname()
if err != nil {
glog.Fatalf("Unable to retrieve hostname: %s", err)
}
appPrefix := fmt.Sprintf("%s.%s", appName, strings.Replace(hostName, ".", "-", -1))
// Runtime metrics.
metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry)
go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, SAMPLE_PERIOD)
go graphite.Graphite(metrics.DefaultRegistry, SAMPLE_PERIOD, appPrefix, addr)
// Uptime.
uptimeGuage := metrics.GetOrRegisterGaugeFloat64("uptime", metrics.DefaultRegistry)
go func() {
startTime := time.Now()
uptimeGuage.Update(0)
for _ = range time.Tick(SAMPLE_PERIOD) {
uptimeGuage.Update(time.Since(startTime).Seconds())
}
}()
}
func main() {
defer common.LogPanic()
common.Init()
args := flag.Args()
if len(args) != 3 {
glog.Errorf("Expected arguments: branch target buildID")
glog.Errorf("i.e.: git_master-skia razor-userdebug 1772442")
os.Exit(1)
}
// Set the arguments necessary to lookup the git hash.
branch := args[0]
target := args[1]
buildID := args[2]
glog.Infof("Branch, target, buildID: %s, %s, %s", branch, target, buildID)
// Set up the oauth client.
var client *http.Client
var err error
// In this case we don't want a backoff transport since the Apiary backend
// seems to fail a lot, so we basically want to fall back to polling if a
// call fails.
transport := &http.Transport{
Dial: util.DialTimeout,
}
if *local {
// Use a local client secret file to load data.
client, err = auth.InstalledAppClient(OAUTH_CACHE_FILEPATH, CLIENT_SECRET_FILEPATH,
transport,
androidbuildinternal.AndroidbuildInternalScope,
storage.CloudPlatformScope)
if err != nil {
glog.Fatalf("Unable to create installed app oauth client:%s", err)
}
} else {
// Use compute engine service account.
client = auth.GCEServiceAccountClient(transport)
}
f, err := androidbuild.New("/tmp/android-gold-ingest", client)
if err != nil {
glog.Fatalf("Failed to construct client: %s", err)
}
for {
r, err := f.Get(branch, target, buildID)
if err != nil {
glog.Errorf("Failed to get requested info: %s", err)
time.Sleep(1 * time.Minute)
continue
}
if r != nil {
glog.Infof("Successfully found: %#v", *r)
}
time.Sleep(1 * time.Minute)
}
}
// getOAuthClient returns an oauth client.
func getOAuthClient(cacheFilePath string) *http.Client {
var client *http.Client
var err error
if *doOauth || *local {
if *local {
// Load client secrets file. The client_secret.json file must come from
// project id: 470362608618, which is whitelisted for access to
// gs://chromium-skia-gm.
client, err = auth.NewClientWithTransport(true, cacheFilePath, *clientSecretFile, nil, auth.SCOPE_READ_ONLY)
if err != nil {
glog.Fatalf("Failed to auth: (Did you download the client_secret.json file from the project with ID: google.com:chrome-skia #470362608618 ?) %s", err)
}
} else {
// Load client id and secret from metadata.
clientId := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_ID))
clientSecret := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_SECRET))
client, err = auth.NewClientFromIdAndSecret(clientId, clientSecret, cacheFilePath, auth.SCOPE_READ_ONLY)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
}
}
return client
}
func pullInit() {
hostname, err := os.Hostname()
if err != nil {
// Never call glog before common.Init*.
os.Exit(1)
}
common.InitWithMetrics("pulld."+hostname, graphiteServer)
glog.Infof("Running with hostname: %s", hostname)
client, err := auth.NewClient(*doOauth, *oauthCacheFile,
storage.DevstorageFullControlScope,
compute.ComputeReadonlyScope)
if err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
glog.Info("Got authenticated client.")
store, err = storage.New(client)
if err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
step(client, store, hostname)
timeCh := time.Tick(time.Second * 60)
go func() {
for {
select {
case <-timeCh:
case <-httpTriggerCh:
}
step(client, store, hostname)
}
}()
}
func main() {
common.InitWithMetrics(filepath.Base(os.Args[0]), graphiteServer)
ts, err := traceservice.NewTraceServiceServer(*db_file)
if err != nil {
glog.Fatalf("Failed to initialize the tracestore server: %s", err)
}
lis, err := net.Listen("tcp", *port)
if err != nil {
glog.Fatalf("failed to listen: %v", err)
}
s := grpc.NewServer()
traceservice.RegisterTraceServiceServer(s, ts)
go func() {
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
glog.Fatalf("Failed to open profiling file: %s", err)
}
if err := pprof.StartCPUProfile(f); err != nil {
glog.Fatalf("Failed to start profiling: %s", err)
}
}
glog.Fatalf("Failure while serving: %s", s.Serve(lis))
}()
// Handle SIGINT and SIGTERM.
ch := make(chan os.Signal)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
log.Println(<-ch)
if *cpuprofile != "" {
pprof.StopCPUProfile()
}
}
func main() {
output, err := exec.RunSimple("gcloud compute project-info describe --format=json")
if err != nil {
glog.Fatalf("Failed to execute gcloud: %s", err)
}
pi := &ProjectInfo{
CommonInstanceMetadata: Metadata{
Items: []*Item{},
},
}
if err := json.Unmarshal([]byte(output), pi); err != nil {
glog.Fatalf("Failed to parse gcloud output: %s", err)
}
items := pi.CommonInstanceMetadata.Items
body := ""
for _, item := range items {
if item.Key == metadata.JWT_SERVICE_ACCOUNT {
body = item.Value
break
}
}
if body == "" {
glog.Fatalf("Failed to find the JST service account in the metadata.")
}
if err := ioutil.WriteFile(OUTPUT_FILENAME, []byte(body), 0600); err != nil {
glog.Fatalf("Failed to write %q: %s", OUTPUT_FILENAME, err)
}
}
func dumpTileToJSON(store tiling.TileStore, nCommits int, nTraces int, fname string) {
tile, err := store.Get(0, -1)
if err != nil {
glog.Fatal("Could not read tile: " + err.Error())
}
newTile := tile
if (nCommits > 0) || (nTraces > 0) {
lastIdx := tile.LastCommitIndex()
if nCommits <= 0 {
nCommits = lastIdx + 1
}
if nTraces <= 0 {
nTraces = len(tile.Traces)
}
commitLen := util.MinInt(nCommits, lastIdx+1)
startCommit := lastIdx + 1 - commitLen
newTraces := map[string]tiling.Trace{}
for key, trace := range tile.Traces {
for i := startCommit; i <= lastIdx; i++ {
if !trace.IsMissing(i) {
newTraces[key] = trace
break
}
}
if len(newTraces) >= nTraces {
break
}
}
newCommits := tile.Commits[startCommit:]
newParamSet := map[string][]string{}
tiling.GetParamSet(newTraces, newParamSet)
newTile = &tiling.Tile{
Traces: newTraces,
ParamSet: newParamSet,
Commits: newCommits,
Scale: tile.Scale,
TileIndex: tile.TileIndex,
}
}
result, err := json.Marshal(newTile)
if err != nil {
glog.Fatalf("Could not marshal to JSON: %s", err)
}
err = ioutil.WriteFile(fname, result, 0644)
if err != nil {
glog.Fatalf("Could not write output file %s", err)
}
fmt.Printf("Commits included: %d\n", len(newTile.Commits))
fmt.Printf("Traces included: %d\n", len(newTile.Traces))
}
func main() {
defer common.LogPanic()
common.Init()
if *emailTokenPath == "" {
glog.Error("Must specify --email_token_path")
return
}
defer glog.Flush()
for _, shiftType := range allShiftTypes {
res, err := http.Get(shiftType.nextSheriffEndpoint)
if err != nil {
glog.Fatalf("Could not HTTP Get: %s", err)
}
defer util.Close(res.Body)
var jsonType map[string]interface{}
if err := json.NewDecoder(res.Body).Decode(&jsonType); err != nil {
glog.Fatalf("Could not unmarshal JSON: %s", err)
}
sheriffEmail, _ := jsonType["username"].(string)
if sheriffEmail == NO_SHERIFF {
glog.Infof("Skipping emailing %s because %s was specified", shiftType.shiftName, NO_SHERIFF)
continue
}
sheriffUsername := strings.Split(string(sheriffEmail), "@")[0]
emailTemplateParsed := template.Must(template.New("sheriff_email").Parse(EMAIL_TEMPLATE))
emailBytes := new(bytes.Buffer)
if err := emailTemplateParsed.Execute(emailBytes, struct {
SheriffName string
SheriffType string
SheriffSchedules string
SheriffDoc string
ScheduleStart string
ScheduleEnd string
}{
SheriffName: sheriffUsername,
SheriffType: shiftType.shiftName,
SheriffSchedules: shiftType.schedulesLink,
SheriffDoc: shiftType.documentationLink,
ScheduleStart: jsonType["schedule_start"].(string),
ScheduleEnd: jsonType["schedule_end"].(string),
}); err != nil {
glog.Errorf("Failed to execute template: %s", err)
return
}
emailSubject := fmt.Sprintf("%s is the next %s", sheriffUsername, shiftType.shiftName)
if err := sendEmail([]string{sheriffEmail, EXTRA_RECIPIENT}, emailSubject, emailBytes.String()); err != nil {
glog.Fatalf("Error sending email to sheriff: %s", err)
}
}
}
// Call once at startup to read requestSalt from the given file.
func MustInitRequestSaltFromFile(filename string) {
saltBase64Bytes, err := ioutil.ReadFile(filename)
if err != nil {
glog.Fatalf("Could not read the webhook request salt file: %s", err)
}
if err = setRequestSaltFromBase64(saltBase64Bytes); err != nil {
glog.Fatalf("Could not decode salt from %s: %v", filename, err)
}
}
func md5Commits(store tiling.TileStore, targetHash string, nCommits int) {
tile, err := store.Get(0, -1)
if err != nil {
glog.Fatal("Could not read tile: " + err.Error())
}
tileLen := tile.LastCommitIndex() + 1
commits := tile.Commits[:tileLen]
// Find the target index.
endIdx := -1
for i, v := range commits {
if strings.HasPrefix(v.Hash, targetHash) {
endIdx = i
break
}
}
if endIdx == -1 {
glog.Fatalf("Unable to find commit %s", targetHash)
}
endIdx++
startIdx := endIdx - nCommits
traceKeys := make([]string, 0, len(tile.Traces))
for k, v := range tile.Traces {
for i := startIdx; i < endIdx; i++ {
if !v.IsMissing(i) {
traceKeys = append(traceKeys, k)
break
}
}
}
sort.Strings(traceKeys)
result := make([][]string, len(traceKeys))
for i, k := range traceKeys {
switch trace := tile.Traces[k].(type) {
case *gtypes.GoldenTrace:
result[i] = trace.Values[startIdx:endIdx]
case *types.PerfTrace:
result[i] = asStringSlice(trace.Values[startIdx:endIdx])
}
}
byteStr, err := getBytes(result)
if err != nil {
glog.Fatalf("Unable to serialize to bytes: %s", err.Error())
}
md5Hash := fmt.Sprintf("%x", md5.Sum(byteStr))
fmt.Printf("Commit Range : %s - %s\n", commits[startIdx].Hash, commits[endIdx-1].Hash)
fmt.Printf("Hash : %s\n", md5Hash)
fmt.Printf("Total traces: %d\n", len(tile.Traces))
fmt.Printf("Non-empty traces: %d\n", len(traceKeys))
}
func getURL(router *mux.Router, name string, pairs ...string) string {
route := router.Get(name)
if route == nil {
glog.Fatalf("Couldn't find any route named %s", name)
}
routeURL, err := route.URL(pairs...)
if err != nil {
glog.Fatalf("Couldn't resolve route %s into a URL", routeURL)
}
return routeURL.String()
}
func main() {
defer common.LogPanic()
// Calls flag.Parse()
common.InitWithMetrics("fuzzer", graphiteServer)
if err := writeFlagsToConfig(); err != nil {
glog.Fatalf("Problem with configuration: %s", err)
}
Init()
if err := setupOAuth(); err != nil {
glog.Fatal(err)
}
go func() {
if err := fcommon.DownloadSkiaVersionForFuzzing(storageClient, config.FrontEnd.SkiaRoot, &config.FrontEnd); err != nil {
glog.Fatalf("Problem downloading Skia: %s", err)
}
fuzzSyncer = syncer.New(storageClient)
fuzzSyncer.Start()
cache, err := fuzzcache.New(config.FrontEnd.BoltDBPath)
if err != nil {
glog.Fatalf("Could not create fuzz report cache at %s: %s", config.FrontEnd.BoltDBPath, err)
}
defer util.Close(cache)
if err := gsloader.LoadFromBoltDB(cache); err != nil {
glog.Errorf("Could not load from boltdb. Loading from source of truth anyway. %s", err)
}
var finder functionnamefinder.Finder
if !*local {
finder = functionnamefinder.NewAsync()
}
gsLoader := gsloader.New(storageClient, finder, cache)
if err := gsLoader.LoadFreshFromGoogleStorage(); err != nil {
glog.Fatalf("Error loading in data from GCS: %s", err)
}
fuzzSyncer.SetGSLoader(gsLoader)
updater := frontend.NewVersionUpdater(gsLoader, fuzzSyncer)
versionWatcher = fcommon.NewVersionWatcher(storageClient, config.FrontEnd.VersionCheckPeriod, updater.HandlePendingVersion, updater.HandleCurrentVersion)
versionWatcher.Start()
err = <-versionWatcher.Status
glog.Fatal(err)
}()
runServer()
}
func cleanupAppLogs(dir string, appLogLevelToSpace map[string]int64, filesToState map[string]fileState) bool {
deletedFiles := false
for appLogLevel := range appLogLevelToSpace {
if appLogLevelToSpace[appLogLevel] > *appLogThreshold {
glog.Infof("App %s is above the threshold. Usage: %d. Threshold: %d", appLogLevel, appLogLevelToSpace[appLogLevel], *appLogThreshold)
tokens := strings.Split(appLogLevel, ".")
app := tokens[0]
logLevel := tokens[1]
logGlob := filepath.Join(dir, app+".*"+logLevel+".*")
matches, err := filepath.Glob(logGlob)
if err != nil {
glog.Fatalf("Could not glob for %s: %s", logGlob, err)
}
fileInfos := make([]os.FileInfo, len(matches))
for i, match := range matches {
fileInfo, err := os.Stat(match)
if err != nil {
glog.Fatalf("Could not stat %s: %s", match, err)
}
fileInfos[i] = fileInfo
}
// Sort by Modified time and keep deleting till we are at
// (threshold - buffer) space left.
sort.Sort(FileInfoModifiedSlice{fileInfos: fileInfos, reverseSort: false})
index := 0
for appLogLevelToSpace[appLogLevel] > *appLogThreshold-*appLogThresholdBuffer {
if index+1 == len(fileInfos) {
glog.Warningf("App %s is above the threshold and has only one file remaining: %s. Not deleting it.", appLogLevel, fileInfos[index].Name())
break
}
fileName := fileInfos[index].Name()
appLogLevelToSpace[appLogLevel] -= fileInfos[index].Size()
if err = os.Remove(filepath.Join(dir, fileName)); err != nil {
glog.Fatalf("Could not delete %s: %s", fileName, err)
}
// Remove the entry from the filesToState map.
delete(filesToState, filepath.Join(dir, fileName))
deletedFiles = true
glog.Infof("Deleted %s", fileName)
index++
}
// Just incase we delete a massive log file.
if appLogLevelToSpace[appLogLevel] < 0 {
appLogLevelToSpace[appLogLevel] = 0
}
}
}
return deletedFiles
}
func Init() {
var err error
dbc, err = dbus.New()
if err != nil {
glog.Fatalf("Failed to initialize dbus: %s", err)
}
hostname, err = os.Hostname()
if err != nil {
glog.Fatalf("Unable to retrieve hostname: %s", err)
}
loadResouces()
}
func main() {
common.Init()
grpclog.Init()
// Load the 0,-1 tile.
tilestore := filetilestore.NewFileTileStore(*tilestore, *dataset, time.Hour)
tile, err := tilestore.Get(0, -1)
if err != nil {
glog.Fatalf("Failed to load tile: %s", err)
}
// Trim to the last 50 commits.
begin := 0
end := tile.LastCommitIndex()
if end >= 49 {
begin = end - 49
}
glog.Infof("Loaded Tile")
tile, err = tile.Trim(begin, end)
// Set up a connection to the server.
conn, err := grpc.Dial(*address, grpc.WithInsecure())
if err != nil {
glog.Fatalf("did not connect: %v", err)
}
defer util.Close(conn)
// Build a TraceService client.
builder := ptypes.PerfTraceBuilder
if *gold {
builder = types.GoldenTraceBuilder
}
ts, err := db.NewTraceServiceDB(conn, builder)
if err != nil {
log.Fatalf("Failed to create db.DB: %s", err)
}
glog.Infof("Opened tracedb")
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
glog.Fatalf("Failed to open profiling file: %s", err)
}
if err := pprof.StartCPUProfile(f); err != nil {
glog.Fatalf("Failed to start profiling: %s", err)
}
defer pprof.StopCPUProfile()
_main(tile, ts)
} else {
_main(tile, ts)
}
}
func main() {
defer common.LogPanic()
// Set up flags.
dbConf := db.DBConfigFromFlags()
// Global init to initialize glog and parse arguments.
common.Init()
if *promptPassword {
if err := dbConf.PromptForPassword(); err != nil {
glog.Fatal(err)
}
}
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatal(err)
}
if *targetVersion < 0 {
// Get the current database version
maxDBVersion := vdb.MaxDBVersion()
glog.Infof("Latest database version: %d", maxDBVersion)
dbVersion, err := vdb.DBVersion()
if err != nil {
glog.Fatalf("Unable to retrieve database version. Error: %s", err)
}
glog.Infof("Current database version: %d", dbVersion)
if dbVersion < maxDBVersion {
glog.Infof("Migrating to version: %d", maxDBVersion)
err = vdb.Migrate(maxDBVersion)
if err != nil {
glog.Fatalf("Unable to retrieve database version. Error: %s", err)
}
}
} else {
glog.Infof("Migrating to version: %d", *targetVersion)
err = vdb.Migrate(*targetVersion)
if err != nil {
glog.Fatalf("Unable to retrieve database version. Error: %s", err)
}
}
glog.Infoln("Database migration finished.")
}
func main() {
defer common.LogPanic()
common.InitWithMetrics("grandcentral", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
if *nsqdAddress == "" {
glog.Fatal("Missing address of nsqd server.")
}
globalEventBus, err := geventbus.NewNSQEventBus(*nsqdAddress)
if err != nil {
glog.Fatalf("Unable to connect to NSQ server: %s", err)
}
eventBus = eventbus.New(globalEventBus)
if *testing {
*useMetadata = false
}
serverURL := "https://" + *host
if *testing {
serverURL = "http://" + *host + *port
}
runServer(serverURL)
}
func main() {
common.Init()
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
if *nsqdAddress == "" {
glog.Fatal("Missing address of nsqd server.")
}
globalEventBus, err := geventbus.NewNSQEventBus(*nsqdAddress)
if err != nil {
glog.Fatalf("Unable to connect to NSQ server: %s", err)
}
eventBus := eventbus.New(globalEventBus)
// Send events every so often.
for _ = range time.Tick(2 * time.Second) {
evData := &event.GoogleStorageEventData{
Bucket: "test-bucket",
Name: "test-name",
Updated: time.Now().String(),
}
eventBus.Publish(event.GLOBAL_GOOGLE_STORAGE, evData)
glog.Infof("Sent Event: %#v ", evData)
}
}
func main() {
common.InitWithMetrics("certpoller", graphiteServer)
client := util.NewTimeoutClient()
certs := []*cert{}
// Populate certs based on cmd-line args.
for _, metadata := range flag.Args() {
c := &cert{
metadata: metadata,
file: fileFromMetadata(metadata),
etag: "",
}
err := get(client, c)
if err != nil {
glog.Fatalf("Failed to retrieve the cert %s: %s", c, err)
}
certs = append(certs, c)
}
for _ = range time.Tick(30 * time.Minute) {
for _, c := range certs {
if err := get(client, c); err != nil {
glog.Errorf("Failed to update cert %s: %s", c.metadata, err)
}
}
}
}
func main() {
common.Init()
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
if *nsqdAddress == "" {
glog.Fatal("Missing address of nsqd server.")
}
globalEventBus, err := geventbus.NewNSQEventBus(*nsqdAddress)
if err != nil {
glog.Fatalf("Unable to connect to NSQ server: %s", err)
}
eventBus := eventbus.New(globalEventBus)
eventBus.SubscribeAsync(event.GLOBAL_GOOGLE_STORAGE, func(evData interface{}) {
data := evData.(*event.GoogleStorageEventData)
glog.Infof("Google Storage notification from bucket\n %s: %s : %s", data.Updated, data.Bucket, data.Name)
})
select {}
}
// NewIngestionProcess creates a Process for ingesting data.
func NewIngestionProcess(git *gitinfo.GitInfo, tileDir, datasetName string, ri ingester.ResultIngester, config map[string]string, every time.Duration, nCommits int, minDuration time.Duration, statusDir, metricName string) ProcessStarter {
return func() {
i, err := ingester.NewIngester(git, tileDir, datasetName, ri, nCommits, minDuration, config, statusDir, metricName)
if err != nil {
glog.Fatalf("Failed to create Ingester: %s", err)
}
glog.Infof("Starting %s ingester. Run every %s.", datasetName, every.String())
// oneStep is a single round of ingestion.
oneStep := func() {
glog.Infof("Running ingester: %s", datasetName)
err := i.Update()
if err != nil {
glog.Error(err)
}
glog.Infof("Finished running ingester: %s", datasetName)
}
// Start the ingester.
go func() {
oneStep()
for _ = range time.Tick(every) {
oneStep()
}
}()
}
}
func main() {
defer common.LogPanic()
common.Init()
args := flag.Args()
if len(args) != 2 {
flag.Usage()
os.Exit(1)
}
bucket, prefix := args[0], args[1]
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
if *nsqdAddress == "" {
glog.Fatal("Missing address of nsqd server.")
}
globalEventBus, err := geventbus.NewNSQEventBus(*nsqdAddress)
if err != nil {
glog.Fatalf("Unable to connect to NSQ server: %s", err)
}
eventBus := eventbus.New(globalEventBus)
eventBus.SubscribeAsync(event.StorageEvent(bucket, prefix), func(evData interface{}) {
data := evData.(*event.GoogleStorageEventData)
glog.Infof("Google Storage notification from bucket\n %s: %s : %s", data.Updated, data.Bucket, data.Name)
})
select {}
}
func main() {
flag.Usage = printUsage
common.Init()
args := flag.Args()
if len(args) == 0 {
printUsage()
os.Exit(1)
}
store := filetilestore.NewFileTileStore(*tileDir, *dataset, 0)
switch args[0] {
case VALIDATE:
if !validator.ValidateDataset(store, *verbose, *echoHashes) {
glog.Fatal("FAILED Validation.")
}
case DUMP_COMMITS:
checkArgs(args, DUMP_COMMITS, 1)
nCommits := parseInt(args[1])
dumpCommits(store, nCommits)
case MD5:
checkArgs(args, MD5, 2)
hash := args[1]
nCommits := parseInt(args[2])
md5Commits(store, hash, nCommits)
case JSON:
checkArgs(args, JSON, 3)
nCommits := parseInt(args[1])
nTraces := parseInt(args[2])
fname := args[3]
dumpTileToJSON(store, nCommits, nTraces, fname)
default:
glog.Fatalf("Unknow command: %s", args[0])
}
}
func parseInt(nStr string) int {
ret, err := strconv.ParseInt(nStr, 10, 0)
if err != nil {
glog.Fatalf("ERROR: %s", err.Error())
}
return int(ret)
}
// issueToRollKey converts an issue number to a BoltDB key.
func issueToRollKey(issue int64) []byte {
var buf bytes.Buffer
if err := binary.Write(&buf, binary.LittleEndian, issue); err != nil {
glog.Fatalf("Failed to serialize int64: %d", issue)
}
return buf.Bytes()
}
// Returns the offset into the color slices (PixelDiffColor,
// or PixelAlphaDiffColor) based on the delta passed in.
//
// The number passed in is the difference between two colors,
// on a scale from 1 to 1024.
func deltaOffset(n int) int {
ret := int(math.Ceil(math.Log(float64(n))/math.Log(3) + 0.5))
if ret < 1 || ret > 7 {
glog.Fatalf("Input: %d", n)
}
return ret - 1
}