Beispiel #1
0
// NewPushBuffer returns PushBuffer that's ready to accept log entries.
func NewPushBuffer(opts PushBufferOptions) PushBuffer {
	if opts.Logger == nil {
		opts.Logger = logging.Null()
	}
	if opts.Clock == nil {
		opts.Clock = clock.GetSystemClock()
	}
	if opts.FlushThreshold == 0 {
		opts.FlushThreshold = DefaultFlushThreshold
	}
	if opts.FlushTimeout == 0 {
		opts.FlushTimeout = DefaultFlushTimeout
	}
	if opts.MaxPushAttempts == 0 {
		opts.MaxPushAttempts = DefaultMaxPushAttempts
	}
	if opts.PushRetryDelay == 0 {
		opts.PushRetryDelay = DefaultPushRetryDelay
	}
	if opts.StopTimeout == 0 {
		opts.StopTimeout = DefaultStopTimeout
	}
	buf := &pushBufferImpl{
		PushBufferOptions: opts,
		input:             make(chan []Entry),
		output:            make(chan error),
		timer:             opts.Clock.NewTimer(),
		stopCh:            make(chan struct{}, 1),
	}
	go buf.loop()
	return buf
}
Beispiel #2
0
func main() {
	start := time.Now()
	flag.Parse()

	duration, err := time.ParseDuration(*durationStr)
	if err != nil {
		log.Errorf("Error parsing duration: %v", err)
		os.Exit(1)
	}

	cycle, err := time.ParseDuration(*cycleStr)
	if err != nil {
		log.Errorf("Error parsing cycle: %v", err)
		os.Exit(1)
	}

	err = readJSONFile(*gatekeeperJSON, &gk)
	if err != nil {
		log.Errorf("Error reading gatekeeper json: %v", err)
		os.Exit(1)
	}

	err = readJSONFile(*gatekeeperTreesJSON, &gkt)
	if err != nil {
		log.Errorf("Error reading gatekeeper json: %v", err)
		os.Exit(1)
	}

	if *snapshot != "" && *replay != "" {
		log.Errorf("Cannot use snapshot and replay flags at the same time.")
		os.Exit(1)
	}

	r := client.NewReader()
	if *snapshot != "" {
		r = client.NewSnapshot(r, *snapshot)
	}

	if *replay != "" {
		r = client.NewReplay(*replay)
	}

	a := analyzer.New(r, 5, 100)

	for masterURL, masterCfgs := range gk.Masters {
		if len(masterCfgs) != 1 {
			log.Errorf("Multiple configs for master: %s", masterURL)
		}
		masterName := masterFromURL(masterURL)
		a.MasterCfgs[masterName] = masterCfgs[0]
	}

	a.MasterOnly = *masterOnly
	a.BuilderOnly = *builderOnly
	a.BuildOnly = *buildOnly

	trees := map[string]bool{}
	if *treesOnly != "" {
		for _, treeOnly := range strings.Split(*treesOnly, ",") {
			trees[treeOnly] = true
		}
	} else {
		for treeName := range gkt {
			trees[treeName] = true
		}
	}

	for tree := range trees {
		if _, ok := gkt[tree]; !ok {
			log.Errorf("Unrecognized tree name: %s", tree)
			os.Exit(1)
		}
	}

	// This is the polling/analysis/alert posting function, which will run in a loop until
	// a timeout or max errors is reached.
	f := func(ctx context.Context) error {
		done := make(chan interface{})
		errs := make(chan error)
		for treeName := range trees {
			go func(tree string) {
				log.Infof("Checking tree: %s", tree)
				masterNames := []string{}
				t := gkt[tree]
				for _, url := range t.Masters {
					masterNames = append(masterNames, masterFromURL(url))
				}

				// TODO(seanmccullough): Plumb ctx through the rest of these calls.
				bes := fetchBuildExtracts(a.Reader, masterNames)
				log.Infof("Build Extracts read: %d", len(bes))

				alerts := &messages.Alerts{}
				for masterName, be := range bes {
					alerts.Alerts = append(alerts.Alerts, analyzeBuildExtract(a, masterName, be)...)
				}
				alerts.Timestamp = messages.TimeToEpochTime(time.Now())

				if *alertsBaseURL == "" {
					log.Infof("No data_url provided. Writing to %s-alerts.json", tree)

					abytes, err := json.MarshalIndent(alerts, "", "\t")
					if err != nil {
						log.Errorf("Couldn't marshal alerts json: %v", err)
						errs <- err
						return
					}

					if err := ioutil.WriteFile(fmt.Sprintf("%s-alerts.json", tree), abytes, 0644); err != nil {
						log.Errorf("Couldn't write to alerts.json: %v", err)
						errs <- err
						return
					}
				} else {
					alertsURL := fmt.Sprintf("%s/%s", *alertsBaseURL, tree)
					w := client.NewWriter(alertsURL)
					log.Infof("Posting alerts to %s", alertsURL)
					err := w.PostAlerts(alerts)
					if err != nil {
						log.Errorf("Couldn't post alerts: %v", err)
						errs <- err
						return
					}
				}

				log.Infof("Filtered failures: %v", filteredFailures)
				a.Reader.DumpStats()
				log.Infof("Elapsed time: %v", time.Since(start))
				done <- nil
			}(treeName)
		}

		for range trees {
			select {
			case err := <-errs:
				return err
			case <-done:
			}
		}
		return nil
	}

	ctx, cancel := context.WithTimeout(context.Background(), duration)
	logging.Set(ctx, log)
	defer cancel()
	loopResults := looper.Run(ctx, f, cycle, *maxErrs, clock.GetSystemClock())

	if !loopResults.Success {
		log.Errorf("Failed to run loop, %v errors", loopResults.Errs)
		os.Exit(1)
	}
}