Пример #1
0
// New creates a new logger that logs to Google Cloud Logging using the application
// default credentials.
//
// See https://developers.google.com/identity/protocols/application-default-credentials
func New(ctx logger.Context) (logger.Logger, error) {
	initGCP()

	var project string
	if projectID != "" {
		project = projectID
	}
	if projectID, found := ctx.Config[projectOptKey]; found {
		project = projectID
	}
	if project == "" {
		return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project")
	}

	c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver")
	if err != nil {
		return nil, err
	}

	if err := c.Ping(); err != nil {
		return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
	}

	l := &gcplogs{
		client: c,
		container: &containerInfo{
			Name:      ctx.ContainerName,
			ID:        ctx.ContainerID,
			ImageName: ctx.ContainerImageName,
			ImageID:   ctx.ContainerImageID,
			Created:   ctx.ContainerCreated,
			Metadata:  ctx.ExtraAttributes(nil),
		},
	}

	if ctx.Config[logCmdKey] == "true" {
		l.container.Command = ctx.Command()
	}

	if onGCE {
		l.instance = &instanceInfo{
			Zone: zone,
			Name: instanceName,
			ID:   instanceID,
		}
	}

	// The logger "overflows" at a rate of 10,000 logs per second and this
	// overflow func is called. We want to surface the error to the user
	// without overly spamming /var/log/docker.log so we log the first time
	// we overflow and every 1000th time after.
	c.Overflow = func(_ *logging.Client, _ logging.Entry) error {
		if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 {
			logrus.Errorf("gcplogs driver has dropped %v logs", i)
		}
		return nil
	}

	return l, nil
}
Пример #2
0
// LogWriter returns an environment-specific io.Writer suitable for passing
// to log.SetOutput. It will also include writing to os.Stderr as well.
func LogWriter() (w io.Writer) {
	w = os.Stderr
	if !env.OnGCE() {
		return
	}
	projID, err := metadata.ProjectID()
	if projID == "" {
		log.Printf("Error getting project ID: %v", err)
		return
	}
	scopes, _ := metadata.Scopes("default")
	haveScope := func(scope string) bool {
		for _, x := range scopes {
			if x == scope {
				return true
			}
		}
		return false
	}
	if !haveScope(logging.Scope) {
		log.Printf("when this Google Compute Engine VM instance was created, it wasn't granted enough access to use Google Cloud Logging (Scope URL: %v).", logging.Scope)
		return
	}

	logc, err := logging.NewClient(context.Background(), projID, "camlistored-stderr")
	if err != nil {
		log.Printf("Error creating Google logging client: %v", err)
		return
	}
	return io.MultiWriter(w, logc.Writer(logging.Debug))
}
Пример #3
0
func NewClient(ctx context.Context, projectID, serviceName, serviceVersion string, opts ...cloud.ClientOption) (*Client, error) {
	l, err := logging.NewClient(ctx, projectID, "errorreports", opts...)
	if err != nil {
		return nil, fmt.Errorf("creating Logging client: %v", err)
	}
	c := &Client{
		loggingClient:  l,
		projectID:      projectID,
		RepanicDefault: true,
		serviceContext: map[string]string{
			"service": serviceName,
		},
	}
	if serviceVersion != "" {
		c.serviceContext["version"] = serviceVersion
	}
	return c, nil
}
Пример #4
0
// LogWriter returns an environment-specific io.Writer suitable for passing
// to log.SetOutput. It will also include writing to os.Stderr as well.
func LogWriter() (w io.Writer) {
	w = os.Stderr
	if !env.OnGCE() {
		return
	}
	projID, err := metadata.ProjectID()
	if projID == "" {
		log.Printf("Error getting project ID: %v", err)
		return
	}
	hc, err := google.DefaultClient(oauth2.NoContext)
	if err != nil {
		log.Printf("Error creating default GCE OAuth2 client: %v", err)
		return
	}
	logc, err := logging.NewClient(cloud.NewContext(projID, hc), "camlistored-stderr")
	if err != nil {
		log.Printf("Error creating Google logging client: %v", err)
		return
	}
	return io.MultiWriter(w, logc.Writer(logging.Debug))
}
Пример #5
0
func maybeSetupGoogleCloudLogging() {
	if flagGCEProjectID == "" && flagGCELogName == "" && flagGCEJWTFile == "" {
		return
	}
	if flagGCEProjectID == "" || flagGCELogName == "" || flagGCEJWTFile == "" {
		exitf("All of --gce_project_id, --gce_log_name, and --gce_jwt_file must be specified for logging on Google Cloud Logging.")
	}
	jsonSlurp, err := ioutil.ReadFile(flagGCEJWTFile)
	if err != nil {
		exitf("Error reading --gce_jwt_file value: %v", err)
	}
	jwtConf, err := google.JWTConfigFromJSON(jsonSlurp, logging.Scope)
	if err != nil {
		exitf("Error reading --gce_jwt_file value: %v", err)
	}
	ctx := cloud.NewContext(flagGCEProjectID, jwtConf.Client(context.Background()))
	logc, err := logging.NewClient(ctx, flagGCEProjectID, flagGCELogName)
	if err != nil {
		exitf("Error creating GCL client: %v", err)
	}
	log.SetOutput(io.MultiWriter(os.Stderr, logc.Writer(logging.Debug)))
}
Пример #6
0
func handle(w http.ResponseWriter, r *http.Request) {
	c, _ := cloudAuthContext(r)
	logc, err := logging.NewClient(c, appengine.AppID(c), "javascript.errors")
	if err != nil {
		http.Error(w, "Cannot connect to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot connect to Google Cloud Logging: %v", err)
		return
	}

	// Note: Error Reporting currently ignores non-GCE and non-AWS logs.
	logc.ServiceName = "compute.googleapis.com"
	logc.CommonLabels = map[string]string{
		"compute.googleapis.com/resource_type": "logger",
		"compute.googleapis.com/resource_id":   "errors"}

	// Fill query params into JSON struct.
	line, _ := strconv.Atoi(r.URL.Query().Get("l"))
	errorType := "default"
	if r.URL.Query().Get("a") == "1" {
		errorType = "assert"
	}
	// By default we log as "INFO" severity, because reports are very spammy
	severity := "INFO"
	level := logging.Info
	// But if the request comes from the cache (and thus only from valid AMP
	// docs) we log as "ERROR".
	if strings.HasPrefix(r.Referer(), "https://cdn.ampproject.org/") {
		severity = "ERROR"
		level = logging.Error
		errorType += "-cdn"
	}

	event := &ErrorEvent{
		Message:     r.URL.Query().Get("m"),
		Exception:   r.URL.Query().Get("s"),
		Version:     r.URL.Query().Get("v"),
		Environment: "prod",
		Application: errorType,
		AppID:       appengine.AppID(c),
		Filename:    r.URL.Query().Get("f"),
		Line:        int32(line),
		Classname:   r.URL.Query().Get("el"),
		Severity:    severity,
	}

	if event.Message == "" && event.Exception == "" {
		http.Error(w, "One of 'message' or 'exception' must be present.",
			http.StatusBadRequest)
		log.Errorf(c, "Malformed request: %v", event)
		return
	}

	// Don't log testing traffic in production
	if event.Version == "$internalRuntimeVersion$" {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	event.Request = &ErrorRequest{
		URL: r.Referer(),
	}
	event.Request.Meta = &ErrorRequestMeta{
		HTTPReferrer:  r.Referer(),
		HTTPUserAgent: r.UserAgent(),
		// Intentionally not logged.
		// RemoteIP:   r.RemoteAddr,
	}

	err = logc.LogSync(logging.Entry{
		Time:    time.Now().UTC(),
		Payload: event,
		Level:   level,
	})

	if err != nil {
		http.Error(w, "Cannot write to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot write to Google Cloud Logging: %v", err)
		return
	}

	// When debug param is present, return a document. This is nicer because
	// browsers otherwise revert the URL during manual testing.
	if r.URL.Query().Get("debug") == "1" {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		fmt.Fprintln(w, "OK\n")
		fmt.Fprintln(w, event)
	} else {
		w.WriteHeader(http.StatusNoContent)
	}
}
Пример #7
0
func main() {
	flag.Parse()

	if *root == "" {
		var err error
		*root, err = os.Getwd()
		if err != nil {
			log.Fatalf("Failed to getwd: %v", err)
		}
	}
	readTemplates()

	mux := http.DefaultServeMux
	mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
	mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
	mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static")))))
	mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks")))))
	mux.Handle(pkgPattern, godocHandler{})
	mux.Handle(cmdPattern, godocHandler{})
	mux.HandleFunc(errPattern, errHandler)

	mux.HandleFunc("/r/", gerritRedirect)
	mux.HandleFunc("/dl/", releaseRedirect)
	mux.HandleFunc("/debugz/ip", ipHandler)
	mux.Handle("/docs/contributing", redirTo("/code#contributing"))
	mux.Handle("/lists", redirTo("/community"))

	mux.HandleFunc("/contributors", contribHandler())
	mux.HandleFunc("/", mainHandler)

	if *buildbotHost != "" && *buildbotBackend != "" {
		buildbotUrl, err := url.Parse(*buildbotBackend)
		if err != nil {
			log.Fatalf("Failed to parse %v as a URL: %v", *buildbotBackend, err)
		}
		buildbotHandler := httputil.NewSingleHostReverseProxy(buildbotUrl)
		bbhpattern := strings.TrimRight(*buildbotHost, "/") + "/"
		mux.Handle(bbhpattern, buildbotHandler)
	}

	if *httpsAddr != "" {
		if launcher := gceDeployHandler("/launch/"); launcher != nil {
			mux.Handle("/launch/", launcher)
		}
	}

	var handler http.Handler = &noWwwHandler{Handler: mux}
	if *logDir != "" || *logStdout {
		handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout))
	}
	if *gceLogName != "" {
		projID := *gceProjectID
		if projID == "" {
			if v, err := metadata.ProjectID(); v == "" || err != nil {
				log.Fatalf("Use of --gce_log_name without specifying --gce_project_id (and not running on GCE); metadata error: %v", err)
			} else {
				projID = v
			}
		}
		var hc *http.Client
		if *gceJWTFile != "" {
			jsonSlurp, err := ioutil.ReadFile(*gceJWTFile)
			if err != nil {
				log.Fatalf("Error reading --gce_jwt_file value: %v", err)
			}
			jwtConf, err := google.JWTConfigFromJSON(jsonSlurp, logging.Scope)
			if err != nil {
				log.Fatalf("Error reading --gce_jwt_file value: %v", err)
			}
			hc = jwtConf.Client(context.Background())
		} else {
			if !metadata.OnGCE() {
				log.Fatal("No --gce_jwt_file and not running on GCE.")
			}
			var err error
			hc, err = google.DefaultClient(oauth2.NoContext)
			if err != nil {
				log.Fatal(err)
			}
		}
		ctx := cloud.NewContext(projID, hc)
		logc, err := logging.NewClient(ctx, projID, *gceLogName)
		if err != nil {
			log.Fatal(err)
		}
		if err := logc.Ping(); err != nil {
			log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
		}
		handler = NewLoggingHandler(handler, gceLogger{logc})
	}

	errc := make(chan error)
	startEmailCommitLoop(errc)

	if *alsoRun != "" {
		runAsChild(*alsoRun)
	}

	httpServer := &http.Server{
		Addr:         *httpAddr,
		Handler:      handler,
		ReadTimeout:  5 * time.Minute,
		WriteTimeout: 30 * time.Minute,
	}
	go func() {
		errc <- httpServer.ListenAndServe()
	}()

	if *httpsAddr != "" {
		log.Printf("Starting TLS server on %s", *httpsAddr)
		httpsServer := new(http.Server)
		*httpsServer = *httpServer
		httpsServer.Addr = *httpsAddr
		go func() {
			errc <- httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile)
		}()
	}

	log.Fatalf("Serve error: %v", <-errc)
}
Пример #8
0
func main() {
	launchConfig.MaybeDeploy()
	flag.Parse()
	setProdFlags()

	if *root == "" {
		var err error
		*root, err = os.Getwd()
		if err != nil {
			log.Fatalf("Failed to getwd: %v", err)
		}
	}
	readTemplates()
	go runDemoBlobserverLoop()

	mux := http.DefaultServeMux
	mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
	mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
	mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static")))))
	mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks")))))
	mux.Handle(pkgPattern, godocHandler{})
	mux.Handle(cmdPattern, godocHandler{})
	mux.HandleFunc(errPattern, errHandler)

	mux.HandleFunc("/r/", gerritRedirect)
	mux.HandleFunc("/dl/", releaseRedirect)
	mux.HandleFunc("/debug/ip", ipHandler)
	mux.HandleFunc("/debug/uptime", uptimeHandler)
	mux.Handle("/docs/contributing", redirTo("/code#contributing"))
	mux.Handle("/lists", redirTo("/community"))

	mux.HandleFunc("/contributors", contribHandler())
	mux.HandleFunc("/", mainHandler)

	if *buildbotHost != "" && *buildbotBackend != "" {
		buildbotUrl, err := url.Parse(*buildbotBackend)
		if err != nil {
			log.Fatalf("Failed to parse %v as a URL: %v", *buildbotBackend, err)
		}
		buildbotHandler := httputil.NewSingleHostReverseProxy(buildbotUrl)
		bbhpattern := strings.TrimRight(*buildbotHost, "/") + "/"
		mux.Handle(bbhpattern, buildbotHandler)
	}

	// ctx initialized now, because gceLauncher needs it first (when in prod).
	// Other users are the GCE logger, and serveHTTPS (in prod).
	var ctx context.Context
	var projID string
	if inProd || *gceLogName != "" {
		projID = projectID()
		ctx = ctxt(projID)
	}

	gceLauncher, err := gceDeployHandler(ctx, "/launch/")
	if err != nil {
		log.Printf("Not installing GCE /launch/ handler: %v", err)
		mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) {
			http.Error(w, fmt.Sprintf("GCE launcher disabled: %v", err), 500)
		})
	} else {
		mux.Handle("/launch/", gceLauncher)
	}

	var handler http.Handler = &noWwwHandler{Handler: mux}
	if *logDir != "" || *logStdout {
		handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout))
	}
	if *gceLogName != "" {
		logc, err := logging.NewClient(ctx, projID, *gceLogName)
		if err != nil {
			log.Fatal(err)
		}
		if err := logc.Ping(); err != nil {
			log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
		}
		handler = NewLoggingHandler(handler, gceLogger{logc})
		if gceLauncher != nil {
			logc, err := logging.NewClient(ctx, projID, *gceLogName)
			if err != nil {
				log.Fatal(err)
			}
			logc.CommonLabels = map[string]string{
				"from": "camli-gce-launcher",
			}
			logger := logc.Logger(logging.Default)
			logger.SetPrefix("launcher: ")
			gceLauncher.SetLogger(logger)
		}
	}

	emailErr := make(chan error)
	startEmailCommitLoop(emailErr)

	if *alsoRun != "" {
		runAsChild(*alsoRun)
	}

	httpServer := &http.Server{
		Addr:         *httpAddr,
		Handler:      handler,
		ReadTimeout:  5 * time.Minute,
		WriteTimeout: 30 * time.Minute,
	}

	httpErr := make(chan error)
	go func() {
		log.Printf("Listening for HTTP on %v", *httpAddr)
		httpErr <- httpServer.ListenAndServe()
	}()

	httpsErr := make(chan error)
	if *httpsAddr != "" {
		go func() {
			httpsErr <- serveHTTPS(ctx, httpServer)
		}()
	}

	if *flagChromeBugRepro {
		go func() {
			log.Printf("Repro handler failed: %v", repro(":8001", "foo:bar"))
		}()
	}

	select {
	case err := <-emailErr:
		log.Fatalf("Error sending emails: %v", err)
	case err := <-httpErr:
		log.Fatalf("Error serving HTTP: %v", err)
	case err := <-httpsErr:
		log.Fatalf("Error serving HTTPS: %v", err)
	}
}
Пример #9
0
func handle(w http.ResponseWriter, r *http.Request) {
	c, _ := cloudAuthContext(r)
	logc, err := logging.NewClient(c, appengine.AppID(c), "javascript.errors")
	if err != nil {
		http.Error(w, "Cannot connect to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot connect to Google Cloud Logging: %v", err)
		return
	}

	// Note: Error Reporting currently ignores non-GCE and non-AWS logs.
	logc.ServiceName = "compute.googleapis.com"
	logc.CommonLabels = map[string]string{
		"compute.googleapis.com/resource_type": "logger",
		"compute.googleapis.com/resource_id":   "errors"}

	// Fill query params into JSON struct.
	line, _ := strconv.Atoi(r.URL.Query().Get("l"))

	event := &ErrorEvent{
		Message:     r.URL.Query().Get("m"),
		Exception:   r.URL.Query().Get("s"),
		Version:     r.URL.Query().Get("v"),
		Environment: "prod",
		Application: appengine.ModuleName(c),
		AppID:       appengine.AppID(c),
		Filename:    r.URL.Query().Get("f"),
		Line:        int32(line),
		Classname:   r.URL.Query().Get("el"),
	}

	if event.Message == "" && event.Exception == "" {
		http.Error(w, "One of 'message' or 'exception' must be present.",
			http.StatusBadRequest)
		log.Errorf(c, "Malformed request: %v", event)
		return
	}

	event.Request = &ErrorRequest{
		URL: r.Referer(),
	}
	event.Request.Meta = &ErrorRequestMeta{
		HTTPReferrer:  r.Referer(),
		HTTPUserAgent: r.UserAgent(),
		// Intentionally not logged.
		// RemoteIP:   r.RemoteAddr,
	}

	err = logc.LogSync(logging.Entry{
		Time:    time.Now().UTC(),
		Payload: event,
	})

	if err != nil {
		http.Error(w, "Cannot write to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot write to Google Cloud Logging: %v", err)
		return
	}

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	fmt.Fprintln(w, "OK")
}