// New creates a new logger that logs to Google Cloud Logging using the application // default credentials. // // See https://developers.google.com/identity/protocols/application-default-credentials func New(ctx logger.Context) (logger.Logger, error) { initGCP() var project string if projectID != "" { project = projectID } if projectID, found := ctx.Config[projectOptKey]; found { project = projectID } if project == "" { return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project") } c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver") if err != nil { return nil, err } if err := c.Ping(); err != nil { return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err) } l := &gcplogs{ client: c, container: &containerInfo{ Name: ctx.ContainerName, ID: ctx.ContainerID, ImageName: ctx.ContainerImageName, ImageID: ctx.ContainerImageID, Created: ctx.ContainerCreated, Metadata: ctx.ExtraAttributes(nil), }, } if ctx.Config[logCmdKey] == "true" { l.container.Command = ctx.Command() } if onGCE { l.instance = &instanceInfo{ Zone: zone, Name: instanceName, ID: instanceID, } } // The logger "overflows" at a rate of 10,000 logs per second and this // overflow func is called. We want to surface the error to the user // without overly spamming /var/log/docker.log so we log the first time // we overflow and every 1000th time after. c.Overflow = func(_ *logging.Client, _ logging.Entry) error { if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 { logrus.Errorf("gcplogs driver has dropped %v logs", i) } return nil } return l, nil }
// LogWriter returns an environment-specific io.Writer suitable for passing // to log.SetOutput. It will also include writing to os.Stderr as well. func LogWriter() (w io.Writer) { w = os.Stderr if !env.OnGCE() { return } projID, err := metadata.ProjectID() if projID == "" { log.Printf("Error getting project ID: %v", err) return } scopes, _ := metadata.Scopes("default") haveScope := func(scope string) bool { for _, x := range scopes { if x == scope { return true } } return false } if !haveScope(logging.Scope) { log.Printf("when this Google Compute Engine VM instance was created, it wasn't granted enough access to use Google Cloud Logging (Scope URL: %v).", logging.Scope) return } logc, err := logging.NewClient(context.Background(), projID, "camlistored-stderr") if err != nil { log.Printf("Error creating Google logging client: %v", err) return } return io.MultiWriter(w, logc.Writer(logging.Debug)) }
func NewClient(ctx context.Context, projectID, serviceName, serviceVersion string, opts ...cloud.ClientOption) (*Client, error) { l, err := logging.NewClient(ctx, projectID, "errorreports", opts...) if err != nil { return nil, fmt.Errorf("creating Logging client: %v", err) } c := &Client{ loggingClient: l, projectID: projectID, RepanicDefault: true, serviceContext: map[string]string{ "service": serviceName, }, } if serviceVersion != "" { c.serviceContext["version"] = serviceVersion } return c, nil }
// LogWriter returns an environment-specific io.Writer suitable for passing // to log.SetOutput. It will also include writing to os.Stderr as well. func LogWriter() (w io.Writer) { w = os.Stderr if !env.OnGCE() { return } projID, err := metadata.ProjectID() if projID == "" { log.Printf("Error getting project ID: %v", err) return } hc, err := google.DefaultClient(oauth2.NoContext) if err != nil { log.Printf("Error creating default GCE OAuth2 client: %v", err) return } logc, err := logging.NewClient(cloud.NewContext(projID, hc), "camlistored-stderr") if err != nil { log.Printf("Error creating Google logging client: %v", err) return } return io.MultiWriter(w, logc.Writer(logging.Debug)) }
func maybeSetupGoogleCloudLogging() { if flagGCEProjectID == "" && flagGCELogName == "" && flagGCEJWTFile == "" { return } if flagGCEProjectID == "" || flagGCELogName == "" || flagGCEJWTFile == "" { exitf("All of --gce_project_id, --gce_log_name, and --gce_jwt_file must be specified for logging on Google Cloud Logging.") } jsonSlurp, err := ioutil.ReadFile(flagGCEJWTFile) if err != nil { exitf("Error reading --gce_jwt_file value: %v", err) } jwtConf, err := google.JWTConfigFromJSON(jsonSlurp, logging.Scope) if err != nil { exitf("Error reading --gce_jwt_file value: %v", err) } ctx := cloud.NewContext(flagGCEProjectID, jwtConf.Client(context.Background())) logc, err := logging.NewClient(ctx, flagGCEProjectID, flagGCELogName) if err != nil { exitf("Error creating GCL client: %v", err) } log.SetOutput(io.MultiWriter(os.Stderr, logc.Writer(logging.Debug))) }
func handle(w http.ResponseWriter, r *http.Request) { c, _ := cloudAuthContext(r) logc, err := logging.NewClient(c, appengine.AppID(c), "javascript.errors") if err != nil { http.Error(w, "Cannot connect to Google Cloud Logging", http.StatusInternalServerError) log.Errorf(c, "Cannot connect to Google Cloud Logging: %v", err) return } // Note: Error Reporting currently ignores non-GCE and non-AWS logs. logc.ServiceName = "compute.googleapis.com" logc.CommonLabels = map[string]string{ "compute.googleapis.com/resource_type": "logger", "compute.googleapis.com/resource_id": "errors"} // Fill query params into JSON struct. line, _ := strconv.Atoi(r.URL.Query().Get("l")) errorType := "default" if r.URL.Query().Get("a") == "1" { errorType = "assert" } // By default we log as "INFO" severity, because reports are very spammy severity := "INFO" level := logging.Info // But if the request comes from the cache (and thus only from valid AMP // docs) we log as "ERROR". if strings.HasPrefix(r.Referer(), "https://cdn.ampproject.org/") { severity = "ERROR" level = logging.Error errorType += "-cdn" } event := &ErrorEvent{ Message: r.URL.Query().Get("m"), Exception: r.URL.Query().Get("s"), Version: r.URL.Query().Get("v"), Environment: "prod", Application: errorType, AppID: appengine.AppID(c), Filename: r.URL.Query().Get("f"), Line: int32(line), Classname: r.URL.Query().Get("el"), Severity: severity, } if event.Message == "" && event.Exception == "" { http.Error(w, "One of 'message' or 'exception' must be present.", http.StatusBadRequest) log.Errorf(c, "Malformed request: %v", event) return } // Don't log testing traffic in production if event.Version == "$internalRuntimeVersion$" { w.WriteHeader(http.StatusNoContent) return } event.Request = &ErrorRequest{ URL: r.Referer(), } event.Request.Meta = &ErrorRequestMeta{ HTTPReferrer: r.Referer(), HTTPUserAgent: r.UserAgent(), // Intentionally not logged. // RemoteIP: r.RemoteAddr, } err = logc.LogSync(logging.Entry{ Time: time.Now().UTC(), Payload: event, Level: level, }) if err != nil { http.Error(w, "Cannot write to Google Cloud Logging", http.StatusInternalServerError) log.Errorf(c, "Cannot write to Google Cloud Logging: %v", err) return } // When debug param is present, return a document. This is nicer because // browsers otherwise revert the URL during manual testing. if r.URL.Query().Get("debug") == "1" { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusOK) fmt.Fprintln(w, "OK\n") fmt.Fprintln(w, event) } else { w.WriteHeader(http.StatusNoContent) } }
func main() { flag.Parse() if *root == "" { var err error *root, err = os.Getwd() if err != nil { log.Fatalf("Failed to getwd: %v", err) } } readTemplates() mux := http.DefaultServeMux mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static")))) mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static")))) mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static"))))) mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks"))))) mux.Handle(pkgPattern, godocHandler{}) mux.Handle(cmdPattern, godocHandler{}) mux.HandleFunc(errPattern, errHandler) mux.HandleFunc("/r/", gerritRedirect) mux.HandleFunc("/dl/", releaseRedirect) mux.HandleFunc("/debugz/ip", ipHandler) mux.Handle("/docs/contributing", redirTo("/code#contributing")) mux.Handle("/lists", redirTo("/community")) mux.HandleFunc("/contributors", contribHandler()) mux.HandleFunc("/", mainHandler) if *buildbotHost != "" && *buildbotBackend != "" { buildbotUrl, err := url.Parse(*buildbotBackend) if err != nil { log.Fatalf("Failed to parse %v as a URL: %v", *buildbotBackend, err) } buildbotHandler := httputil.NewSingleHostReverseProxy(buildbotUrl) bbhpattern := strings.TrimRight(*buildbotHost, "/") + "/" mux.Handle(bbhpattern, buildbotHandler) } if *httpsAddr != "" { if launcher := gceDeployHandler("/launch/"); launcher != nil { mux.Handle("/launch/", launcher) } } var handler http.Handler = &noWwwHandler{Handler: mux} if *logDir != "" || *logStdout { handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout)) } if *gceLogName != "" { projID := *gceProjectID if projID == "" { if v, err := metadata.ProjectID(); v == "" || err != nil { log.Fatalf("Use of --gce_log_name without specifying --gce_project_id (and not running on GCE); metadata error: %v", err) } else { projID = v } } var hc *http.Client if *gceJWTFile != "" { jsonSlurp, err := ioutil.ReadFile(*gceJWTFile) if err != nil { log.Fatalf("Error reading --gce_jwt_file value: %v", err) } jwtConf, err := google.JWTConfigFromJSON(jsonSlurp, logging.Scope) if err != nil { log.Fatalf("Error reading --gce_jwt_file value: %v", err) } hc = jwtConf.Client(context.Background()) } else { if !metadata.OnGCE() { log.Fatal("No --gce_jwt_file and not running on GCE.") } var err error hc, err = google.DefaultClient(oauth2.NoContext) if err != nil { log.Fatal(err) } } ctx := cloud.NewContext(projID, hc) logc, err := logging.NewClient(ctx, projID, *gceLogName) if err != nil { log.Fatal(err) } if err := logc.Ping(); err != nil { log.Fatalf("Failed to ping Google Cloud Logging: %v", err) } handler = NewLoggingHandler(handler, gceLogger{logc}) } errc := make(chan error) startEmailCommitLoop(errc) if *alsoRun != "" { runAsChild(*alsoRun) } httpServer := &http.Server{ Addr: *httpAddr, Handler: handler, ReadTimeout: 5 * time.Minute, WriteTimeout: 30 * time.Minute, } go func() { errc <- httpServer.ListenAndServe() }() if *httpsAddr != "" { log.Printf("Starting TLS server on %s", *httpsAddr) httpsServer := new(http.Server) *httpsServer = *httpServer httpsServer.Addr = *httpsAddr go func() { errc <- httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile) }() } log.Fatalf("Serve error: %v", <-errc) }
func main() { launchConfig.MaybeDeploy() flag.Parse() setProdFlags() if *root == "" { var err error *root, err = os.Getwd() if err != nil { log.Fatalf("Failed to getwd: %v", err) } } readTemplates() go runDemoBlobserverLoop() mux := http.DefaultServeMux mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static")))) mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static")))) mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static"))))) mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks"))))) mux.Handle(pkgPattern, godocHandler{}) mux.Handle(cmdPattern, godocHandler{}) mux.HandleFunc(errPattern, errHandler) mux.HandleFunc("/r/", gerritRedirect) mux.HandleFunc("/dl/", releaseRedirect) mux.HandleFunc("/debug/ip", ipHandler) mux.HandleFunc("/debug/uptime", uptimeHandler) mux.Handle("/docs/contributing", redirTo("/code#contributing")) mux.Handle("/lists", redirTo("/community")) mux.HandleFunc("/contributors", contribHandler()) mux.HandleFunc("/", mainHandler) if *buildbotHost != "" && *buildbotBackend != "" { buildbotUrl, err := url.Parse(*buildbotBackend) if err != nil { log.Fatalf("Failed to parse %v as a URL: %v", *buildbotBackend, err) } buildbotHandler := httputil.NewSingleHostReverseProxy(buildbotUrl) bbhpattern := strings.TrimRight(*buildbotHost, "/") + "/" mux.Handle(bbhpattern, buildbotHandler) } // ctx initialized now, because gceLauncher needs it first (when in prod). // Other users are the GCE logger, and serveHTTPS (in prod). var ctx context.Context var projID string if inProd || *gceLogName != "" { projID = projectID() ctx = ctxt(projID) } gceLauncher, err := gceDeployHandler(ctx, "/launch/") if err != nil { log.Printf("Not installing GCE /launch/ handler: %v", err) mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("GCE launcher disabled: %v", err), 500) }) } else { mux.Handle("/launch/", gceLauncher) } var handler http.Handler = &noWwwHandler{Handler: mux} if *logDir != "" || *logStdout { handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout)) } if *gceLogName != "" { logc, err := logging.NewClient(ctx, projID, *gceLogName) if err != nil { log.Fatal(err) } if err := logc.Ping(); err != nil { log.Fatalf("Failed to ping Google Cloud Logging: %v", err) } handler = NewLoggingHandler(handler, gceLogger{logc}) if gceLauncher != nil { logc, err := logging.NewClient(ctx, projID, *gceLogName) if err != nil { log.Fatal(err) } logc.CommonLabels = map[string]string{ "from": "camli-gce-launcher", } logger := logc.Logger(logging.Default) logger.SetPrefix("launcher: ") gceLauncher.SetLogger(logger) } } emailErr := make(chan error) startEmailCommitLoop(emailErr) if *alsoRun != "" { runAsChild(*alsoRun) } httpServer := &http.Server{ Addr: *httpAddr, Handler: handler, ReadTimeout: 5 * time.Minute, WriteTimeout: 30 * time.Minute, } httpErr := make(chan error) go func() { log.Printf("Listening for HTTP on %v", *httpAddr) httpErr <- httpServer.ListenAndServe() }() httpsErr := make(chan error) if *httpsAddr != "" { go func() { httpsErr <- serveHTTPS(ctx, httpServer) }() } if *flagChromeBugRepro { go func() { log.Printf("Repro handler failed: %v", repro(":8001", "foo:bar")) }() } select { case err := <-emailErr: log.Fatalf("Error sending emails: %v", err) case err := <-httpErr: log.Fatalf("Error serving HTTP: %v", err) case err := <-httpsErr: log.Fatalf("Error serving HTTPS: %v", err) } }
func handle(w http.ResponseWriter, r *http.Request) { c, _ := cloudAuthContext(r) logc, err := logging.NewClient(c, appengine.AppID(c), "javascript.errors") if err != nil { http.Error(w, "Cannot connect to Google Cloud Logging", http.StatusInternalServerError) log.Errorf(c, "Cannot connect to Google Cloud Logging: %v", err) return } // Note: Error Reporting currently ignores non-GCE and non-AWS logs. logc.ServiceName = "compute.googleapis.com" logc.CommonLabels = map[string]string{ "compute.googleapis.com/resource_type": "logger", "compute.googleapis.com/resource_id": "errors"} // Fill query params into JSON struct. line, _ := strconv.Atoi(r.URL.Query().Get("l")) event := &ErrorEvent{ Message: r.URL.Query().Get("m"), Exception: r.URL.Query().Get("s"), Version: r.URL.Query().Get("v"), Environment: "prod", Application: appengine.ModuleName(c), AppID: appengine.AppID(c), Filename: r.URL.Query().Get("f"), Line: int32(line), Classname: r.URL.Query().Get("el"), } if event.Message == "" && event.Exception == "" { http.Error(w, "One of 'message' or 'exception' must be present.", http.StatusBadRequest) log.Errorf(c, "Malformed request: %v", event) return } event.Request = &ErrorRequest{ URL: r.Referer(), } event.Request.Meta = &ErrorRequestMeta{ HTTPReferrer: r.Referer(), HTTPUserAgent: r.UserAgent(), // Intentionally not logged. // RemoteIP: r.RemoteAddr, } err = logc.LogSync(logging.Entry{ Time: time.Now().UTC(), Payload: event, }) if err != nil { http.Error(w, "Cannot write to Google Cloud Logging", http.StatusInternalServerError) log.Errorf(c, "Cannot write to Google Cloud Logging: %v", err) return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusOK) fmt.Fprintln(w, "OK") }