Ejemplo n.º 1
0
// LogWriter returns an environment-specific io.Writer suitable for passing
// to log.SetOutput. It will also include writing to os.Stderr as well.
func LogWriter() (w io.Writer) {
	w = os.Stderr
	if !env.OnGCE() {
		return
	}
	projID, err := metadata.ProjectID()
	if projID == "" {
		log.Printf("Error getting project ID: %v", err)
		return
	}
	scopes, _ := metadata.Scopes("default")
	haveScope := func(scope string) bool {
		for _, x := range scopes {
			if x == scope {
				return true
			}
		}
		return false
	}
	if !haveScope(logging.Scope) {
		log.Printf("when this Google Compute Engine VM instance was created, it wasn't granted enough access to use Google Cloud Logging (Scope URL: %v).", logging.Scope)
		return
	}

	logc, err := logging.NewClient(context.Background(), projID, "camlistored-stderr")
	if err != nil {
		log.Printf("Error creating Google logging client: %v", err)
		return
	}
	return io.MultiWriter(w, logc.Writer(logging.Debug))
}
Ejemplo n.º 2
0
// If we're not running on GCE (e.g. dev mode on localhost) and have
// no other way to get the info, the error value is is errNoRefresh.
func (h *DeployHandler) authenticatedClient() (project string, hc *http.Client, err error) {
	project = os.Getenv("CAMLI_GCE_PROJECT")
	accountFile := os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT")
	if project != "" && accountFile != "" {
		data, errr := ioutil.ReadFile(accountFile)
		err = errr
		if err != nil {
			return
		}
		jwtConf, errr := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/compute.readonly")
		err = errr
		if err != nil {
			return
		}
		hc = jwtConf.Client(context.Background())
		return
	}
	if !metadata.OnGCE() {
		err = errNoRefresh
		return
	}
	project, _ = metadata.ProjectID()
	hc, err = google.DefaultClient(oauth2.NoContext)
	return project, hc, err
}
Ejemplo n.º 3
0
func maybeRemapCloudSQL(host string) (out string, err error) {
	if !strings.HasSuffix(host, cloudSQLSuffix) {
		return host, nil
	}
	inst := strings.TrimSuffix(host, cloudSQLSuffix)
	if !metadata.OnGCE() {
		return "", errors.New("CloudSQL support only available when running on Google Compute Engine.")
	}
	proj, err := metadata.ProjectID()
	if err != nil {
		return "", fmt.Errorf("Failed to lookup GCE project ID: %v", err)
	}

	admin, _ := sqladmin.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource("")))
	listRes, err := admin.Instances.List(proj).Do()
	if err != nil {
		return "", fmt.Errorf("error enumerating Cloud SQL instances: %v", err)
	}
	for _, it := range listRes.Items {
		if !strings.EqualFold(it.Instance, inst) {
			continue
		}
		js, _ := json.Marshal(it)
		log.Printf("Found Cloud SQL instance %s: %s", inst, js)
		for _, ipm := range it.IpAddresses {
			return ipm.IpAddress, nil
		}
		return "", fmt.Errorf("No external IP address for Cloud SQL instances %s", inst)
	}
	var found []string
	for _, it := range listRes.Items {
		found = append(found, it.Instance)
	}
	return "", fmt.Errorf("Cloud SQL instance %q not found. Found: %q", inst, found)
}
Ejemplo n.º 4
0
func logsHandler(w http.ResponseWriter, r *http.Request) {
	suffix := strings.TrimPrefix(r.URL.Path, "/debug/logs/")
	switch suffix {
	case "camlistored":
		projID, err := metadata.ProjectID()
		if err != nil {
			httputil.ServeError(w, r, fmt.Errorf("Error getting project ID: %v", err))
			return
		}
		http.Redirect(w, r,
			"https://console.developers.google.com/logs?project="+projID+"&service=custom.googleapis.com&logName=camlistored-stderr",
			http.StatusFound)
	case "system":
		c := &http.Client{
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					return net.Dial("unix", "/run/camjournald.sock")
				},
			},
		}
		res, err := c.Get("http://journal/entries")
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		io.Copy(w, res.Body)
	default:
		http.Error(w, "no such logs", 404)
	}
}
Ejemplo n.º 5
0
func checkInProduction() bool {
	if !metadata.OnGCE() {
		return false
	}
	proj, _ := metadata.ProjectID()
	inst, _ := metadata.InstanceName()
	log.Printf("Running on GCE: %v / %v", proj, inst)
	return proj == "camlistore-website" && inst == "camweb"
}
Ejemplo n.º 6
0
func (sh *StatusHandler) googleCloudConsole() (string, error) {
	if !env.OnGCE() {
		return "", errors.New("not on GCE")
	}
	projID, err := metadata.ProjectID()
	if err != nil {
		return "", fmt.Errorf("Error getting project ID: %v", err)
	}
	return "https://console.cloud.google.com/compute/instances?project=" + projID, nil
}
Ejemplo n.º 7
0
// projectID returns the GCE project ID used for running this camweb on GCE
// and/or for logging on Google Cloud Logging, if any.
func projectID() string {
	if *gceProjectID != "" {
		return *gceProjectID
	}
	projID, err := metadata.ProjectID()
	if projID == "" || err != nil {
		log.Fatalf("GCE project ID needed but --gce_project_id not specified (and not running on GCE); metadata error: %v", err)
	}
	return projID
}
Ejemplo n.º 8
0
func CreateGCMSink(uri *url.URL) (core.DataSink, error) {
	if len(uri.Scheme) > 0 {
		return nil, fmt.Errorf("scheme should not be set for GCM sink")
	}
	if len(uri.Host) > 0 {
		return nil, fmt.Errorf("host should not be set for GCM sink")
	}

	opts, err := url.ParseQuery(uri.RawQuery)

	metrics := "all"
	if len(opts["metrics"]) > 0 {
		metrics = opts["metrics"][0]
	}
	var metricFilter MetricFilter = metricsAll
	switch metrics {
	case "all":
		metricFilter = metricsAll
	case "autoscaling":
		metricFilter = metricsOnlyAutoscaling
	default:
		return nil, fmt.Errorf("invalid metrics parameter: %s", metrics)
	}

	if err := gce_util.EnsureOnGCE(); err != nil {
		return nil, err
	}

	// Detect project ID
	projectId, err := gce.ProjectID()
	if err != nil {
		return nil, err
	}

	// Create Google Cloud Monitoring service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	gcmService, err := gcm.New(client)
	if err != nil {
		return nil, err
	}

	sink := &gcmSink{
		registered:   false,
		project:      projectId,
		gcmService:   gcmService,
		metricFilter: metricFilter,
	}
	glog.Infof("created GCM sink")
	if err := sink.registerAllMetrics(); err != nil {
		glog.Warningf("Error during metrics registration: %v", err)
	}
	return sink, nil
}
Ejemplo n.º 9
0
func initGCP() {
	initGCPOnce.Do(func() {
		onGCE = metadata.OnGCE()
		if onGCE {
			// These will fail on instances if the metadata service is
			// down or the client is compiled with an API version that
			// has been removed. Since these are not vital, let's ignore
			// them and make their fields in the dockeLogEntry ,omitempty
			projectID, _ = metadata.ProjectID()
			zone, _ = metadata.Zone()
			instanceName, _ = metadata.InstanceName()
			instanceID, _ = metadata.InstanceID()
		}
	})
}
Ejemplo n.º 10
0
func newGcmSource() (dataSource, error) {
	// Detect project ID
	projectId, err := gce.ProjectID()
	if err != nil {
		return nil, err
	}

	// Create Google Cloud Monitoring service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	s, err := gcm.New(client)
	if err != nil {
		return nil, err
	}

	return &gcmSource{
		project:    projectId,
		gcmService: s,
	}, nil
}
Ejemplo n.º 11
0
// newCloudDns creates a new instance of a Google Cloud DNS Interface.
func newCloudDns(config io.Reader) (*Interface, error) {
	projectID, _ := metadata.ProjectID() // On error we get an empty string, which is fine for now.
	var tokenSource oauth2.TokenSource
	// Possibly override defaults with config below
	if config != nil {
		var cfg Config
		if err := gcfg.ReadInto(&cfg, config); err != nil {
			glog.Errorf("Couldn't read config: %v", err)
			return nil, err
		}
		glog.Infof("Using Google Cloud DNS provider config %+v", cfg)
		if cfg.Global.ProjectID != "" {
			projectID = cfg.Global.ProjectID
		}
		if cfg.Global.TokenURL != "" {
			tokenSource = gce.NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
		}
	}
	return CreateInterface(projectID, tokenSource)
}
Ejemplo n.º 12
0
// FindDefaultCredentials searches for "Application Default Credentials".
//
// It looks for credentials in the following places,
// preferring the first location found:
//
//   1. A JSON file whose path is specified by the
//      GOOGLE_APPLICATION_CREDENTIALS environment variable.
//   2. A JSON file in a location known to the gcloud command-line tool.
//      On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
//      On other systems, $HOME/.config/gcloud/application_default_credentials.json.
//   3. On Google App Engine it uses the appengine.AccessToken function.
//   4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
//      credentials from the metadata server.
//      (In this final case any provided scopes are ignored.)
func FindDefaultCredentials(ctx context.Context, scope ...string) (*DefaultCredentials, error) {
	// First, try the environment variable.
	const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
	if filename := os.Getenv(envVar); filename != "" {
		creds, err := readCredentialsFile(ctx, filename, scope)
		if err != nil {
			return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
		}
		return creds, nil
	}

	// Second, try a well-known file.
	filename := wellKnownFile()
	if creds, err := readCredentialsFile(ctx, filename, scope); err == nil {
		return creds, nil
	} else if !os.IsNotExist(err) {
		return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err)
	}

	// Third, if we're on Google App Engine use those credentials.
	if appengineTokenFunc != nil && !appengineVM {
		return &DefaultCredentials{
			ProjectID:   appengineAppIDFunc(ctx),
			TokenSource: AppEngineTokenSource(ctx, scope...),
		}, nil
	}

	// Fourth, if we're on Google Compute Engine use the metadata server.
	if metadata.OnGCE() {
		id, _ := metadata.ProjectID()
		return &DefaultCredentials{
			ProjectID:   id,
			TokenSource: ComputeTokenSource(""),
		}, nil
	}

	// None are found; return helpful error.
	const url = "https://developers.google.com/accounts/docs/application-default-credentials"
	return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url)
}
Ejemplo n.º 13
0
func CreateGCLSink(uri *url.URL) (core.EventSink, error) {
	if err := gce_util.EnsureOnGCE(); err != nil {
		return nil, err
	}

	// Detect project ID
	projectId, err := gce.ProjectID()
	if err != nil {
		return nil, err
	}

	// Create Google Cloud Logging service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	gclService, err := gcl.New(client)
	if err != nil {
		return nil, err
	}

	sink := &gclSink{project: projectId, gclService: gclService}
	glog.Info("created GCL sink")
	return sink, nil
}
Ejemplo n.º 14
0
func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		// The user needs to supply the name of the executable to run.
		flag.Usage()
		return
	}
	if *projectNumber == "" {
		var err error
		*projectNumber, err = metadata.NumericProjectID()
		if err != nil {
			log.Print("Debuglet initialization: ", err)
		}
	}
	if *projectID == "" {
		var err error
		*projectID, err = metadata.ProjectID()
		if err != nil {
			log.Print("Debuglet initialization: ", err)
		}
	}
	sourceContexts, err := readSourceContextFile(*sourceContextFile)
	if err != nil {
		log.Print("Reading source context file: ", err)
	}
	c, err := debuglet.NewController(debuglet.Options{
		ProjectNumber:      *projectNumber,
		ProjectID:          *projectID,
		AppModule:          *appModule,
		AppVersion:         *appVersion,
		SourceContexts:     sourceContexts,
		Verbose:            *verbose,
		ServiceAccountFile: *serviceAccountFile,
	})
	if err != nil {
		log.Fatal("Error connecting to Cloud Debugger: ", err)
	}
	prog, err := local.New(args[0])
	if err != nil {
		log.Fatal("Error loading program: ", err)
	}
	// Load the program, but don't actually start it running yet.
	if _, err = prog.Run(args[1:]...); err != nil {
		log.Fatal("Error loading program: ", err)
	}
	bs := breakpoints.NewBreakpointStore(prog)

	// Seed the random number generator.
	rand.Seed(time.Now().UnixNano())

	// Now we want to do two things: run the user's program, and start sending
	// List requests periodically to the Debuglet Controller to get breakpoints
	// to set.
	//
	// We want to give the Debuglet Controller a chance to give us breakpoints
	// before we start the program, otherwise we would miss any breakpoint
	// triggers that occur during program startup -- for example, a breakpoint on
	// the first line of main. But if the Debuglet Controller is not responding or
	// is returning errors, we don't want to delay starting the program
	// indefinitely.
	//
	// We pass a channel to breakpointListLoop, which will close it when the first
	// List call finishes.  Then we wait until either the channel is closed or a
	// 5-second timer has finished before starting the program.
	ch := make(chan bool)
	// Start a goroutine that sends List requests to the Debuglet Controller, and
	// sets any breakpoints it gets back.
	go breakpointListLoop(c, bs, ch)
	// Wait until 5 seconds have passed or breakpointListLoop has closed ch.
	select {
	case <-time.After(5 * time.Second):
	case <-ch:
	}
	// Run the debuggee.
	programLoop(c, bs, prog)
}
Ejemplo n.º 15
0
Archivo: main.go Proyecto: golang/gddo
func main() {
	flag.Parse()
	log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))

	doc.SetDefaultGOOS(*defaultGOOS)
	httpClient = newHTTPClient()

	var (
		gceLogName string
		projID     string
	)

	if metadata.OnGCE() {
		acct, err := metadata.ProjectAttributeValue("ga-account")
		if err != nil {
			log.Printf("querying metadata for ga-account: %v", err)
		} else {
			gaAccount = acct
		}

		// Get the log name on GCE and setup context for creating a GCE log client.
		if name, err := metadata.ProjectAttributeValue("gce-log-name"); err != nil {
			log.Printf("querying metadata for gce-log-name: %v", err)
		} else {
			gceLogName = name
			if id, err := metadata.ProjectID(); err != nil {
				log.Printf("querying metadata for project ID: %v", err)
			} else {
				projID = id
			}
		}
	} else {
		gaAccount = os.Getenv("GA_ACCOUNT")
	}

	if err := parseHTMLTemplates([][]string{
		{"about.html", "common.html", "layout.html"},
		{"bot.html", "common.html", "layout.html"},
		{"cmd.html", "common.html", "layout.html"},
		{"dir.html", "common.html", "layout.html"},
		{"home.html", "common.html", "layout.html"},
		{"importers.html", "common.html", "layout.html"},
		{"importers_robot.html", "common.html", "layout.html"},
		{"imports.html", "common.html", "layout.html"},
		{"notfound.html", "common.html", "layout.html"},
		{"pkg.html", "common.html", "layout.html"},
		{"results.html", "common.html", "layout.html"},
		{"tools.html", "common.html", "layout.html"},
		{"std.html", "common.html", "layout.html"},
		{"subrepo.html", "common.html", "layout.html"},
		{"graph.html", "common.html"},
	}); err != nil {
		log.Fatal(err)
	}

	if err := parseTextTemplates([][]string{
		{"cmd.txt", "common.txt"},
		{"dir.txt", "common.txt"},
		{"home.txt", "common.txt"},
		{"notfound.txt", "common.txt"},
		{"pkg.txt", "common.txt"},
		{"results.txt", "common.txt"},
	}); err != nil {
		log.Fatal(err)
	}

	var err error
	db, err = database.New()
	if err != nil {
		log.Fatalf("Error opening database: %v", err)
	}

	go runBackgroundTasks()

	staticServer := httputil.StaticServer{
		Dir:    *assetsDir,
		MaxAge: time.Hour,
		MIMETypes: map[string]string{
			".css": "text/css; charset=utf-8",
			".js":  "text/javascript; charset=utf-8",
		},
	}
	statusImageHandlerPNG = staticServer.FileHandler("status.png")
	statusImageHandlerSVG = staticServer.FileHandler("status.svg")

	apiMux := http.NewServeMux()
	apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
	apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
	apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
	apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
	apiMux.Handle("/search", apiHandler(serveAPISearch))
	apiMux.Handle("/packages", apiHandler(serveAPIPackages))
	apiMux.Handle("/importers/", apiHandler(serveAPIImporters))
	apiMux.Handle("/imports/", apiHandler(serveAPIImports))
	apiMux.Handle("/", apiHandler(serveAPIHome))

	mux := http.NewServeMux()
	mux.Handle("/-/site.js", staticServer.FilesHandler(
		"third_party/jquery.timeago.js",
		"site.js"))
	mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
	mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
	mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
	mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
	if *sidebarEnabled {
		mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
	}
	mux.Handle("/-/", http.NotFoundHandler())

	mux.Handle("/-/about", handler(serveAbout))
	mux.Handle("/-/bot", handler(serveBot))
	mux.Handle("/-/go", handler(serveGoIndex))
	mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
	mux.Handle("/-/refresh", handler(serveRefresh))
	mux.Handle("/-/admin/reindex", http.HandlerFunc(runReindex))
	mux.Handle("/-/admin/purgeindex", http.HandlerFunc(runPurgeIndex))
	mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
	mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
	mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
	mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
	mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
	mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
	mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
	mux.Handle("/code.jquery.com/", http.NotFoundHandler())
	mux.Handle("/_ah/health", http.HandlerFunc(serveHealthCheck))
	mux.Handle("/_ah/", http.NotFoundHandler())
	mux.Handle("/", handler(serveHome))

	cacheBusters.Handler = mux

	var root http.Handler = rootHandler{
		{"api.", apiMux},
		{"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
		{"www.godoc.org", otherDomainHandler{"https", "godoc.org"}},
		{"", mux},
	}
	if gceLogName != "" {
		ctx := context.Background()

		logc, err := logging.NewClient(ctx, projID)
		if err != nil {
			log.Fatalf("Failed to create cloud logging client: %v", err)
		}
		logger := logc.Logger(gceLogName)

		if err := logc.Ping(ctx); err != nil {
			log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
		}

		gceLogger = newGCELogger(logger)
	}

	http.Handle("/", root)
	appengine.Main()
}