// LogWriter returns an environment-specific io.Writer suitable for passing
// to log.SetOutput. It will also include writing to os.Stderr as well.
func LogWriter() (w io.Writer) {
w = os.Stderr
if !env.OnGCE() {
return
}
projID, err := metadata.ProjectID()
if projID == "" {
log.Printf("Error getting project ID: %v", err)
return
}
scopes, _ := metadata.Scopes("default")
haveScope := func(scope string) bool {
for _, x := range scopes {
if x == scope {
return true
}
}
return false
}
if !haveScope(logging.Scope) {
log.Printf("when this Google Compute Engine VM instance was created, it wasn't granted enough access to use Google Cloud Logging (Scope URL: %v).", logging.Scope)
return
}
logc, err := logging.NewClient(context.Background(), projID, "camlistored-stderr")
if err != nil {
log.Printf("Error creating Google logging client: %v", err)
return
}
return io.MultiWriter(w, logc.Writer(logging.Debug))
}
// If we're not running on GCE (e.g. dev mode on localhost) and have
// no other way to get the info, the error value is is errNoRefresh.
func (h *DeployHandler) authenticatedClient() (project string, hc *http.Client, err error) {
project = os.Getenv("CAMLI_GCE_PROJECT")
accountFile := os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT")
if project != "" && accountFile != "" {
data, errr := ioutil.ReadFile(accountFile)
err = errr
if err != nil {
return
}
jwtConf, errr := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/compute.readonly")
err = errr
if err != nil {
return
}
hc = jwtConf.Client(context.Background())
return
}
if !metadata.OnGCE() {
err = errNoRefresh
return
}
project, _ = metadata.ProjectID()
hc, err = google.DefaultClient(oauth2.NoContext)
return project, hc, err
}
func maybeRemapCloudSQL(host string) (out string, err error) {
if !strings.HasSuffix(host, cloudSQLSuffix) {
return host, nil
}
inst := strings.TrimSuffix(host, cloudSQLSuffix)
if !metadata.OnGCE() {
return "", errors.New("CloudSQL support only available when running on Google Compute Engine.")
}
proj, err := metadata.ProjectID()
if err != nil {
return "", fmt.Errorf("Failed to lookup GCE project ID: %v", err)
}
admin, _ := sqladmin.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource("")))
listRes, err := admin.Instances.List(proj).Do()
if err != nil {
return "", fmt.Errorf("error enumerating Cloud SQL instances: %v", err)
}
for _, it := range listRes.Items {
if !strings.EqualFold(it.Instance, inst) {
continue
}
js, _ := json.Marshal(it)
log.Printf("Found Cloud SQL instance %s: %s", inst, js)
for _, ipm := range it.IpAddresses {
return ipm.IpAddress, nil
}
return "", fmt.Errorf("No external IP address for Cloud SQL instances %s", inst)
}
var found []string
for _, it := range listRes.Items {
found = append(found, it.Instance)
}
return "", fmt.Errorf("Cloud SQL instance %q not found. Found: %q", inst, found)
}
func logsHandler(w http.ResponseWriter, r *http.Request) {
suffix := strings.TrimPrefix(r.URL.Path, "/debug/logs/")
switch suffix {
case "camlistored":
projID, err := metadata.ProjectID()
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error getting project ID: %v", err))
return
}
http.Redirect(w, r,
"https://console.developers.google.com/logs?project="+projID+"&service=custom.googleapis.com&logName=camlistored-stderr",
http.StatusFound)
case "system":
c := &http.Client{
Transport: &http.Transport{
Dial: func(network, addr string) (net.Conn, error) {
return net.Dial("unix", "/run/camjournald.sock")
},
},
}
res, err := c.Get("http://journal/entries")
if err != nil {
http.Error(w, err.Error(), 500)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
io.Copy(w, res.Body)
default:
http.Error(w, "no such logs", 404)
}
}
func checkInProduction() bool {
if !metadata.OnGCE() {
return false
}
proj, _ := metadata.ProjectID()
inst, _ := metadata.InstanceName()
log.Printf("Running on GCE: %v / %v", proj, inst)
return proj == "camlistore-website" && inst == "camweb"
}
func (sh *StatusHandler) googleCloudConsole() (string, error) {
if !env.OnGCE() {
return "", errors.New("not on GCE")
}
projID, err := metadata.ProjectID()
if err != nil {
return "", fmt.Errorf("Error getting project ID: %v", err)
}
return "https://console.cloud.google.com/compute/instances?project=" + projID, nil
}
// projectID returns the GCE project ID used for running this camweb on GCE
// and/or for logging on Google Cloud Logging, if any.
func projectID() string {
if *gceProjectID != "" {
return *gceProjectID
}
projID, err := metadata.ProjectID()
if projID == "" || err != nil {
log.Fatalf("GCE project ID needed but --gce_project_id not specified (and not running on GCE); metadata error: %v", err)
}
return projID
}
func CreateGCMSink(uri *url.URL) (core.DataSink, error) {
if len(uri.Scheme) > 0 {
return nil, fmt.Errorf("scheme should not be set for GCM sink")
}
if len(uri.Host) > 0 {
return nil, fmt.Errorf("host should not be set for GCM sink")
}
opts, err := url.ParseQuery(uri.RawQuery)
metrics := "all"
if len(opts["metrics"]) > 0 {
metrics = opts["metrics"][0]
}
var metricFilter MetricFilter = metricsAll
switch metrics {
case "all":
metricFilter = metricsAll
case "autoscaling":
metricFilter = metricsOnlyAutoscaling
default:
return nil, fmt.Errorf("invalid metrics parameter: %s", metrics)
}
if err := gce_util.EnsureOnGCE(); err != nil {
return nil, err
}
// Detect project ID
projectId, err := gce.ProjectID()
if err != nil {
return nil, err
}
// Create Google Cloud Monitoring service.
client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
gcmService, err := gcm.New(client)
if err != nil {
return nil, err
}
sink := &gcmSink{
registered: false,
project: projectId,
gcmService: gcmService,
metricFilter: metricFilter,
}
glog.Infof("created GCM sink")
if err := sink.registerAllMetrics(); err != nil {
glog.Warningf("Error during metrics registration: %v", err)
}
return sink, nil
}
func initGCP() {
initGCPOnce.Do(func() {
onGCE = metadata.OnGCE()
if onGCE {
// These will fail on instances if the metadata service is
// down or the client is compiled with an API version that
// has been removed. Since these are not vital, let's ignore
// them and make their fields in the dockeLogEntry ,omitempty
projectID, _ = metadata.ProjectID()
zone, _ = metadata.Zone()
instanceName, _ = metadata.InstanceName()
instanceID, _ = metadata.InstanceID()
}
})
}
func newGcmSource() (dataSource, error) {
// Detect project ID
projectId, err := gce.ProjectID()
if err != nil {
return nil, err
}
// Create Google Cloud Monitoring service.
client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
s, err := gcm.New(client)
if err != nil {
return nil, err
}
return &gcmSource{
project: projectId,
gcmService: s,
}, nil
}
// newCloudDns creates a new instance of a Google Cloud DNS Interface.
func newCloudDns(config io.Reader) (*Interface, error) {
projectID, _ := metadata.ProjectID() // On error we get an empty string, which is fine for now.
var tokenSource oauth2.TokenSource
// Possibly override defaults with config below
if config != nil {
var cfg Config
if err := gcfg.ReadInto(&cfg, config); err != nil {
glog.Errorf("Couldn't read config: %v", err)
return nil, err
}
glog.Infof("Using Google Cloud DNS provider config %+v", cfg)
if cfg.Global.ProjectID != "" {
projectID = cfg.Global.ProjectID
}
if cfg.Global.TokenURL != "" {
tokenSource = gce.NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
}
}
return CreateInterface(projectID, tokenSource)
}
// FindDefaultCredentials searches for "Application Default Credentials".
//
// It looks for credentials in the following places,
// preferring the first location found:
//
// 1. A JSON file whose path is specified by the
// GOOGLE_APPLICATION_CREDENTIALS environment variable.
// 2. A JSON file in a location known to the gcloud command-line tool.
// On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
// On other systems, $HOME/.config/gcloud/application_default_credentials.json.
// 3. On Google App Engine it uses the appengine.AccessToken function.
// 4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
// credentials from the metadata server.
// (In this final case any provided scopes are ignored.)
func FindDefaultCredentials(ctx context.Context, scope ...string) (*DefaultCredentials, error) {
// First, try the environment variable.
const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
if filename := os.Getenv(envVar); filename != "" {
creds, err := readCredentialsFile(ctx, filename, scope)
if err != nil {
return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
}
return creds, nil
}
// Second, try a well-known file.
filename := wellKnownFile()
if creds, err := readCredentialsFile(ctx, filename, scope); err == nil {
return creds, nil
} else if !os.IsNotExist(err) {
return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err)
}
// Third, if we're on Google App Engine use those credentials.
if appengineTokenFunc != nil && !appengineVM {
return &DefaultCredentials{
ProjectID: appengineAppIDFunc(ctx),
TokenSource: AppEngineTokenSource(ctx, scope...),
}, nil
}
// Fourth, if we're on Google Compute Engine use the metadata server.
if metadata.OnGCE() {
id, _ := metadata.ProjectID()
return &DefaultCredentials{
ProjectID: id,
TokenSource: ComputeTokenSource(""),
}, nil
}
// None are found; return helpful error.
const url = "https://developers.google.com/accounts/docs/application-default-credentials"
return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url)
}
func CreateGCLSink(uri *url.URL) (core.EventSink, error) {
if err := gce_util.EnsureOnGCE(); err != nil {
return nil, err
}
// Detect project ID
projectId, err := gce.ProjectID()
if err != nil {
return nil, err
}
// Create Google Cloud Logging service.
client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
gclService, err := gcl.New(client)
if err != nil {
return nil, err
}
sink := &gclSink{project: projectId, gclService: gclService}
glog.Info("created GCL sink")
return sink, nil
}
func main() {
flag.Usage = usage
flag.Parse()
args := flag.Args()
if len(args) == 0 {
// The user needs to supply the name of the executable to run.
flag.Usage()
return
}
if *projectNumber == "" {
var err error
*projectNumber, err = metadata.NumericProjectID()
if err != nil {
log.Print("Debuglet initialization: ", err)
}
}
if *projectID == "" {
var err error
*projectID, err = metadata.ProjectID()
if err != nil {
log.Print("Debuglet initialization: ", err)
}
}
sourceContexts, err := readSourceContextFile(*sourceContextFile)
if err != nil {
log.Print("Reading source context file: ", err)
}
c, err := debuglet.NewController(debuglet.Options{
ProjectNumber: *projectNumber,
ProjectID: *projectID,
AppModule: *appModule,
AppVersion: *appVersion,
SourceContexts: sourceContexts,
Verbose: *verbose,
ServiceAccountFile: *serviceAccountFile,
})
if err != nil {
log.Fatal("Error connecting to Cloud Debugger: ", err)
}
prog, err := local.New(args[0])
if err != nil {
log.Fatal("Error loading program: ", err)
}
// Load the program, but don't actually start it running yet.
if _, err = prog.Run(args[1:]...); err != nil {
log.Fatal("Error loading program: ", err)
}
bs := breakpoints.NewBreakpointStore(prog)
// Seed the random number generator.
rand.Seed(time.Now().UnixNano())
// Now we want to do two things: run the user's program, and start sending
// List requests periodically to the Debuglet Controller to get breakpoints
// to set.
//
// We want to give the Debuglet Controller a chance to give us breakpoints
// before we start the program, otherwise we would miss any breakpoint
// triggers that occur during program startup -- for example, a breakpoint on
// the first line of main. But if the Debuglet Controller is not responding or
// is returning errors, we don't want to delay starting the program
// indefinitely.
//
// We pass a channel to breakpointListLoop, which will close it when the first
// List call finishes. Then we wait until either the channel is closed or a
// 5-second timer has finished before starting the program.
ch := make(chan bool)
// Start a goroutine that sends List requests to the Debuglet Controller, and
// sets any breakpoints it gets back.
go breakpointListLoop(c, bs, ch)
// Wait until 5 seconds have passed or breakpointListLoop has closed ch.
select {
case <-time.After(5 * time.Second):
case <-ch:
}
// Run the debuggee.
programLoop(c, bs, prog)
}
func main() {
flag.Parse()
log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
doc.SetDefaultGOOS(*defaultGOOS)
httpClient = newHTTPClient()
var (
gceLogName string
projID string
)
if metadata.OnGCE() {
acct, err := metadata.ProjectAttributeValue("ga-account")
if err != nil {
log.Printf("querying metadata for ga-account: %v", err)
} else {
gaAccount = acct
}
// Get the log name on GCE and setup context for creating a GCE log client.
if name, err := metadata.ProjectAttributeValue("gce-log-name"); err != nil {
log.Printf("querying metadata for gce-log-name: %v", err)
} else {
gceLogName = name
if id, err := metadata.ProjectID(); err != nil {
log.Printf("querying metadata for project ID: %v", err)
} else {
projID = id
}
}
} else {
gaAccount = os.Getenv("GA_ACCOUNT")
}
if err := parseHTMLTemplates([][]string{
{"about.html", "common.html", "layout.html"},
{"bot.html", "common.html", "layout.html"},
{"cmd.html", "common.html", "layout.html"},
{"dir.html", "common.html", "layout.html"},
{"home.html", "common.html", "layout.html"},
{"importers.html", "common.html", "layout.html"},
{"importers_robot.html", "common.html", "layout.html"},
{"imports.html", "common.html", "layout.html"},
{"notfound.html", "common.html", "layout.html"},
{"pkg.html", "common.html", "layout.html"},
{"results.html", "common.html", "layout.html"},
{"tools.html", "common.html", "layout.html"},
{"std.html", "common.html", "layout.html"},
{"subrepo.html", "common.html", "layout.html"},
{"graph.html", "common.html"},
}); err != nil {
log.Fatal(err)
}
if err := parseTextTemplates([][]string{
{"cmd.txt", "common.txt"},
{"dir.txt", "common.txt"},
{"home.txt", "common.txt"},
{"notfound.txt", "common.txt"},
{"pkg.txt", "common.txt"},
{"results.txt", "common.txt"},
}); err != nil {
log.Fatal(err)
}
var err error
db, err = database.New()
if err != nil {
log.Fatalf("Error opening database: %v", err)
}
go runBackgroundTasks()
staticServer := httputil.StaticServer{
Dir: *assetsDir,
MaxAge: time.Hour,
MIMETypes: map[string]string{
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
},
}
statusImageHandlerPNG = staticServer.FileHandler("status.png")
statusImageHandlerSVG = staticServer.FileHandler("status.svg")
apiMux := http.NewServeMux()
apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
apiMux.Handle("/search", apiHandler(serveAPISearch))
apiMux.Handle("/packages", apiHandler(serveAPIPackages))
apiMux.Handle("/importers/", apiHandler(serveAPIImporters))
apiMux.Handle("/imports/", apiHandler(serveAPIImports))
apiMux.Handle("/", apiHandler(serveAPIHome))
mux := http.NewServeMux()
mux.Handle("/-/site.js", staticServer.FilesHandler(
"third_party/jquery.timeago.js",
"site.js"))
mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
if *sidebarEnabled {
mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
}
mux.Handle("/-/", http.NotFoundHandler())
mux.Handle("/-/about", handler(serveAbout))
mux.Handle("/-/bot", handler(serveBot))
mux.Handle("/-/go", handler(serveGoIndex))
mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
mux.Handle("/-/refresh", handler(serveRefresh))
mux.Handle("/-/admin/reindex", http.HandlerFunc(runReindex))
mux.Handle("/-/admin/purgeindex", http.HandlerFunc(runPurgeIndex))
mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
mux.Handle("/code.jquery.com/", http.NotFoundHandler())
mux.Handle("/_ah/health", http.HandlerFunc(serveHealthCheck))
mux.Handle("/_ah/", http.NotFoundHandler())
mux.Handle("/", handler(serveHome))
cacheBusters.Handler = mux
var root http.Handler = rootHandler{
{"api.", apiMux},
{"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
{"www.godoc.org", otherDomainHandler{"https", "godoc.org"}},
{"", mux},
}
if gceLogName != "" {
ctx := context.Background()
logc, err := logging.NewClient(ctx, projID)
if err != nil {
log.Fatalf("Failed to create cloud logging client: %v", err)
}
logger := logc.Logger(gceLogName)
if err := logc.Ping(ctx); err != nil {
log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
}
gceLogger = newGCELogger(logger)
}
http.Handle("/", root)
appengine.Main()
}