Пример #1
0
func detectGCE() {
	if !metadata.OnGCE() {
		return
	}
	v, _ := metadata.InstanceAttributeValue("camlistore-config-dir")
	isGCE = v != ""
}
Пример #2
0
func (cs computeSource) Token() (*oauth2.Token, error) {
	if !metadata.OnGCE() {
		return nil, errors.New("oauth2/google: can't get a token from the metadata service; not running on GCE")
	}
	acct := cs.account
	if acct == "" {
		acct = "default"
	}
	tokenJSON, err := metadata.Get("instance/service-accounts/" + acct + "/token")
	if err != nil {
		return nil, err
	}
	var res struct {
		AccessToken  string `json:"access_token"`
		ExpiresInSec int    `json:"expires_in"`
		TokenType    string `json:"token_type"`
	}
	err = json.NewDecoder(strings.NewReader(tokenJSON)).Decode(&res)
	if err != nil {
		return nil, fmt.Errorf("oauth2/google: invalid token JSON from metadata: %v", err)
	}
	if res.ExpiresInSec == 0 || res.AccessToken == "" {
		return nil, fmt.Errorf("oauth2/google: incomplete token received from metadata")
	}
	return &oauth2.Token{
		AccessToken: res.AccessToken,
		TokenType:   res.TokenType,
		Expiry:      time.Now().Add(time.Duration(res.ExpiresInSec) * time.Second),
	}, nil
}
Пример #3
0
// If we're not running on GCE (e.g. dev mode on localhost) and have
// no other way to get the info, the error value is is errNoRefresh.
func (h *DeployHandler) authenticatedClient() (project string, hc *http.Client, err error) {
	project = os.Getenv("CAMLI_GCE_PROJECT")
	accountFile := os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT")
	if project != "" && accountFile != "" {
		data, errr := ioutil.ReadFile(accountFile)
		err = errr
		if err != nil {
			return
		}
		jwtConf, errr := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/compute.readonly")
		err = errr
		if err != nil {
			return
		}
		hc = jwtConf.Client(context.Background())
		return
	}
	if !metadata.OnGCE() {
		err = errNoRefresh
		return
	}
	project, _ = metadata.ProjectID()
	hc, err = google.DefaultClient(oauth2.NoContext)
	return project, hc, err
}
Пример #4
0
func maybeRemapCloudSQL(host string) (out string, err error) {
	if !strings.HasSuffix(host, cloudSQLSuffix) {
		return host, nil
	}
	inst := strings.TrimSuffix(host, cloudSQLSuffix)
	if !metadata.OnGCE() {
		return "", errors.New("CloudSQL support only available when running on Google Compute Engine.")
	}
	proj, err := metadata.ProjectID()
	if err != nil {
		return "", fmt.Errorf("Failed to lookup GCE project ID: %v", err)
	}

	admin, _ := sqladmin.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource("")))
	listRes, err := admin.Instances.List(proj).Do()
	if err != nil {
		return "", fmt.Errorf("error enumerating Cloud SQL instances: %v", err)
	}
	for _, it := range listRes.Items {
		if !strings.EqualFold(it.Instance, inst) {
			continue
		}
		js, _ := json.Marshal(it)
		log.Printf("Found Cloud SQL instance %s: %s", inst, js)
		for _, ipm := range it.IpAddresses {
			return ipm.IpAddress, nil
		}
		return "", fmt.Errorf("No external IP address for Cloud SQL instances %s", inst)
	}
	var found []string
	for _, it := range listRes.Items {
		found = append(found, it.Instance)
	}
	return "", fmt.Errorf("Cloud SQL instance %q not found. Found: %q", inst, found)
}
Пример #5
0
func checkInProduction() bool {
	if !metadata.OnGCE() {
		return false
	}
	proj, _ := metadata.ProjectID()
	inst, _ := metadata.InstanceName()
	log.Printf("Running on GCE: %v / %v", proj, inst)
	return proj == "camlistore-website" && inst == "camweb"
}
Пример #6
0
func EnsureOnGCE() error {
	for start := time.Now(); time.Since(start) < waitForGCETimeout; time.Sleep(waitForGCEInterval) {
		glog.Infof("Waiting for GCE metadata to be available")
		if metadata.OnGCE() {
			return nil
		}
	}
	return fmt.Errorf("not running on GCE")
}
Пример #7
0
func TestWriteRead(t *testing.T) {
	if !metadata.OnGCE() {
		t.Skipf("Not testing on GCE")
	}
	if *flagBucket == "" {
		t.Skipf("No bucket specified")
	}
	ctx := context.Background()
	cl, err := storage.NewClient(ctx)
	it := cl.Bucket(*flagBucket).Objects(ctx, nil)
	if _, err := it.Next(); err != iterator.Done {
		if err == nil {
			t.Fatalf("Bucket %v is not empty, aborting test.", *flagBucket)
		}
		t.Fatalf("unexpected bucket iteration error: %v", err)
	}
	filename := "camli-gcs_test.txt"
	defer func() {
		if err := cl.Bucket(*flagBucket).Object(filename).Delete(ctx); err != nil {
			t.Fatalf("error while cleaning up: %v", err)
		}
	}()

	// Write to camli-gcs_test.txt
	gcsPath := "/gcs/" + *flagBucket + "/" + filename
	f, err := wkfs.Create(gcsPath)
	if err != nil {
		t.Fatalf("error creating %v: %v", gcsPath, err)
	}
	data := "Hello World"
	if _, err := io.Copy(f, strings.NewReader(data)); err != nil {
		t.Fatalf("error writing to %v: %v", gcsPath, err)
	}
	if err := f.Close(); err != nil {
		t.Fatalf("error closing %v: %v", gcsPath, err)
	}

	// Read back from camli-gcs_test.txt
	g, err := wkfs.Open(gcsPath)
	if err != nil {
		t.Fatalf("error opening %v: %v", gcsPath, err)
	}
	defer g.Close()
	var buf bytes.Buffer
	if _, err := io.Copy(&buf, g); err != nil {
		t.Fatalf("error reading %v: %v", gcsPath, err)
	}
	if buf.String() != data {
		t.Fatalf("error with %v contents: got %v, wanted %v", gcsPath, buf.String(), data)
	}
}
Пример #8
0
func initGCP() {
	initGCPOnce.Do(func() {
		onGCE = metadata.OnGCE()
		if onGCE {
			// These will fail on instances if the metadata service is
			// down or the client is compiled with an API version that
			// has been removed. Since these are not vital, let's ignore
			// them and make their fields in the dockeLogEntry ,omitempty
			projectID, _ = metadata.ProjectID()
			zone, _ = metadata.Zone()
			instanceName, _ = metadata.InstanceName()
			instanceID, _ = metadata.InstanceID()
		}
	})
}
Пример #9
0
func init() {
	if !metadata.OnGCE() {
		return
	}
	hc, err := google.DefaultClient(oauth2.NoContext)
	if err != nil {
		registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
		return
	}
	ctx := context.Background()
	sc, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
	if err != nil {
		registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
		return
	}
	wkfs.RegisterFS("/gcs/", &gcsFS{
		ctx: ctx,
		sc:  sc,
	})
}
Пример #10
0
// DefaultTokenSource is a token source that uses
// "Application Default Credentials".
//
// It looks for credentials in the following places,
// preferring the first location found:
//
//   1. A JSON file whose path is specified by the
//      GOOGLE_APPLICATION_CREDENTIALS environment variable.
//   2. A JSON file in a location known to the gcloud command-line tool.
//      On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
//      On other systems, $HOME/.config/gcloud/application_default_credentials.json.
//   3. On Google App Engine it uses the appengine.AccessToken function.
//   4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
//      credentials from the metadata server.
//      (In this final case any provided scopes are ignored.)
//
// For more details, see:
// https://developers.google.com/accounts/docs/application-default-credentials
//
func DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) {
	// First, try the environment variable.
	const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
	if filename := os.Getenv(envVar); filename != "" {
		ts, err := tokenSourceFromFile(ctx, filename, scope)
		if err != nil {
			return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
		}
		return ts, nil
	}

	// Second, try a well-known file.
	filename := wellKnownFile()
	_, err := os.Stat(filename)
	if err == nil {
		ts, err2 := tokenSourceFromFile(ctx, filename, scope)
		if err2 == nil {
			return ts, nil
		}
		err = err2
	} else if os.IsNotExist(err) {
		err = nil // ignore this error
	}
	if err != nil {
		return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err)
	}

	// Third, if we're on Google App Engine use those credentials.
	if appengineTokenFunc != nil && !appengineVM {
		return AppEngineTokenSource(ctx, scope...), nil
	}

	// Fourth, if we're on Google Compute Engine use the metadata server.
	if metadata.OnGCE() {
		return ComputeTokenSource(""), nil
	}

	// None are found; return helpful error.
	const url = "https://developers.google.com/accounts/docs/application-default-credentials"
	return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url)
}
Пример #11
0
// NewServiceClient returns a Client for use when running on Google
// Compute Engine.  This client can access buckets owned by the same
// project ID as the VM.
func NewServiceClient() (*Client, error) {
	if !metadata.OnGCE() {
		return nil, errors.New("not running on Google Compute Engine")
	}
	scopes, _ := metadata.Scopes("default")
	haveScope := func(scope string) bool {
		for _, x := range scopes {
			if x == scope {
				return true
			}
		}
		return false
	}
	if !haveScope("https://www.googleapis.com/auth/devstorage.full_control") &&
		!haveScope("https://www.googleapis.com/auth/devstorage.read_write") {
		return nil, errors.New("when this Google Compute Engine VM instance was created, it wasn't granted access to Cloud Storage")
	}
	client := oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))
	service, _ := api.New(client)
	return &Client{client: client, service: service}, nil
}
Пример #12
0
// FindDefaultCredentials searches for "Application Default Credentials".
//
// It looks for credentials in the following places,
// preferring the first location found:
//
//   1. A JSON file whose path is specified by the
//      GOOGLE_APPLICATION_CREDENTIALS environment variable.
//   2. A JSON file in a location known to the gcloud command-line tool.
//      On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
//      On other systems, $HOME/.config/gcloud/application_default_credentials.json.
//   3. On Google App Engine it uses the appengine.AccessToken function.
//   4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
//      credentials from the metadata server.
//      (In this final case any provided scopes are ignored.)
func FindDefaultCredentials(ctx context.Context, scope ...string) (*DefaultCredentials, error) {
	// First, try the environment variable.
	const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
	if filename := os.Getenv(envVar); filename != "" {
		creds, err := readCredentialsFile(ctx, filename, scope)
		if err != nil {
			return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
		}
		return creds, nil
	}

	// Second, try a well-known file.
	filename := wellKnownFile()
	if creds, err := readCredentialsFile(ctx, filename, scope); err == nil {
		return creds, nil
	} else if !os.IsNotExist(err) {
		return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err)
	}

	// Third, if we're on Google App Engine use those credentials.
	if appengineTokenFunc != nil && !appengineVM {
		return &DefaultCredentials{
			ProjectID:   appengineAppIDFunc(ctx),
			TokenSource: AppEngineTokenSource(ctx, scope...),
		}, nil
	}

	// Fourth, if we're on Google Compute Engine use the metadata server.
	if metadata.OnGCE() {
		id, _ := metadata.ProjectID()
		return &DefaultCredentials{
			ProjectID:   id,
			TokenSource: ComputeTokenSource(""),
		}, nil
	}

	// None are found; return helpful error.
	const url = "https://developers.google.com/accounts/docs/application-default-credentials"
	return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url)
}
func newInstance() *Instance {
	var i = new(Instance)
	if !metadata.OnGCE() {
		i.Error = "Not running on GCE"
		return i
	}

	a := &assigner{}
	i.Id = a.assign(metadata.InstanceID)
	i.Zone = a.assign(metadata.Zone)
	i.Name = a.assign(metadata.InstanceName)
	i.Hostname = a.assign(metadata.Hostname)
	i.Project = a.assign(metadata.ProjectID)
	i.InternalIP = a.assign(metadata.InternalIP)
	i.ExternalIP = a.assign(metadata.ExternalIP)
	i.Version = version

	if a.err != nil {
		i.Error = a.err.Error()
	}
	return i
}
Пример #14
0
func main() {
	if !metadata.OnGCE() {
		bucket := os.Getenv("GCSBUCKET")
		if bucket == "" {
			log.Fatal("You need to set the GCSBUCKET env var to specify the Google Cloud Storage bucket to serve from.")
		}
		projectID := os.Getenv("GCEPROJECTID")
		if projectID == "" {
			log.Fatal("You need to set the GCEPROJECTID env var to specify the Google Cloud project where the instance will run.")
		}
		(&cloudlaunch.Config{
			Name:         "serveoncloud",
			BinaryBucket: bucket,
			GCEProjectID: projectID,
			Scopes: []string{
				storageapi.DevstorageFullControlScope,
				compute.ComputeScope,
			},
		}).MaybeDeploy()
		return
	}

	flag.Parse()

	storageURLRxp := regexp.MustCompile(`https://storage.googleapis.com/(.+?)/serveoncloud.*`)
	cloudConfig, err := metadata.InstanceAttributeValue("user-data")
	if err != nil || cloudConfig == "" {
		log.Fatalf("could not get cloud config from metadata: %v", err)
	}
	m := storageURLRxp.FindStringSubmatch(cloudConfig)
	if len(m) < 2 {
		log.Fatal("storage URL not found in cloud config")
	}
	gcsBucket = m[1]

	http.HandleFunc("/", serveHTTP)

	log.Fatal(http.ListenAndServe(*httpAddr, nil))
}
Пример #15
0
func testStorage(t *testing.T, bucketDir string) {
	if *bucket == "" && *configFile == "" {
		t.Skip("Skipping test without --bucket or --config flag")
	}
	var refreshToken string
	if *configFile != "" {
		data, err := ioutil.ReadFile(*configFile)
		if err != nil {
			t.Fatalf("Error reading config file %v: %v", *configFile, err)
		}
		var conf Config
		if err := json.Unmarshal(data, &conf); err != nil {
			t.Fatalf("Error decoding config file %v: %v", *configFile, err)
		}
		*clientID = conf.Auth.ClientID
		*clientSecret = conf.Auth.ClientSecret
		refreshToken = conf.Auth.RefreshToken
		*bucket = conf.Bucket
	}
	if *bucket == "" {
		t.Fatal("bucket not provided in config file or as a flag.")
	}
	if *clientID == "" {
		if !metadata.OnGCE() {
			if *clientSecret == "" {
				t.Fatal("client ID and client secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
			}
		} else {
			*clientID = "auto"
		}
	}
	if *configFile == "" {
		config := &oauth2.Config{
			Scopes:       []string{storage.ScopeReadWrite},
			Endpoint:     google.Endpoint,
			ClientID:     *clientID,
			ClientSecret: *clientSecret,
			RedirectURL:  oauthutil.TitleBarRedirectURL,
		}
		if !metadata.OnGCE() {
			token, err := oauth2.ReuseTokenSource(nil,
				&oauthutil.TokenSource{
					Config:    config,
					CacheFile: *tokenCache,
					AuthCode: func() string {
						if *authCode == "" {
							t.Skipf("Re-run using --auth_code= with the value obtained from %s",
								config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
							return ""
						}
						return *authCode
					},
				}).Token()
			if err != nil {
				t.Fatalf("could not acquire token: %v", err)
			}
			refreshToken = token.RefreshToken
		}
	}

	bucketWithDir := path.Join(*bucket, bucketDir)

	storagetest.TestOpt(t, storagetest.Opts{
		New: func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
			sto, err := newFromConfig(nil, jsonconfig.Obj{
				"bucket": bucketWithDir,
				"auth": map[string]interface{}{
					"client_id":     *clientID,
					"client_secret": *clientSecret,
					"refresh_token": refreshToken,
				},
			})
			if err != nil {
				t.Fatal(err)
			}
			if !testing.Short() {
				log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
			}
			// Bail if bucket is not empty
			ctx := context.Background()
			stor := sto.(*Storage)
			objs, err := stor.client.Bucket(stor.bucket).List(ctx, nil)
			if err != nil {
				t.Fatalf("Error checking if bucket is empty: %v", err)
			}
			if len(objs.Results) != 0 {
				t.Fatalf("Refusing to run test: bucket %v is not empty", *bucket)
			}
			if bucketWithDir != *bucket {
				// Adding "a", and "c" objects in the bucket to make sure objects out of the
				// "directory" are not touched and have no influence.
				for _, key := range []string{"a", "c"} {
					w := stor.client.Bucket(stor.bucket).Object(key).NewWriter(ctx)
					if _, err := io.Copy(w, strings.NewReader(key)); err != nil {
						t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
					}
					if err := w.Close(); err != nil {
						t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
					}
				}
			}

			clearBucket := func(beforeTests bool) func() {
				return func() {
					var all []blob.Ref
					blobserver.EnumerateAll(context.TODO(), sto, func(sb blob.SizedRef) error {
						t.Logf("Deleting: %v", sb.Ref)
						all = append(all, sb.Ref)
						return nil
					})
					if err := sto.RemoveBlobs(all); err != nil {
						t.Fatalf("Error removing blobs during cleanup: %v", err)
					}
					if beforeTests {
						return
					}
					if bucketWithDir != *bucket {
						// checking that "a" and "c" at the root were left untouched.
						for _, key := range []string{"a", "c"} {
							rc, err := stor.client.Bucket(stor.bucket).Object(key).NewReader(ctx)
							if err != nil {
								t.Fatalf("could not find object %s after tests: %v", key, err)
							}
							if _, err := io.Copy(ioutil.Discard, rc); err != nil {
								t.Fatalf("could not find object %s after tests: %v", key, err)
							}
							if err := stor.client.Bucket(stor.bucket).Object(key).Delete(ctx); err != nil {
								t.Fatalf("could not remove object %s after tests: %v", key, err)
							}

						}
					}
				}
			}
			clearBucket(true)()
			return sto, clearBucket(false)
		},
	})
}
Пример #16
0
func main() {
	launchConfig.MaybeDeploy()
	flag.Parse()
	setProdFlags()

	if *root == "" {
		var err error
		*root, err = os.Getwd()
		if err != nil {
			log.Fatalf("Failed to getwd: %v", err)
		}
	}
	readTemplates()
	if err := initGithubSyncing(); err != nil {
		log.Fatalf("error setting up syncing to github: %v")
	}
	go runDemoBlobserverLoop()

	mux := http.DefaultServeMux
	mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
	mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
	mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static")))))
	mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks")))))
	mux.Handle(pkgPattern, godocHandler{})
	mux.Handle(cmdPattern, godocHandler{})
	mux.Handle(appPattern, godocHandler{})
	mux.HandleFunc(errPattern, errHandler)

	mux.HandleFunc("/r/", gerritRedirect)
	mux.HandleFunc("/dl/", releaseRedirect)
	mux.HandleFunc("/debug/ip", ipHandler)
	mux.HandleFunc("/debug/uptime", uptimeHandler)
	mux.Handle("/doc/contributing", redirTo("/code#contributing"))
	mux.Handle("/lists", redirTo("/community"))

	mux.HandleFunc("/contributors", contribHandler())
	mux.HandleFunc("/doc/", docHandler)
	mux.HandleFunc("/", mainHandler)

	if buildbotHost != "" && buildbotBackend != "" {
		if _, err := url.Parse(buildbotBackend); err != nil {
			log.Fatalf("Failed to parse %v as a URL: %v", buildbotBackend, err)
		}
		bbhpattern := strings.TrimRight(buildbotHost, "/") + "/"
		mux.HandleFunc(bbhpattern, func(w http.ResponseWriter, r *http.Request) {
			http.Redirect(w, r, buildbotBackend, http.StatusFound)
		})
	}

	gceLauncher, err := gceDeployHandler("/launch/")
	if err != nil {
		log.Printf("Not installing GCE /launch/ handler: %v", err)
		mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) {
			http.Error(w, fmt.Sprintf("GCE launcher disabled: %v", err), 500)
		})
	} else {
		mux.Handle("/launch/", gceLauncher)
	}

	var handler http.Handler = &redirectRootHandler{Handler: mux}
	if *logDir != "" || *logStdout {
		handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout))
	}
	if *gceLogName != "" {
		projID := projectID()
		var hc *http.Client
		if !metadata.OnGCE() {
			hc = httpClient(projID)
		}
		ctx := context.Background()
		var logc *logging.Client
		if metadata.OnGCE() {
			logc, err = logging.NewClient(ctx, projID, *gceLogName)
		} else {
			logc, err = logging.NewClient(ctx, projID, *gceLogName, option.WithHTTPClient(hc))
		}
		if err != nil {
			log.Fatal(err)
		}
		if err := logc.Ping(); err != nil {
			log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
		}
		handler = NewLoggingHandler(handler, gceLogger{logc})
		if gceLauncher != nil {
			var logc *logging.Client
			if metadata.OnGCE() {
				logc, err = logging.NewClient(ctx, projID, *gceLogName)
			} else {
				logc, err = logging.NewClient(ctx, projID, *gceLogName, option.WithHTTPClient(hc))
			}
			if err != nil {
				log.Fatal(err)
			}
			logc.CommonLabels = map[string]string{
				"from": "camli-gce-launcher",
			}
			logger := logc.Logger(logging.Default)
			logger.SetPrefix("launcher: ")
			gceLauncher.SetLogger(logger)
		}
	}

	emailErr := make(chan error)
	startEmailCommitLoop(emailErr)

	if *alsoRun != "" {
		runAsChild(*alsoRun)
	}

	httpServer := &http.Server{
		Addr:         *httpAddr,
		Handler:      handler,
		ReadTimeout:  5 * time.Minute,
		WriteTimeout: 30 * time.Minute,
	}

	httpErr := make(chan error)
	go func() {
		log.Printf("Listening for HTTP on %v", *httpAddr)
		httpErr <- httpServer.ListenAndServe()
	}()

	httpsErr := make(chan error)
	if *httpsAddr != "" {
		go func() {
			httpsErr <- serveHTTPS(httpServer)
		}()
	}

	if *flagChromeBugRepro {
		go func() {
			log.Printf("Repro handler failed: %v", repro(":8001", "foo:bar"))
		}()
	}

	select {
	case err := <-emailErr:
		log.Fatalf("Error sending emails: %v", err)
	case err := <-httpErr:
		log.Fatalf("Error serving HTTP: %v", err)
	case err := <-httpsErr:
		log.Fatalf("Error serving HTTPS: %v", err)
	}
}
Пример #17
0
func main() {
	flag.Parse()
	log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))

	doc.SetDefaultGOOS(*defaultGOOS)
	httpClient = newHTTPClient()

	var (
		gceLogName string
		projID     string
	)

	if metadata.OnGCE() {
		acct, err := metadata.ProjectAttributeValue("ga-account")
		if err != nil {
			log.Printf("querying metadata for ga-account: %v", err)
		} else {
			gaAccount = acct
		}

		// Get the log name on GCE and setup context for creating a GCE log client.
		if name, err := metadata.ProjectAttributeValue("gce-log-name"); err != nil {
			log.Printf("querying metadata for gce-log-name: %v", err)
		} else {
			gceLogName = name
			if id, err := metadata.ProjectID(); err != nil {
				log.Printf("querying metadata for project ID: %v", err)
			} else {
				projID = id
			}
		}
	} else {
		gaAccount = os.Getenv("GA_ACCOUNT")
	}

	if err := parseHTMLTemplates([][]string{
		{"about.html", "common.html", "layout.html"},
		{"bot.html", "common.html", "layout.html"},
		{"cmd.html", "common.html", "layout.html"},
		{"dir.html", "common.html", "layout.html"},
		{"home.html", "common.html", "layout.html"},
		{"importers.html", "common.html", "layout.html"},
		{"importers_robot.html", "common.html", "layout.html"},
		{"imports.html", "common.html", "layout.html"},
		{"notfound.html", "common.html", "layout.html"},
		{"pkg.html", "common.html", "layout.html"},
		{"results.html", "common.html", "layout.html"},
		{"tools.html", "common.html", "layout.html"},
		{"std.html", "common.html", "layout.html"},
		{"subrepo.html", "common.html", "layout.html"},
		{"graph.html", "common.html"},
	}); err != nil {
		log.Fatal(err)
	}

	if err := parseTextTemplates([][]string{
		{"cmd.txt", "common.txt"},
		{"dir.txt", "common.txt"},
		{"home.txt", "common.txt"},
		{"notfound.txt", "common.txt"},
		{"pkg.txt", "common.txt"},
		{"results.txt", "common.txt"},
	}); err != nil {
		log.Fatal(err)
	}

	var err error
	db, err = database.New()
	if err != nil {
		log.Fatalf("Error opening database: %v", err)
	}

	go runBackgroundTasks()

	staticServer := httputil.StaticServer{
		Dir:    *assetsDir,
		MaxAge: time.Hour,
		MIMETypes: map[string]string{
			".css": "text/css; charset=utf-8",
			".js":  "text/javascript; charset=utf-8",
		},
	}
	statusImageHandlerPNG = staticServer.FileHandler("status.png")
	statusImageHandlerSVG = staticServer.FileHandler("status.svg")

	apiMux := http.NewServeMux()
	apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
	apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
	apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
	apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
	apiMux.Handle("/search", apiHandler(serveAPISearch))
	apiMux.Handle("/packages", apiHandler(serveAPIPackages))
	apiMux.Handle("/importers/", apiHandler(serveAPIImporters))
	apiMux.Handle("/imports/", apiHandler(serveAPIImports))
	apiMux.Handle("/", apiHandler(serveAPIHome))

	mux := http.NewServeMux()
	mux.Handle("/-/site.js", staticServer.FilesHandler(
		"third_party/jquery.timeago.js",
		"site.js"))
	mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
	mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
	mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
	mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
	if *sidebarEnabled {
		mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
	}
	mux.Handle("/-/", http.NotFoundHandler())

	mux.Handle("/-/about", handler(serveAbout))
	mux.Handle("/-/bot", handler(serveBot))
	mux.Handle("/-/go", handler(serveGoIndex))
	mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
	mux.Handle("/-/refresh", handler(serveRefresh))
	mux.Handle("/-/admin/reindex", http.HandlerFunc(runReindex))
	mux.Handle("/-/admin/purgeindex", http.HandlerFunc(runPurgeIndex))
	mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
	mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
	mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
	mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
	mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
	mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
	mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
	mux.Handle("/code.jquery.com/", http.NotFoundHandler())
	mux.Handle("/_ah/health", http.HandlerFunc(serveHealthCheck))
	mux.Handle("/_ah/", http.NotFoundHandler())
	mux.Handle("/", handler(serveHome))

	cacheBusters.Handler = mux

	var root http.Handler = rootHandler{
		{"api.", apiMux},
		{"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
		{"www.godoc.org", otherDomainHandler{"https", "godoc.org"}},
		{"", mux},
	}
	if gceLogName != "" {
		ctx := context.Background()

		logc, err := logging.NewClient(ctx, projID)
		if err != nil {
			log.Fatalf("Failed to create cloud logging client: %v", err)
		}
		logger := logc.Logger(gceLogName)

		if err := logc.Ping(ctx); err != nil {
			log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
		}

		gceLogger = newGCELogger(logger)
	}

	http.Handle("/", root)
	appengine.Main()
}
Пример #18
0
func main() {
	launchConfig.MaybeDeploy()
	flag.Parse()

	var kv keyValue
	var httpsListenAddr string
	if metadata.OnGCE() {
		httpsListenAddr = ":443"
		dsClient, err := datastore.NewClient(context.Background(), GCEProjectID)
		if err != nil {
			log.Fatalf("Error creating datastore client for records: %v", err)
		}
		kv = cachedStore{
			dsClient: dsClient,
			cache:    lru.New(cacheSize),
		}
	} else {
		httpsListenAddr = ":4430"
		kv = memkv{skv: sorted.NewMemoryKeyValue()}
	}
	if err := kv.Set("6401800c.camlistore.net.", "159.203.246.79"); err != nil {
		log.Fatalf("Error adding %v:%v record: %v", "6401800c.camlistore.net.", "159.203.246.79", err)
	}
	if err := kv.Set(domain, *flagServerIP); err != nil {
		log.Fatalf("Error adding %v:%v record: %v", domain, *flagServerIP, err)
	}
	if err := kv.Set("www.camlistore.net.", *flagServerIP); err != nil {
		log.Fatalf("Error adding %v:%v record: %v", "www.camlistore.net.", *flagServerIP, err)
	}

	ds := newDNSServer(kv)
	cs := &gpgchallenge.Server{
		OnSuccess: func(identity string, address string) error {
			log.Printf("Adding %v.camlistore.net. as %v", identity, address)
			return ds.dataSource.Set(strings.ToLower(identity+".camlistore.net."), address)
		},
	}

	tcperr := make(chan error, 1)
	udperr := make(chan error, 1)
	httperr := make(chan error, 1)
	log.Printf("serving DNS on %s\n", *addr)
	go func() {
		tcperr <- dns.ListenAndServe(*addr, "tcp", ds)
	}()
	go func() {
		udperr <- dns.ListenAndServe(*addr, "udp", ds)
	}()
	if metadata.OnGCE() {
		// TODO(mpl): if we want to get a cert for anything
		// *.camlistore.net, it's a bit of a chicken and egg problem, since
		// we need camnetdns itself to be already running and answering DNS
		// queries. It's probably doable, but easier for now to just ask
		// one for camnetdns.camlistore.org, since that name is not
		// resolved by camnetdns.
		hostname := strings.TrimSuffix(authorityNS, ".")
		m := autocert.Manager{
			Prompt:     autocert.AcceptTOS,
			HostPolicy: autocert.HostWhitelist(hostname),
			Cache:      autocert.DirCache(osutil.DefaultLetsEncryptCache()),
		}
		ln, err := tls.Listen("tcp", httpsListenAddr, &tls.Config{
			Rand:           rand.Reader,
			Time:           time.Now,
			NextProtos:     []string{http2.NextProtoTLS, "http/1.1"},
			MinVersion:     tls.VersionTLS12,
			GetCertificate: m.GetCertificate,
		})
		if err != nil {
			log.Fatalf("Error listening on %v: %v", httpsListenAddr, err)
		}
		go func() {
			httperr <- http.Serve(ln, cs)
		}()
	}
	select {
	case err := <-tcperr:
		log.Fatalf("DNS over TCP error: %v", err)
	case err := <-udperr:
		log.Fatalf("DNS error: %v", err)
	case err := <-httperr:
		log.Fatalf("HTTP server error: %v", err)
	}
}
Пример #19
0
func defaultListenAddr() string {
	if metadata.OnGCE() {
		return ":53"
	}
	return ":5300"
}
Пример #20
0
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
	var (
		auth      = config.RequiredObject("auth")
		bucket    = config.RequiredString("bucket")
		cacheSize = config.OptionalInt64("cacheSize", 32<<20)

		clientID     = auth.RequiredString("client_id") // or "auto" for service accounts
		clientSecret = auth.OptionalString("client_secret", "")
		refreshToken = auth.OptionalString("refresh_token", "")
	)

	if err := config.Validate(); err != nil {
		return nil, err
	}
	if err := auth.Validate(); err != nil {
		return nil, err
	}

	var dirPrefix string
	if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
		dirPrefix = parts[1]
		bucket = parts[0]
	}
	if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
		dirPrefix += "/"
	}
	gs := &Storage{
		bucket:    bucket,
		dirPrefix: dirPrefix,
	}

	var (
		ctx = context.Background()
		ts  oauth2.TokenSource
		cl  *storage.Client
		err error
	)
	if clientID == "auto" {
		if !metadata.OnGCE() {
			return nil, errors.New(`Cannot use "auto" client_id when not running on GCE`)
		}
		ts, err = google.DefaultTokenSource(ctx, storage.ScopeReadWrite)
		if err != nil {
			return nil, err
		}
		cl, err = storage.NewClient(ctx)
		if err != nil {
			return nil, err
		}
	} else {
		if clientSecret == "" {
			return nil, errors.New("missing required parameter 'client_secret'")
		}
		if refreshToken == "" {
			return nil, errors.New("missing required parameter 'refresh_token'")
		}
		ts = oauthutil.NewRefreshTokenSource(&oauth2.Config{
			Scopes:       []string{storage.ScopeReadWrite},
			Endpoint:     google.Endpoint,
			ClientID:     clientID,
			ClientSecret: clientSecret,
			RedirectURL:  oauthutil.TitleBarRedirectURL,
		}, refreshToken)
		cl, err = storage.NewClient(ctx, option.WithTokenSource(ts))
		if err != nil {
			return nil, err
		}
	}

	gs.baseHTTPClient = oauth2.NewClient(ctx, ts)
	gs.client = cl

	if cacheSize != 0 {
		gs.cache = memory.NewCache(cacheSize)
	}

	ba, err := gs.client.Bucket(gs.bucket).Attrs(ctx)
	if err != nil {
		return nil, fmt.Errorf("error statting bucket %q: %v", gs.bucket, err)
	}
	hash := sha1.New()
	fmt.Fprintf(hash, "%v%v", ba.Created, ba.MetaGeneration)
	gs.genRandom = fmt.Sprintf("%x", hash.Sum(nil))
	gs.genTime = ba.Created

	return gs, nil
}
Пример #21
0
// NewAuthTransport gives new AuthTransport created with GitHub credentials
// read from GCE metadata when the metadata server is accessible (we're on GCE)
// or read from environment varialbes otherwise.
func NewAuthTransport(base http.RoundTripper) *AuthTransport {
	if metadata.OnGCE() {
		return NewAuthTransportFromMetadata(base)
	}
	return NewAuthTransportFromEnvironment(base)
}