func detectGCE() {
if !metadata.OnGCE() {
return
}
v, _ := metadata.InstanceAttributeValue("camlistore-config-dir")
isGCE = v != ""
}
func (cs computeSource) Token() (*oauth2.Token, error) {
if !metadata.OnGCE() {
return nil, errors.New("oauth2/google: can't get a token from the metadata service; not running on GCE")
}
acct := cs.account
if acct == "" {
acct = "default"
}
tokenJSON, err := metadata.Get("instance/service-accounts/" + acct + "/token")
if err != nil {
return nil, err
}
var res struct {
AccessToken string `json:"access_token"`
ExpiresInSec int `json:"expires_in"`
TokenType string `json:"token_type"`
}
err = json.NewDecoder(strings.NewReader(tokenJSON)).Decode(&res)
if err != nil {
return nil, fmt.Errorf("oauth2/google: invalid token JSON from metadata: %v", err)
}
if res.ExpiresInSec == 0 || res.AccessToken == "" {
return nil, fmt.Errorf("oauth2/google: incomplete token received from metadata")
}
return &oauth2.Token{
AccessToken: res.AccessToken,
TokenType: res.TokenType,
Expiry: time.Now().Add(time.Duration(res.ExpiresInSec) * time.Second),
}, nil
}
// If we're not running on GCE (e.g. dev mode on localhost) and have
// no other way to get the info, the error value is is errNoRefresh.
func (h *DeployHandler) authenticatedClient() (project string, hc *http.Client, err error) {
project = os.Getenv("CAMLI_GCE_PROJECT")
accountFile := os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT")
if project != "" && accountFile != "" {
data, errr := ioutil.ReadFile(accountFile)
err = errr
if err != nil {
return
}
jwtConf, errr := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/compute.readonly")
err = errr
if err != nil {
return
}
hc = jwtConf.Client(context.Background())
return
}
if !metadata.OnGCE() {
err = errNoRefresh
return
}
project, _ = metadata.ProjectID()
hc, err = google.DefaultClient(oauth2.NoContext)
return project, hc, err
}
func maybeRemapCloudSQL(host string) (out string, err error) {
if !strings.HasSuffix(host, cloudSQLSuffix) {
return host, nil
}
inst := strings.TrimSuffix(host, cloudSQLSuffix)
if !metadata.OnGCE() {
return "", errors.New("CloudSQL support only available when running on Google Compute Engine.")
}
proj, err := metadata.ProjectID()
if err != nil {
return "", fmt.Errorf("Failed to lookup GCE project ID: %v", err)
}
admin, _ := sqladmin.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource("")))
listRes, err := admin.Instances.List(proj).Do()
if err != nil {
return "", fmt.Errorf("error enumerating Cloud SQL instances: %v", err)
}
for _, it := range listRes.Items {
if !strings.EqualFold(it.Instance, inst) {
continue
}
js, _ := json.Marshal(it)
log.Printf("Found Cloud SQL instance %s: %s", inst, js)
for _, ipm := range it.IpAddresses {
return ipm.IpAddress, nil
}
return "", fmt.Errorf("No external IP address for Cloud SQL instances %s", inst)
}
var found []string
for _, it := range listRes.Items {
found = append(found, it.Instance)
}
return "", fmt.Errorf("Cloud SQL instance %q not found. Found: %q", inst, found)
}
func checkInProduction() bool {
if !metadata.OnGCE() {
return false
}
proj, _ := metadata.ProjectID()
inst, _ := metadata.InstanceName()
log.Printf("Running on GCE: %v / %v", proj, inst)
return proj == "camlistore-website" && inst == "camweb"
}
func EnsureOnGCE() error {
for start := time.Now(); time.Since(start) < waitForGCETimeout; time.Sleep(waitForGCEInterval) {
glog.Infof("Waiting for GCE metadata to be available")
if metadata.OnGCE() {
return nil
}
}
return fmt.Errorf("not running on GCE")
}
func TestWriteRead(t *testing.T) {
if !metadata.OnGCE() {
t.Skipf("Not testing on GCE")
}
if *flagBucket == "" {
t.Skipf("No bucket specified")
}
ctx := context.Background()
cl, err := storage.NewClient(ctx)
it := cl.Bucket(*flagBucket).Objects(ctx, nil)
if _, err := it.Next(); err != iterator.Done {
if err == nil {
t.Fatalf("Bucket %v is not empty, aborting test.", *flagBucket)
}
t.Fatalf("unexpected bucket iteration error: %v", err)
}
filename := "camli-gcs_test.txt"
defer func() {
if err := cl.Bucket(*flagBucket).Object(filename).Delete(ctx); err != nil {
t.Fatalf("error while cleaning up: %v", err)
}
}()
// Write to camli-gcs_test.txt
gcsPath := "/gcs/" + *flagBucket + "/" + filename
f, err := wkfs.Create(gcsPath)
if err != nil {
t.Fatalf("error creating %v: %v", gcsPath, err)
}
data := "Hello World"
if _, err := io.Copy(f, strings.NewReader(data)); err != nil {
t.Fatalf("error writing to %v: %v", gcsPath, err)
}
if err := f.Close(); err != nil {
t.Fatalf("error closing %v: %v", gcsPath, err)
}
// Read back from camli-gcs_test.txt
g, err := wkfs.Open(gcsPath)
if err != nil {
t.Fatalf("error opening %v: %v", gcsPath, err)
}
defer g.Close()
var buf bytes.Buffer
if _, err := io.Copy(&buf, g); err != nil {
t.Fatalf("error reading %v: %v", gcsPath, err)
}
if buf.String() != data {
t.Fatalf("error with %v contents: got %v, wanted %v", gcsPath, buf.String(), data)
}
}
func initGCP() {
initGCPOnce.Do(func() {
onGCE = metadata.OnGCE()
if onGCE {
// These will fail on instances if the metadata service is
// down or the client is compiled with an API version that
// has been removed. Since these are not vital, let's ignore
// them and make their fields in the dockeLogEntry ,omitempty
projectID, _ = metadata.ProjectID()
zone, _ = metadata.Zone()
instanceName, _ = metadata.InstanceName()
instanceID, _ = metadata.InstanceID()
}
})
}
func init() {
if !metadata.OnGCE() {
return
}
hc, err := google.DefaultClient(oauth2.NoContext)
if err != nil {
registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
return
}
ctx := context.Background()
sc, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
if err != nil {
registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
return
}
wkfs.RegisterFS("/gcs/", &gcsFS{
ctx: ctx,
sc: sc,
})
}
// DefaultTokenSource is a token source that uses
// "Application Default Credentials".
//
// It looks for credentials in the following places,
// preferring the first location found:
//
// 1. A JSON file whose path is specified by the
// GOOGLE_APPLICATION_CREDENTIALS environment variable.
// 2. A JSON file in a location known to the gcloud command-line tool.
// On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
// On other systems, $HOME/.config/gcloud/application_default_credentials.json.
// 3. On Google App Engine it uses the appengine.AccessToken function.
// 4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
// credentials from the metadata server.
// (In this final case any provided scopes are ignored.)
//
// For more details, see:
// https://developers.google.com/accounts/docs/application-default-credentials
//
func DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) {
// First, try the environment variable.
const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
if filename := os.Getenv(envVar); filename != "" {
ts, err := tokenSourceFromFile(ctx, filename, scope)
if err != nil {
return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
}
return ts, nil
}
// Second, try a well-known file.
filename := wellKnownFile()
_, err := os.Stat(filename)
if err == nil {
ts, err2 := tokenSourceFromFile(ctx, filename, scope)
if err2 == nil {
return ts, nil
}
err = err2
} else if os.IsNotExist(err) {
err = nil // ignore this error
}
if err != nil {
return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err)
}
// Third, if we're on Google App Engine use those credentials.
if appengineTokenFunc != nil && !appengineVM {
return AppEngineTokenSource(ctx, scope...), nil
}
// Fourth, if we're on Google Compute Engine use the metadata server.
if metadata.OnGCE() {
return ComputeTokenSource(""), nil
}
// None are found; return helpful error.
const url = "https://developers.google.com/accounts/docs/application-default-credentials"
return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url)
}
// NewServiceClient returns a Client for use when running on Google
// Compute Engine. This client can access buckets owned by the same
// project ID as the VM.
func NewServiceClient() (*Client, error) {
if !metadata.OnGCE() {
return nil, errors.New("not running on Google Compute Engine")
}
scopes, _ := metadata.Scopes("default")
haveScope := func(scope string) bool {
for _, x := range scopes {
if x == scope {
return true
}
}
return false
}
if !haveScope("https://www.googleapis.com/auth/devstorage.full_control") &&
!haveScope("https://www.googleapis.com/auth/devstorage.read_write") {
return nil, errors.New("when this Google Compute Engine VM instance was created, it wasn't granted access to Cloud Storage")
}
client := oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))
service, _ := api.New(client)
return &Client{client: client, service: service}, nil
}
// FindDefaultCredentials searches for "Application Default Credentials".
//
// It looks for credentials in the following places,
// preferring the first location found:
//
// 1. A JSON file whose path is specified by the
// GOOGLE_APPLICATION_CREDENTIALS environment variable.
// 2. A JSON file in a location known to the gcloud command-line tool.
// On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
// On other systems, $HOME/.config/gcloud/application_default_credentials.json.
// 3. On Google App Engine it uses the appengine.AccessToken function.
// 4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
// credentials from the metadata server.
// (In this final case any provided scopes are ignored.)
func FindDefaultCredentials(ctx context.Context, scope ...string) (*DefaultCredentials, error) {
// First, try the environment variable.
const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
if filename := os.Getenv(envVar); filename != "" {
creds, err := readCredentialsFile(ctx, filename, scope)
if err != nil {
return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err)
}
return creds, nil
}
// Second, try a well-known file.
filename := wellKnownFile()
if creds, err := readCredentialsFile(ctx, filename, scope); err == nil {
return creds, nil
} else if !os.IsNotExist(err) {
return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err)
}
// Third, if we're on Google App Engine use those credentials.
if appengineTokenFunc != nil && !appengineVM {
return &DefaultCredentials{
ProjectID: appengineAppIDFunc(ctx),
TokenSource: AppEngineTokenSource(ctx, scope...),
}, nil
}
// Fourth, if we're on Google Compute Engine use the metadata server.
if metadata.OnGCE() {
id, _ := metadata.ProjectID()
return &DefaultCredentials{
ProjectID: id,
TokenSource: ComputeTokenSource(""),
}, nil
}
// None are found; return helpful error.
const url = "https://developers.google.com/accounts/docs/application-default-credentials"
return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url)
}
func newInstance() *Instance {
var i = new(Instance)
if !metadata.OnGCE() {
i.Error = "Not running on GCE"
return i
}
a := &assigner{}
i.Id = a.assign(metadata.InstanceID)
i.Zone = a.assign(metadata.Zone)
i.Name = a.assign(metadata.InstanceName)
i.Hostname = a.assign(metadata.Hostname)
i.Project = a.assign(metadata.ProjectID)
i.InternalIP = a.assign(metadata.InternalIP)
i.ExternalIP = a.assign(metadata.ExternalIP)
i.Version = version
if a.err != nil {
i.Error = a.err.Error()
}
return i
}
func main() {
if !metadata.OnGCE() {
bucket := os.Getenv("GCSBUCKET")
if bucket == "" {
log.Fatal("You need to set the GCSBUCKET env var to specify the Google Cloud Storage bucket to serve from.")
}
projectID := os.Getenv("GCEPROJECTID")
if projectID == "" {
log.Fatal("You need to set the GCEPROJECTID env var to specify the Google Cloud project where the instance will run.")
}
(&cloudlaunch.Config{
Name: "serveoncloud",
BinaryBucket: bucket,
GCEProjectID: projectID,
Scopes: []string{
storageapi.DevstorageFullControlScope,
compute.ComputeScope,
},
}).MaybeDeploy()
return
}
flag.Parse()
storageURLRxp := regexp.MustCompile(`https://storage.googleapis.com/(.+?)/serveoncloud.*`)
cloudConfig, err := metadata.InstanceAttributeValue("user-data")
if err != nil || cloudConfig == "" {
log.Fatalf("could not get cloud config from metadata: %v", err)
}
m := storageURLRxp.FindStringSubmatch(cloudConfig)
if len(m) < 2 {
log.Fatal("storage URL not found in cloud config")
}
gcsBucket = m[1]
http.HandleFunc("/", serveHTTP)
log.Fatal(http.ListenAndServe(*httpAddr, nil))
}
func testStorage(t *testing.T, bucketDir string) {
if *bucket == "" && *configFile == "" {
t.Skip("Skipping test without --bucket or --config flag")
}
var refreshToken string
if *configFile != "" {
data, err := ioutil.ReadFile(*configFile)
if err != nil {
t.Fatalf("Error reading config file %v: %v", *configFile, err)
}
var conf Config
if err := json.Unmarshal(data, &conf); err != nil {
t.Fatalf("Error decoding config file %v: %v", *configFile, err)
}
*clientID = conf.Auth.ClientID
*clientSecret = conf.Auth.ClientSecret
refreshToken = conf.Auth.RefreshToken
*bucket = conf.Bucket
}
if *bucket == "" {
t.Fatal("bucket not provided in config file or as a flag.")
}
if *clientID == "" {
if !metadata.OnGCE() {
if *clientSecret == "" {
t.Fatal("client ID and client secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
}
} else {
*clientID = "auto"
}
}
if *configFile == "" {
config := &oauth2.Config{
Scopes: []string{storage.ScopeReadWrite},
Endpoint: google.Endpoint,
ClientID: *clientID,
ClientSecret: *clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
if !metadata.OnGCE() {
token, err := oauth2.ReuseTokenSource(nil,
&oauthutil.TokenSource{
Config: config,
CacheFile: *tokenCache,
AuthCode: func() string {
if *authCode == "" {
t.Skipf("Re-run using --auth_code= with the value obtained from %s",
config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
return ""
}
return *authCode
},
}).Token()
if err != nil {
t.Fatalf("could not acquire token: %v", err)
}
refreshToken = token.RefreshToken
}
}
bucketWithDir := path.Join(*bucket, bucketDir)
storagetest.TestOpt(t, storagetest.Opts{
New: func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
sto, err := newFromConfig(nil, jsonconfig.Obj{
"bucket": bucketWithDir,
"auth": map[string]interface{}{
"client_id": *clientID,
"client_secret": *clientSecret,
"refresh_token": refreshToken,
},
})
if err != nil {
t.Fatal(err)
}
if !testing.Short() {
log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
}
// Bail if bucket is not empty
ctx := context.Background()
stor := sto.(*Storage)
objs, err := stor.client.Bucket(stor.bucket).List(ctx, nil)
if err != nil {
t.Fatalf("Error checking if bucket is empty: %v", err)
}
if len(objs.Results) != 0 {
t.Fatalf("Refusing to run test: bucket %v is not empty", *bucket)
}
if bucketWithDir != *bucket {
// Adding "a", and "c" objects in the bucket to make sure objects out of the
// "directory" are not touched and have no influence.
for _, key := range []string{"a", "c"} {
w := stor.client.Bucket(stor.bucket).Object(key).NewWriter(ctx)
if _, err := io.Copy(w, strings.NewReader(key)); err != nil {
t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
}
if err := w.Close(); err != nil {
t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
}
}
}
clearBucket := func(beforeTests bool) func() {
return func() {
var all []blob.Ref
blobserver.EnumerateAll(context.TODO(), sto, func(sb blob.SizedRef) error {
t.Logf("Deleting: %v", sb.Ref)
all = append(all, sb.Ref)
return nil
})
if err := sto.RemoveBlobs(all); err != nil {
t.Fatalf("Error removing blobs during cleanup: %v", err)
}
if beforeTests {
return
}
if bucketWithDir != *bucket {
// checking that "a" and "c" at the root were left untouched.
for _, key := range []string{"a", "c"} {
rc, err := stor.client.Bucket(stor.bucket).Object(key).NewReader(ctx)
if err != nil {
t.Fatalf("could not find object %s after tests: %v", key, err)
}
if _, err := io.Copy(ioutil.Discard, rc); err != nil {
t.Fatalf("could not find object %s after tests: %v", key, err)
}
if err := stor.client.Bucket(stor.bucket).Object(key).Delete(ctx); err != nil {
t.Fatalf("could not remove object %s after tests: %v", key, err)
}
}
}
}
}
clearBucket(true)()
return sto, clearBucket(false)
},
})
}
func main() {
launchConfig.MaybeDeploy()
flag.Parse()
setProdFlags()
if *root == "" {
var err error
*root, err = os.Getwd()
if err != nil {
log.Fatalf("Failed to getwd: %v", err)
}
}
readTemplates()
if err := initGithubSyncing(); err != nil {
log.Fatalf("error setting up syncing to github: %v")
}
go runDemoBlobserverLoop()
mux := http.DefaultServeMux
mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static")))))
mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks")))))
mux.Handle(pkgPattern, godocHandler{})
mux.Handle(cmdPattern, godocHandler{})
mux.Handle(appPattern, godocHandler{})
mux.HandleFunc(errPattern, errHandler)
mux.HandleFunc("/r/", gerritRedirect)
mux.HandleFunc("/dl/", releaseRedirect)
mux.HandleFunc("/debug/ip", ipHandler)
mux.HandleFunc("/debug/uptime", uptimeHandler)
mux.Handle("/doc/contributing", redirTo("/code#contributing"))
mux.Handle("/lists", redirTo("/community"))
mux.HandleFunc("/contributors", contribHandler())
mux.HandleFunc("/doc/", docHandler)
mux.HandleFunc("/", mainHandler)
if buildbotHost != "" && buildbotBackend != "" {
if _, err := url.Parse(buildbotBackend); err != nil {
log.Fatalf("Failed to parse %v as a URL: %v", buildbotBackend, err)
}
bbhpattern := strings.TrimRight(buildbotHost, "/") + "/"
mux.HandleFunc(bbhpattern, func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, buildbotBackend, http.StatusFound)
})
}
gceLauncher, err := gceDeployHandler("/launch/")
if err != nil {
log.Printf("Not installing GCE /launch/ handler: %v", err)
mux.HandleFunc("/launch/", func(w http.ResponseWriter, r *http.Request) {
http.Error(w, fmt.Sprintf("GCE launcher disabled: %v", err), 500)
})
} else {
mux.Handle("/launch/", gceLauncher)
}
var handler http.Handler = &redirectRootHandler{Handler: mux}
if *logDir != "" || *logStdout {
handler = NewLoggingHandler(handler, NewApacheLogger(*logDir, *logStdout))
}
if *gceLogName != "" {
projID := projectID()
var hc *http.Client
if !metadata.OnGCE() {
hc = httpClient(projID)
}
ctx := context.Background()
var logc *logging.Client
if metadata.OnGCE() {
logc, err = logging.NewClient(ctx, projID, *gceLogName)
} else {
logc, err = logging.NewClient(ctx, projID, *gceLogName, option.WithHTTPClient(hc))
}
if err != nil {
log.Fatal(err)
}
if err := logc.Ping(); err != nil {
log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
}
handler = NewLoggingHandler(handler, gceLogger{logc})
if gceLauncher != nil {
var logc *logging.Client
if metadata.OnGCE() {
logc, err = logging.NewClient(ctx, projID, *gceLogName)
} else {
logc, err = logging.NewClient(ctx, projID, *gceLogName, option.WithHTTPClient(hc))
}
if err != nil {
log.Fatal(err)
}
logc.CommonLabels = map[string]string{
"from": "camli-gce-launcher",
}
logger := logc.Logger(logging.Default)
logger.SetPrefix("launcher: ")
gceLauncher.SetLogger(logger)
}
}
emailErr := make(chan error)
startEmailCommitLoop(emailErr)
if *alsoRun != "" {
runAsChild(*alsoRun)
}
httpServer := &http.Server{
Addr: *httpAddr,
Handler: handler,
ReadTimeout: 5 * time.Minute,
WriteTimeout: 30 * time.Minute,
}
httpErr := make(chan error)
go func() {
log.Printf("Listening for HTTP on %v", *httpAddr)
httpErr <- httpServer.ListenAndServe()
}()
httpsErr := make(chan error)
if *httpsAddr != "" {
go func() {
httpsErr <- serveHTTPS(httpServer)
}()
}
if *flagChromeBugRepro {
go func() {
log.Printf("Repro handler failed: %v", repro(":8001", "foo:bar"))
}()
}
select {
case err := <-emailErr:
log.Fatalf("Error sending emails: %v", err)
case err := <-httpErr:
log.Fatalf("Error serving HTTP: %v", err)
case err := <-httpsErr:
log.Fatalf("Error serving HTTPS: %v", err)
}
}
func main() {
flag.Parse()
log.Printf("Starting server, os.Args=%s", strings.Join(os.Args, " "))
doc.SetDefaultGOOS(*defaultGOOS)
httpClient = newHTTPClient()
var (
gceLogName string
projID string
)
if metadata.OnGCE() {
acct, err := metadata.ProjectAttributeValue("ga-account")
if err != nil {
log.Printf("querying metadata for ga-account: %v", err)
} else {
gaAccount = acct
}
// Get the log name on GCE and setup context for creating a GCE log client.
if name, err := metadata.ProjectAttributeValue("gce-log-name"); err != nil {
log.Printf("querying metadata for gce-log-name: %v", err)
} else {
gceLogName = name
if id, err := metadata.ProjectID(); err != nil {
log.Printf("querying metadata for project ID: %v", err)
} else {
projID = id
}
}
} else {
gaAccount = os.Getenv("GA_ACCOUNT")
}
if err := parseHTMLTemplates([][]string{
{"about.html", "common.html", "layout.html"},
{"bot.html", "common.html", "layout.html"},
{"cmd.html", "common.html", "layout.html"},
{"dir.html", "common.html", "layout.html"},
{"home.html", "common.html", "layout.html"},
{"importers.html", "common.html", "layout.html"},
{"importers_robot.html", "common.html", "layout.html"},
{"imports.html", "common.html", "layout.html"},
{"notfound.html", "common.html", "layout.html"},
{"pkg.html", "common.html", "layout.html"},
{"results.html", "common.html", "layout.html"},
{"tools.html", "common.html", "layout.html"},
{"std.html", "common.html", "layout.html"},
{"subrepo.html", "common.html", "layout.html"},
{"graph.html", "common.html"},
}); err != nil {
log.Fatal(err)
}
if err := parseTextTemplates([][]string{
{"cmd.txt", "common.txt"},
{"dir.txt", "common.txt"},
{"home.txt", "common.txt"},
{"notfound.txt", "common.txt"},
{"pkg.txt", "common.txt"},
{"results.txt", "common.txt"},
}); err != nil {
log.Fatal(err)
}
var err error
db, err = database.New()
if err != nil {
log.Fatalf("Error opening database: %v", err)
}
go runBackgroundTasks()
staticServer := httputil.StaticServer{
Dir: *assetsDir,
MaxAge: time.Hour,
MIMETypes: map[string]string{
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
},
}
statusImageHandlerPNG = staticServer.FileHandler("status.png")
statusImageHandlerSVG = staticServer.FileHandler("status.svg")
apiMux := http.NewServeMux()
apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
apiMux.Handle("/search", apiHandler(serveAPISearch))
apiMux.Handle("/packages", apiHandler(serveAPIPackages))
apiMux.Handle("/importers/", apiHandler(serveAPIImporters))
apiMux.Handle("/imports/", apiHandler(serveAPIImports))
apiMux.Handle("/", apiHandler(serveAPIHome))
mux := http.NewServeMux()
mux.Handle("/-/site.js", staticServer.FilesHandler(
"third_party/jquery.timeago.js",
"site.js"))
mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
if *sidebarEnabled {
mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
}
mux.Handle("/-/", http.NotFoundHandler())
mux.Handle("/-/about", handler(serveAbout))
mux.Handle("/-/bot", handler(serveBot))
mux.Handle("/-/go", handler(serveGoIndex))
mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
mux.Handle("/-/refresh", handler(serveRefresh))
mux.Handle("/-/admin/reindex", http.HandlerFunc(runReindex))
mux.Handle("/-/admin/purgeindex", http.HandlerFunc(runPurgeIndex))
mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
mux.Handle("/code.jquery.com/", http.NotFoundHandler())
mux.Handle("/_ah/health", http.HandlerFunc(serveHealthCheck))
mux.Handle("/_ah/", http.NotFoundHandler())
mux.Handle("/", handler(serveHome))
cacheBusters.Handler = mux
var root http.Handler = rootHandler{
{"api.", apiMux},
{"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
{"www.godoc.org", otherDomainHandler{"https", "godoc.org"}},
{"", mux},
}
if gceLogName != "" {
ctx := context.Background()
logc, err := logging.NewClient(ctx, projID)
if err != nil {
log.Fatalf("Failed to create cloud logging client: %v", err)
}
logger := logc.Logger(gceLogName)
if err := logc.Ping(ctx); err != nil {
log.Fatalf("Failed to ping Google Cloud Logging: %v", err)
}
gceLogger = newGCELogger(logger)
}
http.Handle("/", root)
appengine.Main()
}
func main() {
launchConfig.MaybeDeploy()
flag.Parse()
var kv keyValue
var httpsListenAddr string
if metadata.OnGCE() {
httpsListenAddr = ":443"
dsClient, err := datastore.NewClient(context.Background(), GCEProjectID)
if err != nil {
log.Fatalf("Error creating datastore client for records: %v", err)
}
kv = cachedStore{
dsClient: dsClient,
cache: lru.New(cacheSize),
}
} else {
httpsListenAddr = ":4430"
kv = memkv{skv: sorted.NewMemoryKeyValue()}
}
if err := kv.Set("6401800c.camlistore.net.", "159.203.246.79"); err != nil {
log.Fatalf("Error adding %v:%v record: %v", "6401800c.camlistore.net.", "159.203.246.79", err)
}
if err := kv.Set(domain, *flagServerIP); err != nil {
log.Fatalf("Error adding %v:%v record: %v", domain, *flagServerIP, err)
}
if err := kv.Set("www.camlistore.net.", *flagServerIP); err != nil {
log.Fatalf("Error adding %v:%v record: %v", "www.camlistore.net.", *flagServerIP, err)
}
ds := newDNSServer(kv)
cs := &gpgchallenge.Server{
OnSuccess: func(identity string, address string) error {
log.Printf("Adding %v.camlistore.net. as %v", identity, address)
return ds.dataSource.Set(strings.ToLower(identity+".camlistore.net."), address)
},
}
tcperr := make(chan error, 1)
udperr := make(chan error, 1)
httperr := make(chan error, 1)
log.Printf("serving DNS on %s\n", *addr)
go func() {
tcperr <- dns.ListenAndServe(*addr, "tcp", ds)
}()
go func() {
udperr <- dns.ListenAndServe(*addr, "udp", ds)
}()
if metadata.OnGCE() {
// TODO(mpl): if we want to get a cert for anything
// *.camlistore.net, it's a bit of a chicken and egg problem, since
// we need camnetdns itself to be already running and answering DNS
// queries. It's probably doable, but easier for now to just ask
// one for camnetdns.camlistore.org, since that name is not
// resolved by camnetdns.
hostname := strings.TrimSuffix(authorityNS, ".")
m := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(hostname),
Cache: autocert.DirCache(osutil.DefaultLetsEncryptCache()),
}
ln, err := tls.Listen("tcp", httpsListenAddr, &tls.Config{
Rand: rand.Reader,
Time: time.Now,
NextProtos: []string{http2.NextProtoTLS, "http/1.1"},
MinVersion: tls.VersionTLS12,
GetCertificate: m.GetCertificate,
})
if err != nil {
log.Fatalf("Error listening on %v: %v", httpsListenAddr, err)
}
go func() {
httperr <- http.Serve(ln, cs)
}()
}
select {
case err := <-tcperr:
log.Fatalf("DNS over TCP error: %v", err)
case err := <-udperr:
log.Fatalf("DNS error: %v", err)
case err := <-httperr:
log.Fatalf("HTTP server error: %v", err)
}
}
func defaultListenAddr() string {
if metadata.OnGCE() {
return ":53"
}
return ":5300"
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
var (
auth = config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
cacheSize = config.OptionalInt64("cacheSize", 32<<20)
clientID = auth.RequiredString("client_id") // or "auto" for service accounts
clientSecret = auth.OptionalString("client_secret", "")
refreshToken = auth.OptionalString("refresh_token", "")
)
if err := config.Validate(); err != nil {
return nil, err
}
if err := auth.Validate(); err != nil {
return nil, err
}
var dirPrefix string
if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
dirPrefix = parts[1]
bucket = parts[0]
}
if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
dirPrefix += "/"
}
gs := &Storage{
bucket: bucket,
dirPrefix: dirPrefix,
}
var (
ctx = context.Background()
ts oauth2.TokenSource
cl *storage.Client
err error
)
if clientID == "auto" {
if !metadata.OnGCE() {
return nil, errors.New(`Cannot use "auto" client_id when not running on GCE`)
}
ts, err = google.DefaultTokenSource(ctx, storage.ScopeReadWrite)
if err != nil {
return nil, err
}
cl, err = storage.NewClient(ctx)
if err != nil {
return nil, err
}
} else {
if clientSecret == "" {
return nil, errors.New("missing required parameter 'client_secret'")
}
if refreshToken == "" {
return nil, errors.New("missing required parameter 'refresh_token'")
}
ts = oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{storage.ScopeReadWrite},
Endpoint: google.Endpoint,
ClientID: clientID,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}, refreshToken)
cl, err = storage.NewClient(ctx, option.WithTokenSource(ts))
if err != nil {
return nil, err
}
}
gs.baseHTTPClient = oauth2.NewClient(ctx, ts)
gs.client = cl
if cacheSize != 0 {
gs.cache = memory.NewCache(cacheSize)
}
ba, err := gs.client.Bucket(gs.bucket).Attrs(ctx)
if err != nil {
return nil, fmt.Errorf("error statting bucket %q: %v", gs.bucket, err)
}
hash := sha1.New()
fmt.Fprintf(hash, "%v%v", ba.Created, ba.MetaGeneration)
gs.genRandom = fmt.Sprintf("%x", hash.Sum(nil))
gs.genTime = ba.Created
return gs, nil
}
// NewAuthTransport gives new AuthTransport created with GitHub credentials
// read from GCE metadata when the metadata server is accessible (we're on GCE)
// or read from environment varialbes otherwise.
func NewAuthTransport(base http.RoundTripper) *AuthTransport {
if metadata.OnGCE() {
return NewAuthTransportFromMetadata(base)
}
return NewAuthTransportFromEnvironment(base)
}