Exemplo n.º 1
0
// All webhooks are sent a "ping" event on creation
func pingHook(userName, repoName string, repoData repoStorageData, content []byte) {
	c, done := context.WithCancel(appengine.BackgroundContext())
	defer done()

	errorf := makeErrorf(c, userName, repoName)

	var payload struct {
		Zen    string `json:"zen"`
		HookID int    `json:"hook_id"`
	}

	err := json.Unmarshal(content, &payload)
	if err != nil {
		errorf("Can't parse payload for ping hook: %s, %s", err.Error(), content)
		return
	}

	err = modifyRepoData(c, userName, repoName, func(item *repoStorageData) {
		item.Status = statusInitializing
	})

	if err != nil {
		log.Errorf(c, "Can't set repo %s/%s to initializing: %s", userName, repoName, err.Error())
	}

	// Pass of to initialization
	go initialize(userName, repoName)
}
Exemplo n.º 2
0
// restartAbandonedOperations runs when the web server starts.
// It goes through the repos in the data store and checks their statuses.
// If they're validating or initializing, those processes will restart.
// If they actually finished validating / initializing but didn't write
// to the store that's fine, since all operations are indempotent; we
// can redo it.
func restartAbandonedOperations() {
	c, done := context.WithCancel(appengine.BackgroundContext())
	defer done()

	log.Infof(c, "Restarting abandoned operations...")

	repos, err := getAllRepoData(c)
	if err != nil {
		log.Errorf(c, "Can't load repos: %s", err.Error)
		return
	}

	for _, repo := range repos {
		switch repo.Status {
		case statusReady:
			log.Infof(c, "Repo ready: %s/%s", repo.User, repo.Repo)
		case statusError:
			log.Infof(c, "Repo errored out: %s/%s", repo.User, repo.Repo)
		case statusValidating:
			log.Infof(c, "Repo requires validation: %s/%s", repo.User, repo.Repo)
			go validate(repo.User, repo.Repo)
		case statusInitializing:
			log.Infof(c, "Repo requires initialization: %s/%s", repo.User, repo.Repo)
			go initialize(repo.User, repo.Repo)
		case statusHooksInitializing:
			log.Infof(c, "Repo requires hook initialization: %s/%s", repo.User, repo.Repo)
			go createHooks(repo.User, repo.Repo)
		default:
			log.Errorf(c, "Unrecognized status for repo %s/%s: %s", repo.User, repo.Repo, repo.Status)
		}
	}
}
Exemplo n.º 3
0
// deactivate deletes webhooks and forgets data for a given repository
func deactivate(userName, repoName string) {
	c, done := context.WithCancel(appengine.BackgroundContext())
	defer done()

	errorf := makeErrorf(c, userName, repoName)

	repoData, err := getRepoData(c, userName, repoName)
	if err != nil {
		errorf("Can't load repo to deactivate: %s", err.Error())
		return
	}

	client := github.NewClient(oauth2.NewClient(c, oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: repoData.Token},
	)))

	log.Infof(c, "Deleting hook for repository %s/%s", userName, repoName)
	err = retry(c, func() (resp *github.Response, err error) {
		resp, err = client.Repositories.DeleteHook(userName, repoName, repoData.HookID)
		return
	})
	if err != nil {
		errorf("Can't delete webhook: %s", err.Error())
		// Keep going; we should still delete the repository data
	} else {
		log.Infof(c, "Deleting hook for repository %s/%s succeeded")
	}

	log.Infof(c, "Deleting repository data for %s/%s", userName, repoName)
	err = deleteRepoData(c, userName, repoName)
	if err != nil {
		errorf("Can't delete repository data: %s", err.Error())
		return
	}
}
Exemplo n.º 4
0
func initStorage() error {
	c, done := context.WithCancel(appengine.BackgroundContext())
	defer done()

	rootKey := makeReposRootKey(c)
	_, err := datastore.Put(c, rootKey, &struct{}{})
	return err
}
Exemplo n.º 5
0
func main() {
	var err error
	api.Tracer, err = trace.NewClient(appengine.BackgroundContext(), "ellies-pad")
	if err != nil {
		panic(err)
	}

	appengine.Main()
}
Exemplo n.º 6
0
// die removes the WebSocket references by id from the global pool and logs
// that it did so. If there is no such WebSocket, it does nothing.
func die(id id, err error) {
	if _, ok := lookup(id); !ok {
		return
	}

	log.Debugf(appengine.BackgroundContext(), "ws: connection for %v died: %s", id, err)
	conns.Lock()
	delete(conns.m, id)
	conns.Unlock()
}
Exemplo n.º 7
0
// ping will loop infinitely and send ping messages over ws. If a write takes
// longer than writeWait, it will remove ws from the connection pool.
func ping(ws *websocket.Conn, id id) {
	b := make([]byte, 2)
	for range time.Tick(pingPeriod) {
		rand.Read(b)
		deadline := time.Now().Add(writeWait)
		if err := ws.WriteControl(websocket.PingMessage, b, deadline); err != nil {
			die(id, err)
			return
		}
		log.Debugf(appengine.BackgroundContext(), "ws: ping %x for %s(%d %q) sent", b, id.kind, id.intID, id.stringID)
	}
}
Exemplo n.º 8
0
Arquivo: gcp.go Projeto: ycaihua/gizmo
// NewContext will attempt create a new context from
// a the Token or JSONAuthPath fields if provided. If the FlexibleAE flag
// is set to designate this is a 'flexible' App Engine VM,
// appengine.BackgroundContext() will be used. Otherwise, google.DefaultClient
// will be used, which should work for standard App Engine environments and GCE.
func (g Config) NewContext(scopes ...string) (context.Context, error) {
	if len(g.Token) > 0 {
		return g.contextFromToken(scopes...)
	}

	if len(g.JSONAuthPath) > 0 {
		return g.contextFromJSON(scopes...)
	}

	if g.FlexibleVM {
		return appengine.BackgroundContext(), nil
	}

	if len(scopes) == 0 {
		scopes = append(scopes, compute.ComputeScope)
	}

	client, err := google.DefaultClient(oauth2.NoContext, scopes...)
	if err != nil {
		return nil, err
	}
	return cloud.NewContext(g.ProjectID, client), nil
}
Exemplo n.º 9
0
func purgeIndex() {
	c := appengine.BackgroundContext()
	if err := database.PurgeIndex(c); err != nil {
		log.Println("purgeIndex:", err)
	}
}
Exemplo n.º 10
0
func reindex() {
	c := appengine.BackgroundContext()
	if err := db.Reindex(c); err != nil {
		log.Println("reindex:", err)
	}
}
Exemplo n.º 11
0
// UseBackground is the same as Use except that it activates production
// implementations which aren't associated with any particular request.
//
// This is only available on Managed VMs.
func UseBackground(c context.Context) context.Context {
	return setupAECtx(c, appengine.BackgroundContext())
}
Exemplo n.º 12
0
func NewContext(r ...*http.Request) Context {
	context := appengine.BackgroundContext()
	return Context{context, MockConnection{}}
}
Exemplo n.º 13
0
// hook sets up webhooks for a given repository
func createHooks(userName, repoName string) {
	c, done := context.WithCancel(appengine.BackgroundContext())
	defer done()

	errorf := makeErrorf(c, userName, repoName)
	repoData, err := getRepoData(c, userName, repoName)
	if err != nil {
		errorf("Can't load repo to hook: %s", err.Error())
		return
	}

	client := github.NewClient(oauth2.NewClient(c, oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: repoData.Token},
	)))

	active := true

	secret := make([]byte, secretSize)
	_, err = rand.Read(secret)
	if err != nil {
		errorf("Can't create secret key: %s", err.Error())
		return
	}
	secretHex := hex.EncodeToString(secret)

	// TODO allow non-appspot urls?
	url := fmt.Sprintf("https://github-mirror-dot-%s.appspot.com/hook/%s/%s", appengine.AppID(c), userName, repoName)

	log.Infof(c, "Creating hook for %s/%s: url `%s`", userName, repoName, url)

	var hook *github.Hook
	err = retry(c, func() (resp *github.Response, err error) {
		hook, resp, err = client.Repositories.CreateHook(userName, repoName, &github.Hook{
			Name: &hookType,
			Events: []string{
				eventPing,
				eventStatus,
				eventPullRequest,
				eventDiffComment,
				eventIssueComment,
			},
			Active: &active,
			Config: map[string]interface{}{
				"url":          url,
				"content_type": "json",
				"secret":       secretHex,
				"insecure_ssl": false,
			},
		})
		return
	})
	if err != nil {
		errorf("Can't create hook: %s", err.Error())
		return
	}

	if hook.ID == nil {
		errorf("No hook ID for new hook")
		return
	}

	log.Infof(c, "Hook creation for %s/%s successful", userName, repoName)

	err = modifyRepoData(c, userName, repoName, func(item *repoStorageData) {
		item.HookSecret = secretHex
		item.HookID = *hook.ID
	})

	if err != nil {
		errorf("Can't set repo status to ready: %s", err.Error())
		return
	}

	log.Infof(c, "Repo waiting for hook ping: %s/%s", userName, repoName)
}
Exemplo n.º 14
0
// initialize performs initial reading and commiting for the repository
func initialize(userName, repoName string) {
	c, done := context.WithCancel(appengine.BackgroundContext())
	defer done()

	errorf := makeErrorf(c, userName, repoName)
	repoData, err := getRepoData(c, userName, repoName)
	if err != nil {
		errorf("Can't load repo to initialize: %s", err.Error())
		return
	}

	repo, err := clone(c, userName, repoName, userName, repoData.Token)
	if err != nil {
		errorf("Can't clone repo: %s", err.Error())
		return
	}

	client := github.NewClient(oauth2.NewClient(c, oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: repoData.Token},
	)))

	errChan := make(chan error, 1000)
	nErrors := 0
	go func() {
		for err := range errChan {
			errorf(err.Error())
			nErrors++
		}
	}()

	reviews, err := mirror.GetAllPullRequests(repo, userName, repoName, client, errChan)
	if err != nil {
		errorf("Can't get PRs: %s", err.Error())
		return
	}

	statuses, err := mirror.GetAllStatuses(userName, repoName, client, errChan)
	if err != nil {
		errorf("Can't get statuses: %s", err.Error())
		return
	}
	close(errChan)

	nStatuses := len(statuses)
	nReviews := len(reviews)
	logChan := make(chan string, 1000)
	go func() {
		for msg := range logChan {
			log.Infof(c, msg)
		}
	}()
	log.Infof(c, "Done reading! Read %d statuses, %d PRs", nStatuses, nReviews)
	log.Infof(c, "Committing...\n")
	if err := mirror.WriteNewReports(statuses, repo, logChan); err != nil {
		errorf(err.Error())
		return
	}
	if err := mirror.WriteNewReviews(reviews, repo, logChan); err != nil {
		errorf(err.Error())
		return
	}
	close(logChan)
	err = syncNotes(repo)
	if err != nil {
		errorf("Error pushing initialization changes for %s/%s: %s",
			userName,
			repoName,
			err.Error())
		return
	}
	log.Infof(c, "Success initializing %s/%s", userName, repoName)

	err = modifyRepoData(c, userName, repoName, func(item *repoStorageData) {
		item.Status = statusReady
	})

	if err != nil {
		errorf("Can't change repo status for %s/%s: %s",
			userName,
			repoName,
			err.Error(),
		)
	}
}
Exemplo n.º 15
0
// validate ensures that the repo is accessible
func validate(user, repo string) {
	c, done := context.WithCancel(appengine.BackgroundContext())
	defer done()

	log.Infof(c, "Validating repo %s/%s", user, repo)

	errorf := makeErrorf(c, user, repo)

	repoData, err := getRepoData(c, user, repo)
	if err != nil {
		errorf("Can't load repo to validate: %s", err.Error())
		return
	}

	httpClient := oauth2.NewClient(c, oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: repoData.Token},
	))

	githubClient := github.NewClient(httpClient)

	var resp *github.Response
	err = retry(c, func() (*github.Response, error) {
		// APIMeta will always succeed and will tell us what scopes
		// we have.
		_, resp, err = githubClient.APIMeta()
		return resp, err
	})

	if err != nil {
		errorf("Can't validate repo %s/%s: %s", user, repo, err.Error())
		return
	}

	scopesHeader := resp.Header["X-Oauth-Scopes"]

	if len(scopesHeader) == 0 {
		// No scopes means that a token has access to all *public* repositories.
		// It's simplest to just require private access.
		errorf("Invalid token, missing scopes: `repo`, `write:repo_hook`")
		return
	}

	// The token has scopes.
	// Let's make sure it has all the ones we need enabled.
	// Note that strictly speaking, we need the repo, public_repo,
	// write:repo_hook, and repo:status scopes, but repo and
	// write:repo_hook subsume the others.

	// Necessary because github makes things comma-delimited instead
	// of semicolon-delimited for some reason.
	scopes := strings.Split(scopesHeader[0], ", ")

	var hasRepo bool
	var hasWriteRepoHook bool
	for _, scope := range scopes {
		switch scope {
		case "repo":
			hasRepo = true
		case "write:repo_hook":
			hasWriteRepoHook = true
		}
	}

	if !hasRepo || !hasWriteRepoHook {
		var missingScopes string
		if !hasRepo && !hasWriteRepoHook {
			missingScopes = "repo, write:repo_hook"
		} else if !hasRepo {
			missingScopes = "repo"
		} else {
			missingScopes = "write:repo_hook"
		}
		errorf("Invalid token for %s/%s, missing scopes: %s",
			user,
			repo,
			missingScopes,
		)
		return
	}

	log.Infof(c, "Validated repo %s/%s", user, repo)

	var remoteRepo *github.Repository
	err = retry(c, func() (resp *github.Response, err error) {
		remoteRepo, resp, err = githubClient.Repositories.Get(user, repo)
		return
	})

	if err != nil {
		errorf("Can't validate repo %s/%s: %s", user, repo, err.Error())
	}

	err = modifyRepoData(c, user, repo, func(item *repoStorageData) {
		item.Status = statusHooksInitializing
	})

	if err != nil {
		errorf("Can't change repo status: %s", err.Error())
	}

	go createHooks(user, repo)
}