forked from icecrime/vossibility-collector
/
sync.go
62 lines (53 loc) · 2.02 KB
/
sync.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
package main
import (
"strings"
"cmd/vossibility-collector/github"
"cmd/vossibility-collector/storage"
log "github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
)
var syncCommand = cli.Command{
Name: "sync",
Usage: "sync storage with the GitHub repositories",
Action: doSyncCommand,
Flags: []cli.Flag{
cli.IntFlag{Name: "from", Value: 1, Usage: "issue number to start from"},
cli.IntFlag{Name: "sleep", Value: 0, Usage: "sleep delay between each GitHub page queried"},
},
}
// doSyncCommand runs a synchronization job: it fetches all GitHub issues and
// pull requests starting with the From index. It uses the API pagination to
// reduce API calls, and allows a Sleep delay between each page to avoid
// triggering the abuse detection mechanism.
func doSyncCommand(c *cli.Context) {
config := ParseConfigOrDie(c.GlobalString("config"))
client := github.NewClient(config.GitHubAPIToken)
blobStore := storage.NewTransformingBlobStore()
// Get the list of repositories from command-line (defaults to all).
repoToSync := c.Args()
if len(repoToSync) == 0 {
repoToSync = make([]string, 0, len(config.Repositories))
for givenName, _ := range config.Repositories {
repoToSync = append(repoToSync, givenName)
}
}
// Get the repositories instances from their given names.
repos := make([]*storage.Repository, 0, len(repoToSync))
for _, givenName := range repoToSync {
r, ok := config.Repositories[givenName]
if !ok {
log.Fatalf("unknown repository %q", givenName)
}
repos = append(repos, r)
}
// Configure a syncJob taking all issues (opened and closed) and storing
// in the snapshot store.
syncOptions := github.DefaultSyncOptions
syncOptions.From = c.Int("from")
syncOptions.SleepPerPage = c.Int("sleep")
syncOptions.State = github.GitHubStateFilterAll
syncOptions.Storage = storage.StoreSnapshot
// Create and run the synchronization job.
log.Warnf("running sync jobs on repositories %s", strings.Join(repoToSync, ", "))
github.NewSyncCommandWithOptions(client, blobStore, &syncOptions).Run(repos)
}