func updateLastCommit() {
fmt.Println("Updating LastCommit")
rows, err := c.DB.Table("repo_stats").Select(`
repository_id, max(week) as week
`).Group("repository_id").Rows()
c.PanicOn(err)
for rows.Next() {
var repository_id int64
var week time.Time
rows.Scan(&repository_id, &week)
var repo c.Repository
c.DB.Where("id = ?", repository_id).First(&repo)
if repo.LastCommit.Time != week {
c.DB.Model(&repo).Update("last_commit", week)
}
}
rows.Close()
}
func generateCommitCountPerUser() {
fmt.Println("Generating commit count per user")
rows, err := c.DB.Table("repo_stats").Select(`
user_id, array_agg(distinct o.login) as organization_list, sum(commits) as commit_count
`).Joins(`
inner join repositories r
on r.id = repo_stats.repository_id
inner join organizations o
on o.id = r.organization_id
`).Group("user_id").Rows()
c.PanicOn(err)
for rows.Next() {
var user c.User
var user_id, commit_count int64
var org_list string
rows.Scan(&user_id, &org_list, &commit_count)
if user_id == 0 {
continue
}
c.DB.Where("id = ?", user_id).First(&user)
if commit_count != user.CommitCount {
user.CommitCount = commit_count
user.OrgList = org_list
c.DB.Save(&user)
}
}
rows.Close()
}
func generateCommitCount() {
fmt.Println("Generating commit count per org")
rows, err := c.DB.Table("repo_stats").Select(`
week, organization_id, sum(commits) as commit_count`).Joins(`
inner join repositories
on repositories.id = repo_stats.repository_id
`).Group("week, repository_id, organization_id").Rows()
c.PanicOn(err)
for rows.Next() {
var week time.Time
var organization_id, commit_count int64
rows.Scan(&week, &organization_id, &commit_count)
stat := c.CommitOrgStats{
Week: week,
OrganizationId: organization_id,
}
c.DB.Where(stat).FirstOrInit(&stat)
stat.CommitCount = commit_count
c.DB.Save(&stat)
}
rows.Close()
}
func updateLastPull() {
fmt.Println("Updating LastPull")
rows, err := c.DB.Table("pulls").Select(`
repository_id, max(gh_created_at) as latest_pull
`).Group("repository_id").Rows()
c.PanicOn(err)
for rows.Next() {
var repository_id int64
var latest_pull time.Time
rows.Scan(&repository_id, &latest_pull)
var repo c.Repository
c.DB.Where("id = ?", repository_id).First(&repo)
if repo.LastPull.Time != latest_pull {
repo.LastPull.Time = latest_pull
repo.LastPull.Valid = true
c.DB.Save(&repo)
}
}
rows.Close()
}
func getAccounts() (orgs []string) {
url := "http://registry.usa.gov/accounts.json?service_id=github"
res, err := http.Get(url)
c.PanicOn(err)
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
c.PanicOn(err)
var data AccountData
err = json.Unmarshal(body, &data)
c.PanicOn(err)
orgs = append(orgs, []string{"arcticlcc", "bbginnovate",
"blue-button", "ca-cst-sii", "cocomans", "cooperhewitt",
"eeoc", "federal-aviation-administration", "globegit",
"gopleader", "govxteam", "greatsmokymountainsnationalpark",
"imdprojects",
"irsgov", "jbei", "kbase", "m-o-s-e-s", "measureauthoringtool",
"nasa-gibs", "nasa-rdt", "ncbitools",
"ncpp", "ncrn", "ndar", "neogeographytoolkit", "nersc", "ngds",
"nhanes", "nist-ics-sec-tb", "nmml",
"nrel-cookbooks", "ozone-development", "pm-master",
"servir", "sunpy", "usbr", "usdeptveteransaffairs", "usgcrp",
"usgin-models", "usgs-astrogeology", "usgs-cida", "usgs-owi", "usindianaffairs",
"usps", "vhainnovations", "visionworkbench",
"wfmrda"}...)
for _, e := range data.Accounts {
found := false
for _, o := range orgs {
if o == e.Account {
found = true
break
}
}
if !found {
orgs = append(orgs, e.Account)
}
}
return orgs
}
func importRepos(org *c.Organization, client *github.Client, page int) {
// Load the repos for a given org
opt := &github.RepositoryListByOrgOptions{Type: "public"}
opt.ListOptions = github.ListOptions{Page: page, PerPage: 100}
repos, response, err := client.Repositories.ListByOrg(org.Login, opt)
fmt.Printf("Getting Repos for org Login %s at page %d\n", org.Login, page)
if err != nil {
fmt.Println("Error with org: ", org.Login)
c.PanicOn(err)
}
// There are more pages, lets fetch the next one
if response.NextPage > 0 {
importRepos(org, client, response.NextPage)
}
for _, r := range repos {
if !*r.Fork {
var repo c.Repository
c.DB.Where("name = ? and organization_id = ?", *r.Name, org.Id).First(&repo)
if repo.Id == 0 {
repo.GhId = int64(*r.ID)
repo.Name = *r.Name
if r.Description != nil {
repo.Description = *r.Description
}
repo.OrganizationId = org.Id
repo.Ignore = false
repo.GhCreatedAt.Scan(*r.CreatedAt)
}
repo.Forks = int64(*r.ForksCount)
repo.Watchers = int64(*r.WatchersCount)
repo.Stargazers = int64(*r.StargazersCount)
repo.Size = int64(*r.Size)
repo.OpenIssues = int64(*r.OpenIssuesCount)
repo.Language = getStr(r.Language)
repo.GhUpdatedAt.Scan(*r.CreatedAt)
c.DB.Save(&repo)
}
}
}
