func packageToDoc(p *gcse.Package) gcse.DocInfo {
// copy Package as a DocInfo
d := gcse.DocInfo{
Package: p.Package,
Name: p.Name,
Synopsis: p.Synopsis,
Description: p.Doc,
LastUpdated: time.Now(),
Author: gcse.AuthorOfPackage(p.Package),
ProjectURL: p.ProjectURL,
StarCount: p.StarCount,
ReadmeFn: p.ReadmeFn,
ReadmeData: p.ReadmeData,
Exported: p.Exported,
}
d.Imports = nil
for _, imp := range p.Imports {
if doc.IsValidRemotePath(imp) {
d.Imports = append(d.Imports, imp)
}
}
d.TestImports = nil
for _, imp := range p.TestImports {
if doc.IsValidRemotePath(imp) {
d.TestImports = append(d.TestImports, imp)
}
}
// append new authors
if strings.HasPrefix(d.Package, "github.com/") {
cDB.AppendPerson("github.com", d.Author)
} else if strings.HasPrefix(d.Package, "bitbucket.org/") {
cDB.AppendPerson("bitbucket.org", d.Author)
}
for _, imp := range d.Imports {
appendPackage(imp)
}
for _, imp := range d.TestImports {
appendPackage(imp)
}
log.Printf("[pushPackage] References: %v", p.References)
for _, ref := range p.References {
appendPackage(ref)
}
schedulePackageNextCrawl(d.Package, p.Etag)
return d
}
func processDocument(d *gcse.DocInfo) error {
pkg := d.Package
// fetch saved DocInfo
var savedD gcse.DocInfo
exists := docDB.Get(pkg, &savedD)
if exists && d.StarCount < 0 {
d.StarCount = savedD.StarCount
}
if d.StarCount < 0 {
d.StarCount = 0
}
log.Printf("Package %s processed!", pkg)
docDB.Put(pkg, *d)
return nil
}