// feed uploads the file to the corresponding dcs-package-importer.
func feed(pkg, filename string, reader io.Reader) error {
shard := shards[shardmapping.TaskIdxForPackage(pkg, len(shards))]
url := fmt.Sprintf("http://%s/import/%s/%s", shard, pkg, filename)
request, err := http.NewRequest("PUT", url, reader)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(request)
if err != nil {
return err
}
defer resp.Body.Close()
log.Printf("HTTP response for %q: %q\n", url, resp.Status)
if resp.StatusCode == 200 && strings.HasSuffix(filename, ".dsc") {
requestMerge(shard)
}
return nil
}
func Show(w http.ResponseWriter, r *http.Request) {
query := r.URL
filename := query.Query().Get("file")
line64, err := strconv.ParseInt(query.Query().Get("line"), 10, 0)
if err != nil {
log.Printf("%v\n", err)
return
}
line := int(line64)
log.Printf("Showing file %s, line %d\n", filename, line)
if *common.UseSourcesDebianNet && health.IsHealthy("sources.debian.net") {
destination := fmt.Sprintf("http://sources.debian.net/src/%s?hl=%d#L%d",
strings.Replace(filename, "_", "/", 1), line, line)
log.Printf("SDN is healthy. Redirecting to %s\n", destination)
http.Redirect(w, r, destination, 302)
return
}
idx := strings.Index(filename, "/")
if idx == -1 {
http.Error(w, "Filename does not contain a package", http.StatusInternalServerError)
return
}
pkg := filename[:idx]
shard := common.SourceBackendStubs[shardmapping.TaskIdxForPackage(pkg, len(common.SourceBackendStubs))]
resp, err := shard.File(context.Background(), &proto.FileRequest{
Path: filename,
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// NB: contents is untrusted as it can contain the contents of any file
// within any Debian package. Converting it to string is not a problem,
// though, see http://golang.org/ref/spec#Conversions, "Conversions to and
// from a string type": "Converting a slice of bytes to a string type
// yields a string whose successive bytes are the elements of the slice.".
// We don’t iterate over this string, we just pass it directly to the
// user’s browser, which can then deal with the bytes :-).
lines := strings.Split(string(resp.Contents), "\n")
highestLineNr := fmt.Sprintf("%d", len(lines))
// Since Go templates don’t offer any way to use {{$idx+1}}, we need to
// pre-calculate line numbers starting from 1 here.
lineNumbers := make([]int, len(lines))
for idx, _ := range lines {
lineNumbers[idx] = idx + 1
}
err = common.Templates.ExecuteTemplate(w, "show.html", map[string]interface{}{
"line": line,
"lines": lines,
"numbers": lineNumbers,
"lnrwidth": len(highestLineNr),
"filename": filename,
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
func checkSources() {
log.Printf("checking sources\n")
lastSanityCheckStarted.Set(float64(time.Now().Unix()))
// Store packages by shard.
type pkgStatus int
const (
NotPresent pkgStatus = iota
Present
Confirmed
)
packages := make(map[string]map[string]pkgStatus)
for _, shard := range shards {
url := fmt.Sprintf("http://%s/listpkgs", shard)
resp, err := http.Get(url)
if err != nil {
log.Printf("Could not get list of packages from %q: %v\n", url, err)
continue
}
defer resp.Body.Close()
type ListPackageReply struct {
Packages []string
}
var reply ListPackageReply
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(&reply); err != nil {
log.Printf("Invalid json from %q: %v\n", url, err)
continue
}
packages[shard] = make(map[string]pkgStatus)
for _, foundpkg := range reply.Packages {
packages[shard][foundpkg] = Present
}
log.Printf("shard %q has %d packages currently\n", shard, len(reply.Packages))
}
sourcesSuffix := "/dists/sid/main/source/Sources.gz"
resp, err := http.Get(*mirrorUrl + sourcesSuffix)
if err != nil {
log.Printf("Could not get Sources.gz: %v\n", err)
return
}
defer resp.Body.Close()
reader, err := gzip.NewReader(resp.Body)
if err != nil {
log.Printf("Could not initialize gzip reader: %v\n", err)
return
}
defer reader.Close()
sourcePackages, err := godebiancontrol.Parse(reader)
if err != nil {
log.Printf("Could not parse Sources.gz: %v\n", err)
return
}
// for every package, calculate who’d be responsible and see if it’s present on that shard.
for _, pkg := range sourcePackages {
if strings.HasSuffix(pkg["Package"], "-data") {
continue
}
p := pkg["Package"] + "_" + pkg["Version"]
shardIdx := shardmapping.TaskIdxForPackage(p, len(shards))
shard := shards[shardIdx]
// Skip shards that are offline (= for which we have no package list).
if _, online := packages[shard]; !online {
continue
}
status := packages[shard][p]
//log.Printf("package %s: shard %d (%s), status %v\n", p, shardIdx, shard, status)
if status == Present {
packages[shard][p] = Confirmed
} else if status == NotPresent {
log.Printf("Feeding package %s to shard %d (%s)\n", p, shardIdx, shard)
var pkgfiles []string
for _, line := range strings.Split(pkg["Files"], "\n") {
parts := strings.Split(strings.TrimSpace(line), " ")
// pkg["Files"] has a newline at the end, so we get one empty line.
if len(parts) < 3 {
continue
}
url := *mirrorUrl + "/" + pkg["Directory"] + "/" + parts[2]
// Append the .dsc to the end, prepend the other files.
if strings.HasSuffix(url, ".dsc") {
pkgfiles = append(pkgfiles, url)
} else {
pkgfiles = append([]string{url}, pkgfiles...)
}
}
feedfiles(p, pkgfiles)
successfulSanityFeed.Inc()
}
}
// Garbage-collect all packages that have not been confirmed.
for _, shard := range shards {
for p, status := range packages[shard] {
if status != Present {
continue
}
log.Printf("garbage-collecting %q on shard %s\n", p, shard)
shard := shards[shardmapping.TaskIdxForPackage(p, len(shards))]
url := fmt.Sprintf("http://%s/garbagecollect", shard)
if _, err := http.PostForm(url, net_url.Values{"package": {p}}); err != nil {
log.Printf("Could not garbage-collect package %q on shard %s: %v\n", p, shard, err)
continue
}
successfulGarbageCollect.Inc()
}
}
}
