示例#1
0
文件: feeder.go 项目: jamessan/dcs
// feed uploads the file to the corresponding dcs-package-importer.
func feed(pkg, filename string, reader io.Reader) error {
	shard := shards[shardmapping.TaskIdxForPackage(pkg, len(shards))]
	url := fmt.Sprintf("http://%s/import/%s/%s", shard, pkg, filename)
	request, err := http.NewRequest("PUT", url, reader)
	if err != nil {
		return err
	}
	resp, err := http.DefaultClient.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	log.Printf("HTTP response for %q: %q\n", url, resp.Status)

	if resp.StatusCode == 200 && strings.HasSuffix(filename, ".dsc") {
		requestMerge(shard)
	}

	return nil
}
示例#2
0
文件: show.go 项目: Debian/dcs
func Show(w http.ResponseWriter, r *http.Request) {
	query := r.URL
	filename := query.Query().Get("file")
	line64, err := strconv.ParseInt(query.Query().Get("line"), 10, 0)
	if err != nil {
		log.Printf("%v\n", err)
		return
	}
	line := int(line64)
	log.Printf("Showing file %s, line %d\n", filename, line)

	if *common.UseSourcesDebianNet && health.IsHealthy("sources.debian.net") {
		destination := fmt.Sprintf("http://sources.debian.net/src/%s?hl=%d#L%d",
			strings.Replace(filename, "_", "/", 1), line, line)
		log.Printf("SDN is healthy. Redirecting to %s\n", destination)
		http.Redirect(w, r, destination, 302)
		return
	}

	idx := strings.Index(filename, "/")
	if idx == -1 {
		http.Error(w, "Filename does not contain a package", http.StatusInternalServerError)
		return
	}
	pkg := filename[:idx]
	shard := common.SourceBackendStubs[shardmapping.TaskIdxForPackage(pkg, len(common.SourceBackendStubs))]
	resp, err := shard.File(context.Background(), &proto.FileRequest{
		Path: filename,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// NB: contents is untrusted as it can contain the contents of any file
	// within any Debian package. Converting it to string is not a problem,
	// though, see http://golang.org/ref/spec#Conversions, "Conversions to and
	// from a string type": "Converting a slice of bytes to a string type
	// yields a string whose successive bytes are the elements of the slice.".
	// We don’t iterate over this string, we just pass it directly to the
	// user’s browser, which can then deal with the bytes :-).
	lines := strings.Split(string(resp.Contents), "\n")
	highestLineNr := fmt.Sprintf("%d", len(lines))

	// Since Go templates don’t offer any way to use {{$idx+1}}, we need to
	// pre-calculate line numbers starting from 1 here.
	lineNumbers := make([]int, len(lines))
	for idx, _ := range lines {
		lineNumbers[idx] = idx + 1
	}

	err = common.Templates.ExecuteTemplate(w, "show.html", map[string]interface{}{
		"line":     line,
		"lines":    lines,
		"numbers":  lineNumbers,
		"lnrwidth": len(highestLineNr),
		"filename": filename,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
示例#3
0
文件: feeder.go 项目: jamessan/dcs
func checkSources() {
	log.Printf("checking sources\n")
	lastSanityCheckStarted.Set(float64(time.Now().Unix()))

	// Store packages by shard.
	type pkgStatus int
	const (
		NotPresent pkgStatus = iota
		Present
		Confirmed
	)
	packages := make(map[string]map[string]pkgStatus)

	for _, shard := range shards {
		url := fmt.Sprintf("http://%s/listpkgs", shard)
		resp, err := http.Get(url)
		if err != nil {
			log.Printf("Could not get list of packages from %q: %v\n", url, err)
			continue
		}
		defer resp.Body.Close()

		type ListPackageReply struct {
			Packages []string
		}

		var reply ListPackageReply
		decoder := json.NewDecoder(resp.Body)
		if err := decoder.Decode(&reply); err != nil {
			log.Printf("Invalid json from %q: %v\n", url, err)
			continue
		}
		packages[shard] = make(map[string]pkgStatus)
		for _, foundpkg := range reply.Packages {
			packages[shard][foundpkg] = Present
		}
		log.Printf("shard %q has %d packages currently\n", shard, len(reply.Packages))
	}

	sourcesSuffix := "/dists/sid/main/source/Sources.gz"
	resp, err := http.Get(*mirrorUrl + sourcesSuffix)
	if err != nil {
		log.Printf("Could not get Sources.gz: %v\n", err)
		return
	}
	defer resp.Body.Close()
	reader, err := gzip.NewReader(resp.Body)
	if err != nil {
		log.Printf("Could not initialize gzip reader: %v\n", err)
		return
	}
	defer reader.Close()

	sourcePackages, err := godebiancontrol.Parse(reader)
	if err != nil {
		log.Printf("Could not parse Sources.gz: %v\n", err)
		return
	}

	// for every package, calculate who’d be responsible and see if it’s present on that shard.
	for _, pkg := range sourcePackages {
		if strings.HasSuffix(pkg["Package"], "-data") {
			continue
		}
		p := pkg["Package"] + "_" + pkg["Version"]
		shardIdx := shardmapping.TaskIdxForPackage(p, len(shards))
		shard := shards[shardIdx]
		// Skip shards that are offline (= for which we have no package list).
		if _, online := packages[shard]; !online {
			continue
		}
		status := packages[shard][p]
		//log.Printf("package %s: shard %d (%s), status %v\n", p, shardIdx, shard, status)
		if status == Present {
			packages[shard][p] = Confirmed
		} else if status == NotPresent {
			log.Printf("Feeding package %s to shard %d (%s)\n", p, shardIdx, shard)

			var pkgfiles []string
			for _, line := range strings.Split(pkg["Files"], "\n") {
				parts := strings.Split(strings.TrimSpace(line), " ")
				// pkg["Files"] has a newline at the end, so we get one empty line.
				if len(parts) < 3 {
					continue
				}
				url := *mirrorUrl + "/" + pkg["Directory"] + "/" + parts[2]

				// Append the .dsc to the end, prepend the other files.
				if strings.HasSuffix(url, ".dsc") {
					pkgfiles = append(pkgfiles, url)
				} else {
					pkgfiles = append([]string{url}, pkgfiles...)
				}
			}
			feedfiles(p, pkgfiles)

			successfulSanityFeed.Inc()
		}
	}

	// Garbage-collect all packages that have not been confirmed.
	for _, shard := range shards {
		for p, status := range packages[shard] {
			if status != Present {
				continue
			}

			log.Printf("garbage-collecting %q on shard %s\n", p, shard)

			shard := shards[shardmapping.TaskIdxForPackage(p, len(shards))]
			url := fmt.Sprintf("http://%s/garbagecollect", shard)
			if _, err := http.PostForm(url, net_url.Values{"package": {p}}); err != nil {
				log.Printf("Could not garbage-collect package %q on shard %s: %v\n", p, shard, err)
				continue
			}

			successfulGarbageCollect.Inc()
		}
	}
}