Exemplo n.º 1
0
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
	const (
		put = iota
		del
		touch
	)
	action := put

	etag := ""
	if pdoc != nil {
		etag = pdoc.Etag
	}

	var d time.Duration

	if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
		// Go source tree mirror.
		pdoc = nil
		action = del
	} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
		// Go Frontend source tree mirror.
		pdoc = nil
		action = del
	} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
		// Suffix of path matches another package.
		pdoc = nil
		action = del
	} else {
		t := time.Now()
		pdocNew, err := doc.Get(httpClient, path, etag)
		d = time.Since(t) / time.Millisecond

		// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
		httpTransport.CloseIdleConnections()

		switch err {
		case doc.ErrPackageNotModified:
			action = touch
		case doc.ErrPackageNotFound:
			pdoc = nil
			action = del
		case nil:
			pdoc = pdocNew
			action = put
		default:
			log.Printf("%s error  %q %q %dms %v", source, path, etag, d, err)
			return nil, err
		}

		if pdoc != nil && !hasSubdirs {
			if pdoc.Name == "" {
				// Handle directories with no child directories as not found.
				pdoc = nil
				action = del
			} else if pdoc.IsCmd && pdoc.Synopsis == "" {
				///Don't store commands with no documentation and no children.
				action = del
			}
		}
	}

	switch action {
	case put:
		log.Printf("%s put    %q %q %dms", source, path, etag, d)
		if err := db.Put(pdoc); err != nil {
			log.Printf("ERROR db.Put(%q): %v", path, err)
		}
	case touch:
		log.Printf("%s touch  %q %q %dms", source, path, etag, d)
		if err := db.TouchLastCrawl(path); err != nil {
			log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
		}
	case del:
		log.Printf("%s delete %q %q %dms", source, path, etag, d)
		if err := db.Delete(path); err != nil {
			log.Printf("ERROR db.Delete(%q): %v", path, err)
		}
	default:
		panic("should not get here")
	}

	return pdoc, nil
}
Exemplo n.º 2
0
// crawlDoc fetches the package documentation from the VCS and updates the database.
func crawlDoc(source string, path string, pdoc *doc.Package, hasSubdirs bool) (*doc.Package, error) {
	etag := ""
	if pdoc != nil {
		etag = pdoc.Etag
	}

	var d time.Duration
	var err error

	if i := strings.Index(path, "/src/pkg/"); i > 0 && doc.IsGoRepoPath(path[i+len("/src/pkg/"):]) {
		// Go source tree mirror.
		pdoc = nil
		err = doc.NotFoundError{"Go source tree mirror."}
	} else if i := strings.Index(path, "/libgo/go/"); i > 0 && doc.IsGoRepoPath(path[i+len("/libgo/go/"):]) {
		// Go Frontend source tree mirror.
		pdoc = nil
		err = doc.NotFoundError{"Go Frontend source tree mirror."}
	} else if m := nestedProjectPat.FindStringIndex(path); m != nil && exists(path[m[0]+1:]) {
		pdoc = nil
		err = doc.NotFoundError{"Copy of other project."}
	} else if blocked, e := db.IsBlocked(path); blocked && e == nil {
		pdoc = nil
		err = doc.NotFoundError{"Blocked."}
	} else {
		t := time.Now()
		var pdocNew *doc.Package
		pdocNew, err = doc.Get(httpClient, path, etag)
		d = time.Since(t) / time.Millisecond

		// For timeout logic in client.go to work, we cannot leave connections idling. This is ugly.
		httpTransport.CloseIdleConnections()

		if err != doc.ErrNotModified {
			pdoc = pdocNew
		}
	}

	switch {
	case err == nil:
		log.Printf("%s put    %q %q -> %q %dms", source, path, etag, pdoc.Etag, d)
		if err := db.Put(pdoc); err != nil {
			log.Printf("ERROR db.Put(%q): %v", path, err)
		}
	case err == doc.ErrNotModified:
		log.Printf("%s touch  %q %q %dms", source, path, etag, d)
		if err := db.TouchLastCrawl(pdoc); err != nil {
			log.Printf("ERROR db.TouchLastCrawl(%q): %v", path, err)
		}
	case doc.IsNotFound(err):
		pdoc = nil
		log.Printf("%s delete %q %s %dms", source, path, err.Error(), d)
		if err := db.Delete(path); err != nil {
			log.Printf("ERROR db.Delete(%q): %v", path, err)
		}
	default:
		log.Printf("%s error  %q %q %dms %v", source, path, etag, d, err)
		return nil, err
	}

	return pdoc, nil
}