func setup() {
cachedir := "./_cache"
c := diskcache.New(cachedir)
tp := httpforcecache.NewTransport(c)
client := &http.Client{Transport: tp}
h = hitomi.New(client)
}
func InitAPI(userAgentIn string) {
client.UserAgent = userAgentIn
tempDir, err := ioutil.TempDir("", "goztivo_")
if err != nil {
panic(err)
}
cache := diskcache.New(tempDir)
transport = httpcache.NewTransport(cache)
client = ozClient{}
client.Client = transport.Client()
client.Mutex = &sync.Mutex{}
client.LastFetch = time.Now()
dataList = &DataList{}
dataList.Mutex = &sync.Mutex{}
go func() {
for {
dataList.Mutex.Lock()
err := getDataList()
dataList.Mutex.Unlock()
if err != nil {
log.Println(err)
return
}
time.Sleep(time.Hour * 24)
}
}()
}
func newTransportContext() context.Context {
var cache httpcache.Cache = httpcache.NewMemoryCache()
if cachePath := os.Getenv("GITHUB_CACHE_PATH"); cachePath != "" {
cache = diskcache.New(cachePath)
}
tr := httpcache.NewTransport(cache)
c := &http.Client{Transport: tr}
return context.WithValue(context.Background(), oauth2.HTTPClient, c)
}
func CachingHttpClient() *http.Client {
if *diskCacheDir == "" {
return nil
}
if *diskCacheDir == defaultDiskCacheDir {
user, err := user.Current()
d.Chk.NoError(err)
*diskCacheDir = path.Join(user.HomeDir, "noms", "httpcache")
}
return httpcache.NewTransport(diskcache.New(*diskCacheDir)).Client()
}
// Client initializes the authorization with the GitHub API
func (g GitHub) Client() *octokat.Client {
var cache httpcache.Cache
if cachePath := os.Getenv("GITHUB_CACHE_PATH"); cachePath != "" {
cache = diskcache.New(cachePath)
} else {
cache = httpcache.NewMemoryCache()
}
tr := httpcache.NewTransport(cache)
c := &http.Client{Transport: tr}
gh := octokat.NewClient()
gh = gh.WithToken(g.AuthToken)
gh = gh.WithHTTPClient(c)
return gh
}
func main() {
flag.Usage = usage
flag.Parse()
// If we can have access to a cache directory on this system, use it for
// caching HTTP requests of presenters.
cacheDir, err := ospath.CacheDir("github.com/shurcooL/Go-Package-Store")
if err != nil {
log.Println("skipping persistent on-disk caching, because unable to acquire a cache dir:", err)
cacheDir = ""
}
// Set GitHub presenter client.
{
var transport http.RoundTripper
// Optionally, perform GitHub API authentication with provided token.
if token := os.Getenv("GO_PACKAGE_STORE_GITHUB_TOKEN"); token != "" {
transport = &oauth2.Transport{
Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}),
}
}
if cacheDir != "" {
transport = &httpcache.Transport{
Transport: transport,
Cache: diskcache.New(filepath.Join(cacheDir, "github-presenter")),
MarkCachedResponses: true,
}
}
github.SetClient(&http.Client{Transport: transport})
}
// Set Gitiles presenter client.
{
var transport http.RoundTripper
if cacheDir != "" {
transport = &httpcache.Transport{
Transport: transport,
Cache: diskcache.New(filepath.Join(cacheDir, "gitiles-presenter")),
MarkCachedResponses: true,
}
}
gitiles.SetClient(&http.Client{Transport: transport})
}
switch {
case !production:
fmt.Println("Using no real packages (hit /mock.html endpoint for mocks).")
pipeline.Done()
updater = repo.MockUpdater{}
default:
fmt.Println("Using all Go packages in GOPATH.")
go func() { // This needs to happen in the background because sending input will be blocked on processing.
forEachRepository(func(r Repo) {
pipeline.AddRepository(r)
})
pipeline.Done()
}()
updater = repo.GopathUpdater{}
case *stdinFlag:
fmt.Println("Reading the list of newline separated Go packages from stdin.")
go func() { // This needs to happen in the background because sending input will be blocked on processing.
br := bufio.NewReader(os.Stdin)
for line, err := br.ReadString('\n'); err == nil; line, err = br.ReadString('\n') {
importPath := line[:len(line)-1] // Trim last newline.
pipeline.Add(importPath)
}
pipeline.Done()
}()
updater = repo.GopathUpdater{}
case *godepsFlag != "":
fmt.Println("Reading the list of Go packages from Godeps.json file:", *godepsFlag)
g, err := readGodeps(*godepsFlag)
if err != nil {
log.Fatalln("Failed to read Godeps.json file", err)
}
go func() { // This needs to happen in the background because sending input will be blocked on processing.
for _, dependency := range g.Deps {
pipeline.AddRevision(dependency.ImportPath, dependency.Rev)
}
pipeline.Done()
}()
updater = nil
case *govendorFlag != "":
fmt.Println("Reading the list of Go packages from vendor.json file:", *govendorFlag)
v, err := readGovendor(*govendorFlag)
if err != nil {
log.Fatalln("Failed to read vendor.json file:", err)
}
go func() { // This needs to happen in the background because sending input will be blocked on processing.
for _, dependency := range v.Package {
pipeline.AddRevision(dependency.Path, dependency.Revision)
}
pipeline.Done()
}()
if gu, err := repo.NewGovendorUpdater(""); err == nil {
updater = gu
} else {
log.Println("govendor updater is not available:", err)
}
}
err = loadTemplates()
if err != nil {
log.Fatalln("loadTemplates:", err)
}
http.HandleFunc("/index.html", mainHandler)
http.Handle("/favicon.ico", http.NotFoundHandler())
fileServer := httpgzip.FileServer(assets, httpgzip.FileServerOptions{ServeError: httpgzip.Detailed})
http.Handle("/assets/", fileServer)
http.Handle("/assets/octicons/", http.StripPrefix("/assets", fileServer))
http.Handle("/opened", websocket.Handler(openedHandler)) // Exit server when client tab is closed.
if updater != nil {
http.HandleFunc("/-/update", updateHandler)
go updateWorker()
}
// Start listening first.
listener, err := net.Listen("tcp", *httpFlag)
if err != nil {
log.Fatalf("failed to listen on %q: %v\n", *httpFlag, err)
}
if production {
// Open a browser tab and navigate to the main page.
go open.Open("http://" + *httpFlag + "/index.html")
}
fmt.Println("Go Package Store server is running at http://" + *httpFlag + "/index.html.")
err = http.Serve(listener, nil)
if err != nil {
log.Fatalln(err)
}
}
func main() {
clientID := flag.String("id", "", "Github client ID")
clientSecret := flag.String("secret", "", "Github client secret")
output := flag.String("output", "gddoexp.out", "Output file")
progress := flag.Bool("progress", false, "Show a progress bar")
flag.Parse()
var auth *gddoexp.GithubAuth
if (clientID != nil && *clientID != "") || (clientSecret != nil && *clientSecret != "") {
if *clientID == "" || *clientSecret == "" {
fmt.Println("to enable Gthub authentication, you need to inform the id and secret")
flag.PrintDefaults()
return
}
auth = &gddoexp.GithubAuth{
ID: *clientID,
Secret: *clientSecret,
}
}
// add cache to avoid repeated requests to Github
gddoexp.HTTPClient = &http.Client{
Transport: httpcache.NewTransport(
diskcache.New(path.Join(os.Getenv("HOME"), ".gddoexp")),
),
}
db, err := database.New()
if err != nil {
fmt.Println("error connecting to database:", err)
return
}
pkgs, err := db.AllPackages()
if err != nil {
fmt.Println("error retrieving all packages:", err)
return
}
file, err := os.OpenFile(*output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
fmt.Println("error creating output file:", err)
return
}
defer file.Close()
log.SetOutput(file)
log.Println("BEGIN")
log.Printf("%d packages will be analyzed", len(pkgs))
var progressBar *pb.ProgressBar
if progress != nil && *progress {
progressBar = pb.StartNew(len(pkgs))
}
var cache int
for response := range gddoexp.ShouldSuppressPackages(pkgs, db, auth) {
if progress != nil && *progress {
progressBar.Increment()
}
if response.Cache {
cache++
}
if response.Error != nil {
log.Println(response.Error)
} else if response.Suppress {
log.Printf("package “%s” should be suppressed\n", response.Package.Path)
if progress != nil && !*progress {
fmt.Println(response.Package.Path)
}
}
}
if progress != nil && *progress {
progressBar.Finish()
}
log.Println("Cache hits:", cache)
log.Println("END")
}