func setup() { cachedir := "./_cache" c := diskcache.New(cachedir) tp := httpforcecache.NewTransport(c) client := &http.Client{Transport: tp} h = hitomi.New(client) }
func InitAPI(userAgentIn string) { client.UserAgent = userAgentIn tempDir, err := ioutil.TempDir("", "goztivo_") if err != nil { panic(err) } cache := diskcache.New(tempDir) transport = httpcache.NewTransport(cache) client = ozClient{} client.Client = transport.Client() client.Mutex = &sync.Mutex{} client.LastFetch = time.Now() dataList = &DataList{} dataList.Mutex = &sync.Mutex{} go func() { for { dataList.Mutex.Lock() err := getDataList() dataList.Mutex.Unlock() if err != nil { log.Println(err) return } time.Sleep(time.Hour * 24) } }() }
func newTransportContext() context.Context { var cache httpcache.Cache = httpcache.NewMemoryCache() if cachePath := os.Getenv("GITHUB_CACHE_PATH"); cachePath != "" { cache = diskcache.New(cachePath) } tr := httpcache.NewTransport(cache) c := &http.Client{Transport: tr} return context.WithValue(context.Background(), oauth2.HTTPClient, c) }
func CachingHttpClient() *http.Client { if *diskCacheDir == "" { return nil } if *diskCacheDir == defaultDiskCacheDir { user, err := user.Current() d.Chk.NoError(err) *diskCacheDir = path.Join(user.HomeDir, "noms", "httpcache") } return httpcache.NewTransport(diskcache.New(*diskCacheDir)).Client() }
// Client initializes the authorization with the GitHub API func (g GitHub) Client() *octokat.Client { var cache httpcache.Cache if cachePath := os.Getenv("GITHUB_CACHE_PATH"); cachePath != "" { cache = diskcache.New(cachePath) } else { cache = httpcache.NewMemoryCache() } tr := httpcache.NewTransport(cache) c := &http.Client{Transport: tr} gh := octokat.NewClient() gh = gh.WithToken(g.AuthToken) gh = gh.WithHTTPClient(c) return gh }
func main() { flag.Usage = usage flag.Parse() // If we can have access to a cache directory on this system, use it for // caching HTTP requests of presenters. cacheDir, err := ospath.CacheDir("github.com/shurcooL/Go-Package-Store") if err != nil { log.Println("skipping persistent on-disk caching, because unable to acquire a cache dir:", err) cacheDir = "" } // Set GitHub presenter client. { var transport http.RoundTripper // Optionally, perform GitHub API authentication with provided token. if token := os.Getenv("GO_PACKAGE_STORE_GITHUB_TOKEN"); token != "" { transport = &oauth2.Transport{ Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), } } if cacheDir != "" { transport = &httpcache.Transport{ Transport: transport, Cache: diskcache.New(filepath.Join(cacheDir, "github-presenter")), MarkCachedResponses: true, } } github.SetClient(&http.Client{Transport: transport}) } // Set Gitiles presenter client. { var transport http.RoundTripper if cacheDir != "" { transport = &httpcache.Transport{ Transport: transport, Cache: diskcache.New(filepath.Join(cacheDir, "gitiles-presenter")), MarkCachedResponses: true, } } gitiles.SetClient(&http.Client{Transport: transport}) } switch { case !production: fmt.Println("Using no real packages (hit /mock.html endpoint for mocks).") pipeline.Done() updater = repo.MockUpdater{} default: fmt.Println("Using all Go packages in GOPATH.") go func() { // This needs to happen in the background because sending input will be blocked on processing. forEachRepository(func(r Repo) { pipeline.AddRepository(r) }) pipeline.Done() }() updater = repo.GopathUpdater{} case *stdinFlag: fmt.Println("Reading the list of newline separated Go packages from stdin.") go func() { // This needs to happen in the background because sending input will be blocked on processing. br := bufio.NewReader(os.Stdin) for line, err := br.ReadString('\n'); err == nil; line, err = br.ReadString('\n') { importPath := line[:len(line)-1] // Trim last newline. pipeline.Add(importPath) } pipeline.Done() }() updater = repo.GopathUpdater{} case *godepsFlag != "": fmt.Println("Reading the list of Go packages from Godeps.json file:", *godepsFlag) g, err := readGodeps(*godepsFlag) if err != nil { log.Fatalln("Failed to read Godeps.json file", err) } go func() { // This needs to happen in the background because sending input will be blocked on processing. for _, dependency := range g.Deps { pipeline.AddRevision(dependency.ImportPath, dependency.Rev) } pipeline.Done() }() updater = nil case *govendorFlag != "": fmt.Println("Reading the list of Go packages from vendor.json file:", *govendorFlag) v, err := readGovendor(*govendorFlag) if err != nil { log.Fatalln("Failed to read vendor.json file:", err) } go func() { // This needs to happen in the background because sending input will be blocked on processing. for _, dependency := range v.Package { pipeline.AddRevision(dependency.Path, dependency.Revision) } pipeline.Done() }() if gu, err := repo.NewGovendorUpdater(""); err == nil { updater = gu } else { log.Println("govendor updater is not available:", err) } } err = loadTemplates() if err != nil { log.Fatalln("loadTemplates:", err) } http.HandleFunc("/index.html", mainHandler) http.Handle("/favicon.ico", http.NotFoundHandler()) fileServer := httpgzip.FileServer(assets, httpgzip.FileServerOptions{ServeError: httpgzip.Detailed}) http.Handle("/assets/", fileServer) http.Handle("/assets/octicons/", http.StripPrefix("/assets", fileServer)) http.Handle("/opened", websocket.Handler(openedHandler)) // Exit server when client tab is closed. if updater != nil { http.HandleFunc("/-/update", updateHandler) go updateWorker() } // Start listening first. listener, err := net.Listen("tcp", *httpFlag) if err != nil { log.Fatalf("failed to listen on %q: %v\n", *httpFlag, err) } if production { // Open a browser tab and navigate to the main page. go open.Open("http://" + *httpFlag + "/index.html") } fmt.Println("Go Package Store server is running at http://" + *httpFlag + "/index.html.") err = http.Serve(listener, nil) if err != nil { log.Fatalln(err) } }
func main() { clientID := flag.String("id", "", "Github client ID") clientSecret := flag.String("secret", "", "Github client secret") output := flag.String("output", "gddoexp.out", "Output file") progress := flag.Bool("progress", false, "Show a progress bar") flag.Parse() var auth *gddoexp.GithubAuth if (clientID != nil && *clientID != "") || (clientSecret != nil && *clientSecret != "") { if *clientID == "" || *clientSecret == "" { fmt.Println("to enable Gthub authentication, you need to inform the id and secret") flag.PrintDefaults() return } auth = &gddoexp.GithubAuth{ ID: *clientID, Secret: *clientSecret, } } // add cache to avoid repeated requests to Github gddoexp.HTTPClient = &http.Client{ Transport: httpcache.NewTransport( diskcache.New(path.Join(os.Getenv("HOME"), ".gddoexp")), ), } db, err := database.New() if err != nil { fmt.Println("error connecting to database:", err) return } pkgs, err := db.AllPackages() if err != nil { fmt.Println("error retrieving all packages:", err) return } file, err := os.OpenFile(*output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { fmt.Println("error creating output file:", err) return } defer file.Close() log.SetOutput(file) log.Println("BEGIN") log.Printf("%d packages will be analyzed", len(pkgs)) var progressBar *pb.ProgressBar if progress != nil && *progress { progressBar = pb.StartNew(len(pkgs)) } var cache int for response := range gddoexp.ShouldSuppressPackages(pkgs, db, auth) { if progress != nil && *progress { progressBar.Increment() } if response.Cache { cache++ } if response.Error != nil { log.Println(response.Error) } else if response.Suppress { log.Printf("package “%s” should be suppressed\n", response.Package.Path) if progress != nil && !*progress { fmt.Println(response.Package.Path) } } } if progress != nil && *progress { progressBar.Finish() } log.Println("Cache hits:", cache) log.Println("END") }