func configure() { loglevel := "INFO" flag.StringVar(&loglevel, "loglevel", "INFO", "Log level") cachedir := ".gopancache" flag.StringVar(&cachedir, "cachedir", ".gopancache", "GoPAN cache directory") index := "index" flag.StringVar(&index, "index", "index", "Name of the primary GoPAN index file") bind := ":7050" flag.StringVar(&bind, "bind", ":7050", "Interface to bind to") cpan := "cpan_index.gz" flag.StringVar(&cpan, "cpan", "cpan_index.gz", "Name of CPAN index to support readthrough") backpan := "backpan_index.gz" flag.StringVar(&backpan, "backpan", "backpan_index.gz", "Name of BackPAN index to support readthrough") remote := "http://localhost:7050" flag.StringVar(&remote, "remote", "http://localhost:7050", "Remote host and port to connect to") loglayout := "[%d] [%p] %m" flag.StringVar(&loglayout, "loglayout", loglayout, "Log layout (for github.com/ian-kent/go-log pattern layout)") indexes := make([]string, 0) flag.Var((*gopan.AppendSliceValue)(&indexes), "s-index", "Secondary indexes to load (can be provided multiple times)") testdeps := true flag.BoolVar(&testdeps, "test-deps", true, "Test dependencies on startup") flag.Parse() config = &Config{ LogLevel: loglevel, CacheDir: cachedir, Index: index, Bind: bind, Indexes: indexes, CanUpdate: false, LatestRelease: "0.0", CurrentRelease: "0.0", // set in main.go so its in one place UpdateURL: "", ImportAvailable: false, CPANIndex: cpan, HasCPANIndex: false, CPANIndexDate: "", BackPANIndex: backpan, HasBackPANIndex: false, BackPANIndexDate: "", BackPANStatus: "Unavailable", CPANStatus: "Unavailable", RemoteHost: remote, TestDeps: testdeps, } // log level and layout log.Logger().Appender().SetLayout(layout.Pattern(loglayout)) log.Logger().SetLevel(log.Stol(loglevel)) }
func main() { // Create our Gotcha application var app = gotcha.Create(Asset) // Set the logger output pattern log.Logger().Appender().SetLayout(layout.Pattern("[%d] [%p] %m")) log.Logger().SetLevel(log.Stol("TRACE")) // Get the router r := app.Router // Create some routes r.Get("/", example) r.Post("/", examplepost) r.Get("/foo", example2) r.Get("/bar", example3) r.Get("/stream", streamed) r.Get("/err", err) // Serve static content (but really use a CDN) r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}")) r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}")) // Listen to some events app.On(events.BeforeHandler, func(session *http.Session, next func()) { n := 0 c, ok := session.Request.Cookies["test"] if ok { n, _ = strconv.Atoi(c.Value) } session.Stash["test"] = n log.Printf("Got BeforeHandler event! n = %d", n) next() }) app.On(events.AfterHandler, func(session *http.Session, next func()) { n := session.Stash["test"].(int) + 1 session.Response.Cookies.Set(&nethttp.Cookie{ Name: "test", Value: strconv.Itoa(n), }) log.Println("Got AfterHandler event!") next() }) app.On(events.AfterResponse, func(session *http.Session, next func()) { log.Println("Got AfterResponse event!") next() }) // Start our application app.Start() <-make(chan int) }
func main() { configure() indexes = make(map[string]map[string]*gopan.Source) indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex) log.Logger().SetLevel(log.Stol(config.LogLevel)) log.Info("Using log level: %s", config.LogLevel) // FIXME inefficient _, _, tpkg, _ := gopan.CountIndex(indexes) npkg := 0 nmod := 0 var pc = func() float64 { return float64(nmod) / float64(tpkg) * 100 } log.Info("Writing packages index file") out, err := os.Create(config.CacheDir + "/" + config.OutputIndex) if err != nil { log.Error("Error creating packages index: %s", err.Error()) return } for fname, _ := range indexes { log.Debug("File: %s", fname) for _, idx := range indexes[fname] { log.Debug("Index: %s", idx) out.Write([]byte(idx.Name + " [" + idx.URL + "]\n")) for _, auth := range idx.Authors { log.Debug("Author %s", auth) out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n")) for _, pkg := range auth.Packages { out.Write([]byte(" " + pkg.Name + " => " + pkg.URL + "\n")) log.Debug("Package: %s", pkg) if !config.Flatten { if len(pkg.Provides) == 0 { // TODO better handling of filenames modnm := strings.TrimSuffix(pkg.Name, ".tar.gz") tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name if _, err := os.Stat(tgzpath); err != nil { log.Error("File not found: %s", tgzpath) return } extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name log.Trace("=> tgzpath: %s", tgzpath) log.Trace(" > extpath: %s", extpath) log.Trace(" > dirpath: %s", dirpath) // Only index packages if they don't already exist if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil { log.Error("Error retrieving package list: %s", err) continue } } npkg += len(pkg.Provides) nmod += 1 for p, pk := range pkg.Provides { out.Write([]byte(" " + p + " (" + pk.Version + "): " + pk.File + "\n")) } if nmod > 0 && nmod%100 == 0 { log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg) } } } } } } out.Close() log.Info("Found %d packages from %d modules", npkg, nmod) }
func main() { configure() log.Logger().SetLevel(log.Stol(config.LogLevel)) log.Info("Using log level: %s", config.LogLevel) indexes = make(map[string]map[string]*gopan.Source) if !config.NoCache { indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index) } if config.NoCache || config.Update { for _, s := range config.Sources { b := strings.SplitN(s, "=", 2) if len(b) < 2 { log.Error("Expected Name=URL pair, got: %s", s) return } if idx, ok := indexes[config.Index][b[0]]; ok { log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1]) idx.URL = b[1] } else { indexes[config.Index][b[0]] = &gopan.Source{ Name: b[0], URL: b[1], Authors: make(map[string]*gopan.Author, 0), } } } if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN { log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN") config.CPAN = true config.BackPAN = true } if config.CPAN { if _, ok := indexes[config.Index]["CPAN"]; !ok { log.Debug("Adding CPAN index") indexes[config.Index]["CPAN"] = gopan.CPANSource() } else { log.Debug("CPAN index already exists") } } if config.BackPAN { if _, ok := indexes[config.Index]["BackPAN"]; !ok { log.Debug("Adding BackPAN index") indexes[config.Index]["BackPAN"] = gopan.BackPANSource() } else { log.Debug("BackPAN index already exists") } } log.Info("Using sources:") for fname, _ := range indexes { log.Info("From %s", fname) for _, source := range indexes[fname] { log.Info("=> %s", source.String()) } } newAuthors := getAuthors() newPackages := getPackages() os.MkdirAll(config.CacheDir, 0777) if !config.NoCache { gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index]) } if config.Update { log.Info("Found %d new packages by %d new authors", newAuthors, newPackages) } } nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes) log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc) if !config.NoMirror { mirrorPan() } }
func Configure() *Config { conf := DefaultConfig() cpan := make([]string, 0) flag.Var((*gopan.AppendSliceValue)(&cpan), "cpan", "A CPAN mirror (can be specified multiple times)") backpan := make([]string, 0) flag.Var((*gopan.AppendSliceValue)(&backpan), "backpan", "A BackPAN mirror (can be specified multiple times)") smart := make([]string, 0) flag.Var((*gopan.AppendSliceValue)(&smart), "smart", "A SmartPAN mirror (can be specified multiple times)") test := make([]string, 0) flag.Var((*gopan.AppendSliceValue)(&test), "test", "A module to install with testing (can be specified multiple times)") tests := false flag.BoolVar(&tests, "tests", false, "Enables all tests during installation phase") nocpan := false flag.BoolVar(&nocpan, "nocpan", false, "Disables the default CPAN source") nobackpan := false flag.BoolVar(&nobackpan, "nobackpan", false, "Disables the default BackCPAN source") cpanfile := "cpanfile" flag.StringVar(&cpanfile, "cpanfile", "cpanfile", "The cpanfile to install") numcpus := runtime.NumCPU() flag.IntVar(&numcpus, "cpus", numcpus, "The number of CPUs to use, defaults to "+strconv.Itoa(numcpus)+" for your environment") nodepdump := false flag.BoolVar(&nodepdump, "nodepdump", nodepdump, "Disables dumping resolved dependencies listing") noinstall := false flag.BoolVar(&noinstall, "noinstall", noinstall, "Disables installation phase") metacpan := false flag.BoolVar(&metacpan, "metacpan", metacpan, "Enable resolving source via MetaCPAN") loglevel := "INFO" flag.StringVar(&loglevel, "loglevel", loglevel, "Log output level (ERROR, INFO, WARN, DEBUG, TRACE)") loglayout := "[%d] [%p] %m" flag.StringVar(&loglayout, "loglayout", loglayout, "Log layout (for github.com/ian-kent/go-log pattern layout)") cachedir := "./.gopancache" flag.StringVar(&cachedir, "cachedir", cachedir, "Cache directory for CPAN modules") installdir := "./local" flag.StringVar(&installdir, "installdir", installdir, "Install directory for CPAN modules") flag.Parse() if nocpan || nobackpan { conf.Sources = DefaultSources(!nocpan, !nobackpan) } // parse cpan mirrors for _, url := range cpan { mirror := strings.TrimSuffix(url, "/") m := NewSource("CPAN", mirror+"/modules/02packages.details.txt.gz", mirror) conf.Sources = append(conf.Sources, m) } // parse backpan mirrors for _, url := range backpan { mirror := strings.TrimSuffix(url, "/") m := NewSource("BackPAN", mirror+"/backpan-index", mirror) // FIXME conf.Sources = append(conf.Sources, m) } // parse smartpan mirrors for _, url := range smart { mirror := strings.TrimSuffix(url, "/") m := NewSource("SmartPAN", mirror, mirror) conf.Sources = append(conf.Sources, m) } // resolve via metacpan conf.MetaCPAN = metacpan if metacpan { m := NewSource("MetaCPAN", "", "") conf.Sources = append(conf.Sources, m) c := NewMetaSource("cpan", "", "http://www.cpan.org", m.ModuleList) conf.MetaSources = append(conf.MetaSources, c) b := NewMetaSource("backpan", "", "http://backpan.perl.org", m.ModuleList) conf.MetaSources = append(conf.MetaSources, b) } sort.Sort(sortSources(conf.Sources)) // parse notest for _, n := range test { conf.Test.Modules[n] = true } conf.Test.Global = tests // parse cpanfile conf.CPANFile = cpanfile // numcpus conf.CPUs = numcpus // nodepdeump conf.NoDepdump = nodepdump // noinstall conf.NoInstall = noinstall // install dir conf.InstallDir = installdir // cache dir conf.CacheDir = cachedir // log level and layout log.Logger().Appender().SetLayout(layout.Pattern(loglayout)) log.Logger().SetLevel(log.Stol(loglevel)) // create cache dir os.MkdirAll(conf.CacheDir, 0777) config = conf return conf }
func main() { configure() args := flag.Args() if len(args) > 0 && args[0] == "init" { log.Info("Initialising SmartPAN") log.Info("=> Installing Perl dependencies") // FIXME most of this is repeated from getpan/main.go cfg := getpan.DefaultConfig() cfg.CacheDir = config.CacheDir for _, source := range cfg.Sources { if err := source.Load(); err != nil { log.Error("Error loading sources: %s", err) os.Exit(1) return } } deps := &getpan.DependencyList{ Dependencies: make([]*getpan.Dependency, 0), } d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "") d2, _ := getpan.DependencyFromString("JSON::XS", "") deps.AddDependency(d1) deps.AddDependency(d2) if err := deps.Resolve(); err != nil { log.Error("Error resolving dependencies: %s", err) os.Exit(1) return } _, err := deps.Install() if err != nil { log.Error("Error installing dependencies: %s", err) os.Exit(2) return } log.Info(" - Installed %d modules", deps.UniqueInstalled()) log.Info("SmartPAN initialisation complete") return } if config.TestDeps { perldeps := gopan.TestPerlDeps() perldeps.Dump() if !perldeps.Ok { log.Error("Required perl dependencies are missing") os.Exit(1) return } } if len(args) > 0 && args[0] == "import" { if len(args) < 4 { log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX") return } fname := args[1] log.Info("Importing module from %s", fname) log.Info("Author ID: %s", args[2]) log.Info("Index : %s", args[3]) extraParams := map[string]string{ "importinto": args[3], "authorid": args[2], "newindex": "", "cpanmirror": "", "importurl": "", "fromdir": "", } if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") { log.Info("URL: %s", fname) extraParams["importurl"] = fname request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams) if err != nil { log.Error("Create request error: %s", err.Error()) return } client := &nethttp.Client{} resp, err := client.Do(request) if err != nil { log.Error("Error connecting to host: %s", err.Error()) return } else { // TODO stream this body := &bytes.Buffer{} _, err := body.ReadFrom(resp.Body) if err != nil { log.Error("Error reading response: %s", err.Error()) return } resp.Body.Close() //log.Info("%d", resp.StatusCode) //log.Info("%s", resp.Header) log.Info("%s", body.String()) } } else { fname = strings.TrimPrefix(fname, "file://") log.Info("File: %s", fname) if _, err := os.Stat(fname); err != nil { log.Error("File not found: %s", err.Error()) return } request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname) if err != nil { log.Error("Create upload error: %s", err.Error()) return } client := &nethttp.Client{} resp, err := client.Do(request) if err != nil { log.Error("Error connecting to host: %s", err.Error()) return } else { // TODO stream this body := &bytes.Buffer{} _, err := body.ReadFrom(resp.Body) if err != nil { log.Error("Error reading response: %s", err.Error()) return } resp.Body.Close() //log.Info("%d", resp.StatusCode) //log.Info("%s", resp.Header) log.Info("%s", body.String()) } } return } config.CurrentRelease = CurrentRelease var wg sync.WaitGroup load_index = func(index string, file string) { indexes[index] = gopan.LoadIndex(file) } wg.Add(1) go func() { defer wg.Done() indexes = make(map[string]map[string]*gopan.Source) // Load CPAN index if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil { config.HasCPANIndex = true config.CPANIndexDate = fi.ModTime().String() config.CPANStatus = "Loading" wg.Add(1) go func() { defer wg.Done() load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex) config.CPANStatus = "Loaded" }() } // Load BackPAN index if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil { config.HasBackPANIndex = true config.BackPANIndexDate = fi.ModTime().String() config.BackPANStatus = "Loading" wg.Add(1) go func() { defer wg.Done() load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex) config.BackPANStatus = "Loaded" }() } // Load our secondary indexes for _, idx := range config.Indexes { wg.Add(1) go func() { defer wg.Done() load_index(idx, config.CacheDir+"/"+idx) }() } // Load our primary index (this is the only index written back to) wg.Add(1) go func() { defer wg.Done() load_index(config.Index, config.CacheDir+"/"+config.Index) }() }() update_indexes = func() { wg.Wait() wg.Add(1) go func() { wg.Wait() config.ImportAvailable = true nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes) // TODO should probably be in the index - needs to udpate when index changes summary = &Summary{nsrc, nauth, npkg, nprov} // Do this now so changing the level doesn't interfere with index load log.Logger().SetLevel(log.Stol(config.LogLevel)) }() defer wg.Done() // Create in-memory indexes for UI/search etc for fname, _ := range indexes { for idn, idx := range indexes[fname] { mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author) for _, auth := range idx.Authors { // author name if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok { mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author) } if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok { mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author) } mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth // wildcards if _, ok := mapped[idx.Name]["*"]; !ok { mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author) } if _, ok := mapped[idx.Name]["*"]["**"]; !ok { mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author) } mapped[idx.Name]["*"]["**"][auth.Name] = auth // combos if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok { mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author) } if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok { mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author) } mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth for _, pkg := range auth.Packages { filemap[pkg.AuthorURL()] = idn for _, prov := range pkg.Provides { parts := strings.Split(prov.Name, "::") log.Trace("PACKAGE: %s", prov.Name) if _, ok := packages[parts[0]]; !ok { packages[parts[0]] = &PkgSpace{ Namespace: parts[0], Packages: make([]*gopan.PerlPackage, 0), Children: make(map[string]*PkgSpace), Parent: nil, Versions: make(map[float64]*gopan.PerlPackage), } } if _, ok := idxpackages[idx.Name]; !ok { idxpackages[idx.Name] = make(map[string]*PkgSpace) } if _, ok := idxpackages[idx.Name][parts[0]]; !ok { idxpackages[idx.Name][parts[0]] = &PkgSpace{ Namespace: parts[0], Packages: make([]*gopan.PerlPackage, 0), Children: make(map[string]*PkgSpace), Parent: nil, Versions: make(map[float64]*gopan.PerlPackage), } } if len(parts) == 1 { packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov) packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov) idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name) } else { packages[parts[0]].Populate(parts[1:], prov) idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov) } } } } } } } go update_indexes() // Get latest SmartPAN version go func() { res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases") if err != nil { log.Error("Error getting latest version: %s", err.Error()) return } defer res.Body.Close() b, err := ioutil.ReadAll(res.Body) if err != nil { log.Error("Error reading stream: %s", err.Error()) return } var r Releases if err = json.Unmarshal(b, &r); err != nil { log.Error("Error unmarshalling JSON: %s", err.Error()) return } log.Info("Current release: %s", config.CurrentRelease) rel := strings.TrimPrefix(r[0].TagName, "v") log.Info("Latest release: %s", rel) config.LatestRelease = rel config.UpdateURL = r[0].URL if config.CurrentRelease < rel { config.CanUpdate = true log.Info("Your version of SmartPAN can be updated.") } }() // Create our Gotcha application var app = gotcha.Create(Asset) app.Config.Listen = config.Bind summary = &Summary{0, 0, 0, 0} app.On(events.BeforeHandler, func(session *http.Session, next func()) { session.Stash["summary"] = summary session.Stash["config"] = config next() }) // Get the router r := app.Router // Create some routes r.Get("/", search) r.Post("/", search) r.Get("/help", help) r.Get("/settings", settings) r.Get("/browse", browse) r.Get("/import", import1) r.Post("/import", import1) r.Get("/import/(?P<jobid>[^/]+)", import2) r.Get("/import/(?P<jobid>[^/]+)/stream", importstream) r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex) // Serve static content (but really use a CDN) r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}")) r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}")) // JSON endpoints r.Get("/where/(?P<module>[^/]+)/?", where) r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where) // Put these last so they only match /{repo} if nothing else matches r.Get("/(?P<repo>[^/]+)/?", browse) r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse) r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex) r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download) r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file) r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse) // Start our application app.Start() <-make(chan int) }