func main() { configure() indexes = make(map[string]map[string]*gopan.Source) indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex) log.Logger().SetLevel(log.Stol(config.LogLevel)) log.Info("Using log level: %s", config.LogLevel) // FIXME inefficient _, _, tpkg, _ := gopan.CountIndex(indexes) npkg := 0 nmod := 0 var pc = func() float64 { return float64(nmod) / float64(tpkg) * 100 } log.Info("Writing packages index file") out, err := os.Create(config.CacheDir + "/" + config.OutputIndex) if err != nil { log.Error("Error creating packages index: %s", err.Error()) return } for fname, _ := range indexes { log.Debug("File: %s", fname) for _, idx := range indexes[fname] { log.Debug("Index: %s", idx) out.Write([]byte(idx.Name + " [" + idx.URL + "]\n")) for _, auth := range idx.Authors { log.Debug("Author %s", auth) out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n")) for _, pkg := range auth.Packages { out.Write([]byte(" " + pkg.Name + " => " + pkg.URL + "\n")) log.Debug("Package: %s", pkg) if !config.Flatten { if len(pkg.Provides) == 0 { // TODO better handling of filenames modnm := strings.TrimSuffix(pkg.Name, ".tar.gz") tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name if _, err := os.Stat(tgzpath); err != nil { log.Error("File not found: %s", tgzpath) return } extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name log.Trace("=> tgzpath: %s", tgzpath) log.Trace(" > extpath: %s", extpath) log.Trace(" > dirpath: %s", dirpath) // Only index packages if they don't already exist if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil { log.Error("Error retrieving package list: %s", err) continue } } npkg += len(pkg.Provides) nmod += 1 for p, pk := range pkg.Provides { out.Write([]byte(" " + p + " (" + pk.Version + "): " + pk.File + "\n")) } if nmod > 0 && nmod%100 == 0 { log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg) } } } } } } out.Close() log.Info("Found %d packages from %d modules", npkg, nmod) }
func main() { configure() log.Logger().SetLevel(log.Stol(config.LogLevel)) log.Info("Using log level: %s", config.LogLevel) indexes = make(map[string]map[string]*gopan.Source) if !config.NoCache { indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index) } if config.NoCache || config.Update { for _, s := range config.Sources { b := strings.SplitN(s, "=", 2) if len(b) < 2 { log.Error("Expected Name=URL pair, got: %s", s) return } if idx, ok := indexes[config.Index][b[0]]; ok { log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1]) idx.URL = b[1] } else { indexes[config.Index][b[0]] = &gopan.Source{ Name: b[0], URL: b[1], Authors: make(map[string]*gopan.Author, 0), } } } if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN { log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN") config.CPAN = true config.BackPAN = true } if config.CPAN { if _, ok := indexes[config.Index]["CPAN"]; !ok { log.Debug("Adding CPAN index") indexes[config.Index]["CPAN"] = gopan.CPANSource() } else { log.Debug("CPAN index already exists") } } if config.BackPAN { if _, ok := indexes[config.Index]["BackPAN"]; !ok { log.Debug("Adding BackPAN index") indexes[config.Index]["BackPAN"] = gopan.BackPANSource() } else { log.Debug("BackPAN index already exists") } } log.Info("Using sources:") for fname, _ := range indexes { log.Info("From %s", fname) for _, source := range indexes[fname] { log.Info("=> %s", source.String()) } } newAuthors := getAuthors() newPackages := getPackages() os.MkdirAll(config.CacheDir, 0777) if !config.NoCache { gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index]) } if config.Update { log.Info("Found %d new packages by %d new authors", newAuthors, newPackages) } } nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes) log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc) if !config.NoMirror { mirrorPan() } }
func mirrorPan() { log.Info("Mirroring *PAN") // FIXME inefficient _, _, npkg, _ := gopan.CountIndex(indexes) mirrored := 0 var pc = func() int { return mirrored / npkg * 100 } for fname, _ := range indexes { log.Debug("File: %s", fname) for _, source := range indexes[fname] { log.Debug("Index: %s", source) wg.Add(1) go func(source *gopan.Source) { defer wg.Done() for _, author := range source.Authors { log.Debug("=> %s", author) wg.Add(1) go func(author *gopan.Author) { cachedir := config.CacheDir + "/" + source.Name + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" os.MkdirAll(cachedir, 0777) defer wg.Done() for _, pkg := range author.Packages { wg.Add(1) go func(pkg *gopan.Package) { defer wg.Done() cache := cachedir + pkg.Name log.Trace(" - Caching to: %s", cache) if _, err := os.Stat(cache); err == nil { log.Debug("%d%% |> %s", pc(), pkg) log.Trace(" - Already exists in cache") mirrored++ return } sem <- 1 mirrored++ log.Debug("%d%% => %s", pc(), pkg) url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" + pkg.Name log.Trace(" - From URL: %s", url) out, err := os.Create(cache) defer out.Close() if err != nil { log.Error("CREATE - %s", err.Error()) <-sem return } resp, err := http.Get(url) if err != nil { log.Error("HTTP GET - %s", err.Error()) <-sem return } _, err = io.Copy(out, resp.Body) if err != nil { log.Error("IO COPY - %s", err.Error()) } <-sem }(pkg) } }(author) } }(source) } } wg.Wait() log.Info("Finished mirroring *PAN") }
func do_import(session *http.Session, job *ImportJob) { log.Info("Running import job %s", job.Id) reponame := job.Form.ImportInto if reponame == "new_index" { reponame = job.Form.NewIndex } msg := func(m string) { if m != ":DONE" { job.History = append(job.History, m) log.Info(m) } for _, w := range job.Watchers { w(m) } } mods := make([]*getpan.Module, 0) // TODO cpanm mirror when using getpan_import if len(job.Form.Cpanfile) > 0 { msg("Parsing cpanfile input") _, modules := getpan_import(job, msg) mods = append(mods, modules...) } if len(job.Form.ImportURL) > 0 { msg("Importing from URL: " + job.Form.ImportURL) // TODO support cpanfile urls nauth := job.Form.AuthorID if len(nauth) < 3 { // FIXME move to form validation msg("Author ID must be at least 3 characters") msg(":DONE") job.Complete = true return } npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth _, fn := filepath.Split(job.Form.ImportURL) nfile := npath + "/" + fn msg("Caching to " + nfile) if _, err := os.Stat(nfile); err != nil { os.MkdirAll(npath, 0777) out, err := os.Create(nfile) if err != nil { msg(err.Error()) msg(":DONE") job.Complete = true return } url := job.Form.ImportURL log.Trace("Downloading: %s", url) resp, err := nethttp.Get(url) if err != nil { msg(err.Error()) msg(":DONE") job.Complete = true return } _, err = io.Copy(out, resp.Body) if err != nil { msg(err.Error()) msg(":DONE") job.Complete = true return } out.Close() resp.Body.Close() } else { log.Trace("File already exists in cache: %s", nfile) } fn = strings.TrimSuffix(fn, ".tar.gz") bits := strings.Split(fn, "-") name := strings.Join(bits[0:len(bits)-1], "-") version := bits[len(bits)-1] s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "") m := &getpan.Module{ Source: s, Name: name, Version: version, Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn, Cached: nfile, Dir: npath, } m.Deps = &getpan.DependencyList{ Parent: m, Dependencies: make([]*getpan.Dependency, 0), } mods = append(mods, m) } if len(job.Form.FromDir) > 0 { msg("Importing from local directory: " + job.Form.FromDir) // TODO support cpanfile paths nauth := job.Form.AuthorID if len(nauth) < 3 { // FIXME move to form validation msg("Author ID must be at least 3 characters") msg(":DONE") job.Complete = true return } npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth _, fn := filepath.Split(job.Form.FromDir) nfile := npath + "/" + fn msg("Caching to " + nfile) _, err := CopyFile(nfile, job.Form.FromDir) if err != nil { msg(err.Error()) msg(":DONE") job.Complete = true return } fn = strings.TrimSuffix(fn, ".tar.gz") bits := strings.Split(fn, "-") name := strings.Join(bits[0:len(bits)-1], "-") version := bits[len(bits)-1] s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "") m := &getpan.Module{ Source: s, Name: name, Version: version, Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn, Cached: nfile, Dir: npath, } m.Deps = &getpan.DependencyList{ Parent: m, Dependencies: make([]*getpan.Dependency, 0), } mods = append(mods, m) } if f, fh, err := session.Request.File("fromfile"); err == nil { fn := fh.Filename msg("Importing from uploaded module/cpanfile: " + fn) if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" { msg("Only cpanfile and *.tar.gz files are supported") msg(":DONE") job.Complete = true return } if fn == "cpanfile" { msg("Importing cpanfile") b, _ := ioutil.ReadAll(f) f.Close() job.Form.Cpanfile = string(b) _, modules := getpan_import(job, msg) mods = append(mods, modules...) } else { msg("Importing .tar.gz") nauth := job.Form.AuthorID if len(nauth) < 3 { // FIXME move to form validation msg("Author ID must be at least 3 characters") msg(":DONE") job.Complete = true return } npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth _, fn = filepath.Split(fn) nfile := npath + "/" + fn msg("Caching to " + nfile) os.MkdirAll(npath, 0777) _, err := CopyToFile(nfile, f) if err != nil { msg(err.Error()) msg(":DONE") job.Complete = true return } fn = strings.TrimSuffix(fn, ".tar.gz") bits := strings.Split(fn, "-") name := strings.Join(bits[0:len(bits)-1], "-") version := bits[len(bits)-1] s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "") m := &getpan.Module{ Source: s, Name: name, Version: version, Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn, Cached: nfile, Dir: npath, } m.Deps = &getpan.DependencyList{ Parent: m, Dependencies: make([]*getpan.Dependency, 0), } mods = append(mods, m) } } else { // there is no file... so no error //msg("Error importing file upload: " + err.Error()) } if len(mods) == 0 { msg("Nothing to do") msg(":DONE") job.Complete = true return } msg("Adding modules to GoPAN index") for _, m := range mods { msg("=> " + m.Name + " (" + m.Cached + ")") dn, fn := filepath.Split(m.Cached) dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator)) auth := dnb[len(dnb)-1] ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth npath := ndir + "/" + fn if _, err := os.Stat(npath); err == nil { msg(" | Already exists in repository") } else { os.MkdirAll(ndir, 0777) msg(" | Copying to " + npath) _, err := CopyFile(npath, m.Cached) if err != nil { msg(" ! " + err.Error()) continue } } if _, ok := indexes[config.Index][reponame]; !ok { msg(" | Creating index: " + reponame) indexes[config.Index][reponame] = &gopan.Source{ Name: reponame, URL: "/authors/id", Authors: make(map[string]*gopan.Author), } mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author) } if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok { msg(" | Creating author: " + auth) author := &gopan.Author{ Source: indexes[config.Index][reponame], Name: auth, Packages: make(map[string]*gopan.Package), URL: "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/", } indexes[config.Index][reponame].Authors[auth] = author if _, ok := mapped[reponame]; !ok { mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author) } // author name if _, ok := mapped[reponame][author.Name[:1]]; !ok { mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author) } if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok { mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author) } mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author // wildcards if _, ok := mapped[reponame]["*"]; !ok { mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author) } if _, ok := mapped[reponame]["*"]["**"]; !ok { mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author) } mapped[reponame]["*"]["**"][author.Name] = author // combos if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok { mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author) } if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok { mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author) } mapped[reponame][author.Name[:1]]["**"][author.Name] = author mapped[reponame]["*"][author.Name[:2]][author.Name] = author } if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok { msg(" | Creating module: " + fn) indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{ Author: indexes[config.Index][reponame].Authors[auth], Name: fn, URL: indexes[config.Index][reponame].Authors[auth].URL + fn, Provides: make(map[string]*gopan.PerlPackage), } msg(" | Getting list of packages") modnm := strings.TrimSuffix(fn, ".tar.gz") pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn] if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil { msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error()) } //pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn] msg(" | Adding packages to index") if _, ok := idxpackages[reponame]; !ok { idxpackages[reponame] = make(map[string]*PkgSpace) } filemap[pkg.AuthorURL()] = reponame for _, prov := range pkg.Provides { parts := strings.Split(prov.Name, "::") if _, ok := packages[parts[0]]; !ok { packages[parts[0]] = &PkgSpace{ Namespace: parts[0], Packages: make([]*gopan.PerlPackage, 0), Children: make(map[string]*PkgSpace), Parent: nil, Versions: make(map[float64]*gopan.PerlPackage), } } if _, ok := idxpackages[reponame][parts[0]]; !ok { idxpackages[reponame][parts[0]] = &PkgSpace{ Namespace: parts[0], Packages: make([]*gopan.PerlPackage, 0), Children: make(map[string]*PkgSpace), Parent: nil, Versions: make(map[float64]*gopan.PerlPackage), } } if len(parts) == 1 { packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov) packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov) idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov } else { packages[parts[0]].Populate(parts[1:], prov) idxpackages[reponame][parts[0]].Populate(parts[1:], prov) } } msg(" | Writing to index file") gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn]) } msg(" | Imported module") } nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes) // TODO should probably be in the index - needs to udpate when index changes summary = &Summary{nsrc, nauth, npkg, nprov} msg(":DONE") job.Complete = true }
func main() { configure() args := flag.Args() if len(args) > 0 && args[0] == "init" { log.Info("Initialising SmartPAN") log.Info("=> Installing Perl dependencies") // FIXME most of this is repeated from getpan/main.go cfg := getpan.DefaultConfig() cfg.CacheDir = config.CacheDir for _, source := range cfg.Sources { if err := source.Load(); err != nil { log.Error("Error loading sources: %s", err) os.Exit(1) return } } deps := &getpan.DependencyList{ Dependencies: make([]*getpan.Dependency, 0), } d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "") d2, _ := getpan.DependencyFromString("JSON::XS", "") deps.AddDependency(d1) deps.AddDependency(d2) if err := deps.Resolve(); err != nil { log.Error("Error resolving dependencies: %s", err) os.Exit(1) return } _, err := deps.Install() if err != nil { log.Error("Error installing dependencies: %s", err) os.Exit(2) return } log.Info(" - Installed %d modules", deps.UniqueInstalled()) log.Info("SmartPAN initialisation complete") return } if config.TestDeps { perldeps := gopan.TestPerlDeps() perldeps.Dump() if !perldeps.Ok { log.Error("Required perl dependencies are missing") os.Exit(1) return } } if len(args) > 0 && args[0] == "import" { if len(args) < 4 { log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX") return } fname := args[1] log.Info("Importing module from %s", fname) log.Info("Author ID: %s", args[2]) log.Info("Index : %s", args[3]) extraParams := map[string]string{ "importinto": args[3], "authorid": args[2], "newindex": "", "cpanmirror": "", "importurl": "", "fromdir": "", } if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") { log.Info("URL: %s", fname) extraParams["importurl"] = fname request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams) if err != nil { log.Error("Create request error: %s", err.Error()) return } client := &nethttp.Client{} resp, err := client.Do(request) if err != nil { log.Error("Error connecting to host: %s", err.Error()) return } else { // TODO stream this body := &bytes.Buffer{} _, err := body.ReadFrom(resp.Body) if err != nil { log.Error("Error reading response: %s", err.Error()) return } resp.Body.Close() //log.Info("%d", resp.StatusCode) //log.Info("%s", resp.Header) log.Info("%s", body.String()) } } else { fname = strings.TrimPrefix(fname, "file://") log.Info("File: %s", fname) if _, err := os.Stat(fname); err != nil { log.Error("File not found: %s", err.Error()) return } request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname) if err != nil { log.Error("Create upload error: %s", err.Error()) return } client := &nethttp.Client{} resp, err := client.Do(request) if err != nil { log.Error("Error connecting to host: %s", err.Error()) return } else { // TODO stream this body := &bytes.Buffer{} _, err := body.ReadFrom(resp.Body) if err != nil { log.Error("Error reading response: %s", err.Error()) return } resp.Body.Close() //log.Info("%d", resp.StatusCode) //log.Info("%s", resp.Header) log.Info("%s", body.String()) } } return } config.CurrentRelease = CurrentRelease var wg sync.WaitGroup load_index = func(index string, file string) { indexes[index] = gopan.LoadIndex(file) } wg.Add(1) go func() { defer wg.Done() indexes = make(map[string]map[string]*gopan.Source) // Load CPAN index if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil { config.HasCPANIndex = true config.CPANIndexDate = fi.ModTime().String() config.CPANStatus = "Loading" wg.Add(1) go func() { defer wg.Done() load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex) config.CPANStatus = "Loaded" }() } // Load BackPAN index if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil { config.HasBackPANIndex = true config.BackPANIndexDate = fi.ModTime().String() config.BackPANStatus = "Loading" wg.Add(1) go func() { defer wg.Done() load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex) config.BackPANStatus = "Loaded" }() } // Load our secondary indexes for _, idx := range config.Indexes { wg.Add(1) go func() { defer wg.Done() load_index(idx, config.CacheDir+"/"+idx) }() } // Load our primary index (this is the only index written back to) wg.Add(1) go func() { defer wg.Done() load_index(config.Index, config.CacheDir+"/"+config.Index) }() }() update_indexes = func() { wg.Wait() wg.Add(1) go func() { wg.Wait() config.ImportAvailable = true nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes) // TODO should probably be in the index - needs to udpate when index changes summary = &Summary{nsrc, nauth, npkg, nprov} // Do this now so changing the level doesn't interfere with index load log.Logger().SetLevel(log.Stol(config.LogLevel)) }() defer wg.Done() // Create in-memory indexes for UI/search etc for fname, _ := range indexes { for idn, idx := range indexes[fname] { mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author) for _, auth := range idx.Authors { // author name if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok { mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author) } if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok { mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author) } mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth // wildcards if _, ok := mapped[idx.Name]["*"]; !ok { mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author) } if _, ok := mapped[idx.Name]["*"]["**"]; !ok { mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author) } mapped[idx.Name]["*"]["**"][auth.Name] = auth // combos if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok { mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author) } if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok { mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author) } mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth for _, pkg := range auth.Packages { filemap[pkg.AuthorURL()] = idn for _, prov := range pkg.Provides { parts := strings.Split(prov.Name, "::") log.Trace("PACKAGE: %s", prov.Name) if _, ok := packages[parts[0]]; !ok { packages[parts[0]] = &PkgSpace{ Namespace: parts[0], Packages: make([]*gopan.PerlPackage, 0), Children: make(map[string]*PkgSpace), Parent: nil, Versions: make(map[float64]*gopan.PerlPackage), } } if _, ok := idxpackages[idx.Name]; !ok { idxpackages[idx.Name] = make(map[string]*PkgSpace) } if _, ok := idxpackages[idx.Name][parts[0]]; !ok { idxpackages[idx.Name][parts[0]] = &PkgSpace{ Namespace: parts[0], Packages: make([]*gopan.PerlPackage, 0), Children: make(map[string]*PkgSpace), Parent: nil, Versions: make(map[float64]*gopan.PerlPackage), } } if len(parts) == 1 { packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov) packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov) idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name) } else { packages[parts[0]].Populate(parts[1:], prov) idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov) } } } } } } } go update_indexes() // Get latest SmartPAN version go func() { res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases") if err != nil { log.Error("Error getting latest version: %s", err.Error()) return } defer res.Body.Close() b, err := ioutil.ReadAll(res.Body) if err != nil { log.Error("Error reading stream: %s", err.Error()) return } var r Releases if err = json.Unmarshal(b, &r); err != nil { log.Error("Error unmarshalling JSON: %s", err.Error()) return } log.Info("Current release: %s", config.CurrentRelease) rel := strings.TrimPrefix(r[0].TagName, "v") log.Info("Latest release: %s", rel) config.LatestRelease = rel config.UpdateURL = r[0].URL if config.CurrentRelease < rel { config.CanUpdate = true log.Info("Your version of SmartPAN can be updated.") } }() // Create our Gotcha application var app = gotcha.Create(Asset) app.Config.Listen = config.Bind summary = &Summary{0, 0, 0, 0} app.On(events.BeforeHandler, func(session *http.Session, next func()) { session.Stash["summary"] = summary session.Stash["config"] = config next() }) // Get the router r := app.Router // Create some routes r.Get("/", search) r.Post("/", search) r.Get("/help", help) r.Get("/settings", settings) r.Get("/browse", browse) r.Get("/import", import1) r.Post("/import", import1) r.Get("/import/(?P<jobid>[^/]+)", import2) r.Get("/import/(?P<jobid>[^/]+)/stream", importstream) r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex) // Serve static content (but really use a CDN) r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}")) r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}")) // JSON endpoints r.Get("/where/(?P<module>[^/]+)/?", where) r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where) // Put these last so they only match /{repo} if nothing else matches r.Get("/(?P<repo>[^/]+)/?", browse) r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse) r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex) r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download) r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file) r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse) // Start our application app.Start() <-make(chan int) }
func delete_file(session *http.Session) { session.Stash["Title"] = "Delete file" html, _ := session.RenderTemplate("delete.html") repo := session.Stash["repo"].(string) file := session.Stash["file"].(string) auth1 := session.Stash["auth1"].(string) auth2 := session.Stash["auth2"].(string) auth3 := session.Stash["auth3"].(string) fname := config.CacheDir + "/" + repo + "/" + auth1 + "/" + auth2 + "/" + auth3 + "/" + file if _, err := os.Stat(fname); err != nil { session.RenderNotFound() return } // Remove file from indexes for f, _ := range indexes { if _, ok := indexes[f][repo]; !ok { continue } if _, ok := indexes[f][repo].Authors[auth3]; !ok { continue } if _, ok := indexes[f][repo].Authors[auth3].Packages[file]; !ok { continue } log.Debug("Removing from index: %s", repo) pkg := indexes[f][repo].Authors[auth3].Packages[file] delete(indexes[f][repo].Authors[auth3].Packages, file) if len(indexes[f][repo].Authors[auth3].Packages) == 0 { log.Debug("Removing author") delete(indexes[f][repo].Authors, auth3) } if len(indexes[f][repo].Authors) == 0 { log.Debug("Removing index") delete(indexes[f], repo) } if auth, ok := mapped[repo][auth1][auth2][auth3]; ok { if len(auth.Packages) == 0 { log.Debug("Removing author from mapped index") delete(mapped[repo][auth1][auth2], auth3) delete(mapped[repo]["*"][auth2], auth3) delete(mapped[repo][auth1]["**"], auth3) delete(mapped[repo]["*"]["**"], auth3) } if len(mapped[repo][auth1][auth2]) == 0 { log.Debug("Removing auth1/auth2 from mapped index") delete(mapped[repo][auth1], auth2) } if len(mapped[repo]["*"][auth2]) == 0 { log.Debug("Removing author **/auth2 from mapped index") delete(mapped[repo][auth1], auth2) } if len(mapped[repo][auth1]["**"]) == 0 { log.Debug("Removing author auth1/** from mapped index") delete(mapped[repo][auth1], auth2) } if len(mapped[repo]["*"]["**"]) == 0 { log.Debug("Removing author */** from mapped index") delete(mapped[repo][auth1], auth2) } if len(mapped[repo]["*"]) == 0 { log.Debug("Removing author * from mapped index") delete(mapped[repo][auth1], auth2) } if len(mapped[repo][auth1]) == 1 { log.Debug("Removing author auth1 from mapped index") delete(mapped[repo], auth1) } if len(mapped[repo]) == 1 { log.Debug("Removing repo from mapped index") delete(mapped, repo) } } for _, prov := range pkg.Provides { parts := strings.Split(prov.Name, "::") // TODO remove from packages/idxpackages if ctx, ok := packages[parts[0]]; ok { parts = parts[1:] for len(parts) > 0 { if c, ok := ctx.Children[parts[0]]; ok { ctx = c } else { log.Debug("Package not found in packages: %s", parts) break } parts = parts[1:] } if len(parts) == 0 { for ctx != nil { for pi, p := range ctx.Packages { if p.Package == pkg { log.Debug("Removing package from packages: %s", ctx.FullName()) ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...) break } } if len(ctx.Packages) == 0 { log.Debug("Removing PkgSpace from packages: %s", ctx.FullName()) if ctx.Parent == nil { delete(packages, ctx.Namespace) } else { delete(ctx.Parent.Children, ctx.Namespace) } } ctx = ctx.Parent } } } parts = strings.Split(prov.Name, "::") if _, ok := idxpackages[repo]; ok { if ctx, ok := idxpackages[repo][parts[0]]; ok { parts = parts[1:] for len(parts) > 0 { if c, ok := ctx.Children[parts[0]]; ok { ctx = c } else { log.Debug("PkgSpace not found in idxpackages") break } parts = parts[1:] } if len(parts) == 0 { for ctx != nil { for pi, p := range ctx.Packages { if p.Package == pkg { log.Debug("Removing package from idxpackages") ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...) break } } if len(ctx.Packages) == 0 { log.Debug("Removing PkgSpace from idxpackages: %s", ctx.FullName()) if ctx.Parent == nil { delete(idxpackages, ctx.Namespace) } else { delete(ctx.Parent.Children, ctx.Namespace) } } ctx = ctx.Parent } } } } } if _, ok := filemap[auth1+"/"+auth2+"/"+auth3+"/"+file]; ok { log.Debug("Removing file from filemap") // FIXME filemap should be map[string][]string, so we know if // the file exists in multiple indexes delete(filemap, auth1+"/"+auth2+"/"+auth3+"/"+file) } // write remove to index gopan.RemoveModule(config.CacheDir+"/"+config.Index, pkg.Author.Source, pkg.Author, pkg) } log.Debug("Removing file from gopancache: %s", fname) // TODO move file deletion to shared gopan package if err := os.Remove(fname); err != nil { log.Error("Error removing file: %s", err) } // TODO maybe clean up author tree (is this smartpans responsibility?) nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes) // TODO should probably be in the index - needs to udpate when index changes summary = &Summary{nsrc, nauth, npkg, nprov} session.Stash["Page"] = "Browse" session.Stash["Content"] = template.HTML(html) session.Render("layout.html") }