func main() {
configure()
indexes = make(map[string]map[string]*gopan.Source)
indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)
log.Logger().SetLevel(log.Stol(config.LogLevel))
log.Info("Using log level: %s", config.LogLevel)
// FIXME inefficient
_, _, tpkg, _ := gopan.CountIndex(indexes)
npkg := 0
nmod := 0
var pc = func() float64 {
return float64(nmod) / float64(tpkg) * 100
}
log.Info("Writing packages index file")
out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
if err != nil {
log.Error("Error creating packages index: %s", err.Error())
return
}
for fname, _ := range indexes {
log.Debug("File: %s", fname)
for _, idx := range indexes[fname] {
log.Debug("Index: %s", idx)
out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
for _, auth := range idx.Authors {
log.Debug("Author %s", auth)
out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
for _, pkg := range auth.Packages {
out.Write([]byte(" " + pkg.Name + " => " + pkg.URL + "\n"))
log.Debug("Package: %s", pkg)
if !config.Flatten {
if len(pkg.Provides) == 0 {
// TODO better handling of filenames
modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")
tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name
if _, err := os.Stat(tgzpath); err != nil {
log.Error("File not found: %s", tgzpath)
return
}
extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name
log.Trace("=> tgzpath: %s", tgzpath)
log.Trace(" > extpath: %s", extpath)
log.Trace(" > dirpath: %s", dirpath)
// Only index packages if they don't already exist
if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
log.Error("Error retrieving package list: %s", err)
continue
}
}
npkg += len(pkg.Provides)
nmod += 1
for p, pk := range pkg.Provides {
out.Write([]byte(" " + p + " (" + pk.Version + "): " + pk.File + "\n"))
}
if nmod > 0 && nmod%100 == 0 {
log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
}
}
}
}
}
}
out.Close()
log.Info("Found %d packages from %d modules", npkg, nmod)
}
func main() {
configure()
log.Logger().SetLevel(log.Stol(config.LogLevel))
log.Info("Using log level: %s", config.LogLevel)
indexes = make(map[string]map[string]*gopan.Source)
if !config.NoCache {
indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
}
if config.NoCache || config.Update {
for _, s := range config.Sources {
b := strings.SplitN(s, "=", 2)
if len(b) < 2 {
log.Error("Expected Name=URL pair, got: %s", s)
return
}
if idx, ok := indexes[config.Index][b[0]]; ok {
log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1])
idx.URL = b[1]
} else {
indexes[config.Index][b[0]] = &gopan.Source{
Name: b[0],
URL: b[1],
Authors: make(map[string]*gopan.Author, 0),
}
}
}
if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN {
log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN")
config.CPAN = true
config.BackPAN = true
}
if config.CPAN {
if _, ok := indexes[config.Index]["CPAN"]; !ok {
log.Debug("Adding CPAN index")
indexes[config.Index]["CPAN"] = gopan.CPANSource()
} else {
log.Debug("CPAN index already exists")
}
}
if config.BackPAN {
if _, ok := indexes[config.Index]["BackPAN"]; !ok {
log.Debug("Adding BackPAN index")
indexes[config.Index]["BackPAN"] = gopan.BackPANSource()
} else {
log.Debug("BackPAN index already exists")
}
}
log.Info("Using sources:")
for fname, _ := range indexes {
log.Info("From %s", fname)
for _, source := range indexes[fname] {
log.Info("=> %s", source.String())
}
}
newAuthors := getAuthors()
newPackages := getPackages()
os.MkdirAll(config.CacheDir, 0777)
if !config.NoCache {
gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index])
}
if config.Update {
log.Info("Found %d new packages by %d new authors", newAuthors, newPackages)
}
}
nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes)
log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc)
if !config.NoMirror {
mirrorPan()
}
}
func mirrorPan() {
log.Info("Mirroring *PAN")
// FIXME inefficient
_, _, npkg, _ := gopan.CountIndex(indexes)
mirrored := 0
var pc = func() int {
return mirrored / npkg * 100
}
for fname, _ := range indexes {
log.Debug("File: %s", fname)
for _, source := range indexes[fname] {
log.Debug("Index: %s", source)
wg.Add(1)
go func(source *gopan.Source) {
defer wg.Done()
for _, author := range source.Authors {
log.Debug("=> %s", author)
wg.Add(1)
go func(author *gopan.Author) {
cachedir := config.CacheDir + "/" + source.Name + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
os.MkdirAll(cachedir, 0777)
defer wg.Done()
for _, pkg := range author.Packages {
wg.Add(1)
go func(pkg *gopan.Package) {
defer wg.Done()
cache := cachedir + pkg.Name
log.Trace(" - Caching to: %s", cache)
if _, err := os.Stat(cache); err == nil {
log.Debug("%d%% |> %s", pc(), pkg)
log.Trace(" - Already exists in cache")
mirrored++
return
}
sem <- 1
mirrored++
log.Debug("%d%% => %s", pc(), pkg)
url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" + pkg.Name
log.Trace(" - From URL: %s", url)
out, err := os.Create(cache)
defer out.Close()
if err != nil {
log.Error("CREATE - %s", err.Error())
<-sem
return
}
resp, err := http.Get(url)
if err != nil {
log.Error("HTTP GET - %s", err.Error())
<-sem
return
}
_, err = io.Copy(out, resp.Body)
if err != nil {
log.Error("IO COPY - %s", err.Error())
}
<-sem
}(pkg)
}
}(author)
}
}(source)
}
}
wg.Wait()
log.Info("Finished mirroring *PAN")
}
func do_import(session *http.Session, job *ImportJob) {
log.Info("Running import job %s", job.Id)
reponame := job.Form.ImportInto
if reponame == "new_index" {
reponame = job.Form.NewIndex
}
msg := func(m string) {
if m != ":DONE" {
job.History = append(job.History, m)
log.Info(m)
}
for _, w := range job.Watchers {
w(m)
}
}
mods := make([]*getpan.Module, 0)
// TODO cpanm mirror when using getpan_import
if len(job.Form.Cpanfile) > 0 {
msg("Parsing cpanfile input")
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
}
if len(job.Form.ImportURL) > 0 {
msg("Importing from URL: " + job.Form.ImportURL)
// TODO support cpanfile urls
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.ImportURL)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
if _, err := os.Stat(nfile); err != nil {
os.MkdirAll(npath, 0777)
out, err := os.Create(nfile)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
url := job.Form.ImportURL
log.Trace("Downloading: %s", url)
resp, err := nethttp.Get(url)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
_, err = io.Copy(out, resp.Body)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
out.Close()
resp.Body.Close()
} else {
log.Trace("File already exists in cache: %s", nfile)
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if len(job.Form.FromDir) > 0 {
msg("Importing from local directory: " + job.Form.FromDir)
// TODO support cpanfile paths
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.FromDir)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
_, err := CopyFile(nfile, job.Form.FromDir)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if f, fh, err := session.Request.File("fromfile"); err == nil {
fn := fh.Filename
msg("Importing from uploaded module/cpanfile: " + fn)
if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
msg("Only cpanfile and *.tar.gz files are supported")
msg(":DONE")
job.Complete = true
return
}
if fn == "cpanfile" {
msg("Importing cpanfile")
b, _ := ioutil.ReadAll(f)
f.Close()
job.Form.Cpanfile = string(b)
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
} else {
msg("Importing .tar.gz")
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn = filepath.Split(fn)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
os.MkdirAll(npath, 0777)
_, err := CopyToFile(nfile, f)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
} else {
// there is no file... so no error
//msg("Error importing file upload: " + err.Error())
}
if len(mods) == 0 {
msg("Nothing to do")
msg(":DONE")
job.Complete = true
return
}
msg("Adding modules to GoPAN index")
for _, m := range mods {
msg("=> " + m.Name + " (" + m.Cached + ")")
dn, fn := filepath.Split(m.Cached)
dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
auth := dnb[len(dnb)-1]
ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
npath := ndir + "/" + fn
if _, err := os.Stat(npath); err == nil {
msg(" | Already exists in repository")
} else {
os.MkdirAll(ndir, 0777)
msg(" | Copying to " + npath)
_, err := CopyFile(npath, m.Cached)
if err != nil {
msg(" ! " + err.Error())
continue
}
}
if _, ok := indexes[config.Index][reponame]; !ok {
msg(" | Creating index: " + reponame)
indexes[config.Index][reponame] = &gopan.Source{
Name: reponame,
URL: "/authors/id",
Authors: make(map[string]*gopan.Author),
}
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
msg(" | Creating author: " + auth)
author := &gopan.Author{
Source: indexes[config.Index][reponame],
Name: auth,
Packages: make(map[string]*gopan.Package),
URL: "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
}
indexes[config.Index][reponame].Authors[auth] = author
if _, ok := mapped[reponame]; !ok {
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
// author name
if _, ok := mapped[reponame][author.Name[:1]]; !ok {
mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author
// wildcards
if _, ok := mapped[reponame]["*"]; !ok {
mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"]["**"]; !ok {
mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
}
mapped[reponame]["*"]["**"][author.Name] = author
// combos
if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]]["**"][author.Name] = author
mapped[reponame]["*"][author.Name[:2]][author.Name] = author
}
if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
msg(" | Creating module: " + fn)
indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
Author: indexes[config.Index][reponame].Authors[auth],
Name: fn,
URL: indexes[config.Index][reponame].Authors[auth].URL + fn,
Provides: make(map[string]*gopan.PerlPackage),
}
msg(" | Getting list of packages")
modnm := strings.TrimSuffix(fn, ".tar.gz")
pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
}
//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
msg(" | Adding packages to index")
if _, ok := idxpackages[reponame]; !ok {
idxpackages[reponame] = make(map[string]*PkgSpace)
}
filemap[pkg.AuthorURL()] = reponame
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
if _, ok := packages[parts[0]]; !ok {
packages[parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if _, ok := idxpackages[reponame][parts[0]]; !ok {
idxpackages[reponame][parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if len(parts) == 1 {
packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
} else {
packages[parts[0]].Populate(parts[1:], prov)
idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
}
}
msg(" | Writing to index file")
gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
}
msg(" | Imported module")
}
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
msg(":DONE")
job.Complete = true
}
func main() {
configure()
args := flag.Args()
if len(args) > 0 && args[0] == "init" {
log.Info("Initialising SmartPAN")
log.Info("=> Installing Perl dependencies")
// FIXME most of this is repeated from getpan/main.go
cfg := getpan.DefaultConfig()
cfg.CacheDir = config.CacheDir
for _, source := range cfg.Sources {
if err := source.Load(); err != nil {
log.Error("Error loading sources: %s", err)
os.Exit(1)
return
}
}
deps := &getpan.DependencyList{
Dependencies: make([]*getpan.Dependency, 0),
}
d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "")
d2, _ := getpan.DependencyFromString("JSON::XS", "")
deps.AddDependency(d1)
deps.AddDependency(d2)
if err := deps.Resolve(); err != nil {
log.Error("Error resolving dependencies: %s", err)
os.Exit(1)
return
}
_, err := deps.Install()
if err != nil {
log.Error("Error installing dependencies: %s", err)
os.Exit(2)
return
}
log.Info(" - Installed %d modules", deps.UniqueInstalled())
log.Info("SmartPAN initialisation complete")
return
}
if config.TestDeps {
perldeps := gopan.TestPerlDeps()
perldeps.Dump()
if !perldeps.Ok {
log.Error("Required perl dependencies are missing")
os.Exit(1)
return
}
}
if len(args) > 0 && args[0] == "import" {
if len(args) < 4 {
log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX")
return
}
fname := args[1]
log.Info("Importing module from %s", fname)
log.Info("Author ID: %s", args[2])
log.Info("Index : %s", args[3])
extraParams := map[string]string{
"importinto": args[3],
"authorid": args[2],
"newindex": "",
"cpanmirror": "",
"importurl": "",
"fromdir": "",
}
if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
log.Info("URL: %s", fname)
extraParams["importurl"] = fname
request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams)
if err != nil {
log.Error("Create request error: %s", err.Error())
return
}
client := &nethttp.Client{}
resp, err := client.Do(request)
if err != nil {
log.Error("Error connecting to host: %s", err.Error())
return
} else {
// TODO stream this
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
log.Error("Error reading response: %s", err.Error())
return
}
resp.Body.Close()
//log.Info("%d", resp.StatusCode)
//log.Info("%s", resp.Header)
log.Info("%s", body.String())
}
} else {
fname = strings.TrimPrefix(fname, "file://")
log.Info("File: %s", fname)
if _, err := os.Stat(fname); err != nil {
log.Error("File not found: %s", err.Error())
return
}
request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname)
if err != nil {
log.Error("Create upload error: %s", err.Error())
return
}
client := &nethttp.Client{}
resp, err := client.Do(request)
if err != nil {
log.Error("Error connecting to host: %s", err.Error())
return
} else {
// TODO stream this
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
log.Error("Error reading response: %s", err.Error())
return
}
resp.Body.Close()
//log.Info("%d", resp.StatusCode)
//log.Info("%s", resp.Header)
log.Info("%s", body.String())
}
}
return
}
config.CurrentRelease = CurrentRelease
var wg sync.WaitGroup
load_index = func(index string, file string) {
indexes[index] = gopan.LoadIndex(file)
}
wg.Add(1)
go func() {
defer wg.Done()
indexes = make(map[string]map[string]*gopan.Source)
// Load CPAN index
if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil {
config.HasCPANIndex = true
config.CPANIndexDate = fi.ModTime().String()
config.CPANStatus = "Loading"
wg.Add(1)
go func() {
defer wg.Done()
load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex)
config.CPANStatus = "Loaded"
}()
}
// Load BackPAN index
if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil {
config.HasBackPANIndex = true
config.BackPANIndexDate = fi.ModTime().String()
config.BackPANStatus = "Loading"
wg.Add(1)
go func() {
defer wg.Done()
load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex)
config.BackPANStatus = "Loaded"
}()
}
// Load our secondary indexes
for _, idx := range config.Indexes {
wg.Add(1)
go func() {
defer wg.Done()
load_index(idx, config.CacheDir+"/"+idx)
}()
}
// Load our primary index (this is the only index written back to)
wg.Add(1)
go func() {
defer wg.Done()
load_index(config.Index, config.CacheDir+"/"+config.Index)
}()
}()
update_indexes = func() {
wg.Wait()
wg.Add(1)
go func() {
wg.Wait()
config.ImportAvailable = true
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
// Do this now so changing the level doesn't interfere with index load
log.Logger().SetLevel(log.Stol(config.LogLevel))
}()
defer wg.Done()
// Create in-memory indexes for UI/search etc
for fname, _ := range indexes {
for idn, idx := range indexes[fname] {
mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
for _, auth := range idx.Authors {
// author name
if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth
// wildcards
if _, ok := mapped[idx.Name]["*"]; !ok {
mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
}
mapped[idx.Name]["*"]["**"][auth.Name] = auth
// combos
if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
}
if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth
for _, pkg := range auth.Packages {
filemap[pkg.AuthorURL()] = idn
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
log.Trace("PACKAGE: %s", prov.Name)
if _, ok := packages[parts[0]]; !ok {
packages[parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if _, ok := idxpackages[idx.Name]; !ok {
idxpackages[idx.Name] = make(map[string]*PkgSpace)
}
if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
idxpackages[idx.Name][parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if len(parts) == 1 {
packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
} else {
packages[parts[0]].Populate(parts[1:], prov)
idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
}
}
}
}
}
}
}
go update_indexes()
// Get latest SmartPAN version
go func() {
res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases")
if err != nil {
log.Error("Error getting latest version: %s", err.Error())
return
}
defer res.Body.Close()
b, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Error("Error reading stream: %s", err.Error())
return
}
var r Releases
if err = json.Unmarshal(b, &r); err != nil {
log.Error("Error unmarshalling JSON: %s", err.Error())
return
}
log.Info("Current release: %s", config.CurrentRelease)
rel := strings.TrimPrefix(r[0].TagName, "v")
log.Info("Latest release: %s", rel)
config.LatestRelease = rel
config.UpdateURL = r[0].URL
if config.CurrentRelease < rel {
config.CanUpdate = true
log.Info("Your version of SmartPAN can be updated.")
}
}()
// Create our Gotcha application
var app = gotcha.Create(Asset)
app.Config.Listen = config.Bind
summary = &Summary{0, 0, 0, 0}
app.On(events.BeforeHandler, func(session *http.Session, next func()) {
session.Stash["summary"] = summary
session.Stash["config"] = config
next()
})
// Get the router
r := app.Router
// Create some routes
r.Get("/", search)
r.Post("/", search)
r.Get("/help", help)
r.Get("/settings", settings)
r.Get("/browse", browse)
r.Get("/import", import1)
r.Post("/import", import1)
r.Get("/import/(?P<jobid>[^/]+)", import2)
r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)
r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex)
// Serve static content (but really use a CDN)
r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))
// JSON endpoints
r.Get("/where/(?P<module>[^/]+)/?", where)
r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)
// Put these last so they only match /{repo} if nothing else matches
r.Get("/(?P<repo>[^/]+)/?", browse)
r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)
// Start our application
app.Start()
<-make(chan int)
}
func delete_file(session *http.Session) {
session.Stash["Title"] = "Delete file"
html, _ := session.RenderTemplate("delete.html")
repo := session.Stash["repo"].(string)
file := session.Stash["file"].(string)
auth1 := session.Stash["auth1"].(string)
auth2 := session.Stash["auth2"].(string)
auth3 := session.Stash["auth3"].(string)
fname := config.CacheDir + "/" + repo + "/" + auth1 + "/" + auth2 + "/" + auth3 + "/" + file
if _, err := os.Stat(fname); err != nil {
session.RenderNotFound()
return
}
// Remove file from indexes
for f, _ := range indexes {
if _, ok := indexes[f][repo]; !ok {
continue
}
if _, ok := indexes[f][repo].Authors[auth3]; !ok {
continue
}
if _, ok := indexes[f][repo].Authors[auth3].Packages[file]; !ok {
continue
}
log.Debug("Removing from index: %s", repo)
pkg := indexes[f][repo].Authors[auth3].Packages[file]
delete(indexes[f][repo].Authors[auth3].Packages, file)
if len(indexes[f][repo].Authors[auth3].Packages) == 0 {
log.Debug("Removing author")
delete(indexes[f][repo].Authors, auth3)
}
if len(indexes[f][repo].Authors) == 0 {
log.Debug("Removing index")
delete(indexes[f], repo)
}
if auth, ok := mapped[repo][auth1][auth2][auth3]; ok {
if len(auth.Packages) == 0 {
log.Debug("Removing author from mapped index")
delete(mapped[repo][auth1][auth2], auth3)
delete(mapped[repo]["*"][auth2], auth3)
delete(mapped[repo][auth1]["**"], auth3)
delete(mapped[repo]["*"]["**"], auth3)
}
if len(mapped[repo][auth1][auth2]) == 0 {
log.Debug("Removing auth1/auth2 from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"][auth2]) == 0 {
log.Debug("Removing author **/auth2 from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo][auth1]["**"]) == 0 {
log.Debug("Removing author auth1/** from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"]["**"]) == 0 {
log.Debug("Removing author */** from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"]) == 0 {
log.Debug("Removing author * from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo][auth1]) == 1 {
log.Debug("Removing author auth1 from mapped index")
delete(mapped[repo], auth1)
}
if len(mapped[repo]) == 1 {
log.Debug("Removing repo from mapped index")
delete(mapped, repo)
}
}
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
// TODO remove from packages/idxpackages
if ctx, ok := packages[parts[0]]; ok {
parts = parts[1:]
for len(parts) > 0 {
if c, ok := ctx.Children[parts[0]]; ok {
ctx = c
} else {
log.Debug("Package not found in packages: %s", parts)
break
}
parts = parts[1:]
}
if len(parts) == 0 {
for ctx != nil {
for pi, p := range ctx.Packages {
if p.Package == pkg {
log.Debug("Removing package from packages: %s", ctx.FullName())
ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
break
}
}
if len(ctx.Packages) == 0 {
log.Debug("Removing PkgSpace from packages: %s", ctx.FullName())
if ctx.Parent == nil {
delete(packages, ctx.Namespace)
} else {
delete(ctx.Parent.Children, ctx.Namespace)
}
}
ctx = ctx.Parent
}
}
}
parts = strings.Split(prov.Name, "::")
if _, ok := idxpackages[repo]; ok {
if ctx, ok := idxpackages[repo][parts[0]]; ok {
parts = parts[1:]
for len(parts) > 0 {
if c, ok := ctx.Children[parts[0]]; ok {
ctx = c
} else {
log.Debug("PkgSpace not found in idxpackages")
break
}
parts = parts[1:]
}
if len(parts) == 0 {
for ctx != nil {
for pi, p := range ctx.Packages {
if p.Package == pkg {
log.Debug("Removing package from idxpackages")
ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
break
}
}
if len(ctx.Packages) == 0 {
log.Debug("Removing PkgSpace from idxpackages: %s", ctx.FullName())
if ctx.Parent == nil {
delete(idxpackages, ctx.Namespace)
} else {
delete(ctx.Parent.Children, ctx.Namespace)
}
}
ctx = ctx.Parent
}
}
}
}
}
if _, ok := filemap[auth1+"/"+auth2+"/"+auth3+"/"+file]; ok {
log.Debug("Removing file from filemap")
// FIXME filemap should be map[string][]string, so we know if
// the file exists in multiple indexes
delete(filemap, auth1+"/"+auth2+"/"+auth3+"/"+file)
}
// write remove to index
gopan.RemoveModule(config.CacheDir+"/"+config.Index, pkg.Author.Source, pkg.Author, pkg)
}
log.Debug("Removing file from gopancache: %s", fname)
// TODO move file deletion to shared gopan package
if err := os.Remove(fname); err != nil {
log.Error("Error removing file: %s", err)
}
// TODO maybe clean up author tree (is this smartpans responsibility?)
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
session.Stash["Page"] = "Browse"
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}