func main() {
configure()
indexes = make(map[string]map[string]*gopan.Source)
indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)
log.Logger().SetLevel(log.Stol(config.LogLevel))
log.Info("Using log level: %s", config.LogLevel)
// FIXME inefficient
_, _, tpkg, _ := gopan.CountIndex(indexes)
npkg := 0
nmod := 0
var pc = func() float64 {
return float64(nmod) / float64(tpkg) * 100
}
log.Info("Writing packages index file")
out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
if err != nil {
log.Error("Error creating packages index: %s", err.Error())
return
}
for fname, _ := range indexes {
log.Debug("File: %s", fname)
for _, idx := range indexes[fname] {
log.Debug("Index: %s", idx)
out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
for _, auth := range idx.Authors {
log.Debug("Author %s", auth)
out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
for _, pkg := range auth.Packages {
out.Write([]byte(" " + pkg.Name + " => " + pkg.URL + "\n"))
log.Debug("Package: %s", pkg)
if !config.Flatten {
if len(pkg.Provides) == 0 {
// TODO better handling of filenames
modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")
tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name
if _, err := os.Stat(tgzpath); err != nil {
log.Error("File not found: %s", tgzpath)
return
}
extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name
log.Trace("=> tgzpath: %s", tgzpath)
log.Trace(" > extpath: %s", extpath)
log.Trace(" > dirpath: %s", dirpath)
// Only index packages if they don't already exist
if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
log.Error("Error retrieving package list: %s", err)
continue
}
}
npkg += len(pkg.Provides)
nmod += 1
for p, pk := range pkg.Provides {
out.Write([]byte(" " + p + " (" + pk.Version + "): " + pk.File + "\n"))
}
if nmod > 0 && nmod%100 == 0 {
log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
}
}
}
}
}
}
out.Close()
log.Info("Found %d packages from %d modules", npkg, nmod)
}
func do_import(session *http.Session, job *ImportJob) {
log.Info("Running import job %s", job.Id)
reponame := job.Form.ImportInto
if reponame == "new_index" {
reponame = job.Form.NewIndex
}
msg := func(m string) {
if m != ":DONE" {
job.History = append(job.History, m)
log.Info(m)
}
for _, w := range job.Watchers {
w(m)
}
}
mods := make([]*getpan.Module, 0)
// TODO cpanm mirror when using getpan_import
if len(job.Form.Cpanfile) > 0 {
msg("Parsing cpanfile input")
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
}
if len(job.Form.ImportURL) > 0 {
msg("Importing from URL: " + job.Form.ImportURL)
// TODO support cpanfile urls
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.ImportURL)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
if _, err := os.Stat(nfile); err != nil {
os.MkdirAll(npath, 0777)
out, err := os.Create(nfile)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
url := job.Form.ImportURL
log.Trace("Downloading: %s", url)
resp, err := nethttp.Get(url)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
_, err = io.Copy(out, resp.Body)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
out.Close()
resp.Body.Close()
} else {
log.Trace("File already exists in cache: %s", nfile)
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if len(job.Form.FromDir) > 0 {
msg("Importing from local directory: " + job.Form.FromDir)
// TODO support cpanfile paths
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.FromDir)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
_, err := CopyFile(nfile, job.Form.FromDir)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if f, fh, err := session.Request.File("fromfile"); err == nil {
fn := fh.Filename
msg("Importing from uploaded module/cpanfile: " + fn)
if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
msg("Only cpanfile and *.tar.gz files are supported")
msg(":DONE")
job.Complete = true
return
}
if fn == "cpanfile" {
msg("Importing cpanfile")
b, _ := ioutil.ReadAll(f)
f.Close()
job.Form.Cpanfile = string(b)
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
} else {
msg("Importing .tar.gz")
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn = filepath.Split(fn)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
os.MkdirAll(npath, 0777)
_, err := CopyToFile(nfile, f)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
} else {
// there is no file... so no error
//msg("Error importing file upload: " + err.Error())
}
if len(mods) == 0 {
msg("Nothing to do")
msg(":DONE")
job.Complete = true
return
}
msg("Adding modules to GoPAN index")
for _, m := range mods {
msg("=> " + m.Name + " (" + m.Cached + ")")
dn, fn := filepath.Split(m.Cached)
dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
auth := dnb[len(dnb)-1]
ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
npath := ndir + "/" + fn
if _, err := os.Stat(npath); err == nil {
msg(" | Already exists in repository")
} else {
os.MkdirAll(ndir, 0777)
msg(" | Copying to " + npath)
_, err := CopyFile(npath, m.Cached)
if err != nil {
msg(" ! " + err.Error())
continue
}
}
if _, ok := indexes[config.Index][reponame]; !ok {
msg(" | Creating index: " + reponame)
indexes[config.Index][reponame] = &gopan.Source{
Name: reponame,
URL: "/authors/id",
Authors: make(map[string]*gopan.Author),
}
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
msg(" | Creating author: " + auth)
author := &gopan.Author{
Source: indexes[config.Index][reponame],
Name: auth,
Packages: make(map[string]*gopan.Package),
URL: "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
}
indexes[config.Index][reponame].Authors[auth] = author
if _, ok := mapped[reponame]; !ok {
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
// author name
if _, ok := mapped[reponame][author.Name[:1]]; !ok {
mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author
// wildcards
if _, ok := mapped[reponame]["*"]; !ok {
mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"]["**"]; !ok {
mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
}
mapped[reponame]["*"]["**"][author.Name] = author
// combos
if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]]["**"][author.Name] = author
mapped[reponame]["*"][author.Name[:2]][author.Name] = author
}
if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
msg(" | Creating module: " + fn)
indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
Author: indexes[config.Index][reponame].Authors[auth],
Name: fn,
URL: indexes[config.Index][reponame].Authors[auth].URL + fn,
Provides: make(map[string]*gopan.PerlPackage),
}
msg(" | Getting list of packages")
modnm := strings.TrimSuffix(fn, ".tar.gz")
pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
}
//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
msg(" | Adding packages to index")
if _, ok := idxpackages[reponame]; !ok {
idxpackages[reponame] = make(map[string]*PkgSpace)
}
filemap[pkg.AuthorURL()] = reponame
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
if _, ok := packages[parts[0]]; !ok {
packages[parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if _, ok := idxpackages[reponame][parts[0]]; !ok {
idxpackages[reponame][parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if len(parts) == 1 {
packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
} else {
packages[parts[0]].Populate(parts[1:], prov)
idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
}
}
msg(" | Writing to index file")
gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
}
msg(" | Imported module")
}
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
msg(":DONE")
job.Complete = true
}