func getpan_import(job *ImportJob, msg func(string)) (*getpan.CPANFile, []*getpan.Module) {
cfg := getpan.DefaultConfig()
cfg.CacheDir = config.CacheDir
if len(job.Form.CPANMirror) > 0 {
msg("Adding CPAN mirror: " + job.Form.CPANMirror)
cfg.Sources = append(cfg.Sources, getpan.NewSource("CPAN", job.Form.CPANMirror+"/modules/02packages.details.txt.gz", job.Form.CPANMirror))
}
defer func(job *ImportJob) {
job.Complete = true
msg("GetPAN import complete")
}(job)
for _, source := range cfg.Sources {
if err := source.Load(); err != nil {
m := fmt.Sprintf("Error loading sources: %s", err)
msg(m)
return nil, nil
}
}
m := "Loaded sources"
msg(m)
deps, err := getpan.ParseCPANLines(strings.Split(job.Form.Cpanfile, "\n"))
if err != nil {
m := fmt.Sprintf("Error parsing cpanfile: %s", err)
msg(m)
return nil, nil
}
m = "Parsed cpanfile"
msg(m)
if err = deps.Resolve(); err != nil {
m := fmt.Sprintf("Error resolving dependencies: %s", err)
msg(m)
return deps, nil
}
m = "Resolved dependency tree:"
msg(m)
modules := make([]*getpan.Module, 0)
PrintCPANFile(job, deps, 0, msg, &modules)
return deps, modules
}
func do_import(session *http.Session, job *ImportJob) {
log.Info("Running import job %s", job.Id)
reponame := job.Form.ImportInto
if reponame == "new_index" {
reponame = job.Form.NewIndex
}
msg := func(m string) {
if m != ":DONE" {
job.History = append(job.History, m)
log.Info(m)
}
for _, w := range job.Watchers {
w(m)
}
}
mods := make([]*getpan.Module, 0)
// TODO cpanm mirror when using getpan_import
if len(job.Form.Cpanfile) > 0 {
msg("Parsing cpanfile input")
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
}
if len(job.Form.ImportURL) > 0 {
msg("Importing from URL: " + job.Form.ImportURL)
// TODO support cpanfile urls
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.ImportURL)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
if _, err := os.Stat(nfile); err != nil {
os.MkdirAll(npath, 0777)
out, err := os.Create(nfile)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
url := job.Form.ImportURL
log.Trace("Downloading: %s", url)
resp, err := nethttp.Get(url)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
_, err = io.Copy(out, resp.Body)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
out.Close()
resp.Body.Close()
} else {
log.Trace("File already exists in cache: %s", nfile)
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if len(job.Form.FromDir) > 0 {
msg("Importing from local directory: " + job.Form.FromDir)
// TODO support cpanfile paths
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.FromDir)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
_, err := CopyFile(nfile, job.Form.FromDir)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if f, fh, err := session.Request.File("fromfile"); err == nil {
fn := fh.Filename
msg("Importing from uploaded module/cpanfile: " + fn)
if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
msg("Only cpanfile and *.tar.gz files are supported")
msg(":DONE")
job.Complete = true
return
}
if fn == "cpanfile" {
msg("Importing cpanfile")
b, _ := ioutil.ReadAll(f)
f.Close()
job.Form.Cpanfile = string(b)
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
} else {
msg("Importing .tar.gz")
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn = filepath.Split(fn)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
os.MkdirAll(npath, 0777)
_, err := CopyToFile(nfile, f)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
} else {
// there is no file... so no error
//msg("Error importing file upload: " + err.Error())
}
if len(mods) == 0 {
msg("Nothing to do")
msg(":DONE")
job.Complete = true
return
}
msg("Adding modules to GoPAN index")
for _, m := range mods {
msg("=> " + m.Name + " (" + m.Cached + ")")
dn, fn := filepath.Split(m.Cached)
dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
auth := dnb[len(dnb)-1]
ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
npath := ndir + "/" + fn
if _, err := os.Stat(npath); err == nil {
msg(" | Already exists in repository")
} else {
os.MkdirAll(ndir, 0777)
msg(" | Copying to " + npath)
_, err := CopyFile(npath, m.Cached)
if err != nil {
msg(" ! " + err.Error())
continue
}
}
if _, ok := indexes[config.Index][reponame]; !ok {
msg(" | Creating index: " + reponame)
indexes[config.Index][reponame] = &gopan.Source{
Name: reponame,
URL: "/authors/id",
Authors: make(map[string]*gopan.Author),
}
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
msg(" | Creating author: " + auth)
author := &gopan.Author{
Source: indexes[config.Index][reponame],
Name: auth,
Packages: make(map[string]*gopan.Package),
URL: "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
}
indexes[config.Index][reponame].Authors[auth] = author
if _, ok := mapped[reponame]; !ok {
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
// author name
if _, ok := mapped[reponame][author.Name[:1]]; !ok {
mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author
// wildcards
if _, ok := mapped[reponame]["*"]; !ok {
mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"]["**"]; !ok {
mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
}
mapped[reponame]["*"]["**"][author.Name] = author
// combos
if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]]["**"][author.Name] = author
mapped[reponame]["*"][author.Name[:2]][author.Name] = author
}
if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
msg(" | Creating module: " + fn)
indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
Author: indexes[config.Index][reponame].Authors[auth],
Name: fn,
URL: indexes[config.Index][reponame].Authors[auth].URL + fn,
Provides: make(map[string]*gopan.PerlPackage),
}
msg(" | Getting list of packages")
modnm := strings.TrimSuffix(fn, ".tar.gz")
pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
}
//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
msg(" | Adding packages to index")
if _, ok := idxpackages[reponame]; !ok {
idxpackages[reponame] = make(map[string]*PkgSpace)
}
filemap[pkg.AuthorURL()] = reponame
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
if _, ok := packages[parts[0]]; !ok {
packages[parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if _, ok := idxpackages[reponame][parts[0]]; !ok {
idxpackages[reponame][parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if len(parts) == 1 {
packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
} else {
packages[parts[0]].Populate(parts[1:], prov)
idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
}
}
msg(" | Writing to index file")
gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
}
msg(" | Imported module")
}
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
msg(":DONE")
job.Complete = true
}