Example #1
0
func getpan_import(job *ImportJob, msg func(string)) (*getpan.CPANFile, []*getpan.Module) {
	cfg := getpan.DefaultConfig()
	cfg.CacheDir = config.CacheDir

	if len(job.Form.CPANMirror) > 0 {
		msg("Adding CPAN mirror: " + job.Form.CPANMirror)
		cfg.Sources = append(cfg.Sources, getpan.NewSource("CPAN", job.Form.CPANMirror+"/modules/02packages.details.txt.gz", job.Form.CPANMirror))
	}

	defer func(job *ImportJob) {
		job.Complete = true
		msg("GetPAN import complete")
	}(job)

	for _, source := range cfg.Sources {
		if err := source.Load(); err != nil {
			m := fmt.Sprintf("Error loading sources: %s", err)
			msg(m)
			return nil, nil
		}
	}

	m := "Loaded sources"
	msg(m)

	deps, err := getpan.ParseCPANLines(strings.Split(job.Form.Cpanfile, "\n"))
	if err != nil {
		m := fmt.Sprintf("Error parsing cpanfile: %s", err)
		msg(m)
		return nil, nil
	}

	m = "Parsed cpanfile"
	msg(m)

	if err = deps.Resolve(); err != nil {
		m := fmt.Sprintf("Error resolving dependencies: %s", err)
		msg(m)
		return deps, nil
	}

	m = "Resolved dependency tree:"
	msg(m)

	modules := make([]*getpan.Module, 0)

	PrintCPANFile(job, deps, 0, msg, &modules)

	return deps, modules
}
Example #2
0
func do_import(session *http.Session, job *ImportJob) {
	log.Info("Running import job %s", job.Id)

	reponame := job.Form.ImportInto
	if reponame == "new_index" {
		reponame = job.Form.NewIndex
	}

	msg := func(m string) {
		if m != ":DONE" {
			job.History = append(job.History, m)
			log.Info(m)
		}
		for _, w := range job.Watchers {
			w(m)
		}
	}

	mods := make([]*getpan.Module, 0)

	// TODO cpanm mirror when using getpan_import

	if len(job.Form.Cpanfile) > 0 {
		msg("Parsing cpanfile input")
		_, modules := getpan_import(job, msg)
		mods = append(mods, modules...)
	}

	if len(job.Form.ImportURL) > 0 {
		msg("Importing from URL: " + job.Form.ImportURL)

		// TODO support cpanfile urls

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.ImportURL)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		if _, err := os.Stat(nfile); err != nil {
			os.MkdirAll(npath, 0777)
			out, err := os.Create(nfile)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			url := job.Form.ImportURL
			log.Trace("Downloading: %s", url)
			resp, err := nethttp.Get(url)

			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			_, err = io.Copy(out, resp.Body)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			out.Close()
			resp.Body.Close()
		} else {
			log.Trace("File already exists in cache: %s", nfile)
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if len(job.Form.FromDir) > 0 {
		msg("Importing from local directory: " + job.Form.FromDir)

		// TODO support cpanfile paths

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.FromDir)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		_, err := CopyFile(nfile, job.Form.FromDir)
		if err != nil {
			msg(err.Error())
			msg(":DONE")
			job.Complete = true
			return
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if f, fh, err := session.Request.File("fromfile"); err == nil {
		fn := fh.Filename

		msg("Importing from uploaded module/cpanfile: " + fn)

		if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
			msg("Only cpanfile and *.tar.gz files are supported")
			msg(":DONE")
			job.Complete = true
			return
		}

		if fn == "cpanfile" {
			msg("Importing cpanfile")
			b, _ := ioutil.ReadAll(f)
			f.Close()
			job.Form.Cpanfile = string(b)
			_, modules := getpan_import(job, msg)
			mods = append(mods, modules...)
		} else {
			msg("Importing .tar.gz")

			nauth := job.Form.AuthorID

			if len(nauth) < 3 {
				// FIXME move to form validation
				msg("Author ID must be at least 3 characters")
				msg(":DONE")
				job.Complete = true
				return
			}

			npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
			_, fn = filepath.Split(fn)
			nfile := npath + "/" + fn

			msg("Caching to " + nfile)

			os.MkdirAll(npath, 0777)

			_, err := CopyToFile(nfile, f)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			fn = strings.TrimSuffix(fn, ".tar.gz")
			bits := strings.Split(fn, "-")
			name := strings.Join(bits[0:len(bits)-1], "-")
			version := bits[len(bits)-1]

			s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
			m := &getpan.Module{
				Source:  s,
				Name:    name,
				Version: version,
				Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
				Cached:  nfile,
				Dir:     npath,
			}
			m.Deps = &getpan.DependencyList{
				Parent:       m,
				Dependencies: make([]*getpan.Dependency, 0),
			}
			mods = append(mods, m)
		}
	} else {
		// there is no file... so no error
		//msg("Error importing file upload: " + err.Error())
	}

	if len(mods) == 0 {
		msg("Nothing to do")
		msg(":DONE")
		job.Complete = true
		return
	}

	msg("Adding modules to GoPAN index")

	for _, m := range mods {
		msg("=> " + m.Name + " (" + m.Cached + ")")

		dn, fn := filepath.Split(m.Cached)
		dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
		auth := dnb[len(dnb)-1]
		ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
		npath := ndir + "/" + fn

		if _, err := os.Stat(npath); err == nil {
			msg(" | Already exists in repository")
		} else {
			os.MkdirAll(ndir, 0777)

			msg(" | Copying to " + npath)
			_, err := CopyFile(npath, m.Cached)
			if err != nil {
				msg(" ! " + err.Error())
				continue
			}
		}

		if _, ok := indexes[config.Index][reponame]; !ok {
			msg(" | Creating index: " + reponame)
			indexes[config.Index][reponame] = &gopan.Source{
				Name:    reponame,
				URL:     "/authors/id",
				Authors: make(map[string]*gopan.Author),
			}

			mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
			msg(" | Creating author: " + auth)
			author := &gopan.Author{
				Source:   indexes[config.Index][reponame],
				Name:     auth,
				Packages: make(map[string]*gopan.Package),
				URL:      "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
			}
			indexes[config.Index][reponame].Authors[auth] = author

			if _, ok := mapped[reponame]; !ok {
				mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
			}

			// author name
			if _, ok := mapped[reponame][author.Name[:1]]; !ok {
				mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
				mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author

			// wildcards
			if _, ok := mapped[reponame]["*"]; !ok {
				mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"]["**"]; !ok {
				mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
			}
			mapped[reponame]["*"]["**"][author.Name] = author

			// combos
			if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
				mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
				mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]]["**"][author.Name] = author
			mapped[reponame]["*"][author.Name[:2]][author.Name] = author
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
			msg(" | Creating module: " + fn)
			indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
				Author:   indexes[config.Index][reponame].Authors[auth],
				Name:     fn,
				URL:      indexes[config.Index][reponame].Authors[auth].URL + fn,
				Provides: make(map[string]*gopan.PerlPackage),
			}

			msg(" | Getting list of packages")
			modnm := strings.TrimSuffix(fn, ".tar.gz")
			pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
				msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
			}

			//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			msg(" | Adding packages to index")
			if _, ok := idxpackages[reponame]; !ok {
				idxpackages[reponame] = make(map[string]*PkgSpace)
			}
			filemap[pkg.AuthorURL()] = reponame
			for _, prov := range pkg.Provides {
				parts := strings.Split(prov.Name, "::")
				if _, ok := packages[parts[0]]; !ok {
					packages[parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if _, ok := idxpackages[reponame][parts[0]]; !ok {
					idxpackages[reponame][parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if len(parts) == 1 {
					packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
					packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
					idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
					idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
				} else {
					packages[parts[0]].Populate(parts[1:], prov)
					idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
				}
			}

			msg(" | Writing to index file")
			gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
		}

		msg(" | Imported module")
	}

	nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
	// TODO should probably be in the index - needs to udpate when index changes
	summary = &Summary{nsrc, nauth, npkg, nprov}

	msg(":DONE")
	job.Complete = true
}