Exemple #1
0
func main() {
	configure()

	indexes = make(map[string]map[string]*gopan.Source)
	indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	// FIXME inefficient
	_, _, tpkg, _ := gopan.CountIndex(indexes)

	npkg := 0
	nmod := 0

	var pc = func() float64 {
		return float64(nmod) / float64(tpkg) * 100
	}

	log.Info("Writing packages index file")

	out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
	if err != nil {
		log.Error("Error creating packages index: %s", err.Error())
		return
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, idx := range indexes[fname] {
			log.Debug("Index: %s", idx)
			out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
			for _, auth := range idx.Authors {
				log.Debug("Author %s", auth)
				out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
				for _, pkg := range auth.Packages {
					out.Write([]byte("  " + pkg.Name + " => " + pkg.URL + "\n"))
					log.Debug("Package: %s", pkg)

					if !config.Flatten {
						if len(pkg.Provides) == 0 {
							// TODO better handling of filenames
							modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")

							tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name

							if _, err := os.Stat(tgzpath); err != nil {
								log.Error("File not found: %s", tgzpath)
								return
							}

							extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
							dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name

							log.Trace("=> tgzpath: %s", tgzpath)
							log.Trace(" > extpath: %s", extpath)
							log.Trace(" > dirpath: %s", dirpath)

							// Only index packages if they don't already exist
							if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
								log.Error("Error retrieving package list: %s", err)
								continue
							}
						}

						npkg += len(pkg.Provides)
						nmod += 1

						for p, pk := range pkg.Provides {
							out.Write([]byte("   " + p + " (" + pk.Version + "): " + pk.File + "\n"))
						}

						if nmod > 0 && nmod%100 == 0 {
							log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
						}

					}
				}
			}
		}
	}

	out.Close()

	log.Info("Found %d packages from %d modules", npkg, nmod)
}
Exemple #2
0
func do_import(session *http.Session, job *ImportJob) {
	log.Info("Running import job %s", job.Id)

	reponame := job.Form.ImportInto
	if reponame == "new_index" {
		reponame = job.Form.NewIndex
	}

	msg := func(m string) {
		if m != ":DONE" {
			job.History = append(job.History, m)
			log.Info(m)
		}
		for _, w := range job.Watchers {
			w(m)
		}
	}

	mods := make([]*getpan.Module, 0)

	// TODO cpanm mirror when using getpan_import

	if len(job.Form.Cpanfile) > 0 {
		msg("Parsing cpanfile input")
		_, modules := getpan_import(job, msg)
		mods = append(mods, modules...)
	}

	if len(job.Form.ImportURL) > 0 {
		msg("Importing from URL: " + job.Form.ImportURL)

		// TODO support cpanfile urls

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.ImportURL)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		if _, err := os.Stat(nfile); err != nil {
			os.MkdirAll(npath, 0777)
			out, err := os.Create(nfile)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			url := job.Form.ImportURL
			log.Trace("Downloading: %s", url)
			resp, err := nethttp.Get(url)

			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			_, err = io.Copy(out, resp.Body)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			out.Close()
			resp.Body.Close()
		} else {
			log.Trace("File already exists in cache: %s", nfile)
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if len(job.Form.FromDir) > 0 {
		msg("Importing from local directory: " + job.Form.FromDir)

		// TODO support cpanfile paths

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.FromDir)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		_, err := CopyFile(nfile, job.Form.FromDir)
		if err != nil {
			msg(err.Error())
			msg(":DONE")
			job.Complete = true
			return
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if f, fh, err := session.Request.File("fromfile"); err == nil {
		fn := fh.Filename

		msg("Importing from uploaded module/cpanfile: " + fn)

		if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
			msg("Only cpanfile and *.tar.gz files are supported")
			msg(":DONE")
			job.Complete = true
			return
		}

		if fn == "cpanfile" {
			msg("Importing cpanfile")
			b, _ := ioutil.ReadAll(f)
			f.Close()
			job.Form.Cpanfile = string(b)
			_, modules := getpan_import(job, msg)
			mods = append(mods, modules...)
		} else {
			msg("Importing .tar.gz")

			nauth := job.Form.AuthorID

			if len(nauth) < 3 {
				// FIXME move to form validation
				msg("Author ID must be at least 3 characters")
				msg(":DONE")
				job.Complete = true
				return
			}

			npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
			_, fn = filepath.Split(fn)
			nfile := npath + "/" + fn

			msg("Caching to " + nfile)

			os.MkdirAll(npath, 0777)

			_, err := CopyToFile(nfile, f)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			fn = strings.TrimSuffix(fn, ".tar.gz")
			bits := strings.Split(fn, "-")
			name := strings.Join(bits[0:len(bits)-1], "-")
			version := bits[len(bits)-1]

			s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
			m := &getpan.Module{
				Source:  s,
				Name:    name,
				Version: version,
				Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
				Cached:  nfile,
				Dir:     npath,
			}
			m.Deps = &getpan.DependencyList{
				Parent:       m,
				Dependencies: make([]*getpan.Dependency, 0),
			}
			mods = append(mods, m)
		}
	} else {
		// there is no file... so no error
		//msg("Error importing file upload: " + err.Error())
	}

	if len(mods) == 0 {
		msg("Nothing to do")
		msg(":DONE")
		job.Complete = true
		return
	}

	msg("Adding modules to GoPAN index")

	for _, m := range mods {
		msg("=> " + m.Name + " (" + m.Cached + ")")

		dn, fn := filepath.Split(m.Cached)
		dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
		auth := dnb[len(dnb)-1]
		ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
		npath := ndir + "/" + fn

		if _, err := os.Stat(npath); err == nil {
			msg(" | Already exists in repository")
		} else {
			os.MkdirAll(ndir, 0777)

			msg(" | Copying to " + npath)
			_, err := CopyFile(npath, m.Cached)
			if err != nil {
				msg(" ! " + err.Error())
				continue
			}
		}

		if _, ok := indexes[config.Index][reponame]; !ok {
			msg(" | Creating index: " + reponame)
			indexes[config.Index][reponame] = &gopan.Source{
				Name:    reponame,
				URL:     "/authors/id",
				Authors: make(map[string]*gopan.Author),
			}

			mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
			msg(" | Creating author: " + auth)
			author := &gopan.Author{
				Source:   indexes[config.Index][reponame],
				Name:     auth,
				Packages: make(map[string]*gopan.Package),
				URL:      "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
			}
			indexes[config.Index][reponame].Authors[auth] = author

			if _, ok := mapped[reponame]; !ok {
				mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
			}

			// author name
			if _, ok := mapped[reponame][author.Name[:1]]; !ok {
				mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
				mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author

			// wildcards
			if _, ok := mapped[reponame]["*"]; !ok {
				mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"]["**"]; !ok {
				mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
			}
			mapped[reponame]["*"]["**"][author.Name] = author

			// combos
			if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
				mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
				mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]]["**"][author.Name] = author
			mapped[reponame]["*"][author.Name[:2]][author.Name] = author
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
			msg(" | Creating module: " + fn)
			indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
				Author:   indexes[config.Index][reponame].Authors[auth],
				Name:     fn,
				URL:      indexes[config.Index][reponame].Authors[auth].URL + fn,
				Provides: make(map[string]*gopan.PerlPackage),
			}

			msg(" | Getting list of packages")
			modnm := strings.TrimSuffix(fn, ".tar.gz")
			pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
				msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
			}

			//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			msg(" | Adding packages to index")
			if _, ok := idxpackages[reponame]; !ok {
				idxpackages[reponame] = make(map[string]*PkgSpace)
			}
			filemap[pkg.AuthorURL()] = reponame
			for _, prov := range pkg.Provides {
				parts := strings.Split(prov.Name, "::")
				if _, ok := packages[parts[0]]; !ok {
					packages[parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if _, ok := idxpackages[reponame][parts[0]]; !ok {
					idxpackages[reponame][parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if len(parts) == 1 {
					packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
					packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
					idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
					idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
				} else {
					packages[parts[0]].Populate(parts[1:], prov)
					idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
				}
			}

			msg(" | Writing to index file")
			gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
		}

		msg(" | Imported module")
	}

	nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
	// TODO should probably be in the index - needs to udpate when index changes
	summary = &Summary{nsrc, nauth, npkg, nprov}

	msg(":DONE")
	job.Complete = true
}