Example #1
0
func main() {
	configure()

	indexes = make(map[string]map[string]*gopan.Source)
	indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	// FIXME inefficient
	_, _, tpkg, _ := gopan.CountIndex(indexes)

	npkg := 0
	nmod := 0

	var pc = func() float64 {
		return float64(nmod) / float64(tpkg) * 100
	}

	log.Info("Writing packages index file")

	out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
	if err != nil {
		log.Error("Error creating packages index: %s", err.Error())
		return
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, idx := range indexes[fname] {
			log.Debug("Index: %s", idx)
			out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
			for _, auth := range idx.Authors {
				log.Debug("Author %s", auth)
				out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
				for _, pkg := range auth.Packages {
					out.Write([]byte("  " + pkg.Name + " => " + pkg.URL + "\n"))
					log.Debug("Package: %s", pkg)

					if !config.Flatten {
						if len(pkg.Provides) == 0 {
							// TODO better handling of filenames
							modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")

							tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name

							if _, err := os.Stat(tgzpath); err != nil {
								log.Error("File not found: %s", tgzpath)
								return
							}

							extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
							dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name

							log.Trace("=> tgzpath: %s", tgzpath)
							log.Trace(" > extpath: %s", extpath)
							log.Trace(" > dirpath: %s", dirpath)

							// Only index packages if they don't already exist
							if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
								log.Error("Error retrieving package list: %s", err)
								continue
							}
						}

						npkg += len(pkg.Provides)
						nmod += 1

						for p, pk := range pkg.Provides {
							out.Write([]byte("   " + p + " (" + pk.Version + "): " + pk.File + "\n"))
						}

						if nmod > 0 && nmod%100 == 0 {
							log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
						}

					}
				}
			}
		}
	}

	out.Close()

	log.Info("Found %d packages from %d modules", npkg, nmod)
}
Example #2
0
func main() {
	configure()

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	indexes = make(map[string]map[string]*gopan.Source)
	if !config.NoCache {
		indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
	}

	if config.NoCache || config.Update {
		for _, s := range config.Sources {
			b := strings.SplitN(s, "=", 2)
			if len(b) < 2 {
				log.Error("Expected Name=URL pair, got: %s", s)
				return
			}

			if idx, ok := indexes[config.Index][b[0]]; ok {
				log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1])
				idx.URL = b[1]
			} else {
				indexes[config.Index][b[0]] = &gopan.Source{
					Name:    b[0],
					URL:     b[1],
					Authors: make(map[string]*gopan.Author, 0),
				}
			}
		}

		if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN {
			log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN")
			config.CPAN = true
			config.BackPAN = true
		}

		if config.CPAN {
			if _, ok := indexes[config.Index]["CPAN"]; !ok {
				log.Debug("Adding CPAN index")
				indexes[config.Index]["CPAN"] = gopan.CPANSource()
			} else {
				log.Debug("CPAN index already exists")
			}
		}

		if config.BackPAN {
			if _, ok := indexes[config.Index]["BackPAN"]; !ok {
				log.Debug("Adding BackPAN index")
				indexes[config.Index]["BackPAN"] = gopan.BackPANSource()
			} else {
				log.Debug("BackPAN index already exists")
			}
		}

		log.Info("Using sources:")
		for fname, _ := range indexes {
			log.Info("From %s", fname)
			for _, source := range indexes[fname] {
				log.Info("=> %s", source.String())
			}
		}

		newAuthors := getAuthors()
		newPackages := getPackages()

		os.MkdirAll(config.CacheDir, 0777)

		if !config.NoCache {
			gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index])
		}

		if config.Update {
			log.Info("Found %d new packages by %d new authors", newAuthors, newPackages)
		}
	}

	nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes)
	log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc)

	if !config.NoMirror {
		mirrorPan()
	}
}
Example #3
0
func mirrorPan() {
	log.Info("Mirroring *PAN")

	// FIXME inefficient
	_, _, npkg, _ := gopan.CountIndex(indexes)

	mirrored := 0
	var pc = func() int {
		return mirrored / npkg * 100
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, source := range indexes[fname] {
			log.Debug("Index: %s", source)
			wg.Add(1)
			go func(source *gopan.Source) {
				defer wg.Done()
				for _, author := range source.Authors {
					log.Debug("=> %s", author)
					wg.Add(1)
					go func(author *gopan.Author) {
						cachedir := config.CacheDir + "/" + source.Name + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
						os.MkdirAll(cachedir, 0777)

						defer wg.Done()
						for _, pkg := range author.Packages {
							wg.Add(1)
							go func(pkg *gopan.Package) {
								defer wg.Done()

								cache := cachedir + pkg.Name
								log.Trace("    - Caching to: %s", cache)

								if _, err := os.Stat(cache); err == nil {
									log.Debug("%d%%  |> %s", pc(), pkg)
									log.Trace("    - Already exists in cache")
									mirrored++
									return
								}

								sem <- 1

								mirrored++

								log.Debug("%d%%  => %s", pc(), pkg)

								url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" + pkg.Name
								log.Trace("    - From URL: %s", url)

								out, err := os.Create(cache)
								defer out.Close()
								if err != nil {
									log.Error("CREATE - %s", err.Error())
									<-sem
									return
								}

								resp, err := http.Get(url)
								if err != nil {
									log.Error("HTTP GET - %s", err.Error())
									<-sem
									return
								}

								_, err = io.Copy(out, resp.Body)
								if err != nil {
									log.Error("IO COPY - %s", err.Error())
								}

								<-sem
							}(pkg)
						}
					}(author)
				}
			}(source)
		}
	}

	wg.Wait()
	log.Info("Finished mirroring *PAN")
}
Example #4
0
func do_import(session *http.Session, job *ImportJob) {
	log.Info("Running import job %s", job.Id)

	reponame := job.Form.ImportInto
	if reponame == "new_index" {
		reponame = job.Form.NewIndex
	}

	msg := func(m string) {
		if m != ":DONE" {
			job.History = append(job.History, m)
			log.Info(m)
		}
		for _, w := range job.Watchers {
			w(m)
		}
	}

	mods := make([]*getpan.Module, 0)

	// TODO cpanm mirror when using getpan_import

	if len(job.Form.Cpanfile) > 0 {
		msg("Parsing cpanfile input")
		_, modules := getpan_import(job, msg)
		mods = append(mods, modules...)
	}

	if len(job.Form.ImportURL) > 0 {
		msg("Importing from URL: " + job.Form.ImportURL)

		// TODO support cpanfile urls

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.ImportURL)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		if _, err := os.Stat(nfile); err != nil {
			os.MkdirAll(npath, 0777)
			out, err := os.Create(nfile)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			url := job.Form.ImportURL
			log.Trace("Downloading: %s", url)
			resp, err := nethttp.Get(url)

			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			_, err = io.Copy(out, resp.Body)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			out.Close()
			resp.Body.Close()
		} else {
			log.Trace("File already exists in cache: %s", nfile)
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if len(job.Form.FromDir) > 0 {
		msg("Importing from local directory: " + job.Form.FromDir)

		// TODO support cpanfile paths

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.FromDir)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		_, err := CopyFile(nfile, job.Form.FromDir)
		if err != nil {
			msg(err.Error())
			msg(":DONE")
			job.Complete = true
			return
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if f, fh, err := session.Request.File("fromfile"); err == nil {
		fn := fh.Filename

		msg("Importing from uploaded module/cpanfile: " + fn)

		if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
			msg("Only cpanfile and *.tar.gz files are supported")
			msg(":DONE")
			job.Complete = true
			return
		}

		if fn == "cpanfile" {
			msg("Importing cpanfile")
			b, _ := ioutil.ReadAll(f)
			f.Close()
			job.Form.Cpanfile = string(b)
			_, modules := getpan_import(job, msg)
			mods = append(mods, modules...)
		} else {
			msg("Importing .tar.gz")

			nauth := job.Form.AuthorID

			if len(nauth) < 3 {
				// FIXME move to form validation
				msg("Author ID must be at least 3 characters")
				msg(":DONE")
				job.Complete = true
				return
			}

			npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
			_, fn = filepath.Split(fn)
			nfile := npath + "/" + fn

			msg("Caching to " + nfile)

			os.MkdirAll(npath, 0777)

			_, err := CopyToFile(nfile, f)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			fn = strings.TrimSuffix(fn, ".tar.gz")
			bits := strings.Split(fn, "-")
			name := strings.Join(bits[0:len(bits)-1], "-")
			version := bits[len(bits)-1]

			s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
			m := &getpan.Module{
				Source:  s,
				Name:    name,
				Version: version,
				Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
				Cached:  nfile,
				Dir:     npath,
			}
			m.Deps = &getpan.DependencyList{
				Parent:       m,
				Dependencies: make([]*getpan.Dependency, 0),
			}
			mods = append(mods, m)
		}
	} else {
		// there is no file... so no error
		//msg("Error importing file upload: " + err.Error())
	}

	if len(mods) == 0 {
		msg("Nothing to do")
		msg(":DONE")
		job.Complete = true
		return
	}

	msg("Adding modules to GoPAN index")

	for _, m := range mods {
		msg("=> " + m.Name + " (" + m.Cached + ")")

		dn, fn := filepath.Split(m.Cached)
		dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
		auth := dnb[len(dnb)-1]
		ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
		npath := ndir + "/" + fn

		if _, err := os.Stat(npath); err == nil {
			msg(" | Already exists in repository")
		} else {
			os.MkdirAll(ndir, 0777)

			msg(" | Copying to " + npath)
			_, err := CopyFile(npath, m.Cached)
			if err != nil {
				msg(" ! " + err.Error())
				continue
			}
		}

		if _, ok := indexes[config.Index][reponame]; !ok {
			msg(" | Creating index: " + reponame)
			indexes[config.Index][reponame] = &gopan.Source{
				Name:    reponame,
				URL:     "/authors/id",
				Authors: make(map[string]*gopan.Author),
			}

			mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
			msg(" | Creating author: " + auth)
			author := &gopan.Author{
				Source:   indexes[config.Index][reponame],
				Name:     auth,
				Packages: make(map[string]*gopan.Package),
				URL:      "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
			}
			indexes[config.Index][reponame].Authors[auth] = author

			if _, ok := mapped[reponame]; !ok {
				mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
			}

			// author name
			if _, ok := mapped[reponame][author.Name[:1]]; !ok {
				mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
				mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author

			// wildcards
			if _, ok := mapped[reponame]["*"]; !ok {
				mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"]["**"]; !ok {
				mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
			}
			mapped[reponame]["*"]["**"][author.Name] = author

			// combos
			if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
				mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
				mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]]["**"][author.Name] = author
			mapped[reponame]["*"][author.Name[:2]][author.Name] = author
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
			msg(" | Creating module: " + fn)
			indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
				Author:   indexes[config.Index][reponame].Authors[auth],
				Name:     fn,
				URL:      indexes[config.Index][reponame].Authors[auth].URL + fn,
				Provides: make(map[string]*gopan.PerlPackage),
			}

			msg(" | Getting list of packages")
			modnm := strings.TrimSuffix(fn, ".tar.gz")
			pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
				msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
			}

			//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			msg(" | Adding packages to index")
			if _, ok := idxpackages[reponame]; !ok {
				idxpackages[reponame] = make(map[string]*PkgSpace)
			}
			filemap[pkg.AuthorURL()] = reponame
			for _, prov := range pkg.Provides {
				parts := strings.Split(prov.Name, "::")
				if _, ok := packages[parts[0]]; !ok {
					packages[parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if _, ok := idxpackages[reponame][parts[0]]; !ok {
					idxpackages[reponame][parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if len(parts) == 1 {
					packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
					packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
					idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
					idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
				} else {
					packages[parts[0]].Populate(parts[1:], prov)
					idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
				}
			}

			msg(" | Writing to index file")
			gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
		}

		msg(" | Imported module")
	}

	nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
	// TODO should probably be in the index - needs to udpate when index changes
	summary = &Summary{nsrc, nauth, npkg, nprov}

	msg(":DONE")
	job.Complete = true
}
Example #5
0
func main() {
	configure()

	args := flag.Args()

	if len(args) > 0 && args[0] == "init" {
		log.Info("Initialising SmartPAN")

		log.Info("=> Installing Perl dependencies")

		// FIXME most of this is repeated from getpan/main.go
		cfg := getpan.DefaultConfig()
		cfg.CacheDir = config.CacheDir

		for _, source := range cfg.Sources {
			if err := source.Load(); err != nil {
				log.Error("Error loading sources: %s", err)
				os.Exit(1)
				return
			}
		}

		deps := &getpan.DependencyList{
			Dependencies: make([]*getpan.Dependency, 0),
		}

		d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "")
		d2, _ := getpan.DependencyFromString("JSON::XS", "")
		deps.AddDependency(d1)
		deps.AddDependency(d2)

		if err := deps.Resolve(); err != nil {
			log.Error("Error resolving dependencies: %s", err)
			os.Exit(1)
			return
		}

		_, err := deps.Install()
		if err != nil {
			log.Error("Error installing dependencies: %s", err)
			os.Exit(2)
			return
		}

		log.Info("   - Installed %d modules", deps.UniqueInstalled())

		log.Info("SmartPAN initialisation complete")

		return
	}

	if config.TestDeps {
		perldeps := gopan.TestPerlDeps()
		perldeps.Dump()
		if !perldeps.Ok {
			log.Error("Required perl dependencies are missing")
			os.Exit(1)
			return
		}
	}

	if len(args) > 0 && args[0] == "import" {
		if len(args) < 4 {
			log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX")
			return
		}

		fname := args[1]
		log.Info("Importing module from %s", fname)
		log.Info("Author ID: %s", args[2])
		log.Info("Index    : %s", args[3])

		extraParams := map[string]string{
			"importinto": args[3],
			"authorid":   args[2],
			"newindex":   "",
			"cpanmirror": "",
			"importurl":  "",
			"fromdir":    "",
		}

		if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
			log.Info("URL: %s", fname)

			extraParams["importurl"] = fname

			request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams)
			if err != nil {
				log.Error("Create request error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)

			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		} else {
			fname = strings.TrimPrefix(fname, "file://")
			log.Info("File: %s", fname)

			if _, err := os.Stat(fname); err != nil {
				log.Error("File not found: %s", err.Error())
				return
			}

			request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname)
			if err != nil {
				log.Error("Create upload error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)
			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		}

		return
	}

	config.CurrentRelease = CurrentRelease

	var wg sync.WaitGroup

	load_index = func(index string, file string) {
		indexes[index] = gopan.LoadIndex(file)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()

		indexes = make(map[string]map[string]*gopan.Source)

		// Load CPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil {
			config.HasCPANIndex = true
			config.CPANIndexDate = fi.ModTime().String()
			config.CPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex)
				config.CPANStatus = "Loaded"
			}()
		}

		// Load BackPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil {
			config.HasBackPANIndex = true
			config.BackPANIndexDate = fi.ModTime().String()
			config.BackPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex)
				config.BackPANStatus = "Loaded"
			}()
		}

		// Load our secondary indexes
		for _, idx := range config.Indexes {
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(idx, config.CacheDir+"/"+idx)
			}()
		}

		// Load our primary index (this is the only index written back to)
		wg.Add(1)
		go func() {
			defer wg.Done()
			load_index(config.Index, config.CacheDir+"/"+config.Index)
		}()
	}()

	update_indexes = func() {
		wg.Wait()
		wg.Add(1)
		go func() {
			wg.Wait()
			config.ImportAvailable = true

			nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
			// TODO should probably be in the index - needs to udpate when index changes
			summary = &Summary{nsrc, nauth, npkg, nprov}

			// Do this now so changing the level doesn't interfere with index load
			log.Logger().SetLevel(log.Stol(config.LogLevel))
		}()
		defer wg.Done()
		// Create in-memory indexes for UI/search etc
		for fname, _ := range indexes {
			for idn, idx := range indexes[fname] {
				mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
				for _, auth := range idx.Authors {
					// author name
					if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
						mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
						mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth

					// wildcards
					if _, ok := mapped[idx.Name]["*"]; !ok {
						mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
						mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name]["*"]["**"][auth.Name] = auth

					// combos
					if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
						mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
						mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
					mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth

					for _, pkg := range auth.Packages {
						filemap[pkg.AuthorURL()] = idn
						for _, prov := range pkg.Provides {
							parts := strings.Split(prov.Name, "::")
							log.Trace("PACKAGE: %s", prov.Name)

							if _, ok := packages[parts[0]]; !ok {
								packages[parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if _, ok := idxpackages[idx.Name]; !ok {
								idxpackages[idx.Name] = make(map[string]*PkgSpace)
							}
							if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
								idxpackages[idx.Name][parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if len(parts) == 1 {
								packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
								packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
								idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
							} else {
								packages[parts[0]].Populate(parts[1:], prov)
								idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
							}
						}
					}
				}
			}
		}
	}
	go update_indexes()

	// Get latest SmartPAN version
	go func() {
		res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases")
		if err != nil {
			log.Error("Error getting latest version: %s", err.Error())
			return
		}
		defer res.Body.Close()
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Error("Error reading stream: %s", err.Error())
			return
		}

		var r Releases
		if err = json.Unmarshal(b, &r); err != nil {
			log.Error("Error unmarshalling JSON: %s", err.Error())
			return
		}

		log.Info("Current release: %s", config.CurrentRelease)
		rel := strings.TrimPrefix(r[0].TagName, "v")
		log.Info("Latest release: %s", rel)
		config.LatestRelease = rel
		config.UpdateURL = r[0].URL

		if config.CurrentRelease < rel {
			config.CanUpdate = true
			log.Info("Your version of SmartPAN can be updated.")
		}
	}()

	// Create our Gotcha application
	var app = gotcha.Create(Asset)
	app.Config.Listen = config.Bind

	summary = &Summary{0, 0, 0, 0}

	app.On(events.BeforeHandler, func(session *http.Session, next func()) {
		session.Stash["summary"] = summary
		session.Stash["config"] = config

		next()
	})

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", search)
	r.Post("/", search)

	r.Get("/help", help)
	r.Get("/settings", settings)
	r.Get("/browse", browse)

	r.Get("/import", import1)
	r.Post("/import", import1)

	r.Get("/import/(?P<jobid>[^/]+)", import2)
	r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)

	r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	// JSON endpoints
	r.Get("/where/(?P<module>[^/]+)/?", where)
	r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)

	// Put these last so they only match /{repo} if nothing else matches
	r.Get("/(?P<repo>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
	r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
	r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)

	// Start our application
	app.Start()

	<-make(chan int)
}
Example #6
0
func delete_file(session *http.Session) {
	session.Stash["Title"] = "Delete file"
	html, _ := session.RenderTemplate("delete.html")

	repo := session.Stash["repo"].(string)
	file := session.Stash["file"].(string)
	auth1 := session.Stash["auth1"].(string)
	auth2 := session.Stash["auth2"].(string)
	auth3 := session.Stash["auth3"].(string)

	fname := config.CacheDir + "/" + repo + "/" + auth1 + "/" + auth2 + "/" + auth3 + "/" + file

	if _, err := os.Stat(fname); err != nil {
		session.RenderNotFound()
		return
	}

	// Remove file from indexes
	for f, _ := range indexes {
		if _, ok := indexes[f][repo]; !ok {
			continue
		}
		if _, ok := indexes[f][repo].Authors[auth3]; !ok {
			continue
		}
		if _, ok := indexes[f][repo].Authors[auth3].Packages[file]; !ok {
			continue
		}
		log.Debug("Removing from index: %s", repo)

		pkg := indexes[f][repo].Authors[auth3].Packages[file]
		delete(indexes[f][repo].Authors[auth3].Packages, file)

		if len(indexes[f][repo].Authors[auth3].Packages) == 0 {
			log.Debug("Removing author")
			delete(indexes[f][repo].Authors, auth3)
		}

		if len(indexes[f][repo].Authors) == 0 {
			log.Debug("Removing index")
			delete(indexes[f], repo)
		}

		if auth, ok := mapped[repo][auth1][auth2][auth3]; ok {
			if len(auth.Packages) == 0 {
				log.Debug("Removing author from mapped index")
				delete(mapped[repo][auth1][auth2], auth3)
				delete(mapped[repo]["*"][auth2], auth3)
				delete(mapped[repo][auth1]["**"], auth3)
				delete(mapped[repo]["*"]["**"], auth3)
			}

			if len(mapped[repo][auth1][auth2]) == 0 {
				log.Debug("Removing auth1/auth2 from mapped index")
				delete(mapped[repo][auth1], auth2)
			}

			if len(mapped[repo]["*"][auth2]) == 0 {
				log.Debug("Removing author **/auth2 from mapped index")
				delete(mapped[repo][auth1], auth2)
			}
			if len(mapped[repo][auth1]["**"]) == 0 {
				log.Debug("Removing author auth1/** from mapped index")
				delete(mapped[repo][auth1], auth2)
			}
			if len(mapped[repo]["*"]["**"]) == 0 {
				log.Debug("Removing author */** from mapped index")
				delete(mapped[repo][auth1], auth2)
			}
			if len(mapped[repo]["*"]) == 0 {
				log.Debug("Removing author * from mapped index")
				delete(mapped[repo][auth1], auth2)
			}

			if len(mapped[repo][auth1]) == 1 {
				log.Debug("Removing author auth1 from mapped index")
				delete(mapped[repo], auth1)
			}

			if len(mapped[repo]) == 1 {
				log.Debug("Removing repo from mapped index")
				delete(mapped, repo)
			}
		}

		for _, prov := range pkg.Provides {
			parts := strings.Split(prov.Name, "::")
			// TODO remove from packages/idxpackages
			if ctx, ok := packages[parts[0]]; ok {
				parts = parts[1:]
				for len(parts) > 0 {
					if c, ok := ctx.Children[parts[0]]; ok {
						ctx = c
					} else {
						log.Debug("Package not found in packages: %s", parts)
						break
					}
					parts = parts[1:]
				}
				if len(parts) == 0 {
					for ctx != nil {
						for pi, p := range ctx.Packages {
							if p.Package == pkg {
								log.Debug("Removing package from packages: %s", ctx.FullName())
								ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
								break
							}
						}
						if len(ctx.Packages) == 0 {
							log.Debug("Removing PkgSpace from packages: %s", ctx.FullName())
							if ctx.Parent == nil {
								delete(packages, ctx.Namespace)
							} else {
								delete(ctx.Parent.Children, ctx.Namespace)
							}
						}

						ctx = ctx.Parent
					}
				}
			}
			parts = strings.Split(prov.Name, "::")
			if _, ok := idxpackages[repo]; ok {
				if ctx, ok := idxpackages[repo][parts[0]]; ok {
					parts = parts[1:]
					for len(parts) > 0 {
						if c, ok := ctx.Children[parts[0]]; ok {
							ctx = c
						} else {
							log.Debug("PkgSpace not found in idxpackages")
							break
						}
						parts = parts[1:]
					}
					if len(parts) == 0 {
						for ctx != nil {
							for pi, p := range ctx.Packages {
								if p.Package == pkg {
									log.Debug("Removing package from idxpackages")
									ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
									break
								}
							}
							if len(ctx.Packages) == 0 {
								log.Debug("Removing PkgSpace from idxpackages: %s", ctx.FullName())
								if ctx.Parent == nil {
									delete(idxpackages, ctx.Namespace)
								} else {
									delete(ctx.Parent.Children, ctx.Namespace)
								}
							}

							ctx = ctx.Parent
						}
					}
				}
			}
		}

		if _, ok := filemap[auth1+"/"+auth2+"/"+auth3+"/"+file]; ok {
			log.Debug("Removing file from filemap")
			// FIXME filemap should be map[string][]string, so we know if
			// the file exists in multiple indexes
			delete(filemap, auth1+"/"+auth2+"/"+auth3+"/"+file)
		}

		// write remove to index
		gopan.RemoveModule(config.CacheDir+"/"+config.Index, pkg.Author.Source, pkg.Author, pkg)
	}

	log.Debug("Removing file from gopancache: %s", fname)
	// TODO move file deletion to shared gopan package
	if err := os.Remove(fname); err != nil {
		log.Error("Error removing file: %s", err)
	}

	// TODO maybe clean up author tree (is this smartpans responsibility?)

	nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
	// TODO should probably be in the index - needs to udpate when index changes
	summary = &Summary{nsrc, nauth, npkg, nprov}

	session.Stash["Page"] = "Browse"
	session.Stash["Content"] = template.HTML(html)
	session.Render("layout.html")
}