コード例 #1
0
ファイル: smart.go プロジェクト: companieshouse/gopan
func where(session *http.Session) {
	module := session.Stash["module"].(string)
	log.Info("Looking for module: %s", module)

	ns := strings.Split(module, "::")
	if _, ok := packages[ns[0]]; !ok {
		log.Info("Top-level namespace [%s] not found", ns[0])
		session.Response.Status = 404
		session.Response.Send()
		return
	}

	mod := packages[ns[0]]

	ns = ns[1:]
	for len(ns) > 0 {
		if _, ok := mod.Children[ns[0]]; !ok {
			log.Info("Child namespace [%s] not found", ns[0])
			session.Response.Status = 404
			session.Response.Send()
			return
		}
		log.Info("Found child namespace [%s]", ns[0])
		mod = mod.Children[ns[0]]
		ns = ns[1:]
	}

	var version string
	if _, ok := session.Stash["version"]; ok {
		version = session.Stash["version"].(string)
		if strings.HasPrefix(version, "v") {
			version = strings.TrimPrefix(version, "v")
		}
		log.Info("Looking for version: %s", version)
	}

	if len(mod.Versions) == 0 {
		log.Info("Module has no versions in index")
		session.Response.Status = 404
		session.Response.Send()
		return
	}

	versions := make([]*VersionOutput, 0)
	lv := float64(0)
	if len(version) > 0 {
		if ">0" == version {
			// Take account of packages that have a version of 'undef'
			for _, pkg := range mod.Packages {
				packageURL := pkg.Package.VirtualURL()
				urlmatches := packageUrlRe.FindStringSubmatch(packageURL)
				if nil == urlmatches {
					log.Info("Version requested [%s] not found", version)
					session.Response.Status = 404
					session.Response.Send()
					return
				}
				ver := gopan.VersionFromString(urlmatches[2])
				log.Info("Found version: %f", ver)
				versions = append(versions, &VersionOutput{
					Index:   pkg.Package.Author.Source.Name,
					URL:     packageURL,
					Path:    pkg.Package.AuthorURL(),
					Version: ver,
				})
				if ver > lv {
					lv = ver
				}
			}
		} else {
			dep, _ := getpan.DependencyFromString(module, version)
			for _, md := range mod.Versions {
				var sver string = md.Version
				if `undef` == md.Version {
					sver = fmt.Sprintf("%.2f", md.Package.Version())
				}
				log.Info("Matching [%s] against derived version [%s] (md.Version [%s], md.Package.Version [%f])", dep.Version, sver, md.Version, md.Package.Version())
				if dep.MatchesVersion(sver) {
					vout := &VersionOutput{
						Index:   md.Package.Author.Source.Name,
						URL:     md.Package.VirtualURL(),
						Path:    md.Package.AuthorURL(),
						Version: gopan.VersionFromString(sver),
					}
					versions = append(versions, vout)
					ver := gopan.VersionFromString(sver)
					if ver > lv {
						lv = ver
					}
				}
			}
		}

		if len(versions) == 0 {
			log.Info("Version requested [%s] not found", version)
			session.Response.Status = 404
			session.Response.Send()
			return
		}
	} else {
		for v, pkg := range mod.Versions {
			log.Info("Found version: %f", v)
			versions = append(versions, &VersionOutput{
				Index:   pkg.Package.Author.Source.Name,
				URL:     pkg.Package.VirtualURL(),
				Path:    pkg.Package.AuthorURL(),
				Version: gopan.VersionFromString(pkg.Version),
			})
			if v > lv {
				lv = v
			}
		}
	}

	session.Response.Headers.Set("Content-Type", "application/json")

	o := &WhereOutput{
		Module:   mod.FullName(),
		Latest:   lv,
		Versions: versions,
	}

	b, err := json.MarshalIndent(o, "", "  ")

	log.Info("Output: %s", string(b))

	if err != nil {
		log.Error("Failed encoding JSON: %s", err.Error())
		session.Response.Status = 500
		session.Response.Send()
		return
	}

	session.Response.Status = 200
	session.Response.Write(b)
}
コード例 #2
0
ファイル: main.go プロジェクト: companieshouse/gopan
func main() {
	config = getpan.Configure()
	config.Dump()

	mods := flag.Args()

	if len(mods) == 0 {
		if _, err := os.Stat(config.CPANFile); os.IsNotExist(err) {
			log.Error("cpanfile not found: %s", config.CPANFile)
			os.Exit(1)
		}
	}

	if len(mods) > 0 && mods[0] == "exec" {
		log.Debug("getpan exec => " + strings.Join(mods[1:], " "))

		cmd := exec.Command(mods[1], mods[2:]...)

		cmd.Env = os.Environ()
		cmd.Env = append(cmd.Env, "PERL5LIB="+config.InstallDir+"/lib/perl5")
		cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")+":"+config.InstallDir+"/bin")

		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		err := cmd.Run()

		if err != nil {
			// debug so it doesn't show up in stdout/stderr unless -loglevel is used
			log.Debug("Error in exec: %s", err.Error())
			os.Exit(10)
		}

		return
	}

	for _, source := range config.Sources {
		err := source.Load()
		if err != nil {
			log.Error("Error loading sources: %s", err)
			os.Exit(1)
			return
		}
	}

	var deps *getpan.DependencyList

	if len(mods) == 0 {
		log.Info("Installing from cpanfile: %s", config.CPANFile)
		d, err := getpan.ParseCPANFile(config.CPANFile)
		if err != nil {
			log.Error("Error parsing cpanfile: %s", err)
			os.Exit(2)
			return
		}
		deps = &d.DependencyList
	} else {
		log.Info("Installing from command line args")
		deps = &getpan.DependencyList{
			Dependencies: make([]*getpan.Dependency, 0),
		}
		for _, arg := range mods {
			dependency, err := getpan.DependencyFromString(arg, "")
			if err != nil {
				log.Error("Unable to parse input: %s", arg)
				continue
			}
			deps.AddDependency(dependency)
		}
	}

	err := deps.Resolve()
	if err != nil {
		log.Error("Error resolving dependencies: %s", err)
		os.Exit(3)
		return
	}

	if false == config.NoDepdump {
		log.Info("Resolved dependency tree:")
		deps.PrintDeps(0)
	}

	if config.NoInstall {
		log.Info("Skipping installation phase")
		return
	}

	_, err = deps.Install()

	if err != nil {
		log.Error("Error installing dependencies: %s", err)
		os.Exit(4)
		return
	}

	// FIXME hacky, need a better way of tracking installed deps
	log.Info("Successfully installed %d modules", deps.UniqueInstalled())
}
コード例 #3
0
ファイル: main.go プロジェクト: companieshouse/gopan
func main() {
	configure()

	args := flag.Args()

	if len(args) > 0 && args[0] == "init" {
		log.Info("Initialising SmartPAN")

		log.Info("=> Installing Perl dependencies")

		// FIXME most of this is repeated from getpan/main.go
		cfg := getpan.DefaultConfig()
		cfg.CacheDir = config.CacheDir

		for _, source := range cfg.Sources {
			if err := source.Load(); err != nil {
				log.Error("Error loading sources: %s", err)
				os.Exit(1)
				return
			}
		}

		deps := &getpan.DependencyList{
			Dependencies: make([]*getpan.Dependency, 0),
		}

		d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "")
		d2, _ := getpan.DependencyFromString("JSON::XS", "")
		deps.AddDependency(d1)
		deps.AddDependency(d2)

		if err := deps.Resolve(); err != nil {
			log.Error("Error resolving dependencies: %s", err)
			os.Exit(1)
			return
		}

		_, err := deps.Install()
		if err != nil {
			log.Error("Error installing dependencies: %s", err)
			os.Exit(2)
			return
		}

		log.Info("   - Installed %d modules", deps.UniqueInstalled())

		log.Info("SmartPAN initialisation complete")

		return
	}

	if config.TestDeps {
		perldeps := gopan.TestPerlDeps()
		perldeps.Dump()
		if !perldeps.Ok {
			log.Error("Required perl dependencies are missing")
			os.Exit(1)
			return
		}
	}

	if len(args) > 0 && args[0] == "import" {
		if len(args) < 4 {
			log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX")
			return
		}

		fname := args[1]
		log.Info("Importing module from %s", fname)
		log.Info("Author ID: %s", args[2])
		log.Info("Index    : %s", args[3])

		extraParams := map[string]string{
			"importinto": args[3],
			"authorid":   args[2],
			"newindex":   "",
			"cpanmirror": "",
			"importurl":  "",
			"fromdir":    "",
		}

		if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
			log.Info("URL: %s", fname)

			extraParams["importurl"] = fname

			request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams)
			if err != nil {
				log.Error("Create request error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)

			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		} else {
			fname = strings.TrimPrefix(fname, "file://")
			log.Info("File: %s", fname)

			if _, err := os.Stat(fname); err != nil {
				log.Error("File not found: %s", err.Error())
				return
			}

			request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname)
			if err != nil {
				log.Error("Create upload error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)
			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		}

		return
	}

	config.CurrentRelease = CurrentRelease

	var wg sync.WaitGroup

	load_index = func(index string, file string) {
		indexes[index] = gopan.LoadIndex(file)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()

		indexes = make(map[string]map[string]*gopan.Source)

		// Load CPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil {
			config.HasCPANIndex = true
			config.CPANIndexDate = fi.ModTime().String()
			config.CPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex)
				config.CPANStatus = "Loaded"
			}()
		}

		// Load BackPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil {
			config.HasBackPANIndex = true
			config.BackPANIndexDate = fi.ModTime().String()
			config.BackPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex)
				config.BackPANStatus = "Loaded"
			}()
		}

		// Load our secondary indexes
		for _, idx := range config.Indexes {
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(idx, config.CacheDir+"/"+idx)
			}()
		}

		// Load our primary index (this is the only index written back to)
		wg.Add(1)
		go func() {
			defer wg.Done()
			load_index(config.Index, config.CacheDir+"/"+config.Index)
		}()
	}()

	update_indexes = func() {
		wg.Wait()
		wg.Add(1)
		go func() {
			wg.Wait()
			config.ImportAvailable = true

			nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
			// TODO should probably be in the index - needs to udpate when index changes
			summary = &Summary{nsrc, nauth, npkg, nprov}

			// Do this now so changing the level doesn't interfere with index load
			log.Logger().SetLevel(log.Stol(config.LogLevel))
		}()
		defer wg.Done()
		// Create in-memory indexes for UI/search etc
		for fname, _ := range indexes {
			for idn, idx := range indexes[fname] {
				mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
				for _, auth := range idx.Authors {
					// author name
					if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
						mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
						mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth

					// wildcards
					if _, ok := mapped[idx.Name]["*"]; !ok {
						mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
						mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name]["*"]["**"][auth.Name] = auth

					// combos
					if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
						mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
						mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
					mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth

					for _, pkg := range auth.Packages {
						filemap[pkg.AuthorURL()] = idn
						for _, prov := range pkg.Provides {
							parts := strings.Split(prov.Name, "::")
							log.Trace("PACKAGE: %s", prov.Name)

							if _, ok := packages[parts[0]]; !ok {
								packages[parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if _, ok := idxpackages[idx.Name]; !ok {
								idxpackages[idx.Name] = make(map[string]*PkgSpace)
							}
							if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
								idxpackages[idx.Name][parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if len(parts) == 1 {
								packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
								packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
								idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
							} else {
								packages[parts[0]].Populate(parts[1:], prov)
								idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
							}
						}
					}
				}
			}
		}
	}
	go update_indexes()

	// Get latest SmartPAN version
	go func() {
		res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases")
		if err != nil {
			log.Error("Error getting latest version: %s", err.Error())
			return
		}
		defer res.Body.Close()
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Error("Error reading stream: %s", err.Error())
			return
		}

		var r Releases
		if err = json.Unmarshal(b, &r); err != nil {
			log.Error("Error unmarshalling JSON: %s", err.Error())
			return
		}

		log.Info("Current release: %s", config.CurrentRelease)
		rel := strings.TrimPrefix(r[0].TagName, "v")
		log.Info("Latest release: %s", rel)
		config.LatestRelease = rel
		config.UpdateURL = r[0].URL

		if config.CurrentRelease < rel {
			config.CanUpdate = true
			log.Info("Your version of SmartPAN can be updated.")
		}
	}()

	// Create our Gotcha application
	var app = gotcha.Create(Asset)
	app.Config.Listen = config.Bind

	summary = &Summary{0, 0, 0, 0}

	app.On(events.BeforeHandler, func(session *http.Session, next func()) {
		session.Stash["summary"] = summary
		session.Stash["config"] = config

		next()
	})

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", search)
	r.Post("/", search)

	r.Get("/help", help)
	r.Get("/settings", settings)
	r.Get("/browse", browse)

	r.Get("/import", import1)
	r.Post("/import", import1)

	r.Get("/import/(?P<jobid>[^/]+)", import2)
	r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)

	r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	// JSON endpoints
	r.Get("/where/(?P<module>[^/]+)/?", where)
	r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)

	// Put these last so they only match /{repo} if nothing else matches
	r.Get("/(?P<repo>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
	r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
	r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)

	// Start our application
	app.Start()

	<-make(chan int)
}