func where(session *http.Session) {
module := session.Stash["module"].(string)
log.Info("Looking for module: %s", module)
ns := strings.Split(module, "::")
if _, ok := packages[ns[0]]; !ok {
log.Info("Top-level namespace [%s] not found", ns[0])
session.Response.Status = 404
session.Response.Send()
return
}
mod := packages[ns[0]]
ns = ns[1:]
for len(ns) > 0 {
if _, ok := mod.Children[ns[0]]; !ok {
log.Info("Child namespace [%s] not found", ns[0])
session.Response.Status = 404
session.Response.Send()
return
}
log.Info("Found child namespace [%s]", ns[0])
mod = mod.Children[ns[0]]
ns = ns[1:]
}
var version string
if _, ok := session.Stash["version"]; ok {
version = session.Stash["version"].(string)
if strings.HasPrefix(version, "v") {
version = strings.TrimPrefix(version, "v")
}
log.Info("Looking for version: %s", version)
}
if len(mod.Versions) == 0 {
log.Info("Module has no versions in index")
session.Response.Status = 404
session.Response.Send()
return
}
versions := make([]*VersionOutput, 0)
lv := float64(0)
if len(version) > 0 {
if ">0" == version {
// Take account of packages that have a version of 'undef'
for _, pkg := range mod.Packages {
packageURL := pkg.Package.VirtualURL()
urlmatches := packageUrlRe.FindStringSubmatch(packageURL)
if nil == urlmatches {
log.Info("Version requested [%s] not found", version)
session.Response.Status = 404
session.Response.Send()
return
}
ver := gopan.VersionFromString(urlmatches[2])
log.Info("Found version: %f", ver)
versions = append(versions, &VersionOutput{
Index: pkg.Package.Author.Source.Name,
URL: packageURL,
Path: pkg.Package.AuthorURL(),
Version: ver,
})
if ver > lv {
lv = ver
}
}
} else {
dep, _ := getpan.DependencyFromString(module, version)
for _, md := range mod.Versions {
var sver string = md.Version
if `undef` == md.Version {
sver = fmt.Sprintf("%.2f", md.Package.Version())
}
log.Info("Matching [%s] against derived version [%s] (md.Version [%s], md.Package.Version [%f])", dep.Version, sver, md.Version, md.Package.Version())
if dep.MatchesVersion(sver) {
vout := &VersionOutput{
Index: md.Package.Author.Source.Name,
URL: md.Package.VirtualURL(),
Path: md.Package.AuthorURL(),
Version: gopan.VersionFromString(sver),
}
versions = append(versions, vout)
ver := gopan.VersionFromString(sver)
if ver > lv {
lv = ver
}
}
}
}
if len(versions) == 0 {
log.Info("Version requested [%s] not found", version)
session.Response.Status = 404
session.Response.Send()
return
}
} else {
for v, pkg := range mod.Versions {
log.Info("Found version: %f", v)
versions = append(versions, &VersionOutput{
Index: pkg.Package.Author.Source.Name,
URL: pkg.Package.VirtualURL(),
Path: pkg.Package.AuthorURL(),
Version: gopan.VersionFromString(pkg.Version),
})
if v > lv {
lv = v
}
}
}
session.Response.Headers.Set("Content-Type", "application/json")
o := &WhereOutput{
Module: mod.FullName(),
Latest: lv,
Versions: versions,
}
b, err := json.MarshalIndent(o, "", " ")
log.Info("Output: %s", string(b))
if err != nil {
log.Error("Failed encoding JSON: %s", err.Error())
session.Response.Status = 500
session.Response.Send()
return
}
session.Response.Status = 200
session.Response.Write(b)
}
func main() {
config = getpan.Configure()
config.Dump()
mods := flag.Args()
if len(mods) == 0 {
if _, err := os.Stat(config.CPANFile); os.IsNotExist(err) {
log.Error("cpanfile not found: %s", config.CPANFile)
os.Exit(1)
}
}
if len(mods) > 0 && mods[0] == "exec" {
log.Debug("getpan exec => " + strings.Join(mods[1:], " "))
cmd := exec.Command(mods[1], mods[2:]...)
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "PERL5LIB="+config.InstallDir+"/lib/perl5")
cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")+":"+config.InstallDir+"/bin")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
// debug so it doesn't show up in stdout/stderr unless -loglevel is used
log.Debug("Error in exec: %s", err.Error())
os.Exit(10)
}
return
}
for _, source := range config.Sources {
err := source.Load()
if err != nil {
log.Error("Error loading sources: %s", err)
os.Exit(1)
return
}
}
var deps *getpan.DependencyList
if len(mods) == 0 {
log.Info("Installing from cpanfile: %s", config.CPANFile)
d, err := getpan.ParseCPANFile(config.CPANFile)
if err != nil {
log.Error("Error parsing cpanfile: %s", err)
os.Exit(2)
return
}
deps = &d.DependencyList
} else {
log.Info("Installing from command line args")
deps = &getpan.DependencyList{
Dependencies: make([]*getpan.Dependency, 0),
}
for _, arg := range mods {
dependency, err := getpan.DependencyFromString(arg, "")
if err != nil {
log.Error("Unable to parse input: %s", arg)
continue
}
deps.AddDependency(dependency)
}
}
err := deps.Resolve()
if err != nil {
log.Error("Error resolving dependencies: %s", err)
os.Exit(3)
return
}
if false == config.NoDepdump {
log.Info("Resolved dependency tree:")
deps.PrintDeps(0)
}
if config.NoInstall {
log.Info("Skipping installation phase")
return
}
_, err = deps.Install()
if err != nil {
log.Error("Error installing dependencies: %s", err)
os.Exit(4)
return
}
// FIXME hacky, need a better way of tracking installed deps
log.Info("Successfully installed %d modules", deps.UniqueInstalled())
}
func main() {
configure()
args := flag.Args()
if len(args) > 0 && args[0] == "init" {
log.Info("Initialising SmartPAN")
log.Info("=> Installing Perl dependencies")
// FIXME most of this is repeated from getpan/main.go
cfg := getpan.DefaultConfig()
cfg.CacheDir = config.CacheDir
for _, source := range cfg.Sources {
if err := source.Load(); err != nil {
log.Error("Error loading sources: %s", err)
os.Exit(1)
return
}
}
deps := &getpan.DependencyList{
Dependencies: make([]*getpan.Dependency, 0),
}
d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "")
d2, _ := getpan.DependencyFromString("JSON::XS", "")
deps.AddDependency(d1)
deps.AddDependency(d2)
if err := deps.Resolve(); err != nil {
log.Error("Error resolving dependencies: %s", err)
os.Exit(1)
return
}
_, err := deps.Install()
if err != nil {
log.Error("Error installing dependencies: %s", err)
os.Exit(2)
return
}
log.Info(" - Installed %d modules", deps.UniqueInstalled())
log.Info("SmartPAN initialisation complete")
return
}
if config.TestDeps {
perldeps := gopan.TestPerlDeps()
perldeps.Dump()
if !perldeps.Ok {
log.Error("Required perl dependencies are missing")
os.Exit(1)
return
}
}
if len(args) > 0 && args[0] == "import" {
if len(args) < 4 {
log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX")
return
}
fname := args[1]
log.Info("Importing module from %s", fname)
log.Info("Author ID: %s", args[2])
log.Info("Index : %s", args[3])
extraParams := map[string]string{
"importinto": args[3],
"authorid": args[2],
"newindex": "",
"cpanmirror": "",
"importurl": "",
"fromdir": "",
}
if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
log.Info("URL: %s", fname)
extraParams["importurl"] = fname
request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams)
if err != nil {
log.Error("Create request error: %s", err.Error())
return
}
client := &nethttp.Client{}
resp, err := client.Do(request)
if err != nil {
log.Error("Error connecting to host: %s", err.Error())
return
} else {
// TODO stream this
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
log.Error("Error reading response: %s", err.Error())
return
}
resp.Body.Close()
//log.Info("%d", resp.StatusCode)
//log.Info("%s", resp.Header)
log.Info("%s", body.String())
}
} else {
fname = strings.TrimPrefix(fname, "file://")
log.Info("File: %s", fname)
if _, err := os.Stat(fname); err != nil {
log.Error("File not found: %s", err.Error())
return
}
request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname)
if err != nil {
log.Error("Create upload error: %s", err.Error())
return
}
client := &nethttp.Client{}
resp, err := client.Do(request)
if err != nil {
log.Error("Error connecting to host: %s", err.Error())
return
} else {
// TODO stream this
body := &bytes.Buffer{}
_, err := body.ReadFrom(resp.Body)
if err != nil {
log.Error("Error reading response: %s", err.Error())
return
}
resp.Body.Close()
//log.Info("%d", resp.StatusCode)
//log.Info("%s", resp.Header)
log.Info("%s", body.String())
}
}
return
}
config.CurrentRelease = CurrentRelease
var wg sync.WaitGroup
load_index = func(index string, file string) {
indexes[index] = gopan.LoadIndex(file)
}
wg.Add(1)
go func() {
defer wg.Done()
indexes = make(map[string]map[string]*gopan.Source)
// Load CPAN index
if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil {
config.HasCPANIndex = true
config.CPANIndexDate = fi.ModTime().String()
config.CPANStatus = "Loading"
wg.Add(1)
go func() {
defer wg.Done()
load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex)
config.CPANStatus = "Loaded"
}()
}
// Load BackPAN index
if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil {
config.HasBackPANIndex = true
config.BackPANIndexDate = fi.ModTime().String()
config.BackPANStatus = "Loading"
wg.Add(1)
go func() {
defer wg.Done()
load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex)
config.BackPANStatus = "Loaded"
}()
}
// Load our secondary indexes
for _, idx := range config.Indexes {
wg.Add(1)
go func() {
defer wg.Done()
load_index(idx, config.CacheDir+"/"+idx)
}()
}
// Load our primary index (this is the only index written back to)
wg.Add(1)
go func() {
defer wg.Done()
load_index(config.Index, config.CacheDir+"/"+config.Index)
}()
}()
update_indexes = func() {
wg.Wait()
wg.Add(1)
go func() {
wg.Wait()
config.ImportAvailable = true
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
// Do this now so changing the level doesn't interfere with index load
log.Logger().SetLevel(log.Stol(config.LogLevel))
}()
defer wg.Done()
// Create in-memory indexes for UI/search etc
for fname, _ := range indexes {
for idn, idx := range indexes[fname] {
mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
for _, auth := range idx.Authors {
// author name
if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth
// wildcards
if _, ok := mapped[idx.Name]["*"]; !ok {
mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
}
mapped[idx.Name]["*"]["**"][auth.Name] = auth
// combos
if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
}
if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth
for _, pkg := range auth.Packages {
filemap[pkg.AuthorURL()] = idn
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
log.Trace("PACKAGE: %s", prov.Name)
if _, ok := packages[parts[0]]; !ok {
packages[parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if _, ok := idxpackages[idx.Name]; !ok {
idxpackages[idx.Name] = make(map[string]*PkgSpace)
}
if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
idxpackages[idx.Name][parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if len(parts) == 1 {
packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
} else {
packages[parts[0]].Populate(parts[1:], prov)
idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
}
}
}
}
}
}
}
go update_indexes()
// Get latest SmartPAN version
go func() {
res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases")
if err != nil {
log.Error("Error getting latest version: %s", err.Error())
return
}
defer res.Body.Close()
b, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Error("Error reading stream: %s", err.Error())
return
}
var r Releases
if err = json.Unmarshal(b, &r); err != nil {
log.Error("Error unmarshalling JSON: %s", err.Error())
return
}
log.Info("Current release: %s", config.CurrentRelease)
rel := strings.TrimPrefix(r[0].TagName, "v")
log.Info("Latest release: %s", rel)
config.LatestRelease = rel
config.UpdateURL = r[0].URL
if config.CurrentRelease < rel {
config.CanUpdate = true
log.Info("Your version of SmartPAN can be updated.")
}
}()
// Create our Gotcha application
var app = gotcha.Create(Asset)
app.Config.Listen = config.Bind
summary = &Summary{0, 0, 0, 0}
app.On(events.BeforeHandler, func(session *http.Session, next func()) {
session.Stash["summary"] = summary
session.Stash["config"] = config
next()
})
// Get the router
r := app.Router
// Create some routes
r.Get("/", search)
r.Post("/", search)
r.Get("/help", help)
r.Get("/settings", settings)
r.Get("/browse", browse)
r.Get("/import", import1)
r.Post("/import", import1)
r.Get("/import/(?P<jobid>[^/]+)", import2)
r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)
r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex)
// Serve static content (but really use a CDN)
r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))
// JSON endpoints
r.Get("/where/(?P<module>[^/]+)/?", where)
r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)
// Put these last so they only match /{repo} if nothing else matches
r.Get("/(?P<repo>[^/]+)/?", browse)
r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)
// Start our application
app.Start()
<-make(chan int)
}
