func (r *Response) Chunked() chan []byte {
c := make(chan []byte)
r.IsChunked = true
r.Send()
go func() {
for b := range c {
if len(b) == 0 {
log.Trace("Chunk stream ended")
if r.Gzipped {
r.gzwriter.Close()
}
break
}
log.Trace("Writing chunk: %d bytes", len(b))
if r.Gzipped {
r.gzwriter.Write(b)
} else {
r.Write(b)
}
if f, ok := r.writer.(nethttp.Flusher); ok {
f.Flush()
}
}
}()
return c
}
func (h *Router) Static(filename string) HandlerFunc {
return func(session *http.Session) {
// TODO beware of ..?
re := regexp.MustCompile("{{(\\w+)}}")
fcopy := re.ReplaceAllStringFunc(filename, func(m string) string {
parts := re.FindStringSubmatch(m)
log.Trace("Found var: %s; name: %s", m, parts[1])
if val, ok := session.Stash[parts[1]]; ok {
log.Trace("Value found in stash for %s: %s", parts[1], val)
return val.(string)
}
log.Trace("No value found in stash for var: %s", parts[1])
return m
})
asset, err := h.Config.AssetLoader(fcopy)
if err != nil {
log.Debug("Static file not found: %s", fcopy)
session.RenderNotFound()
} else {
m := MIME.TypeFromFilename(fcopy)
if len(m) > 0 {
log.Debug("Setting Content-Type: %s", m)
session.Response.Headers.Add("Content-Type", m[0])
}
session.Response.Write(asset)
}
}
}
func ParseCPANLines(lines []string) (*CPANFile, error) {
cpanfile := &CPANFile{}
for _, l := range lines {
if len(l) == 0 {
continue
}
log.Trace("Parsing line: %s", l)
dep, err := ParseCPANLine(l)
if err != nil {
log.Error("=> Error parsing line: %s", err)
continue
}
if dep != nil {
log.Info("=> Found dependency: %s", dep)
cpanfile.AddDependency(dep)
continue
}
log.Trace("=> No error and no dependency found")
}
log.Info("Found %d dependencies in cpanfile", len(cpanfile.Dependencies))
return cpanfile, nil
}
func (fh *FormHelper) parseRules() {
s := reflect.TypeOf(fh.Model).Elem()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i).Name
tag := s.Field(i).Tag
if len(tag) > 0 {
log.Trace("Model field [%s] has validation tag: %s", f, tag)
s := strings.Split(string(tag), ";")
for _, rule := range s {
p := strings.SplitN(rule, ":", 2)
if len(p) == 1 {
rule = strings.TrimSpace(p[0])
log.Trace("Found rule [%s]", rule)
} else {
rule = strings.TrimSpace(p[0])
params := strings.TrimSpace(p[1])
log.Trace("Found rule [%s] with parameters [%s]", rule, params)
if _, ok := fh.Rules[f]; !ok {
fh.Rules[f] = make([]*Rule, 0)
}
switch rule {
case "minlength":
fh.Rules[f] = append(fh.Rules[f], MinLength(params))
case "maxlength":
fh.Rules[f] = append(fh.Rules[f], MaxLength(params))
}
}
}
}
}
}
func (fh *FormHelper) Populate(multipart bool) *FormHelper {
// TODO nested form values
t := reflect.TypeOf(fh.Model)
v := reflect.ValueOf(fh.Model)
s := t.Elem()
w := v.Elem()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i).Name
fl := strings.ToLower(f)
log.Trace("Model field [%s] mapped to form field [%s]", f, fl)
var val []string
if multipart {
val = fh.Session.Request.MultipartForm().Value[fl]
} else {
val = fh.Session.Request.PostForm()[fl]
}
if len(val) > 0 {
fh.Values[f] = val[0]
w.Field(i).SetString(val[0])
log.Trace("Field [%s] had value [%s]", fl, fh.Values[fl])
}
}
return fh
}
func SaveIndex(index string, indexes map[string]*Source) {
// TODO append, but needs to know which stuff is new
//out, err := os.OpenFile(".gopancache/index", os.O_RDWR|os.O_APPEND, 0660)
out, err := os.Create(index)
if err != nil {
log.Error("Error creating index: %s", err.Error())
}
for _, source := range indexes {
out.Write([]byte(source.Name + " [" + source.URL + "]\n"))
log.Trace(source.Name)
for _, author := range source.Authors {
out.Write([]byte(" " + author.Name + " [" + author.URL + "]\n"))
log.Trace(" %s", author.Name)
for _, pkg := range author.Packages {
out.Write([]byte(" " + pkg.Name + " => " + pkg.URL + "\n"))
log.Trace(" %s => %s", pkg.Name, pkg.URL)
for p, pk := range pkg.Provides {
out.Write([]byte(" " + p + " (" + pk.Version + "): " + pk.File + "\n"))
log.Trace(" %s (%s): %s", p, pk.Version, pk.File)
}
}
}
}
out.Close()
}
func (fh *FormHelper) Validate() *FormHelper {
for k, f := range fh.Rules {
log.Trace("Validating field [%s] with value [%s]", k, fh.Values[k])
valid := true
for _, r := range f {
log.Trace("Executing rule [%s] with parameters [%s]", r.Name, r.Parameters)
err := r.Function(fh.Values[k])
if err != nil {
log.Trace("Rule [%s] failed: %s", r.Name, err)
fh.HasErrors = true
valid = false
if _, ok := fh.Errors[k]; !ok {
fh.Errors[k] = make(map[string]error)
}
fh.Errors[k][r.Name] = err
} else {
log.Trace("Rule [%s] passed", r.Name)
}
}
log.Trace("Field [%s] is valid: %t", k, valid)
}
return fh
}
func (p *Package) Version() float64 {
if p.cachedVer > 0 {
return p.cachedVer
}
v := ""
// try and match against a provided package
for _, prov := range p.Provides {
if len(prov.Version) > 0 && prov.Version != "undef" {
if len(v) == 0 {
log.Trace("No version cached, using first version found [%s] from [%s]", prov.Version, prov.Name)
v = prov.Version
} else {
if p.Name == strings.Replace(prov.Name, "::", "-", -1)+"-"+prov.Version {
log.Trace("Version cached but found better match, using [%s] from [%s]", prov.Version, prov.Name)
v = prov.Version
}
}
}
}
if len(v) == 0 {
matches := fnToVer.FindStringSubmatch(p.Name)
if len(matches) >= 3 {
log.Trace("Found regex match: %s", matches[2])
v = matches[2]
}
}
p.cachedVer = VersionFromString(v)
return p.cachedVer
}
func (m *Module) getCmd() *exec.Cmd {
var c *exec.Cmd
if _, ok := config.Test.Modules[m.Name]; ok || config.Test.Global {
log.Trace("Executing cpanm install without --notest flag for %s", m.Cached)
c = exec.Command("cpanm", "-L", config.InstallDir, m.Cached)
} else {
log.Trace("Executing cpanm install with --notest flag for %s", m.Cached)
c = exec.Command("cpanm", "--notest", "-L", config.InstallDir, m.Cached)
}
return c
}
func (r *Response) Send() {
if r.headerSent {
return
}
r.headerSent = true
r.SessionID()
r.writeSessionData()
for k, v := range r.Headers {
for _, h := range v {
log.Trace("Adding header [%s]: [%s]", k, h)
r.writer.Header().Add(k, h)
}
}
for _, c := range r.Cookies {
nethttp.SetCookie(r.writer, c)
}
r.writer.WriteHeader(r.Status)
if r.Gzipped {
r.gzwriter.Write(r.buffer.Bytes())
} else {
r.writer.Write(r.buffer.Bytes())
}
if !r.IsChunked && !r.IsEventStream {
if r.Gzipped {
r.gzwriter.Close()
}
}
}
func (session *Session) render(asset string) error {
asset = "assets/templates/" + asset
var t *template.Template
c, ok := session.Config.Cache["template:"+asset]
if !ok {
log.Trace("Loading asset: %s", asset)
a, err := session.Config.AssetLoader(asset)
log.Trace("Creating template: %s", asset)
t = template.New(asset)
t.Delims(session.Config.LeftDelim, session.Config.RightDelim)
if err != nil || a == nil {
log.Error("Failed loading template %s: %s", asset, err)
return err
}
log.Trace("Parsing template: %s", asset)
_, err = t.Parse(string(a))
if err != nil {
log.Error("Failed parsing template %s: %s", asset, err)
return err
}
log.Trace("Template parsed successfully: %s", asset)
session.Config.Cache["template:"+asset] = t
} else {
t = c.(*template.Template)
log.Trace("Template loaded from cache: %s", asset)
}
var b bytes.Buffer
err := t.Execute(&b, session.Stash)
if err != nil {
log.Error("Failed executing template %s: %s", asset, err)
return err
}
_, err = session.Response.Write(b.Bytes())
if err != nil {
log.Error("Error writing output for template %s: %s", asset, err)
return err
}
return nil
}
func (d *DependencyList) AddDependency(dep *Dependency) {
if _, ok := perl_core[dep.Name]; ok {
log.Trace("Dependency " + dep.Name + " is from perl core")
return
}
if d.Dependencies == nil {
d.Dependencies = make([]*Dependency, 0)
}
d.Dependencies = append(d.Dependencies, dep)
}
func ParseCPANLine(line string) (*Dependency, error) {
if len(line) == 0 {
return nil, nil
}
matches := re.FindStringSubmatch(line)
if len(matches) == 0 {
log.Trace("Unable to parse line: %s", line)
return nil, nil
}
module := matches[2]
version := strings.Replace(matches[4], " ", "", -1)
comment := matches[5]
dependency, err := DependencyFromString(module, version)
if strings.HasPrefix(strings.Trim(comment, " "), "# REQS: ") {
comment = strings.TrimPrefix(strings.Trim(comment, " "), "# REQS: ")
log.Trace("Found additional dependencies: %s", comment)
for _, req := range strings.Split(comment, ";") {
req = strings.Trim(req, " ")
bits := strings.Split(req, "-")
new_dep, err := DependencyFromString(bits[0], bits[1])
if err != nil {
log.Error("Error parsing REQS dependency: %s", req)
continue
}
log.Trace("Added dependency: %s", new_dep)
dependency.Additional = append(dependency.Additional, new_dep)
}
}
if err != nil {
return nil, err
}
log.Info("%s (%s %s)", module, dependency.Modifier, dependency.Version)
return dependency, err
}
func (h *Router) Serve(session *http.Session) {
for _, route := range h.Routes {
if matches := route.Route.Pattern.FindStringSubmatch(session.Request.URL.Path); len(matches) > 0 {
_, ok := route.Route.Methods[session.Request.Method]
if ok {
for i, named := range route.Route.Pattern.SubexpNames() {
if len(named) > 0 {
log.Trace("Matched named pattern '%s': %s", named, matches[i])
session.Stash[named] = matches[i]
}
}
session.Route = route.Route
defer func() {
if e := recover(); e != nil {
switch e.(type) {
case string:
session.RenderException(500, errors.New(e.(string)))
default:
session.RenderException(500, e.(error))
}
session.Response.Send()
session.Response.Close()
}
}()
var wg sync.WaitGroup
wg.Add(1)
// func() will be executed only if *all* event handlers call next()
h.Config.Events.Emit(session, events.BeforeHandler, func() {
route.Handler.ServeHTTP(session)
h.Config.Events.Emit(session, events.AfterHandler, func() {
session.Response.Send()
session.Response.Close()
wg.Done()
})
})
wg.Wait()
return
}
}
}
// no pattern matched; send 404 response
h.Config.Events.Emit(session, events.BeforeHandler, func() {
session.RenderNotFound()
h.Config.Events.Emit(session, events.AfterHandler, func() {
session.Response.Send()
})
})
}
// Populates a package namespace, e.g. constructing each part of the namespace
// when passed []string{'Mojolicious','Plugin','PODRenderer'}
func (p *PkgSpace) Populate(parts []string, pkg *gopan.PerlPackage) {
if len(parts) > 0 {
if _, ok := p.Children[parts[0]]; !ok {
p.Children[parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: p,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if len(parts) == 1 {
p.Children[parts[0]].Packages = append(p.Children[parts[0]].Packages, pkg)
p.Children[parts[0]].Versions[gopan.VersionFromString(pkg.Version)] = pkg
log.Trace("Version linked: %f for %s in %s", gopan.VersionFromString(pkg.Version), pkg.Name, p.Namespace)
} else {
p.Children[parts[0]].Populate(parts[1:], pkg)
}
}
}
func (v *Dependency) MatchesVersion(version string) bool {
dversion := v.Version
dv := gopan.VersionFromString(dversion)
mv := gopan.VersionFromString(version)
valid := false
switch v.Modifier {
case "==":
log.Trace("Matches: %f == %f", mv, dv)
if mv == dv {
valid = true
}
case "<=":
log.Trace("Matches: %f <= %f", mv, dv)
if mv <= dv {
valid = true
}
case ">=":
log.Trace("Matches: %f >= %f", mv, dv)
if mv >= dv {
valid = true
}
case ">":
log.Trace("Matches: %f > %f", mv, dv)
if mv > dv {
valid = true
}
case "<":
log.Trace("Matches: %f < %f", mv, dv)
if mv < dv {
valid = true
}
}
log.Trace("=> Result: %t", valid)
return valid
}
func mirrorPan() {
log.Info("Mirroring *PAN")
// FIXME inefficient
_, _, npkg, _ := gopan.CountIndex(indexes)
mirrored := 0
var pc = func() int {
return mirrored / npkg * 100
}
for fname, _ := range indexes {
log.Debug("File: %s", fname)
for _, source := range indexes[fname] {
log.Debug("Index: %s", source)
wg.Add(1)
go func(source *gopan.Source) {
defer wg.Done()
for _, author := range source.Authors {
log.Debug("=> %s", author)
wg.Add(1)
go func(author *gopan.Author) {
cachedir := config.CacheDir + "/" + source.Name + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
os.MkdirAll(cachedir, 0777)
defer wg.Done()
for _, pkg := range author.Packages {
wg.Add(1)
go func(pkg *gopan.Package) {
defer wg.Done()
cache := cachedir + pkg.Name
log.Trace(" - Caching to: %s", cache)
if _, err := os.Stat(cache); err == nil {
log.Debug("%d%% |> %s", pc(), pkg)
log.Trace(" - Already exists in cache")
mirrored++
return
}
sem <- 1
mirrored++
log.Debug("%d%% => %s", pc(), pkg)
url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" + pkg.Name
log.Trace(" - From URL: %s", url)
out, err := os.Create(cache)
defer out.Close()
if err != nil {
log.Error("CREATE - %s", err.Error())
<-sem
return
}
resp, err := http.Get(url)
if err != nil {
log.Error("HTTP GET - %s", err.Error())
<-sem
return
}
_, err = io.Copy(out, resp.Body)
if err != nil {
log.Error("IO COPY - %s", err.Error())
}
<-sem
}(pkg)
}
}(author)
}
}(source)
}
}
wg.Wait()
log.Info("Finished mirroring *PAN")
}
func (session *Session) Redirect(url *neturl.URL) {
log.Trace("Redirect to: %s", url)
session.Response.Redirect(url, nethttp.StatusFound)
}
func (s *Source) Find(d *Dependency) (*Module, error) {
log.Debug("Finding dependency: %s", d)
switch s.Type {
case "SmartPAN":
log.Debug("=> Using SmartPAN source")
url := s.URL
if !strings.HasSuffix(s.URL, "/") {
url += "/"
}
url += "where/" + d.Name + "/" + d.Modifier + d.Version
log.Info("Query: %s", url)
res, err := http.Get(url)
if err != nil {
log.Error("Error querying SmartPAN: %s", err.Error())
return nil, err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
log.Trace("Got response: %s", string(body))
if res.StatusCode != http.StatusOK {
log.Info("Module not found in SmartPAN: %s", d.Name)
return nil, nil
}
var v *WhereOutput
if err = json.Unmarshal(body, &v); err != nil {
log.Error("Error parsing JSON: %s", err.Error())
return nil, err
}
log.Trace("Found module %s", v.Module)
if len(v.Versions) == 0 {
log.Info("Found module but no versions returned")
return nil, nil
}
var lv *VersionOutput
for _, ver := range v.Versions {
if ver.Version == v.Latest {
log.Info("Using latest version of %s: %f", v.Module, ver.Version)
lv = ver
break
}
}
if lv == nil {
log.Info("Couldn't find latest version, selecting first available")
lv = v.Versions[0]
}
return &Module{
Name: d.Name,
Version: fmt.Sprintf("%f", lv.Version),
Source: s,
Url: lv.URL,
}, nil
case "CPAN":
log.Debug("=> Using CPAN source")
if mod, ok := s.ModuleList[d.Name]; ok {
log.Trace("=> Found in source: %s", mod)
if d.Matches(mod) {
log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
return mod, nil
}
log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
return nil, nil
}
case "BackPAN":
log.Debug("=> Using BackPAN source")
// TODO better version matching - new backpan index?
if mod, ok := s.ModuleList[d.Name+"-"+d.Version]; ok {
log.Trace("=> Found in source: %s", mod)
if d.Matches(mod) {
log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
return mod, nil
}
log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
return nil, nil
}
case "MetaCPAN":
log.Debug("=> Using MetaCPAN source")
var sout, serr bytes.Buffer
var cpanm_args string = fmt.Sprintf("-L %s --info %s~\"%s%s\"", config.InstallDir, d.Name, d.Modifier, d.Version)
cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
if err != nil {
log.Error("Failed to get absolute path of gopan cache directory: %s", err)
return nil, err
}
log.Trace("About to exec: cpanm %s", cpanm_args)
os.Setenv("CPANM_INFO_ARGS", cpanm_args)
os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)
cmd := exec.Command("bash", "-c", `eval cpanm $CPANM_INFO_ARGS`)
cmd.Stdout = &sout
cmd.Stderr = &serr
if err := cmd.Run(); err != nil {
log.Error("cpanm %s: %s,\n%s\n", cpanm_args, err, serr.String())
return nil, nil
}
if 0 == len(sout.String()) {
log.Warn("No author/module from cpanm")
return nil, nil
}
author_module := strings.TrimRight(sout.String(), "\n")
mematches := metacpanRe.FindStringSubmatch(author_module)
if nil == mematches {
log.Error("Match failed for: %s", author_module)
return nil, nil
}
log.Trace("Resolved: %s", author_module)
for _, mesource := range config.MetaSources {
meurl := fmt.Sprintf("authors/id/%s/%s/%s",
mematches[1][0:1],
mematches[1][0:2],
mematches[0])
archive_url := fmt.Sprintf("%s/%s", mesource.URL, meurl)
log.Trace("Checking: " + archive_url)
resp, err := http.Head(archive_url)
if err != nil {
log.Trace(err)
continue
}
log.Trace("HEAD status code: %d", resp.StatusCode)
if 200 == resp.StatusCode {
// No module/version check since 'cpanm --info' may resolve to
// archive and version that may not match source
return &Module{
Name: mematches[2],
Version: mematches[3],
Source: mesource,
Url: meurl,
}, nil
}
}
log.Error("Could not get archive URL via 'cpanm %s'", cpanm_args)
return nil, nil
default:
log.Error("Unrecognised source type: %s", s.Type)
return nil, errors.New(fmt.Sprintf("Unrecognised source: %s", s))
}
log.Trace("=> Not found in source")
return nil, nil
}
func (tr *TaskRun) Start(exitCh chan int) {
tr.Started = time.Now()
stdout, err := tr.Cmd.StdoutPipe()
if err != nil {
tr.Error = err
exitCh <- 1
return
}
stderr, err := tr.Cmd.StderrPipe()
if err != nil {
tr.Error = err
exitCh <- 1
return
}
if len(tr.Stdout) > 0 {
wr, err := NewFileLogWriter(tr.Stdout)
if err != nil {
log.Error("Unable to open file %s: %s", tr.Stdout, err.Error())
tr.StdoutBuf = NewInMemoryLogWriter()
} else {
tr.StdoutBuf = wr
}
} else {
tr.StdoutBuf = NewInMemoryLogWriter()
}
if len(tr.Stderr) > 0 {
wr, err := NewFileLogWriter(tr.Stderr)
if err != nil {
log.Error("Unable to open file %s: %s", tr.Stderr, err.Error())
tr.StderrBuf = NewInMemoryLogWriter()
} else {
tr.StderrBuf = wr
}
} else {
tr.StderrBuf = NewInMemoryLogWriter()
}
if len(tr.Pwd) > 0 {
log.Info("Setting pwd: %s", tr.Pwd)
tr.Cmd.Dir = tr.Pwd
}
for k, v := range tr.Environment {
log.Info("Adding env var %s = %s", k, v)
tr.Cmd.Env = append(tr.Cmd.Env, k+"="+v)
}
err = tr.Cmd.Start()
if tr.Cmd.Process != nil {
ev := &Event{time.Now(), fmt.Sprintf("Process %d started: %s", tr.Cmd.Process.Pid, tr.Command)}
log.Info(ev.Message)
tr.Events = append(tr.Events, ev)
}
if err != nil {
tr.Error = err
log.Error(err.Error())
tr.StdoutBuf.Close()
tr.StderrBuf.Close()
exitCh <- 1
return
}
go func() {
go io.Copy(tr.StdoutBuf, stdout)
go io.Copy(tr.StderrBuf, stderr)
tr.Cmd.Wait()
tr.StdoutBuf.Close()
tr.StderrBuf.Close()
log.Trace("STDOUT: %s", tr.StdoutBuf.String())
log.Trace("STDERR: %s", tr.StderrBuf.String())
ps := tr.Cmd.ProcessState
sy := ps.Sys().(syscall.WaitStatus)
ev := &Event{time.Now(), fmt.Sprintf("Process %d exited with status %d", ps.Pid(), sy.ExitStatus())}
log.Info(ev.Message)
tr.Events = append(tr.Events, ev)
log.Info(ps.String())
tr.Stopped = time.Now()
exitCh <- 1
}()
}
func (d *DependencyList) Install() (int, error) {
if d == nil {
log.Debug("No dependencies to install")
return 0, nil
}
n := 0
if install_semaphore == nil {
install_semaphore = make(chan int, config.CPUs)
}
var wg sync.WaitGroup
var errorLock sync.Mutex
errs := make([]string, 0)
for _, dep := range d.Dependencies {
log.Debug("Installing dependency: %s", dep)
wg.Add(1)
go func(dep *Dependency) {
defer wg.Done()
defer func(mod *Module) {
if mod != nil {
log.Debug("Resuming installation of %s", mod)
}
}(d.Parent)
_, ok1 := global_installed[dep.Module.Cached]
_, ok2 := global_installed[dep.Module.Name+"-"+dep.Module.Version]
if ok1 || ok2 {
log.Trace("Module is already installed: %s", dep.Module)
return
}
log.Trace("Aquiring install lock for module %s", dep.Module)
install_lock.Lock()
if mt, ok := install_mutex[dep.Module.Cached]; ok {
install_lock.Unlock()
log.Trace("Waiting on existing installation for %s", dep.Module)
log.Trace("Path: %s", dep.Module.Path())
mt.Lock()
mt.Unlock()
log.Trace("Existing installation complete for %s", dep.Module)
return
}
log.Trace("Creating new installation lock for module %s", dep.Module)
install_mutex[dep.Module.Cached] = new(sync.Mutex)
install_mutex[dep.Module.Cached].Lock()
//log.Trace("%s:: Sending semaphore", dep.module)
install_semaphore <- 1
install_lock.Unlock()
o, err := dep.Module.Install()
//log.Trace("%s:: Waiting on semaphore", dep.module)
<-install_semaphore
//log.Trace("%s:: Got semaphore", dep.module)
global_installed[dep.Module.Name+"-"+dep.Module.Version] = dep.Module
global_installed[dep.Module.Cached] = dep.Module
global_unique[dep.Module.Name] = 1
n += o
if err != nil {
log.Error("Error installing module: %s", err)
errorLock.Lock()
errs = append(errs, dep.Module.String())
errorLock.Unlock()
}
install_lock.Lock()
install_mutex[dep.Module.Cached].Unlock()
install_lock.Unlock()
n++
}(dep)
}
wg.Wait()
if len(errs) > 0 {
log.Error("Failed to install dependencies:")
for _, err := range errs {
log.Error("=> %s", err)
}
return n, errors.New("Failed to install dependencies")
}
return n, nil
}
// Resolve a dependency (i.e. one module), trying all sources
func (d *Dependency) Resolve(p *Module) error {
if gm, ok := global_modules[d.Name+"-"+d.Version]; ok {
log.Trace("Dependency %s already resolved (S1): %s", d, gm)
d.Module = gm
return nil
}
log.Trace("Resolving dependency: %s", d)
for _, s := range config.Sources {
log.Trace("=> Trying source: %s", s)
m, err := s.Find(d)
if err != nil {
log.Trace("=> Error from source: %s", err)
continue
}
if m != nil {
log.Trace("=> Resolved dependency: %s", m)
d.Module = m
break
}
}
if d.Module == nil {
log.Error("Failed to resolve dependency: %s", d)
return fmt.Errorf("Dependency not found from any source: %s", d)
}
if gm, ok := global_modules[d.Module.Name+"-"+d.Module.Version+"~"+d.Module.Source.URL]; ok {
log.Trace("Dependency %s already resolved (S2): %s", d, gm)
d.Module = gm
} else if gm, ok := global_modules[d.Module.Name]; ok {
log.Trace("Dependency %s already resolved (S3): %s", d, gm)
// See if the already resolved version is acceptable
if !d.MatchesVersion(gm.Version) {
errstr := fmt.Sprintf("Version conflict in dependency tree: %s => %s", d, gm)
log.Error(errstr)
return errors.New(errstr)
}
log.Trace("Version %s matches %s", d.Module, gm.Version)
// TODO See if downloading a new version would be better
d.Module = gm
} else {
log.Debug("Downloading: %s", d.Module)
if err := d.Module.Download(); err != nil {
log.Error("Error downloading module %s: %s", d.Module, err)
return err
}
if p != nil {
if p.IsCircular(d.Module) {
log.Error("Detected circular dependency %s from module %s", d.Module, p)
return fmt.Errorf("Detected circular dependency %s from module %s", d.Module, p)
}
}
// module can't exist because of global_lock
global_modules[d.Module.Name] = d.Module
global_modules[d.Module.Name+"-"+d.Module.Version] = d.Module
global_modules[d.Module.Name+"-"+d.Module.Version+"~"+d.Module.Source.URL] = d.Module
log.Debug("Resolving module dependencies: %s", d.Module)
d.Module.Deps = &DependencyList{
Parent: d.Module,
Dependencies: make([]*Dependency, 0),
}
if d.Additional != nil && len(d.Additional) > 0 {
log.Trace("Adding cpanfile additional REQS")
for _, additional := range d.Additional {
log.Trace("Adding additional dependency from cpanfile: %s", additional)
d.Module.Deps.AddDependency(additional)
}
}
if err := d.Module.loadDependencies(); err != nil {
return err
}
}
return nil
}
func main() {
configure()
indexes = make(map[string]map[string]*gopan.Source)
indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)
log.Logger().SetLevel(log.Stol(config.LogLevel))
log.Info("Using log level: %s", config.LogLevel)
// FIXME inefficient
_, _, tpkg, _ := gopan.CountIndex(indexes)
npkg := 0
nmod := 0
var pc = func() float64 {
return float64(nmod) / float64(tpkg) * 100
}
log.Info("Writing packages index file")
out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
if err != nil {
log.Error("Error creating packages index: %s", err.Error())
return
}
for fname, _ := range indexes {
log.Debug("File: %s", fname)
for _, idx := range indexes[fname] {
log.Debug("Index: %s", idx)
out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
for _, auth := range idx.Authors {
log.Debug("Author %s", auth)
out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
for _, pkg := range auth.Packages {
out.Write([]byte(" " + pkg.Name + " => " + pkg.URL + "\n"))
log.Debug("Package: %s", pkg)
if !config.Flatten {
if len(pkg.Provides) == 0 {
// TODO better handling of filenames
modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")
tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name
if _, err := os.Stat(tgzpath); err != nil {
log.Error("File not found: %s", tgzpath)
return
}
extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name
log.Trace("=> tgzpath: %s", tgzpath)
log.Trace(" > extpath: %s", extpath)
log.Trace(" > dirpath: %s", dirpath)
// Only index packages if they don't already exist
if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
log.Error("Error retrieving package list: %s", err)
continue
}
}
npkg += len(pkg.Provides)
nmod += 1
for p, pk := range pkg.Provides {
out.Write([]byte(" " + p + " (" + pk.Version + "): " + pk.File + "\n"))
}
if nmod > 0 && nmod%100 == 0 {
log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
}
}
}
}
}
}
out.Close()
log.Info("Found %d packages from %d modules", npkg, nmod)
}
func (r *Response) EventStream() chan []byte {
c := make(chan []byte)
r.IsEventStream = true
r.Headers.Add("Content-Type", "text/event-stream")
r.Headers.Add("Cache-Control", "no-cache")
r.Headers.Add("Connection", "keep-alive")
//r.Write([]byte("\n\n"))
r.Send()
hj, ok := r.writer.(nethttp.Hijacker)
if !ok {
log.Warn("Connection unsuitable for hijack")
return nil
}
conn, bufrw, err := hj.Hijack()
if err != nil {
log.Warn("Connection hijack failed")
return nil
}
r.esBufrw = bufrw
r.esConn = conn
go func() {
for b := range c {
if len(b) == 0 {
log.Trace("Event stream ended")
r.esConn.Close()
break
}
lines := strings.Split(string(b), "\n")
data := ""
for _, l := range lines {
data += "data: " + l + "\n"
}
data += "\n"
sz := len(data) + 1
log.Info("Event stream message is %d bytes", sz)
size := fmt.Sprintf("%X", sz)
r.esBufrw.Write([]byte(size + "\r\n"))
lines = strings.Split(data, "\n")
for _, ln := range lines {
r.esBufrw.Write([]byte(ln + "\n"))
}
_, err := r.esBufrw.Write([]byte("\r\n"))
if err != nil {
log.Error("Error writing to connection: %s\n", err)
r.esConn.Close()
break
}
err = r.esBufrw.Flush()
if err != nil {
log.Error("Error flushing buffer: %s\n", err)
r.esConn.Close()
break
}
}
}()
return c
}
func (m *Module) Download() error {
m.Dir = config.CacheDir + "/" + path.Dir(m.Url)
p := strings.TrimSuffix(path.Base(m.Url), ".tar.gz") // FIXME
p = strings.TrimSuffix(p, ".tgz")
m.Extracted = m.Dir + "/" + p
m.Cached = config.CacheDir + "/" + m.Url
log.Trace("Downloading to: %s", m.Dir)
log.Trace("Cached file: %s", m.Cached)
log.Trace("Extracting to: %s", m.Extracted)
log.Trace("Aquiring lock on download: %s", m.Cached)
file_lock.Lock()
if mtx, ok := file_get[m.Cached]; ok {
file_lock.Unlock()
log.Trace("Waiting for existing download: %s", m.Cached)
mtx.Lock()
mtx.Unlock()
log.Trace("Existing download complete: %s", m.Cached)
return nil
} else {
log.Trace("Creating new lock")
file_get[m.Cached] = new(sync.Mutex)
file_get[m.Cached].Lock()
defer file_get[m.Cached].Unlock()
file_lock.Unlock()
log.Trace("Lock aquired: %s", m.Cached)
}
if _, err := os.Stat(m.Cached); err != nil {
os.MkdirAll(m.Dir, 0777)
out, err := os.Create(m.Cached)
if err != nil {
return err
}
url := m.Source.URL + "/" + m.Url
log.Trace("Downloading: %s", url)
resp, err := http.Get(url)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return errors.New("404 not found")
}
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}
c := exec.Command("tar", "-zxf", m.Cached, "-C", m.Dir)
var stdout2 bytes.Buffer
var stderr2 bytes.Buffer
c.Stderr = &stderr2
c.Stdout = &stdout2
if err := c.Start(); err != nil {
return fmt.Errorf("Error extracting %s (%s): %s", m.Name, m.Version, err)
}
if err := c.Wait(); err != nil {
return fmt.Errorf("Error extracting %s %s: %s\nSTDERR:\n%sSTDOUT:\n%s", m.Name, m.Version, err, stderr2.String(), stdout2.String())
}
out.Close()
resp.Body.Close()
log.Trace("File extracted to: %s", m.Extracted)
} else {
log.Trace("File already cached: %s", m.Cached)
}
return nil
}
func getPackages() int {
newpkg := 0
var pl func(*html.Node, *gopan.Source, *gopan.Author)
pl = func(n *html.Node, source *gopan.Source, author *gopan.Author) {
log.Trace("NODE: %s [%s, %s, %s]", n.DataAtom, n.Type, n.Data)
if n.Type == html.ElementNode && n.Data == "a" {
//log.Info("NODE IS ELEMENTNODE")
for _, attr := range n.Attr {
// FIXME stuff that isn't .tar.gz?
if attr.Key == "href" && strings.HasSuffix(attr.Val, ".tar.gz") {
log.Trace("==> HREF: %s", n.FirstChild.Data)
pkg := strings.TrimSuffix(n.FirstChild.Data, "/")
if _, ok := author.Packages[pkg]; !ok {
author.Packages[pkg] = &gopan.Package{
Name: pkg,
Author: author,
URL: author.URL + "/" + pkg,
}
newpkg++
log.Debug("Found package: %s", pkg)
}
}
}
//log.Info("%s", n.Data)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
pl(c, source, author)
}
}
log.Info("Building package list")
for fname, _ := range indexes {
for _, source := range indexes[fname] {
log.Debug("Index: %s", source)
wg.Add(1)
go func(source *gopan.Source) {
defer wg.Done()
for _, author := range source.Authors {
wg.Add(1)
go func(author *gopan.Author) {
defer wg.Done()
sem <- 1
log.Trace("=> %s", author)
url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
log.Trace("Getting URL: %s", url)
res, err := http.Get(url)
if err != nil {
log.Error("HTTP GET - %s", err.Error())
<-sem
return
}
doc, err := html.Parse(res.Body)
if err != nil {
log.Error("HTML PARSE - %s", err.Error())
<-sem
return
}
pl(doc, source, author)
<-sem
}(author)
}
}(source)
}
}
wg.Wait()
log.Info("Finished building package list")
return newpkg
}
func (m *Module) loadDependencies() error {
yml, err := ioutil.ReadFile(m.Extracted + "/META.yml")
if err != nil {
// TODO this isnt an error (it shouldnt make build fail)
log.Error("Error opening META.yml for %s: %s", m.Name, err)
// return nil to prevent build fail
return nil
}
meta := make(map[interface{}]interface{})
err = yaml.Unmarshal(yml, &meta)
if err != nil {
// TODO this isnt a real error, probably
log.Error("Error parsing YAML: %s", err)
// return nil to prevent build fail
return nil
}
if reqs, ok := meta["requires"]; ok {
log.Debug("Found dependencies for module %s", m.Name)
switch reqs.(type) {
case map[interface{}]interface{}:
for req, ver := range reqs.(map[interface{}]interface{}) {
v := float64(0)
switch ver.(type) {
case string:
v = gopan.VersionFromString(ver.(string))
case int:
v = float64(ver.(int))
}
log.Printf("=> %s (%f)", req, v)
dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
if err != nil {
log.Error("Error parsing dependency: %s", err)
continue
}
if _, ok := perl_core[dep.Name]; ok {
log.Trace("Module is from perl core: %s", dep.Name)
continue
}
m.Deps.AddDependency(dep)
}
}
log.Debug("Resolving module dependency list")
if err := m.Deps.Resolve(); err != nil {
log.Error("Error resolving dependency list [%s]: %s", m.Name, err)
return err
}
return nil
}
// FIXME repeat of block above, just with more nested levels
if p, ok := meta["prereqs"]; ok {
if r, ok := p.(map[interface{}]interface{})["runtime"]; ok {
if reqs, ok := r.(map[interface{}]interface{})["requires"]; ok {
log.Debug("Found dependencies for module %s", m.Name)
switch reqs.(type) {
case map[interface{}]interface{}:
for req, ver := range reqs.(map[interface{}]interface{}) {
v := float64(0)
switch ver.(type) {
case string:
v = gopan.VersionFromString(ver.(string))
case int:
v = float64(ver.(int))
}
log.Printf("=> %s (%f)", req, v)
dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
if err != nil {
log.Error("Error parsing dependency: %s", err)
continue
}
if _, ok := perl_core[dep.Name]; ok {
log.Trace("Module is from perl core: %s", dep.Name)
continue
}
m.Deps.AddDependency(dep)
}
}
}
}
if t, ok := p.(map[interface{}]interface{})["test"]; ok {
if reqs, ok := t.(map[interface{}]interface{})["requires"]; ok {
log.Debug("Found dependencies for module %s", m.Name)
switch reqs.(type) {
case map[interface{}]interface{}:
for req, ver := range reqs.(map[interface{}]interface{}) {
v := float64(0)
switch ver.(type) {
case string:
v = gopan.VersionFromString(ver.(string))
case int:
v = float64(ver.(int))
}
log.Printf("=> %s (%f)", req, v)
dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
if err != nil {
log.Error("Error parsing dependency: %s", err)
continue
}
if _, ok := perl_core[dep.Name]; ok {
log.Trace("Module is from perl core: %s", dep.Name)
continue
}
m.Deps.AddDependency(dep)
}
}
}
}
log.Debug("Resolving module dependency list")
if err := m.Deps.Resolve(); err != nil {
log.Error("Error resolving dependency list: %s", err)
return err
}
return nil
}
log.Debug("No dependencies for module %s", m.Name)
return nil
}
func (m *Module) Install() (int, error) {
log.Debug("Installing module: %s", m)
n := 0
if m.Deps != nil {
log.Trace("Installing module dependencies for %s", m)
<-install_semaphore
o, err := m.Deps.Install()
install_semaphore <- 1
n += o
if err != nil {
log.Error("Error installing module dependencies for %s: %s", m, err)
return n, err
}
}
var c *exec.Cmd
var stdout *bytes.Buffer
var stderr *bytes.Buffer
cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
if err != nil {
log.Error("Failed to get absolute path of gopan cache directory: %s", err)
return n, err
}
os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)
done := false
attempts := 0
for !done {
time.Sleep(time.Duration(100) * time.Millisecond)
c = m.getCmd()
stdout = new(bytes.Buffer)
stderr = new(bytes.Buffer)
c.Stderr = stderr
c.Stdout = stdout
// brute force cpanm text file busy errors
attempts++
if err := c.Start(); err != nil {
if attempts > 10 {
log.Error("Error installing module %s: %s", m, err)
return n, err
}
} else {
done = true
}
}
if err := c.Wait(); err != nil {
if !strings.HasPrefix(strings.ToLower(stderr.String()), "plenv: cannot rehash:") && !strings.Contains(strings.ToLower(stderr.String()), "text file busy") &&
!strings.HasPrefix(strings.ToLower(stdout.String()), "plenv: cannot rehash:") && !strings.Contains(strings.ToLower(stdout.String()), "text file busy") {
log.Error(m.Name + "-" + m.Version + " failed to install")
log.Error("Error installing %s %s: %s\nSTDERR:\n%sSTDOUT:\n%s", m.Name, m.Version, err, stderr.String(), stdout.String())
return n, err
}
}
n++
log.Printf("Installed " + m.Name + " (" + m.Version + ")")
return n, nil
}
func Provides(pkg *gopan.Package, tgzpath string, extpath string, dirpath string) error {
// not required? path should already exist
os.MkdirAll(dirpath, 0770)
var stdout1 bytes.Buffer
var stderr1 bytes.Buffer
extract := exec.Command("tar", "-zxf", tgzpath, "-C", dirpath)
extract.Stdout = &stdout1
extract.Stderr = &stderr1
if err := extract.Run(); err != nil {
log.Error("Extract run: %s", err.Error())
log.Trace(stdout1.String())
log.Error(stderr1.String())
return err
}
log.Trace(stdout1.String())
log.Trace(stderr1.String())
defer func() {
var stdout3 bytes.Buffer
var stderr3 bytes.Buffer
clean := exec.Command("rm", "-rf", extpath)
clean.Stdout = &stdout3
clean.Stderr = &stderr3
if err := clean.Run(); err != nil {
log.Error("Clean run: %s", err.Error())
}
log.Trace(stdout3.String())
log.Trace(stderr3.String())
}()
//var stdout2 bytes.Buffer
var stderr2 bytes.Buffer
if len(pldArgs) == 0 {
if len(os.Getenv("GOPAN_ALLOW_DEV_VERSIONS")) > 0 {
pldArgs = "({ALLOW_DEV_VERSION=>1})"
} else {
pldArgs = "()"
}
}
log.Trace("pldArgs: %s", pldArgs)
//cmd := exec.Command("perl", "-MModule::Metadata", "-MJSON::XS", "-e", "print encode_json(Module::Metadata->provides(version => 2, prefix => \"\", dir => $ARGV[0]))", extpath)
cmd := exec.Command("perl", "-MParse::LocalDistribution", "-MJSON::XS", "-e", "print encode_json(Parse::LocalDistribution->new"+pldArgs+"->parse($ARGV[0]))", extpath)
//cmd.Stdout = &stdout2
cmd.Stderr = &stderr2
stdout, err := cmd.StdoutPipe()
defer stdout.Close()
if err != nil {
log.Error("StdoutPipe: %s", err.Error())
return err
}
if err := cmd.Start(); err != nil {
log.Error("Start: %s", err.Error())
return err
}
var pld PLD
if err := json.NewDecoder(stdout).Decode(&pld); err != nil {
log.Error("JSON decoder error: %s", err.Error())
return err
}
if err := cmd.Wait(); err != nil {
log.Error("Wait: %s", err.Error())
return err
}
//log.Trace(stdout2.String())
log.Trace(stderr2.String())
pkg.Provides = make(map[string]*gopan.PerlPackage)
for p, pk := range pld {
pp := &gopan.PerlPackage{
Package: pkg,
Name: p,
Version: pk.Version,
File: pk.Infile,
}
pkg.Provides[p] = pp
log.Trace("%s: %s %s", p, pp.Version, pp.File)
}
log.Debug("%s provides %d packages", pkg, len(pkg.Provides))
return nil
}
func do_import(session *http.Session, job *ImportJob) {
log.Info("Running import job %s", job.Id)
reponame := job.Form.ImportInto
if reponame == "new_index" {
reponame = job.Form.NewIndex
}
msg := func(m string) {
if m != ":DONE" {
job.History = append(job.History, m)
log.Info(m)
}
for _, w := range job.Watchers {
w(m)
}
}
mods := make([]*getpan.Module, 0)
// TODO cpanm mirror when using getpan_import
if len(job.Form.Cpanfile) > 0 {
msg("Parsing cpanfile input")
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
}
if len(job.Form.ImportURL) > 0 {
msg("Importing from URL: " + job.Form.ImportURL)
// TODO support cpanfile urls
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.ImportURL)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
if _, err := os.Stat(nfile); err != nil {
os.MkdirAll(npath, 0777)
out, err := os.Create(nfile)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
url := job.Form.ImportURL
log.Trace("Downloading: %s", url)
resp, err := nethttp.Get(url)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
_, err = io.Copy(out, resp.Body)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
out.Close()
resp.Body.Close()
} else {
log.Trace("File already exists in cache: %s", nfile)
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if len(job.Form.FromDir) > 0 {
msg("Importing from local directory: " + job.Form.FromDir)
// TODO support cpanfile paths
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn := filepath.Split(job.Form.FromDir)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
_, err := CopyFile(nfile, job.Form.FromDir)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
if f, fh, err := session.Request.File("fromfile"); err == nil {
fn := fh.Filename
msg("Importing from uploaded module/cpanfile: " + fn)
if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
msg("Only cpanfile and *.tar.gz files are supported")
msg(":DONE")
job.Complete = true
return
}
if fn == "cpanfile" {
msg("Importing cpanfile")
b, _ := ioutil.ReadAll(f)
f.Close()
job.Form.Cpanfile = string(b)
_, modules := getpan_import(job, msg)
mods = append(mods, modules...)
} else {
msg("Importing .tar.gz")
nauth := job.Form.AuthorID
if len(nauth) < 3 {
// FIXME move to form validation
msg("Author ID must be at least 3 characters")
msg(":DONE")
job.Complete = true
return
}
npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
_, fn = filepath.Split(fn)
nfile := npath + "/" + fn
msg("Caching to " + nfile)
os.MkdirAll(npath, 0777)
_, err := CopyToFile(nfile, f)
if err != nil {
msg(err.Error())
msg(":DONE")
job.Complete = true
return
}
fn = strings.TrimSuffix(fn, ".tar.gz")
bits := strings.Split(fn, "-")
name := strings.Join(bits[0:len(bits)-1], "-")
version := bits[len(bits)-1]
s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
m := &getpan.Module{
Source: s,
Name: name,
Version: version,
Url: "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
Cached: nfile,
Dir: npath,
}
m.Deps = &getpan.DependencyList{
Parent: m,
Dependencies: make([]*getpan.Dependency, 0),
}
mods = append(mods, m)
}
} else {
// there is no file... so no error
//msg("Error importing file upload: " + err.Error())
}
if len(mods) == 0 {
msg("Nothing to do")
msg(":DONE")
job.Complete = true
return
}
msg("Adding modules to GoPAN index")
for _, m := range mods {
msg("=> " + m.Name + " (" + m.Cached + ")")
dn, fn := filepath.Split(m.Cached)
dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
auth := dnb[len(dnb)-1]
ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
npath := ndir + "/" + fn
if _, err := os.Stat(npath); err == nil {
msg(" | Already exists in repository")
} else {
os.MkdirAll(ndir, 0777)
msg(" | Copying to " + npath)
_, err := CopyFile(npath, m.Cached)
if err != nil {
msg(" ! " + err.Error())
continue
}
}
if _, ok := indexes[config.Index][reponame]; !ok {
msg(" | Creating index: " + reponame)
indexes[config.Index][reponame] = &gopan.Source{
Name: reponame,
URL: "/authors/id",
Authors: make(map[string]*gopan.Author),
}
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
msg(" | Creating author: " + auth)
author := &gopan.Author{
Source: indexes[config.Index][reponame],
Name: auth,
Packages: make(map[string]*gopan.Package),
URL: "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
}
indexes[config.Index][reponame].Authors[auth] = author
if _, ok := mapped[reponame]; !ok {
mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
}
// author name
if _, ok := mapped[reponame][author.Name[:1]]; !ok {
mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author
// wildcards
if _, ok := mapped[reponame]["*"]; !ok {
mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"]["**"]; !ok {
mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
}
mapped[reponame]["*"]["**"][author.Name] = author
// combos
if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
}
if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
}
mapped[reponame][author.Name[:1]]["**"][author.Name] = author
mapped[reponame]["*"][author.Name[:2]][author.Name] = author
}
if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
msg(" | Creating module: " + fn)
indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
Author: indexes[config.Index][reponame].Authors[auth],
Name: fn,
URL: indexes[config.Index][reponame].Authors[auth].URL + fn,
Provides: make(map[string]*gopan.PerlPackage),
}
msg(" | Getting list of packages")
modnm := strings.TrimSuffix(fn, ".tar.gz")
pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
}
//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
msg(" | Adding packages to index")
if _, ok := idxpackages[reponame]; !ok {
idxpackages[reponame] = make(map[string]*PkgSpace)
}
filemap[pkg.AuthorURL()] = reponame
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
if _, ok := packages[parts[0]]; !ok {
packages[parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if _, ok := idxpackages[reponame][parts[0]]; !ok {
idxpackages[reponame][parts[0]] = &PkgSpace{
Namespace: parts[0],
Packages: make([]*gopan.PerlPackage, 0),
Children: make(map[string]*PkgSpace),
Parent: nil,
Versions: make(map[float64]*gopan.PerlPackage),
}
}
if len(parts) == 1 {
packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
} else {
packages[parts[0]].Populate(parts[1:], prov)
idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
}
}
msg(" | Writing to index file")
gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
}
msg(" | Imported module")
}
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
msg(":DONE")
job.Complete = true
}
