Exemple #1
0
func (r *Response) Chunked() chan []byte {
	c := make(chan []byte)
	r.IsChunked = true
	r.Send()
	go func() {
		for b := range c {
			if len(b) == 0 {
				log.Trace("Chunk stream ended")
				if r.Gzipped {
					r.gzwriter.Close()
				}
				break
			}
			log.Trace("Writing chunk: %d bytes", len(b))
			if r.Gzipped {
				r.gzwriter.Write(b)
			} else {
				r.Write(b)
			}

			if f, ok := r.writer.(nethttp.Flusher); ok {
				f.Flush()
			}
		}
	}()
	return c
}
Exemple #2
0
func (h *Router) Static(filename string) HandlerFunc {
	return func(session *http.Session) {
		// TODO beware of ..?
		re := regexp.MustCompile("{{(\\w+)}}")
		fcopy := re.ReplaceAllStringFunc(filename, func(m string) string {
			parts := re.FindStringSubmatch(m)
			log.Trace("Found var: %s; name: %s", m, parts[1])
			if val, ok := session.Stash[parts[1]]; ok {
				log.Trace("Value found in stash for %s: %s", parts[1], val)
				return val.(string)
			}
			log.Trace("No value found in stash for var: %s", parts[1])
			return m
		})
		asset, err := h.Config.AssetLoader(fcopy)
		if err != nil {
			log.Debug("Static file not found: %s", fcopy)
			session.RenderNotFound()
		} else {
			m := MIME.TypeFromFilename(fcopy)
			if len(m) > 0 {
				log.Debug("Setting Content-Type: %s", m)
				session.Response.Headers.Add("Content-Type", m[0])
			}
			session.Response.Write(asset)
		}
	}
}
Exemple #3
0
func ParseCPANLines(lines []string) (*CPANFile, error) {
	cpanfile := &CPANFile{}

	for _, l := range lines {
		if len(l) == 0 {
			continue
		}

		log.Trace("Parsing line: %s", l)
		dep, err := ParseCPANLine(l)

		if err != nil {
			log.Error("=> Error parsing line: %s", err)
			continue
		}

		if dep != nil {
			log.Info("=> Found dependency: %s", dep)
			cpanfile.AddDependency(dep)
			continue
		}

		log.Trace("=> No error and no dependency found")
	}

	log.Info("Found %d dependencies in cpanfile", len(cpanfile.Dependencies))

	return cpanfile, nil
}
Exemple #4
0
func (fh *FormHelper) parseRules() {
	s := reflect.TypeOf(fh.Model).Elem()

	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i).Name

		tag := s.Field(i).Tag
		if len(tag) > 0 {
			log.Trace("Model field [%s] has validation tag: %s", f, tag)

			s := strings.Split(string(tag), ";")
			for _, rule := range s {
				p := strings.SplitN(rule, ":", 2)
				if len(p) == 1 {
					rule = strings.TrimSpace(p[0])
					log.Trace("Found rule [%s]", rule)
				} else {
					rule = strings.TrimSpace(p[0])
					params := strings.TrimSpace(p[1])
					log.Trace("Found rule [%s] with parameters [%s]", rule, params)
					if _, ok := fh.Rules[f]; !ok {
						fh.Rules[f] = make([]*Rule, 0)
					}
					switch rule {
					case "minlength":
						fh.Rules[f] = append(fh.Rules[f], MinLength(params))
					case "maxlength":
						fh.Rules[f] = append(fh.Rules[f], MaxLength(params))
					}
				}
			}
		}
	}
}
Exemple #5
0
func (fh *FormHelper) Populate(multipart bool) *FormHelper {
	// TODO nested form values

	t := reflect.TypeOf(fh.Model)
	v := reflect.ValueOf(fh.Model)
	s := t.Elem()
	w := v.Elem()

	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i).Name
		fl := strings.ToLower(f)
		log.Trace("Model field [%s] mapped to form field [%s]", f, fl)

		var val []string
		if multipart {
			val = fh.Session.Request.MultipartForm().Value[fl]
		} else {
			val = fh.Session.Request.PostForm()[fl]
		}
		if len(val) > 0 {
			fh.Values[f] = val[0]
			w.Field(i).SetString(val[0])
			log.Trace("Field [%s] had value [%s]", fl, fh.Values[fl])
		}
	}

	return fh
}
Exemple #6
0
func SaveIndex(index string, indexes map[string]*Source) {
	// TODO append, but needs to know which stuff is new
	//out, err := os.OpenFile(".gopancache/index", os.O_RDWR|os.O_APPEND, 0660)
	out, err := os.Create(index)
	if err != nil {
		log.Error("Error creating index: %s", err.Error())
	}
	for _, source := range indexes {
		out.Write([]byte(source.Name + " [" + source.URL + "]\n"))
		log.Trace(source.Name)
		for _, author := range source.Authors {
			out.Write([]byte(" " + author.Name + " [" + author.URL + "]\n"))
			log.Trace("    %s", author.Name)
			for _, pkg := range author.Packages {
				out.Write([]byte("  " + pkg.Name + " => " + pkg.URL + "\n"))
				log.Trace("        %s => %s", pkg.Name, pkg.URL)
				for p, pk := range pkg.Provides {
					out.Write([]byte("   " + p + " (" + pk.Version + "): " + pk.File + "\n"))
					log.Trace("          %s (%s): %s", p, pk.Version, pk.File)
				}
			}
		}
	}
	out.Close()
}
Exemple #7
0
func (fh *FormHelper) Validate() *FormHelper {
	for k, f := range fh.Rules {
		log.Trace("Validating field [%s] with value [%s]", k, fh.Values[k])

		valid := true
		for _, r := range f {
			log.Trace("Executing rule [%s] with parameters [%s]", r.Name, r.Parameters)

			err := r.Function(fh.Values[k])

			if err != nil {
				log.Trace("Rule [%s] failed: %s", r.Name, err)
				fh.HasErrors = true
				valid = false
				if _, ok := fh.Errors[k]; !ok {
					fh.Errors[k] = make(map[string]error)
				}
				fh.Errors[k][r.Name] = err
			} else {
				log.Trace("Rule [%s] passed", r.Name)
			}
		}

		log.Trace("Field [%s] is valid: %t", k, valid)
	}

	return fh
}
Exemple #8
0
func (p *Package) Version() float64 {
	if p.cachedVer > 0 {
		return p.cachedVer
	}

	v := ""

	// try and match against a provided package
	for _, prov := range p.Provides {
		if len(prov.Version) > 0 && prov.Version != "undef" {
			if len(v) == 0 {
				log.Trace("No version cached, using first version found [%s] from [%s]", prov.Version, prov.Name)
				v = prov.Version
			} else {
				if p.Name == strings.Replace(prov.Name, "::", "-", -1)+"-"+prov.Version {
					log.Trace("Version cached but found better match, using [%s] from [%s]", prov.Version, prov.Name)
					v = prov.Version
				}
			}
		}
	}

	if len(v) == 0 {
		matches := fnToVer.FindStringSubmatch(p.Name)
		if len(matches) >= 3 {
			log.Trace("Found regex match: %s", matches[2])
			v = matches[2]
		}
	}

	p.cachedVer = VersionFromString(v)

	return p.cachedVer
}
Exemple #9
0
func (m *Module) getCmd() *exec.Cmd {
	var c *exec.Cmd
	if _, ok := config.Test.Modules[m.Name]; ok || config.Test.Global {
		log.Trace("Executing cpanm install without --notest flag for %s", m.Cached)
		c = exec.Command("cpanm", "-L", config.InstallDir, m.Cached)
	} else {
		log.Trace("Executing cpanm install with --notest flag for %s", m.Cached)
		c = exec.Command("cpanm", "--notest", "-L", config.InstallDir, m.Cached)
	}
	return c
}
Exemple #10
0
func (r *Response) Send() {
	if r.headerSent {
		return
	}
	r.headerSent = true

	r.SessionID()
	r.writeSessionData()

	for k, v := range r.Headers {
		for _, h := range v {
			log.Trace("Adding header [%s]: [%s]", k, h)
			r.writer.Header().Add(k, h)
		}
	}
	for _, c := range r.Cookies {
		nethttp.SetCookie(r.writer, c)
	}

	r.writer.WriteHeader(r.Status)

	if r.Gzipped {
		r.gzwriter.Write(r.buffer.Bytes())
	} else {
		r.writer.Write(r.buffer.Bytes())
	}

	if !r.IsChunked && !r.IsEventStream {
		if r.Gzipped {
			r.gzwriter.Close()
		}
	}
}
Exemple #11
0
func (session *Session) render(asset string) error {
	asset = "assets/templates/" + asset

	var t *template.Template

	c, ok := session.Config.Cache["template:"+asset]
	if !ok {
		log.Trace("Loading asset: %s", asset)
		a, err := session.Config.AssetLoader(asset)
		log.Trace("Creating template: %s", asset)
		t = template.New(asset)
		t.Delims(session.Config.LeftDelim, session.Config.RightDelim)
		if err != nil || a == nil {
			log.Error("Failed loading template %s: %s", asset, err)
			return err
		}
		log.Trace("Parsing template: %s", asset)
		_, err = t.Parse(string(a))
		if err != nil {
			log.Error("Failed parsing template %s: %s", asset, err)
			return err
		}
		log.Trace("Template parsed successfully: %s", asset)
		session.Config.Cache["template:"+asset] = t
	} else {
		t = c.(*template.Template)
		log.Trace("Template loaded from cache: %s", asset)
	}

	var b bytes.Buffer
	err := t.Execute(&b, session.Stash)
	if err != nil {
		log.Error("Failed executing template %s: %s", asset, err)
		return err
	}

	_, err = session.Response.Write(b.Bytes())

	if err != nil {
		log.Error("Error writing output for template %s: %s", asset, err)
		return err
	}

	return nil
}
Exemple #12
0
func (d *DependencyList) AddDependency(dep *Dependency) {
	if _, ok := perl_core[dep.Name]; ok {
		log.Trace("Dependency " + dep.Name + " is from perl core")
		return
	}
	if d.Dependencies == nil {
		d.Dependencies = make([]*Dependency, 0)
	}
	d.Dependencies = append(d.Dependencies, dep)
}
Exemple #13
0
func ParseCPANLine(line string) (*Dependency, error) {
	if len(line) == 0 {
		return nil, nil
	}

	matches := re.FindStringSubmatch(line)
	if len(matches) == 0 {
		log.Trace("Unable to parse line: %s", line)
		return nil, nil
	}

	module := matches[2]
	version := strings.Replace(matches[4], " ", "", -1)
	comment := matches[5]

	dependency, err := DependencyFromString(module, version)

	if strings.HasPrefix(strings.Trim(comment, " "), "# REQS: ") {
		comment = strings.TrimPrefix(strings.Trim(comment, " "), "# REQS: ")
		log.Trace("Found additional dependencies: %s", comment)
		for _, req := range strings.Split(comment, ";") {
			req = strings.Trim(req, " ")
			bits := strings.Split(req, "-")
			new_dep, err := DependencyFromString(bits[0], bits[1])
			if err != nil {
				log.Error("Error parsing REQS dependency: %s", req)
				continue
			}
			log.Trace("Added dependency: %s", new_dep)
			dependency.Additional = append(dependency.Additional, new_dep)
		}
	}

	if err != nil {
		return nil, err
	}

	log.Info("%s (%s %s)", module, dependency.Modifier, dependency.Version)

	return dependency, err
}
Exemple #14
0
func (h *Router) Serve(session *http.Session) {
	for _, route := range h.Routes {
		if matches := route.Route.Pattern.FindStringSubmatch(session.Request.URL.Path); len(matches) > 0 {
			_, ok := route.Route.Methods[session.Request.Method]
			if ok {
				for i, named := range route.Route.Pattern.SubexpNames() {
					if len(named) > 0 {
						log.Trace("Matched named pattern '%s': %s", named, matches[i])
						session.Stash[named] = matches[i]
					}
				}
				session.Route = route.Route
				defer func() {
					if e := recover(); e != nil {
						switch e.(type) {
						case string:
							session.RenderException(500, errors.New(e.(string)))
						default:
							session.RenderException(500, e.(error))
						}
						session.Response.Send()
						session.Response.Close()
					}
				}()
				var wg sync.WaitGroup
				wg.Add(1)
				// func() will be executed only if *all* event handlers call next()
				h.Config.Events.Emit(session, events.BeforeHandler, func() {
					route.Handler.ServeHTTP(session)
					h.Config.Events.Emit(session, events.AfterHandler, func() {
						session.Response.Send()
						session.Response.Close()
						wg.Done()
					})
				})
				wg.Wait()
				return
			}
		}
	}

	// no pattern matched; send 404 response
	h.Config.Events.Emit(session, events.BeforeHandler, func() {
		session.RenderNotFound()
		h.Config.Events.Emit(session, events.AfterHandler, func() {
			session.Response.Send()
		})
	})
}
Exemple #15
0
// Populates a package namespace, e.g. constructing each part of the namespace
// when passed []string{'Mojolicious','Plugin','PODRenderer'}
func (p *PkgSpace) Populate(parts []string, pkg *gopan.PerlPackage) {
	if len(parts) > 0 {
		if _, ok := p.Children[parts[0]]; !ok {
			p.Children[parts[0]] = &PkgSpace{
				Namespace: parts[0],
				Packages:  make([]*gopan.PerlPackage, 0),
				Children:  make(map[string]*PkgSpace),
				Parent:    p,
				Versions:  make(map[float64]*gopan.PerlPackage),
			}
		}
		if len(parts) == 1 {
			p.Children[parts[0]].Packages = append(p.Children[parts[0]].Packages, pkg)
			p.Children[parts[0]].Versions[gopan.VersionFromString(pkg.Version)] = pkg
			log.Trace("Version linked: %f for %s in %s", gopan.VersionFromString(pkg.Version), pkg.Name, p.Namespace)
		} else {
			p.Children[parts[0]].Populate(parts[1:], pkg)
		}
	}
}
Exemple #16
0
func (v *Dependency) MatchesVersion(version string) bool {
	dversion := v.Version

	dv := gopan.VersionFromString(dversion)
	mv := gopan.VersionFromString(version)

	valid := false
	switch v.Modifier {
	case "==":
		log.Trace("Matches: %f == %f", mv, dv)
		if mv == dv {
			valid = true
		}
	case "<=":
		log.Trace("Matches: %f <= %f", mv, dv)
		if mv <= dv {
			valid = true
		}
	case ">=":
		log.Trace("Matches: %f >= %f", mv, dv)
		if mv >= dv {
			valid = true
		}
	case ">":
		log.Trace("Matches: %f > %f", mv, dv)
		if mv > dv {
			valid = true
		}
	case "<":
		log.Trace("Matches: %f < %f", mv, dv)
		if mv < dv {
			valid = true
		}
	}
	log.Trace("=> Result: %t", valid)
	return valid
}
Exemple #17
0
func mirrorPan() {
	log.Info("Mirroring *PAN")

	// FIXME inefficient
	_, _, npkg, _ := gopan.CountIndex(indexes)

	mirrored := 0
	var pc = func() int {
		return mirrored / npkg * 100
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, source := range indexes[fname] {
			log.Debug("Index: %s", source)
			wg.Add(1)
			go func(source *gopan.Source) {
				defer wg.Done()
				for _, author := range source.Authors {
					log.Debug("=> %s", author)
					wg.Add(1)
					go func(author *gopan.Author) {
						cachedir := config.CacheDir + "/" + source.Name + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
						os.MkdirAll(cachedir, 0777)

						defer wg.Done()
						for _, pkg := range author.Packages {
							wg.Add(1)
							go func(pkg *gopan.Package) {
								defer wg.Done()

								cache := cachedir + pkg.Name
								log.Trace("    - Caching to: %s", cache)

								if _, err := os.Stat(cache); err == nil {
									log.Debug("%d%%  |> %s", pc(), pkg)
									log.Trace("    - Already exists in cache")
									mirrored++
									return
								}

								sem <- 1

								mirrored++

								log.Debug("%d%%  => %s", pc(), pkg)

								url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" + pkg.Name
								log.Trace("    - From URL: %s", url)

								out, err := os.Create(cache)
								defer out.Close()
								if err != nil {
									log.Error("CREATE - %s", err.Error())
									<-sem
									return
								}

								resp, err := http.Get(url)
								if err != nil {
									log.Error("HTTP GET - %s", err.Error())
									<-sem
									return
								}

								_, err = io.Copy(out, resp.Body)
								if err != nil {
									log.Error("IO COPY - %s", err.Error())
								}

								<-sem
							}(pkg)
						}
					}(author)
				}
			}(source)
		}
	}

	wg.Wait()
	log.Info("Finished mirroring *PAN")
}
Exemple #18
0
func (session *Session) Redirect(url *neturl.URL) {
	log.Trace("Redirect to: %s", url)
	session.Response.Redirect(url, nethttp.StatusFound)
}
Exemple #19
0
func (s *Source) Find(d *Dependency) (*Module, error) {
	log.Debug("Finding dependency: %s", d)

	switch s.Type {
	case "SmartPAN":
		log.Debug("=> Using SmartPAN source")

		url := s.URL
		if !strings.HasSuffix(s.URL, "/") {
			url += "/"
		}
		url += "where/" + d.Name + "/" + d.Modifier + d.Version

		log.Info("Query: %s", url)
		res, err := http.Get(url)

		if err != nil {
			log.Error("Error querying SmartPAN: %s", err.Error())
			return nil, err
		}

		defer res.Body.Close()

		body, err := ioutil.ReadAll(res.Body)
		log.Trace("Got response: %s", string(body))

		if res.StatusCode != http.StatusOK {
			log.Info("Module not found in SmartPAN: %s", d.Name)
			return nil, nil
		}

		var v *WhereOutput
		if err = json.Unmarshal(body, &v); err != nil {
			log.Error("Error parsing JSON: %s", err.Error())
			return nil, err
		}

		log.Trace("Found module %s", v.Module)

		if len(v.Versions) == 0 {
			log.Info("Found module but no versions returned")
			return nil, nil
		}

		var lv *VersionOutput
		for _, ver := range v.Versions {
			if ver.Version == v.Latest {
				log.Info("Using latest version of %s: %f", v.Module, ver.Version)
				lv = ver
				break
			}
		}
		if lv == nil {
			log.Info("Couldn't find latest version, selecting first available")
			lv = v.Versions[0]
		}

		return &Module{
			Name:    d.Name,
			Version: fmt.Sprintf("%f", lv.Version),
			Source:  s,
			Url:     lv.URL,
		}, nil
	case "CPAN":
		log.Debug("=> Using CPAN source")
		if mod, ok := s.ModuleList[d.Name]; ok {
			log.Trace("=> Found in source: %s", mod)
			if d.Matches(mod) {
				log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
				return mod, nil
			}
			log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
			return nil, nil
		}
	case "BackPAN":
		log.Debug("=> Using BackPAN source")
		// TODO better version matching - new backpan index?
		if mod, ok := s.ModuleList[d.Name+"-"+d.Version]; ok {
			log.Trace("=> Found in source: %s", mod)
			if d.Matches(mod) {
				log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
				return mod, nil
			}
			log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
			return nil, nil
		}
	case "MetaCPAN":
		log.Debug("=> Using MetaCPAN source")

		var sout, serr bytes.Buffer
		var cpanm_args string = fmt.Sprintf("-L %s --info %s~\"%s%s\"", config.InstallDir, d.Name, d.Modifier, d.Version)

		cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
		if err != nil {
			log.Error("Failed to get absolute path of gopan cache directory: %s", err)
			return nil, err
		}

		log.Trace("About to exec: cpanm %s", cpanm_args)
		os.Setenv("CPANM_INFO_ARGS", cpanm_args)
		os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)
		cmd := exec.Command("bash", "-c", `eval cpanm $CPANM_INFO_ARGS`)
		cmd.Stdout = &sout
		cmd.Stderr = &serr

		if err := cmd.Run(); err != nil {
			log.Error("cpanm %s: %s,\n%s\n", cpanm_args, err, serr.String())
			return nil, nil
		}

		if 0 == len(sout.String()) {
			log.Warn("No author/module from cpanm")
			return nil, nil
		}

		author_module := strings.TrimRight(sout.String(), "\n")
		mematches := metacpanRe.FindStringSubmatch(author_module)
		if nil == mematches {
			log.Error("Match failed for: %s", author_module)
			return nil, nil
		}

		log.Trace("Resolved: %s", author_module)
		for _, mesource := range config.MetaSources {

			meurl := fmt.Sprintf("authors/id/%s/%s/%s",
				mematches[1][0:1],
				mematches[1][0:2],
				mematches[0])

			archive_url := fmt.Sprintf("%s/%s", mesource.URL, meurl)

			log.Trace("Checking: " + archive_url)
			resp, err := http.Head(archive_url)
			if err != nil {
				log.Trace(err)
				continue
			}

			log.Trace("HEAD status code: %d", resp.StatusCode)
			if 200 == resp.StatusCode {
				// No module/version check since 'cpanm --info' may resolve to
				// archive and version that may not match source
				return &Module{
					Name:    mematches[2],
					Version: mematches[3],
					Source:  mesource,
					Url:     meurl,
				}, nil
			}

		}
		log.Error("Could not get archive URL via 'cpanm %s'", cpanm_args)
		return nil, nil
	default:
		log.Error("Unrecognised source type: %s", s.Type)
		return nil, errors.New(fmt.Sprintf("Unrecognised source: %s", s))
	}
	log.Trace("=> Not found in source")
	return nil, nil
}
Exemple #20
0
func (tr *TaskRun) Start(exitCh chan int) {
	tr.Started = time.Now()

	stdout, err := tr.Cmd.StdoutPipe()
	if err != nil {
		tr.Error = err
		exitCh <- 1
		return
	}
	stderr, err := tr.Cmd.StderrPipe()
	if err != nil {
		tr.Error = err
		exitCh <- 1
		return
	}

	if len(tr.Stdout) > 0 {
		wr, err := NewFileLogWriter(tr.Stdout)
		if err != nil {
			log.Error("Unable to open file %s: %s", tr.Stdout, err.Error())
			tr.StdoutBuf = NewInMemoryLogWriter()
		} else {
			tr.StdoutBuf = wr
		}
	} else {
		tr.StdoutBuf = NewInMemoryLogWriter()
	}
	if len(tr.Stderr) > 0 {
		wr, err := NewFileLogWriter(tr.Stderr)
		if err != nil {
			log.Error("Unable to open file %s: %s", tr.Stderr, err.Error())
			tr.StderrBuf = NewInMemoryLogWriter()
		} else {
			tr.StderrBuf = wr
		}
	} else {
		tr.StderrBuf = NewInMemoryLogWriter()
	}

	if len(tr.Pwd) > 0 {
		log.Info("Setting pwd: %s", tr.Pwd)
		tr.Cmd.Dir = tr.Pwd
	}

	for k, v := range tr.Environment {
		log.Info("Adding env var %s = %s", k, v)
		tr.Cmd.Env = append(tr.Cmd.Env, k+"="+v)
	}

	err = tr.Cmd.Start()
	if tr.Cmd.Process != nil {
		ev := &Event{time.Now(), fmt.Sprintf("Process %d started: %s", tr.Cmd.Process.Pid, tr.Command)}
		log.Info(ev.Message)
		tr.Events = append(tr.Events, ev)
	}
	if err != nil {
		tr.Error = err
		log.Error(err.Error())
		tr.StdoutBuf.Close()
		tr.StderrBuf.Close()
		exitCh <- 1
		return
	}
	go func() {
		go io.Copy(tr.StdoutBuf, stdout)
		go io.Copy(tr.StderrBuf, stderr)

		tr.Cmd.Wait()

		tr.StdoutBuf.Close()
		tr.StderrBuf.Close()

		log.Trace("STDOUT: %s", tr.StdoutBuf.String())
		log.Trace("STDERR: %s", tr.StderrBuf.String())

		ps := tr.Cmd.ProcessState
		sy := ps.Sys().(syscall.WaitStatus)

		ev := &Event{time.Now(), fmt.Sprintf("Process %d exited with status %d", ps.Pid(), sy.ExitStatus())}
		log.Info(ev.Message)
		tr.Events = append(tr.Events, ev)
		log.Info(ps.String())

		tr.Stopped = time.Now()
		exitCh <- 1
	}()
}
Exemple #21
0
func (d *DependencyList) Install() (int, error) {
	if d == nil {
		log.Debug("No dependencies to install")
		return 0, nil
	}

	n := 0

	if install_semaphore == nil {
		install_semaphore = make(chan int, config.CPUs)
	}

	var wg sync.WaitGroup
	var errorLock sync.Mutex

	errs := make([]string, 0)

	for _, dep := range d.Dependencies {
		log.Debug("Installing dependency: %s", dep)
		wg.Add(1)
		go func(dep *Dependency) {
			defer wg.Done()
			defer func(mod *Module) {
				if mod != nil {
					log.Debug("Resuming installation of %s", mod)
				}
			}(d.Parent)

			_, ok1 := global_installed[dep.Module.Cached]
			_, ok2 := global_installed[dep.Module.Name+"-"+dep.Module.Version]
			if ok1 || ok2 {
				log.Trace("Module is already installed: %s", dep.Module)
				return
			}

			log.Trace("Aquiring install lock for module %s", dep.Module)
			install_lock.Lock()
			if mt, ok := install_mutex[dep.Module.Cached]; ok {
				install_lock.Unlock()
				log.Trace("Waiting on existing installation for %s", dep.Module)
				log.Trace("Path: %s", dep.Module.Path())
				mt.Lock()
				mt.Unlock()
				log.Trace("Existing installation complete for %s", dep.Module)
				return
			}

			log.Trace("Creating new installation lock for module %s", dep.Module)
			install_mutex[dep.Module.Cached] = new(sync.Mutex)
			install_mutex[dep.Module.Cached].Lock()

			//log.Trace("%s:: Sending semaphore", dep.module)
			install_semaphore <- 1
			install_lock.Unlock()

			o, err := dep.Module.Install()
			//log.Trace("%s:: Waiting on semaphore", dep.module)
			<-install_semaphore
			//log.Trace("%s:: Got semaphore", dep.module)

			global_installed[dep.Module.Name+"-"+dep.Module.Version] = dep.Module
			global_installed[dep.Module.Cached] = dep.Module
			global_unique[dep.Module.Name] = 1

			n += o
			if err != nil {
				log.Error("Error installing module: %s", err)
				errorLock.Lock()
				errs = append(errs, dep.Module.String())
				errorLock.Unlock()
			}

			install_lock.Lock()
			install_mutex[dep.Module.Cached].Unlock()
			install_lock.Unlock()

			n++
		}(dep)
	}

	wg.Wait()

	if len(errs) > 0 {
		log.Error("Failed to install dependencies:")
		for _, err := range errs {
			log.Error("=> %s", err)
		}
		return n, errors.New("Failed to install dependencies")
	}

	return n, nil
}
Exemple #22
0
// Resolve a dependency (i.e. one module), trying all sources
func (d *Dependency) Resolve(p *Module) error {
	if gm, ok := global_modules[d.Name+"-"+d.Version]; ok {
		log.Trace("Dependency %s already resolved (S1): %s", d, gm)
		d.Module = gm
		return nil
	}

	log.Trace("Resolving dependency: %s", d)

	for _, s := range config.Sources {
		log.Trace("=> Trying source: %s", s)
		m, err := s.Find(d)
		if err != nil {
			log.Trace("=> Error from source: %s", err)
			continue
		}
		if m != nil {
			log.Trace("=> Resolved dependency: %s", m)
			d.Module = m
			break
		}
	}
	if d.Module == nil {
		log.Error("Failed to resolve dependency: %s", d)
		return fmt.Errorf("Dependency not found from any source: %s", d)
	}

	if gm, ok := global_modules[d.Module.Name+"-"+d.Module.Version+"~"+d.Module.Source.URL]; ok {
		log.Trace("Dependency %s already resolved (S2): %s", d, gm)
		d.Module = gm
	} else if gm, ok := global_modules[d.Module.Name]; ok {
		log.Trace("Dependency %s already resolved (S3): %s", d, gm)

		// See if the already resolved version is acceptable
		if !d.MatchesVersion(gm.Version) {
			errstr := fmt.Sprintf("Version conflict in dependency tree: %s => %s", d, gm)
			log.Error(errstr)
			return errors.New(errstr)
		}

		log.Trace("Version %s matches %s", d.Module, gm.Version)

		// TODO See if downloading a new version would be better
		d.Module = gm
	} else {
		log.Debug("Downloading: %s", d.Module)
		if err := d.Module.Download(); err != nil {
			log.Error("Error downloading module %s: %s", d.Module, err)
			return err
		}

		if p != nil {
			if p.IsCircular(d.Module) {
				log.Error("Detected circular dependency %s from module %s", d.Module, p)
				return fmt.Errorf("Detected circular dependency %s from module %s", d.Module, p)
			}
		}

		// module can't exist because of global_lock
		global_modules[d.Module.Name] = d.Module
		global_modules[d.Module.Name+"-"+d.Module.Version] = d.Module
		global_modules[d.Module.Name+"-"+d.Module.Version+"~"+d.Module.Source.URL] = d.Module

		log.Debug("Resolving module dependencies: %s", d.Module)
		d.Module.Deps = &DependencyList{
			Parent:       d.Module,
			Dependencies: make([]*Dependency, 0),
		}

		if d.Additional != nil && len(d.Additional) > 0 {
			log.Trace("Adding cpanfile additional REQS")
			for _, additional := range d.Additional {
				log.Trace("Adding additional dependency from cpanfile: %s", additional)
				d.Module.Deps.AddDependency(additional)
			}
		}

		if err := d.Module.loadDependencies(); err != nil {
			return err
		}
	}

	return nil
}
Exemple #23
0
func main() {
	configure()

	indexes = make(map[string]map[string]*gopan.Source)
	indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	// FIXME inefficient
	_, _, tpkg, _ := gopan.CountIndex(indexes)

	npkg := 0
	nmod := 0

	var pc = func() float64 {
		return float64(nmod) / float64(tpkg) * 100
	}

	log.Info("Writing packages index file")

	out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
	if err != nil {
		log.Error("Error creating packages index: %s", err.Error())
		return
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, idx := range indexes[fname] {
			log.Debug("Index: %s", idx)
			out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
			for _, auth := range idx.Authors {
				log.Debug("Author %s", auth)
				out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
				for _, pkg := range auth.Packages {
					out.Write([]byte("  " + pkg.Name + " => " + pkg.URL + "\n"))
					log.Debug("Package: %s", pkg)

					if !config.Flatten {
						if len(pkg.Provides) == 0 {
							// TODO better handling of filenames
							modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")

							tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name

							if _, err := os.Stat(tgzpath); err != nil {
								log.Error("File not found: %s", tgzpath)
								return
							}

							extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
							dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name

							log.Trace("=> tgzpath: %s", tgzpath)
							log.Trace(" > extpath: %s", extpath)
							log.Trace(" > dirpath: %s", dirpath)

							// Only index packages if they don't already exist
							if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
								log.Error("Error retrieving package list: %s", err)
								continue
							}
						}

						npkg += len(pkg.Provides)
						nmod += 1

						for p, pk := range pkg.Provides {
							out.Write([]byte("   " + p + " (" + pk.Version + "): " + pk.File + "\n"))
						}

						if nmod > 0 && nmod%100 == 0 {
							log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
						}

					}
				}
			}
		}
	}

	out.Close()

	log.Info("Found %d packages from %d modules", npkg, nmod)
}
Exemple #24
0
func (r *Response) EventStream() chan []byte {
	c := make(chan []byte)

	r.IsEventStream = true
	r.Headers.Add("Content-Type", "text/event-stream")
	r.Headers.Add("Cache-Control", "no-cache")
	r.Headers.Add("Connection", "keep-alive")
	//r.Write([]byte("\n\n"))
	r.Send()

	hj, ok := r.writer.(nethttp.Hijacker)
	if !ok {
		log.Warn("Connection unsuitable for hijack")
		return nil
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		log.Warn("Connection hijack failed")
		return nil
	}

	r.esBufrw = bufrw
	r.esConn = conn

	go func() {
		for b := range c {
			if len(b) == 0 {
				log.Trace("Event stream ended")
				r.esConn.Close()
				break
			}

			lines := strings.Split(string(b), "\n")
			data := ""
			for _, l := range lines {
				data += "data: " + l + "\n"
			}
			data += "\n"

			sz := len(data) + 1
			log.Info("Event stream message is %d bytes", sz)
			size := fmt.Sprintf("%X", sz)
			r.esBufrw.Write([]byte(size + "\r\n"))

			lines = strings.Split(data, "\n")
			for _, ln := range lines {
				r.esBufrw.Write([]byte(ln + "\n"))
			}
			_, err := r.esBufrw.Write([]byte("\r\n"))

			if err != nil {
				log.Error("Error writing to connection: %s\n", err)
				r.esConn.Close()
				break
			}

			err = r.esBufrw.Flush()
			if err != nil {
				log.Error("Error flushing buffer: %s\n", err)
				r.esConn.Close()
				break
			}
		}
	}()
	return c
}
Exemple #25
0
func (m *Module) Download() error {
	m.Dir = config.CacheDir + "/" + path.Dir(m.Url)
	p := strings.TrimSuffix(path.Base(m.Url), ".tar.gz") // FIXME
	p = strings.TrimSuffix(p, ".tgz")
	m.Extracted = m.Dir + "/" + p
	m.Cached = config.CacheDir + "/" + m.Url

	log.Trace("Downloading to: %s", m.Dir)
	log.Trace("Cached file: %s", m.Cached)
	log.Trace("Extracting to: %s", m.Extracted)

	log.Trace("Aquiring lock on download: %s", m.Cached)
	file_lock.Lock()
	if mtx, ok := file_get[m.Cached]; ok {
		file_lock.Unlock()
		log.Trace("Waiting for existing download: %s", m.Cached)
		mtx.Lock()
		mtx.Unlock()
		log.Trace("Existing download complete: %s", m.Cached)
		return nil
	} else {
		log.Trace("Creating new lock")
		file_get[m.Cached] = new(sync.Mutex)
		file_get[m.Cached].Lock()
		defer file_get[m.Cached].Unlock()
		file_lock.Unlock()
		log.Trace("Lock aquired: %s", m.Cached)
	}

	if _, err := os.Stat(m.Cached); err != nil {
		os.MkdirAll(m.Dir, 0777)
		out, err := os.Create(m.Cached)
		if err != nil {
			return err
		}

		url := m.Source.URL + "/" + m.Url
		log.Trace("Downloading: %s", url)
		resp, err := http.Get(url)

		if err != nil {
			return err
		}

		if resp.StatusCode != 200 {
			return errors.New("404 not found")
		}

		_, err = io.Copy(out, resp.Body)
		if err != nil {
			return err
		}

		c := exec.Command("tar", "-zxf", m.Cached, "-C", m.Dir)

		var stdout2 bytes.Buffer
		var stderr2 bytes.Buffer

		c.Stderr = &stderr2
		c.Stdout = &stdout2

		if err := c.Start(); err != nil {
			return fmt.Errorf("Error extracting %s (%s): %s", m.Name, m.Version, err)
		}

		if err := c.Wait(); err != nil {
			return fmt.Errorf("Error extracting %s %s: %s\nSTDERR:\n%sSTDOUT:\n%s", m.Name, m.Version, err, stderr2.String(), stdout2.String())
		}

		out.Close()
		resp.Body.Close()

		log.Trace("File extracted to: %s", m.Extracted)
	} else {
		log.Trace("File already cached: %s", m.Cached)
	}

	return nil
}
Exemple #26
0
func getPackages() int {
	newpkg := 0

	var pl func(*html.Node, *gopan.Source, *gopan.Author)
	pl = func(n *html.Node, source *gopan.Source, author *gopan.Author) {
		log.Trace("NODE: %s [%s, %s, %s]", n.DataAtom, n.Type, n.Data)
		if n.Type == html.ElementNode && n.Data == "a" {
			//log.Info("NODE IS ELEMENTNODE")
			for _, attr := range n.Attr {
				// FIXME stuff that isn't .tar.gz?
				if attr.Key == "href" && strings.HasSuffix(attr.Val, ".tar.gz") {
					log.Trace("==> HREF: %s", n.FirstChild.Data)
					pkg := strings.TrimSuffix(n.FirstChild.Data, "/")
					if _, ok := author.Packages[pkg]; !ok {
						author.Packages[pkg] = &gopan.Package{
							Name:   pkg,
							Author: author,
							URL:    author.URL + "/" + pkg,
						}
						newpkg++
						log.Debug("Found package: %s", pkg)
					}
				}
			}
			//log.Info("%s", n.Data)
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			pl(c, source, author)
		}
	}

	log.Info("Building package list")

	for fname, _ := range indexes {
		for _, source := range indexes[fname] {
			log.Debug("Index: %s", source)
			wg.Add(1)
			go func(source *gopan.Source) {
				defer wg.Done()
				for _, author := range source.Authors {
					wg.Add(1)
					go func(author *gopan.Author) {
						defer wg.Done()
						sem <- 1
						log.Trace("=> %s", author)

						url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
						log.Trace("Getting URL: %s", url)

						res, err := http.Get(url)
						if err != nil {
							log.Error("HTTP GET - %s", err.Error())
							<-sem
							return
						}

						doc, err := html.Parse(res.Body)
						if err != nil {
							log.Error("HTML PARSE - %s", err.Error())
							<-sem
							return
						}

						pl(doc, source, author)

						<-sem
					}(author)
				}
			}(source)
		}
	}

	wg.Wait()

	log.Info("Finished building package list")

	return newpkg
}
Exemple #27
0
func (m *Module) loadDependencies() error {
	yml, err := ioutil.ReadFile(m.Extracted + "/META.yml")
	if err != nil {
		// TODO this isnt an error (it shouldnt make build fail)
		log.Error("Error opening META.yml for %s: %s", m.Name, err)
		// return nil to prevent build fail
		return nil
	}

	meta := make(map[interface{}]interface{})
	err = yaml.Unmarshal(yml, &meta)
	if err != nil {
		// TODO this isnt a real error, probably
		log.Error("Error parsing YAML: %s", err)
		// return nil to prevent build fail
		return nil
	}

	if reqs, ok := meta["requires"]; ok {
		log.Debug("Found dependencies for module %s", m.Name)
		switch reqs.(type) {
		case map[interface{}]interface{}:
			for req, ver := range reqs.(map[interface{}]interface{}) {
				v := float64(0)
				switch ver.(type) {
				case string:
					v = gopan.VersionFromString(ver.(string))
				case int:
					v = float64(ver.(int))
				}
				log.Printf("=> %s (%f)", req, v)
				dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
				if err != nil {
					log.Error("Error parsing dependency: %s", err)
					continue
				}
				if _, ok := perl_core[dep.Name]; ok {
					log.Trace("Module is from perl core: %s", dep.Name)
					continue
				}
				m.Deps.AddDependency(dep)
			}
		}

		log.Debug("Resolving module dependency list")

		if err := m.Deps.Resolve(); err != nil {
			log.Error("Error resolving dependency list [%s]: %s", m.Name, err)
			return err
		}

		return nil
	}

	// FIXME repeat of block above, just with more nested levels
	if p, ok := meta["prereqs"]; ok {
		if r, ok := p.(map[interface{}]interface{})["runtime"]; ok {
			if reqs, ok := r.(map[interface{}]interface{})["requires"]; ok {
				log.Debug("Found dependencies for module %s", m.Name)
				switch reqs.(type) {
				case map[interface{}]interface{}:
					for req, ver := range reqs.(map[interface{}]interface{}) {
						v := float64(0)
						switch ver.(type) {
						case string:
							v = gopan.VersionFromString(ver.(string))
						case int:
							v = float64(ver.(int))
						}
						log.Printf("=> %s (%f)", req, v)
						dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
						if err != nil {
							log.Error("Error parsing dependency: %s", err)
							continue
						}
						if _, ok := perl_core[dep.Name]; ok {
							log.Trace("Module is from perl core: %s", dep.Name)
							continue
						}
						m.Deps.AddDependency(dep)
					}
				}
			}
		}
		if t, ok := p.(map[interface{}]interface{})["test"]; ok {
			if reqs, ok := t.(map[interface{}]interface{})["requires"]; ok {
				log.Debug("Found dependencies for module %s", m.Name)
				switch reqs.(type) {
				case map[interface{}]interface{}:
					for req, ver := range reqs.(map[interface{}]interface{}) {
						v := float64(0)
						switch ver.(type) {
						case string:
							v = gopan.VersionFromString(ver.(string))
						case int:
							v = float64(ver.(int))
						}
						log.Printf("=> %s (%f)", req, v)
						dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
						if err != nil {
							log.Error("Error parsing dependency: %s", err)
							continue
						}
						if _, ok := perl_core[dep.Name]; ok {
							log.Trace("Module is from perl core: %s", dep.Name)
							continue
						}
						m.Deps.AddDependency(dep)
					}
				}
			}
		}

		log.Debug("Resolving module dependency list")
		if err := m.Deps.Resolve(); err != nil {
			log.Error("Error resolving dependency list: %s", err)
			return err
		}

		return nil
	}

	log.Debug("No dependencies for module %s", m.Name)
	return nil
}
Exemple #28
0
func (m *Module) Install() (int, error) {
	log.Debug("Installing module: %s", m)

	n := 0

	if m.Deps != nil {
		log.Trace("Installing module dependencies for %s", m)

		<-install_semaphore
		o, err := m.Deps.Install()
		install_semaphore <- 1

		n += o
		if err != nil {
			log.Error("Error installing module dependencies for %s: %s", m, err)
			return n, err
		}
	}

	var c *exec.Cmd
	var stdout *bytes.Buffer
	var stderr *bytes.Buffer

	cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
	if err != nil {
		log.Error("Failed to get absolute path of gopan cache directory: %s", err)
		return n, err
	}

	os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)

	done := false
	attempts := 0
	for !done {
		time.Sleep(time.Duration(100) * time.Millisecond)

		c = m.getCmd()
		stdout = new(bytes.Buffer)
		stderr = new(bytes.Buffer)
		c.Stderr = stderr
		c.Stdout = stdout

		// brute force cpanm text file busy errors
		attempts++
		if err := c.Start(); err != nil {
			if attempts > 10 {
				log.Error("Error installing module %s: %s", m, err)
				return n, err
			}
		} else {
			done = true
		}
	}

	if err := c.Wait(); err != nil {
		if !strings.HasPrefix(strings.ToLower(stderr.String()), "plenv: cannot rehash:") && !strings.Contains(strings.ToLower(stderr.String()), "text file busy") &&
			!strings.HasPrefix(strings.ToLower(stdout.String()), "plenv: cannot rehash:") && !strings.Contains(strings.ToLower(stdout.String()), "text file busy") {
			log.Error(m.Name + "-" + m.Version + " failed to install")
			log.Error("Error installing %s %s: %s\nSTDERR:\n%sSTDOUT:\n%s", m.Name, m.Version, err, stderr.String(), stdout.String())
			return n, err
		}
	}

	n++

	log.Printf("Installed " + m.Name + " (" + m.Version + ")")
	return n, nil
}
Exemple #29
0
func Provides(pkg *gopan.Package, tgzpath string, extpath string, dirpath string) error {
	// not required? path should already exist
	os.MkdirAll(dirpath, 0770)

	var stdout1 bytes.Buffer
	var stderr1 bytes.Buffer

	extract := exec.Command("tar", "-zxf", tgzpath, "-C", dirpath)
	extract.Stdout = &stdout1
	extract.Stderr = &stderr1

	if err := extract.Run(); err != nil {
		log.Error("Extract run: %s", err.Error())
		log.Trace(stdout1.String())
		log.Error(stderr1.String())
		return err
	}

	log.Trace(stdout1.String())
	log.Trace(stderr1.String())

	defer func() {
		var stdout3 bytes.Buffer
		var stderr3 bytes.Buffer

		clean := exec.Command("rm", "-rf", extpath)
		clean.Stdout = &stdout3
		clean.Stderr = &stderr3

		if err := clean.Run(); err != nil {
			log.Error("Clean run: %s", err.Error())
		}

		log.Trace(stdout3.String())
		log.Trace(stderr3.String())
	}()

	//var stdout2 bytes.Buffer
	var stderr2 bytes.Buffer

	if len(pldArgs) == 0 {
		if len(os.Getenv("GOPAN_ALLOW_DEV_VERSIONS")) > 0 {
			pldArgs = "({ALLOW_DEV_VERSION=>1})"
		} else {
			pldArgs = "()"
		}
	}
	log.Trace("pldArgs: %s", pldArgs)
	//cmd := exec.Command("perl", "-MModule::Metadata", "-MJSON::XS", "-e", "print encode_json(Module::Metadata->provides(version => 2, prefix => \"\", dir => $ARGV[0]))", extpath)
	cmd := exec.Command("perl", "-MParse::LocalDistribution", "-MJSON::XS", "-e", "print encode_json(Parse::LocalDistribution->new"+pldArgs+"->parse($ARGV[0]))", extpath)
	//cmd.Stdout = &stdout2
	cmd.Stderr = &stderr2

	stdout, err := cmd.StdoutPipe()
	defer stdout.Close()
	if err != nil {
		log.Error("StdoutPipe: %s", err.Error())
		return err
	}

	if err := cmd.Start(); err != nil {
		log.Error("Start: %s", err.Error())
		return err
	}

	var pld PLD
	if err := json.NewDecoder(stdout).Decode(&pld); err != nil {
		log.Error("JSON decoder error: %s", err.Error())
		return err
	}

	if err := cmd.Wait(); err != nil {
		log.Error("Wait: %s", err.Error())
		return err
	}

	//log.Trace(stdout2.String())
	log.Trace(stderr2.String())

	pkg.Provides = make(map[string]*gopan.PerlPackage)
	for p, pk := range pld {
		pp := &gopan.PerlPackage{
			Package: pkg,
			Name:    p,
			Version: pk.Version,
			File:    pk.Infile,
		}
		pkg.Provides[p] = pp
		log.Trace("%s: %s %s", p, pp.Version, pp.File)
	}

	log.Debug("%s provides %d packages", pkg, len(pkg.Provides))
	return nil
}
Exemple #30
0
func do_import(session *http.Session, job *ImportJob) {
	log.Info("Running import job %s", job.Id)

	reponame := job.Form.ImportInto
	if reponame == "new_index" {
		reponame = job.Form.NewIndex
	}

	msg := func(m string) {
		if m != ":DONE" {
			job.History = append(job.History, m)
			log.Info(m)
		}
		for _, w := range job.Watchers {
			w(m)
		}
	}

	mods := make([]*getpan.Module, 0)

	// TODO cpanm mirror when using getpan_import

	if len(job.Form.Cpanfile) > 0 {
		msg("Parsing cpanfile input")
		_, modules := getpan_import(job, msg)
		mods = append(mods, modules...)
	}

	if len(job.Form.ImportURL) > 0 {
		msg("Importing from URL: " + job.Form.ImportURL)

		// TODO support cpanfile urls

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.ImportURL)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		if _, err := os.Stat(nfile); err != nil {
			os.MkdirAll(npath, 0777)
			out, err := os.Create(nfile)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			url := job.Form.ImportURL
			log.Trace("Downloading: %s", url)
			resp, err := nethttp.Get(url)

			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			_, err = io.Copy(out, resp.Body)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			out.Close()
			resp.Body.Close()
		} else {
			log.Trace("File already exists in cache: %s", nfile)
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if len(job.Form.FromDir) > 0 {
		msg("Importing from local directory: " + job.Form.FromDir)

		// TODO support cpanfile paths

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.FromDir)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		_, err := CopyFile(nfile, job.Form.FromDir)
		if err != nil {
			msg(err.Error())
			msg(":DONE")
			job.Complete = true
			return
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if f, fh, err := session.Request.File("fromfile"); err == nil {
		fn := fh.Filename

		msg("Importing from uploaded module/cpanfile: " + fn)

		if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
			msg("Only cpanfile and *.tar.gz files are supported")
			msg(":DONE")
			job.Complete = true
			return
		}

		if fn == "cpanfile" {
			msg("Importing cpanfile")
			b, _ := ioutil.ReadAll(f)
			f.Close()
			job.Form.Cpanfile = string(b)
			_, modules := getpan_import(job, msg)
			mods = append(mods, modules...)
		} else {
			msg("Importing .tar.gz")

			nauth := job.Form.AuthorID

			if len(nauth) < 3 {
				// FIXME move to form validation
				msg("Author ID must be at least 3 characters")
				msg(":DONE")
				job.Complete = true
				return
			}

			npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
			_, fn = filepath.Split(fn)
			nfile := npath + "/" + fn

			msg("Caching to " + nfile)

			os.MkdirAll(npath, 0777)

			_, err := CopyToFile(nfile, f)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			fn = strings.TrimSuffix(fn, ".tar.gz")
			bits := strings.Split(fn, "-")
			name := strings.Join(bits[0:len(bits)-1], "-")
			version := bits[len(bits)-1]

			s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
			m := &getpan.Module{
				Source:  s,
				Name:    name,
				Version: version,
				Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
				Cached:  nfile,
				Dir:     npath,
			}
			m.Deps = &getpan.DependencyList{
				Parent:       m,
				Dependencies: make([]*getpan.Dependency, 0),
			}
			mods = append(mods, m)
		}
	} else {
		// there is no file... so no error
		//msg("Error importing file upload: " + err.Error())
	}

	if len(mods) == 0 {
		msg("Nothing to do")
		msg(":DONE")
		job.Complete = true
		return
	}

	msg("Adding modules to GoPAN index")

	for _, m := range mods {
		msg("=> " + m.Name + " (" + m.Cached + ")")

		dn, fn := filepath.Split(m.Cached)
		dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
		auth := dnb[len(dnb)-1]
		ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
		npath := ndir + "/" + fn

		if _, err := os.Stat(npath); err == nil {
			msg(" | Already exists in repository")
		} else {
			os.MkdirAll(ndir, 0777)

			msg(" | Copying to " + npath)
			_, err := CopyFile(npath, m.Cached)
			if err != nil {
				msg(" ! " + err.Error())
				continue
			}
		}

		if _, ok := indexes[config.Index][reponame]; !ok {
			msg(" | Creating index: " + reponame)
			indexes[config.Index][reponame] = &gopan.Source{
				Name:    reponame,
				URL:     "/authors/id",
				Authors: make(map[string]*gopan.Author),
			}

			mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
			msg(" | Creating author: " + auth)
			author := &gopan.Author{
				Source:   indexes[config.Index][reponame],
				Name:     auth,
				Packages: make(map[string]*gopan.Package),
				URL:      "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
			}
			indexes[config.Index][reponame].Authors[auth] = author

			if _, ok := mapped[reponame]; !ok {
				mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
			}

			// author name
			if _, ok := mapped[reponame][author.Name[:1]]; !ok {
				mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
				mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author

			// wildcards
			if _, ok := mapped[reponame]["*"]; !ok {
				mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"]["**"]; !ok {
				mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
			}
			mapped[reponame]["*"]["**"][author.Name] = author

			// combos
			if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
				mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
				mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]]["**"][author.Name] = author
			mapped[reponame]["*"][author.Name[:2]][author.Name] = author
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
			msg(" | Creating module: " + fn)
			indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
				Author:   indexes[config.Index][reponame].Authors[auth],
				Name:     fn,
				URL:      indexes[config.Index][reponame].Authors[auth].URL + fn,
				Provides: make(map[string]*gopan.PerlPackage),
			}

			msg(" | Getting list of packages")
			modnm := strings.TrimSuffix(fn, ".tar.gz")
			pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
				msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
			}

			//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			msg(" | Adding packages to index")
			if _, ok := idxpackages[reponame]; !ok {
				idxpackages[reponame] = make(map[string]*PkgSpace)
			}
			filemap[pkg.AuthorURL()] = reponame
			for _, prov := range pkg.Provides {
				parts := strings.Split(prov.Name, "::")
				if _, ok := packages[parts[0]]; !ok {
					packages[parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if _, ok := idxpackages[reponame][parts[0]]; !ok {
					idxpackages[reponame][parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if len(parts) == 1 {
					packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
					packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
					idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
					idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
				} else {
					packages[parts[0]].Populate(parts[1:], prov)
					idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
				}
			}

			msg(" | Writing to index file")
			gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
		}

		msg(" | Imported module")
	}

	nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
	// TODO should probably be in the index - needs to udpate when index changes
	summary = &Summary{nsrc, nauth, npkg, nprov}

	msg(":DONE")
	job.Complete = true
}