Пример #1
0
func ParseCPANLines(lines []string) (*CPANFile, error) {
	cpanfile := &CPANFile{}

	for _, l := range lines {
		if len(l) == 0 {
			continue
		}

		log.Trace("Parsing line: %s", l)
		dep, err := ParseCPANLine(l)

		if err != nil {
			log.Error("=> Error parsing line: %s", err)
			continue
		}

		if dep != nil {
			log.Info("=> Found dependency: %s", dep)
			cpanfile.AddDependency(dep)
			continue
		}

		log.Trace("=> No error and no dependency found")
	}

	log.Info("Found %d dependencies in cpanfile", len(cpanfile.Dependencies))

	return cpanfile, nil
}
Пример #2
0
// Listen binds to httpBindAddr
func Listen(httpBindAddr string, Asset func(string) ([]byte, error), exitCh chan int, registerCallback func(http.Handler)) {
	log.Info("[HTTP] Binding to address: %s", httpBindAddr)

	pat := pat.New()
	registerCallback(pat)

	f := func(w http.ResponseWriter, req *http.Request) {
		if Authorised == nil {
			pat.ServeHTTP(w, req)
			return
		}

		u, pw, ok := req.BasicAuth()
		if !ok || !Authorised(u, pw) {
			w.Header().Set("WWW-Authenticate", "Basic")
			w.WriteHeader(401)
			return
		}
		pat.ServeHTTP(w, req)
	}

	err := http.ListenAndServe(httpBindAddr, http.HandlerFunc(f))
	if err != nil {
		log.Fatalf("[HTTP] Error binding to address %s: %s", httpBindAddr, err)
	}
}
Пример #3
0
func (s *Source) loadBackPANSource() error {
	log.Info("Loading BackPAN index: backpan-index")

	file, err := os.Open("backpan-index")
	if err != nil {
		log.Warn(err.Error())
		return nil
	}

	index, err := ioutil.ReadAll(file)
	file.Close()
	if err != nil {
		log.Fatal(err)
	}

	for _, p := range strings.Split(string(index), "\n") {
		if !strings.HasPrefix(p, "authors/id/") {
			continue
		}

		//log.Printf("Parsing: %s\n", p)
		m := s.ModuleFromBackPANIndex(p)
		if m != nil {
			s.ModuleList[m.Name+"-"+m.Version] = m
		}
	}

	log.Printf("Found %d packages for source: %s", len(s.ModuleList), s)
	return nil
}
Пример #4
0
func (ws *Workspace) ExecFunction(task *Task, name string, args ...string) string {
	log.Info("Executing function %s: %s", name, args)
	var fn *Function
	if f, ok := ws.Functions[name]; ok {
		fn = f
	} else if f, ok := GlobalWorkspace.Functions[name]; ok {
		fn = f
	} else {
		log.Warn("Function not found: %s", name)
		return ""
	}

	argmap := make(map[string]string)
	for i, arg := range fn.Args {
		argmap[arg] = args[i]
	}

	for k, v := range argmap {
		log.Info("argmap: %s => %s", k, v)
		for t, m := range task.Metadata {
			log.Info("meta: %s => %s", t, m)
			v = strings.Replace(v, "$"+t, m, -1)
		}
		argmap[k] = v
	}

	c := fn.Command
	for k, v := range argmap {
		log.Info("ARG: %s => %s", k, v)
		c = strings.Replace(c, k, v, -1)
	}

	var funcEnvironment map[string]string
	if ws.InheritEnvironment {
		funcEnvironment = ws.Environment
	} else if GlobalWorkspace.InheritEnvironment {
		funcEnvironment = GlobalWorkspace.Environment
	} else {
		funcEnvironment = make(map[string]string)
	}

	tsk := NewTask(nil, "Function$"+name, fn.Executor, c, funcEnvironment, false, "", "", make(map[string]string), "")
	ch := tsk.Start()
	<-ch
	return tsk.TaskRuns[0].StdoutBuf.String()
}
Пример #5
0
func (s *Session) loadSessionData() {
	if sid_cookie, ok := s.Request.Cookies["__SID"]; ok {
		s.SessionID = sid_cookie.Value
		log.Info("Retrieved session ID (__SID): %s", s.SessionID)
	}

	s.readSessionData()
}
Пример #6
0
func (deps *DependencyList) PrintDeps(d int) {
	for _, dep := range deps.Dependencies {
		if dep.Module == nil {
			log.Info(MkIndent(0)+"%s not found", dep.Name)
			continue
		}
		dep.Module.PrintDeps(d + 1)
	}
}
Пример #7
0
func (s *Source) loadCPANSource() error {
	log.Info("Loading CPAN index: %s", s.Index)

	res, err := http.Get(s.Index)
	if err != nil {
		log.Warn("Error loading index: %s", err)
		return nil
	}

	// TODO optional gzip
	r, err := gzip.NewReader(res.Body)
	if err != nil {
		log.Warn(err.Error())
		b, _ := ioutil.ReadAll(res.Body)
		log.Info("%s", string(b))
		return nil
	}

	packages, err := ioutil.ReadAll(r)
	res.Body.Close()
	if err != nil {
		log.Warn(err)
		return nil
	}

	foundnl := false
	for _, p := range strings.Split(string(packages), "\n") {
		if !foundnl && len(p) == 0 {
			foundnl = true
			continue
		}
		if !foundnl || len(p) == 0 {
			continue
		}
		m := s.ModuleFromCPANIndex(p)
		s.ModuleList[m.Name] = m
	}

	log.Info("Found %d packages for source: %s", len(s.ModuleList), s)
	return nil
}
Пример #8
0
func (r *Response) createSessionId() {
	bytes := make([]byte, 256)
	rand.Read(bytes)
	s, _ := bcrypt.GenerateFromPassword(bytes, 11)
	r.session.SessionID = string(s)
	log.Info("Generated session ID (__SID): %s", r.session.SessionID)
	r.Cookies.Set(&nethttp.Cookie{
		Name:  "__SID",
		Value: r.session.SessionID,
		Path:  "/",
	})
}
Пример #9
0
func (ws *Workspace) GetColumn(task *Task, name string) string {
	log.Info("GetColumn: %s => %s", task.Name, name)
	col := ws.Columns[name]
	var fn []string
	var nm string
	for n, args := range col {
		nm = n
		fn = args
		break
	}
	return ws.ExecFunction(task, nm, fn...)
}
Пример #10
0
func LoadConfig(global string, workspaces []string) {
	// Load global environment
	log.Info("Loading global environment file: %s", global)
	cfg, err := LoadConfigFile(global)
	if err != nil {
		log.Error("Error loading global configuration: %s", err.Error())
	}
	if cfg != nil {
		GlobalConfigWorkspace = cfg
	}

	// Load workspaces
	for _, conf := range workspaces {
		log.Info("Loading workspace file: %s", conf)
		cfg, err := LoadConfigFile(conf)
		if err != nil {
			log.Error("Error loading workspace: %s", err.Error())
		}
		if cfg != nil {
			ConfigWorkspaces[cfg.Name] = cfg
		}
	}
}
Пример #11
0
func (s *Session) readSessionData() {
	sd, ok := s.Request.Cookies["__SD"]
	if ok && sd.Value != "" {
		log.Info("Retrieved session data (__SD): %s", sd.Value)
		sdata := make(map[string]string)
		err := json.Unmarshal([]byte(fromBase64(sd.Value)), &sdata)

		if err != nil {
			s.RenderException(500, err)
			return
		}

		s.SessionData = sdata
	}
}
Пример #12
0
Файл: main.go Проект: djui/necd
func runLoop() {
	conf, err := readConf()
	AssertNoErr(err, "Failed to read config file")
	log.Debug("Global conf: %#v", conf)

	nif := conf["config"]["if"]
	interval := atoi(conf["config"]["interval"], 5)

	for name := range notifyOnChange(nif, interval) {
		log.Info("Network changed: %s", name)

		if section, ok := conf["ssid:"+name]; ok {
			log.Debug("Found section: %v", section)
			ApplyCmds(section)
		} else {
			log.Debug("Undefined section for: %s", name)
		}
	}
}
Пример #13
0
func ParseCPANLine(line string) (*Dependency, error) {
	if len(line) == 0 {
		return nil, nil
	}

	matches := re.FindStringSubmatch(line)
	if len(matches) == 0 {
		log.Trace("Unable to parse line: %s", line)
		return nil, nil
	}

	module := matches[2]
	version := strings.Replace(matches[4], " ", "", -1)
	comment := matches[5]

	dependency, err := DependencyFromString(module, version)

	if strings.HasPrefix(strings.Trim(comment, " "), "# REQS: ") {
		comment = strings.TrimPrefix(strings.Trim(comment, " "), "# REQS: ")
		log.Trace("Found additional dependencies: %s", comment)
		for _, req := range strings.Split(comment, ";") {
			req = strings.Trim(req, " ")
			bits := strings.Split(req, "-")
			new_dep, err := DependencyFromString(bits[0], bits[1])
			if err != nil {
				log.Error("Error parsing REQS dependency: %s", req)
				continue
			}
			log.Trace("Added dependency: %s", new_dep)
			dependency.Additional = append(dependency.Additional, new_dep)
		}
	}

	if err != nil {
		return nil, err
	}

	log.Info("%s (%s %s)", module, dependency.Modifier, dependency.Version)

	return dependency, err
}
Пример #14
0
func (s *Source) Find(d *Dependency) (*Module, error) {
	log.Debug("Finding dependency: %s", d)

	switch s.Type {
	case "SmartPAN":
		log.Debug("=> Using SmartPAN source")

		url := s.URL
		if !strings.HasSuffix(s.URL, "/") {
			url += "/"
		}
		url += "where/" + d.Name + "/" + d.Modifier + d.Version

		log.Info("Query: %s", url)
		res, err := http.Get(url)

		if err != nil {
			log.Error("Error querying SmartPAN: %s", err.Error())
			return nil, err
		}

		defer res.Body.Close()

		body, err := ioutil.ReadAll(res.Body)
		log.Trace("Got response: %s", string(body))

		if res.StatusCode != http.StatusOK {
			log.Info("Module not found in SmartPAN: %s", d.Name)
			return nil, nil
		}

		var v *WhereOutput
		if err = json.Unmarshal(body, &v); err != nil {
			log.Error("Error parsing JSON: %s", err.Error())
			return nil, err
		}

		log.Trace("Found module %s", v.Module)

		if len(v.Versions) == 0 {
			log.Info("Found module but no versions returned")
			return nil, nil
		}

		var lv *VersionOutput
		for _, ver := range v.Versions {
			if ver.Version == v.Latest {
				log.Info("Using latest version of %s: %f", v.Module, ver.Version)
				lv = ver
				break
			}
		}
		if lv == nil {
			log.Info("Couldn't find latest version, selecting first available")
			lv = v.Versions[0]
		}

		return &Module{
			Name:    d.Name,
			Version: fmt.Sprintf("%f", lv.Version),
			Source:  s,
			Url:     lv.URL,
		}, nil
	case "CPAN":
		log.Debug("=> Using CPAN source")
		if mod, ok := s.ModuleList[d.Name]; ok {
			log.Trace("=> Found in source: %s", mod)
			if d.Matches(mod) {
				log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
				return mod, nil
			}
			log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
			return nil, nil
		}
	case "BackPAN":
		log.Debug("=> Using BackPAN source")
		// TODO better version matching - new backpan index?
		if mod, ok := s.ModuleList[d.Name+"-"+d.Version]; ok {
			log.Trace("=> Found in source: %s", mod)
			if d.Matches(mod) {
				log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
				return mod, nil
			}
			log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
			return nil, nil
		}
	case "MetaCPAN":
		log.Debug("=> Using MetaCPAN source")

		var sout, serr bytes.Buffer
		var cpanm_args string = fmt.Sprintf("-L %s --info %s~\"%s%s\"", config.InstallDir, d.Name, d.Modifier, d.Version)

		cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
		if err != nil {
			log.Error("Failed to get absolute path of gopan cache directory: %s", err)
			return nil, err
		}

		log.Trace("About to exec: cpanm %s", cpanm_args)
		os.Setenv("CPANM_INFO_ARGS", cpanm_args)
		os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)
		cmd := exec.Command("bash", "-c", `eval cpanm $CPANM_INFO_ARGS`)
		cmd.Stdout = &sout
		cmd.Stderr = &serr

		if err := cmd.Run(); err != nil {
			log.Error("cpanm %s: %s,\n%s\n", cpanm_args, err, serr.String())
			return nil, nil
		}

		if 0 == len(sout.String()) {
			log.Warn("No author/module from cpanm")
			return nil, nil
		}

		author_module := strings.TrimRight(sout.String(), "\n")
		mematches := metacpanRe.FindStringSubmatch(author_module)
		if nil == mematches {
			log.Error("Match failed for: %s", author_module)
			return nil, nil
		}

		log.Trace("Resolved: %s", author_module)
		for _, mesource := range config.MetaSources {

			meurl := fmt.Sprintf("authors/id/%s/%s/%s",
				mematches[1][0:1],
				mematches[1][0:2],
				mematches[0])

			archive_url := fmt.Sprintf("%s/%s", mesource.URL, meurl)

			log.Trace("Checking: " + archive_url)
			resp, err := http.Head(archive_url)
			if err != nil {
				log.Trace(err)
				continue
			}

			log.Trace("HEAD status code: %d", resp.StatusCode)
			if 200 == resp.StatusCode {
				// No module/version check since 'cpanm --info' may resolve to
				// archive and version that may not match source
				return &Module{
					Name:    mematches[2],
					Version: mematches[3],
					Source:  mesource,
					Url:     meurl,
				}, nil
			}

		}
		log.Error("Could not get archive URL via 'cpanm %s'", cpanm_args)
		return nil, nil
	default:
		log.Error("Unrecognised source type: %s", s.Type)
		return nil, errors.New(fmt.Sprintf("Unrecognised source: %s", s))
	}
	log.Trace("=> Not found in source")
	return nil, nil
}
Пример #15
0
func LoadIndex(index string) map[string]*Source {
	indexes := make(map[string]*Source)

	log.Info("Loading cached index file %s", index)

	if _, err := os.Stat(index); err != nil {
		log.Error("Cached index file not found")
		return indexes
	}

	var bytes []byte
	if strings.HasSuffix(index, ".gz") {
		fi, err := os.Open(index)
		if err != nil {
			log.Error("Error reading index: %s", err.Error())
			return indexes
		}
		defer fi.Close()
		gz, err := gzip.NewReader(fi)
		if err != nil {
			log.Error("Error creating gzip reader: %s", err.Error())
			return indexes
		}

		bytes, err = ioutil.ReadAll(gz)
		if err != nil {
			log.Error("Error reading from gzip: %s", err.Error())
			return indexes
		}
	} else {
		var err error
		bytes, err = ioutil.ReadFile(index)
		if err != nil {
			log.Error("Error reading index: %s", err.Error())
			return indexes
		}
	}

	lines := strings.Split(string(bytes), "\n")
	var csource *Source
	var cauth *Author
	var cpkg *Package
	resrcauth := regexp.MustCompile("^\\s*(.*)\\s\\[(.*)\\]\\s*$")
	repackage := regexp.MustCompile("^\\s*(.*)\\s=>\\s(.*)\\s*$")
	reprovides := regexp.MustCompile("^\\s*(.*)\\s\\((.*)\\):\\s(.*)\\s*$")
	for _, l := range lines {
		log.Trace("Line: %s", l)
		if strings.HasPrefix(l, "   ") {
			// provides
			log.Trace("=> Provides")
			match := reprovides.FindStringSubmatch(l)
			if len(match) > 0 {
				if strings.HasPrefix(match[1], "-") {
					log.Trace("  - Is a removal")
					match[1] = strings.TrimPrefix(match[1], "-")
					if _, ok := cpkg.Provides[match[1]]; ok {
						delete(cpkg.Provides, match[1])
					}
					if len(cpkg.Provides) == 0 {
						delete(cauth.Packages, cpkg.Name)
						cpkg = nil
					}
					if len(cauth.Packages) == 0 {
						delete(csource.Authors, cauth.Name)
						cauth = nil
					}
					if len(csource.Authors) == 0 {
						delete(indexes, csource.Name)
						csource = nil
					}
				} else {
					cpkg.Provides[match[1]] = &PerlPackage{
						Name:    match[1],
						Version: match[2],
						Package: cpkg,
						File:    match[3],
					}
				}
			}
		} else if strings.HasPrefix(l, "  ") {
			// its a package
			log.Trace("=> Package")
			match := repackage.FindStringSubmatch(l)
			if len(match) > 0 {
				if strings.HasPrefix(match[1], "-") {
					log.Trace("  - Is a removal")
					match[1] = strings.TrimPrefix(match[1], "-")
					if _, ok := cauth.Packages[match[1]]; ok {
						delete(cauth.Packages, match[1])
					}
					if len(cauth.Packages) == 0 {
						delete(csource.Authors, cauth.Name)
						cauth = nil
					}
					if len(csource.Authors) == 0 {
						delete(indexes, csource.Name)
						csource = nil
					}
				} else {
					if _, ok := cauth.Packages[match[1]]; ok {
						// we've seen this package before
						log.Trace("Seen this package before: %s", match[1])
						cpkg = cauth.Packages[match[1]]
						continue
					}
					cpkg = &Package{
						Name:     match[1],
						URL:      match[2],
						Author:   cauth,
						Provides: make(map[string]*PerlPackage),
					}
					cauth.Packages[match[1]] = cpkg
				}
			}
		} else if strings.HasPrefix(l, " ") {
			// its an author
			log.Trace("=> Author")
			match := resrcauth.FindStringSubmatch(l)
			if len(match) > 0 {
				if strings.HasPrefix(match[1], "-") {
					log.Trace("  - Is a removal")
					match[1] = strings.TrimPrefix(match[1], "-")
					if _, ok := csource.Authors[match[1]]; ok {
						delete(csource.Authors, match[1])
					}
					if len(csource.Authors) == 0 {
						delete(indexes, csource.Name)
						csource = nil
					}
				} else {
					if _, ok := csource.Authors[match[1]]; ok {
						// we've seen this author before
						log.Trace("Seen this author before: %s", match[1])
						cauth = csource.Authors[match[1]]
						continue
					}
					cauth = &Author{
						Name:     match[1],
						URL:      match[2],
						Source:   csource,
						Packages: make(map[string]*Package, 0),
					}
					csource.Authors[match[1]] = cauth
				}
			}
		} else {
			// its a source
			log.Trace("=> Source")
			match := resrcauth.FindStringSubmatch(l)
			if len(match) > 0 {
				if strings.HasPrefix(match[1], "-") {
					log.Trace("  - Is a removal")
					match[1] = strings.TrimPrefix(match[1], "-")
					if _, ok := indexes[match[1]]; ok {
						delete(indexes, match[1])
					}
				} else {
					seen := false
					for _, idx := range indexes {
						if idx.Name == match[1] {
							// we've seen this source before
							log.Trace("Seen this source before: %s", idx.Name)
							csource = idx
							seen = true
							break
						}
					}
					if seen {
						continue
					}
					csource = &Source{
						Name:    match[1],
						URL:     match[2],
						Authors: make(map[string]*Author, 0),
					}
					indexes[csource.Name] = csource
				}
			}
		}
	}

	for _, source := range indexes {
		log.Trace(source.Name)
		for _, author := range source.Authors {
			log.Trace("    %s", author.Name)
			for _, pkg := range author.Packages {
				log.Trace("        %s => %s", pkg.Name, pkg.URL)
			}
		}
	}

	return indexes
}
Пример #16
0
func where(session *http.Session) {
	module := session.Stash["module"].(string)
	log.Info("Looking for module: %s", module)

	ns := strings.Split(module, "::")
	if _, ok := packages[ns[0]]; !ok {
		log.Info("Top-level namespace [%s] not found", ns[0])
		session.Response.Status = 404
		session.Response.Send()
		return
	}

	mod := packages[ns[0]]

	ns = ns[1:]
	for len(ns) > 0 {
		if _, ok := mod.Children[ns[0]]; !ok {
			log.Info("Child namespace [%s] not found", ns[0])
			session.Response.Status = 404
			session.Response.Send()
			return
		}
		log.Info("Found child namespace [%s]", ns[0])
		mod = mod.Children[ns[0]]
		ns = ns[1:]
	}

	var version string
	if _, ok := session.Stash["version"]; ok {
		version = session.Stash["version"].(string)
		if strings.HasPrefix(version, "v") {
			version = strings.TrimPrefix(version, "v")
		}
		log.Info("Looking for version: %s", version)
	}

	if len(mod.Versions) == 0 {
		log.Info("Module has no versions in index")
		session.Response.Status = 404
		session.Response.Send()
		return
	}

	versions := make([]*VersionOutput, 0)
	lv := float64(0)
	if len(version) > 0 {
		if ">0" == version {
			// Take account of packages that have a version of 'undef'
			for _, pkg := range mod.Packages {
				packageURL := pkg.Package.VirtualURL()
				urlmatches := packageUrlRe.FindStringSubmatch(packageURL)
				if nil == urlmatches {
					log.Info("Version requested [%s] not found", version)
					session.Response.Status = 404
					session.Response.Send()
					return
				}
				ver := gopan.VersionFromString(urlmatches[2])
				log.Info("Found version: %f", ver)
				versions = append(versions, &VersionOutput{
					Index:   pkg.Package.Author.Source.Name,
					URL:     packageURL,
					Path:    pkg.Package.AuthorURL(),
					Version: ver,
				})
				if ver > lv {
					lv = ver
				}
			}
		} else {
			dep, _ := getpan.DependencyFromString(module, version)
			for _, md := range mod.Versions {
				var sver string = md.Version
				if `undef` == md.Version {
					sver = fmt.Sprintf("%.2f", md.Package.Version())
				}
				log.Info("Matching [%s] against derived version [%s] (md.Version [%s], md.Package.Version [%f])", dep.Version, sver, md.Version, md.Package.Version())
				if dep.MatchesVersion(sver) {
					vout := &VersionOutput{
						Index:   md.Package.Author.Source.Name,
						URL:     md.Package.VirtualURL(),
						Path:    md.Package.AuthorURL(),
						Version: gopan.VersionFromString(sver),
					}
					versions = append(versions, vout)
					ver := gopan.VersionFromString(sver)
					if ver > lv {
						lv = ver
					}
				}
			}
		}

		if len(versions) == 0 {
			log.Info("Version requested [%s] not found", version)
			session.Response.Status = 404
			session.Response.Send()
			return
		}
	} else {
		for v, pkg := range mod.Versions {
			log.Info("Found version: %f", v)
			versions = append(versions, &VersionOutput{
				Index:   pkg.Package.Author.Source.Name,
				URL:     pkg.Package.VirtualURL(),
				Path:    pkg.Package.AuthorURL(),
				Version: gopan.VersionFromString(pkg.Version),
			})
			if v > lv {
				lv = v
			}
		}
	}

	session.Response.Headers.Set("Content-Type", "application/json")

	o := &WhereOutput{
		Module:   mod.FullName(),
		Latest:   lv,
		Versions: versions,
	}

	b, err := json.MarshalIndent(o, "", "  ")

	log.Info("Output: %s", string(b))

	if err != nil {
		log.Error("Failed encoding JSON: %s", err.Error())
		session.Response.Status = 500
		session.Response.Send()
		return
	}

	session.Response.Status = 200
	session.Response.Write(b)
}
Пример #17
0
func main() {
	config = getpan.Configure()
	config.Dump()

	mods := flag.Args()

	if len(mods) == 0 {
		if _, err := os.Stat(config.CPANFile); os.IsNotExist(err) {
			log.Error("cpanfile not found: %s", config.CPANFile)
			os.Exit(1)
		}
	}

	if len(mods) > 0 && mods[0] == "exec" {
		log.Debug("getpan exec => " + strings.Join(mods[1:], " "))

		cmd := exec.Command(mods[1], mods[2:]...)

		cmd.Env = os.Environ()
		cmd.Env = append(cmd.Env, "PERL5LIB="+config.InstallDir+"/lib/perl5")
		cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")+":"+config.InstallDir+"/bin")

		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		err := cmd.Run()

		if err != nil {
			// debug so it doesn't show up in stdout/stderr unless -loglevel is used
			log.Debug("Error in exec: %s", err.Error())
			os.Exit(10)
		}

		return
	}

	for _, source := range config.Sources {
		err := source.Load()
		if err != nil {
			log.Error("Error loading sources: %s", err)
			os.Exit(1)
			return
		}
	}

	var deps *getpan.DependencyList

	if len(mods) == 0 {
		log.Info("Installing from cpanfile: %s", config.CPANFile)
		d, err := getpan.ParseCPANFile(config.CPANFile)
		if err != nil {
			log.Error("Error parsing cpanfile: %s", err)
			os.Exit(2)
			return
		}
		deps = &d.DependencyList
	} else {
		log.Info("Installing from command line args")
		deps = &getpan.DependencyList{
			Dependencies: make([]*getpan.Dependency, 0),
		}
		for _, arg := range mods {
			dependency, err := getpan.DependencyFromString(arg, "")
			if err != nil {
				log.Error("Unable to parse input: %s", arg)
				continue
			}
			deps.AddDependency(dependency)
		}
	}

	err := deps.Resolve()
	if err != nil {
		log.Error("Error resolving dependencies: %s", err)
		os.Exit(3)
		return
	}

	if false == config.NoDepdump {
		log.Info("Resolved dependency tree:")
		deps.PrintDeps(0)
	}

	if config.NoInstall {
		log.Info("Skipping installation phase")
		return
	}

	_, err = deps.Install()

	if err != nil {
		log.Error("Error installing dependencies: %s", err)
		os.Exit(4)
		return
	}

	// FIXME hacky, need a better way of tracking installed deps
	log.Info("Successfully installed %d modules", deps.UniqueInstalled())
}
Пример #18
0
func main() {
	log.Logger().SetAppender(NewAppender())

	global := "websysd.json"
	flag.StringVar(&global, "global", global, "global environment configuration")

	workspaces := make([]string, 0)
	flag.Var((*AppendSliceValue)(&workspaces), "workspace", "websysd workspace file (can be specified multiple times), defaults to './workspace.json'")

	// Create our Gotcha application
	var app = gotcha.Create(Asset)

	if len(workspaces) == 0 {
		workspaces = append(workspaces, "./workspace.json")
	}

	LoadConfig(global, workspaces)

	GlobalWorkspace = NewWorkspace(GlobalConfigWorkspace.Name, GlobalConfigWorkspace.Environment, make(map[string]map[string][]string), GlobalConfigWorkspace.InheritEnvironment)
	for fn, args := range GlobalConfigWorkspace.Functions {
		log.Info("=> Creating global function: %s", fn)
		GlobalWorkspace.Functions[fn] = &Function{
			Name:     fn,
			Args:     args.Args,
			Command:  args.Command,
			Executor: args.Executor,
		}
	}

	if GlobalWorkspace.InheritEnvironment {
		log.Info("=> Inheriting process environment into global workspace")
		for _, k := range os.Environ() {
			p := strings.SplitN(k, "=", 2)
			if strings.TrimSpace(p[0]) == "" {
				log.Warn("Skipping empty environment key")
				continue
			}
			log.Info("  %s = %s", p[0], p[1])
			// TODO variable subst for current env vars
			if _, ok := GlobalWorkspace.Environment[p[0]]; !ok {
				GlobalWorkspace.Environment[p[0]] = p[1]
			}
		}
	}

	for _, ws := range ConfigWorkspaces {
		log.Info("=> Creating workspace: %s", ws.Name)

		var workspace *Workspace
		if wks, ok := Workspaces[ws.Name]; ok {
			log.Warn("Workspace %s already exists, merging tasks and environment")
			workspace = wks
		} else {
			workspace = NewWorkspace(ws.Name, ws.Environment, ws.Columns, ws.InheritEnvironment)
			Workspaces[ws.Name] = workspace
		}

		workspace.IsLocked = ws.IsLocked

		if workspace.InheritEnvironment && !GlobalWorkspace.InheritEnvironment {
			log.Info("=> Inheriting process environment into workspace")
			for _, k := range os.Environ() {
				p := strings.SplitN(k, "=", 2)
				if strings.TrimSpace(p[0]) == "" {
					log.Warn("Skipping empty environment key")
					continue
				}
				log.Info("  %s = %s", p[0], p[1])
				// TODO variable subst for current env vars
				if _, ok := workspace.Environment[p[0]]; !ok {
					workspace.Environment[p[0]] = p[1]
				}
			}
		}

		for fn, args := range ws.Functions {
			log.Info("=> Creating workspace function: %s", fn)
			workspace.Functions[fn] = &Function{
				Name:     fn,
				Args:     args.Args,
				Command:  args.Command,
				Executor: args.Executor,
			}
		}

		for _, t := range ws.Tasks {
			log.Info("=> Creating task: %s", t.Name)

			if _, ok := workspace.Tasks[t.Name]; ok {
				log.Warn("Task %s already exists, overwriting")
			}

			env := make(map[string]string)
			for k, v := range GlobalWorkspace.Environment {
				env[k] = v
			}
			for k, v := range ws.Environment {
				env[k] = v
			}
			for k, v := range t.Environment {
				env[k] = v
			}

			task := NewTask(workspace, t.Name, t.Executor, t.Command, env, t.Service, t.Stdout, t.Stderr, t.Metadata, t.Pwd)
			workspace.Tasks[t.Name] = task
		}
	}

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", list_workspaces)
	r.Get("/favicon.ico", r.Static("assets/favicon.ico"))
	r.Get("/log", show_log)
	r.Get("/workspace/(?P<workspace>[^/]+)", list_tasks)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/start", startTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/stop", stopTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/enable", enableServiceTask)
	r.Post("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/disable", disableServiceTask)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)", taskHistory)

	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)", taskRun)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stdout", taskRunStdout)
	r.Get("/workspace/(?P<workspace>[^/]+)/task/(?P<task>[^/]+)/run/(?P<run>\\d+)/stderr", taskRunStderr)

	// Start our application
	app.Start()

	defer func() {
		for _, ws := range Workspaces {
			for _, t := range ws.Tasks {
				if t.ActiveTask != nil && t.ActiveTask.Cmd != nil && t.ActiveTask.Cmd.Process != nil {
					t.ActiveTask.Cmd.Process.Kill()
				}
			}
		}
	}()

	<-make(chan int)
}
Пример #19
0
func (tr *TaskRun) Start(exitCh chan int) {
	tr.Started = time.Now()

	stdout, err := tr.Cmd.StdoutPipe()
	if err != nil {
		tr.Error = err
		exitCh <- 1
		return
	}
	stderr, err := tr.Cmd.StderrPipe()
	if err != nil {
		tr.Error = err
		exitCh <- 1
		return
	}

	if len(tr.Stdout) > 0 {
		wr, err := NewFileLogWriter(tr.Stdout)
		if err != nil {
			log.Error("Unable to open file %s: %s", tr.Stdout, err.Error())
			tr.StdoutBuf = NewInMemoryLogWriter()
		} else {
			tr.StdoutBuf = wr
		}
	} else {
		tr.StdoutBuf = NewInMemoryLogWriter()
	}
	if len(tr.Stderr) > 0 {
		wr, err := NewFileLogWriter(tr.Stderr)
		if err != nil {
			log.Error("Unable to open file %s: %s", tr.Stderr, err.Error())
			tr.StderrBuf = NewInMemoryLogWriter()
		} else {
			tr.StderrBuf = wr
		}
	} else {
		tr.StderrBuf = NewInMemoryLogWriter()
	}

	if len(tr.Pwd) > 0 {
		log.Info("Setting pwd: %s", tr.Pwd)
		tr.Cmd.Dir = tr.Pwd
	}

	for k, v := range tr.Environment {
		log.Info("Adding env var %s = %s", k, v)
		tr.Cmd.Env = append(tr.Cmd.Env, k+"="+v)
	}

	err = tr.Cmd.Start()
	if tr.Cmd.Process != nil {
		ev := &Event{time.Now(), fmt.Sprintf("Process %d started: %s", tr.Cmd.Process.Pid, tr.Command)}
		log.Info(ev.Message)
		tr.Events = append(tr.Events, ev)
	}
	if err != nil {
		tr.Error = err
		log.Error(err.Error())
		tr.StdoutBuf.Close()
		tr.StderrBuf.Close()
		exitCh <- 1
		return
	}
	go func() {
		go io.Copy(tr.StdoutBuf, stdout)
		go io.Copy(tr.StderrBuf, stderr)

		tr.Cmd.Wait()

		tr.StdoutBuf.Close()
		tr.StderrBuf.Close()

		log.Trace("STDOUT: %s", tr.StdoutBuf.String())
		log.Trace("STDERR: %s", tr.StderrBuf.String())

		ps := tr.Cmd.ProcessState
		sy := ps.Sys().(syscall.WaitStatus)

		ev := &Event{time.Now(), fmt.Sprintf("Process %d exited with status %d", ps.Pid(), sy.ExitStatus())}
		log.Info(ev.Message)
		tr.Events = append(tr.Events, ev)
		log.Info(ps.String())

		tr.Stopped = time.Now()
		exitCh <- 1
	}()
}
Пример #20
0
func main() {
	configure()

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	indexes = make(map[string]map[string]*gopan.Source)
	if !config.NoCache {
		indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
	}

	if config.NoCache || config.Update {
		for _, s := range config.Sources {
			b := strings.SplitN(s, "=", 2)
			if len(b) < 2 {
				log.Error("Expected Name=URL pair, got: %s", s)
				return
			}

			if idx, ok := indexes[config.Index][b[0]]; ok {
				log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1])
				idx.URL = b[1]
			} else {
				indexes[config.Index][b[0]] = &gopan.Source{
					Name:    b[0],
					URL:     b[1],
					Authors: make(map[string]*gopan.Author, 0),
				}
			}
		}

		if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN {
			log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN")
			config.CPAN = true
			config.BackPAN = true
		}

		if config.CPAN {
			if _, ok := indexes[config.Index]["CPAN"]; !ok {
				log.Debug("Adding CPAN index")
				indexes[config.Index]["CPAN"] = gopan.CPANSource()
			} else {
				log.Debug("CPAN index already exists")
			}
		}

		if config.BackPAN {
			if _, ok := indexes[config.Index]["BackPAN"]; !ok {
				log.Debug("Adding BackPAN index")
				indexes[config.Index]["BackPAN"] = gopan.BackPANSource()
			} else {
				log.Debug("BackPAN index already exists")
			}
		}

		log.Info("Using sources:")
		for fname, _ := range indexes {
			log.Info("From %s", fname)
			for _, source := range indexes[fname] {
				log.Info("=> %s", source.String())
			}
		}

		newAuthors := getAuthors()
		newPackages := getPackages()

		os.MkdirAll(config.CacheDir, 0777)

		if !config.NoCache {
			gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index])
		}

		if config.Update {
			log.Info("Found %d new packages by %d new authors", newAuthors, newPackages)
		}
	}

	nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes)
	log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc)

	if !config.NoMirror {
		mirrorPan()
	}
}
Пример #21
0
func main() {
	configure()

	indexes = make(map[string]map[string]*gopan.Source)
	indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)

	log.Logger().SetLevel(log.Stol(config.LogLevel))
	log.Info("Using log level: %s", config.LogLevel)

	// FIXME inefficient
	_, _, tpkg, _ := gopan.CountIndex(indexes)

	npkg := 0
	nmod := 0

	var pc = func() float64 {
		return float64(nmod) / float64(tpkg) * 100
	}

	log.Info("Writing packages index file")

	out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
	if err != nil {
		log.Error("Error creating packages index: %s", err.Error())
		return
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, idx := range indexes[fname] {
			log.Debug("Index: %s", idx)
			out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
			for _, auth := range idx.Authors {
				log.Debug("Author %s", auth)
				out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
				for _, pkg := range auth.Packages {
					out.Write([]byte("  " + pkg.Name + " => " + pkg.URL + "\n"))
					log.Debug("Package: %s", pkg)

					if !config.Flatten {
						if len(pkg.Provides) == 0 {
							// TODO better handling of filenames
							modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")

							tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name

							if _, err := os.Stat(tgzpath); err != nil {
								log.Error("File not found: %s", tgzpath)
								return
							}

							extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
							dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name

							log.Trace("=> tgzpath: %s", tgzpath)
							log.Trace(" > extpath: %s", extpath)
							log.Trace(" > dirpath: %s", dirpath)

							// Only index packages if they don't already exist
							if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
								log.Error("Error retrieving package list: %s", err)
								continue
							}
						}

						npkg += len(pkg.Provides)
						nmod += 1

						for p, pk := range pkg.Provides {
							out.Write([]byte("   " + p + " (" + pk.Version + "): " + pk.File + "\n"))
						}

						if nmod > 0 && nmod%100 == 0 {
							log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
						}

					}
				}
			}
		}
	}

	out.Close()

	log.Info("Found %d packages from %d modules", npkg, nmod)
}
Пример #22
0
func (r *Response) EventStream() chan []byte {
	c := make(chan []byte)

	r.IsEventStream = true
	r.Headers.Add("Content-Type", "text/event-stream")
	r.Headers.Add("Cache-Control", "no-cache")
	r.Headers.Add("Connection", "keep-alive")
	//r.Write([]byte("\n\n"))
	r.Send()

	hj, ok := r.writer.(nethttp.Hijacker)
	if !ok {
		log.Warn("Connection unsuitable for hijack")
		return nil
	}
	conn, bufrw, err := hj.Hijack()
	if err != nil {
		log.Warn("Connection hijack failed")
		return nil
	}

	r.esBufrw = bufrw
	r.esConn = conn

	go func() {
		for b := range c {
			if len(b) == 0 {
				log.Trace("Event stream ended")
				r.esConn.Close()
				break
			}

			lines := strings.Split(string(b), "\n")
			data := ""
			for _, l := range lines {
				data += "data: " + l + "\n"
			}
			data += "\n"

			sz := len(data) + 1
			log.Info("Event stream message is %d bytes", sz)
			size := fmt.Sprintf("%X", sz)
			r.esBufrw.Write([]byte(size + "\r\n"))

			lines = strings.Split(data, "\n")
			for _, ln := range lines {
				r.esBufrw.Write([]byte(ln + "\n"))
			}
			_, err := r.esBufrw.Write([]byte("\r\n"))

			if err != nil {
				log.Error("Error writing to connection: %s\n", err)
				r.esConn.Close()
				break
			}

			err = r.esBufrw.Flush()
			if err != nil {
				log.Error("Error flushing buffer: %s\n", err)
				r.esConn.Close()
				break
			}
		}
	}()
	return c
}
Пример #23
0
func (s *settings) ValidateAndUseFile(filePath string) {
	cfg := loadConfigFile(filePath)
	cfg.Validate()
	s.c = cfg
	log.Info("Config loaded " + filePath)
}
Пример #24
0
func examplepost(session *http.Session) {
	m := &ExampleForm{}
	session.Stash["fh"] = form.New(session, m).Populate(false).Validate()
	log.Info("Got posted title: %s", m.Title)
	example(session)
}
Пример #25
0
func do_import(session *http.Session, job *ImportJob) {
	log.Info("Running import job %s", job.Id)

	reponame := job.Form.ImportInto
	if reponame == "new_index" {
		reponame = job.Form.NewIndex
	}

	msg := func(m string) {
		if m != ":DONE" {
			job.History = append(job.History, m)
			log.Info(m)
		}
		for _, w := range job.Watchers {
			w(m)
		}
	}

	mods := make([]*getpan.Module, 0)

	// TODO cpanm mirror when using getpan_import

	if len(job.Form.Cpanfile) > 0 {
		msg("Parsing cpanfile input")
		_, modules := getpan_import(job, msg)
		mods = append(mods, modules...)
	}

	if len(job.Form.ImportURL) > 0 {
		msg("Importing from URL: " + job.Form.ImportURL)

		// TODO support cpanfile urls

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.ImportURL)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		if _, err := os.Stat(nfile); err != nil {
			os.MkdirAll(npath, 0777)
			out, err := os.Create(nfile)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			url := job.Form.ImportURL
			log.Trace("Downloading: %s", url)
			resp, err := nethttp.Get(url)

			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			_, err = io.Copy(out, resp.Body)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			out.Close()
			resp.Body.Close()
		} else {
			log.Trace("File already exists in cache: %s", nfile)
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if len(job.Form.FromDir) > 0 {
		msg("Importing from local directory: " + job.Form.FromDir)

		// TODO support cpanfile paths

		nauth := job.Form.AuthorID

		if len(nauth) < 3 {
			// FIXME move to form validation
			msg("Author ID must be at least 3 characters")
			msg(":DONE")
			job.Complete = true
			return
		}

		npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
		_, fn := filepath.Split(job.Form.FromDir)
		nfile := npath + "/" + fn

		msg("Caching to " + nfile)

		_, err := CopyFile(nfile, job.Form.FromDir)
		if err != nil {
			msg(err.Error())
			msg(":DONE")
			job.Complete = true
			return
		}

		fn = strings.TrimSuffix(fn, ".tar.gz")
		bits := strings.Split(fn, "-")
		name := strings.Join(bits[0:len(bits)-1], "-")
		version := bits[len(bits)-1]

		s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
		m := &getpan.Module{
			Source:  s,
			Name:    name,
			Version: version,
			Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
			Cached:  nfile,
			Dir:     npath,
		}
		m.Deps = &getpan.DependencyList{
			Parent:       m,
			Dependencies: make([]*getpan.Dependency, 0),
		}
		mods = append(mods, m)
	}

	if f, fh, err := session.Request.File("fromfile"); err == nil {
		fn := fh.Filename

		msg("Importing from uploaded module/cpanfile: " + fn)

		if !strings.HasSuffix(fn, ".tar.gz") && fn != "cpanfile" {
			msg("Only cpanfile and *.tar.gz files are supported")
			msg(":DONE")
			job.Complete = true
			return
		}

		if fn == "cpanfile" {
			msg("Importing cpanfile")
			b, _ := ioutil.ReadAll(f)
			f.Close()
			job.Form.Cpanfile = string(b)
			_, modules := getpan_import(job, msg)
			mods = append(mods, modules...)
		} else {
			msg("Importing .tar.gz")

			nauth := job.Form.AuthorID

			if len(nauth) < 3 {
				// FIXME move to form validation
				msg("Author ID must be at least 3 characters")
				msg(":DONE")
				job.Complete = true
				return
			}

			npath := config.CacheDir + "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth
			_, fn = filepath.Split(fn)
			nfile := npath + "/" + fn

			msg("Caching to " + nfile)

			os.MkdirAll(npath, 0777)

			_, err := CopyToFile(nfile, f)
			if err != nil {
				msg(err.Error())
				msg(":DONE")
				job.Complete = true
				return
			}

			fn = strings.TrimSuffix(fn, ".tar.gz")
			bits := strings.Split(fn, "-")
			name := strings.Join(bits[0:len(bits)-1], "-")
			version := bits[len(bits)-1]

			s := getpan.NewSource("CPAN", "/modules/02packages.details.txt.gz", "")
			m := &getpan.Module{
				Source:  s,
				Name:    name,
				Version: version,
				Url:     "/authors/id/" + nauth[:1] + "/" + nauth[:2] + "/" + nauth + "/" + fn,
				Cached:  nfile,
				Dir:     npath,
			}
			m.Deps = &getpan.DependencyList{
				Parent:       m,
				Dependencies: make([]*getpan.Dependency, 0),
			}
			mods = append(mods, m)
		}
	} else {
		// there is no file... so no error
		//msg("Error importing file upload: " + err.Error())
	}

	if len(mods) == 0 {
		msg("Nothing to do")
		msg(":DONE")
		job.Complete = true
		return
	}

	msg("Adding modules to GoPAN index")

	for _, m := range mods {
		msg("=> " + m.Name + " (" + m.Cached + ")")

		dn, fn := filepath.Split(m.Cached)
		dnb := strings.Split(strings.TrimSuffix(dn, string(os.PathSeparator)), string(os.PathSeparator))
		auth := dnb[len(dnb)-1]
		ndir := config.CacheDir + "/" + reponame + "/" + auth[:1] + "/" + auth[:2] + "/" + auth
		npath := ndir + "/" + fn

		if _, err := os.Stat(npath); err == nil {
			msg(" | Already exists in repository")
		} else {
			os.MkdirAll(ndir, 0777)

			msg(" | Copying to " + npath)
			_, err := CopyFile(npath, m.Cached)
			if err != nil {
				msg(" ! " + err.Error())
				continue
			}
		}

		if _, ok := indexes[config.Index][reponame]; !ok {
			msg(" | Creating index: " + reponame)
			indexes[config.Index][reponame] = &gopan.Source{
				Name:    reponame,
				URL:     "/authors/id",
				Authors: make(map[string]*gopan.Author),
			}

			mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth]; !ok {
			msg(" | Creating author: " + auth)
			author := &gopan.Author{
				Source:   indexes[config.Index][reponame],
				Name:     auth,
				Packages: make(map[string]*gopan.Package),
				URL:      "/authors/id/" + auth[:1] + "/" + auth[:2] + "/" + auth + "/",
			}
			indexes[config.Index][reponame].Authors[auth] = author

			if _, ok := mapped[reponame]; !ok {
				mapped[reponame] = make(map[string]map[string]map[string]*gopan.Author)
			}

			// author name
			if _, ok := mapped[reponame][author.Name[:1]]; !ok {
				mapped[reponame][author.Name[:1]] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame][author.Name[:1]][author.Name[:2]]; !ok {
				mapped[reponame][author.Name[:1]][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]][author.Name[:2]][author.Name] = author

			// wildcards
			if _, ok := mapped[reponame]["*"]; !ok {
				mapped[reponame]["*"] = make(map[string]map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"]["**"]; !ok {
				mapped[reponame]["*"]["**"] = make(map[string]*gopan.Author)
			}
			mapped[reponame]["*"]["**"][author.Name] = author

			// combos
			if _, ok := mapped[reponame][author.Name[:1]]["**"]; !ok {
				mapped[reponame][author.Name[:1]]["**"] = make(map[string]*gopan.Author)
			}
			if _, ok := mapped[reponame]["*"][author.Name[:2]]; !ok {
				mapped[reponame]["*"][author.Name[:2]] = make(map[string]*gopan.Author)
			}
			mapped[reponame][author.Name[:1]]["**"][author.Name] = author
			mapped[reponame]["*"][author.Name[:2]][author.Name] = author
		}

		if _, ok := indexes[config.Index][reponame].Authors[auth].Packages[fn]; !ok {
			msg(" | Creating module: " + fn)
			indexes[config.Index][reponame].Authors[auth].Packages[fn] = &gopan.Package{
				Author:   indexes[config.Index][reponame].Authors[auth],
				Name:     fn,
				URL:      indexes[config.Index][reponame].Authors[auth].URL + fn,
				Provides: make(map[string]*gopan.PerlPackage),
			}

			msg(" | Getting list of packages")
			modnm := strings.TrimSuffix(fn, ".tar.gz")
			pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			if err := pandex.Provides(pkg, npath, ndir+"/"+modnm, ndir); err != nil {
				msg(" ! Error retrieving package list for " + pkg.Name + ": " + err.Error())
			}

			//pkg := indexes[config.Index][reponame].Authors[auth].Packages[fn]
			msg(" | Adding packages to index")
			if _, ok := idxpackages[reponame]; !ok {
				idxpackages[reponame] = make(map[string]*PkgSpace)
			}
			filemap[pkg.AuthorURL()] = reponame
			for _, prov := range pkg.Provides {
				parts := strings.Split(prov.Name, "::")
				if _, ok := packages[parts[0]]; !ok {
					packages[parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if _, ok := idxpackages[reponame][parts[0]]; !ok {
					idxpackages[reponame][parts[0]] = &PkgSpace{
						Namespace: parts[0],
						Packages:  make([]*gopan.PerlPackage, 0),
						Children:  make(map[string]*PkgSpace),
						Parent:    nil,
						Versions:  make(map[float64]*gopan.PerlPackage),
					}
				}
				if len(parts) == 1 {
					packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
					packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
					idxpackages[reponame][parts[0]].Packages = append(idxpackages[reponame][parts[0]].Packages, prov)
					idxpackages[reponame][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
				} else {
					packages[parts[0]].Populate(parts[1:], prov)
					idxpackages[reponame][parts[0]].Populate(parts[1:], prov)
				}
			}

			msg(" | Writing to index file")
			gopan.AppendToIndex(config.CacheDir+"/"+config.Index, indexes[config.Index][reponame], indexes[config.Index][reponame].Authors[auth], indexes[config.Index][reponame].Authors[auth].Packages[fn])
		}

		msg(" | Imported module")
	}

	nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
	// TODO should probably be in the index - needs to udpate when index changes
	summary = &Summary{nsrc, nauth, npkg, nprov}

	msg(":DONE")
	job.Complete = true
}
Пример #26
0
func import1(session *http.Session) {
	session.Stash["indexes"] = indexes
	session.Stash["Title"] = "SmartPAN Import"

	m := &ImportForm{}
	f := form.New(session, m)
	session.Stash["fh"] = f

	log.Info("Headers: %s", session.Request.Header())

	if session.Request.Method != "POST" {
		render_import(session)
		return
	}

	f.Populate(true)
	f.Validate()

	if f.HasErrors {
		render_import(session)
		return
	}

	log.Info("Importing into: %s", m.ImportInto)
	if m.ImportInto == "new_index" {
		if len(m.NewIndex) == 0 {
			f.HasErrors = true
			f.Errors["NewIndex"] = make(map[string]error)
			f.Errors["NewIndex"]["required"] = errors.New("Please give the new repository a name")
			render_import(session)
			return
		}
		log.Info("=> Creating new index: %s", m.NewIndex)
	}

	b := make([]byte, 20)
	rand.Read(b)
	en := base64.URLEncoding
	d := make([]byte, en.EncodedLen(len(b)))
	en.Encode(d, b)

	job := &ImportJob{
		Form:     m,
		Complete: false,
		Id:       string(d),
		Watchers: make([]func(string), 0),
	}

	if len(m.Cpanfile) > 0 {
		log.Info("Got cpanfile:")
		log.Info(m.Cpanfile)
	}

	log.Info("=> Created import job: %s", job.Id)
	imports[job.Id] = job

	go do_import(session, job)

	//render_import(session)
	if _, ok := session.Request.Form()["stream"]; ok {
		session.Redirect(&url.URL{Path: "/import/" + job.Id + "/stream", RawQuery: "raw=y"})
	} else {
		session.Redirect(&url.URL{Path: "/import/" + job.Id})
	}
}
Пример #27
0
func main() {
	configure()

	args := flag.Args()

	if len(args) > 0 && args[0] == "init" {
		log.Info("Initialising SmartPAN")

		log.Info("=> Installing Perl dependencies")

		// FIXME most of this is repeated from getpan/main.go
		cfg := getpan.DefaultConfig()
		cfg.CacheDir = config.CacheDir

		for _, source := range cfg.Sources {
			if err := source.Load(); err != nil {
				log.Error("Error loading sources: %s", err)
				os.Exit(1)
				return
			}
		}

		deps := &getpan.DependencyList{
			Dependencies: make([]*getpan.Dependency, 0),
		}

		d1, _ := getpan.DependencyFromString("Parse::LocalDistribution", "")
		d2, _ := getpan.DependencyFromString("JSON::XS", "")
		deps.AddDependency(d1)
		deps.AddDependency(d2)

		if err := deps.Resolve(); err != nil {
			log.Error("Error resolving dependencies: %s", err)
			os.Exit(1)
			return
		}

		_, err := deps.Install()
		if err != nil {
			log.Error("Error installing dependencies: %s", err)
			os.Exit(2)
			return
		}

		log.Info("   - Installed %d modules", deps.UniqueInstalled())

		log.Info("SmartPAN initialisation complete")

		return
	}

	if config.TestDeps {
		perldeps := gopan.TestPerlDeps()
		perldeps.Dump()
		if !perldeps.Ok {
			log.Error("Required perl dependencies are missing")
			os.Exit(1)
			return
		}
	}

	if len(args) > 0 && args[0] == "import" {
		if len(args) < 4 {
			log.Error("Invalid arguments, expecting: smartpan import FILE AUTHORID INDEX")
			return
		}

		fname := args[1]
		log.Info("Importing module from %s", fname)
		log.Info("Author ID: %s", args[2])
		log.Info("Index    : %s", args[3])

		extraParams := map[string]string{
			"importinto": args[3],
			"authorid":   args[2],
			"newindex":   "",
			"cpanmirror": "",
			"importurl":  "",
			"fromdir":    "",
		}

		if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
			log.Info("URL: %s", fname)

			extraParams["importurl"] = fname

			request, err := newFormPostRequest(config.RemoteHost+"/import?stream=y", extraParams)
			if err != nil {
				log.Error("Create request error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)

			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		} else {
			fname = strings.TrimPrefix(fname, "file://")
			log.Info("File: %s", fname)

			if _, err := os.Stat(fname); err != nil {
				log.Error("File not found: %s", err.Error())
				return
			}

			request, err := newfileUploadRequest(config.RemoteHost+"/import?stream=y", extraParams, "fromfile", fname)
			if err != nil {
				log.Error("Create upload error: %s", err.Error())
				return
			}

			client := &nethttp.Client{}
			resp, err := client.Do(request)
			if err != nil {
				log.Error("Error connecting to host: %s", err.Error())
				return
			} else {
				// TODO stream this
				body := &bytes.Buffer{}
				_, err := body.ReadFrom(resp.Body)
				if err != nil {
					log.Error("Error reading response: %s", err.Error())
					return
				}
				resp.Body.Close()
				//log.Info("%d", resp.StatusCode)
				//log.Info("%s", resp.Header)
				log.Info("%s", body.String())
			}
		}

		return
	}

	config.CurrentRelease = CurrentRelease

	var wg sync.WaitGroup

	load_index = func(index string, file string) {
		indexes[index] = gopan.LoadIndex(file)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()

		indexes = make(map[string]map[string]*gopan.Source)

		// Load CPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.CPANIndex); err == nil {
			config.HasCPANIndex = true
			config.CPANIndexDate = fi.ModTime().String()
			config.CPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex)
				config.CPANStatus = "Loaded"
			}()
		}

		// Load BackPAN index
		if fi, err := os.Stat(config.CacheDir + "/" + config.BackPANIndex); err == nil {
			config.HasBackPANIndex = true
			config.BackPANIndexDate = fi.ModTime().String()
			config.BackPANStatus = "Loading"
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex)
				config.BackPANStatus = "Loaded"
			}()
		}

		// Load our secondary indexes
		for _, idx := range config.Indexes {
			wg.Add(1)
			go func() {
				defer wg.Done()
				load_index(idx, config.CacheDir+"/"+idx)
			}()
		}

		// Load our primary index (this is the only index written back to)
		wg.Add(1)
		go func() {
			defer wg.Done()
			load_index(config.Index, config.CacheDir+"/"+config.Index)
		}()
	}()

	update_indexes = func() {
		wg.Wait()
		wg.Add(1)
		go func() {
			wg.Wait()
			config.ImportAvailable = true

			nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
			// TODO should probably be in the index - needs to udpate when index changes
			summary = &Summary{nsrc, nauth, npkg, nprov}

			// Do this now so changing the level doesn't interfere with index load
			log.Logger().SetLevel(log.Stol(config.LogLevel))
		}()
		defer wg.Done()
		// Create in-memory indexes for UI/search etc
		for fname, _ := range indexes {
			for idn, idx := range indexes[fname] {
				mapped[idx.Name] = make(map[string]map[string]map[string]*gopan.Author)
				for _, auth := range idx.Authors {
					// author name
					if _, ok := mapped[idx.Name][auth.Name[:1]]; !ok {
						mapped[idx.Name][auth.Name[:1]] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name][auth.Name[:1]][auth.Name[:2]]; !ok {
						mapped[idx.Name][auth.Name[:1]][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]][auth.Name[:2]][auth.Name] = auth

					// wildcards
					if _, ok := mapped[idx.Name]["*"]; !ok {
						mapped[idx.Name]["*"] = make(map[string]map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"]["**"]; !ok {
						mapped[idx.Name]["*"]["**"] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name]["*"]["**"][auth.Name] = auth

					// combos
					if _, ok := mapped[idx.Name][auth.Name[:1]]["**"]; !ok {
						mapped[idx.Name][auth.Name[:1]]["**"] = make(map[string]*gopan.Author)
					}
					if _, ok := mapped[idx.Name]["*"][auth.Name[:2]]; !ok {
						mapped[idx.Name]["*"][auth.Name[:2]] = make(map[string]*gopan.Author)
					}
					mapped[idx.Name][auth.Name[:1]]["**"][auth.Name] = auth
					mapped[idx.Name]["*"][auth.Name[:2]][auth.Name] = auth

					for _, pkg := range auth.Packages {
						filemap[pkg.AuthorURL()] = idn
						for _, prov := range pkg.Provides {
							parts := strings.Split(prov.Name, "::")
							log.Trace("PACKAGE: %s", prov.Name)

							if _, ok := packages[parts[0]]; !ok {
								packages[parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if _, ok := idxpackages[idx.Name]; !ok {
								idxpackages[idx.Name] = make(map[string]*PkgSpace)
							}
							if _, ok := idxpackages[idx.Name][parts[0]]; !ok {
								idxpackages[idx.Name][parts[0]] = &PkgSpace{
									Namespace: parts[0],
									Packages:  make([]*gopan.PerlPackage, 0),
									Children:  make(map[string]*PkgSpace),
									Parent:    nil,
									Versions:  make(map[float64]*gopan.PerlPackage),
								}
							}
							if len(parts) == 1 {
								packages[parts[0]].Packages = append(packages[parts[0]].Packages, prov)
								packages[parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								idxpackages[idx.Name][parts[0]].Packages = append(idxpackages[idx.Name][parts[0]].Packages, prov)
								idxpackages[idx.Name][parts[0]].Versions[gopan.VersionFromString(prov.Version)] = prov
								log.Trace("Version linked: %f for %s", gopan.VersionFromString(prov.Version), prov.Name)
							} else {
								packages[parts[0]].Populate(parts[1:], prov)
								idxpackages[idx.Name][parts[0]].Populate(parts[1:], prov)
							}
						}
					}
				}
			}
		}
	}
	go update_indexes()

	// Get latest SmartPAN version
	go func() {
		res, err := nethttp.Get("https://api.github.com/repos/companieshouse/gopan/releases")
		if err != nil {
			log.Error("Error getting latest version: %s", err.Error())
			return
		}
		defer res.Body.Close()
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Error("Error reading stream: %s", err.Error())
			return
		}

		var r Releases
		if err = json.Unmarshal(b, &r); err != nil {
			log.Error("Error unmarshalling JSON: %s", err.Error())
			return
		}

		log.Info("Current release: %s", config.CurrentRelease)
		rel := strings.TrimPrefix(r[0].TagName, "v")
		log.Info("Latest release: %s", rel)
		config.LatestRelease = rel
		config.UpdateURL = r[0].URL

		if config.CurrentRelease < rel {
			config.CanUpdate = true
			log.Info("Your version of SmartPAN can be updated.")
		}
	}()

	// Create our Gotcha application
	var app = gotcha.Create(Asset)
	app.Config.Listen = config.Bind

	summary = &Summary{0, 0, 0, 0}

	app.On(events.BeforeHandler, func(session *http.Session, next func()) {
		session.Stash["summary"] = summary
		session.Stash["config"] = config

		next()
	})

	// Get the router
	r := app.Router

	// Create some routes
	r.Get("/", search)
	r.Post("/", search)

	r.Get("/help", help)
	r.Get("/settings", settings)
	r.Get("/browse", browse)

	r.Get("/import", import1)
	r.Post("/import", import1)

	r.Get("/import/(?P<jobid>[^/]+)", import2)
	r.Get("/import/(?P<jobid>[^/]+)/stream", importstream)

	r.Post("/get-index/(?P<index>(CPAN|BackPAN))/?", getindex)

	// Serve static content (but really use a CDN)
	r.Get("/images/(?P<file>.*)", r.Static("assets/images/{{file}}"))
	r.Get("/css/(?P<file>.*)", r.Static("assets/css/{{file}}"))

	// JSON endpoints
	r.Get("/where/(?P<module>[^/]+)/?", where)
	r.Get("/where/(?P<module>[^/]+)/(?P<version>[^/]+)/?", where)

	// Put these last so they only match /{repo} if nothing else matches
	r.Get("/(?P<repo>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/?", browse)
	r.Get("/(?P<repo>[^/]+)/modules/02packages\\.details\\.txt(?P<gz>\\.gz)?", pkgindex)
	r.Get("/(?P<repo>[^/]+)/authors/id/(?P<file>.*\\.tar\\.gz)", download)
	r.Post("/delete/(?P<repo>[^/]+)/authors/id/(?P<auth1>[^/]+)/(?P<auth2>[^/]+)/(?P<auth3>[^/]+)/(?P<file>.*\\.tar\\.gz)", delete_file)
	r.Get("/(?P<repo>[^/]+)/(?P<type>[^/]+)/(?P<path>.*)/?", browse)

	// Start our application
	app.Start()

	<-make(chan int)
}
Пример #28
0
func (m *Module) PrintDeps(d int) {
	log.Info(MkIndent(d)+"%s (%s): %s", m.Name, m.Version, m.Cached)
	if m.Deps != nil {
		m.Deps.PrintDeps(d + 1)
	}
}
Пример #29
0
func getPackages() int {
	newpkg := 0

	var pl func(*html.Node, *gopan.Source, *gopan.Author)
	pl = func(n *html.Node, source *gopan.Source, author *gopan.Author) {
		log.Trace("NODE: %s [%s, %s, %s]", n.DataAtom, n.Type, n.Data)
		if n.Type == html.ElementNode && n.Data == "a" {
			//log.Info("NODE IS ELEMENTNODE")
			for _, attr := range n.Attr {
				// FIXME stuff that isn't .tar.gz?
				if attr.Key == "href" && strings.HasSuffix(attr.Val, ".tar.gz") {
					log.Trace("==> HREF: %s", n.FirstChild.Data)
					pkg := strings.TrimSuffix(n.FirstChild.Data, "/")
					if _, ok := author.Packages[pkg]; !ok {
						author.Packages[pkg] = &gopan.Package{
							Name:   pkg,
							Author: author,
							URL:    author.URL + "/" + pkg,
						}
						newpkg++
						log.Debug("Found package: %s", pkg)
					}
				}
			}
			//log.Info("%s", n.Data)
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			pl(c, source, author)
		}
	}

	log.Info("Building package list")

	for fname, _ := range indexes {
		for _, source := range indexes[fname] {
			log.Debug("Index: %s", source)
			wg.Add(1)
			go func(source *gopan.Source) {
				defer wg.Done()
				for _, author := range source.Authors {
					wg.Add(1)
					go func(author *gopan.Author) {
						defer wg.Done()
						sem <- 1
						log.Trace("=> %s", author)

						url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
						log.Trace("Getting URL: %s", url)

						res, err := http.Get(url)
						if err != nil {
							log.Error("HTTP GET - %s", err.Error())
							<-sem
							return
						}

						doc, err := html.Parse(res.Body)
						if err != nil {
							log.Error("HTML PARSE - %s", err.Error())
							<-sem
							return
						}

						pl(doc, source, author)

						<-sem
					}(author)
				}
			}(source)
		}
	}

	wg.Wait()

	log.Info("Finished building package list")

	return newpkg
}
Пример #30
0
func mirrorPan() {
	log.Info("Mirroring *PAN")

	// FIXME inefficient
	_, _, npkg, _ := gopan.CountIndex(indexes)

	mirrored := 0
	var pc = func() int {
		return mirrored / npkg * 100
	}

	for fname, _ := range indexes {
		log.Debug("File: %s", fname)
		for _, source := range indexes[fname] {
			log.Debug("Index: %s", source)
			wg.Add(1)
			go func(source *gopan.Source) {
				defer wg.Done()
				for _, author := range source.Authors {
					log.Debug("=> %s", author)
					wg.Add(1)
					go func(author *gopan.Author) {
						cachedir := config.CacheDir + "/" + source.Name + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
						os.MkdirAll(cachedir, 0777)

						defer wg.Done()
						for _, pkg := range author.Packages {
							wg.Add(1)
							go func(pkg *gopan.Package) {
								defer wg.Done()

								cache := cachedir + pkg.Name
								log.Trace("    - Caching to: %s", cache)

								if _, err := os.Stat(cache); err == nil {
									log.Debug("%d%%  |> %s", pc(), pkg)
									log.Trace("    - Already exists in cache")
									mirrored++
									return
								}

								sem <- 1

								mirrored++

								log.Debug("%d%%  => %s", pc(), pkg)

								url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" + pkg.Name
								log.Trace("    - From URL: %s", url)

								out, err := os.Create(cache)
								defer out.Close()
								if err != nil {
									log.Error("CREATE - %s", err.Error())
									<-sem
									return
								}

								resp, err := http.Get(url)
								if err != nil {
									log.Error("HTTP GET - %s", err.Error())
									<-sem
									return
								}

								_, err = io.Copy(out, resp.Body)
								if err != nil {
									log.Error("IO COPY - %s", err.Error())
								}

								<-sem
							}(pkg)
						}
					}(author)
				}
			}(source)
		}
	}

	wg.Wait()
	log.Info("Finished mirroring *PAN")
}