// NetworkName returns the SSID of the wifi network.
func NetworkName(nif string) string {
if nif == "" {
nif = C.GoString(C.guessWifiInterfaceName())
}
if nif == "" {
log.Debug("Could not find Wi-Fi network interface")
return ""
}
active := C.getWifiActive(C.CString(nif))
powerOn := C.getWifiPowerOn(C.CString(nif))
if !active {
log.Debug("Wi-Fi network interface is not active")
return ""
}
if !powerOn {
log.Debug("Wi-Fi network interface is not powered on")
return ""
}
ssid := C.GoString(C.getWifiSSID(C.CString(nif)))
if ssid == "" {
log.Error("Wi-Fi network interface ssid empty")
}
return ssid
}
func (h *Router) Static(filename string) HandlerFunc {
return func(session *http.Session) {
// TODO beware of ..?
re := regexp.MustCompile("{{(\\w+)}}")
fcopy := re.ReplaceAllStringFunc(filename, func(m string) string {
parts := re.FindStringSubmatch(m)
log.Trace("Found var: %s; name: %s", m, parts[1])
if val, ok := session.Stash[parts[1]]; ok {
log.Trace("Value found in stash for %s: %s", parts[1], val)
return val.(string)
}
log.Trace("No value found in stash for var: %s", parts[1])
return m
})
asset, err := h.Config.AssetLoader(fcopy)
if err != nil {
log.Debug("Static file not found: %s", fcopy)
session.RenderNotFound()
} else {
m := MIME.TypeFromFilename(fcopy)
if len(m) > 0 {
log.Debug("Setting Content-Type: %s", m)
session.Response.Headers.Add("Content-Type", m[0])
}
session.Response.Write(asset)
}
}
}
func (pd *PerlDeps) Dump() {
log.Debug("Perl dependencies:")
log.Debug(" Found perl => %t", pd.HasPerl)
log.Debug(" Perl version => %s", pd.PerlVersion)
log.Debug(" JSON::XS => %t", pd.HasJSONXS)
log.Debug(" Parse::LocalDistribution => %t", pd.HasParseLocalDistribution)
}
// Resolve dependencies in a dependency list
// Resolves dependencies in order they occured originally
func (d *DependencyList) Resolve() error {
if d == nil {
log.Debug("No dependencies to resolve")
return nil
}
log.Debug("Resolving dependencies")
errs := make([]string, 0)
for _, dep := range d.Dependencies {
log.Debug("Resolving module dependency: %s", dep)
if err := dep.Resolve(d.Parent); err != nil {
log.Error("Error resolving module dependencies [%s]: %s", dep, err)
errs = append(errs, dep.String())
break
}
}
if len(errs) > 0 {
log.Error("Failed to find dependencies:")
for _, err := range errs {
log.Error("=> %s", err)
}
return errors.New("Failed to find dependencies")
}
return nil
}
// SetBrightness takes a float value betweem [0,1] and sets the global screen brightness.
func SetBrightness(vNorm float64) {
v := C.float(constrain(vNorm, 0, 1))
log.Debug("Setting brightness to %v (%v)", vNorm, v)
if res := C.setBrightness(v); int(res) != 0 {
log.Fatalf("Failed to set brightness: %d", res)
}
}
// SetVolume takes a float value between [0,1] and sets the global output volume.
func SetVolume(vNorm float64) {
v := C.float(constrain(vNorm, 0, 1))
log.Debug("Setting volume to %v (%v)", vNorm, v)
if res := C.setVolume(v); int(res) != 0 {
log.Fatalf("Failed to set volume: %d", res)
}
}
func runLoop() {
conf, err := readConf()
AssertNoErr(err, "Failed to read config file")
log.Debug("Global conf: %#v", conf)
nif := conf["config"]["if"]
interval := atoi(conf["config"]["interval"], 5)
for name := range notifyOnChange(nif, interval) {
log.Info("Network changed: %s", name)
if section, ok := conf["ssid:"+name]; ok {
log.Debug("Found section: %v", section)
ApplyCmds(section)
} else {
log.Debug("Undefined section for: %s", name)
}
}
}
func (s *Source) Load() error {
log.Debug("Loading source: %s", s)
switch s.Type {
case "CPAN":
log.Debug("=> Got CPAN source")
return s.loadCPANSource()
case "BackPAN":
log.Debug("=> Got BackPAN source")
return s.loadBackPANSource()
case "SmartPAN":
log.Debug("=> Got SmartPAN source")
return nil
case "MetaCPAN":
log.Debug("=> Got MetaCPAN source")
return nil
default:
log.Error("Unrecognised source type: %s", s.Type)
return errors.New(fmt.Sprintf("Unrecognised source: %s", s))
}
}
func pkgindex(session *http.Session) {
if _, ok := session.Stash["repo"]; !ok {
session.RenderNotFound()
return
}
repo := session.Stash["repo"].(string)
for fname, _ := range indexes {
if _, ok := indexes[fname][repo]; !ok && repo != "SmartPAN" {
session.RenderNotFound()
return
}
}
if g, ok := session.Stash["gz"]; ok {
if len(g.(string)) > 0 {
// cheat and hijack gotchas gzip support
session.Response.Headers.Set("Content-Type", "application/gzip")
session.Response.Send()
session.Response.Gzip()
session.Response.Headers.Remove("Content-Encoding")
log.Debug("Using gzip")
}
}
session.Response.WriteText("File: 02packages.details.txt\n")
session.Response.WriteText("Description: Package names found in directory " + repo + "/authors/id\n")
session.Response.WriteText("Columns: package name, version, path\n")
session.Response.WriteText("Written-By: SmartPAN (from GoPAN)\n")
session.Response.WriteText("Line-Count: " + strconv.Itoa(summary.Packages) + "\n") // FIXME wrong count
session.Response.WriteText("\n")
if repo == "SmartPAN" {
for _, pkg := range packages {
writepkgindex(session, pkg)
}
} else {
for _, pkg := range idxpackages[repo] {
writepkgindex(session, pkg)
}
}
}
func notifyOnChange(nif string, interval int) <-chan (string) {
previousName := NetworkName(nif)
c := make(chan string, 1)
if previousName != "" {
c <- previousName
}
go func() {
for _ = range time.Tick(time.Duration(interval) * time.Second) {
currentName := NetworkName(nif)
log.Debug("Current name: %s", currentName)
if previousName != currentName && currentName != "" {
previousName = currentName
c <- currentName
}
}
}()
return c
}
func Provides(pkg *gopan.Package, tgzpath string, extpath string, dirpath string) error {
// not required? path should already exist
os.MkdirAll(dirpath, 0770)
var stdout1 bytes.Buffer
var stderr1 bytes.Buffer
extract := exec.Command("tar", "-zxf", tgzpath, "-C", dirpath)
extract.Stdout = &stdout1
extract.Stderr = &stderr1
if err := extract.Run(); err != nil {
log.Error("Extract run: %s", err.Error())
log.Trace(stdout1.String())
log.Error(stderr1.String())
return err
}
log.Trace(stdout1.String())
log.Trace(stderr1.String())
defer func() {
var stdout3 bytes.Buffer
var stderr3 bytes.Buffer
clean := exec.Command("rm", "-rf", extpath)
clean.Stdout = &stdout3
clean.Stderr = &stderr3
if err := clean.Run(); err != nil {
log.Error("Clean run: %s", err.Error())
}
log.Trace(stdout3.String())
log.Trace(stderr3.String())
}()
//var stdout2 bytes.Buffer
var stderr2 bytes.Buffer
if len(pldArgs) == 0 {
if len(os.Getenv("GOPAN_ALLOW_DEV_VERSIONS")) > 0 {
pldArgs = "({ALLOW_DEV_VERSION=>1})"
} else {
pldArgs = "()"
}
}
log.Trace("pldArgs: %s", pldArgs)
//cmd := exec.Command("perl", "-MModule::Metadata", "-MJSON::XS", "-e", "print encode_json(Module::Metadata->provides(version => 2, prefix => \"\", dir => $ARGV[0]))", extpath)
cmd := exec.Command("perl", "-MParse::LocalDistribution", "-MJSON::XS", "-e", "print encode_json(Parse::LocalDistribution->new"+pldArgs+"->parse($ARGV[0]))", extpath)
//cmd.Stdout = &stdout2
cmd.Stderr = &stderr2
stdout, err := cmd.StdoutPipe()
defer stdout.Close()
if err != nil {
log.Error("StdoutPipe: %s", err.Error())
return err
}
if err := cmd.Start(); err != nil {
log.Error("Start: %s", err.Error())
return err
}
var pld PLD
if err := json.NewDecoder(stdout).Decode(&pld); err != nil {
log.Error("JSON decoder error: %s", err.Error())
return err
}
if err := cmd.Wait(); err != nil {
log.Error("Wait: %s", err.Error())
return err
}
//log.Trace(stdout2.String())
log.Trace(stderr2.String())
pkg.Provides = make(map[string]*gopan.PerlPackage)
for p, pk := range pld {
pp := &gopan.PerlPackage{
Package: pkg,
Name: p,
Version: pk.Version,
File: pk.Infile,
}
pkg.Provides[p] = pp
log.Trace("%s: %s %s", p, pp.Version, pp.File)
}
log.Debug("%s provides %d packages", pkg, len(pkg.Provides))
return nil
}
func (s *Source) Find(d *Dependency) (*Module, error) {
log.Debug("Finding dependency: %s", d)
switch s.Type {
case "SmartPAN":
log.Debug("=> Using SmartPAN source")
url := s.URL
if !strings.HasSuffix(s.URL, "/") {
url += "/"
}
url += "where/" + d.Name + "/" + d.Modifier + d.Version
log.Info("Query: %s", url)
res, err := http.Get(url)
if err != nil {
log.Error("Error querying SmartPAN: %s", err.Error())
return nil, err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
log.Trace("Got response: %s", string(body))
if res.StatusCode != http.StatusOK {
log.Info("Module not found in SmartPAN: %s", d.Name)
return nil, nil
}
var v *WhereOutput
if err = json.Unmarshal(body, &v); err != nil {
log.Error("Error parsing JSON: %s", err.Error())
return nil, err
}
log.Trace("Found module %s", v.Module)
if len(v.Versions) == 0 {
log.Info("Found module but no versions returned")
return nil, nil
}
var lv *VersionOutput
for _, ver := range v.Versions {
if ver.Version == v.Latest {
log.Info("Using latest version of %s: %f", v.Module, ver.Version)
lv = ver
break
}
}
if lv == nil {
log.Info("Couldn't find latest version, selecting first available")
lv = v.Versions[0]
}
return &Module{
Name: d.Name,
Version: fmt.Sprintf("%f", lv.Version),
Source: s,
Url: lv.URL,
}, nil
case "CPAN":
log.Debug("=> Using CPAN source")
if mod, ok := s.ModuleList[d.Name]; ok {
log.Trace("=> Found in source: %s", mod)
if d.Matches(mod) {
log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
return mod, nil
}
log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
return nil, nil
}
case "BackPAN":
log.Debug("=> Using BackPAN source")
// TODO better version matching - new backpan index?
if mod, ok := s.ModuleList[d.Name+"-"+d.Version]; ok {
log.Trace("=> Found in source: %s", mod)
if d.Matches(mod) {
log.Trace("=> Version (%s) matches dependency: %s", mod.Version, d)
return mod, nil
}
log.Trace("=> Version (%s) doesn't match dependency: %s", mod.Version, d)
return nil, nil
}
case "MetaCPAN":
log.Debug("=> Using MetaCPAN source")
var sout, serr bytes.Buffer
var cpanm_args string = fmt.Sprintf("-L %s --info %s~\"%s%s\"", config.InstallDir, d.Name, d.Modifier, d.Version)
cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
if err != nil {
log.Error("Failed to get absolute path of gopan cache directory: %s", err)
return nil, err
}
log.Trace("About to exec: cpanm %s", cpanm_args)
os.Setenv("CPANM_INFO_ARGS", cpanm_args)
os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)
cmd := exec.Command("bash", "-c", `eval cpanm $CPANM_INFO_ARGS`)
cmd.Stdout = &sout
cmd.Stderr = &serr
if err := cmd.Run(); err != nil {
log.Error("cpanm %s: %s,\n%s\n", cpanm_args, err, serr.String())
return nil, nil
}
if 0 == len(sout.String()) {
log.Warn("No author/module from cpanm")
return nil, nil
}
author_module := strings.TrimRight(sout.String(), "\n")
mematches := metacpanRe.FindStringSubmatch(author_module)
if nil == mematches {
log.Error("Match failed for: %s", author_module)
return nil, nil
}
log.Trace("Resolved: %s", author_module)
for _, mesource := range config.MetaSources {
meurl := fmt.Sprintf("authors/id/%s/%s/%s",
mematches[1][0:1],
mematches[1][0:2],
mematches[0])
archive_url := fmt.Sprintf("%s/%s", mesource.URL, meurl)
log.Trace("Checking: " + archive_url)
resp, err := http.Head(archive_url)
if err != nil {
log.Trace(err)
continue
}
log.Trace("HEAD status code: %d", resp.StatusCode)
if 200 == resp.StatusCode {
// No module/version check since 'cpanm --info' may resolve to
// archive and version that may not match source
return &Module{
Name: mematches[2],
Version: mematches[3],
Source: mesource,
Url: meurl,
}, nil
}
}
log.Error("Could not get archive URL via 'cpanm %s'", cpanm_args)
return nil, nil
default:
log.Error("Unrecognised source type: %s", s.Type)
return nil, errors.New(fmt.Sprintf("Unrecognised source: %s", s))
}
log.Trace("=> Not found in source")
return nil, nil
}
func getAuthors() int {
newauth := 0
var al func(*html.Node, *gopan.Source, string)
al = func(n *html.Node, source *gopan.Source, prefix string) {
log.Trace("NODE: %s [%s, %s, %s]", n.DataAtom, n.Type, n.Data)
if n.Type == html.ElementNode && n.Data == "a" {
//log.Trace("NODE IS ELEMENTNODE")
for _, attr := range n.Attr {
log.Trace("==> TEXT: %s", n.FirstChild.Data)
if attr.Key == "href" && strings.HasPrefix(n.FirstChild.Data, prefix) {
log.Trace("==> HREF: %s", attr.Val)
author := strings.TrimSuffix(n.FirstChild.Data, "/")
if _, ok := source.Authors[author]; !ok {
source.Authors[author] = &gopan.Author{
Name: author,
Source: source,
Packages: make(map[string]*gopan.Package),
URL: source.URL + "/" + author[:1] + "/" + author[:2] + "/" + author + "/",
}
newauth++
log.Debug("Found author: %s", author)
}
}
}
//log.Trace("%s", n.Data)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
al(c, source, prefix)
}
}
log.Info("Building author list")
for fname, _ := range indexes {
for _, source := range indexes[fname] {
log.Info("Generating index: %s", source.Name)
wg.Add(1)
go func(source *gopan.Source) {
defer wg.Done()
for _, p1 := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" {
wg.Add(1)
go func(p1 rune, source *gopan.Source) {
defer wg.Done()
for _, p2 := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" {
wg.Add(1)
go func(p2 rune) {
defer wg.Done()
sem <- 1
log.Trace("=> %s%s", string(p1), string(p2))
url := source.URL + "/" + string(p1) + "/" + string(p1) + string(p2) + "/"
log.Trace("Getting URL: %s", url)
res, err := http.Get(url)
if err != nil {
log.Error("HTTP GET - %s", err.Error())
<-sem
return
}
doc, err := html.Parse(res.Body)
if err != nil {
log.Error("HTML PARSE - %s", err.Error())
<-sem
return
}
al(doc, source, string(p1)+string(p2))
<-sem
}(p2)
}
}(p1, source)
}
}(source)
}
}
wg.Wait()
log.Info("Finished building author list")
return newauth
}
func main() {
config = getpan.Configure()
config.Dump()
mods := flag.Args()
if len(mods) == 0 {
if _, err := os.Stat(config.CPANFile); os.IsNotExist(err) {
log.Error("cpanfile not found: %s", config.CPANFile)
os.Exit(1)
}
}
if len(mods) > 0 && mods[0] == "exec" {
log.Debug("getpan exec => " + strings.Join(mods[1:], " "))
cmd := exec.Command(mods[1], mods[2:]...)
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "PERL5LIB="+config.InstallDir+"/lib/perl5")
cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")+":"+config.InstallDir+"/bin")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
// debug so it doesn't show up in stdout/stderr unless -loglevel is used
log.Debug("Error in exec: %s", err.Error())
os.Exit(10)
}
return
}
for _, source := range config.Sources {
err := source.Load()
if err != nil {
log.Error("Error loading sources: %s", err)
os.Exit(1)
return
}
}
var deps *getpan.DependencyList
if len(mods) == 0 {
log.Info("Installing from cpanfile: %s", config.CPANFile)
d, err := getpan.ParseCPANFile(config.CPANFile)
if err != nil {
log.Error("Error parsing cpanfile: %s", err)
os.Exit(2)
return
}
deps = &d.DependencyList
} else {
log.Info("Installing from command line args")
deps = &getpan.DependencyList{
Dependencies: make([]*getpan.Dependency, 0),
}
for _, arg := range mods {
dependency, err := getpan.DependencyFromString(arg, "")
if err != nil {
log.Error("Unable to parse input: %s", arg)
continue
}
deps.AddDependency(dependency)
}
}
err := deps.Resolve()
if err != nil {
log.Error("Error resolving dependencies: %s", err)
os.Exit(3)
return
}
if false == config.NoDepdump {
log.Info("Resolved dependency tree:")
deps.PrintDeps(0)
}
if config.NoInstall {
log.Info("Skipping installation phase")
return
}
_, err = deps.Install()
if err != nil {
log.Error("Error installing dependencies: %s", err)
os.Exit(4)
return
}
// FIXME hacky, need a better way of tracking installed deps
log.Info("Successfully installed %d modules", deps.UniqueInstalled())
}
func download(session *http.Session) {
if _, ok := session.Stash["repo"]; !ok {
session.RenderNotFound()
return
}
if _, ok := session.Stash["file"]; !ok {
session.RenderNotFound()
return
}
repo := session.Stash["repo"].(string)
file := session.Stash["file"].(string)
if repo == "SmartPAN" {
if _, ok := filemap[file]; !ok {
log.Debug("SmartPAN repo - file [%s] not found in any index", file)
session.RenderNotFound()
return
}
repo = filemap[file]
log.Debug("SmartPAN repo - file [%s] found in [%s]", file, repo)
}
log.Debug("Repo [%s], file [%s]", repo, file)
nfile := config.CacheDir + "/" + repo + "/" + file
if _, err := os.Stat(nfile); err != nil {
log.Debug("File not found on disk, considering readthrough")
for fn, _ := range indexes {
log.Debug("Trying file: %s", fn)
if src, ok := indexes[fn][repo]; ok {
log.Debug("Found matching repo")
if strings.HasPrefix(src.URL, "http:") {
log.Debug("Found HTTP URL, trying: %s", src.URL+"/"+file)
res, err := nethttp.Get(src.URL + "/" + file)
if err != nil {
log.Debug("Error on readthrough: %s", err.Error())
continue
}
defer res.Body.Close()
b, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Debug("Error reading body: %s", err.Error())
continue
}
session.Response.Write(b)
return
}
}
}
log.Debug("No readthrough available")
session.RenderNotFound()
return
}
f, err := os.Open(nfile)
if err != nil {
log.Error(err.Error())
session.RenderNotFound()
return
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
log.Error(err.Error())
session.RenderNotFound()
return
}
session.Response.Write(b)
}
func delete_file(session *http.Session) {
session.Stash["Title"] = "Delete file"
html, _ := session.RenderTemplate("delete.html")
repo := session.Stash["repo"].(string)
file := session.Stash["file"].(string)
auth1 := session.Stash["auth1"].(string)
auth2 := session.Stash["auth2"].(string)
auth3 := session.Stash["auth3"].(string)
fname := config.CacheDir + "/" + repo + "/" + auth1 + "/" + auth2 + "/" + auth3 + "/" + file
if _, err := os.Stat(fname); err != nil {
session.RenderNotFound()
return
}
// Remove file from indexes
for f, _ := range indexes {
if _, ok := indexes[f][repo]; !ok {
continue
}
if _, ok := indexes[f][repo].Authors[auth3]; !ok {
continue
}
if _, ok := indexes[f][repo].Authors[auth3].Packages[file]; !ok {
continue
}
log.Debug("Removing from index: %s", repo)
pkg := indexes[f][repo].Authors[auth3].Packages[file]
delete(indexes[f][repo].Authors[auth3].Packages, file)
if len(indexes[f][repo].Authors[auth3].Packages) == 0 {
log.Debug("Removing author")
delete(indexes[f][repo].Authors, auth3)
}
if len(indexes[f][repo].Authors) == 0 {
log.Debug("Removing index")
delete(indexes[f], repo)
}
if auth, ok := mapped[repo][auth1][auth2][auth3]; ok {
if len(auth.Packages) == 0 {
log.Debug("Removing author from mapped index")
delete(mapped[repo][auth1][auth2], auth3)
delete(mapped[repo]["*"][auth2], auth3)
delete(mapped[repo][auth1]["**"], auth3)
delete(mapped[repo]["*"]["**"], auth3)
}
if len(mapped[repo][auth1][auth2]) == 0 {
log.Debug("Removing auth1/auth2 from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"][auth2]) == 0 {
log.Debug("Removing author **/auth2 from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo][auth1]["**"]) == 0 {
log.Debug("Removing author auth1/** from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"]["**"]) == 0 {
log.Debug("Removing author */** from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"]) == 0 {
log.Debug("Removing author * from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo][auth1]) == 1 {
log.Debug("Removing author auth1 from mapped index")
delete(mapped[repo], auth1)
}
if len(mapped[repo]) == 1 {
log.Debug("Removing repo from mapped index")
delete(mapped, repo)
}
}
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
// TODO remove from packages/idxpackages
if ctx, ok := packages[parts[0]]; ok {
parts = parts[1:]
for len(parts) > 0 {
if c, ok := ctx.Children[parts[0]]; ok {
ctx = c
} else {
log.Debug("Package not found in packages: %s", parts)
break
}
parts = parts[1:]
}
if len(parts) == 0 {
for ctx != nil {
for pi, p := range ctx.Packages {
if p.Package == pkg {
log.Debug("Removing package from packages: %s", ctx.FullName())
ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
break
}
}
if len(ctx.Packages) == 0 {
log.Debug("Removing PkgSpace from packages: %s", ctx.FullName())
if ctx.Parent == nil {
delete(packages, ctx.Namespace)
} else {
delete(ctx.Parent.Children, ctx.Namespace)
}
}
ctx = ctx.Parent
}
}
}
parts = strings.Split(prov.Name, "::")
if _, ok := idxpackages[repo]; ok {
if ctx, ok := idxpackages[repo][parts[0]]; ok {
parts = parts[1:]
for len(parts) > 0 {
if c, ok := ctx.Children[parts[0]]; ok {
ctx = c
} else {
log.Debug("PkgSpace not found in idxpackages")
break
}
parts = parts[1:]
}
if len(parts) == 0 {
for ctx != nil {
for pi, p := range ctx.Packages {
if p.Package == pkg {
log.Debug("Removing package from idxpackages")
ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
break
}
}
if len(ctx.Packages) == 0 {
log.Debug("Removing PkgSpace from idxpackages: %s", ctx.FullName())
if ctx.Parent == nil {
delete(idxpackages, ctx.Namespace)
} else {
delete(ctx.Parent.Children, ctx.Namespace)
}
}
ctx = ctx.Parent
}
}
}
}
}
if _, ok := filemap[auth1+"/"+auth2+"/"+auth3+"/"+file]; ok {
log.Debug("Removing file from filemap")
// FIXME filemap should be map[string][]string, so we know if
// the file exists in multiple indexes
delete(filemap, auth1+"/"+auth2+"/"+auth3+"/"+file)
}
// write remove to index
gopan.RemoveModule(config.CacheDir+"/"+config.Index, pkg.Author.Source, pkg.Author, pkg)
}
log.Debug("Removing file from gopancache: %s", fname)
// TODO move file deletion to shared gopan package
if err := os.Remove(fname); err != nil {
log.Error("Error removing file: %s", err)
}
// TODO maybe clean up author tree (is this smartpans responsibility?)
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
session.Stash["Page"] = "Browse"
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
func main() {
configure()
indexes = make(map[string]map[string]*gopan.Source)
indexes[config.InputIndex] = gopan.LoadIndex(config.CacheDir + "/" + config.InputIndex)
log.Logger().SetLevel(log.Stol(config.LogLevel))
log.Info("Using log level: %s", config.LogLevel)
// FIXME inefficient
_, _, tpkg, _ := gopan.CountIndex(indexes)
npkg := 0
nmod := 0
var pc = func() float64 {
return float64(nmod) / float64(tpkg) * 100
}
log.Info("Writing packages index file")
out, err := os.Create(config.CacheDir + "/" + config.OutputIndex)
if err != nil {
log.Error("Error creating packages index: %s", err.Error())
return
}
for fname, _ := range indexes {
log.Debug("File: %s", fname)
for _, idx := range indexes[fname] {
log.Debug("Index: %s", idx)
out.Write([]byte(idx.Name + " [" + idx.URL + "]\n"))
for _, auth := range idx.Authors {
log.Debug("Author %s", auth)
out.Write([]byte(" " + auth.Name + " [" + auth.URL + "]\n"))
for _, pkg := range auth.Packages {
out.Write([]byte(" " + pkg.Name + " => " + pkg.URL + "\n"))
log.Debug("Package: %s", pkg)
if !config.Flatten {
if len(pkg.Provides) == 0 {
// TODO better handling of filenames
modnm := strings.TrimSuffix(pkg.Name, ".tar.gz")
tgzpath := config.CacheDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name
if _, err := os.Stat(tgzpath); err != nil {
log.Error("File not found: %s", tgzpath)
return
}
extpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + modnm
dirpath := config.ExtDir + "/" + idx.Name + "/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name
log.Trace("=> tgzpath: %s", tgzpath)
log.Trace(" > extpath: %s", extpath)
log.Trace(" > dirpath: %s", dirpath)
// Only index packages if they don't already exist
if err := pandex.Provides(pkg, tgzpath, extpath, dirpath); err != nil {
log.Error("Error retrieving package list: %s", err)
continue
}
}
npkg += len(pkg.Provides)
nmod += 1
for p, pk := range pkg.Provides {
out.Write([]byte(" " + p + " (" + pk.Version + "): " + pk.File + "\n"))
}
if nmod > 0 && nmod%100 == 0 {
log.Info("%f%% Done %d/%d packages (%d provided so far)", pc(), nmod, tpkg, npkg)
}
}
}
}
}
}
out.Close()
log.Info("Found %d packages from %d modules", npkg, nmod)
}
func main() {
configure()
log.Logger().SetLevel(log.Stol(config.LogLevel))
log.Info("Using log level: %s", config.LogLevel)
indexes = make(map[string]map[string]*gopan.Source)
if !config.NoCache {
indexes[config.Index] = gopan.LoadIndex(config.CacheDir + "/" + config.Index)
}
if config.NoCache || config.Update {
for _, s := range config.Sources {
b := strings.SplitN(s, "=", 2)
if len(b) < 2 {
log.Error("Expected Name=URL pair, got: %s", s)
return
}
if idx, ok := indexes[config.Index][b[0]]; ok {
log.Warn("Index [%s] already exists with URL [%s], updating to [%s]", idx.URL, b[1])
idx.URL = b[1]
} else {
indexes[config.Index][b[0]] = &gopan.Source{
Name: b[0],
URL: b[1],
Authors: make(map[string]*gopan.Author, 0),
}
}
}
if len(config.Sources) == 0 && !config.CPAN && !config.BackPAN {
log.Debug("No -source, -cpan, -backpan parameters, adding default CPAN/BackPAN")
config.CPAN = true
config.BackPAN = true
}
if config.CPAN {
if _, ok := indexes[config.Index]["CPAN"]; !ok {
log.Debug("Adding CPAN index")
indexes[config.Index]["CPAN"] = gopan.CPANSource()
} else {
log.Debug("CPAN index already exists")
}
}
if config.BackPAN {
if _, ok := indexes[config.Index]["BackPAN"]; !ok {
log.Debug("Adding BackPAN index")
indexes[config.Index]["BackPAN"] = gopan.BackPANSource()
} else {
log.Debug("BackPAN index already exists")
}
}
log.Info("Using sources:")
for fname, _ := range indexes {
log.Info("From %s", fname)
for _, source := range indexes[fname] {
log.Info("=> %s", source.String())
}
}
newAuthors := getAuthors()
newPackages := getPackages()
os.MkdirAll(config.CacheDir, 0777)
if !config.NoCache {
gopan.SaveIndex(config.CacheDir+"/"+config.Index, indexes[config.Index])
}
if config.Update {
log.Info("Found %d new packages by %d new authors", newAuthors, newPackages)
}
}
nsrc, nauth, nmod, npkg := gopan.CountIndex(indexes)
log.Info("Found %d packages in %d modules by %d authors from %d sources", npkg, nmod, nauth, nsrc)
if !config.NoMirror {
mirrorPan()
}
}
func (c *Config) Dump() {
log.Debug("GoPAN configuration:")
log.Debug("=> Sources")
for _, s := range c.Sources {
log.Debug(" - %s", s)
}
log.Debug("=> Test")
if c.Test.Global {
log.Debug(" - Global tests are enabled")
} else {
log.Debug(" - Global tests are disabled")
for m, _ := range c.Test.Modules {
log.Debug(" - %s tests are enabled", m)
}
}
log.Debug("=> NoInstall: %t", c.NoInstall)
log.Debug("=> MetaCPAN: %t", c.MetaCPAN)
log.Debug("=> NoDepdump: %t", c.NoDepdump)
log.Debug("=> CPANFile: %s", c.CPANFile)
log.Debug("=> LogLevel: %s", c.LogLevel)
log.Debug("=> Parallelism: %d", c.CPUs)
log.Debug("=> CacheDir: %s", c.CacheDir)
log.Debug("=> InstallDir: %s", c.InstallDir)
}
func (d *DependencyList) Install() (int, error) {
if d == nil {
log.Debug("No dependencies to install")
return 0, nil
}
n := 0
if install_semaphore == nil {
install_semaphore = make(chan int, config.CPUs)
}
var wg sync.WaitGroup
var errorLock sync.Mutex
errs := make([]string, 0)
for _, dep := range d.Dependencies {
log.Debug("Installing dependency: %s", dep)
wg.Add(1)
go func(dep *Dependency) {
defer wg.Done()
defer func(mod *Module) {
if mod != nil {
log.Debug("Resuming installation of %s", mod)
}
}(d.Parent)
_, ok1 := global_installed[dep.Module.Cached]
_, ok2 := global_installed[dep.Module.Name+"-"+dep.Module.Version]
if ok1 || ok2 {
log.Trace("Module is already installed: %s", dep.Module)
return
}
log.Trace("Aquiring install lock for module %s", dep.Module)
install_lock.Lock()
if mt, ok := install_mutex[dep.Module.Cached]; ok {
install_lock.Unlock()
log.Trace("Waiting on existing installation for %s", dep.Module)
log.Trace("Path: %s", dep.Module.Path())
mt.Lock()
mt.Unlock()
log.Trace("Existing installation complete for %s", dep.Module)
return
}
log.Trace("Creating new installation lock for module %s", dep.Module)
install_mutex[dep.Module.Cached] = new(sync.Mutex)
install_mutex[dep.Module.Cached].Lock()
//log.Trace("%s:: Sending semaphore", dep.module)
install_semaphore <- 1
install_lock.Unlock()
o, err := dep.Module.Install()
//log.Trace("%s:: Waiting on semaphore", dep.module)
<-install_semaphore
//log.Trace("%s:: Got semaphore", dep.module)
global_installed[dep.Module.Name+"-"+dep.Module.Version] = dep.Module
global_installed[dep.Module.Cached] = dep.Module
global_unique[dep.Module.Name] = 1
n += o
if err != nil {
log.Error("Error installing module: %s", err)
errorLock.Lock()
errs = append(errs, dep.Module.String())
errorLock.Unlock()
}
install_lock.Lock()
install_mutex[dep.Module.Cached].Unlock()
install_lock.Unlock()
n++
}(dep)
}
wg.Wait()
if len(errs) > 0 {
log.Error("Failed to install dependencies:")
for _, err := range errs {
log.Error("=> %s", err)
}
return n, errors.New("Failed to install dependencies")
}
return n, nil
}
func getPackages() int {
newpkg := 0
var pl func(*html.Node, *gopan.Source, *gopan.Author)
pl = func(n *html.Node, source *gopan.Source, author *gopan.Author) {
log.Trace("NODE: %s [%s, %s, %s]", n.DataAtom, n.Type, n.Data)
if n.Type == html.ElementNode && n.Data == "a" {
//log.Info("NODE IS ELEMENTNODE")
for _, attr := range n.Attr {
// FIXME stuff that isn't .tar.gz?
if attr.Key == "href" && strings.HasSuffix(attr.Val, ".tar.gz") {
log.Trace("==> HREF: %s", n.FirstChild.Data)
pkg := strings.TrimSuffix(n.FirstChild.Data, "/")
if _, ok := author.Packages[pkg]; !ok {
author.Packages[pkg] = &gopan.Package{
Name: pkg,
Author: author,
URL: author.URL + "/" + pkg,
}
newpkg++
log.Debug("Found package: %s", pkg)
}
}
}
//log.Info("%s", n.Data)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
pl(c, source, author)
}
}
log.Info("Building package list")
for fname, _ := range indexes {
for _, source := range indexes[fname] {
log.Debug("Index: %s", source)
wg.Add(1)
go func(source *gopan.Source) {
defer wg.Done()
for _, author := range source.Authors {
wg.Add(1)
go func(author *gopan.Author) {
defer wg.Done()
sem <- 1
log.Trace("=> %s", author)
url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
log.Trace("Getting URL: %s", url)
res, err := http.Get(url)
if err != nil {
log.Error("HTTP GET - %s", err.Error())
<-sem
return
}
doc, err := html.Parse(res.Body)
if err != nil {
log.Error("HTML PARSE - %s", err.Error())
<-sem
return
}
pl(doc, source, author)
<-sem
}(author)
}
}(source)
}
}
wg.Wait()
log.Info("Finished building package list")
return newpkg
}
func mirrorPan() {
log.Info("Mirroring *PAN")
// FIXME inefficient
_, _, npkg, _ := gopan.CountIndex(indexes)
mirrored := 0
var pc = func() int {
return mirrored / npkg * 100
}
for fname, _ := range indexes {
log.Debug("File: %s", fname)
for _, source := range indexes[fname] {
log.Debug("Index: %s", source)
wg.Add(1)
go func(source *gopan.Source) {
defer wg.Done()
for _, author := range source.Authors {
log.Debug("=> %s", author)
wg.Add(1)
go func(author *gopan.Author) {
cachedir := config.CacheDir + "/" + source.Name + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/"
os.MkdirAll(cachedir, 0777)
defer wg.Done()
for _, pkg := range author.Packages {
wg.Add(1)
go func(pkg *gopan.Package) {
defer wg.Done()
cache := cachedir + pkg.Name
log.Trace(" - Caching to: %s", cache)
if _, err := os.Stat(cache); err == nil {
log.Debug("%d%% |> %s", pc(), pkg)
log.Trace(" - Already exists in cache")
mirrored++
return
}
sem <- 1
mirrored++
log.Debug("%d%% => %s", pc(), pkg)
url := source.URL + "/" + author.Name[:1] + "/" + author.Name[:2] + "/" + author.Name + "/" + pkg.Name
log.Trace(" - From URL: %s", url)
out, err := os.Create(cache)
defer out.Close()
if err != nil {
log.Error("CREATE - %s", err.Error())
<-sem
return
}
resp, err := http.Get(url)
if err != nil {
log.Error("HTTP GET - %s", err.Error())
<-sem
return
}
_, err = io.Copy(out, resp.Body)
if err != nil {
log.Error("IO COPY - %s", err.Error())
}
<-sem
}(pkg)
}
}(author)
}
}(source)
}
}
wg.Wait()
log.Info("Finished mirroring *PAN")
}
func (m *Module) Install() (int, error) {
log.Debug("Installing module: %s", m)
n := 0
if m.Deps != nil {
log.Trace("Installing module dependencies for %s", m)
<-install_semaphore
o, err := m.Deps.Install()
install_semaphore <- 1
n += o
if err != nil {
log.Error("Error installing module dependencies for %s: %s", m, err)
return n, err
}
}
var c *exec.Cmd
var stdout *bytes.Buffer
var stderr *bytes.Buffer
cpanm_cache_dir, err := filepath.Abs(config.CacheDir)
if err != nil {
log.Error("Failed to get absolute path of gopan cache directory: %s", err)
return n, err
}
os.Setenv("PERL_CPANM_HOME", cpanm_cache_dir)
done := false
attempts := 0
for !done {
time.Sleep(time.Duration(100) * time.Millisecond)
c = m.getCmd()
stdout = new(bytes.Buffer)
stderr = new(bytes.Buffer)
c.Stderr = stderr
c.Stdout = stdout
// brute force cpanm text file busy errors
attempts++
if err := c.Start(); err != nil {
if attempts > 10 {
log.Error("Error installing module %s: %s", m, err)
return n, err
}
} else {
done = true
}
}
if err := c.Wait(); err != nil {
if !strings.HasPrefix(strings.ToLower(stderr.String()), "plenv: cannot rehash:") && !strings.Contains(strings.ToLower(stderr.String()), "text file busy") &&
!strings.HasPrefix(strings.ToLower(stdout.String()), "plenv: cannot rehash:") && !strings.Contains(strings.ToLower(stdout.String()), "text file busy") {
log.Error(m.Name + "-" + m.Version + " failed to install")
log.Error("Error installing %s %s: %s\nSTDERR:\n%sSTDOUT:\n%s", m.Name, m.Version, err, stderr.String(), stdout.String())
return n, err
}
}
n++
log.Printf("Installed " + m.Name + " (" + m.Version + ")")
return n, nil
}
func (m *Module) loadDependencies() error {
yml, err := ioutil.ReadFile(m.Extracted + "/META.yml")
if err != nil {
// TODO this isnt an error (it shouldnt make build fail)
log.Error("Error opening META.yml for %s: %s", m.Name, err)
// return nil to prevent build fail
return nil
}
meta := make(map[interface{}]interface{})
err = yaml.Unmarshal(yml, &meta)
if err != nil {
// TODO this isnt a real error, probably
log.Error("Error parsing YAML: %s", err)
// return nil to prevent build fail
return nil
}
if reqs, ok := meta["requires"]; ok {
log.Debug("Found dependencies for module %s", m.Name)
switch reqs.(type) {
case map[interface{}]interface{}:
for req, ver := range reqs.(map[interface{}]interface{}) {
v := float64(0)
switch ver.(type) {
case string:
v = gopan.VersionFromString(ver.(string))
case int:
v = float64(ver.(int))
}
log.Printf("=> %s (%f)", req, v)
dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
if err != nil {
log.Error("Error parsing dependency: %s", err)
continue
}
if _, ok := perl_core[dep.Name]; ok {
log.Trace("Module is from perl core: %s", dep.Name)
continue
}
m.Deps.AddDependency(dep)
}
}
log.Debug("Resolving module dependency list")
if err := m.Deps.Resolve(); err != nil {
log.Error("Error resolving dependency list [%s]: %s", m.Name, err)
return err
}
return nil
}
// FIXME repeat of block above, just with more nested levels
if p, ok := meta["prereqs"]; ok {
if r, ok := p.(map[interface{}]interface{})["runtime"]; ok {
if reqs, ok := r.(map[interface{}]interface{})["requires"]; ok {
log.Debug("Found dependencies for module %s", m.Name)
switch reqs.(type) {
case map[interface{}]interface{}:
for req, ver := range reqs.(map[interface{}]interface{}) {
v := float64(0)
switch ver.(type) {
case string:
v = gopan.VersionFromString(ver.(string))
case int:
v = float64(ver.(int))
}
log.Printf("=> %s (%f)", req, v)
dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
if err != nil {
log.Error("Error parsing dependency: %s", err)
continue
}
if _, ok := perl_core[dep.Name]; ok {
log.Trace("Module is from perl core: %s", dep.Name)
continue
}
m.Deps.AddDependency(dep)
}
}
}
}
if t, ok := p.(map[interface{}]interface{})["test"]; ok {
if reqs, ok := t.(map[interface{}]interface{})["requires"]; ok {
log.Debug("Found dependencies for module %s", m.Name)
switch reqs.(type) {
case map[interface{}]interface{}:
for req, ver := range reqs.(map[interface{}]interface{}) {
v := float64(0)
switch ver.(type) {
case string:
v = gopan.VersionFromString(ver.(string))
case int:
v = float64(ver.(int))
}
log.Printf("=> %s (%f)", req, v)
dep, err := DependencyFromString(req.(string), fmt.Sprintf("%f", ver))
if err != nil {
log.Error("Error parsing dependency: %s", err)
continue
}
if _, ok := perl_core[dep.Name]; ok {
log.Trace("Module is from perl core: %s", dep.Name)
continue
}
m.Deps.AddDependency(dep)
}
}
}
}
log.Debug("Resolving module dependency list")
if err := m.Deps.Resolve(); err != nil {
log.Error("Error resolving dependency list: %s", err)
return err
}
return nil
}
log.Debug("No dependencies for module %s", m.Name)
return nil
}
// Resolve a dependency (i.e. one module), trying all sources
func (d *Dependency) Resolve(p *Module) error {
if gm, ok := global_modules[d.Name+"-"+d.Version]; ok {
log.Trace("Dependency %s already resolved (S1): %s", d, gm)
d.Module = gm
return nil
}
log.Trace("Resolving dependency: %s", d)
for _, s := range config.Sources {
log.Trace("=> Trying source: %s", s)
m, err := s.Find(d)
if err != nil {
log.Trace("=> Error from source: %s", err)
continue
}
if m != nil {
log.Trace("=> Resolved dependency: %s", m)
d.Module = m
break
}
}
if d.Module == nil {
log.Error("Failed to resolve dependency: %s", d)
return fmt.Errorf("Dependency not found from any source: %s", d)
}
if gm, ok := global_modules[d.Module.Name+"-"+d.Module.Version+"~"+d.Module.Source.URL]; ok {
log.Trace("Dependency %s already resolved (S2): %s", d, gm)
d.Module = gm
} else if gm, ok := global_modules[d.Module.Name]; ok {
log.Trace("Dependency %s already resolved (S3): %s", d, gm)
// See if the already resolved version is acceptable
if !d.MatchesVersion(gm.Version) {
errstr := fmt.Sprintf("Version conflict in dependency tree: %s => %s", d, gm)
log.Error(errstr)
return errors.New(errstr)
}
log.Trace("Version %s matches %s", d.Module, gm.Version)
// TODO See if downloading a new version would be better
d.Module = gm
} else {
log.Debug("Downloading: %s", d.Module)
if err := d.Module.Download(); err != nil {
log.Error("Error downloading module %s: %s", d.Module, err)
return err
}
if p != nil {
if p.IsCircular(d.Module) {
log.Error("Detected circular dependency %s from module %s", d.Module, p)
return fmt.Errorf("Detected circular dependency %s from module %s", d.Module, p)
}
}
// module can't exist because of global_lock
global_modules[d.Module.Name] = d.Module
global_modules[d.Module.Name+"-"+d.Module.Version] = d.Module
global_modules[d.Module.Name+"-"+d.Module.Version+"~"+d.Module.Source.URL] = d.Module
log.Debug("Resolving module dependencies: %s", d.Module)
d.Module.Deps = &DependencyList{
Parent: d.Module,
Dependencies: make([]*Dependency, 0),
}
if d.Additional != nil && len(d.Additional) > 0 {
log.Trace("Adding cpanfile additional REQS")
for _, additional := range d.Additional {
log.Trace("Adding additional dependency from cpanfile: %s", additional)
d.Module.Deps.AddDependency(additional)
}
}
if err := d.Module.loadDependencies(); err != nil {
return err
}
}
return nil
}