func render_import(session *http.Session) {
html, _ := session.RenderTemplate("import.html")
session.Stash["Page"] = "Import"
session.Stash["ImportAvailable"] = config.ImportAvailable
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
func settings(session *http.Session) {
session.Stash["Title"] = "SmartPAN Settings"
html, _ := session.RenderTemplate("settings.html")
session.Stash["Page"] = "Settings"
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
func help(session *http.Session) {
session.Stash["Title"] = "SmartPAN Help"
html, _ := session.RenderTemplate("help.html")
session.Stash["Page"] = "Help"
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
func importstream(session *http.Session) {
job := imports[session.Stash["jobid"].(string)]
session.Stash["Job"] = job
c := session.Response.Chunked()
session.Response.Headers.Add("Content-Type", "text/plain")
raw := false
if _, ok := session.Request.Form()["raw"]; ok {
raw = true
}
if !raw {
header, _ := session.RenderTemplate("layout_streamstart.html")
c <- []byte(header)
}
for _, s := range job.History {
if !raw {
c <- []byte(s + "<br />\n")
} else {
c <- []byte(s + "\n")
}
}
var wg sync.WaitGroup
if !job.Complete {
wg.Add(1)
job.Watchers = append(job.Watchers, func(m string) {
if m == ":DONE" {
wg.Done()
return
}
if !raw {
c <- []byte(m + "<br />\n")
} else {
c <- []byte(m + "\n")
}
})
}
wg.Wait()
if !raw {
footer, _ := session.RenderTemplate("layout_streamend.html")
c <- []byte(footer)
}
c <- make([]byte, 0)
}
func pkgindex(session *http.Session) {
if _, ok := session.Stash["repo"]; !ok {
session.RenderNotFound()
return
}
repo := session.Stash["repo"].(string)
for fname, _ := range indexes {
if _, ok := indexes[fname][repo]; !ok && repo != "SmartPAN" {
session.RenderNotFound()
return
}
}
if g, ok := session.Stash["gz"]; ok {
if len(g.(string)) > 0 {
// cheat and hijack gotchas gzip support
session.Response.Headers.Set("Content-Type", "application/gzip")
session.Response.Send()
session.Response.Gzip()
session.Response.Headers.Remove("Content-Encoding")
log.Debug("Using gzip")
}
}
session.Response.WriteText("File: 02packages.details.txt\n")
session.Response.WriteText("Description: Package names found in directory " + repo + "/authors/id\n")
session.Response.WriteText("Columns: package name, version, path\n")
session.Response.WriteText("Written-By: SmartPAN (from GoPAN)\n")
session.Response.WriteText("Line-Count: " + strconv.Itoa(summary.Packages) + "\n") // FIXME wrong count
session.Response.WriteText("\n")
if repo == "SmartPAN" {
for _, pkg := range packages {
writepkgindex(session, pkg)
}
} else {
for _, pkg := range idxpackages[repo] {
writepkgindex(session, pkg)
}
}
}
func import2(session *http.Session) {
if !config.ImportAvailable {
session.Render("error.html")
session.Stash["GoPANError"] = "Import is currently unavailable"
return
}
session.Stash["Page"] = "Import"
session.Stash["Title"] = "Import job " + session.Stash["jobid"].(string)
job := imports[session.Stash["jobid"].(string)]
session.Stash["Job"] = job
html, _ := session.RenderTemplate("import2.html")
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
func download(session *http.Session) {
if _, ok := session.Stash["repo"]; !ok {
session.RenderNotFound()
return
}
if _, ok := session.Stash["file"]; !ok {
session.RenderNotFound()
return
}
repo := session.Stash["repo"].(string)
file := session.Stash["file"].(string)
if repo == "SmartPAN" {
if _, ok := filemap[file]; !ok {
log.Debug("SmartPAN repo - file [%s] not found in any index", file)
session.RenderNotFound()
return
}
repo = filemap[file]
log.Debug("SmartPAN repo - file [%s] found in [%s]", file, repo)
}
log.Debug("Repo [%s], file [%s]", repo, file)
nfile := config.CacheDir + "/" + repo + "/" + file
if _, err := os.Stat(nfile); err != nil {
log.Debug("File not found on disk, considering readthrough")
for fn, _ := range indexes {
log.Debug("Trying file: %s", fn)
if src, ok := indexes[fn][repo]; ok {
log.Debug("Found matching repo")
if strings.HasPrefix(src.URL, "http:") {
log.Debug("Found HTTP URL, trying: %s", src.URL+"/"+file)
res, err := nethttp.Get(src.URL + "/" + file)
if err != nil {
log.Debug("Error on readthrough: %s", err.Error())
continue
}
defer res.Body.Close()
b, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Debug("Error reading body: %s", err.Error())
continue
}
session.Response.Write(b)
return
}
}
}
log.Debug("No readthrough available")
session.RenderNotFound()
return
}
f, err := os.Open(nfile)
if err != nil {
log.Error(err.Error())
session.RenderNotFound()
return
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
log.Error(err.Error())
session.RenderNotFound()
return
}
session.Response.Write(b)
}
func import1(session *http.Session) {
session.Stash["indexes"] = indexes
session.Stash["Title"] = "SmartPAN Import"
m := &ImportForm{}
f := form.New(session, m)
session.Stash["fh"] = f
log.Info("Headers: %s", session.Request.Header())
if session.Request.Method != "POST" {
render_import(session)
return
}
f.Populate(true)
f.Validate()
if f.HasErrors {
render_import(session)
return
}
log.Info("Importing into: %s", m.ImportInto)
if m.ImportInto == "new_index" {
if len(m.NewIndex) == 0 {
f.HasErrors = true
f.Errors["NewIndex"] = make(map[string]error)
f.Errors["NewIndex"]["required"] = errors.New("Please give the new repository a name")
render_import(session)
return
}
log.Info("=> Creating new index: %s", m.NewIndex)
}
b := make([]byte, 20)
rand.Read(b)
en := base64.URLEncoding
d := make([]byte, en.EncodedLen(len(b)))
en.Encode(d, b)
job := &ImportJob{
Form: m,
Complete: false,
Id: string(d),
Watchers: make([]func(string), 0),
}
if len(m.Cpanfile) > 0 {
log.Info("Got cpanfile:")
log.Info(m.Cpanfile)
}
log.Info("=> Created import job: %s", job.Id)
imports[job.Id] = job
go do_import(session, job)
//render_import(session)
if _, ok := session.Request.Form()["stream"]; ok {
session.Redirect(&url.URL{Path: "/import/" + job.Id + "/stream", RawQuery: "raw=y"})
} else {
session.Redirect(&url.URL{Path: "/import/" + job.Id})
}
}
func getindex(session *http.Session) {
idx := session.Stash["index"]
switch idx {
case "CPAN":
go func() {
config.CPANStatus = "Downloading"
res, err := nethttp.Get("https://s3-eu-west-1.amazonaws.com/gopan/cpan_index.gz")
if err != nil {
log.Error("Error downloading index: %s", err.Error())
session.RenderException(500, errors.New("Error downloading CPAN index: "+err.Error()))
config.CPANStatus = "Failed"
return
}
defer res.Body.Close()
b, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Error("Error reading index: %s", err.Error())
session.RenderException(500, errors.New("Error reading CPAN index: "+err.Error()))
config.CPANStatus = "Failed"
return
}
fi, err := os.Create(config.CacheDir + "/" + config.CPANIndex)
if err != nil {
log.Error("Error creating output file: %s", err.Error())
session.RenderException(500, errors.New("Error creating output file: "+err.Error()))
config.CPANStatus = "Failed"
return
}
defer fi.Close()
fi.Write(b)
config.CPANStatus = "Downloaded"
config.HasCPANIndex = true
config.CPANIndexDate = time.Now().String()
config.CPANStatus = "Loading"
load_index(config.CPANIndex, config.CacheDir+"/"+config.CPANIndex)
config.CPANStatus = "Indexing"
update_indexes()
config.CPANStatus = "Loaded"
}()
session.Redirect(&url.URL{Path: "/settings"})
return
case "BackPAN":
go func() {
config.BackPANStatus = "Downloading"
res, err := nethttp.Get("https://s3-eu-west-1.amazonaws.com/gopan/backpan_index.gz")
if err != nil {
log.Error("Error downloading index: %s", err.Error())
session.RenderException(500, errors.New("Error downloading BackPAN index: "+err.Error()))
config.BackPANStatus = "Failed"
return
}
defer res.Body.Close()
b, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Error("Error reading index: %s", err.Error())
session.RenderException(500, errors.New("Error reading BackPAN index: "+err.Error()))
config.BackPANStatus = "Failed"
return
}
fi, err := os.Create(config.CacheDir + "/" + config.BackPANIndex)
if err != nil {
log.Error("Error creating output file: %s", err.Error())
session.RenderException(500, errors.New("Error creating output file: "+err.Error()))
config.BackPANStatus = "Failed"
return
}
defer fi.Close()
fi.Write(b)
config.BackPANStatus = "Downloaded"
config.HasBackPANIndex = true
config.BackPANIndexDate = time.Now().String()
config.BackPANStatus = "Loading"
load_index(config.BackPANIndex, config.CacheDir+"/"+config.BackPANIndex)
config.BackPANStatus = "Indexing"
update_indexes()
config.BackPANStatus = "Loaded"
}()
session.Redirect(&url.URL{Path: "/settings"})
return
}
session.RenderNotFound()
}
func delete_file(session *http.Session) {
session.Stash["Title"] = "Delete file"
html, _ := session.RenderTemplate("delete.html")
repo := session.Stash["repo"].(string)
file := session.Stash["file"].(string)
auth1 := session.Stash["auth1"].(string)
auth2 := session.Stash["auth2"].(string)
auth3 := session.Stash["auth3"].(string)
fname := config.CacheDir + "/" + repo + "/" + auth1 + "/" + auth2 + "/" + auth3 + "/" + file
if _, err := os.Stat(fname); err != nil {
session.RenderNotFound()
return
}
// Remove file from indexes
for f, _ := range indexes {
if _, ok := indexes[f][repo]; !ok {
continue
}
if _, ok := indexes[f][repo].Authors[auth3]; !ok {
continue
}
if _, ok := indexes[f][repo].Authors[auth3].Packages[file]; !ok {
continue
}
log.Debug("Removing from index: %s", repo)
pkg := indexes[f][repo].Authors[auth3].Packages[file]
delete(indexes[f][repo].Authors[auth3].Packages, file)
if len(indexes[f][repo].Authors[auth3].Packages) == 0 {
log.Debug("Removing author")
delete(indexes[f][repo].Authors, auth3)
}
if len(indexes[f][repo].Authors) == 0 {
log.Debug("Removing index")
delete(indexes[f], repo)
}
if auth, ok := mapped[repo][auth1][auth2][auth3]; ok {
if len(auth.Packages) == 0 {
log.Debug("Removing author from mapped index")
delete(mapped[repo][auth1][auth2], auth3)
delete(mapped[repo]["*"][auth2], auth3)
delete(mapped[repo][auth1]["**"], auth3)
delete(mapped[repo]["*"]["**"], auth3)
}
if len(mapped[repo][auth1][auth2]) == 0 {
log.Debug("Removing auth1/auth2 from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"][auth2]) == 0 {
log.Debug("Removing author **/auth2 from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo][auth1]["**"]) == 0 {
log.Debug("Removing author auth1/** from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"]["**"]) == 0 {
log.Debug("Removing author */** from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo]["*"]) == 0 {
log.Debug("Removing author * from mapped index")
delete(mapped[repo][auth1], auth2)
}
if len(mapped[repo][auth1]) == 1 {
log.Debug("Removing author auth1 from mapped index")
delete(mapped[repo], auth1)
}
if len(mapped[repo]) == 1 {
log.Debug("Removing repo from mapped index")
delete(mapped, repo)
}
}
for _, prov := range pkg.Provides {
parts := strings.Split(prov.Name, "::")
// TODO remove from packages/idxpackages
if ctx, ok := packages[parts[0]]; ok {
parts = parts[1:]
for len(parts) > 0 {
if c, ok := ctx.Children[parts[0]]; ok {
ctx = c
} else {
log.Debug("Package not found in packages: %s", parts)
break
}
parts = parts[1:]
}
if len(parts) == 0 {
for ctx != nil {
for pi, p := range ctx.Packages {
if p.Package == pkg {
log.Debug("Removing package from packages: %s", ctx.FullName())
ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
break
}
}
if len(ctx.Packages) == 0 {
log.Debug("Removing PkgSpace from packages: %s", ctx.FullName())
if ctx.Parent == nil {
delete(packages, ctx.Namespace)
} else {
delete(ctx.Parent.Children, ctx.Namespace)
}
}
ctx = ctx.Parent
}
}
}
parts = strings.Split(prov.Name, "::")
if _, ok := idxpackages[repo]; ok {
if ctx, ok := idxpackages[repo][parts[0]]; ok {
parts = parts[1:]
for len(parts) > 0 {
if c, ok := ctx.Children[parts[0]]; ok {
ctx = c
} else {
log.Debug("PkgSpace not found in idxpackages")
break
}
parts = parts[1:]
}
if len(parts) == 0 {
for ctx != nil {
for pi, p := range ctx.Packages {
if p.Package == pkg {
log.Debug("Removing package from idxpackages")
ctx.Packages = append(ctx.Packages[:pi], ctx.Packages[pi+1:]...)
break
}
}
if len(ctx.Packages) == 0 {
log.Debug("Removing PkgSpace from idxpackages: %s", ctx.FullName())
if ctx.Parent == nil {
delete(idxpackages, ctx.Namespace)
} else {
delete(ctx.Parent.Children, ctx.Namespace)
}
}
ctx = ctx.Parent
}
}
}
}
}
if _, ok := filemap[auth1+"/"+auth2+"/"+auth3+"/"+file]; ok {
log.Debug("Removing file from filemap")
// FIXME filemap should be map[string][]string, so we know if
// the file exists in multiple indexes
delete(filemap, auth1+"/"+auth2+"/"+auth3+"/"+file)
}
// write remove to index
gopan.RemoveModule(config.CacheDir+"/"+config.Index, pkg.Author.Source, pkg.Author, pkg)
}
log.Debug("Removing file from gopancache: %s", fname)
// TODO move file deletion to shared gopan package
if err := os.Remove(fname); err != nil {
log.Error("Error removing file: %s", err)
}
// TODO maybe clean up author tree (is this smartpans responsibility?)
nsrc, nauth, npkg, nprov := gopan.CountIndex(indexes)
// TODO should probably be in the index - needs to udpate when index changes
summary = &Summary{nsrc, nauth, npkg, nprov}
session.Stash["Page"] = "Browse"
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
func search(session *http.Session) {
if q, ok := session.Request.Form()["q"]; ok {
query := strings.ToLower(q[0])
session.Stash["Query"] = q[0]
results := make([]*SearchResult, 0)
var lock sync.Mutex
tStart := time.Now().UnixNano()
log.Trace("Searching for [%s]", query)
var wg sync.WaitGroup
for fname, _ := range indexes {
log.Trace("=> Searching file: %s", fname)
for _, idx := range indexes[fname] {
log.Trace("=> Searching index: %s", idx.Name)
wg.Add(1)
go func(idx *gopan.Source) {
defer wg.Done()
if strings.Contains(strings.ToLower(idx.Name), query) {
lock.Lock()
results = append(results, &SearchResult{
Name: idx.Name,
Type: "Index",
URL: idx.Name,
Obj: idx,
Glyph: "list",
})
lock.Unlock()
}
for _, auth := range idx.Authors {
wg.Add(1)
go func(idx *gopan.Source, auth *gopan.Author) {
defer wg.Done()
if strings.Contains(strings.ToLower(auth.Name), query) {
lock.Lock()
results = append(results, &SearchResult{
Name: auth.Name,
Type: "Author",
URL: idx.Name + "/authors/id/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name,
Obj: auth,
Glyph: "user",
})
lock.Unlock()
}
for _, pkg := range auth.Packages {
if strings.Contains(strings.ToLower(pkg.Name), query) {
lock.Lock()
results = append(results, &SearchResult{
Name: pkg.Name,
Type: "Module",
URL: idx.Name + "/authors/id/" + auth.Name[:1] + "/" + auth.Name[:2] + "/" + auth.Name + "/" + pkg.Name,
Obj: pkg,
Glyph: "compressed",
})
lock.Unlock()
}
for _, prov := range pkg.Provides {
if strings.Contains(strings.ToLower(prov.Name), query) {
lock.Lock()
results = append(results, &SearchResult{
Name: prov.Name,
Type: "Package",
URL: idx.Name + "/modules/" + strings.Replace(prov.Name, "::", "/", -1),
Obj: prov,
Glyph: "briefcase",
})
lock.Unlock()
}
}
}
}(idx, auth)
}
}(idx)
}
}
wg.Wait()
t := float64(time.Now().UnixNano()-tStart) / 100000 // ms
sort.Sort(ByName(results))
session.Stash["Results"] = results
session.Stash["Duration"] = t
}
session.Stash["Title"] = "SmartPAN"
html, _ := session.RenderTemplate("search.html")
session.Stash["Page"] = "Search"
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
func browse(session *http.Session) {
session.Stash["Title"] = "SmartPAN"
path := ""
repo := ""
itype := ""
fpath := ""
if r, ok := session.Stash["repo"]; ok {
repo = r.(string)
fpath += repo + "/"
found := false
if repo == "SmartPAN" {
found = true
} else {
for fname, _ := range indexes {
if _, ok := indexes[fname][repo]; ok {
found = true
break
}
}
}
if !found {
session.RenderNotFound()
return
}
}
if i, ok := session.Stash["type"]; ok {
itype = i.(string)
fpath += itype + "/"
}
if p, ok := session.Stash["path"]; ok {
path = p.(string)
fpath += path + "/"
}
fpath = strings.TrimSuffix(fpath, "/")
session.Stash["path"] = fpath
bits := strings.Split(path, "/")
fbits := strings.Split(fpath, "/")
dirs := make(map[string]map[string]string, 0)
files := make(map[string]map[string]string, 0)
log.Info("Path: %s, Bits: %d", path, len(bits))
if repo == "" {
dirs = toplevelRepo()
} else if itype == "" {
dirs, files = tlRepo1(repo)
} else {
switch itype {
case "authors":
if len(path) == 0 {
dirs, files = tlRepo2(repo, itype)
} else {
switch len(bits) {
case 1:
log.Info("tlAuthor1")
dirs = tlAuthor1(repo)
case 2:
log.Info("tlAuthor2: %s", bits[1])
dirs = tlAuthor2(repo, bits[1])
case 3:
log.Info("tlAuthor3: %s, %s", bits[1], bits[2])
dirs = tlAuthor3(repo, bits[1], bits[2])
case 4:
log.Info("tlModuleList: %s, %s", repo, bits[1], bits[2], bits[3])
files = tlModuleList(repo, bits[1], bits[2], bits[3])
default:
log.Info("Invalid path - rendering not found")
session.RenderNotFound()
}
}
case "modules":
if path == "" {
dirs, files = tlRepo2(repo, itype)
} else {
if repo == "SmartPAN" {
rbits := bits[1:]
ctx := packages[bits[0]]
log.Info("Starting with context: %s", ctx.Namespace)
for len(rbits) > 0 {
ctx = ctx.Children[rbits[0]]
rbits = rbits[1:]
}
log.Info("Stashing package context: %s", ctx.Namespace)
session.Stash["Package"] = ctx
for ns, _ := range ctx.Children {
files[ns] = map[string]string{
"Name": ns,
"Path": "/" + repo + "/modules/" + path + "/" + ns,
}
}
} else {
rbits := bits[1:]
ctx := idxpackages[repo][bits[0]]
log.Info("Starting with context: %s", ctx.Namespace)
for len(rbits) > 0 {
ctx = ctx.Children[rbits[0]]
rbits = rbits[1:]
}
session.Stash["Package"] = ctx
log.Info("Stashing package context: %s", ctx.Namespace)
for ns, _ := range ctx.Children {
files[ns] = map[string]string{
"Name": ns,
"Path": "/" + repo + "/modules/" + path + "/" + ns,
}
}
}
}
default:
session.RenderNotFound()
}
}
session.Stash["Dirs"] = dirs
session.Stash["Files"] = files
pp := make([]map[string]string, 0)
cp := ""
for _, b := range fbits {
cp = cp + "/" + b
pp = append(pp, map[string]string{
"Name": b,
"Path": cp,
})
}
if len(pp) > 0 && len(pp[0]["Name"]) > 0 {
session.Stash["PathBits"] = pp
}
html, _ := session.RenderTemplate("browse.html")
session.Stash["Page"] = "Browse"
session.Stash["Content"] = template.HTML(html)
session.Render("layout.html")
}
