func (r *Repo) Load(callGithub githubCallFunc) error {
log.Info("loading repo: %s", r.Path)
url := "https://github.com/" + r.Path + "/tarball/master"
resp, err := httpClient.Get(url)
if err != nil {
log.StandardError(err)
return err
}
defer resp.Body.Close()
zf, err := gzip.NewReader(resp.Body)
if err != nil {
log.Error("couldn't find a valid repo tarball at %s -- %s", url, err)
return err
}
tr := tar.NewReader(zf)
r.Avatars = map[string]string{}
r.Orderings = map[string]*Ordering{}
r.Planfiles = map[string]*Planfile{}
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
log.Error("reading tarball: %s", err)
return err
}
filename, ext := rsplit(hdr.Name, ".")
_, filename = rsplit(filename, "/")
if ext == "md" || ext == "order" {
log.Info("parsing: %s", filename)
data, err := ioutil.ReadAll(tr)
if err != nil {
log.Error("reading tarball file %q: %s", hdr.Name, err)
continue
}
if ext == "md" {
r.AddPlanfile(filename+".md", data, callGithub)
} else {
r.AddOrdering(filename+".order", data)
}
}
}
log.Info("successfully loaded repo: %s", r.Path)
return nil
}
func runHTTP(name string, host string, port int, handler http.Handler, suffix string) {
addr := fmt.Sprintf("%s:%d", host, port)
listener, err := net.Listen("tcp", addr)
if err != nil {
runtime.Error("Cannot listen on %s: %v", addr, err)
}
go serveHTTP(name, listener, handler)
if host == "" {
host = "localhost"
}
log.Info("%s running on http://%s:%d%s", name, host, port, suffix)
}
func renderMarkdown(input []byte) (out []byte, err error) {
defer func() {
if e := recover(); e != nil {
if e, ok := e.(error); ok {
log.Info("hilite error: restarting, due to: %s", e)
setupPygments()
err = e
}
}
}()
r := &Html{blackfriday.HtmlRenderer(DefaultHtmlFlags, "", "")}
out = blackfriday.Markdown(input, r, DefaultExtensions)
return
}
func (db *DB) Run(host string, port int, limit int64, master MasterController) error {
if host == "" {
host = runtime.GetIP()
}
db.id = getNodeID(host, port)
db.limit = limit << 10
db.limitNotify = (db.limit * 2) / 3
addr, listener := runtime.GetAddrListener(host, port)
defer listener.Close()
go master.Run(db.id)
log.Info("MatchDB running on %s", addr)
delay := 1 * time.Millisecond
maxDelay := 1 * time.Second
for {
conn, err := listener.Accept()
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Temporary() {
if delay == 0 {
delay = 5 * time.Millisecond
} else {
delay *= 2
}
if delay > maxDelay {
delay = maxDelay
}
log.Error("Accept error: %v; retrying in %v", err, delay)
time.Sleep(delay)
continue
}
log.Error("Accept error: %v", err)
return err
}
delay = 0
go db.Handle(conn)
}
}
func main() {
opts := optparse.Parser("Usage: genapi [options]", "0.1")
root := opts.String(
[]string{"-r", "--root"}, "../src/espra",
"path to the root package directory for the app", "PATH")
ignoreList := opts.String(
[]string{"-i", "--ignore"}, "api.go html.go",
"space-separated list of files/subdirectories to ignore", "LIST")
digestFile := opts.String(
[]string{"-d", "--digest"}, "../etc/app/version.digest",
"path to write the digest of the API version", "PATH")
os.Args[0] = "genapi"
opts.Parse(os.Args)
log.AddConsoleLogger()
ignore := strings.Split(*ignoreList, " ")
for i, path := range ignore {
ignore[i] = filepath.Join(*root, path)
}
pkgpath := strings.Split(*root, "/")
pkgname := pkgpath[len(pkgpath)-1]
if err := parseDirectory(pkgname, pkgname, *root, ignore); err != nil {
runtime.StandardError(err)
}
for _, method := range methods {
log.Info("%#v", method)
}
log.Wait()
_ = *digestFile
}
func parseFile(path string, force bool) {
dir, filename := filepath.Split(path)
if !strings.HasSuffix(filename, ".go") {
runtime.Error("%s does not look like a go file", filename)
}
log.Info("Parsing %s", path)
fset := token.NewFileSet()
pkg, err := parser.ParseFile(fset, path, nil, 0)
if err != nil {
runtime.StandardError(err)
}
newpath := filepath.Join(dir, fmt.Sprintf("%s_marshal.go", filename[:len(filename)-3]))
exists, err := fsutil.FileExists(newpath)
if exists && !force {
runtime.Error("%s already exists! please specify --force to overwrite", newpath)
}
prev := ""
models := []*model{}
ast.Inspect(pkg, func(n ast.Node) bool {
if s, ok := n.(*ast.StructType); ok {
fields := []fieldInfo{}
for _, field := range s.Fields.List {
if field.Names == nil {
continue
}
name := field.Names[0].Name
dbName := ""
kind := ""
if field.Tag != nil {
tag := field.Tag.Value[1 : len(field.Tag.Value)-1]
if tag == "-" {
continue
}
dbName = tag
}
if dbName == "" {
dbName = name
rune, _ := utf8.DecodeRuneInString(name)
if !unicode.IsUpper(rune) {
continue
}
}
switch expr := field.Type.(type) {
case *ast.Ident:
switch expr.Name {
case "bool", "string", "int", "int64", "uint", "uint64":
kind = expr.Name
}
case *ast.ArrayType:
if expr.Len == nil { // slice type
switch iexpr := expr.Elt.(type) {
case *ast.Ident:
switch iexpr.Name {
case "byte", "bool", "string", "int", "int64", "uint", "uint64":
kind = "[]" + iexpr.Name
}
case *ast.ArrayType:
if iexpr.Len == nil {
if iiexpr, ok := iexpr.Elt.(*ast.Ident); ok {
if iiexpr.Name == "byte" {
kind = "[][]byte"
}
}
}
}
}
case *ast.SelectorExpr:
if lexpr, ok := expr.X.(*ast.Ident); ok {
if lexpr.Name == "time" && expr.Sel.Name == "Time" {
kind = "time"
}
}
}
if kind == "" {
log.Error("unsupported: %v field (%s.%s)", field.Type, prev, name)
continue
}
fields = append(fields, fieldInfo{
dbName: dbName,
kind: kind,
name: name,
})
}
model := &model{
fields: fields,
name: prev,
}
models = append(models, model)
}
switch x := n.(type) {
case *ast.Ident:
prev = x.Name
}
return true
})
buf := &bytes.Buffer{}
buf.Write(header)
buf.Write([]byte(pkg.Name.Name))
buf.Write([]byte("\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"strconv\"\n\t\"time\"\n\t\"unicode/utf8\"\n)\n\n"))
for _, model := range models {
ref := strings.ToLower(string(model.name[0]))
fmt.Fprintf(buf, "func (%s *%s) Encode(buf *bytes.Buffer) {\n", ref, model.name)
last := len(model.fields) - 1
close := `{"`
written := false
for idx, field := range model.fields {
dbKind, ok := kindMap[field.kind]
if !ok {
log.Error("unsupported kind: %s", field.kind)
continue
}
prefix := `"`
suffix := `"`
if len(dbKind) == 2 {
prefix = "["
suffix = "]"
}
open := fmt.Sprintf(`%s%s":{"%s":%s`, close, field.dbName, dbKind, prefix)
comma := ","
if idx == last {
comma = ""
}
fmt.Fprintf(buf, "\tbuf.WriteString(`%s`)\n", open)
close = fmt.Sprintf(`%s}%s"`, suffix, comma)
written = true
selector := fmt.Sprintf("%s.%s", ref, field.name)
if len(dbKind) == 2 {
fmt.Fprintf(buf, "\tfor idx, elem := range %s {\n", selector)
fmt.Fprint(buf, "\t\tbuf.WriteByte('\"')\n")
write(buf, "\t\t", field.kind[2:], "elem")
fmt.Fprintf(buf, "\t\tif idx == len(%s)-1 {\n", selector)
fmt.Fprint(buf, "\t\t\tbuf.WriteByte('\"')\n")
fmt.Fprint(buf, "\t\t} else {\n")
fmt.Fprint(buf, "\t\t\tbuf.WriteString(`\",`)\n")
fmt.Fprint(buf, "\t\t}\n")
fmt.Fprint(buf, "\t}\n")
} else {
write(buf, "\t", field.kind, selector)
}
}
if written {
fmt.Fprintf(buf, "\tbuf.WriteString(`%s}`)\n", close[:len(close)-1])
}
fmt.Fprintf(buf, "}\n\n")
fmt.Fprintf(buf, "func (%s *%s) Decode(data map[string]map[string]interface{}) {\n", ref, model.name)
close = ""
for _, field := range model.fields {
dbKind, ok := kindMap[field.kind]
if !ok {
continue
}
selector := fmt.Sprintf("%s.%s", ref, field.name)
if len(dbKind) == 2 {
fmt.Fprintf(buf, "%s\tif vals, ok := data[\"%s\"][\"%s\"].([]interface{}); ok {\n", close, field.dbName, dbKind)
fmt.Fprint(buf, "\t\tfor _, sval := range vals {\n")
fmt.Fprint(buf, "\t\t\tval := sval.(string)\n")
readMulti(buf, "\t\t\t", field.kind, selector)
fmt.Fprint(buf, "\t\t}\n")
} else {
fmt.Fprintf(buf, "%s\tif val, ok := data[\"%s\"][\"%s\"].(string); ok {\n", close, field.dbName, dbKind)
read(buf, "\t\t", field.kind, selector)
}
close = "\t}\n"
}
fmt.Fprintf(buf, "%s}\n\n", close)
}
buf.Write(jsonSupport)
log.Info("Writing %s", newpath)
newfile, err := os.Create(newpath)
if err != nil {
runtime.StandardError(err)
}
newfile.Write(buf.Bytes())
newfile.Close()
}
func main() {
opts := optparse.Parser("Usage: html2domly [options]", "v")
outputFile := opts.String([]string{"-o", "--output"}, "../coffee/templates.coffee", "coffeescript file to compile to", "PATH")
templatesSrcDir := opts.String([]string{"-i", "--input"}, "../etc/domly", "template source directory", "PATH")
printJSON := opts.Bool([]string{"--print"}, false, "Print the JSON nicely to the output logger")
os.Args[0] = "html2domly"
opts.Parse(os.Args)
log.AddConsoleLogger()
var (
data []byte
err error
prettyStr string
basename string
pretty bytes.Buffer
)
dir, err := os.Open(*templatesSrcDir)
if err != nil {
runtime.StandardError(err)
}
defer dir.Close()
out, err := os.Create(*outputFile)
if err != nil {
runtime.StandardError(err)
}
defer out.Close()
names, err := dir.Readdirnames(0)
if err != nil {
runtime.StandardError(err)
}
out.Write([]byte("define 'templates', (exports, root) ->\n"))
for _, name := range names {
if strings.HasSuffix(name, ".html") {
basename = strings.TrimSuffix(name, ".html")
} else {
log.Error("file %v does not end in .html", name)
continue
}
templatePath := filepath.Join(*templatesSrcDir, name)
data, err = ui.ParseTemplate(templatePath)
if err != nil {
log.StandardError(err)
} else {
err := json.Indent(&pretty, data, ">", " ")
if err != nil {
log.StandardError(err)
log.Info("%v", data)
} else if *printJSON {
prettyStr = pretty.String()
pretty.Reset()
log.Info("%v", prettyStr)
}
out.Write([]byte(fmt.Sprintf(" exports['%s'] = `%s`\n", basename, data)))
log.Info("compiled '%s'", name)
}
}
out.Write([]byte(" return"))
outPath, _ := filepath.Abs(*outputFile)
log.Info("compiled domly written to: %v", outPath)
log.Wait()
}
func main() {
// Define the options for the command line and config file options parser.
opts := optparse.New(
"Usage: planfile <config.yaml> [options]\n",
"planfile 0.0.1")
cookieKeyFile := opts.StringConfig("cookie-key-file", "cookie.key",
"the file containing the key to sign cookie values [cookie.key]")
gaHost := opts.StringConfig("ga-host", "",
"the google analytics hostname to use")
gaID := opts.StringConfig("ga-id", "",
"the google analytics id to use")
httpAddr := opts.StringConfig("http-addr", ":8888",
"the address to bind the http server [:8888]")
oauthID := opts.Required().StringConfig("oauth-id", "",
"the oauth client id for github")
oauthSecret := opts.Required().StringConfig("oauth-secret", "",
"the oauth client secret for github")
redirectURL := opts.StringConfig("redirect-url", "/.oauth",
"the redirect url for handling oauth [/.oauth]")
repository := opts.Required().StringConfig("repository", "",
"the username/repository on github")
secureMode := opts.BoolConfig("secure-mode",
"enable hsts and secure cookies [false]")
title := opts.StringConfig("title", "Planfile",
"the title for the web app [Planfile]")
refreshKey := opts.StringConfig("refresh-key", "",
"key for anonymously calling refresh at /.refresh?key=<refresh-key>")
refreshOpt := opts.IntConfig("refresh-interval", 1,
"the number of through-the-web edits before a full refresh [1]")
debug, instanceDirectory, _, logPath, _ = runtime.DefaultOpts("planfile", opts, os.Args, true)
service := &oauth.OAuthService{
ClientID: *oauthID,
ClientSecret: *oauthSecret,
Scope: "public_repo",
AuthURL: "https://github.com/login/oauth/authorize",
TokenURL: "https://github.com/login/oauth/access_token",
RedirectURL: *redirectURL,
AcceptHeader: "application/json",
}
assets := map[string]string{}
json.Unmarshal(readFile("assets.json"), &assets)
setupPygments()
mutex := sync.RWMutex{}
repo := &Repo{Path: *repository}
err := repo.Load(callGithubAnon)
if err != nil {
runtime.Exit(1)
}
repo.Title = *title
repo.Updated = time.Now().UTC()
repoJSON, err := json.Marshal(repo)
if err != nil {
runtime.StandardError(err)
}
refreshCount := 0
refreshInterval := *refreshOpt
refreshKeySet := *refreshKey != ""
refreshKeyBytes := []byte(*refreshKey)
secret := readFile(*cookieKeyFile)
newContext := func(w http.ResponseWriter, r *http.Request) *Context {
return &Context{
r: r,
w: w,
secret: secret,
secure: *secureMode,
}
}
register := func(path string, handler func(*Context), usegzip ...bool) {
gzippable := len(usegzip) > 0 && usegzip[0]
http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
log.Info("serving %s", r.URL)
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if gzippable && httputil.Parse(r, "Accept-Encoding").Accepts("gzip") {
buf := &bytes.Buffer{}
enc := gzip.NewWriter(buf)
handler(newContext(GzipWriter{enc, w}, r))
enc.Close()
w.Header().Set("Content-Encoding", "gzip")
w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
buf.WriteTo(w)
} else {
handler(newContext(w, r))
}
})
}
anon := []byte(", null, null, '', false")
authFalse := []byte("', false")
authTrue := []byte("', true")
header := []byte(`<!doctype html>
<meta charset=utf-8>
<title>` + html.EscapeString(*title) + `</title>
<link href="//fonts.googleapis.com/css?family=Abel|Coustard:400" rel=stylesheet>
<link href=/.static/` + assets["planfile.css"] + ` rel=stylesheet>
<body><script>DATA = ['` + *gaHost + `', '` + *gaID + `', `)
footer := []byte(`];</script>
<script src=/.static/` + assets["planfile.js"] + `></script>
<noscript>Sorry, your browser needs <a href=http://enable-javascript.com>JavaScript enabled</a>.</noscript>
`)
register("/", func(ctx *Context) {
mutex.RLock()
defer mutex.RUnlock()
ctx.Write(header)
ctx.Write(repoJSON)
avatar := ctx.GetCookie("avatar")
user := ctx.GetCookie("user")
if avatar != "" && user != "" {
ctx.Write([]byte(", '" + user + "', '" + avatar + "', '" + ctx.GetCookie("xsrf")))
if ctx.IsAuthorised(repo) {
ctx.Write(authTrue)
} else {
ctx.Write(authFalse)
}
} else {
ctx.Write(anon)
}
ctx.Write(footer)
}, true)
register("/.api", func(ctx *Context) {
mutex.RLock()
defer mutex.RUnlock()
if cb := ctx.FormValue("callback"); cb != "" {
ctx.Write([]byte(cb))
ctx.Write([]byte{'('})
ctx.Write(repoJSON)
ctx.Write([]byte{')', ';'})
} else {
ctx.Write(repoJSON)
}
}, true)
register("/.login", func(ctx *Context) {
b := make([]byte, 20)
if n, err := rand.Read(b); err != nil || n != 20 {
ctx.Error("Couldn't access cryptographic device", err)
return
}
s := hex.EncodeToString(b)
ctx.SetCookie("xsrf", s)
ctx.Redirect(service.AuthCodeURL(s))
})
register("/.logout", func(ctx *Context) {
ctx.ExpireCookie("auth")
ctx.ExpireCookie("avatar")
ctx.ExpireCookie("token")
ctx.ExpireCookie("user")
ctx.ExpireCookie("xsrf")
ctx.Redirect("/")
})
notAuthorised := []byte("ERROR: Not Authorised!")
savedHeader := []byte(`<!doctype html>
<meta charset=utf-8>
<title>` + html.EscapeString(*title) + `</title>
<body><script>SAVED="`)
savedFooter := []byte(`"</script><script src=/.static/` + assets["planfile.js"] + `></script>`)
exportRepo := func(ctx *Context) bool {
repo.Updated = time.Now().UTC()
repoJSON, err = json.Marshal(repo)
if err != nil {
ctx.Error("Couldn't encode repo data during refresh", err)
return false
}
return true
}
refresh := func(ctx *Context) {
err := repo.Load(ctx.CreateCallGithub())
if err != nil {
log.Error("couldn't rebuild planfile info: %s", err)
ctx.Write([]byte("ERROR: " + err.Error()))
return
}
exportRepo(ctx)
}
saveItem := func(ctx *Context, update bool) {
mutex.Lock()
defer mutex.Unlock()
if !ctx.IsAuthorised(repo) {
ctx.Write(notAuthorised)
return
}
if !isEqual([]byte(ctx.FormValue("xsrf")), []byte(ctx.GetCookie("xsrf"))) {
ctx.Write(notAuthorised)
return
}
callGithub := ctx.CreateCallGithub()
err := repo.UpdateInfo(callGithub)
if err != nil {
ctx.Error("Couldn't update repo info", err)
return
}
var id, path, message string
if update {
id = ctx.FormValue("id")
path = ctx.FormValue("path")
} else {
baseID := ctx.FormValue("id")
id = baseID
count := 0
for repo.Exists(id + ".md") {
count += 1
id = fmt.Sprintf("%s%d", baseID, count)
}
path = id + ".md"
}
content := strings.Replace(ctx.FormValue("content"), "\r\n", "\n", -1)
tags := ctx.FormValue("tags")
title := ctx.FormValue("title")
redir := "/"
if ctx.FormValue("summary") == "yes" {
if id != "/" {
content = fmt.Sprintf(`---
title: %s
---
%s`, title, content)
if strings.HasPrefix(id, "summary.") {
redir = "/" + id[8:]
} else {
// Shouldn't ever happen. But just in case...
redir = "/" + id
}
}
} else {
redir = "/.item." + id
content = fmt.Sprintf(`---
id: %s
tags: %s
title: %s
---
%s`, id, tags, title, content)
}
if title == "" {
title = id
}
if update {
message = "update: " + title + "."
} else {
message = "add: " + title + "."
}
log.Info("SAVE PATH: %q for %q", path, title)
err = repo.Modify(ctx, path, content, message)
if err != nil {
if update {
ctx.Error("<a href='/.refresh'>Try refreshing.</a> Couldn't update item", err)
} else {
ctx.Error("<a href='/.refresh'>Try refreshing.</a> Couldn't save new item", err)
}
return
}
refreshCount++
if refreshCount%refreshInterval == 0 {
refresh(ctx)
} else {
repo.AddPlanfile(path, []byte(content), callGithub)
if !exportRepo(ctx) {
return
}
}
ctx.Write(savedHeader)
ctx.Write([]byte(html.EscapeString(redir)))
ctx.Write(savedFooter)
}
register("/.modify", func(ctx *Context) {
saveItem(ctx, true)
})
register("/.new", func(ctx *Context) {
saveItem(ctx, false)
})
register("/.oauth", func(ctx *Context) {
s := ctx.FormValue("state")
if s == "" {
ctx.Redirect("/.login")
return
}
if !isEqual([]byte(s), []byte(ctx.GetCookie("xsrf"))) {
ctx.ExpireCookie("xsrf")
ctx.Redirect("/.login")
return
}
t := &oauth.Transport{OAuthService: service}
tok, err := t.ExchangeAuthorizationCode(ctx.FormValue("code"))
if err != nil {
ctx.Error("Auth Exchange Error", err)
return
}
jtok, err := json.Marshal(tok)
if err != nil {
ctx.Error("Couldn't encode token", err)
return
}
ctx.SetCookie("token", hex.EncodeToString(jtok))
ctx.token = tok
user := &User{}
err = ctx.Call("/user", user, nil, false)
if err != nil {
ctx.Error("Couldn't load user info", err)
return
}
ctx.SetCookie("avatar", user.AvatarURL)
ctx.SetCookie("user", user.Login)
ctx.Redirect("/")
})
register("/.preview", func(ctx *Context) {
rendered, err := renderMarkdown([]byte(ctx.FormValue("content")))
if err != nil {
ctx.Error("Couldn't render Markdown", err)
return
}
ctx.Write(rendered)
}, true)
register("/.refresh", func(ctx *Context) {
if !ctx.IsAuthorised(repo) {
if !(refreshKeySet && isEqual(refreshKeyBytes, []byte(ctx.FormValue("key")))) {
ctx.Write(notAuthorised)
return
}
}
mutex.Lock()
defer mutex.Unlock()
refresh(ctx)
ctx.Redirect("/")
})
mimetypes := map[string]string{
"css": "text/css",
"gif": "image/gif",
"ico": "image/x-icon",
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"js": "text/javascript",
"png": "image/png",
"swf": "application/x-shockwave-flash",
"txt": "text/plain",
}
registerStatic := func(filepath, urlpath string) {
_, ext := rsplit(filepath, ".")
ctype, ok := mimetypes[ext]
if !ok {
ctype = "application/octet-stream"
}
if debug {
register(urlpath, func(ctx *Context) {
ctx.SetHeader("Content-Type", ctype)
ctx.Write(readFile(filepath))
}, strings.HasPrefix(ctype, "text/"))
} else {
content := readFile(filepath)
register(urlpath, func(ctx *Context) {
ctx.SetHeader("Cache-Control", "public, max-age=86400")
ctx.SetHeader("Content-Type", ctype)
ctx.Write(content)
}, strings.HasPrefix(ctype, "text/"))
}
}
for _, path := range assets {
registerStatic(filepath.Join(instanceDirectory, "static", path), "/.static/"+path)
}
wwwPath := filepath.Join(instanceDirectory, "www")
if files, err := ioutil.ReadDir(wwwPath); err == nil {
for _, file := range files {
if !file.IsDir() {
registerStatic(filepath.Join(wwwPath, file.Name()), "/"+file.Name())
}
}
}
log.Info("Listening on %s", *httpAddr)
server := &http.Server{
Addr: *httpAddr,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
err = server.ListenAndServe()
if err != nil {
runtime.Error("couldn't bind to tcp socket: %s", err)
}
}