func (r *Repo) Load(callGithub githubCallFunc) error { log.Info("loading repo: %s", r.Path) url := "https://github.com/" + r.Path + "/tarball/master" resp, err := httpClient.Get(url) if err != nil { log.StandardError(err) return err } defer resp.Body.Close() zf, err := gzip.NewReader(resp.Body) if err != nil { log.Error("couldn't find a valid repo tarball at %s -- %s", url, err) return err } tr := tar.NewReader(zf) r.Avatars = map[string]string{} r.Orderings = map[string]*Ordering{} r.Planfiles = map[string]*Planfile{} for { hdr, err := tr.Next() if err == io.EOF { break } if err != nil { log.Error("reading tarball: %s", err) return err } filename, ext := rsplit(hdr.Name, ".") _, filename = rsplit(filename, "/") if ext == "md" || ext == "order" { log.Info("parsing: %s", filename) data, err := ioutil.ReadAll(tr) if err != nil { log.Error("reading tarball file %q: %s", hdr.Name, err) continue } if ext == "md" { r.AddPlanfile(filename+".md", data, callGithub) } else { r.AddOrdering(filename+".order", data) } } } log.Info("successfully loaded repo: %s", r.Path) return nil }
func runHTTP(name string, host string, port int, handler http.Handler, suffix string) { addr := fmt.Sprintf("%s:%d", host, port) listener, err := net.Listen("tcp", addr) if err != nil { runtime.Error("Cannot listen on %s: %v", addr, err) } go serveHTTP(name, listener, handler) if host == "" { host = "localhost" } log.Info("%s running on http://%s:%d%s", name, host, port, suffix) }
func renderMarkdown(input []byte) (out []byte, err error) { defer func() { if e := recover(); e != nil { if e, ok := e.(error); ok { log.Info("hilite error: restarting, due to: %s", e) setupPygments() err = e } } }() r := &Html{blackfriday.HtmlRenderer(DefaultHtmlFlags, "", "")} out = blackfriday.Markdown(input, r, DefaultExtensions) return }
func (db *DB) Run(host string, port int, limit int64, master MasterController) error { if host == "" { host = runtime.GetIP() } db.id = getNodeID(host, port) db.limit = limit << 10 db.limitNotify = (db.limit * 2) / 3 addr, listener := runtime.GetAddrListener(host, port) defer listener.Close() go master.Run(db.id) log.Info("MatchDB running on %s", addr) delay := 1 * time.Millisecond maxDelay := 1 * time.Second for { conn, err := listener.Accept() if err != nil { if ne, ok := err.(net.Error); ok && ne.Temporary() { if delay == 0 { delay = 5 * time.Millisecond } else { delay *= 2 } if delay > maxDelay { delay = maxDelay } log.Error("Accept error: %v; retrying in %v", err, delay) time.Sleep(delay) continue } log.Error("Accept error: %v", err) return err } delay = 0 go db.Handle(conn) } }
func main() { opts := optparse.Parser("Usage: genapi [options]", "0.1") root := opts.String( []string{"-r", "--root"}, "../src/espra", "path to the root package directory for the app", "PATH") ignoreList := opts.String( []string{"-i", "--ignore"}, "api.go html.go", "space-separated list of files/subdirectories to ignore", "LIST") digestFile := opts.String( []string{"-d", "--digest"}, "../etc/app/version.digest", "path to write the digest of the API version", "PATH") os.Args[0] = "genapi" opts.Parse(os.Args) log.AddConsoleLogger() ignore := strings.Split(*ignoreList, " ") for i, path := range ignore { ignore[i] = filepath.Join(*root, path) } pkgpath := strings.Split(*root, "/") pkgname := pkgpath[len(pkgpath)-1] if err := parseDirectory(pkgname, pkgname, *root, ignore); err != nil { runtime.StandardError(err) } for _, method := range methods { log.Info("%#v", method) } log.Wait() _ = *digestFile }
func parseFile(path string, force bool) { dir, filename := filepath.Split(path) if !strings.HasSuffix(filename, ".go") { runtime.Error("%s does not look like a go file", filename) } log.Info("Parsing %s", path) fset := token.NewFileSet() pkg, err := parser.ParseFile(fset, path, nil, 0) if err != nil { runtime.StandardError(err) } newpath := filepath.Join(dir, fmt.Sprintf("%s_marshal.go", filename[:len(filename)-3])) exists, err := fsutil.FileExists(newpath) if exists && !force { runtime.Error("%s already exists! please specify --force to overwrite", newpath) } prev := "" models := []*model{} ast.Inspect(pkg, func(n ast.Node) bool { if s, ok := n.(*ast.StructType); ok { fields := []fieldInfo{} for _, field := range s.Fields.List { if field.Names == nil { continue } name := field.Names[0].Name dbName := "" kind := "" if field.Tag != nil { tag := field.Tag.Value[1 : len(field.Tag.Value)-1] if tag == "-" { continue } dbName = tag } if dbName == "" { dbName = name rune, _ := utf8.DecodeRuneInString(name) if !unicode.IsUpper(rune) { continue } } switch expr := field.Type.(type) { case *ast.Ident: switch expr.Name { case "bool", "string", "int", "int64", "uint", "uint64": kind = expr.Name } case *ast.ArrayType: if expr.Len == nil { // slice type switch iexpr := expr.Elt.(type) { case *ast.Ident: switch iexpr.Name { case "byte", "bool", "string", "int", "int64", "uint", "uint64": kind = "[]" + iexpr.Name } case *ast.ArrayType: if iexpr.Len == nil { if iiexpr, ok := iexpr.Elt.(*ast.Ident); ok { if iiexpr.Name == "byte" { kind = "[][]byte" } } } } } case *ast.SelectorExpr: if lexpr, ok := expr.X.(*ast.Ident); ok { if lexpr.Name == "time" && expr.Sel.Name == "Time" { kind = "time" } } } if kind == "" { log.Error("unsupported: %v field (%s.%s)", field.Type, prev, name) continue } fields = append(fields, fieldInfo{ dbName: dbName, kind: kind, name: name, }) } model := &model{ fields: fields, name: prev, } models = append(models, model) } switch x := n.(type) { case *ast.Ident: prev = x.Name } return true }) buf := &bytes.Buffer{} buf.Write(header) buf.Write([]byte(pkg.Name.Name)) buf.Write([]byte("\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"strconv\"\n\t\"time\"\n\t\"unicode/utf8\"\n)\n\n")) for _, model := range models { ref := strings.ToLower(string(model.name[0])) fmt.Fprintf(buf, "func (%s *%s) Encode(buf *bytes.Buffer) {\n", ref, model.name) last := len(model.fields) - 1 close := `{"` written := false for idx, field := range model.fields { dbKind, ok := kindMap[field.kind] if !ok { log.Error("unsupported kind: %s", field.kind) continue } prefix := `"` suffix := `"` if len(dbKind) == 2 { prefix = "[" suffix = "]" } open := fmt.Sprintf(`%s%s":{"%s":%s`, close, field.dbName, dbKind, prefix) comma := "," if idx == last { comma = "" } fmt.Fprintf(buf, "\tbuf.WriteString(`%s`)\n", open) close = fmt.Sprintf(`%s}%s"`, suffix, comma) written = true selector := fmt.Sprintf("%s.%s", ref, field.name) if len(dbKind) == 2 { fmt.Fprintf(buf, "\tfor idx, elem := range %s {\n", selector) fmt.Fprint(buf, "\t\tbuf.WriteByte('\"')\n") write(buf, "\t\t", field.kind[2:], "elem") fmt.Fprintf(buf, "\t\tif idx == len(%s)-1 {\n", selector) fmt.Fprint(buf, "\t\t\tbuf.WriteByte('\"')\n") fmt.Fprint(buf, "\t\t} else {\n") fmt.Fprint(buf, "\t\t\tbuf.WriteString(`\",`)\n") fmt.Fprint(buf, "\t\t}\n") fmt.Fprint(buf, "\t}\n") } else { write(buf, "\t", field.kind, selector) } } if written { fmt.Fprintf(buf, "\tbuf.WriteString(`%s}`)\n", close[:len(close)-1]) } fmt.Fprintf(buf, "}\n\n") fmt.Fprintf(buf, "func (%s *%s) Decode(data map[string]map[string]interface{}) {\n", ref, model.name) close = "" for _, field := range model.fields { dbKind, ok := kindMap[field.kind] if !ok { continue } selector := fmt.Sprintf("%s.%s", ref, field.name) if len(dbKind) == 2 { fmt.Fprintf(buf, "%s\tif vals, ok := data[\"%s\"][\"%s\"].([]interface{}); ok {\n", close, field.dbName, dbKind) fmt.Fprint(buf, "\t\tfor _, sval := range vals {\n") fmt.Fprint(buf, "\t\t\tval := sval.(string)\n") readMulti(buf, "\t\t\t", field.kind, selector) fmt.Fprint(buf, "\t\t}\n") } else { fmt.Fprintf(buf, "%s\tif val, ok := data[\"%s\"][\"%s\"].(string); ok {\n", close, field.dbName, dbKind) read(buf, "\t\t", field.kind, selector) } close = "\t}\n" } fmt.Fprintf(buf, "%s}\n\n", close) } buf.Write(jsonSupport) log.Info("Writing %s", newpath) newfile, err := os.Create(newpath) if err != nil { runtime.StandardError(err) } newfile.Write(buf.Bytes()) newfile.Close() }
func main() { opts := optparse.Parser("Usage: html2domly [options]", "v") outputFile := opts.String([]string{"-o", "--output"}, "../coffee/templates.coffee", "coffeescript file to compile to", "PATH") templatesSrcDir := opts.String([]string{"-i", "--input"}, "../etc/domly", "template source directory", "PATH") printJSON := opts.Bool([]string{"--print"}, false, "Print the JSON nicely to the output logger") os.Args[0] = "html2domly" opts.Parse(os.Args) log.AddConsoleLogger() var ( data []byte err error prettyStr string basename string pretty bytes.Buffer ) dir, err := os.Open(*templatesSrcDir) if err != nil { runtime.StandardError(err) } defer dir.Close() out, err := os.Create(*outputFile) if err != nil { runtime.StandardError(err) } defer out.Close() names, err := dir.Readdirnames(0) if err != nil { runtime.StandardError(err) } out.Write([]byte("define 'templates', (exports, root) ->\n")) for _, name := range names { if strings.HasSuffix(name, ".html") { basename = strings.TrimSuffix(name, ".html") } else { log.Error("file %v does not end in .html", name) continue } templatePath := filepath.Join(*templatesSrcDir, name) data, err = ui.ParseTemplate(templatePath) if err != nil { log.StandardError(err) } else { err := json.Indent(&pretty, data, ">", " ") if err != nil { log.StandardError(err) log.Info("%v", data) } else if *printJSON { prettyStr = pretty.String() pretty.Reset() log.Info("%v", prettyStr) } out.Write([]byte(fmt.Sprintf(" exports['%s'] = `%s`\n", basename, data))) log.Info("compiled '%s'", name) } } out.Write([]byte(" return")) outPath, _ := filepath.Abs(*outputFile) log.Info("compiled domly written to: %v", outPath) log.Wait() }
func main() { // Define the options for the command line and config file options parser. opts := optparse.New( "Usage: planfile <config.yaml> [options]\n", "planfile 0.0.1") cookieKeyFile := opts.StringConfig("cookie-key-file", "cookie.key", "the file containing the key to sign cookie values [cookie.key]") gaHost := opts.StringConfig("ga-host", "", "the google analytics hostname to use") gaID := opts.StringConfig("ga-id", "", "the google analytics id to use") httpAddr := opts.StringConfig("http-addr", ":8888", "the address to bind the http server [:8888]") oauthID := opts.Required().StringConfig("oauth-id", "", "the oauth client id for github") oauthSecret := opts.Required().StringConfig("oauth-secret", "", "the oauth client secret for github") redirectURL := opts.StringConfig("redirect-url", "/.oauth", "the redirect url for handling oauth [/.oauth]") repository := opts.Required().StringConfig("repository", "", "the username/repository on github") secureMode := opts.BoolConfig("secure-mode", "enable hsts and secure cookies [false]") title := opts.StringConfig("title", "Planfile", "the title for the web app [Planfile]") refreshKey := opts.StringConfig("refresh-key", "", "key for anonymously calling refresh at /.refresh?key=<refresh-key>") refreshOpt := opts.IntConfig("refresh-interval", 1, "the number of through-the-web edits before a full refresh [1]") debug, instanceDirectory, _, logPath, _ = runtime.DefaultOpts("planfile", opts, os.Args, true) service := &oauth.OAuthService{ ClientID: *oauthID, ClientSecret: *oauthSecret, Scope: "public_repo", AuthURL: "https://github.com/login/oauth/authorize", TokenURL: "https://github.com/login/oauth/access_token", RedirectURL: *redirectURL, AcceptHeader: "application/json", } assets := map[string]string{} json.Unmarshal(readFile("assets.json"), &assets) setupPygments() mutex := sync.RWMutex{} repo := &Repo{Path: *repository} err := repo.Load(callGithubAnon) if err != nil { runtime.Exit(1) } repo.Title = *title repo.Updated = time.Now().UTC() repoJSON, err := json.Marshal(repo) if err != nil { runtime.StandardError(err) } refreshCount := 0 refreshInterval := *refreshOpt refreshKeySet := *refreshKey != "" refreshKeyBytes := []byte(*refreshKey) secret := readFile(*cookieKeyFile) newContext := func(w http.ResponseWriter, r *http.Request) *Context { return &Context{ r: r, w: w, secret: secret, secure: *secureMode, } } register := func(path string, handler func(*Context), usegzip ...bool) { gzippable := len(usegzip) > 0 && usegzip[0] http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { log.Info("serving %s", r.URL) w.Header().Set("Content-Type", "text/html; charset=utf-8") if gzippable && httputil.Parse(r, "Accept-Encoding").Accepts("gzip") { buf := &bytes.Buffer{} enc := gzip.NewWriter(buf) handler(newContext(GzipWriter{enc, w}, r)) enc.Close() w.Header().Set("Content-Encoding", "gzip") w.Header().Set("Content-Length", strconv.Itoa(buf.Len())) buf.WriteTo(w) } else { handler(newContext(w, r)) } }) } anon := []byte(", null, null, '', false") authFalse := []byte("', false") authTrue := []byte("', true") header := []byte(`<!doctype html> <meta charset=utf-8> <title>` + html.EscapeString(*title) + `</title> <link href="//fonts.googleapis.com/css?family=Abel|Coustard:400" rel=stylesheet> <link href=/.static/` + assets["planfile.css"] + ` rel=stylesheet> <body><script>DATA = ['` + *gaHost + `', '` + *gaID + `', `) footer := []byte(`];</script> <script src=/.static/` + assets["planfile.js"] + `></script> <noscript>Sorry, your browser needs <a href=http://enable-javascript.com>JavaScript enabled</a>.</noscript> `) register("/", func(ctx *Context) { mutex.RLock() defer mutex.RUnlock() ctx.Write(header) ctx.Write(repoJSON) avatar := ctx.GetCookie("avatar") user := ctx.GetCookie("user") if avatar != "" && user != "" { ctx.Write([]byte(", '" + user + "', '" + avatar + "', '" + ctx.GetCookie("xsrf"))) if ctx.IsAuthorised(repo) { ctx.Write(authTrue) } else { ctx.Write(authFalse) } } else { ctx.Write(anon) } ctx.Write(footer) }, true) register("/.api", func(ctx *Context) { mutex.RLock() defer mutex.RUnlock() if cb := ctx.FormValue("callback"); cb != "" { ctx.Write([]byte(cb)) ctx.Write([]byte{'('}) ctx.Write(repoJSON) ctx.Write([]byte{')', ';'}) } else { ctx.Write(repoJSON) } }, true) register("/.login", func(ctx *Context) { b := make([]byte, 20) if n, err := rand.Read(b); err != nil || n != 20 { ctx.Error("Couldn't access cryptographic device", err) return } s := hex.EncodeToString(b) ctx.SetCookie("xsrf", s) ctx.Redirect(service.AuthCodeURL(s)) }) register("/.logout", func(ctx *Context) { ctx.ExpireCookie("auth") ctx.ExpireCookie("avatar") ctx.ExpireCookie("token") ctx.ExpireCookie("user") ctx.ExpireCookie("xsrf") ctx.Redirect("/") }) notAuthorised := []byte("ERROR: Not Authorised!") savedHeader := []byte(`<!doctype html> <meta charset=utf-8> <title>` + html.EscapeString(*title) + `</title> <body><script>SAVED="`) savedFooter := []byte(`"</script><script src=/.static/` + assets["planfile.js"] + `></script>`) exportRepo := func(ctx *Context) bool { repo.Updated = time.Now().UTC() repoJSON, err = json.Marshal(repo) if err != nil { ctx.Error("Couldn't encode repo data during refresh", err) return false } return true } refresh := func(ctx *Context) { err := repo.Load(ctx.CreateCallGithub()) if err != nil { log.Error("couldn't rebuild planfile info: %s", err) ctx.Write([]byte("ERROR: " + err.Error())) return } exportRepo(ctx) } saveItem := func(ctx *Context, update bool) { mutex.Lock() defer mutex.Unlock() if !ctx.IsAuthorised(repo) { ctx.Write(notAuthorised) return } if !isEqual([]byte(ctx.FormValue("xsrf")), []byte(ctx.GetCookie("xsrf"))) { ctx.Write(notAuthorised) return } callGithub := ctx.CreateCallGithub() err := repo.UpdateInfo(callGithub) if err != nil { ctx.Error("Couldn't update repo info", err) return } var id, path, message string if update { id = ctx.FormValue("id") path = ctx.FormValue("path") } else { baseID := ctx.FormValue("id") id = baseID count := 0 for repo.Exists(id + ".md") { count += 1 id = fmt.Sprintf("%s%d", baseID, count) } path = id + ".md" } content := strings.Replace(ctx.FormValue("content"), "\r\n", "\n", -1) tags := ctx.FormValue("tags") title := ctx.FormValue("title") redir := "/" if ctx.FormValue("summary") == "yes" { if id != "/" { content = fmt.Sprintf(`--- title: %s --- %s`, title, content) if strings.HasPrefix(id, "summary.") { redir = "/" + id[8:] } else { // Shouldn't ever happen. But just in case... redir = "/" + id } } } else { redir = "/.item." + id content = fmt.Sprintf(`--- id: %s tags: %s title: %s --- %s`, id, tags, title, content) } if title == "" { title = id } if update { message = "update: " + title + "." } else { message = "add: " + title + "." } log.Info("SAVE PATH: %q for %q", path, title) err = repo.Modify(ctx, path, content, message) if err != nil { if update { ctx.Error("<a href='/.refresh'>Try refreshing.</a> Couldn't update item", err) } else { ctx.Error("<a href='/.refresh'>Try refreshing.</a> Couldn't save new item", err) } return } refreshCount++ if refreshCount%refreshInterval == 0 { refresh(ctx) } else { repo.AddPlanfile(path, []byte(content), callGithub) if !exportRepo(ctx) { return } } ctx.Write(savedHeader) ctx.Write([]byte(html.EscapeString(redir))) ctx.Write(savedFooter) } register("/.modify", func(ctx *Context) { saveItem(ctx, true) }) register("/.new", func(ctx *Context) { saveItem(ctx, false) }) register("/.oauth", func(ctx *Context) { s := ctx.FormValue("state") if s == "" { ctx.Redirect("/.login") return } if !isEqual([]byte(s), []byte(ctx.GetCookie("xsrf"))) { ctx.ExpireCookie("xsrf") ctx.Redirect("/.login") return } t := &oauth.Transport{OAuthService: service} tok, err := t.ExchangeAuthorizationCode(ctx.FormValue("code")) if err != nil { ctx.Error("Auth Exchange Error", err) return } jtok, err := json.Marshal(tok) if err != nil { ctx.Error("Couldn't encode token", err) return } ctx.SetCookie("token", hex.EncodeToString(jtok)) ctx.token = tok user := &User{} err = ctx.Call("/user", user, nil, false) if err != nil { ctx.Error("Couldn't load user info", err) return } ctx.SetCookie("avatar", user.AvatarURL) ctx.SetCookie("user", user.Login) ctx.Redirect("/") }) register("/.preview", func(ctx *Context) { rendered, err := renderMarkdown([]byte(ctx.FormValue("content"))) if err != nil { ctx.Error("Couldn't render Markdown", err) return } ctx.Write(rendered) }, true) register("/.refresh", func(ctx *Context) { if !ctx.IsAuthorised(repo) { if !(refreshKeySet && isEqual(refreshKeyBytes, []byte(ctx.FormValue("key")))) { ctx.Write(notAuthorised) return } } mutex.Lock() defer mutex.Unlock() refresh(ctx) ctx.Redirect("/") }) mimetypes := map[string]string{ "css": "text/css", "gif": "image/gif", "ico": "image/x-icon", "jpeg": "image/jpeg", "jpg": "image/jpeg", "js": "text/javascript", "png": "image/png", "swf": "application/x-shockwave-flash", "txt": "text/plain", } registerStatic := func(filepath, urlpath string) { _, ext := rsplit(filepath, ".") ctype, ok := mimetypes[ext] if !ok { ctype = "application/octet-stream" } if debug { register(urlpath, func(ctx *Context) { ctx.SetHeader("Content-Type", ctype) ctx.Write(readFile(filepath)) }, strings.HasPrefix(ctype, "text/")) } else { content := readFile(filepath) register(urlpath, func(ctx *Context) { ctx.SetHeader("Cache-Control", "public, max-age=86400") ctx.SetHeader("Content-Type", ctype) ctx.Write(content) }, strings.HasPrefix(ctype, "text/")) } } for _, path := range assets { registerStatic(filepath.Join(instanceDirectory, "static", path), "/.static/"+path) } wwwPath := filepath.Join(instanceDirectory, "www") if files, err := ioutil.ReadDir(wwwPath); err == nil { for _, file := range files { if !file.IsDir() { registerStatic(filepath.Join(wwwPath, file.Name()), "/"+file.Name()) } } } log.Info("Listening on %s", *httpAddr) server := &http.Server{ Addr: *httpAddr, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, } err = server.ListenAndServe() if err != nil { runtime.Error("couldn't bind to tcp socket: %s", err) } }