func initHashKey(s string) { if len(s) != 32 { runtime.Error("The hash-key value %q is not a 32-byte string.", s) } k, err := hex.DecodeString(s) if err != nil { runtime.Error("The hash-key value %q is not a hex-encoded string: %s", s, err) } hashKey0 = binary.LittleEndian.Uint64(k[:8]) hashKey1 = binary.LittleEndian.Uint64(k[8:]) }
func setUpstreamInfo(s *LiveServer, host string, port int, useTLS bool, id int) { if id == 0 { runtime.Error("The publish-cluster-id cannot be 0") } if (id & (id - 1)) != 0 { runtime.Error("The publish-cluster-id is not a power of 2") } var baseURL string if useTLS && port == 443 { baseURL = fmt.Sprintf("https://%s/", host) } else if useTLS { baseURL = fmt.Sprintf("https://%s:%d/", host, port) } else { baseURL = fmt.Sprintf("http://%s:%d/", host, port) } s.publishAck = []byte(fmt.Sprintf("%s,%d,%d", string(s.publishKey), id, id)) s.publishAckURL = baseURL + "ack_publish" }
func NewMaster(singleNode, awsAccess, awsSecret, dynamoRegion, dynamoTable string, masterTimeout time.Duration) MasterController { if awsAcess != "" || awsSecret != "" || dynamoTable != "" { if awsAcess == "" { runtime.Error("Distributed options set, but aws-access-key hasn't been specified") } if awsSecret == "" { runtime.Error("Distributed options set, but aws-secret-key hasn't been specified") } if dynamoTable == "" { runtime.Error("Distributed options set, but master-table hasn't been specified") } region, ok := aws.Regions[dynamoRegion] if !ok { runtime.Error("AWS Region %q not known", dynamoRegion) } return &Master{} } return &LocalMaster{addr: singleNode} }
func getNodeID(host string, port int) []byte { ip := net.ParseIP(host).To4() if ip == nil || len(ip) != 4 { runtime.Error("Could not parse host %q into a 32-bit IPv4 address", host) } id := make([]byte, 14) copy(id[:4], ip) binary.LittleEndian.PutUint16(id[4:6], uint16(port)) binary.LittleEndian.PutUint64(id[6:], uint64(time.Now().UnixNano()<<1)) return id }
func runHTTP(name string, host string, port int, handler http.Handler, suffix string) { addr := fmt.Sprintf("%s:%d", host, port) listener, err := net.Listen("tcp", addr) if err != nil { runtime.Error("Cannot listen on %s: %v", addr, err) } go serveHTTP(name, listener, handler) if host == "" { host = "localhost" } log.Info("%s running on http://%s:%d%s", name, host, port, suffix) }
func parseFile(path string, force bool) { dir, filename := filepath.Split(path) if !strings.HasSuffix(filename, ".go") { runtime.Error("%s does not look like a go file", filename) } log.Info("Parsing %s", path) fset := token.NewFileSet() pkg, err := parser.ParseFile(fset, path, nil, 0) if err != nil { runtime.StandardError(err) } newpath := filepath.Join(dir, fmt.Sprintf("%s_marshal.go", filename[:len(filename)-3])) exists, err := fsutil.FileExists(newpath) if exists && !force { runtime.Error("%s already exists! please specify --force to overwrite", newpath) } prev := "" models := []*model{} ast.Inspect(pkg, func(n ast.Node) bool { if s, ok := n.(*ast.StructType); ok { fields := []fieldInfo{} for _, field := range s.Fields.List { if field.Names == nil { continue } name := field.Names[0].Name dbName := "" kind := "" if field.Tag != nil { tag := field.Tag.Value[1 : len(field.Tag.Value)-1] if tag == "-" { continue } dbName = tag } if dbName == "" { dbName = name rune, _ := utf8.DecodeRuneInString(name) if !unicode.IsUpper(rune) { continue } } switch expr := field.Type.(type) { case *ast.Ident: switch expr.Name { case "bool", "string", "int", "int64", "uint", "uint64": kind = expr.Name } case *ast.ArrayType: if expr.Len == nil { // slice type switch iexpr := expr.Elt.(type) { case *ast.Ident: switch iexpr.Name { case "byte", "bool", "string", "int", "int64", "uint", "uint64": kind = "[]" + iexpr.Name } case *ast.ArrayType: if iexpr.Len == nil { if iiexpr, ok := iexpr.Elt.(*ast.Ident); ok { if iiexpr.Name == "byte" { kind = "[][]byte" } } } } } case *ast.SelectorExpr: if lexpr, ok := expr.X.(*ast.Ident); ok { if lexpr.Name == "time" && expr.Sel.Name == "Time" { kind = "time" } } } if kind == "" { log.Error("unsupported: %v field (%s.%s)", field.Type, prev, name) continue } fields = append(fields, fieldInfo{ dbName: dbName, kind: kind, name: name, }) } model := &model{ fields: fields, name: prev, } models = append(models, model) } switch x := n.(type) { case *ast.Ident: prev = x.Name } return true }) buf := &bytes.Buffer{} buf.Write(header) buf.Write([]byte(pkg.Name.Name)) buf.Write([]byte("\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"strconv\"\n\t\"time\"\n\t\"unicode/utf8\"\n)\n\n")) for _, model := range models { ref := strings.ToLower(string(model.name[0])) fmt.Fprintf(buf, "func (%s *%s) Encode(buf *bytes.Buffer) {\n", ref, model.name) last := len(model.fields) - 1 close := `{"` written := false for idx, field := range model.fields { dbKind, ok := kindMap[field.kind] if !ok { log.Error("unsupported kind: %s", field.kind) continue } prefix := `"` suffix := `"` if len(dbKind) == 2 { prefix = "[" suffix = "]" } open := fmt.Sprintf(`%s%s":{"%s":%s`, close, field.dbName, dbKind, prefix) comma := "," if idx == last { comma = "" } fmt.Fprintf(buf, "\tbuf.WriteString(`%s`)\n", open) close = fmt.Sprintf(`%s}%s"`, suffix, comma) written = true selector := fmt.Sprintf("%s.%s", ref, field.name) if len(dbKind) == 2 { fmt.Fprintf(buf, "\tfor idx, elem := range %s {\n", selector) fmt.Fprint(buf, "\t\tbuf.WriteByte('\"')\n") write(buf, "\t\t", field.kind[2:], "elem") fmt.Fprintf(buf, "\t\tif idx == len(%s)-1 {\n", selector) fmt.Fprint(buf, "\t\t\tbuf.WriteByte('\"')\n") fmt.Fprint(buf, "\t\t} else {\n") fmt.Fprint(buf, "\t\t\tbuf.WriteString(`\",`)\n") fmt.Fprint(buf, "\t\t}\n") fmt.Fprint(buf, "\t}\n") } else { write(buf, "\t", field.kind, selector) } } if written { fmt.Fprintf(buf, "\tbuf.WriteString(`%s}`)\n", close[:len(close)-1]) } fmt.Fprintf(buf, "}\n\n") fmt.Fprintf(buf, "func (%s *%s) Decode(data map[string]map[string]interface{}) {\n", ref, model.name) close = "" for _, field := range model.fields { dbKind, ok := kindMap[field.kind] if !ok { continue } selector := fmt.Sprintf("%s.%s", ref, field.name) if len(dbKind) == 2 { fmt.Fprintf(buf, "%s\tif vals, ok := data[\"%s\"][\"%s\"].([]interface{}); ok {\n", close, field.dbName, dbKind) fmt.Fprint(buf, "\t\tfor _, sval := range vals {\n") fmt.Fprint(buf, "\t\t\tval := sval.(string)\n") readMulti(buf, "\t\t\t", field.kind, selector) fmt.Fprint(buf, "\t\t}\n") } else { fmt.Fprintf(buf, "%s\tif val, ok := data[\"%s\"][\"%s\"].(string); ok {\n", close, field.dbName, dbKind) read(buf, "\t\t", field.kind, selector) } close = "\t}\n" } fmt.Fprintf(buf, "%s}\n\n", close) } buf.Write(jsonSupport) log.Info("Writing %s", newpath) newfile, err := os.Create(newpath) if err != nil { runtime.StandardError(err) } newfile.Write(buf.Bytes()) newfile.Close() }
func main() { // Define the options for the command line and config file options parser. opts := optparse.Parser( "Usage: matchweb <config.yaml> [options]\n", "matchweb 0.0.1") host := opts.StringConfig("host", "", "the host to bind the matchweb to") port := opts.IntConfig("port", 9040, "the port to bind the matchweb to [9040]") redirectURL := opts.StringConfig("redirect-url", "", "the URL that the HTTP Redirector redirects to") redirectorHost := opts.StringConfig("redirector-host", "", "the host to bind the HTTP Redirector to") redirectorPort := opts.IntConfig("redirector-port", 9080, "the port to bind the HTTP Redirector to [9080]") hstsMaxAge := opts.IntConfig("hsts-max-age", 31536000, "max-age value of HSTS in number of seconds [0 (disabled)]") clusterID := opts.StringConfig("cluster-id", "", "the cluster id to use when responding to ping requests") matchdbServer := opts.StringConfig("matchdb-server", "", "the address for a single-node MatchDB server setup") hashKey := opts.StringConfig("hash-key", "", "16-byte hash key encoded as a 32-byte hex string") awsAccessKey := opts.StringConfig("aws-access-key", "", "the AWS Access Key for DynamoDB") awsSecretKey := opts.StringConfig("aws-secret-key", "", "the AWS Secret Key for DynamoDB") awsRegion := opts.StringConfig("aws-region", "us-east-1", "the AWS Region for DynamoDB [us-east-1]") masterTable := opts.StringConfig("master-table", "", "the DynamoDB table for the master lock") masterTimeout := opts.IntConfig("master-timeout", 6000, "timeout in milliseconds for the master lock [6000]") routingTimeout := opts.IntConfig("routing-timeout", 3000, "timeout in milliseconds for routing entries [3000]") publishKey := opts.StringConfig("publish-key", "", "the shared secret for publishing new items") publishClusterID := opts.IntConfig("publish-cluster-id", 0, "the cluster id to use when acknowledging publish requests") upstreamHost := opts.StringConfig("upstream-host", "localhost", "the upstream host to connect to [localhost]") upstreamPort := opts.IntConfig("upstream-port", 8080, "the upstream port to connect to [8080]") upstreamTLS := opts.BoolConfig("upstream-tls", false, "use TLS when connecting to upstream [false]") websocketOrigin := opts.StringConfig("websocket-origin", "", "limit websocket calls to the given origin if specified") maintenanceMode := opts.BoolConfig("maintenance", false, "start up in maintenance mode [false]") // Setup the console log filter. log.ConsoleFilters[logPrefix] = func(items []interface{}) (bool, []interface{}) { return true, items[2 : len(items)-2] } // Parse the command line options. os.Args[0] = "matchweb" runtime.DefaultOpts("matchweb", opts, os.Args) // Initialise the TLS config. tlsconf.Init() // Initialise ping/pong variables. setupPong("matchweb", *clusterID) // Initialise the key for hashing slots. initHashKey(*hashKey) // Ensure required config values. if *publishKey == "" { runtime.Error("The publish-key cannot be empty") } server := &LiveServer{ httpClient: &http.Client{Transport: &http.Transport{TLSClientConfig: tlsconf.Config}}, publishKey: []byte(*publishKey), publishKeyLength: len(*publishKey), websocketOrigin: *websocketOrigin, } setUpstreamInfo(server, *upstreamHost, *upstreamPort, *upstreamTLS, *publishClusterID) if *hstsMaxAge != 0 { server.hstsEnabled = true server.hsts = fmt.Sprintf("max-age=%d", *hstsMaxAge) } // Enable maintenance handling. frontends := []Maintainable{server} handleMaintenance(frontends, *maintenanceMode) // Setup the HTTP Redirector. runRedirector(*redirectorHost, *redirectorPort, *redirectURL, *hstsMaxAge) // Run the Live Server. runHTTP("Live Server", *host, *port, server, "") // Enter the wait loop for the process to be killed. loopForever := make(chan bool, 1) <-loopForever }
func serveHTTP(name string, listener net.Listener, handler http.Handler) { err := http.Serve(listener, handler) if err != nil { runtime.Error("Couldn't serve %s: %s", name, err) } }
func main() { // Define the options for the command line and config file options parser. opts := optparse.New( "Usage: planfile <config.yaml> [options]\n", "planfile 0.0.1") cookieKeyFile := opts.StringConfig("cookie-key-file", "cookie.key", "the file containing the key to sign cookie values [cookie.key]") gaHost := opts.StringConfig("ga-host", "", "the google analytics hostname to use") gaID := opts.StringConfig("ga-id", "", "the google analytics id to use") httpAddr := opts.StringConfig("http-addr", ":8888", "the address to bind the http server [:8888]") oauthID := opts.Required().StringConfig("oauth-id", "", "the oauth client id for github") oauthSecret := opts.Required().StringConfig("oauth-secret", "", "the oauth client secret for github") redirectURL := opts.StringConfig("redirect-url", "/.oauth", "the redirect url for handling oauth [/.oauth]") repository := opts.Required().StringConfig("repository", "", "the username/repository on github") secureMode := opts.BoolConfig("secure-mode", "enable hsts and secure cookies [false]") title := opts.StringConfig("title", "Planfile", "the title for the web app [Planfile]") refreshKey := opts.StringConfig("refresh-key", "", "key for anonymously calling refresh at /.refresh?key=<refresh-key>") refreshOpt := opts.IntConfig("refresh-interval", 1, "the number of through-the-web edits before a full refresh [1]") debug, instanceDirectory, _, logPath, _ = runtime.DefaultOpts("planfile", opts, os.Args, true) service := &oauth.OAuthService{ ClientID: *oauthID, ClientSecret: *oauthSecret, Scope: "public_repo", AuthURL: "https://github.com/login/oauth/authorize", TokenURL: "https://github.com/login/oauth/access_token", RedirectURL: *redirectURL, AcceptHeader: "application/json", } assets := map[string]string{} json.Unmarshal(readFile("assets.json"), &assets) setupPygments() mutex := sync.RWMutex{} repo := &Repo{Path: *repository} err := repo.Load(callGithubAnon) if err != nil { runtime.Exit(1) } repo.Title = *title repo.Updated = time.Now().UTC() repoJSON, err := json.Marshal(repo) if err != nil { runtime.StandardError(err) } refreshCount := 0 refreshInterval := *refreshOpt refreshKeySet := *refreshKey != "" refreshKeyBytes := []byte(*refreshKey) secret := readFile(*cookieKeyFile) newContext := func(w http.ResponseWriter, r *http.Request) *Context { return &Context{ r: r, w: w, secret: secret, secure: *secureMode, } } register := func(path string, handler func(*Context), usegzip ...bool) { gzippable := len(usegzip) > 0 && usegzip[0] http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { log.Info("serving %s", r.URL) w.Header().Set("Content-Type", "text/html; charset=utf-8") if gzippable && httputil.Parse(r, "Accept-Encoding").Accepts("gzip") { buf := &bytes.Buffer{} enc := gzip.NewWriter(buf) handler(newContext(GzipWriter{enc, w}, r)) enc.Close() w.Header().Set("Content-Encoding", "gzip") w.Header().Set("Content-Length", strconv.Itoa(buf.Len())) buf.WriteTo(w) } else { handler(newContext(w, r)) } }) } anon := []byte(", null, null, '', false") authFalse := []byte("', false") authTrue := []byte("', true") header := []byte(`<!doctype html> <meta charset=utf-8> <title>` + html.EscapeString(*title) + `</title> <link href="//fonts.googleapis.com/css?family=Abel|Coustard:400" rel=stylesheet> <link href=/.static/` + assets["planfile.css"] + ` rel=stylesheet> <body><script>DATA = ['` + *gaHost + `', '` + *gaID + `', `) footer := []byte(`];</script> <script src=/.static/` + assets["planfile.js"] + `></script> <noscript>Sorry, your browser needs <a href=http://enable-javascript.com>JavaScript enabled</a>.</noscript> `) register("/", func(ctx *Context) { mutex.RLock() defer mutex.RUnlock() ctx.Write(header) ctx.Write(repoJSON) avatar := ctx.GetCookie("avatar") user := ctx.GetCookie("user") if avatar != "" && user != "" { ctx.Write([]byte(", '" + user + "', '" + avatar + "', '" + ctx.GetCookie("xsrf"))) if ctx.IsAuthorised(repo) { ctx.Write(authTrue) } else { ctx.Write(authFalse) } } else { ctx.Write(anon) } ctx.Write(footer) }, true) register("/.api", func(ctx *Context) { mutex.RLock() defer mutex.RUnlock() if cb := ctx.FormValue("callback"); cb != "" { ctx.Write([]byte(cb)) ctx.Write([]byte{'('}) ctx.Write(repoJSON) ctx.Write([]byte{')', ';'}) } else { ctx.Write(repoJSON) } }, true) register("/.login", func(ctx *Context) { b := make([]byte, 20) if n, err := rand.Read(b); err != nil || n != 20 { ctx.Error("Couldn't access cryptographic device", err) return } s := hex.EncodeToString(b) ctx.SetCookie("xsrf", s) ctx.Redirect(service.AuthCodeURL(s)) }) register("/.logout", func(ctx *Context) { ctx.ExpireCookie("auth") ctx.ExpireCookie("avatar") ctx.ExpireCookie("token") ctx.ExpireCookie("user") ctx.ExpireCookie("xsrf") ctx.Redirect("/") }) notAuthorised := []byte("ERROR: Not Authorised!") savedHeader := []byte(`<!doctype html> <meta charset=utf-8> <title>` + html.EscapeString(*title) + `</title> <body><script>SAVED="`) savedFooter := []byte(`"</script><script src=/.static/` + assets["planfile.js"] + `></script>`) exportRepo := func(ctx *Context) bool { repo.Updated = time.Now().UTC() repoJSON, err = json.Marshal(repo) if err != nil { ctx.Error("Couldn't encode repo data during refresh", err) return false } return true } refresh := func(ctx *Context) { err := repo.Load(ctx.CreateCallGithub()) if err != nil { log.Error("couldn't rebuild planfile info: %s", err) ctx.Write([]byte("ERROR: " + err.Error())) return } exportRepo(ctx) } saveItem := func(ctx *Context, update bool) { mutex.Lock() defer mutex.Unlock() if !ctx.IsAuthorised(repo) { ctx.Write(notAuthorised) return } if !isEqual([]byte(ctx.FormValue("xsrf")), []byte(ctx.GetCookie("xsrf"))) { ctx.Write(notAuthorised) return } callGithub := ctx.CreateCallGithub() err := repo.UpdateInfo(callGithub) if err != nil { ctx.Error("Couldn't update repo info", err) return } var id, path, message string if update { id = ctx.FormValue("id") path = ctx.FormValue("path") } else { baseID := ctx.FormValue("id") id = baseID count := 0 for repo.Exists(id + ".md") { count += 1 id = fmt.Sprintf("%s%d", baseID, count) } path = id + ".md" } content := strings.Replace(ctx.FormValue("content"), "\r\n", "\n", -1) tags := ctx.FormValue("tags") title := ctx.FormValue("title") redir := "/" if ctx.FormValue("summary") == "yes" { if id != "/" { content = fmt.Sprintf(`--- title: %s --- %s`, title, content) if strings.HasPrefix(id, "summary.") { redir = "/" + id[8:] } else { // Shouldn't ever happen. But just in case... redir = "/" + id } } } else { redir = "/.item." + id content = fmt.Sprintf(`--- id: %s tags: %s title: %s --- %s`, id, tags, title, content) } if title == "" { title = id } if update { message = "update: " + title + "." } else { message = "add: " + title + "." } log.Info("SAVE PATH: %q for %q", path, title) err = repo.Modify(ctx, path, content, message) if err != nil { if update { ctx.Error("<a href='/.refresh'>Try refreshing.</a> Couldn't update item", err) } else { ctx.Error("<a href='/.refresh'>Try refreshing.</a> Couldn't save new item", err) } return } refreshCount++ if refreshCount%refreshInterval == 0 { refresh(ctx) } else { repo.AddPlanfile(path, []byte(content), callGithub) if !exportRepo(ctx) { return } } ctx.Write(savedHeader) ctx.Write([]byte(html.EscapeString(redir))) ctx.Write(savedFooter) } register("/.modify", func(ctx *Context) { saveItem(ctx, true) }) register("/.new", func(ctx *Context) { saveItem(ctx, false) }) register("/.oauth", func(ctx *Context) { s := ctx.FormValue("state") if s == "" { ctx.Redirect("/.login") return } if !isEqual([]byte(s), []byte(ctx.GetCookie("xsrf"))) { ctx.ExpireCookie("xsrf") ctx.Redirect("/.login") return } t := &oauth.Transport{OAuthService: service} tok, err := t.ExchangeAuthorizationCode(ctx.FormValue("code")) if err != nil { ctx.Error("Auth Exchange Error", err) return } jtok, err := json.Marshal(tok) if err != nil { ctx.Error("Couldn't encode token", err) return } ctx.SetCookie("token", hex.EncodeToString(jtok)) ctx.token = tok user := &User{} err = ctx.Call("/user", user, nil, false) if err != nil { ctx.Error("Couldn't load user info", err) return } ctx.SetCookie("avatar", user.AvatarURL) ctx.SetCookie("user", user.Login) ctx.Redirect("/") }) register("/.preview", func(ctx *Context) { rendered, err := renderMarkdown([]byte(ctx.FormValue("content"))) if err != nil { ctx.Error("Couldn't render Markdown", err) return } ctx.Write(rendered) }, true) register("/.refresh", func(ctx *Context) { if !ctx.IsAuthorised(repo) { if !(refreshKeySet && isEqual(refreshKeyBytes, []byte(ctx.FormValue("key")))) { ctx.Write(notAuthorised) return } } mutex.Lock() defer mutex.Unlock() refresh(ctx) ctx.Redirect("/") }) mimetypes := map[string]string{ "css": "text/css", "gif": "image/gif", "ico": "image/x-icon", "jpeg": "image/jpeg", "jpg": "image/jpeg", "js": "text/javascript", "png": "image/png", "swf": "application/x-shockwave-flash", "txt": "text/plain", } registerStatic := func(filepath, urlpath string) { _, ext := rsplit(filepath, ".") ctype, ok := mimetypes[ext] if !ok { ctype = "application/octet-stream" } if debug { register(urlpath, func(ctx *Context) { ctx.SetHeader("Content-Type", ctype) ctx.Write(readFile(filepath)) }, strings.HasPrefix(ctype, "text/")) } else { content := readFile(filepath) register(urlpath, func(ctx *Context) { ctx.SetHeader("Cache-Control", "public, max-age=86400") ctx.SetHeader("Content-Type", ctype) ctx.Write(content) }, strings.HasPrefix(ctype, "text/")) } } for _, path := range assets { registerStatic(filepath.Join(instanceDirectory, "static", path), "/.static/"+path) } wwwPath := filepath.Join(instanceDirectory, "www") if files, err := ioutil.ReadDir(wwwPath); err == nil { for _, file := range files { if !file.IsDir() { registerStatic(filepath.Join(wwwPath, file.Name()), "/"+file.Name()) } } } log.Info("Listening on %s", *httpAddr) server := &http.Server{ Addr: *httpAddr, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, } err = server.ListenAndServe() if err != nil { runtime.Error("couldn't bind to tcp socket: %s", err) } }