func reloadConfig() { for _ = range time.Tick(time.Minute) { if err := updateConfig(); err != nil && !gomemcached.IsNotFound(err) { log.Printf("Error updating config: %v", err) } } }
func sendTagNotification(bugid, tagName, actor string) { b, err := getBug(bugid) if err != nil { log.Printf("Error getting bug %v for tag notification: %v", bugid, err) return } tag := Tag{} err = db.Get("tag-"+tagName, &tag) if err != nil { if !gomemcached.IsNotFound(err) { log.Printf("Error sending notification for tag %v: %v", tagName, err) } return } to := filterUnprivelegedEmails(b, removeFromList(tag.Subscribers, actor)) sendNotifications("tag_notification", to, map[string]interface{}{ "Bug": b, "Tag": tagName, "Actor": actor, }) }
func errorCode(err error) int { switch { case err == bugNotVisible: return 401 case gomemcached.IsNotFound(err): return 404 } return 500 }
// Send a custom request and get the response. func (client *Client) Send(req *gomemcached.MCRequest) (rv *gomemcached.MCResponse, err error) { err = transmitRequest(client.conn, req) if err != nil { client.healthy = false return } resp, err := getResponse(client.conn, client.hdrBuf) if err != nil && !gomemcached.IsNotFound(err) { client.healthy = false } return resp, err }
func doGetConfig(w http.ResponseWriter, req *http.Request) { err := updateConfig() if err != nil && !gomemcached.IsNotFound(err) { log.Printf("Error updating config: %v", err) } w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(200) e := json.NewEncoder(w) err = e.Encode(&globalConfig) if err != nil { log.Printf("Error sending config: %v", err) } }
func doGetBackupInfo(w http.ResponseWriter, req *http.Request) { b := backups{} err := couchbase.Get(backupKey, &b) if err != nil { code := 500 if gomemcached.IsNotFound(err) { code = 404 } http.Error(w, err.Error(), code) return } removeDeadBackups(&b) sendJson(w, req, &b) }
func removeDeadBackups(b *backups) { // Keep only backups we're pretty sure still exist. obn := b.Backups b.Backups = nil for _, bi := range obn { fm := fileMeta{} err := couchbase.Get(shortName(bi.Fn), &fm) if gomemcached.IsNotFound(err) { log.Printf("Dropping previous (deleted) backup: %v", bi.Fn) } else { b.Backups = append(b.Backups, bi) } } }
func storeBackupObject(fn, h string) error { b := backups{} err := couchbase.Get(backupKey, &b) if err != nil && !gomemcached.IsNotFound(err) { log.Printf("Weird: %v", err) // return err } removeDeadBackups(&b) ob := backupItem{fn, h, time.Now().UTC(), *globalConfig} b.Latest = ob b.Backups = append(b.Backups, ob) return couchbase.Set(backupKey, 0, &b) }
func doFileInfo(w http.ResponseWriter, req *http.Request, fn string) { fm := fileMeta{} err := couchbase.Get(shortName(fn), &fm) switch { case err == nil: case gomemcached.IsNotFound(err): http.Error(w, "not found", 404) return default: http.Error(w, err.Error(), 500) return } sendJson(w, req, map[string]interface{}{ "path": fn, "meta": fm, }) }
func doLinkFile(w http.ResponseWriter, req *http.Request) { fn := req.URL.Path h := req.FormValue("blob") t := req.FormValue("type") for len(fn) > 0 && fn[0] == '/' { fn = fn[1:] } blob, err := referenceBlob(h) if err != nil { estat := 500 if gomemcached.IsNotFound(err) { estat = 404 } http.Error(w, err.Error(), estat) } fm := fileMeta{ Headers: http.Header{ "Content-Type": []string{t}, }, OID: h, Length: blob.Length, Modified: time.Now().UTC(), } exp := getExpiration(req.Header) if exp != 0 { fm.Headers.Set("X-CBFS-Expiration", strconv.Itoa(exp)) } err = maybeStoreMeta(fn, fm, exp, true) if err != nil { http.Error(w, err.Error(), 500) return } w.WriteHeader(201) }
func db_init() { var ddocVersion uint64 err := Bucket.Get(db_ddoc_version_key, &ddocVersion) if err != nil && !gomemcached.IsNotFound(err) { log.Fatalf("fatal error getting ddoc version from database: %v", err) } if wanted := uint64(1); ddocVersion != wanted { log.Printf("updating ddoc from version %d to %d", ddocVersion, wanted) err = Bucket.PutDDoc(db_ddoc_name, couchbase.DDocJSON{ Views: map[string]couchbase.ViewDefinition{ "discussion-posts": { Map: `function( doc, meta ) { if ( doc.Type == "post" ) { emit( doc.Discussion, null ); } }`, Reduce: `_count`, }, "updated-discussions": { Map: `function( doc, meta ) { if ( doc.Type == "discussion" ) { emit( dateToArray( doc.Modified ), null ); } }`, }, }, }) if err != nil { log.Fatalf("fatal error storing ddoc: %v", err) } err = Bucket.Set(db_ddoc_version_key, 0, wanted) if err != nil { log.Fatalf("fatal error storing ddoc version: %v", err) } } }
func main() { flag.Parse() rand.Seed(time.Now().UnixNano()) initLogger(*useSyslog) initNodeListKeys() http.DefaultTransport = TimeoutTransport(*internodeTimeout) expvar.Publish("httpclients", httputil.InitHTTPTracker(false)) if getHash() == nil { fmt.Fprintf(os.Stderr, "Unsupported hash specified: %v. Supported hashes:\n", globalConfig.Hash) for h := range hashBuilders { fmt.Fprintf(os.Stderr, " * %v\n", h) } os.Exit(1) } err := initServerId() if err != nil { log.Fatalf("Error initializing server ID: %v", err) } if *maxStorageString != "" { ms, err := humanize.ParseBytes(*maxStorageString) if err != nil { log.Fatalf("Error parsing max storage parameter: %v", err) } maxStorage = int64(ms) } couchbase, err = dbConnect() if err != nil { log.Fatalf("Can't connect to couchbase: %v", err) } if err = os.MkdirAll(*root, 0777); err != nil { log.Fatalf("Couldn't create storage dir: %v", err) } err = updateConfig() if err != nil && !gomemcached.IsNotFound(err) { log.Printf("Error updating initial config, using default: %v", err) } if *verbose { log.Printf("Server config:") globalConfig.Dump(os.Stdout) } go reloadConfig() go dnsServices() internodeTaskQueue = make(chan internodeTask, *taskWorkers*1024) initTaskQueueWorkers() go heartbeat() go startTasks() time.AfterFunc(time.Second*time.Duration(rand.Intn(30)+5), grabSomeData) go serveFrame() s := &http.Server{ Addr: *bindAddr, Handler: http.HandlerFunc(httpHandler), ReadTimeout: *readTimeout, } log.Printf("Listening to web requests on %s as server %s", *bindAddr, serverId) l, err := rateListen("tcp", *bindAddr) if err != nil { log.Fatalf("Error listening: %v", err) } log.Fatal(s.Serve(l)) }
func loadExistingHashes() (*hashset.Hashset, error) { b := backups{} err := couchbase.Get(backupKey, &b) if err != nil && !gomemcached.IsNotFound(err) { return nil, err } oids := make(chan string) hsch := make(chan *hashset.Hashset) visitch := make(chan int) errch := make(chan error) wg := sync.WaitGroup{} for i := 0; i < 4; i++ { wg.Add(1) go func() { defer wg.Done() for o := range oids { h, v, e := loadBackupHashes(o) if e == nil { hsch <- h } else { errch <- e } visitch <- v } }() } go func() { wg.Wait() close(hsch) close(visitch) close(errch) }() go func() { for _, i := range b.Backups { log.Printf("Loading backups from %v / %v", i.Oid, i.Fn) oids <- i.Oid } close(oids) }() visited := 0 hs := &hashset.Hashset{} for { // Done getting all the things if hsch == nil && visitch == nil && errch == nil { break } select { case v, ok := <-visitch: visited += v if !ok { visitch = nil } case e, ok := <-errch: err = e if !ok { errch = nil } case h, ok := <-hsch: if ok { log.Printf("Got %v hashes from a backup", h.Len()) hs.AddAll(h) } else { hsch = nil } } } log.Printf("Visited %v obs, kept %v", visited, hs.Len()) return hs, err }
func serveBugDeletion(w http.ResponseWriter, r *http.Request) { me := whoami(r) bugid := mux.Vars(r)["bugid"] _, err := getBugOrDisplayErr(bugid, me, w, r) if err != nil { return } cherr := make(chan error) deleted := make(chan couchbase.ViewRow) delatt := make(chan couchbase.ViewRow) wg := &sync.WaitGroup{} wg.Add(1) go deleteDocsMatching("cbugg", "comments", map[string]interface{}{ "stale": false, "reduce": false, "start_key": []interface{}{bugid}, "end_key": []interface{}{bugid, map[string]string{}}, }, deleted, cherr, wg) wg.Add(1) go deleteDocsMatching("cbugg", "attachments", map[string]interface{}{ "stale": false, "reduce": false, "start_key": []interface{}{bugid}, "end_key": []interface{}{bugid, map[string]string{}}, "include_docs": true, }, delatt, cherr, wg) wg.Add(1) go deleteDocsMatching("cbugg", "bug_history", map[string]interface{}{ "stale": false, "reduce": false, "start_key": []interface{}{bugid}, "end_key": []interface{}{bugid, map[string]string{}}, }, deleted, cherr, wg) go func() { wg.Wait() close(cherr) }() running := true for running { select { case del := <-deleted: log.Printf("Deleted %v", del.ID) case del := <-delatt: log.Printf("Deleted attachment %v", del.ID) mo := (*del.Doc).(map[string]interface{}) mi := mo["json"].(map[string]interface{}) u := mi["url"].(string) err := deleteAttachmentFile(u) if err != nil { log.Printf("Error deleting attachment %v: %v", u, err) } case err, ok := <-cherr: if !ok { running = false } if err != nil { log.Printf("Error: %v", err) } } } log.Printf("Done deleting children.") // This bug should already be deleted as a bug is also bug // history. But give it a go anyway for clarity. err = db.Delete(bugid) if err != nil && !gomemcached.IsNotFound(err) { log.Printf("Error deleting the bug.") } w.WriteHeader(204) }