func reloadConfig() { for _ = range time.Tick(time.Minute) { if err := updateConfig(); err != nil && !gomemcached.IsNotFound(err) { log.Printf("Error updating config: %v", err) } } }
func (c *CfgCB) unlockedLoad() error { bucket, err := c.getBucket() if err != nil { return err } buf, err := bucket.GetRaw(c.cfgKey) if err != nil && !gomemcached.IsNotFound(err) { return err } cfgMem := NewCfgMem() if buf != nil { err = json.Unmarshal(buf, cfgMem) if err != nil { return err } } cfgMemPrev := c.cfgMem cfgMemPrev.m.Lock() defer cfgMemPrev.m.Unlock() cfgMem.subscriptions = cfgMemPrev.subscriptions c.cfgMem = cfgMem return nil }
func (c *CfgCB) Get(key string, cas uint64) ( []byte, uint64, error) { c.logger("cfg_cb: Get, key: %s, cas: %d", key, cas) c.m.Lock() defer c.m.Unlock() bucket, err := c.getBucket() if err != nil { c.logger("cfg_cb: Get, key: %s, cas: %d, getBucket, err: %v", key, cas, err) return nil, 0, err } cfgBuf, _, cfgCAS, err := bucket.GetsRaw(c.cfgKey) if err != nil && !gomemcached.IsNotFound(err) { c.logger("cfg_cb: Get, key: %s, cas: %d, GetsRaw, err: %v", key, cas, err) return nil, 0, err } if cas != 0 && cas != cfgCAS { c.logger("cfg_cb: Get, key: %s, cas: %d, CASError, cfgCAS: %d", key, cas, cfgCAS) return nil, 0, &CfgCASError{} } cfgMem := NewCfgMem() if cfgBuf != nil { c.logger("cfg_cb: Get, key: %s, cas: %d, len(cfgBuf): %d", key, cas, len(cfgBuf)) err = json.Unmarshal(cfgBuf, cfgMem) if err != nil { c.logger("cfg_cb: Get, key: %s, cas: %d, JSONError, err: %v", key, cas, err) return nil, 0, err } } else { c.logger("cfg_cb: Get, key: %s, cas: %d, cfgBuf nil", key, cas) } val, _, err := cfgMem.Get(key, 0) if err != nil { c.logger("cfg_cb: Get, key: %s, cas: %d, cfgMem.Get() err: %v", key, cas, err) return nil, 0, err } c.logger("cfg_cb: Get, key: %s, cas: %d, cfgCAS: %d, val: %s", key, cas, cfgCAS, val) return val, cfgCAS, nil }
func doGetConfig(w http.ResponseWriter, req *http.Request) { err := updateConfig() if err != nil && !gomemcached.IsNotFound(err) { log.Printf("Error updating config: %v", err) } w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(200) e := json.NewEncoder(w) err = e.Encode(&globalConfig) if err != nil { log.Printf("Error sending config: %v", err) } }
func doGetBackupInfo(w http.ResponseWriter, req *http.Request) { b := backups{} err := couchbase.Get(backupKey, &b) if err != nil { code := 500 if gomemcached.IsNotFound(err) { code = 404 } http.Error(w, err.Error(), code) return } removeDeadBackups(&b) sendJson(w, req, &b) }
func removeDeadBackups(b *backups) { // Keep only backups we're pretty sure still exist. obn := b.Backups b.Backups = nil for _, bi := range obn { fm := fileMeta{} err := couchbase.Get(shortName(bi.Fn), &fm) if gomemcached.IsNotFound(err) { log.Printf("Dropping previous (deleted) backup: %v", bi.Fn) } else { b.Backups = append(b.Backups, bi) } } }
func storeBackupObject(fn, h string) error { b := backups{} err := couchbase.Get(backupKey, &b) if err != nil && !gomemcached.IsNotFound(err) { log.Printf("Weird: %v", err) // return err } removeDeadBackups(&b) ob := backupItem{fn, h, time.Now().UTC(), *globalConfig} b.Latest = ob b.Backups = append(b.Backups, ob) return couchbase.Set(backupKey, 0, &b) }
func doFileInfo(w http.ResponseWriter, req *http.Request, fn string) { fm := fileMeta{} err := couchbase.Get(shortName(fn), &fm) switch { case err == nil: case gomemcached.IsNotFound(err): http.Error(w, "not found", 404) return default: http.Error(w, err.Error(), 500) return } sendJson(w, req, map[string]interface{}{ "path": fn, "meta": fm, }) }
func doLinkFile(w http.ResponseWriter, req *http.Request) { fn := req.URL.Path h := req.FormValue("blob") t := req.FormValue("type") for len(fn) > 0 && fn[0] == '/' { fn = fn[1:] } blob, err := referenceBlob(h) if err != nil { estat := 500 if gomemcached.IsNotFound(err) { estat = 404 } http.Error(w, err.Error(), estat) } fm := fileMeta{ Headers: http.Header{ "Content-Type": []string{t}, }, OID: h, Length: blob.Length, Modified: time.Now().UTC(), } exp := getExpiration(req.Header) if exp != 0 { fm.Headers.Set("X-CBFS-Expiration", strconv.Itoa(exp)) } err = maybeStoreMeta(fn, fm, exp, true) if err != nil { http.Error(w, err.Error(), 500) return } w.WriteHeader(201) }
func (c *CfgCB) Del(key string, cas uint64) error { c.m.Lock() defer c.m.Unlock() bucket, err := c.getBucket() if err != nil { c.logger("cfg_cb: Del, key: %s, cas: %d, getBucket, err: %v", key, cas, err) return err } cfgBuf, _, cfgCAS, err := bucket.GetsRaw(c.cfgKey) if err != nil && !gomemcached.IsNotFound(err) { c.logger("cfg_cb: Del, key: %s, cas: %d, GetsRaw, err: %v", key, cas, err) return err } if cas != 0 && cas != cfgCAS { c.logger("cfg_cb: Del, key: %s, cas: %d, CASError, cfgCAS: %d", key, cas, cfgCAS) return &CfgCASError{} } cfgMem := NewCfgMem() if cfgBuf != nil { c.logger("cfg_cb: Del, key: %s, cas: %d, len(cfgBuf): %d", key, cas, len(cfgBuf)) err = json.Unmarshal(cfgBuf, cfgMem) if err != nil { c.logger("cfg_cb: Del, key: %s, cas: %d, JSONError, err: %v", key, cas, err) return err } } else { c.logger("cfg_cb: Del, key: %s, cas: %d, cfgBuf nil", key, cas) } err = cfgMem.Del(key, 0) if err != nil { c.logger("cfg_cb: Del, key: %s, cas: %d, cfgMem.Del() err: %v", key, cas, err) return err } nextCAS, err := bucket.Cas(c.cfgKey, 0, cfgCAS, cfgMem) if err != nil { c.logger("cfg_cb: Del, key: %s, cas: %d, cfgCAS: %d, Cas err: %v", key, cas, cfgCAS, err) if res, ok := err.(*gomemcached.MCResponse); ok { if res.Status == gomemcached.KEY_EEXISTS { c.logger("cfg_cb: Del, key: %s, cas: %d, cfgCAS: %d, Cas KEY_EEXISTS", key, cas, cfgCAS) return &CfgCASError{} } } return err } c.logger("cfg_cb: Del, key: %s, cas: %d, cfgCAS: %d, nextCAS: %d", key, cas, cfgCAS, nextCAS) c.fireEvent(key, nextCAS, nil) return err }
func loadExistingHashes() (*hashset.Hashset, error) { b := backups{} err := couchbase.Get(backupKey, &b) if err != nil && !gomemcached.IsNotFound(err) { return nil, err } oids := make(chan string) hsch := make(chan *hashset.Hashset) visitch := make(chan int) errch := make(chan error) wg := sync.WaitGroup{} for i := 0; i < 4; i++ { wg.Add(1) go func() { defer wg.Done() for o := range oids { h, v, e := loadBackupHashes(o) if e == nil { hsch <- h } else { errch <- e } visitch <- v } }() } go func() { wg.Wait() close(hsch) close(visitch) close(errch) }() go func() { for _, i := range b.Backups { log.Printf("Loading backups from %v / %v", i.Oid, i.Fn) oids <- i.Oid } close(oids) }() visited := 0 hs := &hashset.Hashset{} for { // Done getting all the things if hsch == nil && visitch == nil && errch == nil { break } select { case v, ok := <-visitch: visited += v if !ok { visitch = nil } case e, ok := <-errch: err = e if !ok { errch = nil } case h, ok := <-hsch: if ok { log.Printf("Got %v hashes from a backup", h.Len()) hs.AddAll(h) } else { hsch = nil } } } log.Printf("Visited %v obs, kept %v", visited, hs.Len()) return hs, err }
func main() { flag.Parse() rand.Seed(time.Now().UnixNano()) initLogger(*useSyslog) initNodeListKeys() http.DefaultTransport = TimeoutTransport(*internodeTimeout) expvar.Publish("httpclients", httputil.InitHTTPTracker(false)) if getHash() == nil { fmt.Fprintf(os.Stderr, "Unsupported hash specified: %v. Supported hashes:\n", globalConfig.Hash) for h := range hashBuilders { fmt.Fprintf(os.Stderr, " * %v\n", h) } os.Exit(1) } err := initServerId() if err != nil { log.Fatalf("Error initializing server ID: %v", err) } if *maxStorageString != "" { ms, err := humanize.ParseBytes(*maxStorageString) if err != nil { log.Fatalf("Error parsing max storage parameter: %v", err) } maxStorage = int64(ms) } couchbase, err = dbConnect() if err != nil { log.Fatalf("Can't connect to couchbase: %v", err) } if err = os.MkdirAll(*root, 0777); err != nil { log.Fatalf("Couldn't create storage dir: %v", err) } err = updateConfig() if err != nil && !gomemcached.IsNotFound(err) { log.Printf("Error updating initial config, using default: %v", err) } if *verbose { log.Printf("Server config:") globalConfig.Dump(os.Stdout) } go reloadConfig() go dnsServices() internodeTaskQueue = make(chan internodeTask, *taskWorkers*1024) initTaskQueueWorkers() go heartbeat() go startTasks() time.AfterFunc(time.Second*time.Duration(rand.Intn(30)+5), grabSomeData) go serveFrame() s := &http.Server{ Addr: *bindAddr, Handler: http.HandlerFunc(httpHandler), ReadTimeout: *readTimeout, } log.Printf("Listening to web requests on %s as server %s", *bindAddr, serverId) l, err := rateListen("tcp", *bindAddr) if err != nil { log.Fatalf("Error listening: %v", err) } log.Fatal(s.Serve(l)) }