Beispiel #1
0
Datei: tasks.go Projekt: 40a/cbfs
func reloadConfig() {
	for _ = range time.Tick(time.Minute) {
		if err := updateConfig(); err != nil && !gomemcached.IsNotFound(err) {
			log.Printf("Error updating config: %v", err)
		}
	}
}
Beispiel #2
0
func (c *CfgCB) unlockedLoad() error {
	bucket, err := c.getBucket()
	if err != nil {
		return err
	}

	buf, err := bucket.GetRaw(c.cfgKey)
	if err != nil && !gomemcached.IsNotFound(err) {
		return err
	}

	cfgMem := NewCfgMem()
	if buf != nil {
		err = json.Unmarshal(buf, cfgMem)
		if err != nil {
			return err
		}
	}

	cfgMemPrev := c.cfgMem
	cfgMemPrev.m.Lock()
	defer cfgMemPrev.m.Unlock()

	cfgMem.subscriptions = cfgMemPrev.subscriptions

	c.cfgMem = cfgMem

	return nil
}
Beispiel #3
0
func (c *CfgCB) Get(key string, cas uint64) (
	[]byte, uint64, error) {
	c.logger("cfg_cb: Get, key: %s, cas: %d", key, cas)

	c.m.Lock()
	defer c.m.Unlock()

	bucket, err := c.getBucket()
	if err != nil {
		c.logger("cfg_cb: Get, key: %s, cas: %d, getBucket, err: %v", key, cas, err)

		return nil, 0, err
	}

	cfgBuf, _, cfgCAS, err := bucket.GetsRaw(c.cfgKey)
	if err != nil && !gomemcached.IsNotFound(err) {
		c.logger("cfg_cb: Get, key: %s, cas: %d, GetsRaw, err: %v", key, cas, err)

		return nil, 0, err
	}

	if cas != 0 && cas != cfgCAS {
		c.logger("cfg_cb: Get, key: %s, cas: %d, CASError, cfgCAS: %d", key, cas, cfgCAS)

		return nil, 0, &CfgCASError{}
	}

	cfgMem := NewCfgMem()
	if cfgBuf != nil {
		c.logger("cfg_cb: Get, key: %s, cas: %d, len(cfgBuf): %d", key, cas, len(cfgBuf))

		err = json.Unmarshal(cfgBuf, cfgMem)
		if err != nil {
			c.logger("cfg_cb: Get, key: %s, cas: %d, JSONError, err: %v", key, cas, err)

			return nil, 0, err
		}
	} else {
		c.logger("cfg_cb: Get, key: %s, cas: %d, cfgBuf nil", key, cas)
	}

	val, _, err := cfgMem.Get(key, 0)
	if err != nil {
		c.logger("cfg_cb: Get, key: %s, cas: %d, cfgMem.Get() err: %v", key, cas, err)

		return nil, 0, err
	}

	c.logger("cfg_cb: Get, key: %s, cas: %d, cfgCAS: %d, val: %s", key, cas, cfgCAS, val)

	return val, cfgCAS, nil
}
Beispiel #4
0
func doGetConfig(w http.ResponseWriter, req *http.Request) {
	err := updateConfig()
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Error updating config: %v", err)
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(200)

	e := json.NewEncoder(w)
	err = e.Encode(&globalConfig)
	if err != nil {
		log.Printf("Error sending config: %v", err)
	}
}
Beispiel #5
0
func doGetBackupInfo(w http.ResponseWriter, req *http.Request) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil {
		code := 500
		if gomemcached.IsNotFound(err) {
			code = 404
		}
		http.Error(w, err.Error(), code)
		return
	}

	removeDeadBackups(&b)

	sendJson(w, req, &b)
}
Beispiel #6
0
func removeDeadBackups(b *backups) {
	// Keep only backups we're pretty sure still exist.
	obn := b.Backups
	b.Backups = nil
	for _, bi := range obn {
		fm := fileMeta{}
		err := couchbase.Get(shortName(bi.Fn), &fm)
		if gomemcached.IsNotFound(err) {
			log.Printf("Dropping previous (deleted) backup: %v",
				bi.Fn)
		} else {
			b.Backups = append(b.Backups, bi)
		}
	}

}
Beispiel #7
0
func storeBackupObject(fn, h string) error {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Weird: %v", err)
		// return err
	}

	removeDeadBackups(&b)

	ob := backupItem{fn, h, time.Now().UTC(), *globalConfig}

	b.Latest = ob
	b.Backups = append(b.Backups, ob)

	return couchbase.Set(backupKey, 0, &b)
}
Beispiel #8
0
func doFileInfo(w http.ResponseWriter, req *http.Request, fn string) {
	fm := fileMeta{}
	err := couchbase.Get(shortName(fn), &fm)
	switch {
	case err == nil:
	case gomemcached.IsNotFound(err):
		http.Error(w, "not found", 404)
		return
	default:
		http.Error(w, err.Error(), 500)
		return
	}
	sendJson(w, req, map[string]interface{}{
		"path": fn,
		"meta": fm,
	})
}
Beispiel #9
0
Datei: http.go Projekt: 40a/cbfs
func doLinkFile(w http.ResponseWriter, req *http.Request) {
	fn := req.URL.Path
	h := req.FormValue("blob")
	t := req.FormValue("type")

	for len(fn) > 0 && fn[0] == '/' {
		fn = fn[1:]
	}

	blob, err := referenceBlob(h)
	if err != nil {
		estat := 500
		if gomemcached.IsNotFound(err) {
			estat = 404
		}
		http.Error(w, err.Error(), estat)
	}

	fm := fileMeta{
		Headers: http.Header{
			"Content-Type": []string{t},
		},
		OID:      h,
		Length:   blob.Length,
		Modified: time.Now().UTC(),
	}

	exp := getExpiration(req.Header)
	if exp != 0 {
		fm.Headers.Set("X-CBFS-Expiration", strconv.Itoa(exp))
	}

	err = maybeStoreMeta(fn, fm, exp, true)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.WriteHeader(201)
}
Beispiel #10
0
func (c *CfgCB) Del(key string, cas uint64) error {
	c.m.Lock()
	defer c.m.Unlock()

	bucket, err := c.getBucket()
	if err != nil {
		c.logger("cfg_cb: Del, key: %s, cas: %d, getBucket, err: %v", key, cas, err)

		return err
	}

	cfgBuf, _, cfgCAS, err := bucket.GetsRaw(c.cfgKey)
	if err != nil && !gomemcached.IsNotFound(err) {
		c.logger("cfg_cb: Del, key: %s, cas: %d, GetsRaw, err: %v", key, cas, err)

		return err
	}

	if cas != 0 && cas != cfgCAS {
		c.logger("cfg_cb: Del, key: %s, cas: %d, CASError, cfgCAS: %d", key, cas, cfgCAS)

		return &CfgCASError{}
	}

	cfgMem := NewCfgMem()
	if cfgBuf != nil {
		c.logger("cfg_cb: Del, key: %s, cas: %d, len(cfgBuf): %d", key, cas, len(cfgBuf))

		err = json.Unmarshal(cfgBuf, cfgMem)
		if err != nil {
			c.logger("cfg_cb: Del, key: %s, cas: %d, JSONError, err: %v", key, cas, err)

			return err
		}
	} else {
		c.logger("cfg_cb: Del, key: %s, cas: %d, cfgBuf nil", key, cas)
	}

	err = cfgMem.Del(key, 0)
	if err != nil {
		c.logger("cfg_cb: Del, key: %s, cas: %d, cfgMem.Del() err: %v", key, cas, err)

		return err
	}

	nextCAS, err := bucket.Cas(c.cfgKey, 0, cfgCAS, cfgMem)
	if err != nil {
		c.logger("cfg_cb: Del, key: %s, cas: %d, cfgCAS: %d, Cas err: %v",
			key, cas, cfgCAS, err)

		if res, ok := err.(*gomemcached.MCResponse); ok {
			if res.Status == gomemcached.KEY_EEXISTS {
				c.logger("cfg_cb: Del, key: %s, cas: %d, cfgCAS: %d, Cas KEY_EEXISTS",
					key, cas, cfgCAS)

				return &CfgCASError{}
			}
		}
		return err
	}

	c.logger("cfg_cb: Del, key: %s, cas: %d, cfgCAS: %d, nextCAS: %d",
		key, cas, cfgCAS, nextCAS)

	c.fireEvent(key, nextCAS, nil)

	return err
}
Beispiel #11
0
func loadExistingHashes() (*hashset.Hashset, error) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		return nil, err
	}

	oids := make(chan string)
	hsch := make(chan *hashset.Hashset)
	visitch := make(chan int)
	errch := make(chan error)

	wg := sync.WaitGroup{}

	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for o := range oids {
				h, v, e := loadBackupHashes(o)
				if e == nil {
					hsch <- h
				} else {
					errch <- e
				}
				visitch <- v
			}
		}()
	}
	go func() {
		wg.Wait()
		close(hsch)
		close(visitch)
		close(errch)
	}()

	go func() {
		for _, i := range b.Backups {
			log.Printf("Loading backups from %v / %v", i.Oid, i.Fn)
			oids <- i.Oid
		}
		close(oids)
	}()

	visited := 0
	hs := &hashset.Hashset{}
	for {
		// Done getting all the things
		if hsch == nil && visitch == nil && errch == nil {
			break
		}
		select {
		case v, ok := <-visitch:
			visited += v
			if !ok {
				visitch = nil
			}
		case e, ok := <-errch:
			err = e
			if !ok {
				errch = nil
			}
		case h, ok := <-hsch:
			if ok {
				log.Printf("Got %v hashes from a backup",
					h.Len())
				hs.AddAll(h)
			} else {
				hsch = nil
			}
		}
	}

	log.Printf("Visited %v obs, kept %v", visited, hs.Len())

	return hs, err
}
Beispiel #12
0
Datei: main.go Projekt: 40a/cbfs
func main() {
	flag.Parse()

	rand.Seed(time.Now().UnixNano())

	initLogger(*useSyslog)
	initNodeListKeys()

	http.DefaultTransport = TimeoutTransport(*internodeTimeout)
	expvar.Publish("httpclients", httputil.InitHTTPTracker(false))

	if getHash() == nil {
		fmt.Fprintf(os.Stderr,
			"Unsupported hash specified: %v.  Supported hashes:\n",
			globalConfig.Hash)
		for h := range hashBuilders {
			fmt.Fprintf(os.Stderr, " * %v\n", h)
		}
		os.Exit(1)
	}

	err := initServerId()
	if err != nil {
		log.Fatalf("Error initializing server ID: %v", err)
	}

	if *maxStorageString != "" {
		ms, err := humanize.ParseBytes(*maxStorageString)
		if err != nil {
			log.Fatalf("Error parsing max storage parameter: %v",
				err)
		}
		maxStorage = int64(ms)
	}

	couchbase, err = dbConnect()
	if err != nil {
		log.Fatalf("Can't connect to couchbase: %v", err)
	}

	if err = os.MkdirAll(*root, 0777); err != nil {
		log.Fatalf("Couldn't create storage dir: %v", err)
	}

	err = updateConfig()
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Error updating initial config, using default: %v",
			err)
	}
	if *verbose {
		log.Printf("Server config:")
		globalConfig.Dump(os.Stdout)
	}

	go reloadConfig()

	go dnsServices()

	internodeTaskQueue = make(chan internodeTask, *taskWorkers*1024)
	initTaskQueueWorkers()

	go heartbeat()
	go startTasks()

	time.AfterFunc(time.Second*time.Duration(rand.Intn(30)+5), grabSomeData)

	go serveFrame()

	s := &http.Server{
		Addr:        *bindAddr,
		Handler:     http.HandlerFunc(httpHandler),
		ReadTimeout: *readTimeout,
	}
	log.Printf("Listening to web requests on %s as server %s",
		*bindAddr, serverId)

	l, err := rateListen("tcp", *bindAddr)
	if err != nil {
		log.Fatalf("Error listening: %v", err)
	}
	log.Fatal(s.Serve(l))
}