Beispiel #1
0
func reloadConfig() {
	for _ = range time.Tick(time.Minute) {
		if err := updateConfig(); err != nil && !gomemcached.IsNotFound(err) {
			log.Printf("Error updating config: %v", err)
		}
	}
}
Beispiel #2
0
func sendTagNotification(bugid, tagName, actor string) {
	b, err := getBug(bugid)
	if err != nil {
		log.Printf("Error getting bug %v for tag notification: %v",
			bugid, err)
		return
	}

	tag := Tag{}
	err = db.Get("tag-"+tagName, &tag)
	if err != nil {
		if !gomemcached.IsNotFound(err) {
			log.Printf("Error sending notification for tag %v: %v",
				tagName, err)
		}
		return
	}

	to := filterUnprivelegedEmails(b, removeFromList(tag.Subscribers, actor))

	sendNotifications("tag_notification", to,
		map[string]interface{}{
			"Bug":   b,
			"Tag":   tagName,
			"Actor": actor,
		})
}
Beispiel #3
0
func errorCode(err error) int {
	switch {
	case err == bugNotVisible:
		return 401
	case gomemcached.IsNotFound(err):
		return 404
	}
	return 500
}
Beispiel #4
0
// Send a custom request and get the response.
func (client *Client) Send(req *gomemcached.MCRequest) (rv *gomemcached.MCResponse, err error) {
	err = transmitRequest(client.conn, req)
	if err != nil {
		client.healthy = false
		return
	}
	resp, err := getResponse(client.conn, client.hdrBuf)
	if err != nil && !gomemcached.IsNotFound(err) {
		client.healthy = false
	}
	return resp, err
}
Beispiel #5
0
func doGetConfig(w http.ResponseWriter, req *http.Request) {
	err := updateConfig()
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Error updating config: %v", err)
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(200)

	e := json.NewEncoder(w)
	err = e.Encode(&globalConfig)
	if err != nil {
		log.Printf("Error sending config: %v", err)
	}
}
Beispiel #6
0
func doGetBackupInfo(w http.ResponseWriter, req *http.Request) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil {
		code := 500
		if gomemcached.IsNotFound(err) {
			code = 404
		}
		http.Error(w, err.Error(), code)
		return
	}

	removeDeadBackups(&b)

	sendJson(w, req, &b)
}
Beispiel #7
0
func removeDeadBackups(b *backups) {
	// Keep only backups we're pretty sure still exist.
	obn := b.Backups
	b.Backups = nil
	for _, bi := range obn {
		fm := fileMeta{}
		err := couchbase.Get(shortName(bi.Fn), &fm)
		if gomemcached.IsNotFound(err) {
			log.Printf("Dropping previous (deleted) backup: %v",
				bi.Fn)
		} else {
			b.Backups = append(b.Backups, bi)
		}
	}

}
Beispiel #8
0
func storeBackupObject(fn, h string) error {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Weird: %v", err)
		// return err
	}

	removeDeadBackups(&b)

	ob := backupItem{fn, h, time.Now().UTC(), *globalConfig}

	b.Latest = ob
	b.Backups = append(b.Backups, ob)

	return couchbase.Set(backupKey, 0, &b)
}
Beispiel #9
0
func doFileInfo(w http.ResponseWriter, req *http.Request, fn string) {
	fm := fileMeta{}
	err := couchbase.Get(shortName(fn), &fm)
	switch {
	case err == nil:
	case gomemcached.IsNotFound(err):
		http.Error(w, "not found", 404)
		return
	default:
		http.Error(w, err.Error(), 500)
		return
	}
	sendJson(w, req, map[string]interface{}{
		"path": fn,
		"meta": fm,
	})
}
Beispiel #10
0
func doLinkFile(w http.ResponseWriter, req *http.Request) {
	fn := req.URL.Path
	h := req.FormValue("blob")
	t := req.FormValue("type")

	for len(fn) > 0 && fn[0] == '/' {
		fn = fn[1:]
	}

	blob, err := referenceBlob(h)
	if err != nil {
		estat := 500
		if gomemcached.IsNotFound(err) {
			estat = 404
		}
		http.Error(w, err.Error(), estat)
	}

	fm := fileMeta{
		Headers: http.Header{
			"Content-Type": []string{t},
		},
		OID:      h,
		Length:   blob.Length,
		Modified: time.Now().UTC(),
	}

	exp := getExpiration(req.Header)
	if exp != 0 {
		fm.Headers.Set("X-CBFS-Expiration", strconv.Itoa(exp))
	}

	err = maybeStoreMeta(fn, fm, exp, true)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.WriteHeader(201)
}
Beispiel #11
0
func db_init() {
	var ddocVersion uint64
	err := Bucket.Get(db_ddoc_version_key, &ddocVersion)
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Fatalf("fatal error getting ddoc version from database: %v", err)
	}
	if wanted := uint64(1); ddocVersion != wanted {
		log.Printf("updating ddoc from version %d to %d", ddocVersion, wanted)
		err = Bucket.PutDDoc(db_ddoc_name, couchbase.DDocJSON{
			Views: map[string]couchbase.ViewDefinition{
				"discussion-posts": {
					Map: `function( doc, meta ) {
	if ( doc.Type == "post" ) {
		emit( doc.Discussion, null );
	}
}`,
					Reduce: `_count`,
				},

				"updated-discussions": {
					Map: `function( doc, meta ) {
	if ( doc.Type == "discussion" ) {
		emit( dateToArray( doc.Modified ), null );
	}
}`,
				},
			},
		})
		if err != nil {
			log.Fatalf("fatal error storing ddoc: %v", err)
		}
		err = Bucket.Set(db_ddoc_version_key, 0, wanted)
		if err != nil {
			log.Fatalf("fatal error storing ddoc version: %v", err)
		}
	}
}
Beispiel #12
0
func main() {
	flag.Parse()

	rand.Seed(time.Now().UnixNano())

	initLogger(*useSyslog)
	initNodeListKeys()

	http.DefaultTransport = TimeoutTransport(*internodeTimeout)
	expvar.Publish("httpclients", httputil.InitHTTPTracker(false))

	if getHash() == nil {
		fmt.Fprintf(os.Stderr,
			"Unsupported hash specified: %v.  Supported hashes:\n",
			globalConfig.Hash)
		for h := range hashBuilders {
			fmt.Fprintf(os.Stderr, " * %v\n", h)
		}
		os.Exit(1)
	}

	err := initServerId()
	if err != nil {
		log.Fatalf("Error initializing server ID: %v", err)
	}

	if *maxStorageString != "" {
		ms, err := humanize.ParseBytes(*maxStorageString)
		if err != nil {
			log.Fatalf("Error parsing max storage parameter: %v",
				err)
		}
		maxStorage = int64(ms)
	}

	couchbase, err = dbConnect()
	if err != nil {
		log.Fatalf("Can't connect to couchbase: %v", err)
	}

	if err = os.MkdirAll(*root, 0777); err != nil {
		log.Fatalf("Couldn't create storage dir: %v", err)
	}

	err = updateConfig()
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Error updating initial config, using default: %v",
			err)
	}
	if *verbose {
		log.Printf("Server config:")
		globalConfig.Dump(os.Stdout)
	}

	go reloadConfig()

	go dnsServices()

	internodeTaskQueue = make(chan internodeTask, *taskWorkers*1024)
	initTaskQueueWorkers()

	go heartbeat()
	go startTasks()

	time.AfterFunc(time.Second*time.Duration(rand.Intn(30)+5), grabSomeData)

	go serveFrame()

	s := &http.Server{
		Addr:        *bindAddr,
		Handler:     http.HandlerFunc(httpHandler),
		ReadTimeout: *readTimeout,
	}
	log.Printf("Listening to web requests on %s as server %s",
		*bindAddr, serverId)

	l, err := rateListen("tcp", *bindAddr)
	if err != nil {
		log.Fatalf("Error listening: %v", err)
	}
	log.Fatal(s.Serve(l))
}
Beispiel #13
0
func loadExistingHashes() (*hashset.Hashset, error) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		return nil, err
	}

	oids := make(chan string)
	hsch := make(chan *hashset.Hashset)
	visitch := make(chan int)
	errch := make(chan error)

	wg := sync.WaitGroup{}

	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for o := range oids {
				h, v, e := loadBackupHashes(o)
				if e == nil {
					hsch <- h
				} else {
					errch <- e
				}
				visitch <- v
			}
		}()
	}
	go func() {
		wg.Wait()
		close(hsch)
		close(visitch)
		close(errch)
	}()

	go func() {
		for _, i := range b.Backups {
			log.Printf("Loading backups from %v / %v", i.Oid, i.Fn)
			oids <- i.Oid
		}
		close(oids)
	}()

	visited := 0
	hs := &hashset.Hashset{}
	for {
		// Done getting all the things
		if hsch == nil && visitch == nil && errch == nil {
			break
		}
		select {
		case v, ok := <-visitch:
			visited += v
			if !ok {
				visitch = nil
			}
		case e, ok := <-errch:
			err = e
			if !ok {
				errch = nil
			}
		case h, ok := <-hsch:
			if ok {
				log.Printf("Got %v hashes from a backup",
					h.Len())
				hs.AddAll(h)
			} else {
				hsch = nil
			}
		}
	}

	log.Printf("Visited %v obs, kept %v", visited, hs.Len())

	return hs, err
}
Beispiel #14
0
func serveBugDeletion(w http.ResponseWriter, r *http.Request) {
	me := whoami(r)
	bugid := mux.Vars(r)["bugid"]
	_, err := getBugOrDisplayErr(bugid, me, w, r)
	if err != nil {
		return
	}

	cherr := make(chan error)
	deleted := make(chan couchbase.ViewRow)
	delatt := make(chan couchbase.ViewRow)

	wg := &sync.WaitGroup{}

	wg.Add(1)
	go deleteDocsMatching("cbugg", "comments",
		map[string]interface{}{
			"stale":     false,
			"reduce":    false,
			"start_key": []interface{}{bugid},
			"end_key":   []interface{}{bugid, map[string]string{}},
		}, deleted, cherr, wg)

	wg.Add(1)
	go deleteDocsMatching("cbugg", "attachments",
		map[string]interface{}{
			"stale":        false,
			"reduce":       false,
			"start_key":    []interface{}{bugid},
			"end_key":      []interface{}{bugid, map[string]string{}},
			"include_docs": true,
		}, delatt, cherr, wg)

	wg.Add(1)
	go deleteDocsMatching("cbugg", "bug_history",
		map[string]interface{}{
			"stale":     false,
			"reduce":    false,
			"start_key": []interface{}{bugid},
			"end_key":   []interface{}{bugid, map[string]string{}},
		}, deleted, cherr, wg)

	go func() {
		wg.Wait()
		close(cherr)
	}()

	running := true
	for running {
		select {
		case del := <-deleted:
			log.Printf("Deleted %v", del.ID)
		case del := <-delatt:
			log.Printf("Deleted attachment %v", del.ID)
			mo := (*del.Doc).(map[string]interface{})
			mi := mo["json"].(map[string]interface{})
			u := mi["url"].(string)
			err := deleteAttachmentFile(u)
			if err != nil {
				log.Printf("Error deleting attachment %v: %v",
					u, err)
			}
		case err, ok := <-cherr:
			if !ok {
				running = false
			}
			if err != nil {
				log.Printf("Error:  %v", err)
			}
		}
	}

	log.Printf("Done deleting children.")

	// This bug should already be deleted as a bug is also bug
	// history.  But give it a go anyway for clarity.
	err = db.Delete(bugid)
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Error deleting the bug.")
	}

	w.WriteHeader(204)
}