func reloadConfig() {
for _ = range time.Tick(time.Minute) {
if err := updateConfig(); err != nil && !gomemcached.IsNotFound(err) {
log.Printf("Error updating config: %v", err)
}
}
}
func sendTagNotification(bugid, tagName, actor string) {
b, err := getBug(bugid)
if err != nil {
log.Printf("Error getting bug %v for tag notification: %v",
bugid, err)
return
}
tag := Tag{}
err = db.Get("tag-"+tagName, &tag)
if err != nil {
if !gomemcached.IsNotFound(err) {
log.Printf("Error sending notification for tag %v: %v",
tagName, err)
}
return
}
to := filterUnprivelegedEmails(b, removeFromList(tag.Subscribers, actor))
sendNotifications("tag_notification", to,
map[string]interface{}{
"Bug": b,
"Tag": tagName,
"Actor": actor,
})
}
func errorCode(err error) int {
switch {
case err == bugNotVisible:
return 401
case gomemcached.IsNotFound(err):
return 404
}
return 500
}
// Send a custom request and get the response.
func (client *Client) Send(req *gomemcached.MCRequest) (rv *gomemcached.MCResponse, err error) {
err = transmitRequest(client.conn, req)
if err != nil {
client.healthy = false
return
}
resp, err := getResponse(client.conn, client.hdrBuf)
if err != nil && !gomemcached.IsNotFound(err) {
client.healthy = false
}
return resp, err
}
func doGetConfig(w http.ResponseWriter, req *http.Request) {
err := updateConfig()
if err != nil && !gomemcached.IsNotFound(err) {
log.Printf("Error updating config: %v", err)
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(200)
e := json.NewEncoder(w)
err = e.Encode(&globalConfig)
if err != nil {
log.Printf("Error sending config: %v", err)
}
}
func doGetBackupInfo(w http.ResponseWriter, req *http.Request) {
b := backups{}
err := couchbase.Get(backupKey, &b)
if err != nil {
code := 500
if gomemcached.IsNotFound(err) {
code = 404
}
http.Error(w, err.Error(), code)
return
}
removeDeadBackups(&b)
sendJson(w, req, &b)
}
func removeDeadBackups(b *backups) {
// Keep only backups we're pretty sure still exist.
obn := b.Backups
b.Backups = nil
for _, bi := range obn {
fm := fileMeta{}
err := couchbase.Get(shortName(bi.Fn), &fm)
if gomemcached.IsNotFound(err) {
log.Printf("Dropping previous (deleted) backup: %v",
bi.Fn)
} else {
b.Backups = append(b.Backups, bi)
}
}
}
func storeBackupObject(fn, h string) error {
b := backups{}
err := couchbase.Get(backupKey, &b)
if err != nil && !gomemcached.IsNotFound(err) {
log.Printf("Weird: %v", err)
// return err
}
removeDeadBackups(&b)
ob := backupItem{fn, h, time.Now().UTC(), *globalConfig}
b.Latest = ob
b.Backups = append(b.Backups, ob)
return couchbase.Set(backupKey, 0, &b)
}
func doFileInfo(w http.ResponseWriter, req *http.Request, fn string) {
fm := fileMeta{}
err := couchbase.Get(shortName(fn), &fm)
switch {
case err == nil:
case gomemcached.IsNotFound(err):
http.Error(w, "not found", 404)
return
default:
http.Error(w, err.Error(), 500)
return
}
sendJson(w, req, map[string]interface{}{
"path": fn,
"meta": fm,
})
}
func doLinkFile(w http.ResponseWriter, req *http.Request) {
fn := req.URL.Path
h := req.FormValue("blob")
t := req.FormValue("type")
for len(fn) > 0 && fn[0] == '/' {
fn = fn[1:]
}
blob, err := referenceBlob(h)
if err != nil {
estat := 500
if gomemcached.IsNotFound(err) {
estat = 404
}
http.Error(w, err.Error(), estat)
}
fm := fileMeta{
Headers: http.Header{
"Content-Type": []string{t},
},
OID: h,
Length: blob.Length,
Modified: time.Now().UTC(),
}
exp := getExpiration(req.Header)
if exp != 0 {
fm.Headers.Set("X-CBFS-Expiration", strconv.Itoa(exp))
}
err = maybeStoreMeta(fn, fm, exp, true)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
w.WriteHeader(201)
}
func db_init() {
var ddocVersion uint64
err := Bucket.Get(db_ddoc_version_key, &ddocVersion)
if err != nil && !gomemcached.IsNotFound(err) {
log.Fatalf("fatal error getting ddoc version from database: %v", err)
}
if wanted := uint64(1); ddocVersion != wanted {
log.Printf("updating ddoc from version %d to %d", ddocVersion, wanted)
err = Bucket.PutDDoc(db_ddoc_name, couchbase.DDocJSON{
Views: map[string]couchbase.ViewDefinition{
"discussion-posts": {
Map: `function( doc, meta ) {
if ( doc.Type == "post" ) {
emit( doc.Discussion, null );
}
}`,
Reduce: `_count`,
},
"updated-discussions": {
Map: `function( doc, meta ) {
if ( doc.Type == "discussion" ) {
emit( dateToArray( doc.Modified ), null );
}
}`,
},
},
})
if err != nil {
log.Fatalf("fatal error storing ddoc: %v", err)
}
err = Bucket.Set(db_ddoc_version_key, 0, wanted)
if err != nil {
log.Fatalf("fatal error storing ddoc version: %v", err)
}
}
}
func main() {
flag.Parse()
rand.Seed(time.Now().UnixNano())
initLogger(*useSyslog)
initNodeListKeys()
http.DefaultTransport = TimeoutTransport(*internodeTimeout)
expvar.Publish("httpclients", httputil.InitHTTPTracker(false))
if getHash() == nil {
fmt.Fprintf(os.Stderr,
"Unsupported hash specified: %v. Supported hashes:\n",
globalConfig.Hash)
for h := range hashBuilders {
fmt.Fprintf(os.Stderr, " * %v\n", h)
}
os.Exit(1)
}
err := initServerId()
if err != nil {
log.Fatalf("Error initializing server ID: %v", err)
}
if *maxStorageString != "" {
ms, err := humanize.ParseBytes(*maxStorageString)
if err != nil {
log.Fatalf("Error parsing max storage parameter: %v",
err)
}
maxStorage = int64(ms)
}
couchbase, err = dbConnect()
if err != nil {
log.Fatalf("Can't connect to couchbase: %v", err)
}
if err = os.MkdirAll(*root, 0777); err != nil {
log.Fatalf("Couldn't create storage dir: %v", err)
}
err = updateConfig()
if err != nil && !gomemcached.IsNotFound(err) {
log.Printf("Error updating initial config, using default: %v",
err)
}
if *verbose {
log.Printf("Server config:")
globalConfig.Dump(os.Stdout)
}
go reloadConfig()
go dnsServices()
internodeTaskQueue = make(chan internodeTask, *taskWorkers*1024)
initTaskQueueWorkers()
go heartbeat()
go startTasks()
time.AfterFunc(time.Second*time.Duration(rand.Intn(30)+5), grabSomeData)
go serveFrame()
s := &http.Server{
Addr: *bindAddr,
Handler: http.HandlerFunc(httpHandler),
ReadTimeout: *readTimeout,
}
log.Printf("Listening to web requests on %s as server %s",
*bindAddr, serverId)
l, err := rateListen("tcp", *bindAddr)
if err != nil {
log.Fatalf("Error listening: %v", err)
}
log.Fatal(s.Serve(l))
}
func loadExistingHashes() (*hashset.Hashset, error) {
b := backups{}
err := couchbase.Get(backupKey, &b)
if err != nil && !gomemcached.IsNotFound(err) {
return nil, err
}
oids := make(chan string)
hsch := make(chan *hashset.Hashset)
visitch := make(chan int)
errch := make(chan error)
wg := sync.WaitGroup{}
for i := 0; i < 4; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for o := range oids {
h, v, e := loadBackupHashes(o)
if e == nil {
hsch <- h
} else {
errch <- e
}
visitch <- v
}
}()
}
go func() {
wg.Wait()
close(hsch)
close(visitch)
close(errch)
}()
go func() {
for _, i := range b.Backups {
log.Printf("Loading backups from %v / %v", i.Oid, i.Fn)
oids <- i.Oid
}
close(oids)
}()
visited := 0
hs := &hashset.Hashset{}
for {
// Done getting all the things
if hsch == nil && visitch == nil && errch == nil {
break
}
select {
case v, ok := <-visitch:
visited += v
if !ok {
visitch = nil
}
case e, ok := <-errch:
err = e
if !ok {
errch = nil
}
case h, ok := <-hsch:
if ok {
log.Printf("Got %v hashes from a backup",
h.Len())
hs.AddAll(h)
} else {
hsch = nil
}
}
}
log.Printf("Visited %v obs, kept %v", visited, hs.Len())
return hs, err
}
func serveBugDeletion(w http.ResponseWriter, r *http.Request) {
me := whoami(r)
bugid := mux.Vars(r)["bugid"]
_, err := getBugOrDisplayErr(bugid, me, w, r)
if err != nil {
return
}
cherr := make(chan error)
deleted := make(chan couchbase.ViewRow)
delatt := make(chan couchbase.ViewRow)
wg := &sync.WaitGroup{}
wg.Add(1)
go deleteDocsMatching("cbugg", "comments",
map[string]interface{}{
"stale": false,
"reduce": false,
"start_key": []interface{}{bugid},
"end_key": []interface{}{bugid, map[string]string{}},
}, deleted, cherr, wg)
wg.Add(1)
go deleteDocsMatching("cbugg", "attachments",
map[string]interface{}{
"stale": false,
"reduce": false,
"start_key": []interface{}{bugid},
"end_key": []interface{}{bugid, map[string]string{}},
"include_docs": true,
}, delatt, cherr, wg)
wg.Add(1)
go deleteDocsMatching("cbugg", "bug_history",
map[string]interface{}{
"stale": false,
"reduce": false,
"start_key": []interface{}{bugid},
"end_key": []interface{}{bugid, map[string]string{}},
}, deleted, cherr, wg)
go func() {
wg.Wait()
close(cherr)
}()
running := true
for running {
select {
case del := <-deleted:
log.Printf("Deleted %v", del.ID)
case del := <-delatt:
log.Printf("Deleted attachment %v", del.ID)
mo := (*del.Doc).(map[string]interface{})
mi := mo["json"].(map[string]interface{})
u := mi["url"].(string)
err := deleteAttachmentFile(u)
if err != nil {
log.Printf("Error deleting attachment %v: %v",
u, err)
}
case err, ok := <-cherr:
if !ok {
running = false
}
if err != nil {
log.Printf("Error: %v", err)
}
}
}
log.Printf("Done deleting children.")
// This bug should already be deleted as a bug is also bug
// history. But give it a go anyway for clarity.
err = db.Delete(bugid)
if err != nil && !gomemcached.IsNotFound(err) {
log.Printf("Error deleting the bug.")
}
w.WriteHeader(204)
}