// StartAggregator.Start is a worker function that starts an infinite loop,
// which polls all reducers for their local top ten posts, computes the global
// top ten posts, and prints them out.
func (StartAggregator) Start(reducerAnchor string) {
circuit.Daemonize(func() {
for {
time.Sleep(2 * time.Second)
// Read anchor directory containing all live reducers
d, err := anchorfs.OpenDir(reducerAnchor)
if err != nil {
println("opendir:", err.Error())
continue
}
// List all anchor files; they correspond to circuit workers hosting Reducer objects
_, files, err := d.Files()
if err != nil {
println("files:", err.Error())
continue
}
// Fetch top ten posts from each reducer, in parallel
var (
l limiter.Limiter
lk sync.Mutex
top SortablePosts
)
println("Starting parallel aggregation")
l.Init(10) // At most 10 concurrent reducer requests at a time
for _, f_ := range files {
println("f=", f_.Owner().String())
f := f_ // Explain...
l.Go(func() { getReducerTop(f.Owner(), &lk, &top) })
}
l.Wait()
top = top[:min(10, len(top))]
println("Completed aggregation of", len(top), "best posts")
// Print the global top ten
fmt.Printf("Top ten, %s:\n", time.Now().Format(time.UnixDate))
for i, p := range top {
fmt.Printf("#% 2d: % 30s id=%d\n", i, p.Name, p.ID)
}
}
})
}
func Replenish(c *vena.Config, f *Config) []*ReplenishResult {
var (
lk sync.Mutex
lmtr limiter.Limiter
)
r := make([]*ReplenishResult, len(f.Workers))
lmtr.Init(20)
for i_, w_ := range f.Workers {
i, w := i_, w_
lmtr.Go(
func() {
re, addr, err := replenish(c, f.Workers[i], f.WorkerAnchor(i))
lk.Lock()
defer lk.Unlock()
r[i] = &ReplenishResult{Config: w, Addr: addr, Re: re, Err: err}
},
)
}
lmtr.Wait()
return r
}
// durableFile is the name of the durable file describing the SUMR server cluster
func Replenish(durableFile string, c *Config) []*Replenished {
var (
lk sync.Mutex
lmtr limiter.Limiter
)
r := make([]*Replenished, len(c.Workers))
lmtr.Init(20)
for i_, wcfg_ := range c.Workers {
i, wcfg := i_, wcfg_
lmtr.Go(
func() {
re, addr, err := replenishWorker(durableFile, c, i)
lk.Lock()
defer lk.Unlock()
r[i] = &Replenished{Config: wcfg, Addr: addr, Replenished: re, Err: err}
},
)
}
lmtr.Wait()
return r
}
func (StartAggregator) Start(reducerAnchor string) {
circuit.Daemonize(func() {
for {
time.Sleep(2 * time.Second)
d, err := anchorfs.OpenDir(reducerAnchor)
if err != nil {
println("opendir:", err.Error())
continue
}
_, files, err := d.Files()
if err != nil {
println("files:", err.Error())
continue
}
var (
l limiter.Limiter
lk sync.Mutex
top SortablePosts
)
println("Starting parallel aggregation")
l.Init(10)
for _, f_ := range files {
println("f=", f_.Owner().String())
f := f_ // Explain...
l.Go(func() { getReducerTop(f.Owner(), &lk, &top) })
}
l.Wait()
sort.Sort(top)
top = top[:min(10, len(top))]
println("Completed aggregation of", len(top), "best posts")
fmt.Printf("Top ten, %s:\n", time.Now().Format(time.UnixDate))
for i, p := range top {
fmt.Printf("#% 2d: % 30s id=%d Score=%d\n", i, p.Name, p.ID, p.Score)
}
}
})
}
// notify is called under a lock.
func (fs *FS) notify(issue *issuefs.Issue) {
// fetch listeners
emails, _, err := fs.z.Children(path.Join(fs.root, "listener"))
if err != nil {
panic(err)
}
// Email listeners
var lmtr limiter.Limiter
lmtr.Init(5)
for _, e_ := range emails {
e := e_
lmtr.Go(func() {
var tagline string
if len(issue.Anchor) > 0 {
tagline = issue.Anchor[0]
}
sendmail(e, fmt.Sprintf("CIRCUIT ISSUE: ", tagline), issue.String())
})
}
lmtr.Wait()
}
