Beispiel #1
0
// StartAggregator.Start is a worker function that starts an infinite loop,
// which polls all reducers for their local top ten posts, computes the global
// top ten posts, and prints them out.
func (StartAggregator) Start(reducerAnchor string) {
	circuit.Daemonize(func() {
		for {
			time.Sleep(2 * time.Second)

			// Read anchor directory containing all live reducers
			d, err := anchorfs.OpenDir(reducerAnchor)
			if err != nil {
				println("opendir:", err.Error())
				continue
			}
			// List all anchor files; they correspond to circuit workers hosting Reducer objects
			_, files, err := d.Files()
			if err != nil {
				println("files:", err.Error())
				continue
			}
			// Fetch top ten posts from each reducer, in parallel
			var (
				l   limiter.Limiter
				lk  sync.Mutex
				top SortablePosts
			)
			println("Starting parallel aggregation")
			l.Init(10) // At most 10 concurrent reducer requests at a time
			for _, f_ := range files {
				println("f=", f_.Owner().String())
				f := f_ // Explain...
				l.Go(func() { getReducerTop(f.Owner(), &lk, &top) })
			}
			l.Wait()
			top = top[:min(10, len(top))]
			println("Completed aggregation of", len(top), "best posts")
			// Print the global top ten
			fmt.Printf("Top ten, %s:\n", time.Now().Format(time.UnixDate))
			for i, p := range top {
				fmt.Printf("#% 2d: % 30s id=%d\n", i, p.Name, p.ID)
			}
		}
	})
}
Beispiel #2
0
func Replenish(c *vena.Config, f *Config) []*ReplenishResult {
	var (
		lk   sync.Mutex
		lmtr limiter.Limiter
	)
	r := make([]*ReplenishResult, len(f.Workers))
	lmtr.Init(20)
	for i_, w_ := range f.Workers {
		i, w := i_, w_
		lmtr.Go(
			func() {
				re, addr, err := replenish(c, f.Workers[i], f.WorkerAnchor(i))
				lk.Lock()
				defer lk.Unlock()
				r[i] = &ReplenishResult{Config: w, Addr: addr, Re: re, Err: err}
			},
		)
	}
	lmtr.Wait()
	return r
}
Beispiel #3
0
// durableFile is the name of the durable file describing the SUMR server cluster
func Replenish(durableFile string, c *Config) []*Replenished {
	var (
		lk   sync.Mutex
		lmtr limiter.Limiter
	)
	r := make([]*Replenished, len(c.Workers))
	lmtr.Init(20)
	for i_, wcfg_ := range c.Workers {
		i, wcfg := i_, wcfg_
		lmtr.Go(
			func() {
				re, addr, err := replenishWorker(durableFile, c, i)
				lk.Lock()
				defer lk.Unlock()
				r[i] = &Replenished{Config: wcfg, Addr: addr, Replenished: re, Err: err}
			},
		)
	}
	lmtr.Wait()
	return r
}
Beispiel #4
0
func (StartAggregator) Start(reducerAnchor string) {
	circuit.Daemonize(func() {
		for {
			time.Sleep(2 * time.Second)

			d, err := anchorfs.OpenDir(reducerAnchor)
			if err != nil {
				println("opendir:", err.Error())
				continue
			}
			_, files, err := d.Files()
			if err != nil {
				println("files:", err.Error())
				continue
			}
			var (
				l   limiter.Limiter
				lk  sync.Mutex
				top SortablePosts
			)
			println("Starting parallel aggregation")
			l.Init(10)
			for _, f_ := range files {
				println("f=", f_.Owner().String())
				f := f_ // Explain...
				l.Go(func() { getReducerTop(f.Owner(), &lk, &top) })
			}
			l.Wait()
			sort.Sort(top)
			top = top[:min(10, len(top))]
			println("Completed aggregation of", len(top), "best posts")
			fmt.Printf("Top ten, %s:\n", time.Now().Format(time.UnixDate))
			for i, p := range top {
				fmt.Printf("#% 2d: % 30s id=%d Score=%d\n", i, p.Name, p.ID, p.Score)
			}
		}
	})
}
Beispiel #5
0
// notify is called under a lock.
func (fs *FS) notify(issue *issuefs.Issue) {

	// fetch listeners
	emails, _, err := fs.z.Children(path.Join(fs.root, "listener"))
	if err != nil {
		panic(err)
	}

	// Email listeners
	var lmtr limiter.Limiter
	lmtr.Init(5)
	for _, e_ := range emails {
		e := e_
		lmtr.Go(func() {
			var tagline string
			if len(issue.Anchor) > 0 {
				tagline = issue.Anchor[0]
			}
			sendmail(e, fmt.Sprintf("CIRCUIT ISSUE: ", tagline), issue.String())
		})
	}
	lmtr.Wait()
}