func Spawn(config *vena.Config) { lmtr := limiter.New(20) for _, sh_ := range config.Shard { sh := sh_ lmtr.Go(func() { spawn(sh, config.ShardAnchor(sh.Key)) }) } lmtr.Wait() }
func (t *Listener) loop() { lmtr := limiter.New(MaxParallelHandshakes) for { c, err := t.listener.AcceptTCP() if err != nil { panic(err) // Best not to be quiet about it } lmtr.Go(func() { t.accept(c) }) } }
func Install(i *config.InstallConfig, b *config.BuildConfig, hosts []string) { l := limiter.New(limitParallelTasks) for _, host_ := range hosts { host := host_ l.Go(func() { fmt.Printf("Installing on %s\n", host) if err := installHost(i, b, host); err != nil { fmt.Fprintf(os.Stderr, "Issue on %s: %s\n", host, err) } }) } l.Wait() }
// SumBatch sends a batch of SUM requests to the sumr database func (cli *Client) SumBatch(batch []SumRequest) []float64 { var lk sync.Mutex r := make([]float64, len(batch)) blmtr := limiter.New(10) for i_, a_ := range batch { i, a := i_, a_ blmtr.Go(func() { q := cli.Sum(a.Key) lk.Lock() r[i] = q lk.Unlock() }) } blmtr.Wait() return r }
// AddBatch sends a batch of ADD requests to the sumr database func (cli *Client) AddBatch(batch []AddRequest) []float64 { var lk sync.Mutex r := make([]float64, len(batch)) blmtr := limiter.New(10) for i_, a_ := range batch { i, a := i_, a_ blmtr.Go(func() { q := cli.Add(a.UpdateTime, a.Key, a.Value) lk.Lock() r[i] = q lk.Unlock() }) } blmtr.Wait() return r }
func (t *worker) schedule() { println("Scheduling") lmtr := limiter.New(MaxOutstandingRequests) for { var job *request select { case job = <-t.fwdch: case job = <-t.apich: } lmtr.Open() go func(job *request) { defer lmtr.Close() job.ReturnResponse(t.fwd.Forward(job.Query, job.Source == "fwd")) }(job) } }
func (t *worker) schedule() { println("Scheduling") var ( lk sync.Mutex nxqb int64 nxqe int64 ) lmtr := limiter.New(MaxOutstandingRequests) for { var job interface{} select { case job = <-t.fCreate: case job = <-t.hCreate: //println("+ Processing HTTP-originated XCreatePost") case job = <-t.xCreate: case job = <-t.xQuery: //println("+ Processing X-originated XTimelineQuery") case job = <-t.hQuery: //println("+ Processing HTTP-originated XTimelineQuery") } lk.Lock() nxqb++ lk.Unlock() lmtr.Open() go func(job interface{}) { defer lmtr.Close() switch q := job.(type) { case *createRequest: q.ReturnResponse(t.fwd.Forward(q.Post, q.Forwarded)) case *queryRequest: q.ReturnResponse(t.srv.Query(q.Query)) default: panic("naah") } lk.Lock() nxqe++ if nxqb%1000 == 0 { println("+ Finished", nxqe, "/", nxqb) } lk.Unlock() }(job) } }
func listenTSDB(addr string, reply Replier) { l, err := net.Listen("tcp", addr) if err != nil { panic(err) } // Accept incoming requests go func() { lmtr := limiter.New(100) // At most 100 concurrent connections for { lmtr.Open() conn, err := l.Accept() if err != nil { panic(err) } // Serve individual connection go func() { defer lmtr.Close() defer conn.Close() defer recover() // Recover from panics in reply logic // Read request, send reply r := bufio.NewReader(conn) for { line, err := r.ReadString('\n') if err != nil { println("read line", err.Error()) break } cmd, err := parse(line) if err != nil { println("parse", err.Error()) break } if cmd == nil { continue } switch p := cmd.(type) { case diediedie: reply.DieDieDie() case *put: reply.Put(p.Time, p.Metric, p.Tags, p.Value) } } }() } }() }
// New creates a new server instance backed by a local directory in diskPath. // Keys not updated for forgetAfter duration are evicted from the in-memory replica of the shard's data. // Keys not in memory are not reflected in read operations. func New(diskPath string, forgetAfter time.Duration) (*Server, error) { s := &Server{} os.MkdirAll(diskPath, 0700) // Mount disk disk, err := diskfs.Mount(diskPath, false) if err != nil { return nil, err } // Make db block if s.block, err = block.NewBlock(disk, forgetAfter); err != nil { return nil, err } // Prepare incoming call rate-limiter s.lmtr = limiter.New(10) return s, nil }
func (srv *DashboardServer) Query(xq *proto.XDashboardQuery) ([]*proto.Post, error) { if xq.Limit < 1 { return nil, errors.New("zero query limit") } // Retrieve followed timelines from network service var err error follows := xq.Follows if follows == nil { if follows, err = srv.Follows(xq.DashboardID); err != nil { return nil, err } } // Concurrently fetch the most recent limit+1 posts from each followed timeline. // Put them in the results slice. var rlk sync.Mutex var err0 error results := make([]*proto.Post, 0, xq.Limit*len(follows)) l := limiter.New(MaxConcurrentTimelineQueries) for _, followedTimelineID := range follows { followedID := followedTimelineID l.Go(func() { posts, err := srv.queryTimeline(followedID, xq.BeforePostID, xq.Limit) rlk.Lock() defer rlk.Unlock() if err != nil { if err0 == nil { err0 = err } return } results = append(results, posts...) }) } l.Wait() // Sort all results from most recent to least (descending PostID) sort.Sort(proto.ChronoPosts(results)) return results[:min(xq.Limit, len(results))], err0 }