func (b *bee) handoffNonPersistent(to uint64) error {
s, err := b.stateL1.Save()
if err != nil {
return err
}
if _, err = b.qee.sendCmdToBee(to, cmdRestoreState{State: s}); err != nil {
return err
}
oldc := b.colony()
newc := oldc.DeepCopy()
newc.Leader = to
up := updateColony{
Term: b.term(),
Old: oldc,
New: newc,
}
ctx, cnl := context.WithTimeout(context.Background(),
10*b.hive.config.RaftElectTimeout())
defer cnl()
if _, err := b.hive.proposeAmongHives(ctx, up); err != nil {
return err
}
b.becomeProxy()
return nil
}
func (h *DeQHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
q, ok := mux.Vars(r)["queue"]
if !ok {
http.Error(w, "unkown queue", http.StatusBadRequest)
return
}
ctx, cnl := context.WithTimeout(context.Background(), httpTimeout)
defer cnl()
d := Deque{Queue: Queue(q)}
res, err := h.Hive.Sync(ctx, d)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
switch res := res.(type) {
case Error:
http.Error(w, res.Message, http.StatusInternalServerError)
case Dequed:
if err = json.NewEncoder(w).Encode(res.Task); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
}
func (h *EnQHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
q, ok := mux.Vars(r)["queue"]
if !ok {
http.Error(w, "unkown queue", http.StatusBadRequest)
return
}
b, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, "cannot read request body", http.StatusBadRequest)
return
}
e := Enque{
Queue: Queue(q),
Body: b,
}
ctx, cnl := context.WithTimeout(context.Background(), httpTimeout)
defer cnl()
res, err := h.Hive.Sync(ctx, e)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
switch res := res.(type) {
case Error:
http.Error(w, res.Message, http.StatusInternalServerError)
case Enqued:
fmt.Fprintf(w, "task %v enqueued\n", res.TaskID)
}
}
// ProposeRetry proposes the request to the given group. It retires maxRetries
// times using the given timeout. If maxRetries is -1 it will keep proposing the
// request until it retrieves a response.
func (n *MultiNode) ProposeRetry(group uint64, req interface{},
timeout time.Duration, maxRetries int) (res interface{}, err error) {
for {
ctx, ccl := context.WithTimeout(context.Background(), timeout)
defer ccl()
if status := n.Status(group); status == nil || status.SoftState.Lead == 0 {
// wait with the hope that the group will be created.
n.waitElection(ctx, group)
} else {
res, err = n.Propose(ctx, group, req)
if err != context.DeadlineExceeded {
return
}
}
if maxRetries < 0 {
continue
}
maxRetries--
if maxRetries == 0 {
break
}
}
return nil, context.DeadlineExceeded
}
func (b *bee) addFollower(bid uint64, hid uint64) error {
oldc := b.colony()
if oldc.Leader != b.beeID {
return fmt.Errorf("%v is not the leader", b)
}
newc := oldc.DeepCopy()
if !newc.AddFollower(bid) {
return ErrDuplicateBee
}
t := 10 * b.hive.config.RaftElectTimeout()
upctx, upcnl := context.WithTimeout(context.Background(), t)
defer upcnl()
gid := oldc.ID
// TODO(soheil): It's important to have a proper order here. Or launch both in
// parallel and cancel them on error.
up := updateColony{
Term: b.term(),
Old: oldc,
New: newc,
}
if _, err := b.hive.proposeAmongHives(upctx, up); err != nil {
glog.Errorf("%v cannot update its colony: %v", b, err)
return err
}
cfgctx, cfgcnl := context.WithTimeout(context.Background(), t)
defer cfgcnl()
if err := b.hive.node.AddNodeToGroup(cfgctx, hid, gid, bid); err != nil {
return err
}
cmd := cmd{
Hive: hid,
App: b.app.Name(),
Bee: bid,
Data: cmdJoinColony{Colony: newc},
}
if _, err := b.hive.client.sendCmd(cmd); err != nil {
return err
}
b.setColony(newc)
return nil
}
func (s *rpcServer) ProcessRaft(batch raft.Batch, dummy *bool) (err error) {
if batch.To != s.h.ID() {
glog.Fatalf("%v recieves a raft message for %v", s.h, msg.To)
}
glog.V(3).Infof("%v handles a batch from %v", s.h, batch.From)
ctx, cnl := context.WithTimeout(context.Background(),
s.h.config.RaftHBTimeout())
err = s.h.node.StepBatch(ctx, batch, 2*s.h.config.RaftHBTimeout())
cnl()
return
}
func ExampleWithTimeout() {
// Pass a context with a timeout to tell a blocking function that it
// should abandon its work after the timeout elapses.
ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond)
select {
case <-time.After(200 * time.Millisecond):
fmt.Println("overslept")
case <-ctx.Done():
fmt.Println(ctx.Err()) // prints "context deadline exceeded"
}
// Output:
// context deadline exceeded
}
func (s *KVStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
k, ok := mux.Vars(r)["key"]
if !ok {
http.Error(w, "no key in the url", http.StatusBadRequest)
return
}
ctx, cnl := context.WithTimeout(context.Background(), 30*time.Second)
var res interface{}
var err error
switch r.Method {
case "GET":
res, err = s.Hive.Sync(ctx, Get(k))
case "PUT":
var v []byte
v, err = ioutil.ReadAll(r.Body)
if err != nil {
break
}
res, err = s.Hive.Sync(ctx, Put{Key: k, Val: string(v)})
case "DELETE":
res, err = s.Hive.Sync(ctx, Del(k))
}
cnl()
if err != nil {
switch {
case err.Error() == errKeyNotFound.Error():
http.Error(w, err.Error(), http.StatusNotFound)
return
case err.Error() == errInternal.Error():
http.Error(w, err.Error(), http.StatusInternalServerError)
return
case err.Error() == errInvalid.Error():
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
if res == nil {
return
}
js, err := json.Marshal(res)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(js)
}
func (b *bee) replicate() error {
glog.V(2).Infof("%v replicates transaction", b)
b.Lock()
if b.stateL2 != nil {
err := b.commitTxL2()
b.stateL2 = nil
if err != nil && err != state.ErrNoTx {
b.Unlock()
return err
}
}
if b.stateL1.TxStatus() != state.TxOpen {
b.Unlock()
return state.ErrNoTx
}
stx := b.stateL1.Tx()
if len(stx.Ops) == 0 {
err := b.commitTxL1()
b.Unlock()
return err
}
b.Unlock()
if err := b.maybeRecruitFollowers(); err != nil {
return err
}
msgs := make([]*msg, len(b.msgBufL1))
copy(msgs, b.msgBufL1)
tx := tx{
Tx: stx,
Msgs: msgs,
}
ctx, cnl := context.WithTimeout(context.Background(),
10*b.hive.config.RaftElectTimeout())
defer cnl()
commit := commitTx{
Tx: tx,
Term: b.term(),
}
if _, err := b.hive.node.Propose(ctx, b.group(), commit); err != nil {
glog.Errorf("%v cannot replicate the transaction: %v", b, err)
return err
}
glog.V(2).Infof("%v successfully replicates transaction", b)
return nil
}
func (n *MultiNode) handleBatch(bt batchTimeout) {
ctx, cnl := context.WithTimeout(context.Background(), bt.timeout)
for g, msgs := range bt.batch.Messages {
if _, ok := n.groups[g]; !ok {
glog.Errorf("group %v is not created on %v", g, n)
continue
}
for _, m := range msgs {
if err := n.node.Step(ctx, g, m); err != nil {
glog.Errorf("%v cannot step group %v: %v", n, g, err)
if err == context.DeadlineExceeded || err == context.Canceled {
return
}
}
}
}
cnl()
}
func (h *AckHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
q, ok := vars["queue"]
if !ok {
http.Error(w, "unkown queue", http.StatusBadRequest)
return
}
t, ok := vars["id"]
if !ok {
http.Error(w, "unkown task", http.StatusBadRequest)
return
}
id, err := strconv.ParseUint(t, 10, 64)
if err != nil {
http.Error(w, fmt.Sprintf("invalid task id: %v", err),
http.StatusBadRequest)
return
}
ctx, cnl := context.WithTimeout(context.Background(), httpTimeout)
defer cnl()
a := Ack{
TaskID: TaskID(id),
Queue: Queue(q),
}
res, err := h.Hive.Sync(ctx, a)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
switch res := res.(type) {
case Error:
http.Error(w, res.Message, http.StatusInternalServerError)
case Acked:
fmt.Fprintf(w, "task %v acknowledged\n", id)
}
}
func (h *statHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx, ccl := context.WithTimeout(context.Background(), 10*time.Second)
defer ccl()
res, err := h.hive.Sync(ctx, statRequest{})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
jsonres := make(map[string]map[string]uint64)
for to, m := range res.(statResponse).Matrix {
jsonresto := make(map[string]uint64)
jsonres[strconv.FormatUint(to, 10)] = jsonresto
for from, cnt := range m {
jsonresto[strconv.FormatUint(from, 10)] = cnt
}
}
b, err := json.Marshal(jsonres)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(b)
}
func (b *bee) handleCmdLocal(cc cmdAndChannel) {
glog.V(2).Infof("%v handles command %v", b, cc.cmd)
var err error
var data interface{}
switch cmd := cc.cmd.Data.(type) {
case cmdStop:
b.status = beeStatusStopped
b.disableEmit()
glog.V(2).Infof("%v stopped", b)
case cmdStart:
b.status = beeStatusStarted
glog.V(2).Infof("%v started", b)
case cmdSync:
err = b.raftBarrier()
case cmdRestoreState:
err = b.stateL1.Restore(cmd.State)
case cmdCampaign:
ctx, cnl := context.WithTimeout(context.Background(),
b.hive.config.RaftElectTimeout())
err = b.hive.node.Campaign(ctx, b.group())
cnl()
case cmdHandoff:
err = b.handoff(cmd.To)
case cmdJoinColony:
if !cmd.Colony.Contains(b.ID()) {
err = fmt.Errorf("%v is not in this colony %v", b, cmd.Colony)
break
}
if !b.isColonyNil() {
err = fmt.Errorf("%v is already in colony %v", b, b.colony())
break
}
b.setColony(cmd.Colony)
if b.app.persistent() {
if err = b.createGroup(); err != nil {
break
}
}
if cmd.Colony.Leader == b.ID() {
b.becomeLeader()
} else {
b.becomeFollower()
}
case cmdAddMappedCells:
b.addMappedCells(cmd.Cells)
case cmdRefreshRole:
c := b.colony()
if c.Leader == b.ID() {
b.becomeLeader()
} else {
b.becomeFollower()
}
case cmdAddFollower:
err = b.addFollower(cmd.Bee, cmd.Hive)
default:
err = fmt.Errorf("unknown bee command %#v", cmd)
}
if err != nil {
glog.Errorf("%v cannot handle %v: %v", b, cc.cmd, err)
}
if cc.ch != nil {
cc.ch <- cmdResult{
Data: data,
Err: err,
}
}
}