Example #1
0
func (s *EtcdServer) run() {
	snap, err := s.r.raftStorage.Snapshot()
	if err != nil {
		plog.Panicf("get snapshot from raft storage error: %v", err)
	}
	confState := snap.Metadata.ConfState
	snapi := snap.Metadata.Index
	appliedi := snapi
	// TODO: get rid of the raft initialization in etcd server
	s.r.s = s
	s.r.applyc = make(chan apply)
	s.r.stopped = make(chan struct{})
	s.r.done = make(chan struct{})
	go s.r.run()
	defer func() {
		s.r.stopped <- struct{}{}
		<-s.r.done
		close(s.done)
	}()

	var shouldstop bool
	for {
		select {
		case apply := <-s.r.apply():
			// apply snapshot
			if !raft.IsEmptySnap(apply.snapshot) {
				if apply.snapshot.Metadata.Index <= appliedi {
					plog.Panicf("snapshot index [%d] should > appliedi[%d] + 1",
						apply.snapshot.Metadata.Index, appliedi)
				}

				if err := s.store.Recovery(apply.snapshot.Data); err != nil {
					plog.Panicf("recovery store error: %v", err)
				}
				s.cluster.Recover()

				// recover raft transport
				s.r.transport.RemoveAllPeers()
				for _, m := range s.cluster.Members() {
					if m.ID == s.ID() {
						continue
					}
					s.r.transport.AddPeer(m.ID, m.PeerURLs)
				}

				appliedi = apply.snapshot.Metadata.Index
				snapi = appliedi
				confState = apply.snapshot.Metadata.ConfState
				plog.Infof("recovered from incoming snapshot at index %d", snapi)
			}

			// apply entries
			if len(apply.entries) != 0 {
				firsti := apply.entries[0].Index
				if firsti > appliedi+1 {
					plog.Panicf("first index of committed entry[%d] should <= appliedi[%d] + 1", firsti, appliedi)
				}
				var ents []raftpb.Entry
				if appliedi+1-firsti < uint64(len(apply.entries)) {
					ents = apply.entries[appliedi+1-firsti:]
				}
				if appliedi, shouldstop = s.apply(ents, &confState); shouldstop {
					go s.stopWithDelay(10*100*time.Millisecond, fmt.Errorf("the member has been permanently removed from the cluster"))
				}
			}

			// wait for the raft routine to finish the disk writes before triggering a
			// snapshot. or applied index might be greater than the last index in raft
			// storage, since the raft routine might be slower than apply routine.
			apply.done <- struct{}{}

			// trigger snapshot
			if appliedi-snapi > s.snapCount {
				plog.Infof("start to snapshot (applied: %d, lastsnap: %d)", appliedi, snapi)
				s.snapshot(appliedi, confState)
				snapi = appliedi
			}
		case err := <-s.errorc:
			plog.Errorf("%s", err)
			plog.Infof("the data-dir used by this member must be removed.")
			return
		case <-s.stop:
			return
		}
	}
}
Example #2
0
func (r *raftNode) run() {
	var syncC <-chan time.Time

	defer r.stop()
	for {
		select {
		case <-r.ticker:
			r.Tick()
		case rd := <-r.Ready():
			if rd.SoftState != nil {
				if lead := atomic.LoadUint64(&r.lead); rd.SoftState.Lead != raft.None && lead != rd.SoftState.Lead {
					r.mu.Lock()
					r.lt = time.Now()
					r.mu.Unlock()
				}
				atomic.StoreUint64(&r.lead, rd.SoftState.Lead)
				if rd.RaftState == raft.StateLeader {
					syncC = r.s.SyncTicker
					// TODO: remove the nil checking
					// current test utility does not provide the stats
					if r.s.stats != nil {
						r.s.stats.BecomeLeader()
					}
				} else {
					syncC = nil
				}
			}

			apply := apply{
				entries:  rd.CommittedEntries,
				snapshot: rd.Snapshot,
				done:     make(chan struct{}),
			}

			select {
			case r.applyc <- apply:
			case <-r.stopped:
				return
			}

			if !raft.IsEmptySnap(rd.Snapshot) {
				if err := r.storage.SaveSnap(rd.Snapshot); err != nil {
					plog.Fatalf("raft save snapshot error: %v", err)
				}
				r.raftStorage.ApplySnapshot(rd.Snapshot)
				plog.Infof("raft applied incoming snapshot at index %d", rd.Snapshot.Metadata.Index)
			}
			if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
				plog.Fatalf("raft save state and entries error: %v", err)
			}
			r.raftStorage.Append(rd.Entries)

			r.s.send(rd.Messages)

			select {
			case <-apply.done:
			case <-r.stopped:
				return
			}
			r.Advance()
		case <-syncC:
			r.s.sync(defaultSyncTimeout)
		case <-r.stopped:
			return
		}
	}
}
Example #3
0
func (s *Snapshotter) SaveSnap(snapshot raftpb.Snapshot) error {
	if raft.IsEmptySnap(snapshot) {
		return nil
	}
	return s.save(&snapshot)
}
Example #4
0
func (s *EtcdServer) run() {
	var syncC <-chan time.Time
	var shouldstop bool

	// load initial state from raft storage
	snap, err := s.r.raftStorage.Snapshot()
	if err != nil {
		log.Panicf("etcdserver: get snapshot from raft storage error: %v", err)
	}
	// snapi indicates the index of the last submitted snapshot request
	snapi := snap.Metadata.Index
	appliedi := snap.Metadata.Index
	confState := snap.Metadata.ConfState

	defer func() {
		s.r.Stop()
		s.r.transport.Stop()
		if err := s.r.storage.Close(); err != nil {
			log.Panicf("etcdserver: close storage error: %v", err)
		}
		close(s.done)
	}()
	// TODO: make raft loop a method on raftNode
	for {
		select {
		case <-s.r.ticker:
			s.r.Tick()
		case rd := <-s.r.Ready():
			if rd.SoftState != nil {
				atomic.StoreUint64(&s.r.lead, rd.SoftState.Lead)
				if rd.RaftState == raft.StateLeader {
					syncC = s.SyncTicker
					// TODO: remove the nil checking
					// current test utility does not provide the stats
					if s.stats != nil {
						s.stats.BecomeLeader()
					}
				} else {
					syncC = nil
				}
			}

			// apply snapshot to storage if it is more updated than current snapi
			if !raft.IsEmptySnap(rd.Snapshot) && rd.Snapshot.Metadata.Index > snapi {
				if err := s.r.storage.SaveSnap(rd.Snapshot); err != nil {
					log.Fatalf("etcdserver: save snapshot error: %v", err)
				}
				s.r.raftStorage.ApplySnapshot(rd.Snapshot)
				snapi = rd.Snapshot.Metadata.Index
				log.Printf("etcdserver: saved incoming snapshot at index %d", snapi)
			}

			if err := s.r.storage.Save(rd.HardState, rd.Entries); err != nil {
				log.Fatalf("etcdserver: save state and entries error: %v", err)
			}
			s.r.raftStorage.Append(rd.Entries)

			s.send(rd.Messages)

			// recover from snapshot if it is more updated than current applied
			if !raft.IsEmptySnap(rd.Snapshot) && rd.Snapshot.Metadata.Index > appliedi {
				if err := s.store.Recovery(rd.Snapshot.Data); err != nil {
					log.Panicf("recovery store error: %v", err)
				}
				s.Cluster.Recover()

				// recover raft transport
				s.r.transport.RemoveAllPeers()
				for _, m := range s.Cluster.Members() {
					if m.ID == s.ID() {
						continue
					}
					s.r.transport.AddPeer(m.ID, m.PeerURLs)
				}

				appliedi = rd.Snapshot.Metadata.Index
				confState = rd.Snapshot.Metadata.ConfState
				log.Printf("etcdserver: recovered from incoming snapshot at index %d", snapi)
			}
			// TODO(bmizerany): do this in the background, but take
			// care to apply entries in a single goroutine, and not
			// race them.
			if len(rd.CommittedEntries) != 0 {
				firsti := rd.CommittedEntries[0].Index
				if firsti > appliedi+1 {
					log.Panicf("etcdserver: first index of committed entry[%d] should <= appliedi[%d] + 1", firsti, appliedi)
				}
				var ents []raftpb.Entry
				if appliedi+1-firsti < uint64(len(rd.CommittedEntries)) {
					ents = rd.CommittedEntries[appliedi+1-firsti:]
				}
				if len(ents) > 0 {
					if appliedi, shouldstop = s.apply(ents, &confState); shouldstop {
						go s.stopWithDelay(10*100*time.Millisecond, fmt.Errorf("the member has been permanently removed from the cluster"))
					}
				}
			}

			s.r.Advance()

			if appliedi-snapi > s.r.snapCount {
				log.Printf("etcdserver: start to snapshot (applied: %d, lastsnap: %d)", appliedi, snapi)
				s.snapshot(appliedi, &confState)
				snapi = appliedi
			}
		case <-syncC:
			s.sync(defaultSyncTimeout)
		case err := <-s.errorc:
			log.Printf("etcdserver: %s", err)
			log.Printf("etcdserver: the data-dir used by this member must be removed.")
			return
		case <-s.stop:
			return
		}
	}
}