// SynchronizedWithNetwork returns an error if the time of any of |peers| is
// too far off the local time.
func SynchronizedWithNetwork(peerAddr string, peers []string, networkPassword string) error {
var collectPeers []string
for _, peer := range peers {
if peer != peerAddr {
collectPeers = append(collectPeers, peer)
}
}
log.Printf("Collecting time of remaining nodes %v\n", collectPeers)
results, err := collectTime(collectPeers, networkPassword)
if err != nil {
glog.Warningf("Could not collect time from all of %v: %v\n", collectPeers, err)
}
return synchronizedWithNetwork(results)
}
// SynchronizedWithMasterAndNetwork returns an error if the time of either
// |join| or any other node in the network is too far off the local time.
func SynchronizedWithMasterAndNetwork(peerAddr, join, networkPassword string) error {
result, status, err := getServerTime(join, networkPassword)
if err != nil {
log.Fatalf("Could not join %q: %v\n", join, err)
}
var peers []string
for _, peer := range status.Peers {
if peer != peerAddr && peer != join {
peers = append(peers, peer)
}
}
log.Printf("Collecting time of remaining nodes %v\n", peers)
results, err := collectTime(peers, networkPassword)
if err != nil {
glog.Warningf("Could not collect time from all of %v: %v\n", peers, err)
}
results = append(results, result)
return synchronizedWithNetwork(results)
}
func handleGetMessages(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
// Avoid sessionOrProxy() because GetMessages can be answered on any raft
// node, it’s a read-only request.
session, err := session(r, ps)
if err != nil {
if err == ircserver.ErrSessionNotYetSeen {
http.Error(w, err.Error(), http.StatusInternalServerError)
} else {
http.Error(w, err.Error(), http.StatusNotFound)
}
return
}
remoteAddr := r.RemoteAddr
getMessagesRequestsMu.Lock()
GetMessageRequests[remoteAddr] = GetMessageStats{
Session: session,
Nick: ircServer.GetNick(session),
Started: time.Now(),
UserAgent: r.Header.Get("User-Agent"),
}
getMessagesRequestsMu.Unlock()
defer func() {
getMessagesRequestsMu.Lock()
delete(GetMessageRequests, remoteAddr)
getMessagesRequestsMu.Unlock()
}()
lastSeen := ircServer.GetStartId(session)
lastSeenStr := r.FormValue("lastseen")
if lastSeenStr != "0.0" && lastSeenStr != "" {
parts := strings.Split(lastSeenStr, ".")
if len(parts) != 2 {
log.Printf("cannot parse %q\n", lastSeenStr)
http.Error(w, fmt.Sprintf("Malformed lastseen value (%q)", lastSeenStr),
http.StatusInternalServerError)
return
}
first, err := strconv.ParseInt(parts[0], 0, 64)
if err != nil {
log.Printf("cannot parse %q\n", lastSeenStr)
http.Error(w, fmt.Sprintf("Malformed lastseen value (%q)", lastSeenStr),
http.StatusInternalServerError)
return
}
last, err := strconv.ParseInt(parts[1], 0, 64)
if err != nil {
log.Printf("cannot parse %q\n", lastSeenStr)
http.Error(w, fmt.Sprintf("Malformed lastseen value (%q)", lastSeenStr),
http.StatusInternalServerError)
return
}
lastSeen = types.RobustId{
Id: first,
Reply: last,
}
log.Printf("Trying to resume at %v\n", lastSeen)
}
enc := json.NewEncoder(w)
flushTimer := time.NewTimer(1 * time.Second)
flushTimer.Stop()
var lastFlush time.Time
willFlush := false
msgschan := make(chan []*types.RobustMessage)
ctx, cancel := context.WithCancel(context.Background())
go func() {
var msgs []*types.RobustMessage
pingDone := make(chan bool)
go func() {
pingTicker := time.NewTicker(pingInterval)
for {
select {
case <-pingTicker.C:
msgschan <- []*types.RobustMessage{pingMessage()}
case <-pingDone:
pingTicker.Stop()
return
}
}
}()
// With the following output messages stored for a session:
// Id Reply
// 1431542836610113945.1
// 1431542836610113945.2 |lastSeen|
// 1431542836610113945.3
// 1431542836691955391.1
// …when resuming, GetNext(1431542836610113945.2) will return
// 1431542836691955391.*, skipping the remaining messages with
// Id=1431542836610113945.
// Hence, we need to Get(1431542836610113945.2) to send
// 1431542836610113945.3 and following to the client.
if msgs, ok := ircServer.Get(lastSeen); ok && int(lastSeen.Reply) < len(msgs) {
msgschan <- msgs[lastSeen.Reply:]
}
for {
if ctx.Err() != nil {
pingDone <- true
close(msgschan)
return
}
msgs = ircServer.GetNext(ctx, lastSeen)
if len(msgs) == 0 {
continue
}
// This check prevents replaying old messages in the scenario where
// the client has seen newer messages than the server, for example
// because the server is currently recovering from a snapshot after
// being restarted, or because the server’s network connection to
// the rest of the network is currently slow.
if msgs[0].Id.Id < lastSeen.Id {
glog.Warningf("lastSeen (%d) more recent than GetNext() result %d\n", lastSeen.Id, msgs[0].Id.Id)
glog.Warningf("This should only happen while the server is recovering from a snapshot\n")
glog.Warningf("The message in question is %v\n", msgs[0])
// Prevent busylooping while new messages are applied.
time.Sleep(250 * time.Millisecond)
continue
}
lastSeen = msgs[0].Id
msgschan <- msgs
}
}()
defer func() {
cancel()
ircServer.InterruptGetNext()
for _ = range msgschan {
}
}()
for {
select {
case msgs := <-msgschan:
for _, msg := range msgs {
if msg.Type != types.RobustPing && !msg.InterestingFor[session.Id] {
continue
}
if err := enc.Encode(msg); err != nil {
log.Printf("Error encoding JSON: %v\n", err)
return
}
}
if _, err := ircServer.GetSession(session); err != nil {
// Session was deleted in the meanwhile, abort this request.
return
}
updateLastContact()
// The 10 seconds threshold is arbitrary. The only criterion is
// that it must be higher than raft’s HeartbeatTimeout of 2s. The
// higher it is chosen, the longer users have to wait until they
// can connect to a different node. Note that in the worst case,
// |pingInterval| = 20s needs to pass before this threshold is
// evaluated.
if node.State() != raft.Leader && time.Since(lastContact) > 10*time.Second {
// This node is neither the leader nor was it recently in
// contact with the master, indicating that it is partitioned
// from the rest of the network. We abort this GetMessages
// request so that clients can connect to a different server
// and receive new messages.
log.Printf("Aborting GetMessages request due to LastContact (%v) too long ago\n", node.LastContact())
return
}
if time.Since(lastFlush) > 100*time.Millisecond {
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
lastFlush = time.Now()
} else if !willFlush {
// Delay flushing by 10ms to avoid flushing too often, which
// results in lots of write() syscall.
flushTimer.Reset(10 * time.Millisecond)
willFlush = true
}
case <-flushTimer.C:
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
willFlush = false
lastFlush = time.Now()
}
}
}