func (h *Harness) processPending(results map[int]*result) { for { result, ok := results[h.nextSequenceNumber] if !ok { return } output, err := h.sql.Execute("harness", result.query) if err != nil { h.losef("[%d] Could not execute statement that %v claimed was fine: %s", h.nextSequenceNumber, result.node, err) } if !bytes.Equal(result.body, output.Stdout) { h.losef(`[%d] Received incorrect output from %v for query %s Output: %s Correct output: %s`, h.nextSequenceNumber, result.node, result.query, util.FmtOutput(result.body), util.FmtOutput(output.Stdout)) } else { state.RecordCorrectQuery() log.Printf(`[harness] [%d] Received correct output from %v for query %s Output: %s`, h.nextSequenceNumber, result.node, result.query, util.FmtOutput(result.body)) } // Update bookkeeping delete(results, h.nextSequenceNumber) h.nextSequenceNumber += 1 } }
// Rules for sequence numbers: // // - Gaps are temporarily OK, but not in the long run. func (h *Harness) resultHandler() { results := make(map[int]*result) for { result := <-h.result sequenceNumber, body, err := h.parseResponse(result.resp) if err != nil { h.lose(err.Error()) return } result.body = body if sequenceNumber < h.nextSequenceNumber { h.losef(`[%d] Received an already-processed sequence number from %v in response to %s Output: %s`, sequenceNumber, result.node, result.query, util.FmtOutput(result.resp)) return } if old, ok := results[sequenceNumber]; ok { h.losef(`[%d] Received a still-pending sequence number from %v in response to %s Output: %s This sequence number was originally received in response to %s Original output: %s`, sequenceNumber, result.node, result.query, util.FmtOutput(result.resp), old.query, util.FmtOutput(old.resp)) return } if sequenceNumber > h.nextSequenceNumber { log.Printf("[%d] Result from %v waiting on sequence number %d", sequenceNumber, result.node, h.nextSequenceNumber) } // Notify SPOF monkey that we got a valid request select { case state.GotRequest() <- result.reqid: default: } results[sequenceNumber] = result h.processPending(results) } }