func (h *Harness) processPending(results map[int]*result) { for { result, ok := results[h.nextSequenceNumber] if !ok { return } output, err := h.sql.Execute("harness", result.query) if err != nil { h.losef("[%d] Could not execute statement that %v claimed was fine: %s", h.nextSequenceNumber, result.node, err) } if !bytes.Equal(result.body, output.Stdout) { h.losef(`[%d] Received incorrect output from %v for query %s Output: %s Correct output: %s`, h.nextSequenceNumber, result.node, result.query, util.FmtOutput(result.body), util.FmtOutput(output.Stdout)) } else { state.RecordCorrectQuery() log.Printf(`[harness] [%d] Received correct output from %v for query %s Output: %s`, h.nextSequenceNumber, result.node, result.query, util.FmtOutput(result.body)) } // Update bookkeeping delete(results, h.nextSequenceNumber) h.nextSequenceNumber += 1 } }
func (stream *stream) writeTo(sock *fullDuplex) { for { var buf []byte select { case <-stream.conn.kill: stream.closeLater(sock) return case buf = <-stream.out: } // This is the special "close" notification if buf == nil { sock.CloseWrite() return } n, err := sock.Write(buf) state.RecordWrite(n) log.Debugf("Wrote %s %d bytes: %#v", stream.l, n, string(buf)) if err != nil || n != len(buf) { log.Printf("writeTo %s: %s", stream.l, err) break } } // Close the other end of the connection time.Sleep(stream.conn.link.Delay()) stream.other(sock).CloseRead() }
func AfterParse() { conf.args = flag.Args() if conf.seed == 0 { conf.seed = randomSeed() } if conf.nodeCount < 2 { log.Fatal("You need at least two agents") } if conf.sqlcluster == "" { conf.sqlcluster = sqlclusterPath() if conf.sqlcluster == "" { log.Fatalf("No `sqlcluster' binary provided, and could not find one heuristically. Please provide a `-r <sqlcluster>' argument, or move your stripe-ctf.com/sqlcluster Go package into your GOPATH.", scriptDir()) } else { log.Printf("Found sqlcluster binary at %s", conf.sqlcluster) } } if conf.containerRoot == "" { conf.containerRoot = conf.root } if conf.containerIds == "" { conf.parsedContainerIds = make([]string, 0) } else { conf.parsedContainerIds = strings.Split(conf.containerIds, ",") } log.SetVerbose(conf.verbose) log.Printf("Using seed %v", conf.seed) // Always seed the default RNG. It controls comparatively little, so // don't worry about exposing it rand.Seed(randomSeed()) conf.wg = exit.NewWaitGroup() conf.gotRequest = make(chan int, 1) recordConfig() }
func (h *Harness) issueQuery(reqid int, node *agent.Agent, query string) bool { log.Debugf("[harness] Making request to %v: %#v", node, query) b := strings.NewReader(query) url := node.ConnectionString + "/sql" start := time.Now() resp, err := unixClient.Post(url, "application/octet-stream", b) end := time.Now() if err != nil { log.Printf("[harness] Sleeping 100ms after request error from %s (in response to %#v): %s", node, query, err) return false } body, err := ioutil.ReadAll(resp.Body) resp.Body.Close() if err != nil { log.Printf("[harness] Error reading body from %v (in response to %#v): %s", node, query, err) } if resp.StatusCode != 200 { log.Printf("[harness] Sleeping 100ms after HTTP %d status code from %s (in response to %#v): %s", resp.StatusCode, node, query, body) return false } log.Debugf("[harness] Received response to %v (%#v): %s", node, query, body) h.result <- &result{ reqid: reqid, node: node, query: query, start: start, end: end, resp: body, } return true }
func (sql *SQL) Execute(tag string, command string) (*Output, error) { // TODO: make sure I can catch non-lock issuez sql.mutex.Lock() defer sql.mutex.Unlock() defer func() { sql.sequenceNumber += 1 }() if tag == "primary" || log.Verbose() { log.Printf("[%s] [%d] Executing %#v", tag, sql.sequenceNumber, command) } subprocess := exec.Command("sqlite3", sql.path) subprocess.Stdin = strings.NewReader(command + ";") var stdout, stderr bytes.Buffer subprocess.Stdout = &stdout subprocess.Stderr = &stderr if err := subprocess.Start(); err != nil { log.Panic(err) } var o, e []byte if err := subprocess.Wait(); err != nil { exitstatus := getExitstatus(err) switch true { case exitstatus < 0: log.Panic(err) case exitstatus == 1: fallthrough case exitstatus == 2: o = stderr.Bytes() e = nil } } else { o = stdout.Bytes() e = stderr.Bytes() } output := &Output{ Stdout: o, Stderr: e, SequenceNumber: sql.sequenceNumber, } return output, nil }
// Rules for sequence numbers: // // - Gaps are temporarily OK, but not in the long run. func (h *Harness) resultHandler() { results := make(map[int]*result) for { result := <-h.result sequenceNumber, body, err := h.parseResponse(result.resp) if err != nil { h.lose(err.Error()) return } result.body = body if sequenceNumber < h.nextSequenceNumber { h.losef(`[%d] Received an already-processed sequence number from %v in response to %s Output: %s`, sequenceNumber, result.node, result.query, util.FmtOutput(result.resp)) return } if old, ok := results[sequenceNumber]; ok { h.losef(`[%d] Received a still-pending sequence number from %v in response to %s Output: %s This sequence number was originally received in response to %s Original output: %s`, sequenceNumber, result.node, result.query, util.FmtOutput(result.resp), old.query, util.FmtOutput(old.resp)) return } if sequenceNumber > h.nextSequenceNumber { log.Printf("[%d] Result from %v waiting on sequence number %d", sequenceNumber, result.node, h.nextSequenceNumber) } // Notify SPOF monkey that we got a valid request select { case state.GotRequest() <- result.reqid: default: } results[sequenceNumber] = result h.processPending(results) } }
func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, ` Usage: %s [options] [...args] Octopus will run your sqlcluster and issue queries to it. It simulates a lossy network, and to make things even more fun, will turn loose the following chaotic monkeys: NET MONKEYS - latency: adjusts base latency between nodes - jitter: adjusts latency variance between nodes - lagsplit: spikes latency along a network partition - link: destroys individual links between nodes - netsplit: destroys links along a network partition NODE MONKEYS - freeze: stops nodes (by sending SIGTSTP) - murder: ruthlessly murders nodes (by sending SIGTERM) Any positional arguments given to Octopus will be passed through to SQLCluster. This should be mostly useful for passing a -v flag, like so: ./octopus -- -v OPTIONS: `, os.Args[0]) flag.PrintDefaults() } state.AddFlags() flag.Parse() state.AfterParse() // Handle SIGINT sigchan := make(chan os.Signal) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) d := director.NewDirector() if state.Dryrun() { d.Dryrun() return } // Create the working directory if err := os.MkdirAll(state.Root(), os.ModeDir|0755); err != nil { log.Fatal(err) } d.Start() go func() { select { case <-sigchan: log.Println("Terminating due to signal") case <-state.WaitGroup().Quit: // Someone else requested an exit case <-time.After(state.Duration()): // Time's up! log.Printf("The allotted %s have elapsed. Exiting!", state.Duration()) } state.WaitGroup().Exit() }() h := harness.New(d.Agents()) h.Start() d.StartMonkeys() <-state.WaitGroup().Quit if state.Write() != "" { results := state.JSONResults() if err := ioutil.WriteFile(state.Write(), results, 0755); err != nil { log.Fatalf("Could not write resuts: %s", err) } } else { results := state.PrettyPrintResults() log.Println(results) } state.WaitGroup().Exit() }