func (src *NextReaderSource) Run() error { //This operator always stops the read nexter before exiting. //But can't defer here since in the case of a hardstop readnexter.Stop() was already called defer src.CloseOutput() var count uint32 count = 0 slog.Debugf("Reading up to %d %s", src.MaxItems, " tuples") for { b, eofReached, err := src.readnexter.ReadNext() //if I've been stopped, exit no matter what I've read select { case <-src.StopNotifier: //In this case readNexter was stopped return nil default: } if err != nil { slog.Errorf("Reader encountered error %v", err) src.stopReadNexter.Do(src.readnexter.Stop) return err } else if len(b) > 0 { count++ src.Out() <- b } if eofReached || (count >= src.MaxItems) { slog.Debugf("Got eof in Next Reader Source %d, %d", count, src.MaxItems) src.stopReadNexter.Do(src.readnexter.Stop) return nil } } }
func (s *WeightedEra) NormalizeAndPopulateMap() { total := float32(0) scalar := float32(1) for _, n := range s.nodes { wn := n.(*WeightedNode) total += float32(wn.weight) } if total == 0 { slog.Errorf("Total Node Weight 0") return } slog.Debugf("Total Node Weight %f", total) if total < MAX_WEIGHT { // Scale weights up scalar = MAX_WEIGHT / total total = MAX_WEIGHT } lastPosit := 0 for _, n := range s.nodes { wn := n.(*WeightedNode) wn.weight = uint32(((float32(wn.weight) * scalar) / total) * MAX_WEIGHT) slog.Debugf("New Weight %d", wn.weight) for i := lastPosit; uint32(i) < wn.weight && i < MAX_WEIGHT; i++ { s.nodeMap[i] = wn lastPosit++ } } return }
func (c *DynamicBBManager) keepErasCurrent() { for { time.Sleep(60 * time.Second) slog.Debugf("Updating to new era") err := c.pullLatestEra() if err != nil { slog.Errorf("Cannot get a valid era %v", err) } } }
func (p lengthDelimMultiPartValueWriter) writeValue(msgs [][]byte, writer io.Writer) error { total := 0 for _, msg := range msgs { total += len(msg) } err := binary.Write(writer, binary.LittleEndian, uint32(total)) if err != nil { return err } for _, msg := range msgs { _, err = writer.Write(msg) if err != nil { return err } } slog.Debugf("Write Returned %v, %v", time.Now(), time.Now().UnixNano()) return nil }
func (c *DynamicBBManager) pullLatestEra() (err error) { for _, url := range c.BBHosts { if resp, err := http.Get(url); err == nil { if bbbody, err := ioutil.ReadAll(resp.Body); err == nil { // Try parsing this. bbr := BBResult{} if err := json.Unmarshal(bbbody, &bbr); err == nil { ctime := time.Now() we := NewWeightedEra() for _, node := range bbr.Nodes { n := NewWeightedNode(node.Name, node.Ip, strconv.Itoa(node.Port), node.Disk_free, node.Load) slog.Debugf("Trasport LOG INFO %v", n) we.Add(n) } // Once all the nodes are in for this era, re-weight the Era we.NormalizeAndPopulateMap() c.Eras[ctime] = we c.CurrentTime = ctime c.ErasAdded = append(c.ErasAdded, ctime) // And Remove any super old eras if len(c.ErasAdded) > MAX_ERAS_SAVED { delete(c.Eras, c.ErasAdded[0]) c.ErasAdded = append(c.ErasAdded[:1], c.ErasAdded[2:]...) } // Once we have hit one BB server with no error, no need to try any others. break } else { slog.Errorf("Unmarshal Error %v", err) } } else { slog.Errorf("Read Error %v", err) } } else { slog.Errorf("Network GET Error %v", err) } } return }
func (src *Client) connect() error { defer func() { src.retries++ }() conn, err := net.Dial("tcp", src.addr) if err != nil { slog.Errorf("Cannot establish a connection with %s %v", src.addr, err) return err } wg_sub := &sync.WaitGroup{} defer wg_sub.Wait() rcvChData := make(chan stream.Object, 10) receiver := source.NewIOReaderSourceLengthDelim(conn) receiver.SetOut(rcvChData) rcvChCloseNotifier := make(chan bool) wg_sub.Add(1) go func() { defer wg_sub.Done() defer close(rcvChCloseNotifier) err := receiver.Run() if err != nil { slog.Errorf("Error in client reciever: %v", err) } }() //receiver will be closed by the sender after it is done sending. receiver closed via a hard stop. writeNotifier := stream.NewNonBlockingProcessedNotifier(2) sndChData := make(chan stream.Object, src.hwm) sndChCloseNotifier := make(chan bool) defer close(sndChData) sender := sink.NewMultiPartWriterSink(conn) sender.CompletedNotifier = writeNotifier sender.SetIn(sndChData) wg_sub.Add(1) go func() { defer receiver.Stop() //close receiver defer wg_sub.Done() defer close(sndChCloseNotifier) err := sender.Run() if err != nil { slog.Errorf("Error in client sender: %v", err) } }() //sender closed by closing the sndChData channel or by a hard stop if src.buf.Len() > 0 { leftover := src.buf.Reset() for i, value := range leftover { sendData(sndChData, value, i+1) } } timer := src.resetAckTimer() closing := false //defer log.Println("Exiting client loop") opName := stream.Name(src) writesNotCompleted := uint(0) for { upstreamCh := src.In() if !src.buf.CanAdd() || closing { //disable upstream listening upstreamCh = nil } if closing && src.buf.Len() == 0 { sendClose(sndChData, 100) return nil } select { case msg, ok := <-upstreamCh: if !ok { //softClose //make sure everything was sent closing = true } else { bytes := msg.([]byte) seq, err := src.buf.Add(bytes) if err != nil { slog.Fatalf("Error adding item to buffer %v", err) return err } sendData(sndChData, bytes, seq) writesNotCompleted += 1 metrics.Gm.Event(&opName) // These are batched //slog.Logf(logger.Levels.Debug, "Sent batch -- length %d seq %d", len(bytes), seq) } case cnt := <-writeNotifier.NotificationChannel(): writesNotCompleted -= cnt if timer == nil { slog.Debugf("Seting timer %v, %v", time.Now(), time.Now().UnixNano()) timer = src.resetAckTimer() } case obj, ok := <-rcvChData: slog.Debugf("in Rcv: %v", ok) if !ok { return errors.New("Connection to Server was Broken in Recieve Direction") } command, seq, _, err := parseMsg(obj.([]byte)) if err != nil { slog.Fatalf("%v", err) } if command == ACK { if src.processAck(seq) { timer = src.resetAckTimer() } } else { slog.Fatalf("Unknown Command: %v", command) } case <-rcvChCloseNotifier: //connection threw an eof to the reader? return errors.New("In Select: Recieve Closed") case <-sndChCloseNotifier: return errors.New("Connection to Server was Broken in Send Direction") case <-timer: return errors.New(fmt.Sprintf("Time Out Waiting For Ack, %d %v %v", len(rcvChData), time.Now(), time.Now().UnixNano())) case <-src.StopNotifier: sender.Stop() return nil } } }
func sendClose(sndCh chan<- stream.Object, seq int) { slog.Debugf("Sending Close %d", seq) sendMsg(sndCh, CLOSE, seq, []byte{}) }
func sendAck(sndCh chan<- stream.Object, seq int) { slog.Debugf("Sending back ack %d", seq) sendMsg(sndCh, ACK, seq, []byte{}) }
func (op *BatcherOperator) Run() error { defer close(op.Out()) /* batchExpired puts a lower bound on how often flushes occur */ var batchExpired <-chan time.Time batchExpired = nil //INVARIANT: if container.HasItems() then it will be flushed eventually //We create a state machine with 3 boolean states, state hi = container.HasItems(), wcb = op.DownstreamWillCallback(), bne = (batchExpired != nil) (batch not expired) //invariants required: // INVARIANT LIMITED_DRCB => repeated DRCB calls will eventually cause wcb == false // INVARIANT !wcb can only become wcb after a FLUSH // INVARIANT CAN FLUSH OR WAIT => either DownstreamWillCallback or DownstreamCanAcceptFlush is true //lets analyse cases where hi == true: // wcb && bne => // Case IN => wcb && bne [Case BE or DRCB will eventually happen] // Case BE => PROGRESS 1 || wcb && !bne // Case DRCB => wcb && bne [can't recurse indefinitely by LIMITED_DRCB] || !wcb && bne // wcb && !bne => // Case IN => wcb && !bne [case DRCB will eventually happen] // Case BE => impossible // Case DRCB => // DownstreamCanAcceptFlush => PROGRESS 2 // else: wcb && bne || wcb && !bne [can't recurse indef by LIMITED_DRCB] || !wcb && bne //!wcb && bne // case IN => !wcb && bne [case BE will eventually happen] // case BE => // !DownstreamCanAcceptFlush => impossible [INVARIANT CANFLUSH_OR_WAIT] // else => PROGRESS 2 //case DRCB => impossisible (!wcb) //!wcb && !bne => impossible (all cases disallow this) //liveness: has items => either batch_expired != nil or DownstreamWillCallback for { in := op.In() if op.container.IsFull() { if op.DownstreamCanAcceptFlush() { //PROGRESS 1 op.Flush() batchExpired = time.After(op.minWaitBetweenFlushes) } else { if !op.DownstreamWillCallback() && batchExpired == nil { panic("Batcher deadlocked. Should not happen") } in = nil } } select { //case IN case obj, ok := <-in: if ok { op.container.Add(obj) if !op.DownstreamWillCallback() && op.container.HasItems() && batchExpired == nil { //used by first item batchExpired = time.After(op.minWaitAfterFirstItem) } //IMPOSSIBLE: hi && !wcb && !bne } else { if op.container.HasItems() { op.LastFlush() } if op.container.HasItems() { slog.Fatalf("Last flush did not empty container, some stuff will never be sent") } slog.Debugf("Batch Operator ", op.name, " flushed ", op.total_flushes) return nil } //case BE case <-batchExpired: batchExpired = nil if op.DownstreamCanAcceptFlush() { //PROGRESS 1 op.Flush() batchExpired = time.After(op.minWaitBetweenFlushes) } if !op.DownstreamWillCallback() && op.container.HasItems() && batchExpired == nil { batchExpired = time.After(op.minWaitForLeftover) } //impossibe: hi && !wcb && !bne case <-op.StopNotifier: //INVARIANT and PROGRESS Violated. Hard Stop return nil //case DRCB case count := <-op.processedDownstream.NotificationChannel(): if op.outstanding == 0 { panic("Should never happen, will cause underflow") } op.outstanding -= count if op.DownstreamCanAcceptFlush() && op.container.HasItems() && batchExpired == nil { op.Flush() batchExpired = time.After(op.minWaitBetweenFlushes) } if !op.DownstreamWillCallback() && op.container.HasItems() && batchExpired == nil { batchExpired = time.After(op.minWaitForLeftover) } //impossibe: hi && !wcb && !bne } } }