func (src *Client) Run() error {
src.running = true
defer func() {
src.running = false
}()
metrics.Gm.Register(stream.Name(src))
go func(op string, s *Client) { // Update the queue depth on input for each phase
for {
metrics.Gm.Update(&op, s.GetInDepth())
time.Sleep(1 * time.Second)
}
}(stream.Name(src), src)
for src.retries < RETRY_MAX {
err := src.connect()
if err == nil {
slog.Warnf("Connection failed without error")
return err
} else {
slog.Errorf("Connection failed with error, retrying: %s", err)
time.Sleep(1 * time.Second)
}
}
slog.Errorf("Connection failed retries exceeded. Leftover: %d", src.buf.Len())
return nil //>>>>>>>>>>>>>>???????????????????????
}
func (s *WeightedEra) NormalizeAndPopulateMap() {
total := float32(0)
scalar := float32(1)
for _, n := range s.nodes {
wn := n.(*WeightedNode)
total += float32(wn.weight)
}
if total == 0 {
slog.Errorf("Total Node Weight 0")
return
}
slog.Debugf("Total Node Weight %f", total)
if total < MAX_WEIGHT {
// Scale weights up
scalar = MAX_WEIGHT / total
total = MAX_WEIGHT
}
lastPosit := 0
for _, n := range s.nodes {
wn := n.(*WeightedNode)
wn.weight = uint32(((float32(wn.weight) * scalar) / total) * MAX_WEIGHT)
slog.Debugf("New Weight %d", wn.weight)
for i := lastPosit; uint32(i) < wn.weight && i < MAX_WEIGHT; i++ {
s.nodeMap[i] = wn
lastPosit++
}
}
return
}
func (src *NextReaderSource) Run() error {
//This operator always stops the read nexter before exiting.
//But can't defer here since in the case of a hardstop readnexter.Stop() was already called
defer src.CloseOutput()
var count uint32
count = 0
slog.Debugf("Reading up to %d %s", src.MaxItems, " tuples")
for {
b, eofReached, err := src.readnexter.ReadNext()
//if I've been stopped, exit no matter what I've read
select {
case <-src.StopNotifier:
//In this case readNexter was stopped
return nil
default:
}
if err != nil {
slog.Errorf("Reader encountered error %v", err)
src.stopReadNexter.Do(src.readnexter.Stop)
return err
} else if len(b) > 0 {
count++
src.Out() <- b
}
if eofReached || (count >= src.MaxItems) {
slog.Debugf("Got eof in Next Reader Source %d, %d", count, src.MaxItems)
src.stopReadNexter.Do(src.readnexter.Stop)
return nil
}
}
}
func (e *Executor) Exec(sql string, args ...interface{}) driver.Result {
res, err := e.ExecErr(sql, args...)
if err != nil {
slog.Errorf("Sql: %v Err: %v", sql, err)
}
return res
}
func (sink MultiPartWriterSink) Run() error {
defer func() {
closer, ok := sink.writer.(io.Closer)
if ok {
closer.Close()
}
}()
for {
select {
case msg, ok := <-sink.In():
if ok {
if err := sink.writeValue(msg.([][]byte), sink.writer); err != nil {
slog.Errorf("Writer got error %v", err)
return err
}
if sink.CompletedNotifier != nil {
sink.CompletedNotifier.Notify(1)
}
} else {
return nil
}
case <-sink.StopNotifier:
return nil
}
}
}
func JsonGeneralDecoder() func([]byte, interface{}) {
fn := func(input []byte, to_populate interface{}) {
err := json.Unmarshal(input, to_populate)
if err != nil {
slog.Errorf("Error unmarshaling json: %v %v\n", err.Error(), string(input))
}
}
return fn
}
func (c *DynamicBBManager) keepErasCurrent() {
for {
time.Sleep(60 * time.Second)
slog.Debugf("Updating to new era")
err := c.pullLatestEra()
if err != nil {
slog.Errorf("Cannot get a valid era %v", err)
}
}
}
func (c *DynamicBBManager) pullLatestEra() (err error) {
for _, url := range c.BBHosts {
if resp, err := http.Get(url); err == nil {
if bbbody, err := ioutil.ReadAll(resp.Body); err == nil {
// Try parsing this.
bbr := BBResult{}
if err := json.Unmarshal(bbbody, &bbr); err == nil {
ctime := time.Now()
we := NewWeightedEra()
for _, node := range bbr.Nodes {
n := NewWeightedNode(node.Name, node.Ip, strconv.Itoa(node.Port), node.Disk_free, node.Load)
slog.Debugf("Trasport LOG INFO %v", n)
we.Add(n)
}
// Once all the nodes are in for this era, re-weight the Era
we.NormalizeAndPopulateMap()
c.Eras[ctime] = we
c.CurrentTime = ctime
c.ErasAdded = append(c.ErasAdded, ctime)
// And Remove any super old eras
if len(c.ErasAdded) > MAX_ERAS_SAVED {
delete(c.Eras, c.ErasAdded[0])
c.ErasAdded = append(c.ErasAdded[:1], c.ErasAdded[2:]...)
}
// Once we have hit one BB server with no error, no need to try any others.
break
} else {
slog.Errorf("Unmarshal Error %v", err)
}
} else {
slog.Errorf("Read Error %v", err)
}
} else {
slog.Errorf("Network GET Error %v", err)
}
}
return
}
func statsSender(metricsAddr *string, processName *string) {
rep, err := zmq.NewSocket(zmq.REP)
if err != nil {
slog.Errorf("Stats Sender error: %v", err.Error())
return
}
defer rep.Close()
err = rep.Bind(*metricsAddr)
if err != nil {
slog.Errorf("Stats Sender error: %v", err.Error())
return
}
slog.Infof("Stats sender, listening on %s", *metricsAddr)
// Loop, printing the stats on request
for {
_, err := rep.Recv(0)
if err != nil {
slog.Errorf("%v", err.Error())
} else {
timestamp := time.Now().Unix() - Gm.StartTime
dBag := statsPkg{*processName, timestamp, map[string]interface{}{}}
for k, v := range Gm.OpGroups {
dBag.OpMetrics[k] = map[string]int64{"Events": v.Events.Count(), "Errors": v.Errors.Count(), "Queue": v.QueueLength.Value()}
}
stats, err := json.Marshal(dBag)
if err == nil {
_, err = rep.SendBytes(stats, zmq.DONTWAIT)
if err != nil {
slog.Errorf("%v", err.Error())
}
}
}
}
}
func (r *FailSilentRunner) AsyncRun(op Operator) {
if r.finished {
panic("FailSilentRunner finished")
}
r.wg.Add(1)
r.finOnce.Do(func() {
if r.finHandler != nil {
go func() {
r.wg.Wait()
r.finHandler()
r.finished = true
}()
}
})
go func() {
defer r.wg.Done()
err := op.Run()
slog.Errorf("The following operator exited (%v, %v) in runner %s: %v", op, reflect.TypeOf(op), r.Name, err)
if err != nil {
//slog.Errorf("Got an err from a child (%v, %v) in runner %s: %v", op, reflect.TypeOf(op), r.Name, err)
if r.errorHandler != nil {
r.errorHandler(err)
}
r.firstErrorOnce.Do(func() {
r.firstError = err
})
}
if r.opCloseHandler != nil {
r.opCloseHandler(op, err)
}
}()
}
func (src *Client) connect() error {
defer func() {
src.retries++
}()
conn, err := net.Dial("tcp", src.addr)
if err != nil {
slog.Errorf("Cannot establish a connection with %s %v", src.addr, err)
return err
}
wg_sub := &sync.WaitGroup{}
defer wg_sub.Wait()
rcvChData := make(chan stream.Object, 10)
receiver := source.NewIOReaderSourceLengthDelim(conn)
receiver.SetOut(rcvChData)
rcvChCloseNotifier := make(chan bool)
wg_sub.Add(1)
go func() {
defer wg_sub.Done()
defer close(rcvChCloseNotifier)
err := receiver.Run()
if err != nil {
slog.Errorf("Error in client reciever: %v", err)
}
}()
//receiver will be closed by the sender after it is done sending. receiver closed via a hard stop.
writeNotifier := stream.NewNonBlockingProcessedNotifier(2)
sndChData := make(chan stream.Object, src.hwm)
sndChCloseNotifier := make(chan bool)
defer close(sndChData)
sender := sink.NewMultiPartWriterSink(conn)
sender.CompletedNotifier = writeNotifier
sender.SetIn(sndChData)
wg_sub.Add(1)
go func() {
defer receiver.Stop() //close receiver
defer wg_sub.Done()
defer close(sndChCloseNotifier)
err := sender.Run()
if err != nil {
slog.Errorf("Error in client sender: %v", err)
}
}()
//sender closed by closing the sndChData channel or by a hard stop
if src.buf.Len() > 0 {
leftover := src.buf.Reset()
for i, value := range leftover {
sendData(sndChData, value, i+1)
}
}
timer := src.resetAckTimer()
closing := false
//defer log.Println("Exiting client loop")
opName := stream.Name(src)
writesNotCompleted := uint(0)
for {
upstreamCh := src.In()
if !src.buf.CanAdd() || closing {
//disable upstream listening
upstreamCh = nil
}
if closing && src.buf.Len() == 0 {
sendClose(sndChData, 100)
return nil
}
select {
case msg, ok := <-upstreamCh:
if !ok {
//softClose
//make sure everything was sent
closing = true
} else {
bytes := msg.([]byte)
seq, err := src.buf.Add(bytes)
if err != nil {
slog.Fatalf("Error adding item to buffer %v", err)
return err
}
sendData(sndChData, bytes, seq)
writesNotCompleted += 1
metrics.Gm.Event(&opName) // These are batched
//slog.Logf(logger.Levels.Debug, "Sent batch -- length %d seq %d", len(bytes), seq)
}
case cnt := <-writeNotifier.NotificationChannel():
writesNotCompleted -= cnt
if timer == nil {
slog.Debugf("Seting timer %v, %v", time.Now(), time.Now().UnixNano())
timer = src.resetAckTimer()
}
case obj, ok := <-rcvChData:
slog.Debugf("in Rcv: %v", ok)
if !ok {
return errors.New("Connection to Server was Broken in Recieve Direction")
}
command, seq, _, err := parseMsg(obj.([]byte))
if err != nil {
slog.Fatalf("%v", err)
}
if command == ACK {
if src.processAck(seq) {
timer = src.resetAckTimer()
}
} else {
slog.Fatalf("Unknown Command: %v", command)
}
case <-rcvChCloseNotifier:
//connection threw an eof to the reader?
return errors.New("In Select: Recieve Closed")
case <-sndChCloseNotifier:
return errors.New("Connection to Server was Broken in Send Direction")
case <-timer:
return errors.New(fmt.Sprintf("Time Out Waiting For Ack, %d %v %v", len(rcvChData), time.Now(), time.Now().UnixNano()))
case <-src.StopNotifier:
sender.Stop()
return nil
}
}
}
