示例#1
0
func (src *Client) Run() error {
	src.running = true
	defer func() {
		src.running = false
	}()

	metrics.Gm.Register(stream.Name(src))
	go func(op string, s *Client) { // Update the queue depth on input for each phase
		for {
			metrics.Gm.Update(&op, s.GetInDepth())
			time.Sleep(1 * time.Second)
		}
	}(stream.Name(src), src)

	for src.retries < RETRY_MAX {
		err := src.connect()
		if err == nil {
			slog.Warnf("Connection failed without error")
			return err
		} else {
			slog.Errorf("Connection failed with error, retrying: %s", err)
			time.Sleep(1 * time.Second)
		}
	}
	slog.Errorf("Connection failed retries exceeded. Leftover: %d", src.buf.Len())
	return nil //>>>>>>>>>>>>>>???????????????????????
}
示例#2
0
文件: era.go 项目: sgichohi/go-stream
func (s *WeightedEra) NormalizeAndPopulateMap() {
	total := float32(0)
	scalar := float32(1)
	for _, n := range s.nodes {
		wn := n.(*WeightedNode)
		total += float32(wn.weight)
	}

	if total == 0 {
		slog.Errorf("Total Node Weight 0")
		return
	}

	slog.Debugf("Total Node Weight %f", total)
	if total < MAX_WEIGHT {
		// Scale weights up
		scalar = MAX_WEIGHT / total
		total = MAX_WEIGHT
	}

	lastPosit := 0
	for _, n := range s.nodes {
		wn := n.(*WeightedNode)
		wn.weight = uint32(((float32(wn.weight) * scalar) / total) * MAX_WEIGHT)
		slog.Debugf("New Weight %d", wn.weight)
		for i := lastPosit; uint32(i) < wn.weight && i < MAX_WEIGHT; i++ {
			s.nodeMap[i] = wn
			lastPosit++
		}
	}

	return
}
示例#3
0
func (src *NextReaderSource) Run() error {
	//This operator always stops the read nexter before exiting.
	//But can't defer here since in the case of a hardstop readnexter.Stop() was already called

	defer src.CloseOutput()
	var count uint32
	count = 0
	slog.Debugf("Reading up to %d %s", src.MaxItems, " tuples")
	for {
		b, eofReached, err := src.readnexter.ReadNext()
		//if I've been stopped, exit no matter what I've read
		select {
		case <-src.StopNotifier:
			//In this case readNexter was stopped
			return nil
		default:
		}
		if err != nil {
			slog.Errorf("Reader encountered error %v", err)
			src.stopReadNexter.Do(src.readnexter.Stop)
			return err
		} else if len(b) > 0 {
			count++
			src.Out() <- b
		}
		if eofReached || (count >= src.MaxItems) {
			slog.Debugf("Got eof in Next Reader Source %d, %d", count, src.MaxItems)
			src.stopReadNexter.Do(src.readnexter.Stop)
			return nil
		}
	}

}
示例#4
0
func (e *Executor) Exec(sql string, args ...interface{}) driver.Result {
	res, err := e.ExecErr(sql, args...)
	if err != nil {
		slog.Errorf("Sql: %v Err: %v", sql, err)
	}
	return res
}
示例#5
0
func (sink MultiPartWriterSink) Run() error {
	defer func() {
		closer, ok := sink.writer.(io.Closer)
		if ok {
			closer.Close()
		}
	}()

	for {
		select {
		case msg, ok := <-sink.In():
			if ok {
				if err := sink.writeValue(msg.([][]byte), sink.writer); err != nil {
					slog.Errorf("Writer got error %v", err)
					return err
				}
				if sink.CompletedNotifier != nil {
					sink.CompletedNotifier.Notify(1)
				}
			} else {
				return nil
			}
		case <-sink.StopNotifier:
			return nil
		}

	}

}
示例#6
0
func JsonGeneralDecoder() func([]byte, interface{}) {
	fn := func(input []byte, to_populate interface{}) {
		err := json.Unmarshal(input, to_populate)
		if err != nil {
			slog.Errorf("Error unmarshaling json: %v %v\n", err.Error(), string(input))
		}
	}
	return fn
}
示例#7
0
func (c *DynamicBBManager) keepErasCurrent() {
	for {
		time.Sleep(60 * time.Second)
		slog.Debugf("Updating to new era")
		err := c.pullLatestEra()
		if err != nil {
			slog.Errorf("Cannot get a valid era %v", err)
		}
	}
}
示例#8
0
func (c *DynamicBBManager) pullLatestEra() (err error) {
	for _, url := range c.BBHosts {
		if resp, err := http.Get(url); err == nil {
			if bbbody, err := ioutil.ReadAll(resp.Body); err == nil {
				// Try parsing this.
				bbr := BBResult{}
				if err := json.Unmarshal(bbbody, &bbr); err == nil {
					ctime := time.Now()

					we := NewWeightedEra()
					for _, node := range bbr.Nodes {
						n := NewWeightedNode(node.Name, node.Ip, strconv.Itoa(node.Port), node.Disk_free, node.Load)
						slog.Debugf("Trasport LOG INFO %v", n)
						we.Add(n)
					}

					// Once all the nodes are in for this era, re-weight the Era
					we.NormalizeAndPopulateMap()
					c.Eras[ctime] = we
					c.CurrentTime = ctime
					c.ErasAdded = append(c.ErasAdded, ctime)

					// And Remove any super old eras
					if len(c.ErasAdded) > MAX_ERAS_SAVED {
						delete(c.Eras, c.ErasAdded[0])
						c.ErasAdded = append(c.ErasAdded[:1], c.ErasAdded[2:]...)
					}

					// Once we have hit one BB server with no error, no need to try any others.
					break
				} else {
					slog.Errorf("Unmarshal Error %v", err)
				}
			} else {
				slog.Errorf("Read Error %v", err)
			}
		} else {
			slog.Errorf("Network GET Error %v", err)
		}
	}
	return
}
示例#9
0
func statsSender(metricsAddr *string, processName *string) {
	rep, err := zmq.NewSocket(zmq.REP)
	if err != nil {
		slog.Errorf("Stats Sender error: %v", err.Error())
		return
	}
	defer rep.Close()
	err = rep.Bind(*metricsAddr)
	if err != nil {
		slog.Errorf("Stats Sender error: %v", err.Error())
		return
	}

	slog.Infof("Stats sender, listening on %s", *metricsAddr)

	// Loop, printing the stats on request
	for {
		_, err := rep.Recv(0)
		if err != nil {
			slog.Errorf("%v", err.Error())
		} else {
			timestamp := time.Now().Unix() - Gm.StartTime
			dBag := statsPkg{*processName, timestamp, map[string]interface{}{}}
			for k, v := range Gm.OpGroups {
				dBag.OpMetrics[k] = map[string]int64{"Events": v.Events.Count(), "Errors": v.Errors.Count(), "Queue": v.QueueLength.Value()}
			}
			stats, err := json.Marshal(dBag)
			if err == nil {
				_, err = rep.SendBytes(stats, zmq.DONTWAIT)
				if err != nil {
					slog.Errorf("%v", err.Error())
				}
			}
		}
	}
}
示例#10
0
func (r *FailSilentRunner) AsyncRun(op Operator) {
	if r.finished {
		panic("FailSilentRunner finished")
	}

	r.wg.Add(1)

	r.finOnce.Do(func() {
		if r.finHandler != nil {
			go func() {
				r.wg.Wait()
				r.finHandler()
				r.finished = true
			}()
		}
	})

	go func() {
		defer r.wg.Done()
		err := op.Run()
		slog.Errorf("The following operator exited (%v, %v) in runner %s: %v", op, reflect.TypeOf(op), r.Name, err)
		if err != nil {
			//slog.Errorf("Got an err from a child (%v, %v) in runner %s: %v", op, reflect.TypeOf(op), r.Name, err)
			if r.errorHandler != nil {
				r.errorHandler(err)
			}
			r.firstErrorOnce.Do(func() {
				r.firstError = err
			})
		}

		if r.opCloseHandler != nil {
			r.opCloseHandler(op, err)
		}
	}()
}
示例#11
0
func (src *Client) connect() error {
	defer func() {
		src.retries++
	}()

	conn, err := net.Dial("tcp", src.addr)
	if err != nil {
		slog.Errorf("Cannot establish a connection with %s %v", src.addr, err)
		return err
	}

	wg_sub := &sync.WaitGroup{}
	defer wg_sub.Wait()

	rcvChData := make(chan stream.Object, 10)
	receiver := source.NewIOReaderSourceLengthDelim(conn)
	receiver.SetOut(rcvChData)
	rcvChCloseNotifier := make(chan bool)
	wg_sub.Add(1)
	go func() {
		defer wg_sub.Done()
		defer close(rcvChCloseNotifier)
		err := receiver.Run()
		if err != nil {
			slog.Errorf("Error in client reciever: %v", err)
		}
	}()
	//receiver will be closed by the sender after it is done sending. receiver closed via a hard stop.

	writeNotifier := stream.NewNonBlockingProcessedNotifier(2)
	sndChData := make(chan stream.Object, src.hwm)
	sndChCloseNotifier := make(chan bool)
	defer close(sndChData)
	sender := sink.NewMultiPartWriterSink(conn)
	sender.CompletedNotifier = writeNotifier
	sender.SetIn(sndChData)
	wg_sub.Add(1)
	go func() {
		defer receiver.Stop() //close receiver
		defer wg_sub.Done()
		defer close(sndChCloseNotifier)
		err := sender.Run()
		if err != nil {
			slog.Errorf("Error in client sender: %v", err)
		}
	}()
	//sender closed by closing the sndChData channel or by a hard stop

	if src.buf.Len() > 0 {
		leftover := src.buf.Reset()
		for i, value := range leftover {
			sendData(sndChData, value, i+1)
		}
	}

	timer := src.resetAckTimer()

	closing := false

	//defer log.Println("Exiting client loop")
	opName := stream.Name(src)
	writesNotCompleted := uint(0)
	for {
		upstreamCh := src.In()
		if !src.buf.CanAdd() || closing {
			//disable upstream listening
			upstreamCh = nil
		}
		if closing && src.buf.Len() == 0 {
			sendClose(sndChData, 100)
			return nil
		}
		select {
		case msg, ok := <-upstreamCh:
			if !ok {
				//softClose
				//make sure everything was sent
				closing = true
			} else {
				bytes := msg.([]byte)
				seq, err := src.buf.Add(bytes)
				if err != nil {
					slog.Fatalf("Error adding item to buffer %v", err)
					return err
				}
				sendData(sndChData, bytes, seq)
				writesNotCompleted += 1
				metrics.Gm.Event(&opName) // These are batched
				//slog.Logf(logger.Levels.Debug, "Sent batch -- length %d seq %d", len(bytes), seq)
			}
		case cnt := <-writeNotifier.NotificationChannel():
			writesNotCompleted -= cnt
			if timer == nil {
				slog.Debugf("Seting timer %v, %v", time.Now(), time.Now().UnixNano())
				timer = src.resetAckTimer()
			}
		case obj, ok := <-rcvChData:
			slog.Debugf("in Rcv: %v", ok)
			if !ok {
				return errors.New("Connection to Server was Broken in Recieve Direction")
			}

			command, seq, _, err := parseMsg(obj.([]byte))
			if err != nil {
				slog.Fatalf("%v", err)
			}
			if command == ACK {
				if src.processAck(seq) {
					timer = src.resetAckTimer()
				}
			} else {
				slog.Fatalf("Unknown Command: %v", command)
			}
		case <-rcvChCloseNotifier:
			//connection threw an eof to the reader?
			return errors.New("In Select: Recieve Closed")
		case <-sndChCloseNotifier:
			return errors.New("Connection to Server was Broken in Send Direction")
		case <-timer:
			return errors.New(fmt.Sprintf("Time Out Waiting For Ack, %d %v %v", len(rcvChData), time.Now(), time.Now().UnixNano()))
		case <-src.StopNotifier:
			sender.Stop()
			return nil
		}
	}
}