Exemple #1
0
// SliceIQ is an infiniteQueue - Based on:
// github.com/kylelemons/iq
func (n *Queue) processQ() {

	log.Info("notification", "Process Q started")

recv:
	for {
		// Ensure that pending always has values so the select can
		// multiplex between the receiver and sender properly
		if len(n.Pending) == 0 {
			v, ok := <-n.In
			if !ok {
				// in is closed, flush values
				break
			}

			// We now have something to send
			n.Pending = append(n.Pending, v)
		}

		select {
		// Queue incoming values
		case v, ok := <-n.In:
			if !ok {
				// in is closed, flush values
				break recv
			}
			n.Pending = append(n.Pending, v)

			// Send queued values
		case n.next <- n.Pending[0]:
			n.Pending = n.Pending[1:]

		//stop closed, which means we need to exit without flushing
		case <-n.stop:
			log.Infof("notification", "Process Queue got stop, pending: %v", len(n.Pending))
			return
		}
	}

	// After in is closed, we may still have events to send
	log.Infof("notification", "Flushing queue. length: %v", len(n.Pending))
	for _, v := range n.Pending {
		select {
		case n.next <- v:
		case <-n.stop:
			//stop called...finish
			return
		}
	}

	//Lastly, we close the next channel to tell the notifier we are done
	close(n.next)
}
Exemple #2
0
// Builds the file list for the backup
// Listens for the cancel chanel to close to cancel walk
// Walks the file tree in JobPaths and sends any found file that isn't excluded on the return chan
// If there is an error, it sends the error on the error channel and returns
func buildBackupFileList(cancel <-chan struct{}, jobPaths []spec.BackupPath) (<-chan string, <-chan error) {

	paths := make(chan string)
	errc := make(chan error, 1)

	go func() {
		log.Debug("backupJob", "file list routine started")
		defer close(paths)

		for _, jobPath := range jobPaths {

			log.Debugf("backupJob", "Walking filepath: %v", jobPath)

			errc <- filepath.Walk(jobPath.Path, func(path string, info os.FileInfo, err error) error {

				log.Debugf("backupJob", "Walk Found: %v", path)

				if err != nil {
					return err
				}

				if !info.Mode().IsRegular() {
					return nil
				}

				if shouldExclude(path, jobPath.Excludes) {
					return nil
				}

				select {
				case paths <- path:
				case <-cancel:
					log.Info("backupJob", "Walk Canceled")
					return errors.New("Walk Canceled")
				}
				return nil
			})
		}
	}()

	return paths, errc
}
Exemple #3
0
// Run starts processing the q
func (n *Queue) Run() {
	log.Info("notification", "Starting")
	go n.processQ()
	go n.notify()
}
Exemple #4
0
//Finish closes the in channel and starts a queue flush
func (n *Queue) Finish(f *Notification) {
	log.Info("notification", "Received finish")
	n.In <- f
	close(n.In)
}
Exemple #5
0
// Notify send the actual data to Cooridnator
// No guarantee that the data will be sent sequentually if errors occur when making the attempts
func (n *Queue) notify() {
	log.Info("notification", "Notify started")
	t := try.New(3)
	timeoutNum := 1
	var waitTime time.Duration
notify:
	for {
		select {
		case msg, ok := <-n.next:
			if !ok {
				//next was closed, we are done
				n.Finished <- true
				break notify
			}

			err := t.Do(func(attempt int) (bool, error) {
				sig, err := n.keyManager.Sign(string(msg.Payload))
				if err != nil {
					//log error
					log.Errorf("notification", "Unable to sign message: %v", err)
					return false, err
				}
				req := &httpapi.APIRequest{Address: n.Coordinator.Address, Body: msg.Payload, Signature: sig}

				_, err = req.POST(msg.Endpoint)
				if err != nil {
					log.Errorf("notification", "Unable to notify: %v", err)
					return false, err
				}

				return false, nil
			})

			if try.IsMaxRetries(err) {

				//requeue the message
				n.In <- msg

				//Communication issues, sleep then try again
				if timeoutNum < 10 {
					waitTime = time.Second * time.Duration(timeoutNum*timeoutNum)
				} else if timeoutNum < 15 {
					waitTime = time.Minute * time.Duration(timeoutNum)
				} else if timeoutNum < 21 {
					waitTime = time.Minute * time.Duration(timeoutNum*timeoutNum)
				} else {
					waitTime = time.Hour * time.Duration(timeoutNum)
					timeoutNum--
				}

				n.waitTimer = time.NewTimer(waitTime)
				<-n.waitTimer.C
				timeoutNum++
			}
		case <-n.stop:
			//Quit
			log.Info("notify", "Notify received stop call")
			break notify
		}
	}
}