// SliceIQ is an infiniteQueue - Based on:
// github.com/kylelemons/iq
func (n *Queue) processQ() {
log.Info("notification", "Process Q started")
recv:
for {
// Ensure that pending always has values so the select can
// multiplex between the receiver and sender properly
if len(n.Pending) == 0 {
v, ok := <-n.In
if !ok {
// in is closed, flush values
break
}
// We now have something to send
n.Pending = append(n.Pending, v)
}
select {
// Queue incoming values
case v, ok := <-n.In:
if !ok {
// in is closed, flush values
break recv
}
n.Pending = append(n.Pending, v)
// Send queued values
case n.next <- n.Pending[0]:
n.Pending = n.Pending[1:]
//stop closed, which means we need to exit without flushing
case <-n.stop:
log.Infof("notification", "Process Queue got stop, pending: %v", len(n.Pending))
return
}
}
// After in is closed, we may still have events to send
log.Infof("notification", "Flushing queue. length: %v", len(n.Pending))
for _, v := range n.Pending {
select {
case n.next <- v:
case <-n.stop:
//stop called...finish
return
}
}
//Lastly, we close the next channel to tell the notifier we are done
close(n.next)
}
// Builds the file list for the backup
// Listens for the cancel chanel to close to cancel walk
// Walks the file tree in JobPaths and sends any found file that isn't excluded on the return chan
// If there is an error, it sends the error on the error channel and returns
func buildBackupFileList(cancel <-chan struct{}, jobPaths []spec.BackupPath) (<-chan string, <-chan error) {
paths := make(chan string)
errc := make(chan error, 1)
go func() {
log.Debug("backupJob", "file list routine started")
defer close(paths)
for _, jobPath := range jobPaths {
log.Debugf("backupJob", "Walking filepath: %v", jobPath)
errc <- filepath.Walk(jobPath.Path, func(path string, info os.FileInfo, err error) error {
log.Debugf("backupJob", "Walk Found: %v", path)
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
if shouldExclude(path, jobPath.Excludes) {
return nil
}
select {
case paths <- path:
case <-cancel:
log.Info("backupJob", "Walk Canceled")
return errors.New("Walk Canceled")
}
return nil
})
}
}()
return paths, errc
}
// Run starts processing the q
func (n *Queue) Run() {
log.Info("notification", "Starting")
go n.processQ()
go n.notify()
}
//Finish closes the in channel and starts a queue flush
func (n *Queue) Finish(f *Notification) {
log.Info("notification", "Received finish")
n.In <- f
close(n.In)
}
// Notify send the actual data to Cooridnator
// No guarantee that the data will be sent sequentually if errors occur when making the attempts
func (n *Queue) notify() {
log.Info("notification", "Notify started")
t := try.New(3)
timeoutNum := 1
var waitTime time.Duration
notify:
for {
select {
case msg, ok := <-n.next:
if !ok {
//next was closed, we are done
n.Finished <- true
break notify
}
err := t.Do(func(attempt int) (bool, error) {
sig, err := n.keyManager.Sign(string(msg.Payload))
if err != nil {
//log error
log.Errorf("notification", "Unable to sign message: %v", err)
return false, err
}
req := &httpapi.APIRequest{Address: n.Coordinator.Address, Body: msg.Payload, Signature: sig}
_, err = req.POST(msg.Endpoint)
if err != nil {
log.Errorf("notification", "Unable to notify: %v", err)
return false, err
}
return false, nil
})
if try.IsMaxRetries(err) {
//requeue the message
n.In <- msg
//Communication issues, sleep then try again
if timeoutNum < 10 {
waitTime = time.Second * time.Duration(timeoutNum*timeoutNum)
} else if timeoutNum < 15 {
waitTime = time.Minute * time.Duration(timeoutNum)
} else if timeoutNum < 21 {
waitTime = time.Minute * time.Duration(timeoutNum*timeoutNum)
} else {
waitTime = time.Hour * time.Duration(timeoutNum)
timeoutNum--
}
n.waitTimer = time.NewTimer(waitTime)
<-n.waitTimer.C
timeoutNum++
}
case <-n.stop:
//Quit
log.Info("notify", "Notify received stop call")
break notify
}
}
}