func (sc *StartupContext) Run() error {
//process top halves of local updates so the files are saved off at least locally
localEvents := []*asink.Event{}
initialWalkIncomplete := true
for initialWalkIncomplete {
select {
case event := <-sc.localUpdatesChan:
//process top half of local event
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
return err
}
}
}
if event.LocalStatus&asink.DISCARDED == 0 {
localEvents = append(localEvents, event)
}
case <-sc.initialWalkComplete:
initialWalkIncomplete = false
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
}
}
//find any files that have been deleted since the last time we ran
deletedFiles := []*asink.Event{}
resultChan := make(chan *asink.Event)
errorChan := make(chan error)
go sc.globals.db.DatabaseGetAllFiles(resultChan, errorChan)
deletionWalkIncomplete := true
for deletionWalkIncomplete {
select {
case oldEvent := <-resultChan:
if oldEvent == nil {
deletionWalkIncomplete = false
break
}
//if the file still exists, disregard this event
absolutePath := path.Join(sc.globals.syncDir, oldEvent.Path)
if _, err := os.Stat(absolutePath); err == nil {
break
}
event := new(asink.Event)
event.Path = absolutePath
event.Type = asink.DELETE
event.Timestamp = time.Now().UnixNano()
deletedFiles = append(deletedFiles, event)
case err := <-errorChan:
return ProcessingError{PERMANENT, err}
}
}
for _, event := range deletedFiles {
//make sure we don't need to exit
select {
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
default:
}
//process top half of local event
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
return err
}
}
}
if event.LocalStatus&asink.DISCARDED == 0 {
localEvents = append(localEvents, event)
}
}
//then process remote events (possibly taking a break whenever a local one comes in to process it)
timeout := time.NewTimer(1 * time.Second)
timedOut := false
for !timedOut {
select {
case event := <-sc.localUpdatesChan:
//process top half of local event
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
return err
}
}
}
if event.LocalStatus&asink.DISCARDED == 0 {
localEvents = append(localEvents, event)
}
timeout.Reset(1 * time.Second)
case event := <-sc.remoteUpdatesChan:
err := ProcessRemoteEvent(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessRemoteEvent(sc.globals, event)
if err != nil {
return err
}
}
}
timeout.Reset(1 * time.Second)
case <-timeout.C:
timedOut = true
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
}
}
for _, event := range localEvents {
//make sure we don't need to exit
select {
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
default:
}
err := ProcessLocalEvent_Lower(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Lower(sc.globals, event)
if err != nil {
return err
}
}
}
}
//finally, process the bottom halves of local updates
return nil
}
func StartWatching(watchDir string, fileUpdates chan *asink.Event, initialWalkComplete chan int) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
panic("Failed to create fsnotify watcher")
}
//function called by filepath.Walk to start watching a directory and all subdirectories
watchDirFn := func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
err = watcher.Watch(path)
if err != nil {
if e, ok := err.(syscall.Errno); ok && e == 28 {
//If we reach here, it means we've received ENOSPC from the Linux kernel:
//ENOSPC The user limit on the total number of inotify watches was reached or the kernel failed to allocate a needed resource.
panic("Exhausted the allowed number of inotify watches, please increase /proc/sys/fs/inotify/max_user_watches")
}
panic("Failed to watch " + path)
}
} else if info.Mode().IsRegular() {
event := new(asink.Event)
event.Path = path
event.Type = asink.UPDATE
event.Timestamp = time.Now().UnixNano()
fileUpdates <- event
}
return nil
}
//processes all the fsnotify events into asink events
go func() {
for {
select {
case ev := <-watcher.Event:
//if a directory was created, begin recursively watching all its subdirectories
if fi, err := os.Stat(ev.Name); err == nil && fi.IsDir() {
if ev.IsCreate() {
//Note: even though filepath.Walk will visit root, we must watch root first so we catch files/directories created after the walk begins but before this directory begins being watched
err = watcher.Watch(ev.Name)
if err != nil {
if e, ok := err.(syscall.Errno); ok && e == 28 {
//If we reach here, it means we've received ENOSPC from the Linux kernel:
//ENOSPC The user limit on the total number of inotify watches was reached or the kernel failed to allocate a needed resource.
panic("Exhausted the allowed number of inotify watches, please increase /proc/sys/fs/inotify/max_user_watches")
}
panic("Failed to watch " + ev.Name)
}
//scan this directory to ensure any file events we missed before starting to watch this directory are caught
filepath.Walk(ev.Name, watchDirFn)
}
continue
}
event := new(asink.Event)
if ev.IsCreate() || ev.IsModify() {
event.Type = asink.UPDATE
} else if ev.IsDelete() || ev.IsRename() {
event.Type = asink.DELETE
} else {
panic("Unknown fsnotify event type")
}
event.Path = ev.Name
event.Timestamp = time.Now().UnixNano()
fileUpdates <- event
case err := <-watcher.Error:
panic(err)
}
}
}()
//start watching the directory passed in
filepath.Walk(watchDir, watchDirFn)
initialWalkComplete <- 0
}
