Пример #1
0
func (sc *StartupContext) Run() error {
	//process top halves of local updates so the files are saved off at least locally
	localEvents := []*asink.Event{}
	initialWalkIncomplete := true
	for initialWalkIncomplete {
		select {
		case event := <-sc.localUpdatesChan:
			//process top half of local event
			err := ProcessLocalEvent_Upper(sc.globals, event)
			if err != nil {
				if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
					return err
				} else {
					//if error was temporary, retry once
					event.LocalStatus = 0
					err := ProcessLocalEvent_Upper(sc.globals, event)
					if err != nil {
						return err
					}
				}
			}
			if event.LocalStatus&asink.DISCARDED == 0 {
				localEvents = append(localEvents, event)
			}
		case <-sc.initialWalkComplete:
			initialWalkIncomplete = false
		case <-sc.exitChan:
			return ProcessingError{EXITED, nil}
		}
	}

	//find any files that have been deleted since the last time we ran
	deletedFiles := []*asink.Event{}
	resultChan := make(chan *asink.Event)
	errorChan := make(chan error)
	go sc.globals.db.DatabaseGetAllFiles(resultChan, errorChan)
	deletionWalkIncomplete := true
	for deletionWalkIncomplete {
		select {
		case oldEvent := <-resultChan:
			if oldEvent == nil {
				deletionWalkIncomplete = false
				break
			}

			//if the file still exists, disregard this event
			absolutePath := path.Join(sc.globals.syncDir, oldEvent.Path)
			if _, err := os.Stat(absolutePath); err == nil {
				break
			}

			event := new(asink.Event)
			event.Path = absolutePath
			event.Type = asink.DELETE
			event.Timestamp = time.Now().UnixNano()
			deletedFiles = append(deletedFiles, event)
		case err := <-errorChan:
			return ProcessingError{PERMANENT, err}
		}
	}

	for _, event := range deletedFiles {
		//make sure we don't need to exit
		select {
		case <-sc.exitChan:
			return ProcessingError{EXITED, nil}
		default:
		}
		//process top half of local event
		err := ProcessLocalEvent_Upper(sc.globals, event)
		if err != nil {
			if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
				return err
			} else {
				//if error was temporary, retry once
				event.LocalStatus = 0
				err := ProcessLocalEvent_Upper(sc.globals, event)
				if err != nil {
					return err
				}
			}
		}
		if event.LocalStatus&asink.DISCARDED == 0 {
			localEvents = append(localEvents, event)
		}
	}

	//then process remote events (possibly taking a break whenever a local one comes in to process it)
	timeout := time.NewTimer(1 * time.Second)
	timedOut := false
	for !timedOut {
		select {
		case event := <-sc.localUpdatesChan:
			//process top half of local event
			err := ProcessLocalEvent_Upper(sc.globals, event)
			if err != nil {
				if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
					return err
				} else {
					//if error was temporary, retry once
					event.LocalStatus = 0
					err := ProcessLocalEvent_Upper(sc.globals, event)
					if err != nil {
						return err
					}
				}
			}
			if event.LocalStatus&asink.DISCARDED == 0 {
				localEvents = append(localEvents, event)
			}
			timeout.Reset(1 * time.Second)
		case event := <-sc.remoteUpdatesChan:
			err := ProcessRemoteEvent(sc.globals, event)
			if err != nil {
				if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
					return err
				} else {
					//if error was temporary, retry once
					event.LocalStatus = 0
					err := ProcessRemoteEvent(sc.globals, event)
					if err != nil {
						return err
					}
				}
			}
			timeout.Reset(1 * time.Second)
		case <-timeout.C:
			timedOut = true
		case <-sc.exitChan:
			return ProcessingError{EXITED, nil}
		}
	}

	for _, event := range localEvents {
		//make sure we don't need to exit
		select {
		case <-sc.exitChan:
			return ProcessingError{EXITED, nil}
		default:
		}
		err := ProcessLocalEvent_Lower(sc.globals, event)
		if err != nil {
			if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
				return err
			} else {
				//if error was temporary, retry once
				event.LocalStatus = 0
				err := ProcessLocalEvent_Lower(sc.globals, event)
				if err != nil {
					return err
				}
			}
		}
	}

	//finally, process the bottom halves of local updates
	return nil
}
Пример #2
0
func StartWatching(watchDir string, fileUpdates chan *asink.Event, initialWalkComplete chan int) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		panic("Failed to create fsnotify watcher")
	}

	//function called by filepath.Walk to start watching a directory and all subdirectories
	watchDirFn := func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			err = watcher.Watch(path)
			if err != nil {
				if e, ok := err.(syscall.Errno); ok && e == 28 {
					//If we reach here, it means we've received ENOSPC from the Linux kernel:
					//ENOSPC The user limit on the total number of inotify watches was reached or the kernel failed to allocate a needed resource.
					panic("Exhausted the allowed number of inotify watches, please increase /proc/sys/fs/inotify/max_user_watches")
				}
				panic("Failed to watch " + path)
			}
		} else if info.Mode().IsRegular() {
			event := new(asink.Event)
			event.Path = path
			event.Type = asink.UPDATE
			event.Timestamp = time.Now().UnixNano()
			fileUpdates <- event
		}
		return nil
	}

	//processes all the fsnotify events into asink events
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				//if a directory was created, begin recursively watching all its subdirectories
				if fi, err := os.Stat(ev.Name); err == nil && fi.IsDir() {
					if ev.IsCreate() {
						//Note: even though filepath.Walk will visit root, we must watch root first so we catch files/directories created after the walk begins but before this directory begins being watched
						err = watcher.Watch(ev.Name)
						if err != nil {
							if e, ok := err.(syscall.Errno); ok && e == 28 {
								//If we reach here, it means we've received ENOSPC from the Linux kernel:
								//ENOSPC The user limit on the total number of inotify watches was reached or the kernel failed to allocate a needed resource.
								panic("Exhausted the allowed number of inotify watches, please increase /proc/sys/fs/inotify/max_user_watches")
							}
							panic("Failed to watch " + ev.Name)
						}
						//scan this directory to ensure any file events we missed before starting to watch this directory are caught
						filepath.Walk(ev.Name, watchDirFn)
					}
					continue
				}

				event := new(asink.Event)
				if ev.IsCreate() || ev.IsModify() {
					event.Type = asink.UPDATE
				} else if ev.IsDelete() || ev.IsRename() {
					event.Type = asink.DELETE
				} else {
					panic("Unknown fsnotify event type")
				}

				event.Path = ev.Name
				event.Timestamp = time.Now().UnixNano()

				fileUpdates <- event

			case err := <-watcher.Error:
				panic(err)
			}
		}
	}()

	//start watching the directory passed in
	filepath.Walk(watchDir, watchDirFn)
	initialWalkComplete <- 0
}