func ProcessLocalEvent_Upper(globals *AsinkGlobals, event *asink.Event) error {
var err error
StatStartLocalUpdate()
defer StatStopLocalUpdate()
//make the path relative before we save/send it anywhere
absolutePath := event.Path
event.Path, err = filepath.Rel(globals.syncDir, event.Path)
if err != nil {
return ProcessingError{TEMPORARY, err}
}
latestLocal := LockPath(event.Path, true)
defer func() {
if err != nil {
event.LocalStatus |= asink.DISCARDED
}
event.LocalStatus |= asink.NOSAVE //make sure event doesn't get saved back until lower half
UnlockPath(event)
}()
err = processLocalEvent_Upper(globals, event, latestLocal, absolutePath)
return err
}
func (adb *AsinkDB) DatabaseAddEvent(e *asink.Event) (err error) {
adb.lock.Lock()
tx, err := adb.db.Begin()
if err != nil {
return err
}
//make sure the transaction gets rolled back on error, and the database gets unlocked
defer func() {
if err != nil {
tx.Rollback()
}
adb.lock.Unlock()
}()
result, err := tx.Exec("INSERT INTO events (id, type, localstatus, path, hash, predecessor, timestamp, permissions) VALUES (?,?,?,?,?,?,?,?);", e.Id, e.Type, e.LocalStatus, e.Path, e.Hash, e.Predecessor, e.Timestamp, e.Permissions)
if err != nil {
return err
}
id, err := result.LastInsertId()
if err != nil {
return err
}
err = tx.Commit()
if err != nil {
return err
}
e.LocalId = id
e.InDB = true
return nil
}
func ProcessLocalEvent(globals *AsinkGlobals, event *asink.Event) error {
var err error
StatStartLocalUpdate()
defer StatStopLocalUpdate()
//make the path relative before we save/send it anywhere
absolutePath := event.Path
event.Path, err = filepath.Rel(globals.syncDir, event.Path)
if err != nil {
return ProcessingError{TEMPORARY, err}
}
latestLocal := LockPath(event.Path, true)
defer func() {
if err != nil {
event.LocalStatus |= asink.DISCARDED
}
UnlockPath(event)
}()
err = processLocalEvent_Upper(globals, event, latestLocal, absolutePath)
if err != nil {
return err
}
//don't process the second half if the first half discarded it
if event.LocalStatus&asink.DISCARDED != 0 {
return nil
}
err = processLocalEvent_Lower(globals, event, latestLocal)
return err
}
func ProcessLocalEvent_Lower(globals *AsinkGlobals, event *asink.Event) error {
var err error
StatStartLocalUpdate()
defer StatStopLocalUpdate()
latestLocal := LockPath(event.Path, true)
defer func() {
if err != nil {
event.LocalStatus |= asink.DISCARDED
}
event.LocalStatus &= ^asink.NOSAVE //clear NOSAVE set in upper half
UnlockPath(event)
}()
err = processLocalEvent_Lower(globals, event, latestLocal)
return err
}
//handle a conflict by copying the loser event to another file
func handleConflict(globals *AsinkGlobals, loser *asink.Event, copyFrom string) error {
if loser.IsUpdate() {
//come up with new file name
conflictedPath := path.Join(globals.syncDir, loser.Path) + "_conflicted_copy_" + time.Now().Format("2006-01-02_15:04:05.000000")
//copy file to new filename
src, err := os.Open(copyFrom)
if err != nil {
return err
}
defer src.Close()
sink, err := os.Create(conflictedPath)
if err != nil {
return err
}
defer sink.Close()
_, err = io.Copy(sink, src)
return err
}
return nil
}
func (sc *StartupContext) Run() error {
//process top halves of local updates so the files are saved off at least locally
localEvents := []*asink.Event{}
initialWalkIncomplete := true
for initialWalkIncomplete {
select {
case event := <-sc.localUpdatesChan:
//process top half of local event
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
return err
}
}
}
if event.LocalStatus&asink.DISCARDED == 0 {
localEvents = append(localEvents, event)
}
case <-sc.initialWalkComplete:
initialWalkIncomplete = false
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
}
}
//find any files that have been deleted since the last time we ran
deletedFiles := []*asink.Event{}
resultChan := make(chan *asink.Event)
errorChan := make(chan error)
go sc.globals.db.DatabaseGetAllFiles(resultChan, errorChan)
deletionWalkIncomplete := true
for deletionWalkIncomplete {
select {
case oldEvent := <-resultChan:
if oldEvent == nil {
deletionWalkIncomplete = false
break
}
//if the file still exists, disregard this event
absolutePath := path.Join(sc.globals.syncDir, oldEvent.Path)
if _, err := os.Stat(absolutePath); err == nil {
break
}
event := new(asink.Event)
event.Path = absolutePath
event.Type = asink.DELETE
event.Timestamp = time.Now().UnixNano()
deletedFiles = append(deletedFiles, event)
case err := <-errorChan:
return ProcessingError{PERMANENT, err}
}
}
for _, event := range deletedFiles {
//make sure we don't need to exit
select {
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
default:
}
//process top half of local event
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
return err
}
}
}
if event.LocalStatus&asink.DISCARDED == 0 {
localEvents = append(localEvents, event)
}
}
//then process remote events (possibly taking a break whenever a local one comes in to process it)
timeout := time.NewTimer(1 * time.Second)
timedOut := false
for !timedOut {
select {
case event := <-sc.localUpdatesChan:
//process top half of local event
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Upper(sc.globals, event)
if err != nil {
return err
}
}
}
if event.LocalStatus&asink.DISCARDED == 0 {
localEvents = append(localEvents, event)
}
timeout.Reset(1 * time.Second)
case event := <-sc.remoteUpdatesChan:
err := ProcessRemoteEvent(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessRemoteEvent(sc.globals, event)
if err != nil {
return err
}
}
}
timeout.Reset(1 * time.Second)
case <-timeout.C:
timedOut = true
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
}
}
for _, event := range localEvents {
//make sure we don't need to exit
select {
case <-sc.exitChan:
return ProcessingError{EXITED, nil}
default:
}
err := ProcessLocalEvent_Lower(sc.globals, event)
if err != nil {
if e, ok := err.(ProcessingError); !ok || e.ErrorType != TEMPORARY {
return err
} else {
//if error was temporary, retry once
event.LocalStatus = 0
err := ProcessLocalEvent_Lower(sc.globals, event)
if err != nil {
return err
}
}
}
}
//finally, process the bottom halves of local updates
return nil
}
func ProcessRemoteEvent(globals *AsinkGlobals, event *asink.Event) error {
var err error
StatStartRemoteUpdate()
defer StatStopRemoteUpdate()
latestLocal := LockPath(event.Path, false)
defer func() {
if err != nil {
event.LocalStatus |= asink.DISCARDED
}
UnlockPath(event)
}()
//get the absolute path because we may need it later
absolutePath := path.Join(globals.syncDir, event.Path)
//if we already have this event, or if it is older than our most recent event, bail out
if latestLocal != nil {
if event.Timestamp < latestLocal.Timestamp {
event.LocalStatus |= asink.DISCARDED
return nil
}
if event.IsSameEvent(latestLocal) {
return nil
}
if latestLocal.Hash != event.Predecessor && latestLocal.Hash != event.Hash {
err = handleConflict(globals, latestLocal, path.Join(globals.cacheDir, latestLocal.Hash))
if err != nil {
return ProcessingError{PERMANENT, err}
}
}
}
//Download event
if event.IsUpdate() {
if latestLocal == nil || event.Hash != latestLocal.Hash {
outfile, err := ioutil.TempFile(globals.tmpDir, "asink")
if err != nil {
return ProcessingError{CONFIG, err}
}
tmpfilename := outfile.Name()
StatStartDownload()
downloadReadCloser, err := globals.storage.Get(event.Hash)
if err != nil {
StatStopDownload()
return ProcessingError{STORAGE, err}
}
defer downloadReadCloser.Close()
if globals.encrypted {
decrypter, err := NewDecrypter(downloadReadCloser, globals.key)
if err != nil {
StatStopDownload()
return ProcessingError{STORAGE, err}
}
_, err = io.Copy(outfile, decrypter)
} else {
_, err = io.Copy(outfile, downloadReadCloser)
}
outfile.Close()
StatStopDownload()
if err != nil {
return ProcessingError{STORAGE, err}
}
//rename to local hashed filename
hashedFilename := path.Join(globals.cacheDir, event.Hash)
err = os.Rename(tmpfilename, hashedFilename)
if err != nil {
err = os.Remove(tmpfilename)
if err != nil {
return ProcessingError{PERMANENT, err}
}
return ProcessingError{PERMANENT, err}
}
//copy hashed file to another tmp, then rename it to the actual file.
tmpfilename, err = util.CopyToTmp(hashedFilename, globals.tmpDir)
if err != nil {
return ProcessingError{PERMANENT, err}
}
//make sure containing directory exists
err = util.EnsureDirExists(path.Dir(absolutePath))
if err != nil {
return ProcessingError{PERMANENT, err}
}
err = os.Rename(tmpfilename, absolutePath)
if err != nil {
err2 := os.Remove(tmpfilename)
if err2 != nil {
return ProcessingError{PERMANENT, err2}
}
return ProcessingError{PERMANENT, err}
}
}
if latestLocal == nil || event.Hash != latestLocal.Hash || event.Permissions != latestLocal.Permissions {
err = os.Chmod(absolutePath, event.Permissions)
if err != nil && !util.ErrorFileNotFound(err) {
return ProcessingError{PERMANENT, err}
}
}
} else {
//intentionally ignore errors in case this file has been deleted out from under us
os.Remove(absolutePath)
//delete the directory previously containing this file if its the last file
util.RecursiveRemoveEmptyDirs(path.Dir(absolutePath))
}
return nil
}
func processLocalEvent_Lower(globals *AsinkGlobals, event *asink.Event, latestLocal *asink.Event) error {
var err error
//if we already have this event, or if it is older than our most recent event, bail out
if latestLocal != nil {
if event.Timestamp < latestLocal.Timestamp {
event.LocalStatus |= asink.DISCARDED
return nil
}
//if the remote side snuck in an event that has the same hash
//as we do, disregard our event
if event.Hash == latestLocal.Hash {
event.LocalStatus |= asink.DISCARDED
return nil
}
//if our predecessor has changed, it means we have received a
//remote event for this file since the top half of processing
//this local event. If this is true, we have a conflict we
//can't resolve without user intervention.
if latestLocal.Hash != event.Predecessor {
err = handleConflict(globals, event, path.Join(globals.cacheDir, event.Hash))
event.LocalStatus |= asink.DISCARDED
if err != nil {
return ProcessingError{PERMANENT, err}
}
return nil
}
}
if event.IsUpdate() {
//upload file to remote storage
StatStartUpload()
done := make(chan error, 1)
uploadWriteCloser, err := globals.storage.Put(event.Hash, done)
if err != nil {
return ProcessingError{STORAGE, err}
}
cachedFilename := path.Join(globals.cacheDir, event.Hash)
uploadFile, err := os.Open(cachedFilename)
if err != nil {
uploadWriteCloser.Close()
return ProcessingError{STORAGE, err}
}
if globals.encrypted {
encrypter, err := NewEncrypter(uploadWriteCloser, globals.key)
if err != nil {
uploadWriteCloser.Close()
uploadFile.Close()
return ProcessingError{STORAGE, err}
}
_, err = io.Copy(encrypter, uploadFile)
encrypter.Close()
} else {
_, err = io.Copy(uploadWriteCloser, uploadFile)
}
uploadFile.Close()
uploadWriteCloser.Close()
//ensure the upload is observable by other clients before proceeding
err = <-done
if err != nil {
return ProcessingError{STORAGE, err}
}
StatStopUpload()
if err != nil {
return ProcessingError{STORAGE, err}
}
}
//finally, send it off to the server
StatStartSending()
err = SendEvent(globals, event)
StatStopSending()
if err != nil {
return ProcessingError{NETWORK, err}
}
return nil
}
func processLocalEvent_Upper(globals *AsinkGlobals, event *asink.Event, latestLocal *asink.Event, absolutePath string) error {
if latestLocal != nil {
event.Predecessor = latestLocal.Hash
if event.Timestamp < latestLocal.Timestamp {
fmt.Printf("trying to send event older than latestLocal:\n")
fmt.Printf("OLD %+v\n", latestLocal)
fmt.Printf("NEW %+v\n", event)
}
}
if event.IsUpdate() {
//copy to tmp
//TODO upload in chunks and check modification times to make sure it hasn't been changed instead of copying the whole thing off
tmpfilename, err := util.CopyToTmp(absolutePath, globals.tmpDir)
if err != nil {
//bail out if the file we are trying to upload already got deleted
if util.ErrorFileNotFound(err) {
event.LocalStatus |= asink.DISCARDED
return nil
}
return err
}
//try to collect the file's permissions
fileinfo, err := os.Stat(absolutePath)
if err != nil {
//bail out if the file we are trying to upload already got deleted
if util.ErrorFileNotFound(err) {
event.LocalStatus |= asink.DISCARDED
return nil
}
return ProcessingError{PERMANENT, err}
} else {
event.Permissions = fileinfo.Mode()
}
//get the file's hash
hash, err := HashFile(tmpfilename)
if err != nil {
return ProcessingError{TEMPORARY, err}
}
event.Hash = hash
//If the hash is the same, don't try to upload the event again
if latestLocal != nil && event.Hash == latestLocal.Hash {
os.Remove(tmpfilename)
//If neither the file contents nor permissions changed, squash this event completely
if event.Permissions == latestLocal.Permissions {
event.LocalStatus |= asink.DISCARDED
return nil
}
} else {
//rename to local cache w/ filename=hash
cachedFilename := path.Join(globals.cacheDir, event.Hash)
err = os.Rename(tmpfilename, cachedFilename)
if err != nil {
err = os.Remove(tmpfilename)
if err != nil {
return ProcessingError{PERMANENT, err}
}
return ProcessingError{PERMANENT, err}
}
}
} else {
//if we're trying to delete a file that we thought was already deleted, there's no need to delete it again
if latestLocal != nil && latestLocal.IsDelete() {
event.LocalStatus |= asink.DISCARDED
return nil
}
}
return nil
}
func StartWatching(watchDir string, fileUpdates chan *asink.Event, initialWalkComplete chan int) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
panic("Failed to create fsnotify watcher")
}
//function called by filepath.Walk to start watching a directory and all subdirectories
watchDirFn := func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
err = watcher.Watch(path)
if err != nil {
if e, ok := err.(syscall.Errno); ok && e == 28 {
//If we reach here, it means we've received ENOSPC from the Linux kernel:
//ENOSPC The user limit on the total number of inotify watches was reached or the kernel failed to allocate a needed resource.
panic("Exhausted the allowed number of inotify watches, please increase /proc/sys/fs/inotify/max_user_watches")
}
panic("Failed to watch " + path)
}
} else if info.Mode().IsRegular() {
event := new(asink.Event)
event.Path = path
event.Type = asink.UPDATE
event.Timestamp = time.Now().UnixNano()
fileUpdates <- event
}
return nil
}
//processes all the fsnotify events into asink events
go func() {
for {
select {
case ev := <-watcher.Event:
//if a directory was created, begin recursively watching all its subdirectories
if fi, err := os.Stat(ev.Name); err == nil && fi.IsDir() {
if ev.IsCreate() {
//Note: even though filepath.Walk will visit root, we must watch root first so we catch files/directories created after the walk begins but before this directory begins being watched
err = watcher.Watch(ev.Name)
if err != nil {
if e, ok := err.(syscall.Errno); ok && e == 28 {
//If we reach here, it means we've received ENOSPC from the Linux kernel:
//ENOSPC The user limit on the total number of inotify watches was reached or the kernel failed to allocate a needed resource.
panic("Exhausted the allowed number of inotify watches, please increase /proc/sys/fs/inotify/max_user_watches")
}
panic("Failed to watch " + ev.Name)
}
//scan this directory to ensure any file events we missed before starting to watch this directory are caught
filepath.Walk(ev.Name, watchDirFn)
}
continue
}
event := new(asink.Event)
if ev.IsCreate() || ev.IsModify() {
event.Type = asink.UPDATE
} else if ev.IsDelete() || ev.IsRename() {
event.Type = asink.DELETE
} else {
panic("Unknown fsnotify event type")
}
event.Path = ev.Name
event.Timestamp = time.Now().UnixNano()
fileUpdates <- event
case err := <-watcher.Error:
panic(err)
}
}
}()
//start watching the directory passed in
filepath.Walk(watchDir, watchDirFn)
initialWalkComplete <- 0
}