func main() {
if err := resolveSettings(pars); err != nil {
log.Fatal(err)
}
conn, err := initSSH(pars)
if err != nil {
log.Fatal(err)
}
sftp, err := sftp.NewClient(conn)
if err != nil {
log.Fatal(err)
}
defer sftp.Close()
finStat, err := os.Stat(inPath)
if err != nil {
log.Fatal(err)
}
origSize := finStat.Size()
fin, err := os.Open(inPath)
if err != nil {
log.Fatal(err)
}
defer fin.Close()
outPath := path.Clean(path.Join(pars.DstFolder, filepath.Base(inPath)))
fout, err := sftp.Create(outPath)
if err != nil {
log.Fatal(err)
}
if _, err := io.Copy(fout, fin); err != nil {
log.Fatal(err)
}
foutStat, err := sftp.Lstat(outPath)
if err != nil {
log.Fatal(err)
}
if finalSize := foutStat.Size(); origSize != finalSize {
if err := sftp.Remove(outPath); err != nil {
log.Printf("[ERROR] Unable to remove target file: %v", err)
}
log.Fatalf("Failed to upload %s to %s: expected %d bytes, got %d (missing %d)",
inPath, outPath, origSize, finalSize, origSize-finalSize)
}
fmt.Println("Successfully uploaded", outPath)
}
By("reading the file")
file, err = sftp.Open(target)
Expect(err).NotTo(HaveOccurred())
buffer := &bytes.Buffer{}
_, err = buffer.ReadFrom(file)
Expect(err).NotTo(HaveOccurred())
err = file.Close()
Expect(err).NotTo(HaveOccurred())
Expect(buffer.Bytes()).To(Equal(fileContents))
By("removing the file")
err = sftp.Remove(target)
Expect(err).NotTo(HaveOccurred())
_, err = os.Stat(target)
Expect(err).To(HaveOccurred())
Expect(os.IsNotExist(err)).To(BeTrue())
})
})
Describe("invalid session channel requests", func() {
var channel ssh.Channel
var requests <-chan *ssh.Request
BeforeEach(func() {
var err error
channel, requests, err = client.OpenChannel("session", nil)
func Synchronise(ssh *ssh.Client, supervised, pushOnly bool, localDir, remoteDir string) {
/*
Algorithm:
- Get all local files and dirs
- Get all remote files and dirs
- Determine: ndl := newest mod time of all local files
- Determine: ndr := newest mod time of all remote files
- Free remote space
- Are remote files present, which are locally not present
and these remote files are older than ndl?
- Meaning: You have worked locally and remotely are older files which are not locally present.
Interpretation of change: You have deleted these files locally, because you own local
files which are newer that these files.
- delete these remote files! (may ask the user)
- Free local space
- Are local files present, which are remotely not present
and these local files are older than ndr?
- Meaning: You have worked remotely (or have synced with another PC), thus, local files are older
and these are not present remotely. Interpretation of change: You have delete these files
remotely, because your remotely files are newer that these files.
- delete these local files! (may ask the user)
- Download any new files
- Are remote files present, which are locally not present?
- Meaning: These files are new to the local side
- Download these files! (may ask the user)
- Upload any new files
- Are local files present, which are remotely not present?
- Meaning: These files are new to the remote side
- Upload these files! (may ask the user)
- Changed remote files
- Are remote and local files present, where the remote file is newer?
- Meaning: These files are changed on the remote side
- Download these files and replace the local copies (may ask the user)
- Changed local files
- Are local and remote files present, where the local file is newer?
- Meaning: These files are changed on the local side
- Upload these files and replace the remote copies (may ask the user)
*/
//
// Read the local files ============================================================================================
//
log.Println("Try to read all local files now...")
localFiles = make(map[string]os.FileInfo)
filepath.Walk(localDir, walkerlocal)
log.Printf("Found %d local files.\n", len(localFiles))
//
// Connect to the server ============================================================================================
//
sftp, sftpError := sftp.NewClient(ssh)
if sftpError != nil {
log.Println("Was not able to connect to the server: " + sftpError.Error())
os.Exit(7)
return
}
defer sftp.Close()
//
// Read the remote files ============================================================================================
//
log.Println("Try to read all remote files now...")
remoteFiles = make(map[string]os.FileInfo)
counterRemoteFile := 0
walker := sftp.Walk(remoteDir)
for walker.Step() {
counterRemoteFile++
if walker.Err() != nil {
continue
}
remoteFiles[walker.Path()] = walker.Stat()
if counterRemoteFile%512 == 0 {
fmt.Printf("%05d.\n", counterRemoteFile)
} else if counterRemoteFile%16 == 0 {
fmt.Printf("%05d.", counterRemoteFile)
}
}
fmt.Println()
log.Printf("Found %d remote files.\n", len(remoteFiles))
//
// Normalise all local paths ============================================================================================
//
localFilesNormalised = make(map[string]os.FileInfo)
for key, value := range localFiles {
normalised := normalisePath(localDir, key)
if normalised != `` {
localFilesNormalised[normalised] = value
normalised2localFiles[normalised] = key
}
}
//
// Normalise all remote paths ============================================================================================
//
remoteFilesNormalised = make(map[string]os.FileInfo)
for key, value := range remoteFiles {
normalised := normalisePath(remoteDir, key)
if normalised != `` {
remoteFilesNormalised[normalised] = value
normalised2remoteFiles[normalised] = key
}
}
//
// Determine ndl and ndr ============================================================================================
//
ndl := time.Date(1, time.January, 1, 1, 1, 1, 1, time.UTC)
ndr := ndl
for _, remoteFileInfo := range remoteFiles {
if remoteFileInfo.ModTime().UTC().After(ndr) {
ndr = remoteFileInfo.ModTime().UTC()
}
}
for _, localFileInfo := range localFiles {
if localFileInfo.ModTime().UTC().After(ndl) {
ndl = localFileInfo.ModTime().UTC()
}
}
log.Printf("The newest local file's time: %v\n", ndl)
log.Printf("The newest remote file's time: %v\n", ndr)
//
// Free remote space ============================================================================================
//
deleteRemoteFiles := make([]string, 0)
for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised {
localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
if localFileNormaliesed == nil {
if remoteFileInfo.ModTime().UTC().Before(ndl) {
deleteRemoteFiles = append(deleteRemoteFiles, remoteFileNormalised)
}
}
}
log.Printf("Found %d remote files to delete.\n", len(deleteRemoteFiles))
if len(deleteRemoteFiles) > 0 {
sort.Strings(deleteRemoteFiles)
shouldDeleteRemoteFiles := true
if supervised {
fmt.Println(`=================================================================`)
for _, file := range deleteRemoteFiles {
fmt.Println(normalised2remoteFiles[file])
}
fmt.Print("Should I delete these remote files? <y|n> ")
shouldDeleteRemoteFiles = readYesNoAnswer(false)
}
if shouldDeleteRemoteFiles {
// 1. Remove all files
for _, remoteFileNormalised := range deleteRemoteFiles {
// Skip all directories:
if remoteFilesNormalised[remoteFileNormalised].IsDir() {
continue
}
removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised])
if removeError != nil {
log.Printf("Was not able to delete the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error())
} else {
log.Printf("Deleted the remote file %s\n", normalised2remoteFiles[remoteFileNormalised])
}
}
// 2. Remove all directories
for _, remoteFileNormalised := range deleteRemoteFiles {
// Skip all files:
if !remoteFilesNormalised[remoteFileNormalised].IsDir() {
continue
}
// TODO: Does not work: File an issue!
removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised] + `/`)
if removeError != nil {
log.Printf("Was not able to delete the remote directory %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error())
} else {
log.Printf("Deleted the remote directory %s\n", normalised2remoteFiles[remoteFileNormalised])
}
}
for _, remoteFileNormalised := range deleteRemoteFiles {
delete(remoteFiles, normalised2remoteFiles[remoteFileNormalised])
delete(remoteFilesNormalised, remoteFileNormalised)
}
}
}
//
// Free local space ============================================================================================
//
if !pushOnly {
deleteLocalFiles := make([]string, 0)
for localFileNormalised, localFileInfo := range localFilesNormalised {
remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
if remoteFileNormaliesed == nil {
if localFileInfo.ModTime().UTC().Before(ndr) {
deleteLocalFiles = append(deleteLocalFiles, localFileNormalised)
}
}
}
log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles))
if len(deleteLocalFiles) > 0 {
sort.Strings(deleteLocalFiles)
shouldDeleteLocalFiles := true
if supervised {
fmt.Println(`=================================================================`)
for _, file := range deleteLocalFiles {
fmt.Println(normalised2localFiles[file])
}
fmt.Print("Should I delete these local files? <y|n> ")
shouldDeleteLocalFiles = readYesNoAnswer(false)
}
if shouldDeleteLocalFiles {
for _, localFileNormalised := range deleteLocalFiles {
// Skip all directories:
if localFilesNormalised[localFileNormalised].IsDir() {
continue
}
removeError := os.Remove(normalised2localFiles[localFileNormalised])
if removeError != nil {
log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
} else {
log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised])
}
}
for _, localFileNormalised := range deleteLocalFiles {
// Skip all files:
if !localFilesNormalised[localFileNormalised].IsDir() {
continue
}
removeError := os.Remove(normalised2localFiles[localFileNormalised])
if removeError != nil {
log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
} else {
log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised])
}
}
for _, localFileNormalised := range deleteLocalFiles {
delete(localFiles, normalised2localFiles[localFileNormalised])
delete(localFilesNormalised, localFileNormalised)
}
}
}
}
//
// Download new files ============================================================================================
//
if !pushOnly {
downloadRemoteFiles := make([]string, 0)
for remoteFileNormalised, _ := range remoteFilesNormalised {
localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
if localFileNormaliesed == nil {
downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised)
}
}
log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles))
if len(downloadRemoteFiles) > 0 {
sort.Strings(downloadRemoteFiles)
shouldDownloadRemoteFiles := true
if supervised {
fmt.Println(`=================================================================`)
for _, file := range downloadRemoteFiles {
fmt.Println(normalised2remoteFiles[file])
}
fmt.Print("Should I download these new remote files? <y|n> ")
shouldDownloadRemoteFiles = readYesNoAnswer(false)
}
if shouldDownloadRemoteFiles {
// 1. Create all new directories
for _, remoteFileNormalised := range downloadRemoteFiles {
// Skip all files
if !remoteFilesNormalised[remoteFileNormalised].IsDir() {
continue
}
newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
log.Printf("Try to create the new local directory %s...\n", newLocalDir)
if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil {
log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error())
}
}
// 2. All new files
for _, remoteFileNormalised := range downloadRemoteFiles {
// Skip all directories
if remoteFilesNormalised[remoteFileNormalised].IsDir() {
continue
}
log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
if remoteFileHandleError != nil {
log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
continue
}
_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
newLocalFile := filepath.Join(localDir, path, filename)
localFileHandle, localFileHandleError := os.Create(newLocalFile)
if localFileHandleError != nil {
log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error())
remoteFileHandle.Close()
continue
}
_, copyError := io.Copy(localFileHandle, remoteFileHandle)
if copyError != nil {
log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error())
remoteFileHandle.Close()
localFileHandle.Close()
continue
}
remoteFileHandle.Close()
localFileHandle.Close()
}
}
}
}
//
// Upload new files ============================================================================================
//
uploadLocalFiles := make([]string, 0)
for localFileNormalised, _ := range localFilesNormalised {
remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
if remoteFileNormaliesed == nil {
uploadLocalFiles = append(uploadLocalFiles, localFileNormalised)
}
}
log.Printf("Found %d new local files to upload.\n", len(uploadLocalFiles))
if len(uploadLocalFiles) > 0 {
sort.Strings(uploadLocalFiles)
shouldUploadLocalFiles := true
if supervised {
fmt.Println(`=================================================================`)
for _, file := range uploadLocalFiles {
fmt.Println(normalised2localFiles[file])
}
fmt.Print("Should I upload these local new files? <y|n> ")
shouldUploadLocalFiles = readYesNoAnswer(false)
}
if shouldUploadLocalFiles {
// 1. Create new directories
for _, localFileNormalised := range uploadLocalFiles {
// Skip all files
if !localFilesNormalised[localFileNormalised].IsDir() {
continue
}
newRemoteDir := filepath.ToSlash(filepath.Join(remoteDir, strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)))
log.Printf("Try to create the new remote directory %s...\n", newRemoteDir)
if mkdirError := sftp.Mkdir(newRemoteDir); mkdirError != nil {
log.Printf("Was not able to create the remote directory %s: %s\n", newRemoteDir, mkdirError.Error())
}
}
// 2. All new files
for _, localFileNormalised := range uploadLocalFiles {
// Skip all directories
if localFilesNormalised[localFileNormalised].IsDir() {
continue
}
log.Printf("Try to upload the new local file %s...\n", normalised2localFiles[localFileNormalised])
_, filename := filepath.Split(normalised2localFiles[localFileNormalised])
path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))
newRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename))
remoteFileHandle, remoteFileHandleError := sftp.Create(newRemoteFile)
if remoteFileHandleError != nil {
log.Printf("Was not able to create the remote file %s: %s\n", newRemoteFile, remoteFileHandleError.Error())
continue
}
localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised])
if localFileHandleError != nil {
log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error())
remoteFileHandle.Close()
continue
}
_, copyError := io.Copy(remoteFileHandle, localFileHandle)
if copyError != nil {
log.Printf("Was not able to upload the new local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], newRemoteFile, copyError.Error())
remoteFileHandle.Close()
localFileHandle.Close()
continue
}
remoteFileHandle.Close()
localFileHandle.Close()
}
}
}
//
// Changed files on the remote side ============================================================================================
//
if !pushOnly {
changedRemoteFiles := make([]string, 0)
for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised {
localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() {
if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) {
changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised)
}
}
}
log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles))
if len(changedRemoteFiles) > 0 {
sort.Strings(changedRemoteFiles)
shouldDownloadRemoteFiles := true
if supervised {
fmt.Println(`=================================================================`)
for _, file := range changedRemoteFiles {
fmt.Println(normalised2remoteFiles[file])
}
fmt.Print("Should I download these changed remote files? <y|n> ")
shouldDownloadRemoteFiles = readYesNoAnswer(false)
}
if shouldDownloadRemoteFiles {
for _, remoteFileNormalised := range changedRemoteFiles {
log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
if remoteFileHandleError != nil {
log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
continue
}
_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
existingLocalFile := filepath.Join(localDir, path, filename)
localFileHandle, localFileHandleError := os.Create(existingLocalFile)
if localFileHandleError != nil {
log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error())
remoteFileHandle.Close()
continue
}
_, copyError := io.Copy(localFileHandle, remoteFileHandle)
if copyError != nil {
log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error())
remoteFileHandle.Close()
localFileHandle.Close()
continue
}
remoteFileHandle.Close()
localFileHandle.Close()
}
}
}
}
//
// Changed files on the local side ============================================================================================
//
changedLocalFiles := make([]string, 0)
for localFileNormalised, localFileInfo := range localFilesNormalised {
remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
if remoteFileNormaliesed != nil && !remoteFileNormaliesed.IsDir() {
if localFileInfo.ModTime().UTC().After(remoteFileNormaliesed.ModTime().UTC()) {
changedLocalFiles = append(changedLocalFiles, localFileNormalised)
}
}
}
log.Printf("Found %d local files which are changed.\n", len(changedLocalFiles))
if len(changedLocalFiles) > 0 {
sort.Strings(changedLocalFiles)
shouldUploadLocalFiles := true
if supervised {
fmt.Println(`=================================================================`)
for _, file := range changedLocalFiles {
fmt.Println(normalised2localFiles[file])
}
fmt.Print("Should I upload these changed local files? <y|n> ")
shouldUploadLocalFiles = readYesNoAnswer(false)
}
if shouldUploadLocalFiles {
for _, localFileNormalised := range changedLocalFiles {
log.Printf("Try to upload the changed local file %s...\n", normalised2localFiles[localFileNormalised])
_, filename := filepath.Split(normalised2localFiles[localFileNormalised])
path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))
existingRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename))
remoteFileHandle, remoteFileHandleError := sftp.Create(existingRemoteFile)
if remoteFileHandleError != nil {
log.Printf("Was not able to overwrite the remote file %s: %s\n", existingRemoteFile, remoteFileHandleError.Error())
continue
}
localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised])
if localFileHandleError != nil {
log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error())
remoteFileHandle.Close()
continue
}
_, copyError := io.Copy(remoteFileHandle, localFileHandle)
if copyError != nil {
log.Printf("Was not able to upload the changed local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], existingRemoteFile, copyError.Error())
remoteFileHandle.Close()
localFileHandle.Close()
continue
}
remoteFileHandle.Close()
localFileHandle.Close()
}
}
}
}