Exemple #1
0
func main() {
	if err := resolveSettings(pars); err != nil {
		log.Fatal(err)
	}

	conn, err := initSSH(pars)
	if err != nil {
		log.Fatal(err)
	}

	sftp, err := sftp.NewClient(conn)
	if err != nil {
		log.Fatal(err)
	}
	defer sftp.Close()

	finStat, err := os.Stat(inPath)
	if err != nil {
		log.Fatal(err)
	}
	origSize := finStat.Size()

	fin, err := os.Open(inPath)
	if err != nil {
		log.Fatal(err)
	}
	defer fin.Close()

	outPath := path.Clean(path.Join(pars.DstFolder, filepath.Base(inPath)))
	fout, err := sftp.Create(outPath)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := io.Copy(fout, fin); err != nil {
		log.Fatal(err)
	}

	foutStat, err := sftp.Lstat(outPath)
	if err != nil {
		log.Fatal(err)
	}
	if finalSize := foutStat.Size(); origSize != finalSize {
		if err := sftp.Remove(outPath); err != nil {
			log.Printf("[ERROR] Unable to remove target file: %v", err)
		}
		log.Fatalf("Failed to upload %s to %s: expected %d bytes, got %d (missing %d)",
			inPath, outPath, origSize, finalSize, origSize-finalSize)
	}

	fmt.Println("Successfully uploaded", outPath)
}
			By("reading the file")
			file, err = sftp.Open(target)
			Expect(err).NotTo(HaveOccurred())

			buffer := &bytes.Buffer{}
			_, err = buffer.ReadFrom(file)
			Expect(err).NotTo(HaveOccurred())

			err = file.Close()
			Expect(err).NotTo(HaveOccurred())

			Expect(buffer.Bytes()).To(Equal(fileContents))

			By("removing the file")
			err = sftp.Remove(target)
			Expect(err).NotTo(HaveOccurred())

			_, err = os.Stat(target)
			Expect(err).To(HaveOccurred())
			Expect(os.IsNotExist(err)).To(BeTrue())
		})
	})

	Describe("invalid session channel requests", func() {
		var channel ssh.Channel
		var requests <-chan *ssh.Request

		BeforeEach(func() {
			var err error
			channel, requests, err = client.OpenChannel("session", nil)
func Synchronise(ssh *ssh.Client, supervised, pushOnly bool, localDir, remoteDir string) {
	/*
		Algorithm:
		- Get all local files and dirs
		- Get all remote files and dirs
		- Determine: ndl := newest mod time of all local files
		- Determine: ndr := newest mod time of all remote files
		- Free remote space
			- Are remote files present, which are locally not present
				and these remote files are older than ndl?
					- Meaning: You have worked locally and remotely are older files which are not locally present.
							   Interpretation of change: You have deleted these files locally, because you own local
							   files which are newer that these files.
					- delete these remote files! (may ask the user)
		- Free local space
			- Are local files present, which are remotely not present
				and these local files are older than ndr?
					- Meaning: You have worked remotely (or have synced with another PC), thus, local files are older
							   and these are not present remotely. Interpretation of change: You have delete these files
							   remotely, because your remotely files are newer that these files.
					- delete these local files! (may ask the user)
		- Download any new files
			- Are remote files present, which are locally not present?
					- Meaning: These files are new to the local side
					- Download these files! (may ask the user)
		- Upload any new files
			- Are local files present, which are remotely not present?
					- Meaning: These files are new to the remote side
					- Upload these files! (may ask the user)
		- Changed remote files
			- Are remote and local files present, where the remote file is newer?
					- Meaning: These files are changed on the remote side
					- Download these files and replace the local copies (may ask the user)
		- Changed local files
			- Are local and  remote files present, where the local file is newer?
					- Meaning: These files are changed on the local side
					- Upload these files and replace the remote copies (may ask the user)
	*/

	//
	// Read the local files ============================================================================================
	//
	log.Println("Try to read all local files now...")
	localFiles = make(map[string]os.FileInfo)
	filepath.Walk(localDir, walkerlocal)
	log.Printf("Found %d local files.\n", len(localFiles))

	//
	// Connect to the server ============================================================================================
	//
	sftp, sftpError := sftp.NewClient(ssh)
	if sftpError != nil {
		log.Println("Was not able to connect to the server: " + sftpError.Error())
		os.Exit(7)
		return
	}

	defer sftp.Close()

	//
	// Read the remote files ============================================================================================
	//
	log.Println("Try to read all remote files now...")
	remoteFiles = make(map[string]os.FileInfo)
	counterRemoteFile := 0
	walker := sftp.Walk(remoteDir)
	for walker.Step() {
		counterRemoteFile++
		if walker.Err() != nil {
			continue
		}

		remoteFiles[walker.Path()] = walker.Stat()
		if counterRemoteFile%512 == 0 {
			fmt.Printf("%05d.\n", counterRemoteFile)
		} else if counterRemoteFile%16 == 0 {
			fmt.Printf("%05d.", counterRemoteFile)
		}
	}

	fmt.Println()
	log.Printf("Found %d remote files.\n", len(remoteFiles))

	//
	// Normalise all local paths ============================================================================================
	//
	localFilesNormalised = make(map[string]os.FileInfo)
	for key, value := range localFiles {
		normalised := normalisePath(localDir, key)
		if normalised != `` {
			localFilesNormalised[normalised] = value
			normalised2localFiles[normalised] = key
		}
	}

	//
	// Normalise all remote paths ============================================================================================
	//
	remoteFilesNormalised = make(map[string]os.FileInfo)
	for key, value := range remoteFiles {
		normalised := normalisePath(remoteDir, key)
		if normalised != `` {
			remoteFilesNormalised[normalised] = value
			normalised2remoteFiles[normalised] = key
		}
	}

	//
	// Determine ndl and ndr ============================================================================================
	//
	ndl := time.Date(1, time.January, 1, 1, 1, 1, 1, time.UTC)
	ndr := ndl
	for _, remoteFileInfo := range remoteFiles {
		if remoteFileInfo.ModTime().UTC().After(ndr) {
			ndr = remoteFileInfo.ModTime().UTC()
		}
	}

	for _, localFileInfo := range localFiles {
		if localFileInfo.ModTime().UTC().After(ndl) {
			ndl = localFileInfo.ModTime().UTC()
		}
	}

	log.Printf("The newest local file's time: %v\n", ndl)
	log.Printf("The newest remote file's time: %v\n", ndr)

	//
	// Free remote space ============================================================================================
	//
	deleteRemoteFiles := make([]string, 0)
	for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised {
		localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
		if localFileNormaliesed == nil {
			if remoteFileInfo.ModTime().UTC().Before(ndl) {
				deleteRemoteFiles = append(deleteRemoteFiles, remoteFileNormalised)
			}
		}
	}

	log.Printf("Found %d remote files to delete.\n", len(deleteRemoteFiles))

	if len(deleteRemoteFiles) > 0 {
		sort.Strings(deleteRemoteFiles)
		shouldDeleteRemoteFiles := true
		if supervised {
			fmt.Println(`=================================================================`)
			for _, file := range deleteRemoteFiles {
				fmt.Println(normalised2remoteFiles[file])
			}

			fmt.Print("Should I delete these remote files? <y|n> ")
			shouldDeleteRemoteFiles = readYesNoAnswer(false)
		}

		if shouldDeleteRemoteFiles {
			// 1. Remove all files
			for _, remoteFileNormalised := range deleteRemoteFiles {

				// Skip all directories:
				if remoteFilesNormalised[remoteFileNormalised].IsDir() {
					continue
				}

				removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised])
				if removeError != nil {
					log.Printf("Was not able to delete the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error())
				} else {
					log.Printf("Deleted the remote file %s\n", normalised2remoteFiles[remoteFileNormalised])
				}
			}

			// 2. Remove all directories
			for _, remoteFileNormalised := range deleteRemoteFiles {

				// Skip all files:
				if !remoteFilesNormalised[remoteFileNormalised].IsDir() {
					continue
				}

				// TODO: Does not work: File an issue!
				removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised] + `/`)
				if removeError != nil {
					log.Printf("Was not able to delete the remote directory %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error())
				} else {
					log.Printf("Deleted the remote directory %s\n", normalised2remoteFiles[remoteFileNormalised])
				}
			}

			for _, remoteFileNormalised := range deleteRemoteFiles {
				delete(remoteFiles, normalised2remoteFiles[remoteFileNormalised])
				delete(remoteFilesNormalised, remoteFileNormalised)
			}
		}
	}

	//
	// Free local space ============================================================================================
	//
	if !pushOnly {
		deleteLocalFiles := make([]string, 0)
		for localFileNormalised, localFileInfo := range localFilesNormalised {
			remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
			if remoteFileNormaliesed == nil {
				if localFileInfo.ModTime().UTC().Before(ndr) {
					deleteLocalFiles = append(deleteLocalFiles, localFileNormalised)
				}
			}
		}

		log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles))

		if len(deleteLocalFiles) > 0 {
			sort.Strings(deleteLocalFiles)
			shouldDeleteLocalFiles := true
			if supervised {
				fmt.Println(`=================================================================`)
				for _, file := range deleteLocalFiles {
					fmt.Println(normalised2localFiles[file])
				}

				fmt.Print("Should I delete these local files? <y|n> ")
				shouldDeleteLocalFiles = readYesNoAnswer(false)
			}

			if shouldDeleteLocalFiles {
				for _, localFileNormalised := range deleteLocalFiles {

					// Skip all directories:
					if localFilesNormalised[localFileNormalised].IsDir() {
						continue
					}

					removeError := os.Remove(normalised2localFiles[localFileNormalised])
					if removeError != nil {
						log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
					} else {
						log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised])
					}
				}

				for _, localFileNormalised := range deleteLocalFiles {

					// Skip all files:
					if !localFilesNormalised[localFileNormalised].IsDir() {
						continue
					}

					removeError := os.Remove(normalised2localFiles[localFileNormalised])
					if removeError != nil {
						log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
					} else {
						log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised])
					}
				}

				for _, localFileNormalised := range deleteLocalFiles {
					delete(localFiles, normalised2localFiles[localFileNormalised])
					delete(localFilesNormalised, localFileNormalised)
				}
			}
		}
	}

	//
	// Download new files ============================================================================================
	//

	if !pushOnly {
		downloadRemoteFiles := make([]string, 0)
		for remoteFileNormalised, _ := range remoteFilesNormalised {
			localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
			if localFileNormaliesed == nil {
				downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised)
			}
		}

		log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles))

		if len(downloadRemoteFiles) > 0 {
			sort.Strings(downloadRemoteFiles)
			shouldDownloadRemoteFiles := true
			if supervised {
				fmt.Println(`=================================================================`)
				for _, file := range downloadRemoteFiles {
					fmt.Println(normalised2remoteFiles[file])
				}

				fmt.Print("Should I download these new remote files? <y|n> ")
				shouldDownloadRemoteFiles = readYesNoAnswer(false)
			}

			if shouldDownloadRemoteFiles {

				// 1. Create all new directories
				for _, remoteFileNormalised := range downloadRemoteFiles {

					// Skip all files
					if !remoteFilesNormalised[remoteFileNormalised].IsDir() {
						continue
					}

					newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
					log.Printf("Try to create the new local directory %s...\n", newLocalDir)
					if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil {
						log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error())
					}
				}

				// 2. All new files
				for _, remoteFileNormalised := range downloadRemoteFiles {

					// Skip all directories
					if remoteFilesNormalised[remoteFileNormalised].IsDir() {
						continue
					}

					log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
					remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
					if remoteFileHandleError != nil {
						log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
						continue
					}

					_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
					path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
					newLocalFile := filepath.Join(localDir, path, filename)
					localFileHandle, localFileHandleError := os.Create(newLocalFile)
					if localFileHandleError != nil {
						log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error())
						remoteFileHandle.Close()
						continue
					}

					_, copyError := io.Copy(localFileHandle, remoteFileHandle)
					if copyError != nil {
						log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error())
						remoteFileHandle.Close()
						localFileHandle.Close()
						continue
					}

					remoteFileHandle.Close()
					localFileHandle.Close()
				}
			}
		}
	}

	//
	// Upload new files ============================================================================================
	//

	uploadLocalFiles := make([]string, 0)
	for localFileNormalised, _ := range localFilesNormalised {
		remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
		if remoteFileNormaliesed == nil {
			uploadLocalFiles = append(uploadLocalFiles, localFileNormalised)
		}
	}

	log.Printf("Found %d new local files to upload.\n", len(uploadLocalFiles))

	if len(uploadLocalFiles) > 0 {
		sort.Strings(uploadLocalFiles)
		shouldUploadLocalFiles := true
		if supervised {
			fmt.Println(`=================================================================`)
			for _, file := range uploadLocalFiles {
				fmt.Println(normalised2localFiles[file])
			}

			fmt.Print("Should I upload these local new files? <y|n> ")
			shouldUploadLocalFiles = readYesNoAnswer(false)
		}

		if shouldUploadLocalFiles {

			// 1. Create new directories
			for _, localFileNormalised := range uploadLocalFiles {

				// Skip all files
				if !localFilesNormalised[localFileNormalised].IsDir() {
					continue
				}

				newRemoteDir := filepath.ToSlash(filepath.Join(remoteDir, strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)))
				log.Printf("Try to create the new remote directory %s...\n", newRemoteDir)
				if mkdirError := sftp.Mkdir(newRemoteDir); mkdirError != nil {
					log.Printf("Was not able to create the remote directory %s: %s\n", newRemoteDir, mkdirError.Error())
				}
			}

			// 2. All new files
			for _, localFileNormalised := range uploadLocalFiles {

				// Skip all directories
				if localFilesNormalised[localFileNormalised].IsDir() {
					continue
				}

				log.Printf("Try to upload the new local file %s...\n", normalised2localFiles[localFileNormalised])

				_, filename := filepath.Split(normalised2localFiles[localFileNormalised])
				path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))
				newRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename))
				remoteFileHandle, remoteFileHandleError := sftp.Create(newRemoteFile)
				if remoteFileHandleError != nil {
					log.Printf("Was not able to create the remote file %s: %s\n", newRemoteFile, remoteFileHandleError.Error())
					continue
				}

				localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised])
				if localFileHandleError != nil {
					log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error())
					remoteFileHandle.Close()
					continue
				}

				_, copyError := io.Copy(remoteFileHandle, localFileHandle)
				if copyError != nil {
					log.Printf("Was not able to upload the new local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], newRemoteFile, copyError.Error())
					remoteFileHandle.Close()
					localFileHandle.Close()
					continue
				}

				remoteFileHandle.Close()
				localFileHandle.Close()
			}
		}
	}

	//
	// Changed files on the remote side ============================================================================================
	//

	if !pushOnly {
		changedRemoteFiles := make([]string, 0)
		for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised {
			localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
			if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() {
				if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) {
					changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised)
				}
			}
		}

		log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles))

		if len(changedRemoteFiles) > 0 {
			sort.Strings(changedRemoteFiles)
			shouldDownloadRemoteFiles := true
			if supervised {
				fmt.Println(`=================================================================`)
				for _, file := range changedRemoteFiles {
					fmt.Println(normalised2remoteFiles[file])
				}

				fmt.Print("Should I download these changed remote files? <y|n> ")
				shouldDownloadRemoteFiles = readYesNoAnswer(false)
			}

			if shouldDownloadRemoteFiles {
				for _, remoteFileNormalised := range changedRemoteFiles {
					log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
					remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
					if remoteFileHandleError != nil {
						log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
						continue
					}

					_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
					path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
					existingLocalFile := filepath.Join(localDir, path, filename)
					localFileHandle, localFileHandleError := os.Create(existingLocalFile)
					if localFileHandleError != nil {
						log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error())
						remoteFileHandle.Close()
						continue
					}

					_, copyError := io.Copy(localFileHandle, remoteFileHandle)
					if copyError != nil {
						log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error())
						remoteFileHandle.Close()
						localFileHandle.Close()
						continue
					}

					remoteFileHandle.Close()
					localFileHandle.Close()
				}
			}
		}
	}

	//
	// Changed files on the local side ============================================================================================
	//

	changedLocalFiles := make([]string, 0)
	for localFileNormalised, localFileInfo := range localFilesNormalised {
		remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
		if remoteFileNormaliesed != nil && !remoteFileNormaliesed.IsDir() {
			if localFileInfo.ModTime().UTC().After(remoteFileNormaliesed.ModTime().UTC()) {
				changedLocalFiles = append(changedLocalFiles, localFileNormalised)
			}
		}
	}

	log.Printf("Found %d local files which are changed.\n", len(changedLocalFiles))

	if len(changedLocalFiles) > 0 {
		sort.Strings(changedLocalFiles)
		shouldUploadLocalFiles := true
		if supervised {
			fmt.Println(`=================================================================`)
			for _, file := range changedLocalFiles {
				fmt.Println(normalised2localFiles[file])
			}

			fmt.Print("Should I upload these changed local files? <y|n> ")
			shouldUploadLocalFiles = readYesNoAnswer(false)
		}

		if shouldUploadLocalFiles {
			for _, localFileNormalised := range changedLocalFiles {
				log.Printf("Try to upload the changed local file %s...\n", normalised2localFiles[localFileNormalised])

				_, filename := filepath.Split(normalised2localFiles[localFileNormalised])
				path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))
				existingRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename))
				remoteFileHandle, remoteFileHandleError := sftp.Create(existingRemoteFile)
				if remoteFileHandleError != nil {
					log.Printf("Was not able to overwrite the remote file %s: %s\n", existingRemoteFile, remoteFileHandleError.Error())
					continue
				}

				localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised])
				if localFileHandleError != nil {
					log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error())
					remoteFileHandle.Close()
					continue
				}

				_, copyError := io.Copy(remoteFileHandle, localFileHandle)
				if copyError != nil {
					log.Printf("Was not able to upload the changed local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], existingRemoteFile, copyError.Error())
					remoteFileHandle.Close()
					localFileHandle.Close()
					continue
				}

				remoteFileHandle.Close()
				localFileHandle.Close()
			}
		}
	}
}