func (i *blueBoxInstance) UploadScript(ctx gocontext.Context, script []byte) error {
	client, err := i.sshClient(ctx)
	if err != nil {
		return err
	}
	defer client.Close()

	sftp, err := sftp.NewClient(client)
	if err != nil {
		return err
	}
	defer sftp.Close()

	_, err = sftp.Lstat("build.sh")
	if err == nil {
		return ErrStaleVM
	}

	f, err := sftp.Create("build.sh")
	if err != nil {
		return err
	}

	_, err = f.Write(script)
	if err != nil {
		return err
	}

	return err
}
Exemple #2
0
func (conn *SshConfig) Put(local, remote string) error {
	sftp, err := conn.transport()
	if err != nil {
		return err
	}

	defer sftp.Close()

	src, err := ioutil.ReadFile(local)
	if err != nil {
		return err
	}

	dst, err := sftp.Create(remote)
	if err != nil {
		return errors.New("Could not create the remote file.")
	}

	fmt.Printf("%s --> %s:%s\n", local, conn.Name, remote)
	if _, err := dst.Write(src); err != nil {
		return err
	}

	return nil
}
Exemple #3
0
// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created. `dir` must
// be delimited by forward slashes ("/"), which is required by sftp.
func Create(dir string, program string, args ...string) (*SFTP, error) {
	debug.Log("%v %v", program, args)
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if config file already exists
	_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp blobs
	for _, d := range paths(dir) {
		err = sftp.mkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	err = sftp.Close()
	if err != nil {
		return nil, errors.Wrap(err, "Close")
	}

	// open backend
	return Open(dir, program, args...)
}
func (c *DockerDeployClient) copyFile(source string, target string) error {
	sftp, err := sftp.NewClient(c.sshClient)
	if err != nil {
		log.Fatalf("Could not initialize SFTP connection: %v", err)
	}
	defer sftp.Close()

	tf, err := sftp.Create(target)
	if err != nil {
		return err
	}
	defer tf.Close()

	sf, err := os.Open(source)
	if err != nil {
		return err
	}
	defer sf.Close()

	n, err := io.Copy(tf, sf)
	if err != nil {
		return err
	}

	log.Printf("Artifact from %v to %v copied. %v Bytes transferred.", source, target, n)
	return nil
}
Exemple #5
0
func Rollback(rollback int, deploy_config setup.DeployConfiguration, commands setup.CommandConfig) {
	/* remove the current directory, and switch back to version current - number
	 */
	connection, err := NewConnection(deploy_config.TargetHost, deploy_config.TargetPort, deploy_config.Username, deploy_config.Password, deploy_config.PublicKeyPath)
	if err != nil {
		log.Panic(err)
	}
	sftp, err := sftp.NewClient(connection)
	if err != nil {
		log.Fatal(err)
	}
	defer sftp.Close()

	var build_list []string

	w, err := sftp.ReadDir(deploy_config.TargetDirectory)
	for _, folder := range w {
		if folder.Name() == "current" || !folder.IsDir() {
			continue
		}
		build_list = append(build_list, folder.Name())
	}
	sort.Sort(ByDate(build_list))

	rollback_version := build_list[len(build_list)-rollback] //list is ordered old to new, so we want to index in relation to the end of the list.

	//Create a symbolic link to the new directory.
	cmd := fmt.Sprintf("unlink %s", deploy_config.TargetDirectory+"/current")
	ExecuteCmd(connection, cmd, nil, nil)

	cmd = fmt.Sprintf("ln -s -f %s %s", deploy_config.TargetDirectory+"/"+rollback_version, deploy_config.TargetDirectory+"/current")
	ExecuteCmd(connection, cmd, nil, nil)
}
Exemple #6
0
func Example(conn *ssh.ClientConn) {
	// open an SFTP sesison over an existing ssh connection.
	sftp, err := sftp.NewClient(conn)
	if err != nil {
		log.Fatal(err)
	}
	defer sftp.Close()

	// walk a directory
	w := sftp.Walk("/home/user")
	for w.Step() {
		if w.Err() != nil {
			continue
		}
		log.Println(w.Path())
	}

	// leave your mark
	f, err := sftp.Create("hello.txt")
	if err != nil {
		log.Fatal(err)
	}
	if _, err := f.Write([]byte("Hello world!")); err != nil {
		log.Fatal(err)
	}

	// check it's there
	fi, err := sftp.Lstat("hello.txt")
	if err != nil {
		log.Fatal(err)
	}
	log.Println(fi)
}
func (c *sshClientImpl) UploadFile(username string, password string, ip string, srcFile string, destFile string) error {
	config := &myssh.ClientConfig{
		User: username,
		Auth: []myssh.AuthMethod{
			myssh.Password(password),
		},
	}
	if !IsIP(ip) {
		return errors.New("invalid IP address")
	}

	if IsDir(srcFile) || IsDir(destFile) {
		return errors.New("Is a directory")
	}

	client, err := myssh.Dial("tcp", ip+":22", config)
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer client.Close()

	sftp, err := sftp.NewClient(client)
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer sftp.Close()

	data, err := ioutil.ReadFile(srcFile)
	if err != nil {
		log.Fatal(err)
		return err
	}

	f, err := sftp.Create(destFile)
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer f.Close()

	_, err = f.Write([]byte(data))
	if err != nil {
		log.Fatal(err)
		return err
	}

	_, err = sftp.Lstat(destFile)
	if err != nil {
		log.Fatal(err)
		return err
	}
	return nil
}
Exemple #8
0
/*
	Create a new version, remove oldest if there are more than *Rollback
	return the new version number so that the deploy knows where to place shit.
*/
func create_version(deploy_config setup.DeployConfiguration) string {
	connection, err := NewConnection(deploy_config.TargetHost, deploy_config.TargetPort, deploy_config.Username, deploy_config.Password, deploy_config.PublicKeyPath)
	if err != nil {
		log.Panic(err)
	}
	sftp, err := sftp.NewClient(connection)
	if err != nil {
		log.Fatal(err)
	}
	defer sftp.Close()

	new_version := time.Now().Format("20060102150405")

	var build_list []string

	w, err := sftp.ReadDir(deploy_config.TargetDirectory)
	for _, folder := range w {
		if folder.Name() == "current" || !folder.IsDir() {
			continue
		}
		build_list = append(build_list, folder.Name())
	}
	sort.Sort(ByDate(build_list))

	//Remove oldest in build_list
	if len(build_list)+1 > deploy_config.Rollbacks+1 {
		remove_list := build_list[:len(build_list)-deploy_config.Rollbacks] //remove any old that exceeds Rollbaks, which are in the start of the list
		for _, version := range remove_list {
			log.Printf("Removing version: %s\n", deploy_config.TargetDirectory+"/"+version)
			//err :=sftp.Remove(deploy_config.TargetDirectory + "/" + version)
			cmd := fmt.Sprintf("rm -r %s", deploy_config.TargetDirectory+"/"+version)
			err := ExecuteCmd(connection, cmd, nil, nil)
			if err != nil {
				log.Println(err)
			}
		}
	}

	log.Printf("Creating folder: %s\n", deploy_config.TargetDirectory+"/"+new_version)
	sftp.Mkdir(deploy_config.TargetDirectory + "/" + new_version)

	//Create a symbolic link to the new directory.
	cmd := fmt.Sprintf("unlink %s", deploy_config.TargetDirectory+"/current")
	//log.Printf("Removing old symlink: %s", cmd)
	ExecuteCmd(connection, cmd, nil, nil)

	cmd = fmt.Sprintf("ln -s -f %s %s", deploy_config.TargetDirectory+"/"+new_version, deploy_config.TargetDirectory+"/current")
	//log.Printf("Creating symlink: %s", cmd)
	ExecuteCmd(connection, cmd, nil, nil)

	return new_version
}
Exemple #9
0
func main() {
	if err := resolveSettings(pars); err != nil {
		log.Fatal(err)
	}

	conn, err := initSSH(pars)
	if err != nil {
		log.Fatal(err)
	}

	sftp, err := sftp.NewClient(conn)
	if err != nil {
		log.Fatal(err)
	}
	defer sftp.Close()

	finStat, err := os.Stat(inPath)
	if err != nil {
		log.Fatal(err)
	}
	origSize := finStat.Size()

	fin, err := os.Open(inPath)
	if err != nil {
		log.Fatal(err)
	}
	defer fin.Close()

	outPath := path.Clean(path.Join(pars.DstFolder, filepath.Base(inPath)))
	fout, err := sftp.Create(outPath)
	if err != nil {
		log.Fatal(err)
	}
	if _, err := io.Copy(fout, fin); err != nil {
		log.Fatal(err)
	}

	foutStat, err := sftp.Lstat(outPath)
	if err != nil {
		log.Fatal(err)
	}
	if finalSize := foutStat.Size(); origSize != finalSize {
		if err := sftp.Remove(outPath); err != nil {
			log.Printf("[ERROR] Unable to remove target file: %v", err)
		}
		log.Fatalf("Failed to upload %s to %s: expected %d bytes, got %d (missing %d)",
			inPath, outPath, origSize, finalSize, origSize-finalSize)
	}

	fmt.Println("Successfully uploaded", outPath)
}
func (c *sshClientImpl) DownloadFile(username string, password string, ip string, srcFile string, destFile string) error {
	config := &myssh.ClientConfig{
		User: username,
		Auth: []myssh.AuthMethod{
			myssh.Password(password),
		},
	}

	if !IsIP(ip) {
		return errors.New("invalid IP address")
	}

	if IsDir(srcFile) || IsDir(destFile) {
		return errors.New("Is a directory")
	}

	client, err := myssh.Dial("tcp", ip+":22", config)
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer client.Close()

	sftp, err := sftp.NewClient(client)
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer sftp.Close()

	pFile, err := sftp.Open(srcFile)
	if err != nil {
		log.Fatal(err)
		return err
	}
	defer pFile.Close()

	data, err := ioutil.ReadAll(pFile)
	if err != nil {
		log.Fatal(err)
		return err
	}

	err = ioutil.WriteFile(destFile, data, 0755)
	if err != nil {
		log.Fatal(err)
		return err
	}

	return nil
}
Exemple #11
0
func deploy_local(deploy_config setup.DeployConfiguration, commands setup.CommandConfig) {
	connection, err := NewConnection(deploy_config.TargetHost, deploy_config.TargetPort, deploy_config.Username, deploy_config.Password, deploy_config.PublicKeyPath)
	if err != nil {
		log.Panic(err)
	}

	//If this is the first time deploying, we need to create the path to the target directory
	if !CheckTargetDirectory(connection, deploy_config) {
		cmd := fmt.Sprintf("mkdir -p %s", deploy_config.TargetDirectory)
		ExecuteCmd(connection, cmd, nil, nil)
		//cmd = fmt.Sprintf("mkdir -p %s", deploy_config.TargetDirectory+"/current")
		//ExecuteCmd(connection, cmd, nil, nil)
	}

	new_version := create_version(deploy_config)

	sftp, err := sftp.NewClient(connection)
	if err != nil {
		log.Fatal(err)
	}
	defer sftp.Close()

	files := list_local_directory(deploy_config.Ignores)

	//TODO use "current" instead of the new_version directory, since we have a symbolic link
	for _, file := range files {
		content, _ := ioutil.ReadFile(file)
		file = strings.Replace(file, `\`, `/`, -1)
		target := deploy_config.TargetDirectory + "/" + new_version + "/" + file
		tree := deploy_config.TargetDirectory + "/" + new_version + "/"

		for _, dir := range strings.Split(file, "/")[:len(strings.Split(file, "/"))-1] {
			tree += dir + "/"
			log.Printf("Creating directory: %s\n", tree)
			err := sftp.Mkdir(tree)
			if err != nil {

			}
		}
		log.Printf("Creating file: %s\n", target)
		f, err := sftp.Create(target)
		if err != nil {
			log.Panic(err)
		}
		if _, err := f.Write([]byte(content)); err != nil {
			log.Panic(err)
		}
	}
}
Exemple #12
0
// Read a remote file and return the contents.
func (c *Client) ReadAll(filepath string) ([]byte, error) {
	sftp, err := c.SFTPClient()
	if err != nil {
		panic(err)
	}
	defer sftp.Close()

	file, err := sftp.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ioutil.ReadAll(file)
}
Exemple #13
0
func main() {

	// Attempt to open the input file
	flag.Parse()

	csv, err := os.OpenFile(*input, os.O_RDONLY, 0)
	if err != nil {
		log.Fatalln(err)
	}
	defer csv.Close()
	reader := bufio.NewReader(csv)
	if err != nil {
		log.Fatalln(err)
	}

	// SSH connection.
	config := &ssh.ClientConfig{
		User: *user,
		Auth: []ssh.AuthMethod{
			ssh.Password(*secret),
		},
	}

	conn, err := ssh.Dial("tcp", *address, config)
	if err != nil {
		log.Fatalln(err)
	}
	defer conn.Close()

	// SFTP client.
	sftp, err := sftp.NewClient(conn)
	if err != nil {
		log.Fatalln(err)
	}
	defer sftp.Close()

	// Upload over SFTP.
	sftpFile, err := sftp.OpenFile(filepath.Join(*directory, *file), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
	if err != nil {
		log.Fatalln(err)
	}
	defer sftpFile.Close()
	sftpFile.ReadFrom(reader)
}
Exemple #14
0
func copyToRemote(c *cli.Context) {
	for _, flag := range []string{"keyfile", "to", "host", "user"} {
		if c.Generic(flag) == nil {
			log.Fatalf("flag %s is required\n", flag)
		}
	}
	// process the keyfile
	buf, err := ioutil.ReadFile(c.String("keyfile"))
	if err != nil {
		log.Fatalf("error in reading private key file %s\n", err)
	}
	key, err := ssh.ParsePrivateKey(buf)
	if err != nil {
		log.Fatalf("error in parsing private key %s\n", key)
	}
	// client config
	config := &ssh.ClientConfig{
		User: c.String("user"),
		Auth: []ssh.AuthMethod{ssh.PublicKeys(key)},
	}
	// connection
	client, err := ssh.Dial("tcp", c.String("host")+":22", config)
	if err != nil {
		log.Fatalf("error in ssh connection %s\n", err)
	}
	defer client.Close()
	// sftp handler
	sftp, err := sftp.NewClient(client)
	if err != nil {
		log.Fatalf("error in sftp connection %s\n", err)
	}
	defer sftp.Close()
	// Remote file
	r, err := sftp.Create(filepath.Join(c.String("to"), "go_sftp.txt"))
	if err != nil {
		log.Fatalf("error in creating remote file %s\n", err)
	}
	l := strings.NewReader("Writing through golang sftp")
	if _, err := io.Copy(r, l); err != nil {
		log.Fatalf("error in writing file to remote system %s\n", err)
	}
	log.Println("written new file to remote system")
}
func (i *jupiterBrainInstance) UploadScript(ctx context.Context, script []byte) error {
	client, err := i.sshClient()
	if err != nil {
		return err
	}
	defer client.Close()

	sftp, err := sftp.NewClient(client)
	if err != nil {
		return err
	}
	defer sftp.Close()

	_, err = sftp.Lstat("build.sh")
	if err == nil {
		return ErrStaleVM
	}

	f, err := sftp.Create("build.sh")
	if err != nil {
		return err
	}

	_, err = f.Write(script)
	if err != nil {
		return err
	}

	f, err = sftp.Create("wrapper.sh")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(f, wrapperSh)

	return err
}
			Expect(err).NotTo(HaveOccurred())
			defer session.Close()

			accepted, err := session.SendRequest("subsystem", true, ssh.Marshal(subsysMsg{Subsystem: "sftp"}))
			Expect(err).NotTo(HaveOccurred())
			Expect(accepted).To(BeTrue())
		})

		It("starts an sftp server in write mode", func() {
			tempDir, err := ioutil.TempDir("", "sftp")
			Expect(err).NotTo(HaveOccurred())
			defer os.RemoveAll(tempDir)

			sftp, err := sftp.NewClient(client)
			Expect(err).NotTo(HaveOccurred())
			defer sftp.Close()

			By("creating the file")
			target := filepath.Join(tempDir, "textfile.txt")
			file, err := sftp.Create(target)
			Expect(err).NotTo(HaveOccurred())

			fileContents := []byte("---\nthis is a simple file\n\n")
			_, err = file.Write(fileContents)
			Expect(err).NotTo(HaveOccurred())

			err = file.Close()
			Expect(err).NotTo(HaveOccurred())

			Expect(ioutil.ReadFile(target)).To(Equal(fileContents))
Exemple #17
0
func (conn *SshConfig) Get(rPath string, env *Enviroment) error {
	sftp, err := conn.transport()
	if err != nil {
		return err
	}

	defer sftp.Close()

	// handler for *glob in rpath
	if GlobIn(PathChomp(rPath)) {
		err := conn.GlobHandler(rPath, sftp, env)
		if err != nil {
			return err
		}

		return nil
	}

	remoteFile, err := sftp.Open(rPath)
	if err != nil {
		return err
	}

	fStat, err := remoteFile.Stat()
	if err != nil {
		return err
	}

	if _, err := os.Stat(conn.Name); os.IsNotExist(err) {
		os.Mkdir(conn.Name, 0755)
	}

	switch {

	case (!IsRegularFile(fStat)):
		w := sftp.Walk(rPath)
		for w.Step() {
			if w.Err() != nil {
				continue
			}

			if IsRegularFile(w.Stat()) {
				err := conn.GetFile(w.Path(), PrependParent(conn.Name, w.Path()), sftp, env)
				if err != nil {
					return err
				}

			} else {
				_, err := os.Stat(PrependParent(conn.Name, w.Path()))

				if os.IsNotExist(err) {
					os.Mkdir(PrependParent(conn.Name, w.Path()), 0755)
				}
			}
		}

	default:
		conn.GetFile(rPath, filepath.Join(conn.Name, PathChomp(rPath)), sftp, env)
	}

	return nil
}
Exemple #18
0
func (r *Resource) Handle() {
	Log := log.New(os.Stdout, fmt.Sprintf("%s[%d] ", r.Parent.URL, r.DeviceID), log.Ltime|log.Lshortfile)

	for {
		time.Sleep(1 * time.Second)

		if !r.Parent.Enabled || r.InUse {
			continue
		}

		// Get Job
		var jobInstance *JobInstance = nil
		select {
		case r := <-r.JobQueue:
			jobInstance = r
		default:

		}

		if jobInstance == nil {
			select {
			case a := <-JobQueue:
				jobInstance = a
			default:
			}
		}

		if jobInstance == nil {
			continue
		}
		r.InUse = true

		Log.Println(jobInstance)

		if jobInstance.PID == -1 {
			// Send Model Data
			Log.Println("Uploading Model")
			sftp, err := sftp.NewClient(r.Parent.Client)
			if err != nil {
				Log.Fatal(err)
			}

			sftp.Mkdir(sftp.Join(r.Parent.WorkingDirectory, "model"))
			sftp.Mkdir(sftp.Join(r.Parent.WorkingDirectory, "model", strings.ToLower(jobInstance.Parent.Model.Name)))

			for _, file := range jobInstance.Parent.Model.Files {
				fIn, err := os.Open(fmt.Sprintf("data/%s/%s", jobInstance.Parent.Model.Name, file))
				if err != nil {
					Log.Fatalln(err)
				}

				fOut, err := sftp.Create(sftp.Join(r.Parent.WorkingDirectory, "model", strings.ToLower(jobInstance.Parent.Model.Name), file))
				if err != nil {
					Log.Fatalln(err)
				}

				io.Copy(fOut, fIn)

				fIn.Close()
				fIn.Close()
			}

			// Setup Job Type Parameters
			executable := ""
			configuration := ""

			if jobInstance.Parent.Template.IsProtoMol() {
				executable = "ProtoMol"
				configuration = "sim.conf"
			} else {
				executable = "python"
				configuration = "sim.py"
			}

			// Send Template Data
			template, err := jobInstance.Parent.Template.Process(r.DeviceID, *jobInstance.Parent)
			if err != nil {
				Log.Fatalln(err)
			}

			sftp.Mkdir(sftp.Join(r.Parent.WorkingDirectory, "job"))
			sftp.Mkdir(sftp.Join(r.Parent.WorkingDirectory, "job", strings.ToLower(jobInstance.Parent.Name)))
			sftp.Mkdir(sftp.Join(r.Parent.WorkingDirectory, "job", strings.ToLower(jobInstance.Parent.Name), strings.ToLower(fmt.Sprintf("%d", jobInstance.NumberInSequence()))))

			fOut, err := sftp.Create(sftp.Join(r.Parent.WorkingDirectory, "job", strings.ToLower(jobInstance.Parent.Name), strings.ToLower(fmt.Sprintf("%d", jobInstance.NumberInSequence())), configuration))
			if err != nil {
				Log.Fatalln(err)
			}
			fOut.Write(template)
			fOut.Close()

			sftp.Close()

			// Start job and retrieve PID
			Log.Println("Starting Job")
			session, err := r.Parent.Client.NewSession()
			if err != nil {
				Log.Fatalln(err)
			}

			command := "/bin/bash\n"
			command += "source ~/.bash_profile\n"
			if r.Parent.WorkingDirectory != "" {
				command += "cd " + r.Parent.WorkingDirectory + "\n"
			}
			command += strings.ToLower(fmt.Sprintf("cd job/%s/%d\n", jobInstance.Parent.Name, jobInstance.NumberInSequence()))
			command += "bash -c '(" + executable + " " + configuration + " &> log.txt 2>&1 & echo $! > pidfile); sleep 1; wait $(cat pidfile); echo $? > exit-status' &> /dev/null &\n"
			command += "sleep 1\n"
			command += "cat pidfile"

			sPID, err := session.CombinedOutput(command)
			if err != nil {
				Log.Fatalln(string(sPID), err)
			}
			session.Close()

			// Parse PID
			pid, err := strconv.Atoi(strings.TrimSpace(string(sPID)))
			if err != nil {
				Log.Fatalln(err)
			}
			Log.Println("PID:", pid)

			jobInstance.PID = pid
			if _, err := DB.Exec("update job_instance set pid = ? where id = ?", jobInstance.PID, jobInstance.ID); err != nil {
				Log.Fatalln(err)
			}

			if _, err := DB.Exec("update job_instance set resource_id = ? where id = ?", r.UUID, jobInstance.ID); err != nil {
				Log.Fatalln(err)
			}
		}

		// Wait for completion
		Log.Println("Waiting for completion")
		for {
			session, err := r.Parent.Client.NewSession()
			if err != nil {
				Log.Println(err)
				r.Parent.Connect()
			}

			command := ""
			if r.Parent.WorkingDirectory != "" {
				command += "cd " + r.Parent.WorkingDirectory + "\n"
			}
			command += strings.ToLower(fmt.Sprintf("cd job/%s/%d\n", jobInstance.Parent.Name, jobInstance.NumberInSequence()))
			command += fmt.Sprintf("if [[ ( ! -d /proc/%d ) || ( ! -z `grep zombie /proc/%d/status` ) ]]; then cat exit-status; fi", jobInstance.PID, jobInstance.PID)

			output, err := session.CombinedOutput(command)
			if err != nil {
				Log.Fatalln(string(output), err)
			}
			session.Close()

			if string(output) != "" {
				exitcode, err := strconv.Atoi(strings.TrimSpace(string(output)))
				if err != nil {
					Log.Fatalln(err)
				}
				Log.Println("Exit Code:", exitcode)

				break
			}

			time.Sleep(30 * time.Second)
		}

		// Copy Results to Archives
		for _, archive := range Archives {
			if !archive.Enabled {
				continue
			}

			jobFtp, err := sftp.NewClient(r.Parent.Client)
			if err != nil {
				Log.Fatal(err)
			}

			archiveFtp, err := sftp.NewClient(archive.Client)
			if err != nil {
				Log.Fatal(err)
			}

			workingPath := jobFtp.Join(strings.ToLower(jobInstance.Parent.Name), strings.ToLower(fmt.Sprintf("%d", jobInstance.NumberInSequence())))

			// Create Directory
			archiveFtp.Mkdir(archiveFtp.Join(archive.WorkingDirectory, strings.ToLower(jobInstance.Parent.Name)))
			archiveFtp.Mkdir(archiveFtp.Join(archive.WorkingDirectory, workingPath))

			// Find Files
			files, err := jobFtp.ReadDir(jobFtp.Join(r.Parent.WorkingDirectory, "job", workingPath))
			if err != nil {
				log.Fatal(err)
			}

			// Copy Files
			for _, file := range files {
				fIn, err := jobFtp.Open(jobFtp.Join(r.Parent.WorkingDirectory, "job", workingPath, file.Name()))
				if err != nil {
					Log.Fatalln(err)
				}
				defer fIn.Close()

				fOut, err := archiveFtp.Create(archiveFtp.Join(archive.WorkingDirectory, workingPath, file.Name()))
				if err != nil {
					Log.Fatalln(err)
				}
				defer fOut.Close()

				io.Copy(fOut, fIn)
			}
		}

		jobInstance.Completed = true
		if _, err := DB.Exec("update job_instance set completed = ? where id = ?", jobInstance.Completed, jobInstance.ID); err != nil {
			Log.Fatalln(err)
		}
		r.InUse = false
	}
}
Exemple #19
0
// source is the source of the file/folder
// destination is the path of the FOLDER where source will be transferred to
// NOTE: source will keep it original name when transferred
func (sshTransport *SSHTransport) Put(source, destination string, srcType string) error {
	client, session, err := sshTransport.getClientSession()
	if err != nil {
		return HenchErr(err, map[string]interface{}{
			"host": sshTransport.Host,
		}, fmt.Sprintf("Couldn't dial into %s", sshTransport.Host))
	}
	defer client.Close()
	defer session.Close()

	sftp, err := sftp.NewClient(client)
	if err != nil {
		return HenchErr(err, map[string]interface{}{
			"host": sshTransport.Host,
		}, "Error creating sftp client")
	}
	defer sftp.Close()

	dstPieces := strings.Split(destination, "/")

	// it will always equal home
	if dstPieces[0] == "${HOME}" {
		dstPieces[0] = "."
	}

	newDst := strings.Join(dstPieces, "/")

	// if the src is a dir
	// tar the dir, copy it over, untar it
	// remove tar files on both ends
	if srcType == "dir" {
		// This is done so hosts won't remove tars used by other file
		tarredFile := sshTransport.Host + "_" + filepath.Base(source) + ".tar"

		// create the local tar
		err := tarit(source, tarredFile, nil)

		if err != nil {
			return HenchErr(err, map[string]interface{}{
				"dir": source,
			}, "Failed to tar dir to transport")
		}
		defer os.Remove(tarredFile)

		sourceBuf, err := ioutil.ReadFile(tarredFile)
		if err != nil {
			return HenchErr(err, map[string]interface{}{
				"dir":  source,
				"file": tarredFile,
			}, "Failed to read tar")
		}

		tarredFilePath := filepath.Join(newDst, tarredFile)
		f, err := sftp.Create(tarredFilePath)
		if err != nil {
			return HenchErr(err, map[string]interface{}{
				"host": sshTransport.Host,
				"file": newDst,
			}, "Failed to create remote file")
		}

		if _, err := f.Write(sourceBuf); err != nil {
			return HenchErr(err, map[string]interface{}{
				"host":   sshTransport.Host,
				"file":   destination,
				"source": source,
			}, "Error writing to remote file ")
		}

		cmd := fmt.Sprintf("tar -xvf %s -C %s && rm -rf %s", tarredFilePath, newDst, tarredFilePath)
		_, err = session.CombinedOutput(cmd)
		if err != nil {
			return HenchErr(err, map[string]interface{}{
				"host": sshTransport.Host,
				"file": tarredFilePath,
			}, "While untarring on remote")
		}
	} else {
		f, err := sftp.Create(filepath.Join(newDst, filepath.Base(source)))
		if err != nil {
			return HenchErr(err, map[string]interface{}{
				"host": sshTransport.Host,
				"file": destination,
			}, "Failed to create remote file")
		}

		sourceBuf, err := ioutil.ReadFile(source)
		if err != nil {
			return HenchErr(err, map[string]interface{}{
				"file": source,
			}, "")
		}

		if _, err := f.Write(sourceBuf); err != nil {
			return HenchErr(err, map[string]interface{}{
				"host":   sshTransport.Host,
				"file":   destination,
				"source": source,
			}, "Error writing to remote file ")
		}
	}

	return nil
}
Exemple #20
0
func Synchronise(ssh *ssh.Client, supervised, pushOnly bool, localDir, remoteDir string) {
	/*
		Algorithm:
		- Get all local files and dirs
		- Get all remote files and dirs
		- Determine: ndl := newest mod time of all local files
		- Determine: ndr := newest mod time of all remote files
		- Free remote space
			- Are remote files present, which are locally not present
				and these remote files are older than ndl?
					- Meaning: You have worked locally and remotely are older files which are not locally present.
							   Interpretation of change: You have deleted these files locally, because you own local
							   files which are newer that these files.
					- delete these remote files! (may ask the user)
		- Free local space
			- Are local files present, which are remotely not present
				and these local files are older than ndr?
					- Meaning: You have worked remotely (or have synced with another PC), thus, local files are older
							   and these are not present remotely. Interpretation of change: You have delete these files
							   remotely, because your remotely files are newer that these files.
					- delete these local files! (may ask the user)
		- Download any new files
			- Are remote files present, which are locally not present?
					- Meaning: These files are new to the local side
					- Download these files! (may ask the user)
		- Upload any new files
			- Are local files present, which are remotely not present?
					- Meaning: These files are new to the remote side
					- Upload these files! (may ask the user)
		- Changed remote files
			- Are remote and local files present, where the remote file is newer?
					- Meaning: These files are changed on the remote side
					- Download these files and replace the local copies (may ask the user)
		- Changed local files
			- Are local and  remote files present, where the local file is newer?
					- Meaning: These files are changed on the local side
					- Upload these files and replace the remote copies (may ask the user)
	*/

	//
	// Read the local files ============================================================================================
	//
	log.Println("Try to read all local files now...")
	localFiles = make(map[string]os.FileInfo)
	filepath.Walk(localDir, walkerlocal)
	log.Printf("Found %d local files.\n", len(localFiles))

	//
	// Connect to the server ============================================================================================
	//
	sftp, sftpError := sftp.NewClient(ssh)
	if sftpError != nil {
		log.Println("Was not able to connect to the server: " + sftpError.Error())
		os.Exit(7)
		return
	}

	defer sftp.Close()

	//
	// Read the remote files ============================================================================================
	//
	log.Println("Try to read all remote files now...")
	remoteFiles = make(map[string]os.FileInfo)
	counterRemoteFile := 0
	walker := sftp.Walk(remoteDir)
	for walker.Step() {
		counterRemoteFile++
		if walker.Err() != nil {
			continue
		}

		remoteFiles[walker.Path()] = walker.Stat()
		if counterRemoteFile%512 == 0 {
			fmt.Printf("%05d.\n", counterRemoteFile)
		} else if counterRemoteFile%16 == 0 {
			fmt.Printf("%05d.", counterRemoteFile)
		}
	}

	fmt.Println()
	log.Printf("Found %d remote files.\n", len(remoteFiles))

	//
	// Normalise all local paths ============================================================================================
	//
	localFilesNormalised = make(map[string]os.FileInfo)
	for key, value := range localFiles {
		normalised := normalisePath(localDir, key)
		if normalised != `` {
			localFilesNormalised[normalised] = value
			normalised2localFiles[normalised] = key
		}
	}

	//
	// Normalise all remote paths ============================================================================================
	//
	remoteFilesNormalised = make(map[string]os.FileInfo)
	for key, value := range remoteFiles {
		normalised := normalisePath(remoteDir, key)
		if normalised != `` {
			remoteFilesNormalised[normalised] = value
			normalised2remoteFiles[normalised] = key
		}
	}

	//
	// Determine ndl and ndr ============================================================================================
	//
	ndl := time.Date(1, time.January, 1, 1, 1, 1, 1, time.UTC)
	ndr := ndl
	for _, remoteFileInfo := range remoteFiles {
		if remoteFileInfo.ModTime().UTC().After(ndr) {
			ndr = remoteFileInfo.ModTime().UTC()
		}
	}

	for _, localFileInfo := range localFiles {
		if localFileInfo.ModTime().UTC().After(ndl) {
			ndl = localFileInfo.ModTime().UTC()
		}
	}

	log.Printf("The newest local file's time: %v\n", ndl)
	log.Printf("The newest remote file's time: %v\n", ndr)

	//
	// Free remote space ============================================================================================
	//
	deleteRemoteFiles := make([]string, 0)
	for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised {
		localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
		if localFileNormaliesed == nil {
			if remoteFileInfo.ModTime().UTC().Before(ndl) {
				deleteRemoteFiles = append(deleteRemoteFiles, remoteFileNormalised)
			}
		}
	}

	log.Printf("Found %d remote files to delete.\n", len(deleteRemoteFiles))

	if len(deleteRemoteFiles) > 0 {
		sort.Strings(deleteRemoteFiles)
		shouldDeleteRemoteFiles := true
		if supervised {
			fmt.Println(`=================================================================`)
			for _, file := range deleteRemoteFiles {
				fmt.Println(normalised2remoteFiles[file])
			}

			fmt.Print("Should I delete these remote files? <y|n> ")
			shouldDeleteRemoteFiles = readYesNoAnswer(false)
		}

		if shouldDeleteRemoteFiles {
			// 1. Remove all files
			for _, remoteFileNormalised := range deleteRemoteFiles {

				// Skip all directories:
				if remoteFilesNormalised[remoteFileNormalised].IsDir() {
					continue
				}

				removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised])
				if removeError != nil {
					log.Printf("Was not able to delete the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error())
				} else {
					log.Printf("Deleted the remote file %s\n", normalised2remoteFiles[remoteFileNormalised])
				}
			}

			// 2. Remove all directories
			for _, remoteFileNormalised := range deleteRemoteFiles {

				// Skip all files:
				if !remoteFilesNormalised[remoteFileNormalised].IsDir() {
					continue
				}

				// TODO: Does not work: File an issue!
				removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised] + `/`)
				if removeError != nil {
					log.Printf("Was not able to delete the remote directory %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error())
				} else {
					log.Printf("Deleted the remote directory %s\n", normalised2remoteFiles[remoteFileNormalised])
				}
			}

			for _, remoteFileNormalised := range deleteRemoteFiles {
				delete(remoteFiles, normalised2remoteFiles[remoteFileNormalised])
				delete(remoteFilesNormalised, remoteFileNormalised)
			}
		}
	}

	//
	// Free local space ============================================================================================
	//
	if !pushOnly {
		deleteLocalFiles := make([]string, 0)
		for localFileNormalised, localFileInfo := range localFilesNormalised {
			remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
			if remoteFileNormaliesed == nil {
				if localFileInfo.ModTime().UTC().Before(ndr) {
					deleteLocalFiles = append(deleteLocalFiles, localFileNormalised)
				}
			}
		}

		log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles))

		if len(deleteLocalFiles) > 0 {
			sort.Strings(deleteLocalFiles)
			shouldDeleteLocalFiles := true
			if supervised {
				fmt.Println(`=================================================================`)
				for _, file := range deleteLocalFiles {
					fmt.Println(normalised2localFiles[file])
				}

				fmt.Print("Should I delete these local files? <y|n> ")
				shouldDeleteLocalFiles = readYesNoAnswer(false)
			}

			if shouldDeleteLocalFiles {
				for _, localFileNormalised := range deleteLocalFiles {

					// Skip all directories:
					if localFilesNormalised[localFileNormalised].IsDir() {
						continue
					}

					removeError := os.Remove(normalised2localFiles[localFileNormalised])
					if removeError != nil {
						log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
					} else {
						log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised])
					}
				}

				for _, localFileNormalised := range deleteLocalFiles {

					// Skip all files:
					if !localFilesNormalised[localFileNormalised].IsDir() {
						continue
					}

					removeError := os.Remove(normalised2localFiles[localFileNormalised])
					if removeError != nil {
						log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
					} else {
						log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised])
					}
				}

				for _, localFileNormalised := range deleteLocalFiles {
					delete(localFiles, normalised2localFiles[localFileNormalised])
					delete(localFilesNormalised, localFileNormalised)
				}
			}
		}
	}

	//
	// Download new files ============================================================================================
	//

	if !pushOnly {
		downloadRemoteFiles := make([]string, 0)
		for remoteFileNormalised, _ := range remoteFilesNormalised {
			localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
			if localFileNormaliesed == nil {
				downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised)
			}
		}

		log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles))

		if len(downloadRemoteFiles) > 0 {
			sort.Strings(downloadRemoteFiles)
			shouldDownloadRemoteFiles := true
			if supervised {
				fmt.Println(`=================================================================`)
				for _, file := range downloadRemoteFiles {
					fmt.Println(normalised2remoteFiles[file])
				}

				fmt.Print("Should I download these new remote files? <y|n> ")
				shouldDownloadRemoteFiles = readYesNoAnswer(false)
			}

			if shouldDownloadRemoteFiles {

				// 1. Create all new directories
				for _, remoteFileNormalised := range downloadRemoteFiles {

					// Skip all files
					if !remoteFilesNormalised[remoteFileNormalised].IsDir() {
						continue
					}

					newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
					log.Printf("Try to create the new local directory %s...\n", newLocalDir)
					if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil {
						log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error())
					}
				}

				// 2. All new files
				for _, remoteFileNormalised := range downloadRemoteFiles {

					// Skip all directories
					if remoteFilesNormalised[remoteFileNormalised].IsDir() {
						continue
					}

					log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
					remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
					if remoteFileHandleError != nil {
						log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
						continue
					}

					_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
					path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
					newLocalFile := filepath.Join(localDir, path, filename)
					localFileHandle, localFileHandleError := os.Create(newLocalFile)
					if localFileHandleError != nil {
						log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error())
						remoteFileHandle.Close()
						continue
					}

					_, copyError := io.Copy(localFileHandle, remoteFileHandle)
					if copyError != nil {
						log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error())
						remoteFileHandle.Close()
						localFileHandle.Close()
						continue
					}

					remoteFileHandle.Close()
					localFileHandle.Close()
				}
			}
		}
	}

	//
	// Upload new files ============================================================================================
	//

	uploadLocalFiles := make([]string, 0)
	for localFileNormalised, _ := range localFilesNormalised {
		remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
		if remoteFileNormaliesed == nil {
			uploadLocalFiles = append(uploadLocalFiles, localFileNormalised)
		}
	}

	log.Printf("Found %d new local files to upload.\n", len(uploadLocalFiles))

	if len(uploadLocalFiles) > 0 {
		sort.Strings(uploadLocalFiles)
		shouldUploadLocalFiles := true
		if supervised {
			fmt.Println(`=================================================================`)
			for _, file := range uploadLocalFiles {
				fmt.Println(normalised2localFiles[file])
			}

			fmt.Print("Should I upload these local new files? <y|n> ")
			shouldUploadLocalFiles = readYesNoAnswer(false)
		}

		if shouldUploadLocalFiles {

			// 1. Create new directories
			for _, localFileNormalised := range uploadLocalFiles {

				// Skip all files
				if !localFilesNormalised[localFileNormalised].IsDir() {
					continue
				}

				newRemoteDir := filepath.ToSlash(filepath.Join(remoteDir, strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)))
				log.Printf("Try to create the new remote directory %s...\n", newRemoteDir)
				if mkdirError := sftp.Mkdir(newRemoteDir); mkdirError != nil {
					log.Printf("Was not able to create the remote directory %s: %s\n", newRemoteDir, mkdirError.Error())
				}
			}

			// 2. All new files
			for _, localFileNormalised := range uploadLocalFiles {

				// Skip all directories
				if localFilesNormalised[localFileNormalised].IsDir() {
					continue
				}

				log.Printf("Try to upload the new local file %s...\n", normalised2localFiles[localFileNormalised])

				_, filename := filepath.Split(normalised2localFiles[localFileNormalised])
				path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))
				newRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename))
				remoteFileHandle, remoteFileHandleError := sftp.Create(newRemoteFile)
				if remoteFileHandleError != nil {
					log.Printf("Was not able to create the remote file %s: %s\n", newRemoteFile, remoteFileHandleError.Error())
					continue
				}

				localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised])
				if localFileHandleError != nil {
					log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error())
					remoteFileHandle.Close()
					continue
				}

				_, copyError := io.Copy(remoteFileHandle, localFileHandle)
				if copyError != nil {
					log.Printf("Was not able to upload the new local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], newRemoteFile, copyError.Error())
					remoteFileHandle.Close()
					localFileHandle.Close()
					continue
				}

				remoteFileHandle.Close()
				localFileHandle.Close()
			}
		}
	}

	//
	// Changed files on the remote side ============================================================================================
	//

	if !pushOnly {
		changedRemoteFiles := make([]string, 0)
		for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised {
			localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
			if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() {
				if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) {
					changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised)
				}
			}
		}

		log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles))

		if len(changedRemoteFiles) > 0 {
			sort.Strings(changedRemoteFiles)
			shouldDownloadRemoteFiles := true
			if supervised {
				fmt.Println(`=================================================================`)
				for _, file := range changedRemoteFiles {
					fmt.Println(normalised2remoteFiles[file])
				}

				fmt.Print("Should I download these changed remote files? <y|n> ")
				shouldDownloadRemoteFiles = readYesNoAnswer(false)
			}

			if shouldDownloadRemoteFiles {
				for _, remoteFileNormalised := range changedRemoteFiles {
					log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
					remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
					if remoteFileHandleError != nil {
						log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
						continue
					}

					_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
					path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
					existingLocalFile := filepath.Join(localDir, path, filename)
					localFileHandle, localFileHandleError := os.Create(existingLocalFile)
					if localFileHandleError != nil {
						log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error())
						remoteFileHandle.Close()
						continue
					}

					_, copyError := io.Copy(localFileHandle, remoteFileHandle)
					if copyError != nil {
						log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error())
						remoteFileHandle.Close()
						localFileHandle.Close()
						continue
					}

					remoteFileHandle.Close()
					localFileHandle.Close()
				}
			}
		}
	}

	//
	// Changed files on the local side ============================================================================================
	//

	changedLocalFiles := make([]string, 0)
	for localFileNormalised, localFileInfo := range localFilesNormalised {
		remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
		if remoteFileNormaliesed != nil && !remoteFileNormaliesed.IsDir() {
			if localFileInfo.ModTime().UTC().After(remoteFileNormaliesed.ModTime().UTC()) {
				changedLocalFiles = append(changedLocalFiles, localFileNormalised)
			}
		}
	}

	log.Printf("Found %d local files which are changed.\n", len(changedLocalFiles))

	if len(changedLocalFiles) > 0 {
		sort.Strings(changedLocalFiles)
		shouldUploadLocalFiles := true
		if supervised {
			fmt.Println(`=================================================================`)
			for _, file := range changedLocalFiles {
				fmt.Println(normalised2localFiles[file])
			}

			fmt.Print("Should I upload these changed local files? <y|n> ")
			shouldUploadLocalFiles = readYesNoAnswer(false)
		}

		if shouldUploadLocalFiles {
			for _, localFileNormalised := range changedLocalFiles {
				log.Printf("Try to upload the changed local file %s...\n", normalised2localFiles[localFileNormalised])

				_, filename := filepath.Split(normalised2localFiles[localFileNormalised])
				path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))
				existingRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename))
				remoteFileHandle, remoteFileHandleError := sftp.Create(existingRemoteFile)
				if remoteFileHandleError != nil {
					log.Printf("Was not able to overwrite the remote file %s: %s\n", existingRemoteFile, remoteFileHandleError.Error())
					continue
				}

				localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised])
				if localFileHandleError != nil {
					log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error())
					remoteFileHandle.Close()
					continue
				}

				_, copyError := io.Copy(remoteFileHandle, localFileHandle)
				if copyError != nil {
					log.Printf("Was not able to upload the changed local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], existingRemoteFile, copyError.Error())
					remoteFileHandle.Close()
					localFileHandle.Close()
					continue
				}

				remoteFileHandle.Close()
				localFileHandle.Close()
			}
		}
	}
}