Example #1
0
File: veb.go Project: spydez/veb
// Sets the remote repository for this veb repo.
// Remote repository must already exist for it to be set.
func Remote(index *veb.Index, remote string, log *veb.Log) error {
	defer log.Un(log.Trace(REMOTE))
	var timer veb.Timer
	timer.Start()

	// make remote an absolute path
	if !path.IsAbs(remote) {
		remote = path.Join(index.Root, remote)
	}

	// check to see if remote exists
	fi, err := os.Stat(remote)
	if err != nil {
		if os.IsNotExist(err) {
			log.Err().Println(err)
			return fmt.Errorf("veb remote dir does not exist: %v", err)
		} else {
			log.Err().Println(err)
			return err
		}
	} else if !fi.IsDir() {
		// ain't a directory
		log.Err().Println(remote, "isn't a directory")
		return fmt.Errorf("veb remote must be a folder: %s is not a folder", remote)
	}

	// check to see if it's a veb repo
	remoteRepo := path.Join(remote, veb.META_FOLDER)
	fi, err = os.Stat(remoteRepo)
	if err != nil {
		if os.IsNotExist(err) {
			log.Err().Println(err)
			fmt.Println("veb remote needs to be initialized as a veb repository",
				"\n  (use 'veb init' in remote dir)")
			return err
		} else {
			log.Err().Println(err)
			return err
		}
	} else if !fi.IsDir() {
		// ain't a directory
		log.Err().Println(remoteRepo, "isn't a directory")
		fmt.Println("veb remote needs", remoteRepo, "to be a folder",
			"\nDelete or rename that file and run 'veb init' from", remote, "\n")
		return fmt.Errorf("%s isn't a directory", remoteRepo)
	}

	// set remote
	index.Remote = remote
	index.Save()
	fmt.Println("veb added", remote, "as the remote")

	// info log
	timer.Stop()
	log.Info().Printf("%s took %v\n",
		REMOTE, timer.Duration())
	return nil
}
Example #2
0
File: veb.go Project: spydez/veb
// Compares local index against remote index, then copies the differing files
// to the remote location if remote doesn't have same checksum.
// Updates remote's index with the new file information after each file success,
// but doesn't /save/ remote's index to disk until finished.
func Push(local *veb.Index, log *veb.Log) error {
	defer log.Un(log.Trace(PUSH))
	var timer veb.Timer
	timer.Start()

	// open remote's index
	if local.Remote == "" {
		return fmt.Errorf("No remote veb repository. Use 'veb remote' to set one.")
	}
	remote, err := veb.Load(local.Remote, log) // TODO: have log indicate local vs remote
	if err != nil {
		return fmt.Errorf("veb could not load remote index: %v", err)
	}

	// get new/changed files for local & remote
	// we'll ignore these, as they haven't been committed
	locIgnore := make(chan veb.IndexEntry, CHAN_SIZE)
	remIgnore := make(chan veb.IndexEntry, CHAN_SIZE)
	go local.Check(locIgnore)
	go remote.Check(remIgnore)

	// notify user of ignored files
	cmt := true
	cmtMsg := func() {
		if cmt {
			fmt.Println("use 'veb status' to check new/changed files")
			fmt.Println("use 'veb commit' to add new/changed files to repository")
			cmt = false
		}
	}
	first := true
	locFilter := make(map[string]bool)
	remFilter := make(map[string]bool)
	for f := range locIgnore {
		if first {
			cmtMsg()
			fmt.Println("\n--------------------")
			fmt.Println("LOCAL ignored files:")
			fmt.Println("--------------------")
			first = false
		}
		fmt.Println(INDENT_F, f.Path) // filename
		locFilter[f.Path] = true      // add to filter
	}
	first = true
	for f := range remIgnore {
		if first {
			cmtMsg()
			fmt.Println("\n---------------------")
			fmt.Println("REMOTE ignored files:")
			fmt.Println("---------------------")
			first = false
		}
		fmt.Println(INDENT_F, f.Path) // filename
		remFilter[f.Path] = true      // add to filter
	}

	// make list of files to check
	files := make(chan veb.IndexEntry, CHAN_SIZE)
	numIgnored := 0
	go func() {
		for p, f := range local.Files {
			// ignore if it's one of the new/changed files
			_, skipL := locFilter[p]
			_, skipR := remFilter[p]
			if skipL || skipR {
				numIgnored++
			} else {
				files <- f
			}
		}
		close(files)
	}()

	// send files to remote if xsums differ
	done := make(chan int, MAX_HANDLERS)
	updates := make(chan veb.IndexEntry, CHAN_SIZE)
	errored := make(chan veb.IndexEntry, CHAN_SIZE)
	numErrored := 0
	numPushed := 0
	numNoChange := 0
	for i := 0; i < MAX_HANDLERS; i++ {
		go func() {
			for f := range files {
				// compare checksum hashes
				_, ok := remote.Files[f.Path] // does file exist in remote yet?
				if !ok || !bytes.Equal(f.Xsum, remote.Files[f.Path].Xsum) {
					// TODO: verify f.Xsum == local file's actual xsum
					//  - don't want corrupted files getting across.

					err := pushFile(local.Root, remote.Root, f, log)
					if err != nil {
						// notify of error, but continue with rest of files
						// TODO: get the error out too
						errored <- f
						numErrored++
					} else {
						// save entry so index can be updated
						updates <- f
						numPushed++
					}
				} else {
					numNoChange++
				}
			}
			done <- 1
		}()
	}

	// listeners
	var retVal error = nil
	numListeners := 3
	quit := make(chan int, numListeners)
	go func() {
		for i := 0; i < MAX_HANDLERS; i++ {
			<-done
		}
		close(updates)
		close(errored)
		quit <- 1
	}()
	go func() {
		for f := range errored {
			// clear status line w/ 80 spaces & carriage return
			fmt.Printf("\r%sError: could not push to remote: %s\n",
				"                                                                                \r",
				f.Path)

			if retVal == nil {
				retVal = fmt.Errorf("error transferring files to remote")
			}
		}
		quit <- 1
	}()
	go func() {
		first := true
		for f := range updates {
			if first {
				fmt.Printf("%s%s\n%s\n%s\n",
					"\r                                                                                \r",
					"\n-------------",
					"Pushed files:",
					"-------------")
				first = false
			}

			// clear status line w/ 80 spaces & carriage return
			fmt.Printf("\r%s%s %s\n",
				"                                                                                \r",
				INDENT_F, f.Path)
			remote.Update(&f)
		}
		quit <- 1
	}()

	// print status while waiting for everyone to finish
	for len(quit) < numListeners {
		fmt.Printf("\rstatus: %4d ignored, %4d errors, %4d pushed, %4d unchanged",
			numIgnored, numErrored, numPushed, numNoChange)
	}

	// save remote index's updates
	remote.Save()

	// print outro
	timer.Stop()
	fmt.Println("\r                                                                                ")
	fmt.Printf("status: %4d ignored, %4d errors, %4d pushed, %4d unchanged in %v\n",
		numIgnored, numErrored, numPushed, numNoChange, timer.Duration())

	// info log
	timer.Stop()
	log.Info().Printf("%s (%d ignored, %d errors, %d pushed, %d unchanged) took %v\n",
		PUSH, numIgnored, numErrored, numPushed, numNoChange, timer.Duration())
	return retVal
}
Example #3
0
File: veb.go Project: spydez/veb
// Runs every file in index through hashing algorithm and compares the result
// against the xsum saved in the index.
// Does not verify new files.
// Could take a while. It chews through files in parallel, but it'll still take
// time to go through gigs of data.
// Allows early quitting by listening for QUIT_RUNE on stdin.
func Verify(index *veb.Index, log *veb.Log) error {
	defer log.Un(log.Trace(VERIFY))
	var timer veb.Timer
	timer.Start()

	// start listener for user's quit signal
	quit := make(chan int)
	go func() {
		for {
			var input string
			fmt.Scan(&input)
			if input[0] == QUIT_RUNE {
				quit <- 1
			}
		}
	}()

	// print intro
	fmt.Println("Verifying file checksums against those stored in veb index...")
	fmt.Println("Note: new files (as shown by 'veb status') will not be checked.\n")

	// bail early for empty index
	if len(index.Files) == 0 {
		fmt.Println("No files in veb index. Nothing to verify.")
		return nil
	}

	// toss everything in index into input channel
	files := make(chan veb.IndexEntry, CHAN_SIZE)
	go func() {
		for _, f := range index.Files {
			files <- f
		}
		close(files)
	}()

	// start handler pool working on checking files
	changed := make(chan veb.IndexEntry, CHAN_SIZE)
	done := make(chan int, MAX_HANDLERS)
	for i := 0; i < MAX_HANDLERS; i++ {
		go verifyHandler(index.Root, files, changed, done, log)
	}

	// done listener signals quit when all handlers are done
	go func() {
		for i := 0; i < MAX_HANDLERS; i++ {
			<-done
		}
		quit <- 1
	}()

	// receive & print info
	first := true
	totalFiles := len(index.Files)
	changedFiles := 0
	scannedFiles := 0
verify_receive_loop:
	for {
		// TODO: Rework to not need select. Race condition between quit signal and printing all changes.
		select {
		case <-quit:
			// We're done! Either by finishing or user interrupt.
			break verify_receive_loop

		case f := <-changed:
			// clear status line w/ carriage return & 80 spaces
			fmt.Println("\r                                                                                \r")

			// print header once first file is encountered
			if first {
				fmt.Println("----------------------")
				fmt.Println("Files with new hashes:")
				fmt.Println("----------------------")
				first = false
			}

			// TODO
			//  - move all this to a 'print changed' function?
			//    - has some in common with Status's printing

			// print file name
			fmt.Println(INDENT_F, f.Path)

			// figure out filesize
			curSize := ByteSize(f.Size)
			prevSize := ByteSize(index.Files[f.Path].Size)
			sizeChange := curSize - prevSize
			direction := "increased"
			if sizeChange < 0 {
				direction = "decreased"
				sizeChange = -sizeChange // absolute value
			}

			sanity := false

			// print size change
			// e.g.
			//     - filesize decreased 4.00MB (6.02GB -> 6.01GB)
			if sizeChange != 0 {
				fmt.Printf("%s filesize %s %s (%s -> %s)\n",
					INDENT_I, direction, sizeChange, prevSize, curSize)
				sanity = true
			}

			// print mtime
			if index.Files[f.Path].ModTime != f.ModTime {
				fmt.Printf("%s modified on (%v)\n", INDENT_I, f.ModTime)
				sanity = true
			}

			// print mode
			if index.Files[f.Path].Mode != f.Mode {
				fmt.Printf("%s file mode changed (%v -> %v)\n",
					INDENT_I, index.Files[f.Path].Mode, f.Mode)
				sanity = true
			}

			// sanity check & snark
			if !sanity {
				fmt.Printf("%s ...well /something/ changed. Dunno what. *shrugs*\n", INDENT_I)
			}

			// print xsums
			// TODO: dynamic hash name instead of hard 'SHA1'
			fmt.Printf("%s previous SHA1: %x\n", INDENT_I, index.Files[f.Path].Xsum)
			fmt.Printf("%s current  SHA1: %x\n", INDENT_I, f.Xsum)

			fmt.Printf("\n")

			// status line
			changedFiles++
			scannedFiles = totalFiles - len(files)
			fmt.Printf("\rscanned: %6d of %6d files (%d changed) (type 'q' to quit): ",
				scannedFiles, totalFiles, changedFiles)

		default:
			// status line
			scannedFiles = totalFiles - len(files)
			fmt.Printf("\rscanned: %6d of %6d files (%d changed) (type 'q' to quit): ",
				scannedFiles, totalFiles, changedFiles)
		}
	}

	notChecked := totalFiles - scannedFiles
	okFiles := totalFiles - changedFiles - notChecked

	// print outro
	timer.Stop()
	fmt.Println("\n\nMAKE SURE CHANGED FILES ARE THINGS YOU'VE ACTUALLY CHANGED")
	fmt.Println("  (use 'veb fix <file>' if a file has been corrupted in this repository)")
	fmt.Println("  (use 'veb push', 'veb pull', or 'veb sync' to commit changed/new files)")
	fmt.Printf("\nsummary: %d ok, %d changed, %d not checked in %v\n",
		okFiles, changedFiles, notChecked, timer.Duration())

	// info log
	log.Info().Printf("%s (%d ok, %d changed, %d not checked) took %v\n",
		VERIFY, okFiles, changedFiles, notChecked, timer.Duration())
	return nil
}
Example #4
0
File: veb.go Project: spydez/veb
// Saves all updated/new files to index, so they are available for push/pull.
// Saves new file stats & current checksum of the file shown as new/changed.
func Commit(index *veb.Index, log *veb.Log) error {
	defer log.Un(log.Trace(COMMIT))
	var timer veb.Timer
	timer.Start()

	// check for changes
	files := make(chan veb.IndexEntry, CHAN_SIZE)
	go index.Check(files)

	// start handler pool working on files
	updates := make(chan veb.IndexEntry, CHAN_SIZE)
	done := make(chan int, MAX_HANDLERS)
	for i := 0; i < MAX_HANDLERS; i++ {
		go func() {
			for f := range files {
				// calculate checksum hash
				err := veb.Xsum(&f, log)
				if err != nil {
					log.Err().Println("checksum for verify failed:", err)
				}

				updates <- f
			}
			done <- 1
		}()
	}

	// done listener
	go func() {
		for i := 0; i < MAX_HANDLERS; i++ {
			<-done
		}
		close(updates)
	}()

	// update index
	var retVal error = nil
	numCommits := 0
	numErrors := 0
	first := true
	for f := range updates {
		err := index.Update(&f)
		if err != nil {
			log.Err().Println("index update failed:", err)
			fmt.Println("Error: Couldn't commit", f.Path, ":", err)
			retVal = fmt.Errorf("veb commit failed")
			numErrors++
		} else {
			if first {
				fmt.Println("----------------")
				fmt.Println("Committed files:")
				fmt.Println("----------------")
				first = false
			}
			fmt.Println(INDENT_F, f.Path)
			numCommits++
		}
	}

	// save index once everything's done
	index.Save()

	// info
	timer.Stop()
	fmt.Println("\nsummary:", numCommits, "commits,", numErrors,
		"errors in", timer.Duration())
	log.Info().Printf("%s (%d commits, %d errors) took %v\n",
		COMMIT, numCommits, numErrors, timer.Duration())
	return retVal
}
Example #5
0
File: veb.go Project: spydez/veb
// Check for updated/new files in repo, then nicely print out results.
// Doesn't check file content (that's saved for verify). This is just
// to /quickly/ find new or modified files via file.Lstat().
func Status(index *veb.Index, log *veb.Log) error {
	defer log.Un(log.Trace(STATUS))
	var timer veb.Timer
	timer.Start()

	// check for changes
	files := make(chan veb.IndexEntry, CHAN_SIZE)
	go index.Check(files)

	// parse into new vs changed
	newFiles := make([]string, 0)
	changedFiles := make([]string, 0)
	var noTime time.Time
	for f := range files {
		// new file?
		if f.Name == "" && f.Size == 0 && f.Mode == 0 && f.ModTime == noTime {
			// new file.
			newFiles = append(newFiles, f.Path)
		} else {
			// changed file.
			changedFiles = append(changedFiles, f.Path)
		}
	}

	// print new files
	if len(newFiles) > 0 {
		fmt.Println("----------")
		fmt.Println("New files:")
		fmt.Println("----------")
		for _, f := range newFiles {
			// print file name
			fmt.Println(INDENT_F, f)

			// get stats for file info line
			fi, err := os.Stat(f)
			if err != nil {
				log.Err().Println(err)
				// print file info oops
				fmt.Println(INDENT_I, "could not get file info")
			} else {
				// print file info
				size := ByteSize(fi.Size())
				fmt.Printf("%s %s, modified on (%v)\n", INDENT_I, size, fi.ModTime())
			}
			fmt.Printf("\n")
		}
		fmt.Printf("\n")
	}

	// print changed files
	if len(changedFiles) > 0 {
		fmt.Println("--------------")
		fmt.Println("Changed files:")
		fmt.Println("--------------")
		for _, f := range changedFiles {
			// print file name
			fmt.Println(INDENT_F, f)

			// get stats for file info line
			fi, err := os.Stat(f)
			if err != nil {
				log.Err().Println(err)
				// print file info oops
				fmt.Println(INDENT_I, "could not get file info")
			} else {
				// figure out filesize
				curSize := ByteSize(fi.Size())
				prevSize := ByteSize(index.Files[f].Size)
				sizeChange := curSize - prevSize
				direction := "increased"
				if sizeChange < 0 {
					direction = "decreased"
					sizeChange = -sizeChange // absolute value
				}

				sanity := false

				// print size change
				// e.g.
				//     - filesize decreased 4.00MB (6.02GB -> 6.01GB)
				if sizeChange != 0 {
					fmt.Printf("%s filesize %s %s (%s -> %s)\n",
						INDENT_I, direction, sizeChange, prevSize, curSize)
					sanity = true
				}

				// print mtime
				if index.Files[f].ModTime != fi.ModTime() {
					fmt.Printf("%s modified on (%v)\n", INDENT_I, fi.ModTime())
					sanity = true
				}

				// print mode
				if index.Files[f].Mode != fi.Mode() {
					fmt.Printf("%s file mode changed (%v -> %v)\n",
						INDENT_I, index.Files[f].Mode, fi.Mode())
					sanity = true
				}

				// sanity check & snark
				if !sanity {
					fmt.Printf("%s ...well /something/ changed. Dunno what. *shrugs*\n", INDENT_I)
				}
			}
			fmt.Printf("\n")
		}
		fmt.Printf("\n")
	}

	// print outro
	if len(changedFiles) == 0 && len(newFiles) == 0 {
		fmt.Println("No changes or new files.")
	} else {
		fmt.Println("MAKE SURE CHANGED FILES ARE THINGS YOU'VE ACTUALLY CHANGED")
		fmt.Println("  (use 'veb fix <file>' if a file has been corrupted in this repository)")
		fmt.Println("  (use 'veb push', 'veb pull', or 'veb sync' to commit changed/new files)")
	}
	timer.Stop()
	fmt.Printf("\nsummary: %d new, %d changed (%v)\n",
		len(newFiles), len(changedFiles), timer.Duration())

	log.Info().Printf("%s (%d new, %d changed) took %v\n",
		STATUS, len(newFiles), len(changedFiles), timer.Duration())
	return nil
}