// Pushes local committed file that are changed/new to remote repository.
// TODO: Don't use Copy. Use rsync. 'rsync -qa' perhaps.
func pushFile(localRoot, remoteRoot string, entry veb.IndexEntry, log *veb.Log) error {
// open local file
local, err := os.Open(path.Join(localRoot, entry.Path))
if err != nil {
log.Err().Println(err)
return err
}
defer local.Close()
// make remote dirs, if they don't exist
err = os.MkdirAll(path.Dir(path.Join(remoteRoot, entry.Path)), 0755)
if err != nil {
log.Err().Println(err)
return err
}
// open remote file
remote, err := os.OpenFile(path.Join(remoteRoot, entry.Path),
os.O_WRONLY|os.O_TRUNC|os.O_CREATE, entry.Mode)
if err != nil {
log.Err().Println(err)
return err
}
defer remote.Close()
// send it!
_, err = io.Copy(remote, local)
if err != nil {
log.Err().Println(err)
return err
}
return err
}
// Sets the remote repository for this veb repo.
// Remote repository must already exist for it to be set.
func Remote(index *veb.Index, remote string, log *veb.Log) error {
defer log.Un(log.Trace(REMOTE))
var timer veb.Timer
timer.Start()
// make remote an absolute path
if !path.IsAbs(remote) {
remote = path.Join(index.Root, remote)
}
// check to see if remote exists
fi, err := os.Stat(remote)
if err != nil {
if os.IsNotExist(err) {
log.Err().Println(err)
return fmt.Errorf("veb remote dir does not exist: %v", err)
} else {
log.Err().Println(err)
return err
}
} else if !fi.IsDir() {
// ain't a directory
log.Err().Println(remote, "isn't a directory")
return fmt.Errorf("veb remote must be a folder: %s is not a folder", remote)
}
// check to see if it's a veb repo
remoteRepo := path.Join(remote, veb.META_FOLDER)
fi, err = os.Stat(remoteRepo)
if err != nil {
if os.IsNotExist(err) {
log.Err().Println(err)
fmt.Println("veb remote needs to be initialized as a veb repository",
"\n (use 'veb init' in remote dir)")
return err
} else {
log.Err().Println(err)
return err
}
} else if !fi.IsDir() {
// ain't a directory
log.Err().Println(remoteRepo, "isn't a directory")
fmt.Println("veb remote needs", remoteRepo, "to be a folder",
"\nDelete or rename that file and run 'veb init' from", remote, "\n")
return fmt.Errorf("%s isn't a directory", remoteRepo)
}
// set remote
index.Remote = remote
index.Save()
fmt.Println("veb added", remote, "as the remote")
// info log
timer.Stop()
log.Info().Printf("%s took %v\n",
REMOTE, timer.Duration())
return nil
}
// Calculates checksums of item in files chan, then puts file stats & xsum
// of changed files out on the changed chan.
// Does not look at file stats to determine change. This is purely about xsums.
func verifyHandler(root string, files, changed chan veb.IndexEntry, done chan int, log *veb.Log) {
for f := range files {
// save off old xsum for comparison
oldXsum := f.Xsum
// get file size & such
err := veb.SetStats(root, &f)
if err != nil {
log.Err().Println("couldn't get stats:", err)
}
// calculate checksum hash
err = veb.Xsum(&f, log)
if err != nil {
log.Err().Println("checksum for verify failed:", err)
}
// see if it changed...
if !bytes.Equal(f.Xsum, oldXsum) {
changed <- f
}
}
done <- 1
}
// Compares local index against remote index, then copies the differing files
// to the remote location if remote doesn't have same checksum.
// Updates remote's index with the new file information after each file success,
// but doesn't /save/ remote's index to disk until finished.
func Push(local *veb.Index, log *veb.Log) error {
defer log.Un(log.Trace(PUSH))
var timer veb.Timer
timer.Start()
// open remote's index
if local.Remote == "" {
return fmt.Errorf("No remote veb repository. Use 'veb remote' to set one.")
}
remote, err := veb.Load(local.Remote, log) // TODO: have log indicate local vs remote
if err != nil {
return fmt.Errorf("veb could not load remote index: %v", err)
}
// get new/changed files for local & remote
// we'll ignore these, as they haven't been committed
locIgnore := make(chan veb.IndexEntry, CHAN_SIZE)
remIgnore := make(chan veb.IndexEntry, CHAN_SIZE)
go local.Check(locIgnore)
go remote.Check(remIgnore)
// notify user of ignored files
cmt := true
cmtMsg := func() {
if cmt {
fmt.Println("use 'veb status' to check new/changed files")
fmt.Println("use 'veb commit' to add new/changed files to repository")
cmt = false
}
}
first := true
locFilter := make(map[string]bool)
remFilter := make(map[string]bool)
for f := range locIgnore {
if first {
cmtMsg()
fmt.Println("\n--------------------")
fmt.Println("LOCAL ignored files:")
fmt.Println("--------------------")
first = false
}
fmt.Println(INDENT_F, f.Path) // filename
locFilter[f.Path] = true // add to filter
}
first = true
for f := range remIgnore {
if first {
cmtMsg()
fmt.Println("\n---------------------")
fmt.Println("REMOTE ignored files:")
fmt.Println("---------------------")
first = false
}
fmt.Println(INDENT_F, f.Path) // filename
remFilter[f.Path] = true // add to filter
}
// make list of files to check
files := make(chan veb.IndexEntry, CHAN_SIZE)
numIgnored := 0
go func() {
for p, f := range local.Files {
// ignore if it's one of the new/changed files
_, skipL := locFilter[p]
_, skipR := remFilter[p]
if skipL || skipR {
numIgnored++
} else {
files <- f
}
}
close(files)
}()
// send files to remote if xsums differ
done := make(chan int, MAX_HANDLERS)
updates := make(chan veb.IndexEntry, CHAN_SIZE)
errored := make(chan veb.IndexEntry, CHAN_SIZE)
numErrored := 0
numPushed := 0
numNoChange := 0
for i := 0; i < MAX_HANDLERS; i++ {
go func() {
for f := range files {
// compare checksum hashes
_, ok := remote.Files[f.Path] // does file exist in remote yet?
if !ok || !bytes.Equal(f.Xsum, remote.Files[f.Path].Xsum) {
// TODO: verify f.Xsum == local file's actual xsum
// - don't want corrupted files getting across.
err := pushFile(local.Root, remote.Root, f, log)
if err != nil {
// notify of error, but continue with rest of files
// TODO: get the error out too
errored <- f
numErrored++
} else {
// save entry so index can be updated
updates <- f
numPushed++
}
} else {
numNoChange++
}
}
done <- 1
}()
}
// listeners
var retVal error = nil
numListeners := 3
quit := make(chan int, numListeners)
go func() {
for i := 0; i < MAX_HANDLERS; i++ {
<-done
}
close(updates)
close(errored)
quit <- 1
}()
go func() {
for f := range errored {
// clear status line w/ 80 spaces & carriage return
fmt.Printf("\r%sError: could not push to remote: %s\n",
" \r",
f.Path)
if retVal == nil {
retVal = fmt.Errorf("error transferring files to remote")
}
}
quit <- 1
}()
go func() {
first := true
for f := range updates {
if first {
fmt.Printf("%s%s\n%s\n%s\n",
"\r \r",
"\n-------------",
"Pushed files:",
"-------------")
first = false
}
// clear status line w/ 80 spaces & carriage return
fmt.Printf("\r%s%s %s\n",
" \r",
INDENT_F, f.Path)
remote.Update(&f)
}
quit <- 1
}()
// print status while waiting for everyone to finish
for len(quit) < numListeners {
fmt.Printf("\rstatus: %4d ignored, %4d errors, %4d pushed, %4d unchanged",
numIgnored, numErrored, numPushed, numNoChange)
}
// save remote index's updates
remote.Save()
// print outro
timer.Stop()
fmt.Println("\r ")
fmt.Printf("status: %4d ignored, %4d errors, %4d pushed, %4d unchanged in %v\n",
numIgnored, numErrored, numPushed, numNoChange, timer.Duration())
// info log
timer.Stop()
log.Info().Printf("%s (%d ignored, %d errors, %d pushed, %d unchanged) took %v\n",
PUSH, numIgnored, numErrored, numPushed, numNoChange, timer.Duration())
return retVal
}
// Saves all updated/new files to index, so they are available for push/pull.
// Saves new file stats & current checksum of the file shown as new/changed.
func Commit(index *veb.Index, log *veb.Log) error {
defer log.Un(log.Trace(COMMIT))
var timer veb.Timer
timer.Start()
// check for changes
files := make(chan veb.IndexEntry, CHAN_SIZE)
go index.Check(files)
// start handler pool working on files
updates := make(chan veb.IndexEntry, CHAN_SIZE)
done := make(chan int, MAX_HANDLERS)
for i := 0; i < MAX_HANDLERS; i++ {
go func() {
for f := range files {
// calculate checksum hash
err := veb.Xsum(&f, log)
if err != nil {
log.Err().Println("checksum for verify failed:", err)
}
updates <- f
}
done <- 1
}()
}
// done listener
go func() {
for i := 0; i < MAX_HANDLERS; i++ {
<-done
}
close(updates)
}()
// update index
var retVal error = nil
numCommits := 0
numErrors := 0
first := true
for f := range updates {
err := index.Update(&f)
if err != nil {
log.Err().Println("index update failed:", err)
fmt.Println("Error: Couldn't commit", f.Path, ":", err)
retVal = fmt.Errorf("veb commit failed")
numErrors++
} else {
if first {
fmt.Println("----------------")
fmt.Println("Committed files:")
fmt.Println("----------------")
first = false
}
fmt.Println(INDENT_F, f.Path)
numCommits++
}
}
// save index once everything's done
index.Save()
// info
timer.Stop()
fmt.Println("\nsummary:", numCommits, "commits,", numErrors,
"errors in", timer.Duration())
log.Info().Printf("%s (%d commits, %d errors) took %v\n",
COMMIT, numCommits, numErrors, timer.Duration())
return retVal
}
// Runs every file in index through hashing algorithm and compares the result
// against the xsum saved in the index.
// Does not verify new files.
// Could take a while. It chews through files in parallel, but it'll still take
// time to go through gigs of data.
// Allows early quitting by listening for QUIT_RUNE on stdin.
func Verify(index *veb.Index, log *veb.Log) error {
defer log.Un(log.Trace(VERIFY))
var timer veb.Timer
timer.Start()
// start listener for user's quit signal
quit := make(chan int)
go func() {
for {
var input string
fmt.Scan(&input)
if input[0] == QUIT_RUNE {
quit <- 1
}
}
}()
// print intro
fmt.Println("Verifying file checksums against those stored in veb index...")
fmt.Println("Note: new files (as shown by 'veb status') will not be checked.\n")
// bail early for empty index
if len(index.Files) == 0 {
fmt.Println("No files in veb index. Nothing to verify.")
return nil
}
// toss everything in index into input channel
files := make(chan veb.IndexEntry, CHAN_SIZE)
go func() {
for _, f := range index.Files {
files <- f
}
close(files)
}()
// start handler pool working on checking files
changed := make(chan veb.IndexEntry, CHAN_SIZE)
done := make(chan int, MAX_HANDLERS)
for i := 0; i < MAX_HANDLERS; i++ {
go verifyHandler(index.Root, files, changed, done, log)
}
// done listener signals quit when all handlers are done
go func() {
for i := 0; i < MAX_HANDLERS; i++ {
<-done
}
quit <- 1
}()
// receive & print info
first := true
totalFiles := len(index.Files)
changedFiles := 0
scannedFiles := 0
verify_receive_loop:
for {
// TODO: Rework to not need select. Race condition between quit signal and printing all changes.
select {
case <-quit:
// We're done! Either by finishing or user interrupt.
break verify_receive_loop
case f := <-changed:
// clear status line w/ carriage return & 80 spaces
fmt.Println("\r \r")
// print header once first file is encountered
if first {
fmt.Println("----------------------")
fmt.Println("Files with new hashes:")
fmt.Println("----------------------")
first = false
}
// TODO
// - move all this to a 'print changed' function?
// - has some in common with Status's printing
// print file name
fmt.Println(INDENT_F, f.Path)
// figure out filesize
curSize := ByteSize(f.Size)
prevSize := ByteSize(index.Files[f.Path].Size)
sizeChange := curSize - prevSize
direction := "increased"
if sizeChange < 0 {
direction = "decreased"
sizeChange = -sizeChange // absolute value
}
sanity := false
// print size change
// e.g.
// - filesize decreased 4.00MB (6.02GB -> 6.01GB)
if sizeChange != 0 {
fmt.Printf("%s filesize %s %s (%s -> %s)\n",
INDENT_I, direction, sizeChange, prevSize, curSize)
sanity = true
}
// print mtime
if index.Files[f.Path].ModTime != f.ModTime {
fmt.Printf("%s modified on (%v)\n", INDENT_I, f.ModTime)
sanity = true
}
// print mode
if index.Files[f.Path].Mode != f.Mode {
fmt.Printf("%s file mode changed (%v -> %v)\n",
INDENT_I, index.Files[f.Path].Mode, f.Mode)
sanity = true
}
// sanity check & snark
if !sanity {
fmt.Printf("%s ...well /something/ changed. Dunno what. *shrugs*\n", INDENT_I)
}
// print xsums
// TODO: dynamic hash name instead of hard 'SHA1'
fmt.Printf("%s previous SHA1: %x\n", INDENT_I, index.Files[f.Path].Xsum)
fmt.Printf("%s current SHA1: %x\n", INDENT_I, f.Xsum)
fmt.Printf("\n")
// status line
changedFiles++
scannedFiles = totalFiles - len(files)
fmt.Printf("\rscanned: %6d of %6d files (%d changed) (type 'q' to quit): ",
scannedFiles, totalFiles, changedFiles)
default:
// status line
scannedFiles = totalFiles - len(files)
fmt.Printf("\rscanned: %6d of %6d files (%d changed) (type 'q' to quit): ",
scannedFiles, totalFiles, changedFiles)
}
}
notChecked := totalFiles - scannedFiles
okFiles := totalFiles - changedFiles - notChecked
// print outro
timer.Stop()
fmt.Println("\n\nMAKE SURE CHANGED FILES ARE THINGS YOU'VE ACTUALLY CHANGED")
fmt.Println(" (use 'veb fix <file>' if a file has been corrupted in this repository)")
fmt.Println(" (use 'veb push', 'veb pull', or 'veb sync' to commit changed/new files)")
fmt.Printf("\nsummary: %d ok, %d changed, %d not checked in %v\n",
okFiles, changedFiles, notChecked, timer.Duration())
// info log
log.Info().Printf("%s (%d ok, %d changed, %d not checked) took %v\n",
VERIFY, okFiles, changedFiles, notChecked, timer.Duration())
return nil
}
// Check for updated/new files in repo, then nicely print out results.
// Doesn't check file content (that's saved for verify). This is just
// to /quickly/ find new or modified files via file.Lstat().
func Status(index *veb.Index, log *veb.Log) error {
defer log.Un(log.Trace(STATUS))
var timer veb.Timer
timer.Start()
// check for changes
files := make(chan veb.IndexEntry, CHAN_SIZE)
go index.Check(files)
// parse into new vs changed
newFiles := make([]string, 0)
changedFiles := make([]string, 0)
var noTime time.Time
for f := range files {
// new file?
if f.Name == "" && f.Size == 0 && f.Mode == 0 && f.ModTime == noTime {
// new file.
newFiles = append(newFiles, f.Path)
} else {
// changed file.
changedFiles = append(changedFiles, f.Path)
}
}
// print new files
if len(newFiles) > 0 {
fmt.Println("----------")
fmt.Println("New files:")
fmt.Println("----------")
for _, f := range newFiles {
// print file name
fmt.Println(INDENT_F, f)
// get stats for file info line
fi, err := os.Stat(f)
if err != nil {
log.Err().Println(err)
// print file info oops
fmt.Println(INDENT_I, "could not get file info")
} else {
// print file info
size := ByteSize(fi.Size())
fmt.Printf("%s %s, modified on (%v)\n", INDENT_I, size, fi.ModTime())
}
fmt.Printf("\n")
}
fmt.Printf("\n")
}
// print changed files
if len(changedFiles) > 0 {
fmt.Println("--------------")
fmt.Println("Changed files:")
fmt.Println("--------------")
for _, f := range changedFiles {
// print file name
fmt.Println(INDENT_F, f)
// get stats for file info line
fi, err := os.Stat(f)
if err != nil {
log.Err().Println(err)
// print file info oops
fmt.Println(INDENT_I, "could not get file info")
} else {
// figure out filesize
curSize := ByteSize(fi.Size())
prevSize := ByteSize(index.Files[f].Size)
sizeChange := curSize - prevSize
direction := "increased"
if sizeChange < 0 {
direction = "decreased"
sizeChange = -sizeChange // absolute value
}
sanity := false
// print size change
// e.g.
// - filesize decreased 4.00MB (6.02GB -> 6.01GB)
if sizeChange != 0 {
fmt.Printf("%s filesize %s %s (%s -> %s)\n",
INDENT_I, direction, sizeChange, prevSize, curSize)
sanity = true
}
// print mtime
if index.Files[f].ModTime != fi.ModTime() {
fmt.Printf("%s modified on (%v)\n", INDENT_I, fi.ModTime())
sanity = true
}
// print mode
if index.Files[f].Mode != fi.Mode() {
fmt.Printf("%s file mode changed (%v -> %v)\n",
INDENT_I, index.Files[f].Mode, fi.Mode())
sanity = true
}
// sanity check & snark
if !sanity {
fmt.Printf("%s ...well /something/ changed. Dunno what. *shrugs*\n", INDENT_I)
}
}
fmt.Printf("\n")
}
fmt.Printf("\n")
}
// print outro
if len(changedFiles) == 0 && len(newFiles) == 0 {
fmt.Println("No changes or new files.")
} else {
fmt.Println("MAKE SURE CHANGED FILES ARE THINGS YOU'VE ACTUALLY CHANGED")
fmt.Println(" (use 'veb fix <file>' if a file has been corrupted in this repository)")
fmt.Println(" (use 'veb push', 'veb pull', or 'veb sync' to commit changed/new files)")
}
timer.Stop()
fmt.Printf("\nsummary: %d new, %d changed (%v)\n",
len(newFiles), len(changedFiles), timer.Duration())
log.Info().Printf("%s (%d new, %d changed) took %v\n",
STATUS, len(newFiles), len(changedFiles), timer.Duration())
return nil
}