示例#1
0
文件: veb.go 项目: spydez/veb
// Compares local index against remote index, then copies the differing files
// to the remote location if remote doesn't have same checksum.
// Updates remote's index with the new file information after each file success,
// but doesn't /save/ remote's index to disk until finished.
func Push(local *veb.Index, log *veb.Log) error {
	defer log.Un(log.Trace(PUSH))
	var timer veb.Timer
	timer.Start()

	// open remote's index
	if local.Remote == "" {
		return fmt.Errorf("No remote veb repository. Use 'veb remote' to set one.")
	}
	remote, err := veb.Load(local.Remote, log) // TODO: have log indicate local vs remote
	if err != nil {
		return fmt.Errorf("veb could not load remote index: %v", err)
	}

	// get new/changed files for local & remote
	// we'll ignore these, as they haven't been committed
	locIgnore := make(chan veb.IndexEntry, CHAN_SIZE)
	remIgnore := make(chan veb.IndexEntry, CHAN_SIZE)
	go local.Check(locIgnore)
	go remote.Check(remIgnore)

	// notify user of ignored files
	cmt := true
	cmtMsg := func() {
		if cmt {
			fmt.Println("use 'veb status' to check new/changed files")
			fmt.Println("use 'veb commit' to add new/changed files to repository")
			cmt = false
		}
	}
	first := true
	locFilter := make(map[string]bool)
	remFilter := make(map[string]bool)
	for f := range locIgnore {
		if first {
			cmtMsg()
			fmt.Println("\n--------------------")
			fmt.Println("LOCAL ignored files:")
			fmt.Println("--------------------")
			first = false
		}
		fmt.Println(INDENT_F, f.Path) // filename
		locFilter[f.Path] = true      // add to filter
	}
	first = true
	for f := range remIgnore {
		if first {
			cmtMsg()
			fmt.Println("\n---------------------")
			fmt.Println("REMOTE ignored files:")
			fmt.Println("---------------------")
			first = false
		}
		fmt.Println(INDENT_F, f.Path) // filename
		remFilter[f.Path] = true      // add to filter
	}

	// make list of files to check
	files := make(chan veb.IndexEntry, CHAN_SIZE)
	numIgnored := 0
	go func() {
		for p, f := range local.Files {
			// ignore if it's one of the new/changed files
			_, skipL := locFilter[p]
			_, skipR := remFilter[p]
			if skipL || skipR {
				numIgnored++
			} else {
				files <- f
			}
		}
		close(files)
	}()

	// send files to remote if xsums differ
	done := make(chan int, MAX_HANDLERS)
	updates := make(chan veb.IndexEntry, CHAN_SIZE)
	errored := make(chan veb.IndexEntry, CHAN_SIZE)
	numErrored := 0
	numPushed := 0
	numNoChange := 0
	for i := 0; i < MAX_HANDLERS; i++ {
		go func() {
			for f := range files {
				// compare checksum hashes
				_, ok := remote.Files[f.Path] // does file exist in remote yet?
				if !ok || !bytes.Equal(f.Xsum, remote.Files[f.Path].Xsum) {
					// TODO: verify f.Xsum == local file's actual xsum
					//  - don't want corrupted files getting across.

					err := pushFile(local.Root, remote.Root, f, log)
					if err != nil {
						// notify of error, but continue with rest of files
						// TODO: get the error out too
						errored <- f
						numErrored++
					} else {
						// save entry so index can be updated
						updates <- f
						numPushed++
					}
				} else {
					numNoChange++
				}
			}
			done <- 1
		}()
	}

	// listeners
	var retVal error = nil
	numListeners := 3
	quit := make(chan int, numListeners)
	go func() {
		for i := 0; i < MAX_HANDLERS; i++ {
			<-done
		}
		close(updates)
		close(errored)
		quit <- 1
	}()
	go func() {
		for f := range errored {
			// clear status line w/ 80 spaces & carriage return
			fmt.Printf("\r%sError: could not push to remote: %s\n",
				"                                                                                \r",
				f.Path)

			if retVal == nil {
				retVal = fmt.Errorf("error transferring files to remote")
			}
		}
		quit <- 1
	}()
	go func() {
		first := true
		for f := range updates {
			if first {
				fmt.Printf("%s%s\n%s\n%s\n",
					"\r                                                                                \r",
					"\n-------------",
					"Pushed files:",
					"-------------")
				first = false
			}

			// clear status line w/ 80 spaces & carriage return
			fmt.Printf("\r%s%s %s\n",
				"                                                                                \r",
				INDENT_F, f.Path)
			remote.Update(&f)
		}
		quit <- 1
	}()

	// print status while waiting for everyone to finish
	for len(quit) < numListeners {
		fmt.Printf("\rstatus: %4d ignored, %4d errors, %4d pushed, %4d unchanged",
			numIgnored, numErrored, numPushed, numNoChange)
	}

	// save remote index's updates
	remote.Save()

	// print outro
	timer.Stop()
	fmt.Println("\r                                                                                ")
	fmt.Printf("status: %4d ignored, %4d errors, %4d pushed, %4d unchanged in %v\n",
		numIgnored, numErrored, numPushed, numNoChange, timer.Duration())

	// info log
	timer.Stop()
	log.Info().Printf("%s (%d ignored, %d errors, %d pushed, %d unchanged) took %v\n",
		PUSH, numIgnored, numErrored, numPushed, numNoChange, timer.Duration())
	return retVal
}
示例#2
0
文件: veb.go 项目: spydez/veb
// Command veb to do things with your stuff!
// See all the commands in const, above. Also flags.
// 'veb help' for a pretty print of flags/commands on command line.
func main() {
	out := log.New(os.Stdout, "", 0)

	// define flags
	// TODO
	//  - max CPUs
	//  - verbose

	// parse flags & args
	flag.Parse()

	// setup goroutine parallization
	// cuz it just runs on one processor out of the box...
	NUM_CPUS := runtime.NumCPU()
	runtime.GOMAXPROCS(NUM_CPUS)
	MAX_HANDLERS = NUM_CPUS
	if NUM_CPUS > 4 {
		// TODO: is running on all procs a good idea? (will it starve the system?)
		// using half for now
		// probably go back to all when niced.
		MAX_HANDLERS /= 2
	}

	// sanity check
	if len(flag.Args()) == 0 {
		// TODO
		out.Fatal("INSERT HELP HERE")
	}

	// init is a bit different (no pre-existing index),
	// so take care of it here instead of inside switch
	if flag.Args()[0] == INIT {
		err := Init()
		if err != nil {
			out.Fatal(err)
		}
		pwd, _ := os.Getwd()
		out.Println("Initialized empty veb repository at", pwd)
		return // done
	}

	// find veb repo
	root, err := cdBaseDir()
	if err != nil {
		fmt.Println(err, "\n")
		out.Fatal("Use 'veb init' to create this veb repository.")
	}

	// make the logger
	logf, err := os.OpenFile(path.Join(root, veb.META_FOLDER, veb.LOG_FILE),
		os.O_RDWR|os.O_APPEND|os.O_CREATE, 0644)
	log := veb.NewLog(log.New(logf, "", log.LstdFlags|log.Lshortfile))
	defer log.Info().Println("done\n\n")

	// load the index
	index, err := veb.Load(root, log)
	if err != nil {
		out.Fatal("veb could not load index")
	}

	// print intro
	fmt.Println("veb repository at", root, "\n")

	// act on command
	switch flag.Args()[0] {
	case STATUS:
		err = Status(index, log)
		if err != nil {
			out.Fatal(err)
		}

	case VERIFY:
		err = Verify(index, log)
		if err != nil {
			out.Fatal(err)
		}

	case COMMIT:
		err = Commit(index, log)
		if err != nil {
			out.Fatal(err)
		}

	case REMOTE:
		if len(flag.Args()) < 2 {
			out.Fatal(REMOTE, " needs a path to the backup repository",
				"\n  e.g. 'veb remote ~/backups/music'")
		}
		err = Remote(index, flag.Args()[1], log)
		if err != nil {
			out.Fatal(err)
		}

	case PUSH:
		err = Push(index, log)
		if err != nil {
			out.Fatal(err)
		}

	case FIX:
		// TODO: implement
		out.Fatal("this command is not yet implemented")
	case PULL:
		// TODO: implement
		out.Fatal("this command is not yet implemented")
	case SYNC:
		// TODO: implement
		out.Fatal("this command is not yet implemented")
	case HELP:
		// may provide per-cmd help later, but for now, just the default help
		// TODO: per command help
		fallthrough
	default:
		// TODO
		fmt.Println("INSERT HELP HERE")
	}
}