Exemple #1
0
// buildSHA256List walks pathname tree and fills out
// gDigests [DigestFullPathT] as side effect.
func buildSHA256List(pathname string, info os.FileInfo, err error) error {
	time.Sleep(niceValue) // don't hog CPU
	if info == nil {
		fmt.Printf("WARNING --->  no stat info: %s\n", pathname)
		os.Exit(1)
	}
	if info.IsDir() {
		// do nothing
	} else {
		// TODO(mdr) test for regular file? skip pipes and devices etc
		digest, err := mdr.FileSHA256(pathname)
		if err != nil {
			fmt.Printf("!Err --->  SHA256 failed on %s\n", pathname)
			os.Exit(1)
		}
		var digPath DigestFullPathT = DigestFullPathT{
			Digest:   digest,
			FullPath: pathname,
		}
		gDigests = append(gDigests, &digPath)
		Verbose.Printf("adding %d %v\n", len(gDigests), digPath)
	}
	return nil
}
Exemple #2
0
func main() {
	// fmt.Printf("%s will run for 10 minutes then quit\n", Version)
	flag.Parse()
	err := flagSetup()
	if err != nil {
		usage()
		log.Panicf("flagSetup failed\n")
	}

	requestChan = make(chan SyncRequest)
	replyChan = make(chan SyncReply)
	// this is the receiving side
	if hasDest && hasPort {
		Verbose.Printf("Destination setup\n")
		buildDirMap(flagDest)
		go doServer(requestChan, replyChan)
		fmt.Printf("%s finished normally\n", invokedByName)
	}
	time.Sleep(200 * time.Millisecond) // allow a bit for receiver to finish setup

	// this is the sending side - not recursive for test phase
	if hasSource && hasPort {
		Verbose.Printf("Source setup\n")
		fileList, err := filepath.Glob(flagSource + "/*")
		if err != nil {
			log.Panic("here")
		}
		fileList2, err := filepath.Glob(flagSource + "/moreBins/*")
		if err != nil {
			log.Panic("here")
		}
		for i := 0; i < len(fileList2); i++ {
			fileList = append(fileList, fileList2[i])
		}
		for i := 0; i < len(fileList); i++ {
			fmt.Printf("sending %d %s\n", i, fileList[i])
		}
		for i := 0; i < len(fileList); i++ {
			// skip directories if any
			fname := fileList[i]
			info, err := os.Stat(fname)
			if err != nil {
				log.Panicf("!Err --->  stat failed on %s\n", fname)
			}
			if !info.Mode().IsRegular() {
				log.Printf("!Warning ---> %s is not a regular file\n", fname)
				continue
			}

			fmt.Printf("\n\n")
			// create name relative to top of source tree
			relativeName := fileList[i]
			relativeName = relativeName[len(flagSource)+1:] // remove sending directory
			log.Printf("Requesting dest's hash of %s\n", relativeName)

			var req SyncRequest = SyncRequest{
				Id:       i,
				CmdType:  LookupDigest,
				FilePath: relativeName,
			}
			myHash, err := mdr.FileSHA256(fileList[i]) // use full path here
			if err != nil {
				log.Panic("here")
			}
			log.Printf("sending hash is %s\n", myHash)
			requestChan <- req
			select {
			case reply := <-replyChan:
				Verbose.Printf("reply is: Id(%d) Digest(%s) File(%s)\n", reply.Id,
					reply.Digest, reply.FilePath)
				if myHash != reply.Digest {
					sourceName := filepath.Join(flagSource, req.FilePath)
					destName := filepath.Join(flagDest, req.FilePath)
					log.Printf("Need to copy %s to %s\n", sourceName, destName)
					err := safeCopy(sourceName, destName)
					if err != nil {
						log.Printf("!Err ---> safeCopy failed with %v\n", err)
						return
					}
					if true {
						var req SyncRequest = SyncRequest{
							Id:       i,
							CmdType:  RefreshDigest,
							FilePath: reply.FilePath,
						}
						log.Printf("requesting hash refresh for %s\n", req.FilePath)
						requestChan <- req
					}
				} else {
					//fmt.Printf("%s has same digest in both source and dest\n", req.FilePath)
				}
			}
		}

		fmt.Printf("%s finished normally\n", invokedByName)
	}
	//time.Sleep(3 * time.Second) // server will shutdown on wakeup
	// should send quit signal to doServer instead or use waitgroup
	fmt.Printf("%s finished normally\n", invokedByName)
}
Exemple #3
0
// doServer runs on destination system, serves up digests on request.
func doServer(requestChan chan SyncRequest, replyChan chan SyncReply) {
	// loops forever
	for {
		for {
			if serverMapIsReady {
				break
			}
			time.Sleep(time.Second)
		}
		// select should block till a request arrives
		select {
		case request := <-requestChan:
			Verbose.Printf("Rcvd a request: Id(%d) CmdType(%d) File %s\n", request.Id,
				request.CmdType, request.FilePath)
			requestedFile := filepath.Join(flagDest, request.FilePath)
			switch request.CmdType {
			case RefreshDigest:
				hashUpdate.Lock()
				log.Printf("Refreshing digest for %s\n", requestedFile)
				info, err := os.Stat(requestedFile)
				if err != nil {
					log.Printf("!Err --->  stat failed on %s\n", requestedFile)
					os.Exit(1)
				}
				if !info.Mode().IsRegular() {
					log.Printf("!Err ---> %s is not a regular file\n", requestedFile)
					return
				}
				digest, err := mdr.FileSHA256(requestedFile)
				if err != nil {
					fmt.Printf("!Err --->  SHA256 failed on %s\n", requestedFile)
					os.Exit(1)
				}
				//var digPath DigestFullPathT = DigestFullPathT{
				//	Digest:   digest,
				//	FullPath: requestedFile,
				//}
				gDigestMap[requestedFile] = digest
				hashUpdate.Unlock()
			case LookupDigest:
				hashUpdate.Lock()
				val, ok := gDigestMap[requestedFile]
				if ok {
					Verbose.Printf("Digest for %s is : %s\n", requestedFile, val)
					var reply SyncReply = SyncReply{
						Id:       request.Id,
						Digest:   val,
						FilePath: request.FilePath,
					}
					replyChan <- reply
				} else {
					fmt.Printf("Digest for %s is unknown\n", requestedFile)
					var reply SyncReply = SyncReply{
						Id:       request.Id,
						Digest:   "",
						FilePath: request.FilePath,
					}
					replyChan <- reply
				}
				hashUpdate.Unlock()
			}
		}
	}
}
Exemple #4
0
func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		fmt.Printf("Nothing to do - No arguments in command line\n")
		usage()
		return
	}
	if g_verboseFlag {
		Verbose = true
	}
	flagSetup()

	// BUG(mdr): TODO? flag for relative path or flag for abs path?
	pathName, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatalf("cant get absolute path for %s\n", flag.Arg(0))
	}

	lo := make(chan digestType, 5)
	//fmt.Printf("before lo %d gort running\n",runtime.NumGoroutine())
	go lineOut(lo)

	Verbose.Printf("Checking paths in %s\n", pathName)
	dirInfo, err := os.Stat(pathName)
	if err != nil {
		log.Fatalf("cant stat the directory %s\n", pathName)
	}
	dMode := dirInfo.Mode()
	if dMode.IsDir() == false {
		log.Fatalf("Path %s must be a directory (but isn't)\n", pathName)
	} else {
		Verbose.Printf("%s is a directory, walking starts now\n", pathName)
	}
	filepath.Walk(pathName, CheckPath) // builds g_argList
	var filesProcessed int64 = 0
	var bytesProcessed int64 = 0
	fmt.Fprintf(os.Stderr, "# nCPU = %d\n", nCPU)
	throttle := make(chan int, nCPU)
	startTime := time.Now()
	for _, fname := range g_argList {
		//fmt.Printf("Goroutines active = %d\n", runtime.NumGoroutine())
		throttle <- 1
		loop.Add(1)
		go func(fullpath string, accel chan int) {
			defer loop.Done()
			var tmp digestType
			tmp.pathname = fullpath
			stats, err := os.Stat(fullpath)
			if err != nil {
				log.Fatalf("Can't get fileinfo for %s\n", fullpath)
			}
			// check time for sanity (date < now()
			tmp.fileDate = stats.ModTime()
			if tmp.fileDate.After(startTime) {
				fmt.Printf("# bad date %s for %s\n", tmp.fileDate.String(), fullpath)
				// BUG(mdr): TODO? save bad dates in a list for appending
				g_badDateCt++
			}
			if g_noSHAFlag {
				// do nothing
			} else {
				tmp.dig256, err = mdr.FileSHA256(fullpath)
				if err != nil {
					log.Fatalf("SHA256 failed on %s\n", fullpath)
				}
			}
			tmp.fileLength = stats.Size()
			g_tmMutex.Lock()
			bytesProcessed += tmp.fileLength
			filesProcessed++
			g_tmMutex.Unlock()
			lo <- tmp
			<-accel // free a core
		}(fname, throttle)
	}
	loop.Wait()
	var doneRec digestType
	doneRec.pathname = ""
	lo <- doneRec
	outPut.Wait()
	//time.Sleep(1 * time.Second) // not necessary
	// wrapup
	elapsedTime := time.Now().Sub(startTime)
	elapsedSeconds := elapsedTime.Seconds()
	fmt.Printf("# %s Rundate=%s\n", G_version, startTime.String())
	fmt.Printf("# Processed %s files with %s bytes in %s for %.2g bytes/sec\n",
		mdr.CommaFmtInt64(filesProcessed), mdr.CommaFmtInt64(bytesProcessed), mdr.HumanTime(elapsedTime), float32(bytesProcessed)/float32(elapsedSeconds))
	fmt.Printf("# nCPU[%d]  BadDates[%d]\n", nCPU, g_badDateCt)
}