// buildSHA256List walks pathname tree and fills out
// gDigests [DigestFullPathT] as side effect.
func buildSHA256List(pathname string, info os.FileInfo, err error) error {
time.Sleep(niceValue) // don't hog CPU
if info == nil {
fmt.Printf("WARNING ---> no stat info: %s\n", pathname)
os.Exit(1)
}
if info.IsDir() {
// do nothing
} else {
// TODO(mdr) test for regular file? skip pipes and devices etc
digest, err := mdr.FileSHA256(pathname)
if err != nil {
fmt.Printf("!Err ---> SHA256 failed on %s\n", pathname)
os.Exit(1)
}
var digPath DigestFullPathT = DigestFullPathT{
Digest: digest,
FullPath: pathname,
}
gDigests = append(gDigests, &digPath)
Verbose.Printf("adding %d %v\n", len(gDigests), digPath)
}
return nil
}
func main() {
// fmt.Printf("%s will run for 10 minutes then quit\n", Version)
flag.Parse()
err := flagSetup()
if err != nil {
usage()
log.Panicf("flagSetup failed\n")
}
requestChan = make(chan SyncRequest)
replyChan = make(chan SyncReply)
// this is the receiving side
if hasDest && hasPort {
Verbose.Printf("Destination setup\n")
buildDirMap(flagDest)
go doServer(requestChan, replyChan)
fmt.Printf("%s finished normally\n", invokedByName)
}
time.Sleep(200 * time.Millisecond) // allow a bit for receiver to finish setup
// this is the sending side - not recursive for test phase
if hasSource && hasPort {
Verbose.Printf("Source setup\n")
fileList, err := filepath.Glob(flagSource + "/*")
if err != nil {
log.Panic("here")
}
fileList2, err := filepath.Glob(flagSource + "/moreBins/*")
if err != nil {
log.Panic("here")
}
for i := 0; i < len(fileList2); i++ {
fileList = append(fileList, fileList2[i])
}
for i := 0; i < len(fileList); i++ {
fmt.Printf("sending %d %s\n", i, fileList[i])
}
for i := 0; i < len(fileList); i++ {
// skip directories if any
fname := fileList[i]
info, err := os.Stat(fname)
if err != nil {
log.Panicf("!Err ---> stat failed on %s\n", fname)
}
if !info.Mode().IsRegular() {
log.Printf("!Warning ---> %s is not a regular file\n", fname)
continue
}
fmt.Printf("\n\n")
// create name relative to top of source tree
relativeName := fileList[i]
relativeName = relativeName[len(flagSource)+1:] // remove sending directory
log.Printf("Requesting dest's hash of %s\n", relativeName)
var req SyncRequest = SyncRequest{
Id: i,
CmdType: LookupDigest,
FilePath: relativeName,
}
myHash, err := mdr.FileSHA256(fileList[i]) // use full path here
if err != nil {
log.Panic("here")
}
log.Printf("sending hash is %s\n", myHash)
requestChan <- req
select {
case reply := <-replyChan:
Verbose.Printf("reply is: Id(%d) Digest(%s) File(%s)\n", reply.Id,
reply.Digest, reply.FilePath)
if myHash != reply.Digest {
sourceName := filepath.Join(flagSource, req.FilePath)
destName := filepath.Join(flagDest, req.FilePath)
log.Printf("Need to copy %s to %s\n", sourceName, destName)
err := safeCopy(sourceName, destName)
if err != nil {
log.Printf("!Err ---> safeCopy failed with %v\n", err)
return
}
if true {
var req SyncRequest = SyncRequest{
Id: i,
CmdType: RefreshDigest,
FilePath: reply.FilePath,
}
log.Printf("requesting hash refresh for %s\n", req.FilePath)
requestChan <- req
}
} else {
//fmt.Printf("%s has same digest in both source and dest\n", req.FilePath)
}
}
}
fmt.Printf("%s finished normally\n", invokedByName)
}
//time.Sleep(3 * time.Second) // server will shutdown on wakeup
// should send quit signal to doServer instead or use waitgroup
fmt.Printf("%s finished normally\n", invokedByName)
}
// doServer runs on destination system, serves up digests on request.
func doServer(requestChan chan SyncRequest, replyChan chan SyncReply) {
// loops forever
for {
for {
if serverMapIsReady {
break
}
time.Sleep(time.Second)
}
// select should block till a request arrives
select {
case request := <-requestChan:
Verbose.Printf("Rcvd a request: Id(%d) CmdType(%d) File %s\n", request.Id,
request.CmdType, request.FilePath)
requestedFile := filepath.Join(flagDest, request.FilePath)
switch request.CmdType {
case RefreshDigest:
hashUpdate.Lock()
log.Printf("Refreshing digest for %s\n", requestedFile)
info, err := os.Stat(requestedFile)
if err != nil {
log.Printf("!Err ---> stat failed on %s\n", requestedFile)
os.Exit(1)
}
if !info.Mode().IsRegular() {
log.Printf("!Err ---> %s is not a regular file\n", requestedFile)
return
}
digest, err := mdr.FileSHA256(requestedFile)
if err != nil {
fmt.Printf("!Err ---> SHA256 failed on %s\n", requestedFile)
os.Exit(1)
}
//var digPath DigestFullPathT = DigestFullPathT{
// Digest: digest,
// FullPath: requestedFile,
//}
gDigestMap[requestedFile] = digest
hashUpdate.Unlock()
case LookupDigest:
hashUpdate.Lock()
val, ok := gDigestMap[requestedFile]
if ok {
Verbose.Printf("Digest for %s is : %s\n", requestedFile, val)
var reply SyncReply = SyncReply{
Id: request.Id,
Digest: val,
FilePath: request.FilePath,
}
replyChan <- reply
} else {
fmt.Printf("Digest for %s is unknown\n", requestedFile)
var reply SyncReply = SyncReply{
Id: request.Id,
Digest: "",
FilePath: request.FilePath,
}
replyChan <- reply
}
hashUpdate.Unlock()
}
}
}
}
func main() {
flag.Parse()
if flag.NArg() == 0 {
fmt.Printf("Nothing to do - No arguments in command line\n")
usage()
return
}
if g_verboseFlag {
Verbose = true
}
flagSetup()
// BUG(mdr): TODO? flag for relative path or flag for abs path?
pathName, err := filepath.Abs(flag.Arg(0))
if err != nil {
log.Fatalf("cant get absolute path for %s\n", flag.Arg(0))
}
lo := make(chan digestType, 5)
//fmt.Printf("before lo %d gort running\n",runtime.NumGoroutine())
go lineOut(lo)
Verbose.Printf("Checking paths in %s\n", pathName)
dirInfo, err := os.Stat(pathName)
if err != nil {
log.Fatalf("cant stat the directory %s\n", pathName)
}
dMode := dirInfo.Mode()
if dMode.IsDir() == false {
log.Fatalf("Path %s must be a directory (but isn't)\n", pathName)
} else {
Verbose.Printf("%s is a directory, walking starts now\n", pathName)
}
filepath.Walk(pathName, CheckPath) // builds g_argList
var filesProcessed int64 = 0
var bytesProcessed int64 = 0
fmt.Fprintf(os.Stderr, "# nCPU = %d\n", nCPU)
throttle := make(chan int, nCPU)
startTime := time.Now()
for _, fname := range g_argList {
//fmt.Printf("Goroutines active = %d\n", runtime.NumGoroutine())
throttle <- 1
loop.Add(1)
go func(fullpath string, accel chan int) {
defer loop.Done()
var tmp digestType
tmp.pathname = fullpath
stats, err := os.Stat(fullpath)
if err != nil {
log.Fatalf("Can't get fileinfo for %s\n", fullpath)
}
// check time for sanity (date < now()
tmp.fileDate = stats.ModTime()
if tmp.fileDate.After(startTime) {
fmt.Printf("# bad date %s for %s\n", tmp.fileDate.String(), fullpath)
// BUG(mdr): TODO? save bad dates in a list for appending
g_badDateCt++
}
if g_noSHAFlag {
// do nothing
} else {
tmp.dig256, err = mdr.FileSHA256(fullpath)
if err != nil {
log.Fatalf("SHA256 failed on %s\n", fullpath)
}
}
tmp.fileLength = stats.Size()
g_tmMutex.Lock()
bytesProcessed += tmp.fileLength
filesProcessed++
g_tmMutex.Unlock()
lo <- tmp
<-accel // free a core
}(fname, throttle)
}
loop.Wait()
var doneRec digestType
doneRec.pathname = ""
lo <- doneRec
outPut.Wait()
//time.Sleep(1 * time.Second) // not necessary
// wrapup
elapsedTime := time.Now().Sub(startTime)
elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# %s Rundate=%s\n", G_version, startTime.String())
fmt.Printf("# Processed %s files with %s bytes in %s for %.2g bytes/sec\n",
mdr.CommaFmtInt64(filesProcessed), mdr.CommaFmtInt64(bytesProcessed), mdr.HumanTime(elapsedTime), float32(bytesProcessed)/float32(elapsedSeconds))
fmt.Printf("# nCPU[%d] BadDates[%d]\n", nCPU, g_badDateCt)
}
