// mod_ListMatches is single threaded
// It does list and maintains count as well
func mod_ListMatches(fname string) {
startTime := time.Now()
fmt.Printf("Doing ListMatches()\n")
allTargets := getTargets(fname)
fmt.Printf("searching %d different targets\n", len(allTargets))
for _, target := range allTargets {
matchCt := int64(0)
matchBytes := int64(0)
re, err := regexp.Compile(target)
if err != nil {
log.Fatalf("can't compile regexp - crashed here\n")
}
for _, r := range g_nameList {
if re.MatchString(r.Name) {
fmt.Printf("%16s bytes : %s\n", mdr.CommaFmtInt64(r.Size), r.Name)
matchCt++
matchBytes += r.Size
}
}
showMatchCounts(matchBytes, matchCt, target)
g_totalMatches += matchCt
g_totalBytes += matchBytes
}
elapsedTime := time.Now().Sub(startTime)
//elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# modList run time is %s\n", mdr.HumanTime(elapsedTime))
}
func main() {
startTime := time.Now()
flag.Parse()
for i := 0; i < flag.NArg(); i++ {
g_argList = append(g_argList, flag.Arg(i))
}
flagSetup()
tok = tokenbucket.New(time.Millisecond*100, 20)
Verbose.Printf("arglist %v\n", g_argList)
if len(g_argList) < 1 {
usage()
}
go handleQuit()
for _, arg := range g_argList {
if quitTime {
fmt.Printf("Returning from quit signal\n")
break
}
fmt.Printf("loading %s\n", arg)
var err error
g_nameList, err = LoadSHA256Names(arg)
if err != nil {
log.Fatalf("Crashed loading %s\n", arg)
}
/*
if len(countFile) > 0 {
modGort_CountMatches(countFile)
}
if len(countFile) > 0 {
modSplit_CountMatches(countFile)
}
*/
if len(countFile) > 0 {
modPool_CountMatches(countFile)
}
if len(killFile) > 0 {
fmt.Printf("calling mod_KillMatches\n")
mod_KillMatches(killFile)
}
if len(listFile) > 0 {
mod_ListMatches(listFile)
}
}
fmt.Printf("# Final summary for all args:\n")
fmt.Printf("# Total bytes matched = %s in %d files\n",
mdr.CommaFmtInt64(g_totalBytes), g_totalMatches)
fmt.Printf("# Deleted %s total bytes in %s files\n",
mdr.CommaFmtInt64(g_totalKillBytes), mdr.CommaFmtInt64(g_totalKillCt))
elapsedTime := time.Now().Sub(startTime)
//elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# Elapsed run time is %s\n", mdr.HumanTime(elapsedTime))
}
// modPool_CountMatches uses multi-cores when available
// It does not list by default but will if Verbose
// Uses mdr.JobSplit
func modSplit_CountMatches(fname string) {
startTime := time.Now()
fmt.Printf("Doing modSplit_CountMatches()\n")
var wg sync.WaitGroup
allTargets := getTargets(fname)
nCPU := runtime.NumCPU()
fmt.Printf("searching %d different targets with %d CPUs\n", len(allTargets), nCPU)
runtime.GOMAXPROCS(nCPU)
splits := mdr.JobSplit(len(g_nameList), nCPU)
for _, target := range allTargets {
if quitTime {
fmt.Printf("Returning from quit signal\n")
return
}
Verbose.Printf("goroutine starting with regexp %s\n", target)
re, err := regexp.Compile(target)
if err != nil {
fmt.Printf("!Err ---> can't compile regexp : %s \n", target)
continue
}
var matchCt = int64(0)
var matchBytes = int64(0)
for j := 0; j < nCPU; j++ {
wg.Add(1)
go func(lo, hi int) {
defer wg.Done()
var fileCt = int64(0)
var byteCt = int64(0)
for i := lo; i <= hi; i++ {
r := g_nameList[i]
if re.MatchString(r.Name) {
fileCt++
byteCt += r.Size
}
}
g_tmMutex.Lock()
matchCt += fileCt
matchBytes += byteCt
g_tmMutex.Unlock()
}(splits[j].X, splits[j].Y)
}
wg.Wait()
showMatchCounts(matchBytes, matchCt, target)
g_totalMatches += matchCt
g_totalBytes += matchBytes
}
elapsedTime := time.Now().Sub(startTime)
//elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# modSplit run time is %s\n", mdr.HumanTime(elapsedTime))
}
// mod_KillMatches will delete files that match
// single thread so no mutex protection of global vars
func mod_KillMatches(fname string) {
startTime := time.Now()
fmt.Printf("Doing KillMatches()\n")
allTargets := getTargets(fname)
fmt.Printf("searching %d different targets\n", len(allTargets))
for _, target := range allTargets {
re, err := regexp.Compile(target)
if err != nil {
fmt.Printf("!Err ---> can't compile regexp : %s \n", target)
continue
}
var matchCt = int64(0)
var killCt = int64(0)
var matchBytes = int64(0)
var killBytes = int64(0)
for _, r := range g_nameList {
if re.MatchString(r.Name) {
matchCt++
matchBytes += r.Size
Verbose.Printf("\tkilling %s\n", r.Name)
if niceFlag > 0 {
time.Sleep(tok.Take(int64(niceFlag))) // rate limiter if used
}
if doReALLy {
err := os.Remove(r.Name)
if err != nil {
if os.IsNotExist(err) == false {
log.Fatalf("Can't remove %s\n", r.Name)
}
}
killCt++
killBytes += r.Size
g_totalKillCt++
g_totalKillBytes += r.Size
} else {
fmt.Printf("need flag -doReALLy to actually remove %s\n", r.Name)
}
matchCt++
}
}
showMatchCounts(matchBytes, matchCt, target)
fmt.Printf("Kill %s had %d matches, %d files actually deleted with %s bytes\n",
target, matchCt, killCt, mdr.CommaFmtInt64(killBytes))
}
elapsedTime := time.Now().Sub(startTime)
//elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# modKill run time is %s\n", mdr.HumanTime(elapsedTime))
}
// modPool_CountMatches uses multi-cores (with throttle to nCPU) when > 2 cores available
// It does not list by default but will if Verbose
// Uses worker pool
func modPool_CountMatches(fname string) {
startTime := time.Now()
fmt.Printf("Doing modPool_CountMatches()\n")
var wg sync.WaitGroup
allTargets := getTargets(fname)
fmt.Printf("searching %d different targets\n", len(allTargets))
nCPU := runtime.NumCPU()
runtime.GOMAXPROCS(nCPU)
workers := make(chan int, nCPU)
fmt.Printf("using %d cores\n", nCPU)
for _, t := range allTargets {
if quitTime {
fmt.Printf("Returning from quit signal\n")
return
}
workers <- 1 // blocks if no workers available
Verbose.Printf("goroutine starting with regexp %s\n", t)
wg.Add(1)
go func(target string) {
defer wg.Done()
re, err := regexp.Compile(target)
if err != nil {
fmt.Printf("!Err ---> can't compile regexp : %s \n", target)
<-workers // free a worker core
return
}
var matchCt = int64(0)
var matchBytes = int64(0)
for _, r := range g_nameList {
if re.MatchString(r.Name) {
matchCt++
matchBytes += r.Size
}
}
showMatchCounts(matchBytes, matchCt, target)
g_tmMutex.Lock()
g_totalMatches += matchCt
g_totalBytes += matchBytes
g_tmMutex.Unlock()
<-workers // free a worker core
}(t)
}
wg.Wait()
elapsedTime := time.Now().Sub(startTime)
//elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# modPool run time is %s\n", mdr.HumanTime(elapsedTime))
}
// modGort_CountMatches similar to pool but doesn't try to limit the
// number of active goroutines.
//
func modGort_CountMatches(fname string) {
startTime := time.Now()
fmt.Printf("Doing modGort_CountMatches()\n")
var wg sync.WaitGroup
allTargets := getTargets(fname)
nCPU := runtime.NumCPU()
fmt.Printf("search ing %d different targets with %d CPUs\n", len(allTargets), nCPU)
// test passes, throttles down when nCPU >= 2
runtime.GOMAXPROCS(nCPU)
for _, t := range allTargets {
if quitTime {
fmt.Printf("Returning from quit signal\n")
return
}
Verbose.Printf("goroutine starting with regexp %s\n", t)
wg.Add(1)
go func(target string) {
defer wg.Done()
re, err := regexp.Compile(target)
if err != nil {
log.Fatalf("can't compile regexp - crashed here\n")
}
var matchCt = int64(0)
var matchBytes = int64(0)
for _, r := range g_nameList {
if re.MatchString(r.Name) {
matchCt++
matchBytes += r.Size
}
}
showMatchCounts(matchBytes, matchCt, target)
g_tmMutex.Lock()
g_totalMatches += matchCt
g_totalBytes += matchBytes
g_tmMutex.Unlock()
}(t)
}
wg.Wait()
elapsedTime := time.Now().Sub(startTime)
//elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# modGort run time is %s\n", mdr.HumanTime(elapsedTime))
}
func main() {
flag.Parse()
if flag.NArg() == 0 {
fmt.Printf("Nothing to do - No arguments in command line\n")
usage()
return
}
if g_verboseFlag {
Verbose = true
}
flagSetup()
// BUG(mdr): TODO? flag for relative path or flag for abs path?
pathName, err := filepath.Abs(flag.Arg(0))
if err != nil {
log.Fatalf("cant get absolute path for %s\n", flag.Arg(0))
}
lo := make(chan digestType, 5)
//fmt.Printf("before lo %d gort running\n",runtime.NumGoroutine())
go lineOut(lo)
Verbose.Printf("Checking paths in %s\n", pathName)
dirInfo, err := os.Stat(pathName)
if err != nil {
log.Fatalf("cant stat the directory %s\n", pathName)
}
dMode := dirInfo.Mode()
if dMode.IsDir() == false {
log.Fatalf("Path %s must be a directory (but isn't)\n", pathName)
} else {
Verbose.Printf("%s is a directory, walking starts now\n", pathName)
}
filepath.Walk(pathName, CheckPath) // builds g_argList
var filesProcessed int64 = 0
var bytesProcessed int64 = 0
fmt.Fprintf(os.Stderr, "# nCPU = %d\n", nCPU)
throttle := make(chan int, nCPU)
startTime := time.Now()
for _, fname := range g_argList {
//fmt.Printf("Goroutines active = %d\n", runtime.NumGoroutine())
throttle <- 1
loop.Add(1)
go func(fullpath string, accel chan int) {
defer loop.Done()
var tmp digestType
tmp.pathname = fullpath
stats, err := os.Stat(fullpath)
if err != nil {
log.Fatalf("Can't get fileinfo for %s\n", fullpath)
}
// check time for sanity (date < now()
tmp.fileDate = stats.ModTime()
if tmp.fileDate.After(startTime) {
fmt.Printf("# bad date %s for %s\n", tmp.fileDate.String(), fullpath)
// BUG(mdr): TODO? save bad dates in a list for appending
g_badDateCt++
}
if g_noSHAFlag {
// do nothing
} else {
tmp.dig256, err = mdr.FileSHA256(fullpath)
if err != nil {
log.Fatalf("SHA256 failed on %s\n", fullpath)
}
}
tmp.fileLength = stats.Size()
g_tmMutex.Lock()
bytesProcessed += tmp.fileLength
filesProcessed++
g_tmMutex.Unlock()
lo <- tmp
<-accel // free a core
}(fname, throttle)
}
loop.Wait()
var doneRec digestType
doneRec.pathname = ""
lo <- doneRec
outPut.Wait()
//time.Sleep(1 * time.Second) // not necessary
// wrapup
elapsedTime := time.Now().Sub(startTime)
elapsedSeconds := elapsedTime.Seconds()
fmt.Printf("# %s Rundate=%s\n", G_version, startTime.String())
fmt.Printf("# Processed %s files with %s bytes in %s for %.2g bytes/sec\n",
mdr.CommaFmtInt64(filesProcessed), mdr.CommaFmtInt64(bytesProcessed), mdr.HumanTime(elapsedTime), float32(bytesProcessed)/float32(elapsedSeconds))
fmt.Printf("# nCPU[%d] BadDates[%d]\n", nCPU, g_badDateCt)
}
