func (r *Run) fileWorker(files chan string) error {
compiledInstructions, err := r.compileInstruction()
if err != nil {
return fmt.Errorf("failed to compile instructions: %s", err)
}
var fileFilterRe, fileIgnoreRe *regexp.Regexp
if r.FileFilter != "" {
fileFilterRe, err = regexp.Compile(r.FileFilter)
if err != nil {
return fmt.Errorf("failed to compile file filter regexp: %s", err)
}
}
if r.FileIgnore != "" {
fileIgnoreRe, err = regexp.Compile(r.FileIgnore)
if err != nil {
return fmt.Errorf("failed to compile file ignore regexp: %s", err)
}
}
for f := range files {
if fileFilterRe != nil && fileFilterRe.MatchString(f, true, true) < 0 {
continue
}
if fileIgnoreRe != nil && fileIgnoreRe.MatchString(f, true, true) > -1 {
continue
}
if err = r.processFile(f, compiledInstructions); err != nil {
return fmt.Errorf("failed to process file %s: %s", f, err)
}
}
return nil
}
func query(patterns []string, fFlag string, iFlag bool, out io.Writer, limit int, timelimit time.Duration) (lines int) {
var fre *regexp.Regexp
var err error
if fFlag != "" {
fre, err = regexp.Compile(fFlag)
if err != nil {
return
}
}
outchan := make(chan string) // all output ist collected here.
matchchan := make(chan bool) // grep's tell whether thy found sth.
stopchan := make(chan bool) // grep's listen here to be stopped
timeout := make(chan bool) // delivers a timeout for this function
go func() {
time.Sleep(timelimit)
timeout <- true
}()
g := make([]*Grep, 0, len(patterns))
for _, v := range patterns {
pat := "(?m)" + v
if iFlag {
pat = "(?i)" + pat
}
re, err := regexp.Compile(pat)
if err != nil {
continue
}
log.Printf("Grepping for %s\n", re)
g = append(g, &Grep{
Regexp: re,
Stdout: outchan,
Matched: matchchan,
Stop: stopchan,
Stderr: os.Stderr,
})
}
if len(g) == 0 {
return
}
q := index.RegexpQuery(g[0].Regexp.Syntax)
for _, v := range g[1:] {
q = q.And(index.RegexpQuery(v.Regexp.Syntax))
}
if *verboseFlag {
log.Printf("query: %s\n", q)
}
ix := index.Open(index.File())
ix.Verbose = *verboseFlag
var post []uint32
if *bruteFlag {
post = ix.PostingQuery(&index.Query{Op: index.QAll})
} else {
post = ix.PostingQuery(q)
}
if *verboseFlag {
log.Printf("post query identified %d possible files\n", len(post))
}
if fre != nil {
fnames := make([]uint32, 0, len(post))
for _, fileid := range post {
name := ix.Name(fileid)
if fre.MatchString(name, true, true) < 0 {
continue
}
fnames = append(fnames, fileid)
}
if *verboseFlag {
log.Printf("filename regexp matched %d files\n", len(fnames))
}
post = fnames
}
output := make([]string, 0, 10)
lines = 0
timeoutFlag := false
for _, fileid := range post {
output = output[:0]
name := ix.Name(fileid)
for _, grep := range g {
go grep.File(name)
}
runningcount := len(g)
// Counting is critical here. Read once from matchchan and write once
// to stopchan for ech grep - or everything will deadlock.
matched := true
for runningcount > 0 {
select {
case s := <-outchan:
output = append(output, s)
case match := <-matchchan:
runningcount--
if !match {
matched = false
runningcount = 0
}
case <-timeout:
runningcount = 0
timeoutFlag = true
}
}
//log.Println("Stopping all greps")
stopcount := len(g)
for stopcount > 0 {
select {
case stopchan <- true:
stopcount--
case <-outchan:
case <-matchchan:
}
}
//log.Println("All greps stopped")
if matched {
if *verboseFlag {
log.Printf("writing %d lines of output from %s\n", len(output), name)
}
for _, s := range output {
fmt.Fprint(out, s)
lines++
limit--
if limit == 0 {
fmt.Fprint(out, "... :0: Even More.\n")
return
}
}
}
if timeoutFlag {
fmt.Fprintf(out, "... :0: Timeout: %dms.\n", timelimit/time.Millisecond)
break
}
}
return
}
func Main() {
g := regexp.Grep{
Stdout: os.Stdout,
Stderr: os.Stderr,
}
g.AddFlags()
flag.Usage = usage
flag.Parse()
args := flag.Args()
if len(args) != 1 {
usage()
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
pat := "(?m)" + args[0]
if *iFlag {
pat = "(?i)" + pat
}
re, err := regexp.Compile(pat)
if err != nil {
log.Fatal(err)
}
g.Regexp = re
var fre *regexp.Regexp
if *fFlag != "" {
fre, err = regexp.Compile(*fFlag)
if err != nil {
log.Fatal(err)
}
}
q := index.RegexpQuery(re.Syntax)
if *verboseFlag {
log.Printf("query: %s\n", q)
}
ix := index.Open(index.File())
ix.Verbose = *verboseFlag
var post []uint32
if *bruteFlag {
post = ix.PostingQuery(&index.Query{Op: index.QAll})
} else {
post = ix.PostingQuery(q)
}
if *verboseFlag {
log.Printf("post query identified %d possible files\n", len(post))
}
if fre != nil {
fnames := make([]uint32, 0, len(post))
for _, fileid := range post {
name := ix.Name(fileid)
if fre.MatchString(name, true, true) < 0 {
continue
}
fnames = append(fnames, fileid)
}
if *verboseFlag {
log.Printf("filename regexp matched %d files\n", len(fnames))
}
post = fnames
}
for _, fileid := range post {
name := ix.Name(fileid)
g.File(name)
}
matches = g.Match
}