func TestRun(t *testing.T) {
t.Parallel()
for _, test := range cases {
tmp, err := test.MakeTempCopy()
if err != nil {
t.Fatalf("faled to make temp copy for %s: %s", test.Name, err)
}
ixFile := filepath.Join(tmp, ".csearchindex")
ixw := index.Create(ixFile)
ixw.AddPaths([]string{tmp})
filepath.Walk(tmp, func(path string, info os.FileInfo, err error) error {
if _, elem := filepath.Split(path); elem != "" {
// Skip various temporary or "hidden" files or directories.
if elem[0] == '.' || elem[0] == '#' || elem[0] == '~' || elem[len(elem)-1] == '~' {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
}
if err != nil {
t.Fatal(err)
}
if info != nil && info.Mode()&os.ModeType == 0 {
ixw.AddFile(path)
}
return nil
})
ixw.Flush()
run := &pie.Run{
Index: index.Open(ixFile),
Instruction: test.Instruction,
FileIgnore: test.FileIgnore,
FileFilter: test.FileFilter,
NumWorkers: test.NumWorkers,
}
err = run.Run()
if err != nil {
t.Fatalf("run for %s failed: %s", test.Name, err)
}
same, err := test.Compare(tmp)
if err != nil {
t.Fatalf("compare for %s failed: %s", test.Name, err)
}
if !same {
t.Fatalf("did not get expected result for %s", test.Name)
}
if *removeTemp {
os.RemoveAll(tmp)
}
}
}
func query(patterns []string, fFlag string, iFlag bool, out io.Writer, limit int, timelimit time.Duration) (lines int) {
var fre *regexp.Regexp
var err error
if fFlag != "" {
fre, err = regexp.Compile(fFlag)
if err != nil {
return
}
}
outchan := make(chan string) // all output ist collected here.
matchchan := make(chan bool) // grep's tell whether thy found sth.
stopchan := make(chan bool) // grep's listen here to be stopped
timeout := make(chan bool) // delivers a timeout for this function
go func() {
time.Sleep(timelimit)
timeout <- true
}()
g := make([]*Grep, 0, len(patterns))
for _, v := range patterns {
pat := "(?m)" + v
if iFlag {
pat = "(?i)" + pat
}
re, err := regexp.Compile(pat)
if err != nil {
continue
}
log.Printf("Grepping for %s\n", re)
g = append(g, &Grep{
Regexp: re,
Stdout: outchan,
Matched: matchchan,
Stop: stopchan,
Stderr: os.Stderr,
})
}
if len(g) == 0 {
return
}
q := index.RegexpQuery(g[0].Regexp.Syntax)
for _, v := range g[1:] {
q = q.And(index.RegexpQuery(v.Regexp.Syntax))
}
if *verboseFlag {
log.Printf("query: %s\n", q)
}
ix := index.Open(index.File())
ix.Verbose = *verboseFlag
var post []uint32
if *bruteFlag {
post = ix.PostingQuery(&index.Query{Op: index.QAll})
} else {
post = ix.PostingQuery(q)
}
if *verboseFlag {
log.Printf("post query identified %d possible files\n", len(post))
}
if fre != nil {
fnames := make([]uint32, 0, len(post))
for _, fileid := range post {
name := ix.Name(fileid)
if fre.MatchString(name, true, true) < 0 {
continue
}
fnames = append(fnames, fileid)
}
if *verboseFlag {
log.Printf("filename regexp matched %d files\n", len(fnames))
}
post = fnames
}
output := make([]string, 0, 10)
lines = 0
timeoutFlag := false
for _, fileid := range post {
output = output[:0]
name := ix.Name(fileid)
for _, grep := range g {
go grep.File(name)
}
runningcount := len(g)
// Counting is critical here. Read once from matchchan and write once
// to stopchan for ech grep - or everything will deadlock.
matched := true
for runningcount > 0 {
select {
case s := <-outchan:
output = append(output, s)
case match := <-matchchan:
runningcount--
if !match {
matched = false
runningcount = 0
}
case <-timeout:
runningcount = 0
timeoutFlag = true
}
}
//log.Println("Stopping all greps")
stopcount := len(g)
for stopcount > 0 {
select {
case stopchan <- true:
stopcount--
case <-outchan:
case <-matchchan:
}
}
//log.Println("All greps stopped")
if matched {
if *verboseFlag {
log.Printf("writing %d lines of output from %s\n", len(output), name)
}
for _, s := range output {
fmt.Fprint(out, s)
lines++
limit--
if limit == 0 {
fmt.Fprint(out, "... :0: Even More.\n")
return
}
}
}
if timeoutFlag {
fmt.Fprintf(out, "... :0: Timeout: %dms.\n", timelimit/time.Millisecond)
break
}
}
return
}
func Main() {
g := regexp.Grep{
Stdout: os.Stdout,
Stderr: os.Stderr,
}
g.AddFlags()
flag.Usage = usage
flag.Parse()
args := flag.Args()
if len(args) != 1 {
usage()
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
pat := "(?m)" + args[0]
if *iFlag {
pat = "(?i)" + pat
}
re, err := regexp.Compile(pat)
if err != nil {
log.Fatal(err)
}
g.Regexp = re
var fre *regexp.Regexp
if *fFlag != "" {
fre, err = regexp.Compile(*fFlag)
if err != nil {
log.Fatal(err)
}
}
q := index.RegexpQuery(re.Syntax)
if *verboseFlag {
log.Printf("query: %s\n", q)
}
ix := index.Open(index.File())
ix.Verbose = *verboseFlag
var post []uint32
if *bruteFlag {
post = ix.PostingQuery(&index.Query{Op: index.QAll})
} else {
post = ix.PostingQuery(q)
}
if *verboseFlag {
log.Printf("post query identified %d possible files\n", len(post))
}
if fre != nil {
fnames := make([]uint32, 0, len(post))
for _, fileid := range post {
name := ix.Name(fileid)
if fre.MatchString(name, true, true) < 0 {
continue
}
fnames = append(fnames, fileid)
}
if *verboseFlag {
log.Printf("filename regexp matched %d files\n", len(fnames))
}
post = fnames
}
for _, fileid := range post {
name := ix.Name(fileid)
g.File(name)
}
matches = g.Match
}
func Main() error {
var (
goMaxProcs = flag.Int("gomaxprocs", runtime.NumCPU(), "gomaxprocs")
ignoreRe = flag.String("ignore", "", "file full path ignore regexp")
filterRe = flag.String("filter", "", "file full path filter regexp")
cpuProfile = flag.String("cpuprofile", "", "write cpu profile to this file")
inFile = flag.String("input", "", "read instruction pairs from this file")
indexFile = flag.String("index", defaultIndexFile(), "default index file location")
roots = flag.String("root", defaultRoot(), "comma separated target paths")
)
flag.Usage = usage
flag.Parse()
args := flag.Args()
runtime.GOMAXPROCS(*goMaxProcs)
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
// parse replacement instructions
r := &pie.Run{
FileFilter: *filterRe,
FileIgnore: *ignoreRe,
}
var err error
if *inFile != "" {
f, err := os.Open(*inFile)
if err != nil {
return err
}
r.Instruction, err = pie.InstructionFromReader(f)
if err != nil {
return err
}
} else {
r.Instruction, err = pie.InstructionFromArgs(args)
if err != nil {
return err
}
}
if len(r.Instruction) == 0 {
flag.Usage()
}
// make the index
iw := index.Create(*indexFile)
for _, arg := range strings.Split(*roots, ",") {
filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
if _, elem := filepath.Split(path); elem != "" {
// Skip various temporary or "hidden" files or directories.
if elem[0] == '.' || elem[0] == '#' || elem[0] == '~' || elem[len(elem)-1] == '~' {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
}
if err != nil {
log.Printf("%s: %s", path, err)
return nil
}
if info != nil && info.Mode()&os.ModeType == 0 {
iw.AddFile(path)
}
return nil
})
}
iw.Flush()
defer os.Remove(*indexFile)
r.Index = index.Open(*indexFile)
return r.Run()
}