Beispiel #1
0
Datei: pie.go Projekt: daaku/pie
func (r *Run) fileWorker(files chan string) error {
	compiledInstructions, err := r.compileInstruction()
	if err != nil {
		return fmt.Errorf("failed to compile instructions: %s", err)
	}

	var fileFilterRe, fileIgnoreRe *regexp.Regexp
	if r.FileFilter != "" {
		fileFilterRe, err = regexp.Compile(r.FileFilter)
		if err != nil {
			return fmt.Errorf("failed to compile file filter regexp: %s", err)
		}
	}
	if r.FileIgnore != "" {
		fileIgnoreRe, err = regexp.Compile(r.FileIgnore)
		if err != nil {
			return fmt.Errorf("failed to compile file ignore regexp: %s", err)
		}
	}

	for f := range files {
		if fileFilterRe != nil && fileFilterRe.MatchString(f, true, true) < 0 {
			continue
		}
		if fileIgnoreRe != nil && fileIgnoreRe.MatchString(f, true, true) > -1 {
			continue
		}
		if err = r.processFile(f, compiledInstructions); err != nil {
			return fmt.Errorf("failed to process file %s: %s", f, err)
		}
	}
	return nil
}
Beispiel #2
0
func query(patterns []string, fFlag string, iFlag bool, out io.Writer, limit int, timelimit time.Duration) (lines int) {

	var fre *regexp.Regexp
	var err error
	if fFlag != "" {
		fre, err = regexp.Compile(fFlag)
		if err != nil {
			return
		}
	}
	outchan := make(chan string) // all output ist collected here.
	matchchan := make(chan bool) // grep's tell whether thy found sth.
	stopchan := make(chan bool)  // grep's listen here to be stopped
	timeout := make(chan bool)   // delivers a timeout for this function
	go func() {
		time.Sleep(timelimit)
		timeout <- true
	}()

	g := make([]*Grep, 0, len(patterns))
	for _, v := range patterns {
		pat := "(?m)" + v
		if iFlag {
			pat = "(?i)" + pat
		}
		re, err := regexp.Compile(pat)
		if err != nil {
			continue
		}
		log.Printf("Grepping for %s\n", re)
		g = append(g, &Grep{
			Regexp:  re,
			Stdout:  outchan,
			Matched: matchchan,
			Stop:    stopchan,
			Stderr:  os.Stderr,
		})
	}
	if len(g) == 0 {
		return
	}

	q := index.RegexpQuery(g[0].Regexp.Syntax)
	for _, v := range g[1:] {
		q = q.And(index.RegexpQuery(v.Regexp.Syntax))
	}
	if *verboseFlag {
		log.Printf("query: %s\n", q)
	}

	ix := index.Open(index.File())
	ix.Verbose = *verboseFlag
	var post []uint32
	if *bruteFlag {
		post = ix.PostingQuery(&index.Query{Op: index.QAll})
	} else {
		post = ix.PostingQuery(q)
	}
	if *verboseFlag {
		log.Printf("post query identified %d possible files\n", len(post))
	}

	if fre != nil {
		fnames := make([]uint32, 0, len(post))

		for _, fileid := range post {
			name := ix.Name(fileid)
			if fre.MatchString(name, true, true) < 0 {
				continue
			}
			fnames = append(fnames, fileid)
		}

		if *verboseFlag {
			log.Printf("filename regexp matched %d files\n", len(fnames))
		}
		post = fnames
	}

	output := make([]string, 0, 10)
	lines = 0
	timeoutFlag := false
	for _, fileid := range post {
		output = output[:0]
		name := ix.Name(fileid)

		for _, grep := range g {
			go grep.File(name)
		}
		runningcount := len(g)

		// Counting is critical here. Read once from matchchan and write once
		// to stopchan for ech grep - or everything will deadlock.
		matched := true
		for runningcount > 0 {
			select {
			case s := <-outchan:
				output = append(output, s)
			case match := <-matchchan:
				runningcount--
				if !match {
					matched = false
					runningcount = 0
				}
			case <-timeout:
				runningcount = 0
				timeoutFlag = true
			}

		}
		//log.Println("Stopping all greps")
		stopcount := len(g)
		for stopcount > 0 {
			select {
			case stopchan <- true:
				stopcount--
			case <-outchan:
			case <-matchchan:
			}
		}
		//log.Println("All greps stopped")
		if matched {
			if *verboseFlag {
				log.Printf("writing %d lines of output from %s\n", len(output), name)
			}
			for _, s := range output {
				fmt.Fprint(out, s)
				lines++
				limit--
				if limit == 0 {
					fmt.Fprint(out, "... :0: Even More.\n")
					return
				}
			}
		}
		if timeoutFlag {
			fmt.Fprintf(out, "... :0: Timeout: %dms.\n", timelimit/time.Millisecond)
			break
		}

	}
	return
}
Beispiel #3
0
func Main() {
	g := regexp.Grep{
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	g.AddFlags()

	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if len(args) != 1 {
		usage()
	}

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	pat := "(?m)" + args[0]
	if *iFlag {
		pat = "(?i)" + pat
	}
	re, err := regexp.Compile(pat)
	if err != nil {
		log.Fatal(err)
	}
	g.Regexp = re
	var fre *regexp.Regexp
	if *fFlag != "" {
		fre, err = regexp.Compile(*fFlag)
		if err != nil {
			log.Fatal(err)
		}
	}
	q := index.RegexpQuery(re.Syntax)
	if *verboseFlag {
		log.Printf("query: %s\n", q)
	}

	ix := index.Open(index.File())
	ix.Verbose = *verboseFlag
	var post []uint32
	if *bruteFlag {
		post = ix.PostingQuery(&index.Query{Op: index.QAll})
	} else {
		post = ix.PostingQuery(q)
	}
	if *verboseFlag {
		log.Printf("post query identified %d possible files\n", len(post))
	}

	if fre != nil {
		fnames := make([]uint32, 0, len(post))

		for _, fileid := range post {
			name := ix.Name(fileid)
			if fre.MatchString(name, true, true) < 0 {
				continue
			}
			fnames = append(fnames, fileid)
		}

		if *verboseFlag {
			log.Printf("filename regexp matched %d files\n", len(fnames))
		}
		post = fnames
	}

	for _, fileid := range post {
		name := ix.Name(fileid)
		g.File(name)
	}

	matches = g.Match
}