コード例 #1
0
ファイル: pie_test.go プロジェクト: daaku/pie
func TestRun(t *testing.T) {
	t.Parallel()
	for _, test := range cases {
		tmp, err := test.MakeTempCopy()
		if err != nil {
			t.Fatalf("faled to make temp copy for %s: %s", test.Name, err)
		}

		ixFile := filepath.Join(tmp, ".csearchindex")
		ixw := index.Create(ixFile)
		ixw.AddPaths([]string{tmp})
		filepath.Walk(tmp, func(path string, info os.FileInfo, err error) error {
			if _, elem := filepath.Split(path); elem != "" {
				// Skip various temporary or "hidden" files or directories.
				if elem[0] == '.' || elem[0] == '#' || elem[0] == '~' || elem[len(elem)-1] == '~' {
					if info.IsDir() {
						return filepath.SkipDir
					}
					return nil
				}
			}
			if err != nil {
				t.Fatal(err)
			}
			if info != nil && info.Mode()&os.ModeType == 0 {
				ixw.AddFile(path)
			}
			return nil
		})
		ixw.Flush()

		run := &pie.Run{
			Index:       index.Open(ixFile),
			Instruction: test.Instruction,
			FileIgnore:  test.FileIgnore,
			FileFilter:  test.FileFilter,
			NumWorkers:  test.NumWorkers,
		}
		err = run.Run()
		if err != nil {
			t.Fatalf("run for %s failed: %s", test.Name, err)
		}
		same, err := test.Compare(tmp)
		if err != nil {
			t.Fatalf("compare for %s failed: %s", test.Name, err)
		}
		if !same {
			t.Fatalf("did not get expected result for %s", test.Name)
		}
		if *removeTemp {
			os.RemoveAll(tmp)
		}
	}
}
コード例 #2
0
ファイル: cwebsearch.go プロジェクト: ceemos/cwebsearch
func query(patterns []string, fFlag string, iFlag bool, out io.Writer, limit int, timelimit time.Duration) (lines int) {

	var fre *regexp.Regexp
	var err error
	if fFlag != "" {
		fre, err = regexp.Compile(fFlag)
		if err != nil {
			return
		}
	}
	outchan := make(chan string) // all output ist collected here.
	matchchan := make(chan bool) // grep's tell whether thy found sth.
	stopchan := make(chan bool)  // grep's listen here to be stopped
	timeout := make(chan bool)   // delivers a timeout for this function
	go func() {
		time.Sleep(timelimit)
		timeout <- true
	}()

	g := make([]*Grep, 0, len(patterns))
	for _, v := range patterns {
		pat := "(?m)" + v
		if iFlag {
			pat = "(?i)" + pat
		}
		re, err := regexp.Compile(pat)
		if err != nil {
			continue
		}
		log.Printf("Grepping for %s\n", re)
		g = append(g, &Grep{
			Regexp:  re,
			Stdout:  outchan,
			Matched: matchchan,
			Stop:    stopchan,
			Stderr:  os.Stderr,
		})
	}
	if len(g) == 0 {
		return
	}

	q := index.RegexpQuery(g[0].Regexp.Syntax)
	for _, v := range g[1:] {
		q = q.And(index.RegexpQuery(v.Regexp.Syntax))
	}
	if *verboseFlag {
		log.Printf("query: %s\n", q)
	}

	ix := index.Open(index.File())
	ix.Verbose = *verboseFlag
	var post []uint32
	if *bruteFlag {
		post = ix.PostingQuery(&index.Query{Op: index.QAll})
	} else {
		post = ix.PostingQuery(q)
	}
	if *verboseFlag {
		log.Printf("post query identified %d possible files\n", len(post))
	}

	if fre != nil {
		fnames := make([]uint32, 0, len(post))

		for _, fileid := range post {
			name := ix.Name(fileid)
			if fre.MatchString(name, true, true) < 0 {
				continue
			}
			fnames = append(fnames, fileid)
		}

		if *verboseFlag {
			log.Printf("filename regexp matched %d files\n", len(fnames))
		}
		post = fnames
	}

	output := make([]string, 0, 10)
	lines = 0
	timeoutFlag := false
	for _, fileid := range post {
		output = output[:0]
		name := ix.Name(fileid)

		for _, grep := range g {
			go grep.File(name)
		}
		runningcount := len(g)

		// Counting is critical here. Read once from matchchan and write once
		// to stopchan for ech grep - or everything will deadlock.
		matched := true
		for runningcount > 0 {
			select {
			case s := <-outchan:
				output = append(output, s)
			case match := <-matchchan:
				runningcount--
				if !match {
					matched = false
					runningcount = 0
				}
			case <-timeout:
				runningcount = 0
				timeoutFlag = true
			}

		}
		//log.Println("Stopping all greps")
		stopcount := len(g)
		for stopcount > 0 {
			select {
			case stopchan <- true:
				stopcount--
			case <-outchan:
			case <-matchchan:
			}
		}
		//log.Println("All greps stopped")
		if matched {
			if *verboseFlag {
				log.Printf("writing %d lines of output from %s\n", len(output), name)
			}
			for _, s := range output {
				fmt.Fprint(out, s)
				lines++
				limit--
				if limit == 0 {
					fmt.Fprint(out, "... :0: Even More.\n")
					return
				}
			}
		}
		if timeoutFlag {
			fmt.Fprintf(out, "... :0: Timeout: %dms.\n", timelimit/time.Millisecond)
			break
		}

	}
	return
}
コード例 #3
0
ファイル: csearch.go プロジェクト: santazhang/sandbox
func Main() {
	g := regexp.Grep{
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	g.AddFlags()

	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if len(args) != 1 {
		usage()
	}

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	pat := "(?m)" + args[0]
	if *iFlag {
		pat = "(?i)" + pat
	}
	re, err := regexp.Compile(pat)
	if err != nil {
		log.Fatal(err)
	}
	g.Regexp = re
	var fre *regexp.Regexp
	if *fFlag != "" {
		fre, err = regexp.Compile(*fFlag)
		if err != nil {
			log.Fatal(err)
		}
	}
	q := index.RegexpQuery(re.Syntax)
	if *verboseFlag {
		log.Printf("query: %s\n", q)
	}

	ix := index.Open(index.File())
	ix.Verbose = *verboseFlag
	var post []uint32
	if *bruteFlag {
		post = ix.PostingQuery(&index.Query{Op: index.QAll})
	} else {
		post = ix.PostingQuery(q)
	}
	if *verboseFlag {
		log.Printf("post query identified %d possible files\n", len(post))
	}

	if fre != nil {
		fnames := make([]uint32, 0, len(post))

		for _, fileid := range post {
			name := ix.Name(fileid)
			if fre.MatchString(name, true, true) < 0 {
				continue
			}
			fnames = append(fnames, fileid)
		}

		if *verboseFlag {
			log.Printf("filename regexp matched %d files\n", len(fnames))
		}
		post = fnames
	}

	for _, fileid := range post {
		name := ix.Name(fileid)
		g.File(name)
	}

	matches = g.Match
}
コード例 #4
0
ファイル: pie.go プロジェクト: daaku/pie
func Main() error {
	var (
		goMaxProcs = flag.Int("gomaxprocs", runtime.NumCPU(), "gomaxprocs")
		ignoreRe   = flag.String("ignore", "", "file full path ignore regexp")
		filterRe   = flag.String("filter", "", "file full path filter regexp")
		cpuProfile = flag.String("cpuprofile", "", "write cpu profile to this file")
		inFile     = flag.String("input", "", "read instruction pairs from this file")
		indexFile  = flag.String("index", defaultIndexFile(), "default index file location")
		roots      = flag.String("root", defaultRoot(), "comma separated target paths")
	)

	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	runtime.GOMAXPROCS(*goMaxProcs)

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// parse replacement instructions
	r := &pie.Run{
		FileFilter: *filterRe,
		FileIgnore: *ignoreRe,
	}
	var err error
	if *inFile != "" {
		f, err := os.Open(*inFile)
		if err != nil {
			return err
		}
		r.Instruction, err = pie.InstructionFromReader(f)
		if err != nil {
			return err
		}
	} else {
		r.Instruction, err = pie.InstructionFromArgs(args)
		if err != nil {
			return err
		}
	}
	if len(r.Instruction) == 0 {
		flag.Usage()
	}

	// make the index
	iw := index.Create(*indexFile)
	for _, arg := range strings.Split(*roots, ",") {
		filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
			if _, elem := filepath.Split(path); elem != "" {
				// Skip various temporary or "hidden" files or directories.
				if elem[0] == '.' || elem[0] == '#' || elem[0] == '~' || elem[len(elem)-1] == '~' {
					if info.IsDir() {
						return filepath.SkipDir
					}
					return nil
				}
			}
			if err != nil {
				log.Printf("%s: %s", path, err)
				return nil
			}
			if info != nil && info.Mode()&os.ModeType == 0 {
				iw.AddFile(path)
			}
			return nil
		})
	}
	iw.Flush()
	defer os.Remove(*indexFile)
	r.Index = index.Open(*indexFile)

	return r.Run()
}