示例#1
0
文件: cmd.go 项目: timtadh/sfp
func countReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hf:",
		[]string{
			"help",
			"filename=",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	filename := "count"
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-f", "--filename":
			filename = oa.Arg()
		default:
			errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	r, err := reporters.NewCount(conf, filename)
	if err != nil {
		errors.Logf("ERROR", "There was error creating output files\n")
		errors.Logf("ERROR", "%v", err)
		os.Exit(1)
	}
	return r, args
}
示例#2
0
func main() {
	short := "h"
	long := []string{
		"help",
		"allow-dups",
	}
	args, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}

	port := -1
	dups := false
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "--allow-dups":
			dups = true
		}
	}

	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "You must specify a port")
		Usage(ErrorCodes["opts"])
	}
	port = parse_int(args[0])

	fmt.Println("starting")
	server := net.NewServer(func() net.Queue { return queue.NewQueue(dups) })
	server.Start(port)
}
示例#3
0
文件: main.go 项目: timtadh/sfp
func uniproxMode(argv []string, conf *config.Config) (miners.Miner, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hw:",
		[]string{
			"help",
			"walks=",
			"max",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}
	walks := 15
	max := false
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "-w", "--walks":
			walks = cmd.ParseInt(oa.Arg())
		case "--max":
			max = true
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}
	miner, err := uniprox.NewWalker(conf, walks, max)
	if err != nil {
		log.Fatal(err)
	}
	return miner, args
}
示例#4
0
文件: main.go 项目: timtadh/sfp
func premuskMode(argv []string, conf *config.Config) (miners.Miner, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
			"teleport=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}
	teleport := .01
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "--teleport":
			teleport = cmd.ParseFloat(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}
	miner := premusk.NewWalker(conf, teleport)
	return miner, args
}
示例#5
0
文件: cmd.go 项目: timtadh/sfp
func logReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hl:p:",
		[]string{
			"help",
			"level=",
			"prefix=",
			"show-pr",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}
	level := "INFO"
	prefix := ""
	showPr := false
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-l", "--level":
			level = oa.Arg()
		case "-p", "--prefix":
			prefix = oa.Arg()
		case "--show-pr":
			showPr = true
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	return reporters.NewLog(fmtr, showPr, level, prefix), args
}
示例#6
0
func main() {
	short := "hm:"
	long := []string{
		"help", "mode=",
	}

	args, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}

	var mode string
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-m", "--mode":
			switch oa.Arg() {
			case "create", "read":
				mode = oa.Arg()
			default:
				fmt.Fprintf(os.Stderr, "mode %v not supported\n", oa.Arg())
				Usage(ErrorCodes["opts"])
			}
		}

	}

	if len(args) != 2 {
		fmt.Fprintf(os.Stderr, "Must supply exactly two file paths")
		Usage(ErrorCodes["opts"])
	}
	list_path := args[0]
	keys_path := args[1]

	f, err := os.Create("stress-" + mode + ".prof")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	profile_writer = f

	switch mode {
	case "create":
		assert_file_does_not_exist(list_path)
		assert_file_does_not_exist(keys_path)
		create(list_path, keys_path)
	case "read":
		assert_file_exist(list_path)
		assert_file_exist(keys_path)
		read(list_path, keys_path)
	}
}
示例#7
0
文件: cmd.go 项目: timtadh/sfp
func fileReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hp:e:n:",
		[]string{
			"help",
			"patterns=",
			"embeddings=",
			"names=",
			"matrices=",
			"probabilities=",
			"show-pr",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	patterns := "patterns"
	embeddings := "embeddings"
	names := "names.txt"
	matrices := "matrices.json"
	probabilities := "probabilities.prs"
	showPr := false
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-p", "--patterns":
			patterns = oa.Arg()
		case "-e", "--embeddings":
			embeddings = oa.Arg()
		case "-n", "--names":
			names = oa.Arg()
		case "--matrices":
			matrices = oa.Arg()
		case "--probabilites":
			probabilities = oa.Arg()
		case "--show-pr":
			showPr = true
		default:
			errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	fr, err := reporters.NewFile(conf, fmtr, showPr, patterns, embeddings, names, matrices, probabilities)
	if err != nil {
		errors.Logf("ERROR", "There was error creating output files\n")
		errors.Logf("ERROR", "%v\n", err)
		os.Exit(1)
	}
	return fr, args
}
示例#8
0
func main() {

	short := "hp:"
	long := []string{
		"help",
		"pattern=",
	}

	_, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
	if err != nil {
		log.Print(os.Stderr, err)
		Usage(1)
	}

	patterns := make([]string, 0, 10)
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-p", "--pattern":
			patterns = append(patterns, oa.Arg())
		}
	}

	if len(patterns) <= 0 {
		log.Print("Must supply some regulars expressions!")
		Usage(1)
	}

	asts := make([]frontend.AST, 0, len(patterns))
	for _, p := range patterns {
		ast, err := frontend.Parse([]byte(p))
		if err != nil {
			log.Fatal(err)
		}
		asts = append(asts, ast)
	}

	lexast := asts[len(asts)-1]
	for i := len(asts) - 2; i >= 0; i-- {
		lexast = frontend.NewAltMatch(asts[i], lexast)
	}

	program, err := frontend.Generate(lexast)
	if err != nil {
		log.Fatal(err)
	}

	for i, inst := range program {
		fmt.Printf("%3d %s\n", i, inst.Serialize())
	}
}
示例#9
0
文件: cmd.go 项目: timtadh/sfp
func dbscanReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hf:e:g:a:",
		[]string{
			"help",
			"filename=",
			"epsilon=",
			"gamma=",
			"attr=",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	clusters := "clusters"
	metrics := "metrics"
	attr := ""
	epsilon := 0.2
	gamma := 0.2
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-c", "--clusters-name":
			clusters = oa.Arg()
		case "-m", "--metrics-name":
			metrics = oa.Arg()
		case "-a", "--attr":
			attr = oa.Arg()
		case "-e", "--epsilon":
			epsilon = ParseFloat(oa.Arg())
		case "-g", "--gamma":
			gamma = ParseFloat(oa.Arg())
		default:
			errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	if attr == "" {
		errors.Logf("ERROR", "You must supply --attr=<attr> to dbscan")
		Usage(ErrorCodes["opts"])
	}
	r, err := reporters.NewDbScan(conf, fmtr, clusters, metrics, attr, epsilon, gamma)
	if err != nil {
		errors.Logf("ERROR", "There was error creating output files\n")
		errors.Logf("ERROR", "%v", err)
		os.Exit(1)
	}
	return r, args
}
示例#10
0
文件: cmd.go 项目: timtadh/sfp
func uniqueReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
			"histogram=",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	histogram := ""
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "--histogram":
			histogram = oa.Arg()
		default:
			errors.Logf("ERROR", "Unknown flag '%v'", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	var rptr miners.Reporter
	if len(args) == 0 {
		errors.Logf("ERROR", "You must supply an inner reporter to unique")
		fmt.Fprintln(os.Stderr, "try: unique file")
		Usage(ErrorCodes["opts"])
	} else if _, has := reports[args[0]]; !has {
		errors.Logf("ERROR", "Unknown reporter '%v'", args[0])
		fmt.Fprintln(os.Stderr, "Reporters:")
		for k := range reports {
			fmt.Fprintln(os.Stderr, "  ", k)
		}
		Usage(ErrorCodes["opts"])
	} else {
		rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
	}
	uniq, err := reporters.NewUnique(conf, fmtr, rptr, histogram)
	if err != nil {
		errors.Logf("ERROR", "Error creating unique reporter '%v'\n", err)
		Usage(ErrorCodes["opts"])
	}
	return uniq, args
}
示例#11
0
文件: cmd.go 项目: timtadh/sfp
func chainReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		default:
			errors.Logf("ERROR", "Unknown flag '%v'", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	rptrs := make([]miners.Reporter, 0, 10)
	for len(args) >= 1 {
		if args[0] == "endchain" {
			args = args[1:]
			break
		}
		if _, has := reports[args[0]]; !has {
			errors.Logf("ERROR", "Unknown reporter '%v'\n", args[0])
			fmt.Fprintln(os.Stderr, "Reporters:")
			for k := range reports {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			Usage(ErrorCodes["opts"])
		}
		var rptr miners.Reporter
		rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
		rptrs = append(rptrs, rptr)
	}
	if len(rptrs) == 0 {
		errors.Logf("ERROR", "Empty chain")
		fmt.Fprintln(os.Stderr, "try: chain log file")
		Usage(ErrorCodes["opts"])
	}
	return &reporters.Chain{rptrs}, args
}
示例#12
0
文件: cmd.go 项目: timtadh/sfp
func itemsetType(argv []string, conf *config.Config) (lattice.Loader, func(lattice.DataType, lattice.PrFormatter) lattice.Formatter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hl:", []string{"help", "loader=", "min-items=", "max-items="},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}

	loaderType := "int"
	min := 0
	max := int(math.MaxInt32)
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-l", "--loader":
			loaderType = oa.Arg()
		case "--min-items":
			min = ParseInt(oa.Arg())
		case "--max-items":
			max = ParseInt(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}

	var loader lattice.Loader
	switch loaderType {
	case "int":
		loader, err = itemset.NewIntLoader(conf, min, max)
	default:
		fmt.Fprintf(os.Stderr, "Unknown itemset loader '%v'\n", loaderType)
		Usage(ErrorCodes["opts"])
	}
	if err != nil {
		log.Panic(err)
	}
	fmtr := func(_ lattice.DataType, prfmt lattice.PrFormatter) lattice.Formatter {
		return &itemset.Formatter{prfmt}
	}
	return loader, fmtr, args
}
示例#13
0
文件: cmd.go 项目: timtadh/sfp
func heapProfileReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hp:a:e:",
		[]string{
			"help",
			"profile=",
			"after=",
			"every=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}
	after := 0
	every := 1
	profile := ""
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-p", "--patterns":
			profile = oa.Arg()
		case "-a", "--after":
			after = ParseInt(oa.Arg())
		case "-e", "--every":
			every = ParseInt(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	if profile == "" {
		fmt.Fprintf(os.Stderr, "You must supply a location to write the profile (-p) in heap-profile.\n")
		os.Exit(1)
	}
	r, err := reporters.NewHeapProfile(profile, after, every)
	if err != nil {
		fmt.Fprintf(os.Stderr, "There was error creating output files\n")
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	return r, args
}
示例#14
0
文件: cmd.go 项目: timtadh/sfp
func skipReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hs:",
		[]string{
			"help",
			"skip=",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	skip := 0
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-s", "--skip":
			skip = ParseInt(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	var rptr miners.Reporter
	if len(args) == 0 {
		fmt.Fprintln(os.Stderr, "You must supply an inner reporter to skip")
		fmt.Fprintln(os.Stderr, "try: skip log")
		Usage(ErrorCodes["opts"])
	} else if _, has := reports[args[0]]; !has {
		fmt.Fprintf(os.Stderr, "Unknown reporter '%v'\n", args[0])
		fmt.Fprintln(os.Stderr, "Reporters:")
		for k := range reports {
			fmt.Fprintln(os.Stderr, "  ", k)
		}
		Usage(ErrorCodes["opts"])
	} else {
		rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
	}
	r := reporters.NewSkip(skip, rptr)
	return r, args
}
示例#15
0
文件: cmd.go 项目: timtadh/sfp
func canonMaxReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	var rptr miners.Reporter
	if len(args) == 0 {
		fmt.Fprintln(os.Stderr, "You must supply an inner reporter to canon-max")
		fmt.Fprintln(os.Stderr, "try: unique file")
		Usage(ErrorCodes["opts"])
	} else if _, has := reports[args[0]]; !has {
		fmt.Fprintf(os.Stderr, "Unknown reporter '%v'\n", args[0])
		fmt.Fprintln(os.Stderr, "Reporters:")
		for k := range reports {
			fmt.Fprintln(os.Stderr, "  ", k)
		}
		Usage(ErrorCodes["opts"])
	} else {
		rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
	}
	m, err := reporters.NewCanonMax(rptr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating canon-max reporter '%v'\n", err)
		Usage(ErrorCodes["opts"])
	}
	return m, args
}
示例#16
0
文件: main.go 项目: timtadh/sfp
func qsplorMode(argv []string, conf *config.Config) (miners.Miner, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hs:m:",
		[]string{
			"help",
			"score-function=",
			"max-queue-size=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}
	var scorer qsplor.Scorer = qsplor.Scorers["random"]
	var maxQueueSize int = 10
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "-m", "--max-queue-size":
			maxQueueSize = cmd.ParseInt(oa.Arg())
		case "-s", "--score-function":
			if _, has := qsplor.Scorers[oa.Arg()]; !has {
				fmt.Fprintf(os.Stderr, "Unknown score function: %v\n", oa.Arg())
				fmt.Fprintf(os.Stderr, "Valid score functions:\n")
				for name, _ := range qsplor.Scorers {
					fmt.Fprintf(os.Stderr, "%v\n", name)
				}
				cmd.Usage(cmd.ErrorCodes["opts"])
			}
			scorer = qsplor.Scorers[oa.Arg()]
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}
	return qsplor.NewMiner(conf, scorer, maxQueueSize), args
}
示例#17
0
文件: main.go 项目: timtadh/sfp
func fastmaxMode(argv []string, conf *config.Config) (miners.Miner, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}
	return fastmax.NewWalker(conf), args
}
示例#18
0
文件: cmd.go 项目: timtadh/sfp
func dirReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hd:",
		[]string{
			"help",
			"dir-name=",
			"show-pr",
		},
	)
	if err != nil {
		errors.Logf("ERROR", "%v", err)
		Usage(ErrorCodes["opts"])
	}
	dir := "samples"
	showPr := false
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-d", "--dir-name":
			dir = oa.Arg()
		case "--show-pr":
			showPr = true
		default:
			errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}
	fr, err := reporters.NewDir(conf, fmtr, showPr, dir)
	if err != nil {
		errors.Logf("ERROR", "There was error creating output files\n")
		errors.Logf("ERROR", "%v", err)
		os.Exit(1)
	}
	return fr, args
}
示例#19
0
文件: main.go 项目: timtadh/sfp
func muskMode(argv []string, conf *config.Config) (miners.Miner, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"h",
		[]string{
			"help",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}
	miner := walker.NewWalker(conf, musk.MakeMaxUniformWalk(musk.Next, nil))
	return miner, args
}
示例#20
0
func main() {
	// Destroy after `main()` runs.
	defer unboundInstance.Destroy()

	// Parse and validate args.
	leftovers, optargs, err := getopt.GetOpt(os.Args[1:], "p", []string{"preserve"})
	if err != nil {
		SyncPrintf("error: %s\n", err)
		return
	} else if len(leftovers) > 0 {
		SyncPrintf("error: unrecognized parameter: %s\n", leftovers)
		return
	}
	if len(optargs) > 0 && optargs[0].Opt() == "-p" {
		//SyncPrintf("Found opt!\n")
		preserveInput = true
	}

	domains := ReadLinesFromStdin(func(line string) string {
		return strings.TrimSpace(line)
	})

	resultMap := make(map[string]bool)
	for _, d := range domains {
		resultMap[d] = false
	}

	tasks := make(chan string, CONCURRENCY) //len(domains))

	// Spawn worker goroutines.
	wg := new(sync.WaitGroup)

	// Adding routines to workgroup and running then.
	for i := 0; i < CONCURRENCY; i++ {
		wg.Add(1)
		go worker(tasks, wg)
	}

	receiver := func(numDomains int) {
		defer wg.Done()

		i := 0
	Loop:
		for {
			select {
			case result := <-ch:
				//log.Println(result.response)
				//domain, ips, err := ParseResponse(result.domain, result.response.String())
				if err != nil {
					SyncPrintf("failed :: domain=%s :: error=%s\n", result.domain, err.Error())
					if preserveInput {
						SyncPrintf("%s %s\n", result.originalLine, strings.Join(result.addresses, " "))
					} else {
						SyncPrintf("%s %s\n", result.domain, strings.Join(result.addresses, " "))
					}
				} else {
					if preserveInput {
						SyncPrintf("%s %s\n", result.originalLine, strings.Join(result.addresses, " "))
					} else {
						SyncPrintf("%s %s\n", result.domain, strings.Join(result.addresses, " "))
					}
				}
				resultMap[result.domain] = true
				i++
				//fmt.Printf("%v/%v\n", i, numDomains)
				if i == numDomains {
					break Loop
				}
				/*for d, _ := range resultMap {
					if resultMap[d] == false {
						fmt.Printf("still waiting on: %v\n", d)
					}
				}*/
			}
		}
	}

	wg.Add(1)
	go receiver(len(domains))

	// Processing all links by spreading them to `free` goroutines
	for _, domain := range domains {
		tasks <- domain
	}

	close(tasks)

	// Wait for the workers to finish.
	wg.Wait()
}
示例#21
0
文件: cmd.go 项目: timtadh/sfp
func digraphType(argv []string, conf *config.Config) (lattice.Loader, func(lattice.DataType, lattice.PrFormatter) lattice.Formatter, []string) {
	args, optargs, err := getopt.GetOpt(
		argv,
		"hl:c:i:e:",
		[]string{"help",
			"loader=",
			"count-mode=",
			"fully-optimistic",
			"overlap-pruning",
			"extension-pruning",
			"unsup-embs-pruning",
			"extend-from-embeddings",
			"extend-from-freq-edges",
			"no-caching",
			"emb-search-starting-point=",
			"min-edges=",
			"max-edges=",
			"min-vertices=",
			"max-vertices=",
			"include=",
			"exclude=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}

	loaderType := "veg"
	modeStr := "MNI"
	overlapPruning := false
	extensionPruning := false
	unsupEmbsPruning := false
	extendFromEmb := false
	extendFromEdges := false
	embSearchStartingPoint := subgraph.MostConnected
	caching := true
	minE := 0
	maxE := int(math.MaxInt32)
	minV := 0
	maxV := int(math.MaxInt32)
	includes := make([]string, 0, 10)
	excludes := make([]string, 0, 10)
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-l", "--loader":
			loaderType = oa.Arg()
		case "-c", "--count-mode":
			modeStr = oa.Arg()
		case "--overlap-pruning":
			overlapPruning = true
		case "--extension-pruning":
			extensionPruning = true
		case "--unsup-embs-pruning":
			unsupEmbsPruning = true
		case "--emb-search-starting-point":
			switch oa.Arg() {
			case "random-start":
				embSearchStartingPoint = subgraph.RandomStart
			case "most-connected":
				embSearchStartingPoint = subgraph.MostConnected
			case "least-connected":
				embSearchStartingPoint = subgraph.LeastConnected
			case "most-frequent":
				embSearchStartingPoint = subgraph.MostFrequent
			case "least-frequent":
				embSearchStartingPoint = subgraph.LeastFrequent
			case "most-extensions":
				embSearchStartingPoint = subgraph.MostExtensions
			case "fewest-extensions":
				embSearchStartingPoint = subgraph.FewestExtensions
			case "lowest-cardinality":
				embSearchStartingPoint = subgraph.LowestCardinality
			case "highest-cardinality":
				embSearchStartingPoint = subgraph.HighestCardinality
			default:
				fmt.Fprintf(os.Stderr, "unknown mode for --emb-search-starting-point %v", oa.Arg())
				fmt.Fprintln(os.Stderr, "valid modes: random-start, (most|least)-connected, (most|least)-frequent")
				fmt.Fprintln(os.Stderr, "             (most|fewest)-extensions, (lowest|highest)-cardinality")
				Usage(ErrorCodes["opts"])
			}
		case "--no-caching":
			caching = false
		case "--min-edges":
			minE = ParseInt(oa.Arg())
		case "--max-edges":
			maxE = ParseInt(oa.Arg())
		case "--min-vertices":
			minV = ParseInt(oa.Arg())
		case "--max-vertices":
			maxV = ParseInt(oa.Arg())
		case "--extend-from-embeddings":
			extendFromEmb = true
		case "--extend-from-freq-edges":
			extendFromEdges = true
		case "-i", "--include":
			includes = append(includes, "("+AssertRegex(oa.Arg())+")")
		case "-e", "--exclude":
			excludes = append(excludes, "("+AssertRegex(oa.Arg())+")")
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}

	var mode digraph.Mode
	if extendFromEmb && extendFromEdges {
		fmt.Fprintf(os.Stderr, "Cannot have both --extend-from-embeddings and --extend-from-freq-edges\n")
		Usage(ErrorCodes["opts"])
	} else if extendFromEmb {
		mode |= digraph.ExtFromEmb
	} else if extendFromEdges {
		mode |= digraph.ExtFromFreqEdges
	} else {
		mode |= digraph.ExtFromEmb
	}

	switch modeStr {
	case "MNI":
		mode |= digraph.MNI
	case "FIS":
		mode |= digraph.FIS
	case "GIS":
		mode |= digraph.GIS
	default:
		fmt.Fprintf(os.Stderr, "Unknown support mode '%v'\n", modeStr)
		fmt.Fprintf(os.Stderr, "support modes: MNI (min-image support), FIS (fully independent subgraphs)\n")
		fmt.Fprintf(os.Stderr, "               GIS (greedy independent subgraphs)\n")
		Usage(ErrorCodes["opts"])
	}
	if overlapPruning {
		mode |= digraph.OverlapPruning
	}
	if extensionPruning {
		mode |= digraph.ExtensionPruning
	}
	if unsupEmbsPruning {
		mode |= digraph.EmbeddingPruning
	}
	if caching {
		mode |= digraph.Caching
	}

	var include *regexp.Regexp = nil
	var exclude *regexp.Regexp = nil
	if len(includes) > 0 {
		include = regexp.MustCompile(strings.Join(includes, "|"))
		errors.Logf("INFO", "including labels matching '%v'", include)
	}
	if len(excludes) > 0 {
		exclude = regexp.MustCompile(strings.Join(excludes, "|"))
		errors.Logf("INFO", "excluding labels matching '%v'", exclude)
	}

	dc := &digraph.Config{
		MinEdges:            minE,
		MaxEdges:            maxE,
		MinVertices:         minV,
		MaxVertices:         maxV,
		Mode:                mode,
		Include:             include,
		Exclude:             exclude,
		EmbSearchStartPoint: embSearchStartingPoint,
	}

	var loader lattice.Loader
	switch loaderType {
	case "veg":
		loader, err = digraph.NewVegLoader(conf, dc)
	case "dot":
		loader, err = digraph.NewDotLoader(conf, dc)
	default:
		fmt.Fprintf(os.Stderr, "Unknown itemset loader '%v'\n", loaderType)
		Usage(ErrorCodes["opts"])
	}
	if err != nil {
		log.Panic(err)
	}
	fmtr := func(dt lattice.DataType, prfmt lattice.PrFormatter) lattice.Formatter {
		g := dt.(*digraph.Digraph)
		return digraph.NewFormatter(g, prfmt)
	}
	return loader, fmtr, args
}
示例#22
0
func main() {
	_, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"hl:a:c:s:",
		[]string{"help", "listen=", "assets", "clones=", "src="},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, "error parsing command line flags", err)
		Usage(ErrorCodes["opts"])
	}

	listen := "0.0.0.0:80"
	assets := ""
	clones := ""
	source := ""
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
			os.Exit(0)
		case "-l", "--listen":
			listen = oa.Arg()
		case "-a", "--assets":
			assets, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "assets path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		case "-c", "--clones":
			clones, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "clones path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		case "-s", "--src":
			source, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "source path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}

	if assets == "" {
		fmt.Fprintln(os.Stderr, "You must supply a path to the assets")
		Usage(ErrorCodes["opts"])
	}

	if clones == "" {
		fmt.Fprintln(os.Stderr, "You must supply a path to the clones")
		Usage(ErrorCodes["opts"])
	}

	if source == "" {
		fmt.Fprintln(os.Stderr, "You must supply a path to the source")
		Usage(ErrorCodes["opts"])
	}

	handler := views.Routes(assets, clones, source)

	server := &http.Server{
		Addr:           listen,
		Handler:        handler,
		ReadTimeout:    1 * time.Second,
		WriteTimeout:   1 * time.Second,
		MaxHeaderBytes: http.DefaultMaxHeaderBytes,
		TLSConfig:      nil,
		TLSNextProto:   nil,
		ConnState:      nil,
		ErrorLog:       nil,
	}

	err = server.ListenAndServe()
	if err != nil {
		log.Fatal(err)
	}
}
示例#23
0
文件: main.go 项目: timtadh/sfp
func run() int {
	modes := map[string]cmd.Mode{
		"dfs":         dfsMode,
		"index-speed": indexSpeedMode,
		"vsigram":     vsigramMode,
		"qsplor":      qsplorMode,
	}

	args, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"ho:c:p:",
		[]string{
			"help",
			"output=", "cache=",
			"support=",
			"modes", "types", "reporters",
			"skip-log=",
			"cpu-profile=",
			"parallelism=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		fmt.Fprintln(os.Stderr, "could not process your arguments (perhaps you forgot a mode?) try:")
		fmt.Fprintf(os.Stderr, "$ %v breadth %v\n", os.Args[0], strings.Join(os.Args[1:], " "))
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	output := ""
	cache := ""
	support := 0
	cpuProfile := ""
	parallelism := -1
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "-o", "--output":
			output = cmd.EmptyDir(oa.Arg())
		case "-c", "--cache":
			cache = cmd.EmptyDir(oa.Arg())
		case "-p", "--parallelism":
			parallelism = cmd.ParseInt(oa.Arg())
		case "--support":
			support = cmd.ParseInt(oa.Arg())
		case "--types":
			fmt.Fprintln(os.Stderr, "Types:")
			for k := range cmd.Types {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			os.Exit(0)
		case "--modes":
			fmt.Fprintln(os.Stderr, "Modes:")
			for k := range modes {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			os.Exit(0)
		case "--reporters":
			fmt.Fprintln(os.Stderr, "Reporters:")
			for k := range cmd.Reporters {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			os.Exit(0)
		case "--skip-log":
			level := oa.Arg()
			errors.Logf("INFO", "not logging level %v", level)
			errors.SkipLogging[level] = true
		case "--cpu-profile":
			cpuProfile = cmd.AssertFile(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}

	if support <= 0 {
		fmt.Fprintf(os.Stderr, "Support <= 0, must be > 0\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if output == "" {
		fmt.Fprintf(os.Stderr, "You must supply an output dir (-o)\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if cpuProfile != "" {
		defer cmd.CPUProfile(cpuProfile)()
	}

	conf := &config.Config{
		Cache:       cache,
		Output:      output,
		Support:     support,
		Parallelism: parallelism,
	}

	return cmd.Main(args, conf, modes)
}
示例#24
0
文件: main.go 项目: timtadh/sfp
func run() int {
	args, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"h:p:n:",
		[]string{
			"help",
			"pattern=",
			"cpu-profile=",
			"names=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	patterns := make([]string, 0, 10)
	namesPath := ""
	cpuProfile := ""
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "-p", "--pattern":
			patterns = append(patterns, oa.Arg())
		case "-n", "--names":
			namesPath = cmd.AssertFileExists(oa.Arg())
		case "--cpu-profile":
			cpuProfile = cmd.AssertFile(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}

	if namesPath != "" && len(patterns) > 0 {
		fmt.Fprintf(os.Stderr, "You cannot supply patterns with both (-p) and (-n)\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if len(patterns) == 0 && namesPath == "" {
		fmt.Fprintf(os.Stderr, "You must supply a pattern (-p, -n)\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if namesPath != "" {
		var err error
		patterns, _, err = loadNames(namesPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "There was error loading the probability file\n")
			fmt.Fprintf(os.Stderr, "%v\n", err)
			return 1
		}
	}

	conf := &config.Config{}

	graphs := make([]*digraph.Digraph, 0, 10)
	for len(args) > 0 {
		loadDt, as := cmd.ParseType(args, conf)
		args = as
		dt, _ := loadDt(nil)
		graph := dt.(*digraph.Digraph)
		graphs = append(graphs, graph)
	}

	if cpuProfile != "" {
		defer cmd.CPUProfile(cpuProfile)()
	}

	errors.Logf("INFO", "looking for embeddings")
	for _, graph := range graphs {
		for _, pattern := range patterns {
			sg, err := subgraph.ParsePretty(pattern, graph.Labels)
			if err != nil {
				fmt.Fprintf(os.Stderr, "There was error during the parsing the pattern '%v'\n", pattern)
				fmt.Fprintf(os.Stderr, "%v\n", err)
				return 1
			}
			if sg.Pretty(graph.Labels) != pattern {
				errors.Logf("ERROR", "bad load of pattern")
				errors.Logf("ERROR", "expected %v", pattern)
				errors.Logf("ERROR", "got      %v", sg.Pretty(graph.Labels))
				return 1
			}
			errors.Logf("INFO", "cur sg: %v", sg.Pretty(graph.Labels))
			ei, _ := sg.IterEmbeddings(subgraph.MostConnected, graph.Indices, nil, nil, nil)
			c := 0
			for _, next := ei(false); next != nil; _, next = next(false) {
				c++
			}
			errors.Logf("EMB", "total embs: %v", c)
			fmt.Println(sg.Dotty(graph.Labels, nil, nil))
		}
	}

	return 0
}
示例#25
0
文件: main.go 项目: timtadh/sfp
func run() int {
	args, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"h:p:v:",
		[]string{
			"help",
			"pattern=",
			"cpu-profile=",
			"visualize=",
			"probabilities=",
			"samples=",
			"names=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	visual := ""
	patterns := make([]string, 0, 10)
	prPath := ""
	namesPath := ""
	cpuProfile := ""
	samples := -1
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "-p", "--pattern":
			patterns = append(patterns, oa.Arg())
		case "--probabilities":
			prPath = cmd.AssertFileExists(oa.Arg())
		case "--names":
			namesPath = cmd.AssertFileExists(oa.Arg())
		case "--samples":
			samples = cmd.ParseInt(oa.Arg())
		case "--cpu-profile":
			cpuProfile = cmd.AssertFile(oa.Arg())
		case "-v", "--visual":
			visual = cmd.AssertFile(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}

	if (prPath != "" || namesPath != "") && len(patterns) > 0 {
		fmt.Fprintf(os.Stderr, "You cannot supply patterns with both (-p) and (--probabilities)\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if len(patterns) == 0 && prPath == "" && namesPath == "" {
		fmt.Fprintf(os.Stderr, "You must supply a pattern (-p, --names, --probabilities)\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	var patternCount int = len(patterns)
	var prs []float64 = nil
	if prPath != "" {
		var err error
		prs, patterns, patternCount, err = loadProbabilities(prPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "There was error loading the probability file\n")
			fmt.Fprintf(os.Stderr, "%v\n", err)
			return 1
		}
	} else if namesPath != "" {
		var err error
		prs, patterns, patternCount, err = loadNames(namesPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "There was error loading the probability file\n")
			fmt.Fprintf(os.Stderr, "%v\n", err)
			return 1
		}
	}

	if samples < len(prs) {
		errors.Errorf("INFO", "assuming # of samples is the total number of patterns supplied: %v", patternCount)
		samples = patternCount
	}

	conf := &config.Config{}

	graphs := make([]*digraph.Digraph, 0, 10)
	for len(args) > 0 {
		loadDt, as := cmd.ParseType(args, conf)
		args = as
		dt, _ := loadDt(nil)
		graph := dt.(*digraph.Digraph)
		graphs = append(graphs, graph)
	}

	if cpuProfile != "" {
		defer cmd.CPUProfile(cpuProfile)()
	}

	var visualize io.Writer = nil
	if visual != "" {
		f, err := os.Create(visual)
		if err != nil {
			fmt.Fprintf(os.Stderr, "There was error opening the visualization output file\n")
			fmt.Fprintf(os.Stderr, "%v\n", err)
			return 1
		}
		defer f.Close()
		visualize = f
	}

	matches := make([]float64, 0, len(patterns))
	matched := make([]*subgraph.SubGraph, 0, len(patterns))
	sgEdges := make([]float64, 0, len(patterns))
	total := 0.0
	totalEdges := 0.0
	for _, graph := range graphs {
		for _, pattern := range patterns {
			sg, err := subgraph.ParsePretty(pattern, graph.Labels)
			if err != nil {
				fmt.Fprintf(os.Stderr, "There was error during the parsing the pattern '%v'\n", pattern)
				fmt.Fprintf(os.Stderr, "%v\n", err)
				return 1
			}
			match, csg, err := sg.EstimateMatch(graph.Indices)
			match = match * float64(len(sg.E))
			if err != nil {
				errors.Logf("ERROR", "%v", err)
				return 1
			}
			matches = append(matches, match)
			matched = append(matched, csg)
			sgEdges = append(sgEdges, float64(len(sg.E)))
			// fmt.Printf("%v, %v, %v\n", i+1, match, pattern)
			total += match
			totalEdges += float64(len(sg.E))
			if visualize != nil {
				dotty, err := csg.VisualizeEmbedding(sg.AsIndices(), graph.Labels)
				if err != nil {
					fmt.Fprintf(os.Stderr, "There was error visualizing the embedding '%v'\n", csg)
					fmt.Fprintf(os.Stderr, "%v\n", err)
					return 1
				}
				fmt.Fprintln(visualize, dotty)
			}
		}
	}
	errors.Logf("DEBUG", "prs %v", sum(prs))
	fmt.Printf(", %v, sample total covered edges\n", total)
	fmt.Printf(", %v, sample total edges\n", totalEdges)
	fmt.Printf(", %v, sample covered/total\n", total/totalEdges)
	fmt.Printf(", %v, sample avg covered\n", total/float64(len(patterns)))
	fmt.Printf(", %v, sample avg edges\n", totalEdges/float64(len(patterns)))

	if len(prs) > 0 {
		pis := samplingPrs(samples, prs)
		jpis := jointSamplingPrs(samples, prs, pis)
		estN := estPopSize(pis)
		estTotalMatch := estPopTotal(pis, matches)
		estVarTotalMatch := estVarTotal(pis, jpis, matches)
		estTotalEdges := estPopTotal(pis, sgEdges)
		estVarTotalEdges := estVarTotal(pis, jpis, sgEdges)

		fmt.Printf("\n")
		fmt.Printf(", %v, estimated population total of matched edges\n", estTotalMatch)
		fmt.Printf(", %v, estimated population total of total edges\n", estTotalEdges)
		fmt.Printf(", %v, estimated var population total of match edges\n", estVarTotalMatch)
		fmt.Printf(", %v, estimated var population total of total edges\n", estVarTotalEdges)
		fmt.Printf(", %v, estimated std population total of match edges\n", math.Sqrt(estVarTotalMatch))
		fmt.Printf(", %v, estimated std population total of total edges\n", math.Sqrt(estVarTotalEdges))
		fmt.Printf(", %v, estimated population mean\n", estTotalMatch/estTotalEdges)

		estMeanMatch := estPopMean(estTotalMatch, estN)
		estMeanEdges := estPopMean(estTotalEdges, estN)
		fmt.Printf("\n")
		fmt.Printf(", %v, est. mean matches\n", estMeanMatch)
		fmt.Printf(", %v, est. mean edges\n", estMeanEdges)
		fmt.Printf(", %v, est. cover\n", estMeanMatch/estMeanEdges)

		varMeanMatch := estVarMean(estN, estMeanMatch, pis, jpis, matches)
		varMeanEdges := estVarMean(estN, estMeanEdges, pis, jpis, sgEdges)
		stdMeanMatch := math.Sqrt(varMeanMatch)
		stdMeanEdges := math.Sqrt(varMeanEdges)
		fmt.Printf("\n")
		fmt.Printf(", %v, var. mean matches\n", varMeanMatch)
		fmt.Printf(", %v, var. mean edges\n", varMeanEdges)
		fmt.Printf(", %v, std. mean matches\n", stdMeanMatch)
		fmt.Printf(", %v, std. mean edges\n", stdMeanEdges)

		t := t_alpha_05[samples-1]
		fmt.Printf("\n")
		fmt.Printf(", %v - %v, interval. mean matches\n",
			estMeanMatch-t*stdMeanMatch,
			estMeanMatch+t*stdMeanMatch)
		fmt.Printf(", %v - %v, interval. mean edges\n",
			estMeanEdges-t*stdMeanEdges,
			estMeanEdges+t*stdMeanEdges)
		fmt.Printf(", %v - %v, interval. cover\n",
			math.Max((estMeanMatch-t*stdMeanMatch)/(estMeanEdges+t*stdMeanEdges), 0.0),
			math.Min((estMeanMatch+t*stdMeanMatch)/(estMeanEdges-t*stdMeanEdges), 1.0))
	}
	return 0
}
示例#26
0
文件: main.go 项目: timtadh/sfp
func run() int {
	modes := map[string]cmd.Mode{
		"graple":  grapleMode,
		"fastmax": fastmaxMode,
		"musk":    muskMode,
		"ospace":  ospaceMode,
		"premusk": premuskMode,
		"uniprox": uniproxMode,
	}

	args, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"ho:c:p:",
		[]string{
			"help",
			"output=", "cache=",
			"modes", "types", "reporters",
			"non-unique",
			"support=",
			"samples=",
			"skip-log=",
			"cpu-profile=",
			"parallelism=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		fmt.Fprintln(os.Stderr, "could not process your arguments (perhaps you forgot a mode?) try:")
		fmt.Fprintf(os.Stderr, "$ %v breadth %v\n", os.Args[0], strings.Join(os.Args[1:], " "))
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	output := ""
	cache := ""
	unique := true
	support := 0
	samples := 0
	cpuProfile := ""
	parallelism := -1
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "-o", "--output":
			output = cmd.EmptyDir(oa.Arg())
		case "-c", "--cache":
			cache = cmd.EmptyDir(oa.Arg())
		case "-p", "--parallelism":
			parallelism = cmd.ParseInt(oa.Arg())
		case "--support":
			support = cmd.ParseInt(oa.Arg())
		case "--samples":
			samples = cmd.ParseInt(oa.Arg())
		case "--non-unique":
			unique = false
		case "--types":
			fmt.Fprintln(os.Stderr, "Types:")
			for k := range cmd.Types {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			os.Exit(0)
		case "--modes":
			fmt.Fprintln(os.Stderr, "Modes:")
			for k := range modes {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			os.Exit(0)
		case "--reporters":
			fmt.Fprintln(os.Stderr, "Reporters:")
			for k := range cmd.Reporters {
				fmt.Fprintln(os.Stderr, "  ", k)
			}
			os.Exit(0)
		case "--skip-log":
			level := oa.Arg()
			errors.Logf("INFO", "not logging level %v", level)
			errors.SkipLogging[level] = true
		case "--cpu-profile":
			cpuProfile = cmd.AssertFile(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}

	if support <= 0 {
		fmt.Fprintf(os.Stderr, "Support <= 0, must be > 0\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if samples <= 0 {
		fmt.Fprintf(os.Stderr, "Samples <= 0, must be > 0\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if output == "" {
		fmt.Fprintf(os.Stderr, "You must supply an output dir (-o)\n")
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	if cpuProfile != "" {
		errors.Logf("DEBUG", "starting cpu profile: %v", cpuProfile)
		f, err := os.Create(cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Fatal(err)
		}
		sigs := make(chan os.Signal, 1)
		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
		go func() {
			sig:=<-sigs
			errors.Logf("DEBUG", "closing cpu profile")
			pprof.StopCPUProfile()
			err := f.Close()
			errors.Logf("DEBUG", "closed cpu profile, err: %v", err)
			panic(errors.Errorf("caught signal: %v", sig))
		}()
		defer func() {
			errors.Logf("DEBUG", "closing cpu profile")
			pprof.StopCPUProfile()
			err := f.Close()
			errors.Logf("DEBUG", "closed cpu profile, err: %v", err)
		}()
	}

	conf := &config.Config{
		Cache:   cache,
		Output:  output,
		Support: support,
		Samples: samples,
		Unique:  unique,
		Parallelism: parallelism,
	}
	return cmd.Main(args, conf, modes)
}
示例#27
0
func main() {
	args, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"hl:d:o:f:s:r:c:",
		[]string{"help", "directory=", "output=", "format=",
			"column-sort=", "row-group=", "chart-group=",
			"overlap-columns="},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, "error parsing command line flags", err)
		Usage(1)
	}
	if len(args) > 0 {
		fmt.Fprintf(os.Stderr, "unexpected trailing args `%v`\n", strings.Join(args, " "))
		Usage(1)
	}

	format, err := ingest.ParseFormatString("$(slide) $(subject) $(region) $(stain).tif")
	if err != nil {
		log.Fatal(err)
	}
	directory := ""
	rowGroup := Vars("region")
	chartGroup := Vars("subject,slide")
	columnSort := Vars("stain")
	overlapCols := Vars("")
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
			os.Exit(0)
		case "-f", "--format":
			format, err = ingest.ParseFormatString(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "Invalid format string (%v) '%v'\n", oa.Opt(), oa.Arg())
				fmt.Fprintln(os.Stderr, err)
				Usage(1)
			}
		case "-d", "--directory":
			directory, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "Bad directory (%v) '%v' supplied\n", oa.Opt(), oa.Arg())
				Usage(1)
			}
			if _, err := os.Stat(directory); err != nil && os.IsNotExist(err) {
				fmt.Fprintf(os.Stderr, "Bad directory (%v) '%v' supplied\n", oa.Opt(), oa.Arg())
				Usage(1)
			} else if err != nil {
				fmt.Fprintln(os.Stderr, err)
				fmt.Fprintf(os.Stderr, "Bad directory (%v) '%v' supplied\n", oa.Opt(), oa.Arg())
				Usage(1)
			}
		case "-s", "--column-sort":
			columnSort = Vars(oa.Arg())
		case "-r", "--row-group":
			rowGroup = Vars(oa.Arg())
		case "-c", "--chart-group":
			chartGroup = Vars(oa.Arg())
		case "--overlap-columns":
			overlapCols = Vars(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(1)
		}
	}

	log.Println(directory)

	files, err := ingest.Ingest(directory, format)
	if err != nil {
		log.Fatal(err)
	}
	log.Println(files)
	for _, img := range files {
		log.Println(img)
	}

	C := charts.MakeCharts(files, chartGroup, rowGroup, columnSort, overlapCols)
	for _, chart := range C {
		log.Println("chart", chart.Meta())
		for _, row := range chart.Rows() {
			log.Println("row", row.Meta())
			for _, img := range row.Images() {
				log.Println(img)
			}
			log.Println()
		}
		log.Println()
	}

	log.Println("done")
	fmt.Println(charts.ChartsHTML(C))
}
示例#28
0
文件: main.go 项目: timtadh/sfp
func run() int {
	args, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"h:i:o:",
		[]string{
			"input=",
			"output=",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}
	if len(args) != 0 {
		fmt.Fprintln(os.Stderr, "trailing args: %v", args)
		cmd.Usage(cmd.ErrorCodes["opts"])
	}

	inputPath := ""
	outputPath := ""
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			cmd.Usage(0)
		case "-i", "--input":
			inputPath = cmd.AssertFile(oa.Arg())
		case "-o", "--output":
			outputPath = cmd.AssertFile(oa.Arg())
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			cmd.Usage(cmd.ErrorCodes["opts"])
		}
	}

	var input io.Reader
	if inputPath != "" {
		inputf, err := os.Open(inputPath)
		if err != nil {
			errors.Logf("ERROR", "could not open %v : %v", inputPath, err)
			return 1
		}
		defer inputf.Close()
		input = inputf
	} else {
		inputPath = "<stdin>"
		input = os.Stdin
	}

	var output io.Writer
	if outputPath != "" {
		outputf, err := os.Create(outputPath)
		if err != nil {
			errors.Logf("ERROR", "could not open %v : %v", outputPath, err)
			return 1
		}
		defer outputf.Close()
		if strings.HasSuffix(outputPath, ".gz") {
			z := gzip.NewWriter(outputf)
			defer z.Close()
			output = z
		} else {
			output = outputf
		}
	} else {
		outputPath = "<stdout>"
		output = os.Stdout
	}

	errors.Logf("INFO", "converting %v writing to %v", inputPath, outputPath)
	err = convert(input, output)
	if err != nil {
		errors.Logf("ERROR", "error converting dot to veg %v", err)
		return 1
	}
	return 0
}
示例#29
0
文件: main.go 项目: timtadh/cc-survey
func main() {
	_, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"hl:a:c:s:",
		[]string{"help", "listen=", "assets", "clones=", "src=",
			"private-ssl-key=", "ssl-cert="},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, "error parsing command line flags", err)
		Usage(ErrorCodes["opts"])
	}

	listen := "0.0.0.0:80"
	assets := ""
	clones := ""
	source := ""
	ssl_key := ""
	ssl_cert := ""
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
			os.Exit(0)
		case "-l", "--listen":
			listen = oa.Arg()
		case "-a", "--assets":
			assets, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "assets path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		case "-c", "--clones":
			clones, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "clones path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		case "-s", "--src":
			source, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "source path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		case "--private-ssl-key":
			ssl_key, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "private-ssl-key path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
			_, err = os.Stat(ssl_key)
			if os.IsNotExist(err) {
				fmt.Fprintf(os.Stderr, "private-ssl-key path does not exist. %v", ssl_key)
				Usage(ErrorCodes["opts"])
			} else if err != nil {
				fmt.Fprintf(os.Stderr, "private-ssl-key path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		case "--ssl-cert":
			log.Println("ssl-cert", oa.Arg())
			ssl_cert, err = filepath.Abs(oa.Arg())
			if err != nil {
				fmt.Fprintf(os.Stderr, "ssl-cert path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
			_, err = os.Stat(ssl_cert)
			if os.IsNotExist(err) {
				fmt.Fprintf(os.Stderr, "ssl-cert path does not exist. %v", ssl_cert)
				Usage(ErrorCodes["opts"])
			} else if err != nil {
				fmt.Fprintf(os.Stderr, "ssl-cert path was bad: %v", err)
				Usage(ErrorCodes["opts"])
			}
		default:
			fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
			Usage(ErrorCodes["opts"])
		}
	}

	if assets == "" {
		fmt.Fprintln(os.Stderr, "You must supply a path to the assets")
		Usage(ErrorCodes["opts"])
	}

	if clones == "" {
		fmt.Fprintln(os.Stderr, "You must supply a path to the clones")
		Usage(ErrorCodes["opts"])
	}

	if source == "" {
		fmt.Fprintln(os.Stderr, "You must supply a path to the source")
		Usage(ErrorCodes["opts"])
	}

	if (ssl_key == "" && ssl_cert != "") || (ssl_key != "" && ssl_cert == "") {
		fmt.Fprintln(os.Stderr, "To use ssl you must supply key and cert")
		Usage(ErrorCodes["opts"])
	}

	handler := views.Routes(assets, clones, source)

	server := &http.Server{
		Addr:           listen,
		Handler:        handler,
		ReadTimeout:    1 * time.Second,
		WriteTimeout:   1 * time.Second,
		MaxHeaderBytes: http.DefaultMaxHeaderBytes,
		TLSConfig:      nil,
		TLSNextProto:   nil,
		ConnState:      nil,
		ErrorLog:       nil,
	}

	if ssl_key == "" {
		err = server.ListenAndServe()
		if err != nil {
			log.Fatal(err)
		}
	} else {
		log.Println(ssl_cert, ssl_key)
		err = server.ListenAndServeTLS(ssl_cert, ssl_key)
		if err != nil {
			log.Panic(err)
		}
	}
}
示例#30
0
文件: main.go 项目: timtadh/graple
func main() {
	args, optargs, err := getopt.GetOpt(
		os.Args[1:],
		"hs:m:o:c:",
		[]string{
			"help",
			"support=",
			"cache=",
			"min-vertices=",
			"sample-size=",
			"mem-profile=",
			"cpu-profile=",
			"output=",
			"probabilities",
		},
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		Usage(ErrorCodes["opts"])
	}
	log.Printf("Number of goroutines = %v", runtime.NumGoroutine())

	support := -1
	minVertices := -1
	sampleSize := -1
	memProfile := ""
	cpuProfile := ""
	outputDir := ""
	cache := ""
	compute_prs := false
	for _, oa := range optargs {
		switch oa.Opt() {
		case "-h", "--help":
			Usage(0)
		case "-o", "--output":
			outputDir = EmptyDir(AssertDir(oa.Arg()))
		case "-s", "--support":
			support = ParseInt(oa.Arg())
		case "-m", "--min-vertices":
			minVertices = ParseInt(oa.Arg())
		case "-c", "--cache":
			cache = AssertDir(oa.Arg())
		case "--probabilities":
			compute_prs = true
		case "--sample-size":
			sampleSize = ParseInt(oa.Arg())
		case "--mem-profile":
			memProfile = AssertFile(oa.Arg())
		case "--cpu-profile":
			cpuProfile = AssertFile(oa.Arg())
		}
	}

	if support < 1 {
		fmt.Fprintf(os.Stderr, "You must supply a support greater than 0, you gave %v\n", support)
		Usage(ErrorCodes["opts"])
	}

	if sampleSize < 1 {
		fmt.Fprintf(os.Stderr, "You must supply a sample-size greater than 0, you gave %v\n", sampleSize)
		Usage(ErrorCodes["opts"])
	}

	if outputDir == "" {
		fmt.Fprintf(os.Stderr, "You must supply an output file (use -o)\n")
		Usage(ErrorCodes["opts"])
	}

	if cache == "" {
		fmt.Fprintln(os.Stderr, "you must supply a --cache=<dir>")
		Usage(ErrorCodes["opts"])
	}

	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "Expected a path to the graph file")
		Usage(ErrorCodes["opts"])
	}

	getReader := func() (io.Reader, func()) { return Input(args[0]) }

	if cpuProfile != "" {
		f, err := os.Create(cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Fatal(err)
		}
		defer pprof.StopCPUProfile()
	}

	var memProfFile io.WriteCloser
	if memProfile != "" {
		f, err := os.Create(memProfile)
		if err != nil {
			log.Fatal(err)
		}
		memProfFile = f
		defer f.Close()
	}

	nodePath := path.Join(outputDir, "node-attrs.bptree")

	nodeBf, err := fmap.CreateBlockFile(nodePath)
	if err != nil {
		log.Fatal(err)
	}
	defer nodeBf.Close()
	nodeAttrs, err := bptree.New(nodeBf, 4, -1)
	if err != nil {
		log.Fatal(err)
	}

	G, err := graph.LoadGraph(getReader, "", nodeAttrs, nil)
	if err != nil {
		log.Println("Error loading the graph")
		log.Panic(err)
	}
	log.Print("Loaded graph, about to start mining")

	sgCount := 0
	sgMaker := func() store.SubGraphs {
		name := fmt.Sprintf("subgraphs-%d.b+tree", sgCount)
		sgCount++
		path := path.Join(cache, name)
		s := store.NewFs2BpTree(G, path)
		// os.Remove(path)
		// s, err := store.NewSqlite(G, path)
		// if err != nil {
		// 	log.Panic(err)
		// }
		return s
	}

	idxCount := 0
	idxMaker := func() store.UniqueIndex {
		name := fmt.Sprintf("unique-idx-%d.b+tree", idxCount)
		idxCount++
		path := path.Join(cache, name)
		s := store.NewFs2UniqueIndex(G, path)
		// os.Remove(path)
		// s, err := store.NewSqlite(G, path)
		// if err != nil {
		// 	log.Panic(err)
		// }
		return s
	}

	setsCount := 0
	setsMaker := func() store.SetsMap {
		name := fmt.Sprintf("sets-%d.b+tree", setsCount)
		setsCount++
		path := path.Join(cache, name)
		s := store.NewFs2Sets(path)
		// os.Remove(path)
		// s, err := store.NewSqlite(G, path)
		// if err != nil {
		// 	log.Panic(err)
		// }
		return s
	}

	// memFsMaker := func() store.SubGraphs {
	// 	return store.AnonFs2BpTree(G)
	// }

	m := mine.RandomWalk(
		G,
		support,
		minVertices,
		sampleSize,
		memProfFile,
		sgMaker,
		idxMaker,
		setsMaker,
	)
	keys := list.NewSorted(10, false)
	counts := hashtable.NewLinearHash()
	for label := range m.Report {
		key := types.ByteSlice(label)
		count := 0
		if counts.Has(key) {
			c, err := counts.Get(key)
			if err != nil {
				log.Panic(err)
			}
			count = c.(int)
		}
		counts.Put(key, count+1)
		keys.Add(key)
	}
	log.Println("Tries", m.Tries)
	triesPath := path.Join(outputDir, "tries")
	if f, e := os.Create(triesPath); e != nil {
		log.Fatal(err)
	} else {
		fmt.Fprintln(f, m.Tries)
		f.Close()
	}
	{
		log.Println("Finished mining! Writing output...")
		keyCh := make(chan []byte)
		go func() {
			for k, next := keys.Items()(); next != nil; k, next = next() {
				keyCh <- []byte(k.(types.ByteSlice))
			}
			close(keyCh)
		}()
		writeMaximalPatterns(keyCh, m.AllEmbeddings, nodeAttrs, outputDir)
	}

	if !compute_prs {
		log.Println("Done!")
		return
	}

	log.Println("Finished writing patterns. Computing probabilities...")
	count := 0
	for k, next := keys.Items()(); next != nil; k, next = next() {
		patDir := path.Join(outputDir, fmt.Sprintf("%d", count))
		log.Println("-----------------------------------")
		c, err := counts.Get(k)
		if err != nil {
			log.Fatal(err)
		}
		key := []byte(k.(types.ByteSlice))
		dupCount := c.(int)
		// if max.Count(key) < support {
		// 	log.Println("wat not enough subgraphs", max.Count(key))
		// 	continue
		// }
		if c, err := os.Create(path.Join(patDir, "duplicates")); err != nil {
			log.Fatal(err)
		} else {
			fmt.Fprintln(c, dupCount)
			c.Close()
		}
		for _, sg, next := m.AllEmbeddings.Find(key)(); next != nil; _, sg, next = next() {
			vp, Q, R, u, err := m.PrMatrices(sg)
			if err != nil {
				log.Println(err)
				errPath := path.Join(patDir, "error")
				if f, e := os.Create(errPath); e != nil {
					log.Fatal(err)
				} else {
					fmt.Fprintln(f, err)
					f.Close()
				}
			} else {
				bytes, err := json.Marshal(map[string]interface{}{
					"Q":              Q,
					"R":              R,
					"u":              u,
					"startingPoints": vp,
				})
				if err != nil {
					log.Fatal(err)
				}
				matPath := path.Join(patDir, "matrices.json")
				if m, err := os.Create(matPath); err != nil {
					log.Fatal(err)
				} else {
					_, err := m.Write(bytes)
					if err != nil {
						m.Close()
						log.Fatal(err)
					}
					m.Close()
				}
			}
			break
		}
		count++
	}
	log.Println("Done!")
}