func countReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hf:",
[]string{
"help",
"filename=",
},
)
if err != nil {
errors.Logf("ERROR", "%v", err)
Usage(ErrorCodes["opts"])
}
filename := "count"
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-f", "--filename":
filename = oa.Arg()
default:
errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
r, err := reporters.NewCount(conf, filename)
if err != nil {
errors.Logf("ERROR", "There was error creating output files\n")
errors.Logf("ERROR", "%v", err)
os.Exit(1)
}
return r, args
}
func main() {
short := "h"
long := []string{
"help",
"allow-dups",
}
args, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
port := -1
dups := false
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "--allow-dups":
dups = true
}
}
if len(args) != 1 {
fmt.Fprintln(os.Stderr, "You must specify a port")
Usage(ErrorCodes["opts"])
}
port = parse_int(args[0])
fmt.Println("starting")
server := net.NewServer(func() net.Queue { return queue.NewQueue(dups) })
server.Start(port)
}
func uniproxMode(argv []string, conf *config.Config) (miners.Miner, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hw:",
[]string{
"help",
"walks=",
"max",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
walks := 15
max := false
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "-w", "--walks":
walks = cmd.ParseInt(oa.Arg())
case "--max":
max = true
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
miner, err := uniprox.NewWalker(conf, walks, max)
if err != nil {
log.Fatal(err)
}
return miner, args
}
func premuskMode(argv []string, conf *config.Config) (miners.Miner, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"h",
[]string{
"help",
"teleport=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
teleport := .01
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "--teleport":
teleport = cmd.ParseFloat(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
miner := premusk.NewWalker(conf, teleport)
return miner, args
}
func logReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hl:p:",
[]string{
"help",
"level=",
"prefix=",
"show-pr",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
level := "INFO"
prefix := ""
showPr := false
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-l", "--level":
level = oa.Arg()
case "-p", "--prefix":
prefix = oa.Arg()
case "--show-pr":
showPr = true
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
return reporters.NewLog(fmtr, showPr, level, prefix), args
}
func main() {
short := "hm:"
long := []string{
"help", "mode=",
}
args, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
var mode string
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-m", "--mode":
switch oa.Arg() {
case "create", "read":
mode = oa.Arg()
default:
fmt.Fprintf(os.Stderr, "mode %v not supported\n", oa.Arg())
Usage(ErrorCodes["opts"])
}
}
}
if len(args) != 2 {
fmt.Fprintf(os.Stderr, "Must supply exactly two file paths")
Usage(ErrorCodes["opts"])
}
list_path := args[0]
keys_path := args[1]
f, err := os.Create("stress-" + mode + ".prof")
if err != nil {
panic(err)
}
defer f.Close()
profile_writer = f
switch mode {
case "create":
assert_file_does_not_exist(list_path)
assert_file_does_not_exist(keys_path)
create(list_path, keys_path)
case "read":
assert_file_exist(list_path)
assert_file_exist(keys_path)
read(list_path, keys_path)
}
}
func fileReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hp:e:n:",
[]string{
"help",
"patterns=",
"embeddings=",
"names=",
"matrices=",
"probabilities=",
"show-pr",
},
)
if err != nil {
errors.Logf("ERROR", "%v", err)
Usage(ErrorCodes["opts"])
}
patterns := "patterns"
embeddings := "embeddings"
names := "names.txt"
matrices := "matrices.json"
probabilities := "probabilities.prs"
showPr := false
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-p", "--patterns":
patterns = oa.Arg()
case "-e", "--embeddings":
embeddings = oa.Arg()
case "-n", "--names":
names = oa.Arg()
case "--matrices":
matrices = oa.Arg()
case "--probabilites":
probabilities = oa.Arg()
case "--show-pr":
showPr = true
default:
errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
fr, err := reporters.NewFile(conf, fmtr, showPr, patterns, embeddings, names, matrices, probabilities)
if err != nil {
errors.Logf("ERROR", "There was error creating output files\n")
errors.Logf("ERROR", "%v\n", err)
os.Exit(1)
}
return fr, args
}
func main() {
short := "hp:"
long := []string{
"help",
"pattern=",
}
_, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
if err != nil {
log.Print(os.Stderr, err)
Usage(1)
}
patterns := make([]string, 0, 10)
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-p", "--pattern":
patterns = append(patterns, oa.Arg())
}
}
if len(patterns) <= 0 {
log.Print("Must supply some regulars expressions!")
Usage(1)
}
asts := make([]frontend.AST, 0, len(patterns))
for _, p := range patterns {
ast, err := frontend.Parse([]byte(p))
if err != nil {
log.Fatal(err)
}
asts = append(asts, ast)
}
lexast := asts[len(asts)-1]
for i := len(asts) - 2; i >= 0; i-- {
lexast = frontend.NewAltMatch(asts[i], lexast)
}
program, err := frontend.Generate(lexast)
if err != nil {
log.Fatal(err)
}
for i, inst := range program {
fmt.Printf("%3d %s\n", i, inst.Serialize())
}
}
func dbscanReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hf:e:g:a:",
[]string{
"help",
"filename=",
"epsilon=",
"gamma=",
"attr=",
},
)
if err != nil {
errors.Logf("ERROR", "%v", err)
Usage(ErrorCodes["opts"])
}
clusters := "clusters"
metrics := "metrics"
attr := ""
epsilon := 0.2
gamma := 0.2
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-c", "--clusters-name":
clusters = oa.Arg()
case "-m", "--metrics-name":
metrics = oa.Arg()
case "-a", "--attr":
attr = oa.Arg()
case "-e", "--epsilon":
epsilon = ParseFloat(oa.Arg())
case "-g", "--gamma":
gamma = ParseFloat(oa.Arg())
default:
errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
if attr == "" {
errors.Logf("ERROR", "You must supply --attr=<attr> to dbscan")
Usage(ErrorCodes["opts"])
}
r, err := reporters.NewDbScan(conf, fmtr, clusters, metrics, attr, epsilon, gamma)
if err != nil {
errors.Logf("ERROR", "There was error creating output files\n")
errors.Logf("ERROR", "%v", err)
os.Exit(1)
}
return r, args
}
func uniqueReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"h",
[]string{
"help",
"histogram=",
},
)
if err != nil {
errors.Logf("ERROR", "%v", err)
Usage(ErrorCodes["opts"])
}
histogram := ""
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "--histogram":
histogram = oa.Arg()
default:
errors.Logf("ERROR", "Unknown flag '%v'", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
var rptr miners.Reporter
if len(args) == 0 {
errors.Logf("ERROR", "You must supply an inner reporter to unique")
fmt.Fprintln(os.Stderr, "try: unique file")
Usage(ErrorCodes["opts"])
} else if _, has := reports[args[0]]; !has {
errors.Logf("ERROR", "Unknown reporter '%v'", args[0])
fmt.Fprintln(os.Stderr, "Reporters:")
for k := range reports {
fmt.Fprintln(os.Stderr, " ", k)
}
Usage(ErrorCodes["opts"])
} else {
rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
}
uniq, err := reporters.NewUnique(conf, fmtr, rptr, histogram)
if err != nil {
errors.Logf("ERROR", "Error creating unique reporter '%v'\n", err)
Usage(ErrorCodes["opts"])
}
return uniq, args
}
func chainReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"h",
[]string{
"help",
},
)
if err != nil {
errors.Logf("ERROR", "%v", err)
Usage(ErrorCodes["opts"])
}
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
default:
errors.Logf("ERROR", "Unknown flag '%v'", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
rptrs := make([]miners.Reporter, 0, 10)
for len(args) >= 1 {
if args[0] == "endchain" {
args = args[1:]
break
}
if _, has := reports[args[0]]; !has {
errors.Logf("ERROR", "Unknown reporter '%v'\n", args[0])
fmt.Fprintln(os.Stderr, "Reporters:")
for k := range reports {
fmt.Fprintln(os.Stderr, " ", k)
}
Usage(ErrorCodes["opts"])
}
var rptr miners.Reporter
rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
rptrs = append(rptrs, rptr)
}
if len(rptrs) == 0 {
errors.Logf("ERROR", "Empty chain")
fmt.Fprintln(os.Stderr, "try: chain log file")
Usage(ErrorCodes["opts"])
}
return &reporters.Chain{rptrs}, args
}
func itemsetType(argv []string, conf *config.Config) (lattice.Loader, func(lattice.DataType, lattice.PrFormatter) lattice.Formatter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hl:", []string{"help", "loader=", "min-items=", "max-items="},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
loaderType := "int"
min := 0
max := int(math.MaxInt32)
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-l", "--loader":
loaderType = oa.Arg()
case "--min-items":
min = ParseInt(oa.Arg())
case "--max-items":
max = ParseInt(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
var loader lattice.Loader
switch loaderType {
case "int":
loader, err = itemset.NewIntLoader(conf, min, max)
default:
fmt.Fprintf(os.Stderr, "Unknown itemset loader '%v'\n", loaderType)
Usage(ErrorCodes["opts"])
}
if err != nil {
log.Panic(err)
}
fmtr := func(_ lattice.DataType, prfmt lattice.PrFormatter) lattice.Formatter {
return &itemset.Formatter{prfmt}
}
return loader, fmtr, args
}
func heapProfileReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hp:a:e:",
[]string{
"help",
"profile=",
"after=",
"every=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
after := 0
every := 1
profile := ""
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-p", "--patterns":
profile = oa.Arg()
case "-a", "--after":
after = ParseInt(oa.Arg())
case "-e", "--every":
every = ParseInt(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
if profile == "" {
fmt.Fprintf(os.Stderr, "You must supply a location to write the profile (-p) in heap-profile.\n")
os.Exit(1)
}
r, err := reporters.NewHeapProfile(profile, after, every)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error creating output files\n")
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
return r, args
}
func skipReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hs:",
[]string{
"help",
"skip=",
},
)
if err != nil {
errors.Logf("ERROR", "%v", err)
Usage(ErrorCodes["opts"])
}
skip := 0
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-s", "--skip":
skip = ParseInt(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
var rptr miners.Reporter
if len(args) == 0 {
fmt.Fprintln(os.Stderr, "You must supply an inner reporter to skip")
fmt.Fprintln(os.Stderr, "try: skip log")
Usage(ErrorCodes["opts"])
} else if _, has := reports[args[0]]; !has {
fmt.Fprintf(os.Stderr, "Unknown reporter '%v'\n", args[0])
fmt.Fprintln(os.Stderr, "Reporters:")
for k := range reports {
fmt.Fprintln(os.Stderr, " ", k)
}
Usage(ErrorCodes["opts"])
} else {
rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
}
r := reporters.NewSkip(skip, rptr)
return r, args
}
func canonMaxReporter(reports map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"h",
[]string{
"help",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
var rptr miners.Reporter
if len(args) == 0 {
fmt.Fprintln(os.Stderr, "You must supply an inner reporter to canon-max")
fmt.Fprintln(os.Stderr, "try: unique file")
Usage(ErrorCodes["opts"])
} else if _, has := reports[args[0]]; !has {
fmt.Fprintf(os.Stderr, "Unknown reporter '%v'\n", args[0])
fmt.Fprintln(os.Stderr, "Reporters:")
for k := range reports {
fmt.Fprintln(os.Stderr, " ", k)
}
Usage(ErrorCodes["opts"])
} else {
rptr, args = reports[args[0]](reports, args[1:], fmtr, conf)
}
m, err := reporters.NewCanonMax(rptr)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating canon-max reporter '%v'\n", err)
Usage(ErrorCodes["opts"])
}
return m, args
}
func qsplorMode(argv []string, conf *config.Config) (miners.Miner, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hs:m:",
[]string{
"help",
"score-function=",
"max-queue-size=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
var scorer qsplor.Scorer = qsplor.Scorers["random"]
var maxQueueSize int = 10
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "-m", "--max-queue-size":
maxQueueSize = cmd.ParseInt(oa.Arg())
case "-s", "--score-function":
if _, has := qsplor.Scorers[oa.Arg()]; !has {
fmt.Fprintf(os.Stderr, "Unknown score function: %v\n", oa.Arg())
fmt.Fprintf(os.Stderr, "Valid score functions:\n")
for name, _ := range qsplor.Scorers {
fmt.Fprintf(os.Stderr, "%v\n", name)
}
cmd.Usage(cmd.ErrorCodes["opts"])
}
scorer = qsplor.Scorers[oa.Arg()]
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
return qsplor.NewMiner(conf, scorer, maxQueueSize), args
}
func fastmaxMode(argv []string, conf *config.Config) (miners.Miner, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"h",
[]string{
"help",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
return fastmax.NewWalker(conf), args
}
func dirReporter(rptrs map[string]Reporter, argv []string, fmtr lattice.Formatter, conf *config.Config) (miners.Reporter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hd:",
[]string{
"help",
"dir-name=",
"show-pr",
},
)
if err != nil {
errors.Logf("ERROR", "%v", err)
Usage(ErrorCodes["opts"])
}
dir := "samples"
showPr := false
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-d", "--dir-name":
dir = oa.Arg()
case "--show-pr":
showPr = true
default:
errors.Logf("ERROR", "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
fr, err := reporters.NewDir(conf, fmtr, showPr, dir)
if err != nil {
errors.Logf("ERROR", "There was error creating output files\n")
errors.Logf("ERROR", "%v", err)
os.Exit(1)
}
return fr, args
}
func muskMode(argv []string, conf *config.Config) (miners.Miner, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"h",
[]string{
"help",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
miner := walker.NewWalker(conf, musk.MakeMaxUniformWalk(musk.Next, nil))
return miner, args
}
func main() {
// Destroy after `main()` runs.
defer unboundInstance.Destroy()
// Parse and validate args.
leftovers, optargs, err := getopt.GetOpt(os.Args[1:], "p", []string{"preserve"})
if err != nil {
SyncPrintf("error: %s\n", err)
return
} else if len(leftovers) > 0 {
SyncPrintf("error: unrecognized parameter: %s\n", leftovers)
return
}
if len(optargs) > 0 && optargs[0].Opt() == "-p" {
//SyncPrintf("Found opt!\n")
preserveInput = true
}
domains := ReadLinesFromStdin(func(line string) string {
return strings.TrimSpace(line)
})
resultMap := make(map[string]bool)
for _, d := range domains {
resultMap[d] = false
}
tasks := make(chan string, CONCURRENCY) //len(domains))
// Spawn worker goroutines.
wg := new(sync.WaitGroup)
// Adding routines to workgroup and running then.
for i := 0; i < CONCURRENCY; i++ {
wg.Add(1)
go worker(tasks, wg)
}
receiver := func(numDomains int) {
defer wg.Done()
i := 0
Loop:
for {
select {
case result := <-ch:
//log.Println(result.response)
//domain, ips, err := ParseResponse(result.domain, result.response.String())
if err != nil {
SyncPrintf("failed :: domain=%s :: error=%s\n", result.domain, err.Error())
if preserveInput {
SyncPrintf("%s %s\n", result.originalLine, strings.Join(result.addresses, " "))
} else {
SyncPrintf("%s %s\n", result.domain, strings.Join(result.addresses, " "))
}
} else {
if preserveInput {
SyncPrintf("%s %s\n", result.originalLine, strings.Join(result.addresses, " "))
} else {
SyncPrintf("%s %s\n", result.domain, strings.Join(result.addresses, " "))
}
}
resultMap[result.domain] = true
i++
//fmt.Printf("%v/%v\n", i, numDomains)
if i == numDomains {
break Loop
}
/*for d, _ := range resultMap {
if resultMap[d] == false {
fmt.Printf("still waiting on: %v\n", d)
}
}*/
}
}
}
wg.Add(1)
go receiver(len(domains))
// Processing all links by spreading them to `free` goroutines
for _, domain := range domains {
tasks <- domain
}
close(tasks)
// Wait for the workers to finish.
wg.Wait()
}
func digraphType(argv []string, conf *config.Config) (lattice.Loader, func(lattice.DataType, lattice.PrFormatter) lattice.Formatter, []string) {
args, optargs, err := getopt.GetOpt(
argv,
"hl:c:i:e:",
[]string{"help",
"loader=",
"count-mode=",
"fully-optimistic",
"overlap-pruning",
"extension-pruning",
"unsup-embs-pruning",
"extend-from-embeddings",
"extend-from-freq-edges",
"no-caching",
"emb-search-starting-point=",
"min-edges=",
"max-edges=",
"min-vertices=",
"max-vertices=",
"include=",
"exclude=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
loaderType := "veg"
modeStr := "MNI"
overlapPruning := false
extensionPruning := false
unsupEmbsPruning := false
extendFromEmb := false
extendFromEdges := false
embSearchStartingPoint := subgraph.MostConnected
caching := true
minE := 0
maxE := int(math.MaxInt32)
minV := 0
maxV := int(math.MaxInt32)
includes := make([]string, 0, 10)
excludes := make([]string, 0, 10)
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-l", "--loader":
loaderType = oa.Arg()
case "-c", "--count-mode":
modeStr = oa.Arg()
case "--overlap-pruning":
overlapPruning = true
case "--extension-pruning":
extensionPruning = true
case "--unsup-embs-pruning":
unsupEmbsPruning = true
case "--emb-search-starting-point":
switch oa.Arg() {
case "random-start":
embSearchStartingPoint = subgraph.RandomStart
case "most-connected":
embSearchStartingPoint = subgraph.MostConnected
case "least-connected":
embSearchStartingPoint = subgraph.LeastConnected
case "most-frequent":
embSearchStartingPoint = subgraph.MostFrequent
case "least-frequent":
embSearchStartingPoint = subgraph.LeastFrequent
case "most-extensions":
embSearchStartingPoint = subgraph.MostExtensions
case "fewest-extensions":
embSearchStartingPoint = subgraph.FewestExtensions
case "lowest-cardinality":
embSearchStartingPoint = subgraph.LowestCardinality
case "highest-cardinality":
embSearchStartingPoint = subgraph.HighestCardinality
default:
fmt.Fprintf(os.Stderr, "unknown mode for --emb-search-starting-point %v", oa.Arg())
fmt.Fprintln(os.Stderr, "valid modes: random-start, (most|least)-connected, (most|least)-frequent")
fmt.Fprintln(os.Stderr, " (most|fewest)-extensions, (lowest|highest)-cardinality")
Usage(ErrorCodes["opts"])
}
case "--no-caching":
caching = false
case "--min-edges":
minE = ParseInt(oa.Arg())
case "--max-edges":
maxE = ParseInt(oa.Arg())
case "--min-vertices":
minV = ParseInt(oa.Arg())
case "--max-vertices":
maxV = ParseInt(oa.Arg())
case "--extend-from-embeddings":
extendFromEmb = true
case "--extend-from-freq-edges":
extendFromEdges = true
case "-i", "--include":
includes = append(includes, "("+AssertRegex(oa.Arg())+")")
case "-e", "--exclude":
excludes = append(excludes, "("+AssertRegex(oa.Arg())+")")
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
var mode digraph.Mode
if extendFromEmb && extendFromEdges {
fmt.Fprintf(os.Stderr, "Cannot have both --extend-from-embeddings and --extend-from-freq-edges\n")
Usage(ErrorCodes["opts"])
} else if extendFromEmb {
mode |= digraph.ExtFromEmb
} else if extendFromEdges {
mode |= digraph.ExtFromFreqEdges
} else {
mode |= digraph.ExtFromEmb
}
switch modeStr {
case "MNI":
mode |= digraph.MNI
case "FIS":
mode |= digraph.FIS
case "GIS":
mode |= digraph.GIS
default:
fmt.Fprintf(os.Stderr, "Unknown support mode '%v'\n", modeStr)
fmt.Fprintf(os.Stderr, "support modes: MNI (min-image support), FIS (fully independent subgraphs)\n")
fmt.Fprintf(os.Stderr, " GIS (greedy independent subgraphs)\n")
Usage(ErrorCodes["opts"])
}
if overlapPruning {
mode |= digraph.OverlapPruning
}
if extensionPruning {
mode |= digraph.ExtensionPruning
}
if unsupEmbsPruning {
mode |= digraph.EmbeddingPruning
}
if caching {
mode |= digraph.Caching
}
var include *regexp.Regexp = nil
var exclude *regexp.Regexp = nil
if len(includes) > 0 {
include = regexp.MustCompile(strings.Join(includes, "|"))
errors.Logf("INFO", "including labels matching '%v'", include)
}
if len(excludes) > 0 {
exclude = regexp.MustCompile(strings.Join(excludes, "|"))
errors.Logf("INFO", "excluding labels matching '%v'", exclude)
}
dc := &digraph.Config{
MinEdges: minE,
MaxEdges: maxE,
MinVertices: minV,
MaxVertices: maxV,
Mode: mode,
Include: include,
Exclude: exclude,
EmbSearchStartPoint: embSearchStartingPoint,
}
var loader lattice.Loader
switch loaderType {
case "veg":
loader, err = digraph.NewVegLoader(conf, dc)
case "dot":
loader, err = digraph.NewDotLoader(conf, dc)
default:
fmt.Fprintf(os.Stderr, "Unknown itemset loader '%v'\n", loaderType)
Usage(ErrorCodes["opts"])
}
if err != nil {
log.Panic(err)
}
fmtr := func(dt lattice.DataType, prfmt lattice.PrFormatter) lattice.Formatter {
g := dt.(*digraph.Digraph)
return digraph.NewFormatter(g, prfmt)
}
return loader, fmtr, args
}
func main() {
_, optargs, err := getopt.GetOpt(
os.Args[1:],
"hl:a:c:s:",
[]string{"help", "listen=", "assets", "clones=", "src="},
)
if err != nil {
fmt.Fprintln(os.Stderr, "error parsing command line flags", err)
Usage(ErrorCodes["opts"])
}
listen := "0.0.0.0:80"
assets := ""
clones := ""
source := ""
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
os.Exit(0)
case "-l", "--listen":
listen = oa.Arg()
case "-a", "--assets":
assets, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "assets path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
case "-c", "--clones":
clones, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "clones path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
case "-s", "--src":
source, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "source path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
if assets == "" {
fmt.Fprintln(os.Stderr, "You must supply a path to the assets")
Usage(ErrorCodes["opts"])
}
if clones == "" {
fmt.Fprintln(os.Stderr, "You must supply a path to the clones")
Usage(ErrorCodes["opts"])
}
if source == "" {
fmt.Fprintln(os.Stderr, "You must supply a path to the source")
Usage(ErrorCodes["opts"])
}
handler := views.Routes(assets, clones, source)
server := &http.Server{
Addr: listen,
Handler: handler,
ReadTimeout: 1 * time.Second,
WriteTimeout: 1 * time.Second,
MaxHeaderBytes: http.DefaultMaxHeaderBytes,
TLSConfig: nil,
TLSNextProto: nil,
ConnState: nil,
ErrorLog: nil,
}
err = server.ListenAndServe()
if err != nil {
log.Fatal(err)
}
}
func run() int {
modes := map[string]cmd.Mode{
"dfs": dfsMode,
"index-speed": indexSpeedMode,
"vsigram": vsigramMode,
"qsplor": qsplorMode,
}
args, optargs, err := getopt.GetOpt(
os.Args[1:],
"ho:c:p:",
[]string{
"help",
"output=", "cache=",
"support=",
"modes", "types", "reporters",
"skip-log=",
"cpu-profile=",
"parallelism=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
fmt.Fprintln(os.Stderr, "could not process your arguments (perhaps you forgot a mode?) try:")
fmt.Fprintf(os.Stderr, "$ %v breadth %v\n", os.Args[0], strings.Join(os.Args[1:], " "))
cmd.Usage(cmd.ErrorCodes["opts"])
}
output := ""
cache := ""
support := 0
cpuProfile := ""
parallelism := -1
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "-o", "--output":
output = cmd.EmptyDir(oa.Arg())
case "-c", "--cache":
cache = cmd.EmptyDir(oa.Arg())
case "-p", "--parallelism":
parallelism = cmd.ParseInt(oa.Arg())
case "--support":
support = cmd.ParseInt(oa.Arg())
case "--types":
fmt.Fprintln(os.Stderr, "Types:")
for k := range cmd.Types {
fmt.Fprintln(os.Stderr, " ", k)
}
os.Exit(0)
case "--modes":
fmt.Fprintln(os.Stderr, "Modes:")
for k := range modes {
fmt.Fprintln(os.Stderr, " ", k)
}
os.Exit(0)
case "--reporters":
fmt.Fprintln(os.Stderr, "Reporters:")
for k := range cmd.Reporters {
fmt.Fprintln(os.Stderr, " ", k)
}
os.Exit(0)
case "--skip-log":
level := oa.Arg()
errors.Logf("INFO", "not logging level %v", level)
errors.SkipLogging[level] = true
case "--cpu-profile":
cpuProfile = cmd.AssertFile(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
if support <= 0 {
fmt.Fprintf(os.Stderr, "Support <= 0, must be > 0\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if output == "" {
fmt.Fprintf(os.Stderr, "You must supply an output dir (-o)\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if cpuProfile != "" {
defer cmd.CPUProfile(cpuProfile)()
}
conf := &config.Config{
Cache: cache,
Output: output,
Support: support,
Parallelism: parallelism,
}
return cmd.Main(args, conf, modes)
}
func run() int {
args, optargs, err := getopt.GetOpt(
os.Args[1:],
"h:p:n:",
[]string{
"help",
"pattern=",
"cpu-profile=",
"names=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
patterns := make([]string, 0, 10)
namesPath := ""
cpuProfile := ""
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "-p", "--pattern":
patterns = append(patterns, oa.Arg())
case "-n", "--names":
namesPath = cmd.AssertFileExists(oa.Arg())
case "--cpu-profile":
cpuProfile = cmd.AssertFile(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
if namesPath != "" && len(patterns) > 0 {
fmt.Fprintf(os.Stderr, "You cannot supply patterns with both (-p) and (-n)\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if len(patterns) == 0 && namesPath == "" {
fmt.Fprintf(os.Stderr, "You must supply a pattern (-p, -n)\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if namesPath != "" {
var err error
patterns, _, err = loadNames(namesPath)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error loading the probability file\n")
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
}
conf := &config.Config{}
graphs := make([]*digraph.Digraph, 0, 10)
for len(args) > 0 {
loadDt, as := cmd.ParseType(args, conf)
args = as
dt, _ := loadDt(nil)
graph := dt.(*digraph.Digraph)
graphs = append(graphs, graph)
}
if cpuProfile != "" {
defer cmd.CPUProfile(cpuProfile)()
}
errors.Logf("INFO", "looking for embeddings")
for _, graph := range graphs {
for _, pattern := range patterns {
sg, err := subgraph.ParsePretty(pattern, graph.Labels)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error during the parsing the pattern '%v'\n", pattern)
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
if sg.Pretty(graph.Labels) != pattern {
errors.Logf("ERROR", "bad load of pattern")
errors.Logf("ERROR", "expected %v", pattern)
errors.Logf("ERROR", "got %v", sg.Pretty(graph.Labels))
return 1
}
errors.Logf("INFO", "cur sg: %v", sg.Pretty(graph.Labels))
ei, _ := sg.IterEmbeddings(subgraph.MostConnected, graph.Indices, nil, nil, nil)
c := 0
for _, next := ei(false); next != nil; _, next = next(false) {
c++
}
errors.Logf("EMB", "total embs: %v", c)
fmt.Println(sg.Dotty(graph.Labels, nil, nil))
}
}
return 0
}
func run() int {
args, optargs, err := getopt.GetOpt(
os.Args[1:],
"h:p:v:",
[]string{
"help",
"pattern=",
"cpu-profile=",
"visualize=",
"probabilities=",
"samples=",
"names=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
visual := ""
patterns := make([]string, 0, 10)
prPath := ""
namesPath := ""
cpuProfile := ""
samples := -1
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "-p", "--pattern":
patterns = append(patterns, oa.Arg())
case "--probabilities":
prPath = cmd.AssertFileExists(oa.Arg())
case "--names":
namesPath = cmd.AssertFileExists(oa.Arg())
case "--samples":
samples = cmd.ParseInt(oa.Arg())
case "--cpu-profile":
cpuProfile = cmd.AssertFile(oa.Arg())
case "-v", "--visual":
visual = cmd.AssertFile(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
if (prPath != "" || namesPath != "") && len(patterns) > 0 {
fmt.Fprintf(os.Stderr, "You cannot supply patterns with both (-p) and (--probabilities)\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if len(patterns) == 0 && prPath == "" && namesPath == "" {
fmt.Fprintf(os.Stderr, "You must supply a pattern (-p, --names, --probabilities)\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
var patternCount int = len(patterns)
var prs []float64 = nil
if prPath != "" {
var err error
prs, patterns, patternCount, err = loadProbabilities(prPath)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error loading the probability file\n")
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
} else if namesPath != "" {
var err error
prs, patterns, patternCount, err = loadNames(namesPath)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error loading the probability file\n")
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
}
if samples < len(prs) {
errors.Errorf("INFO", "assuming # of samples is the total number of patterns supplied: %v", patternCount)
samples = patternCount
}
conf := &config.Config{}
graphs := make([]*digraph.Digraph, 0, 10)
for len(args) > 0 {
loadDt, as := cmd.ParseType(args, conf)
args = as
dt, _ := loadDt(nil)
graph := dt.(*digraph.Digraph)
graphs = append(graphs, graph)
}
if cpuProfile != "" {
defer cmd.CPUProfile(cpuProfile)()
}
var visualize io.Writer = nil
if visual != "" {
f, err := os.Create(visual)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error opening the visualization output file\n")
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
defer f.Close()
visualize = f
}
matches := make([]float64, 0, len(patterns))
matched := make([]*subgraph.SubGraph, 0, len(patterns))
sgEdges := make([]float64, 0, len(patterns))
total := 0.0
totalEdges := 0.0
for _, graph := range graphs {
for _, pattern := range patterns {
sg, err := subgraph.ParsePretty(pattern, graph.Labels)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error during the parsing the pattern '%v'\n", pattern)
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
match, csg, err := sg.EstimateMatch(graph.Indices)
match = match * float64(len(sg.E))
if err != nil {
errors.Logf("ERROR", "%v", err)
return 1
}
matches = append(matches, match)
matched = append(matched, csg)
sgEdges = append(sgEdges, float64(len(sg.E)))
// fmt.Printf("%v, %v, %v\n", i+1, match, pattern)
total += match
totalEdges += float64(len(sg.E))
if visualize != nil {
dotty, err := csg.VisualizeEmbedding(sg.AsIndices(), graph.Labels)
if err != nil {
fmt.Fprintf(os.Stderr, "There was error visualizing the embedding '%v'\n", csg)
fmt.Fprintf(os.Stderr, "%v\n", err)
return 1
}
fmt.Fprintln(visualize, dotty)
}
}
}
errors.Logf("DEBUG", "prs %v", sum(prs))
fmt.Printf(", %v, sample total covered edges\n", total)
fmt.Printf(", %v, sample total edges\n", totalEdges)
fmt.Printf(", %v, sample covered/total\n", total/totalEdges)
fmt.Printf(", %v, sample avg covered\n", total/float64(len(patterns)))
fmt.Printf(", %v, sample avg edges\n", totalEdges/float64(len(patterns)))
if len(prs) > 0 {
pis := samplingPrs(samples, prs)
jpis := jointSamplingPrs(samples, prs, pis)
estN := estPopSize(pis)
estTotalMatch := estPopTotal(pis, matches)
estVarTotalMatch := estVarTotal(pis, jpis, matches)
estTotalEdges := estPopTotal(pis, sgEdges)
estVarTotalEdges := estVarTotal(pis, jpis, sgEdges)
fmt.Printf("\n")
fmt.Printf(", %v, estimated population total of matched edges\n", estTotalMatch)
fmt.Printf(", %v, estimated population total of total edges\n", estTotalEdges)
fmt.Printf(", %v, estimated var population total of match edges\n", estVarTotalMatch)
fmt.Printf(", %v, estimated var population total of total edges\n", estVarTotalEdges)
fmt.Printf(", %v, estimated std population total of match edges\n", math.Sqrt(estVarTotalMatch))
fmt.Printf(", %v, estimated std population total of total edges\n", math.Sqrt(estVarTotalEdges))
fmt.Printf(", %v, estimated population mean\n", estTotalMatch/estTotalEdges)
estMeanMatch := estPopMean(estTotalMatch, estN)
estMeanEdges := estPopMean(estTotalEdges, estN)
fmt.Printf("\n")
fmt.Printf(", %v, est. mean matches\n", estMeanMatch)
fmt.Printf(", %v, est. mean edges\n", estMeanEdges)
fmt.Printf(", %v, est. cover\n", estMeanMatch/estMeanEdges)
varMeanMatch := estVarMean(estN, estMeanMatch, pis, jpis, matches)
varMeanEdges := estVarMean(estN, estMeanEdges, pis, jpis, sgEdges)
stdMeanMatch := math.Sqrt(varMeanMatch)
stdMeanEdges := math.Sqrt(varMeanEdges)
fmt.Printf("\n")
fmt.Printf(", %v, var. mean matches\n", varMeanMatch)
fmt.Printf(", %v, var. mean edges\n", varMeanEdges)
fmt.Printf(", %v, std. mean matches\n", stdMeanMatch)
fmt.Printf(", %v, std. mean edges\n", stdMeanEdges)
t := t_alpha_05[samples-1]
fmt.Printf("\n")
fmt.Printf(", %v - %v, interval. mean matches\n",
estMeanMatch-t*stdMeanMatch,
estMeanMatch+t*stdMeanMatch)
fmt.Printf(", %v - %v, interval. mean edges\n",
estMeanEdges-t*stdMeanEdges,
estMeanEdges+t*stdMeanEdges)
fmt.Printf(", %v - %v, interval. cover\n",
math.Max((estMeanMatch-t*stdMeanMatch)/(estMeanEdges+t*stdMeanEdges), 0.0),
math.Min((estMeanMatch+t*stdMeanMatch)/(estMeanEdges-t*stdMeanEdges), 1.0))
}
return 0
}
func run() int {
modes := map[string]cmd.Mode{
"graple": grapleMode,
"fastmax": fastmaxMode,
"musk": muskMode,
"ospace": ospaceMode,
"premusk": premuskMode,
"uniprox": uniproxMode,
}
args, optargs, err := getopt.GetOpt(
os.Args[1:],
"ho:c:p:",
[]string{
"help",
"output=", "cache=",
"modes", "types", "reporters",
"non-unique",
"support=",
"samples=",
"skip-log=",
"cpu-profile=",
"parallelism=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
fmt.Fprintln(os.Stderr, "could not process your arguments (perhaps you forgot a mode?) try:")
fmt.Fprintf(os.Stderr, "$ %v breadth %v\n", os.Args[0], strings.Join(os.Args[1:], " "))
cmd.Usage(cmd.ErrorCodes["opts"])
}
output := ""
cache := ""
unique := true
support := 0
samples := 0
cpuProfile := ""
parallelism := -1
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "-o", "--output":
output = cmd.EmptyDir(oa.Arg())
case "-c", "--cache":
cache = cmd.EmptyDir(oa.Arg())
case "-p", "--parallelism":
parallelism = cmd.ParseInt(oa.Arg())
case "--support":
support = cmd.ParseInt(oa.Arg())
case "--samples":
samples = cmd.ParseInt(oa.Arg())
case "--non-unique":
unique = false
case "--types":
fmt.Fprintln(os.Stderr, "Types:")
for k := range cmd.Types {
fmt.Fprintln(os.Stderr, " ", k)
}
os.Exit(0)
case "--modes":
fmt.Fprintln(os.Stderr, "Modes:")
for k := range modes {
fmt.Fprintln(os.Stderr, " ", k)
}
os.Exit(0)
case "--reporters":
fmt.Fprintln(os.Stderr, "Reporters:")
for k := range cmd.Reporters {
fmt.Fprintln(os.Stderr, " ", k)
}
os.Exit(0)
case "--skip-log":
level := oa.Arg()
errors.Logf("INFO", "not logging level %v", level)
errors.SkipLogging[level] = true
case "--cpu-profile":
cpuProfile = cmd.AssertFile(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
if support <= 0 {
fmt.Fprintf(os.Stderr, "Support <= 0, must be > 0\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if samples <= 0 {
fmt.Fprintf(os.Stderr, "Samples <= 0, must be > 0\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if output == "" {
fmt.Fprintf(os.Stderr, "You must supply an output dir (-o)\n")
cmd.Usage(cmd.ErrorCodes["opts"])
}
if cpuProfile != "" {
errors.Logf("DEBUG", "starting cpu profile: %v", cpuProfile)
f, err := os.Create(cpuProfile)
if err != nil {
log.Fatal(err)
}
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatal(err)
}
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig:=<-sigs
errors.Logf("DEBUG", "closing cpu profile")
pprof.StopCPUProfile()
err := f.Close()
errors.Logf("DEBUG", "closed cpu profile, err: %v", err)
panic(errors.Errorf("caught signal: %v", sig))
}()
defer func() {
errors.Logf("DEBUG", "closing cpu profile")
pprof.StopCPUProfile()
err := f.Close()
errors.Logf("DEBUG", "closed cpu profile, err: %v", err)
}()
}
conf := &config.Config{
Cache: cache,
Output: output,
Support: support,
Samples: samples,
Unique: unique,
Parallelism: parallelism,
}
return cmd.Main(args, conf, modes)
}
func main() {
args, optargs, err := getopt.GetOpt(
os.Args[1:],
"hl:d:o:f:s:r:c:",
[]string{"help", "directory=", "output=", "format=",
"column-sort=", "row-group=", "chart-group=",
"overlap-columns="},
)
if err != nil {
fmt.Fprintln(os.Stderr, "error parsing command line flags", err)
Usage(1)
}
if len(args) > 0 {
fmt.Fprintf(os.Stderr, "unexpected trailing args `%v`\n", strings.Join(args, " "))
Usage(1)
}
format, err := ingest.ParseFormatString("$(slide) $(subject) $(region) $(stain).tif")
if err != nil {
log.Fatal(err)
}
directory := ""
rowGroup := Vars("region")
chartGroup := Vars("subject,slide")
columnSort := Vars("stain")
overlapCols := Vars("")
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
os.Exit(0)
case "-f", "--format":
format, err = ingest.ParseFormatString(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid format string (%v) '%v'\n", oa.Opt(), oa.Arg())
fmt.Fprintln(os.Stderr, err)
Usage(1)
}
case "-d", "--directory":
directory, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "Bad directory (%v) '%v' supplied\n", oa.Opt(), oa.Arg())
Usage(1)
}
if _, err := os.Stat(directory); err != nil && os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Bad directory (%v) '%v' supplied\n", oa.Opt(), oa.Arg())
Usage(1)
} else if err != nil {
fmt.Fprintln(os.Stderr, err)
fmt.Fprintf(os.Stderr, "Bad directory (%v) '%v' supplied\n", oa.Opt(), oa.Arg())
Usage(1)
}
case "-s", "--column-sort":
columnSort = Vars(oa.Arg())
case "-r", "--row-group":
rowGroup = Vars(oa.Arg())
case "-c", "--chart-group":
chartGroup = Vars(oa.Arg())
case "--overlap-columns":
overlapCols = Vars(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(1)
}
}
log.Println(directory)
files, err := ingest.Ingest(directory, format)
if err != nil {
log.Fatal(err)
}
log.Println(files)
for _, img := range files {
log.Println(img)
}
C := charts.MakeCharts(files, chartGroup, rowGroup, columnSort, overlapCols)
for _, chart := range C {
log.Println("chart", chart.Meta())
for _, row := range chart.Rows() {
log.Println("row", row.Meta())
for _, img := range row.Images() {
log.Println(img)
}
log.Println()
}
log.Println()
}
log.Println("done")
fmt.Println(charts.ChartsHTML(C))
}
func run() int {
args, optargs, err := getopt.GetOpt(
os.Args[1:],
"h:i:o:",
[]string{
"input=",
"output=",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
cmd.Usage(cmd.ErrorCodes["opts"])
}
if len(args) != 0 {
fmt.Fprintln(os.Stderr, "trailing args: %v", args)
cmd.Usage(cmd.ErrorCodes["opts"])
}
inputPath := ""
outputPath := ""
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
cmd.Usage(0)
case "-i", "--input":
inputPath = cmd.AssertFile(oa.Arg())
case "-o", "--output":
outputPath = cmd.AssertFile(oa.Arg())
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
cmd.Usage(cmd.ErrorCodes["opts"])
}
}
var input io.Reader
if inputPath != "" {
inputf, err := os.Open(inputPath)
if err != nil {
errors.Logf("ERROR", "could not open %v : %v", inputPath, err)
return 1
}
defer inputf.Close()
input = inputf
} else {
inputPath = "<stdin>"
input = os.Stdin
}
var output io.Writer
if outputPath != "" {
outputf, err := os.Create(outputPath)
if err != nil {
errors.Logf("ERROR", "could not open %v : %v", outputPath, err)
return 1
}
defer outputf.Close()
if strings.HasSuffix(outputPath, ".gz") {
z := gzip.NewWriter(outputf)
defer z.Close()
output = z
} else {
output = outputf
}
} else {
outputPath = "<stdout>"
output = os.Stdout
}
errors.Logf("INFO", "converting %v writing to %v", inputPath, outputPath)
err = convert(input, output)
if err != nil {
errors.Logf("ERROR", "error converting dot to veg %v", err)
return 1
}
return 0
}
func main() {
_, optargs, err := getopt.GetOpt(
os.Args[1:],
"hl:a:c:s:",
[]string{"help", "listen=", "assets", "clones=", "src=",
"private-ssl-key=", "ssl-cert="},
)
if err != nil {
fmt.Fprintln(os.Stderr, "error parsing command line flags", err)
Usage(ErrorCodes["opts"])
}
listen := "0.0.0.0:80"
assets := ""
clones := ""
source := ""
ssl_key := ""
ssl_cert := ""
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
os.Exit(0)
case "-l", "--listen":
listen = oa.Arg()
case "-a", "--assets":
assets, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "assets path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
case "-c", "--clones":
clones, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "clones path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
case "-s", "--src":
source, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "source path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
case "--private-ssl-key":
ssl_key, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "private-ssl-key path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
_, err = os.Stat(ssl_key)
if os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "private-ssl-key path does not exist. %v", ssl_key)
Usage(ErrorCodes["opts"])
} else if err != nil {
fmt.Fprintf(os.Stderr, "private-ssl-key path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
case "--ssl-cert":
log.Println("ssl-cert", oa.Arg())
ssl_cert, err = filepath.Abs(oa.Arg())
if err != nil {
fmt.Fprintf(os.Stderr, "ssl-cert path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
_, err = os.Stat(ssl_cert)
if os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "ssl-cert path does not exist. %v", ssl_cert)
Usage(ErrorCodes["opts"])
} else if err != nil {
fmt.Fprintf(os.Stderr, "ssl-cert path was bad: %v", err)
Usage(ErrorCodes["opts"])
}
default:
fmt.Fprintf(os.Stderr, "Unknown flag '%v'\n", oa.Opt())
Usage(ErrorCodes["opts"])
}
}
if assets == "" {
fmt.Fprintln(os.Stderr, "You must supply a path to the assets")
Usage(ErrorCodes["opts"])
}
if clones == "" {
fmt.Fprintln(os.Stderr, "You must supply a path to the clones")
Usage(ErrorCodes["opts"])
}
if source == "" {
fmt.Fprintln(os.Stderr, "You must supply a path to the source")
Usage(ErrorCodes["opts"])
}
if (ssl_key == "" && ssl_cert != "") || (ssl_key != "" && ssl_cert == "") {
fmt.Fprintln(os.Stderr, "To use ssl you must supply key and cert")
Usage(ErrorCodes["opts"])
}
handler := views.Routes(assets, clones, source)
server := &http.Server{
Addr: listen,
Handler: handler,
ReadTimeout: 1 * time.Second,
WriteTimeout: 1 * time.Second,
MaxHeaderBytes: http.DefaultMaxHeaderBytes,
TLSConfig: nil,
TLSNextProto: nil,
ConnState: nil,
ErrorLog: nil,
}
if ssl_key == "" {
err = server.ListenAndServe()
if err != nil {
log.Fatal(err)
}
} else {
log.Println(ssl_cert, ssl_key)
err = server.ListenAndServeTLS(ssl_cert, ssl_key)
if err != nil {
log.Panic(err)
}
}
}
func main() {
args, optargs, err := getopt.GetOpt(
os.Args[1:],
"hs:m:o:c:",
[]string{
"help",
"support=",
"cache=",
"min-vertices=",
"sample-size=",
"mem-profile=",
"cpu-profile=",
"output=",
"probabilities",
},
)
if err != nil {
fmt.Fprintln(os.Stderr, err)
Usage(ErrorCodes["opts"])
}
log.Printf("Number of goroutines = %v", runtime.NumGoroutine())
support := -1
minVertices := -1
sampleSize := -1
memProfile := ""
cpuProfile := ""
outputDir := ""
cache := ""
compute_prs := false
for _, oa := range optargs {
switch oa.Opt() {
case "-h", "--help":
Usage(0)
case "-o", "--output":
outputDir = EmptyDir(AssertDir(oa.Arg()))
case "-s", "--support":
support = ParseInt(oa.Arg())
case "-m", "--min-vertices":
minVertices = ParseInt(oa.Arg())
case "-c", "--cache":
cache = AssertDir(oa.Arg())
case "--probabilities":
compute_prs = true
case "--sample-size":
sampleSize = ParseInt(oa.Arg())
case "--mem-profile":
memProfile = AssertFile(oa.Arg())
case "--cpu-profile":
cpuProfile = AssertFile(oa.Arg())
}
}
if support < 1 {
fmt.Fprintf(os.Stderr, "You must supply a support greater than 0, you gave %v\n", support)
Usage(ErrorCodes["opts"])
}
if sampleSize < 1 {
fmt.Fprintf(os.Stderr, "You must supply a sample-size greater than 0, you gave %v\n", sampleSize)
Usage(ErrorCodes["opts"])
}
if outputDir == "" {
fmt.Fprintf(os.Stderr, "You must supply an output file (use -o)\n")
Usage(ErrorCodes["opts"])
}
if cache == "" {
fmt.Fprintln(os.Stderr, "you must supply a --cache=<dir>")
Usage(ErrorCodes["opts"])
}
if len(args) != 1 {
fmt.Fprintln(os.Stderr, "Expected a path to the graph file")
Usage(ErrorCodes["opts"])
}
getReader := func() (io.Reader, func()) { return Input(args[0]) }
if cpuProfile != "" {
f, err := os.Create(cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
err = pprof.StartCPUProfile(f)
if err != nil {
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
var memProfFile io.WriteCloser
if memProfile != "" {
f, err := os.Create(memProfile)
if err != nil {
log.Fatal(err)
}
memProfFile = f
defer f.Close()
}
nodePath := path.Join(outputDir, "node-attrs.bptree")
nodeBf, err := fmap.CreateBlockFile(nodePath)
if err != nil {
log.Fatal(err)
}
defer nodeBf.Close()
nodeAttrs, err := bptree.New(nodeBf, 4, -1)
if err != nil {
log.Fatal(err)
}
G, err := graph.LoadGraph(getReader, "", nodeAttrs, nil)
if err != nil {
log.Println("Error loading the graph")
log.Panic(err)
}
log.Print("Loaded graph, about to start mining")
sgCount := 0
sgMaker := func() store.SubGraphs {
name := fmt.Sprintf("subgraphs-%d.b+tree", sgCount)
sgCount++
path := path.Join(cache, name)
s := store.NewFs2BpTree(G, path)
// os.Remove(path)
// s, err := store.NewSqlite(G, path)
// if err != nil {
// log.Panic(err)
// }
return s
}
idxCount := 0
idxMaker := func() store.UniqueIndex {
name := fmt.Sprintf("unique-idx-%d.b+tree", idxCount)
idxCount++
path := path.Join(cache, name)
s := store.NewFs2UniqueIndex(G, path)
// os.Remove(path)
// s, err := store.NewSqlite(G, path)
// if err != nil {
// log.Panic(err)
// }
return s
}
setsCount := 0
setsMaker := func() store.SetsMap {
name := fmt.Sprintf("sets-%d.b+tree", setsCount)
setsCount++
path := path.Join(cache, name)
s := store.NewFs2Sets(path)
// os.Remove(path)
// s, err := store.NewSqlite(G, path)
// if err != nil {
// log.Panic(err)
// }
return s
}
// memFsMaker := func() store.SubGraphs {
// return store.AnonFs2BpTree(G)
// }
m := mine.RandomWalk(
G,
support,
minVertices,
sampleSize,
memProfFile,
sgMaker,
idxMaker,
setsMaker,
)
keys := list.NewSorted(10, false)
counts := hashtable.NewLinearHash()
for label := range m.Report {
key := types.ByteSlice(label)
count := 0
if counts.Has(key) {
c, err := counts.Get(key)
if err != nil {
log.Panic(err)
}
count = c.(int)
}
counts.Put(key, count+1)
keys.Add(key)
}
log.Println("Tries", m.Tries)
triesPath := path.Join(outputDir, "tries")
if f, e := os.Create(triesPath); e != nil {
log.Fatal(err)
} else {
fmt.Fprintln(f, m.Tries)
f.Close()
}
{
log.Println("Finished mining! Writing output...")
keyCh := make(chan []byte)
go func() {
for k, next := keys.Items()(); next != nil; k, next = next() {
keyCh <- []byte(k.(types.ByteSlice))
}
close(keyCh)
}()
writeMaximalPatterns(keyCh, m.AllEmbeddings, nodeAttrs, outputDir)
}
if !compute_prs {
log.Println("Done!")
return
}
log.Println("Finished writing patterns. Computing probabilities...")
count := 0
for k, next := keys.Items()(); next != nil; k, next = next() {
patDir := path.Join(outputDir, fmt.Sprintf("%d", count))
log.Println("-----------------------------------")
c, err := counts.Get(k)
if err != nil {
log.Fatal(err)
}
key := []byte(k.(types.ByteSlice))
dupCount := c.(int)
// if max.Count(key) < support {
// log.Println("wat not enough subgraphs", max.Count(key))
// continue
// }
if c, err := os.Create(path.Join(patDir, "duplicates")); err != nil {
log.Fatal(err)
} else {
fmt.Fprintln(c, dupCount)
c.Close()
}
for _, sg, next := m.AllEmbeddings.Find(key)(); next != nil; _, sg, next = next() {
vp, Q, R, u, err := m.PrMatrices(sg)
if err != nil {
log.Println(err)
errPath := path.Join(patDir, "error")
if f, e := os.Create(errPath); e != nil {
log.Fatal(err)
} else {
fmt.Fprintln(f, err)
f.Close()
}
} else {
bytes, err := json.Marshal(map[string]interface{}{
"Q": Q,
"R": R,
"u": u,
"startingPoints": vp,
})
if err != nil {
log.Fatal(err)
}
matPath := path.Join(patDir, "matrices.json")
if m, err := os.Create(matPath); err != nil {
log.Fatal(err)
} else {
_, err := m.Write(bytes)
if err != nil {
m.Close()
log.Fatal(err)
}
m.Close()
}
}
break
}
count++
}
log.Println("Done!")
}