func main() {
flag.Parse()
if gs == nil && *entriesFile == "" {
flagutil.UsageError("missing --graphstore or --entries")
} else if gs != nil && *entriesFile != "" {
flagutil.UsageError("--graphstore and --entries are mutually exclusive")
} else if *tablePath == "" {
flagutil.UsageError("missing required --out flag")
}
db, err := leveldb.Open(*tablePath, nil)
if err != nil {
log.Fatal(err)
}
defer db.Close()
ctx := context.Background()
if err := profile.Start(ctx); err != nil {
log.Fatal(err)
}
defer profile.Stop()
var rd stream.EntryReader
if gs != nil {
rd = func(f func(e *spb.Entry) error) error {
defer gs.Close(ctx)
return gs.Scan(ctx, &spb.ScanRequest{}, f)
}
} else {
f, err := vfs.Open(ctx, *entriesFile)
if err != nil {
log.Fatalf("Error opening %q: %v", *entriesFile, err)
}
defer f.Close()
rd = stream.NewReader(f)
}
if err := pipeline.Run(ctx, rd, db, &pipeline.Options{
Verbose: *verbose,
MaxPageSize: *maxPageSize,
CompressShards: *compressShards,
MaxShardSize: *maxShardSize,
IOBufferSize: int(shardIOBufferSize.Bytes()),
}); err != nil {
log.Fatal("FATAL ERROR: ", err)
}
}
func main() {
flag.Parse()
if len(flag.Args()) > 0 {
flagutil.UsageErrorf("unknown arguments: %v", flag.Args())
}
in := bufio.NewReaderSize(os.Stdin, 2*4096)
out := bufio.NewWriter(os.Stdout)
var rd stream.EntryReader
if *readJSON {
rd = stream.NewJSONReader(in)
} else {
rd = stream.NewReader(in)
}
if *sortStream || *entrySets || *uniqEntries {
var err error
rd, err = sortEntries(rd)
failOnErr(err)
}
if *uniqEntries {
rd = dedupEntries(rd)
}
switch {
case *countOnly:
var count int
failOnErr(rd(func(_ *spb.Entry) error {
count++
return nil
}))
fmt.Println(count)
case *entrySets:
encoder := json.NewEncoder(out)
var set entrySet
failOnErr(rd(func(entry *spb.Entry) error {
if !compare.VNamesEqual(set.Source, entry.Source) || !compare.VNamesEqual(set.Target, entry.Target) || set.EdgeKind != entry.EdgeKind {
if len(set.Properties) != 0 {
if err := encoder.Encode(set); err != nil {
return err
}
}
set.Source = entry.Source
set.EdgeKind = entry.EdgeKind
set.Target = entry.Target
set.Properties = make(map[string]string)
}
set.Properties[entry.FactName] = string(entry.FactValue)
return nil
}))
if len(set.Properties) != 0 {
failOnErr(encoder.Encode(set))
}
case *writeJSON:
encoder := json.NewEncoder(out)
failOnErr(rd(func(entry *spb.Entry) error {
return encoder.Encode(entry)
}))
default:
wr := delimited.NewWriter(out)
failOnErr(rd(func(entry *spb.Entry) error {
rec, err := proto.Marshal(entry)
if err != nil {
return err
}
return wr.Put(rec)
}))
}
failOnErr(out.Flush())
}