func main() { flag.Parse() if len(flag.Args()) == 0 { flagutil.UsageError("missing kindex-file path") } else if len(flag.Args()) > 1 { flagutil.UsageErrorf("unknown arguments: %v", flag.Args()[1:]) } path := flag.Arg(0) idx, err := kindex.Open(context.Background(), path) if err != nil { log.Fatalf("Error reading %q: %v", path, err) } en := json.NewEncoder(os.Stdout) if *printFiles { if err := en.Encode(idx); err != nil { log.Fatalf("Error encoding JSON: %v", err) } } else { if err := en.Encode(idx.Proto); err != nil { log.Fatalf("Error encoding JSON compilation: %v", err) } } }
func main() { flag.Parse() if len(flag.Args()) != 0 { flagutil.UsageErrorf("unknown arguments: %v", flag.Args()) } written := make(map[[sha512.Size384]byte]struct{}) var skipped uint64 rd := delimited.NewReader(os.Stdin) wr := delimited.NewWriter(os.Stdout) for { rec, err := rd.Next() if err == io.EOF { break } else if err != nil { log.Fatal(err) } hash := sha512.Sum384(rec) if _, ok := written[hash]; ok { skipped++ continue } if err := wr.Put(rec); err != nil { log.Fatal(err) } written[hash] = struct{}{} } log.Printf("dedup_stream: skipped %d records", skipped) }
func main() { log.SetPrefix("write_entries: ") flag.Parse() if *numWorkers < 1 { flagutil.UsageErrorf("Invalid number of --workers %d (must be ≥ 1)", *numWorkers) } else if *batchSize < 1 { flagutil.UsageErrorf("Invalid --batch_size %d (must be ≥ 1)", *batchSize) } else if gs == nil { flagutil.UsageError("Missing --graphstore") } defer gsutil.LogClose(context.Background(), gs) gsutil.EnsureGracefulExit(gs) if *profCPU != "" { f, err := vfs.Create(context.Background(), *profCPU) if err != nil { log.Fatal(err) } defer f.Close() pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } writes := graphstore.BatchWrites(stream.ReadEntries(os.Stdin), *batchSize) var ( wg sync.WaitGroup numEntries uint64 ) wg.Add(*numWorkers) for i := 0; i < *numWorkers; i++ { go func() { defer wg.Done() num, err := writeEntries(context.Background(), gs, writes) if err != nil { log.Fatal(err) } atomic.AddUint64(&numEntries, num) }() } wg.Wait() log.Printf("Wrote %d entries", numEntries) }
func main() { flag.Parse() if len(flag.Args()) > 0 { flagutil.UsageErrorf("unknown arguments: %v", flag.Args()) } in := os.Stdin var entries <-chan *spb.Entry if *readJSON { entries = stream.ReadJSONEntries(in) } else { entries = stream.ReadEntries(in) } if *sortStream || *entrySets { entries = sortEntries(entries) } encoder := json.NewEncoder(os.Stdout) wr := delimited.NewWriter(os.Stdout) var set entrySet entryCount := 0 for entry := range entries { if *countOnly { entryCount++ } else if *entrySets { if compare.VNamesEqual(set.Source, entry.Source) || !compare.VNamesEqual(set.Target, entry.Target) || set.EdgeKind != entry.EdgeKind { if len(set.Properties) != 0 { failOnErr(encoder.Encode(set)) } set.Source = entry.Source set.EdgeKind = entry.EdgeKind set.Target = entry.Target set.Properties = make(map[string]string) } set.Properties[entry.FactName] = string(entry.FactValue) } else if *writeJSON { failOnErr(encoder.Encode(entry)) } else { rec, err := proto.Marshal(entry) failOnErr(err) failOnErr(wr.Put(rec)) } } if len(set.Properties) != 0 { failOnErr(encoder.Encode(set)) } if *countOnly { fmt.Println(entryCount) } }
func main() { flag.Parse() if flag.NArg() != 0 { flagutil.UsageErrorf("unknown arguments: %v", flag.Args()) } rd, err := delimited.NewUniqReader(delimited.NewReader(os.Stdin), int(cacheSize.Bytes())) if err != nil { log.Fatalf("Error creating UniqReader: %v", err) } wr := delimited.NewWriter(os.Stdout) if err := delimited.Copy(wr, rd); err != nil { log.Fatal(err) } log.Printf("dedup_stream: skipped %d records", rd.Skipped()) }
func main() { flag.Parse() if len(flag.Args()) > 0 { flagutil.UsageErrorf("unknown arguments: %v", flag.Args()) } in := bufio.NewReaderSize(os.Stdin, 2*4096) out := bufio.NewWriter(os.Stdout) var rd stream.EntryReader if *readJSON { rd = stream.NewJSONReader(in) } else { rd = stream.NewReader(in) } if *sortStream || *entrySets || *uniqEntries { var err error rd, err = sortEntries(rd) failOnErr(err) } if *uniqEntries { rd = dedupEntries(rd) } switch { case *countOnly: var count int failOnErr(rd(func(_ *spb.Entry) error { count++ return nil })) fmt.Println(count) case *entrySets: encoder := json.NewEncoder(out) var set entrySet failOnErr(rd(func(entry *spb.Entry) error { if !compare.VNamesEqual(set.Source, entry.Source) || !compare.VNamesEqual(set.Target, entry.Target) || set.EdgeKind != entry.EdgeKind { if len(set.Properties) != 0 { if err := encoder.Encode(set); err != nil { return err } } set.Source = entry.Source set.EdgeKind = entry.EdgeKind set.Target = entry.Target set.Properties = make(map[string]string) } set.Properties[entry.FactName] = string(entry.FactValue) return nil })) if len(set.Properties) != 0 { failOnErr(encoder.Encode(set)) } case *writeJSON: encoder := json.NewEncoder(out) failOnErr(rd(func(entry *spb.Entry) error { return encoder.Encode(entry) })) default: wr := delimited.NewWriter(out) failOnErr(rd(func(entry *spb.Entry) error { rec, err := proto.Marshal(entry) if err != nil { return err } return wr.Put(rec) })) } failOnErr(out.Flush()) }
func main() { flag.Parse() if flag.NArg() > 0 { flagutil.UsageErrorf("unknown non-flag argument(s): %v", flag.Args()) } else if *offset < 0 && (*lineNumber < 0 || *columnOffset < 0) { flagutil.UsageError("non-negative --offset (or --line and --column) required") } else if *signature == "" && *path == "" { flagutil.UsageError("must provide at least --path or --signature") } defer (*apiFlag).Close() xs, idx = *apiFlag, *apiFlag relPath := *path if *localRepoRoot != "NONE" { if _, err := os.Stat(relPath); err == nil { absPath, err := filepath.Abs(relPath) if err != nil { log.Fatal(err) } if *dirtyBuffer == "" { *dirtyBuffer = absPath } kytheRoot := *localRepoRoot if kytheRoot == "" { kytheRoot = findKytheRoot(filepath.Dir(absPath)) } if kytheRoot != "" { relPath, err = filepath.Rel(filepath.Join(kytheRoot, *root), absPath) if err != nil { log.Fatal(err) } } } } partialFile := &spb.VName{ Signature: *signature, Corpus: *corpus, Root: *root, Path: relPath, Language: *language, } reply, err := idx.Search(ctx, &spb.SearchRequest{ Partial: partialFile, Fact: fileFacts, }) if err != nil { log.Fatalf("Error locating file {%v}: %v", partialFile, err) } if len(reply.Ticket) == 0 { log.Fatalf("Could not locate file {%v}", partialFile) } else if len(reply.Ticket) > 1 { log.Fatalf("Ambiguous file {%v}; multiple results: %v", partialFile, reply.Ticket) } fileTicket := reply.Ticket[0] text := readDirtyBuffer(ctx) decor, err := xs.Decorations(ctx, &xpb.DecorationsRequest{ // TODO(schroederc): limit Location to a SPAN around *offset Location: &xpb.Location{Ticket: fileTicket}, References: true, SourceText: true, DirtyBuffer: text, Filter: []string{ schema.NodeKindFact, schema.SubkindFact, schema.AnchorLocFilter, // TODO(schroederc): remove once backwards-compatibility fix below is removed }, }) if err != nil { log.Fatal(err) } if text == nil { text = decor.SourceText } nodes := xrefs.NodesMap(decor.Node) // Normalize point within source text point := normalizedPoint(text) en := json.NewEncoder(os.Stdout) for _, ref := range decor.Reference { var start, end int if ref.AnchorStart == nil || ref.AnchorEnd == nil { // TODO(schroederc): remove this backwards-compatibility fix start, end = parseAnchorSpan(nodes[ref.SourceTicket]) } else { start, end = int(ref.AnchorStart.ByteOffset), int(ref.AnchorEnd.ByteOffset) } if start <= point && point < end { var r reference r.Span.Start = start r.Span.End = end r.Span.Text = string(text[start:end]) r.Kind = strings.TrimPrefix(ref.Kind, schema.EdgePrefix) r.Node.Ticket = ref.TargetTicket node := nodes[ref.TargetTicket] r.Node.Kind = string(node[schema.NodeKindFact]) r.Node.Subkind = string(node[schema.SubkindFact]) if eReply, err := xrefs.AllEdges(ctx, xs, &xpb.EdgesRequest{ Ticket: []string{ref.TargetTicket}, Kind: []string{schema.NamedEdge, schema.TypedEdge, definedAtEdge, definedBindingAtEdge}, }); err != nil { log.Printf("WARNING: error getting edges for %q: %v", ref.TargetTicket, err) } else { edges := xrefs.EdgesMap(eReply.EdgeSet)[ref.TargetTicket] for _, name := range edges[schema.NamedEdge] { if uri, err := kytheuri.Parse(name); err != nil { log.Printf("WARNING: named node ticket (%q) could not be parsed: %v", name, err) } else { r.Node.Names = append(r.Node.Names, uri.Signature) } } if typed := edges[schema.TypedEdge]; len(typed) > 0 { r.Node.Typed = typed[0] } if !*skipDefinitions { defs := edges[definedAtEdge] if len(defs) == 0 { defs = edges[definedBindingAtEdge] } for _, defAnchor := range defs { def, err := completeDefinition(defAnchor) if err != nil { log.Printf("WARNING: failed to complete definition for %q: %v", defAnchor, err) } else { r.Node.Definitions = append(r.Node.Definitions, def) } } } } if err := en.Encode(r); err != nil { log.Fatal(err) } } } }
func main() { flag.Parse() if flag.NArg() > 0 { flagutil.UsageErrorf("unknown non-flag argument(s): %v", flag.Args()) } else if *offset < 0 && (*lineNumber < 0 || *columnOffset < 0) { flagutil.UsageError("non-negative --offset (or --line and --column) required") } else if *path == "" { flagutil.UsageError("must provide --path") } defer (*apiFlag).Close() xs = *apiFlag relPath := *path if *localRepoRoot != "NONE" { if _, err := os.Stat(relPath); err == nil { absPath, err := filepath.Abs(relPath) if err != nil { log.Fatal(err) } if *dirtyBuffer == "" { *dirtyBuffer = absPath } kytheRoot := *localRepoRoot if kytheRoot == "" { kytheRoot = findKytheRoot(filepath.Dir(absPath)) } if kytheRoot != "" { relPath, err = filepath.Rel(filepath.Join(kytheRoot, *root), absPath) if err != nil { log.Fatal(err) } } } } fileTicket := (&kytheuri.URI{Corpus: *corpus, Root: *root, Path: relPath}).String() point := &xpb.Location_Point{ ByteOffset: int32(*offset), LineNumber: int32(*lineNumber), ColumnOffset: int32(*columnOffset), } dirtyBuffer := readDirtyBuffer(ctx) decor, err := xs.Decorations(ctx, &xpb.DecorationsRequest{ Location: &xpb.Location{ Ticket: fileTicket, Kind: xpb.Location_SPAN, Start: point, End: point, }, SpanKind: xpb.DecorationsRequest_AROUND_SPAN, References: true, SourceText: true, DirtyBuffer: dirtyBuffer, Filter: []string{ facts.NodeKind, facts.Subkind, }, }) if err != nil { log.Fatal(err) } nodes := xrefs.NodesMap(decor.Nodes) en := json.NewEncoder(os.Stdout) for _, ref := range decor.Reference { start, end := int(ref.AnchorStart.ByteOffset), int(ref.AnchorEnd.ByteOffset) var r reference r.Span.Start = start r.Span.End = end if len(dirtyBuffer) > 0 { r.Span.Text = string(dirtyBuffer[start:end]) } // TODO(schroederc): add option to get anchor text from DecorationsReply r.Kind = strings.TrimPrefix(ref.Kind, edges.Prefix) r.Node.Ticket = ref.TargetTicket node := nodes[ref.TargetTicket] r.Node.Kind = string(node[facts.NodeKind]) r.Node.Subkind = string(node[facts.Subkind]) // TODO(schroederc): use CrossReferences method if eReply, err := xrefs.AllEdges(ctx, xs, &gpb.EdgesRequest{ Ticket: []string{ref.TargetTicket}, Kind: []string{edges.Named, edges.Typed, definedAtEdge, definedBindingAtEdge}, }); err != nil { log.Printf("WARNING: error getting edges for %q: %v", ref.TargetTicket, err) } else { matching := xrefs.EdgesMap(eReply.EdgeSets)[ref.TargetTicket] for name := range matching[edges.Named] { if uri, err := kytheuri.Parse(name); err != nil { log.Printf("WARNING: named node ticket (%q) could not be parsed: %v", name, err) } else { r.Node.Names = append(r.Node.Names, uri.Signature) } } for typed := range matching[edges.Typed] { r.Node.Typed = typed break } if !*skipDefinitions { defs := matching[definedAtEdge] if len(defs) == 0 { defs = matching[definedBindingAtEdge] } for defAnchor := range defs { def, err := completeDefinition(defAnchor) if err != nil { log.Printf("WARNING: failed to complete definition for %q: %v", defAnchor, err) } else { r.Node.Definitions = append(r.Node.Definitions, def) } } } } if err := en.Encode(r); err != nil { log.Fatal(err) } } }
func main() { flag.Parse() if *servingTable == "" && gs == nil { flagutil.UsageError("missing either --serving_table or --graphstore") } else if *httpListeningAddr == "" && *grpcListeningAddr == "" && *tlsListeningAddr == "" { flagutil.UsageError("missing either --listen, --tls_listen, or --grpc_listen argument") } else if *servingTable != "" && gs != nil { flagutil.UsageError("--serving_table and --graphstore are mutually exclusive") } else if *tlsListeningAddr != "" && (*tlsCertFile == "" || *tlsKeyFile == "") { flagutil.UsageError("--tls_cert_file and --tls_key_file are required if given --tls_listen") } else if flag.NArg() > 0 { flagutil.UsageErrorf("unknown non-flag arguments given: %v", flag.Args()) } var ( xs xrefs.Service ft filetree.Service sr search.Service ) ctx := context.Background() if *servingTable != "" { db, err := leveldb.Open(*servingTable, nil) if err != nil { log.Fatalf("Error opening db at %q: %v", *servingTable, err) } defer db.Close() tbl := table.ProtoBatchParallel{&table.KVProto{db}} xs = xsrv.NewCombinedTable(tbl) ft = &ftsrv.Table{tbl} sr = &srchsrv.Table{&table.KVInverted{db}} } else { log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table") if f, ok := gs.(filetree.Service); ok { log.Printf("Using %T directly as filetree service", gs) ft = f } else { m := filetree.NewMap() if err := m.Populate(ctx, gs); err != nil { log.Fatalf("Error populating file tree from GraphStore: %v", err) } ft = m } if x, ok := gs.(xrefs.Service); ok { log.Printf("Using %T directly as xrefs service", gs) xs = x } else { if err := xstore.EnsureReverseEdges(ctx, gs); err != nil { log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err) } xs = xstore.NewGraphStoreService(gs) } if s, ok := gs.(search.Service); ok { log.Printf("Using %T directly as search service", gs) sr = s } } if sr == nil { log.Println("Search API not supported") } if *grpcListeningAddr != "" { srv := grpc.NewServer() xpb.RegisterXRefServiceServer(srv, xs) ftpb.RegisterFileTreeServiceServer(srv, ft) if sr != nil { spb.RegisterSearchServiceServer(srv, sr) } go startGRPC(srv) } if *httpListeningAddr != "" || *tlsListeningAddr != "" { apiMux := http.NewServeMux() http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if *httpAllowOrigin != "" { w.Header().Set("Access-Control-Allow-Origin", *httpAllowOrigin) } apiMux.ServeHTTP(w, r) }) xrefs.RegisterHTTPHandlers(ctx, xs, apiMux) filetree.RegisterHTTPHandlers(ctx, ft, apiMux) if sr != nil { search.RegisterHTTPHandlers(ctx, sr, apiMux) } if *publicResources != "" { log.Println("Serving public resources at", *publicResources) if s, err := os.Stat(*publicResources); err != nil { log.Fatalf("ERROR: could not get FileInfo for %q: %v", *publicResources, err) } else if !s.IsDir() { log.Fatalf("ERROR: %q is not a directory", *publicResources) } apiMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path))) }) } } if *httpListeningAddr != "" { go startHTTP() } if *tlsListeningAddr != "" { go startTLS() } select {} // block forever }
func main() { flag.Parse() if flag.NArg() == 0 { flagutil.UsageError("Missing path to LevelDB") } var protoValueType reflect.Type if *protoValue != "" { protoValueType = proto.MessageType(*protoValue) if protoValueType == nil { flagutil.UsageErrorf("could not understand protocol buffer type: %q", *protoValue) } } var en *json.Encoder if *emitJSON { en = json.NewEncoder(os.Stdout) } for _, path := range flag.Args() { func() { db, err := leveldb.Open(path, nil) if err != nil { log.Fatalf("Error opening %q: %v", path, err) } defer db.Close() it, err := db.ScanPrefix([]byte(*keyPrefix), nil) if err != nil { log.Fatalf("Error creating iterator for %q: v", path, err) } defer it.Close() for { key, val, err := it.Next() if err == io.EOF { break } else if err != nil { log.Fatalf("Error during scan of %q: %v", path, err) } var k, v interface{} if protoValueType == nil { if *stringKey { k = strconv.Quote(string(key)) } else { k = base64.StdEncoding.EncodeToString(key) } if *stringValue { v = strconv.Quote(string(val)) } else { v = base64.StdEncoding.EncodeToString(val) } } else { p := reflect.New(protoValueType.Elem()).Interface().(proto.Message) if err := proto.Unmarshal(val, p); err != nil { log.Fatalf("Error unmarshaling value to %q: %v", *protoValue, err) } k, v = string(key), p } if en == nil { fmt.Println(strings.NewReplacer( "@key@", fmt.Sprintf("%s", k), "@value@", fmt.Sprintf("%s", v), ).Replace(*lineFormat)) } else { en.Encode(keyValue{k, v}) } } }() } }