Example #1
0
func main() {
	flag.Parse()
	if gs == nil {
		flagutil.UsageError("missing required --graphstore flag")
	} else if *tablePath == "" {
		flagutil.UsageError("missing required --out flag")
	}

	db, err := leveldb.Open(*tablePath, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	ctx := context.Background()

	if err := profile.Start(ctx); err != nil {
		log.Fatal(err)
	}
	defer profile.Stop()

	if err := pipeline.Run(ctx, gs, db, &pipeline.Options{
		MaxEdgePageSize: *maxEdgePageSize,
	}); err != nil {
		log.Fatal("FATAL ERROR: ", err)
	}
}
Example #2
0
func main() {
	flag.Parse()
	if *servingTable == "" {
		flagutil.UsageError("missing --serving_table")
	} else if *outPath == "" {
		flagutil.UsageError("missing --out")
	}

	ctx := context.Background()
	db, err := leveldb.Open(*servingTable, nil)
	if err != nil {
		log.Fatalf("Error opening db at %q: %v", *servingTable, err)
	}
	defer db.Close()
	tbl := &table.KVProto{db}
	esr := &esrchsrv.Table{&table.KVInverted{db}}

	ix := index.Create(*outPath)
	for _, ticket := range filesInTable(ctx, esr) {
		var fd srvpb.FileDecorations
		if err := tbl.Lookup(ctx, xsrv.DecorationsKey(ticket), &fd); err != nil {
			log.Fatalf("Error looking up decoration for %q: %v", ticket, err)
		}
		ix.Add(ticket, bytes.NewReader(fd.SourceText))
	}
	ix.Flush()
}
Example #3
0
func main() {
	flag.Parse()
	if gs == nil && *entriesFile == "" {
		flagutil.UsageError("missing --graphstore or --entries")
	} else if gs != nil && *entriesFile != "" {
		flagutil.UsageError("--graphstore and --entries are mutually exclusive")
	} else if *tablePath == "" {
		flagutil.UsageError("missing required --out flag")
	}

	db, err := leveldb.Open(*tablePath, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	ctx := context.Background()

	if err := profile.Start(ctx); err != nil {
		log.Fatal(err)
	}
	defer profile.Stop()

	var rd stream.EntryReader
	if gs != nil {
		rd = func(f func(e *spb.Entry) error) error {
			defer gs.Close(ctx)
			return gs.Scan(ctx, &spb.ScanRequest{}, f)
		}
	} else {
		f, err := vfs.Open(ctx, *entriesFile)
		if err != nil {
			log.Fatalf("Error opening %q: %v", *entriesFile, err)
		}
		defer f.Close()
		rd = stream.NewReader(f)
	}

	if err := pipeline.Run(ctx, rd, db, &pipeline.Options{
		Verbose:        *verbose,
		MaxPageSize:    *maxPageSize,
		CompressShards: *compressShards,
		MaxShardSize:   *maxShardSize,
		IOBufferSize:   int(shardIOBufferSize.Bytes()),
	}); err != nil {
		log.Fatal("FATAL ERROR: ", err)
	}
}
Example #4
0
func main() {
	flag.Parse()

	// done is sent a value when the analyzer should exit
	done := make(chan struct{}, 1)
	defer func() { done <- struct{}{} }()

	analyzerBin, analyzerArgs, compilations := parseAnalyzerCommand()
	if len(compilations) == 0 {
		flagutil.UsageError("Missing kindex-file paths")
	}

	cmd := exec.Command(analyzerBin, analyzerArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	var proc *os.Process
	if err := process.StartAsync(cmd, &process.Callbacks{
		OnStart: func(p *os.Process) {
			log.Printf("Starting analyzer subprocess: %s", strings.Join(cmd.Args, " "))
			proc = p
		},
		OnExit: func(state *os.ProcessState, err error) {
			select {
			case <-done:
			default:
				log.Fatalf("Analyzer subprocess exited unexpectedly (state:%v; error:%v)", state, err)
			}
		},
	}); err != nil {
		log.Fatalf("Error starting analyzer: %v", err)
	}

	addr := fmt.Sprintf("localhost:%d", *analyzerPort)
	conn, err := grpc.Dial(addr, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("Error dialing analyzer %q: %v", addr, err)
	}
	defer conn.Close()

	queue := local.NewKIndexQueue(compilations)
	fdsAddr := launchFileDataService(queue)

	wr := delimited.NewWriter(os.Stdout)

	driver := &driver.Driver{
		Analyzer: &remote.Analyzer{aspb.NewCompilationAnalyzerClient(conn)},
		Output:   func(_ context.Context, out *apb.AnalysisOutput) error { return wr.Put(out.Value) },

		FileDataService: fdsAddr,
		Compilations:    queue,
	}

	if err := driver.Run(context.Background()); err != nil {
		log.Fatal(err)
	}

	if err := proc.Signal(os.Interrupt); err != nil {
		log.Fatalf("Failed to send interrupt to analyzer: %v", err)
	}
}
Example #5
0
func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		flagutil.UsageError("missing kindex-file path")
	} else if len(flag.Args()) > 1 {
		flagutil.UsageErrorf("unknown arguments: %v", flag.Args()[1:])
	}

	path := flag.Arg(0)
	idx, err := kindex.Open(context.Background(), path)
	if err != nil {
		log.Fatalf("Error reading %q: %v", path, err)
	}

	en := json.NewEncoder(os.Stdout)
	if *printFiles {
		if err := en.Encode(idx); err != nil {
			log.Fatalf("Error encoding JSON: %v", err)
		}
	} else {
		if err := en.Encode(idx.Proto); err != nil {
			log.Fatalf("Error encoding JSON compilation: %v", err)
		}
	}
}
Example #6
0
func main() {
	flag.Parse()
	if gs == nil {
		flagutil.UsageError("missing required --graphstore flag")
	} else if *tablePath == "" {
		flagutil.UsageError("missing required --out flag")
	}

	db, err := leveldb.Open(*tablePath, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	if err := pipeline.Run(context.Background(), gs, db); err != nil {
		log.Fatal(err)
	}
}
Example #7
0
func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		flagutil.UsageError("not given any files")
	}

	xs := xrefs.WebClient(*remoteAPI)

	for _, file := range flag.Args() {
		ticket := (&kytheuri.URI{Corpus: *corpus, Path: file}).String()
		decor, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
			Location:   &xpb.Location{Ticket: ticket},
			SourceText: true,
			References: true,
		})
		if err != nil {
			log.Fatalf("Failed to get decorations for file %q", file)
		}

		nodes := xrefs.NodesMap(decor.Nodes)
		emitted := stringset.New()

		for _, r := range decor.Reference {
			if r.Kind != schema.DefinesBindingEdge || emitted.Contains(r.TargetTicket) {
				continue
			}

			ident := string(nodes[r.TargetTicket][identifierFact])
			if ident == "" {
				continue
			}

			offset, err := strconv.Atoi(string(nodes[r.SourceTicket][schema.AnchorStartFact]))
			if err != nil {
				log.Printf("Invalid start offset for anchor %q", r.SourceTicket)
				continue
			}

			fields, err := getTagFields(xs, r.TargetTicket)
			if err != nil {
				log.Printf("Failed to get tagfields for %q: %v", r.TargetTicket, err)
			}

			fmt.Printf("%s\t%s\t%d;\"\t%s\n",
				ident, file, offsetLine(decor.SourceText, offset), strings.Join(fields, "\t"))
			emitted.Add(r.TargetTicket)
		}
	}
}
Example #8
0
func main() {
	log.SetPrefix("write_entries: ")

	flag.Parse()
	if *numWorkers < 1 {
		flagutil.UsageErrorf("Invalid number of --workers %d (must be ≥ 1)", *numWorkers)
	} else if *batchSize < 1 {
		flagutil.UsageErrorf("Invalid --batch_size %d (must be ≥ 1)", *batchSize)
	} else if gs == nil {
		flagutil.UsageError("Missing --graphstore")
	}

	defer gsutil.LogClose(context.Background(), gs)
	gsutil.EnsureGracefulExit(gs)

	if *profCPU != "" {
		f, err := vfs.Create(context.Background(), *profCPU)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	writes := graphstore.BatchWrites(stream.ReadEntries(os.Stdin), *batchSize)

	var (
		wg         sync.WaitGroup
		numEntries uint64
	)
	wg.Add(*numWorkers)
	for i := 0; i < *numWorkers; i++ {
		go func() {
			defer wg.Done()
			num, err := writeEntries(context.Background(), gs, writes)
			if err != nil {
				log.Fatal(err)
			}

			atomic.AddUint64(&numEntries, num)
		}()
	}
	wg.Wait()

	log.Printf("Wrote %d entries", numEntries)
}
Example #9
0
func main() {
	flag.Parse()
	if *offset < 0 {
		flagutil.UsageError("non-negative --offset required")
	} else if *signature == "" && *path == "" {
		flagutil.UsageError("must provide at least --path or --signature")
	}

	if strings.HasPrefix(*remoteAPI, "http://") || strings.HasPrefix(*remoteAPI, "https://") {
		xs = xrefs.WebClient(*remoteAPI)
		idx = search.WebClient(*remoteAPI)
	} else {
		conn, err := grpc.Dial(*remoteAPI)
		if err != nil {
			log.Fatalf("Error connecting to remote API %q: %v", *remoteAPI, err)
		}
		defer conn.Close()
		xs = xrefs.GRPC(xpb.NewXRefServiceClient(conn))
		idx = search.GRPC(spb.NewSearchServiceClient(conn))
	}

	relPath := *path
	if !*ignoreLocalRepo {
		if _, err := os.Stat(relPath); err == nil {
			absPath, err := filepath.Abs(relPath)
			if err != nil {
				log.Fatal(err)
			}
			kytheRoot := findKytheRoot(filepath.Dir(absPath))
			if kytheRoot != "" {
				relPath, err = filepath.Rel(filepath.Join(kytheRoot, *root), absPath)
				if err != nil {
					log.Fatal(err)
				}
			}
		}
	}
	partialFile := &spb.VName{
		Signature: *signature,
		Corpus:    *corpus,
		Root:      *root,
		Path:      relPath,
		Language:  *language,
	}
	reply, err := idx.Search(ctx, &spb.SearchRequest{
		Partial: partialFile,
		Fact:    fileFacts,
	})
	if err != nil {
		log.Fatalf("Error locating file {%v}: %v", partialFile, err)
	}
	if len(reply.Ticket) == 0 {
		log.Fatalf("Could not locate file {%v}", partialFile)
	} else if len(reply.Ticket) > 1 {
		log.Fatalf("Ambiguous file {%v}; multiple results: %v", partialFile, reply.Ticket)
	}

	fileTicket := reply.Ticket[0]
	decor, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
		// TODO(schroederc): limit Location to a SPAN around *offset
		Location:    &xpb.Location{Ticket: fileTicket},
		References:  true,
		SourceText:  true,
		DirtyBuffer: readDirtyBuffer(ctx),
	})
	if err != nil {
		log.Fatal(err)
	}
	nodes := xrefs.NodesMap(decor.Node)

	en := json.NewEncoder(os.Stdout)
	for _, ref := range decor.Reference {
		start, end := parseAnchorSpan(nodes[ref.SourceTicket])

		if start <= *offset && *offset < end {
			var r reference
			r.Span.Start = start
			r.Span.End = end
			r.Span.Text = string(decor.SourceText[start:end])
			r.Kind = strings.TrimPrefix(ref.Kind, schema.EdgePrefix)
			r.Node.Ticket = ref.TargetTicket

			node := nodes[ref.TargetTicket]
			r.Node.Kind = string(node[schema.NodeKindFact])
			r.Node.Subkind = string(node[schema.SubkindFact])

			if eReply, err := xs.Edges(ctx, &xpb.EdgesRequest{
				Ticket: []string{ref.TargetTicket},
				Kind:   []string{schema.NamedEdge, definedAtEdge},
			}); err != nil {
				log.Printf("WARNING: error getting edges for %q: %v", ref.TargetTicket, err)
			} else {
				edges := xrefs.EdgesMap(eReply.EdgeSet)[ref.TargetTicket]
				for _, name := range edges[schema.NamedEdge] {
					if uri, err := kytheuri.Parse(name); err != nil {
						log.Printf("WARNING: named node ticket (%q) could not be parsed: %v", name, err)
					} else {
						r.Node.Names = append(r.Node.Names, uri.Signature)
					}
				}

				if !*skipDefinitions {
					for _, defAnchor := range edges[definedAtEdge] {
						def, err := completeDefinition(defAnchor)
						if err != nil {
							log.Printf("WARNING: failed to complete definition for %q: %v", defAnchor, err)
						} else {
							r.Node.Definitions = append(r.Node.Definitions, def)
						}
					}
				}
			}

			if err := en.Encode(r); err != nil {
				log.Fatal(err)
			}
		}
	}
}
Example #10
0
func main() {
	flag.Parse()
	if *servingTable == "" && gs == nil {
		flagutil.UsageError("missing either --serving_table or --graphstore")
	} else if *httpListeningAddr == "" && *grpcListeningAddr == "" {
		flagutil.UsageError("missing either --listen or --grpc_listen argument")
	} else if *servingTable != "" && gs != nil {
		flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
	}

	var (
		xs xrefs.Service
		ft filetree.Service
		sr search.Service
	)

	ctx := context.Background()
	if *servingTable != "" {
		db, err := leveldb.Open(*servingTable, nil)
		if err != nil {
			log.Fatalf("Error opening db at %q: %v", *servingTable, err)
		}
		defer db.Close()
		tbl := table.ProtoBatchParallel{&table.KVProto{db}}
		xs = xsrv.NewCombinedTable(tbl)
		ft = &ftsrv.Table{tbl}
		sr = &srchsrv.Table{&table.KVInverted{db}}
	} else {
		log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
		if f, ok := gs.(filetree.Service); ok {
			log.Printf("Using %T directly as filetree service", gs)
			ft = f
		} else {
			m := filetree.NewMap()
			if err := m.Populate(ctx, gs); err != nil {
				log.Fatalf("Error populating file tree from GraphStore: %v", err)
			}
			ft = m
		}

		if x, ok := gs.(xrefs.Service); ok {
			log.Printf("Using %T directly as xrefs service", gs)
			xs = x
		} else {
			if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
				log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
			}
			xs = xstore.NewGraphStoreService(gs)
		}

		if s, ok := gs.(search.Service); ok {
			log.Printf("Using %T directly as search service", gs)
			sr = s
		}
	}

	if sr == nil {
		log.Println("Search API not supported")
	}

	if *grpcListeningAddr != "" {
		srv := grpc.NewServer()
		xpb.RegisterXRefServiceServer(srv, xs)
		ftpb.RegisterFileTreeServiceServer(srv, ft)
		if sr != nil {
			spb.RegisterSearchServiceServer(srv, sr)
		}
		go startGRPC(srv)
	}

	if *httpListeningAddr != "" {
		xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux)
		filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux)
		if sr != nil {
			search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux)
		}
		go startHTTP()
	}

	select {} // block forever
}
Example #11
0
func main() {
	flag.Parse()
	if flag.NArg() > 0 {
		flagutil.UsageErrorf("unknown non-flag argument(s): %v", flag.Args())
	} else if *offset < 0 && (*lineNumber < 0 || *columnOffset < 0) {
		flagutil.UsageError("non-negative --offset (or --line and --column) required")
	} else if *signature == "" && *path == "" {
		flagutil.UsageError("must provide at least --path or --signature")
	}

	defer (*apiFlag).Close()
	xs, idx = *apiFlag, *apiFlag

	relPath := *path
	if *localRepoRoot != "NONE" {
		if _, err := os.Stat(relPath); err == nil {
			absPath, err := filepath.Abs(relPath)
			if err != nil {
				log.Fatal(err)
			}
			if *dirtyBuffer == "" {
				*dirtyBuffer = absPath
			}

			kytheRoot := *localRepoRoot
			if kytheRoot == "" {
				kytheRoot = findKytheRoot(filepath.Dir(absPath))
			}
			if kytheRoot != "" {
				relPath, err = filepath.Rel(filepath.Join(kytheRoot, *root), absPath)
				if err != nil {
					log.Fatal(err)
				}
			}
		}
	}
	partialFile := &spb.VName{
		Signature: *signature,
		Corpus:    *corpus,
		Root:      *root,
		Path:      relPath,
		Language:  *language,
	}
	reply, err := idx.Search(ctx, &spb.SearchRequest{
		Partial: partialFile,
		Fact:    fileFacts,
	})
	if err != nil {
		log.Fatalf("Error locating file {%v}: %v", partialFile, err)
	}
	if len(reply.Ticket) == 0 {
		log.Fatalf("Could not locate file {%v}", partialFile)
	} else if len(reply.Ticket) > 1 {
		log.Fatalf("Ambiguous file {%v}; multiple results: %v", partialFile, reply.Ticket)
	}

	fileTicket := reply.Ticket[0]
	text := readDirtyBuffer(ctx)
	decor, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
		// TODO(schroederc): limit Location to a SPAN around *offset
		Location:    &xpb.Location{Ticket: fileTicket},
		References:  true,
		SourceText:  true,
		DirtyBuffer: text,
		Filter: []string{
			schema.NodeKindFact,
			schema.SubkindFact,
			schema.AnchorLocFilter, // TODO(schroederc): remove once backwards-compatibility fix below is removed
		},
	})
	if err != nil {
		log.Fatal(err)
	}
	if text == nil {
		text = decor.SourceText
	}
	nodes := xrefs.NodesMap(decor.Node)

	// Normalize point within source text
	point := normalizedPoint(text)

	en := json.NewEncoder(os.Stdout)
	for _, ref := range decor.Reference {
		var start, end int
		if ref.AnchorStart == nil || ref.AnchorEnd == nil {
			// TODO(schroederc): remove this backwards-compatibility fix
			start, end = parseAnchorSpan(nodes[ref.SourceTicket])
		} else {
			start, end = int(ref.AnchorStart.ByteOffset), int(ref.AnchorEnd.ByteOffset)
		}

		if start <= point && point < end {
			var r reference
			r.Span.Start = start
			r.Span.End = end
			r.Span.Text = string(text[start:end])
			r.Kind = strings.TrimPrefix(ref.Kind, schema.EdgePrefix)
			r.Node.Ticket = ref.TargetTicket

			node := nodes[ref.TargetTicket]
			r.Node.Kind = string(node[schema.NodeKindFact])
			r.Node.Subkind = string(node[schema.SubkindFact])

			if eReply, err := xrefs.AllEdges(ctx, xs, &xpb.EdgesRequest{
				Ticket: []string{ref.TargetTicket},
				Kind:   []string{schema.NamedEdge, schema.TypedEdge, definedAtEdge, definedBindingAtEdge},
			}); err != nil {
				log.Printf("WARNING: error getting edges for %q: %v", ref.TargetTicket, err)
			} else {
				edges := xrefs.EdgesMap(eReply.EdgeSet)[ref.TargetTicket]
				for _, name := range edges[schema.NamedEdge] {
					if uri, err := kytheuri.Parse(name); err != nil {
						log.Printf("WARNING: named node ticket (%q) could not be parsed: %v", name, err)
					} else {
						r.Node.Names = append(r.Node.Names, uri.Signature)
					}
				}

				if typed := edges[schema.TypedEdge]; len(typed) > 0 {
					r.Node.Typed = typed[0]
				}

				if !*skipDefinitions {
					defs := edges[definedAtEdge]
					if len(defs) == 0 {
						defs = edges[definedBindingAtEdge]
					}
					for _, defAnchor := range defs {
						def, err := completeDefinition(defAnchor)
						if err != nil {
							log.Printf("WARNING: failed to complete definition for %q: %v", defAnchor, err)
						} else {
							r.Node.Definitions = append(r.Node.Definitions, def)
						}
					}
				}
			}

			if err := en.Encode(r); err != nil {
				log.Fatal(err)
			}
		}
	}
}
Example #12
0
func main() {
	flag.Parse()
	if flag.NArg() > 0 {
		flagutil.UsageErrorf("unknown non-flag argument(s): %v", flag.Args())
	} else if *offset < 0 && (*lineNumber < 0 || *columnOffset < 0) {
		flagutil.UsageError("non-negative --offset (or --line and --column) required")
	} else if *path == "" {
		flagutil.UsageError("must provide --path")
	}

	defer (*apiFlag).Close()
	xs = *apiFlag

	relPath := *path
	if *localRepoRoot != "NONE" {
		if _, err := os.Stat(relPath); err == nil {
			absPath, err := filepath.Abs(relPath)
			if err != nil {
				log.Fatal(err)
			}
			if *dirtyBuffer == "" {
				*dirtyBuffer = absPath
			}

			kytheRoot := *localRepoRoot
			if kytheRoot == "" {
				kytheRoot = findKytheRoot(filepath.Dir(absPath))
			}
			if kytheRoot != "" {
				relPath, err = filepath.Rel(filepath.Join(kytheRoot, *root), absPath)
				if err != nil {
					log.Fatal(err)
				}
			}
		}
	}

	fileTicket := (&kytheuri.URI{Corpus: *corpus, Root: *root, Path: relPath}).String()
	point := &xpb.Location_Point{
		ByteOffset:   int32(*offset),
		LineNumber:   int32(*lineNumber),
		ColumnOffset: int32(*columnOffset),
	}
	dirtyBuffer := readDirtyBuffer(ctx)
	decor, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
		Location: &xpb.Location{
			Ticket: fileTicket,
			Kind:   xpb.Location_SPAN,
			Start:  point,
			End:    point,
		},
		SpanKind:    xpb.DecorationsRequest_AROUND_SPAN,
		References:  true,
		SourceText:  true,
		DirtyBuffer: dirtyBuffer,
		Filter: []string{
			facts.NodeKind,
			facts.Subkind,
		},
	})
	if err != nil {
		log.Fatal(err)
	}
	nodes := xrefs.NodesMap(decor.Nodes)

	en := json.NewEncoder(os.Stdout)
	for _, ref := range decor.Reference {
		start, end := int(ref.AnchorStart.ByteOffset), int(ref.AnchorEnd.ByteOffset)

		var r reference
		r.Span.Start = start
		r.Span.End = end
		if len(dirtyBuffer) > 0 {
			r.Span.Text = string(dirtyBuffer[start:end])
		} // TODO(schroederc): add option to get anchor text from DecorationsReply
		r.Kind = strings.TrimPrefix(ref.Kind, edges.Prefix)
		r.Node.Ticket = ref.TargetTicket

		node := nodes[ref.TargetTicket]
		r.Node.Kind = string(node[facts.NodeKind])
		r.Node.Subkind = string(node[facts.Subkind])

		// TODO(schroederc): use CrossReferences method
		if eReply, err := xrefs.AllEdges(ctx, xs, &gpb.EdgesRequest{
			Ticket: []string{ref.TargetTicket},
			Kind:   []string{edges.Named, edges.Typed, definedAtEdge, definedBindingAtEdge},
		}); err != nil {
			log.Printf("WARNING: error getting edges for %q: %v", ref.TargetTicket, err)
		} else {
			matching := xrefs.EdgesMap(eReply.EdgeSets)[ref.TargetTicket]
			for name := range matching[edges.Named] {
				if uri, err := kytheuri.Parse(name); err != nil {
					log.Printf("WARNING: named node ticket (%q) could not be parsed: %v", name, err)
				} else {
					r.Node.Names = append(r.Node.Names, uri.Signature)
				}
			}

			for typed := range matching[edges.Typed] {
				r.Node.Typed = typed
				break
			}

			if !*skipDefinitions {
				defs := matching[definedAtEdge]
				if len(defs) == 0 {
					defs = matching[definedBindingAtEdge]
				}
				for defAnchor := range defs {
					def, err := completeDefinition(defAnchor)
					if err != nil {
						log.Printf("WARNING: failed to complete definition for %q: %v", defAnchor, err)
					} else {
						r.Node.Definitions = append(r.Node.Definitions, def)
					}
				}
			}
		}

		if err := en.Encode(r); err != nil {
			log.Fatal(err)
		}
	}
}
Example #13
0
File: ktags.go Project: jwatt/kythe
func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		flagutil.UsageError("not given any files")
	}

	xs := xrefs.WebClient(*remoteAPI)
	idx := search.WebClient(*remoteAPI)

	for _, file := range flag.Args() {
		results, err := idx.Search(ctx, &spb.SearchRequest{
			Partial: &spb.VName{Path: file},
			Fact: []*spb.SearchRequest_Fact{{
				Name:  schema.NodeKindFact,
				Value: []byte(schema.FileKind),
			}},
		})
		if err != nil {
			log.Fatalf("Error searching for ticket of file %q", file)
		} else if len(results.Ticket) == 0 {
			log.Printf("Could not find ticket for file %q", file)
			continue
		} else if len(results.Ticket) != 1 {
			log.Printf("Multiple tickets found for file %q; choosing first from %v", file, results.Ticket)
		}

		ticket := results.Ticket[0]
		decor, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
			Location:   &xpb.Location{Ticket: ticket},
			SourceText: true,
			References: true,
		})
		if err != nil {
			log.Fatalf("Failed to get decorations for file %q", file)
		}

		nodes := xrefs.NodesMap(decor.Node)
		emitted := stringset.New()

		for _, r := range decor.Reference {
			if r.Kind != schema.DefinesBindingEdge || emitted.Contains(r.TargetTicket) {
				continue
			}

			ident := string(nodes[r.TargetTicket][identifierFact])
			if ident == "" {
				continue
			}

			offset, err := strconv.Atoi(string(nodes[r.SourceTicket][schema.AnchorStartFact]))
			if err != nil {
				log.Printf("Invalid start offset for anchor %q", r.SourceTicket)
				continue
			}

			fields, err := getTagFields(xs, r.TargetTicket)
			if err != nil {
				log.Printf("Failed to get tagfields for %q: %v", r.TargetTicket, err)
			}

			fmt.Printf("%s\t%s\t%d;\"\t%s\n",
				ident, file, offsetLine(decor.SourceText, offset), strings.Join(fields, "\t"))
			emitted.Add(r.TargetTicket)
		}
	}
}
Example #14
0
func main() {
	flag.Parse()
	if *servingTable == "" && gs == nil {
		flagutil.UsageError("missing either --serving_table or --graphstore")
	} else if *httpListeningAddr == "" && *grpcListeningAddr == "" && *tlsListeningAddr == "" {
		flagutil.UsageError("missing either --listen, --tls_listen, or --grpc_listen argument")
	} else if *servingTable != "" && gs != nil {
		flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
	} else if *tlsListeningAddr != "" && (*tlsCertFile == "" || *tlsKeyFile == "") {
		flagutil.UsageError("--tls_cert_file and --tls_key_file are required if given --tls_listen")
	} else if flag.NArg() > 0 {
		flagutil.UsageErrorf("unknown non-flag arguments given: %v", flag.Args())
	}

	var (
		xs xrefs.Service
		ft filetree.Service
		sr search.Service
	)

	ctx := context.Background()
	if *servingTable != "" {
		db, err := leveldb.Open(*servingTable, nil)
		if err != nil {
			log.Fatalf("Error opening db at %q: %v", *servingTable, err)
		}
		defer db.Close()
		tbl := table.ProtoBatchParallel{&table.KVProto{db}}
		xs = xsrv.NewCombinedTable(tbl)
		ft = &ftsrv.Table{tbl}
		sr = &srchsrv.Table{&table.KVInverted{db}}
	} else {
		log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
		if f, ok := gs.(filetree.Service); ok {
			log.Printf("Using %T directly as filetree service", gs)
			ft = f
		} else {
			m := filetree.NewMap()
			if err := m.Populate(ctx, gs); err != nil {
				log.Fatalf("Error populating file tree from GraphStore: %v", err)
			}
			ft = m
		}

		if x, ok := gs.(xrefs.Service); ok {
			log.Printf("Using %T directly as xrefs service", gs)
			xs = x
		} else {
			if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
				log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
			}
			xs = xstore.NewGraphStoreService(gs)
		}

		if s, ok := gs.(search.Service); ok {
			log.Printf("Using %T directly as search service", gs)
			sr = s
		}
	}

	if sr == nil {
		log.Println("Search API not supported")
	}

	if *grpcListeningAddr != "" {
		srv := grpc.NewServer()
		xpb.RegisterXRefServiceServer(srv, xs)
		ftpb.RegisterFileTreeServiceServer(srv, ft)
		if sr != nil {
			spb.RegisterSearchServiceServer(srv, sr)
		}
		go startGRPC(srv)
	}

	if *httpListeningAddr != "" || *tlsListeningAddr != "" {
		apiMux := http.NewServeMux()
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			if *httpAllowOrigin != "" {
				w.Header().Set("Access-Control-Allow-Origin", *httpAllowOrigin)
			}
			apiMux.ServeHTTP(w, r)
		})

		xrefs.RegisterHTTPHandlers(ctx, xs, apiMux)
		filetree.RegisterHTTPHandlers(ctx, ft, apiMux)
		if sr != nil {
			search.RegisterHTTPHandlers(ctx, sr, apiMux)
		}
		if *publicResources != "" {
			log.Println("Serving public resources at", *publicResources)
			if s, err := os.Stat(*publicResources); err != nil {
				log.Fatalf("ERROR: could not get FileInfo for %q: %v", *publicResources, err)
			} else if !s.IsDir() {
				log.Fatalf("ERROR: %q is not a directory", *publicResources)
			}
			apiMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
				http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path)))
			})
		}
	}
	if *httpListeningAddr != "" {
		go startHTTP()
	}
	if *tlsListeningAddr != "" {
		go startTLS()
	}

	select {} // block forever
}
Example #15
0
func main() {
	flag.Parse()
	if gs == nil {
		flagutil.UsageError("missing --graphstore")
	} else if *shardsToFiles != "" && *shards <= 0 {
		flagutil.UsageError("--sharded_file and --shards must be given together")
	} else if *shards > 0 && len(flag.Args()) > 0 {
		flagutil.UsageError("--shards and giving tickets for reads are mutually exclusive")
	}

	ctx := context.Background()

	wr := delimited.NewWriter(os.Stdout)
	var total int64
	if *shards <= 0 {
		entryFunc := func(entry *spb.Entry) error {
			if *count {
				total++
				return nil
			}
			return wr.PutProto(entry)
		}
		if len(flag.Args()) > 0 {
			if *targetTicket != "" || *factPrefix != "" {
				log.Fatal("--target and --fact_prefix are unsupported when given tickets")
			}
			if err := readEntries(ctx, gs, entryFunc, *edgeKind, flag.Args()); err != nil {
				log.Fatal(err)
			}
		} else {
			if err := scanEntries(ctx, gs, entryFunc, *edgeKind, *targetTicket, *factPrefix); err != nil {
				log.Fatal(err)
			}
		}
		if *count {
			fmt.Println(total)
		}
		return
	}

	sgs, ok := gs.(graphstore.Sharded)
	if !ok {
		log.Fatalf("Sharding unsupported for given GraphStore type: %T", gs)
	} else if *shardIndex >= *shards {
		log.Fatalf("Invalid shard index for %d shards: %d", *shards, *shardIndex)
	}

	if *count {
		cnt, err := sgs.Count(ctx, &spb.CountRequest{Index: *shardIndex, Shards: *shards})
		if err != nil {
			log.Fatalf("ERROR: %v", err)
		}
		fmt.Println(cnt)
		return
	} else if *shardsToFiles != "" {
		var wg sync.WaitGroup
		wg.Add(int(*shards))
		for i := int64(0); i < *shards; i++ {
			go func(i int64) {
				defer wg.Done()
				path := fmt.Sprintf("%s-%.5d-of-%.5d", *shardsToFiles, i, *shards)
				f, err := vfs.Create(ctx, path)
				if err != nil {
					log.Fatalf("Failed to create file %q: %v", path, err)
				}
				defer f.Close()
				wr := delimited.NewWriter(f)
				if err := sgs.Shard(ctx, &spb.ShardRequest{
					Index:  i,
					Shards: *shards,
				}, func(entry *spb.Entry) error {
					return wr.PutProto(entry)
				}); err != nil {
					log.Fatalf("GraphStore shard scan error: %v", err)
				}
			}(i)
		}
		wg.Wait()
		return
	}

	if err := sgs.Shard(ctx, &spb.ShardRequest{
		Index:  *shardIndex,
		Shards: *shards,
	}, func(entry *spb.Entry) error {
		return wr.PutProto(entry)
	}); err != nil {
		log.Fatalf("GraphStore shard scan error: %v", err)
	}
}
Example #16
0
func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		flagutil.UsageError("Missing path to LevelDB")
	}

	var protoValueType reflect.Type
	if *protoValue != "" {
		protoValueType = proto.MessageType(*protoValue)
		if protoValueType == nil {
			flagutil.UsageErrorf("could not understand protocol buffer type: %q", *protoValue)
		}
	}

	var en *json.Encoder
	if *emitJSON {
		en = json.NewEncoder(os.Stdout)
	}

	for _, path := range flag.Args() {
		func() {
			db, err := leveldb.Open(path, nil)
			if err != nil {
				log.Fatalf("Error opening %q: %v", path, err)
			}
			defer db.Close()

			it, err := db.ScanPrefix([]byte(*keyPrefix), nil)
			if err != nil {
				log.Fatalf("Error creating iterator for %q: v", path, err)
			}
			defer it.Close()

			for {
				key, val, err := it.Next()
				if err == io.EOF {
					break
				} else if err != nil {
					log.Fatalf("Error during scan of %q: %v", path, err)
				}

				var k, v interface{}

				if protoValueType == nil {
					if *stringKey {
						k = strconv.Quote(string(key))
					} else {
						k = base64.StdEncoding.EncodeToString(key)
					}
					if *stringValue {
						v = strconv.Quote(string(val))
					} else {
						v = base64.StdEncoding.EncodeToString(val)
					}
				} else {
					p := reflect.New(protoValueType.Elem()).Interface().(proto.Message)
					if err := proto.Unmarshal(val, p); err != nil {
						log.Fatalf("Error unmarshaling value to %q: %v", *protoValue, err)
					}

					k, v = string(key), p
				}

				if en == nil {
					fmt.Println(strings.NewReplacer(
						"@key@", fmt.Sprintf("%s", k),
						"@value@", fmt.Sprintf("%s", v),
					).Replace(*lineFormat))
				} else {
					en.Encode(keyValue{k, v})
				}
			}
		}()
	}
}