func withTempTable(t *testing.T, f func(*Table)) {
dir, err := ioutil.TempDir("", "table.test")
if err != nil {
t.Errorf("error creating temporary directory: %v", err)
}
defer func() {
if err := os.RemoveAll(dir); err != nil {
t.Errorf("error removing temporary directory: %v", err)
}
}()
db, err := leveldb.Open(dir, nil)
if err != nil {
if err := os.RemoveAll(dir); err != nil {
log.Printf("error removing temporary directory: %v", err)
}
t.Errorf("error opening temporary table: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
t.Errorf("error closing temporary table: %v", err)
}
}()
tbl := &Table{&table.KVInverted{db}}
for _, n := range testNodes {
if err := IndexNode(tbl, n); err != nil {
t.Error(err)
}
}
f(tbl)
}
func main() {
flag.Parse()
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := &table.KVProto{db}
ctx := context.Background()
xrefs.RegisterHTTPHandlers(ctx, &xsrv.Table{tbl}, http.DefaultServeMux)
filetree.RegisterHTTPHandlers(ctx, &ftsrv.Table{tbl}, http.DefaultServeMux)
search.RegisterHTTPHandlers(ctx, &srchsrv.Table{&table.KVInverted{db}}, http.DefaultServeMux)
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path)))
})
http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "ok")
})
http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "sure, we'll start!")
})
log.Printf("Server listening on %q", *listeningAddr)
log.Fatal(http.ListenAndServe(*listeningAddr, nil))
}
func main() {
flag.Parse()
if *servingTable == "" {
flagutil.UsageError("missing --serving_table")
} else if *outPath == "" {
flagutil.UsageError("missing --out")
}
ctx := context.Background()
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := &table.KVProto{db}
esr := &esrchsrv.Table{&table.KVInverted{db}}
ix := index.Create(*outPath)
for _, ticket := range filesInTable(ctx, esr) {
var fd srvpb.FileDecorations
if err := tbl.Lookup(ctx, xsrv.DecorationsKey(ticket), &fd); err != nil {
log.Fatalf("Error looking up decoration for %q: %v", ticket, err)
}
ix.Add(ticket, bytes.NewReader(fd.SourceText))
}
ix.Flush()
}
// ParseSpec parses the given specification and returns an opened handle to an
// API Interface. The following formats are currently supported:
// - http:// URL pointed at a JSON web API
// - https:// URL pointed at a JSON web API
// - host:port pointed at a GRPC API
// - local path to a LevelDB serving table
func ParseSpec(apiSpec string) (Interface, error) {
api := &apiCloser{}
if strings.HasPrefix(apiSpec, "http://") || strings.HasPrefix(apiSpec, "https://") {
api.xs = xrefs.WebClient(apiSpec)
api.ft = filetree.WebClient(apiSpec)
} else if _, err := os.Stat(apiSpec); err == nil {
db, err := leveldb.Open(apiSpec, nil)
if err != nil {
return nil, fmt.Errorf("error opening local DB at %q: %v", apiSpec, err)
}
api.closer = func() error { return db.Close() }
tbl := table.ProtoBatchParallel{&table.KVProto{db}}
api.xs = xsrv.NewCombinedTable(tbl)
api.ft = &ftsrv.Table{tbl, true}
} else {
conn, err := grpc.Dial(apiSpec, grpc.WithInsecure())
if err != nil {
return nil, fmt.Errorf("error connecting to remote API %q: %v", apiSpec, err)
}
api.closer = func() error { conn.Close(); return nil }
api.xs = xrefs.GRPC(xpb.NewXRefServiceClient(conn), gpb.NewGraphServiceClient(conn))
api.ft = filetree.GRPC(ftpb.NewFileTreeServiceClient(conn))
}
return api, nil
}
func main() {
flag.Parse()
if gs == nil {
flagutil.UsageError("missing required --graphstore flag")
} else if *tablePath == "" {
flagutil.UsageError("missing required --out flag")
}
db, err := leveldb.Open(*tablePath, nil)
if err != nil {
log.Fatal(err)
}
defer db.Close()
ctx := context.Background()
if err := profile.Start(ctx); err != nil {
log.Fatal(err)
}
defer profile.Stop()
if err := pipeline.Run(ctx, gs, db, &pipeline.Options{
MaxEdgePageSize: *maxEdgePageSize,
}); err != nil {
log.Fatal("FATAL ERROR: ", err)
}
}
func tempTable(name string) (keyvalue.DB, error) {
tempDir, err := ioutil.TempDir("", "kythe.pipeline."+name)
if err != nil {
return nil, fmt.Errorf("failed to create temporary directory: %v", err)
}
tbl, err := leveldb.Open(tempDir, nil)
if err != nil {
return nil, fmt.Errorf("failed to create temporary table: %v", err)
}
return &deleteOnClose{tbl, tempDir}, nil
}
func main() {
flag.Parse()
if gs == nil && *entriesFile == "" {
flagutil.UsageError("missing --graphstore or --entries")
} else if gs != nil && *entriesFile != "" {
flagutil.UsageError("--graphstore and --entries are mutually exclusive")
} else if *tablePath == "" {
flagutil.UsageError("missing required --out flag")
}
db, err := leveldb.Open(*tablePath, nil)
if err != nil {
log.Fatal(err)
}
defer db.Close()
ctx := context.Background()
if err := profile.Start(ctx); err != nil {
log.Fatal(err)
}
defer profile.Stop()
var rd stream.EntryReader
if gs != nil {
rd = func(f func(e *spb.Entry) error) error {
defer gs.Close(ctx)
return gs.Scan(ctx, &spb.ScanRequest{}, f)
}
} else {
f, err := vfs.Open(ctx, *entriesFile)
if err != nil {
log.Fatalf("Error opening %q: %v", *entriesFile, err)
}
defer f.Close()
rd = stream.NewReader(f)
}
if err := pipeline.Run(ctx, rd, db, &pipeline.Options{
Verbose: *verbose,
MaxPageSize: *maxPageSize,
CompressShards: *compressShards,
MaxShardSize: *maxShardSize,
IOBufferSize: int(shardIOBufferSize.Bytes()),
}); err != nil {
log.Fatal("FATAL ERROR: ", err)
}
}
func main() {
flag.Parse()
if gs == nil {
flagutil.UsageError("missing required --graphstore flag")
} else if *tablePath == "" {
flagutil.UsageError("missing required --out flag")
}
db, err := leveldb.Open(*tablePath, nil)
if err != nil {
log.Fatal(err)
}
defer db.Close()
if err := pipeline.Run(context.Background(), gs, db); err != nil {
log.Fatal(err)
}
}
func main() {
flag.Parse()
if *servingTable == "" {
log.Fatal("Missing --serving_table argument")
} else if *portFile == "" {
log.Fatal("Missing --port_file argument")
}
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := &table.KVProto{db}
xs := xsrv.NewCombinedTable(tbl)
ft := &ftsrv.Table{tbl}
sr := &srchsrv.Table{&table.KVInverted{db}}
ctx := context.Background()
xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux)
filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux)
search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux)
web.RegisterQuitHandler(http.DefaultServeMux)
http.HandleFunc("/alive", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "ok")
})
l, err := net.Listen("tcp", "localhost:0")
if err != nil {
log.Fatal(err)
}
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
log.Fatal(err)
}
if err := ioutil.WriteFile(*portFile, []byte(port+"\n"), 0777); err != nil {
log.Fatal(err)
}
defer os.Remove(*portFile) // ignore errors
log.Fatal(http.Serve(l, nil))
}
func main() {
flag.Parse()
if len(flag.Args()) == 0 {
flag.Usage()
os.Exit(0)
} else if *servingTable == "" && *remoteAPI == "" {
log.Fatal("One of --serving_table or --api is required")
}
if *servingTable == "" {
if strings.HasPrefix(*remoteAPI, "http://") || strings.HasPrefix(*remoteAPI, "https://") {
xs = xrefs.WebClient(*remoteAPI)
ft = filetree.WebClient(*remoteAPI)
idx = search.WebClient(*remoteAPI)
} else {
conn, err := grpc.Dial(*remoteAPI)
if err != nil {
log.Fatalf("Error connecting to remote API %q: %v", *remoteAPI, err)
}
defer conn.Close()
xs = xrefs.GRPC(xpb.NewXRefServiceClient(conn))
ft = filetree.GRPC(ftpb.NewFileTreeServiceClient(conn))
idx = search.GRPC(spb.NewSearchServiceClient(conn))
}
} else {
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := &table.KVProto{db}
xs = &xsrv.Table{tbl}
ft = &ftsrv.Table{tbl}
idx = &srchsrv.Table{&table.KVInverted{db}}
}
if err := getCommand(flag.Arg(0)).run(); err != nil {
log.Fatal("ERROR: ", err)
}
}
func main() {
flag.Parse()
if *servingTable == "" && gs == nil {
flagutil.UsageError("missing either --serving_table or --graphstore")
} else if *httpListeningAddr == "" && *grpcListeningAddr == "" {
flagutil.UsageError("missing either --listen or --grpc_listen argument")
} else if *servingTable != "" && gs != nil {
flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
}
var (
xs xrefs.Service
ft filetree.Service
sr search.Service
)
ctx := context.Background()
if *servingTable != "" {
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := table.ProtoBatchParallel{&table.KVProto{db}}
xs = xsrv.NewCombinedTable(tbl)
ft = &ftsrv.Table{tbl}
sr = &srchsrv.Table{&table.KVInverted{db}}
} else {
log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
if f, ok := gs.(filetree.Service); ok {
log.Printf("Using %T directly as filetree service", gs)
ft = f
} else {
m := filetree.NewMap()
if err := m.Populate(ctx, gs); err != nil {
log.Fatalf("Error populating file tree from GraphStore: %v", err)
}
ft = m
}
if x, ok := gs.(xrefs.Service); ok {
log.Printf("Using %T directly as xrefs service", gs)
xs = x
} else {
if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
}
xs = xstore.NewGraphStoreService(gs)
}
if s, ok := gs.(search.Service); ok {
log.Printf("Using %T directly as search service", gs)
sr = s
}
}
if sr == nil {
log.Println("Search API not supported")
}
if *grpcListeningAddr != "" {
srv := grpc.NewServer()
xpb.RegisterXRefServiceServer(srv, xs)
ftpb.RegisterFileTreeServiceServer(srv, ft)
if sr != nil {
spb.RegisterSearchServiceServer(srv, sr)
}
go startGRPC(srv)
}
if *httpListeningAddr != "" {
xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux)
filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux)
if sr != nil {
search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux)
}
go startHTTP()
}
select {} // block forever
}
func main() {
flag.Parse()
if *servingTable == "" && gs == nil {
flagutil.UsageError("missing either --serving_table or --graphstore")
} else if *httpListeningAddr == "" && *grpcListeningAddr == "" && *tlsListeningAddr == "" {
flagutil.UsageError("missing either --listen, --tls_listen, or --grpc_listen argument")
} else if *servingTable != "" && gs != nil {
flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
} else if *tlsListeningAddr != "" && (*tlsCertFile == "" || *tlsKeyFile == "") {
flagutil.UsageError("--tls_cert_file and --tls_key_file are required if given --tls_listen")
} else if flag.NArg() > 0 {
flagutil.UsageErrorf("unknown non-flag arguments given: %v", flag.Args())
}
var (
xs xrefs.Service
ft filetree.Service
sr search.Service
)
ctx := context.Background()
if *servingTable != "" {
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := table.ProtoBatchParallel{&table.KVProto{db}}
xs = xsrv.NewCombinedTable(tbl)
ft = &ftsrv.Table{tbl}
sr = &srchsrv.Table{&table.KVInverted{db}}
} else {
log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
if f, ok := gs.(filetree.Service); ok {
log.Printf("Using %T directly as filetree service", gs)
ft = f
} else {
m := filetree.NewMap()
if err := m.Populate(ctx, gs); err != nil {
log.Fatalf("Error populating file tree from GraphStore: %v", err)
}
ft = m
}
if x, ok := gs.(xrefs.Service); ok {
log.Printf("Using %T directly as xrefs service", gs)
xs = x
} else {
if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
}
xs = xstore.NewGraphStoreService(gs)
}
if s, ok := gs.(search.Service); ok {
log.Printf("Using %T directly as search service", gs)
sr = s
}
}
if sr == nil {
log.Println("Search API not supported")
}
if *grpcListeningAddr != "" {
srv := grpc.NewServer()
xpb.RegisterXRefServiceServer(srv, xs)
ftpb.RegisterFileTreeServiceServer(srv, ft)
if sr != nil {
spb.RegisterSearchServiceServer(srv, sr)
}
go startGRPC(srv)
}
if *httpListeningAddr != "" || *tlsListeningAddr != "" {
apiMux := http.NewServeMux()
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if *httpAllowOrigin != "" {
w.Header().Set("Access-Control-Allow-Origin", *httpAllowOrigin)
}
apiMux.ServeHTTP(w, r)
})
xrefs.RegisterHTTPHandlers(ctx, xs, apiMux)
filetree.RegisterHTTPHandlers(ctx, ft, apiMux)
if sr != nil {
search.RegisterHTTPHandlers(ctx, sr, apiMux)
}
if *publicResources != "" {
log.Println("Serving public resources at", *publicResources)
if s, err := os.Stat(*publicResources); err != nil {
log.Fatalf("ERROR: could not get FileInfo for %q: %v", *publicResources, err)
} else if !s.IsDir() {
log.Fatalf("ERROR: %q is not a directory", *publicResources)
}
apiMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path)))
})
}
}
if *httpListeningAddr != "" {
go startHTTP()
}
if *tlsListeningAddr != "" {
go startTLS()
}
select {} // block forever
}
func main() {
flag.Parse()
if flag.NArg() == 0 {
flagutil.UsageError("Missing path to LevelDB")
}
var protoValueType reflect.Type
if *protoValue != "" {
protoValueType = proto.MessageType(*protoValue)
if protoValueType == nil {
flagutil.UsageErrorf("could not understand protocol buffer type: %q", *protoValue)
}
}
var en *json.Encoder
if *emitJSON {
en = json.NewEncoder(os.Stdout)
}
for _, path := range flag.Args() {
func() {
db, err := leveldb.Open(path, nil)
if err != nil {
log.Fatalf("Error opening %q: %v", path, err)
}
defer db.Close()
it, err := db.ScanPrefix([]byte(*keyPrefix), nil)
if err != nil {
log.Fatalf("Error creating iterator for %q: v", path, err)
}
defer it.Close()
for {
key, val, err := it.Next()
if err == io.EOF {
break
} else if err != nil {
log.Fatalf("Error during scan of %q: %v", path, err)
}
var k, v interface{}
if protoValueType == nil {
if *stringKey {
k = strconv.Quote(string(key))
} else {
k = base64.StdEncoding.EncodeToString(key)
}
if *stringValue {
v = strconv.Quote(string(val))
} else {
v = base64.StdEncoding.EncodeToString(val)
}
} else {
p := reflect.New(protoValueType.Elem()).Interface().(proto.Message)
if err := proto.Unmarshal(val, p); err != nil {
log.Fatalf("Error unmarshaling value to %q: %v", *protoValue, err)
}
k, v = string(key), p
}
if en == nil {
fmt.Println(strings.NewReplacer(
"@key@", fmt.Sprintf("%s", k),
"@value@", fmt.Sprintf("%s", v),
).Replace(*lineFormat))
} else {
en.Encode(keyValue{k, v})
}
}
}()
}
}