func main() {
flag.Parse()
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := table.ProtoBatchParallel{&table.KVProto{db}}
ctx := context.Background()
xrefs.RegisterHTTPHandlers(ctx, xsrv.NewCombinedTable(tbl), http.DefaultServeMux)
filetree.RegisterHTTPHandlers(ctx, &ftsrv.Table{tbl, true}, http.DefaultServeMux)
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path)))
})
http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "ok")
})
http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "sure, we'll start!")
})
log.Printf("Server listening on %q", *listeningAddr)
log.Fatal(http.ListenAndServe(*listeningAddr, nil))
}
// ParseSpec parses the given specification and returns an opened handle to an
// API Interface. The following formats are currently supported:
// - http:// URL pointed at a JSON web API
// - https:// URL pointed at a JSON web API
// - host:port pointed at a GRPC API
// - local path to a LevelDB serving table
func ParseSpec(apiSpec string) (Interface, error) {
api := &apiCloser{}
if strings.HasPrefix(apiSpec, "http://") || strings.HasPrefix(apiSpec, "https://") {
api.xs = xrefs.WebClient(apiSpec)
api.ft = filetree.WebClient(apiSpec)
} else if _, err := os.Stat(apiSpec); err == nil {
db, err := leveldb.Open(apiSpec, nil)
if err != nil {
return nil, fmt.Errorf("error opening local DB at %q: %v", apiSpec, err)
}
api.closer = func() error { return db.Close() }
tbl := table.ProtoBatchParallel{&table.KVProto{db}}
api.xs = xsrv.NewCombinedTable(tbl)
api.ft = &ftsrv.Table{tbl, true}
} else {
conn, err := grpc.Dial(apiSpec, grpc.WithInsecure())
if err != nil {
return nil, fmt.Errorf("error connecting to remote API %q: %v", apiSpec, err)
}
api.closer = func() error { conn.Close(); return nil }
api.xs = xrefs.GRPC(xpb.NewXRefServiceClient(conn), gpb.NewGraphServiceClient(conn))
api.ft = filetree.GRPC(ftpb.NewFileTreeServiceClient(conn))
}
return api, nil
}
func main() {
flag.Parse()
if *servingTable == "" {
log.Fatal("Missing --serving_table argument")
} else if *portFile == "" {
log.Fatal("Missing --port_file argument")
}
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := &table.KVProto{db}
xs := xsrv.NewCombinedTable(tbl)
ft := &ftsrv.Table{tbl}
sr := &srchsrv.Table{&table.KVInverted{db}}
ctx := context.Background()
xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux)
filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux)
search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux)
web.RegisterQuitHandler(http.DefaultServeMux)
http.HandleFunc("/alive", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "ok")
})
l, err := net.Listen("tcp", "localhost:0")
if err != nil {
log.Fatal(err)
}
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
log.Fatal(err)
}
if err := ioutil.WriteFile(*portFile, []byte(port+"\n"), 0777); err != nil {
log.Fatal(err)
}
defer os.Remove(*portFile) // ignore errors
log.Fatal(http.Serve(l, nil))
}
func main() {
flag.Parse()
if *servingTable == "" && gs == nil {
flagutil.UsageError("missing either --serving_table or --graphstore")
} else if *httpListeningAddr == "" && *grpcListeningAddr == "" {
flagutil.UsageError("missing either --listen or --grpc_listen argument")
} else if *servingTable != "" && gs != nil {
flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
}
var (
xs xrefs.Service
ft filetree.Service
sr search.Service
)
ctx := context.Background()
if *servingTable != "" {
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := table.ProtoBatchParallel{&table.KVProto{db}}
xs = xsrv.NewCombinedTable(tbl)
ft = &ftsrv.Table{tbl}
sr = &srchsrv.Table{&table.KVInverted{db}}
} else {
log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
if f, ok := gs.(filetree.Service); ok {
log.Printf("Using %T directly as filetree service", gs)
ft = f
} else {
m := filetree.NewMap()
if err := m.Populate(ctx, gs); err != nil {
log.Fatalf("Error populating file tree from GraphStore: %v", err)
}
ft = m
}
if x, ok := gs.(xrefs.Service); ok {
log.Printf("Using %T directly as xrefs service", gs)
xs = x
} else {
if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
}
xs = xstore.NewGraphStoreService(gs)
}
if s, ok := gs.(search.Service); ok {
log.Printf("Using %T directly as search service", gs)
sr = s
}
}
if sr == nil {
log.Println("Search API not supported")
}
if *grpcListeningAddr != "" {
srv := grpc.NewServer()
xpb.RegisterXRefServiceServer(srv, xs)
ftpb.RegisterFileTreeServiceServer(srv, ft)
if sr != nil {
spb.RegisterSearchServiceServer(srv, sr)
}
go startGRPC(srv)
}
if *httpListeningAddr != "" {
xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux)
filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux)
if sr != nil {
search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux)
}
go startHTTP()
}
select {} // block forever
}
// Run writes the xrefs and filetree serving tables to db based on the given
// graphstore.Service.
func Run(ctx context.Context, gs graphstore.Service, db keyvalue.DB) error {
log.Println("Starting serving pipeline")
tbl := &table.KVProto{db}
// TODO(schroederc): for large corpora, this won't fit in memory
var files []string
entries := make(chan *spb.Entry)
ftIn, nIn, eIn := make(chan *spb.VName), make(chan *spb.Entry), make(chan *spb.Entry)
go func() {
for entry := range entries {
if graphstore.IsNodeFact(entry) {
nIn <- entry
if entry.FactName == schema.NodeKindFact && string(entry.FactValue) == "file" {
ftIn <- entry.Source
files = append(files, kytheuri.ToString(entry.Source))
}
} else {
eIn <- entry
}
}
close(ftIn)
close(nIn)
close(eIn)
}()
log.Println("Scanning GraphStore")
var sErr error
go func() {
sErr = gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
entries <- e
return nil
})
if sErr != nil {
sErr = fmt.Errorf("error scanning GraphStore: %v", sErr)
}
close(entries)
}()
var (
ftErr, nErr, eErr error
ftWG, edgeNodeWG sync.WaitGroup
)
ftWG.Add(1)
go func() {
defer ftWG.Done()
ftErr = writeFileTree(ctx, tbl, ftIn)
if ftErr != nil {
ftErr = fmt.Errorf("error writing FileTree: %v", ftErr)
} else {
log.Println("Wrote FileTree")
}
}()
edgeNodeWG.Add(2)
nodes := make(chan *srvpb.Node)
go func() {
defer edgeNodeWG.Done()
nErr = writeNodes(ctx, tbl, nIn, nodes)
if nErr != nil {
nErr = fmt.Errorf("error writing Nodes: %v", nErr)
} else {
log.Println("Wrote Nodes")
}
}()
go func() {
defer edgeNodeWG.Done()
eErr = writeEdges(ctx, tbl, eIn)
if eErr != nil {
eErr = fmt.Errorf("error writing Edges: %v", eErr)
} else {
log.Println("Wrote Edges")
}
}()
var (
idxWG sync.WaitGroup
idxErr error
)
idxWG.Add(1)
go func() {
defer idxWG.Done()
idxErr = writeIndex(ctx, &table.KVInverted{db}, nodes)
if idxErr != nil {
idxErr = fmt.Errorf("error writing Search Index: %v", idxErr)
} else {
log.Println("Wrote Search Index")
}
}()
edgeNodeWG.Wait()
if eErr != nil {
return eErr
} else if nErr != nil {
return nErr
}
es := xrefs.NodesEdgesService(xsrv.NewCombinedTable(tbl))
if err := writeDecorations(ctx, tbl, es, files); err != nil {
return fmt.Errorf("error writing FileDecorations: %v", err)
}
ftWG.Wait()
if ftErr != nil {
return ftErr
}
idxWG.Wait()
if idxErr != nil {
return idxErr
}
return sErr
}
func main() {
flag.Parse()
if *servingTable == "" && gs == nil {
flagutil.UsageError("missing either --serving_table or --graphstore")
} else if *httpListeningAddr == "" && *grpcListeningAddr == "" && *tlsListeningAddr == "" {
flagutil.UsageError("missing either --listen, --tls_listen, or --grpc_listen argument")
} else if *servingTable != "" && gs != nil {
flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
} else if *tlsListeningAddr != "" && (*tlsCertFile == "" || *tlsKeyFile == "") {
flagutil.UsageError("--tls_cert_file and --tls_key_file are required if given --tls_listen")
} else if flag.NArg() > 0 {
flagutil.UsageErrorf("unknown non-flag arguments given: %v", flag.Args())
}
var (
xs xrefs.Service
ft filetree.Service
sr search.Service
)
ctx := context.Background()
if *servingTable != "" {
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := table.ProtoBatchParallel{&table.KVProto{db}}
xs = xsrv.NewCombinedTable(tbl)
ft = &ftsrv.Table{tbl}
sr = &srchsrv.Table{&table.KVInverted{db}}
} else {
log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
if f, ok := gs.(filetree.Service); ok {
log.Printf("Using %T directly as filetree service", gs)
ft = f
} else {
m := filetree.NewMap()
if err := m.Populate(ctx, gs); err != nil {
log.Fatalf("Error populating file tree from GraphStore: %v", err)
}
ft = m
}
if x, ok := gs.(xrefs.Service); ok {
log.Printf("Using %T directly as xrefs service", gs)
xs = x
} else {
if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
}
xs = xstore.NewGraphStoreService(gs)
}
if s, ok := gs.(search.Service); ok {
log.Printf("Using %T directly as search service", gs)
sr = s
}
}
if sr == nil {
log.Println("Search API not supported")
}
if *grpcListeningAddr != "" {
srv := grpc.NewServer()
xpb.RegisterXRefServiceServer(srv, xs)
ftpb.RegisterFileTreeServiceServer(srv, ft)
if sr != nil {
spb.RegisterSearchServiceServer(srv, sr)
}
go startGRPC(srv)
}
if *httpListeningAddr != "" || *tlsListeningAddr != "" {
apiMux := http.NewServeMux()
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if *httpAllowOrigin != "" {
w.Header().Set("Access-Control-Allow-Origin", *httpAllowOrigin)
}
apiMux.ServeHTTP(w, r)
})
xrefs.RegisterHTTPHandlers(ctx, xs, apiMux)
filetree.RegisterHTTPHandlers(ctx, ft, apiMux)
if sr != nil {
search.RegisterHTTPHandlers(ctx, sr, apiMux)
}
if *publicResources != "" {
log.Println("Serving public resources at", *publicResources)
if s, err := os.Stat(*publicResources); err != nil {
log.Fatalf("ERROR: could not get FileInfo for %q: %v", *publicResources, err)
} else if !s.IsDir() {
log.Fatalf("ERROR: %q is not a directory", *publicResources)
}
apiMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path)))
})
}
}
if *httpListeningAddr != "" {
go startHTTP()
}
if *tlsListeningAddr != "" {
go startTLS()
}
select {} // block forever
}