func withTempTable(t *testing.T, f func(*Table)) { dir, err := ioutil.TempDir("", "table.test") if err != nil { t.Errorf("error creating temporary directory: %v", err) } defer func() { if err := os.RemoveAll(dir); err != nil { t.Errorf("error removing temporary directory: %v", err) } }() db, err := leveldb.Open(dir, nil) if err != nil { if err := os.RemoveAll(dir); err != nil { log.Printf("error removing temporary directory: %v", err) } t.Errorf("error opening temporary table: %v", err) } defer func() { if err := db.Close(); err != nil { t.Errorf("error closing temporary table: %v", err) } }() tbl := &Table{&table.KVInverted{db}} for _, n := range testNodes { if err := IndexNode(tbl, n); err != nil { t.Error(err) } } f(tbl) }
func main() { flag.Parse() db, err := leveldb.Open(*servingTable, nil) if err != nil { log.Fatalf("Error opening db at %q: %v", *servingTable, err) } defer db.Close() tbl := &table.KVProto{db} ctx := context.Background() xrefs.RegisterHTTPHandlers(ctx, &xsrv.Table{tbl}, http.DefaultServeMux) filetree.RegisterHTTPHandlers(ctx, &ftsrv.Table{tbl}, http.DefaultServeMux) search.RegisterHTTPHandlers(ctx, &srchsrv.Table{&table.KVInverted{db}}, http.DefaultServeMux) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path))) }) http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "ok") }) http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "sure, we'll start!") }) log.Printf("Server listening on %q", *listeningAddr) log.Fatal(http.ListenAndServe(*listeningAddr, nil)) }
func main() { flag.Parse() if *servingTable == "" { flagutil.UsageError("missing --serving_table") } else if *outPath == "" { flagutil.UsageError("missing --out") } ctx := context.Background() db, err := leveldb.Open(*servingTable, nil) if err != nil { log.Fatalf("Error opening db at %q: %v", *servingTable, err) } defer db.Close() tbl := &table.KVProto{db} esr := &esrchsrv.Table{&table.KVInverted{db}} ix := index.Create(*outPath) for _, ticket := range filesInTable(ctx, esr) { var fd srvpb.FileDecorations if err := tbl.Lookup(ctx, xsrv.DecorationsKey(ticket), &fd); err != nil { log.Fatalf("Error looking up decoration for %q: %v", ticket, err) } ix.Add(ticket, bytes.NewReader(fd.SourceText)) } ix.Flush() }
// ParseSpec parses the given specification and returns an opened handle to an // API Interface. The following formats are currently supported: // - http:// URL pointed at a JSON web API // - https:// URL pointed at a JSON web API // - host:port pointed at a GRPC API // - local path to a LevelDB serving table func ParseSpec(apiSpec string) (Interface, error) { api := &apiCloser{} if strings.HasPrefix(apiSpec, "http://") || strings.HasPrefix(apiSpec, "https://") { api.xs = xrefs.WebClient(apiSpec) api.ft = filetree.WebClient(apiSpec) } else if _, err := os.Stat(apiSpec); err == nil { db, err := leveldb.Open(apiSpec, nil) if err != nil { return nil, fmt.Errorf("error opening local DB at %q: %v", apiSpec, err) } api.closer = func() error { return db.Close() } tbl := table.ProtoBatchParallel{&table.KVProto{db}} api.xs = xsrv.NewCombinedTable(tbl) api.ft = &ftsrv.Table{tbl, true} } else { conn, err := grpc.Dial(apiSpec, grpc.WithInsecure()) if err != nil { return nil, fmt.Errorf("error connecting to remote API %q: %v", apiSpec, err) } api.closer = func() error { conn.Close(); return nil } api.xs = xrefs.GRPC(xpb.NewXRefServiceClient(conn), gpb.NewGraphServiceClient(conn)) api.ft = filetree.GRPC(ftpb.NewFileTreeServiceClient(conn)) } return api, nil }
func main() { flag.Parse() if gs == nil { flagutil.UsageError("missing required --graphstore flag") } else if *tablePath == "" { flagutil.UsageError("missing required --out flag") } db, err := leveldb.Open(*tablePath, nil) if err != nil { log.Fatal(err) } defer db.Close() ctx := context.Background() if err := profile.Start(ctx); err != nil { log.Fatal(err) } defer profile.Stop() if err := pipeline.Run(ctx, gs, db, &pipeline.Options{ MaxEdgePageSize: *maxEdgePageSize, }); err != nil { log.Fatal("FATAL ERROR: ", err) } }
func tempTable(name string) (keyvalue.DB, error) { tempDir, err := ioutil.TempDir("", "kythe.pipeline."+name) if err != nil { return nil, fmt.Errorf("failed to create temporary directory: %v", err) } tbl, err := leveldb.Open(tempDir, nil) if err != nil { return nil, fmt.Errorf("failed to create temporary table: %v", err) } return &deleteOnClose{tbl, tempDir}, nil }
func main() { flag.Parse() if gs == nil && *entriesFile == "" { flagutil.UsageError("missing --graphstore or --entries") } else if gs != nil && *entriesFile != "" { flagutil.UsageError("--graphstore and --entries are mutually exclusive") } else if *tablePath == "" { flagutil.UsageError("missing required --out flag") } db, err := leveldb.Open(*tablePath, nil) if err != nil { log.Fatal(err) } defer db.Close() ctx := context.Background() if err := profile.Start(ctx); err != nil { log.Fatal(err) } defer profile.Stop() var rd stream.EntryReader if gs != nil { rd = func(f func(e *spb.Entry) error) error { defer gs.Close(ctx) return gs.Scan(ctx, &spb.ScanRequest{}, f) } } else { f, err := vfs.Open(ctx, *entriesFile) if err != nil { log.Fatalf("Error opening %q: %v", *entriesFile, err) } defer f.Close() rd = stream.NewReader(f) } if err := pipeline.Run(ctx, rd, db, &pipeline.Options{ Verbose: *verbose, MaxPageSize: *maxPageSize, CompressShards: *compressShards, MaxShardSize: *maxShardSize, IOBufferSize: int(shardIOBufferSize.Bytes()), }); err != nil { log.Fatal("FATAL ERROR: ", err) } }
func main() { flag.Parse() if gs == nil { flagutil.UsageError("missing required --graphstore flag") } else if *tablePath == "" { flagutil.UsageError("missing required --out flag") } db, err := leveldb.Open(*tablePath, nil) if err != nil { log.Fatal(err) } defer db.Close() if err := pipeline.Run(context.Background(), gs, db); err != nil { log.Fatal(err) } }
func main() { flag.Parse() if *servingTable == "" { log.Fatal("Missing --serving_table argument") } else if *portFile == "" { log.Fatal("Missing --port_file argument") } db, err := leveldb.Open(*servingTable, nil) if err != nil { log.Fatalf("Error opening db at %q: %v", *servingTable, err) } defer db.Close() tbl := &table.KVProto{db} xs := xsrv.NewCombinedTable(tbl) ft := &ftsrv.Table{tbl} sr := &srchsrv.Table{&table.KVInverted{db}} ctx := context.Background() xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux) filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux) search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux) web.RegisterQuitHandler(http.DefaultServeMux) http.HandleFunc("/alive", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "ok") }) l, err := net.Listen("tcp", "localhost:0") if err != nil { log.Fatal(err) } _, port, err := net.SplitHostPort(l.Addr().String()) if err != nil { log.Fatal(err) } if err := ioutil.WriteFile(*portFile, []byte(port+"\n"), 0777); err != nil { log.Fatal(err) } defer os.Remove(*portFile) // ignore errors log.Fatal(http.Serve(l, nil)) }
func main() { flag.Parse() if len(flag.Args()) == 0 { flag.Usage() os.Exit(0) } else if *servingTable == "" && *remoteAPI == "" { log.Fatal("One of --serving_table or --api is required") } if *servingTable == "" { if strings.HasPrefix(*remoteAPI, "http://") || strings.HasPrefix(*remoteAPI, "https://") { xs = xrefs.WebClient(*remoteAPI) ft = filetree.WebClient(*remoteAPI) idx = search.WebClient(*remoteAPI) } else { conn, err := grpc.Dial(*remoteAPI) if err != nil { log.Fatalf("Error connecting to remote API %q: %v", *remoteAPI, err) } defer conn.Close() xs = xrefs.GRPC(xpb.NewXRefServiceClient(conn)) ft = filetree.GRPC(ftpb.NewFileTreeServiceClient(conn)) idx = search.GRPC(spb.NewSearchServiceClient(conn)) } } else { db, err := leveldb.Open(*servingTable, nil) if err != nil { log.Fatalf("Error opening db at %q: %v", *servingTable, err) } defer db.Close() tbl := &table.KVProto{db} xs = &xsrv.Table{tbl} ft = &ftsrv.Table{tbl} idx = &srchsrv.Table{&table.KVInverted{db}} } if err := getCommand(flag.Arg(0)).run(); err != nil { log.Fatal("ERROR: ", err) } }
func main() { flag.Parse() if *servingTable == "" && gs == nil { flagutil.UsageError("missing either --serving_table or --graphstore") } else if *httpListeningAddr == "" && *grpcListeningAddr == "" { flagutil.UsageError("missing either --listen or --grpc_listen argument") } else if *servingTable != "" && gs != nil { flagutil.UsageError("--serving_table and --graphstore are mutually exclusive") } var ( xs xrefs.Service ft filetree.Service sr search.Service ) ctx := context.Background() if *servingTable != "" { db, err := leveldb.Open(*servingTable, nil) if err != nil { log.Fatalf("Error opening db at %q: %v", *servingTable, err) } defer db.Close() tbl := table.ProtoBatchParallel{&table.KVProto{db}} xs = xsrv.NewCombinedTable(tbl) ft = &ftsrv.Table{tbl} sr = &srchsrv.Table{&table.KVInverted{db}} } else { log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table") if f, ok := gs.(filetree.Service); ok { log.Printf("Using %T directly as filetree service", gs) ft = f } else { m := filetree.NewMap() if err := m.Populate(ctx, gs); err != nil { log.Fatalf("Error populating file tree from GraphStore: %v", err) } ft = m } if x, ok := gs.(xrefs.Service); ok { log.Printf("Using %T directly as xrefs service", gs) xs = x } else { if err := xstore.EnsureReverseEdges(ctx, gs); err != nil { log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err) } xs = xstore.NewGraphStoreService(gs) } if s, ok := gs.(search.Service); ok { log.Printf("Using %T directly as search service", gs) sr = s } } if sr == nil { log.Println("Search API not supported") } if *grpcListeningAddr != "" { srv := grpc.NewServer() xpb.RegisterXRefServiceServer(srv, xs) ftpb.RegisterFileTreeServiceServer(srv, ft) if sr != nil { spb.RegisterSearchServiceServer(srv, sr) } go startGRPC(srv) } if *httpListeningAddr != "" { xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux) filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux) if sr != nil { search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux) } go startHTTP() } select {} // block forever }
func main() { flag.Parse() if *servingTable == "" && gs == nil { flagutil.UsageError("missing either --serving_table or --graphstore") } else if *httpListeningAddr == "" && *grpcListeningAddr == "" && *tlsListeningAddr == "" { flagutil.UsageError("missing either --listen, --tls_listen, or --grpc_listen argument") } else if *servingTable != "" && gs != nil { flagutil.UsageError("--serving_table and --graphstore are mutually exclusive") } else if *tlsListeningAddr != "" && (*tlsCertFile == "" || *tlsKeyFile == "") { flagutil.UsageError("--tls_cert_file and --tls_key_file are required if given --tls_listen") } else if flag.NArg() > 0 { flagutil.UsageErrorf("unknown non-flag arguments given: %v", flag.Args()) } var ( xs xrefs.Service ft filetree.Service sr search.Service ) ctx := context.Background() if *servingTable != "" { db, err := leveldb.Open(*servingTable, nil) if err != nil { log.Fatalf("Error opening db at %q: %v", *servingTable, err) } defer db.Close() tbl := table.ProtoBatchParallel{&table.KVProto{db}} xs = xsrv.NewCombinedTable(tbl) ft = &ftsrv.Table{tbl} sr = &srchsrv.Table{&table.KVInverted{db}} } else { log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table") if f, ok := gs.(filetree.Service); ok { log.Printf("Using %T directly as filetree service", gs) ft = f } else { m := filetree.NewMap() if err := m.Populate(ctx, gs); err != nil { log.Fatalf("Error populating file tree from GraphStore: %v", err) } ft = m } if x, ok := gs.(xrefs.Service); ok { log.Printf("Using %T directly as xrefs service", gs) xs = x } else { if err := xstore.EnsureReverseEdges(ctx, gs); err != nil { log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err) } xs = xstore.NewGraphStoreService(gs) } if s, ok := gs.(search.Service); ok { log.Printf("Using %T directly as search service", gs) sr = s } } if sr == nil { log.Println("Search API not supported") } if *grpcListeningAddr != "" { srv := grpc.NewServer() xpb.RegisterXRefServiceServer(srv, xs) ftpb.RegisterFileTreeServiceServer(srv, ft) if sr != nil { spb.RegisterSearchServiceServer(srv, sr) } go startGRPC(srv) } if *httpListeningAddr != "" || *tlsListeningAddr != "" { apiMux := http.NewServeMux() http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if *httpAllowOrigin != "" { w.Header().Set("Access-Control-Allow-Origin", *httpAllowOrigin) } apiMux.ServeHTTP(w, r) }) xrefs.RegisterHTTPHandlers(ctx, xs, apiMux) filetree.RegisterHTTPHandlers(ctx, ft, apiMux) if sr != nil { search.RegisterHTTPHandlers(ctx, sr, apiMux) } if *publicResources != "" { log.Println("Serving public resources at", *publicResources) if s, err := os.Stat(*publicResources); err != nil { log.Fatalf("ERROR: could not get FileInfo for %q: %v", *publicResources, err) } else if !s.IsDir() { log.Fatalf("ERROR: %q is not a directory", *publicResources) } apiMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path))) }) } } if *httpListeningAddr != "" { go startHTTP() } if *tlsListeningAddr != "" { go startTLS() } select {} // block forever }
func main() { flag.Parse() if flag.NArg() == 0 { flagutil.UsageError("Missing path to LevelDB") } var protoValueType reflect.Type if *protoValue != "" { protoValueType = proto.MessageType(*protoValue) if protoValueType == nil { flagutil.UsageErrorf("could not understand protocol buffer type: %q", *protoValue) } } var en *json.Encoder if *emitJSON { en = json.NewEncoder(os.Stdout) } for _, path := range flag.Args() { func() { db, err := leveldb.Open(path, nil) if err != nil { log.Fatalf("Error opening %q: %v", path, err) } defer db.Close() it, err := db.ScanPrefix([]byte(*keyPrefix), nil) if err != nil { log.Fatalf("Error creating iterator for %q: v", path, err) } defer it.Close() for { key, val, err := it.Next() if err == io.EOF { break } else if err != nil { log.Fatalf("Error during scan of %q: %v", path, err) } var k, v interface{} if protoValueType == nil { if *stringKey { k = strconv.Quote(string(key)) } else { k = base64.StdEncoding.EncodeToString(key) } if *stringValue { v = strconv.Quote(string(val)) } else { v = base64.StdEncoding.EncodeToString(val) } } else { p := reflect.New(protoValueType.Elem()).Interface().(proto.Message) if err := proto.Unmarshal(val, p); err != nil { log.Fatalf("Error unmarshaling value to %q: %v", *protoValue, err) } k, v = string(key), p } if en == nil { fmt.Println(strings.NewReplacer( "@key@", fmt.Sprintf("%s", k), "@value@", fmt.Sprintf("%s", v), ).Replace(*lineFormat)) } else { en.Encode(keyValue{k, v}) } } }() } }