Exemplo n.º 1
0
func main() {
	flag.Parse()

	db, err := leveldb.Open(*servingTable, nil)
	if err != nil {
		log.Fatalf("Error opening db at %q: %v", *servingTable, err)
	}
	defer db.Close()
	tbl := table.ProtoBatchParallel{&table.KVProto{db}}

	ctx := context.Background()
	xrefs.RegisterHTTPHandlers(ctx, xsrv.NewCombinedTable(tbl), http.DefaultServeMux)
	filetree.RegisterHTTPHandlers(ctx, &ftsrv.Table{tbl, true}, http.DefaultServeMux)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path)))
	})

	http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "ok")
	})
	http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "sure, we'll start!")
	})

	log.Printf("Server listening on %q", *listeningAddr)
	log.Fatal(http.ListenAndServe(*listeningAddr, nil))
}
Exemplo n.º 2
0
Arquivo: api.go Projeto: benjyw/kythe
// ParseSpec parses the given specification and returns an opened handle to an
// API Interface.  The following formats are currently supported:
//   - http:// URL pointed at a JSON web API
//   - https:// URL pointed at a JSON web API
//   - host:port pointed at a GRPC API
//   - local path to a LevelDB serving table
func ParseSpec(apiSpec string) (Interface, error) {
	api := &apiCloser{}
	if strings.HasPrefix(apiSpec, "http://") || strings.HasPrefix(apiSpec, "https://") {
		api.xs = xrefs.WebClient(apiSpec)
		api.ft = filetree.WebClient(apiSpec)
	} else if _, err := os.Stat(apiSpec); err == nil {
		db, err := leveldb.Open(apiSpec, nil)
		if err != nil {
			return nil, fmt.Errorf("error opening local DB at %q: %v", apiSpec, err)
		}
		api.closer = func() error { return db.Close() }

		tbl := table.ProtoBatchParallel{&table.KVProto{db}}
		api.xs = xsrv.NewCombinedTable(tbl)
		api.ft = &ftsrv.Table{tbl, true}
	} else {
		conn, err := grpc.Dial(apiSpec, grpc.WithInsecure())
		if err != nil {
			return nil, fmt.Errorf("error connecting to remote API %q: %v", apiSpec, err)
		}
		api.closer = func() error { conn.Close(); return nil }

		api.xs = xrefs.GRPC(xpb.NewXRefServiceClient(conn), gpb.NewGraphServiceClient(conn))
		api.ft = filetree.GRPC(ftpb.NewFileTreeServiceClient(conn))
	}
	return api, nil
}
Exemplo n.º 3
0
func main() {
	flag.Parse()
	if *servingTable == "" {
		log.Fatal("Missing --serving_table argument")
	} else if *portFile == "" {
		log.Fatal("Missing --port_file argument")
	}

	db, err := leveldb.Open(*servingTable, nil)
	if err != nil {
		log.Fatalf("Error opening db at %q: %v", *servingTable, err)
	}
	defer db.Close()
	tbl := &table.KVProto{db}
	xs := xsrv.NewCombinedTable(tbl)
	ft := &ftsrv.Table{tbl}
	sr := &srchsrv.Table{&table.KVInverted{db}}

	ctx := context.Background()
	xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux)
	filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux)
	search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux)
	web.RegisterQuitHandler(http.DefaultServeMux)
	http.HandleFunc("/alive", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "ok")
	})

	l, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		log.Fatal(err)
	}

	_, port, err := net.SplitHostPort(l.Addr().String())
	if err != nil {
		log.Fatal(err)
	}

	if err := ioutil.WriteFile(*portFile, []byte(port+"\n"), 0777); err != nil {
		log.Fatal(err)
	}
	defer os.Remove(*portFile) // ignore errors
	log.Fatal(http.Serve(l, nil))
}
Exemplo n.º 4
0
func main() {
	flag.Parse()
	if *servingTable == "" && gs == nil {
		flagutil.UsageError("missing either --serving_table or --graphstore")
	} else if *httpListeningAddr == "" && *grpcListeningAddr == "" {
		flagutil.UsageError("missing either --listen or --grpc_listen argument")
	} else if *servingTable != "" && gs != nil {
		flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
	}

	var (
		xs xrefs.Service
		ft filetree.Service
		sr search.Service
	)

	ctx := context.Background()
	if *servingTable != "" {
		db, err := leveldb.Open(*servingTable, nil)
		if err != nil {
			log.Fatalf("Error opening db at %q: %v", *servingTable, err)
		}
		defer db.Close()
		tbl := table.ProtoBatchParallel{&table.KVProto{db}}
		xs = xsrv.NewCombinedTable(tbl)
		ft = &ftsrv.Table{tbl}
		sr = &srchsrv.Table{&table.KVInverted{db}}
	} else {
		log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
		if f, ok := gs.(filetree.Service); ok {
			log.Printf("Using %T directly as filetree service", gs)
			ft = f
		} else {
			m := filetree.NewMap()
			if err := m.Populate(ctx, gs); err != nil {
				log.Fatalf("Error populating file tree from GraphStore: %v", err)
			}
			ft = m
		}

		if x, ok := gs.(xrefs.Service); ok {
			log.Printf("Using %T directly as xrefs service", gs)
			xs = x
		} else {
			if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
				log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
			}
			xs = xstore.NewGraphStoreService(gs)
		}

		if s, ok := gs.(search.Service); ok {
			log.Printf("Using %T directly as search service", gs)
			sr = s
		}
	}

	if sr == nil {
		log.Println("Search API not supported")
	}

	if *grpcListeningAddr != "" {
		srv := grpc.NewServer()
		xpb.RegisterXRefServiceServer(srv, xs)
		ftpb.RegisterFileTreeServiceServer(srv, ft)
		if sr != nil {
			spb.RegisterSearchServiceServer(srv, sr)
		}
		go startGRPC(srv)
	}

	if *httpListeningAddr != "" {
		xrefs.RegisterHTTPHandlers(ctx, xs, http.DefaultServeMux)
		filetree.RegisterHTTPHandlers(ctx, ft, http.DefaultServeMux)
		if sr != nil {
			search.RegisterHTTPHandlers(ctx, sr, http.DefaultServeMux)
		}
		go startHTTP()
	}

	select {} // block forever
}
Exemplo n.º 5
0
// Run writes the xrefs and filetree serving tables to db based on the given
// graphstore.Service.
func Run(ctx context.Context, gs graphstore.Service, db keyvalue.DB) error {
	log.Println("Starting serving pipeline")
	tbl := &table.KVProto{db}

	// TODO(schroederc): for large corpora, this won't fit in memory
	var files []string

	entries := make(chan *spb.Entry)
	ftIn, nIn, eIn := make(chan *spb.VName), make(chan *spb.Entry), make(chan *spb.Entry)
	go func() {
		for entry := range entries {
			if graphstore.IsNodeFact(entry) {
				nIn <- entry
				if entry.FactName == schema.NodeKindFact && string(entry.FactValue) == "file" {
					ftIn <- entry.Source
					files = append(files, kytheuri.ToString(entry.Source))
				}
			} else {
				eIn <- entry
			}
		}
		close(ftIn)
		close(nIn)
		close(eIn)
	}()
	log.Println("Scanning GraphStore")
	var sErr error
	go func() {
		sErr = gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
			entries <- e
			return nil
		})
		if sErr != nil {
			sErr = fmt.Errorf("error scanning GraphStore: %v", sErr)
		}
		close(entries)
	}()

	var (
		ftErr, nErr, eErr error
		ftWG, edgeNodeWG  sync.WaitGroup
	)
	ftWG.Add(1)
	go func() {
		defer ftWG.Done()
		ftErr = writeFileTree(ctx, tbl, ftIn)
		if ftErr != nil {
			ftErr = fmt.Errorf("error writing FileTree: %v", ftErr)
		} else {
			log.Println("Wrote FileTree")
		}
	}()
	edgeNodeWG.Add(2)
	nodes := make(chan *srvpb.Node)
	go func() {
		defer edgeNodeWG.Done()
		nErr = writeNodes(ctx, tbl, nIn, nodes)
		if nErr != nil {
			nErr = fmt.Errorf("error writing Nodes: %v", nErr)
		} else {
			log.Println("Wrote Nodes")
		}
	}()
	go func() {
		defer edgeNodeWG.Done()
		eErr = writeEdges(ctx, tbl, eIn)
		if eErr != nil {
			eErr = fmt.Errorf("error writing Edges: %v", eErr)
		} else {
			log.Println("Wrote Edges")
		}
	}()

	var (
		idxWG  sync.WaitGroup
		idxErr error
	)
	idxWG.Add(1)
	go func() {
		defer idxWG.Done()
		idxErr = writeIndex(ctx, &table.KVInverted{db}, nodes)
		if idxErr != nil {
			idxErr = fmt.Errorf("error writing Search Index: %v", idxErr)
		} else {
			log.Println("Wrote Search Index")
		}
	}()

	edgeNodeWG.Wait()
	if eErr != nil {
		return eErr
	} else if nErr != nil {
		return nErr
	}

	es := xrefs.NodesEdgesService(xsrv.NewCombinedTable(tbl))
	if err := writeDecorations(ctx, tbl, es, files); err != nil {
		return fmt.Errorf("error writing FileDecorations: %v", err)
	}

	ftWG.Wait()
	if ftErr != nil {
		return ftErr
	}
	idxWG.Wait()
	if idxErr != nil {
		return idxErr
	}

	return sErr
}
Exemplo n.º 6
0
func main() {
	flag.Parse()
	if *servingTable == "" && gs == nil {
		flagutil.UsageError("missing either --serving_table or --graphstore")
	} else if *httpListeningAddr == "" && *grpcListeningAddr == "" && *tlsListeningAddr == "" {
		flagutil.UsageError("missing either --listen, --tls_listen, or --grpc_listen argument")
	} else if *servingTable != "" && gs != nil {
		flagutil.UsageError("--serving_table and --graphstore are mutually exclusive")
	} else if *tlsListeningAddr != "" && (*tlsCertFile == "" || *tlsKeyFile == "") {
		flagutil.UsageError("--tls_cert_file and --tls_key_file are required if given --tls_listen")
	} else if flag.NArg() > 0 {
		flagutil.UsageErrorf("unknown non-flag arguments given: %v", flag.Args())
	}

	var (
		xs xrefs.Service
		ft filetree.Service
		sr search.Service
	)

	ctx := context.Background()
	if *servingTable != "" {
		db, err := leveldb.Open(*servingTable, nil)
		if err != nil {
			log.Fatalf("Error opening db at %q: %v", *servingTable, err)
		}
		defer db.Close()
		tbl := table.ProtoBatchParallel{&table.KVProto{db}}
		xs = xsrv.NewCombinedTable(tbl)
		ft = &ftsrv.Table{tbl}
		sr = &srchsrv.Table{&table.KVInverted{db}}
	} else {
		log.Println("WARNING: serving directly from a GraphStore can be slow; you may want to use a --serving_table")
		if f, ok := gs.(filetree.Service); ok {
			log.Printf("Using %T directly as filetree service", gs)
			ft = f
		} else {
			m := filetree.NewMap()
			if err := m.Populate(ctx, gs); err != nil {
				log.Fatalf("Error populating file tree from GraphStore: %v", err)
			}
			ft = m
		}

		if x, ok := gs.(xrefs.Service); ok {
			log.Printf("Using %T directly as xrefs service", gs)
			xs = x
		} else {
			if err := xstore.EnsureReverseEdges(ctx, gs); err != nil {
				log.Fatalf("Error ensuring reverse edges in GraphStore: %v", err)
			}
			xs = xstore.NewGraphStoreService(gs)
		}

		if s, ok := gs.(search.Service); ok {
			log.Printf("Using %T directly as search service", gs)
			sr = s
		}
	}

	if sr == nil {
		log.Println("Search API not supported")
	}

	if *grpcListeningAddr != "" {
		srv := grpc.NewServer()
		xpb.RegisterXRefServiceServer(srv, xs)
		ftpb.RegisterFileTreeServiceServer(srv, ft)
		if sr != nil {
			spb.RegisterSearchServiceServer(srv, sr)
		}
		go startGRPC(srv)
	}

	if *httpListeningAddr != "" || *tlsListeningAddr != "" {
		apiMux := http.NewServeMux()
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			if *httpAllowOrigin != "" {
				w.Header().Set("Access-Control-Allow-Origin", *httpAllowOrigin)
			}
			apiMux.ServeHTTP(w, r)
		})

		xrefs.RegisterHTTPHandlers(ctx, xs, apiMux)
		filetree.RegisterHTTPHandlers(ctx, ft, apiMux)
		if sr != nil {
			search.RegisterHTTPHandlers(ctx, sr, apiMux)
		}
		if *publicResources != "" {
			log.Println("Serving public resources at", *publicResources)
			if s, err := os.Stat(*publicResources); err != nil {
				log.Fatalf("ERROR: could not get FileInfo for %q: %v", *publicResources, err)
			} else if !s.IsDir() {
				log.Fatalf("ERROR: %q is not a directory", *publicResources)
			}
			apiMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
				http.ServeFile(w, r, filepath.Join(*publicResources, filepath.Clean(r.URL.Path)))
			})
		}
	}
	if *httpListeningAddr != "" {
		go startHTTP()
	}
	if *tlsListeningAddr != "" {
		go startTLS()
	}

	select {} // block forever
}