Beispiel #1
0
func main() {
	flag.Parse()

	// 得到 CrawlClient
	conn, err := grpc.Dial(*address, grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	client := pb.NewCrawlClient(conn)

	log.Printf("开始抓取")
	request := pb.CrawlRequest{
		Url:        *url,
		Timeout:    10000,
		RecrawlTtl: *ttl,
		Method:     pb.Method(pb.Method_value[*method]),
	}
	response, err := client.Crawl(context.Background(), &request)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("抓取完毕")
	log.Printf("%+v", response.Metadata)
	log.Printf("%d", len(response.Content))
}
Beispiel #2
0
func (zc *ZergClient) Get(url string) (pb.CrawlClient, error) {
	// 检查是否已经初始化
	if !zc.initialized {
		return nil, errors.New("DistCrawlClient 没有初始化")
	}

	node, err := zc.consistentService.GetNode(url)
	if err != nil {
		return nil, err
	}

	if _, ok := zc.conns[node]; !ok {
		conn, err := grpc.Dial(node, grpc.WithInsecure())
		if err != nil {
			return nil, err
		}
		zc.conns[node] = conn
		client := pb.NewCrawlClient(conn)
		zc.clients[node] = client
	}

	return zc.clients[node], err
}
Beispiel #3
0
func (zc *ZergClient) Crawl(in *pb.CrawlRequest, opts ...grpc.CallOption) (*pb.CrawlResponse, error) {
	// 检查是否已经初始化
	if !zc.initialized {
		return nil, errors.New("DistCrawlClient 没有初始化")
	}

	node, err := zc.consistentService.GetNode(in.Url)
	if err != nil {
		return nil, err
	}

	if _, ok := zc.conns[node]; !ok {
		conn, err := grpc.Dial(node, grpc.WithInsecure())
		if err != nil {
			return nil, err
		}
		zc.conns[node] = conn
		client := pb.NewCrawlClient(conn)
		zc.clients[node] = client
	}

	return zc.clients[node].Crawl(context.Background(), in, opts...)
}