func main() {
flag.Parse()
// 得到 CrawlClient
conn, err := grpc.Dial(*address, grpc.WithInsecure())
if err != nil {
log.Fatal(err)
}
client := pb.NewCrawlClient(conn)
log.Printf("开始抓取")
request := pb.CrawlRequest{
Url: *url,
Timeout: 10000,
RecrawlTtl: *ttl,
Method: pb.Method(pb.Method_value[*method]),
}
response, err := client.Crawl(context.Background(), &request)
if err != nil {
log.Fatal(err)
}
log.Printf("抓取完毕")
log.Printf("%+v", response.Metadata)
log.Printf("%d", len(response.Content))
}
func (zc *ZergClient) Get(url string) (pb.CrawlClient, error) {
// 检查是否已经初始化
if !zc.initialized {
return nil, errors.New("DistCrawlClient 没有初始化")
}
node, err := zc.consistentService.GetNode(url)
if err != nil {
return nil, err
}
if _, ok := zc.conns[node]; !ok {
conn, err := grpc.Dial(node, grpc.WithInsecure())
if err != nil {
return nil, err
}
zc.conns[node] = conn
client := pb.NewCrawlClient(conn)
zc.clients[node] = client
}
return zc.clients[node], err
}
func (zc *ZergClient) Crawl(in *pb.CrawlRequest, opts ...grpc.CallOption) (*pb.CrawlResponse, error) {
// 检查是否已经初始化
if !zc.initialized {
return nil, errors.New("DistCrawlClient 没有初始化")
}
node, err := zc.consistentService.GetNode(in.Url)
if err != nil {
return nil, err
}
if _, ok := zc.conns[node]; !ok {
conn, err := grpc.Dial(node, grpc.WithInsecure())
if err != nil {
return nil, err
}
zc.conns[node] = conn
client := pb.NewCrawlClient(conn)
zc.clients[node] = client
}
return zc.clients[node].Crawl(context.Background(), in, opts...)
}
