func TestPush(t *testing.T) {
cfg, err := config.Get()
if err != nil {
t.Error(err.Error())
}
nodes := cfg.SSDBNodes
clients := ssdb.GetClients(nodes, cfg, ssdb.Params{MaxActive: 100, MaxIdle: 100})
req := initRequest()
r := initResponse(req)
err = Push(clients, r)
if err != nil {
t.Error("TestPush fail ", err.Error())
} else {
t.Log("TestPush ok")
}
}
func BenchmarkPush(b *testing.B) {
cfg, err := config.Get("/etc/yascrapy/core.json")
if err != nil {
b.Error(err.Error())
}
nodes := cfg.SSDBNodes
clients := ssdb.GetClients(nodes, cfg, ssdb.Params{MaxActive: 100, MaxIdle: 100})
for i := 0; i < b.N; i++ {
req := initRequest()
req.Url = fmt.Sprintf("%s%d", req.Url, i)
resp := initResponse(req)
err := Push(clients, resp)
if err != nil {
b.Log("TestPush fail ", err.Error())
}
}
}
func run(cfgFile string, consumerNumber int, crawlerName string) {
var conn *amqp.Connection
var consumers []*core_consumer.CoreConsumer
cfg, err := config.Get(cfgFile)
if err != nil {
utils.Error.Println("read config file fail ", err.Error(), cfgFile)
return
}
proxyNode := config.SSDBNode{
Host: cfg.ProxyRedisIp,
Port: cfg.ProxyRedisPort,
}
ssdbNodes := cfg.SSDBNodes
if len(ssdbNodes) == 0 {
utils.Error.Println("ssdb nodes can not be empty")
return
}
ssdbClients := ssdb.GetClients(ssdbNodes, cfg, ssdb.Params{MaxIdle: 5000, MaxActive: 5000})
proxyClient := ssdb.GetProxyClient(proxyNode, ssdb.Params{MaxIdle: 5000, MaxActive: 5000})
connPool := &rabbitmq.ConnPool{
MaxIdle: 10,
MaxActive: 50,
Dial: rabbitmq.CreateConn,
Cfg: cfg,
}
maxConn := 1000
httpClients := CrawlerHttpClients(consumerNumber, maxConn)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, os.Interrupt)
signal.Notify(sigs, syscall.SIGTERM)
go func() {
<-sigs
cleanup(cfg, proxyClient)
os.Exit(0)
}()
for {
reqQueues, err := core_queue.GetQueues("http_request:", cfg)
if err != nil {
utils.Error.Printf("GetReqQueues error:%s\n", err.Error())
time.Sleep(1 * time.Second)
continue
} else {
utils.Debug.Println("GetReqQueues success")
}
respQueues, err := core_queue.GetQueues("http_response:", cfg)
if err != nil {
utils.Error.Printf("GetRespQueues error:%s\n", err.Error())
time.Sleep(1 * time.Second)
continue
} else {
utils.Debug.Println("GetRespQueues success")
}
conn = connPool.Get().(*amqp.Connection)
reqQueues, err = core_queue.UpdateQueueCount(reqQueues, conn)
if err != nil {
utils.Error.Println(err.Error())
time.Sleep(1 * time.Second)
continue
} else {
utils.Debug.Println("UpdateQueueCount success")
}
connPool.Release(conn)
reqQueues = core_queue.RemoveEmptyQueues(reqQueues)
utils.Debug.Println(reqQueues)
if len(reqQueues) == 0 {
utils.Info.Println("req queues empty")
time.Sleep(1 * time.Second)
continue
} else {
utils.Debug.Println("RemoveEmptyQueues success")
}
core_queue.DeclareResponseQueues(conn, reqQueues, 5)
conn = connPool.Get().(*amqp.Connection)
respQueues, err = core_queue.UpdateQueueCount(respQueues, conn)
utils.Debug.Println(respQueues)
if err != nil {
conn = nil
utils.Error.Println(err.Error())
time.Sleep(1 * time.Second)
continue
}
connPool.Release(conn)
if len(respQueues) == 0 {
utils.Info.Println("resp queues empty")
time.Sleep(1 * time.Second)
continue
}
consumers = core_consumer.AliveCrawlerConsumers(consumers, crawlerName)
utils.Debug.Println("total consumers: ", len(consumers), ", need start consumers:", consumerNumber-len(consumers))
newConsumers := consumerNumber - len(consumers)
for i := 0; i < newConsumers; i++ {
reqQueueName, err := core_queue.GetRandReqQueue(reqQueues, crawlerName)
if err != nil {
utils.Error.Printf(err.Error())
continue
}
utils.Debug.Printf("GetRandReqQueue %s\n", reqQueueName)
respQueueName, err := core_queue.GetRandRespQueue(respQueues, crawlerName)
if err != nil {
utils.Error.Printf(err.Error())
continue
}
utils.Debug.Printf("GetRandRespQueue %s\n", respQueueName)
index := i / maxConn
c := core_consumer.CreateCoreConsumer(i, crawlerName, reqQueueName, respQueueName)
consumers = append(consumers, c)
utils.Debug.Println("add consumer", c)
go consumer(c, connPool, ssdbClients, httpClients[index], proxyClient)
}
if len(consumers) < consumerNumber {
utils.Warning.Printf("aliveConsumers now %d, lower than %d\n", len(consumers), consumerNumber)
}
utils.Debug.Println("total consumers ", len(consumers))
time.Sleep(1 * time.Second)
}
}