func TestPushKey(t *testing.T) {
queueName := "test_crawler"
req := initRequest()
r := initResponse(req)
cfg, err := config.Get("/etc/yascrapy/core.json")
if err != nil {
t.Error(err.Error())
}
conn, err := rabbitmq.GetConn(cfg)
if err != nil {
t.Error(err.Error())
}
err = rabbitmq.DeclareQueue(conn, queueName)
if err != nil {
t.Error(err.Error())
}
ch, err := rabbitmq.MakeChannel(conn)
defer ch.Close()
if err != nil {
t.Error("TestPushKey fail %s", err.Error())
}
err = PushKey(ch, r, queueName)
if err != nil {
t.Error("TestPushKey fail ", err.Error())
} else {
t.Log("TestPushKey ok")
}
}
func BenchmarkPushKey(b *testing.B) {
queueName := "test_crawler"
cfg, err := config.Get("/etc/yascrapy/core.json")
if err != nil {
b.Error(err.Error())
}
conn, err := rabbitmq.GetConn(cfg)
if err != nil {
b.Error(err.Error())
}
err = rabbitmq.DeclareQueue(conn, queueName)
if err != nil {
b.Error(err.Error())
}
ch, err := rabbitmq.MakeChannel(conn)
defer ch.Close()
if err != nil {
b.Error("TestPush fail", err.Error())
}
for i := 0; i < b.N; i++ {
req := initRequest()
req.Url = fmt.Sprintf("%s%d", req.Url, i)
resp := initResponse(req)
err = PushKey(ch, resp, queueName)
if err != nil {
b.Log(err.Error())
}
}
}
func TestCoreConsumer(t *testing.T) {
cfg, err := config.Get("/etc/yascrapy/core.json")
if err != nil {
t.Error(err.Error())
}
var consumers []*CoreConsumer
var conn *amqp.Connection
conn, err = rabbitmq.GetConn(cfg)
if err != nil {
t.Error("rabbitmq GetConn fail", err.Error())
}
queues, err := core_queue.GetQueues("http_request:", cfg)
t.Log("queues cnt", len(queues))
if err != nil {
t.Error("Get Queues fail", err.Error())
}
queues, err = core_queue.UpdateQueueCount(queues, conn)
if err != nil {
t.Error("UpdateQueueCount fail", err.Error())
}
queues = core_queue.RemoveEmptyQueues(queues)
t.Log("queues cnt", len(queues))
rand.Seed(time.Now().Unix())
consumers = AliveCoreConsumers(consumers)
respQueues, err := core_queue.GetQueues("http_response:", cfg)
if err != nil {
t.Error("http response error", err.Error())
}
for i := 0; i < cfg.RabbitmqConsumers; i++ {
reqQueue, err := core_queue.GetRandReqQueue(queues)
if err != nil {
t.Error(err.Error())
}
crawler := core_queue.GetCrawler(reqQueue, "http_request:")
respQueueName, err := core_queue.GetRandRespQueue(respQueues, crawler)
aliveReqConsumers := AliveReqConsumers(reqQueue.Name, consumers)
aliveRespConsumers := AliveRespConsumers(respQueueName, consumers)
if err != nil {
t.Error(err.Error())
}
t.Log("req queue", reqQueue.Name, "resp queue", respQueueName)
t.Log("aliveReqConsumers ", len(aliveReqConsumers), "aliveRespConsumers", len(aliveRespConsumers))
c := CreateCoreConsumer(i, crawler, reqQueue.Name, respQueueName)
consumers = append(consumers, c)
}
t.Error("to see output")
}