/
indeed_test.go
72 lines (63 loc) · 1.64 KB
/
indeed_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
package main
import (
eatonconfig "github.com/ECLabs/Eaton-Feeder/config"
eatonevents "github.com/ECLabs/Eaton-Feeder/events"
"testing"
)
func TestIndeedClientPipeline(t *testing.T) {
eatonconfig.KafkaLoggerTopic = "logs-test"
err := eatonevents.Init()
if err != nil {
t.Fatal("failed to create event publisher: ", err)
}
i := new(IndeedClient)
scraper := new(IndeedScraper)
k, err := NewKafkaProducer()
if err != nil {
t.Fatal("failed to create new kafka producer: ", err)
}
c, err := NewKafkaConsumer()
if err != nil {
t.Fatal("failed to create new kafka consumer: ", err)
}
go func() {
consumerErrorChannel := c.ConsumeMessages()
for err := range consumerErrorChannel {
t.Log("failed to consume msg: ", err)
}
}()
errChannel, jobResultChannel := i.GetResults()
scraperErrChannel, scraperJobResultChannel := scraper.GetFullJobSummary(jobResultChannel)
kafkaErrChannel, kafkaDoneChannel := k.SendMessages(scraperJobResultChannel)
go func() {
for err := range errChannel {
t.Log("error thrown while polling indeed api: ", err)
}
}()
go func() {
for err := range kafkaErrChannel {
t.Fatal("Kafka ERROR: ", err)
}
}()
go func() {
for jobResult := range scraperJobResultChannel {
t.Log("Received JobResult: ", jobResult.JobKey)
if jobResult.IsLast() {
t.Log("and that was the last job result!")
}
}
}()
go func() {
for err := range scraperErrChannel {
t.Fatal("scraper error: ", err)
}
}()
go func() {
for jobResult := range jobResultChannel {
t.Log("jobResult: ", jobResult)
}
}()
for kafkaDone := range kafkaDoneChannel {
t.Log("finished sending to kafka!", kafkaDone)
}
}