func (this *PipelineConsole) Process(items *page_items.PageItems, t com_interfaces.Task) {
println("----------------------------------------------------------------------------------------------")
println("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
println("Crawled result : ")
for key, value := range items.GetAll() {
println(key + "\t:\t" + value)
}
}
func (this *PipelineFile) Process(items *page_items.PageItems, t com_interfaces.Task) {
this.pFile.WriteString("--------------------------------------------------------\n")
this.pFile.WriteString("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
this.pFile.WriteString("Crawled result : \n")
for key, value := range items.GetAll() {
this.pFile.WriteString(key + "\t:\t" + value + "\n")
}
}
func (this *PipelineRedis) Process(items *page_items.PageItems, t com_interfaces.Task) {
//r := rand.New(rand.NewSource(time.Now().UnixNano()))
//ts := r.Intn(100)
//ta := time.Now().Unix()
//tt := strconv.FormatInt(ta, 10) + strconv.Itoa(ts)
body, _ := json.Marshal(items.GetAll())
println("*******************")
println(body)
this.client.Lpush("list", body)
}
func (this *PipelineKafka) Process(items *page_items.PageItems, t com_interfaces.Task) {
res := ""
for k, v := range items.GetAll() {
res = res + k + ":" + v + ","
}
var message sarama.ProducerMessage
message.Topic = this.db
message.Value = sarama.StringEncoder(res)
this.producer.Input() <- &message
}
