func (this *PipelineConsole) Process(items *page_items.PageItems, t com_interfaces.Task) {
	println("----------------------------------------------------------------------------------------------")
	println("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
	println("Crawled result : ")
	for key, value := range items.GetAll() {
		println(key + "\t:\t" + value)
	}
}
Exemplo n.º 2
0
func (this *PipelineFile) Process(items *page_items.PageItems, t com_interfaces.Task) {
	this.pFile.WriteString("--------------------------------------------------------\n")
	this.pFile.WriteString("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
	this.pFile.WriteString("Crawled result : \n")
	for key, value := range items.GetAll() {
		this.pFile.WriteString(key + "\t:\t" + value + "\n")
	}
}
Exemplo n.º 3
0
func (this *PipelineRedis) Process(items *page_items.PageItems, t com_interfaces.Task) {
	//r := rand.New(rand.NewSource(time.Now().UnixNano()))
	//ts := r.Intn(100)
	//ta := time.Now().Unix()
	//tt := strconv.FormatInt(ta, 10) + strconv.Itoa(ts)
	body, _ := json.Marshal(items.GetAll())
	println("*******************")
	println(body)
	this.client.Lpush("list", body)
}
Exemplo n.º 4
0
func (this *PipelineKafka) Process(items *page_items.PageItems, t com_interfaces.Task) {
	res := ""
	for k, v := range items.GetAll() {
		res = res + k + ":" + v + ","
	}
	var message sarama.ProducerMessage
	message.Topic = this.db
	message.Value = sarama.StringEncoder(res)
	this.producer.Input() <- &message
}