Golang PageItems.GetAll Examples

Example #1

File: pipeline_console.go Project: ZuochaoLee/visual_spider_go

func (this *PipelineConsole) Process(items *page_items.PageItems, t com_interfaces.Task) {
	println("----------------------------------------------------------------------------------------------")
	println("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
	println("Crawled result : ")
	for key, value := range items.GetAll() {
		println(key + "\t:\t" + value)
	}
}

Example #2

File: pipeline_file.go Project: ZuochaoLee/visual_spider_go

func (this *PipelineFile) Process(items *page_items.PageItems, t com_interfaces.Task) {
	this.pFile.WriteString("--------------------------------------------------------\n")
	this.pFile.WriteString("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
	this.pFile.WriteString("Crawled result : \n")
	for key, value := range items.GetAll() {
		this.pFile.WriteString(key + "\t:\t" + value + "\n")
	}
}

Example #3

File: pipline_redis.go Project: ZuochaoLee/visual_spider_go

func (this *PipelineRedis) Process(items *page_items.PageItems, t com_interfaces.Task) {
	//r := rand.New(rand.NewSource(time.Now().UnixNano()))
	//ts := r.Intn(100)
	//ta := time.Now().Unix()
	//tt := strconv.FormatInt(ta, 10) + strconv.Itoa(ts)
	body, _ := json.Marshal(items.GetAll())
	println("*******************")
	println(body)
	this.client.Lpush("list", body)
}

Example #4

File: pipeline_kafka.go Project: ZuochaoLee/visual_spider_go

func (this *PipelineKafka) Process(items *page_items.PageItems, t com_interfaces.Task) {
	res := ""
	for k, v := range items.GetAll() {
		res = res + k + ":" + v + ","
	}
	var message sarama.ProducerMessage
	message.Topic = this.db
	message.Value = sarama.StringEncoder(res)
	this.producer.Input() <- &message
}