示例#1
0
func (self *PipelineFile) Process(items *robot.PageItems, t robot.Task) {
	self.file.WriteString("----------------------------------------------------------------------------------------------\n")
	self.file.WriteString("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
	self.file.WriteString("Crawled result : \n")
	for key, value := range items.GetAll() {
		self.file.WriteString(key + "\t:\t" + value + "\n")
	}
}
示例#2
0
func (self *PipelineConsole) Process(items *robot.PageItems, t robot.Task) {
	println("----------------------------------------------------------------------------------------------")
	println("Crawled url :\t" + items.GetRequest().GetUrl() + "\n")
	println("Crawled result : ")
	for key, value := range items.GetAll() {
		println(key + "\t:\t" + value)
	}
}
示例#3
0
文件: pipeline.go 项目: kjfcpua/robot
//添加小说表
func (self *PipelineMySQL) novelProcess(pageitems *robot.PageItems, task robot.Task, firstid, secondid int64, picname string) (int64, error) {
	items := pageitems.GetAll()
	o := orm.NewOrm()
	novel := &Novel{
		Title:        items["title"],
		Firstid:      firstid,
		Secondid:     secondid,
		Author:       items["author"],
		Introduction: items["introduction"],
		Picture:      picname,
		Novelsource:  items["novelsource"],
		Createtime:   time.Now(),
	}
	//如果数据不存在 则创建
	_, id, err := o.ReadOrCreate(novel, "novelsource")
	return id, err
}
示例#4
0
文件: pipeline.go 项目: kjfcpua/robot
func (self *PipelineMySQL) contentProcess(pageitems *robot.PageItems, task robot.Task, novelid, firstid, secondid int64) {
	items := pageitems.GetAll()
	chapter, _ := strconv.Atoi(items["chapter"])
	o := orm.NewOrm()
	content := &Content{
		Novelid:       novelid,
		Title:         items["title"],
		Firstid:       firstid,
		Secondid:      secondid,
		Chapter:       chapter,
		Subtitle:      items["subtitle"],
		Text:          items["content"],
		Contentsource: items["contenturl"],
		Createtime:    time.Now(),
	}
	//如果数据不存在 则创建
	_, _, err := o.ReadOrCreate(content, "contentsource")
	if err == nil {
		log.Println("创建content成功 subtitle:", items["subtitle"])
	}
}