func (this *Spider) Run() {
if this.threadnum == 0 {
this.threadnum = 1
}
this.mc = resource_manage.NewResourceManageChan(this.threadnum)
//init db by sorawa
for {
req := this.pScheduler.Poll()
// mc is not atomic
if this.mc.Has() == 0 && req == nil && this.exitWhenComplete {
mlog.StraceInst().Println("** executed callback **")
this.pPageProcesser.Finish()
mlog.StraceInst().Println("** end spider **")
break
} else if req == nil {
time.Sleep(500 * time.Millisecond)
//mlog.StraceInst().Println("scheduler is empty")
continue
}
this.mc.GetOne()
// Asynchronous fetching
go func(req *request.Request) {
defer this.mc.FreeOne()
//time.Sleep( time.Duration(rand.Intn(5)) * time.Second)
mlog.StraceInst().Println("start crawl : " + req.GetUrl())
this.pageProcess(req)
}(req)
}
this.close()
}
func TestResourceManage(t *testing.T) {
var mc *resource_manage.ResourceManageChan
mc = resource_manage.NewResourceManageChan(1)
mc.GetOne()
println("incr")
mc.FreeOne()
println("decr")
mc.GetOne()
println("incr")
}