func (j *job) crawl() {
c, e := client.New()
if j.failOn(e) {
return
}
j.quotas = j.makeQuotas()
j.resChan = make(chan *task, 300)
defer close(j.resChan)
// launch the jobs
go func() {
for i, d := range j.domains {
quota := <-j.quotas
task := &task{
domain: d,
client: c,
job: j,
quota: quota,
id: i + 1,
}
go task.run()
}
}()
j.writeOut()
}
func (self *Console) Main() {
s := bufio.NewScanner(os.Stdin)
if self.Term == nil {
c, e := client.New()
noError(e)
self.Term = term.New(c)
self.Term.Log = nil
self.Term.Out = os.Stdout
}
for {
fmt.Print("d8> ")
if !s.Scan() {
break
}
self.line(strings.TrimSpace(s.Text()))
if self.Exit {
break
}
}
noError(s.Err())
fmt.Println()
}
func makeStd() *Term {
if std == nil {
c, e := client.New()
if e != nil {
panic(e)
}
std = New(c)
std.Log = os.Stdout
std.Out = os.Stdout
}
return std
}
func single() {
c, e := client.New()
noError(e)
t := term.New(c)
t.Log = nil
t.Out = os.Stdout
for _, s := range os.Args[1:] {
d, e := domain.Parse(s)
if e != nil {
fmt.Fprintln(os.Stderr, e)
continue
}
fmt.Printf("// %v\n", d)
_, e = t.T(tasks.NewInfo(d))
if e != nil {
fmt.Fprintln(os.Stderr, e)
}
}
}
func (self *Crawler) Crawl() error {
// load input
list, e := LoadList(self.In, self.Log)
if e != nil {
return e
}
c, e := client.New()
if e != nil {
return e
}
quotas := self.quotas()
lock := new(sync.Mutex)
idFmt := fmt.Sprintf("%%0%dd", digits(len(list)))
for id, d := range list {
q := <-quotas
task := &crawlTask{
id: fmt.Sprintf(idFmt, id+1),
domain: d,
client: c,
deflate: self.Deflate,
lock: lock,
quota: q,
quotaReturn: quotas,
}
go task.run()
}
// join
for i := 0; i < cap(quotas); i++ {
<-quotas
}
return nil
}