func (j *job) crawl() { c, e := client.New() if j.failOn(e) { return } j.quotas = j.makeQuotas() j.resChan = make(chan *task, 300) defer close(j.resChan) // launch the jobs go func() { for i, d := range j.domains { quota := <-j.quotas task := &task{ domain: d, client: c, job: j, quota: quota, id: i + 1, } go task.run() } }() j.writeOut() }
func (self *Console) Main() { s := bufio.NewScanner(os.Stdin) if self.Term == nil { c, e := client.New() noError(e) self.Term = term.New(c) self.Term.Log = nil self.Term.Out = os.Stdout } for { fmt.Print("d8> ") if !s.Scan() { break } self.line(strings.TrimSpace(s.Text())) if self.Exit { break } } noError(s.Err()) fmt.Println() }
func makeStd() *Term { if std == nil { c, e := client.New() if e != nil { panic(e) } std = New(c) std.Log = os.Stdout std.Out = os.Stdout } return std }
func single() { c, e := client.New() noError(e) t := term.New(c) t.Log = nil t.Out = os.Stdout for _, s := range os.Args[1:] { d, e := domain.Parse(s) if e != nil { fmt.Fprintln(os.Stderr, e) continue } fmt.Printf("// %v\n", d) _, e = t.T(tasks.NewInfo(d)) if e != nil { fmt.Fprintln(os.Stderr, e) } } }
func (self *Crawler) Crawl() error { // load input list, e := LoadList(self.In, self.Log) if e != nil { return e } c, e := client.New() if e != nil { return e } quotas := self.quotas() lock := new(sync.Mutex) idFmt := fmt.Sprintf("%%0%dd", digits(len(list))) for id, d := range list { q := <-quotas task := &crawlTask{ id: fmt.Sprintf(idFmt, id+1), domain: d, client: c, deflate: self.Deflate, lock: lock, quota: q, quotaReturn: quotas, } go task.run() } // join for i := 0; i < cap(quotas); i++ { <-quotas } return nil }