Beispiel #1
0
Datei: job.go Projekt: h8liu/d8
func (j *job) crawl() {
	c, e := client.New()
	if j.failOn(e) {
		return
	}

	j.quotas = j.makeQuotas()
	j.resChan = make(chan *task, 300)
	defer close(j.resChan)

	// launch the jobs
	go func() {
		for i, d := range j.domains {
			quota := <-j.quotas
			task := &task{
				domain: d,
				client: c,
				job:    j,
				quota:  quota,
				id:     i + 1,
			}
			go task.run()
		}
	}()

	j.writeOut()
}
Beispiel #2
0
func (self *Console) Main() {
	s := bufio.NewScanner(os.Stdin)
	if self.Term == nil {
		c, e := client.New()
		noError(e)
		self.Term = term.New(c)
		self.Term.Log = nil
		self.Term.Out = os.Stdout
	}

	for {
		fmt.Print("d8> ")
		if !s.Scan() {
			break
		}

		self.line(strings.TrimSpace(s.Text()))
		if self.Exit {
			break
		}
	}

	noError(s.Err())

	fmt.Println()
}
Beispiel #3
0
Datei: term.go Projekt: h8liu/d8
func makeStd() *Term {
	if std == nil {
		c, e := client.New()
		if e != nil {
			panic(e)
		}

		std = New(c)
		std.Log = os.Stdout
		std.Out = os.Stdout
	}

	return std
}
Beispiel #4
0
Datei: main.go Projekt: h8liu/d8
func single() {
	c, e := client.New()
	noError(e)
	t := term.New(c)
	t.Log = nil
	t.Out = os.Stdout

	for _, s := range os.Args[1:] {
		d, e := domain.Parse(s)
		if e != nil {
			fmt.Fprintln(os.Stderr, e)
			continue
		}
		fmt.Printf("// %v\n", d)

		_, e = t.T(tasks.NewInfo(d))
		if e != nil {
			fmt.Fprintln(os.Stderr, e)
		}
	}
}
Beispiel #5
0
func (self *Crawler) Crawl() error {
	// load input
	list, e := LoadList(self.In, self.Log)
	if e != nil {
		return e
	}

	c, e := client.New()
	if e != nil {
		return e
	}

	quotas := self.quotas()
	lock := new(sync.Mutex)

	idFmt := fmt.Sprintf("%%0%dd", digits(len(list)))

	for id, d := range list {
		q := <-quotas
		task := &crawlTask{
			id:          fmt.Sprintf(idFmt, id+1),
			domain:      d,
			client:      c,
			deflate:     self.Deflate,
			lock:        lock,
			quota:       q,
			quotaReturn: quotas,
		}

		go task.run()
	}

	// join
	for i := 0; i < cap(quotas); i++ {
		<-quotas
	}

	return nil
}