Golang Spider Examples

Example #1

File: basic.go Project: gnagel/spider

// addSpider adds a spider.Spider to the list of spiders.
func (bs *BasicScheduler) addSpider(s spider.Spider, schedule spider.SpiderScheduler, baseCtx *spider.Context) (spider.Spider, error) {
	bs.Lock()
	defer bs.Unlock()
	bs.schedules[s] = schedule
	ctx, err := s.Setup(baseCtx)
	if err != nil {
		return nil, err
	}
	bs.contexts[s] = ctx
	return s, nil
}

Example #2

File: basic.go Project: gnagel/spider

func (bs *BasicScheduler) spin(s spider.Spider, baseCtx *spider.Context, errChan chan error) {
	if err := s.Spin(baseCtx); err != nil {
		errChan <- err
	}
}