Пример #1
0
func (sd *scheduler) retry(ctx *Context) (*queue.Item, bool, error) {
	defer ctx.free()

	var cnt int
	if err := sd.cw.store.UpdateFunc(ctx.url, func(uu *URL) {
		uu.NumRetry++
		cnt = uu.NumRetry
	}); err != nil {
		return nil, false, err
	}

	delay, max := sd.cw.ctrl.Retry(ctx)
	if cnt >= max {
		sd.logger.Error(
			"exceed maximum number of retries",
			"err", ctx.err, "url", ctx.url, "retries", cnt,
		)
		err := sd.cw.store.Complete(ctx.url)
		return nil, false, err
	}

	sd.logger.Error(
		"retry due to error",
		"err", ctx.err, "url", ctx.url, "retries", cnt,
	)
	item := queue.NewItem()
	item.URL = ctx.url
	item.Ctx = ctx.C
	item.Next = time.Now().Add(delay)
	return item, true, nil
}
Пример #2
0
func (sd *scheduler) sched(r *Response, u *url.URL) *queue.Item {
	item := queue.NewItem()
	item.URL = u
	t := sd.cw.ctrl.Sched(r, u)
	if t.Ctx == nil {
		t.Ctx = context.Background()
	}
	item.Next, item.Score, item.Ctx = t.At, t.Score, t.Ctx
	return item
}
Пример #3
0
func (sd *scheduler) resched(r *Response) (
	item *queue.Item, done bool, err error,
) {
	defer func() {
		r.ctx.free()
		r.free()
	}()

	var last time.Time
	if err = sd.cw.store.UpdateFunc(r.URL, func(u *URL) {
		u.NumVisit++
		u.NumRetry = 0
		last = u.Last
		u.Last = r.Timestamp
	}); err != nil {
		return
	}

	var t Ticket
	done, t = sd.cw.ctrl.Resched(r)
	if done {
		err = sd.cw.store.Complete(r.URL)
		return
	} else if t.Ctx == nil {
		t.Ctx = context.Background()
	}

	item = queue.NewItem()
	item.URL = r.URL
	item.Next, item.Score, item.Ctx = t.At, t.Score, t.Ctx
	min := last.Add(sd.cw.opt.MinDelay)
	if item.Next.Before(min) {
		item.Next = min
	}
	return
}
Пример #4
0
func (e *element) decode(b []byte) error {
	e.item = queue.NewItem()
	return json.Unmarshal(b, e.item)
}