func (sd *scheduler) retry(ctx *Context) (*queue.Item, bool, error) {
defer ctx.free()
var cnt int
if err := sd.cw.store.UpdateFunc(ctx.url, func(uu *URL) {
uu.NumRetry++
cnt = uu.NumRetry
}); err != nil {
return nil, false, err
}
delay, max := sd.cw.ctrl.Retry(ctx)
if cnt >= max {
sd.logger.Error(
"exceed maximum number of retries",
"err", ctx.err, "url", ctx.url, "retries", cnt,
)
err := sd.cw.store.Complete(ctx.url)
return nil, false, err
}
sd.logger.Error(
"retry due to error",
"err", ctx.err, "url", ctx.url, "retries", cnt,
)
item := queue.NewItem()
item.URL = ctx.url
item.Ctx = ctx.C
item.Next = time.Now().Add(delay)
return item, true, nil
}
func (sd *scheduler) sched(r *Response, u *url.URL) *queue.Item {
item := queue.NewItem()
item.URL = u
t := sd.cw.ctrl.Sched(r, u)
if t.Ctx == nil {
t.Ctx = context.Background()
}
item.Next, item.Score, item.Ctx = t.At, t.Score, t.Ctx
return item
}
func (sd *scheduler) resched(r *Response) (
item *queue.Item, done bool, err error,
) {
defer func() {
r.ctx.free()
r.free()
}()
var last time.Time
if err = sd.cw.store.UpdateFunc(r.URL, func(u *URL) {
u.NumVisit++
u.NumRetry = 0
last = u.Last
u.Last = r.Timestamp
}); err != nil {
return
}
var t Ticket
done, t = sd.cw.ctrl.Resched(r)
if done {
err = sd.cw.store.Complete(r.URL)
return
} else if t.Ctx == nil {
t.Ctx = context.Background()
}
item = queue.NewItem()
item.URL = r.URL
item.Next, item.Score, item.Ctx = t.At, t.Score, t.Ctx
min := last.Add(sd.cw.opt.MinDelay)
if item.Next.Before(min) {
item.Next = min
}
return
}
func (e *element) decode(b []byte) error {
e.item = queue.NewItem()
return json.Unmarshal(b, e.item)
}