func (w *wrapper) fromURL(u *crawler.URL) {
w.Scheme = u.URL.Scheme
w.Host = u.URL.Host
w.Path = u.URL.EscapedPath()
w.Query = u.Query().Encode()
w.Depth = u.Depth
w.Done = u.Done
w.Status = u.Status
w.Last = u.Last
w.NumVisit = u.NumVisit
w.NumError = u.NumRetry
}
func (s *LevelStore) Complete(u *url.URL) (err error) {
tx, err := s.DB.OpenTransaction()
if err != nil {
return
}
commit := false
defer func() {
if !commit && err != nil {
tx.Discard() // TODO: handle error
}
}()
key := keyURL(u)
v, err := tx.Get(key, nil)
if err != nil {
return
}
uu := crawler.URL{}
if err = s.codec.Unmarshal(v, &uu); err != nil {
return
}
uu.Done = true
if v, err = s.codec.Marshal(&uu); err != nil {
return
}
if err = tx.Put(key, v, nil); err != nil {
return
}
if v, err = tx.Get(keyFinishCount, nil); err != nil {
return
}
cnt := util.Btoi64(v) + 1
if err = tx.Put(keyFinishCount, util.I64tob(cnt), nil); err == nil {
commit = true
err = tx.Commit()
}
return
}
