Example #1
0
func (w *wrapper) fromURL(u *crawler.URL) {
	w.Scheme = u.URL.Scheme
	w.Host = u.URL.Host
	w.Path = u.URL.EscapedPath()
	w.Query = u.Query().Encode()
	w.Depth = u.Depth
	w.Done = u.Done
	w.Status = u.Status
	w.Last = u.Last
	w.NumVisit = u.NumVisit
	w.NumError = u.NumRetry
}
Example #2
0
func (s *LevelStore) Complete(u *url.URL) (err error) {
	tx, err := s.DB.OpenTransaction()
	if err != nil {
		return
	}
	commit := false
	defer func() {
		if !commit && err != nil {
			tx.Discard() // TODO: handle error
		}
	}()

	key := keyURL(u)
	v, err := tx.Get(key, nil)
	if err != nil {
		return
	}
	uu := crawler.URL{}
	if err = s.codec.Unmarshal(v, &uu); err != nil {
		return
	}
	uu.Done = true
	if v, err = s.codec.Marshal(&uu); err != nil {
		return
	}
	if err = tx.Put(key, v, nil); err != nil {
		return
	}
	if v, err = tx.Get(keyFinishCount, nil); err != nil {
		return
	}
	cnt := util.Btoi64(v) + 1
	if err = tx.Put(keyFinishCount, util.I64tob(cnt), nil); err == nil {
		commit = true
		err = tx.Commit()
	}
	return
}