示例#1
0
文件: main.go 项目: pakohan/go-libs
func main() {
	/*
		client := &http.Client{
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return errors.New("")
			},
		}
	*/

	for _, url := range urls {
		str := fmt.Sprintf("http://kleinanzeigen.ebay.de%s", url)
		//fmt.Println(str)
		doc, err := loadDocument(str)

		if err != nil {
			panic(err)
		}

		c := C{}

		offer, err := flatscan.GetOffer(doc, c)
		if err != nil {
			panic(err)
		}

		fmt.Printf("%+v\n", *offer)
	}
}
示例#2
0
func loadList(url string, c appengine.Context, s []Setting) (i int, err error) {
	i = 0
	client := urlfetch.Client(c)
	var doc *goquery.Document
	doc, err = LoadDocumentGAE(url, client)
	if err != nil {
		return
	}

	for _, offerPath := range flatscan.ExtractLinks(doc) {
		md5Writer := md5.New()
		io.WriteString(md5Writer, offerPath)
		md5Sum := fmt.Sprintf("%x", md5Writer.Sum(nil))

		key := datastore.NewKey(c, entitiyFlatOffer, md5Sum, 0, nil)

		amount, err := CheckAmountGAE(key, c)

		if amount > 0 || err != nil {
			i++
			continue
		}

		offerUrl := fmt.Sprintf("%s%s", base, offerPath)
		doc, err = LoadDocumentGAE(offerUrl, client)
		if err != nil {
			c.Errorf(err.Error())
			i++
			continue
		}

		offer, err := flatscan.GetOffer(doc, c)
		if err != nil {
			c.Errorf("%s: %s", err.Error(), offerPath)
			i++
			continue
		}

		offer.Url = offerPath
		c.Infof("Title: '%s', url: %s", offer.Title, offer.Url)

		offer.ID = md5Sum

		for _, setting := range s {
			b, err := setting.CheckOffer(*offer)
			if b {
				buf := bytes.NewBufferString("")
				err = emailTemplate.Execute(buf, offer)
				if err != nil {
					c.Errorf(err.Error())
				}

				msg := &mail.Message{
					Sender:  "Flat Scan Sender <*****@*****.**>",
					To:      []string{setting.Email},
					Subject: "Found a Flat",
					Body:    buf.String(),
				}

				c.Infof(msg.Body)

				err = mail.Send(c, msg)
				if err != nil {
					c.Errorf(err.Error())
					sendErrorMail(c, err)
					//return i, err
				} else {
					c.Infof("Mail sent %+v", *msg)
				}
			}
		}

		key, err = datastore.Put(c, key, offer)
		if err != nil {
			c.Errorf(err.Error())
			sendErrorMail(c, err)
			//return i, err
		}
	}

	return i, nil
}