func main() {
/*
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return errors.New("")
},
}
*/
for _, url := range urls {
str := fmt.Sprintf("http://kleinanzeigen.ebay.de%s", url)
//fmt.Println(str)
doc, err := loadDocument(str)
if err != nil {
panic(err)
}
c := C{}
offer, err := flatscan.GetOffer(doc, c)
if err != nil {
panic(err)
}
fmt.Printf("%+v\n", *offer)
}
}
func loadList(url string, c appengine.Context, s []Setting) (i int, err error) {
i = 0
client := urlfetch.Client(c)
var doc *goquery.Document
doc, err = LoadDocumentGAE(url, client)
if err != nil {
return
}
for _, offerPath := range flatscan.ExtractLinks(doc) {
md5Writer := md5.New()
io.WriteString(md5Writer, offerPath)
md5Sum := fmt.Sprintf("%x", md5Writer.Sum(nil))
key := datastore.NewKey(c, entitiyFlatOffer, md5Sum, 0, nil)
amount, err := CheckAmountGAE(key, c)
if amount > 0 || err != nil {
i++
continue
}
offerUrl := fmt.Sprintf("%s%s", base, offerPath)
doc, err = LoadDocumentGAE(offerUrl, client)
if err != nil {
c.Errorf(err.Error())
i++
continue
}
offer, err := flatscan.GetOffer(doc, c)
if err != nil {
c.Errorf("%s: %s", err.Error(), offerPath)
i++
continue
}
offer.Url = offerPath
c.Infof("Title: '%s', url: %s", offer.Title, offer.Url)
offer.ID = md5Sum
for _, setting := range s {
b, err := setting.CheckOffer(*offer)
if b {
buf := bytes.NewBufferString("")
err = emailTemplate.Execute(buf, offer)
if err != nil {
c.Errorf(err.Error())
}
msg := &mail.Message{
Sender: "Flat Scan Sender <*****@*****.**>",
To: []string{setting.Email},
Subject: "Found a Flat",
Body: buf.String(),
}
c.Infof(msg.Body)
err = mail.Send(c, msg)
if err != nil {
c.Errorf(err.Error())
sendErrorMail(c, err)
//return i, err
} else {
c.Infof("Mail sent %+v", *msg)
}
}
}
key, err = datastore.Put(c, key, offer)
if err != nil {
c.Errorf(err.Error())
sendErrorMail(c, err)
//return i, err
}
}
return i, nil
}