示例#1
0
func (scraper Scraper) scrapeArticle(a *entity.Article) error {
	log.Println(scraper.host + a.URL)
	doc, err := goquery.NewDocument(scraper.host + a.URL)
	if err != nil {
		return err
	}

	articleNode := doc.Find("main article.article")
	contentNode := articleNode.Find(".article-content")

	if articleNode.Length() == 0 {
		log.Printf("article %s has no content", a.URL)
	}

	contentHTML, err := contentNode.Html()
	if err != nil {
		return err
	}

	a.Headline = strings.TrimSpace(articleNode.Find(".article--title").Text())
	a.Excerpt = strings.TrimSpace(contentNode.Find(".article--teaser").Text())
	a.Content = strings.TrimSpace(contentHTML)

	authorNode := articleNode.Find(".author .author--link")
	authorURL, _ := authorNode.Attr("href")
	authorName := strings.TrimSpace(authorNode.Text())

	idMatches := idRegex.FindStringSubmatch(authorURL)
	if len(idMatches) != 2 {
		log.Printf("couldn't parse id for author %s\n", authorURL)
	}

	// ID
	authorID, err := strconv.Atoi(idMatches[1])
	if err != nil {
		log.Printf("couldn't parse id for author %s\n", authorURL)
	}

	author := entity.Author{
		ID:   authorID,
		Name: authorName,
		URL:  authorURL,
	}
	scraper.db.Preload("Images").Preload("Crawl").FirstOrCreate(&author)

	if author.Crawl.ID == 0 {
		author.Crawl = entity.Crawl{Next: time.Now()}
	}

	scraper.db.Save(&author)

	a.Crawl.Next = time.Now().Add(time.Duration(float64(rand.Intn(18000))+30*time.Minute.Seconds()) * time.Second)
	a.AuthorID = author.ID
	scraper.db.Save(&a)

	return nil
}
示例#2
0
func (r Repository) SaveAllAuthors(authors []*entity.Author) error {
	tx := r.DB.Begin()
	for _, a := range authors {
		author := entity.Author{ID: a.ID}
		tx.Preload("Crawl").Preload("Images").FirstOrCreate(&author)

		author.Ordering = a.Ordering
		author.Name = a.Name
		author.Title = a.Title
		author.URL = a.URL

		for _, i := range a.Images {
			author.AddImage(i)
		}

		if author.Crawl.ID == 0 {
			author.Crawl = entity.Crawl{Next: time.Now()}
		}

		tx.Save(&author)
	}
	tx.Commit()

	return nil
}
示例#3
0
func (scraper Scraper) scrapeAuthor(a *entity.Author) error {
	log.Println(scraper.host + a.URL)
	doc, err := goquery.NewDocument(scraper.host + a.URL)
	if err != nil {
		return err
	}

	scraper.db.Preload("Images").Preload("Crawl").FirstOrCreate(&a)

	authorNode := doc.Find("main .island .author")
	imageNode := authorNode.Find("img.author__img")

	a.Biography = strings.TrimSpace(authorNode.Find("p").First().Text())

	html, err := authorNode.Find("p.meta").Html()
	if err != nil {
		return err
	}

	a.SocialMedia = strings.TrimSpace(html)
	var images []entity.Image
	if imageNode.Length() > 0 {
		srcset, _ := imageNode.Attr("srcset")
		images, err = ParseAuthorImages(srcset)
		if err != nil {
			return err
		}
	}

	for _, i := range images {
		a.AddImage(i)
	}

	a.Crawl.Next = time.Now().Add(time.Duration(float64(rand.Intn(18000))+30*time.Minute.Seconds()) * time.Second)
	scraper.db.Save(&a)

	return nil
}