Example #1
0
func (t *TechCrunch) Posts(url string) (posts []models.Post) {
	doc := t.Load(url)

	channel := make(chan models.Post)
	s := doc.Find("ul.river li .block-content")

	for i := range s.Nodes {
		go func(s *goquery.Selection) {
			title := s.Find("h2.post-title").Text()
			url, exists := s.Find("h2.post-title > a").Attr("href")

			post := models.Post{Title: title, Url: url, Origin: name}
			if exists {
				t.Meta(url, &post)
			}
			t.debug("%s", post.Json())

			channel <- post
		}(s.Eq(i))
	}

	for {
		select {
		case p := <-channel:
			posts = append(posts, p)
			t.debug("Length types.Posts: %d", len(posts))
			t.debug("Length Nodes: %d", len(s.Nodes))
			if len(s.Nodes) == len(posts) {
				t.spinner.Stop()
				return posts
			}
		case <-time.After(50 * time.Millisecond):
			t.spinner.Start()
		}
	}
}
Example #2
0
func (t *TechCrunch) Meta(url string, post *models.Post) {
	doc := t.Load(url)

	t.debug("Loading: %s", url)

	m := new(models.Meta)

	doc.Find("meta").Each(func(i int, s *goquery.Selection) {
		t, e := s.Attr("name")
		if e {
			val, _ := s.Attr("content")
			switch t {
			case "tag":
				m.Tags = append(m.Tags, val)
				break
			case "category":
				m.Categories = append(m.Categories, val)
				break
			case "timestamp":
				t, _ := time.Parse(layout, val)
				m.Time = t
				break
			case "author":
				m.Author = val
				break
			case "description":
				m.Description = val
				break
			case "content":
				m.Content = val
				break
			}

		}
	})

	post.Meta = *m
}