func ParseHtmlComments(p *post.Post) (err error) {
if p.WebPostId == "" {
return errors.New(fmt.Sprintf("p.WebPostId is empty in post '%s'", p.String("PC: ")))
}
// Get comments from hackernews
geturl := fmt.Sprintf("http://news.ycombinator.com/item?id=%s", p.WebPostId)
// DEBUG
//geturl := fmt.Sprintf("https://news.ycombinator.com/item?id=9751858")
if DebugLevel > 2 {
fmt.Printf("START GET COMMENTS FROM '%s'\n", geturl)
}
body, err := GetHtmlBody(geturl)
if err != nil {
return errors.New("GetHtmlBody: " + err.Error())
}
// Create a qoquery document to parse from an io.Reader
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return errors.New("Failed to parse HTML: " + err.Error())
}
// Find hackernews comments = elements with class "athing"
thing := doc.Find(".athing")
for iThing := range thing.Nodes {
// use `singlecomment` as a selection of one single post
singlecomment := thing.Eq(iThing)
comment := post.NewComment()
//p.Comments = append(p.Comments, &comment)
comheads := singlecomment.Find(".comhead a")
for i := range comheads.Nodes {
comhead := comheads.Eq(i)
t, _ := comhead.Html()
s, exists := comhead.Attr("href")
if exists {
if strings.HasPrefix(s, "user?id") {
comment.User = t
continue
}
if strings.HasPrefix(s, "item?id") {
if strings.Contains(t, "ago") {
var commentDate time.Time
commentDate, err = GetDateFromCreatedAgo(t)
if err != nil {
comment.Err = errors.New(fmt.Sprintf("Failed to convert to date: %s: %s\n", t, err.Error()))
err = nil
continue
}
comment.CommentDate = commentDate
if len(strings.Split(s, "=")) > 1 {
comment.WebCommentId = strings.Split(s, "=")[1]
}
//comment.Err = err
}
}
}
comments := singlecomment.Find("span.comment")
removeReplySelection := comments.Find("span div.reply")
removeReplySelection.Remove()
var sep string
for iComment, _ := range comments.Nodes {
s := comments.Eq(iComment)
h, _ := s.Html()
if !utf8.ValidString(s.Text()) {
comment.Err = errors.New(fmt.Sprintf("Ignoring invalid UTF-8: '%s'", s.Text()))
break
}
h, err = HtmlToMarkdown(h)
if err != nil {
comment.Err = errors.New(fmt.Sprintf("Ignoring markdownifier: '%s'", err.Error()))
break
}
if h != "" {
comment.Body = comment.Body + sep + h
}
sep = "\n"
}
//fmt.Printf("POST %s BODY = %s\n", p.WebPostId, comment.Body)
if comment.Err == nil && len(comment.WebCommentId) > 0 && len(comment.Body) > 0 {
p.Comments = append(p.Comments, &comment)
} else {
p.CommentParseErrors = append(p.CommentParseErrors, &comment)
}
}
}
if DebugLevel > 0 {
fmt.Printf("GET COMMENTS FROM '%s' yielded %d comments\n", geturl, len(p.Comments))
}
return err
}
