func CreateNullTime(v time.Time) pq.NullTime {
var nt pq.NullTime
nt.Valid = true
nt.Time = v
return nt
}
// convertArt converts a slurped article into jl form
func convertArt(src *slurp.Article) (*jl.Article, []*jl.UnresolvedJourno) {
now := time.Now()
bestURL := src.CanonicalURL
if bestURL == "" {
bestURL = src.URLs[0]
}
pub := jl.NewPublication(src.Publication.Domain, src.Publication.Name)
var pubDate pq.NullTime
t, err := time.Parse(time.RFC3339, src.Published)
if err == nil {
pubDate.Time = t
pubDate.Valid = true
} // else pubDate is NULL
wordCnt := sql.NullInt64{0, false}
rawTxt := ""
if src.Content != "" {
// parse and render to raw text
doc, err := html.Parse(strings.NewReader(src.Content))
if err == nil {
rawTxt = htmlutil.RenderNode(doc)
}
}
tags := []jl.Tag{}
if rawTxt != "" {
// count words
cnt := len(wordPat.FindAllString(rawTxt, -1))
if cnt > 0 {
wordCnt = sql.NullInt64{int64(cnt), true}
}
tags = jl.ExtractTagsFromText(rawTxt)
}
art := &jl.Article{
ID: 0, // note: we generate our own IDs at the JL end
Title: src.Headline,
Byline: "",
Description: "",
Content: src.Content,
PubDate: pubDate,
FirstSeen: now,
LastSeen: now,
Permalink: bestURL,
SrcURL: bestURL,
URLs: make([]string, len(src.URLs)),
Publication: pub,
LastScraped: pq.NullTime{now, true},
WordCount: wordCnt,
Status: 'a',
Tags: tags,
}
copy(art.URLs, src.URLs)
// authors
authors := []*jl.UnresolvedJourno{}
for _, a := range src.Authors {
u := &jl.UnresolvedJourno{
Name: a.Name,
// TODO: Email, rel-author, twitter, etc..
}
authors = append(authors, u)
}
return art, authors
}
