func CreateNullTime(v time.Time) pq.NullTime {
	var nt pq.NullTime
	nt.Valid = true
	nt.Time = v
	return nt
}
Exemple #2
0
// convertArt converts a slurped article into jl form
func convertArt(src *slurp.Article) (*jl.Article, []*jl.UnresolvedJourno) {
	now := time.Now()

	bestURL := src.CanonicalURL
	if bestURL == "" {
		bestURL = src.URLs[0]
	}

	pub := jl.NewPublication(src.Publication.Domain, src.Publication.Name)

	var pubDate pq.NullTime
	t, err := time.Parse(time.RFC3339, src.Published)
	if err == nil {
		pubDate.Time = t
		pubDate.Valid = true
	} // else pubDate is NULL

	wordCnt := sql.NullInt64{0, false}
	rawTxt := ""
	if src.Content != "" {
		// parse and render to raw text
		doc, err := html.Parse(strings.NewReader(src.Content))
		if err == nil {
			rawTxt = htmlutil.RenderNode(doc)
		}
	}

	tags := []jl.Tag{}
	if rawTxt != "" {
		// count words
		cnt := len(wordPat.FindAllString(rawTxt, -1))
		if cnt > 0 {
			wordCnt = sql.NullInt64{int64(cnt), true}
		}

		tags = jl.ExtractTagsFromText(rawTxt)
	}

	art := &jl.Article{
		ID:          0, // note: we generate our own IDs at the JL end
		Title:       src.Headline,
		Byline:      "",
		Description: "",
		Content:     src.Content,
		PubDate:     pubDate,
		FirstSeen:   now,
		LastSeen:    now,
		Permalink:   bestURL,
		SrcURL:      bestURL,
		URLs:        make([]string, len(src.URLs)),
		Publication: pub,
		LastScraped: pq.NullTime{now, true},

		WordCount: wordCnt,
		Status:    'a',
		Tags:      tags,
	}
	copy(art.URLs, src.URLs)

	// authors
	authors := []*jl.UnresolvedJourno{}
	for _, a := range src.Authors {
		u := &jl.UnresolvedJourno{
			Name: a.Name,
			// TODO: Email, rel-author, twitter, etc..
		}
		authors = append(authors, u)
	}
	return art, authors
}