func NewProxyHTTP(l webfw.Logger, urlTemplate string) (ProxyHTTP, error) {
l.Infof("URL Template: %s\n", urlTemplate)
t, err := template.New("proxy-http-url-template").Parse(urlTemplate)
if err != nil {
return ProxyHTTP{}, err
}
return ProxyHTTP{logger: l, urlTemplate: t}, nil
}
func getReadeefUser(repo content.Repo, md5hex string, log webfw.Logger) content.User {
md5, err := hex.DecodeString(md5hex)
if err != nil {
log.Printf("Error decoding hex api_key")
return nil
}
user := repo.UserByMD5Api(md5)
if user.HasErr() {
log.Printf("Error getting user by md5api field: %v\n", user.Err())
return nil
}
return user
}
func NewSearchIndex(repo content.Repo, config Config, logger webfw.Logger) (SearchIndex, error) {
var err error
var index bleve.Index
si := SearchIndex{}
_, err = os.Stat(config.SearchIndex.BlevePath)
if err == nil {
logger.Infoln("Opening search index " + config.SearchIndex.BlevePath)
index, err = bleve.Open(config.SearchIndex.BlevePath)
if err != nil {
return EmptySearchIndex, errors.New(fmt.Sprintf("Error opening search index: %v\n", err))
}
} else if os.IsNotExist(err) {
mapping := bleve.NewIndexMapping()
docMapping := bleve.NewDocumentMapping()
idfieldmapping := bleve.NewTextFieldMapping()
idfieldmapping.IncludeInAll = false
docMapping.AddFieldMappingsAt("FeedId", idfieldmapping)
docMapping.AddFieldMappingsAt("ArticleId", idfieldmapping)
mapping.AddDocumentMapping(mapping.DefaultType, docMapping)
logger.Infoln("Creating search index " + config.SearchIndex.BlevePath)
index, err = bleve.New(config.SearchIndex.BlevePath, mapping)
if err != nil {
return EmptySearchIndex, errors.New(fmt.Sprintf("Error creating search index: %v\n", err))
}
si.newIndex = true
} else {
return EmptySearchIndex, errors.New(
fmt.Sprintf("Error getting stat of '%s': %v\n", config.SearchIndex.BlevePath, err))
}
si.logger = logger
si.repo = repo
si.Index = index
si.batchSize = config.SearchIndex.BatchSize
return si, nil
}
func insertThumbnailTarget(d *goquery.Document, thumbnailLink string, logger webfw.Logger) bool {
changed := false
if d.Find(".top-image").Length() > 0 {
return changed
}
thumbDoc, err := goquery.NewDocumentFromReader(strings.NewReader(fmt.Sprintf(`<img src="%s">`, thumbnailLink)))
if err != nil {
logger.Infof("Error generating thumbnail image node: %v\n", err)
return changed
}
d.Find("body").PrependSelection(thumbDoc.Find("img"))
changed = true
return changed
}
func updateArticle(a content.Article, tx *sqlx.Tx, db *db.DB, logger webfw.Logger) {
if a.HasErr() {
return
}
if err := a.Validate(); err != nil {
a.Err(err)
return
}
logger.Infof("Updating article %s\n", a)
d := a.Data()
s := db.SQL()
stmt, err := tx.Preparex(s.Article.Update)
if err != nil {
a.Err(err)
return
}
defer stmt.Close()
res, err := stmt.Exec(d.Title, d.Description, d.Date, d.Guid, d.Link, d.FeedId)
if err != nil {
a.Err(err)
return
}
if num, err := res.RowsAffected(); err != nil && err == sql.ErrNoRows || num == 0 {
logger.Infof("Creating article %s\n", a)
aId, err := db.CreateWithId(tx, s.Article.Create, d.FeedId, d.Link, d.Guid,
d.Title, d.Description, d.Date)
if err != nil {
a.Err(fmt.Errorf("Error updating article %s (guid - %v, link - %s): %v", a, d.Guid, d.Link, err))
return
}
d.Id = data.ArticleId(aId)
d.IsNew = true
a.Data(d)
}
}
func NewBleve(path string, size int64, logger webfw.Logger) (content.SearchProvider, error) {
var err error
var exists bool
var index bleve.Index
_, err = os.Stat(path)
if err == nil {
logger.Infoln("Opening search index " + path)
index, err = bleve.Open(path)
if err != nil {
return nil, errors.New(fmt.Sprintf("Error opening search index: %v\n", err))
}
exists = true
} else if os.IsNotExist(err) {
mapping := bleve.NewIndexMapping()
docMapping := bleve.NewDocumentMapping()
idfieldmapping := bleve.NewTextFieldMapping()
idfieldmapping.IncludeInAll = false
docMapping.AddFieldMappingsAt("FeedId", idfieldmapping)
docMapping.AddFieldMappingsAt("ArticleId", idfieldmapping)
mapping.AddDocumentMapping(mapping.DefaultType, docMapping)
logger.Infoln("Creating search index " + path)
index, err = bleve.NewUsing(path, mapping, upside_down.Name, goleveldb.Name, nil)
if err != nil {
return nil, errors.New(fmt.Sprintf("Error creating search index: %v\n", err))
}
} else {
return nil, errors.New(
fmt.Sprintf("Error getting stat of '%s': %v\n", path, err))
}
return &Bleve{logger: logger, index: index, batchSize: size, newIndex: !exists}, nil
}
func RegisterControllers(config readeef.Config, dispatcher *webfw.Dispatcher, logger webfw.Logger) error {
repo, err := repo.New(config.DB.Driver, config.DB.Connect, logger)
if err != nil {
return err
}
capabilities := capabilities{
I18N: len(dispatcher.Config.I18n.Languages) > 1,
Popularity: len(config.Popularity.Providers) > 0,
}
var ap []content.ArticleProcessor
for _, p := range config.Content.ArticleProcessors {
switch p {
case "relative-url":
ap = append(ap, contentProcessor.NewRelativeUrl(logger))
case "proxy-http":
template := config.Content.ProxyHTTPURLTemplate
if template != "" {
p, err := contentProcessor.NewProxyHTTP(logger, template)
if err != nil {
return fmt.Errorf("Error initializing Proxy HTTP article processor: %v", err)
}
ap = append(ap, p)
capabilities.ProxyHTTP = true
}
case "insert-thumbnail-target":
ap = append(ap, contentProcessor.NewInsertThumbnailTarget(logger))
}
}
repo.ArticleProcessors(ap)
if err := initAdminUser(repo, []byte(config.Auth.Secret)); err != nil {
return err
}
mw := make([]string, 0, len(dispatcher.Config.Dispatcher.Middleware))
for _, m := range dispatcher.Config.Dispatcher.Middleware {
switch m {
case "I18N", "Static", "Url", "Sitemap":
case "Session":
if capabilities.ProxyHTTP {
mw = append(mw, m)
}
default:
mw = append(mw, m)
}
}
dispatcher.Config.Dispatcher.Middleware = mw
dispatcher.Context.SetGlobal(readeef.CtxKey("config"), config)
dispatcher.Context.SetGlobal(context.BaseCtxKey("readeefConfig"), config)
dispatcher.Context.SetGlobal(readeef.CtxKey("repo"), repo)
fm := readeef.NewFeedManager(repo, config, logger)
var processors []parser.Processor
for _, p := range config.FeedParser.Processors {
switch p {
case "relative-url":
processors = append(processors, processor.NewRelativeUrl(logger))
case "proxy-http":
template := config.FeedParser.ProxyHTTPURLTemplate
if template != "" {
p, err := processor.NewProxyHTTP(logger, template)
if err != nil {
return fmt.Errorf("Error initializing Proxy HTTP processor: %v", err)
}
processors = append(processors, p)
capabilities.ProxyHTTP = true
}
case "cleanup":
processors = append(processors, processor.NewCleanup(logger))
case "top-image-marker":
processors = append(processors, processor.NewTopImageMarker(logger))
}
}
fm.ParserProcessors(processors)
var sp content.SearchProvider
switch config.Content.SearchProvider {
case "elastic":
if sp, err = search.NewElastic(config.Content.ElasticURL, config.Content.SearchBatchSize, logger); err != nil {
logger.Printf("Error initializing Elastic search: %v\n", err)
}
case "bleve":
fallthrough
default:
if sp, err = search.NewBleve(config.Content.BlevePath, config.Content.SearchBatchSize, logger); err != nil {
logger.Printf("Error initializing Bleve search: %v\n", err)
}
}
if sp != nil {
if sp.IsNewIndex() {
go func() {
sp.IndexAllFeeds(repo)
}()
}
}
var ce content.Extractor
switch config.Content.Extractor {
case "readability":
if ce, err = extractor.NewReadability(config.Content.ReadabilityKey); err != nil {
return fmt.Errorf("Error initializing Readability extractor: %v\n", err)
}
case "goose":
fallthrough
default:
if ce, err = extractor.NewGoose(dispatcher.Config.Renderer.Dir); err != nil {
return fmt.Errorf("Error initializing Goose extractor: %v\n", err)
}
}
if ce != nil {
capabilities.Extractor = true
}
var t content.Thumbnailer
switch config.Content.Thumbnailer {
case "extract":
if t, err = thumbnailer.NewExtract(ce, logger); err != nil {
return fmt.Errorf("Error initializing Extract thumbnailer: %v\n", err)
}
case "description":
fallthrough
default:
t = thumbnailer.NewDescription(logger)
}
monitors := []content.FeedMonitor{monitor.NewUnread(repo, logger)}
for _, m := range config.FeedManager.Monitors {
switch m {
case "index":
if sp != nil {
monitors = append(monitors, monitor.NewIndex(sp, logger))
capabilities.Search = true
}
case "thumbnailer":
if t != nil {
monitors = append(monitors, monitor.NewThumbnailer(t, logger))
}
}
}
webSocket := NewWebSocket(fm, sp, ce, capabilities)
dispatcher.Handle(webSocket)
monitors = append(monitors, webSocket)
if config.Hubbub.CallbackURL != "" {
hubbub := readeef.NewHubbub(repo, config, logger, dispatcher.Pattern,
fm.RemoveFeedChannel())
if err := hubbub.InitSubscriptions(); err != nil {
return fmt.Errorf("Error initializing hubbub subscriptions: %v", err)
}
hubbub.FeedMonitors(monitors)
fm.Hubbub(hubbub)
}
fm.FeedMonitors(monitors)
fm.Start()
nonce := readeef.NewNonce()
controllers := []webfw.Controller{
NewAuth(capabilities),
NewFeed(fm, sp),
NewArticle(config, ce),
NewUser(),
NewUserSettings(),
NewNonce(nonce),
}
if fm.Hubbub() != nil {
controllers = append(controllers, NewHubbubController(fm.Hubbub(), config.Hubbub.RelativePath,
fm.AddFeedChannel(), fm.RemoveFeedChannel()))
}
for _, e := range config.API.Emulators {
switch e {
case "tt-rss":
controllers = append(controllers, NewTtRss(fm, sp))
case "fever":
controllers = append(controllers, NewFever())
}
}
for _, c := range controllers {
dispatcher.Handle(c)
}
middleware.InitializeDefault(dispatcher)
dispatcher.RegisterMiddleware(readeef.Auth{Pattern: dispatcher.Pattern, Nonce: nonce, IgnoreURLPrefix: config.Auth.IgnoreURLPrefix})
dispatcher.Renderer = renderer.NewRenderer(dispatcher.Config.Renderer.Dir,
dispatcher.Config.Renderer.Base)
dispatcher.Renderer.Delims("{%", "%}")
go func() {
for {
select {
case <-time.After(5 * time.Minute):
nonce.Clean(45 * time.Second)
}
}
}()
return nil
}
func readState(u content.User, dbo *db.DB, logger webfw.Logger, opts data.ArticleUpdateStateOptions, read bool, join, joinPredicate, deleteJoin, deleteWhere string, insertArgs, deleteArgs []interface{}) {
if u.HasErr() {
return
}
s := dbo.SQL()
var err error
if readStateInsertTemplate == nil {
readStateInsertTemplate, err = template.New("read-state-insert-sql").
Parse(s.User.ReadStateInsertTemplate)
if err != nil {
u.Err(fmt.Errorf("Error generating read-state-insert template: %v", err))
return
}
}
if readStateDeleteTemplate == nil {
readStateDeleteTemplate, err = template.New("read-state-delete-sql").
Parse(s.User.ReadStateDeleteTemplate)
if err != nil {
u.Err(fmt.Errorf("Error generating read-state-delete template: %v", err))
return
}
}
tx, err := dbo.Beginx()
if err != nil {
u.Err(err)
return
}
defer tx.Rollback()
if read {
args := append([]interface{}{u.Data().Login}, deleteArgs...)
buf := util.BufferPool.GetBuffer()
defer util.BufferPool.Put(buf)
data := readStateDeleteData{}
if deleteJoin != "" {
data.Join = deleteJoin
}
if opts.FavoriteOnly {
data.Join += s.User.ReadStateDeleteFavoriteJoin
}
if opts.UntaggedOnly {
data.Join += s.User.ReadStateDeleteUntaggedJoin
}
where := []string{}
if deleteWhere != "" {
where = append(where, deleteWhere)
}
if !opts.BeforeDate.IsZero() {
where = append(where, fmt.Sprintf("(a.date IS NULL OR a.date < $%d)", len(args)+1))
args = append(args, opts.BeforeDate)
}
if !opts.AfterDate.IsZero() {
where = append(where, fmt.Sprintf("a.date > $%d", len(args)+1))
args = append(args, opts.AfterDate)
}
if opts.BeforeId > 0 {
where = append(where, fmt.Sprintf("a.id < $%d", len(args)+1))
args = append(args, opts.BeforeId)
}
if opts.AfterId > 0 {
where = append(where, fmt.Sprintf("a.id > $%d", len(args)+1))
args = append(args, opts.AfterId)
}
if opts.FavoriteOnly {
where = append(where, "af.article_id IS NOT NULL")
}
if opts.UntaggedOnly {
where = append(where, "uft.feed_id IS NULL")
}
if len(where) > 0 {
data.Where = " WHERE " + strings.Join(where, " AND ")
}
if err := readStateDeleteTemplate.Execute(buf, data); err != nil {
u.Err(fmt.Errorf("Error executing read-state-delete template: %v", err))
return
}
sql := buf.String()
logger.Debugf("Read state delete SQL:\n%s\nArgs:%v\n", sql, args)
stmt, err := tx.Preparex(sql)
if err != nil {
u.Err(err)
return
}
defer stmt.Close()
_, err = stmt.Exec(args...)
if err != nil {
u.Err(err)
return
}
} else {
args := append([]interface{}{u.Data().Login}, insertArgs...)
buf := util.BufferPool.GetBuffer()
defer util.BufferPool.Put(buf)
data := readStateInsertData{}
if joinPredicate != "" {
data.JoinPredicate = " AND " + joinPredicate
}
if opts.FavoriteOnly {
data.Join += s.User.ReadStateInsertFavoriteJoin
}
if opts.UntaggedOnly {
data.Join += s.User.ReadStateInsertUntaggedJoin
}
if join != "" {
data.Join += joinPredicate
}
where := []string{}
if !opts.BeforeDate.IsZero() {
where = append(where, fmt.Sprintf("(a.date IS NULL OR a.date < $%d)", len(args)+1))
args = append(args, opts.BeforeDate)
}
if !opts.AfterDate.IsZero() {
where = append(where, fmt.Sprintf("a.date > $%d", len(args)+1))
args = append(args, opts.AfterDate)
}
if opts.BeforeId > 0 {
where = append(where, fmt.Sprintf("a.id < $%d", len(args)+1))
args = append(args, opts.BeforeId)
}
if opts.AfterId > 0 {
where = append(where, fmt.Sprintf("a.id > $%d", len(args)+1))
args = append(args, opts.AfterId)
}
if opts.FavoriteOnly {
where = append(where, "af.article_id IS NOT NULL")
}
if opts.UntaggedOnly {
where = append(where, "uft.feed_id IS NULL")
}
if len(where) > 0 {
data.Where = " WHERE " + strings.Join(where, " AND ")
}
if err := readStateInsertTemplate.Execute(buf, data); err != nil {
u.Err(fmt.Errorf("Error executing read-state-insert template: %v", err))
return
}
sql := buf.String()
logger.Debugf("Read state insert SQL:\n%s\nArgs:%q\n", sql, args)
stmt, err := tx.Preparex(sql)
if err != nil {
u.Err(err)
return
}
defer stmt.Close()
_, err = stmt.Exec(args...)
if err != nil {
u.Err(err)
return
}
}
if err = tx.Commit(); err != nil {
u.Err(err)
}
}
func articleCount(u content.User, dbo *db.DB, logger webfw.Logger, opts data.ArticleCountOptions, join, where string, args []interface{}) (count int64) {
if u.HasErr() {
return
}
s := dbo.SQL()
var err error
if articleCountTemplate == nil {
articleCountTemplate, err = template.New("article-count-sql").
Parse(s.User.ArticleCountTemplate)
if err != nil {
u.Err(fmt.Errorf("Error generating article-count template: %v", err))
return
}
}
renderData := articleCountData{}
containsUserFeeds := !opts.UnreadOnly && !opts.FavoriteOnly
if containsUserFeeds {
renderData.Join += s.User.ArticleCountUserFeedsJoin
} else {
if opts.UnreadOnly {
renderData.Join += s.User.ArticleCountUnreadJoin
}
if opts.FavoriteOnly {
renderData.Join += s.User.ArticleCountFavoriteJoin
}
}
if opts.UntaggedOnly {
renderData.Join += s.User.ArticleCountUntaggedJoin
}
if join != "" {
renderData.Join += " " + join
}
args = append([]interface{}{u.Data().Login}, args...)
whereSlice := []string{}
if opts.UnreadOnly {
whereSlice = append(whereSlice, "au.article_id IS NOT NULL AND au.user_login = $1")
}
if opts.FavoriteOnly {
whereSlice = append(whereSlice, "af.article_id IS NOT NULL AND af.user_login = $1")
}
if opts.UntaggedOnly {
whereSlice = append(whereSlice, "uft.feed_id IS NULL")
}
if where != "" {
whereSlice = append(whereSlice, where)
}
if opts.BeforeId > 0 {
whereSlice = append(whereSlice, fmt.Sprintf("a.id < $%d", len(args)+1))
args = append(args, opts.BeforeId)
}
if opts.AfterId > 0 {
whereSlice = append(whereSlice, fmt.Sprintf("a.id > $%d", len(args)+1))
args = append(args, opts.AfterId)
}
if !opts.BeforeDate.IsZero() {
whereSlice = append(whereSlice, fmt.Sprintf("(a.date IS NULL OR a.date < $%d)", len(args)+1))
args = append(args, opts.BeforeDate)
}
if !opts.AfterDate.IsZero() {
whereSlice = append(whereSlice, fmt.Sprintf("a.date > $%d", len(args)+1))
args = append(args, opts.AfterDate)
}
if len(whereSlice) > 0 {
renderData.Where = "WHERE " + strings.Join(whereSlice, " AND ")
}
buf := util.BufferPool.GetBuffer()
defer util.BufferPool.Put(buf)
if err := articleCountTemplate.Execute(buf, renderData); err != nil {
u.Err(fmt.Errorf("Error executing article-count template: %v", err))
return
}
sql := buf.String()
logger.Debugf("Article count SQL:\n%s\nArgs:%v\n", sql, args)
if err := dbo.Get(&count, sql, args...); err != nil {
u.Err(err)
return
}
return
}
func internalGetArticles(u content.User, dbo *db.DB, logger webfw.Logger, opts data.ArticleQueryOptions, sorting content.ArticleSorting, join, where string, args []interface{}) (ua []content.UserArticle) {
renderData := getArticlesData{}
s := dbo.SQL()
if opts.IncludeScores {
renderData.Columns += ", asco.score"
renderData.Join += s.User.GetArticlesScoreJoin
}
if opts.UntaggedOnly {
renderData.Join += s.User.GetArticlesUntaggedJoin
}
if join != "" {
renderData.Join += " " + join
}
args = append([]interface{}{u.Data().Login}, args...)
whereSlice := []string{}
if opts.UnreadOnly {
whereSlice = append(whereSlice, "au.article_id IS NOT NULL")
} else if opts.ReadOnly {
whereSlice = append(whereSlice, "au.article_id IS NULL")
}
if opts.UntaggedOnly {
whereSlice = append(whereSlice, "uft.feed_id IS NULL")
}
if where != "" {
whereSlice = append(whereSlice, where)
}
if opts.BeforeId > 0 {
whereSlice = append(whereSlice, fmt.Sprintf("a.id < $%d", len(args)+1))
args = append(args, opts.BeforeId)
}
if opts.AfterId > 0 {
whereSlice = append(whereSlice, fmt.Sprintf("a.id > $%d", len(args)+1))
args = append(args, opts.AfterId)
}
if opts.FavoriteOnly {
whereSlice = append(whereSlice, "af.article_id IS NOT NULL")
}
if !opts.BeforeDate.IsZero() {
whereSlice = append(whereSlice, fmt.Sprintf("(a.date IS NULL OR a.date < $%d)", len(args)+1))
args = append(args, opts.BeforeDate)
}
if !opts.AfterDate.IsZero() {
whereSlice = append(whereSlice, fmt.Sprintf("a.date > $%d", len(args)+1))
args = append(args, opts.AfterDate)
}
if len(whereSlice) > 0 {
renderData.Where = "WHERE " + strings.Join(whereSlice, " AND ")
}
sortingField := sorting.Field()
sortingOrder := sorting.Order()
fields := []string{}
if opts.IncludeScores && opts.HighScoredFirst {
field := "asco.score"
if sortingOrder == data.DescendingOrder {
field += " DESC"
}
fields = append(fields, field)
}
if opts.UnreadFirst {
fields = append(fields, "read")
}
switch sortingField {
case data.SortById:
fields = append(fields, "a.id")
case data.SortByDate:
fields = append(fields, "a.date")
}
if len(fields) > 0 {
renderData.Order = " ORDER BY " + strings.Join(fields, ", ")
if sortingOrder == data.DescendingOrder {
renderData.Order += " DESC"
}
}
if opts.Limit > 0 {
renderData.Limit = fmt.Sprintf(" LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, opts.Limit, opts.Offset)
}
buf := util.BufferPool.GetBuffer()
defer util.BufferPool.Put(buf)
if err := getArticlesTemplate.Execute(buf, renderData); err != nil {
u.Err(fmt.Errorf("Error executing get-articles template: %v", err))
return
}
sql := buf.String()
var data []data.Article
logger.Debugf("Articles SQL:\n%s\nArgs:%v\n", sql, args)
if err := dbo.Select(&data, sql, args...); err != nil {
u.Err(err)
return
}
ua = make([]content.UserArticle, len(data))
for i := range data {
ua[i] = u.Repo().UserArticle(u)
ua[i].Data(data[i])
}
processors := u.Repo().ArticleProcessors()
if !opts.SkipProcessors && len(processors) > 0 {
for _, p := range processors {
if opts.SkipSessionProcessors {
if _, ok := p.(processor.ProxyHTTP); ok {
continue
}
}
ua = p.ProcessArticles(ua)
}
}
return
}
func getArticles(u content.User, dbo *db.DB, logger webfw.Logger, sorting content.ArticleSorting, columns, join, where, order string, args []interface{}, paging ...int) (ua []content.UserArticle) {
if u.HasErr() {
return
}
sql := dbo.SQL("get_article_columns")
if columns != "" {
sql += ", " + columns
}
sql += dbo.SQL("get_article_tables")
if join != "" {
sql += " " + join
}
sql += dbo.SQL("get_article_joins")
args = append([]interface{}{u.Data().Login}, args...)
if where != "" {
sql += " AND " + where
}
sortingField := sorting.Field()
sortingOrder := sorting.Order()
fields := []string{}
if order != "" {
fields = append(fields, order)
}
switch sortingField {
case data.SortById:
fields = append(fields, "a.id")
case data.SortByDate:
fields = append(fields, "a.date")
}
if len(fields) > 0 {
sql += " ORDER BY "
sql += strings.Join(fields, ",")
if sortingOrder == data.DescendingOrder {
sql += " DESC"
}
}
if len(paging) > 0 {
limit, offset := pagingLimit(paging)
sql += fmt.Sprintf(" LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, limit, offset)
}
var data []data.Article
logger.Debugf("Articles SQL:\n%s\nArgs:%q\n", sql, args)
if err := dbo.Select(&data, sql, args...); err != nil {
u.Err(err)
return
}
ua = make([]content.UserArticle, len(data))
for i := range data {
ua[i] = u.Repo().UserArticle(u)
ua[i].Data(data[i])
}
return
}