func recentRecordsQuery() *entities.Query {
return IEPG.Query().Filter("EndAt >=", request.Value(func(*wcg.Request) interface{} {
return lib.Today().Add(-recentRecordsTimeWindow)
})).Filter("EndAt <", request.Value(func(*wcg.Request) interface{} {
return lib.Today().Add(recentRecordsTimeWindow * 2)
})).Order("-EndAt")
}
reflect.TypeOf(float64(0)): func(s string) (reflect.Value, error) {
v, err := strconv.ParseFloat(s, 64)
return reflect.ValueOf(v), err
},
// time
reflect.TypeOf(time.Duration(0)): func(s string) (reflect.Value, error) {
v, err := time.ParseDuration(s)
return reflect.ValueOf(v), err
},
reflect.TypeOf(time.Now()): func(s string) (reflect.Value, error) {
switch s {
case _ValueMacroTimeNow:
return reflect.ValueOf(lib.Now()), nil
case _ValueMacroTimeToday:
return reflect.ValueOf(lib.Today()), nil
case "":
return reflect.ValueOf(time.Time{}), nil
default:
if t, err := wcg.ParseDateTime(s); err == nil {
return reflect.ValueOf(t), err
} else if t, err := wcg.ParseDate(s); err == nil {
return reflect.ValueOf(t), err
} else {
// as JSON representation
var t time.Time
err := json.Unmarshal([]byte(fmt.Sprintf("\"%s\"", s)), &t)
return reflect.ValueOf(t), err
}
}
},
func _selectAmebloPosts(req *wcg.Request, limit int) ([]*hplink.AmebloPost, error) {
var targets []*hplink.AmebloPost
// Posts with Images but failed to save images on gphoto.
pagination, err := entities.AmebloPost.Query().Filter("Images.Height=", 0).Limit(limit).Execute(req)
if err != nil {
return nil, err
}
posts := pagination.Data.([]hplink.AmebloPost)
for i := range posts {
targets = append(targets, &posts[i])
}
remains := limit - len(targets)
if remains > 0 {
// Uncrawled posts
pagination, err := entities.AmebloPost.Query().Filter("IsContentsCrawled=", false).Order("UpdatedAt").Limit(remains).Execute(req)
if err != nil {
return nil, err
}
posts = pagination.Data.([]hplink.AmebloPost)
for i := range posts {
targets = append(targets, &posts[i])
}
}
remains = limit - len(targets)
if remains > 0 {
// TODO: we can remove this code when all of records has non-zero time value.
// Crawled Posts but PostAt field is still zero time due to parser bug.
pagination, err := entities.AmebloPost.Query().Filter("PostAt=", time.Time{}).Order("UpdatedAt").Limit(remains).Execute(req)
if err != nil {
return nil, err
}
posts = pagination.Data.([]hplink.AmebloPost)
for i := range posts {
targets = append(targets, &posts[i])
}
}
if remains > 0 {
// Recent Posts
aweekago := lib.Today().Add(-7 * 24 * time.Hour)
pagination, err = entities.AmebloPost.Query().Filter("PostAt >=", aweekago).Execute(req)
if err != nil {
req.Logger.Errorf("Could not query the AmebloPosts within a week ago: %v", err)
return targets, nil
}
// then select older posts (we cannot use PostAt fileter and UpdatedAt sort in a single query due to datastore spec)
pagination.SetLessFunc(func(a interface{}, b interface{}) bool {
return a.(*hplink.AmebloPost).UpdatedAt.Before(b.(*hplink.AmebloPost).UpdatedAt)
})
sort.Sort(pagination)
posts := pagination.Data.([]hplink.AmebloPost)
for i := range posts {
targets = append(targets, &posts[i])
remains--
if remains <= 0 {
break
}
}
}
return targets, nil
}
