// RunInBackground a wrapper for runtime.RunInBackground
func RunInBackground(req *wcg.Request, f func(*wcg.Request)) {
bgReq := req.Clone()
runtime.RunInBackground(NewContext(req), func(ctx context.Context) {
bgReq.SetLocal("__gaebg_context", ctx)
f(bgReq)
})
}
func (pi *PageResponse) GetTemplates(req *wcg.Request) []string {
if pi.StatusCode >= 400 {
file := fmt.Sprintf("%d.html", pi.StatusCode)
_, err := os.Stat(path.Join(wcg.ViewConfig.BaseDir, file))
if os.IsNotExist(err) {
return []string{"500.html", "header.html", "footer.html"}
}
return []string{file, "header.html", "footer.html"}
}
// succses case
var _p = pi.templatePath
if _p == "" {
_p = req.URL().Path
}
if strings.HasPrefix(_p, "/") {
_p = fmt.Sprintf(".%s", _p) // make path (/path/to/endpoint/) relative (./path/to/endpoint)
}
if path.Ext(_p) == "" {
_p = path.Join(_p, "/index.html")
}
_, err := os.Stat(path.Join(wcg.ViewConfig.BaseDir, _p))
if os.IsNotExist(err) {
return []string{"default.html", "header.html", "footer.html"}
}
return []string{_p, "header.html", "footer.html"}
}
func canSkipSessionMiddleware(req *wcg.Request) bool {
return req.Header(request.APITokenHeader) != "" ||
request.IsStatic(req) ||
request.IsHook(req) ||
request.IsCron(req) ||
request.IsTask(req)
}
func getAPITokenHeader(req *wcg.Request) string {
token := req.Header(request.APITokenHeader)
if wcg.IsUUID(token) {
return token
}
return ""
}
func debugger(res *wcg.Response, req *wcg.Request) {
if !gates.CanPass(req, "debugger") {
return
}
if req.Header("x-ajax-request") == "true" {
// TODO: implement debugger here
}
}
// getIEPGURLResolver returns iepgURLResolver object.
// the mock object in '__test_iepg_url_resolver' local variable in the request can be used for testing.
func getIEPGURLResolver(req *wcg.Request) *iepgURLResolver {
if req.IsTest() {
if v := req.Local("__test_iepg_url_resolver"); v != nil {
if v1, ok := v.(*iepgURLResolver); ok {
return v1
}
}
}
return defaultIEPGURLResolver
}
// PushTask to push a task into the queue.
func (queue *PushQueue) PushTask(req *wcg.Request, urlPath string, form url.Values) error {
var queueName string
if req.IsTest() || lib.IsOnLocalGAE() {
queueName = "default"
} else {
queueName = queue.Name
}
if _, err := taskqueue.Add(gae.NewContext(req), taskqueue.NewPOSTTask(urlPath, form), queueName); err != nil {
req.Logger.Errorf("[Queue] Error adding a task (%s) into the queue (%q): %v", urlPath, queueName, err)
return err
}
return nil
}
func sendToBigQuery(req *wcg.Request) error {
var errors = make([]error, 0)
var svc *bigquery.Service
var records []*wcg.LogRecord
var tmp interface{}
data := make(map[string][]*bigquery.TableDataInsertAllRequestRows)
if tmp = req.Local(requestLocalLogCacheKey); tmp == nil {
return nil
}
records = tmp.([]*wcg.LogRecord)
svc, err := bq.NewService(req)
if err != nil {
return err
}
for _, r := range records {
tableID, row, err := bqTransform(r)
if err != nil {
errors = append(errors, err)
continue
}
if rows, ok := data[tableID]; ok {
rows = append(rows, row)
} else {
data[tableID] = []*bigquery.TableDataInsertAllRequestRows{row}
}
}
for tableID, rows := range data {
ret, err := svc.Tabledata.InsertAll(
LogSinkConfig.BigqueryProject,
LogSinkConfig.BigqueryDataset,
tableID,
&bigquery.TableDataInsertAllRequest{
Kind: "bigquery#tableDataInsertAllRequest",
Rows: rows,
}).Do()
if err != nil {
errors = append(errors, err)
continue
}
if len(ret.InsertErrors) > 0 {
errors = append(errors, fmt.Errorf("InsertErrors: %s", ret.InsertErrors[0]))
continue
}
}
if len(errors) > 0 {
return logTransmissionError(errors)
}
return nil
}
// NewTaskUser creatse a *APITokenUser for the task
func NewTaskUser(req *wcg.Request) *APITokenUser {
return &APITokenUser{
&models.APIToken{
Token: "x-appengine-taskqueue",
Description: fmt.Sprintf(
"TaskName=%q, QueueName=%q",
req.Header("X-AppEngine-TaskName"),
req.Header("X-AppEngine-QueueName"),
),
CreatedAt: time.Now(),
AlertOn: time.Duration(0),
LastAccess: time.Now(),
},
}
}
func authorizeByAPIToken(req *wcg.Request) {
tokenString := req.Header(request.APITokenHeader)
if !lib.IsOnGAE() && !lib.IsProduction() && tokenString == string(request.APITestUser.Token) {
req.User = request.APITestUser
return
}
if !wcg.IsUUID(tokenString) {
// Do not access APIToken entity if it's not a valid UUID.
return
}
_, ent, err := entities.APIToken.Get().Key(tokenString).Cache(true).One(req)
if err == nil {
req.User = &request.APITokenUser{ent.(*models.APIToken)}
return
}
}
// NewService returns a new bigquery service accessor instance
func NewService(req *wcg.Request) (*bigquery.Service, error) {
var ctx context.Context
tmp := req.Local("__gaetest__context")
if tmp != nil {
ctx = tmp.(context.Context)
} else {
ctx = appengine.NewContext(req.HTTPRequest())
}
client, err := NewHTTPClient(ctx)
if err != nil {
return nil, err
}
svc, err := bigquery.New(client)
if err != nil {
return nil, err
}
return svc, nil
}
func newRemoteAPIContext(req *wcg.Request, host string) (context.Context, error) {
// TODO: migrate file to datastore?
if !lib.IsOnLocalGAE() && !req.IsTest() {
return nil, fmt.Errorf("RemoteAPI use on GAE is not supported yet.")
}
var hc *http.Client
var err error
const defaultCredentialEnvKey = "GOOGLE_APPLICATION_CREDENTIALS"
if os.Getenv(defaultCredentialEnvKey) != "" {
hc, err = google.DefaultClient(gae.NewContext(req), remoteCtxScopes...)
} else {
return nil, ErrNoRemoteAPIKeyIsConfigured
}
if err != nil {
return nil, err
}
return remote_api.NewRemoteContext(host, hc)
}
func newMediaClient(req *wcg.Request) (*media.Client, error) {
const fallbackServer = "speedland-ng-dev.appspot.com"
kind := entities.OAuth2ClientSettings
if lib.IsOnLocalGAE() || req.IsTest() {
kind = kind.NewRemote(fallbackServer)
}
_, settings, err := kind.Get().Key(OAuth2ClientKey).One(req)
if err != nil {
return nil, err
}
if settings == nil {
if lib.IsOnLocalGAE() || req.IsTest() {
return nil, fmt.Errorf("OAuthClient `%s` is not configured on %s", OAuth2ClientKey, fallbackServer)
}
return nil, fmt.Errorf("OAuthClient `%s` is not configured", OAuth2ClientKey)
}
return media.NewClient(
settings.(*models.OAuth2ClientSettings).GenClient(urlfetch.NewContext(req)),
), nil
}
func findArtistAndMember(req *wcg.Request) *hplink.Member {
var artist *hplink.Artist
var member *hplink.Member
var funcs = []func(){
func() {
_, ent := entities.Artist.Get().Key(
req.Param(paramKeyArtist),
).MustOne(req)
if ent != nil {
artist = ent.(*hplink.Artist)
}
},
func() {
_, ent := entities.Member.Get().Key(
fmt.Sprintf("%s.%s", req.Param(paramKeyArtist), req.Param(paramKeyMember)),
).MustOne(req)
if ent != nil {
member = ent.(*hplink.Member)
}
},
}
iterator.ParallelSlice(funcs, func(i int, v func()) error {
v()
return nil
})
if artist == nil || member == nil {
return nil
}
member.Artist = artist
return member
}
// NewContext a wrapper for appengine.NewContext
func NewContext(req *wcg.Request) context.Context {
ctx := req.Local("__gaetest__context")
if ctx != nil {
return ctx.(context.Context)
}
ctx = req.Local("__gaebg_context")
if ctx != nil {
return ctx.(context.Context)
}
return appengine.NewContext(req.HTTPRequest())
}
func execStatsQuery(req *wcg.Request, query *entities.Query) response.Response {
now := lib.Now()
until := wcg.ParseDateTimeOr(req.Query("until"), now)
if until.After(now) {
until = now
}
sinceDefault := until.Add(StatsQueryDefaultTimeWindow)
sinceMin := until.Add(StatsQueryMaxTimeWindow)
since := wcg.ParseDateTimeOr(req.Query("since"), until.Add(-24*time.Hour))
if since.After(until) {
since = sinceDefault
} else if sinceMin.After(since) {
since = sinceMin
}
query = query.Filter("Timestamp >=", since).Filter("Timestamp <", until)
if req.Query("device") != "" {
query = query.Filter("Device =", req.Query("device"))
}
return response.NewJSONResponse(query.MustExecute(req).Data)
}
func processEntityPut(req *wcg.Request, put *entities.Put, keyParams string, create bool) response.Response {
key := req.Param(keyParams)
_, ent := put.Kind().Get().Key(key).UseDefaultIfNil(create).MustOne(req)
if ent == nil {
return response.APINotFound
}
if req.HTTPRequest().Form == nil {
req.Logger.Warnf("You should use ParseForm middleware to avoid the error in EntityPut|EntityPutOrCreate")
return response.InternalServerError(req, ErrInvalidFormParameters).(response.Response)
}
err := put.Kind().UpdateEntityFromForm(ent, req.HTTPRequest().Form)
if err != nil {
return response.BadRequest(req, ErrInvalidFormParameters)
}
_, ent_ := put.Key(key).MustUpdate(req, ent)
return response.NewJSONResponseWithStatus(ent_, 200)
}
// IsCronRequest returns true if the request is comming from cron.
func IsCron(req *wcg.Request) bool {
return strings.HasPrefix(req.URL().Path, CronPathPrefix) && req.Header(CronHeader) != ""
}
func IsStatic(req *wcg.Request) bool {
return strings.HasPrefix(req.URL().Path, StaticPathPrefix)
}
func IsHook(req *wcg.Request) bool {
return strings.HasPrefix(req.URL().Path, HookPathPrefix)
}
func runTasksCrawlersAmebloEntryLists(req *wcg.Request, task *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
const FollowLinkKey = "fl"
const SettingsKey = "s"
const URLKey = "u"
var query = req.HTTPRequest().URL.Query()
var settingsList []*hplink.CrawlerSettings
var urlList []string
if settingsKeys, ok := query[SettingsKey]; ok {
_, _list := entities.CrawlerSettings.GetMulti().Keys(settingsKeys...).MustList(req)
settingsList = _list.([]*hplink.CrawlerSettings)
} else {
query := entities.CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo)
if pagination := query.MustExecute(req); pagination.Length() > 0 {
list := pagination.Data.([]hplink.CrawlerSettings)
settingsList = make([]*hplink.CrawlerSettings, len(list))
for i := range list {
settingsList[i] = &list[i]
}
}
}
var numList = len(settingsList)
urlList = make([]string, numList)
if urls, ok := query[URLKey]; ok {
if numList != len(urls) {
return nil, fmt.Errorf("List mismatch - found %d settings but %d urls are specified", numList, len(urls))
}
urlList = query[URLKey]
} else {
for i := range settingsList {
urlList[i] = (*hplink.AmebloCrawlerSettings)(settingsList[i]).GetEntryListURL()
}
}
startTime := lib.Now()
nextParamSettingsKeys := make([]string, numList)
nextParamURLs := make([]string, numList)
err := iterator.ParallelSlice(settingsList, func(i int, v *hplink.CrawlerSettings) error {
next, err := _crawlAmebloEntryList(req, v, urlList[i])
if err != nil {
settingsList[i].Error = []byte(fmt.Sprintf("%v", err))
settingsList[i].Status = hplink.CrawlerStatusFailure
settingsList[i].LastRun = lib.Now()
return err
}
settingsList[i].Error = nil
settingsList[i].Status = hplink.CrawlerStatusSuccess
settingsList[i].LastRun = lib.Now()
if next != "" {
nextParamSettingsKeys[i] = v.URL
nextParamURLs[i] = next
}
return nil
})
entities.CrawlerSettings.PutMulti().MustUpdate(req, settingsList)
if err != nil {
return nil, err
}
if req.Query(FollowLinkKey) != "true" {
return nil, err
}
// fl=true make a recursive call to follow next links
// reduce empty urls from nextParam* and return it for recursive call
var fixedNextParamSettingsKeys []string
var fixedNextParamURLs []string
var hasNext = false
for i := range nextParamURLs {
if nextParamURLs[i] != "" {
hasNext = true
fixedNextParamSettingsKeys = append(fixedNextParamSettingsKeys, nextParamSettingsKeys[i])
fixedNextParamURLs = append(fixedNextParamURLs, nextParamURLs[i])
}
}
var progress models.AsyncAPITaskProgress
var lastProgress = task.LastProgress()
if lastProgress == nil {
progress.Current = len(urlList)
progress.Total = 0
} else {
progress.Current = lastProgress.Current + len(urlList)
}
if hasNext {
progress.Next = url.Values{
FollowLinkKey: []string{"true"},
SettingsKey: fixedNextParamSettingsKeys,
URLKey: fixedNextParamURLs,
}
wait := configs.GetIntValue(req, "hplink.ameblo_crawler_url_wait", 2, 0, 10)
lib.WaitAndEnsureAfter(startTime, time.Duration(wait)*time.Second)
}
req.Logger.Infof("No more URL needs to be crawled.")
return &progress, nil
}
// IsTaskRequest returns true if the request is comming from TaskQueue.
func IsTask(req *wcg.Request) bool {
return req.Header(TaskQueueHeader) != ""
}
// IsAPIRequest returns true if the request is comming to an API endpoint.
func IsAPI(req *wcg.Request) bool {
return strings.HasPrefix(req.URL().Path, APIPathPrefix)
}