Пример #1
0
// RunInBackground a wrapper for runtime.RunInBackground
func RunInBackground(req *wcg.Request, f func(*wcg.Request)) {
	bgReq := req.Clone()
	runtime.RunInBackground(NewContext(req), func(ctx context.Context) {
		bgReq.SetLocal("__gaebg_context", ctx)
		f(bgReq)
	})
}
Пример #2
0
func (pi *PageResponse) GetTemplates(req *wcg.Request) []string {
	if pi.StatusCode >= 400 {
		file := fmt.Sprintf("%d.html", pi.StatusCode)
		_, err := os.Stat(path.Join(wcg.ViewConfig.BaseDir, file))
		if os.IsNotExist(err) {
			return []string{"500.html", "header.html", "footer.html"}
		}
		return []string{file, "header.html", "footer.html"}
	}
	// succses case
	var _p = pi.templatePath
	if _p == "" {
		_p = req.URL().Path
	}
	if strings.HasPrefix(_p, "/") {
		_p = fmt.Sprintf(".%s", _p) // make path (/path/to/endpoint/) relative (./path/to/endpoint)
	}
	if path.Ext(_p) == "" {
		_p = path.Join(_p, "/index.html")
	}
	_, err := os.Stat(path.Join(wcg.ViewConfig.BaseDir, _p))
	if os.IsNotExist(err) {
		return []string{"default.html", "header.html", "footer.html"}
	}
	return []string{_p, "header.html", "footer.html"}
}
Пример #3
0
func canSkipSessionMiddleware(req *wcg.Request) bool {
	return req.Header(request.APITokenHeader) != "" ||
		request.IsStatic(req) ||
		request.IsHook(req) ||
		request.IsCron(req) ||
		request.IsTask(req)
}
Пример #4
0
func getAPITokenHeader(req *wcg.Request) string {
	token := req.Header(request.APITokenHeader)
	if wcg.IsUUID(token) {
		return token
	}
	return ""
}
Пример #5
0
func debugger(res *wcg.Response, req *wcg.Request) {
	if !gates.CanPass(req, "debugger") {
		return
	}
	if req.Header("x-ajax-request") == "true" {
		// TODO: implement debugger here
	}
}
Пример #6
0
// getIEPGURLResolver returns iepgURLResolver object.
// the mock object in '__test_iepg_url_resolver' local variable in the request can be used for testing.
func getIEPGURLResolver(req *wcg.Request) *iepgURLResolver {
	if req.IsTest() {
		if v := req.Local("__test_iepg_url_resolver"); v != nil {
			if v1, ok := v.(*iepgURLResolver); ok {
				return v1
			}
		}
	}
	return defaultIEPGURLResolver
}
Пример #7
0
// PushTask to push a task into the queue.
func (queue *PushQueue) PushTask(req *wcg.Request, urlPath string, form url.Values) error {
	var queueName string
	if req.IsTest() || lib.IsOnLocalGAE() {
		queueName = "default"
	} else {
		queueName = queue.Name
	}
	if _, err := taskqueue.Add(gae.NewContext(req), taskqueue.NewPOSTTask(urlPath, form), queueName); err != nil {
		req.Logger.Errorf("[Queue] Error adding a task (%s) into the queue (%q): %v", urlPath, queueName, err)
		return err
	}
	return nil
}
Пример #8
0
func sendToBigQuery(req *wcg.Request) error {
	var errors = make([]error, 0)
	var svc *bigquery.Service
	var records []*wcg.LogRecord
	var tmp interface{}
	data := make(map[string][]*bigquery.TableDataInsertAllRequestRows)

	if tmp = req.Local(requestLocalLogCacheKey); tmp == nil {
		return nil
	}
	records = tmp.([]*wcg.LogRecord)
	svc, err := bq.NewService(req)
	if err != nil {
		return err
	}

	for _, r := range records {
		tableID, row, err := bqTransform(r)
		if err != nil {
			errors = append(errors, err)
			continue
		}
		if rows, ok := data[tableID]; ok {
			rows = append(rows, row)
		} else {
			data[tableID] = []*bigquery.TableDataInsertAllRequestRows{row}
		}
	}
	for tableID, rows := range data {
		ret, err := svc.Tabledata.InsertAll(
			LogSinkConfig.BigqueryProject,
			LogSinkConfig.BigqueryDataset,
			tableID,
			&bigquery.TableDataInsertAllRequest{
				Kind: "bigquery#tableDataInsertAllRequest",
				Rows: rows,
			}).Do()
		if err != nil {
			errors = append(errors, err)
			continue
		}
		if len(ret.InsertErrors) > 0 {
			errors = append(errors, fmt.Errorf("InsertErrors: %s", ret.InsertErrors[0]))
			continue
		}
	}
	if len(errors) > 0 {
		return logTransmissionError(errors)
	}
	return nil
}
Пример #9
0
// NewTaskUser creatse a *APITokenUser for the task
func NewTaskUser(req *wcg.Request) *APITokenUser {
	return &APITokenUser{
		&models.APIToken{
			Token: "x-appengine-taskqueue",
			Description: fmt.Sprintf(
				"TaskName=%q, QueueName=%q",
				req.Header("X-AppEngine-TaskName"),
				req.Header("X-AppEngine-QueueName"),
			),
			CreatedAt:  time.Now(),
			AlertOn:    time.Duration(0),
			LastAccess: time.Now(),
		},
	}
}
Пример #10
0
func authorizeByAPIToken(req *wcg.Request) {
	tokenString := req.Header(request.APITokenHeader)
	if !lib.IsOnGAE() && !lib.IsProduction() && tokenString == string(request.APITestUser.Token) {
		req.User = request.APITestUser
		return
	}
	if !wcg.IsUUID(tokenString) {
		// Do not access APIToken entity if it's not a valid UUID.
		return
	}

	_, ent, err := entities.APIToken.Get().Key(tokenString).Cache(true).One(req)
	if err == nil {
		req.User = &request.APITokenUser{ent.(*models.APIToken)}
		return
	}
}
Пример #11
0
// NewService returns a new bigquery service accessor instance
func NewService(req *wcg.Request) (*bigquery.Service, error) {
	var ctx context.Context
	tmp := req.Local("__gaetest__context")
	if tmp != nil {
		ctx = tmp.(context.Context)
	} else {
		ctx = appengine.NewContext(req.HTTPRequest())
	}

	client, err := NewHTTPClient(ctx)
	if err != nil {
		return nil, err
	}
	svc, err := bigquery.New(client)
	if err != nil {
		return nil, err
	}
	return svc, nil
}
Пример #12
0
func newRemoteAPIContext(req *wcg.Request, host string) (context.Context, error) {
	// TODO: migrate file to datastore?
	if !lib.IsOnLocalGAE() && !req.IsTest() {
		return nil, fmt.Errorf("RemoteAPI use on GAE is not supported yet.")
	}

	var hc *http.Client
	var err error
	const defaultCredentialEnvKey = "GOOGLE_APPLICATION_CREDENTIALS"
	if os.Getenv(defaultCredentialEnvKey) != "" {
		hc, err = google.DefaultClient(gae.NewContext(req), remoteCtxScopes...)
	} else {
		return nil, ErrNoRemoteAPIKeyIsConfigured
	}
	if err != nil {
		return nil, err
	}
	return remote_api.NewRemoteContext(host, hc)
}
Пример #13
0
func newMediaClient(req *wcg.Request) (*media.Client, error) {
	const fallbackServer = "speedland-ng-dev.appspot.com"
	kind := entities.OAuth2ClientSettings
	if lib.IsOnLocalGAE() || req.IsTest() {
		kind = kind.NewRemote(fallbackServer)
	}
	_, settings, err := kind.Get().Key(OAuth2ClientKey).One(req)
	if err != nil {
		return nil, err
	}
	if settings == nil {
		if lib.IsOnLocalGAE() || req.IsTest() {
			return nil, fmt.Errorf("OAuthClient `%s` is not configured on %s", OAuth2ClientKey, fallbackServer)
		}
		return nil, fmt.Errorf("OAuthClient `%s` is not configured", OAuth2ClientKey)
	}
	return media.NewClient(
		settings.(*models.OAuth2ClientSettings).GenClient(urlfetch.NewContext(req)),
	), nil
}
Пример #14
0
func findArtistAndMember(req *wcg.Request) *hplink.Member {
	var artist *hplink.Artist
	var member *hplink.Member
	var funcs = []func(){
		func() {
			_, ent := entities.Artist.Get().Key(
				req.Param(paramKeyArtist),
			).MustOne(req)
			if ent != nil {
				artist = ent.(*hplink.Artist)
			}
		},
		func() {
			_, ent := entities.Member.Get().Key(
				fmt.Sprintf("%s.%s", req.Param(paramKeyArtist), req.Param(paramKeyMember)),
			).MustOne(req)
			if ent != nil {
				member = ent.(*hplink.Member)
			}
		},
	}
	iterator.ParallelSlice(funcs, func(i int, v func()) error {
		v()
		return nil
	})
	if artist == nil || member == nil {
		return nil
	}
	member.Artist = artist
	return member
}
Пример #15
0
// NewContext a wrapper for appengine.NewContext
func NewContext(req *wcg.Request) context.Context {
	ctx := req.Local("__gaetest__context")
	if ctx != nil {
		return ctx.(context.Context)
	}
	ctx = req.Local("__gaebg_context")
	if ctx != nil {
		return ctx.(context.Context)
	}
	return appengine.NewContext(req.HTTPRequest())
}
Пример #16
0
func execStatsQuery(req *wcg.Request, query *entities.Query) response.Response {
	now := lib.Now()
	until := wcg.ParseDateTimeOr(req.Query("until"), now)
	if until.After(now) {
		until = now
	}
	sinceDefault := until.Add(StatsQueryDefaultTimeWindow)
	sinceMin := until.Add(StatsQueryMaxTimeWindow)
	since := wcg.ParseDateTimeOr(req.Query("since"), until.Add(-24*time.Hour))
	if since.After(until) {
		since = sinceDefault
	} else if sinceMin.After(since) {
		since = sinceMin
	}
	query = query.Filter("Timestamp >=", since).Filter("Timestamp <", until)
	if req.Query("device") != "" {
		query = query.Filter("Device =", req.Query("device"))
	}
	return response.NewJSONResponse(query.MustExecute(req).Data)
}
Пример #17
0
func processEntityPut(req *wcg.Request, put *entities.Put, keyParams string, create bool) response.Response {
	key := req.Param(keyParams)
	_, ent := put.Kind().Get().Key(key).UseDefaultIfNil(create).MustOne(req)
	if ent == nil {
		return response.APINotFound
	}
	if req.HTTPRequest().Form == nil {
		req.Logger.Warnf("You should use ParseForm middleware to avoid the error in EntityPut|EntityPutOrCreate")
		return response.InternalServerError(req, ErrInvalidFormParameters).(response.Response)
	}
	err := put.Kind().UpdateEntityFromForm(ent, req.HTTPRequest().Form)
	if err != nil {
		return response.BadRequest(req, ErrInvalidFormParameters)
	}
	_, ent_ := put.Key(key).MustUpdate(req, ent)
	return response.NewJSONResponseWithStatus(ent_, 200)
}
Пример #18
0
// IsCronRequest returns true if the request is comming from cron.
func IsCron(req *wcg.Request) bool {
	return strings.HasPrefix(req.URL().Path, CronPathPrefix) && req.Header(CronHeader) != ""
}
Пример #19
0
func IsStatic(req *wcg.Request) bool {
	return strings.HasPrefix(req.URL().Path, StaticPathPrefix)
}
Пример #20
0
func IsHook(req *wcg.Request) bool {
	return strings.HasPrefix(req.URL().Path, HookPathPrefix)
}
func runTasksCrawlersAmebloEntryLists(req *wcg.Request, task *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
	const FollowLinkKey = "fl"
	const SettingsKey = "s"
	const URLKey = "u"

	var query = req.HTTPRequest().URL.Query()
	var settingsList []*hplink.CrawlerSettings
	var urlList []string
	if settingsKeys, ok := query[SettingsKey]; ok {
		_, _list := entities.CrawlerSettings.GetMulti().Keys(settingsKeys...).MustList(req)
		settingsList = _list.([]*hplink.CrawlerSettings)
	} else {
		query := entities.CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo)
		if pagination := query.MustExecute(req); pagination.Length() > 0 {
			list := pagination.Data.([]hplink.CrawlerSettings)
			settingsList = make([]*hplink.CrawlerSettings, len(list))
			for i := range list {
				settingsList[i] = &list[i]
			}
		}
	}

	var numList = len(settingsList)
	urlList = make([]string, numList)
	if urls, ok := query[URLKey]; ok {
		if numList != len(urls) {
			return nil, fmt.Errorf("List mismatch - found %d settings but %d urls are specified", numList, len(urls))
		}
		urlList = query[URLKey]
	} else {
		for i := range settingsList {
			urlList[i] = (*hplink.AmebloCrawlerSettings)(settingsList[i]).GetEntryListURL()
		}
	}

	startTime := lib.Now()
	nextParamSettingsKeys := make([]string, numList)
	nextParamURLs := make([]string, numList)
	err := iterator.ParallelSlice(settingsList, func(i int, v *hplink.CrawlerSettings) error {
		next, err := _crawlAmebloEntryList(req, v, urlList[i])
		if err != nil {
			settingsList[i].Error = []byte(fmt.Sprintf("%v", err))
			settingsList[i].Status = hplink.CrawlerStatusFailure
			settingsList[i].LastRun = lib.Now()
			return err
		}
		settingsList[i].Error = nil
		settingsList[i].Status = hplink.CrawlerStatusSuccess
		settingsList[i].LastRun = lib.Now()
		if next != "" {
			nextParamSettingsKeys[i] = v.URL
			nextParamURLs[i] = next
		}
		return nil
	})
	entities.CrawlerSettings.PutMulti().MustUpdate(req, settingsList)
	if err != nil {
		return nil, err
	}
	if req.Query(FollowLinkKey) != "true" {
		return nil, err
	}
	// fl=true make a recursive call to follow next links
	// reduce empty urls from nextParam* and return it for recursive call
	var fixedNextParamSettingsKeys []string
	var fixedNextParamURLs []string
	var hasNext = false
	for i := range nextParamURLs {
		if nextParamURLs[i] != "" {
			hasNext = true
			fixedNextParamSettingsKeys = append(fixedNextParamSettingsKeys, nextParamSettingsKeys[i])
			fixedNextParamURLs = append(fixedNextParamURLs, nextParamURLs[i])
		}
	}
	var progress models.AsyncAPITaskProgress
	var lastProgress = task.LastProgress()
	if lastProgress == nil {
		progress.Current = len(urlList)
		progress.Total = 0
	} else {
		progress.Current = lastProgress.Current + len(urlList)
	}
	if hasNext {
		progress.Next = url.Values{
			FollowLinkKey: []string{"true"},
			SettingsKey:   fixedNextParamSettingsKeys,
			URLKey:        fixedNextParamURLs,
		}
		wait := configs.GetIntValue(req, "hplink.ameblo_crawler_url_wait", 2, 0, 10)
		lib.WaitAndEnsureAfter(startTime, time.Duration(wait)*time.Second)
	}
	req.Logger.Infof("No more URL needs to be crawled.")
	return &progress, nil
}
Пример #22
0
// IsTaskRequest returns true if the request is comming from TaskQueue.
func IsTask(req *wcg.Request) bool {
	return req.Header(TaskQueueHeader) != ""
}
Пример #23
0
// IsAPIRequest returns true if the request is comming to an API endpoint.
func IsAPI(req *wcg.Request) bool {
	return strings.HasPrefix(req.URL().Path, APIPathPrefix)
}