// NewAPINotifier returns a new instance of *APINotifier
func NewAPINotifier(baseURL string, token string) *APINotifier {
logger := wcg.NewLoggerWithPrefix(nil, "pt.Notifier")
return &APINotifier{
client: api.NewClient(baseURL, token, logger),
logger: logger,
}
}
func NewRegularIntervalAgent(repeatable Repeatable, interval time.Duration) Agent {
return &RegularIntervalAgent{
Interval: interval,
repeatable: repeatable,
stopChannel: nil,
isRunning: false,
logger: wcg.NewLoggerWithPrefix(nil, fmt.Sprintf("%s", repeatable.Name())),
}
}
// NewPT1Recorder returns a new instance of *PT1Recorder
func NewPT1Recorder(iepg *pt.IEPG, cfg *Config, notifier Notifier) Recorder {
if cfg == nil {
cfg = DefaultConfig
}
recorder := &PT1Recorder{}
recorder.iepg = iepg
recorder.stats = iepg.NewStats()
recorder.cfg = cfg
recorder.notifier = notifier
recorder.logger = wcg.NewLoggerWithPrefix(nil, fmt.Sprintf("pt1recorder.%s", iepg.ID))
return recorder
}
// NewImageCacher cretes a new *ImageCacher available in a request.
func NewImageCacher(req *wcg.Request, kind *entities.Kind) (*ImageCacher, error) {
client, err := newMediaClient(req)
if err != nil {
return nil, err
}
logger := wcg.NewLoggerWithPrefix(req, "ImageCache")
return &ImageCacher{
kind: kind,
req: req,
client: client,
memc: memcache.NewDriver(gae.NewContext(req), logger),
logger: logger,
}, nil
}
// NewAgent creats a new instance of *Agent
func NewAgent(baseURL, token string, cfg *Config) *Agent {
logger := wcg.NewLoggerWithPrefix(nil, agentName)
if cfg == nil {
cfg = DefaultConfig
}
return &Agent{
subscriber: NewSubscriber(baseURL, token),
recorders: make(map[string]Recorder),
recorderFactory: func(iepg *pt.IEPG) Recorder {
return NewPT1Recorder(iepg, cfg, NewAPINotifier(baseURL, token))
},
cfg: cfg,
logger: logger,
}
}
// Define endpoints to process an async task.
func (helper *AsyncAPIHelper) Define(path string) *AsyncAPIConfig {
if !strings.HasSuffix(path, "/") {
panic(fmt.Errorf("Task endpoint must end with '/' (%s)", path))
}
api := helper.app.API()
// /path/to/something/ => api-path-to-something
p := api.Path(path)
name := strings.Replace(p[1:len(p)-1], ":", "", -1) // remove parameters
name = strings.Replace(name, "/", "-", -1) // replace '/' with '-'
config := &AsyncAPIConfig{
Queue: helper.app.Queue().DefinePush(name),
app: helper.app,
path: path,
}
// Endpoint that trigger the task.
api.POST(path, Handler(
func(req *wcg.Request) response.Response {
for _, h := range config.triggerHandlers {
resp := h.Handle(req)
if resp != nil {
req.Logger.Debugf("Async Task is not triggered.")
return resp
}
}
task := &models.AsyncAPITask{}
task.ID = wcg.NewUUID()
task.Path = path
task.Query = req.HTTPRequest().URL.RawQuery
task.Status = models.AsyncAPIStatusReady
entities.AsyncAPITask.Put().Key(string(task.ID)).MustUpdate(req, task)
// Push a task
taskPath := fmt.Sprintf("%s%s.json?%s", req.URL().Path, string(task.ID), req.HTTPRequest().URL.Query().Encode())
if err := config.Queue.PushTask(req, taskPath, nil); err != nil {
return response.InternalServerError(req, err)
}
req.Logger.Infof("Triggered an async task [id:%s path:%s]", task.ID, taskPath)
return response.NewJSONResponseWithStatus(
AsyncTaskTriggerResponse{task.ID},
201,
)
},
))
// Endpoint that executes the task in background
// This endpoint must be called by PushQueue
api.POST(path+":taskid.json", Handler(
func(req *wcg.Request) response.Response {
if !request.IsTask(req) {
req.Logger.Warnf(
"Non system call to the async task endpoint: UserAgent=%s, IP=%s",
req.HTTPRequest().UserAgent(),
req.HTTPRequest().RemoteAddr,
)
return response.APIOK
}
_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
if one == nil {
req.Logger.Warnf(
"Task %q not found: UserAgent=%s, IP=%s",
req.Param("taskid"),
req.HTTPRequest().UserAgent(),
req.HTTPRequest().RemoteAddr,
)
return response.APIOK
}
task := one.(*models.AsyncAPITask)
logger := wcg.NewLoggerWithPrefix(req, fmt.Sprintf("[AsyncTask: %s (queue: %s)]", task.ID, name))
if task.Status != models.AsyncAPIStatusReady && task.Status != models.AsyncAPIStatusRunning {
logger.Warnf("Task is not ready: Status=%s", task.Status)
return response.APIOK
}
if task.Status == models.AsyncAPIStatusReady {
task.StartAt = lib.Now()
task.Status = models.AsyncAPIStatusRunning
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
}
var err error
var progress *models.AsyncAPITaskProgress
var resp response.Response
func() {
defer func() {
if x := recover(); x != nil {
logger.Errorf("Unhandle error recovered from the task: %v", x)
err = fmt.Errorf("%v", x)
}
}()
progress, err = config.processHandler(req, task)
}()
if progress != nil {
task.Progress = append(task.Progress, *progress)
}
if progress != nil && progress.Next != nil {
// update the entity (Progress field)
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
// Push a next request into the queue
// This is not required in test environment as test code manually call the next request (see testhelper.AsyncTaskTestRunner#Run code)
// The test environment uses the response JSON to call the next URL and that response is used only for this purpose.
// If the code cannot push task for the next request, we should stop the task as failure.
logger.Infof("Task needs to call recursively: Next parameter is %v", progress.Next)
if !req.IsTest() {
err = config.Queue.PushTask(req, fmt.Sprintf("%s?%s", req.URL().Path, progress.Next.Encode()), nil)
}
if err == nil {
return response.NewJSONResponse(progress.Next)
}
// if an error occurrs by PushTask, just stopping the task and update the datastore as failure.
}
// finished the task
task.FinishAt = lib.Now()
tt := task.FinishAt.Sub(task.StartAt)
if err == nil {
task.Status = models.AsyncAPIStatusSuccess
logger.Infof("Task finished successfully (time: %s)", tt.String())
resp = response.APIOK
} else {
task.Error = err.Error()
task.Status = models.AsyncAPIStatusFailure
logger.Errorf("Task failed (time: %s): %s", tt.String(), task.Error)
resp = response.APIInternalServerError
}
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
return resp
},
))
// Endpoint that returns the task status.
api.GET(path+":taskid.json", Handler(
func(req *wcg.Request) response.Response {
for _, h := range config.triggerHandlers {
resp := h.Handle(req)
if resp != nil {
return resp
}
}
_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
if one == nil {
return response.NotFound(req)
}
task := one.(*models.AsyncAPITask)
return response.NewJSONResponse(
AsyncTaskMonitorResponse{
ID: task.ID,
Status: task.Status,
Progress: task.LastProgress(),
},
)
},
))
return config
}
func setup20161009Backfill(app *server.App) string {
const batchSize = 800
const cursorKey = "c"
const path = "/tasks/20161009backfill/"
var API = app.API()
var resolveSettingsURL = func(url string, settingsList []hplink.CrawlerSettings) string {
for _, settings := range settingsList {
if strings.HasPrefix(url, settings.URL) {
return settings.URL
}
}
return ""
}
API.POST(
path,
handlers.AdminGate,
server.Handler(func(req *wcg.Request) response.Response {
cursor := req.Query(cursorKey)
logger := wcg.NewLoggerWithPrefix(req, "20161009backfill")
logger.Infof("Start operation (c=%q)", cursor)
p := entities.CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo).MustExecute(req)
if p.Len() == 0 {
logger.Infof("No CrawlerSettings found.")
return nil
}
settingsList := p.Data.([]hplink.CrawlerSettings)
q := entities.AmebloPost.Query().Order("-UpdatedAt")
if cursor != "" {
q = q.StartCursor(cursor)
}
iter := q.MustRun(req)
var updates []hplink.AmebloPost
var needMoreIteration = true
for i := 0; i < batchSize; i++ {
_, _post := iter.MustNext()
if _post == nil {
needMoreIteration = false
break
}
post := _post.(*hplink.AmebloPost)
if post.SettingsURL == "" {
logger.Infof("Set %q for SettingsURL on %q", post.SettingsURL, post.URL)
post.SettingsURL = resolveSettingsURL(post.URL, settingsList)
}
updates = append(updates, *post)
}
entities.AmebloPost.PutMulti().DoNotUpdateTimestamp(true).MustUpdate(req, updates)
if needMoreIteration {
_path := fmt.Sprintf("%s?%s", API.Path(path), url.Values{
cursorKey: []string{iter.MustCursorString()},
}.Encode())
if err := server.RunInOpsQueue(req, _path, nil); err != nil {
logger.Errorf("Failed to continue the task: %v", err)
}
}
return response.APIOK
}),
)
return API.Path(path)
}