示例#1
0
// NewAPINotifier returns a new instance of *APINotifier
func NewAPINotifier(baseURL string, token string) *APINotifier {
	logger := wcg.NewLoggerWithPrefix(nil, "pt.Notifier")
	return &APINotifier{
		client: api.NewClient(baseURL, token, logger),
		logger: logger,
	}
}
示例#2
0
func NewRegularIntervalAgent(repeatable Repeatable, interval time.Duration) Agent {
	return &RegularIntervalAgent{
		Interval:    interval,
		repeatable:  repeatable,
		stopChannel: nil,
		isRunning:   false,
		logger:      wcg.NewLoggerWithPrefix(nil, fmt.Sprintf("%s", repeatable.Name())),
	}
}
示例#3
0
// NewPT1Recorder returns a new instance of *PT1Recorder
func NewPT1Recorder(iepg *pt.IEPG, cfg *Config, notifier Notifier) Recorder {
	if cfg == nil {
		cfg = DefaultConfig
	}
	recorder := &PT1Recorder{}
	recorder.iepg = iepg
	recorder.stats = iepg.NewStats()
	recorder.cfg = cfg
	recorder.notifier = notifier
	recorder.logger = wcg.NewLoggerWithPrefix(nil, fmt.Sprintf("pt1recorder.%s", iepg.ID))
	return recorder
}
示例#4
0
// NewImageCacher cretes a new *ImageCacher available in a request.
func NewImageCacher(req *wcg.Request, kind *entities.Kind) (*ImageCacher, error) {
	client, err := newMediaClient(req)
	if err != nil {
		return nil, err
	}
	logger := wcg.NewLoggerWithPrefix(req, "ImageCache")
	return &ImageCacher{
		kind:   kind,
		req:    req,
		client: client,
		memc:   memcache.NewDriver(gae.NewContext(req), logger),
		logger: logger,
	}, nil
}
示例#5
0
// NewAgent creats a new instance of *Agent
func NewAgent(baseURL, token string, cfg *Config) *Agent {
	logger := wcg.NewLoggerWithPrefix(nil, agentName)
	if cfg == nil {
		cfg = DefaultConfig
	}
	return &Agent{
		subscriber: NewSubscriber(baseURL, token),
		recorders:  make(map[string]Recorder),
		recorderFactory: func(iepg *pt.IEPG) Recorder {
			return NewPT1Recorder(iepg, cfg, NewAPINotifier(baseURL, token))
		},
		cfg:    cfg,
		logger: logger,
	}
}
示例#6
0
// Define endpoints to process an async task.
func (helper *AsyncAPIHelper) Define(path string) *AsyncAPIConfig {
	if !strings.HasSuffix(path, "/") {
		panic(fmt.Errorf("Task endpoint must end with '/' (%s)", path))
	}
	api := helper.app.API()
	// /path/to/something/ => api-path-to-something
	p := api.Path(path)
	name := strings.Replace(p[1:len(p)-1], ":", "", -1) // remove parameters
	name = strings.Replace(name, "/", "-", -1)          // replace '/' with '-'
	config := &AsyncAPIConfig{
		Queue: helper.app.Queue().DefinePush(name),
		app:   helper.app,
		path:  path,
	}

	// Endpoint that trigger the task.
	api.POST(path, Handler(
		func(req *wcg.Request) response.Response {
			for _, h := range config.triggerHandlers {
				resp := h.Handle(req)
				if resp != nil {
					req.Logger.Debugf("Async Task is not triggered.")
					return resp
				}
			}
			task := &models.AsyncAPITask{}
			task.ID = wcg.NewUUID()
			task.Path = path
			task.Query = req.HTTPRequest().URL.RawQuery
			task.Status = models.AsyncAPIStatusReady
			entities.AsyncAPITask.Put().Key(string(task.ID)).MustUpdate(req, task)
			// Push a task
			taskPath := fmt.Sprintf("%s%s.json?%s", req.URL().Path, string(task.ID), req.HTTPRequest().URL.Query().Encode())
			if err := config.Queue.PushTask(req, taskPath, nil); err != nil {
				return response.InternalServerError(req, err)
			}
			req.Logger.Infof("Triggered an async task [id:%s path:%s]", task.ID, taskPath)
			return response.NewJSONResponseWithStatus(
				AsyncTaskTriggerResponse{task.ID},
				201,
			)
		},
	))

	// Endpoint that executes the task in background
	// This endpoint must be called by PushQueue
	api.POST(path+":taskid.json", Handler(
		func(req *wcg.Request) response.Response {
			if !request.IsTask(req) {
				req.Logger.Warnf(
					"Non system call to the async task endpoint: UserAgent=%s, IP=%s",
					req.HTTPRequest().UserAgent(),
					req.HTTPRequest().RemoteAddr,
				)
				return response.APIOK
			}

			_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
			if one == nil {
				req.Logger.Warnf(
					"Task %q not found: UserAgent=%s, IP=%s",
					req.Param("taskid"),
					req.HTTPRequest().UserAgent(),
					req.HTTPRequest().RemoteAddr,
				)
				return response.APIOK
			}
			task := one.(*models.AsyncAPITask)
			logger := wcg.NewLoggerWithPrefix(req, fmt.Sprintf("[AsyncTask: %s (queue: %s)]", task.ID, name))
			if task.Status != models.AsyncAPIStatusReady && task.Status != models.AsyncAPIStatusRunning {
				logger.Warnf("Task is not ready: Status=%s", task.Status)
				return response.APIOK
			}

			if task.Status == models.AsyncAPIStatusReady {
				task.StartAt = lib.Now()
				task.Status = models.AsyncAPIStatusRunning
				entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
			}

			var err error
			var progress *models.AsyncAPITaskProgress
			var resp response.Response
			func() {
				defer func() {
					if x := recover(); x != nil {
						logger.Errorf("Unhandle error recovered from the task: %v", x)
						err = fmt.Errorf("%v", x)
					}
				}()
				progress, err = config.processHandler(req, task)
			}()

			if progress != nil {
				task.Progress = append(task.Progress, *progress)
			}
			if progress != nil && progress.Next != nil {
				// update the entity (Progress field)
				entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)

				// Push a next request into the queue
				// This is not required in test environment as test code manually call the next request (see testhelper.AsyncTaskTestRunner#Run code)
				// The test environment uses the response JSON to call the next URL and that response is used only for this purpose.
				// If the code cannot push task for the next request, we should stop the task as failure.
				logger.Infof("Task needs to call recursively: Next parameter is %v", progress.Next)
				if !req.IsTest() {
					err = config.Queue.PushTask(req, fmt.Sprintf("%s?%s", req.URL().Path, progress.Next.Encode()), nil)
				}
				if err == nil {
					return response.NewJSONResponse(progress.Next)
				}
				// if an error occurrs by PushTask, just stopping the task and update the datastore as failure.
			}
			// finished the task
			task.FinishAt = lib.Now()
			tt := task.FinishAt.Sub(task.StartAt)
			if err == nil {
				task.Status = models.AsyncAPIStatusSuccess
				logger.Infof("Task finished successfully (time: %s)", tt.String())
				resp = response.APIOK
			} else {
				task.Error = err.Error()
				task.Status = models.AsyncAPIStatusFailure
				logger.Errorf("Task failed (time: %s): %s", tt.String(), task.Error)
				resp = response.APIInternalServerError
			}
			entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
			return resp
		},
	))

	// Endpoint that returns the task status.
	api.GET(path+":taskid.json", Handler(
		func(req *wcg.Request) response.Response {
			for _, h := range config.triggerHandlers {
				resp := h.Handle(req)
				if resp != nil {
					return resp
				}
			}
			_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
			if one == nil {
				return response.NotFound(req)
			}
			task := one.(*models.AsyncAPITask)
			return response.NewJSONResponse(
				AsyncTaskMonitorResponse{
					ID:       task.ID,
					Status:   task.Status,
					Progress: task.LastProgress(),
				},
			)
		},
	))

	return config
}
示例#7
0
func setup20161009Backfill(app *server.App) string {
	const batchSize = 800
	const cursorKey = "c"
	const path = "/tasks/20161009backfill/"
	var API = app.API()
	var resolveSettingsURL = func(url string, settingsList []hplink.CrawlerSettings) string {
		for _, settings := range settingsList {
			if strings.HasPrefix(url, settings.URL) {
				return settings.URL
			}
		}
		return ""
	}
	API.POST(
		path,
		handlers.AdminGate,
		server.Handler(func(req *wcg.Request) response.Response {
			cursor := req.Query(cursorKey)
			logger := wcg.NewLoggerWithPrefix(req, "20161009backfill")
			logger.Infof("Start operation (c=%q)", cursor)
			p := entities.CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo).MustExecute(req)
			if p.Len() == 0 {
				logger.Infof("No CrawlerSettings found.")
				return nil
			}
			settingsList := p.Data.([]hplink.CrawlerSettings)
			q := entities.AmebloPost.Query().Order("-UpdatedAt")
			if cursor != "" {
				q = q.StartCursor(cursor)
			}
			iter := q.MustRun(req)

			var updates []hplink.AmebloPost
			var needMoreIteration = true
			for i := 0; i < batchSize; i++ {
				_, _post := iter.MustNext()
				if _post == nil {
					needMoreIteration = false
					break
				}
				post := _post.(*hplink.AmebloPost)
				if post.SettingsURL == "" {
					logger.Infof("Set %q for SettingsURL on %q", post.SettingsURL, post.URL)
					post.SettingsURL = resolveSettingsURL(post.URL, settingsList)
				}
				updates = append(updates, *post)
			}
			entities.AmebloPost.PutMulti().DoNotUpdateTimestamp(true).MustUpdate(req, updates)
			if needMoreIteration {
				_path := fmt.Sprintf("%s?%s", API.Path(path), url.Values{
					cursorKey: []string{iter.MustCursorString()},
				}.Encode())
				if err := server.RunInOpsQueue(req, _path, nil); err != nil {
					logger.Errorf("Failed to continue the task: %v", err)
				}
			}
			return response.APIOK
		}),
	)
	return API.Path(path)

}