// worker executes the Attempts. func (s *Scheduler) worker(attempt *models.Attempt) { result := make(chan *models.Attempt) defer close(result) var wg sync.WaitGroup wg.Add(1) db := s.store.DB() defer db.Session.Close() b := models.NewBase(db) // Start a goroutine to touch/reserve the Attempt. go func() { defer wg.Done() for { select { case attempt := <-result: if attempt == nil { return } _, err := b.NextAttemptForTask(attempt.TaskID, attempt.Status) if err != nil { fmt.Println(err) } return case <-time.After(time.Duration(s.touchInterval) * time.Second): b.TouchAttempt(attempt.ID, s.touchInterval*2) } } }() attempt, err := b.DoAttempt(attempt) if err != nil { fmt.Println(err) } result <- attempt wg.Wait() }
func (mw *BaseMiddleware) MiddlewareFunc(next rest.HandlerFunc) rest.HandlerFunc { return func(w rest.ResponseWriter, r *rest.Request) { db := mw.Store.DB() defer db.Session.Close() r.Env["MODELS_BASE"] = models.NewBase(db) next(w, r) } }
// worker executes the Attempts. func (s *Scheduler) worker(attempt *models.Attempt) { result := make(chan *models.Attempt) defer close(result) var wg sync.WaitGroup wg.Add(1) db := s.store.DB() defer db.Session.Close() b := models.NewBase(db) // Start a goroutine to touch/reserve the Attempt. go func(attempt *models.Attempt) { defer wg.Done() for { select { case attempt := <-result: if attempt == nil { return } _, err := b.NextAttemptForTask(attempt) if err != nil && err != models.ErrDatabase { log.Printf("Scheduler error with NextAttemptForTask: %#v\n", err) } return case <-time.After(time.Duration(s.touchInterval) * time.Second): b.TouchAttempt(attempt.ID, s.touchInterval*2) } } }(attempt) err := b.DoAttempt(attempt) if err == nil { result <- attempt } else { if err != models.ErrDatabase { log.Printf("Scheduler error with DoAttempt: %#v\n", err) } result <- nil } wg.Wait() }
// New creates a new instance of the Rest API. func New(s *store.Store, adminPassword string) (*rest.Api, error) { db := s.DB() models.NewBase(db).EnsureIndex() db.Session.Close() api := rest.NewApi() api.Use(rest.DefaultDevStack...) api.Use(&BaseMiddleware{ Store: s, }) api.Use(&rest.JsonpMiddleware{ CallbackNameKey: "cb", }) api.Use(&rest.CorsMiddleware{ RejectNonCorsRequests: false, OriginValidator: func(origin string, request *rest.Request) bool { return true }, AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, AllowedHeaders: []string{"Accept", "Content-Type", "Origin", "Authorization"}, AccessControlAllowCredentials: true, AccessControlMaxAge: 3600, }) authBasic := &AuthBasicMiddleware{ Realm: "Hooky", Authenticator: authenticate(adminPassword), Authorizator: authorize(adminPassword), } api.Use(&rest.IfMiddleware{ Condition: func(r *rest.Request) bool { return r.URL.Path != "/status" }, IfTrue: authBasic, }) router, err := rest.MakeRouter( rest.Get("/authenticate", Authenticate), rest.Post("/accounts", PostAccount), rest.Get("/accounts", GetAccounts), rest.Get("/accounts/:account", GetAccount), rest.Patch("/accounts/:account", PatchAccount), rest.Delete("/accounts/:account", DeleteAccount), rest.Delete("/accounts/:account/applications", DeleteApplications), rest.Get("/accounts/:account/applications", GetApplications), rest.Get("/accounts/:account/applications/:application", GetApplication), rest.Put("/accounts/:account/applications/:application", PutApplication), rest.Delete("/accounts/:account/applications/:application", DeleteApplication), rest.Get("/accounts/:account/applications/:application/queues", GetQueues), rest.Put("/accounts/:account/applications/:application/queues/:queue", PutQueue), rest.Delete("/accounts/:account/applications/:application/queues/:queue", DeleteQueue), rest.Get("/accounts/:account/applications/:application/queues/:queue", GetQueue), rest.Delete("/accounts/:account/applications/:application/queues", DeleteQueues), rest.Post("/accounts/:account/applications/:application/tasks", PutTask), rest.Get("/accounts/:account/applications/:application/tasks", GetTasks), rest.Delete("/accounts/:account/applications/:application/tasks", DeleteTasks), rest.Put("/accounts/:account/applications/:application/tasks/:task", PutTask), rest.Get("/accounts/:account/applications/:application/tasks/:task", GetTask), rest.Delete("/accounts/:account/applications/:application/tasks/:task", DeleteTask), rest.Post("/accounts/:account/applications/:application/tasks/:task/attempts", PostAttempt), rest.Get("/accounts/:account/applications/:application/tasks/:task/attempts", GetAttempts), rest.Get("/status", GetStatus), ) if err != nil { return nil, err } api.SetApp(router) return api, nil }
// Start starts the Scheduler. func (s *Scheduler) Start() { // Cleaner go func() { clean := func() { db := s.store.DB() defer db.Session.Close() b := models.NewBase(db) if _, err := b.CleanFinishedAttempts(s.cleanFinishedAttempts); err != nil && err != models.ErrDatabase { log.Printf("Scheduler error with CleanFinishedAttempts: %s\n", err) } if err := b.CleanDeletedRessources(); err != nil && err != models.ErrDatabase { log.Printf("Scheduler error with CleanDeletedRessources: %s\n", err) } } clean() for { select { case <-s.quit: return case <-time.After(time.Second * 60): clean() } } }() // Fixer go func() { fix := func() { db := s.store.DB() defer db.Session.Close() b := models.NewBase(db) if err := b.FixIntegrity(); err != nil && err != models.ErrDatabase { log.Printf("Scheduler error with FixIntegrity: %s\n", err) } if err := b.FixQueues(); err != nil && err != models.ErrDatabase { log.Printf("Scheduler error with FixQueues: %s\n", err) } } fix() for { select { case <-s.quit: return case <-time.After(time.Second * 60): fix() } } }() // Attempts scheduler go func() { for { select { case <-s.quit: return case s.querierSem <- true: go func() { s.workerSem <- true s.wg.Add(1) defer s.wg.Done() db := s.store.DB() b := models.NewBase(db) attempt, err := b.NextAttempt(s.touchInterval * 2) db.Session.Close() if attempt != nil { s.wg.Add(1) go func() { defer s.wg.Done() s.worker(attempt) <-s.workerSem }() <-s.querierSem return } else if err != nil && err != models.ErrDatabase { log.Printf("Scheduler error with NextAttempt: %#v\n", err) } <-s.workerSem <-s.querierSem }() } } }() }
// Start starts the Scheduler. func (s *Scheduler) Start() { // Cleaner go func() { clean := func() { db := s.store.DB() b := models.NewBase(db) if _, err := b.CleanFinishedAttempts(s.cleanFinishedAttempts); err != nil { fmt.Println(err) } if err := b.CleanDeletedRessources(); err != nil { fmt.Println(err) } db.Session.Close() } clean() for { select { case <-s.quit: return case <-time.After(time.Second * 60): clean() } } }() // Attempts scheduler go func() { for { select { case <-s.quit: return case s.querierSem <- true: go func() { s.workerSem <- true s.wg.Add(1) defer s.wg.Done() db := s.store.DB() b := models.NewBase(db) attempt, err := b.NextAttempt(s.touchInterval * 2) db.Session.Close() if attempt != nil { s.wg.Add(1) go func() { defer s.wg.Done() s.worker(attempt) <-s.workerSem }() <-s.querierSem return } else if err != nil { fmt.Println(err) } <-s.workerSem time.Sleep(100 * time.Millisecond) <-s.querierSem }() } } }() }
func main() { app := cli.NewApp() app.Name = "hooky" app.Usage = "the webhooks scheduler" app.Version = "0.1" app.Author = "Sébastien Estienne" app.Email = "*****@*****.**" app.Flags = []cli.Flag{ cli.StringFlag{ Name: "bind-address", Value: "", Usage: "host address to bind on", EnvVar: "HOOKY_BIND_ADDRESS", }, cli.StringFlag{ Name: "bind-port", Value: "8000", Usage: "port number to bind on", EnvVar: "HOOKY_BIND_PORT,PORT", }, cli.StringFlag{ Name: "mongo-uri", Value: "mongodb://127.0.0.1/hooky", Usage: "MongoDB URI to connect to", EnvVar: "HOOKY_MONGO_URI", }, cli.StringFlag{ Name: "admin-password", Value: "admin", Usage: "admin password", EnvVar: "HOOKY_ADMIN_PASSWORD", }, cli.StringFlag{ Name: "accesslog-format", Value: "none", Usage: "format of the access log: json, apache-fancy, apache-combined, apache-common or none", EnvVar: "HOOKY_ACCESSLOG_FORMAT", }, cli.IntFlag{ Name: "max-mongo-query", Value: 1, Usage: "maximum number of parallel queries on MongoDB", EnvVar: "HOOKY_MAX_MONGO_QUERY", }, cli.IntFlag{ Name: "max-http-request", Value: 20, Usage: "maximum number of parallel HTTP requests", EnvVar: "HOOKY_MAX_HTTP_REQUEST", }, cli.IntFlag{ Name: "touch-interval", Value: 5, Usage: "frequency to update the tasks reservation duration in seconds", EnvVar: "HOOKY_TOUCH_INTERVAL", }, cli.IntFlag{ Name: "clean-finished-attempts", Value: 7 * 24, Usage: "delete finished attempts that are older than this age in hours", EnvVar: "HOOKY_CLEAN_FINISHED_ATTEMPTS", }, } app.Action = func(c *cli.Context) { s, err := store.New(c.String("mongo-uri")) if err != nil { log.Fatal(err) } db := s.DB() if err := models.NewBase(db).Bootstrap(); err != nil { log.Fatal(err) } db.Session.Close() sched := scheduler.New(s, c.Int("max-mongo-query"), c.Int("max-http-request"), c.Int("touch-interval"), c.Int("clean-finished-attempts")*3600) sched.Start() ra, err := restapi.New(s, c.String("admin-password"), c.String("accesslog-format")) if err != nil { log.Fatal(err) } server := &graceful.Server{ Timeout: 10 * time.Second, Server: &http.Server{ Addr: c.String("bind-host") + ":" + c.String("bind-port"), Handler: ra.MakeHandler(), }, } err = server.ListenAndServe() if err != nil { log.Println(err) } log.Println("exiting...") sched.Stop() log.Println("exited") } app.Run(os.Args) }