// worker executes the Attempts.
func (s *Scheduler) worker(attempt *models.Attempt) {
result := make(chan *models.Attempt)
defer close(result)
var wg sync.WaitGroup
wg.Add(1)
db := s.store.DB()
defer db.Session.Close()
b := models.NewBase(db)
// Start a goroutine to touch/reserve the Attempt.
go func() {
defer wg.Done()
for {
select {
case attempt := <-result:
if attempt == nil {
return
}
_, err := b.NextAttemptForTask(attempt.TaskID, attempt.Status)
if err != nil {
fmt.Println(err)
}
return
case <-time.After(time.Duration(s.touchInterval) * time.Second):
b.TouchAttempt(attempt.ID, s.touchInterval*2)
}
}
}()
attempt, err := b.DoAttempt(attempt)
if err != nil {
fmt.Println(err)
}
result <- attempt
wg.Wait()
}
func (mw *BaseMiddleware) MiddlewareFunc(next rest.HandlerFunc) rest.HandlerFunc {
return func(w rest.ResponseWriter, r *rest.Request) {
db := mw.Store.DB()
defer db.Session.Close()
r.Env["MODELS_BASE"] = models.NewBase(db)
next(w, r)
}
}
// worker executes the Attempts.
func (s *Scheduler) worker(attempt *models.Attempt) {
result := make(chan *models.Attempt)
defer close(result)
var wg sync.WaitGroup
wg.Add(1)
db := s.store.DB()
defer db.Session.Close()
b := models.NewBase(db)
// Start a goroutine to touch/reserve the Attempt.
go func(attempt *models.Attempt) {
defer wg.Done()
for {
select {
case attempt := <-result:
if attempt == nil {
return
}
_, err := b.NextAttemptForTask(attempt)
if err != nil && err != models.ErrDatabase {
log.Printf("Scheduler error with NextAttemptForTask: %#v\n", err)
}
return
case <-time.After(time.Duration(s.touchInterval) * time.Second):
b.TouchAttempt(attempt.ID, s.touchInterval*2)
}
}
}(attempt)
err := b.DoAttempt(attempt)
if err == nil {
result <- attempt
} else {
if err != models.ErrDatabase {
log.Printf("Scheduler error with DoAttempt: %#v\n", err)
}
result <- nil
}
wg.Wait()
}
// New creates a new instance of the Rest API.
func New(s *store.Store, adminPassword string) (*rest.Api, error) {
db := s.DB()
models.NewBase(db).EnsureIndex()
db.Session.Close()
api := rest.NewApi()
api.Use(rest.DefaultDevStack...)
api.Use(&BaseMiddleware{
Store: s,
})
api.Use(&rest.JsonpMiddleware{
CallbackNameKey: "cb",
})
api.Use(&rest.CorsMiddleware{
RejectNonCorsRequests: false,
OriginValidator: func(origin string, request *rest.Request) bool {
return true
},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Content-Type", "Origin", "Authorization"},
AccessControlAllowCredentials: true,
AccessControlMaxAge: 3600,
})
authBasic := &AuthBasicMiddleware{
Realm: "Hooky",
Authenticator: authenticate(adminPassword),
Authorizator: authorize(adminPassword),
}
api.Use(&rest.IfMiddleware{
Condition: func(r *rest.Request) bool {
return r.URL.Path != "/status"
},
IfTrue: authBasic,
})
router, err := rest.MakeRouter(
rest.Get("/authenticate", Authenticate),
rest.Post("/accounts", PostAccount),
rest.Get("/accounts", GetAccounts),
rest.Get("/accounts/:account", GetAccount),
rest.Patch("/accounts/:account", PatchAccount),
rest.Delete("/accounts/:account", DeleteAccount),
rest.Delete("/accounts/:account/applications", DeleteApplications),
rest.Get("/accounts/:account/applications", GetApplications),
rest.Get("/accounts/:account/applications/:application", GetApplication),
rest.Put("/accounts/:account/applications/:application", PutApplication),
rest.Delete("/accounts/:account/applications/:application", DeleteApplication),
rest.Get("/accounts/:account/applications/:application/queues", GetQueues),
rest.Put("/accounts/:account/applications/:application/queues/:queue", PutQueue),
rest.Delete("/accounts/:account/applications/:application/queues/:queue", DeleteQueue),
rest.Get("/accounts/:account/applications/:application/queues/:queue", GetQueue),
rest.Delete("/accounts/:account/applications/:application/queues", DeleteQueues),
rest.Post("/accounts/:account/applications/:application/tasks", PutTask),
rest.Get("/accounts/:account/applications/:application/tasks", GetTasks),
rest.Delete("/accounts/:account/applications/:application/tasks", DeleteTasks),
rest.Put("/accounts/:account/applications/:application/tasks/:task", PutTask),
rest.Get("/accounts/:account/applications/:application/tasks/:task", GetTask),
rest.Delete("/accounts/:account/applications/:application/tasks/:task", DeleteTask),
rest.Post("/accounts/:account/applications/:application/tasks/:task/attempts", PostAttempt),
rest.Get("/accounts/:account/applications/:application/tasks/:task/attempts", GetAttempts),
rest.Get("/status", GetStatus),
)
if err != nil {
return nil, err
}
api.SetApp(router)
return api, nil
}
// Start starts the Scheduler.
func (s *Scheduler) Start() {
// Cleaner
go func() {
clean := func() {
db := s.store.DB()
defer db.Session.Close()
b := models.NewBase(db)
if _, err := b.CleanFinishedAttempts(s.cleanFinishedAttempts); err != nil && err != models.ErrDatabase {
log.Printf("Scheduler error with CleanFinishedAttempts: %s\n", err)
}
if err := b.CleanDeletedRessources(); err != nil && err != models.ErrDatabase {
log.Printf("Scheduler error with CleanDeletedRessources: %s\n", err)
}
}
clean()
for {
select {
case <-s.quit:
return
case <-time.After(time.Second * 60):
clean()
}
}
}()
// Fixer
go func() {
fix := func() {
db := s.store.DB()
defer db.Session.Close()
b := models.NewBase(db)
if err := b.FixIntegrity(); err != nil && err != models.ErrDatabase {
log.Printf("Scheduler error with FixIntegrity: %s\n", err)
}
if err := b.FixQueues(); err != nil && err != models.ErrDatabase {
log.Printf("Scheduler error with FixQueues: %s\n", err)
}
}
fix()
for {
select {
case <-s.quit:
return
case <-time.After(time.Second * 60):
fix()
}
}
}()
// Attempts scheduler
go func() {
for {
select {
case <-s.quit:
return
case s.querierSem <- true:
go func() {
s.workerSem <- true
s.wg.Add(1)
defer s.wg.Done()
db := s.store.DB()
b := models.NewBase(db)
attempt, err := b.NextAttempt(s.touchInterval * 2)
db.Session.Close()
if attempt != nil {
s.wg.Add(1)
go func() {
defer s.wg.Done()
s.worker(attempt)
<-s.workerSem
}()
<-s.querierSem
return
} else if err != nil && err != models.ErrDatabase {
log.Printf("Scheduler error with NextAttempt: %#v\n", err)
}
<-s.workerSem
<-s.querierSem
}()
}
}
}()
}
// Start starts the Scheduler.
func (s *Scheduler) Start() {
// Cleaner
go func() {
clean := func() {
db := s.store.DB()
b := models.NewBase(db)
if _, err := b.CleanFinishedAttempts(s.cleanFinishedAttempts); err != nil {
fmt.Println(err)
}
if err := b.CleanDeletedRessources(); err != nil {
fmt.Println(err)
}
db.Session.Close()
}
clean()
for {
select {
case <-s.quit:
return
case <-time.After(time.Second * 60):
clean()
}
}
}()
// Attempts scheduler
go func() {
for {
select {
case <-s.quit:
return
case s.querierSem <- true:
go func() {
s.workerSem <- true
s.wg.Add(1)
defer s.wg.Done()
db := s.store.DB()
b := models.NewBase(db)
attempt, err := b.NextAttempt(s.touchInterval * 2)
db.Session.Close()
if attempt != nil {
s.wg.Add(1)
go func() {
defer s.wg.Done()
s.worker(attempt)
<-s.workerSem
}()
<-s.querierSem
return
} else if err != nil {
fmt.Println(err)
}
<-s.workerSem
time.Sleep(100 * time.Millisecond)
<-s.querierSem
}()
}
}
}()
}
func main() {
app := cli.NewApp()
app.Name = "hooky"
app.Usage = "the webhooks scheduler"
app.Version = "0.1"
app.Author = "Sébastien Estienne"
app.Email = "*****@*****.**"
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "bind-address",
Value: "",
Usage: "host address to bind on",
EnvVar: "HOOKY_BIND_ADDRESS",
},
cli.StringFlag{
Name: "bind-port",
Value: "8000",
Usage: "port number to bind on",
EnvVar: "HOOKY_BIND_PORT,PORT",
},
cli.StringFlag{
Name: "mongo-uri",
Value: "mongodb://127.0.0.1/hooky",
Usage: "MongoDB URI to connect to",
EnvVar: "HOOKY_MONGO_URI",
},
cli.StringFlag{
Name: "admin-password",
Value: "admin",
Usage: "admin password",
EnvVar: "HOOKY_ADMIN_PASSWORD",
},
cli.StringFlag{
Name: "accesslog-format",
Value: "none",
Usage: "format of the access log: json, apache-fancy, apache-combined, apache-common or none",
EnvVar: "HOOKY_ACCESSLOG_FORMAT",
},
cli.IntFlag{
Name: "max-mongo-query",
Value: 1,
Usage: "maximum number of parallel queries on MongoDB",
EnvVar: "HOOKY_MAX_MONGO_QUERY",
},
cli.IntFlag{
Name: "max-http-request",
Value: 20,
Usage: "maximum number of parallel HTTP requests",
EnvVar: "HOOKY_MAX_HTTP_REQUEST",
},
cli.IntFlag{
Name: "touch-interval",
Value: 5,
Usage: "frequency to update the tasks reservation duration in seconds",
EnvVar: "HOOKY_TOUCH_INTERVAL",
},
cli.IntFlag{
Name: "clean-finished-attempts",
Value: 7 * 24,
Usage: "delete finished attempts that are older than this age in hours",
EnvVar: "HOOKY_CLEAN_FINISHED_ATTEMPTS",
},
}
app.Action = func(c *cli.Context) {
s, err := store.New(c.String("mongo-uri"))
if err != nil {
log.Fatal(err)
}
db := s.DB()
if err := models.NewBase(db).Bootstrap(); err != nil {
log.Fatal(err)
}
db.Session.Close()
sched := scheduler.New(s, c.Int("max-mongo-query"), c.Int("max-http-request"), c.Int("touch-interval"), c.Int("clean-finished-attempts")*3600)
sched.Start()
ra, err := restapi.New(s, c.String("admin-password"), c.String("accesslog-format"))
if err != nil {
log.Fatal(err)
}
server := &graceful.Server{
Timeout: 10 * time.Second,
Server: &http.Server{
Addr: c.String("bind-host") + ":" + c.String("bind-port"),
Handler: ra.MakeHandler(),
},
}
err = server.ListenAndServe()
if err != nil {
log.Println(err)
}
log.Println("exiting...")
sched.Stop()
log.Println("exited")
}
app.Run(os.Args)
}