Beispiel #1
0
// worker executes the Attempts.
func (s *Scheduler) worker(attempt *models.Attempt) {
	result := make(chan *models.Attempt)
	defer close(result)
	var wg sync.WaitGroup
	wg.Add(1)
	db := s.store.DB()
	defer db.Session.Close()
	b := models.NewBase(db)
	// Start a goroutine to touch/reserve the Attempt.
	go func() {
		defer wg.Done()
		for {
			select {
			case attempt := <-result:
				if attempt == nil {
					return
				}
				_, err := b.NextAttemptForTask(attempt.TaskID, attempt.Status)
				if err != nil {
					fmt.Println(err)
				}
				return
			case <-time.After(time.Duration(s.touchInterval) * time.Second):
				b.TouchAttempt(attempt.ID, s.touchInterval*2)
			}
		}
	}()
	attempt, err := b.DoAttempt(attempt)
	if err != nil {
		fmt.Println(err)
	}
	result <- attempt
	wg.Wait()
}
Beispiel #2
0
func (mw *BaseMiddleware) MiddlewareFunc(next rest.HandlerFunc) rest.HandlerFunc {
	return func(w rest.ResponseWriter, r *rest.Request) {
		db := mw.Store.DB()
		defer db.Session.Close()
		r.Env["MODELS_BASE"] = models.NewBase(db)
		next(w, r)
	}
}
Beispiel #3
0
// worker executes the Attempts.
func (s *Scheduler) worker(attempt *models.Attempt) {
	result := make(chan *models.Attempt)
	defer close(result)
	var wg sync.WaitGroup
	wg.Add(1)
	db := s.store.DB()
	defer db.Session.Close()
	b := models.NewBase(db)
	// Start a goroutine to touch/reserve the Attempt.
	go func(attempt *models.Attempt) {
		defer wg.Done()
		for {
			select {
			case attempt := <-result:
				if attempt == nil {
					return
				}
				_, err := b.NextAttemptForTask(attempt)
				if err != nil && err != models.ErrDatabase {
					log.Printf("Scheduler error with NextAttemptForTask: %#v\n", err)
				}
				return
			case <-time.After(time.Duration(s.touchInterval) * time.Second):
				b.TouchAttempt(attempt.ID, s.touchInterval*2)
			}
		}
	}(attempt)
	err := b.DoAttempt(attempt)
	if err == nil {
		result <- attempt
	} else {
		if err != models.ErrDatabase {
			log.Printf("Scheduler error with DoAttempt: %#v\n", err)
		}
		result <- nil
	}
	wg.Wait()
}
Beispiel #4
0
// New creates a new instance of the Rest API.
func New(s *store.Store, adminPassword string) (*rest.Api, error) {
	db := s.DB()
	models.NewBase(db).EnsureIndex()
	db.Session.Close()

	api := rest.NewApi()
	api.Use(rest.DefaultDevStack...)
	api.Use(&BaseMiddleware{
		Store: s,
	})
	api.Use(&rest.JsonpMiddleware{
		CallbackNameKey: "cb",
	})
	api.Use(&rest.CorsMiddleware{
		RejectNonCorsRequests: false,
		OriginValidator: func(origin string, request *rest.Request) bool {
			return true
		},
		AllowedMethods:                []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
		AllowedHeaders:                []string{"Accept", "Content-Type", "Origin", "Authorization"},
		AccessControlAllowCredentials: true,
		AccessControlMaxAge:           3600,
	})
	authBasic := &AuthBasicMiddleware{
		Realm:         "Hooky",
		Authenticator: authenticate(adminPassword),
		Authorizator:  authorize(adminPassword),
	}
	api.Use(&rest.IfMiddleware{
		Condition: func(r *rest.Request) bool {
			return r.URL.Path != "/status"
		},
		IfTrue: authBasic,
	})
	router, err := rest.MakeRouter(
		rest.Get("/authenticate", Authenticate),
		rest.Post("/accounts", PostAccount),
		rest.Get("/accounts", GetAccounts),
		rest.Get("/accounts/:account", GetAccount),
		rest.Patch("/accounts/:account", PatchAccount),
		rest.Delete("/accounts/:account", DeleteAccount),
		rest.Delete("/accounts/:account/applications", DeleteApplications),
		rest.Get("/accounts/:account/applications", GetApplications),
		rest.Get("/accounts/:account/applications/:application", GetApplication),
		rest.Put("/accounts/:account/applications/:application", PutApplication),
		rest.Delete("/accounts/:account/applications/:application", DeleteApplication),
		rest.Get("/accounts/:account/applications/:application/queues", GetQueues),
		rest.Put("/accounts/:account/applications/:application/queues/:queue", PutQueue),
		rest.Delete("/accounts/:account/applications/:application/queues/:queue", DeleteQueue),
		rest.Get("/accounts/:account/applications/:application/queues/:queue", GetQueue),
		rest.Delete("/accounts/:account/applications/:application/queues", DeleteQueues),
		rest.Post("/accounts/:account/applications/:application/tasks", PutTask),
		rest.Get("/accounts/:account/applications/:application/tasks", GetTasks),
		rest.Delete("/accounts/:account/applications/:application/tasks", DeleteTasks),
		rest.Put("/accounts/:account/applications/:application/tasks/:task", PutTask),
		rest.Get("/accounts/:account/applications/:application/tasks/:task", GetTask),
		rest.Delete("/accounts/:account/applications/:application/tasks/:task", DeleteTask),
		rest.Post("/accounts/:account/applications/:application/tasks/:task/attempts", PostAttempt),
		rest.Get("/accounts/:account/applications/:application/tasks/:task/attempts", GetAttempts),
		rest.Get("/status", GetStatus),
	)
	if err != nil {
		return nil, err
	}
	api.SetApp(router)

	return api, nil
}
Beispiel #5
0
// Start starts the Scheduler.
func (s *Scheduler) Start() {
	// Cleaner
	go func() {
		clean := func() {
			db := s.store.DB()
			defer db.Session.Close()
			b := models.NewBase(db)
			if _, err := b.CleanFinishedAttempts(s.cleanFinishedAttempts); err != nil && err != models.ErrDatabase {
				log.Printf("Scheduler error with CleanFinishedAttempts: %s\n", err)
			}
			if err := b.CleanDeletedRessources(); err != nil && err != models.ErrDatabase {
				log.Printf("Scheduler error with CleanDeletedRessources: %s\n", err)
			}
		}
		clean()
		for {
			select {
			case <-s.quit:
				return
			case <-time.After(time.Second * 60):
				clean()
			}
		}
	}()

	// Fixer
	go func() {
		fix := func() {
			db := s.store.DB()
			defer db.Session.Close()
			b := models.NewBase(db)
			if err := b.FixIntegrity(); err != nil && err != models.ErrDatabase {
				log.Printf("Scheduler error with FixIntegrity: %s\n", err)
			}
			if err := b.FixQueues(); err != nil && err != models.ErrDatabase {
				log.Printf("Scheduler error with FixQueues: %s\n", err)
			}
		}
		fix()
		for {
			select {
			case <-s.quit:
				return
			case <-time.After(time.Second * 60):
				fix()
			}
		}
	}()

	// Attempts scheduler
	go func() {
		for {
			select {
			case <-s.quit:
				return

			case s.querierSem <- true:
				go func() {
					s.workerSem <- true
					s.wg.Add(1)
					defer s.wg.Done()
					db := s.store.DB()
					b := models.NewBase(db)
					attempt, err := b.NextAttempt(s.touchInterval * 2)
					db.Session.Close()
					if attempt != nil {
						s.wg.Add(1)
						go func() {
							defer s.wg.Done()
							s.worker(attempt)
							<-s.workerSem
						}()
						<-s.querierSem
						return
					} else if err != nil && err != models.ErrDatabase {
						log.Printf("Scheduler error with NextAttempt: %#v\n", err)
					}
					<-s.workerSem
					<-s.querierSem
				}()
			}
		}
	}()
}
Beispiel #6
0
// Start starts the Scheduler.
func (s *Scheduler) Start() {
	// Cleaner
	go func() {
		clean := func() {
			db := s.store.DB()
			b := models.NewBase(db)
			if _, err := b.CleanFinishedAttempts(s.cleanFinishedAttempts); err != nil {
				fmt.Println(err)
			}
			if err := b.CleanDeletedRessources(); err != nil {
				fmt.Println(err)
			}
			db.Session.Close()
		}
		clean()
		for {
			select {
			case <-s.quit:
				return
			case <-time.After(time.Second * 60):
				clean()
			}
		}
	}()

	// Attempts scheduler
	go func() {
		for {
			select {
			case <-s.quit:
				return

			case s.querierSem <- true:
				go func() {
					s.workerSem <- true
					s.wg.Add(1)
					defer s.wg.Done()
					db := s.store.DB()
					b := models.NewBase(db)
					attempt, err := b.NextAttempt(s.touchInterval * 2)
					db.Session.Close()
					if attempt != nil {
						s.wg.Add(1)
						go func() {
							defer s.wg.Done()
							s.worker(attempt)
							<-s.workerSem
						}()
						<-s.querierSem
						return
					} else if err != nil {
						fmt.Println(err)
					}
					<-s.workerSem
					time.Sleep(100 * time.Millisecond)
					<-s.querierSem
				}()
			}
		}
	}()
}
Beispiel #7
0
func main() {
	app := cli.NewApp()
	app.Name = "hooky"
	app.Usage = "the webhooks scheduler"
	app.Version = "0.1"
	app.Author = "Sébastien Estienne"
	app.Email = "*****@*****.**"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "bind-address",
			Value:  "",
			Usage:  "host address to bind on",
			EnvVar: "HOOKY_BIND_ADDRESS",
		},
		cli.StringFlag{
			Name:   "bind-port",
			Value:  "8000",
			Usage:  "port number to bind on",
			EnvVar: "HOOKY_BIND_PORT,PORT",
		},
		cli.StringFlag{
			Name:   "mongo-uri",
			Value:  "mongodb://127.0.0.1/hooky",
			Usage:  "MongoDB URI to connect to",
			EnvVar: "HOOKY_MONGO_URI",
		},
		cli.StringFlag{
			Name:   "admin-password",
			Value:  "admin",
			Usage:  "admin password",
			EnvVar: "HOOKY_ADMIN_PASSWORD",
		},
		cli.StringFlag{
			Name:   "accesslog-format",
			Value:  "none",
			Usage:  "format of the access log: json, apache-fancy, apache-combined, apache-common or none",
			EnvVar: "HOOKY_ACCESSLOG_FORMAT",
		},
		cli.IntFlag{
			Name:   "max-mongo-query",
			Value:  1,
			Usage:  "maximum number of parallel queries on MongoDB",
			EnvVar: "HOOKY_MAX_MONGO_QUERY",
		},
		cli.IntFlag{
			Name:   "max-http-request",
			Value:  20,
			Usage:  "maximum number of parallel HTTP requests",
			EnvVar: "HOOKY_MAX_HTTP_REQUEST",
		},
		cli.IntFlag{
			Name:   "touch-interval",
			Value:  5,
			Usage:  "frequency to update the tasks reservation duration in seconds",
			EnvVar: "HOOKY_TOUCH_INTERVAL",
		},
		cli.IntFlag{
			Name:   "clean-finished-attempts",
			Value:  7 * 24,
			Usage:  "delete finished attempts that are older than this age in hours",
			EnvVar: "HOOKY_CLEAN_FINISHED_ATTEMPTS",
		},
	}
	app.Action = func(c *cli.Context) {
		s, err := store.New(c.String("mongo-uri"))
		if err != nil {
			log.Fatal(err)
		}

		db := s.DB()
		if err := models.NewBase(db).Bootstrap(); err != nil {
			log.Fatal(err)
		}
		db.Session.Close()

		sched := scheduler.New(s, c.Int("max-mongo-query"), c.Int("max-http-request"), c.Int("touch-interval"), c.Int("clean-finished-attempts")*3600)
		sched.Start()
		ra, err := restapi.New(s, c.String("admin-password"), c.String("accesslog-format"))
		if err != nil {
			log.Fatal(err)
		}
		server := &graceful.Server{
			Timeout: 10 * time.Second,
			Server: &http.Server{
				Addr:    c.String("bind-host") + ":" + c.String("bind-port"),
				Handler: ra.MakeHandler(),
			},
		}
		err = server.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
		log.Println("exiting...")
		sched.Stop()
		log.Println("exited")
	}
	app.Run(os.Args)
}