func testSupervisor(t *testing.T, nWorkers int, test func(*Supervisor)) {
tempdir, err := ioutil.TempDir("", "")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(tempdir)
client, err := tlsconfig.NewClient(tempdir)
if err != nil {
t.Error(err)
return
}
testStorage = NewTestStorage()
c := SupervisorConfig{
Storage: testStorage,
TLSHandshaker: client,
// Discard logs for tests
Logger: log.New(ioutil.Discard, "", 0),
}
supervisor, err := NewSupervisor(c)
if err != nil {
t.Errorf("failed to create supervisor: %v", err)
return
}
// discard logs for tests
supervisor.Logger = log.New(ioutil.Discard, "", 0)
for i := 0; i < nWorkers; i++ {
mps, err := mps.NewMPS()
if err != nil {
t.Error(err)
return
}
defer mps.Reset()
// configure server's TLS config directly rather than with a handshake
server := httptest.NewUnstartedServer(mps)
server.TLS, err = client.ServerConfig(net.ParseIP("127.0.0.1"))
if err != nil {
t.Error(err)
return
}
server.StartTLS()
defer server.Close()
addr := strings.TrimPrefix(server.URL, "https://")
if err := supervisor.AddWorker(addr, int64(i)); err != nil {
t.Errorf("could not add worker: %v", err)
return
}
}
test(supervisor)
}
func TestSupervisorReboot(t *testing.T) {
tempdir, err := ioutil.TempDir("", "")
if err != nil {
t.Error(err)
return
}
defer os.RemoveAll(tempdir)
client, err := tlsconfig.NewClient(tempdir)
if err != nil {
t.Error(err)
return
}
testStorage = NewTestStorage()
c := SupervisorConfig{
Storage: testStorage,
TLSHandshaker: client,
// Discard logs for tests
Logger: log.New(ioutil.Discard, "", 0),
}
supervisor1, err := NewSupervisor(c)
if err != nil {
t.Errorf("failed to create supervisor: %v", err)
return
}
addrs := [5]string{"", "", "", "", ""}
for i := 0; i < 5; i++ {
mps, err := mps.NewMPS()
if err != nil {
t.Error(err)
return
}
defer mps.Reset()
// configure server's TLS config directly rather than with a handshake
server := httptest.NewUnstartedServer(mps)
server.TLS, err = client.ServerConfig(net.ParseIP("127.0.0.1"))
if err != nil {
t.Error(err)
return
}
server.StartTLS()
defer server.Close()
addrs[i] = strings.TrimPrefix(server.URL, "https://")
}
type Model struct {
User, Model string
Version int
Instances []int64
}
deployments := map[int64]*Model{}
for i, addr := range addrs {
if err := supervisor1.AddWorker(addr, int64(i)); err != nil {
t.Errorf("could not add worker: %v", err)
return
}
}
user := "******"
// Deploy 5 different models
modelnames := []string{
"hellor_0", "hellor_1", "hellor_2", "hellor_3", "hellor_4",
}
// Override the storage's NewDeployment function so we can track which
// deployment is associated with which model version.
// This is normally done by the database.
nInstances := 2
var nextId int64 = 1
var nextDeployId int64 = 1
testStorage.newDeployment = func(user, model string, version int) (deployId int64, instIds []int64, err error) {
deployId = nextDeployId
nextDeployId++
ids := make([]int64, nInstances)
instIds = make([]int64, nInstances)
for i := range ids {
ids[i] = nextId
instIds[i] = nextId
nextId++
}
deployments[deployId] = &Model{user, model, version, ids}
return deployId, instIds, nil
}
for _, model := range modelnames {
if err := supervisor1.Deploy(user, model, 1); err != nil {
t.Errorf("could not deploy: %v", err)
return
}
}
nExp := len(modelnames)
if n := len(deployments); n != nExp {
t.Errorf("expected %d deployments, got: %d", nExp, n)
return
}
for _, model := range deployments {
if n := len(model.Instances); n != nInstances {
t.Errorf("expected %d deployments per model, got %d", nInstances, n)
return
}
}
testPred := func(super *Supervisor) {
for _, model := range modelnames {
hf := func(w http.ResponseWriter, r *http.Request) {
super.Predict(user, model, w, r)
}
s := httptest.NewServer(http.HandlerFunc(hf))
defer s.Close()
for i := 0; i < 100; i++ {
body := bytes.NewReader([]byte(`{"name":"bigdatabob"}`))
resp, err := http.Post(s.URL, "application/json", body)
if err != nil {
t.Errorf("request failed: %v", err)
return
}
respbody, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
t.Errorf("could not read body: %v", err)
return
}
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200, got %s: %s", resp.Status, respbody)
return
}
}
}
}
testPred(supervisor1)
supervisor1.shutdown()
workers := make([]db.Worker, len(addrs))
for i, addr := range addrs {
workers[i] = db.Worker{int64(i), addr}
}
deploymentReqs := make([]db.DeploymentReq, len(deployments))
i := 0
for deployId, model := range deployments {
deploymentReqs[i] = db.DeploymentReq{
Username: model.User,
Modelname: model.Model,
Version: model.Version,
LastDeployId: deployId,
ValidInstanceIds: model.Instances,
}
i++
}
c2 := SupervisorConfig{
Storage: testStorage,
TLSHandshaker: client,
Workers: workers,
Deployments: deploymentReqs,
// Discard logs for tests
Logger: log.New(ioutil.Discard, "", 0),
}
supervisor2, err := NewSupervisor(c2)
if err != nil {
t.Errorf("failed to start second supervisor: %v", err)
return
}
// discard logs for tests
supervisor2.Logger = log.New(ioutil.Discard, "", 0)
testPred(supervisor2)
}
func NewApp(config *AppConfig) (*App, error) {
if config.ModelReplication < 1 {
return nil, fmt.Errorf("model replication level cannot be less than 1")
}
var accessLog io.Writer
if config.AccessLog == "" {
accessLog = os.Stderr
} else {
flags := os.O_WRONLY | os.O_APPEND | os.O_CREATE
file, err := os.OpenFile(config.AccessLog, flags, 0644)
if err != nil {
return nil, err
}
log.Println("using access log:", config.AccessLog)
accessLog = file
}
appDB, err := sqlutil.NewReconnectingDB("mysql", config.DBConnStr)
if err != nil {
return nil, fmt.Errorf("could not connect to database: %v", err)
}
// do the static directories exist?
for _, dir := range []string{"css", "js", "fonts", "img", "model"} {
d := filepath.Join(config.StaticDir, dir)
_, err := os.Stat(d)
if err != nil {
return nil, fmt.Errorf("directory %s doesn't exist", d)
}
}
serveDir := func(dirName string) http.Handler {
d := filepath.Join(config.StaticDir, dirName)
return http.StripPrefix("/"+dirName, http.FileServer(http.Dir(d)))
}
store := sessions.NewCookieStore([]byte(config.SecretSessionKey))
client, err := tlsconfig.NewClient(config.CertDir)
if err != nil {
return nil, fmt.Errorf("certificate loading failed: %v", err)
}
app := App{
store: store,
db: appDB,
staticDir: config.StaticDir,
bundleDir: config.BundleDir,
modelLogsDir: config.ModelLogs,
modelReplication: config.ModelReplication,
isdev: config.IsDev,
serviceName: config.ServiceName,
sharingDisabled: config.DisableSharing,
}
tx, err := app.db.Begin()
if err != nil {
return nil, err
}
workers, err := db.Workers(tx)
if err != nil {
return nil, err
}
deploymentReqs, err := db.DeploymentRequests(tx)
if err != nil {
return nil, err
}
tx.Rollback()
c := alb.SupervisorConfig{
Storage: &storage{new(sync.Mutex), &app},
TLSHandshaker: client,
Workers: workers,
Deployments: deploymentReqs,
}
app.sup, err = alb.NewSupervisor(c)
if err != nil {
return nil, fmt.Errorf("failed to initialize communcation with MPSs: %v", err)
}
app.sup.Logger = log.New(os.Stderr, "supervisor: ", log.LstdFlags)
r := mux.NewRouter()
r.PathPrefix("/css/").Handler(serveDir("css"))
r.PathPrefix("/js/").Handler(serveDir("js"))
r.PathPrefix("/fonts/").Handler(serveDir("fonts"))
r.PathPrefix("/img/").Handler(serveDir("img"))
r.HandleFunc("/login", app.handleLogin)
r.HandleFunc("/register", app.handleRegister)
r.HandleFunc("/verify-password", app.handleVerifyPassword)
r.HandleFunc("/logout", app.handleLogout)
r.Handle("/favicon.ico", app.serveFile("favicon.ico"))
// deployment routes
r.PathPrefix("/deployer").HandlerFunc(app.handleDeployment)
r.HandleFunc("/verify", app.handleOldVerify)
authedRouter := mux.NewRouter()
authedRouter.NotFoundHandler = app.serveFile("404.html")
authedRouter.PathPrefix("/model/").Handler(serveDir("model"))
authedRouter.Handle("/", app.serveFile("index.html"))
authedRouter.Handle("/account", app.serveFile("account.html"))
authedRouter.HandleFunc("/user.json", app.handleUser)
authedRouter.HandleFunc("/users/{name}", app.handleUserByName)
// apikey stuff
authedRouter.HandleFunc("/apikey", app.handleApikey)
// model specific pages
authedRouter.Handle("/models/{name}", app.serveFile("model/index.html"))
authedRouter.HandleFunc("/models/{name}/json", app.handleModel)
authedRouter.Handle("/models/{name}/scoring", app.serveFile("model/scoring.html"))
authedRouter.Handle("/models/{name}/versions", app.serveFile("model/versions.html"))
authedRouter.HandleFunc("/models/{name}/versions.json", app.handleModelVersions)
authedRouter.Handle("/models/{name}/logs", app.serveFile("model/logs.html"))
authedRouter.HandleFunc("/models/{name}/logs/json", app.handleModelLogs)
authedRouter.Handle("/models/{name}/settings", app.serveFile("model/settings.html"))
authedRouter.Handle("/models/{name}/form-builder", app.serveFile("model/form-builder.html"))
authedRouter.HandleFunc("/models/{name}/redeploy/{version}", app.handleModelRedeploy)
authedRouter.HandleFunc("/models/{name}/shared", app.handleModelSharedUsers)
authedRouter.HandleFunc("/models/{name}/startshare/{user}", app.handleModelStartSharing)
authedRouter.HandleFunc("/models/{name}/stopshare/{user}", app.handleModelStopSharing)
authedRouter.HandleFunc("/models/{name}/example", app.handleModelExample)
// model actions
authedRouter.HandleFunc("/models/{name}/action/{action}", app.handleModelStateChange)
authedRouter.HandleFunc("/models", app.handleUserModels)
authedRouter.HandleFunc("/shared", app.handleSharedModels)
authedRouter.HandleFunc("/whoami", app.handleWhoami)
adminRouter := mux.NewRouter()
authedRouter.PathPrefix("/admin").Handler(app.restrictAdmin(adminRouter))
adminRouter.Handle("/admin", app.serveFile("admin/index.html"))
adminRouter.Handle("/admin/users", app.serveFile("admin/users.html"))
adminRouter.HandleFunc("/admin/users.json", app.handleUsersData)
adminRouter.HandleFunc("/admin/users/create", app.handleUsersCreate)
adminRouter.HandleFunc("/admin/users/delete", app.handleUserDelete)
adminRouter.HandleFunc("/admin/users/makeadmin", app.handleUserMakeAdmin)
adminRouter.HandleFunc("/admin/users/unmakeadmin", app.handleUserUnmakeAdmin)
adminRouter.HandleFunc("/admin/users/setpass", app.handleUserSetPass)
adminRouter.Handle("/admin/models", app.serveFile("admin/models.html"))
adminRouter.HandleFunc("/admin/models.json", app.handleAllUserModels)
adminRouter.HandleFunc("/admin/servers", app.handleServers)
adminRouter.HandleFunc("/admin/servers.json", app.handleServersData)
adminRouter.HandleFunc("/admin/servers/remove", app.handleServersRemove)
// get initial authentication for models
auth, err := func() (*db.PredictionAuth, error) {
tx, err := app.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
return db.GetAuth(tx)
}()
if err != nil {
return nil, err
}
// must enforce no caching for jsx pages
noCaching := func(handler http.Handler) http.Handler {
hf := func(w http.ResponseWriter, r *http.Request) {
// see: http://goo.gl/itaIDo
w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
w.Header().Set("Pragma", "no-cache")
w.Header().Set("Expires", "0")
handler.ServeHTTP(w, r)
}
return http.HandlerFunc(hf)
}
h := handlers.LoggingHandler(accessLog, noCaching(app.restrict(authedRouter)))
r.NotFoundHandler = h
predRouter := alb.NewPredictionRouter(r, app.sup, auth)
// TODO: add api routes
app.router = predRouter
return &app, nil
}
