// handleDeployment listens on the /deployer route and handles requests
// from the R and Python clients.
func (app *App) handleDeployment(w http.ResponseWriter, r *http.Request) {
// for phone home metrics
start := time.Now()
username, apikey, ok := r.BasicAuth()
if !ok || username == "" {
username = r.FormValue("username")
apikey = r.FormValue("apikey")
}
if username == "" {
http.Error(w, "No auth provided", http.StatusUnauthorized)
return
}
tx, err := app.db.Begin()
if err != nil {
app.dbError(w, r, err)
return
}
defer tx.Rollback()
user, err := db.GetUser(tx, username)
if err != nil {
if db.IsNotFound(err) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("user not found"))
} else {
app.dbError(w, r, err)
}
return
}
if apikey != user.Apikey {
http.Error(w, "APIKEY did not match", http.StatusUnauthorized)
return
}
info, bundle, err := mps.ReadBundle(r)
if err != nil {
http.Error(w, "Could not parse upload request "+err.Error(), http.StatusBadRequest)
return
}
modelSize := int64(len(bundle))
// validate model info
if !isValidModelName(info.Modelname) {
http.Error(w, fmt.Sprintf("'%s' is an invalid model name", info.Modelname), http.StatusBadRequest)
return
}
// Write bundle to disk
timestamp := time.Now().UTC().Format("2006_01_02_15_04_05")
bundleFilename := user.Name + "_" + info.Modelname + "_" + timestamp + ".json"
b := filepath.Join(app.bundleDir, bundleFilename)
log.Println(b)
if err := ioutil.WriteFile(b, bundle, 0644); err != nil {
http.Error(w, fmt.Sprintf("could not write model file to disk %v", err), http.StatusInternalServerError)
return
}
// add bundle to database
p := db.NewVersionParams{
UserId: user.Id,
Model: info.Modelname,
Lang: info.Lang,
LangPackages: info.LanguagePackages,
UbuntuPackages: info.UbuntuPackages,
SourceCode: info.SourceCode,
BundleFilename: bundleFilename,
}
versionNum, err := db.NewModelVersion(tx, &p)
if err != nil {
http.Error(w, "could not create new model version: "+err.Error(), http.StatusInternalServerError)
return
}
if err := tx.Commit(); err != nil {
app.dbError(w, r, err)
return
}
clientIP := r.RemoteAddr
go func(user, model string, version int) {
app.logf("deployment of %s:%s:%d started", user, model, version)
err := app.sup.Deploy(user, model, version)
if err != nil {
app.logf("deployment of %s:%s:%d failed %v", user, model, version, err)
} else {
app.logf("deployment of %s:%s:%d succeeded", user, model, version)
}
if app.isdev {
return
}
// Send phone home metrics
errmsg := ""
if err != nil {
errmsg = err.Error()
}
d := &events.Deployment{
StartTime: start.Unix(),
EndTime: time.Now().Unix(),
Username: user,
ModelName: model,
ModelLang: info.Lang,
ModelVer: int64(version),
Service: app.serviceName,
ClientIP: clientIP,
Error: errmsg,
ModelSize: modelSize,
ModelDeps: packages(info.LanguagePackages).String(),
}
data, err := json.Marshal(&d)
if err != nil {
app.logf("could not marshal deployment: %v", err)
return
}
resp, err := http.Post(events.Endpoint, "application/json", bytes.NewReader(data))
if err != nil {
app.logf("failed to make request: %v", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
app.logf("could not read body: %v", err)
return
}
app.logf("could not send info: %s %v", resp.Status, body)
}
}(user.Name, info.Modelname, versionNum)
// send a JSON response to the user
resp := struct {
Status string `json:"status"`
Version int `json:"version"`
}{"Successfully deployed", versionNum}
data, err := json.Marshal(&resp)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(data)
}
// - [x] create N users
// - [x] create N apikeys & read-only apikeys
// - [x] create M models for each user
// - [x] create J versions for model
// - [x] create model status for each model
// - [ ] associate model w/ MPS (?)
// - [x] give (some) models JSON/HTML examples
// - [x] share model with K users
func seedFunc(conn *sql.DB) error {
tx, err := conn.Begin()
if err != nil {
return fmt.Errorf("cannot begin transaction %v", err)
}
defer tx.Rollback()
log.Println("truncating tables in db")
tx.Exec(`use scienceops;`)
tx.Exec(`SET FOREIGN_KEY_CHECKS=0;`)
rows, err := tx.Query(`show tables;`)
if err != nil {
return fmt.Errorf("could not truncate database: ", err)
}
var tables []string
defer rows.Close()
for rows.Next() {
var t string
rows.Scan(&t)
tables = append(tables, t)
}
for _, t := range tables {
q := fmt.Sprintf("TRUNCATE TABLE %s;", t)
if _, err := tx.Exec(q); err != nil {
fmt.Println(err)
return fmt.Errorf("Could not truncate table %s: %v", t, err)
}
}
tx.Exec(`SET FOREIGN_KEY_CHECKS=1;`)
log.Println("seeding db")
nUsers := 3
nModels := 10
nVersions := 15
os.MkdirAll("/tmp/bundles/", 0777)
hashedPass := "******"
for _, user := range []string{"eric", "ryan", "greg", "sush", "colin", "brandon", "austin", "charlie"} {
_, err := db.NewUser(tx, user, hashedPass, user+"@yhathq.com", true)
if err != nil {
return fmt.Errorf("could not create user: %v", err)
}
}
for _, user := range []string{"bigdatabob"} {
_, err := db.NewUser(tx, user, hashedPass, user+"@yhathq.com", false)
if err != nil {
return fmt.Errorf("could not create user: %v", err)
}
}
for i := 0; i < nUsers; i++ {
username := fmt.Sprintf("user-%d", i)
email := fmt.Sprintf("*****@*****.**", username)
user, err := db.NewUser(tx, username, hashedPass, email, true)
if err != nil {
return fmt.Errorf("could not create user: %v", err)
}
log.Printf("Created user %s", user.Name)
for j := 0; j < nModels; j++ {
name := randomdata.SillyName()
params := &db.NewVersionParams{
UserId: user.Id,
Model: name,
Lang: db.LangPython2,
SourceCode: "print HI!",
BundleFilename: "/foobar/bundle.json",
}
for v := 0; v < nVersions; v++ {
if _, err := db.NewModelVersion(tx, params); err != nil {
return fmt.Errorf("could not create version: %v", err)
}
}
model, err := db.GetModel(tx, username, name)
if err != nil {
return fmt.Errorf("could not get model %s/%s: %v", username, name, err)
}
err = db.SetModelStatus(tx, model.Id, "online")
if err != nil {
fmt.Println(err)
return fmt.Errorf("could not insert model status: %v", err)
}
}
}
log.Printf("added %d users to db\n", nUsers)
return tx.Commit()
}