Beispiel #1
0
// handleDeployment listens on the /deployer route and handles requests
// from the R and Python clients.
func (app *App) handleDeployment(w http.ResponseWriter, r *http.Request) {

	// for phone home metrics
	start := time.Now()

	username, apikey, ok := r.BasicAuth()
	if !ok || username == "" {
		username = r.FormValue("username")
		apikey = r.FormValue("apikey")
	}

	if username == "" {
		http.Error(w, "No auth provided", http.StatusUnauthorized)
		return
	}

	tx, err := app.db.Begin()
	if err != nil {
		app.dbError(w, r, err)
		return
	}
	defer tx.Rollback()

	user, err := db.GetUser(tx, username)
	if err != nil {
		if db.IsNotFound(err) {
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte("user not found"))
		} else {
			app.dbError(w, r, err)
		}
		return
	}
	if apikey != user.Apikey {
		http.Error(w, "APIKEY did not match", http.StatusUnauthorized)
		return
	}

	info, bundle, err := mps.ReadBundle(r)
	if err != nil {
		http.Error(w, "Could not parse upload request "+err.Error(), http.StatusBadRequest)
		return
	}

	modelSize := int64(len(bundle))

	// validate model info
	if !isValidModelName(info.Modelname) {
		http.Error(w, fmt.Sprintf("'%s' is an invalid model name", info.Modelname), http.StatusBadRequest)
		return
	}

	// Write bundle to disk
	timestamp := time.Now().UTC().Format("2006_01_02_15_04_05")
	bundleFilename := user.Name + "_" + info.Modelname + "_" + timestamp + ".json"
	b := filepath.Join(app.bundleDir, bundleFilename)
	log.Println(b)
	if err := ioutil.WriteFile(b, bundle, 0644); err != nil {
		http.Error(w, fmt.Sprintf("could not write model file to disk %v", err), http.StatusInternalServerError)
		return
	}

	// add bundle to database
	p := db.NewVersionParams{
		UserId:         user.Id,
		Model:          info.Modelname,
		Lang:           info.Lang,
		LangPackages:   info.LanguagePackages,
		UbuntuPackages: info.UbuntuPackages,
		SourceCode:     info.SourceCode,
		BundleFilename: bundleFilename,
	}
	versionNum, err := db.NewModelVersion(tx, &p)
	if err != nil {
		http.Error(w, "could not create new model version: "+err.Error(), http.StatusInternalServerError)
		return
	}
	if err := tx.Commit(); err != nil {
		app.dbError(w, r, err)
		return
	}

	clientIP := r.RemoteAddr

	go func(user, model string, version int) {
		app.logf("deployment of %s:%s:%d started", user, model, version)
		err := app.sup.Deploy(user, model, version)
		if err != nil {
			app.logf("deployment of %s:%s:%d failed %v", user, model, version, err)
		} else {
			app.logf("deployment of %s:%s:%d succeeded", user, model, version)
		}

		if app.isdev {
			return
		}

		// Send phone home metrics
		errmsg := ""
		if err != nil {
			errmsg = err.Error()
		}

		d := &events.Deployment{
			StartTime: start.Unix(),
			EndTime:   time.Now().Unix(),
			Username:  user,
			ModelName: model,
			ModelLang: info.Lang,
			ModelVer:  int64(version),
			Service:   app.serviceName,
			ClientIP:  clientIP,
			Error:     errmsg,
			ModelSize: modelSize,
			ModelDeps: packages(info.LanguagePackages).String(),
		}
		data, err := json.Marshal(&d)
		if err != nil {
			app.logf("could not marshal deployment: %v", err)
			return
		}

		resp, err := http.Post(events.Endpoint, "application/json", bytes.NewReader(data))
		if err != nil {
			app.logf("failed to make request: %v", err)
			return
		}
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				app.logf("could not read body: %v", err)
				return
			}
			app.logf("could not send info: %s %v", resp.Status, body)
		}

	}(user.Name, info.Modelname, versionNum)

	// send a JSON response to the user
	resp := struct {
		Status  string `json:"status"`
		Version int    `json:"version"`
	}{"Successfully deployed", versionNum}
	data, err := json.Marshal(&resp)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(data)
}
Beispiel #2
0
// - [x] create N users
// - [x] create N apikeys & read-only apikeys
// - [x] create M models for each user
//     - [x] create J versions for model
//     - [x] create model status for each model
//     - [ ] associate model w/ MPS (?)
//     - [x] give (some) models JSON/HTML examples
//     - [x] share model with K users
func seedFunc(conn *sql.DB) error {
	tx, err := conn.Begin()
	if err != nil {
		return fmt.Errorf("cannot begin transaction %v", err)
	}
	defer tx.Rollback()

	log.Println("truncating tables in db")
	tx.Exec(`use scienceops;`)
	tx.Exec(`SET FOREIGN_KEY_CHECKS=0;`)
	rows, err := tx.Query(`show tables;`)
	if err != nil {
		return fmt.Errorf("could not truncate database: ", err)
	}
	var tables []string

	defer rows.Close()
	for rows.Next() {
		var t string
		rows.Scan(&t)
		tables = append(tables, t)
	}
	for _, t := range tables {
		q := fmt.Sprintf("TRUNCATE TABLE %s;", t)
		if _, err := tx.Exec(q); err != nil {
			fmt.Println(err)
			return fmt.Errorf("Could not truncate table %s: %v", t, err)
		}
	}
	tx.Exec(`SET FOREIGN_KEY_CHECKS=1;`)

	log.Println("seeding db")

	nUsers := 3
	nModels := 10
	nVersions := 15

	os.MkdirAll("/tmp/bundles/", 0777)

	hashedPass := "******"
	for _, user := range []string{"eric", "ryan", "greg", "sush", "colin", "brandon", "austin", "charlie"} {
		_, err := db.NewUser(tx, user, hashedPass, user+"@yhathq.com", true)
		if err != nil {
			return fmt.Errorf("could not create user: %v", err)
		}
	}
	for _, user := range []string{"bigdatabob"} {
		_, err := db.NewUser(tx, user, hashedPass, user+"@yhathq.com", false)
		if err != nil {
			return fmt.Errorf("could not create user: %v", err)
		}
	}

	for i := 0; i < nUsers; i++ {
		username := fmt.Sprintf("user-%d", i)
		email := fmt.Sprintf("*****@*****.**", username)
		user, err := db.NewUser(tx, username, hashedPass, email, true)
		if err != nil {
			return fmt.Errorf("could not create user: %v", err)
		}
		log.Printf("Created user %s", user.Name)

		for j := 0; j < nModels; j++ {
			name := randomdata.SillyName()

			params := &db.NewVersionParams{
				UserId:         user.Id,
				Model:          name,
				Lang:           db.LangPython2,
				SourceCode:     "print HI!",
				BundleFilename: "/foobar/bundle.json",
			}

			for v := 0; v < nVersions; v++ {
				if _, err := db.NewModelVersion(tx, params); err != nil {
					return fmt.Errorf("could not create version: %v", err)
				}
			}
			model, err := db.GetModel(tx, username, name)
			if err != nil {
				return fmt.Errorf("could not get model %s/%s: %v", username, name, err)
			}
			err = db.SetModelStatus(tx, model.Id, "online")
			if err != nil {
				fmt.Println(err)
				return fmt.Errorf("could not insert model status: %v", err)
			}

		}

	}
	log.Printf("added %d users to db\n", nUsers)
	return tx.Commit()
}