func ensureConfigExists(database *pgx.Tx, config Config) *requestError {
dbConfig, reqErr := fetchConfig(database, config.Name)
if reqErr != nil {
return reqErr
}
if dbConfig != nil {
if !config.isSameAs(dbConfig) {
return badRequestError("Config does not match the one in the database")
}
return nil
}
envVars, err := json.Marshal(config.MonoEnvironmentVariables)
if err != nil {
return internalServerError("Could not marshal mono environment variables to JSON")
}
_, err = database.Exec("insertConfig", config.Name, config.MonoExecutable, string(envVars), config.MonoOptions)
if err != nil {
fmt.Printf("config insert error: %s\n", err.Error())
return internalServerError("Could not insert config")
}
return nil
}
func FindModel(model models.Model, db *pgx.ConnPool, tx *pgx.Tx, where string, fields ...string) error {
fieldNames, fieldValues := model.Fields(fields...)
query := SqlSelect(model.TableName(), fieldNames)
where = fmt.Sprintf(" WHERE %s = ? %s", model.PrimaryName(), where)
query = FormateToPQuery(query + where)
args := []interface{}{model.PrimaryValue()}
var err error
if tx != nil {
err = tx.QueryRow(query, args...).Scan(fieldValues...)
} else {
err = db.QueryRow(query, args...).Scan(fieldValues...)
}
if err == pgx.ErrNoRows {
return models.ErrNotFound
}
if err != nil {
return err
}
return nil
}
func updateRunSet(database *pgx.Tx, runSetID int32, rs *RunSet) *requestError {
_, err := database.Exec("updateRunSet", runSetID, rs.LogURLs, rs.TimedOutBenchmarks, rs.CrashedBenchmarks)
if err != nil {
fmt.Printf("update run set error: %s\n", err)
return internalServerError("Could not update run set")
}
return nil
}
func fetchConfig(database *pgx.Tx, name string) (*Config, *requestError) {
dbConfig := Config{Name: name}
err := database.QueryRow("queryConfig", name).Scan(&dbConfig.MonoExecutable, &dbConfig.MonoEnvironmentVariables, &dbConfig.MonoOptions)
if err == pgx.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, internalServerError("Database query error")
}
return &dbConfig, nil
}
func (d *pgDataStore) addCertWithTx(tx *pgx.Tx, c *router.Certificate) error {
c.Cert = strings.Trim(c.Cert, " \n")
c.Key = strings.Trim(c.Key, " \n")
if _, err := tls.X509KeyPair([]byte(c.Cert), []byte(c.Key)); err != nil {
return httphelper.JSONError{
Code: httphelper.ValidationErrorCode,
Message: "Certificate invalid: " + err.Error(),
}
}
tlsCertSHA256 := sha256.Sum256([]byte(c.Cert))
if err := tx.QueryRow("select_certificate_by_sha", tlsCertSHA256[:]).Scan(&c.ID, &c.CreatedAt, &c.UpdatedAt); err != nil {
if err := tx.QueryRow("insert_certificate", c.Cert, c.Key, tlsCertSHA256[:]).Scan(&c.ID, &c.CreatedAt, &c.UpdatedAt); err != nil {
return err
}
}
for _, rid := range c.Routes {
if _, err := tx.Exec("delete_route_certificate_by_route_id", rid); err != nil {
return err
}
if _, err := tx.Exec("insert_route_certificate", rid, c.ID); err != nil {
return err
}
}
return nil
}
func fetchRunSetSummaries(database *pgx.Tx, machine string, config string) ([]RunSetSummary, *requestError) {
rows, err := database.Query("queryRunSetSummaries", machine, config)
if err != nil {
return nil, internalServerError("Could not fetch run set summaries")
}
var summaries []RunSetSummary
for rows.Next() {
var rss RunSetSummary
if err = rows.Scan(&rss.ID, &rss.MainProduct.Commit); err != nil {
return nil, internalServerError("Could not scan run set summary")
}
summaries = append(summaries, rss)
}
return summaries, nil
}
func (r *Run) ensureBenchmarksAndMetricsExist(database *pgx.Tx, benchmarks map[string]bool) *requestError {
if benchmarks != nil && !benchmarks[r.Benchmark] {
_, err := database.Exec("insertBenchmark", r.Benchmark)
if err != nil {
return badRequestError("Couldn't insert Benchmark: " + r.Benchmark)
}
benchmarks[r.Benchmark] = true
}
for m, v := range r.Results {
if !metricIsAllowed(m, v) {
return badRequestError("Metric not supported or results of wrong type: " + m)
}
}
return nil
}
func insertResults(database *pgx.Tx, runID int32, results Results, update bool) *requestError {
idByMetric := make(map[string]int32)
if update {
rows, err := database.Query("queryRunMetricsForRun", runID)
if err != nil {
return internalServerError("Could not query run metrics")
}
for rows.Next() {
var runMetricID int32
var metric string
err = rows.Scan(&runMetricID, &metric)
if err != nil {
return internalServerError("Could not scan run metric")
}
idByMetric[metric] = runMetricID
}
}
for m, v := range results {
var err error
runMetricID, exists := idByMetric[m]
if metricIsArray(m) {
var arr []float64
for _, x := range v.([]interface{}) {
arr = append(arr, x.(float64))
}
if exists {
_, err = database.Exec("updateRunMetricArray", runMetricID, arr)
} else {
_, err = database.Exec("insertRunMetricArray", runID, m, arr)
}
} else {
if exists {
_, err = database.Exec("updateRunMetricNumber", runMetricID, v)
} else {
_, err = database.Exec("insertRunMetricNumber", runID, m, v)
}
}
if err != nil {
fmt.Printf("run metric insert error: %s\n", err)
return internalServerError("Could not insert run metric")
}
}
return nil
}
func insertRuns(database *pgx.Tx, runSetID int32, runs []Run) ([]int32, *requestError) {
var runIDs []int32
for _, run := range runs {
var runID int32
err := database.QueryRow("insertRun", run.Benchmark, runSetID).Scan(&runID)
if err != nil {
fmt.Printf("run insert error: %s\n", err)
return nil, internalServerError("Could not insert run")
}
runIDs = append(runIDs, runID)
reqErr := insertResults(database, runID, run.Results, false)
if reqErr != nil {
return nil, reqErr
}
}
return runIDs, nil
}
func fetchBenchmarks(database *pgx.Tx) (map[string]bool, *requestError) {
rows, err := database.Query("queryBenchmarks")
if err != nil {
return nil, internalServerError("Could not get benchmarks")
}
defer rows.Close()
names := make(map[string]bool)
for rows.Next() {
var name string
if err = rows.Scan(&name); err != nil {
return nil, internalServerError("Could not scan benchmark name")
}
names[name] = true
}
return names, nil
}
func updateOrCreateModel(model models.Model, db *pgx.ConnPool, tx *pgx.Tx, isNew, isRemove bool, where string, fields ...string) error {
if isNew {
model.BeforeCreate()
}
if isRemove {
model.BeforeDelete()
}
model.BeforeSave()
fieldNames, fieldValues := model.Fields(fields...)
var query string = SqlUpdate(model.TableName(), fieldNames)
if isNew {
query = SqlInsert(model.TableName(), fieldNames)
}
where = fmt.Sprintf(" WHERE %[1]s = ? %[2]s RETURNING %[1]s", model.PrimaryName(), where)
if isNew {
where = fmt.Sprintf(" RETURNING %s", model.PrimaryName())
} else {
fieldValues = append(fieldValues, model.PrimaryValue())
}
query = FormateToPQuery(query + where)
var err error
if tx != nil {
err = tx.QueryRow(query, fieldValues...).Scan(model.Maps()[model.PrimaryName()])
} else {
err = db.QueryRow(query, fieldValues...).Scan(model.Maps()[model.PrimaryName()])
}
if err != nil {
return err
}
return nil
}
func ensureProductExists(database *pgx.Tx, product Product) (string, *requestError) {
var commitDate time.Time
// FIXME: check and update mergeBaseHash
err := database.QueryRow("queryCommitWithProduct", product.Commit, product.Name).Scan(&commitDate)
if err == nil {
return product.Commit, nil
}
if err == pgx.ErrNoRows {
var sha string
fmt.Printf("Commit %s for product %s not found in DB.\n", product.Commit, product.Name)
owner, repo := githubRepoForProduct(product.Name)
if owner == "" || repo == "" {
return "", badRequestError("Unknown product")
}
commit, _, err := githubClient.Git.GetCommit(owner, repo, product.Commit)
if err == nil {
sha = *commit.SHA
commitDate = *commit.Committer.Date
} else {
// try to fall back to provided commitDate
if product.CommitDate != nil {
sha = product.Commit
commitDate = *product.CommitDate
} else {
return "", badRequestError("Could not get date for provided commit " + product.Commit)
}
}
_, err = database.Exec("insertCommit", sha, commitDate, product.Name, product.MergeBaseHash)
if err != nil {
return "", internalServerError("Couldn't insert commit")
}
return sha, nil
}
return "", internalServerError("Database query error")
}
func ensureMachineExists(database *pgx.Tx, machine Machine) *requestError {
var architecture string
err := database.QueryRow("queryMachine", machine.Name).Scan(&architecture)
if err == nil {
if machine.Architecture != architecture {
return badRequestError("Machine has a different architecture than the one in the database")
}
return nil
}
if err == pgx.ErrNoRows {
_, err = database.Exec("insertMachine", machine.Name, machine.Architecture)
if err != nil {
return internalServerError("Could not insert machine")
}
return nil
}
return internalServerError("Database query error")
}
func deleteRunSet(database *pgx.Tx, runSetID int32) (int64, int64, *requestError) {
res, err := database.Exec("deleteRunMetricByRunSetId", runSetID)
if err != nil {
fmt.Printf("delete runmetric by runsetid: %s\n", err)
return 0, 0, internalServerError("Could not delete runset")
}
numMetrics := res.RowsAffected()
res, err = database.Exec("deleteRunByRunSetId", runSetID)
if err != nil {
fmt.Printf("delete run by runsetid: %s\n", err)
return 0, 0, internalServerError("Could not delete runset")
}
numRuns := res.RowsAffected()
_, err = database.Exec("deleteRunSet", runSetID)
if err != nil {
fmt.Printf("delete runset: %s\n", err)
return 0, 0, internalServerError("Could not delete runset")
}
return numMetrics, numRuns, nil
}
func (p *Process) assumePrimary(downstream *discoverd.Instance) (err error) {
log := p.log.New("fn", "assumePrimary")
if downstream != nil {
log = log.New("downstream", downstream.Addr)
}
if p.running() && p.config().Role == state.RoleSync {
log.Info("promoting to primary")
if err := ioutil.WriteFile(p.triggerPath(), nil, 0655); err != nil {
log.Error("error creating trigger file", "path", p.triggerPath(), "err", err)
return err
}
p.waitForSync(downstream, true)
return nil
}
log.Info("starting as primary")
if p.running() {
panic(fmt.Sprintf("unexpected state running role=%s", p.config().Role))
}
if err := p.initDB(); err != nil {
return err
}
if err := os.Remove(p.recoveryConfPath()); err != nil && !os.IsNotExist(err) {
log.Error("error removing recovery.conf", "path", p.recoveryConfPath(), "err", err)
return err
}
if err := p.writeConfig(configData{ReadOnly: downstream != nil}); err != nil {
log.Error("error writing postgres.conf", "path", p.configPath(), "err", err)
return err
}
if err := p.start(); err != nil {
return err
}
var tx *pgx.Tx
defer func() {
if err != nil {
if tx != nil {
tx.Rollback()
}
p.db.Close()
if err := p.stop(); err != nil {
log.Debug("ignoring error stopping postgres", "err", err)
}
}
}()
tx, err = p.db.Begin()
if err != nil {
log.Error("error acquiring connection", "err", err)
return err
}
if _, err := tx.Exec("SET TRANSACTION READ WRITE"); err != nil {
log.Error("error setting transaction read-write", "err", err)
return err
}
if _, err := tx.Exec(fmt.Sprintf(`
DO
$body$
BEGIN
IF NOT EXISTS (
SELECT * FROM pg_catalog.pg_user
WHERE usename = 'flynn')
THEN
CREATE USER flynn WITH SUPERUSER CREATEDB CREATEROLE REPLICATION PASSWORD '%s';
END IF;
END
$body$;
`, p.password)); err != nil {
log.Error("error creating superuser", "err", err)
return err
}
if err := tx.Commit(); err != nil {
log.Error("error committing transaction", "err", err)
return err
}
if downstream != nil {
p.waitForSync(downstream, true)
}
return nil
}
func runSetPutHandler(database *pgx.Tx, w http.ResponseWriter, r *http.Request, body []byte) (bool, *requestError) {
var params RunSet
if err := json.Unmarshal(body, ¶ms); err != nil {
fmt.Printf("Unmarshal error: %s\n", err.Error())
return false, badRequestError("Could not parse request body")
}
reqErr := ensureMachineExists(database, params.Machine)
if reqErr != nil {
return false, reqErr
}
mainCommit, reqErr := ensureProductExists(database, params.MainProduct)
if reqErr != nil {
return false, reqErr
}
var secondaryCommits []string
for _, p := range params.SecondaryProducts {
commit, reqErr := ensureProductExists(database, p)
if reqErr != nil {
return false, reqErr
}
secondaryCommits = append(secondaryCommits, commit)
}
reqErr = params.ensureBenchmarksAndMetricsExist(database)
if reqErr != nil {
return false, reqErr
}
reqErr = ensureConfigExists(database, params.Config)
if reqErr != nil {
return false, reqErr
}
var pullRequestID *int32
if params.PullRequest != nil {
var prID int32
pullRequestID = &prID
err := database.QueryRow("insertPullRequest",
params.PullRequest.BaselineRunSetID,
params.PullRequest.URL).Scan(pullRequestID)
if err != nil {
fmt.Printf("pull request insert error: %s\n", err)
return false, internalServerError("Could not insert pull request")
}
}
var runSetID int32
err := database.QueryRow("insertRunSet",
params.StartedAt, params.FinishedAt,
params.BuildURL, params.LogURLs,
mainCommit, secondaryCommits, params.Machine.Name, params.Config.Name,
params.TimedOutBenchmarks, params.CrashedBenchmarks,
pullRequestID).Scan(&runSetID)
if err != nil {
fmt.Printf("run set insert error: %s\n", err)
return false, internalServerError("Could not insert run set")
}
runIDs, reqErr := insertRuns(database, runSetID, params.Runs)
if reqErr != nil {
return false, reqErr
}
resp := runsetPostResponse{RunSetID: runSetID, RunIDs: runIDs, PullRequestID: pullRequestID}
respBytes, err := json.Marshal(&resp)
if err != nil {
return false, internalServerError("Could not produce JSON for response")
}
w.WriteHeader(http.StatusCreated)
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.Write(respBytes)
return true, nil
}
func fetchRunSet(database *pgx.Tx, id int32, withRuns bool) (*RunSet, *requestError) {
var rs RunSet
var secondaryCommits []string
var pullRequestID *int32
err := database.QueryRow("queryRunSet", id).Scan(&rs.StartedAt, &rs.FinishedAt,
&rs.BuildURL, &rs.LogURLs,
&rs.MainProduct.Commit, &secondaryCommits, &rs.Machine.Name, &rs.Config.Name,
&rs.TimedOutBenchmarks, &rs.CrashedBenchmarks,
&pullRequestID)
if err == pgx.ErrNoRows {
return nil, badRequestError("Run set does not exist")
}
if err != nil {
fmt.Printf("run set query error: %s\n", err.Error())
return nil, internalServerError("Could not get run set")
}
// FIXME: combine these two queries with the above one
err = database.QueryRow("queryCommit", rs.MainProduct.Commit).Scan(&rs.MainProduct.Name)
if err != nil {
return nil, internalServerError("Could not get commit")
}
for _, c := range secondaryCommits {
product := Product{Commit: c}
err = database.QueryRow("queryCommit", c).Scan(&product.Name)
if err != nil {
return nil, internalServerError("Could not get commit")
}
rs.SecondaryProducts = append(rs.SecondaryProducts, product)
}
err = database.QueryRow("queryMachine", rs.Machine.Name).Scan(&rs.Machine.Architecture)
if err != nil {
return nil, internalServerError("Could not get machine")
}
config, reqErr := fetchConfig(database, rs.Config.Name)
if reqErr != nil {
return nil, nil
}
rs.Config = *config
if withRuns {
rows, err := database.Query("queryRunMetrics", id)
if err != nil {
return nil, internalServerError("Could not get run metrics")
}
defer rows.Close()
runs := make(map[int32]*Run)
for rows.Next() {
var runID int32
var benchmark string
var metric string
var resultNumber *float64
var resultArray []float64
if err = rows.Scan(&runID, &benchmark, &metric, &resultNumber, &resultArray); err != nil {
return nil, internalServerError("Could not scan run metric: " + err.Error())
}
var resultValue interface{}
if metricIsArray(metric) {
resultValue = resultArray
} else if resultNumber != nil {
resultValue = *resultNumber
} else {
return nil, internalServerError("Run metric data is inconsistent")
}
run := runs[runID]
if run == nil {
run = &Run{Benchmark: benchmark, Results: make(map[string]interface{})}
runs[runID] = run
}
run.Results[metric] = resultValue
}
for _, r := range runs {
rs.Runs = append(rs.Runs, *r)
}
}
return &rs, nil
}