func (e *ETCDToSQL) migrateDomains(logger lager.Logger) error {
logger = logger.Session("migrating-domains")
logger.Debug("starting")
defer logger.Debug("finished")
response, err := e.storeClient.Get(etcd.DomainSchemaRoot, false, true)
if err != nil {
logger.Error("failed-fetching-domains", err)
}
if response != nil {
for _, node := range response.Node.Nodes {
domain := path.Base(node.Key)
expireTime := e.clock.Now().UnixNano() + int64(time.Second)*node.TTL
_, err := e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
INSERT INTO domains
(domain, expire_time)
VALUES (?, ?)
`, e.dbFlavor), domain, expireTime)
if err != nil {
logger.Error("failed-inserting-domain", err)
continue
}
}
}
return nil
}
func createTables(logger lager.Logger, db *sql.DB, flavor string) error {
var createTablesSQL = []string{
sqldb.RebindForFlavor(createDomainSQL, flavor),
sqldb.RebindForFlavor(createDesiredLRPsSQL, flavor),
sqldb.RebindForFlavor(createActualLRPsSQL, flavor),
sqldb.RebindForFlavor(createTasksSQL, flavor),
}
logger.Info("creating-tables")
for _, query := range createTablesSQL {
logger.Info("creating the table", lager.Data{"query": query})
_, err := db.Exec(query)
if err != nil {
logger.Error("failed-creating-tables", err)
return err
}
logger.Info("created the table", lager.Data{"query": query})
}
return nil
}
func (e *ETCDToSQL) migrateActualLRPs(logger lager.Logger) error {
logger = logger.Session("migrating-actual-lrps")
logger.Debug("starting")
defer logger.Debug("finished")
response, err := e.storeClient.Get(etcd.ActualLRPSchemaRoot, false, true)
if err != nil {
logger.Error("failed-fetching-actual-lrps", err)
}
if response != nil {
for _, parent := range response.Node.Nodes {
for _, indices := range parent.Nodes {
for _, node := range indices.Nodes {
// we're going to explicitly ignore evacuating lrps for simplicity's sake
if path.Base(node.Key) == "instance" {
actualLRP := new(models.ActualLRP)
err := e.serializer.Unmarshal(logger, []byte(node.Value), actualLRP)
if err != nil {
logger.Error("failed-to-deserialize-actual-lrp", err)
continue
}
netInfoData, err := e.serializer.Marshal(logger, format.ENCRYPTED_PROTO, &actualLRP.ActualLRPNetInfo)
if err != nil {
logger.Error("failed-to-marshal-net-info", err)
}
_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
INSERT INTO actual_lrps
(process_guid, instance_index, domain, instance_guid, cell_id,
net_info, crash_count, crash_reason, state, placement_error, since,
modification_tag_epoch, modification_tag_index)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, e.dbFlavor), actualLRP.ProcessGuid, actualLRP.Index, actualLRP.Domain, actualLRP.InstanceGuid,
actualLRP.CellId, netInfoData, actualLRP.CrashCount, actualLRP.CrashReason,
actualLRP.State, actualLRP.PlacementError, actualLRP.Since,
actualLRP.ModificationTag.Epoch, actualLRP.ModificationTag.Index)
if err != nil {
logger.Error("failed-inserting-actual-lrp", err)
continue
}
}
}
}
}
}
return nil
}
func (e *EncryptRoutes) Up(logger lager.Logger) error {
logger = logger.Session("encrypt-route-column")
logger.Info("starting")
defer logger.Info("completed")
query := fmt.Sprintf("SELECT process_guid, routes FROM desired_lrps")
rows, err := e.rawSQLDB.Query(query)
if err != nil {
logger.Error("failed-query", err)
return err
}
defer rows.Close()
var processGuid string
var routeData []byte
for rows.Next() {
err := rows.Scan(&processGuid, &routeData)
if err != nil {
logger.Error("failed-reading-row", err)
continue
}
encodedData, err := e.encoder.Encode(format.BASE64_ENCRYPTED, routeData)
if err != nil {
logger.Error("failed-encrypting-routes", err)
return models.ErrBadRequest
}
bindings := make([]interface{}, 0, 3)
updateQuery := fmt.Sprintf("UPDATE desired_lrps SET routes = ? WHERE process_guid = ?")
bindings = append(bindings, encodedData)
bindings = append(bindings, processGuid)
_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(updateQuery, e.dbFlavor), bindings...)
if err != nil {
logger.Error("failed-updating-desired-lrp-record", err)
return models.ErrBadRequest
}
}
if rows.Err() != nil {
logger.Error("failed-fetching-row", rows.Err())
return rows.Err()
}
return nil
}
func (e *ETCDToSQL) migrateTasks(logger lager.Logger) error {
logger = logger.Session("migrating-tasks")
logger.Debug("starting")
defer logger.Debug("finished")
response, err := e.storeClient.Get(etcd.TaskSchemaRoot, false, true)
if err != nil {
logger.Error("failed-fetching-tasks", err)
}
if response != nil {
for _, node := range response.Node.Nodes {
task := new(models.Task)
err := e.serializer.Unmarshal(logger, []byte(node.Value), task)
if err != nil {
logger.Error("failed-to-deserialize-task", err)
continue
}
definitionData, err := e.serializer.Marshal(logger, format.ENCRYPTED_PROTO, task.TaskDefinition)
_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
INSERT INTO tasks
(guid, domain, updated_at, created_at, first_completed_at,
state, cell_id, result, failed, failure_reason,
task_definition)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, e.dbFlavor),
task.TaskGuid, task.Domain, task.UpdatedAt, task.CreatedAt,
task.FirstCompletedAt, task.State, task.CellId, task.Result,
task.Failed, task.FailureReason, definitionData)
if err != nil {
logger.Error("failed-inserting-task", err)
continue
}
}
}
return nil
}
err = json.Unmarshal([]byte(versionData), &actualVersion)
Expect(err).NotTo(HaveOccurred())
Expect(actualVersion).To(Equal(*expectedVersion))
})
})
Context("when a version is already set", func() {
var existingVersion *models.Version
BeforeEach(func() {
existingVersion = &models.Version{CurrentVersion: 99, TargetVersion: 100}
versionJSON, err := json.Marshal(existingVersion)
Expect(err).NotTo(HaveOccurred())
queryStr := "UPDATE configurations SET value = ? WHERE id = ?"
_, err = db.Exec(sqldb.RebindForFlavor(queryStr, dbDriverName), versionJSON, sqldb.VersionID)
Expect(err).NotTo(HaveOccurred())
})
It("updates the version in the db", func() {
version := &models.Version{CurrentVersion: 20, TargetVersion: 1001}
err := sqlDB.SetVersion(logger, version)
Expect(err).NotTo(HaveOccurred())
queryStr := "SELECT value FROM configurations WHERE id = ?"
if test_helpers.UsePostgres() {
queryStr = test_helpers.ReplaceQuestionMarks(queryStr)
}
rows, err := db.Query(queryStr, sqldb.VersionID)
Expect(err).NotTo(HaveOccurred())
It("does not error out", func() {
Expect(migErr).NotTo(HaveOccurred())
})
It("should add a placement_tags column to desired lrps", func() {
placementTags := []string{"tag-1"}
jsonData, err := json.Marshal(placementTags)
Expect(err).NotTo(HaveOccurred())
_, err = rawSQLDB.Exec(
sqldb.RebindForFlavor(
`INSERT INTO desired_lrps
(process_guid, domain, placement_tags, log_guid, instances, memory_mb,
disk_mb, rootfs, routes, volume_placement, modification_tag_epoch, run_info)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
flavor,
),
"guid", "domain",
jsonData,
"log guid", 2, 1, 1, "rootfs", "routes", "volumes yo", 1, "run info",
)
Expect(err).NotTo(HaveOccurred())
var fetchedJSONData string
query := sqldb.RebindForFlavor("select placement_tags from desired_lrps limit 1", flavor)
row := rawSQLDB.QueryRow(query)
Expect(row.Scan(&fetchedJSONData)).NotTo(HaveOccurred())
Expect(fetchedJSONData).To(BeEquivalentTo(jsonData))
})
routes TEXT NOT NULL,
volume_placement TEXT NOT NULL,
run_info TEXT NOT NULL
);`,
}
for _, st := range createStatements {
_, err := rawSQLDB.Exec(st)
Expect(err).NotTo(HaveOccurred())
}
})
It("does not error out", func() {
Expect(migrationErr).NotTo(HaveOccurred())
})
It("should change the size of all text columns ", func() {
value := strings.Repeat("x", 65536*2)
query := sqldb.RebindForFlavor("insert into desired_lrps(annotation, routes, volume_placement, run_info) values('', '', '', ?)", flavor)
_, err := rawSQLDB.Exec(query, value)
Expect(err).NotTo(HaveOccurred())
})
})
Describe("Down", func() {
It("returns a not implemented error", func() {
Expect(migration.Down(logger)).To(HaveOccurred())
})
})
}
})
BeforeEach(func() {
createStatement := `CREATE TABLE actual_lrps(
placement_error VARCHAR(255) NOT NULL DEFAULT '',
crash_reason VARCHAR(255) NOT NULL DEFAULT ''
);`
_, err := rawSQLDB.Exec(createStatement)
Expect(err).NotTo(HaveOccurred())
})
testTableAndColumn := func(table, column string) {
title := fmt.Sprintf("should change the size of %s column ", column)
It(title, func() {
value := strings.Repeat("x", 1024)
insertQuery := fmt.Sprintf("insert into %s(%s) values(?)", table, column)
query := sqldb.RebindForFlavor(insertQuery, flavor)
_, err := rawSQLDB.Exec(query, value)
Expect(err).NotTo(HaveOccurred())
selectQuery := fmt.Sprintf("select %s from %s", column, table)
row := rawSQLDB.QueryRow(selectQuery)
Expect(err).NotTo(HaveOccurred())
var actualValue string
Expect(row.Scan(&actualValue)).To(Succeed())
Expect(actualValue).To(Equal(value))
})
}
testTableAndColumn("actual_lrps", "crash_reason")
testTableAndColumn("actual_lrps", "placement_error")
It("does not change the default", func() {
func (e *ETCDToSQL) migrateDesiredLRPs(logger lager.Logger) error {
logger = logger.Session("migrating-desired-lrp-scheduling-infos")
logger.Debug("starting")
defer logger.Debug("finished")
response, err := e.storeClient.Get(etcd.DesiredLRPSchedulingInfoSchemaRoot, false, true)
if err != nil {
logger.Error("failed-fetching-desired-lrp-scheduling-infos", err)
}
schedInfos := make(map[string]*models.DesiredLRPSchedulingInfo)
if response != nil {
for _, node := range response.Node.Nodes {
model := new(models.DesiredLRPSchedulingInfo)
err := e.serializer.Unmarshal(logger, []byte(node.Value), model)
if err != nil {
logger.Error("failed-to-deserialize-desired-lrp-scheduling-info", err)
continue
}
schedInfos[path.Base(node.Key)] = model
}
}
logger.Info("migrating-desired-lrp-run-infos")
response, err = e.storeClient.Get(etcd.DesiredLRPRunInfoSchemaRoot, false, true)
if err != nil {
logger.Error("failed-fetching-desired-lrp-run-infos", err)
}
if response != nil {
for _, node := range response.Node.Nodes {
schedInfo := schedInfos[path.Base(node.Key)]
routeData, err := json.Marshal(schedInfo.Routes)
if err != nil {
logger.Error("failed-to-marshal-routes", err)
continue
}
if schedInfo.VolumePlacement == nil {
schedInfo.VolumePlacement = &models.VolumePlacement{}
}
volumePlacementData, err := e.serializer.Marshal(logger, format.ENCRYPTED_PROTO, schedInfo.VolumePlacement)
if err != nil {
logger.Error("failed-marshalling-volume-placements", err)
}
_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
INSERT INTO desired_lrps
(process_guid, domain, log_guid, annotation, instances, memory_mb,
disk_mb, rootfs, volume_placement, routes, modification_tag_epoch,
modification_tag_index, run_info)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, e.dbFlavor), schedInfo.ProcessGuid, schedInfo.Domain, schedInfo.LogGuid, schedInfo.Annotation,
schedInfo.Instances, schedInfo.MemoryMb, schedInfo.DiskMb, schedInfo.RootFs, volumePlacementData,
routeData, schedInfo.ModificationTag.Epoch, schedInfo.ModificationTag.Index, []byte(node.Value))
if err != nil {
logger.Error("failed-inserting-desired-lrp", err)
continue
}
}
}
return nil
}