Esempio n. 1
0
func (e *ETCDToSQL) migrateDomains(logger lager.Logger) error {
	logger = logger.Session("migrating-domains")
	logger.Debug("starting")
	defer logger.Debug("finished")

	response, err := e.storeClient.Get(etcd.DomainSchemaRoot, false, true)
	if err != nil {
		logger.Error("failed-fetching-domains", err)
	}

	if response != nil {
		for _, node := range response.Node.Nodes {
			domain := path.Base(node.Key)
			expireTime := e.clock.Now().UnixNano() + int64(time.Second)*node.TTL

			_, err := e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
				INSERT INTO domains
				(domain, expire_time)
				VALUES (?, ?)
		  `, e.dbFlavor), domain, expireTime)
			if err != nil {
				logger.Error("failed-inserting-domain", err)
				continue
			}
		}
	}

	return nil
}
Esempio n. 2
0
func createTables(logger lager.Logger, db *sql.DB, flavor string) error {
	var createTablesSQL = []string{
		sqldb.RebindForFlavor(createDomainSQL, flavor),
		sqldb.RebindForFlavor(createDesiredLRPsSQL, flavor),
		sqldb.RebindForFlavor(createActualLRPsSQL, flavor),
		sqldb.RebindForFlavor(createTasksSQL, flavor),
	}

	logger.Info("creating-tables")
	for _, query := range createTablesSQL {
		logger.Info("creating the table", lager.Data{"query": query})
		_, err := db.Exec(query)
		if err != nil {
			logger.Error("failed-creating-tables", err)
			return err
		}
		logger.Info("created the table", lager.Data{"query": query})
	}

	return nil
}
Esempio n. 3
0
func (e *ETCDToSQL) migrateActualLRPs(logger lager.Logger) error {
	logger = logger.Session("migrating-actual-lrps")
	logger.Debug("starting")
	defer logger.Debug("finished")
	response, err := e.storeClient.Get(etcd.ActualLRPSchemaRoot, false, true)
	if err != nil {
		logger.Error("failed-fetching-actual-lrps", err)
	}

	if response != nil {
		for _, parent := range response.Node.Nodes {
			for _, indices := range parent.Nodes {
				for _, node := range indices.Nodes {
					// we're going to explicitly ignore evacuating lrps for simplicity's sake
					if path.Base(node.Key) == "instance" {
						actualLRP := new(models.ActualLRP)
						err := e.serializer.Unmarshal(logger, []byte(node.Value), actualLRP)
						if err != nil {
							logger.Error("failed-to-deserialize-actual-lrp", err)
							continue
						}

						netInfoData, err := e.serializer.Marshal(logger, format.ENCRYPTED_PROTO, &actualLRP.ActualLRPNetInfo)
						if err != nil {
							logger.Error("failed-to-marshal-net-info", err)
						}

						_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
							INSERT INTO actual_lrps
								(process_guid, instance_index, domain, instance_guid, cell_id,
								net_info, crash_count, crash_reason, state, placement_error, since,
								modification_tag_epoch, modification_tag_index)
							VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
						`, e.dbFlavor), actualLRP.ProcessGuid, actualLRP.Index, actualLRP.Domain, actualLRP.InstanceGuid,
							actualLRP.CellId, netInfoData, actualLRP.CrashCount, actualLRP.CrashReason,
							actualLRP.State, actualLRP.PlacementError, actualLRP.Since,
							actualLRP.ModificationTag.Epoch, actualLRP.ModificationTag.Index)
						if err != nil {
							logger.Error("failed-inserting-actual-lrp", err)
							continue
						}
					}
				}
			}
		}
	}

	return nil
}
func (e *EncryptRoutes) Up(logger lager.Logger) error {
	logger = logger.Session("encrypt-route-column")
	logger.Info("starting")
	defer logger.Info("completed")

	query := fmt.Sprintf("SELECT process_guid, routes FROM desired_lrps")

	rows, err := e.rawSQLDB.Query(query)
	if err != nil {
		logger.Error("failed-query", err)
		return err
	}
	defer rows.Close()

	var processGuid string
	var routeData []byte

	for rows.Next() {
		err := rows.Scan(&processGuid, &routeData)
		if err != nil {
			logger.Error("failed-reading-row", err)
			continue
		}
		encodedData, err := e.encoder.Encode(format.BASE64_ENCRYPTED, routeData)
		if err != nil {
			logger.Error("failed-encrypting-routes", err)
			return models.ErrBadRequest
		}

		bindings := make([]interface{}, 0, 3)
		updateQuery := fmt.Sprintf("UPDATE desired_lrps SET routes = ? WHERE process_guid = ?")
		bindings = append(bindings, encodedData)
		bindings = append(bindings, processGuid)
		_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(updateQuery, e.dbFlavor), bindings...)
		if err != nil {
			logger.Error("failed-updating-desired-lrp-record", err)
			return models.ErrBadRequest
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-fetching-row", rows.Err())
		return rows.Err()
	}
	return nil
}
Esempio n. 5
0
func (e *ETCDToSQL) migrateTasks(logger lager.Logger) error {
	logger = logger.Session("migrating-tasks")
	logger.Debug("starting")
	defer logger.Debug("finished")
	response, err := e.storeClient.Get(etcd.TaskSchemaRoot, false, true)
	if err != nil {
		logger.Error("failed-fetching-tasks", err)
	}

	if response != nil {
		for _, node := range response.Node.Nodes {
			task := new(models.Task)
			err := e.serializer.Unmarshal(logger, []byte(node.Value), task)
			if err != nil {
				logger.Error("failed-to-deserialize-task", err)
				continue
			}

			definitionData, err := e.serializer.Marshal(logger, format.ENCRYPTED_PROTO, task.TaskDefinition)

			_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
							INSERT INTO tasks
								(guid, domain, updated_at, created_at, first_completed_at,
								state, cell_id, result, failed, failure_reason,
								task_definition)
							VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
						`, e.dbFlavor),
				task.TaskGuid, task.Domain, task.UpdatedAt, task.CreatedAt,
				task.FirstCompletedAt, task.State, task.CellId, task.Result,
				task.Failed, task.FailureReason, definitionData)
			if err != nil {
				logger.Error("failed-inserting-task", err)
				continue
			}
		}
	}

	return nil
}
Esempio n. 6
0
				err = json.Unmarshal([]byte(versionData), &actualVersion)
				Expect(err).NotTo(HaveOccurred())

				Expect(actualVersion).To(Equal(*expectedVersion))
			})
		})

		Context("when a version is already set", func() {
			var existingVersion *models.Version
			BeforeEach(func() {
				existingVersion = &models.Version{CurrentVersion: 99, TargetVersion: 100}
				versionJSON, err := json.Marshal(existingVersion)
				Expect(err).NotTo(HaveOccurred())

				queryStr := "UPDATE configurations SET value = ? WHERE id = ?"
				_, err = db.Exec(sqldb.RebindForFlavor(queryStr, dbDriverName), versionJSON, sqldb.VersionID)
				Expect(err).NotTo(HaveOccurred())
			})

			It("updates the version in the db", func() {
				version := &models.Version{CurrentVersion: 20, TargetVersion: 1001}

				err := sqlDB.SetVersion(logger, version)
				Expect(err).NotTo(HaveOccurred())

				queryStr := "SELECT value FROM configurations WHERE id = ?"
				if test_helpers.UsePostgres() {
					queryStr = test_helpers.ReplaceQuestionMarks(queryStr)
				}
				rows, err := db.Query(queryStr, sqldb.VersionID)
				Expect(err).NotTo(HaveOccurred())
		It("does not error out", func() {
			Expect(migErr).NotTo(HaveOccurred())
		})

		It("should add a placement_tags column to desired lrps", func() {
			placementTags := []string{"tag-1"}

			jsonData, err := json.Marshal(placementTags)
			Expect(err).NotTo(HaveOccurred())

			_, err = rawSQLDB.Exec(
				sqldb.RebindForFlavor(
					`INSERT INTO desired_lrps
						  (process_guid, domain, placement_tags, log_guid, instances, memory_mb,
							  disk_mb, rootfs, routes, volume_placement, modification_tag_epoch, run_info)
						  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
					flavor,
				),
				"guid", "domain",
				jsonData,
				"log guid", 2, 1, 1, "rootfs", "routes", "volumes yo", 1, "run info",
			)
			Expect(err).NotTo(HaveOccurred())

			var fetchedJSONData string
			query := sqldb.RebindForFlavor("select placement_tags from desired_lrps limit 1", flavor)
			row := rawSQLDB.QueryRow(query)
			Expect(row.Scan(&fetchedJSONData)).NotTo(HaveOccurred())
			Expect(fetchedJSONData).To(BeEquivalentTo(jsonData))
		})
	routes TEXT NOT NULL,
	volume_placement TEXT NOT NULL,
	run_info TEXT NOT NULL
);`,
				}
				for _, st := range createStatements {
					_, err := rawSQLDB.Exec(st)
					Expect(err).NotTo(HaveOccurred())
				}
			})

			It("does not error out", func() {
				Expect(migrationErr).NotTo(HaveOccurred())
			})

			It("should change the size of all text columns ", func() {
				value := strings.Repeat("x", 65536*2)
				query := sqldb.RebindForFlavor("insert into desired_lrps(annotation, routes, volume_placement, run_info) values('', '', '', ?)", flavor)
				_, err := rawSQLDB.Exec(query, value)
				Expect(err).NotTo(HaveOccurred())
			})
		})

		Describe("Down", func() {
			It("returns a not implemented error", func() {
				Expect(migration.Down(logger)).To(HaveOccurred())
			})
		})
	}
})
			BeforeEach(func() {
				createStatement := `CREATE TABLE actual_lrps(
	placement_error VARCHAR(255) NOT NULL DEFAULT '',
	crash_reason VARCHAR(255) NOT NULL DEFAULT ''
);`
				_, err := rawSQLDB.Exec(createStatement)
				Expect(err).NotTo(HaveOccurred())
			})

			testTableAndColumn := func(table, column string) {
				title := fmt.Sprintf("should change the size of %s column ", column)
				It(title, func() {
					value := strings.Repeat("x", 1024)
					insertQuery := fmt.Sprintf("insert into %s(%s) values(?)", table, column)
					query := sqldb.RebindForFlavor(insertQuery, flavor)
					_, err := rawSQLDB.Exec(query, value)
					Expect(err).NotTo(HaveOccurred())
					selectQuery := fmt.Sprintf("select %s from %s", column, table)
					row := rawSQLDB.QueryRow(selectQuery)
					Expect(err).NotTo(HaveOccurred())
					var actualValue string
					Expect(row.Scan(&actualValue)).To(Succeed())
					Expect(actualValue).To(Equal(value))
				})
			}

			testTableAndColumn("actual_lrps", "crash_reason")
			testTableAndColumn("actual_lrps", "placement_error")

			It("does not change the default", func() {
Esempio n. 10
0
func (e *ETCDToSQL) migrateDesiredLRPs(logger lager.Logger) error {
	logger = logger.Session("migrating-desired-lrp-scheduling-infos")
	logger.Debug("starting")
	defer logger.Debug("finished")
	response, err := e.storeClient.Get(etcd.DesiredLRPSchedulingInfoSchemaRoot, false, true)
	if err != nil {
		logger.Error("failed-fetching-desired-lrp-scheduling-infos", err)
	}

	schedInfos := make(map[string]*models.DesiredLRPSchedulingInfo)

	if response != nil {
		for _, node := range response.Node.Nodes {
			model := new(models.DesiredLRPSchedulingInfo)
			err := e.serializer.Unmarshal(logger, []byte(node.Value), model)
			if err != nil {
				logger.Error("failed-to-deserialize-desired-lrp-scheduling-info", err)
				continue
			}
			schedInfos[path.Base(node.Key)] = model
		}
	}

	logger.Info("migrating-desired-lrp-run-infos")
	response, err = e.storeClient.Get(etcd.DesiredLRPRunInfoSchemaRoot, false, true)
	if err != nil {
		logger.Error("failed-fetching-desired-lrp-run-infos", err)
	}

	if response != nil {
		for _, node := range response.Node.Nodes {
			schedInfo := schedInfos[path.Base(node.Key)]
			routeData, err := json.Marshal(schedInfo.Routes)
			if err != nil {
				logger.Error("failed-to-marshal-routes", err)
				continue
			}

			if schedInfo.VolumePlacement == nil {
				schedInfo.VolumePlacement = &models.VolumePlacement{}
			}

			volumePlacementData, err := e.serializer.Marshal(logger, format.ENCRYPTED_PROTO, schedInfo.VolumePlacement)
			if err != nil {
				logger.Error("failed-marshalling-volume-placements", err)
			}

			_, err = e.rawSQLDB.Exec(sqldb.RebindForFlavor(`
				INSERT INTO desired_lrps
					(process_guid, domain, log_guid, annotation, instances, memory_mb,
					disk_mb, rootfs, volume_placement, routes, modification_tag_epoch,
					modification_tag_index, run_info)
				VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
			`, e.dbFlavor), schedInfo.ProcessGuid, schedInfo.Domain, schedInfo.LogGuid, schedInfo.Annotation,
				schedInfo.Instances, schedInfo.MemoryMb, schedInfo.DiskMb, schedInfo.RootFs, volumePlacementData,
				routeData, schedInfo.ModificationTag.Epoch, schedInfo.ModificationTag.Index, []byte(node.Value))
			if err != nil {
				logger.Error("failed-inserting-desired-lrp", err)
				continue
			}
		}
	}

	return nil
}