func TestMycnf(t *testing.T) {
dbaConfig := dbconfigs.DefaultDBConfigs.Dba
replConfig := dbconfigs.DefaultDBConfigs.Repl
tablet0 := NewMysqld(NewMycnf(0, 6802), &dbaConfig, &replConfig)
root, err := env.VtRoot()
if err != nil {
t.Errorf("err: %v", err)
}
cnfTemplatePaths := []string{
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"),
}
data, err := tablet0.config.makeMycnf(cnfTemplatePaths)
if err != nil {
t.Errorf("err: %v", err)
} else {
t.Logf("data: %v", data)
}
err = ioutil.WriteFile(MYCNF_PATH, []byte(data), 0666)
if err != nil {
t.Errorf("failed creating my.cnf %v", err)
}
_, err = ioutil.ReadFile(MYCNF_PATH)
if err != nil {
t.Errorf("failed reading, err %v", err)
return
}
mycnf, err := ReadMycnf(MYCNF_PATH)
if err != nil {
t.Errorf("failed reading, err %v", err)
} else {
t.Logf("socket file %v", mycnf.SocketFile)
}
}
func (zkd *Zkd) makeCfg() (string, error) {
root, err := env.VtRoot()
if err != nil {
return "", err
}
cnfTemplatePaths := []string{path.Join(root, "config/zkcfg/zoo.cfg")}
return MakeZooCfg(cnfTemplatePaths, zkd.config, "# generated by vt")
}
// Init will create the default directory structure for the mysqld process,
// generate / configure a my.cnf file, unpack a skeleton database,
// and create some management tables.
func (mysqld *Mysqld) Init(mysqlWaitTime time.Duration, bootstrapArchive string, skipSchema bool) error {
log.Infof("mysqlctl.Init")
err := mysqld.createDirs()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
root, err := vtenv.VtRoot()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
// Set up config files.
if err = mysqld.initConfig(root); err != nil {
log.Errorf("failed creating %v: %v", mysqld.config.path, err)
return err
}
// Unpack bootstrap DB files.
dbTbzPath := path.Join(root, "data/bootstrap/"+bootstrapArchive)
log.Infof("decompress bootstrap db %v", dbTbzPath)
args := []string{"-xj", "-C", mysqld.TabletDir, "-f", dbTbzPath}
if _, err = execCmd("tar", args, []string{}, ""); err != nil {
log.Errorf("failed unpacking %v: %v", dbTbzPath, err)
return err
}
// Start mysqld.
if err = mysqld.Start(mysqlWaitTime); err != nil {
log.Errorf("failed starting, check %v", mysqld.config.ErrorLogPath)
return err
}
// Load initial schema.
if skipSchema {
return nil
}
schemaPath := path.Join(root, "data/bootstrap/_vt_schema.sql")
schema, err := ioutil.ReadFile(schemaPath)
if err != nil {
return err
}
sqlCmds := make([]string, 0, 10)
log.Infof("initial schema: %v", string(schema))
for _, cmd := range strings.Split(string(schema), ";") {
cmd = strings.TrimSpace(cmd)
if cmd == "" {
continue
}
sqlCmds = append(sqlCmds, cmd)
}
return mysqld.ExecuteSuperQueryList(sqlCmds)
}
func (agent *ActionAgent) resolvePaths() error {
vtroot, err := env.VtRoot()
if err != nil {
return err
}
path := path.Join(vtroot, "bin/vtaction")
if _, err := os.Stat(path); err != nil {
return fmt.Errorf("vtaction binary %s not found: %v", path, err)
}
agent.vtActionBinFile = path
return nil
}
// Init will create the default directory structure for the mysqld process,
// generate / configure a my.cnf file, install a skeleton database,
// and apply the provided initial SQL file.
func (mysqld *Mysqld) Init(ctx context.Context, initDBSQLFile string) error {
log.Infof("mysqlctl.Init")
err := mysqld.createDirs()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
root, err := vtenv.VtRoot()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
// Set up config files.
if err = mysqld.initConfig(root); err != nil {
log.Errorf("failed creating %v: %v", mysqld.config.path, err)
return err
}
// Install data dir.
mysqlRoot, err := vtenv.VtMysqlRoot()
if err != nil {
log.Errorf("%v", err)
return err
}
log.Infof("Installing data dir with mysql_install_db")
args := []string{
"--defaults-file=" + mysqld.config.path,
"--basedir=" + mysqlRoot,
}
if _, err = execCmd(path.Join(mysqlRoot, "bin/mysql_install_db"), args, nil, mysqlRoot, nil); err != nil {
log.Errorf("mysql_install_db failed: %v", err)
return err
}
// Start mysqld.
if err = mysqld.Start(ctx); err != nil {
log.Errorf("failed starting, check %v", mysqld.config.ErrorLogPath)
return err
}
// Run initial SQL file.
sqlFile, err := os.Open(initDBSQLFile)
if err != nil {
return fmt.Errorf("can't open init_db_sql_file (%v): %v", initDBSQLFile, err)
}
defer sqlFile.Close()
if err := mysqld.executeMysqlScript("root", sqlFile); err != nil {
return fmt.Errorf("can't run init_db_sql_file (%v): %v", initDBSQLFile, err)
}
return nil
}
/*
ZOO_LOG_DIR=""
ZOO_CFG="/.../zoo.cfg"
ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
java -DZOO_LOG_DIR=${ZOO_LOG_DIR} -cp $CLASSPATH $ZOOMAIN $YT_ZK_CFG
*/
func (zkd *Zkd) Start() error {
log.Infof("zkctl.Start")
// NOTE(msolomon) use a script here so we can detach and continue to run
// if the wrangler process dies. this pretty much the same as mysqld_safe.
args := []string{
zkd.config.LogDir(),
zkd.config.ConfigFile(),
zkd.config.PidFile(),
}
root, err := env.VtRoot()
if err != nil {
return err
}
dir := path.Join(root, "bin")
cmd := exec.Command(path.Join(dir, "zksrv.sh"), args...)
cmd.Env = os.Environ()
cmd.Dir = dir
if err = cmd.Start(); err != nil {
return err
}
// give it some time to succeed - usually by the time the socket emerges
// we are in good shape
for i := 0; i < StartWaitTime; i++ {
zkAddr := fmt.Sprintf(":%v", zkd.config.ClientPort)
conn, connErr := net.Dial("tcp", zkAddr)
if connErr != nil {
err = connErr
time.Sleep(time.Second)
continue
} else {
err = nil
conn.Write([]byte("ruok"))
reply := make([]byte, 4)
conn.Read(reply)
if string(reply) != "imok" {
err = fmt.Errorf("local zk unhealthy: %v %v", zkAddr, reply)
}
conn.Close()
break
}
}
zkd.done = make(chan struct{})
go func(done chan<- struct{}) {
// wait so we don't get a bunch of defunct processes
cmd.Wait()
close(done)
}(zkd.done)
return err
}
func TestMycnf(t *testing.T) {
os.Setenv("MYSQL_FLAVOR", "MariaDB")
dbaConfig := dbconfigs.DefaultDBConfigs.Dba
appConfig := dbconfigs.DefaultDBConfigs.App.ConnParams
replConfig := dbconfigs.DefaultDBConfigs.Repl
cnf := NewMycnf(11111, 6802)
// Assigning ServerID to be different from tablet UID to make sure that there are no
// assumptions in the code that those IDs are the same.
cnf.ServerID = 22222
tablet0 := NewMysqld("Dba", "App", cnf, &dbaConfig, &appConfig, &replConfig)
defer tablet0.Close()
root, err := env.VtRoot()
if err != nil {
t.Errorf("err: %v", err)
}
cnfTemplatePaths := []string{
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"),
}
data, err := tablet0.config.makeMycnf(cnfTemplatePaths)
if err != nil {
t.Errorf("err: %v", err)
} else {
t.Logf("data: %v", data)
}
err = ioutil.WriteFile(MycnfPath, []byte(data), 0666)
if err != nil {
t.Errorf("failed creating my.cnf %v", err)
}
_, err = ioutil.ReadFile(MycnfPath)
if err != nil {
t.Errorf("failed reading, err %v", err)
return
}
mycnf, err := ReadMycnf(MycnfPath)
if err != nil {
t.Errorf("failed reading, err %v", err)
} else {
t.Logf("socket file %v", mycnf.SocketFile)
}
// Tablet UID should be 11111, which determines tablet/data dir.
if got, want := mycnf.DataDir, "/vt_0000011111/"; !strings.Contains(got, want) {
t.Errorf("mycnf.DataDir = %v, want *%v*", got, want)
}
// MySQL server-id should be 22222, different from Tablet UID.
if got, want := mycnf.ServerID, uint32(22222); got != want {
t.Errorf("mycnf.ServerID = %v, want %v", got, want)
}
}
func (agent *ActionAgent) resolvePaths() error {
var p string
if *vtactionBinaryPath != "" {
p = *vtactionBinaryPath
} else {
vtroot, err := env.VtRoot()
if err != nil {
return err
}
p = path.Join(vtroot, "bin/vtaction")
}
if _, err := os.Stat(p); err != nil {
return fmt.Errorf("vtaction binary %s not found: %v", p, err)
}
agent.vtActionBinFile = p
return nil
}
// Init will create the default directory structure for the mysqld process,
// generate / configure a my.cnf file, install a skeleton database,
// and apply the provided initial SQL file.
func (mysqld *Mysqld) Init(ctx context.Context, initDBSQLFile string) error {
log.Infof("mysqlctl.Init")
err := mysqld.createDirs()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
root, err := vtenv.VtRoot()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
// Set up config files.
if err = mysqld.initConfig(root); err != nil {
log.Errorf("failed creating %v: %v", mysqld.config.path, err)
return err
}
// Install data dir.
if err = mysqld.installDataDir(); err != nil {
return err
}
// Start mysqld.
if err = mysqld.Start(ctx); err != nil {
log.Errorf("failed starting mysqld (check %v for more info): %v", mysqld.config.ErrorLogPath, err)
return err
}
// Run initial SQL file.
sqlFile, err := os.Open(initDBSQLFile)
if err != nil {
return fmt.Errorf("can't open init_db_sql_file (%v): %v", initDBSQLFile, err)
}
defer sqlFile.Close()
if err := mysqld.executeMysqlScript(&sqldb.ConnParams{Uname: "root", Pass: ""}, sqlFile); err != nil {
return fmt.Errorf("can't run init_db_sql_file (%v): %v", initDBSQLFile, err)
}
return nil
}
func TestMycnf(t *testing.T) {
os.Setenv("MYSQL_FLAVOR", "MariaDB")
dbaConfig := dbconfigs.DefaultDBConfigs.Dba
appConfig := dbconfigs.DefaultDBConfigs.App.ConnParams
replConfig := dbconfigs.DefaultDBConfigs.Repl
tablet0 := NewMysqld("Dba", "App", NewMycnf(0, 6802), &dbaConfig, &appConfig, &replConfig)
defer tablet0.Close()
root, err := env.VtRoot()
if err != nil {
t.Errorf("err: %v", err)
}
cnfTemplatePaths := []string{
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"),
}
data, err := tablet0.config.makeMycnf(cnfTemplatePaths)
if err != nil {
t.Errorf("err: %v", err)
} else {
t.Logf("data: %v", data)
}
err = ioutil.WriteFile(MycnfPath, []byte(data), 0666)
if err != nil {
t.Errorf("failed creating my.cnf %v", err)
}
_, err = ioutil.ReadFile(MycnfPath)
if err != nil {
t.Errorf("failed reading, err %v", err)
return
}
mycnf, err := ReadMycnf(MycnfPath)
if err != nil {
t.Errorf("failed reading, err %v", err)
} else {
t.Logf("socket file %v", mycnf.SocketFile)
}
}
// ReinitConfig updates the config file as if Mysqld is initializing. At the
// moment it only randomizes ServerID because it's not safe to restore a replica
// from a backup and then give it the same ServerID as before, MySQL can then
// skip transactions in the replication stream with the same server_id.
func (mysqld *Mysqld) ReinitConfig(ctx context.Context) error {
log.Infof("Mysqld.ReinitConfig")
// Execute as remote action on mysqlctld if requested.
if *socketFile != "" {
log.Infof("executing Mysqld.ReinitConfig() remotely via mysqlctld server: %v", *socketFile)
client, err := mysqlctlclient.New("unix", *socketFile)
if err != nil {
return fmt.Errorf("can't dial mysqlctld: %v", err)
}
defer client.Close()
return client.ReinitConfig(ctx)
}
if err := mysqld.config.RandomizeMysqlServerID(); err != nil {
return err
}
root, err := vtenv.VtRoot()
if err != nil {
return err
}
return mysqld.initConfig(root)
}
func TestMycnf(t *testing.T) {
var vtRepl VtReplParams
vtRepl.StartKey = ""
vtRepl.EndKey = ""
tablet0 := NewMysqld(NewMycnf(0, 6802, vtRepl), DefaultDbaParams, DefaultReplParams)
root, err := env.VtRoot()
if err != nil {
t.Errorf("err: %v", err)
}
cnfTemplatePaths := []string{
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"),
path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"),
}
data, err := MakeMycnf(tablet0.config, cnfTemplatePaths)
if err != nil {
t.Errorf("err: %v", err)
} else {
t.Logf("data: %v", data)
}
err = ioutil.WriteFile(MYCNF_PATH, []byte(data), 0666)
if err != nil {
t.Errorf("failed creating my.cnf %v", err)
}
_, err = ioutil.ReadFile(MYCNF_PATH)
if err != nil {
t.Errorf("failed reading, err %v", err)
return
}
mycnf, err := ReadMycnf(MYCNF_PATH)
if err != nil {
t.Errorf("failed reading, err %v", err)
} else {
t.Logf("socket file %v", mycnf.SocketFile)
}
}
// Init will create the default directory structure for the mysqld process,
// generate / configure a my.cnf file, unpack a skeleton database,
// and create some management tables.
func (mysqld *Mysqld) Init(ctx context.Context, bootstrapArchive string) error {
log.Infof("mysqlctl.Init")
err := mysqld.createDirs()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
root, err := vtenv.VtRoot()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
// Set up config files.
if err = mysqld.initConfig(root); err != nil {
log.Errorf("failed creating %v: %v", mysqld.config.path, err)
return err
}
// Unpack bootstrap DB files.
dbTbzPath := path.Join(root, "data/bootstrap/"+bootstrapArchive)
log.Infof("decompress bootstrap db %v", dbTbzPath)
args := []string{"-xj", "-C", mysqld.TabletDir, "-f", dbTbzPath}
if _, err = execCmd("tar", args, []string{}, ""); err != nil {
log.Errorf("failed unpacking %v: %v", dbTbzPath, err)
return err
}
// Start mysqld.
if err = mysqld.Start(ctx); err != nil {
log.Errorf("failed starting, check %v", mysqld.config.ErrorLogPath)
return err
}
return nil
}
// findVtMysqlbinlogDir finds the directory that contains vt_mysqlbinlog:
// could be with the mysql distribution, or with the vt distribution
func findVtMysqlbinlogDir() (string, error) {
// first look in VtRoot
dir, err := vtenv.VtRoot()
if err == nil {
if _, err = os.Stat(path.Join(dir, "bin/vt_mysqlbinlog")); err == nil {
return dir, nil
}
}
// then look in VtMysqlRoot
dir, err = vtenv.VtMysqlRoot()
if err == nil {
if _, err = os.Stat(path.Join(dir, "bin/vt_mysqlbinlog")); err == nil {
return dir, nil
}
}
// then try current directory + bin/vt_mysqlbinlog
if _, err = os.Stat("bin/vt_mysqlbinlog"); err == nil {
return "", nil
}
return "", fmt.Errorf("Cannot find vt_mysqlbinlog binary")
}
func Init(mt *Mysqld, mysqlWaitTime time.Duration) error {
log.Infof("mysqlctl.Init")
err := mt.createDirs()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
root, err := vtenv.VtRoot()
if err != nil {
log.Errorf("%s", err.Error())
return err
}
hr := hook.NewSimpleHook("make_mycnf").Execute()
configData := ""
if hr.ExitStatus == hook.HOOK_DOES_NOT_EXIST {
log.Infof("make_mycnf hook doesn't exist")
cnfTemplatePaths := []string{
path.Join(root, "config/mycnf/default.cnf"),
path.Join(root, "config/mycnf/master.cnf"),
path.Join(root, "config/mycnf/replica.cnf"),
}
if extraCnf := os.Getenv("EXTRA_MY_CNF"); extraCnf != "" {
parts := strings.Split(extraCnf, ":")
cnfTemplatePaths = append(cnfTemplatePaths, parts...)
}
configData, err = MakeMycnf(mt.config, cnfTemplatePaths)
} else if hr.ExitStatus == hook.HOOK_SUCCESS {
configData, err = fillMycnfTemplate(mt.config, hr.Stdout)
} else {
err = fmt.Errorf("make_mycnf hook failed(%v): %v", hr.ExitStatus, hr.Stderr)
}
if err == nil {
err = ioutil.WriteFile(mt.config.path, []byte(configData), 0664)
}
if err != nil {
log.Errorf("failed creating %v: %v", mt.config.path, err)
return err
}
dbTbzPath := path.Join(root, "data/bootstrap/mysql-db-dir.tbz")
log.Infof("decompress bootstrap db %v", dbTbzPath)
args := []string{"-xj", "-C", mt.config.DataDir, "-f", dbTbzPath}
_, tarErr := execCmd("tar", args, []string{}, "")
if tarErr != nil {
log.Errorf("failed unpacking %v: %v", dbTbzPath, tarErr)
return tarErr
}
if err = Start(mt, mysqlWaitTime); err != nil {
log.Errorf("failed starting, check %v", mt.config.ErrorLogPath)
return err
}
schemaPath := path.Join(root, "data/bootstrap/_vt_schema.sql")
schema, err := ioutil.ReadFile(schemaPath)
if err != nil {
return err
}
sqlCmds := make([]string, 0, 10)
log.Infof("initial schema: %v", string(schema))
for _, cmd := range strings.Split(string(schema), ";") {
cmd = strings.TrimSpace(cmd)
if cmd == "" {
continue
}
sqlCmds = append(sqlCmds, cmd)
}
return mt.executeSuperQueryList(sqlCmds)
}
func (hook *Hook) Execute() (result *HookResult) {
result = &HookResult{}
// also check for bad string here on the server side, to be sure
if strings.Contains(hook.Name, "/") {
result.ExitStatus = HOOK_INVALID_NAME
result.Stderr = "Hooks cannot contains '/'\n"
return result
}
// find our root
root, err := vtenv.VtRoot()
if err != nil {
result.ExitStatus = HOOK_VTROOT_ERROR
result.Stdout = "Cannot get VTROOT: " + err.Error() + "\n"
return result
}
// see if the hook exists
vthook := path.Join(root, "vthook", hook.Name)
_, err = os.Stat(vthook)
if err != nil {
if os.IsNotExist(err) {
result.ExitStatus = HOOK_DOES_NOT_EXIST
result.Stdout = "Skipping missing hook: " + vthook + "\n"
return result
}
result.ExitStatus = HOOK_STAT_FAILED
result.Stderr = "Cannot stat hook: " + vthook + ": " + err.Error() + "\n"
return result
}
// build the args, run it
args := make([]string, 0, 10)
for key, value := range hook.Parameters {
if value != "" {
args = append(args, "--"+key+"="+value)
} else {
args = append(args, "--"+key)
}
}
log.Infof("hook: executing hook: %v %v", vthook, strings.Join(args, " "))
cmd := exec.Command(vthook, args...)
if len(hook.ExtraEnv) > 0 {
cmd.Env = os.Environ()
for key, value := range hook.ExtraEnv {
cmd.Env = append(cmd.Env, key+"="+value)
}
}
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err = cmd.Run()
result.Stdout = stdout.String()
result.Stderr = stderr.String()
if err == nil {
result.ExitStatus = HOOK_SUCCESS
} else {
if cmd.ProcessState != nil && cmd.ProcessState.Sys() != nil {
result.ExitStatus = cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
} else {
result.ExitStatus = HOOK_CANNOT_GET_EXIT_STATUS
}
result.Stderr += "ERROR: " + err.Error() + "\n"
}
log.Infof("hook: result is %v", result.String())
return result
}