func TestMycnf(t *testing.T) { dbaConfig := dbconfigs.DefaultDBConfigs.Dba replConfig := dbconfigs.DefaultDBConfigs.Repl tablet0 := NewMysqld(NewMycnf(0, 6802), &dbaConfig, &replConfig) root, err := env.VtRoot() if err != nil { t.Errorf("err: %v", err) } cnfTemplatePaths := []string{ path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"), } data, err := tablet0.config.makeMycnf(cnfTemplatePaths) if err != nil { t.Errorf("err: %v", err) } else { t.Logf("data: %v", data) } err = ioutil.WriteFile(MYCNF_PATH, []byte(data), 0666) if err != nil { t.Errorf("failed creating my.cnf %v", err) } _, err = ioutil.ReadFile(MYCNF_PATH) if err != nil { t.Errorf("failed reading, err %v", err) return } mycnf, err := ReadMycnf(MYCNF_PATH) if err != nil { t.Errorf("failed reading, err %v", err) } else { t.Logf("socket file %v", mycnf.SocketFile) } }
func (zkd *Zkd) makeCfg() (string, error) { root, err := env.VtRoot() if err != nil { return "", err } cnfTemplatePaths := []string{path.Join(root, "config/zkcfg/zoo.cfg")} return MakeZooCfg(cnfTemplatePaths, zkd.config, "# generated by vt") }
// Init will create the default directory structure for the mysqld process, // generate / configure a my.cnf file, unpack a skeleton database, // and create some management tables. func (mysqld *Mysqld) Init(mysqlWaitTime time.Duration, bootstrapArchive string, skipSchema bool) error { log.Infof("mysqlctl.Init") err := mysqld.createDirs() if err != nil { log.Errorf("%s", err.Error()) return err } root, err := vtenv.VtRoot() if err != nil { log.Errorf("%s", err.Error()) return err } // Set up config files. if err = mysqld.initConfig(root); err != nil { log.Errorf("failed creating %v: %v", mysqld.config.path, err) return err } // Unpack bootstrap DB files. dbTbzPath := path.Join(root, "data/bootstrap/"+bootstrapArchive) log.Infof("decompress bootstrap db %v", dbTbzPath) args := []string{"-xj", "-C", mysqld.TabletDir, "-f", dbTbzPath} if _, err = execCmd("tar", args, []string{}, ""); err != nil { log.Errorf("failed unpacking %v: %v", dbTbzPath, err) return err } // Start mysqld. if err = mysqld.Start(mysqlWaitTime); err != nil { log.Errorf("failed starting, check %v", mysqld.config.ErrorLogPath) return err } // Load initial schema. if skipSchema { return nil } schemaPath := path.Join(root, "data/bootstrap/_vt_schema.sql") schema, err := ioutil.ReadFile(schemaPath) if err != nil { return err } sqlCmds := make([]string, 0, 10) log.Infof("initial schema: %v", string(schema)) for _, cmd := range strings.Split(string(schema), ";") { cmd = strings.TrimSpace(cmd) if cmd == "" { continue } sqlCmds = append(sqlCmds, cmd) } return mysqld.ExecuteSuperQueryList(sqlCmds) }
func (agent *ActionAgent) resolvePaths() error { vtroot, err := env.VtRoot() if err != nil { return err } path := path.Join(vtroot, "bin/vtaction") if _, err := os.Stat(path); err != nil { return fmt.Errorf("vtaction binary %s not found: %v", path, err) } agent.vtActionBinFile = path return nil }
// Init will create the default directory structure for the mysqld process, // generate / configure a my.cnf file, install a skeleton database, // and apply the provided initial SQL file. func (mysqld *Mysqld) Init(ctx context.Context, initDBSQLFile string) error { log.Infof("mysqlctl.Init") err := mysqld.createDirs() if err != nil { log.Errorf("%s", err.Error()) return err } root, err := vtenv.VtRoot() if err != nil { log.Errorf("%s", err.Error()) return err } // Set up config files. if err = mysqld.initConfig(root); err != nil { log.Errorf("failed creating %v: %v", mysqld.config.path, err) return err } // Install data dir. mysqlRoot, err := vtenv.VtMysqlRoot() if err != nil { log.Errorf("%v", err) return err } log.Infof("Installing data dir with mysql_install_db") args := []string{ "--defaults-file=" + mysqld.config.path, "--basedir=" + mysqlRoot, } if _, err = execCmd(path.Join(mysqlRoot, "bin/mysql_install_db"), args, nil, mysqlRoot, nil); err != nil { log.Errorf("mysql_install_db failed: %v", err) return err } // Start mysqld. if err = mysqld.Start(ctx); err != nil { log.Errorf("failed starting, check %v", mysqld.config.ErrorLogPath) return err } // Run initial SQL file. sqlFile, err := os.Open(initDBSQLFile) if err != nil { return fmt.Errorf("can't open init_db_sql_file (%v): %v", initDBSQLFile, err) } defer sqlFile.Close() if err := mysqld.executeMysqlScript("root", sqlFile); err != nil { return fmt.Errorf("can't run init_db_sql_file (%v): %v", initDBSQLFile, err) } return nil }
/* ZOO_LOG_DIR="" ZOO_CFG="/.../zoo.cfg" ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" java -DZOO_LOG_DIR=${ZOO_LOG_DIR} -cp $CLASSPATH $ZOOMAIN $YT_ZK_CFG */ func (zkd *Zkd) Start() error { log.Infof("zkctl.Start") // NOTE(msolomon) use a script here so we can detach and continue to run // if the wrangler process dies. this pretty much the same as mysqld_safe. args := []string{ zkd.config.LogDir(), zkd.config.ConfigFile(), zkd.config.PidFile(), } root, err := env.VtRoot() if err != nil { return err } dir := path.Join(root, "bin") cmd := exec.Command(path.Join(dir, "zksrv.sh"), args...) cmd.Env = os.Environ() cmd.Dir = dir if err = cmd.Start(); err != nil { return err } // give it some time to succeed - usually by the time the socket emerges // we are in good shape for i := 0; i < StartWaitTime; i++ { zkAddr := fmt.Sprintf(":%v", zkd.config.ClientPort) conn, connErr := net.Dial("tcp", zkAddr) if connErr != nil { err = connErr time.Sleep(time.Second) continue } else { err = nil conn.Write([]byte("ruok")) reply := make([]byte, 4) conn.Read(reply) if string(reply) != "imok" { err = fmt.Errorf("local zk unhealthy: %v %v", zkAddr, reply) } conn.Close() break } } zkd.done = make(chan struct{}) go func(done chan<- struct{}) { // wait so we don't get a bunch of defunct processes cmd.Wait() close(done) }(zkd.done) return err }
func TestMycnf(t *testing.T) { os.Setenv("MYSQL_FLAVOR", "MariaDB") dbaConfig := dbconfigs.DefaultDBConfigs.Dba appConfig := dbconfigs.DefaultDBConfigs.App.ConnParams replConfig := dbconfigs.DefaultDBConfigs.Repl cnf := NewMycnf(11111, 6802) // Assigning ServerID to be different from tablet UID to make sure that there are no // assumptions in the code that those IDs are the same. cnf.ServerID = 22222 tablet0 := NewMysqld("Dba", "App", cnf, &dbaConfig, &appConfig, &replConfig) defer tablet0.Close() root, err := env.VtRoot() if err != nil { t.Errorf("err: %v", err) } cnfTemplatePaths := []string{ path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"), } data, err := tablet0.config.makeMycnf(cnfTemplatePaths) if err != nil { t.Errorf("err: %v", err) } else { t.Logf("data: %v", data) } err = ioutil.WriteFile(MycnfPath, []byte(data), 0666) if err != nil { t.Errorf("failed creating my.cnf %v", err) } _, err = ioutil.ReadFile(MycnfPath) if err != nil { t.Errorf("failed reading, err %v", err) return } mycnf, err := ReadMycnf(MycnfPath) if err != nil { t.Errorf("failed reading, err %v", err) } else { t.Logf("socket file %v", mycnf.SocketFile) } // Tablet UID should be 11111, which determines tablet/data dir. if got, want := mycnf.DataDir, "/vt_0000011111/"; !strings.Contains(got, want) { t.Errorf("mycnf.DataDir = %v, want *%v*", got, want) } // MySQL server-id should be 22222, different from Tablet UID. if got, want := mycnf.ServerID, uint32(22222); got != want { t.Errorf("mycnf.ServerID = %v, want %v", got, want) } }
func (agent *ActionAgent) resolvePaths() error { var p string if *vtactionBinaryPath != "" { p = *vtactionBinaryPath } else { vtroot, err := env.VtRoot() if err != nil { return err } p = path.Join(vtroot, "bin/vtaction") } if _, err := os.Stat(p); err != nil { return fmt.Errorf("vtaction binary %s not found: %v", p, err) } agent.vtActionBinFile = p return nil }
// Init will create the default directory structure for the mysqld process, // generate / configure a my.cnf file, install a skeleton database, // and apply the provided initial SQL file. func (mysqld *Mysqld) Init(ctx context.Context, initDBSQLFile string) error { log.Infof("mysqlctl.Init") err := mysqld.createDirs() if err != nil { log.Errorf("%s", err.Error()) return err } root, err := vtenv.VtRoot() if err != nil { log.Errorf("%s", err.Error()) return err } // Set up config files. if err = mysqld.initConfig(root); err != nil { log.Errorf("failed creating %v: %v", mysqld.config.path, err) return err } // Install data dir. if err = mysqld.installDataDir(); err != nil { return err } // Start mysqld. if err = mysqld.Start(ctx); err != nil { log.Errorf("failed starting mysqld (check %v for more info): %v", mysqld.config.ErrorLogPath, err) return err } // Run initial SQL file. sqlFile, err := os.Open(initDBSQLFile) if err != nil { return fmt.Errorf("can't open init_db_sql_file (%v): %v", initDBSQLFile, err) } defer sqlFile.Close() if err := mysqld.executeMysqlScript(&sqldb.ConnParams{Uname: "root", Pass: ""}, sqlFile); err != nil { return fmt.Errorf("can't run init_db_sql_file (%v): %v", initDBSQLFile, err) } return nil }
func TestMycnf(t *testing.T) { os.Setenv("MYSQL_FLAVOR", "MariaDB") dbaConfig := dbconfigs.DefaultDBConfigs.Dba appConfig := dbconfigs.DefaultDBConfigs.App.ConnParams replConfig := dbconfigs.DefaultDBConfigs.Repl tablet0 := NewMysqld("Dba", "App", NewMycnf(0, 6802), &dbaConfig, &appConfig, &replConfig) defer tablet0.Close() root, err := env.VtRoot() if err != nil { t.Errorf("err: %v", err) } cnfTemplatePaths := []string{ path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"), } data, err := tablet0.config.makeMycnf(cnfTemplatePaths) if err != nil { t.Errorf("err: %v", err) } else { t.Logf("data: %v", data) } err = ioutil.WriteFile(MycnfPath, []byte(data), 0666) if err != nil { t.Errorf("failed creating my.cnf %v", err) } _, err = ioutil.ReadFile(MycnfPath) if err != nil { t.Errorf("failed reading, err %v", err) return } mycnf, err := ReadMycnf(MycnfPath) if err != nil { t.Errorf("failed reading, err %v", err) } else { t.Logf("socket file %v", mycnf.SocketFile) } }
// ReinitConfig updates the config file as if Mysqld is initializing. At the // moment it only randomizes ServerID because it's not safe to restore a replica // from a backup and then give it the same ServerID as before, MySQL can then // skip transactions in the replication stream with the same server_id. func (mysqld *Mysqld) ReinitConfig(ctx context.Context) error { log.Infof("Mysqld.ReinitConfig") // Execute as remote action on mysqlctld if requested. if *socketFile != "" { log.Infof("executing Mysqld.ReinitConfig() remotely via mysqlctld server: %v", *socketFile) client, err := mysqlctlclient.New("unix", *socketFile) if err != nil { return fmt.Errorf("can't dial mysqlctld: %v", err) } defer client.Close() return client.ReinitConfig(ctx) } if err := mysqld.config.RandomizeMysqlServerID(); err != nil { return err } root, err := vtenv.VtRoot() if err != nil { return err } return mysqld.initConfig(root) }
func TestMycnf(t *testing.T) { var vtRepl VtReplParams vtRepl.StartKey = "" vtRepl.EndKey = "" tablet0 := NewMysqld(NewMycnf(0, 6802, vtRepl), DefaultDbaParams, DefaultReplParams) root, err := env.VtRoot() if err != nil { t.Errorf("err: %v", err) } cnfTemplatePaths := []string{ path.Join(root, "src/github.com/youtube/vitess/config/mycnf/default.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/replica.cnf"), path.Join(root, "src/github.com/youtube/vitess/config/mycnf/master.cnf"), } data, err := MakeMycnf(tablet0.config, cnfTemplatePaths) if err != nil { t.Errorf("err: %v", err) } else { t.Logf("data: %v", data) } err = ioutil.WriteFile(MYCNF_PATH, []byte(data), 0666) if err != nil { t.Errorf("failed creating my.cnf %v", err) } _, err = ioutil.ReadFile(MYCNF_PATH) if err != nil { t.Errorf("failed reading, err %v", err) return } mycnf, err := ReadMycnf(MYCNF_PATH) if err != nil { t.Errorf("failed reading, err %v", err) } else { t.Logf("socket file %v", mycnf.SocketFile) } }
// Init will create the default directory structure for the mysqld process, // generate / configure a my.cnf file, unpack a skeleton database, // and create some management tables. func (mysqld *Mysqld) Init(ctx context.Context, bootstrapArchive string) error { log.Infof("mysqlctl.Init") err := mysqld.createDirs() if err != nil { log.Errorf("%s", err.Error()) return err } root, err := vtenv.VtRoot() if err != nil { log.Errorf("%s", err.Error()) return err } // Set up config files. if err = mysqld.initConfig(root); err != nil { log.Errorf("failed creating %v: %v", mysqld.config.path, err) return err } // Unpack bootstrap DB files. dbTbzPath := path.Join(root, "data/bootstrap/"+bootstrapArchive) log.Infof("decompress bootstrap db %v", dbTbzPath) args := []string{"-xj", "-C", mysqld.TabletDir, "-f", dbTbzPath} if _, err = execCmd("tar", args, []string{}, ""); err != nil { log.Errorf("failed unpacking %v: %v", dbTbzPath, err) return err } // Start mysqld. if err = mysqld.Start(ctx); err != nil { log.Errorf("failed starting, check %v", mysqld.config.ErrorLogPath) return err } return nil }
// findVtMysqlbinlogDir finds the directory that contains vt_mysqlbinlog: // could be with the mysql distribution, or with the vt distribution func findVtMysqlbinlogDir() (string, error) { // first look in VtRoot dir, err := vtenv.VtRoot() if err == nil { if _, err = os.Stat(path.Join(dir, "bin/vt_mysqlbinlog")); err == nil { return dir, nil } } // then look in VtMysqlRoot dir, err = vtenv.VtMysqlRoot() if err == nil { if _, err = os.Stat(path.Join(dir, "bin/vt_mysqlbinlog")); err == nil { return dir, nil } } // then try current directory + bin/vt_mysqlbinlog if _, err = os.Stat("bin/vt_mysqlbinlog"); err == nil { return "", nil } return "", fmt.Errorf("Cannot find vt_mysqlbinlog binary") }
func Init(mt *Mysqld, mysqlWaitTime time.Duration) error { log.Infof("mysqlctl.Init") err := mt.createDirs() if err != nil { log.Errorf("%s", err.Error()) return err } root, err := vtenv.VtRoot() if err != nil { log.Errorf("%s", err.Error()) return err } hr := hook.NewSimpleHook("make_mycnf").Execute() configData := "" if hr.ExitStatus == hook.HOOK_DOES_NOT_EXIST { log.Infof("make_mycnf hook doesn't exist") cnfTemplatePaths := []string{ path.Join(root, "config/mycnf/default.cnf"), path.Join(root, "config/mycnf/master.cnf"), path.Join(root, "config/mycnf/replica.cnf"), } if extraCnf := os.Getenv("EXTRA_MY_CNF"); extraCnf != "" { parts := strings.Split(extraCnf, ":") cnfTemplatePaths = append(cnfTemplatePaths, parts...) } configData, err = MakeMycnf(mt.config, cnfTemplatePaths) } else if hr.ExitStatus == hook.HOOK_SUCCESS { configData, err = fillMycnfTemplate(mt.config, hr.Stdout) } else { err = fmt.Errorf("make_mycnf hook failed(%v): %v", hr.ExitStatus, hr.Stderr) } if err == nil { err = ioutil.WriteFile(mt.config.path, []byte(configData), 0664) } if err != nil { log.Errorf("failed creating %v: %v", mt.config.path, err) return err } dbTbzPath := path.Join(root, "data/bootstrap/mysql-db-dir.tbz") log.Infof("decompress bootstrap db %v", dbTbzPath) args := []string{"-xj", "-C", mt.config.DataDir, "-f", dbTbzPath} _, tarErr := execCmd("tar", args, []string{}, "") if tarErr != nil { log.Errorf("failed unpacking %v: %v", dbTbzPath, tarErr) return tarErr } if err = Start(mt, mysqlWaitTime); err != nil { log.Errorf("failed starting, check %v", mt.config.ErrorLogPath) return err } schemaPath := path.Join(root, "data/bootstrap/_vt_schema.sql") schema, err := ioutil.ReadFile(schemaPath) if err != nil { return err } sqlCmds := make([]string, 0, 10) log.Infof("initial schema: %v", string(schema)) for _, cmd := range strings.Split(string(schema), ";") { cmd = strings.TrimSpace(cmd) if cmd == "" { continue } sqlCmds = append(sqlCmds, cmd) } return mt.executeSuperQueryList(sqlCmds) }
func (hook *Hook) Execute() (result *HookResult) { result = &HookResult{} // also check for bad string here on the server side, to be sure if strings.Contains(hook.Name, "/") { result.ExitStatus = HOOK_INVALID_NAME result.Stderr = "Hooks cannot contains '/'\n" return result } // find our root root, err := vtenv.VtRoot() if err != nil { result.ExitStatus = HOOK_VTROOT_ERROR result.Stdout = "Cannot get VTROOT: " + err.Error() + "\n" return result } // see if the hook exists vthook := path.Join(root, "vthook", hook.Name) _, err = os.Stat(vthook) if err != nil { if os.IsNotExist(err) { result.ExitStatus = HOOK_DOES_NOT_EXIST result.Stdout = "Skipping missing hook: " + vthook + "\n" return result } result.ExitStatus = HOOK_STAT_FAILED result.Stderr = "Cannot stat hook: " + vthook + ": " + err.Error() + "\n" return result } // build the args, run it args := make([]string, 0, 10) for key, value := range hook.Parameters { if value != "" { args = append(args, "--"+key+"="+value) } else { args = append(args, "--"+key) } } log.Infof("hook: executing hook: %v %v", vthook, strings.Join(args, " ")) cmd := exec.Command(vthook, args...) if len(hook.ExtraEnv) > 0 { cmd.Env = os.Environ() for key, value := range hook.ExtraEnv { cmd.Env = append(cmd.Env, key+"="+value) } } var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr err = cmd.Run() result.Stdout = stdout.String() result.Stderr = stderr.String() if err == nil { result.ExitStatus = HOOK_SUCCESS } else { if cmd.ProcessState != nil && cmd.ProcessState.Sys() != nil { result.ExitStatus = cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() } else { result.ExitStatus = HOOK_CANNOT_GET_EXIT_STATUS } result.Stderr += "ERROR: " + err.Error() + "\n" } log.Infof("hook: result is %v", result.String()) return result }