func (s *S) TestRemoveUnits(c *C) {
tmpdir, err := commandmocker.Add("juju", "removed")
c.Assert(err, IsNil)
defer commandmocker.Remove(tmpdir)
app := NewFakeApp("xanadu", "rush", 4)
p := JujuProvisioner{}
units, err := p.RemoveUnits(app, 3)
c.Assert(err, IsNil)
expected := []string{
"remove-unit", "xanadu/0", "xanadu/1", "xanadu/2",
"terminate-machine", "1",
"terminate-machine", "2",
"terminate-machine", "3",
}
ran := make(chan bool, 1)
go func() {
for {
if reflect.DeepEqual(commandmocker.Parameters(tmpdir), expected) {
ran <- true
}
}
}()
select {
case <-ran:
case <-time.After(2e9):
params := commandmocker.Parameters(tmpdir)
c.Fatalf("Did not run terminate-machine commands after 2 seconds. Parameters: %#v", params)
}
c.Assert(units, DeepEquals, []int{0, 1, 2})
}
func (s *S) TestDestroy(c *C) {
tmpdir, err := commandmocker.Add("juju", "$*")
c.Assert(err, IsNil)
defer commandmocker.Remove(tmpdir)
app := NewFakeApp("cribcaged", "python", 3)
p := JujuProvisioner{}
err = p.Destroy(app)
c.Assert(err, IsNil)
c.Assert(commandmocker.Ran(tmpdir), Equals, true)
expected := []string{
"destroy-service", "cribcaged",
"terminate-machine", "1",
"terminate-machine", "2",
"terminate-machine", "3",
}
ran := make(chan bool, 1)
go func() {
for {
if reflect.DeepEqual(commandmocker.Parameters(tmpdir), expected) {
ran <- true
}
}
}()
select {
case <-ran:
case <-time.After(2e9):
c.Errorf("Did not run terminate-machine commands after 2 seconds.")
}
c.Assert(commandmocker.Parameters(tmpdir), DeepEquals, expected)
}
func (s *S) TestRemoveUnit(c *gocheck.C) {
tmpdir, err := commandmocker.Add("juju", "removed")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
app := testing.NewFakeApp("two", "rush", 3)
p := JujuProvisioner{}
conn, collection := p.unitsCollection()
defer conn.Close()
err = collection.Insert(instance{UnitName: "two/2", InstanceId: "i-00000439"})
c.Assert(err, gocheck.IsNil)
err = p.RemoveUnit(app, "two/2")
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
expected := []string{"remove-unit", "two/2", "terminate-machine", "3"}
ran := make(chan bool, 1)
go func() {
for {
if reflect.DeepEqual(commandmocker.Parameters(tmpdir), expected) {
ran <- true
}
time.Sleep(1e3)
}
}()
n, err := collection.Find(bson.M{"_id": "two/2"}).Count()
c.Assert(err, gocheck.IsNil)
c.Assert(n, gocheck.Equals, 0)
select {
case <-ran:
case <-time.After(2e9):
c.Errorf("Did not run terminate-machine command after 2 seconds.")
}
}
func (s *S) TestBootstrapProvisionHealerHeal(c *gocheck.C) {
p := JujuProvisioner{}
m := machine{
AgentState: "not-started",
IpAddress: "localhost",
InstanceId: "i-00000376",
InstanceState: "running",
}
p.saveBootstrapMachine(m)
conn, collection := p.bootstrapCollection()
defer conn.Close()
defer collection.Remove(m)
sshTmpdir, err := commandmocker.Add("ssh", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(sshTmpdir)
sshOutput := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"localhost",
"sudo",
"start",
"juju-provision-agent",
}
h := bootstrapProvisionHealer{}
err = h.Heal()
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(sshTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(sshTmpdir), gocheck.DeepEquals, sshOutput)
}
func (s *S) TestDestroy(c *gocheck.C) {
tmpdir, err := commandmocker.Add("juju", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
app := testing.NewFakeApp("cribcaged", "python", 3)
p := JujuProvisioner{}
conn, collection := p.unitsCollection()
defer conn.Close()
err = collection.Insert(
instance{UnitName: "cribcaged/0"},
instance{UnitName: "cribcaged/1"},
instance{UnitName: "cribcaged/2"},
)
c.Assert(err, gocheck.IsNil)
err = p.Destroy(app)
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
expected := []string{
"destroy-service", "cribcaged",
"terminate-machine", "1",
"terminate-machine", "2",
"terminate-machine", "3",
}
ran := make(chan bool, 1)
go func() {
for {
if reflect.DeepEqual(commandmocker.Parameters(tmpdir), expected) {
ran <- true
}
time.Sleep(1e3)
}
}()
n, err := collection.Find(bson.M{
"_id": bson.M{
"$in": []string{"cribcaged/0", "cribcaged/1", "cribcaged/2"},
},
}).Count()
c.Assert(err, gocheck.IsNil)
c.Assert(n, gocheck.Equals, 0)
select {
case <-ran:
case <-time.After(2e9):
c.Errorf("Did not run terminate-machine commands after 2 seconds.")
}
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, expected)
}
func (s *S) TestProvisionSetup(c *gocheck.C) {
tmpdir, err := commandmocker.Add("scp", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
sshTempDir, err := commandmocker.Add("ssh", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(sshTempDir)
p := LocalProvisioner{}
formulasPath := "/home/ubuntu/formulas"
config.Set("local:formulas-path", formulasPath)
err = p.setup("10.10.10.10", "static")
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
cmds := []string{
"-q",
"-o",
"StrictHostKeyChecking no",
"-r",
formulasPath + "/static/hooks",
"[email protected]:/var/lib/tsuru",
}
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, cmds)
c.Assert(commandmocker.Ran(sshTempDir), gocheck.Equals, true)
cmds = []string{
"-q",
"-o",
"StrictHostKeyChecking no",
"-l",
"ubuntu",
"10.10.10.10",
"sudo mkdir -p /var/lib/tsuru/hooks",
"-q",
"-o",
"StrictHostKeyChecking no",
"-l",
"ubuntu",
"10.10.10.10",
"sudo chown -R ubuntu /var/lib/tsuru/hooks",
}
c.Assert(commandmocker.Parameters(sshTempDir), gocheck.DeepEquals, cmds)
}
func (s *S) TestExecuteWithoutArgs(c *gocheck.C) {
tmpdir, err := commandmocker.Add("ls", "ok")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
var e OsExecutor
var b bytes.Buffer
err = e.Execute("ls", nil, nil, &b, &b)
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(tmpdir), gocheck.IsNil)
c.Assert(b.String(), gocheck.Equals, "ok")
}
func (s *S) TestExecute(c *gocheck.C) {
tmpdir, err := commandmocker.Add("ls", "ok")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
var e OsExecutor
var b bytes.Buffer
err = e.Execute("ls", []string{"-lsa"}, nil, &b, &b)
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
expected := []string{"-lsa"}
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, expected)
c.Assert(b.String(), gocheck.Equals, "ok")
}
func (s *S) TestInstanceMachineHeal(c *gocheck.C) {
jujuTmpdir, err := commandmocker.Add("juju", collectOutputInstanceDown)
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(jujuTmpdir)
sshTmpdir, err := commandmocker.Add("ssh", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(sshTmpdir)
jujuOutput := []string{
"status", // for juju status that gets the output
}
sshOutput := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"10.10.10.163",
"sudo",
"stop",
"juju-machine-agent",
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"10.10.10.163",
"sudo",
"start",
"juju-machine-agent",
}
h := instanceMachineHealer{}
err = h.Heal()
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(jujuTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(jujuTmpdir), gocheck.DeepEquals, jujuOutput)
c.Assert(commandmocker.Ran(sshTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(sshTmpdir), gocheck.DeepEquals, sshOutput)
}
func (s *S) TestRestart(c *gocheck.C) {
tmpdir, err := commandmocker.Add("juju", "restart")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
app := testing.NewFakeApp("cribcaged", "python", 1)
p := JujuProvisioner{}
err = p.Restart(app)
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
expected := []string{
"ssh", "-o", "StrictHostKeyChecking no", "-q", "1", "/var/lib/tsuru/hooks/restart",
}
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, expected)
}
func (s *S) TestZookeeperHealerHeal(c *gocheck.C) {
ln, err := net.Listen("tcp", ":2181")
c.Assert(err, gocheck.IsNil)
defer ln.Close()
go func() {
conn, _ := ln.Accept()
fmt.Fprintln(conn, "notok")
conn.Close()
}()
p := JujuProvisioner{}
m := machine{
AgentState: "not-started",
IpAddress: "localhost",
InstanceId: "i-00000376",
InstanceState: "running",
}
p.saveBootstrapMachine(m)
conn, collection := p.bootstrapCollection()
defer conn.Close()
defer collection.Remove(m)
sshTmpdir, err := commandmocker.Add("ssh", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(sshTmpdir)
sshOutput := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"localhost",
"sudo",
"stop",
"zookeeper",
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"localhost",
"sudo",
"start",
"zookeeper",
}
h := zookeeperHealer{}
err = h.Heal()
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(sshTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(sshTmpdir), gocheck.DeepEquals, sshOutput)
}
func (s *S) TestProvisionerRestart(c *gocheck.C) {
var p LocalProvisioner
tmpdir, err := commandmocker.Add("ssh", "ok")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
app := testing.NewFakeApp("almah", "static", 1)
err = p.Restart(app)
c.Assert(err, gocheck.IsNil)
ip := app.ProvisionUnits()[0].GetIp()
expected := []string{
"-l", "ubuntu", "-q", "-o", "StrictHostKeyChecking no", ip, "/var/lib/tsuru/hooks/restart",
}
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, expected)
}
func (s *S) TestInstanceMachineHealWhenEverythingIsOk(c *gocheck.C) {
fexec := &etesting.FakeExecutor{}
setExecut(fexec)
defer setExecut(nil)
jujuTmpdir, err := commandmocker.Add("juju", collectOutput)
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(jujuTmpdir)
h := instanceMachineHealer{}
err = h.Heal()
c.Assert(err, gocheck.IsNil)
args := []string{
"status", // for juju status that gets the output
}
c.Assert(commandmocker.Ran(jujuTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(jujuTmpdir), gocheck.DeepEquals, args)
}
func (s *S) TestDeploy(c *gocheck.C) {
tmpdir, err := commandmocker.Add("juju", "")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
config.Set("git:unit-repo", "test/dir")
defer func() {
config.Unset("git:unit-repo")
}()
app := testing.NewFakeApp("cribcaged", "python", 1)
w := &bytes.Buffer{}
p := JujuProvisioner{}
err = p.Deploy(app, "f83ac40", w)
c.Assert(err, gocheck.IsNil)
expected := []string{"set", app.GetName(), "app-version=f83ac40"}
c.Assert(commandmocker.Parameters(tmpdir)[:3], gocheck.DeepEquals, expected)
}
func (s *S) TestInstanceMachineHealWhenEverythingIsOk(c *gocheck.C) {
jujuTmpdir, err := commandmocker.Add("juju", collectOutput)
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(jujuTmpdir)
sshTmpdir, err := commandmocker.Add("ssh", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(sshTmpdir)
jujuOutput := []string{
"status", // for juju status that gets the output
}
h := instanceMachineHealer{}
err = h.Heal()
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(jujuTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(jujuTmpdir), gocheck.DeepEquals, jujuOutput)
c.Assert(commandmocker.Ran(sshTmpdir), gocheck.Equals, false)
}
func (s *S) TestProvision(c *gocheck.C) {
config.Set("juju:charms-path", "/etc/juju/charms")
defer config.Unset("juju:charms-path")
config.Set("host", "somehost")
defer config.Unset("host")
tmpdir, err := commandmocker.Add("juju", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
app := testing.NewFakeApp("trace", "python", 0)
p := JujuProvisioner{}
err = p.Provision(app)
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
expectedParams := []string{
"deploy", "--repository", "/etc/juju/charms", "local:python", "trace",
"set", "trace", "app-repo=" + repository.GetReadOnlyUrl("trace"),
}
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, expectedParams)
}
func (s *S) TestProvisionStart(c *gocheck.C) {
tmpdir, err := commandmocker.Add("ssh", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
p := LocalProvisioner{}
err = p.start("10.10.10.10")
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(tmpdir), gocheck.Equals, true)
cmds := []string{
"-q",
"-o",
"StrictHostKeyChecking no",
"-l",
"ubuntu",
"10.10.10.10",
"sudo /var/lib/tsuru/hooks/start",
}
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, cmds)
}
func (s *S) TestInstanceMachineHeal(c *gocheck.C) {
fexec := &etesting.FakeExecutor{}
execut = fexec
defer func() {
execut = nil
}()
jujuTmpdir, err := commandmocker.Add("juju", collectOutputInstanceDown)
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(jujuTmpdir)
h := instanceMachineHealer{}
err = h.Heal()
c.Assert(err, gocheck.IsNil)
args := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"10.10.10.163",
"sudo",
"stop",
"juju-machine-agent",
}
c.Assert(fexec.ExecutedCmd("ssh", args), gocheck.Equals, true)
args = []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"10.10.10.163",
"sudo",
"start",
"juju-machine-agent",
}
c.Assert(fexec.ExecutedCmd("ssh", args), gocheck.Equals, true)
args = []string{
"status", // for juju status that gets the output
}
c.Assert(commandmocker.Ran(jujuTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(jujuTmpdir), gocheck.DeepEquals, args)
}
func (s *S) TestInstaceUnitHeal(c *gocheck.C) {
conn, err := db.Conn()
c.Assert(err, gocheck.IsNil)
a := app.App{
Name: "as_i_rise",
Units: []app.Unit{{Name: "as_i_rise/0", State: "down", Ip: "server-1081.novalocal"}},
}
err = conn.Apps().Insert(&a)
c.Assert(err, gocheck.IsNil)
defer conn.Apps().Remove(bson.M{"name": "as_i_rise"})
sshTmpdir, err := commandmocker.Add("ssh", "$*")
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(sshTmpdir)
sshOutput := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"server-1081.novalocal",
"sudo",
"stop",
"juju-as_i_rise-0",
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"server-1081.novalocal",
"sudo",
"start",
"juju-as_i_rise-0",
}
h := instanceUnitHealer{}
err = h.Heal()
c.Assert(err, gocheck.IsNil)
c.Assert(commandmocker.Ran(sshTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(sshTmpdir), gocheck.DeepEquals, sshOutput)
}
func (s *S) TestAddUnits(c *C) {
tmpdir, err := commandmocker.Add("juju", addUnitsOutput)
c.Assert(err, IsNil)
defer commandmocker.Remove(tmpdir)
app := NewFakeApp("resist", "rush", 0)
p := JujuProvisioner{}
units, err := p.AddUnits(app, 4)
c.Assert(err, IsNil)
c.Assert(units, HasLen, 4)
names := make([]string, len(units))
for i, unit := range units {
names[i] = unit.Name
}
expected := []string{"resist/3", "resist/4", "resist/5", "resist/6"}
c.Assert(names, DeepEquals, expected)
c.Assert(commandmocker.Ran(tmpdir), Equals, true)
expectedParams := []string{
"set", "resist", "app-repo=" + repository.GetReadOnlyUrl("resist"),
"add-unit", "resist", "--num-units", "4",
}
c.Assert(commandmocker.Parameters(tmpdir), DeepEquals, expectedParams)
}
func (s *S) TestAddUnits(c *gocheck.C) {
tmpdir, err := commandmocker.Add("juju", addUnitsOutput)
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(tmpdir)
app := testing.NewFakeApp("resist", "rush", 0)
p := JujuProvisioner{}
units, err := p.AddUnits(app, 4)
c.Assert(err, gocheck.IsNil)
c.Assert(units, gocheck.HasLen, 4)
names := make([]string, len(units))
for i, unit := range units {
names[i] = unit.Name
}
expected := []string{"resist/3", "resist/4", "resist/5", "resist/6"}
c.Assert(names, gocheck.DeepEquals, expected)
args := []string{
"add-unit", "resist", "--num-units", "4",
}
c.Assert(commandmocker.Parameters(tmpdir), gocheck.DeepEquals, args)
_, err = getQueue(queueName).Get(1e6)
c.Assert(err, gocheck.NotNil)
}
func (s *S) TestRemoveUnit(c *C) {
tmpdir, err := commandmocker.Add("juju", "removed")
c.Assert(err, IsNil)
defer commandmocker.Remove(tmpdir)
app := NewFakeApp("two", "rush", 3)
p := JujuProvisioner{}
err = p.RemoveUnit(app, "two/2")
c.Assert(err, IsNil)
c.Assert(commandmocker.Ran(tmpdir), Equals, true)
expected := []string{"remove-unit", "two/2", "terminate-machine", "3"}
ran := make(chan bool, 1)
go func() {
for {
if reflect.DeepEqual(commandmocker.Parameters(tmpdir), expected) {
ran <- true
}
}
}()
select {
case <-ran:
case <-time.After(2e9):
c.Errorf("Did not run terminate-machine command after 2 seconds.")
}
}
func (s *S) TestInstanceAgenstConfigHealerHeal(c *gocheck.C) {
server, err := ec2test.NewServer()
c.Assert(err, gocheck.IsNil)
defer server.Quit()
id := server.NewInstances(1, "small", "ami-123", ec2.InstanceState{Code: 16, Name: "running"}, nil)[0]
p := JujuProvisioner{}
m := machine{
AgentState: "not-started",
IpAddress: "localhost",
InstanceId: id,
InstanceState: "running",
}
p.saveBootstrapMachine(m)
conn, collection := p.bootstrapCollection()
defer conn.Close()
defer collection.Remove(m)
a := app.App{
Name: "as_i_rise",
Units: []app.Unit{{Name: "as_i_rise/0", State: "down", Ip: "server-1081.novalocal"}},
}
err = s.conn.Apps().Insert(&a)
c.Assert(err, gocheck.IsNil)
defer s.conn.Apps().Remove(bson.M{"name": "as_i_rise"})
sshTmpdir, err := commandmocker.Error("ssh", "", 1)
c.Assert(err, gocheck.IsNil)
defer commandmocker.Remove(sshTmpdir)
h := instanceAgentsConfigHealer{}
auth := aws.Auth{AccessKey: "access", SecretKey: "secret"}
region := aws.SAEast
region.EC2Endpoint = server.URL()
h.e = ec2.New(auth, region)
err = h.Heal()
c.Assert(err, gocheck.IsNil)
sshOutput := []string{
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"server-1081.novalocal",
"grep",
"",
"/etc/init/juju-machine-agent.conf",
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"server-1081.novalocal",
"sudo",
"sed",
"-i",
"'s/env JUJU_ZOOKEEPER=.*/env JUJU_ZOOKEEPER=\":2181\"/g'",
"/etc/init/juju-machine-agent.conf",
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"server-1081.novalocal",
"grep",
"",
"/etc/init/juju-as_i_rise-0.conf",
"-o",
"StrictHostKeyChecking no",
"-q",
"-l",
"ubuntu",
"server-1081.novalocal",
"sudo",
"sed",
"-i",
"'s/env JUJU_ZOOKEEPER=.*/env JUJU_ZOOKEEPER=\":2181\"/g'",
"/etc/init/juju-as_i_rise-0.conf",
}
c.Assert(commandmocker.Ran(sshTmpdir), gocheck.Equals, true)
c.Assert(commandmocker.Parameters(sshTmpdir), gocheck.DeepEquals, sshOutput)
}