func (s *S) TestAddNewRouteForwardNoWeb(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
imageName := "tsuru/app-" + app.GetName()
customData := map[string]interface{}{
"processes": map[string]interface{}{
"api": "python myapi.py",
},
}
err := saveImageCustomData(imageName, customData)
c.Assert(err, check.IsNil)
cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "api", HostAddr: "127.0.0.1", HostPort: "1234"}
cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "api", HostAddr: "127.0.0.2", HostPort: "4321"}
defer cont1.Remove(s.p)
defer cont2.Remove(s.p)
args := changeUnitsPipelineArgs{
app: app,
provisioner: s.p,
imageId: imageName,
}
context := action.FWContext{Previous: []container.Container{cont1, cont2}, Params: []interface{}{args}}
r, err := addNewRoutes.Forward(context)
c.Assert(err, check.IsNil)
containers := r.([]container.Container)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
c.Assert(hasRoute, check.Equals, true)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
c.Assert(hasRoute, check.Equals, true)
c.Assert(containers, check.HasLen, 2)
c.Assert(containers[0].Routable, check.Equals, true)
c.Assert(containers[0].ID, check.Equals, "ble-1")
c.Assert(containers[1].Routable, check.Equals, true)
c.Assert(containers[1].ID, check.Equals, "ble-2")
}
func (s *S) TestCreateContainerForward(c *check.C) {
err := s.newFakeImage(s.p, "tsuru/python", nil)
c.Assert(err, check.IsNil)
client, err := docker.NewClient(s.server.URL())
c.Assert(err, check.IsNil)
images, err := client.ListImages(docker.ListImagesOptions{All: true})
c.Assert(err, check.IsNil)
cmds := []string{"ps", "-ef"}
app := provisiontest.NewFakeApp("myapp", "python", 1)
cont := container.Container{Name: "myName", AppName: app.GetName(), Type: app.GetPlatform(), Status: "created"}
args := runContainerActionsArgs{
app: app,
imageID: images[0].ID,
commands: cmds,
provisioner: s.p,
}
context := action.FWContext{Previous: cont, Params: []interface{}{args}}
r, err := createContainer.Forward(context)
c.Assert(err, check.IsNil)
cont = r.(container.Container)
defer cont.Remove(s.p)
c.Assert(cont, check.FitsTypeOf, container.Container{})
c.Assert(cont.ID, check.Not(check.Equals), "")
c.Assert(cont.HostAddr, check.Equals, "127.0.0.1")
dcli, err := docker.NewClient(s.server.URL())
c.Assert(err, check.IsNil)
cc, err := dcli.InspectContainer(cont.ID)
c.Assert(err, check.IsNil)
c.Assert(cc.State.Running, check.Equals, false)
}
func (p *dockerProvisioner) fixContainer(container *container.Container, info container.NetworkInfo) error {
if info.HTTPHostPort == "" {
return nil
}
appInstance, err := app.GetByName(container.AppName)
if err != nil {
return err
}
r, err := getRouterForApp(appInstance)
if err != nil {
return err
}
err = r.RemoveRoute(container.AppName, container.Address())
if err != nil && err != router.ErrRouteNotFound {
return err
}
container.IP = info.IP
container.HostPort = info.HTTPHostPort
err = r.AddRoute(container.AppName, container.Address())
if err != nil && err != router.ErrRouteExists {
return err
}
coll := p.Collection()
defer coll.Close()
return coll.Update(bson.M{"id": container.ID}, container)
}
func (s *S) TestFollowLogsAndCommitForward(c *check.C) {
err := s.newFakeImage(s.p, "tsuru/python", nil)
c.Assert(err, check.IsNil)
app := provisiontest.NewFakeApp("mightyapp", "python", 1)
nextImgName, err := appNewImageName(app.GetName())
c.Assert(err, check.IsNil)
cont := container.Container{AppName: "mightyapp", ID: "myid123", BuildingImage: nextImgName}
err = cont.Create(&container.CreateArgs{
App: app,
ImageID: "tsuru/python",
Commands: []string{"foo"},
Provisioner: s.p,
})
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
args := runContainerActionsArgs{writer: buf, provisioner: s.p}
context := action.FWContext{Params: []interface{}{args}, Previous: cont}
imageId, err := followLogsAndCommit.Forward(context)
c.Assert(err, check.IsNil)
c.Assert(imageId, check.Equals, "tsuru/app-mightyapp:v1")
c.Assert(buf.String(), check.Not(check.Equals), "")
var dbCont container.Container
coll := s.p.Collection()
defer coll.Close()
err = coll.Find(bson.M{"id": cont.ID}).One(&dbCont)
c.Assert(err, check.NotNil)
c.Assert(err.Error(), check.Equals, "not found")
_, err = s.p.Cluster().InspectContainer(cont.ID)
c.Assert(err, check.NotNil)
c.Assert(err.Error(), check.Matches, "No such container.*")
err = s.p.Cluster().RemoveImage("tsuru/app-mightyapp:v1")
c.Assert(err, check.IsNil)
}
func (h *ContainerHealer) healContainerIfNeeded(cont container.Container) error {
if cont.LastSuccessStatusUpdate.IsZero() {
if !cont.MongoID.Time().Before(time.Now().Add(-h.maxUnresponsiveTime)) {
return nil
}
}
isAsExpected, err := h.isAsExpected(cont)
if err != nil {
log.Errorf("Containers healing: couldn't verify running processes in container %q: %s", cont.ID, err)
}
if isAsExpected {
cont.SetStatus(h.provisioner, cont.ExpectedStatus(), true)
return nil
}
locked := h.locker.Lock(cont.AppName)
if !locked {
return errors.Errorf("Containers healing: unable to heal %q couldn't lock app %s", cont.ID, cont.AppName)
}
defer h.locker.Unlock(cont.AppName)
// Sanity check, now we have a lock, let's find out if the container still exists
_, err = h.provisioner.GetContainer(cont.ID)
if err != nil {
if _, isNotFound := err.(*provision.UnitNotFoundError); isNotFound {
return nil
}
return errors.Wrapf(err, "Containers healing: unable to heal %q couldn't verify it still exists", cont.ID)
}
a, err := app.GetByName(cont.AppName)
if err != nil {
return errors.Wrapf(err, "Containers healing: unable to heal %q couldn't get app %q", cont.ID, cont.AppName)
}
log.Errorf("Initiating healing process for container %q, unresponsive since %s.", cont.ID, cont.LastSuccessStatusUpdate)
evt, err := event.NewInternal(&event.Opts{
Target: event.Target{Type: event.TargetTypeContainer, Value: cont.ID},
InternalKind: "healer",
CustomData: cont,
Allowed: event.Allowed(permission.PermAppReadEvents, append(permission.Contexts(permission.CtxTeam, a.Teams),
permission.Context(permission.CtxApp, a.Name),
permission.Context(permission.CtxPool, a.Pool),
)...),
})
if err != nil {
return errors.Wrap(err, "Error trying to insert container healing event, healing aborted")
}
newCont, healErr := h.healContainer(cont)
if healErr != nil {
healErr = errors.Errorf("Error healing container %q: %s", cont.ID, healErr.Error())
}
err = evt.DoneCustomData(healErr, newCont)
if err != nil {
log.Errorf("Error trying to update containers healing event: %s", err)
}
return healErr
}
func (s *S) TestCreateContainerForward(c *check.C) {
config.Set("docker:user", "ubuntu")
defer config.Unset("docker:user")
err := s.newFakeImage(s.p, "tsuru/python", nil)
c.Assert(err, check.IsNil)
client, err := docker.NewClient(s.server.URL())
c.Assert(err, check.IsNil)
images, err := client.ListImages(docker.ListImagesOptions{All: true})
c.Assert(err, check.IsNil)
cmds := []string{"ps", "-ef"}
app := provisiontest.NewFakeApp("myapp", "python", 1)
cont := container.Container{Name: "myName", AppName: app.GetName(), Type: app.GetPlatform(), Status: "created"}
args := runContainerActionsArgs{
app: app,
imageID: images[0].ID,
commands: cmds,
provisioner: s.p,
buildingImage: images[0].ID,
isDeploy: true,
}
context := action.FWContext{Previous: cont, Params: []interface{}{args}}
r, err := createContainer.Forward(context)
c.Assert(err, check.IsNil)
cont = r.(container.Container)
defer cont.Remove(s.p)
c.Assert(cont, check.FitsTypeOf, container.Container{})
c.Assert(cont.ID, check.Not(check.Equals), "")
c.Assert(cont.HostAddr, check.Equals, "127.0.0.1")
dcli, err := docker.NewClient(s.server.URL())
c.Assert(err, check.IsNil)
cc, err := dcli.InspectContainer(cont.ID)
c.Assert(err, check.IsNil)
c.Assert(cc.State.Running, check.Equals, false)
c.Assert(cc.Config.User, check.Equals, "ubuntu")
args = runContainerActionsArgs{
app: app,
imageID: images[0].ID,
commands: cmds,
provisioner: s.p,
}
optsPull := docker.PullImageOptions{Repository: images[0].ID, OutputStream: nil}
err = s.p.Cluster().PullImage(optsPull, docker.AuthConfiguration{})
c.Assert(err, check.IsNil)
cont = container.Container{Name: "myName2", AppName: app.GetName(), Type: app.GetPlatform(), Status: "created"}
context = action.FWContext{Previous: cont, Params: []interface{}{args}}
r, err = createContainer.Forward(context)
c.Assert(err, check.IsNil)
cont = r.(container.Container)
defer cont.Remove(s.p)
cc, err = dcli.InspectContainer(cont.ID)
c.Assert(err, check.IsNil)
c.Assert(cc.Config.User, check.Equals, "")
}
func (p *dockerProvisioner) runRestartAfterHooks(cont *container.Container, w io.Writer) error {
yamlData, err := getImageTsuruYamlData(cont.Image)
if err != nil {
return err
}
cmds := yamlData.Hooks.Restart.After
for _, cmd := range cmds {
err := cont.Exec(p, w, w, cmd)
if err != nil {
return fmt.Errorf("couldn't execute restart:after hook %q(%s): %s", cmd, cont.ShortID(), err.Error())
}
}
return nil
}
func (h *ContainerHealer) isAsExpected(cont container.Container) (bool, error) {
container, err := h.provisioner.Cluster().InspectContainer(cont.ID)
if err != nil {
return false, err
}
if container.State.Dead || container.State.RemovalInProgress {
return false, nil
}
isRunning := container.State.Running || container.State.Restarting
if cont.ExpectedStatus() == provision.StatusStopped {
return !isRunning, nil
}
return isRunning, nil
}
func (p *dockerProvisioner) fixContainer(container *container.Container, info container.NetworkInfo) error {
if info.HTTPHostPort == "" {
return nil
}
container.IP = info.IP
container.HostPort = info.HTTPHostPort
coll := p.Collection()
defer coll.Close()
err := coll.Update(bson.M{"id": container.ID}, bson.M{
"$set": bson.M{"hostport": container.HostPort, "ip": container.IP},
})
lockedRoutesRebuildOrEnqueue(container.AppName)
return err
}
func (h *ContainerHealer) healContainerIfNeeded(cont container.Container) error {
if cont.LastSuccessStatusUpdate.IsZero() {
if !cont.MongoID.Time().Before(time.Now().Add(-h.maxUnresponsiveTime)) {
return nil
}
}
isRunning, err := h.isRunning(cont)
if err != nil {
log.Errorf("Containers healing: couldn't verify running processes in container %s: %s", cont.ID, err.Error())
}
if isRunning {
cont.SetStatus(h.provisioner, provision.StatusStarted, true)
return nil
}
healingCounter, err := healingCountFor("container", cont.ID, consecutiveHealingsTimeframe)
if err != nil {
return fmt.Errorf("Containers healing: couldn't verify number of previous healings for %s: %s", cont.ID, err.Error())
}
if healingCounter > consecutiveHealingsLimitInTimeframe {
return fmt.Errorf("Containers healing: number of healings for container %s in the last %d minutes exceeds limit of %d: %d",
cont.ID, consecutiveHealingsTimeframe/time.Minute, consecutiveHealingsLimitInTimeframe, healingCounter)
}
locked := h.locker.Lock(cont.AppName)
if !locked {
return fmt.Errorf("Containers healing: unable to heal %s couldn't lock app %s", cont.ID, cont.AppName)
}
defer h.locker.Unlock(cont.AppName)
// Sanity check, now we have a lock, let's find out if the container still exists
_, err = h.provisioner.GetContainer(cont.ID)
if err != nil {
if err == mgo.ErrNotFound {
return nil
}
return fmt.Errorf("Containers healing: unable to heal %s couldn't verify it still exists.", cont.ID)
}
log.Errorf("Initiating healing process for container %s, unresponsive since %s.", cont.ID, cont.LastSuccessStatusUpdate)
evt, err := NewHealingEvent(cont)
if err != nil {
return fmt.Errorf("Error trying to insert container healing event, healing aborted: %s", err.Error())
}
newCont, healErr := h.healContainer(cont)
if healErr != nil {
healErr = fmt.Errorf("Error healing container %s: %s", cont.ID, healErr.Error())
}
err = evt.Update(newCont, healErr)
if err != nil {
log.Errorf("Error trying to update containers healing event: %s", err.Error())
}
return healErr
}
func (p *dockerProvisioner) Shell(opts provision.ShellOptions) error {
var (
c *container.Container
err error
)
if opts.Unit != "" {
c, err = p.GetContainer(opts.Unit)
} else {
c, err = p.getOneContainerByAppName(opts.App.GetName())
}
if err != nil {
return err
}
return c.Shell(p, opts.Conn, opts.Conn, opts.Conn, container.Pty{Width: opts.Width, Height: opts.Height, Term: opts.Term})
}
func (p *dockerProvisioner) checkContainer(container *container.Container) error {
if container.Available() {
info, err := container.NetworkInfo(p)
if err != nil {
return err
}
if info.HTTPHostPort != container.HostPort || info.IP != container.IP {
err = p.fixContainer(container, info)
if err != nil {
log.Errorf("error on fix container hostport for [container %s]", container.ID)
return err
}
}
}
return nil
}
func (s *S) TestUpdateContainerInDBForward(c *check.C) {
cont := container.Container{Name: "myName"}
coll := s.p.Collection()
defer coll.Close()
err := coll.Insert(cont)
c.Assert(err, check.IsNil)
cont.ID = "myID"
context := action.FWContext{Previous: cont, Params: []interface{}{runContainerActionsArgs{
provisioner: s.p,
}}}
r, err := updateContainerInDB.Forward(context)
c.Assert(r, check.FitsTypeOf, container.Container{})
retrieved, err := s.p.GetContainer(cont.ID)
c.Assert(err, check.IsNil)
c.Assert(retrieved.ID, check.Equals, cont.ID)
}
func (s *S) TestMoveOneContainerNoActionNeeded(c *check.C) {
p, err := NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
cont := container.Container{ID: "something"}
p.SetContainers("localhost", []container.Container{cont})
errors := make(chan error, 1)
result := p.MoveOneContainer(cont, "localhost", errors, nil, nil, nil)
cont.HostAddr = "localhost"
c.Assert(result, check.DeepEquals, cont)
select {
case err := <-errors:
c.Error(err)
default:
}
}
func (s *S) TestAllContainers(c *check.C) {
p, err := NewFakeDockerProvisioner()
c.Assert(err, check.IsNil)
defer p.Destroy()
cont1 := container.Container{ID: "cont1"}
cont2 := container.Container{ID: "cont2"}
p.SetContainers("localhost", []container.Container{cont1})
p.SetContainers("remotehost", []container.Container{cont2})
cont1.HostAddr = "localhost"
cont2.HostAddr = "remotehost"
containers := p.AllContainers()
expected := []container.Container{cont1, cont2}
if expected[0].HostAddr != containers[0].HostAddr {
expected = []container.Container{cont2, cont1}
}
c.Assert(containers, check.DeepEquals, expected)
}
func (s *S) TestFollowLogsAndCommitForwardWaitFailure(c *check.C) {
s.server.PrepareFailure("failed to wait for the container", "/containers/.*/wait")
defer s.server.ResetFailure("failed to wait for the container")
err := s.newFakeImage(s.p, "tsuru/python", nil)
c.Assert(err, check.IsNil)
app := provisiontest.NewFakeApp("myapp", "python", 1)
cont := container.Container{AppName: "mightyapp"}
err = cont.Create(&container.CreateArgs{
App: app,
ImageID: "tsuru/python",
Commands: []string{"foo"},
Provisioner: s.p,
})
c.Assert(err, check.IsNil)
err = cont.Start(&container.StartArgs{
Provisioner: s.p,
App: app,
})
c.Assert(err, check.IsNil)
err = cont.Stop(s.p)
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
args := runContainerActionsArgs{writer: buf, provisioner: s.p}
context := action.FWContext{Params: []interface{}{args}, Previous: cont}
imageId, err := followLogsAndCommit.Forward(context)
c.Assert(err, check.ErrorMatches, `.*failed to wait for the container\n$`)
c.Assert(imageId, check.IsNil)
}
func (s *S) TestAddNewRouteForwardDoesNotAddWhenHostPortIsEmpty(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "", HostAddr: "addr1", HostPort: ""}
cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "", HostAddr: "addr2", HostPort: "4321"}
defer cont.Remove(s.p)
defer cont2.Remove(s.p)
args := changeUnitsPipelineArgs{
app: app,
provisioner: s.p,
}
prevContainers := []container.Container{cont, cont2}
context := action.FWContext{Previous: prevContainers, Params: []interface{}{args}}
_, err := addNewRoutes.Forward(context)
c.Assert(err, check.Equals, nil)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
c.Assert(hasRoute, check.Equals, false)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
c.Assert(hasRoute, check.Equals, true)
}
func (s *S) TestFollowLogsAndCommitForwardNonZeroStatus(c *check.C) {
err := s.newFakeImage(s.p, "tsuru/python", nil)
c.Assert(err, check.IsNil)
app := provisiontest.NewFakeApp("myapp", "python", 1)
cont := container.Container{AppName: "mightyapp"}
err = cont.Create(&container.CreateArgs{
App: app,
ImageID: "tsuru/python",
Commands: []string{"foo"},
Provisioner: s.p,
})
c.Assert(err, check.IsNil)
err = s.server.MutateContainer(cont.ID, docker.State{ExitCode: 1})
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
args := runContainerActionsArgs{writer: buf, provisioner: s.p}
context := action.FWContext{Params: []interface{}{args}, Previous: cont}
imageId, err := followLogsAndCommit.Forward(context)
c.Assert(err, check.NotNil)
c.Assert(err.Error(), check.Equals, "Exit status 1")
c.Assert(imageId, check.IsNil)
}
func (s *S) TestRemoveOldRoutesForwardNoImageData(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
err := appendAppImageName(app.GetName(), "img1")
c.Assert(err, check.IsNil)
err = pullAppImageNames(app.GetName(), []string{"img1"})
c.Assert(err, check.IsNil)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "", HostAddr: "127.0.0.1", HostPort: ""}
args := changeUnitsPipelineArgs{
app: app,
toRemove: []container.Container{cont1},
provisioner: s.p,
}
context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
r, err := removeOldRoutes.Forward(context)
c.Assert(err, check.IsNil)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
c.Assert(hasRoute, check.Equals, false)
containers := r.([]container.Container)
c.Assert(containers, check.DeepEquals, []container.Container{})
c.Assert(args.toRemove[0].Routable, check.Equals, false)
}
func (s *S) TestFixContainer(c *check.C) {
cleanup, server, p := startDocker("9999")
defer cleanup()
coll := p.Collection()
defer coll.Close()
cont := container.Container{
ID: "9930c24f1c4x",
AppName: "makea",
Type: "python",
Status: provision.StatusStarted.String(),
IP: "127.0.0.4",
HostPort: "9025",
HostAddr: "127.0.0.1",
}
err := coll.Insert(cont)
c.Assert(err, check.IsNil)
defer coll.RemoveAll(bson.M{"appname": cont.AppName})
err = s.storage.Apps().Insert(&app.App{Name: cont.AppName})
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp(cont.AppName, cont.Type, 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
var storage cluster.MapStorage
storage.StoreContainer(cont.ID, server.URL)
p.cluster, err = cluster.New(nil, &storage,
cluster.Node{Address: server.URL},
)
c.Assert(err, check.IsNil)
info, err := cont.NetworkInfo(p)
c.Assert(err, check.IsNil)
err = p.fixContainer(&cont, info)
c.Assert(err, check.IsNil)
conta, err := p.GetContainer("9930c24f1c4x")
c.Assert(err, check.IsNil)
c.Assert(conta.IP, check.Equals, "127.0.0.9")
c.Assert(conta.HostPort, check.Equals, "9999")
}
func (p *FakeDockerProvisioner) moveOneContainer(cont container.Container, toHost string) (container.Container, error) {
cont, index, err := p.findContainer(cont.ID)
if err != nil {
return cont, err
}
if cont.HostAddr == toHost {
return cont, nil
}
if toHost == "" {
for host := range p.containers {
if host != cont.HostAddr {
toHost = host
break
}
}
}
originHost := cont.HostAddr
moving := ContainerMoving{
ContainerID: cont.ID,
HostFrom: originHost,
HostTo: toHost,
}
p.movings = append(p.movings, moving)
if toHost == "" {
cont.ID += "-recreated"
p.containers[originHost][index] = cont
return cont, nil
}
cont.HostAddr = toHost
cont.ID += "-moved"
last := len(p.containers[originHost]) - 1
p.containers[originHost][index] = p.containers[originHost][last]
p.containers[originHost] = p.containers[originHost][:last]
p.containers[cont.HostAddr] = append(p.containers[cont.HostAddr], cont)
return cont, nil
}
func (s *S) TestGetRemovableContainer(c *check.C) {
a1 := app.App{Name: "impius", Teams: []string{"tsuruteam", "nodockerforme"}, Pool: "pool1"}
cont1 := container.Container{ID: "1", Name: "impius1", AppName: a1.Name, ProcessName: "web"}
cont2 := container.Container{ID: "2", Name: "mirror1", AppName: a1.Name, ProcessName: "worker"}
a2 := app.App{Name: "notimpius", Teams: []string{"tsuruteam", "nodockerforme"}, Pool: "pool1"}
cont3 := container.Container{ID: "3", Name: "dedication1", AppName: a2.Name, ProcessName: "web"}
cont4 := container.Container{ID: "4", Name: "dedication2", AppName: a2.Name, ProcessName: "worker"}
err := s.storage.Apps().Insert(a1)
c.Assert(err, check.IsNil)
err = s.storage.Apps().Insert(a2)
c.Assert(err, check.IsNil)
defer s.storage.Apps().RemoveAll(bson.M{"name": a1.Name})
defer s.storage.Apps().RemoveAll(bson.M{"name": a2.Name})
p := provision.Pool{Name: "pool1", Teams: []string{
"tsuruteam",
"nodockerforme",
}}
o := provision.AddPoolOptions{Name: p.Name}
err = provision.AddPool(o)
c.Assert(err, check.IsNil)
err = provision.AddTeamsToPool(p.Name, p.Teams)
defer provision.RemovePool(p.Name)
contColl := s.p.Collection()
defer contColl.Close()
err = contColl.Insert(
cont1, cont2, cont3, cont4,
)
c.Assert(err, check.IsNil)
defer contColl.RemoveAll(bson.M{"name": bson.M{"$in": []string{cont1.Name, cont2.Name, cont3.Name, cont4.Name}}})
scheduler := segregatedScheduler{provisioner: s.p}
clusterInstance, err := cluster.New(&scheduler, &cluster.MapStorage{})
s.p.cluster = clusterInstance
c.Assert(err, check.IsNil)
server1, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server1.Stop()
server2, err := testing.NewServer("127.0.0.1:0", nil, nil)
c.Assert(err, check.IsNil)
defer server2.Stop()
localURL := strings.Replace(server2.URL(), "127.0.0.1", "localhost", -1)
err = clusterInstance.Register(cluster.Node{
Address: server1.URL(),
Metadata: map[string]string{"pool": "pool1"},
})
c.Assert(err, check.IsNil)
err = clusterInstance.Register(cluster.Node{
Address: localURL,
Metadata: map[string]string{"pool": "pool1"},
})
c.Assert(err, check.IsNil)
opts := docker.CreateContainerOptions{Name: cont1.Name}
_, err = scheduler.Schedule(clusterInstance, opts, []string{a1.Name, cont1.ProcessName})
c.Assert(err, check.IsNil)
opts = docker.CreateContainerOptions{Name: cont2.Name}
_, err = scheduler.Schedule(clusterInstance, opts, []string{a1.Name, cont2.ProcessName})
c.Assert(err, check.IsNil)
opts = docker.CreateContainerOptions{Name: cont3.Name}
_, err = scheduler.Schedule(clusterInstance, opts, []string{a2.Name, cont3.ProcessName})
c.Assert(err, check.IsNil)
opts = docker.CreateContainerOptions{Name: cont4.Name}
_, err = scheduler.Schedule(clusterInstance, opts, []string{a2.Name, cont4.ProcessName})
c.Assert(err, check.IsNil)
cont, err := scheduler.GetRemovableContainer(a1.Name, "web")
c.Assert(err, check.IsNil)
c.Assert(cont, check.Equals, cont1.ID)
err = cont1.Remove(s.p)
c.Assert(err, check.IsNil)
_, err = scheduler.GetRemovableContainer(a1.Name, "web")
c.Assert(err, check.NotNil)
}
func runHealthcheck(cont *container.Container, w io.Writer) error {
yamlData, err := getImageTsuruYamlData(cont.Image)
if err != nil {
return err
}
path := yamlData.Healthcheck.Path
method := yamlData.Healthcheck.Method
match := yamlData.Healthcheck.Match
status := yamlData.Healthcheck.Status
allowedFailures := yamlData.Healthcheck.AllowedFailures
if path == "" {
return nil
}
path = strings.TrimSpace(strings.TrimLeft(path, "/"))
if method == "" {
method = "get"
}
method = strings.ToUpper(method)
if status == 0 && match == "" {
status = 200
}
var matchRE *regexp.Regexp
if match != "" {
match = "(?s)" + match
matchRE, err = regexp.Compile(match)
if err != nil {
return err
}
}
maxWaitTime, _ := config.GetInt("docker:healthcheck:max-time")
if maxWaitTime == 0 {
maxWaitTime = 120
}
maxWaitTime = maxWaitTime * int(time.Second)
sleepTime := 3 * time.Second
startedTime := time.Now()
url := fmt.Sprintf("http://%s:%s/%s", cont.HostAddr, cont.HostPort, path)
for {
var lastError error = nil
req, err := http.NewRequest(method, url, nil)
if err != nil {
return err
}
rsp, err := timeoutHttpClient.Do(req)
if err != nil {
lastError = fmt.Errorf("healthcheck fail(%s): %s", cont.ShortID(), err.Error())
} else {
defer rsp.Body.Close()
if status != 0 && rsp.StatusCode != status {
lastError = fmt.Errorf("healthcheck fail(%s): wrong status code, expected %d, got: %d", cont.ShortID(), status, rsp.StatusCode)
} else if matchRE != nil {
result, err := ioutil.ReadAll(rsp.Body)
if err != nil {
lastError = err
}
if !matchRE.Match(result) {
lastError = fmt.Errorf("healthcheck fail(%s): unexpected result, expected %q, got: %s", cont.ShortID(), match, string(result))
}
}
if lastError != nil {
if allowedFailures == 0 {
return lastError
}
allowedFailures--
}
}
if lastError == nil {
fmt.Fprintf(w, " ---> healthcheck successful(%s)\n", cont.ShortID())
return nil
}
if time.Now().Sub(startedTime) > time.Duration(maxWaitTime) {
return lastError
}
fmt.Fprintf(w, " ---> %s. Trying again in %s\n", lastError.Error(), sleepTime)
time.Sleep(sleepTime)
}
}
func (s *S) removeTestContainer(c *container.Container) error {
routertest.FakeRouter.RemoveBackend(c.AppName)
return c.Remove(s.p)
}
func (s *S) newContainer(opts *newContainerOpts, p *dockerProvisioner) (*container.Container, error) {
container := container.Container{
ID: "id",
IP: "10.10.10.10",
HostPort: "3333",
HostAddr: "127.0.0.1",
ProcessName: "web",
ExposedPort: "8888/tcp",
}
if p == nil {
p = s.p
}
image := "tsuru/python:latest"
var customData map[string]interface{}
if opts != nil {
if opts.Image != "" {
image = opts.Image
}
container.Status = opts.Status
container.AppName = opts.AppName
container.ProcessName = opts.ProcessName
customData = opts.ImageCustomData
if opts.Provisioner != nil {
p = opts.Provisioner
}
}
err := s.newFakeImage(p, image, customData)
if err != nil {
return nil, err
}
if container.AppName == "" {
container.AppName = "container"
}
routertest.FakeRouter.AddBackend(container.AppName)
routertest.FakeRouter.AddRoute(container.AppName, container.Address())
ports := map[docker.Port]struct{}{
docker.Port(s.port + "/tcp"): {},
}
config := docker.Config{
Image: image,
Cmd: []string{"ps"},
ExposedPorts: ports,
}
createOptions := docker.CreateContainerOptions{Config: &config}
createOptions.Name = randomString()
_, c, err := p.Cluster().CreateContainer(createOptions)
if err != nil {
return nil, err
}
container.ID = c.ID
container.Image = image
container.Name = createOptions.Name
conn, err := db.Conn()
if err != nil {
return nil, err
}
defer conn.Close()
err = conn.Collection(s.collName).Insert(&container)
if err != nil {
return nil, err
}
imageId, err := appCurrentImageName(container.AppName)
if err != nil {
return nil, err
}
err = s.newFakeImage(p, imageId, nil)
if err != nil {
return nil, err
}
return &container, nil
}
func (s *S) TestRemoveOldRoutesBackward(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web"}
cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web"}
defer cont.Remove(s.p)
defer cont2.Remove(s.p)
cont.Routable = true
cont2.Routable = true
args := changeUnitsPipelineArgs{
app: app,
toRemove: []container.Container{cont, cont2},
provisioner: s.p,
}
context := action.BWContext{Params: []interface{}{args}}
removeOldRoutes.Backward(context)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
c.Assert(hasRoute, check.Equals, true)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
c.Assert(hasRoute, check.Equals, true)
}
func (s *S) TestRemoveOldRoutesForwardFailInMiddle(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr1"}
cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr2"}
defer cont.Remove(s.p)
defer cont2.Remove(s.p)
err := routertest.FakeRouter.AddRoute(app.GetName(), cont.Address())
c.Assert(err, check.IsNil)
err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
c.Assert(err, check.IsNil)
routertest.FakeRouter.FailForIp(cont2.Address().String())
args := changeUnitsPipelineArgs{
app: app,
toRemove: []container.Container{cont, cont2},
provisioner: s.p,
}
context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
_, err = removeOldRoutes.Forward(context)
c.Assert(err, check.Equals, routertest.ErrForcedFailure)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
c.Assert(hasRoute, check.Equals, true)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
c.Assert(hasRoute, check.Equals, true)
c.Assert(args.toRemove[0].Routable, check.Equals, true)
c.Assert(args.toRemove[1].Routable, check.Equals, false)
}
func (s *S) TestRemoveOldRoutesForward(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.1", HostPort: "1234"}
cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.2", HostPort: "4321"}
cont3 := container.Container{ID: "ble-3", AppName: app.GetName(), ProcessName: "worker", HostAddr: "127.0.0.3", HostPort: "8080"}
defer cont1.Remove(s.p)
defer cont2.Remove(s.p)
defer cont3.Remove(s.p)
err := routertest.FakeRouter.AddRoute(app.GetName(), cont1.Address())
c.Assert(err, check.IsNil)
err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
c.Assert(err, check.IsNil)
args := changeUnitsPipelineArgs{
app: app,
toRemove: []container.Container{cont1, cont2, cont3},
provisioner: s.p,
}
context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
r, err := removeOldRoutes.Forward(context)
c.Assert(err, check.IsNil)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
c.Assert(hasRoute, check.Equals, false)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
c.Assert(hasRoute, check.Equals, false)
containers := r.([]container.Container)
c.Assert(containers, check.DeepEquals, []container.Container{})
c.Assert(args.toRemove[0].Routable, check.Equals, true)
c.Assert(args.toRemove[1].Routable, check.Equals, true)
c.Assert(args.toRemove[2].Routable, check.Equals, false)
}
func (s *S) TestAddNewRouteBackward(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.1", HostPort: "1234"}
cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.2", HostPort: "4321"}
cont3 := container.Container{ID: "ble-3", AppName: app.GetName(), ProcessName: "worker", HostAddr: "127.0.0.3", HostPort: "8080"}
defer cont1.Remove(s.p)
defer cont2.Remove(s.p)
defer cont3.Remove(s.p)
err := routertest.FakeRouter.AddRoute(app.GetName(), cont1.Address())
c.Assert(err, check.IsNil)
err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
c.Assert(err, check.IsNil)
args := changeUnitsPipelineArgs{
app: app,
provisioner: s.p,
}
cont1.Routable = true
cont2.Routable = true
context := action.BWContext{FWResult: []container.Container{cont1, cont2, cont3}, Params: []interface{}{args}}
addNewRoutes.Backward(context)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
c.Assert(hasRoute, check.Equals, false)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
c.Assert(hasRoute, check.Equals, false)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont3.Address().String())
c.Assert(hasRoute, check.Equals, false)
}
func (s *S) TestRemoveOldRoutesForwardFailInMiddle(c *check.C) {
app := provisiontest.NewFakeApp("myapp", "python", 1)
imageName := "tsuru/app-" + app.GetName()
customData := map[string]interface{}{
"processes": map[string]interface{}{
"web": "python myapi.py",
"worker": "tail -f /dev/null",
},
}
err := saveImageCustomData(imageName, customData)
c.Assert(err, check.IsNil)
routertest.FakeRouter.AddBackend(app.GetName())
defer routertest.FakeRouter.RemoveBackend(app.GetName())
cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr1", HostPort: "1234"}
cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr2", HostPort: "1234"}
defer cont.Remove(s.p)
defer cont2.Remove(s.p)
err = routertest.FakeRouter.AddRoute(app.GetName(), cont.Address())
c.Assert(err, check.IsNil)
err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
c.Assert(err, check.IsNil)
routertest.FakeRouter.FailForIp(cont2.Address().String())
args := changeUnitsPipelineArgs{
app: app,
toRemove: []container.Container{cont, cont2},
provisioner: s.p,
}
context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
_, err = removeOldRoutes.Forward(context)
c.Assert(err, check.Equals, routertest.ErrForcedFailure)
hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
c.Assert(hasRoute, check.Equals, true)
hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
c.Assert(hasRoute, check.Equals, true)
c.Assert(args.toRemove[0].Routable, check.Equals, true)
c.Assert(args.toRemove[1].Routable, check.Equals, true)
}