func (s *S) TestRebalanceContainersManyApps(c *check.C) {
p, err := s.startMultipleServersCluster()
c.Assert(err, check.IsNil)
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
err = s.newFakeImage(p, "tsuru/app-otherapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
appInstance2 := provisiontest.NewFakeApp("otherapp", "python", 0)
defer p.Destroy(appInstance2)
p.Provision(appInstance2)
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 1}},
app: appInstance,
imageId: imageId,
provisioner: p,
})
c.Assert(err, check.IsNil)
imageId2, err := image.AppCurrentImageName(appInstance2.GetName())
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 1}},
app: appInstance2,
imageId: imageId2,
provisioner: p,
})
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
Pool: "test-default",
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
appStruct2 := &app.App{
Name: appInstance2.GetName(),
Pool: "test-default",
}
err = s.storage.Apps().Insert(appStruct2)
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
c1, err := p.listContainersByHost("localhost")
c.Assert(err, check.IsNil)
c.Assert(c1, check.HasLen, 2)
err = p.rebalanceContainers(buf, false)
c.Assert(err, check.IsNil)
c1, err = p.listContainersByHost("localhost")
c.Assert(err, check.IsNil)
c.Assert(c1, check.HasLen, 1)
c2, err := p.listContainersByHost("127.0.0.1")
c.Assert(err, check.IsNil)
c.Assert(c2, check.HasLen, 1)
}
func (s *S) TestAppCurrentImageName(c *check.C) {
err := image.AppendAppImageName("myapp", "tsuru/app-myapp:v1")
c.Assert(err, check.IsNil)
img1, err := image.AppCurrentImageName("myapp")
c.Assert(err, check.IsNil)
c.Assert(img1, check.Equals, "tsuru/app-myapp:v1")
err = image.AppendAppImageName("myapp", "tsuru/app-myapp:v2")
c.Assert(err, check.IsNil)
img2, err := image.AppCurrentImageName("myapp")
c.Assert(err, check.IsNil)
c.Assert(img2, check.Equals, "tsuru/app-myapp:v2")
}
func (s *S) TestMoveContainerErrorStarted(c *check.C) {
p, err := s.startMultipleServersCluster()
c.Assert(err, check.IsNil)
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
p.Provision(appInstance)
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
addedConts, err := addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 2}},
app: appInstance,
imageId: imageId,
provisioner: p,
})
c.Assert(err, check.IsNil)
err = addedConts[0].SetStatus(p, provision.StatusError, true)
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
_, err = p.moveContainer(addedConts[0].ID[:6], "127.0.0.1", buf)
c.Assert(err, check.IsNil)
containers, err := p.listContainersByHost("localhost")
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
containers, err = p.listContainersByHost("127.0.0.1")
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
c.Assert(containers[0].Status, check.Equals, provision.StatusStarting.String())
}
func (p *dockerProvisioner) ExecuteCommandIsolated(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
imageID, err := image.AppCurrentImageName(app.GetName())
if err != nil {
return err
}
return p.runCommandInContainer(imageID, cmd, app, stdout, stderr)
}
func (p *swarmProvisioner) ExecuteCommandIsolated(stdout, stderr io.Writer, a provision.App, cmd string, args ...string) error {
if a.GetDeploys() == 0 {
return errors.New("commands can only be executed after the first deploy")
}
img, err := image.AppCurrentImageName(a.GetName())
if err != nil {
return err
}
client, err := chooseDBSwarmNode()
if err != nil {
return err
}
opts := tsuruServiceOpts{
app: a,
image: img,
isIsolatedRun: true,
}
cmds := []string{"/bin/bash", "-lc", cmd}
cmds = append(cmds, args...)
serviceID, _, err := runOnceCmds(client, opts, cmds, stdout, stderr)
if serviceID != "" {
removeServiceAndLog(client, serviceID)
}
return err
}
func (s *S) TestRebalanceContainersByHost(c *check.C) {
otherServer, err := dtesting.NewServer("localhost:0", nil, nil)
c.Assert(err, check.IsNil)
defer otherServer.Stop()
otherUrl := strings.Replace(otherServer.URL(), "127.0.0.1", "localhost", 1)
p := &dockerProvisioner{}
err = p.Initialize()
c.Assert(err, check.IsNil)
p.storage = &cluster.MapStorage{}
p.scheduler = &segregatedScheduler{provisioner: p}
p.cluster, err = cluster.New(p.scheduler, p.storage,
cluster.Node{Address: s.server.URL(), Metadata: map[string]string{"pool": "pool1"}},
cluster.Node{Address: otherUrl, Metadata: map[string]string{"pool": "pool1"}},
)
c.Assert(err, check.IsNil)
opts := provision.AddPoolOptions{Name: "pool1"}
err = provision.AddPool(opts)
c.Assert(err, check.IsNil)
err = provision.AddTeamsToPool("pool1", []string{"team1"})
c.Assert(err, check.IsNil)
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 5}},
app: appInstance,
imageId: imageId,
provisioner: p,
})
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
TeamOwner: "team1",
Pool: "pool1",
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
c1, err := p.listContainersByHost("localhost")
c.Assert(err, check.IsNil)
c.Assert(c1, check.HasLen, 5)
c2, err := p.listContainersByHost("127.0.0.1")
c.Assert(err, check.IsNil)
c.Assert(c2, check.HasLen, 0)
err = p.Cluster().Unregister(otherUrl)
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = p.rebalanceContainersByHost(net.URLToHost(otherUrl), buf)
c.Assert(err, check.IsNil)
c.Assert(p.scheduler.ignoredContainers, check.IsNil)
c2, err = p.listContainersByHost("127.0.0.1")
c.Assert(err, check.IsNil)
c.Assert(c2, check.HasLen, 5)
}
func (s *S) TestRebalanceContainersDryBodyHandler(c *check.C) {
p, err := s.startMultipleServersCluster()
c.Assert(err, check.IsNil)
mainDockerProvisioner = p
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
coll := p.Collection()
defer coll.Close()
coll.Insert(container.Container{ID: "container-id", AppName: appInstance.GetName(), Version: "container-version", Image: "tsuru/python", ProcessName: "web"})
defer coll.RemoveAll(bson.M{"appname": appInstance.GetName()})
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
units, err := addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 5}},
app: appInstance,
imageId: imageId,
provisioner: p,
})
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
Platform: appInstance.GetPlatform(),
Pool: "test-default",
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
err = s.storage.Apps().Update(
bson.M{"name": appStruct.Name},
bson.M{"$set": bson.M{"units": units}},
)
c.Assert(err, check.IsNil)
recorder := httptest.NewRecorder()
opts := rebalanceOptions{Dry: true}
v, err := form.EncodeToValues(&opts)
c.Assert(err, check.IsNil)
b := strings.NewReader(v.Encode())
request, err := http.NewRequest("POST", "/docker/containers/rebalance", b)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
server := api.RunServer(true)
server.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
body, err := ioutil.ReadAll(recorder.Body)
c.Assert(err, check.IsNil)
validJson := fmt.Sprintf("[%s]", strings.Replace(strings.Trim(string(body), "\n "), "\n", ",", -1))
var result []tsuruIo.SimpleJsonMessage
err = json.Unmarshal([]byte(validJson), &result)
c.Assert(err, check.IsNil)
c.Assert(result, check.HasLen, 8)
c.Assert(result[0].Message, check.Equals, "Rebalancing 6 units...\n")
c.Assert(result[1].Message, check.Matches, "(?s)Would move unit .*")
c.Assert(result[7].Message, check.Equals, "Containers successfully rebalanced!\n")
}
func (s *S) TestMoveContainer(c *check.C) {
p, err := s.startMultipleServersCluster()
c.Assert(err, check.IsNil)
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
coll := p.Collection()
defer coll.Close()
coll.Insert(container.Container{
ID: "container-id",
AppName: appInstance.GetName(),
Version: "container-version",
Image: "tsuru/python",
})
defer coll.RemoveAll(bson.M{"appname": appInstance.GetName()})
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
addedConts, err := addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 2}},
app: appInstance,
imageId: imageId,
provisioner: p,
})
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
var serviceBodies []string
var serviceMethods []string
rollback := s.addServiceInstance(c, appInstance.GetName(), []string{addedConts[0].ID}, func(w http.ResponseWriter, r *http.Request) {
data, _ := ioutil.ReadAll(r.Body)
serviceBodies = append(serviceBodies, string(data))
serviceMethods = append(serviceMethods, r.Method)
w.WriteHeader(http.StatusOK)
})
defer rollback()
_, err = p.moveContainer(addedConts[0].ID[:6], "127.0.0.1", buf)
c.Assert(err, check.IsNil)
containers, err := p.listContainersByHost("localhost")
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
containers, err = p.listContainersByHost("127.0.0.1")
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 1)
c.Assert(serviceBodies, check.HasLen, 2)
c.Assert(serviceMethods, check.HasLen, 2)
c.Assert(serviceMethods[0], check.Equals, "POST")
c.Assert(serviceBodies[0], check.Matches, ".*unit-host=127.0.0.1")
c.Assert(serviceMethods[1], check.Equals, "DELETE")
c.Assert(serviceBodies[1], check.Matches, ".*unit-host=localhost")
}
func (s *S) TestRebalanceContainersDry(c *check.C) {
p, err := s.startMultipleServersCluster()
c.Assert(err, check.IsNil)
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
args := changeUnitsPipelineArgs{
app: appInstance,
toAdd: map[string]*containersToAdd{"web": {Quantity: 5}},
imageId: imageId,
provisioner: p,
toHost: "localhost",
}
pipeline := action.NewPipeline(
&provisionAddUnitsToHost,
&bindAndHealthcheck,
&addNewRoutes,
&setRouterHealthcheck,
&updateAppImage,
)
err = pipeline.Execute(args)
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
Pool: "test-default",
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
router, err := getRouterForApp(appInstance)
c.Assert(err, check.IsNil)
beforeRoutes, err := router.Routes(appStruct.Name)
c.Assert(err, check.IsNil)
c.Assert(beforeRoutes, check.HasLen, 5)
var serviceCalled bool
rollback := s.addServiceInstance(c, appInstance.GetName(), nil, func(w http.ResponseWriter, r *http.Request) {
serviceCalled = true
w.WriteHeader(http.StatusOK)
})
defer rollback()
buf := safe.NewBuffer(nil)
err = p.rebalanceContainers(buf, true)
c.Assert(err, check.IsNil)
c1, err := p.listContainersByHost("localhost")
c.Assert(err, check.IsNil)
c2, err := p.listContainersByHost("127.0.0.1")
c.Assert(err, check.IsNil)
c.Assert(c1, check.HasLen, 5)
c.Assert(c2, check.HasLen, 0)
routes, err := router.Routes(appStruct.Name)
c.Assert(err, check.IsNil)
c.Assert(routes, check.DeepEquals, beforeRoutes)
c.Assert(serviceCalled, check.Equals, false)
}
func (s *S) TestMoveContainers(c *check.C) {
p, err := s.startMultipleServersCluster()
c.Assert(err, check.IsNil)
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
coll := p.Collection()
defer coll.Close()
coll.Insert(container.Container{
ID: "container-id",
AppName: appInstance.GetName(),
Version: "container-version",
Image: "tsuru/python",
})
defer coll.RemoveAll(bson.M{"appname": appInstance.GetName()})
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
_, err = addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 2}},
app: appInstance,
imageId: imageId,
provisioner: p,
})
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
buf := safe.NewBuffer(nil)
err = p.MoveContainers("localhost", "127.0.0.1", buf)
c.Assert(err, check.IsNil)
containers, err := p.listContainersByHost("localhost")
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 0)
containers, err = p.listContainersByHost("127.0.0.1")
c.Assert(err, check.IsNil)
c.Assert(containers, check.HasLen, 2)
parts := strings.Split(buf.String(), "\n")
c.Assert(parts[0], check.Matches, ".*Moving 2 units.*")
var matches int
movingRegexp := regexp.MustCompile(`.*Moving unit.*for.*myapp.*localhost.*127.0.0.1.*`)
for _, line := range parts[1:] {
if movingRegexp.MatchString(line) {
matches++
}
}
c.Assert(matches, check.Equals, 2)
}
func (p *swarmProvisioner) RoutableAddresses(a provision.App) ([]url.URL, error) {
client, err := chooseDBSwarmNode()
if err != nil {
return nil, err
}
imgID, err := image.AppCurrentImageName(a.GetName())
if err != nil {
if err != image.ErrNoImagesAvailable {
return nil, err
}
return nil, nil
}
webProcessName, err := image.GetImageWebProcessName(imgID)
if err != nil {
return nil, err
}
if webProcessName == "" {
return nil, nil
}
srvName := serviceNameForApp(a, webProcessName)
srv, err := client.InspectService(srvName)
if err != nil {
return nil, err
}
var pubPort uint32
if len(srv.Endpoint.Ports) > 0 {
pubPort = srv.Endpoint.Ports[0].PublishedPort
}
if pubPort == 0 {
return nil, nil
}
nodes, err := listValidNodes(client)
if err != nil {
return nil, err
}
for i := len(nodes) - 1; i >= 0; i-- {
if nodes[i].Spec.Annotations.Labels[labelNodePoolName.String()] != a.GetPool() {
nodes[i], nodes[len(nodes)-1] = nodes[len(nodes)-1], nodes[i]
nodes = nodes[:len(nodes)-1]
}
}
addrs := make([]url.URL, len(nodes))
for i, n := range nodes {
addr := n.Spec.Labels[labelNodeDockerAddr.String()]
host := tsuruNet.URLToHost(addr)
addrs[i] = url.URL{
Scheme: "http",
Host: fmt.Sprintf("%s:%d", host, pubPort),
}
}
return addrs, nil
}
func allAppProcesses(appName string) ([]string, error) {
var processes []string
imgID, err := image.AppCurrentImageName(appName)
if err != nil {
return nil, errors.WithStack(err)
}
data, err := image.GetImageCustomData(imgID)
if err != nil {
return nil, errors.WithStack(err)
}
for procName := range data.Processes {
processes = append(processes, procName)
}
return processes, nil
}
func deployProcesses(client *docker.Client, a provision.App, newImg string, updateSpec processSpec) error {
curImg, err := image.AppCurrentImageName(a.GetName())
if err != nil {
return err
}
currentImageData, err := image.GetImageCustomData(curImg)
if err != nil {
return err
}
currentSpec := processSpec{}
for p := range currentImageData.Processes {
currentSpec[p] = processState{}
}
newImageData, err := image.GetImageCustomData(newImg)
if err != nil {
return err
}
if len(newImageData.Processes) == 0 {
return errors.Errorf("no process information found deploying image %q", newImg)
}
newSpec := processSpec{}
for p := range newImageData.Processes {
newSpec[p] = processState{start: true}
if updateSpec != nil {
newSpec[p] = updateSpec[p]
}
}
pipeline := action.NewPipeline(
updateServices,
updateImageInDB,
removeOldServices,
)
return pipeline.Execute(&pipelineArgs{
client: client,
app: a,
newImage: newImg,
newImageSpec: newSpec,
currentImage: curImg,
currentImageSpec: currentSpec,
})
}
func (p *dockerProvisioner) RoutableAddresses(app provision.App) ([]url.URL, error) {
imageId, err := image.AppCurrentImageName(app.GetName())
if err != nil && err != image.ErrNoImagesAvailable {
return nil, err
}
webProcessName, err := image.GetImageWebProcessName(imageId)
if err != nil {
return nil, err
}
containers, err := p.listContainersByApp(app.GetName())
if err != nil {
return nil, err
}
addrs := make([]url.URL, 0, len(containers))
for _, container := range containers {
if container.ProcessName == webProcessName && container.ValidAddr() {
addrs = append(addrs, *container.Address())
}
}
return addrs, nil
}
func (p *dockerProvisioner) AddUnits(a provision.App, units uint, process string, w io.Writer) error {
if a.GetDeploys() == 0 {
return errors.New("New units can only be added after the first deployment")
}
if units == 0 {
return errors.New("Cannot add 0 units")
}
if w == nil {
w = ioutil.Discard
}
writer := io.MultiWriter(w, &app.LogWriter{App: a})
imageId, err := image.AppCurrentImageName(a.GetName())
if err != nil {
return err
}
imageData, err := image.GetImageCustomData(imageId)
if err != nil {
return err
}
_, err = p.runCreateUnitsPipeline(writer, a, map[string]*containersToAdd{process: {Quantity: int(units)}}, imageId, imageData.ExposedPort)
return err
}
func changeUnits(a provision.App, units int, processName string, w io.Writer) error {
if a.GetDeploys() == 0 {
return errors.New("units can only be modified after the first deploy")
}
if units == 0 {
return errors.New("cannot change 0 units")
}
client, err := chooseDBSwarmNode()
if err != nil {
return err
}
imageId, err := image.AppCurrentImageName(a.GetName())
if err != nil {
return err
}
if processName == "" {
_, processName, err = dockercommon.ProcessCmdForImage(processName, imageId)
if err != nil {
return errors.WithStack(err)
}
}
return deployProcesses(client, a, imageId, processSpec{processName: processState{increment: units}})
}
func (p *dockerProvisioner) Restart(a provision.App, process string, w io.Writer) error {
containers, err := p.listContainersByProcess(a.GetName(), process)
if err != nil {
return err
}
imageId, err := image.AppCurrentImageName(a.GetName())
if err != nil {
return err
}
if w == nil {
w = ioutil.Discard
}
writer := io.MultiWriter(w, &app.LogWriter{App: a})
toAdd := make(map[string]*containersToAdd, len(containers))
for _, c := range containers {
if _, ok := toAdd[c.ProcessName]; !ok {
toAdd[c.ProcessName] = &containersToAdd{Quantity: 0}
}
toAdd[c.ProcessName].Quantity++
toAdd[c.ProcessName].Status = provision.StatusStarted
}
_, err = p.runReplaceUnitsPipeline(writer, a, toAdd, containers, imageId)
return err
}
func changeAppState(a provision.App, process string, state processState) error {
client, err := chooseDBSwarmNode()
if err != nil {
return err
}
var processes []string
if process == "" {
processes, err = allAppProcesses(a.GetName())
if err != nil {
return err
}
} else {
processes = []string{process}
}
imgID, err := image.AppCurrentImageName(a.GetName())
if err != nil {
return errors.WithStack(err)
}
spec := processSpec{}
for _, procName := range processes {
spec[procName] = state
}
return deployProcesses(client, a, imgID, spec)
}
func (s *S) TestAppCurrentImageNameWithoutImage(c *check.C) {
img1, err := image.AppCurrentImageName("myapp")
c.Assert(err, check.IsNil)
c.Assert(img1, check.Equals, "tsuru/app-myapp")
}
func (p *dockerProvisioner) RemoveUnits(a provision.App, units uint, processName string, w io.Writer) error {
if a == nil {
return errors.New("remove units: app should not be nil")
}
if units == 0 {
return errors.New("cannot remove zero units")
}
var err error
if w == nil {
w = ioutil.Discard
}
imgId, err := image.AppCurrentImageName(a.GetName())
if err != nil {
return err
}
_, processName, err = dockercommon.ProcessCmdForImage(processName, imgId)
if err != nil {
return err
}
containers, err := p.listContainersByProcess(a.GetName(), processName)
if err != nil {
return err
}
if len(containers) < int(units) {
return errors.Errorf("cannot remove %d units from process %q, only %d available", units, processName, len(containers))
}
fmt.Fprintf(w, "\n---- Removing %d %s ----\n", units, pluralize("unit", int(units)))
p, err = p.cloneProvisioner(nil)
if err != nil {
return err
}
toRemove := make([]container.Container, 0, units)
for i := 0; i < int(units); i++ {
var (
containerID string
cont *container.Container
)
containerID, err = p.scheduler.GetRemovableContainer(a.GetName(), processName)
if err != nil {
return err
}
cont, err = p.GetContainer(containerID)
if err != nil {
return err
}
p.scheduler.ignoredContainers = append(p.scheduler.ignoredContainers, cont.ID)
toRemove = append(toRemove, *cont)
}
args := changeUnitsPipelineArgs{
app: a,
toRemove: toRemove,
writer: w,
provisioner: p,
}
pipeline := action.NewPipeline(
&removeOldRoutes,
&provisionRemoveOldUnits,
&provisionUnbindOldUnits,
)
err = pipeline.Execute(args)
if err != nil {
return errors.Wrap(err, "error removing routes, units weren't removed")
}
return nil
}
func (s *S) TestRebalanceContainersFilters(c *check.C) {
p, err := s.startMultipleServersClusterSeggregated()
c.Assert(err, check.IsNil)
mainDockerProvisioner = p
err = s.newFakeImage(p, "tsuru/app-myapp", nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp("myapp", "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
coll := p.Collection()
defer coll.Close()
defer coll.RemoveAll(bson.M{"appname": appInstance.GetName()})
imageId, err := image.AppCurrentImageName(appInstance.GetName())
c.Assert(err, check.IsNil)
units, err := addContainersWithHost(&changeUnitsPipelineArgs{
toHost: "localhost",
toAdd: map[string]*containersToAdd{"web": {Quantity: 5}},
app: appInstance,
imageId: imageId,
provisioner: p,
})
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
Platform: appInstance.GetPlatform(),
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
err = s.storage.Apps().Update(
bson.M{"name": appStruct.Name},
bson.M{"$set": bson.M{"units": units}},
)
c.Assert(err, check.IsNil)
opts := rebalanceOptions{
MetadataFilter: map[string]string{"pool": "pool1"},
}
v, err := form.EncodeToValues(&opts)
c.Assert(err, check.IsNil)
b := strings.NewReader(v.Encode())
recorder := httptest.NewRecorder()
request, err := http.NewRequest("POST", "/docker/containers/rebalance", b)
c.Assert(err, check.IsNil)
request.Header.Set("Authorization", "bearer "+s.token.GetValue())
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
server := api.RunServer(true)
server.ServeHTTP(recorder, request)
c.Assert(recorder.Code, check.Equals, http.StatusOK)
body, err := ioutil.ReadAll(recorder.Body)
c.Assert(err, check.IsNil)
validJson := fmt.Sprintf("[%s]", strings.Replace(strings.Trim(string(body), "\n "), "\n", ",", -1))
var result []tsuruIo.SimpleJsonMessage
err = json.Unmarshal([]byte(validJson), &result)
c.Assert(err, check.IsNil)
c.Assert(result, check.HasLen, 2)
c.Assert(result[0].Message, check.Equals, "No containers found to rebalance\n")
c.Assert(result[1].Message, check.Equals, "Containers successfully rebalanced!\n")
c.Assert(eventtest.EventDesc{
Target: event.Target{Type: event.TargetTypePool, Value: "pool1"},
Owner: s.token.GetUserName(),
Kind: "node.update.rebalance",
StartCustomData: []map[string]interface{}{
{"name": "MetadataFilter.pool", "value": "pool1"},
},
}, eventtest.HasEvent)
}
func (s *S) newContainer(opts *newContainerOpts, p *dockerProvisioner) (*container.Container, error) {
container := container.Container{
ID: "id",
IP: "10.10.10.10",
HostPort: "3333",
HostAddr: "127.0.0.1",
ProcessName: "web",
ExposedPort: "8888/tcp",
}
if p == nil {
p = s.p
}
imageName := "tsuru/python:latest"
var customData map[string]interface{}
if opts != nil {
if opts.Image != "" {
imageName = opts.Image
}
container.AppName = opts.AppName
container.ProcessName = opts.ProcessName
customData = opts.ImageCustomData
if opts.Provisioner != nil {
p = opts.Provisioner
}
container.SetStatus(p, provision.Status(opts.Status), false)
}
err := s.newFakeImage(p, imageName, customData)
if err != nil {
return nil, err
}
if container.AppName == "" {
container.AppName = "container"
}
routertest.FakeRouter.AddBackend(container.AppName)
routertest.FakeRouter.AddRoute(container.AppName, container.Address())
ports := map[docker.Port]struct{}{
docker.Port(s.port + "/tcp"): {},
}
config := docker.Config{
Image: imageName,
Cmd: []string{"ps"},
ExposedPorts: ports,
}
createOptions := docker.CreateContainerOptions{Config: &config}
createOptions.Name = randomString()
_, c, err := p.Cluster().CreateContainer(createOptions, net.StreamInactivityTimeout)
if err != nil {
return nil, err
}
container.ID = c.ID
container.Image = imageName
container.Name = createOptions.Name
conn, err := db.Conn()
if err != nil {
return nil, err
}
defer conn.Close()
err = conn.Collection(s.collName).Insert(&container)
if err != nil {
return nil, err
}
imageId, err := image.AppCurrentImageName(container.AppName)
if err != nil {
return nil, err
}
err = s.newFakeImage(p, imageId, nil)
if err != nil {
return nil, err
}
return &container, nil
}
fmt.Fprintf(writer, "\n---- Setting router healthcheck (%s) ----\n", msg)
err = hcRouter.SetHealthcheck(args.app.GetName(), hcData)
return newContainers, err
},
Backward: func(ctx action.BWContext) {
args := ctx.Params[0].(changeUnitsPipelineArgs)
r, err := getRouterForApp(args.app)
if err != nil {
log.Errorf("[set-router-healthcheck:Backward] Error getting router: %s", err)
return
}
hcRouter, ok := r.(router.CustomHealthcheckRouter)
if !ok {
return
}
currentImageName, _ := image.AppCurrentImageName(args.app.GetName())
yamlData, err := image.GetImageTsuruYamlData(currentImageName)
if err != nil {
log.Errorf("[set-router-healthcheck:Backward] Error getting yaml data: %s", err)
}
hcData := yamlData.Healthcheck.ToRouterHC()
err = hcRouter.SetHealthcheck(args.app.GetName(), hcData)
if err != nil {
log.Errorf("[set-router-healthcheck:Backward] Error setting healthcheck: %s", err)
}
},
}
var removeOldRoutes = action.Action{
Name: "remove-old-routes",
Forward: func(ctx action.FWContext) (result action.Result, err error) {
func (s *S) TestRebalanceContainersManyAppsSegStress(c *check.C) {
var nodes []cluster.Node
var nodeHosts []string
for i := 0; i < 6; i++ {
newIp := fmt.Sprintf("127.0.0.%d", i+1)
otherServer, err := dtesting.NewServer(newIp+":0", nil, nil)
c.Assert(err, check.IsNil)
defer otherServer.Stop()
nodes = append(nodes, cluster.Node{Address: otherServer.URL(), Metadata: map[string]string{"pool": "pool1"}})
nodeHosts = append(nodeHosts, net.URLToHost(otherServer.URL()))
}
var err error
p := &dockerProvisioner{}
err = p.Initialize()
c.Assert(err, check.IsNil)
p.storage, err = buildClusterStorage()
c.Assert(err, check.IsNil)
p.scheduler = &segregatedScheduler{provisioner: p}
p.cluster, err = cluster.New(p.scheduler, p.storage, nodes...)
c.Assert(err, check.IsNil)
opts := provision.AddPoolOptions{Name: "pool1"}
err = provision.AddPool(opts)
c.Assert(err, check.IsNil)
err = provision.AddTeamsToPool("pool1", []string{"team1"})
c.Assert(err, check.IsNil)
variation := []int{10, 20, 30, 40, 50, 100}
maxContainers := 40
for i := 0; i < maxContainers; i++ {
appName := fmt.Sprintf("myapp-%d", i)
err = s.newFakeImage(p, "tsuru/app-"+appName, nil)
c.Assert(err, check.IsNil)
appInstance := provisiontest.NewFakeApp(appName, "python", 0)
defer p.Destroy(appInstance)
p.Provision(appInstance)
imageId, aErr := image.AppCurrentImageName(appInstance.GetName())
c.Assert(aErr, check.IsNil)
var chosenNode string
for j := range variation {
if i < (maxContainers*variation[j])/100 {
chosenNode = nodeHosts[j]
break
}
}
args := changeUnitsPipelineArgs{
app: appInstance,
toAdd: map[string]*containersToAdd{"web": {Quantity: 6}},
imageId: imageId,
provisioner: p,
toHost: chosenNode,
}
pipeline := action.NewPipeline(
&provisionAddUnitsToHost,
&bindAndHealthcheck,
&addNewRoutes,
&setRouterHealthcheck,
&updateAppImage,
)
err = pipeline.Execute(args)
c.Assert(err, check.IsNil)
appStruct := &app.App{
Name: appInstance.GetName(),
TeamOwner: "team1",
Pool: "pool1",
}
err = s.storage.Apps().Insert(appStruct)
c.Assert(err, check.IsNil)
}
buf := safe.NewBuffer(nil)
cloneProv, err := p.rebalanceContainersByFilter(buf, []string{}, map[string]string{"pool": "pool1"}, false)
c.Assert(err, check.IsNil)
c.Assert(cloneProv.cluster.Healer, check.Equals, p.cluster.Healer)
for i := range nodeHosts {
conts, err := p.listContainersByHost(nodeHosts[i])
c.Assert(err, check.IsNil)
c.Logf("containers in %q: %d", nodeHosts[i], len(conts))
c.Check(len(conts) >= 39 || len(conts) <= 41, check.Equals, true)
}
}
