Beispiel #1
0
func (s *S) TestAddNewRouteForwardNoWeb(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	imageName := "tsuru/app-" + app.GetName()
	customData := map[string]interface{}{
		"processes": map[string]interface{}{
			"api": "python myapi.py",
		},
	}
	err := saveImageCustomData(imageName, customData)
	c.Assert(err, check.IsNil)
	cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "api", HostAddr: "127.0.0.1", HostPort: "1234"}
	cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "api", HostAddr: "127.0.0.2", HostPort: "4321"}
	defer cont1.Remove(s.p)
	defer cont2.Remove(s.p)
	args := changeUnitsPipelineArgs{
		app:         app,
		provisioner: s.p,
		imageId:     imageName,
	}
	context := action.FWContext{Previous: []container.Container{cont1, cont2}, Params: []interface{}{args}}
	r, err := addNewRoutes.Forward(context)
	c.Assert(err, check.IsNil)
	containers := r.([]container.Container)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
	c.Assert(hasRoute, check.Equals, true)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
	c.Assert(hasRoute, check.Equals, true)
	c.Assert(containers, check.HasLen, 2)
	c.Assert(containers[0].Routable, check.Equals, true)
	c.Assert(containers[0].ID, check.Equals, "ble-1")
	c.Assert(containers[1].Routable, check.Equals, true)
	c.Assert(containers[1].ID, check.Equals, "ble-2")
}
Beispiel #2
0
func (s *S) TestCreateContainerForward(c *check.C) {
	err := s.newFakeImage(s.p, "tsuru/python", nil)
	c.Assert(err, check.IsNil)
	client, err := docker.NewClient(s.server.URL())
	c.Assert(err, check.IsNil)
	images, err := client.ListImages(docker.ListImagesOptions{All: true})
	c.Assert(err, check.IsNil)
	cmds := []string{"ps", "-ef"}
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	cont := container.Container{Name: "myName", AppName: app.GetName(), Type: app.GetPlatform(), Status: "created"}
	args := runContainerActionsArgs{
		app:         app,
		imageID:     images[0].ID,
		commands:    cmds,
		provisioner: s.p,
	}
	context := action.FWContext{Previous: cont, Params: []interface{}{args}}
	r, err := createContainer.Forward(context)
	c.Assert(err, check.IsNil)
	cont = r.(container.Container)
	defer cont.Remove(s.p)
	c.Assert(cont, check.FitsTypeOf, container.Container{})
	c.Assert(cont.ID, check.Not(check.Equals), "")
	c.Assert(cont.HostAddr, check.Equals, "127.0.0.1")
	dcli, err := docker.NewClient(s.server.URL())
	c.Assert(err, check.IsNil)
	cc, err := dcli.InspectContainer(cont.ID)
	c.Assert(err, check.IsNil)
	c.Assert(cc.State.Running, check.Equals, false)
}
Beispiel #3
0
func (p *dockerProvisioner) fixContainer(container *container.Container, info container.NetworkInfo) error {
	if info.HTTPHostPort == "" {
		return nil
	}
	appInstance, err := app.GetByName(container.AppName)
	if err != nil {
		return err
	}
	r, err := getRouterForApp(appInstance)
	if err != nil {
		return err
	}
	err = r.RemoveRoute(container.AppName, container.Address())
	if err != nil && err != router.ErrRouteNotFound {
		return err
	}
	container.IP = info.IP
	container.HostPort = info.HTTPHostPort
	err = r.AddRoute(container.AppName, container.Address())
	if err != nil && err != router.ErrRouteExists {
		return err
	}
	coll := p.Collection()
	defer coll.Close()
	return coll.Update(bson.M{"id": container.ID}, container)
}
Beispiel #4
0
func (s *S) TestFollowLogsAndCommitForward(c *check.C) {
	err := s.newFakeImage(s.p, "tsuru/python", nil)
	c.Assert(err, check.IsNil)
	app := provisiontest.NewFakeApp("mightyapp", "python", 1)
	nextImgName, err := appNewImageName(app.GetName())
	c.Assert(err, check.IsNil)
	cont := container.Container{AppName: "mightyapp", ID: "myid123", BuildingImage: nextImgName}
	err = cont.Create(&container.CreateArgs{
		App:         app,
		ImageID:     "tsuru/python",
		Commands:    []string{"foo"},
		Provisioner: s.p,
	})
	c.Assert(err, check.IsNil)
	buf := safe.NewBuffer(nil)
	args := runContainerActionsArgs{writer: buf, provisioner: s.p}
	context := action.FWContext{Params: []interface{}{args}, Previous: cont}
	imageId, err := followLogsAndCommit.Forward(context)
	c.Assert(err, check.IsNil)
	c.Assert(imageId, check.Equals, "tsuru/app-mightyapp:v1")
	c.Assert(buf.String(), check.Not(check.Equals), "")
	var dbCont container.Container
	coll := s.p.Collection()
	defer coll.Close()
	err = coll.Find(bson.M{"id": cont.ID}).One(&dbCont)
	c.Assert(err, check.NotNil)
	c.Assert(err.Error(), check.Equals, "not found")
	_, err = s.p.Cluster().InspectContainer(cont.ID)
	c.Assert(err, check.NotNil)
	c.Assert(err.Error(), check.Matches, "No such container.*")
	err = s.p.Cluster().RemoveImage("tsuru/app-mightyapp:v1")
	c.Assert(err, check.IsNil)
}
Beispiel #5
0
func (h *ContainerHealer) healContainerIfNeeded(cont container.Container) error {
	if cont.LastSuccessStatusUpdate.IsZero() {
		if !cont.MongoID.Time().Before(time.Now().Add(-h.maxUnresponsiveTime)) {
			return nil
		}
	}
	isAsExpected, err := h.isAsExpected(cont)
	if err != nil {
		log.Errorf("Containers healing: couldn't verify running processes in container %q: %s", cont.ID, err)
	}
	if isAsExpected {
		cont.SetStatus(h.provisioner, cont.ExpectedStatus(), true)
		return nil
	}
	locked := h.locker.Lock(cont.AppName)
	if !locked {
		return errors.Errorf("Containers healing: unable to heal %q couldn't lock app %s", cont.ID, cont.AppName)
	}
	defer h.locker.Unlock(cont.AppName)
	// Sanity check, now we have a lock, let's find out if the container still exists
	_, err = h.provisioner.GetContainer(cont.ID)
	if err != nil {
		if _, isNotFound := err.(*provision.UnitNotFoundError); isNotFound {
			return nil
		}
		return errors.Wrapf(err, "Containers healing: unable to heal %q couldn't verify it still exists", cont.ID)
	}
	a, err := app.GetByName(cont.AppName)
	if err != nil {
		return errors.Wrapf(err, "Containers healing: unable to heal %q couldn't get app %q", cont.ID, cont.AppName)
	}
	log.Errorf("Initiating healing process for container %q, unresponsive since %s.", cont.ID, cont.LastSuccessStatusUpdate)
	evt, err := event.NewInternal(&event.Opts{
		Target:       event.Target{Type: event.TargetTypeContainer, Value: cont.ID},
		InternalKind: "healer",
		CustomData:   cont,
		Allowed: event.Allowed(permission.PermAppReadEvents, append(permission.Contexts(permission.CtxTeam, a.Teams),
			permission.Context(permission.CtxApp, a.Name),
			permission.Context(permission.CtxPool, a.Pool),
		)...),
	})
	if err != nil {
		return errors.Wrap(err, "Error trying to insert container healing event, healing aborted")
	}
	newCont, healErr := h.healContainer(cont)
	if healErr != nil {
		healErr = errors.Errorf("Error healing container %q: %s", cont.ID, healErr.Error())
	}
	err = evt.DoneCustomData(healErr, newCont)
	if err != nil {
		log.Errorf("Error trying to update containers healing event: %s", err)
	}
	return healErr
}
Beispiel #6
0
func (s *S) TestCreateContainerForward(c *check.C) {
	config.Set("docker:user", "ubuntu")
	defer config.Unset("docker:user")
	err := s.newFakeImage(s.p, "tsuru/python", nil)
	c.Assert(err, check.IsNil)
	client, err := docker.NewClient(s.server.URL())
	c.Assert(err, check.IsNil)
	images, err := client.ListImages(docker.ListImagesOptions{All: true})
	c.Assert(err, check.IsNil)
	cmds := []string{"ps", "-ef"}
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	cont := container.Container{Name: "myName", AppName: app.GetName(), Type: app.GetPlatform(), Status: "created"}
	args := runContainerActionsArgs{
		app:           app,
		imageID:       images[0].ID,
		commands:      cmds,
		provisioner:   s.p,
		buildingImage: images[0].ID,
		isDeploy:      true,
	}
	context := action.FWContext{Previous: cont, Params: []interface{}{args}}
	r, err := createContainer.Forward(context)
	c.Assert(err, check.IsNil)
	cont = r.(container.Container)
	defer cont.Remove(s.p)
	c.Assert(cont, check.FitsTypeOf, container.Container{})
	c.Assert(cont.ID, check.Not(check.Equals), "")
	c.Assert(cont.HostAddr, check.Equals, "127.0.0.1")
	dcli, err := docker.NewClient(s.server.URL())
	c.Assert(err, check.IsNil)
	cc, err := dcli.InspectContainer(cont.ID)
	c.Assert(err, check.IsNil)
	c.Assert(cc.State.Running, check.Equals, false)
	c.Assert(cc.Config.User, check.Equals, "ubuntu")
	args = runContainerActionsArgs{
		app:         app,
		imageID:     images[0].ID,
		commands:    cmds,
		provisioner: s.p,
	}
	optsPull := docker.PullImageOptions{Repository: images[0].ID, OutputStream: nil}
	err = s.p.Cluster().PullImage(optsPull, docker.AuthConfiguration{})
	c.Assert(err, check.IsNil)
	cont = container.Container{Name: "myName2", AppName: app.GetName(), Type: app.GetPlatform(), Status: "created"}
	context = action.FWContext{Previous: cont, Params: []interface{}{args}}
	r, err = createContainer.Forward(context)
	c.Assert(err, check.IsNil)
	cont = r.(container.Container)
	defer cont.Remove(s.p)
	cc, err = dcli.InspectContainer(cont.ID)
	c.Assert(err, check.IsNil)
	c.Assert(cc.Config.User, check.Equals, "")
}
Beispiel #7
0
func (p *dockerProvisioner) runRestartAfterHooks(cont *container.Container, w io.Writer) error {
	yamlData, err := getImageTsuruYamlData(cont.Image)
	if err != nil {
		return err
	}
	cmds := yamlData.Hooks.Restart.After
	for _, cmd := range cmds {
		err := cont.Exec(p, w, w, cmd)
		if err != nil {
			return fmt.Errorf("couldn't execute restart:after hook %q(%s): %s", cmd, cont.ShortID(), err.Error())
		}
	}
	return nil
}
Beispiel #8
0
func (h *ContainerHealer) isAsExpected(cont container.Container) (bool, error) {
	container, err := h.provisioner.Cluster().InspectContainer(cont.ID)
	if err != nil {
		return false, err
	}
	if container.State.Dead || container.State.RemovalInProgress {
		return false, nil
	}
	isRunning := container.State.Running || container.State.Restarting
	if cont.ExpectedStatus() == provision.StatusStopped {
		return !isRunning, nil
	}
	return isRunning, nil
}
Beispiel #9
0
func (p *dockerProvisioner) fixContainer(container *container.Container, info container.NetworkInfo) error {
	if info.HTTPHostPort == "" {
		return nil
	}
	container.IP = info.IP
	container.HostPort = info.HTTPHostPort
	coll := p.Collection()
	defer coll.Close()
	err := coll.Update(bson.M{"id": container.ID}, bson.M{
		"$set": bson.M{"hostport": container.HostPort, "ip": container.IP},
	})
	lockedRoutesRebuildOrEnqueue(container.AppName)
	return err
}
Beispiel #10
0
func (h *ContainerHealer) healContainerIfNeeded(cont container.Container) error {
	if cont.LastSuccessStatusUpdate.IsZero() {
		if !cont.MongoID.Time().Before(time.Now().Add(-h.maxUnresponsiveTime)) {
			return nil
		}
	}
	isRunning, err := h.isRunning(cont)
	if err != nil {
		log.Errorf("Containers healing: couldn't verify running processes in container %s: %s", cont.ID, err.Error())
	}
	if isRunning {
		cont.SetStatus(h.provisioner, provision.StatusStarted, true)
		return nil
	}
	healingCounter, err := healingCountFor("container", cont.ID, consecutiveHealingsTimeframe)
	if err != nil {
		return fmt.Errorf("Containers healing: couldn't verify number of previous healings for %s: %s", cont.ID, err.Error())
	}
	if healingCounter > consecutiveHealingsLimitInTimeframe {
		return fmt.Errorf("Containers healing: number of healings for container %s in the last %d minutes exceeds limit of %d: %d",
			cont.ID, consecutiveHealingsTimeframe/time.Minute, consecutiveHealingsLimitInTimeframe, healingCounter)
	}
	locked := h.locker.Lock(cont.AppName)
	if !locked {
		return fmt.Errorf("Containers healing: unable to heal %s couldn't lock app %s", cont.ID, cont.AppName)
	}
	defer h.locker.Unlock(cont.AppName)
	// Sanity check, now we have a lock, let's find out if the container still exists
	_, err = h.provisioner.GetContainer(cont.ID)
	if err != nil {
		if err == mgo.ErrNotFound {
			return nil
		}
		return fmt.Errorf("Containers healing: unable to heal %s couldn't verify it still exists.", cont.ID)
	}
	log.Errorf("Initiating healing process for container %s, unresponsive since %s.", cont.ID, cont.LastSuccessStatusUpdate)
	evt, err := NewHealingEvent(cont)
	if err != nil {
		return fmt.Errorf("Error trying to insert container healing event, healing aborted: %s", err.Error())
	}
	newCont, healErr := h.healContainer(cont)
	if healErr != nil {
		healErr = fmt.Errorf("Error healing container %s: %s", cont.ID, healErr.Error())
	}
	err = evt.Update(newCont, healErr)
	if err != nil {
		log.Errorf("Error trying to update containers healing event: %s", err.Error())
	}
	return healErr
}
Beispiel #11
0
func (p *dockerProvisioner) Shell(opts provision.ShellOptions) error {
	var (
		c   *container.Container
		err error
	)
	if opts.Unit != "" {
		c, err = p.GetContainer(opts.Unit)
	} else {
		c, err = p.getOneContainerByAppName(opts.App.GetName())
	}
	if err != nil {
		return err
	}
	return c.Shell(p, opts.Conn, opts.Conn, opts.Conn, container.Pty{Width: opts.Width, Height: opts.Height, Term: opts.Term})
}
Beispiel #12
0
func (p *dockerProvisioner) checkContainer(container *container.Container) error {
	if container.Available() {
		info, err := container.NetworkInfo(p)
		if err != nil {
			return err
		}
		if info.HTTPHostPort != container.HostPort || info.IP != container.IP {
			err = p.fixContainer(container, info)
			if err != nil {
				log.Errorf("error on fix container hostport for [container %s]", container.ID)
				return err
			}
		}
	}
	return nil
}
Beispiel #13
0
func (s *S) TestUpdateContainerInDBForward(c *check.C) {
	cont := container.Container{Name: "myName"}
	coll := s.p.Collection()
	defer coll.Close()
	err := coll.Insert(cont)
	c.Assert(err, check.IsNil)
	cont.ID = "myID"
	context := action.FWContext{Previous: cont, Params: []interface{}{runContainerActionsArgs{
		provisioner: s.p,
	}}}
	r, err := updateContainerInDB.Forward(context)
	c.Assert(r, check.FitsTypeOf, container.Container{})
	retrieved, err := s.p.GetContainer(cont.ID)
	c.Assert(err, check.IsNil)
	c.Assert(retrieved.ID, check.Equals, cont.ID)
}
Beispiel #14
0
func (s *S) TestMoveOneContainerNoActionNeeded(c *check.C) {
	p, err := NewFakeDockerProvisioner()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	cont := container.Container{ID: "something"}
	p.SetContainers("localhost", []container.Container{cont})
	errors := make(chan error, 1)
	result := p.MoveOneContainer(cont, "localhost", errors, nil, nil, nil)
	cont.HostAddr = "localhost"
	c.Assert(result, check.DeepEquals, cont)
	select {
	case err := <-errors:
		c.Error(err)
	default:
	}
}
Beispiel #15
0
func (s *S) TestAllContainers(c *check.C) {
	p, err := NewFakeDockerProvisioner()
	c.Assert(err, check.IsNil)
	defer p.Destroy()
	cont1 := container.Container{ID: "cont1"}
	cont2 := container.Container{ID: "cont2"}
	p.SetContainers("localhost", []container.Container{cont1})
	p.SetContainers("remotehost", []container.Container{cont2})
	cont1.HostAddr = "localhost"
	cont2.HostAddr = "remotehost"
	containers := p.AllContainers()
	expected := []container.Container{cont1, cont2}
	if expected[0].HostAddr != containers[0].HostAddr {
		expected = []container.Container{cont2, cont1}
	}
	c.Assert(containers, check.DeepEquals, expected)
}
Beispiel #16
0
func (s *S) TestFollowLogsAndCommitForwardWaitFailure(c *check.C) {
	s.server.PrepareFailure("failed to wait for the container", "/containers/.*/wait")
	defer s.server.ResetFailure("failed to wait for the container")
	err := s.newFakeImage(s.p, "tsuru/python", nil)
	c.Assert(err, check.IsNil)
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	cont := container.Container{AppName: "mightyapp"}
	err = cont.Create(&container.CreateArgs{
		App:         app,
		ImageID:     "tsuru/python",
		Commands:    []string{"foo"},
		Provisioner: s.p,
	})
	c.Assert(err, check.IsNil)
	err = cont.Start(&container.StartArgs{
		Provisioner: s.p,
		App:         app,
	})
	c.Assert(err, check.IsNil)
	err = cont.Stop(s.p)
	c.Assert(err, check.IsNil)
	buf := safe.NewBuffer(nil)
	args := runContainerActionsArgs{writer: buf, provisioner: s.p}
	context := action.FWContext{Params: []interface{}{args}, Previous: cont}
	imageId, err := followLogsAndCommit.Forward(context)
	c.Assert(err, check.ErrorMatches, `.*failed to wait for the container\n$`)
	c.Assert(imageId, check.IsNil)
}
Beispiel #17
0
func (s *S) TestAddNewRouteForwardDoesNotAddWhenHostPortIsEmpty(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "", HostAddr: "addr1", HostPort: ""}
	cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "", HostAddr: "addr2", HostPort: "4321"}
	defer cont.Remove(s.p)
	defer cont2.Remove(s.p)
	args := changeUnitsPipelineArgs{
		app:         app,
		provisioner: s.p,
	}
	prevContainers := []container.Container{cont, cont2}
	context := action.FWContext{Previous: prevContainers, Params: []interface{}{args}}
	_, err := addNewRoutes.Forward(context)
	c.Assert(err, check.Equals, nil)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
	c.Assert(hasRoute, check.Equals, false)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
	c.Assert(hasRoute, check.Equals, true)
}
Beispiel #18
0
func (s *S) TestFollowLogsAndCommitForwardNonZeroStatus(c *check.C) {
	err := s.newFakeImage(s.p, "tsuru/python", nil)
	c.Assert(err, check.IsNil)
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	cont := container.Container{AppName: "mightyapp"}
	err = cont.Create(&container.CreateArgs{
		App:         app,
		ImageID:     "tsuru/python",
		Commands:    []string{"foo"},
		Provisioner: s.p,
	})
	c.Assert(err, check.IsNil)
	err = s.server.MutateContainer(cont.ID, docker.State{ExitCode: 1})
	c.Assert(err, check.IsNil)
	buf := safe.NewBuffer(nil)
	args := runContainerActionsArgs{writer: buf, provisioner: s.p}
	context := action.FWContext{Params: []interface{}{args}, Previous: cont}
	imageId, err := followLogsAndCommit.Forward(context)
	c.Assert(err, check.NotNil)
	c.Assert(err.Error(), check.Equals, "Exit status 1")
	c.Assert(imageId, check.IsNil)
}
Beispiel #19
0
func (s *S) TestRemoveOldRoutesForwardNoImageData(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	err := appendAppImageName(app.GetName(), "img1")
	c.Assert(err, check.IsNil)
	err = pullAppImageNames(app.GetName(), []string{"img1"})
	c.Assert(err, check.IsNil)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "", HostAddr: "127.0.0.1", HostPort: ""}
	args := changeUnitsPipelineArgs{
		app:         app,
		toRemove:    []container.Container{cont1},
		provisioner: s.p,
	}
	context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
	r, err := removeOldRoutes.Forward(context)
	c.Assert(err, check.IsNil)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
	c.Assert(hasRoute, check.Equals, false)
	containers := r.([]container.Container)
	c.Assert(containers, check.DeepEquals, []container.Container{})
	c.Assert(args.toRemove[0].Routable, check.Equals, false)
}
Beispiel #20
0
func (s *S) TestFixContainer(c *check.C) {
	cleanup, server, p := startDocker("9999")
	defer cleanup()
	coll := p.Collection()
	defer coll.Close()
	cont := container.Container{
		ID:       "9930c24f1c4x",
		AppName:  "makea",
		Type:     "python",
		Status:   provision.StatusStarted.String(),
		IP:       "127.0.0.4",
		HostPort: "9025",
		HostAddr: "127.0.0.1",
	}
	err := coll.Insert(cont)
	c.Assert(err, check.IsNil)
	defer coll.RemoveAll(bson.M{"appname": cont.AppName})
	err = s.storage.Apps().Insert(&app.App{Name: cont.AppName})
	c.Assert(err, check.IsNil)
	appInstance := provisiontest.NewFakeApp(cont.AppName, cont.Type, 0)
	defer p.Destroy(appInstance)
	p.Provision(appInstance)
	var storage cluster.MapStorage
	storage.StoreContainer(cont.ID, server.URL)
	p.cluster, err = cluster.New(nil, &storage,
		cluster.Node{Address: server.URL},
	)
	c.Assert(err, check.IsNil)
	info, err := cont.NetworkInfo(p)
	c.Assert(err, check.IsNil)
	err = p.fixContainer(&cont, info)
	c.Assert(err, check.IsNil)
	conta, err := p.GetContainer("9930c24f1c4x")
	c.Assert(err, check.IsNil)
	c.Assert(conta.IP, check.Equals, "127.0.0.9")
	c.Assert(conta.HostPort, check.Equals, "9999")
}
Beispiel #21
0
func (p *FakeDockerProvisioner) moveOneContainer(cont container.Container, toHost string) (container.Container, error) {
	cont, index, err := p.findContainer(cont.ID)
	if err != nil {
		return cont, err
	}
	if cont.HostAddr == toHost {
		return cont, nil
	}
	if toHost == "" {
		for host := range p.containers {
			if host != cont.HostAddr {
				toHost = host
				break
			}
		}
	}
	originHost := cont.HostAddr
	moving := ContainerMoving{
		ContainerID: cont.ID,
		HostFrom:    originHost,
		HostTo:      toHost,
	}
	p.movings = append(p.movings, moving)
	if toHost == "" {
		cont.ID += "-recreated"
		p.containers[originHost][index] = cont
		return cont, nil
	}
	cont.HostAddr = toHost
	cont.ID += "-moved"
	last := len(p.containers[originHost]) - 1
	p.containers[originHost][index] = p.containers[originHost][last]
	p.containers[originHost] = p.containers[originHost][:last]
	p.containers[cont.HostAddr] = append(p.containers[cont.HostAddr], cont)
	return cont, nil
}
Beispiel #22
0
func (s *S) TestGetRemovableContainer(c *check.C) {
	a1 := app.App{Name: "impius", Teams: []string{"tsuruteam", "nodockerforme"}, Pool: "pool1"}
	cont1 := container.Container{ID: "1", Name: "impius1", AppName: a1.Name, ProcessName: "web"}
	cont2 := container.Container{ID: "2", Name: "mirror1", AppName: a1.Name, ProcessName: "worker"}
	a2 := app.App{Name: "notimpius", Teams: []string{"tsuruteam", "nodockerforme"}, Pool: "pool1"}
	cont3 := container.Container{ID: "3", Name: "dedication1", AppName: a2.Name, ProcessName: "web"}
	cont4 := container.Container{ID: "4", Name: "dedication2", AppName: a2.Name, ProcessName: "worker"}
	err := s.storage.Apps().Insert(a1)
	c.Assert(err, check.IsNil)
	err = s.storage.Apps().Insert(a2)
	c.Assert(err, check.IsNil)
	defer s.storage.Apps().RemoveAll(bson.M{"name": a1.Name})
	defer s.storage.Apps().RemoveAll(bson.M{"name": a2.Name})
	p := provision.Pool{Name: "pool1", Teams: []string{
		"tsuruteam",
		"nodockerforme",
	}}
	o := provision.AddPoolOptions{Name: p.Name}
	err = provision.AddPool(o)
	c.Assert(err, check.IsNil)
	err = provision.AddTeamsToPool(p.Name, p.Teams)
	defer provision.RemovePool(p.Name)
	contColl := s.p.Collection()
	defer contColl.Close()
	err = contColl.Insert(
		cont1, cont2, cont3, cont4,
	)
	c.Assert(err, check.IsNil)
	defer contColl.RemoveAll(bson.M{"name": bson.M{"$in": []string{cont1.Name, cont2.Name, cont3.Name, cont4.Name}}})
	scheduler := segregatedScheduler{provisioner: s.p}
	clusterInstance, err := cluster.New(&scheduler, &cluster.MapStorage{})
	s.p.cluster = clusterInstance
	c.Assert(err, check.IsNil)
	server1, err := testing.NewServer("127.0.0.1:0", nil, nil)
	c.Assert(err, check.IsNil)
	defer server1.Stop()
	server2, err := testing.NewServer("127.0.0.1:0", nil, nil)
	c.Assert(err, check.IsNil)
	defer server2.Stop()
	localURL := strings.Replace(server2.URL(), "127.0.0.1", "localhost", -1)
	err = clusterInstance.Register(cluster.Node{
		Address:  server1.URL(),
		Metadata: map[string]string{"pool": "pool1"},
	})
	c.Assert(err, check.IsNil)
	err = clusterInstance.Register(cluster.Node{
		Address:  localURL,
		Metadata: map[string]string{"pool": "pool1"},
	})
	c.Assert(err, check.IsNil)
	opts := docker.CreateContainerOptions{Name: cont1.Name}
	_, err = scheduler.Schedule(clusterInstance, opts, []string{a1.Name, cont1.ProcessName})
	c.Assert(err, check.IsNil)
	opts = docker.CreateContainerOptions{Name: cont2.Name}
	_, err = scheduler.Schedule(clusterInstance, opts, []string{a1.Name, cont2.ProcessName})
	c.Assert(err, check.IsNil)
	opts = docker.CreateContainerOptions{Name: cont3.Name}
	_, err = scheduler.Schedule(clusterInstance, opts, []string{a2.Name, cont3.ProcessName})
	c.Assert(err, check.IsNil)
	opts = docker.CreateContainerOptions{Name: cont4.Name}
	_, err = scheduler.Schedule(clusterInstance, opts, []string{a2.Name, cont4.ProcessName})
	c.Assert(err, check.IsNil)
	cont, err := scheduler.GetRemovableContainer(a1.Name, "web")
	c.Assert(err, check.IsNil)
	c.Assert(cont, check.Equals, cont1.ID)
	err = cont1.Remove(s.p)
	c.Assert(err, check.IsNil)
	_, err = scheduler.GetRemovableContainer(a1.Name, "web")
	c.Assert(err, check.NotNil)
}
Beispiel #23
0
func runHealthcheck(cont *container.Container, w io.Writer) error {
	yamlData, err := getImageTsuruYamlData(cont.Image)
	if err != nil {
		return err
	}
	path := yamlData.Healthcheck.Path
	method := yamlData.Healthcheck.Method
	match := yamlData.Healthcheck.Match
	status := yamlData.Healthcheck.Status
	allowedFailures := yamlData.Healthcheck.AllowedFailures
	if path == "" {
		return nil
	}
	path = strings.TrimSpace(strings.TrimLeft(path, "/"))
	if method == "" {
		method = "get"
	}
	method = strings.ToUpper(method)
	if status == 0 && match == "" {
		status = 200
	}
	var matchRE *regexp.Regexp
	if match != "" {
		match = "(?s)" + match
		matchRE, err = regexp.Compile(match)
		if err != nil {
			return err
		}
	}
	maxWaitTime, _ := config.GetInt("docker:healthcheck:max-time")
	if maxWaitTime == 0 {
		maxWaitTime = 120
	}
	maxWaitTime = maxWaitTime * int(time.Second)
	sleepTime := 3 * time.Second
	startedTime := time.Now()
	url := fmt.Sprintf("http://%s:%s/%s", cont.HostAddr, cont.HostPort, path)
	for {
		var lastError error = nil
		req, err := http.NewRequest(method, url, nil)
		if err != nil {
			return err
		}
		rsp, err := timeoutHttpClient.Do(req)
		if err != nil {
			lastError = fmt.Errorf("healthcheck fail(%s): %s", cont.ShortID(), err.Error())
		} else {
			defer rsp.Body.Close()
			if status != 0 && rsp.StatusCode != status {
				lastError = fmt.Errorf("healthcheck fail(%s): wrong status code, expected %d, got: %d", cont.ShortID(), status, rsp.StatusCode)
			} else if matchRE != nil {
				result, err := ioutil.ReadAll(rsp.Body)

				if err != nil {
					lastError = err
				}
				if !matchRE.Match(result) {
					lastError = fmt.Errorf("healthcheck fail(%s): unexpected result, expected %q, got: %s", cont.ShortID(), match, string(result))
				}
			}
			if lastError != nil {
				if allowedFailures == 0 {
					return lastError
				}
				allowedFailures--
			}
		}
		if lastError == nil {
			fmt.Fprintf(w, " ---> healthcheck successful(%s)\n", cont.ShortID())
			return nil
		}
		if time.Now().Sub(startedTime) > time.Duration(maxWaitTime) {
			return lastError
		}
		fmt.Fprintf(w, " ---> %s. Trying again in %s\n", lastError.Error(), sleepTime)
		time.Sleep(sleepTime)
	}
}
Beispiel #24
0
func (s *S) removeTestContainer(c *container.Container) error {
	routertest.FakeRouter.RemoveBackend(c.AppName)
	return c.Remove(s.p)
}
Beispiel #25
0
func (s *S) newContainer(opts *newContainerOpts, p *dockerProvisioner) (*container.Container, error) {
	container := container.Container{
		ID:          "id",
		IP:          "10.10.10.10",
		HostPort:    "3333",
		HostAddr:    "127.0.0.1",
		ProcessName: "web",
		ExposedPort: "8888/tcp",
	}
	if p == nil {
		p = s.p
	}
	image := "tsuru/python:latest"
	var customData map[string]interface{}
	if opts != nil {
		if opts.Image != "" {
			image = opts.Image
		}
		container.Status = opts.Status
		container.AppName = opts.AppName
		container.ProcessName = opts.ProcessName
		customData = opts.ImageCustomData
		if opts.Provisioner != nil {
			p = opts.Provisioner
		}
	}
	err := s.newFakeImage(p, image, customData)
	if err != nil {
		return nil, err
	}
	if container.AppName == "" {
		container.AppName = "container"
	}
	routertest.FakeRouter.AddBackend(container.AppName)
	routertest.FakeRouter.AddRoute(container.AppName, container.Address())
	ports := map[docker.Port]struct{}{
		docker.Port(s.port + "/tcp"): {},
	}
	config := docker.Config{
		Image:        image,
		Cmd:          []string{"ps"},
		ExposedPorts: ports,
	}
	createOptions := docker.CreateContainerOptions{Config: &config}
	createOptions.Name = randomString()
	_, c, err := p.Cluster().CreateContainer(createOptions)
	if err != nil {
		return nil, err
	}
	container.ID = c.ID
	container.Image = image
	container.Name = createOptions.Name
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	err = conn.Collection(s.collName).Insert(&container)
	if err != nil {
		return nil, err
	}
	imageId, err := appCurrentImageName(container.AppName)
	if err != nil {
		return nil, err
	}
	err = s.newFakeImage(p, imageId, nil)
	if err != nil {
		return nil, err
	}
	return &container, nil
}
Beispiel #26
0
func (s *S) TestRemoveOldRoutesBackward(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web"}
	cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web"}
	defer cont.Remove(s.p)
	defer cont2.Remove(s.p)
	cont.Routable = true
	cont2.Routable = true
	args := changeUnitsPipelineArgs{
		app:         app,
		toRemove:    []container.Container{cont, cont2},
		provisioner: s.p,
	}
	context := action.BWContext{Params: []interface{}{args}}
	removeOldRoutes.Backward(context)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
	c.Assert(hasRoute, check.Equals, true)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
	c.Assert(hasRoute, check.Equals, true)
}
Beispiel #27
0
func (s *S) TestRemoveOldRoutesForwardFailInMiddle(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr1"}
	cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr2"}
	defer cont.Remove(s.p)
	defer cont2.Remove(s.p)
	err := routertest.FakeRouter.AddRoute(app.GetName(), cont.Address())
	c.Assert(err, check.IsNil)
	err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
	c.Assert(err, check.IsNil)
	routertest.FakeRouter.FailForIp(cont2.Address().String())
	args := changeUnitsPipelineArgs{
		app:         app,
		toRemove:    []container.Container{cont, cont2},
		provisioner: s.p,
	}
	context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
	_, err = removeOldRoutes.Forward(context)
	c.Assert(err, check.Equals, routertest.ErrForcedFailure)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
	c.Assert(hasRoute, check.Equals, true)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
	c.Assert(hasRoute, check.Equals, true)
	c.Assert(args.toRemove[0].Routable, check.Equals, true)
	c.Assert(args.toRemove[1].Routable, check.Equals, false)
}
Beispiel #28
0
func (s *S) TestRemoveOldRoutesForward(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.1", HostPort: "1234"}
	cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.2", HostPort: "4321"}
	cont3 := container.Container{ID: "ble-3", AppName: app.GetName(), ProcessName: "worker", HostAddr: "127.0.0.3", HostPort: "8080"}
	defer cont1.Remove(s.p)
	defer cont2.Remove(s.p)
	defer cont3.Remove(s.p)
	err := routertest.FakeRouter.AddRoute(app.GetName(), cont1.Address())
	c.Assert(err, check.IsNil)
	err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
	c.Assert(err, check.IsNil)
	args := changeUnitsPipelineArgs{
		app:         app,
		toRemove:    []container.Container{cont1, cont2, cont3},
		provisioner: s.p,
	}
	context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
	r, err := removeOldRoutes.Forward(context)
	c.Assert(err, check.IsNil)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
	c.Assert(hasRoute, check.Equals, false)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
	c.Assert(hasRoute, check.Equals, false)
	containers := r.([]container.Container)
	c.Assert(containers, check.DeepEquals, []container.Container{})
	c.Assert(args.toRemove[0].Routable, check.Equals, true)
	c.Assert(args.toRemove[1].Routable, check.Equals, true)
	c.Assert(args.toRemove[2].Routable, check.Equals, false)
}
Beispiel #29
0
func (s *S) TestAddNewRouteBackward(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	cont1 := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.1", HostPort: "1234"}
	cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "127.0.0.2", HostPort: "4321"}
	cont3 := container.Container{ID: "ble-3", AppName: app.GetName(), ProcessName: "worker", HostAddr: "127.0.0.3", HostPort: "8080"}
	defer cont1.Remove(s.p)
	defer cont2.Remove(s.p)
	defer cont3.Remove(s.p)
	err := routertest.FakeRouter.AddRoute(app.GetName(), cont1.Address())
	c.Assert(err, check.IsNil)
	err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
	c.Assert(err, check.IsNil)
	args := changeUnitsPipelineArgs{
		app:         app,
		provisioner: s.p,
	}
	cont1.Routable = true
	cont2.Routable = true
	context := action.BWContext{FWResult: []container.Container{cont1, cont2, cont3}, Params: []interface{}{args}}
	addNewRoutes.Backward(context)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont1.Address().String())
	c.Assert(hasRoute, check.Equals, false)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
	c.Assert(hasRoute, check.Equals, false)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont3.Address().String())
	c.Assert(hasRoute, check.Equals, false)
}
Beispiel #30
0
func (s *S) TestRemoveOldRoutesForwardFailInMiddle(c *check.C) {
	app := provisiontest.NewFakeApp("myapp", "python", 1)
	imageName := "tsuru/app-" + app.GetName()
	customData := map[string]interface{}{
		"processes": map[string]interface{}{
			"web":    "python myapi.py",
			"worker": "tail -f /dev/null",
		},
	}
	err := saveImageCustomData(imageName, customData)
	c.Assert(err, check.IsNil)
	routertest.FakeRouter.AddBackend(app.GetName())
	defer routertest.FakeRouter.RemoveBackend(app.GetName())
	cont := container.Container{ID: "ble-1", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr1", HostPort: "1234"}
	cont2 := container.Container{ID: "ble-2", AppName: app.GetName(), ProcessName: "web", HostAddr: "addr2", HostPort: "1234"}
	defer cont.Remove(s.p)
	defer cont2.Remove(s.p)
	err = routertest.FakeRouter.AddRoute(app.GetName(), cont.Address())
	c.Assert(err, check.IsNil)
	err = routertest.FakeRouter.AddRoute(app.GetName(), cont2.Address())
	c.Assert(err, check.IsNil)
	routertest.FakeRouter.FailForIp(cont2.Address().String())
	args := changeUnitsPipelineArgs{
		app:         app,
		toRemove:    []container.Container{cont, cont2},
		provisioner: s.p,
	}
	context := action.FWContext{Previous: []container.Container{}, Params: []interface{}{args}}
	_, err = removeOldRoutes.Forward(context)
	c.Assert(err, check.Equals, routertest.ErrForcedFailure)
	hasRoute := routertest.FakeRouter.HasRoute(app.GetName(), cont.Address().String())
	c.Assert(hasRoute, check.Equals, true)
	hasRoute = routertest.FakeRouter.HasRoute(app.GetName(), cont2.Address().String())
	c.Assert(hasRoute, check.Equals, true)
	c.Assert(args.toRemove[0].Routable, check.Equals, true)
	c.Assert(args.toRemove[1].Routable, check.Equals, true)
}