Beispiel #1
0
func (a *autoScaleConfig) rebalanceIfNeeded(evt *event.Event, pool string, nodes []*cluster.Node, sResult *scalerResult) error {
	if len(sResult.ToRemove) > 0 {
		return nil
	}
	if sResult.ToAdd > 0 {
		sResult.ToRebalance = true
	}
	rebalanceFilter := map[string]string{poolMetadataName: pool}
	if !sResult.ToRebalance {
		// No action yet, check if we need rebalance
		_, gap, err := a.provisioner.containerGapInNodes(nodes)
		if err != nil {
			return errors.Wrapf(err, "unable to obtain container gap in nodes")
		}
		buf := safe.NewBuffer(nil)
		dryProvisioner, err := a.provisioner.rebalanceContainersByFilter(buf, nil, rebalanceFilter, true)
		if err != nil {
			return errors.Wrapf(err, "unable to run dry rebalance to check if rebalance is needed. log: %s", buf.String())
		}
		if dryProvisioner == nil {
			return nil
		}
		_, gapAfter, err := dryProvisioner.containerGapInNodes(nodes)
		if err != nil {
			return errors.Wrap(err, "couldn't find containers from rebalanced nodes")
		}
		if math.Abs((float64)(gap-gapAfter)) > 2.0 {
			sResult.ToRebalance = true
			if sResult.Reason == "" {
				sResult.Reason = fmt.Sprintf("gap is %d, after rebalance gap will be %d", gap, gapAfter)
			}
		}
	}
	if sResult.ToRebalance {
		evt.Logf("running rebalance, for %q: %#v", pool, sResult)
		buf := safe.NewBuffer(nil)
		writer := io.MultiWriter(buf, evt)
		_, err := a.provisioner.rebalanceContainersByFilter(writer, nil, rebalanceFilter, false)
		if err != nil {
			return errors.Wrapf(err, "unable to rebalance containers. log: %s", buf.String())
		}
	}
	return nil
}
Beispiel #2
0
func (a *autoScaleConfig) removeMultipleNodes(evt *event.Event, chosenNodes []cluster.Node) error {
	nodeAddrs := make([]string, len(chosenNodes))
	nodeHosts := make([]string, len(chosenNodes))
	for i, node := range chosenNodes {
		_, hasIaas := node.Metadata["iaas"]
		if !hasIaas {
			return errors.Errorf("no IaaS information in node (%s) metadata: %#v", node.Address, node.Metadata)
		}
		nodeAddrs[i] = node.Address
		nodeHosts[i] = net.URLToHost(node.Address)
	}
	err := a.provisioner.Cluster().UnregisterNodes(nodeAddrs...)
	if err != nil {
		return errors.Wrapf(err, "unable to unregister nodes (%s) for removal", strings.Join(nodeAddrs, ", "))
	}
	buf := safe.NewBuffer(nil)
	err = a.provisioner.moveContainersFromHosts(nodeHosts, "", buf)
	if err != nil {
		for _, node := range chosenNodes {
			a.provisioner.Cluster().Register(node)
		}
		return errors.Wrapf(err, "unable to move containers from nodes (%s). log: %s", strings.Join(nodeAddrs, ", "), buf.String())
	}
	wg := sync.WaitGroup{}
	for i := range chosenNodes {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			node := chosenNodes[i]
			m, err := iaas.FindMachineByIdOrAddress(node.Metadata["iaas-id"], net.URLToHost(node.Address))
			if err != nil {
				evt.Logf("unable to find machine for removal in iaas: %s", err)
				return
			}
			err = m.Destroy()
			if err != nil {
				evt.Logf("unable to destroy machine in IaaS: %s", err)
			}
		}(i)
	}
	wg.Wait()
	return nil
}
Beispiel #3
0
func (a *autoScaleConfig) addNode(evt *event.Event, modelNodes []*cluster.Node) (*cluster.Node, error) {
	metadata, err := chooseMetadataFromNodes(modelNodes)
	if err != nil {
		return nil, err
	}
	_, hasIaas := metadata["iaas"]
	if !hasIaas {
		return nil, errors.Errorf("no IaaS information in nodes metadata: %#v", metadata)
	}
	machine, err := iaas.CreateMachineForIaaS(metadata["iaas"], metadata)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create machine")
	}
	newAddr := machine.FormatNodeAddress()
	evt.Logf("new machine created: %s - Waiting for docker to start...", newAddr)
	createOpts := provision.AddNodeOptions{
		Address:    newAddr,
		Metadata:   metadata,
		WaitTO:     a.WaitTimeNewMachine,
		CaCert:     machine.CaCert,
		ClientCert: machine.ClientCert,
		ClientKey:  machine.ClientKey,
	}
	err = a.provisioner.AddNode(createOpts)
	if err != nil {
		machine.Destroy()
		a.provisioner.Cluster().Unregister(newAddr)
		return nil, errors.Wrapf(err, "error adding new node %s", newAddr)
	}
	createdNode, err := a.provisioner.Cluster().GetNode(newAddr)
	if err != nil {
		machine.Destroy()
		a.provisioner.Cluster().Unregister(newAddr)
		return nil, errors.Wrapf(err, "error retrieving new node %s", newAddr)
	}
	evt.Logf("new machine created: %s - started!", newAddr)
	return &createdNode, nil
}