func (u *Updater) rollbackUpdate(ctx context.Context, serviceID, message string) {
log.G(ctx).Debugf("starting rollback of service %s", serviceID)
var service *api.Service
err := u.store.Update(func(tx store.Tx) error {
service = store.GetService(tx, serviceID)
if service == nil {
return nil
}
if service.UpdateStatus == nil {
// The service was updated since we started this update
return nil
}
service.UpdateStatus.State = api.UpdateStatus_ROLLBACK_STARTED
service.UpdateStatus.Message = message
if service.PreviousSpec == nil {
return errors.New("cannot roll back service because no previous spec is available")
}
service.Spec = *service.PreviousSpec
service.PreviousSpec = nil
return store.UpdateService(tx, service)
})
if err != nil {
log.G(ctx).WithError(err).Errorf("failed to start rollback of service %s", serviceID)
return
}
}
// UpdateService updates a Service referenced by ServiceID with the given ServiceSpec.
// - Returns `NotFound` if the Service is not found.
// - Returns `InvalidArgument` if the ServiceSpec is malformed.
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) {
if request.ServiceID == "" || request.ServiceVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateServiceSpec(request.Spec); err != nil {
return nil, err
}
var service *api.Service
err := s.store.Update(func(tx store.Tx) error {
service = store.GetService(tx, request.ServiceID)
if service == nil {
return nil
}
// temporary disable network update
if request.Spec != nil && !reflect.DeepEqual(request.Spec.Networks, service.Spec.Networks) {
return errNetworkUpdateNotSupported
}
service.Meta.Version = *request.ServiceVersion
service.Spec = *request.Spec.Copy()
return store.UpdateService(tx, service)
})
if err != nil {
return nil, err
}
if service == nil {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
return &api.UpdateServiceResponse{
Service: service,
}, nil
}
// UpdateService updates a Service referenced by ServiceID with the given ServiceSpec.
// - Returns `NotFound` if the Service is not found.
// - Returns `InvalidArgument` if the ServiceSpec is malformed.
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) {
if request.ServiceID == "" || request.ServiceVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateServiceSpec(request.Spec); err != nil {
return nil, err
}
var service *api.Service
err := s.store.Update(func(tx store.Tx) error {
service = store.GetService(tx, request.ServiceID)
if service == nil {
return nil
}
service.Meta.Version = *request.ServiceVersion
service.Spec = *request.Spec.Copy()
return store.UpdateService(tx, service)
})
if err != nil {
return nil, err
}
if service == nil {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
return &api.UpdateServiceResponse{
Service: service,
}, nil
}
// UpdateService updates a Service referenced by ServiceID with the given ServiceSpec.
// - Returns `NotFound` if the Service is not found.
// - Returns `InvalidArgument` if the ServiceSpec is malformed.
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) {
if request.ServiceID == "" || request.ServiceVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateServiceSpec(request.Spec); err != nil {
return nil, err
}
var service *api.Service
s.store.View(func(tx store.ReadTx) {
service = store.GetService(tx, request.ServiceID)
})
if service == nil {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
if request.Spec.Endpoint != nil && !reflect.DeepEqual(request.Spec.Endpoint, service.Spec.Endpoint) {
if err := s.checkPortConflicts(request.Spec, request.ServiceID); err != nil {
return nil, err
}
}
err := s.store.Update(func(tx store.Tx) error {
service = store.GetService(tx, request.ServiceID)
if service == nil {
return nil
}
// temporary disable network update
if request.Spec != nil && !reflect.DeepEqual(request.Spec.Networks, service.Spec.Networks) {
return errNetworkUpdateNotSupported
}
// orchestrator is designed to be stateless, so it should not deal
// with service mode change (comparing current config with previous config).
// proper way to change service mode is to delete and re-add.
if request.Spec != nil && reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) {
return errModeChangeNotAllowed
}
service.Meta.Version = *request.ServiceVersion
service.Spec = *request.Spec.Copy()
// Reset update status
service.UpdateStatus = nil
return store.UpdateService(tx, service)
})
if err != nil {
return nil, err
}
if service == nil {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
return &api.UpdateServiceResponse{
Service: service,
}, nil
}
// NewUpdater creates a new Updater.
func NewUpdater(store *store.MemoryStore, restartSupervisor *RestartSupervisor, cluster *api.Cluster, newService *api.Service) *Updater {
return &Updater{
store: store,
watchQueue: store.WatchQueue(),
restarts: restartSupervisor,
cluster: cluster.Copy(),
newService: newService.Copy(),
stopChan: make(chan struct{}),
doneChan: make(chan struct{}),
}
}
func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s *api.Service) error {
if s.Spec.Endpoint != nil {
if s.Endpoint == nil {
s.Endpoint = &api.Endpoint{
Spec: s.Spec.Endpoint.Copy(),
}
}
// The service is trying to expose ports to the external
// world. Automatically attach the service to the ingress
// network only if it is not already done.
if len(s.Spec.Endpoint.Ports) != 0 {
var found bool
for _, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == ingressNetwork.ID {
found = true
break
}
}
if !found {
s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs,
&api.Endpoint_VirtualIP{NetworkID: ingressNetwork.ID})
}
}
}
if err := nc.nwkAllocator.ServiceAllocate(s); err != nil {
nc.unallocatedServices[s.ID] = s
return err
}
if err := a.store.Update(func(tx store.Tx) error {
for {
err := store.UpdateService(tx, s)
if err != nil && err != store.ErrSequenceConflict {
return fmt.Errorf("failed updating state in store transaction for service %s: %v", s.ID, err)
}
if err == store.ErrSequenceConflict {
storeService := store.GetService(tx, s.ID)
storeService.Endpoint = s.Endpoint
s = storeService
continue
}
break
}
return nil
}); err != nil {
if err := nc.nwkAllocator.ServiceDeallocate(s); err != nil {
log.G(ctx).WithError(err).Errorf("failed rolling back allocation of service %s: %v", s.ID, err)
}
return err
}
return nil
}
func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
nc := a.netCtx
if s.Spec.Endpoint != nil {
// service has user-defined endpoint
if s.Endpoint == nil {
// service currently has no allocated endpoint, need allocated.
s.Endpoint = &api.Endpoint{
Spec: s.Spec.Endpoint.Copy(),
}
}
// The service is trying to expose ports to the external
// world. Automatically attach the service to the ingress
// network only if it is not already done.
if isIngressNetworkNeeded(s) {
var found bool
for _, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == nc.ingressNetwork.ID {
found = true
break
}
}
if !found {
s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs,
&api.Endpoint_VirtualIP{NetworkID: nc.ingressNetwork.ID})
}
}
} else if s.Endpoint != nil {
// service has no user-defined endpoints while has already allocated network resources,
// need deallocated.
if err := nc.nwkAllocator.ServiceDeallocate(s); err != nil {
return err
}
}
if err := nc.nwkAllocator.ServiceAllocate(s); err != nil {
nc.unallocatedServices[s.ID] = s
return err
}
// If the service doesn't expose ports any more and if we have
// any lingering virtual IP references for ingress network
// clean them up here.
if !isIngressNetworkNeeded(s) {
if s.Endpoint != nil {
for i, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == nc.ingressNetwork.ID {
n := len(s.Endpoint.VirtualIPs)
s.Endpoint.VirtualIPs[i], s.Endpoint.VirtualIPs[n-1] = s.Endpoint.VirtualIPs[n-1], nil
s.Endpoint.VirtualIPs = s.Endpoint.VirtualIPs[:n-1]
break
}
}
}
}
return nil
}
// ServiceAllocate allocates all the network resources such as virtual
// IP and ports needed by the service.
func (na *NetworkAllocator) ServiceAllocate(s *api.Service) (err error) {
if err = na.portAllocator.serviceAllocatePorts(s); err != nil {
return
}
defer func() {
if err != nil {
na.ServiceDeallocate(s)
}
}()
// If ResolutionMode is DNSRR do not try allocating VIPs.
if s.Spec.Endpoint != nil && s.Spec.Endpoint.Mode == api.ResolutionModeDNSRoundRobin {
return
}
if s.Endpoint == nil {
s.Endpoint = &api.Endpoint{
Spec: s.Spec.Endpoint.Copy(),
}
}
// First allocate VIPs for all the pre-populated endpoint attachments
for _, eAttach := range s.Endpoint.VirtualIPs {
if err = na.allocateVIP(eAttach); err != nil {
return
}
}
outer:
for _, nAttach := range s.Spec.Networks {
for _, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == nAttach.Target {
continue outer
}
}
vip := &api.Endpoint_VirtualIP{NetworkID: nAttach.Target}
if err = na.allocateVIP(vip); err != nil {
return
}
s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs, vip)
}
s.Endpoint.Spec = s.Spec.Endpoint.Copy()
na.services[s.ID] = struct{}{}
return
}
func (pa *portAllocator) serviceAllocatePorts(s *api.Service) (err error) {
if s.Spec.Endpoint == nil {
return nil
}
// We might have previous allocations which we want to stick
// to if possible. So instead of strictly going by port
// configs in the Spec reconcile the list of port configs from
// both the Spec and runtime state.
portConfigs := reconcilePortConfigs(s)
// Port configuration might have changed. Cleanup all old allocations first.
pa.serviceDeallocatePorts(s)
defer func() {
if err != nil {
// Free all the ports allocated so far which
// should be present in s.Endpoints.ExposedPorts
pa.serviceDeallocatePorts(s)
}
}()
for _, portConfig := range portConfigs {
// Make a copy of port config to create runtime state
portState := portConfig.Copy()
// Do an actual allocation only if the PublishMode is Ingress
if portConfig.PublishMode == api.PublishModeIngress {
if err = pa.portSpaces[portState.Protocol].allocate(portState); err != nil {
return
}
}
if s.Endpoint == nil {
s.Endpoint = &api.Endpoint{}
}
s.Endpoint.Ports = append(s.Endpoint.Ports, portState)
}
return nil
}
// This function prepares the service object for being updated when the change regards
// the published ports in host mode: It resets the runtime state ports (s.Endpoint.Ports)
// to the current ingress mode runtime state ports plus the newly configured publish mode ports,
// so that the service allocation invoked on this new service object will trigger the deallocation
// of any old publish mode port and allocation of any new one.
func updatePortsInHostPublishMode(s *api.Service) {
if s.Endpoint != nil {
var portConfigs []*api.PortConfig
for _, portConfig := range s.Endpoint.Ports {
if portConfig.PublishMode == api.PublishModeIngress {
portConfigs = append(portConfigs, portConfig)
}
}
s.Endpoint.Ports = portConfigs
}
if s.Spec.Endpoint != nil {
if s.Endpoint == nil {
s.Endpoint = &api.Endpoint{}
}
for _, portConfig := range s.Spec.Endpoint.Ports {
if portConfig.PublishMode == api.PublishModeIngress {
continue
}
s.Endpoint.Ports = append(s.Endpoint.Ports, portConfig.Copy())
}
s.Endpoint.Spec = s.Spec.Endpoint.Copy()
}
}
// ServiceAllocate allocates all the network resources such as virtual
// IP and ports needed by the service.
func (na *NetworkAllocator) ServiceAllocate(s *api.Service) (err error) {
if err = na.portAllocator.serviceAllocatePorts(s); err != nil {
return
}
defer func() {
if err != nil {
na.ServiceDeallocate(s)
}
}()
if s.Endpoint == nil {
s.Endpoint = &api.Endpoint{}
}
s.Endpoint.Spec = s.Spec.Endpoint.Copy()
// If ResolutionMode is DNSRR do not try allocating VIPs, but
// free any VIP from previous state.
if s.Spec.Endpoint != nil && s.Spec.Endpoint.Mode == api.ResolutionModeDNSRoundRobin {
if s.Endpoint != nil {
for _, vip := range s.Endpoint.VirtualIPs {
if err := na.deallocateVIP(vip); err != nil {
// don't bail here, deallocate as many as possible.
log.L.WithError(err).
WithField("vip.network", vip.NetworkID).
WithField("vip.addr", vip.Addr).Error("error deallocating vip")
}
}
s.Endpoint.VirtualIPs = nil
}
delete(na.services, s.ID)
return
}
// First allocate VIPs for all the pre-populated endpoint attachments
for _, eAttach := range s.Endpoint.VirtualIPs {
if err = na.allocateVIP(eAttach); err != nil {
return
}
}
// Always prefer NetworkAttachmentConfig in the TaskSpec
specNetworks := s.Spec.Task.Networks
if len(specNetworks) == 0 && s != nil && len(s.Spec.Networks) != 0 {
specNetworks = s.Spec.Networks
}
outer:
for _, nAttach := range specNetworks {
for _, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == nAttach.Target {
continue outer
}
}
vip := &api.Endpoint_VirtualIP{NetworkID: nAttach.Target}
if err = na.allocateVIP(vip); err != nil {
return
}
s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs, vip)
}
na.services[s.ID] = struct{}{}
return
}
func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
nc := a.netCtx
if s.Spec.Endpoint != nil {
// service has user-defined endpoint
if s.Endpoint == nil {
// service currently has no allocated endpoint, need allocated.
s.Endpoint = &api.Endpoint{
Spec: s.Spec.Endpoint.Copy(),
}
}
// The service is trying to expose ports to the external
// world. Automatically attach the service to the ingress
// network only if it is not already done.
if len(s.Spec.Endpoint.Ports) != 0 {
var found bool
for _, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == nc.ingressNetwork.ID {
found = true
break
}
}
if !found {
s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs,
&api.Endpoint_VirtualIP{NetworkID: nc.ingressNetwork.ID})
}
}
} else if s.Endpoint != nil {
// service has no user-defined endpoints while has already allocated network resources,
// need deallocated.
if err := nc.nwkAllocator.ServiceDeallocate(s); err != nil {
return err
}
}
if err := nc.nwkAllocator.ServiceAllocate(s); err != nil {
nc.unallocatedServices[s.ID] = s
return err
}
// If the service doesn't expose ports any more and if we have
// any lingering virtual IP references for ingress network
// clean them up here.
if s.Spec.Endpoint == nil || len(s.Spec.Endpoint.Ports) == 0 {
if s.Endpoint != nil {
for i, vip := range s.Endpoint.VirtualIPs {
if vip.NetworkID == nc.ingressNetwork.ID {
n := len(s.Endpoint.VirtualIPs)
s.Endpoint.VirtualIPs[i], s.Endpoint.VirtualIPs[n-1] = s.Endpoint.VirtualIPs[n-1], nil
s.Endpoint.VirtualIPs = s.Endpoint.VirtualIPs[:n-1]
break
}
}
}
}
if err := a.store.Update(func(tx store.Tx) error {
for {
err := store.UpdateService(tx, s)
if err != nil && err != store.ErrSequenceConflict {
return fmt.Errorf("failed updating state in store transaction for service %s: %v", s.ID, err)
}
if err == store.ErrSequenceConflict {
storeService := store.GetService(tx, s.ID)
storeService.Endpoint = s.Endpoint
s = storeService
continue
}
break
}
return nil
}); err != nil {
if err := nc.nwkAllocator.ServiceDeallocate(s); err != nil {
log.G(ctx).WithError(err).Errorf("failed rolling back allocation of service %s: %v", s.ID, err)
}
return err
}
return nil
}
// UpdateService updates a Service referenced by ServiceID with the given ServiceSpec.
// - Returns `NotFound` if the Service is not found.
// - Returns `InvalidArgument` if the ServiceSpec is malformed.
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) {
if request.ServiceID == "" || request.ServiceVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateServiceSpec(request.Spec); err != nil {
return nil, err
}
var service *api.Service
s.store.View(func(tx store.ReadTx) {
service = store.GetService(tx, request.ServiceID)
})
if service == nil {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
if request.Spec.Endpoint != nil && !reflect.DeepEqual(request.Spec.Endpoint, service.Spec.Endpoint) {
if err := s.checkPortConflicts(request.Spec, request.ServiceID); err != nil {
return nil, err
}
}
err := s.store.Update(func(tx store.Tx) error {
service = store.GetService(tx, request.ServiceID)
if service == nil {
return grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
// temporary disable network update
requestSpecNetworks := request.Spec.Task.Networks
if len(requestSpecNetworks) == 0 {
requestSpecNetworks = request.Spec.Networks
}
specNetworks := service.Spec.Task.Networks
if len(specNetworks) == 0 {
specNetworks = service.Spec.Networks
}
if !reflect.DeepEqual(requestSpecNetworks, specNetworks) {
return grpc.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error())
}
// Check to see if all the secrets being added exist as objects
// in our datastore
err := s.checkSecretExistence(tx, request.Spec)
if err != nil {
return err
}
// orchestrator is designed to be stateless, so it should not deal
// with service mode change (comparing current config with previous config).
// proper way to change service mode is to delete and re-add.
if reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) {
return grpc.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error())
}
if service.Spec.Annotations.Name != request.Spec.Annotations.Name {
return grpc.Errorf(codes.Unimplemented, errRenameNotSupported.Error())
}
service.Meta.Version = *request.ServiceVersion
service.PreviousSpec = service.Spec.Copy()
service.Spec = *request.Spec.Copy()
// Reset update status
service.UpdateStatus = nil
return store.UpdateService(tx, service)
})
if err != nil {
return nil, err
}
return &api.UpdateServiceResponse{
Service: service,
}, nil
}