func (a *Allocator) taskCreateNetworkAttachments(t *api.Task, s *api.Service) {
// If task network attachments have already been filled in no
// need to do anything else.
if len(t.Networks) != 0 {
return
}
var networks []*api.NetworkAttachment
if isIngressNetworkNeeded(s) {
networks = append(networks, &api.NetworkAttachment{Network: a.netCtx.ingressNetwork})
}
a.store.View(func(tx store.ReadTx) {
// Always prefer NetworkAttachmentConfig in the TaskSpec
specNetworks := t.Spec.Networks
if len(specNetworks) == 0 && s != nil && len(s.Spec.Networks) != 0 {
specNetworks = s.Spec.Networks
}
for _, na := range specNetworks {
n := store.GetNetwork(tx, na.Target)
if n == nil {
continue
}
attachment := api.NetworkAttachment{Network: n}
attachment.Aliases = append(attachment.Aliases, na.Aliases...)
attachment.Addresses = append(attachment.Addresses, na.Addresses...)
networks = append(networks, &attachment)
}
})
taskUpdateNetworks(t, networks)
}
func (a *Allocator) taskCreateNetworkAttachments(t *api.Task, s *api.Service) {
// If service is nil or if task network attachments have
// already been filled in no need to do anything else.
if s == nil || len(t.Networks) != 0 {
return
}
var networks []*api.NetworkAttachment
// The service to which this task belongs is trying to expose
// ports to the external world. Automatically attach the task
// to the ingress network.
if s.Spec.Endpoint != nil && len(s.Spec.Endpoint.Ports) != 0 {
networks = append(networks, &api.NetworkAttachment{Network: a.netCtx.ingressNetwork})
}
a.store.View(func(tx store.ReadTx) {
for _, na := range s.Spec.Networks {
n := store.GetNetwork(tx, na.Target)
if n != nil {
var aliases []string
for _, a := range na.Aliases {
aliases = append(aliases, a)
}
networks = append(networks, &api.NetworkAttachment{Network: n, Aliases: aliases})
}
}
})
taskUpdateNetworks(t, networks)
}
// AttachNetwork allows the node to request the resources
// allocation needed for a network attachment on the specific node.
// - Returns `InvalidArgument` if the Spec is malformed.
// - Returns `NotFound` if the Network is not found.
// - Returns `PermissionDenied` if the Network is not manually attachable.
// - Returns an error if the creation fails.
func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.AttachNetworkRequest) (*api.AttachNetworkResponse, error) {
nodeInfo, err := ca.RemoteNode(ctx)
if err != nil {
return nil, err
}
var network *api.Network
ra.store.View(func(tx store.ReadTx) {
network = store.GetNetwork(tx, request.Config.Target)
if network == nil {
if networks, err := store.FindNetworks(tx, store.ByName(request.Config.Target)); err == nil && len(networks) == 1 {
network = networks[0]
}
}
})
if network == nil {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.Config.Target)
}
if !network.Spec.Attachable {
return nil, grpc.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target)
}
t := &api.Task{
ID: identity.NewID(),
NodeID: nodeInfo.NodeID,
Spec: api.TaskSpec{
Runtime: &api.TaskSpec_Attachment{
Attachment: &api.NetworkAttachmentSpec{
ContainerID: request.ContainerID,
},
},
Networks: []*api.NetworkAttachmentConfig{
{
Target: network.ID,
Addresses: request.Config.Addresses,
},
},
},
Status: api.TaskStatus{
State: api.TaskStateNew,
Timestamp: ptypes.MustTimestampProto(time.Now()),
Message: "created",
},
DesiredState: api.TaskStateRunning,
// TODO: Add Network attachment.
}
if err := ra.store.Update(func(tx store.Tx) error {
return store.CreateTask(tx, t)
}); err != nil {
return nil, err
}
return &api.AttachNetworkResponse{AttachmentID: t.ID}, nil
}
// RemoveNetwork removes a Network referenced by NetworkID.
// - Returns `InvalidArgument` if NetworkID is not provided.
// - Returns `NotFound` if the Network is not found.
// - Returns an error if the deletion fails.
func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRequest) (*api.RemoveNetworkResponse, error) {
var (
services []*api.Service
err error
)
if request.NetworkID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
s.store.View(func(tx store.ReadTx) {
services, err = store.FindServices(tx, store.All)
})
if err != nil {
return nil, grpc.Errorf(codes.Internal, "could not find services using network %s", request.NetworkID)
}
for _, s := range services {
specNetworks := s.Spec.Task.Networks
if len(specNetworks) == 0 {
specNetworks = s.Spec.Networks
}
for _, na := range specNetworks {
if na.Target == request.NetworkID {
return nil, grpc.Errorf(codes.FailedPrecondition, "network %s is in use", request.NetworkID)
}
}
}
err = s.store.Update(func(tx store.Tx) error {
nw := store.GetNetwork(tx, request.NetworkID)
if _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
networkDescription := nw.ID
if nw.Spec.Annotations.Name != "" {
networkDescription = fmt.Sprintf("%s (%s)", nw.Spec.Annotations.Name, nw.ID)
}
return grpc.Errorf(codes.PermissionDenied, "%s is a pre-defined network and cannot be removed", networkDescription)
}
return store.DeleteNetwork(tx, request.NetworkID)
})
if err != nil {
if err == store.ErrNotExist {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
return nil, err
}
return &api.RemoveNetworkResponse{}, nil
}
// GetNetwork returns a Network given a NetworkID.
// - Returns `InvalidArgument` if NetworkID is not provided.
// - Returns `NotFound` if the Network is not found.
func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) (*api.GetNetworkResponse, error) {
if request.NetworkID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var n *api.Network
s.store.View(func(tx store.ReadTx) {
n = store.GetNetwork(tx, request.NetworkID)
})
if n == nil {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
return &api.GetNetworkResponse{
Network: n,
}, nil
}
func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error {
for _, na := range networks {
var network *api.Network
s.store.View(func(tx store.ReadTx) {
network = store.GetNetwork(tx, na.Target)
})
if network == nil {
continue
}
if _, ok := network.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
return grpc.Errorf(codes.InvalidArgument,
"Service cannot be explicitly attached to %q network which is a swarm internal network",
network.Spec.Annotations.Name)
}
}
return nil
}
// RemoveNetwork removes a Network referenced by NetworkID.
// - Returns `InvalidArgument` if NetworkID is not provided.
// - Returns `NotFound` if the Network is not found.
// - Returns an error if the deletion fails.
func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRequest) (*api.RemoveNetworkResponse, error) {
if request.NetworkID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
err := s.store.Update(func(tx store.Tx) error {
services, err := store.FindServices(tx, store.ByReferencedNetworkID(request.NetworkID))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", request.NetworkID, err)
}
if len(services) != 0 {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", request.NetworkID, services[0].ID)
}
tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(request.NetworkID))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", request.NetworkID, err)
}
if len(tasks) != 0 {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", request.NetworkID, tasks[0].ID)
}
nw := store.GetNetwork(tx, request.NetworkID)
if _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
networkDescription := nw.ID
if nw.Spec.Annotations.Name != "" {
networkDescription = fmt.Sprintf("%s (%s)", nw.Spec.Annotations.Name, nw.ID)
}
return grpc.Errorf(codes.PermissionDenied, "%s is a pre-defined network and cannot be removed", networkDescription)
}
return store.DeleteNetwork(tx, request.NetworkID)
})
if err != nil {
if err == store.ErrNotExist {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
return nil, err
}
return &api.RemoveNetworkResponse{}, nil
}
func (a *Allocator) taskCreateNetworkAttachments(t *api.Task, s *api.Service) {
// If task network attachments have already been filled in no
// need to do anything else.
if len(t.Networks) != 0 {
return
}
var networks []*api.NetworkAttachment
// The service to which this task belongs is trying to expose
// ports to the external world. Automatically attach the task
// to the ingress network.
if s != nil && s.Spec.Endpoint != nil && len(s.Spec.Endpoint.Ports) != 0 {
networks = append(networks, &api.NetworkAttachment{Network: a.netCtx.ingressNetwork})
}
a.store.View(func(tx store.ReadTx) {
// Always prefer NetworkAttachmentConfig in the TaskSpec
specNetworks := t.Spec.Networks
if len(specNetworks) == 0 && s != nil && len(s.Spec.Networks) != 0 {
specNetworks = s.Spec.Networks
}
for _, na := range specNetworks {
n := store.GetNetwork(tx, na.Target)
if n == nil {
continue
}
attachment := api.NetworkAttachment{Network: n}
attachment.Aliases = append(attachment.Aliases, na.Aliases...)
attachment.Addresses = append(attachment.Addresses, na.Addresses...)
networks = append(networks, &attachment)
}
})
taskUpdateNetworks(t, networks)
}
func (a *Allocator) allocateTask(ctx context.Context, t *api.Task) (err error) {
taskUpdated := false
nc := a.netCtx
// We might be here even if a task allocation has already
// happened but wasn't successfully committed to store. In such
// cases skip allocation and go straight ahead to updating the
// store.
if !nc.nwkAllocator.IsTaskAllocated(t) {
a.store.View(func(tx store.ReadTx) {
if t.ServiceID != "" {
s := store.GetService(tx, t.ServiceID)
if s == nil {
err = fmt.Errorf("could not find service %s", t.ServiceID)
return
}
if !nc.nwkAllocator.IsServiceAllocated(s) {
err = fmt.Errorf("service %s to which this task %s belongs has pending allocations", s.ID, t.ID)
return
}
if s.Endpoint != nil {
taskUpdateEndpoint(t, s.Endpoint)
taskUpdated = true
}
}
for _, na := range t.Networks {
n := store.GetNetwork(tx, na.Network.ID)
if n == nil {
err = fmt.Errorf("failed to retrieve network %s while allocating task %s", na.Network.ID, t.ID)
return
}
if !nc.nwkAllocator.IsAllocated(n) {
err = fmt.Errorf("network %s attached to task %s not allocated yet", n.ID, t.ID)
return
}
na.Network = n
}
if err = nc.nwkAllocator.AllocateTask(t); err != nil {
err = errors.Wrapf(err, "failed during networktask allocation for task %s", t.ID)
return
}
if nc.nwkAllocator.IsTaskAllocated(t) {
taskUpdated = true
}
})
if err != nil {
return err
}
}
// Update the network allocations and moving to
// PENDING state on top of the latest store state.
if a.taskAllocateVote(networkVoter, t.ID) {
if t.Status.State < api.TaskStatePending {
updateTaskStatus(t, api.TaskStatePending, allocatedStatusMessage)
taskUpdated = true
}
}
if !taskUpdated {
return errNoChanges
}
return nil
}
func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx store.Tx, t *api.Task) (*api.Task, error) {
taskUpdated := false
// Get the latest task state from the store before updating.
storeT := store.GetTask(tx, t.ID)
if storeT == nil {
return nil, fmt.Errorf("could not find task %s while trying to update network allocation", t.ID)
}
// We might be here even if a task allocation has already
// happened but wasn't successfully committed to store. In such
// cases skip allocation and go straight ahead to updating the
// store.
if !nc.nwkAllocator.IsTaskAllocated(t) {
if t.ServiceID != "" {
s := store.GetService(tx, t.ServiceID)
if s == nil {
return nil, fmt.Errorf("could not find service %s", t.ServiceID)
}
if !nc.nwkAllocator.IsServiceAllocated(s) {
return nil, fmt.Errorf("service %s to which this task %s belongs has pending allocations", s.ID, t.ID)
}
taskUpdateEndpoint(t, s.Endpoint)
}
for _, na := range t.Networks {
n := store.GetNetwork(tx, na.Network.ID)
if n == nil {
return nil, fmt.Errorf("failed to retrieve network %s while allocating task %s", na.Network.ID, t.ID)
}
if !nc.nwkAllocator.IsAllocated(n) {
return nil, fmt.Errorf("network %s attached to task %s not allocated yet", n.ID, t.ID)
}
na.Network = n
}
if err := nc.nwkAllocator.AllocateTask(t); err != nil {
return nil, fmt.Errorf("failed during networktask allocation for task %s: %v", t.ID, err)
}
if nc.nwkAllocator.IsTaskAllocated(t) {
taskUpdateNetworks(storeT, t.Networks)
taskUpdateEndpoint(storeT, t.Endpoint)
taskUpdated = true
}
}
// Update the network allocations and moving to
// ALLOCATED state on top of the latest store state.
if a.taskAllocateVote(networkVoter, t.ID) {
if storeT.Status.State < api.TaskStateAllocated {
updateTaskStatus(storeT, api.TaskStateAllocated, "allocated")
taskUpdated = true
}
}
if taskUpdated {
if err := store.UpdateTask(tx, storeT); err != nil {
return nil, fmt.Errorf("failed updating state in store transaction for task %s: %v", storeT.ID, err)
}
}
return storeT, nil
}
func TestAllocator(t *testing.T) {
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
a, err := New(s)
assert.NoError(t, err)
assert.NotNil(t, a)
// Try adding some objects to store before allocator is started
assert.NoError(t, s.Update(func(tx store.Tx) error {
n1 := &api.Network{
ID: "testID1",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test1",
},
},
}
assert.NoError(t, store.CreateNetwork(tx, n1))
s1 := &api.Service{
ID: "testServiceID1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "service1",
},
Task: api.TaskSpec{
Networks: []*api.NetworkAttachmentConfig{
{
Target: "testID1",
},
},
},
Endpoint: &api.EndpointSpec{},
},
}
assert.NoError(t, store.CreateService(tx, s1))
t1 := &api.Task{
ID: "testTaskID1",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Networks: []*api.NetworkAttachment{
{
Network: n1,
},
},
}
assert.NoError(t, store.CreateTask(tx, t1))
return nil
}))
netWatch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateNetwork{}, state.EventDeleteNetwork{})
defer cancel()
taskWatch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{}, state.EventDeleteTask{})
defer cancel()
serviceWatch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateService{}, state.EventDeleteService{})
defer cancel()
// Start allocator
go func() {
assert.NoError(t, a.Run(context.Background()))
}()
// Now verify if we get network and tasks updated properly
watchNetwork(t, netWatch, false, isValidNetwork)
watchTask(t, s, taskWatch, false, isValidTask)
watchService(t, serviceWatch, false, nil)
// Add new networks/tasks/services after allocator is started.
assert.NoError(t, s.Update(func(tx store.Tx) error {
n2 := &api.Network{
ID: "testID2",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test2",
},
},
}
assert.NoError(t, store.CreateNetwork(tx, n2))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
assert.NoError(t, s.Update(func(tx store.Tx) error {
s2 := &api.Service{
ID: "testServiceID2",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "service2",
},
Networks: []*api.NetworkAttachmentConfig{
{
Target: "testID2",
},
},
Endpoint: &api.EndpointSpec{},
},
}
assert.NoError(t, store.CreateService(tx, s2))
return nil
}))
watchService(t, serviceWatch, false, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t2 := &api.Task{
ID: "testTaskID2",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
ServiceID: "testServiceID2",
DesiredState: api.TaskStateRunning,
}
assert.NoError(t, store.CreateTask(tx, t2))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
// Now try adding a task which depends on a network before adding the network.
n3 := &api.Network{
ID: "testID3",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test3",
},
},
}
assert.NoError(t, s.Update(func(tx store.Tx) error {
t3 := &api.Task{
ID: "testTaskID3",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
Networks: []*api.NetworkAttachment{
{
Network: n3,
},
},
}
assert.NoError(t, store.CreateTask(tx, t3))
return nil
}))
// Wait for a little bit of time before adding network just to
// test network is not available while task allocation is
// going through
time.Sleep(10 * time.Millisecond)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNetwork(tx, n3))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteTask(tx, "testTaskID3"))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t5 := &api.Task{
ID: "testTaskID5",
Spec: api.TaskSpec{
Networks: []*api.NetworkAttachmentConfig{
{
Target: "testID2",
},
},
},
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
ServiceID: "testServiceID2",
}
assert.NoError(t, store.CreateTask(tx, t5))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteNetwork(tx, "testID3"))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteService(tx, "testServiceID2"))
return nil
}))
watchService(t, serviceWatch, false, nil)
// Try to create a task with no network attachments and test
// that it moves to ALLOCATED state.
assert.NoError(t, s.Update(func(tx store.Tx) error {
t4 := &api.Task{
ID: "testTaskID4",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
}
assert.NoError(t, store.CreateTask(tx, t4))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
n2 := store.GetNetwork(tx, "testID2")
require.NotEqual(t, nil, n2)
assert.NoError(t, store.UpdateNetwork(tx, n2))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
watchNetwork(t, netWatch, true, nil)
// Try updating task which is already allocated
assert.NoError(t, s.Update(func(tx store.Tx) error {
t2 := store.GetTask(tx, "testTaskID2")
require.NotEqual(t, nil, t2)
assert.NoError(t, store.UpdateTask(tx, t2))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
watchTask(t, s, taskWatch, true, nil)
// Try adding networks with conflicting network resources and
// add task which attaches to a network which gets allocated
// later and verify if task reconciles and moves to ALLOCATED.
n4 := &api.Network{
ID: "testID4",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test4",
},
DriverConfig: &api.Driver{
Name: "overlay",
Options: map[string]string{
"com.docker.network.driver.overlay.vxlanid_list": "328",
},
},
},
}
n5 := n4.Copy()
n5.ID = "testID5"
n5.Spec.Annotations.Name = "test5"
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNetwork(tx, n4))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNetwork(tx, n5))
return nil
}))
watchNetwork(t, netWatch, true, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t6 := &api.Task{
ID: "testTaskID6",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
Networks: []*api.NetworkAttachment{
{
Network: n5,
},
},
}
assert.NoError(t, store.CreateTask(tx, t6))
return nil
}))
watchTask(t, s, taskWatch, true, nil)
// Now remove the conflicting network.
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteNetwork(tx, n4.ID))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
watchTask(t, s, taskWatch, false, isValidTask)
// Try adding services with conflicting port configs and add
// task which is part of the service whose allocation hasn't
// happened and when that happens later and verify if task
// reconciles and moves to ALLOCATED.
s3 := &api.Service{
ID: "testServiceID3",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "service3",
},
Endpoint: &api.EndpointSpec{
Ports: []*api.PortConfig{
{
Name: "http",
TargetPort: 80,
PublishedPort: 8080,
},
},
},
},
}
s4 := s3.Copy()
s4.ID = "testServiceID4"
s4.Spec.Annotations.Name = "service4"
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, s3))
return nil
}))
watchService(t, serviceWatch, false, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, s4))
return nil
}))
watchService(t, serviceWatch, true, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t7 := &api.Task{
ID: "testTaskID7",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
ServiceID: "testServiceID4",
DesiredState: api.TaskStateRunning,
}
assert.NoError(t, store.CreateTask(tx, t7))
return nil
}))
watchTask(t, s, taskWatch, true, nil)
// Now remove the conflicting service.
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteService(tx, s3.ID))
return nil
}))
watchService(t, serviceWatch, false, nil)
watchTask(t, s, taskWatch, false, isValidTask)
a.Stop()
}