// CreateService creates and return a Service based on the provided ServiceSpec.
// - Returns `InvalidArgument` if the ServiceSpec is malformed.
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
// - Returns `AlreadyExists` if the ServiceID conflicts.
// - Returns an error if the creation fails.
func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRequest) (*api.CreateServiceResponse, error) {
if err := validateServiceSpec(request.Spec); err != nil {
return nil, err
}
if err := s.checkPortConflicts(request.Spec, ""); err != nil {
return nil, err
}
// TODO(aluzzardi): Consider using `Name` as a primary key to handle
// duplicate creations. See #65
service := &api.Service{
ID: identity.NewID(),
Spec: *request.Spec,
}
err := s.store.Update(func(tx store.Tx) error {
return store.CreateService(tx, service)
})
if err != nil {
return nil, err
}
return &api.CreateServiceResponse{
Service: service,
}, nil
}
// CreateService creates and return a Service based on the provided ServiceSpec.
// - Returns `InvalidArgument` if the ServiceSpec is malformed.
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
// - Returns `AlreadyExists` if the ServiceID conflicts.
// - Returns an error if the creation fails.
func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRequest) (*api.CreateServiceResponse, error) {
if err := validateServiceSpec(request.Spec); err != nil {
return nil, err
}
if err := s.validateNetworks(request.Spec.Networks); err != nil {
return nil, err
}
if err := s.checkPortConflicts(request.Spec, ""); err != nil {
return nil, err
}
// TODO(aluzzardi): Consider using `Name` as a primary key to handle
// duplicate creations. See #65
service := &api.Service{
ID: identity.NewID(),
Spec: *request.Spec,
}
err := s.store.Update(func(tx store.Tx) error {
// Check to see if all the secrets being added exist as objects
// in our datastore
err := s.checkSecretExistence(tx, request.Spec)
if err != nil {
return err
}
return store.CreateService(tx, service)
})
if err != nil {
return nil, err
}
return &api.CreateServiceResponse{
Service: service,
}, nil
}
func TestUpdater(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
// Move tasks to their desired state.
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
go func() {
for {
select {
case e := <-watch:
task := e.(state.EventUpdateTask).Task
if task.Status.State == task.DesiredState {
continue
}
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = task.DesiredState
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
}
}
}()
instances := 3
cluster := &api.Cluster{
// test cluster configuration propagation to task creation.
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: "default",
},
},
}
service := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: uint64(instances),
},
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "v:1",
// This won't apply in this test because we set the old tasks to DEAD.
StopGracePeriod: ptypes.DurationProto(time.Hour),
},
},
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateCluster(tx, cluster))
assert.NoError(t, store.CreateService(tx, service))
for i := 0; i < instances; i++ {
assert.NoError(t, store.CreateTask(tx, newTask(cluster, service, uint64(i))))
}
return nil
})
assert.NoError(t, err)
originalTasks := getRunnableServiceTasks(t, s, service)
for _, task := range originalTasks {
assert.Equal(t, "v:1", task.Spec.GetContainer().Image)
assert.Nil(t, task.LogDriver) // should be left alone
}
service.Spec.Task.GetContainer().Image = "v:2"
service.Spec.Task.LogDriver = &api.Driver{Name: "tasklogdriver"}
updater := NewUpdater(s, NewRestartSupervisor(s))
updater.Run(ctx, cluster, service, getRunnableServiceTasks(t, s, service))
updatedTasks := getRunnableServiceTasks(t, s, service)
for _, task := range updatedTasks {
assert.Equal(t, "v:2", task.Spec.GetContainer().Image)
assert.Equal(t, service.Spec.Task.LogDriver, task.LogDriver) // pick up from task
}
service.Spec.Task.GetContainer().Image = "v:3"
cluster.Spec.DefaultLogDriver = &api.Driver{Name: "clusterlogdriver"} // make cluster default logdriver.
service.Spec.Update = &api.UpdateConfig{
Parallelism: 1,
}
updater = NewUpdater(s, NewRestartSupervisor(s))
updater.Run(ctx, cluster, service, getRunnableServiceTasks(t, s, service))
updatedTasks = getRunnableServiceTasks(t, s, service)
for _, task := range updatedTasks {
assert.Equal(t, "v:3", task.Spec.GetContainer().Image)
assert.Equal(t, service.Spec.Task.LogDriver, task.LogDriver) // still pick up from task
}
service.Spec.Task.GetContainer().Image = "v:4"
service.Spec.Task.LogDriver = nil // use cluster default now.
service.Spec.Update = &api.UpdateConfig{
Parallelism: 1,
Delay: *ptypes.DurationProto(10 * time.Millisecond),
}
updater = NewUpdater(s, NewRestartSupervisor(s))
updater.Run(ctx, cluster, service, getRunnableServiceTasks(t, s, service))
updatedTasks = getRunnableServiceTasks(t, s, service)
for _, task := range updatedTasks {
assert.Equal(t, "v:4", task.Spec.GetContainer().Image)
assert.Equal(t, cluster.Spec.DefaultLogDriver, task.LogDriver) // pick up from cluster
}
}
func TestUpdaterStopGracePeriod(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
// Move tasks to their desired state.
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
go func() {
for {
select {
case e := <-watch:
task := e.(state.EventUpdateTask).Task
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
// Explicitly do not set task state to
// DEAD to trigger StopGracePeriod
if task.DesiredState == api.TaskStateRunning && task.Status.State != api.TaskStateRunning {
task.Status.State = api.TaskStateRunning
return store.UpdateTask(tx, task)
}
return nil
})
assert.NoError(t, err)
}
}
}()
var instances uint64 = 3
service := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "v:1",
StopGracePeriod: ptypes.DurationProto(100 * time.Millisecond),
},
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: instances,
},
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, service))
for i := uint64(0); i < instances; i++ {
task := newTask(nil, service, uint64(i))
task.Status.State = api.TaskStateRunning
assert.NoError(t, store.CreateTask(tx, task))
}
return nil
})
assert.NoError(t, err)
originalTasks := getRunnableServiceTasks(t, s, service)
for _, task := range originalTasks {
assert.Equal(t, "v:1", task.Spec.GetContainer().Image)
}
before := time.Now()
service.Spec.Task.GetContainer().Image = "v:2"
updater := NewUpdater(s, NewRestartSupervisor(s))
// Override the default (1 minute) to speed up the test.
updater.restarts.taskTimeout = 100 * time.Millisecond
updater.Run(ctx, nil, service, getRunnableServiceTasks(t, s, service))
updatedTasks := getRunnableServiceTasks(t, s, service)
for _, task := range updatedTasks {
assert.Equal(t, "v:2", task.Spec.GetContainer().Image)
}
after := time.Now()
// At least 100 ms should have elapsed. Only check the lower bound,
// because the system may be slow and it could have taken longer.
if after.Sub(before) < 100*time.Millisecond {
t.Fatal("stop timeout should have elapsed")
}
}
func addService(t *testing.T, s *store.MemoryStore, service *api.Service) {
s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, service))
return nil
})
}
func TestTaskHistory(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
assert.NoError(t, s.Update(func(tx store.Tx) error {
store.CreateCluster(tx, &api.Cluster{
ID: identity.NewID(),
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: store.DefaultClusterName,
},
Orchestration: api.OrchestrationConfig{
TaskHistoryRetentionLimit: 2,
},
},
})
return nil
}))
taskReaper := NewTaskReaper(s)
defer taskReaper.Stop()
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/)
defer cancel()
// Create a service with two instances specified before the orchestrator is
// started. This should result in two tasks when the orchestrator
// starts up.
err := s.Update(func(tx store.Tx) error {
j1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 2,
},
},
Task: api.TaskSpec{
Restart: &api.RestartPolicy{
Condition: api.RestartOnAny,
Delay: ptypes.DurationProto(0),
},
},
},
}
assert.NoError(t, store.CreateService(tx, j1))
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
go taskReaper.Run()
observedTask1 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask1.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1")
observedTask2 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask2.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1")
// Fail both tasks. They should both get restarted.
updatedTask1 := observedTask1.Copy()
updatedTask1.Status.State = api.TaskStateFailed
updatedTask1.ServiceAnnotations = api.Annotations{Name: "original"}
updatedTask2 := observedTask2.Copy()
updatedTask2.Status.State = api.TaskStateFailed
updatedTask2.ServiceAnnotations = api.Annotations{Name: "original"}
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask1))
assert.NoError(t, store.UpdateTask(tx, updatedTask2))
return nil
})
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
observedTask3 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask3.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1")
expectTaskUpdate(t, watch)
observedTask4 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask4.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1")
// Fail these replacement tasks. Since TaskHistory is set to 2, this
// should cause the oldest tasks for each instance to get deleted.
updatedTask3 := observedTask3.Copy()
updatedTask3.Status.State = api.TaskStateFailed
updatedTask4 := observedTask4.Copy()
updatedTask4.Status.State = api.TaskStateFailed
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask3))
assert.NoError(t, store.UpdateTask(tx, updatedTask4))
return nil
})
deletedTask1 := watchTaskDelete(t, watch)
deletedTask2 := watchTaskDelete(t, watch)
assert.Equal(t, api.TaskStateFailed, deletedTask1.Status.State)
assert.Equal(t, "original", deletedTask1.ServiceAnnotations.Name)
assert.Equal(t, api.TaskStateFailed, deletedTask2.Status.State)
assert.Equal(t, "original", deletedTask2.ServiceAnnotations.Name)
var foundTasks []*api.Task
s.View(func(tx store.ReadTx) {
foundTasks, err = store.FindTasks(tx, store.All)
})
assert.NoError(t, err)
assert.Len(t, foundTasks, 4)
}
func TestIsStateDirty(t *testing.T) {
ctx := context.Background()
temp, err := ioutil.TempFile("", "test-socket")
assert.NoError(t, err)
assert.NoError(t, temp.Close())
assert.NoError(t, os.Remove(temp.Name()))
defer os.RemoveAll(temp.Name())
stateDir, err := ioutil.TempDir("", "test-raft")
assert.NoError(t, err)
defer os.RemoveAll(stateDir)
tc := testutils.NewTestCA(t, func(p ca.CertPaths) *ca.KeyReadWriter {
return ca.NewKeyReadWriter(p, []byte("kek"), nil)
})
defer tc.Stop()
managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
m, err := New(&Config{
RemoteAPI: &RemoteAddrs{ListenAddr: "127.0.0.1:0"},
ControlAPI: temp.Name(),
StateDir: stateDir,
SecurityConfig: managerSecurityConfig,
AutoLockManagers: true,
UnlockKey: []byte("kek"),
})
assert.NoError(t, err)
assert.NotNil(t, m)
go m.Run(ctx)
defer m.Stop(ctx, false)
// State should never be dirty just after creating the manager
isDirty, err := m.IsStateDirty()
assert.NoError(t, err)
assert.False(t, isDirty)
// Wait for cluster and node to be created.
watch, cancel := state.Watch(m.raftNode.MemoryStore().WatchQueue())
defer cancel()
<-watch
<-watch
// Updating the node should not cause the state to become dirty
assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
node := store.GetNode(tx, m.config.SecurityConfig.ClientTLSCreds.NodeID())
require.NotNil(t, node)
node.Spec.Availability = api.NodeAvailabilityPause
return store.UpdateNode(tx, node)
}))
isDirty, err = m.IsStateDirty()
assert.NoError(t, err)
assert.False(t, isDirty)
// Adding a service should cause the state to become dirty
assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
return store.CreateService(tx, &api.Service{ID: "foo"})
}))
isDirty, err = m.IsStateDirty()
assert.NoError(t, err)
assert.True(t, isDirty)
}
func TestReplicatedOrchestrator(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/)
defer cancel()
// Create a service with two instances specified before the orchestrator is
// started. This should result in two tasks when the orchestrator
// starts up.
err := s.Update(func(tx store.Tx) error {
s1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 2,
},
},
},
}
assert.NoError(t, store.CreateService(tx, s1))
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
observedTask1 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask1.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1")
observedTask2 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask2.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1")
// Create a second service.
err = s.Update(func(tx store.Tx) error {
s2 := &api.Service{
ID: "id2",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name2",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 1,
},
},
},
}
assert.NoError(t, store.CreateService(tx, s2))
return nil
})
assert.NoError(t, err)
observedTask3 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask3.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name2")
// Update a service to scale it out to 3 instances
err = s.Update(func(tx store.Tx) error {
s2 := &api.Service{
ID: "id2",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name2",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 3,
},
},
},
}
assert.NoError(t, store.UpdateService(tx, s2))
return nil
})
assert.NoError(t, err)
observedTask4 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask4.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name2")
observedTask5 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask5.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask5.ServiceAnnotations.Name, "name2")
// Now scale it back down to 1 instance
err = s.Update(func(tx store.Tx) error {
s2 := &api.Service{
ID: "id2",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name2",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 1,
},
},
},
}
assert.NoError(t, store.UpdateService(tx, s2))
return nil
})
assert.NoError(t, err)
observedDeletion1 := watchShutdownTask(t, watch)
assert.Equal(t, observedDeletion1.Status.State, api.TaskStateNew)
assert.Equal(t, observedDeletion1.ServiceAnnotations.Name, "name2")
observedDeletion2 := watchShutdownTask(t, watch)
assert.Equal(t, observedDeletion2.Status.State, api.TaskStateNew)
assert.Equal(t, observedDeletion2.ServiceAnnotations.Name, "name2")
// There should be one remaining task attached to service id2/name2.
var liveTasks []*api.Task
s.View(func(readTx store.ReadTx) {
var tasks []*api.Task
tasks, err = store.FindTasks(readTx, store.ByServiceID("id2"))
for _, t := range tasks {
if t.DesiredState == api.TaskStateRunning {
liveTasks = append(liveTasks, t)
}
}
})
assert.NoError(t, err)
assert.Len(t, liveTasks, 1)
// Delete the remaining task directly. It should be recreated by the
// orchestrator.
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteTask(tx, liveTasks[0].ID))
return nil
})
assert.NoError(t, err)
observedTask6 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask6.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name2")
// Delete the service. Its remaining task should go away.
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteService(tx, "id2"))
return nil
})
assert.NoError(t, err)
deletedTask := watchTaskDelete(t, watch)
assert.Equal(t, deletedTask.Status.State, api.TaskStateNew)
assert.Equal(t, deletedTask.ServiceAnnotations.Name, "name2")
}
func TestLogBrokerSelector(t *testing.T) {
ctx, ca, _, serverAddr, brokerAddr, done := testLogBrokerEnv(t)
defer done()
client, clientDone := testLogClient(t, serverAddr)
defer clientDone()
agent1, agent1Security, agent1Done := testBrokerClient(t, ca, brokerAddr)
defer agent1Done()
agent1subscriptions := listenSubscriptions(ctx, t, agent1)
agent2, agent2Security, agent2Done := testBrokerClient(t, ca, brokerAddr)
defer agent2Done()
agent2subscriptions := listenSubscriptions(ctx, t, agent2)
// Subscribe to a task.
require.NoError(t, ca.MemoryStore.Update(func(tx store.Tx) error {
return store.CreateTask(tx, &api.Task{
ID: "task",
})
}))
_, err := client.SubscribeLogs(ctx, &api.SubscribeLogsRequest{
Options: &api.LogSubscriptionOptions{
Follow: true,
},
Selector: &api.LogSelector{
TaskIDs: []string{"task"},
},
})
require.NoError(t, err)
// Since it's not assigned to any agent, nobody should receive it.
ensureNoSubscription(t, agent1subscriptions)
ensureNoSubscription(t, agent2subscriptions)
// Assign the task to agent-1. Make sure it's received by agent-1 but *not*
// agent-2.
require.NoError(t, ca.MemoryStore.Update(func(tx store.Tx) error {
task := store.GetTask(tx, "task")
require.NotNil(t, task)
task.NodeID = agent1Security.ServerTLSCreds.NodeID()
return store.UpdateTask(tx, task)
}))
ensureSubscription(t, agent1subscriptions)
ensureNoSubscription(t, agent2subscriptions)
// Subscribe to a service.
require.NoError(t, ca.MemoryStore.Update(func(tx store.Tx) error {
return store.CreateService(tx, &api.Service{
ID: "service",
})
}))
_, err = client.SubscribeLogs(ctx, &api.SubscribeLogsRequest{
Options: &api.LogSubscriptionOptions{
Follow: true,
},
Selector: &api.LogSelector{
ServiceIDs: []string{"service"},
},
})
require.NoError(t, err)
// Since there are no corresponding tasks, nobody should receive it.
ensureNoSubscription(t, agent1subscriptions)
ensureNoSubscription(t, agent2subscriptions)
// Create a task that does *NOT* belong to our service and assign it to node-1.
require.NoError(t, ca.MemoryStore.Update(func(tx store.Tx) error {
return store.CreateTask(tx, &api.Task{
ID: "wrong-task",
ServiceID: "wrong-service",
NodeID: agent1Security.ServerTLSCreds.NodeID(),
})
}))
// Ensure agent-1 doesn't receive it.
ensureNoSubscription(t, agent1subscriptions)
// Now create another task that does belong to our service and assign it to node-1.
require.NoError(t, ca.MemoryStore.Update(func(tx store.Tx) error {
return store.CreateTask(tx, &api.Task{
ID: "service-task-1",
ServiceID: "service",
NodeID: agent1Security.ServerTLSCreds.NodeID(),
})
}))
// Make sure agent-1 receives it...
ensureSubscription(t, agent1subscriptions)
// ...and agent-2 does not.
ensureNoSubscription(t, agent2subscriptions)
// Create another task, same as above.
require.NoError(t, ca.MemoryStore.Update(func(tx store.Tx) error {
return store.CreateTask(tx, &api.Task{
ID: "service-task-2",
ServiceID: "service",
NodeID: agent1Security.ServerTLSCreds.NodeID(),
})
}))
// agent-1 should *not* receive it anymore since the subscription was already delivered.
// agent-2 should still not get it.
ensureNoSubscription(t, agent1subscriptions)
ensureNoSubscription(t, agent2subscriptions)
// Now, create another one and assign it to agent-2.
require.NoError(t, ca.MemoryStore.Update(func(tx store.Tx) error {
return store.CreateTask(tx, &api.Task{
ID: "service-task-3",
ServiceID: "service",
NodeID: agent2Security.ServerTLSCreds.NodeID(),
})
}))
// Make sure it's delivered to agent-2.
ensureSubscription(t, agent2subscriptions)
// it shouldn't do anything for agent-1.
ensureNoSubscription(t, agent1subscriptions)
}
func TestOrchestratorRestartOnNone(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/)
defer cancel()
// Create a service with two instances specified before the orchestrator is
// started. This should result in two tasks when the orchestrator
// starts up.
err := s.Update(func(tx store.Tx) error {
j1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
Restart: &api.RestartPolicy{
Condition: api.RestartOnNone,
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 2,
},
},
},
}
assert.NoError(t, store.CreateService(tx, j1))
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
observedTask1 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask1.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1")
observedTask2 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask2.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1")
// Fail the first task. Confirm that it does not get restarted.
updatedTask1 := observedTask1.Copy()
updatedTask1.Status.State = api.TaskStateFailed
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask1))
return nil
})
assert.NoError(t, err)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
select {
case <-watch:
t.Fatal("got unexpected event")
case <-time.After(100 * time.Millisecond):
}
// Mark the second task as completed. Confirm that it does not get restarted.
updatedTask2 := observedTask2.Copy()
updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted}
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask2))
return nil
})
assert.NoError(t, err)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
select {
case <-watch:
t.Fatal("got unexpected event")
case <-time.After(100 * time.Millisecond):
}
}
func TestOrchestratorRestartOnAny(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/)
defer cancel()
// Create a service with two instances specified before the orchestrator is
// started. This should result in two tasks when the orchestrator
// starts up.
err := s.Update(func(tx store.Tx) error {
j1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
Restart: &api.RestartPolicy{
Condition: api.RestartOnAny,
Delay: ptypes.DurationProto(0),
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 2,
},
},
},
}
assert.NoError(t, store.CreateService(tx, j1))
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
observedTask1 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask1.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1")
observedTask2 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask2.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1")
// Fail the first task. Confirm that it gets restarted.
updatedTask1 := observedTask1.Copy()
updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed}
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask1))
return nil
})
assert.NoError(t, err)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
observedTask3 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask3.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1")
expectCommit(t, watch)
observedTask4 := watchTaskUpdate(t, watch)
assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning)
assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1")
// Mark the second task as completed. Confirm that it gets restarted.
updatedTask2 := observedTask2.Copy()
updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted}
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask2))
return nil
})
assert.NoError(t, err)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
observedTask5 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask5.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask5.ServiceAnnotations.Name, "name1")
expectCommit(t, watch)
observedTask6 := watchTaskUpdate(t, watch)
assert.Equal(t, observedTask6.DesiredState, api.TaskStateRunning)
assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name1")
}
func TestUpdaterFailureAction(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
// Fail new tasks the updater tries to run
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
go func() {
for {
select {
case e := <-watch:
task := e.(state.EventUpdateTask).Task
if task.DesiredState == api.TaskStateRunning && task.Status.State != api.TaskStateFailed {
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = api.TaskStateFailed
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
} else if task.DesiredState > api.TaskStateRunning {
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = task.DesiredState
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
}
}
}
}()
instances := 3
cluster := &api.Cluster{
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: "default",
},
},
}
service := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: uint64(instances),
},
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "v:1",
// This won't apply in this test because we set the old tasks to DEAD.
StopGracePeriod: ptypes.DurationProto(time.Hour),
},
},
},
Update: &api.UpdateConfig{
FailureAction: api.UpdateConfig_PAUSE,
Parallelism: 1,
Delay: *ptypes.DurationProto(500 * time.Millisecond),
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateCluster(tx, cluster))
assert.NoError(t, store.CreateService(tx, service))
for i := 0; i < instances; i++ {
assert.NoError(t, store.CreateTask(tx, newTask(cluster, service, uint64(i))))
}
return nil
})
assert.NoError(t, err)
originalTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range originalTasks {
for _, task := range slot {
assert.Equal(t, "v:1", task.Spec.GetContainer().Image)
}
}
service.Spec.Task.GetContainer().Image = "v:2"
updater := NewUpdater(s, NewRestartSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks := getRunnableSlotSlice(t, s, service)
v1Counter := 0
v2Counter := 0
for _, slot := range updatedTasks {
for _, task := range slot {
if task.Spec.GetContainer().Image == "v:1" {
v1Counter++
} else if task.Spec.GetContainer().Image == "v:2" {
v2Counter++
}
}
}
assert.Equal(t, instances-1, v1Counter)
assert.Equal(t, 1, v2Counter)
s.View(func(tx store.ReadTx) {
service = store.GetService(tx, service.ID)
})
assert.Equal(t, api.UpdateStatus_PAUSED, service.UpdateStatus.State)
// Updating again should do nothing while the update is PAUSED
updater = NewUpdater(s, NewRestartSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
v1Counter = 0
v2Counter = 0
for _, slot := range updatedTasks {
for _, task := range slot {
if task.Spec.GetContainer().Image == "v:1" {
v1Counter++
} else if task.Spec.GetContainer().Image == "v:2" {
v2Counter++
}
}
}
assert.Equal(t, instances-1, v1Counter)
assert.Equal(t, 1, v2Counter)
// Switch to a service with FailureAction: CONTINUE
err = s.Update(func(tx store.Tx) error {
service = store.GetService(tx, service.ID)
service.Spec.Update.FailureAction = api.UpdateConfig_CONTINUE
service.UpdateStatus = nil
assert.NoError(t, store.UpdateService(tx, service))
return nil
})
assert.NoError(t, err)
service.Spec.Task.GetContainer().Image = "v:3"
updater = NewUpdater(s, NewRestartSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
v2Counter = 0
v3Counter := 0
for _, slot := range updatedTasks {
for _, task := range slot {
if task.Spec.GetContainer().Image == "v:2" {
v2Counter++
} else if task.Spec.GetContainer().Image == "v:3" {
v3Counter++
}
}
}
assert.Equal(t, 0, v2Counter)
assert.Equal(t, instances, v3Counter)
}
func TestReplicatedScaleDown(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
s1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 6,
},
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, s1))
nodes := []*api.Node{
{
ID: "node1",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name1",
},
Availability: api.NodeAvailabilityActive,
},
Status: api.NodeStatus{
State: api.NodeStatus_READY,
},
},
{
ID: "node2",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name2",
},
Availability: api.NodeAvailabilityActive,
},
Status: api.NodeStatus{
State: api.NodeStatus_READY,
},
},
{
ID: "node3",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name3",
},
Availability: api.NodeAvailabilityActive,
},
Status: api.NodeStatus{
State: api.NodeStatus_READY,
},
},
}
for _, node := range nodes {
assert.NoError(t, store.CreateNode(tx, node))
}
// task1 is assigned to node1
// task2 - task3 are assigned to node2
// task4 - task6 are assigned to node3
// task7 is unassigned
tasks := []*api.Task{
{
ID: "task1",
Slot: 1,
DesiredState: api.TaskStateRunning,
Status: api.TaskStatus{
State: api.TaskStateStarting,
},
ServiceAnnotations: api.Annotations{
Name: "task1",
},
ServiceID: "id1",
NodeID: "node1",
},
{
ID: "task2",
Slot: 2,
DesiredState: api.TaskStateRunning,
Status: api.TaskStatus{
State: api.TaskStateRunning,
},
ServiceAnnotations: api.Annotations{
Name: "task2",
},
ServiceID: "id1",
NodeID: "node2",
},
{
ID: "task3",
Slot: 3,
DesiredState: api.TaskStateRunning,
Status: api.TaskStatus{
State: api.TaskStateRunning,
},
ServiceAnnotations: api.Annotations{
Name: "task3",
},
ServiceID: "id1",
NodeID: "node2",
},
{
ID: "task4",
Slot: 4,
DesiredState: api.TaskStateRunning,
Status: api.TaskStatus{
State: api.TaskStateRunning,
},
ServiceAnnotations: api.Annotations{
Name: "task4",
},
ServiceID: "id1",
NodeID: "node3",
},
{
ID: "task5",
Slot: 5,
DesiredState: api.TaskStateRunning,
Status: api.TaskStatus{
State: api.TaskStateRunning,
},
ServiceAnnotations: api.Annotations{
Name: "task5",
},
ServiceID: "id1",
NodeID: "node3",
},
{
ID: "task6",
Slot: 6,
DesiredState: api.TaskStateRunning,
Status: api.TaskStatus{
State: api.TaskStateRunning,
},
ServiceAnnotations: api.Annotations{
Name: "task6",
},
ServiceID: "id1",
NodeID: "node3",
},
{
ID: "task7",
Slot: 7,
DesiredState: api.TaskStateRunning,
Status: api.TaskStatus{
State: api.TaskStateNew,
},
ServiceAnnotations: api.Annotations{
Name: "task7",
},
ServiceID: "id1",
},
}
for _, task := range tasks {
assert.NoError(t, store.CreateTask(tx, task))
}
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
// Replicas was set to 6, but we started with 7 tasks. task7 should
// be the one the orchestrator chose to shut down because it was not
// assigned yet.
observedShutdown := watchShutdownTask(t, watch)
assert.Equal(t, "task7", observedShutdown.ID)
// Now scale down to 2 instances.
err = s.Update(func(tx store.Tx) error {
s1.Spec.Mode = &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 2,
},
}
assert.NoError(t, store.UpdateService(tx, s1))
return nil
})
assert.NoError(t, err)
// Tasks should be shut down in a way that balances the remaining tasks.
// node2 and node3 should be preferred over node1 because node1's task
// is not running yet.
shutdowns := make(map[string]int)
for i := 0; i != 4; i++ {
observedShutdown := watchShutdownTask(t, watch)
shutdowns[observedShutdown.NodeID]++
}
assert.Equal(t, 1, shutdowns["node1"])
assert.Equal(t, 1, shutdowns["node2"])
assert.Equal(t, 2, shutdowns["node3"])
// There should be remaining tasks on node2 and node3.
s.View(func(readTx store.ReadTx) {
tasks, err := store.FindTasks(readTx, store.ByDesiredState(api.TaskStateRunning))
require.NoError(t, err)
require.Len(t, tasks, 2)
if tasks[0].NodeID == "node2" {
assert.Equal(t, "node3", tasks[1].NodeID)
} else {
assert.Equal(t, "node3", tasks[0].NodeID)
assert.Equal(t, "node2", tasks[1].NodeID)
}
})
}
func TestAllocator(t *testing.T) {
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
a, err := New(s)
assert.NoError(t, err)
assert.NotNil(t, a)
// Try adding some objects to store before allocator is started
assert.NoError(t, s.Update(func(tx store.Tx) error {
n1 := &api.Network{
ID: "testID1",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test1",
},
},
}
assert.NoError(t, store.CreateNetwork(tx, n1))
s1 := &api.Service{
ID: "testServiceID1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "service1",
},
Task: api.TaskSpec{
Networks: []*api.NetworkAttachmentConfig{
{
Target: "testID1",
},
},
},
Endpoint: &api.EndpointSpec{},
},
}
assert.NoError(t, store.CreateService(tx, s1))
t1 := &api.Task{
ID: "testTaskID1",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Networks: []*api.NetworkAttachment{
{
Network: n1,
},
},
}
assert.NoError(t, store.CreateTask(tx, t1))
return nil
}))
netWatch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateNetwork{}, state.EventDeleteNetwork{})
defer cancel()
taskWatch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{}, state.EventDeleteTask{})
defer cancel()
serviceWatch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateService{}, state.EventDeleteService{})
defer cancel()
// Start allocator
go func() {
assert.NoError(t, a.Run(context.Background()))
}()
// Now verify if we get network and tasks updated properly
watchNetwork(t, netWatch, false, isValidNetwork)
watchTask(t, s, taskWatch, false, isValidTask)
watchService(t, serviceWatch, false, nil)
// Add new networks/tasks/services after allocator is started.
assert.NoError(t, s.Update(func(tx store.Tx) error {
n2 := &api.Network{
ID: "testID2",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test2",
},
},
}
assert.NoError(t, store.CreateNetwork(tx, n2))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
assert.NoError(t, s.Update(func(tx store.Tx) error {
s2 := &api.Service{
ID: "testServiceID2",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "service2",
},
Networks: []*api.NetworkAttachmentConfig{
{
Target: "testID2",
},
},
Endpoint: &api.EndpointSpec{},
},
}
assert.NoError(t, store.CreateService(tx, s2))
return nil
}))
watchService(t, serviceWatch, false, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t2 := &api.Task{
ID: "testTaskID2",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
ServiceID: "testServiceID2",
DesiredState: api.TaskStateRunning,
}
assert.NoError(t, store.CreateTask(tx, t2))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
// Now try adding a task which depends on a network before adding the network.
n3 := &api.Network{
ID: "testID3",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test3",
},
},
}
assert.NoError(t, s.Update(func(tx store.Tx) error {
t3 := &api.Task{
ID: "testTaskID3",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
Networks: []*api.NetworkAttachment{
{
Network: n3,
},
},
}
assert.NoError(t, store.CreateTask(tx, t3))
return nil
}))
// Wait for a little bit of time before adding network just to
// test network is not available while task allocation is
// going through
time.Sleep(10 * time.Millisecond)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNetwork(tx, n3))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteTask(tx, "testTaskID3"))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t5 := &api.Task{
ID: "testTaskID5",
Spec: api.TaskSpec{
Networks: []*api.NetworkAttachmentConfig{
{
Target: "testID2",
},
},
},
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
ServiceID: "testServiceID2",
}
assert.NoError(t, store.CreateTask(tx, t5))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteNetwork(tx, "testID3"))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteService(tx, "testServiceID2"))
return nil
}))
watchService(t, serviceWatch, false, nil)
// Try to create a task with no network attachments and test
// that it moves to ALLOCATED state.
assert.NoError(t, s.Update(func(tx store.Tx) error {
t4 := &api.Task{
ID: "testTaskID4",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
}
assert.NoError(t, store.CreateTask(tx, t4))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
assert.NoError(t, s.Update(func(tx store.Tx) error {
n2 := store.GetNetwork(tx, "testID2")
require.NotEqual(t, nil, n2)
assert.NoError(t, store.UpdateNetwork(tx, n2))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
watchNetwork(t, netWatch, true, nil)
// Try updating task which is already allocated
assert.NoError(t, s.Update(func(tx store.Tx) error {
t2 := store.GetTask(tx, "testTaskID2")
require.NotEqual(t, nil, t2)
assert.NoError(t, store.UpdateTask(tx, t2))
return nil
}))
watchTask(t, s, taskWatch, false, isValidTask)
watchTask(t, s, taskWatch, true, nil)
// Try adding networks with conflicting network resources and
// add task which attaches to a network which gets allocated
// later and verify if task reconciles and moves to ALLOCATED.
n4 := &api.Network{
ID: "testID4",
Spec: api.NetworkSpec{
Annotations: api.Annotations{
Name: "test4",
},
DriverConfig: &api.Driver{
Name: "overlay",
Options: map[string]string{
"com.docker.network.driver.overlay.vxlanid_list": "328",
},
},
},
}
n5 := n4.Copy()
n5.ID = "testID5"
n5.Spec.Annotations.Name = "test5"
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNetwork(tx, n4))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNetwork(tx, n5))
return nil
}))
watchNetwork(t, netWatch, true, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t6 := &api.Task{
ID: "testTaskID6",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
DesiredState: api.TaskStateRunning,
Networks: []*api.NetworkAttachment{
{
Network: n5,
},
},
}
assert.NoError(t, store.CreateTask(tx, t6))
return nil
}))
watchTask(t, s, taskWatch, true, nil)
// Now remove the conflicting network.
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteNetwork(tx, n4.ID))
return nil
}))
watchNetwork(t, netWatch, false, isValidNetwork)
watchTask(t, s, taskWatch, false, isValidTask)
// Try adding services with conflicting port configs and add
// task which is part of the service whose allocation hasn't
// happened and when that happens later and verify if task
// reconciles and moves to ALLOCATED.
s3 := &api.Service{
ID: "testServiceID3",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "service3",
},
Endpoint: &api.EndpointSpec{
Ports: []*api.PortConfig{
{
Name: "http",
TargetPort: 80,
PublishedPort: 8080,
},
},
},
},
}
s4 := s3.Copy()
s4.ID = "testServiceID4"
s4.Spec.Annotations.Name = "service4"
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, s3))
return nil
}))
watchService(t, serviceWatch, false, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, s4))
return nil
}))
watchService(t, serviceWatch, true, nil)
assert.NoError(t, s.Update(func(tx store.Tx) error {
t7 := &api.Task{
ID: "testTaskID7",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
ServiceID: "testServiceID4",
DesiredState: api.TaskStateRunning,
}
assert.NoError(t, store.CreateTask(tx, t7))
return nil
}))
watchTask(t, s, taskWatch, true, nil)
// Now remove the conflicting service.
assert.NoError(t, s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteService(tx, s3.ID))
return nil
}))
watchService(t, serviceWatch, false, nil)
watchTask(t, s, taskWatch, false, isValidTask)
a.Stop()
}
func TestOrchestratorRestartDelay(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/)
defer cancel()
// Create a service with two instances specified before the orchestrator is
// started. This should result in two tasks when the orchestrator
// starts up.
err := s.Update(func(tx store.Tx) error {
j1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
Restart: &api.RestartPolicy{
Condition: api.RestartOnAny,
Delay: ptypes.DurationProto(100 * time.Millisecond),
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 2,
},
},
},
}
assert.NoError(t, store.CreateService(tx, j1))
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
observedTask1 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask1.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1")
observedTask2 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask2.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1")
// Fail the first task. Confirm that it gets restarted.
updatedTask1 := observedTask1.Copy()
updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed}
before := time.Now()
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask1))
return nil
})
assert.NoError(t, err)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
observedTask3 := watchTaskCreate(t, watch)
expectCommit(t, watch)
assert.Equal(t, observedTask3.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady)
assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1")
observedTask4 := watchTaskUpdate(t, watch)
after := time.Now()
// At least 100 ms should have elapsed. Only check the lower bound,
// because the system may be slow and it could have taken longer.
if after.Sub(before) < 100*time.Millisecond {
t.Fatalf("restart delay should have elapsed. Got: %v", after.Sub(before))
}
assert.Equal(t, observedTask4.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning)
assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1")
}
func TestUpdaterRollback(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
var (
failImage1 uint32
failImage2 uint32
)
watchCreate, cancelCreate := state.Watch(s.WatchQueue(), state.EventCreateTask{})
defer cancelCreate()
watchServiceUpdate, cancelServiceUpdate := state.Watch(s.WatchQueue(), state.EventUpdateService{})
defer cancelServiceUpdate()
// Fail new tasks the updater tries to run
watchUpdate, cancelUpdate := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancelUpdate()
go func() {
failedLast := false
for {
select {
case e := <-watchUpdate:
task := e.(state.EventUpdateTask).Task
if task.DesiredState == task.Status.State {
continue
}
if task.DesiredState == api.TaskStateRunning && task.Status.State != api.TaskStateFailed && task.Status.State != api.TaskStateRunning {
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
// Never fail two image2 tasks in a row, so there's a mix of
// failed and successful tasks for the rollback.
if task.Spec.GetContainer().Image == "image1" && atomic.LoadUint32(&failImage1) == 1 {
task.Status.State = api.TaskStateFailed
failedLast = true
} else if task.Spec.GetContainer().Image == "image2" && atomic.LoadUint32(&failImage2) == 1 && !failedLast {
task.Status.State = api.TaskStateFailed
failedLast = true
} else {
task.Status.State = task.DesiredState
failedLast = false
}
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
} else if task.DesiredState > api.TaskStateRunning {
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = task.DesiredState
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
}
}
}
}()
// Create a service with four replicas specified before the orchestrator
// is started. This should result in two tasks when the orchestrator
// starts up.
err := s.Update(func(tx store.Tx) error {
s1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "image1",
},
},
Restart: &api.RestartPolicy{
Condition: api.RestartOnNone,
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 4,
},
},
Update: &api.UpdateConfig{
FailureAction: api.UpdateConfig_ROLLBACK,
Parallelism: 1,
Delay: *ptypes.DurationProto(10 * time.Millisecond),
Monitor: ptypes.DurationProto(500 * time.Millisecond),
MaxFailureRatio: 0.4,
},
},
}
assert.NoError(t, store.CreateService(tx, s1))
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
observedTask := testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
atomic.StoreUint32(&failImage2, 1)
// Start a rolling update
err = s.Update(func(tx store.Tx) error {
s1 := store.GetService(tx, "id1")
require.NotNil(t, s1)
s1.PreviousSpec = s1.Spec.Copy()
s1.UpdateStatus = nil
s1.Spec.Task.GetContainer().Image = "image2"
assert.NoError(t, store.UpdateService(tx, s1))
return nil
})
assert.NoError(t, err)
// Should see three tasks started, then a rollback
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2")
// Should get to the ROLLBACK_STARTED state
for {
e := <-watchServiceUpdate
if e.(state.EventUpdateService).Service.UpdateStatus == nil {
continue
}
if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_STARTED {
break
}
}
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
// Should end up in ROLLBACK_COMPLETED state
for {
e := <-watchServiceUpdate
if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_COMPLETED {
break
}
}
atomic.StoreUint32(&failImage1, 1)
// Repeat the rolling update but this time fail the tasks that the
// rollback creates. It should end up in ROLLBACK_PAUSED.
err = s.Update(func(tx store.Tx) error {
s1 := store.GetService(tx, "id1")
require.NotNil(t, s1)
s1.PreviousSpec = s1.Spec.Copy()
s1.UpdateStatus = nil
s1.Spec.Task.GetContainer().Image = "image2"
assert.NoError(t, store.UpdateService(tx, s1))
return nil
})
assert.NoError(t, err)
// Should see three tasks started, then a rollback
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2")
// Should get to the ROLLBACK_STARTED state
for {
e := <-watchServiceUpdate
if e.(state.EventUpdateService).Service.UpdateStatus == nil {
continue
}
if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_STARTED {
break
}
}
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
observedTask = testutils.WatchTaskCreate(t, watchCreate)
assert.Equal(t, observedTask.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1")
// Should end up in ROLLBACK_PAUSED state
for {
e := <-watchServiceUpdate
if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_PAUSED {
break
}
}
}
func TestOrchestratorRestartWindow(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/)
defer cancel()
// Create a service with two instances specified before the orchestrator is
// started. This should result in two tasks when the orchestrator
// starts up.
err := s.Update(func(tx store.Tx) error {
j1 := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 2,
},
},
Task: api.TaskSpec{
Restart: &api.RestartPolicy{
Condition: api.RestartOnAny,
Delay: ptypes.DurationProto(100 * time.Millisecond),
MaxAttempts: 1,
Window: ptypes.DurationProto(500 * time.Millisecond),
},
},
},
}
assert.NoError(t, store.CreateService(tx, j1))
return nil
})
assert.NoError(t, err)
// Start the orchestrator.
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
observedTask1 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask1.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1")
observedTask2 := watchTaskCreate(t, watch)
assert.Equal(t, observedTask2.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1")
// Fail the first task. Confirm that it gets restarted.
updatedTask1 := observedTask1.Copy()
updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed}
before := time.Now()
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask1))
return nil
})
assert.NoError(t, err)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
observedTask3 := watchTaskCreate(t, watch)
expectCommit(t, watch)
assert.Equal(t, observedTask3.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady)
assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1")
observedTask4 := watchTaskUpdate(t, watch)
after := time.Now()
// At least 100 ms should have elapsed. Only check the lower bound,
// because the system may be slow and it could have taken longer.
if after.Sub(before) < 100*time.Millisecond {
t.Fatal("restart delay should have elapsed")
}
assert.Equal(t, observedTask4.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning)
assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1")
// Fail the second task. Confirm that it gets restarted.
updatedTask2 := observedTask2.Copy()
updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed}
before = time.Now()
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask2))
return nil
})
assert.NoError(t, err)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
observedTask5 := watchTaskCreate(t, watch)
expectCommit(t, watch)
assert.Equal(t, observedTask5.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask5.DesiredState, api.TaskStateReady)
assert.Equal(t, observedTask5.ServiceAnnotations.Name, "name1")
observedTask6 := watchTaskUpdate(t, watch) // task gets started after a delay
expectCommit(t, watch)
assert.Equal(t, observedTask6.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask6.DesiredState, api.TaskStateRunning)
assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name1")
// Fail the first instance again. It should not be restarted.
updatedTask1 = observedTask3.Copy()
updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed}
before = time.Now()
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask1))
return nil
})
assert.NoError(t, err)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
select {
case <-watch:
t.Fatal("got unexpected event")
case <-time.After(200 * time.Millisecond):
}
time.Sleep(time.Second)
// Fail the second instance again. It should get restarted because
// enough time has elapsed since the last restarts.
updatedTask2 = observedTask5.Copy()
updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed}
before = time.Now()
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.UpdateTask(tx, updatedTask2))
return nil
})
assert.NoError(t, err)
expectTaskUpdate(t, watch)
expectCommit(t, watch)
expectTaskUpdate(t, watch)
observedTask7 := watchTaskCreate(t, watch)
expectCommit(t, watch)
assert.Equal(t, observedTask7.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask7.DesiredState, api.TaskStateReady)
observedTask8 := watchTaskUpdate(t, watch)
after = time.Now()
// At least 100 ms should have elapsed. Only check the lower bound,
// because the system may be slow and it could have taken longer.
if after.Sub(before) < 100*time.Millisecond {
t.Fatal("restart delay should have elapsed")
}
assert.Equal(t, observedTask8.Status.State, api.TaskStateNew)
assert.Equal(t, observedTask8.DesiredState, api.TaskStateRunning)
assert.Equal(t, observedTask8.ServiceAnnotations.Name, "name1")
}
func TestDrain(t *testing.T) {
ctx := context.Background()
initialService := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{},
},
Restart: &api.RestartPolicy{
Condition: api.RestartOnNone,
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: 6,
},
},
},
}
initialNodeSet := []*api.Node{
{
ID: "id1",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name1",
},
Availability: api.NodeAvailabilityActive,
},
Status: api.NodeStatus{
State: api.NodeStatus_READY,
},
},
{
ID: "id2",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name2",
},
Availability: api.NodeAvailabilityActive,
},
Status: api.NodeStatus{
State: api.NodeStatus_DOWN,
},
},
// We should NOT kick out tasks on UNKNOWN nodes.
{
ID: "id3",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name3",
},
Availability: api.NodeAvailabilityActive,
},
Status: api.NodeStatus{
State: api.NodeStatus_UNKNOWN,
},
},
{
ID: "id4",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name4",
},
Availability: api.NodeAvailabilityPause,
},
Status: api.NodeStatus{
State: api.NodeStatus_READY,
},
},
{
ID: "id5",
Spec: api.NodeSpec{
Annotations: api.Annotations{
Name: "name5",
},
Availability: api.NodeAvailabilityDrain,
},
Status: api.NodeStatus{
State: api.NodeStatus_READY,
},
},
}
initialTaskSet := []*api.Task{
// Task not assigned to any node
{
ID: "id0",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Slot: 1,
ServiceAnnotations: api.Annotations{
Name: "name0",
},
ServiceID: "id1",
},
// Tasks assigned to the nodes defined above
{
ID: "id1",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Slot: 2,
ServiceAnnotations: api.Annotations{
Name: "name1",
},
ServiceID: "id1",
NodeID: "id1",
},
{
ID: "id2",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Slot: 3,
ServiceAnnotations: api.Annotations{
Name: "name2",
},
ServiceID: "id1",
NodeID: "id2",
},
{
ID: "id3",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Slot: 4,
ServiceAnnotations: api.Annotations{
Name: "name3",
},
ServiceID: "id1",
NodeID: "id3",
},
{
ID: "id4",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Slot: 5,
ServiceAnnotations: api.Annotations{
Name: "name4",
},
ServiceID: "id1",
NodeID: "id4",
},
{
ID: "id5",
Status: api.TaskStatus{
State: api.TaskStateNew,
},
Slot: 6,
ServiceAnnotations: api.Annotations{
Name: "name5",
},
ServiceID: "id1",
NodeID: "id5",
},
}
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
err := s.Update(func(tx store.Tx) error {
// Prepopulate service
assert.NoError(t, store.CreateService(tx, initialService))
// Prepoulate nodes
for _, n := range initialNodeSet {
assert.NoError(t, store.CreateNode(tx, n))
}
// Prepopulate tasks
for _, task := range initialTaskSet {
assert.NoError(t, store.CreateTask(tx, task))
}
return nil
})
assert.NoError(t, err)
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
orchestrator := NewReplicatedOrchestrator(s)
defer orchestrator.Stop()
go func() {
assert.NoError(t, orchestrator.Run(ctx))
}()
// id2 and id5 should be killed immediately
deletion1 := watchShutdownTask(t, watch)
deletion2 := watchShutdownTask(t, watch)
assert.Regexp(t, "id(2|5)", deletion1.ID)
assert.Regexp(t, "id(2|5)", deletion1.NodeID)
assert.Regexp(t, "id(2|5)", deletion2.ID)
assert.Regexp(t, "id(2|5)", deletion2.NodeID)
// Create a new task, assigned to node id2
err = s.Update(func(tx store.Tx) error {
task := initialTaskSet[2].Copy()
task.ID = "newtask"
task.NodeID = "id2"
assert.NoError(t, store.CreateTask(tx, task))
return nil
})
assert.NoError(t, err)
deletion3 := watchShutdownTask(t, watch)
assert.Equal(t, "newtask", deletion3.ID)
assert.Equal(t, "id2", deletion3.NodeID)
// Set node id4 to the DRAINED state
err = s.Update(func(tx store.Tx) error {
n := initialNodeSet[3].Copy()
n.Spec.Availability = api.NodeAvailabilityDrain
assert.NoError(t, store.UpdateNode(tx, n))
return nil
})
assert.NoError(t, err)
deletion4 := watchShutdownTask(t, watch)
assert.Equal(t, "id4", deletion4.ID)
assert.Equal(t, "id4", deletion4.NodeID)
// Delete node id1
err = s.Update(func(tx store.Tx) error {
assert.NoError(t, store.DeleteNode(tx, "id1"))
return nil
})
assert.NoError(t, err)
deletion5 := watchShutdownTask(t, watch)
assert.Equal(t, "id1", deletion5.ID)
assert.Equal(t, "id1", deletion5.NodeID)
}
func TestUpdater(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
// Move tasks to their desired state.
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
go func() {
for {
select {
case e := <-watch:
task := e.(state.EventUpdateTask).Task
if task.Status.State == task.DesiredState {
continue
}
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = task.DesiredState
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
}
}
}()
instances := 3
cluster := &api.Cluster{
// test cluster configuration propagation to task creation.
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: "default",
},
},
}
service := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: uint64(instances),
},
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "v:1",
},
},
},
Update: &api.UpdateConfig{
// avoid having Run block for a long time to watch for failures
Monitor: gogotypes.DurationProto(50 * time.Millisecond),
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateCluster(tx, cluster))
assert.NoError(t, store.CreateService(tx, service))
for i := 0; i < instances; i++ {
assert.NoError(t, store.CreateTask(tx, orchestrator.NewTask(cluster, service, uint64(i), "")))
}
return nil
})
assert.NoError(t, err)
originalTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range originalTasks {
for _, task := range slot {
assert.Equal(t, "v:1", task.Spec.GetContainer().Image)
assert.Nil(t, task.LogDriver) // should be left alone
}
}
service.Spec.Task.GetContainer().Image = "v:2"
service.Spec.Task.LogDriver = &api.Driver{Name: "tasklogdriver"}
updater := NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range updatedTasks {
for _, task := range slot {
assert.Equal(t, "v:2", task.Spec.GetContainer().Image)
assert.Equal(t, service.Spec.Task.LogDriver, task.LogDriver) // pick up from task
}
}
service.Spec.Task.GetContainer().Image = "v:3"
cluster.Spec.TaskDefaults.LogDriver = &api.Driver{Name: "clusterlogdriver"} // make cluster default logdriver.
service.Spec.Update = &api.UpdateConfig{
Parallelism: 1,
Monitor: gogotypes.DurationProto(50 * time.Millisecond),
}
updater = NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
for _, slot := range updatedTasks {
for _, task := range slot {
assert.Equal(t, "v:3", task.Spec.GetContainer().Image)
assert.Equal(t, service.Spec.Task.LogDriver, task.LogDriver) // still pick up from task
}
}
service.Spec.Task.GetContainer().Image = "v:4"
service.Spec.Task.LogDriver = nil // use cluster default now.
service.Spec.Update = &api.UpdateConfig{
Parallelism: 1,
Delay: 10 * time.Millisecond,
Monitor: gogotypes.DurationProto(50 * time.Millisecond),
}
updater = NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
for _, slot := range updatedTasks {
for _, task := range slot {
assert.Equal(t, "v:4", task.Spec.GetContainer().Image)
assert.Equal(t, cluster.Spec.TaskDefaults.LogDriver, task.LogDriver) // pick up from cluster
}
}
}