func TestMethods(t *testing.T) {
bus := setup(t)
errc := make(chan error, 1)
instA := flux.InstanceID("steamy-windows-89")
mockA := &platform.MockPlatform{
AllServicesAnswer: []platform.Service{platform.Service{}},
RegradeError: platform.RegradeError{flux.ServiceID("foo/bar"): errors.New("foo barred")},
}
subscribe(t, bus, errc, instA, mockA)
plat, err := bus.Connect(instA)
if err != nil {
t.Fatal(err)
}
ss, err := plat.AllServices("", nil)
if err != nil {
t.Fatal(err)
}
if len(mockA.AllServicesAnswer) != len(ss) {
t.Fatalf("Expected %d result, got %d", len(mockA.AllServicesAnswer), len(ss))
}
err = plat.Regrade([]platform.RegradeSpec{})
if _, ok := err.(platform.RegradeError); !ok {
t.Fatalf("expected RegradeError, got %+v", err)
}
mockB := &platform.MockPlatform{
AllServicesError: errors.New("just didn't feel like it"),
SomeServicesAnswer: []platform.Service{platform.Service{}, platform.Service{}},
}
instB := flux.InstanceID("smokey-water-72")
subscribe(t, bus, errc, instB, mockB)
platB, err := bus.Connect(instB)
if err != nil {
t.Fatal(err)
}
ss, err = platB.SomeServices([]flux.ServiceID{})
if err != nil {
t.Fatal(err)
}
if len(mockB.SomeServicesAnswer) != len(ss) {
t.Fatalf("Expected %d result, got %d", len(mockB.SomeServicesAnswer), len(ss))
}
ss, err = platB.AllServices("", nil)
if err == nil {
t.Fatal("expected error but didn't get one")
}
close(errc)
err = <-errc
if err != nil {
t.Fatalf("expected nil from subscription channel, but got err %v", err)
}
}
func TestHistoryLog(t *testing.T) {
instance := flux.InstanceID("instance")
db := newSQL(t)
defer db.Close()
bailIfErr(t, db.LogEvent(instance, "namespace", "service", "event 1"))
bailIfErr(t, db.LogEvent(instance, "namespace", "other", "event 3"))
bailIfErr(t, db.LogEvent(instance, "namespace", "service", "event 2"))
es, err := db.EventsForService(instance, "namespace", "service")
if err != nil {
t.Fatal(err)
}
if len(es) != 2 {
t.Fatalf("Expected 2 events, got %d\n", len(es))
}
checkInDescOrder(t, es)
es, err = db.AllEvents(instance)
if err != nil {
t.Fatal(err)
}
if len(es) != 3 {
t.Fatalf("Expected 3 events, got %#v\n", es)
}
checkInDescOrder(t, es)
}
func TestPing(t *testing.T) {
bus := setup(t)
errc := make(chan error)
instID := flux.InstanceID("wirey-bird-68")
platA := &platform.MockPlatform{}
subscribe(t, bus, errc, instID, platA)
// AwaitPresence uses Ping, so we have to install our error after
// subscribe succeeds.
platA.PingError = platform.FatalError{errors.New("ping problem")}
if err := platA.Ping(); err == nil {
t.Fatalf("expected error from directly calling ping, got nil")
}
err := bus.Ping(instID)
if err == nil {
t.Errorf("expected error from ping, got nil")
} else if err.Error() != "ping problem" {
t.Errorf("got the wrong error: %s", err.Error())
}
select {
case err := <-errc:
if err == nil {
t.Fatal("expected error return from subscription but didn't get one")
}
case <-time.After(100 * time.Millisecond):
t.Fatal("expected error return from subscription but didn't get one")
}
}
func TestFatalErrorDisconnects(t *testing.T) {
bus := setup(t)
errc := make(chan error)
instA := flux.InstanceID("golden-years-75")
mockA := &platform.MockPlatform{
SomeServicesError: platform.FatalError{errors.New("Disaster.")},
}
subscribe(t, bus, errc, instA, mockA)
plat, err := bus.Connect(instA)
if err != nil {
t.Fatal(err)
}
_, err = plat.SomeServices([]flux.ServiceID{})
if err == nil {
t.Error("expected error, got nil")
} else if _, ok := err.(platform.FatalError); !ok {
t.Errorf("expected platform.FatalError, got %v", err)
}
select {
case err = <-errc:
if err == nil {
t.Error("expected error from subscription being killed, got nil")
}
case <-time.After(1 * time.Second):
t.Error("timed out waiting for expected error from subscription closing")
}
}
func TestNewConnectionKicks(t *testing.T) {
bus := setup(t)
instA := flux.InstanceID("foo")
mockA := &platform.MockPlatform{}
errA := make(chan error)
subscribe(t, bus, errA, instA, mockA)
mockB := &platform.MockPlatform{}
errB := make(chan error)
subscribe(t, bus, errB, instA, mockB)
select {
case <-errA:
break
case <-time.After(1 * time.Second):
t.Error("timed out waiting for connection to be kicked")
}
close(errB)
err := <-errB
if err != nil {
t.Errorf("expected no error from second connection, but got %q", err)
}
}
func getInstanceID(req *http.Request) flux.InstanceID {
s := req.Header.Get(flux.InstanceIDHeaderKey)
if s == "" {
return flux.DefaultInstanceID
}
return flux.InstanceID(s)
}
func (db *DB) All() ([]instance.NamedConfig, error) {
rows, err := db.conn.Query(`SELECT instance, config FROM config`)
if err != nil {
return nil, err
}
defer rows.Close()
instances := []instance.NamedConfig{}
for rows.Next() {
var (
id, confStr string
conf instance.Config
)
err = rows.Scan(&id, &confStr)
if err == nil {
err = json.Unmarshal([]byte(confStr), &conf)
}
if err != nil {
return nil, err
}
instances = append(instances, instance.NamedConfig{
ID: flux.InstanceID(id),
Config: conf,
})
}
return instances, rows.Err()
}
func TestJobEncodingDecoding(t *testing.T) {
// Check it can serialize/deserialize release jobs
now := time.Now().UTC()
expected := Job{
Instance: flux.InstanceID("instance"),
ID: NewJobID(),
Queue: DefaultQueue,
Method: ReleaseJob,
Params: ReleaseJobParams{
ServiceSpec: flux.ServiceSpecAll,
ImageSpec: flux.ImageSpecLatest,
Kind: flux.ReleaseKindExecute,
},
ScheduledAt: now,
Priority: PriorityInteractive,
Key: "key1",
Submitted: now,
Claimed: now,
Heartbeat: now,
Finished: now,
Log: []string{"log1"},
Status: "status",
Done: true,
Success: true,
}
b, err := json.Marshal(expected)
bailIfErr(t, err)
var got Job
bailIfErr(t, json.Unmarshal(b, &got))
if !reflect.DeepEqual(got, expected) {
t.Errorf("got %q, expected %q", got, expected)
}
}
func TestUpdateOK(t *testing.T) {
db := newDB(t)
inst := flux.InstanceID("floaty-womble-abc123")
service := flux.MakeServiceID("namespace", "service")
services := map[flux.ServiceID]instance.ServiceConfig{}
services[service] = instance.ServiceConfig{
Automated: true,
Locked: true,
}
c := instance.Config{
Services: services,
}
err := db.UpdateConfig(inst, func(_ instance.Config) (instance.Config, error) {
return c, nil
})
if err != nil {
t.Fatal(err)
}
c1, err := db.GetConfig(inst)
if err != nil {
t.Fatal(err)
}
if _, found := c1.Services[service]; !found {
t.Fatalf("did not find instance config after setting")
}
if !c1.Services[service].Automated {
t.Fatalf("expected service config %#v, got %#v", c.Services[service], c1.Services[service])
}
if !c1.Services[service].Locked {
t.Fatalf("expected service config %#v, got %#v", c.Services[service], c1.Services[service])
}
}
func TestStandaloneMessageBus(t *testing.T) {
instID := flux.InstanceID("instance")
bus := NewStandaloneMessageBus(NewBusMetrics())
p := &MockPlatform{}
done := make(chan error)
bus.Subscribe(instID, p, done)
if err := bus.Ping(instID); err != nil {
t.Fatal(err)
}
// subscribing another connection kicks the first one off
p2 := &MockPlatform{PingError: errors.New("ping failed")}
done2 := make(chan error, 2)
bus.Subscribe(instID, p2, done2)
select {
case <-done:
break
case <-time.After(1 * time.Second):
t.Error("expected connection to be kicked when subsequent connection arrived, but it wasn't")
}
err := bus.Ping(instID)
if err == nil {
t.Error("expected error from pinging mock platform, but got nil")
}
done2 <- nil
err = <-done2
if err != nil {
t.Error("did not expect subscription error after application-level error")
}
// Now test that a FatalError does shut the subscription down
p3 := &MockPlatform{PingError: FatalError{errors.New("ping failed")}}
done3 := make(chan error)
bus.Subscribe(instID, p3, done3)
select {
case <-done2:
break
case <-time.After(1 * time.Second):
t.Error("expected connection to be kicked when subsequent connection arrived, but it wasn't")
}
err = bus.Ping(instID)
if err == nil {
t.Error("expected error from pinging mock platform, but got nil")
}
select {
case <-done3:
break
case <-time.After(1 * time.Second):
t.Error("expected error from connection on error, got none")
}
}
func TestDatabaseStoreExpiresHeartbeatedButCrashedJobs(t *testing.T) {
instance := flux.InstanceID("instance")
db := Setup(t)
defer Cleanup(t, db)
// Mock time, so we can mess around with it
now := time.Now()
db.now = func(_ dbProxy) (time.Time, error) {
return now, nil
}
// Put a job
jobID, err := db.PutJob(instance, Job{
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: PriorityInteractive,
})
bailIfErr(t, err)
// Take it, so it is claimed
_, err = db.NextJob(nil)
bailIfErr(t, err)
// Heartbeat the job
now = now.Add(1 * time.Minute)
bailIfErr(t, db.Heartbeat(jobID))
// GC should not remove it (heartbeat should keep it alive longer)
now = now.Add(30 * time.Second)
bailIfErr(t, db.GC())
_, err = db.GetJob(instance, jobID)
bailIfErr(t, err)
// GC should remove it after gc time
now = now.Add(2 * time.Minute)
bailIfErr(t, db.GC())
// - should be removed
_, err = db.GetJob(instance, jobID)
if err != ErrNoSuchJob {
t.Errorf("expected ErrNoSuchJob, got %q", err)
}
}
"github.com/spf13/pflag"
"github.com/weaveworks/flux"
"github.com/weaveworks/flux/api"
transport "github.com/weaveworks/flux/http"
)
type rootOpts struct {
URL string
Token string
API api.ClientService
}
// fluxctl never sends an instance ID directly; it's always blank, and
// optionally gets populated by an intermediating authfe from the token.
const noInstanceID = flux.InstanceID("")
type serviceOpts struct {
*rootOpts
}
func newService(parent *rootOpts) *serviceOpts {
return &serviceOpts{rootOpts: parent}
}
func newRoot() *rootOpts {
return &rootOpts{}
}
var rootLongHelp = strings.TrimSpace(`
fluxctl helps you deploy your code.
// Take the next job from specified queues. If queues is nil, all queues are
// used.
func (s *DatabaseStore) NextJob(queues []string) (Job, error) {
var job Job
err := s.Transaction(func(s *DatabaseStore) error {
now, err := s.now(s.conn)
if err != nil {
return errors.Wrap(err, "getting current time")
}
var (
instanceID string
jobID string
queue string
method string
paramsBytes []byte
scheduledAt time.Time
priority int
key string
submittedAt time.Time
claimedAt nullTime
heartbeatAt nullTime
finishedAt nullTime
logStr string
status string
done sql.NullBool
success sql.NullBool
)
if err := s.conn.QueryRow(`
SELECT instance_id, id, queue, method, params,
scheduled_at, priority, key, submitted_at,
claimed_at, heartbeat_at, finished_at, log, status,
done, success
FROM jobs
-- Only unclaimed/unfinished jobs are available
WHERE claimed_at IS NULL
AND finished_at IS NULL
-- Don't make jobs available until after they are scheduled
AND scheduled_at <= $1
-- Only one job at a time per instance
AND instance_id NOT IN (
SELECT instance_id
FROM jobs
WHERE claimed_at IS NOT NULL
AND finished_at IS NULL
GROUP BY instance_id
)
-- subtraction is to work around for ql, not being able to sort
-- multiple columns in different ways.
ORDER BY (-1 * priority), scheduled_at, submitted_at
LIMIT 1`,
now,
).Scan(
&instanceID,
&jobID,
&queue,
&method,
¶msBytes,
&scheduledAt,
&priority,
&key,
&submittedAt,
&claimedAt,
&heartbeatAt,
&finishedAt,
&logStr,
&status,
&done,
&success,
); err == sql.ErrNoRows {
return ErrNoJobAvailable
} else if err != nil {
return errors.Wrap(err, "dequeueing next job")
}
params, err := s.scanParams(method, paramsBytes)
if err != nil {
return errors.Wrap(err, "unmarshaling params")
}
var log []string
if err := json.NewDecoder(strings.NewReader(logStr)).Decode(&log); err != nil {
return errors.Wrap(err, "unmarshaling log")
}
job = Job{
Instance: flux.InstanceID(instanceID),
ID: JobID(jobID),
Queue: queue,
Method: method,
Params: params,
ScheduledAt: scheduledAt,
Priority: priority,
Key: key,
Submitted: submittedAt,
Claimed: claimedAt.Time,
Heartbeat: heartbeatAt.Time,
Finished: finishedAt.Time,
Log: log,
Status: status,
Done: done.Bool,
Success: success.Bool,
}
if res, err := s.conn.Exec(`
UPDATE jobs
SET claimed_at = $1
WHERE id = $2
AND instance_id = $3
`, now, jobID, instanceID); err != nil {
return errors.Wrap(err, "marking job as claimed")
} else if n, err := res.RowsAffected(); err != nil {
return errors.Wrap(err, "after update, checking affected rows")
} else if n != 1 {
return errors.Errorf("wanted to affect 1 row; affected %d", n)
}
return nil
})
return job, err
}
func TestDatabaseStore(t *testing.T) {
instance := flux.InstanceID("instance")
instance2 := flux.InstanceID("instance2")
db := Setup(t)
defer Cleanup(t, db)
// Get a job when there are none
_, err := db.NextJob(nil)
if err != ErrNoJobAvailable {
t.Fatalf("Expected ErrNoJobAvailable, got %q", err)
}
// Put some jobs
backgroundJobID, err := db.PutJob(instance2, Job{
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: PriorityBackground,
})
bailIfErr(t, err)
interactiveJobID, err := db.PutJob(instance, Job{
Key: "2",
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: PriorityInteractive,
})
bailIfErr(t, err)
// Put a duplicate
duplicateID, err := db.PutJob(instance, Job{
Key: "2",
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: PriorityInteractive,
})
if err != ErrJobAlreadyQueued {
t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
}
if string(duplicateID) != "" {
t.Errorf("Expected no id for duplicate job, got: %q", duplicateID)
}
// Take one
interactiveJob, err := db.NextJob(nil)
bailIfErr(t, err)
// - It should be the highest priority
if interactiveJob.ID != interactiveJobID {
t.Errorf("Got a lower priority job when a higher one was available")
}
// - It should have a default queue
if interactiveJob.Queue != DefaultQueue {
t.Errorf("job default queue (%q) was not expected (%q)", interactiveJob.Queue, DefaultQueue)
}
// - It should have been scheduled in the past
now, err := db.now(db.conn)
bailIfErr(t, err)
if interactiveJob.ScheduledAt.IsZero() || interactiveJob.ScheduledAt.After(now) {
t.Errorf("expected job to be scheduled in the past")
}
// - It should have a log and status
if len(interactiveJob.Log) == 0 || interactiveJob.Status == "" {
t.Errorf("expected job to have a log and status")
}
// Put a duplicate (when existing is claimed, but not finished)
// - It should fail
_, err = db.PutJob(instance, Job{
Key: "2",
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: 1, // low priority, so it won't interfere with other jobs
})
if err != ErrJobAlreadyQueued {
t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
}
// Put a duplicate (For another instance)
// - It should succeed
_, err = db.PutJob(instance2, Job{
Key: "2",
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: 1, // low priority, so it won't interfere with other jobs
})
bailIfErr(t, err)
// Put a duplicate (Ignoring duplicates)
// - It should succeed
_, err = db.PutJobIgnoringDuplicates(instance, Job{
Key: "2",
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: 1, // low priority, so it won't interfere with other jobs
})
bailIfErr(t, err)
// Update the job
newStatus := "Being used in testing"
interactiveJob.Status = newStatus
interactiveJob.Log = append(interactiveJob.Log, newStatus)
bailIfErr(t, db.UpdateJob(interactiveJob))
// - It should have saved the changes
interactiveJob, err = db.GetJob(instance, interactiveJobID)
bailIfErr(t, err)
if interactiveJob.Status != newStatus || len(interactiveJob.Log) != 2 || interactiveJob.Log[1] != interactiveJob.Status {
t.Errorf("expected job to have new log and status")
}
// Heartbeat the job
oldHeartbeat := interactiveJob.Heartbeat
bailIfErr(t, db.Heartbeat(interactiveJobID))
// - Heartbeat time should be updated
interactiveJob, err = db.GetJob(instance, interactiveJobID)
bailIfErr(t, err)
if !interactiveJob.Heartbeat.After(oldHeartbeat) {
t.Errorf("expected job heartbeat to have been updated")
}
// Take the next
backgroundJob, err := db.NextJob(nil)
bailIfErr(t, err)
// - It should be different
if backgroundJob.ID != backgroundJobID {
t.Errorf("Got a different job than expected")
}
// Finish one
backgroundJob.Done = true
backgroundJob.Success = true
bailIfErr(t, db.UpdateJob(backgroundJob))
// - Status should be changed
backgroundJob, err = db.GetJob(instance2, backgroundJobID)
bailIfErr(t, err)
if !backgroundJob.Done || !backgroundJob.Success {
t.Errorf("expected job to have been marked as done")
}
// GC
// - Advance time so we can gc stuff
db.now = func(_ dbProxy) (time.Time, error) {
return time.Now().Add(2 * time.Minute), nil
}
bailIfErr(t, db.GC())
// - Finished should be removed
_, err = db.GetJob(instance, backgroundJobID)
if err != ErrNoSuchJob {
t.Errorf("expected ErrNoSuchJob, got %q", err)
}
}
func TestDatabaseStoreFairScheduling(t *testing.T) {
instance1 := flux.InstanceID("instance1")
instance2 := flux.InstanceID("instance2")
db := Setup(t)
defer Cleanup(t, db)
// Put some jobs for instance 1
job1ID, err := db.PutJob(instance1, Job{
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: PriorityInteractive,
})
bailIfErr(t, err)
job2ID, err := db.PutJob(instance1, Job{
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: PriorityInteractive,
})
bailIfErr(t, err)
// Put a job for instance 2
job3ID, err := db.PutJob(instance2, Job{
Method: ReleaseJob,
Params: ReleaseJobParams{},
Priority: PriorityInteractive,
})
bailIfErr(t, err)
// Take one
// - It should be instance1's first job
job1, err := db.NextJob(nil)
bailIfErr(t, err)
if job1.ID != job1ID {
t.Errorf("Got a newer job when an older one was available")
}
// Take another (while instance1 has one in-progress)
// - It should be instance2's first job
job3, err := db.NextJob(nil)
bailIfErr(t, err)
if job3.ID != job3ID {
t.Errorf("Got an unexpected job id")
}
// Take another (while instance1, and instance2 has one in-progress)
// - It should say none are available, because both are in-progress
_, err = db.NextJob(nil)
if err != ErrNoJobAvailable {
t.Fatalf("Expected ErrNoJobAvailable, got %q", err)
}
// Finish instance1's job
job1.Done = true
job1.Success = true
bailIfErr(t, db.UpdateJob(job1))
// - Status should be changed
job1, err = db.GetJob(instance1, job1ID)
bailIfErr(t, err)
if !job1.Done || !job1.Success {
t.Errorf("expected job to have been marked as done")
}
// Take another
// - It should be instance1's next job
job2, err := db.NextJob(nil)
bailIfErr(t, err)
// - It should be the next job for instance1
if job2.ID != job2ID {
t.Errorf("Got an unexpected job id")
}
}
func TestDatabaseStoreScheduledJobs(t *testing.T) {
instance := flux.InstanceID("instance")
now := time.Now()
for _, example := range []struct {
name string // name of this test
jobs []Job // Jobs to put in the queue
offset time.Duration // Amount to travel forward
expectedIndex int // Index of expected job
}{
{
"basics",
[]Job{
{
Method: ReleaseJob,
Params: ReleaseJobParams{},
ScheduledAt: now.Add(1 * time.Minute),
},
},
2 * time.Minute,
0,
},
{
"higher priorities",
[]Job{
{
Method: ReleaseJob,
Params: ReleaseJobParams{},
ScheduledAt: now.Add(1 * time.Minute),
Priority: 1,
},
{
Method: ReleaseJob,
Params: ReleaseJobParams{},
ScheduledAt: now.Add(1 * time.Minute),
Priority: 10,
},
},
2 * time.Minute,
1,
},
{
"scheduled first",
[]Job{
{
Method: ReleaseJob,
Params: ReleaseJobParams{},
ScheduledAt: now.Add(1 * time.Minute),
},
{
Method: ReleaseJob,
Params: ReleaseJobParams{},
ScheduledAt: now.Add(5 * time.Second),
},
},
2 * time.Minute,
1,
},
{
"submitted first",
[]Job{
{
Method: ReleaseJob,
Params: ReleaseJobParams{},
ScheduledAt: now.Add(1 * time.Minute),
},
{
Method: ReleaseJob,
Params: ReleaseJobParams{},
ScheduledAt: now.Add(1 * time.Minute),
},
},
2 * time.Minute,
0,
},
} {
db := Setup(t)
// Stub now so we can time-travel
db.now = func(_ dbProxy) (time.Time, error) {
return now, nil
}
// Put some scheduled jobs
var ids []JobID
for i, job := range example.jobs {
id, err := db.PutJob(instance, job)
if err != nil {
t.Errorf("[%s] putting job onto queue: %v", example.name, err)
} else {
ids = append(ids, id)
}
// Advance time so each job has a different submission timestamp
db.now = func(_ dbProxy) (time.Time, error) {
return now.Add(time.Duration(i) * time.Second), nil
}
}
// Check nothing is available
if _, err := db.NextJob(nil); err != ErrNoJobAvailable {
t.Fatalf("[%s] Expected ErrNoJobAvailable, got %q", example.name, err)
}
// Advance time so it is available
db.now = func(_ dbProxy) (time.Time, error) {
return now.Add(example.offset), nil
}
// It should be available
job, err := db.NextJob(nil)
if err != nil {
t.Errorf("[%s] getting job from queue: %v", example.name, err)
continue
}
if job.ID != ids[example.expectedIndex] {
t.Fatalf("[%s] Expected scheduled job, got %q", example.name, job.ID)
}
Cleanup(t, db)
}
}