Beispiel #1
0
func TestMethods(t *testing.T) {
	bus := setup(t)

	errc := make(chan error, 1)

	instA := flux.InstanceID("steamy-windows-89")
	mockA := &platform.MockPlatform{
		AllServicesAnswer: []platform.Service{platform.Service{}},
		RegradeError:      platform.RegradeError{flux.ServiceID("foo/bar"): errors.New("foo barred")},
	}
	subscribe(t, bus, errc, instA, mockA)

	plat, err := bus.Connect(instA)
	if err != nil {
		t.Fatal(err)
	}
	ss, err := plat.AllServices("", nil)
	if err != nil {
		t.Fatal(err)
	}
	if len(mockA.AllServicesAnswer) != len(ss) {
		t.Fatalf("Expected %d result, got %d", len(mockA.AllServicesAnswer), len(ss))
	}

	err = plat.Regrade([]platform.RegradeSpec{})
	if _, ok := err.(platform.RegradeError); !ok {
		t.Fatalf("expected RegradeError, got %+v", err)
	}

	mockB := &platform.MockPlatform{
		AllServicesError:   errors.New("just didn't feel like it"),
		SomeServicesAnswer: []platform.Service{platform.Service{}, platform.Service{}},
	}
	instB := flux.InstanceID("smokey-water-72")
	subscribe(t, bus, errc, instB, mockB)
	platB, err := bus.Connect(instB)
	if err != nil {
		t.Fatal(err)
	}

	ss, err = platB.SomeServices([]flux.ServiceID{})
	if err != nil {
		t.Fatal(err)
	}
	if len(mockB.SomeServicesAnswer) != len(ss) {
		t.Fatalf("Expected %d result, got %d", len(mockB.SomeServicesAnswer), len(ss))
	}

	ss, err = platB.AllServices("", nil)
	if err == nil {
		t.Fatal("expected error but didn't get one")
	}

	close(errc)
	err = <-errc
	if err != nil {
		t.Fatalf("expected nil from subscription channel, but got err %v", err)
	}
}
Beispiel #2
0
func TestHistoryLog(t *testing.T) {
	instance := flux.InstanceID("instance")
	db := newSQL(t)
	defer db.Close()

	bailIfErr(t, db.LogEvent(instance, "namespace", "service", "event 1"))
	bailIfErr(t, db.LogEvent(instance, "namespace", "other", "event 3"))
	bailIfErr(t, db.LogEvent(instance, "namespace", "service", "event 2"))

	es, err := db.EventsForService(instance, "namespace", "service")
	if err != nil {
		t.Fatal(err)
	}
	if len(es) != 2 {
		t.Fatalf("Expected 2 events, got %d\n", len(es))
	}
	checkInDescOrder(t, es)

	es, err = db.AllEvents(instance)
	if err != nil {
		t.Fatal(err)
	}
	if len(es) != 3 {
		t.Fatalf("Expected 3 events, got %#v\n", es)
	}
	checkInDescOrder(t, es)
}
Beispiel #3
0
func TestPing(t *testing.T) {
	bus := setup(t)

	errc := make(chan error)
	instID := flux.InstanceID("wirey-bird-68")
	platA := &platform.MockPlatform{}
	subscribe(t, bus, errc, instID, platA)

	// AwaitPresence uses Ping, so we have to install our error after
	// subscribe succeeds.
	platA.PingError = platform.FatalError{errors.New("ping problem")}
	if err := platA.Ping(); err == nil {
		t.Fatalf("expected error from directly calling ping, got nil")
	}

	err := bus.Ping(instID)
	if err == nil {
		t.Errorf("expected error from ping, got nil")
	} else if err.Error() != "ping problem" {
		t.Errorf("got the wrong error: %s", err.Error())
	}

	select {
	case err := <-errc:
		if err == nil {
			t.Fatal("expected error return from subscription but didn't get one")
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("expected error return from subscription but didn't get one")
	}
}
Beispiel #4
0
func TestFatalErrorDisconnects(t *testing.T) {
	bus := setup(t)

	errc := make(chan error)

	instA := flux.InstanceID("golden-years-75")
	mockA := &platform.MockPlatform{
		SomeServicesError: platform.FatalError{errors.New("Disaster.")},
	}
	subscribe(t, bus, errc, instA, mockA)

	plat, err := bus.Connect(instA)
	if err != nil {
		t.Fatal(err)
	}

	_, err = plat.SomeServices([]flux.ServiceID{})
	if err == nil {
		t.Error("expected error, got nil")
	} else if _, ok := err.(platform.FatalError); !ok {
		t.Errorf("expected platform.FatalError, got %v", err)
	}

	select {
	case err = <-errc:
		if err == nil {
			t.Error("expected error from subscription being killed, got nil")
		}
	case <-time.After(1 * time.Second):
		t.Error("timed out waiting for expected error from subscription closing")
	}
}
Beispiel #5
0
func TestNewConnectionKicks(t *testing.T) {
	bus := setup(t)

	instA := flux.InstanceID("foo")

	mockA := &platform.MockPlatform{}
	errA := make(chan error)
	subscribe(t, bus, errA, instA, mockA)

	mockB := &platform.MockPlatform{}
	errB := make(chan error)
	subscribe(t, bus, errB, instA, mockB)

	select {
	case <-errA:
		break
	case <-time.After(1 * time.Second):
		t.Error("timed out waiting for connection to be kicked")
	}

	close(errB)
	err := <-errB
	if err != nil {
		t.Errorf("expected no error from second connection, but got %q", err)
	}
}
Beispiel #6
0
func getInstanceID(req *http.Request) flux.InstanceID {
	s := req.Header.Get(flux.InstanceIDHeaderKey)
	if s == "" {
		return flux.DefaultInstanceID
	}
	return flux.InstanceID(s)
}
Beispiel #7
0
func (db *DB) All() ([]instance.NamedConfig, error) {
	rows, err := db.conn.Query(`SELECT instance, config FROM config`)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	instances := []instance.NamedConfig{}
	for rows.Next() {
		var (
			id, confStr string
			conf        instance.Config
		)
		err = rows.Scan(&id, &confStr)
		if err == nil {
			err = json.Unmarshal([]byte(confStr), &conf)
		}
		if err != nil {
			return nil, err
		}

		instances = append(instances, instance.NamedConfig{
			ID:     flux.InstanceID(id),
			Config: conf,
		})
	}
	return instances, rows.Err()
}
Beispiel #8
0
func TestJobEncodingDecoding(t *testing.T) {
	// Check it can serialize/deserialize release jobs
	now := time.Now().UTC()
	expected := Job{
		Instance: flux.InstanceID("instance"),
		ID:       NewJobID(),
		Queue:    DefaultQueue,
		Method:   ReleaseJob,
		Params: ReleaseJobParams{
			ServiceSpec: flux.ServiceSpecAll,
			ImageSpec:   flux.ImageSpecLatest,
			Kind:        flux.ReleaseKindExecute,
		},
		ScheduledAt: now,
		Priority:    PriorityInteractive,
		Key:         "key1",
		Submitted:   now,
		Claimed:     now,
		Heartbeat:   now,
		Finished:    now,
		Log:         []string{"log1"},
		Status:      "status",
		Done:        true,
		Success:     true,
	}
	b, err := json.Marshal(expected)
	bailIfErr(t, err)
	var got Job
	bailIfErr(t, json.Unmarshal(b, &got))

	if !reflect.DeepEqual(got, expected) {
		t.Errorf("got %q, expected %q", got, expected)
	}
}
Beispiel #9
0
func TestUpdateOK(t *testing.T) {
	db := newDB(t)

	inst := flux.InstanceID("floaty-womble-abc123")
	service := flux.MakeServiceID("namespace", "service")
	services := map[flux.ServiceID]instance.ServiceConfig{}
	services[service] = instance.ServiceConfig{
		Automated: true,
		Locked:    true,
	}
	c := instance.Config{
		Services: services,
	}
	err := db.UpdateConfig(inst, func(_ instance.Config) (instance.Config, error) {
		return c, nil
	})
	if err != nil {
		t.Fatal(err)
	}
	c1, err := db.GetConfig(inst)
	if err != nil {
		t.Fatal(err)
	}
	if _, found := c1.Services[service]; !found {
		t.Fatalf("did not find instance config after setting")
	}
	if !c1.Services[service].Automated {
		t.Fatalf("expected service config %#v, got %#v", c.Services[service], c1.Services[service])
	}
	if !c1.Services[service].Locked {
		t.Fatalf("expected service config %#v, got %#v", c.Services[service], c1.Services[service])
	}
}
Beispiel #10
0
func TestStandaloneMessageBus(t *testing.T) {
	instID := flux.InstanceID("instance")
	bus := NewStandaloneMessageBus(NewBusMetrics())
	p := &MockPlatform{}

	done := make(chan error)
	bus.Subscribe(instID, p, done)

	if err := bus.Ping(instID); err != nil {
		t.Fatal(err)
	}

	// subscribing another connection kicks the first one off
	p2 := &MockPlatform{PingError: errors.New("ping failed")}
	done2 := make(chan error, 2)
	bus.Subscribe(instID, p2, done2)

	select {
	case <-done:
		break
	case <-time.After(1 * time.Second):
		t.Error("expected connection to be kicked when subsequent connection arrived, but it wasn't")
	}

	err := bus.Ping(instID)
	if err == nil {
		t.Error("expected error from pinging mock platform, but got nil")
	}

	done2 <- nil
	err = <-done2
	if err != nil {
		t.Error("did not expect subscription error after application-level error")
	}

	// Now test that a FatalError does shut the subscription down
	p3 := &MockPlatform{PingError: FatalError{errors.New("ping failed")}}
	done3 := make(chan error)
	bus.Subscribe(instID, p3, done3)

	select {
	case <-done2:
		break
	case <-time.After(1 * time.Second):
		t.Error("expected connection to be kicked when subsequent connection arrived, but it wasn't")
	}

	err = bus.Ping(instID)
	if err == nil {
		t.Error("expected error from pinging mock platform, but got nil")
	}

	select {
	case <-done3:
		break
	case <-time.After(1 * time.Second):
		t.Error("expected error from connection on error, got none")
	}
}
Beispiel #11
0
func TestDatabaseStoreExpiresHeartbeatedButCrashedJobs(t *testing.T) {
	instance := flux.InstanceID("instance")
	db := Setup(t)
	defer Cleanup(t, db)

	// Mock time, so we can mess around with it
	now := time.Now()
	db.now = func(_ dbProxy) (time.Time, error) {
		return now, nil
	}

	// Put a job
	jobID, err := db.PutJob(instance, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Take it, so it is claimed
	_, err = db.NextJob(nil)
	bailIfErr(t, err)

	// Heartbeat the job
	now = now.Add(1 * time.Minute)
	bailIfErr(t, db.Heartbeat(jobID))

	// GC should not remove it (heartbeat should keep it alive longer)
	now = now.Add(30 * time.Second)
	bailIfErr(t, db.GC())
	_, err = db.GetJob(instance, jobID)
	bailIfErr(t, err)

	// GC should remove it after gc time
	now = now.Add(2 * time.Minute)
	bailIfErr(t, db.GC())
	// - should be removed
	_, err = db.GetJob(instance, jobID)
	if err != ErrNoSuchJob {
		t.Errorf("expected ErrNoSuchJob, got %q", err)
	}
}
Beispiel #12
0
	"github.com/spf13/pflag"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/api"
	transport "github.com/weaveworks/flux/http"
)

type rootOpts struct {
	URL   string
	Token string
	API   api.ClientService
}

// fluxctl never sends an instance ID directly; it's always blank, and
// optionally gets populated by an intermediating authfe from the token.
const noInstanceID = flux.InstanceID("")

type serviceOpts struct {
	*rootOpts
}

func newService(parent *rootOpts) *serviceOpts {
	return &serviceOpts{rootOpts: parent}
}

func newRoot() *rootOpts {
	return &rootOpts{}
}

var rootLongHelp = strings.TrimSpace(`
fluxctl helps you deploy your code.
Beispiel #13
0
// Take the next job from specified queues. If queues is nil, all queues are
// used.
func (s *DatabaseStore) NextJob(queues []string) (Job, error) {
	var job Job
	err := s.Transaction(func(s *DatabaseStore) error {
		now, err := s.now(s.conn)
		if err != nil {
			return errors.Wrap(err, "getting current time")
		}
		var (
			instanceID  string
			jobID       string
			queue       string
			method      string
			paramsBytes []byte
			scheduledAt time.Time
			priority    int
			key         string
			submittedAt time.Time
			claimedAt   nullTime
			heartbeatAt nullTime
			finishedAt  nullTime
			logStr      string
			status      string
			done        sql.NullBool
			success     sql.NullBool
		)
		if err := s.conn.QueryRow(`
			SELECT instance_id, id, queue, method, params,
						 scheduled_at, priority, key, submitted_at,
						 claimed_at, heartbeat_at, finished_at, log, status,
						 done, success
			FROM jobs

			-- Only unclaimed/unfinished jobs are available
			WHERE claimed_at IS NULL
			AND finished_at IS NULL

			-- Don't make jobs available until after they are scheduled
			AND scheduled_at <= $1

			-- Only one job at a time per instance
			AND instance_id NOT IN (
				SELECT instance_id
				FROM jobs
				WHERE claimed_at IS NOT NULL
				AND finished_at IS NULL
				GROUP BY instance_id
			)

			-- subtraction is to work around for ql, not being able to sort
			-- multiple columns in different ways.
			ORDER BY (-1 * priority), scheduled_at, submitted_at
			LIMIT 1`,
			now,
		).Scan(
			&instanceID,
			&jobID,
			&queue,
			&method,
			&paramsBytes,
			&scheduledAt,
			&priority,
			&key,
			&submittedAt,
			&claimedAt,
			&heartbeatAt,
			&finishedAt,
			&logStr,
			&status,
			&done,
			&success,
		); err == sql.ErrNoRows {
			return ErrNoJobAvailable
		} else if err != nil {
			return errors.Wrap(err, "dequeueing next job")
		}

		params, err := s.scanParams(method, paramsBytes)
		if err != nil {
			return errors.Wrap(err, "unmarshaling params")
		}

		var log []string
		if err := json.NewDecoder(strings.NewReader(logStr)).Decode(&log); err != nil {
			return errors.Wrap(err, "unmarshaling log")
		}

		job = Job{
			Instance:    flux.InstanceID(instanceID),
			ID:          JobID(jobID),
			Queue:       queue,
			Method:      method,
			Params:      params,
			ScheduledAt: scheduledAt,
			Priority:    priority,
			Key:         key,
			Submitted:   submittedAt,
			Claimed:     claimedAt.Time,
			Heartbeat:   heartbeatAt.Time,
			Finished:    finishedAt.Time,
			Log:         log,
			Status:      status,
			Done:        done.Bool,
			Success:     success.Bool,
		}

		if res, err := s.conn.Exec(`
			UPDATE jobs
				 SET claimed_at = $1
			 WHERE id = $2
				 AND instance_id = $3
		`, now, jobID, instanceID); err != nil {
			return errors.Wrap(err, "marking job as claimed")
		} else if n, err := res.RowsAffected(); err != nil {
			return errors.Wrap(err, "after update, checking affected rows")
		} else if n != 1 {
			return errors.Errorf("wanted to affect 1 row; affected %d", n)
		}
		return nil
	})
	return job, err
}
Beispiel #14
0
func TestDatabaseStore(t *testing.T) {
	instance := flux.InstanceID("instance")
	instance2 := flux.InstanceID("instance2")
	db := Setup(t)
	defer Cleanup(t, db)

	// Get a job when there are none
	_, err := db.NextJob(nil)
	if err != ErrNoJobAvailable {
		t.Fatalf("Expected ErrNoJobAvailable, got %q", err)
	}

	// Put some jobs
	backgroundJobID, err := db.PutJob(instance2, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityBackground,
	})
	bailIfErr(t, err)
	interactiveJobID, err := db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Put a duplicate
	duplicateID, err := db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	if err != ErrJobAlreadyQueued {
		t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
	}
	if string(duplicateID) != "" {
		t.Errorf("Expected no id for duplicate job, got: %q", duplicateID)
	}

	// Take one
	interactiveJob, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be the highest priority
	if interactiveJob.ID != interactiveJobID {
		t.Errorf("Got a lower priority job when a higher one was available")
	}
	// - It should have a default queue
	if interactiveJob.Queue != DefaultQueue {
		t.Errorf("job default queue (%q) was not expected (%q)", interactiveJob.Queue, DefaultQueue)
	}
	// - It should have been scheduled in the past
	now, err := db.now(db.conn)
	bailIfErr(t, err)
	if interactiveJob.ScheduledAt.IsZero() || interactiveJob.ScheduledAt.After(now) {
		t.Errorf("expected job to be scheduled in the past")
	}
	// - It should have a log and status
	if len(interactiveJob.Log) == 0 || interactiveJob.Status == "" {
		t.Errorf("expected job to have a log and status")
	}

	// Put a duplicate (when existing is claimed, but not finished)
	// - It should fail
	_, err = db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	if err != ErrJobAlreadyQueued {
		t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
	}

	// Put a duplicate (For another instance)
	// - It should succeed
	_, err = db.PutJob(instance2, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	bailIfErr(t, err)

	// Put a duplicate (Ignoring duplicates)
	// - It should succeed
	_, err = db.PutJobIgnoringDuplicates(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	bailIfErr(t, err)

	// Update the job
	newStatus := "Being used in testing"
	interactiveJob.Status = newStatus
	interactiveJob.Log = append(interactiveJob.Log, newStatus)
	bailIfErr(t, db.UpdateJob(interactiveJob))
	// - It should have saved the changes
	interactiveJob, err = db.GetJob(instance, interactiveJobID)
	bailIfErr(t, err)
	if interactiveJob.Status != newStatus || len(interactiveJob.Log) != 2 || interactiveJob.Log[1] != interactiveJob.Status {
		t.Errorf("expected job to have new log and status")
	}

	// Heartbeat the job
	oldHeartbeat := interactiveJob.Heartbeat
	bailIfErr(t, db.Heartbeat(interactiveJobID))
	// - Heartbeat time should be updated
	interactiveJob, err = db.GetJob(instance, interactiveJobID)
	bailIfErr(t, err)
	if !interactiveJob.Heartbeat.After(oldHeartbeat) {
		t.Errorf("expected job heartbeat to have been updated")
	}

	// Take the next
	backgroundJob, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be different
	if backgroundJob.ID != backgroundJobID {
		t.Errorf("Got a different job than expected")
	}

	// Finish one
	backgroundJob.Done = true
	backgroundJob.Success = true
	bailIfErr(t, db.UpdateJob(backgroundJob))
	// - Status should be changed
	backgroundJob, err = db.GetJob(instance2, backgroundJobID)
	bailIfErr(t, err)
	if !backgroundJob.Done || !backgroundJob.Success {
		t.Errorf("expected job to have been marked as done")
	}

	// GC
	// - Advance time so we can gc stuff
	db.now = func(_ dbProxy) (time.Time, error) {
		return time.Now().Add(2 * time.Minute), nil
	}
	bailIfErr(t, db.GC())
	// - Finished should be removed
	_, err = db.GetJob(instance, backgroundJobID)
	if err != ErrNoSuchJob {
		t.Errorf("expected ErrNoSuchJob, got %q", err)
	}
}
Beispiel #15
0
func TestDatabaseStoreFairScheduling(t *testing.T) {
	instance1 := flux.InstanceID("instance1")
	instance2 := flux.InstanceID("instance2")
	db := Setup(t)
	defer Cleanup(t, db)

	// Put some jobs for instance 1
	job1ID, err := db.PutJob(instance1, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)
	job2ID, err := db.PutJob(instance1, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Put a job for instance 2
	job3ID, err := db.PutJob(instance2, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Take one
	// - It should be instance1's first job
	job1, err := db.NextJob(nil)
	bailIfErr(t, err)
	if job1.ID != job1ID {
		t.Errorf("Got a newer job when an older one was available")
	}
	// Take another (while instance1 has one in-progress)
	// - It should be instance2's first job
	job3, err := db.NextJob(nil)
	bailIfErr(t, err)
	if job3.ID != job3ID {
		t.Errorf("Got an unexpected job id")
	}

	// Take another (while instance1, and instance2 has one in-progress)
	// - It should say none are available, because both are in-progress
	_, err = db.NextJob(nil)
	if err != ErrNoJobAvailable {
		t.Fatalf("Expected ErrNoJobAvailable, got %q", err)
	}

	// Finish instance1's job
	job1.Done = true
	job1.Success = true
	bailIfErr(t, db.UpdateJob(job1))
	// - Status should be changed
	job1, err = db.GetJob(instance1, job1ID)
	bailIfErr(t, err)
	if !job1.Done || !job1.Success {
		t.Errorf("expected job to have been marked as done")
	}

	// Take another
	// - It should be instance1's next job
	job2, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be the next job for instance1
	if job2.ID != job2ID {
		t.Errorf("Got an unexpected job id")
	}
}
Beispiel #16
0
func TestDatabaseStoreScheduledJobs(t *testing.T) {
	instance := flux.InstanceID("instance")
	now := time.Now()

	for _, example := range []struct {
		name          string        // name of this test
		jobs          []Job         // Jobs to put in the queue
		offset        time.Duration // Amount to travel forward
		expectedIndex int           // Index of expected job
	}{
		{
			"basics",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
			},
			2 * time.Minute,
			0,
		},
		{
			"higher priorities",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
					Priority:    1,
				},
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
					Priority:    10,
				},
			},
			2 * time.Minute,
			1,
		},
		{
			"scheduled first",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(5 * time.Second),
				},
			},
			2 * time.Minute,
			1,
		},
		{
			"submitted first",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
			},
			2 * time.Minute,
			0,
		},
	} {
		db := Setup(t)

		// Stub now so we can time-travel
		db.now = func(_ dbProxy) (time.Time, error) {
			return now, nil
		}

		// Put some scheduled jobs
		var ids []JobID
		for i, job := range example.jobs {
			id, err := db.PutJob(instance, job)
			if err != nil {
				t.Errorf("[%s] putting job onto queue: %v", example.name, err)
			} else {
				ids = append(ids, id)
			}

			// Advance time so each job has a different submission timestamp
			db.now = func(_ dbProxy) (time.Time, error) {
				return now.Add(time.Duration(i) * time.Second), nil
			}
		}

		// Check nothing is available
		if _, err := db.NextJob(nil); err != ErrNoJobAvailable {
			t.Fatalf("[%s] Expected ErrNoJobAvailable, got %q", example.name, err)
		}

		// Advance time so it is available
		db.now = func(_ dbProxy) (time.Time, error) {
			return now.Add(example.offset), nil
		}

		// It should be available
		job, err := db.NextJob(nil)
		if err != nil {
			t.Errorf("[%s] getting job from queue: %v", example.name, err)
			continue
		}
		if job.ID != ids[example.expectedIndex] {
			t.Fatalf("[%s] Expected scheduled job, got %q", example.name, job.ID)
		}

		Cleanup(t, db)
	}
}