Beispiel #1
0
func TestDatabaseStoreExpiresHeartbeatedButCrashedJobs(t *testing.T) {
	instance := flux.InstanceID("instance")
	db := Setup(t)
	defer Cleanup(t, db)

	// Mock time, so we can mess around with it
	now := time.Now()
	db.now = func(_ dbProxy) (time.Time, error) {
		return now, nil
	}

	// Put a job
	jobID, err := db.PutJob(instance, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Take it, so it is claimed
	_, err = db.NextJob(nil)
	bailIfErr(t, err)

	// Heartbeat the job
	now = now.Add(1 * time.Minute)
	bailIfErr(t, db.Heartbeat(jobID))

	// GC should not remove it (heartbeat should keep it alive longer)
	now = now.Add(30 * time.Second)
	bailIfErr(t, db.GC())
	_, err = db.GetJob(instance, jobID)
	bailIfErr(t, err)

	// GC should remove it after gc time
	now = now.Add(2 * time.Minute)
	bailIfErr(t, db.GC())
	// - should be removed
	_, err = db.GetJob(instance, jobID)
	if err != ErrNoSuchJob {
		t.Errorf("expected ErrNoSuchJob, got %q", err)
	}
}
Beispiel #2
0
func TestDatabaseStore(t *testing.T) {
	instance := flux.InstanceID("instance")
	instance2 := flux.InstanceID("instance2")
	db := Setup(t)
	defer Cleanup(t, db)

	// Get a job when there are none
	_, err := db.NextJob(nil)
	if err != ErrNoJobAvailable {
		t.Fatalf("Expected ErrNoJobAvailable, got %q", err)
	}

	// Put some jobs
	backgroundJobID, err := db.PutJob(instance2, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityBackground,
	})
	bailIfErr(t, err)
	interactiveJobID, err := db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Put a duplicate
	duplicateID, err := db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	if err != ErrJobAlreadyQueued {
		t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
	}
	if string(duplicateID) != "" {
		t.Errorf("Expected no id for duplicate job, got: %q", duplicateID)
	}

	// Take one
	interactiveJob, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be the highest priority
	if interactiveJob.ID != interactiveJobID {
		t.Errorf("Got a lower priority job when a higher one was available")
	}
	// - It should have a default queue
	if interactiveJob.Queue != DefaultQueue {
		t.Errorf("job default queue (%q) was not expected (%q)", interactiveJob.Queue, DefaultQueue)
	}
	// - It should have been scheduled in the past
	now, err := db.now(db.conn)
	bailIfErr(t, err)
	if interactiveJob.ScheduledAt.IsZero() || interactiveJob.ScheduledAt.After(now) {
		t.Errorf("expected job to be scheduled in the past")
	}
	// - It should have a log and status
	if len(interactiveJob.Log) == 0 || interactiveJob.Status == "" {
		t.Errorf("expected job to have a log and status")
	}

	// Put a duplicate (when existing is claimed, but not finished)
	// - It should fail
	_, err = db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	if err != ErrJobAlreadyQueued {
		t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
	}

	// Put a duplicate (For another instance)
	// - It should succeed
	_, err = db.PutJob(instance2, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	bailIfErr(t, err)

	// Put a duplicate (Ignoring duplicates)
	// - It should succeed
	_, err = db.PutJobIgnoringDuplicates(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	bailIfErr(t, err)

	// Update the job
	newStatus := "Being used in testing"
	interactiveJob.Status = newStatus
	interactiveJob.Log = append(interactiveJob.Log, newStatus)
	bailIfErr(t, db.UpdateJob(interactiveJob))
	// - It should have saved the changes
	interactiveJob, err = db.GetJob(instance, interactiveJobID)
	bailIfErr(t, err)
	if interactiveJob.Status != newStatus || len(interactiveJob.Log) != 2 || interactiveJob.Log[1] != interactiveJob.Status {
		t.Errorf("expected job to have new log and status")
	}

	// Heartbeat the job
	oldHeartbeat := interactiveJob.Heartbeat
	bailIfErr(t, db.Heartbeat(interactiveJobID))
	// - Heartbeat time should be updated
	interactiveJob, err = db.GetJob(instance, interactiveJobID)
	bailIfErr(t, err)
	if !interactiveJob.Heartbeat.After(oldHeartbeat) {
		t.Errorf("expected job heartbeat to have been updated")
	}

	// Take the next
	backgroundJob, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be different
	if backgroundJob.ID != backgroundJobID {
		t.Errorf("Got a different job than expected")
	}

	// Finish one
	backgroundJob.Done = true
	backgroundJob.Success = true
	bailIfErr(t, db.UpdateJob(backgroundJob))
	// - Status should be changed
	backgroundJob, err = db.GetJob(instance2, backgroundJobID)
	bailIfErr(t, err)
	if !backgroundJob.Done || !backgroundJob.Success {
		t.Errorf("expected job to have been marked as done")
	}

	// GC
	// - Advance time so we can gc stuff
	db.now = func(_ dbProxy) (time.Time, error) {
		return time.Now().Add(2 * time.Minute), nil
	}
	bailIfErr(t, db.GC())
	// - Finished should be removed
	_, err = db.GetJob(instance, backgroundJobID)
	if err != ErrNoSuchJob {
		t.Errorf("expected ErrNoSuchJob, got %q", err)
	}
}
Beispiel #3
0
func TestDatabaseStoreFairScheduling(t *testing.T) {
	instance1 := flux.InstanceID("instance1")
	instance2 := flux.InstanceID("instance2")
	db := Setup(t)
	defer Cleanup(t, db)

	// Put some jobs for instance 1
	job1ID, err := db.PutJob(instance1, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)
	job2ID, err := db.PutJob(instance1, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Put a job for instance 2
	job3ID, err := db.PutJob(instance2, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Take one
	// - It should be instance1's first job
	job1, err := db.NextJob(nil)
	bailIfErr(t, err)
	if job1.ID != job1ID {
		t.Errorf("Got a newer job when an older one was available")
	}
	// Take another (while instance1 has one in-progress)
	// - It should be instance2's first job
	job3, err := db.NextJob(nil)
	bailIfErr(t, err)
	if job3.ID != job3ID {
		t.Errorf("Got an unexpected job id")
	}

	// Take another (while instance1, and instance2 has one in-progress)
	// - It should say none are available, because both are in-progress
	_, err = db.NextJob(nil)
	if err != ErrNoJobAvailable {
		t.Fatalf("Expected ErrNoJobAvailable, got %q", err)
	}

	// Finish instance1's job
	job1.Done = true
	job1.Success = true
	bailIfErr(t, db.UpdateJob(job1))
	// - Status should be changed
	job1, err = db.GetJob(instance1, job1ID)
	bailIfErr(t, err)
	if !job1.Done || !job1.Success {
		t.Errorf("expected job to have been marked as done")
	}

	// Take another
	// - It should be instance1's next job
	job2, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be the next job for instance1
	if job2.ID != job2ID {
		t.Errorf("Got an unexpected job id")
	}
}
Beispiel #4
0
func TestDatabaseStoreScheduledJobs(t *testing.T) {
	instance := flux.InstanceID("instance")
	now := time.Now()

	for _, example := range []struct {
		name          string        // name of this test
		jobs          []Job         // Jobs to put in the queue
		offset        time.Duration // Amount to travel forward
		expectedIndex int           // Index of expected job
	}{
		{
			"basics",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
			},
			2 * time.Minute,
			0,
		},
		{
			"higher priorities",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
					Priority:    1,
				},
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
					Priority:    10,
				},
			},
			2 * time.Minute,
			1,
		},
		{
			"scheduled first",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(5 * time.Second),
				},
			},
			2 * time.Minute,
			1,
		},
		{
			"submitted first",
			[]Job{
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
				{
					Method:      ReleaseJob,
					Params:      ReleaseJobParams{},
					ScheduledAt: now.Add(1 * time.Minute),
				},
			},
			2 * time.Minute,
			0,
		},
	} {
		db := Setup(t)

		// Stub now so we can time-travel
		db.now = func(_ dbProxy) (time.Time, error) {
			return now, nil
		}

		// Put some scheduled jobs
		var ids []JobID
		for i, job := range example.jobs {
			id, err := db.PutJob(instance, job)
			if err != nil {
				t.Errorf("[%s] putting job onto queue: %v", example.name, err)
			} else {
				ids = append(ids, id)
			}

			// Advance time so each job has a different submission timestamp
			db.now = func(_ dbProxy) (time.Time, error) {
				return now.Add(time.Duration(i) * time.Second), nil
			}
		}

		// Check nothing is available
		if _, err := db.NextJob(nil); err != ErrNoJobAvailable {
			t.Fatalf("[%s] Expected ErrNoJobAvailable, got %q", example.name, err)
		}

		// Advance time so it is available
		db.now = func(_ dbProxy) (time.Time, error) {
			return now.Add(example.offset), nil
		}

		// It should be available
		job, err := db.NextJob(nil)
		if err != nil {
			t.Errorf("[%s] getting job from queue: %v", example.name, err)
			continue
		}
		if job.ID != ids[example.expectedIndex] {
			t.Fatalf("[%s] Expected scheduled job, got %q", example.name, job.ID)
		}

		Cleanup(t, db)
	}
}