Example #1
0
func TestDatabaseStoreExpiresHeartbeatedButCrashedJobs(t *testing.T) {
	instance := flux.InstanceID("instance")
	db := Setup(t)
	defer Cleanup(t, db)

	// Mock time, so we can mess around with it
	now := time.Now()
	db.now = func(_ dbProxy) (time.Time, error) {
		return now, nil
	}

	// Put a job
	jobID, err := db.PutJob(instance, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Take it, so it is claimed
	_, err = db.NextJob(nil)
	bailIfErr(t, err)

	// Heartbeat the job
	now = now.Add(1 * time.Minute)
	bailIfErr(t, db.Heartbeat(jobID))

	// GC should not remove it (heartbeat should keep it alive longer)
	now = now.Add(30 * time.Second)
	bailIfErr(t, db.GC())
	_, err = db.GetJob(instance, jobID)
	bailIfErr(t, err)

	// GC should remove it after gc time
	now = now.Add(2 * time.Minute)
	bailIfErr(t, db.GC())
	// - should be removed
	_, err = db.GetJob(instance, jobID)
	if err != ErrNoSuchJob {
		t.Errorf("expected ErrNoSuchJob, got %q", err)
	}
}
Example #2
0
func TestDatabaseStore(t *testing.T) {
	instance := flux.InstanceID("instance")
	instance2 := flux.InstanceID("instance2")
	db := Setup(t)
	defer Cleanup(t, db)

	// Get a job when there are none
	_, err := db.NextJob(nil)
	if err != ErrNoJobAvailable {
		t.Fatalf("Expected ErrNoJobAvailable, got %q", err)
	}

	// Put some jobs
	backgroundJobID, err := db.PutJob(instance2, Job{
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityBackground,
	})
	bailIfErr(t, err)
	interactiveJobID, err := db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	bailIfErr(t, err)

	// Put a duplicate
	duplicateID, err := db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: PriorityInteractive,
	})
	if err != ErrJobAlreadyQueued {
		t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
	}
	if string(duplicateID) != "" {
		t.Errorf("Expected no id for duplicate job, got: %q", duplicateID)
	}

	// Take one
	interactiveJob, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be the highest priority
	if interactiveJob.ID != interactiveJobID {
		t.Errorf("Got a lower priority job when a higher one was available")
	}
	// - It should have a default queue
	if interactiveJob.Queue != DefaultQueue {
		t.Errorf("job default queue (%q) was not expected (%q)", interactiveJob.Queue, DefaultQueue)
	}
	// - It should have been scheduled in the past
	now, err := db.now(db.conn)
	bailIfErr(t, err)
	if interactiveJob.ScheduledAt.IsZero() || interactiveJob.ScheduledAt.After(now) {
		t.Errorf("expected job to be scheduled in the past")
	}
	// - It should have a log and status
	if len(interactiveJob.Log) == 0 || interactiveJob.Status == "" {
		t.Errorf("expected job to have a log and status")
	}

	// Put a duplicate (when existing is claimed, but not finished)
	// - It should fail
	_, err = db.PutJob(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	if err != ErrJobAlreadyQueued {
		t.Errorf("Expected duplicate job to return ErrJobAlreadyQueued, got: %q", err)
	}

	// Put a duplicate (For another instance)
	// - It should succeed
	_, err = db.PutJob(instance2, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	bailIfErr(t, err)

	// Put a duplicate (Ignoring duplicates)
	// - It should succeed
	_, err = db.PutJobIgnoringDuplicates(instance, Job{
		Key:      "2",
		Method:   ReleaseJob,
		Params:   ReleaseJobParams{},
		Priority: 1, // low priority, so it won't interfere with other jobs
	})
	bailIfErr(t, err)

	// Update the job
	newStatus := "Being used in testing"
	interactiveJob.Status = newStatus
	interactiveJob.Log = append(interactiveJob.Log, newStatus)
	bailIfErr(t, db.UpdateJob(interactiveJob))
	// - It should have saved the changes
	interactiveJob, err = db.GetJob(instance, interactiveJobID)
	bailIfErr(t, err)
	if interactiveJob.Status != newStatus || len(interactiveJob.Log) != 2 || interactiveJob.Log[1] != interactiveJob.Status {
		t.Errorf("expected job to have new log and status")
	}

	// Heartbeat the job
	oldHeartbeat := interactiveJob.Heartbeat
	bailIfErr(t, db.Heartbeat(interactiveJobID))
	// - Heartbeat time should be updated
	interactiveJob, err = db.GetJob(instance, interactiveJobID)
	bailIfErr(t, err)
	if !interactiveJob.Heartbeat.After(oldHeartbeat) {
		t.Errorf("expected job heartbeat to have been updated")
	}

	// Take the next
	backgroundJob, err := db.NextJob(nil)
	bailIfErr(t, err)
	// - It should be different
	if backgroundJob.ID != backgroundJobID {
		t.Errorf("Got a different job than expected")
	}

	// Finish one
	backgroundJob.Done = true
	backgroundJob.Success = true
	bailIfErr(t, db.UpdateJob(backgroundJob))
	// - Status should be changed
	backgroundJob, err = db.GetJob(instance2, backgroundJobID)
	bailIfErr(t, err)
	if !backgroundJob.Done || !backgroundJob.Success {
		t.Errorf("expected job to have been marked as done")
	}

	// GC
	// - Advance time so we can gc stuff
	db.now = func(_ dbProxy) (time.Time, error) {
		return time.Now().Add(2 * time.Minute), nil
	}
	bailIfErr(t, db.GC())
	// - Finished should be removed
	_, err = db.GetJob(instance, backgroundJobID)
	if err != ErrNoSuchJob {
		t.Errorf("expected ErrNoSuchJob, got %q", err)
	}
}