func TestPurgeJobQueue(t *testing.T) {
setup()
defer teardown()
Convey("Given job queue, do purge", t, func() {
ctx := context.Background()
key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
job.Key = key.String()
srv.EnqueJob(ctx, job)
srv.ListJobQueue(store.TableJobFeed)
cmd := &pb.CommandRequest{
Command: "PurgeJobs",
}
srv.Command(ctx, cmd)
jobs, err := srv.ListJobQueue(store.TableJobRunning)
So(err, ShouldBeNil)
So(len(jobs), ShouldEqual, 0)
srv.EnqueJob(ctx, job)
worker := &pb.Worker{
Id: "123456",
}
srv.GetFeedJob(ctx, worker)
srv.dequeJob()
srv.Command(ctx, cmd)
jobs, err = srv.ListJobQueue(store.TableJobRunning)
So(err, ShouldBeNil)
So(len(jobs), ShouldEqual, 0)
})
}
func TestServerJob(t *testing.T) {
setup()
defer teardown()
Convey("Manual enqueue to database", t, func() {
key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
job.Key = key.String()
bytes, err := proto.Marshal(job)
So(err, ShouldBeNil)
err = srv.mdb.Put(key.Bytes(), bytes)
So(err, ShouldBeNil)
bytes, err = srv.mdb.Get(key.Bytes())
So(err, ShouldBeNil)
err = proto.Unmarshal(bytes, job)
So(err, ShouldBeNil)
key = store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
// iter := srv.mdb.Iterator()
// iter.Seek(key.Prefix().Bytes())
// defer iter.Close()
// So(iter.Valid(), ShouldBeTrue)
got, err := srv.dequeJob()
So(err, ShouldBeNil)
So(got.Key, ShouldEqual, job.Key)
So(got.Id, ShouldEqual, job.Id)
So(got.RemoteKey, ShouldEqual, job.RemoteKey)
got, err = srv.dequeJob()
So(err, ShouldNotBeNil)
})
}
func (s *ApiServer) EnqueJob(ctx context.Context, job *pb.FeedJob) (*pb.FeedJob, error) {
// Time ordered job queue
key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId())
job.Key = key.String()
job.Created = time.Now().Unix()
job.Updated = time.Now().Unix()
bytes, err := proto.Marshal(job)
if err != nil {
return nil, err
}
s.mdb.Put(key.Bytes(), bytes)
return job, nil
}
func TestFinishJobQueue(t *testing.T) {
setup()
defer teardown()
Convey("Given ApiServer, enqueue job, deque job, finish job", t, func() {
ctx := context.Background()
key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
job.Key = key.String()
job.TargetId = "targetId"
srv.EnqueJob(ctx, job)
worker := &pb.Worker{
Id: "123456",
}
// running job
newjob, err := srv.GetFeedJob(ctx, worker)
So(err, ShouldBeNil)
So(newjob.Id, ShouldEqual, job.Id)
So(newjob.RemoteKey, ShouldEqual, job.RemoteKey)
So(newjob.Key, ShouldNotEqual, job.Key)
// finished job
key1 := job.Key
newjob.Id = "name"
newjob.Uuid = "c6f8dca854f011ddb489003048343a40"
finjob, err := srv.FinishJob(ctx, newjob)
So(err, ShouldBeNil)
So(finjob.Key, ShouldNotEqual, key1)
So(finjob.Status, ShouldEqual, "done")
dbjob, err := store.GetArchiveHistory(srv.mdb, newjob.TargetId)
So(err, ShouldBeNil)
So(finjob.Key, ShouldEqual, dbjob.Key)
So(dbjob.Status, ShouldEqual, "done")
// check running job states
jobs, err := srv.ListJobQueue(store.TableJobRunning)
So(err, ShouldBeNil)
So(len(jobs), ShouldEqual, 0)
})
}
func TestJobQueue(t *testing.T) {
setup()
defer teardown()
Convey("Given ApiServer, When enqueue job, should deque the same job", t, func() {
ctx := context.Background()
key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
job.Key = key.String()
srv.EnqueJob(ctx, job)
jobs, err := srv.ListJobQueue(store.TableJobFeed)
So(err, ShouldBeNil)
So(len(jobs), ShouldEqual, 1)
worker := &pb.Worker{
Id: "123456",
}
got, err := srv.GetFeedJob(ctx, worker)
So(err, ShouldBeNil)
So(got.Id, ShouldEqual, job.Id)
So(got.RemoteKey, ShouldEqual, job.RemoteKey)
_, err = srv.dequeJob()
So(err, ShouldNotBeNil)
jobs, err = srv.ListJobQueue(store.TableJobRunning)
So(err, ShouldBeNil)
So(len(jobs), ShouldEqual, 1)
// reopen to check data
Convey("reopen db should got the same result: no job available", func() {
srv.Shutdown()
srv = NewApiServer(dbpath, mcFile)
_, err = srv.dequeJob()
So(err, ShouldNotBeNil)
})
})
}
func (s *ApiServer) dequeJob() (*pb.FeedJob, error) {
var job *pb.FeedJob
key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId())
store.ForwardTableScan(s.mdb, key.Prefix(), func(i int, k, v []byte) error {
job = &pb.FeedJob{}
if err := proto.Unmarshal(v, job); err != nil {
return err
}
return &store.Error{"ok", store.StopIteration}
})
if job == nil {
return nil, fmt.Errorf("No more job available")
}
kb, _ := hex.DecodeString(job.Key)
if err := s.mdb.Delete(kb); err != nil {
return nil, err
}
return job, nil
}
func TestMdbReopen(t *testing.T) {
setup()
defer teardown()
Convey("mdb reopen bug: Corruption on wrong key size", t, func() {
key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
job.Key = key.String()
bytes, err := proto.Marshal(job)
err = srv.mdb.Put(key.Bytes(), bytes)
So(err, ShouldBeNil)
// reopen to check data
srv.Shutdown()
srv = NewApiServer(dbpath, mcFile)
got, err := srv.dequeJob()
So(err, ShouldBeNil)
So(got.Key, ShouldEqual, job.Key)
So(got.Id, ShouldEqual, job.Id)
So(got.RemoteKey, ShouldEqual, job.RemoteKey)
})
}
func TestReopenDeque(t *testing.T) {
setup()
defer teardown()
Convey("mdb redeque", t, func() {
key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
job.Key = key.String()
mdb := srv.mdb
bytes, err := proto.Marshal(job)
err = mdb.Put(key.Bytes(), bytes)
_, err = srv.dequeJob()
So(err, ShouldBeNil)
// reopen to check data
srv.Shutdown()
srv = NewApiServer(dbpath, mcFile)
_, err = srv.dequeJob()
So(err, ShouldNotBeNil)
})
}
func (s *ApiServer) GetFeedJob(ctx context.Context, in *pb.Worker) (*pb.FeedJob, error) {
s.Lock()
defer s.Unlock()
job, err := s.dequeJob()
if err != nil {
return nil, err
}
// Time ordered running job
key := store.NewFlakeKey(store.TableJobRunning, s.mdb.NextId())
job.Key = key.String()
job.Worker = in.Id
job.Created = time.Now().Unix()
job.Updated = time.Now().Unix()
bytes, err := proto.Marshal(job)
if err != nil {
return nil, err
}
s.mdb.Put(key.Bytes(), bytes)
return job, nil
}