func TestPurgeJobQueue(t *testing.T) { setup() defer teardown() Convey("Given job queue, do purge", t, func() { ctx := context.Background() key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId()) job.Key = key.String() srv.EnqueJob(ctx, job) srv.ListJobQueue(store.TableJobFeed) cmd := &pb.CommandRequest{ Command: "PurgeJobs", } srv.Command(ctx, cmd) jobs, err := srv.ListJobQueue(store.TableJobRunning) So(err, ShouldBeNil) So(len(jobs), ShouldEqual, 0) srv.EnqueJob(ctx, job) worker := &pb.Worker{ Id: "123456", } srv.GetFeedJob(ctx, worker) srv.dequeJob() srv.Command(ctx, cmd) jobs, err = srv.ListJobQueue(store.TableJobRunning) So(err, ShouldBeNil) So(len(jobs), ShouldEqual, 0) }) }
func TestServerJob(t *testing.T) { setup() defer teardown() Convey("Manual enqueue to database", t, func() { key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId()) job.Key = key.String() bytes, err := proto.Marshal(job) So(err, ShouldBeNil) err = srv.mdb.Put(key.Bytes(), bytes) So(err, ShouldBeNil) bytes, err = srv.mdb.Get(key.Bytes()) So(err, ShouldBeNil) err = proto.Unmarshal(bytes, job) So(err, ShouldBeNil) key = store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId()) // iter := srv.mdb.Iterator() // iter.Seek(key.Prefix().Bytes()) // defer iter.Close() // So(iter.Valid(), ShouldBeTrue) got, err := srv.dequeJob() So(err, ShouldBeNil) So(got.Key, ShouldEqual, job.Key) So(got.Id, ShouldEqual, job.Id) So(got.RemoteKey, ShouldEqual, job.RemoteKey) got, err = srv.dequeJob() So(err, ShouldNotBeNil) }) }
func (s *ApiServer) EnqueJob(ctx context.Context, job *pb.FeedJob) (*pb.FeedJob, error) { // Time ordered job queue key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId()) job.Key = key.String() job.Created = time.Now().Unix() job.Updated = time.Now().Unix() bytes, err := proto.Marshal(job) if err != nil { return nil, err } s.mdb.Put(key.Bytes(), bytes) return job, nil }
func TestFinishJobQueue(t *testing.T) { setup() defer teardown() Convey("Given ApiServer, enqueue job, deque job, finish job", t, func() { ctx := context.Background() key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId()) job.Key = key.String() job.TargetId = "targetId" srv.EnqueJob(ctx, job) worker := &pb.Worker{ Id: "123456", } // running job newjob, err := srv.GetFeedJob(ctx, worker) So(err, ShouldBeNil) So(newjob.Id, ShouldEqual, job.Id) So(newjob.RemoteKey, ShouldEqual, job.RemoteKey) So(newjob.Key, ShouldNotEqual, job.Key) // finished job key1 := job.Key newjob.Id = "name" newjob.Uuid = "c6f8dca854f011ddb489003048343a40" finjob, err := srv.FinishJob(ctx, newjob) So(err, ShouldBeNil) So(finjob.Key, ShouldNotEqual, key1) So(finjob.Status, ShouldEqual, "done") dbjob, err := store.GetArchiveHistory(srv.mdb, newjob.TargetId) So(err, ShouldBeNil) So(finjob.Key, ShouldEqual, dbjob.Key) So(dbjob.Status, ShouldEqual, "done") // check running job states jobs, err := srv.ListJobQueue(store.TableJobRunning) So(err, ShouldBeNil) So(len(jobs), ShouldEqual, 0) }) }
func TestJobQueue(t *testing.T) { setup() defer teardown() Convey("Given ApiServer, When enqueue job, should deque the same job", t, func() { ctx := context.Background() key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId()) job.Key = key.String() srv.EnqueJob(ctx, job) jobs, err := srv.ListJobQueue(store.TableJobFeed) So(err, ShouldBeNil) So(len(jobs), ShouldEqual, 1) worker := &pb.Worker{ Id: "123456", } got, err := srv.GetFeedJob(ctx, worker) So(err, ShouldBeNil) So(got.Id, ShouldEqual, job.Id) So(got.RemoteKey, ShouldEqual, job.RemoteKey) _, err = srv.dequeJob() So(err, ShouldNotBeNil) jobs, err = srv.ListJobQueue(store.TableJobRunning) So(err, ShouldBeNil) So(len(jobs), ShouldEqual, 1) // reopen to check data Convey("reopen db should got the same result: no job available", func() { srv.Shutdown() srv = NewApiServer(dbpath, mcFile) _, err = srv.dequeJob() So(err, ShouldNotBeNil) }) }) }
func (s *ApiServer) dequeJob() (*pb.FeedJob, error) { var job *pb.FeedJob key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId()) store.ForwardTableScan(s.mdb, key.Prefix(), func(i int, k, v []byte) error { job = &pb.FeedJob{} if err := proto.Unmarshal(v, job); err != nil { return err } return &store.Error{"ok", store.StopIteration} }) if job == nil { return nil, fmt.Errorf("No more job available") } kb, _ := hex.DecodeString(job.Key) if err := s.mdb.Delete(kb); err != nil { return nil, err } return job, nil }
func TestMdbReopen(t *testing.T) { setup() defer teardown() Convey("mdb reopen bug: Corruption on wrong key size", t, func() { key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId()) job.Key = key.String() bytes, err := proto.Marshal(job) err = srv.mdb.Put(key.Bytes(), bytes) So(err, ShouldBeNil) // reopen to check data srv.Shutdown() srv = NewApiServer(dbpath, mcFile) got, err := srv.dequeJob() So(err, ShouldBeNil) So(got.Key, ShouldEqual, job.Key) So(got.Id, ShouldEqual, job.Id) So(got.RemoteKey, ShouldEqual, job.RemoteKey) }) }
func TestReopenDeque(t *testing.T) { setup() defer teardown() Convey("mdb redeque", t, func() { key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId()) job.Key = key.String() mdb := srv.mdb bytes, err := proto.Marshal(job) err = mdb.Put(key.Bytes(), bytes) _, err = srv.dequeJob() So(err, ShouldBeNil) // reopen to check data srv.Shutdown() srv = NewApiServer(dbpath, mcFile) _, err = srv.dequeJob() So(err, ShouldNotBeNil) }) }
func (s *ApiServer) GetFeedJob(ctx context.Context, in *pb.Worker) (*pb.FeedJob, error) { s.Lock() defer s.Unlock() job, err := s.dequeJob() if err != nil { return nil, err } // Time ordered running job key := store.NewFlakeKey(store.TableJobRunning, s.mdb.NextId()) job.Key = key.String() job.Worker = in.Id job.Created = time.Now().Unix() job.Updated = time.Now().Unix() bytes, err := proto.Marshal(job) if err != nil { return nil, err } s.mdb.Put(key.Bytes(), bytes) return job, nil }