Beispiel #1
0
func TestPurgeJobQueue(t *testing.T) {
	setup()
	defer teardown()

	Convey("Given job queue, do purge", t, func() {
		ctx := context.Background()
		key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
		job.Key = key.String()

		srv.EnqueJob(ctx, job)
		srv.ListJobQueue(store.TableJobFeed)

		cmd := &pb.CommandRequest{
			Command: "PurgeJobs",
		}
		srv.Command(ctx, cmd)
		jobs, err := srv.ListJobQueue(store.TableJobRunning)
		So(err, ShouldBeNil)
		So(len(jobs), ShouldEqual, 0)

		srv.EnqueJob(ctx, job)
		worker := &pb.Worker{
			Id: "123456",
		}
		srv.GetFeedJob(ctx, worker)
		srv.dequeJob()

		srv.Command(ctx, cmd)

		jobs, err = srv.ListJobQueue(store.TableJobRunning)
		So(err, ShouldBeNil)
		So(len(jobs), ShouldEqual, 0)
	})
}
Beispiel #2
0
func TestServerJob(t *testing.T) {
	setup()
	defer teardown()

	Convey("Manual enqueue to database", t, func() {
		key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())

		job.Key = key.String()

		bytes, err := proto.Marshal(job)
		So(err, ShouldBeNil)

		err = srv.mdb.Put(key.Bytes(), bytes)
		So(err, ShouldBeNil)

		bytes, err = srv.mdb.Get(key.Bytes())
		So(err, ShouldBeNil)

		err = proto.Unmarshal(bytes, job)
		So(err, ShouldBeNil)

		key = store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
		// iter := srv.mdb.Iterator()
		// iter.Seek(key.Prefix().Bytes())
		// defer iter.Close()
		// So(iter.Valid(), ShouldBeTrue)

		got, err := srv.dequeJob()
		So(err, ShouldBeNil)
		So(got.Key, ShouldEqual, job.Key)
		So(got.Id, ShouldEqual, job.Id)
		So(got.RemoteKey, ShouldEqual, job.RemoteKey)

		got, err = srv.dequeJob()
		So(err, ShouldNotBeNil)
	})
}
Beispiel #3
0
func (s *ApiServer) EnqueJob(ctx context.Context, job *pb.FeedJob) (*pb.FeedJob, error) {
	// Time ordered job queue
	key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId())

	job.Key = key.String()
	job.Created = time.Now().Unix()
	job.Updated = time.Now().Unix()

	bytes, err := proto.Marshal(job)
	if err != nil {
		return nil, err
	}
	s.mdb.Put(key.Bytes(), bytes)
	return job, nil
}
Beispiel #4
0
func TestFinishJobQueue(t *testing.T) {
	setup()
	defer teardown()

	Convey("Given ApiServer, enqueue job, deque job, finish job", t, func() {
		ctx := context.Background()
		key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
		job.Key = key.String()
		job.TargetId = "targetId"

		srv.EnqueJob(ctx, job)

		worker := &pb.Worker{
			Id: "123456",
		}

		// running job
		newjob, err := srv.GetFeedJob(ctx, worker)
		So(err, ShouldBeNil)
		So(newjob.Id, ShouldEqual, job.Id)
		So(newjob.RemoteKey, ShouldEqual, job.RemoteKey)
		So(newjob.Key, ShouldNotEqual, job.Key)

		// finished job
		key1 := job.Key
		newjob.Id = "name"
		newjob.Uuid = "c6f8dca854f011ddb489003048343a40"
		finjob, err := srv.FinishJob(ctx, newjob)
		So(err, ShouldBeNil)
		So(finjob.Key, ShouldNotEqual, key1)
		So(finjob.Status, ShouldEqual, "done")

		dbjob, err := store.GetArchiveHistory(srv.mdb, newjob.TargetId)
		So(err, ShouldBeNil)
		So(finjob.Key, ShouldEqual, dbjob.Key)
		So(dbjob.Status, ShouldEqual, "done")

		// check running job states
		jobs, err := srv.ListJobQueue(store.TableJobRunning)
		So(err, ShouldBeNil)
		So(len(jobs), ShouldEqual, 0)
	})
}
Beispiel #5
0
func TestJobQueue(t *testing.T) {
	setup()
	defer teardown()

	Convey("Given ApiServer, When enqueue job, should deque the same job", t, func() {
		ctx := context.Background()
		key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
		job.Key = key.String()

		srv.EnqueJob(ctx, job)
		jobs, err := srv.ListJobQueue(store.TableJobFeed)
		So(err, ShouldBeNil)
		So(len(jobs), ShouldEqual, 1)

		worker := &pb.Worker{
			Id: "123456",
		}
		got, err := srv.GetFeedJob(ctx, worker)
		So(err, ShouldBeNil)
		So(got.Id, ShouldEqual, job.Id)
		So(got.RemoteKey, ShouldEqual, job.RemoteKey)

		_, err = srv.dequeJob()
		So(err, ShouldNotBeNil)

		jobs, err = srv.ListJobQueue(store.TableJobRunning)
		So(err, ShouldBeNil)
		So(len(jobs), ShouldEqual, 1)

		// reopen to check data
		Convey("reopen db should got the same result: no job available", func() {
			srv.Shutdown()
			srv = NewApiServer(dbpath, mcFile)

			_, err = srv.dequeJob()
			So(err, ShouldNotBeNil)
		})
	})
}
Beispiel #6
0
func (s *ApiServer) dequeJob() (*pb.FeedJob, error) {
	var job *pb.FeedJob

	key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId())
	store.ForwardTableScan(s.mdb, key.Prefix(), func(i int, k, v []byte) error {
		job = &pb.FeedJob{}
		if err := proto.Unmarshal(v, job); err != nil {
			return err
		}
		return &store.Error{"ok", store.StopIteration}
	})

	if job == nil {
		return nil, fmt.Errorf("No more job available")
	}

	kb, _ := hex.DecodeString(job.Key)
	if err := s.mdb.Delete(kb); err != nil {
		return nil, err
	}
	return job, nil
}
Beispiel #7
0
func TestMdbReopen(t *testing.T) {
	setup()
	defer teardown()

	Convey("mdb reopen bug: Corruption on wrong key size", t, func() {
		key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())
		job.Key = key.String()

		bytes, err := proto.Marshal(job)
		err = srv.mdb.Put(key.Bytes(), bytes)
		So(err, ShouldBeNil)

		// reopen to check data
		srv.Shutdown()
		srv = NewApiServer(dbpath, mcFile)

		got, err := srv.dequeJob()
		So(err, ShouldBeNil)
		So(got.Key, ShouldEqual, job.Key)
		So(got.Id, ShouldEqual, job.Id)
		So(got.RemoteKey, ShouldEqual, job.RemoteKey)
	})
}
Beispiel #8
0
func TestReopenDeque(t *testing.T) {
	setup()
	defer teardown()

	Convey("mdb redeque", t, func() {
		key := store.NewFlakeKey(store.TableJobFeed, srv.mdb.NextId())

		job.Key = key.String()
		mdb := srv.mdb

		bytes, err := proto.Marshal(job)
		err = mdb.Put(key.Bytes(), bytes)

		_, err = srv.dequeJob()
		So(err, ShouldBeNil)

		// reopen to check data
		srv.Shutdown()
		srv = NewApiServer(dbpath, mcFile)

		_, err = srv.dequeJob()
		So(err, ShouldNotBeNil)
	})
}
Beispiel #9
0
func (s *ApiServer) GetFeedJob(ctx context.Context, in *pb.Worker) (*pb.FeedJob, error) {
	s.Lock()
	defer s.Unlock()

	job, err := s.dequeJob()
	if err != nil {
		return nil, err
	}

	// Time ordered running job
	key := store.NewFlakeKey(store.TableJobRunning, s.mdb.NextId())

	job.Key = key.String()
	job.Worker = in.Id
	job.Created = time.Now().Unix()
	job.Updated = time.Now().Unix()

	bytes, err := proto.Marshal(job)
	if err != nil {
		return nil, err
	}
	s.mdb.Put(key.Bytes(), bytes)
	return job, nil
}