func (s *ApiServer) FixTooMuchJobs() error {
log.Println("too much jobs: purging peridoc jobs...")
prefix := store.TableJobFeed
_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
job := &pb.FeedJob{}
if err := proto.Unmarshal(v, job); err != nil {
return err
}
if int(job.MaxLimit) == 99 {
return s.mdb.Delete(k)
}
return nil
})
if err != nil {
return err
}
prefix = store.TableJobRunning
_, err = store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
job := &pb.FeedJob{}
if err := proto.Unmarshal(v, job); err != nil {
return err
}
if int(job.MaxLimit) == 99 {
return s.mdb.Delete(k)
}
return nil
})
if err != nil {
return err
}
return nil
}
func (s *ApiServer) FixJobs() error {
log.Println("purging all jobs...")
prefix := store.TableJobFeed
_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
job := &pb.FeedJob{}
if err := proto.Unmarshal(v, job); err != nil {
return err
}
if job.RemoteKey == "" {
return s.mdb.Delete(k)
}
return nil
})
if err != nil {
return err
}
prefix = store.TableJobRunning
_, err = store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
job := &pb.FeedJob{}
if err := proto.Unmarshal(v, job); err != nil {
return err
}
if job.RemoteKey == "" {
return s.mdb.Delete(k)
}
return nil
})
if err != nil {
return err
}
return nil
}
func (s *ApiServer) ForwardFetchFeed(ctx context.Context, req *pb.FeedRequest) (*pb.Feed, error) {
if req.PageSize <= 0 || req.PageSize >= 100 {
req.PageSize = 50
}
profile, err := store.GetProfile(s.mdb, req.Id)
if err != nil {
return nil, err
}
uuid1, _ := uuid.FromString(profile.Uuid)
preKey := store.NewUUIDKey(store.TableReverseEntryIndex, uuid1)
log.Println("forward seeking:", preKey.String())
start := req.Start
var entries []*pb.Entry
_, err = store.ForwardTableScan(s.rdb, preKey, func(i int, k, v []byte) error {
if start > 0 {
start--
return nil // continue
}
entry := new(pb.Entry)
rawdata, err := s.rdb.Get(v) // index value point to entry key
if err != nil || len(rawdata) == 0 {
return fmt.Errorf("entry data missing")
}
if err := proto.Unmarshal(rawdata, entry); err != nil {
return err
}
if err = FormatFeedEntry(s.mdb, req, entry); err != nil {
return err
}
entries = append(entries, entry)
if i > int(req.PageSize+req.Start) {
return &store.Error{"ok", store.StopIteration}
}
return nil
})
if err != nil {
return nil, err
}
feed := &pb.Feed{
Uuid: profile.Uuid,
Id: profile.Id,
Name: profile.Name,
Picture: profile.Picture,
Type: profile.Type,
Private: profile.Private,
SupId: profile.SupId,
Description: profile.Description,
Entries: entries[:],
}
return feed, nil
}
func (s *ApiServer) PurgeJobs() error {
log.Println("purging all jobs...")
prefix := store.TableJobFeed
_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error {
return s.mdb.Delete(key)
})
if err != nil {
return err
}
prefix = store.TableJobRunning
_, err = store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error {
return s.mdb.Delete(key)
})
if err != nil {
return err
}
return nil
}
func (s *ApiServer) ListJobQueue(prefix store.Key) (jobs []*pb.FeedJob, err error) {
log.Println("listing running job...")
store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error {
job := &pb.FeedJob{}
if err := proto.Unmarshal(value, job); err != nil {
return err
}
// if err = s.mdb.Delete(key); err != nil {
// return err
// }
jobs = append(jobs, job)
log.Println("found entry:", hex.EncodeToString(key))
return nil
})
return
}
func (s *ApiServer) RefetchFriendFeed() error {
prefix := store.TableProfile
j := 0
n, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
profile := &pb.Profile{}
if err := proto.Unmarshal(v, profile); err != nil {
return err
}
if profile.RemoteKey != "" {
return nil
}
oldjob, err := store.GetArchiveHistory(s.mdb, profile.Id)
if err != nil {
return err
}
if oldjob.Id == "" || oldjob.RemoteKey == "" {
log.Println("Refetch Friendfeed: unknown remote key")
return nil
}
job := &pb.FeedJob{
Uuid: profile.Uuid,
Id: oldjob.Id,
RemoteKey: oldjob.RemoteKey,
TargetId: profile.Id,
Start: 0,
PageSize: 100,
MaxLimit: 99,
ForceUpdate: true,
Created: time.Now().Unix(),
Updated: time.Now().Unix(),
}
log.Println(job)
_, err = s.EnqueJob(context.Background(), job)
j++
return err
})
if err != nil {
log.Println("Error on scanning user profiles:", err)
}
log.Printf("Jobs pulled: %d scanned, %d friendfeed feeds scheduled.", n, j)
return err
}
func (s *ApiServer) RefetchUserFeed() error {
prefix := store.TableProfile
j := 0
n, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
profile := &pb.Profile{}
if err := proto.Unmarshal(v, profile); err != nil {
return err
}
feedinfo, _ := store.GetFeedinfo(s.rdb, profile.Uuid)
// only sync twitter service
graph := BuildGraph(feedinfo)
if _, ok := graph.Services["twitter"]; !ok {
return nil
}
service := graph.Services["twitter"]
if service.Oauth == nil {
return nil
}
job := &pb.FeedJob{
Uuid: feedinfo.Uuid,
Id: feedinfo.Id,
Profile: profile,
Service: service,
Start: 0,
Created: time.Now().Unix(),
Updated: time.Now().Unix(),
}
_, err := s.EnqueJob(context.Background(), job)
j++
return err
})
if err != nil {
log.Println("Error on scanning user profiles:", err)
}
log.Printf("Jobs pulled: %d scanned, %d user feeds scheduled.", n, j)
return err
}
func (s *ApiServer) RedoFailedJob() error {
log.Println("redo failed jobs...")
prefix := store.TableJobRunning
_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
job := &pb.FeedJob{}
if err := proto.Unmarshal(v, job); err != nil {
return err
}
_, err := s.EnqueJob(context.Background(), job)
if err != nil {
return s.mdb.Delete(k)
}
return nil
})
if err != nil {
return err
}
return nil
}
func (s *ApiServer) dequeJob() (*pb.FeedJob, error) {
var job *pb.FeedJob
key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId())
store.ForwardTableScan(s.mdb, key.Prefix(), func(i int, k, v []byte) error {
job = &pb.FeedJob{}
if err := proto.Unmarshal(v, job); err != nil {
return err
}
return &store.Error{"ok", store.StopIteration}
})
if job == nil {
return nil, fmt.Errorf("No more job available")
}
kb, _ := hex.DecodeString(job.Key)
if err := s.mdb.Delete(kb); err != nil {
return nil, err
}
return job, nil
}