func (s *ApiServer) FixTooMuchJobs() error { log.Println("too much jobs: purging peridoc jobs...") prefix := store.TableJobFeed _, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error { job := &pb.FeedJob{} if err := proto.Unmarshal(v, job); err != nil { return err } if int(job.MaxLimit) == 99 { return s.mdb.Delete(k) } return nil }) if err != nil { return err } prefix = store.TableJobRunning _, err = store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error { job := &pb.FeedJob{} if err := proto.Unmarshal(v, job); err != nil { return err } if int(job.MaxLimit) == 99 { return s.mdb.Delete(k) } return nil }) if err != nil { return err } return nil }
func (s *ApiServer) FixJobs() error { log.Println("purging all jobs...") prefix := store.TableJobFeed _, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error { job := &pb.FeedJob{} if err := proto.Unmarshal(v, job); err != nil { return err } if job.RemoteKey == "" { return s.mdb.Delete(k) } return nil }) if err != nil { return err } prefix = store.TableJobRunning _, err = store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error { job := &pb.FeedJob{} if err := proto.Unmarshal(v, job); err != nil { return err } if job.RemoteKey == "" { return s.mdb.Delete(k) } return nil }) if err != nil { return err } return nil }
func (s *ApiServer) ForwardFetchFeed(ctx context.Context, req *pb.FeedRequest) (*pb.Feed, error) { if req.PageSize <= 0 || req.PageSize >= 100 { req.PageSize = 50 } profile, err := store.GetProfile(s.mdb, req.Id) if err != nil { return nil, err } uuid1, _ := uuid.FromString(profile.Uuid) preKey := store.NewUUIDKey(store.TableReverseEntryIndex, uuid1) log.Println("forward seeking:", preKey.String()) start := req.Start var entries []*pb.Entry _, err = store.ForwardTableScan(s.rdb, preKey, func(i int, k, v []byte) error { if start > 0 { start-- return nil // continue } entry := new(pb.Entry) rawdata, err := s.rdb.Get(v) // index value point to entry key if err != nil || len(rawdata) == 0 { return fmt.Errorf("entry data missing") } if err := proto.Unmarshal(rawdata, entry); err != nil { return err } if err = FormatFeedEntry(s.mdb, req, entry); err != nil { return err } entries = append(entries, entry) if i > int(req.PageSize+req.Start) { return &store.Error{"ok", store.StopIteration} } return nil }) if err != nil { return nil, err } feed := &pb.Feed{ Uuid: profile.Uuid, Id: profile.Id, Name: profile.Name, Picture: profile.Picture, Type: profile.Type, Private: profile.Private, SupId: profile.SupId, Description: profile.Description, Entries: entries[:], } return feed, nil }
func (s *ApiServer) PurgeJobs() error { log.Println("purging all jobs...") prefix := store.TableJobFeed _, err := store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error { return s.mdb.Delete(key) }) if err != nil { return err } prefix = store.TableJobRunning _, err = store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error { return s.mdb.Delete(key) }) if err != nil { return err } return nil }
func (s *ApiServer) ListJobQueue(prefix store.Key) (jobs []*pb.FeedJob, err error) { log.Println("listing running job...") store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error { job := &pb.FeedJob{} if err := proto.Unmarshal(value, job); err != nil { return err } // if err = s.mdb.Delete(key); err != nil { // return err // } jobs = append(jobs, job) log.Println("found entry:", hex.EncodeToString(key)) return nil }) return }
func (s *ApiServer) RefetchFriendFeed() error { prefix := store.TableProfile j := 0 n, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error { profile := &pb.Profile{} if err := proto.Unmarshal(v, profile); err != nil { return err } if profile.RemoteKey != "" { return nil } oldjob, err := store.GetArchiveHistory(s.mdb, profile.Id) if err != nil { return err } if oldjob.Id == "" || oldjob.RemoteKey == "" { log.Println("Refetch Friendfeed: unknown remote key") return nil } job := &pb.FeedJob{ Uuid: profile.Uuid, Id: oldjob.Id, RemoteKey: oldjob.RemoteKey, TargetId: profile.Id, Start: 0, PageSize: 100, MaxLimit: 99, ForceUpdate: true, Created: time.Now().Unix(), Updated: time.Now().Unix(), } log.Println(job) _, err = s.EnqueJob(context.Background(), job) j++ return err }) if err != nil { log.Println("Error on scanning user profiles:", err) } log.Printf("Jobs pulled: %d scanned, %d friendfeed feeds scheduled.", n, j) return err }
func (s *ApiServer) RefetchUserFeed() error { prefix := store.TableProfile j := 0 n, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error { profile := &pb.Profile{} if err := proto.Unmarshal(v, profile); err != nil { return err } feedinfo, _ := store.GetFeedinfo(s.rdb, profile.Uuid) // only sync twitter service graph := BuildGraph(feedinfo) if _, ok := graph.Services["twitter"]; !ok { return nil } service := graph.Services["twitter"] if service.Oauth == nil { return nil } job := &pb.FeedJob{ Uuid: feedinfo.Uuid, Id: feedinfo.Id, Profile: profile, Service: service, Start: 0, Created: time.Now().Unix(), Updated: time.Now().Unix(), } _, err := s.EnqueJob(context.Background(), job) j++ return err }) if err != nil { log.Println("Error on scanning user profiles:", err) } log.Printf("Jobs pulled: %d scanned, %d user feeds scheduled.", n, j) return err }
func (s *ApiServer) RedoFailedJob() error { log.Println("redo failed jobs...") prefix := store.TableJobRunning _, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error { job := &pb.FeedJob{} if err := proto.Unmarshal(v, job); err != nil { return err } _, err := s.EnqueJob(context.Background(), job) if err != nil { return s.mdb.Delete(k) } return nil }) if err != nil { return err } return nil }
func (s *ApiServer) dequeJob() (*pb.FeedJob, error) { var job *pb.FeedJob key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId()) store.ForwardTableScan(s.mdb, key.Prefix(), func(i int, k, v []byte) error { job = &pb.FeedJob{} if err := proto.Unmarshal(v, job); err != nil { return err } return &store.Error{"ok", store.StopIteration} }) if job == nil { return nil, fmt.Errorf("No more job available") } kb, _ := hex.DecodeString(job.Key) if err := s.mdb.Delete(kb); err != nil { return nil, err } return job, nil }