Beispiel #1
0
func (s *ApiServer) FixTooMuchJobs() error {
	log.Println("too much jobs: purging peridoc jobs...")

	prefix := store.TableJobFeed
	_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
		job := &pb.FeedJob{}
		if err := proto.Unmarshal(v, job); err != nil {
			return err
		}
		if int(job.MaxLimit) == 99 {
			return s.mdb.Delete(k)
		}
		return nil
	})
	if err != nil {
		return err
	}

	prefix = store.TableJobRunning
	_, err = store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
		job := &pb.FeedJob{}
		if err := proto.Unmarshal(v, job); err != nil {
			return err
		}
		if int(job.MaxLimit) == 99 {
			return s.mdb.Delete(k)
		}
		return nil
	})

	if err != nil {
		return err
	}
	return nil
}
Beispiel #2
0
func (s *ApiServer) FixJobs() error {
	log.Println("purging all jobs...")

	prefix := store.TableJobFeed
	_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
		job := &pb.FeedJob{}
		if err := proto.Unmarshal(v, job); err != nil {
			return err
		}
		if job.RemoteKey == "" {
			return s.mdb.Delete(k)
		}
		return nil
	})
	if err != nil {
		return err
	}

	prefix = store.TableJobRunning
	_, err = store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
		job := &pb.FeedJob{}
		if err := proto.Unmarshal(v, job); err != nil {
			return err
		}
		if job.RemoteKey == "" {
			return s.mdb.Delete(k)
		}
		return nil
	})

	if err != nil {
		return err
	}
	return nil
}
Beispiel #3
0
func (s *ApiServer) ForwardFetchFeed(ctx context.Context, req *pb.FeedRequest) (*pb.Feed, error) {
	if req.PageSize <= 0 || req.PageSize >= 100 {
		req.PageSize = 50
	}

	profile, err := store.GetProfile(s.mdb, req.Id)
	if err != nil {
		return nil, err
	}

	uuid1, _ := uuid.FromString(profile.Uuid)
	preKey := store.NewUUIDKey(store.TableReverseEntryIndex, uuid1)
	log.Println("forward seeking:", preKey.String())

	start := req.Start
	var entries []*pb.Entry
	_, err = store.ForwardTableScan(s.rdb, preKey, func(i int, k, v []byte) error {
		if start > 0 {
			start--
			return nil // continue
		}

		entry := new(pb.Entry)
		rawdata, err := s.rdb.Get(v) // index value point to entry key
		if err != nil || len(rawdata) == 0 {
			return fmt.Errorf("entry data missing")
		}
		if err := proto.Unmarshal(rawdata, entry); err != nil {
			return err
		}
		if err = FormatFeedEntry(s.mdb, req, entry); err != nil {
			return err
		}

		entries = append(entries, entry)
		if i > int(req.PageSize+req.Start) {
			return &store.Error{"ok", store.StopIteration}
		}
		return nil
	})

	if err != nil {
		return nil, err
	}

	feed := &pb.Feed{
		Uuid:        profile.Uuid,
		Id:          profile.Id,
		Name:        profile.Name,
		Picture:     profile.Picture,
		Type:        profile.Type,
		Private:     profile.Private,
		SupId:       profile.SupId,
		Description: profile.Description,
		Entries:     entries[:],
	}
	return feed, nil
}
Beispiel #4
0
func (s *ApiServer) PurgeJobs() error {
	log.Println("purging all jobs...")

	prefix := store.TableJobFeed
	_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error {
		return s.mdb.Delete(key)
	})
	if err != nil {
		return err
	}

	prefix = store.TableJobRunning
	_, err = store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error {
		return s.mdb.Delete(key)
	})

	if err != nil {
		return err
	}
	return nil
}
Beispiel #5
0
func (s *ApiServer) ListJobQueue(prefix store.Key) (jobs []*pb.FeedJob, err error) {
	log.Println("listing running job...")
	store.ForwardTableScan(s.mdb, prefix, func(i int, key, value []byte) error {
		job := &pb.FeedJob{}
		if err := proto.Unmarshal(value, job); err != nil {
			return err
		}
		// if err = s.mdb.Delete(key); err != nil {
		// 	return err
		// }
		jobs = append(jobs, job)
		log.Println("found entry:", hex.EncodeToString(key))
		return nil
	})
	return
}
Beispiel #6
0
func (s *ApiServer) RefetchFriendFeed() error {
	prefix := store.TableProfile
	j := 0
	n, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
		profile := &pb.Profile{}
		if err := proto.Unmarshal(v, profile); err != nil {
			return err
		}

		if profile.RemoteKey != "" {
			return nil
		}

		oldjob, err := store.GetArchiveHistory(s.mdb, profile.Id)
		if err != nil {
			return err
		}

		if oldjob.Id == "" || oldjob.RemoteKey == "" {
			log.Println("Refetch Friendfeed: unknown remote key")
			return nil
		}

		job := &pb.FeedJob{
			Uuid:        profile.Uuid,
			Id:          oldjob.Id,
			RemoteKey:   oldjob.RemoteKey,
			TargetId:    profile.Id,
			Start:       0,
			PageSize:    100,
			MaxLimit:    99,
			ForceUpdate: true,
			Created:     time.Now().Unix(),
			Updated:     time.Now().Unix(),
		}

		log.Println(job)
		_, err = s.EnqueJob(context.Background(), job)
		j++
		return err
	})
	if err != nil {
		log.Println("Error on scanning user profiles:", err)
	}
	log.Printf("Jobs pulled: %d scanned, %d friendfeed feeds scheduled.", n, j)
	return err
}
Beispiel #7
0
func (s *ApiServer) RefetchUserFeed() error {
	prefix := store.TableProfile
	j := 0
	n, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
		profile := &pb.Profile{}
		if err := proto.Unmarshal(v, profile); err != nil {
			return err
		}

		feedinfo, _ := store.GetFeedinfo(s.rdb, profile.Uuid)
		// only sync twitter service
		graph := BuildGraph(feedinfo)
		if _, ok := graph.Services["twitter"]; !ok {
			return nil
		}

		service := graph.Services["twitter"]
		if service.Oauth == nil {
			return nil
		}
		job := &pb.FeedJob{
			Uuid:    feedinfo.Uuid,
			Id:      feedinfo.Id,
			Profile: profile,
			Service: service,
			Start:   0,
			Created: time.Now().Unix(),
			Updated: time.Now().Unix(),
		}

		_, err := s.EnqueJob(context.Background(), job)
		j++
		return err
	})
	if err != nil {
		log.Println("Error on scanning user profiles:", err)
	}
	log.Printf("Jobs pulled: %d scanned, %d user feeds scheduled.", n, j)
	return err
}
Beispiel #8
0
func (s *ApiServer) RedoFailedJob() error {
	log.Println("redo failed jobs...")

	prefix := store.TableJobRunning
	_, err := store.ForwardTableScan(s.mdb, prefix, func(i int, k, v []byte) error {
		job := &pb.FeedJob{}
		if err := proto.Unmarshal(v, job); err != nil {
			return err
		}

		_, err := s.EnqueJob(context.Background(), job)
		if err != nil {
			return s.mdb.Delete(k)
		}
		return nil
	})

	if err != nil {
		return err
	}
	return nil
}
Beispiel #9
0
func (s *ApiServer) dequeJob() (*pb.FeedJob, error) {
	var job *pb.FeedJob

	key := store.NewFlakeKey(store.TableJobFeed, s.mdb.NextId())
	store.ForwardTableScan(s.mdb, key.Prefix(), func(i int, k, v []byte) error {
		job = &pb.FeedJob{}
		if err := proto.Unmarshal(v, job); err != nil {
			return err
		}
		return &store.Error{"ok", store.StopIteration}
	})

	if job == nil {
		return nil, fmt.Errorf("No more job available")
	}

	kb, _ := hex.DecodeString(job.Key)
	if err := s.mdb.Delete(kb); err != nil {
		return nil, err
	}
	return job, nil
}