// Might be worth putting a defer in here incase a job breaks // I.E to Clean up is processing func Learn(s *session.Session) string { if s.SetLearning() { // New log.Println("Set up learning") status := http.GetStatus(s) s.SetLimits(int(status.Follows), int(status.Followed_by)) } if !s.SetProcessing() { // Show we're still working return " *" } switch s.GetLearningStep() { case utils.APPRAISE: jobs.MinePeople(s) return StatusBar(s, "Mining Followers") case utils.SCORN: jobs.MinePeople(s) return StatusBar(s, "Mining Following") case utils.BUILD: // Logistic Regression // Get records and run go jobs.LogisticRegression(s) s.IncrementStep() return "* Running Logistic Regression" case utils.GOODTAGS: go jobs.CrawlTags(s, true) return StatusBar(s, "Finding Good Tags") case utils.BADTAGS: go jobs.CrawlTags(s, false) return StatusBar(s, "Finding Bad Tags") case utils.COMPUTETAGS: go jobs.WeightTags(s) return "* Ranking Tags" case utils.SHARE: go s.Share() s.IncrementStep() s.StopProcessing() return "Sharing" } return "Stop" }
func MinePeople(s *session.Session) { // Set up channel next := make(chan *http.Users) var batch http.Users if s.GetNext() == "" { if s.GetState() > 0 { s.IncrementStep() } if s.GetLearningStep() == utils.APPRAISE { batch = http.GetFollowers(s) } else { batch = http.GetFollowing(s) } } else { batch = http.GetNext(s, s.GetNext()) } go listen(s, next, 0, s.IncrementState()) next <- &batch }
// Async set up multi calls func listen(s *session.Session, next chan *http.Users, calls int, follows float64) { for { select { case users := <-next: i := len(users.Data) - 1 s.IncrementCount() go process(s, users, i, follows) close(next) if calls == utils.MAXPEOPLEGRAB { s.SetNext(users.Pagination.Next_url) return } var batch http.Users nxt := make(chan *http.Users) if users.Pagination.Next_url != "" { log.Println("Getting another batch") batch = http.GetNext(s, users.Pagination.Next_url) } else if follows == 0 { // follows == float64(s.GetLearningStep()) then have a array of functions log.Println("Proceeding to next Step") s.IncrementStep() s.IncrementState() batch = http.GetFollowing(s) follows = float64(s.GetLearningStep()) } else { s.SetNext("") return } go listen(s, nxt, calls+1, follows) nxt <- &batch return } } }