Example #1
0
func getPublishersHandler(w http.ResponseWriter, r *http.Request) {

	publishers, err := rpc.GetFollowing(adminTwisterUsername)
	if err != nil {
		log.Println("Error: getPublishersHandler can not fetch following users for", adminTwisterUsername, "from Twister RPC server.", err)
		http.Error(w, "getPublishersHandler RPC err", 500)
		return
	}

	data, _ := json.Marshal(publishers)

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Write(data)

}
Example #2
0
func syncTimeLine() {
	// Update publishers
	twisterUsers, err := rpc.GetFollowing(adminTwisterUsername)
	if err != nil {
		log.Println("Error: can not fetch following users for", adminTwisterUsername, "from Twister RPC server.", err)
	}
	log.Println(twisterUsers)
	log.Println("Get", len(*twisterUsers), "following users for", adminTwisterUsername)
	db.AddPublishersIfNotExist(twisterUsers)

	// Get all publishers' username, max id, since id and latest id (k), from DB
	publishers, err := db.GetPublishers()
	if err != nil {
		log.Println(err)
		log.Printf("Error: syncTimeLine GetPublishers: %+v\n", publishers)
	}
	//log.Printf("%+v\n", publishers)

	// Get posts from Twister for all publishers
	var gup getUserPosts
	for name, data := range publishers {
		gup.addUser(name, data.Max, data.Since)
	}

	newPosts, err := rpc.GetPosts(sync_posts_number, gup.data)
	//log.Printf("%+v\n", presult)
	if err != nil {
		log.Println(err)
		log.Printf("Error: syncTimeLine GetPosts: %+v\n", gup.data)
		return
	}
	log.Println(len(*newPosts), "new posts")

	// Update publishers sync data with info from retrieved posts.
	var newShPosts db.ShPosts
	var newShPubReplyPosts db.ShPubReplyPosts
	hasNewPostsPublishers := make(map[string]updatePostsInfo)
	for _, tp := range *newPosts {

		// Make sure current post's username is in the publishers list.(May contain promoted post)
		_, ok := publishers[tp.Userpost.N]
		if !ok {
			continue
		}

		// Get lastK, may be nil pointer.
		var lastK int64
		if tp.Userpost.Lastk == nil {
			lastK = -1
		} else {
			lastK = *(tp.Userpost.Lastk)
		}

		upi, ok := hasNewPostsPublishers[tp.Userpost.N]
		if ok {
			upi.lastK = lastK
			if upi.maxK < tp.Userpost.K {
				upi.maxK = tp.Userpost.K
			}
			if upi.minK > tp.Userpost.K {
				upi.minK = tp.Userpost.K
			}
			hasNewPostsPublishers[tp.Userpost.N] = upi
		} else {
			hasNewPostsPublishers[tp.Userpost.N] = updatePostsInfo{
				lastK: lastK,
				maxK:  tp.Userpost.K,
				minK:  tp.Userpost.K,
			}

		}

		// Add New ShPost
		if tp.Userpost.Reply == nil && tp.Userpost.Rt == nil {
			btih, category, fileSize, title, ok := retrieveMagnetInfo(tp.Userpost.Msg)
			if ok {
				shPost := db.ShPost{
					Msg:      tp.Userpost.Msg,
					N:        tp.Userpost.N,
					K:        tp.Userpost.K,
					Lastk:    lastK,
					Time:     tp.Userpost.Time,
					Category: category,
					Title:    title,
					Magnet:   btih,
					Size:     fileSize,
				}

				newShPosts = append(newShPosts, shPost)
				//fmt.Println(index, tp.Userpost.N, tp.Userpost.K, lastK)
			}
		}

		// Add new ShPubReplyPost
		if tp.Userpost.Reply != nil {
			if tp.Userpost.Reply.N == tp.Userpost.N {
				shPubReplyPosts := db.ShPubReplyPost{
					Msg:   tp.Userpost.Msg,
					Time:  tp.Userpost.Time,
					N:     tp.Userpost.N,
					K:     tp.Userpost.Reply.K,
					Lastk: lastK,
				}
				newShPubReplyPosts = append(newShPubReplyPosts, shPubReplyPosts)
				//fmt.Println(index, tp.Userpost.N, tp.Userpost.K, lastK)
			}
		}
	}

	// Save posts to DB
	db.AddPosts(&newShPosts)

	err = db.AddPublishersReplyPosts(&newShPubReplyPosts)
	if err != nil {
		log.Println(err)
		log.Printf("Error: syncTimeLine db.AddPublishersReplyPosts: %+v\n", len(newShPubReplyPosts))
	}

	//log.Printf("%+v\n", hasNewPostsPublishers)

	// Update publishers in DB
	updatedPublishers := make(map[string]db.SyncData)
	for name, postsInfo := range hasNewPostsPublishers {
		var newSd db.SyncData
		oldSd := publishers[name]

		newSd.Latest = oldSd.Latest
		if postsInfo.maxK > oldSd.Latest {
			newSd.Latest = postsInfo.maxK
		}

		newSd.Since = oldSd.Since
		lastK := min(postsInfo.lastK, postsInfo.minK-1)
		if lastK <= oldSd.Since {
			newSd.Since = newSd.Latest
			newSd.Max = max_post_id
		} else {
			newSd.Max = lastK
		}

		updatedPublishers[name] = newSd
	}

	err = db.UpdatePublishers(&updatedPublishers)
	if err != nil {
		log.Println(err)
		log.Printf("Error: syncTimeLine: %+v\n", updatedPublishers)
	}

	// Indexing posts
	search.Index(&newShPosts)

}