// Run Stat Command Implementation
func (c *TotalCommand) Run(args []string) int {
cmdFlags := flag.NewFlagSet("total", flag.ContinueOnError)
cmdFlags.Usage = func() { c.UI.Output(c.Help()) }
env.ConfigFlags(cmdFlags)
if err := cmdFlags.Parse(args); err != nil {
fmt.Println("Could not parse config: ", err)
return 1
}
environment, err := env.GetEnvironment()
if err != nil {
fmt.Println("Could not parse config ", err)
return 1
}
mysql, err = env.GetConnection(environment)
if err != nil {
fmt.Println("Could not Get DB Connection: ", err)
return 1
}
fixPostsWithNoGroup()
//loop users
totalStatUsers()
//loop groups
totalStatGroups()
return 0
}
// Run Serve Command Implementation
func (c *ServeCommand) Run(args []string) int {
cmdFlags := flag.NewFlagSet("serve", flag.ContinueOnError)
cmdFlags.Usage = func() { c.UI.Output(c.Help()) }
env.ConfigFlags(cmdFlags)
if err := cmdFlags.Parse(args); err != nil {
fmt.Println("Could not parse config: ", err)
return 1
}
environment, err := env.GetEnvironment()
if err != nil {
fmt.Println("Could not parse config ", err)
return 1
}
mysql, err = env.GetConnection(environment)
if err != nil {
fmt.Println("Could not Get DB Connection: ", err)
return 1
}
fs := http.FileServer(http.Dir("../public"))
http.Handle("/public/", http.StripPrefix("/public/", fs))
http.HandleFunc("/", homeEntry)
http.HandleFunc("/search/do", doSearchHandler)
http.HandleFunc("/rank/do", doRankHandler)
err = http.ListenAndServe(environment.HTTP, nil)
if err != nil {
fmt.Println("Server Failed to Start, ", err.Error())
return 1
}
return 0
}
// Run Stat Command Implementation
func (c *StatCommand) Run(args []string) int {
var startDate string
var endDate string
cmdFlags := flag.NewFlagSet("stat", flag.ContinueOnError)
cmdFlags.Usage = func() { c.UI.Output(c.Help()) }
env.ConfigFlags(cmdFlags)
cmdFlags.StringVar(&startDate, "start", "", "Start Date.")
cmdFlags.StringVar(&endDate, "end", "", "End Date.")
if err := cmdFlags.Parse(args); err != nil {
fmt.Println("Could not parse config: ", err)
return 1
}
environment, err := env.GetEnvironment()
if err != nil {
fmt.Println("Could not parse config ", err)
return 1
}
mysql, err = env.GetConnection(environment)
if err != nil {
fmt.Println("Could not Get DB Connection: ", err)
return 1
}
startTime := time.Now().Add(time.Hour * -24)
endTime := time.Now().Add(time.Hour * -24)
if startDate != "" {
startTime = parseDate(startDate)
}
if endDate != "" {
endTime = parseDate(endDate).Add(time.Hour * 24)
}
fixPostsWithNoGroup()
for startTime.Before(endTime) {
currentDate := times.Format("Y-m-d", startTime)
fmt.Println("Running date: ", currentDate, " ...")
//loop users
statUsers(currentDate)
//loop groups
statGroups(currentDate)
startTime = startTime.Add(time.Hour * 24)
}
return 0
}
//Run Crawl Command Run
func (c *CrawlCommand) Run(args []string) int {
cmdFlags := flag.NewFlagSet("crawl", flag.ContinueOnError)
cmdFlags.Usage = func() { c.UI.Output(c.Help()) }
env.ConfigFlags(cmdFlags)
if err := cmdFlags.Parse(args); err != nil {
fmt.Println("Could not parse config: ", err)
return 1
}
environment, err := env.GetEnvironment()
if err != nil {
fmt.Println("Could not parse config ", err)
return 1
}
db, err := env.GetConnection(environment)
if err != nil {
l4g.Error("Could not Get DB Connection: ", err)
return 1
}
sharedHTTPClient = &http.Client{}
messageChan = make(chan *schema.MessageBase)
groupChan = make(chan *schema.GroupBase)
quitChan = make(chan int)
resumeChan = make(chan int)
idChan = make(chan int)
isRunning = false
continuousFailedIds = []int{}
go stopper()
go scheduler()
go func() {
resumeChan <- 1
}()
for {
select {
case id := <-idChan:
go crawler(id + 1)
case m := <-messageChan:
m.Message.Save(db)
go next(m.ID)
case g := <-groupChan:
g.Group.Save(db)
go next(g.ID)
case <-quitChan:
time.Sleep(1 * time.Second)
l4g.Info("[crawl.scheduler] Goodbye!")
time.Sleep(100 * time.Millisecond)
return 0
case status := <-resumeChan:
// status = 1 will trigger crawling start.
l4g.Info("[crawl.scheduler] Resume Signal %d", status)
if status == 1 && !isRunning {
//check db for max(id).
var maxID int64
statment, err := db.Prepare(`
SELECT MAX(id) FROM
(SELECT MAX(id) AS id FROM users u
UNION
SELECT MAX(id) AS id FROM threads t
UNION
SELECT MAX(id) AS id FROM posts p)
utp
`)
if err != nil {
l4g.Error("[mysql.getMax] %s", err)
}
err = statment.QueryRow().Scan(&maxID)
if err != nil {
l4g.Error("[mysql.getMax.scan] %s", err)
}
statment.Close()
l4g.Info("[mysql.getMax] Get MaxId %d", maxID)
l4g.Info("[crawl.scheduler] Resumed from %d", maxID)
isRunning = true
go next(int(maxID))
}
if status == 0 && isRunning {
l4g.Info("[crawl.scheduler] Paused")
isRunning = false
}
}
}
}
func doRankHandler(w http.ResponseWriter, r *http.Request) {
type Rank struct {
Rank int `json:"rank"`
Target interface{} `json:"target"`
Value int `json:"value"`
}
type RankItem struct {
ID schema.Stat `json:"id"`
Title string `json:"title"`
Items []Rank `json:"items"`
}
type RankResult struct {
Date string `json:"date"`
UserRanks []RankItem `json:"user_ranks"`
GroupRanks []RankItem `json:"group_ranks"`
}
statDate := times.Format("Y-m-d", time.Now().Add(time.Hour*-24))
rankResult := RankResult{}
rankResult.Date = statDate
environment, _ := env.GetEnvironment()
redis := goredis.Client{}
redis.Addr = environment.RedisServer + ":" + strconv.Itoa(environment.RedisPort)
redisKey := "spored:ranks"
redisValue, _ := redis.Get(redisKey)
if redisValue == nil || string(redisValue) == "" {
//select target date related user stats
userWhereDict := map[schema.Stat]string{
schema.StatThreads: "1970-01-02",
schema.StatPosts: "1970-01-04",
schema.StatLives: "1970-01-06",
schema.StatAttachments: "1970-01-08",
}
sql := "SELECT owner_id, stat_value FROM stats WHERE date = ? AND stat_type = ? AND owner_type = ? ORDER BY stat_value DESC LIMIT ?"
stmt, _ := mysql.Prepare(sql)
defer stmt.Close()
for statType, date := range userWhereDict {
rankSeq := 0
userRank := RankItem{}
userRank.ID = statType
switch statType {
case 2:
userRank.Title = "用户帖子数排行"
case 4:
userRank.Title = "用户回复数排行"
case 6:
userRank.Title = "用户活跃版数排行"
case 8:
userRank.Title = "用户附件数排行"
}
rows, _ := stmt.Query(date, statType, schema.OwnerUser, 50)
for rows.Next() {
rankSeq++
var ownerID, statValue int
rank := Rank{}
rows.Scan(&ownerID, &statValue)
rank.Rank = rankSeq
rank.Value = statValue
user := schema.User{}
user.New(mysql, ownerID)
rank.Target = user
userRank.Items = append(userRank.Items, rank)
}
rankResult.UserRanks = append(rankResult.UserRanks, userRank)
}
//select target date related group stats
groupWhereDict := map[schema.Stat]string{
schema.StatThreads: "1970-01-02",
schema.StatPosts: "1970-01-04",
schema.StatLives: "1970-01-06",
}
for statType, date := range groupWhereDict {
rankSeq := 0
groupRank := RankItem{}
groupRank.ID = statType
switch statType {
case 2:
groupRank.Title = "群组帖子数排行"
case 4:
groupRank.Title = "群组回复数排行"
case 6:
groupRank.Title = "群组活跃用户版数排行"
}
rows, _ := stmt.Query(date, statType, schema.OwnerGroup, 20)
for rows.Next() {
rankSeq++
var ownerID, statValue int
rank := Rank{}
rows.Scan(&ownerID, &statValue)
rank.Rank = rankSeq
rank.Value = statValue
group := schema.Group{}
group.New(mysql, ownerID)
rank.Target = group
groupRank.Items = append(groupRank.Items, rank)
}
rankResult.GroupRanks = append(rankResult.GroupRanks, groupRank)
}
resultJSON, _ := json.Marshal(rankResult)
fmt.Fprintf(w, string(resultJSON))
redis.Set(redisKey, resultJSON)
redis.Expire(redisKey, times.StrToLocalTime(times.Format("Y-m-d", time.Now().Add(24*time.Hour))+" 01:00:00").Unix()-time.Now().Unix())
return
}
fmt.Fprintf(w, string(redisValue))
}
func doSearchHandler(w http.ResponseWriter, r *http.Request) {
type SearchResult struct {
Total int `json:"total"`
Groups map[string]uint `json:"groups"`
Results []interface{} `json:"results"`
Page int `json:"page"`
Error string `json:"error"`
}
environment, _ := env.GetEnvironment()
sphinx := gosphinx.NewSphinxClient()
sphinx.SetServer(environment.SphinxServer, environment.SphinxPort)
sphinx.Open()
defer sphinx.Close()
searchType := r.URL.Query().Get("t")
searchWord := r.URL.Query().Get("q")
page := 1
if r.URL.Query().Get("page") != "" {
page, _ = strconv.Atoi(r.URL.Query().Get("page"))
if page == 0 {
page = 1
}
}
if searchWord == "" {
result := SearchResult{Error: "请输入要搜索的关键词"}
resultJSON, _ := json.Marshal(result)
fmt.Fprintf(w, string(resultJSON))
return
}
result := SearchResult{Error: ""}
if searchType == "" {
sphinx.SetGroupBy("id_type", gosphinx.SPH_GROUPBY_ATTR, "@group asc")
res, err := sphinx.Query(searchWord, "spored", "Group Query for "+searchWord)
if err != nil {
result.Error = "Sphinx Error " + err.Error()
resultJSON, _ := json.Marshal(result)
fmt.Fprintf(w, string(resultJSON))
return
}
groups := map[string]uint{}
for _, r := range res.Matches {
typeRef := reflect.ValueOf(r.AttrValues[0])
switch typeRef.Uint() {
case 1:
ref := reflect.ValueOf(r.AttrValues[2])
groups["users"] = uint(ref.Uint())
case 2:
ref := reflect.ValueOf(r.AttrValues[2])
groups["groups"] = uint(ref.Uint())
case 3:
ref := reflect.ValueOf(r.AttrValues[2])
groups["threads"] = uint(ref.Uint())
}
}
result.Groups = groups
}
sphinx.ResetFilters()
sphinx.ResetGroupBy()
if searchType != "" {
searchTypeInt, _ := strconv.Atoi(searchType)
sphinx.SetFilter("id_type", []uint64{uint64(searchTypeInt)}, false)
}
sphinx.SetLimits((page-1)*10, 10, 5000, 5000)
res, err := sphinx.Query(searchWord, "spored", "Query for "+searchWord)
if err != nil {
result.Error = "Sphinx Error" + err.Error()
resultJSON, _ := json.Marshal(result)
fmt.Fprintf(w, string(resultJSON))
return
}
results := []interface{}{}
for _, r := range res.Matches {
redis := goredis.Client{}
redis.Addr = environment.RedisServer + ":" + strconv.Itoa(environment.RedisPort)
idTypeRef := reflect.ValueOf(r.AttrValues[0])
idType := idTypeRef.Uint()
redisKey := fmt.Sprintf("spored:%d_%d", idType, r.DocId)
redisValue, _ := redis.Get(redisKey)
if redisValue == nil || string(redisValue) == "" {
//update cache
var jsonBytes []byte
switch idType {
case 1:
object := schema.User{}
object.New(mysql, int(r.DocId))
jsonBytes, _ = json.Marshal(object)
results = append(results, object)
case 2:
object := schema.Group{}
object.New(mysql, int(r.DocId))
jsonBytes, _ = json.Marshal(object)
results = append(results, object)
case 3:
object := schema.Message{}
object.New(mysql, int(r.DocId))
jsonBytes, _ = json.Marshal(object)
results = append(results, object)
}
if jsonBytes != nil {
redis.Set(redisKey, jsonBytes)
redis.Expire(redisKey, 24*3600)
}
} else {
switch idType {
case 1:
object := schema.User{}
json.Unmarshal(redisValue, &object)
results = append(results, object)
case 2:
object := schema.Group{}
json.Unmarshal(redisValue, &object)
results = append(results, object)
case 3:
object := schema.Message{}
json.Unmarshal(redisValue, &object)
results = append(results, object)
}
}
}
result.Results = results
result.Page = page
result.Total = res.Total
resultJSON, _ := json.Marshal(result)
fmt.Fprintf(w, string(resultJSON))
}
