// Sensivite 用于 echo 框架的过滤发布敏感词(广告) func Sensivite() echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(ctx echo.Context) error { content := ctx.FormValue("content") title := ctx.FormValue("title") user := ctx.Get("user").(*model.Me) if title != "" { for _, s := range titleSensitives { if hasSensitiveChar(title, s) { // 把账号冻结 logic.DefaultUser.UpdateUserStatus(ctx, user.Uid, model.UserStatusFreeze) logger.Infoln("user="******"publish ad, title=", title, ". freeze") return errors.New("对不起,您的账号已被冻结!") } } } if hasSensitive(title, contentSensitives) || hasSensitive(content, contentSensitives) { // 把账号冻结 logic.DefaultUser.UpdateUserStatus(ctx, user.Uid, model.UserStatusFreeze) logger.Infoln("user="******"publish ad, title=", title, ";content=", content, ". freeze") return errors.New("对不起,您的账号已被冻结!") } if err := next(ctx); err != nil { return err } return nil } } }
func indexing(isAll bool) { logger.Infoln("indexing start...") start := time.Now() defer func() { logger.Infoln("indexing spend time:", time.Now().Sub(start)) }() logic.DefaultSearcher.Indexing(isAll) }
// 将所有 角色拥有的权限 加载到内存中;后台修改时,重新加载一次 func LoadRoleAuthorities() error { roleAuthorities := make([]*model.RoleAuthority, 0) err := MasterDB.Find(&roleAuthorities) if err != nil { logger.Errorln("LoadRoleAuthorities role_authority read fail:", err) return err } roleAuthLocker.Lock() defer roleAuthLocker.Unlock() RoleAuthorities = make(map[int][]int) for _, roleAuth := range roleAuthorities { roleId := roleAuth.Roleid if authorities, ok := RoleAuthorities[roleId]; ok { RoleAuthorities[roleId] = append(authorities, roleAuth.Aid) } else { RoleAuthorities[roleId] = []int{roleAuth.Aid} } } logger.Infoln("LoadRoleAuthorities successfully!") return nil }
// 将所有 角色 加载到内存中;后台修改角色时,重新加载一次 func LoadRoles() error { roles := make([]*model.Role, 0) err := MasterDB.Find(&roles) if err != nil { logger.Errorln("LoadRoles role read fail:", err) return err } if len(roles) == 0 { logger.Errorln("LoadRoles role read fail: num is 0") return errors.New("no role") } roleLocker.Lock() defer roleLocker.Unlock() maxRoleid := roles[len(roles)-1].Roleid Roles = make([]*model.Role, maxRoleid) // 由于角色不多,而且一般角色id是连续自增的,因此这里以角色id当slice的index for _, role := range roles { Roles[role.Roleid-1] = role } logger.Infoln("LoadRoles successfully!") return nil }
// 给某个用户发送一条消息 func (this *book) PostMessage(uid int, message *Message) { this.rwMutex.RLock() defer this.rwMutex.RUnlock() if userData, ok := this.users[uid]; ok { logger.Infoln("post message to", uid, message) go userData.SendMessage(message) } }
// 给所有用户广播消息 func (this *book) BroadcastAllUsersMessage(message *Message) { logger.Infoln("BroadcastAllUsersMessage message", message) this.rwMutex.RLock() defer this.rwMutex.RUnlock() for _, userData := range this.users { userData.SendMessage(message) } }
func autocrawl(needAll bool, crawlConfFile string, whichSite string) { content, err := ioutil.ReadFile(crawlConfFile) if err != nil { log.Fatalln("parse crawl config read file error:", err) } err = json.Unmarshal(content, &websites) if err != nil { log.Fatalln("parse crawl config json parse error:", err) } if needAll { // 全量 for website, wbconf := range websites { if whichSite != "" && whichSite != website { continue } logger.Infoln("all crawl", website) go doCrawl(wbconf, true) } } // 定时增量 c := cron.New() c.AddFunc(config.ConfigFile.MustValue("crawl", "spec", "0 0 */1 * * ?"), func() { // 抓取 reddit go logic.DefaultReddit.Parse("") // 抓取 www.oschina.net/project go logic.DefaultProject.ParseProjectList("http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time") for website, wbconf := range websites { if whichSite != "" && whichSite != website { continue } logger.Infoln("do crawl", website) go doCrawl(wbconf, false) } }) c.Start() }
func parseArticleList(url, listselector, resultselector string, isAuto bool) (err error) { logger.Infoln("parse url:", url) var doc *goquery.Document if strings.Contains(url, "oschina.net") { req, err := http.NewRequest("GET", url, nil) if err != nil { return err } req.Header.Add("Referer", "http://www.oschina.net/search?q=go&scope=blog&onlytitle=1&sort_by_time=1") resp, err := http.DefaultClient.Do(req) if err != nil { return err } doc, err = goquery.NewDocumentFromResponse(resp) } else { doc, err = goquery.NewDocument(url) } if err != nil { return } doc.Find(listselector).Each(func(i int, contentSelection *goquery.Selection) { aSelection := contentSelection.Find(resultselector) if isAuto { title := aSelection.Text() matched, err := regexp.MatchString(pattern, title) if err != nil { logger.Errorln(err) return } if !matched { return } } articleUrl, ok := aSelection.Attr("href") if ok { pos := strings.LastIndex(articleUrl, "?") if pos != -1 { articleUrl = articleUrl[:pos] } logic.DefaultArticle.ParseArticle(context.Background(), articleUrl, isAuto) } }) return }
// 给除了自己的其他用户广播消息 func (this *book) BroadcastToOthersMessage(message *Message, myself int) { logger.Infoln("BroadcastToOthersMessage message", message) this.rwMutex.RLock() defer this.rwMutex.RUnlock() for uid, userData := range this.users { if uid == myself { continue } userData.SendMessage(message) } }
func (this *UserData) SendMessage(message *Message) { this.rwMutex.RLock() defer this.rwMutex.RUnlock() for serverId, messageQueue := range this.serverMsgQueue { // 有可能用户已经退出,导致 messageQueue满,阻塞 if len(messageQueue) < MessageQueueLen { messageQueue <- message } else { logger.Infoln("server_id:", serverId, "had close") } } }
// 将所有 资源分类信息 加载到内存中:后台修改节点时,重新加载一次 func LoadCategories() (err error) { categories := make([]*model.ResourceCat, 0) err = MasterDB.Find(&categories) if err != nil { logger.Errorln("LoadCategories category read fail:", err) return } catRWMutex.Lock() defer catRWMutex.Unlock() AllCategory = categories logger.Infoln("LoadCategories successfully!") return }
// 将所有 权限 加载到内存中;后台修改权限时,重新加载一次 func LoadAuthorities() error { authorities := make([]*model.Authority, 0) err := MasterDB.Find(&authorities) if err != nil { logger.Errorln("LoadAuthorities authority read fail:", err) return err } authLocker.Lock() defer authLocker.Unlock() Authorities = authorities logger.Infoln("LoadAuthorities successfully!") return nil }
// websocket,统计在线用户数 // uri: /ws func (this *WebsocketController) Ws(wsConn *websocket.Conn) { defer wsConn.Close() this.mutex.Lock() this.ServerId++ serverId := this.ServerId this.mutex.Unlock() req := wsConn.Request() user := goutils.MustInt(req.FormValue("uid")) if user == 0 { user = int(goutils.Ip2long(goutils.RemoteIp(req))) } userData := logic.Book.AddUser(user, serverId) // 给自己发送消息,告诉当前在线用户数、历史最高在线人数 onlineInfo := map[string]int{"online": logic.Book.Len(), "maxonline": logic.MaxOnlineNum()} message := logic.NewMessage(logic.WsMsgOnline, onlineInfo) err := websocket.JSON.Send(wsConn, message) if err != nil { logger.Errorln("Sending onlineusers error:", err) } var clientClosed = false for { select { case message := <-userData.MessageQueue(serverId): if err := websocket.JSON.Send(wsConn, message); err != nil { clientClosed = true } // 心跳 case <-time.After(30e9): if err := websocket.JSON.Send(wsConn, ""); err != nil { clientClosed = true } } if clientClosed { logic.Book.DelUser(user, serverId) logger.Infoln("user:"******"client close") break } } // 用户退出时需要变更其他用户看到的在线用户数 if !logic.Book.UserIsOnline(user) { message := logic.NewMessage(logic.WsMsgOnline, map[string]int{"online": logic.Book.Len()}) go logic.Book.BroadcastAllUsersMessage(message) } }
// SendMail 发送电子邮件 func (EmailLogic) SendMail(subject, content string, tos []string) error { emailConfig, _ := config.ConfigFile.GetSection("email") message := `From: Go语言中文网 | Golang中文社区 | Go语言学习园地<` + emailConfig["from_email"] + `> To: ` + strings.Join(tos, ",") + ` Subject: ` + subject + ` Content-Type: text/html;charset=UTF-8 ` + content smtpAddr := emailConfig["smtp_host"] + ":" + emailConfig["smtp_port"] auth := smtp.PlainAuth("", emailConfig["smtp_username"], emailConfig["smtp_password"], emailConfig["smtp_host"]) err := smtp.SendMail(smtpAddr, auth, emailConfig["from_email"], tos, []byte(message)) if err != nil { logger.Errorln("Send Mail to", strings.Join(tos, ","), "error:", err) return err } logger.Infoln("Send Mail to", strings.Join(tos, ","), "Successfully") return nil }
// ForgetPasswd 忘记密码 func (AccountController) ForgetPasswd(ctx echo.Context) error { if _, ok := ctx.Get("user").(*model.Me); ok { return ctx.Redirect(http.StatusSeeOther, "/") } contentTpl := "user/forget_pwd.html" data := map[string]interface{}{"activeUsers": "active"} email := ctx.FormValue("email") if email == "" || ctx.Request().Method() != "POST" { return render(ctx, contentTpl, data) } // 校验email是否存在 if logic.DefaultUser.UserExists(ctx, "email", email) { var uuid string for { uuid = guuid.NewV4().String() if _, ok := resetPwdMap[uuid]; !ok { resetPwdMap[uuid] = email break } logger.Infoln("forget passwd GenUUID 冲突....") } var emailUrl string if strings.HasSuffix(email, "@gmail.com") { emailUrl = "http://mail.google.com" } else { pos := strings.LastIndex(email, "@") emailUrl = "http://mail." + email[pos+1:] } data["success"] = template.HTML(`一封包含了重设密码链接的邮件已经发送到您的注册邮箱,按照邮件中的提示,即可重设您的密码。<a href="` + emailUrl + `" target="_blank">立即前往邮箱</a>`) go logic.DefaultEmail.SendResetpwdMail(email, uuid) } else { data["error"] = "该邮箱没有在本社区注册过!" } return render(ctx, contentTpl, data) }
// 将所有 节点信息 加载到内存中:后台修改节点时,重新加载一次 func LoadNodes() error { nodeList := make([]*model.TopicNode, 0) err := MasterDB.Find(&nodeList) if err != nil { logger.Errorln("LoadNodes node read fail:", err) return err } nodeNum := len(nodeList) tmpNodeList := make(map[int]*model.TopicNode, nodeNum) for _, node := range nodeList { tmpNodeList[node.Nid] = node } nodeRWMutex.Lock() defer nodeRWMutex.Unlock() AllNode = make([]map[string]interface{}, nodeNum) for i, node := range nodeList { nodeMap := make(map[string]interface{}, 5) nodeMap["pid"] = node.Parent if node.Parent == 0 { nodeMap["parent"] = "根节点" } else { nodeMap["parent"] = tmpNodeList[node.Parent].Name } nodeMap["nid"] = node.Nid nodeMap["name"] = node.Name nodeMap["intro"] = node.Intro nodeMap["ctime"] = node.Ctime AllNode[i] = nodeMap } logger.Infoln("LoadNodes successfully!") return nil }
// ParseArticle 获取 url 对应的文章并根据规则进行解析 func (ArticleLogic) ParseArticle(ctx context.Context, articleUrl string, auto bool) (*model.Article, error) { articleUrl = strings.TrimSpace(articleUrl) if !strings.HasPrefix(articleUrl, "http") { articleUrl = "http://" + articleUrl } tmpArticle := &model.Article{} _, err := MasterDB.Where("url=?", articleUrl).Get(tmpArticle) if err != nil || tmpArticle.Id != 0 { logger.Infoln(articleUrl, "has exists:", err) return nil, errors.New("has exists!") } urlPaths := strings.SplitN(articleUrl, "/", 5) domain := urlPaths[2] for k, v := range domainPatch { if strings.Contains(domain, k) && !strings.Contains(domain, "www."+k) { domain = v break } } rule := &model.CrawlRule{} _, err = MasterDB.Where("domain=?", domain).Get(rule) if err != nil { logger.Errorln("find rule by domain error:", err) return nil, err } if rule.Id == 0 { logger.Errorln("domain:", domain, "not exists!") return nil, errors.New("domain not exists") } var doc *goquery.Document if doc, err = goquery.NewDocument(articleUrl); err != nil { logger.Errorln("goquery newdocument error:", err) return nil, err } author, authorTxt := "", "" if rule.InUrl { index, err := strconv.Atoi(rule.Author) if err != nil { logger.Errorln("author rule is illegal:", rule.Author, "error:", err) return nil, err } author = urlPaths[index] authorTxt = author } else { if strings.HasPrefix(rule.Author, ".") || strings.HasPrefix(rule.Author, "#") { authorSelection := doc.Find(rule.Author) author, err = authorSelection.Html() if err != nil { logger.Errorln("goquery parse author error:", err) return nil, err } author = strings.TrimSpace(author) authorTxt = strings.TrimSpace(authorSelection.Text()) } else { // 某些个人博客,页面中没有作者的信息,因此,规则中 author 即为 作者 author = rule.Author authorTxt = rule.Author } } title := "" doc.Find(rule.Title).Each(func(i int, selection *goquery.Selection) { if title != "" { return } tmpTitle := strings.TrimSpace(selection.Text()) tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "原")) tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "荐")) tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "转")) tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "顶")) if tmpTitle != "" { title = tmpTitle } }) if title == "" { logger.Errorln("url:", articleUrl, "parse title error:", err) return nil, err } replacer := strings.NewReplacer("[置顶]", "", "[原]", "", "[转]", "") title = strings.TrimSpace(replacer.Replace(title)) contentSelection := doc.Find(rule.Content) // relative url -> abs url contentSelection.Find("img").Each(func(i int, s *goquery.Selection) { if v, ok := s.Attr("src"); ok { if !strings.HasPrefix(v, "http") { s.SetAttr("src", domain+v) } } }) content, err := contentSelection.Html() if err != nil { logger.Errorln("goquery parse content error:", err) return nil, err } content = strings.TrimSpace(content) txt := strings.TrimSpace(contentSelection.Text()) txt = articleRe.ReplaceAllLiteralString(txt, " ") txt = articleSpaceRe.ReplaceAllLiteralString(txt, " ") // 自动抓取,内容长度不能少于 300 字 if auto && len(txt) < 300 { logger.Errorln(articleUrl, "content is short") return nil, errors.New("content is short") } if auto && strings.Count(txt, "http://") > 10 { logger.Errorln(articleUrl, "content contains too many link!") return nil, errors.New("content contains too many link") } pubDate := times.Format("Y-m-d H:i:s") if rule.PubDate != "" { pubDate = strings.TrimSpace(doc.Find(rule.PubDate).First().Text()) // oschina patch re := regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}") submatches := re.FindStringSubmatch(pubDate) if len(submatches) > 0 { pubDate = submatches[0] } else { // oschina 多少之前忽略 pubDate = "" } } if pubDate == "" { pubDate = times.Format("Y-m-d H:i:s") } else { // YYYYY-MM-dd HH:mm if len(pubDate) == 16 && auto { // 三个月之前不入库 pubTime, err := time.ParseInLocation("2006-01-02 15:04", pubDate, time.Local) if err == nil { if pubTime.Add(3 * 30 * 86400 * time.Second).Before(time.Now()) { return nil, errors.New("article is old!") } } } } article := &model.Article{ Domain: domain, Name: rule.Name, Author: author, AuthorTxt: authorTxt, Title: title, Content: content, Txt: txt, PubDate: pubDate, Url: articleUrl, Lang: rule.Lang, } _, err = MasterDB.Insert(article) if err != nil { logger.Errorln("insert article error:", err) return nil, err } return article, nil }
// ParseOneProject 处理单个 project func (ProjectLogic) ParseOneProject(projectUrl string) error { if !strings.HasPrefix(projectUrl, "http") { projectUrl = OsChinaDomain + projectUrl } var ( doc *goquery.Document err error ) // 加上 ?fromerr=xfwefs,否则页面有 js 重定向 if doc, err = goquery.NewDocument(projectUrl + "?fromerr=xfwefs"); err != nil { return errors.New("goquery fetch " + projectUrl + " error:" + err.Error()) } // 标题 category := strings.TrimSpace(doc.Find(".Project .name").Text()) name := strings.TrimSpace(doc.Find(".Project .name u").Text()) if category == "" && name == "" { return errors.New("projectUrl:" + projectUrl + " category and name are empty") } tmpIndex := strings.LastIndex(category, name) if tmpIndex != -1 { category = category[:tmpIndex] } // uri uri := projectUrl[strings.LastIndex(projectUrl, "/")+1:] project := &model.OpenProject{} _, err = MasterDB.Where("uri=?", uri).Get(project) // 已经存在 if project.Id != 0 { logger.Infoln("url", projectUrl, "has exists!") return nil } logoSelection := doc.Find(".Project .PN img") if logoSelection.AttrOr("title", "") != "" { project.Logo = logoSelection.AttrOr("src", "") if !strings.HasPrefix(project.Logo, "http") { project.Logo = OsChinaDomain + project.Logo } project.Logo, err = DefaultUploader.TransferUrl(nil, project.Logo, ProjectLogoPrefix) if err != nil { logger.Errorln("project logo upload error:", err) } } // 获取项目相关链接 doc.Find("#Body .urls li").Each(func(i int, liSelection *goquery.Selection) { aSelection := liSelection.Find("a") uri := util.FetchRealUrl(OsChinaDomain + aSelection.AttrOr("href", "")) switch aSelection.Text() { case "软件首页": project.Home = uri case "软件文档": project.Doc = uri case "软件下载": project.Download = uri } }) ctime := time.Now() doc.Find("#Body .attrs li").Each(func(i int, liSelection *goquery.Selection) { aSelection := liSelection.Find("a") txt := aSelection.Text() if i == 0 { project.Licence = txt if txt == "未知" { project.Licence = "其他" } } else if i == 1 { project.Lang = txt } else if i == 2 { project.Os = txt } else if i == 3 { dtime, err := time.ParseInLocation("2006年01月02日", aSelection.Last().Text(), time.Local) if err != nil { logger.Errorln("parse ctime error:", err) } else { ctime = dtime.Local() } } }) project.Name = name project.Category = category project.Uri = uri project.Repo = strings.TrimSpace(doc.Find("#Body .github-widget").AttrOr("data-repo", "")) project.Src = "https://github.com/" + project.Repo pos := strings.Index(project.Repo, "/") if pos > -1 { project.Author = project.Repo[:pos] } else { project.Author = "网友" } if project.Doc == "" { // TODO:暂时认为一定是 Go 语言 project.Doc = "https://godoc.org/" + project.Src[8:] } desc := "" doc.Find("#Body .detail").Find("p").NextAll().Each(func(i int, domSelection *goquery.Selection) { doc.FindSelection(domSelection).WrapHtml(`<div id="tmp` + strconv.Itoa(i) + `"></div>`) domHtml, _ := doc.Find("#tmp" + strconv.Itoa(i)).Html() if domSelection.Is("pre") { desc += domHtml + "\n\n" } else { desc += html2md.Convert(domHtml) + "\n\n" } }) project.Desc = strings.TrimSpace(desc) project.Username = PresetUsernames[rand.Intn(4)] project.Status = model.ProjectStatusOnline project.Ctime = model.OftenTime(ctime) _, err = MasterDB.Insert(project) if err != nil { return errors.New("insert into open project error:" + err.Error()) } return nil }
// 订阅邮件通知 func (self EmailLogic) EmailNotice() { beginDate := time.Now().Add(-7 * 24 * time.Hour).Format("2006-01-02") endDate := time.Now().Add(-24 * time.Hour).Format("2006-01-02") beginTime := beginDate + " 00:00:00" // 本周晨读(过去 7 天) readings, err := DefaultReading.FindLastList(beginTime) if err != nil { logger.Errorln("find morning reading error:", err) } // 本周精彩文章 articles, err := DefaultArticle.FindLastList(beginTime, 10) if err != nil { logger.Errorln("find article error:", err) } // 本周热门主题 topics, err := DefaultTopic.FindLastList(beginTime, 10) if err != nil { logger.Errorln("find topic error:", err) } data := map[string]interface{}{ "readings": readings, "articles": articles, "topics": topics, "beginDate": beginDate, "endDate": endDate, } // 给所有用户发送邮件 var ( lastUid = 0 limit = 500 users = make([]*model.User, 0) ) for { err = MasterDB.Where("uid>?", lastUid).Asc("uid").Limit(limit).Find(&users) if err != nil { logger.Errorln("find user error:", err) continue } if len(users) == 0 { break } for _, user := range users { if lastUid < user.Uid { lastUid = user.Uid } if user.Unsubscribe == 1 { logger.Infoln("user unsubscribe", user) continue } if user.Status != model.UserStatusAudit { logger.Infoln("user is not normal:", user) continue } data["email"] = user.Email data["token"] = self.GenUnsubscribeToken(user) content, err := self.genEmailContent(data) if err != nil { logger.Errorln("from email.html gen email content error:", err) continue } self.SendMail("每周精选", content, []string{user.Email}) // 控制发信速度 time.Sleep(60 * time.Second) } users = make([]*model.User, 0) } }
func (this *SolrClient) Post() error { stringBuilder := goutils.NewBuffer().Append("{") needComma := false for _, addCommand := range this.addCommands { commandJson, err := json.Marshal(addCommand) if err != nil { continue } if stringBuilder.Len() == 1 { needComma = false } else { needComma = true } if needComma { stringBuilder.Append(",") } stringBuilder.Append(`"add":`).Append(commandJson) } for _, delCommand := range this.delCommands { commandJson, err := json.Marshal(delCommand) if err != nil { continue } if stringBuilder.Len() == 1 { needComma = false } else { needComma = true } if needComma { stringBuilder.Append(",") } stringBuilder.Append(`"delete":`).Append(commandJson) } if stringBuilder.Len() == 1 { logger.Errorln("post docs:no right addcommand") return errors.New("no right addcommand") } stringBuilder.Append("}") logger.Infoln("start post data to solr...") resp, err := http.Post(config.ConfigFile.MustValue("search", "engine_url")+"/update?wt=json&commit=true", "application/json", stringBuilder) if err != nil { logger.Errorln("post error:", err) return err } defer resp.Body.Close() var result map[string]interface{} err = json.NewDecoder(resp.Body).Decode(&result) if err != nil { logger.Errorln("parse response error:", err) return err } logger.Infoln("post data result:", result) return nil }