예제 #1
0
// Sensivite 用于 echo 框架的过滤发布敏感词(广告)
func Sensivite() echo.MiddlewareFunc {
	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(ctx echo.Context) error {
			content := ctx.FormValue("content")
			title := ctx.FormValue("title")

			user := ctx.Get("user").(*model.Me)

			if title != "" {
				for _, s := range titleSensitives {
					if hasSensitiveChar(title, s) {
						// 把账号冻结
						logic.DefaultUser.UpdateUserStatus(ctx, user.Uid, model.UserStatusFreeze)
						logger.Infoln("user="******"publish ad, title=", title, ". freeze")
						return errors.New("对不起,您的账号已被冻结!")
					}
				}
			}

			if hasSensitive(title, contentSensitives) || hasSensitive(content, contentSensitives) {
				// 把账号冻结
				logic.DefaultUser.UpdateUserStatus(ctx, user.Uid, model.UserStatusFreeze)
				logger.Infoln("user="******"publish ad, title=", title, ";content=", content, ". freeze")
				return errors.New("对不起,您的账号已被冻结!")
			}

			if err := next(ctx); err != nil {
				return err
			}

			return nil
		}
	}
}
예제 #2
0
func indexing(isAll bool) {
	logger.Infoln("indexing start...")

	start := time.Now()
	defer func() {
		logger.Infoln("indexing spend time:", time.Now().Sub(start))
	}()

	logic.DefaultSearcher.Indexing(isAll)
}
예제 #3
0
// 将所有 角色拥有的权限 加载到内存中;后台修改时,重新加载一次
func LoadRoleAuthorities() error {
	roleAuthorities := make([]*model.RoleAuthority, 0)
	err := MasterDB.Find(&roleAuthorities)
	if err != nil {
		logger.Errorln("LoadRoleAuthorities role_authority read fail:", err)
		return err
	}

	roleAuthLocker.Lock()
	defer roleAuthLocker.Unlock()

	RoleAuthorities = make(map[int][]int)

	for _, roleAuth := range roleAuthorities {
		roleId := roleAuth.Roleid

		if authorities, ok := RoleAuthorities[roleId]; ok {
			RoleAuthorities[roleId] = append(authorities, roleAuth.Aid)
		} else {
			RoleAuthorities[roleId] = []int{roleAuth.Aid}
		}
	}

	logger.Infoln("LoadRoleAuthorities successfully!")

	return nil
}
예제 #4
0
// 将所有 角色 加载到内存中;后台修改角色时,重新加载一次
func LoadRoles() error {
	roles := make([]*model.Role, 0)
	err := MasterDB.Find(&roles)
	if err != nil {
		logger.Errorln("LoadRoles role read fail:", err)
		return err
	}

	if len(roles) == 0 {
		logger.Errorln("LoadRoles role read fail: num is 0")
		return errors.New("no role")
	}

	roleLocker.Lock()
	defer roleLocker.Unlock()

	maxRoleid := roles[len(roles)-1].Roleid
	Roles = make([]*model.Role, maxRoleid)

	// 由于角色不多,而且一般角色id是连续自增的,因此这里以角色id当slice的index
	for _, role := range roles {
		Roles[role.Roleid-1] = role
	}

	logger.Infoln("LoadRoles successfully!")

	return nil
}
예제 #5
0
// 给某个用户发送一条消息
func (this *book) PostMessage(uid int, message *Message) {
	this.rwMutex.RLock()
	defer this.rwMutex.RUnlock()
	if userData, ok := this.users[uid]; ok {
		logger.Infoln("post message to", uid, message)
		go userData.SendMessage(message)
	}
}
예제 #6
0
// 给所有用户广播消息
func (this *book) BroadcastAllUsersMessage(message *Message) {
	logger.Infoln("BroadcastAllUsersMessage message", message)

	this.rwMutex.RLock()
	defer this.rwMutex.RUnlock()
	for _, userData := range this.users {
		userData.SendMessage(message)
	}
}
예제 #7
0
func autocrawl(needAll bool, crawlConfFile string, whichSite string) {
	content, err := ioutil.ReadFile(crawlConfFile)
	if err != nil {
		log.Fatalln("parse crawl config read file error:", err)
	}

	err = json.Unmarshal(content, &websites)
	if err != nil {
		log.Fatalln("parse crawl config json parse error:", err)
	}

	if needAll {
		// 全量
		for website, wbconf := range websites {
			if whichSite != "" && whichSite != website {
				continue
			}

			logger.Infoln("all crawl", website)
			go doCrawl(wbconf, true)
		}
	}

	// 定时增量
	c := cron.New()
	c.AddFunc(config.ConfigFile.MustValue("crawl", "spec", "0 0 */1 * * ?"), func() {
		// 抓取 reddit
		go logic.DefaultReddit.Parse("")

		// 抓取 www.oschina.net/project
		go logic.DefaultProject.ParseProjectList("http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time")

		for website, wbconf := range websites {
			if whichSite != "" && whichSite != website {
				continue
			}

			logger.Infoln("do crawl", website)
			go doCrawl(wbconf, false)
		}
	})
	c.Start()
}
예제 #8
0
func parseArticleList(url, listselector, resultselector string, isAuto bool) (err error) {

	logger.Infoln("parse url:", url)

	var doc *goquery.Document

	if strings.Contains(url, "oschina.net") {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return err
		}
		req.Header.Add("Referer", "http://www.oschina.net/search?q=go&scope=blog&onlytitle=1&sort_by_time=1")
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}
		doc, err = goquery.NewDocumentFromResponse(resp)
	} else {
		doc, err = goquery.NewDocument(url)
	}

	if err != nil {
		return
	}

	doc.Find(listselector).Each(func(i int, contentSelection *goquery.Selection) {

		aSelection := contentSelection.Find(resultselector)

		if isAuto {
			title := aSelection.Text()

			matched, err := regexp.MatchString(pattern, title)
			if err != nil {
				logger.Errorln(err)
				return
			}

			if !matched {
				return
			}
		}

		articleUrl, ok := aSelection.Attr("href")
		if ok {
			pos := strings.LastIndex(articleUrl, "?")
			if pos != -1 {
				articleUrl = articleUrl[:pos]
			}
			logic.DefaultArticle.ParseArticle(context.Background(), articleUrl, isAuto)
		}
	})

	return
}
예제 #9
0
// 给除了自己的其他用户广播消息
func (this *book) BroadcastToOthersMessage(message *Message, myself int) {
	logger.Infoln("BroadcastToOthersMessage message", message)

	this.rwMutex.RLock()
	defer this.rwMutex.RUnlock()
	for uid, userData := range this.users {
		if uid == myself {
			continue
		}
		userData.SendMessage(message)
	}
}
예제 #10
0
func (this *UserData) SendMessage(message *Message) {
	this.rwMutex.RLock()
	defer this.rwMutex.RUnlock()

	for serverId, messageQueue := range this.serverMsgQueue {
		// 有可能用户已经退出,导致 messageQueue满,阻塞
		if len(messageQueue) < MessageQueueLen {
			messageQueue <- message
		} else {
			logger.Infoln("server_id:", serverId, "had close")
		}
	}
}
예제 #11
0
// 将所有 资源分类信息 加载到内存中:后台修改节点时,重新加载一次
func LoadCategories() (err error) {
	categories := make([]*model.ResourceCat, 0)
	err = MasterDB.Find(&categories)
	if err != nil {
		logger.Errorln("LoadCategories category read fail:", err)
		return
	}

	catRWMutex.Lock()
	defer catRWMutex.Unlock()

	AllCategory = categories

	logger.Infoln("LoadCategories successfully!")

	return
}
예제 #12
0
// 将所有 权限 加载到内存中;后台修改权限时,重新加载一次
func LoadAuthorities() error {
	authorities := make([]*model.Authority, 0)
	err := MasterDB.Find(&authorities)
	if err != nil {
		logger.Errorln("LoadAuthorities authority read fail:", err)
		return err
	}

	authLocker.Lock()
	defer authLocker.Unlock()

	Authorities = authorities

	logger.Infoln("LoadAuthorities successfully!")

	return nil
}
예제 #13
0
// websocket,统计在线用户数
// uri: /ws
func (this *WebsocketController) Ws(wsConn *websocket.Conn) {
	defer wsConn.Close()

	this.mutex.Lock()
	this.ServerId++
	serverId := this.ServerId
	this.mutex.Unlock()
	req := wsConn.Request()
	user := goutils.MustInt(req.FormValue("uid"))
	if user == 0 {
		user = int(goutils.Ip2long(goutils.RemoteIp(req)))
	}
	userData := logic.Book.AddUser(user, serverId)
	// 给自己发送消息,告诉当前在线用户数、历史最高在线人数
	onlineInfo := map[string]int{"online": logic.Book.Len(), "maxonline": logic.MaxOnlineNum()}
	message := logic.NewMessage(logic.WsMsgOnline, onlineInfo)
	err := websocket.JSON.Send(wsConn, message)
	if err != nil {
		logger.Errorln("Sending onlineusers error:", err)
	}
	var clientClosed = false
	for {
		select {
		case message := <-userData.MessageQueue(serverId):
			if err := websocket.JSON.Send(wsConn, message); err != nil {
				clientClosed = true
			}
			// 心跳
		case <-time.After(30e9):
			if err := websocket.JSON.Send(wsConn, ""); err != nil {
				clientClosed = true
			}
		}
		if clientClosed {
			logic.Book.DelUser(user, serverId)
			logger.Infoln("user:"******"client close")
			break
		}
	}
	// 用户退出时需要变更其他用户看到的在线用户数
	if !logic.Book.UserIsOnline(user) {
		message := logic.NewMessage(logic.WsMsgOnline, map[string]int{"online": logic.Book.Len()})
		go logic.Book.BroadcastAllUsersMessage(message)
	}
}
예제 #14
0
// SendMail 发送电子邮件
func (EmailLogic) SendMail(subject, content string, tos []string) error {
	emailConfig, _ := config.ConfigFile.GetSection("email")
	message := `From: Go语言中文网 | Golang中文社区 | Go语言学习园地<` + emailConfig["from_email"] + `>
To: ` + strings.Join(tos, ",") + `
Subject: ` + subject + `
Content-Type: text/html;charset=UTF-8

` + content

	smtpAddr := emailConfig["smtp_host"] + ":" + emailConfig["smtp_port"]
	auth := smtp.PlainAuth("", emailConfig["smtp_username"], emailConfig["smtp_password"], emailConfig["smtp_host"])
	err := smtp.SendMail(smtpAddr, auth, emailConfig["from_email"], tos, []byte(message))
	if err != nil {
		logger.Errorln("Send Mail to", strings.Join(tos, ","), "error:", err)
		return err
	}
	logger.Infoln("Send Mail to", strings.Join(tos, ","), "Successfully")
	return nil
}
예제 #15
0
// ForgetPasswd 忘记密码
func (AccountController) ForgetPasswd(ctx echo.Context) error {
	if _, ok := ctx.Get("user").(*model.Me); ok {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	contentTpl := "user/forget_pwd.html"
	data := map[string]interface{}{"activeUsers": "active"}

	email := ctx.FormValue("email")
	if email == "" || ctx.Request().Method() != "POST" {
		return render(ctx, contentTpl, data)
	}

	// 校验email是否存在
	if logic.DefaultUser.UserExists(ctx, "email", email) {
		var uuid string
		for {
			uuid = guuid.NewV4().String()
			if _, ok := resetPwdMap[uuid]; !ok {
				resetPwdMap[uuid] = email
				break
			}
			logger.Infoln("forget passwd GenUUID 冲突....")
		}
		var emailUrl string
		if strings.HasSuffix(email, "@gmail.com") {
			emailUrl = "http://mail.google.com"
		} else {
			pos := strings.LastIndex(email, "@")
			emailUrl = "http://mail." + email[pos+1:]
		}
		data["success"] = template.HTML(`一封包含了重设密码链接的邮件已经发送到您的注册邮箱,按照邮件中的提示,即可重设您的密码。<a href="` + emailUrl + `" target="_blank">立即前往邮箱</a>`)
		go logic.DefaultEmail.SendResetpwdMail(email, uuid)
	} else {
		data["error"] = "该邮箱没有在本社区注册过!"
	}

	return render(ctx, contentTpl, data)
}
예제 #16
0
// 将所有 节点信息 加载到内存中:后台修改节点时,重新加载一次
func LoadNodes() error {
	nodeList := make([]*model.TopicNode, 0)
	err := MasterDB.Find(&nodeList)
	if err != nil {
		logger.Errorln("LoadNodes node read fail:", err)
		return err
	}
	nodeNum := len(nodeList)
	tmpNodeList := make(map[int]*model.TopicNode, nodeNum)
	for _, node := range nodeList {
		tmpNodeList[node.Nid] = node
	}

	nodeRWMutex.Lock()
	defer nodeRWMutex.Unlock()

	AllNode = make([]map[string]interface{}, nodeNum)
	for i, node := range nodeList {
		nodeMap := make(map[string]interface{}, 5)
		nodeMap["pid"] = node.Parent
		if node.Parent == 0 {
			nodeMap["parent"] = "根节点"
		} else {
			nodeMap["parent"] = tmpNodeList[node.Parent].Name
		}
		nodeMap["nid"] = node.Nid
		nodeMap["name"] = node.Name
		nodeMap["intro"] = node.Intro
		nodeMap["ctime"] = node.Ctime
		AllNode[i] = nodeMap
	}

	logger.Infoln("LoadNodes successfully!")

	return nil
}
예제 #17
0
// ParseArticle 获取 url 对应的文章并根据规则进行解析
func (ArticleLogic) ParseArticle(ctx context.Context, articleUrl string, auto bool) (*model.Article, error) {
	articleUrl = strings.TrimSpace(articleUrl)
	if !strings.HasPrefix(articleUrl, "http") {
		articleUrl = "http://" + articleUrl
	}

	tmpArticle := &model.Article{}
	_, err := MasterDB.Where("url=?", articleUrl).Get(tmpArticle)
	if err != nil || tmpArticle.Id != 0 {
		logger.Infoln(articleUrl, "has exists:", err)
		return nil, errors.New("has exists!")
	}

	urlPaths := strings.SplitN(articleUrl, "/", 5)
	domain := urlPaths[2]

	for k, v := range domainPatch {
		if strings.Contains(domain, k) && !strings.Contains(domain, "www."+k) {
			domain = v
			break
		}
	}

	rule := &model.CrawlRule{}
	_, err = MasterDB.Where("domain=?", domain).Get(rule)
	if err != nil {
		logger.Errorln("find rule by domain error:", err)
		return nil, err
	}

	if rule.Id == 0 {
		logger.Errorln("domain:", domain, "not exists!")
		return nil, errors.New("domain not exists")
	}

	var doc *goquery.Document
	if doc, err = goquery.NewDocument(articleUrl); err != nil {
		logger.Errorln("goquery newdocument error:", err)
		return nil, err
	}

	author, authorTxt := "", ""
	if rule.InUrl {
		index, err := strconv.Atoi(rule.Author)
		if err != nil {
			logger.Errorln("author rule is illegal:", rule.Author, "error:", err)
			return nil, err
		}
		author = urlPaths[index]
		authorTxt = author
	} else {
		if strings.HasPrefix(rule.Author, ".") || strings.HasPrefix(rule.Author, "#") {
			authorSelection := doc.Find(rule.Author)
			author, err = authorSelection.Html()
			if err != nil {
				logger.Errorln("goquery parse author error:", err)
				return nil, err
			}

			author = strings.TrimSpace(author)
			authorTxt = strings.TrimSpace(authorSelection.Text())
		} else {
			// 某些个人博客,页面中没有作者的信息,因此,规则中 author 即为 作者
			author = rule.Author
			authorTxt = rule.Author
		}
	}

	title := ""
	doc.Find(rule.Title).Each(func(i int, selection *goquery.Selection) {
		if title != "" {
			return
		}

		tmpTitle := strings.TrimSpace(selection.Text())
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "原"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "荐"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "转"))
		tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "顶"))
		if tmpTitle != "" {
			title = tmpTitle
		}
	})

	if title == "" {
		logger.Errorln("url:", articleUrl, "parse title error:", err)
		return nil, err
	}

	replacer := strings.NewReplacer("[置顶]", "", "[原]", "", "[转]", "")
	title = strings.TrimSpace(replacer.Replace(title))

	contentSelection := doc.Find(rule.Content)

	// relative url -> abs url
	contentSelection.Find("img").Each(func(i int, s *goquery.Selection) {
		if v, ok := s.Attr("src"); ok {
			if !strings.HasPrefix(v, "http") {
				s.SetAttr("src", domain+v)
			}
		}
	})

	content, err := contentSelection.Html()
	if err != nil {
		logger.Errorln("goquery parse content error:", err)
		return nil, err
	}
	content = strings.TrimSpace(content)
	txt := strings.TrimSpace(contentSelection.Text())
	txt = articleRe.ReplaceAllLiteralString(txt, " ")
	txt = articleSpaceRe.ReplaceAllLiteralString(txt, " ")

	// 自动抓取,内容长度不能少于 300 字
	if auto && len(txt) < 300 {
		logger.Errorln(articleUrl, "content is short")
		return nil, errors.New("content is short")
	}

	if auto && strings.Count(txt, "http://") > 10 {
		logger.Errorln(articleUrl, "content contains too many link!")
		return nil, errors.New("content contains too many link")
	}

	pubDate := times.Format("Y-m-d H:i:s")
	if rule.PubDate != "" {
		pubDate = strings.TrimSpace(doc.Find(rule.PubDate).First().Text())

		// oschina patch
		re := regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}")
		submatches := re.FindStringSubmatch(pubDate)
		if len(submatches) > 0 {
			pubDate = submatches[0]
		} else {
			// oschina 多少之前忽略
			pubDate = ""
		}
	}

	if pubDate == "" {
		pubDate = times.Format("Y-m-d H:i:s")
	} else {
		// YYYYY-MM-dd HH:mm
		if len(pubDate) == 16 && auto {
			// 三个月之前不入库
			pubTime, err := time.ParseInLocation("2006-01-02 15:04", pubDate, time.Local)
			if err == nil {
				if pubTime.Add(3 * 30 * 86400 * time.Second).Before(time.Now()) {
					return nil, errors.New("article is old!")
				}
			}
		}
	}

	article := &model.Article{
		Domain:    domain,
		Name:      rule.Name,
		Author:    author,
		AuthorTxt: authorTxt,
		Title:     title,
		Content:   content,
		Txt:       txt,
		PubDate:   pubDate,
		Url:       articleUrl,
		Lang:      rule.Lang,
	}

	_, err = MasterDB.Insert(article)
	if err != nil {
		logger.Errorln("insert article error:", err)
		return nil, err
	}

	return article, nil
}
예제 #18
0
// ParseOneProject 处理单个 project
func (ProjectLogic) ParseOneProject(projectUrl string) error {
	if !strings.HasPrefix(projectUrl, "http") {
		projectUrl = OsChinaDomain + projectUrl
	}

	var (
		doc *goquery.Document
		err error
	)

	// 加上 ?fromerr=xfwefs,否则页面有 js 重定向
	if doc, err = goquery.NewDocument(projectUrl + "?fromerr=xfwefs"); err != nil {
		return errors.New("goquery fetch " + projectUrl + " error:" + err.Error())
	}

	// 标题
	category := strings.TrimSpace(doc.Find(".Project .name").Text())
	name := strings.TrimSpace(doc.Find(".Project .name u").Text())
	if category == "" && name == "" {
		return errors.New("projectUrl:" + projectUrl + " category and name are empty")
	}

	tmpIndex := strings.LastIndex(category, name)
	if tmpIndex != -1 {
		category = category[:tmpIndex]
	}

	// uri
	uri := projectUrl[strings.LastIndex(projectUrl, "/")+1:]

	project := &model.OpenProject{}

	_, err = MasterDB.Where("uri=?", uri).Get(project)
	// 已经存在
	if project.Id != 0 {
		logger.Infoln("url", projectUrl, "has exists!")
		return nil
	}

	logoSelection := doc.Find(".Project .PN img")
	if logoSelection.AttrOr("title", "") != "" {
		project.Logo = logoSelection.AttrOr("src", "")

		if !strings.HasPrefix(project.Logo, "http") {
			project.Logo = OsChinaDomain + project.Logo
		}

		project.Logo, err = DefaultUploader.TransferUrl(nil, project.Logo, ProjectLogoPrefix)
		if err != nil {
			logger.Errorln("project logo upload error:", err)
		}
	}

	// 获取项目相关链接
	doc.Find("#Body .urls li").Each(func(i int, liSelection *goquery.Selection) {
		aSelection := liSelection.Find("a")
		uri := util.FetchRealUrl(OsChinaDomain + aSelection.AttrOr("href", ""))
		switch aSelection.Text() {
		case "软件首页":
			project.Home = uri
		case "软件文档":
			project.Doc = uri
		case "软件下载":
			project.Download = uri
		}
	})

	ctime := time.Now()
	doc.Find("#Body .attrs li").Each(func(i int, liSelection *goquery.Selection) {
		aSelection := liSelection.Find("a")
		txt := aSelection.Text()
		if i == 0 {
			project.Licence = txt
			if txt == "未知" {
				project.Licence = "其他"
			}
		} else if i == 1 {
			project.Lang = txt
		} else if i == 2 {
			project.Os = txt
		} else if i == 3 {
			dtime, err := time.ParseInLocation("2006年01月02日", aSelection.Last().Text(), time.Local)
			if err != nil {
				logger.Errorln("parse ctime error:", err)
			} else {
				ctime = dtime.Local()
			}
		}
	})

	project.Name = name
	project.Category = category
	project.Uri = uri
	project.Repo = strings.TrimSpace(doc.Find("#Body .github-widget").AttrOr("data-repo", ""))
	project.Src = "https://github.com/" + project.Repo

	pos := strings.Index(project.Repo, "/")
	if pos > -1 {
		project.Author = project.Repo[:pos]
	} else {
		project.Author = "网友"
	}

	if project.Doc == "" {
		// TODO:暂时认为一定是 Go 语言
		project.Doc = "https://godoc.org/" + project.Src[8:]
	}

	desc := ""
	doc.Find("#Body .detail").Find("p").NextAll().Each(func(i int, domSelection *goquery.Selection) {
		doc.FindSelection(domSelection).WrapHtml(`<div id="tmp` + strconv.Itoa(i) + `"></div>`)
		domHtml, _ := doc.Find("#tmp" + strconv.Itoa(i)).Html()
		if domSelection.Is("pre") {
			desc += domHtml + "\n\n"
		} else {
			desc += html2md.Convert(domHtml) + "\n\n"
		}
	})

	project.Desc = strings.TrimSpace(desc)
	project.Username = PresetUsernames[rand.Intn(4)]
	project.Status = model.ProjectStatusOnline
	project.Ctime = model.OftenTime(ctime)

	_, err = MasterDB.Insert(project)
	if err != nil {
		return errors.New("insert into open project error:" + err.Error())
	}

	return nil
}
예제 #19
0
// 订阅邮件通知
func (self EmailLogic) EmailNotice() {

	beginDate := time.Now().Add(-7 * 24 * time.Hour).Format("2006-01-02")
	endDate := time.Now().Add(-24 * time.Hour).Format("2006-01-02")

	beginTime := beginDate + " 00:00:00"

	// 本周晨读(过去 7 天)
	readings, err := DefaultReading.FindLastList(beginTime)
	if err != nil {
		logger.Errorln("find morning reading error:", err)
	}

	// 本周精彩文章
	articles, err := DefaultArticle.FindLastList(beginTime, 10)
	if err != nil {
		logger.Errorln("find article error:", err)
	}

	// 本周热门主题
	topics, err := DefaultTopic.FindLastList(beginTime, 10)
	if err != nil {
		logger.Errorln("find topic error:", err)
	}

	data := map[string]interface{}{
		"readings":  readings,
		"articles":  articles,
		"topics":    topics,
		"beginDate": beginDate,
		"endDate":   endDate,
	}

	// 给所有用户发送邮件
	var (
		lastUid = 0
		limit   = 500
		users   = make([]*model.User, 0)
	)

	for {
		err = MasterDB.Where("uid>?", lastUid).Asc("uid").Limit(limit).Find(&users)
		if err != nil {
			logger.Errorln("find user error:", err)
			continue
		}

		if len(users) == 0 {
			break
		}

		for _, user := range users {
			if lastUid < user.Uid {
				lastUid = user.Uid
			}

			if user.Unsubscribe == 1 {
				logger.Infoln("user unsubscribe", user)
				continue
			}

			if user.Status != model.UserStatusAudit {
				logger.Infoln("user is not normal:", user)
				continue
			}

			data["email"] = user.Email
			data["token"] = self.GenUnsubscribeToken(user)

			content, err := self.genEmailContent(data)
			if err != nil {
				logger.Errorln("from email.html gen email content error:", err)
				continue
			}

			self.SendMail("每周精选", content, []string{user.Email})

			// 控制发信速度
			time.Sleep(60 * time.Second)
		}

		users = make([]*model.User, 0)
	}

}
예제 #20
0
func (this *SolrClient) Post() error {
	stringBuilder := goutils.NewBuffer().Append("{")

	needComma := false
	for _, addCommand := range this.addCommands {
		commandJson, err := json.Marshal(addCommand)
		if err != nil {
			continue
		}

		if stringBuilder.Len() == 1 {
			needComma = false
		} else {
			needComma = true
		}

		if needComma {
			stringBuilder.Append(",")
		}

		stringBuilder.Append(`"add":`).Append(commandJson)
	}

	for _, delCommand := range this.delCommands {
		commandJson, err := json.Marshal(delCommand)
		if err != nil {
			continue
		}

		if stringBuilder.Len() == 1 {
			needComma = false
		} else {
			needComma = true
		}

		if needComma {
			stringBuilder.Append(",")
		}

		stringBuilder.Append(`"delete":`).Append(commandJson)
	}

	if stringBuilder.Len() == 1 {
		logger.Errorln("post docs:no right addcommand")
		return errors.New("no right addcommand")
	}

	stringBuilder.Append("}")

	logger.Infoln("start post data to solr...")

	resp, err := http.Post(config.ConfigFile.MustValue("search", "engine_url")+"/update?wt=json&commit=true", "application/json", stringBuilder)
	if err != nil {
		logger.Errorln("post error:", err)
		return err
	}

	defer resp.Body.Close()

	var result map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&result)
	if err != nil {
		logger.Errorln("parse response error:", err)
		return err
	}

	logger.Infoln("post data result:", result)

	return nil
}