func SaveAuthority(form url.Values, opUser string) (errMsg string, err error) {
authority := model.NewAuthority()
err = util.ConvertAssign(authority, form)
if err != nil {
logger.Errorln("authority ConvertAssign error", err)
errMsg = err.Error()
return
}
authority.OpUser = opUser
if authority.Aid != 0 {
err = authority.Persist(authority)
} else {
authority.Ctime = util.TimeNow()
_, err = authority.Insert()
}
if err != nil {
errMsg = "内部服务器错误"
logger.Errorln(errMsg, ":", err)
return
}
global.AuthorityChan <- struct{}{}
return
}
// 发表评论（或回复）。
// objid 注册的评论对象
// uid 评论人
func PostComment(uid, objid int, form url.Values) (*model.Comment, error) {
comment := model.NewComment()
comment.Objid = objid
objtype := util.MustInt(form.Get("objtype"))
comment.Objtype = objtype
comment.Uid = uid
comment.Content = form.Get("content")
// TODO:评论楼层怎么处理，避免冲突？最后的楼层信息保存在内存中？
// 暂时只是从数据库中取出最后的评论楼层
stringBuilder := util.NewBuffer()
stringBuilder.Append("objid=").AppendInt(objid).Append(" AND objtype=").AppendInt(objtype)
tmpCmt, err := model.NewComment().Where(stringBuilder.String()).Order("ctime DESC").Find()
if err != nil {
logger.Errorln("post comment service error:", err)
return nil, err
} else {
comment.Floor = tmpCmt.Floor + 1
}
// 入评论库
cid, err := comment.Insert()
if err != nil {
logger.Errorln("post comment service error:", err)
return nil, err
}
comment.Cid = cid
comment.Ctime = util.TimeNow()
decodeCmtContent(comment)
// 回调，不关心处理结果（有些对象可能不需要回调）
if commenter, ok := commenters[objtype]; ok {
logger.Debugf("评论[objid:%d] [objtype:%d] [uid:%d] 成功，通知被评论者更新", objid, objtype, uid)
go commenter.UpdateComment(cid, objid, uid, time.Now().Format("2006-01-02 15:04:05"))
}
// 发评论，活跃度+5
go IncUserWeight("uid="+strconv.Itoa(uid), 5)
// 给被评论对象所有者发系统消息
ext := map[string]interface{}{
"objid": objid,
"objtype": objtype,
"cid": cid,
"uid": uid,
}
go SendSystemMsgTo(0, objtype, ext)
// @某人 发系统消息
go SendSysMsgAtUids(form.Get("uid"), ext)
go SendSysMsgAtUsernames(form.Get("usernames"), ext)
return comment, nil
}
func SaveRole(form url.Values, opUser string) (errMsg string, err error) {
role := model.NewRole()
role.Name = form.Get("name")
role.OpUser = opUser
roleid := form.Get("roleid")
isNew := roleid == ""
if isNew {
role.Ctime = util.TimeNow()
_, err = role.Insert()
} else {
role.Roleid, err = strconv.Atoi(roleid)
if err != nil {
errMsg = "roleid invalid"
logger.Errorln(errMsg, ":", err)
return
}
err = role.Persist(role)
}
if err != nil {
errMsg = "内部服务器错误"
logger.Errorln(errMsg, ":", err)
return
}
roleAuth := model.NewRoleAuthority()
if !isNew {
// 如果是更新角色，将之前的角色权限都删除
roleAuth.Where("roleid=" + strconv.Itoa(role.Roleid)).Delete()
}
roleAuth.Roleid = role.Roleid
roleAuth.OpUser = opUser
// 增加角色拥有的权限
for _, aid := range form["authorities[]"] {
aid, err := strconv.Atoi(aid)
if err != nil {
continue
}
roleAuth.Aid = aid
roleAuth.Insert()
}
global.RoleChan <- struct{}{}
global.RoleAuthChan <- struct{}{}
return
}
func CreateUser(form url.Values) (errMsg string, err error) {
if EmailExists(form.Get("email")) {
err = errors.New("该邮箱已注册过")
return
}
if UsernameExists(form.Get("username")) {
err = errors.New("用户名已存在")
return
}
// 存用户基本信息，产生自增长UID
user := model.NewUser()
err = util.ConvertAssign(user, form)
if err != nil {
logger.Errorln("user ConvertAssign error", err)
errMsg = err.Error()
return
}
user.Ctime = util.TimeNow()
// 随机给一个默认头像
user.Avatar = DefaultAvatars[rand.Intn(len(DefaultAvatars))]
uid, err := user.Insert()
if err != nil {
errMsg = "内部服务器错误"
logger.Errorln(errMsg, "：", err)
return
}
// 存用户登录信息
userLogin := model.NewUserLogin()
err = util.ConvertAssign(userLogin, form)
if err != nil {
errMsg = err.Error()
logger.Errorln("CreateUser error:", err)
return
}
userLogin.Uid = uid
_, err = userLogin.Insert()
if err != nil {
errMsg = "内部服务器错误"
logger.Errorln(errMsg, "：", err)
return
}
// 存用户角色信息
userRole := model.NewUserRole()
// 默认为初级会员
userRole.Roleid = Roles[len(Roles)-1].Roleid
userRole.Uid = uid
if _, err = userRole.Insert(); err != nil {
logger.Errorln("userRole insert Error:", err)
}
// 存用户活跃信息，初始活跃+2
userActive := model.NewUserActive()
userActive.Uid = uid
userActive.Username = user.Username
userActive.Avatar = user.Avatar
userActive.Email = user.Email
userActive.Weight = 2
if _, err = userActive.Insert(); err != nil {
logger.Errorln("UserActive insert Error:", err)
}
return
}
// 处理 Reddit 中的一条资源
func dealRedditOneResource(contentSelection *goquery.Selection) error {
aSelection := contentSelection.Find(".title a.title")
title := aSelection.Text()
if title == "" {
return errors.New("title is empty")
}
resourceUrl, ok := aSelection.Attr("href")
if !ok || resourceUrl == "" {
return errors.New("resource url is empty")
}
isReddit := false
resource := model.NewResource()
// Reddit 自身的内容
if contentSelection.HasClass("self") {
isReddit = true
resourceUrl = Reddit + resourceUrl
}
err := resource.Where("url=?", resourceUrl).Find("id")
// 已经存在
if resource.Id != 0 {
// 如果是 reddit 本身的，可以更新评论信息
if !isReddit {
return errors.New("url" + resourceUrl + "has exists!")
}
}
if isReddit {
resource.Form = model.ContentForm
var doc *goquery.Document
if doc, err = goquery.NewDocument(resourceUrl); err != nil {
return errors.New("goquery reddit.com/r/golang self newdocument error:" + err.Error())
}
content, err := doc.Find("#siteTable .usertext .md").Html()
if err != nil {
return err
}
doc.Find(".commentarea .comment .usertext .md").Each(func(i int, contentSel *goquery.Selection) {
if i == 0 {
content += `<hr/>**评论：**<br/><br/>`
}
comment, err := contentSel.Html()
if err != nil {
return
}
comment = strings.TrimSpace(comment)
comment = resourceRe.ReplaceAllLiteralString(comment, "\n")
author := contentSel.ParentsFiltered(".usertext").Prev().Find(".author").Text()
content += author + ": <pre>" + comment + "</pre>"
})
if strings.TrimSpace(content) == "" {
return errors.New("goquery reddit.com/r/golang self newdocument(" + resourceUrl + ") error: content is empty")
}
resource.Content = content
// reddit 本身的，当做其他资源
resource.Catid = 4
} else {
resource.Form = model.LinkForm
// Github，是开源项目
if contentSelection.Find(".title .domain a").Text() == "github.com" {
resource.Catid = 2
} else {
resource.Catid = 1
}
}
resource.Title = title
resource.Url = resourceUrl
resource.Uid = PresetUids[rand.Intn(4)]
ctime := util.TimeNow()
datetime, ok := contentSelection.Find(".tagline time").Attr("datetime")
if ok {
dtime, err := time.ParseInLocation(time.RFC3339, datetime, time.UTC)
if err != nil {
logger.Errorln("parse ctime error:", err)
} else {
ctime = dtime.Local().Format("2006-01-02 15:04:05")
}
}
resource.Ctime = ctime
if resource.Id == 0 {
var id int64
id, err = resource.Insert()
if err != nil {
return errors.New("insert into Resource error:" + err.Error())
}
// 存扩展信息
resourceEx := model.NewResourceEx()
resourceEx.Id = int(id)
if _, err = resourceEx.Insert(); err != nil {
return errors.New("insert into ResourceEx error:" + err.Error())
}
} else {
if err = resource.Persist(resource); err != nil {
return errors.New("persist resource:" + strconv.Itoa(resource.Id) + " error:" + err.Error())
}
}
return nil
}
// 获取url对应的文章并根据规则进行解析
func ParseArticle(articleUrl string, auto bool) (*model.Article, error) {
articleUrl = strings.TrimSpace(articleUrl)
if !strings.HasPrefix(articleUrl, "http") {
articleUrl = "http://" + articleUrl
}
tmpArticle := model.NewArticle()
err := tmpArticle.Where("url=" + articleUrl).Find("id")
if err != nil || tmpArticle.Id != 0 {
logger.Errorln(articleUrl, "has exists:", err)
return nil, errors.New("has exists!")
}
urlPaths := strings.SplitN(articleUrl, "/", 5)
domain := urlPaths[2]
for k, v := range domainPatch {
if strings.Contains(domain, k) && !strings.Contains(domain, "www."+k) {
domain = v
break
}
}
rule := model.NewCrawlRule()
err = rule.Where("domain=" + domain).Find()
if err != nil {
logger.Errorln("find rule by domain error:", err)
return nil, err
}
if rule.Id == 0 {
logger.Errorln("domain:", domain, "not exists!")
return nil, errors.New("domain not exists")
}
var doc *goquery.Document
if doc, err = goquery.NewDocument(articleUrl); err != nil {
logger.Errorln("goquery newdocument error:", err)
return nil, err
}
author, authorTxt := "", ""
if rule.InUrl {
index, err := strconv.Atoi(rule.Author)
if err != nil {
logger.Errorln("author rule is illegal:", rule.Author, "error:", err)
return nil, err
}
author = urlPaths[index]
authorTxt = author
} else {
if strings.HasPrefix(rule.Author, ".") || strings.HasPrefix(rule.Author, "#") {
authorSelection := doc.Find(rule.Author)
author, err = authorSelection.Html()
if err != nil {
logger.Errorln("goquery parse author error:", err)
return nil, err
}
author = strings.TrimSpace(author)
authorTxt = strings.TrimSpace(authorSelection.Text())
} else {
// 某些个人博客，页面中没有作者的信息，因此，规则中 author 即为 作者
author = rule.Author
authorTxt = rule.Author
}
}
title := ""
doc.Find(rule.Title).Each(func(i int, selection *goquery.Selection) {
if title != "" {
return
}
tmpTitle := strings.TrimSpace(strings.TrimPrefix(selection.Text(), "原"))
tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "荐"))
tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "转"))
tmpTitle = strings.TrimSpace(strings.TrimPrefix(tmpTitle, "顶"))
if tmpTitle != "" {
title = tmpTitle
}
})
if title == "" {
logger.Errorln("url:", articleUrl, "parse title error:", err)
return nil, err
}
replacer := strings.NewReplacer("[置顶]", "", "[原]", "", "[转]", "")
title = strings.TrimSpace(replacer.Replace(title))
contentSelection := doc.Find(rule.Content)
// relative url -> abs url
contentSelection.Find("img").Each(func(i int, s *goquery.Selection) {
if v, ok := s.Attr("src"); ok {
if !strings.HasPrefix(v, "http") {
s.SetAttr("src", domain+v)
}
}
})
content, err := contentSelection.Html()
if err != nil {
logger.Errorln("goquery parse content error:", err)
return nil, err
}
content = strings.TrimSpace(content)
txt := strings.TrimSpace(contentSelection.Text())
txt = articleRe.ReplaceAllLiteralString(txt, " ")
txt = articleSpaceRe.ReplaceAllLiteralString(txt, " ")
// 自动抓取，内容长度不能少于 300 字
if auto && len(txt) < 300 {
logger.Infoln(articleUrl, "content is short")
return nil, errors.New("content is short")
}
pubDate := util.TimeNow()
if rule.PubDate != "" {
pubDate = strings.TrimSpace(doc.Find(rule.PubDate).First().Text())
// sochina patch
re := regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}")
submatches := re.FindStringSubmatch(pubDate)
if len(submatches) > 0 {
pubDate = submatches[0]
}
}
if pubDate == "" {
pubDate = util.TimeNow()
}
article := model.NewArticle()
article.Domain = domain
article.Name = rule.Name
article.Author = author
article.AuthorTxt = authorTxt
article.Title = title
article.Content = content
article.Txt = txt
article.PubDate = pubDate
article.Url = articleUrl
article.Lang = rule.Lang
article.Ctime = util.TimeNow()
_, err = article.Insert()
if err != nil {
logger.Errorln("insert article error:", err)
return nil, err
}
return article, nil
}
// 增加（修改）资源
func PublishResource(user map[string]interface{}, form url.Values) (err error) {
uid := user["uid"].(int)
resource := model.NewResource()
if form.Get("id") != "" {
err = resource.Where("id=?", form.Get("id")).Find()
if err != nil {
logger.Errorln("Publish Resource find error:", err)
return
}
isAdmin := false
if _, ok := user["isadmin"]; ok {
isAdmin = user["isadmin"].(bool)
}
if resource.Uid != uid && !isAdmin {
err = NotModifyAuthorityErr
return
}
fields := []string{"title", "catid", "form", "url", "content"}
if form.Get("form") == model.LinkForm {
form.Set("content", "")
} else {
form.Set("url", "")
}
id := form.Get("id")
query, args := updateSetClause(form, fields)
err = resource.Set(query, args...).Where("id=?", id).Update()
if err != nil {
logger.Errorf("更新資源 【%s】 信息失败：%s\n", id, err)
return
}
// 修改資源，活跃度+2
go IncUserWeight("uid="+strconv.Itoa(uid), 2)
} else {
util.ConvertAssign(resource, form)
resource.Uid = uid
resource.Ctime = util.TimeNow()
var id int64
id, err = resource.Insert()
if err != nil {
logger.Errorln("Publish Resource error:", err)
return
}
// 存扩展信息
resourceEx := model.NewResourceEx()
resourceEx.Id = int(id)
if _, err = resourceEx.Insert(); err != nil {
logger.Errorln("PublishResource Ex error:", err)
return
}
// 给 被@用户 发系统消息
/*
ext := map[string]interface{}{
"objid": id,
"objtype": model.TYPE_RESOURCE,
"uid": user["uid"],
"msgtype": model.MsgtypePublishAtMe,
}
go SendSysMsgAtUsernames(form.Get("usernames"), ext)
*/
// 发布主题，活跃度+10
go IncUserWeight("uid="+strconv.Itoa(uid), 10)
}
return
}
func PublishProject(user map[string]interface{}, form url.Values) (err error) {
id := form.Get("id")
isModify := id != ""
if !isModify && ProjectUriExists(form.Get("uri")) {
err = errors.New("uri存在")
return
}
username := user["username"].(string)
project := model.NewOpenProject()
if isModify {
err = project.Where("id=?", id).Find()
if err != nil {
logger.Errorln("Publish Project find error:", err)
return
}
isAdmin := false
if _, ok := user["isadmin"]; ok {
isAdmin = user["isadmin"].(bool)
}
if project.Username != username && !isAdmin {
err = NotModifyAuthorityErr
return
}
util.ConvertAssign(project, form)
} else {
util.ConvertAssign(project, form)
project.Username = username
project.Ctime = util.TimeNow()
}
project.Uri = strings.ToLower(project.Uri)
github := "github.com"
pos := strings.Index(project.Src, github)
if pos != -1 {
project.Repo = project.Src[pos+len(github)+1:]
}
if !isModify {
_, err = project.Insert()
} else {
err = project.Persist(project)
}
if err != nil {
logger.Errorln("Publish Project error:", err)
}
// 发布項目，活跃度+10
if uid, ok := user["uid"].(int); ok {
weight := 10
if isModify {
weight = 2
}
go IncUserWeight("uid="+strconv.Itoa(uid), weight)
}
return
}
// ParseOneProject 处理单个 project
func ParseOneProject(projectUrl string) error {
if !strings.HasPrefix(projectUrl, "http") {
projectUrl = OsChinaDomain + projectUrl
}
var (
doc *goquery.Document
err error
)
// 加上 ?fromerr=xfwefs，否则页面有 js 重定向
if doc, err = goquery.NewDocument(projectUrl + "?fromerr=xfwefs"); err != nil {
return errors.New("goquery fetch " + projectUrl + " error:" + err.Error())
}
// 标题
category := strings.TrimSpace(doc.Find(".Project .name").Text())
name := strings.TrimSpace(doc.Find(".Project .name u").Text())
if category == "" && name == "" {
return errors.New("projectUrl:" + projectUrl + " category and name are empty")
}
tmpIndex := strings.LastIndex(category, name)
if tmpIndex != -1 {
category = category[:tmpIndex]
}
// uri
uri := projectUrl[strings.LastIndex(projectUrl, "/")+1:]
project := model.NewOpenProject()
err = project.Where("uri=?", uri).Find("id")
// 已经存在
if project.Id != 0 {
return errors.New("url" + projectUrl + "has exists!")
}
logoSelection := doc.Find(".Project .PN img")
if logoSelection.AttrOr("title", "") != "" {
project.Logo = logoSelection.AttrOr("src", "")
if !strings.HasPrefix(project.Logo, "http") {
project.Logo = OsChinaDomain + project.Logo
}
project.Logo, err = UploadUrlFile(project.Logo, ProjectLogoPrefix)
if err != nil {
logger.Errorln("project logo upload error:", err)
}
}
// 获取项目相关链接
doc.Find("#Body .urls li").Each(func(i int, liSelection *goquery.Selection) {
aSelection := liSelection.Find("a")
uri := util.FetchRealUrl(OsChinaDomain + aSelection.AttrOr("href", ""))
switch aSelection.Text() {
case "软件首页":
project.Home = uri
case "软件文档":
project.Doc = uri
case "软件下载":
project.Download = uri
}
})
ctime := util.TimeNow()
doc.Find("#Body .attrs li").Each(func(i int, liSelection *goquery.Selection) {
aSelection := liSelection.Find("a")
txt := aSelection.Text()
if i == 0 {
project.Licence = txt
if txt == "未知" {
project.Licence = "其他"
}
} else if i == 1 {
project.Lang = txt
} else if i == 2 {
project.Os = txt
} else if i == 3 {
dtime, err := time.ParseInLocation("2006年01月02日", aSelection.Last().Text(), time.Local)
if err != nil {
logger.Errorln("parse ctime error:", err)
} else {
ctime = dtime.Local().Format("2006-01-02 15:04:05")
}
}
})
project.Name = name
project.Category = category
project.Uri = uri
project.Repo = strings.TrimSpace(doc.Find("#Body .github-widget").AttrOr("data-repo", ""))
project.Src = "https://github.com/" + project.Repo
pos := strings.Index(project.Repo, "/")
if pos > -1 {
project.Author = project.Repo[:pos]
} else {
project.Author = "网友"
}
if project.Doc == "" {
// TODO：暂时认为一定是 Go 语言
project.Doc = "https://godoc.org/" + project.Src[8:]
}
desc := ""
doc.Find("#Body .detail").Find("p").NextAll().Each(func(i int, domSelection *goquery.Selection) {
doc.FindSelection(domSelection).WrapHtml(`<div id="tmp` + strconv.Itoa(i) + `"></div>`)
domHtml, _ := doc.Find("#tmp" + strconv.Itoa(i)).Html()
if domSelection.Is("pre") {
desc += domHtml + "\n\n"
} else {
desc += html2md.Convert(domHtml) + "\n\n"
}
})
project.Desc = strings.TrimSpace(desc)
project.Username = PresetUsernames[rand.Intn(4)]
project.Status = model.ProjectStatusOnline
project.Ctime = ctime
_, err = project.Insert()
if err != nil {
return errors.New("insert into open project error:" + err.Error())
}
return nil
}
// 发布主题。入topics和topics_ex库
func PublishTopic(user map[string]interface{}, form url.Values) (err error) {
uid := user["uid"].(int)
topic := model.NewTopic()
if form.Get("tid") != "" {
err = topic.Where("tid=?", form.Get("tid")).Find()
if err != nil {
logger.Errorln("Publish Topic find error:", err)
return
}
isAdmin := false
if _, ok := user["isadmin"]; ok {
isAdmin = user["isadmin"].(bool)
}
if topic.Uid != uid && !isAdmin {
err = NotModifyAuthorityErr
return
}
_, err = ModifyTopic(user, form)
if err != nil {
logger.Errorln("Publish Topic error:", err)
return
}
} else {
util.ConvertAssign(topic, form)
topic.Uid = uid
topic.Ctime = util.TimeNow()
var tid int
tid, err = topic.Insert()
if err != nil {
logger.Errorln("Publish Topic error:", err)
return
}
// 存扩展信息
topicEx := model.NewTopicEx()
topicEx.Tid = tid
_, err = topicEx.Insert()
if err != nil {
logger.Errorln("Insert TopicEx error:", err)
return
}
// 给 被@用户 发系统消息
ext := map[string]interface{}{
"objid": tid,
"objtype": model.TYPE_TOPIC,
"uid": user["uid"],
"msgtype": model.MsgtypePublishAtMe,
}
go SendSysMsgAtUsernames(form.Get("usernames"), ext)
// 发布主题，活跃度+10
go IncUserWeight("uid="+strconv.Itoa(uid), 10)
}
return
}