func CommentsWorker(pos int64, limit int, done chan bool) {
defer func() {
fmt.Println("Comment", pos, "to", pos+int64(limit), "done")
done <- true
}()
bbcodeCompiler := bbcode.NewCompiler(true, true)
var discuzComments []DiscuzComment
csvfile := csvdatafile.New("comment", fmt.Sprintf("/tmp/comment-%d.csv", pos))
if err := csvfile.Create(); err != nil {
fmt.Println("Coment Worker Create File:", err)
}
sql := "select pid, tid, message, authorid, author, useip, dateline from pre_forum_post where first = 0 limit ?,?"
if _, err := conf.Orm.Raw(sql, pos, limit).QueryRows(&discuzComments); err != nil {
fmt.Println("CommentsWorker Error:", err)
return
}
csvfile.Fields = []string{
"id",
"topic_id",
"content_hex",
"user_id",
"username",
"ip",
"created"}
for _, discuzComment := range discuzComments {
content := &models.Content{Message: bbcodeCompiler.Compile(discuzComment.Message), TopicId: discuzComment.Tid, CommentId: discuzComment.Pid}
var contentHex string
var err error
if contentHex, err = content.Insert(); err != nil {
fmt.Println("Comments Worker Content Insert:", err)
}
if err := csvfile.AppendRow(
discuzComment.Pid,
discuzComment.Tid,
contentHex,
discuzComment.Authorid,
discuzComment.Author,
discuzComment.Useip,
time.Unix(discuzComment.Dateline, 0).Format("2006-01-02 15:04:05"),
); err != nil {
fmt.Println(err)
}
if err := csvfile.Flush(); err != nil {
fmt.Println(err)
}
}
csvfile.Close()
if err := csvfile.LoadToMySQL(conf.OrmGotalk); err != nil {
fmt.Println(err)
}
csvfile.Remove()
}
func TopicsWorker(pos int64, limit int, done chan bool) {
defer func() {
fmt.Println("Topic", pos, "to", pos+int64(limit), "done")
done <- true
}()
bbcodeCompiler := bbcode.NewCompiler(true, true)
var discuzTopics []DiscuzTopic
csvfile := csvdatafile.New("topic", fmt.Sprintf("/tmp/topic-%d.csv", pos))
if err := csvfile.Create(); err != nil {
fmt.Println("Topic Worker Create File:", err)
}
sql := "select t.tid, t.subject, p.message, t.authorid, t.author, t.fid, t.views, t.replies, t.favtimes, t.digest, t.closed, t.lastposter, t.lastpost, t.dateline,p.useip from pre_forum_thread as t left join pre_forum_post as p on t.tid=p.tid where p.first = 1 limit ?,?"
if _, err := conf.Orm.Raw(sql, pos, limit).QueryRows(&discuzTopics); err != nil {
fmt.Println("TopicsWorker Error:", err)
return
}
csvfile.Fields = []string{
"id",
"title",
"content_hex",
"user_id",
"username",
"category_id",
"pv_count",
"comment_count",
"bookmark_count",
"is_excellent",
"is_closed",
"last_reply_username",
"last_reply_at",
"created",
"updated",
"ip"}
for _, discuzTopic := range discuzTopics {
content := &models.Content{Message: bbcodeCompiler.Compile(discuzTopic.Message), TopicId: discuzTopic.Tid}
var contentHex string
var err error
if contentHex, err = content.Insert(); err != nil {
fmt.Println("Topic Worker Content Insert:", err)
}
if err := csvfile.AppendRow(
discuzTopic.Tid,
discuzTopic.Subject,
contentHex,
discuzTopic.Authorid,
discuzTopic.Author,
discuzTopic.Fid,
discuzTopic.Views,
discuzTopic.Replies,
discuzTopic.Favtimes,
Btoi(discuzTopic.Digest),
Btoi(discuzTopic.Closed),
discuzTopic.Lastposter,
time.Unix(discuzTopic.Lastpost, 0).Format("2006-01-02 15:04:05"),
time.Unix(discuzTopic.Dateline, 0).Format("2006-01-02 15:04:05"),
time.Unix(discuzTopic.Lastpost, 0).Format("2006-01-02 15:04:05"),
discuzTopic.Useip); err != nil {
fmt.Println(err)
}
if err := csvfile.Flush(); err != nil {
fmt.Println(err)
}
}
csvfile.Close()
if err := csvfile.LoadToMySQL(conf.OrmGotalk); err != nil {
fmt.Println(err)
}
csvfile.Remove()
}