func (this *Pixivspy) WaitDownload() {
log4go.Info("等待图片下载完成...")
go func() {
tick := time.Tick(30 * time.Second)
for range tick {
log4go.Info("剩下[%d]任务未完成", atomic.LoadInt32(&this.maxTaskNum))
}
}()
this.waitDownloader.Wait()
}
func safeHandler(fn http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
defer func() {
e := recover()
if err, ok := e.(error); ok {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Info("WARN: panic in %v. - %v", fn, err)
log.Info(string(debug.Stack()))
}
}()
fn(w, r)
}
}
func main() {
// Load the configuration (isn't this easy?)
l4g.LoadConfiguration("example.xml")
l4g.Finest("This will only go to those of you really cool UDP kids! If you change enabled=true.")
l4g.Info("About .....")
l4g.Trace("About that time, eh chaps?")
l4g.Trace("About that time, eh chaps?")
l4g.Trace("About that time, eh chaps?")
l4g.Debug("About that time, eh chaps?")
l4g.Info("About that time, eh chaps?")
l4g.Warn("About that time, eh chaps?")
select {}
}
func loginHandler(w http.ResponseWriter, r *http.Request) {
milliseconds := time.Now().UnixNano() / 10E5
if r.Method == "GET" {
locals := make(map[string]interface{})
err := renderHtml(w, "login", locals)
check(err)
return
}
if r.Method == "POST" {
//mongodb opration
mongoCon, err := mgo.Dial("127.0.0.1:27018")
check(err)
defer mongoCon.Close()
mongoCon.SetMode(mgo.Monotonic, true)
l := mongoCon.DB("test_go").C("user")
user := r.FormValue("username")
passwd := r.FormValue("password")
result := UserData{}
err = l.Find(&bson.M{"username": user}).One(&result)
if err != nil {
log.Info("not found the username " + user)
panic(err)
}
if passwd == result.Password {
//set cookie
// create random string
randNum := rand.New(rand.NewSource(milliseconds))
h := sha1.New()
randString := result.Password + strconv.Itoa(randNum.Intn(100))
log.Info("rand string " + randString)
io.WriteString(h, randString)
hashString := fmt.Sprintf("%x", h.Sum(nil))
log.Info(hashString)
err = l.Update(bson.M{"username": user}, bson.M{"$set": bson.M{"random": hashString}})
check(err)
cookie1 := http.Cookie{Name: "value", Value: hashString}
cookie2 := http.Cookie{Name: "username", Value: user}
http.SetCookie(w, &cookie1)
http.SetCookie(w, &cookie2)
log.Info("set cookie!")
http.Redirect(w, r, "/", http.StatusFound)
} else {
log.Info("password is not right " + passwd)
}
return
}
}
func (this *Pixivspy) checkLogin() bool {
req, err := http.NewRequest(config.Info.CheckLoginPageUrl.Method,
config.Info.CheckLoginPageUrl.Url,
nil)
if err != nil {
log4go.Debug("可能也许大概内存不足导致发送请求失败!")
os.Exit(-1)
}
log4go.Info("登录中")
this.setHtmlHeader(req)
body, err := this.download(req)
if err != nil {
return false
}
query, err := goquery.ParseString(string(body))
if err != nil {
log4go.Error("解析登录数据失败", this.username)
return false
}
item := query.Find(".user")
if item.Length() == 0 {
return false
}
return true
}
func (this *Pixivspy) Login() bool {
postValue := url.Values{}
postValue.Set("mode", "login")
postValue.Add("pixiv_id", this.username)
postValue.Add("pass", this.password)
postValue.Add("skip", "1")
req, err := http.NewRequest(config.Info.LoginPageUrl.Method,
config.Info.LoginPageUrl.Url,
strings.NewReader(postValue.Encode()))
if err != nil {
log4go.Debug("可能也许大概内存不足导致发送请求失败!")
return false
}
log4go.Info("正在登录: %s", this.username)
this.setHtmlHeader(req)
client := http.Client{Jar: this.cookie}
resp := &http.Response{}
retry := 0
for retry < 5 {
resp, err = client.Do(req)
if err == nil {
break
}
if resp != nil {
resp.Body.Close()
}
retry++
}
if retry == 5 || !this.checkLogin() {
log4go.Error("帐号: %s 登录失败", this.username)
return false
}
log4go.Info("帐号: %s 登录成功", this.username)
return true
}
func main() {
SetLog()
defer log.Close()
var name = "skoo"
log.Debug("log4go by %s", name)
log.Info("log4go by %s", name)
log.Warn("log4go by %s", name)
log.Error("log4go by %s", name)
log.Fatal("log4go by %s", name)
}
func Create(daemon *daemon.Daemon, addr string) (server *Server, err os.Error) {
// create structure
server = new(Server)
// set daemon
server.daemon = daemon
// try to listen on the socket
log.Info("Trying to listen on %v", addr)
server.listener, err = net.Listen("tcp", addr)
return
}
func main() {
defer log4go.Close()
pixiv := pixivspy.NewPixivspy(config.Info.User, config.Info.Password)
if !pixiv.Login() {
return
}
downloadPath := util.FixPathName(config.Info.DownloadPath)
log4go.Info("检查下载目录: %s", downloadPath)
if !util.CheckFileIsExist(downloadPath) {
log4go.Info("创建下载目录: %s", downloadPath)
os.MkdirAll(downloadPath, 0777)
}
switch config.Info.Mode {
case "id":
for _, id := range config.Info.IDS {
pixiv.DownloadMemberIllusts(id)
}
case "star":
if len(config.Info.IDS) == 0 {
pixiv.DownloadStar("")
} else {
for _, id := range config.Info.IDS {
pixiv.DownloadStar(id)
}
}
default:
log4go.Error("Mode必须为[star | id]")
}
pixiv.WaitDownload()
}
func main() {
SetLog()
defer log.Close()
var name = "skoo"
for {
log.Debug("log4go by %s", name)
log.Info("log4go by %s", name)
log.Warn("log4go by %s", name)
log.Error("log4go by %s", name)
log.Fatal("log4go by %s", name)
time.Sleep(time.Second * 1)
}
}
func (server *Server) Serve() {
for {
// accept connection
conn, err := server.listener.Accept()
if err != nil {
log.Warn("Error in accept() loop: %v", err)
continue
}
// output this new connection
log.Info("Got connection from %v on local socket %v", conn.RemoteAddr(), conn.LocalAddr())
// start client
connection.HandleConnection(server.daemon, conn)
}
}
func registerHandler(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
locals := make(map[string]interface{})
err := renderHtml(w, "register", locals)
check(err)
return
}
if r.Method == "POST" {
//mongodb opration
mongoCon, err := mgo.Dial("127.0.0.1:27018")
check(err)
defer mongoCon.Close()
mongoCon.SetMode(mgo.Monotonic, true)
// 获取数据库,获取集合
l := mongoCon.DB("test_go").C("user")
i := mongoCon.DB("test_go").C("index")
var doc IndexData
change := mgo.Change{
Update: bson.M{"$inc": bson.M{"num": 1}},
ReturnNew: true,
}
_, err = i.Find(bson.M{"index": "uid"}).Apply(change, &doc)
if err != nil {
indexUid := IndexData{
Index: "uid",
Num: 1,
}
err := i.Insert(indexUid)
check(err)
doc.Num = 1
}
log.Info(doc.Num)
users := UserData{
Uid: doc.Num,
Username: r.FormValue("username"),
Password: r.FormValue("password"),
Email: r.FormValue("email"),
}
err = l.Insert(users)
check(err)
http.Redirect(w, r, "login", http.StatusFound)
return
}
}
func (this *Pixivspy) parseMemberIllusts(memberID string) []*pixivIllust {
works := []*pixivIllust{}
pageNum := 1
for {
page, ok := this.downloadMemberPage(memberID, pageNum)
if ok {
log4go.Debug("正在解析[%s]P主作品第[%d]页", memberID, pageNum)
query, err := goquery.ParseString(string(page))
if err != nil {
log4go.Debug("解析[%s]P主作品第[%d]页失败", memberID, pageNum)
return works
}
workItem := query.Find(".work")
if workItem.Length() == 0 {
log4go.Info("已经访问到最后一页")
return works
}
illustNameItem := query.Find("a h1.title")
for i := 0; i < workItem.Length(); i++ {
href := workItem.Eq(i).Attr("href")
class := workItem.Eq(i).Attr("class")
id := href[strings.Index(href, "illust_id=")+len("illust_id="):]
works = append(works,
&pixivIllust{
href: href,
class: class,
id: id,
title: illustNameItem.Eq(i).Text(),
})
}
pageNum++
} else {
log4go.Warn("访问[%s]P主主页失败", memberID)
return works
}
}
return works
}
func init() {
fileInfoArr, err := ioutil.ReadDir(TEMPLATE_DIR)
if err != nil {
panic(err)
return
}
var templateName, templatePath string
for _, fileInfo := range fileInfoArr {
templateName = fileInfo.Name()
if ext := path.Ext(templateName); ext != ".html" {
continue
}
templatePath = TEMPLATE_DIR + "/" + templateName
log.Info("Loading template:" + templatePath)
t := template.Must(template.ParseFiles(templatePath))
tmpl := strings.Split(templateName, ".html")[0]
templates[tmpl] = t
}
}
func init() {
runtime.GOMAXPROCS(runtime.NumCPU())
if config.Info.FW.On == true {
w := log4go.NewFileWriter()
w.SetPathPattern(config.Info.FW.LogPath)
log4go.Register(w)
}
if config.Info.CW.On == true {
w := log4go.NewConsoleWriter()
w.SetColor(config.Info.CW.Color)
log4go.Register(w)
}
switch config.Info.Level {
case "Debug":
log4go.SetLevel(log4go.DEBUG)
case "Info":
log4go.SetLevel(log4go.INFO)
case "Warning":
log4go.SetLevel(log4go.WARNING)
case "Error":
log4go.SetLevel(log4go.ERROR)
case "Fatal":
log4go.SetLevel(log4go.FATAL)
default:
log4go.Info("日志配置错误。")
}
log4go.Info("服务：%s", NAME)
log4go.Info("描述: %s", DESCRIPTION)
log4go.Info("版本号：%s", VERSION)
log4go.Info("操作系统: %s", runtime.GOOS)
log4go.Info("CPU: %s", runtime.GOARCH)
log4go.Info("核心数: %d", runtime.NumCPU())
}
func Handler(session *session.Session, msg packets.Packet) {
// try to cast to an Request
req, ok := msg.(*packet.Request)
if ok != true {
panic("BUG!!11!!1: failed cast to Request.. something is very messed up!")
}
// log handshake packet
log.Info("Got handshake<%s>", req.Username)
// answer handshake
res := &packet.Response{
Hash: "-",
}
// now add login handler
session.SetHandler(login_p.REQ_PID, login.Handler)
// FIXME: disable handshake?
session.Transmit(res)
}
func (this *Pixivspy) parseStarPage(url string) []string {
members := []string{}
for i := 1; true; i++ {
url = fmt.Sprintf(url+"&p=%d", i)
req, _ := http.NewRequest(config.Info.StarPageUrl.Method, url, nil)
this.setHtmlHeader(req)
page, err := this.download(req)
if page == nil || err != nil {
log4go.Error("下载关注页面失败")
break
}
query, err := goquery.ParseString(string(page))
if err != nil {
log4go.Debug("解析关注页面失败")
break
}
membersItem := query.Find("div.userdata")
if membersItem.Length() == 0 {
break
}
log4go.Info("正在解析第[%d]页关注页面", i)
for i := 0; i < membersItem.Length(); i++ {
a := membersItem.Eq(i)
memberID := a.Find("a").Attr("data-user_id")
members = append(members, memberID)
}
}
return members
}
func userListHandler(w http.ResponseWriter, r *http.Request) {
username, err := url.QueryUnescape(r.FormValue("username"))
check(err)
//mongodb opration
mongoCon, err := mgo.Dial("127.0.0.1:27018")
check(err)
defer mongoCon.Close()
mongoCon.SetMode(mgo.Monotonic, true)
// 获取数据库,获取集合
l := mongoCon.DB("test_go").C("list")
var outputResult []ListData
err = l.Find(&bson.M{"username": username}).All(&outputResult)
output := make(map[string]interface{})
output["msg"] = "true"
log.Info("output data", outputResult)
output["data"] = outputResult
outputJSON, err := json.Marshal(output)
check(err)
w.Write(outputJSON)
}
func main() {
// Get a new logger instance
//log := l4g.NewLogger()
// Create a default logger that is logging messages of FINE or higher
log.AddFilter("file", log.FINE, log.NewFileLogWriter(filename, false))
//log.Close()
/* Can also specify manually via the following: (these are the defaults) */
flw := log.NewFileLogWriter(filename, false)
//flw.SetFormat("[%D %T] [%L] (%S) %M")
//flw.SetRotate(false)
//flw.SetRotateSize(0)
//flw.SetRotateLines(0)
//flw.SetRotateDaily(false)
log.AddFilter("file", log.FINE, flw)
mux := http.NewServeMux()
staticDirHandler(mux, "/public/", "./public", 0)
mux.HandleFunc("/", safeHandler(indexHandler))
mux.HandleFunc("/register", safeHandler(registerHandler))
mux.HandleFunc("/login", safeHandler(loginHandler))
mux.HandleFunc("/save", safeHandler(saveHandler))
mux.HandleFunc("/user_list", safeHandler(userListHandler))
lis, err := net.Listen("tcp", ":9090")
check(err)
go func() {
http.Serve(lis, mux)
//err := http.ListenAndServe(":8080", mux)
if err != nil {
log.Critical("ListenAndServe: ", err.Error())
}
}()
ch := make(chan os.Signal)
signal.Notify(ch, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
//#WORKER is a new process tag.
//newArgs := append(os.Args, "#WORKER")
attr := syscall.ProcAttr{
Env: os.Environ(),
}
for {
sig := <-ch
//log.Info("Signal received:", sig)
switch sig {
case syscall.SIGHUP:
log.Info("get sighup sighup")
case syscall.SIGINT:
log.Info("get SIGINT ,exit!")
os.Exit(1)
case syscall.SIGUSR1:
log.Info("usr1")
//close the net
lis.Close()
log.Info("close connect")
if _, _, err := syscall.StartProcess(os.Args[0], os.Args, &attr); err != nil {
check(err)
}
//exit current process.
return
case syscall.SIGUSR2:
log.Info("usr2 ")
}
}
}
func saveHandler(w http.ResponseWriter, r *http.Request) {
postData, err := url.QueryUnescape(r.FormValue("data"))
check(err)
username, err := url.QueryUnescape(r.FormValue("username"))
check(err)
var data GetData
err = json.Unmarshal([]byte(postData), &data)
check(err)
//mongodb opration
mongoCon, err := mgo.Dial("127.0.0.1:27018")
check(err)
defer mongoCon.Close()
mongoCon.SetMode(mgo.Monotonic, true)
// 获取数据库,获取集合
l := mongoCon.DB("test_go").C("list")
for _, va := range data.Item {
va.Username = username
switch va.Change {
case "ADD":
log.Info("add item " + va.List)
// 存储数据
//result := ListData{}
//err = l.Find(&bson.M{"gid": va.Gid}).One(&result)
i := mongoCon.DB("test_go").C("index")
var doc IndexData
change := mgo.Change{
Update: bson.M{"$inc": bson.M{"num": 1}},
ReturnNew: true,
}
_, err = i.Find(bson.M{"index": "gid"}).Apply(change, &doc)
if err != nil {
indexUid := IndexData{
Index: "gid",
Num: 1000,
}
err := i.Insert(indexUid)
check(err)
doc.Num = 1000
}
va.Gid = doc.Num
va.Time = time.Now().UnixNano() / 10E5
va.Change = "SAVED"
va.Version = 1
err = l.Insert(&va)
if err != nil {
log.Info("find the same list " + va.List)
}
case "MODIFY":
log.Info("Modify item " + va.List)
// 读取数据
result := ListData{}
err = l.Find(&bson.M{"gid": va.Gid}).One(&result)
if err != nil {
log.Info("can't find ", va.Gid)
err = l.Insert(&va)
check(err)
} else {
if va.Version > result.Version || va.Time > result.Time {
//update mongodb data
oldList := bson.M{"gid": va.Gid}
va.Time = time.Now().UnixNano() / 10E5
va.Version = result.Version + 1
va.Change = "SAVED"
err := l.Update(oldList, &va)
check(err)
}
}
case "DEL":
log.Info("delete item " + va.List)
result := ListData{}
err = l.Find(&bson.M{"gid": va.Gid}).One(&result)
if err == nil {
err = l.Remove(result)
check(err)
log.Info("have deleted it")
}
default:
log.Info("change content is error " + va.Change)
return
}
}
userListHandler(w, r)
}
func (this *Pixivspy) DownloadMemberIllusts(memberID string) {
// 获取P主名称
MemberName := this.parseMemberNameFromMemberID(memberID)
log4go.Info("获取P主的昵称: %s", MemberName)
// 获取P主图片集
illusts := this.parseMemberIllusts(memberID)
if illusts == nil {
return
}
log4go.Info("准备解析[%s]P主的图片集", memberID)
ch := make(chan *pixivImage, 8)
go func() {
defer close(ch)
// 匹配图片集并获取图片地址
for _, v := range illusts {
page, ok := this.parseIllustPage(v)
if ok {
query, err := goquery.ParseString(page)
if err != nil {
log4go.Warn("解析[%s]图片集失败", v.title)
return
}
downloadPath := util.FixPathName(config.Info.DownloadPath + "/" + MemberName + "/" + v.title)
log4go.Info("检查图片集目录: %s", downloadPath)
if !util.CheckFileIsExist(downloadPath) {
log4go.Info("创建图片集目录: %s", downloadPath)
os.MkdirAll(downloadPath, 0777)
}
imageItem := query.Find(".original-image")
for i := 0; i < imageItem.Length(); i++ {
url := imageItem.Eq(i).Attr("data-src")
title := imageItem.Eq(i).Attr("alt")
width := imageItem.Eq(i).Attr("width")
height := imageItem.Eq(i).Attr("height")
ch <- &pixivImage{url: url,
title: title,
width: width,
height: height,
downloadPath: downloadPath,
href: config.Info.HostPageUrl.Url + v.href}
}
}
}
}()
for v := range ch {
pictureName := v.width + "_" + v.height + "_" + v.url[strings.LastIndex(v.url, "/")+1:]
downloadName := util.FixPathName(v.downloadPath + "/" + pictureName)
this.waitDownloader.Add(1)
atomic.AddInt32(&this.maxTaskNum, 1)
log4go.Info("当前任务数:%d, 正在获取空闲任务下载图片[%s]", atomic.LoadInt32(&this.maxTaskNum), downloadName)
for atomic.LoadInt32(&this.maxTaskNum) >= config.Info.MaxTaskNum {
time.Sleep(time.Second)
}
go func() {
defer this.waitDownloader.Done()
defer atomic.AddInt32(&this.maxTaskNum, -1)
if util.CheckFileIsExist(downloadName) {
log4go.Info("图片[%s]已存在, 跳过本次下载", downloadName)
return
}
log4go.Info("正在下载图片[%s]", pictureName)
req, _ := http.NewRequest("GET", v.url, nil)
this.setImageHeader(req, v.href)
image, err := this.download(req)
if err != nil {
log4go.Warn("图片[%s]下载失败", pictureName)
} else {
ioutil.WriteFile(downloadName, image, 0777)
}
}()
}
}
