func (this *Pixivspy) checkLogin() bool {
req, err := http.NewRequest(config.Info.CheckLoginPageUrl.Method,
config.Info.CheckLoginPageUrl.Url,
nil)
if err != nil {
log4go.Debug("可能也许大概内存不足导致发送请求失败!")
os.Exit(-1)
}
log4go.Info("登录中")
this.setHtmlHeader(req)
body, err := this.download(req)
if err != nil {
return false
}
query, err := goquery.ParseString(string(body))
if err != nil {
log4go.Error("解析登录数据失败", this.username)
return false
}
item := query.Find(".user")
if item.Length() == 0 {
return false
}
return true
}
func (this *Pixivspy) parseMemberNameFromMemberID(memberID string) string {
page, ok := this.downloadMemberPage(memberID, 1)
if ok {
query, err := goquery.ParseString(string(page))
if err != nil {
log4go.Debug("解析[%s]P主主页失败", memberID)
return "Unkown"
}
workItem := query.Find(".user")
return workItem.Text()
}
return "Unkown"
}
func (this *Pixivspy) parseMemberIllusts(memberID string) []*pixivIllust {
works := []*pixivIllust{}
pageNum := 1
for {
page, ok := this.downloadMemberPage(memberID, pageNum)
if ok {
log4go.Debug("正在解析[%s]P主作品第[%d]页", memberID, pageNum)
query, err := goquery.ParseString(string(page))
if err != nil {
log4go.Debug("解析[%s]P主作品第[%d]页失败", memberID, pageNum)
return works
}
workItem := query.Find(".work")
if workItem.Length() == 0 {
log4go.Info("已经访问到最后一页")
return works
}
illustNameItem := query.Find("a h1.title")
for i := 0; i < workItem.Length(); i++ {
href := workItem.Eq(i).Attr("href")
class := workItem.Eq(i).Attr("class")
id := href[strings.Index(href, "illust_id=")+len("illust_id="):]
works = append(works,
&pixivIllust{
href: href,
class: class,
id: id,
title: illustNameItem.Eq(i).Text(),
})
}
pageNum++
} else {
log4go.Warn("访问[%s]P主主页失败", memberID)
return works
}
}
return works
}
func (this *Pixivspy) parseStarPage(url string) []string {
members := []string{}
for i := 1; true; i++ {
url = fmt.Sprintf(url+"&p=%d", i)
req, _ := http.NewRequest(config.Info.StarPageUrl.Method, url, nil)
this.setHtmlHeader(req)
page, err := this.download(req)
if page == nil || err != nil {
log4go.Error("下载关注页面失败")
break
}
query, err := goquery.ParseString(string(page))
if err != nil {
log4go.Debug("解析关注页面失败")
break
}
membersItem := query.Find("div.userdata")
if membersItem.Length() == 0 {
break
}
log4go.Info("正在解析第[%d]页关注页面", i)
for i := 0; i < membersItem.Length(); i++ {
a := membersItem.Eq(i)
memberID := a.Find("a").Attr("data-user_id")
members = append(members, memberID)
}
}
return members
}
func (this *Pixivspy) DownloadMemberIllusts(memberID string) {
// 获取P主名称
MemberName := this.parseMemberNameFromMemberID(memberID)
log4go.Info("获取P主的昵称: %s", MemberName)
// 获取P主图片集
illusts := this.parseMemberIllusts(memberID)
if illusts == nil {
return
}
log4go.Info("准备解析[%s]P主的图片集", memberID)
ch := make(chan *pixivImage, 8)
go func() {
defer close(ch)
// 匹配图片集并获取图片地址
for _, v := range illusts {
page, ok := this.parseIllustPage(v)
if ok {
query, err := goquery.ParseString(page)
if err != nil {
log4go.Warn("解析[%s]图片集失败", v.title)
return
}
downloadPath := util.FixPathName(config.Info.DownloadPath + "/" + MemberName + "/" + v.title)
log4go.Info("检查图片集目录: %s", downloadPath)
if !util.CheckFileIsExist(downloadPath) {
log4go.Info("创建图片集目录: %s", downloadPath)
os.MkdirAll(downloadPath, 0777)
}
imageItem := query.Find(".original-image")
for i := 0; i < imageItem.Length(); i++ {
url := imageItem.Eq(i).Attr("data-src")
title := imageItem.Eq(i).Attr("alt")
width := imageItem.Eq(i).Attr("width")
height := imageItem.Eq(i).Attr("height")
ch <- &pixivImage{url: url,
title: title,
width: width,
height: height,
downloadPath: downloadPath,
href: config.Info.HostPageUrl.Url + v.href}
}
}
}
}()
for v := range ch {
pictureName := v.width + "_" + v.height + "_" + v.url[strings.LastIndex(v.url, "/")+1:]
downloadName := util.FixPathName(v.downloadPath + "/" + pictureName)
this.waitDownloader.Add(1)
atomic.AddInt32(&this.maxTaskNum, 1)
log4go.Info("当前任务数:%d, 正在获取空闲任务下载图片[%s]", atomic.LoadInt32(&this.maxTaskNum), downloadName)
for atomic.LoadInt32(&this.maxTaskNum) >= config.Info.MaxTaskNum {
time.Sleep(time.Second)
}
go func() {
defer this.waitDownloader.Done()
defer atomic.AddInt32(&this.maxTaskNum, -1)
if util.CheckFileIsExist(downloadName) {
log4go.Info("图片[%s]已存在, 跳过本次下载", downloadName)
return
}
log4go.Info("正在下载图片[%s]", pictureName)
req, _ := http.NewRequest("GET", v.url, nil)
this.setImageHeader(req, v.href)
image, err := this.download(req)
if err != nil {
log4go.Warn("图片[%s]下载失败", pictureName)
} else {
ioutil.WriteFile(downloadName, image, 0777)
}
}()
}
}