Beispiel #1
0
func (this *Pixivspy) checkLogin() bool {
	req, err := http.NewRequest(config.Info.CheckLoginPageUrl.Method,
		config.Info.CheckLoginPageUrl.Url,
		nil)

	if err != nil {
		log4go.Debug("可能也许大概内存不足导致发送请求失败!")
		os.Exit(-1)
	}

	log4go.Info("登录中")

	this.setHtmlHeader(req)

	body, err := this.download(req)
	if err != nil {
		return false
	}

	query, err := goquery.ParseString(string(body))
	if err != nil {
		log4go.Error("解析登录数据失败", this.username)
		return false
	}

	item := query.Find(".user")
	if item.Length() == 0 {
		return false
	}

	return true
}
Beispiel #2
0
func (this *Pixivspy) parseMemberNameFromMemberID(memberID string) string {
	page, ok := this.downloadMemberPage(memberID, 1)

	if ok {
		query, err := goquery.ParseString(string(page))
		if err != nil {
			log4go.Debug("解析[%s]P主主页失败", memberID)
			return "Unkown"
		}

		workItem := query.Find(".user")

		return workItem.Text()
	}

	return "Unkown"
}
Beispiel #3
0
func (this *Pixivspy) parseMemberIllusts(memberID string) []*pixivIllust {
	works := []*pixivIllust{}
	pageNum := 1

	for {
		page, ok := this.downloadMemberPage(memberID, pageNum)

		if ok {
			log4go.Debug("正在解析[%s]P主作品第[%d]页", memberID, pageNum)

			query, err := goquery.ParseString(string(page))
			if err != nil {
				log4go.Debug("解析[%s]P主作品第[%d]页失败", memberID, pageNum)
				return works
			}

			workItem := query.Find(".work")
			if workItem.Length() == 0 {
				log4go.Info("已经访问到最后一页")
				return works
			}
			illustNameItem := query.Find("a h1.title")

			for i := 0; i < workItem.Length(); i++ {
				href := workItem.Eq(i).Attr("href")
				class := workItem.Eq(i).Attr("class")
				id := href[strings.Index(href, "illust_id=")+len("illust_id="):]

				works = append(works,
					&pixivIllust{
						href:  href,
						class: class,
						id:    id,
						title: illustNameItem.Eq(i).Text(),
					})
			}
			pageNum++
		} else {
			log4go.Warn("访问[%s]P主主页失败", memberID)
			return works
		}
	}

	return works
}
Beispiel #4
0
func (this *Pixivspy) parseStarPage(url string) []string {
	members := []string{}

	for i := 1; true; i++ {
		url = fmt.Sprintf(url+"&p=%d", i)

		req, _ := http.NewRequest(config.Info.StarPageUrl.Method, url, nil)

		this.setHtmlHeader(req)

		page, err := this.download(req)
		if page == nil || err != nil {
			log4go.Error("下载关注页面失败")
			break
		}

		query, err := goquery.ParseString(string(page))
		if err != nil {
			log4go.Debug("解析关注页面失败")
			break
		}

		membersItem := query.Find("div.userdata")

		if membersItem.Length() == 0 {
			break
		}

		log4go.Info("正在解析第[%d]页关注页面", i)

		for i := 0; i < membersItem.Length(); i++ {
			a := membersItem.Eq(i)
			memberID := a.Find("a").Attr("data-user_id")
			members = append(members, memberID)
		}
	}

	return members
}
Beispiel #5
0
func (this *Pixivspy) DownloadMemberIllusts(memberID string) {

	// 获取P主名称
	MemberName := this.parseMemberNameFromMemberID(memberID)
	log4go.Info("获取P主的昵称: %s", MemberName)

	// 获取P主图片集
	illusts := this.parseMemberIllusts(memberID)
	if illusts == nil {
		return
	}

	log4go.Info("准备解析[%s]P主的图片集", memberID)

	ch := make(chan *pixivImage, 8)
	go func() {
		defer close(ch)
		// 匹配图片集并获取图片地址
		for _, v := range illusts {
			page, ok := this.parseIllustPage(v)

			if ok {
				query, err := goquery.ParseString(page)
				if err != nil {
					log4go.Warn("解析[%s]图片集失败", v.title)
					return
				}

				downloadPath := util.FixPathName(config.Info.DownloadPath + "/" + MemberName + "/" + v.title)

				log4go.Info("检查图片集目录: %s", downloadPath)
				if !util.CheckFileIsExist(downloadPath) {
					log4go.Info("创建图片集目录: %s", downloadPath)
					os.MkdirAll(downloadPath, 0777)
				}

				imageItem := query.Find(".original-image")
				for i := 0; i < imageItem.Length(); i++ {
					url := imageItem.Eq(i).Attr("data-src")
					title := imageItem.Eq(i).Attr("alt")
					width := imageItem.Eq(i).Attr("width")
					height := imageItem.Eq(i).Attr("height")
					ch <- &pixivImage{url: url,
						title:        title,
						width:        width,
						height:       height,
						downloadPath: downloadPath,
						href:         config.Info.HostPageUrl.Url + v.href}
				}
			}
		}
	}()

	for v := range ch {
		pictureName := v.width + "_" + v.height + "_" + v.url[strings.LastIndex(v.url, "/")+1:]
		downloadName := util.FixPathName(v.downloadPath + "/" + pictureName)

		this.waitDownloader.Add(1)
		atomic.AddInt32(&this.maxTaskNum, 1)

		log4go.Info("当前任务数:%d, 正在获取空闲任务下载图片[%s]", atomic.LoadInt32(&this.maxTaskNum), downloadName)

		for atomic.LoadInt32(&this.maxTaskNum) >= config.Info.MaxTaskNum {
			time.Sleep(time.Second)
		}

		go func() {
			defer this.waitDownloader.Done()
			defer atomic.AddInt32(&this.maxTaskNum, -1)

			if util.CheckFileIsExist(downloadName) {
				log4go.Info("图片[%s]已存在, 跳过本次下载", downloadName)
				return
			}

			log4go.Info("正在下载图片[%s]", pictureName)

			req, _ := http.NewRequest("GET", v.url, nil)
			this.setImageHeader(req, v.href)
			image, err := this.download(req)
			if err != nil {
				log4go.Warn("图片[%s]下载失败", pictureName)
			} else {
				ioutil.WriteFile(downloadName, image, 0777)
			}
		}()
	}
}