Example #1
0
func (this *Pixivspy) WaitDownload() {
	log4go.Info("等待图片下载完成...")

	go func() {
		tick := time.Tick(30 * time.Second)
		for range tick {
			log4go.Info("剩下[%d]任务未完成", atomic.LoadInt32(&this.maxTaskNum))
		}
	}()

	this.waitDownloader.Wait()
}
Example #2
0
func safeHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			e := recover()
			if err, ok := e.(error); ok {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				log.Info("WARN: panic in %v. - %v", fn, err)
				log.Info(string(debug.Stack()))
			}
		}()
		fn(w, r)
	}
}
func main() {
	// Load the configuration (isn't this easy?)
	l4g.LoadConfiguration("example.xml")

	l4g.Finest("This will only go to those of you really cool UDP kids!  If you change enabled=true.")
	l4g.Info("About .....")
	l4g.Trace("About that time, eh chaps?")
	l4g.Trace("About that time, eh chaps?")
	l4g.Trace("About that time, eh chaps?")
	l4g.Debug("About that time, eh chaps?")
	l4g.Info("About that time, eh chaps?")
	l4g.Warn("About that time, eh chaps?")
	select {}
}
Example #4
0
func loginHandler(w http.ResponseWriter, r *http.Request) {
	milliseconds := time.Now().UnixNano() / 10E5
	if r.Method == "GET" {
		locals := make(map[string]interface{})
		err := renderHtml(w, "login", locals)
		check(err)
		return
	}

	if r.Method == "POST" {
		//mongodb opration
		mongoCon, err := mgo.Dial("127.0.0.1:27018")
		check(err)
		defer mongoCon.Close()
		mongoCon.SetMode(mgo.Monotonic, true)
		l := mongoCon.DB("test_go").C("user")
		user := r.FormValue("username")
		passwd := r.FormValue("password")
		result := UserData{}
		err = l.Find(&bson.M{"username": user}).One(&result)
		if err != nil {
			log.Info("not found the username " + user)
			panic(err)
		}

		if passwd == result.Password {
			//set cookie
			// create random string
			randNum := rand.New(rand.NewSource(milliseconds))
			h := sha1.New()
			randString := result.Password + strconv.Itoa(randNum.Intn(100))
			log.Info("rand string " + randString)
			io.WriteString(h, randString)
			hashString := fmt.Sprintf("%x", h.Sum(nil))
			log.Info(hashString)
			err = l.Update(bson.M{"username": user}, bson.M{"$set": bson.M{"random": hashString}})
			check(err)
			cookie1 := http.Cookie{Name: "value", Value: hashString}
			cookie2 := http.Cookie{Name: "username", Value: user}
			http.SetCookie(w, &cookie1)
			http.SetCookie(w, &cookie2)
			log.Info("set cookie!")
			http.Redirect(w, r, "/", http.StatusFound)
		} else {
			log.Info("password is not right " + passwd)
		}
		return
	}
}
Example #5
0
func (this *Pixivspy) checkLogin() bool {
	req, err := http.NewRequest(config.Info.CheckLoginPageUrl.Method,
		config.Info.CheckLoginPageUrl.Url,
		nil)

	if err != nil {
		log4go.Debug("可能也许大概内存不足导致发送请求失败!")
		os.Exit(-1)
	}

	log4go.Info("登录中")

	this.setHtmlHeader(req)

	body, err := this.download(req)
	if err != nil {
		return false
	}

	query, err := goquery.ParseString(string(body))
	if err != nil {
		log4go.Error("解析登录数据失败", this.username)
		return false
	}

	item := query.Find(".user")
	if item.Length() == 0 {
		return false
	}

	return true
}
Example #6
0
func (this *Pixivspy) Login() bool {
	postValue := url.Values{}
	postValue.Set("mode", "login")
	postValue.Add("pixiv_id", this.username)
	postValue.Add("pass", this.password)
	postValue.Add("skip", "1")

	req, err := http.NewRequest(config.Info.LoginPageUrl.Method,
		config.Info.LoginPageUrl.Url,
		strings.NewReader(postValue.Encode()))

	if err != nil {
		log4go.Debug("可能也许大概内存不足导致发送请求失败!")
		return false
	}

	log4go.Info("正在登录: %s", this.username)

	this.setHtmlHeader(req)

	client := http.Client{Jar: this.cookie}

	resp := &http.Response{}
	retry := 0
	for retry < 5 {
		resp, err = client.Do(req)
		if err == nil {
			break
		}

		if resp != nil {
			resp.Body.Close()
		}

		retry++
	}

	if retry == 5 || !this.checkLogin() {
		log4go.Error("帐号: %s 登录失败", this.username)
		return false
	}

	log4go.Info("帐号: %s 登录成功", this.username)

	return true
}
Example #7
0
func main() {
	SetLog()
	defer log.Close()

	var name = "skoo"
	log.Debug("log4go by %s", name)
	log.Info("log4go by %s", name)
	log.Warn("log4go by %s", name)
	log.Error("log4go by %s", name)
	log.Fatal("log4go by %s", name)
}
Example #8
0
func Create(daemon *daemon.Daemon, addr string) (server *Server, err os.Error) {
	// create structure
	server = new(Server)

	// set daemon
	server.daemon = daemon

	// try to listen on the socket
	log.Info("Trying to listen on %v", addr)
	server.listener, err = net.Listen("tcp", addr)
	return
}
Example #9
0
func main() {
	defer log4go.Close()

	pixiv := pixivspy.NewPixivspy(config.Info.User, config.Info.Password)

	if !pixiv.Login() {
		return
	}

	downloadPath := util.FixPathName(config.Info.DownloadPath)

	log4go.Info("检查下载目录: %s", downloadPath)
	if !util.CheckFileIsExist(downloadPath) {
		log4go.Info("创建下载目录: %s", downloadPath)
		os.MkdirAll(downloadPath, 0777)
	}

	switch config.Info.Mode {
	case "id":
		for _, id := range config.Info.IDS {
			pixiv.DownloadMemberIllusts(id)
		}
	case "star":
		if len(config.Info.IDS) == 0 {
			pixiv.DownloadStar("")
		} else {
			for _, id := range config.Info.IDS {
				pixiv.DownloadStar(id)
			}
		}
	default:
		log4go.Error("Mode必须为[star | id]")
	}

	pixiv.WaitDownload()
}
Example #10
0
func main() {
	SetLog()
	defer log.Close()

	var name = "skoo"

	for {
		log.Debug("log4go by %s", name)
		log.Info("log4go by %s", name)
		log.Warn("log4go by %s", name)
		log.Error("log4go by %s", name)
		log.Fatal("log4go by %s", name)

		time.Sleep(time.Second * 1)
	}
}
Example #11
0
func (server *Server) Serve() {
	for {
		// accept connection
		conn, err := server.listener.Accept()
		if err != nil {
			log.Warn("Error in accept() loop: %v", err)
			continue
		}

		// output this new connection
		log.Info("Got connection from %v on local socket %v", conn.RemoteAddr(), conn.LocalAddr())

		// start client
		connection.HandleConnection(server.daemon, conn)
	}
}
Example #12
0
func registerHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		locals := make(map[string]interface{})
		err := renderHtml(w, "register", locals)
		check(err)
		return
	}

	if r.Method == "POST" {
		//mongodb opration
		mongoCon, err := mgo.Dial("127.0.0.1:27018")
		check(err)
		defer mongoCon.Close()
		mongoCon.SetMode(mgo.Monotonic, true)

		// 获取数据库,获取集合
		l := mongoCon.DB("test_go").C("user")
		i := mongoCon.DB("test_go").C("index")
		var doc IndexData
		change := mgo.Change{
			Update:    bson.M{"$inc": bson.M{"num": 1}},
			ReturnNew: true,
		}
		_, err = i.Find(bson.M{"index": "uid"}).Apply(change, &doc)
		if err != nil {
			indexUid := IndexData{
				Index: "uid",
				Num:   1,
			}
			err := i.Insert(indexUid)
			check(err)
			doc.Num = 1
		}
		log.Info(doc.Num)
		users := UserData{
			Uid:      doc.Num,
			Username: r.FormValue("username"),
			Password: r.FormValue("password"),
			Email:    r.FormValue("email"),
		}
		err = l.Insert(users)
		check(err)
		http.Redirect(w, r, "login", http.StatusFound)
		return
	}

}
Example #13
0
func (this *Pixivspy) parseMemberIllusts(memberID string) []*pixivIllust {
	works := []*pixivIllust{}
	pageNum := 1

	for {
		page, ok := this.downloadMemberPage(memberID, pageNum)

		if ok {
			log4go.Debug("正在解析[%s]P主作品第[%d]页", memberID, pageNum)

			query, err := goquery.ParseString(string(page))
			if err != nil {
				log4go.Debug("解析[%s]P主作品第[%d]页失败", memberID, pageNum)
				return works
			}

			workItem := query.Find(".work")
			if workItem.Length() == 0 {
				log4go.Info("已经访问到最后一页")
				return works
			}
			illustNameItem := query.Find("a h1.title")

			for i := 0; i < workItem.Length(); i++ {
				href := workItem.Eq(i).Attr("href")
				class := workItem.Eq(i).Attr("class")
				id := href[strings.Index(href, "illust_id=")+len("illust_id="):]

				works = append(works,
					&pixivIllust{
						href:  href,
						class: class,
						id:    id,
						title: illustNameItem.Eq(i).Text(),
					})
			}
			pageNum++
		} else {
			log4go.Warn("访问[%s]P主主页失败", memberID)
			return works
		}
	}

	return works
}
Example #14
0
func init() {

	fileInfoArr, err := ioutil.ReadDir(TEMPLATE_DIR)
	if err != nil {
		panic(err)
		return
	}
	var templateName, templatePath string
	for _, fileInfo := range fileInfoArr {
		templateName = fileInfo.Name()
		if ext := path.Ext(templateName); ext != ".html" {
			continue
		}
		templatePath = TEMPLATE_DIR + "/" + templateName
		log.Info("Loading template:" + templatePath)
		t := template.Must(template.ParseFiles(templatePath))
		tmpl := strings.Split(templateName, ".html")[0]
		templates[tmpl] = t
	}
}
Example #15
0
func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	if config.Info.FW.On == true {
		w := log4go.NewFileWriter()
		w.SetPathPattern(config.Info.FW.LogPath)
		log4go.Register(w)
	}

	if config.Info.CW.On == true {
		w := log4go.NewConsoleWriter()
		w.SetColor(config.Info.CW.Color)
		log4go.Register(w)
	}

	switch config.Info.Level {
	case "Debug":
		log4go.SetLevel(log4go.DEBUG)

	case "Info":
		log4go.SetLevel(log4go.INFO)

	case "Warning":
		log4go.SetLevel(log4go.WARNING)

	case "Error":
		log4go.SetLevel(log4go.ERROR)

	case "Fatal":
		log4go.SetLevel(log4go.FATAL)

	default:
		log4go.Info("日志配置错误。")
	}

	log4go.Info("服务:%s", NAME)
	log4go.Info("描述: %s", DESCRIPTION)
	log4go.Info("版本号:%s", VERSION)
	log4go.Info("操作系统: %s", runtime.GOOS)
	log4go.Info("CPU: %s", runtime.GOARCH)
	log4go.Info("核心数: %d", runtime.NumCPU())
}
Example #16
0
func Handler(session *session.Session, msg packets.Packet) {
	// try to cast to an Request
	req, ok := msg.(*packet.Request)
	if ok != true {
		panic("BUG!!11!!1: failed cast to Request.. something is very messed up!")
	}

	// log handshake packet
	log.Info("Got handshake<%s>", req.Username)

	// answer handshake
	res := &packet.Response{
		Hash: "-",
	}

	// now add login handler
	session.SetHandler(login_p.REQ_PID, login.Handler)

	// FIXME: disable handshake?

	session.Transmit(res)
}
Example #17
0
func (this *Pixivspy) parseStarPage(url string) []string {
	members := []string{}

	for i := 1; true; i++ {
		url = fmt.Sprintf(url+"&p=%d", i)

		req, _ := http.NewRequest(config.Info.StarPageUrl.Method, url, nil)

		this.setHtmlHeader(req)

		page, err := this.download(req)
		if page == nil || err != nil {
			log4go.Error("下载关注页面失败")
			break
		}

		query, err := goquery.ParseString(string(page))
		if err != nil {
			log4go.Debug("解析关注页面失败")
			break
		}

		membersItem := query.Find("div.userdata")

		if membersItem.Length() == 0 {
			break
		}

		log4go.Info("正在解析第[%d]页关注页面", i)

		for i := 0; i < membersItem.Length(); i++ {
			a := membersItem.Eq(i)
			memberID := a.Find("a").Attr("data-user_id")
			members = append(members, memberID)
		}
	}

	return members
}
Example #18
0
func userListHandler(w http.ResponseWriter, r *http.Request) {
	username, err := url.QueryUnescape(r.FormValue("username"))
	check(err)

	//mongodb opration
	mongoCon, err := mgo.Dial("127.0.0.1:27018")
	check(err)

	defer mongoCon.Close()

	mongoCon.SetMode(mgo.Monotonic, true)

	// 获取数据库,获取集合
	l := mongoCon.DB("test_go").C("list")
	var outputResult []ListData
	err = l.Find(&bson.M{"username": username}).All(&outputResult)
	output := make(map[string]interface{})
	output["msg"] = "true"
	log.Info("output data", outputResult)
	output["data"] = outputResult
	outputJSON, err := json.Marshal(output)
	check(err)
	w.Write(outputJSON)
}
Example #19
0
func main() {

	// Get a new logger instance
	//log := l4g.NewLogger()

	// Create a default logger that is logging messages of FINE or higher
	log.AddFilter("file", log.FINE, log.NewFileLogWriter(filename, false))
	//log.Close()

	/* Can also specify manually via the following: (these are the defaults) */
	flw := log.NewFileLogWriter(filename, false)
	//flw.SetFormat("[%D %T] [%L] (%S) %M")
	//flw.SetRotate(false)
	//flw.SetRotateSize(0)
	//flw.SetRotateLines(0)
	//flw.SetRotateDaily(false)
	log.AddFilter("file", log.FINE, flw)

	mux := http.NewServeMux()
	staticDirHandler(mux, "/public/", "./public", 0)
	mux.HandleFunc("/", safeHandler(indexHandler))
	mux.HandleFunc("/register", safeHandler(registerHandler))
	mux.HandleFunc("/login", safeHandler(loginHandler))
	mux.HandleFunc("/save", safeHandler(saveHandler))
	mux.HandleFunc("/user_list", safeHandler(userListHandler))

	lis, err := net.Listen("tcp", ":9090")
	check(err)

	go func() {
		http.Serve(lis, mux)
		//err := http.ListenAndServe(":8080", mux)
		if err != nil {
			log.Critical("ListenAndServe: ", err.Error())
		}
	}()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
	//#WORKER is a new process tag.
	//newArgs := append(os.Args, "#WORKER")
	attr := syscall.ProcAttr{
		Env: os.Environ(),
	}
	for {
		sig := <-ch
		//log.Info("Signal received:", sig)
		switch sig {
		case syscall.SIGHUP:
			log.Info("get sighup sighup")
		case syscall.SIGINT:
			log.Info("get SIGINT ,exit!")
			os.Exit(1)
		case syscall.SIGUSR1:
			log.Info("usr1")
			//close the net
			lis.Close()
			log.Info("close connect")
			if _, _, err := syscall.StartProcess(os.Args[0], os.Args, &attr); err != nil {
				check(err)
			}
			//exit current process.
			return
		case syscall.SIGUSR2:
			log.Info("usr2 ")
		}
	}
}
Example #20
0
func saveHandler(w http.ResponseWriter, r *http.Request) {
	postData, err := url.QueryUnescape(r.FormValue("data"))
	check(err)
	username, err := url.QueryUnescape(r.FormValue("username"))
	check(err)

	var data GetData
	err = json.Unmarshal([]byte(postData), &data)
	check(err)
	//mongodb opration
	mongoCon, err := mgo.Dial("127.0.0.1:27018")
	check(err)

	defer mongoCon.Close()

	mongoCon.SetMode(mgo.Monotonic, true)

	// 获取数据库,获取集合
	l := mongoCon.DB("test_go").C("list")

	for _, va := range data.Item {
		va.Username = username
		switch va.Change {
		case "ADD":
			log.Info("add item " + va.List)
			// 存储数据
			//result := ListData{}
			//err = l.Find(&bson.M{"gid": va.Gid}).One(&result)
			i := mongoCon.DB("test_go").C("index")
			var doc IndexData
			change := mgo.Change{
				Update:    bson.M{"$inc": bson.M{"num": 1}},
				ReturnNew: true,
			}
			_, err = i.Find(bson.M{"index": "gid"}).Apply(change, &doc)
			if err != nil {
				indexUid := IndexData{
					Index: "gid",
					Num:   1000,
				}
				err := i.Insert(indexUid)
				check(err)
				doc.Num = 1000
			}
			va.Gid = doc.Num
			va.Time = time.Now().UnixNano() / 10E5
			va.Change = "SAVED"
			va.Version = 1
			err = l.Insert(&va)
			if err != nil {
				log.Info("find the same list " + va.List)
			}
		case "MODIFY":
			log.Info("Modify item " + va.List)
			//  读取数据
			result := ListData{}
			err = l.Find(&bson.M{"gid": va.Gid}).One(&result)
			if err != nil {
				log.Info("can't find ", va.Gid)
				err = l.Insert(&va)
				check(err)
			} else {
				if va.Version > result.Version || va.Time > result.Time {
					//update mongodb data
					oldList := bson.M{"gid": va.Gid}
					va.Time = time.Now().UnixNano() / 10E5
					va.Version = result.Version + 1
					va.Change = "SAVED"
					err := l.Update(oldList, &va)
					check(err)
				}
			}
		case "DEL":
			log.Info("delete item " + va.List)
			result := ListData{}
			err = l.Find(&bson.M{"gid": va.Gid}).One(&result)
			if err == nil {
				err = l.Remove(result)
				check(err)
				log.Info("have deleted it")
			}
		default:
			log.Info("change content is error " + va.Change)
			return
		}
	}
	userListHandler(w, r)
}
Example #21
0
func (this *Pixivspy) DownloadMemberIllusts(memberID string) {

	// 获取P主名称
	MemberName := this.parseMemberNameFromMemberID(memberID)
	log4go.Info("获取P主的昵称: %s", MemberName)

	// 获取P主图片集
	illusts := this.parseMemberIllusts(memberID)
	if illusts == nil {
		return
	}

	log4go.Info("准备解析[%s]P主的图片集", memberID)

	ch := make(chan *pixivImage, 8)
	go func() {
		defer close(ch)
		// 匹配图片集并获取图片地址
		for _, v := range illusts {
			page, ok := this.parseIllustPage(v)

			if ok {
				query, err := goquery.ParseString(page)
				if err != nil {
					log4go.Warn("解析[%s]图片集失败", v.title)
					return
				}

				downloadPath := util.FixPathName(config.Info.DownloadPath + "/" + MemberName + "/" + v.title)

				log4go.Info("检查图片集目录: %s", downloadPath)
				if !util.CheckFileIsExist(downloadPath) {
					log4go.Info("创建图片集目录: %s", downloadPath)
					os.MkdirAll(downloadPath, 0777)
				}

				imageItem := query.Find(".original-image")
				for i := 0; i < imageItem.Length(); i++ {
					url := imageItem.Eq(i).Attr("data-src")
					title := imageItem.Eq(i).Attr("alt")
					width := imageItem.Eq(i).Attr("width")
					height := imageItem.Eq(i).Attr("height")
					ch <- &pixivImage{url: url,
						title:        title,
						width:        width,
						height:       height,
						downloadPath: downloadPath,
						href:         config.Info.HostPageUrl.Url + v.href}
				}
			}
		}
	}()

	for v := range ch {
		pictureName := v.width + "_" + v.height + "_" + v.url[strings.LastIndex(v.url, "/")+1:]
		downloadName := util.FixPathName(v.downloadPath + "/" + pictureName)

		this.waitDownloader.Add(1)
		atomic.AddInt32(&this.maxTaskNum, 1)

		log4go.Info("当前任务数:%d, 正在获取空闲任务下载图片[%s]", atomic.LoadInt32(&this.maxTaskNum), downloadName)

		for atomic.LoadInt32(&this.maxTaskNum) >= config.Info.MaxTaskNum {
			time.Sleep(time.Second)
		}

		go func() {
			defer this.waitDownloader.Done()
			defer atomic.AddInt32(&this.maxTaskNum, -1)

			if util.CheckFileIsExist(downloadName) {
				log4go.Info("图片[%s]已存在, 跳过本次下载", downloadName)
				return
			}

			log4go.Info("正在下载图片[%s]", pictureName)

			req, _ := http.NewRequest("GET", v.url, nil)
			this.setImageHeader(req, v.href)
			image, err := this.download(req)
			if err != nil {
				log4go.Warn("图片[%s]下载失败", pictureName)
			} else {
				ioutil.WriteFile(downloadName, image, 0777)
			}
		}()
	}
}