Exemple #1
0
func appInit() {
	app.LogicApp = app.New().SetLog(Lsc).AsyncLog(true)
	spiderMenu = func() (spmenu []map[string]string) {
		// 获取蜘蛛家族
		for _, sp := range app.LogicApp.GetSpiderLib() {
			spmenu = append(spmenu, map[string]string{"name": sp.GetName(), "description": sp.GetDescription()})
		}
		return spmenu
	}()
}
Exemple #2
0
// [spider frame (golang)] Pholcus(幽灵蛛)是一款纯Go语言编写的高并发、分布式、重量级爬虫软件,支持单机、服务端、客户端三种运行模式,拥有Web、GUI、命令行三种操作界面;规则简单灵活、批量任务并发、输出方式丰富(mysql/mongodb/csv/excel等)、有大量Demo共享;同时她还支持横纵向两种抓取模式,支持模拟登录和任务暂停、取消等一系列高级功能;
//(官方QQ群:Go大数据 42731170,欢迎加入我们的讨论)。
// GUI界面版。
package gui

import (
	"log"
	"runtime"

	"github.com/henrylee2cn/pholcus/app"
	"github.com/henrylee2cn/pholcus/app/spider"
	"github.com/lxn/walk"
)

var LogicApp = app.New().AsyncLog(true)

func Run() {
	// 开启最大核心数运行
	runtime.GOMAXPROCS(runtime.NumCPU())
	runmodeWindow()
}

func Init() {
	LogicApp.Init(Input.RunMode, Input.Port, Input.Master)
}

func SetTaskConf() {
	// 纠正协程数
	if Input.ThreadNum == 0 {
		Input.ThreadNum = 1
	}
	}(conn)

	for {
		var req map[string]interface{}

		if err := ws.JSON.Receive(conn, &req); err != nil {
			reporter.Printf("websocket接收出错断开 (%v) !", err)
			return
		}

		reporter.Printf("Received from web: %v", req)
		wsApi[util.Atoa(req["operate"])](conn, req)
	}
}

var logicApp = app.New()
var spiderMenu = make([]map[string]string, 0)
var wsApi = map[string]func(*ws.Conn, map[string]interface{}){}

func init() {
	// 设置log输出目标
	logicApp.SetLog(Log)

	// 初始化运行
	wsApi["init"] = func(conn *ws.Conn, req map[string]interface{}) {
		var mode = util.Atoi(req["mode"])
		var port = util.Atoi(req["port"])
		var master = util.Atoa(req["ip"]) //服务器(主节点)地址,不含端口
		currMode := logicApp.GetRunMode()
		if currMode == -1 {
			logicApp.Init(mode, port, master) // 运行模式初始化
Exemple #4
0
import (
	"flag"
	// "bufio"
	"log"
	// "os"
	// "fmt"
	"runtime"
	"strconv"
	"strings"

	"github.com/henrylee2cn/pholcus/app"
	"github.com/henrylee2cn/pholcus/spider"
)

var LogicApp = app.New()

func Run() {
	// 开启最大核心数运行
	runtime.GOMAXPROCS(runtime.NumCPU())

	// //运行模式
	// modeflag := flag.Int("运行模式", 0, "*运行模式: [0] 单机    [1] 服务端    [2] 客户端\r\n")
	LogicApp.Init(1, 0, "")

	// //端口号,非单机模式填写
	// portflag := flag.Int("端口号", 0, "端口号: 只填写数字即可,不含冒号\r\n")

	// //主节点ip,客户端模式填写
	// masterflag := flag.String("服务端IP", "127.0.0.1", "主节点IP: 服务端IP地址,不含端口\r\n")
import (
	"github.com/henrylee2cn/pholcus/app"
	"github.com/henrylee2cn/pholcus/app/spider"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/status"
	ws "github.com/henrylee2cn/websocket.google"
)

var (
	wchan       chan interface{}
	wchanClosed bool
	isRunning   bool

	logicApp   = app.New().SetLog(Log).AsyncLog(true)
	spiderMenu = func() (spmenu []map[string]string) {
		// 获取蜘蛛家族
		for _, sp := range logicApp.GetSpiderLib() {
			spmenu = append(spmenu, map[string]string{"name": sp.GetName(), "description": sp.GetDescription()})
		}
		return spmenu
	}()

	wsApi = map[string]func(*ws.Conn, map[string]interface{}){}
)

func wsHandle(conn *ws.Conn) {
	wchanClosed = false
	defer func() {
		// 连接断开前关闭正在运行的任务
Exemple #6
0
import (
	"flag"
	// "bufio"
	// "os"
	// "fmt"
	"runtime"
	"strconv"
	"strings"

	"github.com/henrylee2cn/pholcus/app"
	"github.com/henrylee2cn/pholcus/app/spider"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/status"
)

var LogicApp = app.New().Init(status.OFFLINE, 0, "")

func Run() {
	// 开启最大核心数运行
	runtime.GOMAXPROCS(runtime.NumCPU())

	// //运行模式
	// modeflag := flag.Int("运行模式", 0, "*运行模式: [0] 单机    [1] 服务端    [2] 客户端\r\n")

	// //端口号,非单机模式填写
	// portflag := flag.Int("端口号", 0, "端口号: 只填写数字即可,不含冒号\r\n")

	// //主节点ip,客户端模式填写
	// masterflag := flag.String("服务端IP", "127.0.0.1", "主节点IP: 服务端IP地址,不含端口\r\n")

	// 蜘蛛列表