예제 #1
0
// 配置运行参数
func setConf(req map[string]interface{}) bool {
	if tn := util.Atoui(req["threadNum"]); tn == 0 {
		app.LogicApp.SetAppConf("threadNum", 1)
	} else {
		app.LogicApp.SetAppConf("threadNum", tn)
	}

	app.LogicApp.
		SetAppConf("Pausetime", [2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])}).
		SetAppConf("OutType", util.Atoa(req["output"])).
		SetAppConf("DockerCap", util.Atoui(req["dockerCap"])).
		SetAppConf("MaxPage", util.Atoi(req["maxPage"])).
		SetAppConf("Keywords", util.Atoa(req["keywords"])).
		SetAppConf("DeduplicationTarget", req["deduplicationTarget"])

	var inheritDeduplication bool
	if req["inheritDeduplication"] == "true" {
		inheritDeduplication = true
	}
	app.LogicApp.SetAppConf("InheritDeduplication", inheritDeduplication)

	if !setSpiderQueue(req) {
		return false
	}
	return true
}
예제 #2
0
func wsHandle(conn *ws.Conn) {
	defer func() {
		close(wchan)
		conn.Close()
	}()
	wchan = make(chan interface{}, 1024)

	go func(conn *ws.Conn) {
		var err error
		defer func() {
			reporter.Printf("websocket发送出错断开 (%v) !", err)
		}()
		for info := range wchan {
			if _, err = ws.JSON.Send(conn, info); err != nil {
				return
			}
		}
	}(conn)

	for {
		var req map[string]interface{}

		if err := ws.JSON.Receive(conn, &req); err != nil {
			reporter.Printf("websocket接收出错断开 (%v) !", err)
			return
		}

		reporter.Printf("Received from web: %v", req)
		wsApi[util.Atoa(req["operate"])](conn, req)
	}
}
예제 #3
0
func wsHandle(conn *ws.Conn) {
	defer func() {
		if p := recover(); p != nil {
			logs.Log.Error("%v", p)
		}
	}()
	sess, _ := globalSessions.SessionStart(nil, conn.Request())
	sessID := sess.SessionID()
	if Sc.GetConn(sessID) == nil {
		Sc.Add(sessID, conn)
	}

	defer Sc.Remove(sessID, conn)

	go func() {
		var err error
		for info := range Sc.GetWchan(sessID).wchan {
			if _, err = ws.JSON.Send(conn, info); err != nil {
				return
			}
		}
	}()

	for {
		var req map[string]interface{}

		if err := ws.JSON.Receive(conn, &req); err != nil {
			// logs.Log.Debug("websocket接收出错断开 (%v) !", err)
			return
		}

		// log.Log.Debug("Received from web: %v", req)
		wsApi[util.Atoa(req["operate"])](sessID, req)
	}
}
예제 #4
0
func wsHandle(conn *ws.Conn) {
	wchanClosed = false
	defer func() {
		// 连接断开前关闭正在运行的任务
		// if isRunning {
		// 	isRunning = false
		// 	logicApp.LogRest().Stop()
		// }
		wchanClosed = true
		close(wchan)
		conn.Close()
	}()
	wchan = make(chan interface{}, 1024)

	go func(conn *ws.Conn) {
		var err error
		for info := range wchan {
			if _, err = ws.JSON.Send(conn, info); err != nil {
				return
			}
		}
	}(conn)

	for {
		var req map[string]interface{}

		if err := ws.JSON.Receive(conn, &req); err != nil {
			// logs.Log.Debug("websocket接收出错断开 (%v) !", err)
			return
		}

		// log.Log.Debug("Received from web: %v", req)
		wsApi[util.Atoa(req["operate"])](conn, req)
	}
}
예제 #5
0
// 配置运行参数
func setConf(req map[string]interface{}) {
	if tn := util.Atoi(req["ThreadNum"]); tn == 0 {
		app.LogicApp.SetAppConf("ThreadNum", 1)
	} else {
		app.LogicApp.SetAppConf("ThreadNum", tn)
	}

	app.LogicApp.
		SetAppConf("Pausetime", int64(util.Atoi(req["Pausetime"]))).
		SetAppConf("ProxyMinute", int64(util.Atoi(req["ProxyMinute"]))).
		SetAppConf("OutType", util.Atoa(req["OutType"])).
		SetAppConf("DockerCap", util.Atoi(req["DockerCap"])).
		SetAppConf("Limit", int64(util.Atoi(req["Limit"]))).
		SetAppConf("Keyins", util.Atoa(req["Keyins"])).
		SetAppConf("SuccessInherit", req["SuccessInherit"] == "true").
		SetAppConf("FailureInherit", req["FailureInherit"] == "true")

	setSpiderQueue(req)
}
예제 #6
0
// 配置运行参数
func setConf(req map[string]interface{}) bool {
	if tn := util.Atoui(req["threadNum"]); tn == 0 {
		logicApp.SetAppConf("threadNum", 1)
	} else {
		logicApp.SetAppConf("threadNum", tn)
	}

	logicApp.
		SetAppConf("Pausetime", [2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])}).
		SetAppConf("OutType", util.Atoa(req["output"])).
		SetAppConf("DockerCap", util.Atoui(req["dockerCap"])).
		SetAppConf("MaxPage", util.Atoi(req["maxPage"])).
		SetAppConf("Keywords", util.Atoa(req["keywords"]))

	if !setSpiderQueue(req) {
		return false
	}
	return true
}
예제 #7
0
func setSpiderQueue(req map[string]interface{}) bool {
	spNames, ok := req["spiders"].([]interface{})
	if !ok {
		log.Println(" *     —— 亲,任务列表不能为空哦~")
		return false
	}
	spiders := []*spider.Spider{}
	for _, sp := range logicApp.GetAllSpiders() {
		for _, spName := range spNames {
			if util.Atoa(spName) == sp.GetName() {
				spiders = append(spiders, sp.Gost())
			}
		}
	}
	logicApp.SpiderPrepare(spiders, util.Atoa(req["keywords"]))
	if logicApp.SpiderQueueLen() == 0 {
		log.Println(" *     —— 亲,任务列表不能为空哦~")
		return false
	}
	return true
}
예제 #8
0
// 配置运行参数
func setConf(req map[string]interface{}) bool {
	if tn := util.Atoi(req["ThreadNum"]); tn == 0 {
		app.LogicApp.SetAppConf("ThreadNum", 1)
	} else {
		app.LogicApp.SetAppConf("ThreadNum", tn)
	}

	app.LogicApp.
		SetAppConf("Pausetime", int64(util.Atoi(req["Pausetime"]))).
		SetAppConf("ProxyMinute", int64(util.Atoi(req["ProxyMinute"]))).
		SetAppConf("OutType", util.Atoa(req["OutType"])).
		SetAppConf("DockerCap", util.Atoi(req["DockerCap"])).
		SetAppConf("MaxPage", int64(util.Atoi(req["MaxPage"]))).
		SetAppConf("Keywords", util.Atoa(req["Keywords"])).
		SetAppConf("SuccessInherit", req["SuccessInherit"] == "true").
		SetAppConf("FailureInherit", req["FailureInherit"] == "true")

	if !setSpiderQueue(req) {
		return false
	}
	return true
}
예제 #9
0
func setSpiderQueue(req map[string]interface{}) {
	spNames, ok := req["spiders"].([]interface{})
	if !ok {
		return
	}
	spiders := []*spider.Spider{}
	for _, sp := range app.LogicApp.GetSpiderLib() {
		for _, spName := range spNames {
			if util.Atoa(spName) == sp.GetName() {
				spiders = append(spiders, sp.Copy())
			}
		}
	}
	app.LogicApp.SpiderPrepare(spiders)
}
예제 #10
0
// 配置运行参数
func setConf(req map[string]interface{}) bool {
	if tn := util.Atoui(req["threadNum"]); tn == 0 {
		logicApp.SetThreadNum(1)
	} else {
		logicApp.SetThreadNum(tn)
	}
	logicApp.SetPausetime([2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])})
	logicApp.SetOutType(util.Atoa(req["output"]))
	logicApp.SetDockerCap(util.Atoui(req["dockerCap"])) //分段转储容器容量
	// 选填项
	logicApp.SetMaxPage(util.Atoi(req["maxPage"]))
	if !setSpiderQueue(req) {
		return false
	}
	return true
}
예제 #11
0
func setSpiderQueue(req map[string]interface{}) bool {
	spNames, ok := req["spiders"].([]interface{})
	if !ok {
		logs.Log.Warning(" *     —— 亲,任务列表不能为空哦~")
		return false
	}
	spiders := []*spider.Spider{}
	for _, sp := range app.LogicApp.GetSpiderLib() {
		for _, spName := range spNames {
			if util.Atoa(spName) == sp.GetName() {
				spiders = append(spiders, sp.Gost())
			}
		}
	}
	app.LogicApp.SpiderPrepare(spiders)
	if app.LogicApp.GetSpiderQueue().Len() == 0 {
		logs.Log.Warning(" *     —— 亲,任务列表不能为空哦~")
		return false
	}
	return true
}
예제 #12
0
func init() {
	// 设置log输出目标
	logicApp.SetLog(Log)

	// 初始化运行
	wsApi["init"] = func(conn *ws.Conn, req map[string]interface{}) {
		var mode = util.Atoi(req["mode"])
		var port = util.Atoi(req["port"])
		var master = util.Atoa(req["ip"]) //服务器(主节点)地址,不含端口
		currMode := logicApp.GetRunMode()
		if currMode == -1 {
			logicApp.Init(mode, port, master) // 运行模式初始化
			// 获取蜘蛛家族
			for _, sp := range logicApp.GetAllSpiders() {
				spiderMenu = append(spiderMenu, map[string]string{"name": sp.GetName(), "description": sp.GetDescription()})
			}
		} else if currMode != mode {
			logicApp = logicApp.ReInit(mode, port, master) // 切换运行模式
		}

		// 输出到前端的信息
		var info = map[string]interface{}{"operate": "init", "mode": mode}

		// 运行模式标题
		switch mode {
		case status.OFFLINE:
			info["title"] = config.APP_FULL_NAME + "                                                          【 运行模式 ->  单机 】"
		case status.SERVER:
			info["title"] = config.APP_FULL_NAME + "                                                          【 运行模式 ->  服务端 】"
		case status.CLIENT:
			info["title"] = config.APP_FULL_NAME + "                                                          【 运行模式 ->  客户端 】"
		}

		if mode == status.CLIENT {
			go logicApp.Run()
			goto send
		}

		// 蜘蛛家族清单
		info["spiderMenu"] = spiderMenu
		// 输出方式清单
		info["outputMenu"] = logicApp.GetOutputLib()
		// 并发协程上限
		info["threadNum"] = map[string]uint{
			"max":     999999,
			"min":     1,
			"default": defaultConfig.ThreadNum,
		}
		// 暂停时间,单位ms
		info["sleepTime"] = map[string][]uint{
			"base":    []uint{0, 100, 300, 500, 1000, 3000, 5000, 10000, 15000, 20000, 30000, 60000},
			"random":  []uint{0, 100, 300, 500, 1000, 3000, 5000, 10000, 15000, 20000, 30000, 60000},
			"default": []uint{defaultConfig.Pausetime[0], defaultConfig.Pausetime[1]},
		}
		// 分批输出的容量
		info["dockerCap"] = map[string]uint{"min": 1, "max": 5000000, "default": defaultConfig.DockerCap}

	send:
		// 写入发送通道
		wchan <- info
	}

	wsApi["run"] = func(conn *ws.Conn, req map[string]interface{}) {
		if logicApp.GetRunMode() != status.CLIENT {
			if !setConf(req) {
				wchan <- map[string]interface{}{"mode": logicApp.GetRunMode(), "status": 0}
				return
			}
		}

		if logicApp.GetRunMode() == status.OFFLINE {
			wchan <- map[string]interface{}{"operate": "run", "mode": status.OFFLINE, "status": 1}
		}

		go func() {
			logicApp.Run()
			if logicApp.GetRunMode() == status.OFFLINE {
				wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": 1}
			}
		}()
	}

	// 终止当前任务,现仅支持单机模式
	wsApi["stop"] = func(conn *ws.Conn, req map[string]interface{}) {
		if logicApp.GetRunMode() != status.OFFLINE {
			wchan <- map[string]interface{}{"operate": "stop", "mode": logicApp.GetRunMode(), "status": 0}
			return
		}
		wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": 1}
		logicApp.Stop()
	}

	// 终止当前任务,现仅支持单机模式
	wsApi["pauseRecover"] = func(conn *ws.Conn, req map[string]interface{}) {
		if logicApp.GetRunMode() != status.OFFLINE {
			return
		}
		logicApp.PauseRecover()
	}
}
예제 #13
0
func init() {

	// 初始化运行
	wsApi["refresh"] = func(sessID string, req map[string]interface{}) {
		// 写入发送通道
		Sc.Write(sessID, tplData(app.LogicApp.GetAppConf("mode").(int)), 1)
	}

	// 初始化运行
	wsApi["init"] = func(sessID string, req map[string]interface{}) {
		var mode = util.Atoi(req["mode"])
		var port = util.Atoi(req["port"])
		var master = util.Atoa(req["ip"]) //服务器(主节点)地址,不含端口
		currMode := app.LogicApp.GetAppConf("mode").(int)
		if currMode == status.UNSET {
			app.LogicApp.Init(mode, port, master, Lsc) // 运行模式初始化,设置log输出目标
		} else {
			app.LogicApp = app.LogicApp.ReInit(mode, port, master) // 切换运行模式
		}

		if mode == status.CLIENT {
			go app.LogicApp.Run()
		}

		// 写入发送通道
		Sc.Write(sessID, tplData(mode))
	}

	wsApi["run"] = func(sessID string, req map[string]interface{}) {
		if app.LogicApp.GetAppConf("mode").(int) != status.CLIENT {
			setConf(req)
		}

		if app.LogicApp.GetAppConf("mode").(int) == status.OFFLINE {
			Sc.Write(sessID, map[string]interface{}{"operate": "run"})
		}

		go func() {
			app.LogicApp.Run()
			if app.LogicApp.GetAppConf("mode").(int) == status.OFFLINE {
				Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
			}
		}()
	}

	// 终止当前任务,现仅支持单机模式
	wsApi["stop"] = func(sessID string, req map[string]interface{}) {
		if app.LogicApp.GetAppConf("mode").(int) != status.OFFLINE {
			Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
			return
		} else {
			app.LogicApp.Stop()
			Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
		}
	}

	// 任务暂停与恢复,目前仅支持单机模式
	wsApi["pauseRecover"] = func(sessID string, req map[string]interface{}) {
		if app.LogicApp.GetAppConf("mode").(int) != status.OFFLINE {
			return
		}
		app.LogicApp.PauseRecover()
		Sc.Write(sessID, map[string]interface{}{"operate": "pauseRecover"})
	}

	// 退出当前模式
	wsApi["exit"] = func(sessID string, req map[string]interface{}) {
		app.LogicApp = app.LogicApp.ReInit(status.UNSET, 0, "")
		Sc.Write(sessID, map[string]interface{}{"operate": "exit"})
	}
}
예제 #14
0
func init() {
	// 初始化运行
	wsApi["goon"] = func(conn *ws.Conn, req map[string]interface{}) {
		// 写入发送通道
		if !wchanClosed {
			wchan <- tplData(logicApp.GetAppConf("mode").(int))
		}
	}

	// 初始化运行
	wsApi["init"] = func(conn *ws.Conn, req map[string]interface{}) {
		var mode = util.Atoi(req["mode"])
		var port = util.Atoi(req["port"])
		var master = util.Atoa(req["ip"]) //服务器(主节点)地址,不含端口
		currMode := logicApp.GetAppConf("mode").(int)
		if currMode == status.UNSET {
			logicApp.Init(mode, port, master, Log) // 运行模式初始化,设置log输出目标
		} else {
			logicApp = logicApp.ReInit(mode, port, master) // 切换运行模式
		}

		if mode == status.CLIENT {
			go logicApp.Run()
		}

		// 写入发送通道
		if !wchanClosed {
			wchan <- tplData(mode)
		}
	}

	wsApi["run"] = func(conn *ws.Conn, req map[string]interface{}) {
		if logicApp.GetAppConf("mode").(int) != status.CLIENT && !wchanClosed {
			if !setConf(req) {
				wchan <- map[string]interface{}{"mode": logicApp.GetAppConf("mode").(int), "status": status.UNKNOW}
				return
			}
		}

		if logicApp.GetAppConf("mode").(int) == status.OFFLINE && !wchanClosed {
			wchan <- map[string]interface{}{"operate": "run", "mode": status.OFFLINE, "status": status.RUN}
		}

		go func() {
			isRunning = true
			logicApp.Run()
			if logicApp.GetAppConf("mode").(int) == status.OFFLINE && !wchanClosed {
				isRunning = false
				wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": status.STOP}
			}
		}()
	}

	// 终止当前任务,现仅支持单机模式
	wsApi["stop"] = func(conn *ws.Conn, req map[string]interface{}) {
		if logicApp.GetAppConf("mode").(int) != status.OFFLINE && !wchanClosed {
			wchan <- map[string]interface{}{"operate": "stop", "mode": logicApp.GetAppConf("mode").(int), "status": status.UNKNOW}
			return
		} else if !wchanClosed {
			logicApp.Stop()
			wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": status.STOP}
		}
	}

	// 终止当前任务,现仅支持单机模式
	wsApi["pauseRecover"] = func(conn *ws.Conn, req map[string]interface{}) {
		if logicApp.GetAppConf("mode").(int) != status.OFFLINE {
			return
		}
		logicApp.PauseRecover()
	}

	// 退出当前模式
	wsApi["exit"] = func(conn *ws.Conn, req map[string]interface{}) {
		logicApp = logicApp.ReInit(status.UNSET, 0, "")
	}
}