// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoui(req["threadNum"]); tn == 0 {
app.LogicApp.SetAppConf("threadNum", 1)
} else {
app.LogicApp.SetAppConf("threadNum", tn)
}
app.LogicApp.
SetAppConf("Pausetime", [2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])}).
SetAppConf("OutType", util.Atoa(req["output"])).
SetAppConf("DockerCap", util.Atoui(req["dockerCap"])).
SetAppConf("MaxPage", util.Atoi(req["maxPage"])).
SetAppConf("Keywords", util.Atoa(req["keywords"])).
SetAppConf("DeduplicationTarget", req["deduplicationTarget"])
var inheritDeduplication bool
if req["inheritDeduplication"] == "true" {
inheritDeduplication = true
}
app.LogicApp.SetAppConf("InheritDeduplication", inheritDeduplication)
if !setSpiderQueue(req) {
return false
}
return true
}
// 配置运行参数
func setConf(req map[string]interface{}) {
if tn := util.Atoi(req["ThreadNum"]); tn == 0 {
app.LogicApp.SetAppConf("ThreadNum", 1)
} else {
app.LogicApp.SetAppConf("ThreadNum", tn)
}
app.LogicApp.
SetAppConf("Pausetime", int64(util.Atoi(req["Pausetime"]))).
SetAppConf("ProxyMinute", int64(util.Atoi(req["ProxyMinute"]))).
SetAppConf("OutType", util.Atoa(req["OutType"])).
SetAppConf("DockerCap", util.Atoi(req["DockerCap"])).
SetAppConf("Limit", int64(util.Atoi(req["Limit"]))).
SetAppConf("Keyins", util.Atoa(req["Keyins"])).
SetAppConf("SuccessInherit", req["SuccessInherit"] == "true").
SetAppConf("FailureInherit", req["FailureInherit"] == "true")
setSpiderQueue(req)
}
// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoi(req["ThreadNum"]); tn == 0 {
app.LogicApp.SetAppConf("ThreadNum", 1)
} else {
app.LogicApp.SetAppConf("ThreadNum", tn)
}
app.LogicApp.
SetAppConf("Pausetime", int64(util.Atoi(req["Pausetime"]))).
SetAppConf("ProxyMinute", int64(util.Atoi(req["ProxyMinute"]))).
SetAppConf("OutType", util.Atoa(req["OutType"])).
SetAppConf("DockerCap", util.Atoi(req["DockerCap"])).
SetAppConf("MaxPage", int64(util.Atoi(req["MaxPage"]))).
SetAppConf("Keywords", util.Atoa(req["Keywords"])).
SetAppConf("SuccessInherit", req["SuccessInherit"] == "true").
SetAppConf("FailureInherit", req["FailureInherit"] == "true")
if !setSpiderQueue(req) {
return false
}
return true
}
// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoui(req["threadNum"]); tn == 0 {
logicApp.SetThreadNum(1)
} else {
logicApp.SetThreadNum(tn)
}
logicApp.SetPausetime([2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])})
logicApp.SetOutType(util.Atoa(req["output"]))
logicApp.SetDockerCap(util.Atoui(req["dockerCap"])) //分段转储容器容量
// 选填项
logicApp.SetMaxPage(util.Atoi(req["maxPage"]))
if !setSpiderQueue(req) {
return false
}
return true
}
// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoui(req["threadNum"]); tn == 0 {
logicApp.SetAppConf("threadNum", 1)
} else {
logicApp.SetAppConf("threadNum", tn)
}
logicApp.
SetAppConf("Pausetime", [2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])}).
SetAppConf("OutType", util.Atoa(req["output"])).
SetAppConf("DockerCap", util.Atoui(req["dockerCap"])).
SetAppConf("MaxPage", util.Atoi(req["maxPage"])).
SetAppConf("Keywords", util.Atoa(req["keywords"]))
if !setSpiderQueue(req) {
return false
}
return true
}
func init() {
// 设置log输出目标
logicApp.SetLog(Log)
// 初始化运行
wsApi["init"] = func(conn *ws.Conn, req map[string]interface{}) {
var mode = util.Atoi(req["mode"])
var port = util.Atoi(req["port"])
var master = util.Atoa(req["ip"]) //服务器(主节点)地址，不含端口
currMode := logicApp.GetRunMode()
if currMode == -1 {
logicApp.Init(mode, port, master) // 运行模式初始化
// 获取蜘蛛家族
for _, sp := range logicApp.GetAllSpiders() {
spiderMenu = append(spiderMenu, map[string]string{"name": sp.GetName(), "description": sp.GetDescription()})
}
} else if currMode != mode {
logicApp = logicApp.ReInit(mode, port, master) // 切换运行模式
}
// 输出到前端的信息
var info = map[string]interface{}{"operate": "init", "mode": mode}
// 运行模式标题
switch mode {
case status.OFFLINE:
info["title"] = config.APP_FULL_NAME + " 【 运行模式 -> 单机 】"
case status.SERVER:
info["title"] = config.APP_FULL_NAME + " 【 运行模式 -> 服务端 】"
case status.CLIENT:
info["title"] = config.APP_FULL_NAME + " 【 运行模式 -> 客户端 】"
}
if mode == status.CLIENT {
go logicApp.Run()
goto send
}
// 蜘蛛家族清单
info["spiderMenu"] = spiderMenu
// 输出方式清单
info["outputMenu"] = logicApp.GetOutputLib()
// 并发协程上限
info["threadNum"] = map[string]uint{
"max": 999999,
"min": 1,
"default": defaultConfig.ThreadNum,
}
// 暂停时间，单位ms
info["sleepTime"] = map[string][]uint{
"base": []uint{0, 100, 300, 500, 1000, 3000, 5000, 10000, 15000, 20000, 30000, 60000},
"random": []uint{0, 100, 300, 500, 1000, 3000, 5000, 10000, 15000, 20000, 30000, 60000},
"default": []uint{defaultConfig.Pausetime[0], defaultConfig.Pausetime[1]},
}
// 分批输出的容量
info["dockerCap"] = map[string]uint{"min": 1, "max": 5000000, "default": defaultConfig.DockerCap}
send:
// 写入发送通道
wchan <- info
}
wsApi["run"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetRunMode() != status.CLIENT {
if !setConf(req) {
wchan <- map[string]interface{}{"mode": logicApp.GetRunMode(), "status": 0}
return
}
}
if logicApp.GetRunMode() == status.OFFLINE {
wchan <- map[string]interface{}{"operate": "run", "mode": status.OFFLINE, "status": 1}
}
go func() {
logicApp.Run()
if logicApp.GetRunMode() == status.OFFLINE {
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": 1}
}
}()
}
// 终止当前任务，现仅支持单机模式
wsApi["stop"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetRunMode() != status.OFFLINE {
wchan <- map[string]interface{}{"operate": "stop", "mode": logicApp.GetRunMode(), "status": 0}
return
}
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": 1}
logicApp.Stop()
}
// 终止当前任务，现仅支持单机模式
wsApi["pauseRecover"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetRunMode() != status.OFFLINE {
return
}
logicApp.PauseRecover()
}
}
func init() {
// 初始化运行
wsApi["refresh"] = func(sessID string, req map[string]interface{}) {
// 写入发送通道
Sc.Write(sessID, tplData(app.LogicApp.GetAppConf("mode").(int)), 1)
}
// 初始化运行
wsApi["init"] = func(sessID string, req map[string]interface{}) {
var mode = util.Atoi(req["mode"])
var port = util.Atoi(req["port"])
var master = util.Atoa(req["ip"]) //服务器(主节点)地址，不含端口
currMode := app.LogicApp.GetAppConf("mode").(int)
if currMode == status.UNSET {
app.LogicApp.Init(mode, port, master, Lsc) // 运行模式初始化，设置log输出目标
} else {
app.LogicApp = app.LogicApp.ReInit(mode, port, master) // 切换运行模式
}
if mode == status.CLIENT {
go app.LogicApp.Run()
}
// 写入发送通道
Sc.Write(sessID, tplData(mode))
}
wsApi["run"] = func(sessID string, req map[string]interface{}) {
if app.LogicApp.GetAppConf("mode").(int) != status.CLIENT {
setConf(req)
}
if app.LogicApp.GetAppConf("mode").(int) == status.OFFLINE {
Sc.Write(sessID, map[string]interface{}{"operate": "run"})
}
go func() {
app.LogicApp.Run()
if app.LogicApp.GetAppConf("mode").(int) == status.OFFLINE {
Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
}
}()
}
// 终止当前任务，现仅支持单机模式
wsApi["stop"] = func(sessID string, req map[string]interface{}) {
if app.LogicApp.GetAppConf("mode").(int) != status.OFFLINE {
Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
return
} else {
app.LogicApp.Stop()
Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
}
}
// 任务暂停与恢复，目前仅支持单机模式
wsApi["pauseRecover"] = func(sessID string, req map[string]interface{}) {
if app.LogicApp.GetAppConf("mode").(int) != status.OFFLINE {
return
}
app.LogicApp.PauseRecover()
Sc.Write(sessID, map[string]interface{}{"operate": "pauseRecover"})
}
// 退出当前模式
wsApi["exit"] = func(sessID string, req map[string]interface{}) {
app.LogicApp = app.LogicApp.ReInit(status.UNSET, 0, "")
Sc.Write(sessID, map[string]interface{}{"operate": "exit"})
}
}
func init() {
// 初始化运行
wsApi["goon"] = func(conn *ws.Conn, req map[string]interface{}) {
// 写入发送通道
if !wchanClosed {
wchan <- tplData(logicApp.GetAppConf("mode").(int))
}
}
// 初始化运行
wsApi["init"] = func(conn *ws.Conn, req map[string]interface{}) {
var mode = util.Atoi(req["mode"])
var port = util.Atoi(req["port"])
var master = util.Atoa(req["ip"]) //服务器(主节点)地址，不含端口
currMode := logicApp.GetAppConf("mode").(int)
if currMode == status.UNSET {
logicApp.Init(mode, port, master, Log) // 运行模式初始化，设置log输出目标
} else {
logicApp = logicApp.ReInit(mode, port, master) // 切换运行模式
}
if mode == status.CLIENT {
go logicApp.Run()
}
// 写入发送通道
if !wchanClosed {
wchan <- tplData(mode)
}
}
wsApi["run"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetAppConf("mode").(int) != status.CLIENT && !wchanClosed {
if !setConf(req) {
wchan <- map[string]interface{}{"mode": logicApp.GetAppConf("mode").(int), "status": status.UNKNOW}
return
}
}
if logicApp.GetAppConf("mode").(int) == status.OFFLINE && !wchanClosed {
wchan <- map[string]interface{}{"operate": "run", "mode": status.OFFLINE, "status": status.RUN}
}
go func() {
isRunning = true
logicApp.Run()
if logicApp.GetAppConf("mode").(int) == status.OFFLINE && !wchanClosed {
isRunning = false
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": status.STOP}
}
}()
}
// 终止当前任务，现仅支持单机模式
wsApi["stop"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetAppConf("mode").(int) != status.OFFLINE && !wchanClosed {
wchan <- map[string]interface{}{"operate": "stop", "mode": logicApp.GetAppConf("mode").(int), "status": status.UNKNOW}
return
} else if !wchanClosed {
logicApp.Stop()
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": status.STOP}
}
}
// 终止当前任务，现仅支持单机模式
wsApi["pauseRecover"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetAppConf("mode").(int) != status.OFFLINE {
return
}
logicApp.PauseRecover()
}
// 退出当前模式
wsApi["exit"] = func(conn *ws.Conn, req map[string]interface{}) {
logicApp = logicApp.ReInit(status.UNSET, 0, "")
}
}