// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoui(req["threadNum"]); tn == 0 {
app.LogicApp.SetAppConf("threadNum", 1)
} else {
app.LogicApp.SetAppConf("threadNum", tn)
}
app.LogicApp.
SetAppConf("Pausetime", [2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])}).
SetAppConf("OutType", util.Atoa(req["output"])).
SetAppConf("DockerCap", util.Atoui(req["dockerCap"])).
SetAppConf("MaxPage", util.Atoi(req["maxPage"])).
SetAppConf("Keywords", util.Atoa(req["keywords"])).
SetAppConf("DeduplicationTarget", req["deduplicationTarget"])
var inheritDeduplication bool
if req["inheritDeduplication"] == "true" {
inheritDeduplication = true
}
app.LogicApp.SetAppConf("InheritDeduplication", inheritDeduplication)
if !setSpiderQueue(req) {
return false
}
return true
}
func wsHandle(conn *ws.Conn) {
defer func() {
close(wchan)
conn.Close()
}()
wchan = make(chan interface{}, 1024)
go func(conn *ws.Conn) {
var err error
defer func() {
reporter.Printf("websocket发送出错断开 (%v) !", err)
}()
for info := range wchan {
if _, err = ws.JSON.Send(conn, info); err != nil {
return
}
}
}(conn)
for {
var req map[string]interface{}
if err := ws.JSON.Receive(conn, &req); err != nil {
reporter.Printf("websocket接收出错断开 (%v) !", err)
return
}
reporter.Printf("Received from web: %v", req)
wsApi[util.Atoa(req["operate"])](conn, req)
}
}
func wsHandle(conn *ws.Conn) {
defer func() {
if p := recover(); p != nil {
logs.Log.Error("%v", p)
}
}()
sess, _ := globalSessions.SessionStart(nil, conn.Request())
sessID := sess.SessionID()
if Sc.GetConn(sessID) == nil {
Sc.Add(sessID, conn)
}
defer Sc.Remove(sessID, conn)
go func() {
var err error
for info := range Sc.GetWchan(sessID).wchan {
if _, err = ws.JSON.Send(conn, info); err != nil {
return
}
}
}()
for {
var req map[string]interface{}
if err := ws.JSON.Receive(conn, &req); err != nil {
// logs.Log.Debug("websocket接收出错断开 (%v) !", err)
return
}
// log.Log.Debug("Received from web: %v", req)
wsApi[util.Atoa(req["operate"])](sessID, req)
}
}
func wsHandle(conn *ws.Conn) {
wchanClosed = false
defer func() {
// 连接断开前关闭正在运行的任务
// if isRunning {
// isRunning = false
// logicApp.LogRest().Stop()
// }
wchanClosed = true
close(wchan)
conn.Close()
}()
wchan = make(chan interface{}, 1024)
go func(conn *ws.Conn) {
var err error
for info := range wchan {
if _, err = ws.JSON.Send(conn, info); err != nil {
return
}
}
}(conn)
for {
var req map[string]interface{}
if err := ws.JSON.Receive(conn, &req); err != nil {
// logs.Log.Debug("websocket接收出错断开 (%v) !", err)
return
}
// log.Log.Debug("Received from web: %v", req)
wsApi[util.Atoa(req["operate"])](conn, req)
}
}
// 配置运行参数
func setConf(req map[string]interface{}) {
if tn := util.Atoi(req["ThreadNum"]); tn == 0 {
app.LogicApp.SetAppConf("ThreadNum", 1)
} else {
app.LogicApp.SetAppConf("ThreadNum", tn)
}
app.LogicApp.
SetAppConf("Pausetime", int64(util.Atoi(req["Pausetime"]))).
SetAppConf("ProxyMinute", int64(util.Atoi(req["ProxyMinute"]))).
SetAppConf("OutType", util.Atoa(req["OutType"])).
SetAppConf("DockerCap", util.Atoi(req["DockerCap"])).
SetAppConf("Limit", int64(util.Atoi(req["Limit"]))).
SetAppConf("Keyins", util.Atoa(req["Keyins"])).
SetAppConf("SuccessInherit", req["SuccessInherit"] == "true").
SetAppConf("FailureInherit", req["FailureInherit"] == "true")
setSpiderQueue(req)
}
// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoui(req["threadNum"]); tn == 0 {
logicApp.SetAppConf("threadNum", 1)
} else {
logicApp.SetAppConf("threadNum", tn)
}
logicApp.
SetAppConf("Pausetime", [2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])}).
SetAppConf("OutType", util.Atoa(req["output"])).
SetAppConf("DockerCap", util.Atoui(req["dockerCap"])).
SetAppConf("MaxPage", util.Atoi(req["maxPage"])).
SetAppConf("Keywords", util.Atoa(req["keywords"]))
if !setSpiderQueue(req) {
return false
}
return true
}
func setSpiderQueue(req map[string]interface{}) bool {
spNames, ok := req["spiders"].([]interface{})
if !ok {
log.Println(" * —— 亲，任务列表不能为空哦~")
return false
}
spiders := []*spider.Spider{}
for _, sp := range logicApp.GetAllSpiders() {
for _, spName := range spNames {
if util.Atoa(spName) == sp.GetName() {
spiders = append(spiders, sp.Gost())
}
}
}
logicApp.SpiderPrepare(spiders, util.Atoa(req["keywords"]))
if logicApp.SpiderQueueLen() == 0 {
log.Println(" * —— 亲，任务列表不能为空哦~")
return false
}
return true
}
// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoi(req["ThreadNum"]); tn == 0 {
app.LogicApp.SetAppConf("ThreadNum", 1)
} else {
app.LogicApp.SetAppConf("ThreadNum", tn)
}
app.LogicApp.
SetAppConf("Pausetime", int64(util.Atoi(req["Pausetime"]))).
SetAppConf("ProxyMinute", int64(util.Atoi(req["ProxyMinute"]))).
SetAppConf("OutType", util.Atoa(req["OutType"])).
SetAppConf("DockerCap", util.Atoi(req["DockerCap"])).
SetAppConf("MaxPage", int64(util.Atoi(req["MaxPage"]))).
SetAppConf("Keywords", util.Atoa(req["Keywords"])).
SetAppConf("SuccessInherit", req["SuccessInherit"] == "true").
SetAppConf("FailureInherit", req["FailureInherit"] == "true")
if !setSpiderQueue(req) {
return false
}
return true
}
func setSpiderQueue(req map[string]interface{}) {
spNames, ok := req["spiders"].([]interface{})
if !ok {
return
}
spiders := []*spider.Spider{}
for _, sp := range app.LogicApp.GetSpiderLib() {
for _, spName := range spNames {
if util.Atoa(spName) == sp.GetName() {
spiders = append(spiders, sp.Copy())
}
}
}
app.LogicApp.SpiderPrepare(spiders)
}
// 配置运行参数
func setConf(req map[string]interface{}) bool {
if tn := util.Atoui(req["threadNum"]); tn == 0 {
logicApp.SetThreadNum(1)
} else {
logicApp.SetThreadNum(tn)
}
logicApp.SetPausetime([2]uint{(util.Atoui(req["baseSleeptime"])), util.Atoui(req["randomSleepPeriod"])})
logicApp.SetOutType(util.Atoa(req["output"]))
logicApp.SetDockerCap(util.Atoui(req["dockerCap"])) //分段转储容器容量
// 选填项
logicApp.SetMaxPage(util.Atoi(req["maxPage"]))
if !setSpiderQueue(req) {
return false
}
return true
}
func setSpiderQueue(req map[string]interface{}) bool {
spNames, ok := req["spiders"].([]interface{})
if !ok {
logs.Log.Warning(" * —— 亲，任务列表不能为空哦~")
return false
}
spiders := []*spider.Spider{}
for _, sp := range app.LogicApp.GetSpiderLib() {
for _, spName := range spNames {
if util.Atoa(spName) == sp.GetName() {
spiders = append(spiders, sp.Gost())
}
}
}
app.LogicApp.SpiderPrepare(spiders)
if app.LogicApp.GetSpiderQueue().Len() == 0 {
logs.Log.Warning(" * —— 亲，任务列表不能为空哦~")
return false
}
return true
}
func init() {
// 设置log输出目标
logicApp.SetLog(Log)
// 初始化运行
wsApi["init"] = func(conn *ws.Conn, req map[string]interface{}) {
var mode = util.Atoi(req["mode"])
var port = util.Atoi(req["port"])
var master = util.Atoa(req["ip"]) //服务器(主节点)地址，不含端口
currMode := logicApp.GetRunMode()
if currMode == -1 {
logicApp.Init(mode, port, master) // 运行模式初始化
// 获取蜘蛛家族
for _, sp := range logicApp.GetAllSpiders() {
spiderMenu = append(spiderMenu, map[string]string{"name": sp.GetName(), "description": sp.GetDescription()})
}
} else if currMode != mode {
logicApp = logicApp.ReInit(mode, port, master) // 切换运行模式
}
// 输出到前端的信息
var info = map[string]interface{}{"operate": "init", "mode": mode}
// 运行模式标题
switch mode {
case status.OFFLINE:
info["title"] = config.APP_FULL_NAME + " 【 运行模式 -> 单机 】"
case status.SERVER:
info["title"] = config.APP_FULL_NAME + " 【 运行模式 -> 服务端 】"
case status.CLIENT:
info["title"] = config.APP_FULL_NAME + " 【 运行模式 -> 客户端 】"
}
if mode == status.CLIENT {
go logicApp.Run()
goto send
}
// 蜘蛛家族清单
info["spiderMenu"] = spiderMenu
// 输出方式清单
info["outputMenu"] = logicApp.GetOutputLib()
// 并发协程上限
info["threadNum"] = map[string]uint{
"max": 999999,
"min": 1,
"default": defaultConfig.ThreadNum,
}
// 暂停时间，单位ms
info["sleepTime"] = map[string][]uint{
"base": []uint{0, 100, 300, 500, 1000, 3000, 5000, 10000, 15000, 20000, 30000, 60000},
"random": []uint{0, 100, 300, 500, 1000, 3000, 5000, 10000, 15000, 20000, 30000, 60000},
"default": []uint{defaultConfig.Pausetime[0], defaultConfig.Pausetime[1]},
}
// 分批输出的容量
info["dockerCap"] = map[string]uint{"min": 1, "max": 5000000, "default": defaultConfig.DockerCap}
send:
// 写入发送通道
wchan <- info
}
wsApi["run"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetRunMode() != status.CLIENT {
if !setConf(req) {
wchan <- map[string]interface{}{"mode": logicApp.GetRunMode(), "status": 0}
return
}
}
if logicApp.GetRunMode() == status.OFFLINE {
wchan <- map[string]interface{}{"operate": "run", "mode": status.OFFLINE, "status": 1}
}
go func() {
logicApp.Run()
if logicApp.GetRunMode() == status.OFFLINE {
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": 1}
}
}()
}
// 终止当前任务，现仅支持单机模式
wsApi["stop"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetRunMode() != status.OFFLINE {
wchan <- map[string]interface{}{"operate": "stop", "mode": logicApp.GetRunMode(), "status": 0}
return
}
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": 1}
logicApp.Stop()
}
// 终止当前任务，现仅支持单机模式
wsApi["pauseRecover"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetRunMode() != status.OFFLINE {
return
}
logicApp.PauseRecover()
}
}
func init() {
// 初始化运行
wsApi["refresh"] = func(sessID string, req map[string]interface{}) {
// 写入发送通道
Sc.Write(sessID, tplData(app.LogicApp.GetAppConf("mode").(int)), 1)
}
// 初始化运行
wsApi["init"] = func(sessID string, req map[string]interface{}) {
var mode = util.Atoi(req["mode"])
var port = util.Atoi(req["port"])
var master = util.Atoa(req["ip"]) //服务器(主节点)地址，不含端口
currMode := app.LogicApp.GetAppConf("mode").(int)
if currMode == status.UNSET {
app.LogicApp.Init(mode, port, master, Lsc) // 运行模式初始化，设置log输出目标
} else {
app.LogicApp = app.LogicApp.ReInit(mode, port, master) // 切换运行模式
}
if mode == status.CLIENT {
go app.LogicApp.Run()
}
// 写入发送通道
Sc.Write(sessID, tplData(mode))
}
wsApi["run"] = func(sessID string, req map[string]interface{}) {
if app.LogicApp.GetAppConf("mode").(int) != status.CLIENT {
setConf(req)
}
if app.LogicApp.GetAppConf("mode").(int) == status.OFFLINE {
Sc.Write(sessID, map[string]interface{}{"operate": "run"})
}
go func() {
app.LogicApp.Run()
if app.LogicApp.GetAppConf("mode").(int) == status.OFFLINE {
Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
}
}()
}
// 终止当前任务，现仅支持单机模式
wsApi["stop"] = func(sessID string, req map[string]interface{}) {
if app.LogicApp.GetAppConf("mode").(int) != status.OFFLINE {
Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
return
} else {
app.LogicApp.Stop()
Sc.Write(sessID, map[string]interface{}{"operate": "stop"})
}
}
// 任务暂停与恢复，目前仅支持单机模式
wsApi["pauseRecover"] = func(sessID string, req map[string]interface{}) {
if app.LogicApp.GetAppConf("mode").(int) != status.OFFLINE {
return
}
app.LogicApp.PauseRecover()
Sc.Write(sessID, map[string]interface{}{"operate": "pauseRecover"})
}
// 退出当前模式
wsApi["exit"] = func(sessID string, req map[string]interface{}) {
app.LogicApp = app.LogicApp.ReInit(status.UNSET, 0, "")
Sc.Write(sessID, map[string]interface{}{"operate": "exit"})
}
}
func init() {
// 初始化运行
wsApi["goon"] = func(conn *ws.Conn, req map[string]interface{}) {
// 写入发送通道
if !wchanClosed {
wchan <- tplData(logicApp.GetAppConf("mode").(int))
}
}
// 初始化运行
wsApi["init"] = func(conn *ws.Conn, req map[string]interface{}) {
var mode = util.Atoi(req["mode"])
var port = util.Atoi(req["port"])
var master = util.Atoa(req["ip"]) //服务器(主节点)地址，不含端口
currMode := logicApp.GetAppConf("mode").(int)
if currMode == status.UNSET {
logicApp.Init(mode, port, master, Log) // 运行模式初始化，设置log输出目标
} else {
logicApp = logicApp.ReInit(mode, port, master) // 切换运行模式
}
if mode == status.CLIENT {
go logicApp.Run()
}
// 写入发送通道
if !wchanClosed {
wchan <- tplData(mode)
}
}
wsApi["run"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetAppConf("mode").(int) != status.CLIENT && !wchanClosed {
if !setConf(req) {
wchan <- map[string]interface{}{"mode": logicApp.GetAppConf("mode").(int), "status": status.UNKNOW}
return
}
}
if logicApp.GetAppConf("mode").(int) == status.OFFLINE && !wchanClosed {
wchan <- map[string]interface{}{"operate": "run", "mode": status.OFFLINE, "status": status.RUN}
}
go func() {
isRunning = true
logicApp.Run()
if logicApp.GetAppConf("mode").(int) == status.OFFLINE && !wchanClosed {
isRunning = false
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": status.STOP}
}
}()
}
// 终止当前任务，现仅支持单机模式
wsApi["stop"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetAppConf("mode").(int) != status.OFFLINE && !wchanClosed {
wchan <- map[string]interface{}{"operate": "stop", "mode": logicApp.GetAppConf("mode").(int), "status": status.UNKNOW}
return
} else if !wchanClosed {
logicApp.Stop()
wchan <- map[string]interface{}{"operate": "stop", "mode": status.OFFLINE, "status": status.STOP}
}
}
// 终止当前任务，现仅支持单机模式
wsApi["pauseRecover"] = func(conn *ws.Conn, req map[string]interface{}) {
if logicApp.GetAppConf("mode").(int) != status.OFFLINE {
return
}
logicApp.PauseRecover()
}
// 退出当前模式
wsApi["exit"] = func(conn *ws.Conn, req map[string]interface{}) {
logicApp = logicApp.ReInit(status.UNSET, 0, "")
}
}