func newMatrix(spiderName, spiderSubName string, maxPage int64) *Matrix {
matrix := &Matrix{
spiderName: spiderName,
maxPage: maxPage,
reqs: make(map[int][]*request.Request),
priorities: []int{},
history: history.New(spiderName, spiderSubName),
tempHistory: make(map[string]bool),
failures: make(map[string]*request.Request),
}
if cache.Task.Mode != status.SERVER {
matrix.history.ReadSuccess(cache.Task.OutType, cache.Task.SuccessInherit)
matrix.history.ReadFailure(cache.Task.OutType, cache.Task.FailureInherit)
matrix.setFailures(matrix.history.PullFailure())
}
return matrix
}
count chan bool
// 运行状态
status int
// 全局代理IP
proxy *proxy.Proxy
// 标记是否使用代理IP
useProxy bool
// 全局历史记录
history history.Historier
// 全局读写锁
sync.RWMutex
}
// 定义全局调度
var sdl = &scheduler{
history: history.New(),
status: status.RUN,
count: make(chan bool, cache.Task.ThreadNum),
}
func init() {
go func() {
// 等待main中的init()执行完毕
cache.WaitInit(0)
sdl.proxy = proxy.New()
}()
}
func Init() {
for sdl.proxy == nil {
runtime.Gosched()
}