func newMatrix(spiderName, spiderSubName string, maxPage int64) *Matrix { matrix := &Matrix{ spiderName: spiderName, maxPage: maxPage, reqs: make(map[int][]*request.Request), priorities: []int{}, history: history.New(spiderName, spiderSubName), tempHistory: make(map[string]bool), failures: make(map[string]*request.Request), } if cache.Task.Mode != status.SERVER { matrix.history.ReadSuccess(cache.Task.OutType, cache.Task.SuccessInherit) matrix.history.ReadFailure(cache.Task.OutType, cache.Task.FailureInherit) matrix.setFailures(matrix.history.PullFailure()) } return matrix }
count chan bool // 运行状态 status int // 全局代理IP proxy *proxy.Proxy // 标记是否使用代理IP useProxy bool // 全局历史记录 history history.Historier // 全局读写锁 sync.RWMutex } // 定义全局调度 var sdl = &scheduler{ history: history.New(), status: status.RUN, count: make(chan bool, cache.Task.ThreadNum), } func init() { go func() { // 等待main中的init()执行完毕 cache.WaitInit(0) sdl.proxy = proxy.New() }() } func Init() { for sdl.proxy == nil { runtime.Gosched() }