func init() {
logger = logging.NewSimpleLogger()
}
"github.com/PuerkitoBio/goquery"
"io"
"logging"
"net/http"
"net/url"
"strings"
"time"
"webcrawler/analyzer"
base "webcrawler/base"
pipeline "webcrawler/itempipeline"
sched "webcrawler/scheduler"
"webcrawler/tool"
)
// 日志记录器。
var logger logging.Logger = logging.NewSimpleLogger()
// 条目处理器。
func processItem(item base.Item) (result base.Item, err error) {
if item == nil {
return nil, errors.New("Invalid item!")
}
// 生成结果
result = make(map[string]interface{})
for k, v := range item {
result[k] = v
}
if _, ok := result["number"]; !ok {
result["number"] = len(result)
}
time.Sleep(10 * time.Millisecond)
// 创建日志记录器。
func NewLogger() logging.Logger {
return logging.NewSimpleLogger()
}