// 取得服務。
func NewService(key string) *Service {
if len(key) == 0 {
logger.Printf("%s has empty key.", alu.Caller())
}
return &Service{
Key: key,
Logger: alu.NewLogger("log"),
}
}
func NewCrawler(config *Config) *Crawler {
runtime.GOMAXPROCS(runtime.NumCPU())
c := &Crawler{
user_agent: fmt.Sprintf("%s %s", config.Email, config.URL),
Response: make(chan *Response, config.Concurrency),
queue: make([]*http.Request, 0),
config: config,
logger: alu.NewLogger("crawler.log"),
}
/*
* TODO
* The transport of client supports MaxIdleConnsPerHost property.
* It controls the concurrent connections,use few transports instead of
* clients.
*
*/
// Init clients
c.clients = make([]*client, 0)
for i := 0; i < config.Concurrency; i++ {
client := NewClient(c.logger, uint8(i), c.Response)
c.clients = append(c.clients, client)
}
currentClient := c.clients[0]
for i := 0; i < len(c.clients); i++ {
if i != len(c.clients)-1 {
c.clients[i].Next = c.clients[i+1]
} else {
c.clients[i].Next = c.clients[0]
}
}
c.currentClient = currentClient
return c
}
func NewRobots() *Robots {
return &Robots{
hosts: make(map[string]map[string]map[string]bool),
logger: alu.NewLogger("robots.log"),
}
}
// 取得組態服務。
func NewConfigService() *ConfigService {
logger = alu.NewLogger("log")
return &ConfigService{}
}