Пример #1
0
// 取得服務。
func NewService(key string) *Service {
	if len(key) == 0 {
		logger.Printf("%s has empty key.", alu.Caller())
	}

	return &Service{
		Key:    key,
		Logger: alu.NewLogger("log"),
	}
}
Пример #2
0
func NewCrawler(config *Config) *Crawler {
	runtime.GOMAXPROCS(runtime.NumCPU())

	c := &Crawler{
		user_agent: fmt.Sprintf("%s %s", config.Email, config.URL),
		Response:   make(chan *Response, config.Concurrency),
		queue:      make([]*http.Request, 0),
		config:     config,
		logger:     alu.NewLogger("crawler.log"),
	}

	/*
	 * TODO
	 * The transport of client supports MaxIdleConnsPerHost property.
	 * It controls the concurrent connections,use few transports instead of
	 * clients.
	 *
	 */
	// Init clients
	c.clients = make([]*client, 0)
	for i := 0; i < config.Concurrency; i++ {
		client := NewClient(c.logger, uint8(i), c.Response)
		c.clients = append(c.clients, client)
	}

	currentClient := c.clients[0]
	for i := 0; i < len(c.clients); i++ {
		if i != len(c.clients)-1 {
			c.clients[i].Next = c.clients[i+1]
		} else {
			c.clients[i].Next = c.clients[0]
		}
	}
	c.currentClient = currentClient

	return c
}
Пример #3
0
func NewRobots() *Robots {
	return &Robots{
		hosts:  make(map[string]map[string]map[string]bool),
		logger: alu.NewLogger("robots.log"),
	}
}
Пример #4
0
// 取得組態服務。
func NewConfigService() *ConfigService {
	logger = alu.NewLogger("log")
	return &ConfigService{}
}