func (m *hostsManager) initByHostName(db proxy.DbHost, hostName string) error {
baseURL, err := m.resolveURL(hostName)
if err != nil {
return err
}
statusCode, body, err := m.readRobotTxt(hostName)
if err != nil {
return err
}
robot, err := robotstxt.FromStatusAndBytes(statusCode, body)
if err != nil {
return werrors.NewDetails(ErrCreateRobotsTxtFromURL, err)
}
host := proxy.NewHost(hostName, statusCode, body)
hostID, err := db.AddHost(host, baseURL)
if err == nil {
m.hosts[hostName] = hostID
m.robotsTxt[hostID] = robot.FindGroup("Googlebot")
}
return err
}
func (m *hostsManager) initByDb(db proxy.DbHost) error {
hosts, err := db.GetHosts()
if err != nil {
return err
}
for id, host := range hosts {
hostName := host.GetName()
robot, err := robotstxt.FromStatusAndBytes(host.GetRobotsTxt())
if err != nil {
return werrors.NewFields(ErrCreateRobotsTxtFromDb,
zap.String("host", hostName),
zap.String("details", err.Error()))
}
m.hosts[hostName] = id
m.robotsTxt[id] = robot.FindGroup("Googlebot")
}
return nil
}