Пример #1
0
func (m *hostsManager) initByHostName(db proxy.DbHost, hostName string) error {
	baseURL, err := m.resolveURL(hostName)
	if err != nil {
		return err
	}

	statusCode, body, err := m.readRobotTxt(hostName)
	if err != nil {
		return err
	}

	robot, err := robotstxt.FromStatusAndBytes(statusCode, body)
	if err != nil {
		return werrors.NewDetails(ErrCreateRobotsTxtFromURL, err)
	}

	host := proxy.NewHost(hostName, statusCode, body)
	hostID, err := db.AddHost(host, baseURL)

	if err == nil {
		m.hosts[hostName] = hostID
		m.robotsTxt[hostID] = robot.FindGroup("Googlebot")
	}

	return err
}
Пример #2
0
func (m *hostsManager) initByDb(db proxy.DbHost) error {
	hosts, err := db.GetHosts()
	if err != nil {
		return err
	}
	for id, host := range hosts {
		hostName := host.GetName()
		robot, err := robotstxt.FromStatusAndBytes(host.GetRobotsTxt())
		if err != nil {
			return werrors.NewFields(ErrCreateRobotsTxtFromDb,
				zap.String("host", hostName),
				zap.String("details", err.Error()))
		}
		m.hosts[hostName] = id
		m.robotsTxt[id] = robot.FindGroup("Googlebot")
	}

	return nil
}