Ejemplo n.º 1
0
func (e *Engine) makeUrl(eng interfaces.EngineInterface) string {
	uri := eng.GetUrl()

	uri = strings.Replace(uri, "[WORD]", url.QueryEscape(e.config.Text), -1)
	uri = strings.Replace(uri, "[PAGE]", strconv.Itoa(e.config.Page), -1)

	return uri
}
Ejemplo n.º 2
0
func (e *Engine) parseHTML(html string, eng interfaces.EngineInterface) []string {
	matched := eng.GetRegex().FindAllStringSubmatch(html, -1)

	var list []string
	for _, link := range matched {
		list = append(list, link[1])
	}

	return list
}
Ejemplo n.º 3
0
func (e *Engine) downloadHTML(eng interfaces.EngineInterface) string {
	uri := eng.GetUrl()
	client := &http.Client{}
	req, err := http.NewRequest("GET", uri, nil)

	if err != nil {
		log.Fatalf("Error to create a new request for [%s] -> %s\n", uri, err.Error())
	}

	req.Header.Set("User-Agent", e.config.UserAgent)
	res, err := client.Do(req)
	if err != nil {
		log.Fatalf("Error to read data of body -> %s\n", err.Error())
	}
	defer res.Body.Close()
	data, err := ioutil.ReadAll(res.Body)

	return string(data)
}