Example #1
0
func DownLoad(url string) (pageContent string, err error) {
	resp, err := http.Get(url)
	if err != nil {
		logs.Error("http get error")
		return "", errors.New("http get error")
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logs.Error("read response body error")
		return "", errors.New("read response body error")
	}
	return string(body), nil
}
Example #2
0
func SaveDataToLocalDB(paperTitle string, paperContent string, source_title string, source_address string) {

	isExist, err := DataInDB(source_address)
	if err != nil {
		logs.Error("judge url in db or not error")
	} else if isExist == true {
		logs.Debug("current url is exsist in database")
		return
	}

	sql := "insert into zhiliaoyuan(id, paper_title, source_title, source_url, add_time) values("
	sql += "nextval('zhiliaoyuan_id_seq'),'" + paperTitle + "','"
	sql += source_title + "','" + source_address + "',now());"

	err = db.ExecuteSql(sql)
	if err != nil {
		logs.Error(err.Error())
	}
}