func main() {
fmt.Println("Get index...")
s, statusCode := core.Get("http://www.yifan100.com/dir/15136/")
if statusCode != 200 {
return
}
index, _ := findIndex(s)
fmt.Println(`Get contents and write to file...`)
for _, itm := range index {
fmt.Printf("Get content %s from %s and write to file.\n", itm.title, itm.url)
fileName := fmt.Sprintf("c:\\%s.txt", itm.title)
content := readContent(itm.url)
ioutil.WriteFile(fileName, []byte(content), 0644)
fmt.Printf("Finish writing to %s.\n", fileName)
}
}
func readContent(url string) (content string) {
raw, statusCode := core.Get(url)
if statusCode != 200 {
fmt.Print("Fail to get the raw data from", url, "\n")
return
}
match := ptnContentRough.FindStringSubmatch(raw)
if match != nil {
content = match[1]
} else {
return
}
content = ptnBrTag.ReplaceAllString(content, "\r\n")
content = ptnHTMLTag.ReplaceAllString(content, "")
content = ptnSpace.ReplaceAllString(content, "")
return
}