Example #1
0
func (this *Tingo) handle(url string, needException bool) (children []string) {
	children = nil
	url = this.trimUrl(url)
	if this.isNotNeedUrl(url, needException) {
		return
	}

	// 文件是否已存在
	// url = a.com/a/?id=12&id=1221, 那么genUrl=a.com/a/index.html?id=121
	genUrl := this.genUrl(url)
	if this.isExists(genUrl) {
		return
	}

	// 得到内容
	fullUrl := this.agreement + url
	if needException {
		log.Println("正在处理 `异常` " + fullUrl)
	} else {
		log.Println("正在处理 " + fullUrl)
	}

	content, err := this.getContent(fullUrl)
	if !needException && (err != nil || content == "") { // !needException防止处理异常时无限循环
		this.exceptionUrl[url] = true
		return
	}

	this.hadHandleUrl[url] = true

	ext := strings.ToLower(filepath.Ext(this.trimQueryParams(url)))
	if ext == ".css" {
		children = this.handleCSS(url, content)
		return
	}

	// 如果是js, image文件就不往下执行了
	if com.InArray(this.noChildrenFileExts, ext) {
		// 保存该文件
		if !this.writeFile(url, content) {
			return
		}
		return
	}

	if this.t == 1 {
		// 解析html里的href, src
		children = this.handleHTML(url, genUrl, content)
	}

	return

}
Example #2
0
func (this *Tingo) genFilename(url string) (string, bool) {
	urlArr := strings.Split(url, "/")
	if urlArr != nil {
		last := urlArr[len(urlArr)-1]
		ext := strings.ToLower(filepath.Ext(url)) //获取后缀
		if ext == "" {
			return this.defaultFilename, true // 需要append到url后面
		} else if com.InArray([]string{".php", ".jsp", ".asp", ".aspx"}, ext) {
			filename := filepath.Base(last)                            // a.php
			filename = com.Substr(filename, 0, len(filename)-len(ext)) // a
			return filename + ".html", false
		}
	}
	return "", true
}