Exemplo n.º 1
0
func GetMP3URL(data []byte) string {

	type Result struct {
		XMLName xml.Name `xml:"result"`
		Count   int      `xml:"count"`
		Encode  []string `xml:"url>encode"`
		Decode  []string `xml:"url>decode"`
		Typed   []string `xml:"url>type"`
	}
	var v Result

	out := make([]byte, len(data))

	iconv.Convert(data, out, "gb2312", "utf-8")
	fmt.Printf("%v\n", string(out))
	fString := strings.Replace(string(out), "gb2312", "utf-8", 1)
	fmt.Printf("%v\n", fString)
	err := xml.Unmarshal([]byte(fString), &v)
	if err != nil {
		fmt.Printf("error: %v", err)
		return "error"
	}
	if v.Count < 0 {
		return "没有找到"
	}

	for i, value := range v.Typed {
		if value == "8" {
			return v.Encode[i][0:strings.LastIndex(v.Encode[i], "/")+1] + v.Decode[i]
		}
	}
	return "没有mp3格式"

}
Exemplo n.º 2
0
func ConvertEncodingAuto(src []byte) ([]byte, error) {
	out := make([]byte, 64*1024)
	for _, encoding := range []string{"CP932", "utf-8"} {
		_, bytes, err := iconv.Convert(src, out, encoding, "utf-8")
		if err == nil {
			return out[0:bytes], nil
		}
	}
	return nil, errors.New("cannot convert string")
}
Exemplo n.º 3
0
func json2Filef(url string, save_to string, maxPage string) (int, error) {
	//fmt.Println(url, maxPage)
	for {
		ll = append(ll, time.Now())
		statusCode, body, err := fasthttp.Get(nil, url)
		if err != nil {
			return 0, err
		}
		if 200 != statusCode {
			return 0, fmt.Errorf("!200")
		}

		anti_spider := "smPolicy=tmallrateweb-rate-anti_Spider-checklogin"
		idx_anti_spider := bytes.Index(body, []byte(anti_spider))
		paginator_empty := "\"paginator\":\"\""
		idx_paginator_empty := bytes.Index(body, []byte(paginator_empty))
		if idx_anti_spider >= 0 || idx_paginator_empty >= 0 {
			time.Sleep(5 * time.Second)
			continue
		}

		out := make([]byte, len(body)*2)
		_, bytesWritten, err := iconv.Convert(body, out, "gbk", "utf-8")

		//fmt.Println(bytesRead, bytesWritten, err)
		idx := bytes.IndexByte(out, '(')
		if idx < 0 || bytesWritten < idx+1 {
			return 0, fmt.Errorf("idx error")
		}
		out = out[idx+1 : bytesWritten]
		//fmt.Println(string(out))
		idx_end := bytes.LastIndexByte(out, ')')
		if idx_end < 0 {
			return 0, fmt.Errorf("idx_end<0, )")
		}
		out = out[:idx_end]
		fmt.Println(save_to)
		ioutil.WriteFile(save_to, out, 0666)

		time.Sleep(1 * time.Second)
		if len(maxPage) > 0 {
			return strconv.Atoi(pickString(body, maxPage, ":", ","))
		} else {
			return 0, nil
		}
	}
}
Exemplo n.º 4
0
func json2Filef(t *ShopType, sellerId, itemId string) (int, error) {
	if ex := os.MkdirAll(fmt.Sprintf("%s/%s/%s/%s", root, t.Type, sellerId, itemId), 0777); ex != nil {
		fmt.Println(ex)
		return 0, ex
	}

	for p, max := 1, 1; p <= max; {
		url := fmt.Sprintf(t.RateFormat, itemId, sellerId, p)
		// fmt.Println(url)
		if body, ok := fetchData(url); ok {
			if len(body) < 100 {
				if "jsonp({\"status\":1111,\"wait\":5})" == string(body) {
					time.Sleep(5 * time.Minute)
				}
			}
			if p == 1 {
				lastPage := pickString(body, "\"lastPage\":", ",\"page\"")
				fmt.Println("lastPage", lastPage)
				if v_max, err := strconv.Atoi(lastPage); err == nil {
					max = v_max
				} else {
					fmt.Println(err)
				}
			}
			out := make([]byte, len(body)*2)
			_, bytesWritten, _ := iconv.Convert(body, out, "gbk", "utf-8")
			idx := bytes.IndexByte(out, '(')
			if idx < 0 || bytesWritten < idx+1 {
				return 0, fmt.Errorf("idx error")
			}
			out = out[idx+1 : bytesWritten]
			idx_end := bytes.LastIndexByte(out, ')')
			if idx_end < 0 {
				return 0, fmt.Errorf("idx_end<0, )")
			}
			out = out[:idx_end]

			save_to := fmt.Sprintf("%s/%s/%s/%s/%s.%d.log", root, t.Type, sellerId, itemId, itemId, p)
			fmt.Printf("%s %d/%d [%d]\n%s\n", url, p, max, len(out), save_to)
			ioutil.WriteFile(save_to, out, 0666)
			time.Sleep(time.Duration(rand.Intn(5)) * time.Second)

			p++
		}
	}
	return 0, nil
}
Exemplo n.º 5
0
//return filepath, media_type, content_disposition, error
func (this *Curler) download_html(uri string) (filepath, media_type, content_dispos string, err error) {
	//	var charset string
	ofn, media_type, charset, content_dispos, err := this.download_imp(uri)
	if err != nil {
		return
	}
	of, err := os.Open(ofn)
	if err != nil {
		return
	}
	defer of.Close()

	if len(charset) == 0 {
		_, charset = extract_charset(detect_content_type(of))
		of.Seek(0, 0)
	}
	//	log.Println("detected content-type", charset)

	//某些技术网站使用繁体,但标识gb2312
	if len(charset) == 0 || charset == "gb2312" {
		charset = "gbk"
	}

	tf, err := ioutil.TempFile(this.TempDir, this.Prefix)
	if err != nil {
		return
	}
	defer tf.Close()

	if charset != "utf-8" {
		in, _ := ioutil.ReadAll(of)
		out := make([]byte, len(in)*2)

		var w int
		_, w, err = iconv.Convert(in, out, charset, "utf-8")
		if err != nil {
			return
		}
		out = out[:w]

		io.Copy(tf, bytes.NewReader(out))
	} else {
		io.Copy(tf, of)
	}
	return tf.Name(), media_type, content_dispos, nil
}
Exemplo n.º 6
0
func ExportCsv(head []string, data [][]string) (out []byte, err error) {
	fmt.Println(len(head), len(data))
	if len(head) == 0 {
		err = errors.New("ExportCsv Head is nil")
		return
	}

	columnCount := len(head)

	dataStr := bytes.NewBufferString("")
	//添加头
	for index, headElem := range head {
		separate := ","
		if index == columnCount-1 {
			separate = "\n"
		}
		dataStr.WriteString(headElem + separate)
	}

	//添加数据行
	for _, dataArray := range data {
		if len(dataArray) != columnCount { //数据项数小于列数
			err = errors.New("ExportCsv data format is error.")
		}
		for index, dataElem := range dataArray {
			separate := ","
			if index == columnCount-1 {
				separate = "\n"
			}
			dataStr.WriteString(dataElem + separate)
		}
	}

	//处理编码
	out = make([]byte, len(dataStr.Bytes()))
	iconv.Convert(dataStr.Bytes(), out, "utf-8", OUT_ENCODING)
	return
}
Exemplo n.º 7
0
func serveHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Println("starting....")
	r.ParseForm()
	queryArr := r.Form["w"]
	fmt.Println(queryArr)
	if len(queryArr) < 1 {
		return
	}
	querystr := queryArr[0]
	in := []byte(querystr)
	out := make([]byte, len(in))

	iconv.Convert(in, out, "utf-8", "gbk")

	fmt.Printf("%v\n", string(out))

	v := url.Values{}
	v.Add("p", "1")
	v.Add("source", "1")
	v.Add("t", "1")
	v.Add("w", string(out))
	//s := "andy"
	url := "http://cgi.music.soso.com/fcgi-bin/m.q?" + v.Encode()

	//fmt.Println(url)

	//response, _ := http.Get(url)

	client := &http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(req)
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	cgiUrl := "http://soso.music.qq.com/fcgi-bin/fcg_song.fcg"
	cgiRes, _ := http.Get(cgiUrl)
	defer cgiRes.Body.Close()
	fmt.Printf(":::::%v\n", cgiRes.Header)
	urls, songName, singer, _ := GetSosoMP3URL(string(body))
	//w.Header().Add("Referer", "http://soso.music.qq.com/fcgi-bin/fcg_song.fcg")
	//w.Header().Add("Set-Cookie", "qqmusic_sosokey=4D96476733A6D833E90FEA9E590408D171B92452775E15FB")
	//w.Header().Add("Set-Cookie", "qqmusic_fromtag=10")
	//w.Header().Add("Set-Cookie", "domain=qq.com")
	fmt.Fprintln(w, "<!DOCTYPE HTML>")
	fmt.Fprintln(w, "<html>")

	fmt.Fprintln(w, "<body>")

	for i, value := range urls {
		fmt.Fprintln(w, "<audio src=\""+value+"\" controls=\"controls\">")
		fmt.Fprintln(w, "Your browser does not support the audio tag.")
		fmt.Fprintln(w, "</audio>")
		fmt.Fprintln(w, songName[i])
		fmt.Fprintln(w, "&nbsp&nbsp&nbsp")
		fmt.Fprintln(w, singer[i])
		fmt.Fprintln(w, "<br>")
	}
	fmt.Fprintln(w, "</body>")
	fmt.Fprintln(w, "</html>")

}
Exemplo n.º 8
0
func main() {
	// read bytes from sample.utf8
	utf8Bytes, err := ioutil.ReadFile("sample.utf8")

	if err != nil {
		fmt.Println("Could not open 'sample.utf8': ", err)
	}

	// read bytes from sample.ebcdic-us
	ebcdicBytes, err := ioutil.ReadFile("sample.ebcdic-us")

	if err != nil {
		fmt.Println("Could not open 'sample.ebcdic-us': ", err)
	}

	// use iconv to check conversions both ways
	utf8String := string(utf8Bytes)
	ebcdicString := string(ebcdicBytes)

	// convert from utf-8 to ebcdic
	utf8ConvertedString, err := iconv.ConvertString(utf8String, "utf-8", "ebcdic-us")

	if err != nil || ebcdicString != utf8ConvertedString {
		// generate hex string
		ebcdicHexString := hex.EncodeToString(ebcdicBytes)
		utf8ConvertedHexString := hex.EncodeToString([]byte(utf8ConvertedString))

		fmt.Println("utf-8 was not properly converted to ebcdic-us by iconv.ConvertString, error: ", err)
		fmt.Println(ebcdicHexString, " - ", len(ebcdicString))
		fmt.Println(utf8ConvertedHexString, " - ", len(utf8ConvertedString))
	} else {
		fmt.Println("utf-8 was properly converted to ebcdic-us by iconv.ConvertString")
	}

	// convert from ebcdic to utf-8
	ebcdicConvertedString, err := iconv.ConvertString(ebcdicString, "ebcdic-us", "utf-8")

	if err != nil || utf8String != ebcdicConvertedString {
		// generate hex string
		utf8HexString := hex.EncodeToString(utf8Bytes)
		ebcdicConvertedHexString := hex.EncodeToString([]byte(ebcdicConvertedString))

		fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.ConvertString, error: ", err)
		fmt.Println(utf8HexString, " - ", len(utf8String))
		fmt.Println(ebcdicConvertedHexString, " - ", len(ebcdicConvertedString))
	} else {
		fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.ConvertString")
	}

	testBuffer := make([]byte, len(ebcdicBytes)*2)

	// convert from ebdic bytes to utf-8 bytes
	bytesRead, bytesWritten, err := iconv.Convert(ebcdicBytes, testBuffer, "ebcdic-us", "utf-8")

	if err != nil || bytesRead != len(ebcdicBytes) || bytesWritten != len(utf8Bytes) {
		fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.Convert, error: ", err)
	} else {
		fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.Convert")
	}

	// convert from utf-8 bytes to ebcdic bytes
	bytesRead, bytesWritten, err = iconv.Convert(utf8Bytes, testBuffer, "utf-8", "ebcdic-us")

	if err != nil || bytesRead != len(utf8Bytes) || bytesWritten != len(ebcdicBytes) {
		fmt.Println("utf-8 was not properly converted to ebcdic-us by iconv.Convert, error: ", err)
	} else {
		fmt.Println("utf-8 was properly converted to ebcdic-us by iconv.Convert")
	}

	// test iconv.Reader
	utf8File, _ := os.Open("sample.utf8")
	utf8Reader, _ := iconv.NewReader(utf8File, "utf-8", "ebcdic-us")
	bytesRead, err = utf8Reader.Read(testBuffer)

	if err != nil || bytesRead != len(ebcdicBytes) {
		fmt.Println("utf8 was not properly converted to ebcdic-us by iconv.Reader", err)
	} else {
		fmt.Println("utf8 was property converted to ebcdic-us by iconv.Reader")
	}

	ebcdicFile, _ := os.Open("sample.ebcdic-us")
	ebcdicReader, _ := iconv.NewReader(ebcdicFile, "ebcdic-us", "utf-8")
	bytesRead, err = ebcdicReader.Read(testBuffer)

	if err != nil || bytesRead != len(utf8Bytes) {
		fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.Reader: ", err)

		if bytesRead > 0 {
			fmt.Println(string(testBuffer[:bytesRead]))
			fmt.Println(hex.EncodeToString(testBuffer[:bytesRead]))
			fmt.Println(hex.EncodeToString(utf8Bytes))
		}
	} else {
		fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.Reader")
	}
}
Exemplo n.º 9
0
func convert(src []byte) []byte {
	x := make([]byte, 100)
	_, n, _ := iconv.Convert(src, x, "GB18030", "utf-8")
	return x[:n]
}