func GetMP3URL(data []byte) string {
type Result struct {
XMLName xml.Name `xml:"result"`
Count int `xml:"count"`
Encode []string `xml:"url>encode"`
Decode []string `xml:"url>decode"`
Typed []string `xml:"url>type"`
}
var v Result
out := make([]byte, len(data))
iconv.Convert(data, out, "gb2312", "utf-8")
fmt.Printf("%v\n", string(out))
fString := strings.Replace(string(out), "gb2312", "utf-8", 1)
fmt.Printf("%v\n", fString)
err := xml.Unmarshal([]byte(fString), &v)
if err != nil {
fmt.Printf("error: %v", err)
return "error"
}
if v.Count < 0 {
return "没有找到"
}
for i, value := range v.Typed {
if value == "8" {
return v.Encode[i][0:strings.LastIndex(v.Encode[i], "/")+1] + v.Decode[i]
}
}
return "没有mp3格式"
}
func ConvertEncodingAuto(src []byte) ([]byte, error) {
out := make([]byte, 64*1024)
for _, encoding := range []string{"CP932", "utf-8"} {
_, bytes, err := iconv.Convert(src, out, encoding, "utf-8")
if err == nil {
return out[0:bytes], nil
}
}
return nil, errors.New("cannot convert string")
}
func json2Filef(url string, save_to string, maxPage string) (int, error) {
//fmt.Println(url, maxPage)
for {
ll = append(ll, time.Now())
statusCode, body, err := fasthttp.Get(nil, url)
if err != nil {
return 0, err
}
if 200 != statusCode {
return 0, fmt.Errorf("!200")
}
anti_spider := "smPolicy=tmallrateweb-rate-anti_Spider-checklogin"
idx_anti_spider := bytes.Index(body, []byte(anti_spider))
paginator_empty := "\"paginator\":\"\""
idx_paginator_empty := bytes.Index(body, []byte(paginator_empty))
if idx_anti_spider >= 0 || idx_paginator_empty >= 0 {
time.Sleep(5 * time.Second)
continue
}
out := make([]byte, len(body)*2)
_, bytesWritten, err := iconv.Convert(body, out, "gbk", "utf-8")
//fmt.Println(bytesRead, bytesWritten, err)
idx := bytes.IndexByte(out, '(')
if idx < 0 || bytesWritten < idx+1 {
return 0, fmt.Errorf("idx error")
}
out = out[idx+1 : bytesWritten]
//fmt.Println(string(out))
idx_end := bytes.LastIndexByte(out, ')')
if idx_end < 0 {
return 0, fmt.Errorf("idx_end<0, )")
}
out = out[:idx_end]
fmt.Println(save_to)
ioutil.WriteFile(save_to, out, 0666)
time.Sleep(1 * time.Second)
if len(maxPage) > 0 {
return strconv.Atoi(pickString(body, maxPage, ":", ","))
} else {
return 0, nil
}
}
}
func json2Filef(t *ShopType, sellerId, itemId string) (int, error) {
if ex := os.MkdirAll(fmt.Sprintf("%s/%s/%s/%s", root, t.Type, sellerId, itemId), 0777); ex != nil {
fmt.Println(ex)
return 0, ex
}
for p, max := 1, 1; p <= max; {
url := fmt.Sprintf(t.RateFormat, itemId, sellerId, p)
// fmt.Println(url)
if body, ok := fetchData(url); ok {
if len(body) < 100 {
if "jsonp({\"status\":1111,\"wait\":5})" == string(body) {
time.Sleep(5 * time.Minute)
}
}
if p == 1 {
lastPage := pickString(body, "\"lastPage\":", ",\"page\"")
fmt.Println("lastPage", lastPage)
if v_max, err := strconv.Atoi(lastPage); err == nil {
max = v_max
} else {
fmt.Println(err)
}
}
out := make([]byte, len(body)*2)
_, bytesWritten, _ := iconv.Convert(body, out, "gbk", "utf-8")
idx := bytes.IndexByte(out, '(')
if idx < 0 || bytesWritten < idx+1 {
return 0, fmt.Errorf("idx error")
}
out = out[idx+1 : bytesWritten]
idx_end := bytes.LastIndexByte(out, ')')
if idx_end < 0 {
return 0, fmt.Errorf("idx_end<0, )")
}
out = out[:idx_end]
save_to := fmt.Sprintf("%s/%s/%s/%s/%s.%d.log", root, t.Type, sellerId, itemId, itemId, p)
fmt.Printf("%s %d/%d [%d]\n%s\n", url, p, max, len(out), save_to)
ioutil.WriteFile(save_to, out, 0666)
time.Sleep(time.Duration(rand.Intn(5)) * time.Second)
p++
}
}
return 0, nil
}
//return filepath, media_type, content_disposition, error
func (this *Curler) download_html(uri string) (filepath, media_type, content_dispos string, err error) {
// var charset string
ofn, media_type, charset, content_dispos, err := this.download_imp(uri)
if err != nil {
return
}
of, err := os.Open(ofn)
if err != nil {
return
}
defer of.Close()
if len(charset) == 0 {
_, charset = extract_charset(detect_content_type(of))
of.Seek(0, 0)
}
// log.Println("detected content-type", charset)
//某些技术网站使用繁体，但标识gb2312
if len(charset) == 0 || charset == "gb2312" {
charset = "gbk"
}
tf, err := ioutil.TempFile(this.TempDir, this.Prefix)
if err != nil {
return
}
defer tf.Close()
if charset != "utf-8" {
in, _ := ioutil.ReadAll(of)
out := make([]byte, len(in)*2)
var w int
_, w, err = iconv.Convert(in, out, charset, "utf-8")
if err != nil {
return
}
out = out[:w]
io.Copy(tf, bytes.NewReader(out))
} else {
io.Copy(tf, of)
}
return tf.Name(), media_type, content_dispos, nil
}
func ExportCsv(head []string, data [][]string) (out []byte, err error) {
fmt.Println(len(head), len(data))
if len(head) == 0 {
err = errors.New("ExportCsv Head is nil")
return
}
columnCount := len(head)
dataStr := bytes.NewBufferString("")
//添加头
for index, headElem := range head {
separate := ","
if index == columnCount-1 {
separate = "\n"
}
dataStr.WriteString(headElem + separate)
}
//添加数据行
for _, dataArray := range data {
if len(dataArray) != columnCount { //数据项数小于列数
err = errors.New("ExportCsv data format is error.")
}
for index, dataElem := range dataArray {
separate := ","
if index == columnCount-1 {
separate = "\n"
}
dataStr.WriteString(dataElem + separate)
}
}
//处理编码
out = make([]byte, len(dataStr.Bytes()))
iconv.Convert(dataStr.Bytes(), out, "utf-8", OUT_ENCODING)
return
}
func serveHTTP(w http.ResponseWriter, r *http.Request) {
fmt.Println("starting....")
r.ParseForm()
queryArr := r.Form["w"]
fmt.Println(queryArr)
if len(queryArr) < 1 {
return
}
querystr := queryArr[0]
in := []byte(querystr)
out := make([]byte, len(in))
iconv.Convert(in, out, "utf-8", "gbk")
fmt.Printf("%v\n", string(out))
v := url.Values{}
v.Add("p", "1")
v.Add("source", "1")
v.Add("t", "1")
v.Add("w", string(out))
//s := "andy"
url := "http://cgi.music.soso.com/fcgi-bin/m.q?" + v.Encode()
//fmt.Println(url)
//response, _ := http.Get(url)
client := &http.Client{}
req, _ := http.NewRequest("GET", url, nil)
resp, err := client.Do(req)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println(err)
return
}
cgiUrl := "http://soso.music.qq.com/fcgi-bin/fcg_song.fcg"
cgiRes, _ := http.Get(cgiUrl)
defer cgiRes.Body.Close()
fmt.Printf(":::::%v\n", cgiRes.Header)
urls, songName, singer, _ := GetSosoMP3URL(string(body))
//w.Header().Add("Referer", "http://soso.music.qq.com/fcgi-bin/fcg_song.fcg")
//w.Header().Add("Set-Cookie", "qqmusic_sosokey=4D96476733A6D833E90FEA9E590408D171B92452775E15FB")
//w.Header().Add("Set-Cookie", "qqmusic_fromtag=10")
//w.Header().Add("Set-Cookie", "domain=qq.com")
fmt.Fprintln(w, "<!DOCTYPE HTML>")
fmt.Fprintln(w, "<html>")
fmt.Fprintln(w, "<body>")
for i, value := range urls {
fmt.Fprintln(w, "<audio src=\""+value+"\" controls=\"controls\">")
fmt.Fprintln(w, "Your browser does not support the audio tag.")
fmt.Fprintln(w, "</audio>")
fmt.Fprintln(w, songName[i])
fmt.Fprintln(w, "   ")
fmt.Fprintln(w, singer[i])
fmt.Fprintln(w, "<br>")
}
fmt.Fprintln(w, "</body>")
fmt.Fprintln(w, "</html>")
}
func main() {
// read bytes from sample.utf8
utf8Bytes, err := ioutil.ReadFile("sample.utf8")
if err != nil {
fmt.Println("Could not open 'sample.utf8': ", err)
}
// read bytes from sample.ebcdic-us
ebcdicBytes, err := ioutil.ReadFile("sample.ebcdic-us")
if err != nil {
fmt.Println("Could not open 'sample.ebcdic-us': ", err)
}
// use iconv to check conversions both ways
utf8String := string(utf8Bytes)
ebcdicString := string(ebcdicBytes)
// convert from utf-8 to ebcdic
utf8ConvertedString, err := iconv.ConvertString(utf8String, "utf-8", "ebcdic-us")
if err != nil || ebcdicString != utf8ConvertedString {
// generate hex string
ebcdicHexString := hex.EncodeToString(ebcdicBytes)
utf8ConvertedHexString := hex.EncodeToString([]byte(utf8ConvertedString))
fmt.Println("utf-8 was not properly converted to ebcdic-us by iconv.ConvertString, error: ", err)
fmt.Println(ebcdicHexString, " - ", len(ebcdicString))
fmt.Println(utf8ConvertedHexString, " - ", len(utf8ConvertedString))
} else {
fmt.Println("utf-8 was properly converted to ebcdic-us by iconv.ConvertString")
}
// convert from ebcdic to utf-8
ebcdicConvertedString, err := iconv.ConvertString(ebcdicString, "ebcdic-us", "utf-8")
if err != nil || utf8String != ebcdicConvertedString {
// generate hex string
utf8HexString := hex.EncodeToString(utf8Bytes)
ebcdicConvertedHexString := hex.EncodeToString([]byte(ebcdicConvertedString))
fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.ConvertString, error: ", err)
fmt.Println(utf8HexString, " - ", len(utf8String))
fmt.Println(ebcdicConvertedHexString, " - ", len(ebcdicConvertedString))
} else {
fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.ConvertString")
}
testBuffer := make([]byte, len(ebcdicBytes)*2)
// convert from ebdic bytes to utf-8 bytes
bytesRead, bytesWritten, err := iconv.Convert(ebcdicBytes, testBuffer, "ebcdic-us", "utf-8")
if err != nil || bytesRead != len(ebcdicBytes) || bytesWritten != len(utf8Bytes) {
fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.Convert, error: ", err)
} else {
fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.Convert")
}
// convert from utf-8 bytes to ebcdic bytes
bytesRead, bytesWritten, err = iconv.Convert(utf8Bytes, testBuffer, "utf-8", "ebcdic-us")
if err != nil || bytesRead != len(utf8Bytes) || bytesWritten != len(ebcdicBytes) {
fmt.Println("utf-8 was not properly converted to ebcdic-us by iconv.Convert, error: ", err)
} else {
fmt.Println("utf-8 was properly converted to ebcdic-us by iconv.Convert")
}
// test iconv.Reader
utf8File, _ := os.Open("sample.utf8")
utf8Reader, _ := iconv.NewReader(utf8File, "utf-8", "ebcdic-us")
bytesRead, err = utf8Reader.Read(testBuffer)
if err != nil || bytesRead != len(ebcdicBytes) {
fmt.Println("utf8 was not properly converted to ebcdic-us by iconv.Reader", err)
} else {
fmt.Println("utf8 was property converted to ebcdic-us by iconv.Reader")
}
ebcdicFile, _ := os.Open("sample.ebcdic-us")
ebcdicReader, _ := iconv.NewReader(ebcdicFile, "ebcdic-us", "utf-8")
bytesRead, err = ebcdicReader.Read(testBuffer)
if err != nil || bytesRead != len(utf8Bytes) {
fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.Reader: ", err)
if bytesRead > 0 {
fmt.Println(string(testBuffer[:bytesRead]))
fmt.Println(hex.EncodeToString(testBuffer[:bytesRead]))
fmt.Println(hex.EncodeToString(utf8Bytes))
}
} else {
fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.Reader")
}
}
func convert(src []byte) []byte {
x := make([]byte, 100)
_, n, _ := iconv.Convert(src, x, "GB18030", "utf-8")
return x[:n]
}
