Exemplo n.º 1
0
// FetchStockData 抓取数据
func FetchStockData(code string) (*StockIndustry, *StockICB, error) {
	// http://stockData.stock.hexun.com/600028.shtml
	formt := "http://stockData.stock.hexun.com/%s.shtml"
	resp, err := wget.Get(fmt.Sprintf(formt, code))
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	tr := doc.Find("#list3 table.box6 tr")

	stockIndustry := &StockIndustry{}
	stockICB := &StockICB{}

	stockIndustry.Name, err = iconv.ConvertString(tr.Eq(7).Find("td").Eq(1).Text(), "gb2312", "utf-8")
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	stockICB.Name, err = iconv.ConvertString(tr.Eq(8).Find("td").Eq(1).Text(), "gb2312", "utf-8")
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	return stockIndustry, stockICB, nil
}
Exemplo n.º 2
0
func (this *TextMessageController) Post() {
	r := this.Ctx.Request
	//this.ParseForm()
	r.ParseForm()
	//beego.Trace("ToUserName", this.GetString("ToUserName"))
	//beego.Trace("ToUserName", r.PostFormValue("ToUserName"))
	//beego.Trace("MsgType", this.GetString("MsgType"))
	//beego.Trace("MsgType", r.PostFormValue("MsgType"))
	for k, v := range r.Form {
		beego.Trace("key:", k)
		beego.Trace("val:", strings.Join(v, ""))
	}

	//for k, v := range r.PostForm {
	//	beego.Trace("key:", k)
	//	beego.Trace("val:", strings.Join(v, ""))
	//}
	//for i, s := range string(this.Ctx.Input.RequestBody) {
	//	beego.Trace(i, ":", s, ":", string(s))
	//}
	//beego.Trace("body:", string(this.Ctx.Input.RequestBody))
	output, _ := iconv.ConvertString(string(this.Ctx.Input.RequestBody), "utf8", "gbk")
	//beego.Trace(charmap.Windows1252.NewEncoder())
	//encoding.UTF8Validator.Transform()
	beego.Trace(output)
	this.Data["Out"] = "sdfasdfasdfasdfasdfa"
	this.TplNames = "out.tpl"
}
Exemplo n.º 3
0
func GetProxyRegion(url string) string { //得到IP的地区,从另一个网站爬的
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(time.Second * 10)
				c, err := net.DialTimeout(netw, addr, time.Second*10)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
			ResponseHeaderTimeout: time.Second * 10,
		},
	}
	resp, getError := c.Get(url)
	body, goqueryError := goquery.NewDocumentFromResponse(resp)
	if getError != nil || goqueryError != nil {
		return ""
	}
	preRegion := body.Find(".ul1").Find("li").Eq(0).Text() //爬取的网站正好是gb2312格式的
	address, err11 := iconv.ConvertString(preRegion, "gb2312", "utf-8")
	if err11 != nil {
		return ""
	}
	region := strings.Split(address, ":")
	if strings.Contains(region[1], "省") || strings.Contains(region[1], "市") {
		region[1] = "中国" + region[1]
	}
	return region[1]
}
Exemplo n.º 4
0
Arquivo: convert.go Projeto: npk/tidb
// Eval implements the Expression Eval interface.
func (f *FunctionConvert) Eval(ctx context.Context, args map[interface{}]interface{}) (interface{}, error) {
	value, err := f.Expr.Eval(ctx, args)
	if err != nil {
		return nil, err
	}

	// Casting nil to any type returns nil
	if value == nil {
		return nil, nil
	}
	str, ok := value.(string)
	if !ok {
		return nil, nil
	}
	if strings.ToLower(f.Charset) == "ascii" {
		return value, nil
	} else if strings.ToLower(f.Charset) == "utf8mb4" {
		return value, nil
	}
	target, err := iconv.ConvertString(str, "utf-8", f.Charset)
	if err != nil {
		log.Errorf("Convert %s to %s with error: %v", str, f.Charset, err)
		return nil, errors.Trace(err)
	}
	return target, nil
}
Exemplo n.º 5
0
func ConvertToUtf8(src, encoding string) string {
	if len(src) < 1 {
		return src
	}

	if encoding == "utf-8" {
		return src
	}

	lines := strings.Split(src, "\n")
	var output string
	for _, line := range lines {
		o, err := iconv.ConvertString(line, encoding, "utf-8")

		// if error
		if err != nil {
			log.Println(err)
			output += line
			continue
		}

		output += o
	}

	return output
}
Exemplo n.º 6
0
//转换字符编码格式,例如文本是gb2312的,现在转换为utf-8:ConvretCharacterEncoding(str, "gb2312", "utf-8")
func ConvertCharacterEncoding(msg, oldEncoding, newEncoding string) string {
	newMsg, err := iconv.ConvertString(msg, oldEncoding, newEncoding)
	if err != nil {
		log.Println("convert string Encoding error: ", err)
		return ""
	}
	return newMsg
}
Exemplo n.º 7
0
// parse a Part
func (p *aParse) parsePart(part Part, contentType string) (string, bool) {
	if part.ContentType().ToString() == contentType && part.Filename() == "" {
		payload := string(p.rawPart(part))

		// convert charset
		targetCharset := "utf-8"
		sourceCharset := strings.ToLower(part.ContentType().Parameter("charset"))
		if sourceCharset != targetCharset {
			payload, _ = iconv.ConvertString(payload, sourceCharset, targetCharset)
		}
		return payload, true
	}

	return "", false
}
Exemplo n.º 8
0
func (h *HDUJudger) GetProblems() error {
	vidsModel := &model.VIdsModel{}
	StartId, err := vidsModel.GetLastID("HDU")
	if err == model.DBErr {
		return err
	} else if StartId < 1000 {
		StartId = 999
	}
	errCnt := 0
	lastId := StartId
	for i := 1; ; i++ {
		pid := strconv.Itoa(StartId + i)
		page, err := h.GetProblemPage(pid)
		if err != nil { //offline
			hdulogger.Println("pid["+pid+"]: ", err, ".")
			return err
		}
		cpage, err := iconv.ConvertString(page, "gb2312", "utf-8")
		if err != nil { //Although getting error, continue proccess it.
			hdulogger.Println("pid["+pid+"]: ", "encode convert error.")
			cpage = page
		}

		if h.IsExist(cpage) {
			err := h.SetDetail(pid, cpage)
			if err != nil {
				hdulogger.Println("pid["+pid+"]: ", "import error.")
			} else {
				lastId = StartId + i
				vidsModel.SetLastID("HDU", lastId)
			}
			errCnt = 0
		} else {
			hdulogger.Println("pid["+pid+"]: ", "not exist.")
			errCnt++
		}

		if errCnt >= 100 { //If "not exist" continuously repeat 100 times, terminate it.
			break
		}
	}

	hdulogger.Println("import terminated. Last pid is ", lastId, ".")
	return nil
}
Exemplo n.º 9
0
// ReadContentUFT8 reads the contents from the file and pass these contents to UTF-8 encoding.
func ReadContentUTF8(path string, encoding string) (contents string, err error) {

	// Read the contents of the file, whatever encoding it has
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatalf("[PropRef:toolset] error reading file contents: %v", err)
		return "", errors.New("[PropRef:tools] Error reading file " + err.Error())
	}

	// We need to convert that to UTF-8 if the encoding is different to UTF-8
	data, err := iconv.ConvertString(string(dat), encoding, "utf-8")
	if err != nil {
		log.Fatalf("[PropRef:tools] error converting file %v with encoding %v to UTF-8: %v.", path, encoding, err)
		return "", errors.New("[PropRef:tools] Error converting file encoding " + err.Error())
	}
	return data, nil

}
Exemplo n.º 10
0
func scanDiapazon(ch chan storage.IpItem, quit chan uint32, ipstart, ipend uint32) {
	var ip uint32
	var ipItem storage.IpItem
	proxy, _ := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, proxy.Direct)
	for ip = ipstart; ip < ipend; ip++ {
		ipItem.IP = ip
		conn, err := proxy.Dial("tcp", ipItem.String()+":23")
		if err != nil {
		} else {
			hello, err := bufio.NewReader(conn).ReadString(':')
			if err == nil {
				output, _ := iconv.ConvertString(hello, "latin1", "utf-8")
				ipItem.Hello = output
				conn.Close()
				ch <- ipItem
			}
		}
	}
	quit <- ipstart / BLOCK_SIZE
}
Exemplo n.º 11
0
// 打印响应
func (to *TcpObj) printResponse(r *http.Response, err error) {
	if nil != err {
		fmt.Println("Error Happend!\n", err)
		return
	}

	// 打印头部
	if to.OutputResponse {
		fmt.Println(r.Proto, r.Status)
		fmt.Println(sep("-", 80), ":resp.HEADER")
		for key, _ := range r.Header {
			fmt.Printf("%20s : %s\n", key, r.Header.Get(key))
		}
		// 打印分隔行
		fmt.Println(sep("-", 80), ":resp.BODY")
	}

	// 分析编码和内容
	rContentType := strings.TrimSpace(r.Header.Get("Content-Type"))
	reg := regexp.MustCompile(`^([a-z/]+)(;[ ]*)(charset=)(.*)$`)
	grps := reg.FindAllStringSubmatch(rContentType, -1)
	var charset string
	//var contentType string
	if nil != grps {
		//contentType = grps[0][1]
		charset = grps[0][4]
	} else {
		charset = "utf8"
		//contentType = "text/html"
	}

	// 打印 body
	bs, _ := ioutil.ReadAll(r.Body)
	r.Body.Close()
	str, _ := iconv.ConvertString(string(bs), charset, "utf8")

	fmt.Println(str)

}
Exemplo n.º 12
0
func searchbyYear(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	Year := params["year"]
	ID := params["id"]
	FN := params["fn"]
	LN := params["ln"]
	//fmt.Printf("%s %s %s %s",Year,ID,FN,LN)
	Surl := "http://refundedcheque.rd.go.th/itp_x_tw/SearchTaxpayerServlet"
	query := fmt.Sprintf("nid=%s&fName=%s&lName=%s&taxYear=%s&searchType=null&effDate=null", ID, FN, LN, Year)
	log.Println(query)
	req, err := http.NewRequest("POST", Surl, strings.NewReader(query))
	req.Header.Set("Referer", "http://refundedcheque.rd.go.th/itp_x_tw/pages/ITPtaxresult.jsp")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("X-Requested-With", "XMLHttpRequest")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	body, _ := ioutil.ReadAll(resp.Body)
	output, _ := iconv.ConvertString(string(body), "tis-620", "utf-8")
	fmt.Println("response Body:", string(output))

	if len(output) == 0 {
		log.Println("Data not found")
		w.Header().Set("Content-Type", "application/json")
		w.Write([]byte("[]"))
	}
	w.Write([]byte(output))

}
Exemplo n.º 13
0
func fetchAndFillFundRank(page int, quarter string) (int, error) {
	// http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=50&date=2015-09-30&orgType=&stateType=null&titType=null&page=2&callback=hxbase_json7
	formt := "http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=%s&date=%s&orgType=&stateType=null&titType=null&page=%d&callback=hxbase_json7"
	pageLimit := 100
	body, err := wget.GetBody(fmt.Sprintf(formt, pageLimit, quarter, page))
	if err != nil {
		return -1, err
	}

	exists := db.NewExistsBuilder("funds")
	ist := db.NewInsertBuilder("funds")
	query := db.NewQueryBuilder("funds")

	var fund *Fund
	var fundRank *FundRank
	var row db.DataRow

	// hxbase_json7(
	str, err := iconv.ConvertString(string(body), "gb2312", "utf-8")
	if err != nil {
		return -1, err
	}

	src := []byte(str)
	src = src[13 : len(src)-1]
	src = bytes.Replace(src, []byte(":'"), []byte(`":"`), -1)
	src = bytes.Replace(src, []byte("',"), []byte(`","`), -1)
	src = bytes.Replace(src, []byte("'}"), []byte(`"}`), -1)
	src = bytes.Replace(src, []byte("{"), []byte(`{"`), -1)
	src = bytes.Replace(src, []byte("sum:"), []byte(`sum":`), 1)
	src = bytes.Replace(src, []byte("list:"), []byte(`"list":`), 1)

	v := &JsonFund{}
	err = json.Unmarshal(src, v)
	if err != nil {
		fmt.Println(string(src))
		return -1, err
	}
	// {RankTd:'51',OrgName:'法国巴黎银行',OrgNameLink:'o-QF000031.shtml',OrgType:'QFII',ShareHoldingNum:'3',ShareHoldingNumLink:'otherDetail.aspx?OrgNo=QF000031',TotalHoldings:'48,388.00',TotalMarketValue:'1,100,735.00',OrgAlt:'法国巴黎银行'}
	for _, item := range v.List {
		fund = &Fund{Code: item["OrgNameLink"], Name: string(item["OrgName"]), TypeID: FundType(item["OrgType"])}
		fund.Code = fund.Code[2 : len(fund.Code)-6]
		fmt.Println("----")
		if !exists.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).Exists() {
			fmt.Println("insert", fund)
			ist.Table("funds").Insert(fund)
		}

		row, _ = query.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).QueryOne()
		if row.Empty() {
			return -1, fmt.Errorf("code %s not found", fund.Code)
		}

		fundRank = &FundRank{
			FundID: row.GetInt64("id"),
			Date:   quarter,
			Rank:   int(util.ParseMoney(item["RankTd"])),
			Count:  int(util.ParseMoney(item["ShareHoldingNum"])),
			MH:     int64(util.ParseMoney(item["TotalHoldings"])),
			MV:     int64(util.ParseMoney(item["TotalMarketValue"])),
		}

		if !exists.Table("fund_rank").Where("fund_id=? and date=?", fundRank.FundID, fundRank.Date).Exists() {
			fmt.Println("insert", fundRank)
			ist.Table("fund_rank").Insert(fundRank)
		}

	}
	return int(math.Ceil(float64(v.Sum / pageLimit))), nil
}
Exemplo n.º 14
0
func (t *Connection) Submit(src, dst, msg string, opt *SubmitOptions) (
	err error) {
	glog.Infof("Submit %s -> %s: %s", src, dst, msg)
	body := new(bytes.Buffer)
	// MsgType
	err = body.WriteByte(opt.MsgType)
	if err != nil {
		return
	}
	// NeedReport = 0 for now
	err = body.WriteByte(0)
	if err != nil {
		return
	}
	// Priority
	if !(opt.Priority >= 0 && opt.Priority <= 3) {
		glog.Warningf("Incorrect priority %d, set to 1, normal.",
			opt.Priority)
		opt.Priority = 1
	}
	err = body.WriteByte(opt.Priority)
	if err != nil {
		return
	}
	// ServiceID 10 octetString
	b, err := octetString(opt.ServiceID, 10)
	if err != nil {
		return
	}
	_, err = body.Write(b)
	if err != nil {
		return
	}
	// FeeType 2 octetString
	if len(opt.FeeType) != 2 {
		err = errors.New("len(FeeType) should be 2")
		return
	}
	_, err = body.WriteString(opt.FeeType)
	if err != nil {
		return
	}
	// FeeCode 6 octetString
	b, err = octetString(opt.FeeCode, 6)
	if err != nil {
		return
	}
	_, err = body.Write(b)
	if err != nil {
		return
	}
	// FixedFee 6 octetString
	b, err = octetString(opt.FixedFee, 6)
	if err != nil {
		return
	}
	_, err = body.Write(b)
	if err != nil {
		return
	}
	// MsgFormat byte, text message required 15 for GB18030
	err = body.WriteByte(15)
	if err != nil {
		return
	}
	// ValidTime 17 octetString “YYMMDDhhmmsstnnp”
	_, err = body.Write([]byte{
		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return
	}
	// AtTime 17 octetString
	_, err = body.Write([]byte{
		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return
	}
	// SrcTermID 21 octetString
	b, err = octetString(src, 21)
	if err != nil {
		return
	}
	_, err = body.Write(b)
	if err != nil {
		return
	}
	// ChargeTermID 21 octetString
	_, err = body.Write([]byte{
		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
		0})
	if err != nil {
		return
	}
	// DestTermIDCount byte
	err = body.WriteByte(0x01)
	if err != nil {
		return
	}
	// DestTermID 21 octetString * count
	b, err = octetString(dst, 21)
	if err != nil {
		return
	}
	_, err = body.Write(b)
	if err != nil {
		return
	}
	// convert msg to GB18030
	msg, err = iconv.ConvertString(msg, "UTF-8", "GB18030")
	if err != nil {
		return
	}
	if len(msg) > 0xff {
		err = fmt.Errorf("Message too long: %d bytes", len(msg))
	}
	// MsgLength byte
	err = body.WriteByte(byte(len(msg)))
	if err != nil {
		return
	}
	// MsgContent octetString
	_, err = body.WriteString(msg)
	if err != nil {
		return
	}
	// Reserve 8 octetString
	_, err = body.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return
	}
	seq, err := t.writeRequest(REQID_SUBMIT, body.Bytes())
	if err != nil {
		return
	}
	t.pool[seq] = nil
	glog.Infof("Submit seq %d %s -> %s: %s", seq, src, dst, msg)
	return
}
Exemplo n.º 15
0
func main() {
	// read bytes from sample.utf8
	utf8Bytes, err := ioutil.ReadFile("sample.utf8")

	if err != nil {
		fmt.Println("Could not open 'sample.utf8': ", err)
	}

	// read bytes from sample.ebcdic-us
	ebcdicBytes, err := ioutil.ReadFile("sample.ebcdic-us")

	if err != nil {
		fmt.Println("Could not open 'sample.ebcdic-us': ", err)
	}

	// use iconv to check conversions both ways
	utf8String := string(utf8Bytes)
	ebcdicString := string(ebcdicBytes)

	// convert from utf-8 to ebcdic
	utf8ConvertedString, err := iconv.ConvertString(utf8String, "utf-8", "ebcdic-us")

	if err != nil || ebcdicString != utf8ConvertedString {
		// generate hex string
		ebcdicHexString := hex.EncodeToString(ebcdicBytes)
		utf8ConvertedHexString := hex.EncodeToString([]byte(utf8ConvertedString))

		fmt.Println("utf-8 was not properly converted to ebcdic-us by iconv.ConvertString, error: ", err)
		fmt.Println(ebcdicHexString, " - ", len(ebcdicString))
		fmt.Println(utf8ConvertedHexString, " - ", len(utf8ConvertedString))
	} else {
		fmt.Println("utf-8 was properly converted to ebcdic-us by iconv.ConvertString")
	}

	// convert from ebcdic to utf-8
	ebcdicConvertedString, err := iconv.ConvertString(ebcdicString, "ebcdic-us", "utf-8")

	if err != nil || utf8String != ebcdicConvertedString {
		// generate hex string
		utf8HexString := hex.EncodeToString(utf8Bytes)
		ebcdicConvertedHexString := hex.EncodeToString([]byte(ebcdicConvertedString))

		fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.ConvertString, error: ", err)
		fmt.Println(utf8HexString, " - ", len(utf8String))
		fmt.Println(ebcdicConvertedHexString, " - ", len(ebcdicConvertedString))
	} else {
		fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.ConvertString")
	}

	testBuffer := make([]byte, len(ebcdicBytes)*2)

	// convert from ebdic bytes to utf-8 bytes
	bytesRead, bytesWritten, err := iconv.Convert(ebcdicBytes, testBuffer, "ebcdic-us", "utf-8")

	if err != nil || bytesRead != len(ebcdicBytes) || bytesWritten != len(utf8Bytes) {
		fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.Convert, error: ", err)
	} else {
		fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.Convert")
	}

	// convert from utf-8 bytes to ebcdic bytes
	bytesRead, bytesWritten, err = iconv.Convert(utf8Bytes, testBuffer, "utf-8", "ebcdic-us")

	if err != nil || bytesRead != len(utf8Bytes) || bytesWritten != len(ebcdicBytes) {
		fmt.Println("utf-8 was not properly converted to ebcdic-us by iconv.Convert, error: ", err)
	} else {
		fmt.Println("utf-8 was properly converted to ebcdic-us by iconv.Convert")
	}

	// test iconv.Reader
	utf8File, _ := os.Open("sample.utf8")
	utf8Reader, _ := iconv.NewReader(utf8File, "utf-8", "ebcdic-us")
	bytesRead, err = utf8Reader.Read(testBuffer)

	if err != nil || bytesRead != len(ebcdicBytes) {
		fmt.Println("utf8 was not properly converted to ebcdic-us by iconv.Reader", err)
	} else {
		fmt.Println("utf8 was property converted to ebcdic-us by iconv.Reader")
	}

	ebcdicFile, _ := os.Open("sample.ebcdic-us")
	ebcdicReader, _ := iconv.NewReader(ebcdicFile, "ebcdic-us", "utf-8")
	bytesRead, err = ebcdicReader.Read(testBuffer)

	if err != nil || bytesRead != len(utf8Bytes) {
		fmt.Println("ebcdic-us was not properly converted to utf-8 by iconv.Reader: ", err)

		if bytesRead > 0 {
			fmt.Println(string(testBuffer[:bytesRead]))
			fmt.Println(hex.EncodeToString(testBuffer[:bytesRead]))
			fmt.Println(hex.EncodeToString(utf8Bytes))
		}
	} else {
		fmt.Println("ebcdic-us was properly converted to utf-8 by iconv.Reader")
	}
}
Exemplo n.º 16
0
func ConvertUtf(in string, code string) (output string, err error) {
	output, err = iconv.ConvertString(in, "utf-8", code)
	return
}
Exemplo n.º 17
0
func ToCp874(str string) string {
	str, _ = iconv.ConvertString(str, "utf-8", "cp874")
	return str
}
Exemplo n.º 18
0
func main() {
	iconv.ConvertString("我是UTF-8", "utf-8", "gbk")
}
Exemplo n.º 19
0
// FetchDzjy 抓取数据
func FetchDzjy(date time.Time) ([]*dzjyVO, error) {
	formt := "http://data.eastmoney.com/dzjy/%s.html"

	resp, err := wget.Get(fmt.Sprintf(formt, date.Format("200601")))
	if err != nil {
		return nil, gos.DoError(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, gos.DoError(err)
	}
	var td *goquery.Selection
	var dzjy *dzjyVO
	var dateStr string
	var stockCode string
	var row db.DataRow
	query := db.NewQueryBuilder("stock")
	datalist := make([]*dzjyVO, 0)
	buy := ""
	sell := ""
	var priceNow float64
	var price float64
	var amount float64
	var total float64
	var length int

	doc.Find("#content div.list").Eq(2).Find("table tr.list_eve").Each(func(i int, tr *goquery.Selection) {
		td = tr.Find("td")
		length = td.Length()

		if length == 10 {
			dateStr = td.Eq(0).Text()
			stockCode = td.Eq(1).Text()
		} else if length == 9 {
			stockCode = td.Eq(0).Text()
		}

		row, _ = query.Where("code=?", stockCode).QueryOne()
		if row.Empty() {
			return
		}

		switch length {
		case 10:
			priceNow = util.ParseMoney(td.Eq(4).Text())
			price = util.ParseMoney(td.Eq(5).Text())
			amount = util.ParseMoney(td.Eq(6).Text())
			total = util.ParseMoney(td.Eq(7).Text())
			buy, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
			sell, err = iconv.ConvertString(td.Eq(9).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}

		case 9:
			priceNow = util.ParseMoney(td.Eq(3).Text())
			price = util.ParseMoney(td.Eq(4).Text())
			amount = util.ParseMoney(td.Eq(5).Text())
			total = util.ParseMoney(td.Eq(6).Text())
			buy, err = iconv.ConvertString(td.Eq(7).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
			sell, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
		case 5:
			price = util.ParseMoney(td.Eq(0).Text())
			amount = util.ParseMoney(td.Eq(1).Text())
			total = util.ParseMoney(td.Eq(2).Text())
			buy, err = iconv.ConvertString(td.Eq(3).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
			sell, err = iconv.ConvertString(td.Eq(4).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
		default:
			return
		}

		dzjy = &dzjyVO{
			StockID:  row.GetInt64("id"),
			Date:     dateStr,
			PriceNow: priceNow,
			Price:    price,
			Amount:   amount,
			Total:    total,
			Buy:      buy,
			Sell:     sell,
		}

		datalist = append(datalist, dzjy)
	})

	return datalist, nil
}
Exemplo n.º 20
0
func (qw *QQWry) QueryIP(ip string) (areaInfo, extraInfo string) {
	var first, last, current int64

	addr := ip2Int64(ip)
	if addr == -1 {
		return "Unknown", "Unknown"
	}

	first = qw.first
	last = qw.last
	current = ((last-first)/7/2)*7 + first
	for current > first {
		if read4byte(qw, current) > addr {
			last = current
			current = ((last-first)/7/2)*7 + first
		} else {
			first = current
			current = ((last-first)/7/2)*7 + first
		}
	}

	offset := read3byte(qw, current+4)
	end := read4byte(qw, offset)
	if addr > end {
		return "Unknown", "Unknown"
	}

	offset += 4
	mode := read1byte(qw, offset)
	country := make([]byte, 256)
	var area []byte
	n := 0
	if mode == _REDIRECT_MODE_1 {
		offset = read3byte(qw, offset+1)
		if read1byte(qw, offset) == _REDIRECT_MODE_2 {
			off := read3byte(qw, offset+1)
			qw.ReadAt(country, off)
			n = bytes.IndexByte(country, 0x00)
			country = country[:n]
			offset += 4
		} else {
			qw.ReadAt(country, offset)
			n = bytes.IndexByte(country, 0x00)
			country = country[:n]
			offset += int64(n + 1)
		}
	} else if mode == _REDIRECT_MODE_2 {
		off := read3byte(qw, offset+1)
		qw.ReadAt(country, off)
		n = bytes.IndexByte(country, 0x00)
		country = country[:n]
		offset += 4
	} else {
		qw.ReadAt(country, offset)
		n = bytes.IndexByte(country, 0x00)
		country = country[:n]
		offset += int64(n + 1)
	}

	mode = read1byte(qw, offset)
	if mode == _REDIRECT_MODE_1 || mode == _REDIRECT_MODE_2 {
		offset = read3byte(qw, offset+1)
	}

	if offset != 0 {
		area = make([]byte, 256)
		qw.ReadAt(area, offset)
		n = bytes.IndexByte(area, 0x00)
		area = area[:n]
	}

	cz88 := []byte("CZ88.NET")
	if len(country) > 1 && bytes.Compare(country[1:], cz88) != 0 {
		areaInfo, _ = iconv.ConvertString(string(country), "GB18030", "utf-8")
	}

	if len(area) > 1 && bytes.Compare(area[1:], cz88) != 0 {
		extraInfo, _ = iconv.ConvertString(string(area), "GB18030", "utf-8")
	}

	return areaInfo, extraInfo
}
Exemplo n.º 21
0
Arquivo: hq.go Projeto: jack1582/hq
func run() {
	var (
		doc           *goquery.Document
		doc_err, err  error
		fd            *os.File
		file_encoding string = "-"
	)
	if *url == "-" {
		doc, doc_err = goquery.NewDocumentFromReader(os.Stdin)
	} else if fd, err = os.Open(*url); err == nil {
		doc, doc_err = goquery.NewDocumentFromReader(fd)
		defer fd.Close()
	} else {
		tmp_url := *url
		if !strings.HasPrefix(tmp_url, "http") {
			tmp_url = "http://" + tmp_url
		}
		doc, doc_err = goquery.NewDocument(tmp_url)
	}
	if doc_err != nil {
		log.Fatal("goquery NewDocument err:", doc_err)
	}

	if *debug {
		log.Printf("tag=[%s]\n", selector)
	}

	if !*noenc {
		file_encoding = get_html_enc(doc)
	}

	doc.Find(selector).Map(func(i int, sel *goquery.Selection) string {
		output := ""
		switch fun {
		case "html":
			if output, err = sel.Html(); err != nil {
				log.Fatal("select err:", err)
			}
			output = strings.TrimSpace(output)
		case "ohtml":
			if output, err = goquery.OuterHtml(sel); err != nil {
				log.Fatal("select err:", err)
			}
			output = strings.TrimSpace(output)
		case "text":
			output = strings.TrimSpace(sel.Text())
		case "attr":
			attr_list := strings.Split(*attr, ",")
			for _, attr_i := range attr_list {
				var output_i string
				if attr_i == "text" { // a hardcode case for convenience scrapy
					output_i = strings.TrimSpace(sel.Text())
				} else {
					output_i = sel.AttrOr(strings.TrimSpace(attr_i), "-")
				}
				if output_i == "" {
					output_i = "-"
				}
				output += output_i + "\t"
			}
		}

		if !*noenc && file_encoding != "" && file_encoding != "utf8" {
			if output, err = iconv.ConvertString(output, file_encoding, "utf8"); err != nil {
				log.Fatal("encoding invalid", err)
			}
		}
		fmt.Println(output)
		return ""
	})
}