// FetchStockData 抓取数据
func FetchStockData(code string) (*StockIndustry, *StockICB, error) {
// http://stockData.stock.hexun.com/600028.shtml
formt := "http://stockData.stock.hexun.com/%s.shtml"
resp, err := wget.Get(fmt.Sprintf(formt, code))
if err != nil {
return nil, nil, gos.DoError(err)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, nil, gos.DoError(err)
}
tr := doc.Find("#list3 table.box6 tr")
stockIndustry := &StockIndustry{}
stockICB := &StockICB{}
stockIndustry.Name, err = iconv.ConvertString(tr.Eq(7).Find("td").Eq(1).Text(), "gb2312", "utf-8")
if err != nil {
return nil, nil, gos.DoError(err)
}
stockICB.Name, err = iconv.ConvertString(tr.Eq(8).Find("td").Eq(1).Text(), "gb2312", "utf-8")
if err != nil {
return nil, nil, gos.DoError(err)
}
return stockIndustry, stockICB, nil
}
// GetRzrqStockData 抓取数据
func GetRzrqStockData(code string) ([]*RzrqStockData, error) {
if isStockCached(code) {
log.App.Info("rzrq stock cached", code)
return stockDataCached[code], nil
}
src, err := FetchRzrqStockData(code, 1)
if err != nil {
return nil, gos.DoError(err)
}
v := &RzrqStockJSONData{}
// [{stats:false}]
if len(src) == 15 && string(src) == "[{stats:false}]" {
return nil, fmt.Errorf("您所查找的股票代码 %s 不存在", code)
}
if err = json.Unmarshal(src, &v); err != nil {
return nil, gos.DoError(err)
}
var dataSet []*RzrqStockData
dataSet, err = v.ParseSumData()
if err != nil {
return nil, err
}
stockDataCached[code] = dataSet
return dataSet, err
}
func Post(url string, val url.Values) (*http.Response, error) {
resp, err := http.PostForm(url, val)
if err != nil {
return nil, gos.DoError(err)
}
if resp.StatusCode != 200 {
return resp, gos.DoError(fmt.Sprintf("Post failed:%d", resp.StatusCode))
}
return resp, nil
}
func GetBody(url string) ([]byte, error) {
resp, err := Get(url)
if err != nil {
return nil, gos.DoError(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, gos.DoError(err)
}
return body, nil
}
func PostBody(url string, val url.Values) ([]byte, error) {
resp, err := Post(url, val)
if err != nil {
return nil, gos.DoError(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, gos.DoError(err)
}
return body, nil
}
// FillStockData 抓取数据
func FillPingGuData() {
// http://stockData.stock.hexun.com/600028.shtml
var stockID int64
var pinggu *PingGuStruct
var err error
ist := db.NewInsertBuilder("pinggu")
exists := db.NewExistsBuilder("pinggu")
dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
for _, data := range dataset {
if exists.Where("stock_id=? and date=?", data.GetInt64("id"), util.StockQuarter()).Exists() {
continue
}
pinggu, err = FetchPingGuData(data.GetString("code"))
if err != nil {
gos.DoError(err)
continue
}
stockID = data.GetInt64("id")
for _, row := range pinggu.Content {
row.StockID = stockID
ist.Insert(row)
}
fmt.Println("insert ", data.GetString("code"))
time.Sleep(1 * time.Second)
}
}
// ParseSumData 解析两市汇总信息
func (r RzrqStockJSONData) ParseSumData() ([]*RzrqStockData, error) {
if len(r) == 0 {
return nil, nil
}
var err error
var dataSet = make([]*RzrqStockData, 0)
var tmp []string
var itemData *RzrqStockData
var x float64
var val reflect.Value
var mapping *stockDataMapping
var isFirst = true
for _, item := range r {
tmp = strings.Split(item, ",")
itemData = &RzrqStockData{}
itemData.Date, err = time.Parse("2006/1/2", tmp[4])
if err != nil {
return nil, gos.DoError(err)
}
if isFirst {
itemData.Code = tmp[0]
itemData.Name = tmp[2]
isFirst = false
}
val = reflect.ValueOf(itemData).Elem()
for _, mapping = range rzrqStockDataMapping {
if tmp[mapping.Index] == "" || tmp[mapping.Index] == "-" {
continue
}
if x, err = strconv.ParseFloat(tmp[mapping.Index], 64); err != nil {
return nil, gos.DoError(err)
}
val.FieldByName(mapping.Name).SetFloat(x)
}
dataSet = append(dataSet, itemData)
}
return dataSet, nil
}
// FillYyb 抓取数据
func FillGgcg() error {
v, err := FetchGgcg(1)
if err != nil {
return err
}
var item *ggcgVO
l := v.Parse()
// create
for _, item = range l {
item.Init(item)
item.Create()
if err != nil {
gos.DoError(err)
}
}
time.Sleep(1 * time.Second)
pages := v.Pages
if pages > 20 {
pages = 20
}
for i := 2; i < pages+1; i++ {
v, err = FetchGgcg(i)
if err != nil {
return err
}
l = v.Parse()
for _, item = range l {
item.Init(item)
item.Create()
if err != nil {
gos.DoError(err)
}
}
time.Sleep(1 * time.Second)
}
return err
}
// Wget 抓取数据
func Get(url string) (*http.Response, error) {
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
req.Header.Add("User-Agent", `Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36`)
resp, err := client.Do(req)
if err != nil {
return nil, gos.DoError("error:", err)
} else if resp.Body == nil {
return nil, gos.DoError("error: resp.Body is empty")
}
if resp.StatusCode != 200 {
return resp, gos.DoError(fmt.Sprintf("Get failed:%d", resp.StatusCode))
}
// var OKPJKmpr={pages:0,data:[{stats:false}]}
return resp, nil
}
// RzrqSumData 抓取数据
func GetRzrqSumData() ([]*RzrqSumItemData, error) {
if isCached() {
log.App.Info("rzrq sum cached")
return sumdataCached, nil
}
src, err := FetchRzrqSumData(1)
if err != nil {
return nil, gos.DoError(err)
}
v := &RzrqSumJSONData{}
if err = json.Unmarshal(src, &v); err != nil {
return nil, gos.DoError(err)
}
sumdataCached, err = v.ParseSumData()
return sumdataCached, err
}
func main() {
data, err := cninfo.FetchFinancialData("600718", "2015-09-30")
if err != nil {
gos.DoError(err).LogError()
fmt.Println(err)
return
}
fmt.Println(data)
}
// ParseSumData 解析两市汇总信息
func (r RzrqSumJSONData) ParseSumData() ([]*RzrqSumItemData, error) {
if len(r) == 0 {
return nil, nil
}
var err error
var dataSet = make([]*RzrqSumItemData, 0)
var tmp []string
var itemData *RzrqSumItemData
var x int64
var val reflect.Value
for _, item := range r {
tmp = strings.Split(item, ",")
if len(tmp) != 13 {
return nil, fmt.Errorf("parse data error")
}
itemData = &RzrqSumItemData{}
itemData.Date, err = time.Parse("2006-01-02", tmp[0])
if err != nil {
return nil, gos.DoError(err)
}
val = reflect.ValueOf(itemData).Elem()
for _, mapping := range rzrqSumDataMapping {
if tmp[mapping.Index] == "-" || tmp[mapping.Index] == "" {
val.FieldByName(mapping.Name).SetInt(-1)
continue
}
if x, err = util.Str2Int64(tmp[mapping.Index]); err != nil {
return nil, gos.DoError(err)
}
val.FieldByName(mapping.Name).SetInt(x)
}
dataSet = append(dataSet, itemData)
}
return dataSet, nil
}
// FetchPingGuData 抓取行业评估数据
func FetchPingGuData(code string) (*PingGuStruct, error) {
// http://stockData.stock.hexun.com/600028.shtml
now := gos.NowInLocation()
formt := "http://pinggu.stock.hexun.com/DataProvider/StockFinanceNewFlash.ashx?&code=%s&s=%d"
body, err := wget.GetBody(fmt.Sprintf(formt, code, now.Unix()))
if err != nil {
return nil, gos.DoError(err)
}
v := &PingGuStruct{}
err = xml.Unmarshal(body, v)
if err != nil {
return nil, gos.DoError(err)
}
quarter := util.StockQuarter()
for _, item := range v.Content {
item.Date = quarter
switch item.Name {
case "综合能力":
item.Itype = 1
case "盈利能力":
item.Itype = 2
case "偿债能力":
item.Itype = 3
case "成长能力":
item.Itype = 4
case "资产经营":
item.Itype = 5
case "市场表现":
item.Itype = 6
case "投资收益":
item.Itype = 7
}
}
return v, nil
}
// FetchRzrqSumData 抓取数据
func FetchRzrqSumData(page int) ([]byte, error) {
st := gos.NowInLocation().Unix() / 30
formt := "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=%s&st=0&sr=1&p=%d&ps=50&js=var%%20ruOtumOo={pages:(pc),data:[(x)]}&rt=%d"
url := fmt.Sprintf(formt, "SHSZHSSUM", page, st)
body, err := wget.GetBody(url)
if err != nil {
return nil, gos.DoError(err)
}
return body[bytes.Index(body, []byte("[")) : len(body)-1], nil
}
// FillStockData 抓取数据
func FillStockData() {
// http://stockData.stock.hexun.com/600028.shtml
upd := db.NewUpdateBuilder("stock")
ist := db.NewInsertBuilder("")
q := db.NewQueryBuilder("")
dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
for _, data := range dataset {
if data.GetInt("industry_id") == 0 || data.GetInt("icb_id") == 0 {
industry, icb, err := FetchStockData(data.GetString("code"))
if err != nil {
gos.DoError(err)
continue
}
if data.GetInt("industry_id") == 0 {
ist.Table("industry").Insert(industry)
row, err := q.Table("industry").Where("name=?", industry.Name).QueryOne()
if err != nil {
gos.DoError(err)
} else if !row.Empty() {
upd.Where("id=?", data.GetInt64("id")).Update(db.DataRow{"industry_id": row.GetInt64("id")})
}
}
if data.GetInt("icb_id") == 0 {
ist.Table("icb").Insert(icb)
row, err := q.Table("icb").Where("name=?", icb.Name).QueryOne()
if err != nil {
gos.DoError(err)
} else if !row.Empty() {
upd.Where("id=?", data.GetInt64("id")).Update(db.DataRow{"icb_id": row.GetInt64("id")})
}
}
}
}
}
func (u *ImageModel) DeleteByName(name string) (bool, error) {
img, err := db.NewQueryBuilder(u.Table()).Where("name=?", name).QueryOne()
if err != nil {
return false, err
}
if img.Empty() {
return false, gos.DoError("没有找到相应的记录")
}
date := img.GetTime("created_at").Format("20060102")
filename := fmt.Sprint(name, img.GetString("ext"))
os.Remove(path.Join(gos.HomeDir, "/webroot/uploads/pictures", date, filename))
_, err = db.NewDeleteBuilder(u.Table()).Where("id=?", img.GetInt64("id")).Delete()
return true, err
}
func (a *ArticleApi) GetByCid(cid string) (db.DataRow, error) {
q := db.NewQueryBuilder("articles").Select("cid,user_id,title,html,status,created_at,updated_at")
row, err := q.Where("cid=?", cid).QueryOne()
if err != nil {
return nil, err
}
if row.Empty() {
return nil, gos.DoError("没有找到相应的记录")
}
row = row.Bytes2String()
user, err := user.NewUserModel().GetById(row.GetInt64("user_id"))
if err != nil {
return nil, err
}
user = user.Bytes2String()
row["user"] = user
return row, err
}
func (a *WriterApi) GetByCid(cid string) (db.DataRow, error) {
q := db.NewQueryBuilder("articles").Select("id,cid,user_id,title,body,status,created_at,updated_at")
row, err := q.Where("cid=?", cid).QueryOne()
if err != nil {
return nil, err
}
if row.Empty() {
return nil, gos.DoError("没有找到相应的记录")
}
row = row.Bytes2String()
q1 := db.NewQueryBuilder("images").Select("name,ext,to_char(created_at, 'YYYYMMDD') as date")
images, err := q1.Where("article_id=?", row.GetInt64("id")).Query()
if err != nil {
return nil, err
}
row["images"] = images.Bytes2String()
return row, err
}
// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=SHT&sty=SHTTMYE&rt=0.6341747129336
// curl "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=SHT&sty=SHTTMYE&rt=0.6341747129336" -H "Pragma: no-cache" -H "Accept-Encoding: gzip, deflate, sdch" -H "Accept-Language: en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36" -H "Accept: */*" -H "Referer: http://data.eastmoney.com/bkzj/hgt.html" -H "X-Requested-With: ShockwaveFlash/18.0.0.232" -H "Cookie: HAList=f-0-000003-B"%"u80A1"%"u6307"%"u6570; pgv_pvi=3511567662; pgv_info=ssi=s7100762258" -H "Connection: keep-alive" -H "Cache-Control: no-cache" --compressed
// RzrqSumData 抓取数据
func GetHgtAmount() ([]*HgtAmount, error) {
r := rand.New(rand.NewSource(99))
formt := "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=SHT&sty=SHTTMYE&rt=%v"
body, err := wget.GetBody(fmt.Sprintf(formt, r.Float64()))
if err != nil {
return nil, gos.DoError(err)
}
now := gos.NowInLocation()
arr := bytes.Split(body, []byte("\r\n"))
var tmp [][]byte
result := make([]*HgtAmount, len(arr))
var amountA float64
var amountB float64
var date time.Time
for i, v := range arr {
tmp = bytes.Split(v, gos.B_SEMICOLON)
amountA, err = util.Str2Float64(string(tmp[1]))
if err != nil {
amountA = 0
}
amountB, err = util.Str2Float64(string(tmp[2]))
if err != nil {
amountA = 0
}
date, err = time.ParseInLocation("2006/1/2 15:04", fmt.Sprint(now.Format("2006/1/2"), " ", string(tmp[0])), gos.GetSite().Location)
if err != nil {
date = now
}
result[i] = &HgtAmount{
Date: date,
AmountA: amountA,
AmountB: amountB,
}
}
return result, nil
}
func (a *WriterApi) Save(args util.MapData) (string, error) {
cid := args.GetString("cid")
title := args.GetString("title")
body := args.GetBytes("body")
images := args.GetStringSlice("images")
au := a.GetUserAuth()
var err error
model := article.NewArticleModel()
if cid == "" {
cid, err = model.New(au.UserID(), title, body)
} else {
_, err = model.Update(cid, title, body)
}
if err != nil {
return cid, err
}
article, _ := db.NewQueryBuilder(model.Table()).Select("id").Where("cid=?", cid).QueryOne()
if article.Empty() {
return cid, gos.DoError("没有找到相应的记录")
}
count := len(images)
if count > 0 {
image := db.DataRow{}
image["article_id"] = article.GetInt64("id")
for i := 0; i < count; i++ {
images[i] = fmt.Sprint("'", images[i], "'")
}
db.NewUpdateBuilder("images").
Where(fmt.Sprint("name in (", strings.Join(images, ","), ")")).
Update(image)
}
return cid, nil
}
func (a *ArticleApi) List(args util.MapData) (db.DataSet, error) {
nick := args.GetString("nick")
page := args.GetInt("page")
q := db.NewQueryBuilder("articles").
Page(page, 20).
Order("id desc").
Select("cid,title,description,status,created_at")
if nick != "" {
u, err := user.NewUserModel().GetByNickAllField(nick)
if err != nil {
return nil, err
}
if u.Empty() {
return nil, gos.DoError("没有找到相应的记录", nick)
}
q.Where("user_id=?", u.GetInt64("id"))
}
ds, err := q.Query()
return ds.Bytes2String(), err
}
func RunAlertHgt() {
var errCh = make(chan error)
c := time.Tick(1 * time.Minute)
conf := gos.Configuration.GetConf("other")
minute := conf.GetInt("hgt_check_minute")
amount := conf.GetFloat("hgt_check_amount")
go func() {
for range c {
errCh <- alertAtHgtChanged(minute, amount)
}
}()
go func() {
for {
select {
case err := <-errCh:
if err != nil {
gos.DoError(err)
}
}
}
}()
}
// FetchRzrqStockData 抓取数据
func FetchRzrqStockData(code string, page int) ([]byte, error) {
st := time.Now().Unix() / 30
// var% OKPJKmpr={pages:10,data:
// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=1&code=600718&st=0&sr=1&p=5&ps=50&js=var%20OKPJKmpr={pages:(pc),data:[(x)]}&rt=48027423
// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=2&code=002161&st=0&sr=1&p=5&ps=50&js=var%20QOwpoBrj={pages:(pc),data:[(x)]}&rt=48154207
// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=2&code=300001&st=0&sr=1&p=5&ps=50&js=var%20laheCxDp={pages:(pc),data:[(x)]}&rt=48154224
formt := "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=%d&code=%s&st=0&sr=1&p=%d&ps=50&js=var%%20OKPJKmpr={pages:(pc),data:[(x)]}&rt=%d"
mkt := 2
switch code[0:1] {
case "6":
mkt = 1
}
url := fmt.Sprintf(formt, mkt, code, page, st)
body, err := wget.GetBody(url)
if err != nil {
return nil, gos.DoError(err)
}
// var OKPJKmpr={pages:0,data:[{stats:false}]}
return body[bytes.Index(body, []byte("[")) : len(body)-1], nil
}
// FetchRzrqSumData 抓取数据
func FetchCwfx(code, ctype string) ([]*Cwzb, []*Zcfzb, []*Lrb, []*Xjllb, []*FinPercent, error) {
formt := "http://f10.eastmoney.com/f10_v2/FinanceAnalysis.aspx?code=%s%s"
resp, err := wget.Get(fmt.Sprintf(formt, ctype, code))
if err != nil {
return nil, nil, nil, nil, nil, gos.DoError(err)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, nil, nil, nil, nil, gos.DoError(err)
}
cwzbList := make([]*Cwzb, 0)
zcfzbList := make([]*Zcfzb, 0)
lrbList := make([]*Lrb, 0)
xjllbList := make([]*Xjllb, 0)
finPercentList := make([]*FinPercent, 0)
index := 0
doc.Find("#F10MainTargetDiv table tr").Each(func(i int, tr *goquery.Selection) {
if i == 0 {
tr.Find("th.tips-fieldname-Right").Each(func(k int, th *goquery.Selection) {
cwzbList = append(cwzbList, &Cwzb{Date: fmt.Sprint("20", th.Text())})
})
return
}
if _, isOk := tr.Attr("onclick"); isOk {
tr.Find("td.tips-data-Right").Each(func(k int, td *goquery.Selection) {
reflect.ValueOf(cwzbList[k]).
Elem().Field(index + 1).SetFloat(util.ParseMoneyCN(td.Text()))
})
index++
}
})
label := ""
trlist := doc.Find("#BBMX_table tr")
if trlist.Length() == 60 {
trlist.Each(func(i int, tr *goquery.Selection) {
if i == 0 {
tr.Find("th.tips-fieldname-Right").Each(func(k int, th *goquery.Selection) {
date := fmt.Sprint("20", th.Text())
zcfzbList = append(zcfzbList, &Zcfzb{Date: date})
lrbList = append(lrbList, &Lrb{Date: date})
xjllbList = append(xjllbList, &Xjllb{Date: date})
})
label = "资产负债表"
index = 0
return
}
if tr.Find("th.tips-colname-Left").Length() > 0 {
label = tr.Find("th").First().Text()
index = 0
return
}
tr.Find("td.tips-data-Right").Each(func(k int, td *goquery.Selection) {
switch label {
case "资产负债表":
// fmt.Println(label, util.ParseMoneyCN(td.Text()), index, k)
reflect.ValueOf(zcfzbList[k]).
Elem().Field(index + 1).SetFloat(util.ParseMoneyCN(td.Text()))
case "利润表":
reflect.ValueOf(lrbList[k]).
Elem().Field(index + 1).SetFloat(util.ParseMoneyCN(td.Text()))
case "现金流量表":
reflect.ValueOf(xjllbList[k]).
Elem().Field(index + 1).SetFloat(util.ParseMoneyCN(td.Text()))
}
})
index++
})
}
trlist = doc.Find("#PPTable tr")
if trlist.Length() == 20 {
trlist.Each(func(i int, tr *goquery.Selection) {
if i == 0 {
tr.Find("th.tips-dataC").Each(func(k int, th *goquery.Selection) {
finPercentList = append(finPercentList, &FinPercent{Date: fmt.Sprint("20", th.Text())})
})
return
}
if i == 1 {
return
}
index = 0
tr.Find("td.tips-data-Right").Each(func(k int, td *goquery.Selection) {
if k%2 == 0 {
return
}
reflect.ValueOf(finPercentList[index]).
Elem().Field(i - 1).SetFloat(util.ParsePercent(td.Text()))
index++
})
})
}
return cwzbList, zcfzbList, lrbList, xjllbList, finPercentList, nil
}
// FillCwfx 抓取并且填充数据
func FillCwfx(lastDate string) {
ist := db.NewInsertBuilder("fin_cwzb")
exist := db.NewExistsBuilder("fin_cwzb")
dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
for i, data := range dataset {
fmt.Println("--", i, "--", data.GetString("code"), data.GetString("ctype"))
if exist.Where("stock_id=? and date=?", data.GetInt64("id"), lastDate).Exists() {
gos.Log.Info("exists", lastDate, "return")
continue
}
cwzbList, zcfzbList, lrbList, xjllbList, finPercentList, err := FetchCwfx(data.GetString("code"), data.GetString("ctype"))
if err != nil {
gos.DoError(err)
continue
}
if len(cwzbList) == 0 {
continue
}
for _, row := range cwzbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_cwzb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_cwzb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_cwzb", row.Date)
ist.Table("fin_cwzb").Insert(row)
}
for _, row := range zcfzbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_zcfzb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_zcfzb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_zcfzb", row.Date)
ist.Table("fin_zcfzb").Insert(row)
}
for _, row := range lrbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_lrb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
fmt.Println("exists fin_lrb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_lrb", row.Date)
ist.Table("fin_lrb").Insert(row)
}
for _, row := range xjllbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_xjllb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_xjllb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_xjllb", row.Date)
ist.Table("fin_xjllb").Insert(row)
}
for _, row := range finPercentList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_percent").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_percent", row.Date, "return")
continue
}
gos.Log.Info("insert fin_percent", row.Date)
ist.Table("fin_percent").Insert(row)
}
if len(cwzbList) > 0 {
time.Sleep(1 * time.Second)
}
}
}
// FetchDzjy 抓取数据
func FetchDzjy(date time.Time) ([]*dzjyVO, error) {
formt := "http://data.eastmoney.com/dzjy/%s.html"
resp, err := wget.Get(fmt.Sprintf(formt, date.Format("200601")))
if err != nil {
return nil, gos.DoError(err)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, gos.DoError(err)
}
var td *goquery.Selection
var dzjy *dzjyVO
var dateStr string
var stockCode string
var row db.DataRow
query := db.NewQueryBuilder("stock")
datalist := make([]*dzjyVO, 0)
buy := ""
sell := ""
var priceNow float64
var price float64
var amount float64
var total float64
var length int
doc.Find("#content div.list").Eq(2).Find("table tr.list_eve").Each(func(i int, tr *goquery.Selection) {
td = tr.Find("td")
length = td.Length()
if length == 10 {
dateStr = td.Eq(0).Text()
stockCode = td.Eq(1).Text()
} else if length == 9 {
stockCode = td.Eq(0).Text()
}
row, _ = query.Where("code=?", stockCode).QueryOne()
if row.Empty() {
return
}
switch length {
case 10:
priceNow = util.ParseMoney(td.Eq(4).Text())
price = util.ParseMoney(td.Eq(5).Text())
amount = util.ParseMoney(td.Eq(6).Text())
total = util.ParseMoney(td.Eq(7).Text())
buy, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
if err != nil {
return
}
sell, err = iconv.ConvertString(td.Eq(9).Text(), "gb2312", "utf-8")
if err != nil {
return
}
case 9:
priceNow = util.ParseMoney(td.Eq(3).Text())
price = util.ParseMoney(td.Eq(4).Text())
amount = util.ParseMoney(td.Eq(5).Text())
total = util.ParseMoney(td.Eq(6).Text())
buy, err = iconv.ConvertString(td.Eq(7).Text(), "gb2312", "utf-8")
if err != nil {
return
}
sell, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
if err != nil {
return
}
case 5:
price = util.ParseMoney(td.Eq(0).Text())
amount = util.ParseMoney(td.Eq(1).Text())
total = util.ParseMoney(td.Eq(2).Text())
buy, err = iconv.ConvertString(td.Eq(3).Text(), "gb2312", "utf-8")
if err != nil {
return
}
sell, err = iconv.ConvertString(td.Eq(4).Text(), "gb2312", "utf-8")
if err != nil {
return
}
default:
return
}
dzjy = &dzjyVO{
StockID: row.GetInt64("id"),
Date: dateStr,
PriceNow: priceNow,
Price: price,
Amount: amount,
Total: total,
Buy: buy,
Sell: sell,
}
datalist = append(datalist, dzjy)
})
return datalist, nil
}
func (this *ImageUpload) DoUpload() {
defer func() {
if r := recover(); r != nil {
gos.DoError(r)
}
}()
now := time.Now()
nowStr := now.Format("20060102")
tagName := "file"
tempPath := path.Join(gos.HomeDir, this.StoreRoot, "/temp")
storePath := path.Join(gos.HomeDir, this.StoreRoot, "/pictures/", nowStr)
if this.Ctx.Request.Method == "GET" {
if err := this.ResumableJsTest(storePath); err == nil {
this.ResumableJsCheckAndMeger(storePath, tempPath)
this.Ctx.ResponseWriter.WriteHeader(200)
} else {
this.Ctx.ResponseWriter.WriteHeader(204)
}
return
}
websvr := &gos.WebApi{Ctx: this.Ctx}
reply := map[string]string{}
sf, err := this.ResumableJs(tagName, storePath, tempPath)
if err != nil {
gos.DoError(err)
websvr.Reply(gos.DoError(err).Data(), nil)
return
}
// sf.Store()
if sf == nil {
reply["filename"] = ""
reply["ext"] = ""
reply["url"] = ""
} else {
reply["filename"] = sf.StoreName
reply["ext"] = sf.Ext
folder := fmt.Sprint("/uploads/pictures/", nowStr, "/")
reply["url"] = fmt.Sprint(folder, reply["filename"], "-m", sf.Ext)
// err = resizeImage(storePath, sf.StoreName, sf.Ext)
from := path.Join(gos.HomeDir, "webroot", folder, fmt.Sprint(reply["filename"], sf.Ext))
to := path.Join(gos.HomeDir, "webroot", folder, fmt.Sprint(reply["filename"], "-m", sf.Ext))
cmdStr := fmt.Sprint("convert -resize 800x600 ", from, " ", to)
cmd := exec.Command("/bin/sh", "-c", cmdStr)
_, err := cmd.Output()
if err != nil {
gos.DoError(err, cmdStr)
}
au := auth.New(this.Ctx)
row := db.DataRow{}
row["name"] = sf.StoreName
row["ext"] = sf.Ext
row["user_id"] = au.UserID()
row["created_at"] = now
_, err = db.NewInsertBuilder("images").Insert(row)
if err != nil {
websvr.Reply(gos.DoError(err).Data(), nil)
return
}
}
websvr.Reply(reply, nil)
}