// FillStockData 抓取数据
func FillPingGuData() {
// http://stockData.stock.hexun.com/600028.shtml
var stockID int64
var pinggu *PingGuStruct
var err error
ist := db.NewInsertBuilder("pinggu")
exists := db.NewExistsBuilder("pinggu")
dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
for _, data := range dataset {
if exists.Where("stock_id=? and date=?", data.GetInt64("id"), util.StockQuarter()).Exists() {
continue
}
pinggu, err = FetchPingGuData(data.GetString("code"))
if err != nil {
gos.DoError(err)
continue
}
stockID = data.GetInt64("id")
for _, row := range pinggu.Content {
row.StockID = stockID
ist.Insert(row)
}
fmt.Println("insert ", data.GetString("code"))
time.Sleep(1 * time.Second)
}
}
func (a *WriterApi) GetByCid(cid string) (db.DataRow, error) {
q := db.NewQueryBuilder("articles").Select("id,cid,user_id,title,body,status,created_at,updated_at")
row, err := q.Where("cid=?", cid).QueryOne()
if err != nil {
return nil, err
}
if row.Empty() {
return nil, gos.DoError("没有找到相应的记录")
}
row = row.Bytes2String()
q1 := db.NewQueryBuilder("images").Select("name,ext,to_char(created_at, 'YYYYMMDD') as date")
images, err := q1.Where("article_id=?", row.GetInt64("id")).Query()
if err != nil {
return nil, err
}
row["images"] = images.Bytes2String()
return row, err
}
// FillStockData 抓取数据
func FillStockData() {
// http://stockData.stock.hexun.com/600028.shtml
upd := db.NewUpdateBuilder("stock")
ist := db.NewInsertBuilder("")
q := db.NewQueryBuilder("")
dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
for _, data := range dataset {
if data.GetInt("industry_id") == 0 || data.GetInt("icb_id") == 0 {
industry, icb, err := FetchStockData(data.GetString("code"))
if err != nil {
gos.DoError(err)
continue
}
if data.GetInt("industry_id") == 0 {
ist.Table("industry").Insert(industry)
row, err := q.Table("industry").Where("name=?", industry.Name).QueryOne()
if err != nil {
gos.DoError(err)
} else if !row.Empty() {
upd.Where("id=?", data.GetInt64("id")).Update(db.DataRow{"industry_id": row.GetInt64("id")})
}
}
if data.GetInt("icb_id") == 0 {
ist.Table("icb").Insert(icb)
row, err := q.Table("icb").Where("name=?", icb.Name).QueryOne()
if err != nil {
gos.DoError(err)
} else if !row.Empty() {
upd.Where("id=?", data.GetInt64("id")).Update(db.DataRow{"icb_id": row.GetInt64("id")})
}
}
}
}
}
func main() {
c := conf.Load("../../app/app.conf")
db.Init("app", c.GetConf("db").MapData())
db.Current().Log = log.NewEmpty()
lastDate := "2015-09-30"
ist := db.NewInsertBuilder("fin_dubang")
exist := db.NewExistsBuilder("fin_dubang")
dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
isIst := false
for i, data := range dataset {
fmt.Println("--", i, "--", data.GetString("code"))
if exist.Where("stock_id=? and date=?", data.GetInt64("id"), lastDate).Exists() {
fmt.Println("exists", lastDate, "return")
continue
}
dubangList, err := dfcf.FetchDubang(data.GetString("code"), data.GetString("ctype"))
if err != nil {
fmt.Println(err)
continue
}
if len(dubangList) == 0 {
continue
}
isIst = false
for _, dubang := range dubangList {
dubang.StockId = data.GetInt64("id")
if exist.Where("stock_id=? and date=?", dubang.StockId, dubang.Date).Exists() {
fmt.Println("exists", dubang.Date, "return")
continue
}
fmt.Println("insert", dubang.Date)
ist.Insert(dubang)
isIst = true
}
if isIst {
time.Sleep(1 * time.Second)
}
}
}
func (u *ImageModel) DeleteByName(name string) (bool, error) {
img, err := db.NewQueryBuilder(u.Table()).Where("name=?", name).QueryOne()
if err != nil {
return false, err
}
if img.Empty() {
return false, gos.DoError("没有找到相应的记录")
}
date := img.GetTime("created_at").Format("20060102")
filename := fmt.Sprint(name, img.GetString("ext"))
os.Remove(path.Join(gos.HomeDir, "/webroot/uploads/pictures", date, filename))
_, err = db.NewDeleteBuilder(u.Table()).Where("id=?", img.GetInt64("id")).Delete()
return true, err
}
func (s *jsonGgcgData) Parse() []*ggcgVO {
dataset := make([]*ggcgVO, 0)
query := db.NewQueryBuilder("stock")
upd := db.NewUpdateBuilder("stock")
var tmp []string
var arr []string
var row db.DataRow
var ggcg *ggcgVO
for _, item := range s.Data {
arr = strings.Split(item, ",")
tmp = strings.Split(strings.ToLower(arr[15]), ".")
row, _ = query.Where("code=? and ctype=?", tmp[0], tmp[1]).QueryOne()
if row.Empty() {
continue
}
upd.Where("id=?", row.GetInt64("id")).Update(db.DataRow{"name": arr[9]})
// "0.00444,谢飞鹏,002420,谢杏思,A股,2015-11-16,-17800, 0 ,9.44, 毅昌股份,父母,YCGF,竞价交易,-168032,董事,002420.SZ"
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// "0.00025,邓伦明,002539,邓伦德,A股,2015-11-17,-2500, 0, 14.65, 新都化工,兄弟姐妹,XDHG,竞价交易,-36625,监事,002539.SZ",
// "0.00012,马东杰,002771,马东伟,A股,2015-11-16, 100, 0, 129.12, 真视通,兄弟姐妹,ZST,竞价交易,12912,监事,002771.SZ",
// "0.00012,马东杰,002771,马东伟,A股,2015-11-16, -100, 0, 129.8, 真视通,兄弟姐妹,ZST,竞价交易,-12980,监事,002771.SZ",
// "0.00628,郝先进,002690,郝先进,A股,2015-11-16,42430,23581180,30.26,美亚光电,本人,MYGD,竞价交易,1283931.8,董事、高管,002690.SZ"
ggcg = &ggcgVO{
StockID: row.GetInt64("id"),
Date: arr[5],
Person: arr[3],
Price: util.ParseMoney(arr[8]),
Amount: util.ParseMoney(arr[6]),
Total: util.ParseMoney(arr[13]),
Remain: int64(util.ParseMoney(arr[7])),
Proportion: util.ParseMoney(arr[0]),
Reason: arr[12],
PersonRel: arr[1],
PersonRelType: arr[10],
Job: arr[14],
}
dataset = append(dataset, ggcg)
}
return dataset
}
func (a *ArticleApi) GetByCid(cid string) (db.DataRow, error) {
q := db.NewQueryBuilder("articles").Select("cid,user_id,title,html,status,created_at,updated_at")
row, err := q.Where("cid=?", cid).QueryOne()
if err != nil {
return nil, err
}
if row.Empty() {
return nil, gos.DoError("没有找到相应的记录")
}
row = row.Bytes2String()
user, err := user.NewUserModel().GetById(row.GetInt64("user_id"))
if err != nil {
return nil, err
}
user = user.Bytes2String()
row["user"] = user
return row, err
}
func (a *WriterApi) Save(args util.MapData) (string, error) {
cid := args.GetString("cid")
title := args.GetString("title")
body := args.GetBytes("body")
images := args.GetStringSlice("images")
au := a.GetUserAuth()
var err error
model := article.NewArticleModel()
if cid == "" {
cid, err = model.New(au.UserID(), title, body)
} else {
_, err = model.Update(cid, title, body)
}
if err != nil {
return cid, err
}
article, _ := db.NewQueryBuilder(model.Table()).Select("id").Where("cid=?", cid).QueryOne()
if article.Empty() {
return cid, gos.DoError("没有找到相应的记录")
}
count := len(images)
if count > 0 {
image := db.DataRow{}
image["article_id"] = article.GetInt64("id")
for i := 0; i < count; i++ {
images[i] = fmt.Sprint("'", images[i], "'")
}
db.NewUpdateBuilder("images").
Where(fmt.Sprint("name in (", strings.Join(images, ","), ")")).
Update(image)
}
return cid, nil
}
func (a *ArticleApi) List(args util.MapData) (db.DataSet, error) {
nick := args.GetString("nick")
page := args.GetInt("page")
q := db.NewQueryBuilder("articles").
Page(page, 20).
Order("id desc").
Select("cid,title,description,status,created_at")
if nick != "" {
u, err := user.NewUserModel().GetByNickAllField(nick)
if err != nil {
return nil, err
}
if u.Empty() {
return nil, gos.DoError("没有找到相应的记录", nick)
}
q.Where("user_id=?", u.GetInt64("id"))
}
ds, err := q.Query()
return ds.Bytes2String(), err
}
func (u *UserModel) GetById(id int64) (db.DataRow, error) {
return db.NewQueryBuilder(u.Table()).Cache().
Select(userFields).
Where("id=?", id).QueryOne()
}
func (a *UserAuth) QueryById(id int64) db.DataRow {
r, _ := db.NewQueryBuilder(a.GetOptions().Table).Where("id=?", id).CacheExpire(a.GetExpire()).QueryOne()
return r
}
func (a *UserAuth) ClearCache() {
a.ClearCacheByKeys([]string{"nick"}, []interface{}{a.User().GetString("nick")})
db.NewQueryBuilder(a.GetOptions().Table).Where("id=?", a.UserId()).ClearCache()
}
// FillCwfx 抓取并且填充数据
func FillCwfx(lastDate string) {
ist := db.NewInsertBuilder("fin_cwzb")
exist := db.NewExistsBuilder("fin_cwzb")
dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
for i, data := range dataset {
fmt.Println("--", i, "--", data.GetString("code"), data.GetString("ctype"))
if exist.Where("stock_id=? and date=?", data.GetInt64("id"), lastDate).Exists() {
gos.Log.Info("exists", lastDate, "return")
continue
}
cwzbList, zcfzbList, lrbList, xjllbList, finPercentList, err := FetchCwfx(data.GetString("code"), data.GetString("ctype"))
if err != nil {
gos.DoError(err)
continue
}
if len(cwzbList) == 0 {
continue
}
for _, row := range cwzbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_cwzb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_cwzb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_cwzb", row.Date)
ist.Table("fin_cwzb").Insert(row)
}
for _, row := range zcfzbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_zcfzb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_zcfzb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_zcfzb", row.Date)
ist.Table("fin_zcfzb").Insert(row)
}
for _, row := range lrbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_lrb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
fmt.Println("exists fin_lrb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_lrb", row.Date)
ist.Table("fin_lrb").Insert(row)
}
for _, row := range xjllbList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_xjllb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_xjllb", row.Date, "return")
continue
}
gos.Log.Info("insert fin_xjllb", row.Date)
ist.Table("fin_xjllb").Insert(row)
}
for _, row := range finPercentList {
row.StockID = data.GetInt64("id")
if exist.Table("fin_percent").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
gos.Log.Info("exists fin_percent", row.Date, "return")
continue
}
gos.Log.Info("insert fin_percent", row.Date)
ist.Table("fin_percent").Insert(row)
}
if len(cwzbList) > 0 {
time.Sleep(1 * time.Second)
}
}
}
// FetchDzjy 抓取数据
func FetchDzjy(date time.Time) ([]*dzjyVO, error) {
formt := "http://data.eastmoney.com/dzjy/%s.html"
resp, err := wget.Get(fmt.Sprintf(formt, date.Format("200601")))
if err != nil {
return nil, gos.DoError(err)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, gos.DoError(err)
}
var td *goquery.Selection
var dzjy *dzjyVO
var dateStr string
var stockCode string
var row db.DataRow
query := db.NewQueryBuilder("stock")
datalist := make([]*dzjyVO, 0)
buy := ""
sell := ""
var priceNow float64
var price float64
var amount float64
var total float64
var length int
doc.Find("#content div.list").Eq(2).Find("table tr.list_eve").Each(func(i int, tr *goquery.Selection) {
td = tr.Find("td")
length = td.Length()
if length == 10 {
dateStr = td.Eq(0).Text()
stockCode = td.Eq(1).Text()
} else if length == 9 {
stockCode = td.Eq(0).Text()
}
row, _ = query.Where("code=?", stockCode).QueryOne()
if row.Empty() {
return
}
switch length {
case 10:
priceNow = util.ParseMoney(td.Eq(4).Text())
price = util.ParseMoney(td.Eq(5).Text())
amount = util.ParseMoney(td.Eq(6).Text())
total = util.ParseMoney(td.Eq(7).Text())
buy, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
if err != nil {
return
}
sell, err = iconv.ConvertString(td.Eq(9).Text(), "gb2312", "utf-8")
if err != nil {
return
}
case 9:
priceNow = util.ParseMoney(td.Eq(3).Text())
price = util.ParseMoney(td.Eq(4).Text())
amount = util.ParseMoney(td.Eq(5).Text())
total = util.ParseMoney(td.Eq(6).Text())
buy, err = iconv.ConvertString(td.Eq(7).Text(), "gb2312", "utf-8")
if err != nil {
return
}
sell, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
if err != nil {
return
}
case 5:
price = util.ParseMoney(td.Eq(0).Text())
amount = util.ParseMoney(td.Eq(1).Text())
total = util.ParseMoney(td.Eq(2).Text())
buy, err = iconv.ConvertString(td.Eq(3).Text(), "gb2312", "utf-8")
if err != nil {
return
}
sell, err = iconv.ConvertString(td.Eq(4).Text(), "gb2312", "utf-8")
if err != nil {
return
}
default:
return
}
dzjy = &dzjyVO{
StockID: row.GetInt64("id"),
Date: dateStr,
PriceNow: priceNow,
Price: price,
Amount: amount,
Total: total,
Buy: buy,
Sell: sell,
}
datalist = append(datalist, dzjy)
})
return datalist, nil
}
func fetchAndFillFundRank(page int, quarter string) (int, error) {
// http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=50&date=2015-09-30&orgType=&stateType=null&titType=null&page=2&callback=hxbase_json7
formt := "http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=%s&date=%s&orgType=&stateType=null&titType=null&page=%d&callback=hxbase_json7"
pageLimit := 100
body, err := wget.GetBody(fmt.Sprintf(formt, pageLimit, quarter, page))
if err != nil {
return -1, err
}
exists := db.NewExistsBuilder("funds")
ist := db.NewInsertBuilder("funds")
query := db.NewQueryBuilder("funds")
var fund *Fund
var fundRank *FundRank
var row db.DataRow
// hxbase_json7(
str, err := iconv.ConvertString(string(body), "gb2312", "utf-8")
if err != nil {
return -1, err
}
src := []byte(str)
src = src[13 : len(src)-1]
src = bytes.Replace(src, []byte(":'"), []byte(`":"`), -1)
src = bytes.Replace(src, []byte("',"), []byte(`","`), -1)
src = bytes.Replace(src, []byte("'}"), []byte(`"}`), -1)
src = bytes.Replace(src, []byte("{"), []byte(`{"`), -1)
src = bytes.Replace(src, []byte("sum:"), []byte(`sum":`), 1)
src = bytes.Replace(src, []byte("list:"), []byte(`"list":`), 1)
v := &JsonFund{}
err = json.Unmarshal(src, v)
if err != nil {
fmt.Println(string(src))
return -1, err
}
// {RankTd:'51',OrgName:'法国巴黎银行',OrgNameLink:'o-QF000031.shtml',OrgType:'QFII',ShareHoldingNum:'3',ShareHoldingNumLink:'otherDetail.aspx?OrgNo=QF000031',TotalHoldings:'48,388.00',TotalMarketValue:'1,100,735.00',OrgAlt:'法国巴黎银行'}
for _, item := range v.List {
fund = &Fund{Code: item["OrgNameLink"], Name: string(item["OrgName"]), TypeID: FundType(item["OrgType"])}
fund.Code = fund.Code[2 : len(fund.Code)-6]
fmt.Println("----")
if !exists.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).Exists() {
fmt.Println("insert", fund)
ist.Table("funds").Insert(fund)
}
row, _ = query.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).QueryOne()
if row.Empty() {
return -1, fmt.Errorf("code %s not found", fund.Code)
}
fundRank = &FundRank{
FundID: row.GetInt64("id"),
Date: quarter,
Rank: int(util.ParseMoney(item["RankTd"])),
Count: int(util.ParseMoney(item["ShareHoldingNum"])),
MH: int64(util.ParseMoney(item["TotalHoldings"])),
MV: int64(util.ParseMoney(item["TotalMarketValue"])),
}
if !exists.Table("fund_rank").Where("fund_id=? and date=?", fundRank.FundID, fundRank.Date).Exists() {
fmt.Println("insert", fundRank)
ist.Table("fund_rank").Insert(fundRank)
}
}
return int(math.Ceil(float64(v.Sum / pageLimit))), nil
}
// fetchAndFillYyb 抓取数据
func fetchAndFillYyb(page int) (int, error) {
// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=2&ps=50&js=var%20XvAVhGPE={%22data%22:[(x)],%22pages%22:%22(pc)%22,%22update%22:%22(ud)%22}&rt=48257541
formt := `http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=%d&ps=%d&js=var%%20XvAVhGPE={"data":[(x)],"pages":"(pc)","update":"(ud)"}&rt=%d`
pageLimit := 100
body, err := wget.GetBody(fmt.Sprintf(formt, page, pageLimit, time.Now().Unix()))
if err != nil {
return -1, err
}
exists := db.NewExistsBuilder("yyb")
ist := db.NewInsertBuilder("yyb")
upd := db.NewUpdateBuilder("yyb")
query := db.NewQueryBuilder("yyb")
var yyb *yybStruct
var yybLub *yybLhbStruct
var arr []string
var row db.DataRow
src := body[bytes.Index(body, []byte("={"))+1:]
v := &jsonYybData{}
err = json.Unmarshal(src, v)
if err != nil {
return -1, err
}
// "80136686,2,216,135,七喜控股|协鑫集成|恒信移动,2991067145.735,广东,80000073,440000,105,6089657316.805,3098590171.07,华泰证券股份有限公司广州天河东路证券营业部,002027.SZ|002506.SZ|300081.SZ"
for i, item := range v.Data {
arr = strings.Split(item, ",")
yyb = &yybStruct{Code: arr[0], Name: arr[12], Area: arr[6]}
if !exists.Table("yyb").Where("code=?", yyb.Code).Exists() {
gos.Log.Info("insert", yyb.Code)
ist.Table("yyb").Insert(yyb)
}
row, _ = query.Table("yyb").Where("code=?", yyb.Code).QueryOne()
if row.Empty() {
return -1, fmt.Errorf("code %s not found", yyb.Code)
}
yybLub = &yybLhbStruct{
YybID: row.GetInt64("id"),
Updated: v.Update,
Rank: pageLimit*(page-1) + i + 1,
Amount: util.ParseMoney(arr[10]),
Buy: util.ParseMoney(arr[11]),
Sell: util.ParseMoney(arr[5]),
Num: int(util.ParseMoney(arr[2])),
NumBuy: int(util.ParseMoney(arr[3])),
NumSell: int(util.ParseMoney(arr[9])),
}
if exists.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Exists() {
gos.Log.Info("update", yyb.Code, yybLub.YybID)
upd.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Update(yybLub)
} else {
gos.Log.Info("insert", yyb.Code, yybLub.YybID)
ist.Table("yyb_lhb").Insert(yybLub)
}
}
return int(util.ParseMoney(v.Pages)), nil
}
func (u *UserModel) GetByNickAllField(nick string) (db.DataRow, error) {
return db.NewQueryBuilder(u.Table()).Cache().
Where("nick=?", nick).QueryOne()
}