Example #1
0
// FillStockData 抓取数据
func FillPingGuData() {
	// http://stockData.stock.hexun.com/600028.shtml
	var stockID int64
	var pinggu *PingGuStruct
	var err error

	ist := db.NewInsertBuilder("pinggu")
	exists := db.NewExistsBuilder("pinggu")

	dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()

	for _, data := range dataset {
		if exists.Where("stock_id=? and date=?", data.GetInt64("id"), util.StockQuarter()).Exists() {
			continue
		}
		pinggu, err = FetchPingGuData(data.GetString("code"))
		if err != nil {
			gos.DoError(err)
			continue
		}
		stockID = data.GetInt64("id")
		for _, row := range pinggu.Content {
			row.StockID = stockID
			ist.Insert(row)
		}
		fmt.Println("insert ", data.GetString("code"))
		time.Sleep(1 * time.Second)
	}
}
Example #2
0
func main() {
	c := conf.Load("../../app/app.conf")
	db.Init("app", c.GetConf("db").MapData())
	db.Current().Log = log.NewEmpty()

	lastDate := "2015-09-30"
	ist := db.NewInsertBuilder("fin_dubang")
	exist := db.NewExistsBuilder("fin_dubang")
	dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()
	isIst := false

	for i, data := range dataset {
		fmt.Println("--", i, "--", data.GetString("code"))
		if exist.Where("stock_id=? and date=?", data.GetInt64("id"), lastDate).Exists() {
			fmt.Println("exists", lastDate, "return")
			continue
		}

		dubangList, err := dfcf.FetchDubang(data.GetString("code"), data.GetString("ctype"))
		if err != nil {
			fmt.Println(err)
			continue
		}

		if len(dubangList) == 0 {
			continue
		}

		isIst = false
		for _, dubang := range dubangList {
			dubang.StockId = data.GetInt64("id")

			if exist.Where("stock_id=? and date=?", dubang.StockId, dubang.Date).Exists() {
				fmt.Println("exists", dubang.Date, "return")
				continue
			}

			fmt.Println("insert", dubang.Date)
			ist.Insert(dubang)
			isIst = true
		}

		if isIst {
			time.Sleep(1 * time.Second)
		}
	}

}
Example #3
0
func Regist(cipher string) error {
	_, b, err := gos.PraseCipher([]byte(cipher))
	if err != nil {
		return gos.NewError(0, err)
	}

	arr := bytes.Split(b, separator)

	obj := db.DataRow{}
	obj["nick"] = string(arr[0])
	obj["status"] = 1
	obj["created_at"] = time.Now()
	obj["salt"], obj["token"] = UserToken(string(arr[0]), arr[1])
	_, err = db.NewInsertBuilder("users").Insert(obj)
	return err
}
Example #4
0
func (u *ArticleModel) New(userId int64, title string, body []byte) (string, error) {
	cid := util.Unique()
	vo := NewArticleVO(userId, title, body)

	vo.Cid = cid
	vo.Html = blackfriday.MarkdownCommon(body)

	reg, _ := regexp.Compile("\\<[\\S\\s]+?\\>")
	vo.Description = reg.ReplaceAll(vo.Html, []byte(""))

	_, err := db.NewInsertBuilder(u.Table()).Insert(vo)
	if err != nil {
		return cid, err
	}

	return cid, err
}
Example #5
0
File: fin.go Project: jiorry/gotock
func FillStockInfo() error {
	name := `C:\Program360\dzh2_hlzq\Download\FIN\full.FIN`
	l, err := ReadData(name)
	if err != nil {
		return err
	}

	rows := make([]db.DataRow, 0)
	rowsFin := make([]*FinData, 0)

	insert := db.NewInsertBuilder("stock")

	var k int = 0

	for _, finData := range l {
		if k == 100 {
			insert.Table("stock").InsertM(rows)
			for _, fin := range rowsFin {
				insert.Table("finance_info").Insert(fin)
			}
			k = 0
			rows = make([]db.DataRow, 0)
			rowsFin = make([]*FinData, 0)
		}

		if finData.Fin3 == 0 {
			continue
		}

		rows = append(rows, db.DataRow{"code": finData.Code[2:], "ctype": strings.ToLower(finData.Code[:2])})
		rowsFin = append(rowsFin, finData)

		k++
	}

	if k > 0 {
		fmt.Println(k, "finished")
		insert.Table("stock").InsertM(rows)
		for _, fin := range rowsFin {
			insert.Table("finance_info").Insert(fin)
		}
	}

	return nil
}
Example #6
0
func (a *StoreDaily) fill(code string, result db.DataSet) error {
	if len(result) == 0 {
		return nil
	}

	id := a.prepareStock(a.StoreBase.prefix, code)
	if id < 0 {
		return fmt.Errorf("stock not found %s %s", a.StoreBase.prefix, code)
	}

	insert := db.NewInsertBuilder(a.table)
	// exists := db.NewExistsBuilder(a.table)
	fmt.Println("begin write records:", a.prefix, code, " total: ", len(result))

	date := ""
	rows := db.DataSet{}
	k := 0
	for _, item := range result {
		item["stock_id"] = id
		date = item.GetString("date")
		// if exists.Where("stock_id=? and date=?", id, date).Exists() {
		// 	fmt.Println("skip:", code, id, date)
		// 	continue
		// }

		rows = append(rows, item)
		// fmt.Println("insert:", code, id, date)

		if k == 20 {
			k = 0
			insert.InsertM(rows)
			fmt.Println(code, id, date, "---- InsertM ----")
			rows = db.DataSet{}
		}
		k++
	}
	if len(rows) > 0 {
		insert.InsertM(rows)
		fmt.Println("--------------- InsertM ---------------")
	}
	return nil
}
Example #7
0
// FillStockData 抓取数据
func FillStockData() {
	// http://stockData.stock.hexun.com/600028.shtml
	upd := db.NewUpdateBuilder("stock")
	ist := db.NewInsertBuilder("")
	q := db.NewQueryBuilder("")
	dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()

	for _, data := range dataset {
		if data.GetInt("industry_id") == 0 || data.GetInt("icb_id") == 0 {
			industry, icb, err := FetchStockData(data.GetString("code"))
			if err != nil {
				gos.DoError(err)
				continue
			}

			if data.GetInt("industry_id") == 0 {
				ist.Table("industry").Insert(industry)
				row, err := q.Table("industry").Where("name=?", industry.Name).QueryOne()
				if err != nil {
					gos.DoError(err)
				} else if !row.Empty() {
					upd.Where("id=?", data.GetInt64("id")).Update(db.DataRow{"industry_id": row.GetInt64("id")})
				}
			}

			if data.GetInt("icb_id") == 0 {
				ist.Table("icb").Insert(icb)
				row, err := q.Table("icb").Where("name=?", icb.Name).QueryOne()
				if err != nil {
					gos.DoError(err)
				} else if !row.Empty() {
					upd.Where("id=?", data.GetInt64("id")).Update(db.DataRow{"icb_id": row.GetInt64("id")})
				}
			}
		}

	}
}
Example #8
0
func fetchAndFillFundRank(page int, quarter string) (int, error) {
	// http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=50&date=2015-09-30&orgType=&stateType=null&titType=null&page=2&callback=hxbase_json7
	formt := "http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=%s&date=%s&orgType=&stateType=null&titType=null&page=%d&callback=hxbase_json7"
	pageLimit := 100
	body, err := wget.GetBody(fmt.Sprintf(formt, pageLimit, quarter, page))
	if err != nil {
		return -1, err
	}

	exists := db.NewExistsBuilder("funds")
	ist := db.NewInsertBuilder("funds")
	query := db.NewQueryBuilder("funds")

	var fund *Fund
	var fundRank *FundRank
	var row db.DataRow

	// hxbase_json7(
	str, err := iconv.ConvertString(string(body), "gb2312", "utf-8")
	if err != nil {
		return -1, err
	}

	src := []byte(str)
	src = src[13 : len(src)-1]
	src = bytes.Replace(src, []byte(":'"), []byte(`":"`), -1)
	src = bytes.Replace(src, []byte("',"), []byte(`","`), -1)
	src = bytes.Replace(src, []byte("'}"), []byte(`"}`), -1)
	src = bytes.Replace(src, []byte("{"), []byte(`{"`), -1)
	src = bytes.Replace(src, []byte("sum:"), []byte(`sum":`), 1)
	src = bytes.Replace(src, []byte("list:"), []byte(`"list":`), 1)

	v := &JsonFund{}
	err = json.Unmarshal(src, v)
	if err != nil {
		fmt.Println(string(src))
		return -1, err
	}
	// {RankTd:'51',OrgName:'法国巴黎银行',OrgNameLink:'o-QF000031.shtml',OrgType:'QFII',ShareHoldingNum:'3',ShareHoldingNumLink:'otherDetail.aspx?OrgNo=QF000031',TotalHoldings:'48,388.00',TotalMarketValue:'1,100,735.00',OrgAlt:'法国巴黎银行'}
	for _, item := range v.List {
		fund = &Fund{Code: item["OrgNameLink"], Name: string(item["OrgName"]), TypeID: FundType(item["OrgType"])}
		fund.Code = fund.Code[2 : len(fund.Code)-6]
		fmt.Println("----")
		if !exists.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).Exists() {
			fmt.Println("insert", fund)
			ist.Table("funds").Insert(fund)
		}

		row, _ = query.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).QueryOne()
		if row.Empty() {
			return -1, fmt.Errorf("code %s not found", fund.Code)
		}

		fundRank = &FundRank{
			FundID: row.GetInt64("id"),
			Date:   quarter,
			Rank:   int(util.ParseMoney(item["RankTd"])),
			Count:  int(util.ParseMoney(item["ShareHoldingNum"])),
			MH:     int64(util.ParseMoney(item["TotalHoldings"])),
			MV:     int64(util.ParseMoney(item["TotalMarketValue"])),
		}

		if !exists.Table("fund_rank").Where("fund_id=? and date=?", fundRank.FundID, fundRank.Date).Exists() {
			fmt.Println("insert", fundRank)
			ist.Table("fund_rank").Insert(fundRank)
		}

	}
	return int(math.Ceil(float64(v.Sum / pageLimit))), nil
}
Example #9
0
File: yyb.go Project: jiorry/gotock
// fetchAndFillYyb 抓取数据
func fetchAndFillYyb(page int) (int, error) {
	// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=2&ps=50&js=var%20XvAVhGPE={%22data%22:[(x)],%22pages%22:%22(pc)%22,%22update%22:%22(ud)%22}&rt=48257541
	formt := `http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=%d&ps=%d&js=var%%20XvAVhGPE={"data":[(x)],"pages":"(pc)","update":"(ud)"}&rt=%d`
	pageLimit := 100

	body, err := wget.GetBody(fmt.Sprintf(formt, page, pageLimit, time.Now().Unix()))
	if err != nil {
		return -1, err
	}

	exists := db.NewExistsBuilder("yyb")
	ist := db.NewInsertBuilder("yyb")
	upd := db.NewUpdateBuilder("yyb")
	query := db.NewQueryBuilder("yyb")
	var yyb *yybStruct
	var yybLub *yybLhbStruct
	var arr []string
	var row db.DataRow
	src := body[bytes.Index(body, []byte("={"))+1:]

	v := &jsonYybData{}
	err = json.Unmarshal(src, v)
	if err != nil {
		return -1, err
	}
	// "80136686,2,216,135,七喜控股|协鑫集成|恒信移动,2991067145.735,广东,80000073,440000,105,6089657316.805,3098590171.07,华泰证券股份有限公司广州天河东路证券营业部,002027.SZ|002506.SZ|300081.SZ"
	for i, item := range v.Data {
		arr = strings.Split(item, ",")
		yyb = &yybStruct{Code: arr[0], Name: arr[12], Area: arr[6]}

		if !exists.Table("yyb").Where("code=?", yyb.Code).Exists() {
			gos.Log.Info("insert", yyb.Code)
			ist.Table("yyb").Insert(yyb)
		}

		row, _ = query.Table("yyb").Where("code=?", yyb.Code).QueryOne()
		if row.Empty() {
			return -1, fmt.Errorf("code %s not found", yyb.Code)
		}

		yybLub = &yybLhbStruct{
			YybID:   row.GetInt64("id"),
			Updated: v.Update,
			Rank:    pageLimit*(page-1) + i + 1,
			Amount:  util.ParseMoney(arr[10]),
			Buy:     util.ParseMoney(arr[11]),
			Sell:    util.ParseMoney(arr[5]),
			Num:     int(util.ParseMoney(arr[2])),
			NumBuy:  int(util.ParseMoney(arr[3])),
			NumSell: int(util.ParseMoney(arr[9])),
		}

		if exists.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Exists() {
			gos.Log.Info("update", yyb.Code, yybLub.YybID)
			upd.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Update(yybLub)
		} else {
			gos.Log.Info("insert", yyb.Code, yybLub.YybID)
			ist.Table("yyb_lhb").Insert(yybLub)
		}

	}
	return int(util.ParseMoney(v.Pages)), nil
}
Example #10
0
func (this *ImageUpload) DoUpload() {
	defer func() {
		if r := recover(); r != nil {
			gos.DoError(r)
		}
	}()
	now := time.Now()
	nowStr := now.Format("20060102")
	tagName := "file"
	tempPath := path.Join(gos.HomeDir, this.StoreRoot, "/temp")
	storePath := path.Join(gos.HomeDir, this.StoreRoot, "/pictures/", nowStr)

	if this.Ctx.Request.Method == "GET" {
		if err := this.ResumableJsTest(storePath); err == nil {
			this.ResumableJsCheckAndMeger(storePath, tempPath)
			this.Ctx.ResponseWriter.WriteHeader(200)
		} else {
			this.Ctx.ResponseWriter.WriteHeader(204)
		}
		return
	}

	websvr := &gos.WebApi{Ctx: this.Ctx}
	reply := map[string]string{}
	sf, err := this.ResumableJs(tagName, storePath, tempPath)
	if err != nil {
		gos.DoError(err)
		websvr.Reply(gos.DoError(err).Data(), nil)
		return
	}

	// sf.Store()

	if sf == nil {
		reply["filename"] = ""
		reply["ext"] = ""
		reply["url"] = ""
	} else {
		reply["filename"] = sf.StoreName
		reply["ext"] = sf.Ext

		folder := fmt.Sprint("/uploads/pictures/", nowStr, "/")
		reply["url"] = fmt.Sprint(folder, reply["filename"], "-m", sf.Ext)

		// err = resizeImage(storePath, sf.StoreName, sf.Ext)
		from := path.Join(gos.HomeDir, "webroot", folder, fmt.Sprint(reply["filename"], sf.Ext))
		to := path.Join(gos.HomeDir, "webroot", folder, fmt.Sprint(reply["filename"], "-m", sf.Ext))
		cmdStr := fmt.Sprint("convert -resize 800x600 ", from, " ", to)
		cmd := exec.Command("/bin/sh", "-c", cmdStr)
		_, err := cmd.Output()
		if err != nil {
			gos.DoError(err, cmdStr)
		}

		au := auth.New(this.Ctx)
		row := db.DataRow{}
		row["name"] = sf.StoreName
		row["ext"] = sf.Ext
		row["user_id"] = au.UserID()
		row["created_at"] = now

		_, err = db.NewInsertBuilder("images").Insert(row)
		if err != nil {
			websvr.Reply(gos.DoError(err).Data(), nil)
			return
		}
	}

	websvr.Reply(reply, nil)
}
Example #11
0
// FillCwfx 抓取并且填充数据
func FillCwfx(lastDate string) {
	ist := db.NewInsertBuilder("fin_cwzb")
	exist := db.NewExistsBuilder("fin_cwzb")
	dataset, _ := db.NewQueryBuilder("stock").Select("id,code,ctype").Limit(0).Query()

	for i, data := range dataset {
		fmt.Println("--", i, "--", data.GetString("code"), data.GetString("ctype"))
		if exist.Where("stock_id=? and date=?", data.GetInt64("id"), lastDate).Exists() {
			gos.Log.Info("exists", lastDate, "return")
			continue
		}

		cwzbList, zcfzbList, lrbList, xjllbList, finPercentList, err := FetchCwfx(data.GetString("code"), data.GetString("ctype"))
		if err != nil {
			gos.DoError(err)
			continue
		}

		if len(cwzbList) == 0 {
			continue
		}

		for _, row := range cwzbList {
			row.StockID = data.GetInt64("id")

			if exist.Table("fin_cwzb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
				gos.Log.Info("exists fin_cwzb", row.Date, "return")
				continue
			}

			gos.Log.Info("insert fin_cwzb", row.Date)
			ist.Table("fin_cwzb").Insert(row)
		}
		for _, row := range zcfzbList {
			row.StockID = data.GetInt64("id")

			if exist.Table("fin_zcfzb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
				gos.Log.Info("exists fin_zcfzb", row.Date, "return")
				continue
			}

			gos.Log.Info("insert fin_zcfzb", row.Date)
			ist.Table("fin_zcfzb").Insert(row)
		}
		for _, row := range lrbList {
			row.StockID = data.GetInt64("id")

			if exist.Table("fin_lrb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
				fmt.Println("exists fin_lrb", row.Date, "return")
				continue
			}

			gos.Log.Info("insert fin_lrb", row.Date)
			ist.Table("fin_lrb").Insert(row)
		}
		for _, row := range xjllbList {
			row.StockID = data.GetInt64("id")

			if exist.Table("fin_xjllb").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
				gos.Log.Info("exists fin_xjllb", row.Date, "return")
				continue
			}

			gos.Log.Info("insert fin_xjllb", row.Date)
			ist.Table("fin_xjllb").Insert(row)
		}
		for _, row := range finPercentList {
			row.StockID = data.GetInt64("id")

			if exist.Table("fin_percent").Where("stock_id=? and date=?", row.StockID, row.Date).Exists() {
				gos.Log.Info("exists fin_percent", row.Date, "return")
				continue
			}

			gos.Log.Info("insert fin_percent", row.Date)
			ist.Table("fin_percent").Insert(row)
		}

		if len(cwzbList) > 0 {
			time.Sleep(1 * time.Second)
		}
	}
}