Example #1
0
func (s *jsonGgcgData) Parse() []*ggcgVO {
	dataset := make([]*ggcgVO, 0)

	query := db.NewQueryBuilder("stock")
	upd := db.NewUpdateBuilder("stock")

	var tmp []string
	var arr []string
	var row db.DataRow
	var ggcg *ggcgVO

	for _, item := range s.Data {
		arr = strings.Split(item, ",")
		tmp = strings.Split(strings.ToLower(arr[15]), ".")
		row, _ = query.Where("code=? and ctype=?", tmp[0], tmp[1]).QueryOne()

		if row.Empty() {
			continue
		}
		upd.Where("id=?", row.GetInt64("id")).Update(db.DataRow{"name": arr[9]})
		// "0.00444,谢飞鹏,002420,谢杏思,A股,2015-11-16,-17800,  0   ,9.44,  毅昌股份,父母,YCGF,竞价交易,-168032,董事,002420.SZ"
		//     0      1     2      3    4      5         6     7   8        9    10   11    12       13   14    15
		// "0.00025,邓伦明,002539,邓伦德,A股,2015-11-17,-2500,   0,  14.65,  新都化工,兄弟姐妹,XDHG,竞价交易,-36625,监事,002539.SZ",
		// "0.00012,马东杰,002771,马东伟,A股,2015-11-16,  100,   0,  129.12, 真视通,兄弟姐妹,ZST,竞价交易,12912,监事,002771.SZ",
		// "0.00012,马东杰,002771,马东伟,A股,2015-11-16, -100,   0,  129.8,  真视通,兄弟姐妹,ZST,竞价交易,-12980,监事,002771.SZ",
		// "0.00628,郝先进,002690,郝先进,A股,2015-11-16,42430,23581180,30.26,美亚光电,本人,MYGD,竞价交易,1283931.8,董事、高管,002690.SZ"

		ggcg = &ggcgVO{
			StockID:       row.GetInt64("id"),
			Date:          arr[5],
			Person:        arr[3],
			Price:         util.ParseMoney(arr[8]),
			Amount:        util.ParseMoney(arr[6]),
			Total:         util.ParseMoney(arr[13]),
			Remain:        int64(util.ParseMoney(arr[7])),
			Proportion:    util.ParseMoney(arr[0]),
			Reason:        arr[12],
			PersonRel:     arr[1],
			PersonRelType: arr[10],
			Job:           arr[14],
		}

		dataset = append(dataset, ggcg)
	}

	return dataset
}
Example #2
0
func fetchAndFillFundRank(page int, quarter string) (int, error) {
	// http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=50&date=2015-09-30&orgType=&stateType=null&titType=null&page=2&callback=hxbase_json7
	formt := "http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=%s&date=%s&orgType=&stateType=null&titType=null&page=%d&callback=hxbase_json7"
	pageLimit := 100
	body, err := wget.GetBody(fmt.Sprintf(formt, pageLimit, quarter, page))
	if err != nil {
		return -1, err
	}

	exists := db.NewExistsBuilder("funds")
	ist := db.NewInsertBuilder("funds")
	query := db.NewQueryBuilder("funds")

	var fund *Fund
	var fundRank *FundRank
	var row db.DataRow

	// hxbase_json7(
	str, err := iconv.ConvertString(string(body), "gb2312", "utf-8")
	if err != nil {
		return -1, err
	}

	src := []byte(str)
	src = src[13 : len(src)-1]
	src = bytes.Replace(src, []byte(":'"), []byte(`":"`), -1)
	src = bytes.Replace(src, []byte("',"), []byte(`","`), -1)
	src = bytes.Replace(src, []byte("'}"), []byte(`"}`), -1)
	src = bytes.Replace(src, []byte("{"), []byte(`{"`), -1)
	src = bytes.Replace(src, []byte("sum:"), []byte(`sum":`), 1)
	src = bytes.Replace(src, []byte("list:"), []byte(`"list":`), 1)

	v := &JsonFund{}
	err = json.Unmarshal(src, v)
	if err != nil {
		fmt.Println(string(src))
		return -1, err
	}
	// {RankTd:'51',OrgName:'法国巴黎银行',OrgNameLink:'o-QF000031.shtml',OrgType:'QFII',ShareHoldingNum:'3',ShareHoldingNumLink:'otherDetail.aspx?OrgNo=QF000031',TotalHoldings:'48,388.00',TotalMarketValue:'1,100,735.00',OrgAlt:'法国巴黎银行'}
	for _, item := range v.List {
		fund = &Fund{Code: item["OrgNameLink"], Name: string(item["OrgName"]), TypeID: FundType(item["OrgType"])}
		fund.Code = fund.Code[2 : len(fund.Code)-6]
		fmt.Println("----")
		if !exists.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).Exists() {
			fmt.Println("insert", fund)
			ist.Table("funds").Insert(fund)
		}

		row, _ = query.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).QueryOne()
		if row.Empty() {
			return -1, fmt.Errorf("code %s not found", fund.Code)
		}

		fundRank = &FundRank{
			FundID: row.GetInt64("id"),
			Date:   quarter,
			Rank:   int(util.ParseMoney(item["RankTd"])),
			Count:  int(util.ParseMoney(item["ShareHoldingNum"])),
			MH:     int64(util.ParseMoney(item["TotalHoldings"])),
			MV:     int64(util.ParseMoney(item["TotalMarketValue"])),
		}

		if !exists.Table("fund_rank").Where("fund_id=? and date=?", fundRank.FundID, fundRank.Date).Exists() {
			fmt.Println("insert", fundRank)
			ist.Table("fund_rank").Insert(fundRank)
		}

	}
	return int(math.Ceil(float64(v.Sum / pageLimit))), nil
}
Example #3
0
File: yyb.go Project: jiorry/gotock
// fetchAndFillYyb 抓取数据
func fetchAndFillYyb(page int) (int, error) {
	// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=2&ps=50&js=var%20XvAVhGPE={%22data%22:[(x)],%22pages%22:%22(pc)%22,%22update%22:%22(ud)%22}&rt=48257541
	formt := `http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=%d&ps=%d&js=var%%20XvAVhGPE={"data":[(x)],"pages":"(pc)","update":"(ud)"}&rt=%d`
	pageLimit := 100

	body, err := wget.GetBody(fmt.Sprintf(formt, page, pageLimit, time.Now().Unix()))
	if err != nil {
		return -1, err
	}

	exists := db.NewExistsBuilder("yyb")
	ist := db.NewInsertBuilder("yyb")
	upd := db.NewUpdateBuilder("yyb")
	query := db.NewQueryBuilder("yyb")
	var yyb *yybStruct
	var yybLub *yybLhbStruct
	var arr []string
	var row db.DataRow
	src := body[bytes.Index(body, []byte("={"))+1:]

	v := &jsonYybData{}
	err = json.Unmarshal(src, v)
	if err != nil {
		return -1, err
	}
	// "80136686,2,216,135,七喜控股|协鑫集成|恒信移动,2991067145.735,广东,80000073,440000,105,6089657316.805,3098590171.07,华泰证券股份有限公司广州天河东路证券营业部,002027.SZ|002506.SZ|300081.SZ"
	for i, item := range v.Data {
		arr = strings.Split(item, ",")
		yyb = &yybStruct{Code: arr[0], Name: arr[12], Area: arr[6]}

		if !exists.Table("yyb").Where("code=?", yyb.Code).Exists() {
			gos.Log.Info("insert", yyb.Code)
			ist.Table("yyb").Insert(yyb)
		}

		row, _ = query.Table("yyb").Where("code=?", yyb.Code).QueryOne()
		if row.Empty() {
			return -1, fmt.Errorf("code %s not found", yyb.Code)
		}

		yybLub = &yybLhbStruct{
			YybID:   row.GetInt64("id"),
			Updated: v.Update,
			Rank:    pageLimit*(page-1) + i + 1,
			Amount:  util.ParseMoney(arr[10]),
			Buy:     util.ParseMoney(arr[11]),
			Sell:    util.ParseMoney(arr[5]),
			Num:     int(util.ParseMoney(arr[2])),
			NumBuy:  int(util.ParseMoney(arr[3])),
			NumSell: int(util.ParseMoney(arr[9])),
		}

		if exists.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Exists() {
			gos.Log.Info("update", yyb.Code, yybLub.YybID)
			upd.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Update(yybLub)
		} else {
			gos.Log.Info("insert", yyb.Code, yybLub.YybID)
			ist.Table("yyb_lhb").Insert(yybLub)
		}

	}
	return int(util.ParseMoney(v.Pages)), nil
}
Example #4
0
// FetchDzjy 抓取数据
func FetchDzjy(date time.Time) ([]*dzjyVO, error) {
	formt := "http://data.eastmoney.com/dzjy/%s.html"

	resp, err := wget.Get(fmt.Sprintf(formt, date.Format("200601")))
	if err != nil {
		return nil, gos.DoError(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, gos.DoError(err)
	}
	var td *goquery.Selection
	var dzjy *dzjyVO
	var dateStr string
	var stockCode string
	var row db.DataRow
	query := db.NewQueryBuilder("stock")
	datalist := make([]*dzjyVO, 0)
	buy := ""
	sell := ""
	var priceNow float64
	var price float64
	var amount float64
	var total float64
	var length int

	doc.Find("#content div.list").Eq(2).Find("table tr.list_eve").Each(func(i int, tr *goquery.Selection) {
		td = tr.Find("td")
		length = td.Length()

		if length == 10 {
			dateStr = td.Eq(0).Text()
			stockCode = td.Eq(1).Text()
		} else if length == 9 {
			stockCode = td.Eq(0).Text()
		}

		row, _ = query.Where("code=?", stockCode).QueryOne()
		if row.Empty() {
			return
		}

		switch length {
		case 10:
			priceNow = util.ParseMoney(td.Eq(4).Text())
			price = util.ParseMoney(td.Eq(5).Text())
			amount = util.ParseMoney(td.Eq(6).Text())
			total = util.ParseMoney(td.Eq(7).Text())
			buy, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
			sell, err = iconv.ConvertString(td.Eq(9).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}

		case 9:
			priceNow = util.ParseMoney(td.Eq(3).Text())
			price = util.ParseMoney(td.Eq(4).Text())
			amount = util.ParseMoney(td.Eq(5).Text())
			total = util.ParseMoney(td.Eq(6).Text())
			buy, err = iconv.ConvertString(td.Eq(7).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
			sell, err = iconv.ConvertString(td.Eq(8).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
		case 5:
			price = util.ParseMoney(td.Eq(0).Text())
			amount = util.ParseMoney(td.Eq(1).Text())
			total = util.ParseMoney(td.Eq(2).Text())
			buy, err = iconv.ConvertString(td.Eq(3).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
			sell, err = iconv.ConvertString(td.Eq(4).Text(), "gb2312", "utf-8")
			if err != nil {
				return
			}
		default:
			return
		}

		dzjy = &dzjyVO{
			StockID:  row.GetInt64("id"),
			Date:     dateStr,
			PriceNow: priceNow,
			Price:    price,
			Amount:   amount,
			Total:    total,
			Buy:      buy,
			Sell:     sell,
		}

		datalist = append(datalist, dzjy)
	})

	return datalist, nil
}