func (um *Model) GetRegisteredUsers() (users map[string]*User) {
if rows, err := um.table.Scan(nil); err == nil {
users = make(map[string]*User)
for _, row := range rows {
uid := row["uid"].Value
user := &User{
uid: uid,
key: row["key"].Value,
Enabled: row["enabled"].Value,
logs: make(map[string][]*LogLine),
billHist: []*Billing{},
md: um,
}
if err := json.Unmarshal([]byte(row["info"].Value), &user); err != nil {
log.Error("Problem trying to retieve the user information for user:"******"Error:", err)
return nil
}
if err = json.Unmarshal([]byte(row["logs"].Value), &user.logs); err != nil {
log.Error("Problem trying to unmarshal the user logs for user:"******"Error:", err)
return nil
}
if err = json.Unmarshal([]byte(row["bill_hist"].Value), &user.billHist); err != nil {
log.Error("Problem trying to unmarshal the billing history for user:"******"Error:", err)
return nil
}
users[uid] = user
}
}
return
}
func (bi *Billing) getToken() {
client := &http.Client{}
req, err := http.NewRequest("POST", bi.baseUri+"oauth2/token", strings.NewReader("grant_type=client_credentials"))
req.Header.Add("Accept", `application/json`)
req.Header.Add("Accept-Language", `en_US`)
req.SetBasicAuth(bi.clientId, bi.secret)
resp, err := client.Do(req)
if err != nil {
log.Error("Problem trying to request a new token from the PayPal API, Error:", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Error("Problem trying to parse the response form the PayPal API, Error:", err)
}
responseInfo := make(map[string]interface{})
err = json.Unmarshal(body, &responseInfo)
if err != nil {
log.Error("Problem trying to read token from PayPal API, Error:", err)
}
bi.token = responseInfo["access_token"].(string)
bi.tokenExpireTs = time.Now().Unix() + int64(responseInfo["expires_in"].(float64))
}
func GetTrainerCorrelations(trainingFile string, TimeRangeToStudySecs int64) TrainerInt {
log.Debug("Initializing trainer...")
TimeRangeToStudySecs *= tsMultToSecs
feedsFile, err := os.Open(trainingFile)
log.Debug("File:", trainingFile)
if err != nil {
log.Fatal("Problem reading the logs file")
}
scanner := bufio.NewScanner(feedsFile)
feeds := &TrainerCorrelations{
feeds: make(map[string][]*charont.CurrVal),
centroidsCurr: make(map[string][][]float64),
centroidsCurrSell: make(map[string][][]float64),
centroidsForAsk: make(map[string][]int),
centroidsForSell: make(map[string][]int),
maxWinByCentroid: make(map[string][]float64),
maxLossByCentroid: make(map[string][]float64),
maxWinByCentroidSell: make(map[string][]float64),
maxLossByCentroidSell: make(map[string][]float64),
}
i := 0
for {
var feed *charont.CurrVal
if !scanner.Scan() {
break
}
lineParts := strings.SplitN(scanner.Text(), ":", 2)
curr := lineParts[0]
if len(lineParts) < 2 {
log.Error("The line:", i, "can't be parsed")
continue
}
if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
log.Error("The feeds response body is not a JSON valid, Error:", err, "Line:", i)
continue
}
if _, ok := feeds.feeds[curr]; !ok {
feeds.feeds[curr] = []*charont.CurrVal{}
}
feeds.feeds[curr] = append(feeds.feeds[curr], feed)
i++
if i%10000 == 0 {
log.Debug("Lines:", i)
}
}
for curr, scores := range feeds.feeds {
log.Debug("Curr:", curr, "Scores:", len(scores))
}
feeds.studyCurrencies(TimeRangeToStudySecs)
return feeds
}
func (im *Model) updateInstances() {
if rows, err := im.table.Scan(nil); err == nil {
instances := []string{}
for _, row := range rows {
lastTs, _ := strconv.ParseInt(row["ts"].Value, 10, 64)
if lastTs, _ = strconv.ParseInt(row["ts"].Value, 10, 64); lastTs+cTTL > time.Now().Unix() {
instances = append(instances, row[cPrimKey].Value)
} else if row[cPrimKey].Value != hostName {
log.Info("Outdated instance detected, removing it, name:", row[cPrimKey].Value)
attKey := &dynamodb.Key{
HashKey: row[cPrimKey].Value,
RangeKey: "",
}
_, err = im.table.DeleteItem(attKey)
if err != nil {
log.Error("The instance:", row[cPrimKey].Value, "can't be removed, Error:", err)
}
}
}
sort.Sort(byName(instances))
im.mutex.Lock()
im.instancesAlive = instances
im.mutex.Unlock()
} else {
log.Error("Problem trying to get the list of instances from Dynamo DB, Error:", err)
}
}
func (um *Model) delTable() {
if tableDesc, err := um.conn.DescribeTable(um.tableName); err == nil {
if _, err = um.conn.DeleteTable(*tableDesc); err != nil {
log.Error("Can't remove Dynamo table:", um.tableName, "Error:", err)
}
} else {
log.Error("Can't remove Dynamo table:", um.tableName, "Error:", err)
}
}
func (rc *Recommender) uncompress(data []byte) (result []byte) {
gz, err := gzip.NewReader(strings.NewReader(string(data)))
if err != nil {
log.Error("The data can't be uncompressed on shard:", rc.identifier, "Error:", err)
}
defer gz.Close()
if result, err = ioutil.ReadAll(gz); err != nil {
log.Error("The data can't be uncompressed on shard:", rc.identifier, "Error:", err)
}
return
}
func (rc *Recommender) compress(data []byte) (result []byte) {
var b bytes.Buffer
gz := gzip.NewWriter(&b)
if _, err := gz.Write(data); err != nil {
log.Error("The data can't be compressed on shard:", rc.identifier, "Error:", err)
}
if err := gz.Flush(); err != nil {
log.Error("The data can't be compressed on shard:", rc.identifier, "Error:", err)
}
if err := gz.Close(); err != nil {
log.Error("The data can't be compressed on shard:", rc.identifier, "Error:", err)
}
return b.Bytes()
}
func (rc *Recommender) LoadBackup() (success bool) {
log.Info("Loading backup from S3:", rc.identifier)
auth, err := aws.EnvAuth()
if err != nil {
log.Error("Problem trying to connect with AWS:", err)
return false
}
s := s3.New(auth, rc.s3Region)
bucket := s.Bucket(S3BUCKET)
jsonData, err := bucket.Get(rc.getS3Path())
if err != nil {
log.Info("Problem trying to get backup from S3:", err)
return false
}
dataFromJSON := [][]uint64{}
json.Unmarshal(rc.uncompress(jsonData), &dataFromJSON)
log.Info("Data loaded from S3:", rc.identifier, "len:", len(dataFromJSON))
recs := 0
for _, record := range dataFromJSON {
scores := make(map[uint64]uint8)
for i := 1; i < len(record); i += 2 {
scores[record[i]] = uint8(record[i+1])
}
recs += len(scores)
rc.AddRecord(record[0], scores)
}
return true
}
func (mg *Manager) SendEmail(to, body, subject string) (success bool) {
auth := smtp.PlainAuth(
mg.mailFromAddr,
mg.mailFromAddr,
os.Getenv("PIT_MAIL_PASS"),
mg.mailServerAddr,
)
body = fmt.Sprintf(
"Subject: %s\r\n\r\n\r\n%s",
subject,
[]byte(body))
err := smtp.SendMail(
fmt.Sprintf("%s:%d", mg.mailServerAddr, mg.mailServerPort),
auth,
mg.mailFromAddr,
[]string{to},
[]byte(body),
)
if err != nil {
log.Error("Problem trying to send a verification e-mail:", err)
return false
}
return true
}
func InitAndKeepAlive(prefix string, awsRegion string, keepAlive bool) (im *Model) {
if awsAuth, err := aws.EnvAuth(); err == nil {
im = &Model{
prefix: prefix,
tableName: fmt.Sprintf("%s_%s", prefix, cTable),
conn: &dynamodb.Server{
Auth: awsAuth,
Region: aws.Regions[awsRegion],
},
}
im.initTable()
if keepAlive {
im.registerHostName(hostName)
}
im.updateInstances()
if keepAlive {
go func() {
for {
im.registerHostName(hostName)
im.updateInstances()
time.Sleep(time.Second)
}
}()
}
} else {
log.Error("Problem trying to connect with DynamoDB, Error:", err)
return
}
return
}
// GetFloat Returns the value of the section, subsection as float
func GetFloat(sec, subsec string) (v float64) {
if v, err := strconv.ParseFloat(loadSection(sec).ValueOf(subsec), 64); err == nil {
return v
}
log.Error("Configuration parameter:", sec, subsec, "can't be parsed as integer")
return
}
func InitOandaApi(endpoint string, authToken string, accountId int, currencies []string, currLogsFile string) (api *Oanda, err error) {
var resp []byte
api = &Oanda{
endpoint: endpoint,
openOrders: make(map[int64]*Order),
authToken: authToken,
currencies: currencies,
listeners: make(map[string][]func(currency string, ts int64)),
currentWin: 0,
simulatedOrders: 0,
}
if currLogsFile != "" {
api.currLogsFile, err = os.Create(currLogsFile)
if err != nil {
log.Error("Currency logs file can't be open, Error:", err)
return
}
}
api.mutex.Lock()
if accountId == -1 {
var accInfo map[string]interface{}
respHttp, err := http.PostForm(fmt.Sprintf(FAKE_GENERATE_ACCOUNT_URL, api.endpoint), nil)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(respHttp.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(body, &accInfo)
if err != nil {
return nil, err
}
resp, err = api.doRequest("GET", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), nil)
log.Info("New account generated:", int(accInfo["accountId"].(float64)))
} else {
resp, err = api.doRequest("GET", fmt.Sprintf("%s%d", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), accountId), nil)
}
if err != nil {
return
}
err = json.Unmarshal(resp, &api.account)
if err != nil {
return
}
api.mutex.Unlock()
return
}
func (um *Model) AdminGetUserInfoByID(uid string) (user *User) {
um.mutex.Lock()
defer um.mutex.Unlock()
if us, ok := um.cache[uid]; ok {
return us
}
attKey := &dynamodb.Key{
HashKey: uid,
RangeKey: "",
}
if data, err := um.table.GetItemConsistent(attKey, false); err == nil {
user = &User{
uid: uid,
key: data["key"].Value,
Enabled: data["enabled"].Value,
logs: make(map[string][]*LogLine),
billHist: []*Billing{},
md: um,
}
if err := json.Unmarshal([]byte(data["info"].Value), &user); err != nil {
log.Error("Problem trying to retieve the user information for user:"******"Error:", err)
return nil
}
if _, ok := data["logs"]; ok {
if err = json.Unmarshal([]byte(data["logs"].Value), &user.logs); err != nil {
log.Error("Problem trying to unmarshal the user logs for user:"******"Error:", err)
}
}
if _, ok := data["bill_hist"]; ok {
if err = json.Unmarshal([]byte(data["bill_hist"].Value), &user.billHist); err != nil {
log.Error("Problem trying to unmarshal the user billing history for user:"******"Error:", err)
}
}
} else {
log.Error("Problem trying to read the user information for user:"******"Error:", err)
}
um.cache[uid] = user
return
}
func (rc *Recommender) SaveBackup() {
log.Info("Storing backup on S3:", rc.identifier)
rc.mutex.Lock()
records := make([][]uint64, len(rc.records))
i := 0
for recID, record := range rc.records {
records[i] = make([]uint64, len(record.scores)*2+1)
records[i][0] = recID
elemPos := 1
for k, v := range record.scores {
records[i][elemPos] = k
records[i][elemPos+1] = uint64(v)
elemPos += 2
}
i++
}
rc.mutex.Unlock()
jsonToUpload, err := json.Marshal(records)
auth, err := aws.EnvAuth()
if err != nil {
log.Error("Problem trying to connect with AWS:", err)
return
}
s := s3.New(auth, rc.s3Region)
bucket := s.Bucket(S3BUCKET)
err = bucket.Put(
rc.getS3Path(),
rc.compress(jsonToUpload),
"text/plain",
s3.BucketOwnerFull,
s3.Options{})
if err != nil {
log.Error("Problem trying to upload backup to S3 from:", rc.identifier, "Error:", err)
}
log.Info("New backup stored on S3, bucket:", S3BUCKET, "Path:", rc.getS3Path())
}
func (mock *Mock) ratesCollector() {
mock.mutex.Lock()
mock.currencyValues = make(map[string][]*CurrVal)
for _, curr := range mock.currencies {
mock.mutexCurr[curr] = new(sync.Mutex)
}
mock.mutex.Unlock()
scanner := bufio.NewScanner(mock.currLogsFile)
scanner.Scan()
log.Info("Parsing currencies from the mock file...")
i := 0
lastWinVal := mock.currentWin
for {
var feed CurrVal
scanner.Scan()
lineParts := strings.SplitN(scanner.Text(), ":", 2)
curr := lineParts[0]
if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
log.Error("The feeds response body is not a JSON valid, Error:", err)
continue
}
mock.mutexCurr[curr].Lock()
//log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
mock.currencyValues[curr] = append(mock.currencyValues[curr], &CurrVal{
Ts: feed.Ts,
Bid: feed.Bid,
Ask: feed.Ask,
})
if listeners, ok := mock.listeners[curr]; ok {
for _, listener := range listeners {
listener(curr, feed.Ts)
}
}
if len(mock.currencyValues[curr]) > MAX_RATES_TO_STORE {
mock.currencyValues[curr] = mock.currencyValues[curr][1:]
}
mock.mutexCurr[curr].Unlock()
if lastWinVal != mock.currentWin {
log.Info("CurrentWin:", i, mock.currentWin)
lastWinVal = mock.currentWin
}
i++
}
}
func (bi *Billing) callToApi(uri string, method string, reqBody string) (body []byte, err error) {
if time.Now().Unix() > bi.tokenExpireTs {
bi.getToken()
}
client := &http.Client{}
url := bi.baseUri + uri
log.Info(reqBody)
req, err := http.NewRequest(method, url, strings.NewReader(reqBody))
req.Header.Add("Content-Type", `application/json`)
req.Header.Add("Authorization", "Bearer "+bi.token)
resp, err := client.Do(req)
if err != nil {
log.Error("Problem trying to do a request to the PayPal API, Url:", url, "Error:", err)
}
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
log.Error("Problem trying to parse the response form the PayPal API, Error:", err)
}
return
}
func (um *Model) initTable() {
pKey := dynamodb.PrimaryKey{dynamodb.NewStringAttribute(cPrimKey, ""), nil}
um.table = um.conn.NewTable(um.tableName, pKey)
res, err := um.table.DescribeTable()
if err != nil {
log.Info("Creating a new table on DynamoDB:", um.tableName)
td := dynamodb.TableDescriptionT{
TableName: um.tableName,
AttributeDefinitions: []dynamodb.AttributeDefinitionT{
dynamodb.AttributeDefinitionT{cPrimKey, "S"},
},
KeySchema: []dynamodb.KeySchemaT{
dynamodb.KeySchemaT{cPrimKey, "HASH"},
},
ProvisionedThroughput: dynamodb.ProvisionedThroughputT{
ReadCapacityUnits: cDefaultWRCapacity,
WriteCapacityUnits: cDefaultWRCapacity,
},
}
if _, err := um.conn.CreateTable(td); err != nil {
log.Error("Error trying to create a table on Dynamo DB, table:", um.tableName, "Error:", err)
}
if res, err = um.table.DescribeTable(); err != nil {
log.Error("Error trying to describe a table on Dynamo DB, table:", um.tableName, "Error:", err)
}
}
for "ACTIVE" != res.TableStatus {
if res, err = um.table.DescribeTable(); err != nil {
log.Error("Can't describe Dynamo DB instances table, Error:", err)
}
log.Debug("Waiting for active table, current status:", res.TableStatus)
time.Sleep(time.Second)
}
}
func TestCollabInsertion(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
f, err := os.Open(TESTSET)
if err != nil {
log.Error("Can't read the the test set file:", TESTSET, "Error:", err)
t.Fail()
}
r := bufio.NewReader(f)
s, e := Readln(r)
records := []map[uint64]uint8{}
log.Info("Parsing test file...")
for i := 0; e == nil && i < 10000; i++ {
//for i := 0; e == nil && i < 480187; i++ {
s, e = Readln(r)
_, scores := parseLine(s)
records = append(records, scores)
}
log.Info("Generating tree...")
tr, _ := ProcessNewTrees(records, 50, MAXSCORE, 3)
tr.setTestMode()
log.Info("Tree generated...")
quadError := 0.0
comparedItems := 0
for i := 0; e == nil && i < 1000; i++ {
s, e = Readln(r)
_, scores := parseLine(s)
elements := tr.GetBestRecommendation(scores, 10)
for _, elemID := range elements {
if score, rated := scores[elemID]; rated {
quadError += (1.0 - (float64(score) / MAXSCORE)) * (1.0 - (float64(score) / MAXSCORE))
comparedItems++
}
}
}
// Estimate the Root-mean-square deviation, we will use 0.3 for this test because the training set and the number of trees is too low
rmsd := math.Sqrt(quadError / float64(comparedItems))
if rmsd > 0.3 {
t.Error("The RMSD is bigger than the expected, obtained:", rmsd)
}
return
}
func (rc *Recommender) DestroyS3Backup() (success bool) {
log.Info("Destroying backup on S3:", rc.identifier)
auth, err := aws.EnvAuth()
if err != nil {
log.Error("Problem trying to connect with AWS:", err)
return false
}
s := s3.New(auth, rc.s3Region)
bucket := s.Bucket(S3BUCKET)
if err := bucket.Del(rc.getS3Path()); err != nil {
log.Info("Problem trying to remove backup from S3:", err)
return false
}
return true
}
func GetMock(feedsFile string, feedsBySecond int, currencies []string, httpPort int) (mock *Mock) {
var err error
mock = &Mock{
orders: 0,
currentWin: 0,
feedsBySecond: feedsBySecond,
mutexCurr: make(map[string]*sync.Mutex),
ordersByCurr: make(map[string][]*Order),
currencies: currencies,
listeners: make(map[string][]func(currency string, ts int64)),
openOrders: make(map[int64]*Order),
}
for _, curr := range currencies {
mock.ordersByCurr[curr] = []*Order{}
}
if feedsFile != "" {
mock.currLogsFile, err = os.Open(feedsFile)
if err != nil {
log.Error("Currency logs file can't be open, Error:", err)
return
}
}
http.HandleFunc("/get_curr_values_orders", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Content-Type", "application/json")
curr := r.FormValue("curr")
info, _ := json.Marshal(&currOpsInfo{
Prices: mock.currencyValues[curr],
Orders: mock.ordersByCurr[curr],
})
w.Write(info)
})
go http.ListenAndServe(fmt.Sprintf(":%d", httpPort), nil)
log.Info("Mock HTTP server listening on:", httpPort)
return
}
func GetModel(prefix string, awsRegion string) (um *Model) {
if awsAuth, err := aws.EnvAuth(); err == nil {
um = &Model{
prefix: prefix,
tableName: fmt.Sprintf("%s_%s", prefix, cTable),
secret: []byte(os.Getenv("PIT_SECRET")),
cache: make(map[string]*User),
conn: &dynamodb.Server{
Auth: awsAuth,
Region: aws.Regions[awsRegion],
},
}
um.initTable()
go um.cacheManager()
} else {
log.Error("Problem trying to connect with DynamoDB, Error:", err)
}
return
}
func (us *User) persist() bool {
userJSONInfo, _ := json.Marshal(us)
userJSONLogs, _ := json.Marshal(us.logs)
userJSONBillHist, _ := json.Marshal(us.billHist)
attribs := []dynamodb.Attribute{
*dynamodb.NewStringAttribute(cPrimKey, us.uid),
*dynamodb.NewStringAttribute("key", us.key),
*dynamodb.NewStringAttribute("info", string(userJSONInfo)),
*dynamodb.NewStringAttribute("bill_hist", string(userJSONBillHist)),
*dynamodb.NewStringAttribute("logs", string(userJSONLogs)),
*dynamodb.NewStringAttribute("enabled", string(us.Enabled)),
}
if _, err := us.md.table.PutItem(us.uid, cPrimKey, attribs); err != nil {
log.Error("A new user can't be registered on the users table, Error:", err)
return false
}
return true
}
func (bi *Billing) SendNewBill(invTitle, targetEmail string, items []*InvItem) (billId string, err error) {
inv := &Invoice{
MerchantInfo: &InvMerchantInfo{
Email: bi.email,
BusinessName: bi.addrAddr,
Address: map[string]string{
"line1": bi.addrAddr,
"city": bi.addrCity,
"state": bi.addrState,
"postal_code": bi.addrZip,
"country_code": bi.addrCountryCode,
},
},
BillingInfo: []map[string]string{
map[string]string{
"email": targetEmail,
},
},
Items: items,
Note: invTitle,
}
invStr, _ := json.Marshal(inv)
if body, err := bi.callToApi("invoicing/invoices", "POST", string(invStr)); err == nil {
responseInfo := make(map[string]interface{})
err = json.Unmarshal(body, &responseInfo)
if err != nil {
log.Error("Problem trying to response from PayPal API, Error:", err)
}
billId = responseInfo["id"].(string)
sendRes, err := bi.callToApi(fmt.Sprintf("invoicing/invoices/%s/send", billId), "POST", "")
log.Info("Sent!!!:", fmt.Sprintf("invoicing/invoices/%s/send", billId), string(sendRes), err)
}
return
}
func (api *Oanda) CloseOrder(ord *Order, ts int64) (err error) {
var realOrder string
ord.SellTs = ts
ord.Open = false
if ord.Real {
resp, err := api.doRequest("DELETE", fmt.Sprintf(CHECK_ORDER_URL, api.endpoint, api.account.AccountId, ord.Id), nil)
if err != nil {
log.Error("Problem trying to close an open position, Error:", err)
return err
}
generic := map[string]float64{}
json.Unmarshal(resp, &generic)
ord.CloseRate = generic["price"]
ord.Profit = generic["profit"] / float64(ord.Units)
api.mutex.Lock()
delete(api.openOrders, ord.Id)
api.mutex.Unlock()
api.currentWin += ord.Profit
realOrder = "Real"
} else {
api.mutex.Lock()
lastPrice := api.currencyValues[ord.Curr[4:]][len(api.currencyValues[ord.Curr[4:]])-1]
api.mutex.Unlock()
if ord.Type == "buy" {
ord.CloseRate = lastPrice.Bid
} else {
ord.Price = lastPrice.Ask
}
ord.Profit = ord.CloseRate/ord.Price - 1
realOrder = "Simultaion"
}
log.Debug("Closed Order:", ord.Id, "BuyTs:", time.Unix(ord.BuyTs/tsMultToSecs, 0), "TimeToSell:", (ord.SellTs-ord.BuyTs)/tsMultToSecs, "Curr:", ord.Curr, "OpenRate:", ord.Price, "Close rate:", ord.CloseRate, "And Profit:", ord.Profit, "Current Win:", api.currentWin, "Type:", realOrder)
return
}
func (api *Oanda) placeMarketOrder(inst string, units int, side string, price float64, realOps bool, ts int64) (order *Order, err error) {
if !realOps {
api.mutex.Lock()
defer api.mutex.Unlock()
api.openOrders[api.simulatedOrders] = &Order{
Id: api.simulatedOrders,
Units: units,
Open: true,
Type: side,
Real: false,
BuyTs: ts,
Curr: inst,
}
if side == "buy" {
api.openOrders[api.simulatedOrders].Price = price
} else {
api.openOrders[api.simulatedOrders].CloseRate = price
}
api.simulatedOrders++
return api.openOrders[api.simulatedOrders-1], nil
}
var orderInfo orderStruc
var bound string
if side == "sell" {
bound = "lowerBound"
} else {
bound = "upperBound"
}
resp, err := api.doRequest("POST", fmt.Sprintf(PLACE_ORDER_URL, api.endpoint, api.account.AccountId),
url.Values{
"instrument": {inst},
"units": {fmt.Sprintf("%d", int(units))},
"side": {side},
"type": {"market"},
bound: {fmt.Sprintf("%f", price)},
})
if err != nil {
log.Error("Problem trying to place a new order, Error:", err)
return
}
err = json.Unmarshal(resp, &orderInfo)
if err != nil || orderInfo.Info == nil {
log.Error("The response from the server to place an order can't be parsed:", string(resp), "Error:", err)
return
}
log.Debug("Values: instrument:", inst, "units", units, "side:", side, "type: market ID:", orderInfo, "\nOrder response:", string(resp))
order = &Order{
Id: orderInfo.Info.Id,
Price: orderInfo.Price,
Units: units,
Open: true,
Type: side,
BuyTs: ts,
Curr: inst,
Real: true,
}
if side == "buy" {
order.Price = orderInfo.Price
} else {
order.CloseRate = orderInfo.Price
}
api.mutex.Lock()
api.openOrders[order.Id] = order
api.mutex.Unlock()
return
}
func (api *Oanda) ratesCollector() {
var feeds map[string][]feedStruc
api.mutex.Lock()
api.currencyValues = make(map[string][]*CurrVal)
currExange := make([]string, len(api.currencies))
lasCurrPriceA := make(map[string]float64)
lasCurrPriceB := make(map[string]float64)
log.Debug("Curr:", api.currencies)
for i, curr := range api.currencies {
api.currencyValues[curr] = []*CurrVal{}
currExange[i] = fmt.Sprintf("%s_%s", api.account.AccountCurrency, curr)
lasCurrPriceA[curr] = 0
lasCurrPriceB[curr] = 0
}
api.mutex.Unlock()
feedsUrl := fmt.Sprintf(FEEDS_URL, api.endpoint) + strings.Join(currExange, "%2C")
log.Info("Parsing currencies from the feeds URL:", feedsUrl)
c := time.Tick((1000 / COLLECT_BY_SECOND) * time.Millisecond)
for _ = range c {
resp, err := api.doRequest("GET", feedsUrl, nil)
if err != nil {
log.Error("The feeds URL can't be parsed, Error:", err)
continue
}
if err = json.Unmarshal(resp, &feeds); err != nil {
log.Error("The feeds response body is not a JSON valid, Error:", err, "Resp:", string(resp))
continue
}
// Ok, all fine, we are going to parse the feeds
for _, feed := range feeds["prices"] {
curr := feed.Instrument[len(api.account.AccountCurrency)+1:]
if lasCurrPriceA[curr] != feed.Ask || lasCurrPriceB[curr] != feed.Bid {
log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
api.mutex.Lock()
api.currencyValues[curr] = append(api.currencyValues[curr], &CurrVal{
Ts: time.Now().UnixNano(),
Bid: feed.Bid,
Ask: feed.Ask,
})
api.mutex.Unlock()
if api.currLogsFile != nil {
api.mutex.Lock()
b, _ := json.Marshal(api.currencyValues[curr][len(api.currencyValues[curr])-1])
_, err := api.currLogsFile.WriteString(fmt.Sprintf("%s:%s\n", curr, string(b)))
api.mutex.Unlock()
if err != nil {
log.Error("Can't write into the currencies logs file, Error:", err)
}
}
if listeners, ok := api.listeners[curr]; ok {
for _, listener := range listeners {
go listener(curr, time.Now().UnixNano())
}
}
api.mutex.Lock()
if len(api.currencyValues[curr]) > MAX_RATES_TO_STORE {
api.currencyValues[curr] = api.currencyValues[curr][1:]
}
api.mutex.Unlock()
lasCurrPriceA[curr] = feed.Ask
lasCurrPriceB[curr] = feed.Bid
}
}
}
}
func TestRecommenderSaveLoad(t *testing.T) {
maxClassifications := uint64(1000000)
runtime.GOMAXPROCS(runtime.NumCPU())
sh := NewShard("/testing", "test_collab_insertion", maxClassifications, 5, "eu-west-1")
f, err := os.Open(TESTSET)
if err != nil {
log.Error("Can't read the the test set file:", TESTSET, "Error:", err)
t.Fail()
}
r := bufio.NewReader(f)
s, e := Readln(r)
i := 0
for e == nil && i < 100000 {
s, e = Readln(r)
recID, scores := parseLine(s)
sh.AddRecord(recID, scores)
i++
if i%1000 == 0 {
log.Debug("Lines processed:", i)
}
}
time.Sleep(time.Second)
if sh.totalClassif > maxClassifications {
t.Error(
"Problem with the garbage collection, the total number of stored stores are:",
sh.totalClassif, "and the max defined boundary is:", maxClassifications)
}
if sh.status != STATUS_SARTING {
t.Error("The expectede status was:", STATUS_SARTING, "but the actual one is:", sh.status)
}
log.Debug("Processing tree...")
sh.RecalculateTree()
if sh.status != STATUS_ACTIVE {
t.Error("The expectede status was:", STATUS_ACTIVE, "but the actual one is:", sh.status)
}
s, e = Readln(r)
recID, scores := parseLine(s)
recomendationsBef := sh.CalcScores(recID, scores, 10)
if len(recomendationsBef) != 10 {
t.Error("The expected recommendations was 10, but:", len(recomendationsBef), "obtained.")
}
prevScores := sh.totalClassif
sh.SaveBackup()
sh = NewShard("/testing", "test_collab_insertion", maxClassifications, 5, "eu-west-1")
sh.RecalculateTree()
if sh.status != STATUS_NORECORDS {
t.Error("The expectede status was:", STATUS_NORECORDS, "but the actual one is:", sh.status)
}
sh.LoadBackup()
sh.RecalculateTree()
if prevScores != sh.totalClassif {
t.Error(
"Before store a backup the number of records was:", prevScores,
"but after load the backup is:", sh.totalClassif)
}
recomendationsAfter := sh.CalcScores(recID, scores, 10)
if len(recomendationsAfter) != 10 {
t.Error("The expected recommendations was 10, but:", len(recomendationsAfter), "obtained.")
}
log.Debug("Classifications:", sh.maxClassif)
}