func GetTrainerCorrelations(trainingFile string, TimeRangeToStudySecs int64) TrainerInt {
log.Debug("Initializing trainer...")
TimeRangeToStudySecs *= tsMultToSecs
feedsFile, err := os.Open(trainingFile)
log.Debug("File:", trainingFile)
if err != nil {
log.Fatal("Problem reading the logs file")
}
scanner := bufio.NewScanner(feedsFile)
feeds := &TrainerCorrelations{
feeds: make(map[string][]*charont.CurrVal),
centroidsCurr: make(map[string][][]float64),
centroidsCurrSell: make(map[string][][]float64),
centroidsForAsk: make(map[string][]int),
centroidsForSell: make(map[string][]int),
maxWinByCentroid: make(map[string][]float64),
maxLossByCentroid: make(map[string][]float64),
maxWinByCentroidSell: make(map[string][]float64),
maxLossByCentroidSell: make(map[string][]float64),
}
i := 0
for {
var feed *charont.CurrVal
if !scanner.Scan() {
break
}
lineParts := strings.SplitN(scanner.Text(), ":", 2)
curr := lineParts[0]
if len(lineParts) < 2 {
log.Error("The line:", i, "can't be parsed")
continue
}
if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
log.Error("The feeds response body is not a JSON valid, Error:", err, "Line:", i)
continue
}
if _, ok := feeds.feeds[curr]; !ok {
feeds.feeds[curr] = []*charont.CurrVal{}
}
feeds.feeds[curr] = append(feeds.feeds[curr], feed)
i++
if i%10000 == 0 {
log.Debug("Lines:", i)
}
}
for curr, scores := range feeds.feeds {
log.Debug("Curr:", curr, "Scores:", len(scores))
}
feeds.studyCurrencies(TimeRangeToStudySecs)
return feeds
}
func (mock *Mock) CloseOrder(ord *Order, ts int64) (err error) {
var realOrder string
currVals := mock.currencyValues[ord.Curr]
if ord.Type == "buy" {
ord.CloseRate = currVals[len(currVals)-1].Bid
} else {
ord.Price = currVals[len(currVals)-1].Ask
}
ord.Profit = ord.CloseRate/ord.Price - 1
ord.SellTs = ts
mock.mutex.Lock()
mock.ordersByCurr[ord.Curr] = append(mock.ordersByCurr[ord.Curr], ord)
delete(mock.openOrders, ord.Id)
mock.mutex.Unlock()
if ord.Real {
mock.currentWin += ord.Profit * float64(ord.Units)
realOrder = "Real"
} else {
realOrder = "Simultaion"
}
log.Debug("Closed Order:", ord.Id, "TypeOrd:", ord.Type, "BuyTs:", time.Unix(ord.BuyTs/tsMultToSecs, 0), "TimeToSell:", (ord.SellTs-ord.BuyTs)/tsMultToSecs, "Curr:", ord.Curr, "OpenRate:", ord.Price, "Close rate:", ord.CloseRate, "And Profit:", ord.Profit, "Current Win:", mock.currentWin, "Type:", realOrder)
return
}
func (hades *Hades) CloseAllOpenOrdersAndFinish() {
hades.tradesThatCanPlay = 0
allFinished := false
for !allFinished {
time.Sleep(time.Second)
allFinished = true
for _, trader := range hades.tradersPlaying {
if !trader.StopPlaying() {
log.Debug("Trader:", trader.GetID(), "still playing...")
allFinished = false
}
}
}
log.Debug("All the traders are done, close the system")
}
func (tr *Tree) printTree() {
queue := []*tNode{tr.tree[0]}
for len(queue) > 0 {
node := queue[0]
queue = queue[1:]
recsL := make([][2]float64, len(node.bestRecL))
for i := 0; i < len(recsL); i++ {
recsL[i][0] = node.bestRecL[i].score
recsL[i][1] = node.bestRecL[i].avg
}
recsD := make([][2]float64, len(node.bestRecD))
for i := 0; i < len(recsD); i++ {
recsD[i][0] = node.bestRecD[i].score
recsD[i][1] = node.bestRecD[i].avg
}
recsU := make([][2]float64, len(node.bestRecU))
for i := 0; i < len(recsU); i++ {
recsU[i][0] = node.bestRecU[i].score
recsU[i][1] = node.bestRecU[i].avg
}
log.Debug("From:", node.value)
log.Debug("Like:", recsL)
log.Debug("Disl:", recsD)
log.Debug("Unkn:", recsU)
if node.like != nil {
queue = append(queue, node.like)
}
if node.unknown != nil {
queue = append(queue, node.unknown)
}
if node.dislike != nil {
queue = append(queue, node.dislike)
}
}
}
func (wt *windowTrader) NewPrices(curr string, ts int64) {
var realOpsStr string
wt.mutex.Lock()
defer wt.mutex.Unlock()
if wt.realOps {
realOpsStr = "Real"
} else {
realOpsStr = "Simulation"
}
currVals := wt.collector.GetAllCurrVals()
lastVal := currVals[curr][len(currVals[curr])-1]
if wt.opRunning == nil {
// Check if we can buy
if should, typeOper := wt.trainer.ShouldIOperate(curr, currVals, wt.id); should {
log.Debug("Buy:", curr, "ID:", wt.id, "Price:", lastVal.Ask, "Type:", realOpsStr)
if typeOper == "buy" {
wt.opRunning, _ = wt.collector.Buy(curr, wt.unitsToUse, lastVal.Ask, wt.realOps, lastVal.Ts)
} else {
wt.opRunning, _ = wt.collector.Sell(curr, wt.unitsToUse, lastVal.Bid, wt.realOps, lastVal.Ts)
}
wt.askVal = lastVal
}
} else {
// Check if we can sell
if wt.trainer.ShouldIClose(curr, wt.askVal, currVals, wt.id, wt.opRunning) {
scoreBefSell := wt.GetScore(3)
totalProfitBefSell := wt.GetTotalProfit()
if err := wt.collector.CloseOrder(wt.opRunning, lastVal.Ts); err == nil {
wt.ops = append(wt.ops, wt.opRunning)
log.Debug("Selling:", curr, "Trader:", wt.id, "Profit:", wt.ops[len(wt.ops)-1].Profit, "Time:", float64(lastVal.Ts-wt.askVal.Ts)/tsMultToSecs, "TotalProfit:", wt.GetTotalProfit(), "Score:", wt.GetScore(3), "scoreBefSell:", scoreBefSell, "totalProfitBefSell:", totalProfitBefSell, "Real:", realOpsStr)
wt.opRunning = nil
}
}
}
}
func (rc *Recommender) AddRecord(recID uint64, scores map[uint64]uint8) {
var sc *score
var existingRecord bool
rc.dirty = true
// If the system is cloning the data to process the tree, just leave
// the data on the buffer
if rc.cloning {
rc.cloningBuffer[recID] = scores
return
}
rc.mutex.Lock()
if sc, existingRecord = rc.records[recID]; existingRecord {
if sc.prev != nil {
sc.prev.next = sc.next
} else {
// This is the older elem
rc.older = rc.older.next
}
if sc.next != nil {
sc.next.prev = sc.prev
} else {
// This is the last elem
rc.newer = rc.newer.prev
}
rc.totalClassif += uint64(len(scores) - len(sc.scores))
sc.scores = scores
} else {
sc = &score{
recID: recID,
scores: scores,
}
rc.records[recID] = sc
rc.totalClassif += uint64(len(scores))
}
if rc.newer != nil {
sc.prev = rc.newer
rc.newer.next = sc
rc.newer = sc
} else {
rc.newer = sc
rc.older = sc
}
rc.mutex.Unlock()
log.Debug("Stored elements:", rc.totalClassif, "Max stored elements:", rc.maxClassif)
}
func (im *Model) GetMaxShardsToAcquire(totalShards int) (total int) {
im.mutex.Lock()
defer im.mutex.Unlock()
log.Debug("Instances alive:", im.instancesAlive)
if len(im.instancesAlive) == 0 {
return 0
}
total = totalShards / len(im.instancesAlive)
if im.instancesAlive[len(im.instancesAlive)-1] == hostName {
total += totalShards % len(im.instancesAlive)
}
return
}
func (tr *TrainerCorrelations) ShouldIClose(curr string, askVal *charont.CurrVal, vals map[string][]*charont.CurrVal, traderID int, ord *charont.Order) bool {
var centroid, traderCentroid int
var currentWin float64
currVal := vals[curr][len(vals[curr])-1]
traderAvgDiv := float64(traderID % clustersToUse)
secondsUsed := (currVal.Ts - askVal.Ts) / tsMultToSecs
if secondsUsed > secsToWaitUntilForceSell {
// Out of time...
log.Debug("Selling by time: Centroid:", traderCentroid, secondsUsed, secsToWaitUntilForceSell)
return true
}
if ord.Type == "buy" {
traderCentroid = (traderID - (TrainersToRun / 2)) / clustersToUse
currentWin = (currVal.Bid / ord.Price) - 1
centroid = tr.centroidsForAsk[curr][traderCentroid]
if currentWin > tr.maxWinByCentroid[curr][centroid]/traderAvgDiv {
// More than the AVG profit/3
log.Debug("Selling by profit > avg/", traderAvgDiv, ", Centroid:", traderCentroid, "Profit:", currentWin, "Avg:", tr.maxWinByCentroid[curr][tr.centroidsForAsk[curr][traderCentroid]])
return true
}
if currentWin < tr.maxLossByCentroid[curr][traderCentroid] {
log.Debug("Selling by max loss, loss:", currentWin, "Centroid:", traderCentroid, "Max Loss:", maxLoss, "Max loss Avg by centroid:", tr.maxLossByCentroid[curr][traderCentroid])
return true
}
} else {
traderCentroid = traderID / clustersToUse
currentWin = (ord.CloseRate / currVal.Ask) - 1
centroid = tr.centroidsForSell[curr][traderCentroid]
if currentWin > tr.maxWinByCentroidSell[curr][centroid]/traderAvgDiv {
// More than the AVG profit/3
log.Debug("Selling by profit > avg/", traderAvgDiv, ", Centroid:", traderCentroid, "Profit:", currentWin, "Avg:", tr.maxWinByCentroidSell[curr][tr.centroidsForAsk[curr][traderCentroid]])
return true
}
if currentWin < tr.maxLossByCentroidSell[curr][traderCentroid] {
log.Debug("Selling by max loss, loss:", currentWin, "Centroid:", traderCentroid, "Max Loss:", maxLoss, "Max loss Avg by centroid:", tr.maxLossByCentroidSell[curr][traderCentroid])
return true
}
}
if secondsUsed > secsToWaitUntilForceSell/2 && currentWin > 0 {
// More than the half of the time and some profit
log.Debug("Selling by profit > 1 and time > totalTime/2, secs used", secondsUsed, "Centroid:", traderCentroid, "Secs to wait:", secsToWaitUntilForceSell/2, "Profit:", currentWin, "Avg:", tr.maxWinByCentroid[curr][centroid])
return true
}
return false
}
func GetHades(trainer philoctetes.TrainerInt, traders int, from int, collector charont.Int, unitsToUse, samplesToConsiderer, lastOpsToConsider, tradesThatCanPlay, maxSecsToWait int) (hades *Hades) {
hades = &Hades{
traders: make([]hermes.Int, philoctetes.TrainersToRun*len(collector.GetCurrencies())),
collector: collector,
tradesThatCanPlay: tradesThatCanPlay,
lastOpsToConsider: lastOpsToConsider,
tradersPlaying: make(map[int]hermes.Int),
}
for i, curr := range collector.GetCurrencies() {
for t := 0; t < philoctetes.TrainersToRun; t++ {
log.Debug("Launching trader:", curr, "Id:", t, "TotalToLaunch:", len(hades.traders), i*t)
hades.traders[i*philoctetes.TrainersToRun+t] = hermes.GetWindowTrader(t, trainer, curr, collector, unitsToUse, samplesToConsiderer, maxSecsToWait)
}
}
go collector.Run()
go hades.manageTraders()
return
}
func (api *Oanda) CloseOrder(ord *Order, ts int64) (err error) {
var realOrder string
ord.SellTs = ts
ord.Open = false
if ord.Real {
resp, err := api.doRequest("DELETE", fmt.Sprintf(CHECK_ORDER_URL, api.endpoint, api.account.AccountId, ord.Id), nil)
if err != nil {
log.Error("Problem trying to close an open position, Error:", err)
return err
}
generic := map[string]float64{}
json.Unmarshal(resp, &generic)
ord.CloseRate = generic["price"]
ord.Profit = generic["profit"] / float64(ord.Units)
api.mutex.Lock()
delete(api.openOrders, ord.Id)
api.mutex.Unlock()
api.currentWin += ord.Profit
realOrder = "Real"
} else {
api.mutex.Lock()
lastPrice := api.currencyValues[ord.Curr[4:]][len(api.currencyValues[ord.Curr[4:]])-1]
api.mutex.Unlock()
if ord.Type == "buy" {
ord.CloseRate = lastPrice.Bid
} else {
ord.Price = lastPrice.Ask
}
ord.Profit = ord.CloseRate/ord.Price - 1
realOrder = "Simultaion"
}
log.Debug("Closed Order:", ord.Id, "BuyTs:", time.Unix(ord.BuyTs/tsMultToSecs, 0), "TimeToSell:", (ord.SellTs-ord.BuyTs)/tsMultToSecs, "Curr:", ord.Curr, "OpenRate:", ord.Price, "Close rate:", ord.CloseRate, "And Profit:", ord.Profit, "Current Win:", api.currentWin, "Type:", realOrder)
return
}
func (um *Model) initTable() {
pKey := dynamodb.PrimaryKey{dynamodb.NewStringAttribute(cPrimKey, ""), nil}
um.table = um.conn.NewTable(um.tableName, pKey)
res, err := um.table.DescribeTable()
if err != nil {
log.Info("Creating a new table on DynamoDB:", um.tableName)
td := dynamodb.TableDescriptionT{
TableName: um.tableName,
AttributeDefinitions: []dynamodb.AttributeDefinitionT{
dynamodb.AttributeDefinitionT{cPrimKey, "S"},
},
KeySchema: []dynamodb.KeySchemaT{
dynamodb.KeySchemaT{cPrimKey, "HASH"},
},
ProvisionedThroughput: dynamodb.ProvisionedThroughputT{
ReadCapacityUnits: cDefaultWRCapacity,
WriteCapacityUnits: cDefaultWRCapacity,
},
}
if _, err := um.conn.CreateTable(td); err != nil {
log.Error("Error trying to create a table on Dynamo DB, table:", um.tableName, "Error:", err)
}
if res, err = um.table.DescribeTable(); err != nil {
log.Error("Error trying to describe a table on Dynamo DB, table:", um.tableName, "Error:", err)
}
}
for "ACTIVE" != res.TableStatus {
if res, err = um.table.DescribeTable(); err != nil {
log.Error("Can't describe Dynamo DB instances table, Error:", err)
}
log.Debug("Waiting for active table, current status:", res.TableStatus)
time.Sleep(time.Second)
}
}
func TestPlaceOrder(t *testing.T) {
cfg.Init("v", "dev")
api, err := InitOandaApi(
cfg.GetStr("oanda", "endpoint"),
cfg.GetStr("oanda", "token"),
int(cfg.GetInt("oanda", "account-id")),
strings.Split(cfg.GetStr("oanda", "currencies"), ","),
cfg.GetStr("oanda", "exanges-log"),
)
if err != nil {
t.Error("Problem connecting with oanda, Error:", err)
}
curr := api.GetBaseCurrency()
if curr != "EUR" {
t.Error("The configured value on the test account was EUR, but:", curr, "was returned")
}
currs := api.GetCurrencies()
log.Debug(currs)
order, err := api.Buy("USD", 1, 1.3, true, time.Now().Unix())
if err != nil {
t.Error("Problem placing an order, Error:", err)
}
err = api.CloseOrder(order, time.Now().Unix())
if err != nil {
t.Error("Problem closing an order, Error:", err)
}
order, err = api.Sell("USD", 1, 1.0, true, time.Now().Unix())
if err != nil {
t.Error("Problem placing an order, Error:", err)
}
err = api.CloseOrder(order, time.Now().Unix())
if err != nil {
t.Error("Problem closing an order, Error:", err)
}
order, err = api.Buy("USD", 1, 1.3, false, time.Now().Unix())
if err != nil {
t.Error("Problem placing an order, Error:", err)
}
err = api.CloseOrder(order, time.Now().Unix())
if err != nil {
t.Error("Problem closing an order, Error:", err)
}
order, err = api.Sell("USD", 1, 1.0, false, time.Now().Unix())
if err != nil {
t.Error("Problem placing an order, Error:", err)
}
err = api.CloseOrder(order, time.Now().Unix())
if err != nil {
t.Error("Problem closing an order, Error:", err)
}
}
func (tr *TrainerCorrelations) getCentroids(valsForScore ByScore) (centroidsCurr [][]float64, maxWinByCentroid []float64, maxLossByCentroid []float64, centroidsForAsk []int) {
sort.Sort(valsForScore)
// Using k-means try to find 2 centroids:
// - Buy
// - Don't buy
// Identify what centroid is what
// Check the Precission and recall obtained using this 2 centroids
// Init the clusters centroids
log.Debug("Moving centroids")
for c := 0; c < clusters; c++ {
pos := c * ((len(valsForScore) - 1) / (clusters - 1))
centroidsCurr = append(centroidsCurr, []float64{
valsForScore[pos].charAskMin, // ask min relation
valsForScore[pos].charAskMax, // ask max relation
valsForScore[pos].charAskMean, // ask mean relation
valsForScore[pos].charAskMode, // ask mode relation
})
}
modified := true
for modified {
scoresByCentroid := make([][]*ScoreCounter, clusters)
for _, score := range valsForScore {
centroid := tr.getClosestCentroid(score, centroidsCurr)
scoresByCentroid[centroid] = append(scoresByCentroid[centroid], score)
}
for i := 0; i < clusters; i++ {
log.Debug("Items centroid:", i, len(scoresByCentroid[i]))
}
// Move the centroids
modified = false
for c := 0; c < clusters; c++ {
log.Debug("scoresByCentroid:", c, len(scoresByCentroid[c]))
oldCentroid := centroidsCurr[c]
centroidsCurr[c] = []float64{
0.0,
0.0,
0.0,
0.0,
}
scoresCentroid := float64(len(scoresByCentroid[c]))
for _, score := range scoresByCentroid[c] {
centroidsCurr[c][0] += score.charAskMin / scoresCentroid
centroidsCurr[c][1] += score.charAskMax / scoresCentroid
centroidsCurr[c][2] += score.charAskMean / scoresCentroid
centroidsCurr[c][3] += score.charAskMode / scoresCentroid
}
// Check if the centroid was moved or not
for i := 0; i < len(oldCentroid); i++ {
if oldCentroid[i] != centroidsCurr[c][i] {
modified = true
}
}
log.Debug("Centroids:", c, oldCentroid, centroidsCurr[c])
}
}
// With the clsters initted, try to estimate the score by centroid
avgScoreCent := make([]float64, clusters)
avgMaxWin := make([]float64, clusters)
maxWinByCentroid = make([]float64, clusters)
maxLossByCentroid = make([]float64, clusters)
scoresByCentroid := make(map[int][]*ScoreCounter)
for _, score := range valsForScore {
centroid := tr.getClosestCentroid(score, centroidsCurr)
avgScoreCent[centroid] += score.score
avgMaxWin[centroid] += score.maxWinNoNorm
maxWinByCentroid[centroid] += score.maxWinNoNorm
if maxLossByCentroid[centroid] > score.maxLossNoNorm {
maxLossByCentroid[centroid] = score.maxLossNoNorm
}
if _, ok := scoresByCentroid[centroid]; ok {
scoresByCentroid[centroid] = append(scoresByCentroid[centroid], score)
} else {
scoresByCentroid[centroid] = []*ScoreCounter{score}
}
}
// Try to reduce the max loss until contain the cMinLossRange points
// inside the range
for c, scores := range scoresByCentroid {
usedScores := len(scores)
valsToInclude := int(float64(len(scores)) * cMinLossRange)
for usedScores != valsToInclude && usedScores > valsToInclude {
maxLossByCentroid[c] *= 0.99
usedScores = 0
for _, score := range scores {
if maxLossByCentroid[c] < score.maxLossNoNorm {
usedScores++
}
}
}
log.Debug("Used Scores for MinLoss:", usedScores, "of:", len(scores), "Perc:", float64(usedScores)/float64(len(scores)), "Max Loss:", maxLossByCentroid[c])
}
scores := make([]float64, clusters)
for c := 0; c < clusters; c++ {
maxWinByCentroid[c] /= math.Abs(float64(len(scoresByCentroid[c])))
avgScoreCent[c] /= float64(len(scoresByCentroid[c]))
avgMaxWin[c] /= float64(len(scoresByCentroid[c]))
scores[c] = avgMaxWin[c]
log.Debug("Centroid:", c, "Items:", len(scoresByCentroid[c]), "Score", avgScoreCent[c], "Avg MaxWin:", avgMaxWin[c], "CentroidPos:", centroidsCurr[c])
}
sort.Float64s(scores)
centroidsForAsk = []int{}
for _, score := range scores[clusters-clustersToUse:] {
for k, v := range avgMaxWin {
if v == score {
centroidsForAsk = append(centroidsForAsk, k)
}
}
}
return
}
func (tr *TrainerCorrelations) studyCurrencies(TimeRangeToStudySecs int64) {
log.Debug(len(tr.feeds["USD"]), tr.feeds["USD"][0])
currsTrained := 0
for curr, vals := range tr.feeds {
/*if curr != "USD" {
continue
}*/
go func(curr string, vals []*charont.CurrVal) {
valsForScore := ByScore{}
// Params to Buy
minMaxWin := [2]float64{math.Inf(1), math.Inf(-1)}
minMaxD := [2]float64{math.Inf(1), math.Inf(-1)}
minMaxW := [2]float64{math.Inf(1), math.Inf(-1)}
lastWindowFirstPosUsed := 0
// Params to Sell
valsForScoreSell := ByScore{}
minMaxWinSell := [2]float64{math.Inf(1), math.Inf(-1)}
minMaxDSell := [2]float64{math.Inf(1), math.Inf(-1)}
minMaxWSell := [2]float64{math.Inf(1), math.Inf(-1)}
pointToStudyLoop:
for i, val := range vals {
if i%1000 == 0 {
log.Debug("Processed:", i, "points for currency:", curr)
}
charAskMin, charAskMax, charAskMean, charAskMode, noPossibleToStudy, firstWindowPos := tr.getPointCharacteristics(val, vals[lastWindowFirstPosUsed:i])
//log.Debug("Study - Val:", val, "lastWindowFirstPosUsed:", lastWindowFirstPosUsed, "Vals:", len(vals[lastWindowFirstPosUsed:i]), charAskMin, charAskMax, charAskMean, charAskMode, noPossibleToStudy, firstWindowPos)
if noPossibleToStudy {
log.Debug("No study, no possible")
continue pointToStudyLoop
}
lastWindowFirstPosUsed += firstWindowPos
// Get the range to Buy
found := int64(-1)
w := -1.0
maxWin := -1.0
maxLoss := 10.0
winningRangStudy:
for _, futureVal := range vals[i+1:] {
currWin := futureVal.Bid / val.Ask
if currWin > maxWin {
maxWin = currWin
}
if currWin < maxLoss {
maxLoss = currWin
}
if found != -1 {
if futureVal.Bid < val.Ask {
w = float64(futureVal.Ts - found)
break winningRangStudy
}
} else {
if currWin > 1 {
found = futureVal.Ts
}
}
}
// Get the range to Sell
foundSell := int64(-1)
wSell := -1.0
maxWinSell := -1.0
maxLossSell := 10.0
winningRangStudySell:
for _, futureVal := range vals[i+1:] {
currWin := val.Bid / futureVal.Ask
if currWin > maxWinSell {
maxWinSell = currWin
}
if currWin < maxLossSell {
maxLossSell = currWin
}
if foundSell != -1 {
if val.Bid < futureVal.Ask {
wSell = float64(futureVal.Ts - foundSell)
break winningRangStudySell
}
} else {
if currWin > 1 {
foundSell = futureVal.Ts
}
}
}
// Calculate the scores for Buy
maxWinFlo := maxWin - 1
maxLossFlo := maxLoss - 1
if found != -1 {
if w != -1 {
//log.Debug("New Range:", curr, w/1000000000, float64(found-val.Ts)/1000000000, maxWin)
dFlo := float64(found - val.Ts)
valsForScore = append(valsForScore, &ScoreCounter{
val: val,
w: w,
d: dFlo,
maxWin: maxWinFlo,
maxWinNoNorm: maxWinFlo,
maxLossNoNorm: maxLossFlo,
charAskMin: charAskMin,
charAskMax: charAskMax,
charAskMean: charAskMean,
charAskMode: charAskMode,
})
if w < minMaxW[0] {
minMaxW[0] = w
}
if w > minMaxW[1] {
minMaxW[1] = w
}
if dFlo < minMaxD[0] {
minMaxD[0] = dFlo
}
if dFlo > minMaxD[1] {
minMaxD[1] = dFlo
}
if maxWinFlo < minMaxWin[0] {
minMaxWin[0] = maxWinFlo
}
if maxWinFlo > minMaxWin[1] {
minMaxWin[1] = maxWinFlo
}
}
} else {
// This is a bad point, we need to keep track of this also
valsForScore = append(valsForScore, &ScoreCounter{
val: val,
w: 0,
d: 1,
maxWin: maxWinFlo,
maxWinNoNorm: maxWinFlo,
// We don't want to know the
// max loss for bad points,
// since it is going to be used
// as boundary to sell
maxLossNoNorm: 0,
charAskMin: charAskMin,
charAskMax: charAskMax,
charAskMean: charAskMean,
charAskMode: charAskMode,
})
}
// Calculate the scores for Sell
maxWinFloSell := maxWinSell - 1
maxLossFloSell := maxLossSell - 1
if foundSell != -1 {
if wSell != -1 {
//log.Debug("New Range:", curr, w/1000000000, float64(found-val.Ts)/1000000000, maxWin)
dFloSell := float64(foundSell - val.Ts)
valsForScoreSell = append(valsForScoreSell, &ScoreCounter{
val: val,
w: wSell,
d: dFloSell,
maxWin: maxWinFloSell,
maxWinNoNorm: maxWinFloSell,
maxLossNoNorm: maxLossFloSell,
charAskMin: charAskMin,
charAskMax: charAskMax,
charAskMean: charAskMean,
charAskMode: charAskMode,
})
if wSell < minMaxWSell[0] {
minMaxWSell[0] = wSell
}
if wSell > minMaxWSell[1] {
minMaxWSell[1] = wSell
}
if dFloSell < minMaxDSell[0] {
minMaxDSell[0] = dFloSell
}
if dFloSell > minMaxDSell[1] {
minMaxDSell[1] = dFloSell
}
if maxWinFloSell < minMaxWinSell[0] {
minMaxWinSell[0] = maxWinFloSell
}
if maxWinFloSell > minMaxWinSell[1] {
minMaxWinSell[1] = maxWinFloSell
}
}
} else {
// This is a bad point, we need to keep track of this also
valsForScore = append(valsForScore, &ScoreCounter{
val: val,
w: 0,
d: 1,
maxWin: maxWinFloSell,
maxWinNoNorm: maxWinFloSell,
maxLossNoNorm: 0,
charAskMin: charAskMin,
charAskMax: charAskMax,
charAskMean: charAskMean,
charAskMode: charAskMode,
})
}
}
// Now normalise and prepare the scores to Buy
for i := 0; i < len(valsForScore); i++ {
if valsForScore[i].w != 0 {
//log.Debug("Before Normalise:", valsForScore[i])
valsForScore[i].maxWin = (valsForScore[i].maxWin - minMaxWin[0]) / (minMaxWin[1] - minMaxWin[0])
valsForScore[i].d = (valsForScore[i].d - minMaxD[0]) / (minMaxD[1] - minMaxD[0])
valsForScore[i].w = (valsForScore[i].w - minMaxW[0]) / (minMaxW[1] - minMaxW[0])
if valsForScore[i].maxWin != 0 && valsForScore[i].d != 0 && valsForScore[i].w != 0 {
valsForScore[i].score = (valsForScore[i].maxWin * valsForScore[i].w) / valsForScore[i].d
//log.Debug("After Normalise:", valsForScore[i])
} else {
valsForScore[i].score = -1
}
}
}
// Now normalise and prepare the scores to Sell
log.Debug("Normalise:", minMaxWinSell)
log.Debug("ValsForSell:", valsForScoreSell[:10])
for i := 0; i < len(valsForScoreSell); i++ {
if valsForScoreSell[i].w != 0 {
//log.Debug("Before Normalise:", valsForScoreSell[i])
valsForScoreSell[i].maxWin = (valsForScoreSell[i].maxWin - minMaxWinSell[0]) / (minMaxWinSell[1] - minMaxWinSell[0])
valsForScoreSell[i].d = (valsForScoreSell[i].d - minMaxDSell[0]) / (minMaxDSell[1] - minMaxDSell[0])
valsForScoreSell[i].w = (valsForScoreSell[i].w - minMaxWSell[0]) / (minMaxWSell[1] - minMaxWSell[0])
if valsForScoreSell[i].maxWin != 0 && valsForScoreSell[i].d != 0 && valsForScoreSell[i].w != 0 {
valsForScoreSell[i].score = (valsForScoreSell[i].maxWin * valsForScoreSell[i].w) / valsForScoreSell[i].d
//log.Debug("After Normalise:", valsForScoreSell[i])
} else {
valsForScoreSell[i].score = -1
}
}
}
tr.mutex.Lock()
tr.centroidsCurr[curr], tr.maxWinByCentroid[curr], tr.maxLossByCentroid[curr], tr.centroidsForAsk[curr] = tr.getCentroids(valsForScore)
log.Debug("Centroides to buy:", curr, tr.centroidsForAsk[curr], "Max loss:", tr.maxLossByCentroid[curr])
tr.centroidsCurrSell[curr], tr.maxWinByCentroidSell[curr], tr.maxLossByCentroidSell[curr], tr.centroidsForSell[curr] = tr.getCentroids(valsForScoreSell)
log.Debug("Centroides to sell:", curr, tr.centroidsForSell[curr], "Max loss:", tr.maxLossByCentroidSell[curr])
tr.mutex.Unlock()
currsTrained++
}(curr, vals)
}
for currsTrained < len(tr.feeds) {
time.Sleep(time.Second)
}
}
func TestRecommenderSaveLoad(t *testing.T) {
maxClassifications := uint64(1000000)
runtime.GOMAXPROCS(runtime.NumCPU())
sh := NewShard("/testing", "test_collab_insertion", maxClassifications, 5, "eu-west-1")
f, err := os.Open(TESTSET)
if err != nil {
log.Error("Can't read the the test set file:", TESTSET, "Error:", err)
t.Fail()
}
r := bufio.NewReader(f)
s, e := Readln(r)
i := 0
for e == nil && i < 100000 {
s, e = Readln(r)
recID, scores := parseLine(s)
sh.AddRecord(recID, scores)
i++
if i%1000 == 0 {
log.Debug("Lines processed:", i)
}
}
time.Sleep(time.Second)
if sh.totalClassif > maxClassifications {
t.Error(
"Problem with the garbage collection, the total number of stored stores are:",
sh.totalClassif, "and the max defined boundary is:", maxClassifications)
}
if sh.status != STATUS_SARTING {
t.Error("The expectede status was:", STATUS_SARTING, "but the actual one is:", sh.status)
}
log.Debug("Processing tree...")
sh.RecalculateTree()
if sh.status != STATUS_ACTIVE {
t.Error("The expectede status was:", STATUS_ACTIVE, "but the actual one is:", sh.status)
}
s, e = Readln(r)
recID, scores := parseLine(s)
recomendationsBef := sh.CalcScores(recID, scores, 10)
if len(recomendationsBef) != 10 {
t.Error("The expected recommendations was 10, but:", len(recomendationsBef), "obtained.")
}
prevScores := sh.totalClassif
sh.SaveBackup()
sh = NewShard("/testing", "test_collab_insertion", maxClassifications, 5, "eu-west-1")
sh.RecalculateTree()
if sh.status != STATUS_NORECORDS {
t.Error("The expectede status was:", STATUS_NORECORDS, "but the actual one is:", sh.status)
}
sh.LoadBackup()
sh.RecalculateTree()
if prevScores != sh.totalClassif {
t.Error(
"Before store a backup the number of records was:", prevScores,
"but after load the backup is:", sh.totalClassif)
}
recomendationsAfter := sh.CalcScores(recID, scores, 10)
if len(recomendationsAfter) != 10 {
t.Error("The expected recommendations was 10, but:", len(recomendationsAfter), "obtained.")
}
log.Debug("Classifications:", sh.maxClassif)
}
func (api *Oanda) ratesCollector() {
var feeds map[string][]feedStruc
api.mutex.Lock()
api.currencyValues = make(map[string][]*CurrVal)
currExange := make([]string, len(api.currencies))
lasCurrPriceA := make(map[string]float64)
lasCurrPriceB := make(map[string]float64)
log.Debug("Curr:", api.currencies)
for i, curr := range api.currencies {
api.currencyValues[curr] = []*CurrVal{}
currExange[i] = fmt.Sprintf("%s_%s", api.account.AccountCurrency, curr)
lasCurrPriceA[curr] = 0
lasCurrPriceB[curr] = 0
}
api.mutex.Unlock()
feedsUrl := fmt.Sprintf(FEEDS_URL, api.endpoint) + strings.Join(currExange, "%2C")
log.Info("Parsing currencies from the feeds URL:", feedsUrl)
c := time.Tick((1000 / COLLECT_BY_SECOND) * time.Millisecond)
for _ = range c {
resp, err := api.doRequest("GET", feedsUrl, nil)
if err != nil {
log.Error("The feeds URL can't be parsed, Error:", err)
continue
}
if err = json.Unmarshal(resp, &feeds); err != nil {
log.Error("The feeds response body is not a JSON valid, Error:", err, "Resp:", string(resp))
continue
}
// Ok, all fine, we are going to parse the feeds
for _, feed := range feeds["prices"] {
curr := feed.Instrument[len(api.account.AccountCurrency)+1:]
if lasCurrPriceA[curr] != feed.Ask || lasCurrPriceB[curr] != feed.Bid {
log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
api.mutex.Lock()
api.currencyValues[curr] = append(api.currencyValues[curr], &CurrVal{
Ts: time.Now().UnixNano(),
Bid: feed.Bid,
Ask: feed.Ask,
})
api.mutex.Unlock()
if api.currLogsFile != nil {
api.mutex.Lock()
b, _ := json.Marshal(api.currencyValues[curr][len(api.currencyValues[curr])-1])
_, err := api.currLogsFile.WriteString(fmt.Sprintf("%s:%s\n", curr, string(b)))
api.mutex.Unlock()
if err != nil {
log.Error("Can't write into the currencies logs file, Error:", err)
}
}
if listeners, ok := api.listeners[curr]; ok {
for _, listener := range listeners {
go listener(curr, time.Now().UnixNano())
}
}
api.mutex.Lock()
if len(api.currencyValues[curr]) > MAX_RATES_TO_STORE {
api.currencyValues[curr] = api.currencyValues[curr][1:]
}
api.mutex.Unlock()
lasCurrPriceA[curr] = feed.Ask
lasCurrPriceB[curr] = feed.Bid
}
}
}
}
// ProcessNewTrees Creates a new set of numberOfTrees decission trees with a
// max deep of maxDeep. Specify on maxScore the max possible score for the
// elements
func ProcessNewTrees(records []map[uint64]uint8, maxDeep int, maxScore uint8, numberOfTrees int) (tr *Tree, avgScores map[uint64]float64) {
avgScores = make(map[uint64]float64)
elementsTotals := []elemTotals{}
elemsPos := make(map[uint64]int)
i := 0
log.Debug("Records:", len(records))
for _, record := range records {
for k, v := range record {
vUint64 := uint64(v)
v2Uint64 := vUint64 * vUint64
if p, ok := elemsPos[k]; ok {
elementsTotals[p].sum += vUint64
elementsTotals[p].sum2 += v2Uint64
elementsTotals[p].err += uint64((maxScore - v) * (maxScore - v))
elementsTotals[p].n++
} else {
elementsTotals = append(elementsTotals, elemTotals{
elemID: k,
sum: vUint64,
sum2: v2Uint64,
err: uint64((maxScore - v) * (maxScore - v)),
n: 1,
})
elemsPos[k] = i
i++
}
}
}
for _, v := range elementsTotals {
avgScores[v.elemID] = float64(v.sum) / float64(v.n)
}
tr = &Tree{
maxDeep: maxDeep,
maxScore: maxScore,
totalRecs: len(records),
tree: make(map[uint64]*tNode),
testMode: false,
}
if len(elemsPos) < numberOfTrees {
tr.numOfTrees = len(elemsPos)
} else {
tr.numOfTrees = numberOfTrees
}
i = 0
studiedIds := make(map[uint64]bool)
for {
if i >= tr.numOfTrees {
return
}
max := uint64(0)
maxKey := uint64(0)
for _, info := range elementsTotals {
if max < info.n {
if _, ok := studiedIds[info.elemID]; !ok {
max = info.n
maxKey = info.elemID
}
}
}
studiedIds[maxKey] = true
pos := elemsPos[maxKey]
log.Debug("----->>>> Building tree from:", maxKey, i, elementsTotals[pos].n)
delete(elemsPos, maxKey)
tr.tree[maxKey] = tr.getTreeNode(maxKey, elemsPos, elementsTotals, records, 1)
elemsPos[maxKey] = pos
i++
}
return
}
func (api *Oanda) placeMarketOrder(inst string, units int, side string, price float64, realOps bool, ts int64) (order *Order, err error) {
if !realOps {
api.mutex.Lock()
defer api.mutex.Unlock()
api.openOrders[api.simulatedOrders] = &Order{
Id: api.simulatedOrders,
Units: units,
Open: true,
Type: side,
Real: false,
BuyTs: ts,
Curr: inst,
}
if side == "buy" {
api.openOrders[api.simulatedOrders].Price = price
} else {
api.openOrders[api.simulatedOrders].CloseRate = price
}
api.simulatedOrders++
return api.openOrders[api.simulatedOrders-1], nil
}
var orderInfo orderStruc
var bound string
if side == "sell" {
bound = "lowerBound"
} else {
bound = "upperBound"
}
resp, err := api.doRequest("POST", fmt.Sprintf(PLACE_ORDER_URL, api.endpoint, api.account.AccountId),
url.Values{
"instrument": {inst},
"units": {fmt.Sprintf("%d", int(units))},
"side": {side},
"type": {"market"},
bound: {fmt.Sprintf("%f", price)},
})
if err != nil {
log.Error("Problem trying to place a new order, Error:", err)
return
}
err = json.Unmarshal(resp, &orderInfo)
if err != nil || orderInfo.Info == nil {
log.Error("The response from the server to place an order can't be parsed:", string(resp), "Error:", err)
return
}
log.Debug("Values: instrument:", inst, "units", units, "side:", side, "type: market ID:", orderInfo, "\nOrder response:", string(resp))
order = &Order{
Id: orderInfo.Info.Id,
Price: orderInfo.Price,
Units: units,
Open: true,
Type: side,
BuyTs: ts,
Curr: inst,
Real: true,
}
if side == "buy" {
order.Price = orderInfo.Price
} else {
order.CloseRate = orderInfo.Price
}
api.mutex.Lock()
api.openOrders[order.Id] = order
api.mutex.Unlock()
return
}