Example #1
0
func GetTrainerCorrelations(trainingFile string, TimeRangeToStudySecs int64) TrainerInt {
	log.Debug("Initializing trainer...")

	TimeRangeToStudySecs *= tsMultToSecs
	feedsFile, err := os.Open(trainingFile)
	log.Debug("File:", trainingFile)
	if err != nil {
		log.Fatal("Problem reading the logs file")
	}

	scanner := bufio.NewScanner(feedsFile)

	feeds := &TrainerCorrelations{
		feeds:                 make(map[string][]*charont.CurrVal),
		centroidsCurr:         make(map[string][][]float64),
		centroidsCurrSell:     make(map[string][][]float64),
		centroidsForAsk:       make(map[string][]int),
		centroidsForSell:      make(map[string][]int),
		maxWinByCentroid:      make(map[string][]float64),
		maxLossByCentroid:     make(map[string][]float64),
		maxWinByCentroidSell:  make(map[string][]float64),
		maxLossByCentroidSell: make(map[string][]float64),
	}

	i := 0
	for {
		var feed *charont.CurrVal

		if !scanner.Scan() {
			break
		}
		lineParts := strings.SplitN(scanner.Text(), ":", 2)
		curr := lineParts[0]
		if len(lineParts) < 2 {
			log.Error("The line:", i, "can't be parsed")
			continue
		}
		if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
			log.Error("The feeds response body is not a JSON valid, Error:", err, "Line:", i)
			continue
		}

		if _, ok := feeds.feeds[curr]; !ok {
			feeds.feeds[curr] = []*charont.CurrVal{}
		}
		feeds.feeds[curr] = append(feeds.feeds[curr], feed)
		i++
		if i%10000 == 0 {
			log.Debug("Lines:", i)
		}
	}

	for curr, scores := range feeds.feeds {
		log.Debug("Curr:", curr, "Scores:", len(scores))
	}

	feeds.studyCurrencies(TimeRangeToStudySecs)

	return feeds
}
Example #2
0
func (mock *Mock) CloseOrder(ord *Order, ts int64) (err error) {
	var realOrder string

	currVals := mock.currencyValues[ord.Curr]
	if ord.Type == "buy" {
		ord.CloseRate = currVals[len(currVals)-1].Bid
	} else {
		ord.Price = currVals[len(currVals)-1].Ask
	}
	ord.Profit = ord.CloseRate/ord.Price - 1
	ord.SellTs = ts

	mock.mutex.Lock()
	mock.ordersByCurr[ord.Curr] = append(mock.ordersByCurr[ord.Curr], ord)

	delete(mock.openOrders, ord.Id)
	mock.mutex.Unlock()

	if ord.Real {
		mock.currentWin += ord.Profit * float64(ord.Units)
		realOrder = "Real"
	} else {
		realOrder = "Simultaion"
	}

	log.Debug("Closed Order:", ord.Id, "TypeOrd:", ord.Type, "BuyTs:", time.Unix(ord.BuyTs/tsMultToSecs, 0), "TimeToSell:", (ord.SellTs-ord.BuyTs)/tsMultToSecs, "Curr:", ord.Curr, "OpenRate:", ord.Price, "Close rate:", ord.CloseRate, "And Profit:", ord.Profit, "Current Win:", mock.currentWin, "Type:", realOrder)
	return
}
Example #3
0
func (hades *Hades) CloseAllOpenOrdersAndFinish() {
	hades.tradesThatCanPlay = 0

	allFinished := false
	for !allFinished {
		time.Sleep(time.Second)

		allFinished = true
		for _, trader := range hades.tradersPlaying {
			if !trader.StopPlaying() {
				log.Debug("Trader:", trader.GetID(), "still playing...")
				allFinished = false
			}
		}
	}

	log.Debug("All the traders are done, close the system")
}
Example #4
0
func (tr *Tree) printTree() {
	queue := []*tNode{tr.tree[0]}

	for len(queue) > 0 {
		node := queue[0]
		queue = queue[1:]

		recsL := make([][2]float64, len(node.bestRecL))
		for i := 0; i < len(recsL); i++ {
			recsL[i][0] = node.bestRecL[i].score
			recsL[i][1] = node.bestRecL[i].avg
		}
		recsD := make([][2]float64, len(node.bestRecD))
		for i := 0; i < len(recsD); i++ {
			recsD[i][0] = node.bestRecD[i].score
			recsD[i][1] = node.bestRecD[i].avg
		}
		recsU := make([][2]float64, len(node.bestRecU))
		for i := 0; i < len(recsU); i++ {
			recsU[i][0] = node.bestRecU[i].score
			recsU[i][1] = node.bestRecU[i].avg
		}
		log.Debug("From:", node.value)
		log.Debug("Like:", recsL)
		log.Debug("Disl:", recsD)
		log.Debug("Unkn:", recsU)

		if node.like != nil {
			queue = append(queue, node.like)
		}
		if node.unknown != nil {
			queue = append(queue, node.unknown)
		}
		if node.dislike != nil {
			queue = append(queue, node.dislike)
		}
	}
}
Example #5
0
func (wt *windowTrader) NewPrices(curr string, ts int64) {
	var realOpsStr string

	wt.mutex.Lock()
	defer wt.mutex.Unlock()

	if wt.realOps {
		realOpsStr = "Real"
	} else {
		realOpsStr = "Simulation"
	}
	currVals := wt.collector.GetAllCurrVals()
	lastVal := currVals[curr][len(currVals[curr])-1]
	if wt.opRunning == nil {
		// Check if we can buy
		if should, typeOper := wt.trainer.ShouldIOperate(curr, currVals, wt.id); should {
			log.Debug("Buy:", curr, "ID:", wt.id, "Price:", lastVal.Ask, "Type:", realOpsStr)
			if typeOper == "buy" {
				wt.opRunning, _ = wt.collector.Buy(curr, wt.unitsToUse, lastVal.Ask, wt.realOps, lastVal.Ts)
			} else {
				wt.opRunning, _ = wt.collector.Sell(curr, wt.unitsToUse, lastVal.Bid, wt.realOps, lastVal.Ts)
			}
			wt.askVal = lastVal
		}
	} else {
		// Check if we can sell
		if wt.trainer.ShouldIClose(curr, wt.askVal, currVals, wt.id, wt.opRunning) {
			scoreBefSell := wt.GetScore(3)
			totalProfitBefSell := wt.GetTotalProfit()
			if err := wt.collector.CloseOrder(wt.opRunning, lastVal.Ts); err == nil {
				wt.ops = append(wt.ops, wt.opRunning)
				log.Debug("Selling:", curr, "Trader:", wt.id, "Profit:", wt.ops[len(wt.ops)-1].Profit, "Time:", float64(lastVal.Ts-wt.askVal.Ts)/tsMultToSecs, "TotalProfit:", wt.GetTotalProfit(), "Score:", wt.GetScore(3), "scoreBefSell:", scoreBefSell, "totalProfitBefSell:", totalProfitBefSell, "Real:", realOpsStr)
				wt.opRunning = nil
			}
		}
	}
}
Example #6
0
func (rc *Recommender) AddRecord(recID uint64, scores map[uint64]uint8) {
	var sc *score
	var existingRecord bool

	rc.dirty = true
	// If the system is cloning the data to process the tree, just leave
	// the data on the buffer
	if rc.cloning {
		rc.cloningBuffer[recID] = scores
		return
	}
	rc.mutex.Lock()
	if sc, existingRecord = rc.records[recID]; existingRecord {
		if sc.prev != nil {
			sc.prev.next = sc.next
		} else {
			// This is the older elem
			rc.older = rc.older.next
		}
		if sc.next != nil {
			sc.next.prev = sc.prev
		} else {
			// This is the last elem
			rc.newer = rc.newer.prev
		}

		rc.totalClassif += uint64(len(scores) - len(sc.scores))
		sc.scores = scores
	} else {
		sc = &score{
			recID:  recID,
			scores: scores,
		}
		rc.records[recID] = sc
		rc.totalClassif += uint64(len(scores))
	}

	if rc.newer != nil {
		sc.prev = rc.newer
		rc.newer.next = sc
		rc.newer = sc
	} else {
		rc.newer = sc
		rc.older = sc
	}
	rc.mutex.Unlock()

	log.Debug("Stored elements:", rc.totalClassif, "Max stored elements:", rc.maxClassif)
}
Example #7
0
func (im *Model) GetMaxShardsToAcquire(totalShards int) (total int) {
	im.mutex.Lock()
	defer im.mutex.Unlock()

	log.Debug("Instances alive:", im.instancesAlive)
	if len(im.instancesAlive) == 0 {
		return 0
	}

	total = totalShards / len(im.instancesAlive)
	if im.instancesAlive[len(im.instancesAlive)-1] == hostName {
		total += totalShards % len(im.instancesAlive)
	}

	return
}
Example #8
0
func (tr *TrainerCorrelations) ShouldIClose(curr string, askVal *charont.CurrVal, vals map[string][]*charont.CurrVal, traderID int, ord *charont.Order) bool {
	var centroid, traderCentroid int
	var currentWin float64

	currVal := vals[curr][len(vals[curr])-1]

	traderAvgDiv := float64(traderID % clustersToUse)
	secondsUsed := (currVal.Ts - askVal.Ts) / tsMultToSecs

	if secondsUsed > secsToWaitUntilForceSell {
		// Out of time...
		log.Debug("Selling by time: Centroid:", traderCentroid, secondsUsed, secsToWaitUntilForceSell)
		return true
	}

	if ord.Type == "buy" {
		traderCentroid = (traderID - (TrainersToRun / 2)) / clustersToUse
		currentWin = (currVal.Bid / ord.Price) - 1
		centroid = tr.centroidsForAsk[curr][traderCentroid]

		if currentWin > tr.maxWinByCentroid[curr][centroid]/traderAvgDiv {
			// More than the AVG profit/3
			log.Debug("Selling by profit > avg/", traderAvgDiv, ", Centroid:", traderCentroid, "Profit:", currentWin, "Avg:", tr.maxWinByCentroid[curr][tr.centroidsForAsk[curr][traderCentroid]])
			return true
		}

		if currentWin < tr.maxLossByCentroid[curr][traderCentroid] {
			log.Debug("Selling by max loss, loss:", currentWin, "Centroid:", traderCentroid, "Max Loss:", maxLoss, "Max loss Avg by centroid:", tr.maxLossByCentroid[curr][traderCentroid])
			return true
		}
	} else {
		traderCentroid = traderID / clustersToUse
		currentWin = (ord.CloseRate / currVal.Ask) - 1
		centroid = tr.centroidsForSell[curr][traderCentroid]

		if currentWin > tr.maxWinByCentroidSell[curr][centroid]/traderAvgDiv {
			// More than the AVG profit/3
			log.Debug("Selling by profit > avg/", traderAvgDiv, ", Centroid:", traderCentroid, "Profit:", currentWin, "Avg:", tr.maxWinByCentroidSell[curr][tr.centroidsForAsk[curr][traderCentroid]])
			return true
		}

		if currentWin < tr.maxLossByCentroidSell[curr][traderCentroid] {
			log.Debug("Selling by max loss, loss:", currentWin, "Centroid:", traderCentroid, "Max Loss:", maxLoss, "Max loss Avg by centroid:", tr.maxLossByCentroidSell[curr][traderCentroid])
			return true
		}
	}

	if secondsUsed > secsToWaitUntilForceSell/2 && currentWin > 0 {
		// More than the half of the time and some profit
		log.Debug("Selling by profit > 1 and time > totalTime/2, secs used", secondsUsed, "Centroid:", traderCentroid, "Secs to wait:", secsToWaitUntilForceSell/2, "Profit:", currentWin, "Avg:", tr.maxWinByCentroid[curr][centroid])
		return true
	}

	return false
}
Example #9
0
func GetHades(trainer philoctetes.TrainerInt, traders int, from int, collector charont.Int, unitsToUse, samplesToConsiderer, lastOpsToConsider, tradesThatCanPlay, maxSecsToWait int) (hades *Hades) {
	hades = &Hades{
		traders:           make([]hermes.Int, philoctetes.TrainersToRun*len(collector.GetCurrencies())),
		collector:         collector,
		tradesThatCanPlay: tradesThatCanPlay,
		lastOpsToConsider: lastOpsToConsider,
		tradersPlaying:    make(map[int]hermes.Int),
	}

	for i, curr := range collector.GetCurrencies() {
		for t := 0; t < philoctetes.TrainersToRun; t++ {
			log.Debug("Launching trader:", curr, "Id:", t, "TotalToLaunch:", len(hades.traders), i*t)
			hades.traders[i*philoctetes.TrainersToRun+t] = hermes.GetWindowTrader(t, trainer, curr, collector, unitsToUse, samplesToConsiderer, maxSecsToWait)
		}
	}

	go collector.Run()
	go hades.manageTraders()

	return
}
Example #10
0
func (api *Oanda) CloseOrder(ord *Order, ts int64) (err error) {
	var realOrder string

	ord.SellTs = ts
	ord.Open = false
	if ord.Real {
		resp, err := api.doRequest("DELETE", fmt.Sprintf(CHECK_ORDER_URL, api.endpoint, api.account.AccountId, ord.Id), nil)
		if err != nil {
			log.Error("Problem trying to close an open position, Error:", err)
			return err
		}
		generic := map[string]float64{}
		json.Unmarshal(resp, &generic)

		ord.CloseRate = generic["price"]
		ord.Profit = generic["profit"] / float64(ord.Units)
		api.mutex.Lock()
		delete(api.openOrders, ord.Id)
		api.mutex.Unlock()

		api.currentWin += ord.Profit
		realOrder = "Real"
	} else {
		api.mutex.Lock()
		lastPrice := api.currencyValues[ord.Curr[4:]][len(api.currencyValues[ord.Curr[4:]])-1]
		api.mutex.Unlock()
		if ord.Type == "buy" {
			ord.CloseRate = lastPrice.Bid
		} else {
			ord.Price = lastPrice.Ask
		}
		ord.Profit = ord.CloseRate/ord.Price - 1
		realOrder = "Simultaion"
	}
	log.Debug("Closed Order:", ord.Id, "BuyTs:", time.Unix(ord.BuyTs/tsMultToSecs, 0), "TimeToSell:", (ord.SellTs-ord.BuyTs)/tsMultToSecs, "Curr:", ord.Curr, "OpenRate:", ord.Price, "Close rate:", ord.CloseRate, "And Profit:", ord.Profit, "Current Win:", api.currentWin, "Type:", realOrder)

	return
}
Example #11
0
File: users.go Project: postfix/pit
func (um *Model) initTable() {
	pKey := dynamodb.PrimaryKey{dynamodb.NewStringAttribute(cPrimKey, ""), nil}
	um.table = um.conn.NewTable(um.tableName, pKey)

	res, err := um.table.DescribeTable()
	if err != nil {
		log.Info("Creating a new table on DynamoDB:", um.tableName)
		td := dynamodb.TableDescriptionT{
			TableName: um.tableName,
			AttributeDefinitions: []dynamodb.AttributeDefinitionT{
				dynamodb.AttributeDefinitionT{cPrimKey, "S"},
			},
			KeySchema: []dynamodb.KeySchemaT{
				dynamodb.KeySchemaT{cPrimKey, "HASH"},
			},
			ProvisionedThroughput: dynamodb.ProvisionedThroughputT{
				ReadCapacityUnits:  cDefaultWRCapacity,
				WriteCapacityUnits: cDefaultWRCapacity,
			},
		}

		if _, err := um.conn.CreateTable(td); err != nil {
			log.Error("Error trying to create a table on Dynamo DB, table:", um.tableName, "Error:", err)
		}
		if res, err = um.table.DescribeTable(); err != nil {
			log.Error("Error trying to describe a table on Dynamo DB, table:", um.tableName, "Error:", err)
		}
	}
	for "ACTIVE" != res.TableStatus {
		if res, err = um.table.DescribeTable(); err != nil {
			log.Error("Can't describe Dynamo DB instances table, Error:", err)
		}
		log.Debug("Waiting for active table, current status:", res.TableStatus)
		time.Sleep(time.Second)
	}
}
Example #12
0
func TestPlaceOrder(t *testing.T) {
	cfg.Init("v", "dev")

	api, err := InitOandaApi(
		cfg.GetStr("oanda", "endpoint"),
		cfg.GetStr("oanda", "token"),
		int(cfg.GetInt("oanda", "account-id")),
		strings.Split(cfg.GetStr("oanda", "currencies"), ","),
		cfg.GetStr("oanda", "exanges-log"),
	)

	if err != nil {
		t.Error("Problem connecting with oanda, Error:", err)
	}

	curr := api.GetBaseCurrency()
	if curr != "EUR" {
		t.Error("The configured value on the test account was EUR, but:", curr, "was returned")
	}

	currs := api.GetCurrencies()
	log.Debug(currs)

	order, err := api.Buy("USD", 1, 1.3, true, time.Now().Unix())
	if err != nil {
		t.Error("Problem placing an order, Error:", err)
	}

	err = api.CloseOrder(order, time.Now().Unix())
	if err != nil {
		t.Error("Problem closing an order, Error:", err)
	}

	order, err = api.Sell("USD", 1, 1.0, true, time.Now().Unix())
	if err != nil {
		t.Error("Problem placing an order, Error:", err)
	}

	err = api.CloseOrder(order, time.Now().Unix())
	if err != nil {
		t.Error("Problem closing an order, Error:", err)
	}

	order, err = api.Buy("USD", 1, 1.3, false, time.Now().Unix())
	if err != nil {
		t.Error("Problem placing an order, Error:", err)
	}

	err = api.CloseOrder(order, time.Now().Unix())
	if err != nil {
		t.Error("Problem closing an order, Error:", err)
	}

	order, err = api.Sell("USD", 1, 1.0, false, time.Now().Unix())
	if err != nil {
		t.Error("Problem placing an order, Error:", err)
	}

	err = api.CloseOrder(order, time.Now().Unix())
	if err != nil {
		t.Error("Problem closing an order, Error:", err)
	}
}
Example #13
0
func (tr *TrainerCorrelations) getCentroids(valsForScore ByScore) (centroidsCurr [][]float64, maxWinByCentroid []float64, maxLossByCentroid []float64, centroidsForAsk []int) {
	sort.Sort(valsForScore)

	// Using k-means try to find 2 centroids:
	//  - Buy
	//  - Don't buy
	// Identify what centroid is what
	// Check the Precission and recall obtained using this 2 centroids

	// Init the clusters centroids
	log.Debug("Moving centroids")
	for c := 0; c < clusters; c++ {
		pos := c * ((len(valsForScore) - 1) / (clusters - 1))
		centroidsCurr = append(centroidsCurr, []float64{
			valsForScore[pos].charAskMin,  // ask min relation
			valsForScore[pos].charAskMax,  // ask max relation
			valsForScore[pos].charAskMean, // ask mean relation
			valsForScore[pos].charAskMode, // ask mode relation
		})
	}

	modified := true
	for modified {
		scoresByCentroid := make([][]*ScoreCounter, clusters)
		for _, score := range valsForScore {
			centroid := tr.getClosestCentroid(score, centroidsCurr)
			scoresByCentroid[centroid] = append(scoresByCentroid[centroid], score)
		}

		for i := 0; i < clusters; i++ {
			log.Debug("Items centroid:", i, len(scoresByCentroid[i]))
		}

		// Move the centroids
		modified = false
		for c := 0; c < clusters; c++ {
			log.Debug("scoresByCentroid:", c, len(scoresByCentroid[c]))
			oldCentroid := centroidsCurr[c]
			centroidsCurr[c] = []float64{
				0.0,
				0.0,
				0.0,
				0.0,
			}
			scoresCentroid := float64(len(scoresByCentroid[c]))
			for _, score := range scoresByCentroid[c] {
				centroidsCurr[c][0] += score.charAskMin / scoresCentroid
				centroidsCurr[c][1] += score.charAskMax / scoresCentroid
				centroidsCurr[c][2] += score.charAskMean / scoresCentroid
				centroidsCurr[c][3] += score.charAskMode / scoresCentroid
			}

			// Check if the centroid was moved or not
			for i := 0; i < len(oldCentroid); i++ {
				if oldCentroid[i] != centroidsCurr[c][i] {
					modified = true
				}
			}
			log.Debug("Centroids:", c, oldCentroid, centroidsCurr[c])
		}
	}

	// With the clsters initted, try to estimate the score by centroid
	avgScoreCent := make([]float64, clusters)
	avgMaxWin := make([]float64, clusters)
	maxWinByCentroid = make([]float64, clusters)
	maxLossByCentroid = make([]float64, clusters)
	scoresByCentroid := make(map[int][]*ScoreCounter)
	for _, score := range valsForScore {
		centroid := tr.getClosestCentroid(score, centroidsCurr)
		avgScoreCent[centroid] += score.score
		avgMaxWin[centroid] += score.maxWinNoNorm
		maxWinByCentroid[centroid] += score.maxWinNoNorm

		if maxLossByCentroid[centroid] > score.maxLossNoNorm {
			maxLossByCentroid[centroid] = score.maxLossNoNorm
		}
		if _, ok := scoresByCentroid[centroid]; ok {
			scoresByCentroid[centroid] = append(scoresByCentroid[centroid], score)
		} else {
			scoresByCentroid[centroid] = []*ScoreCounter{score}
		}
	}

	// Try to reduce the max loss until contain the cMinLossRange points
	// inside the range
	for c, scores := range scoresByCentroid {
		usedScores := len(scores)
		valsToInclude := int(float64(len(scores)) * cMinLossRange)
		for usedScores != valsToInclude && usedScores > valsToInclude {
			maxLossByCentroid[c] *= 0.99
			usedScores = 0
			for _, score := range scores {
				if maxLossByCentroid[c] < score.maxLossNoNorm {
					usedScores++
				}
			}
		}

		log.Debug("Used Scores for MinLoss:", usedScores, "of:", len(scores), "Perc:", float64(usedScores)/float64(len(scores)), "Max Loss:", maxLossByCentroid[c])
	}

	scores := make([]float64, clusters)
	for c := 0; c < clusters; c++ {
		maxWinByCentroid[c] /= math.Abs(float64(len(scoresByCentroid[c])))
		avgScoreCent[c] /= float64(len(scoresByCentroid[c]))
		avgMaxWin[c] /= float64(len(scoresByCentroid[c]))
		scores[c] = avgMaxWin[c]
		log.Debug("Centroid:", c, "Items:", len(scoresByCentroid[c]), "Score", avgScoreCent[c], "Avg MaxWin:", avgMaxWin[c], "CentroidPos:", centroidsCurr[c])
	}

	sort.Float64s(scores)

	centroidsForAsk = []int{}
	for _, score := range scores[clusters-clustersToUse:] {
		for k, v := range avgMaxWin {
			if v == score {
				centroidsForAsk = append(centroidsForAsk, k)
			}
		}
	}

	return
}
Example #14
0
func (tr *TrainerCorrelations) studyCurrencies(TimeRangeToStudySecs int64) {
	log.Debug(len(tr.feeds["USD"]), tr.feeds["USD"][0])
	currsTrained := 0

	for curr, vals := range tr.feeds {
		/*if curr != "USD" {
			continue
		}*/
		go func(curr string, vals []*charont.CurrVal) {
			valsForScore := ByScore{}

			// Params to Buy
			minMaxWin := [2]float64{math.Inf(1), math.Inf(-1)}
			minMaxD := [2]float64{math.Inf(1), math.Inf(-1)}
			minMaxW := [2]float64{math.Inf(1), math.Inf(-1)}
			lastWindowFirstPosUsed := 0

			// Params to Sell
			valsForScoreSell := ByScore{}
			minMaxWinSell := [2]float64{math.Inf(1), math.Inf(-1)}
			minMaxDSell := [2]float64{math.Inf(1), math.Inf(-1)}
			minMaxWSell := [2]float64{math.Inf(1), math.Inf(-1)}

		pointToStudyLoop:
			for i, val := range vals {
				if i%1000 == 0 {
					log.Debug("Processed:", i, "points for currency:", curr)
				}
				charAskMin, charAskMax, charAskMean, charAskMode, noPossibleToStudy, firstWindowPos := tr.getPointCharacteristics(val, vals[lastWindowFirstPosUsed:i])
				//log.Debug("Study - Val:", val, "lastWindowFirstPosUsed:", lastWindowFirstPosUsed, "Vals:", len(vals[lastWindowFirstPosUsed:i]), charAskMin, charAskMax, charAskMean, charAskMode, noPossibleToStudy, firstWindowPos)
				if noPossibleToStudy {
					log.Debug("No study, no possible")
					continue pointToStudyLoop
				}
				lastWindowFirstPosUsed += firstWindowPos

				// Get the range to Buy
				found := int64(-1)
				w := -1.0
				maxWin := -1.0
				maxLoss := 10.0
			winningRangStudy:
				for _, futureVal := range vals[i+1:] {
					currWin := futureVal.Bid / val.Ask
					if currWin > maxWin {
						maxWin = currWin
					}
					if currWin < maxLoss {
						maxLoss = currWin
					}

					if found != -1 {
						if futureVal.Bid < val.Ask {
							w = float64(futureVal.Ts - found)
							break winningRangStudy
						}
					} else {
						if currWin > 1 {
							found = futureVal.Ts
						}
					}
				}

				// Get the range to Sell
				foundSell := int64(-1)
				wSell := -1.0
				maxWinSell := -1.0
				maxLossSell := 10.0
			winningRangStudySell:
				for _, futureVal := range vals[i+1:] {
					currWin := val.Bid / futureVal.Ask
					if currWin > maxWinSell {
						maxWinSell = currWin
					}
					if currWin < maxLossSell {
						maxLossSell = currWin
					}

					if foundSell != -1 {
						if val.Bid < futureVal.Ask {
							wSell = float64(futureVal.Ts - foundSell)
							break winningRangStudySell
						}
					} else {
						if currWin > 1 {
							foundSell = futureVal.Ts
						}
					}
				}

				// Calculate the scores for Buy
				maxWinFlo := maxWin - 1
				maxLossFlo := maxLoss - 1
				if found != -1 {
					if w != -1 {
						//log.Debug("New Range:", curr, w/1000000000, float64(found-val.Ts)/1000000000, maxWin)
						dFlo := float64(found - val.Ts)
						valsForScore = append(valsForScore, &ScoreCounter{
							val:           val,
							w:             w,
							d:             dFlo,
							maxWin:        maxWinFlo,
							maxWinNoNorm:  maxWinFlo,
							maxLossNoNorm: maxLossFlo,

							charAskMin:  charAskMin,
							charAskMax:  charAskMax,
							charAskMean: charAskMean,
							charAskMode: charAskMode,
						})

						if w < minMaxW[0] {
							minMaxW[0] = w
						}
						if w > minMaxW[1] {
							minMaxW[1] = w
						}
						if dFlo < minMaxD[0] {
							minMaxD[0] = dFlo
						}
						if dFlo > minMaxD[1] {
							minMaxD[1] = dFlo
						}
						if maxWinFlo < minMaxWin[0] {
							minMaxWin[0] = maxWinFlo
						}
						if maxWinFlo > minMaxWin[1] {
							minMaxWin[1] = maxWinFlo
						}
					}
				} else {
					// This is a bad point, we need to keep track of this also
					valsForScore = append(valsForScore, &ScoreCounter{
						val:          val,
						w:            0,
						d:            1,
						maxWin:       maxWinFlo,
						maxWinNoNorm: maxWinFlo,
						// We don't want to know the
						// max loss for bad points,
						// since it is going to be used
						// as boundary to sell
						maxLossNoNorm: 0,

						charAskMin:  charAskMin,
						charAskMax:  charAskMax,
						charAskMean: charAskMean,
						charAskMode: charAskMode,
					})
				}

				// Calculate the scores for Sell
				maxWinFloSell := maxWinSell - 1
				maxLossFloSell := maxLossSell - 1
				if foundSell != -1 {
					if wSell != -1 {
						//log.Debug("New Range:", curr, w/1000000000, float64(found-val.Ts)/1000000000, maxWin)
						dFloSell := float64(foundSell - val.Ts)
						valsForScoreSell = append(valsForScoreSell, &ScoreCounter{
							val:           val,
							w:             wSell,
							d:             dFloSell,
							maxWin:        maxWinFloSell,
							maxWinNoNorm:  maxWinFloSell,
							maxLossNoNorm: maxLossFloSell,

							charAskMin:  charAskMin,
							charAskMax:  charAskMax,
							charAskMean: charAskMean,
							charAskMode: charAskMode,
						})

						if wSell < minMaxWSell[0] {
							minMaxWSell[0] = wSell
						}
						if wSell > minMaxWSell[1] {
							minMaxWSell[1] = wSell
						}
						if dFloSell < minMaxDSell[0] {
							minMaxDSell[0] = dFloSell
						}
						if dFloSell > minMaxDSell[1] {
							minMaxDSell[1] = dFloSell
						}
						if maxWinFloSell < minMaxWinSell[0] {
							minMaxWinSell[0] = maxWinFloSell
						}
						if maxWinFloSell > minMaxWinSell[1] {
							minMaxWinSell[1] = maxWinFloSell
						}
					}
				} else {
					// This is a bad point, we need to keep track of this also
					valsForScore = append(valsForScore, &ScoreCounter{
						val:           val,
						w:             0,
						d:             1,
						maxWin:        maxWinFloSell,
						maxWinNoNorm:  maxWinFloSell,
						maxLossNoNorm: 0,

						charAskMin:  charAskMin,
						charAskMax:  charAskMax,
						charAskMean: charAskMean,
						charAskMode: charAskMode,
					})
				}
			}

			// Now normalise and prepare the scores to Buy
			for i := 0; i < len(valsForScore); i++ {
				if valsForScore[i].w != 0 {
					//log.Debug("Before Normalise:", valsForScore[i])
					valsForScore[i].maxWin = (valsForScore[i].maxWin - minMaxWin[0]) / (minMaxWin[1] - minMaxWin[0])
					valsForScore[i].d = (valsForScore[i].d - minMaxD[0]) / (minMaxD[1] - minMaxD[0])
					valsForScore[i].w = (valsForScore[i].w - minMaxW[0]) / (minMaxW[1] - minMaxW[0])
					if valsForScore[i].maxWin != 0 && valsForScore[i].d != 0 && valsForScore[i].w != 0 {
						valsForScore[i].score = (valsForScore[i].maxWin * valsForScore[i].w) / valsForScore[i].d
						//log.Debug("After Normalise:", valsForScore[i])
					} else {
						valsForScore[i].score = -1
					}
				}
			}

			// Now normalise and prepare the scores to Sell
			log.Debug("Normalise:", minMaxWinSell)
			log.Debug("ValsForSell:", valsForScoreSell[:10])
			for i := 0; i < len(valsForScoreSell); i++ {
				if valsForScoreSell[i].w != 0 {
					//log.Debug("Before Normalise:", valsForScoreSell[i])
					valsForScoreSell[i].maxWin = (valsForScoreSell[i].maxWin - minMaxWinSell[0]) / (minMaxWinSell[1] - minMaxWinSell[0])
					valsForScoreSell[i].d = (valsForScoreSell[i].d - minMaxDSell[0]) / (minMaxDSell[1] - minMaxDSell[0])
					valsForScoreSell[i].w = (valsForScoreSell[i].w - minMaxWSell[0]) / (minMaxWSell[1] - minMaxWSell[0])
					if valsForScoreSell[i].maxWin != 0 && valsForScoreSell[i].d != 0 && valsForScoreSell[i].w != 0 {
						valsForScoreSell[i].score = (valsForScoreSell[i].maxWin * valsForScoreSell[i].w) / valsForScoreSell[i].d
						//log.Debug("After Normalise:", valsForScoreSell[i])
					} else {
						valsForScoreSell[i].score = -1
					}
				}
			}

			tr.mutex.Lock()
			tr.centroidsCurr[curr], tr.maxWinByCentroid[curr], tr.maxLossByCentroid[curr], tr.centroidsForAsk[curr] = tr.getCentroids(valsForScore)
			log.Debug("Centroides to buy:", curr, tr.centroidsForAsk[curr], "Max loss:", tr.maxLossByCentroid[curr])
			tr.centroidsCurrSell[curr], tr.maxWinByCentroidSell[curr], tr.maxLossByCentroidSell[curr], tr.centroidsForSell[curr] = tr.getCentroids(valsForScoreSell)
			log.Debug("Centroides to sell:", curr, tr.centroidsForSell[curr], "Max loss:", tr.maxLossByCentroidSell[curr])
			tr.mutex.Unlock()

			currsTrained++
		}(curr, vals)
	}

	for currsTrained < len(tr.feeds) {
		time.Sleep(time.Second)
	}
}
Example #15
0
func TestRecommenderSaveLoad(t *testing.T) {
	maxClassifications := uint64(1000000)
	runtime.GOMAXPROCS(runtime.NumCPU())

	sh := NewShard("/testing", "test_collab_insertion", maxClassifications, 5, "eu-west-1")

	f, err := os.Open(TESTSET)
	if err != nil {
		log.Error("Can't read the the test set file:", TESTSET, "Error:", err)
		t.Fail()
	}
	r := bufio.NewReader(f)
	s, e := Readln(r)
	i := 0
	for e == nil && i < 100000 {
		s, e = Readln(r)
		recID, scores := parseLine(s)
		sh.AddRecord(recID, scores)
		i++
		if i%1000 == 0 {
			log.Debug("Lines processed:", i)
		}
	}

	time.Sleep(time.Second)

	if sh.totalClassif > maxClassifications {
		t.Error(
			"Problem with the garbage collection, the total number of stored stores are:",
			sh.totalClassif, "and the max defined boundary is:", maxClassifications)
	}

	if sh.status != STATUS_SARTING {
		t.Error("The expectede status was:", STATUS_SARTING, "but the actual one is:", sh.status)
	}

	log.Debug("Processing tree...")
	sh.RecalculateTree()

	if sh.status != STATUS_ACTIVE {
		t.Error("The expectede status was:", STATUS_ACTIVE, "but the actual one is:", sh.status)
	}

	s, e = Readln(r)
	recID, scores := parseLine(s)
	recomendationsBef := sh.CalcScores(recID, scores, 10)
	if len(recomendationsBef) != 10 {
		t.Error("The expected recommendations was 10, but:", len(recomendationsBef), "obtained.")
	}

	prevScores := sh.totalClassif
	sh.SaveBackup()

	sh = NewShard("/testing", "test_collab_insertion", maxClassifications, 5, "eu-west-1")
	sh.RecalculateTree()

	if sh.status != STATUS_NORECORDS {
		t.Error("The expectede status was:", STATUS_NORECORDS, "but the actual one is:", sh.status)
	}

	sh.LoadBackup()

	sh.RecalculateTree()

	if prevScores != sh.totalClassif {
		t.Error(
			"Before store a backup the number of records was:", prevScores,
			"but after load the backup is:", sh.totalClassif)
	}

	recomendationsAfter := sh.CalcScores(recID, scores, 10)
	if len(recomendationsAfter) != 10 {
		t.Error("The expected recommendations was 10, but:", len(recomendationsAfter), "obtained.")
	}

	log.Debug("Classifications:", sh.maxClassif)
}
Example #16
0
func (api *Oanda) ratesCollector() {
	var feeds map[string][]feedStruc

	api.mutex.Lock()
	api.currencyValues = make(map[string][]*CurrVal)
	currExange := make([]string, len(api.currencies))
	lasCurrPriceA := make(map[string]float64)
	lasCurrPriceB := make(map[string]float64)

	log.Debug("Curr:", api.currencies)
	for i, curr := range api.currencies {
		api.currencyValues[curr] = []*CurrVal{}
		currExange[i] = fmt.Sprintf("%s_%s", api.account.AccountCurrency, curr)
		lasCurrPriceA[curr] = 0
		lasCurrPriceB[curr] = 0
	}
	api.mutex.Unlock()

	feedsUrl := fmt.Sprintf(FEEDS_URL, api.endpoint) + strings.Join(currExange, "%2C")
	log.Info("Parsing currencies from the feeds URL:", feedsUrl)

	c := time.Tick((1000 / COLLECT_BY_SECOND) * time.Millisecond)
	for _ = range c {
		resp, err := api.doRequest("GET", feedsUrl, nil)
		if err != nil {
			log.Error("The feeds URL can't be parsed, Error:", err)
			continue
		}

		if err = json.Unmarshal(resp, &feeds); err != nil {
			log.Error("The feeds response body is not a JSON valid, Error:", err, "Resp:", string(resp))
			continue
		}

		// Ok, all fine, we are going to parse the feeds
		for _, feed := range feeds["prices"] {
			curr := feed.Instrument[len(api.account.AccountCurrency)+1:]
			if lasCurrPriceA[curr] != feed.Ask || lasCurrPriceB[curr] != feed.Bid {
				log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
				api.mutex.Lock()
				api.currencyValues[curr] = append(api.currencyValues[curr], &CurrVal{
					Ts:  time.Now().UnixNano(),
					Bid: feed.Bid,
					Ask: feed.Ask,
				})
				api.mutex.Unlock()

				if api.currLogsFile != nil {
					api.mutex.Lock()
					b, _ := json.Marshal(api.currencyValues[curr][len(api.currencyValues[curr])-1])
					_, err := api.currLogsFile.WriteString(fmt.Sprintf("%s:%s\n", curr, string(b)))
					api.mutex.Unlock()
					if err != nil {
						log.Error("Can't write into the currencies logs file, Error:", err)
					}
				}

				if listeners, ok := api.listeners[curr]; ok {
					for _, listener := range listeners {
						go listener(curr, time.Now().UnixNano())
					}
				}
				api.mutex.Lock()
				if len(api.currencyValues[curr]) > MAX_RATES_TO_STORE {
					api.currencyValues[curr] = api.currencyValues[curr][1:]
				}
				api.mutex.Unlock()
				lasCurrPriceA[curr] = feed.Ask
				lasCurrPriceB[curr] = feed.Bid
			}
		}
	}
}
Example #17
0
// ProcessNewTrees Creates a new set of numberOfTrees decission trees with a
// max deep of maxDeep. Specify on maxScore the max possible score for the
// elements
func ProcessNewTrees(records []map[uint64]uint8, maxDeep int, maxScore uint8, numberOfTrees int) (tr *Tree, avgScores map[uint64]float64) {
	avgScores = make(map[uint64]float64)
	elementsTotals := []elemTotals{}
	elemsPos := make(map[uint64]int)

	i := 0
	log.Debug("Records:", len(records))
	for _, record := range records {
		for k, v := range record {
			vUint64 := uint64(v)
			v2Uint64 := vUint64 * vUint64
			if p, ok := elemsPos[k]; ok {
				elementsTotals[p].sum += vUint64
				elementsTotals[p].sum2 += v2Uint64
				elementsTotals[p].err += uint64((maxScore - v) * (maxScore - v))
				elementsTotals[p].n++
			} else {
				elementsTotals = append(elementsTotals, elemTotals{
					elemID: k,
					sum:    vUint64,
					sum2:   v2Uint64,
					err:    uint64((maxScore - v) * (maxScore - v)),
					n:      1,
				})
				elemsPos[k] = i
				i++
			}
		}
	}

	for _, v := range elementsTotals {
		avgScores[v.elemID] = float64(v.sum) / float64(v.n)
	}

	tr = &Tree{
		maxDeep:   maxDeep,
		maxScore:  maxScore,
		totalRecs: len(records),
		tree:      make(map[uint64]*tNode),
		testMode:  false,
	}

	if len(elemsPos) < numberOfTrees {
		tr.numOfTrees = len(elemsPos)
	} else {
		tr.numOfTrees = numberOfTrees
	}

	i = 0
	studiedIds := make(map[uint64]bool)
	for {
		if i >= tr.numOfTrees {
			return
		}

		max := uint64(0)
		maxKey := uint64(0)
		for _, info := range elementsTotals {
			if max < info.n {
				if _, ok := studiedIds[info.elemID]; !ok {
					max = info.n
					maxKey = info.elemID
				}
			}
		}
		studiedIds[maxKey] = true
		pos := elemsPos[maxKey]
		log.Debug("----->>>> Building tree from:", maxKey, i, elementsTotals[pos].n)
		delete(elemsPos, maxKey)
		tr.tree[maxKey] = tr.getTreeNode(maxKey, elemsPos, elementsTotals, records, 1)
		elemsPos[maxKey] = pos

		i++
	}

	return
}
Example #18
0
func (api *Oanda) placeMarketOrder(inst string, units int, side string, price float64, realOps bool, ts int64) (order *Order, err error) {
	if !realOps {
		api.mutex.Lock()
		defer api.mutex.Unlock()

		api.openOrders[api.simulatedOrders] = &Order{
			Id:    api.simulatedOrders,
			Units: units,
			Open:  true,
			Type:  side,
			Real:  false,
			BuyTs: ts,
			Curr:  inst,
		}
		if side == "buy" {
			api.openOrders[api.simulatedOrders].Price = price
		} else {
			api.openOrders[api.simulatedOrders].CloseRate = price
		}
		api.simulatedOrders++
		return api.openOrders[api.simulatedOrders-1], nil
	}

	var orderInfo orderStruc
	var bound string

	if side == "sell" {
		bound = "lowerBound"
	} else {
		bound = "upperBound"
	}

	resp, err := api.doRequest("POST", fmt.Sprintf(PLACE_ORDER_URL, api.endpoint, api.account.AccountId),
		url.Values{
			"instrument": {inst},
			"units":      {fmt.Sprintf("%d", int(units))},
			"side":       {side},
			"type":       {"market"},
			bound:        {fmt.Sprintf("%f", price)},
		})

	if err != nil {
		log.Error("Problem trying to place a new order, Error:", err)
		return
	}

	err = json.Unmarshal(resp, &orderInfo)
	if err != nil || orderInfo.Info == nil {
		log.Error("The response from the server to place an order can't be parsed:", string(resp), "Error:", err)
		return
	}
	log.Debug("Values: instrument:", inst, "units", units, "side:", side, "type: market ID:", orderInfo, "\nOrder response:", string(resp))

	order = &Order{
		Id:    orderInfo.Info.Id,
		Price: orderInfo.Price,
		Units: units,
		Open:  true,
		Type:  side,
		BuyTs: ts,
		Curr:  inst,
		Real:  true,
	}
	if side == "buy" {
		order.Price = orderInfo.Price
	} else {
		order.CloseRate = orderInfo.Price
	}

	api.mutex.Lock()
	api.openOrders[order.Id] = order
	api.mutex.Unlock()

	return
}