func (rc *Recommender) LoadBackup() (success bool) {
log.Info("Loading backup from S3:", rc.identifier)
auth, err := aws.EnvAuth()
if err != nil {
log.Error("Problem trying to connect with AWS:", err)
return false
}
s := s3.New(auth, rc.s3Region)
bucket := s.Bucket(S3BUCKET)
jsonData, err := bucket.Get(rc.getS3Path())
if err != nil {
log.Info("Problem trying to get backup from S3:", err)
return false
}
dataFromJSON := [][]uint64{}
json.Unmarshal(rc.uncompress(jsonData), &dataFromJSON)
log.Info("Data loaded from S3:", rc.identifier, "len:", len(dataFromJSON))
recs := 0
for _, record := range dataFromJSON {
scores := make(map[uint64]uint8)
for i := 1; i < len(record); i += 2 {
scores[record[i]] = uint8(record[i+1])
}
recs += len(scores)
rc.AddRecord(record[0], scores)
}
return true
}
func main() {
if len(os.Args) > 1 {
cfg.Init("pit", os.Args[1])
log.SetLogger(
log.Levels[cfg.GetStr("logger", "level")],
cfg.GetStr("logger", "log_file"),
cfg.GetInt("logger", "max_log_size_mb"),
)
} else {
cfg.Init("pit", "dev")
}
runtime.GOMAXPROCS(runtime.NumCPU())
usersModel := users.GetModel(
cfg.GetStr("aws", "prefix"),
cfg.GetStr("aws", "region"))
accountsManager := accountsmanager.Init(
cfg.GetStr("rec-api", "base-url"),
cfg.GetStr("mail", "addr"),
cfg.GetStr("mail", "server"),
cfg.GetInt("mail", "port"),
usersModel)
shardsManager := shardsmanager.Init(
cfg.GetStr("aws", "prefix"),
cfg.GetStr("aws", "region"),
cfg.GetStr("aws", "s3-backups-path"),
int(cfg.GetInt("rec-api", "port")),
usersModel,
cfg.GetStr("mail", "addr"))
api.Init(
shardsManager,
accountsManager,
cfg.GetStr("rec-api", "static"),
int(cfg.GetInt("rec-api", "port")),
int(cfg.GetInt("rec-api", "ssl-port")),
cfg.GetStr("rec-api", "ssl-cert"),
cfg.GetStr("rec-api", "ssl-key"))
log.Info("System started...")
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)
// Block until a signal is received.
<-c
log.Info("Stopping all the services")
shardsManager.Stop()
}
func (mock *Mock) ratesCollector() {
mock.mutex.Lock()
mock.currencyValues = make(map[string][]*CurrVal)
for _, curr := range mock.currencies {
mock.mutexCurr[curr] = new(sync.Mutex)
}
mock.mutex.Unlock()
scanner := bufio.NewScanner(mock.currLogsFile)
scanner.Scan()
log.Info("Parsing currencies from the mock file...")
i := 0
lastWinVal := mock.currentWin
for {
var feed CurrVal
scanner.Scan()
lineParts := strings.SplitN(scanner.Text(), ":", 2)
curr := lineParts[0]
if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
log.Error("The feeds response body is not a JSON valid, Error:", err)
continue
}
mock.mutexCurr[curr].Lock()
//log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
mock.currencyValues[curr] = append(mock.currencyValues[curr], &CurrVal{
Ts: feed.Ts,
Bid: feed.Bid,
Ask: feed.Ask,
})
if listeners, ok := mock.listeners[curr]; ok {
for _, listener := range listeners {
listener(curr, feed.Ts)
}
}
if len(mock.currencyValues[curr]) > MAX_RATES_TO_STORE {
mock.currencyValues[curr] = mock.currencyValues[curr][1:]
}
mock.mutexCurr[curr].Unlock()
if lastWinVal != mock.currentWin {
log.Info("CurrentWin:", i, mock.currentWin)
lastWinVal = mock.currentWin
}
i++
}
}
func (im *Model) updateInstances() {
if rows, err := im.table.Scan(nil); err == nil {
instances := []string{}
for _, row := range rows {
lastTs, _ := strconv.ParseInt(row["ts"].Value, 10, 64)
if lastTs, _ = strconv.ParseInt(row["ts"].Value, 10, 64); lastTs+cTTL > time.Now().Unix() {
instances = append(instances, row[cPrimKey].Value)
} else if row[cPrimKey].Value != hostName {
log.Info("Outdated instance detected, removing it, name:", row[cPrimKey].Value)
attKey := &dynamodb.Key{
HashKey: row[cPrimKey].Value,
RangeKey: "",
}
_, err = im.table.DeleteItem(attKey)
if err != nil {
log.Error("The instance:", row[cPrimKey].Value, "can't be removed, Error:", err)
}
}
}
sort.Sort(byName(instances))
im.mutex.Lock()
im.instancesAlive = instances
im.mutex.Unlock()
} else {
log.Error("Problem trying to get the list of instances from Dynamo DB, Error:", err)
}
}
func TestCollabInsertion(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
f, err := os.Open(TESTSET)
if err != nil {
log.Error("Can't read the the test set file:", TESTSET, "Error:", err)
t.Fail()
}
r := bufio.NewReader(f)
s, e := Readln(r)
records := []map[uint64]uint8{}
log.Info("Parsing test file...")
for i := 0; e == nil && i < 10000; i++ {
//for i := 0; e == nil && i < 480187; i++ {
s, e = Readln(r)
_, scores := parseLine(s)
records = append(records, scores)
}
log.Info("Generating tree...")
tr, _ := ProcessNewTrees(records, 50, MAXSCORE, 3)
tr.setTestMode()
log.Info("Tree generated...")
quadError := 0.0
comparedItems := 0
for i := 0; e == nil && i < 1000; i++ {
s, e = Readln(r)
_, scores := parseLine(s)
elements := tr.GetBestRecommendation(scores, 10)
for _, elemID := range elements {
if score, rated := scores[elemID]; rated {
quadError += (1.0 - (float64(score) / MAXSCORE)) * (1.0 - (float64(score) / MAXSCORE))
comparedItems++
}
}
}
// Estimate the Root-mean-square deviation, we will use 0.3 for this test because the training set and the number of trees is too low
rmsd := math.Sqrt(quadError / float64(comparedItems))
if rmsd > 0.3 {
t.Error("The RMSD is bigger than the expected, obtained:", rmsd)
}
return
}
func InitOandaApi(endpoint string, authToken string, accountId int, currencies []string, currLogsFile string) (api *Oanda, err error) {
var resp []byte
api = &Oanda{
endpoint: endpoint,
openOrders: make(map[int64]*Order),
authToken: authToken,
currencies: currencies,
listeners: make(map[string][]func(currency string, ts int64)),
currentWin: 0,
simulatedOrders: 0,
}
if currLogsFile != "" {
api.currLogsFile, err = os.Create(currLogsFile)
if err != nil {
log.Error("Currency logs file can't be open, Error:", err)
return
}
}
api.mutex.Lock()
if accountId == -1 {
var accInfo map[string]interface{}
respHttp, err := http.PostForm(fmt.Sprintf(FAKE_GENERATE_ACCOUNT_URL, api.endpoint), nil)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(respHttp.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(body, &accInfo)
if err != nil {
return nil, err
}
resp, err = api.doRequest("GET", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), nil)
log.Info("New account generated:", int(accInfo["accountId"].(float64)))
} else {
resp, err = api.doRequest("GET", fmt.Sprintf("%s%d", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), accountId), nil)
}
if err != nil {
return
}
err = json.Unmarshal(resp, &api.account)
if err != nil {
return
}
api.mutex.Unlock()
return
}
func (rc *Recommender) DestroyS3Backup() (success bool) {
log.Info("Destroying backup on S3:", rc.identifier)
auth, err := aws.EnvAuth()
if err != nil {
log.Error("Problem trying to connect with AWS:", err)
return false
}
s := s3.New(auth, rc.s3Region)
bucket := s.Bucket(S3BUCKET)
if err := bucket.Del(rc.getS3Path()); err != nil {
log.Info("Problem trying to remove backup from S3:", err)
return false
}
return true
}
func (rc *Recommender) SaveBackup() {
log.Info("Storing backup on S3:", rc.identifier)
rc.mutex.Lock()
records := make([][]uint64, len(rc.records))
i := 0
for recID, record := range rc.records {
records[i] = make([]uint64, len(record.scores)*2+1)
records[i][0] = recID
elemPos := 1
for k, v := range record.scores {
records[i][elemPos] = k
records[i][elemPos+1] = uint64(v)
elemPos += 2
}
i++
}
rc.mutex.Unlock()
jsonToUpload, err := json.Marshal(records)
auth, err := aws.EnvAuth()
if err != nil {
log.Error("Problem trying to connect with AWS:", err)
return
}
s := s3.New(auth, rc.s3Region)
bucket := s.Bucket(S3BUCKET)
err = bucket.Put(
rc.getS3Path(),
rc.compress(jsonToUpload),
"text/plain",
s3.BucketOwnerFull,
s3.Options{})
if err != nil {
log.Error("Problem trying to upload backup to S3 from:", rc.identifier, "Error:", err)
}
log.Info("New backup stored on S3, bucket:", S3BUCKET, "Path:", rc.getS3Path())
}
func (rc *Recommender) RecalculateTree() {
// No new record was added, so is not necessary to calculate the tree
// again
if !rc.dirty {
log.Info("Tree not dirty:", rc.identifier)
return
}
log.Info("Recalculating tree for:", rc.identifier)
if len(rc.records) < cMinRecordsToStart {
rc.dirty = false
rc.status = StatusNoRecords
return
}
rc.cloning = true
rc.mutex.Lock()
records := make([]map[uint64]uint8, len(rc.records))
i := 0
for _, record := range rc.records {
records[i] = record.scores
i++
}
rc.mutex.Unlock()
rc.cloning = false
for recID, scores := range rc.cloningBuffer {
rc.AddRecord(recID, scores)
}
rc.cloningBuffer = make(map[uint64]map[uint64]uint8)
rc.recTree, rc.avgScoreElems = rectree.ProcessNewTrees(records, cRecTreeMaxDeep, rc.maxScore, cRecTreeNumOfTrees)
rc.status = StatusActive
log.Info("Tree recalculation finished:", rc.identifier)
rc.dirty = false
}
func GetMock(feedsFile string, feedsBySecond int, currencies []string, httpPort int) (mock *Mock) {
var err error
mock = &Mock{
orders: 0,
currentWin: 0,
feedsBySecond: feedsBySecond,
mutexCurr: make(map[string]*sync.Mutex),
ordersByCurr: make(map[string][]*Order),
currencies: currencies,
listeners: make(map[string][]func(currency string, ts int64)),
openOrders: make(map[int64]*Order),
}
for _, curr := range currencies {
mock.ordersByCurr[curr] = []*Order{}
}
if feedsFile != "" {
mock.currLogsFile, err = os.Open(feedsFile)
if err != nil {
log.Error("Currency logs file can't be open, Error:", err)
return
}
}
http.HandleFunc("/get_curr_values_orders", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Content-Type", "application/json")
curr := r.FormValue("curr")
info, _ := json.Marshal(&currOpsInfo{
Prices: mock.currencyValues[curr],
Orders: mock.ordersByCurr[curr],
})
w.Write(info)
})
go http.ListenAndServe(fmt.Sprintf(":%d", httpPort), nil)
log.Info("Mock HTTP server listening on:", httpPort)
return
}
func NewShard(s3Path string, identifier string, maxClassif uint64, maxScore uint8, s3Region string) (rc *Recommender) {
log.Info("Starting shard:", identifier, "With max number of elements:", maxClassif)
rc = &Recommender{
identifier: identifier,
maxClassif: maxClassif,
totalClassif: 0,
maxScore: maxScore,
records: make(map[uint64]*score),
status: StatusStarting,
s3Path: s3Path,
s3Region: aws.Regions[s3Region],
dirty: true,
running: true,
cloning: false,
cloningBuffer: make(map[uint64]map[uint64]uint8),
}
go rc.checkAndExpire()
return
}
func (bi *Billing) SendNewBill(invTitle, targetEmail string, items []*InvItem) (billId string, err error) {
inv := &Invoice{
MerchantInfo: &InvMerchantInfo{
Email: bi.email,
BusinessName: bi.addrAddr,
Address: map[string]string{
"line1": bi.addrAddr,
"city": bi.addrCity,
"state": bi.addrState,
"postal_code": bi.addrZip,
"country_code": bi.addrCountryCode,
},
},
BillingInfo: []map[string]string{
map[string]string{
"email": targetEmail,
},
},
Items: items,
Note: invTitle,
}
invStr, _ := json.Marshal(inv)
if body, err := bi.callToApi("invoicing/invoices", "POST", string(invStr)); err == nil {
responseInfo := make(map[string]interface{})
err = json.Unmarshal(body, &responseInfo)
if err != nil {
log.Error("Problem trying to response from PayPal API, Error:", err)
}
billId = responseInfo["id"].(string)
sendRes, err := bi.callToApi(fmt.Sprintf("invoicing/invoices/%s/send", billId), "POST", "")
log.Info("Sent!!!:", fmt.Sprintf("invoicing/invoices/%s/send", billId), string(sendRes), err)
}
return
}
func (um *Model) initTable() {
pKey := dynamodb.PrimaryKey{dynamodb.NewStringAttribute(cPrimKey, ""), nil}
um.table = um.conn.NewTable(um.tableName, pKey)
res, err := um.table.DescribeTable()
if err != nil {
log.Info("Creating a new table on DynamoDB:", um.tableName)
td := dynamodb.TableDescriptionT{
TableName: um.tableName,
AttributeDefinitions: []dynamodb.AttributeDefinitionT{
dynamodb.AttributeDefinitionT{cPrimKey, "S"},
},
KeySchema: []dynamodb.KeySchemaT{
dynamodb.KeySchemaT{cPrimKey, "HASH"},
},
ProvisionedThroughput: dynamodb.ProvisionedThroughputT{
ReadCapacityUnits: cDefaultWRCapacity,
WriteCapacityUnits: cDefaultWRCapacity,
},
}
if _, err := um.conn.CreateTable(td); err != nil {
log.Error("Error trying to create a table on Dynamo DB, table:", um.tableName, "Error:", err)
}
if res, err = um.table.DescribeTable(); err != nil {
log.Error("Error trying to describe a table on Dynamo DB, table:", um.tableName, "Error:", err)
}
}
for "ACTIVE" != res.TableStatus {
if res, err = um.table.DescribeTable(); err != nil {
log.Error("Can't describe Dynamo DB instances table, Error:", err)
}
log.Debug("Waiting for active table, current status:", res.TableStatus)
time.Sleep(time.Second)
}
}
func (bi *Billing) callToApi(uri string, method string, reqBody string) (body []byte, err error) {
if time.Now().Unix() > bi.tokenExpireTs {
bi.getToken()
}
client := &http.Client{}
url := bi.baseUri + uri
log.Info(reqBody)
req, err := http.NewRequest(method, url, strings.NewReader(reqBody))
req.Header.Add("Content-Type", `application/json`)
req.Header.Add("Authorization", "Bearer "+bi.token)
resp, err := client.Do(req)
if err != nil {
log.Error("Problem trying to do a request to the PayPal API, Url:", url, "Error:", err)
}
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
log.Error("Problem trying to parse the response form the PayPal API, Error:", err)
}
return
}
func (api *Oanda) ratesCollector() {
var feeds map[string][]feedStruc
api.mutex.Lock()
api.currencyValues = make(map[string][]*CurrVal)
currExange := make([]string, len(api.currencies))
lasCurrPriceA := make(map[string]float64)
lasCurrPriceB := make(map[string]float64)
log.Debug("Curr:", api.currencies)
for i, curr := range api.currencies {
api.currencyValues[curr] = []*CurrVal{}
currExange[i] = fmt.Sprintf("%s_%s", api.account.AccountCurrency, curr)
lasCurrPriceA[curr] = 0
lasCurrPriceB[curr] = 0
}
api.mutex.Unlock()
feedsUrl := fmt.Sprintf(FEEDS_URL, api.endpoint) + strings.Join(currExange, "%2C")
log.Info("Parsing currencies from the feeds URL:", feedsUrl)
c := time.Tick((1000 / COLLECT_BY_SECOND) * time.Millisecond)
for _ = range c {
resp, err := api.doRequest("GET", feedsUrl, nil)
if err != nil {
log.Error("The feeds URL can't be parsed, Error:", err)
continue
}
if err = json.Unmarshal(resp, &feeds); err != nil {
log.Error("The feeds response body is not a JSON valid, Error:", err, "Resp:", string(resp))
continue
}
// Ok, all fine, we are going to parse the feeds
for _, feed := range feeds["prices"] {
curr := feed.Instrument[len(api.account.AccountCurrency)+1:]
if lasCurrPriceA[curr] != feed.Ask || lasCurrPriceB[curr] != feed.Bid {
log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
api.mutex.Lock()
api.currencyValues[curr] = append(api.currencyValues[curr], &CurrVal{
Ts: time.Now().UnixNano(),
Bid: feed.Bid,
Ask: feed.Ask,
})
api.mutex.Unlock()
if api.currLogsFile != nil {
api.mutex.Lock()
b, _ := json.Marshal(api.currencyValues[curr][len(api.currencyValues[curr])-1])
_, err := api.currLogsFile.WriteString(fmt.Sprintf("%s:%s\n", curr, string(b)))
api.mutex.Unlock()
if err != nil {
log.Error("Can't write into the currencies logs file, Error:", err)
}
}
if listeners, ok := api.listeners[curr]; ok {
for _, listener := range listeners {
go listener(curr, time.Now().UnixNano())
}
}
api.mutex.Lock()
if len(api.currencyValues[curr]) > MAX_RATES_TO_STORE {
api.currencyValues[curr] = api.currencyValues[curr][1:]
}
api.mutex.Unlock()
lasCurrPriceA[curr] = feed.Ask
lasCurrPriceB[curr] = feed.Bid
}
}
}
}