Пример #1
0
func (rc *Recommender) LoadBackup() (success bool) {
	log.Info("Loading backup from S3:", rc.identifier)
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return false
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	jsonData, err := bucket.Get(rc.getS3Path())
	if err != nil {
		log.Info("Problem trying to get backup from S3:", err)
		return false
	}

	dataFromJSON := [][]uint64{}
	json.Unmarshal(rc.uncompress(jsonData), &dataFromJSON)

	log.Info("Data loaded from S3:", rc.identifier, "len:", len(dataFromJSON))
	recs := 0
	for _, record := range dataFromJSON {
		scores := make(map[uint64]uint8)
		for i := 1; i < len(record); i += 2 {
			scores[record[i]] = uint8(record[i+1])
		}
		recs += len(scores)
		rc.AddRecord(record[0], scores)
	}

	return true
}
Пример #2
0
Файл: pit.go Проект: postfix/pit
func main() {
	if len(os.Args) > 1 {
		cfg.Init("pit", os.Args[1])

		log.SetLogger(
			log.Levels[cfg.GetStr("logger", "level")],
			cfg.GetStr("logger", "log_file"),
			cfg.GetInt("logger", "max_log_size_mb"),
		)
	} else {
		cfg.Init("pit", "dev")
	}
	runtime.GOMAXPROCS(runtime.NumCPU())

	usersModel := users.GetModel(
		cfg.GetStr("aws", "prefix"),
		cfg.GetStr("aws", "region"))

	accountsManager := accountsmanager.Init(
		cfg.GetStr("rec-api", "base-url"),
		cfg.GetStr("mail", "addr"),
		cfg.GetStr("mail", "server"),
		cfg.GetInt("mail", "port"),
		usersModel)

	shardsManager := shardsmanager.Init(
		cfg.GetStr("aws", "prefix"),
		cfg.GetStr("aws", "region"),
		cfg.GetStr("aws", "s3-backups-path"),
		int(cfg.GetInt("rec-api", "port")),
		usersModel,
		cfg.GetStr("mail", "addr"))

	api.Init(
		shardsManager,
		accountsManager,
		cfg.GetStr("rec-api", "static"),
		int(cfg.GetInt("rec-api", "port")),
		int(cfg.GetInt("rec-api", "ssl-port")),
		cfg.GetStr("rec-api", "ssl-cert"),
		cfg.GetStr("rec-api", "ssl-key"))

	log.Info("System started...")
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)
	// Block until a signal is received.
	<-c

	log.Info("Stopping all the services")
	shardsManager.Stop()
}
Пример #3
0
func (mock *Mock) ratesCollector() {
	mock.mutex.Lock()
	mock.currencyValues = make(map[string][]*CurrVal)
	for _, curr := range mock.currencies {
		mock.mutexCurr[curr] = new(sync.Mutex)
	}
	mock.mutex.Unlock()

	scanner := bufio.NewScanner(mock.currLogsFile)
	scanner.Scan()
	log.Info("Parsing currencies from the mock file...")

	i := 0
	lastWinVal := mock.currentWin
	for {
		var feed CurrVal

		scanner.Scan()
		lineParts := strings.SplitN(scanner.Text(), ":", 2)
		curr := lineParts[0]
		if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
			log.Error("The feeds response body is not a JSON valid, Error:", err)
			continue
		}

		mock.mutexCurr[curr].Lock()
		//log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
		mock.currencyValues[curr] = append(mock.currencyValues[curr], &CurrVal{
			Ts:  feed.Ts,
			Bid: feed.Bid,
			Ask: feed.Ask,
		})

		if listeners, ok := mock.listeners[curr]; ok {
			for _, listener := range listeners {
				listener(curr, feed.Ts)
			}
		}
		if len(mock.currencyValues[curr]) > MAX_RATES_TO_STORE {
			mock.currencyValues[curr] = mock.currencyValues[curr][1:]
		}
		mock.mutexCurr[curr].Unlock()

		if lastWinVal != mock.currentWin {
			log.Info("CurrentWin:", i, mock.currentWin)
			lastWinVal = mock.currentWin
		}
		i++
	}
}
Пример #4
0
func (im *Model) updateInstances() {
	if rows, err := im.table.Scan(nil); err == nil {
		instances := []string{}
		for _, row := range rows {
			lastTs, _ := strconv.ParseInt(row["ts"].Value, 10, 64)
			if lastTs, _ = strconv.ParseInt(row["ts"].Value, 10, 64); lastTs+cTTL > time.Now().Unix() {
				instances = append(instances, row[cPrimKey].Value)
			} else if row[cPrimKey].Value != hostName {
				log.Info("Outdated instance detected, removing it, name:", row[cPrimKey].Value)
				attKey := &dynamodb.Key{
					HashKey:  row[cPrimKey].Value,
					RangeKey: "",
				}

				_, err = im.table.DeleteItem(attKey)
				if err != nil {
					log.Error("The instance:", row[cPrimKey].Value, "can't be removed, Error:", err)
				}
			}
		}

		sort.Sort(byName(instances))
		im.mutex.Lock()
		im.instancesAlive = instances
		im.mutex.Unlock()
	} else {
		log.Error("Problem trying to get the list of instances from Dynamo DB, Error:", err)
	}
}
Пример #5
0
func TestCollabInsertion(t *testing.T) {
	runtime.GOMAXPROCS(runtime.NumCPU())

	f, err := os.Open(TESTSET)
	if err != nil {
		log.Error("Can't read the the test set file:", TESTSET, "Error:", err)
		t.Fail()
	}
	r := bufio.NewReader(f)
	s, e := Readln(r)
	records := []map[uint64]uint8{}
	log.Info("Parsing test file...")
	for i := 0; e == nil && i < 10000; i++ {
		//for i := 0; e == nil && i < 480187; i++ {
		s, e = Readln(r)
		_, scores := parseLine(s)
		records = append(records, scores)
	}
	log.Info("Generating tree...")
	tr, _ := ProcessNewTrees(records, 50, MAXSCORE, 3)
	tr.setTestMode()
	log.Info("Tree generated...")

	quadError := 0.0
	comparedItems := 0
	for i := 0; e == nil && i < 1000; i++ {
		s, e = Readln(r)
		_, scores := parseLine(s)
		elements := tr.GetBestRecommendation(scores, 10)

		for _, elemID := range elements {
			if score, rated := scores[elemID]; rated {
				quadError += (1.0 - (float64(score) / MAXSCORE)) * (1.0 - (float64(score) / MAXSCORE))
				comparedItems++
			}
		}
	}

	// Estimate the Root-mean-square deviation, we will use 0.3 for this test because the training set and the number of trees is too low
	rmsd := math.Sqrt(quadError / float64(comparedItems))
	if rmsd > 0.3 {
		t.Error("The RMSD is bigger than the expected, obtained:", rmsd)
	}

	return
}
Пример #6
0
func InitOandaApi(endpoint string, authToken string, accountId int, currencies []string, currLogsFile string) (api *Oanda, err error) {
	var resp []byte

	api = &Oanda{
		endpoint:        endpoint,
		openOrders:      make(map[int64]*Order),
		authToken:       authToken,
		currencies:      currencies,
		listeners:       make(map[string][]func(currency string, ts int64)),
		currentWin:      0,
		simulatedOrders: 0,
	}

	if currLogsFile != "" {
		api.currLogsFile, err = os.Create(currLogsFile)
		if err != nil {
			log.Error("Currency logs file can't be open, Error:", err)
			return
		}
	}
	api.mutex.Lock()

	if accountId == -1 {
		var accInfo map[string]interface{}

		respHttp, err := http.PostForm(fmt.Sprintf(FAKE_GENERATE_ACCOUNT_URL, api.endpoint), nil)
		if err != nil {
			return nil, err
		}
		body, err := ioutil.ReadAll(respHttp.Body)
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(body, &accInfo)
		if err != nil {
			return nil, err
		}
		resp, err = api.doRequest("GET", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), nil)

		log.Info("New account generated:", int(accInfo["accountId"].(float64)))
	} else {
		resp, err = api.doRequest("GET", fmt.Sprintf("%s%d", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), accountId), nil)
	}

	if err != nil {
		return
	}

	err = json.Unmarshal(resp, &api.account)
	if err != nil {
		return
	}

	api.mutex.Unlock()

	return
}
Пример #7
0
func (rc *Recommender) DestroyS3Backup() (success bool) {
	log.Info("Destroying backup on S3:", rc.identifier)
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return false
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	if err := bucket.Del(rc.getS3Path()); err != nil {
		log.Info("Problem trying to remove backup from S3:", err)
		return false
	}

	return true
}
Пример #8
0
func (rc *Recommender) SaveBackup() {
	log.Info("Storing backup on S3:", rc.identifier)
	rc.mutex.Lock()
	records := make([][]uint64, len(rc.records))
	i := 0
	for recID, record := range rc.records {
		records[i] = make([]uint64, len(record.scores)*2+1)
		records[i][0] = recID
		elemPos := 1
		for k, v := range record.scores {
			records[i][elemPos] = k
			records[i][elemPos+1] = uint64(v)
			elemPos += 2
		}
		i++
	}
	rc.mutex.Unlock()

	jsonToUpload, err := json.Marshal(records)

	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	err = bucket.Put(
		rc.getS3Path(),
		rc.compress(jsonToUpload),
		"text/plain",
		s3.BucketOwnerFull,
		s3.Options{})
	if err != nil {
		log.Error("Problem trying to upload backup to S3 from:", rc.identifier, "Error:", err)
	}

	log.Info("New backup stored on S3, bucket:", S3BUCKET, "Path:", rc.getS3Path())
}
Пример #9
0
func (rc *Recommender) RecalculateTree() {
	// No new record was added, so is not necessary to calculate the tree
	// again
	if !rc.dirty {
		log.Info("Tree not dirty:", rc.identifier)
		return
	}
	log.Info("Recalculating tree for:", rc.identifier)
	if len(rc.records) < cMinRecordsToStart {
		rc.dirty = false
		rc.status = StatusNoRecords
		return
	}

	rc.cloning = true
	rc.mutex.Lock()
	records := make([]map[uint64]uint8, len(rc.records))
	i := 0
	for _, record := range rc.records {
		records[i] = record.scores
		i++
	}
	rc.mutex.Unlock()
	rc.cloning = false

	for recID, scores := range rc.cloningBuffer {
		rc.AddRecord(recID, scores)
	}
	rc.cloningBuffer = make(map[uint64]map[uint64]uint8)

	rc.recTree, rc.avgScoreElems = rectree.ProcessNewTrees(records, cRecTreeMaxDeep, rc.maxScore, cRecTreeNumOfTrees)

	rc.status = StatusActive
	log.Info("Tree recalculation finished:", rc.identifier)
	rc.dirty = false
}
Пример #10
0
func GetMock(feedsFile string, feedsBySecond int, currencies []string, httpPort int) (mock *Mock) {
	var err error

	mock = &Mock{
		orders:        0,
		currentWin:    0,
		feedsBySecond: feedsBySecond,
		mutexCurr:     make(map[string]*sync.Mutex),
		ordersByCurr:  make(map[string][]*Order),
		currencies:    currencies,
		listeners:     make(map[string][]func(currency string, ts int64)),
		openOrders:    make(map[int64]*Order),
	}

	for _, curr := range currencies {
		mock.ordersByCurr[curr] = []*Order{}
	}

	if feedsFile != "" {
		mock.currLogsFile, err = os.Open(feedsFile)
		if err != nil {
			log.Error("Currency logs file can't be open, Error:", err)
			return
		}
	}

	http.HandleFunc("/get_curr_values_orders", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Content-Type", "application/json")
		curr := r.FormValue("curr")
		info, _ := json.Marshal(&currOpsInfo{
			Prices: mock.currencyValues[curr],
			Orders: mock.ordersByCurr[curr],
		})
		w.Write(info)
	})
	go http.ListenAndServe(fmt.Sprintf(":%d", httpPort), nil)
	log.Info("Mock HTTP server listening on:", httpPort)

	return
}
Пример #11
0
func NewShard(s3Path string, identifier string, maxClassif uint64, maxScore uint8, s3Region string) (rc *Recommender) {
	log.Info("Starting shard:", identifier, "With max number of elements:", maxClassif)

	rc = &Recommender{
		identifier:    identifier,
		maxClassif:    maxClassif,
		totalClassif:  0,
		maxScore:      maxScore,
		records:       make(map[uint64]*score),
		status:        StatusStarting,
		s3Path:        s3Path,
		s3Region:      aws.Regions[s3Region],
		dirty:         true,
		running:       true,
		cloning:       false,
		cloningBuffer: make(map[uint64]map[uint64]uint8),
	}

	go rc.checkAndExpire()

	return
}
Пример #12
0
func (bi *Billing) SendNewBill(invTitle, targetEmail string, items []*InvItem) (billId string, err error) {
	inv := &Invoice{
		MerchantInfo: &InvMerchantInfo{
			Email:        bi.email,
			BusinessName: bi.addrAddr,
			Address: map[string]string{
				"line1":        bi.addrAddr,
				"city":         bi.addrCity,
				"state":        bi.addrState,
				"postal_code":  bi.addrZip,
				"country_code": bi.addrCountryCode,
			},
		},
		BillingInfo: []map[string]string{
			map[string]string{
				"email": targetEmail,
			},
		},
		Items: items,
		Note:  invTitle,
	}

	invStr, _ := json.Marshal(inv)

	if body, err := bi.callToApi("invoicing/invoices", "POST", string(invStr)); err == nil {
		responseInfo := make(map[string]interface{})
		err = json.Unmarshal(body, &responseInfo)
		if err != nil {
			log.Error("Problem trying to response from PayPal API, Error:", err)
		}

		billId = responseInfo["id"].(string)
		sendRes, err := bi.callToApi(fmt.Sprintf("invoicing/invoices/%s/send", billId), "POST", "")

		log.Info("Sent!!!:", fmt.Sprintf("invoicing/invoices/%s/send", billId), string(sendRes), err)
	}

	return
}
Пример #13
0
func (um *Model) initTable() {
	pKey := dynamodb.PrimaryKey{dynamodb.NewStringAttribute(cPrimKey, ""), nil}
	um.table = um.conn.NewTable(um.tableName, pKey)

	res, err := um.table.DescribeTable()
	if err != nil {
		log.Info("Creating a new table on DynamoDB:", um.tableName)
		td := dynamodb.TableDescriptionT{
			TableName: um.tableName,
			AttributeDefinitions: []dynamodb.AttributeDefinitionT{
				dynamodb.AttributeDefinitionT{cPrimKey, "S"},
			},
			KeySchema: []dynamodb.KeySchemaT{
				dynamodb.KeySchemaT{cPrimKey, "HASH"},
			},
			ProvisionedThroughput: dynamodb.ProvisionedThroughputT{
				ReadCapacityUnits:  cDefaultWRCapacity,
				WriteCapacityUnits: cDefaultWRCapacity,
			},
		}

		if _, err := um.conn.CreateTable(td); err != nil {
			log.Error("Error trying to create a table on Dynamo DB, table:", um.tableName, "Error:", err)
		}
		if res, err = um.table.DescribeTable(); err != nil {
			log.Error("Error trying to describe a table on Dynamo DB, table:", um.tableName, "Error:", err)
		}
	}
	for "ACTIVE" != res.TableStatus {
		if res, err = um.table.DescribeTable(); err != nil {
			log.Error("Can't describe Dynamo DB instances table, Error:", err)
		}
		log.Debug("Waiting for active table, current status:", res.TableStatus)
		time.Sleep(time.Second)
	}
}
Пример #14
0
func (bi *Billing) callToApi(uri string, method string, reqBody string) (body []byte, err error) {
	if time.Now().Unix() > bi.tokenExpireTs {
		bi.getToken()
	}

	client := &http.Client{}

	url := bi.baseUri + uri
	log.Info(reqBody)
	req, err := http.NewRequest(method, url, strings.NewReader(reqBody))
	req.Header.Add("Content-Type", `application/json`)
	req.Header.Add("Authorization", "Bearer "+bi.token)
	resp, err := client.Do(req)
	if err != nil {
		log.Error("Problem trying to do a request to the PayPal API, Url:", url, "Error:", err)
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Error("Problem trying to parse the response form the PayPal API, Error:", err)
	}

	return
}
Пример #15
0
func (api *Oanda) ratesCollector() {
	var feeds map[string][]feedStruc

	api.mutex.Lock()
	api.currencyValues = make(map[string][]*CurrVal)
	currExange := make([]string, len(api.currencies))
	lasCurrPriceA := make(map[string]float64)
	lasCurrPriceB := make(map[string]float64)

	log.Debug("Curr:", api.currencies)
	for i, curr := range api.currencies {
		api.currencyValues[curr] = []*CurrVal{}
		currExange[i] = fmt.Sprintf("%s_%s", api.account.AccountCurrency, curr)
		lasCurrPriceA[curr] = 0
		lasCurrPriceB[curr] = 0
	}
	api.mutex.Unlock()

	feedsUrl := fmt.Sprintf(FEEDS_URL, api.endpoint) + strings.Join(currExange, "%2C")
	log.Info("Parsing currencies from the feeds URL:", feedsUrl)

	c := time.Tick((1000 / COLLECT_BY_SECOND) * time.Millisecond)
	for _ = range c {
		resp, err := api.doRequest("GET", feedsUrl, nil)
		if err != nil {
			log.Error("The feeds URL can't be parsed, Error:", err)
			continue
		}

		if err = json.Unmarshal(resp, &feeds); err != nil {
			log.Error("The feeds response body is not a JSON valid, Error:", err, "Resp:", string(resp))
			continue
		}

		// Ok, all fine, we are going to parse the feeds
		for _, feed := range feeds["prices"] {
			curr := feed.Instrument[len(api.account.AccountCurrency)+1:]
			if lasCurrPriceA[curr] != feed.Ask || lasCurrPriceB[curr] != feed.Bid {
				log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
				api.mutex.Lock()
				api.currencyValues[curr] = append(api.currencyValues[curr], &CurrVal{
					Ts:  time.Now().UnixNano(),
					Bid: feed.Bid,
					Ask: feed.Ask,
				})
				api.mutex.Unlock()

				if api.currLogsFile != nil {
					api.mutex.Lock()
					b, _ := json.Marshal(api.currencyValues[curr][len(api.currencyValues[curr])-1])
					_, err := api.currLogsFile.WriteString(fmt.Sprintf("%s:%s\n", curr, string(b)))
					api.mutex.Unlock()
					if err != nil {
						log.Error("Can't write into the currencies logs file, Error:", err)
					}
				}

				if listeners, ok := api.listeners[curr]; ok {
					for _, listener := range listeners {
						go listener(curr, time.Now().UnixNano())
					}
				}
				api.mutex.Lock()
				if len(api.currencyValues[curr]) > MAX_RATES_TO_STORE {
					api.currencyValues[curr] = api.currencyValues[curr][1:]
				}
				api.mutex.Unlock()
				lasCurrPriceA[curr] = feed.Ask
				lasCurrPriceB[curr] = feed.Bid
			}
		}
	}
}