Пример #1
0
func Post(url string, data []byte, appHeader string) {
	url = fmt.Sprintf("%s?callback=http://%s:%d/callback", url, conf.Local.Hostname, conf.Local.Port)
	req, err := http.NewRequest("POST", url, bytes.NewReader(data))
	if err != nil {
		pialog.Error(err)
		return
	}
	req.Header.Set("Content-Type", "avro/binary")
	req.Header.Set("Application", appHeader)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		pialog.Error(err)
		return
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		pialog.Error(err)
		return
	}
	pialog.Info("Server replied:", string(body), resp.Status)
}
Пример #2
0
func PushScores(scores []interface{}) {
	SetLocale.SetLocale(SetLocale.LC_ALL, "de_DE")
	dsn := getDSN()
	db, err := sql.Open("oci8", dsn)
	check(err)

	defer db.Close()
	tx, err := db.Begin()
	check(err)
	stmt := PrepareStatement(tx, conf.Database.QueryOut)
	/*for _, score := range scores.Scores {
		ExecuteInsert(stmt, score)
	}*/
	fails_count := 0
	for _, score := range scores {
		if s, ok := score.(map[string]interface{}); ok {
			err = ExecuteScoreInsert(stmt, s)
			if err != nil {
				fails_count += 1
			}
		} else {
			pialog.Error("Not a valid score:\n", score)
		}
	}
	tx.Commit()
	pialog.Info("Score failures count:", fails_count)
}
Пример #3
0
func exportData(data []byte) {
	pialog.Info("Callback received with payload size:", len(data), "-> Will export data to Oracle DB.")
	defer timeTrack(time.Now(), "Data export")

	var j map[string]interface{}
	err := json.Unmarshal(data, &j)
	check(err)

	if scores, ok := j["Score"].([]interface{}); ok {
		PushScores(scores)
	} else {
		pialog.Error("Not valid scores array")
	}

	if var_imps, ok := j["var_imp"].([]interface{}); ok {
		PushVarIMP(var_imps)
	} else {
		pialog.Error("Not valid var_imps array")
	}
}
Пример #4
0
func GetData() {
	SetLocale.SetLocale(SetLocale.LC_ALL, "de_DE")
	dsn := getDSN()
	db, err := sql.Open("oci8", dsn)
	check(err)
	defer db.Close()
	if err != nil {
		pialog.Error(err)
		return
	}
	SelectDBData(db, conf.Database.Query)
}
Пример #5
0
func main() {
	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))

	confpath := flag.String("config-dir", fmt.Sprintf("%s/conf", dir), "Config file path")
	var v bool
	flag.BoolVar(&v, "v", false, "Asks for version")
	flag.Parse()

	if v {
		fmt.Println("Pia-oracle version:", version)
		return
	}

	config.Path = *confpath

	pialog.InitializeLogging()
	pialog.Info("Starting pia-oracle version:", version)
	pialog.Info("Loading config from:", *confpath)

	var err error
	conf, err = config.GetConfig(fmt.Sprintf("%s/pia-oracle.toml", *confpath))
	if err != nil {
		pialog.Error(err)
		pialog.Info("Exiting")
		return
	}

	pialog.Info("Server started:", fmt.Sprintf("%s:%d", conf.Local.Listen, conf.Local.Port))

	http.HandleFunc("/trigger", trigger)
	http.HandleFunc("/callback", callback)
	http.HandleFunc("/predict", predict)
	err = http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Local.Listen, conf.Local.Port), nil)
	if err != nil {
		pialog.Error(err)
	}
	pialog.Info("Exiting")
}
Пример #6
0
func PushVarIMP(var_imps []interface{}) {
	SetLocale.SetLocale(SetLocale.LC_ALL, "de_DE")
	dsn := getDSN()
	db, err := sql.Open("oci8", dsn)
	check(err)
	defer db.Close()

	tx, err := db.Begin()
	check(err)
	stmt := PrepareStatement(tx, conf.Database.QueryOutImp)
	fails_count := 0
	for _, var_imp := range var_imps {
		if s, ok := var_imp.(map[string]interface{}); ok {
			err = ExecuteImpInsert(stmt, s)
			if err != nil {
				fails_count += 1
			}
		} else {
			pialog.Error("Not a valid var_imp:\n", var_imp)
		}
	}
	tx.Commit()
	pialog.Info("IMP failures count:", fails_count)
}
Пример #7
0
func SelectDBData(db *sql.DB, query string) {
	defer timeTrack(time.Now(), "SelectDBData")

	rows, err := db.Query(query)
	check(err)
	defer rows.Close()

	columns, err := rows.Columns()
	check(err)

	values := make([]interface{}, len(columns))
	scanArgs := make([]interface{}, len(values))
	for i := range values {
		scanArgs[i] = &values[i]
	}

	c := 0
	var claims []interface{}
	outerSchema, innerSchema, codec := LoadAvroSchema(conf.Avro.OuterSchema, conf.Avro.InnerSchema)

	for rows.Next() {
		c += 1
		err = rows.Scan(scanArgs...)
		check(err)

		m := make(map[string]interface{})
		claim, err := goavro.NewRecord(innerSchema)
		check(err)
		for i, colName := range columns {
			if val, ok := values[i].(time.Time); ok {
				claim.Set(colName, val.Unix())
				m[colName] = val.Unix()
			} else {
				claim.Set(colName, values[i])
				m[colName] = values[i]
			}
		}

		claims = append(claims, claim)

		if len(claims) == 3000 {

			claims_avro, err := goavro.NewRecord(outerSchema)
			check(err)
			claims_avro.Set("claims", claims)
			buf := new(bytes.Buffer)
			err = codec.Encode(buf, claims_avro)
			go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)
			claims = make([]interface{}, 0)
		}
	}

	if rows.Err() != nil {
		pialog.Error(rows.Err())
	}

	claims_avro, err := goavro.NewRecord(outerSchema)
	check(err)
	claims_avro.Set("claims", claims)

	buf := new(bytes.Buffer)
	err = codec.Encode(buf, claims_avro)
	check(err)

	go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)

	pialog.Info("Total input records:", c)
}