func Post(url string, data []byte, appHeader string) {
url = fmt.Sprintf("%s?callback=http://%s:%d/callback", url, conf.Local.Hostname, conf.Local.Port)
req, err := http.NewRequest("POST", url, bytes.NewReader(data))
if err != nil {
pialog.Error(err)
return
}
req.Header.Set("Content-Type", "avro/binary")
req.Header.Set("Application", appHeader)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
pialog.Error(err)
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
pialog.Error(err)
return
}
pialog.Info("Server replied:", string(body), resp.Status)
}
func PushScores(scores []interface{}) {
SetLocale.SetLocale(SetLocale.LC_ALL, "de_DE")
dsn := getDSN()
db, err := sql.Open("oci8", dsn)
check(err)
defer db.Close()
tx, err := db.Begin()
check(err)
stmt := PrepareStatement(tx, conf.Database.QueryOut)
/*for _, score := range scores.Scores {
ExecuteInsert(stmt, score)
}*/
fails_count := 0
for _, score := range scores {
if s, ok := score.(map[string]interface{}); ok {
err = ExecuteScoreInsert(stmt, s)
if err != nil {
fails_count += 1
}
} else {
pialog.Error("Not a valid score:\n", score)
}
}
tx.Commit()
pialog.Info("Score failures count:", fails_count)
}
func exportData(data []byte) {
pialog.Info("Callback received with payload size:", len(data), "-> Will export data to Oracle DB.")
defer timeTrack(time.Now(), "Data export")
var j map[string]interface{}
err := json.Unmarshal(data, &j)
check(err)
if scores, ok := j["Score"].([]interface{}); ok {
PushScores(scores)
} else {
pialog.Error("Not valid scores array")
}
if var_imps, ok := j["var_imp"].([]interface{}); ok {
PushVarIMP(var_imps)
} else {
pialog.Error("Not valid var_imps array")
}
}
func GetData() {
SetLocale.SetLocale(SetLocale.LC_ALL, "de_DE")
dsn := getDSN()
db, err := sql.Open("oci8", dsn)
check(err)
defer db.Close()
if err != nil {
pialog.Error(err)
return
}
SelectDBData(db, conf.Database.Query)
}
func main() {
dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
confpath := flag.String("config-dir", fmt.Sprintf("%s/conf", dir), "Config file path")
var v bool
flag.BoolVar(&v, "v", false, "Asks for version")
flag.Parse()
if v {
fmt.Println("Pia-oracle version:", version)
return
}
config.Path = *confpath
pialog.InitializeLogging()
pialog.Info("Starting pia-oracle version:", version)
pialog.Info("Loading config from:", *confpath)
var err error
conf, err = config.GetConfig(fmt.Sprintf("%s/pia-oracle.toml", *confpath))
if err != nil {
pialog.Error(err)
pialog.Info("Exiting")
return
}
pialog.Info("Server started:", fmt.Sprintf("%s:%d", conf.Local.Listen, conf.Local.Port))
http.HandleFunc("/trigger", trigger)
http.HandleFunc("/callback", callback)
http.HandleFunc("/predict", predict)
err = http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Local.Listen, conf.Local.Port), nil)
if err != nil {
pialog.Error(err)
}
pialog.Info("Exiting")
}
func PushVarIMP(var_imps []interface{}) {
SetLocale.SetLocale(SetLocale.LC_ALL, "de_DE")
dsn := getDSN()
db, err := sql.Open("oci8", dsn)
check(err)
defer db.Close()
tx, err := db.Begin()
check(err)
stmt := PrepareStatement(tx, conf.Database.QueryOutImp)
fails_count := 0
for _, var_imp := range var_imps {
if s, ok := var_imp.(map[string]interface{}); ok {
err = ExecuteImpInsert(stmt, s)
if err != nil {
fails_count += 1
}
} else {
pialog.Error("Not a valid var_imp:\n", var_imp)
}
}
tx.Commit()
pialog.Info("IMP failures count:", fails_count)
}
func SelectDBData(db *sql.DB, query string) {
defer timeTrack(time.Now(), "SelectDBData")
rows, err := db.Query(query)
check(err)
defer rows.Close()
columns, err := rows.Columns()
check(err)
values := make([]interface{}, len(columns))
scanArgs := make([]interface{}, len(values))
for i := range values {
scanArgs[i] = &values[i]
}
c := 0
var claims []interface{}
outerSchema, innerSchema, codec := LoadAvroSchema(conf.Avro.OuterSchema, conf.Avro.InnerSchema)
for rows.Next() {
c += 1
err = rows.Scan(scanArgs...)
check(err)
m := make(map[string]interface{})
claim, err := goavro.NewRecord(innerSchema)
check(err)
for i, colName := range columns {
if val, ok := values[i].(time.Time); ok {
claim.Set(colName, val.Unix())
m[colName] = val.Unix()
} else {
claim.Set(colName, values[i])
m[colName] = values[i]
}
}
claims = append(claims, claim)
if len(claims) == 3000 {
claims_avro, err := goavro.NewRecord(outerSchema)
check(err)
claims_avro.Set("claims", claims)
buf := new(bytes.Buffer)
err = codec.Encode(buf, claims_avro)
go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)
claims = make([]interface{}, 0)
}
}
if rows.Err() != nil {
pialog.Error(rows.Err())
}
claims_avro, err := goavro.NewRecord(outerSchema)
check(err)
claims_avro.Set("claims", claims)
buf := new(bytes.Buffer)
err = codec.Encode(buf, claims_avro)
check(err)
go Post(conf.Rest.PredictionEndpoint, buf.Bytes(), conf.Rest.AppHeader)
pialog.Info("Total input records:", c)
}