func saverechistory(key string, dt toolkit.M) (err error) {
err = nil
fullfilename := filepath.Join(toolkit.ToString(histConf.Get("recpath", "")), mapRecHistory[key])
if EC_DATA_PATH != "" {
fullfilename = filepath.Join(EC_DATA_PATH, "datagrabber", "historyrec", mapRecHistory[key])
}
// fmt.Println(fullfilename, " - Key - ", key, " - filename - ", mapRecHistory)
cconfig := toolkit.M{"newfile": true, "useheader": true, "delimiter": ","}
conn, err := prepareconnection("csv", fullfilename, "", "", "", cconfig)
if err != nil {
return
}
q := conn.NewQuery().SetConfig("multiexec", true).Insert()
for k, v := range dt {
if toolkit.TypeName(v) == "toolkit.M" {
dt.Set(k, fmt.Sprintf("%v", v))
}
}
err = q.Exec(toolkit.M{}.Set("data", dt))
conn.Close()
return
}
func GetFieldsFromDS(_id string, opt ...string) ([]string, error) {
withSubFields := false
if len(opt) > 0 {
if opt[0] == "with sub fields" {
withSubFields = true
}
}
dataFetch, err := FetchDataFromDS(_id, 1)
if err != nil {
return nil, err
}
var fields []string
for i, val := range dataFetch {
if i > 0 {
break
}
for field, fieldValue := range val {
if withSubFields && (toolkit.TypeName(fieldValue) == "map[string]interface {}") {
for subField := range fieldValue.(map[string]interface{}) {
fields = append(fields, fmt.Sprintf("%s.%s", field, subField))
}
continue
}
fields = append(fields, field)
}
}
return fields, nil
}
func (a *SessionController) GetSession(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
_ = a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
}
tSession := new(acl.Session)
if find := toolkit.ToString(payload["find"]); find != "" {
filter = new(dbox.Filter)
filter = dbox.Contains("loginid", find)
}
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
// c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
// c, err := acl.Find(tSession, nil, toolkit.M{}.Set("take", payload["take"].(int)).Set("skip", payload["skip"].(int)))
c, err := acl.Find(tSession, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
err = c.Fetch(&arrm, 0, false)
for i, val := range arrm {
arrm[i].Set("duration", time.Since(val["created"].(time.Time)).Hours())
arrm[i].Set("status", "ACTIVE")
if val["expired"].(time.Time).Before(time.Now().UTC()) {
arrm[i].Set("duration", val["expired"].(time.Time).Sub(val["created"].(time.Time)).Hours())
arrm[i].Set("status", "EXPIRED")
}
}
c.Close()
c, err = acl.Find(tSession, filter, nil)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
} else {
return helper.CreateResult(true, data, "")
}
}
func (a *AdministrationController) GetAccess(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
}
tAccess := new(acl.Access)
if find := toolkit.ToString(payload["find"]); find != "" {
filter = new(dbox.Filter)
filter = dbox.Or(dbox.Contains("id", find),
dbox.Contains("title", find),
dbox.Contains("group1", find),
dbox.Contains("group2", find),
dbox.Contains("group3", find),
dbox.Contains("specialaccess1", find),
dbox.Contains("specialaccess2", find),
dbox.Contains("specialaccess3", find),
dbox.Contains("specialaccess4", find))
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
c, err := acl.Find(tAccess, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err == nil {
err = c.Fetch(&arrm, 0, false)
}
c.Close()
c, err = acl.Find(tAccess, filter, nil)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
} else {
return helper.CreateResult(true, data, "")
}
}
func getConfig() (err error) {
var result interface{}
if strings.Contains(configpath, "http") {
res, err := http.Get(configpath)
checkfatalerror(err)
defer res.Body.Close()
decoder := json.NewDecoder(res.Body)
err = decoder.Decode(&result)
checkfatalerror(err)
} else {
bytes, err := ioutil.ReadFile(configpath)
checkfatalerror(err)
err = json.Unmarshal(bytes, &result)
checkfatalerror(err)
}
switch toolkit.TypeName(result) {
case "[]interface {}":
isFound := false
for _, eachRaw := range result.([]interface{}) {
each := eachRaw.(map[string]interface{})
if each["_id"].(string) == _id {
m := toolkit.M{}
m, err = toolkit.ToM(each)
checkfatalerror(err)
config = m
isFound = true
}
}
if !isFound {
checkfatalerror(errors.New(fmt.Sprintf("config with _id %s is not found\n%#v", _id, result)))
}
case "map[string]interface {}":
m := toolkit.M{}
m, err = toolkit.ToM(result)
checkfatalerror(err)
config = m
default:
checkfatalerror(errors.New(fmt.Sprintf("invalid config file\n%#v", result)))
}
return
}
func (q *Query) DataType(data interface{}) interface{} {
if data != nil {
rf := toolkit.TypeName(data)
// toolkit.Println("data>", rf)
if rf == "[]uint8" {
uintToString := string(data.([]uint8))
spChar := strings.Contains(uintToString, "\x00")
if spChar {
uintToString = strings.Replace(uintToString, "\x00", "", 1)
}
floatVal, e := strconv.ParseFloat(uintToString, 64)
if e != nil {
} else {
data = floatVal
}
} else {
intVal, e := strconv.Atoi(toolkit.ToString(data))
if e != nil {
e = nil
floatVal, e := strconv.ParseFloat(toolkit.ToString(data), 64)
if e != nil {
e = nil
boolVal, e := strconv.ParseBool(toolkit.ToString(data))
if e != nil {
e = nil
dateVal, e := time.Parse(q.DateFormat, toolkit.ToString(data))
if e != nil {
data = data
} else { /*if string is date*/
data = dateVal
}
} else { /*if string is bool*/
data = boolVal
}
} else { /*if string is float*/
data = floatVal
}
} else { /*if string is int*/
data = intVal
}
}
}
return data
}
func (a *UserController) GetUser(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
_ = a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
if strings.Contains(toolkit.TypeName(payload["find"]), "float") {
payload["find"] = toolkit.ToInt(payload["find"], toolkit.RoundingAuto)
}
tUser := new(acl.User)
if find := toolkit.ToString(payload["find"]); find != "" {
filter = new(dbox.Filter)
filter = dbox.Or(dbox.Contains("id", find),
dbox.Contains("fullname", find),
dbox.Contains("email", find))
}
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
c, err := acl.Find(tUser, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
err = c.Fetch(&arrm, 0, false)
c.Close()
c, err = acl.Find(tUser, filter, nil)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
} else {
return helper.CreateResult(true, data, "")
}
}
func (a *GroupController) Search(r *knot.WebContext) interface{} {
var filter *dbox.Filter
r.Config.OutputType = knot.OutputJson
_ = a.InitialSetDatabase()
payload := map[string]interface{}{}
err := r.GetForms(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
if strings.Contains(toolkit.TypeName(payload["search"]), "float") {
payload["search"] = toolkit.ToInt(payload["search"], toolkit.RoundingAuto)
}
tGroup := new(acl.Group)
if search := toolkit.ToString(payload["search"]); search != "" {
filter = new(dbox.Filter)
filter = dbox.Or(dbox.Contains("_id", search), dbox.Contains("title", search), dbox.Contains("owner", search))
}
fmt.Println(filter)
take := toolkit.ToInt(payload["take"], toolkit.RoundingAuto)
skip := toolkit.ToInt(payload["skip"], toolkit.RoundingAuto)
c, err := acl.Find(tGroup, filter, toolkit.M{}.Set("take", take).Set("skip", skip))
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
data := toolkit.M{}
arrm := make([]toolkit.M, 0, 0)
err = c.Fetch(&arrm, 0, false)
data.Set("Datas", arrm)
data.Set("total", c.Count())
if err != nil {
return helper.CreateResult(true, nil, err.Error())
}
return helper.CreateResult(true, data, "")
}
/* ==========================================
var payload = {
sessionid:"t7AuS0YIE9w8gOWY22HPJaj1pSxEjBNU",
accesscheck:[""],
accessid:""
};
app.ajaxPost("/acl/authenticate", payload)
============================================= */
func (a *AclController) Authenticate(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
var iaccenum acl.AccessTypeEnum
payload := toolkit.M{}
result := toolkit.M{}
result.Set("hasaccess", false)
err := r.GetPayload(&payload)
switch {
case err != nil:
return helper.CreateResult(false, nil, err.Error())
}
switch toolkit.TypeName(payload["accesscheck"]) {
case "[]interface {}":
for _, val := range payload["accesscheck"].([]interface{}) {
tacc := acl.GetAccessEnum(toolkit.ToString(val))
if !acl.Matchaccess(int(tacc), int(iaccenum)) {
iaccenum += tacc
}
}
default:
iaccenum = acl.GetAccessEnum(toolkit.ToString(payload["accesscheck"]))
}
// toolkit.Println("Type name : ", toolkit.TypeName(payload["accesscheck"]))
found := acl.HasAccess(toolkit.ToString(payload["sessionid"]),
acl.IDTypeSession,
toolkit.ToString(payload["accessid"]),
iaccenum)
if found {
result.Set("hasaccess", found)
}
return helper.CreateResult(true, result, "")
}
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
// log.Printf("start parse:\n")
if !toolkit.IsPointer(m) {
// log.Printf("not pointer\n")
return errorlib.Error("", "", "Fetch", "Model object should be pointer")
}
// log.Printf("pointer\n")
slice := false
var ins []string
if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
slice = true
ins = in.([]string)
} else {
ins = append(ins, in.(string))
}
// log.Printf("outputType: %v\n", outputType)
if outputType == CSV {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
reader := csv.NewReader(strings.NewReader(""))
if strings.Contains(data, "','") {
reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
} else {
reader = csv.NewReader(strings.NewReader(data))
}
record, e := reader.Read()
if e != nil {
return e
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
}
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(record[i])
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for _, val := range header {
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else if outputType == JSON {
var temp interface{}
ins = InspectJson(ins)
//for catch multi json in one line
if JsonPart != "" && slice {
for {
tempjsonpart := JsonPart
JsonPart = ""
tempIn := InspectJson([]string{tempjsonpart})
if len(tempIn) == 0 {
break
} else {
for _, tin := range tempIn {
ins = append(ins, tin)
}
}
}
}
forSerde := strings.Join(ins, ",")
if slice {
forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
}
if len(ins) > 0 {
e := json.Unmarshal([]byte(forSerde), &temp)
if e != nil {
return e
}
e = toolkit.Serde(temp, m, "json")
if e != nil {
return e
}
}
} else {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
// log.Printf("v: %v\n", v)
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
// log.Printf("ivs: %v\n", ivs)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
/*log.Printf("data: %v\n", data)
log.Printf("iv: %v\n", iv)*/
splitted := strings.Split(data, "\t")
/*log.Printf("appendData: %v\n", appendData)
log.Printf("kind: %v\n", v.Kind())
log.Printf("test: %v", fmt.Sprintf("%v", v))
//log.Printf("v.Name: %T\n", v)
if fmt.Sprintf("%v", v) == "reflect.Value" {
log.Printf("else: %v\n", "reflect.Value")
for _, val := range header {
log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
log.Printf("appendData: %v\n", appendData)
} else */
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
// log.Printf("struct: %v\n", v.Kind())
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
// log.Printf("i: %v\n", i)
// log.Printf("name: (%v) tag: (%v)\n", appendData.Has(v.Field(i).Name), appendData.Has(tag.Get("tag_name")))
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
// log.Printf("valthis: %v\n", valthis)
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
// log.Printf("else: %v\n", v.Kind())
for _, val := range header {
// log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
// log.Printf("iv result: %v\n", iv)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
// log.Printf("ivs result: %v\n", ivs)
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
// log.Printf("result: %v\n", m)
}
return nil
}
func fetchConfig() (err error) {
switch toolkit.ToString(config.Get("sourcetype", "")) {
case "SourceType_HttpHtml":
SourceType = SourceType_HttpHtml
case "SourceType_HttpJson":
SourceType = SourceType_HttpJson
default:
err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
return
}
Log.AddLog("Start fetch grabconf", "INFO")
if !config.Has("grabconf") {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
return
}
tconfgrab := toolkit.M{}
tconfgrab, err = toolkit.ToM(config["grabconf"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
return
}
grabconfig := sedotan.Config{}
if tconfgrab.Has("formvalues") {
tfromvalues := toolkit.M{}
tfromvalues, err = toolkit.ToM(tconfgrab["formvalues"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, formvalues found error : %v", err.Error()))
return
}
grabconfig.SetFormValues(tfromvalues)
}
if tconfgrab.Has("loginvalues") {
grabconfig.LoginValues, err = toolkit.ToM(tconfgrab["loginvalues"])
}
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, loginvalues found error : %v", err.Error()))
return
}
grabconfig.URL = toolkit.ToString(tconfgrab.Get("url", ""))
grabconfig.CallType = toolkit.ToString(tconfgrab.Get("calltype", ""))
grabconfig.AuthType = toolkit.ToString(tconfgrab.Get("authtype", ""))
grabconfig.AuthUserId = toolkit.ToString(tconfgrab.Get("authuserid", ""))
grabconfig.AuthPassword = toolkit.ToString(tconfgrab.Get("authpassword", ""))
grabconfig.LoginUrl = toolkit.ToString(tconfgrab.Get("loginurl", ""))
grabconfig.LogoutUrl = toolkit.ToString(tconfgrab.Get("logouturl", ""))
Log.AddLog(fmt.Sprintf("Done fetch grabconf : %v", toolkit.JsonString(grabconfig)), "INFO")
wGrabber = sedotan.NewGrabber(grabconfig.URL, grabconfig.CallType, &grabconfig)
Log.AddLog("Start fetch datasettings", "INFO")
if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
wGrabber.DataSettings = make(map[string]*sedotan.DataSetting)
destDboxs = make(map[string]*DestInfo)
for i, xVal := range config["datasettings"].([]interface{}) {
err = nil
tDataSetting := sedotan.DataSetting{}
tDestDbox := DestInfo{}
mVal := toolkit.M{}
mVal, err = toolkit.ToM(xVal)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
continue
}
tnameid := toolkit.ToString(mVal.Get("nameid", ""))
if tnameid == "" {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
continue
}
tDataSetting.RowSelector = toolkit.ToString(mVal.Get("rowselector", ""))
// Fetch columnsettings
if !mVal.Has("columnsettings") || !(toolkit.TypeName(mVal["columnsettings"]) == "[]interface {}") {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : columnsettings is not found or incorrect", i, tnameid), "ERROR")
continue
}
tDataSetting.ColumnSettings = make([]*sedotan.GrabColumn, 0, 0)
for xi, Valcs := range mVal["columnsettings"].([]interface{}) {
mValcs := toolkit.M{}
mValcs, err = toolkit.ToM(Valcs)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : columnsettings is not found or incorrect", i, tnameid, xi), "ERROR")
continue
}
tgrabcolumn := sedotan.GrabColumn{}
tgrabcolumn.Alias = toolkit.ToString(mValcs.Get("alias", ""))
tgrabcolumn.Selector = toolkit.ToString(mValcs.Get("selector", ""))
tgrabcolumn.ValueType = toolkit.ToString(mValcs.Get("valuetype", ""))
tgrabcolumn.AttrName = toolkit.ToString(mValcs.Get("attrname", ""))
tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
tDataSetting.Column(tindex, &tgrabcolumn)
}
//Fetch Filter Condition
if mVal.Has("filtercond") {
tfiltercond := toolkit.M{}
tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
} else {
tDataSetting.SetFilterCond(tfiltercond)
}
}
//Fetch Connection Info
tConnInfo := toolkit.M{}
tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))
tHost := toolkit.ToString(tConnInfo.Get("host", ""))
tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
tSettings := toolkit.M{}
tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection.Close()
destDboxs[tnameid] = &tDestDbox
wGrabber.DataSettings[tnameid] = &tDataSetting
}
err = nil
if len(destDboxs) == 0 || len(wGrabber.DataSettings) == 0 {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
if !config.Has("histconf") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
Log.AddLog("Start fetch histconf", "INFO")
histConf, err = toolkit.ToM(config.Get("histconf", nil))
if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
return
}
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
if !toolkit.IsPointer(m) {
return errorlib.Error("", "", "Fetch", "Model object should be pointer")
}
slice := false
var ins []string
if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
slice = true
ins = in.([]string)
} else {
ins = append(ins, in.(string))
}
if outputType == CSV {
var v reflect.Type
if slice && toolkit.TypeName(m) != "*interface {}" {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
reader := csv.NewReader(strings.NewReader(""))
if strings.Contains(data, "','") {
reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
} else {
reader = csv.NewReader(strings.NewReader(data))
}
record, e := reader.Read()
if e != nil {
return e
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
valthis := appendData[v.Field(i).Name]
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
case reflect.Bool:
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
default:
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for i, val := range header {
appendData[val] = strings.TrimSpace(record[i])
valthis := appendData[val]
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else if outputType == JSON {
var temp interface{}
ins = InspectJson(ins)
//for catch multi json in one line
if JsonPart != "" && slice {
for {
tempjsonpart := JsonPart
JsonPart = ""
tempIn := InspectJson([]string{tempjsonpart})
if len(tempIn) == 0 {
break
} else {
for _, tin := range tempIn {
ins = append(ins, tin)
}
}
}
}
forSerde := strings.Join(ins, ",")
if slice {
forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
}
if len(ins) > 0 {
e := json.Unmarshal([]byte(forSerde), &temp)
if e != nil {
return e
}
e = toolkit.Serde(temp, m, "json")
if e != nil {
return e
}
}
} else {
var v reflect.Type
if slice && toolkit.TypeName(m) != "*interface {}" {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
splitted := strings.Split(data, "\t")
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
valthis := appendData[v.Field(i).Name]
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
case reflect.Bool:
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
default:
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for i, val := range header {
appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
valthis := appendData[val]
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
}
return nil
}
func filterCondition(cond toolkit.M) *dbox.Filter {
fb := new(dbox.Filter)
for key, val := range cond {
if key == "$and" || key == "$or" {
afb := []*dbox.Filter{}
for _, sVal := range val.([]interface{}) {
rVal := sVal.(map[string]interface{})
mVal := toolkit.M{}
for rKey, mapVal := range rVal {
mVal.Set(rKey, mapVal)
}
afb = append(afb, filterCondition(mVal))
}
if key == "$and" {
fb = dbox.And(afb...)
} else {
fb = dbox.Or(afb...)
}
} else {
if toolkit.TypeName(val) == "map[string]interface {}" {
mVal := val.(map[string]interface{})
tomVal, _ := toolkit.ToM(mVal)
switch {
case tomVal.Has("$eq"):
fb = dbox.Eq(key, tomVal["$eq"])
case tomVal.Has("$ne"):
fb = dbox.Ne(key, tomVal["$ne"])
case tomVal.Has("$regex"):
fb = dbox.Contains(key, toolkit.ToString(tomVal["$regex"]))
case tomVal.Has("$gt"):
fb = dbox.Gt(key, tomVal["$gt"])
case tomVal.Has("$gte"):
fb = dbox.Gte(key, tomVal["$gte"])
case tomVal.Has("$lt"):
fb = dbox.Lt(key, tomVal["$lt"])
case tomVal.Has("$lte"):
fb = dbox.Lte(key, tomVal["$lte"])
case tomVal.Has("$in"):
tval := make([]interface{}, 0, 0)
if toolkit.TypeName(tomVal["$in"]) == "[]interface {}" {
for _, tin := range tomVal["$in"].([]interface{}) {
tval = append(tval, tin)
}
} else {
tval = append(tval, tomVal["$in"])
}
fb = dbox.In(key, tval...)
case tomVal.Has("$nin"):
tval := make([]interface{}, 0, 0)
if toolkit.TypeName(tomVal["$nin"]) == "[]interface {}" {
for _, tin := range tomVal["$nin"].([]interface{}) {
tval = append(tval, tin)
}
} else {
tval = append(tval, tomVal["$nin"])
}
fb = dbox.Nin(key, tval...)
}
} else {
fb = dbox.Eq(key, val)
}
}
}
return fb
}
func (d *DataBrowserController) TestQuery(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
data := colonycore.DataBrowser{} //map[string]interface{}{}
err := r.GetPayload(&data)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
conn, datacon, err := d.connToDatabase(data.ConnectionID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
query, err := d.parseQuery(conn, data, datacon)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
cursor, err := query.Cursor(nil)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
defer cursor.Close()
dataFetch := []toolkit.M{}
err = cursor.Fetch(&dataFetch, 1, false)
if err != nil {
cursor.ResetFetch()
err = cursor.Fetch(&dataFetch, 0, false)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
}
metadata := []*colonycore.StructInfo{}
// var dt string
for i, dataFields := range dataFetch {
if i > 0 {
break
}
j := 1
for keyField, dataField := range dataFields {
if strings.Contains(keyField, "id") && !strings.Contains(data.QueryText, "id") &&
!strings.Contains(data.QueryText, "*") && data.TableNames == "" {
continue
}
sInfo := &colonycore.StructInfo{}
sInfo.Field = keyField
sInfo.Label = keyField
rf := "string"
if dataField != nil {
rf = toolkit.TypeName(dataField)
if rf == "time.Time" {
rf = "date"
}
}
sInfo.DataType = rf
sInfo.Format = ""
sInfo.Align = "Left"
sInfo.ShowIndex = toolkit.ToInt(j, toolkit.RoundingAuto)
sInfo.Sortable = false
sInfo.SimpleFilter = false
sInfo.AdvanceFilter = false
sInfo.Aggregate = ""
metadata = append(metadata, sInfo)
j++
}
}
data.MetaData = metadata
return helper.CreateResult(true, data, "")
}
func fetchConfig() (err error) {
switch toolkit.ToString(config.Get("sourcetype", "")) {
case "SourceType_DocExcel":
SourceType = SourceType_DocExcel
case "SourceType_DocMongo":
SourceType = SourceType_DocMongo
default:
err = errors.New(fmt.Sprintf("Fetch Config, Source type is not defined : %v", config.Get("sourcetype", "")))
return
}
Log.AddLog("Start fetch grabconf", "INFO")
if !config.Has("grabconf") {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf not found error"))
return
}
tconfgrab := toolkit.M{}
tconfgrab, err = toolkit.ToM(config["grabconf"])
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, grabconf found error : %v", err.Error()))
return
}
if !tconfgrab.Has("doctype") {
err = errors.New("Fetch Config, doctype not found")
return
}
ci := dbox.ConnectionInfo{}
mapconninfo := toolkit.M{}
mapconninfo, err = toolkit.ToM(tconfgrab.Get("connectioninfo", nil))
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo found error : %v", err.Error()))
return
}
ci.Host = toolkit.ToString(mapconninfo.Get("host", ""))
ci.Database = toolkit.ToString(mapconninfo.Get("database", ""))
ci.UserName = toolkit.ToString(mapconninfo.Get("userName", ""))
ci.Password = toolkit.ToString(mapconninfo.Get("password", ""))
ci.Settings, err = toolkit.ToM(mapconninfo.Get("settings", nil))
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, load connectioninfo.settings found error : %v", err.Error()))
return
}
sGrabber, err = sedotan.NewGetDatabase(ci.Host, toolkit.ToString(tconfgrab.Get("doctype", "")), &ci)
if err != nil {
err = errors.New(fmt.Sprintf("Fetch Config, create new get database found error : %v", err.Error()))
}
Log.AddLog("Start fetch datasettings", "INFO")
if !config.Has("datasettings") || !(toolkit.TypeName(config["datasettings"]) == "[]interface {}") {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
sGrabber.CollectionSettings = make(map[string]*sedotan.CollectionSetting)
destDboxs = make(map[string]*DestInfo)
for i, xVal := range config["datasettings"].([]interface{}) {
err = nil
tCollectionSetting := sedotan.CollectionSetting{}
tDestDbox := DestInfo{}
mVal := toolkit.M{}
mVal, err = toolkit.ToM(xVal)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Found : %v", i, err.Error()), "ERROR")
continue
}
tnameid := toolkit.ToString(mVal.Get("nameid", ""))
if tnameid == "" {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d] Data Setting Id is not found", i), "ERROR")
continue
}
tCollectionSetting.Collection = toolkit.ToString(mVal.Get("collection", ""))
// Fetch mapssettings
if !mVal.Has("mapssettings") || !(toolkit.TypeName(mVal["mapssettings"]) == "[]interface {}") {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : mapssettings is not found or incorrect", i, tnameid), "ERROR")
continue
}
tCollectionSetting.MapsColumns = make([]*sedotan.MapColumn, 0, 0)
for xi, Valcs := range mVal["mapssettings"].([]interface{}) {
mValcs := toolkit.M{}
mValcs, err = toolkit.ToM(Valcs)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v.%v] Found : mapssettings is not found or incorrect", i, tnameid, xi), "ERROR")
continue
}
tgrabcolumn := sedotan.MapColumn{}
tgrabcolumn.Source = toolkit.ToString(mValcs.Get("source", ""))
tgrabcolumn.SType = toolkit.ToString(mValcs.Get("sourcetype", ""))
tgrabcolumn.Destination = toolkit.ToString(mValcs.Get("destination", ""))
tgrabcolumn.DType = toolkit.ToString(mValcs.Get("destinationtype", ""))
// tindex := toolkit.ToInt(mValcs.Get("index", 0), toolkit.RoundingAuto)
tCollectionSetting.MapsColumns = append(tCollectionSetting.MapsColumns, &tgrabcolumn)
}
//Fetch Filter Condition
if mVal.Has("filtercond") {
tfiltercond := toolkit.M{}
tfiltercond, err = toolkit.ToM(mVal.Get("filtercond", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : filter cond is incorrect, %v", i, tnameid, err.Error()), "ERROR")
} else {
tCollectionSetting.SetFilterCond(tfiltercond)
}
}
//Fetch limit data
if mVal.Has("limitrow") {
tLimitrow, err := toolkit.ToM(mVal["limitrow"])
if err == nil {
tCollectionSetting.Take = toolkit.ToInt(tLimitrow.Get("take", 0), toolkit.RoundingAuto)
tCollectionSetting.Skip = toolkit.ToInt(tLimitrow.Get("skip", 0), toolkit.RoundingAuto)
}
}
//Fetch Connection Info
tConnInfo := toolkit.M{}
tConnInfo, err = toolkit.ToM(mVal.Get("connectioninfo", toolkit.M{}))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.desttype = toolkit.ToString(mVal.Get("desttype", ""))
tDestDbox.collection = toolkit.ToString(tConnInfo.Get("collection", ""))
tHost := toolkit.ToString(tConnInfo.Get("host", ""))
tDatabase := toolkit.ToString(tConnInfo.Get("database", ""))
tUserName := toolkit.ToString(tConnInfo.Get("username", ""))
tPassword := toolkit.ToString(tConnInfo.Get("password", ""))
tSettings := toolkit.M{}
tSettings, err = toolkit.ToM(tConnInfo.Get("settings", nil))
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Connection Setting Found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection, err = prepareconnection(tDestDbox.desttype, tHost, tDatabase, tUserName, tPassword, tSettings)
if err != nil {
Log.AddLog(fmt.Sprintf("[Fetch.Ds.%d.%v] Create connection found : %v", i, tnameid, err.Error()), "ERROR")
continue
}
tDestDbox.IConnection.Close()
destDboxs[tnameid] = &tDestDbox
sGrabber.CollectionSettings[tnameid] = &tCollectionSetting
}
err = nil
if len(destDboxs) == 0 || len(sGrabber.CollectionSettings) == 0 {
err = errors.New("Fetch Config, datasettings is not found or have wrong format")
return
}
if !config.Has("histconf") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
Log.AddLog("Start fetch histconf", "INFO")
histConf, err = toolkit.ToM(config.Get("histconf", nil))
if err != nil || len(histConf) == 0 || !histConf.Has("histpath") || !histConf.Has("recpath") || !histConf.Has("filename") || !histConf.Has("filepattern") {
err = errors.New("Fetch Config, history configuration is not found or have wrong format")
return
}
if !config.Has("extcommand") {
return
}
Log.AddLog("Start fetch extcommand", "INFO")
extCommand, _ = toolkit.ToM(config.Get("extcommand", nil))
return
}
func checkconfig(cm toolkit.M) (err error) {
err = nil
if !cm.Has("_id") {
err = errors.New(fmt.Sprintf("_id is not found"))
return
}
if !cm.Has("sourcetype") {
err = errors.New(fmt.Sprintf("sourcetype is not found"))
return
}
if cm.Has("grabconf") {
_, err = toolkit.ToM(cm["grabconf"])
} else {
err = errors.New(fmt.Sprintf("grab config is not found"))
}
if err != nil {
return
}
if cm.Has("intervalconf") {
_, err = toolkit.ToM(cm["intervalconf"])
} else {
err = errors.New(fmt.Sprintf("interval configuration is not found"))
}
if err != nil {
return
}
if cm.Has("logconf") {
_, err = toolkit.ToM(cm["logconf"])
} else {
err = errors.New(fmt.Sprintf("log configuration is not found"))
}
if err != nil {
return
}
if cm.Has("datasettings") {
if toolkit.TypeName(cm["datasettings"]) != "[]interface {}" {
err = errors.New(fmt.Sprintf("data settings must be []interface {}"))
}
} else {
err = errors.New(fmt.Sprintf("data settings is not found"))
}
if err != nil {
return
}
if cm.Has("histconf") {
_, err = toolkit.ToM(cm["histconf"])
} else {
err = errors.New(fmt.Sprintf("history configuration is not found"))
}
if err != nil {
return
}
return
}
func fetchconfig() {
var result interface{}
if strings.Contains(configPath, "http") {
res, configerr := http.Get(configPath)
if configerr != nil {
return
}
defer res.Body.Close()
decoder := json.NewDecoder(res.Body)
configerr = decoder.Decode(&result)
} else {
bytes, configerr := ioutil.ReadFile(configPath)
if configerr != nil {
return
}
configerr = json.Unmarshal(bytes, &result)
}
if configerr != nil {
return
}
switch toolkit.TypeName(result) {
case "[]interface {}":
for i, each := range result.([]interface{}) {
m, err := toolkit.ToM(each)
if err == nil {
err = checkconfig(m)
}
if err == nil {
validconfig += 1
if m.Get("running", false).(bool) {
config = append(config, m)
}
} else {
// tstring := fmt.Sprintf("%v;(Config index %d. %v)", configerr, i, err)
// configerr = errors.New(fmt.Sprintf("%v", tstring))
Log.AddLog(fmt.Sprintf("Config index %d. %v", i, err), "ERROR")
}
}
case "map[string]interface {}":
m, err := toolkit.ToM(result)
if err == nil {
err = checkconfig(m)
}
if err == nil {
validconfig += 1
if m.Get("running", false).(bool) {
config = append(config, m)
}
} else {
Log.AddLog(fmt.Sprintf("Fetch Config Error Found : %v", err), "ERROR")
}
default:
Log.AddLog(fmt.Sprintf("invalid config file\n%#v", result), "ERROR")
}
}
func getJsonFilePath(o interface{}) string {
tns := strings.Split(strings.ToLower(toolkit.TypeName(o)), ".")
fn := tns[len(tns)-1] + ".json"
return filepath.Join(ConfigPath, fn)
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
tableData := []toolkit.M{}
var e error
h := c.sessionHive
if h != nil {
e = h.Exec(c.QueryString, func(x hive.HiveResult) error {
tableData = append(tableData, x.ResultObj.(map[string]interface{}))
return nil
})
} else {
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
dataTypeList := toolkit.M{}
var isStruct bool
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := valueType.Field(i).Type.String()
dataTypeList.Set(namaField, dataType)
}
isStruct = true
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
var ok bool
var b []byte
if val == nil {
v = nil
} else {
b, ok = val.([]byte)
if ok {
v = string(b)
} else {
v = val
}
}
/*mysql always byte, postgres only string that byte, oracle always string, mssql agree with field datatype*/
if (ok && (c.driver == "mysql" || c.driver == "postgres")) || c.driver == "oci8" {
if isStruct {
v = c.structValue(dataTypeList, col, v)
} else {
intVal, e := strconv.Atoi(toolkit.ToString(v))
if e != nil {
e = nil
floatVal, e := strconv.ParseFloat(toolkit.ToString(v), 64)
if e != nil {
e = nil
boolVal, e := strconv.ParseBool(toolkit.ToString(v))
if e != nil {
e = nil
dateVal, e := time.Parse(c.DateFormat, toolkit.ToString(v))
if e != nil {
v = v
} else { /*if string is date*/
v = dateVal
}
} else { /*if string is bool*/
v = boolVal
}
} else { /*if string is float*/
v = floatVal
}
} else { /*if string is int*/
v = intVal
}
}
}
toolkit.Println(col, toolkit.TypeName(v), v)
entry.Set(strings.ToLower(col), v)
}
tableData = append(tableData, entry)
}
}
maxIndex := toolkit.SliceLen(tableData)
if e != nil {
return e
}
end := c.start + n
if end > maxIndex || n == 0 {
end = maxIndex
}
if c.start >= maxIndex {
e = errors.New("No more data to fetched!")
} else {
e = toolkit.Serde(tableData[c.start:end], m, "json")
}
c.start = end
return e
}
func (d *DataSourceController) parseMetadata(data toolkit.M) []*colonycore.FieldInfo {
metadata := []*colonycore.FieldInfo{}
for key, val := range data {
label := key
lookup := new(colonycore.Lookup)
lookup.LookupFields = []string{}
meta := new(colonycore.FieldInfo)
meta.ID = key
meta.Label = label
meta.Type, _ = helper.GetBetterType(val)
meta.Format = ""
meta.Lookup = lookup
meta.Type = "string"
if val != nil {
if toolkit.TypeName(val) == "map[string]interface {}" {
val, _ = toolkit.ToM(val)
}
switch toolkit.TypeName(val) {
case "toolkit.M":
meta.Type = "object"
meta.Sub = d.parseMetadata(val.(toolkit.M))
case "[]interface {}":
meta.Type = "array"
valArray := val.([]interface{})
if len(valArray) > 0 {
if toolkit.TypeName(valArray[0]) == "map[string]interface {}" {
valArray[0], _ = toolkit.ToM(valArray[0])
}
if toolkit.TypeName(valArray[0]) == "toolkit.M" {
meta.Type = "array-objects"
meta.Sub = d.parseMetadata(valArray[0].(toolkit.M))
} else {
switch target := toolkit.TypeName(valArray[0]); {
case strings.Contains(target, "interface"):
case strings.Contains(target, "string"):
meta.Type = "array-string"
case strings.Contains(target, "int"):
meta.Type = "array-int"
case strings.Contains(target, "float"):
case strings.Contains(target, "double"):
meta.Type = "array-double"
default:
meta.Type = "array-string"
}
}
}
}
}
metadata = append(metadata, meta)
}
return metadata
}
