func DetectFormat(in string, dateFormat string) (res string) {
if in != "" {
matchNumber := false
matchFloat := false
matchDate := false
formatDate := "((^(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)[\\d]{4}$)|(^[\\d]{4}(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])$))"
matchDate, _ = regexp.MatchString(formatDate, in)
if !matchDate && dateFormat != "" {
d := cast.String2Date(in, dateFormat)
if d.Year() > 1 {
matchDate = true
}
}
x := strings.Index(in, ".")
if x > 0 {
matchFloat = true
in = strings.Replace(in, ".", "", 1)
}
matchNumber, _ = regexp.MatchString("^\\d+$", in)
if strings.TrimSpace(in) == "true" || strings.TrimSpace(in) == "false" {
res = "bool"
} else {
res = "string"
if matchNumber {
res = "int"
if matchFloat {
res = "float"
}
}
if matchDate {
res = "date"
}
}
}
return res
}
func CheckDataType(inputModel reflect.StructField, inputVal interface{}, dateFormat string) (output string) {
if dateFormat == "" {
dateFormat = "dd/MM/yyyy"
}
output = ""
switch inputModel.Type.Kind() {
case reflect.Int:
temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
output = strconv.FormatInt(temp, 10)
case reflect.Int16:
temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
output = strconv.FormatInt(temp, 10)
case reflect.Int32:
temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
output = strconv.FormatInt(temp, 10)
case reflect.Int64:
temp, _ := strconv.ParseInt(strconv.Itoa(inputVal.(int)), 10, 32)
output = strconv.FormatInt(temp, 10)
case reflect.Float32:
temp, _ := strconv.ParseFloat(strconv.FormatFloat(inputVal.(float64), 'f', 3, 32), 32)
output = strconv.FormatFloat(temp, 'f', 3, 32)
case reflect.Float64:
temp, _ := strconv.ParseFloat(strconv.FormatFloat(inputVal.(float64), 'f', 3, 64), 64)
output = strconv.FormatFloat(temp, 'f', 3, 64)
case reflect.Bool:
temp, _ := strconv.ParseBool(strconv.FormatBool(inputVal.(bool)))
output = strconv.FormatBool(temp)
case reflect.String:
output += "\"" + inputVal.(string) + "\""
default:
dtype := DetectDataType(inputVal.(string), dateFormat)
if dtype == "date" {
output = "\"" + cast.Date2String(cast.String2Date(inputVal.(string), dateFormat), dateFormat) + "\""
}
}
return output
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
fmt.Println(c.QueryString)
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
tableData := []toolkit.M{}
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
// valueint := values
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
// rows.Scan(valuePtrs...)
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
// fmt.Println("Nilai val : ", val)
b, ok := val.([]byte)
if ok {
v = string(b)
} else {
v = val
}
entry.Set(strings.ToLower(col), v)
// entry.Set(col, values[i])
// e = toolkit.DecodeByte(val.([]byte), v)
// toolkit.FromBytes(toolkit.ToBytes(val, ""), "", v)
// entry.Set(col, v)
}
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := strings.ToLower(valueType.Field(i).Type.String())
if entry.Has(namaField) {
fmt.Println("isi entry : ", entry[namaField], dataType)
if strings.Contains(dataType, "int") {
entry.Set(namaField,
cast.ToInt(entry[namaField], cast.RoundingAuto))
} else if strings.Contains(dataType, "time.time") {
entry.Set(namaField,
cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
}
}
}
}
tableData = append(tableData, entry)
}
fmt.Println("Nilai table data : ", tableData)
if e != nil {
return e
}
if n == 0 {
// *m.(*[]map[string]interface{}) = tableData
// toolkit.Unjson(toolkit.Jsonify(tableData), m)
e = toolkit.Serde(tableData, m, "json")
fmt.Println("Nilai Model : ", m)
} else {
end := c.start + n
if end > len(tableData) {
e = errors.New("index out of range")
} else {
// *m.(*[]map[string]interface{}) = tableData[0:n]
//toolkit.Unjson(toolkit.Jsonify(tableData[0:n]), m)
e = toolkit.Serde(tableData[0:n], m, "json")
}
}
return e
}
func (q *Query) setConfigParam() {
ci := q.Connection().(*Connection).Info()
q.newfile = ci.Settings.Get("newfile", false).(bool)
q.isUseHeader = ci.Settings.Get("useheader", false).(bool)
// set header from reader =============== }
dateformat := ci.Settings.Get("dateformat", "").(string)
q.headerColumn = make([]headerstruct, 0, 0)
tdread, e := q.reader.Read()
for i, v := range tdread {
ts := headerstruct{}
ts.name = string(i)
ts.dataType = ""
if q.isUseHeader {
ts.name = v
}
q.headerColumn = append(q.headerColumn, ts)
}
if q.isUseHeader && e == nil {
tdread, e = q.reader.Read()
}
isCheckType := true
ix := 0
for isCheckType && e != io.EOF {
isCheckType = false
for i, v := range tdread {
if v != "" {
matchNumber := false
matchFloat := false
matchDate := false
formatDate := "((^(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)[\\d]{4}$)|(^[\\d]{4}(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])$))"
matchDate, _ = regexp.MatchString(formatDate, v)
if !matchDate && dateformat != "" {
d := cast.String2Date(v, dateformat)
if d.Year() > 1 {
matchDate = true
}
}
x := strings.Index(v, ".")
if x > 0 {
matchFloat = true
v = strings.Replace(v, ".", "", 1)
}
matchNumber, _ = regexp.MatchString("^\\d+$", v)
q.headerColumn[i].dataType = "string"
if matchNumber {
q.headerColumn[i].dataType = "int"
if matchFloat {
q.headerColumn[i].dataType = "float"
}
}
if matchDate {
q.headerColumn[i].dataType = "date"
}
}
}
for _, v := range q.headerColumn {
if v.dataType == "" {
isCheckType = true
}
}
if isCheckType {
tdread, e = q.reader.Read()
}
ix++
if ix > 10 {
break
}
}
for _, v := range q.headerColumn {
if v.dataType == "" {
v.dataType = "string"
}
}
_ = q.resetReader()
// ===================== }
if ci.Settings.Has("mapheader") {
smh := ci.Settings["mapheader"].([]toolkit.M)
for i, val := range smh {
ts := headerstruct{}
for name, dt := range val {
ts.name = name
ts.dataType = cast.ToString(dt)
}
if (i + 1) < len(q.headerColumn) {
q.headerColumn[i] = ts
} else {
q.headerColumn = append(q.headerColumn, ts)
}
}
}
}
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
// log.Printf("start parse:\n")
if !toolkit.IsPointer(m) {
// log.Printf("not pointer\n")
return errorlib.Error("", "", "Fetch", "Model object should be pointer")
}
// log.Printf("pointer\n")
slice := false
var ins []string
if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
slice = true
ins = in.([]string)
} else {
ins = append(ins, in.(string))
}
// log.Printf("outputType: %v\n", outputType)
if outputType == CSV {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
reader := csv.NewReader(strings.NewReader(""))
if strings.Contains(data, "','") {
reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
} else {
reader = csv.NewReader(strings.NewReader(data))
}
record, e := reader.Read()
if e != nil {
return e
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
}
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(record[i])
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for _, val := range header {
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else if outputType == JSON {
var temp interface{}
ins = InspectJson(ins)
//for catch multi json in one line
if JsonPart != "" && slice {
for {
tempjsonpart := JsonPart
JsonPart = ""
tempIn := InspectJson([]string{tempjsonpart})
if len(tempIn) == 0 {
break
} else {
for _, tin := range tempIn {
ins = append(ins, tin)
}
}
}
}
forSerde := strings.Join(ins, ",")
if slice {
forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
}
if len(ins) > 0 {
e := json.Unmarshal([]byte(forSerde), &temp)
if e != nil {
return e
}
e = toolkit.Serde(temp, m, "json")
if e != nil {
return e
}
}
} else {
var v reflect.Type
if slice {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
// log.Printf("v: %v\n", v)
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
// log.Printf("ivs: %v\n", ivs)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
/*log.Printf("data: %v\n", data)
log.Printf("iv: %v\n", iv)*/
splitted := strings.Split(data, "\t")
/*log.Printf("appendData: %v\n", appendData)
log.Printf("kind: %v\n", v.Kind())
log.Printf("test: %v", fmt.Sprintf("%v", v))
//log.Printf("v.Name: %T\n", v)
if fmt.Sprintf("%v", v) == "reflect.Value" {
log.Printf("else: %v\n", "reflect.Value")
for _, val := range header {
log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
log.Printf("appendData: %v\n", appendData)
} else */
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
// log.Printf("struct: %v\n", v.Kind())
for i := 0; i < v.NumField(); i++ {
tag := v.Field(i).Tag
// log.Printf("i: %v\n", i)
// log.Printf("name: (%v) tag: (%v)\n", appendData.Has(v.Field(i).Name), appendData.Has(tag.Get("tag_name")))
if appendData.Has(v.Field(i).Name) || appendData.Has(tag.Get("tag_name")) {
valthis := appendData[v.Field(i).Name]
if valthis == nil {
valthis = appendData[tag.Get("tag_name")]
}
// log.Printf("valthis: %v\n", valthis)
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
}
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
for i, val := range header {
appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
}
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
// log.Printf("else: %v\n", v.Kind())
for _, val := range header {
// log.Printf("val: %v\n", val)
valthis := appendData[val]
dtype := DetectFormat(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
// log.Printf("iv result: %v\n", iv)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
// log.Printf("ivs result: %v\n", ivs)
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
// log.Printf("result: %v\n", m)
}
return nil
}
func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error) {
if !toolkit.IsPointer(m) {
return errorlib.Error("", "", "Fetch", "Model object should be pointer")
}
slice := false
var ins []string
if reflect.ValueOf(m).Elem().Kind() == reflect.Slice || toolkit.TypeName(in) == "[]string" {
slice = true
ins = in.([]string)
} else {
ins = append(ins, in.(string))
}
if outputType == CSV {
var v reflect.Type
if slice && toolkit.TypeName(m) != "*interface {}" {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
reader := csv.NewReader(strings.NewReader(""))
if strings.Contains(data, "','") {
reader = csv.NewReader(strings.NewReader("\"" + strings.Trim(strings.Replace(data, "','", "\",\"", -1), "'") + "\""))
} else {
reader = csv.NewReader(strings.NewReader(data))
}
record, e := reader.Read()
if e != nil {
return e
}
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(record[i])
valthis := appendData[v.Field(i).Name]
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
case reflect.Bool:
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
default:
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for i, val := range header {
appendData[val] = strings.TrimSpace(record[i])
valthis := appendData[val]
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else if outputType == JSON {
var temp interface{}
ins = InspectJson(ins)
//for catch multi json in one line
if JsonPart != "" && slice {
for {
tempjsonpart := JsonPart
JsonPart = ""
tempIn := InspectJson([]string{tempjsonpart})
if len(tempIn) == 0 {
break
} else {
for _, tin := range tempIn {
ins = append(ins, tin)
}
}
}
}
forSerde := strings.Join(ins, ",")
if slice {
forSerde = fmt.Sprintf("[%s]", strings.Join(ins, ","))
}
if len(ins) > 0 {
e := json.Unmarshal([]byte(forSerde), &temp)
if e != nil {
return e
}
e = toolkit.Serde(temp, m, "json")
if e != nil {
return e
}
}
} else {
var v reflect.Type
if slice && toolkit.TypeName(m) != "*interface {}" {
v = reflect.TypeOf(m).Elem().Elem()
} else {
v = reflect.TypeOf(m).Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
for _, data := range ins {
appendData := toolkit.M{}
iv := reflect.New(v).Interface()
splitted := strings.Split(data, "\t")
if v.Kind() == reflect.Struct {
for i := 0; i < v.NumField(); i++ {
appendData[v.Field(i).Name] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
valthis := appendData[v.Field(i).Name]
switch v.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int16:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int32:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Int64:
appendData.Set(v.Field(i).Name, cast.ToInt(valthis, cast.RoundingAuto))
case reflect.Float32:
valf, _ := strconv.ParseFloat(valthis.(string), 32)
appendData.Set(v.Field(i).Name, valf)
case reflect.Float64:
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(v.Field(i).Name, valf)
case reflect.Bool:
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(v.Field(i).Name, valf)
default:
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(v.Field(i).Name, valf)
}
}
}
} else {
if len(header) == 0 {
e = errorlib.Error("", "", "Parse Out", "Header cant be null because object is not struct")
return e
}
for i, val := range header {
appendData[val] = strings.TrimSpace(strings.Trim(splitted[i], " '"))
valthis := appendData[val]
dtype := DetectDataType(valthis.(string), dateFormat)
if dtype == "int" {
appendData.Set(val, cast.ToInt(valthis, cast.RoundingAuto))
} else if dtype == "float" {
valf, _ := strconv.ParseFloat(valthis.(string), 64)
appendData.Set(val, valf)
} else if dtype == "date" {
valf := cast.String2Date(valthis.(string), dateFormat)
appendData.Set(val, valf)
} else if dtype == "bool" {
valf, _ := strconv.ParseBool(valthis.(string))
appendData.Set(val, valf)
}
}
}
toolkit.Serde(appendData, iv, JSON)
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
}
if slice {
reflect.ValueOf(m).Elem().Set(ivs)
} else {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
}
return nil
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
tableData := []toolkit.M{}
// var e error
rows, e := c.session.Query(c.QueryString)
var valueType reflect.Type
if n == 1 {
valueType = reflect.TypeOf(m).Elem()
} else {
valueType = reflect.TypeOf(m).Elem().Elem()
}
if e != nil {
return e
}
defer rows.Close()
columns, e := rows.Columns()
if e != nil {
return e
}
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := toolkit.M{}
for i, col := range columns {
var v interface{}
val := values[i]
// b, ok := val.([]byte)
// // toolkit.Println("i : ", i, " :col: ", col, " :val: ", val, " :b : ", b, " :type data : ", toolkit.Value(val))
// var out interface{}
// e = toolkit.Unjson(b, &out)
// // toolkit.Println("i : ", i, "b : ", b, " :out: ", v, " :: error : ", e)
// if e != nil {
// ok = false
// }
// if ok {
// v = out
// toolkit.Println("error OK :: ", ok, " :v :", v)
// } else {
// toolkit.Println("error OK :: ", ok, " :b :", b)
// v = string(b)
// }
v = val
entry.Set(strings.ToLower(col), v)
}
if valueType.Kind() == reflect.Struct {
for i := 0; i < valueType.NumField(); i++ {
namaField := strings.ToLower(valueType.Field(i).Name)
dataType := strings.ToLower(valueType.Field(i).Type.String())
if entry.Has(namaField) {
if strings.Contains(dataType, "int") {
entry.Set(namaField,
cast.ToInt(entry[namaField], cast.RoundingAuto))
} else if strings.Contains(dataType, "time.time") {
entry.Set(namaField,
cast.String2Date(cast.ToString(entry[namaField]), "2006-01-02 15:04:05"))
}
}
}
}
tableData = append(tableData, entry)
}
// toolkit.Println("... ::: ", tableData)
maxIndex := toolkit.SliceLen(tableData)
var e2 error
if e2 != nil {
return e2
}
end := c.start + n
if end > maxIndex || n == 0 {
end = maxIndex
}
if c.start >= maxIndex {
e2 = errors.New("No more data to fetched!")
} else {
e2 = toolkit.Serde(tableData[c.start:end], m, "json")
}
c.start = end
return e2
}
func (c *Connection) SetHeaderData() {
ci := c.Info()
dateformat := ci.Settings.Get("dateformat", "").(string)
var tempstruct []headerstruct
tempData, e := c.reader.Read()
for i, v := range tempData {
ts := headerstruct{}
// ts.index = i
ts.name = string(i)
ts.dataType = "string"
if c.isUseHeader {
ts.name = v
}
tempstruct = append(tempstruct, ts)
}
if c.isUseHeader && e != io.EOF {
tempData, e = c.reader.Read()
}
isCheckType := true
ix := 0
for isCheckType && e != io.EOF {
ix += 1
isCheckType = false
for i, v := range tempData {
if v != "" {
tempstruct[i].dataType = "string"
matchNumber := false
matchFloat := false
matchF1 := false
matchF2 := false
//dd.mm.yyyy dd/mm/yyyy dd-mm-yyyy
//yyyy.mm.dd yyyy/mm/dd yyyy-mm-dd
// formatDate := "((^(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)[\\d]{4}$)|(^[\\d]{4}(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])$))"
F1 := "(^(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)[\\d]{4}$)"
F2 := "(^[\\d]{4}(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])$)"
matchF1, _ = regexp.MatchString(F1, v)
matchF2, _ = regexp.MatchString(F2, v)
if !matchF1 && !matchF2 && dateformat != "" {
d := cast.String2Date(v, dateformat)
if !d.IsZero() {
tempstruct[i].dataType = "date"
tempstruct[i].format = dateformat
}
}
x := strings.Index(v, ".")
if x > 0 {
matchFloat = true
v = strings.Replace(v, ".", "", 1)
}
matchNumber, _ = regexp.MatchString("^\\d+$", v)
if matchNumber && string(v[0]) != "0" {
tempstruct[i].dataType = "int"
if matchFloat {
tempstruct[i].dataType = "float"
}
}
if matchF1 || matchF2 {
tempstruct[i].dataType = "date"
switch {
case strings.Contains(v, "."):
tempstruct[i].format = "dd.MM.YYYY"
if matchF2 {
tempstruct[i].format = "YYYY.MM.dd"
}
case strings.Contains(v, "-"):
tempstruct[i].format = "dd-MM-YYYY"
if matchF2 {
tempstruct[i].format = "YYYY-MM-dd"
}
case strings.Contains(v, "/"):
tempstruct[i].format = "dd/MM/YYYY"
if matchF2 {
tempstruct[i].format = "YYYY/MM/dd"
}
}
}
}
}
for _, v := range tempstruct {
if v.dataType == "" {
isCheckType = true
}
}
if isCheckType {
tempData, _ = c.reader.Read()
}
// fmt.Println(ix, "-", isCheckType)
// fmt.Println(tempstruct)
if ix > 5 {
break
}
}
c.headerColumn = tempstruct
c.file.Close()
c.file, _ = os.Open(ci.Host)
c.reader = csv.NewReader(c.file)
c.SetReaderParam()
if c.isUseHeader {
tempData, _ = c.reader.Read()
}
}
func (c *Connection) SetHeaderData() {
ci := c.Info()
dateformat := ci.Settings.Get("dateformat", "").(string)
var tempstruct []headerstruct
tempData, e := c.reader.Read()
for i, v := range tempData {
ts := headerstruct{}
// ts.index = i
ts.name = string(i)
ts.dataType = "string"
if c.isUseHeader {
ts.name = v
}
tempstruct = append(tempstruct, ts)
}
if c.isUseHeader && e != io.EOF {
tempData, e = c.reader.Read()
}
isCheckType := true
ix := 0
for isCheckType && e != io.EOF {
ix += 1
isCheckType = false
for i, v := range tempData {
if v != "" {
matchNumber := false
matchFloat := false
matchDate := false
formatDate := "((^(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)[\\d]{4}$)|(^[\\d]{4}(\\.|\\/|-)(0[0-9]|[0-9]|1[0-2])(\\.|\\/|-)(0[0-9]|[0-9]|(1|2)[0-9]|3[0-1])$))"
matchDate, _ = regexp.MatchString(formatDate, v)
if !matchDate && dateformat != "" {
d := cast.String2Date(v, dateformat)
if d.Year() > 1 {
matchDate = true
}
}
x := strings.Index(v, ".")
if x > 0 {
matchFloat = true
v = strings.Replace(v, ".", "", 1)
}
matchNumber, _ = regexp.MatchString("^\\d+$", v)
tempstruct[i].dataType = "string"
if matchNumber {
tempstruct[i].dataType = "int"
if matchFloat {
tempstruct[i].dataType = "float"
}
}
if matchDate {
tempstruct[i].dataType = "date"
}
}
}
for _, v := range tempstruct {
if v.dataType == "" {
isCheckType = true
}
}
if isCheckType {
tempData, _ = c.reader.Read()
}
// fmt.Println(ix, "-", isCheckType)
// fmt.Println(tempstruct)
if ix > 5 {
break
}
}
c.headerColumn = tempstruct
c.file.Close()
c.file, _ = os.Open(ci.Host)
c.reader = csv.NewReader(c.file)
c.SetReaderParam()
if c.isUseHeader {
tempData, _ = c.reader.Read()
}
}
func (w *WebGrabberController) GetLog(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
arrcmd := make([]string, 0, 0)
result := toolkit.M{}
payload := struct {
ID string `json:"_id"`
Date string `json:"date"`
}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
wg := new(colonycore.WebGrabber)
err = colonycore.Get(wg, payload.ID)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
o, err := toolkit.ToM(wg)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
// history := NewHistory(payload.ID)
// logs := history.GetLogHistory([]interface{}{o}, payload.Date)
// apppath := ""
// if runtime.GOOS == "windows" {
// arrcmd = append(arrcmd, "cmd")
// arrcmd = append(arrcmd, "/C")
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread.exe")
// } else {
// apppath = filepath.Join(EC_APP_PATH, "bin", "sedotanread")
// }
// arrcmd = append(arrcmd, apppath)
// arrcmd = append(arrcmd, `-readtype=logfile`)
// arrcmd = append(arrcmd, `-datetime=`+payload.Date)
// arrcmd = append(arrcmd, `-nameid=`+payload.ID)
// arrcmd = append(arrcmd, `-datas=`+toolkit.JsonString([]interface{}{o}))
// cmd := exec.Command(arrcmd[0], arrcmd[1:]...)
// byteoutput, err := cmd.CombinedOutput()
// if err != nil {
// return helper.CreateResult(false, nil, err.Error())
// }
logPath := ""
for _, v := range []interface{}{o} {
vMap, _ := toolkit.ToM(v)
logConf := vMap["logconf"].(map[string]interface{})
dateNowFormat := logConf["filepattern"].(string)
logpathconfig := logConf["logpath"].(string)
theDate := cast.String2Date(payload.Date, "YYYY/MM/dd HH:mm:ss")
theDateString := cast.Date2String(theDate, dateNowFormat)
fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString)
logPath = logpathconfig + fileName
}
client, server, err := w.ConnectToSedotanServer()
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
SshClient := *client
apppath := ""
if server.OS == "linux" {
apppath = server.AppPath + `/cli/sedotanread`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="logfile"`)
arrcmd = append(arrcmd, `-nameid="`+payload.ID+`"`)
arrcmd = append(arrcmd, `-pathfile="`+logPath+`"`)
arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`)
} else {
apppath = server.AppPath + `\bin\sedotanread.exe`
arrcmd = append(arrcmd, apppath)
arrcmd = append(arrcmd, `-readtype="logfile"`)
arrcmd = append(arrcmd, `-nameid=`+payload.ID+`"`)
arrcmd = append(arrcmd, `-pathfile=`+logPath+`"`)
arrcmd = append(arrcmd, `-datetime="`+payload.Date+`"`)
}
cmds := strings.Join(append(arrcmd[:1], arrcmd[1:]...), " ")
fmt.Println("====>", cmds)
output, err := SshClient.GetOutputCommandSsh(cmds)
if err != nil {
fmt.Println(err)
}
err = toolkit.UnjsonFromString(output, &result)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, result["DATA"], "")
}
func (w *WebGrabberController) GetLogHistory(datas []interface{}, date string) interface{} {
for _, v := range datas {
vMap, _ := toolkit.ToM(v)
logConf := vMap["logconf"].(map[string]interface{})
dateNowFormat := logConf["filepattern"].(string)
theDate := cast.String2Date(date, "YYYY/MM/dd HH:mm:ss")
theDateString := cast.Date2String(theDate, dateNowFormat)
fileName := fmt.Sprintf("%s-%s", logConf["filename"], theDateString)
w.logPath = f.Join(EC_DATA_PATH, "webgrabber", "log", fileName)
}
file, err := os.Open(w.logPath)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
defer file.Close()
addMinute := toolkit.String2Date(date, "YYYY/MM/dd HH:mm:ss").Add(1 * time.Minute)
dateAddMinute := toolkit.Date2String(addMinute, "YYYY/MM/dd HH:mm:ss")
getHours := strings.Split(date, ":")
getAddMinute := strings.Split(dateAddMinute, ":")
containString := getHours[0] + ":" + getHours[1]
containString2 := getAddMinute[0] + ":" + getAddMinute[1]
scanner := bufio.NewScanner(file)
lines := 0
containLines := 0
logsseparator := ""
add7Hours := func(s string) string {
t, _ := time.Parse("2006/01/02 15:04", s)
t = t.Add(time.Hour * 7)
return t.Format("2006/01/02 15:04")
}
containString = add7Hours(containString)
containString2 = add7Hours(containString2)
var logs []interface{}
for scanner.Scan() {
lines++
contains := strings.Contains(scanner.Text(), containString)
contains2 := strings.Contains(scanner.Text(), containString2)
if contains {
containLines = lines
logsseparator = containString
}
if contains2 {
containLines = lines
logsseparator = containString2
}
result := toolkit.M{}
if lines == containLines {
arrlogs := strings.Split(scanner.Text(), logsseparator)
result.Set("Type", arrlogs[0])
result.Set("Date", logsseparator+":"+arrlogs[1][1:3])
result.Set("Desc", arrlogs[1][4:len(arrlogs[1])])
logs = append(logs, result)
}
}
if err := scanner.Err(); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
var addSlice = toolkit.M{}
addSlice.Set("logs", logs)
return addSlice
}