func (w *WebGrabberController) GetFetchedData(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
w.PrepareHistoryPath()
payload := struct {
Driver string
Host string
Database string
Collection string
Username string
Password string
}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
var data []toolkit.M
if payload.Driver == "csv" {
config := toolkit.M{"useheader": true, "delimiter": ","}
query := helper.Query("csv", payload.Host, "", "", "", config)
data, err = query.SelectAll("")
} else {
query := helper.Query("mongo", payload.Host, payload.Database, payload.Username, payload.Password)
data, err = query.SelectAll(payload.Collection)
}
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, data, "")
}
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) {
logConf, err := d.getLogger(dataGrabber)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
defer logConf.Close()
}
message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType)
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
dsOrigin := new(colonycore.DataSource)
err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dsDestination := new(colonycore.DataSource)
err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dataDS, _, conn, query, metaSave, err := new(DataSourceController).
ConnectToDataSource(dataGrabber.DataSourceOrigin)
if len(dataDS.QueryInfo) == 0 {
message := "Data source origin has invalid query"
logConf.AddLog(message, "ERROR")
return false, nil, message
}
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer conn.Close()
if metaSave.keyword != "" {
message := `Data source origin query is not "Select"`
logConf.AddLog(message, "ERROR")
return false, nil, message
}
cursor, err := query.Cursor(nil)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer cursor.Close()
data := []toolkit.M{}
err = cursor.Fetch(&data, 0, false)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
arrayContains := func(slice []string, key string) bool {
for _, each := range slice {
if each == key {
return true
}
}
return false
}
connDesc := new(colonycore.Connection)
err = colonycore.Get(connDesc, dsDestination.ConnectionID)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
transformedData := []toolkit.M{}
for _, each := range data {
eachTransformedData := toolkit.M{}
for _, eachMeta := range dsDestination.MetaData {
if arrayContains(dataGrabber.IgnoreFieldsDestination, eachMeta.ID) {
continue
}
fieldFrom := eachMeta.ID
checkMap:
for _, eachMap := range dataGrabber.Map {
if eachMap.FieldDestination == eachMeta.ID {
fieldFrom = eachMap.FieldOrigin
break checkMap
}
}
eachTransformedData.Set(eachMeta.ID, each.Get(fieldFrom))
}
transformedData = append(transformedData, eachTransformedData)
tableName := dsDestination.QueryInfo.GetString("from")
queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.GetString("_id")))
queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
err = queryWrapper.Save(tableName, eachTransformedData)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
}
message = fmt.Sprintf("===> Success transforming %v data", len(transformedData))
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
return true, transformedData, ""
}
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) {
logConf, err := d.getLogger(dataGrabber)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
defer logConf.Close()
}
message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType)
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
dsOrigin := new(colonycore.DataSource)
err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dsDestination := new(colonycore.DataSource)
err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dataDS, _, conn, query, metaSave, err := new(DataSourceController).
ConnectToDataSource(dataGrabber.DataSourceOrigin)
if len(dataDS.QueryInfo) == 0 {
message := "Data source origin has invalid query"
logConf.AddLog(message, "ERROR")
return false, nil, message
}
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer conn.Close()
if metaSave.keyword != "" {
message := `Data source origin query is not "Select"`
logConf.AddLog(message, "ERROR")
return false, nil, message
}
cursor, err := query.Cursor(nil)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer cursor.Close()
data := []toolkit.M{}
err = cursor.Fetch(&data, 0, false)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
connDesc := new(colonycore.Connection)
err = colonycore.Get(connDesc, dsDestination.ConnectionID)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
const FLAG_ARG_DATA string = `%1`
transformedData := []toolkit.M{}
for _, each := range data {
eachTransformedData := toolkit.M{}
for _, eachMap := range dataGrabber.Maps {
var valueEachSourceField interface{}
// ============================================ SOURCE
if !strings.Contains(eachMap.Source, "|") {
// source could be: field, object, array
valueEachSourceField = each.Get(eachMap.Source)
} else {
// source could be: field of object, field of array-objects
prev := strings.Split(eachMap.Source, "|")[0]
next := strings.Split(eachMap.Source, "|")[1]
var fieldInfoDes *colonycore.FieldInfo = nil
for _, eds := range dsOrigin.MetaData {
if eds.ID == prev {
fieldInfoDes = eds
break
}
}
if fieldInfoDes != nil {
// source is field of array-objects
if fieldInfoDes.Type == "array-objects" {
valueObjects := []interface{}{}
if temp, _ := each.Get(prev, nil).([]interface{}); temp != nil {
valueObjects = make([]interface{}, len(temp))
for i, each := range temp {
if tempSub, _ := toolkit.ToM(each); tempSub != nil {
valueObjects[i] = tempSub.Get(next)
}
}
}
valueEachSourceField = valueObjects
} else {
// source is field of object
valueObject := toolkit.M{}
if valueObject, _ = toolkit.ToM(each.Get(prev)); valueObject != nil {
valueEachSourceField = valueObject.Get(next)
}
}
}
}
// ============================================ DESTINATION
if !strings.Contains(eachMap.Destination, "|") {
if eachMap.SourceType == "object" {
sourceObject, _ := toolkit.ToM(valueEachSourceField)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for _, desMeta := range dsDestination.MetaData {
if desMeta.ID == eachMap.Destination {
for _, eachMetaSub := range desMeta.Sub {
// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
}
break
}
}
eachTransformedData.Set(eachMap.Destination, valueObject)
} else if eachMap.SourceType == "array-objects" {
sourceObjects, _ := valueEachSourceField.([]interface{})
if sourceObjects == nil {
sourceObjects = []interface{}{}
}
valueObjects := []interface{}{}
for _, sourceObjectRaw := range sourceObjects {
sourceObject, _ := toolkit.ToM(sourceObjectRaw)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for _, desMeta := range dsDestination.MetaData {
if desMeta.ID == eachMap.Destination {
for _, eachMetaSub := range desMeta.Sub {
// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
}
break
}
}
valueObjects = append(valueObjects, valueObject)
}
eachTransformedData.Set(eachMap.Destination, valueObjects)
} else {
if strings.Contains(eachMap.DestinationType, "array") {
valueObjects := each.Get(eachMap.Source)
eachTransformedData.Set(eachMap.Destination, valueObjects)
} else {
// eachTransformedData.Set(eachMap.Destination, convertDataType(eachMap.DestinationType, eachMap.Source, each))
eachTransformedData.Set(eachMap.Destination, d.convertTo(each.Get(eachMap.Source), eachMap.DestinationType))
}
}
} else {
prev := strings.Split(eachMap.Destination, "|")[0]
next := strings.Split(eachMap.Destination, "|")[1]
var fieldInfoDes *colonycore.FieldInfo = nil
for _, eds := range dsDestination.MetaData {
if eds.ID == prev {
fieldInfoDes = eds
break
}
}
if fieldInfoDes != nil {
if fieldInfoDes.Type == "array-objects" {
valueObjects := []interface{}{}
if temp := eachTransformedData.Get(prev, nil); temp == nil {
valueObjects = []interface{}{}
} else {
valueObjects, _ = temp.([]interface{})
if valueObjects == nil {
valueObjects = []interface{}{}
}
}
if temp, _ := valueEachSourceField.([]interface{}); temp != nil {
for i, eachVal := range temp {
valueObject := toolkit.M{}
if len(valueObjects) > i {
if temp2, _ := toolkit.ToM(valueObjects[i]); temp2 != nil {
valueObject = temp2
// valueObject.Set(next, eachVal)
valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
}
valueObjects[i] = valueObject
} else {
if fieldInfoDes.Sub != nil {
for _, subMeta := range fieldInfoDes.Sub {
valueObject.Set(subMeta.ID, nil)
}
}
// valueObject.Set(next, eachVal)
valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
valueObjects = append(valueObjects, valueObject)
}
}
}
eachTransformedData.Set(prev, valueObjects)
} else {
valueObject, _ := toolkit.ToM(eachTransformedData.Get(prev))
if valueObject == nil {
valueObject = toolkit.M{}
}
//tambahan
prevSource := strings.Split(eachMap.Source, "|")[0]
nextSource := strings.Split(eachMap.Source, "|")[1]
mval, _ := toolkit.ToM(each.Get(prevSource, nil))
//=========
valueObject.Set(next, d.convertTo(mval.Get(nextSource), eachMap.DestinationType))
// valueObject.Set(next, convertDataType(eachMap.DestinationType, nextSource, mval))
eachTransformedData.Set(prev, valueObject)
}
}
}
}
transformedData = append(transformedData, eachTransformedData)
dataToSave := eachTransformedData
// ================ pre transfer command
if dataGrabber.PreTransferCommand != "" {
// jsonTranformedDataBytes, err := json.Marshal(each)
jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
if err != nil {
return false, nil, err.Error()
}
jsonTranformedData := string(jsonTranformedDataBytes)
var preCommand = dataGrabber.PreTransferCommand
if strings.Contains(dataGrabber.PreTransferCommand, FLAG_ARG_DATA) {
preCommand = strings.TrimSpace(strings.Replace(dataGrabber.PreTransferCommand, FLAG_ARG_DATA, "", -1))
}
dataToSave = toolkit.M{}
output, err := toolkit.RunCommand(preCommand, jsonTranformedData)
fmt.Printf("===> Pre Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", preCommand, jsonTranformedData, output)
if err == nil {
postData := toolkit.M{}
if err := json.Unmarshal([]byte(output), &postData); err == nil {
dataToSave = postData
}
}
}
// ================
if len(dataToSave) == 0 {
continue
}
nilFieldDest := eachTransformedData
for _, metadataDest := range dsDestination.MetaData {
if temp := eachTransformedData.Get(metadataDest.ID); temp == nil {
if metadataDest.ID != "_id" {
if metadataDest.Type == "object" {
valueObject := toolkit.M{}
for _, eachMetaSub := range metadataDest.Sub {
valueObject.Set(eachMetaSub.ID, nil)
}
nilFieldDest.Set(metadataDest.ID, valueObject)
} else if metadataDest.Type == "array-objects" {
valueEachSourceField := each.Get(metadataDest.ID)
sourceObjects, _ := valueEachSourceField.([]interface{})
if sourceObjects == nil {
sourceObjects = []interface{}{}
}
valueObjects := []interface{}{}
for _, sourceObjectRaw := range sourceObjects {
sourceObject, _ := toolkit.ToM(sourceObjectRaw)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for keyss, _ := range sourceObject {
valueObject.Set(keyss, nil)
}
valueObjects = append(valueObjects, valueObject)
}
nilFieldDest.Set(metadataDest.ID, valueObjects)
} else {
if strings.Contains(metadataDest.Type, "array") {
valueObjects := []interface{}{}
nilFieldDest.Set(metadataDest.ID, valueObjects)
} else {
nilFieldDest.Set(metadataDest.ID, nil)
}
}
}
}
}
tableName := dsDestination.QueryInfo.GetString("from")
queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
if dataGrabber.InsertMode == "fresh" {
queryWrapper.Delete(tableName, dbox.Or())
}
if eachTransformedData.Has("_id") {
err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.Get("_id")))
}
if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, connDesc.Driver) && strings.HasPrefix(connDesc.Host, "http") {
queryWrapper = helper.Query(connDesc.Driver, connDesc.FileLocation, "", "", "", connDesc.Settings)
} else {
queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
}
if !nilFieldDest.Has("_id") || nilFieldDest.Get("_id") == nil || nilFieldDest.GetString("_id") == "<nil>" {
nilFieldDest.Set("_id", helper.RandomIDWithPrefix(""))
}
err = queryWrapper.Save(tableName, nilFieldDest)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
// ================ post transfer command
if dataGrabber.PostTransferCommand != "" {
eachTransformedData = dataToSave
jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
if err != nil {
return false, nil, err.Error()
}
jsonTranformedData := string(jsonTranformedDataBytes)
var postCommand = dataGrabber.PostTransferCommand
if strings.Contains(dataGrabber.PostTransferCommand, FLAG_ARG_DATA) {
postCommand = strings.TrimSpace(strings.Replace(dataGrabber.PostTransferCommand, FLAG_ARG_DATA, "", -1))
}
output, err := toolkit.RunCommand(postCommand, jsonTranformedData)
fmt.Printf("===> Post Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", postCommand, jsonTranformedData, output)
}
}
message = fmt.Sprintf("===> Success transforming %v data", len(transformedData))
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
return true, transformedData, ""
}
func main() {
var err error
var datastring string
var datatemp []interface{}
var logs interface{}
container := toolkit.M{}
dataset := make([]toolkit.M, 0, 0)
// fReadType := flag.String("readtype", "", "read type sedotan file") //snapshot,history,rechistory,logfile,[daemonlog]
// fLocation := flag.String("pathfile", "", "Full path file location include filename and pattern") //support using environment variable EC_DATA_PATH
// fDateTime := flag.String("datetime", "", "Date time for log file")
// fTake := flag.Int("take", 0, "take for limit data")
// fSkip := flag.Int("skip", 0, "skip for limit data")
flag.Parse()
tReadType := toolkit.ToString(*fReadType)
tLocation = toolkit.ToString(*fLocation)
tNameid = toolkit.ToString(*fNameid)
tDateTime = toolkit.ToString(*fDateTime)
tDatas = toolkit.ToString(*fDatas)
//snapshot,history,rechistory,logfile,[daemonlog]
//=========== Parse other flag ===========
// HERE
//========================================
switch tReadType {
case "snapshot":
module := GetDirSnapshot(tNameid)
SnapShot, err := module.OpenSnapShot(tNameid)
if err != nil {
fmt.Sprintf("ERROR: %s", err)
}
datastring = toolkit.JsonString(SnapShot)
err = toolkit.UnjsonFromString(datastring, &dataset)
container.Set("DATA", dataset)
case "history":
module := NewHistory(tLocation)
datatemp, err = module.OpenHistory()
if err != nil {
fmt.Sprintf("ERROR: %s", err)
}
datastring = toolkit.JsonString(datatemp)
err = toolkit.UnjsonFromString(datastring, &dataset)
container.Set("DATA", dataset)
case "rechistory":
var data []toolkit.M
config := toolkit.M{"useheader": true, "delimiter": ","}
query := helper.Query("csv", tLocation, "", "", "", config)
data, err = query.SelectAll("")
container.Set("DATA", data)
case "logfile":
history := NewHistory(tNameid)
err = toolkit.UnjsonFromString(tDatas, &datatemp)
logs = history.GetLogHistory(datatemp, tDateTime)
container.Set("DATA", logs)
case "daemonlog":
// logs, err := GetLogs()
// if err != nil {
// fmt.Sprintf("ERROR: %s", err)
// }
// datatemp = append(datatemp,logs)
// datastring = toolkit.JsonString(datatemp)
// container.Set("DATA", datastring)
container.Set("DATA", dataset)
err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format"))
default:
container.Set("DATA", dataset)
err = errors.New(fmt.Sprintf("-readtype cannot empty or get wrong format"))
}
container.Set("ERROR", err)
outputstring := toolkit.JsonString(container)
fmt.Printf("%s", outputstring)
}