func (d *DataSourceController) TestConnection(r *knot.WebContext) interface{} {
r.Config.OutputType = knot.OutputJson
payload := map[string]interface{}{}
err := r.GetPayload(&payload)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
driver := payload["Driver"].(string)
host := payload["Host"].(string)
database := payload["Database"].(string)
username := payload["UserName"].(string)
password := payload["Password"].(string)
var settings toolkit.M = nil
if settingsRaw := d.parseSettings(payload["Settings"], nil); settingsRaw != nil {
settings, err = toolkit.ToM(settingsRaw)
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
}
if err := d.checkIfDriverIsSupported(driver); err != nil {
return helper.CreateResult(false, nil, err.Error())
}
fakeDataConn := &colonycore.Connection{
Database: database,
Driver: driver,
Host: host,
UserName: username,
Password: password,
Settings: settings,
}
if driver == "json" || driver == "csv" {
if strings.HasPrefix(host, "http") {
fileTempID := helper.RandomIDWithPrefix("f")
fileType := helper.GetFileExtension(host)
fakeDataConn.FileLocation = fmt.Sprintf("%s.%s", filepath.Join(EC_DATA_PATH, "datasource", "upload", fileTempID), fileType)
file, err := os.Create(fakeDataConn.FileLocation)
if err != nil {
os.Remove(fakeDataConn.FileLocation)
return helper.CreateResult(false, nil, err.Error())
}
defer file.Close()
resp, err := http.Get(host)
if err != nil {
os.Remove(fakeDataConn.FileLocation)
return helper.CreateResult(false, nil, err.Error())
}
defer resp.Body.Close()
_, err = io.Copy(file, resp.Body)
if err != nil {
os.Remove(fakeDataConn.FileLocation)
return helper.CreateResult(false, nil, err.Error())
}
} else {
fakeDataConn.FileLocation = host
}
}
err = helper.ConnectUsingDataConn(fakeDataConn).CheckIfConnected()
if fakeDataConn.FileLocation != "" && strings.HasPrefix(host, "http") {
os.Remove(fakeDataConn.FileLocation)
}
if err != nil {
return helper.CreateResult(false, nil, err.Error())
}
return helper.CreateResult(true, nil, "")
}
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) {
logConf, err := d.getLogger(dataGrabber)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
defer logConf.Close()
}
message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType)
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
dsOrigin := new(colonycore.DataSource)
err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dsDestination := new(colonycore.DataSource)
err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
dataDS, _, conn, query, metaSave, err := new(DataSourceController).
ConnectToDataSource(dataGrabber.DataSourceOrigin)
if len(dataDS.QueryInfo) == 0 {
message := "Data source origin has invalid query"
logConf.AddLog(message, "ERROR")
return false, nil, message
}
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer conn.Close()
if metaSave.keyword != "" {
message := `Data source origin query is not "Select"`
logConf.AddLog(message, "ERROR")
return false, nil, message
}
cursor, err := query.Cursor(nil)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
defer cursor.Close()
data := []toolkit.M{}
err = cursor.Fetch(&data, 0, false)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
connDesc := new(colonycore.Connection)
err = colonycore.Get(connDesc, dsDestination.ConnectionID)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
const FLAG_ARG_DATA string = `%1`
transformedData := []toolkit.M{}
for _, each := range data {
eachTransformedData := toolkit.M{}
for _, eachMap := range dataGrabber.Maps {
var valueEachSourceField interface{}
// ============================================ SOURCE
if !strings.Contains(eachMap.Source, "|") {
// source could be: field, object, array
valueEachSourceField = each.Get(eachMap.Source)
} else {
// source could be: field of object, field of array-objects
prev := strings.Split(eachMap.Source, "|")[0]
next := strings.Split(eachMap.Source, "|")[1]
var fieldInfoDes *colonycore.FieldInfo = nil
for _, eds := range dsOrigin.MetaData {
if eds.ID == prev {
fieldInfoDes = eds
break
}
}
if fieldInfoDes != nil {
// source is field of array-objects
if fieldInfoDes.Type == "array-objects" {
valueObjects := []interface{}{}
if temp, _ := each.Get(prev, nil).([]interface{}); temp != nil {
valueObjects = make([]interface{}, len(temp))
for i, each := range temp {
if tempSub, _ := toolkit.ToM(each); tempSub != nil {
valueObjects[i] = tempSub.Get(next)
}
}
}
valueEachSourceField = valueObjects
} else {
// source is field of object
valueObject := toolkit.M{}
if valueObject, _ = toolkit.ToM(each.Get(prev)); valueObject != nil {
valueEachSourceField = valueObject.Get(next)
}
}
}
}
// ============================================ DESTINATION
if !strings.Contains(eachMap.Destination, "|") {
if eachMap.SourceType == "object" {
sourceObject, _ := toolkit.ToM(valueEachSourceField)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for _, desMeta := range dsDestination.MetaData {
if desMeta.ID == eachMap.Destination {
for _, eachMetaSub := range desMeta.Sub {
// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
}
break
}
}
eachTransformedData.Set(eachMap.Destination, valueObject)
} else if eachMap.SourceType == "array-objects" {
sourceObjects, _ := valueEachSourceField.([]interface{})
if sourceObjects == nil {
sourceObjects = []interface{}{}
}
valueObjects := []interface{}{}
for _, sourceObjectRaw := range sourceObjects {
sourceObject, _ := toolkit.ToM(sourceObjectRaw)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for _, desMeta := range dsDestination.MetaData {
if desMeta.ID == eachMap.Destination {
for _, eachMetaSub := range desMeta.Sub {
// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
}
break
}
}
valueObjects = append(valueObjects, valueObject)
}
eachTransformedData.Set(eachMap.Destination, valueObjects)
} else {
if strings.Contains(eachMap.DestinationType, "array") {
valueObjects := each.Get(eachMap.Source)
eachTransformedData.Set(eachMap.Destination, valueObjects)
} else {
// eachTransformedData.Set(eachMap.Destination, convertDataType(eachMap.DestinationType, eachMap.Source, each))
eachTransformedData.Set(eachMap.Destination, d.convertTo(each.Get(eachMap.Source), eachMap.DestinationType))
}
}
} else {
prev := strings.Split(eachMap.Destination, "|")[0]
next := strings.Split(eachMap.Destination, "|")[1]
var fieldInfoDes *colonycore.FieldInfo = nil
for _, eds := range dsDestination.MetaData {
if eds.ID == prev {
fieldInfoDes = eds
break
}
}
if fieldInfoDes != nil {
if fieldInfoDes.Type == "array-objects" {
valueObjects := []interface{}{}
if temp := eachTransformedData.Get(prev, nil); temp == nil {
valueObjects = []interface{}{}
} else {
valueObjects, _ = temp.([]interface{})
if valueObjects == nil {
valueObjects = []interface{}{}
}
}
if temp, _ := valueEachSourceField.([]interface{}); temp != nil {
for i, eachVal := range temp {
valueObject := toolkit.M{}
if len(valueObjects) > i {
if temp2, _ := toolkit.ToM(valueObjects[i]); temp2 != nil {
valueObject = temp2
// valueObject.Set(next, eachVal)
valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
}
valueObjects[i] = valueObject
} else {
if fieldInfoDes.Sub != nil {
for _, subMeta := range fieldInfoDes.Sub {
valueObject.Set(subMeta.ID, nil)
}
}
// valueObject.Set(next, eachVal)
valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
valueObjects = append(valueObjects, valueObject)
}
}
}
eachTransformedData.Set(prev, valueObjects)
} else {
valueObject, _ := toolkit.ToM(eachTransformedData.Get(prev))
if valueObject == nil {
valueObject = toolkit.M{}
}
//tambahan
prevSource := strings.Split(eachMap.Source, "|")[0]
nextSource := strings.Split(eachMap.Source, "|")[1]
mval, _ := toolkit.ToM(each.Get(prevSource, nil))
//=========
valueObject.Set(next, d.convertTo(mval.Get(nextSource), eachMap.DestinationType))
// valueObject.Set(next, convertDataType(eachMap.DestinationType, nextSource, mval))
eachTransformedData.Set(prev, valueObject)
}
}
}
}
transformedData = append(transformedData, eachTransformedData)
dataToSave := eachTransformedData
// ================ pre transfer command
if dataGrabber.PreTransferCommand != "" {
// jsonTranformedDataBytes, err := json.Marshal(each)
jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
if err != nil {
return false, nil, err.Error()
}
jsonTranformedData := string(jsonTranformedDataBytes)
var preCommand = dataGrabber.PreTransferCommand
if strings.Contains(dataGrabber.PreTransferCommand, FLAG_ARG_DATA) {
preCommand = strings.TrimSpace(strings.Replace(dataGrabber.PreTransferCommand, FLAG_ARG_DATA, "", -1))
}
dataToSave = toolkit.M{}
output, err := toolkit.RunCommand(preCommand, jsonTranformedData)
fmt.Printf("===> Pre Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", preCommand, jsonTranformedData, output)
if err == nil {
postData := toolkit.M{}
if err := json.Unmarshal([]byte(output), &postData); err == nil {
dataToSave = postData
}
}
}
// ================
if len(dataToSave) == 0 {
continue
}
nilFieldDest := eachTransformedData
for _, metadataDest := range dsDestination.MetaData {
if temp := eachTransformedData.Get(metadataDest.ID); temp == nil {
if metadataDest.ID != "_id" {
if metadataDest.Type == "object" {
valueObject := toolkit.M{}
for _, eachMetaSub := range metadataDest.Sub {
valueObject.Set(eachMetaSub.ID, nil)
}
nilFieldDest.Set(metadataDest.ID, valueObject)
} else if metadataDest.Type == "array-objects" {
valueEachSourceField := each.Get(metadataDest.ID)
sourceObjects, _ := valueEachSourceField.([]interface{})
if sourceObjects == nil {
sourceObjects = []interface{}{}
}
valueObjects := []interface{}{}
for _, sourceObjectRaw := range sourceObjects {
sourceObject, _ := toolkit.ToM(sourceObjectRaw)
if sourceObject == nil {
sourceObject = toolkit.M{}
}
valueObject := toolkit.M{}
for keyss, _ := range sourceObject {
valueObject.Set(keyss, nil)
}
valueObjects = append(valueObjects, valueObject)
}
nilFieldDest.Set(metadataDest.ID, valueObjects)
} else {
if strings.Contains(metadataDest.Type, "array") {
valueObjects := []interface{}{}
nilFieldDest.Set(metadataDest.ID, valueObjects)
} else {
nilFieldDest.Set(metadataDest.ID, nil)
}
}
}
}
}
tableName := dsDestination.QueryInfo.GetString("from")
queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
if dataGrabber.InsertMode == "fresh" {
queryWrapper.Delete(tableName, dbox.Or())
}
if eachTransformedData.Has("_id") {
err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.Get("_id")))
}
if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, connDesc.Driver) && strings.HasPrefix(connDesc.Host, "http") {
queryWrapper = helper.Query(connDesc.Driver, connDesc.FileLocation, "", "", "", connDesc.Settings)
} else {
queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
}
if !nilFieldDest.Has("_id") || nilFieldDest.Get("_id") == nil || nilFieldDest.GetString("_id") == "<nil>" {
nilFieldDest.Set("_id", helper.RandomIDWithPrefix(""))
}
err = queryWrapper.Save(tableName, nilFieldDest)
if err != nil {
logConf.AddLog(err.Error(), "ERROR")
return false, nil, err.Error()
}
// ================ post transfer command
if dataGrabber.PostTransferCommand != "" {
eachTransformedData = dataToSave
jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
if err != nil {
return false, nil, err.Error()
}
jsonTranformedData := string(jsonTranformedDataBytes)
var postCommand = dataGrabber.PostTransferCommand
if strings.Contains(dataGrabber.PostTransferCommand, FLAG_ARG_DATA) {
postCommand = strings.TrimSpace(strings.Replace(dataGrabber.PostTransferCommand, FLAG_ARG_DATA, "", -1))
}
output, err := toolkit.RunCommand(postCommand, jsonTranformedData)
fmt.Printf("===> Post Transfer Command Result\n COMMAND -> %s %s\n OUTPUT -> %s\n", postCommand, jsonTranformedData, output)
}
}
message = fmt.Sprintf("===> Success transforming %v data", len(transformedData))
logConf.AddLog(message, "SUCCESS")
fmt.Println(message)
return true, transformedData, ""
}