Exemplo n.º 1
0
func (d *DataSourceController) TestConnection(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	payload := map[string]interface{}{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	driver := payload["Driver"].(string)
	host := payload["Host"].(string)
	database := payload["Database"].(string)
	username := payload["UserName"].(string)
	password := payload["Password"].(string)
	var settings toolkit.M = nil

	if settingsRaw := d.parseSettings(payload["Settings"], nil); settingsRaw != nil {
		settings, err = toolkit.ToM(settingsRaw)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}
	}

	if err := d.checkIfDriverIsSupported(driver); err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	fakeDataConn := &colonycore.Connection{
		Database: database,
		Driver:   driver,
		Host:     host,
		UserName: username,
		Password: password,
		Settings: settings,
	}

	if driver == "json" || driver == "csv" {
		if strings.HasPrefix(host, "http") {
			fileTempID := helper.RandomIDWithPrefix("f")
			fileType := helper.GetFileExtension(host)
			fakeDataConn.FileLocation = fmt.Sprintf("%s.%s", filepath.Join(EC_DATA_PATH, "datasource", "upload", fileTempID), fileType)

			file, err := os.Create(fakeDataConn.FileLocation)
			if err != nil {
				os.Remove(fakeDataConn.FileLocation)
				return helper.CreateResult(false, nil, err.Error())
			}
			defer file.Close()

			resp, err := http.Get(host)
			if err != nil {
				os.Remove(fakeDataConn.FileLocation)
				return helper.CreateResult(false, nil, err.Error())
			}
			defer resp.Body.Close()

			_, err = io.Copy(file, resp.Body)
			if err != nil {
				os.Remove(fakeDataConn.FileLocation)
				return helper.CreateResult(false, nil, err.Error())
			}
		} else {
			fakeDataConn.FileLocation = host
		}
	}

	err = helper.ConnectUsingDataConn(fakeDataConn).CheckIfConnected()

	if fakeDataConn.FileLocation != "" && strings.HasPrefix(host, "http") {
		os.Remove(fakeDataConn.FileLocation)
	}

	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	return helper.CreateResult(true, nil, "")
}
Exemplo n.º 2
0
func (d *DataGrabberController) Transform(dataGrabber *colonycore.DataGrabber) (bool, []toolkit.M, string) {

	logConf, err := d.getLogger(dataGrabber)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")
		defer logConf.Close()
	}

	message := fmt.Sprintf("===> Transformation started! %s -> %s interval %d %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination, dataGrabber.GrabInterval, dataGrabber.IntervalType)
	logConf.AddLog(message, "SUCCESS")
	fmt.Println(message)

	dsOrigin := new(colonycore.DataSource)
	err = colonycore.Get(dsOrigin, dataGrabber.DataSourceOrigin)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	dsDestination := new(colonycore.DataSource)
	err = colonycore.Get(dsDestination, dataGrabber.DataSourceDestination)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	dataDS, _, conn, query, metaSave, err := new(DataSourceController).
		ConnectToDataSource(dataGrabber.DataSourceOrigin)
	if len(dataDS.QueryInfo) == 0 {
		message := "Data source origin has invalid query"
		logConf.AddLog(message, "ERROR")

		return false, nil, message
	}
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}
	defer conn.Close()

	if metaSave.keyword != "" {
		message := `Data source origin query is not "Select"`
		logConf.AddLog(message, "ERROR")

		return false, nil, message
	}

	cursor, err := query.Cursor(nil)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}
	defer cursor.Close()

	data := []toolkit.M{}
	err = cursor.Fetch(&data, 0, false)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	connDesc := new(colonycore.Connection)
	err = colonycore.Get(connDesc, dsDestination.ConnectionID)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")

		return false, nil, err.Error()
	}

	const FLAG_ARG_DATA string = `%1`
	transformedData := []toolkit.M{}

	for _, each := range data {
		eachTransformedData := toolkit.M{}

		for _, eachMap := range dataGrabber.Maps {
			var valueEachSourceField interface{}

			// ============================================ SOURCE
			if !strings.Contains(eachMap.Source, "|") {
				// source could be: field, object, array
				valueEachSourceField = each.Get(eachMap.Source)
			} else {
				// source could be: field of object, field of array-objects
				prev := strings.Split(eachMap.Source, "|")[0]
				next := strings.Split(eachMap.Source, "|")[1]

				var fieldInfoDes *colonycore.FieldInfo = nil
				for _, eds := range dsOrigin.MetaData {
					if eds.ID == prev {
						fieldInfoDes = eds
						break
					}
				}

				if fieldInfoDes != nil {
					// source is field of array-objects
					if fieldInfoDes.Type == "array-objects" {
						valueObjects := []interface{}{}
						if temp, _ := each.Get(prev, nil).([]interface{}); temp != nil {
							valueObjects = make([]interface{}, len(temp))
							for i, each := range temp {
								if tempSub, _ := toolkit.ToM(each); tempSub != nil {
									valueObjects[i] = tempSub.Get(next)
								}
							}
						}
						valueEachSourceField = valueObjects
					} else {
						// source is field of object
						valueObject := toolkit.M{}
						if valueObject, _ = toolkit.ToM(each.Get(prev)); valueObject != nil {
							valueEachSourceField = valueObject.Get(next)
						}
					}
				}
			}

			// ============================================ DESTINATION
			if !strings.Contains(eachMap.Destination, "|") {
				if eachMap.SourceType == "object" {
					sourceObject, _ := toolkit.ToM(valueEachSourceField)
					if sourceObject == nil {
						sourceObject = toolkit.M{}
					}

					valueObject := toolkit.M{}
					for _, desMeta := range dsDestination.MetaData {
						if desMeta.ID == eachMap.Destination {
							for _, eachMetaSub := range desMeta.Sub {
								// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
								valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
							}
							break
						}
					}

					eachTransformedData.Set(eachMap.Destination, valueObject)
				} else if eachMap.SourceType == "array-objects" {
					sourceObjects, _ := valueEachSourceField.([]interface{})
					if sourceObjects == nil {
						sourceObjects = []interface{}{}
					}

					valueObjects := []interface{}{}
					for _, sourceObjectRaw := range sourceObjects {
						sourceObject, _ := toolkit.ToM(sourceObjectRaw)
						if sourceObject == nil {
							sourceObject = toolkit.M{}
						}

						valueObject := toolkit.M{}
						for _, desMeta := range dsDestination.MetaData {
							if desMeta.ID == eachMap.Destination {
								for _, eachMetaSub := range desMeta.Sub {
									// valueObject.Set(eachMetaSub.ID, sourceObject.Get(eachMetaSub.ID))
									valueObject.Set(eachMetaSub.ID, d.convertTo(sourceObject.Get(eachMetaSub.ID), eachMap.DestinationType))
								}
								break
							}
						}

						valueObjects = append(valueObjects, valueObject)
					}

					eachTransformedData.Set(eachMap.Destination, valueObjects)
				} else {
					if strings.Contains(eachMap.DestinationType, "array") {
						valueObjects := each.Get(eachMap.Source)

						eachTransformedData.Set(eachMap.Destination, valueObjects)
					} else {
						// eachTransformedData.Set(eachMap.Destination, convertDataType(eachMap.DestinationType, eachMap.Source, each))
						eachTransformedData.Set(eachMap.Destination, d.convertTo(each.Get(eachMap.Source), eachMap.DestinationType))
					}
				}
			} else {
				prev := strings.Split(eachMap.Destination, "|")[0]
				next := strings.Split(eachMap.Destination, "|")[1]

				var fieldInfoDes *colonycore.FieldInfo = nil
				for _, eds := range dsDestination.MetaData {
					if eds.ID == prev {
						fieldInfoDes = eds
						break
					}
				}

				if fieldInfoDes != nil {
					if fieldInfoDes.Type == "array-objects" {
						valueObjects := []interface{}{}
						if temp := eachTransformedData.Get(prev, nil); temp == nil {
							valueObjects = []interface{}{}
						} else {
							valueObjects, _ = temp.([]interface{})
							if valueObjects == nil {
								valueObjects = []interface{}{}
							}
						}

						if temp, _ := valueEachSourceField.([]interface{}); temp != nil {
							for i, eachVal := range temp {
								valueObject := toolkit.M{}
								if len(valueObjects) > i {
									if temp2, _ := toolkit.ToM(valueObjects[i]); temp2 != nil {
										valueObject = temp2
										// valueObject.Set(next, eachVal)
										valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
									}

									valueObjects[i] = valueObject
								} else {
									if fieldInfoDes.Sub != nil {
										for _, subMeta := range fieldInfoDes.Sub {
											valueObject.Set(subMeta.ID, nil)
										}
									}

									// valueObject.Set(next, eachVal)
									valueObject.Set(next, d.convertTo(eachVal, eachMap.DestinationType))
									valueObjects = append(valueObjects, valueObject)
								}
							}
						}

						eachTransformedData.Set(prev, valueObjects)
					} else {
						valueObject, _ := toolkit.ToM(eachTransformedData.Get(prev))
						if valueObject == nil {
							valueObject = toolkit.M{}
						}

						//tambahan
						prevSource := strings.Split(eachMap.Source, "|")[0]
						nextSource := strings.Split(eachMap.Source, "|")[1]
						mval, _ := toolkit.ToM(each.Get(prevSource, nil))

						//=========
						valueObject.Set(next, d.convertTo(mval.Get(nextSource), eachMap.DestinationType))
						// valueObject.Set(next, convertDataType(eachMap.DestinationType, nextSource, mval))
						eachTransformedData.Set(prev, valueObject)
					}
				}
			}
		}

		transformedData = append(transformedData, eachTransformedData)
		dataToSave := eachTransformedData

		// ================ pre transfer command
		if dataGrabber.PreTransferCommand != "" {
			// jsonTranformedDataBytes, err := json.Marshal(each)
			jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
			if err != nil {
				return false, nil, err.Error()
			}
			jsonTranformedData := string(jsonTranformedDataBytes)

			var preCommand = dataGrabber.PreTransferCommand
			if strings.Contains(dataGrabber.PreTransferCommand, FLAG_ARG_DATA) {
				preCommand = strings.TrimSpace(strings.Replace(dataGrabber.PreTransferCommand, FLAG_ARG_DATA, "", -1))
			}

			dataToSave = toolkit.M{}

			output, err := toolkit.RunCommand(preCommand, jsonTranformedData)
			fmt.Printf("===> Pre Transfer Command Result\n  COMMAND -> %s %s\n  OUTPUT  -> %s\n", preCommand, jsonTranformedData, output)
			if err == nil {
				postData := toolkit.M{}
				if err := json.Unmarshal([]byte(output), &postData); err == nil {
					dataToSave = postData
				}
			}
		}
		// ================

		if len(dataToSave) == 0 {
			continue
		}

		nilFieldDest := eachTransformedData
		for _, metadataDest := range dsDestination.MetaData {
			if temp := eachTransformedData.Get(metadataDest.ID); temp == nil {
				if metadataDest.ID != "_id" {
					if metadataDest.Type == "object" {
						valueObject := toolkit.M{}

						for _, eachMetaSub := range metadataDest.Sub {
							valueObject.Set(eachMetaSub.ID, nil)
						}
						nilFieldDest.Set(metadataDest.ID, valueObject)
					} else if metadataDest.Type == "array-objects" {
						valueEachSourceField := each.Get(metadataDest.ID)

						sourceObjects, _ := valueEachSourceField.([]interface{})
						if sourceObjects == nil {
							sourceObjects = []interface{}{}
						}

						valueObjects := []interface{}{}
						for _, sourceObjectRaw := range sourceObjects {
							sourceObject, _ := toolkit.ToM(sourceObjectRaw)

							if sourceObject == nil {
								sourceObject = toolkit.M{}
							}
							valueObject := toolkit.M{}
							for keyss, _ := range sourceObject {
								valueObject.Set(keyss, nil)
							}

							valueObjects = append(valueObjects, valueObject)
						}

						nilFieldDest.Set(metadataDest.ID, valueObjects)
					} else {
						if strings.Contains(metadataDest.Type, "array") {
							valueObjects := []interface{}{}

							nilFieldDest.Set(metadataDest.ID, valueObjects)
						} else {
							nilFieldDest.Set(metadataDest.ID, nil)
						}
					}
				}
			}
		}

		tableName := dsDestination.QueryInfo.GetString("from")
		queryWrapper := helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
		if dataGrabber.InsertMode == "fresh" {
			queryWrapper.Delete(tableName, dbox.Or())
		}
		if eachTransformedData.Has("_id") {
			err = queryWrapper.Delete(tableName, dbox.Eq("_id", eachTransformedData.Get("_id")))
		}

		if toolkit.HasMember([]string{"json", "jsons", "csv", "csvs"}, connDesc.Driver) && strings.HasPrefix(connDesc.Host, "http") {
			queryWrapper = helper.Query(connDesc.Driver, connDesc.FileLocation, "", "", "", connDesc.Settings)
		} else {
			queryWrapper = helper.Query(connDesc.Driver, connDesc.Host, connDesc.Database, connDesc.UserName, connDesc.Password, connDesc.Settings)
		}

		if !nilFieldDest.Has("_id") || nilFieldDest.Get("_id") == nil || nilFieldDest.GetString("_id") == "<nil>" {
			nilFieldDest.Set("_id", helper.RandomIDWithPrefix(""))
		}

		err = queryWrapper.Save(tableName, nilFieldDest)
		if err != nil {
			logConf.AddLog(err.Error(), "ERROR")

			return false, nil, err.Error()
		}

		// ================ post transfer command
		if dataGrabber.PostTransferCommand != "" {
			eachTransformedData = dataToSave
			jsonTranformedDataBytes, err := json.Marshal(eachTransformedData)
			if err != nil {
				return false, nil, err.Error()
			}
			jsonTranformedData := string(jsonTranformedDataBytes)

			var postCommand = dataGrabber.PostTransferCommand
			if strings.Contains(dataGrabber.PostTransferCommand, FLAG_ARG_DATA) {
				postCommand = strings.TrimSpace(strings.Replace(dataGrabber.PostTransferCommand, FLAG_ARG_DATA, "", -1))
			}

			output, err := toolkit.RunCommand(postCommand, jsonTranformedData)
			fmt.Printf("===> Post Transfer Command Result\n  COMMAND -> %s %s\n  OUTPUT  -> %s\n", postCommand, jsonTranformedData, output)
		}
	}

	message = fmt.Sprintf("===> Success transforming %v data", len(transformedData))
	logConf.AddLog(message, "SUCCESS")
	fmt.Println(message)

	return true, transformedData, ""
}