Exemplo n.º 1
0
func generateWhereCondition(strwhere string) (oprstr string, sstr []string) {

	oprstr = "or"
	r := regexp.MustCompile(`^(?P<lastprocess01>(.*))(?P<firstprocess>(\(.*([Aa][Nn][Dd]|[Oo][Rr]).*\)))(?P<lastprocess02>(.*))$`)
	// ror := regexp.MustCompile(`(.*) (?P<oprandor>([Oo][Rr])) (.*)`)
	// rand := regexp.MustCompile(`(.*) (?P<oprandor>([Aa][Nn][Dd])) (.*)`)
	// if !r.MatchString(strwhere) {
	// 	r = ror
	// }

	// if !r.MatchString(strwhere) {
	// 	r = rand
	// }

	// tempalias := make(toolkit.M, 0, 0)
	tempalias := toolkit.M{}
	for r.MatchString(strwhere) {
		condpart := toolkit.M{}
		temparray := r.FindStringSubmatch(strwhere)

		for i, val := range r.SubexpNames() {
			if val != "" && temparray[i] != "" {
				condpart.Set(val, temparray[i])
			}
		}

		straliaskey := "@" + toolkit.GenerateRandomString("1234567890", 10)
		strwhere = strings.Replace(strwhere, condpart["firstprocess"].(string), straliaskey, -1)
		tempalias.Set(straliaskey, condpart["firstprocess"])
	}

	r = regexp.MustCompile(`(.*) (?P<oprandor>([Oo][Rr])) (.*)`)
	if !r.MatchString(strwhere) {
		r = regexp.MustCompile(`(.*) (?P<oprandor>([Aa][Nn][Dd])) (.*)`)
	}

	condpart := toolkit.M{}
	temparray := r.FindStringSubmatch(strwhere)
	// toolkit.Printf("Connection 319 : %#v\n", temparray)
	for i, val := range r.SubexpNames() {
		if val != "" && temparray[i] != "" {
			condpart.Set(val, temparray[i])
		}
	}

	sstr = strings.Split(strwhere, condpart["oprandor"].(string))
	if strings.ToLower(condpart["oprandor"].(string)) == "and" {
		oprstr = "and"
	}

	for key, val := range tempalias {
		for i, strval := range sstr {
			sstr[i] = strings.Replace(strval, key, val.(string), -1)
		}
	}

	return
}
Exemplo n.º 2
0
func getSessionTokenIdFromCookie(r *WebContext) string {
	tokenId := ""
	c, _ := r.Cookie(SessionCookieId(), "")
	if c == nil {
		tokenId = toolkit.GenerateRandomString("", 32)
		setCookieForSession(r, SessionCookieId(), tokenId, time.Hour*24*30)
	} else {
		tokenId = c.Value
	}
	return tokenId
}
Exemplo n.º 3
0
func getSessionTokenIdFromCookie(r *Request) string {
	tokenId := ""
	c, _ := r.Cookie(SessionCookieId())
	if c == nil {
		tokenId = toolkit.GenerateRandomString("", 32)
		r.SetCookie(SessionCookieId(), tokenId, time.Hour*24*30)
	} else {
		tokenId = c.Value
	}
	return tokenId
}
Exemplo n.º 4
0
func (a *Server) Secret() string {
	if a.secret == "" {
		a.secret = toolkit.GenerateRandomString("", 32)
	}
	return a.secret
}
Exemplo n.º 5
0
func doGenerateMaintenanceDataBrowser(wg *sync.WaitGroup, d *GenDataBrowser, plant PowerPlantCoordinates) {
	var e error
	ctx := d.BaseController.Ctx
	c := ctx.Connection

	saveDatas := []orm.IModel{}
	plantCodeStr := plant.PlantCode
	dataCount := 0

	tmpPlantCondition := conditions.Get(plantCodeStr)
	if tmpPlantCondition != nil {
		plantCondition := tmpPlantCondition.(tk.M)

		length := plantCondition.GetInt("length")
		dets := plantCondition.Get("det").([]tk.M)
		dets = append(dets, tk.M{})

		turbinesCodes := []string{}
		genIDTempTable := tk.GenerateRandomString("", 20)

		detsLen := len(dets) - 1

		tk.Printf("detsLen: %v \n", detsLen)

		for i, det := range dets {
			freeQuery := false
			tk.Println(i)
			tk.Printf("dets: %#v \n", det)
			assets := []FunctionalLocation{}
			systemAssets := []FunctionalLocation{}
			desc := det.GetString("desc")

			query := []*dbox.Filter{}
			query = append(query, dbox.Contains("FunctionalLocationCode", plantCodeStr))
			query = append(query, dbox.Eq("PIPI", plantCodeStr))

			queryStr := "select * from FunctionalLocation where FunctionalLocationCode like('%" + plantCodeStr + "%') and PIPI = '" + plantCodeStr + "' "

			if i == 0 {
				query = append(query, dbox.Contains("Description", desc))
				if plantCodeStr == "2220" {
					query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
				} else {
					query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
				}
			} else if i == 1 && detsLen == 2 {
				query = append(query, dbox.Contains("Description", desc))
				if plantCodeStr == "2220" {
					query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
				} else {
					query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
				}
			} else {
				tk.Printf("turbinesCodes: %v \n", len(turbinesCodes))
				if len(turbinesCodes) > 1 {
					maxLength := 1000
					maxLoop := tk.ToInt(tk.ToFloat64(len(turbinesCodes)/maxLength, 0, tk.RoundingUp), tk.RoundingUp)

					for i := 0; i <= maxLoop; i++ {
						datas := []string{}

						if i != maxLoop {
							datas = turbinesCodes[i*maxLength : (i*maxLength)+maxLength]

						} else {
							datas = turbinesCodes[i*maxLength:]
						}

						tmpNotIn := []orm.IModel{}
						for _, val := range datas {
							tmpData := new(GenDataBrowserNotInTmp)
							tmpData.ID = genIDTempTable
							tmpData.FLCode = val
							tmpNotIn = append(tmpNotIn, tmpData)
						}
						// tk.Printf("tmpNotIn: %v \n", len(tmpNotIn))
						e = ctx.InsertBulk(tmpNotIn)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
					}

					queryStr = queryStr + " and FunctionalLocationCode not in(select flcode from GenDataBrowserNotInTmp where ID = '" + genIDTempTable + "')"
					freeQuery = true
				}
			}

			if freeQuery {
				csrDet, e := c.NewQuery().Command("freequery", tk.M{}.Set("syntax", queryStr)).Cursor(nil)
				ErrorHandler(e, "generateMaintenanceDataBrowser")

				e = csrDet.Fetch(&assets, 0, false)
				ErrorHandler(e, "generateMaintenanceDataBrowser")
				csrDet.Close()
			} else {
				csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(dbox.And(query...)).Cursor(nil)
				ErrorHandler(e, "generateMaintenanceDataBrowser")

				e = csrDet.Fetch(&assets, 0, false)
				ErrorHandler(e, "generateMaintenanceDataBrowser")
				csrDet.Close()
			}

			tk.Printf("-- assets: %v \n", len(assets))

			if len(assets) > 0 {
				relatedAssets := []FunctionalLocation{}

				for _, asset := range assets {

					if plantCodeStr == "2110" {
						if len(asset.FunctionalLocationCode) <= 13 {
							query = []*dbox.Filter{}
							query = append(query,
								dbox.And(
									dbox.Eq("PG", "MP1"),
									dbox.Eq("PIPI", plantCodeStr),
									dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode),
									dbox.And(
										dbox.Lte("LEN(FunctionalLocationCode)", 13),
										dbox.Gte("LEN(FunctionalLocationCode)", 12),
									),
								),
							)

							csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							e = csrDet.Fetch(&systemAssets, 0, false)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							csrDet.Close()
						}

						if systemAssets == nil || len(systemAssets) == 0 {
							query = []*dbox.Filter{}

							if i != 2 {
								query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
							} else {
								query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
							}

							csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							e = csrDet.Fetch(&relatedAssets, 0, false)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							csrDet.Close()

							for _, relasset := range relatedAssets {
								isTurbineSystem := false
								if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
									isTurbineSystem = true
								}

								newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

								if newEquipment != "" {

									if i != 2 {
										turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
									}

									for _, year := range years {
										data := DataBrowser{}
										data.PeriodYear = year
										data.FunctionalLocation = relasset.FunctionalLocationCode
										data.FLDescription = relasset.Description
										data.IsTurbine = false

										// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)

										if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
											data.IsTurbine = true
											// tk.Println(" isTurbine: TRUE")
										} else {
											// tk.Println(" isTurbine: FALSE")
											data.TurbineParent = asset.FunctionalLocationCode
										}

										data.AssetType = "Other"

										if i == 0 {
											data.AssetType = "Steam"
										} else if i == 1 {
											data.AssetType = "Gas"
										}

										data.EquipmentType = newEquipment
										data.EquipmentTypeDescription = newEquipment
										data.Plant = plant
										data.PlantCode = plant.PlantCode

										if data.IsTurbine {
											info := GeneralInfo{}
											substr := ""
											substrValInt := 0
											if data.AssetType == "Steam" {
												substrValInt = 1
												substr = "ST"
											} else if data.AssetType == "Gas" {
												substrValInt = 2
											}

											if substrValInt != 0 {
												tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfo)
													substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
													return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
												}).Exec().Result.Data().([]GeneralInfo)
												if len(tmpInfo) > 0 {
													info = tmpInfo[0]
												}

												if info.Id != "" {
													data.TInfShortName = info.Unit
													data.TInfManufacturer = info.Manufacturer
													data.TInfModel = info.Model
													data.TInfUnitType = info.UnitType
													data.TInfInstalledCapacity = info.InstalledCapacity
													data.TInfOperationalCapacity = info.OperationalCapacity
													data.TInfPrimaryFuel = info.PrimaryFuel1
													data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
													data.TInfBackupFuel = info.BackupFuel
													data.TInfHeatRate = info.HeatRate
													data.TInfEfficiency = info.Efficiency

													commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
													ErrorHandler(e, "generateMaintenanceDataBrowser")
													data.TInfCommisioningDate = commDate

													if info.RetirementPlan != "" {
														retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
														retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
														ErrorHandler(e, "generateMaintenanceDataBrowser")
														data.TInfRetirementPlan = retirementPlan
													}

													installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfInstalledMWH = installedMWH.Value

													actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

													capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfCapacityFactor = capacityFactor.Value

													actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoActualFuelConsumption)
														return y.GenID == info.Id && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

													data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
													data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
													data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
													data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

													fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
														y := x.(FuelCost)
														unitID := strings.Replace(
															strings.Replace(
																strings.Replace(
																	strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

														return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
													})

													data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
														y := x.(FuelCost)
														return y.EnergyNetProduction
													}).Exec().Result.Data().(float64)

													data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
														y := x.(FuelCost)
														return y.PrimaryFuelConsumed
													}).Exec().Result.Data().(float64)
												}
											}
											// Vibrations handled by sql query
										}

										// Maintenance handled by sql query
										// FailureNotifications handled by sql query
										// MROElements handled by sql query
										// Operationals handled by sql query

										// e = ctx.Insert(&data)
										saveDatas = append(saveDatas, &data)
										dataCount++
										if len(saveDatas) == 1000 {
											mugen.Lock()
											e = ctx.InsertBulk(saveDatas)
											mugen.Unlock()
											ErrorHandler(e, "generateMaintenanceDataBrowser")
											tk.Printf("%v : %v \n", plantCodeStr, dataCount)

											saveDatas = []orm.IModel{}
										}

										/*ErrorHandler(e, "generateMaintenanceDataBrowser")
										tk.Println("save 1")*/
									}
								}
							}

						} else {
							// with system assets
							if i != 2 {
								turbinesCodes = append(turbinesCodes, asset.FunctionalLocationCode)
							}

							for _, year := range years {
								data := DataBrowser{}
								data.PeriodYear = year
								data.FunctionalLocation = asset.FunctionalLocationCode
								data.FLDescription = asset.Description
								data.IsTurbine = false
								data.IsSystem = false

								// tk.Printf("%v | %v | ", asset.FunctionalLocationCode, i)

								if asset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
									data.IsTurbine = true
									data.IsSystem = true
									// tk.Println(" isTurbine: TRUE")
								} else {
									// tk.Println(" isTurbine: FALSE")
									data.TurbineParent = asset.FunctionalLocationCode
									data.SystemParent = asset.FunctionalLocationCode
								}

								data.AssetType = "Other"

								if i == 0 {
									data.AssetType = "Steam"
								} else if i == 1 {
									data.AssetType = "Gas"
								}

								data.EquipmentType = "System"
								data.EquipmentTypeDescription = "System"
								data.Plant = plant
								data.PlantCode = plant.PlantCode

								if data.IsTurbine {
									info := GeneralInfo{}
									substr := ""
									substrValInt := 0
									if data.AssetType == "Steam" {
										substrValInt = 1
										substr = "ST"
									} else if data.AssetType == "Gas" {
										substrValInt = 2
									}

									if substrValInt != 0 {
										tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
											y := x.(GeneralInfo)
											substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
											return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
										}).Exec().Result.Data().([]GeneralInfo)
										if len(tmpInfo) > 0 {
											info = tmpInfo[0]
										}

										if info.Id != "" {
											data.TInfShortName = info.Unit
											data.TInfManufacturer = info.Manufacturer
											data.TInfModel = info.Model
											data.TInfUnitType = info.UnitType
											data.TInfInstalledCapacity = info.InstalledCapacity
											data.TInfOperationalCapacity = info.OperationalCapacity
											data.TInfPrimaryFuel = info.PrimaryFuel1
											data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
											data.TInfBackupFuel = info.BackupFuel
											data.TInfHeatRate = info.HeatRate
											data.TInfEfficiency = info.Efficiency

											commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
											ErrorHandler(e, "generateMaintenanceDataBrowser")
											data.TInfCommisioningDate = commDate

											if info.RetirementPlan != "" {
												retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
												retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
												ErrorHandler(e, "generateMaintenanceDataBrowser")
												data.TInfRetirementPlan = retirementPlan
											}

											installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfInstalledMWH = installedMWH.Value

											actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

											capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfCapacityFactor = capacityFactor.Value

											actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoActualFuelConsumption)
												return y.GenID == info.Id && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

											data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
											data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
											data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
											data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

											fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
												y := x.(FuelCost)
												unitID := strings.Replace(
													strings.Replace(
														strings.Replace(
															strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

												return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
											})

											data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
												y := x.(FuelCost)
												return y.EnergyNetProduction
											}).Exec().Result.Data().(float64)

											data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
												y := x.(FuelCost)
												return y.PrimaryFuelConsumed
											}).Exec().Result.Data().(float64)
										}
									}
									// Vibrations handled by sql query
								}

								// Maintenance handled by sql query
								// FailureNotifications handled by sql query
								// MROElements handled by sql query
								// Operationals handled by sql query

								// e = ctx.Insert(&data)
								// dataCount++
								saveDatas = append(saveDatas, &data)
								dataCount++
								if len(saveDatas) == 1000 {
									mugen.Lock()
									e = ctx.InsertBulk(saveDatas)
									mugen.Unlock()
									ErrorHandler(e, "generateMaintenanceDataBrowser")
									tk.Printf("%v : %v \n", plantCodeStr, dataCount)

									saveDatas = []orm.IModel{}
								}

								/*ErrorHandler(e, "generateMaintenanceDataBrowser")
								tk.Println("save 2")*/

							}

							for _, sysAsset := range systemAssets {
								query = []*dbox.Filter{}

								if i != 2 {
									query = append(query, dbox.Contains("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
								} else {
									query = append(query, dbox.Eq("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
								}

								csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
								ErrorHandler(e, "generateMaintenanceDataBrowser")
								e = csrDet.Fetch(&relatedAssets, 0, false)
								ErrorHandler(e, "generateMaintenanceDataBrowser")
								csrDet.Close()

								for _, relasset := range relatedAssets {
									isTurbineSystem := false
									if (relasset.FunctionalLocationCode == asset.FunctionalLocationCode || relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode) && i != 2 {
										isTurbineSystem = true
									}

									newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

									if newEquipment != "" {

										if i != 2 {
											turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
										}

										for _, year := range years {
											data := DataBrowser{}
											data.PeriodYear = year
											data.FunctionalLocation = relasset.FunctionalLocationCode
											data.FLDescription = relasset.Description
											data.IsTurbine = false
											data.IsSystem = false

											// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)

											if relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode && i != 2 {
												data.IsTurbine = true
												data.IsSystem = true
												// tk.Println(" isTurbine: TRUE")
											} else {
												// tk.Println(" isTurbine: FALSE")
												data.TurbineParent = asset.FunctionalLocationCode
												data.SystemParent = asset.FunctionalLocationCode
											}

											data.AssetType = "Other"

											if i == 0 {
												data.AssetType = "Steam"
											} else if i == 1 {
												data.AssetType = "Gas"
											}

											data.EquipmentType = newEquipment
											data.EquipmentTypeDescription = newEquipment
											data.Plant = plant
											data.PlantCode = plant.PlantCode

											if data.IsTurbine {
												info := GeneralInfo{}
												substr := ""
												substrValInt := 0
												if data.AssetType == "Steam" {
													substrValInt = 1
													substr = "ST"
												} else if data.AssetType == "Gas" {
													substrValInt = 2
												}

												if substrValInt != 0 {
													tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfo)
														substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
														return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
													}).Exec().Result.Data().([]GeneralInfo)
													if len(tmpInfo) > 0 {
														info = tmpInfo[0]
													}

													if info.Id != "" {
														data.TInfShortName = info.Unit
														data.TInfManufacturer = info.Manufacturer
														data.TInfModel = info.Model
														data.TInfUnitType = info.UnitType
														data.TInfInstalledCapacity = info.InstalledCapacity
														data.TInfOperationalCapacity = info.OperationalCapacity
														data.TInfPrimaryFuel = info.PrimaryFuel1
														data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
														data.TInfBackupFuel = info.BackupFuel
														data.TInfHeatRate = info.HeatRate
														data.TInfEfficiency = info.Efficiency

														commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
														ErrorHandler(e, "generateMaintenanceDataBrowser")
														data.TInfCommisioningDate = commDate

														if info.RetirementPlan != "" {
															retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
															retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
															ErrorHandler(e, "generateMaintenanceDataBrowser")
															data.TInfRetirementPlan = retirementPlan
														}

														installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfInstalledMWH = installedMWH.Value

														actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

														capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfCapacityFactor = capacityFactor.Value

														actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoActualFuelConsumption)
															return y.GenID == info.Id && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

														data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
														data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
														data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
														data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

														fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
															y := x.(FuelCost)
															unitID := strings.Replace(
																strings.Replace(
																	strings.Replace(
																		strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

															return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
														})

														data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
															y := x.(FuelCost)
															return y.EnergyNetProduction
														}).Exec().Result.Data().(float64)

														data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
															y := x.(FuelCost)
															return y.PrimaryFuelConsumed
														}).Exec().Result.Data().(float64)
													}
												}
												// Vibrations handled by sql query
											}

											// Maintenance handled by sql query
											// FailureNotifications handled by sql query
											// MROElements handled by sql query
											// Operationals handled by sql query

											// e = ctx.Insert(&data)
											saveDatas = append(saveDatas, &data)
											dataCount++
											if len(saveDatas) == 1000 {
												mugen.Lock()
												e = ctx.InsertBulk(saveDatas)
												mugen.Unlock()
												ErrorHandler(e, "generateMaintenanceDataBrowser")
												tk.Printf("%v : %v \n", plantCodeStr, dataCount)

												saveDatas = []orm.IModel{}
											}

											/*ErrorHandler(e, "generateMaintenanceDataBrowser")
											tk.Println("save 3")*/
										}
									}
								}
							}
						}
					} else {
						// another plant(s)

						query = []*dbox.Filter{}

						if i != detsLen {
							query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
						} else {
							query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
						}

						csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
						e = csrDet.Fetch(&relatedAssets, 0, false)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
						csrDet.Close()
						tk.Printf("-- related assets: %v \n", len(relatedAssets))

						for _, relasset := range relatedAssets {
							isTurbineSystem := false
							if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
								isTurbineSystem = true
							}

							newEquipment := ""
							newEquipment = d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

							if newEquipment != "" {

								if i != detsLen {
									turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
								}

								for _, year := range years {
									_ = year
									data := DataBrowser{}
									data.PeriodYear = year
									data.FunctionalLocation = relasset.FunctionalLocationCode
									data.FLDescription = relasset.Description
									data.IsTurbine = false

									if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
										data.IsTurbine = true
									} else {
										data.TurbineParent = asset.FunctionalLocationCode
									}

									data.AssetType = "Other"

									if i == 0 {
										data.AssetType = "Steam"
									} else if i == 1 && detsLen > 1 {
										data.AssetType = "Gas"
									}

									data.EquipmentType = newEquipment
									data.EquipmentTypeDescription = newEquipment
									data.Plant = plant
									data.PlantCode = plant.PlantCode

									if data.IsTurbine {
										info := GeneralInfo{}
										substr := ""
										substrValInt := 0
										if data.AssetType == "Steam" {
											// substrValInt = 1
											substrValInt = 2
											substr = "ST" + substr
										} else if data.AssetType == "Gas" {
											substrValInt = 2
											substr = "GT" + substr
										}

										if substrValInt != 0 {
											tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfo)
												substr = data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
												return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
											}).Exec().Result.Data().([]GeneralInfo)
											if len(tmpInfo) > 0 {
												info = tmpInfo[0]
											}

											if info.Id != "" {
												data.TInfShortName = info.Unit
												data.TInfManufacturer = info.Manufacturer
												data.TInfModel = info.Model
												data.TInfUnitType = info.UnitType
												data.TInfInstalledCapacity = info.InstalledCapacity
												data.TInfOperationalCapacity = info.OperationalCapacity
												data.TInfPrimaryFuel = info.PrimaryFuel1
												data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
												data.TInfBackupFuel = info.BackupFuel
												data.TInfHeatRate = info.HeatRate
												data.TInfEfficiency = info.Efficiency

												commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
												ErrorHandler(e, "generateMaintenanceDataBrowser")

												data.TInfCommisioningDate = commDate

												if info.RetirementPlan != "" {
													retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
													retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
													ErrorHandler(e, "generateMaintenanceDataBrowser")
													data.TInfRetirementPlan = retirementPlan
												}

												installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfInstalledMWH = installedMWH.Value

												actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

												capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfCapacityFactor = capacityFactor.Value

												actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoActualFuelConsumption)
													return y.GenID == info.Id && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

												data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
												data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
												data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
												data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

												fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
													y := x.(FuelCost)
													unitID := strings.Replace(
														strings.Replace(
															strings.Replace(
																strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

													return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
												})

												data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
													y := x.(FuelCost)
													return y.EnergyNetProduction
												}).Exec().Result.Data().(float64)

												data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
													y := x.(FuelCost)
													return y.PrimaryFuelConsumed
												}).Exec().Result.Data().(float64)
											}
										}
										// Vibrations handled by sql query
									}

									// Maintenance handled by sql query
									// FailureNotifications handled by sql query
									// MROElements handled by sql query
									// Operationals handled by sql query

									// e = ctx.Insert(&data)
									saveDatas = append(saveDatas, &data)
									dataCount++
									if len(saveDatas) == 1000 {
										mugen.Lock()
										e = ctx.InsertBulk(saveDatas)
										mugen.Unlock()
										ErrorHandler(e, "generateMaintenanceDataBrowser")
										tk.Printf("%v : %v \n", plantCodeStr, dataCount)

										saveDatas = []orm.IModel{}
									}

									/*ErrorHandler(e, "generateMaintenanceDataBrowser")
									tk.Println("save 4")*/
								}
							}
						}
					}

				}
			}
		}

		e = c.NewQuery().Delete().From(new(GenDataBrowserNotInTmp).TableName()).SetConfig("multiexec", true).Where(dbox.Eq("ID", genIDTempTable)).Exec(nil)

		if len(saveDatas) > 0 {
			e = ctx.InsertBulk(saveDatas)
			ErrorHandler(e, "generateMaintenanceDataBrowser")
			tk.Printf("%v : %v \n", plantCodeStr, dataCount)
		}

	}

	mugen.Lock()
	dataCounts.Set(plantCodeStr, dataCount)
	mugen.Unlock()

	wg.Done()
}
Exemplo n.º 6
0
func savedatagrab() (err error) {
	for key, _ := range wGrabber.Config.DataSettings {

		err = nil
		note := ""
		dt := toolkit.M{}.Set("datasettingname", key).Set("grabdate", thistime).Set("rowgrabbed", 0).
			Set("rowsaved", 0).Set("note", note).Set("grabstatus", "fail").Set("recfile", "")

		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] start save data", key), "INFO")
		docs := []toolkit.M{}
		err = wGrabber.ResultFromHtml(key, &docs)
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get data : %s", key, err.Error())
			Log.AddLog(note, "ERROR")
			dt = dt.Set("note", note)
			_ = savehistory(dt)
			continue
		}

		dt = dt.Set("rowgrabbed", len(docs))

		//Update Total Process
		mutex.Lock()
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] get snapshot for update total process", key), "INFO")
		err = getsnapshot()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}

		if pid == snapshotdata.Pid {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] update total process data : %v", key, len(docs)), "INFO")
			snapshotdata.Cgtotal += len(docs)
			err = savesnapshot()
		}

		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}
		mutex.Unlock()
		//==================== Split to multiexec with goroutine <<---

		err = destDboxs[key].IConnection.Connect()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to connect [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error())
			Log.AddLog(note, "ERROR")
			dt = dt.Set("note", note)
			_ = savehistory(dt)
			continue
		}

		q := destDboxs[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
		if destDboxs[key].collection != "" {
			q = q.From(destDboxs[key].collection)
		}

		iN := 0
		for _, doc := range docs {
			if destDboxs[key].desttype == "mongo" {
				doc["_id"] = toolkit.GenerateRandomString("", 32)
			}

			err = q.Exec(toolkit.M{
				"data": doc,
			})

			if err != nil {
				note = "Error in insert data"
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
				continue
			} else {
				iN += 1
			}

			if math.Mod(float64(iN), 100) == 0 {
				_ = updatesnapshot(iN, key)
				iN = 0
			}
		}

		dt = dt.Set("rowsaved", iN)
		_ = updatesnapshot(iN, key)
		filerec, err := saverechistory(key, docs)
		if err != nil {
			if note != "" {
				note = note + ";" + fmt.Sprintf("[savedatagrab.%s] Unable to save rec history : %s", key, err.Error())
			} else {
				note = fmt.Sprintf("[savedatagrab.%s] Unable to save rec history : %s", key, err.Error())
			}

			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to save rec history : %s", key, err.Error()), "ERROR")
			dt = dt.Set("note", note)
			_ = savehistory(dt)
			continue
		}

		dt = dt.Set("note", note).Set("grabstatus", "done").Set("recfile", filerec)
		err = savehistory(dt)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to save history : %s", key), "ERROR")
		}
		snapshotdata.Rowgrabbed += iN
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Finish save data", key), "INFO")
	}
	return
}
Exemplo n.º 7
0
func (g *GrabService) execService() {
	g.LastGrabStat = false
	go func(g *GrabService) {
		for g.ServiceRunningStat {

			if g.LastGrabStat {
				<-time.After(g.GrabInterval)
			} else {
				<-time.After(g.TimeOutInterval)
			}

			if !g.ServiceRunningStat {
				continue
			}

			g.ErrorNotes = ""
			g.LastGrabExe = time.Now()
			g.NextGrabExe = time.Now().Add(g.GrabInterval)
			g.LastGrabStat = true
			g.Log.AddLog(fmt.Sprintf("[%s] Grab Started %s", g.Name, g.Url), "INFO")
			g.GrabCount += 1

			keySetting := []string{}
			switch g.SourceType {
			case SourceType_HttpHtml, SourceType_HttpJson:
				if e := g.ServGrabber.Grab(nil); e != nil {
					g.ErrorNotes = fmt.Sprintf("[%s] Grab Failed %s, repeat after %s :%s", g.Name, g.Url, g.TimeOutIntervalInfo, e)
					g.Log.AddLog(g.ErrorNotes, "ERROR")
					g.NextGrabExe = time.Now().Add(g.TimeOutInterval)
					g.LastGrabStat = false
					g.ErrorFound += 1
				} else {
					g.Log.AddLog(fmt.Sprintf("[%s] Grab Success %s", g.Name, g.Url), "INFO")
				}
				for key, _ := range g.ServGrabber.Config.DataSettings {
					keySetting = append(keySetting, key)
				}
				// keySetting = g.sGrabber.Config.DataSettings
			case SourceType_DocExcel:
				// e = g.sGetData.ResultFromDatabase(key, &docs)
				// if e != nil {
				// 	g.LastGrabStat = false
				// }
				for key, _ := range g.ServGetData.CollectionSettings {
					keySetting = append(keySetting, key)
				}
			}

			// if e := g.ServGrabber.Grab(nil); e != nil {
			// 	g.ErrorNotes = fmt.Sprintf("[%s] Grab Failed %s, repeat after %s :%s", g.Name, g.Url, g.TimeOutIntervalInfo, e)
			// 	g.Log.AddLog(g.ErrorNotes, "ERROR")
			// 	g.NextGrabExe = time.Now().Add(g.TimeOutInterval)
			// 	g.LastGrabStat = false
			// 	g.ErrorFound += 1
			// 	continue
			// } else {
			// 	g.Log.AddLog(fmt.Sprintf("[%s] Grab Success %s", g.Name, g.Url), "INFO")
			// }

			if g.LastGrabStat {
				for _, key := range keySetting {
					var e error
					g.Log.AddLog(fmt.Sprintf("[%s-%s] Fetch Data to destination started", g.Name, key), "INFO")

					docs := []toolkit.M{}
					switch g.SourceType {
					case SourceType_HttpHtml, SourceType_HttpJson:
						e = g.ServGrabber.ResultFromHtml(key, &docs)
					case SourceType_DocExcel:
						e = g.ServGetData.ResultFromDatabase(key, &docs)
						if e != nil {
							g.LastGrabStat = false
						}
					}
					if e != nil || !(g.LastGrabStat) {
						g.ErrorNotes = fmt.Sprintf("[%s-%s] Fetch Result Failed : ", g.Name, key, e)
						g.Log.AddLog(g.ErrorNotes, "ERROR")
					}

					e = g.DestDbox[key].IConnection.Connect()
					if e != nil {
						g.ErrorNotes = fmt.Sprintf("[%s-%s] Connect to destination failed [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
						g.Log.AddLog(g.ErrorNotes, "ERROR")
					}

					var q dbox.IQuery
					if g.DestDbox[key].Collection == "" {
						q = g.DestDbox[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
					} else {
						q = g.DestDbox[key].IConnection.NewQuery().SetConfig("multiexec", true).From(g.DestDbox[key].Collection).Save()
					}
					xN := 0
					iN := 0
					for _, doc := range docs {
						for key, val := range doc {
							doc[key] = strings.TrimSpace(fmt.Sprintf("%s", val))
						}

						if g.DestDbox[key].Desttype == "mongo" {
							doc["_id"] = toolkit.GenerateRandomString("1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnpqrstuvwxyz", 32)
						}

						e = q.Exec(toolkit.M{
							"data": doc,
						})

						if g.DestDbox[key].Desttype == "mongo" {
							delete(doc, "_id")
						}

						if e != nil {
							g.ErrorNotes = fmt.Sprintf("[%s-%s] Unable to insert [%s-%s]:%s", g.Name, key, g.DestDbox[key].Desttype, g.DestDbox[key].IConnection.Info().Host, e)
							g.Log.AddLog(g.ErrorNotes, "ERROR")
							g.ErrorFound += 1
						} else {
							iN += 1
						}
						xN++
					}
					g.RowGrabbed += xN
					q.Close()
					g.DestDbox[key].IConnection.Close()

					g.Log.AddLog(fmt.Sprintf("[%s-%s] Fetch Data to destination finished with %d record fetch", g.Name, key, xN), "INFO")

					if g.HistoryPath != "" && g.HistoryRecPath != "" {
						recfile := g.AddRecHistory(key, docs)
						historyservice := toolkit.M{}.Set("datasettingname", key).Set("grabdate", g.LastGrabExe).Set("rowgrabbed", g.RowGrabbed).
							Set("rowsaved", iN).Set("note", g.ErrorNotes).Set("grabstatus", "SUCCESS").Set("recfile", recfile)
						if !(g.LastGrabStat) {
							historyservice.Set("grabstatus", "FAILED")
						}
						g.AddHistory(historyservice)
					}
				}
			} else {
				if g.HistoryPath != "" {
					historyservice := toolkit.M{}.Set("datasettingname", "-").Set("grabdate", g.LastGrabExe).Set("rowgrabbed", g.RowGrabbed).
						Set("rowsaved", 0).Set("note", g.ErrorNotes).Set("grabstatus", "FAILED").Set("recfile", "")
					g.AddHistory(historyservice)
				}
			}
		}
	}(g)
}
Exemplo n.º 8
0
func streamsavedata(intms <-chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
	var err error
	iN, note := 0, ""

	for intm := range intms {
		if destDboxs[key].desttype == "mongo" {
			intm.Set("_id", toolkit.GenerateRandomString("", 32))
		}

		if len(intm) == 0 {
			continue
		}
		//Pre Execute Program
		if extCommand.Has("pre") && toolkit.ToString(extCommand["pre"]) != "" {
			sintm := toolkit.JsonString(intm)
			arrcmd := make([]string, 0, 0)

			// if runtime.GOOS == "windows" {
			// 	arrcmd = append(arrcmd, "cmd")
			// 	arrcmd = append(arrcmd, "/C")
			// }

			arrcmd = append(arrcmd, toolkit.ToString(extCommand["pre"]))
			arrcmd = append(arrcmd, sintm)

			// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
			output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute pre external command :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}

			err = toolkit.UnjsonFromString(output, &intm)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get pre external command output :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}
		}

		err = sQ.Exec(toolkit.M{
			"data": intm,
		})

		if err != nil {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
			note = "Error Found"
			continue
		}

		err = saverechistory(key, intm)
		if err != nil {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to insert record data [%s-%s]:%s", key, "csv", destDboxs[key].IConnection.Info().Host, err.Error()), "ERROR")
			note = "Error Found"
		}

		iN += 1
		if math.Mod(float64(iN), 100) == 0 {
			_ = updatesnapshot(iN, key)
			dt = dt.Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
			iN = 0
		}

		//Post Execute Program
		if extCommand.Has("post") {
			sintm := toolkit.JsonString(intm)
			arrcmd := make([]string, 0, 0)

			// if runtime.GOOS == "windows" {
			// 	arrcmd = append(arrcmd, "cmd")
			// 	arrcmd = append(arrcmd, "/C")
			// }

			arrcmd = append(arrcmd, toolkit.ToString(extCommand["post"]))
			arrcmd = append(arrcmd, sintm)

			// output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1:])
			output, err := toolkit.RunCommand(arrcmd[0], arrcmd[1])
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to execute post external command :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}

			err = toolkit.UnjsonFromString(output, &intm)
			if err != nil {
				Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to get post external command output :%s", key, err.Error()), "ERROR")
				note = "Error Found"
				continue
			}
		}
	}
	dt = dt.Set("note", note).
		Set("grabstatus", "done").
		Set("rowsaved", (toolkit.ToInt(dt.Get("rowsaved", 0), toolkit.RoundingAuto) + iN))
	_ = updatesnapshot(iN, key)
	err = savehistory(dt)
	if err != nil {
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Unable to save history : %s", key), "ERROR")
	}
	Log.AddLog(fmt.Sprintf("[savedatagrab.%s] Finish save data", key), "INFO")
	destDboxs[key].IConnection.Close()
}