Пример #1
0
func TestFrom(t *testing.T) {
	for i := 0; i < sampleCount; i++ {
		obj := Obj{}
		obj.F = toolkit.ToFloat64(toolkit.RandInt(1000), 2, toolkit.RoundingAuto)
		obj.I = toolkit.RandInt(1000) + 5
		objs = append(objs, obj)

		if i == 0 {
			min = obj.I
			max = obj.I
			sum = obj.I
			avg = toolkit.ToFloat64(obj.I, 4, toolkit.RoundingAuto)
		} else {
			sum += obj.I
			avg = toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) /
				toolkit.ToFloat64(i+1, 4, toolkit.RoundingAuto)
			if min > obj.I {
				min = obj.I
			}
			if max < obj.I {
				max = obj.I
			}
		}
	}

	c = crowd.From(&objs)
	check(t, c.Error, "")
	toolkit.Printf("Data len: %d, max: %d, min: %d, sum: %d, avg: %5.4f\n", c.Len(), max, min, sum, avg)
}
Пример #2
0
func TestAggr(t *testing.T) {
	skipIfNil(t)
	c2 := c.Min(fn).Max(fn).Sum(fn).Avg(fn).Exec()
	check(t, c2.Error, "Aggr")
	if toolkit.ToInt(c2.Result.Min, toolkit.RoundingAuto) != min ||
		toolkit.ToInt(c2.Result.Max, toolkit.RoundingAuto) != max ||
		c2.Result.Sum != toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) ||
		c2.Result.Avg != avg {
		t.Fatalf("Error aggr. Got %v\n", toolkit.JsonString(c2.Result))
	}
	toolkit.Println("Value: ", toolkit.JsonString(c2.Result))
}
Пример #3
0
func main() {
	jsonarg := toolkit.M{}

	err := toolkit.UnjsonFromString(os.Args[1], &jsonarg)
	if err != nil {
		return
	}

	jsonarg["Date"] = "000000"
	jsonarg[`MB 62% Fe`] = "POST " + toolkit.ToString(jsonarg[`MB 62% Fe`])
	jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto) * 5)
	fmt.Println(toolkit.JsonString(jsonarg))
}
Пример #4
0
func main() {
	jsonarg := toolkit.M{}

	err := toolkit.UnjsonFromString(os.Args[1], &jsonarg)
	if err != nil {
		return
	}

	jsonarg["Date"] = time.Now().Format("020106")
	jsonarg[`MB 62% Fe`] = "FE " + toolkit.ToString(jsonarg[`MB 62% Fe`])
	jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto))
	fmt.Println(toolkit.JsonString(jsonarg))
}
Пример #5
0
func toInterface(data string, dataType string, dateFormat string) interface{} {
	if dataType == "" {
		if strings.HasPrefix(data, "#") && strings.HasSuffix(data, "#") {
			dataType = DataDate
		} else {
			vfloat := toolkit.ToFloat64(dataType, 2, toolkit.RoundingAuto)
			vint := toolkit.ToInt(dataType, toolkit.RoundingAuto)
			if int(vfloat) == vint && vint != 0 {
				dataType = DataInt
			} else if vfloat != 0 {
				// dataType = DataFloat
				b, i := toolkit.MemberIndex(DataFloats, dataType)
				if b {
					for idx, dataFloat := range DataFloats {
						if idx == i {
							dataType = dataFloat
						}
					}
				}
			} else {
				dataType = DataString
			}
		}
	}

	if dataType == DataDate {
		return toolkit.String2Date(data, dateFormat)
	} else if dataType == DataInt {
		return toolkit.ToInt(data, toolkit.RoundingAuto)
	} else if toolkit.HasMember(DataFloats, dataType) {
		return toolkit.ToFloat64(data, 2, toolkit.RoundingAuto)
	} else {
		return data
	}

	return nil
}
Пример #6
0
func doGenerateMaintenanceDataBrowser(wg *sync.WaitGroup, d *GenDataBrowser, plant PowerPlantCoordinates) {
	var e error
	ctx := d.BaseController.Ctx
	c := ctx.Connection

	saveDatas := []orm.IModel{}
	plantCodeStr := plant.PlantCode
	dataCount := 0

	tmpPlantCondition := conditions.Get(plantCodeStr)
	if tmpPlantCondition != nil {
		plantCondition := tmpPlantCondition.(tk.M)

		length := plantCondition.GetInt("length")
		dets := plantCondition.Get("det").([]tk.M)
		dets = append(dets, tk.M{})

		turbinesCodes := []string{}
		genIDTempTable := tk.GenerateRandomString("", 20)

		detsLen := len(dets) - 1

		tk.Printf("detsLen: %v \n", detsLen)

		for i, det := range dets {
			freeQuery := false
			tk.Println(i)
			tk.Printf("dets: %#v \n", det)
			assets := []FunctionalLocation{}
			systemAssets := []FunctionalLocation{}
			desc := det.GetString("desc")

			query := []*dbox.Filter{}
			query = append(query, dbox.Contains("FunctionalLocationCode", plantCodeStr))
			query = append(query, dbox.Eq("PIPI", plantCodeStr))

			queryStr := "select * from FunctionalLocation where FunctionalLocationCode like('%" + plantCodeStr + "%') and PIPI = '" + plantCodeStr + "' "

			if i == 0 {
				query = append(query, dbox.Contains("Description", desc))
				if plantCodeStr == "2220" {
					query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
				} else {
					query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
				}
			} else if i == 1 && detsLen == 2 {
				query = append(query, dbox.Contains("Description", desc))
				if plantCodeStr == "2220" {
					query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
				} else {
					query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
				}
			} else {
				tk.Printf("turbinesCodes: %v \n", len(turbinesCodes))
				if len(turbinesCodes) > 1 {
					maxLength := 1000
					maxLoop := tk.ToInt(tk.ToFloat64(len(turbinesCodes)/maxLength, 0, tk.RoundingUp), tk.RoundingUp)

					for i := 0; i <= maxLoop; i++ {
						datas := []string{}

						if i != maxLoop {
							datas = turbinesCodes[i*maxLength : (i*maxLength)+maxLength]

						} else {
							datas = turbinesCodes[i*maxLength:]
						}

						tmpNotIn := []orm.IModel{}
						for _, val := range datas {
							tmpData := new(GenDataBrowserNotInTmp)
							tmpData.ID = genIDTempTable
							tmpData.FLCode = val
							tmpNotIn = append(tmpNotIn, tmpData)
						}
						// tk.Printf("tmpNotIn: %v \n", len(tmpNotIn))
						e = ctx.InsertBulk(tmpNotIn)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
					}

					queryStr = queryStr + " and FunctionalLocationCode not in(select flcode from GenDataBrowserNotInTmp where ID = '" + genIDTempTable + "')"
					freeQuery = true
				}
			}

			if freeQuery {
				csrDet, e := c.NewQuery().Command("freequery", tk.M{}.Set("syntax", queryStr)).Cursor(nil)
				ErrorHandler(e, "generateMaintenanceDataBrowser")

				e = csrDet.Fetch(&assets, 0, false)
				ErrorHandler(e, "generateMaintenanceDataBrowser")
				csrDet.Close()
			} else {
				csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(dbox.And(query...)).Cursor(nil)
				ErrorHandler(e, "generateMaintenanceDataBrowser")

				e = csrDet.Fetch(&assets, 0, false)
				ErrorHandler(e, "generateMaintenanceDataBrowser")
				csrDet.Close()
			}

			tk.Printf("-- assets: %v \n", len(assets))

			if len(assets) > 0 {
				relatedAssets := []FunctionalLocation{}

				for _, asset := range assets {

					if plantCodeStr == "2110" {
						if len(asset.FunctionalLocationCode) <= 13 {
							query = []*dbox.Filter{}
							query = append(query,
								dbox.And(
									dbox.Eq("PG", "MP1"),
									dbox.Eq("PIPI", plantCodeStr),
									dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode),
									dbox.And(
										dbox.Lte("LEN(FunctionalLocationCode)", 13),
										dbox.Gte("LEN(FunctionalLocationCode)", 12),
									),
								),
							)

							csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							e = csrDet.Fetch(&systemAssets, 0, false)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							csrDet.Close()
						}

						if systemAssets == nil || len(systemAssets) == 0 {
							query = []*dbox.Filter{}

							if i != 2 {
								query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
							} else {
								query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
							}

							csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							e = csrDet.Fetch(&relatedAssets, 0, false)
							ErrorHandler(e, "generateMaintenanceDataBrowser")
							csrDet.Close()

							for _, relasset := range relatedAssets {
								isTurbineSystem := false
								if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
									isTurbineSystem = true
								}

								newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

								if newEquipment != "" {

									if i != 2 {
										turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
									}

									for _, year := range years {
										data := DataBrowser{}
										data.PeriodYear = year
										data.FunctionalLocation = relasset.FunctionalLocationCode
										data.FLDescription = relasset.Description
										data.IsTurbine = false

										// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)

										if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
											data.IsTurbine = true
											// tk.Println(" isTurbine: TRUE")
										} else {
											// tk.Println(" isTurbine: FALSE")
											data.TurbineParent = asset.FunctionalLocationCode
										}

										data.AssetType = "Other"

										if i == 0 {
											data.AssetType = "Steam"
										} else if i == 1 {
											data.AssetType = "Gas"
										}

										data.EquipmentType = newEquipment
										data.EquipmentTypeDescription = newEquipment
										data.Plant = plant
										data.PlantCode = plant.PlantCode

										if data.IsTurbine {
											info := GeneralInfo{}
											substr := ""
											substrValInt := 0
											if data.AssetType == "Steam" {
												substrValInt = 1
												substr = "ST"
											} else if data.AssetType == "Gas" {
												substrValInt = 2
											}

											if substrValInt != 0 {
												tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfo)
													substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
													return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
												}).Exec().Result.Data().([]GeneralInfo)
												if len(tmpInfo) > 0 {
													info = tmpInfo[0]
												}

												if info.Id != "" {
													data.TInfShortName = info.Unit
													data.TInfManufacturer = info.Manufacturer
													data.TInfModel = info.Model
													data.TInfUnitType = info.UnitType
													data.TInfInstalledCapacity = info.InstalledCapacity
													data.TInfOperationalCapacity = info.OperationalCapacity
													data.TInfPrimaryFuel = info.PrimaryFuel1
													data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
													data.TInfBackupFuel = info.BackupFuel
													data.TInfHeatRate = info.HeatRate
													data.TInfEfficiency = info.Efficiency

													commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
													ErrorHandler(e, "generateMaintenanceDataBrowser")
													data.TInfCommisioningDate = commDate

													if info.RetirementPlan != "" {
														retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
														retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
														ErrorHandler(e, "generateMaintenanceDataBrowser")
														data.TInfRetirementPlan = retirementPlan
													}

													installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfInstalledMWH = installedMWH.Value

													actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

													capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoDetails)
														return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoDetails)[0]

													data.TInfCapacityFactor = capacityFactor.Value

													actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfoActualFuelConsumption)
														return y.GenID == info.Id && y.Year == year
													}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

													data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
													data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
													data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
													data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

													fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
														y := x.(FuelCost)
														unitID := strings.Replace(
															strings.Replace(
																strings.Replace(
																	strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

														return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
													})

													data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
														y := x.(FuelCost)
														return y.EnergyNetProduction
													}).Exec().Result.Data().(float64)

													data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
														y := x.(FuelCost)
														return y.PrimaryFuelConsumed
													}).Exec().Result.Data().(float64)
												}
											}
											// Vibrations handled by sql query
										}

										// Maintenance handled by sql query
										// FailureNotifications handled by sql query
										// MROElements handled by sql query
										// Operationals handled by sql query

										// e = ctx.Insert(&data)
										saveDatas = append(saveDatas, &data)
										dataCount++
										if len(saveDatas) == 1000 {
											mugen.Lock()
											e = ctx.InsertBulk(saveDatas)
											mugen.Unlock()
											ErrorHandler(e, "generateMaintenanceDataBrowser")
											tk.Printf("%v : %v \n", plantCodeStr, dataCount)

											saveDatas = []orm.IModel{}
										}

										/*ErrorHandler(e, "generateMaintenanceDataBrowser")
										tk.Println("save 1")*/
									}
								}
							}

						} else {
							// with system assets
							if i != 2 {
								turbinesCodes = append(turbinesCodes, asset.FunctionalLocationCode)
							}

							for _, year := range years {
								data := DataBrowser{}
								data.PeriodYear = year
								data.FunctionalLocation = asset.FunctionalLocationCode
								data.FLDescription = asset.Description
								data.IsTurbine = false
								data.IsSystem = false

								// tk.Printf("%v | %v | ", asset.FunctionalLocationCode, i)

								if asset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
									data.IsTurbine = true
									data.IsSystem = true
									// tk.Println(" isTurbine: TRUE")
								} else {
									// tk.Println(" isTurbine: FALSE")
									data.TurbineParent = asset.FunctionalLocationCode
									data.SystemParent = asset.FunctionalLocationCode
								}

								data.AssetType = "Other"

								if i == 0 {
									data.AssetType = "Steam"
								} else if i == 1 {
									data.AssetType = "Gas"
								}

								data.EquipmentType = "System"
								data.EquipmentTypeDescription = "System"
								data.Plant = plant
								data.PlantCode = plant.PlantCode

								if data.IsTurbine {
									info := GeneralInfo{}
									substr := ""
									substrValInt := 0
									if data.AssetType == "Steam" {
										substrValInt = 1
										substr = "ST"
									} else if data.AssetType == "Gas" {
										substrValInt = 2
									}

									if substrValInt != 0 {
										tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
											y := x.(GeneralInfo)
											substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
											return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
										}).Exec().Result.Data().([]GeneralInfo)
										if len(tmpInfo) > 0 {
											info = tmpInfo[0]
										}

										if info.Id != "" {
											data.TInfShortName = info.Unit
											data.TInfManufacturer = info.Manufacturer
											data.TInfModel = info.Model
											data.TInfUnitType = info.UnitType
											data.TInfInstalledCapacity = info.InstalledCapacity
											data.TInfOperationalCapacity = info.OperationalCapacity
											data.TInfPrimaryFuel = info.PrimaryFuel1
											data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
											data.TInfBackupFuel = info.BackupFuel
											data.TInfHeatRate = info.HeatRate
											data.TInfEfficiency = info.Efficiency

											commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
											ErrorHandler(e, "generateMaintenanceDataBrowser")
											data.TInfCommisioningDate = commDate

											if info.RetirementPlan != "" {
												retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
												retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
												ErrorHandler(e, "generateMaintenanceDataBrowser")
												data.TInfRetirementPlan = retirementPlan
											}

											installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfInstalledMWH = installedMWH.Value

											actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

											capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoDetails)
												return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoDetails)[0]

											data.TInfCapacityFactor = capacityFactor.Value

											actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfoActualFuelConsumption)
												return y.GenID == info.Id && y.Year == year
											}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

											data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
											data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
											data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
											data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

											fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
												y := x.(FuelCost)
												unitID := strings.Replace(
													strings.Replace(
														strings.Replace(
															strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

												return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
											})

											data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
												y := x.(FuelCost)
												return y.EnergyNetProduction
											}).Exec().Result.Data().(float64)

											data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
												y := x.(FuelCost)
												return y.PrimaryFuelConsumed
											}).Exec().Result.Data().(float64)
										}
									}
									// Vibrations handled by sql query
								}

								// Maintenance handled by sql query
								// FailureNotifications handled by sql query
								// MROElements handled by sql query
								// Operationals handled by sql query

								// e = ctx.Insert(&data)
								// dataCount++
								saveDatas = append(saveDatas, &data)
								dataCount++
								if len(saveDatas) == 1000 {
									mugen.Lock()
									e = ctx.InsertBulk(saveDatas)
									mugen.Unlock()
									ErrorHandler(e, "generateMaintenanceDataBrowser")
									tk.Printf("%v : %v \n", plantCodeStr, dataCount)

									saveDatas = []orm.IModel{}
								}

								/*ErrorHandler(e, "generateMaintenanceDataBrowser")
								tk.Println("save 2")*/

							}

							for _, sysAsset := range systemAssets {
								query = []*dbox.Filter{}

								if i != 2 {
									query = append(query, dbox.Contains("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
								} else {
									query = append(query, dbox.Eq("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
								}

								csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
								ErrorHandler(e, "generateMaintenanceDataBrowser")
								e = csrDet.Fetch(&relatedAssets, 0, false)
								ErrorHandler(e, "generateMaintenanceDataBrowser")
								csrDet.Close()

								for _, relasset := range relatedAssets {
									isTurbineSystem := false
									if (relasset.FunctionalLocationCode == asset.FunctionalLocationCode || relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode) && i != 2 {
										isTurbineSystem = true
									}

									newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

									if newEquipment != "" {

										if i != 2 {
											turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
										}

										for _, year := range years {
											data := DataBrowser{}
											data.PeriodYear = year
											data.FunctionalLocation = relasset.FunctionalLocationCode
											data.FLDescription = relasset.Description
											data.IsTurbine = false
											data.IsSystem = false

											// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)

											if relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode && i != 2 {
												data.IsTurbine = true
												data.IsSystem = true
												// tk.Println(" isTurbine: TRUE")
											} else {
												// tk.Println(" isTurbine: FALSE")
												data.TurbineParent = asset.FunctionalLocationCode
												data.SystemParent = asset.FunctionalLocationCode
											}

											data.AssetType = "Other"

											if i == 0 {
												data.AssetType = "Steam"
											} else if i == 1 {
												data.AssetType = "Gas"
											}

											data.EquipmentType = newEquipment
											data.EquipmentTypeDescription = newEquipment
											data.Plant = plant
											data.PlantCode = plant.PlantCode

											if data.IsTurbine {
												info := GeneralInfo{}
												substr := ""
												substrValInt := 0
												if data.AssetType == "Steam" {
													substrValInt = 1
													substr = "ST"
												} else if data.AssetType == "Gas" {
													substrValInt = 2
												}

												if substrValInt != 0 {
													tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
														y := x.(GeneralInfo)
														substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
														return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
													}).Exec().Result.Data().([]GeneralInfo)
													if len(tmpInfo) > 0 {
														info = tmpInfo[0]
													}

													if info.Id != "" {
														data.TInfShortName = info.Unit
														data.TInfManufacturer = info.Manufacturer
														data.TInfModel = info.Model
														data.TInfUnitType = info.UnitType
														data.TInfInstalledCapacity = info.InstalledCapacity
														data.TInfOperationalCapacity = info.OperationalCapacity
														data.TInfPrimaryFuel = info.PrimaryFuel1
														data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
														data.TInfBackupFuel = info.BackupFuel
														data.TInfHeatRate = info.HeatRate
														data.TInfEfficiency = info.Efficiency

														commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
														ErrorHandler(e, "generateMaintenanceDataBrowser")
														data.TInfCommisioningDate = commDate

														if info.RetirementPlan != "" {
															retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
															retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
															ErrorHandler(e, "generateMaintenanceDataBrowser")
															data.TInfRetirementPlan = retirementPlan
														}

														installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfInstalledMWH = installedMWH.Value

														actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

														capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoDetails)
															return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoDetails)[0]

														data.TInfCapacityFactor = capacityFactor.Value

														actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
															y := x.(GeneralInfoActualFuelConsumption)
															return y.GenID == info.Id && y.Year == year
														}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

														data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
														data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
														data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
														data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

														fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
															y := x.(FuelCost)
															unitID := strings.Replace(
																strings.Replace(
																	strings.Replace(
																		strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

															return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
														})

														data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
															y := x.(FuelCost)
															return y.EnergyNetProduction
														}).Exec().Result.Data().(float64)

														data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
															y := x.(FuelCost)
															return y.PrimaryFuelConsumed
														}).Exec().Result.Data().(float64)
													}
												}
												// Vibrations handled by sql query
											}

											// Maintenance handled by sql query
											// FailureNotifications handled by sql query
											// MROElements handled by sql query
											// Operationals handled by sql query

											// e = ctx.Insert(&data)
											saveDatas = append(saveDatas, &data)
											dataCount++
											if len(saveDatas) == 1000 {
												mugen.Lock()
												e = ctx.InsertBulk(saveDatas)
												mugen.Unlock()
												ErrorHandler(e, "generateMaintenanceDataBrowser")
												tk.Printf("%v : %v \n", plantCodeStr, dataCount)

												saveDatas = []orm.IModel{}
											}

											/*ErrorHandler(e, "generateMaintenanceDataBrowser")
											tk.Println("save 3")*/
										}
									}
								}
							}
						}
					} else {
						// another plant(s)

						query = []*dbox.Filter{}

						if i != detsLen {
							query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
						} else {
							query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
						}

						csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
						e = csrDet.Fetch(&relatedAssets, 0, false)
						ErrorHandler(e, "generateMaintenanceDataBrowser")
						csrDet.Close()
						tk.Printf("-- related assets: %v \n", len(relatedAssets))

						for _, relasset := range relatedAssets {
							isTurbineSystem := false
							if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
								isTurbineSystem = true
							}

							newEquipment := ""
							newEquipment = d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)

							if newEquipment != "" {

								if i != detsLen {
									turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
								}

								for _, year := range years {
									_ = year
									data := DataBrowser{}
									data.PeriodYear = year
									data.FunctionalLocation = relasset.FunctionalLocationCode
									data.FLDescription = relasset.Description
									data.IsTurbine = false

									if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
										data.IsTurbine = true
									} else {
										data.TurbineParent = asset.FunctionalLocationCode
									}

									data.AssetType = "Other"

									if i == 0 {
										data.AssetType = "Steam"
									} else if i == 1 && detsLen > 1 {
										data.AssetType = "Gas"
									}

									data.EquipmentType = newEquipment
									data.EquipmentTypeDescription = newEquipment
									data.Plant = plant
									data.PlantCode = plant.PlantCode

									if data.IsTurbine {
										info := GeneralInfo{}
										substr := ""
										substrValInt := 0
										if data.AssetType == "Steam" {
											// substrValInt = 1
											substrValInt = 2
											substr = "ST" + substr
										} else if data.AssetType == "Gas" {
											substrValInt = 2
											substr = "GT" + substr
										}

										if substrValInt != 0 {
											tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
												y := x.(GeneralInfo)
												substr = data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
												return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
											}).Exec().Result.Data().([]GeneralInfo)
											if len(tmpInfo) > 0 {
												info = tmpInfo[0]
											}

											if info.Id != "" {
												data.TInfShortName = info.Unit
												data.TInfManufacturer = info.Manufacturer
												data.TInfModel = info.Model
												data.TInfUnitType = info.UnitType
												data.TInfInstalledCapacity = info.InstalledCapacity
												data.TInfOperationalCapacity = info.OperationalCapacity
												data.TInfPrimaryFuel = info.PrimaryFuel1
												data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
												data.TInfBackupFuel = info.BackupFuel
												data.TInfHeatRate = info.HeatRate
												data.TInfEfficiency = info.Efficiency

												commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
												ErrorHandler(e, "generateMaintenanceDataBrowser")

												data.TInfCommisioningDate = commDate

												if info.RetirementPlan != "" {
													retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
													retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
													ErrorHandler(e, "generateMaintenanceDataBrowser")
													data.TInfRetirementPlan = retirementPlan
												}

												installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfInstalledMWH = installedMWH.Value

												actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfActualEnergyGeneration = actualEnergyGeneration.Value

												capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoDetails)
													return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoDetails)[0]

												data.TInfCapacityFactor = capacityFactor.Value

												actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
													y := x.(GeneralInfoActualFuelConsumption)
													return y.GenID == info.Id && y.Year == year
												}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]

												data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
												data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
												data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
												data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel

												fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
													y := x.(FuelCost)
													unitID := strings.Replace(
														strings.Replace(
															strings.Replace(
																strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)

													return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
												})

												data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
													y := x.(FuelCost)
													return y.EnergyNetProduction
												}).Exec().Result.Data().(float64)

												data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
													y := x.(FuelCost)
													return y.PrimaryFuelConsumed
												}).Exec().Result.Data().(float64)
											}
										}
										// Vibrations handled by sql query
									}

									// Maintenance handled by sql query
									// FailureNotifications handled by sql query
									// MROElements handled by sql query
									// Operationals handled by sql query

									// e = ctx.Insert(&data)
									saveDatas = append(saveDatas, &data)
									dataCount++
									if len(saveDatas) == 1000 {
										mugen.Lock()
										e = ctx.InsertBulk(saveDatas)
										mugen.Unlock()
										ErrorHandler(e, "generateMaintenanceDataBrowser")
										tk.Printf("%v : %v \n", plantCodeStr, dataCount)

										saveDatas = []orm.IModel{}
									}

									/*ErrorHandler(e, "generateMaintenanceDataBrowser")
									tk.Println("save 4")*/
								}
							}
						}
					}

				}
			}
		}

		e = c.NewQuery().Delete().From(new(GenDataBrowserNotInTmp).TableName()).SetConfig("multiexec", true).Where(dbox.Eq("ID", genIDTempTable)).Exec(nil)

		if len(saveDatas) > 0 {
			e = ctx.InsertBulk(saveDatas)
			ErrorHandler(e, "generateMaintenanceDataBrowser")
			tk.Printf("%v : %v \n", plantCodeStr, dataCount)
		}

	}

	mugen.Lock()
	dataCounts.Set(plantCodeStr, dataCount)
	mugen.Unlock()

	wg.Done()
}
Пример #7
0
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) {
	var hasLookup bool
	toolkit.Println("payload : ", payload)
	if payload.Has("haslookup") {
		hasLookup = payload.Get("haslookup").(bool)
	}
	_id := toolkit.ToString(payload.Get("browserid", ""))
	sort := payload.Get("sort")
	search := payload.Get("search")
	_ = search
	take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto)
	skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto)

	TblName := toolkit.M{}
	payload.Unset("browserid")
	//sorter = ""
	if sort != nil {
		tmsort, _ := toolkit.ToM(sort.([]interface{})[0])
		fmt.Printf("====== sort %#v\n", tmsort["dir"])
		if tmsort["dir"] == "asc" {
			sorter = tmsort["field"].(string)
		} else if tmsort["dir"] == "desc" {
			sorter = "-" + tmsort["field"].(string)
		} else if tmsort["dir"] == nil {
			sorter = " "
		}
	} else {
		sorter = " "
	}

	dataDS := new(colonycore.DataBrowser)
	err := colonycore.Get(dataDS, _id)
	if err != nil {
		return 0, nil, nil, err
	}

	dataConn := new(colonycore.Connection)
	err = colonycore.Get(dataConn, dataDS.ConnectionID)
	if err != nil {
		return 0, nil, nil, err
	}

	if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil {
		return 0, nil, nil, err
	}

	connection, err := helper.ConnectUsingDataConn(dataConn).Connect()
	if err != nil {
		return 0, nil, nil, err
	}

	if dataDS.QueryType == "" {
		TblName.Set("from", dataDS.TableNames)
		payload.Set("from", dataDS.TableNames)
	} else if dataDS.QueryType == "Dbox" {
		getTableName := toolkit.M{}
		toolkit.UnjsonFromString(dataDS.QueryText, &getTableName)
		payload.Set("from", getTableName.Get("from").(string))

		if qSelect := getTableName.Get("select", "").(string); qSelect != "" {
			payload.Set("select", getTableName.Get("select").(string))
		}
	} else if dataDS.QueryType == "SQL" {
		var QueryString string
		if dataConn.Driver == "mysql" || dataConn.Driver == "hive" {
			QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip)
		} else if dataConn.Driver == "mssql" {
			QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " +
				toolkit.ToString(take) + " ROWS ONLY "

		} else if dataConn.Driver == "postgres" {
			QueryString = " LIMIT " + toolkit.ToString(take) +
				" OFFSET " + toolkit.ToString(skip)
		}
		stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString)
		payload.Set("freetext", stringQuery)
		// toolkit.Println(stringQuery)
	}

	qcount, _ := d.parseQuery(connection.NewQuery(), TblName)
	query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload)

	var selectfield string
	for _, metadata := range dataDS.MetaData {
		tField := metadata.Field
		if payload.Has(tField) {
			selectfield = toolkit.ToString(tField)
			if toolkit.IsSlice(payload[tField]) {
				query = query.Where(dbox.In(tField, payload[tField].([]interface{})...))
				qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...))
			} else if !toolkit.IsNilOrEmpty(payload[tField]) {
				var hasPattern bool
				for _, val := range querypattern {
					if strings.Contains(toolkit.ToString(payload[tField]), val) {
						hasPattern = true
					}
				}
				if hasPattern {
					query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
						toolkit.ToString(metadata.DataType), ""))
					qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
						toolkit.ToString(metadata.DataType), ""))
				} else {
					switch toolkit.ToString(metadata.DataType) {
					case "int":
						query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
					case "float32":
						query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
					case "float64":
						query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
						qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
					default:
						query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
						qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
					}
				}
			}
		}
	}

	if hasLookup && selectfield != "" {
		if toolkit.HasMember(ds_flatfile, dataConn.Driver) {
			query = query.Select(selectfield)
			qcount = qcount.Select(selectfield)
		} else {
			query = query.Select(selectfield).Group(selectfield)
			qcount = qcount.Select(selectfield).Group(selectfield)
		}

	}

	ccount, err := qcount.Cursor(nil)
	if err != nil {
		return 0, nil, nil, err
	}
	defer ccount.Close()

	dcount := ccount.Count()

	cursor, err := query.Cursor(nil)
	if err != nil {
		return 0, nil, nil, err
	}
	defer cursor.Close()

	data := []toolkit.M{}
	cursor.Fetch(&data, 0, false)

	if err != nil {
		return 0, nil, nil, err
	}

	if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) &&
		!toolkit.HasMember(ds_flatfile, dataConn.Driver) {
		dataMongo := []toolkit.M{}
		for _, val := range data {
			mVal, _ := toolkit.ToM(val.Get("_id"))
			dataMongo = append(dataMongo, mVal)
		}
		data = dataMongo
	} else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) {
		/*distinct value for flat file*/
		dataFlat := []toolkit.M{}
		var existingVal = []string{""}
		for _, val := range data {
			valString := toolkit.ToString(val.Get(selectfield))
			if !toolkit.HasMember(existingVal, valString) {
				dataFlat = append(dataFlat, val)
				existingVal = append(existingVal, valString)
			}
		}
		data = dataFlat
	}

	return dcount, data, dataDS, nil
}
Пример #8
0
func (cmd *Command) Exec(c *Crowd) error {
	if c.data == nil {
		return errors.New("Exec: Data is empty")
	}
	l := c.Len()
	if cmd.CommandType == CommandSum {
		fn := cmd.Fns[0]
		sum := float64(0)
		for i := 0; i < l; i++ {
			el := fn(c.Item(i))
			if !toolkit.IsNumber(el) {
				c.Result.Sum = 0
				return nil
			}
			item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
			sum += item
		}
		c.Result.Sum = sum
	} else if cmd.CommandType == CommandMin {
		fn := cmd.Fns[0]
		var ret interface{}
		for i := 0; i < l; i++ {
			item := fn(c.Item(i))
			if i == 0 {
				ret = item
			} else if toolkit.Compare(ret, item, "$gt") {
				ret = item
			}
		}
		c.Result.Min = ret
	} else if cmd.CommandType == CommandMax {
		fn := cmd.Fns[0]
		var ret interface{}
		for i := 0; i < l; i++ {
			item := fn(c.Item(i))
			if i == 0 {
				ret = item
			} else if toolkit.Compare(ret, item, "$lt") {
				ret = item
			}
		}
		c.Result.Max = ret
	} else if cmd.CommandType == CommandAvg {
		fn := cmd.Fns[0]
		ret := float64(0)
		for i := 0; i < l; i++ {
			el := fn(c.Item(i))
			if !toolkit.IsNumber(el) {
				c.Result.Sum = 0
				return nil
			}
			item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
			ret += item
		}
		c.Result.Avg = ret / toolkit.ToFloat64(l, 0, toolkit.RoundingAuto)
	} else if cmd.CommandType == CommandWhere {
		fn := cmd.Fns[0]
		el, _ := toolkit.GetEmptySliceElement(c.data)
		tel := reflect.TypeOf(el)
		array := reflect.MakeSlice(reflect.SliceOf(tel), 0, 0)
		for i := 0; i < l; i++ {
			item := c.Item(i)
			if fn(item).(bool) {
				array = reflect.Append(array, reflect.ValueOf(item))
			}
		}
		c.Result.data = array.Interface()
	} else if cmd.CommandType == CommandApply {
		fn := cmd.Fns[0]
		var array reflect.Value
		for i := 0; i < l; i++ {
			item := fn(c.Item(i))
			if i == 0 {
				//toolkit.Println(reflect.ValueOf(item).Type().String())
				array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(item)), 0, 0)
			} else {
				array = reflect.Append(array, reflect.ValueOf(item))
			}
		}
		c.Result.data = array.Interface()
		c.data = c.Result.data
	} else if cmd.CommandType == CommandGroup {
		fng := cmd.Fns[0]
		fnc := cmd.Fns[1]
		mvs := map[interface{}]reflect.Value{}
		//mvo := map[interface{}]interface{}{}
		var mvo []KV
		for i := 0; i < l; i++ {
			item := c.Item(i)
			g := fng(item)
			gi := fnc(item)
			array, exist := mvs[g]
			if !exist {
				//array = []interface{}{}
				array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(gi)), 0, 0)
			}
			array = reflect.Append(array, reflect.ValueOf(gi))
			//toolkit.Println("Data:",g,array)
			mvs[g] = array
		}
		for k, v := range mvs {
			mvo = append(mvo, KV{k, v.Interface()})
		}
		c.Result.data = mvo
		c.data = mvo
	} else {
		return errors.New(string(cmd.CommandType) + ": not yet applicable")
	}
	return nil
}
Пример #9
0
func savedatagrab() (err error) {
	var wg, wgstream sync.WaitGroup
	// if len(sGrabber.CollectionSettings) > 0 {
	// 	wg.Add(len(sGrabber.CollectionSettings))
	// }

	mapRecHistory = make(map[string]string, 0)
	historyFileName = fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))

	for key, _ := range sGrabber.CollectionSettings {
		//set history name
		mapRecHistory[key] = fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
		//================
		err = nil
		note := ""
		dt := toolkit.M{}.Set("datasettingname", key).Set("grabdate", thistime).Set("rowgrabbed", 0).
			Set("rowsaved", 0).Set("note", note).Set("grabstatus", "fail").Set("recfile", mapRecHistory[key])

		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] start save data", key), "INFO")
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data source", key), "INFO")
		iQ, err := sGrabber.GetQuery(key)
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get query data : %s", key, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		defer sGrabber.CloseConn()
		csr, err := iQ.Cursor(nil)
		if err != nil || csr == nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to create cursor or cursor nil to get data : %s", key, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data souce done", key), "INFO")
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save", key), "INFO")
		err = destDboxs[key].IConnection.Connect()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to connect [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error())
			errlogsavehistory(note, dt)
			continue
		}

		q := destDboxs[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
		if destDboxs[key].collection != "" {
			q = q.From(destDboxs[key].collection)
		}
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save done", key), "INFO")

		//Update Total Process
		mutex.Lock()
		Log.AddLog(fmt.Sprintf("[savedatagrab.%s] get snapshot for update total process", key), "INFO")
		err = getsnapshot()
		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}

		if pid == snapshotdata.Pid {
			Log.AddLog(fmt.Sprintf("[savedatagrab.%s] update total process data : %v", key, csr.Count()), "INFO")
			snapshotdata.Cgtotal += csr.Count()
			err = savesnapshot()
		}

		if err != nil {
			note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
			Log.AddLog(note, "ERROR")
		}
		mutex.Unlock()

		outtm := make(chan toolkit.M)
		wgstream.Add(1)
		go func(intm chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
			streamsavedata(intm, sQ, key, dt)
			wgstream.Done()
		}(outtm, q, key, dt)

		wg.Add(1)
		go func(outtm chan toolkit.M, csr dbox.ICursor) {
			condition := true

			for condition {
				results := make([]toolkit.M, 0)
				err = csr.Fetch(&results, 1000, false)
				if err != nil {
					note = fmt.Sprintf("[savedatagrab.%s] Unable to fetch data :%s", key, err.Error())
					Log.AddLog(note, "ERROR")
					condition = false
					continue
				}

				for _, result := range results {
					xresult := toolkit.M{}
					for _, column := range sGrabber.CollectionSettings[key].MapsColumns {
						var val, tval interface{}
						strsplits := strings.Split(column.Source, "|")

						ttm := toolkit.M{}
						lskey := ""
						for i, strsplit := range strsplits {
							lskey = strsplit
							if i == 0 && result.Has(strsplit) {
								ttm.Set(strsplit, result[strsplit])
							} else if ttm.Has(strsplit) {
								ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
							}
						}

						strsplits = strings.Split(column.Destination, "|")
						if ttm.Has(lskey) {
							tval = ttm[lskey]
						} else {
							tval = sedotan.DefaultValue(column.DType)
						}

						// if len(strsplits) > 1 {
						// 	if xresult.Has(strsplits[0]) {
						// 		val = xre
						// 	} else {

						// 	}
						// }

						tm := toolkit.M{}
						if xresult.Has(strsplits[0]) {
							tm, _ = toolkit.ToM(xresult[strsplits[0]])
						}
						// xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
						switch column.DType {
						case "int":
							tval = toolkit.ToInt(tval, toolkit.RoundingAuto)
						case "float":
							tval = toolkit.ToFloat64(tval, 2, toolkit.RoundingAuto)
						case "string":
							tval = toolkit.ToString(tval)
						default:
							tval = tval
						}

						val = getresultobj(strsplits, tval, tm)

						// if len(strsplits) == 1 {
						// 	val = tval
						// } else {
						// 	val = getresultobj(strsplits[1:], tval)
						// }

						// if len(strsplits) > 1 && {

						// }
						xresult.Set(strsplits[0], val)

						// for i, strsplit := range strsplits {
						// 	if i == 0 && ttm.Has(lskey) {
						// 		xresult.Set(strsplit, ttm[lskey])
						// 	} else if (i + 1) == len(strsplits) {

						// 	}
						// 	// else if ttm.Has(strsplit) {
						// 	// 	ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
						// 	// }
						// }

					}
					outtm <- xresult
				}

				if len(results) < 1000 || err != nil {
					condition = false
				}
			}
			//
			//

			// ms := []toolkit.M{}
			// for _, val := range results {
			// 	m := toolkit.M{}
			// 	for _, column := range g.CollectionSettings[dataSettingId].MapsColumns {
			// 		m.Set(column.Source, "")
			// 		if val.Has(column.Destination) {
			// 			m.Set(column.Source, val[column.Destination])
			// 		}
			// 	}
			// 	ms = append(ms, m)
			// }

			close(outtm)
			wg.Done()
		}(outtm, csr)

	}

	wgstream.Wait()
	wg.Wait()

	return
}
Пример #10
0
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {

	if closeWhenDone {
		defer c.Close()
	}

	e := c.prepIter()
	if e != nil {
		return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
	}

	if !toolkit.IsPointer(m) {
		return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
	}

	if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
		return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
	}

	var v reflect.Type

	if n == 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
		v = reflect.TypeOf(m).Elem()
	} else {
		v = reflect.TypeOf(m).Elem().Elem()
	}

	ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)

	lineCount := 0

	//=============================
	// fmt.Println("Qursor 191 : ", c.headerColumn)
	for {
		iv := reflect.New(v).Interface()

		isAppend := true
		// c.count += 1
		recData := toolkit.M{}
		appendData := toolkit.M{}

		dataTemp, e := c.reader.Read()

		for i, val := range dataTemp {
			orgname := c.headerColumn[i].name
			lowername := strings.ToLower(c.headerColumn[i].name)

			switch c.headerColumn[i].dataType {
			case "int":
				recData[lowername] = cast.ToInt(val, cast.RoundingAuto)
			case "float":
				decimalPoint := len(val) - (strings.Index(val, ".") + 1)
				recData[lowername] = cast.ToF64(val, decimalPoint, cast.RoundingAuto)
			case "date":
				recData[lowername] = toolkit.String2Date(val, c.headerColumn[i].format) // Just for test
				// fmt.Printf("FOR DEBUG : %v \n", c.headerColumn[i].format)
			default:
				recData[lowername] = val
			}

			if len(c.ConditionVal.Select) == 0 || c.ConditionVal.Select.Get("*", 0).(int) == 1 {
				appendData[orgname] = recData[lowername]
			} else {
				if c.ConditionVal.Select.Get(strings.ToLower(c.headerColumn[i].name), 0).(int) == 1 {
					appendData[orgname] = recData[lowername]
				}
			}
		}

		isAppend = c.ConditionVal.getCondition(recData)

		if isAppend {
			c.count += 1
		}

		if c.count <= c.ConditionVal.skip || (c.count > (c.ConditionVal.skip+c.ConditionVal.limit) && c.ConditionVal.limit > 0) {
			isAppend = false
		}
		// fmt.Printf("%v - %v \n", v.Elem().Kind(), toolkit.TypeName(v))
		if v.Kind() == reflect.Struct || v.Elem().Kind() == reflect.Struct {
			tv := v
			if v.Elem().Kind() == reflect.Struct {
				tv = v.Elem()
			}

			for i := 0; i < tv.NumField(); i++ {
				str := tv.Field(i).Name
				fcond := false
				if appendData.Has(str) {
					fcond = true
				} else if appendData.Has(strings.ToLower(str)) {
					fcond = true
					str = strings.ToLower(str)
				} else if strings.ToLower(str) == "id" && appendData.Has("_id") {
					str = "_id"
					fcond = true
				}

				if fcond {
					switch tv.Field(i).Type.Kind() {
					case reflect.Int:
						appendData.Set(str, cast.ToInt(appendData[str], cast.RoundingAuto))
					case reflect.String:
						appendData.Set(str, toolkit.ToString(appendData[str]))
					case reflect.Float64:
						tstr := toolkit.ToString(appendData[str])
						decimalPoint := len(tstr) - (strings.Index(tstr, ".") + 1)
						appendData.Set(str, toolkit.ToFloat64(tstr, decimalPoint, toolkit.RoundingAuto))
					}
				}
			}
		}

		if e == io.EOF {
			if isAppend && len(appendData) > 0 {
				toolkit.Serde(appendData, iv, "json")
				ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
				lineCount += 1
			}
			break
		} else if e != nil {
			return errorlib.Error(packageName, modCursor,
				"Fetch", e.Error())
		}

		if isAppend && len(appendData) > 0 {
			toolkit.Serde(appendData, iv, "json")
			ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
			lineCount += 1
		}

		if n > 0 {
			if lineCount >= n {
				break
			}
		}
	}

	if e != nil {
		return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
	}

	if n == 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
		if ivs.Len() > 0 {
			reflect.ValueOf(m).Elem().Set(ivs.Index(0))
		}
	} else {
		reflect.ValueOf(m).Elem().Set(ivs)
	}

	return nil
}
Пример #11
0
func (b *BaseController) ConvertMGOToSQLServer(m orm.IModel) error {
	tStart := time.Now()

	tk.Printf("\nConvertMGOToSQLServer: Converting %v \n", m.TableName())
	tk.Println("ConvertMGOToSQLServer: Starting to convert...\n")

	c, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Cursor(nil)

	if e != nil {
		return e
	}

	defer c.Close()

	totalData := c.Count()
	processIter := tk.ToInt(tk.ToFloat64(totalData/maxDataEachProcess, 0, tk.RoundingUp), tk.RoundingUp)

	if maxDataEachProcess == 0 {
		processIter = 0
	}

	for iter := 0; iter < processIter+1; iter++ {

		skip := iter * maxDataEachProcess
		take := maxDataEachProcess

		if maxDataEachProcess == 0 {
			take = totalData
		} else if iter == processIter {
			take = totalData - skip
		}

		csr, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Skip(skip).Take(take).Cursor(nil)

		if e != nil {
			return e
		}

		result := []tk.M{}
		e = csr.Fetch(&result, 0, false)
		csr.Close()

		if e != nil {
			return e
		}

		dtLen := len(result)

		resPart := make([][]tk.M, worker)

		if dtLen < worker {
			resPart = make([][]tk.M, 1)
			resPart[0] = result
		} else {
			workerTaskCount := tk.ToInt(tk.ToFloat64(dtLen/worker, 0, tk.RoundingAuto), tk.RoundingAuto)
			count := 0

			for i := 0; i < worker; i++ {
				if i == worker-1 {
					resPart[i] = result[count:]
				} else {
					resPart[i] = result[count : count+workerTaskCount]
				}
				count += workerTaskCount
			}
		}

		wg := &sync.WaitGroup{}
		wg.Add(len(resPart))

		for _, val := range resPart {
			// go b.Insert(val, m, wg)
			go b.InsertBulk(val, m, wg, tStart)
		}

		wg.Wait()
	}

	tk.Println("\nConvertMGOToSQLServer: Finish.")
	tk.Printf("Completed Success in %v \n", time.Since(tStart))
	return nil
}
Пример #12
0
func (cmd *Command) Exec(c *Crowd) error {
	if c.data == nil {
		return errors.New("Exec: Data is empty")
	}
	l := c.Len()
	if cmd.CommandType == CommandSum {
		fn := cmd.Fns[0]
		sum := float64(0)
		for i := 0; i < l; i++ {
			el := fn(c.Item(i))
			if !toolkit.IsNumber(el) {
				c.Result.Sum = 0
				return nil
			}
			item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
			sum += item
		}
		c.Result.Sum = sum
	} else if cmd.CommandType == CommandMin {
		fn := cmd.Fns[0]
		var ret interface{}
		for i := 0; i < l; i++ {
			item := fn(c.Item(i))
			if i == 0 {
				ret = item
			} else if toolkit.Compare(ret, item, "$gt") {
				ret = item
			}
		}
		c.Result.Min = ret
	} else if cmd.CommandType == CommandMax {
		fn := cmd.Fns[0]
		var ret interface{}
		for i := 0; i < l; i++ {
			item := fn(c.Item(i))
			if i == 0 {
				ret = item
			} else if toolkit.Compare(ret, item, "$lt") {
				ret = item
			}
		}
		c.Result.Max = ret
	} else if cmd.CommandType == CommandAvg {
		fn := cmd.Fns[0]
		ret := float64(0)
		for i := 0; i < l; i++ {
			el := fn(c.Item(i))
			if !toolkit.IsNumber(el) {
				c.Result.Sum = 0
				return nil
			}
			item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
			ret += item
		}
		c.Result.Avg = toolkit.Div(ret, toolkit.ToFloat64(l, 0, toolkit.RoundingAuto))
	} else if cmd.CommandType == CommandWhere {
		fn := cmd.Fns[0]
		el, _ := toolkit.GetEmptySliceElement(c.data)
		tel := reflect.TypeOf(el)
		array := reflect.MakeSlice(reflect.SliceOf(tel), 0, 0)
		for i := 0; i < l; i++ {
			item := c.Item(i)
			if fn(item).(bool) {
				//toolkit.Printfn("Where data %d: %v", i, item)
				array = reflect.Append(array, reflect.ValueOf(item))
			}
		}
		c.Result.data = array.Interface()
	} else if cmd.CommandType == CommandApply {
		fn := cmd.Fns[0]
		var array reflect.Value
		for i := 0; i < l; i++ {
			item := fn(c.Item(i))
			//toolkit.Printfn("Applying data %d of %d: %v", i, l, item)
			if i == 0 {
				//toolkit.Println(reflect.ValueOf(item).Type().String())
				array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(item)), 0, 0)
			}
			array = reflect.Append(array, reflect.ValueOf(item))
		}
		c.Result.data = array.Interface()
		c.data = c.Result.data
	} else if cmd.CommandType == CommandGroup {
		fng := cmd.Fns[0]
		fnc := cmd.Fns[1]
		mvs := map[interface{}]reflect.Value{}
		//mvo := map[interface{}]interface{}{}
		var mvo []KV
		for i := 0; i < l; i++ {
			item := c.Item(i)
			//toolkit.Printfn("Processing data %d of %d: %v", i, l, item)
			g := fng(item)
			gi := fnc(item)
			array, exist := mvs[g]
			if !exist {
				//array = []interface{}{}
				array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(gi)), 0, 0)
			}
			array = reflect.Append(array, reflect.ValueOf(gi))
			//toolkit.Println("Data:",g,array)
			mvs[g] = array
		}
		for k, v := range mvs {
			mvo = append(mvo, KV{k, v.Interface()})
		}
		c.Result.data = mvo
		c.data = mvo
	} else if cmd.CommandType == CommandSort {
		sorter, e := NewSorter(c.data, cmd.Fns[0])
		if e != nil {
			return e
		}

		var direction SortDirection
		direction = cmd.Parms.Get("direction", SortAscending).(SortDirection)
		c.Result.data = sorter.Sort(direction)

		c.data = c.Result.data
	} else if cmd.CommandType == CommandJoin {
		joinData := cmd.Parms.Get("joindata")
		if joinData == nil {
			return errors.New("crowd.join: Right side join data is nil")
		}
		if cmd.FnJoinKey == nil {
			cmd.FnJoinKey = func(x, y interface{}) bool {
				return x == y
			}
		}
		if cmd.FnJoinSelect == nil {
			cmd.FnJoinSelect = func(x, y interface{}) interface{} {
				return toolkit.M{}.Set("data1", x).Set("data2", y)
			}
		}
		l1 := toolkit.SliceLen(joinData)
		var array reflect.Value
		arrayBuilt := false
		for i := 0; i < l; i++ {
			item1 := c.Item(i)
			for i1 := 0; i1 < l1; i1++ {
				item2 := toolkit.SliceItem(joinData, i1)
				joinOK := cmd.FnJoinKey(item1, item2)
				if joinOK {
					outObj := cmd.FnJoinSelect(item1, item2)
					if !arrayBuilt {
						arrayBuilt = true
						array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(outObj)), 0, 0)
					}
					array = reflect.Append(array, reflect.ValueOf(outObj))
				}
			}
		}
		if !arrayBuilt {
			return errors.New("crowd.join: No match")
		}
		c.Result.data = array.Interface()
		c.data = array.Interface()
	} else {
		return errors.New(string(cmd.CommandType) + ": not yet applicable")
	}
	return nil
}