func TestFrom(t *testing.T) {
for i := 0; i < sampleCount; i++ {
obj := Obj{}
obj.F = toolkit.ToFloat64(toolkit.RandInt(1000), 2, toolkit.RoundingAuto)
obj.I = toolkit.RandInt(1000) + 5
objs = append(objs, obj)
if i == 0 {
min = obj.I
max = obj.I
sum = obj.I
avg = toolkit.ToFloat64(obj.I, 4, toolkit.RoundingAuto)
} else {
sum += obj.I
avg = toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) /
toolkit.ToFloat64(i+1, 4, toolkit.RoundingAuto)
if min > obj.I {
min = obj.I
}
if max < obj.I {
max = obj.I
}
}
}
c = crowd.From(&objs)
check(t, c.Error, "")
toolkit.Printf("Data len: %d, max: %d, min: %d, sum: %d, avg: %5.4f\n", c.Len(), max, min, sum, avg)
}
func TestAggr(t *testing.T) {
skipIfNil(t)
c2 := c.Min(fn).Max(fn).Sum(fn).Avg(fn).Exec()
check(t, c2.Error, "Aggr")
if toolkit.ToInt(c2.Result.Min, toolkit.RoundingAuto) != min ||
toolkit.ToInt(c2.Result.Max, toolkit.RoundingAuto) != max ||
c2.Result.Sum != toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) ||
c2.Result.Avg != avg {
t.Fatalf("Error aggr. Got %v\n", toolkit.JsonString(c2.Result))
}
toolkit.Println("Value: ", toolkit.JsonString(c2.Result))
}
func main() {
jsonarg := toolkit.M{}
err := toolkit.UnjsonFromString(os.Args[1], &jsonarg)
if err != nil {
return
}
jsonarg["Date"] = "000000"
jsonarg[`MB 62% Fe`] = "POST " + toolkit.ToString(jsonarg[`MB 62% Fe`])
jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto) * 5)
fmt.Println(toolkit.JsonString(jsonarg))
}
func main() {
jsonarg := toolkit.M{}
err := toolkit.UnjsonFromString(os.Args[1], &jsonarg)
if err != nil {
return
}
jsonarg["Date"] = time.Now().Format("020106")
jsonarg[`MB 62% Fe`] = "FE " + toolkit.ToString(jsonarg[`MB 62% Fe`])
jsonarg[`Platts 62% Fe IODEX`] = toolkit.ToString(toolkit.ToFloat64(jsonarg[`Platts 62% Fe IODEX`], 6, toolkit.RoundingAuto))
fmt.Println(toolkit.JsonString(jsonarg))
}
func toInterface(data string, dataType string, dateFormat string) interface{} {
if dataType == "" {
if strings.HasPrefix(data, "#") && strings.HasSuffix(data, "#") {
dataType = DataDate
} else {
vfloat := toolkit.ToFloat64(dataType, 2, toolkit.RoundingAuto)
vint := toolkit.ToInt(dataType, toolkit.RoundingAuto)
if int(vfloat) == vint && vint != 0 {
dataType = DataInt
} else if vfloat != 0 {
// dataType = DataFloat
b, i := toolkit.MemberIndex(DataFloats, dataType)
if b {
for idx, dataFloat := range DataFloats {
if idx == i {
dataType = dataFloat
}
}
}
} else {
dataType = DataString
}
}
}
if dataType == DataDate {
return toolkit.String2Date(data, dateFormat)
} else if dataType == DataInt {
return toolkit.ToInt(data, toolkit.RoundingAuto)
} else if toolkit.HasMember(DataFloats, dataType) {
return toolkit.ToFloat64(data, 2, toolkit.RoundingAuto)
} else {
return data
}
return nil
}
func doGenerateMaintenanceDataBrowser(wg *sync.WaitGroup, d *GenDataBrowser, plant PowerPlantCoordinates) {
var e error
ctx := d.BaseController.Ctx
c := ctx.Connection
saveDatas := []orm.IModel{}
plantCodeStr := plant.PlantCode
dataCount := 0
tmpPlantCondition := conditions.Get(plantCodeStr)
if tmpPlantCondition != nil {
plantCondition := tmpPlantCondition.(tk.M)
length := plantCondition.GetInt("length")
dets := plantCondition.Get("det").([]tk.M)
dets = append(dets, tk.M{})
turbinesCodes := []string{}
genIDTempTable := tk.GenerateRandomString("", 20)
detsLen := len(dets) - 1
tk.Printf("detsLen: %v \n", detsLen)
for i, det := range dets {
freeQuery := false
tk.Println(i)
tk.Printf("dets: %#v \n", det)
assets := []FunctionalLocation{}
systemAssets := []FunctionalLocation{}
desc := det.GetString("desc")
query := []*dbox.Filter{}
query = append(query, dbox.Contains("FunctionalLocationCode", plantCodeStr))
query = append(query, dbox.Eq("PIPI", plantCodeStr))
queryStr := "select * from FunctionalLocation where FunctionalLocationCode like('%" + plantCodeStr + "%') and PIPI = '" + plantCodeStr + "' "
if i == 0 {
query = append(query, dbox.Contains("Description", desc))
if plantCodeStr == "2220" {
query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
} else {
query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
}
} else if i == 1 && detsLen == 2 {
query = append(query, dbox.Contains("Description", desc))
if plantCodeStr == "2220" {
query = append(query, dbox.Lte("LEN(FunctionalLocationCode)", length))
} else {
query = append(query, dbox.Eq("LEN(FunctionalLocationCode)", length))
}
} else {
tk.Printf("turbinesCodes: %v \n", len(turbinesCodes))
if len(turbinesCodes) > 1 {
maxLength := 1000
maxLoop := tk.ToInt(tk.ToFloat64(len(turbinesCodes)/maxLength, 0, tk.RoundingUp), tk.RoundingUp)
for i := 0; i <= maxLoop; i++ {
datas := []string{}
if i != maxLoop {
datas = turbinesCodes[i*maxLength : (i*maxLength)+maxLength]
} else {
datas = turbinesCodes[i*maxLength:]
}
tmpNotIn := []orm.IModel{}
for _, val := range datas {
tmpData := new(GenDataBrowserNotInTmp)
tmpData.ID = genIDTempTable
tmpData.FLCode = val
tmpNotIn = append(tmpNotIn, tmpData)
}
// tk.Printf("tmpNotIn: %v \n", len(tmpNotIn))
e = ctx.InsertBulk(tmpNotIn)
ErrorHandler(e, "generateMaintenanceDataBrowser")
}
queryStr = queryStr + " and FunctionalLocationCode not in(select flcode from GenDataBrowserNotInTmp where ID = '" + genIDTempTable + "')"
freeQuery = true
}
}
if freeQuery {
csrDet, e := c.NewQuery().Command("freequery", tk.M{}.Set("syntax", queryStr)).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&assets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
} else {
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(dbox.And(query...)).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&assets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
}
tk.Printf("-- assets: %v \n", len(assets))
if len(assets) > 0 {
relatedAssets := []FunctionalLocation{}
for _, asset := range assets {
if plantCodeStr == "2110" {
if len(asset.FunctionalLocationCode) <= 13 {
query = []*dbox.Filter{}
query = append(query,
dbox.And(
dbox.Eq("PG", "MP1"),
dbox.Eq("PIPI", plantCodeStr),
dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode),
dbox.And(
dbox.Lte("LEN(FunctionalLocationCode)", 13),
dbox.Gte("LEN(FunctionalLocationCode)", 12),
),
),
)
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&systemAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
}
if systemAssets == nil || len(systemAssets) == 0 {
query = []*dbox.Filter{}
if i != 2 {
query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
} else {
query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
}
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&relatedAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
for _, relasset := range relatedAssets {
isTurbineSystem := false
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
isTurbineSystem = true
}
newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)
if newEquipment != "" {
if i != 2 {
turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
}
for _, year := range years {
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = relasset.FunctionalLocationCode
data.FLDescription = relasset.Description
data.IsTurbine = false
// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
data.IsTurbine = true
// tk.Println(" isTurbine: TRUE")
} else {
// tk.Println(" isTurbine: FALSE")
data.TurbineParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 {
data.AssetType = "Gas"
}
data.EquipmentType = newEquipment
data.EquipmentTypeDescription = newEquipment
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
substrValInt = 1
substr = "ST"
} else if data.AssetType == "Gas" {
substrValInt = 2
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 1")*/
}
}
}
} else {
// with system assets
if i != 2 {
turbinesCodes = append(turbinesCodes, asset.FunctionalLocationCode)
}
for _, year := range years {
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = asset.FunctionalLocationCode
data.FLDescription = asset.Description
data.IsTurbine = false
data.IsSystem = false
// tk.Printf("%v | %v | ", asset.FunctionalLocationCode, i)
if asset.FunctionalLocationCode == asset.FunctionalLocationCode && i != 2 {
data.IsTurbine = true
data.IsSystem = true
// tk.Println(" isTurbine: TRUE")
} else {
// tk.Println(" isTurbine: FALSE")
data.TurbineParent = asset.FunctionalLocationCode
data.SystemParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 {
data.AssetType = "Gas"
}
data.EquipmentType = "System"
data.EquipmentTypeDescription = "System"
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
substrValInt = 1
substr = "ST"
} else if data.AssetType == "Gas" {
substrValInt = 2
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
// dataCount++
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 2")*/
}
for _, sysAsset := range systemAssets {
query = []*dbox.Filter{}
if i != 2 {
query = append(query, dbox.Contains("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
} else {
query = append(query, dbox.Eq("FunctionalLocationCode", sysAsset.FunctionalLocationCode))
}
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&relatedAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
for _, relasset := range relatedAssets {
isTurbineSystem := false
if (relasset.FunctionalLocationCode == asset.FunctionalLocationCode || relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode) && i != 2 {
isTurbineSystem = true
}
newEquipment := d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)
if newEquipment != "" {
if i != 2 {
turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
}
for _, year := range years {
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = relasset.FunctionalLocationCode
data.FLDescription = relasset.Description
data.IsTurbine = false
data.IsSystem = false
// tk.Printf("%v | %v | ", relasset.FunctionalLocationCode, i)
if relasset.FunctionalLocationCode == sysAsset.FunctionalLocationCode && i != 2 {
data.IsTurbine = true
data.IsSystem = true
// tk.Println(" isTurbine: TRUE")
} else {
// tk.Println(" isTurbine: FALSE")
data.TurbineParent = asset.FunctionalLocationCode
data.SystemParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 {
data.AssetType = "Gas"
}
data.EquipmentType = newEquipment
data.EquipmentTypeDescription = newEquipment
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
substrValInt = 1
substr = "ST"
} else if data.AssetType == "Gas" {
substrValInt = 2
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = substr + data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 3")*/
}
}
}
}
}
} else {
// another plant(s)
query = []*dbox.Filter{}
if i != detsLen {
query = append(query, dbox.Contains("FunctionalLocationCode", asset.FunctionalLocationCode))
} else {
query = append(query, dbox.Eq("FunctionalLocationCode", asset.FunctionalLocationCode))
}
csrDet, e := c.NewQuery().From(new(FunctionalLocation).TableName()).Where(query...).Cursor(nil)
ErrorHandler(e, "generateMaintenanceDataBrowser")
e = csrDet.Fetch(&relatedAssets, 0, false)
ErrorHandler(e, "generateMaintenanceDataBrowser")
csrDet.Close()
tk.Printf("-- related assets: %v \n", len(relatedAssets))
for _, relasset := range relatedAssets {
isTurbineSystem := false
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
isTurbineSystem = true
}
newEquipment := ""
newEquipment = d.getNewEquipmentType(relasset.ObjectType, isTurbineSystem)
if newEquipment != "" {
if i != detsLen {
turbinesCodes = append(turbinesCodes, relasset.FunctionalLocationCode)
}
for _, year := range years {
_ = year
data := DataBrowser{}
data.PeriodYear = year
data.FunctionalLocation = relasset.FunctionalLocationCode
data.FLDescription = relasset.Description
data.IsTurbine = false
if relasset.FunctionalLocationCode == asset.FunctionalLocationCode && i != detsLen {
data.IsTurbine = true
} else {
data.TurbineParent = asset.FunctionalLocationCode
}
data.AssetType = "Other"
if i == 0 {
data.AssetType = "Steam"
} else if i == 1 && detsLen > 1 {
data.AssetType = "Gas"
}
data.EquipmentType = newEquipment
data.EquipmentTypeDescription = newEquipment
data.Plant = plant
data.PlantCode = plant.PlantCode
if data.IsTurbine {
info := GeneralInfo{}
substr := ""
substrValInt := 0
if data.AssetType == "Steam" {
// substrValInt = 1
substrValInt = 2
substr = "ST" + substr
} else if data.AssetType == "Gas" {
substrValInt = 2
substr = "GT" + substr
}
if substrValInt != 0 {
tmpInfo := crowd.From(&generalInfos).Where(func(x interface{}) interface{} {
y := x.(GeneralInfo)
substr = data.FunctionalLocation[len(data.FunctionalLocation)-substrValInt:]
return strings.Contains(strings.ToLower(strings.Trim(y.Plant, " ")), strings.ToLower(plant.PlantName)) && y.Unit == substr
}).Exec().Result.Data().([]GeneralInfo)
if len(tmpInfo) > 0 {
info = tmpInfo[0]
}
if info.Id != "" {
data.TInfShortName = info.Unit
data.TInfManufacturer = info.Manufacturer
data.TInfModel = info.Model
data.TInfUnitType = info.UnitType
data.TInfInstalledCapacity = info.InstalledCapacity
data.TInfOperationalCapacity = info.OperationalCapacity
data.TInfPrimaryFuel = info.PrimaryFuel1
data.TInfPrimaryFuel2 = info.PrimaryFuel2Startup
data.TInfBackupFuel = info.BackupFuel
data.TInfHeatRate = info.HeatRate
data.TInfEfficiency = info.Efficiency
commDate, e := time.Parse("01/02/2006", "01/01"+tk.ToString(info.CommissioningDate))
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfCommisioningDate = commDate
if info.RetirementPlan != "" {
retirementPlanStr := strings.Split(info.RetirementPlan, "(")[0]
retirementPlan, e := time.Parse("01/02/2006", "01/01"+retirementPlanStr)
ErrorHandler(e, "generateMaintenanceDataBrowser")
data.TInfRetirementPlan = retirementPlan
}
installedMWH := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "InstalledMWH" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfInstalledMWH = installedMWH.Value
actualEnergyGeneration := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "ActualEnergyGeneration" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfActualEnergyGeneration = actualEnergyGeneration.Value
capacityFactor := crowd.From(&generalInfoDetails).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoDetails)
return y.GenID == info.Id && y.Type == "CapacityFactor" && y.Year == year
}).Exec().Result.Data().([]GeneralInfoDetails)[0]
data.TInfCapacityFactor = capacityFactor.Value
actualFuelConsumption := crowd.From(&generalInfoActualFuelConsumption).Where(func(x interface{}) interface{} {
y := x.(GeneralInfoActualFuelConsumption)
return y.GenID == info.Id && y.Year == year
}).Exec().Result.Data().([]GeneralInfoActualFuelConsumption)[0]
data.TInfActualFuelConsumption_CrudeBarrel = actualFuelConsumption.CrudeBarrel
data.TInfActualFuelConsumption_DieselBarrel = actualFuelConsumption.DieselBarrel
data.TInfActualFuelConsumption_GASMMSCF = actualFuelConsumption.GASMMSCF
data.TInfActualFuelConsumption_HFOBarrel = actualFuelConsumption.HFOBarrel
fuelCostCrowd := crowd.From(&fuelCosts).Where(func(x interface{}) interface{} {
y := x.(FuelCost)
unitID := strings.Replace(
strings.Replace(
strings.Replace(
strings.Replace(y.UnitId, ".", "", -1), " 0", "", -1), " ", "", -1), "C.C ", "", -1)
return y.Year == year && y.Plant == data.Plant.PlantName && unitID == data.TInfShortName
})
data.TInfUpdateEnergyGeneration = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.EnergyNetProduction
}).Exec().Result.Data().(float64)
data.TInfUpdateFuelConsumption = fuelCostCrowd.Sum(func(x interface{}) interface{} {
y := x.(FuelCost)
return y.PrimaryFuelConsumed
}).Exec().Result.Data().(float64)
}
}
// Vibrations handled by sql query
}
// Maintenance handled by sql query
// FailureNotifications handled by sql query
// MROElements handled by sql query
// Operationals handled by sql query
// e = ctx.Insert(&data)
saveDatas = append(saveDatas, &data)
dataCount++
if len(saveDatas) == 1000 {
mugen.Lock()
e = ctx.InsertBulk(saveDatas)
mugen.Unlock()
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
saveDatas = []orm.IModel{}
}
/*ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Println("save 4")*/
}
}
}
}
}
}
}
e = c.NewQuery().Delete().From(new(GenDataBrowserNotInTmp).TableName()).SetConfig("multiexec", true).Where(dbox.Eq("ID", genIDTempTable)).Exec(nil)
if len(saveDatas) > 0 {
e = ctx.InsertBulk(saveDatas)
ErrorHandler(e, "generateMaintenanceDataBrowser")
tk.Printf("%v : %v \n", plantCodeStr, dataCount)
}
}
mugen.Lock()
dataCounts.Set(plantCodeStr, dataCount)
mugen.Unlock()
wg.Done()
}
func (d *DataSourceController) ConnectToDataSourceDB(payload toolkit.M) (int, []toolkit.M, *colonycore.DataBrowser, error) {
var hasLookup bool
toolkit.Println("payload : ", payload)
if payload.Has("haslookup") {
hasLookup = payload.Get("haslookup").(bool)
}
_id := toolkit.ToString(payload.Get("browserid", ""))
sort := payload.Get("sort")
search := payload.Get("search")
_ = search
take := toolkit.ToInt(payload.Get("take", ""), toolkit.RoundingAuto)
skip := toolkit.ToInt(payload.Get("skip", ""), toolkit.RoundingAuto)
TblName := toolkit.M{}
payload.Unset("browserid")
//sorter = ""
if sort != nil {
tmsort, _ := toolkit.ToM(sort.([]interface{})[0])
fmt.Printf("====== sort %#v\n", tmsort["dir"])
if tmsort["dir"] == "asc" {
sorter = tmsort["field"].(string)
} else if tmsort["dir"] == "desc" {
sorter = "-" + tmsort["field"].(string)
} else if tmsort["dir"] == nil {
sorter = " "
}
} else {
sorter = " "
}
dataDS := new(colonycore.DataBrowser)
err := colonycore.Get(dataDS, _id)
if err != nil {
return 0, nil, nil, err
}
dataConn := new(colonycore.Connection)
err = colonycore.Get(dataConn, dataDS.ConnectionID)
if err != nil {
return 0, nil, nil, err
}
if err := d.checkIfDriverIsSupported(dataConn.Driver); err != nil {
return 0, nil, nil, err
}
connection, err := helper.ConnectUsingDataConn(dataConn).Connect()
if err != nil {
return 0, nil, nil, err
}
if dataDS.QueryType == "" {
TblName.Set("from", dataDS.TableNames)
payload.Set("from", dataDS.TableNames)
} else if dataDS.QueryType == "Dbox" {
getTableName := toolkit.M{}
toolkit.UnjsonFromString(dataDS.QueryText, &getTableName)
payload.Set("from", getTableName.Get("from").(string))
if qSelect := getTableName.Get("select", "").(string); qSelect != "" {
payload.Set("select", getTableName.Get("select").(string))
}
} else if dataDS.QueryType == "SQL" {
var QueryString string
if dataConn.Driver == "mysql" || dataConn.Driver == "hive" {
QueryString = " LIMIT " + toolkit.ToString(take) + " OFFSET " + toolkit.ToString(skip)
} else if dataConn.Driver == "mssql" {
QueryString = " OFFSET " + toolkit.ToString(skip) + " ROWS FETCH NEXT " +
toolkit.ToString(take) + " ROWS ONLY "
} else if dataConn.Driver == "postgres" {
QueryString = " LIMIT " + toolkit.ToString(take) +
" OFFSET " + toolkit.ToString(skip)
}
stringQuery := toolkit.Sprintf("%s %s", dataDS.QueryText, QueryString)
payload.Set("freetext", stringQuery)
// toolkit.Println(stringQuery)
}
qcount, _ := d.parseQuery(connection.NewQuery(), TblName)
query, _ := d.parseQuery(connection.NewQuery() /*.Skip(skip).Take(take) .Order(sorter)*/, payload)
var selectfield string
for _, metadata := range dataDS.MetaData {
tField := metadata.Field
if payload.Has(tField) {
selectfield = toolkit.ToString(tField)
if toolkit.IsSlice(payload[tField]) {
query = query.Where(dbox.In(tField, payload[tField].([]interface{})...))
qcount = qcount.Where(dbox.In(tField, payload[tField].([]interface{})...))
} else if !toolkit.IsNilOrEmpty(payload[tField]) {
var hasPattern bool
for _, val := range querypattern {
if strings.Contains(toolkit.ToString(payload[tField]), val) {
hasPattern = true
}
}
if hasPattern {
query = query.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
qcount = qcount.Where(dbox.ParseFilter(toolkit.ToString(tField), toolkit.ToString(payload[tField]),
toolkit.ToString(metadata.DataType), ""))
} else {
switch toolkit.ToString(metadata.DataType) {
case "int":
query = query.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToInt(payload[tField], toolkit.RoundingAuto)))
case "float32":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat32(payload[tField], 2, toolkit.RoundingAuto)))
case "float64":
query = query.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
qcount = qcount.Where(dbox.Eq(tField, toolkit.ToFloat64(payload[tField], 2, toolkit.RoundingAuto)))
default:
query = query.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
qcount = qcount.Where(dbox.Contains(tField, toolkit.ToString(payload[tField])))
}
}
}
}
}
if hasLookup && selectfield != "" {
if toolkit.HasMember(ds_flatfile, dataConn.Driver) {
query = query.Select(selectfield)
qcount = qcount.Select(selectfield)
} else {
query = query.Select(selectfield).Group(selectfield)
qcount = qcount.Select(selectfield).Group(selectfield)
}
}
ccount, err := qcount.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer ccount.Close()
dcount := ccount.Count()
cursor, err := query.Cursor(nil)
if err != nil {
return 0, nil, nil, err
}
defer cursor.Close()
data := []toolkit.M{}
cursor.Fetch(&data, 0, false)
if err != nil {
return 0, nil, nil, err
}
if hasLookup && selectfield != "" && !toolkit.HasMember(ds_rdbms, dataConn.Driver) &&
!toolkit.HasMember(ds_flatfile, dataConn.Driver) {
dataMongo := []toolkit.M{}
for _, val := range data {
mVal, _ := toolkit.ToM(val.Get("_id"))
dataMongo = append(dataMongo, mVal)
}
data = dataMongo
} else if hasLookup && selectfield != "" && toolkit.HasMember(ds_flatfile, dataConn.Driver) {
/*distinct value for flat file*/
dataFlat := []toolkit.M{}
var existingVal = []string{""}
for _, val := range data {
valString := toolkit.ToString(val.Get(selectfield))
if !toolkit.HasMember(existingVal, valString) {
dataFlat = append(dataFlat, val)
existingVal = append(existingVal, valString)
}
}
data = dataFlat
}
return dcount, data, dataDS, nil
}
func (cmd *Command) Exec(c *Crowd) error {
if c.data == nil {
return errors.New("Exec: Data is empty")
}
l := c.Len()
if cmd.CommandType == CommandSum {
fn := cmd.Fns[0]
sum := float64(0)
for i := 0; i < l; i++ {
el := fn(c.Item(i))
if !toolkit.IsNumber(el) {
c.Result.Sum = 0
return nil
}
item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
sum += item
}
c.Result.Sum = sum
} else if cmd.CommandType == CommandMin {
fn := cmd.Fns[0]
var ret interface{}
for i := 0; i < l; i++ {
item := fn(c.Item(i))
if i == 0 {
ret = item
} else if toolkit.Compare(ret, item, "$gt") {
ret = item
}
}
c.Result.Min = ret
} else if cmd.CommandType == CommandMax {
fn := cmd.Fns[0]
var ret interface{}
for i := 0; i < l; i++ {
item := fn(c.Item(i))
if i == 0 {
ret = item
} else if toolkit.Compare(ret, item, "$lt") {
ret = item
}
}
c.Result.Max = ret
} else if cmd.CommandType == CommandAvg {
fn := cmd.Fns[0]
ret := float64(0)
for i := 0; i < l; i++ {
el := fn(c.Item(i))
if !toolkit.IsNumber(el) {
c.Result.Sum = 0
return nil
}
item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
ret += item
}
c.Result.Avg = ret / toolkit.ToFloat64(l, 0, toolkit.RoundingAuto)
} else if cmd.CommandType == CommandWhere {
fn := cmd.Fns[0]
el, _ := toolkit.GetEmptySliceElement(c.data)
tel := reflect.TypeOf(el)
array := reflect.MakeSlice(reflect.SliceOf(tel), 0, 0)
for i := 0; i < l; i++ {
item := c.Item(i)
if fn(item).(bool) {
array = reflect.Append(array, reflect.ValueOf(item))
}
}
c.Result.data = array.Interface()
} else if cmd.CommandType == CommandApply {
fn := cmd.Fns[0]
var array reflect.Value
for i := 0; i < l; i++ {
item := fn(c.Item(i))
if i == 0 {
//toolkit.Println(reflect.ValueOf(item).Type().String())
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(item)), 0, 0)
} else {
array = reflect.Append(array, reflect.ValueOf(item))
}
}
c.Result.data = array.Interface()
c.data = c.Result.data
} else if cmd.CommandType == CommandGroup {
fng := cmd.Fns[0]
fnc := cmd.Fns[1]
mvs := map[interface{}]reflect.Value{}
//mvo := map[interface{}]interface{}{}
var mvo []KV
for i := 0; i < l; i++ {
item := c.Item(i)
g := fng(item)
gi := fnc(item)
array, exist := mvs[g]
if !exist {
//array = []interface{}{}
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(gi)), 0, 0)
}
array = reflect.Append(array, reflect.ValueOf(gi))
//toolkit.Println("Data:",g,array)
mvs[g] = array
}
for k, v := range mvs {
mvo = append(mvo, KV{k, v.Interface()})
}
c.Result.data = mvo
c.data = mvo
} else {
return errors.New(string(cmd.CommandType) + ": not yet applicable")
}
return nil
}
func savedatagrab() (err error) {
var wg, wgstream sync.WaitGroup
// if len(sGrabber.CollectionSettings) > 0 {
// wg.Add(len(sGrabber.CollectionSettings))
// }
mapRecHistory = make(map[string]string, 0)
historyFileName = fmt.Sprintf("%s-%s.csv", toolkit.ToString(histConf.Get("filename", "")), toolkit.Date2String(sedotan.TimeNow(), toolkit.ToString(histConf.Get("filepattern", ""))))
for key, _ := range sGrabber.CollectionSettings {
//set history name
mapRecHistory[key] = fmt.Sprintf("%s.%s-%s.csv", _id, key, toolkit.Date2String(sedotan.TimeNow(), "YYYYMMddHHmmss"))
//================
err = nil
note := ""
dt := toolkit.M{}.Set("datasettingname", key).Set("grabdate", thistime).Set("rowgrabbed", 0).
Set("rowsaved", 0).Set("note", note).Set("grabstatus", "fail").Set("recfile", mapRecHistory[key])
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] start save data", key), "INFO")
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data source", key), "INFO")
iQ, err := sGrabber.GetQuery(key)
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get query data : %s", key, err.Error())
errlogsavehistory(note, dt)
continue
}
defer sGrabber.CloseConn()
csr, err := iQ.Cursor(nil)
if err != nil || csr == nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to create cursor or cursor nil to get data : %s", key, err.Error())
errlogsavehistory(note, dt)
continue
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare data souce done", key), "INFO")
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save", key), "INFO")
err = destDboxs[key].IConnection.Connect()
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to connect [%s-%s]:%s", key, destDboxs[key].desttype, destDboxs[key].IConnection.Info().Host, err.Error())
errlogsavehistory(note, dt)
continue
}
q := destDboxs[key].IConnection.NewQuery().SetConfig("multiexec", true).Save()
if destDboxs[key].collection != "" {
q = q.From(destDboxs[key].collection)
}
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] prepare destination save done", key), "INFO")
//Update Total Process
mutex.Lock()
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] get snapshot for update total process", key), "INFO")
err = getsnapshot()
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
Log.AddLog(note, "ERROR")
}
if pid == snapshotdata.Pid {
Log.AddLog(fmt.Sprintf("[savedatagrab.%s] update total process data : %v", key, csr.Count()), "INFO")
snapshotdata.Cgtotal += csr.Count()
err = savesnapshot()
}
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to get last snapshot :%s", key, err.Error())
Log.AddLog(note, "ERROR")
}
mutex.Unlock()
outtm := make(chan toolkit.M)
wgstream.Add(1)
go func(intm chan toolkit.M, sQ dbox.IQuery, key string, dt toolkit.M) {
streamsavedata(intm, sQ, key, dt)
wgstream.Done()
}(outtm, q, key, dt)
wg.Add(1)
go func(outtm chan toolkit.M, csr dbox.ICursor) {
condition := true
for condition {
results := make([]toolkit.M, 0)
err = csr.Fetch(&results, 1000, false)
if err != nil {
note = fmt.Sprintf("[savedatagrab.%s] Unable to fetch data :%s", key, err.Error())
Log.AddLog(note, "ERROR")
condition = false
continue
}
for _, result := range results {
xresult := toolkit.M{}
for _, column := range sGrabber.CollectionSettings[key].MapsColumns {
var val, tval interface{}
strsplits := strings.Split(column.Source, "|")
ttm := toolkit.M{}
lskey := ""
for i, strsplit := range strsplits {
lskey = strsplit
if i == 0 && result.Has(strsplit) {
ttm.Set(strsplit, result[strsplit])
} else if ttm.Has(strsplit) {
ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
}
}
strsplits = strings.Split(column.Destination, "|")
if ttm.Has(lskey) {
tval = ttm[lskey]
} else {
tval = sedotan.DefaultValue(column.DType)
}
// if len(strsplits) > 1 {
// if xresult.Has(strsplits[0]) {
// val = xre
// } else {
// }
// }
tm := toolkit.M{}
if xresult.Has(strsplits[0]) {
tm, _ = toolkit.ToM(xresult[strsplits[0]])
}
// xval = val.Set(strsplits[1], getresultobj(strsplits[1:], tval, tm))
switch column.DType {
case "int":
tval = toolkit.ToInt(tval, toolkit.RoundingAuto)
case "float":
tval = toolkit.ToFloat64(tval, 2, toolkit.RoundingAuto)
case "string":
tval = toolkit.ToString(tval)
default:
tval = tval
}
val = getresultobj(strsplits, tval, tm)
// if len(strsplits) == 1 {
// val = tval
// } else {
// val = getresultobj(strsplits[1:], tval)
// }
// if len(strsplits) > 1 && {
// }
xresult.Set(strsplits[0], val)
// for i, strsplit := range strsplits {
// if i == 0 && ttm.Has(lskey) {
// xresult.Set(strsplit, ttm[lskey])
// } else if (i + 1) == len(strsplits) {
// }
// // else if ttm.Has(strsplit) {
// // ttm = toolkit.M{}.Set(strsplit, ttm[strsplit])
// // }
// }
}
outtm <- xresult
}
if len(results) < 1000 || err != nil {
condition = false
}
}
//
//
// ms := []toolkit.M{}
// for _, val := range results {
// m := toolkit.M{}
// for _, column := range g.CollectionSettings[dataSettingId].MapsColumns {
// m.Set(column.Source, "")
// if val.Has(column.Destination) {
// m.Set(column.Source, val[column.Destination])
// }
// }
// ms = append(ms, m)
// }
close(outtm)
wg.Done()
}(outtm, csr)
}
wgstream.Wait()
wg.Wait()
return
}
func (c *Cursor) Fetch(m interface{}, n int, closeWhenDone bool) error {
if closeWhenDone {
defer c.Close()
}
e := c.prepIter()
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if !toolkit.IsPointer(m) {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer")
}
if n != 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
return errorlib.Error(packageName, modCursor, "Fetch", "Model object should be pointer of slice")
}
var v reflect.Type
if n == 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
v = reflect.TypeOf(m).Elem()
} else {
v = reflect.TypeOf(m).Elem().Elem()
}
ivs := reflect.MakeSlice(reflect.SliceOf(v), 0, 0)
lineCount := 0
//=============================
// fmt.Println("Qursor 191 : ", c.headerColumn)
for {
iv := reflect.New(v).Interface()
isAppend := true
// c.count += 1
recData := toolkit.M{}
appendData := toolkit.M{}
dataTemp, e := c.reader.Read()
for i, val := range dataTemp {
orgname := c.headerColumn[i].name
lowername := strings.ToLower(c.headerColumn[i].name)
switch c.headerColumn[i].dataType {
case "int":
recData[lowername] = cast.ToInt(val, cast.RoundingAuto)
case "float":
decimalPoint := len(val) - (strings.Index(val, ".") + 1)
recData[lowername] = cast.ToF64(val, decimalPoint, cast.RoundingAuto)
case "date":
recData[lowername] = toolkit.String2Date(val, c.headerColumn[i].format) // Just for test
// fmt.Printf("FOR DEBUG : %v \n", c.headerColumn[i].format)
default:
recData[lowername] = val
}
if len(c.ConditionVal.Select) == 0 || c.ConditionVal.Select.Get("*", 0).(int) == 1 {
appendData[orgname] = recData[lowername]
} else {
if c.ConditionVal.Select.Get(strings.ToLower(c.headerColumn[i].name), 0).(int) == 1 {
appendData[orgname] = recData[lowername]
}
}
}
isAppend = c.ConditionVal.getCondition(recData)
if isAppend {
c.count += 1
}
if c.count <= c.ConditionVal.skip || (c.count > (c.ConditionVal.skip+c.ConditionVal.limit) && c.ConditionVal.limit > 0) {
isAppend = false
}
// fmt.Printf("%v - %v \n", v.Elem().Kind(), toolkit.TypeName(v))
if v.Kind() == reflect.Struct || v.Elem().Kind() == reflect.Struct {
tv := v
if v.Elem().Kind() == reflect.Struct {
tv = v.Elem()
}
for i := 0; i < tv.NumField(); i++ {
str := tv.Field(i).Name
fcond := false
if appendData.Has(str) {
fcond = true
} else if appendData.Has(strings.ToLower(str)) {
fcond = true
str = strings.ToLower(str)
} else if strings.ToLower(str) == "id" && appendData.Has("_id") {
str = "_id"
fcond = true
}
if fcond {
switch tv.Field(i).Type.Kind() {
case reflect.Int:
appendData.Set(str, cast.ToInt(appendData[str], cast.RoundingAuto))
case reflect.String:
appendData.Set(str, toolkit.ToString(appendData[str]))
case reflect.Float64:
tstr := toolkit.ToString(appendData[str])
decimalPoint := len(tstr) - (strings.Index(tstr, ".") + 1)
appendData.Set(str, toolkit.ToFloat64(tstr, decimalPoint, toolkit.RoundingAuto))
}
}
}
}
if e == io.EOF {
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
break
} else if e != nil {
return errorlib.Error(packageName, modCursor,
"Fetch", e.Error())
}
if isAppend && len(appendData) > 0 {
toolkit.Serde(appendData, iv, "json")
ivs = reflect.Append(ivs, reflect.ValueOf(iv).Elem())
lineCount += 1
}
if n > 0 {
if lineCount >= n {
break
}
}
}
if e != nil {
return errorlib.Error(packageName, modCursor, "Fetch", e.Error())
}
if n == 1 && reflect.ValueOf(m).Elem().Kind() != reflect.Slice {
if ivs.Len() > 0 {
reflect.ValueOf(m).Elem().Set(ivs.Index(0))
}
} else {
reflect.ValueOf(m).Elem().Set(ivs)
}
return nil
}
func (b *BaseController) ConvertMGOToSQLServer(m orm.IModel) error {
tStart := time.Now()
tk.Printf("\nConvertMGOToSQLServer: Converting %v \n", m.TableName())
tk.Println("ConvertMGOToSQLServer: Starting to convert...\n")
c, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Cursor(nil)
if e != nil {
return e
}
defer c.Close()
totalData := c.Count()
processIter := tk.ToInt(tk.ToFloat64(totalData/maxDataEachProcess, 0, tk.RoundingUp), tk.RoundingUp)
if maxDataEachProcess == 0 {
processIter = 0
}
for iter := 0; iter < processIter+1; iter++ {
skip := iter * maxDataEachProcess
take := maxDataEachProcess
if maxDataEachProcess == 0 {
take = totalData
} else if iter == processIter {
take = totalData - skip
}
csr, e := b.MongoCtx.Connection.NewQuery().From(m.TableName()).Skip(skip).Take(take).Cursor(nil)
if e != nil {
return e
}
result := []tk.M{}
e = csr.Fetch(&result, 0, false)
csr.Close()
if e != nil {
return e
}
dtLen := len(result)
resPart := make([][]tk.M, worker)
if dtLen < worker {
resPart = make([][]tk.M, 1)
resPart[0] = result
} else {
workerTaskCount := tk.ToInt(tk.ToFloat64(dtLen/worker, 0, tk.RoundingAuto), tk.RoundingAuto)
count := 0
for i := 0; i < worker; i++ {
if i == worker-1 {
resPart[i] = result[count:]
} else {
resPart[i] = result[count : count+workerTaskCount]
}
count += workerTaskCount
}
}
wg := &sync.WaitGroup{}
wg.Add(len(resPart))
for _, val := range resPart {
// go b.Insert(val, m, wg)
go b.InsertBulk(val, m, wg, tStart)
}
wg.Wait()
}
tk.Println("\nConvertMGOToSQLServer: Finish.")
tk.Printf("Completed Success in %v \n", time.Since(tStart))
return nil
}
func (cmd *Command) Exec(c *Crowd) error {
if c.data == nil {
return errors.New("Exec: Data is empty")
}
l := c.Len()
if cmd.CommandType == CommandSum {
fn := cmd.Fns[0]
sum := float64(0)
for i := 0; i < l; i++ {
el := fn(c.Item(i))
if !toolkit.IsNumber(el) {
c.Result.Sum = 0
return nil
}
item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
sum += item
}
c.Result.Sum = sum
} else if cmd.CommandType == CommandMin {
fn := cmd.Fns[0]
var ret interface{}
for i := 0; i < l; i++ {
item := fn(c.Item(i))
if i == 0 {
ret = item
} else if toolkit.Compare(ret, item, "$gt") {
ret = item
}
}
c.Result.Min = ret
} else if cmd.CommandType == CommandMax {
fn := cmd.Fns[0]
var ret interface{}
for i := 0; i < l; i++ {
item := fn(c.Item(i))
if i == 0 {
ret = item
} else if toolkit.Compare(ret, item, "$lt") {
ret = item
}
}
c.Result.Max = ret
} else if cmd.CommandType == CommandAvg {
fn := cmd.Fns[0]
ret := float64(0)
for i := 0; i < l; i++ {
el := fn(c.Item(i))
if !toolkit.IsNumber(el) {
c.Result.Sum = 0
return nil
}
item := toolkit.ToFloat64(el, 4, toolkit.RoundingAuto)
ret += item
}
c.Result.Avg = toolkit.Div(ret, toolkit.ToFloat64(l, 0, toolkit.RoundingAuto))
} else if cmd.CommandType == CommandWhere {
fn := cmd.Fns[0]
el, _ := toolkit.GetEmptySliceElement(c.data)
tel := reflect.TypeOf(el)
array := reflect.MakeSlice(reflect.SliceOf(tel), 0, 0)
for i := 0; i < l; i++ {
item := c.Item(i)
if fn(item).(bool) {
//toolkit.Printfn("Where data %d: %v", i, item)
array = reflect.Append(array, reflect.ValueOf(item))
}
}
c.Result.data = array.Interface()
} else if cmd.CommandType == CommandApply {
fn := cmd.Fns[0]
var array reflect.Value
for i := 0; i < l; i++ {
item := fn(c.Item(i))
//toolkit.Printfn("Applying data %d of %d: %v", i, l, item)
if i == 0 {
//toolkit.Println(reflect.ValueOf(item).Type().String())
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(item)), 0, 0)
}
array = reflect.Append(array, reflect.ValueOf(item))
}
c.Result.data = array.Interface()
c.data = c.Result.data
} else if cmd.CommandType == CommandGroup {
fng := cmd.Fns[0]
fnc := cmd.Fns[1]
mvs := map[interface{}]reflect.Value{}
//mvo := map[interface{}]interface{}{}
var mvo []KV
for i := 0; i < l; i++ {
item := c.Item(i)
//toolkit.Printfn("Processing data %d of %d: %v", i, l, item)
g := fng(item)
gi := fnc(item)
array, exist := mvs[g]
if !exist {
//array = []interface{}{}
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(gi)), 0, 0)
}
array = reflect.Append(array, reflect.ValueOf(gi))
//toolkit.Println("Data:",g,array)
mvs[g] = array
}
for k, v := range mvs {
mvo = append(mvo, KV{k, v.Interface()})
}
c.Result.data = mvo
c.data = mvo
} else if cmd.CommandType == CommandSort {
sorter, e := NewSorter(c.data, cmd.Fns[0])
if e != nil {
return e
}
var direction SortDirection
direction = cmd.Parms.Get("direction", SortAscending).(SortDirection)
c.Result.data = sorter.Sort(direction)
c.data = c.Result.data
} else if cmd.CommandType == CommandJoin {
joinData := cmd.Parms.Get("joindata")
if joinData == nil {
return errors.New("crowd.join: Right side join data is nil")
}
if cmd.FnJoinKey == nil {
cmd.FnJoinKey = func(x, y interface{}) bool {
return x == y
}
}
if cmd.FnJoinSelect == nil {
cmd.FnJoinSelect = func(x, y interface{}) interface{} {
return toolkit.M{}.Set("data1", x).Set("data2", y)
}
}
l1 := toolkit.SliceLen(joinData)
var array reflect.Value
arrayBuilt := false
for i := 0; i < l; i++ {
item1 := c.Item(i)
for i1 := 0; i1 < l1; i1++ {
item2 := toolkit.SliceItem(joinData, i1)
joinOK := cmd.FnJoinKey(item1, item2)
if joinOK {
outObj := cmd.FnJoinSelect(item1, item2)
if !arrayBuilt {
arrayBuilt = true
array = reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf(outObj)), 0, 0)
}
array = reflect.Append(array, reflect.ValueOf(outObj))
}
}
}
if !arrayBuilt {
return errors.New("crowd.join: No match")
}
c.Result.data = array.Interface()
c.data = array.Interface()
} else {
return errors.New(string(cmd.CommandType) + ": not yet applicable")
}
return nil
}