func loadMeasure(wg *sync.WaitGroup) {
measureDb, err := godbf.NewFromFile("MEASURE.DBF", encoding)
if err != nil {
panic(err)
}
r, _ := regexp.Compile(`([0-9/]*)\s*(.*)`)
for i := 0; i < measureDb.NumberOfRecords(); i++ {
id, _ := measureDb.Float64FieldValueByName(i, "MSR_ID")
fr_des, _ := measureDb.FieldValueByName(i, "MSR_NMF")
en_des, _ := measureDb.FieldValueByName(i, "MSR_NME")
en_res := r.FindStringSubmatch(en_des)
fr_res := r.FindStringSubmatch(fr_des)
/*fmt.Printf("en_des : %s, len = %d :\n", en_des, len(res))
for i := range res {
fmt.Println(res[i])
}*/
measureMap[id] = d.Measure{en_res[1], fr_res[2], en_res[2]}
}
wg.Done()
return
}
func loadNutrientName(wg *sync.WaitGroup) {
ntNmDb, err := godbf.NewFromFile("NT_NM.DBF", encoding)
if err != nil {
panic(err)
}
for i := 0; i < ntNmDb.NumberOfRecords(); i++ {
nut_id, _ := ntNmDb.Float64FieldValueByName(i, "NT_ID")
fr_name, _ := ntNmDb.FieldValueByName(i, "NT_NMF")
en_name, _ := ntNmDb.FieldValueByName(i, "NT_NME")
unit, _ := ntNmDb.FieldValueByName(i, "UNIT")
tag, _ := ntNmDb.FieldValueByName(i, "TAG")
nutNameMap[nut_id] = d.Nutrient{fr_name, en_name, tag, unit}
}
wg.Done()
}
func loadRefuse(wg *sync.WaitGroup) {
var refuseNameMap map[float64]d.Refuse = make(map[float64]d.Refuse)
refuseNameDb, err := godbf.NewFromFile("REFU_NM.DBF", encoding)
if err != nil {
panic(err)
}
for i := 0; i < refuseNameDb.NumberOfRecords(); i++ {
refuseId, _ := refuseNameDb.Float64FieldValueByName(i, "REFUSE_ID")
fr_name, _ := refuseNameDb.FieldValueByName(i, "REFUSE_NM_F")
en_name, _ := refuseNameDb.FieldValueByName(i, "REFUSE_NM_E")
refuseNameMap[refuseId] = d.Refuse{fr_name, en_name, 0}
}
refuseDb, err := godbf.NewFromFile("REFUSE.DBF", encoding)
if err != nil {
panic(err)
}
for i := 0; i < refuseDb.NumberOfRecords(); i++ {
foodId, _ := refuseDb.Float64FieldValueByName(i, "FD_ID")
refuseId, _ := refuseDb.Float64FieldValueByName(i, "REFUSE_ID")
value, _ := refuseDb.Float64FieldValueByName(i, "REFUSE_AMT")
refuseObj := refuseNameMap[refuseId]
refuseObj.Value = int(value)
refuseMap[foodId] = refuseObj
}
wg.Done()
}
func loadFoodGroup(wg *sync.WaitGroup) {
foodGroupDb, err := godbf.NewFromFile("FOOD_GRP.DBF", encoding)
if err != nil {
panic(err)
}
for i := 0; i < foodGroupDb.NumberOfRecords(); i++ {
fr_n, _ := foodGroupDb.FieldValueByName(i, "FD_GRP_NMF")
en_n, _ := foodGroupDb.FieldValueByName(i, "FD_GRP_NME")
id, _ := foodGroupDb.Float64FieldValueByName(i, "FD_GRP_ID")
foodGroupMap[id] = d.FoodGroup{int(id), fr_n, en_n}
}
wg.Done()
return
}
func loadConvFac(wg *sync.WaitGroup) {
convDb, err := godbf.NewFromFile("CONV_FAC.DBF", encoding)
if err != nil {
panic(err)
}
for i := 0; i < convDb.NumberOfRecords(); i++ {
fd_id, _ := convDb.Float64FieldValueByName(i, "FD_ID")
msr_id, _ := convDb.Float64FieldValueByName(i, "MSR_ID")
conv_fac, _ := convDb.Float64FieldValueByName(i, "CONV_FAC")
tmp := foodMap[fd_id]
tmp.ConvData = append(foodMap[fd_id].ConvData, d.Conv{conv_fac, measureMap[msr_id]})
foodMap[fd_id] = tmp
}
wg.Done()
}
func main() {
fmt.Println("Data load started")
var wg sync.WaitGroup
foodNmDb, err := godbf.NewFromFile("FOOD_NM.DBF", encoding)
if err != nil {
panic(err)
}
wg.Add(4)
go loadFoodGroup(&wg)
go loadMeasure(&wg)
go loadRefuse(&wg)
go loadNutrientName(&wg)
wg.Wait()
for i := 0; i < foodNmDb.NumberOfRecords(); i++ {
fr_n, _ := foodNmDb.FieldValueByName(i, "L_FD_NMF")
en_n, _ := foodNmDb.FieldValueByName(i, "L_FD_NME")
ab_fr_n, _ := foodNmDb.FieldValueByName(i, "A_FD_NMF")
ab_en_n, _ := foodNmDb.FieldValueByName(i, "A_FD_NME")
id, _ := foodNmDb.Float64FieldValueByName(i, "FD_ID")
foodGroupId, _ := foodNmDb.Float64FieldValueByName(i, "FD_GRP_ID")
foodMap[id] = d.Food{int(id), fr_n, en_n, ab_fr_n, ab_en_n, foodGroupMap[foodGroupId], refuseMap[id], make([]d.Conv, 0), make(map[string]d.NutrientData)}
}
wg.Add(2)
go loadConvFac(&wg)
go loadNutrients(&wg)
wg.Wait()
fmt.Println("Data load done")
fmt.Println("Beginning db insert")
CommitToMongo()
fmt.Println("Db insert done")
}
func loadNutrients(wg *sync.WaitGroup) {
ntAmtDb, err := godbf.NewFromFile("NT_AMT.DBF", encoding)
if err != nil {
panic(err)
}
for i := 0; i < ntAmtDb.NumberOfRecords(); i++ {
fd_id, _ := ntAmtDb.Float64FieldValueByName(i, "FD_ID")
nut_id, _ := ntAmtDb.Float64FieldValueByName(i, "NT_ID")
val, _ := ntAmtDb.Float64FieldValueByName(i, "NT_VALUE")
tmp := foodMap[fd_id]
tmp.Nutrients[nutNameMap[nut_id].En_name] = d.NutrientData{nutNameMap[nut_id], val}
foodMap[fd_id] = tmp
}
wg.Done()
}