func saveStatus(db database.MongoDB, lineName, status string, source models.Source) {
uri := models.MakeUri(lineName)
result := re.FindStringSubmatch(lineName)
lineNumber, _ := strconv.Atoi(result[0])
q := bson.M{"id": uri}
_, err := db.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"name": lineName,
"linenumber": lineNumber,
},
"$addToSet": bson.M{
"sources": source,
},
}, models.Line{})
log.Println(uri)
parser.CheckError(err)
var statusOld models.Status
err = db.FindOne(bson.M{"line_id": uri}, &statusOld)
statusQ := bson.M{"line_id": uri, "_id": bson.NewObjectId()}
if err == nil && statusOld.Status == status {
statusQ = bson.M{"_id": statusOld.Id, "line_id": uri}
}
_, err = db.Upsert(statusQ, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"status": status,
"line_id": uri,
},
"$addToSet": bson.M{
"sources": source,
},
}, models.Status{})
parser.CheckError(err)
parser.Log.Debug(lineName + " - " + status)
parser.Log.Info("-- Created Status to " + lineName)
parser.Log.Info("Status: " + status)
parser.Log.Info("------")
if uri == "linha11coral" {
saveStatus(db, "Linha 11-Coral-Expresso", status, source)
}
}
func getQuotaPage(id, url string, DB database.MongoDB) {
if parser.IsCached(url) {
return
}
defer parser.DeferedCache(url)
<-time.After(2 * time.Second)
doc, err := goquery.NewDocument(url)
if err != nil {
parser.Log.Error(err.Error(), url)
return
}
var p models.Parliamentarian
DB.FindOne(bson.M{
"id": id,
}, &p)
doc.Find(".espacoPadraoInferior2 tr:not(.celulasCentralizadas)").Each(func(i int, s *goquery.Selection) {
data := s.Find("td")
cnpj := data.Eq(0).Text()
if cnpj == "TOTAL" {
return
}
suplier := data.Eq(1).Text()
orderN := strings.TrimSpace(data.Eq(2).Text())
companyUri := models.MakeUri(suplier)
if cnpj == "" {
cnpj = companyUri
}
_, err := DB.Upsert(bson.M{"id": cnpj}, bson.M{
"$set": bson.M{
"name": suplier,
"uri": companyUri,
},
}, models.Company{})
parser.CheckError(err)
switch len(data.Nodes) {
case 4:
// value := data.Eq(3).Text()
// log.Println("normal:", cnpj, "|", suplier, "|", orderN, value)
// log.Println("skip")
case 7:
sendedAt, _ := time.Parse("2006-01-02", strings.Split(data.Eq(3).Text(), " ")[0])
value := strings.Replace(data.Eq(6).Text(), "R$", "", -1)
value = strings.Replace(value, ".", "", -1)
value = strings.Replace(value, "-", "", -1)
value = strings.TrimSpace(strings.Replace(value, ",", ".", -1))
valueF, _ := strconv.ParseFloat(value, 64)
parser.Log.Debug(orderN)
orderNS := strings.Split(orderN, ":")
var ticket string
if len(orderNS) == 1 {
ticket = strings.TrimSpace(orderNS[0])
} else {
ticket = strings.TrimSpace(orderNS[1])
}
_, err = DB.Upsert(bson.M{"order": orderN, "parliamentarian": p.Id}, bson.M{
"$set": bson.M{
"company": cnpj,
"date": sendedAt,
"passenger_name": data.Eq(4).Text(),
"route": data.Eq(5).Text(),
"value": valueF,
"ticket": ticket,
},
}, models.Quota{})
parser.CheckError(err)
default:
panic(data.Text())
}
})
}