func saveStatus(db database.MongoDB, lineName, status string, source models.Source) {
uri := models.MakeUri(lineName)
result := re.FindStringSubmatch(lineName)
lineNumber, _ := strconv.Atoi(result[0])
q := bson.M{"id": uri}
_, err := db.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"name": lineName,
"linenumber": lineNumber,
},
"$addToSet": bson.M{
"sources": source,
},
}, models.Line{})
log.Println(uri)
parser.CheckError(err)
var statusOld models.Status
err = db.FindOne(bson.M{"line_id": uri}, &statusOld)
statusQ := bson.M{"line_id": uri, "_id": bson.NewObjectId()}
if err == nil && statusOld.Status == status {
statusQ = bson.M{"_id": statusOld.Id, "line_id": uri}
}
_, err = db.Upsert(statusQ, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"status": status,
"line_id": uri,
},
"$addToSet": bson.M{
"sources": source,
},
}, models.Status{})
parser.CheckError(err)
parser.Log.Debug(lineName + " - " + status)
parser.Log.Info("-- Created Status to " + lineName)
parser.Log.Info("Status: " + status)
parser.Log.Info("------")
if uri == "linha11coral" {
saveStatus(db, "Linha 11-Coral-Expresso", status, source)
}
}
func getData(g string, db database.MongoDB) {
url := "http://www.apolo11.com/reservatorios.php?step=" + g
doc, err := goquery.NewDocument(url)
parser.CheckError(err)
doc.Find("body > center:nth-child(1) > table > tbody > tr > td:nth-child(1) > b > table").Each(func(_ int, s *goquery.Selection) {
trs := s.Find("tr")
title := "Sistema " + parser.Titlelize(strings.Replace(trs.Eq(0).Text(), "SISTEMA", "", -1))
uri := models.MakeUri(title)
getInfo := func(i int, ss *goquery.Selection) string {
return ss.Text()
}
percent := trs.Eq(1).Find("font").Map(getInfo)
date := trs.Eq(2).Find("font").Map(getInfo)
data := make([]bson.M, 0)
for i, _ := range percent {
if strings.TrimSpace(date[i]) != "" && strings.TrimSpace(date[i]) != "/" {
data = append(data, bson.M{"percent": percent[i], "date": date[i]})
}
}
query := bson.M{"uri": uri, "granularity_letter": g}
source := models.Source{
Url: "http://www.apolo11.com",
Note: "Apolo11",
}
_, err := db.Upsert(query, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"uri": uri,
"name": title,
"granularity_letter": g,
"granularity": getGranularity(g),
"data": data,
"source": []models.Source{source},
},
}, models.Reservoir{})
parser.CheckError(err)
log.Println(uri, title, data)
})
}
func (p SaveDeputiesFromTransparenciaBrasil) Run(DB database.MongoDB) {
source := models.Source{
Url: "http://dev.transparencia.org.br/",
Note: "Transparencia Brasil",
}
if parser.IsCached("http://dev.transparencia.org.br/") {
parser.Log.Info("SaveDeputiesFromTransparenciaBrasil Cached")
return
}
defer parser.DeferedCache("http://dev.transparencia.org.br/")
parser.Log.Info("Starting SaveDeputiesFromTransparenciaBrasil")
c := transparencia.New("kqOfbdNKSlpf")
query := map[string]string{
"casa": "1",
}
parliamenrians, err := c.Excelencias(query)
parser.CheckError(err)
for _, parliamenrian := range parliamenrians {
uri := models.MakeUri(parliamenrian.Apelido)
parser.Log.Info("Saving %s", parliamenrian.Nome)
_, err := DB.Upsert(bson.M{"id": uri}, bson.M{
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"summary": parliamenrian.MiniBio,
"nationalidentify": parliamenrian.CPF,
},
"$addToSet": bson.M{
"sources": source,
"identifiers": bson.M{
"$each": []bson.M{
{
"identifier": parliamenrian.Id,
"scheme": "TransparenciaBrasilID",
},
{
"identifier": parliamenrian.CPF,
"scheme": "CPF",
},
},
},
},
}, models.Parliamentarian{})
parser.CheckError(err)
}
}
func (p SaveDeputiesFromSearch) Run(DB database.MongoDB) {
searchURL := "http://www2.camara.leg.br/deputados/pesquisa"
if parser.IsCached(searchURL) {
parser.Log.Info("SaveDeputiesFromSearch Cached")
return
}
defer parser.DeferedCache(searchURL)
var doc *goquery.Document
var e error
if doc, e = goquery.NewDocument(searchURL); e != nil {
parser.Log.Critical(e.Error())
}
source := models.Source{
Url: searchURL,
Note: "Pesquisa Câmara",
}
doc.Find("#deputado option").Each(func(i int, s *goquery.Selection) {
value, _ := s.Attr("value")
if value != "" {
info := regexp.MustCompile("=|%23|!|\\||\\?").Split(value, -1)
name := parser.Titlelize(info[0])
q := bson.M{
"id": models.MakeUri(name),
}
_, err := DB.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$addToSet": bson.M{
"sources": source,
"identifiers": models.Identifier{
Identifier: info[2], Scheme: "nMatricula",
},
},
}, models.Parliamentarian{})
parser.CheckError(err)
}
})
}
func (self BasicStateBot) ParseState(db database.MongoDB, stateUrl string) {
doc, err := goquery.NewDocument(stateUrl)
parser.CheckError(err)
source := models.Source{
Url: stateUrl,
Note: "ibge",
}
data := doc.Find("#sintese tr")
pUrl, _ := url.Parse(stateUrl)
id := pUrl.Query().Get("sigla")
capital := parser.ToUtf8(data.Eq(0).Find(".total").Text())
population2014 := data.Eq(1).Find(".total").Text()
population2010 := data.Eq(2).Find(".total").Text()
area := data.Eq(3).Find(".total").Text()
populationDensity := data.Eq(4).Find(".total").Text()
numberOfMunicipalities, _ := strconv.Atoi(data.Eq(5).Find(".total").Text())
log.Println(id, capital, population2014, population2010, area, populationDensity, numberOfMunicipalities)
if STATES_NAME[id] == "" {
panic(id)
}
q := bson.M{"id": id}
_, err = db.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"name": STATES_NAME[id],
"capitalid": models.MakeUri(capital),
"population": toFloat(population2014),
"area": toFloat(area),
"populationdensity": toFloat(populationDensity),
"numberofmunicipalities": numberOfMunicipalities,
},
"$addToSet": bson.M{
"sources": source,
},
}, models.State{})
doc = nil
parser.CheckError(err)
}
func (self BasicCityBot) getCitiesData(db database.MongoDB, url string, stateID string) {
doc, err := goquery.NewDocument(url)
parser.CheckError(err)
source := models.Source{
Url: url,
Note: "ibge",
}
doc.Find("#municipios tbody tr").Each(func(_ int, s *goquery.Selection) {
data := s.Find("td")
name := data.Eq(0).Text()
parser.Log.Debug("Salving: " + name + " (" + stateID + ")")
id := models.MakeUri(name)
ibgecode, _ := strconv.Atoi(data.Eq(1).Text())
q := bson.M{"id": id, "stateid": stateID}
_, err = db.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"name": name,
"ibgecode": ibgecode,
"gentile": data.Eq(2).Text(),
"population": toFloat(data.Eq(3).Text()),
"area": toFloat(data.Eq(4).Text()),
"density": toFloat(data.Eq(5).Text()),
"pib": toFloat(data.Eq(6).Text()),
},
"$addToSet": bson.M{
"sources": source,
},
}, models.City{})
parser.CheckError(err)
})
doc = nil
}
func (p SaveDeputiesQuotas) Run(DB database.MongoDB) {
url := "http://www.camara.gov.br/cota-parlamentar/pg-cota-lista-deputados.jsp"
if parser.IsCached(url) {
return
}
defer parser.DeferedCache(url)
doc, err := goquery.NewDocument(url)
if err != nil {
panic(err)
return
}
doc.Find("#content ul li a").Each(func(_ int, s *goquery.Selection) {
url, _ := s.Attr("href")
name_party := strings.Split(s.Text(), "-")
name := strings.TrimSpace(name_party[0])
id := models.MakeUri(name)
if !strings.Contains(id, "lideranca") {
getPages(CAMARABASEURL+url, id, DB)
}
})
}
func (self SaveSenatorsFromIndex) Run(DB database.MongoDB) {
indexURL := "http://www.senado.gov.br"
if parser.IsCached(indexURL) {
parser.Log.Info("SaveSenatorsFromIndex Cached")
return
}
defer parser.DeferedCache(indexURL)
source := models.Source{
Url: indexURL,
Note: "senado.gov.br website",
}
var doc *goquery.Document
var e error
if doc, e = goquery.NewDocument(indexURL + "/senadores/"); e != nil {
parser.Log.Critical(e.Error())
}
doc.Find("#senadores tbody tr").Each(func(i int, s *goquery.Selection) {
data := s.Find("td")
name := data.Eq(0).Text()
link, okLink := data.Eq(0).Find("a").Attr("href")
if !okLink {
parser.CheckError(errors.New("link not found"))
} else {
link = indexURL + link
}
email, okEmail := data.Eq(6).Find("a").Attr("href")
if !okEmail {
email = ""
} else {
email = strings.Replace(email, "mailto:", "", -1)
}
partyId := models.MakeUri(data.Eq(1).Text())
DB.Upsert(bson.M{"id": partyId}, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"id": partyId,
"classification": "party",
},
}, &models.Party{})
parliamenrianId := models.MakeUri(name)
q := bson.M{
"id": parliamenrianId,
}
re := regexp.MustCompile("paginst/senador(.+)a.asp")
senatorId := re.FindStringSubmatch(link)[1]
_, err := DB.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$addToSet": bson.M{
"sources": source,
"contactdetails": bson.M{
"$each": []models.ContactDetail{
{
Label: "Telefone",
Type: "phone",
Value: data.Eq(4).Text(),
Sources: []models.Source{source},
},
{
Label: "Fax",
Type: "fax",
Value: data.Eq(5).Text(),
Sources: []models.Source{source},
},
},
},
"identifiers": bson.M{
"$each": []models.Identifier{
{Identifier: senatorId, Scheme: "CodSenador"},
},
},
},
"$set": bson.M{
"name": name,
"email": email,
"link": link,
"shortname": models.MakeUri(name),
},
}, models.Parliamentarian{})
parser.CheckError(err)
docDetails, e := goquery.NewDocument(link)
if e != nil {
parser.Log.Critical(e.Error())
}
info := docDetails.Find(".dadosSenador b")
birthdateA := strings.Split(info.Eq(1).Text(), "/")
year, _ := strconv.Atoi(birthdateA[2])
month, _ := strconv.Atoi(birthdateA[1])
day, _ := strconv.Atoi(birthdateA[0])
loc, _ := time.LoadLocation("America/Sao_Paulo")
birthDate := popolo.Date{time.Date(year, time.Month(month), day, 0, 0, 0, 0, loc)}
_, err = DB.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"birthdate": birthDate,
},
"$addToSet": bson.M{
"sources": source,
"othernames": models.OtherNames{
Name: info.Eq(0).Text(),
Note: "Nome de nascimento",
},
"contactdetails": models.ContactDetail{
Label: "Gabinete",
Type: "address",
Value: info.Eq(4).Text(),
Sources: []models.Source{source},
},
},
}, models.Parliamentarian{})
parser.CreateMembermeship(DB, models.Rel{
Id: parliamenrianId,
Link: parser.LinkTo("parliamenrians", parliamenrianId),
}, models.Rel{
Id: partyId,
Link: parser.LinkTo("parties", partyId),
}, source, "Filiado", "Partido")
parser.CheckError(err)
})
}
func (_ StationBot) Run(db database.MongoDB) {
doc, err := goquery.NewDocument("http://www.metro.sp.gov.br/app/trajeto/xt/estacoesTipoXML.asp")
parser.CheckError(err)
doc.Find("estacao").Each(func(_ int, s *goquery.Selection) {
id, _ := s.Attr("estacaoid")
name, _ := s.Attr("nome")
lineId, _ := s.Attr("linhaid")
_lineName, _ := s.Attr("linha")
typeId, _ := s.Attr("tipoid")
typeName, _ := s.Attr("tipo")
if typeId == "3" {
return
}
lineName := "Linha " + strings.Split(_lineName, " ")[0]
uri := models.MakeUri(lineName)
cannonicaluri := uri
names := strings.Split(lineName, "-")
if len(names) == 3 {
cannonicaluri = models.MakeUri(strings.Replace(lineName, names[2], "", -1))
}
lineQ := bson.M{"id": uri}
_, err := db.Upsert(lineQ, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"name": lineName,
"cannonicaluri": cannonicaluri,
"metroid": lineId,
"type": bson.M{
"id": typeId,
"name": typeName,
},
},
}, models.Line{})
parser.CheckError(err)
q := bson.M{"id": models.MakeUri(name)}
_, err = db.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"metroid": id,
"name": name,
},
}, models.Station{})
parser.CheckError(err)
log.Println(id, name, lineId, lineName, typeName)
})
}
func (_ SavePartiesFromTSE) Run(DB database.MongoDB) {
url := "http://www.tse.jus.br/partidos/partidos-politicos/registrados-no-tse"
if parser.IsCached(url) {
parser.Log.Info("SavePartiesFromTSE Cached")
return
}
defer parser.DeferedCache(url)
source := models.Source{
Url: url,
Note: "Tribunal Superior Eleitoral",
}
var doc *goquery.Document
var e error
if doc, e = goquery.NewDocument(url); e != nil {
parser.Log.Critical(e.Error())
}
const (
IDX = iota
SIGLA_IDX
NAME_IDX
DEFERIMENTO_IDX
PRESIDENT_IDX
N_IDX
)
doc.Find("#textoConteudo table tr").Each(func(i int, s *goquery.Selection) {
if s.Find(".titulo_tabela").Length() < 6 && s.Find("td").Length() > 1 {
info := s.Find("td")
parser.Log.Info("%s - %s - %s - %s - %s - %s",
info.Eq(IDX).Text(),
info.Eq(SIGLA_IDX).Text(),
info.Eq(NAME_IDX).Text(),
info.Eq(DEFERIMENTO_IDX).Text(),
info.Eq(PRESIDENT_IDX).Text(),
info.Eq(N_IDX).Text(),
)
partyId := models.MakeUri(info.Eq(SIGLA_IDX).Text())
DB.Upsert(bson.M{"id": partyId}, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"id": partyId,
"name": parser.Titlelize(info.Eq(NAME_IDX).Text()),
"othernames": []bson.M{{
"name": info.Eq(SIGLA_IDX).Text(),
}},
"classification": "party",
},
"$addToSet": bson.M{
"sources": []models.Source{source},
},
}, &models.Party{})
urlDetails, b := info.Eq(SIGLA_IDX).Find("a").Attr("href")
if b {
docDetails, err := goquery.NewDocument(urlDetails)
if err != nil {
parser.Log.Critical(err.Error())
}
sourceDetails := models.Source{
Url: urlDetails,
Note: "Tribunal Superior Eleitoral",
}
contactdetails := make([]bson.M, 0)
details := docDetails.Find("#ancora-text-um p")
address := strings.Split(details.Eq(3).Text(), ":")[1]
contactdetails = append(contactdetails, bson.M{
"label": "Endereço",
"type": "address",
"value": address,
"sources": []models.Source{sourceDetails},
})
contactdetails = append(contactdetails, bson.M{
"label": "CEP",
"type": "zipcode",
"value": findZipcode(0, details),
"sources": []models.Source{sourceDetails},
})
phoneString := strings.Split(details.Eq(5).Text(), ":")[1]
phone := strings.Split(phoneString, "/")[0]
contactdetails = append(contactdetails, bson.M{
"label": "Telefone",
"type": "phone",
"value": phone,
"sources": []models.Source{sourceDetails},
})
faxString := strings.Split(details.Eq(6).Text(), ":")[1]
fax := strings.Split(faxString, "/")[0]
contactdetails = append(contactdetails, bson.M{
"label": "Fax",
"type": "fax",
"value": fax,
"sources": []models.Source{sourceDetails},
})
website, ok := details.Eq(7).Find("a").Attr("href")
if ok {
contactdetails = append(contactdetails, bson.M{
"label": "Site",
"type": "website",
"value": website,
"sources": []models.Source{sourceDetails},
})
}
details.Eq(8).Find("a").Each(func(i int, ss *goquery.Selection) {
email, ok := ss.Attr("href")
if !ok {
return
}
contactdetails = append(contactdetails, bson.M{
"label": "Email",
"type": "email",
"value": email,
"sources": []models.Source{sourceDetails},
})
})
data := bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"contactdetails": contactdetails,
},
}
DB.Upsert(bson.M{"id": partyId}, data, models.Party{})
}
}
})
}
func getQuotaPage(id, url string, DB database.MongoDB) {
if parser.IsCached(url) {
return
}
defer parser.DeferedCache(url)
<-time.After(2 * time.Second)
doc, err := goquery.NewDocument(url)
if err != nil {
parser.Log.Error(err.Error(), url)
return
}
var p models.Parliamentarian
DB.FindOne(bson.M{
"id": id,
}, &p)
doc.Find(".espacoPadraoInferior2 tr:not(.celulasCentralizadas)").Each(func(i int, s *goquery.Selection) {
data := s.Find("td")
cnpj := data.Eq(0).Text()
if cnpj == "TOTAL" {
return
}
suplier := data.Eq(1).Text()
orderN := strings.TrimSpace(data.Eq(2).Text())
companyUri := models.MakeUri(suplier)
if cnpj == "" {
cnpj = companyUri
}
_, err := DB.Upsert(bson.M{"id": cnpj}, bson.M{
"$set": bson.M{
"name": suplier,
"uri": companyUri,
},
}, models.Company{})
parser.CheckError(err)
switch len(data.Nodes) {
case 4:
// value := data.Eq(3).Text()
// log.Println("normal:", cnpj, "|", suplier, "|", orderN, value)
// log.Println("skip")
case 7:
sendedAt, _ := time.Parse("2006-01-02", strings.Split(data.Eq(3).Text(), " ")[0])
value := strings.Replace(data.Eq(6).Text(), "R$", "", -1)
value = strings.Replace(value, ".", "", -1)
value = strings.Replace(value, "-", "", -1)
value = strings.TrimSpace(strings.Replace(value, ",", ".", -1))
valueF, _ := strconv.ParseFloat(value, 64)
parser.Log.Debug(orderN)
orderNS := strings.Split(orderN, ":")
var ticket string
if len(orderNS) == 1 {
ticket = strings.TrimSpace(orderNS[0])
} else {
ticket = strings.TrimSpace(orderNS[1])
}
_, err = DB.Upsert(bson.M{"order": orderN, "parliamentarian": p.Id}, bson.M{
"$set": bson.M{
"company": cnpj,
"date": sendedAt,
"passenger_name": data.Eq(4).Text(),
"route": data.Eq(5).Text(),
"value": valueF,
"ticket": ticket,
},
}, models.Quota{})
parser.CheckError(err)
default:
panic(data.Text())
}
})
}
func (p SaveDeputiesFromXML) Run(DB database.MongoDB) {
xmlURL := "http://www.camara.gov.br/SitCamaraWS/Deputados.asmx/ObterDeputados"
if parser.IsCached(xmlURL) {
parser.Log.Info("SaveDeputiesFromXML Cached")
return
}
defer parser.DeferedCache(xmlURL)
source := models.Source{
Url: xmlURL,
Note: "Câmara API",
}
var doc *goquery.Document
var e error
if doc, e = goquery.NewDocument(xmlURL); e != nil {
parser.Log.Critical(e.Error())
}
doc.Find("deputado").Each(func(i int, s *goquery.Selection) {
name := parser.Titlelize(s.Find("nomeparlamentar").First().Text())
parser.Log.Info("Saving " + name)
partyId := models.MakeUri(s.Find("partido").First().Text())
DB.Upsert(bson.M{"id": partyId}, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"id": partyId,
"classification": "party",
},
}, &models.Party{})
parliamenrianId := models.MakeUri(name)
q := bson.M{
"id": parliamenrianId,
}
fullName := strings.Split(parser.Titlelize(s.Find("nome").First().Text()), " ")
_, err := DB.Upsert(q, bson.M{
"$setOnInsert": bson.M{
"createdat": time.Now(),
},
"$currentDate": bson.M{
"updatedat": true,
},
"$set": bson.M{
"name": &name,
"sortname": &name,
"id": models.MakeUri(name),
"gender": s.Find("sexo").First().Text(),
"image": s.Find("urlFoto").First().Text(),
"email": s.Find("email").First().Text(),
},
"$addToSet": bson.M{
"sources": source,
"identifiers": bson.M{
"$each": []models.Identifier{
{Identifier: s.Find("idParlamentar").First().Text(), Scheme: "idParlamentar"},
{Identifier: s.Find("ideCadastro").First().Text(), Scheme: "ideCadastro"},
},
},
"othernames": models.OtherNames{
Name: parser.Titlelize(s.Find("nome").First().Text()),
FamilyName: fullName[len(fullName)-1:][0],
GivenName: fullName[0],
Note: "Nome de nascimento",
},
"contactdetails": bson.M{
"$each": []models.ContactDetail{
{
Label: "Telefone",
Type: "phone",
Value: s.Find("fone").First().Text(),
Sources: []models.Source{source},
},
{
Label: "Gabinete",
Type: "address",
Value: s.Find("gabinete").First().Text() + ", Anexo " + s.Find("anexo").First().Text(),
Sources: []models.Source{source},
},
},
},
},
}, &models.Parliamentarian{})
parser.CreateMembermeship(DB, models.Rel{
Id: parliamenrianId,
Link: parser.LinkTo("parliamenrians", parliamenrianId),
}, models.Rel{
Id: partyId,
Link: parser.LinkTo("parties", partyId),
}, source, "Filiado", "Partido")
parser.CheckError(err)
})
}