Beispiel #1
0
func getData(g string, db database.MongoDB) {
	url := "http://www.apolo11.com/reservatorios.php?step=" + g
	doc, err := goquery.NewDocument(url)
	parser.CheckError(err)
	doc.Find("body > center:nth-child(1) > table > tbody > tr > td:nth-child(1) > b > table").Each(func(_ int, s *goquery.Selection) {
		trs := s.Find("tr")
		title := "Sistema " + parser.Titlelize(strings.Replace(trs.Eq(0).Text(), "SISTEMA", "", -1))
		uri := models.MakeUri(title)

		getInfo := func(i int, ss *goquery.Selection) string {
			return ss.Text()
		}

		percent := trs.Eq(1).Find("font").Map(getInfo)
		date := trs.Eq(2).Find("font").Map(getInfo)

		data := make([]bson.M, 0)
		for i, _ := range percent {
			if strings.TrimSpace(date[i]) != "" && strings.TrimSpace(date[i]) != "/" {
				data = append(data, bson.M{"percent": percent[i], "date": date[i]})
			}
		}

		query := bson.M{"uri": uri, "granularity_letter": g}

		source := models.Source{
			Url:  "http://www.apolo11.com",
			Note: "Apolo11",
		}

		_, err := db.Upsert(query, bson.M{
			"$setOnInsert": bson.M{
				"createdat": time.Now(),
			},
			"$currentDate": bson.M{
				"updatedat": true,
			},
			"$set": bson.M{
				"uri":                uri,
				"name":               title,
				"granularity_letter": g,
				"granularity":        getGranularity(g),
				"data":               data,
				"source":             []models.Source{source},
			},
		}, models.Reservoir{})
		parser.CheckError(err)

		log.Println(uri, title, data)
	})

}
func (p SaveDeputiesFromSearch) Run(DB database.MongoDB) {
	searchURL := "http://www2.camara.leg.br/deputados/pesquisa"

	if parser.IsCached(searchURL) {
		parser.Log.Info("SaveDeputiesFromSearch Cached")
		return
	}
	defer parser.DeferedCache(searchURL)

	var doc *goquery.Document
	var e error

	if doc, e = goquery.NewDocument(searchURL); e != nil {
		parser.Log.Critical(e.Error())
	}

	source := models.Source{
		Url:  searchURL,
		Note: "Pesquisa Câmara",
	}

	doc.Find("#deputado option").Each(func(i int, s *goquery.Selection) {
		value, _ := s.Attr("value")
		if value != "" {
			info := regexp.MustCompile("=|%23|!|\\||\\?").Split(value, -1)

			name := parser.Titlelize(info[0])
			q := bson.M{
				"id": models.MakeUri(name),
			}

			_, err := DB.Upsert(q, bson.M{
				"$setOnInsert": bson.M{
					"createdat": time.Now(),
				},
				"$currentDate": bson.M{
					"updatedat": true,
				},
				"$addToSet": bson.M{
					"sources": source,
					"identifiers": models.Identifier{
						Identifier: info[2], Scheme: "nMatricula",
					},
				},
			}, models.Parliamentarian{})
			parser.CheckError(err)
		}
	})
}
Beispiel #3
0
func (_ StatusBot) Run(db database.MongoDB) {
	// Metro SP
	doc, err := goquery.NewDocument("http://www.metro.sp.gov.br/Sistemas/direto-do-metro-via4/diretodoMetroHome.aspx")
	parser.CheckError(err)
	metroSource := models.Source{
		Url: "http://www.metro.sp.gov.br/Sistemas/direto-do-metro-via4/diretodoMetroHome.aspx",
	}

	doc.Find("ul li").Each(func(_ int, s *goquery.Selection) {
		lineName := strings.TrimSpace(s.Find(".nomeDaLinha").Text())
		status := strings.TrimSpace(s.Find(".statusDaLinha").Text())
		saveStatus(db, lineName, status, metroSource)
	})

	// CPTM

	doc, err = goquery.NewDocument("http://www.cptm.sp.gov.br/Pages/atendimento.aspx")
	parser.CheckError(err)
	cptmSource := models.Source{
		Url: "http://www.cptm.sp.gov.br/Central-Relacionamento/situacao-linhas.asp",
	}

	lines := map[string]string{
		"rubi":      "7",
		"diamante":  "8",
		"esmeralda": "9",
		"turquesa":  "10",
		"coral":     "11",
		"safira":    "12",
	}

	doc.Find("#atendimento_consumidor .situacao_linhas .col-md-10 div.col-md-2").Each(func(_ int, s *goquery.Selection) {
		class, _ := s.Attr("class")
		uri := strings.TrimSpace(strings.Replace(class, "col-md-2", "", -1))
		name := s.Find(".nome_linha")
		lineNumber := lines[uri]
		status := s.Find("[data-toggle='tooltip']").Text()

		lineName := "Linha " + lineNumber + " - " + parser.Titlelize(strings.TrimSpace(name.Text()))

		saveStatus(db, lineName, status, cptmSource)
	})
}
func (_ SavePartiesFromTSE) Run(DB database.MongoDB) {
	url := "http://www.tse.jus.br/partidos/partidos-politicos/registrados-no-tse"

	if parser.IsCached(url) {
		parser.Log.Info("SavePartiesFromTSE Cached")
		return
	}
	defer parser.DeferedCache(url)

	source := models.Source{
		Url:  url,
		Note: "Tribunal Superior Eleitoral",
	}

	var doc *goquery.Document
	var e error

	if doc, e = goquery.NewDocument(url); e != nil {
		parser.Log.Critical(e.Error())
	}

	const (
		IDX = iota
		SIGLA_IDX
		NAME_IDX
		DEFERIMENTO_IDX
		PRESIDENT_IDX
		N_IDX
	)

	doc.Find("#textoConteudo table tr").Each(func(i int, s *goquery.Selection) {
		if s.Find(".titulo_tabela").Length() < 6 && s.Find("td").Length() > 1 {
			info := s.Find("td")
			parser.Log.Info("%s - %s - %s - %s - %s - %s",
				info.Eq(IDX).Text(),
				info.Eq(SIGLA_IDX).Text(),
				info.Eq(NAME_IDX).Text(),
				info.Eq(DEFERIMENTO_IDX).Text(),
				info.Eq(PRESIDENT_IDX).Text(),
				info.Eq(N_IDX).Text(),
			)

			partyId := models.MakeUri(info.Eq(SIGLA_IDX).Text())
			DB.Upsert(bson.M{"id": partyId}, bson.M{
				"$setOnInsert": bson.M{
					"createdat": time.Now(),
				},
				"$currentDate": bson.M{
					"updatedat": true,
				},
				"$set": bson.M{
					"id":   partyId,
					"name": parser.Titlelize(info.Eq(NAME_IDX).Text()),
					"othernames": []bson.M{{
						"name": info.Eq(SIGLA_IDX).Text(),
					}},
					"classification": "party",
				},
				"$addToSet": bson.M{
					"sources": []models.Source{source},
				},
			}, &models.Party{})

			urlDetails, b := info.Eq(SIGLA_IDX).Find("a").Attr("href")
			if b {
				docDetails, err := goquery.NewDocument(urlDetails)
				if err != nil {
					parser.Log.Critical(err.Error())
				}

				sourceDetails := models.Source{
					Url:  urlDetails,
					Note: "Tribunal Superior Eleitoral",
				}

				contactdetails := make([]bson.M, 0)

				details := docDetails.Find("#ancora-text-um p")

				address := strings.Split(details.Eq(3).Text(), ":")[1]
				contactdetails = append(contactdetails, bson.M{
					"label":   "Endereço",
					"type":    "address",
					"value":   address,
					"sources": []models.Source{sourceDetails},
				})

				contactdetails = append(contactdetails, bson.M{
					"label":   "CEP",
					"type":    "zipcode",
					"value":   findZipcode(0, details),
					"sources": []models.Source{sourceDetails},
				})

				phoneString := strings.Split(details.Eq(5).Text(), ":")[1]
				phone := strings.Split(phoneString, "/")[0]
				contactdetails = append(contactdetails, bson.M{
					"label":   "Telefone",
					"type":    "phone",
					"value":   phone,
					"sources": []models.Source{sourceDetails},
				})

				faxString := strings.Split(details.Eq(6).Text(), ":")[1]
				fax := strings.Split(faxString, "/")[0]
				contactdetails = append(contactdetails, bson.M{
					"label":   "Fax",
					"type":    "fax",
					"value":   fax,
					"sources": []models.Source{sourceDetails},
				})

				website, ok := details.Eq(7).Find("a").Attr("href")
				if ok {
					contactdetails = append(contactdetails, bson.M{
						"label":   "Site",
						"type":    "website",
						"value":   website,
						"sources": []models.Source{sourceDetails},
					})
				}

				details.Eq(8).Find("a").Each(func(i int, ss *goquery.Selection) {
					email, ok := ss.Attr("href")
					if !ok {
						return
					}
					contactdetails = append(contactdetails, bson.M{
						"label":   "Email",
						"type":    "email",
						"value":   email,
						"sources": []models.Source{sourceDetails},
					})
				})

				data := bson.M{
					"$setOnInsert": bson.M{
						"createdat": time.Now(),
					},
					"$currentDate": bson.M{
						"updatedat": true,
					},
					"$set": bson.M{
						"contactdetails": contactdetails,
					},
				}

				DB.Upsert(bson.M{"id": partyId}, data, models.Party{})
			}
		}
	})
}
func (p SaveDeputiesFromXML) Run(DB database.MongoDB) {
	xmlURL := "http://www.camara.gov.br/SitCamaraWS/Deputados.asmx/ObterDeputados"

	if parser.IsCached(xmlURL) {
		parser.Log.Info("SaveDeputiesFromXML Cached")
		return
	}
	defer parser.DeferedCache(xmlURL)

	source := models.Source{
		Url:  xmlURL,
		Note: "Câmara API",
	}

	var doc *goquery.Document
	var e error

	if doc, e = goquery.NewDocument(xmlURL); e != nil {
		parser.Log.Critical(e.Error())
	}

	doc.Find("deputado").Each(func(i int, s *goquery.Selection) {
		name := parser.Titlelize(s.Find("nomeparlamentar").First().Text())
		parser.Log.Info("Saving " + name)
		partyId := models.MakeUri(s.Find("partido").First().Text())
		DB.Upsert(bson.M{"id": partyId}, bson.M{
			"$setOnInsert": bson.M{
				"createdat": time.Now(),
			},
			"$currentDate": bson.M{
				"updatedat": true,
			},
			"$set": bson.M{
				"id":             partyId,
				"classification": "party",
			},
		}, &models.Party{})

		parliamenrianId := models.MakeUri(name)
		q := bson.M{
			"id": parliamenrianId,
		}
		fullName := strings.Split(parser.Titlelize(s.Find("nome").First().Text()), " ")

		_, err := DB.Upsert(q, bson.M{
			"$setOnInsert": bson.M{
				"createdat": time.Now(),
			},
			"$currentDate": bson.M{
				"updatedat": true,
			},
			"$set": bson.M{
				"name":     &name,
				"sortname": &name,
				"id":       models.MakeUri(name),
				"gender":   s.Find("sexo").First().Text(),
				"image":    s.Find("urlFoto").First().Text(),
				"email":    s.Find("email").First().Text(),
			},
			"$addToSet": bson.M{
				"sources": source,
				"identifiers": bson.M{
					"$each": []models.Identifier{
						{Identifier: s.Find("idParlamentar").First().Text(), Scheme: "idParlamentar"},
						{Identifier: s.Find("ideCadastro").First().Text(), Scheme: "ideCadastro"},
					},
				},
				"othernames": models.OtherNames{
					Name:       parser.Titlelize(s.Find("nome").First().Text()),
					FamilyName: fullName[len(fullName)-1:][0],
					GivenName:  fullName[0],
					Note:       "Nome de nascimento",
				},
				"contactdetails": bson.M{
					"$each": []models.ContactDetail{
						{
							Label:   "Telefone",
							Type:    "phone",
							Value:   s.Find("fone").First().Text(),
							Sources: []models.Source{source},
						},
						{
							Label:   "Gabinete",
							Type:    "address",
							Value:   s.Find("gabinete").First().Text() + ", Anexo " + s.Find("anexo").First().Text(),
							Sources: []models.Source{source},
						},
					},
				},
			},
		}, &models.Parliamentarian{})

		parser.CreateMembermeship(DB, models.Rel{
			Id:   parliamenrianId,
			Link: parser.LinkTo("parliamenrians", parliamenrianId),
		}, models.Rel{
			Id:   partyId,
			Link: parser.LinkTo("parties", partyId),
		}, source, "Filiado", "Partido")
		parser.CheckError(err)
	})
}