Beispiel #1
1
func saveStatus(db database.MongoDB, lineName, status string, source models.Source) {
	uri := models.MakeUri(lineName)
	result := re.FindStringSubmatch(lineName)
	lineNumber, _ := strconv.Atoi(result[0])

	q := bson.M{"id": uri}

	_, err := db.Upsert(q, bson.M{
		"$setOnInsert": bson.M{
			"createdat": time.Now(),
		},
		"$currentDate": bson.M{
			"updatedat": true,
		},
		"$set": bson.M{
			"name":       lineName,
			"linenumber": lineNumber,
		},
		"$addToSet": bson.M{
			"sources": source,
		},
	}, models.Line{})

	log.Println(uri)
	parser.CheckError(err)
	var statusOld models.Status
	err = db.FindOne(bson.M{"line_id": uri}, &statusOld)

	statusQ := bson.M{"line_id": uri, "_id": bson.NewObjectId()}
	if err == nil && statusOld.Status == status {
		statusQ = bson.M{"_id": statusOld.Id, "line_id": uri}
	}

	_, err = db.Upsert(statusQ, bson.M{
		"$setOnInsert": bson.M{
			"createdat": time.Now(),
		},
		"$currentDate": bson.M{
			"updatedat": true,
		},
		"$set": bson.M{
			"status":  status,
			"line_id": uri,
		},
		"$addToSet": bson.M{
			"sources": source,
		},
	}, models.Status{})
	parser.CheckError(err)

	parser.Log.Debug(lineName + " - " + status)
	parser.Log.Info("-- Created Status to " + lineName)
	parser.Log.Info("Status: " + status)
	parser.Log.Info("------")

	if uri == "linha11coral" {
		saveStatus(db, "Linha 11-Coral-Expresso", status, source)
	}
}
func getQuotaPage(id, url string, DB database.MongoDB) {
	if parser.IsCached(url) {
		return
	}
	defer parser.DeferedCache(url)

	<-time.After(2 * time.Second)
	doc, err := goquery.NewDocument(url)
	if err != nil {
		parser.Log.Error(err.Error(), url)
		return
	}

	var p models.Parliamentarian
	DB.FindOne(bson.M{
		"id": id,
	}, &p)

	doc.Find(".espacoPadraoInferior2 tr:not(.celulasCentralizadas)").Each(func(i int, s *goquery.Selection) {
		data := s.Find("td")
		cnpj := data.Eq(0).Text()

		if cnpj == "TOTAL" {
			return
		}
		suplier := data.Eq(1).Text()
		orderN := strings.TrimSpace(data.Eq(2).Text())
		companyUri := models.MakeUri(suplier)

		if cnpj == "" {
			cnpj = companyUri
		}

		_, err := DB.Upsert(bson.M{"id": cnpj}, bson.M{
			"$set": bson.M{
				"name": suplier,
				"uri":  companyUri,
			},
		}, models.Company{})
		parser.CheckError(err)

		switch len(data.Nodes) {
		case 4:
			// value :=  data.Eq(3).Text()
			// log.Println("normal:", cnpj, "|", suplier, "|", orderN, value)
			// log.Println("skip")
		case 7:
			sendedAt, _ := time.Parse("2006-01-02", strings.Split(data.Eq(3).Text(), " ")[0])
			value := strings.Replace(data.Eq(6).Text(), "R$", "", -1)
			value = strings.Replace(value, ".", "", -1)
			value = strings.Replace(value, "-", "", -1)
			value = strings.TrimSpace(strings.Replace(value, ",", ".", -1))
			valueF, _ := strconv.ParseFloat(value, 64)

			parser.Log.Debug(orderN)

			orderNS := strings.Split(orderN, ":")
			var ticket string
			if len(orderNS) == 1 {
				ticket = strings.TrimSpace(orderNS[0])
			} else {
				ticket = strings.TrimSpace(orderNS[1])
			}

			_, err = DB.Upsert(bson.M{"order": orderN, "parliamentarian": p.Id}, bson.M{
				"$set": bson.M{
					"company":        cnpj,
					"date":           sendedAt,
					"passenger_name": data.Eq(4).Text(),
					"route":          data.Eq(5).Text(),
					"value":          valueF,
					"ticket":         ticket,
				},
			}, models.Quota{})
			parser.CheckError(err)

		default:
			panic(data.Text())
		}
	})
}