Example #1
0
func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics,
	tagmapping *mapping.Mapping, pbfFile *pbf.Pbf,
	limiter *limit.Limiter,
) {
	nodes := make(chan []element.Node, 4)
	coords := make(chan []element.Node, 4)
	ways := make(chan []element.Way, 4)
	relations := make(chan []element.Relation, 4)

	withLimiter := false
	if limiter != nil {
		withLimiter = true
	}

	if pbfFile.Header.Time.Unix() != 0 {
		log.Printf("reading %s with data till %v", pbfFile.Filename, pbfFile.Header.Time.Local())
	}

	parser := pbf.NewParser(pbfFile, coords, nodes, ways, relations)

	// wait for all coords/nodes to be processed before continuing with
	// ways. required for -limitto checks
	coordsSync := sync.WaitGroup{}
	parser.FinishedCoords(func() {
		for i := 0; int64(i) < nCoords; i++ {
			coords <- nil
		}
		for i := 0; int64(i) < nNodes; i++ {
			nodes <- nil
		}
		coordsSync.Wait()
	})

	// wait for all ways to be processed before continuing with
	// relations. required for -limitto checks
	waysSync := sync.WaitGroup{}
	parser.FinishedWays(func() {
		for i := 0; int64(i) < nWays; i++ {
			ways <- nil
		}
		waysSync.Wait()
	})

	waitWriter := sync.WaitGroup{}

	for i := 0; int64(i) < nWays; i++ {
		waysSync.Add(1)
		waitWriter.Add(1)
		go func() {
			var skip, hit int

			m := tagmapping.WayTagFilter()
			for ws := range ways {
				if ws == nil {
					waysSync.Done()
					waysSync.Wait()
					continue
				}
				if skipWays {
					continue
				}
				for i, _ := range ws {
					m.Filter(&ws[i].Tags)
					if withLimiter {
						cached, err := cache.Coords.FirstRefIsCached(ws[i].Refs)
						if err != nil {
							log.Errorf("error while checking for cached refs of way %d: %v", ws[i].Id, err)
							cached = true // don't skip in case of error
						}
						if cached {
							hit += 1
						} else {
							ws[i].Id = osmcache.SKIP
							skip += 1
						}
					}
				}
				err := cache.Ways.PutWays(ws)
				if err != nil {
					log.Errorf("error while caching ways: %v", err)
				}
				progress.AddWays(len(ws))
			}

			waitWriter.Done()
		}()
	}

	for i := 0; int64(i) < nRels; i++ {
		waitWriter.Add(1)
		go func() {
			var skip, hit int

			m := tagmapping.RelationTagFilter()
			for rels := range relations {
				numWithTags := 0
				for i, _ := range rels {
					m.Filter(&rels[i].Tags)
					if len(rels[i].Tags) > 0 {
						numWithTags += 1
					}
					if withLimiter {
						cached, err := cache.Ways.FirstMemberIsCached(rels[i].Members)
						if err != nil {
							log.Errorf("error while checking for cached members of relation %d: %v", rels[i].Id, err)
							cached = true // don't skip in case of error
						}
						if cached {
							hit += 1
						} else {
							skip += 1
							rels[i].Id = osmcache.SKIP
						}
					}
				}
				err := cache.Relations.PutRelations(rels)
				if err != nil {
					log.Errorf("error while caching relation: %v", err)
				}
				progress.AddRelations(numWithTags)
			}

			waitWriter.Done()
		}()
	}

	for i := 0; int64(i) < nCoords; i++ {
		coordsSync.Add(1)
		waitWriter.Add(1)
		go func() {
			var skip, hit int
			g := geos.NewGeos()
			defer g.Finish()
			for nds := range coords {
				if nds == nil {
					coordsSync.Done()
					coordsSync.Wait()
					continue
				}
				if withLimiter {
					for i, _ := range nds {
						if !limiter.IntersectsBuffer(g, nds[i].Long, nds[i].Lat) {
							skip += 1
							nds[i].Id = osmcache.SKIP
						} else {
							hit += 1
						}
					}
				}
				cache.Coords.PutCoords(nds)
				progress.AddCoords(len(nds))
			}
			waitWriter.Done()
		}()
	}

	for i := 0; int64(i) < nNodes; i++ {
		coordsSync.Add(1)
		waitWriter.Add(1)
		go func() {
			g := geos.NewGeos()
			defer g.Finish()
			m := tagmapping.NodeTagFilter()
			for nds := range nodes {
				if nds == nil {
					coordsSync.Done()
					coordsSync.Wait()
					continue
				}
				numWithTags := 0
				for i, _ := range nds {
					m.Filter(&nds[i].Tags)
					if len(nds[i].Tags) > 0 {
						numWithTags += 1
					}
					if withLimiter {
						if !limiter.IntersectsBuffer(g, nds[i].Long, nds[i].Lat) {
							nds[i].Id = osmcache.SKIP
						}
					}
				}
				cache.Nodes.PutNodes(nds)
				progress.AddNodes(numWithTags)
			}
			waitWriter.Done()
		}()
	}

	parser.Parse()
	waitWriter.Wait()
}
Example #2
0
func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics,
	tagmapping *mapping.Mapping, pbfFile *pbf.Pbf,
	limiter *limit.Limiter,
) {
	nodes := make(chan []element.Node, 4)
	coords := make(chan []element.Node, 4)
	ways := make(chan []element.Way, 4)
	relations := make(chan []element.Relation, 4)

	withLimiter := false
	if limiter != nil {
		withLimiter = true
	}

	if pbfFile.Header.Time.Unix() != 0 {
		log.Printf("reading %s with data till %v", pbfFile.Filename, pbfFile.Header.Time.Local())
	}

	parser := pbf.NewParser(pbfFile, coords, nodes, ways, relations)

	coordsSynced := make(chan bool)
	coordsSync := util.NewSyncPoint(int(nCoords+nNodes), func() {
		coordsSynced <- true
	})
	parser.NotifyWays(func() {
		for i := 0; int64(i) < nCoords; i++ {
			coords <- nil
		}
		for i := 0; int64(i) < nNodes; i++ {
			nodes <- nil
		}
		<-coordsSynced
	})

	waysSynced := make(chan bool)
	waysSync := util.NewSyncPoint(int(nWays), func() {
		waysSynced <- true
	})
	parser.NotifyRelations(func() {
		for i := 0; int64(i) < nWays; i++ {
			ways <- nil
		}
		<-waysSynced
	})

	parser.Start()

	waitWriter := sync.WaitGroup{}

	for i := 0; int64(i) < nWays; i++ {
		waitWriter.Add(1)
		go func() {
			var skip, hit int

			m := tagmapping.WayTagFilter()
			for ws := range ways {
				if ws == nil {
					waysSync.Sync()
					continue
				}
				if skipWays {
					continue
				}
				for i, _ := range ws {
					m.Filter(&ws[i].Tags)
					if withLimiter {
						if !cache.Coords.FirstRefIsCached(ws[i].Refs) {
							ws[i].Id = osmcache.SKIP
							skip += 1

						} else {
							hit += 1
						}
					}
				}
				cache.Ways.PutWays(ws)
				progress.AddWays(len(ws))
			}

			waitWriter.Done()
		}()
	}

	for i := 0; int64(i) < nRels; i++ {
		waitWriter.Add(1)
		go func() {
			var skip, hit int

			m := tagmapping.RelationTagFilter()
			for rels := range relations {
				numWithTags := 0
				for i, _ := range rels {
					m.Filter(&rels[i].Tags)
					if len(rels[i].Tags) > 0 {
						numWithTags += 1
					}
					if withLimiter {
						if !cache.Ways.FirstMemberIsCached(rels[i].Members) {
							skip += 1

							rels[i].Id = osmcache.SKIP
						} else {
							hit += 1

						}
					}
				}
				cache.Relations.PutRelations(rels)
				progress.AddRelations(numWithTags)
			}

			waitWriter.Done()
		}()
	}

	for i := 0; int64(i) < nCoords; i++ {
		waitWriter.Add(1)
		go func() {
			var skip, hit int
			g := geos.NewGeos()
			defer g.Finish()
			for nds := range coords {
				if nds == nil {
					coordsSync.Sync()
					continue
				}
				if withLimiter {
					for i, _ := range nds {
						nd := element.Node{Long: nds[i].Long, Lat: nds[i].Lat}
						proj.NodeToMerc(&nd)
						if !limiter.IntersectsBuffer(g, nd.Long, nd.Lat) {
							skip += 1
							nds[i].Id = osmcache.SKIP
						} else {
							hit += 1
						}
					}
				}
				cache.Coords.PutCoords(nds)
				progress.AddCoords(len(nds))
			}
			waitWriter.Done()
		}()
	}

	for i := 0; int64(i) < nNodes; i++ {
		waitWriter.Add(1)
		go func() {
			g := geos.NewGeos()
			defer g.Finish()
			m := tagmapping.NodeTagFilter()
			for nds := range nodes {
				if nds == nil {
					coordsSync.Sync()
					continue
				}
				numWithTags := 0
				for i, _ := range nds {
					m.Filter(&nds[i].Tags)
					if len(nds[i].Tags) > 0 {
						numWithTags += 1
					}
					if withLimiter {
						nd := element.Node{Long: nds[i].Long, Lat: nds[i].Lat}
						proj.NodeToMerc(&nd)
						if !limiter.IntersectsBuffer(g, nd.Long, nd.Lat) {
							nds[i].Id = osmcache.SKIP
						}
					}
				}
				cache.Nodes.PutNodes(nds)
				progress.AddNodes(numWithTags)
			}
			waitWriter.Done()
		}()
	}

	parser.Close()
	close(relations)
	close(ways)
	close(nodes)
	close(coords)
	waitWriter.Wait()
}