func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics, tagmapping *mapping.Mapping, pbfFile *pbf.Pbf, limiter *limit.Limiter, ) { nodes := make(chan []element.Node, 4) coords := make(chan []element.Node, 4) ways := make(chan []element.Way, 4) relations := make(chan []element.Relation, 4) withLimiter := false if limiter != nil { withLimiter = true } if pbfFile.Header.Time.Unix() != 0 { log.Printf("reading %s with data till %v", pbfFile.Filename, pbfFile.Header.Time.Local()) } parser := pbf.NewParser(pbfFile, coords, nodes, ways, relations) coordsSynced := make(chan bool) coordsSync := util.NewSyncPoint(int(nCoords+nNodes), func() { coordsSynced <- true }) parser.NotifyWays(func() { for i := 0; int64(i) < nCoords; i++ { coords <- nil } for i := 0; int64(i) < nNodes; i++ { nodes <- nil } <-coordsSynced }) waysSynced := make(chan bool) waysSync := util.NewSyncPoint(int(nWays), func() { waysSynced <- true }) parser.NotifyRelations(func() { for i := 0; int64(i) < nWays; i++ { ways <- nil } <-waysSynced }) parser.Start() waitWriter := sync.WaitGroup{} for i := 0; int64(i) < nWays; i++ { waitWriter.Add(1) go func() { var skip, hit int m := tagmapping.WayTagFilter() for ws := range ways { if ws == nil { waysSync.Sync() continue } if skipWays { continue } for i, _ := range ws { m.Filter(&ws[i].Tags) if withLimiter { if !cache.Coords.FirstRefIsCached(ws[i].Refs) { ws[i].Id = osmcache.SKIP skip += 1 } else { hit += 1 } } } cache.Ways.PutWays(ws) progress.AddWays(len(ws)) } waitWriter.Done() }() } for i := 0; int64(i) < nRels; i++ { waitWriter.Add(1) go func() { var skip, hit int m := tagmapping.RelationTagFilter() for rels := range relations { numWithTags := 0 for i, _ := range rels { m.Filter(&rels[i].Tags) if len(rels[i].Tags) > 0 { numWithTags += 1 } if withLimiter { if !cache.Ways.FirstMemberIsCached(rels[i].Members) { skip += 1 rels[i].Id = osmcache.SKIP } else { hit += 1 } } } cache.Relations.PutRelations(rels) progress.AddRelations(numWithTags) } waitWriter.Done() }() } for i := 0; int64(i) < nCoords; i++ { waitWriter.Add(1) go func() { var skip, hit int g := geos.NewGeos() defer g.Finish() for nds := range coords { if nds == nil { coordsSync.Sync() continue } if withLimiter { for i, _ := range nds { nd := element.Node{Long: nds[i].Long, Lat: nds[i].Lat} proj.NodeToMerc(&nd) if !limiter.IntersectsBuffer(g, nd.Long, nd.Lat) { skip += 1 nds[i].Id = osmcache.SKIP } else { hit += 1 } } } cache.Coords.PutCoords(nds) progress.AddCoords(len(nds)) } waitWriter.Done() }() } for i := 0; int64(i) < nNodes; i++ { waitWriter.Add(1) go func() { g := geos.NewGeos() defer g.Finish() m := tagmapping.NodeTagFilter() for nds := range nodes { if nds == nil { coordsSync.Sync() continue } numWithTags := 0 for i, _ := range nds { m.Filter(&nds[i].Tags) if len(nds[i].Tags) > 0 { numWithTags += 1 } if withLimiter { nd := element.Node{Long: nds[i].Long, Lat: nds[i].Lat} proj.NodeToMerc(&nd) if !limiter.IntersectsBuffer(g, nd.Long, nd.Lat) { nds[i].Id = osmcache.SKIP } } } cache.Nodes.PutNodes(nds) progress.AddNodes(numWithTags) } waitWriter.Done() }() } parser.Close() close(relations) close(ways) close(nodes) close(coords) waitWriter.Wait() }
func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapping.Mapping, pbfFile *pbf.Pbf) { nodes := make(chan []element.Node, 4) coords := make(chan []element.Node, 4) ways := make(chan []element.Way, 4) relations := make(chan []element.Relation, 4) if pbfFile.Header.Time.Unix() != 0 { log.Printf("reading %s with data till %v", pbfFile.Filename, pbfFile.Header.Time.Local()) } blocks := pbfFile.BlockPositions() waitParser := sync.WaitGroup{} for i := 0; int64(i) < nParser; i++ { waitParser.Add(1) go func() { for block := range blocks { block.Parse( coords, nodes, ways, relations, ) } waitParser.Done() }() } waitWriter := sync.WaitGroup{} for i := 0; int64(i) < nWays; i++ { waitWriter.Add(1) go func() { m := tagmapping.WayTagFilter() for ws := range ways { if skipWays { continue } for i, _ := range ws { m.Filter(&ws[i].Tags) } cache.Ways.PutWays(ws) progress.AddWays(len(ws)) } waitWriter.Done() }() } for i := 0; int64(i) < nRels; i++ { waitWriter.Add(1) go func() { m := tagmapping.RelationTagFilter() for rels := range relations { numWithTags := 0 for i, _ := range rels { m.Filter(&rels[i].Tags) if len(rels[i].Tags) > 0 { numWithTags += 1 } } cache.Relations.PutRelations(rels) progress.AddRelations(numWithTags) } waitWriter.Done() }() } for i := 0; int64(i) < nCoords; i++ { waitWriter.Add(1) go func() { for nds := range coords { if skipCoords { continue } cache.Coords.PutCoords(nds) progress.AddCoords(len(nds)) } waitWriter.Done() }() } for i := 0; int64(i) < nNodes; i++ { waitWriter.Add(1) go func() { m := tagmapping.NodeTagFilter() for nds := range nodes { numWithTags := 0 for i, _ := range nds { m.Filter(&nds[i].Tags) if len(nds[i].Tags) > 0 { numWithTags += 1 } } cache.Nodes.PutNodes(nds) progress.AddNodes(numWithTags) } waitWriter.Done() }() } waitParser.Wait() close(coords) close(nodes) close(ways) close(relations) waitWriter.Wait() }