func main() {
debug := false // switch to true for a bit of extra information
// grok the command line args
filename, dist, lat, lon, pattern := grok_args(os.Args)
if debug {
fmt.Printf("# d,lat,lon,'pattern': %.3f, %f,%f,'%s'\n", dist, lat, lon, pattern)
}
// start reading the file
f, err := os.Open(filename)
if err != nil {
log.Fatal(err)
}
defer f.Close()
d := osmpbf.NewDecoder(f)
err = d.Start(runtime.GOMAXPROCS(-1))
if err != nil {
log.Fatal(err)
}
var nc, wc, rc uint64
for {
if v, err := d.Decode(); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
} else {
switch v := v.(type) {
case *osmpbf.Node:
handleNode(*v, dist, lat, lon, pattern)
//handleNode(*v, lat, lon, dist)
nc++
case *osmpbf.Way:
// Process Way v.
wc++
case *osmpbf.Relation:
// Process Relation v.
rc++
default:
log.Fatalf("unknown type %T\n", v)
}
}
}
if debug {
fmt.Printf("# Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
}
}
func ParsePbf(path string) (*OsmData, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
d := osmpbf.NewDecoder(f)
err = d.Start(10)
if err != nil {
return nil, err
}
data := &OsmData{
Nodes: map[int64]Node{},
Ways: map[int64]Way{},
Features: map[string][]FeatureRef{},
}
for {
if v, err := d.Decode(); err == io.EOF {
break
} else if err != nil {
return nil, err
} else {
switch v := v.(type) {
case *osmpbf.Node:
node := NodeFromPbf(v)
data.Nodes[node.Id] = node
case *osmpbf.Way:
way := WayFromPbf(v)
fName, ok := way.MatchAny(mapFeatures)
if ok {
data.Ways[way.Id] = way
data.Features[fName] = append(data.Features[fName], FeatureRef{way.Id, ItemTypeWay})
}
case *osmpbf.Relation:
// Ignore
default:
return nil, fmt.Errorf("unknown type %T", v)
}
}
}
return data, nil
}
func Example() {
f, err := os.Open("greater-london-140324.osm.pbf")
if err != nil {
log.Fatal(err)
}
defer f.Close()
d := osmpbf.NewDecoder(f)
// use more memory from the start, it is faster
d.SetBufferSize(osmpbf.MaxBlobSize)
// start decoding with several goroutines, it is faster
err = d.Start(runtime.GOMAXPROCS(-1))
if err != nil {
log.Fatal(err)
}
var nc, wc, rc uint64
for {
if v, err := d.Decode(); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
} else {
switch v := v.(type) {
case *osmpbf.Node:
// Process Node v.
nc++
case *osmpbf.Way:
// Process Way v.
wc++
case *osmpbf.Relation:
// Process Relation v.
rc++
default:
log.Fatalf("unknown type %T\n", v)
}
}
}
fmt.Printf("Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
// Output:
// Nodes: 2729006, Ways: 459055, Relations: 12833
}
func main() {
// configuration
config := getSettings()
// open pbf file
file := openFile(config.PbfPath)
defer file.Close()
decoder := osmpbf.NewDecoder(file)
err := decoder.Start(runtime.GOMAXPROCS(-1)) // use several goroutines for faster decoding
if err != nil {
log.Fatal(err)
}
db := openLevelDB(config.LevedbPath)
defer db.Close()
run(decoder, db, config)
}
func NewExcavator(pbfpath string) (ex *Excavator, err error) {
file, err := os.Open(pbfpath)
if err != nil {
return
}
/*
stat, err := file.Stat()
if err != nil {
return
}
est := stat.Size() >> 8
*/
pbf := osmpbf.NewDecoder(file)
ex = &Excavator{
pbfpath: pbfpath,
pbffile: file,
pbf: pbf,
nodes: make(chan *Node, BlockSize),
ways: make(chan *Way, BlockSize),
relations: make(chan *Relation, BlockSize),
errs: make(chan error),
dredgers: &sync.WaitGroup{},
couriers: &sync.WaitGroup{},
NodeCourier: func(c <-chan *Node) {
for range c {
}
},
WayCourier: func(c <-chan *Way) {
for range c {
}
},
RelationCourier: func(c <-chan *Relation) {
for range c {
}
},
}
return
}
func OsmConverter() {
// http://be.gis-lab.info/data/osm_dump/dump/latest/
f, err := os.Open("./pbf/RU-CHE.osm.pbf")
if err != nil {
log.Fatal(err)
}
defer f.Close()
d := osmpbf.NewDecoder(f)
err = d.Start(runtime.GOMAXPROCS(4))
if err != nil {
log.Fatal(err)
}
var nc, wc, rc uint64
for {
//log.Println("ln")
if v, err := d.Decode(); err == io.EOF {
log.Printf("%#v", v)
break
} else if err != nil {
log.Printf("%#v", v)
log.Fatal(err)
} else {
switch v := v.(type) {
case *osmpbf.Node:
// Process Node v.
if strings.TrimSpace(v.Tags["addr:country"]) != "" {
log.Println(v.Tags["name:ru"], v.Lat, v.Lon, v.Info)
strid := strconv.Itoa(int(v.ID))
points := gegeo.GeGeoPoint{
Id: strid,
Title: v.Tags["name:ru"],
Metadata: v.Tags,
Location: &gegeo.Location{
Latitude: v.Lat,
Longitude: v.Lon,
},
}
AddDocument(points)
}
nc++
case *osmpbf.Way:
// Process Way v.
//log.Println(v.Tags, v.Info, v.NodeIDs)
wc++
case *osmpbf.Relation:
// Process Relation v.
rc++
default:
log.Fatalf("unknown type %T\n", v)
}
}
}
log.Printf("Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
}
// implements the osm.Parser interface
func (p *Pbf) Parse() (o *osm.OSM, err error) {
d := osmpbf.NewDecoder(p.r)
err = d.Start(runtime.GOMAXPROCS(-1))
if err != nil {
return
}
o = osm.NewOSM()
var v interface{}
for {
if v, err = d.Decode(); err == io.EOF {
err = nil
break
} else if err != nil {
return
} else {
switch v := v.(type) {
case *osmpbf.Node:
var t tags.Tags
if v.Tags != nil && len(v.Tags) != 0 {
t = tags.Tags(v.Tags)
}
o.Nodes[v.ID] = &node.Node{
Id_: v.ID,
User_: user.New(int64(v.Info.Uid), v.Info.User),
Position_: point.New(v.Lat, v.Lon),
Timestamp_: v.Info.Timestamp,
Changeset_: v.Info.Changeset,
Version_: int64(v.Info.Version),
Visible_: v.Info.Visible,
Tags_: &t,
}
case *osmpbf.Way:
var t tags.Tags
if v.Tags != nil && len(v.Tags) != 0 {
t = tags.Tags(v.Tags)
}
w := &way.Way{
Id_: v.ID,
User_: user.New(int64(v.Info.Uid), v.Info.User),
Timestamp_: v.Info.Timestamp,
Changeset_: v.Info.Changeset,
Version_: int64(v.Info.Version),
Visible_: v.Info.Visible,
Tags_: &t,
}
var nd []*node.Node
for _, id := range v.NodeIDs {
n := o.Nodes[id]
if n == nil {
err = errors.New(fmt.Sprintf("Missing node #%d in way #%d", id, v.ID))
o = nil
return
}
nd = append(nd, n)
}
w.Nodes_ = nd
o.Ways[v.ID] = w
case *osmpbf.Relation:
var t tags.Tags
if v.Tags != nil && len(v.Tags) != 0 {
t = tags.Tags(v.Tags)
}
r := &relation.Relation{
Id_: v.ID,
User_: user.New(int64(v.Info.Uid), v.Info.User),
Timestamp_: v.Info.Timestamp,
Changeset_: v.Info.Changeset,
Version_: int64(v.Info.Version),
Visible_: v.Info.Visible,
Tags_: &t,
}
var members []*relation.Member
for _, m := range v.Members {
member := &relation.Member{Role: m.Role, Id_: m.ID}
switch m.Type {
case osmpbf.NodeType:
member.Type_ = item.TypeNode
member.Ref = o.GetNode(m.ID)
case osmpbf.WayType:
member.Type_ = item.TypeWay
member.Ref = o.GetWay(m.ID)
case osmpbf.RelationType:
member.Type_ = item.TypeRelation
member.Ref = o.GetRelation(m.ID)
}
if member.Ref == nil {
err = errors.New(fmt.Sprintf("Missing member #%d (%s) in way #%d", m.ID, member.Type(), v.ID))
o = nil
return
}
members = append(members, member)
}
r.Members_ = members
o.Relations[v.ID] = r
default:
log.Printf("ERROR: unknown type %T\n", v)
}
}
}
return
}
func main() {
f, err := os.Open("washington-latest.osm.pbf")
if err != nil {
log.Fatal(err)
os.Exit(1)
}
defer f.Close()
o, err := os.Create("washington-latest-highway.osm")
if err != nil {
log.Fatal(err)
os.Exit(1)
}
osm := goosm.NewOsm()
stat, _ := f.Stat()
size := stat.Size()
bar := pb.New64(size).SetUnits(pb.U_BYTES)
bar.ShowSpeed = true
bar.Start()
d := osmpbf.NewDecoder(bar.NewProxyReader(f))
err = d.Start(1)
if err != nil {
log.Fatal(err)
os.Exit(1)
}
var nc, wc, rc uint64
for {
if v, err := d.Decode(); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
} else {
switch t := v.(type) {
case *osmpbf.Node:
n := v.(*osmpbf.Node)
for k, _ := range n.Tags {
if k == "highway" {
osm.AppendNode(goosm.Node{
Id: fmt.Sprintf("%d", n.ID),
Lat: n.Lat,
Lon: n.Lon,
Visible: n.Info.Visible})
}
}
// Process Node v.
nc++
case *osmpbf.Way:
n := v.(*osmpbf.Way)
for k, _ := range n.Tags {
if k == "highway" {
way := goosm.NewWay()
for _, nd := range n.NodeIDs {
way.AppendNd(string(nd))
}
osm.AppendWay(*way)
wc++
}
}
case *osmpbf.Relation:
n := v.(*osmpbf.Relation)
for k, _ := range n.Tags {
if k == "highway" {
rc++
}
}
default:
log.Fatalf("unknown type %T\n", t)
}
}
}
fmt.Println()
fmt.Printf("Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
osm.Write(o)
}