Esempio n. 1
0
File: grob.go Progetto: wmo/grob
func main() {
	debug := false // switch to true for a bit of extra information

	// grok the command line args
	filename, dist, lat, lon, pattern := grok_args(os.Args)

	if debug {
		fmt.Printf("# d,lat,lon,'pattern': %.3f, %f,%f,'%s'\n", dist, lat, lon, pattern)
	}

	// start reading the file
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	d := osmpbf.NewDecoder(f)
	err = d.Start(runtime.GOMAXPROCS(-1))
	if err != nil {
		log.Fatal(err)
	}

	var nc, wc, rc uint64
	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		} else {
			switch v := v.(type) {
			case *osmpbf.Node:
				handleNode(*v, dist, lat, lon, pattern)
				//handleNode(*v, lat, lon, dist)
				nc++
			case *osmpbf.Way:
				// Process Way v.
				wc++
			case *osmpbf.Relation:
				// Process Relation v.
				rc++
			default:
				log.Fatalf("unknown type %T\n", v)
			}
		}
	}
	if debug {
		fmt.Printf("# Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
	}
}
Esempio n. 2
0
func ParsePbf(path string) (*OsmData, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	d := osmpbf.NewDecoder(f)
	err = d.Start(10)
	if err != nil {
		return nil, err
	}

	data := &OsmData{
		Nodes:    map[int64]Node{},
		Ways:     map[int64]Way{},
		Features: map[string][]FeatureRef{},
	}

	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		} else {
			switch v := v.(type) {
			case *osmpbf.Node:
				node := NodeFromPbf(v)
				data.Nodes[node.Id] = node
			case *osmpbf.Way:
				way := WayFromPbf(v)
				fName, ok := way.MatchAny(mapFeatures)
				if ok {
					data.Ways[way.Id] = way
					data.Features[fName] = append(data.Features[fName], FeatureRef{way.Id, ItemTypeWay})
				}
			case *osmpbf.Relation:
				// Ignore
			default:
				return nil, fmt.Errorf("unknown type %T", v)
			}
		}
	}

	return data, nil
}
Esempio n. 3
0
func Example() {
	f, err := os.Open("greater-london-140324.osm.pbf")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	d := osmpbf.NewDecoder(f)

	// use more memory from the start, it is faster
	d.SetBufferSize(osmpbf.MaxBlobSize)

	// start decoding with several goroutines, it is faster
	err = d.Start(runtime.GOMAXPROCS(-1))
	if err != nil {
		log.Fatal(err)
	}

	var nc, wc, rc uint64
	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		} else {
			switch v := v.(type) {
			case *osmpbf.Node:
				// Process Node v.
				nc++
			case *osmpbf.Way:
				// Process Way v.
				wc++
			case *osmpbf.Relation:
				// Process Relation v.
				rc++
			default:
				log.Fatalf("unknown type %T\n", v)
			}
		}
	}

	fmt.Printf("Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
	// Output:
	// Nodes: 2729006, Ways: 459055, Relations: 12833
}
Esempio n. 4
0
func main() {

	// configuration
	config := getSettings()

	// open pbf file
	file := openFile(config.PbfPath)
	defer file.Close()

	decoder := osmpbf.NewDecoder(file)
	err := decoder.Start(runtime.GOMAXPROCS(-1)) // use several goroutines for faster decoding
	if err != nil {
		log.Fatal(err)
	}

	db := openLevelDB(config.LevedbPath)
	defer db.Close()

	run(decoder, db, config)
}
Esempio n. 5
0
func NewExcavator(pbfpath string) (ex *Excavator, err error) {
	file, err := os.Open(pbfpath)
	if err != nil {
		return
	}
	/*
		stat, err := file.Stat()
		if err != nil {
			return
		}
		est := stat.Size() >> 8
	*/
	pbf := osmpbf.NewDecoder(file)
	ex = &Excavator{
		pbfpath:   pbfpath,
		pbffile:   file,
		pbf:       pbf,
		nodes:     make(chan *Node, BlockSize),
		ways:      make(chan *Way, BlockSize),
		relations: make(chan *Relation, BlockSize),
		errs:      make(chan error),
		dredgers:  &sync.WaitGroup{},
		couriers:  &sync.WaitGroup{},
		NodeCourier: func(c <-chan *Node) {
			for range c {
			}
		},
		WayCourier: func(c <-chan *Way) {
			for range c {
			}
		},
		RelationCourier: func(c <-chan *Relation) {
			for range c {
			}
		},
	}
	return
}
Esempio n. 6
0
func OsmConverter() {

	// http://be.gis-lab.info/data/osm_dump/dump/latest/
	f, err := os.Open("./pbf/RU-CHE.osm.pbf")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	d := osmpbf.NewDecoder(f)
	err = d.Start(runtime.GOMAXPROCS(4))
	if err != nil {
		log.Fatal(err)
	}

	var nc, wc, rc uint64
	for {
		//log.Println("ln")
		if v, err := d.Decode(); err == io.EOF {
			log.Printf("%#v", v)
			break
		} else if err != nil {
			log.Printf("%#v", v)
			log.Fatal(err)
		} else {

			switch v := v.(type) {
			case *osmpbf.Node:
				// Process Node v.

				if strings.TrimSpace(v.Tags["addr:country"]) != "" {
					log.Println(v.Tags["name:ru"], v.Lat, v.Lon, v.Info)

					strid := strconv.Itoa(int(v.ID))

					points := gegeo.GeGeoPoint{
						Id:       strid,
						Title:    v.Tags["name:ru"],
						Metadata: v.Tags,
						Location: &gegeo.Location{
							Latitude:  v.Lat,
							Longitude: v.Lon,
						},
					}

					AddDocument(points)
				}

				nc++
			case *osmpbf.Way:
				// Process Way v.
				//log.Println(v.Tags, v.Info, v.NodeIDs)

				wc++
			case *osmpbf.Relation:
				// Process Relation v.
				rc++
			default:
				log.Fatalf("unknown type %T\n", v)
			}

		}
	}

	log.Printf("Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
}
Esempio n. 7
0
// implements the osm.Parser interface
func (p *Pbf) Parse() (o *osm.OSM, err error) {
	d := osmpbf.NewDecoder(p.r)

	err = d.Start(runtime.GOMAXPROCS(-1))
	if err != nil {
		return
	}

	o = osm.NewOSM()
	var v interface{}
	for {
		if v, err = d.Decode(); err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return
		} else {
			switch v := v.(type) {
			case *osmpbf.Node:
				var t tags.Tags
				if v.Tags != nil && len(v.Tags) != 0 {
					t = tags.Tags(v.Tags)
				}
				o.Nodes[v.ID] = &node.Node{
					Id_:        v.ID,
					User_:      user.New(int64(v.Info.Uid), v.Info.User),
					Position_:  point.New(v.Lat, v.Lon),
					Timestamp_: v.Info.Timestamp,
					Changeset_: v.Info.Changeset,
					Version_:   int64(v.Info.Version),
					Visible_:   v.Info.Visible,
					Tags_:      &t,
				}
			case *osmpbf.Way:
				var t tags.Tags
				if v.Tags != nil && len(v.Tags) != 0 {
					t = tags.Tags(v.Tags)
				}
				w := &way.Way{
					Id_:        v.ID,
					User_:      user.New(int64(v.Info.Uid), v.Info.User),
					Timestamp_: v.Info.Timestamp,
					Changeset_: v.Info.Changeset,
					Version_:   int64(v.Info.Version),
					Visible_:   v.Info.Visible,
					Tags_:      &t,
				}
				var nd []*node.Node
				for _, id := range v.NodeIDs {
					n := o.Nodes[id]
					if n == nil {
						err = errors.New(fmt.Sprintf("Missing node #%d in way #%d", id, v.ID))
						o = nil
						return
					}
					nd = append(nd, n)
				}
				w.Nodes_ = nd
				o.Ways[v.ID] = w
			case *osmpbf.Relation:
				var t tags.Tags
				if v.Tags != nil && len(v.Tags) != 0 {
					t = tags.Tags(v.Tags)
				}
				r := &relation.Relation{
					Id_:        v.ID,
					User_:      user.New(int64(v.Info.Uid), v.Info.User),
					Timestamp_: v.Info.Timestamp,
					Changeset_: v.Info.Changeset,
					Version_:   int64(v.Info.Version),
					Visible_:   v.Info.Visible,
					Tags_:      &t,
				}
				var members []*relation.Member
				for _, m := range v.Members {
					member := &relation.Member{Role: m.Role, Id_: m.ID}
					switch m.Type {
					case osmpbf.NodeType:
						member.Type_ = item.TypeNode
						member.Ref = o.GetNode(m.ID)
					case osmpbf.WayType:
						member.Type_ = item.TypeWay
						member.Ref = o.GetWay(m.ID)
					case osmpbf.RelationType:
						member.Type_ = item.TypeRelation
						member.Ref = o.GetRelation(m.ID)
					}
					if member.Ref == nil {
						err = errors.New(fmt.Sprintf("Missing member #%d (%s) in way #%d", m.ID, member.Type(), v.ID))
						o = nil
						return
					}
					members = append(members, member)
				}
				r.Members_ = members
				o.Relations[v.ID] = r
			default:
				log.Printf("ERROR: unknown type %T\n", v)
			}
		}
	}
	return
}
Esempio n. 8
0
func main() {
	f, err := os.Open("washington-latest.osm.pbf")
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}
	defer f.Close()

	o, err := os.Create("washington-latest-highway.osm")
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	osm := goosm.NewOsm()

	stat, _ := f.Stat()
	size := stat.Size()
	bar := pb.New64(size).SetUnits(pb.U_BYTES)
	bar.ShowSpeed = true
	bar.Start()

	d := osmpbf.NewDecoder(bar.NewProxyReader(f))
	err = d.Start(1)
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	var nc, wc, rc uint64
	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		} else {
			switch t := v.(type) {
			case *osmpbf.Node:
				n := v.(*osmpbf.Node)
				for k, _ := range n.Tags {
					if k == "highway" {
						osm.AppendNode(goosm.Node{
							Id:      fmt.Sprintf("%d", n.ID),
							Lat:     n.Lat,
							Lon:     n.Lon,
							Visible: n.Info.Visible})
					}
				}
				// Process Node v.
				nc++
			case *osmpbf.Way:
				n := v.(*osmpbf.Way)
				for k, _ := range n.Tags {
					if k == "highway" {
						way := goosm.NewWay()
						for _, nd := range n.NodeIDs {
							way.AppendNd(string(nd))
						}
						osm.AppendWay(*way)
						wc++
					}
				}
			case *osmpbf.Relation:
				n := v.(*osmpbf.Relation)
				for k, _ := range n.Tags {
					if k == "highway" {
						rc++
					}
				}
			default:
				log.Fatalf("unknown type %T\n", t)
			}
		}
	}
	fmt.Println()
	fmt.Printf("Nodes: %d, Ways: %d, Relations: %d\n", nc, wc, rc)
	osm.Write(o)
}