func Update(oscFile string, geometryLimiter *limit.Limiter, force bool) {
state, err := diffstate.ParseFromOsc(oscFile)
if err != nil {
log.Fatal(err)
}
lastState, err := diffstate.ParseLastState(config.BaseOptions.CacheDir)
if err != nil {
log.Fatal(err)
}
if lastState != nil && lastState.Sequence != 0 && state != nil && state.Sequence <= lastState.Sequence {
if !force {
log.Warn(state, " already imported")
return
}
}
defer log.StopStep(log.StartStep(fmt.Sprintf("Processing %s", oscFile)))
elems, errc := parser.Parse(oscFile)
osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
err = osmCache.Open()
if err != nil {
log.Fatal("osm cache: ", err)
}
diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir)
err = diffCache.Open()
if err != nil {
log.Fatal("diff cache: ", err)
}
tagmapping, err := mapping.NewMapping(config.BaseOptions.MappingFile)
if err != nil {
log.Fatal(err)
}
connType := database.ConnectionType(config.BaseOptions.Connection)
dbConf := database.Config{
Type: connType,
ConnectionParams: config.BaseOptions.Connection,
Srid: config.BaseOptions.Srid,
}
db, err := database.Open(dbConf, tagmapping)
if err != nil {
log.Fatal("database open: ", err)
}
err = db.Begin()
if err != nil {
log.Fatal(err)
}
delDb, ok := db.(database.Deleter)
if !ok {
log.Fatal("database not deletable")
}
deleter := NewDeleter(
delDb,
osmCache,
diffCache,
tagmapping.PointMatcher(),
tagmapping.LineStringMatcher(),
tagmapping.PolygonMatcher(),
)
progress := stats.NewStatsReporter()
expiredTiles := expire.NewTiles(14)
relTagFilter := tagmapping.RelationTagFilter()
wayTagFilter := tagmapping.WayTagFilter()
nodeTagFilter := tagmapping.NodeTagFilter()
pointsTagMatcher := tagmapping.PointMatcher()
lineStringsTagMatcher := tagmapping.LineStringMatcher()
polygonsTagMatcher := tagmapping.PolygonMatcher()
relations := make(chan *element.Relation)
ways := make(chan *element.Way)
nodes := make(chan *element.Node)
relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
db, polygonsTagMatcher, progress, config.BaseOptions.Srid)
relWriter.SetLimiter(geometryLimiter)
relWriter.SetExpireTiles(expiredTiles)
relWriter.Start()
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
lineStringsTagMatcher, polygonsTagMatcher, progress, config.BaseOptions.Srid)
wayWriter.SetLimiter(geometryLimiter)
wayWriter.SetExpireTiles(expiredTiles)
wayWriter.Start()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
pointsTagMatcher, progress, config.BaseOptions.Srid)
nodeWriter.SetLimiter(geometryLimiter)
nodeWriter.Start()
nodeIds := make(map[int64]bool)
wayIds := make(map[int64]bool)
relIds := make(map[int64]bool)
step := log.StartStep("Parsing changes, updating cache and removing elements")
g := geos.NewGeos()
For:
for {
select {
case elem := <-elems:
if elem.Rel != nil {
relTagFilter.Filter(&elem.Rel.Tags)
progress.AddRelations(1)
} else if elem.Way != nil {
wayTagFilter.Filter(&elem.Way.Tags)
progress.AddWays(1)
} else if elem.Node != nil {
nodeTagFilter.Filter(&elem.Node.Tags)
if len(elem.Node.Tags) > 0 {
progress.AddNodes(1)
}
progress.AddCoords(1)
}
if elem.Del {
deleter.Delete(elem)
if !elem.Add {
if elem.Rel != nil {
if err := osmCache.Relations.DeleteRelation(elem.Rel.Id); err != nil {
log.Fatal(err)
}
} else if elem.Way != nil {
if err := osmCache.Ways.DeleteWay(elem.Way.Id); err != nil {
log.Fatal(err)
}
diffCache.Ways.Delete(elem.Way.Id)
} else if elem.Node != nil {
if err := osmCache.Nodes.DeleteNode(elem.Node.Id); err != nil {
log.Fatal(err)
}
if err := osmCache.Coords.DeleteCoord(elem.Node.Id); err != nil {
log.Fatal(err)
}
}
}
}
if elem.Add {
if elem.Rel != nil {
// check if first member is cached to avoid caching
// unneeded relations (typical outside of our coverage)
if memberIsCached(elem.Rel.Members, osmCache.Ways) {
osmCache.Relations.PutRelation(elem.Rel)
relIds[elem.Rel.Id] = true
}
} else if elem.Way != nil {
// check if first coord is cached to avoid caching
// unneeded ways (typical outside of our coverage)
if coordIsCached(elem.Way.Refs, osmCache.Coords) {
osmCache.Ways.PutWay(elem.Way)
wayIds[elem.Way.Id] = true
}
} else if elem.Node != nil {
if geometryLimiter == nil || geometryLimiter.IntersectsBuffer(g, elem.Node.Long, elem.Node.Lat) {
osmCache.Nodes.PutNode(elem.Node)
osmCache.Coords.PutCoords([]element.Node{*elem.Node})
nodeIds[elem.Node.Id] = true
}
}
}
case err := <-errc:
if err != io.EOF {
log.Fatal(err)
}
break For
}
}
progress.Stop()
log.StopStep(step)
step = log.StartStep("Writing added/modified elements")
progress = stats.NewStatsReporter()
for nodeId, _ := range nodeIds {
node, err := osmCache.Nodes.GetNode(nodeId)
if err != nil {
if err != cache.NotFound {
log.Print(node, err)
}
// missing nodes can still be Coords
// no `continue` here
}
if node != nil {
// insert new node
nodes <- node
}
dependers := diffCache.Coords.Get(nodeId)
// mark depending ways for (re)insert
for _, way := range dependers {
wayIds[way] = true
}
}
for wayId, _ := range wayIds {
way, err := osmCache.Ways.GetWay(wayId)
if err != nil {
if err != cache.NotFound {
log.Print(way, err)
}
continue
}
// insert new way
ways <- way
dependers := diffCache.Ways.Get(wayId)
// mark depending relations for (re)insert
for _, rel := range dependers {
relIds[rel] = true
}
}
for relId, _ := range relIds {
rel, err := osmCache.Relations.GetRelation(relId)
if err != nil {
if err != cache.NotFound {
log.Print(rel, err)
}
continue
}
// insert new relation
relations <- rel
}
close(relations)
close(ways)
close(nodes)
nodeWriter.Close()
relWriter.Close()
wayWriter.Close()
err = db.End()
if err != nil {
log.Fatal(err)
}
err = db.Close()
if err != nil {
log.Fatal(err)
}
osmCache.Close()
diffCache.Close()
log.StopStep(step)
step = log.StartStep("Updating expired tiles db")
expire.WriteTileExpireDb(
expiredTiles.SortedTiles(),
"/tmp/expire_tiles.db",
)
log.StopStep(step)
progress.Stop()
if state != nil {
err = diffstate.WriteLastState(config.BaseOptions.CacheDir, state)
if err != nil {
log.Warn(err) // warn only
}
}
}
func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics,
tagmapping *mapping.Mapping, pbfFile *pbf.Pbf,
limiter *limit.Limiter,
) {
nodes := make(chan []element.Node, 4)
coords := make(chan []element.Node, 4)
ways := make(chan []element.Way, 4)
relations := make(chan []element.Relation, 4)
withLimiter := false
if limiter != nil {
withLimiter = true
}
if pbfFile.Header.Time.Unix() != 0 {
log.Printf("reading %s with data till %v", pbfFile.Filename, pbfFile.Header.Time.Local())
}
parser := pbf.NewParser(pbfFile, coords, nodes, ways, relations)
coordsSynced := make(chan bool)
coordsSync := util.NewSyncPoint(int(nCoords+nNodes), func() {
coordsSynced <- true
})
parser.NotifyWays(func() {
for i := 0; int64(i) < nCoords; i++ {
coords <- nil
}
for i := 0; int64(i) < nNodes; i++ {
nodes <- nil
}
<-coordsSynced
})
waysSynced := make(chan bool)
waysSync := util.NewSyncPoint(int(nWays), func() {
waysSynced <- true
})
parser.NotifyRelations(func() {
for i := 0; int64(i) < nWays; i++ {
ways <- nil
}
<-waysSynced
})
parser.Start()
waitWriter := sync.WaitGroup{}
for i := 0; int64(i) < nWays; i++ {
waitWriter.Add(1)
go func() {
var skip, hit int
m := tagmapping.WayTagFilter()
for ws := range ways {
if ws == nil {
waysSync.Sync()
continue
}
if skipWays {
continue
}
for i, _ := range ws {
m.Filter(&ws[i].Tags)
if withLimiter {
if !cache.Coords.FirstRefIsCached(ws[i].Refs) {
ws[i].Id = osmcache.SKIP
skip += 1
} else {
hit += 1
}
}
}
cache.Ways.PutWays(ws)
progress.AddWays(len(ws))
}
waitWriter.Done()
}()
}
for i := 0; int64(i) < nRels; i++ {
waitWriter.Add(1)
go func() {
var skip, hit int
m := tagmapping.RelationTagFilter()
for rels := range relations {
numWithTags := 0
for i, _ := range rels {
m.Filter(&rels[i].Tags)
if len(rels[i].Tags) > 0 {
numWithTags += 1
}
if withLimiter {
if !cache.Ways.FirstMemberIsCached(rels[i].Members) {
skip += 1
rels[i].Id = osmcache.SKIP
} else {
hit += 1
}
}
}
cache.Relations.PutRelations(rels)
progress.AddRelations(numWithTags)
}
waitWriter.Done()
}()
}
for i := 0; int64(i) < nCoords; i++ {
waitWriter.Add(1)
go func() {
var skip, hit int
g := geos.NewGeos()
defer g.Finish()
for nds := range coords {
if nds == nil {
coordsSync.Sync()
continue
}
if withLimiter {
for i, _ := range nds {
nd := element.Node{Long: nds[i].Long, Lat: nds[i].Lat}
proj.NodeToMerc(&nd)
if !limiter.IntersectsBuffer(g, nd.Long, nd.Lat) {
skip += 1
nds[i].Id = osmcache.SKIP
} else {
hit += 1
}
}
}
cache.Coords.PutCoords(nds)
progress.AddCoords(len(nds))
}
waitWriter.Done()
}()
}
for i := 0; int64(i) < nNodes; i++ {
waitWriter.Add(1)
go func() {
g := geos.NewGeos()
defer g.Finish()
m := tagmapping.NodeTagFilter()
for nds := range nodes {
if nds == nil {
coordsSync.Sync()
continue
}
numWithTags := 0
for i, _ := range nds {
m.Filter(&nds[i].Tags)
if len(nds[i].Tags) > 0 {
numWithTags += 1
}
if withLimiter {
nd := element.Node{Long: nds[i].Long, Lat: nds[i].Lat}
proj.NodeToMerc(&nd)
if !limiter.IntersectsBuffer(g, nd.Long, nd.Lat) {
nds[i].Id = osmcache.SKIP
}
}
}
cache.Nodes.PutNodes(nds)
progress.AddNodes(numWithTags)
}
waitWriter.Done()
}()
}
parser.Close()
close(relations)
close(ways)
close(nodes)
close(coords)
waitWriter.Wait()
}
func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expireor, osmCache *cache.OSMCache, diffCache *cache.DiffCache, force bool) error {
state, err := diffstate.ParseFromOsc(oscFile)
if err != nil {
return err
}
lastState, err := diffstate.ParseLastState(config.BaseOptions.DiffDir)
if err != nil {
log.Warn(err)
}
if lastState != nil && lastState.Sequence != 0 && state != nil && state.Sequence <= lastState.Sequence {
if !force {
log.Warn(state, " already imported")
return nil
}
}
defer log.StopStep(log.StartStep(fmt.Sprintf("Processing %s", oscFile)))
elems, errc := parser.Parse(oscFile)
tagmapping, err := mapping.NewMapping(config.BaseOptions.MappingFile)
if err != nil {
return err
}
dbConf := database.Config{
ConnectionParams: config.BaseOptions.Connection,
Srid: config.BaseOptions.Srid,
// we apply diff imports on the Production schema
ImportSchema: config.BaseOptions.Schemas.Production,
ProductionSchema: config.BaseOptions.Schemas.Production,
BackupSchema: config.BaseOptions.Schemas.Backup,
}
db, err := database.Open(dbConf, tagmapping)
if err != nil {
return errors.New("database open: " + err.Error())
}
defer db.Close()
err = db.Begin()
if err != nil {
return err
}
delDb, ok := db.(database.Deleter)
if !ok {
return errors.New("database not deletable")
}
genDb, ok := db.(database.Generalizer)
if ok {
genDb.EnableGeneralizeUpdates()
}
deleter := NewDeleter(
delDb,
osmCache,
diffCache,
tagmapping.PointMatcher(),
tagmapping.LineStringMatcher(),
tagmapping.PolygonMatcher(),
)
progress := stats.NewStatsReporter()
relTagFilter := tagmapping.RelationTagFilter()
wayTagFilter := tagmapping.WayTagFilter()
nodeTagFilter := tagmapping.NodeTagFilter()
relations := make(chan *element.Relation)
ways := make(chan *element.Way)
nodes := make(chan *element.Node)
relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
db, progress, config.BaseOptions.Srid)
relWriter.SetLimiter(geometryLimiter)
relWriter.SetExpireor(expireor)
relWriter.Start()
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
progress, config.BaseOptions.Srid)
wayWriter.SetLimiter(geometryLimiter)
wayWriter.SetExpireor(expireor)
wayWriter.Start()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
progress, config.BaseOptions.Srid)
nodeWriter.SetLimiter(geometryLimiter)
nodeWriter.SetExpireor(expireor)
nodeWriter.Start()
nodeIds := make(map[int64]bool)
wayIds := make(map[int64]bool)
relIds := make(map[int64]bool)
step := log.StartStep("Parsing changes, updating cache and removing elements")
g := geos.NewGeos()
For:
for {
select {
case elem := <-elems:
if elem.Rel != nil {
relTagFilter.Filter(&elem.Rel.Tags)
progress.AddRelations(1)
} else if elem.Way != nil {
wayTagFilter.Filter(&elem.Way.Tags)
progress.AddWays(1)
} else if elem.Node != nil {
nodeTagFilter.Filter(&elem.Node.Tags)
if len(elem.Node.Tags) > 0 {
progress.AddNodes(1)
}
progress.AddCoords(1)
}
if elem.Del {
if err := deleter.Delete(elem); err != nil {
return err
}
if !elem.Add {
if elem.Rel != nil {
if err := osmCache.Relations.DeleteRelation(elem.Rel.Id); err != nil {
return err
}
} else if elem.Way != nil {
if err := osmCache.Ways.DeleteWay(elem.Way.Id); err != nil {
return err
}
diffCache.Ways.Delete(elem.Way.Id)
} else if elem.Node != nil {
if err := osmCache.Nodes.DeleteNode(elem.Node.Id); err != nil {
return err
}
if err := osmCache.Coords.DeleteCoord(elem.Node.Id); err != nil {
return err
}
}
}
}
if elem.Add {
if elem.Rel != nil {
// check if first member is cached to avoid caching
// unneeded relations (typical outside of our coverage)
if osmCache.Ways.FirstMemberIsCached(elem.Rel.Members) {
osmCache.Relations.PutRelation(elem.Rel)
relIds[elem.Rel.Id] = true
}
} else if elem.Way != nil {
// check if first coord is cached to avoid caching
// unneeded ways (typical outside of our coverage)
if osmCache.Coords.FirstRefIsCached(elem.Way.Refs) {
osmCache.Ways.PutWay(elem.Way)
wayIds[elem.Way.Id] = true
}
} else if elem.Node != nil {
addNode := true
if geometryLimiter != nil {
nd := element.Node{Long: elem.Node.Long, Lat: elem.Node.Lat}
proj.NodeToMerc(&nd)
if !geometryLimiter.IntersectsBuffer(g, nd.Long, nd.Lat) {
addNode = false
}
}
if addNode {
osmCache.Nodes.PutNode(elem.Node)
osmCache.Coords.PutCoords([]element.Node{*elem.Node})
nodeIds[elem.Node.Id] = true
}
}
}
case err := <-errc:
if err != io.EOF {
return err
}
break For
}
}
// mark member ways from deleted relations for re-insert
for id, _ := range deleter.DeletedMemberWays() {
wayIds[id] = true
}
progress.Stop()
log.StopStep(step)
step = log.StartStep("Writing added/modified elements")
progress = stats.NewStatsReporter()
// mark depending ways for (re)insert
for nodeId, _ := range nodeIds {
dependers := diffCache.Coords.Get(nodeId)
for _, way := range dependers {
wayIds[way] = true
}
}
// mark depending relations for (re)insert
for wayId, _ := range wayIds {
dependers := diffCache.Ways.Get(wayId)
// mark depending relations for (re)insert
for _, rel := range dependers {
relIds[rel] = true
}
}
for relId, _ := range relIds {
rel, err := osmCache.Relations.GetRelation(relId)
if err != nil {
if err != cache.NotFound {
log.Print(rel, err)
}
continue
}
// insert new relation
progress.AddRelations(1)
relations <- rel
}
for wayId, _ := range wayIds {
way, err := osmCache.Ways.GetWay(wayId)
if err != nil {
if err != cache.NotFound {
log.Print(way, err)
}
continue
}
// insert new way
progress.AddWays(1)
ways <- way
}
for nodeId, _ := range nodeIds {
node, err := osmCache.Nodes.GetNode(nodeId)
if err != nil {
if err != cache.NotFound {
log.Print(node, err)
}
// missing nodes can still be Coords
// no `continue` here
}
if node != nil {
// insert new node
progress.AddNodes(1)
nodes <- node
}
}
close(relations)
close(ways)
close(nodes)
nodeWriter.Wait()
relWriter.Wait()
wayWriter.Wait()
if genDb != nil {
genDb.GeneralizeUpdates()
}
err = db.End()
if err != nil {
return err
}
err = db.Close()
if err != nil {
return err
}
log.StopStep(step)
progress.Stop()
if state != nil {
if lastState != nil {
state.Url = lastState.Url
}
err = diffstate.WriteLastState(config.BaseOptions.DiffDir, state)
if err != nil {
log.Warn(err) // warn only
}
}
return nil
}