func (s *importTestSuite) diffCache(t *testing.T) *cache.DiffCache {
c := cache.NewDiffCache(s.config.cacheDir)
if err := c.Open(); err != nil {
t.Fatal(err)
}
return c
}
func Query(args []string) {
flags.Usage = Usage
if len(args) == 0 {
Usage()
}
err := flags.Parse(args)
if err != nil {
log.Fatal(err)
}
log.SetFlags(0)
log.SetOutput(os.Stdout)
osmCache := cache.NewOSMCache(*cachedir)
err = osmCache.Open()
if err != nil {
log.Fatal(err)
}
diffCache := cache.NewDiffCache(*cachedir)
err = diffCache.Open()
if err != nil {
log.Fatal(err)
}
if *full && *deps {
log.Fatal("cannot use -full and -deps option together")
}
result := result{}
if *relIds != "" {
ids := splitIds(*relIds)
result.Relations = collectRelations(osmCache, ids, *full)
}
if *wayIds != "" {
ids := splitIds(*wayIds)
result.Ways = collectWays(osmCache, diffCache, ids, *full, *deps)
}
if *nodeIds != "" {
ids := splitIds(*nodeIds)
result.Nodes = collectNodes(osmCache, diffCache, ids, *deps)
}
printJson(result)
}
func Diff() {
if config.BaseOptions.Quiet {
logging.SetQuiet(true)
}
var geometryLimiter *limit.Limiter
if config.BaseOptions.LimitTo != "" {
var err error
step := log.StartStep("Reading limitto geometries")
geometryLimiter, err = limit.NewFromGeoJSON(
config.BaseOptions.LimitTo,
config.BaseOptions.LimitToCacheBuffer,
config.BaseOptions.Srid,
)
if err != nil {
log.Fatal(err)
}
log.StopStep(step)
}
osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
err := osmCache.Open()
if err != nil {
log.Fatal("osm cache: ", err)
}
defer osmCache.Close()
diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir)
err = diffCache.Open()
if err != nil {
log.Fatal("diff cache: ", err)
}
for _, oscFile := range config.DiffFlags.Args() {
err := Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false)
if err != nil {
osmCache.Close()
diffCache.Close()
log.Fatalf("unable to process %s: %v", oscFile, err)
}
}
// explicitly Close since os.Exit prevents defers
osmCache.Close()
diffCache.Close()
}
func Import() {
if config.BaseOptions.Quiet {
logging.SetQuiet(true)
}
if (config.ImportOptions.Write || config.ImportOptions.Read != "") && (config.ImportOptions.RevertDeploy || config.ImportOptions.RemoveBackup) {
log.Fatal("-revertdeploy and -removebackup not compatible with -read/-write")
}
if config.ImportOptions.RevertDeploy && (config.ImportOptions.RemoveBackup || config.ImportOptions.DeployProduction) {
log.Fatal("-revertdeploy not compatible with -deployproduction/-removebackup")
}
var geometryLimiter *limit.Limiter
if (config.ImportOptions.Write || config.ImportOptions.Read != "") && config.BaseOptions.LimitTo != "" {
var err error
step := log.StartStep("Reading limitto geometries")
geometryLimiter, err = limit.NewFromGeoJSON(
config.BaseOptions.LimitTo,
config.BaseOptions.LimitToCacheBuffer,
config.BaseOptions.Srid,
)
if err != nil {
log.Fatal(err)
}
log.StopStep(step)
}
tagmapping, err := mapping.NewMapping(config.BaseOptions.MappingFile)
if err != nil {
log.Fatal("mapping file: ", err)
}
var db database.DB
if config.ImportOptions.Write || config.ImportOptions.DeployProduction || config.ImportOptions.RevertDeploy || config.ImportOptions.RemoveBackup || config.ImportOptions.Optimize {
if config.BaseOptions.Connection == "" {
log.Fatal("missing connection option")
}
conf := database.Config{
ConnectionParams: config.BaseOptions.Connection,
Srid: config.BaseOptions.Srid,
ImportSchema: config.BaseOptions.Schemas.Import,
ProductionSchema: config.BaseOptions.Schemas.Production,
BackupSchema: config.BaseOptions.Schemas.Backup,
}
db, err = database.Open(conf, tagmapping)
if err != nil {
log.Fatal(err)
}
defer db.Close()
}
osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
if config.ImportOptions.Read != "" && osmCache.Exists() {
if config.ImportOptions.Overwritecache {
log.Printf("removing existing cache %s", config.BaseOptions.CacheDir)
err := osmCache.Remove()
if err != nil {
log.Fatal("unable to remove cache:", err)
}
} else if !config.ImportOptions.Appendcache {
log.Fatal("cache already exists use -appendcache or -overwritecache")
}
}
step := log.StartStep("Imposm")
var elementCounts *stats.ElementCounts
if config.ImportOptions.Read != "" {
step := log.StartStep("Reading OSM data")
err = osmCache.Open()
if err != nil {
log.Fatal(err)
}
progress := stats.NewStatsReporter()
pbfFile, err := pbf.Open(config.ImportOptions.Read)
if err != nil {
log.Fatal(err)
}
if !config.ImportOptions.Appendcache {
// enable optimization if we don't append to existing cache
osmCache.Coords.SetLinearImport(true)
}
readLimiter := geometryLimiter
if config.BaseOptions.LimitToCacheBuffer == 0.0 {
readLimiter = nil
}
reader.ReadPbf(osmCache, progress, tagmapping,
pbfFile, readLimiter)
osmCache.Coords.SetLinearImport(false)
elementCounts = progress.Stop()
osmCache.Close()
log.StopStep(step)
if config.ImportOptions.Diff {
diffstate := state.FromPbf(pbfFile, config.ImportOptions.DiffStateBefore)
if diffstate != nil {
os.MkdirAll(config.BaseOptions.DiffDir, 0755)
err := diffstate.WriteToFile(path.Join(config.BaseOptions.DiffDir, "last.state.txt"))
if err != nil {
log.Print("error writing last.state.txt: ", err)
}
}
}
}
if config.ImportOptions.Write {
stepImport := log.StartStep("Importing OSM data")
stepWrite := log.StartStep("Writing OSM data")
progress := stats.NewStatsReporterWithEstimate(elementCounts)
err = db.Init()
if err != nil {
log.Fatal(err)
}
bulkDb, ok := db.(database.BulkBeginner)
if ok {
err = bulkDb.BeginBulk()
} else {
err = db.Begin()
}
if err != nil {
log.Fatal(err)
}
var diffCache *cache.DiffCache
if config.ImportOptions.Diff {
diffCache = cache.NewDiffCache(config.BaseOptions.CacheDir)
if err = diffCache.Remove(); err != nil {
log.Fatal(err)
}
if err = diffCache.Open(); err != nil {
log.Fatal(err)
}
}
err = osmCache.Open()
if err != nil {
log.Fatal(err)
}
if diffCache != nil {
diffCache.Coords.SetLinearImport(true)
diffCache.Ways.SetLinearImport(true)
}
osmCache.Coords.SetReadOnly(true)
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
relations,
db, progress,
tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(),
config.BaseOptions.Srid)
relWriter.SetLimiter(geometryLimiter)
relWriter.EnableConcurrent()
relWriter.Start()
relWriter.Wait() // blocks till the Relations.Iter() finishes
osmCache.Relations.Close()
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
ways, db,
progress,
tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(),
config.BaseOptions.Srid)
wayWriter.SetLimiter(geometryLimiter)
wayWriter.EnableConcurrent()
wayWriter.Start()
wayWriter.Wait() // blocks till the Ways.Iter() finishes
osmCache.Ways.Close()
nodes := osmCache.Nodes.Iter()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
progress,
tagmapping.PointMatcher(),
config.BaseOptions.Srid)
nodeWriter.SetLimiter(geometryLimiter)
nodeWriter.EnableConcurrent()
nodeWriter.Start()
nodeWriter.Wait() // blocks till the Nodes.Iter() finishes
osmCache.Close()
err = db.End()
if err != nil {
log.Fatal(err)
}
progress.Stop()
if config.ImportOptions.Diff {
diffCache.Close()
}
log.StopStep(stepWrite)
if db, ok := db.(database.Generalizer); ok {
if err := db.Generalize(); err != nil {
log.Fatal(err)
}
} else {
log.Fatal("database not generalizeable")
}
if db, ok := db.(database.Finisher); ok {
if err := db.Finish(); err != nil {
log.Fatal(err)
}
} else {
log.Fatal("database not finishable")
}
log.StopStep(stepImport)
}
if config.ImportOptions.Optimize {
if db, ok := db.(database.Optimizer); ok {
if err := db.Optimize(); err != nil {
log.Fatal(err)
}
} else {
log.Fatal("database not optimizable")
}
}
if config.ImportOptions.DeployProduction {
if db, ok := db.(database.Deployer); ok {
if err := db.Deploy(); err != nil {
log.Fatal(err)
}
} else {
log.Fatal("database not deployable")
}
}
if config.ImportOptions.RevertDeploy {
if db, ok := db.(database.Deployer); ok {
if err := db.RevertDeploy(); err != nil {
log.Fatal(err)
}
} else {
log.Fatal("database not deployable")
}
}
if config.ImportOptions.RemoveBackup {
if db, ok := db.(database.Deployer); ok {
if err := db.RemoveBackup(); err != nil {
log.Fatal(err)
}
} else {
log.Fatal("database not deployable")
}
}
log.StopStep(step)
}
func Main(usage func()) {
golog.SetFlags(golog.LstdFlags | golog.Lshortfile)
if os.Getenv("GOMAXPROCS") == "" {
runtime.GOMAXPROCS(runtime.NumCPU())
}
if len(os.Args) <= 1 {
usage()
logging.Shutdown()
os.Exit(1)
}
switch os.Args[1] {
case "import":
config.ParseImport(os.Args[2:])
if config.BaseOptions.Httpprofile != "" {
stats.StartHttpPProf(config.BaseOptions.Httpprofile)
}
import_.Import()
case "diff":
config.ParseDiffImport(os.Args[2:])
if config.BaseOptions.Httpprofile != "" {
stats.StartHttpPProf(config.BaseOptions.Httpprofile)
}
if config.BaseOptions.Quiet {
logging.SetQuiet(true)
}
var geometryLimiter *limit.Limiter
if config.BaseOptions.LimitTo != "" {
var err error
step := log.StartStep("Reading limitto geometries")
geometryLimiter, err = limit.NewFromGeoJsonWithBuffered(
config.BaseOptions.LimitTo,
config.BaseOptions.LimitToCacheBuffer,
)
if err != nil {
log.Fatal(err)
}
log.StopStep(step)
}
osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
err := osmCache.Open()
if err != nil {
log.Fatal("osm cache: ", err)
}
defer osmCache.Close()
diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir)
err = diffCache.Open()
if err != nil {
log.Fatal("diff cache: ", err)
}
for _, oscFile := range config.DiffFlags.Args() {
err := diff.Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false)
if err != nil {
osmCache.Close()
diffCache.Close()
log.Fatal(err)
}
}
// explicitly Close since os.Exit prevents defers
osmCache.Close()
diffCache.Close()
case "query-cache":
query.Query(os.Args[2:])
case "version":
fmt.Println(Version)
os.Exit(0)
default:
usage()
log.Fatalf("invalid command: '%s'", os.Args[1])
}
logging.Shutdown()
os.Exit(0)
}