Example #1
0
func createGobs(targetDir string) {
	var makeGob = func(bmap interface{}, ptr interface{}) interface{} {
		dbutil.BsonMapToObject(bmap, ptr)
		return ptr
	}
	var makeAdminGob = func(bmap interface{}) interface{} {
		return makeGob(bmap, &geoutil.GeoNamesAdminRecord{})
	}
	var makeNameGob = func(bmap interface{}) interface{} {
		return makeGob(bmap, &geoutil.GeoNamesNameRecord{})
	}
	var makeZipGob = func(bmap interface{}) interface{} {
		return makeGob(bmap, &geoutil.GeoNamesZipRecord{})
	}
	var lola numutil.Dvec2
	var box [2][2]float64
	var dbConn *mgo.Session
	var dbName, fbName, fp string
	var geoRecs []interface{}
	dbutil.Panic = true
	dbConn, _ = dbutil.ConnectToGlobal()
	defer dbConn.Close()
	dbName = dbutil.GeoNamesDbName(dbConn, true)
	dbutil.FindAll(dbConn, dbName, "a", nil, &geoRecs)
	gobutil.CreateGobsFile(path.Join(targetDir, "ga"), &geoRecs, makeAdminGob, true)
	for lola.Y = geoutil.LatMin; lola.Y < geoutil.LatMax; lola.Y++ {
		for lola.X = geoutil.LonMin; lola.X < geoutil.LonMax; lola.X++ {
			fbName = geoutil.LoLaFileName(lola.X, lola.Y)
			fmt.Println(fbName)
			geoRecs = nil
			box[0][0] = lola.X
			box[0][1] = lola.Y
			box[1][0] = lola.X + 1
			box[1][1] = lola.Y + 1
			dbutil.FindAll(dbConn, dbName, "n", bson.M{"l": bson.M{"$within": bson.M{"$box": box}}}, &geoRecs)
			if len(geoRecs) == 0 {
				geoRecs = nil
				dbutil.FindOne(dbConn, dbName, "n", bson.M{"l": bson.M{"$near": []float64{lola.X + 0.5, lola.Y + 0.5}}}, &geoRecs)
			}
			fp = path.Join(path.Join(targetDir, "gn"), fbName)
			fmt.Println(fp)
			gobutil.CreateGobsFile(fp, &geoRecs, makeNameGob, true)
			geoRecs = nil
			dbutil.FindAll(dbConn, dbName, "z", bson.M{"l": bson.M{"$within": bson.M{"$box": box}}}, &geoRecs)
			if len(geoRecs) > 0 {
				fp = path.Join(path.Join(targetDir, "gz"), fbName)
				fmt.Println(fp)
				gobutil.CreateGobsFile(fp, &geoRecs, makeZipGob, true)
			}
		}
	}
}
Example #2
0
func main() {
	runtime.GOMAXPROCS(8)
	flag.Parse()
	if *flagFetch != "" {
		fetch(*flagFetch)
	}
	if *flagMakeDb != "" {
		createDb(*flagMakeDb)
	}
	if *flagMakeElev != "" {
		createElevs(*flagMakeElev)
	}
	if *flagMakeGobs != "" {
		createGobs(*flagMakeGobs)
	}
	if *flagTest != "" {
		type tmpType struct {
			name     string
			lo       float64
			la       float64
			hasOld   bool
			hasNew   bool
			hasBathy bool
			hasSrtm  bool
			hasRmw   bool
			hasVfp   bool
		}
		var nold, nnew, nbathy, nsrtm, nrmw, nvfp, tc = 0, 0, 0, 0, 0, 0, 0
		var tmps []tmpType
		var tmp tmpType
		var missings []string
		for lat := geoutil.LatMin; lat < geoutil.LatMax; lat++ {
			for lon := geoutil.LonMin; lon < geoutil.LonMax; lon++ {
				tmp.lo = lon
				tmp.la = lat
				tmp.name = geoutil.LoLaFileName(lon, lat)
				if tmp.hasOld = fileutil.FileExists("/ssd1/m/oe/" + tmp.name + ".png"); tmp.hasOld {
					nold++
				}
				if tmp.hasNew = fileutil.FileExists("/ssd1/m/e/" + tmp.name); tmp.hasNew {
					nnew++
				}
				if tmp.hasOld && !tmp.hasNew {
					missings = append(missings, tmp.name)
				}
				if tmp.hasBathy = fileutil.FileExists("/media/hdx/elev_raw/bathy/hgt/" + tmp.name + ".hgt.le"); tmp.hasBathy {
					nbathy++
				}
				if tmp.hasSrtm = fileutil.FileExists("/media/hdx/elev_raw/srtm/hgt/" + tmp.name + ".hgt.le"); tmp.hasSrtm {
					nsrtm++
				}
				if tmp.hasRmw = fileutil.FileExists("/media/hdx/elev_raw/rmw3/" + tmp.name + ".HGT"); tmp.hasRmw {
					nrmw++
				}
				if tmp.hasVfp = fileutil.FileExists("/media/hdx/elev_raw/vfp/" + tmp.name + ".hgt"); tmp.hasVfp {
					nvfp++
				}
				tmps = append(tmps, tmp)
			}
		}
		fmt.Printf("Old: %d\tNew: %d\tBathy: %d\tSrtm: %d\tRmw: %d\tVfp: %d\t\n", nold, nnew, nbathy, nsrtm, nrmw, nvfp)
		fmt.Println(missings)
		type chk func(t tmpType) bool
		var checks = map[string]chk{
			"srtm only": func(t tmpType) bool { return t.hasSrtm && !(t.hasRmw || t.hasVfp) },
			"rmw only":  func(t tmpType) bool { return t.hasRmw && !(t.hasSrtm || t.hasVfp) },
			"vfp only":  func(t tmpType) bool { return t.hasVfp && !(t.hasSrtm || t.hasRmw) },
			"s+r only":  func(t tmpType) bool { return t.hasRmw && t.hasSrtm && !t.hasVfp },
			"s+v only":  func(t tmpType) bool { return t.hasVfp && t.hasSrtm && !t.hasRmw },
			"r+v only":  func(t tmpType) bool { return t.hasVfp && t.hasRmw && !t.hasSrtm },
			"srv":       func(t tmpType) bool { return t.hasVfp && t.hasRmw && t.hasSrtm },
		}
		for cn, cf := range checks {
			tc = 0
			for _, tmp = range tmps {
				if cf(tmp) {
					tc++
				}
			}
			fmt.Printf("%s: %d\n", cn, tc)
		}
	}
}
Example #3
0
func createElevs(targetDir string) {
	type sourceRec struct {
		dirPath   string
		filePath  string
		fileTime  time.Time
		byteOrder binary.ByteOrder
		fileLen   int64
	}
	var srcBasePath = "/media/hdx/elev_raw/"
	var force = []string{ /*"N64W024"*/ }
	var initSourceRec = func(dirRelPath string) *sourceRec {
		return &sourceRec{path.Join(srcBasePath, dirRelPath), "", time.Time{}, binary.LittleEndian, 0}
	}
	var fexts = []string{".hgt", ".hgt.le", ".HGT", ".HGT.LE"}
	var srcRecs = map[string]*sourceRec{"": initSourceRec(""), "bathy": initSourceRec("bathy/hgt"), "srtm": initSourceRec("srtm/hgt"), "rmw": initSourceRec("rmw3"), "vfp": initSourceRec("vfp")}
	var lola numutil.Dvec2
	var fp, fbName, skey, nkey string
	var hasBathy, hasElev bool
	var srec *sourceRec
	var newest, srtmTime time.Time
	var pngMakers = 0
	var pngChan = make(chan bool)
	var makePng = func(outFilePath string, elevFile *sourceRec, bathyFilePath string) {
		var ps, ps1, psx = 1200, 1201, 1200 * 2400
		var bathyLandVal int16 = 0
		var bathyMapping = []int16{-4500, -2500, -9500, -6500, -10500, -3500, bathyLandVal, -5500, -7500, -200, -8500, -1500, -750}
		var pngSize = image.Rect(0, 0, ps, ps)
		var px, py int
		var bv, ev, fv int16
		var srcFile, bathFile, pngFile *os.File
		var err error
		var pngImage *image.NRGBA
		fmt.Println(outFilePath)
		if pngFile, err = os.Create(outFilePath); err != nil {
			fmt.Println("PANIC", err)
			panic(err)
		}
		if bathFile, err = os.Open(bathyFilePath); err != nil {
			bathFile = nil
		}
		if srcFile, err = os.Open(elevFile.filePath); err != nil {
			srcFile = nil
		}
		if (bathFile != nil) || (srcFile != nil) {
			pngImage = image.NewNRGBA(pngSize)
			for py = 0; py < pngSize.Max.Y; py++ {
				for px = 0; px < pngSize.Max.X; px++ {
					if (bathFile == nil) || !fileutil.ReadFromBinary(bathFile, int64((math.Floor(float64(py)/40)*30)+math.Floor(float64(px)/40))*2, binary.LittleEndian, &bv) {
						bv = bathyLandVal
					} else {
						bv = bathyMapping[bv]
					}
					if (srcFile == nil) || !fileutil.ReadFromBinary(srcFile, (int64((py*coreutil.Ifi(int(elevFile.fileLen) == psx, ps, ps1))+px)*2) /*+ coreutil.Ifl(int(elevFile.fileLen) == psx, 0, 2402)*/, elevFile.byteOrder, &ev) {
						ev = -32768
					}
					fv = coreutil.Ifs(bv == bathyLandVal, ev, coreutil.Ifs((ev != 0) && (ev > bv), ev, bv))
					pngImage.Set(px, py, color.NRGBA{0, byte(fv), byte(fv >> 8), 255})
				}
			}
			if bathFile != nil {
				bathFile.Close()
			}
			if srcFile != nil {
				srcFile.Close()
			}
			err = png.Encode(pngFile, pngImage)
			if err != nil {
				fmt.Println("PANIC", err)
				panic(err)
			}
		}
		pngFile.Close()
		pngChan <- true
	}
	var goMakePng = func(outFilePath string, elevFile *sourceRec, bathyFilePath string) {
		for pngMakers >= 8 {
			<-pngChan
			pngMakers--
		}
		pngMakers++
		go makePng(outFilePath, elevFile, bathyFilePath)
	}
	srtmTime = time.Date(2008, 10, 15, 12, 30, 30, 500, time.Local)
	for lola.Y = geoutil.LatMin; lola.Y < geoutil.LatMax; lola.Y++ {
		for lola.X = geoutil.LonMin; lola.X < geoutil.LonMax; lola.X++ {
			fbName = geoutil.LoLaFileName(lola.X, lola.Y)
			if (len(force) == 0) || (stringutil.InSliceAt(force, fbName) >= 0) {
				fp = path.Join(targetDir, fbName)
				for skey, srec = range srcRecs {
					if len(skey) > 0 {
						srec.filePath, srec.fileTime, srec.fileLen = fileutil.FileExistsPath(srec.dirPath, fbName, fexts, true, false)
						if skey == "srtm" {
							srec.fileTime = srtmTime
						}
					}
				}
				newest = time.Time{}
				nkey = ""
				for skey, srec = range srcRecs {
					if (len(skey) > 0) && (skey != "bathy") && (len(srec.filePath) > 0) && (srec.fileTime.After(newest)) {
						newest = srec.fileTime
						nkey = skey
						if strings.HasSuffix(strings.ToLower(srec.filePath), ".hgt.le") {
							srec.byteOrder = binary.LittleEndian
						} else {
							srec.byteOrder = binary.BigEndian
						}
					}
				}
				if hasBathy, hasElev = len(srcRecs["bathy"].filePath) > 0, len(nkey) > 0; hasBathy || hasElev {
					goMakePng(fp, srcRecs[nkey], srcRecs["bathy"].filePath)
				} else {
					fmt.Println("\tNO BATHY, NO ELEV!\n")
				}
				if (len(srcRecs["rmw"].filePath) > 0) && (len(srcRecs["srtm"].filePath) > 0) && (len(srcRecs["vfp"].filePath) > 0) {
					goMakePng(path.Join(targetDir+"_rsv", fbName+"."+nkey+".r.png"), srcRecs["rmw"], srcRecs["bathy"].filePath)
					goMakePng(path.Join(targetDir+"_rsv", fbName+"."+nkey+".s.png"), srcRecs["srtm"], srcRecs["bathy"].filePath)
					goMakePng(path.Join(targetDir+"_rsv", fbName+"."+nkey+".v.png"), srcRecs["vfp"], srcRecs["bathy"].filePath)
				} else if (len(srcRecs["rmw"].filePath) > 0) && (len(srcRecs["vfp"].filePath) > 0) {
					goMakePng(path.Join(targetDir+"_rv", fbName+"."+nkey+".r.png"), srcRecs["rmw"], srcRecs["bathy"].filePath)
					goMakePng(path.Join(targetDir+"_rv", fbName+"."+nkey+".v.png"), srcRecs["vfp"], srcRecs["bathy"].filePath)
				} else if (len(srcRecs["rmw"].filePath) > 0) && (len(srcRecs["srtm"].filePath) > 0) {
					goMakePng(path.Join(targetDir+"_rs", fbName+"."+nkey+".r.png"), srcRecs["rmw"], srcRecs["bathy"].filePath)
					goMakePng(path.Join(targetDir+"_rs", fbName+"."+nkey+".s.png"), srcRecs["srtm"], srcRecs["bathy"].filePath)
				} else if (len(srcRecs["vfp"].filePath) > 0) && (len(srcRecs["srtm"].filePath) > 0) {
					goMakePng(path.Join(targetDir+"_sv", fbName+"."+nkey+".v.png"), srcRecs["vfp"], srcRecs["bathy"].filePath)
					goMakePng(path.Join(targetDir+"_sv", fbName+"."+nkey+".s.png"), srcRecs["srtm"], srcRecs["bathy"].filePath)
				}
			}
		}
	}
}