func main() {
var data = flag.String("data", "", "The data directory where WOF data lives, required")
var cache_size = flag.Int("cache_size", 1024, "The number of WOF records with large geometries to cache")
var cache_trigger = flag.Int("cache_trigger", 2000, "The minimum number of coordinates in a WOF record that will trigger caching")
var lat = flag.Float64("latitude", 0.0, "")
var lon = flag.Float64("longitude", 0.0, "")
var loglevel = flag.String("loglevel", "info", "...")
flag.Parse()
logger := log.NewWOFLogger("[wof-breaches] ")
logger.AddLogger(os.Stdout, *loglevel)
idx, _ := rtree.NewIndex(*data, *cache_size, *cache_trigger, logger)
for _, path := range flag.Args() {
logger.Info("indexing %s", path)
idx.IndexGeoJSONFile(path)
}
results := idx.GetIntersectsByLatLon(*lat, *lon)
inflated := idx.InflateSpatialResults(results)
for _, r := range inflated {
logger.Info("%v", r)
}
}
func NewPointInPolygonSimple(source string) (*WOFPointInPolygon, error) {
cache_size := 100
cache_trigger := 1000
logger := log.NewWOFLogger("[wof-pip-server]")
logger.AddLogger(os.Stdout, "debug")
return NewPointInPolygon(source, cache_size, cache_trigger, logger)
}
func main() {
var data = flag.String("data", "", "The data directory where WOF data lives, required")
var cache_size = flag.Int("cache_size", 1024, "The number of WOF records with large geometries to cache")
var cache_trigger = flag.Int("cache_trigger", 2000, "The minimum number of coordinates in a WOF record that will trigger caching")
var loglevel = flag.String("loglevel", "debug", "...")
var subject_id = flag.Int("subject", 0, "...")
var clipping_id = flag.Int("clipping", 0, "...")
flag.Parse()
logger := log.NewWOFLogger("[wof-breaches] ")
logger.AddLogger(os.Stdout, *loglevel)
idx, _ := breaches.NewIndex(*data, *cache_size, *cache_trigger, logger)
subject_path := utils.Id2AbsPath(*data, *subject_id)
clipping_path := utils.Id2AbsPath(*data, *clipping_id)
idx.IndexGeoJSONFile(subject_path)
subject, _ := idx.LoadGeoJSON(subject_path)
clipping, _ := idx.LoadGeoJSON(clipping_path)
logger.Info("does %s (clipping) breach %s (subject)", clipping.WOFName(), subject.WOFName())
t1 := time.Now()
results, _ := idx.Breaches(clipping)
if len(results) == 0 {
logger.Info("%s (clipping) DOES NOT breach %s (subject)", clipping.WOFName(), subject.WOFName())
}
for _, r := range results {
subject_path := utils.Id2AbsPath(*data, r.Id)
subject_feature, _ := idx.LoadGeoJSON(subject_path)
logger.Info("%s (%d) breaches %s (%d)", clipping.WOFName(), clipping.WOFId(), subject_feature.WOFName(), subject_feature.WOFId())
}
t2 := time.Since(t1)
logger.Info("time to search %v", t2)
}
func main() {
var port = flag.Int("port", 8080, "Port to listen")
var min = flag.Int("min", 5, "The minimum number of Brooklyn Integers to keep on hand at all times")
var loglevel = flag.String("loglevel", "info", "Log level")
var cors = flag.Bool("cors", false, "Enable CORS headers")
flag.Parse()
writer := io.MultiWriter(os.Stdout)
logger := log.NewWOFLogger("[big-integer] ")
logger.AddLogger(writer, *loglevel)
proxy := NewProxy(int64(*min), logger)
proxy.Init()
handler := func(rsp http.ResponseWriter, r *http.Request) {
i, err := proxy.Integer()
if err != nil {
msg := fmt.Sprintf("Failed to retrieve integer because %v", err)
http.Error(rsp, msg, http.StatusBadRequest)
}
if *cors {
rsp.Header().Set("Access-Control-Allow-Origin", "*")
return
}
io.WriteString(rsp, strconv.Itoa(int(i)))
}
http.HandleFunc("/", handler)
str_port := ":" + strconv.Itoa(*port)
err := http.ListenAndServe(str_port, nil)
if err != nil {
logger.Fatal("Failed to start server, because %v\n", err)
}
}
func main() {
var host = flag.String("host", "localhost", "The hostname to listen for requests on")
var port = flag.Int("port", 8080, "The port number to listen for requests on")
var data = flag.String("data", "", "The data directory where WOF data lives, required")
var cache_size = flag.Int("cache_size", 1024, "The number of WOF records with large geometries to cache")
var cache_trigger = flag.Int("cache_trigger", 2000, "The minimum number of coordinates in a WOF record that will trigger caching")
var strict = flag.Bool("strict", false, "Enable strict placetype checking")
var logs = flag.String("logs", "", "Where to write logs to disk")
var metrics = flag.String("metrics", "", "Where to write (@rcrowley go-metrics style) metrics to disk")
var format = flag.String("metrics-as", "plain", "Format metrics as... ? Valid options are \"json\" and \"plain\"")
var cors = flag.Bool("cors", false, "Enable CORS headers")
var verbose = flag.Bool("verbose", false, "Enable verbose logging, or log level \"info\"")
var verboser = flag.Bool("verboser", false, "Enable really verbose logging, or log level \"debug\"")
flag.Parse()
args := flag.Args()
if *data == "" {
panic("missing data")
}
_, err := os.Stat(*data)
if os.IsNotExist(err) {
panic("data does not exist")
}
loglevel := "status"
if *verbose {
loglevel = "info"
}
if *verboser {
loglevel = "debug"
}
var l_writer io.Writer
var m_writer io.Writer
l_writer = io.MultiWriter(os.Stdout)
if *logs != "" {
l_file, l_err := os.OpenFile(*logs, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
if l_err != nil {
panic(l_err)
}
l_writer = io.MultiWriter(os.Stdout, l_file)
}
logger := log.NewWOFLogger("[wof-pip-server] ")
logger.AddLogger(l_writer, loglevel)
p, p_err := pip.NewPointInPolygon(*data, *cache_size, *cache_trigger, logger)
if p_err != nil {
panic(p_err)
}
if *metrics != "" {
m_file, m_err := os.OpenFile(*metrics, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
if m_err != nil {
panic(m_err)
}
m_writer = io.MultiWriter(m_file)
_ = p.SendMetricsTo(m_writer, 60e9, *format)
}
t1 := time.Now()
for _, path := range args {
p.IndexMetaFile(path)
}
t2 := float64(time.Since(t1)) / 1e9
p.Logger.Status("indexed %d records in %.3f seconds", p.Rtree.Size(), t2)
for pt, count := range p.Placetypes {
p.Logger.Status("indexed %s: %d", pt, count)
}
handler := func(rsp http.ResponseWriter, req *http.Request) {
query := req.URL.Query()
str_lat := query.Get("latitude")
str_lon := query.Get("longitude")
placetype := query.Get("placetype")
if str_lat == "" {
http.Error(rsp, "Missing latitude parameter", http.StatusBadRequest)
return
}
if str_lon == "" {
http.Error(rsp, "Missing longitude parameter", http.StatusBadRequest)
return
}
lat, lat_err := strconv.ParseFloat(str_lat, 64)
lon, lon_err := strconv.ParseFloat(str_lon, 64)
if lat_err != nil {
http.Error(rsp, "Invalid latitude parameter", http.StatusBadRequest)
return
}
if lon_err != nil {
http.Error(rsp, "Invalid longitude parameter", http.StatusBadRequest)
return
}
if lat > 90.0 || lat < -90.0 {
http.Error(rsp, "E_IMPOSSIBLE_LATITUDE", http.StatusBadRequest)
return
}
if lon > 180.0 || lon < -180.0 {
http.Error(rsp, "E_IMPOSSIBLE_LONGITUDE", http.StatusBadRequest)
return
}
if placetype != "" {
if *strict && !p.IsKnownPlacetype(placetype) {
http.Error(rsp, "Unknown placetype", http.StatusBadRequest)
return
}
}
results, timings := p.GetByLatLonForPlacetype(lat, lon, placetype)
count := len(results)
ttp := 0.0
for _, t := range timings {
ttp += t.Duration
}
if placetype != "" {
p.Logger.Debug("time to reverse geocode %f, %f @%s: %d results in %f seconds ", lat, lon, placetype, count, ttp)
} else {
p.Logger.Debug("time to reverse geocode %f, %f: %d results in %f seconds ", lat, lon, count, ttp)
}
js, err := json.Marshal(results)
if err != nil {
http.Error(rsp, err.Error(), http.StatusInternalServerError)
return
}
// maybe this although it seems like it adds functionality for a lot of
// features this server does not need - https://github.com/rs/cors
// (20151022/thisisaaronland)
if *cors {
rsp.Header().Set("Access-Control-Allow-Origin", "*")
}
rsp.Header().Set("Content-Type", "application/json")
rsp.Write(js)
}
endpoint := fmt.Sprintf("%s:%d", *host, *port)
http.HandleFunc("/", handler)
http.ListenAndServe(endpoint, nil)
}
func main() {
var root = flag.String("root", "", "The directory where your Who's On First data is stored")
var bucket = flag.String("bucket", "", "The S3 bucket to sync <root> to")
var prefix = flag.String("prefix", "", "A prefix inside your S3 bucket where things go")
var list = flag.String("file-list", "", "A single file containing a list of files to sync")
var debug = flag.Bool("debug", false, "Be chatty")
var dryrun = flag.Bool("dryrun", false, "Go through the motions but don't actually clone anything")
var tidy = flag.Bool("tidy", false, "Remove -file-list file, if present")
var credentials = flag.String("credentials", "", "Your S3 credentials file")
var procs = flag.Int("processes", (runtime.NumCPU() * 2), "The number of concurrent processes to sync data with")
var loglevel = flag.String("loglevel", "info", "Log level for reporting")
var slack = flag.Bool("slack", false, "Send status updates to Slack (via slackcat)")
var slack_config = flag.String("slack-config", "", "The path to your slackcat config")
flag.Parse()
if *root == "" {
panic("missing root")
}
_, err := os.Stat(*root)
if os.IsNotExist(err) {
panic("root does not exist")
}
if *bucket == "" {
panic("missing bucket")
}
if *credentials != "" {
os.Setenv("AWS_CREDENTIAL_FILE", *credentials)
}
if *debug || *dryrun {
*loglevel = "debug"
}
logger := log.NewWOFLogger("[wof-sync-files] ")
writer := io.MultiWriter(os.Stdout)
logger.AddLogger(writer, *loglevel)
s := s3.WOFSync(*bucket, *prefix, *procs, *debug, logger)
s.MonitorStatus()
if *dryrun {
s.Dryrun = true
}
if *list == "" {
args := flag.Args()
s.SyncFiles(args, *root)
} else {
_, err := os.Stat(*list)
if os.IsNotExist(err) {
panic(err)
}
s.SyncFileList(*list, *root)
if !*debug && *tidy {
os.Remove(*list)
}
}
if *slack {
sl, err := slackcat.NewWriter(*slack_config)
if err != nil {
logger.Warning("failed to create slackcat writer, because %v", err)
} else {
logger.AddLogger(sl, "status")
logger.Status(s.StatusReport())
logger.Status("Time to process %v", s.TimeToProcess)
}
}
}
func main() {
/*
This still lacks metrics collection of strict placetype checking
(20151203/thisisaaronland)
*/
var port = flag.Int("port", 9988, "The port number to listen for requests on")
var data = flag.String("data", "", "The data directory where WOF data lives, required")
var cache_size = flag.Int("cache_size", 1024, "The number of WOF records with large geometries to cache")
var cache_trigger = flag.Int("cache_trigger", 2000, "The minimum number of coordinates in a WOF record that will trigger caching")
// var strict = flag.Bool("strict", false, "Enable strict placetype checking")
var logs = flag.String("logs", "", "Where to write logs to disk")
// var metrics = flag.String("metrics", "", "Where to write (@rcrowley go-metrics style) metrics to disk")
// var format = flag.String("metrics-as", "plain", "Format metrics as... ? Valid options are \"json\" and \"plain\"")
var cors = flag.Bool("cors", false, "Enable CORS headers")
var loglevel = flag.String("loglevel", "info", "")
flag.Parse()
args := flag.Args()
if *data == "" {
panic("missing data")
}
_, err := os.Stat(*data)
if os.IsNotExist(err) {
panic("data does not exist")
}
l_writer := io.MultiWriter(os.Stdout)
if *logs != "" {
l_file, l_err := os.OpenFile(*logs, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
if l_err != nil {
panic(l_err)
}
l_writer = io.MultiWriter(os.Stdout, l_file)
}
logger := log.NewWOFLogger("[wof-breach-server] ")
logger.AddLogger(l_writer, *loglevel)
idx, _ := breaches.NewIndex(*data, *cache_size, *cache_trigger, logger)
for _, meta := range args {
logger.Info("indexing %s", meta)
idx.IndexMetaFile(meta)
}
logger.Info("indexing meta files complete")
var lookups int64
lookups = 0
handler := func(rsp http.ResponseWriter, req *http.Request) {
query := req.URL.Query()
str_wofid := query.Get("wofid")
if str_wofid == "" {
http.Error(rsp, "Missing wofid parameter", http.StatusBadRequest)
return
}
wofid, err := strconv.Atoi(str_wofid)
if err != nil {
http.Error(rsp, "Unable to parse WOF ID", http.StatusBadRequest)
return
}
path := utils.Id2AbsPath(*data, wofid)
feature, err := idx.LoadGeoJSON(path)
if err != nil {
http.Error(rsp, "Invalid WOF ID", http.StatusBadRequest)
return
}
logger.Info("looking up %d (%d)", feature.WOFId(), lookups)
t1 := time.Now()
atomic.AddInt64(&lookups, 1)
results, err := idx.Breaches(feature)
atomic.AddInt64(&lookups, -1)
t2 := time.Since(t1)
logger.Info("time to lookup %d : %v", feature.WOFId(), t2)
if err != nil {
http.Error(rsp, err.Error(), http.StatusInternalServerError)
return
}
logger.Info("%d results breach %d", len(results), feature.WOFId())
js, err := json.Marshal(results)
if err != nil {
http.Error(rsp, err.Error(), http.StatusInternalServerError)
return
}
if *cors {
rsp.Header().Set("Access-Control-Allow-Origin", "*")
}
rsp.Header().Set("Content-Type", "application/json")
rsp.Write(js)
}
str_port := fmt.Sprintf(":%d", *port)
http.HandleFunc("/", handler)
http.ListenAndServe(str_port, nil)
}
func main() {
var data = flag.String("data", "", "The data directory where WOF data lives, required")
var cache_size = flag.Int("cache_size", 1024, "The number of WOF records with large geometries to cache")
var cache_trigger = flag.Int("cache_trigger", 2000, "The minimum number of coordinates in a WOF record that will trigger caching")
var loglevel = flag.String("loglevel", "info", "...")
flag.Parse()
args := flag.Args()
logger := log.NewWOFLogger("[wof-breaches] ")
logger.AddLogger(os.Stdout, *loglevel)
idx, _ := breaches.NewIndex(*data, *cache_size, *cache_trigger, logger)
for _, meta := range args {
logger.Info("indexing %s", meta)
idx.IndexMetaFile(meta)
}
logger.Info("indexing meta files complete")
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
id := scanner.Text()
if id == "" {
continue
}
wofid, err := strconv.Atoi(id)
if err != nil {
logger.Error("failed to convert %s to a WOF ID, because %v", id, err)
continue
}
path := utils.Id2AbsPath(*data, wofid)
feature, err := idx.LoadGeoJSON(path)
if err != nil {
logger.Error("Failed to read %s, because %v", path, err)
continue
}
// Note the WOFID-iness below - please to be waiting for this
// to get pushed to get resolved (20151125/thisisaaronland)
// https://github.com/whosonfirst/go-whosonfirst-geojson/issues/2
logger.Info("what records does %s (%s) breach?", feature.WOFName(), path)
t1 := time.Now()
results, err := idx.Breaches(feature)
if err != nil {
logger.Error("Failed to determined breaches for %s (%d), because %v", feature.WOFName(), feature.WOFId(), err)
continue
}
count := len(results)
if count == 0 {
logger.Status("%s does not breach any other records in the index", feature.WOFName())
} else if count == 1 {
logger.Status("%s breaches one other record in the index", feature.WOFName())
} else {
logger.Status("%s breaches %d other records in the index", feature.WOFName(), count)
}
for _, r := range results {
other_path := utils.Id2AbsPath(*data, r.Id)
other_feature, _ := idx.LoadGeoJSON(other_path)
logger.Info("%s (%d) breaches %s (%d)", feature.WOFName(), feature.WOFId(), other_feature.WOFName(), other_feature.WOFId())
}
t2 := time.Since(t1)
logger.Info("time to search %v", t2)
}
}
func main() {
var host = flag.String("host", "localhost", "The hostname to listen for requests on")
var port = flag.Int("port", 8080, "The port number to listen for requests on")
var data = flag.String("data", "", "The data directory where WOF data lives, required")
var cache_all = flag.Bool("cache_all", false, "Just cache everything, regardless of size")
var cache_size = flag.Int("cache_size", 1024, "The number of WOF records with large geometries to cache")
var cache_trigger = flag.Int("cache_trigger", 2000, "The minimum number of coordinates in a WOF record that will trigger caching")
var strict = flag.Bool("strict", false, "Enable strict placetype checking")
var loglevel = flag.String("loglevel", "info", "Log level for reporting")
var logs = flag.String("logs", "", "Where to write logs to disk")
var metrics = flag.String("metrics", "", "Where to write (@rcrowley go-metrics style) metrics to disk")
var format = flag.String("metrics-as", "plain", "Format metrics as... ? Valid options are \"json\" and \"plain\"")
var cors = flag.Bool("cors", false, "Enable CORS headers")
var procs = flag.Int("procs", (runtime.NumCPU() * 2), "The number of concurrent processes to clone data with")
var pidfile = flag.String("pidfile", "", "Where to write a PID file for wof-pip-server. If empty the PID file will be written to wof-pip-server.pid in the current directory")
var nopid = flag.Bool("nopid", false, "Do not try to write a PID file")
flag.Parse()
args := flag.Args()
if *data == "" {
panic("missing data")
}
_, err := os.Stat(*data)
if os.IsNotExist(err) {
panic("data does not exist")
}
runtime.GOMAXPROCS(*procs)
var l_writer io.Writer
var m_writer io.Writer
l_writer = io.MultiWriter(os.Stdout)
if *logs != "" {
l_file, l_err := os.OpenFile(*logs, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
if l_err != nil {
panic(l_err)
}
l_writer = io.MultiWriter(os.Stdout, l_file)
}
logger := log.NewWOFLogger("[wof-pip-server] ")
logger.AddLogger(l_writer, *loglevel)
if *cache_all {
*cache_size = 0
*cache_trigger = 1
mu := new(sync.Mutex)
wg := new(sync.WaitGroup)
for _, path := range args {
wg.Add(1)
go func(path string) {
defer wg.Done()
count := 0
fh, err := os.Open(path)
if err != nil {
logger.Error("failed to open %s for reading, because %v", path, err)
os.Exit(1)
}
scanner := bufio.NewScanner(fh)
for scanner.Scan() {
count += 1
}
mu.Lock()
*cache_size += count
mu.Unlock()
}(path)
}
wg.Wait()
logger.Status("set cache_size to %d and cache_trigger to %d", *cache_size, *cache_trigger)
}
p, p_err := pip.NewPointInPolygon(*data, *cache_size, *cache_trigger, logger)
if p_err != nil {
panic(p_err)
}
if *metrics != "" {
m_file, m_err := os.OpenFile(*metrics, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
if m_err != nil {
panic(m_err)
}
m_writer = io.MultiWriter(m_file)
_ = p.SendMetricsTo(m_writer, 60e9, *format)
}
indexing := true
ch := make(chan bool)
go func() {
<-ch
indexing = false
}()
go func() {
if *nopid {
t1 := time.Now()
for _, path := range args {
p.Logger.Status("indexing %s", path)
p.IndexMetaFile(path)
}
t2 := float64(time.Since(t1)) / 1e9
p.Logger.Status("indexed %d records in %.3f seconds", p.Rtree.Size(), t2)
ch <- true
return
}
if *pidfile == "" {
cwd, err := os.Getwd()
if err != nil {
panic(err)
}
fname := fmt.Sprintf("%s.pid", os.Args[0])
*pidfile = filepath.Join(cwd, fname)
}
fh, err := os.Create(*pidfile)
if err != nil {
panic(err)
}
defer fh.Close()
t1 := time.Now()
for _, path := range args {
p.Logger.Status("indexing %s", path)
p.IndexMetaFile(path)
}
t2 := float64(time.Since(t1)) / 1e9
p.Logger.Status("indexed %d records in %.3f seconds", p.Rtree.Size(), t2)
pid := os.Getpid()
strpid := strconv.Itoa(pid)
fh.Write([]byte(strpid))
p.Logger.Status("create PID file %s", *pidfile)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigs
p.Logger.Status("remove PID file %s", *pidfile)
os.Remove(*pidfile)
os.Exit(0)
}()
ch <- true
}()
handler := func(rsp http.ResponseWriter, req *http.Request) {
if indexing == true {
http.Error(rsp, "indexing records", http.StatusServiceUnavailable)
return
}
query := req.URL.Query()
str_lat := query.Get("latitude")
str_lon := query.Get("longitude")
placetype := query.Get("placetype")
excluded := query["exclude"] // see the way we're accessing the map directly to get a list? yeah, that
if str_lat == "" {
http.Error(rsp, "Missing latitude parameter", http.StatusBadRequest)
return
}
if str_lon == "" {
http.Error(rsp, "Missing longitude parameter", http.StatusBadRequest)
return
}
lat, lat_err := strconv.ParseFloat(str_lat, 64)
lon, lon_err := strconv.ParseFloat(str_lon, 64)
if lat_err != nil {
http.Error(rsp, "Invalid latitude parameter", http.StatusBadRequest)
return
}
if lon_err != nil {
http.Error(rsp, "Invalid longitude parameter", http.StatusBadRequest)
return
}
if lat > 90.0 || lat < -90.0 {
http.Error(rsp, "E_IMPOSSIBLE_LATITUDE", http.StatusBadRequest)
return
}
if lon > 180.0 || lon < -180.0 {
http.Error(rsp, "E_IMPOSSIBLE_LONGITUDE", http.StatusBadRequest)
return
}
filters := pip.WOFPointInPolygonFilters{}
if placetype != "" {
if *strict && !p.IsKnownPlacetype(placetype) {
http.Error(rsp, "Unknown placetype", http.StatusBadRequest)
return
}
filters["placetype"] = placetype
}
for _, what := range excluded {
if what == "deprecated" || what == "superseded" {
filters[what] = false
}
}
results, timings := p.GetByLatLonFiltered(lat, lon, filters)
count := len(results)
ttp := 0.0
for _, t := range timings {
ttp += t.Duration
}
if placetype != "" {
p.Logger.Debug("time to reverse geocode %f, %f @%s: %d results in %f seconds ", lat, lon, placetype, count, ttp)
} else {
p.Logger.Debug("time to reverse geocode %f, %f: %d results in %f seconds ", lat, lon, count, ttp)
}
js, err := json.Marshal(results)
if err != nil {
http.Error(rsp, err.Error(), http.StatusInternalServerError)
return
}
// maybe this although it seems like it adds functionality for a lot of
// features this server does not need - https://github.com/rs/cors
// (20151022/thisisaaronland)
if *cors {
rsp.Header().Set("Access-Control-Allow-Origin", "*")
}
rsp.Header().Set("Content-Type", "application/json")
rsp.Write(js)
}
endpoint := fmt.Sprintf("%s:%d", *host, *port)
mux := http.NewServeMux()
mux.HandleFunc("/", handler)
gracehttp.Serve(&http.Server{Addr: endpoint, Handler: mux})
os.Exit(0)
}
func main() {
var root = flag.String("root", "", "The directory to sync")
var bucket = flag.String("bucket", "", "The S3 bucket to sync <root> to")
var prefix = flag.String("prefix", "", "A prefix inside your S3 bucket where things go")
var debug = flag.Bool("debug", false, "Don't actually try to sync anything and spew a lot of line noise")
var dryrun = flag.Bool("dryrun", false, "Go through the motions but don't actually clone anything")
var credentials = flag.String("credentials", "", "Your S3 credentials file")
var procs = flag.Int("processes", (runtime.NumCPU() * 2), "The number of concurrent processes to sync data with")
var loglevel = flag.String("loglevel", "info", "Log level for reporting")
var slack = flag.Bool("slack", false, "Send status updates to Slack (via slackcat)")
var slack_config = flag.String("slack-config", "", "The path to your slackcat config")
flag.Parse()
if *root == "" {
golog.Fatal("missing root to sync")
}
_, err := os.Stat(*root)
if os.IsNotExist(err) {
golog.Fatal("root does not exist")
}
if *bucket == "" {
golog.Fatal("missing bucket")
}
if *credentials != "" {
os.Setenv("AWS_CREDENTIAL_FILE", *credentials)
}
if *debug || *dryrun {
*loglevel = "debug"
}
logger := log.NewWOFLogger("[wof-sync-dirs] ")
writer := io.MultiWriter(os.Stdout)
logger.AddLogger(writer, *loglevel)
s := s3.WOFSync(*bucket, *prefix, *procs, *debug, logger)
if *dryrun {
s.Dryrun = true
}
s.MonitorStatus()
err = s.SyncDirectory(*root)
if *slack {
sl, err := slackcat.NewWriter(*slack_config)
if err != nil {
logger.Warning("failed to create slackcat writer, because %v", err)
} else {
logger.AddLogger(sl, "status")
logger.Status(s.StatusReport())
logger.Status("Time to process %v", s.TimeToProcess)
}
}
}