func serverVersionTime() (time.Time, error) {
resp, err := http.Get(ghcnd_version_addr)
if err != nil {
return time.Time{}, errors.Wrap(err, "could not get lat ghcnd version data").SetState(M{logkeys.URL: ghcnd_version_addr})
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return time.Time{}, errors.Wrap(err, "could not read ghcnd version response").SetState(M{logkeys.URL: ghcnd_version_addr})
}
serverVersionTime, err := time.Parse("2006010215", string(versionRegexp.Find(body)))
if err != nil {
return time.Time{}, errors.Wrap(err, "could not parse local ghcnd_all version")
}
return serverVersionTime, nil
}
func localVersionTime(ghcnd_all_dir string) (time.Time, error) {
path := ghcnd_all_dir + "/ghcnd-version.txt"
file, err := ioutil.ReadFile(path)
if err != nil {
return time.Time{}, errors.Wrapf(err, "could not read ghcnd version file: %s", path)
}
localVersionTime, err := time.Parse("2006010215", string(versionRegexp.Find(file)))
if err != nil {
return time.Time{}, errors.Wrap(err, "could not parse local ghcnd_all version")
}
return localVersionTime, nil
}
func downloadLatestStations(ghcnd_stations_path string) error {
fmt.Printf("downloading stations file from %s.\n", ghcnd_all_addr)
response, err := http.Get(ghcnd_stations_addr)
if err != nil {
return errors.Wrapf(err, "could not fetch ghcnd station data: %s", ghcnd_stations_addr)
}
defer response.Body.Close()
b, err := ioutil.ReadAll(response.Body)
if err != nil {
return errors.Wrap(err, "could not read station file data")
}
if err := ioutil.WriteFile(ghcnd_stations_path, b, os.ModePerm); err != nil {
return errors.Wrap(err, "could not write station file")
}
fmt.Printf("station download complete.\n")
return nil
}
func downloadLatestGHCND(ghcnd_all_dir_path string) error {
if ghcnd_all_dir_path == "" {
return errors.New("a directory path in which to download the new ghcnd data is required")
}
fmt.Printf("downloading latest ghcnd_all tar file from %s\n", ghcnd_all_addr)
fmt.Println("START", time.Now())
defer func() {
fmt.Println("END", time.Now())
}()
client := &http.Client{
Timeout: time.Hour,
Transport: &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(netw, addr, time.Second*30)
if err != nil {
return nil, err
}
conn.SetDeadline(time.Now().Add(time.Hour * 2))
conn.SetReadDeadline(time.Now().Add(time.Hour * 2))
return conn, nil
},
},
}
req, err := http.NewRequest("GET", ghcnd_all_addr, nil)
req.Header.Add("Accept-Encoding", "identity")
req.Close = true
response, err := client.Do(req)
//response, err := http.Get(ghcnd_all_addr)
if err != nil {
return errors.Wrapf(err, "could not fetch ghcnd data: %s", ghcnd_all_addr)
}
defer response.Body.Close()
destPath, err := unzipit.UnpackStream(response.Body, ghcnd_all_dir_path)
if err != nil {
return errors.Wrap(err, "could not unpack stream")
}
fmt.Printf("ghcnd download complete: %s.\n", destPath)
return nil
}
func downloadLatestGHCNDIfRequired(ghcnd_stations_path, ghcnd_all_dir_path string) error {
fmt.Printf("does the ghcnd directory exist: %s?\n", ghcnd_all_dir_path)
// Does the directory exist at all?
_, err := os.Stat(ghcnd_all_dir_path)
if err != nil && !os.IsNotExist(err) {
return errors.Wrap(err, "could not stat ghcnd_all directory")
}
dirExists := err == nil || !os.IsNotExist(err)
var isCurrentVersion bool
if dirExists {
fmt.Println("ghcnd directory exists. comparing versions.")
local, err := localVersionTime(ghcnd_all_dir_path)
if err != nil {
return err
}
server, err := serverVersionTime()
if err != nil {
return err
}
isCurrentVersion = local.After(server) || local.Equal(server)
if !isCurrentVersion {
fmt.Println(fmt.Sprintf(
"local ghcnd_all directory version time [%v] is earlier than server version time [%v].", local, server))
} else {
fmt.Println(fmt.Sprintf(
"local ghcnd_all directory version time [%v] is up to date with server version time [%v].", local, server))
}
} else {
fmt.Println("ghcnd directory does not exist.")
}
if !dirExists || !isCurrentVersion {
if err := downloadLatestGHCND(ghcnd_all_dir_path); err != nil {
return err
}
if err := downloadLatestStations(ghcnd_stations_path); err != nil {
return err
}
}
return nil
}