Example #1
0
func serverVersionTime() (time.Time, error) {
	resp, err := http.Get(ghcnd_version_addr)
	if err != nil {
		return time.Time{}, errors.Wrap(err, "could not get lat ghcnd version data").SetState(M{logkeys.URL: ghcnd_version_addr})
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return time.Time{}, errors.Wrap(err, "could not read ghcnd version response").SetState(M{logkeys.URL: ghcnd_version_addr})
	}
	serverVersionTime, err := time.Parse("2006010215", string(versionRegexp.Find(body)))
	if err != nil {
		return time.Time{}, errors.Wrap(err, "could not parse local ghcnd_all version")
	}
	return serverVersionTime, nil
}
Example #2
0
func localVersionTime(ghcnd_all_dir string) (time.Time, error) {
	path := ghcnd_all_dir + "/ghcnd-version.txt"
	file, err := ioutil.ReadFile(path)
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "could not read ghcnd version file: %s", path)
	}
	localVersionTime, err := time.Parse("2006010215", string(versionRegexp.Find(file)))
	if err != nil {
		return time.Time{}, errors.Wrap(err, "could not parse local ghcnd_all version")
	}
	return localVersionTime, nil
}
Example #3
0
func downloadLatestStations(ghcnd_stations_path string) error {

	fmt.Printf("downloading stations file from %s.\n", ghcnd_all_addr)

	response, err := http.Get(ghcnd_stations_addr)
	if err != nil {
		return errors.Wrapf(err, "could not fetch ghcnd station data: %s", ghcnd_stations_addr)
	}
	defer response.Body.Close()

	b, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return errors.Wrap(err, "could not read station file data")
	}

	if err := ioutil.WriteFile(ghcnd_stations_path, b, os.ModePerm); err != nil {
		return errors.Wrap(err, "could not write station file")
	}

	fmt.Printf("station download complete.\n")

	return nil
}
Example #4
0
func downloadLatestGHCND(ghcnd_all_dir_path string) error {

	if ghcnd_all_dir_path == "" {
		return errors.New("a directory path in which to download the new ghcnd data is required")
	}

	fmt.Printf("downloading latest ghcnd_all tar file from %s\n", ghcnd_all_addr)

	fmt.Println("START", time.Now())
	defer func() {
		fmt.Println("END", time.Now())
	}()

	client := &http.Client{
		Timeout: time.Hour,
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				conn, err := net.DialTimeout(netw, addr, time.Second*30)
				if err != nil {
					return nil, err
				}
				conn.SetDeadline(time.Now().Add(time.Hour * 2))
				conn.SetReadDeadline(time.Now().Add(time.Hour * 2))
				return conn, nil
			},
		},
	}
	req, err := http.NewRequest("GET", ghcnd_all_addr, nil)
	req.Header.Add("Accept-Encoding", "identity")
	req.Close = true

	response, err := client.Do(req)
	//response, err := http.Get(ghcnd_all_addr)
	if err != nil {
		return errors.Wrapf(err, "could not fetch ghcnd data: %s", ghcnd_all_addr)
	}
	defer response.Body.Close()

	destPath, err := unzipit.UnpackStream(response.Body, ghcnd_all_dir_path)
	if err != nil {
		return errors.Wrap(err, "could not unpack stream")
	}

	fmt.Printf("ghcnd download complete: %s.\n", destPath)

	return nil

}
Example #5
0
func downloadLatestGHCNDIfRequired(ghcnd_stations_path, ghcnd_all_dir_path string) error {

	fmt.Printf("does the ghcnd directory exist: %s?\n", ghcnd_all_dir_path)

	// Does the directory exist at all?
	_, err := os.Stat(ghcnd_all_dir_path)
	if err != nil && !os.IsNotExist(err) {
		return errors.Wrap(err, "could not stat ghcnd_all directory")
	}
	dirExists := err == nil || !os.IsNotExist(err)
	var isCurrentVersion bool
	if dirExists {
		fmt.Println("ghcnd directory exists. comparing versions.")
		local, err := localVersionTime(ghcnd_all_dir_path)
		if err != nil {
			return err
		}
		server, err := serverVersionTime()
		if err != nil {
			return err
		}
		isCurrentVersion = local.After(server) || local.Equal(server)
		if !isCurrentVersion {
			fmt.Println(fmt.Sprintf(
				"local ghcnd_all directory version time [%v] is earlier than server version time [%v].", local, server))
		} else {
			fmt.Println(fmt.Sprintf(
				"local ghcnd_all directory version time [%v] is up to date with server version time [%v].", local, server))
		}
	} else {
		fmt.Println("ghcnd directory does not exist.")
	}

	if !dirExists || !isCurrentVersion {
		if err := downloadLatestGHCND(ghcnd_all_dir_path); err != nil {
			return err
		}
		if err := downloadLatestStations(ghcnd_stations_path); err != nil {
			return err
		}
	}

	return nil
}