Esempio n. 1
0
func dlyline(line string, elements Elements, times Times, recordC chan<- Record, distanceToReference float64) error {

	if line == "" {
		return nil
	}
	// Encoding can be found here: http://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt
	stationID := line[0:11]

	year, err := strconv.Atoi(line[11:15])
	if err != nil {
		return errors.Wrapf(err, "could not parse year[%s]", line[11:14])
	}
	m, err := strconv.Atoi(line[15:17])
	if err != nil {
		return errors.Wrapf(err, "could not parse month[%s]", line[15:16])
	}
	month := time.Month(m)
	element := Element(line[17:21])

	if !elements.Contains(element) {
		return nil
	}

	for _, i := range times.DaysForMonth(year, month) {
		cursor := 21 + ((i - 1) * 8)
		if len(line) < cursor+8 {
			continue
		}
		// Is value missing?
		if line[cursor:cursor+5] == "-9999" {
			continue
		}
		v, err := strconv.ParseFloat(strings.TrimSpace(line[cursor:cursor+5]), 64)
		if err != nil {
			return errors.Wrapf(err, "could not parse value[%s]", line[cursor:cursor+5])
		}
		r := Record{
			Station: stationID,
			Element: element,
			Date:    Time{time.Date(year, month, i, 0, 0, 0, 0, time.UTC)},
			Value:   v,
			MFlag:   line[cursor+5 : cursor+6],
			QFlag:   line[cursor+6 : cursor+7],
			SFlag:   line[cursor+7 : cursor+8],
			DistanceToReferencePoint: distanceToReference,
		}
		recordC <- r
	}
	return nil

}
Esempio n. 2
0
func localVersionTime(ghcnd_all_dir string) (time.Time, error) {
	path := ghcnd_all_dir + "/ghcnd-version.txt"
	file, err := ioutil.ReadFile(path)
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "could not read ghcnd version file: %s", path)
	}
	localVersionTime, err := time.Parse("2006010215", string(versionRegexp.Find(file)))
	if err != nil {
		return time.Time{}, errors.Wrap(err, "could not parse local ghcnd_all version")
	}
	return localVersionTime, nil
}
Esempio n. 3
0
func downloadLatestGHCND(ghcnd_all_dir_path string) error {

	if ghcnd_all_dir_path == "" {
		return errors.New("a directory path in which to download the new ghcnd data is required")
	}

	fmt.Printf("downloading latest ghcnd_all tar file from %s\n", ghcnd_all_addr)

	fmt.Println("START", time.Now())
	defer func() {
		fmt.Println("END", time.Now())
	}()

	client := &http.Client{
		Timeout: time.Hour,
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				conn, err := net.DialTimeout(netw, addr, time.Second*30)
				if err != nil {
					return nil, err
				}
				conn.SetDeadline(time.Now().Add(time.Hour * 2))
				conn.SetReadDeadline(time.Now().Add(time.Hour * 2))
				return conn, nil
			},
		},
	}
	req, err := http.NewRequest("GET", ghcnd_all_addr, nil)
	req.Header.Add("Accept-Encoding", "identity")
	req.Close = true

	response, err := client.Do(req)
	//response, err := http.Get(ghcnd_all_addr)
	if err != nil {
		return errors.Wrapf(err, "could not fetch ghcnd data: %s", ghcnd_all_addr)
	}
	defer response.Body.Close()

	destPath, err := unzipit.UnpackStream(response.Body, ghcnd_all_dir_path)
	if err != nil {
		return errors.Wrap(err, "could not unpack stream")
	}

	fmt.Printf("ghcnd download complete: %s.\n", destPath)

	return nil

}
Esempio n. 4
0
func downloadLatestStations(ghcnd_stations_path string) error {

	fmt.Printf("downloading stations file from %s.\n", ghcnd_all_addr)

	response, err := http.Get(ghcnd_stations_addr)
	if err != nil {
		return errors.Wrapf(err, "could not fetch ghcnd station data: %s", ghcnd_stations_addr)
	}
	defer response.Body.Close()

	b, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return errors.Wrap(err, "could not read station file data")
	}

	if err := ioutil.WriteFile(ghcnd_stations_path, b, os.ModePerm); err != nil {
		return errors.Wrap(err, "could not write station file")
	}

	fmt.Printf("station download complete.\n")

	return nil
}