func dlyline(line string, elements Elements, times Times, recordC chan<- Record, distanceToReference float64) error {
if line == "" {
return nil
}
// Encoding can be found here: http://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt
stationID := line[0:11]
year, err := strconv.Atoi(line[11:15])
if err != nil {
return errors.Wrapf(err, "could not parse year[%s]", line[11:14])
}
m, err := strconv.Atoi(line[15:17])
if err != nil {
return errors.Wrapf(err, "could not parse month[%s]", line[15:16])
}
month := time.Month(m)
element := Element(line[17:21])
if !elements.Contains(element) {
return nil
}
for _, i := range times.DaysForMonth(year, month) {
cursor := 21 + ((i - 1) * 8)
if len(line) < cursor+8 {
continue
}
// Is value missing?
if line[cursor:cursor+5] == "-9999" {
continue
}
v, err := strconv.ParseFloat(strings.TrimSpace(line[cursor:cursor+5]), 64)
if err != nil {
return errors.Wrapf(err, "could not parse value[%s]", line[cursor:cursor+5])
}
r := Record{
Station: stationID,
Element: element,
Date: Time{time.Date(year, month, i, 0, 0, 0, 0, time.UTC)},
Value: v,
MFlag: line[cursor+5 : cursor+6],
QFlag: line[cursor+6 : cursor+7],
SFlag: line[cursor+7 : cursor+8],
DistanceToReferencePoint: distanceToReference,
}
recordC <- r
}
return nil
}
func localVersionTime(ghcnd_all_dir string) (time.Time, error) {
path := ghcnd_all_dir + "/ghcnd-version.txt"
file, err := ioutil.ReadFile(path)
if err != nil {
return time.Time{}, errors.Wrapf(err, "could not read ghcnd version file: %s", path)
}
localVersionTime, err := time.Parse("2006010215", string(versionRegexp.Find(file)))
if err != nil {
return time.Time{}, errors.Wrap(err, "could not parse local ghcnd_all version")
}
return localVersionTime, nil
}
func downloadLatestGHCND(ghcnd_all_dir_path string) error {
if ghcnd_all_dir_path == "" {
return errors.New("a directory path in which to download the new ghcnd data is required")
}
fmt.Printf("downloading latest ghcnd_all tar file from %s\n", ghcnd_all_addr)
fmt.Println("START", time.Now())
defer func() {
fmt.Println("END", time.Now())
}()
client := &http.Client{
Timeout: time.Hour,
Transport: &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(netw, addr, time.Second*30)
if err != nil {
return nil, err
}
conn.SetDeadline(time.Now().Add(time.Hour * 2))
conn.SetReadDeadline(time.Now().Add(time.Hour * 2))
return conn, nil
},
},
}
req, err := http.NewRequest("GET", ghcnd_all_addr, nil)
req.Header.Add("Accept-Encoding", "identity")
req.Close = true
response, err := client.Do(req)
//response, err := http.Get(ghcnd_all_addr)
if err != nil {
return errors.Wrapf(err, "could not fetch ghcnd data: %s", ghcnd_all_addr)
}
defer response.Body.Close()
destPath, err := unzipit.UnpackStream(response.Body, ghcnd_all_dir_path)
if err != nil {
return errors.Wrap(err, "could not unpack stream")
}
fmt.Printf("ghcnd download complete: %s.\n", destPath)
return nil
}
func downloadLatestStations(ghcnd_stations_path string) error {
fmt.Printf("downloading stations file from %s.\n", ghcnd_all_addr)
response, err := http.Get(ghcnd_stations_addr)
if err != nil {
return errors.Wrapf(err, "could not fetch ghcnd station data: %s", ghcnd_stations_addr)
}
defer response.Body.Close()
b, err := ioutil.ReadAll(response.Body)
if err != nil {
return errors.Wrap(err, "could not read station file data")
}
if err := ioutil.WriteFile(ghcnd_stations_path, b, os.ModePerm); err != nil {
return errors.Wrap(err, "could not write station file")
}
fmt.Printf("station download complete.\n")
return nil
}