// URLToBytes gets the response from urlstr and returns it as a byte string
// TODO wait on a goroutine w/ timeout, instead of blocking main thread
func URLToBytes(urlstr string, secure bool) []byte {
// create http client
transport := &http.Transport{}
if !secure {
transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
client := &http.Client{Transport: transport}
// get response from URL
resp, err := client.Get(urlstr)
if err != nil {
errutil.CouldntReadError(urlstr, err)
}
defer resp.Body.Close()
// read response
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.WithFields(log.Fields{
"URL": urlstr,
"Error": err.Error(),
}).Fatal("Bad response, couldn't read body")
} else if body == nil || bytes.Equal(body, []byte{}) {
log.WithFields(log.Fields{
"URL": urlstr,
}).Warn("Body of response was empty")
}
return body
}
// validateFlags ensures that all options passed via the command line are valid
func validateFlags(file string, URL string, directory string) {
// validatePath ensures that something is at a given path
validatePath := func(path string) {
if _, err := os.Stat(path); err != nil {
errutil.CouldntReadError(path, err)
}
} // validateURL ensures that the given URL is valid, or logs an error
validateURL := func(urlstr string) {
if _, err := url.Parse(urlstr); err != nil {
log.WithFields(log.Fields{
"url": urlstr,
"error": err.Error(),
}).Fatal("Couldn't parse URL")
}
}
if URL != "" {
validateURL(URL)
}
if directory != "" {
validatePath(directory)
}
if file != "" {
validatePath(file)
}
}
// GetFilesWithExtension returns the paths to all the files in the given dir
// that end with the given file extension (with or without dot)
func GetFilesWithExtension(path string, ext string) (paths []string) {
finfos, err := ioutil.ReadDir(path) // list of os.FileInfo
if err != nil {
errutil.CouldntReadError(path, err)
}
for _, finfo := range finfos {
name := finfo.Name()
if strings.HasSuffix(name, ext) {
// TODO path.Join these suckers
paths = append(paths, path+"/"+name)
}
}
return paths
}
// FileToBytes reads a file and handles the error
func FileToBytes(path string) []byte {
data, err := ioutil.ReadFile(path)
errutil.CouldntReadError(path, err)
return data
}
