// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool) {
var expectation expectedRequest
if len(f.expected) > 0 && f.expected[0].URL == url {
expectation, f.expected = f.expected[0], f.expected[1:]
} else if _, ok := f.anyExpected[url]; ok {
expectation = f.anyExpected[url]
delete(f.anyExpected, url)
} else {
result <- fmt.Errorf("unexpected request for %s", url)
return
}
if expectation.Err != nil {
result <- expectation.Err
return
}
err := os.MkdirAll(filepath.Dir(filename), 0755)
if err != nil {
result <- err
return
}
outfile, err := os.Create(filename)
if err != nil {
result <- err
return
}
defer outfile.Close()
cks := utils.NewChecksumWriter()
w := io.MultiWriter(outfile, cks)
_, err = w.Write([]byte(expectation.Response))
if err != nil {
result <- err
return
}
if expected.Size != -1 {
if expected.Size != cks.Sum().Size || expected.MD5 != "" && expected.MD5 != cks.Sum().MD5 ||
expected.SHA1 != "" && expected.SHA1 != cks.Sum().SHA1 || expected.SHA256 != "" && expected.SHA256 != cks.Sum().SHA256 {
if ignoreMismatch {
fmt.Printf("WARNING: checksums don't match: %#v != %#v for %s\n", expected, cks.Sum(), url)
} else {
result <- fmt.Errorf("checksums don't match: %#v != %#v for %s", expected, cks.Sum(), url)
return
}
}
}
result <- nil
return
}
// handleTask processes single download task
func (downloader *downloaderImpl) handleTask(task *downloadTask) {
downloader.progress.Printf("Downloading %s...\n", task.url)
req, err := http.NewRequest("GET", task.url, nil)
if err != nil {
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
req.Close = true
proxyURL, _ := downloader.client.Transport.(*http.Transport).Proxy(req)
if proxyURL == nil && (req.URL.Scheme == "http" || req.URL.Scheme == "https") {
req.URL.Opaque = strings.Replace(req.URL.RequestURI(), "+", "%2b", -1)
req.URL.RawQuery = ""
}
resp, err := downloader.client.Do(req)
if err != nil {
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
if resp.Body != nil {
defer resp.Body.Close()
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
task.result <- &HTTPError{Code: resp.StatusCode, URL: task.url}
return
}
err = os.MkdirAll(filepath.Dir(task.destination), 0755)
if err != nil {
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
temppath := task.destination + ".down"
outfile, err := os.Create(temppath)
if err != nil {
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
defer outfile.Close()
checksummer := utils.NewChecksumWriter()
writers := []io.Writer{outfile, downloader.aggWriter}
if task.expected.Size != -1 {
writers = append(writers, checksummer)
}
w := io.MultiWriter(writers...)
_, err = io.Copy(w, resp.Body)
if err != nil {
os.Remove(temppath)
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
if task.expected.Size != -1 {
actual := checksummer.Sum()
if actual.Size != task.expected.Size {
err = fmt.Errorf("%s: size check mismatch %d != %d", task.url, actual.Size, task.expected.Size)
} else if task.expected.MD5 != "" && actual.MD5 != task.expected.MD5 {
err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", task.url, actual.MD5, task.expected.MD5)
} else if task.expected.SHA1 != "" && actual.SHA1 != task.expected.SHA1 {
err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
}
if err != nil {
if task.ignoreMismatch {
downloader.progress.Printf("WARNING: %s\n", err.Error())
} else {
os.Remove(temppath)
task.result <- err
return
}
}
}
err = os.Rename(temppath, task.destination)
if err != nil {
os.Remove(temppath)
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
task.result <- nil
}
// handleTask processes single download task
func (downloader *downloaderImpl) handleTask(task *downloadTask) {
downloader.progress.Printf("Downloading %s...\n", task.url)
resp, err := downloader.client.Get(task.url)
if err != nil {
task.result <- err
return
}
if resp.Body != nil {
defer resp.Body.Close()
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
task.result <- fmt.Errorf("HTTP code %d while fetching %s", resp.StatusCode, task.url)
return
}
err = os.MkdirAll(filepath.Dir(task.destination), 0755)
if err != nil {
task.result <- err
return
}
temppath := task.destination + ".down"
outfile, err := os.Create(temppath)
if err != nil {
task.result <- err
return
}
defer outfile.Close()
checksummer := utils.NewChecksumWriter()
writers := []io.Writer{outfile, downloader.aggWriter}
if task.expected.Size != -1 {
writers = append(writers, checksummer)
}
w := io.MultiWriter(writers...)
_, err = io.Copy(w, resp.Body)
if err != nil {
os.Remove(temppath)
task.result <- err
return
}
if task.expected.Size != -1 {
actual := checksummer.Sum()
if actual.Size != task.expected.Size {
err = fmt.Errorf("%s: size check mismatch %d != %d", task.url, actual.Size, task.expected.Size)
} else if task.expected.MD5 != "" && actual.MD5 != task.expected.MD5 {
err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", task.url, actual.MD5, task.expected.MD5)
} else if task.expected.SHA1 != "" && actual.SHA1 != task.expected.SHA1 {
err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
}
if err != nil {
if task.ignoreMismatch {
downloader.progress.Printf("WARNING: %s\n", err.Error())
} else {
os.Remove(temppath)
task.result <- err
return
}
}
}
err = os.Rename(temppath, task.destination)
if err != nil {
os.Remove(temppath)
task.result <- err
return
}
task.result <- nil
}
func (downloader *downloaderImpl) downloadTask(req *http.Request, task *downloadTask) (string, error) {
resp, err := downloader.client.Do(req)
if err != nil {
return "", fmt.Errorf("%s: %s", task.url, err)
}
if resp.Body != nil {
defer resp.Body.Close()
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return "", &HTTPError{Code: resp.StatusCode, URL: task.url}
}
err = os.MkdirAll(filepath.Dir(task.destination), 0777)
if err != nil {
return "", fmt.Errorf("%s: %s", task.url, err)
}
temppath := task.destination + ".down"
outfile, err := os.Create(temppath)
if err != nil {
return "", fmt.Errorf("%s: %s", task.url, err)
}
defer outfile.Close()
checksummer := utils.NewChecksumWriter()
writers := []io.Writer{outfile, downloader.aggWriter}
if task.expected.Size != -1 {
writers = append(writers, checksummer)
}
w := io.MultiWriter(writers...)
_, err = io.Copy(w, resp.Body)
if err != nil {
os.Remove(temppath)
return "", fmt.Errorf("%s: %s", task.url, err)
}
if task.expected.Size != -1 {
actual := checksummer.Sum()
if actual.Size != task.expected.Size {
err = fmt.Errorf("%s: size check mismatch %d != %d", task.url, actual.Size, task.expected.Size)
} else if task.expected.MD5 != "" && actual.MD5 != task.expected.MD5 {
err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", task.url, actual.MD5, task.expected.MD5)
} else if task.expected.SHA1 != "" && actual.SHA1 != task.expected.SHA1 {
err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
} else if task.expected.SHA512 != "" && actual.SHA512 != task.expected.SHA512 {
err = fmt.Errorf("%s: sha512 hash mismatch %#v != %#v", task.url, actual.SHA512, task.expected.SHA512)
}
if err != nil {
if task.ignoreMismatch {
downloader.progress.Printf("WARNING: %s\n", err.Error())
} else {
os.Remove(temppath)
return "", err
}
}
}
return temppath, nil
}