/
download.go
131 lines (108 loc) · 3.32 KB
/
download.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
package main
import (
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"regexp"
"code.cloudfoundry.org/bytefmt"
"github.com/kennygrant/sanitize"
"github.com/smira/go-ftp-protocol/protocol"
)
func downloadFromURL(url string, folder string, maxretry int, httpClient *http.Client, fileName string) (path string, newEpisode bool, err error) {
for i := 1; i <= maxretry; i++ {
path, newEpisode, err = download(url, folder, httpClient, fileName)
if err == nil {
break
} else {
logger.Warning.Println("Download failure at attempt "+strconv.Itoa(i)+"/"+strconv.Itoa(maxretry)+" for url "+url, err)
}
}
return path, newEpisode, err
}
func downloadFromURLWithoutName(url string, folder string, maxretry int, httpClient *http.Client) (path string, newEpisode bool, err error) {
fileName := extractResourceNameFromURL(url)
return downloadFromURL(url, folder, maxretry, httpClient, fileName)
}
func extractResourceNameFromURL(uri string) string {
var urlPath string
parsedURL, err := url.Parse(uri)
if err != nil {
logger.Warning.Println("Cannot extract path from url : "+uri+" : ", err)
urlPath = uri
} else {
urlPath = parsedURL.Path
}
tokens := strings.Split(urlPath, "/")
resource := tokens[len(tokens)-1]
return resource
}
/**
Replace // with / in urls - except in http(s)://
*/
func cleanURL(url string) (cleanURL string) {
re := regexp.MustCompile("([^:])(\\/\\/)")
cleanURL = re.ReplaceAllString(url, "$1/")
return cleanURL
}
func download(referenceURI string, folder string, httpClient *http.Client, fileName string) (path string, newEpisode bool, err error) {
fileName = filepath.Join(folder, fileName)
fileName = sanitize.Path(fileName)
uri := cleanURL(referenceURI)
logger.Debug.Println("Local resource path : " + fileName)
tmpFilename := fileName + ".part"
resourceName := filepath.Base(folder) + " - " + filepath.Base(fileName)
defer removeTempFile(tmpFilename)
var resp *http.Response
if !pathExists(fileName) {
logger.Debug.Println("New resource available : " + resourceName)
// TODO: check file existence first with io.IsExist
output, err := os.Create(tmpFilename)
if err != nil {
return fileName, newEpisode, err
}
defer output.Close()
if strings.HasPrefix(uri, "ftp") {
logger.Debug.Println("FTP download detected")
transport := &http.Transport{}
transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
client := &http.Client{Transport: transport}
response, err := client.Get(uri)
if err != nil {
return fileName, newEpisode, err
}
resp = response
} else {
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return fileName, newEpisode, err
}
req.Close = true
response, err := httpClient.Do(req)
if err != nil {
return fileName, newEpisode, err
}
resp = response
defer response.Body.Close()
}
n, err := io.Copy(output, resp.Body)
if err != nil {
return fileName, newEpisode, err
}
logger.Debug.Println("Resource downloaded : " + resourceName + " (" + bytefmt.ByteSize(uint64(n)) + ")")
os.Rename(tmpFilename, fileName)
newEpisode = true
} else {
logger.Debug.Println("No download since the file exists", fileName)
newEpisode = false
}
return fileName, newEpisode, err
}
func removeTempFile(tmpFilename string) {
if pathExists(tmpFilename) {
os.Remove(tmpFilename)
}
}