This repository has been archived by the owner on Apr 23, 2020. It is now read-only.
/
utils.go
98 lines (90 loc) · 1.63 KB
/
utils.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
package main
import (
"crypto/sha512"
"fmt"
"io/ioutil"
"net/http"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/transform"
"github.com/PuerkitoBio/goquery"
)
var (
pt = fmt.Printf
sp = fmt.Sprintf
)
func getBytes(client *http.Client, url string) ([]byte, error) {
retry := 3
get:
resp, err := client.Get(url)
if err != nil {
if retry > 0 {
retry--
goto get
} else {
return nil, me(err, "get")
}
}
defer resp.Body.Close()
bs, err := ioutil.ReadAll(resp.Body)
if err != nil {
if retry > 0 {
retry--
goto get
} else {
return nil, me(err, "get")
}
}
return bs, nil
}
func getDoc(client *http.Client, url string) (*goquery.Document, error) {
retry := 3
get:
resp, err := client.Get(url)
if err != nil {
if retry > 0 {
retry--
goto get
} else {
return nil, me(err, "get")
}
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
if retry > 0 {
retry--
goto get
} else {
return nil, me(err, "new document from response")
}
}
return doc, nil
}
func getGbkDoc(client *http.Client, url string) (*goquery.Document, error) {
retry := 3
get:
resp, err := client.Get(url)
if err != nil {
if retry > 0 {
retry--
goto get
} else {
return nil, me(err, "get")
}
}
defer resp.Body.Close()
r := transform.NewReader(resp.Body, simplifiedchinese.GBK.NewDecoder())
doc, err := goquery.NewDocumentFromReader(r)
if err != nil {
if retry > 0 {
retry--
goto get
} else {
return nil, me(err, "new document from response")
}
}
return doc, nil
}
func hash(in string) []byte {
sum := sha512.Sum512([]byte(in))
return sum[:]
}