func http_get(url string) (res []byte) {
req, _ := http.NewRequest("GET", url, nil)
if SID != "" {
req.AddCookie(&http.Cookie{Name: "sid", Value: SID})
}
r, er := new(http.Client).Do(req)
if er != nil {
println(url, er.Error())
os.Exit(1)
}
if SID == "" {
for i := range r.Cookies() {
if r.Cookies()[i].Name == "sid" {
SID = r.Cookies()[i].Value
//fmt.Println("sid", SID)
}
}
}
if r.StatusCode == 200 {
defer r.Body.Close()
res, _ = ioutil.ReadAll(r.Body)
} else {
println(url, "http.Get returned code", r.StatusCode)
os.Exit(1)
}
return
}
// Sends a notification to the GCM gateway.
func (this *Client) Send(n *Notification) string {
req, _ := http.NewRequest("POST", this.Gateway, strings.NewReader(n.ToJSON()))
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", "key="+this.ApiKey)
resp, err := new(http.Client).Do(req)
defer resp.Body.Close()
if err != nil {
return err.Error()
}
body, _ := ioutil.ReadAll(resp.Body)
return string(body)
}
func Get(url string) (string, error) { // From http://stackoverflow.com/questions/11692860/how-can-i-efficiently-download-a-large-file-using-go
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", fmt.Errorf("Request creation error: %s", err.Error())
}
req.Header.Set("User-Agent", UserAgent)
resp, err := new(http.Client).Do(req)
if err != nil {
return "", fmt.Errorf("HTTP request error: %s", err.Error())
}
defer resp.Body.Close()
buf := new(bytes.Buffer)
_, err = io.Copy(buf, resp.Body)
if err != nil {
return "", fmt.Errorf("Buffer copy error: %s", err.Error())
}
return string(buf.Bytes()), nil
}
func (this *WebAppServer) HandleGet(w http.ResponseWriter, request *http.Request) {
URL := this.WebAppURL + request.URL.Path
if request.URL.RawQuery != "" {
URL += "?" + request.URL.RawQuery
}
req, err := http.NewRequest(request.Method, URL, request.Body)
if err != nil {
w.WriteHeader(500)
fmt.Fprint(w, "bab error :(", err.Error())
return
}
this.copyHeader(request.Header, req.Header)
resp, err := new(http.Client).Do(req)
if err != nil {
w.WriteHeader(500)
fmt.Fprint(w, "bab error :(\n", err.Error())
return
}
this.copyHeader(resp.Header, w.Header())
w.WriteHeader(resp.StatusCode)
defer resp.Body.Close()
io.Copy(w, resp.Body)
}
func GetPics(archive uint64) (rawhtml string, urls []string, err error) {
req, err := http.NewRequest("GET", ArchivePrefix+strconv.FormatUint(archive, 10), nil)
if err != nil {
return "", nil, fmt.Errorf("Request creation error: %s", err.Error())
}
req.Header.Set("User-Agent", UserAgent)
func() {
CookieLock.RLock()
defer CookieLock.RUnlock()
req.AddCookie(&Cookie)
}()
resp, err := new(http.Client).Do(req)
if err != nil {
return "", nil, fmt.Errorf("HTTP request error: %s", err.Error())
}
defer resp.Body.Close()
buf := new(bytes.Buffer)
io.Copy(buf, resp.Body)
rawhtml = buf.String()
var doc *goquery.Document
doc, err = goquery.NewDocumentFromReader(buf)
if err != nil {
return
}
if doc.Find("title").First().Text()[:7] == "You are" {
UpdateCookie(doc)
return GetPics(archive)
}
urls = make([]string, 0)
doc.Find("div .entry-content").Find("img").Each(func(i int, s *goquery.Selection) {
if img, ok := s.Attr("data-lazy-src"); ok {
urls = append(urls, img)
}
})
return
}
func GetArchives(manga uint64) (title string, mangas []Archive, err error) {
req, err := http.NewRequest("GET", MangaPrefix+strconv.FormatUint(manga, 10), nil)
if err != nil {
return "", nil, fmt.Errorf("Request creation error: %s", err.Error())
}
req.Header.Set("User-Agent", UserAgent)
func() {
CookieLock.RLock()
defer CookieLock.RUnlock()
req.AddCookie(&Cookie)
}()
resp, err := new(http.Client).Do(req)
if err != nil {
return "", nil, fmt.Errorf("HTTP request error: %s", err.Error())
}
defer resp.Body.Close()
buf := new(bytes.Buffer)
io.Copy(buf, resp.Body)
var doc *goquery.Document
doc, err = goquery.NewDocumentFromReader(buf)
if err != nil {
return
}
if doc.Find("title").First().Text()[:7] == "You are" {
UpdateCookie(doc)
return GetArchives(manga)
}
links := make([]Archive, 0)
seq := uint64(1)
title = Filter(doc.Find("div .subject").Find("h1").Text(), "_")
if title == "" {
title = Filter(strings.Replace(doc.Find("title").First().Text(), " | SHENCOMICS", "", 1), "_")
if title == "" {
title = "Untitled"
}
}
needFix := make(map[uint64]*struct {
needs []int
fix int
})
doc.Find("div .content").Children().Find("a").Each(func(i int, s *goquery.Selection) {
if l, ok := s.Attr("href"); ok && strings.Index(l, ArchivePrefix) != -1 {
if link, err := strconv.ParseUint(l[strings.Index(l, ArchivePrefix)+len(ArchivePrefix):], 10, 64); err == nil {
sub := regexp.MustCompile("^[\xc2\xa0 \\t]+").ReplaceAllString(s.Text(), "")
if sub == "" {
for _, l := range links {
if l.ID == link {
return
}
}
sub = "Untitled"
if _, ok := needFix[link]; ok {
needFix[link].needs = append(needFix[link].needs, len(links))
} else {
needFix[link] = &struct {
needs []int
fix int
}{
[]int{len(links)},
-1,
}
}
} else {
if index, ok := needFix[link]; ok && index.fix < 0 {
needFix[link].fix = len(links)
}
}
links = append(links, Archive{
ID: link,
Seq: seq,
Title: fmt.Sprintf("%s(%d)", title, manga),
Subject: sub,
})
seq++
}
}
})
for _, st := range needFix {
if st.fix > 0 {
for _, need := range st.needs {
links[need].Subject = links[st.fix].Subject
}
}
}
return title, links, nil
}