// open func openUrl(url string) (data string, err error) { var resp *http.Response var raw []byte var dec mahonia.Decoder = nil debug.Println("Get:", url) resp, err = http.Get(url) if err != nil { return } if resp.StatusCode != http.StatusOK { err = errors.New("Bad Status:" + resp.Status) return } // only handle html files ctype := resp.Header.Get("Content-Type") if -1 == strings.Index(ctype, "text/html") { err = errors.New("Not a html file") return } // try enconding: gbk\big5\utf8 charset := "" if seps := strings.Split(ctype, "="); len(seps) >= 2 { charset = seps[1] charset = strings.ToLower(charset) if strings.HasPrefix(charset, "gb") { charset = "gb18030" dec = gbk } else if strings.HasPrefix(charset, "big") { charset = "big5" dec = big5 } else if strings.HasPrefix(charset, "utf") || charset == "" { charset = "utf8" dec = nil } else { err = errors.New("Unsupported charset:" + charset) return } } else { dec = nil } debug.Println("Using charset:", charset) // TODO gzip handle contentEncoding := resp.Header.Get("Content-Encoding") if contentEncoding == "gzip" { err = errors.New("Content-Encoding:" + contentEncoding + "temporally not supported") return } // read the response if dec != nil { raw, err = ioutil.ReadAll(dec.NewReader(resp.Body)) } else { raw, err = ioutil.ReadAll(resp.Body) } if err != nil { return } defer resp.Body.Close() data = string(raw) debug.Println("Data:", data) return }