func UTF8(cs string, data []byte) ([]byte, error) { if strings.ToUpper(cs) == "UTF-8" { return data, nil } r, err := charset.NewReader(cs, bytes.NewReader(data)) if err != nil { return []byte{}, err } return ioutil.ReadAll(r) }
func parsePart(mediaType, charsetStr, encoding string, part []byte) (html, text []byte, err error) { // deal with charset if strings.ToLower(charsetStr) == "iso-8859-1" { var cr io.Reader cr, err = charset.NewReader("latin1", bytes.NewReader(part)) if err != nil { return } part, err = ioutil.ReadAll(cr) if err != nil { return } } // deal with encoding var body []byte if strings.ToLower(encoding) == "quoted-printable" { dec := qprintable.NewDecoder(qprintable.WindowsTextEncoding, bytes.NewReader(part)) body, err = ioutil.ReadAll(dec) if err != nil { return } } else { body = part } // deal with media type mediaType = strings.ToLower(mediaType) switch { case strings.Contains(mediaType, "text/html"): html = body case strings.Contains(mediaType, "text/plain"): text = body } return }
func Google(query string, start int, num int, ipv6 bool) (ret []GoogleResult, hasNext bool) { url := "https://google.com/search" if ipv6 { url = "https://ipv6.google.com/search" } res, err := httpclient. WithHeader("Accept-Charset", "utf-8"). Get(url, map[string]string{ "hl": "en", "q": query, "start": fmt.Sprintf("%d", start), "num": fmt.Sprintf("%d", num), }) r, _ := charset.NewReader("iso-8859-1", res.Body) //defer r.Close() defer res.Body.Close() if (err != nil) || (res.StatusCode != 200) { return } doc, errDoc := goquery.NewDocumentFromReader(r) if errDoc != nil { return } doc.Find(itemSel).Each(func(i int, sel *goquery.Selection) { link := sel.Find(linkSel) desc := sel.Find(descSel) href, ok := link.Attr("href") if ok { u, err := qs.Parse(href) if err != nil { return } desc.Find("div").Remove() url := strings.Trim(u.Query().Get("q"), " ") // Rule out the 'news' if strings.HasPrefix(url, "http") { ret = append(ret, GoogleResult{ title: link.First().Text(), url: url, summary: desc.Text(), }) } } }) if doc.Find(nextSel).Last().Text() == "Next" { hasNext = true } return }
func paradoxReadTable(path string, table paradoxTable, fixHour int) error { // Read file in slice data, err := ioutil.ReadFile(path) if err != nil { return err } var h header r := bytes.NewReader(data) // RecordSize binary.Read(r, binary.LittleEndian, &h.recordSize) // HeaderSize binary.Read(r, binary.LittleEndian, &h.headerSize) // Num Fields in table r.Seek(offsetNumFields, os.SEEK_SET) binary.Read(r, binary.LittleEndian, &h.numFields) // Get data for fields var fields []field r.Seek(offsetFieldTypeSize, os.SEEK_SET) for i := 0; i < int(h.numFields); i++ { var f field binary.Read(r, binary.LittleEndian, &f.typ) binary.Read(r, binary.LittleEndian, &f.size) fields = append(fields, f) } // Skip trash data: tablenameptr, fieldnameptr r.Seek(int64(4+4*h.numFields+261), os.SEEK_CUR) // Get fields name for i := 0; i < int(h.numFields); i++ { var name string for { b, _ := r.ReadByte() if b == 0x00 { // delimiter break } name += string(b) } fields[i].name = string(name) } // For blocks of data in file nblock := int64(1) for { r.Seek(int64(h.headerSize)*nblock, os.SEEK_SET) nblock++ var bl block err = binary.Read(r, binary.LittleEndian, &bl.nextBlock) if err != nil { // EOF break } binary.Read(r, binary.LittleEndian, &bl.prevBlock) binary.Read(r, binary.LittleEndian, &bl.addSize) if bl.addSize > 32767 { bl.addSize = 0 bl.numRecs = 0 } else { bl.numRecs = (bl.addSize / h.recordSize) + 1 } // For records in blocks for i := 0; i < int(bl.numRecs); i++ { // For fields in record curRecSize := 0 var vals []interface{} for j := 0; j < int(h.numFields); j++ { typ := fields[j].typ size := fields[j].size curRecSize += int(size) sbuf := make([]byte, 1024) r.Read(sbuf[:size]) switch typ { case PX_Field_Type_LongInt, PX_Field_Type_Incremental, PX_Field_Type_Number: if sbuf[0] == 0x80 { sbuf[0] = 0x00 } vals = append(vals, int(intBE32(sbuf[:size]))) case PX_Field_Type_Date: vals = append(vals, "") case PX_Field_Type_Timestamp: sbuf[0] &= 0x7f var timestamp int64 timestamp = int64(intBE64(sbuf[:size])) timestamp >>= 8 timestamp /= 500 timestamp -= 37603860709183 timestamp -= 10800 + int64(fixHour*60*60) // fix hours vals = append(vals, time.Unix(timestamp, 0)) default: sbuf = bytes.Trim(sbuf, "\x00") size = uint8(len(sbuf)) r, _ := charset.NewReader("windows-1251", strings.NewReader(string(sbuf[:size]))) result, _ := ioutil.ReadAll(r) vals = append(vals, string(result)) } } table.appendRow(vals...) if curRecSize < int(h.recordSize) { r.Seek(int64(int(h.recordSize)-curRecSize), os.SEEK_CUR) } } } return nil }