func UTF8(cs string, data []byte) ([]byte, error) {
if strings.ToUpper(cs) == "UTF-8" {
return data, nil
}
r, err := charset.NewReader(cs, bytes.NewReader(data))
if err != nil {
return []byte{}, err
}
return ioutil.ReadAll(r)
}
func parsePart(mediaType, charsetStr, encoding string, part []byte) (html, text []byte, err error) {
// deal with charset
if strings.ToLower(charsetStr) == "iso-8859-1" {
var cr io.Reader
cr, err = charset.NewReader("latin1", bytes.NewReader(part))
if err != nil {
return
}
part, err = ioutil.ReadAll(cr)
if err != nil {
return
}
}
// deal with encoding
var body []byte
if strings.ToLower(encoding) == "quoted-printable" {
dec := qprintable.NewDecoder(qprintable.WindowsTextEncoding, bytes.NewReader(part))
body, err = ioutil.ReadAll(dec)
if err != nil {
return
}
} else {
body = part
}
// deal with media type
mediaType = strings.ToLower(mediaType)
switch {
case strings.Contains(mediaType, "text/html"):
html = body
case strings.Contains(mediaType, "text/plain"):
text = body
}
return
}
func Google(query string, start int, num int, ipv6 bool) (ret []GoogleResult, hasNext bool) {
url := "https://google.com/search"
if ipv6 {
url = "https://ipv6.google.com/search"
}
res, err := httpclient.
WithHeader("Accept-Charset", "utf-8").
Get(url, map[string]string{
"hl": "en",
"q": query,
"start": fmt.Sprintf("%d", start),
"num": fmt.Sprintf("%d", num),
})
r, _ := charset.NewReader("iso-8859-1", res.Body)
//defer r.Close()
defer res.Body.Close()
if (err != nil) || (res.StatusCode != 200) {
return
}
doc, errDoc := goquery.NewDocumentFromReader(r)
if errDoc != nil {
return
}
doc.Find(itemSel).Each(func(i int, sel *goquery.Selection) {
link := sel.Find(linkSel)
desc := sel.Find(descSel)
href, ok := link.Attr("href")
if ok {
u, err := qs.Parse(href)
if err != nil {
return
}
desc.Find("div").Remove()
url := strings.Trim(u.Query().Get("q"), " ")
// Rule out the 'news'
if strings.HasPrefix(url, "http") {
ret = append(ret, GoogleResult{
title: link.First().Text(),
url: url,
summary: desc.Text(),
})
}
}
})
if doc.Find(nextSel).Last().Text() == "Next" {
hasNext = true
}
return
}
func paradoxReadTable(path string, table paradoxTable, fixHour int) error {
// Read file in slice
data, err := ioutil.ReadFile(path)
if err != nil {
return err
}
var h header
r := bytes.NewReader(data)
// RecordSize
binary.Read(r, binary.LittleEndian, &h.recordSize)
// HeaderSize
binary.Read(r, binary.LittleEndian, &h.headerSize)
// Num Fields in table
r.Seek(offsetNumFields, os.SEEK_SET)
binary.Read(r, binary.LittleEndian, &h.numFields)
// Get data for fields
var fields []field
r.Seek(offsetFieldTypeSize, os.SEEK_SET)
for i := 0; i < int(h.numFields); i++ {
var f field
binary.Read(r, binary.LittleEndian, &f.typ)
binary.Read(r, binary.LittleEndian, &f.size)
fields = append(fields, f)
}
// Skip trash data: tablenameptr, fieldnameptr
r.Seek(int64(4+4*h.numFields+261), os.SEEK_CUR)
// Get fields name
for i := 0; i < int(h.numFields); i++ {
var name string
for {
b, _ := r.ReadByte()
if b == 0x00 { // delimiter
break
}
name += string(b)
}
fields[i].name = string(name)
}
// For blocks of data in file
nblock := int64(1)
for {
r.Seek(int64(h.headerSize)*nblock, os.SEEK_SET)
nblock++
var bl block
err = binary.Read(r, binary.LittleEndian, &bl.nextBlock)
if err != nil { // EOF
break
}
binary.Read(r, binary.LittleEndian, &bl.prevBlock)
binary.Read(r, binary.LittleEndian, &bl.addSize)
if bl.addSize > 32767 {
bl.addSize = 0
bl.numRecs = 0
} else {
bl.numRecs = (bl.addSize / h.recordSize) + 1
}
// For records in blocks
for i := 0; i < int(bl.numRecs); i++ {
// For fields in record
curRecSize := 0
var vals []interface{}
for j := 0; j < int(h.numFields); j++ {
typ := fields[j].typ
size := fields[j].size
curRecSize += int(size)
sbuf := make([]byte, 1024)
r.Read(sbuf[:size])
switch typ {
case PX_Field_Type_LongInt, PX_Field_Type_Incremental, PX_Field_Type_Number:
if sbuf[0] == 0x80 {
sbuf[0] = 0x00
}
vals = append(vals, int(intBE32(sbuf[:size])))
case PX_Field_Type_Date:
vals = append(vals, "")
case PX_Field_Type_Timestamp:
sbuf[0] &= 0x7f
var timestamp int64
timestamp = int64(intBE64(sbuf[:size]))
timestamp >>= 8
timestamp /= 500
timestamp -= 37603860709183
timestamp -= 10800 + int64(fixHour*60*60) // fix hours
vals = append(vals, time.Unix(timestamp, 0))
default:
sbuf = bytes.Trim(sbuf, "\x00")
size = uint8(len(sbuf))
r, _ := charset.NewReader("windows-1251", strings.NewReader(string(sbuf[:size])))
result, _ := ioutil.ReadAll(r)
vals = append(vals, string(result))
}
}
table.appendRow(vals...)
if curRecSize < int(h.recordSize) {
r.Seek(int64(int(h.recordSize)-curRecSize), os.SEEK_CUR)
}
}
}
return nil
}