Esempio n. 1
0
func UTF8(cs string, data []byte) ([]byte, error) {
	if strings.ToUpper(cs) == "UTF-8" {
		return data, nil
	}

	r, err := charset.NewReader(cs, bytes.NewReader(data))
	if err != nil {
		return []byte{}, err
	}

	return ioutil.ReadAll(r)

}
Esempio n. 2
0
func parsePart(mediaType, charsetStr, encoding string, part []byte) (html, text []byte, err error) {
	// deal with charset
	if strings.ToLower(charsetStr) == "iso-8859-1" {
		var cr io.Reader
		cr, err = charset.NewReader("latin1", bytes.NewReader(part))
		if err != nil {
			return
		}

		part, err = ioutil.ReadAll(cr)
		if err != nil {
			return
		}
	}

	// deal with encoding
	var body []byte
	if strings.ToLower(encoding) == "quoted-printable" {
		dec := qprintable.NewDecoder(qprintable.WindowsTextEncoding, bytes.NewReader(part))
		body, err = ioutil.ReadAll(dec)
		if err != nil {
			return
		}
	} else {
		body = part
	}

	// deal with media type
	mediaType = strings.ToLower(mediaType)
	switch {
	case strings.Contains(mediaType, "text/html"):
		html = body
	case strings.Contains(mediaType, "text/plain"):
		text = body
	}
	return
}
Esempio n. 3
0
func Google(query string, start int, num int, ipv6 bool) (ret []GoogleResult, hasNext bool) {
	url := "https://google.com/search"

	if ipv6 {
		url = "https://ipv6.google.com/search"
	}

	res, err := httpclient.
		WithHeader("Accept-Charset", "utf-8").
		Get(url, map[string]string{
			"hl":    "en",
			"q":     query,
			"start": fmt.Sprintf("%d", start),
			"num":   fmt.Sprintf("%d", num),
		})

	r, _ := charset.NewReader("iso-8859-1", res.Body)

	//defer r.Close()
	defer res.Body.Close()

	if (err != nil) || (res.StatusCode != 200) {
		return
	}

	doc, errDoc := goquery.NewDocumentFromReader(r)

	if errDoc != nil {
		return
	}

	doc.Find(itemSel).Each(func(i int, sel *goquery.Selection) {
		link := sel.Find(linkSel)
		desc := sel.Find(descSel)
		href, ok := link.Attr("href")

		if ok {
			u, err := qs.Parse(href)

			if err != nil {
				return
			}

			desc.Find("div").Remove()

			url := strings.Trim(u.Query().Get("q"), " ")

			// Rule out the 'news'
			if strings.HasPrefix(url, "http") {
				ret = append(ret, GoogleResult{
					title:   link.First().Text(),
					url:     url,
					summary: desc.Text(),
				})
			}
		}
	})

	if doc.Find(nextSel).Last().Text() == "Next" {
		hasNext = true
	}

	return
}
func paradoxReadTable(path string, table paradoxTable, fixHour int) error {
	// Read file in slice
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	var h header
	r := bytes.NewReader(data)

	// RecordSize
	binary.Read(r, binary.LittleEndian, &h.recordSize)
	// HeaderSize
	binary.Read(r, binary.LittleEndian, &h.headerSize)
	// Num Fields in table
	r.Seek(offsetNumFields, os.SEEK_SET)
	binary.Read(r, binary.LittleEndian, &h.numFields)

	// Get data for fields
	var fields []field
	r.Seek(offsetFieldTypeSize, os.SEEK_SET)
	for i := 0; i < int(h.numFields); i++ {
		var f field
		binary.Read(r, binary.LittleEndian, &f.typ)
		binary.Read(r, binary.LittleEndian, &f.size)
		fields = append(fields, f)
	}

	// Skip trash data: tablenameptr, fieldnameptr
	r.Seek(int64(4+4*h.numFields+261), os.SEEK_CUR)

	// Get fields name
	for i := 0; i < int(h.numFields); i++ {
		var name string
		for {
			b, _ := r.ReadByte()
			if b == 0x00 { // delimiter
				break
			}
			name += string(b)
		}
		fields[i].name = string(name)
	}

	// For blocks of data in file
	nblock := int64(1)
	for {
		r.Seek(int64(h.headerSize)*nblock, os.SEEK_SET)
		nblock++
		var bl block
		err = binary.Read(r, binary.LittleEndian, &bl.nextBlock)
		if err != nil { // EOF
			break
		}
		binary.Read(r, binary.LittleEndian, &bl.prevBlock)
		binary.Read(r, binary.LittleEndian, &bl.addSize)

		if bl.addSize > 32767 {
			bl.addSize = 0
			bl.numRecs = 0
		} else {
			bl.numRecs = (bl.addSize / h.recordSize) + 1
		}

		// For records in blocks
		for i := 0; i < int(bl.numRecs); i++ {
			// For fields in record
			curRecSize := 0
			var vals []interface{}
			for j := 0; j < int(h.numFields); j++ {
				typ := fields[j].typ
				size := fields[j].size
				curRecSize += int(size)
				sbuf := make([]byte, 1024)
				r.Read(sbuf[:size])
				switch typ {
				case PX_Field_Type_LongInt, PX_Field_Type_Incremental, PX_Field_Type_Number:
					if sbuf[0] == 0x80 {
						sbuf[0] = 0x00
					}
					vals = append(vals, int(intBE32(sbuf[:size])))
				case PX_Field_Type_Date:
					vals = append(vals, "")
				case PX_Field_Type_Timestamp:
					sbuf[0] &= 0x7f
					var timestamp int64
					timestamp = int64(intBE64(sbuf[:size]))
					timestamp >>= 8
					timestamp /= 500
					timestamp -= 37603860709183
					timestamp -= 10800 + int64(fixHour*60*60) // fix hours
					vals = append(vals, time.Unix(timestamp, 0))
				default:
					sbuf = bytes.Trim(sbuf, "\x00")
					size = uint8(len(sbuf))
					r, _ := charset.NewReader("windows-1251", strings.NewReader(string(sbuf[:size])))
					result, _ := ioutil.ReadAll(r)
					vals = append(vals, string(result))
				}
			}
			table.appendRow(vals...)
			if curRecSize < int(h.recordSize) {
				r.Seek(int64(int(h.recordSize)-curRecSize), os.SEEK_CUR)
			}
		}
	}
	return nil
}