Esempio n. 1
0
//转码
func ConvertToString(src string, srcCode string, tagCode string) string {
	srcCoder := mahonia.NewDecoder(srcCode)
	srcResult := srcCoder.ConvertString(src)
	tagCoder := mahonia.NewDecoder(tagCode)
	_, cdata, _ := tagCoder.Translate([]byte(srcResult), true)
	result := string(cdata)
	return result
}
Esempio n. 2
0
// Function ProcessFile process the given file and incorporate the information
// into the NGramGenerator g for future N-Gram model generation.
func (g *NGramGenerator) ProcessFile(filename string) error {
	var decoder mahonia.Decoder
	if g.charset != "" {
		decoder = mahonia.NewDecoder(g.charset)
	}
	lineProcessor := func(line string) (bool, error) {
		line = strings.Trim(line, " \t\n\f\b\r")
		if decoder != nil {
			line = decoder.ConvertString(line)
		}
		tokens := strings.Split(line, " ")
		var prevToken string
		for i, t := range tokens {
			//Monogram frequency
			g.uniGram[t]++
			g.uniGramCount++
			//Bigram frequency
			var key BiGramKey
			if i == 0 {
				key = BiGramKey{SentenceStartTag, t}
			} else {
				key = BiGramKey{prevToken, t}
			}
			fmt.Printf("%v\n", key)
			g.biGram[key]++
			g.biGramCount++
			prevToken = t
		}
		return true, nil
	}
	return util.ForEachLineInFile(filename, lineProcessor)
}
Esempio n. 3
0
func NewSegCNCorpus(source_charset string) *SegCNCorpus {
	if source_charset != "" {
		return &SegCNCorpus{nil, 0, mahonia.NewDecoder(source_charset)}
	}
	return &SegCNCorpus{nil, 0, nil}

}
Esempio n. 4
0
File: xml.go Progetto: lanior/upc
func XmlCharsetReader(charset string, input io.Reader) (io.Reader, error) {
	decoder := mahonia.NewDecoder(charset)
	if decoder == nil {
		return nil, fmt.Errorf("Unknown charset %s", charset)
	}
	return decoder.NewReader(input), nil
}
Esempio n. 5
0
func (self *Grepper) Grep() {
	for arg := range self.In {

		fh, err := os.Open(arg.Path)
		f := bufio.NewReader(fh)
		if err != nil {
			panic(err)
		}
		buf := make([]byte, 1024)
		decoder := mahonia.NewDecoder(arg.Encode)

		m := make([]*print.Match, 0)

		var lineNum = 1
		for {
			buf, _, err = f.ReadLine()
			if err != nil {
				break
			}

			s := string(buf)
			if decoder != nil && arg.Encode != file.UTF8 && arg.Encode != file.ASCII {
				s = decoder.ConvertString(s)
			}
			if strings.Contains(s, arg.Pattern) {
				m = append(m, &print.Match{lineNum, s})
			}
			lineNum++
		}
		self.Out <- &print.Params{arg.Pattern, arg.Path, m}
		fh.Close()

	}
	close(self.Out)
}
Esempio n. 6
0
func main() {
	content, err := ioutil.ReadFile("studygolang.xml")
	if err != nil {
		log.Fatal(err)
	}
	var result Result
	err = xml.Unmarshal(content, &result)
	if err != nil {
		log.Fatal(err)
	}
	log.Println(result)
	log.Println(result.Persons[0].Name)

	//input := "<?xml version=\"1.0\" encoding=\"GBK\"?><request><head><h_exch_code>800101</h_exch_code><h_bank_no>1111</h_bank_no><h_user_id>1038738897</h_user_id><h_branch_id>B00008211</h_branch_id><h_fact_date>20110321</h_fact_date><h_fact_time>16:28:30</h_fact_time><h_exch_date>20130929</h_exch_date><h_serial_no>123456</h_serial_no><h_rsp_code>hj123545</h_rsp_code><h_rsp_msg>ar</h_rsp_msg></head><body><record><user_pwd>54f9b3396fe28c208d525db21588965c</user_pwd></record></body></request>"
	//inputReader := strings.NewReader(input)
	content, err = ioutil.ReadFile("request.xml")
	decoder := mahonia.NewDecoder("gb18030")
	r := bufio.NewReader(decoder.NewReader(content))
	if err != nil {
		log.Fatal(err)
	}
	log.Println(content)
	var request Request_800101
	err = xml.Unmarshal(content, &request)
	if err != nil {
		log.Fatal(err)
	}
	log.Println(request)
}
Esempio n. 7
0
func (ur *utf8reader) UTF8Reader(charset string, body io.Reader) (r io.Reader, err error) {
	alias := strings.ToLower(charset)
	newname, ok := charsetaliases[alias]
	if ok {
		charset = newname
	}
	r = mahonia.NewDecoder(charset).NewReader(body)
	return
}
Esempio n. 8
0
func convert(s string) string {
	var dec mahonia.Decoder
	dec = mahonia.NewDecoder("gbk")
	if ret, ok := dec.ConvertStringOK(s); ok {
		fmt.Println("GBK to UTF-8: ", ret)
		return ret
	}

	return s
}
Esempio n. 9
0
func (dt *DbfTable) getNormalizedFieldName(name string) (s string) {
	e := mahonia.NewEncoder(dt.fileEncoding)
	b := []byte(e.ConvertString(name))

	if len(b) > 10 {
		b = b[0:10]
	}

	d := mahonia.NewDecoder(dt.fileEncoding)
	s = d.ConvertString(string(b))

	return
}
Esempio n. 10
0
func (this *QQwry) Find(ip string) {
	if this.filepath == "" {
		return
	}

	file, err := os.OpenFile(this.filepath, os.O_RDONLY, 0400)
	defer file.Close()
	if err != nil {
		return
	}
	this.file = file

	this.Ip = ip
	offset := this.searchIndex(binary.BigEndian.Uint32(net.ParseIP(ip).To4()))
	// log.Println("loc offset:", offset)
	if offset <= 0 {
		return
	}

	var country []byte
	var area []byte

	mode := this.readMode(offset + 4)
	// log.Println("mode", mode)
	if mode == REDIRECT_MODE_1 {
		countryOffset := this.readUInt24()
		mode = this.readMode(countryOffset)
		// log.Println("1 - mode", mode)
		if mode == REDIRECT_MODE_2 {
			c := this.readUInt24()
			country = this.readString(c)
			countryOffset += 4
		} else {
			country = this.readString(countryOffset)
			countryOffset += uint32(len(country) + 1)
		}
		area = this.readArea(countryOffset)
	} else if mode == REDIRECT_MODE_2 {
		countryOffset := this.readUInt24()
		country = this.readString(countryOffset)
		area = this.readArea(offset + 8)
	} else {
		country = this.readString(offset + 4)
		area = this.readArea(offset + uint32(5+len(country)))
	}

	enc := mahonia.NewDecoder("gbk")
	this.Country = enc.ConvertString(string(country))
	this.City = enc.ConvertString(string(area))

}
Esempio n. 11
0
func toUtf8(content []byte) (error, string) {
	detector := chardet.NewTextDetector()
	result, err := detector.DetectBest(content)
	if err != nil {
		return err, ""
	}

	if result.Charset == "utf8" {
		return nil, string(content)
	}

	decoder := mahonia.NewDecoder(result.Charset)
	return nil, decoder.ConvertString(string(content))
}
Esempio n. 12
0
File: mzs2.go Progetto: rose312/mzr
func SelfPage(cururl string) {

	x, _ := goquery.NewDocument(cururl)
	title := x.Find("title").Text()

	if url, b := x.Find("#content a img").Attr("src"); b == true {

		if alt, b := x.Find("#content a img").Attr("alt"); b == true {
			title = alt
		}

		enc := mahonia.NewDecoder("gbk")
		if _, e := AddSpiderData(url, enc.ConvertString(title)); e != nil {
			fmt.Println(e)
		}
	}

}
Esempio n. 13
0
func GetTotalPhysicalMemory() (int64, error) {
	d, err := exec.Command("cmd", "/c", "wmic memphysical get MaxCapacity").Output()
	if err != nil {
		return -1, err
	}

	dec := mahonia.NewDecoder("GBK")

	s := dec.ConvertString(string(d))

	r := strings.NewReader(s)

	var tmp string
	var n int64
	fmt.Fscanln(r, &tmp)
	fmt.Fscanln(r, &n)

	return n, nil
}
Esempio n. 14
0
func Test_quick(t *testing.T) {
	dbfHandle := DBFOpen("../map/bou2_4p.dbf", "rb")
	defer DBFClose(dbfHandle)

	decoder := mahonia.NewDecoder("gbk")

	for i := 0; i < DBFGetRecordCount(dbfHandle); i++ {
		for j := 0; j < DBFGetFieldCount(dbfHandle); j++ {
			name, type_, _, _ := DBFGetFieldInfo(dbfHandle, j)
			switch type_ {
			case String:
				p(name, decoder.ConvertString(string(DBFReadStringAttribute(dbfHandle, i, j))))
			case Integer, Logical:
				p(name, DBFReadIntegerAttribute(dbfHandle, i, j))
			case Double:
				p(name, DBFReadDoubleAttribute(dbfHandle, i, j))
			}
		}
	}
}
Esempio n. 15
0
func (ac *Account) Login() error {
	var pld PreLoginData
	err := pld.Load(ac.UserName)
	if err != nil {
		return err
	}
	client := &http.Client{}
	data := `entry=weibo&gateway=1&from=&savestate=7&useticket=1&pagerefer=http%3A%2F%2Fweibo.com%2Fa%2Fdownload&vsnf=1&su=` + ac.UserName + `&service=miniblog&servertime=` + strconv.FormatInt(pld.Servertime, 10) + `&nonce=` + pld.Nonce + `&pwencode=rsa2&rsakv=` + pld.Rsakv + `&sp=` + pld.GetPwd(ac.Password) + `&encoding=UTF-8&prelt=415&url=http%3A%2F%2Fweibo.com%2Fajaxlogin.php%3Fframelogin%3D1%26callback%3Dparent.sinaSSOController.feedBackUrlCallBack&returntype=META`
	fmt.Printf("form data: %v\n", data)
	req, err := http.NewRequest("POST", SSO_LOGIN_URL+"?"+data, nil) // bytes.NewBuffer([]byte(data)))
	if err != nil {
		return err
	}
	SetHeader(req)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	fmt.Printf("resp: %v\n", resp)
	defer resp.Body.Close()
	fmt.Println("sso login cookie: ", resp.Header["Set-Cookie"])
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	enc := mahonia.NewDecoder("GBK")
	bodyString := enc.ConvertString(string(body))
	sIdx := strings.Index(bodyString, `replace("`)
	if sIdx > 0 {
		ac.loginUrl = bodyString[sIdx+9:]
		sIdx = strings.Index(ac.loginUrl, `")`)
		if sIdx > 0 {
			ac.loginUrl = ac.loginUrl[:sIdx]
			fmt.Println("loginUrl:", ac.loginUrl)
			if strings.Contains(ac.loginUrl, "ticket") {
				return ac.saveCookie()
			}
		}
	}
	return errors.New(fmt.Sprintf("fail to login, loginUrl: %s", ac.loginUrl))
}
Esempio n. 16
0
func reparseHtml(s string) (*xmlpath.Node, error) {
	content := mahonia.NewDecoder("cp932").ConvertString(s)

	doc, err := xhtml.Parse(strings.NewReader(content))
	if err != nil {
		return nil, fmt.Errorf("could not parse HTML for %s ...(snip): %v",
			content[:30], err)
	}

	var b bytes.Buffer
	xhtml.Render(&b, doc)
	fixed := strings.NewReader(b.String())

	root, err := xmlpath.ParseHTML(fixed)
	if err != nil {
		return nil, fmt.Errorf("could not rebuild HTML for %s ...(snip): %v",
			content[:30], err)
	}

	return root, nil
}
Esempio n. 17
0
func readFile(fileName string) (deals []StockDeal) {
	s := []StockDeal{}
	file, err := os.Open(fileName)
	if err != nil {
		fmt.Println("open File Error!")
		return s
	}
	defer file.Close()
	rd := bufio.NewReader(file)
	decoder := mahonia.NewDecoder("gbk")
	if decoder == nil {
		fmt.Println("编码不存在!")
		return s
	}

	for {
		str, err := rd.ReadString('\n')

		str = decoder.ConvertString(str)
		fmt.Println(str)
		if strings.HasPrefix(str, "20") {
			strs := strings.Fields(str)
			fmt.Println(strs)
			var deal StockDeal
			deal.date = strs[0]      //日期
			deal.time = strs[1]      //时间
			deal.code = strs[2]      //代码
			deal.name = strs[3]      //名称
			deal.deal = strs[4]      //买入或者卖出
			deal.callPrice = strs[5] //成交价
			s = append(s, deal)

		}
		if err == io.EOF || err != nil {
			fmt.Println("Read Over")
			break
		}
	}
	return s
}
Esempio n. 18
0
func readWebPageSource(url string, encode ETargetEncodeType) (string, error) {
	for i := 1; i <= HttpTryCnt; i++ {

		rsp, err := http.Get(url)
		if err != nil {
			continue
		}
		defer rsp.Body.Close()

		if rsp.StatusCode == http.StatusOK {
			decoder := mahonia.NewDecoder(string(encode))
			data, e := ioutil.ReadAll(decoder.NewReader(rsp.Body))
			if e != nil {
				continue
			}

			return string(data), e
		}
	}

	return "", fmt.Errorf("read web page source timeout.")
}
Esempio n. 19
0
func (dt *DbfTable) FieldValue(row int, fieldIndex int) (value string) {

	// create decoder to convert bytes to utf-8
	d := mahonia.NewDecoder(dt.fileEncoding)

	offset := int(dt.numberOfBytesInHeader)
	lengthOfRecord := int(dt.lengthOfEachRecord)

	offset = offset + (row * lengthOfRecord)

	recordOffset := 1

	for i := 0; i < len(dt.fields); i++ {
		if i == fieldIndex {
			break
		} else {
			recordOffset += int(dt.fields[i].fieldLength)
		}
	}

	temp := dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.fields[fieldIndex].fieldLength))]

	for i := 0; i < len(temp); i++ {
		if temp[i] == 0x00 {
			temp = temp[0:i]
			break
		}
	}

	s := d.ConvertString(string(temp))
	//fmt.Printf("utf-8 value:[%#v]\n", s)

	value = strings.TrimSpace(s)

	//fmt.Printf("raw value:[%#v]\n", dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.Fields[fieldIndex].fieldLength))])
	//fmt.Printf("utf-8 value:[%#v]\n", []byte(s))
	//value = string(dt.dataStore[(offset + recordOffset):((offset + recordOffset) + int(dt.Fields[fieldIndex].fieldLength))])
	return
}
Esempio n. 20
0
func (requestInfo RequestInformation) DecodeResponseBody(body io.Reader) (*SaltResponse, error) {
	charset := mahonia.NewDecoder(requestInfo.Charset)
	if charset == nil {
		return nil, errors.New("charset is null.")
	}

	r := charset.NewReader(body)
	decoder := xml.NewDecoder(r)

	decoder.CharsetReader = CharsetReader

	for {
		token, err := decoder.Token()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		switch startElement := token.(type) {
		case xml.StartElement:
			if startElement.Name.Space == "http://schemas.xmlsoap.org/soap/envelope/" && startElement.Name.Local == "Body" {
				//nextElementIsBody = true
				responseBody := SaltResponse{}
				err = decoder.DecodeElement(&responseBody, &startElement)

				if err != nil {

					return nil, err

				}
				return &responseBody, nil
			}
		}
	}

	return nil, errors.New("Did not find SOAP body element")
}
Esempio n. 21
0
func readfile() string {
	f, err := os.Open("c:/test/test.txt")
	if err != nil {
		return err.Error()
	}
	defer f.Close()
	buf := make([]byte, 1024)
	//文件ex7.txt的编码是gb18030
	decoder := mahonia.NewDecoder("gb18030")
	if decoder == nil {
		return "编码不存在!"
	}
	var str string = ""
	for {
		n, _ := f.Read(buf)
		if 0 == n {
			break
		}
		//解码为UTF-8
		str += decoder.ConvertString(string(buf[:n]))
	}
	fmt.Println(str)
	return str
}
Esempio n. 22
0
func main() {
	flag.Parse()

	var r io.Reader = os.Stdin
	var w io.Writer = os.Stdout

	if *from != "utf-8" {
		decode := mahonia.NewDecoder(*from)
		if decode == nil {
			log.Fatalf("Could not create decoder for %s", *from)
		}
		r = decode.NewReader(r)
	}

	if *to != "utf-8" {
		encode := mahonia.NewEncoder(*to)
		if encode == nil {
			log.Fatalf("Could not create decoder for %s", *to)
		}
		w = encode.NewWriter(w)
	}

	io.Copy(w, r)
}
Esempio n. 23
0
const (
	SAVE_LOG_FILE       = "save.log"
	SAVE_PATH           = "save"
	URL_ROOT            = "http://www.info.pmda.go.jp"
	URL_PSEARCH         = URL_ROOT + "/psearch/"
	URL_PSEARCH_KENSAKU = URL_PSEARCH + "html/menu_tenpu_kensaku.html"
	HISTORY_PATH        = "history.json"
	SAVED_LIST_PATH     = "tmp_saved.txt"
	DRUG_LOG            = "drugpath.txt"
)

var (
	w, _    = os.Open(SAVE_LOG_FILE)
	SAVELOG = log.New(w, "", 0)
	DECODER = mahonia.NewDecoder("euc-jp")
)

type URL string

func (url URL) Get() *bufio.Reader {
	return get(string(url))
}

func (url URL) GetAndDecode() io.Reader {
	return getanddecode(string(url))
}

func (url URL) GetRaw() io.Reader {
	return getraw(string(url))
}
Esempio n. 24
0
func main() {
	flag.Parse()

	var f *os.File
	var err error

	switch flag.NArg() {
	case 0:
		f = os.Stdin
	case 1:
		f, err = os.Open(flag.Arg(0))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		defer f.Close()
	default:
		flag.Usage()
		os.Exit(1)
	}

	dec := mahonia.NewDecoder(*flag_e)
	if dec == nil {
		fmt.Fprintln(os.Stderr, "Unknown encoding name")
		os.Exit(1)
	}

	t := uint32(0)

	out := syscall.Handle(os.Stdout.Fd())

	var csbi consoleScreenBufferInfo
	procGetConsoleScreenBufferInfo.Call(uintptr(out), uintptr(unsafe.Pointer(&csbi)))
	attr_old := csbi.attributes
	defer func() {
		procSetConsoleTextAttribute.Call(uintptr(out), uintptr(attr_old))
	}()

	timer := time.NewTimer(0)

	quit := make(chan bool)
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, os.Interrupt)
	go func() {
		<-sc
		timer.Stop()
		quit <- true
	}()

	var lastbuf bytes.Buffer
	var scroll *smallRect

loop:
	for {
		var header [3]uint32
		err = binary.Read(f, binary.LittleEndian, &header)
		if err != nil {
			break
		}

		if !*flag_n {
			cur := header[0]*1000000 + header[1]
			if t != 0 {
				timer.Reset(time.Duration(float64(cur-t) / *flag_s) * time.Microsecond)
				select {
				case <-timer.C:
				case <-quit:
					break loop
				}
			}
			t = cur
		}

		var data []byte
		off := 0
		if lastbuf.Len() > 0 {
			data = make([]byte, int(header[2])+lastbuf.Len())
			copy(data[:lastbuf.Len()], lastbuf.Bytes())
			off += lastbuf.Len()
			lastbuf.Reset()
		} else {
			data = make([]byte, header[2])
		}
		for off < int(header[2]) {
			n, err := f.Read(data[off:])
			if err != nil {
				break
			}
			off += n
		}

		er := dec.NewReader(bytes.NewBuffer(data))
	parse:
		for {
			r1, _, err := procGetConsoleScreenBufferInfo.Call(uintptr(out), uintptr(unsafe.Pointer(&csbi)))
			if r1 == 0 {
				break loop
			}

			c1, _, err := er.ReadRune()
			if err != nil {
				break parse
			}
			if c1 != 0x1b {
				switch {
				case c1 == 0x08:
					if csbi.cursorPosition.x > 0 {
						csbi.cursorPosition.x -= 1
					}
					r1, _, _ := procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&csbi.cursorPosition))))
					if r1 == 0 {
						break loop
					}
				case c1 == 0x0a:
					if scroll != nil && csbi.cursorPosition.y == scroll.bottom {
						var ci charInfo
						ci.unicodeChar = ' '
						ci.attributes = csbi.attributes
						move := scroll
						move.top++
						xy := coord{
							x: 0,
							y: scroll.top,
						}
						r1, _, _ = procScrollConsoleScreenBuffer.Call(uintptr(out), uintptr(unsafe.Pointer(&move)), 0, uintptr(*(*int32)(unsafe.Pointer(&xy))), uintptr(unsafe.Pointer(&ci)))
						if r1 == 0 {
							break loop
						}
					} else if csbi.cursorPosition.y < csbi.window.bottom {
						csbi.cursorPosition.y++
						r1, _, _ := procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&csbi.cursorPosition))))
						if r1 == 0 {
							break loop
						}
					} else {
						fmt.Print(string(c1))
					}
				case c1 == '\r' || c1 == '\t' || c1 >= 0x20:
					if *flag_d {
						debug("OUT:" + string(c1))
					}
					fmt.Print(string(c1))
				}
				continue
			}
			c2, _, err := er.ReadRune()
			if err != nil {
				lastbuf.WriteRune(c1)
				break parse
			}

			var buf bytes.Buffer
			var m rune
			switch c2 {
			case 0x5b:
				for {
					c, _, err := er.ReadRune()
					if err != nil {
						lastbuf.WriteRune(c1)
						lastbuf.WriteByte(0x5b)
						lastbuf.Write(buf.Bytes())
						break parse
					}
					if ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '@' {
						m = c
						break
					}
					buf.Write([]byte(string(c)))
				}
			case 0x5d:
				for {
					c, _, err := er.ReadRune()
					if err != nil {
						lastbuf.Write(buf.Bytes())
						break parse
					}
					if c == ';' {
						break
					}
					buf.Write([]byte(string(c)))
				}
				continue
			}

			if *flag_d {
				debug("ESC:" + buf.String() + string(m))
			}
			var n int
			switch m {
			case 'h':
				if _, err := fmt.Sscanf(buf.String(), "%d", &n); err != nil {
					switch n {
					case 47:
						xy := coord{
							y: csbi.window.top,
							x: csbi.window.left,
						}
						procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&xy))))
					}
				}
			case '@':
				if _, err := fmt.Sscanf(buf.String(), "%d", &n); err != nil {
					n = 1
				}
				var ci charInfo
				ci.unicodeChar = ' '
				ci.attributes = csbi.attributes
				var move smallRect
				move.top = csbi.cursorPosition.y
				move.bottom = move.top
				move.left = csbi.cursorPosition.x
				move.right = csbi.size.x - short(n)
				xy := coord{
					x: csbi.cursorPosition.x + short(n),
					y: csbi.cursorPosition.y,
				}
				r1, _, _ = procScrollConsoleScreenBuffer.Call(uintptr(out), uintptr(unsafe.Pointer(&move)), 0, uintptr(*(*int32)(unsafe.Pointer(&xy))), uintptr(unsafe.Pointer(&ci)))
				if r1 == 0 {
					break loop
				}
				r1, _, _ = procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&csbi.cursorPosition))))
				if r1 == 0 {
					break loop
				}
			case 'm':
				attr := csbi.attributes
				cs := buf.String()
				if cs == "" {
					procSetConsoleTextAttribute.Call(uintptr(out), uintptr(attr_old))
					continue
				}
				for _, ns := range strings.Split(cs, ";") {
					if n, err = strconv.Atoi(ns); err == nil {
						switch {
						case n == 0 || n == 100:
							attr = attr_old
						case 1 <= n && n <= 5:
							attr |= foregroundIntensity
						case n == 7:
							attr = ((attr & foregroundMask) << 4) | ((attr & backgroundMask) >> 4)
						case 22 == n || n == 25 || n == 25:
							attr |= foregroundIntensity
						case n == 27:
							attr = ((attr & foregroundMask) << 4) | ((attr & backgroundMask) >> 4)
						case 30 <= n && n <= 37:
							attr = (attr & backgroundMask)
							if (n-30)&1 != 0 {
								attr |= foregroundRed
							}
							if (n-30)&2 != 0 {
								attr |= foregroundGreen
							}
							if (n-30)&4 != 0 {
								attr |= foregroundBlue
							}
						case 40 <= n && n <= 47:
							attr = (attr & foregroundMask)
							if (n-40)&1 != 0 {
								attr |= backgroundRed
							}
							if (n-40)&2 != 0 {
								attr |= backgroundGreen
							}
							if (n-40)&4 != 0 {
								attr |= backgroundBlue
							}
						}
						procSetConsoleTextAttribute.Call(uintptr(out), uintptr(attr))
					}
				}
			case 'A':
				ns, _ := fmt.Sscanf(buf.String(), "%d", &n)
				if ns == 0 {
					csbi.cursorPosition.y--
				} else {
					csbi.cursorPosition.y -= short(n)
				}
				r1, _, _ = procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&csbi.cursorPosition))))
				if r1 == 0 {
					break loop
				}
			case 'B':
				ns, _ := fmt.Sscanf(buf.String(), "%d", &n)
				if ns == 0 {
					csbi.cursorPosition.y++
				} else {
					csbi.cursorPosition.y += short(n)
				}
				r1, _, _ = procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&csbi.cursorPosition))))
				if r1 == 0 {
					break loop
				}
			case 'C':
				ns, _ := fmt.Sscanf(buf.String(), "%d", &n)
				if ns == 0 {
					csbi.cursorPosition.x++
				} else {
					csbi.cursorPosition.x += short(n)
				}
				r1, _, _ = procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&csbi.cursorPosition))))
				if r1 == 0 {
					break loop
				}
			case 'D':
				ns, _ := fmt.Sscanf(buf.String(), "%d", &n)
				if ns == 0 {
					csbi.cursorPosition.x--
				} else {
					csbi.cursorPosition.x -= short(n)
				}
				r1, _, _ = procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&csbi.cursorPosition))))
				if r1 == 0 {
					break loop
				}
			case 'J':
				if _, err = fmt.Sscanf(buf.String(), "%d", &n); err != nil {
					n = 0
				}
				switch n {
				case 0:
					cursor := coord{
						x: csbi.cursorPosition.x,
						y: csbi.cursorPosition.y,
					}
					var count, w dword
					count = dword(csbi.size.x - csbi.cursorPosition.x + (csbi.size.y-csbi.cursorPosition.y)*csbi.size.x)
					r1, _, _ = procFillConsoleOutputCharacter.Call(uintptr(out), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
					r1, _, _ = procFillConsoleOutputAttribute.Call(uintptr(out), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
				case 1:
					cursor := coord{
						x: csbi.window.left,
						y: csbi.window.top,
					}
					var count, w dword
					count = dword(csbi.cursorPosition.x + (csbi.cursorPosition.y-1)*csbi.size.x)
					r1, _, _ = procFillConsoleOutputCharacter.Call(uintptr(out), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
					r1, _, _ = procFillConsoleOutputAttribute.Call(uintptr(out), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
				case 2:
					cursor := coord{
						x: csbi.window.left,
						y: csbi.window.top,
					}
					var count, w dword
					count = dword(csbi.size.x * csbi.size.y)
					r1, _, _ = procFillConsoleOutputCharacter.Call(uintptr(out), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
					r1, _, _ = procFillConsoleOutputAttribute.Call(uintptr(out), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
				}
			case 'K':
				fmt.Sscanf(buf.String(), "%d", &n)
				switch n {
				case 0:
					cursor := coord{
						x: csbi.cursorPosition.x,
						y: csbi.cursorPosition.y,
					}
					var count, w dword
					count = dword(csbi.size.x - csbi.cursorPosition.x)
					r1, _, _ = procFillConsoleOutputCharacter.Call(uintptr(out), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
					r1, _, _ = procFillConsoleOutputAttribute.Call(uintptr(out), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
				case 1:
					cursor := coord{
						x: csbi.window.left,
						y: csbi.window.top + csbi.cursorPosition.y,
					}
					var count, w dword
					count = dword(csbi.cursorPosition.x)
					r1, _, _ = procFillConsoleOutputCharacter.Call(uintptr(out), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
					r1, _, _ = procFillConsoleOutputAttribute.Call(uintptr(out), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
				case 2:
					cursor := coord{
						x: csbi.window.left,
						y: csbi.window.top + csbi.cursorPosition.y,
					}
					var count, w dword
					count = dword(csbi.size.x)
					r1, _, _ = procFillConsoleOutputCharacter.Call(uintptr(out), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
					r1, _, _ = procFillConsoleOutputAttribute.Call(uintptr(out), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
					if r1 == 0 {
						break loop
					}
				}
			case 'H':
				var xy coord
				ns, _ := fmt.Sscanf(buf.String(), "%d;%d", &xy.y, &xy.x)
				if ns == 1 {
					xy.y--
				} else if ns == 2 {
					xy.y--
					xy.x--
				}
				xy.y += csbi.window.top
				xy.x += csbi.window.left
				procSetConsoleCursorPosition.Call(uintptr(out), uintptr(*(*int32)(unsafe.Pointer(&xy))))
			case 'r':
				scroll = &smallRect{}
				ns, _ := fmt.Sscanf(buf.String(), "%d;%d", &scroll.top, &scroll.left)
				scroll.left = csbi.window.left
				scroll.right = csbi.window.right
				if ns == 0 {
					scroll = nil
				} else if ns == 1 {
					scroll.top--
				} else if ns == 2 {
					scroll.bottom--
				}
			}
		}
	}
}
Esempio n. 25
0
func NewFromFile(fileName string, fileEncoding string) (table *DbfTable, err error) {
	// create a decoder to decode file correctly
	d := mahonia.NewDecoder(fileEncoding)

	s, err := readFile(fileName)

	if err != nil {
		return nil, err
	}

	// Create and pupulate DbaseTable struct
	dt := new(DbfTable)

	dt.fileEncoding = fileEncoding

	// read dbase table header information
	dt.fileSignature = s[0]
	dt.updateYear = s[1]
	dt.updateMonth = s[2]
	dt.updateDay = s[3]
	dt.numberOfRecords = uint32(s[4]) | (uint32(s[5]) << 8) | (uint32(s[6]) << 16) | (uint32(s[7]) << 24)
	dt.numberOfBytesInHeader = uint16(s[8]) | (uint16(s[9]) << 8)
	dt.lengthOfEachRecord = uint16(s[10]) | (uint16(s[11]) << 8)

	// create fieldMap to taranslate field name to index
	dt.fieldMap = make(map[string]int)

	// Number of fields in dbase table
	dt.numberOfFields = int((dt.numberOfBytesInHeader - 1 - 32) / 32)

	// populate dbf fields
	for i := 0; i < int(dt.numberOfFields); i++ {
		offset := (i * 32) + 32

		fieldName := strings.Trim(d.ConvertString(string(s[offset:offset+10])), string([]byte{0}))
		dt.fieldMap[fieldName] = i

		var err error

		switch s[offset+11] {
		case 'C':
			err = dt.AddTextField(fieldName, s[offset+16])
		case 'N':
			err = dt.AddNumberField(fieldName, s[offset+16])
		case 'F':
			err = dt.AddFloatField(fieldName, s[offset+16])
		case 'L':
			err = dt.AddBooleanField(fieldName)
		case 'D':
			err = dt.AddDateField(fieldName)
		}

		// Check return value for errors
		if err != nil {
			return nil, err
		}

		//fmt.Printf("Field name:%v\n", fieldName)
		//fmt.Printf("Field data type:%v\n", string(s[offset+11]))
		//fmt.Printf("Field length:%v\n", s[offset+16])
		//fmt.Println("-----------------------------------------------")
	}

	//fmt.Printf("DbfReader:\n%#v\n", dt)
	//fmt.Printf("DbfReader:\n%#v\n", int(dt.Fields[2].fieldLength))

	//fmt.Printf("num records in table:%v\n", (dt.numberOfRecords))
	//fmt.Printf("lenght of each record:%v\n", (dt.lengthOfEachRecord))

	// Since we are reading dbase file from the disk at least at this
	// phase changing schema of dbase file is not allowed.
	dt.dataEntryStarted = true

	// set DbfTable dataStore slice that will store the complete file in memory
	dt.dataStore = s

	return dt, nil
}
Esempio n. 26
0
// Function NewUtf8Converter creates a Utf8 converter that can convert the
// source_charset to Utf8.
func NewUtf8Converter(source_charset string) *Utf8Converter {
	return &Utf8Converter{mahonia.NewDecoder(source_charset)}
}
Esempio n. 27
0
File: gof.go Progetto: dohq/gof
func main() {
	flag.Parse()

	var err error
	cwd := ""

	if flag.NFlag() == 0 && flag.NArg() == 1 {
		*root = flag.Arg(0)
	}

	if *root == "" {
		cwd, err = os.Getwd()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	} else {
		if runtime.GOOS == "windows" && strings.HasPrefix(*root, "/") {
			cwd, _ = os.Getwd()
			cwd = filepath.Join(filepath.VolumeName(cwd), *root)
		} else {
			cwd, err = filepath.Abs(*root)
		}
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		st, err := os.Stat(cwd)
		if err == nil && !st.IsDir() {
			err = fmt.Errorf("Directory not found: %s", cwd)
		}
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		err = os.Chdir(cwd)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

	dirty := false
	terminating := false
	var quit chan bool

	timer := time.AfterFunc(0, func() {
		if dirty {
			filter()
			draw_screen()
			dirty = false
		} else {
			draw_screen()
		}
	})
	timer.Stop()

	is_tty := isatty()

	if !is_tty {
		var buf *bufio.Reader
		if enc := os.Getenv("GOFSTDINENC"); enc != "" {
			buf = bufio.NewReader(mahonia.NewDecoder(enc).NewReader(os.Stdin))
		} else {
			buf = bufio.NewReader(os.Stdin)
		}
		for {
			b, _, err := buf.ReadLine()
			if err != nil {
				break
			}
			mutex.Lock()
			files = append(files, string(b))
			mutex.Unlock()
			dirty = true
			timer.Reset(duration)
		}
		err = tty_ready()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	} else if *launcher {
		home := os.Getenv("HOME")
		if home == "" && runtime.GOOS == "windows" {
			home = os.Getenv("USERPROFILE")
		}
		b, err := ioutil.ReadFile(filepath.Join(home, ".gof-launcher"))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		launcherFiles = strings.Split(string(b), "\n")
		for _, line := range launcherFiles {
			cols := strings.SplitN(line, "\t", 2)
			if len(cols) == 2 {
				files = append(files, cols[0])
			}
		}
		err = tty_ready()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

	err = termbox.Init()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if is_tty {
		termbox.SetInputMode(termbox.InputEsc)
	}

	filter()
	draw_screen()

	// Walk and collect files recursively.
	if len(files) == 0 {
		quit = make(chan bool)
		go func() {
			filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
				if terminating {
					return errors.New("terminate")
				}
				if info == nil {
					return err
				}
				if !info.IsDir() {
					if p, err := filepath.Rel(cwd, path); err == nil {
						path = p
					}
					path = filepath.ToSlash(path)
					mutex.Lock()
					files = append(files, path)
					mutex.Unlock()
					dirty = true
					timer.Reset(duration)
				} else if strings.HasPrefix(info.Name(), ".") {
					return filepath.SkipDir
				}
				return nil
			})
			scanning = -1
			quit <- true
		}()
	}

loop:
	for {
		update := false

		// Polling key events
		switch ev := termbox.PollEvent(); ev.Type {
		case termbox.EventKey:
			switch ev.Key {
			case termbox.KeyEsc, termbox.KeyCtrlD, termbox.KeyCtrlC:
				termbox.Close()
				os.Exit(1)
			case termbox.KeyHome, termbox.KeyCtrlA:
				cursor_x = 0
			case termbox.KeyEnd, termbox.KeyCtrlE:
				cursor_x = len(input)
			case termbox.KeyEnter:
				if cursor_y >= 0 && cursor_y < len(current) {
					if len(selected) == 0 {
						selected = append(selected, current[cursor_y].name)
					}
					break loop
				}
			case termbox.KeyArrowLeft:
				if cursor_x > 0 {
					cursor_x--
				}
			case termbox.KeyArrowRight:
				if cursor_x < len([]rune(input)) {
					cursor_x++
				}
			case termbox.KeyArrowUp, termbox.KeyCtrlK:
				if cursor_y < len(current)-1 {
					if cursor_y < height-4 {
						cursor_y++
					}
				}
			case termbox.KeyArrowDown, termbox.KeyCtrlJ:
				if cursor_y > 0 {
					cursor_y--
				}
			case termbox.KeyCtrlO:
				if cursor_y >= 0 && cursor_y < len(current) {
					*edit = true
					if len(selected) == 0 {
						selected = append(selected, current[cursor_y].name)
					}
					break loop
				}
			case termbox.KeyCtrlI:
				heading = !heading
			case termbox.KeyCtrlL:
				update = true
			case termbox.KeyCtrlU:
				cursor_x = 0
				input = []rune{}
				update = true
			case termbox.KeyCtrlW:
				part := string(input[0:cursor_x])
				rest := input[cursor_x:len(input)]
				pos := regexp.MustCompile(`\s+`).FindStringIndex(part)
				if len(pos) > 0 && pos[len(pos)-1] > 0 {
					println(pos[len(pos)-1])
					input = []rune(part[0 : pos[len(pos)-1]-1])
					input = append(input, rest...)
				} else {
					input = []rune{}
				}
				cursor_x = len(input)
				update = true
			case termbox.KeyCtrlZ:
				found := -1
				name := current[cursor_y].name
				for i, s := range selected {
					if name == s {
						found = i
						break
					}
				}
				if found == -1 {
					selected = append(selected, current[cursor_y].name)
				} else {
					selected = append(selected[:found], selected[found+1:]...)
				}
				update = true
			case termbox.KeyBackspace, termbox.KeyBackspace2:
				if cursor_x > 0 {
					input = append(input[0:cursor_x-1], input[cursor_x:len(input)]...)
					cursor_x--
					update = true
				}
			case termbox.KeyDelete:
				if cursor_x < len([]rune(input)) {
					input = append(input[0:cursor_x], input[cursor_x+1:len(input)]...)
					update = true
				}
			default:
				if ev.Key == termbox.KeySpace {
					ev.Ch = ' '
				}
				if ev.Ch > 0 {
					out := []rune{}
					out = append(out, input[0:cursor_x]...)
					out = append(out, ev.Ch)
					input = append(out, input[cursor_x:len(input)]...)
					cursor_x++
					update = true
				}
			}
		case termbox.EventError:
			update = false
		}

		// If need to update, start timer
		if scanning != -1 {
			if update {
				dirty = true
				timer.Reset(duration)
			} else {
				timer.Reset(1)
			}
		} else {
			if update {
				filter()
			}
			draw_screen()
		}
	}
	timer.Stop()

	// Request terminating
	terminating = true
	if quit != nil {
		<-quit
	}

	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
	termbox.Close()

	tty_term()

	if len(selected) == 0 {
		os.Exit(1)
	}

	if !*launcher {
		for i, f := range selected {
			selected[i] = filepath.Join(cwd, f)
		}
	}

	if *launcher {
		for _, f := range selected {
			for _, line := range launcherFiles {
				cols := strings.SplitN(line, "\t", 2)
				if len(cols) == 2 && cols[0] == f {
					stdin := os.Stdin
					stdout := os.Stdout
					var shell, shellcflag string
					if runtime.GOOS == "windows" {
						stdin, _ = os.Open("CONIN$")
						stdout, _ = os.Open("CONOUT$")
						shell = os.Getenv("COMSPEC")
						if shell == "" {
							shell = "cmd"
						}
						shellcflag = "/c"
					} else {
						stdin = os.Stdin
						shell = os.Getenv("SHELL")
						if shell == "" {
							shell = "sh"
						}
						shellcflag = "-c"
					}
					cmd := exec.Command(shell, shellcflag, strings.TrimSpace(cols[1]))
					cmd.Stdin = stdin
					cmd.Stdout = stdout
					cmd.Stderr = os.Stderr
					err = cmd.Start()
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
					}
					return
				}
			}
		}
	} else if *edit {
		err = edit_file(selected)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	} else if *cat {
		for _, f := range selected {
			f, err := os.Open(f)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				continue
			}
			io.Copy(os.Stdout, f)
			f.Close()
		}
	} else if *remove {
		for _, f := range selected {
			os.Remove(f)
		}
	} else if flag.NArg() > 0 {
		args := flag.Args()
		args = append(args, selected...)
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		err = cmd.Start()
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		cmd.Wait()
	} else {
		for _, f := range selected {
			fmt.Println(f)
		}
	}
}
Esempio n. 28
0
func convertFileName(fname string) string {
	d := mahonia.NewDecoder("cp932")
	return d.ConvertString(fname)
}
Esempio n. 29
0
func (m_client *MyHttpClient) Do() error {
	m_client.ContentBytes = nil

	//New Request
	var req *http.Request
	var req_err error
	if m_client.PostData != nil {
		req, req_err = http.NewRequest(m_client.Method, m_client.Url, strings.NewReader(m_client.PostData.Encode()))
	} else {
		req, req_err = http.NewRequest(m_client.Method, m_client.Url, nil)
	}

	log.Println(req.URL.String())
	if req_err != nil {
		return req_err
	}

	//add header
	if m_client.ContentType == "" {
		m_client.ContentType = "text/html;charset=gb2312"
		m_client.Decode = "gb2312"
	}
	req.Header.Set("Content-Type", m_client.ContentType)
	req.Header.Set("Referer", m_client.Refer)

	if m_client.CookieFile != "" {
		AddCookies(req, m_client.CookieFile)
	}

	//New HTTP CLIENT
	client := &http.Client{}
	resp, client_err := client.Do(req)
	if client_err != nil {
		return client_err
	}
	defer resp.Body.Close()

	//cookie
	if m_client.SaveCookieFlag && len(resp.Cookies()) > 0 {
		SaveCookies(resp.Cookies(), m_client.CookieFile)
	}

	//login result
	resp_body, read_err := ioutil.ReadAll(resp.Body)
	if read_err != nil {
		return read_err
	}

	//convert gb2312 to utf-8
	if m_client.Decode != "" {
		dec := mahonia.NewDecoder(m_client.Decode)
		ret, ok := dec.ConvertStringOK(string(resp_body))
		if !ok {
			return errors.New("convert gb2312 to utf-8 err!")
		}
		m_client.ContentBytes = []byte(ret)
	} else {
		m_client.ContentBytes = resp_body
	}

	return nil
}
Esempio n. 30
0
// http://www.jma.go.jp/jp/amedas_h/today-46211.html
// http://www6.kaiho.mlit.go.jp/03kanku/shimoda/ shift-jis
// http://www6.kaiho.mlit.go.jp/03kanku/yokosuka/kisyou.html
func getWindData(c appengine.Context) (*WindData, error) {
	windData := &WindData{}
	client := urlfetch.Client(c)
	resp, err := client.Get(MICS_URL)
	if err != nil {
		return nil, fmt.Errorf("could not get %s: %v", MICS_URL, err)
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("server responded non-200: %s, %s", MICS_URL, resp.Status)
	}

	defer resp.Body.Close()
	// http://stackoverflow.com/questions/24101721/parse-broken-html-with-golang
	buf := new(bytes.Buffer)
	buf.ReadFrom(resp.Body)
	content := mahonia.NewDecoder("cp932").ConvertString(buf.String())

	doc, err := xhtml.Parse(strings.NewReader(content))
	// https://godoc.org/golang.org/x/net/html
	if err != nil {
		return nil, fmt.Errorf("could not parse HTML for %s: %v", MICS_URL, err)
	}
	var b bytes.Buffer
	xhtml.Render(&b, doc)
	fixed := strings.NewReader(b.String())

	root, err := xmlpath.ParseHTML(fixed)
	if err != nil {
		return nil, fmt.Errorf("could not parse HTML: %s\n Error: %v", content, err)
	}

	path := xmlpath.MustCompile(MICS_TABLE_XPATH)
	table, ok := path.String(root)
	if !ok {
		return nil, fmt.Errorf("could not find table path")
	}
	re := regexp.MustCompile("([^\n])\n")
	windData.Table = re.ReplaceAllString(table, "$1 ")

	path = xmlpath.MustCompile(MICS_DATE_XPATH)
	date, ok := path.String(root)
	if !ok {
		return nil, fmt.Errorf("could not find date")
	}
	windData.Date = date

	imgResp, err := client.Get(MICS_SHIMODA_IMG_URL)
	if err != nil {
		return nil, fmt.Errorf("unable to get img from %s: %v", MICS_SHIMODA_IMG_URL, err)
	}
	if imgResp.StatusCode != 200 {
		return nil, fmt.Errorf("img server responded non-200: %s, %s", MICS_SHIMODA_IMG_URL, imgResp.Status)
	}
	defer imgResp.Body.Close()

	// XXX need to resize the image for Gratina2
	// JPG is more available: http://media.kddi.com/app/publish/torisetsu/pdf/gratina2_torisetsu_shousai.pdf
	// go image packages
	// image/gif, image/jpeg: http://golang.org/pkg/image/gif/#Encode

	pngImg, err := png.Decode(imgResp.Body)
	if err != nil {
		// we can do with only text info
		c.Infof("No image attached. Could not decode png: %v", err)
		return windData, nil
	}
	buf.Reset()
	err = jpeg.Encode(buf, pngImg, &jpeg.Options{Quality: 75})
	if err != nil {
		// we can do with text info only
		c.Infof("No image attached. Could not encode to jpeg: %v", err)
		return windData, nil
	}
	windData.Img = buf.Bytes()
	return windData, nil
}