func main() {
f, err := os.Open("test3.xml")
if err != nil {
fmt.Println(err)
return
}
n, err := xmlpath.Parse(f)
f.Close()
if err != nil {
fmt.Println(err)
return
}
q1 := xmlpath.MustCompile("//item")
if _, ok := q1.String(n); !ok {
fmt.Println("no item")
}
q2 := xmlpath.MustCompile("//price")
for it := q2.Iter(n); it.Next(); {
fmt.Println(it.Node())
}
q3 := xmlpath.MustCompile("//name")
names := []*xmlpath.Node{}
for it := q3.Iter(n); it.Next(); {
names = append(names, it.Node())
}
if len(names) == 0 {
fmt.Println("no names")
}
}
func (hfy *httphfy) Paser(reader io.Reader) *Response {
rsp := new(Response)
root, err := xmlpath.Parse(reader)
if err != nil {
rsp.Status = FAILED
rsp.Msg = "短信平台服务错误"
return rsp
}
pathstatus := xmlpath.MustCompile("//returnsms/returnstatus")
pathmessage := xmlpath.MustCompile("//returnsms/message")
if status, ok := pathstatus.String(root); ok {
if status == "Success" {
rsp.Status = SUCCESS
} else {
rsp.Status = FAILED
}
} else {
rsp.Status = FAILED
rsp.Msg = "短信平台服务错误"
return rsp
}
if message, ok := pathmessage.String(root); ok {
rsp.Msg = "短信平台:" + message
}
return rsp
}
func front_parse(c_front_urls chan string, c_front_page chan []byte, c_doc_urls chan string, wg *sync.WaitGroup) {
front_page := <-c_front_page
//fmt.Printf("%s\n", string(front_page))
//path := xmlpath.MustCompile("/html/body/div/div[2]/div/div[3]/div/div[1]/div[1]/h1/text()") //title
doc_urls_xpath := xmlpath.MustCompile("/html/body/div[@id=\"page_align\"]/div[@id=\"page_width\"]/div[@id=\"ContainerMain\"]/div[@class=\"content-border list\"]/div[@class=\"content-color\"]/div[@class=\"list-lbc\"]//a/@href") //doc urls
next_front_urls_xpath := xmlpath.MustCompile("/html/body/div[@id=\"page_align\"]/div[@id=\"page_width\"]/div[@id=\"ContainerMain\"]/nav/ul[@id=\"paging\"]/li[@class=\"page\"]") //next url
/*
front_page_noscript := remove_noscript(front_page)
fix_html := fix_broken_html(front_page_noscript)
utf8_reader := decode_utf8(fix_html)
root, err := xmlpath.ParseHTML(utf8_reader)*/
utf8_reader := decode_utf8(string(front_page))
doc_page_noscript := remove_noscript(utf8_reader)
fix_html := fix_broken_html(doc_page_noscript)
//fmt.Println(string(fix_html))
root, err := xmlpath.ParseHTML(strings.NewReader(fix_html))
if err != nil {
//log.Println("ca rentre")
log.Fatal("FRONT PAGE", err)
}
doc_urls := doc_urls_xpath.Iter(root)
for doc_urls.Next() {
doc_url := doc_urls.Node().String()
c_doc_urls <- doc_url
//log.Println( "Doc URL:", doc_url) //<-- DOC URL
}
prev_next_front_urls := next_front_urls_xpath.Iter(root)
var node *xmlpath.Node
for prev_next_front_urls.Next() {
node = prev_next_front_urls.Node()
}
href_xpath := xmlpath.MustCompile("a/@href")
if next_front_url, ok := href_xpath.String(node); ok {
c_front_urls <- next_front_url
log.Println("Next Front URL:", next_front_url)
wg.Add(1)
go front_worker(c_front_urls, c_front_page, c_doc_urls, wg)
} else {
log.Println("No Next Front URL")
log.Println("Front DONE")
return
}
}
func tarballsFrom(source tarballSource) ([]*Tarball, error) {
resp, err := http.Get(source.url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("cannot read http response: %v", err)
}
clearScripts(data)
root, err := xmlpath.ParseHTML(bytes.NewBuffer(data))
if err != nil {
return nil, err
}
var tbs []*Tarball
iter := xmlpath.MustCompile(source.xpath).Iter(root)
for iter.Next() {
s := iter.Node().String()
if strings.HasPrefix(s, "//") {
s = "https:" + s
}
if tb, ok := parseURL(s); ok {
tbs = append(tbs, tb)
}
}
if len(tbs) == 0 {
return nil, fmt.Errorf("no downloads available at " + source.url)
}
return tbs, nil
}
func loadXpath(response *http.Response, xpath string) ([]byte, error) {
body, err := ioutil.ReadAll(response.Body)
panicError(err)
// Parse body to see if login worked
// reader := strings.NewReader(body)
root, err := html.Parse(bytes.NewBuffer(body))
if err != nil {
return nil, err
}
var b bytes.Buffer
html.Render(&b, root)
fixedHtml := b
// body = bytes.NewReader(fixedHtml)
xmlroot, xmlerr := xmlpath.ParseHTML(bytes.NewReader(fixedHtml.Bytes()))
if xmlerr != nil {
return nil, xmlerr
}
path := xmlpath.MustCompile(xpath)
if value, ok := path.Bytes(xmlroot); ok {
return value, nil
}
return nil, errors.New("Could not find xpath")
}
func print_simple_selector(pathString string) {
path := xmlpath.MustCompile(pathString)
if value, ok := path.String(read_xml()); ok {
fmt.Println(value)
}
}
func main() {
if os.Getenv("AWS_KEY") == "" || len(os.Args) != 2 {
fmt.Fprint(os.Stderr, "Usage: amzn-item-lookup <itemId>.\nAWS credentials are read from the environment variables AWS_KEY and AWS_SECRET.\n")
os.Exit(-1)
}
// Construct AWS credentials
var cred AWSCredentials
cred.host = "ecs.amazonaws.co.uk"
cred.accessKey = os.Getenv("AWS_KEY")
cred.secret = os.Getenv("AWS_SECRET")
// lookup item by id
itemId := os.Args[1]
itemXml := lookupItem(cred, itemId)
// find ItemAttributes block
xml := xmlpath.MustCompile("//ItemAttributes")
iter := xml.Iter(itemXml)
if !iter.Next() {
panic("Cannot parse response! [Wrong credentials?]")
}
item := parseItemAttributes(iter.Node())
// fmt.Println(item)
// print output
fmt.Print(strings.Join(item.author, ", "), DELIM)
fmt.Print(item.title, DELIM, item.publisher, DELIM, item.edition, " ed", DELIM, item.publicationDate, DELIM)
fmt.Print(item.binding, DELIM, item.pages, " pages", DELIM, item.isbn, DELIM, item.ean, DELIM, item.price, DELIM, item.priceCurrency)
fmt.Println()
}
func (vim *VimClient) FindByInventoryPath(inventoryPath string) (vmId string, err error) {
data := struct {
InventoryPath string
}{
inventoryPath,
}
t := template.Must(template.New("FindByInventoryPath").Parse(FindByInventoryPathRequestTemplate))
log.Printf("Looking for '%s'", inventoryPath)
request, _ := vim.prepareRequest(t, data)
response, err := vim.Do(request)
// defer response.Body.Close()
if err != nil {
err = fmt.Errorf("Error calling FindByInventoryPath: '%s'", err.Error())
return
}
body, _ := ioutil.ReadAll(response.Body)
// log.Printf("RESPONSE BODY BELOW:\n============\n%s\n===========\nEND RESPONSE BODY\n", string(body))
root, _ := xmlpath.Parse(bytes.NewBuffer(body))
path := xmlpath.MustCompile("//*/FindByInventoryPathResponse/returnval")
if vmId, ok := path.String(root); ok {
return vmId, err
} else {
err := fmt.Errorf("Found nothing.")
return vmId, err
}
}
func (s *BasicSuite) TestHTML(c *C) {
node, err := xmlpath.ParseHTML(bytes.NewBuffer(trivialHtml))
c.Assert(err, IsNil)
path := xmlpath.MustCompile("/root/foo")
result, ok := path.String(node)
c.Assert(ok, Equals, true)
c.Assert(result, Equals, "<a>")
}
func (r *XMLResource) XPathIter(xpath string) *XMLIter {
path := xmlpath.MustCompile(xpath)
b := bytes.NewBufferString(string(r.Body()))
root, _ := xmlpath.Parse(b)
return &XMLIter{iter: path.Iter(root)}
}
func (s *BasicSuite) TestRootText(c *C) {
node, err := xmlpath.Parse(bytes.NewBuffer(trivialXml))
c.Assert(err, IsNil)
path := xmlpath.MustCompile("/")
result, ok := path.String(node)
c.Assert(ok, Equals, true)
c.Assert(result, Equals, "abcdefg")
}
func (r *XMLResource) XPath(xpath string) (string, bool) {
path := xmlpath.MustCompile(xpath)
b := bytes.NewBufferString(r.Body())
root, _ := xmlpath.Parse(b)
return path.String(root)
}
func getTitusCsvLink(root *xmlpath.Node) (string, error) {
path := xmlpath.MustCompile(titusCsvXpath)
csvLink, ok := path.String(root)
if ok {
return TitusCgiBaseUrl + "/" + csvLink, nil
} else {
return "", fmt.Errorf("could not find xpath %s", titusCsvXpath)
}
}
func parseVmPowerStateProperty(body *bytes.Buffer) (value string) {
root, _ := xmlpath.Parse(bytes.NewBuffer(body.Bytes()))
path := xmlpath.MustCompile("//*/RetrievePropertiesResponse/returnval/propSet[name='runtime']/val/powerState")
if value, ok := path.String(root); ok {
return value
} else {
return ""
}
}
func parseVmPropertyValue(prop string, body *bytes.Buffer) (value string) {
root, _ := xmlpath.Parse(body)
pathString := strings.Join([]string{"//*/RetrievePropertiesResponse/returnval/propSet[name='", prop, "']/val"}, "")
path := xmlpath.MustCompile(pathString)
if value, ok := path.String(root); ok {
return value
} else {
return ""
}
}
func getXPathValue(xml string, eomfile EomFile, lookupPath string) (string, bool) {
path := xmlpath.MustCompile(lookupPath)
root, err := xmlpath.Parse(strings.NewReader(xml))
if err != nil {
warnLogger.Printf("Cannot parse XML of eomfile using xpath [%v], error: [%v]", err.Error(), lookupPath)
return "", false
}
xpathValue, ok := path.String(root)
return xpathValue, ok
}
func parseTaskIdFromResponse(response *http.Response) (value string) {
body, _ := ioutil.ReadAll(response.Body)
root, _ := xmlpath.Parse(bytes.NewBuffer(body))
path := xmlpath.MustCompile("//*/CloneVM_TaskResponse/returnval")
if value, ok := path.String(root); ok {
return value
} else {
return ""
}
}
func (s *BasicSuite) BenchmarkSimplePathExists(c *C) {
node, err := xmlpath.Parse(bytes.NewBuffer(instancesXml))
c.Assert(err, IsNil)
path := xmlpath.MustCompile("/DescribeInstancesResponse/reservationSet/item/instancesSet/item/instanceType")
var exists bool
c.ResetTimer()
for i := 0; i < c.N; i++ {
exists = path.Exists(node)
}
c.StopTimer()
c.Assert(exists, Equals, true)
}
func (s *BasicSuite) BenchmarkSimplePathString(c *C) {
node, err := xmlpath.Parse(bytes.NewBuffer(instancesXml))
c.Assert(err, IsNil)
path := xmlpath.MustCompile("/DescribeInstancesResponse/reservationSet/item/instancesSet/item/instanceType")
var str string
c.ResetTimer()
for i := 0; i < c.N; i++ {
str, _ = path.String(node)
}
c.StopTimer()
c.Assert(str, Equals, "m1.small")
}
func ParseTorumemo(content io.ReadCloser) (string, string, error) {
datePath := xmlpath.MustCompile(`//div[@class="date"]`)
titlePath := xmlpath.MustCompile(`//div[@class="title"]`)
contentPath := xmlpath.MustCompile(`//div[@class="body"]/p`)
root, err := xmlpath.ParseHTML(content)
if err != nil {
return "", "", err
}
date, _ := datePath.String(root)
title, _ := titlePath.String(root)
date = strings.TrimSpace(date)
title = strings.TrimSpace(title)
iter := contentPath.Iter(root)
var body string
for iter.Next() {
body += iter.Node().String()
}
return date + " " + title, body, err
}
func parseTaskPropertyValues(body []byte, props ...string) map[string]string {
values := make(map[string]string)
for _, prop := range props {
root, _ := xmlpath.Parse(bytes.NewBuffer(body))
// pathString := fmt.Sprintf("//*/RetrievePropertiesResponse/returnval/propSet/val/%s", prop)
pathString := strings.Join([]string{"//*/RetrievePropertiesResponse/returnval/propSet/val/", prop}, "")
path := xmlpath.MustCompile(pathString)
if value, ok := path.String(root); ok {
values[prop] = value
} else {
values[prop] = ""
}
}
return values
}
func print_app_groups_json() {
path := xmlpath.MustCompile("//group/app_root")
app_iter := path.Iter(read_xml())
var entries []map[string]string
for app_iter.Next() {
entries = append(entries, map[string]string{"{#NAME}": app_iter.Node().String()})
}
data := map[string][]map[string]string{"data": entries}
json, _ := json.Marshal(data)
fmt.Println(string(json))
}
func (c *Comp) doCompXPath(r *http.Response) bool {
path := xmlpath.MustCompile(c.Path)
root, err := xmlpath.ParseHTML(r.Body)
if err != nil {
fmt.Printf("doCompXPath: %v\n", err)
return false
}
value, ok := path.String(root)
if !ok {
return false
}
ok = c.dataMatch(value)
return ok
}
// LatestVersion find latest version of the artifact
func (r *MavenRepository) LatestVersion(artifact *Artifact) (version string, err error) {
metadataURL := fmt.Sprintf("%s/maven-metadata.xml", r.artifactURLBase(artifact))
utils.Log("info", fmt.Sprintf("Search latest version to %s: %+v", metadataURL, artifact))
content, err := ValidateDownload(metadataURL)
if err != nil {
return "", err
}
xmlPath := xmlpath.MustCompile(`/metadata/versioning/latest`)
root, err := xmlpath.Parse(bytes.NewReader(content))
if err != nil {
return "", err
}
if latestVersion, ok := xmlPath.String(root); ok {
utils.Log("info", fmt.Sprintf("Found latest version: %s", latestVersion))
return latestVersion, nil
}
return "", fmt.Errorf("Could not find latest version in %s", metadataURL)
}
func crawle(rank, url string, quest chan string, wg *sync.WaitGroup) {
// XPATH of quest name
// '//table/tr[(position() mod 2) = 1]/td[1]/a[1]/text()'
defer wg.Done()
path := xmlpath.MustCompile("//table/tr/td[1]/span/../a[1]/text()")
resp, err := http.Get(url)
if err != nil {
fmt.Println("Error while fetching", url)
return
}
defer resp.Body.Close()
root, err := xmlpath.ParseHTML(resp.Body)
if err != nil {
fmt.Println("Error while parsing", url)
return
}
iter := path.Iter(root)
for iter.Next() {
quest <- fmt.Sprintf("{\"rank\": \"%s\", \"name\": \"%s\"},", rank, iter.Node().String())
}
}
func lookupMap(m map[string]interface{}) {
for n, v := range m {
myStack.push(n)
switch v2 := v.(type) {
case string:
xpath := xmlpath.MustCompile(myStack.String())
if xpath.Exists(root) {
itr := xpath.Iter(root)
isMatched := false
var srcStr string
for itr.Next() {
node := itr.Node()
srcStr = node.String()
if v2 == srcStr {
s := fmt.Sprintf("%s\t[%s]", myStack, v2)
matched = append(matched, s)
isMatched = true
break
}
}
if !isMatched {
s := fmt.Sprintf("%s, source: [%s], target: [%s]", myStack, srcStr, v2)
unmatched = append(unmatched, s)
}
} else {
s := fmt.Sprintf("%s\t[%s]", myStack, v2)
skipped = append(skipped, s)
}
case map[string]interface{}:
lookupMap(v2)
case []interface{}:
lookupSlice(v2)
default:
panic(reflect.TypeOf(v))
}
myStack.pop()
}
}
func read_xml() *xmlpath.Node {
path, err := exec.LookPath("passenger-status")
if err != nil {
// passenger-status not found in path
if _, err := os.Stat("/usr/local/rvm/wrappers/default/passenger-status"); err == nil {
// default rvm wrapper exists so use that!
path = "/usr/local/rvm/wrappers/default/passenger-status"
}
}
cmd := exec.Command(path, "--show=xml")
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
if err := cmd.Start(); err != nil {
log.Fatal(err)
}
// Stuff to handle the iso-8859-1 xml encoding
// http://stackoverflow.com/a/32224438/606167
decoder := xml.NewDecoder(stdout)
decoder.CharsetReader = charset.NewReaderLabel
xmlData, err := xmlpath.ParseDecoder(decoder)
if err != nil {
log.Fatal(err)
}
// Check version
version_path := xmlpath.MustCompile("/info/@version")
if version, ok := version_path.String(xmlData); !ok || (version != "3" && version != "2") {
log.Fatal("Unsupported Passenger version (xml version ", version, ")")
}
return xmlData
}
func parsePathProperty(body *bytes.Buffer) (value string) {
root, _ := xmlpath.Parse(bytes.NewBuffer(body.Bytes()))
path := xmlpath.MustCompile("//RetrievePropertiesResponse//val")
iter := path.Iter(root)
values := make([]string, 0)
// iter.Next() // skip top element "Datacenters"
for {
// iter.Next() // skip id element
ok := iter.Next()
if ok == false {
break
} else {
newVal := []string{iter.Node().String()}
// add new value to the beginning of the slice
// TODO: get rid of ids
// current end value: Datacenters/group-d1/Tukwila/datacenter-2/vm/group-v3/1-templates/group-v53287/Lower/group-v54541/my_new_template_that_packer_built
values = append(newVal, values...)
}
}
value = strings.Join(values, "/")
return value
}
// http://www.jma.go.jp/jp/amedas_h/today-46211.html
// http://www6.kaiho.mlit.go.jp/03kanku/shimoda/ shift-jis
// http://www6.kaiho.mlit.go.jp/03kanku/yokosuka/kisyou.html
func getWindData(c appengine.Context) (*WindData, error) {
windData := &WindData{}
client := urlfetch.Client(c)
resp, err := client.Get(MICS_URL)
if err != nil {
return nil, fmt.Errorf("could not get %s: %v", MICS_URL, err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("server responded non-200: %s, %s", MICS_URL, resp.Status)
}
defer resp.Body.Close()
// http://stackoverflow.com/questions/24101721/parse-broken-html-with-golang
buf := new(bytes.Buffer)
buf.ReadFrom(resp.Body)
content := mahonia.NewDecoder("cp932").ConvertString(buf.String())
doc, err := xhtml.Parse(strings.NewReader(content))
// https://godoc.org/golang.org/x/net/html
if err != nil {
return nil, fmt.Errorf("could not parse HTML for %s: %v", MICS_URL, err)
}
var b bytes.Buffer
xhtml.Render(&b, doc)
fixed := strings.NewReader(b.String())
root, err := xmlpath.ParseHTML(fixed)
if err != nil {
return nil, fmt.Errorf("could not parse HTML: %s\n Error: %v", content, err)
}
path := xmlpath.MustCompile(MICS_TABLE_XPATH)
table, ok := path.String(root)
if !ok {
return nil, fmt.Errorf("could not find table path")
}
re := regexp.MustCompile("([^\n])\n")
windData.Table = re.ReplaceAllString(table, "$1 ")
path = xmlpath.MustCompile(MICS_DATE_XPATH)
date, ok := path.String(root)
if !ok {
return nil, fmt.Errorf("could not find date")
}
windData.Date = date
imgResp, err := client.Get(MICS_SHIMODA_IMG_URL)
if err != nil {
return nil, fmt.Errorf("unable to get img from %s: %v", MICS_SHIMODA_IMG_URL, err)
}
if imgResp.StatusCode != 200 {
return nil, fmt.Errorf("img server responded non-200: %s, %s", MICS_SHIMODA_IMG_URL, imgResp.Status)
}
defer imgResp.Body.Close()
// XXX need to resize the image for Gratina2
// JPG is more available: http://media.kddi.com/app/publish/torisetsu/pdf/gratina2_torisetsu_shousai.pdf
// go image packages
// image/gif, image/jpeg: http://golang.org/pkg/image/gif/#Encode
pngImg, err := png.Decode(imgResp.Body)
if err != nil {
// we can do with only text info
c.Infof("No image attached. Could not decode png: %v", err)
return windData, nil
}
buf.Reset()
err = jpeg.Encode(buf, pngImg, &jpeg.Options{Quality: 75})
if err != nil {
// we can do with text info only
c.Infof("No image attached. Could not encode to jpeg: %v", err)
return windData, nil
}
windData.Img = buf.Bytes()
return windData, nil
}
func (n *XMLNode) XPathNode(xpath string) (string, bool) {
path := xmlpath.MustCompile(xpath)
return path.String(n.node)
}
