Example #1
0
func main() {
	m, err := mxj.NewMapXmlReader(os.Stdin)
	if err != nil {
		util.Fatal(err)
	}
	if err := m.JsonIndentWriter(os.Stdout, "", "  "); err != nil {
		util.Fatal(err)
	}
	if _, err := os.Stdout.WriteString("\n"); err != nil {
		util.Fatal(err)
	}
}
Example #2
0
File: yfws.go Project: IMQS/yfws
func SendRequest(url, msg string, params map[string]string) ([]mxj.Map, error) {

	yfrequest, ok := yfRequests[msg]
	if !ok {
		return nil, fmt.Errorf("Could not find request for %s", msg)
	}

	local := yfrequest.Request

	// Check that all vars are filled
	for _, name := range yfrequest.Params {
		if value, ok := params[name]; ok {
			local = strings.Replace(local, name, value, -1)
		} else {
			return nil, fmt.Errorf("Could not find value for param %s", name)
		}
	}
	req, err := http.NewRequest("POST", url, strings.NewReader(local))
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "text/xml;charset=UTF-8")
	req.Header.Add("SOAPAction", `""`)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		bdy, _ := ioutil.ReadAll(resp.Body)
		fmt.Println(string(bdy))
		resp.Body.Close()
		return nil, fmt.Errorf("Yellowfin Error: HTTP Code %v", resp.StatusCode)
	}
	m, err := mxj.NewMapXmlReader(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}

	response := make([]mxj.Map, 0)

	err = parseResponse(&m, &response, yfrequest.Call, yfrequest.Resource)
	if err != nil {
		return nil, err
	}
	return response, nil

}
func ExampleMap_ValuesForKeyPath() {
	// a snippet from examples/gonuts1.go
	// How to compensate for irregular tag labels in data.
	// Need to extract from an XML stream the values for "netid" and "idnet".
	// Solution: use a wildcard path "data.*" to anonymize the "netid" and "idnet" tags.

	var msg1 = []byte(`
<?xml version="1.0" encoding="UTF-8"?>
<data>
    <netid>
        <disable>no</disable>
        <text1>default:text</text1>
        <word1>default:word</word1>
    </netid>
</data>
`)

	var msg2 = []byte(`
<?xml version="1.0" encoding="UTF-8"?>
<data>
    <idnet>
        <disable>yes</disable>
        <text1>default:text</text1>
        <word1>default:word</word1>
    </idnet>
</data>
`)

	// let's create a message stream
	buf := new(bytes.Buffer)
	// load a couple of messages into it
	_, _ = buf.Write(msg1)
	_, _ = buf.Write(msg2)

	n := 0
	for {
		n++
		// Read the stream as Map values - quit on io.EOF.
		// Get the raw XML as well as the Map value.
		m, merr := mxj.NewMapXmlReader(buf)
		if merr != nil && merr != io.EOF {
			// handle error - for demo we just print it and continue
			fmt.Printf("msg: %d - merr: %s\n", n, merr.Error())
			continue
		} else if merr == io.EOF {
			break
		}

		// get the values for "netid" or "idnet" key using path == "data.*"
		values, _ := m.ValuesForPath("data.*")
		fmt.Println("\nmsg:", n, "> path == data.* - got array of values, len:", len(values))
		for i, val := range values {
			fmt.Println("ValuesForPath result array member -", i, ":", val)
			fmt.Println("              k:v pairs for array member:", i)
			for key, val := range val.(map[string]interface{}) {
				// You'd probably want to process the value, as appropriate.
				// Here we just print it out.
				fmt.Println("\t\t", key, ":", val)
			}
		}
	}
	// Output:
	// msg: 1 > path == data.* - got array of values, len: 1
	// ValuesForPath result array member - 0 : map[disable:no text1:default:text word1:default:word]
	//               k:v pairs for array member: 0
	// 		 disable : no
	// 		 text1 : default:text
	// 		 word1 : default:word
	//
	// msg: 2 > path == data.* - got array of values, len: 1
	// ValuesForPath result array member - 0 : map[disable:yes text1:default:text word1:default:word]
	//               k:v pairs for array member: 0
	// 		 disable : yes
	// 		 text1 : default:text
	// 		 word1 : default:word
}
Example #4
0
func main() {
	err := d.Try(func() {
		spec.RegisterDatabaseFlags(flag.CommandLine)
		profile.RegisterProfileFlags(flag.CommandLine)
		flag.Usage = customUsage
		flag.Parse(true)

		if flag.NArg() != 2 {
			d.CheckError(errors.New("Expected directory path followed by dataset"))
		}
		dir := flag.Arg(0)
		ds, err := spec.GetDataset(flag.Arg(1))
		d.CheckError(err)

		defer profile.MaybeStartProfile().Stop()

		cpuCount := runtime.NumCPU()

		filesChan := make(chan fileIndex, 1024)
		refsChan := make(chan refIndex, 1024)

		getFilePaths := func() {
			index := 0
			err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
				d.PanicIfTrue(err != nil, "Cannot traverse directories")
				if !info.IsDir() && filepath.Ext(path) == ".xml" {
					filesChan <- fileIndex{path, index}
					index++
				}

				return nil
			})
			d.PanicIfError(err)
			close(filesChan)
		}

		wg := sync.WaitGroup{}
		importXML := func() {
			expectedType := types.NewMap()
			for f := range filesChan {
				file, err := os.Open(f.path)
				d.PanicIfTrue(err != nil, "Error getting XML")

				xmlObject, err := mxj.NewMapXmlReader(file)
				d.PanicIfTrue(err != nil, "Error decoding XML")
				object := xmlObject.Old()
				file.Close()

				nomsObj := jsontonoms.NomsValueFromDecodedJSON(object, false)
				d.Chk.IsType(expectedType, nomsObj)

				var r types.Ref
				if !*noIO {
					r = ds.Database().WriteValue(nomsObj)
				}

				refsChan <- refIndex{r, f.index}
			}

			wg.Done()
		}

		go getFilePaths()
		for i := 0; i < cpuCount*8; i++ {
			wg.Add(1)
			go importXML()
		}
		go func() {
			wg.Wait()
			close(refsChan) // done converting xml to noms
		}()

		refList := refIndexList{}
		for r := range refsChan {
			refList = append(refList, r)
		}
		sort.Sort(refList)

		refs := make([]types.Value, len(refList))
		for idx, r := range refList {
			refs[idx] = r.ref
		}

		rl := types.NewList(refs...)

		if !*noIO {
			_, err := ds.CommitValue(rl)
			d.PanicIfError(err)
		}
	})

	if err != nil {
		log.Fatal(err)
	}
}
func main() {
	var file string
	flag.StringVar(&file, "file", "", "file to process")
	flag.Parse()

	fh, fherr := os.Open(file)
	if fherr != nil {
		fmt.Println("fherr:", fherr.Error())
		return
	}
	defer fh.Close()
	fmt.Println(time.Now().String(), "... File Opened:", file)

	/*
		// Get the XML data set from the file.
		fs, _ := fh.Stat()
		xmldata := make([]byte, fs.Size())
		n, frerr := fh.Read(xmldata)
		if frerr != nil {
			fmt.Println("frerr:", frerr.Error())
			return
		}
		if int64(n) != fs.Size() {
			fmt.Println("n:", n, "fs.Size():", fs.Size())
			return
		}
		fmt.Println(time.Now().String(), "... File Read - size:", fs.Size())

		// load XML into a Map value
		m, merr := mxj.NewMapXml(xmldata)
	*/
	// Consume the file using os.File Reader.
	// Note: there is a single record with root tag of "Metrics".
	m, merr := mxj.NewMapXmlReader(fh)
	if merr != nil {
		log.Fatal("merr:", merr.Error())
	}
	fmt.Println(time.Now().String(), "... XML Unmarshaled - len:", len(m))

	// Get just the key values of interest.
	// Could also use m.ValuesForKey("Metric"),
	// since there's just the one path.
	metricVals, err := m.ValuesForPath("Metrics.Metric")
	if err != nil {
		log.Fatal("err:", err.Error())
	}
	fmt.Println(time.Now().String(), "... ValuesFromKeyPath - len:", len(metricVals))

	// now just manipulate Map entries returned as []interface{} array.
	for _, v := range metricVals {
		aMetricVal := v.(map[string]interface{})

		// create file to hold csv data sets
		id := aMetricVal["-id"].(string)
		desc := aMetricVal["-description"].(string)
		mf, mferr := os.OpenFile(id+".csv", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666)
		if mferr != nil {
			fmt.Println("mferr:", mferr.Error())
			return
		}

		fmt.Print(time.Now().String(), " id: ", id, " desc: ", desc)
		mf.WriteString(id + "," + desc + "\n")

		// rescan looking for keys with data: Values or Value
		for key, val := range aMetricVal {
			switch key {
			case "Values":
				// extract the list of "Value" from map
				values := val.(map[string]interface{})["Value"].([]interface{})
				fmt.Println(" len(Values):", len(values))

				// first line in file is the metric label values (keys)
				var gotKeys bool
				for _, vval := range values {
					valueEntry := vval.(map[string]interface{})

					// no guarantee that range on map will follow any sequence
					lv := len(valueEntry)
					type ev [2]string
					list := make([]ev, lv)
					var i int
					for k, v := range valueEntry {
						list[i][0] = k
						list[i][1] = v.(string)
						i++
					}

					// extract keys as column header on first pass
					if !gotKeys {
						// print out the keys
						var gotFirstKey bool
						// for kk, _ := range valueEntry {
						for i := 0; i < lv; i++ {
							if gotFirstKey {
								mf.WriteString(",")
							} else {
								gotFirstKey = true
							}
							// strip prepended hyphen
							mf.WriteString((list[i][0])[1:])
						}
						mf.WriteString("\n")
						gotKeys = true
					}

					// print out values
					var gotFirstVal bool
					// for _, vv := range valueEntry {
					for i := 0; i < lv; i++ {
						if gotFirstVal {
							mf.WriteString(",")
						} else {
							gotFirstVal = true
						}
						mf.WriteString(list[i][1])
					}

					// terminate row of data
					mf.WriteString("\n")
				}
			case "Value":
				vv := val.(map[string]interface{})
				fmt.Println(" len(Value):", len(vv))
				mf.WriteString("value\n" + vv["-value"].(string) + "\n")
			}
		}
		mf.Close()
	}
}
func main() {
	sourceFilePath := flag.String("source-file-path", "", "source file path")
	targetDirPath := flag.String("target-dir-path", "", "target dir path")
	verbose := flag.Bool("verbose", false, "verbose")

	flag.Parse()

	if *sourceFilePath == "" || *targetDirPath == "" {
		flag.PrintDefaults()
		os.Exit(2)
	}

	var err error
	report, err = os.Create("report.txt")
	if err != nil {
		panic(err)
	}
	defer report.Close()

	f, err := os.Open(*sourceFilePath)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	root, err = xmlpath.Parse(f)
	if err != nil {
		panic(err)
	}

	filepath.Walk(*targetDirPath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		if filepath.Base(*sourceFilePath) == filepath.Base(path) {
			fmt.Printf("CHECKING... %s\n", path)
			fmt.Fprintf(report, "\n\nSOURCE : %s\n", *sourceFilePath)
			fmt.Fprintf(report, "TARGET : %s\n", path)

			f, err := os.Open(path)
			if err != nil {
				fmt.Println(err)
				fmt.Fprintln(report, err)
				return nil
			}
			defer f.Close()
			m, err := mxj.NewMapXmlReader(f)
			if err != nil {
				fmt.Println(err)
				fmt.Fprintln(report, err)
				return nil
			}

			matched = matched[:0]
			skipped = skipped[:0]
			unmatched = unmatched[:0]
			lookupMap(m)

			if len(unmatched) != 0 {
				fmt.Fprintf(report, "\n...UNMATCHED...\n")
				for _, s := range unmatched {
					fmt.Fprintf(report, "%s\n", s)
				}
			}
			if len(skipped) != 0 {
				fmt.Fprintf(report, "\n...skipped...\n")
				for _, s := range skipped {
					fmt.Fprintf(report, "%s\n", s)
				}
			}
			if *verbose && len(matched) != 0 {
				fmt.Fprintf(report, "\n...matched...\n")
				for _, s := range matched {
					fmt.Fprintf(report, "%s\n", s)
				}
			}
		}
		return nil
	})

	fmt.Println("DONE... report.txt")
}
Example #7
0
func main() {
	err := d.Try(func() {
		dsFlags := dataset.NewFlags()
		flag.Usage = customUsage
		flag.Parse()
		ds := dsFlags.CreateDataset()
		dir := flag.Arg(0)
		if ds == nil || dir == "" {
			flag.Usage()
			return
		}
		defer ds.Store().Close()

		if util.MaybeStartCPUProfile() {
			defer util.StopCPUProfile()
		}

		cpuCount := runtime.NumCPU()
		runtime.GOMAXPROCS(cpuCount)

		filesChan := make(chan fileIndex, 1024)
		refsChan := make(chan refIndex, 1024)

		getFilePaths := func() {
			index := 0
			err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
				d.Exp.NoError(err, "Cannot traverse directories")
				if !info.IsDir() && filepath.Ext(path) == ".xml" {
					filesChan <- fileIndex{path, index}
					index++
				}

				return nil
			})
			d.Exp.NoError(err)
			close(filesChan)
		}

		wg := sync.WaitGroup{}
		importXml := func() {
			expectedType := util.NewMapOfStringToValue()
			for f := range filesChan {
				file, err := os.Open(f.path)
				d.Exp.NoError(err, "Error getting XML")

				xmlObject, err := mxj.NewMapXmlReader(file)
				d.Exp.NoError(err, "Error decoding XML")
				object := xmlObject.Old()
				file.Close()

				nomsObj := util.NomsValueFromDecodedJSON(object)
				d.Chk.IsType(expectedType, nomsObj)
				r := ref.Ref{}

				if !*noIO {
					r = ds.Store().WriteValue(nomsObj).TargetRef()
				}

				refsChan <- refIndex{r, f.index}
			}

			wg.Done()
		}

		go getFilePaths()
		for i := 0; i < cpuCount*8; i++ {
			wg.Add(1)
			go importXml()
		}
		go func() {
			wg.Wait()
			close(refsChan) // done converting xml to noms
		}()

		refList := refIndexList{}
		for r := range refsChan {
			refList = append(refList, r)
		}
		sort.Sort(refList)

		refs := make(util.ListOfRefOfMapOfStringToValueDef, len(refList))
		for idx, r := range refList {
			refs[idx] = r.ref
		}

		if !*noIO {
			_, err := ds.Commit(refs.New())
			d.Exp.NoError(err)
		}

		util.MaybeWriteMemProfile()
	})

	if err != nil {
		log.Fatal(err)
	}
}