func main() {
m, err := mxj.NewMapXmlReader(os.Stdin)
if err != nil {
util.Fatal(err)
}
if err := m.JsonIndentWriter(os.Stdout, "", " "); err != nil {
util.Fatal(err)
}
if _, err := os.Stdout.WriteString("\n"); err != nil {
util.Fatal(err)
}
}
func SendRequest(url, msg string, params map[string]string) ([]mxj.Map, error) {
yfrequest, ok := yfRequests[msg]
if !ok {
return nil, fmt.Errorf("Could not find request for %s", msg)
}
local := yfrequest.Request
// Check that all vars are filled
for _, name := range yfrequest.Params {
if value, ok := params[name]; ok {
local = strings.Replace(local, name, value, -1)
} else {
return nil, fmt.Errorf("Could not find value for param %s", name)
}
}
req, err := http.NewRequest("POST", url, strings.NewReader(local))
if err != nil {
return nil, err
}
req.Header.Add("Content-Type", "text/xml;charset=UTF-8")
req.Header.Add("SOAPAction", `""`)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
bdy, _ := ioutil.ReadAll(resp.Body)
fmt.Println(string(bdy))
resp.Body.Close()
return nil, fmt.Errorf("Yellowfin Error: HTTP Code %v", resp.StatusCode)
}
m, err := mxj.NewMapXmlReader(resp.Body)
resp.Body.Close()
if err != nil {
return nil, err
}
response := make([]mxj.Map, 0)
err = parseResponse(&m, &response, yfrequest.Call, yfrequest.Resource)
if err != nil {
return nil, err
}
return response, nil
}
func ExampleMap_ValuesForKeyPath() {
// a snippet from examples/gonuts1.go
// How to compensate for irregular tag labels in data.
// Need to extract from an XML stream the values for "netid" and "idnet".
// Solution: use a wildcard path "data.*" to anonymize the "netid" and "idnet" tags.
var msg1 = []byte(`
<?xml version="1.0" encoding="UTF-8"?>
<data>
<netid>
<disable>no</disable>
<text1>default:text</text1>
<word1>default:word</word1>
</netid>
</data>
`)
var msg2 = []byte(`
<?xml version="1.0" encoding="UTF-8"?>
<data>
<idnet>
<disable>yes</disable>
<text1>default:text</text1>
<word1>default:word</word1>
</idnet>
</data>
`)
// let's create a message stream
buf := new(bytes.Buffer)
// load a couple of messages into it
_, _ = buf.Write(msg1)
_, _ = buf.Write(msg2)
n := 0
for {
n++
// Read the stream as Map values - quit on io.EOF.
// Get the raw XML as well as the Map value.
m, merr := mxj.NewMapXmlReader(buf)
if merr != nil && merr != io.EOF {
// handle error - for demo we just print it and continue
fmt.Printf("msg: %d - merr: %s\n", n, merr.Error())
continue
} else if merr == io.EOF {
break
}
// get the values for "netid" or "idnet" key using path == "data.*"
values, _ := m.ValuesForPath("data.*")
fmt.Println("\nmsg:", n, "> path == data.* - got array of values, len:", len(values))
for i, val := range values {
fmt.Println("ValuesForPath result array member -", i, ":", val)
fmt.Println(" k:v pairs for array member:", i)
for key, val := range val.(map[string]interface{}) {
// You'd probably want to process the value, as appropriate.
// Here we just print it out.
fmt.Println("\t\t", key, ":", val)
}
}
}
// Output:
// msg: 1 > path == data.* - got array of values, len: 1
// ValuesForPath result array member - 0 : map[disable:no text1:default:text word1:default:word]
// k:v pairs for array member: 0
// disable : no
// text1 : default:text
// word1 : default:word
//
// msg: 2 > path == data.* - got array of values, len: 1
// ValuesForPath result array member - 0 : map[disable:yes text1:default:text word1:default:word]
// k:v pairs for array member: 0
// disable : yes
// text1 : default:text
// word1 : default:word
}
func main() {
err := d.Try(func() {
spec.RegisterDatabaseFlags(flag.CommandLine)
profile.RegisterProfileFlags(flag.CommandLine)
flag.Usage = customUsage
flag.Parse(true)
if flag.NArg() != 2 {
d.CheckError(errors.New("Expected directory path followed by dataset"))
}
dir := flag.Arg(0)
ds, err := spec.GetDataset(flag.Arg(1))
d.CheckError(err)
defer profile.MaybeStartProfile().Stop()
cpuCount := runtime.NumCPU()
filesChan := make(chan fileIndex, 1024)
refsChan := make(chan refIndex, 1024)
getFilePaths := func() {
index := 0
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
d.PanicIfTrue(err != nil, "Cannot traverse directories")
if !info.IsDir() && filepath.Ext(path) == ".xml" {
filesChan <- fileIndex{path, index}
index++
}
return nil
})
d.PanicIfError(err)
close(filesChan)
}
wg := sync.WaitGroup{}
importXML := func() {
expectedType := types.NewMap()
for f := range filesChan {
file, err := os.Open(f.path)
d.PanicIfTrue(err != nil, "Error getting XML")
xmlObject, err := mxj.NewMapXmlReader(file)
d.PanicIfTrue(err != nil, "Error decoding XML")
object := xmlObject.Old()
file.Close()
nomsObj := jsontonoms.NomsValueFromDecodedJSON(object, false)
d.Chk.IsType(expectedType, nomsObj)
var r types.Ref
if !*noIO {
r = ds.Database().WriteValue(nomsObj)
}
refsChan <- refIndex{r, f.index}
}
wg.Done()
}
go getFilePaths()
for i := 0; i < cpuCount*8; i++ {
wg.Add(1)
go importXML()
}
go func() {
wg.Wait()
close(refsChan) // done converting xml to noms
}()
refList := refIndexList{}
for r := range refsChan {
refList = append(refList, r)
}
sort.Sort(refList)
refs := make([]types.Value, len(refList))
for idx, r := range refList {
refs[idx] = r.ref
}
rl := types.NewList(refs...)
if !*noIO {
_, err := ds.CommitValue(rl)
d.PanicIfError(err)
}
})
if err != nil {
log.Fatal(err)
}
}
func main() {
var file string
flag.StringVar(&file, "file", "", "file to process")
flag.Parse()
fh, fherr := os.Open(file)
if fherr != nil {
fmt.Println("fherr:", fherr.Error())
return
}
defer fh.Close()
fmt.Println(time.Now().String(), "... File Opened:", file)
/*
// Get the XML data set from the file.
fs, _ := fh.Stat()
xmldata := make([]byte, fs.Size())
n, frerr := fh.Read(xmldata)
if frerr != nil {
fmt.Println("frerr:", frerr.Error())
return
}
if int64(n) != fs.Size() {
fmt.Println("n:", n, "fs.Size():", fs.Size())
return
}
fmt.Println(time.Now().String(), "... File Read - size:", fs.Size())
// load XML into a Map value
m, merr := mxj.NewMapXml(xmldata)
*/
// Consume the file using os.File Reader.
// Note: there is a single record with root tag of "Metrics".
m, merr := mxj.NewMapXmlReader(fh)
if merr != nil {
log.Fatal("merr:", merr.Error())
}
fmt.Println(time.Now().String(), "... XML Unmarshaled - len:", len(m))
// Get just the key values of interest.
// Could also use m.ValuesForKey("Metric"),
// since there's just the one path.
metricVals, err := m.ValuesForPath("Metrics.Metric")
if err != nil {
log.Fatal("err:", err.Error())
}
fmt.Println(time.Now().String(), "... ValuesFromKeyPath - len:", len(metricVals))
// now just manipulate Map entries returned as []interface{} array.
for _, v := range metricVals {
aMetricVal := v.(map[string]interface{})
// create file to hold csv data sets
id := aMetricVal["-id"].(string)
desc := aMetricVal["-description"].(string)
mf, mferr := os.OpenFile(id+".csv", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666)
if mferr != nil {
fmt.Println("mferr:", mferr.Error())
return
}
fmt.Print(time.Now().String(), " id: ", id, " desc: ", desc)
mf.WriteString(id + "," + desc + "\n")
// rescan looking for keys with data: Values or Value
for key, val := range aMetricVal {
switch key {
case "Values":
// extract the list of "Value" from map
values := val.(map[string]interface{})["Value"].([]interface{})
fmt.Println(" len(Values):", len(values))
// first line in file is the metric label values (keys)
var gotKeys bool
for _, vval := range values {
valueEntry := vval.(map[string]interface{})
// no guarantee that range on map will follow any sequence
lv := len(valueEntry)
type ev [2]string
list := make([]ev, lv)
var i int
for k, v := range valueEntry {
list[i][0] = k
list[i][1] = v.(string)
i++
}
// extract keys as column header on first pass
if !gotKeys {
// print out the keys
var gotFirstKey bool
// for kk, _ := range valueEntry {
for i := 0; i < lv; i++ {
if gotFirstKey {
mf.WriteString(",")
} else {
gotFirstKey = true
}
// strip prepended hyphen
mf.WriteString((list[i][0])[1:])
}
mf.WriteString("\n")
gotKeys = true
}
// print out values
var gotFirstVal bool
// for _, vv := range valueEntry {
for i := 0; i < lv; i++ {
if gotFirstVal {
mf.WriteString(",")
} else {
gotFirstVal = true
}
mf.WriteString(list[i][1])
}
// terminate row of data
mf.WriteString("\n")
}
case "Value":
vv := val.(map[string]interface{})
fmt.Println(" len(Value):", len(vv))
mf.WriteString("value\n" + vv["-value"].(string) + "\n")
}
}
mf.Close()
}
}
func main() {
sourceFilePath := flag.String("source-file-path", "", "source file path")
targetDirPath := flag.String("target-dir-path", "", "target dir path")
verbose := flag.Bool("verbose", false, "verbose")
flag.Parse()
if *sourceFilePath == "" || *targetDirPath == "" {
flag.PrintDefaults()
os.Exit(2)
}
var err error
report, err = os.Create("report.txt")
if err != nil {
panic(err)
}
defer report.Close()
f, err := os.Open(*sourceFilePath)
if err != nil {
panic(err)
}
defer f.Close()
root, err = xmlpath.Parse(f)
if err != nil {
panic(err)
}
filepath.Walk(*targetDirPath, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
if filepath.Base(*sourceFilePath) == filepath.Base(path) {
fmt.Printf("CHECKING... %s\n", path)
fmt.Fprintf(report, "\n\nSOURCE : %s\n", *sourceFilePath)
fmt.Fprintf(report, "TARGET : %s\n", path)
f, err := os.Open(path)
if err != nil {
fmt.Println(err)
fmt.Fprintln(report, err)
return nil
}
defer f.Close()
m, err := mxj.NewMapXmlReader(f)
if err != nil {
fmt.Println(err)
fmt.Fprintln(report, err)
return nil
}
matched = matched[:0]
skipped = skipped[:0]
unmatched = unmatched[:0]
lookupMap(m)
if len(unmatched) != 0 {
fmt.Fprintf(report, "\n...UNMATCHED...\n")
for _, s := range unmatched {
fmt.Fprintf(report, "%s\n", s)
}
}
if len(skipped) != 0 {
fmt.Fprintf(report, "\n...skipped...\n")
for _, s := range skipped {
fmt.Fprintf(report, "%s\n", s)
}
}
if *verbose && len(matched) != 0 {
fmt.Fprintf(report, "\n...matched...\n")
for _, s := range matched {
fmt.Fprintf(report, "%s\n", s)
}
}
}
return nil
})
fmt.Println("DONE... report.txt")
}
func main() {
err := d.Try(func() {
dsFlags := dataset.NewFlags()
flag.Usage = customUsage
flag.Parse()
ds := dsFlags.CreateDataset()
dir := flag.Arg(0)
if ds == nil || dir == "" {
flag.Usage()
return
}
defer ds.Store().Close()
if util.MaybeStartCPUProfile() {
defer util.StopCPUProfile()
}
cpuCount := runtime.NumCPU()
runtime.GOMAXPROCS(cpuCount)
filesChan := make(chan fileIndex, 1024)
refsChan := make(chan refIndex, 1024)
getFilePaths := func() {
index := 0
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
d.Exp.NoError(err, "Cannot traverse directories")
if !info.IsDir() && filepath.Ext(path) == ".xml" {
filesChan <- fileIndex{path, index}
index++
}
return nil
})
d.Exp.NoError(err)
close(filesChan)
}
wg := sync.WaitGroup{}
importXml := func() {
expectedType := util.NewMapOfStringToValue()
for f := range filesChan {
file, err := os.Open(f.path)
d.Exp.NoError(err, "Error getting XML")
xmlObject, err := mxj.NewMapXmlReader(file)
d.Exp.NoError(err, "Error decoding XML")
object := xmlObject.Old()
file.Close()
nomsObj := util.NomsValueFromDecodedJSON(object)
d.Chk.IsType(expectedType, nomsObj)
r := ref.Ref{}
if !*noIO {
r = ds.Store().WriteValue(nomsObj).TargetRef()
}
refsChan <- refIndex{r, f.index}
}
wg.Done()
}
go getFilePaths()
for i := 0; i < cpuCount*8; i++ {
wg.Add(1)
go importXml()
}
go func() {
wg.Wait()
close(refsChan) // done converting xml to noms
}()
refList := refIndexList{}
for r := range refsChan {
refList = append(refList, r)
}
sort.Sort(refList)
refs := make(util.ListOfRefOfMapOfStringToValueDef, len(refList))
for idx, r := range refList {
refs[idx] = r.ref
}
if !*noIO {
_, err := ds.Commit(refs.New())
d.Exp.NoError(err)
}
util.MaybeWriteMemProfile()
})
if err != nil {
log.Fatal(err)
}
}