func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flags := datas.NewFlags()
flag.Parse()
ds, ok := flags.CreateDataStore()
if !ok || *inputID == "" || *outputID == "" {
flag.Usage()
return
}
defer ds.Close()
err := d.Try(func() {
if util.MaybeStartCPUProfile() {
defer util.StopCPUProfile()
}
inputDataset := dataset.NewDataset(ds, *inputID)
outputDataset := dataset.NewDataset(ds, *outputID)
input := inputDataset.Head().Value().(ListOfRefOfMapOfStringToValue)
output := getIndex(input, ds)
_, err := outputDataset.Commit(output)
d.Exp.NoError(err)
util.MaybeWriteMemProfile()
})
if err != nil {
log.Fatal(err)
}
}
func main() {
cpuCount := runtime.NumCPU()
runtime.GOMAXPROCS(cpuCount)
flag.Parse()
sourceStore, ok := sourceStoreFlags.CreateDataStore()
sink := sinkDsFlags.CreateDataset()
if !ok || sink == nil || *p == 0 || *sourceObject == "" {
flag.Usage()
return
}
defer sourceStore.Close()
defer sink.Store().Close()
err := d.Try(func() {
if util.MaybeStartCPUProfile() {
defer util.StopCPUProfile()
}
sourceRef := ref.Ref{}
if r, ok := ref.MaybeParse(*sourceObject); ok {
if sourceStore.Has(r) {
sourceRef = r
}
} else {
if c, ok := sourceStore.MaybeHead(*sourceObject); ok {
sourceRef = c.Ref()
}
}
d.Exp.False(sourceRef.IsEmpty(), "Unknown source object: %s", *sourceObject)
var err error
*sink, err = sink.Pull(sourceStore, sourceRef, int(*p))
util.MaybeWriteMemProfile()
d.Exp.NoError(err)
})
if err != nil {
log.Fatal(err)
}
}
func main() {
err := d.Try(func() {
dsFlags := dataset.NewFlags()
flag.Usage = customUsage
flag.Parse()
ds := dsFlags.CreateDataset()
dir := flag.Arg(0)
if ds == nil || dir == "" {
flag.Usage()
return
}
defer ds.Store().Close()
if util.MaybeStartCPUProfile() {
defer util.StopCPUProfile()
}
cpuCount := runtime.NumCPU()
runtime.GOMAXPROCS(cpuCount)
filesChan := make(chan fileIndex, 1024)
refsChan := make(chan refIndex, 1024)
getFilePaths := func() {
index := 0
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
d.Exp.NoError(err, "Cannot traverse directories")
if !info.IsDir() && filepath.Ext(path) == ".xml" {
filesChan <- fileIndex{path, index}
index++
}
return nil
})
d.Exp.NoError(err)
close(filesChan)
}
wg := sync.WaitGroup{}
importXml := func() {
expectedType := util.NewMapOfStringToValue()
for f := range filesChan {
file, err := os.Open(f.path)
d.Exp.NoError(err, "Error getting XML")
xmlObject, err := mxj.NewMapXmlReader(file)
d.Exp.NoError(err, "Error decoding XML")
object := xmlObject.Old()
file.Close()
nomsObj := util.NomsValueFromDecodedJSON(object)
d.Chk.IsType(expectedType, nomsObj)
r := ref.Ref{}
if !*noIO {
r = ds.Store().WriteValue(nomsObj).TargetRef()
}
refsChan <- refIndex{r, f.index}
}
wg.Done()
}
go getFilePaths()
for i := 0; i < cpuCount*8; i++ {
wg.Add(1)
go importXml()
}
go func() {
wg.Wait()
close(refsChan) // done converting xml to noms
}()
refList := refIndexList{}
for r := range refsChan {
refList = append(refList, r)
}
sort.Sort(refList)
refs := make(util.ListOfRefOfMapOfStringToValueDef, len(refList))
for idx, r := range refList {
refs[idx] = r.ref
}
if !*noIO {
_, err := ds.Commit(refs.New())
d.Exp.NoError(err)
}
util.MaybeWriteMemProfile()
})
if err != nil {
log.Fatal(err)
}
}