//fetch input data
func moveInputData(work *core.Workunit) (size int64, err error) {
for _, io := range work.Inputs {
inputname := io.FileName
dataUrl, uerr := io.DataUrl()
if uerr != nil {
return 0, uerr
}
if work.Rank > 0 {
dataUrl = fmt.Sprintf("%s&index=%s&part=%s", dataUrl, work.IndexType(), work.Part())
}
inputFilePath := path.Join(work.Path(), inputname)
logger.Debug(2, "mover: fetching input from url:"+dataUrl)
logger.Event(event.FILE_IN, "workid="+work.Id+" url="+dataUrl)
// this gets file from any downloadable url, not just shock
if datamoved, _, err := shock.FetchFile(inputFilePath, dataUrl, work.Info.DataToken, io.Uncompress, false); err != nil {
return size, err
} else {
size += datamoved
}
logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl)
}
return
}
//fetch input data
func MoveInputData(work *core.Workunit) (size int64, err error) {
for _, io := range work.Inputs {
inputname := io.FileName
// skip if NoFile == true
if !io.NoFile { // is file !
dataUrl, uerr := io.DataUrl()
if uerr != nil {
return 0, uerr
}
inputFilePath := fmt.Sprintf("%s/%s", work.Path(), inputname)
if work.Rank == 0 {
if conf.CACHE_ENABLED && io.Node != "" {
if file_path, err := StatCacheFilePath(io.Node); err == nil {
//make a link in work dir from cached file
linkname := fmt.Sprintf("%s/%s", work.Path(), inputname)
fmt.Printf("input found in cache, making link: " + file_path + " -> " + linkname + "\n")
err = os.Symlink(file_path, linkname)
if err == nil {
logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl)
}
return 0, err
}
}
} else {
dataUrl = fmt.Sprintf("%s&index=%s&part=%s", dataUrl, work.IndexType(), work.Part())
}
logger.Debug(2, "mover: fetching input file from url:"+dataUrl)
logger.Event(event.FILE_IN, "workid="+work.Id+";url="+dataUrl)
// download file
if datamoved, _, err := shock.FetchFile(inputFilePath, dataUrl, work.Info.DataToken, io.Uncompress, false); err != nil {
return size, errors.New("shock.FetchFile returned: " + err.Error())
} else {
size += datamoved
}
logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl)
}
// download node attributes if requested
if io.AttrFile != "" {
// get node
node, err := shock.ShockGet(io.Host, io.Node, work.Info.DataToken)
if err != nil {
//return size, err
return size, errors.New("shock.ShockGet (node attributes) returned: " + err.Error())
}
logger.Debug(2, "mover: fetching input attributes from node:"+node.Id)
logger.Event(event.ATTR_IN, "workid="+work.Id+";node="+node.Id)
// print node attributes
attrFilePath := fmt.Sprintf("%s/%s", work.Path(), io.AttrFile)
attr_json, _ := json.Marshal(node.Attributes)
if err := ioutil.WriteFile(attrFilePath, attr_json, 0644); err != nil {
return size, err
}
logger.Event(event.ATTR_READY, "workid="+work.Id+";path="+attrFilePath)
}
}
return
}
//parse workunit, fetch input data, compose command arguments
func ParseWorkunitArgs(work *core.Workunit) (args []string, err error) {
argstr := work.Cmd.Args
if argstr == "" {
return
}
argList := strings.Fields(argstr)
inputsMap := work.Inputs
for _, arg := range argList {
if strings.Contains(arg, "@") { //parse input/output to accessible local file
segs := strings.Split(arg, "@")
if len(segs) > 2 {
return []string{}, errors.New("invalid format in command args, multiple @ within one arg")
}
inputname := segs[1]
if inputsMap.Has(inputname) {
io := inputsMap[inputname]
var dataUrl string
if work.Rank == 0 {
dataUrl = io.DataUrl()
} else {
dataUrl = fmt.Sprintf("%s&index=%s&part=%s", io.DataUrl(), work.IndexType(), work.Part())
}
inputFilePath := fmt.Sprintf("%s/%s", work.Path(), inputname)
logger.Debug(2, "mover: fetching input from url:"+dataUrl)
logger.Event(event.FILE_IN, "workid="+work.Id+" url="+dataUrl)
if err := fetchFile(inputFilePath, dataUrl, work.Info.DataToken); err != nil { //get file from Shock
return []string{}, err
}
logger.Event(event.FILE_READY, "workid="+work.Id+" url="+dataUrl)
parsedArg := fmt.Sprintf("%s%s", segs[0], inputFilePath)
args = append(args, parsedArg)
}
} else { //no @, has nothing to do with input/output, append directly
args = append(args, arg)
}
}
return args, nil
}
//fetch input data
func moveInputData(work *core.Workunit) (size int64, err error) {
for inputname, io := range work.Inputs {
var dataUrl string
if work.Rank == 0 {
dataUrl = io.DataUrl()
} else {
dataUrl = fmt.Sprintf("%s&index=%s&part=%s", io.DataUrl(), work.IndexType(), work.Part())
}
inputFilePath := path.Join(work.Path(), inputname)
logger.Debug(2, "mover: fetching input from url:"+dataUrl)
logger.Event(event.FILE_IN, "workid="+work.Id+" url="+dataUrl)
if datamoved, err := shock.FetchFile(inputFilePath, dataUrl, work.Info.DataToken, io.Uncompress); err != nil {
return size, err
} else {
size += datamoved
}
logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl)
}
return
}