Пример #1
0
//fetch prerequisite data (e.g. reference dbs)
func movePreData(workunit *core.Workunit) (size int64, err error) {
	for name, io := range workunit.Predata {
		predata_directory := path.Join(conf.DATA_PATH, "predata")
		err = os.MkdirAll(predata_directory, 755)
		if err != nil {
			return 0, errors.New("error creating predata_directory: " + err.Error())
		}

		file_path := path.Join(predata_directory, name)
		if !isFileExisting(file_path) {

			size, err = shock.FetchFile(file_path, io.Url, workunit.Info.DataToken, io.Uncompress)
			if err != nil {
				return 0, errors.New("error in fetchFile:" + err.Error())
			}
		}

		use_symlink := false
		linkname := path.Join(workunit.Path(), name)
		if workunit.Cmd.Dockerimage != "" || strings.HasPrefix(workunit.Cmd.Name, "app:") { // TODO need more save way to detect use of docker

			use_symlink = false // TODO mechanism

			if use_symlink {
				file_path = path.Join(conf.DOCKER_WORKUNIT_PREDATA_DIR, name)
				// some tasks want to write in predata dir, thus need symlink
				logger.Debug(1, "dangling symlink:"+linkname+" -> "+file_path)

				// creation of dangling symlinks is not possible with with os.Symlink, thus use system ln
				link_out, err := exec.Command("ln", "-s", file_path, linkname).CombinedOutput()
				logger.Debug(1, fmt.Sprintf("ln returned %s", link_out))

				if err != nil {
					return 0, errors.New("error creating predata file symlink (dangling version): " + err.Error())
				}
			} else {
				// some programs do not accept symlinks (e.g. emirge), need to copy the file into the work directory
				// linkname refers to target file now.
				logger.Debug(1, "copy predata:"+file_path+" -> "+linkname)

				_, err := shock.CopyFile(file_path, linkname)
				if err != nil {
					return 0, fmt.Errorf("error copying file from %s to % s: ", file_path, linkname, err.Error())
				}
			}
		} else {

			//linkname := path.Join(workunit.Path(), name)
			logger.Debug(1, "symlink:"+linkname+" -> "+file_path)

			err = os.Symlink(file_path, linkname)
			if err != nil {
				return 0, errors.New("error creating predata file symlink: " + err.Error())
			}
		}

	}
	return
}
Пример #2
0
//fetch prerequisite data (e.g. reference dbs)
func movePreData(workunit *core.Workunit) (size int64, err error) {
	for _, io := range workunit.Predata {
		name := io.FileName
		predata_directory := path.Join(conf.DATA_PATH, "predata")
		err = os.MkdirAll(predata_directory, 755)
		if err != nil {
			return 0, errors.New("error creating predata_directory: " + err.Error())
		}

		file_path := path.Join(predata_directory, name)
		dataUrl, uerr := io.DataUrl()
		if uerr != nil {
			return 0, uerr
		}

		// get shock and local md5sums
		isShockPredata := true
		node_md5 := ""
		if io.Node == "-" {
			isShockPredata = false
		} else {
			node, err := shock.ShockGet(io.Host, io.Node, workunit.Info.DataToken)
			if err != nil {
				return 0, errors.New("error in ShockGet: " + err.Error())
			}
			// rename file to be md5sum
			node_md5 = node.File.Checksum["md5"]
			file_path = path.Join(predata_directory, node_md5)
		}

		// file does not exist or its md5sum is wrong
		if !isFileExisting(file_path) {
			logger.Debug(2, "mover: fetching predata from url: "+dataUrl)
			logger.Event(event.PRE_IN, "workid="+workunit.Id+" url="+dataUrl)

			var md5sum string
			file_path_part := file_path + ".part" // temporary name
			// this gets file from any downloadable url, not just shock
			size, md5sum, err = shock.FetchFile(file_path_part, dataUrl, workunit.Info.DataToken, io.Uncompress, isShockPredata)
			if err != nil {
				return 0, errors.New("error in fetchFile: " + err.Error())
			}
			os.Rename(file_path_part, file_path)
			if err != nil {
				return 0, errors.New("error renaming after download of preData: " + err.Error())
			}
			if isShockPredata {
				if node_md5 != md5sum {
					return 0, errors.New("error downloaded file md5 does not mach shock md5, node: " + io.Node)
				} else {
					logger.Debug(2, "mover: predata "+name+" has md5sum "+md5sum)
				}
			}
		} else {
			logger.Debug(2, "mover: predata already exists: "+name)
		}

		// timstamp for last access - future caching
		accessfile, err := os.Create(file_path + ".access")
		if err != nil {
			return 0, errors.New("error creating predata access file: " + err.Error())
		}
		defer accessfile.Close()
		accessfile.WriteString(time.Now().String())

		// determine if running with docker
		wants_docker := false
		if workunit.Cmd.Dockerimage != "" || workunit.App != nil { // TODO need more save way to detect use of docker
			wants_docker = true
		}
		if wants_docker && conf.USE_DOCKER == "no" {
			return 0, errors.New("error: use of docker images has been disabled by administrator")
		}
		if wants_docker == false && conf.USE_DOCKER == "only" {
			return 0, errors.New("error: use of docker images is enforced by administrator")
		}

		// copy or create symlink in work dir
		linkname := path.Join(workunit.Path(), name)
		if conf.NO_SYMLINK {
			// some programs do not accept symlinks (e.g. emirge), need to copy the file into the work directory
			logger.Debug(1, "copy predata: "+file_path+" -> "+linkname)
			_, err := shock.CopyFile(file_path, linkname)
			if err != nil {
				return 0, fmt.Errorf("error copying file from %s to % s: ", file_path, linkname, err.Error())
			}
		} else {
			if wants_docker {
				// new filepath for predata dir in container
				var docker_file_path string
				if isShockPredata {
					docker_file_path = path.Join(conf.DOCKER_WORKUNIT_PREDATA_DIR, node_md5)
				} else {
					docker_file_path = path.Join(conf.DOCKER_WORKUNIT_PREDATA_DIR, name)
				}
				logger.Debug(1, "creating dangling symlink: "+linkname+" -> "+docker_file_path)
				// dangling link will give error, we ignore that here
				_ = os.Symlink(docker_file_path, linkname)
			} else {
				logger.Debug(1, "symlink:"+linkname+" -> "+file_path)
				err = os.Symlink(file_path, linkname)
				if err != nil {
					return 0, errors.New("error creating predata file symlink: " + err.Error())
				}
			}
		}

		logger.Event(event.PRE_READY, "workid="+workunit.Id+";url="+dataUrl)
	}
	return
}