func GenerateList(inlist, outlist string) bool {
if !util.IsExist(inlist) {
fmt.Println("input list do not exist! File : ", inlist)
return false
}
ofw, err := os.Create(outlist)
if err != nil {
fmt.Println("output list generatation failed! File: ", outlist)
return false
}
defer ofw.Close()
ofbw := bufio.NewWriter(ofw)
ifr, err := os.Open(inlist)
if err != nil {
fmt.Println("input list can not open! File: ", inlist)
return false
}
defer ifr.Close()
ifbr := bufio.NewScanner(ifr)
for ifbr.Scan() {
key := ifbr.Text()
fmt.Println("Keyword: ", key)
str := fmt.Sprintf("/CDShare/Corpus/monc/clean/NU-%s.raw\n", key)
fmt.Println("Path: ", str)
ofbw.WriteString(str)
}
ofbw.Flush()
return true
}
func main() {
var inlist, outlist, cmd string
flag.StringVar(&inlist, "i", "", "list of the training data keyword")
flag.StringVar(&outlist, "o", "", "path list of the training data keyword")
flag.StringVar(&cmd, "c", "", "path of command")
flag.Parse()
if !util.IsExist(outlist) {
if !monc.GenerateList(inlist, outlist) {
fmt.Println("playlist generation failed")
}
}
ofr, err := os.Open(outlist)
if err != nil {
return
}
defer ofr.Close()
ofrb := bufio.NewScanner(ofr)
fmt.Println("Start the task")
for ofrb.Scan() {
command := exec.Command(cmd, ofrb.Text())
out, _ := command.CombinedOutput()
fmt.Println("Run Cmd : ", cmd, ofrb.Text())
command.Run()
fmt.Println("Out :", string(out))
}
fmt.Println("Finished the task")
}
func FileSize(file string) int64 {
if !util.IsExist(file) {
fmt.Println("No Exist File : ", file)
return -1
}
info, err := os.Stat(file)
if err != nil {
fmt.Println("Info no catched File : ", file)
return -1
}
return info.Size()
}
//获取项目指定branch的commit信息，包括id和message
func GitFileCommitids(projectName, branchName, filepath string) (commitInfos []CommitInfo, err error) {
deployDir := config.GitDeployDir + projectName
gitDir := path.Join(deployDir, ".git")
if ok, _ := util.IsDir(deployDir); !ok {
err = util.NewError("Project[%s] Directory Not Existed", projectName)
return
}
realFilePath := path.Join(deployDir, filepath)
if ok := util.IsExist(realFilePath); !ok {
err = util.NewError("File[%s] Not Exist, Please Check", filepath)
return
}
args := [][]string{
strings.Split(fmt.Sprintf("--git-dir=%s --work-tree=%s log --pretty=format:\"%%H:%%s\" %s %s", gitDir, deployDir, branchName, realFilePath), " "),
}
msg := ""
for _, arg := range args {
so, se, e := util.RunCmd(config.GIT, arg...)
msg += so
if e != nil {
err = e
return
}
if len(se) != 0 {
err = util.NewError("Command Git log exec stderr: %s", se)
return
}
}
infos := strings.Split(msg, "\n")
for _, info := range infos {
ss := strings.Split(strings.Trim(info, "\""), ":")
if len(ss) != 2 {
err = util.NewError("Commit Info [%s] Split Error", info)
return
}
commitInfos = append(commitInfos, CommitInfo{Id: ss[0], Msg: ss[1]})
}
return
}
func NewLogger(prefix, outputFile string, tag int, debug bool) *Logger {
// Remove the old log file.
if util.IsExist(outputFile) {
if err := os.Remove(outputFile); err != nil {
return nil
}
}
// Create new log file.
file, err := os.Create(outputFile)
if err != nil {
return nil
}
// Create the logger.
return &Logger{
logger: log.New(file, prefix, log.Lshortfile|log.LstdFlags),
tag: tag,
debug: debug,
}
}
//通过读取文件获取某个branch最新的commitID
func GetGitBranchHeadCommitid(projectName, branchName string) (commitId string, err error) {
deployDir := config.GitDeployDir + projectName
if ok, _ := util.IsDir(deployDir); !ok {
err = util.NewError("Project[%s] Directory Not Existed", projectName)
return
}
headPath := path.Join(deployDir, fmt.Sprintf(".git/refs/heads/%s", branchName))
if ok := util.IsExist(headPath); !ok {
err = util.NewError("[%s] Branch Head File Not Exist, Please Make Sure The Branch Existed", branchName)
return
}
content, err := ioutil.ReadFile(headPath)
if err != nil {
return
}
commitId = strings.TrimSpace(string(content))
return
}
func MergeRaw(inlist, outdir string) bool {
if !util.IsExist(inlist) {
return false
}
ifr, err := os.Open(inlist)
if err != nil {
return false
}
defer ifr.Close()
var maxSize int64 = MSize(15*60, 8000, 16)
var sumSize int64 = 0
var index int = 0
var fmerge, flist *os.File = nil, nil
var fw, flw *bufio.Writer = nil, nil
var zerobuf [8000 * 2 / 4]byte
isc := bufio.NewScanner(ifr)
for isc.Scan() {
fileSize := FileSize(isc.Text())
if sumSize <= 0 || sumSize+fileSize > maxSize {
if fmerge != nil {
if fw != nil {
fw.Flush()
flw.Flush()
} else {
fmt.Println(" bufio Write Error")
}
flist.Close()
fmerge.Close()
}
// update merge file
mergefile := filepath.Join(outdir, fmt.Sprintf("merge%d.raw", index))
filelist := filepath.Join(outdir, fmt.Sprintf("merge%d.list", index))
fmt.Println("merge file ", mergefile)
flist, _ = os.Create(filelist)
fmerge, _ = os.Create(mergefile)
fw = bufio.NewWriter(fmerge)
flw = bufio.NewWriter(flist)
index++
sumSize = 0
}
fmt.Println("File: ", isc.Text(), "\n Size : ", fileSize)
list := fmt.Sprintf("%s %f\n", isc.Text(), SLen(fileSize, 8000, 16))
fr, _ := os.Open(isc.Text())
io.Copy(fw, fr)
fw.Write(zerobuf[0:])
flw.WriteString(list)
sumSize = sumSize + fileSize
}
if fmerge != nil {
fmt.Println("the last file close")
if fw != nil {
fw.Flush()
flw.Flush()
} else {
fmt.Println(" bufio Write Error")
}
fmerge.Close()
flist.Close()
}
return true
}
func (fs *LocalFS) Exist(name string) bool {
return util.IsExist(fs.root + name)
}