//after call this method,remember to close [Reader,Writer]
func LocalFileReadWriteOssFile(f F, h HadoopFile, ossClient *oss.Client,
bucket string, ossDir string, b bool) error {
timeStamp := fmt.Sprintf("%d", makeTimestamp())
ossPath := ossDir + timeStamp + f.f.Name()
hdfsPath := h.dirPrefix + h.fileInfo.Name()
if b {
dateStr, err := getDateStamp(hdfsPath)
if err != nil {
log.Printf("get date info from hdfsPath %s failed!", hdfsPath)
return err
}
ossNameArray := strings.Split(f.f.Name(), "/")
ossPath = dateStr + "/" + timeStamp + ossNameArray[(len(ossNameArray)-1)]
if err != nil {
return err
}
}
fi, err := os.Open(f.f.Name())
log.Printf("Open file %s \n", f.f.Name())
defer fi.Close()
if err != nil {
return err
}
err = ossClient.PutObject(bucket, ossPath, fi, nil)
if err != nil {
log.Printf("Oss Put Object failed %s!\n", err)
return err
} else {
log.Printf("ossDir:%s put file:%s success!! \n", ossDir, f.f.Name())
}
return nil
}
func writeToOss(h HadoopFile, ossDir string, hdfsClient *hdfs.Client,
ossClient *oss.Client, bucket string, b bool) error {
log.Printf("Start to sync %s to ossDir %s\n", h.dirPrefix+h.fileInfo.Name(), ossDir)
hdfsPath := h.dirPrefix + h.fileInfo.Name()
timeStamp := fmt.Sprintf("%d", makeTimestamp())
ossPath := ossDir + timeStamp + h.fileInfo.Name()
if b {
dateStr, err := getDateStamp(hdfsPath)
if err != nil {
log.Printf("get date info from hdfsPath %s failed!", hdfsPath)
return err
}
ossPath = dateStr + "/" + timeStamp + h.fileInfo.Name()
if err != nil {
return err
}
}
reader, err := hdfsClient.Open(hdfsPath)
if err != nil {
log.Printf("hdfsClient opend failed! err message: %s\n", err)
return err
}
defer reader.Close()
err = ossClient.PutObject(bucket, ossPath, reader, nil)
if err != nil {
log.Printf("Oss Append Object failed %s!\n", err)
return err
}
log.Printf("Finished sync %s to ossDir \n", h.dirPrefix+h.fileInfo.Name(), ossDir)
return nil
}
func WriteFileToOss(filePath string, fileName string, ossClient *oss.Client, ossPath string, bucket string) error {
fi, err := os.Open(filePath)
defer fi.Close()
if err != nil {
return err
}
err = ossClient.PutObject(bucket, ossPath, fi, nil)
if err != nil {
log.Printf("Oss Put Object failed %s!\n", err)
return err
} else {
return nil
}
}
