func listDir(name string, ossClient *oss.Client, bucket string) []fuse.DirEntry { marker := "" result := make([]fuse.DirEntry, 0) for { if name != "" && name[len(name)-1:] != "/" { name = name + "/" } log.Printf("bucket:%s, marker:%s, name:%s, slash:%si \n", bucket, marker, name, "/") rst, err := ossClient.ListBucket(bucket, marker, name, "/") log.Printf("Result len is:%d \n", len(rst.Objects)) if err != nil { return nil } marker = rst.NextMarker for _, prefix := range rst.Prefixes { arrays := strings.Split(prefix[:len(prefix)-1], "/") dirEntry := fuse.DirEntry{Name: arrays[len(arrays)-1], Mode: fuse.S_IFDIR} result = append(result, dirEntry) } for _, obj := range rst.Objects { if obj.Key[len(obj.Key)-1:] != "/" { arrays := strings.Split(obj.Key, "/") //file system only need the last item dirEntry := fuse.DirEntry{Name: arrays[len(arrays)-1], Mode: fuse.S_IFREG} result = append(result, dirEntry) } } if marker == "" { log.Printf("let's break out \n", marker) break } } return result }
func fillBufFile(bufFile *BufFile, off int64, fileSize int64, client *oss.Client, fileName string) error { endOff := off + int64(bufFile.cacheSize) if endOff > fileSize { endOff = fileSize } _, reader, err := client.GetObjectRange(bufFile.bucket, fileName, off, endOff-1) log.Printf("q %d endoff %d \n", off, endOff-1) //callNum = callNum + 1 //if callNum > 1000000 { // return fmt.Errorf("call num exceed!") //} data := make([]byte, endOff-off) if err != nil { log.Printf("make failed!len(data):%d, endoff:%d, off:%d", len(data), endOff, off) return err } num := 0 for { l, err := reader.Read(data[num:]) if err != nil && err != io.EOF { log.Printf("reader.Read failed len(data):%d, num:%d\n", err, len(data), num) return err } num = num + l if num == len(data) { bufFile.data = data bufFile.endOff = endOff bufFile.off = off bufFile.size = int(endOff - off) return nil } } }
//after call this method,remember to close [Reader,Writer] func LocalFileReadWriteOssFile(f F, h HadoopFile, ossClient *oss.Client, bucket string, ossDir string, b bool) error { timeStamp := fmt.Sprintf("%d", makeTimestamp()) ossPath := ossDir + timeStamp + f.f.Name() hdfsPath := h.dirPrefix + h.fileInfo.Name() if b { dateStr, err := getDateStamp(hdfsPath) if err != nil { log.Printf("get date info from hdfsPath %s failed!", hdfsPath) return err } ossNameArray := strings.Split(f.f.Name(), "/") ossPath = dateStr + "/" + timeStamp + ossNameArray[(len(ossNameArray)-1)] if err != nil { return err } } fi, err := os.Open(f.f.Name()) log.Printf("Open file %s \n", f.f.Name()) defer fi.Close() if err != nil { return err } err = ossClient.PutObject(bucket, ossPath, fi, nil) if err != nil { log.Printf("Oss Put Object failed %s!\n", err) return err } else { log.Printf("ossDir:%s put file:%s success!! \n", ossDir, f.f.Name()) } return nil }
func writeToOss(h HadoopFile, ossDir string, hdfsClient *hdfs.Client, ossClient *oss.Client, bucket string, b bool) error { log.Printf("Start to sync %s to ossDir %s\n", h.dirPrefix+h.fileInfo.Name(), ossDir) hdfsPath := h.dirPrefix + h.fileInfo.Name() timeStamp := fmt.Sprintf("%d", makeTimestamp()) ossPath := ossDir + timeStamp + h.fileInfo.Name() if b { dateStr, err := getDateStamp(hdfsPath) if err != nil { log.Printf("get date info from hdfsPath %s failed!", hdfsPath) return err } ossPath = dateStr + "/" + timeStamp + h.fileInfo.Name() if err != nil { return err } } reader, err := hdfsClient.Open(hdfsPath) if err != nil { log.Printf("hdfsClient opend failed! err message: %s\n", err) return err } defer reader.Close() err = ossClient.PutObject(bucket, ossPath, reader, nil) if err != nil { log.Printf("Oss Append Object failed %s!\n", err) return err } log.Printf("Finished sync %s to ossDir \n", h.dirPrefix+h.fileInfo.Name(), ossDir) return nil }
func WriteFileToOss(filePath string, fileName string, ossClient *oss.Client, ossPath string, bucket string) error { fi, err := os.Open(filePath) defer fi.Close() if err != nil { return err } err = ossClient.PutObject(bucket, ossPath, fi, nil) if err != nil { log.Printf("Oss Put Object failed %s!\n", err) return err } else { return nil } }
func CheckGzFile(fileName string, ossClient *oss.Client, bucket string) ([]string, error) { marker := "" errFiles := make([]string, 0) for { rst, err := ossClient.ListBucket(bucket, marker, fileName, "/") if err != nil { log.Printf("err;%s \n", err) return nil, nil } marker = rst.NextMarker if err != nil { log.Printf("err!:%s \n", err) return nil, err } for _, o := range rst.Objects { if err := func() error { _, readerCloser, err := ossClient.GetObject(bucket, o.Key) gzReader, err := gzip.NewReader(readerCloser) defer readerCloser.Close() defer gzReader.Close() if err != nil { log.Printf("new gzreader failed !%s \n", err) return err } if err != nil { log.Printf("GetObject failed !%s \n", err) return err } written, err := io.Copy(ioutil.Discard, gzReader) if err != nil { log.Printf("Have found one error file !so we mark it!written is %d \n", written) errFiles = append(errFiles, o.Key) return err } return nil }(); err != nil { log.Printf("oss get and checked failed!err:%s o is %s\n", err, o) } } if marker == "" { break } } return errFiles, nil }