Example #1
0
func listDir(name string, ossClient *oss.Client, bucket string) []fuse.DirEntry {
	marker := ""
	result := make([]fuse.DirEntry, 0)
	for {
		if name != "" && name[len(name)-1:] != "/" {
			name = name + "/"
		}
		log.Printf("bucket:%s, marker:%s, name:%s, slash:%si \n", bucket, marker, name, "/")
		rst, err := ossClient.ListBucket(bucket, marker, name, "/")
		log.Printf("Result len is:%d \n", len(rst.Objects))
		if err != nil {
			return nil
		}
		marker = rst.NextMarker
		for _, prefix := range rst.Prefixes {
			arrays := strings.Split(prefix[:len(prefix)-1], "/")
			dirEntry := fuse.DirEntry{Name: arrays[len(arrays)-1], Mode: fuse.S_IFDIR}
			result = append(result, dirEntry)
		}
		for _, obj := range rst.Objects {
			if obj.Key[len(obj.Key)-1:] != "/" {
				arrays := strings.Split(obj.Key, "/")
				//file system only need the last item
				dirEntry := fuse.DirEntry{Name: arrays[len(arrays)-1], Mode: fuse.S_IFREG}
				result = append(result, dirEntry)
			}
		}
		if marker == "" {
			log.Printf("let's break out \n", marker)
			break
		}
	}
	return result
}
Example #2
0
func fillBufFile(bufFile *BufFile, off int64, fileSize int64, client *oss.Client, fileName string) error {
	endOff := off + int64(bufFile.cacheSize)
	if endOff > fileSize {
		endOff = fileSize
	}
	_, reader, err := client.GetObjectRange(bufFile.bucket, fileName, off, endOff-1)
	log.Printf("q %d endoff %d \n", off, endOff-1)
	//callNum = callNum + 1
	//if callNum > 1000000 {
	//	return fmt.Errorf("call num exceed!")
	//}
	data := make([]byte, endOff-off)
	if err != nil {
		log.Printf("make failed!len(data):%d, endoff:%d, off:%d", len(data), endOff, off)
		return err
	}
	num := 0
	for {
		l, err := reader.Read(data[num:])
		if err != nil && err != io.EOF {
			log.Printf("reader.Read failed len(data):%d, num:%d\n", err, len(data), num)
			return err
		}
		num = num + l
		if num == len(data) {
			bufFile.data = data
			bufFile.endOff = endOff
			bufFile.off = off
			bufFile.size = int(endOff - off)
			return nil
		}
	}
}
Example #3
0
//after call this method,remember to close [Reader,Writer]
func LocalFileReadWriteOssFile(f F, h HadoopFile, ossClient *oss.Client,
	bucket string, ossDir string, b bool) error {
	timeStamp := fmt.Sprintf("%d", makeTimestamp())
	ossPath := ossDir + timeStamp + f.f.Name()
	hdfsPath := h.dirPrefix + h.fileInfo.Name()
	if b {
		dateStr, err := getDateStamp(hdfsPath)
		if err != nil {
			log.Printf("get date info from hdfsPath %s failed!", hdfsPath)
			return err
		}
		ossNameArray := strings.Split(f.f.Name(), "/")
		ossPath = dateStr + "/" + timeStamp + ossNameArray[(len(ossNameArray)-1)]
		if err != nil {
			return err
		}
	}
	fi, err := os.Open(f.f.Name())
	log.Printf("Open file %s \n", f.f.Name())
	defer fi.Close()
	if err != nil {
		return err
	}
	err = ossClient.PutObject(bucket, ossPath, fi, nil)
	if err != nil {
		log.Printf("Oss Put Object failed %s!\n", err)
		return err
	} else {
		log.Printf("ossDir:%s put file:%s success!! \n", ossDir, f.f.Name())
	}
	return nil
}
Example #4
0
func writeToOss(h HadoopFile, ossDir string, hdfsClient *hdfs.Client,
	ossClient *oss.Client, bucket string, b bool) error {
	log.Printf("Start to sync %s to ossDir %s\n", h.dirPrefix+h.fileInfo.Name(), ossDir)
	hdfsPath := h.dirPrefix + h.fileInfo.Name()
	timeStamp := fmt.Sprintf("%d", makeTimestamp())
	ossPath := ossDir + timeStamp + h.fileInfo.Name()
	if b {
		dateStr, err := getDateStamp(hdfsPath)
		if err != nil {
			log.Printf("get date info from hdfsPath %s failed!", hdfsPath)
			return err
		}
		ossPath = dateStr + "/" + timeStamp + h.fileInfo.Name()
		if err != nil {
			return err
		}
	}
	reader, err := hdfsClient.Open(hdfsPath)
	if err != nil {
		log.Printf("hdfsClient opend failed! err message: %s\n", err)
		return err
	}
	defer reader.Close()
	err = ossClient.PutObject(bucket, ossPath, reader, nil)
	if err != nil {
		log.Printf("Oss Append Object failed %s!\n", err)
		return err
	}
	log.Printf("Finished sync %s to ossDir \n", h.dirPrefix+h.fileInfo.Name(), ossDir)
	return nil
}
Example #5
0
func WriteFileToOss(filePath string, fileName string, ossClient *oss.Client, ossPath string, bucket string) error {
	fi, err := os.Open(filePath)
	defer fi.Close()
	if err != nil {
		return err
	}
	err = ossClient.PutObject(bucket, ossPath, fi, nil)
	if err != nil {
		log.Printf("Oss Put Object failed %s!\n", err)
		return err
	} else {
		return nil
	}
}
Example #6
0
func CheckGzFile(fileName string, ossClient *oss.Client, bucket string) ([]string, error) {
	marker := ""
	errFiles := make([]string, 0)
	for {
		rst, err := ossClient.ListBucket(bucket, marker, fileName, "/")
		if err != nil {
			log.Printf("err;%s \n", err)
			return nil, nil
		}
		marker = rst.NextMarker
		if err != nil {
			log.Printf("err!:%s \n", err)
			return nil, err
		}
		for _, o := range rst.Objects {
			if err := func() error {
				_, readerCloser, err := ossClient.GetObject(bucket, o.Key)
				gzReader, err := gzip.NewReader(readerCloser)
				defer readerCloser.Close()
				defer gzReader.Close()
				if err != nil {
					log.Printf("new gzreader failed !%s \n", err)
					return err
				}
				if err != nil {
					log.Printf("GetObject failed !%s \n", err)
					return err
				}
				written, err := io.Copy(ioutil.Discard, gzReader)
				if err != nil {
					log.Printf("Have found one error file !so we mark it!written is %d \n", written)
					errFiles = append(errFiles, o.Key)
					return err
				}
				return nil
			}(); err != nil {
				log.Printf("oss get and checked failed!err:%s o is %s\n", err, o)
			}
		}
		if marker == "" {
			break
		}
	}
	return errFiles, nil
}