Exemple #1
0
// if full sync ok, return sync offset for later backlog syncing
func (h *Handler) replicationSlaveFullSync(c *conn) (syncOffset int64, resp redis.Resp, err error) {
	// after bgsave, we must send this RDB to slave,
	// so we don't allow others do bgsave before full sync done.
	if ok := h.bgSaveSem.AcquireTimeout(time.Minute); !ok {
		resp, err = toRespErrorf("wait others do bgsave timeout")
		return
	}
	defer h.bgSaveSem.Release()

	// now begin full sync
	h.counters.syncFull.Add(1)

	var rdb *os.File
	rdb, syncOffset, err = h.replicationBgSave()
	if err != nil {
		resp, err = toRespError(err)
		return
	}
	defer rdb.Close()

	// send rdb to slave
	st, _ := rdb.Stat()

	rdbSize := st.Size()

	if err = c.writeRDBFrom(rdbSize, rdb); err != nil {
		// close this connection here???
		log.Errorf("slave %s sync rdb err - %s", c, err)
		c.Close()
		return
	}

	return syncOffset, nil, nil
}
Exemple #2
0
func OpenFileOrStdDev(path string, write bool) *os.File {
	var fp *os.File
	var err error

	if path == "stdin" {
		fp = os.Stdin
		err = nil
	} else if path == "stdout" {
		fp = os.Stdout
		err = nil
	} else {
		if write {
			fp, err = os.Create(CleanPath(path))
		} else {
			fp, err = os.Open(CleanPath(path))
		}
	}

	if err != nil {
		log.Fatalln(err)
	}

	stat, statErr := fp.Stat()

	if statErr != nil {
		log.Fatalln(err)
	}

	if stat.IsDir() {
		log.Fatalf("%s: is a directory\n", path)
	}

	return fp
}
Exemple #3
0
// UploadReleaseAsset creates an asset by uploading a file into a release repository.
// To upload assets that cannot be represented by an os.File, call NewUploadRequest directly.
//
// GitHub API docs : http://developer.github.com/v3/repos/releases/#upload-a-release-asset
func (s *RepositoriesService) UploadReleaseAsset(owner, repo string, id int, opt *UploadOptions, file *os.File) (*ReleaseAsset, *Response, error) {
	u := fmt.Sprintf("repos/%s/%s/releases/%d/assets", owner, repo, id)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	stat, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}
	if stat.IsDir() {
		return nil, nil, errors.New("the asset to upload can't be a directory")
	}

	mediaType := mime.TypeByExtension(filepath.Ext(file.Name()))
	req, err := s.client.NewUploadRequest(u, file, stat.Size(), mediaType)
	if err != nil {
		return nil, nil, err
	}

	asset := new(ReleaseAsset)
	resp, err := s.client.Do(req, asset)
	if err != nil {
		return nil, resp, err
	}
	return asset, resp, err
}
Exemple #4
0
func main() {
	var APIOptions = oss.GetDefaultAPIOptioins()
	APIOptions.AccessID = AccessKeyID
	APIOptions.SecretAccessKey = AccessKeySecret
	var OSSAPI, err = oss.NewAPI(APIOptions)

	if err != nil {
		log.Fatal(err)
	}

	var bucket = "ossgosdklargefile"
	var object = "largefile.bin"
	var bufSize = int64(1024 * 1024 * 2) // 2 M
	var file = "test.bin"
	var fp *os.File

	if err = OSSAPI.PutBucket(bucket, oss.ACLPublicReadWrite, "", nil); err != nil {
		log.Printf("%s\n", err)
	}

	var multi *oss.MultipartUpload
	if multi, err = OSSAPI.NewMultipartUpload(bucket, object, nil); err != nil {
		log.Printf("%s\n", err)
	}

	if fp, err = os.Open(file); err != nil {
		log.Fatal(err)
	}

	var stat, _ = fp.Stat()
	var fileLength = stat.Size()
	var filePart = int(fileLength / bufSize)
	if int64(filePart)*bufSize < fileLength {
		filePart = filePart + 1
	}

	var rd io.Reader
	var parts = make([]oss.Part, filePart)
	var etag string

	for i := 1; i <= filePart; i++ {
		rd = io.LimitReader(fp, bufSize)
		if etag, err = multi.UploadPart(i, rd); err != nil {
			log.Printf("%s\n", err)
		}
		log.Printf("PartNumber: %d, ETag: %s\n", i, etag)
		parts[i-1] = oss.Part{
			PartNumber: i,
			ETag:       etag,
		}
	}

	var result oss.CompleteMultipartUploadResult

	if err = multi.CompleteUpload(parts, &result); err != nil {
		log.Printf("%s\n", err)
	}

	log.Printf("Upload result: %s\n", result)
}
Exemple #5
0
func getSize(f *os.File) (int64, error) {
	fi, err := f.Stat()
	if err != nil {
		return 0, err
	}
	return fi.Size(), nil
}
Exemple #6
0
//----------------------------------------------- parse & load a game data file into dictionary
func parse(file *os.File) {
	// csv 读取器
	csv_reader := csv.NewReader(file)
	records, err := csv_reader.ReadAll()
	if err != nil {
		ERR("cannot parse csv file.", file.Name(), err)
		return
	}

	// 是否为空档
	if len(records) == 0 {
		ERR("csv file is empty", file.Name())
		return
	}

	// 处理表名
	fi, err := file.Stat()
	if err != nil {
		ERR("cannot stat the file", file.Name())
		return
	}
	tblname := strings.TrimSuffix(fi.Name(), path.Ext(file.Name()))

	// 记录数据, 第一行为表头,因此从第二行开始
	for line := 1; line < len(records); line++ {
		for field := 1; field < len(records[line]); field++ { // 每条记录的第一个字段作为行索引
			_set(tblname, records[line][0], records[0][field], records[line][field])
		}
	}
}
Exemple #7
0
func GetFileSize(file *os.File) (size int64, err error) {
	var fi os.FileInfo
	if fi, err = file.Stat(); err == nil {
		size = fi.Size()
	}
	return
}
Exemple #8
0
// NewBufferFile maps a file to shared memory and returns a handle to the shared memory buffer
func NewBufferFile(file *os.File, size, prot int) (Buffer, error) {
	fi, err := file.Stat()
	if err != nil {
		return nil, err
	}
	sys := fi.Sys().(*syscall.Stat_t)
	if sys.Size != int64(size) {
		return nil, errWrongSize
	}

	// Dup to allow file parameter to be closed regardless
	fd, err := syscall.Dup(int(file.Fd()))
	if err != nil {
		return nil, err
	}
	const flags = syscall.MAP_SHARED
	b, err := syscall.Mmap(fd, 0, size, prot, flags)
	if err != nil {
		return nil, err
	}

	// localFile is nil because fd is from somewhere else
	buf := &sharedBuffer{os.NewFile(uintptr(fd), ""), nil, b, stackToKeep()}
	runtime.SetFinalizer(buf, (*sharedBuffer).finalize)

	return buf, nil
}
Exemple #9
0
func (g *GraphTool) tarCp(srcName string, tw *tar.Writer) (int64, error) {
	var (
		src *os.File
		err error
	)

	if src, err = os.Open(srcName); err != nil {
		return 0, err
	}
	defer src.Close()

	srcStat, err := src.Stat()
	if err != nil {
		g.logger.Error(err.Error())
	} else if err := unix.Fadvise(int(src.Fd()), 0, srcStat.Size(), unix.MADV_SEQUENTIAL); err != nil {
		g.logger.Error(err.Error())
	}

	if n, err := io.Copy(tw, src); err != nil {
		g.logger.Error(err.Error())
	} else {
		return n, nil
	}

	return 0, nil
}
Exemple #10
0
func isTTY(file *os.File) bool {
	stat, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	return ((stat.Mode() & os.ModeCharDevice) != 0)
}
Exemple #11
0
func customReader() ([]byte, int) {
	var file *os.File
	var err error

	if section == nil {
		file, err = os.Open(inputSampleFilename)
		if err != nil {
			panic(err)
		}

		fi, err := file.Stat()
		if err != nil {
			panic(err)
		}

		section = io.NewSectionReader(file, 0, fi.Size())
	}

	b := make([]byte, IO_BUFFER_SIZE)

	n, err := section.Read(b)
	if err != nil {
		fmt.Println("section.Read():", err)
		file.Close()
	}

	return b, n
}
Exemple #12
0
func ReadFile(name string) (b []byte, e os.Error) {
	var (
		raw  []byte
		err  os.Error
		file *os.File
		dir  *os.Dir
	)

	file, err = os.Open(name, os.O_RDONLY, 0)
	if err != nil {
		return raw, err
	}

	dir, err = file.Stat()
	if err != nil {
		return raw, err
	}

	var size uint64 = dir.Size
	raw = make([]byte, size)
	_, err = file.Read(raw)
	if err != nil {
		return raw, err
	}

	return raw, err
}
Exemple #13
0
func (c *Conn) StoreBlob(file *os.File) (string, error) {
	reader := hashed.NewSha1FileReader(file)

	info, err := file.Stat()
	if err != nil {
		return "", err
	}
	size := info.Size()

	if err := c.WriteJSONFrame(request{"store", "", size, false}); err != nil {
		return "", err
	}

	if err := c.WriteBlobFrameFrom(reader, size); err != nil {
		return "", err
	}

	key := util.HexHash(reader)
	if err := c.WriteJSONFrame(request{Key: key}); err != nil {
		return "", err
	}

	var res response
	if err := c.ReadJSONFrame(&res); err != nil {
		return "", err
	}

	if res.Val != "ok" {
		return "", responseError(res)
	}

	return key, nil
}
Exemple #14
0
func LoadNeedleMap(file *os.File) *NeedleMap {
	nm := NewNeedleMap(file)
	bytes := make([]byte, 16*RowsToRead)
	count, e := nm.indexFile.Read(bytes)
	if count > 0 {
		fstat, _ := file.Stat()
		log.Println("Loading index file", fstat.Name(), "size", fstat.Size())
	}
	for count > 0 && e == nil {
		for i := 0; i < count; i += 16 {
			key := util.BytesToUint64(bytes[i : i+8])
			offset := util.BytesToUint32(bytes[i+8 : i+12])
			size := util.BytesToUint32(bytes[i+12 : i+16])
			if offset > 0 {
				nm.m.Set(Key(key), offset, size)
				//log.Println("reading key", key, "offset", offset, "size", size)
				nm.fileCounter++
			} else {
				nm.m.Delete(Key(key))
				//log.Println("removing key", key)
				nm.deletionCounter++
			}
		}

		count, e = nm.indexFile.Read(bytes)
	}
	return nm
}
func RawToFixed(f *os.File, options *VHDOptions) {
	info, err := f.Stat()
	check(err)
	size := uint64(info.Size())
	header := CreateFixedHeader(size, options)
	binary.Write(f, binary.BigEndian, header)
}
Exemple #16
0
//Return true if the provided file appears to be a character device
func isInteractive(file *os.File) bool {
	fileInfo, err := file.Stat()
	if err != nil {
		return false
	}
	return fileInfo.Mode()&(os.ModeCharDevice|os.ModeCharDevice) != 0
}
Exemple #17
0
// readTombstoneV2 reads the second version of tombstone files that are capable
// of storing keys and the range of time for the key that points were deleted. This
// format is binary.
func (t *Tombstoner) readTombstoneV2(f *os.File) ([]Tombstone, error) {
	// Skip header, already checked earlier
	if _, err := f.Seek(4, os.SEEK_SET); err != nil {
		return nil, err
	}
	n := int64(4)

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}
	size := fi.Size()

	tombstones := []Tombstone{}
	var (
		min, max int64
		key      string
	)
	b := make([]byte, 4096)
	for {
		if n >= size {
			return tombstones, nil
		}

		if _, err = f.Read(b[:4]); err != nil {
			return nil, err
		}
		n += 4

		keyLen := int(binary.BigEndian.Uint32(b[:4]))
		if keyLen > len(b) {
			b = make([]byte, keyLen)
		}

		if _, err := f.Read(b[:keyLen]); err != nil {
			return nil, err
		}
		key = string(b[:keyLen])
		n += int64(keyLen)

		if _, err := f.Read(b[:8]); err != nil {
			return nil, err
		}
		n += 8

		min = int64(binary.BigEndian.Uint64(b[:8]))

		if _, err := f.Read(b[:8]); err != nil {
			return nil, err
		}
		n += 8
		max = int64(binary.BigEndian.Uint64(b[:8]))

		tombstones = append(tombstones, Tombstone{
			Key: key,
			Min: min,
			Max: max,
		})
	}
}
Exemple #18
0
func (gh *GitHub) UploadReleaseAsset(owner, repository string, release int, name string, file *os.File) (*github.ReleaseAsset, *github.Response, error) {
	url_, err := url.Parse(fmt.Sprintf("repos/%s/%s/releases/%d/assets", owner, repository, release))
	if err != nil {
		return nil, nil, err
	}
	query := url_.Query()
	query.Add("name", name)
	url_.RawQuery = query.Encode()

	stat, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}
	if stat.IsDir() {
		return nil, nil, errors.New("invalid asset: is a directory")
	}

	rq, err := gh.Client.NewUploadRequest(url_.String(), file, stat.Size(), "application/octet-stream")
	if err != nil {
		return nil, nil, err
	}

	asset := new(github.ReleaseAsset)
	rp, err := gh.Client.Do(rq, asset)
	if err != nil {
		return nil, rp, err
	}
	return asset, rp, err
}
Exemple #19
0
func (rr *RemoteRepository) Spool(packageName string, file *os.File) (revision *RevisionInfo, err error) {
	statInfo, err := file.Stat()
	if err != nil {
		fmt.Println("Error stating file", err)
		return
	}

	revisionId, err := buildRevisionId(file)
	if err != nil {
		fmt.Println("Failed to build revision id")
		return
	}

	revision = &RevisionInfo{packageName, revisionId}

	fileName := statInfo.Name()
	nameBase := fileName[:strings.Index(fileName, ".")]

	s3Path := fmt.Sprintf("%s.%s.%s", nameBase, revisionId, fileName[strings.Index(fileName, ".")+1:])
	err = rr.bucket.PutReader(s3Path, file, statInfo.Size(), "application/octet-stream", s3.Private)
	if err != nil {
		fmt.Println("Failed to PUT revision:", err)
		return
	}
	return
}
Exemple #20
0
func (r *Runner) uploadToS3(file *os.File, b *Build, boundary string) string {
	name := fmt.Sprintf("%s-build-%s-%s.txt", b.ID, b.Commit, time.Now().Format("2006-01-02-15-04-05"))
	url := fmt.Sprintf("https://s3.amazonaws.com/%s/%s", logBucket, name)

	if _, err := file.Seek(0, os.SEEK_SET); err != nil {
		log.Printf("failed to seek log file: %s\n", err)
		return ""
	}

	stat, err := file.Stat()
	if err != nil {
		log.Printf("failed to get log file size: %s\n", err)
		return ""
	}

	log.Printf("uploading build log to S3: %s\n", url)
	if err := s3attempts.Run(func() error {
		contentType := "multipart/mixed; boundary=" + boundary
		acl := "public-read"
		_, err := r.s3.PutObject(&s3.PutObjectRequest{
			Key:           &name,
			Body:          file,
			Bucket:        &logBucket,
			ACL:           &acl,
			ContentType:   &contentType,
			ContentLength: typeconv.Int64Ptr(stat.Size()),
		})
		return err
	}); err != nil {
		log.Printf("failed to upload build output to S3: %s\n", err)
	}
	return url
}
Exemple #21
0
// BuildMetadata generates the metadata for a backup archive file.
func BuildMetadata(file *os.File) (*Metadata, error) {

	// Extract the file size.
	fi, err := file.Stat()
	if err != nil {
		return nil, errors.Trace(err)
	}
	size := fi.Size()

	// Extract the timestamp.
	timestamp := fileTimestamp(fi)

	// Get the checksum.
	hasher := sha1.New()
	_, err = io.Copy(hasher, file)
	if err != nil {
		return nil, errors.Trace(err)
	}
	rawsum := hasher.Sum(nil)
	checksum := base64.StdEncoding.EncodeToString(rawsum)

	// Build the metadata.
	meta := NewMetadata()
	meta.Started = time.Time{}
	meta.Origin = UnknownOrigin()
	err = meta.MarkComplete(size, checksum)
	if err != nil {
		return nil, errors.Trace(err)
	}
	meta.Finished = &timestamp
	return meta, nil
}
Exemple #22
0
func newFile(f *os.File, maxSize int64, pgBits uint) (*file, error) {
	if maxSize < 0 {
		panic("internal error")
	}

	pgSize := 1 << pgBits
	switch {
	case sysPage > pgSize:
		pgBits = uint(mathutil.Log2Uint64(uint64(sysPage)))
	default:
		pgBits = uint(mathutil.Log2Uint64(uint64(pgSize / sysPage * sysPage)))
	}
	pgSize = 1 << pgBits
	fi := &file{
		f: f,
		m: fileMap{},
		maxPages: int(mathutil.MinInt64(
			1024,
			mathutil.MaxInt64(maxSize/int64(pgSize), 1)),
		),
		pgBits: pgBits,
		pgMask: pgSize - 1,
		pgSize: pgSize,
	}
	info, err := f.Stat()
	if err != nil {
		return nil, err
	}

	if err = fi.Truncate(info.Size()); err != nil {
		return nil, err
	}

	return fi, nil
}
Exemple #23
0
func (gateway Gateway) NewRequestForFile(method, path, accessToken string, body *os.File) (req *Request, apiErr error) {
	progressReader := NewProgressReader(body, gateway.ui, 5*time.Second)
	progressReader.Seek(0, 0)
	fileStats, err := body.Stat()

	if err != nil {
		apiErr = errors.NewWithError(T("Error getting file info"), err)
		return
	}

	request, err := http.NewRequest(method, path, progressReader)
	if err != nil {
		apiErr = errors.NewWithError(T("Error building request"), err)
		return
	}

	fileSize := fileStats.Size()
	progressReader.SetTotalSize(fileSize)
	request.ContentLength = fileSize

	if err != nil {
		apiErr = errors.NewWithError(T("Error building request"), err)
		return
	}

	return gateway.newRequest(request, accessToken, progressReader)
}
Exemple #24
0
func ReadLastNLines(f *os.File, lines int) (string, error) {
	if lines <= 0 {
		return "", fmt.Errorf("invalid line count")
	}

	stat, err := f.Stat()
	if err != nil {
		return "", err
	}

	data, err := syscall.Mmap(int(f.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		return "", err
	}
	defer syscall.Munmap(data)

	for i := len(data) - 1; i >= 0; i-- {
		if data[i] == '\n' {
			lines--
		}

		if lines < 0 {
			return string(data[i+1 : len(data)]), nil
		}
	}

	return string(data), nil
}
Exemple #25
0
func (a *Archive) AddFile(path string, f *os.File) error {
	info, err := f.Stat()
	if err != nil {
		return err
	}
	return a.AddInfoFile(path, info, f)
}
Exemple #26
0
// ToTempFile writes an action into a generated temporary file and returns its filename
func (a Action) ToTempFile() (filename string, err error) {
	var (
		data []byte
		fd   *os.File
		fi   os.FileInfo
	)
	data, err = json.Marshal(a)
	if err != nil {
		return
	}
	fd, err = ioutil.TempFile("", "migaction_")
	defer fd.Close()
	if err != nil {
		return
	}
	_, err = fd.Write(data)
	if err != nil {
		return
	}
	fi, err = fd.Stat()
	if err != nil {
		return
	}
	filename = fmt.Sprintf("%s/%s", os.TempDir(), fi.Name())
	return
}
Exemple #27
0
// FindLoopDeviceFor returns a loopback device file for the specified file which
// is backing file of a loop back device.
func FindLoopDeviceFor(file *os.File) *os.File {
	stat, err := file.Stat()
	if err != nil {
		return nil
	}
	targetInode := stat.Sys().(*syscall.Stat_t).Ino
	targetDevice := stat.Sys().(*syscall.Stat_t).Dev

	for i := 0; true; i++ {
		path := fmt.Sprintf("/dev/loop%d", i)

		file, err := os.OpenFile(path, os.O_RDWR, 0)
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}

			// Ignore all errors until the first not-exist
			// we want to continue looking for the file
			continue
		}

		dev, inode, err := getLoopbackBackingFile(file)
		if err == nil && dev == targetDevice && inode == targetInode {
			return file
		}
		file.Close()
	}

	return nil
}
Exemple #28
0
func (c *client) getFileSize(file *os.File) (int64, error) {
	fi, err := file.Stat()
	if err != nil {
		return 0, err
	}
	return fi.Size(), nil
}
Exemple #29
0
func getLogReader(logfile string, logf *os.File) (*bufio.Reader, error) {
	var rdr *bufio.Reader
	// Is this a gzip file?
	if path.Ext(logfile) == gzipext {
		gzrdr, err := gzip.NewReader(logf)
		if err != nil {
			return nil, err
		}
		rdr = bufio.NewReader(gzrdr)
	} else {
		// See if the file has shrunk. If so, read from the beginning.
		fi, err := logf.Stat()
		if err != nil {
			return nil, err
		}
		if fi.Size() < pos {
			pos = 0
		}
		logf.Seek(pos, os.SEEK_SET)
		fmt.Printf("Starting read at offset %d\n", pos)
		rdr = bufio.NewReader(logf)
	}

	return rdr, nil
}
Exemple #30
0
func main() {
	flag.Usage = usage
	flag.Parse()

	var fd *os.File
	switch flag.NArg() {
	case 0:
		fd = os.Stdin
	case 1:
		var err error
		fd, err = os.Open(flag.Arg(0))
		ck(err)
		defer fd.Close()

		fi, err := fd.Stat()
		ck(err)
		if fi.IsDir() {
			ck(fmt.Errorf("%v: is a directory", flag.Arg(0)))
		}
	default:
		usage()
	}

	r := bufio.NewReader(fd)
	w := bufio.NewWriter(os.Stdout)
	defer w.Flush()
	if *dflag {
		decode(r, w)
	} else {
		encode(r, w)
	}
}