Exemplo n.º 1
0
func (s *Disk) downloadIndex(ctx context.Context, index types.ObjectIndex) (*os.File, *http.Response, error) {
	vhost, ok := contexts.GetVhost(ctx)
	if !ok {
		return nil, nil, fmt.Errorf("Could not get vhost from context.")
	}

	startOffset := uint64(index.Part) * s.partSize
	endOffset := startOffset + s.partSize - 1
	resp, err := vhost.Upstream.GetRequestPartial(index.ObjID.Path, startOffset, endOffset)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()
	file, err := CreateFile(path.Join(s.path, pathFromIndex(index)))
	if err != nil {
		return nil, nil, err
	}

	size, err := io.Copy(file, resp.Body)
	if err != nil {
		return nil, nil, utils.NewCompositeError(err, file.Close())
	}
	s.logger.Debugf("Storage [%p] downloaded for index %s with size %d", s, index, size)

	_, err = file.Seek(0, os.SEEK_SET)
	if err != nil {
		return nil, nil, utils.NewCompositeError(err, file.Close())
	}

	return file, resp, err
}
Exemplo n.º 2
0
// GetPart returns an io.ReadCloser that will read the specified part of the
// object from the disk.
func (s *Disk) GetPart(idx *types.ObjectIndex) (io.ReadCloser, error) {
	s.logger.Debugf("[DiskStorage] Getting file data for %s...", idx)
	f, err := os.Open(s.getObjectIndexPath(idx))
	if err != nil {
		return nil, err
	}
	stat, err := f.Stat()
	if err != nil {
		return nil, utils.NewCompositeError(err, f.Close())
	}

	if uint64(stat.Size()) > s.partSize {
		err = fmt.Errorf("Object part has invalid size %d", stat.Size())
		return nil, utils.NewCompositeError(err, f.Close(), s.DiscardPart(idx))
	}

	return f, nil
}
Exemplo n.º 3
0
// SavePart writes the contents of the supplied object part to the disk.
func (s *Disk) SavePart(idx *types.ObjectIndex, data io.Reader) error {
	s.logger.Debugf("[DiskStorage] Saving file data for %s...", idx)

	tmpPath := appendRandomSuffix(s.getObjectIndexPath(idx))
	f, err := s.createFile(tmpPath)
	if err != nil {
		return err
	}

	if savedSize, err := io.Copy(f, data); err != nil {
		return utils.NewCompositeError(err, f.Close(), os.Remove(tmpPath))
	} else if uint64(savedSize) > s.partSize {
		err = fmt.Errorf("Object part has invalid size %d", savedSize)
		return utils.NewCompositeError(err, f.Close(), os.Remove(tmpPath))
	} else if err := f.Close(); err != nil {
		return err
	}

	return os.Rename(tmpPath, s.getObjectIndexPath(idx))
}
Exemplo n.º 4
0
Arquivo: utils.go Projeto: na--/nedomi
func (s *Disk) getObjectMetadata(objPath string) (*types.ObjectMetadata, error) {
	f, err := os.Open(objPath)
	if err != nil {
		return nil, err
	}

	obj := &types.ObjectMetadata{}
	if err := json.NewDecoder(f).Decode(&obj); err != nil {
		return nil, utils.NewCompositeError(err, f.Close())
	}

	if filepath.Base(filepath.Dir(objPath)) != obj.ID.StrHash() {
		err := fmt.Errorf("The object %s was in the wrong directory: %s", obj.ID, objPath)
		return nil, utils.NewCompositeError(err, f.Close())
	}
	//!TODO: add more validation? ex. compare the cache key as well? also the
	// data itself may be corrupted or from an old app version

	return obj, f.Close()
}
Exemplo n.º 5
0
func (f *partReadCloser) Close() error {
	var readFromCurrent = uint64(f.readSoFar) % f.partSize
	var err error
	if readFromCurrent != 0 {
		_, err = io.CopyN(
			ioutil.Discard,
			f.ReadCloser,
			int64(f.partSize-readFromCurrent),
		)
	}
	if err == io.EOF {
		err = nil
	}

	return utils.NewCompositeError(err, f.ReadCloser.Close())
}
Exemplo n.º 6
0
Arquivo: utils.go Projeto: na--/nedomi
func (s *Disk) saveSettingsOnDisk(cz *config.CacheZone) error {
	if err := s.checkPreviousDiskSettings(cz); err != nil {
		return err
	}

	filePath := filepath.Join(s.path, diskSettingsFileName)
	f, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY, s.filePermissions)
	if err != nil {
		return err
	}

	if err = json.NewEncoder(f).Encode(cz); err != nil {
		return utils.NewCompositeError(err, f.Close())
	}

	return f.Close()
}
Exemplo n.º 7
0
// SaveMetadata writes the supplied metadata to the disk.
func (s *Disk) SaveMetadata(m *types.ObjectMetadata) error {
	s.logger.Debugf("[DiskStorage] Saving metadata for %s...", m.ID)

	tmpPath := appendRandomSuffix(s.getObjectMetadataPath(m.ID))
	f, err := s.createFile(tmpPath)
	if err != nil {
		return err
	}

	if err = json.NewEncoder(f).Encode(m); err != nil {
		return utils.NewCompositeError(err, f.Close())
	} else if err := f.Close(); err != nil {
		return err
	}

	//!TODO: use a faster encoding than json (some binary marshaller? gob?)

	return os.Rename(tmpPath, s.getObjectMetadataPath(m.ID))
}
Exemplo n.º 8
0
Arquivo: utils.go Projeto: na--/nedomi
func (s *Disk) checkPreviousDiskSettings(newSettings *config.CacheZone) error {
	f, err := os.Open(filepath.Join(s.path, diskSettingsFileName))
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}

	oldSettings := &config.CacheZone{}
	if err := json.NewDecoder(f).Decode(&oldSettings); err != nil {
		return utils.NewCompositeError(err, f.Close())
	}
	if err := f.Close(); err != nil {
		return err
	}

	if oldSettings.PartSize != newSettings.PartSize {
		return fmt.Errorf("Old partsize is %d and new partsize is %d",
			oldSettings.PartSize, newSettings.PartSize)
	}
	//!TODO: more validation?
	return nil
}