func (s *Disk) downloadIndex(ctx context.Context, index types.ObjectIndex) (*os.File, *http.Response, error) {
vhost, ok := contexts.GetVhost(ctx)
if !ok {
return nil, nil, fmt.Errorf("Could not get vhost from context.")
}
startOffset := uint64(index.Part) * s.partSize
endOffset := startOffset + s.partSize - 1
resp, err := vhost.Upstream.GetRequestPartial(index.ObjID.Path, startOffset, endOffset)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
file, err := CreateFile(path.Join(s.path, pathFromIndex(index)))
if err != nil {
return nil, nil, err
}
size, err := io.Copy(file, resp.Body)
if err != nil {
return nil, nil, utils.NewCompositeError(err, file.Close())
}
s.logger.Debugf("Storage [%p] downloaded for index %s with size %d", s, index, size)
_, err = file.Seek(0, os.SEEK_SET)
if err != nil {
return nil, nil, utils.NewCompositeError(err, file.Close())
}
return file, resp, err
}
// GetPart returns an io.ReadCloser that will read the specified part of the
// object from the disk.
func (s *Disk) GetPart(idx *types.ObjectIndex) (io.ReadCloser, error) {
s.logger.Debugf("[DiskStorage] Getting file data for %s...", idx)
f, err := os.Open(s.getObjectIndexPath(idx))
if err != nil {
return nil, err
}
stat, err := f.Stat()
if err != nil {
return nil, utils.NewCompositeError(err, f.Close())
}
if uint64(stat.Size()) > s.partSize {
err = fmt.Errorf("Object part has invalid size %d", stat.Size())
return nil, utils.NewCompositeError(err, f.Close(), s.DiscardPart(idx))
}
return f, nil
}
// SavePart writes the contents of the supplied object part to the disk.
func (s *Disk) SavePart(idx *types.ObjectIndex, data io.Reader) error {
s.logger.Debugf("[DiskStorage] Saving file data for %s...", idx)
tmpPath := appendRandomSuffix(s.getObjectIndexPath(idx))
f, err := s.createFile(tmpPath)
if err != nil {
return err
}
if savedSize, err := io.Copy(f, data); err != nil {
return utils.NewCompositeError(err, f.Close(), os.Remove(tmpPath))
} else if uint64(savedSize) > s.partSize {
err = fmt.Errorf("Object part has invalid size %d", savedSize)
return utils.NewCompositeError(err, f.Close(), os.Remove(tmpPath))
} else if err := f.Close(); err != nil {
return err
}
return os.Rename(tmpPath, s.getObjectIndexPath(idx))
}
func (s *Disk) getObjectMetadata(objPath string) (*types.ObjectMetadata, error) {
f, err := os.Open(objPath)
if err != nil {
return nil, err
}
obj := &types.ObjectMetadata{}
if err := json.NewDecoder(f).Decode(&obj); err != nil {
return nil, utils.NewCompositeError(err, f.Close())
}
if filepath.Base(filepath.Dir(objPath)) != obj.ID.StrHash() {
err := fmt.Errorf("The object %s was in the wrong directory: %s", obj.ID, objPath)
return nil, utils.NewCompositeError(err, f.Close())
}
//!TODO: add more validation? ex. compare the cache key as well? also the
// data itself may be corrupted or from an old app version
return obj, f.Close()
}
func (f *partReadCloser) Close() error {
var readFromCurrent = uint64(f.readSoFar) % f.partSize
var err error
if readFromCurrent != 0 {
_, err = io.CopyN(
ioutil.Discard,
f.ReadCloser,
int64(f.partSize-readFromCurrent),
)
}
if err == io.EOF {
err = nil
}
return utils.NewCompositeError(err, f.ReadCloser.Close())
}
func (s *Disk) saveSettingsOnDisk(cz *config.CacheZone) error {
if err := s.checkPreviousDiskSettings(cz); err != nil {
return err
}
filePath := filepath.Join(s.path, diskSettingsFileName)
f, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY, s.filePermissions)
if err != nil {
return err
}
if err = json.NewEncoder(f).Encode(cz); err != nil {
return utils.NewCompositeError(err, f.Close())
}
return f.Close()
}
// SaveMetadata writes the supplied metadata to the disk.
func (s *Disk) SaveMetadata(m *types.ObjectMetadata) error {
s.logger.Debugf("[DiskStorage] Saving metadata for %s...", m.ID)
tmpPath := appendRandomSuffix(s.getObjectMetadataPath(m.ID))
f, err := s.createFile(tmpPath)
if err != nil {
return err
}
if err = json.NewEncoder(f).Encode(m); err != nil {
return utils.NewCompositeError(err, f.Close())
} else if err := f.Close(); err != nil {
return err
}
//!TODO: use a faster encoding than json (some binary marshaller? gob?)
return os.Rename(tmpPath, s.getObjectMetadataPath(m.ID))
}
func (s *Disk) checkPreviousDiskSettings(newSettings *config.CacheZone) error {
f, err := os.Open(filepath.Join(s.path, diskSettingsFileName))
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
oldSettings := &config.CacheZone{}
if err := json.NewDecoder(f).Decode(&oldSettings); err != nil {
return utils.NewCompositeError(err, f.Close())
}
if err := f.Close(); err != nil {
return err
}
if oldSettings.PartSize != newSettings.PartSize {
return fmt.Errorf("Old partsize is %d and new partsize is %d",
oldSettings.PartSize, newSettings.PartSize)
}
//!TODO: more validation?
return nil
}