Beispiel #1
0
func (t *rpcType) cleanup(request sub.CleanupRequest,
	reply *sub.CleanupResponse) error {
	t.disableScannerFunc(true)
	defer t.disableScannerFunc(false)
	t.rwLock.Lock()
	defer t.rwLock.Unlock()
	t.logger.Printf("Cleanup(): %d objects\n", len(request.Hashes))
	if t.fetchInProgress {
		t.logger.Println("Error: fetch in progress")
		return errors.New("fetch in progress")
	}
	if t.updateInProgress {
		t.logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	for _, hash := range request.Hashes {
		pathname := path.Join(t.objectsDir, objectcache.HashToFilename(hash))
		err := fsutil.ForceRemove(pathname)
		if err == nil {
			t.logger.Printf("Deleted: %s\n", pathname)
		} else {
			t.logger.Println(err)
		}
	}
	return nil
}
Beispiel #2
0
func (objSrv *ObjectServer) addObject(reader io.Reader, length uint64,
	expectedHash *hash.Hash) (hash.Hash, bool, error) {
	hashVal, data, err := objectcache.ReadObject(reader, length, expectedHash)
	if err != nil {
		return hashVal, false, err
	}
	length = uint64(len(data))
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hashVal))
	// Check for existing object and collision.
	fi, err := os.Lstat(filename)
	if err == nil {
		if !fi.Mode().IsRegular() {
			return hashVal, false, errors.New("Existing non-file: " + filename)
		}
		if err := collisionCheck(data, filename, fi.Size()); err != nil {
			return hashVal, false, errors.New(
				"Collision detected: " + err.Error())
		}
		// No collision and no error: it's the same object. Go home early.
		return hashVal, false, nil
	}
	if err = os.MkdirAll(path.Dir(filename), syscall.S_IRWXU); err != nil {
		return hashVal, false, err
	}
	if err := fsutil.CopyToFile(filename, filePerms, bytes.NewReader(data),
		length); err != nil {
		return hashVal, false, err
	}
	objSrv.rwLock.Lock()
	objSrv.sizesMap[hashVal] = uint64(len(data))
	objSrv.rwLock.Unlock()
	return hashVal, true, nil
}
Beispiel #3
0
func makeRegularInode(fullPathname string, inode *filesystem.RegularInode,
	multiplyUsedObjects map[hash.Hash]uint64) {
	var err error
	if inode.Size > 0 {
		objectPathname := path.Join(objectsDir,
			objectcache.HashToFilename(inode.Hash))
		numCopies := multiplyUsedObjects[inode.Hash]
		if numCopies > 1 {
			numCopies--
			objectPathname += strings.Repeat("~", int(numCopies))
			if numCopies < 2 {
				delete(multiplyUsedObjects, inode.Hash)
			}
		}
		err = os.Rename(objectPathname, fullPathname)
	} else {
		_, err = os.Create(fullPathname)
	}
	if err != nil {
		logger.Println(err)
		return
	}
	if err := inode.WriteMetadata(fullPathname); err != nil {
		logger.Println(err)
	} else {
		if inode.Size > 0 {
			logger.Printf("Made inode: %s from: %x\n", fullPathname, inode.Hash)
		} else {
			logger.Printf("Made empty inode: %s\n", fullPathname)
		}
	}
}
Beispiel #4
0
func readOne(objectsDir string, hash hash.Hash, length uint64,
	reader io.Reader) error {
	filename := path.Join(objectsDir, objectcache.HashToFilename(hash))
	dirname := path.Dir(filename)
	if err := os.MkdirAll(dirname, syscall.S_IRWXU); err != nil {
		return err
	}
	return fsutil.CopyToFile(filename, reader, int64(length))
}
Beispiel #5
0
func (objSrv *ObjectServer) deleteObject(hashVal hash.Hash) error {
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hashVal))
	if err := os.Remove(filename); err != nil {
		return err
	}
	objSrv.rwLock.Lock()
	delete(objSrv.sizesMap, hashVal)
	objSrv.rwLock.Unlock()
	return nil
}
Beispiel #6
0
func copyFilesToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))
		if copyFile(destPathname, sourcePathname) {
			logger.Printf("Copied: %s to cache\n", sourcePathname)
		}
	}
}
Beispiel #7
0
func (t *rpcType) copyFilesToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(t.objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))
		if err := copyFile(destPathname, sourcePathname); err != nil {
			t.logger.Println(err)
		} else {
			t.logger.Printf("Copied: %s to cache\n", sourcePathname)
		}
	}
}
Beispiel #8
0
func (objSrv *ObjectServer) addObject(data []byte, expectedHash *hash.Hash) (
	hash.Hash, bool, error) {
	var hash hash.Hash
	if len(data) < 1 {
		return hash, false, errors.New("zero length object cannot be added")
	}
	hasher := sha512.New()
	if hasher.Size() != len(hash) {
		return hash, false, errors.New("Incompatible hash size")
	}
	_, err := hasher.Write(data)
	if err != nil {
		return hash, false, err
	}
	copy(hash[:], hasher.Sum(nil))
	if expectedHash != nil {
		if hash != *expectedHash {
			return hash, false, errors.New(fmt.Sprintf(
				"Hash mismatch. Computed=%x, expected=%x", hash, *expectedHash))
		}
	}
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hash))
	// Check for existing object and collision.
	fi, err := os.Lstat(filename)
	if err == nil {
		if !fi.Mode().IsRegular() {
			return hash, false, errors.New("Existing non-file: " + filename)
		}
		err := collisionCheck(data, filename, fi.Size())
		if err != nil {
			return hash, false, errors.New("Collision detected: " + err.Error())
		}
		// No collision and no error: it's the same object. Go home early.
		return hash, false, nil
	}
	err = os.MkdirAll(path.Dir(filename), 0755)
	if err != nil {
		return hash, false, err
	}
	file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0660)
	if err != nil {
		return hash, false, err
	}
	defer file.Close()
	_, err = file.Write(data)
	if err != nil {
		return hash, false, err
	}
	objSrv.sizesMap[hash] = uint64(len(data))
	return hash, true, nil
}
Beispiel #9
0
func makeObjectCopies(multiplyUsedObjects map[hash.Hash]uint64) {
	for hash, numCopies := range multiplyUsedObjects {
		if numCopies < 2 {
			continue
		}
		objectPathname := path.Join(objectsDir,
			objectcache.HashToFilename(hash))
		for numCopies--; numCopies > 0; numCopies-- {
			ext := strings.Repeat("~", int(numCopies))
			if copyFile(objectPathname+ext, objectPathname) {
				logger.Printf("Copied object: %x%s\n", hash, ext)
			}
		}
	}
}
Beispiel #10
0
func (objSrv *objectServer) AddObject(reader io.Reader, length uint64,
	expectedHash *hash.Hash) (hash.Hash, bool, error) {
	hashVal, data, err := objectcache.ReadObject(reader, length, expectedHash)
	if err != nil {
		return hashVal, false, err
	}
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hashVal))
	if err = os.MkdirAll(path.Dir(filename), dirPerms); err != nil {
		return hashVal, false, err
	}
	if err := fsutil.CopyToFile(filename, filePerms, bytes.NewReader(data),
		length); err != nil {
		return hashVal, false, err
	}
	return hashVal, true, nil
}
Beispiel #11
0
func (t *rpcType) makeObjectCopies(multiplyUsedObjects map[hash.Hash]uint64) {
	for hash, numCopies := range multiplyUsedObjects {
		if numCopies < 2 {
			continue
		}
		objectPathname := path.Join(t.objectsDir,
			objectcache.HashToFilename(hash))
		for numCopies--; numCopies > 0; numCopies-- {
			ext := strings.Repeat("~", int(numCopies))
			if err := copyFile(objectPathname+ext, objectPathname); err != nil {
				t.logger.Println(err)
			} else {
				t.logger.Printf("Copied object: %x%s\n", hash, ext)
			}
		}
	}
}
Beispiel #12
0
func (or *ObjectsReader) nextObject() (uint64, io.ReadCloser, error) {
	or.nextIndex++
	if or.nextIndex >= int64(len(or.hashes)) {
		return 0, nil, errors.New("all objects have been consumed")
	}
	filename := path.Join(or.objectServer.baseDir,
		objectcache.HashToFilename(or.hashes[or.nextIndex]))
	file, err := os.Open(filename)
	if err != nil {
		return 0, nil, err
	}
	fi, err := file.Stat()
	if err != nil {
		file.Close()
		return 0, nil, err
	}
	return uint64(fi.Size()), file, nil
}
Beispiel #13
0
func (t *rpcType) copyFilesToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(t.objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))
		prefix := "Copied"
		if fileToCopy.DoHardlink {
			prefix = "Hardlinked"
		}
		if err := copyFile(destPathname, sourcePathname,
			fileToCopy.DoHardlink); err != nil {
			t.lastUpdateError = err
			t.logger.Println(err)
		} else {
			t.logger.Printf("%s: %s to cache\n", prefix, sourcePathname)
		}
	}
}
Beispiel #14
0
func makeRegularInode(fullPathname string,
	inode *filesystem.RegularInode, multiplyUsedObjects map[hash.Hash]uint64,
	objectsDir string, logger *log.Logger) error {
	var objectPathname string
	if inode.Size > 0 {
		objectPathname = path.Join(objectsDir,
			objectcache.HashToFilename(inode.Hash))
		numCopies := multiplyUsedObjects[inode.Hash]
		if numCopies > 1 {
			numCopies--
			objectPathname += fmt.Sprintf("~%d~", numCopies)
			if numCopies < 2 {
				delete(multiplyUsedObjects, inode.Hash)
			} else {
				multiplyUsedObjects[inode.Hash] = numCopies
			}
		}
	} else {
		objectPathname = fmt.Sprintf("%s.empty.%d", fullPathname, os.Getpid())
		if file, err := os.OpenFile(objectPathname,
			os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600); err != nil {
			return err
		} else {
			file.Close()
		}
	}
	if err := fsutil.ForceRename(objectPathname, fullPathname); err != nil {
		logger.Println(err)
		return err
	}
	if err := inode.WriteMetadata(fullPathname); err != nil {
		logger.Println(err)
		return err
	} else {
		if inode.Size > 0 {
			logger.Printf("Made inode: %s from: %x\n",
				fullPathname, inode.Hash)
		} else {
			logger.Printf("Made empty inode: %s\n", fullPathname)
		}
	}
	return nil
}
Beispiel #15
0
func (objSrv *ObjectServer) checkObject(hash hash.Hash) (uint64, error) {
	if size, ok := objSrv.sizesMap[hash]; ok {
		return size, nil
	}
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hash))
	fi, err := os.Lstat(filename)
	if err != nil {
		return 0, nil
	}
	if fi.Mode().IsRegular() {
		if fi.Size() < 1 {
			return 0, errors.New(fmt.Sprintf("zero length file: %s", filename))
		}
		size := uint64(fi.Size())
		objSrv.sizesMap[hash] = size
		return size, nil
	}
	return 0, nil
}
Beispiel #16
0
func convertToObject(pathname, objectsDir string) error {
	file, err := os.Open(pathname)
	if err != nil {
		return err
	}
	defer file.Close()
	hasher := sha512.New()
	_, err = io.Copy(hasher, file)
	if err != nil {
		return err
	}
	var hashVal hash.Hash
	copy(hashVal[:], hasher.Sum(nil))
	objPathname := path.Join(objectsDir, objectcache.HashToFilename(hashVal))
	if err = os.MkdirAll(path.Dir(objPathname), 0755); err != nil {
		return err
	}
	return fsutil.ForceRename(pathname, objPathname)
}
Beispiel #17
0
func readOne(hash hash.Hash, reader io.Reader) error {
	filename := path.Join(objectsDir, objectcache.HashToFilename(hash))
	dirname := path.Dir(filename)
	err := os.MkdirAll(dirname, syscall.S_IRUSR|syscall.S_IWUSR)
	if err != nil {
		return err
	}
	file, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	writer := bufio.NewWriter(file)
	defer writer.Flush()
	_, err = io.Copy(writer, reader)
	if err != nil {
		return errors.New(fmt.Sprintf("error copying: %s", err.Error()))
	}
	return nil
}
Beispiel #18
0
func (objSrv *ObjectServer) addObject(reader io.Reader, length uint64,
	expectedHash *hash.Hash) (hash.Hash, bool, error) {
	var hash hash.Hash
	if length < 1 {
		return hash, false, errors.New("zero length object cannot be added")
	}
	data := make([]byte, length)
	nRead, err := io.ReadFull(reader, data)
	if err != nil {
		return hash, false, err
	}
	if uint64(nRead) != length {
		return hash, false, errors.New(fmt.Sprintf(
			"failed to read data, wanted: %d, got: %d bytes", length, nRead))
	}
	hasher := sha512.New()
	if hasher.Size() != len(hash) {
		return hash, false, errors.New("Incompatible hash size")
	}
	if _, err := hasher.Write(data); err != nil {
		return hash, false, err
	}
	copy(hash[:], hasher.Sum(nil))
	if expectedHash != nil {
		if hash != *expectedHash {
			return hash, false, errors.New(fmt.Sprintf(
				"Hash mismatch. Computed=%x, expected=%x", hash, *expectedHash))
		}
	}
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hash))
	// Check for existing object and collision.
	fi, err := os.Lstat(filename)
	if err == nil {
		if !fi.Mode().IsRegular() {
			return hash, false, errors.New("Existing non-file: " + filename)
		}
		if err := collisionCheck(data, filename, fi.Size()); err != nil {
			return hash, false, errors.New("Collision detected: " + err.Error())
		}
		// No collision and no error: it's the same object. Go home early.
		return hash, false, nil
	}
	if err = os.MkdirAll(path.Dir(filename), 0755); err != nil {
		return hash, false, err
	}
	tmpFilename := filename + "~"
	file, err := os.OpenFile(tmpFilename, os.O_CREATE|os.O_WRONLY, 0660)
	if err != nil {
		return hash, false, err
	}
	defer os.Remove(tmpFilename)
	defer file.Close()
	nWritten, err := file.Write(data)
	if err != nil {
		return hash, false, err
	}
	if nWritten != len(data) {
		return hash, false, errors.New(fmt.Sprintf(
			"expected length: %d, got: %d for: %s\n",
			len(data), nWritten, tmpFilename))
	}
	objSrv.sizesMap[hash] = uint64(len(data))
	return hash, true, os.Rename(tmpFilename, filename)
}