// CreateSnapshot will create hardlinks for all tsm and tombstone files
// in the path provided
func (f *FileStore) CreateSnapshot() (string, error) {
f.traceLogger.Printf("Creating snapshot in %s", f.dir)
files := f.Files()
f.mu.Lock()
f.currentTempDirID += 1
f.mu.Unlock()
f.mu.RLock()
defer f.mu.RUnlock()
// get a tmp directory name
tmpPath := fmt.Sprintf("%s/%d.tmp", f.dir, f.currentTempDirID)
err := os.Mkdir(tmpPath, 0777)
if err != nil {
return "", err
}
for _, tsmf := range files {
newpath := filepath.Join(tmpPath, filepath.Base(tsmf.Path()))
if err := os.Link(tsmf.Path(), newpath); err != nil {
return "", fmt.Errorf("error creating tsm hard link: %q", err)
}
// Check for tombstones and link those as well
for _, tf := range tsmf.TombstoneFiles() {
newpath := filepath.Join(tmpPath, filepath.Base(tf.Path))
if err := os.Link(tf.Path, newpath); err != nil {
return "", fmt.Errorf("error creating tombstone hard link: %q", err)
}
}
}
return tmpPath, nil
}
func forceLink(oldname, newname string) error {
err := os.Link(oldname, newname)
if err == nil {
return nil
}
if os.IsPermission(err) {
// Blindly attempt to remove immutable attributes.
MakeMutable(oldname, newname)
}
return os.Link(oldname, newname)
}
// DirectoryScanner implements filepath.WalkFunc, necessary to walk and
// register each file in the download directory before beginning the
// download. This lets us know which files are already downloaded, and
// which ones can be hardlinked.
//
// Deprecated; replaced by GetAllCurrentFiles().
func DirectoryScanner(path string, f os.FileInfo, err error) error {
if f == nil { // Only exists if the directory doesn't exist beforehand.
return err
}
if f.IsDir() {
return err
}
if info, ok := FileTracker.m[f.Name()]; ok {
// File exists.
if !os.SameFile(info.FileInfo(), f) {
os.Remove(path)
err := os.Link(info.Path, path)
if err != nil {
log.Fatal(err)
}
}
} else {
// New file.
closedChannel := make(chan struct{})
close(closedChannel)
FileTracker.m[f.Name()] = FileStatus{
Name: f.Name(),
Path: path,
Priority: 0, // TODO(Liru): Add priority to file list when it is implemented
Exists: closedChannel,
}
}
return err
}
func (c *checksums) ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error) {
defer func() {
if err != nil {
logrus.Debugf("could not get checksum for %q with tar-split: %q. Attempting fallback.", id, err)
diffID, size, err = c.checksumForGraphIDNoTarsplit(id, parent, newTarDataPath)
}
}()
if oldTarDataPath == "" {
err = errors.New("no tar-split file")
return
}
tarDataFile, err := os.Open(oldTarDataPath)
if err != nil {
return
}
defer tarDataFile.Close()
uncompressed, err := gzip.NewReader(tarDataFile)
if err != nil {
return
}
dgst := digest.Canonical.New()
err = c.assembleTarTo(id, uncompressed, &size, dgst.Hash())
if err != nil {
return
}
diffID = layer.DiffID(dgst.Digest())
os.RemoveAll(newTarDataPath)
err = os.Link(oldTarDataPath, newTarDataPath)
return
}
// Deal correctly with hard links implied by matching client inode
// numbers.
func TestLinkExisting(t *testing.T) {
tc := NewTestCase(t)
defer tc.Cleanup()
c := RandomData(5)
err := ioutil.WriteFile(tc.orig+"/file1", c, 0644)
if err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
err = os.Link(tc.orig+"/file1", tc.orig+"/file2")
if err != nil {
t.Fatalf("Link failed: %v", err)
}
var s1, s2 syscall.Stat_t
err = syscall.Lstat(tc.mnt+"/file1", &s1)
if err != nil {
t.Fatalf("Lstat failed: %v", err)
}
err = syscall.Lstat(tc.mnt+"/file2", &s2)
if err != nil {
t.Fatalf("Lstat failed: %v", err)
}
if s1.Ino != s2.Ino {
t.Errorf("linked files should have identical inodes %v %v", s1.Ino, s2.Ino)
}
back, err := ioutil.ReadFile(tc.mnt + "/file1")
if err != nil {
t.Fatalf("ReadFile failed: %v", err)
}
CompareSlices(t, back, c)
}
func (config *Config) SaveConfig(path string) error {
path = os.ExpandEnv(path)
os.Remove(path + ".new")
os.Mkdir(filepath.Dir(path), 0700)
file, err := os.Create(path + ".new")
if err != nil {
return err
}
defer file.Close()
defer os.Remove(path + ".new")
data, err := yaml.Marshal(config)
if err != nil {
return err
}
if _, err := file.Write(data); err != nil {
return err
}
os.Remove(path + ".bak")
if err := os.Link(path, path+".bak"); err != nil && !os.IsNotExist(err) {
return err
}
file.Close()
os.Remove(path)
return os.Rename(path+".new", path)
}
// Try to get Lockfile lock. Returns nil, if successful and and error describing the reason, it didn't work out.
// Please note, that existing lockfiles containing pids of dead processes and lockfiles containing no pid at all
// are deleted.
func (l Lockfile) TryLock() error {
name := string(l)
// This has been checked by New already. If we trigger here,
// the caller didn't use New and re-implemented it's functionality badly.
// So panic, that he might find this easily during testing.
if !filepath.IsAbs(string(name)) {
panic(ErrNeedAbsPath)
}
tmplock, err := ioutil.TempFile(filepath.Dir(name), "")
if err != nil {
return err
} else {
defer tmplock.Close()
defer os.Remove(tmplock.Name())
}
_, err = tmplock.WriteString(fmt.Sprintf("%d\n", os.Getpid()))
if err != nil {
return err
}
// return value intentionally ignored, as ignoring it is part of the algorithm
_ = os.Link(tmplock.Name(), name)
fiTmp, err := os.Lstat(tmplock.Name())
if err != nil {
return err
}
fiLock, err := os.Lstat(name)
if err != nil {
return err
}
// Success
if os.SameFile(fiTmp, fiLock) {
return nil
}
_, err = l.GetOwner()
switch err {
default:
// Other errors -> defensively fail and let caller handle this
return err
case nil:
return ErrBusy
case ErrDeadOwner, ErrInvalidPid:
// cases we can fix below
}
// clean stale/invalid lockfile
err = os.Remove(name)
if err != nil {
return err
}
// now that we cleaned up the stale lockfile, let's recurse
return l.TryLock()
}
// Deal correctly with hard links implied by matching client inode
// numbers.
func TestLinkExisting(t *testing.T) {
me := NewTestCase(t)
defer me.Cleanup()
c := "hello"
err := ioutil.WriteFile(me.orig+"/file1", []byte(c), 0644)
CheckSuccess(err)
err = os.Link(me.orig+"/file1", me.orig+"/file2")
CheckSuccess(err)
f1, err := os.Lstat(me.mnt + "/file1")
CheckSuccess(err)
f2, err := os.Lstat(me.mnt + "/file2")
CheckSuccess(err)
if f1.Ino != f2.Ino {
t.Errorf("linked files should have identical inodes %v %v", f1.Ino, f2.Ino)
}
c1, err := ioutil.ReadFile(me.mnt + "/file1")
CheckSuccess(err)
if string(c1) != c {
t.Errorf("Content mismatch relative to original.")
}
}
func copyFile(src, dst string) (err error) {
sfi, err := os.Stat(src)
if err != nil {
return err
}
if !sfi.Mode().IsRegular() {
return fmt.Errorf("non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
}
dfi, err := os.Stat(dst)
if err != nil {
if !os.IsNotExist(err) {
return nil
}
} else {
if !(dfi.Mode().IsRegular()) {
return fmt.Errorf("non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
}
if os.SameFile(sfi, dfi) {
return nil
}
}
if err = os.Link(src, dst); err == nil {
return err
}
return copyFileContents(src, dst)
}
func copyFile(source string, target string) {
source_info, err := os.Stat(source)
if err != nil {
log.Println(err)
return
}
os.MkdirAll(filepath.Dir(target), 0666)
target_info, err := os.Stat(target)
if err != nil {
os.Link(source, target)
} else if !os.SameFile(source_info, target_info) {
unsetReadonly(target)
os.Remove(target)
os.Link(source, target)
}
}
func TestMemUnionFsLink(t *testing.T) {
wd, _, clean := setupMemUfs(t)
defer clean()
content := "blabla"
fn := wd + "/ro/file"
err := ioutil.WriteFile(fn, []byte(content), 0666)
CheckSuccess(err)
err = os.Link(wd+"/mnt/file", wd+"/mnt/linked")
CheckSuccess(err)
var st2 syscall.Stat_t
err = syscall.Lstat(wd+"/mnt/linked", &st2)
CheckSuccess(err)
var st1 syscall.Stat_t
err = syscall.Lstat(wd+"/mnt/file", &st1)
CheckSuccess(err)
if st1.Ino != st2.Ino {
t.Errorf("inode numbers should be equal for linked files %v, %v", st1.Ino, st2.Ino)
}
c, err := ioutil.ReadFile(wd + "/mnt/linked")
if string(c) != content {
t.Errorf("content mismatch got %q want %q", string(c), content)
}
}
func (ln *SomeLn) Exec(inPipe io.Reader, outPipe io.Writer, errPipe io.Writer) error {
if ln.IsSymbolic {
return os.Symlink(ln.target, ln.linkName)
} else {
return os.Link(ln.target, ln.linkName)
}
}
// Deal correctly with hard links implied by matching client inode
// numbers.
func TestLinkExisting(t *testing.T) {
tc := NewTestCase(t)
defer tc.Cleanup()
c := "hello"
err := ioutil.WriteFile(tc.orig+"/file1", []byte(c), 0644)
CheckSuccess(err)
err = os.Link(tc.orig+"/file1", tc.orig+"/file2")
CheckSuccess(err)
var s1, s2 syscall.Stat_t
err = syscall.Lstat(tc.mnt+"/file1", &s1)
CheckSuccess(err)
err = syscall.Lstat(tc.mnt+"/file2", &s2)
CheckSuccess(err)
if s1.Ino != s2.Ino {
t.Errorf("linked files should have identical inodes %v %v", s1.Ino, s2.Ino)
}
c1, err := ioutil.ReadFile(tc.mnt + "/file1")
CheckSuccess(err)
if string(c1) != c {
t.Errorf("Content mismatch relative to original.")
}
}
// GetAllCurrentFiles scans the download directory and parses the files inside
// for possible future linking, if a duplicate is found.
func GetAllCurrentFiles() {
os.MkdirAll(cfg.DownloadDirectory, 0755)
dirs, err := ioutil.ReadDir(cfg.DownloadDirectory)
if err != nil {
panic(err)
}
// TODO: Make GetAllCurrentFiles a LOT more stable. A lot could go wrong, but meh.
for _, d := range dirs {
if !d.IsDir() {
continue
}
dir, err := os.Open(cfg.DownloadDirectory + string(os.PathSeparator) + d.Name())
if err != nil {
log.Fatal(err)
}
// fmt.Println(dir.Name())
files, err := dir.Readdirnames(0)
if err != nil {
log.Fatal(err)
}
for _, f := range files {
if info, ok := FileTracker.m[f]; ok {
// File exists.
p := dir.Name() + string(os.PathSeparator) + f
checkFile, err := os.Stat(p)
if err != nil {
log.Fatal(err)
}
if !os.SameFile(info.FileInfo(), checkFile) {
os.Remove(p)
err := os.Link(info.Path, p)
if err != nil {
log.Fatal(err)
}
}
} else {
// New file.
closedChannel := make(chan struct{})
close(closedChannel)
FileTracker.m[f] = FileStatus{
Name: f,
Path: dir.Name() + string(os.PathSeparator) + f,
Priority: 0, // TODO(Liru): Add priority to file list when it is implemented
Exists: closedChannel,
}
}
}
}
}
func (me *testCase) testLink() {
me.tester.Log("Testing hard links.")
me.writeOrigFile()
err := os.Mkdir(me.origSubdir, 0777)
CheckSuccess(err)
// Link.
err = os.Link(me.mountFile, me.mountSubfile)
CheckSuccess(err)
fi, err := os.Lstat(me.mountFile)
if fi.Nlink != 2 {
me.tester.Errorf("Expect 2 links: %v", fi)
}
f, err := os.Open(me.mountSubfile)
var buf [1024]byte
slice := buf[:]
n, err := f.Read(slice)
f.Close()
strContents := string(slice[:n])
if strContents != contents {
me.tester.Errorf("Content error: %v", slice[:n])
}
me.removeMountSubdir()
me.removeMountFile()
}
func main() {
goopt.Suite = "XQZ coreutils"
goopt.Author = "William Pearson"
goopt.Version = "Link v0.1"
goopt.Summary = "Creates a link to FILE1 called FILE2"
goopt.Usage = func() string {
return fmt.Sprintf("Usage:\t%s FILE1 FILE2\n or:\t%s OPTION\n", os.Args[0], os.Args[0]) + goopt.Summary + "\n\n" + goopt.Help()
}
goopt.Description = func() string {
return goopt.Summary + "\n\nUnless an option is passed to it."
}
goopt.NoArg([]string{"-v", "--version"}, "outputs version information and exits", version)
goopt.Parse(nil)
switch {
case len(os.Args) == 1:
fmt.Println("Missing filenames")
case len(os.Args) == 2:
fmt.Println("Missing filename after '%s'", os.Args[1])
case len(os.Args) > 3:
fmt.Println("Too many filenames")
}
if len(os.Args) != 3 {
os.Exit(1)
}
file1 := os.Args[1]
file2 := os.Args[2]
if err := os.Link(file1, file2); err != nil {
fmt.Println("Encountered an error during linking: %v", err)
os.Exit(1)
}
return
}
// CopyFile copies a file from src to dst. If src and dst files exist, and are
// the same, then return success. Otherise, attempt to create a hard link
// between the two files. If that fail, copy the file contents from src to dst.
func CopyFile(src, dst string) (err error) {
sfi, err := os.Stat(src)
if err != nil {
return
}
if !sfi.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
}
dfi, err := os.Stat(dst)
if err != nil {
if !os.IsNotExist(err) {
return
}
} else {
if !(dfi.Mode().IsRegular()) {
return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
}
if os.SameFile(sfi, dfi) {
return
}
}
if err = os.Link(src, dst); err == nil {
return
}
err = copyFileContents(src, dst)
return
}
func copy(src string, dst string, fi os.FileInfo, sc chan sem) error {
mod := fi.Mode()
switch {
case mod&os.ModeSymlink != 0:
if tgt, err := os.Readlink(src); err != nil {
return err
} else {
if err := os.Symlink(tgt, dst); err != nil {
return err
}
}
case mod&os.ModeDir != 0:
if err := os.Mkdir(dst, mod); err != nil {
return err
}
if err := copyEntries(src, dst, sc); err != nil {
return err
}
default:
if err := os.Link(src, dst); err != nil {
return err
}
}
return nil
}
func (k *KVFiles) Put(key, value []byte) error {
tmp, err := ioutil.TempFile(k.path, "put-")
if err != nil {
return err
}
defer func() {
// silence errcheck
_ = os.Remove(tmp.Name())
}()
_, err = tmp.Write(value)
if err != nil {
return err
}
path := path.Join(k.path, hex.EncodeToString(key)+".data")
err = os.Link(tmp.Name(), path)
if err != nil {
// EEXIST is safe to ignore here, that just means we
// successfully de-duplicated content
if !os.IsExist(err) {
return err
}
}
return nil
}
func deduplicate(f sameFile) error {
by_dev := map[uint64][]int{}
for i, dev := range f.devices {
by_dev[dev] = append(by_dev[dev], i)
}
for _, file_list := range by_dev {
if len(file_list) <= 1 {
continue
}
first_file := file_list[0]
for _, cur_file := range file_list[1:] {
if f.inodes[first_file] == f.inodes[cur_file] {
continue
}
err := os.Remove(f.paths[cur_file])
if err != nil {
return err
}
err = os.Link(f.paths[first_file], f.paths[cur_file])
if err != nil {
panic(fmt.Errorf("Could not link identical file '%s' to '%s': %s", f.paths[first_file], f.paths[cur_file], err.Error()))
}
}
}
return nil
}
func migrateSnapshots(legacySnapDir, snapDir string) error {
// use temporary snaphot directory so initialization appears atomic
tmpdirpath := filepath.Clean(snapDir) + ".tmp"
if fileutil.Exist(tmpdirpath) {
if err := os.RemoveAll(tmpdirpath); err != nil {
return errors.Wrap(err, "could not remove temporary snapshot directory")
}
}
if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
return errors.Wrap(err, "could not create temporary snapshot directory")
}
snapshotNames, err := fileutil.ReadDir(legacySnapDir)
if err != nil {
return errors.Wrapf(err, "could not list snapshot directory %s", legacySnapDir)
}
for _, fname := range snapshotNames {
err := os.Link(filepath.Join(legacySnapDir, fname), filepath.Join(tmpdirpath, fname))
if err != nil {
return errors.Wrap(err, "error linking snapshot file")
}
}
if err := os.Rename(tmpdirpath, snapDir); err != nil {
return err
}
return nil
}
func AtomicReplaceLink(from, target string) error {
newpath := from + ".replace.tmp"
if err := os.Link(from, newpath); err != nil {
return err
}
return os.Rename(newpath, target)
}
//TODO use exec and 'mklink.exe'
func makeLink(target, linkName string, options LnOptions) error {
if options.IsSymbolic {
return os.Symlink(target, linkName)
} else {
return os.Link(target, linkName)
}
}
func (w *baseLayerWriter) AddLink(name string, target string) (err error) {
defer func() {
if err != nil {
w.err = err
}
}()
err = w.closeCurrentFile()
if err != nil {
return err
}
linkpath, err := makeLongAbsPath(filepath.Join(w.root, name))
if err != nil {
return err
}
linktarget, err := makeLongAbsPath(filepath.Join(w.root, target))
if err != nil {
return err
}
return winio.RunWithPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}, func() (err error) {
return os.Link(linktarget, linkpath)
})
}
// Deal correctly with hard links implied by matching client inode
// numbers.
func TestLinkExisting(t *testing.T) {
tc := NewTestCase(t)
defer tc.Cleanup()
c := RandomData(5)
err := ioutil.WriteFile(tc.orig+"/file1", c, 0644)
CheckSuccess(err)
err = os.Link(tc.orig+"/file1", tc.orig+"/file2")
CheckSuccess(err)
var s1, s2 syscall.Stat_t
err = syscall.Lstat(tc.mnt+"/file1", &s1)
CheckSuccess(err)
err = syscall.Lstat(tc.mnt+"/file2", &s2)
CheckSuccess(err)
if s1.Ino != s2.Ino {
t.Errorf("linked files should have identical inodes %v %v", s1.Ino, s2.Ino)
}
back, err := ioutil.ReadFile(tc.mnt + "/file1")
CheckSuccess(err)
CompareSlices(t, back, c)
}
func LinkRecursive(from, to string) error {
st, err := os.Lstat(from)
if err != nil {
return err
}
if st.IsDir() {
err := os.MkdirAll(to, st.Mode())
if err != nil {
return err
}
d, err := os.Open(from)
if err != nil {
return err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, n := range names {
err := LinkRecursive(filepath.Join(from, n), filepath.Join(to, n))
if err != nil {
return err
}
}
return nil
} else {
return os.Link(from, to)
}
}
// Deal correctly with hard links implied by matching client inode
// numbers.
func TestLinkForget(t *testing.T) {
tc := NewTestCase(t)
defer tc.Cleanup()
c := "hello"
err := ioutil.WriteFile(tc.orig+"/file1", []byte(c), 0644)
if err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
err = os.Link(tc.orig+"/file1", tc.orig+"/file2")
if err != nil {
t.Fatalf("Link failed: %v", err)
}
var s1, s2 syscall.Stat_t
err = syscall.Lstat(tc.mnt+"/file1", &s1)
if err != nil {
t.Fatalf("Lstat failed: %v", err)
}
tc.pathFs.ForgetClientInodes()
err = syscall.Lstat(tc.mnt+"/file2", &s2)
if err != nil {
t.Fatalf("Lstat failed: %v", err)
}
if s1.Ino == s2.Ino {
t.Error("After forget, we should not export links")
}
}
func TestRepoIndexCmd(t *testing.T) {
dir, err := ioutil.TempDir("", "helm-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
if err := os.Link("testdata/testcharts/compressedchart-0.1.0.tgz", filepath.Join(dir, "compressedchart-0.1.0.tgz")); err != nil {
t.Fatal(err)
}
buf := bytes.NewBuffer(nil)
c := newRepoIndexCmd(buf)
if err := c.RunE(c, []string{dir}); err != nil {
t.Errorf("%q", err)
}
index, err := repo.LoadIndexFile(filepath.Join(dir, "index.yaml"))
if err != nil {
t.Fatal(err)
}
if len(index.Entries) != 1 {
t.Errorf("expected 1 entry, got %v: %#v", len(index.Entries), index.Entries)
}
}
func TestUnionFsLink(t *testing.T) {
wd, clean := setupUfs(t)
defer clean()
content := "blabla"
fn := wd + "/ro/file"
err := ioutil.WriteFile(fn, []byte(content), 0666)
CheckSuccess(err)
freezeRo(wd + "/ro")
err = os.Link(wd+"/mnt/file", wd+"/mnt/linked")
CheckSuccess(err)
fi2, err := os.Lstat(wd + "/mnt/linked")
CheckSuccess(err)
fi1, err := os.Lstat(wd + "/mnt/file")
CheckSuccess(err)
if fi1.Ino != fi2.Ino {
t.Errorf("inode numbers should be equal for linked files %v, %v", fi1.Ino, fi2.Ino)
}
c, err := ioutil.ReadFile(wd + "/mnt/linked")
if string(c) != content {
t.Errorf("content mismatch got %q want %q", string(c), content)
}
}
func CopyFile(srcPath string, destPath string) error {
srcFile, err := os.Stat(srcPath)
if err != nil {
return err
}
if !srcFile.Mode().IsRegular() {
return fmt.Errorf("Unregular source file: %s (%s)", srcFile.Name(), srcFile.Mode().String())
}
destFile, err := os.Stat(destPath)
if err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
if !(destFile.Mode().IsRegular()) {
return fmt.Errorf("Unregular source file: %s (%s)", srcFile.Name(), srcFile.Mode().String())
}
if os.SameFile(srcFile, destFile) {
return nil
}
}
if err := os.Link(srcPath, destPath); err == nil {
return err
}
return copyFileContents(srcPath, destPath)
}