diff --git a/go.mod b/go.mod index 50d0a7ec..d3bbdd4d 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/stretchr/testify v1.8.2 golang.org/x/crypto v0.45.0 golang.org/x/sync v0.18.0 - golang.org/x/sys v0.38.0 // indirect + golang.org/x/sys v0.38.0 google.golang.org/api v0.116.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 ) diff --git a/nullseed.go b/nullseed.go index c5c108d3..31ff9e5d 100644 --- a/nullseed.go +++ b/nullseed.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "os" "path/filepath" ) @@ -16,7 +15,7 @@ type nullChunkSeed struct { } func newNullChunkSeed(dstFile string, blocksize uint64, max uint64) (*nullChunkSeed, error) { - blockfile, err := ioutil.TempFile(filepath.Dir(dstFile), ".tmp-block") + blockfile, err := os.CreateTemp(filepath.Dir(dstFile), ".tmp-block") if err != nil { return nil, err } @@ -47,12 +46,17 @@ func (s *nullChunkSeed) LongestMatchWith(chunks []IndexChunk) (int, SeedSegment) if len(chunks) == 0 { return 0, nil } - // No limit needed: when isBlank=true, WriteInto skips without copying. - // When isBlank=false, we must still write zeros to overwrite stale data. - // The previous limit of 100 caused chunks beyond the limit to fall - // through to other code paths, leading to incorrect assembly. - var n int + var ( + n int + limit int + ) + if !s.canReflink { + limit = 100 + } for _, c := range chunks { + if limit != 0 && limit == n { + break + } if c.ID != s.id { break } @@ -104,7 +108,7 @@ func (s *nullChunkSection) WriteInto(dst *os.File, offset, length, blocksize uin return 0, 0, fmt.Errorf("unable to copy %d bytes to %s : wrong size", length, dst.Name()) } - // When cloning isn'a available we'd normally have to copy the 0 bytes into + // When cloning isn't available we'd normally have to copy the 0 bytes into // the target range. But if that's already blank (because it's a new/truncated // file) there's no need to copy 0 bytes. if !s.canReflink { @@ -117,7 +121,7 @@ func (s *nullChunkSection) WriteInto(dst *os.File, offset, length, blocksize uin } func (s *nullChunkSection) copy(dst *os.File, offset, length uint64) (uint64, uint64, error) { - if _, err := dst.Seek(int64(offset), os.SEEK_SET); err != nil { + if _, err := dst.Seek(int64(offset), io.SeekStart); err != nil { return 0, 0, err } // Copy using a fixed buffer. Using io.Copy() with a LimitReader will make it diff --git a/preallocate_darwin.go b/preallocate_darwin.go index b249d6c2..226b9c37 100644 --- a/preallocate_darwin.go +++ b/preallocate_darwin.go @@ -3,24 +3,11 @@ package desync import ( + "errors" "fmt" "os" - "syscall" - "unsafe" -) - -type fstore_t struct { - Flags uint32 - Posmode int32 - Offset int64 - Length int64 - Bytesalloc int64 -} -const ( - fAllocateAll = 0x00000004 - fPeofPosmode = 3 - fPreallocate = 42 + "golang.org/x/sys/unix" ) // preallocateFile physically allocates disk blocks and sets the file size. @@ -35,18 +22,29 @@ func preallocateFile(name string, size int64) error { } defer f.Close() - store := fstore_t{ - Flags: fAllocateAll, - Posmode: fPeofPosmode, - Offset: 0, - Length: size, + info, err := f.Stat() + if err != nil { + return err } - _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, - uintptr(f.Fd()), - uintptr(fPreallocate), - uintptr(unsafe.Pointer(&store))) - if errno != 0 { - return fmt.Errorf("F_PREALLOCATE: %w", errno) + + // F_PREALLOCATE with F_PEOFPOSMODE allocates relative to the current + // end of file, so only request the difference. Nothing to allocate if + // the file is already large enough or no growth is needed. + if extra := size - info.Size(); extra > 0 { + store := unix.Fstore_t{ + Flags: unix.F_ALLOCATEALL, + Posmode: unix.F_PEOFPOSMODE, + Offset: 0, + Length: extra, + } + if err := unix.FcntlFstore(f.Fd(), unix.F_PREALLOCATE, &store); err != nil { + // Not all filesystems support F_PREALLOCATE (e.g. SMB or FUSE + // mounts). The sparse-hole issue is specific to APFS, so fall + // back to a plain truncate there. + if !errors.Is(err, unix.ENOTSUP) { + return fmt.Errorf("F_PREALLOCATE %s: %w", name, err) + } + } } return f.Truncate(size) diff --git a/preallocate_darwin_test.go b/preallocate_darwin_test.go new file mode 100644 index 00000000..da610b6d --- /dev/null +++ b/preallocate_darwin_test.go @@ -0,0 +1,48 @@ +//go:build darwin + +package desync + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// TestPreallocateTightDisk re-preallocates an existing file to its current +// size on a nearly-full volume. This must not require additional disk +// space: F_PREALLOCATE with F_PEOFPOSMODE allocates relative to the +// existing end of file, so requesting the full size instead of only the +// missing difference over-allocates and fails with ENOSPC. +func TestPreallocateTightDisk(t *testing.T) { + if _, err := exec.LookPath("hdiutil"); err != nil { + t.Skip("hdiutil not available") + } + dir := t.TempDir() + img := filepath.Join(dir, "small.dmg") + mount := filepath.Join(dir, "mnt") + require.NoError(t, os.Mkdir(mount, 0755)) + + out, err := exec.Command("hdiutil", "create", "-size", "32m", "-fs", "APFS", "-volname", "desync-test", img).CombinedOutput() + require.NoError(t, err, string(out)) + out, err = exec.Command("hdiutil", "attach", img, "-mountpoint", mount, "-nobrowse").CombinedOutput() + require.NoError(t, err, string(out)) + t.Cleanup(func() { _ = exec.Command("hdiutil", "detach", mount, "-force").Run() }) + + var st unix.Statfs_t + require.NoError(t, unix.Statfs(mount, &st)) + free := int64(st.Bavail) * int64(st.Bsize) + + // Fill most of the volume with an existing file, leaving less free + // space than the size of the file itself + size := free * 2 / 3 + name := filepath.Join(mount, "existing") + require.NoError(t, os.WriteFile(name, bytes.Repeat([]byte{0xab}, int(size)), 0666)) + + // The file already has the right size, nothing should be allocated + require.NoError(t, preallocateFile(name, size)) +} diff --git a/preallocate_other.go b/preallocate_other.go index abc7b303..a211981f 100644 --- a/preallocate_other.go +++ b/preallocate_other.go @@ -4,10 +4,15 @@ package desync import "os" -// preallocateFile truncates the file to the given size. -// On Linux (ext4) and other platforms, Truncate produces a file that -// reads back as zeros without sparse-hole issues, so no special -// preallocation is needed. +// preallocateFile truncates the file to the given size, creating it if +// it doesn't exist. On Linux (ext4) and other platforms, Truncate +// produces a file that reads back as zeros without sparse-hole issues, +// so no special preallocation is needed. func preallocateFile(name string, size int64) error { - return os.Truncate(name, size) + f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return err + } + defer f.Close() + return f.Truncate(size) } diff --git a/preallocate_test.go b/preallocate_test.go new file mode 100644 index 00000000..0d2c12b7 --- /dev/null +++ b/preallocate_test.go @@ -0,0 +1,70 @@ +package desync + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPreallocateNewFile(t *testing.T) { + name := filepath.Join(t.TempDir(), "new") + + require.NoError(t, preallocateFile(name, 2*1024*1024)) + + b, err := os.ReadFile(name) + require.NoError(t, err) + require.Len(t, b, 2*1024*1024) + require.Equal(t, make([]byte, 2*1024*1024), b) +} + +func TestPreallocateGrowExistingFile(t *testing.T) { + name := filepath.Join(t.TempDir(), "grow") + data := bytes.Repeat([]byte{0xab}, 4096) + require.NoError(t, os.WriteFile(name, data, 0666)) + + require.NoError(t, preallocateFile(name, 64*1024)) + + b, err := os.ReadFile(name) + require.NoError(t, err) + require.Len(t, b, 64*1024) + // The original content must be preserved and the new region read as zeros + require.Equal(t, data, b[:len(data)]) + require.Equal(t, make([]byte, 64*1024-len(data)), b[len(data):]) +} + +func TestPreallocateShrinkExistingFile(t *testing.T) { + name := filepath.Join(t.TempDir(), "shrink") + data := bytes.Repeat([]byte{0xab}, 64*1024) + require.NoError(t, os.WriteFile(name, data, 0666)) + + require.NoError(t, preallocateFile(name, 4096)) + + b, err := os.ReadFile(name) + require.NoError(t, err) + require.Equal(t, data[:4096], b) +} + +func TestPreallocateSameSize(t *testing.T) { + name := filepath.Join(t.TempDir(), "same") + data := bytes.Repeat([]byte{0xab}, 4096) + require.NoError(t, os.WriteFile(name, data, 0666)) + + require.NoError(t, preallocateFile(name, int64(len(data)))) + + b, err := os.ReadFile(name) + require.NoError(t, err) + require.Equal(t, data, b) +} + +func TestPreallocateEmptyFile(t *testing.T) { + name := filepath.Join(t.TempDir(), "empty") + + require.NoError(t, preallocateFile(name, 0)) + + info, err := os.Stat(name) + require.NoError(t, err) + require.Zero(t, info.Size()) +}