Fix F_PREALLOCATE delta calculation, use x/sys/unix, keep nullseed limit

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/stretchr/testify v1.8.2
 	golang.org/x/crypto v0.45.0
 	golang.org/x/sync v0.18.0
-	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/sys v0.38.0
 	google.golang.org/api v0.116.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 )

nullseed.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 )
@@ -16,7 +15,7 @@ type nullChunkSeed struct {
 }
 
 func newNullChunkSeed(dstFile string, blocksize uint64, max uint64) (*nullChunkSeed, error) {
-	blockfile, err := ioutil.TempFile(filepath.Dir(dstFile), ".tmp-block")
+	blockfile, err := os.CreateTemp(filepath.Dir(dstFile), ".tmp-block")
 	if err != nil {
 		return nil, err
 	}
@@ -47,12 +46,17 @@ func (s *nullChunkSeed) LongestMatchWith(chunks []IndexChunk) (int, SeedSegment)
 	if len(chunks) == 0 {
 		return 0, nil
 	}
-	// No limit needed: when isBlank=true, WriteInto skips without copying.
-	// When isBlank=false, we must still write zeros to overwrite stale data.
-	// The previous limit of 100 caused chunks beyond the limit to fall
-	// through to other code paths, leading to incorrect assembly.
-	var n int
+	var (
+		n     int
+		limit int
+	)
+	if !s.canReflink {
+		limit = 100
+	}
 	for _, c := range chunks {
+		if limit != 0 && limit == n {
+			break
+		}
 		if c.ID != s.id {
 			break
 		}
@@ -104,7 +108,7 @@ func (s *nullChunkSection) WriteInto(dst *os.File, offset, length, blocksize uin
 		return 0, 0, fmt.Errorf("unable to copy %d bytes to %s : wrong size", length, dst.Name())
 	}
 
-	// When cloning isn'a available we'd normally have to copy the 0 bytes into
+	// When cloning isn't available we'd normally have to copy the 0 bytes into
 	// the target range. But if that's already blank (because it's a new/truncated
 	// file) there's no need to copy 0 bytes.
 	if !s.canReflink {
@@ -117,7 +121,7 @@ func (s *nullChunkSection) WriteInto(dst *os.File, offset, length, blocksize uin
 }
 
 func (s *nullChunkSection) copy(dst *os.File, offset, length uint64) (uint64, uint64, error) {
-	if _, err := dst.Seek(int64(offset), os.SEEK_SET); err != nil {
+	if _, err := dst.Seek(int64(offset), io.SeekStart); err != nil {
 		return 0, 0, err
 	}
 	// Copy using a fixed buffer. Using io.Copy() with a LimitReader will make it

preallocate_darwin.go

@@ -3,24 +3,11 @@
 package desync
 
 import (
+	"errors"
 	"fmt"
 	"os"
-	"syscall"
-	"unsafe"
-)
-
-type fstore_t struct {
-	Flags      uint32
-	Posmode    int32
-	Offset     int64
-	Length     int64
-	Bytesalloc int64
-}
 
-const (
-	fAllocateAll = 0x00000004
-	fPeofPosmode = 3
-	fPreallocate = 42
+	"golang.org/x/sys/unix"
 )
 
 // preallocateFile physically allocates disk blocks and sets the file size.
@@ -35,18 +22,29 @@ func preallocateFile(name string, size int64) error {
 	}
 	defer f.Close()
 
-	store := fstore_t{
-		Flags:   fAllocateAll,
-		Posmode: fPeofPosmode,
-		Offset:  0,
-		Length:  size,
+	info, err := f.Stat()
+	if err != nil {
+		return err
 	}
-	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL,
-		uintptr(f.Fd()),
-		uintptr(fPreallocate),
-		uintptr(unsafe.Pointer(&store)))
-	if errno != 0 {
-		return fmt.Errorf("F_PREALLOCATE: %w", errno)
+
+	// F_PREALLOCATE with F_PEOFPOSMODE allocates relative to the current
+	// end of file, so only request the difference. Nothing to allocate if
+	// the file is already large enough or no growth is needed.
+	if extra := size - info.Size(); extra > 0 {
+		store := unix.Fstore_t{
+			Flags:   unix.F_ALLOCATEALL,
+			Posmode: unix.F_PEOFPOSMODE,
+			Offset:  0,
+			Length:  extra,
+		}
+		if err := unix.FcntlFstore(f.Fd(), unix.F_PREALLOCATE, &store); err != nil {
+			// Not all filesystems support F_PREALLOCATE (e.g. SMB or FUSE
+			// mounts). The sparse-hole issue is specific to APFS, so fall
+			// back to a plain truncate there.
+			if !errors.Is(err, unix.ENOTSUP) {
+				return fmt.Errorf("F_PREALLOCATE %s: %w", name, err)
+			}
+		}
 	}
 
 	return f.Truncate(size)

preallocate_darwin_test.go

@@ -0,0 +1,48 @@
+//go:build darwin
+
+package desync
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// TestPreallocateTightDisk re-preallocates an existing file to its current
+// size on a nearly-full volume. This must not require additional disk
+// space: F_PREALLOCATE with F_PEOFPOSMODE allocates relative to the
+// existing end of file, so requesting the full size instead of only the
+// missing difference over-allocates and fails with ENOSPC.
+func TestPreallocateTightDisk(t *testing.T) {
+	if _, err := exec.LookPath("hdiutil"); err != nil {
+		t.Skip("hdiutil not available")
+	}
+	dir := t.TempDir()
+	img := filepath.Join(dir, "small.dmg")
+	mount := filepath.Join(dir, "mnt")
+	require.NoError(t, os.Mkdir(mount, 0755))
+
+	out, err := exec.Command("hdiutil", "create", "-size", "32m", "-fs", "APFS", "-volname", "desync-test", img).CombinedOutput()
+	require.NoError(t, err, string(out))
+	out, err = exec.Command("hdiutil", "attach", img, "-mountpoint", mount, "-nobrowse").CombinedOutput()
+	require.NoError(t, err, string(out))
+	t.Cleanup(func() { _ = exec.Command("hdiutil", "detach", mount, "-force").Run() })
+
+	var st unix.Statfs_t
+	require.NoError(t, unix.Statfs(mount, &st))
+	free := int64(st.Bavail) * int64(st.Bsize)
+
+	// Fill most of the volume with an existing file, leaving less free
+	// space than the size of the file itself
+	size := free * 2 / 3
+	name := filepath.Join(mount, "existing")
+	require.NoError(t, os.WriteFile(name, bytes.Repeat([]byte{0xab}, int(size)), 0666))
+
+	// The file already has the right size, nothing should be allocated
+	require.NoError(t, preallocateFile(name, size))
+}

preallocate_other.go

@@ -4,10 +4,15 @@ package desync
 
 import "os"
 
-// preallocateFile truncates the file to the given size.
-// On Linux (ext4) and other platforms, Truncate produces a file that
-// reads back as zeros without sparse-hole issues, so no special
-// preallocation is needed.
+// preallocateFile truncates the file to the given size, creating it if
+// it doesn't exist. On Linux (ext4) and other platforms, Truncate
+// produces a file that reads back as zeros without sparse-hole issues,
+// so no special preallocation is needed.
 func preallocateFile(name string, size int64) error {
-	return os.Truncate(name, size)
+	f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return f.Truncate(size)
 }

preallocate_test.go

@@ -0,0 +1,70 @@
+package desync
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPreallocateNewFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "new")
+
+	require.NoError(t, preallocateFile(name, 2*1024*1024))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Len(t, b, 2*1024*1024)
+	require.Equal(t, make([]byte, 2*1024*1024), b)
+}
+
+func TestPreallocateGrowExistingFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "grow")
+	data := bytes.Repeat([]byte{0xab}, 4096)
+	require.NoError(t, os.WriteFile(name, data, 0666))
+
+	require.NoError(t, preallocateFile(name, 64*1024))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Len(t, b, 64*1024)
+	// The original content must be preserved and the new region read as zeros
+	require.Equal(t, data, b[:len(data)])
+	require.Equal(t, make([]byte, 64*1024-len(data)), b[len(data):])
+}
+
+func TestPreallocateShrinkExistingFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "shrink")
+	data := bytes.Repeat([]byte{0xab}, 64*1024)
+	require.NoError(t, os.WriteFile(name, data, 0666))
+
+	require.NoError(t, preallocateFile(name, 4096))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Equal(t, data[:4096], b)
+}
+
+func TestPreallocateSameSize(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "same")
+	data := bytes.Repeat([]byte{0xab}, 4096)
+	require.NoError(t, os.WriteFile(name, data, 0666))
+
+	require.NoError(t, preallocateFile(name, int64(len(data))))
+
+	b, err := os.ReadFile(name)
+	require.NoError(t, err)
+	require.Equal(t, data, b)
+}
+
+func TestPreallocateEmptyFile(t *testing.T) {
+	name := filepath.Join(t.TempDir(), "empty")
+
+	require.NoError(t, preallocateFile(name, 0))
+
+	info, err := os.Stat(name)
+	require.NoError(t, err)
+	require.Zero(t, info.Size())
+}