Draft: Identify macOS UNIX-specific system directories

src/HiddenFiles.jl

@@ -55,27 +55,56 @@ include("path.jl")
         # https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
         const UF_HIDDEN = 0x00008000
 
+        struct BSDTimeSpec
+            sec::Int64
+            nsec::Int64
+        end
+
+        # See HiddenFiles.jl#14
+        struct BSDStatStruct
+            st_dev::UInt64
+            st_mode::UInt64
+            st_nlink::UInt64
+            st_uid::UInt64
+            st_gid::UInt64
+            st_rdev::UInt64
+            st_ino::UInt64
+            st_size::UInt64
+            st_blksize::UInt64
+            st_blocks::UInt64
+            st_flags::UInt64
+            st_gen::UInt64
+            st_atime::BSDTimeSpec
+            st_mtim::BSDTimeSpec
+            st_ctim::BSDTimeSpec
+            st_birthtim::BSDTimeSpec
+
+            # https://discourse.julialang.org/t/106893/6
+            # https://discourse.julialang.org/t/75835/6
+            # BSDStatStruct() = new()
+        end
+
         # https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/stat.2.html
         # http://docs.libuv.org/en/v1.x/fs.html#c.uv_stat_t
-        const ST_FLAGS_STAT_OFFSET = 0x15
+        # https://discourse.julialang.org/t/75835/5
+        #
+        # See previous version of _st_flags at f149f1a
         function _st_flags(f::AbstractString)
-            statbuf = Vector{UInt32}(undef, ccall(:jl_sizeof_stat, Int32, ()))
-
+            # Note: sizeof(BSDStatStruct) must equal ccall(:jl_sizeof_stat, Int32, ())
+            stat_t_ref = Ref{BSDStatStruct}()
             # int stat(const char *restrict path, struct stat *restrict buf);
             # int stat(const char * restrict path, struct stat * restrict sb);
-            i = ccall(:jl_stat, Int32, (Cstring, Ptr{Cvoid}), f, statbuf)
+            i = ccall(:jl_lstat, Int32, (Cstring, Ptr{BSDStatStruct}), f, stat_t_ref)
             iszero(i) || Base.uv_error("_st_flags($(repr(f)))", i)
-
-            # st_flags offset is at index 11, or 21 in 32-bit
-            return statbuf[ST_FLAGS_STAT_OFFSET]
+            stat_t = stat_t_ref.x
+            return stat_t.st_flags
         end
 
         # https://github.com/dotnet/runtime/blob/5992145db2cb57956ee444aa0f0c2f3f85ee3673/src/native/libs/System.Native/pal_io.c#L219
         # https://github.com/davidkaya/corefx/blob/4fd3d39f831f3e14f311b0cdc0a33d662e684a9c/src/System.IO.FileSystem/src/System/IO/FileStatus.Unix.cs#L88
-        _isinvisible(f::AbstractString) = (_st_flags(f) & UF_HIDDEN) == UF_HIDDEN
-
-        _ishidden_bsd_related(ps::PathStruct) =
-            _ishidden_unix(ps) || _isinvisible(ps.realpath)
+        _isinvisible_st_flags(f::AbstractString) = (_st_flags(f) & UF_HIDDEN) == UF_HIDDEN
+        _isinvisible_st_flags(ps::PathStruct) = _isinvisible_st_flags(ps.realpath)
+        _ishidden_bsd_related(ps::PathStruct) = _ishidden_unix(ps) || _isinvisible_st_flags(ps)
     end
 
     @static if Sys.isapple()  # macOS/Darwin
@@ -108,7 +137,18 @@ include("path.jl")
         #   - `/var`—Contains log files and other files whose content is variable. (Log
         #            files are typically viewed using the Console app.)
         # TODO
-        _issystemfile(f::AbstractString) = false
+        _issystemdir(f::AbstractString) = false
+        _issystemdir(ps::PathStruct) = _issystemdir(ps.realpath)
+
+        # This _isinvisible function seems to capture some cases (e.g., `/tmp`) that the other _isinvisible function does not
+        function _isinvisible_macos_item_info(f::AbstractString, str_encoding::Unsigned = CF_STRING_ENCODING, path_style::Integer = K_CF_URL_POSIX_PATH_STYLE)
+            cfstr = _cfstring_create_with_cstring(f, str_encoding)
+            url_ref = _cf_url_create_with_file_system_path(cfstr, isdir(f))
+            item_info = _ls_copy_item_info_for_url(url_ref, K_IS_INVISIBLE)
+            return !iszero(item_info[1] & K_IS_INVISIBLE)
+        end
+        _isinvisible_macos_item_info(ps::PathStruct, str_encoding::Unsigned = CF_STRING_ENCODING, path_style::Integer = K_CF_URL_POSIX_PATH_STYLE) =
+            _isinvisible_macos_item_info(ps.realpath, str_encoding, path_style)
 
         #=== Case 3: Explicitly hidden files and directories ===#
         # The Finder may hide specific files or directories that should not be accessed
@@ -117,7 +157,9 @@ include("path.jl")
         # from the command line.  (The Finder provides a different user interface for
         # accessing local disks.)  In macOS 10.7 and later, the Finder also hides the
         # `~/Library` directory—that is, the `Library` directory located in the user’s
-        # home directory.  This case is handled by `_isinvisible`.
+        # home directory.
+        #
+        # This case is handled by `_isinvisible_st_flags`.
 
         #=== Case 4: Packages and bundles ===#
         # Packages and bundles are directories that the Finder presents to the user as if
@@ -181,12 +223,15 @@ include("path.jl")
             end
             return false
         end
+        _exists_inside_package_or_bundle(ps::PathStruct) =
+            _exists_inside_package_or_bundle(ps.realpath)
 
         #=== All macOS cases ===#
         _ishidden_macos(ps::PathStruct) =
             _ishidden_bsd_related(ps) ||
-            _issystemfile(ps.path) ||
-            _exists_inside_package_or_bundle(ps.realpath)
+            _issystemdir(ps) ||
+            _isinvisible_macos_item_info(ps) ||
+            _exists_inside_package_or_bundle(ps)
         _ishidden = _ishidden_macos
     elseif Sys.isbsd()  # BSD; this excludes macOS through control flow (as macOS is checked for first)
         _ishidden_bsd(ps::PathStruct) = _ishidden_bsd_related(ps)

src/utils/darwin.jl

@@ -1,3 +1,7 @@
+# TODO: consider using ObjectiveC.jl  (HiddenFiles.jl#8)
+#   https://discourse.julialang.org/t/95862
+#   https://github.com/JuliaInterop/ObjectiveC.jl
+
 # https://opensource.apple.com/source/CF/CF-635/CFString.h.auto.html
 # https://developer.apple.com/documentation/corefoundation/cfstringbuiltinencodings
 const K_CF_STRING_ENCODING_MAC_ROMAN = 0x0
@@ -210,3 +214,50 @@ function _string_from_cf_string(cfstr::Cstring, encoding::Unsigned = CF_STRING_E
     end
     return String(take!(cfio))
 end
+
+#===============================================#
+
+# https://developer.apple.com/documentation/coreservices/lsiteminfoflags/klsiteminfoisinvisible
+# TODO: convert this to enum: https://developer.apple.com/documentation/coreservices/lsiteminfoflags: https://github.com/phracker/MacOSX-SDKs/blob/041600eda65c6a668f66cb7d56b7d1da3e8bcc93/MacOSX10.6.sdk/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/Headers/LSInfo.h#L95-L111
+const KLS_ITEM_INFO_IS_INVISIBLE = 0x00000040
+
+# https://developer.apple.com/documentation/coreservices/1429609-anonymous/kisinvisible
+const K_IS_INVISIBLE = 0x4000
+
+# https://developer.apple.com/documentation/corefoundation/cfurlpathstyle
+const K_CF_URL_POSIX_PATH_STYLE = zero(Int8)
+
+# https://developer.apple.com/documentation/corefoundation/1543250-cfurlcreatewithfilesystempath
+function _cf_url_create_with_file_system_path(cfstr::Cstring, is_directory::Bool, path_style::Integer = K_CF_URL_POSIX_PATH_STYLE)
+    # TODO: handle error codes
+    # CFURLRef CFURLCreateWithFileSystemPath(CFAllocatorRef allocator, CFStringRef filePath, CFURLPathStyle pathStyle, Boolean isDirectory);
+    url_ref = ccall(:CFURLCreateWithFileSystemPath, Ptr{UInt32},
+                    (Ptr{Cvoid}, Cstring, Int32, Bool),
+                    C_NULL, cfstr, path_style, is_directory)
+    return url_ref
+end
+
+# https://developer.apple.com/documentation/coreservices/lsiteminforecord
+function _ls_item_info_record()
+    @warn "LSItemInfoRecord has been deprecated since macOS 10.11"
+    error("not yet implemented")
+end
+
+# https://developer.apple.com/documentation/coreservices/lsrequestedinfo/klsrequestallflags
+const K_LS_REQUEST_ALL_FLAGS = 0x00000010
+# TODO: convert this to enum: https://developer.apple.com/documentation/coreservices/lsrequestedinfo: https://github.com/phracker/MacOSX-SDKs/blob/041600eda65c6a668f66cb7d56b7d1da3e8bcc93/MacOSX10.6.sdk/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/Headers/LSInfo.h#L83-L93
+
+# https://developer.apple.com/documentation/coreservices/1445685-lscopyiteminfoforurl
+function _ls_copy_item_info_for_url(url_ref::Ptr{UInt32}, requested_info::Unsigned = K_LS_REQUEST_ALL_FLAGS)
+    # TODO: handle error codes
+    requested_info == K_LS_REQUEST_ALL_FLAGS && @warn "kLSRequestAllFlags has been deprecated since macOS 10.11"
+    requested_info == KLS_ITEM_INFO_IS_INVISIBLE && @warn "kLSItemInfoIsInvisible has been deprecated since macOS 10.11; ensure you are using kIsInvisible instead"
+    @warn "LSCopyItemInfoForURL has been deprecated since macOS 10.11"
+    # buf = Vector{UInt32}(undef, 100)
+    buf = zeros(UInt32, 100)
+    # OSStatus LSCopyItemInfoForURL(CFURLRef inURL, LSRequestedInfo inWhichInfo, LSItemInfoRecord *outItemInfo);
+    ptr = ccall(:LSCopyItemInfoForURL, Ptr{UInt32},
+                (Ptr{UInt32}, UInt32, Ptr{Cvoid}),
+                url_ref, requested_info, buf)
+    return buf
+end

test/runtests.jl

@@ -28,14 +28,34 @@ using Test
 
                 # Case 2: UNIX-specific directories
                 # TODO: complete this case
-                @test HiddenFiles.ishidden("/bin/")
-                @test HiddenFiles.ishidden("/dev/")
-                @test HiddenFiles.ishidden("/usr/")
-                @test !HiddenFiles.ishidden("/tmp/")
+                @test HiddenFiles.ishidden("/bin")
+                @test HiddenFiles.ishidden("/dev")
+                @test HiddenFiles.ishidden("/dev")
+                @test HiddenFiles.ishidden("/etc")
+                @test HiddenFiles.ishidden("/sbin")
+                @test HiddenFiles.ishidden("/sbin")
+                @test HiddenFiles.ishidden("/tmp")  # This expands to /private/tmp, but /private is hidden which means /tmp should also be hidden (see 54ced37c)
+                @test HiddenFiles.ishidden("/private")
+                @test HiddenFiles.ishidden("/private/tmp")
+                @test HiddenFiles.ishidden("/private/tmp/")
+                @test HiddenFiles.ishidden("/tmp/")
+                @test HiddenFiles.ishidden("/usr")
+                @test HiddenFiles.ishidden("/var")
+                @test HiddenFiles._isinvisible_macos_item_info("/bin")
+                @test HiddenFiles._isinvisible_macos_item_info("/dev")
+                @test HiddenFiles._isinvisible_macos_item_info("/dev")
+                @test HiddenFiles._isinvisible_macos_item_info("/etc")
+                @test HiddenFiles._isinvisible_macos_item_info("/sbin")
+                @test HiddenFiles._isinvisible_macos_item_info("/sbin")
+                @test HiddenFiles._isinvisible_macos_item_info("/tmp")
+                @test HiddenFiles._isinvisible_macos_item_info("/usr")
+                @test HiddenFiles._isinvisible_macos_item_info("/var")
+                @test !HiddenFiles._isinvisible_macos_item_info("/bin/bash")
 
                 # Case 3: Explicitly hidden files and directories
-                @test HiddenFiles._isinvisible("/Volumes")
+                @test HiddenFiles._isinvisible_st_flags("/Volumes")
                 @test ishidden("/Volumes")
+                @test !HiddenFiles._isinvisible_st_flags(p′)
                 @test !HiddenFiles._isinvisible(p′)
 
                 # Case 4: Packages and bundles
@@ -48,11 +68,12 @@ using Test
                 @test ishidden("/System/Applications/Utilities/Terminal.app/Contents/")  # This should be the same as above, as we expand all paths using realpath
                 @test !HiddenFiles._ispackage_or_bundle("/System/Applications/Utilities/Terminal.app/Contents/")
                 @test HiddenFiles._exists_inside_package_or_bundle("/System/Applications/Utilities/Terminal.app/Contents/")
-                @test !HiddenFiles._exists_inside_package_or_bundle("/bin/")
-                f = randpath()
+                @test !HiddenFiles._exists_inside_package_or_bundle("/bin")
+                @test !HiddenFiles._exists_inside_package_or_bundle("/tmp")
+                f = randpath()  # This path shouldn't exist
                 cfstr_nonexistent = HiddenFiles._cfstring_create_with_cstring(f)
                 @test_throws Exception HiddenFiles._mditem_create(cfstr_nonexistent)
-                encoding_mode_nonexistent = 0x1c000101  # this encoding mode should not exist
+                encoding_mode_nonexistent = 0x1c000101  # This encoding mode should not exist
                 @test_throws Exception HiddenFiles._cfstring_create_with_cstring(
                     "Julia", encoding_mode_nonexistent
                 )
@@ -67,9 +88,9 @@ using Test
             # TODO: should we not only support FreeBSD?  Are we testing on other BSD systems?  OpenBSD?
             @testset "HiddenFiles.jl—FreeBSD" begin
                 @test ishidden(p)
-                @test !HiddenFiles._isinvisible(p)
+                @test !HiddenFiles._isinvisible_st_flags(p)
                 @test ishidden(p′)
-                @test !HiddenFiles._isinvisible(p′)
+                @test !HiddenFiles._isinvisible_st_flags(p′)
                 @test !ishidden(homedir())
                 @test !ishidden("/bin/")
                 @test !ishidden("/dev/")