From ae9f2b2c7ecbff3204ea056bea69a57713d4058d Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Mon, 26 Sep 2022 20:39:25 +1300 Subject: [PATCH 01/16] Error on iszfs call (not yet implemented) --- src/utils/zfs.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 5341127..328c200 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -1,5 +1,6 @@ function iszfs() # TODO + error("not yet implemented") return false end From d4d145976c6879811a3bd5765f5c7b33494707f7 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Mon, 26 Sep 2022 22:53:38 +1300 Subject: [PATCH 02/16] WIP: begin work on implementing statfs calls for UNIX-like OSs --- src/utils/zfs.jl | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 328c200..ea38836 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -1,6 +1,32 @@ -function iszfs() - # TODO - error("not yet implemented") - return false +@static if Sys.isapple() + # macOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/statfs.2.html + + const SIZEOF_STATFS = 2168 + const SIZEOF_STATFS64 = 2168 + const MACOS_F_OTYPE_OFFSET = 13 # + + function iszfs() + error("not yet implemented") + buf = zeros(UInt, 2048); ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x0d] => 0x00 + buf = zeros(UInt32, 2048); ccall(:statfs64, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x13] => 0x73667061 + end +elseif Sys.isbsd() + const SIZEOF_STATFS = 2344 + + # BSD: https://www.freebsd.org/cgi/man.cgi?query=statfs&sektion=2 + const BSD_F_TYPE_OFFSET = 0x03 # 2 in UInt64, but 3 in UInt32 + + function iszfs() + error("not yet implemented") + buf = zeros(UInt32, 2048); ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x03] => 0x10001010 + end +elseif Sys.isunix() + # Linux: https://man7.org/linux/man-pages/man2/statfs.2.html + function iszfs() + error("not yet implemented") + buf = zeros(UInt32, 2048); ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # + end +else + iszfs() = error("Cannot call statfs for non-unix operating systems") end From 9b3e746b5148339f91e1d49efd37409c9ab50042 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 00:39:09 +1300 Subject: [PATCH 03/16] WIP: Implement libuv statfs version; research into ZFS types In Julia 1.6, nice libuv statfs bindings were added to Julia, which allows system-independent statfs calls. In Julia 1.8, this was made even nicer by tucking all of that logic into a struct, behind diskstat. While I would like to use this, it would mean not supporting versions < 1.6 for this functionality. I will have to think on it/ask some friends for their opinions. I also did some research/testing into ZFS type values for once we have the f_type field from statfs. Finally, I have (and will continue to) do testing into system-dependent offsets of the f_type field from statfs, in case we can indeed write some janky solution that will work < 1.6. --- src/utils/zfs.jl | 67 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index ea38836..232d82d 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -1,30 +1,77 @@ +# Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 +# C calls for UV statfs requires Julia 1.6 +# http://docs.libuv.org/en/v1.x/fs.html#c.uv_fs_statfs +struct DiskStat + ftype::UInt64 + bsize::UInt64 + blocks::UInt64 + bfree::UInt64 + bavail::UInt64 + files::UInt64 + ffree::UInt64 + fspare::NTuple{4, UInt64} # reserved +end +function statfs(f::AbstractString) + buf = Vector{UInt8}(undef, ccall(:jl_sizeof_uv_fs_t, Int32, ())) + i = ccall(:uv_fs_statfs, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Ptr{Cvoid}), + C_NULL, buf, f, C_NULL) + i < 0 && Base.uv_error("statfs($(repr(f)))", i) + p = ccall(:jl_uv_fs_t_ptr, Ptr{UInt32}, (Ptr{Cvoid},), buf) + + return unsafe_load(reinterpret(Ptr{DiskStat}, p)) +end + +# https://lists.freebsd.org/pipermail/freebsd-hackers/2018-February/052295.html +const SUN_ZFS_SUPER_MAGIC = 0x1b +const BSD_ZFS_SUPER_MAGIC = 0xde # Obtained from testing +# https://reviews.freebsd.org/rS320069 +const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 + +const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) + +function iszfs(f::AbstractString) + s = statfs(f) + return s.ftype ∈ ZFS_SUPER_MAGICS +end + @static if Sys.isapple() # macOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/statfs.2.html - + const SIZEOF_STATFS = 2168 const SIZEOF_STATFS64 = 2168 const MACOS_F_OTYPE_OFFSET = 13 # - + function iszfs() - error("not yet implemented") - buf = zeros(UInt, 2048); ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x0d] => 0x00 - buf = zeros(UInt32, 2048); ccall(:statfs64, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x13] => 0x73667061 + # 0x1a <- offset 69 in UInt8 array in statfs, 53 in UInt8 array in statfs64 + # error("not yet implemented") + buf = zeros(UInt8, SIZEOF_STATFS) ## 64 + ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x0d] => 0x00 + # return buf + + buf = zeros(UInt8, SIZEOF_STATFS64) ## 32 + ccall(:statfs64, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x13] => 0x73667061 + return buf end elseif Sys.isbsd() const SIZEOF_STATFS = 2344 - + # BSD: https://www.freebsd.org/cgi/man.cgi?query=statfs&sektion=2 const BSD_F_TYPE_OFFSET = 0x03 # 2 in UInt64, but 3 in UInt32 - + function iszfs() - error("not yet implemented") - buf = zeros(UInt32, 2048); ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x03] => 0x10001010 + # 0xde <- offset 5 in UInt8 array + # error("not yet implemented") + buf = zeros(UInt8, SIZEOF_STATFS) ## 32 + ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x03] => 0x10001010 + return buf end elseif Sys.isunix() # Linux: https://man7.org/linux/man-pages/man2/statfs.2.html function iszfs() error("not yet implemented") - buf = zeros(UInt32, 2048); ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # + buf = zeros(UInt8, error("NEED TO TEST sizeof(struct statfs) ON LINUX")) ## 32 + ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # + return buf end else iszfs() = error("Cannot call statfs for non-unix operating systems") From c95cb4fd4e093977a26570a8d07004639853bbbf Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 00:55:37 +1300 Subject: [PATCH 04/16] WIP: f_type offset research Added C programme in comment for offset research. macOS: $ gcc -o statfs statfs.c && ./statfs offsetof(struct statfs, f_type): 0x3c st.f_type: 0x1a BSD: $ gcc11 statfs.c -o statfs && ./statfs offsetof(struct statfs, f_type): 0x4 st.f_type: 0xde --- src/utils/zfs.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 232d82d..4acc3cf 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -34,6 +34,23 @@ function iszfs(f::AbstractString) return s.ftype ∈ ZFS_SUPER_MAGICS end + +#= +#include +#include +#include +#include + +int main() { + struct statfs st; + printf("offsetof(struct statfs, f_type): 0x%lx\n", offsetof(struct statfs, f_type)); + statfs("/", &st); + printf("st.f_type: 0x%llx\n", (uint64_t) st.f_type); + return 0; +} + +=# + @static if Sys.isapple() # macOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/statfs.2.html From d715d975c2caaaf23b2dde2a17ca96b711555809 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 01:40:55 +1300 Subject: [PATCH 05/16] WIP: select iszfs function depending on Julia version I also finished cleaning up the macOS and BSD versions of iszfs. Interestingly, macOS appears to have both f_type and f_fssubtype fields set from statfs (despite the latter not actually being documented. I am still deciding whether or not our < 1.6 version is a good idea. --- src/utils/zfs.jl | 139 ++++++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 4acc3cf..f1851f8 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -1,26 +1,3 @@ -# Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 -# C calls for UV statfs requires Julia 1.6 -# http://docs.libuv.org/en/v1.x/fs.html#c.uv_fs_statfs -struct DiskStat - ftype::UInt64 - bsize::UInt64 - blocks::UInt64 - bfree::UInt64 - bavail::UInt64 - files::UInt64 - ffree::UInt64 - fspare::NTuple{4, UInt64} # reserved -end -function statfs(f::AbstractString) - buf = Vector{UInt8}(undef, ccall(:jl_sizeof_uv_fs_t, Int32, ())) - i = ccall(:uv_fs_statfs, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Ptr{Cvoid}), - C_NULL, buf, f, C_NULL) - i < 0 && Base.uv_error("statfs($(repr(f)))", i) - p = ccall(:jl_uv_fs_t_ptr, Ptr{UInt32}, (Ptr{Cvoid},), buf) - - return unsafe_load(reinterpret(Ptr{DiskStat}, p)) -end - # https://lists.freebsd.org/pipermail/freebsd-hackers/2018-February/052295.html const SUN_ZFS_SUPER_MAGIC = 0x1b const BSD_ZFS_SUPER_MAGIC = 0xde # Obtained from testing @@ -29,12 +6,6 @@ const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) -function iszfs(f::AbstractString) - s = statfs(f) - return s.ftype ∈ ZFS_SUPER_MAGICS -end - - #= #include #include @@ -48,49 +19,81 @@ int main() { printf("st.f_type: 0x%llx\n", (uint64_t) st.f_type); return 0; } - =# -@static if Sys.isapple() - # macOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/statfs.2.html - - const SIZEOF_STATFS = 2168 - const SIZEOF_STATFS64 = 2168 - const MACOS_F_OTYPE_OFFSET = 13 # - - function iszfs() - # 0x1a <- offset 69 in UInt8 array in statfs, 53 in UInt8 array in statfs64 - # error("not yet implemented") - buf = zeros(UInt8, SIZEOF_STATFS) ## 64 - ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x0d] => 0x00 - # return buf - - buf = zeros(UInt8, SIZEOF_STATFS64) ## 32 - ccall(:statfs64, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x13] => 0x73667061 - return buf - end -elseif Sys.isbsd() - const SIZEOF_STATFS = 2344 - - # BSD: https://www.freebsd.org/cgi/man.cgi?query=statfs&sektion=2 - const BSD_F_TYPE_OFFSET = 0x03 # 2 in UInt64, but 3 in UInt32 - - function iszfs() - # 0xde <- offset 5 in UInt8 array - # error("not yet implemented") - buf = zeros(UInt8, SIZEOF_STATFS) ## 32 - ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # buf[0x03] => 0x10001010 - return buf +@static if VERSION ≥ v"1.6" + @static if VERSION < v"1.8" + # Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 + # C calls for UV statfs requires Julia 1.6 + # http://docs.libuv.org/en/v1.x/fs.html#c.uv_fs_statfs + struct DiskStat + ftype::UInt64 + bsize::UInt64 + blocks::UInt64 + bfree::UInt64 + bavail::UInt64 + files::UInt64 + ffree::UInt64 + fspare::NTuple{4, UInt64} # reserved + end + function statfs(f::AbstractString) + buf = Vector{UInt8}(undef, ccall(:jl_sizeof_uv_fs_t, Int32, ())) + i = ccall(:uv_fs_statfs, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Ptr{Cvoid}), + C_NULL, buf, f, C_NULL) + i < 0 && Base.uv_error("statfs($(repr(f)))", i) + p = ccall(:jl_uv_fs_t_ptr, Ptr{UInt32}, (Ptr{Cvoid},), buf) + + return unsafe_load(reinterpret(Ptr{DiskStat}, p)) + end + else + statfs = diskstat end -elseif Sys.isunix() - # Linux: https://man7.org/linux/man-pages/man2/statfs.2.html - function iszfs() - error("not yet implemented") - buf = zeros(UInt8, error("NEED TO TEST sizeof(struct statfs) ON LINUX")) ## 32 - ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), "/", buf) # - return buf + + function iszfs(f::AbstractString) + s = statfs(f) + return s.ftype ∈ ZFS_SUPER_MAGICS end else - iszfs() = error("Cannot call statfs for non-unix operating systems") + # If Julia version < 1.6, we have to write our own, not very nice solution + @static if Sys.isapple() + # macOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/statfs.2.html + const SIZEOF_STATFS = 2168 + const F_TYPE_OFFSET = 0x3d + const F_FSSUBTYPE_OFFSET = 0x45 + + function iszfs(f::AbstractString) + buf = Vector{UInt8}(undef, SIZEOF_STATFS) + # statfs(const char *path, struct statfs *buf); + i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) + i < 0 && Base.uv_error("statfs($(repr(f)))", i) + return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS || buf[F_FSSUBTYPE_OFFSET] ∈ ZFS_SUPER_MAGICS + end + elseif Sys.isbsd() + # https://www.freebsd.org/cgi/man.cgi?query=statfs&sektion=2 + const SIZEOF_STATFS = 2344 + const F_TYPE_OFFSET = 0x05 + + function iszfs(f::AbstractString) + buf = Vector{UInt8}(undef, SIZEOF_STATFS) + # statfs(const char *path, struct statfs *buf); + i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) + i < 0 && Base.uv_error("statfs($(repr(f)))", i) + return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS + end + elseif Sys.isunix() + # Linux: https://man7.org/linux/man-pages/man2/statfs.2.html + const SIZEOF_STATFS = error("not yet implemented") + const F_TYPE_OFFSET = error("not yet implemented") + + function iszfs(f::AbstractString) + buf = zeros(UInt8, SIZEOF_STATFS) + # statfs(const char *path, struct statfs *buf); + i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) + i < 0 && Base.uv_error("statfs($(repr(f)))", i) + return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS + end + else + iszfs() = error("Cannot call statfs for non-Unix operating systems") + end end From 15cebc2d96a789d9edf9b178e3fbe413e9b1e172 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 01:49:59 +1300 Subject: [PATCH 06/16] WIP: rename iszfs -> _iszfs; correct type signatures for _iszfs --- src/HiddenFiles.jl | 6 +----- src/docs.jl | 13 +++++++++++-- src/utils/zfs.jl | 10 +++++----- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/HiddenFiles.jl b/src/HiddenFiles.jl index 0bfe883..6f0451f 100644 --- a/src/HiddenFiles.jl +++ b/src/HiddenFiles.jl @@ -25,15 +25,11 @@ include("docs.jl") @static if Sys.isunix() include("utils/zfs.jl") - if iszfs() # @static breaks here # ZFS - error("not yet implemented") - _ishidden_zfs(f::AbstractString) = error("not yet implemented") - end # Trivial Unix check _isdotfile(f::AbstractString) = startswith(basename(f), '.') # Account for ZFS - _ishidden_unix(f::AbstractString) = _isdotfile(f) || (iszfs() && _ishidden_zfs()) + _ishidden_unix(f::AbstractString) = _isdotfile(f) || (_iszfs(f) && _ishidden_zfs(f)) @static if Sys.isbsd() # BDS-related; this is true for macOS as well # https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/chflags.2.html diff --git a/src/docs.jl b/src/docs.jl index 8a23c92..ad9b669 100644 --- a/src/docs.jl +++ b/src/docs.jl @@ -12,8 +12,17 @@ See also: [`_ishidden_unix`](@ref), [`_ishidden_windows`](@ref), [`_ishidden_mac _ishidden ### ZFS ### -"This function is not yet implemented" -iszfs +""" +```julia +_iszfs(f::AbstractString) -> Bool +``` + +Determines if the file system that `f` exists on is ZFS. + +!!! note + This is only implemented for Unix-like systems, as I have not seen ZFS for Windows. +""" +_iszfs "This function is not yet implemented" _ishidden_zfs diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index f1851f8..3ed88d0 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -49,7 +49,7 @@ int main() { statfs = diskstat end - function iszfs(f::AbstractString) + function _iszfs(f::AbstractString) s = statfs(f) return s.ftype ∈ ZFS_SUPER_MAGICS end @@ -61,7 +61,7 @@ else const F_TYPE_OFFSET = 0x3d const F_FSSUBTYPE_OFFSET = 0x45 - function iszfs(f::AbstractString) + function _iszfs(f::AbstractString) buf = Vector{UInt8}(undef, SIZEOF_STATFS) # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) @@ -73,7 +73,7 @@ else const SIZEOF_STATFS = 2344 const F_TYPE_OFFSET = 0x05 - function iszfs(f::AbstractString) + function _iszfs(f::AbstractString) buf = Vector{UInt8}(undef, SIZEOF_STATFS) # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) @@ -85,7 +85,7 @@ else const SIZEOF_STATFS = error("not yet implemented") const F_TYPE_OFFSET = error("not yet implemented") - function iszfs(f::AbstractString) + function _iszfs(f::AbstractString) buf = zeros(UInt8, SIZEOF_STATFS) # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) @@ -93,7 +93,7 @@ else return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end else - iszfs() = error("Cannot call statfs for non-Unix operating systems") + _iszfs(_f::AbstractString) = error("Cannot call statfs for non-Unix operating systems") end end From 8ab9603e29849da40b689ef1211b1fb846bddcaf Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 01:53:30 +1300 Subject: [PATCH 07/16] Remove C statfs offset helper code from comment Moved to: https://github.com/jakewilliami/HiddenFiles.jl/pull/20#issuecomment-1257996322 --- src/utils/zfs.jl | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 3ed88d0..a71b7bf 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -6,21 +6,6 @@ const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) -#= -#include -#include -#include -#include - -int main() { - struct statfs st; - printf("offsetof(struct statfs, f_type): 0x%lx\n", offsetof(struct statfs, f_type)); - statfs("/", &st); - printf("st.f_type: 0x%llx\n", (uint64_t) st.f_type); - return 0; -} -=# - @static if VERSION ≥ v"1.6" @static if VERSION < v"1.8" # Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 From ac5fda35f22d8860cd987dbe588532c83e3e0a78 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 02:01:13 +1300 Subject: [PATCH 08/16] Added debug print for FS type --- src/utils/zfs.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index a71b7bf..38aeb8a 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -1,3 +1,5 @@ +# TODO: Correct < 1.6 error codes + # https://lists.freebsd.org/pipermail/freebsd-hackers/2018-February/052295.html const SUN_ZFS_SUPER_MAGIC = 0x1b const BSD_ZFS_SUPER_MAGIC = 0xde # Obtained from testing @@ -6,7 +8,7 @@ const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) -@static if VERSION ≥ v"1.6" +@static if VERSION == v"1.6" @static if VERSION < v"1.8" # Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 # C calls for UV statfs requires Julia 1.6 @@ -36,6 +38,7 @@ const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SU function _iszfs(f::AbstractString) s = statfs(f) + @info s.ftype return s.ftype ∈ ZFS_SUPER_MAGICS end else @@ -51,6 +54,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) + @info buf[F_TYPE_OFFSET] buf[F_SSUBTYPE_OFFSET] return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS || buf[F_FSSUBTYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isbsd() @@ -63,6 +67,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) + @info buf[F_TYPE_OFFSET] return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isunix() @@ -75,6 +80,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) + @info buf[F_TYPE_OFFSET] return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end else From 5f6a21e084e2b386a9b2215c6d82ee01d8e2797f Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 02:03:17 +1300 Subject: [PATCH 09/16] Correct debug print of FS type and subtype for macOS --- src/utils/zfs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 38aeb8a..c1f7945 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -54,7 +54,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - @info buf[F_TYPE_OFFSET] buf[F_SSUBTYPE_OFFSET] + @info buf[F_TYPE_OFFSET], buf[F_FSSUBTYPE_OFFSET] return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS || buf[F_FSSUBTYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isbsd() From abcac6be3de6a7796f2ad74303de3c6f72995323 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 02:10:03 +1300 Subject: [PATCH 10/16] WIP: added full buffer in debug prints --- src/utils/zfs.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index c1f7945..b09e938 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -8,6 +8,7 @@ const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) +# TODO: change this back to < 1.6 @static if VERSION == v"1.6" @static if VERSION < v"1.8" # Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 @@ -54,7 +55,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - @info buf[F_TYPE_OFFSET], buf[F_FSSUBTYPE_OFFSET] + @info buf[F_TYPE_OFFSET], buf[F_FSSUBTYPE_OFFSET], buf return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS || buf[F_FSSUBTYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isbsd() @@ -63,11 +64,11 @@ else const F_TYPE_OFFSET = 0x05 function _iszfs(f::AbstractString) - buf = Vector{UInt8}(undef, SIZEOF_STATFS) + buf = Vector{UInt32}(undef, SIZEOF_STATFS) # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - @info buf[F_TYPE_OFFSET] + @info buf[F_TYPE_OFFSET], buf return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isunix() From a4b738b716ccf0d6a795cff71bdb175f054468fa Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 02:19:49 +1300 Subject: [PATCH 11/16] Revert Julia 1.6 checker to allow new _iszfs implementation --- src/utils/zfs.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index b09e938..d9209a0 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -8,8 +8,7 @@ const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) -# TODO: change this back to < 1.6 -@static if VERSION == v"1.6" +@static if VERSION < v"1.6" @static if VERSION < v"1.8" # Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 # C calls for UV statfs requires Julia 1.6 From f2e31959eaba26cd160f6ce6a09d23a907552b14 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 02:23:39 +1300 Subject: [PATCH 12/16] Correct < 1.6 to <= 1.6 in _iszfs implementation --- src/utils/zfs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index d9209a0..0bd9e98 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -8,7 +8,7 @@ const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) -@static if VERSION < v"1.6" +@static if VERSION ≤ v"1.6" @static if VERSION < v"1.8" # Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 # C calls for UV statfs requires Julia 1.6 From bcaba09658e6145f59c52c330e3b38f81b70fe17 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 02:28:51 +1300 Subject: [PATCH 13/16] Correct <= 1.6 to >= 1.6 in _iszfs implementation --- src/utils/zfs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 0bd9e98..1873839 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -8,7 +8,7 @@ const LINUX_ZFS_SUPER_MAGIC = 0x2fc12fc1 const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SUPER_MAGIC) -@static if VERSION ≤ v"1.6" +@static if VERSION ≥ v"1.6" @static if VERSION < v"1.8" # Adapted from Julia 1.8's diskstat: https://github.com/JuliaLang/julia/pull/42248 # C calls for UV statfs requires Julia 1.6 From eda1c22edfe4de6054fac9eea405a52d5be4f0fa Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 08:59:52 +1300 Subject: [PATCH 14/16] Adjust statfs buf offsets according to eltype; finish Linux impl The statfs buffer I implemented previously had elements that were UInt8s, but after implementing Linux's _iszfs function, I realised that some statfs calls needed to store larger-sized integers. As such, we can now dynamically change the element type of the buffer and this will dynamically change the size of the array, and the appropriate offset (cld because Julia is indexed by one) --- src/utils/zfs.jl | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 1873839..9266625 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -45,41 +45,50 @@ else # If Julia version < 1.6, we have to write our own, not very nice solution @static if Sys.isapple() # macOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/statfs.2.html - const SIZEOF_STATFS = 2168 - const F_TYPE_OFFSET = 0x3d - const F_FSSUBTYPE_OFFSET = 0x45 + const STATFS_BUF_ELTYPE = UInt32 + const SIZEOF_STATFS_BUF_ELTYPE = Int8(sizeof(STATFS_BUF_ELTYPE)) + const SIZEOF_STATFS = cld(2168, SIZEOF_STATFS_BUF_ELTYPE) + const F_TYPE_OFFSET = cld(0x3d, SIZEOF_STATFS_BUF_ELTYPE) + const F_FSSUBTYPE_OFFSET = cld(0x45, SIZEOF_STATFS_BUF_ELTYPE) function _iszfs(f::AbstractString) - buf = Vector{UInt8}(undef, SIZEOF_STATFS) + buf = Vector{STATFS_BUF_ELTYPE}(undef, SIZEOF_STATFS) # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) + # return buf @info buf[F_TYPE_OFFSET], buf[F_FSSUBTYPE_OFFSET], buf return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS || buf[F_FSSUBTYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isbsd() # https://www.freebsd.org/cgi/man.cgi?query=statfs&sektion=2 - const SIZEOF_STATFS = 2344 - const F_TYPE_OFFSET = 0x05 + const STATFS_BUF_ELTYPE = UInt32 + const SIZEOF_STATFS_BUF_ELTYPE = Int8(sizeof(STATFS_BUF_ELTYPE)) + const SIZEOF_STATFS = cld(2344, SIZEOF_STATFS_BUF_ELTYPE) + const F_TYPE_OFFSET = cld(0x05, SIZEOF_STATFS_BUF_ELTYPE) function _iszfs(f::AbstractString) - buf = Vector{UInt32}(undef, SIZEOF_STATFS) + buf = Vector{STATFS_BUF_ELTYPE}(undef, SIZEOF_STATFS) # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) + # return buf @info buf[F_TYPE_OFFSET], buf return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isunix() # Linux: https://man7.org/linux/man-pages/man2/statfs.2.html - const SIZEOF_STATFS = error("not yet implemented") - const F_TYPE_OFFSET = error("not yet implemented") + const STATFS_BUF_ELTYPE = UInt32 + const SIZEOF_STATFS_BUF_ELTYPE = Int8(sizeof(STATFS_BUF_ELTYPE)) + const SIZEOF_STATFS = cld(120, SIZEOF_STATFS_BUF_ELTYPE) + const F_TYPE_OFFSET = cld(0x01, SIZEOF_STATFS_BUF_ELTYPE) function _iszfs(f::AbstractString) - buf = zeros(UInt8, SIZEOF_STATFS) + buf = Vector{STATFS_BUF_ELTYPE}(undef, SIZEOF_STATFS) # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) + # return buf @info buf[F_TYPE_OFFSET] return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end From 3cba8564af833b7954363757858a194a9893f2b2 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 09:09:30 +1300 Subject: [PATCH 15/16] Move statfs buf eltype and its size into <1.6 logic These values did not need to be defined for each OS branch if Julia version < 1.6 --- src/utils/zfs.jl | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 9266625..2a241b5 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -43,10 +43,13 @@ const ZFS_SUPER_MAGICS = (SUN_ZFS_SUPER_MAGIC, BSD_ZFS_SUPER_MAGIC, LINUX_ZFS_SU end else # If Julia version < 1.6, we have to write our own, not very nice solution + + # It occurred to me that UInt8 wasn't always enough to store the f_type (eda1c22) + const STATFS_BUF_ELTYPE = UInt32 + const SIZEOF_STATFS_BUF_ELTYPE = Int8(sizeof(STATFS_BUF_ELTYPE)) + @static if Sys.isapple() # macOS: https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/statfs.2.html - const STATFS_BUF_ELTYPE = UInt32 - const SIZEOF_STATFS_BUF_ELTYPE = Int8(sizeof(STATFS_BUF_ELTYPE)) const SIZEOF_STATFS = cld(2168, SIZEOF_STATFS_BUF_ELTYPE) const F_TYPE_OFFSET = cld(0x3d, SIZEOF_STATFS_BUF_ELTYPE) const F_FSSUBTYPE_OFFSET = cld(0x45, SIZEOF_STATFS_BUF_ELTYPE) @@ -56,14 +59,11 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - # return buf @info buf[F_TYPE_OFFSET], buf[F_FSSUBTYPE_OFFSET], buf return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS || buf[F_FSSUBTYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isbsd() # https://www.freebsd.org/cgi/man.cgi?query=statfs&sektion=2 - const STATFS_BUF_ELTYPE = UInt32 - const SIZEOF_STATFS_BUF_ELTYPE = Int8(sizeof(STATFS_BUF_ELTYPE)) const SIZEOF_STATFS = cld(2344, SIZEOF_STATFS_BUF_ELTYPE) const F_TYPE_OFFSET = cld(0x05, SIZEOF_STATFS_BUF_ELTYPE) @@ -72,14 +72,11 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - # return buf @info buf[F_TYPE_OFFSET], buf return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isunix() # Linux: https://man7.org/linux/man-pages/man2/statfs.2.html - const STATFS_BUF_ELTYPE = UInt32 - const SIZEOF_STATFS_BUF_ELTYPE = Int8(sizeof(STATFS_BUF_ELTYPE)) const SIZEOF_STATFS = cld(120, SIZEOF_STATFS_BUF_ELTYPE) const F_TYPE_OFFSET = cld(0x01, SIZEOF_STATFS_BUF_ELTYPE) @@ -88,7 +85,6 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - # return buf @info buf[F_TYPE_OFFSET] return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end From 1b13b0ee8c7e4d149e8d9ed148072ae86877e35c Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Tue, 27 Sep 2022 09:15:52 +1300 Subject: [PATCH 16/16] Add file name to debug print for Julia < 1.6 impl of _iszfs --- src/utils/zfs.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/zfs.jl b/src/utils/zfs.jl index 2a241b5..78fe8af 100644 --- a/src/utils/zfs.jl +++ b/src/utils/zfs.jl @@ -59,7 +59,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - @info buf[F_TYPE_OFFSET], buf[F_FSSUBTYPE_OFFSET], buf + @info buf[F_TYPE_OFFSET], buf[F_FSSUBTYPE_OFFSET], buf, f return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS || buf[F_FSSUBTYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isbsd() @@ -72,7 +72,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - @info buf[F_TYPE_OFFSET], buf + @info buf[F_TYPE_OFFSET], buf, f return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end elseif Sys.isunix() @@ -85,7 +85,7 @@ else # statfs(const char *path, struct statfs *buf); i = ccall(:statfs, Int, (Cstring, Ptr{Cvoid}), f, buf) i < 0 && Base.uv_error("statfs($(repr(f)))", i) - @info buf[F_TYPE_OFFSET] + @info buf[F_TYPE_OFFSET], buf, f return buf[F_TYPE_OFFSET] ∈ ZFS_SUPER_MAGICS end else