From 85e94ce2c3da36d7b3760ff481cc982029f1e881 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 4 Jun 2026 23:52:12 +0200 Subject: [PATCH 1/4] refactor v2, v3 --- src/serve_zarr.jl | 63 ++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/src/serve_zarr.jl b/src/serve_zarr.jl index e56caf9..680d54d 100644 --- a/src/serve_zarr.jl +++ b/src/serve_zarr.jl @@ -1,25 +1,36 @@ -""" - _synthetic_zmetadata(path) -> Union{Vector{UInt8}, Nothing} - -Build the same consolidated metadata dict as `Zarr.consolidate_metadata(s::AbstractStore, d, prefix)` -with `prefix == ""`, then serialize it like `Zarr.consolidate_metadata(s::AbstractStore, p)` -(the `JSON.print` of `metadata` / `zarr_consolidated_format`) but **without** writing `s[p, .zmetadata]` to disk. -`Zarr.zarr_req_handler` instead calls `consolidate_metadata(s)` so the store gains a real `.zmetadata` -file before keys are served. +function _get_metadata(path::String) + cons_meta, cons_key = _read_consolidated_metadata(path) + if !isnothing(cons_meta) + return cons_meta, cons_key + else + created_metadata = _generate_metadata(path) + return created_metadata, cons_key + end +end -Returns `nothing` when `.zmetadata` already exists on disk, a root `zarr.json` exists (v2-only synthesis), -or the directory is not a Zarr v2 group/array. -""" -function _synthetic_zmetadata(path::String) - isfile(joinpath(path, "zarr.json")) && return nothing - is_v2 = isfile(joinpath(path, ".zgroup")) || isfile(joinpath(path, ".zarray")) - is_v2 || return nothing - - meta_path = joinpath(path, ".zmetadata") - if isfile(meta_path) - return read(meta_path) +function _read_consolidated_metadata(path::String) + zarr_json = joinpath(path, "zarr.json") + if isfile(zarr_json) + root = Zarr.JSON.parsefile(zarr_json; dicttype=Dict{String,Any}) + if haskey(root, "consolidated_metadata") + @info "Reading consolidated metadata from zarr.json" path + return read(zarr_json), "zarr.json" + end + return nothing, "zarr.json" end + zmetadata = joinpath(path, ".zmetadata") + if isfile(zmetadata) + @info "Reading consolidated metadata from .zmetadata" path + return read(zmetadata), ".zmetadata" + end + return nothing, ".zmetadata" +end +""" + _generate_metadata(path) -> Union{Vector{UInt8}, Nothing} +""" +function _generate_metadata(path::String) + @info "Serving synthesized .zmetadata for unconsolidated Zarr v2 store" path store = DirectoryStore(path) d = Dict{String, Any}() Zarr.consolidate_metadata(store, d, "") @@ -39,9 +50,7 @@ synthesized `.zmetadata` so HTTP clients can discover the hierarchy without dire """ function serve_zarr(path::String; host::String = "127.0.0.1") path = abspath(path) - synthetic_zmetadata = _synthetic_zmetadata(path) - !isnothing(synthetic_zmetadata) && - @info "Serving synthesized .zmetadata for unconsolidated Zarr v2 store" path + cons_metadata, cons_key = _get_metadata(path) CORS_HEADERS = [ "Access-Control-Allow-Origin" => "*", @@ -62,14 +71,18 @@ function serve_zarr(path::String; host::String = "127.0.0.1") return HTTP.Response(403, CORS_HEADERS, "Forbidden") end - if rel == ".zmetadata" && !isnothing(synthetic_zmetadata) + if !isnothing(cons_key) && rel == cons_key && !isnothing(cons_metadata) + # @info "Serving consolidated metadata" rel cons_key length(cons_metadata) + # verify it parses correctly + # @info "Content preview" preview=String(cons_metadata[1:min(200, end)]) + headers = [ CORS_HEADERS..., "Content-Type" => "application/json", - "Content-Length" => string(length(synthetic_zmetadata)), + "Content-Length" => string(length(cons_metadata)), ] req.method == "HEAD" && return HTTP.Response(200, headers) - return HTTP.Response(200, headers; body = synthetic_zmetadata) + return HTTP.Response(200, headers; body = cons_metadata) end isfile(fpath) || return HTTP.Response(404, CORS_HEADERS, "Not found") From d20a6771d62d7c78819a45961f1a04d0d23a3252 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Fri, 5 Jun 2026 15:34:51 +0200 Subject: [PATCH 2/4] generate only v2 metadata if not available --- src/serve_zarr.jl | 100 +++++++++++++++------------------------------- 1 file changed, 33 insertions(+), 67 deletions(-) diff --git a/src/serve_zarr.jl b/src/serve_zarr.jl index 680d54d..b031a4b 100644 --- a/src/serve_zarr.jl +++ b/src/serve_zarr.jl @@ -1,56 +1,11 @@ -function _get_metadata(path::String) - cons_meta, cons_key = _read_consolidated_metadata(path) - if !isnothing(cons_meta) - return cons_meta, cons_key - else - created_metadata = _generate_metadata(path) - return created_metadata, cons_key - end -end - -function _read_consolidated_metadata(path::String) - zarr_json = joinpath(path, "zarr.json") - if isfile(zarr_json) - root = Zarr.JSON.parsefile(zarr_json; dicttype=Dict{String,Any}) - if haskey(root, "consolidated_metadata") - @info "Reading consolidated metadata from zarr.json" path - return read(zarr_json), "zarr.json" - end - return nothing, "zarr.json" - end - zmetadata = joinpath(path, ".zmetadata") - if isfile(zmetadata) - @info "Reading consolidated metadata from .zmetadata" path - return read(zmetadata), ".zmetadata" - end - return nothing, ".zmetadata" -end - -""" - _generate_metadata(path) -> Union{Vector{UInt8}, Nothing} """ -function _generate_metadata(path::String) - @info "Serving synthesized .zmetadata for unconsolidated Zarr v2 store" path - store = DirectoryStore(path) - d = Dict{String, Any}() - Zarr.consolidate_metadata(store, d, "") - buf = IOBuffer() - Zarr.JSON.print(buf, Dict("metadata" => d, "zarr_consolidated_format" => 1), 4) - return take!(buf) -end + serve_zarr(path::String; host::String = "127.0.0.1") -""" - serve_zarr(path; host="127.0.0.1") - -Spin up a local HTTP server exposing a Zarr store at `path`. -Returns the URL string to pass to `browzarr(; store=...)`. - -Unconsolidated Zarr v2 stores (`.zgroup` / `.zattrs` only) are served with a -synthesized `.zmetadata` so HTTP clients can discover the hierarchy without directory listing. +Serve a local Zarr store as an HTTP server. """ function serve_zarr(path::String; host::String = "127.0.0.1") path = abspath(path) - cons_metadata, cons_key = _get_metadata(path) + synthetic_metadata = _has_consolidated_metadata(path) ? nothing : _generate_metadata(path) CORS_HEADERS = [ "Access-Control-Allow-Origin" => "*", @@ -71,18 +26,14 @@ function serve_zarr(path::String; host::String = "127.0.0.1") return HTTP.Response(403, CORS_HEADERS, "Forbidden") end - if !isnothing(cons_key) && rel == cons_key && !isnothing(cons_metadata) - # @info "Serving consolidated metadata" rel cons_key length(cons_metadata) - # verify it parses correctly - # @info "Content preview" preview=String(cons_metadata[1:min(200, end)]) - + if !isnothing(synthetic_metadata) && !isfile(fpath) headers = [ CORS_HEADERS..., "Content-Type" => "application/json", - "Content-Length" => string(length(cons_metadata)), + "Content-Length" => string(length(synthetic_metadata)), ] req.method == "HEAD" && return HTTP.Response(200, headers) - return HTTP.Response(200, headers; body = cons_metadata) + return HTTP.Response(200, headers; body = synthetic_metadata) end isfile(fpath) || return HTTP.Response(404, CORS_HEADERS, "Not found") @@ -112,26 +63,21 @@ function serve_zarr(path::String; host::String = "127.0.0.1") seek(io, start) read(io, len) end - return HTTP.Response( - 206, [ - headers..., - "Content-Range" => "bytes $start-$stop/$filesize", - "Content-Length" => string(len), - ]; body - ) + return HTTP.Response(206, [ + headers..., + "Content-Range" => "bytes $start-$stop/$filesize", + "Content-Length" => string(len), + ]; body) end end - return HTTP.Response(200, headers; body = read(fpath)) # ? or - # return HTTP.Response(200, headers; body = Mmap.mmap(fpath)) + return HTTP.Response(200, headers; body = read(fpath)) end - - # non-blocking server + s = _serve!(handler, host, 0) port = _port(s) server = _get_server(s) srv = ZarrServer(server, host, port, path) - lock(SERVERS_LOCK) do ZARR_SERVERS[port] = srv end @@ -140,6 +86,26 @@ function serve_zarr(path::String; host::String = "127.0.0.1") return srv end +function _has_consolidated_metadata(path::String) + zarr_json = joinpath(path, "zarr.json") + if isfile(zarr_json) + root = Zarr.JSON.parsefile(zarr_json; dicttype=Dict{String,Any}) + return haskey(root, "consolidated_metadata") + end + return isfile(joinpath(path, ".zmetadata")) +end + +function _generate_metadata(path::String) + store = DirectoryStore(path) + d = Dict{String, Any}() + Zarr.consolidate_metadata(store, d, "") # this should work for v2 and v3 stores, check support in Zarr.jl + isempty(d) && return nothing + @info "Serving synthesized metadata for unconsolidated Zarr store" path + buf = IOBuffer() + Zarr.JSON.print(buf, Dict("metadata" => d, "zarr_consolidated_format" => 1), 4) + return take!(buf) +end + function stop_zarr!(srv::ZarrServer) _forceclose(srv.server) return @info "Zarr server stopped" port = srv.port From 4b06f120059f1899f9b5483c772a4b44858ac439 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Tue, 9 Jun 2026 19:31:20 +0200 Subject: [PATCH 3/4] support only v2 synthetic metadata --- src/serve_zarr.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/serve_zarr.jl b/src/serve_zarr.jl index b031a4b..f1adefd 100644 --- a/src/serve_zarr.jl +++ b/src/serve_zarr.jl @@ -26,7 +26,7 @@ function serve_zarr(path::String; host::String = "127.0.0.1") return HTTP.Response(403, CORS_HEADERS, "Forbidden") end - if !isnothing(synthetic_metadata) && !isfile(fpath) + if rel == ".zmetadata" && !isnothing(synthetic_metadata) && !isfile(fpath) headers = [ CORS_HEADERS..., "Content-Type" => "application/json", @@ -90,12 +90,17 @@ function _has_consolidated_metadata(path::String) zarr_json = joinpath(path, "zarr.json") if isfile(zarr_json) root = Zarr.JSON.parsefile(zarr_json; dicttype=Dict{String,Any}) - return haskey(root, "consolidated_metadata") + cm = get(root, "consolidated_metadata", nothing) + isnothing(cm) && throw(ArgumentError("Missing consolidated_metadata in zarr.json")) + return true end return isfile(joinpath(path, ".zmetadata")) end function _generate_metadata(path::String) + is_v2 = isfile(joinpath(path, ".zgroup")) || isfile(joinpath(path, ".zarray")) + is_v2 || return nothing + store = DirectoryStore(path) d = Dict{String, Any}() Zarr.consolidate_metadata(store, d, "") # this should work for v2 and v3 stores, check support in Zarr.jl From 0868f3d4d83457586a99f3fc0d0dd7f32d343f25 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Tue, 9 Jun 2026 19:39:42 +0200 Subject: [PATCH 4/4] update test --- test/runtests.jl | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f62ac5e..c21fd9e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -56,17 +56,9 @@ end end end -@testset "serve_zarr does not synthesize .zmetadata when zarr.json exists" begin +@testset "serve_zarr rejects zarr.json without consolidated_metadata" begin mktempdir() do dir write(joinpath(dir, "zarr.json"), "{}") - srv = serve_zarr(dir) - url = "http://$(srv.host):$(srv.port)" - port = srv.port - try - meta = HTTP.get("$url/.zmetadata"; retry = false, status_exception = false) - @test meta.status == 404 - finally - stop_zarr!(port) - end + @test_throws ArgumentError serve_zarr(dir) end end