diff --git a/.gitignore b/.gitignore index f9c40a1..bdb566f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ Manifest.toml *.nc *.zarr +dev/ \ No newline at end of file diff --git a/Project.toml b/Project.toml index 82608c9..532a9f1 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,7 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" [compat] -HTTP = "1.11.0, 2" +HTTP = "1, 2" LazyArtifacts = "1.10.11, 1.11.0" Sockets = "1.10.11, 1.11.0" Zarr = "0.9, 0.10.0" diff --git a/README.md b/README.md index 1917248..5303d37 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ browzarr(; store="/absolute/path/to/zarr_file.zarr") # local zarr directory browzarr(; store="https://s3.bucket.de:67/misc/out.zarr") # remote Zarr store ``` +List all `BrowzarrServer`s + +```julia +Browzarr.running_servers() +``` + To stop all running servers: ```julia @@ -58,7 +64,7 @@ To stop a server on a specific port: Browzarr.stop!(3000) ``` -### Setup Local Zarr Server +### Setup a Local Zarr Server You can pass directly the path to your local `zarr` directory ```julia @@ -74,3 +80,23 @@ store = Browzarr.serve_zarr("/absolute/path/to/zarr_file.zarr") # now launch it! browzarr(; store=store) ``` + +List all `ZarrServer`s + +```julia +Browzarr.running_zarr_servers() +``` + +To stop all running `ZarrServer`s: + +```julia +Browzarr.stop_all_zarr!() +``` + +To stop a `ZarrServer` on a specific port: + +```julia +Browzarr.stop_zarr!(16180) +``` + +--- diff --git a/src/Browzarr.jl b/src/Browzarr.jl index 25423f5..f4bb310 100644 --- a/src/Browzarr.jl +++ b/src/Browzarr.jl @@ -6,26 +6,65 @@ using HTTP using Zarr, Sockets using Zarr: DirectoryStore +# compat abstractions between HTTP 1.x and 2.x +const HTTP_V2 = pkgversion(HTTP) >= v"2.0" +const server_ = HTTP_V2 ? HTTP.Server : Sockets.TCPServer + +function _serve!(handler, host, port; kwargs...) + if HTTP_V2 + return HTTP.serve!(handler, host, port; kwargs...) + else + server = Sockets.listen(Sockets.getaddrinfo(host), port) + return server, errormonitor(@async HTTP.serve(handler, host, port, server = server)) + end +end + +_get_server(s) = HTTP_V2 ? s : first(s) + +function _port(s) + if HTTP_V2 + return HTTP.port(s) + else + _, port = getsockname(first(s)) + return port + end +end + +_forceclose(s) = HTTP_V2 ? HTTP.forceclose(s) : close(s) +_escapeuri(p) = HTTP_V2 ? HTTP.escapeuri(p) : HTTP.URIs.escapeuri(p) +_unescapeuri(p) = HTTP_V2 ? HTTP.unescapeuri(p) : HTTP.URIs.unescapeuri(p) +_uri(s) = HTTP_V2 ? HTTP.URI(s) : HTTP.URIs.URI(s) + struct BrowzarrServer - server::HTTP.Servers.Server + server::server_ host::String port::Int store::Union{String, Nothing} format::Union{String, Nothing} end +struct ZarrServer + server::server_ + host::String + port::Int + path::String +end + const SERVERS = Dict{Int, BrowzarrServer}() -const ZARR_SERVERS = Dict{Int, Sockets.TCPServer}() +const ZARR_SERVERS = Dict{Int, ZarrServer}() const SERVERS_LOCK = ReentrantLock() include("mimeTypes.jl") include("serve_zarr.jl") include("servers.jl") -function browzarr(; port::Union{Integer, Nothing} = nothing, open::Union{Bool, Nothing} = nothing, store::Union{String, Nothing} = nothing) +function browzarr(; port::Union{Integer, Nothing} = nothing, open::Union{Bool, Nothing} = nothing, store::Union{String, ZarrServer, Nothing} = nothing) - if !isnothing(store) && isdir(store) - store = serve_zarr(store) + if store isa ZarrServer + store = "http://$(store.host):$(store.port)" + elseif store isa String && isdir(store) + zarr_srv = serve_zarr(store) + store = "http://$(zarr_srv.host):$(zarr_srv.port)" wait_for_server(store) || error("Zarr server failed to start at $store") end @@ -49,4 +88,35 @@ function browzarr(; port::Union{Integer, Nothing} = nothing, open::Union{Bool, N return srv end +function Base.show(io::IO, srv::BrowzarrServer) + color = get(io, :color, false) + + styled(c, s) = color ? "\e[$(c)m$(s)\e[0m" : string(s) + key(s) = styled("1;36", s) # bold cyan + str(s) = styled("32", s) # green + num(s) = styled("33", s) # yellow + sym(s) = styled("38;5;208", s) # orange + bold(s) = styled("1", s) + + print(io, bold("BrowzarrServer"), "(") + print(io, key("host"), "=", str(srv.host), ", ") + print(io, key("port"), "=", num(srv.port), ", ") + print(io, key("store"), "=", sym(srv.store), ", ") + print(io, key("format"), "=", sym(srv.format), ")") +end + +function Base.show(io::IO, srv::ZarrServer) + color = get(io, :color, false) + styled(c, s) = color ? "\e[$(c)m$(s)\e[0m" : string(s) + key(s) = styled("1;36", s) # bold cyan + str(s) = styled("32", s) # green + num(s) = styled("33", s) # yellow + sym(s) = styled("38;5;208", s) # orange + bold(s) = styled("1", s) + print(io, bold("ZarrServer"), "(") + print(io, key("host"), "=", str(srv.host), ", ") + print(io, key("port"), "=", num(srv.port), ", ") + print(io, key("path"), "=", sym(srv.path), ")") +end + end diff --git a/src/serve_zarr.jl b/src/serve_zarr.jl index a2c2708..e56caf9 100644 --- a/src/serve_zarr.jl +++ b/src/serve_zarr.jl @@ -11,11 +11,15 @@ Returns `nothing` when `.zmetadata` already exists on disk, a root `zarr.json` e or the directory is not a Zarr v2 group/array. """ function _synthetic_zmetadata(path::String) - meta_path = joinpath(path, ".zmetadata") - isfile(meta_path) && return nothing isfile(joinpath(path, "zarr.json")) && return nothing is_v2 = isfile(joinpath(path, ".zgroup")) || isfile(joinpath(path, ".zarray")) is_v2 || return nothing + + meta_path = joinpath(path, ".zmetadata") + if isfile(meta_path) + return read(meta_path) + end + store = DirectoryStore(path) d = Dict{String, Any}() Zarr.consolidate_metadata(store, d, "") @@ -39,9 +43,6 @@ function serve_zarr(path::String; host::String = "127.0.0.1") !isnothing(synthetic_zmetadata) && @info "Serving synthesized .zmetadata for unconsolidated Zarr v2 store" path - server = Sockets.listen(Sockets.getaddrinfo(host), 0) # OS picks a free port - _, port = getsockname(server) - CORS_HEADERS = [ "Access-Control-Allow-Origin" => "*", "Access-Control-Allow-Methods" => "GET, OPTIONS", @@ -51,7 +52,7 @@ function serve_zarr(path::String; host::String = "127.0.0.1") function handler(req::HTTP.Request) req.method == "OPTIONS" && return HTTP.Response(200, CORS_HEADERS) - target_path = HTTP.URIs.unescapeuri(HTTP.URIs.URI(req.target).path) + target_path = _unescapeuri(_uri(req.target).path) rel = lstrip(target_path, '/') contains(rel, "..") && return HTTP.Response(403, CORS_HEADERS, "Forbidden") fpath = abspath(joinpath(path, rel)) @@ -107,17 +108,28 @@ function serve_zarr(path::String; host::String = "127.0.0.1") ) end end - - return HTTP.Response(200, headers, open(fpath, "r")) + return HTTP.Response(200, headers; body = read(fpath)) # ? or + # return HTTP.Response(200, headers; body = Mmap.mmap(fpath)) end + + # non-blocking server + s = _serve!(handler, host, 0) + port = _port(s) + server = _get_server(s) + + srv = ZarrServer(server, host, port, path) - errormonitor(@async HTTP.serve(handler, host, port, server = server)) lock(SERVERS_LOCK) do - ZARR_SERVERS[port] = server + ZARR_SERVERS[port] = srv end atexit(() -> stop_zarr!(port)) - @info "Zarr HTTP server started" path url = "http://$host:$port" - return "http://$host:$port" + @info "Zarr HTTP server started" url = "http://$host:$port" + return srv +end + +function stop_zarr!(srv::ZarrServer) + _forceclose(srv.server) + return @info "Zarr server stopped" port = srv.port end """ @@ -129,8 +141,36 @@ function stop_zarr!(port::Integer) return lock(SERVERS_LOCK) do srv = get(ZARR_SERVERS, port, nothing) srv === nothing && return - close(srv) + stop_zarr!(srv) pop!(ZARR_SERVERS, port, nothing) - @info "Zarr server stopped" port end end + +""" + stop_all_zarr!() + +Stop all Zarr HTTP servers. +""" +function stop_all_zarr!() + servers = lock(SERVERS_LOCK) do + s = collect(values(ZARR_SERVERS)) + empty!(ZARR_SERVERS) + s + end + for srv in servers + try + stop_zarr!(srv) + catch e + @warn "Failed to stop Zarr server" port = srv.port exception = e + end + end + return @info "All Zarr servers stopped" +end + +running_zarr_servers() = lock(SERVERS_LOCK) do + collect(values(ZARR_SERVERS)) +end + +get_zarr_server(port::Integer) = lock(SERVERS_LOCK) do + get(ZARR_SERVERS, port, nothing) +end diff --git a/src/servers.jl b/src/servers.jl index 2a07120..83278c7 100644 --- a/src/servers.jl +++ b/src/servers.jl @@ -1,14 +1,17 @@ function start_browzarr(; port::Union{Integer, Nothing} = nothing, host::String = "127.0.0.1", store::Union{String, Nothing} = nothing) return lock(SERVERS_LOCK) do - # Bind to requested port or let OS pick a free one - tcp = Sockets.listen(Sockets.getaddrinfo(host), isnothing(port) ? 0 : port) - _, p = getsockname(tcp) - - haskey(SERVERS, p) && error("Server already running on port $p") # The `browzarr` npm tarball unpacks under `package/` and the static build lives in `out/`. dir = joinpath(artifact"Browzarr", "package", "out") handler = static_handler(dir, store) - server = HTTP.serve!(handler, tcp) + + !isnothing(port) && haskey(SERVERS, port) && error("Server already running on port $port") + + s = _serve!( + handler, host, isnothing(port) ? 0 : port; + listenany = isnothing(port), + ) + p = _port(s) + server = _get_server(s) srv = BrowzarrServer(server, host, p, store, detect_format(store)) SERVERS[p] = srv @@ -29,7 +32,7 @@ function detect_format(store::Union{String, Nothing}) end function stop!(srv::BrowzarrServer) - close(srv.server) + _forceclose(srv.server) return @info "Browzarr server stopped" port = srv.port end @@ -92,7 +95,7 @@ end function server_url(srv::BrowzarrServer) base = "http://$(srv.host):$(srv.port)" isnothing(srv.store) && return base - store = startswith(srv.store, "http") ? srv.store : HTTP.URIs.escapeuri(srv.store) + store = startswith(srv.store, "http") ? srv.store : _escapeuri(srv.store) url = "$base/?store=$store" !isnothing(srv.format) && (url *= "&format=$(srv.format)") return url @@ -143,10 +146,11 @@ function _display_vscode(srv::BrowzarrServer) end function wait_for_server(host, port; timeout = 10.0) + url = "http://$host:$port" deadline = time() + timeout while time() < deadline try - close(HTTP.connect(host, port)) + HTTP.get(url; request_timeout = 1, retry = false, status_exception = false) return true catch sleep(0.05) @@ -157,13 +161,10 @@ function wait_for_server(host, port; timeout = 10.0) end function wait_for_server(url::String; timeout = 10.0) - uri = HTTP.URIs.URI(url) - host = uri.host - port = isempty(uri.port) ? (uri.scheme == "https" ? 443 : 80) : parse(Int, uri.port) deadline = time() + timeout while time() < deadline try - HTTP.get(url; readtimeout = 1, retry = false, status_exception = false) + HTTP.get(url; request_timeout = 1, retry = false, status_exception = false) return true catch sleep(0.05) diff --git a/test/runtests.jl b/test/runtests.jl index 0034f20..f62ac5e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -36,8 +36,9 @@ end zcreate(Float32, g, "temp", 4, 4) @test !isfile(joinpath(dir, ".zmetadata")) - url = serve_zarr(dir) - port = parse(Int, HTTP.URIs.URI(url).port) + srv = serve_zarr(dir) + url = "http://$(srv.host):$(srv.port)" + port = srv.port Browzarr.wait_for_server(url) try meta = HTTP.get("$url/.zmetadata"; retry = false) @@ -58,8 +59,9 @@ end @testset "serve_zarr does not synthesize .zmetadata when zarr.json exists" begin mktempdir() do dir write(joinpath(dir, "zarr.json"), "{}") - url = serve_zarr(dir) - port = parse(Int, HTTP.URIs.URI(url).port) + srv = serve_zarr(dir) + url = "http://$(srv.host):$(srv.port)" + port = srv.port try meta = HTTP.get("$url/.zmetadata"; retry = false, status_exception = false) @test meta.status == 404