From ecd7e038a77e74a78ba72a3ff0648e6c26e34861 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 8 Jul 2026 18:52:00 +0200 Subject: [PATCH] Upgrade to LMDB 1.0 --- Project.toml | 2 +- README.md | 5 ++ docs/src/index.md | 4 + docs/src/lib/environments.md | 3 +- docs/src/lib/errors.md | 6 +- docs/src/lib/lowlevel.md | 25 ++++-- docs/src/man/environments.md | 20 +++-- docs/src/man/essentials.md | 6 +- docs/src/man/lowlevel.md | 11 ++- res/wrap.jl | 3 + src/environment.jl | 21 +++-- src/liblmdb.jl | 145 +++++++++++++++++++++++++++++++---- test/cursor.jl | 1 + test/environment.jl | 7 +- test/runtests.jl | 2 +- 15 files changed, 217 insertions(+), 44 deletions(-) diff --git a/Project.toml b/Project.toml index a7dda65..d1bd43f 100644 --- a/Project.toml +++ b/Project.toml @@ -12,5 +12,5 @@ LMDB_jll = "6206cf0b-f360-5984-af49-5437264c140e" [compat] CEnum = "0.4, 0.5" -LMDB_jll = "0.9.35" +LMDB_jll = "1.0" julia = "1.10" diff --git a/README.md b/README.md index 1c5df8d..4b8fdd4 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ Memory-Mapped Database. LMDB is an embedded, memory-mapped, ACID key-value store developed by Symas for OpenLDAP. It persists to disk while reading at near in-memory speeds. +LMDB 1.0 changed the on-disk file format. Databases created by LMDB 0.9 +must be exported with a 0.9 `mdb_dump` and imported with a 1.0 `mdb_load`; +LMDB 1.0 does not open 0.9 database files directly. + ```julia using Pkg; Pkg.add("LMDB") ``` @@ -125,3 +129,4 @@ ret == 0 || throw(LMDB.LMDBError(ret)) - LMDB upstream: - LMDB API docs: +- LMDB 1.0 upgrade notes: diff --git a/docs/src/index.md b/docs/src/index.md index 85024dd..1c68a68 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -7,6 +7,10 @@ LMDB is an embedded, memory-mapped, ACID key-value store developed by Symas for OpenLDAP. It persists to disk while reading at near in-memory speeds, limited only by the size of the virtual address space. +LMDB 1.0 changed the on-disk file format. Databases created by LMDB 0.9 +must be exported with a 0.9 `mdb_dump` and imported with a 1.0 +`mdb_load`; LMDB 1.0 does not open 0.9 database files directly. + ```julia using Pkg; Pkg.add("LMDB") ``` diff --git a/docs/src/lib/environments.md b/docs/src/lib/environments.md index 6ddccd0..b70340b 100644 --- a/docs/src/lib/environments.md +++ b/docs/src/lib/environments.md @@ -27,7 +27,8 @@ path ## Configuration `Environment` exposes its tunables through `getindex` / `setindex!` with -symbol keys (`:Flags`, `:Readers`, `:MapSize`, `:DBs`, `:KeySize`): +symbol keys (`:Flags`, `:Readers`, `:MapSize`, `:PageSize`, `:DBs`, +`:KeySize`): ```@docs Base.setindex!(::Environment, ::Integer, ::Symbol) diff --git a/docs/src/lib/errors.md b/docs/src/lib/errors.md index c5477c8..3520886 100644 --- a/docs/src/lib/errors.md +++ b/docs/src/lib/errors.md @@ -30,7 +30,11 @@ The rest (`MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_DBS_FULL`, `MDB_READERS_FULL`, `MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, -`MDB_BAD_VALSIZE`, `MDB_BAD_DBI`) live under the `LMDB.` prefix. +`MDB_BAD_VALSIZE`, `MDB_BAD_DBI`, `MDB_PROBLEM`, `MDB_BAD_CHECKSUM`, +`MDB_CRYPTO_FAIL`, `MDB_ENV_ENCRYPTION`, `MDB_TXN_PENDING`, +`MDB_CANT_ROLLBACK`, `MDB_DBIS_BUSY`, `MDB_SHORT_WRITE`, +`MDB_ENV_BUSY`, `MDB_IS_READONLY`, `MDB_ADDR_BUSY`) live under the +`LMDB.` prefix. ## Where errors come from at each surface diff --git a/docs/src/lib/lowlevel.md b/docs/src/lib/lowlevel.md index 9c8c6fb..78f012e 100644 --- a/docs/src/lib/lowlevel.md +++ b/docs/src/lib/lowlevel.md @@ -32,10 +32,12 @@ ret == 0 || throw(LMDB.LMDBError(ret)) Bindings that return non-status data (`mdb_strerror`, `mdb_version`, `mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`, -`mdb_env_get_userctx`, `mdb_cursor_txn`, `mdb_cursor_dbi`) and +`mdb_env_get_userctx`, `mdb_cursor_txn`, `mdb_cursor_dbi`, +`mdb_cursor_is_db`, `mdb_modload`) and `Cvoid`-returning ones (`mdb_env_close`, `mdb_dbi_close`, -`mdb_txn_abort`, `mdb_txn_reset`, `mdb_cursor_close`) are left bare, -since there is nothing to check. +`mdb_txn_abort`, `mdb_txn_reset`, `mdb_cursor_close`, +`mdb_modunload`, `mdb_modsetup`) are left bare, since there is +nothing to check. ## Customisation point: `MDBValueIO` @@ -69,10 +71,12 @@ LMDB.MDB_env # opaque LMDB.MDB_txn # opaque LMDB.MDB_cursor # opaque LMDB.MDB_dbi # = Cuint +LMDB.mdb_size_t # mapsize/page/transaction id integer type LMDB.MDB_val # struct { mv_size::Csize_t; mv_data::Ptr{Cvoid} } LMDB.MDB_stat # struct (page sizes, depth, leaf/branch/overflow page counts, entries) LMDB.MDB_envinfo # struct (mapaddr, mapsize, last_pgno, last_txnid, maxreaders, numreaders) LMDB.MDB_cursor_op # @cenum: MDB_FIRST … MDB_PREV_MULTIPLE (19 variants) +LMDB.MDB_crypto_funcs ``` ### Environment @@ -83,16 +87,19 @@ mdb_env_open mdb_env_close mdb_env_copy mdb_env_copy2 mdb_env_copyfd mdb_env_copyfd2 +mdb_env_incr_dump mdb_env_incr_dumpfd mdb_env_incr_loadfd mdb_env_stat mdb_env_info mdb_env_sync mdb_env_set_flags mdb_env_get_flags mdb_env_get_path mdb_env_get_fd -mdb_env_set_mapsize +mdb_env_set_mapsize mdb_env_set_pagesize mdb_env_set_maxreaders mdb_env_get_maxreaders mdb_env_set_maxdbs mdb_env_get_maxkeysize mdb_env_set_userctx mdb_env_get_userctx mdb_env_set_assert +mdb_env_set_encrypt mdb_env_set_checksum +mdb_env_rollback ``` ### Transaction @@ -101,7 +108,9 @@ mdb_env_set_assert mdb_txn_begin mdb_txn_env mdb_txn_id +mdb_txn_flags mdb_txn_commit +mdb_txn_prepare mdb_txn_abort mdb_txn_reset mdb_txn_renew @@ -135,18 +144,20 @@ mdb_cursor_close mdb_cursor_renew mdb_cursor_txn mdb_cursor_dbi +mdb_cursor_is_db mdb_cursor_get mdb_cursor_put mdb_cursor_del mdb_cursor_count ``` -### Comparators / readers / version +### Comparators / readers / crypto modules / version ```julia mdb_cmp mdb_dcmp mdb_reader_list mdb_reader_check +mdb_modload mdb_modunload mdb_modsetup mdb_version mdb_strerror ``` @@ -155,8 +166,8 @@ mdb_strerror | group | constants | |------|-----------| -| Env flags | `MDB_FIXEDMAP`, `MDB_NOSUBDIR`, `MDB_NOSYNC`, `MDB_RDONLY`, `MDB_NOMETASYNC`, `MDB_WRITEMAP`, `MDB_MAPASYNC`, `MDB_NOTLS`, `MDB_NOLOCK`, `MDB_NORDAHEAD`, `MDB_NOMEMINIT` | +| Env flags | `MDB_FIXEDMAP`, `MDB_ENCRYPT`, `MDB_NOSUBDIR`, `MDB_NOSYNC`, `MDB_RDONLY`, `MDB_NOMETASYNC`, `MDB_WRITEMAP`, `MDB_MAPASYNC`, `MDB_NOTLS`, `MDB_NOLOCK`, `MDB_NORDAHEAD`, `MDB_NOMEMINIT`, `MDB_PREVSNAPSHOT`, `MDB_REMAP_CHUNKS` | | DB flags | `MDB_REVERSEKEY`, `MDB_DUPSORT`, `MDB_INTEGERKEY`, `MDB_DUPFIXED`, `MDB_INTEGERDUP`, `MDB_REVERSEDUP`, `MDB_CREATE` | | Write flags | `MDB_NOOVERWRITE`, `MDB_NODUPDATA`, `MDB_CURRENT`, `MDB_RESERVE`, `MDB_APPEND`, `MDB_APPENDDUP`, `MDB_MULTIPLE` | | Copy flag | `MDB_CP_COMPACT` | -| Status codes | `MDB_SUCCESS=0`, `MDB_KEYEXIST`, `MDB_NOTFOUND`, `MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_MAP_FULL`, `MDB_DBS_FULL`, `MDB_READERS_FULL`, `MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, `MDB_BAD_VALSIZE`, `MDB_BAD_DBI` | +| Status codes | `MDB_SUCCESS=0`, `MDB_KEYEXIST`, `MDB_NOTFOUND`, `MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_MAP_FULL`, `MDB_DBS_FULL`, `MDB_READERS_FULL`, `MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, `MDB_BAD_VALSIZE`, `MDB_BAD_DBI`, `MDB_PROBLEM`, `MDB_BAD_CHECKSUM`, `MDB_CRYPTO_FAIL`, `MDB_ENV_ENCRYPTION`, `MDB_TXN_PENDING`, `MDB_CANT_ROLLBACK`, `MDB_DBIS_BUSY`, `MDB_SHORT_WRITE`, `MDB_ENV_BUSY`, `MDB_IS_READONLY`, `MDB_ADDR_BUSY` | diff --git a/docs/src/man/environments.md b/docs/src/man/environments.md index 4c89430..70172ff 100644 --- a/docs/src/man/environments.md +++ b/docs/src/man/environments.md @@ -15,6 +15,7 @@ the handle, set mapsize/maxreaders/maxdbs/flags, open the directory: ```julia env = Environment("/tmp/mydb"; mapsize = 1 << 30, # 1 GiB virtual map + pagesize = 8192, maxreaders = 510, maxdbs = 8, flags = LMDB.MDB_NOTLS) @@ -23,14 +24,15 @@ env = Environment("/tmp/mydb"; mapsize = 1 << 30, # 1 GiB virtual map If anything fails on the way through, the half-open env is closed before the exception propagates. -After the env is open, `[:Flags]` / `[:Readers]` / `[:MapSize]` / -`[:DBs]` setindex! keys map to `mdb_env_set_flags` / -`mdb_env_set_maxreaders` / `mdb_env_set_mapsize` / `mdb_env_set_maxdbs`, -and `set!` / `unset!` flip individual flag bits. +Before the env is open, `pagesize` maps to `mdb_env_set_pagesize`. +The `[:Flags]` / `[:Readers]` / `[:MapSize]` / `[:PageSize]` / `[:DBs]` +setindex! keys map to `mdb_env_set_flags` / `mdb_env_set_maxreaders` / +`mdb_env_set_mapsize` / `mdb_env_set_pagesize` / `mdb_env_set_maxdbs`, +and `set!` / `unset!` flip individual flag bits after open. `getindex` exposes a few read-only views: `env[:Flags]`, -`env[:Readers]`, and `env[:KeySize]` (the maximum key length, fixed at -compile time of the bundled `LMDB_jll`). +`env[:Readers]`, and `env[:KeySize]` (the maximum key length for the +environment's configured page size). The do-block form `Environment(f, path; kwargs...)` opens the env, calls `f(env)`, and closes on the way out: @@ -66,6 +68,12 @@ LMDB.MDB_RDONLY)` on an open env will return `EINVAL`. on-disk size. Pick a generous power of two (say, 1 GiB or 8 GiB) up front. The on-disk file grows incrementally as data is written. +LMDB 1.0 also lets you choose the DB page size with `pagesize`. The +default is the OS page size; larger values can allow larger keys and +`MDB_DUPSORT` data items. After opening, `stat(env).psize` reports the +actual page size and `env[:KeySize]` reports the corresponding maximum +key size. + If a write txn would exceed `mapsize`, LMDB returns `MDB_MAP_FULL`. To recover, close the env, raise `mapsize`, and reopen. The database itself does not need rewriting. diff --git a/docs/src/man/essentials.md b/docs/src/man/essentials.md index 16a84f2..553abf3 100644 --- a/docs/src/man/essentials.md +++ b/docs/src/man/essentials.md @@ -10,9 +10,13 @@ After importing LMDB.jl, you can immediately query the bundled library: julia> using LMDB julia> LMDB.version() -(v"0.9.33", "LMDB 0.9.33: (May 21, 2024)") +v"1.0.0" ``` +LMDB 1.0 changed the on-disk file format. Databases created by LMDB +0.9 must be exported with a 0.9 `mdb_dump` and imported with a 1.0 +`mdb_load`; LMDB 1.0 does not open 0.9 database files directly. + ## A complete example The easiest entry point is the [`LMDBDict`](@ref), a persistent diff --git a/docs/src/man/lowlevel.md b/docs/src/man/lowlevel.md index 37b6037..52a55f7 100644 --- a/docs/src/man/lowlevel.md +++ b/docs/src/man/lowlevel.md @@ -39,9 +39,11 @@ This is exactly the pattern `get(txn, dbi, key, T, default)` uses internally. Bindings that don't return a status (`mdb_strerror`, `mdb_version`, `mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`, -`mdb_cursor_txn`, `mdb_cursor_dbi`) and `Cvoid`-returning ones +`mdb_cursor_txn`, `mdb_cursor_dbi`, `mdb_cursor_is_db`, `mdb_modload`) +and `Cvoid`-returning ones (`mdb_env_close`, `mdb_dbi_close`, `mdb_txn_abort`, `mdb_txn_reset`, -`mdb_cursor_close`) are left bare; there is nothing to check. +`mdb_cursor_close`, `mdb_modunload`, `mdb_modsetup`) are left bare; +there is nothing to check. ## ccall glue: passing values to `Ptr{MDB_val}` @@ -148,3 +150,8 @@ Julia wrappers deliberately don't include them: - `MDB_GET_MULTIPLE` / `MDB_NEXT_MULTIPLE` cursor ops: reachable by passing the constant directly to `LMDB.mdb_cursor_get`. Useful with `MDB_DUPFIXED` databases for batched reads. +- LMDB 1.0 incremental backup, encryption/checksum hooks, and + two-phase-commit helpers are exposed as raw bindings: + `mdb_env_incr_dump`, `mdb_env_incr_dumpfd`, `mdb_env_incr_loadfd`, + `mdb_env_set_encrypt`, `mdb_env_set_checksum`, `mdb_txn_prepare`, + and `mdb_env_rollback`. diff --git a/res/wrap.jl b/res/wrap.jl index b3908c6..16789f7 100644 --- a/res/wrap.jl +++ b/res/wrap.jl @@ -16,6 +16,7 @@ const UNCHECKED_CINT = ( "mdb_env_get_maxkeysize", # returns the maximum key size, not a status. "mdb_cmp", # returns the comparison result (-1/0/1). "mdb_dcmp", # idem for dup-data. + "mdb_cursor_is_db", # returns a boolean int. ) # Replace `function $name(args)\n @ccall ...::Cint\nend` with @@ -42,6 +43,8 @@ function postprocess!(path::AbstractString) end write(out, SubString(src, last, lastindex(src))) src = String(take!(out)) + src = replace(src, r"^const MDB_SIZE_MAX = SIZE_MAX$"m => + "const MDB_SIZE_MAX = typemax(mdb_size_t)") # Clang.Generators always emits `export $jll_pkg_name` next to its # `using` line; strip it so LMDB_jll stays an implementation detail. src = replace(src, r"^export LMDB_jll\n"m => "") diff --git a/src/environment.jl b/src/environment.jl index f5c61fc..3b6b014 100644 --- a/src/environment.jl +++ b/src/environment.jl @@ -25,19 +25,22 @@ path(env::Environment) = env.path isopen(env::Environment) = env.handle != C_NULL """ - Environment(path::AbstractString; mapsize=nothing, maxreaders=nothing, - maxdbs=nothing, flags=0, mode=0o755) -> Environment + Environment(path::AbstractString; mapsize=nothing, pagesize=nothing, + maxreaders=nothing, maxdbs=nothing, flags=0, + mode=0o755) -> Environment Open the LMDB environment at `path`. The directory must already exist and be writable. The configuration kwargs go through -`mdb_env_set_mapsize`, `mdb_env_set_maxreaders`, and `mdb_env_set_maxdbs`; -`flags` is forwarded to `mdb_env_open`. If anything fails along the -way, the partially-built env is closed before rethrowing. +`mdb_env_set_mapsize`, `mdb_env_set_pagesize`, +`mdb_env_set_maxreaders`, and `mdb_env_set_maxdbs`; `flags` is +forwarded to `mdb_env_open`. If anything fails along the way, the +partially-built env is closed before rethrowing. Matches py-lmdb's `Environment(path, **kwargs)` and lmdb-rs's `EnvironmentBuilder.open(path)`. """ function Environment(path::AbstractString; mapsize::Union{Integer,Nothing} = nothing, + pagesize::Union{Integer,Nothing} = nothing, maxreaders::Union{Integer,Nothing} = nothing, maxdbs::Union{Integer,Nothing} = nothing, flags::Integer = zero(Cuint), @@ -47,6 +50,7 @@ function Environment(path::AbstractString; mapsize::Union{Integer,Nothing} = not env = Environment(env_ref[], "", WeakRef[]) finalizer(close, env) try + pagesize === nothing || (env[:PageSize] = pagesize) mapsize === nothing || (env[:MapSize] = mapsize) maxreaders === nothing || (env[:Readers] = maxreaders) maxdbs === nothing || (env[:DBs] = maxdbs) @@ -124,6 +128,7 @@ end * Flags * Readers * MapSize + * PageSize * DBs * `value` parameter value @@ -137,11 +142,13 @@ function setindex!(env::Environment, val::Integer, option::Symbol) elseif option == :MapSize # MDB_env_set_mapsize takes a size_t; using Cuint truncates >4 GiB # maps on 64-bit platforms (issue #38, PRs #37 / #40). - mdb_env_set_mapsize(env, Csize_t(val)) + mdb_env_set_mapsize(env, mdb_size_t(val)) + elseif option == :PageSize + mdb_env_set_pagesize(env, Cint(val)) elseif option == :DBs mdb_env_set_maxdbs(env, Cuint(val)) else - throw(ArgumentError("Unsupported environment option :$option (supported: :Flags, :Readers, :MapSize, :DBs)")) + throw(ArgumentError("Unsupported environment option :$option (supported: :Flags, :Readers, :MapSize, :PageSize, :DBs)")) end end diff --git a/src/liblmdb.jl b/src/liblmdb.jl index 1706d94..7ba0e69 100644 --- a/src/liblmdb.jl +++ b/src/liblmdb.jl @@ -26,6 +26,8 @@ end const mdb_mode_t = mode_t +const mdb_size_t = Csize_t + const mdb_filehandle_t = Cint mutable struct MDB_cursor end @@ -41,6 +43,12 @@ const MDB_cmp_func = Cvoid # typedef void ( MDB_rel_func ) ( MDB_val * item , void * oldptr , void * newptr , void * relctx ) const MDB_rel_func = Cvoid +# typedef int ( MDB_enc_func ) ( const MDB_val * src , MDB_val * dst , const MDB_val * key , int encdec ) +const MDB_enc_func = Cvoid + +# typedef void ( MDB_sum_func ) ( const MDB_val * src , MDB_val * dst , const MDB_val * key ) +const MDB_sum_func = Cvoid + @cenum MDB_cursor_op::UInt32 begin MDB_FIRST = 0 MDB_FIRST_DUP = 1 @@ -66,17 +74,17 @@ end struct MDB_stat ms_psize::Cuint ms_depth::Cuint - ms_branch_pages::Csize_t - ms_leaf_pages::Csize_t - ms_overflow_pages::Csize_t - ms_entries::Csize_t + ms_branch_pages::mdb_size_t + ms_leaf_pages::mdb_size_t + ms_overflow_pages::mdb_size_t + ms_entries::mdb_size_t end struct MDB_envinfo me_mapaddr::Ptr{Cvoid} - me_mapsize::Csize_t - me_last_pgno::Csize_t - me_last_txnid::Csize_t + me_mapsize::mdb_size_t + me_last_pgno::mdb_size_t + me_last_txnid::mdb_size_t me_maxreaders::Cuint me_numreaders::Cuint end @@ -116,6 +124,19 @@ end flags::Cuint)::Cint end +@checked function mdb_env_incr_dumpfd(env, fd, txnid) + @ccall liblmdb.mdb_env_incr_dumpfd(env::Ptr{MDB_env}, fd::mdb_filehandle_t, + txnid::Csize_t)::Cint +end + +@checked function mdb_env_incr_dump(env, path, txnid) + @ccall liblmdb.mdb_env_incr_dump(env::Ptr{MDB_env}, path::Cstring, txnid::Csize_t)::Cint +end + +@checked function mdb_env_incr_loadfd(env, fd) + @ccall liblmdb.mdb_env_incr_loadfd(env::Ptr{MDB_env}, fd::mdb_filehandle_t)::Cint +end + @checked function mdb_env_stat(env, stat) @ccall liblmdb.mdb_env_stat(env::Ptr{MDB_env}, stat::Ptr{MDB_stat})::Cint end @@ -149,7 +170,11 @@ end end @checked function mdb_env_set_mapsize(env, size) - @ccall liblmdb.mdb_env_set_mapsize(env::Ptr{MDB_env}, size::Csize_t)::Cint + @ccall liblmdb.mdb_env_set_mapsize(env::Ptr{MDB_env}, size::mdb_size_t)::Cint +end + +@checked function mdb_env_set_pagesize(env, size) + @ccall liblmdb.mdb_env_set_pagesize(env::Ptr{MDB_env}, size::Cint)::Cint end @checked function mdb_env_set_maxreaders(env, readers) @@ -183,6 +208,16 @@ const MDB_assert_func = Cvoid @ccall liblmdb.mdb_env_set_assert(env::Ptr{MDB_env}, func::Ptr{MDB_assert_func})::Cint end +@checked function mdb_env_set_encrypt(env, func, key, size) + @ccall liblmdb.mdb_env_set_encrypt(env::Ptr{MDB_env}, func::Ptr{MDB_enc_func}, + key::Ptr{MDB_val}, size::Cuint)::Cint +end + +@checked function mdb_env_set_checksum(env, func, size) + @ccall liblmdb.mdb_env_set_checksum(env::Ptr{MDB_env}, func::Ptr{MDB_sum_func}, + size::Cuint)::Cint +end + @checked function mdb_txn_begin(env, parent, flags, txn) @ccall liblmdb.mdb_txn_begin(env::Ptr{MDB_env}, parent::Ptr{MDB_txn}, flags::Cuint, txn::Ptr{Ptr{MDB_txn}})::Cint @@ -193,13 +228,25 @@ function mdb_txn_env(txn) end function mdb_txn_id(txn) - @ccall liblmdb.mdb_txn_id(txn::Ptr{MDB_txn})::Csize_t + @ccall liblmdb.mdb_txn_id(txn::Ptr{MDB_txn})::mdb_size_t +end + +@checked function mdb_txn_flags(txn, flags) + @ccall liblmdb.mdb_txn_flags(txn::Ptr{MDB_txn}, flags::Ptr{Cuint})::Cint end @checked function mdb_txn_commit(txn) @ccall liblmdb.mdb_txn_commit(txn::Ptr{MDB_txn})::Cint end +@checked function mdb_txn_prepare(txn) + @ccall liblmdb.mdb_txn_prepare(txn::Ptr{MDB_txn})::Cint +end + +@checked function mdb_env_rollback(env, txnid) + @ccall liblmdb.mdb_env_rollback(env::Ptr{MDB_env}, txnid::mdb_size_t)::Cint +end + function mdb_txn_abort(txn) @ccall liblmdb.mdb_txn_abort(txn::Ptr{MDB_txn})::Cvoid end @@ -279,6 +326,10 @@ function mdb_cursor_dbi(cursor) @ccall liblmdb.mdb_cursor_dbi(cursor::Ptr{MDB_cursor})::MDB_dbi end +function mdb_cursor_is_db(cursor) + @ccall liblmdb.mdb_cursor_is_db(cursor::Ptr{MDB_cursor})::Cint +end + @checked function mdb_cursor_get(cursor, key, data, op) @ccall liblmdb.mdb_cursor_get(cursor::Ptr{MDB_cursor}, key::Ptr{MDB_val}, data::Ptr{MDB_val}, op::MDB_cursor_op)::Cint @@ -294,7 +345,7 @@ end end @checked function mdb_cursor_count(cursor, countp) - @ccall liblmdb.mdb_cursor_count(cursor::Ptr{MDB_cursor}, countp::Ptr{Csize_t})::Cint + @ccall liblmdb.mdb_cursor_count(cursor::Ptr{MDB_cursor}, countp::Ptr{mdb_size_t})::Cint end function mdb_cmp(txn, dbi, a, b) @@ -319,16 +370,54 @@ end @ccall liblmdb.mdb_reader_check(env::Ptr{MDB_env}, dead::Ptr{Cint})::Cint end -const MDB_VERSION_MAJOR = 0 +# typedef int ( MDB_str2key_func ) ( const char * passwd , MDB_val * key ) +const MDB_str2key_func = Cvoid + +struct MDB_crypto_funcs + mcf_str2key::Ptr{MDB_str2key_func} + mcf_encfunc::Ptr{MDB_enc_func} + mcf_sumfunc::Ptr{MDB_sum_func} + mcf_keysize::Cint + mcf_esumsize::Cint + mcf_sumsize::Cint +end + +# typedef MDB_crypto_funcs * ( MDB_crypto_hooks ) ( void ) +const MDB_crypto_hooks = Cvoid + +function mdb_modload(file, symname, mcf_ptr, errmsg) + @ccall liblmdb.mdb_modload(file::Cstring, symname::Cstring, + mcf_ptr::Ptr{Ptr{MDB_crypto_funcs}}, + errmsg::Ptr{Cstring})::Ptr{Cvoid} +end + +function mdb_modunload(handle) + @ccall liblmdb.mdb_modunload(handle::Ptr{Cvoid})::Cvoid +end + +function mdb_modsetup(env, mcf_ptr, passphrase) + @ccall liblmdb.mdb_modsetup(env::Ptr{MDB_env}, mcf_ptr::Ptr{MDB_crypto_funcs}, + passphrase::Cstring)::Cvoid +end -const MDB_VERSION_MINOR = 9 +const MDB_FMT_Z = "z" -const MDB_VERSION_PATCH = 33 +const MDB_RPAGE_CACHE = 1 -const MDB_VERSION_DATE = "May 21, 2024" +const MDB_SIZE_MAX = typemax(mdb_size_t) + +const MDB_VERSION_MAJOR = 1 + +const MDB_VERSION_MINOR = 0 + +const MDB_VERSION_PATCH = 0 + +const MDB_VERSION_DATE = "June 30, 2026" const MDB_FIXEDMAP = 0x01 +const MDB_ENCRYPT = Cuint(0x2000) + const MDB_NOSUBDIR = 0x4000 const MDB_NOSYNC = 0x00010000 @@ -349,6 +438,10 @@ const MDB_NORDAHEAD = 0x00800000 const MDB_NOMEMINIT = 0x01000000 +const MDB_PREVSNAPSHOT = 0x02000000 + +const MDB_REMAP_CHUNKS = 0x04000000 + const MDB_REVERSEKEY = 0x02 const MDB_DUPSORT = 0x04 @@ -421,4 +514,26 @@ const MDB_BAD_VALSIZE = -30781 const MDB_BAD_DBI = -30780 -const MDB_LAST_ERRCODE = MDB_BAD_DBI +const MDB_PROBLEM = -30779 + +const MDB_BAD_CHECKSUM = -30778 + +const MDB_CRYPTO_FAIL = -30777 + +const MDB_ENV_ENCRYPTION = -30776 + +const MDB_TXN_PENDING = -30775 + +const MDB_CANT_ROLLBACK = -30774 + +const MDB_DBIS_BUSY = -30773 + +const MDB_SHORT_WRITE = -30772 + +const MDB_ENV_BUSY = -30771 + +const MDB_IS_READONLY = -30770 + +const MDB_ADDR_BUSY = -30769 + +const MDB_LAST_ERRCODE = MDB_ADDR_BUSY diff --git a/test/cursor.jl b/test/cursor.jl index b73f6ee..39049bf 100644 --- a/test/cursor.jl +++ b/test/cursor.jl @@ -60,6 +60,7 @@ mktempdir() do dir LMDB.Cursor(txn, dbi) do cur @test LMDB.seek!(cur, String) == "a" + @test LMDB.mdb_cursor_is_db(cur) == 0 @test LMDB.value(cur, String) == "1" @test LMDB.key(cur, String) == "a" @test LMDB.item(cur, String, String) == ("a" => "1") diff --git a/test/environment.jl b/test/environment.jl index 5e42a9e..88587f9 100644 --- a/test/environment.jl +++ b/test/environment.jl @@ -6,7 +6,8 @@ mktempdir() do dir try @test isopen(env) @test env[:Readers] == 126 - @test env[:KeySize] == 511 + @test env[:KeySize] == LMDB.mdb_env_get_maxkeysize(env) + @test env[:KeySize] > 0 @test env[:Flags] == 0 # Manipulate flags via LMDB.set!/LMDB.unset! after open. @@ -45,12 +46,14 @@ end # High-level LMDB.Environment(path; ...) constructor. mktempdir() do dir big = Csize_t(8) * 1024^3 - env = LMDB.Environment(dir; mapsize = big, maxreaders = 42, maxdbs = 4, + env = LMDB.Environment(dir; mapsize = big, pagesize = 8192, + maxreaders = 42, maxdbs = 4, flags = LMDB.MDB_NOSYNC | LMDB.MDB_NOTLS) try @test isopen(env) @test env[:Readers] == 42 @test LMDB.info(env).mapsize == big + @test LMDB.stat(env).psize == 8192 @test LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) @test LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOTLS)) finally diff --git a/test/runtests.jl b/test/runtests.jl index 49fb415..e74d277 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,7 @@ using Test, LMDB @testset "LMDB" verbose=true begin - @test LMDB.version() >= v"0.9.35" + @test LMDB.version() >= v"1.0.0" # LMDBError ex = LMDBError(Cint(0))