diff --git a/docs/2.drivers/fs.md b/docs/2.drivers/fs.md index f227d3e50..87e170cc7 100644 --- a/docs/2.drivers/fs.md +++ b/docs/2.drivers/fs.md @@ -28,6 +28,7 @@ const storage = createStorage({ - `base`: Base directory to isolate operations on this directory - `ignore`: Ignore patterns for watch - `watchOptions`: Additional [chokidar](https://github.com/paulmillr/chokidar) options. +- `dataSuffix`: Suffix appended to on-disk filenames to prevent key/namespace collisions (see [below](#preventing-keynamespace-collisions)). ## Node.js Filesystem (Lite) @@ -46,3 +47,43 @@ const storage = createStorage({ - `base`: Base directory to isolate operations on this directory - `ignore`: Optional callback function `(path: string) => boolean` +- `dataSuffix`: Suffix appended to on-disk filenames to prevent key/namespace collisions (see [below](#preventing-keynamespace-collisions)). + +## Preventing key/namespace collisions + +Both `fs` drivers map the `:` key separator to `/` on disk, so a key is also a +directory prefix of its sub-keys. That means a key and its own namespace cannot +normally coexist: writing `foo` creates the file `/foo`, and then writing +`foo:bar` needs `/foo` to be a directory, which fails (`ENOTDIR` on +POSIX, `ENOENT` on Windows). + +Set `dataSuffix` (for example `".data"`) to store every value in a suffixed +leaf file so values and namespaces never collide: + +```js +import { createStorage } from "unstorage"; +import fsDriver from "unstorage/drivers/fs"; + +const storage = createStorage({ + driver: fsDriver({ base: "./data", dataSuffix: ".data" }), +}); + +await storage.setItem("foo", "a"); // -> ./data/foo.data +await storage.setItem("foo:bar", "b"); // -> ./data/foo/bar.data (no collision) +``` + +The suffix is transparently added on writes and stripped from the keys returned +by `getKeys()` and watch callbacks. It must be a non-empty string other than `.` +or `..` and must not contain `/`, `\` or `:`. It defaults to `undefined` +(disabled) for backward compatibility. Pick a suffix your keys do not end with: a +key that itself ends in the suffix can still collide with a same-named namespace +(for example `foo` and `foo.data:bar` when the suffix is `.data`). + +::warning +`dataSuffix` changes the on-disk layout. Enabling it on a directory that already +holds un-suffixed files makes those files invisible to the driver (they are +skipped by `getItem`/`getKeys`). Treat enabling it as starting a fresh data +generation, or migrate existing files by renaming them to include the suffix. +`clear()` still removes the entire base directory, including any non-suffixed +files it otherwise ignores. +:: diff --git a/src/drivers/fs-lite.ts b/src/drivers/fs-lite.ts index 79a6cdc85..12d5fcc46 100644 --- a/src/drivers/fs-lite.ts +++ b/src/drivers/fs-lite.ts @@ -8,6 +8,15 @@ export interface FSStorageOptions { ignore?: (path: string) => boolean; readOnly?: boolean; noClear?: boolean; + /** + * Suffix appended to all stored file paths on disk. + * + * When set (e.g. `".data"`), key `foo` is stored as `foo.data` and + * key `foo:bar` as `foo/bar.data`. This prevents file/directory + * collisions that occur when both `foo` and `foo:bar` exist as keys + * (the plain key would need `foo` to be both a file *and* a directory). + */ + dataSuffix?: string; } const PATH_TRAVERSE_RE = /\.\.:|\.\.$/; @@ -20,6 +29,25 @@ const driver: DriverFactory = (opts = {}) => { } opts.base = resolve(opts.base); + const dataSuffix = opts.dataSuffix; + + // `dataSuffix` is appended to leaf filenames, so it must not introduce path + // separators (which would create nesting) or the ":" key separator (which + // would break the round-trip in getKeys/watch). Reject those up front. + if ( + dataSuffix !== undefined && + (typeof dataSuffix !== "string" || + dataSuffix.length === 0 || + dataSuffix === "." || + dataSuffix === ".." || + /[/\\:]/.test(dataSuffix)) + ) { + throw createError( + DRIVER_NAME, + `Invalid dataSuffix: ${JSON.stringify(dataSuffix)}. It must be a non-empty string other than "." or ".." and must not contain "/", "\\" or ":".`, + ); + } + const r = (key: string) => { if (PATH_TRAVERSE_RE.test(key)) { throw createError( @@ -31,6 +59,18 @@ const driver: DriverFactory = (opts = {}) => { return resolved; }; + const rFile = (key: string) => { + const resolved = r(key); + if (!dataSuffix) { + return resolved; + } + // A root/empty key resolves to `base` itself; `base + suffix` would write a + // sibling *outside* base. Keep the suffixed file inside base instead. + return resolved === opts.base + ? join(opts.base!, dataSuffix) + : resolved + dataSuffix; + }; + return { name: DRIVER_NAME, options: opts, @@ -38,17 +78,17 @@ const driver: DriverFactory = (opts = {}) => { maxDepth: true, }, hasItem(key) { - return existsSync(r(key)); + return existsSync(rFile(key)); }, getItem(key) { - return readFile(r(key), "utf8"); + return readFile(rFile(key), "utf8"); }, getItemRaw(key) { - return readFile(r(key)); + return readFile(rFile(key)); }, async getMeta(key) { const { atime, mtime, size, birthtime, ctime } = await fsp - .stat(r(key)) + .stat(rFile(key)) .catch(() => ({}) as Stats); return { atime, mtime, size, birthtime, ctime }; }, @@ -56,22 +96,28 @@ const driver: DriverFactory = (opts = {}) => { if (opts.readOnly) { return; } - return writeFile(r(key), value, "utf8"); + return writeFile(rFile(key), value, "utf8"); }, setItemRaw(key, value) { if (opts.readOnly) { return; } - return writeFile(r(key), value); + return writeFile(rFile(key), value); }, removeItem(key) { if (opts.readOnly) { return; } - return unlink(r(key)) as Promise; + return unlink(rFile(key)) as Promise; }, - getKeys(_base, topts) { - return readdirRecursive(r("."), opts.ignore, topts?.maxDepth); + async getKeys(_base, topts) { + const keys = await readdirRecursive(r("."), opts.ignore, topts?.maxDepth); + if (dataSuffix) { + return keys + .filter((key) => key.endsWith(dataSuffix)) + .map((key) => key.slice(0, -dataSuffix.length)); + } + return keys; }, async clear() { if (opts.readOnly || opts.noClear) { diff --git a/src/drivers/fs.ts b/src/drivers/fs.ts index 3facd0c79..5d852ade9 100644 --- a/src/drivers/fs.ts +++ b/src/drivers/fs.ts @@ -17,6 +17,15 @@ export interface FSStorageOptions { readOnly?: boolean; noClear?: boolean; watchOptions?: ChokidarOptions; + /** + * Suffix appended to all stored file paths on disk. + * + * When set (e.g. `".data"`), key `foo` is stored as `foo.data` and + * key `foo:bar` as `foo/bar.data`. This prevents file/directory + * collisions that occur when both `foo` and `foo:bar` exist as keys + * (the plain key would need `foo` to be both a file *and* a directory). + */ + dataSuffix?: string; } const PATH_TRAVERSE_RE = /\.\.:|\.\.$/; @@ -41,6 +50,25 @@ const driver: DriverFactory = (userOptions = {}) => { }); }; + const dataSuffix = userOptions.dataSuffix; + + // `dataSuffix` is appended to leaf filenames, so it must not introduce path + // separators (which would create nesting) or the ":" key separator (which + // would break the round-trip in getKeys/watch). Reject those up front. + if ( + dataSuffix !== undefined && + (typeof dataSuffix !== "string" || + dataSuffix.length === 0 || + dataSuffix === "." || + dataSuffix === ".." || + /[/\\:]/.test(dataSuffix)) + ) { + throw createError( + DRIVER_NAME, + `Invalid dataSuffix: ${JSON.stringify(dataSuffix)}. It must be a non-empty string other than "." or ".." and must not contain "/", "\\" or ":".`, + ); + } + const r = (key: string) => { if (PATH_TRAVERSE_RE.test(key)) { throw createError( @@ -52,6 +80,16 @@ const driver: DriverFactory = (userOptions = {}) => { return resolved; }; + const rFile = (key: string) => { + const resolved = r(key); + if (!dataSuffix) { + return resolved; + } + // A root/empty key resolves to `base` itself; `base + suffix` would write a + // sibling *outside* base. Keep the suffixed file inside base instead. + return resolved === base ? join(base, dataSuffix) : resolved + dataSuffix; + }; + let _watcher: FSWatcher | undefined; const _unwatch = async () => { if (_watcher) { @@ -67,17 +105,17 @@ const driver: DriverFactory = (userOptions = {}) => { maxDepth: true, }, hasItem(key) { - return existsSync(r(key)); + return existsSync(rFile(key)); }, getItem(key) { - return readFile(r(key), "utf8"); + return readFile(rFile(key), "utf8"); }, getItemRaw(key) { - return readFile(r(key)); + return readFile(rFile(key)); }, async getMeta(key) { const { atime, mtime, size, birthtime, ctime } = await fsp - .stat(r(key)) + .stat(rFile(key)) .catch(() => ({}) as Stats); return { atime, mtime, size, birthtime, ctime }; }, @@ -85,22 +123,28 @@ const driver: DriverFactory = (userOptions = {}) => { if (userOptions.readOnly) { return; } - return writeFile(r(key), value, "utf8"); + return writeFile(rFile(key), value, "utf8"); }, setItemRaw(key, value) { if (userOptions.readOnly) { return; } - return writeFile(r(key), value); + return writeFile(rFile(key), value); }, removeItem(key) { if (userOptions.readOnly) { return; } - return unlink(r(key)) as Promise; + return unlink(rFile(key)) as Promise; }, - getKeys(_base, topts) { - return readdirRecursive(r("."), ignore, topts?.maxDepth); + async getKeys(_base, topts) { + const keys = await readdirRecursive(r("."), ignore, topts?.maxDepth); + if (dataSuffix) { + return keys + .filter((key) => key.endsWith(dataSuffix)) + .map((key) => key.slice(0, -dataSuffix.length)); + } + return keys; }, async clear() { if (userOptions.readOnly || userOptions.noClear) { @@ -139,6 +183,12 @@ const driver: DriverFactory = (userOptions = {}) => { .on("error", reject) .on("all", (eventName, path) => { path = relative(base, path); + if (dataSuffix) { + if (!path.endsWith(dataSuffix)) { + return; // ignore non-suffixed files + } + path = path.slice(0, -dataSuffix.length); + } if (eventName === "change" || eventName === "add") { callback("update", path); } else if (eventName === "unlink") { diff --git a/test/drivers/fs-lite.test.ts b/test/drivers/fs-lite.test.ts index e5e5b3b72..1b15eb020 100644 --- a/test/drivers/fs-lite.test.ts +++ b/test/drivers/fs-lite.test.ts @@ -1,8 +1,10 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, afterEach } from "vitest"; import { resolve } from "node:path"; -import { readFile } from "../../src/drivers/utils/node-fs.ts"; +import { existsSync } from "node:fs"; +import { readFile, writeFile } from "../../src/drivers/utils/node-fs.ts"; import { testDriver } from "./utils.ts"; import driver from "../../src/drivers/fs-lite.ts"; +import { createStorage } from "../../src/storage.ts"; describe("drivers: fs-lite", () => { const dir = resolve(__dirname, "tmp/fs-lite"); @@ -65,4 +67,99 @@ describe("drivers: fs-lite", () => { }); }, }); + + describe("dataSuffix option", () => { + const suffixDir = resolve(__dirname, "tmp/fs-lite-suffix"); + + afterEach(async () => { + const s = createStorage({ + driver: driver({ base: suffixDir, dataSuffix: ".data" }), + }); + await s.clear(); + await s.dispose(); + }); + + it("prevents file/directory collision with dataSuffix", async () => { + const d = driver({ base: suffixDir, dataSuffix: ".data" }); + const storage = createStorage({ driver: d }); + + // This is the key scenario: "foo" and "foo:bar" must coexist. + // Without dataSuffix, "foo" creates a file at /foo, but + // "foo:bar" needs /foo/ to be a directory -> ENOTDIR. + await storage.setItem("foo", "value_foo"); + await storage.setItem("foo:bar", "value_foo_bar"); + + expect(await storage.getItem("foo")).toBe("value_foo"); + expect(await storage.getItem("foo:bar")).toBe("value_foo_bar"); + + // Verify on-disk layout uses suffix + expect(await readFile(resolve(suffixDir, "foo.data"), "utf8")).toBe( + "value_foo", + ); + expect( + await readFile(resolve(suffixDir, "foo/bar.data"), "utf8"), + ).toBe("value_foo_bar"); + + // getKeys should return clean keys without the suffix + const keys = (await storage.getKeys()).sort(); + expect(keys).toEqual(["foo", "foo:bar"]); + + await storage.dispose(); + }); + + it("runs standard driver tests with dataSuffix", async () => { + const d = driver({ base: suffixDir, dataSuffix: ".data" }); + const storage = createStorage({ driver: d }); + + await storage.setItem("s1:a", "test_data"); + expect(await storage.hasItem("s1:a")).toBe(true); + expect(await storage.getItem("s1:a")).toBe("test_data"); + + await storage.removeItem("s1:a"); + expect(await storage.hasItem("s1:a")).toBe(false); + + await storage.dispose(); + }); + + it("rejects an invalid dataSuffix", () => { + for (const bad of ["", ".", "..", "a/b", "a\\b", "a:b"]) { + expect(() => driver({ base: suffixDir, dataSuffix: bad })).toThrow( + /Invalid dataSuffix/, + ); + } + }); + + it("strips exactly one suffix (key already ending in the suffix)", async () => { + const storage = createStorage({ + driver: driver({ base: suffixDir, dataSuffix: ".data" }), + }); + // key "foo.data" -> on-disk "foo.data.data"; getKeys must strip one suffix only + await storage.setItem("foo.data", "v"); + expect(await storage.getItem("foo.data")).toBe("v"); + expect(await readFile(resolve(suffixDir, "foo.data.data"), "utf8")).toBe("v"); + expect(await storage.getKeys()).toEqual(["foo.data"]); + await storage.dispose(); + }); + + it("getKeys ignores files without the suffix", async () => { + const storage = createStorage({ + driver: driver({ base: suffixDir, dataSuffix: ".data" }), + }); + await storage.setItem("foo", "v"); // -> foo.data + await writeFile(resolve(suffixDir, "stray.txt"), "x", "utf8"); // not ours + expect((await storage.getKeys()).sort()).toEqual(["foo"]); + await storage.dispose(); + }); + + it("keeps an empty/root key inside base (no sibling escape)", async () => { + // Regression: rFile must not turn "" into `.data`, a sibling + // written outside the configured base directory. + const d = driver({ base: suffixDir, dataSuffix: ".data" }); + await d.setItem!("", "root", {}); + expect(existsSync(suffixDir + ".data")).toBe(false); // no escape + expect(await d.getItem!("")).toBe("root"); // still addressable + expect(await d.getKeys!("", {})).toEqual([""]); + await d.dispose?.(); + }); + }); }); diff --git a/test/drivers/fs.test.ts b/test/drivers/fs.test.ts index 366f5f0e0..dec59724c 100644 --- a/test/drivers/fs.test.ts +++ b/test/drivers/fs.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { resolve } from "node:path"; +import { existsSync } from "node:fs"; import { readFile, writeFile } from "../../src/drivers/utils/node-fs.ts"; import { testDriver, type TestContext } from "./utils.ts"; import driver from "../../src/drivers/fs.ts"; @@ -108,4 +109,118 @@ describe("drivers: fs", () => { await ctx.storage?.dispose(); await ctx.driver?.dispose?.(); }); + + describe("dataSuffix option", () => { + const suffixDir = resolve(__dirname, "tmp/fs-suffix"); + + afterEach(async () => { + const s = createStorage({ + driver: driver({ base: suffixDir, dataSuffix: ".data" }), + }); + await s.clear(); + await s.dispose(); + }); + + it("prevents file/directory collision with dataSuffix", async () => { + const d = driver({ base: suffixDir, dataSuffix: ".data" }); + const storage = createStorage({ driver: d }); + + // This is the key scenario: "foo" and "foo:bar" must coexist. + // Without dataSuffix, "foo" creates a file at /foo, but + // "foo:bar" needs /foo/ to be a directory -> ENOTDIR. + await storage.setItem("foo", "value_foo"); + await storage.setItem("foo:bar", "value_foo_bar"); + + expect(await storage.getItem("foo")).toBe("value_foo"); + expect(await storage.getItem("foo:bar")).toBe("value_foo_bar"); + + // Verify on-disk layout uses suffix + expect(await readFile(resolve(suffixDir, "foo.data"), "utf8")).toBe( + "value_foo", + ); + expect( + await readFile(resolve(suffixDir, "foo/bar.data"), "utf8"), + ).toBe("value_foo_bar"); + + // getKeys should return clean keys without the suffix + const keys = (await storage.getKeys()).sort(); + expect(keys).toEqual(["foo", "foo:bar"]); + + await storage.dispose(); + }); + + it("runs standard driver tests with dataSuffix", async () => { + const d = driver({ base: suffixDir, dataSuffix: ".data" }); + const storage = createStorage({ driver: d }); + + await storage.setItem("s1:a", "test_data"); + expect(await storage.hasItem("s1:a")).toBe(true); + expect(await storage.getItem("s1:a")).toBe("test_data"); + + await storage.removeItem("s1:a"); + expect(await storage.hasItem("s1:a")).toBe(false); + + await storage.dispose(); + }); + + it("rejects an invalid dataSuffix", () => { + for (const bad of ["", ".", "..", "a/b", "a\\b", "a:b"]) { + expect(() => driver({ base: suffixDir, dataSuffix: bad })).toThrow( + /Invalid dataSuffix/, + ); + } + }); + + it("strips exactly one suffix (key already ending in the suffix)", async () => { + const storage = createStorage({ + driver: driver({ base: suffixDir, dataSuffix: ".data" }), + }); + // key "foo.data" -> on-disk "foo.data.data"; getKeys must strip one suffix only + await storage.setItem("foo.data", "v"); + expect(await storage.getItem("foo.data")).toBe("v"); + expect(await readFile(resolve(suffixDir, "foo.data.data"), "utf8")).toBe("v"); + expect(await storage.getKeys()).toEqual(["foo.data"]); + await storage.dispose(); + }); + + it("getKeys ignores files without the suffix", async () => { + const storage = createStorage({ + driver: driver({ base: suffixDir, dataSuffix: ".data" }), + }); + await storage.setItem("foo", "v"); // -> foo.data + await writeFile(resolve(suffixDir, "stray.txt"), "x", "utf8"); // not ours + expect((await storage.getKeys()).sort()).toEqual(["foo"]); + await storage.dispose(); + }); + + it("watcher strips the suffix and ignores non-suffixed files", async () => { + const storage = createStorage({ + driver: driver({ base: suffixDir, dataSuffix: ".data" }), + }); + const watcher = vi.fn(); + await storage.watch(watcher); + await writeFile(resolve(suffixDir, "s1/random.data"), "random", "utf8"); + await writeFile(resolve(suffixDir, "s1/ignored.txt"), "nope", "utf8"); + await new Promise((resolve) => setTimeout(resolve, 700)); + // separator-agnostic (key may use ":", "/" or "\\" depending on OS/normalization) + const updates = watcher.mock.calls + .filter(([event]) => event === "update") + .map(([, key]) => String(key)); + expect(updates.some((k) => /(^|[:/\\])random$/.test(k))).toBe(true); + expect(updates.every((k) => !k.includes(".data"))).toBe(true); // suffix never leaks + expect(updates.some((k) => k.includes("ignored"))).toBe(false); // non-suffixed ignored + await storage.dispose(); + }); + + it("keeps an empty/root key inside base (no sibling escape)", async () => { + // Regression: rFile must not turn "" into `.data`, a sibling + // written outside the configured base directory. + const d = driver({ base: suffixDir, dataSuffix: ".data" }); + await d.setItem!("", "root", {}); + expect(existsSync(suffixDir + ".data")).toBe(false); // no escape + expect(await d.getItem!("")).toBe("root"); // still addressable + expect(await d.getKeys!("", {})).toEqual([""]); + await d.dispose?.(); + }); + }); });