diff --git a/.gitignore b/.gitignore index 3acd371..676cdd7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ node_modules vite.config.js.timestamp-* vite.config.ts.timestamp-* test-results +sync-dbs/**.* +!sync-dbs/.keep diff --git a/package.json b/package.json index 7721deb..94581aa 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@sveltejs/kit": "^2.0.0", "@sveltejs/vite-plugin-svelte": "^3.0.1", "@testcontainers/postgresql": "^10.4.0", + "@types/better-sqlite3": "^7.6.8", "@types/ws": "^8.5.10", "@typescript-eslint/eslint-plugin": "^6.14.0", "@typescript-eslint/parser": "^6.14.0", @@ -62,6 +63,9 @@ "type": "module", "dependencies": { "@lucia-auth/adapter-postgresql": "^2.0.2", + "@vlcn.io/crsqlite": "^0.16.1", + "@vlcn.io/crsqlite-wasm": "^0.16.0", + "better-sqlite3": "^9.2.2", "bits-ui": "^0.11.8", "clsx": "^2.0.0", "drizzle-orm": "^0.29.1", @@ -77,5 +81,8 @@ "tailwind-variants": "^0.1.19", "ws": "^8.15.1", "zod": "^3.22.4" + }, + "optionalDependencies": { + "bufferutil": "^4.0.8" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 101feaf..b66b21d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,15 @@ dependencies: '@lucia-auth/adapter-postgresql': specifier: ^2.0.2 version: 2.0.2(lucia@2.7.5)(postgres@3.4.3) + '@vlcn.io/crsqlite': + specifier: ^0.16.1 + version: 0.16.1 + '@vlcn.io/crsqlite-wasm': + specifier: ^0.16.0 + version: 0.16.0 + better-sqlite3: + specifier: ^9.2.2 + version: 9.2.2 bits-ui: specifier: ^0.11.8 version: 0.11.8(svelte@4.2.8) @@ -16,7 +25,7 @@ dependencies: version: 2.0.0 drizzle-orm: specifier: ^0.29.1 - version: 0.29.1(postgres@3.4.3) + version: 0.29.1(@types/better-sqlite3@7.6.8)(better-sqlite3@9.2.2)(postgres@3.4.3) faktory-worker: specifier: ^4.5.1 version: 4.5.1 @@ -49,11 +58,16 @@ dependencies: version: 0.1.19(tailwindcss@3.3.6) ws: specifier: ^8.15.1 - version: 8.15.1 + version: 8.15.1(bufferutil@4.0.8) zod: specifier: ^3.22.4 version: 3.22.4 +optionalDependencies: + bufferutil: + specifier: ^4.0.8 + version: 4.0.8 + devDependencies: '@playwright/test': specifier: ^1.40.1 @@ -73,6 +87,9 @@ devDependencies: '@testcontainers/postgresql': specifier: ^10.4.0 version: 10.4.0 + '@types/better-sqlite3': + specifier: ^7.6.8 + version: 7.6.8 '@types/ws': specifier: ^8.5.10 version: 8.5.10 @@ -90,7 +107,7 @@ devDependencies: version: 16.3.1 drizzle-kit: specifier: ^0.20.7 - version: 0.20.7 + version: 0.20.7(bufferutil@4.0.8) eslint: specifier: ^8.56.0 version: 8.56.0 @@ -1285,6 +1302,11 @@ packages: - supports-color dev: true + /@types/better-sqlite3@7.6.8: + resolution: {integrity: sha512-ASndM4rdGrzk7iXXqyNC4fbwt4UEjpK0i3j4q4FyeQrLAthfB6s7EF135ZJE0qQxtKIMFwmyT6x0switET7uIw==} + dependencies: + '@types/node': 20.10.4 + /@types/cookie@0.6.0: resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} @@ -1325,7 +1347,6 @@ packages: resolution: {integrity: sha512-D08YG6rr8X90YB56tSIuBaddy/UXAA9RKJoFvrsnogAum/0pmjkgi4+2nx96A330FmioegBWmEYQ+syqCFaveg==} dependencies: undici-types: 5.26.5 - dev: true /@types/pug@2.0.10: resolution: {integrity: sha512-Sk/uYFOBAB7mb74XcpizmH0KOR2Pv3D2Hmrh1Dmy5BmK3MpdSa5kqZcg6EKBdklU0bFXX9gCfzvpnyUehrPIuA==} @@ -1537,6 +1558,29 @@ packages: pretty-format: 29.7.0 dev: true + /@vlcn.io/crsqlite-wasm@0.16.0: + resolution: {integrity: sha512-5gf52eyMYvZirxuEUo4QS65JhEsw3fndoO+tCtCEOxuiIEtvaKB2/6wuuKGRlMVkxIp4Bls70D3DCF5v9lcJxA==} + dependencies: + '@vlcn.io/wa-sqlite': 0.22.0 + '@vlcn.io/xplat-api': 0.15.0 + async-mutex: 0.4.0 + dev: false + + /@vlcn.io/crsqlite@0.16.1: + resolution: {integrity: sha512-ju6dONV/xq3haiHUVkY/mUuz14lXqak1GAyCsY8YqEoZUUORIx8nXa8aIOM6A+n31+KreTsJ4qSiG0VCnZLveA==} + requiresBuild: true + dev: false + + /@vlcn.io/wa-sqlite@0.22.0: + resolution: {integrity: sha512-OujKro0mAqP7/efUeCGB6zBiyMoSCFVe7jQKPF0n47U9ZhOaIW3kQUVCwF+CmzvzQfN1Vl4PrFQRNNxlSwTCNQ==} + dev: false + + /@vlcn.io/xplat-api@0.15.0: + resolution: {integrity: sha512-2/aE7VgI3EbIO5EcJGrskAJuCa2pteY1rWNWfhovFKMERe9NhJdlDMIB1I31X0sN/WC2DnF30RqcdTXNfYyzhQ==} + dependencies: + comlink: 4.4.1 + dev: false + /acorn-jsx@5.3.2(acorn@8.11.2): resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1673,6 +1717,12 @@ packages: resolution: {integrity: sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ==} dev: true + /async-mutex@0.4.0: + resolution: {integrity: sha512-eJFZ1YhRR8UN8eBLoNzcDPcy/jqjsg6I1AP+KvWQX80BqOSW1oJPJXDylPUEeMr2ZQvHgnQ//Lp6f3RQ1zI7HA==} + dependencies: + tslib: 2.6.2 + dev: false + /async@3.2.5: resolution: {integrity: sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==} dev: true @@ -1707,7 +1757,6 @@ packages: /base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - dev: true /bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} @@ -1715,10 +1764,24 @@ packages: tweetnacl: 0.14.5 dev: true + /better-sqlite3@9.2.2: + resolution: {integrity: sha512-qwjWB46il0lsDkeB4rSRI96HyDQr8sxeu1MkBVLMrwusq1KRu4Bpt1TMI+8zIJkDUtZ3umjAkaEjIlokZKWCQw==} + requiresBuild: true + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.1 + dev: false + /binary-extensions@2.2.0: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} + /bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + dependencies: + file-uri-to-path: 1.0.0 + dev: false + /bits-ui@0.11.8(svelte@4.2.8): resolution: {integrity: sha512-T3YaT88OJguBoUU/MSncf41fiIc+5/ka8Au2LUDo0nSECex+LFY40+hKWLJc5tRT56avkyHsI7x9daA2r9eS/g==} peerDependencies: @@ -1736,7 +1799,6 @@ packages: buffer: 5.7.1 inherits: 2.0.4 readable-stream: 3.6.2 - dev: true /blake3-wasm@2.1.5: resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} @@ -1784,7 +1846,13 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 - dev: true + + /bufferutil@4.0.8: + resolution: {integrity: sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw==} + engines: {node: '>=6.14.2'} + requiresBuild: true + dependencies: + node-gyp-build: 4.7.1 /buildcheck@0.0.6: resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==} @@ -1883,7 +1951,6 @@ packages: /chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - dev: true /cli-color@2.0.3: resolution: {integrity: sha512-OkoZnxyC4ERN3zLzZaY9Emb7f/MhBOIpePv0Ycok0fJYT+Ouo00UBEIwsVsr0yoow++n5YWlSUgST9GKhNHiRQ==} @@ -1926,6 +1993,10 @@ packages: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} dev: true + /comlink@4.4.1: + resolution: {integrity: sha512-+1dlx0aY5Jo1vHy/tSsIGpSkN4tS9rZSW8FIhG0JH/crs9wwweswIo/POr451r7bZww3hFbPAKnTpimzL/mm4Q==} + dev: false + /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} @@ -2038,6 +2109,13 @@ packages: dependencies: ms: 2.1.2 + /decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + dependencies: + mimic-response: 3.1.0 + dev: false + /deep-eql@4.1.3: resolution: {integrity: sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==} engines: {node: '>=6'} @@ -2045,6 +2123,11 @@ packages: type-detect: 4.0.8 dev: true + /deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + dev: false + /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} dev: true @@ -2067,6 +2150,11 @@ packages: engines: {node: '>=8'} dev: true + /detect-libc@2.0.2: + resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} + engines: {node: '>=8'} + dev: false + /devalue@4.3.2: resolution: {integrity: sha512-KqFl6pOgOW+Y6wJgu80rHpo2/3H07vr8ntR9rkkFIRETewbf5GaYYcakYfiKz89K+sLsuPkQIZaXDMjUObZwWg==} @@ -2143,7 +2231,7 @@ packages: wordwrap: 1.0.0 dev: true - /drizzle-kit@0.20.7: + /drizzle-kit@0.20.7(bufferutil@4.0.8): resolution: {integrity: sha512-3LjTvgVAI1jd3JHLG2tMW5ew49NuD7SMymRv+h9xUxb/geS+U/O1yENni0HhyjZH+Gc8hdStL9v1xY9Ob3s3/g==} hasBin: true dependencies: @@ -2159,7 +2247,7 @@ packages: json-diff: 0.9.0 minimatch: 7.4.6 semver: 7.5.4 - wrangler: 3.20.0 + wrangler: 3.20.0(bufferutil@4.0.8) zod: 3.22.4 transitivePeerDependencies: - bufferutil @@ -2167,7 +2255,7 @@ packages: - utf-8-validate dev: true - /drizzle-orm@0.29.1(postgres@3.4.3): + /drizzle-orm@0.29.1(@types/better-sqlite3@7.6.8)(better-sqlite3@9.2.2)(postgres@3.4.3): resolution: {integrity: sha512-yItc4unfHnk8XkDD3/bdC63vdboTY7e7I03lCF1OJYABXSIfQYU9BFTQJXMMovVeb3T1/OJWwfW/70T1XPnuUA==} peerDependencies: '@aws-sdk/client-rds-data': '>=3' @@ -2229,6 +2317,8 @@ packages: sqlite3: optional: true dependencies: + '@types/better-sqlite3': 7.6.8 + better-sqlite3: 9.2.2 postgres: 3.4.3 dev: false @@ -2240,7 +2330,6 @@ packages: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} dependencies: once: 1.4.0 - dev: true /es5-ext@0.10.62: resolution: {integrity: sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==} @@ -2783,6 +2872,11 @@ packages: engines: {node: '>=6'} dev: true + /expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + dev: false + /ext@1.7.0: resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} dependencies: @@ -2842,6 +2936,10 @@ packages: flat-cache: 3.2.0 dev: true + /file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + dev: false + /fill-range@7.0.1: resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} engines: {node: '>=8'} @@ -2893,7 +2991,6 @@ packages: /fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} - dev: true /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -2948,6 +3045,10 @@ packages: resolve-pkg-maps: 1.0.0 dev: true + /github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + dev: false + /glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -3058,7 +3159,6 @@ packages: /ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - dev: true /ignore@5.3.0: resolution: {integrity: sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==} @@ -3091,6 +3191,10 @@ packages: /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + /ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + dev: false + /ioredis@5.3.2: resolution: {integrity: sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==} engines: {node: '>=12.22.0'} @@ -3322,7 +3426,6 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 - dev: true /lru-queue@0.1.0: resolution: {integrity: sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==} @@ -3403,12 +3506,17 @@ packages: engines: {node: '>=12'} dev: true + /mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + dev: false + /min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} dev: true - /miniflare@3.20231030.4: + /miniflare@3.20231030.4(bufferutil@4.0.8): resolution: {integrity: sha512-7MBz0ArLuDop1WJGZC6tFgN6c5MRyDOIlxbm3yp0TRBpvDS/KsTuWCQcCjsxN4QQ5zvL3JTkuIZbQzRRw/j6ow==} engines: {node: '>=16.13'} hasBin: true @@ -3422,7 +3530,7 @@ packages: stoppable: 1.1.0 undici: 5.28.2 workerd: 1.20231030.0 - ws: 8.15.1 + ws: 8.15.1(bufferutil@4.0.8) youch: 3.3.3 zod: 3.22.4 transitivePeerDependencies: @@ -3452,11 +3560,9 @@ packages: /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: true /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - dev: true /mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} @@ -3526,6 +3632,10 @@ packages: hasBin: true dev: false + /napi-build-utils@1.0.2: + resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} + dev: false + /natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true @@ -3534,6 +3644,13 @@ packages: resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==} dev: true + /node-abi@3.52.0: + resolution: {integrity: sha512-JJ98b02z16ILv7859irtXn4oUaFWADtvkzy2c0IAatNVX2Mc9Yoh8z6hZInn3QwvMEYhHuQloYi+TTQy67SIdQ==} + engines: {node: '>=10'} + dependencies: + semver: 7.5.4 + dev: false + /node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -3551,6 +3668,11 @@ packages: engines: {node: '>= 6.13.0'} dev: true + /node-gyp-build@4.7.1: + resolution: {integrity: sha512-wTSrZ+8lsRRa3I3H8Xr65dLWSgCvY2l4AOnaeKdPA9TB/WYMPaTcrzf3rXvFoVvjKNVnu0CcWSx54qq9GKRUYg==} + hasBin: true + requiresBuild: true + /node-releases@2.0.14: resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} dev: true @@ -3836,6 +3958,25 @@ packages: engines: {node: '>=12'} dev: false + /prebuild-install@7.1.1: + resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} + engines: {node: '>=10'} + hasBin: true + dependencies: + detect-libc: 2.0.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 1.0.2 + node-abi: 3.52.0 + pump: 3.0.0 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.0.1 + tunnel-agent: 0.6.0 + dev: false + /prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -3894,7 +4035,6 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 - dev: true /punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} @@ -3908,6 +4048,16 @@ packages: resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} dev: true + /rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + dev: false + /react-is@18.2.0: resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} dev: true @@ -3936,7 +4086,6 @@ packages: inherits: 2.0.4 string_decoder: 1.3.0 util-deprecate: 1.0.2 - dev: true /readdir-glob@1.1.3: resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} @@ -4069,7 +4218,6 @@ packages: /safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - dev: true /safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -4098,7 +4246,6 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 - dev: true /set-cookie-parser@2.6.0: resolution: {integrity: sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==} @@ -4128,6 +4275,18 @@ packages: engines: {node: '>=14'} dev: true + /simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + dev: false + + /simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + dev: false + /sirv@2.0.3: resolution: {integrity: sha512-O9jm9BsID1P+0HOi81VpXPoDxYP374pkOLzACAoyUQ/3OUVndNpsz6wMnY2z+yOxzbllCKZrM+9QrWsv4THnyA==} engines: {node: '>= 10'} @@ -4240,7 +4399,6 @@ packages: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} dependencies: safe-buffer: 5.2.1 - dev: true /strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} @@ -4261,6 +4419,11 @@ packages: min-indent: 1.0.1 dev: true + /strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + dev: false + /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -4531,7 +4694,6 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.0 tar-stream: 2.2.0 - dev: true /tar-fs@3.0.4: resolution: {integrity: sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==} @@ -4550,7 +4712,6 @@ packages: fs-constants: 1.0.0 inherits: 2.0.4 readable-stream: 3.6.2 - dev: true /tar-stream@3.1.6: resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==} @@ -4669,6 +4830,12 @@ packages: esbuild: 0.15.18 dev: true + /tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + dependencies: + safe-buffer: 5.2.1 + dev: false + /tweetnacl@0.14.5: resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} dev: true @@ -4710,7 +4877,6 @@ packages: /undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - dev: true /undici@5.28.2: resolution: {integrity: sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==} @@ -4910,7 +5076,7 @@ packages: '@cloudflare/workerd-windows-64': 1.20231030.0 dev: true - /wrangler@3.20.0: + /wrangler@3.20.0(bufferutil@4.0.8): resolution: {integrity: sha512-7mg25zJByhBmrfG+CbImSid7JNd5lxGovLA167ndtE8Yrqd3TUukrGWL8o0RCQIm0FUcgl2nCzWArJDShlZVKA==} engines: {node: '>=16.17.0'} hasBin: true @@ -4921,7 +5087,7 @@ packages: blake3-wasm: 2.1.5 chokidar: 3.5.3 esbuild: 0.17.19 - miniflare: 3.20231030.4 + miniflare: 3.20231030.4(bufferutil@4.0.8) nanoid: 3.3.7 path-to-regexp: 6.2.1 resolve.exports: 2.0.2 @@ -4940,7 +5106,7 @@ packages: /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - /ws@8.15.1: + /ws@8.15.1(bufferutil@4.0.8): resolution: {integrity: sha512-W5OZiCjXEmk0yZ66ZN82beM5Sz7l7coYxpRkzS+p9PP+ToQry8szKh+61eNktr7EA9DOwvFGhfC605jDHbP6QQ==} engines: {node: '>=10.0.0'} peerDependencies: @@ -4951,6 +5117,8 @@ packages: optional: true utf-8-validate: optional: true + dependencies: + bufferutil: 4.0.8 /xxhash-wasm@1.0.2: resolution: {integrity: sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A==} @@ -4958,7 +5126,6 @@ packages: /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: true /yaml@1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} diff --git a/src/lib/browser-db.ts b/src/lib/browser-db.ts new file mode 100644 index 0000000..ee9d147 --- /dev/null +++ b/src/lib/browser-db.ts @@ -0,0 +1,8 @@ +import initWasm from '@vlcn.io/crsqlite-wasm'; +import wasmUrl from '@vlcn.io/crsqlite-wasm/crsqlite.wasm?url'; + +export async function load(file = 'default.db', paths: { wasm?: string } = {}) { + const sqlite = await initWasm(() => paths?.wasm || wasmUrl); + const db = await sqlite.open(file); + return { db, browser: true }; +} diff --git a/src/lib/server/sync-db/db.ts b/src/lib/server/sync-db/db.ts new file mode 100644 index 0000000..01a9832 --- /dev/null +++ b/src/lib/server/sync-db/db.ts @@ -0,0 +1,17 @@ +import Database from 'better-sqlite3'; +import { extensionPath } from '@vlcn.io/crsqlite'; + +export function dbFrom(filename) { + // TODO: should be an env-var so we can use Render's persistent disk as the path + const db = new Database(`./sync-dbs/${filename}`); + db.pragma('journal_mode = WAL'); + db.loadExtension(extensionPath); + + // TODO: import schema from same place as frontend + db.exec(`CREATE TABLE IF NOT EXISTS todos (id PRIMARY KEY NOT NULL, content, complete); + SELECT crsql_as_crr('todos'); + CREATE TABLE IF NOT EXISTS todonts (id PRIMARY KEY NOT NULL, content, complete); + SELECT crsql_as_crr('todonts'); + `); + return db; +} diff --git a/src/lib/server/websockets/features/offline/flow.md b/src/lib/server/websockets/features/offline/flow.md new file mode 100644 index 0000000..182b3f6 --- /dev/null +++ b/src/lib/server/websockets/features/offline/flow.md @@ -0,0 +1,31 @@ +# Flow + +## Server + +1. WS connection established +2. Server gets clientSiteId from URL +3. Server queries `crsql_tracked_peers` for clientSite version +4. Server queries all changes >= tracked client version +5. Server sends `update` with changes + 1. Client receives `update` + 2. Client merges changes + 3. Client updates server version in `crsql_tracked_peers` +6. Server updates client version in `crsql_tracked_peers` + +## Client + +1. WS connection established +2. Server sends down `connected` message with server siteId +2. Client queries `crsql_tracked_peers` for serverSite version +3. Client queries all changes >= tracked server version +4. Client sends `update` with changes + 1. Server receives `update` + 2. Server merges changes + 3. Server updates client version in `crsql_tracked_peers` + 4. Server gets all tracked peers with version <= server + 5. Server loops through clients + 1. Server queries all changes >= tracked client version + 2. Server sends `update` with changes to client + 3. Server updates client version in `crsql_tracked_peers` +5. Client updates server version in `crsql_tracked_peers` + diff --git a/src/lib/server/websockets/features/offline/sync.ts b/src/lib/server/websockets/features/offline/sync.ts new file mode 100644 index 0000000..37a8afc --- /dev/null +++ b/src/lib/server/websockets/features/offline/sync.ts @@ -0,0 +1,251 @@ +import { client, create } from '../../redis-client'; +import type { ExtendedWebSocket } from '../../../../../../vite-plugins/vite-plugin-svelte-kit-integrated-websocket-server'; +import type { Redis } from 'ioredis'; +import { dbFrom } from '$lib/server/sync-db/db'; +import type Database from 'better-sqlite3'; +import { WebSocket } from 'ws'; +import { latestVersions } from '../../../../websockets/sync-db-store.js'; + +const INSERT_CHANGES = `INSERT INTO crsql_changes VALUES (?, unhex(?), ?, ?, ?, ?, unhex(?), ?, ?)`; + +// TODO: review https://github.com/vlcn-io/js/blob/main/packages/ws-server/src/DB.ts +// see if any edge cases have been missed + +export class Sync { + private stream: string; + private ws: ExtendedWebSocket; + private userId: string; + private redisClient: Redis = client(); + private db: ReturnType; + private version: number; + private siteId: string; + private clientSiteId: string; + + /* + This is the preferred way to instantiate this class for 2 reasons: + 1. If we use the class in multiple places, we want to avoid duplicating all of this setup. + 2. Constructors can not be asynchronous so the only way to encapsulate the setup is through + a static method + */ + static async init({ + ws, + stream, + clientSiteId, + clientVersion + }: { + stream: string; + ws: ExtendedWebSocket; + clientSiteId: string; + clientVersion: number; + }) { + const db = dbFrom(`${ws.session.user.userId}.db`); + const { siteId, version } = db + .prepare('SELECT hex(crsql_site_id()) as siteId, crsql_db_version() as version;') + .get(); + const sync = new Sync({ ws, stream, db, version, siteId, clientSiteId }); + + const subClient = create(); + await subClient.subscribe(sync.stream, (err) => { + if (err) { + console.error('Failed to subscribe: %s', err.message); + } + }); + + const subscription = await subClient.on('messageBuffer', (stream, message) => { + sync.notify(message); + }); + + ws.on('message', async (data) => { + const parsed = JSON.parse(data.toString()); + const changes = parsed.changes; + if (parsed.type === 'update') { + /* + some client is sending an update to the server + which then is forwarded to all clients -> `sync.recieve(data)` + this can be triggered in two ways: + 1. client inserts a new entry and sends an update + 2. client receives a message of `type: 'connected'`, then it sends up all changes + */ + changes.forEach((change, i) => { + db.prepare(INSERT_CHANGES).run(...change); + }); + + const changeSiteVersions = latestVersions(changes); + + changeSiteVersions.forEach(([changeSiteId, changeDbVersion]) => { + db.prepare( + `INSERT INTO crsql_tracked_peers (site_id, version, tag, event) + VALUES (unhex(?), ?, 0, 0) + ON CONFLICT([site_id], [tag], [event]) + DO UPDATE SET version=excluded.version` + ).run(changeSiteId, changeDbVersion); + }); + + await sync.receive(data); + } + }); + + ws.on('close', async () => { + await subscription.unsubscribe(); + }); + + // Make sure this happens AFTER event handlers are declared + // RE-THINK this. + // sync.catchUpClient(clientSiteId); + // catch up server should force catch up the other clients when the server receives the updates + // sync.catchUpServer(clientSiteId); + + return sync; + } + + private constructor({ + ws, + stream, + db, + version, + siteId, + clientSiteId + }: { + ws: ExtendedWebSocket; + stream: string; + db: ReturnType; + version: number; + siteId: string; + clientSiteId: string; + }) { + this.stream = stream; + this.ws = ws; + this.userId = ws.session.user.userId; + this.db = db; + this.version = version; + this.siteId = siteId; + this.clientSiteId = clientSiteId; + } + + /* + STATES + - 1. connection + - 2. server sends `connect` with server siteId + - 3. client receives message and queries tracked_peers for server siteId and compares versions + - A. Client is >= Server + - 1. client queries for changes >= server's version + - 2. client sends `update` with changes + - 1. server receives updates + - 2. server merges updates + - 3. server sets tracked peers + - 4. server sends `update` to all tracked_peers with version <= last change received + - 3. client sets client version tracked_peers to last change sent + - B. Client is <= Server + - 1. client sends `request-for-update` + - 1. server queries client's version in tracked_peers + - 2. server queries for changes >= client's version + - 3. server sends `update` with changes + - 1. server receives `update` + - 2. server merges changes + - 3. server sends `update` to all tracked_peers with version <= last change received + - 4. server updates tracked_peers + - 2. client sets server version in tracked_peers to last change sent + */ + + catchUpServer(clientSiteId) { + // Here we can send down the last seen id or something. + // that way we don't need the entire contents of the db + + const result = this.db + .prepare(`SELECT version FROM crsql_tracked_peers WHERE site_id = unhex(?)`) + .get(clientSiteId); + const version = result?.version ?? 0; + this.notify(JSON.stringify({ type: 'connected', siteId: clientSiteId, version })); + } + + catchUpClient(clientSiteId: string) { + // Maybe we can do something to only send down what's needed. + // just updates after the last update by `${clientSiteId} + + const result = this.db + .prepare(`SELECT version FROM crsql_tracked_peers WHERE site_id = unhex(?)`) + .get(clientSiteId); + const lastVersion = result?.version ?? 0; + + const changes = this.db + .prepare( + `SELECT "table", hex("pk") as pk, "cid", "val", "col_version", "db_version", hex("site_id") as site_id, "cl", "seq" + FROM crsql_changes WHERE site_id != unhex(:clientSiteId) + AND db_version > :lastVersion` + ) + .all({ + clientSiteId, + lastVersion + }); + + const changeSiteVersions = latestVersions(changes.map((change) => Object.values(change))); + + changeSiteVersions.forEach(([changeSiteId, changeDbVersion]) => { + this.db + .prepare( + `INSERT INTO crsql_tracked_peers (site_id, version, tag, event) + VALUES (unhex(?), ?, 0, 0) + ON CONFLICT([site_id], [tag], [event]) + DO UPDATE SET version=excluded.version` + ) + .run(changeSiteId, changeDbVersion); + }); + + if (changes.length) { + const message = JSON.stringify({ + type: 'pull', + siteId: this.siteId, + version: this.version, + changes: changes.map((change) => Object.values(change)) + }); + this.notify(message); + } + } + + private shouldProcess(message) { + const isFromSelf = message?.siteId === this.clientSiteId; + if (isFromSelf && message.type === 'update') { + return true; + } + + if (isFromSelf && message.type !== 'update') { + return true; + } + + if (!isFromSelf) { + return true; + } + + return true; + } + + private messageToObject(message) { + if (typeof message === 'string') { + return JSON.parse(message); + } else if (message.type === 'string') { + return message; + } else if (Buffer.isBuffer(message)) { + return JSON.parse(message.toString()); + } else { + throw 'unknown message format'; + } + } + + private async notify(message) { + if ( + this.ws.readyState === WebSocket.OPEN && + this.shouldProcess(this.messageToObject(message)) + ) { + this.ws.send(message, { binary: true }); + } + } + + private async publishMessage(message: ArrayBufferLike) { + await this.redisClient.lpush(this.stream, message); + await this.redisClient.publish(this.stream, message); + } + + private async receive(message: Buffer) { + await this.publishMessage(message); + } +} diff --git a/src/lib/server/websockets/handler.ts b/src/lib/server/websockets/handler.ts index 52b3d0e..6d2ac7e 100644 --- a/src/lib/server/websockets/handler.ts +++ b/src/lib/server/websockets/handler.ts @@ -5,6 +5,7 @@ import { Chat as ChatStreams } from './features/redis-streams/chat.js'; import { Chat as ChatPubSub } from './features/redis-pub-sub/chat.js'; import type { ExtendedWebSocket } from '../../../../vite-plugins/vite-plugin-svelte-kit-integrated-websocket-server'; import { auth } from '../lucia'; +import { Sync } from './features/offline/sync'; const FEATURE_STRATEGIES = { chat: { @@ -14,6 +15,9 @@ const FEATURE_STRATEGIES = { presence: { 'redis-streams': PresenceStreams, 'redis-pub-sub': PresencePubSub + }, + offline: { + sync: Sync } }; @@ -21,6 +25,8 @@ type Feature = { type: keyof typeof FEATURE_STRATEGIES; strategy: keyof (typeof FEATURE_STRATEGIES)[keyof typeof FEATURE_STRATEGIES]; stream: string; + clientSiteId?: string; + clientVersion?: number; }; function getFeatureFor( @@ -58,7 +64,12 @@ export async function hooksConnectionHandler(ws: ExtendedWebSocket, request: Inc ws.close(1008, `No stream specified for ${feature.type}`); throw new Error(`Invalid feature ${feature.type} - no stream specified`); } - - await getFeatureFor(feature.type, feature.strategy).init({ ws, stream: feature.stream }); + const f = await getFeatureFor(feature.type, feature.strategy); + await f.init({ + ws, + stream: feature.stream, + clientSiteId: feature.clientSiteId, + clientVersion: feature.clientVersion + }); }); } diff --git a/src/lib/server/websockets/redis-client.ts b/src/lib/server/websockets/redis-client.ts index 6ca7afb..5468fa2 100644 --- a/src/lib/server/websockets/redis-client.ts +++ b/src/lib/server/websockets/redis-client.ts @@ -5,7 +5,7 @@ import 'dotenv/config'; const connectionString = process.env.REDIS_WS_SERVER as string; let cli: Redis | null = null; -export const create = () => new Redis(connectionString); +export const create = (options = {}) => new Redis(connectionString, options); export const client = () => { cli = cli ? cli : create(); return cli; diff --git a/src/lib/sync-db.ts b/src/lib/sync-db.ts new file mode 100644 index 0000000..72e9260 --- /dev/null +++ b/src/lib/sync-db.ts @@ -0,0 +1,50 @@ +import initWasm, { DB } from '@vlcn.io/crsqlite-wasm'; +import wasmUrl from '@vlcn.io/crsqlite-wasm/crsqlite.wasm?url'; + +const INSERT_CHANGES = `INSERT INTO crsql_changes VALUES (?, unhex(?), ?, ?, ?, ?, unhex(?), ?, ?)`; + +export class Database { + db: DB; + + static async load({ schema, name }: { schema: string[]; name: string }) { + const sqlite = await initWasm(() => wasmUrl); + const db = await sqlite.open(name); + const database = new Database(db); + await database.db.execMany(schema); + return database; + } + + constructor(db: DB) { + this.db = db; + } + + async version() { + const [[version]] = await this.db.exec(`SELECT crsql_db_version();`); + return version; + } + + async siteId() { + const [[siteId]] = await this.db.exec(`SELECT hex(crsql_site_id());`); + return siteId; + } + + async merge(changes) { + // const trackedPeers = await this.db.exec(`SELECT * FROM crsql_tracked_peers`); + // TODO: USE PREPARED STATEMENTS + await this.db.tx(async (tx) => { + changes.forEach(async (change) => { + await tx.exec(INSERT_CHANGES, change); + }); + }); + } + + async updateTrackedPeer(changeSiteId: string, changeDbVersion: number) { + await this.db.exec( + `INSERT INTO crsql_tracked_peers (site_id, version, tag, event) + VALUES (unhex(?), ?, 0, 0) + ON CONFLICT([site_id], [tag], [event]) + DO UPDATE SET version=excluded.version`, + [changeSiteId, changeDbVersion] + ); + } +} diff --git a/src/lib/websockets/sync-db-store.ts b/src/lib/websockets/sync-db-store.ts new file mode 100644 index 0000000..b734e79 --- /dev/null +++ b/src/lib/websockets/sync-db-store.ts @@ -0,0 +1,191 @@ +import { browser } from '$app/environment'; +import { Database } from '$lib/sync-db'; +import { onDestroy } from 'svelte'; +import { readable, writable } from 'svelte/store'; + +const encoder = new TextEncoder(); + +function wsErrorHandler(error: Event) { + console.error(error); +} + +export function latestVersions(changes) { + return Object.entries( + changes.reduce((acc, change) => { + const siteId = change[6]; + const version = change[5]; + if (acc[siteId]) { + acc[siteId] = acc[siteId] > version ? acc[siteId] : version; + } else { + acc[siteId] = version; + } + return acc; + }, {}) + ); +} + +async function pushOfflineChangesToServer(database, ws, version) { + const changes = await database.db.exec( + `SELECT "table", hex("pk") as pk, "cid", "val", "col_version", "db_version", hex("site_id") as site_id, "cl", "seq" + FROM crsql_changes WHERE site_id = crsql_site_id() AND db_version > ?`, + [version] + ); + + const changeSiteVersions = latestVersions(changes); + // sending so we're using the local db_version + changeSiteVersions.forEach(async ([changeSiteId, changeDbVersion]) => { + // await database.updateTrackedPeer(changeSiteId, changeDbVersion); + await database.db.exec( + `INSERT INTO crsql_tracked_peers (site_id, version, tag, event) + VALUES (unhex(?), ?, 0, 0) + ON CONFLICT([site_id], [tag], [event]) + DO UPDATE SET version=excluded.version`, + [changeSiteId, changeDbVersion] + ); + }); + + // maybe this should be a POST so we can get a nicer + // user experience. that way we can await until the + // changes are here rather than reacting to a server + // sent websocket message + const message = encoder.encode( + JSON.stringify({ + type: 'update', + siteId: await database.siteId(), + version: await database.version(), + changes + }) + ); + ws.send(message); +} + +function wsMessageHandler({ + database, + update +}: { + database: Database; + update: () => Promise; +}) { + return async function (event: Event) { + // Are we over subscribing here? every `store` attaches an event listener + // maybe there's some kind of queue or something we can use to only apply + // appropriate udpates + if (typeof event.data !== 'string') { + const clientSiteId = await database.siteId(); + const m = await event.data.text(); + const { type, changes, siteId, version } = JSON.parse(m); + + if ((type === 'update' && siteId !== clientSiteId) || type === 'pull') { + await database.merge(changes); + + const changeSiteVersions = latestVersions(changes); + + changeSiteVersions.forEach(async ([changeSiteId, changeDbVersion]) => { + // await database.updateTrackedPeer(changeSiteId, changeDbVersion); + await database.db.exec( + `INSERT INTO crsql_tracked_peers (site_id, version, tag, event) + VALUES (unhex(?), ?, 0, 0) + ON CONFLICT([site_id], [tag], [event]) + DO UPDATE SET version=excluded.version`, + [changeSiteId, changeDbVersion] + ); + }); + await update(); + } + + if (type === 'connected') { + await pushOfflineChangesToServer(database, this, version); + } + } + }; +} + +// TODO: probably need re-connect/retry logic if WS server closes connection +async function setupWs({ url, database }: { url: string; database: Promise }) { + const db = await database; + const u = new URL(url); + const features = JSON.parse(u.searchParams.get('features') as string); + features[0].clientSiteId = await db.siteId(); + features[0].clientVersion = await db.version(); + u.searchParams.set('features', JSON.stringify(features)); + const ws = new WebSocket(`${decodeURI(u.href)}`); + ws.addEventListener('error', wsErrorHandler); + return ws; +} + +export function db({ schema, name, wsUrl }) { + if (!browser) { + // No SSR for now. + return { store: () => readable([]) }; + } + + const databasePromise = Database.load({ schema, name }); + const wsPromise = setupWs({ url: wsUrl, database: databasePromise }); + + const store = ({ query, commands }) => { + const q = writable([]); + databasePromise.then(async (database) => { + const ws = await wsPromise; + const update = async () => q.set(await query(database.db)); + // Maybe this should register the listener in a store, + // we may be over subscribing since we add a listener with + // every `store` + ws.addEventListener('message', wsMessageHandler({ database, update })); + await update(); + }); + + const cmds = Object.fromEntries( + Object.entries(commands).map(([name, fn]) => [ + name, + async (args) => { + const db = await databasePromise; + const results = await fn(db.db, args); + q.set(await query(db.db)); + const changes = await db.db.exec( + `SELECT "table", hex("pk") as pk, "cid", "val", "col_version", "db_version", hex("site_id") as site_id, "cl", "seq" + FROM crsql_changes WHERE site_id = crsql_site_id() + AND db_version >= crsql_db_version()` + ); + + const changeSiteVersions = latestVersions(changes); + + changeSiteVersions.forEach(async ([changeSiteId, changeDbVersion]) => { + // await db.updateTrackedPeer(changeSiteId, changeDbVersion); + await db.db.exec( + `INSERT INTO crsql_tracked_peers (site_id, version, tag, event) + VALUES (unhex(?), ?, 0, 0) + ON CONFLICT([site_id], [tag], [event]) + DO UPDATE SET version=excluded.version`, + [changeSiteId, changeDbVersion] + ); + }); + + const ws = await wsPromise; + ws.send( + encoder.encode( + JSON.stringify({ + type: 'update', + siteId: await db.siteId(), + version: await db.version(), + changes + }) + ) + ); + return results; + } + ]) + ); + + return { + subscribe: q.subscribe, + ...cmds + }; + }; + + onDestroy(async () => { + const ws = await wsPromise; + ws.close(); + }); + + return { store }; +} diff --git a/src/lib/websockets/ws-store.ts b/src/lib/websockets/ws-store.ts index 460435f..0744527 100644 --- a/src/lib/websockets/ws-store.ts +++ b/src/lib/websockets/ws-store.ts @@ -48,7 +48,7 @@ export function wsStore({ url }: { url: string }) { // Generic send, can be customized/extended from custom store // see `send` in `chat-store` for example - send(message: string) { + send(message: string | ArrayBufferLike | Blob | ArrayBufferView) { ws?.send(message); } }; diff --git a/src/routes/app/+layout.svelte b/src/routes/app/+layout.svelte index f7655ce..69b6db2 100644 --- a/src/routes/app/+layout.svelte +++ b/src/routes/app/+layout.svelte @@ -66,6 +66,16 @@ href="/app/websocket-example/using-pub-sub">Websocket examples + + + + + + + + {#each $todos as todo} +
+ todos.toggle(todo.id)} /> + {todo.id} + {todo.content} + +
+ {/each} + + +
+

todonts

+
{ + await todonts.insert(newTodont); + newTodont = ''; + }} + > + + +
+ + {#each $todonts as todont} +
+ todonts.toggle(todont.id)} + /> + {todont.id} + {todont.content} + +
+ {/each} +
+ + +

How this works

+
    +
  1. + Setup (catch Server up, catch the Client A up) +
      +
    1. initialize and/or open Client A DB
      2. +
    2. Client A connect web socket (sending up clientAVersion and clientASiteID)
      3. +
    3. Server queries for all updates >= clientAVersion and != clientASiteID
      4. +
    4. Server `push` message with results to Client A
      5. +
    5. Client A merges results and "catches up" on all changes
      6. +
    6. Client A `push` all changes >= serverVersion and == clientASiteId to Server
      7. +
    7. Server merges and `push` changes to Client B and Client C
      8. +
    8. Client B merges change
      9. +
    9. Client C merges change
      10. +
    +
    2. +
  2. + Normal flow +
      +
    1. Client A makes update
      2. +
    2. `push` update to server
      3. +
    3. server merges change
      4. +
    4. server `push` update to Client B and Client C
      5. +
    5. Client B merges change
      6. +
    6. Client C merges change
      7. +
    +
    3. +
  3. + Database migrations +
      +
    1. ???
      2. +
    +
    4. +
diff --git a/src/routes/app/offline-first/README.md b/src/routes/app/offline-first/README.md new file mode 100644 index 0000000..2e1defa --- /dev/null +++ b/src/routes/app/offline-first/README.md @@ -0,0 +1,52 @@ +# Architecture + +## Server + +- ws-server.ts - websocket server +- websockets/handler.ts + - auth/session + - "feature" initializtion (chat, presence, offline sync...) +- features/offline/sync.ts + - WS handlers + - Redis pub/sub (could also be Redis stream? maybe make this an adapter) + - Database interactions + - Sync logic +- +page.server.ts + - construct wss url + features + - lookup/construct `dbName` for logged in user. + + +## Client + +- +page.svelte + - hook into store and react to changes + - render whatever comes back from store + - handle interactions - UI CRUD +- sync-db.ts + - initialize WASM sqlite + - open db + - load schema + - provide useful methods related to syncing + - `.merge` + - `.version` + - `.siteId` + - NOTE: can probably move some of the queries in `/features/offline/sync.ts` and `sync-db-store.ts` to this class + - provides reference to `db` to query +- sync-db-store.ts + - method for pulling `latestVersion` out of array of `changes` + - establish WS connection + - add WS listener + - initiate WASM db load + - returns `store` with bound lookup methods and reactive query + - handles WS message types/syncing logic + +# TODO + +- [ ] keep alive for WS connection +- [ ] move heavy lifting to web worker +- [ ] cache layer? +- [ ] make network layer an adapter. websocket/REST/SSE/webrtc?/etc... +- [ ] error handling/change rejection +- [ ] handle websocket buffer full - not sure what max message size is but make sure +- [ ] websocket compression? +- [ ] nodejs streams api? diff --git a/sync-dbs/.keep b/sync-dbs/.keep new file mode 100644 index 0000000..e69de29