diff --git a/package-lock.json b/package-lock.json index 79a166a5..5c1b860a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "node-llama-cpp": "^3.0.0-beta.44", "node-stream-zip": "^1.15.0", "openai": "^4.79.1", + "pdfjs-dist": "^2.16.105", "systeminformation": "^5.23.5", "tiktoken": "^1.0.16" }, @@ -327,31 +328,31 @@ "license": "MIT" }, "node_modules/@napi-rs/canvas": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.65.tgz", - "integrity": "sha512-YcFhXQcp+b2d38zFOJNbpyPHnIL7KAEkhJQ+UeeKI5IpE9B8Cpf/M6RiHPQXSsSqnYbrfFylnW49dyh2oeSblQ==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.67.tgz", + "integrity": "sha512-VA4Khm/5Kg2bQGx3jXotTC4MloOG8b1Ung80exafUK0k5u6yJmIz3Q2iXeeWZs5weV+LQOEB+CPKsYwEYaGAjw==", "license": "MIT", "optional": true, "engines": { "node": ">= 10" }, "optionalDependencies": { - "@napi-rs/canvas-android-arm64": "0.1.65", - "@napi-rs/canvas-darwin-arm64": "0.1.65", - "@napi-rs/canvas-darwin-x64": "0.1.65", - "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.65", - "@napi-rs/canvas-linux-arm64-gnu": "0.1.65", - "@napi-rs/canvas-linux-arm64-musl": "0.1.65", - "@napi-rs/canvas-linux-riscv64-gnu": "0.1.65", - "@napi-rs/canvas-linux-x64-gnu": "0.1.65", - "@napi-rs/canvas-linux-x64-musl": "0.1.65", - "@napi-rs/canvas-win32-x64-msvc": "0.1.65" + "@napi-rs/canvas-android-arm64": "0.1.67", + "@napi-rs/canvas-darwin-arm64": "0.1.67", + "@napi-rs/canvas-darwin-x64": "0.1.67", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.67", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.67", + "@napi-rs/canvas-linux-arm64-musl": "0.1.67", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.67", + "@napi-rs/canvas-linux-x64-gnu": "0.1.67", + "@napi-rs/canvas-linux-x64-musl": "0.1.67", + "@napi-rs/canvas-win32-x64-msvc": "0.1.67" } }, "node_modules/@napi-rs/canvas-android-arm64": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.65.tgz", - "integrity": "sha512-ZYwqFYEKcT5Zr8lbiaJNJj/poLaeK2TncolY914r+gD2TJNeP7ZqvE7A2SX/1C9MB4E3DQEwm3YhL3WEf0x3MQ==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.67.tgz", + "integrity": "sha512-W+3DFG5h0WU8Vqqb3W5fNmm5/TPH5ECZRinQDK4CAKFSUkc4iZcDwrmyFG9sB4KdHazf1mFVHCpEeVMO6Mk6Zg==", "cpu": [ "arm64" ], @@ -365,9 +366,9 @@ } }, "node_modules/@napi-rs/canvas-darwin-arm64": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.65.tgz", - "integrity": "sha512-Pg1pfiJEyDIsX+V0QaJPRWvXbw5zmWAk3bivFCvt/5pwZb37/sT6E/RqPHT9NnqpDyKW6SriwY9ypjljysUA1Q==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.67.tgz", + "integrity": "sha512-xzrv7QboI47yhIHR5P5u/9KGswokuOKLiKSukr1Ku03RRJxP6lGuVtrAZAgdRg7F9FsuF2REf2yK53YVb6pMlA==", "cpu": [ "arm64" ], @@ -381,9 +382,9 @@ } }, "node_modules/@napi-rs/canvas-darwin-x64": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.65.tgz", - "integrity": "sha512-3Tr+/HjdJN7Z/VKIcsxV2DvDIibZCExgfYTgljCkUSFuoI7iNkOE6Dc1Q6j212EB9PeO8KmfrViBqHYT6IwWkA==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.67.tgz", + "integrity": "sha512-SNk9lYBr84N0gW8MZ2IrjygFtbFBILr3SEqMdHzHHuph20SQmssFvJGPZwSSCMEyKAvyqhogbmlew0te5Z4w9Q==", "cpu": [ "x64" ], @@ -397,9 +398,9 @@ } }, "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.65.tgz", - "integrity": "sha512-3KP+dYObH7CVkZMZWwk1WX9jRjL+EKdQtD43H8MOI+illf+dwqLlecdQ4d9bQRIxELKJ8dyPWY4fOp/Ngufrdg==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.67.tgz", + "integrity": "sha512-qmBlSvUpl567bzH8tNXi82u5FrL4d0qINqd6K9O7GWGGGFmKMJdrgi2/SW3wwCTxqHBasIDdVWc4KSJfwyaoDQ==", "cpu": [ "arm" ], @@ -413,9 +414,9 @@ } }, "node_modules/@napi-rs/canvas-linux-arm64-gnu": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.65.tgz", - "integrity": "sha512-Ka3StKz7Dq7kjTF3nNJCq43UN/VlANS7qGE3dWkn1d+tQNsCRy/wRmyt1TUFzIjRqcTFMQNRbgYq84+53UBA0A==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.67.tgz", + "integrity": "sha512-k3nAPQefkMeFuJ65Rqdnx92KX1JXQhEKjjWeKsCJB+7sIBgQUWtHo9c3etfVLv5pkWJJDFi/Zc2soNkH3E8dRA==", "cpu": [ "arm64" ], @@ -429,9 +430,9 @@ } }, "node_modules/@napi-rs/canvas-linux-arm64-musl": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.65.tgz", - "integrity": "sha512-O4xMASm2JrmqYoiDyxVWi+z5C14H+oVEag2rZ5iIA67dhWqYZB+iO7wCFpBYRj31JPBR29FOsu6X9zL+DwBFdw==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.67.tgz", + "integrity": "sha512-lZwHWR1cCP408l86n3Qbs3X1oFeAYMjJIQvQl1VMZh6wo5PfI+jaZSKBUOd8x44TnVllX9yhLY9unNRztk/sUQ==", "cpu": [ "arm64" ], @@ -445,9 +446,9 @@ } }, "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.65.tgz", - "integrity": "sha512-dblWDaA59ZU8bPbkfM+riSke7sFbNZ70LEevUdI5rgiFEUzYUQlU34gSBzemTACj5rCWt1BYeu0GfkLSjNMBSw==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.67.tgz", + "integrity": "sha512-PdBC9p6bLHA1W3OdA0vTHj701SB/kioGQ1uCFBRMs5KBCaMLb/H4aNi8uaIUIEvBWnxeAjoNcLU7//q0FxEosw==", "cpu": [ "riscv64" ], @@ -461,9 +462,9 @@ } }, "node_modules/@napi-rs/canvas-linux-x64-gnu": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.65.tgz", - "integrity": "sha512-wsp+atutw13OJXGU3DDkdngtBDoEg01IuK5xMe0L6VFPV8maGkh17CXze078OD5QJOc6kFyw3DDscMLOPF8+oA==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.67.tgz", + "integrity": "sha512-kJJX6eWzjipL/LdKOWCJctc88e5yzuXri8+s0V/lN06OwuLGW62TWS3lvi8qlUrGMOfRGabSWWlB4omhASSB8w==", "cpu": [ "x64" ], @@ -477,9 +478,9 @@ } }, "node_modules/@napi-rs/canvas-linux-x64-musl": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.65.tgz", - "integrity": "sha512-odX+nN+IozWzhdj31INcHz3Iy9+EckNw+VqsZcaUxZOTu7/3FmktRNI6aC1qe5minZNv1m05YOS1FVf7fvmjlA==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.67.tgz", + "integrity": "sha512-jLKiPWGeN6ZzhnaLG7ex7eexsiHJ1mdtPK1qKvETIcu45dApMXyUIHvdL6XWB5gFFtj5ScHzLUxv1vkfPZsoxA==", "cpu": [ "x64" ], @@ -493,9 +494,9 @@ } }, "node_modules/@napi-rs/canvas-win32-x64-msvc": { - "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.65.tgz", - "integrity": "sha512-RZQX3luWnlNWgdMnLMQ1hyfQraeAn9lnxWWVCHuUM4tAWEV8UDdeb7cMwmJW7eyt8kAosmjeHt3cylQMHOxGFg==", + "version": "0.1.67", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.67.tgz", + "integrity": "sha512-K/JmkOFbc4iRZYUqJhj0jwqfHA/wNQEmTiGNsgZ6d59yF/IBNp5T0D5eg3B8ghjI8GxDYCiSJ6DNX8mC3Oh2EQ==", "cpu": [ "x64" ], @@ -1043,6 +1044,18 @@ "pdf2md": "lib/pdf2md-cli.js" } }, + "node_modules/@opendocsg/pdf2md/node_modules/pdfjs-dist": { + "version": "4.10.38", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.10.38.tgz", + "integrity": "sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.65" + } + }, "node_modules/@reflink/reflink": { "version": "0.1.19", "resolved": "https://registry.npmjs.org/@reflink/reflink/-/reflink-0.1.19.tgz", @@ -1737,6 +1750,13 @@ "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", "license": "MIT" }, + "node_modules/dommatrix": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/dommatrix/-/dommatrix-1.0.3.tgz", + "integrity": "sha512-l32Xp/TLgWb8ReqbVJAFIvXmY7go4nTxxlWiAFyhoQw9RKEOHBZNnyGvJWqDVSPmq3Y9HlM4npqF/T6VMOXhww==", + "deprecated": "dommatrix is no longer maintained. Please use @thednp/dommatrix.", + "license": "MIT" + }, "node_modules/dotenv": { "version": "16.4.7", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", @@ -2842,15 +2862,30 @@ } }, "node_modules/pdfjs-dist": { - "version": "4.10.38", - "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.10.38.tgz", - "integrity": "sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==", + "version": "2.16.105", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.16.105.tgz", + "integrity": "sha512-J4dn41spsAwUxCpEoVf6GVoz908IAA3mYiLmNxg8J9kfRXc2jxpbUepcP0ocp0alVNLFthTAM8DZ1RaHh8sU0A==", "license": "Apache-2.0", - "engines": { - "node": ">=20" + "dependencies": { + "dommatrix": "^1.0.3", + "web-streams-polyfill": "^3.2.1" }, - "optionalDependencies": { - "@napi-rs/canvas": "^0.1.65" + "peerDependencies": { + "worker-loader": "^3.0.8" + }, + "peerDependenciesMeta": { + "worker-loader": { + "optional": true + } + } + }, + "node_modules/pdfjs-dist/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" } }, "node_modules/pretty-bytes": { diff --git a/package.json b/package.json index 7365a12b..1cbb7a8f 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "node-llama-cpp": "^3.0.0-beta.44", "node-stream-zip": "^1.15.0", "openai": "^4.79.1", + "pdfjs-dist": "^2.16.105", "systeminformation": "^5.23.5", "tiktoken": "^1.0.16" }, diff --git a/scripts/knowledge2character.js b/scripts/knowledge2character.js index c2ca0f30..0b54f823 100755 --- a/scripts/knowledge2character.js +++ b/scripts/knowledge2character.js @@ -64,7 +64,7 @@ const main = async () => { outputFilePath = `${characterName}.knowledge.character.json`; } - character.knowledge = knowledge; + character.knowledge = knowledge.chunks; writeJsonFile(outputFilePath, character);