Support globs for src and ignoring paths

package.json

@@ -71,8 +71,10 @@
     "cross-spawn": "^7.0.6",
     "estree-walker": "^2.0.1",
     "gettext-parser": "^6.0.0",
+    "glob": "13.0.0",
     "hunspell-spellchecker": "^1.0.2",
     "ignore": "^5.1.8",
+    "is-glob": "^4.0.3",
     "koa": "^2.16.4",
     "koa-body": "^7.0.1",
     "koa-router": "^14.0.0",

src/commands/check.ts

@@ -8,6 +8,7 @@ import { extractAll } from "../lib/extract";
 import { checkDuplicateKeys } from "../lib/checkDuplicateKeys";
 import * as c3poTypes from "../types";
 import { parse, PoData } from "../lib/parser";
+import { resolvePaths } from "../lib/utils";
 
 /*
 Run any string in stream through warning first
@@ -61,12 +62,15 @@ Check all keys from pots(keys only files) are present in pofile(files with trans
 */
 async function check(
     pofile: string,
-    paths: string[],
+    src: string[],
+    ignore: string[] | string | undefined,
     lang: string,
     overrideOpts?: c3poTypes.TtagOpts,
     ttagRcOpts?: c3poTypes.TtagRc,
     skip?: "translation"
 ) {
+    const paths = resolvePaths(src, ignore);
+
     const progress: c3poTypes.Progress = ora(
         `[ttag] checking translations from ${paths} ...`
     );

src/commands/extract.ts

@@ -2,20 +2,23 @@ import * as ora from "ora";
 import * as fs from "fs";
 import * as c3poTypes from "../types";
 import { extractAll } from "../lib/extract";
+import { resolvePaths } from "../lib/utils";
 
 async function extract(
     output: string,
-    paths: string[],
+    src: string[],
+    ignore: string[] | string | undefined,
     lang: string = "en",
     ttagOverrideOpts?: c3poTypes.TtagOpts,
     ttagRcOpts?: c3poTypes.TtagRc
 ) {
     const progress: c3poTypes.Progress = ora(
         `[ttag] extracting translations to ${output} ...`
     );
+
     progress.start();
     const result = await extractAll(
-        paths,
+        resolvePaths(src, ignore),
         lang,
         progress,
         ttagOverrideOpts,

src/commands/update.ts

@@ -6,20 +6,30 @@ import { updatePo } from "../lib/update";
 import { parse } from "../lib/parser";
 import { serialize, SerializeOptions } from "../lib/serializer";
 import { checkDuplicateKeys } from "../lib/checkDuplicateKeys";
+import { resolvePaths } from "../lib/utils";
 
 async function update(
     pofile: string,
     src: string[],
+    ignore: string[] | string | undefined,
     lang: string,
     ttagOverrideOpts?: ttagTypes.TtagOpts,
     ttagRcOpts?: ttagTypes.TtagRc,
     serializeOpts?: SerializeOptions
 ) {
+    const paths = resolvePaths(src, ignore);
+
     const progress: ttagTypes.Progress = ora(`[ttag] updating ${pofile} ...`);
     progress.start();
     try {
         const pot = parse(
-            await extractAll(src, lang, progress, ttagOverrideOpts, ttagRcOpts)
+            await extractAll(
+                paths,
+                lang,
+                progress,
+                ttagOverrideOpts,
+                ttagRcOpts
+            )
         );
         const errMessage = checkDuplicateKeys(pot);
 

src/index.ts

@@ -84,12 +84,22 @@ yargs
                 default: "en",
                 description: "sets default lang (ISO format)"
             },
+            src: {
+                description:
+                    "path to source files/directories (supports glob, but needs to be quoted)"
+            },
+            ignore: {
+                alias: "i",
+                description:
+                    "paths to ignore (supports glob, but needs to be quoted)"
+            },
             ...getTtagOptsForYargs()
         },
         (argv: any) => {
             extract(
                 argv.output,
                 argv.src,
+                argv.ignore,
                 argv.lang,
                 parseTtagPluginOpts(argv),
                 parseTtagRcOpts()
@@ -110,12 +120,22 @@ yargs
                 choices: ["translation"],
                 default: undefined
             },
+            src: {
+                description:
+                    "path to source files/directories (supports glob, but needs to be quoted)"
+            },
+            ignore: {
+                alias: "i",
+                description:
+                    "paths to ignore (supports glob, but needs to be quoted)"
+            },
             ...getTtagOptsForYargs()
         },
         (argv: any) => {
             check(
                 argv.pofile,
                 argv.src,
+                argv.ignore,
                 argv.lang,
                 parseTtagPluginOpts(argv),
                 parseTtagRcOpts(),
@@ -226,7 +246,13 @@ yargs
                 description: "path to .po file with translations"
             },
             src: {
-                description: "path to source files/directories"
+                description:
+                    "path to source files/directories (supports glob, but needs to be quoted)"
+            },
+            ignore: {
+                alias: "i",
+                description:
+                    "paths to ignore (supports glob, but needs to be quoted)"
             },
             ...getTtagOptsForYargs(),
             foldLength: {
@@ -238,6 +264,7 @@ yargs
             update(
                 argv.pofile,
                 argv.src,
+                argv.ignore,
                 argv.lang,
                 parseTtagPluginOpts(argv),
                 parseTtagRcOpts(),

src/lib/utils.ts

@@ -1,6 +1,9 @@
 import { Translations, Message, PoData, PoDataCompact } from "./parser";
 import generate from "@babel/generator";
 import { Node } from "@babel/types";
+import { globSync } from "glob";
+import * as fs from "fs";
+const isGlob = require("is-glob");
 
 const pluralNumRegex = /^nplurals ?= ?(\d);/;
 
@@ -70,3 +73,46 @@ export function convert2Compact(poData: PoData): PoDataCompact {
     delete compactPo.contexts[""][""];
     return compactPo;
 }
+
+/**
+ * Helper function that calculates all the file paths from a src array with optional ignore paths.
+ * The folder paths are ignored, since further directory traversing is not necessary.
+ */
+export function resolvePaths(
+    /** Regular paths or globs */
+    src: string[],
+    /**
+     * Could be a string or an array of strings, depending on how many paths are passed by the cli.
+     *
+     * `--ignore path1` => 'path1'
+     *
+     * `--ignore path1 --ignore path2` => ['path1', 'path2']
+     */
+    ignore?: string | string[]
+): string[] {
+    const toGlob = (path: string): string => {
+        if (fs.lstatSync(path).isDirectory()) {
+            if (!path.endsWith("/")) {
+                path = `${path}/`;
+            }
+            return `${path}**/*`;
+        } else {
+            return path;
+        }
+    };
+
+    /** All paths are internally transformed to globs, to keep backwards compatibility with 'src' paths */
+    const srcGlob = src.map(path => (isGlob(path) ? path : toGlob(path)));
+    if (typeof ignore === "string") {
+        ignore = [ignore];
+    }
+    const ignoreGlob = ignore?.map(path =>
+        isGlob(path) ? path : toGlob(path)
+    );
+
+    return globSync(srcGlob, {
+        absolute: true,
+        nodir: true,
+        ignore: ignoreGlob
+    });
+}

tests/commands/__snapshots__/test_check.ts.snap

@@ -1,3 +1,7 @@
 // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
 
+exports[`check from glob source 1`] = `""`;
+
+exports[`check from glob source with ignore 1`] = `""`;
+
 exports[`check when all string are translated 1`] = `""`;

tests/commands/__snapshots__/test_extract.ts.snap

@@ -18,6 +18,37 @@ msgstr \\"\\"
 "
 `;
 
+exports[`extract from glob source with filter (ignore all) 1`] = `
+"msgid \\"\\"
+msgstr \\"\\"
+\\"Content-Type: text/plain; charset=utf-8\\\\n\\"
+\\"Plural-Forms: nplurals=2; plural=(n!=1);\\\\n\\"
+
+#: tests/fixtures/baseTest/test1.js:4
+#, javascript-format
+msgid \\"test translation \${ name }\\"
+msgstr \\"\\"
+
+#: tests/fixtures/baseTest/nested/test2.js:4
+#, javascript-format
+msgid \\"test translation 2 \${ name }\\"
+msgstr \\"\\"
+"
+`;
+
+exports[`extract from glob source with filter 1`] = `
+"msgid \\"\\"
+msgstr \\"\\"
+\\"Content-Type: text/plain; charset=utf-8\\\\n\\"
+\\"Plural-Forms: nplurals=2; plural=(n!=1);\\\\n\\"
+
+#: tests/fixtures/baseTest/nested/test2.js:4
+#, javascript-format
+msgid \\"test translation 2 \${ name }\\"
+msgstr \\"\\"
+"
+`;
+
 exports[`extract from js with another default locale 1`] = `
 "msgid \\"\\"
 msgstr \\"\\"

tests/commands/__snapshots__/test_update.ts.snap

@@ -45,3 +45,26 @@ msgstr[0] \\"\\"
 msgstr[1] \\"\\"
 "
 `;
+
+exports[`test update po with glob 1`] = `
+"msgid \\"\\"
+msgstr \\"\\"
+\\"Content-Type: text/plain; charset=utf-8\\\\n\\"
+\\"Plural-Forms: nplurals=2; plural=(n!=1);\\\\n\\"
+
+#: tests/fixtures/updateTest/test.js:3
+msgid \\"test\\"
+msgstr \\"\\"
+
+#: tests/fixtures/updateTest/test.js:4
+msgid \\"new\\"
+msgstr \\"\\"
+
+#: tests/fixtures/updateTest/test.js:6
+#, javascript-format
+msgid \\"\${ n } banana\\"
+msgid_plural \\"\${ n } bananas\\"
+msgstr[0] \\"\\"
+msgstr[1] \\"\\"
+"
+`;

tests/commands/test_check.ts

@@ -12,6 +12,10 @@ const checkNotPass = path.resolve(
     __dirname,
     "../fixtures/checkTest/check-trans-not-exist.js"
 );
+const checkNotPassGlob = path.resolve(
+    __dirname,
+    "../fixtures/**/check-trans-not-exist.js"
+);
 const checkInvalidFormat = path.resolve(
     __dirname,
     "../fixtures/checkTest/check-trans-invalid-format.js"
@@ -26,6 +30,10 @@ const checkSameKey = path.resolve(
     __dirname,
     "../fixtures/checkTest/check-same-key.js"
 );
+const checkPassGlob = path.resolve(
+    __dirname,
+    "../fixtures/**/check-trans-exist.js"
+);
 
 test("check when all string are translated", () => {
     const result = execSync(
@@ -104,3 +112,18 @@ test("check same key", () => {
         expect(err.stderr.toString()).toContain(errMessage);
     }
 });
+
+test("check from glob source", () => {
+    const result = execSync(
+        `ts-node src/index.ts check ${poPath} ${checkPassGlob}`
+    );
+    expect(result.toString()).toMatchSnapshot();
+});
+
+test("check from glob source with ignore", () => {
+    // this check should normally fail, the ignore makes it pass
+    const result = execSync(
+        `ts-node src/index.ts check ${poPath} ${checkNotPassGlob} --ignore ${checkNotPassGlob}`
+    );
+    expect(result.toString()).toMatchSnapshot();
+});

tests/commands/test_extract.ts

@@ -5,6 +5,7 @@ import { execSync } from "child_process";
 
 const potPath = path.resolve(__dirname, "../../dist/translation.pot");
 const baseTestPath = path.resolve(__dirname, "../fixtures/baseTest");
+const baseTestPathGlob = path.resolve(__dirname, "../fixtures/baseTest/**");
 const sortByMsgidPath = path.resolve(__dirname, "../fixtures/sortByMsgidTest");
 const ukTestPath = path.resolve(__dirname, "../fixtures/ukLocaleTest");
 const jsxPath = path.resolve(__dirname, "../fixtures/testJSXParse.jsx");
@@ -149,3 +150,19 @@ test("extract from ts with declare", () => {
     const result = fs.readFileSync(potPath).toString();
     expect(result).toMatchSnapshot();
 });
+
+test("extract from glob source with filter", () => {
+    execSync(
+        `ts-node src/index.ts extract -o ${potPath} ${baseTestPathGlob} -i **/test1.js`
+    );
+    const result = fs.readFileSync(potPath).toString();
+    expect(result).toMatchSnapshot();
+});
+
+test("extract from glob source with filter (ignore all)", () => {
+    execSync(
+        `ts-node src/index.ts extract -o ${potPath} ${baseTestPathGlob} -i **/*.js`
+    );
+    const result = fs.readFileSync(potPath).toString();
+    expect(result).toMatchSnapshot();
+});

tests/commands/test_update.ts

@@ -17,6 +17,7 @@ msgstr "obsolete trans"
 `;
 
 const srcPath = path.resolve(__dirname, "../fixtures/updateTest/test.js");
+const srcPathGlob = path.resolve(__dirname, "../fixtures/updateTest/test.*");
 
 test("test update po", () => {
     const tmpFile = tmp.fileSync();
@@ -171,3 +172,12 @@ test("should apply useProjectBabelrc opt and fail with react compiler", () => {
         tmpFile.removeCallback();
     }
 });
+
+test("test update po with glob", () => {
+    const tmpFile = tmp.fileSync();
+    fs.writeFileSync(tmpFile.name, originalPo);
+    execSync(`ts-node src/index.ts update ${tmpFile.name} ${srcPathGlob}`);
+    const result = fs.readFileSync(tmpFile.name).toString();
+    expect(result).toMatchSnapshot();
+    tmpFile.removeCallback();
+});

tests/fixtures/utilsTest/root.ts

@@ -0,0 +1 @@
+export const greet = () => console.log('hi')

tests/fixtures/utilsTest/subfolder-1/a.ts

@@ -0,0 +1 @@
+export const greet = () => console.log('hi')

tests/fixtures/utilsTest/subfolder-1/b.js

@@ -0,0 +1 @@
+export const greet = () => console.log('hi')

tests/fixtures/utilsTest/subfolder-2/a.ts

@@ -0,0 +1 @@
+export const greet = () => console.log('hi')

tests/fixtures/utilsTest/subfolder-2/b.js

@@ -0,0 +1 @@
+export const greet = () => console.log('hi')