feat(@formatjs/cli): allow extract to take in --in-file

longlho · longlho · commit bbfaef62fa7d · 2025-05-04T23:28:29.000-04:00
diff --git a/packages/cli-lib/src/cli.ts b/packages/cli-lib/src/cli.ts
@@ -6,6 +6,8 @@ import compileFolder from './compile_folder'
 import {debug} from './console_utils'
 import extract, {ExtractCLIOptions} from './extract'
 import {verify, VerifyOpts} from './verify'
+import {readFileSync} from 'fs-extra'
+import {resolve} from 'path'
 
 const KNOWN_COMMANDS = ['extract']
 
@@ -47,6 +49,11 @@ type FormatFn = <T = Record<string, MessageDescriptor>>(
 This is especially useful to convert from our extracted format to a TMS-specific format.
 `
     )
+    .option(
+      '--in-file <path>',
+      `The file containing list of files to extract from, separated by newlines. This is mainly
+      to deal with the case where you have a large number of files to extract from and bash chokes.`
+    )
     .option(
       '--out-file <path>',
       `The target file path where the plugin will output an aggregated 
@@ -116,9 +123,25 @@ sentences are not translator-friendly.`
     .action(async (filePatterns: string[], cmdObj: ExtractCLIOptions) => {
       debug('File pattern:', filePatterns)
       debug('Options:', cmdObj)
-      const files = globSync(filePatterns, {
-        ignore: cmdObj.ignore,
-      })
+      const files: string[] = []
+      if (filePatterns.length) {
+        files.push(
+          ...globSync(filePatterns, {
+            ignore: cmdObj.ignore,
+          })
+        )
+      }
+
+      if (cmdObj.inFile) {
+        debug('Reading inFile:', cmdObj.inFile)
+        const inFile = readFileSync(cmdObj.inFile, 'utf8')
+        files.push(
+          ...inFile
+            .split('\n')
+            .filter(Boolean)
+            .map(f => resolve(f))
+        )
+      }
 
       debug('Files to extract:', files)
 
@@ -135,7 +158,7 @@ sentences are not translator-friendly.`
         format: cmdObj.format,
         // It is possible that the glob pattern does NOT match anything.
         // But so long as the glob pattern is provided, don't read from stdin.
-        readFromStdin: filePatterns.length === 0,
+        readFromStdin: files.length === 0,
         preserveWhitespace: cmdObj.preserveWhitespace,
         flatten: cmdObj.flatten,
       })
diff --git a/packages/cli-lib/src/extract.ts b/packages/cli-lib/src/extract.ts
@@ -46,6 +46,10 @@ export type ExtractCLIOptions = Omit<
    * Output File
    */
   outFile?: string
+  /**
+   * Input File
+   */
+  inFile?: string
   /**
    * Ignore file glob pattern
    */
diff --git a/packages/cli/integration-tests/extract/__snapshots__/integration.test.ts.snap b/packages/cli/integration-tests/extract/__snapshots__/integration.test.ts.snap
@@ -283,6 +283,8 @@ Options:
   \`\`\` 
   This is especially useful to convert from our extracted format to a TMS-specific format.
   
+  --in-file <path>                                      The file containing list of files to extract from, separated by newlines. This is mainly
+        to deal with the case where you have a large number of files to extract from and bash chokes.
   --out-file <path>                                     The target file path where the plugin will output an aggregated 
   \`.json\` file of all the translations from the \`files\` supplied.
   --id-interpolation-pattern <pattern>                  If certain message descriptors don't have id, this \`pattern\` will be used to automatically
@@ -320,6 +322,52 @@ Options:
 }
 `;
 
+exports[`basic case: inFile 1`] = `
+{
+  "stderr": "",
+  "stdout": "",
+}
+`;
+
+exports[`basic case: inFile 2`] = `
+{
+  "286VP8": {
+    "defaultMessage": "No ID",
+    "description": "no ID",
+  },
+  "OvR0NS": {
+    "defaultMessage": "I have {count, plural, one{a dog} other{many dogs}}",
+  },
+  "app.home.kittens": {
+    "defaultMessage": "{count, plural, =0 {😭} one {# kitten} other {# kittens}}",
+    "description": "Counts kittens",
+  },
+  "escaped.apostrophe": {
+    "defaultMessage": "A quoted value ''{value}'",
+    "description": "Escaped apostrophe",
+  },
+  "foo": {
+    "defaultMessage": "Foo",
+  },
+  "foo.bar.baz": {
+    "defaultMessage": "Hello World!",
+    "description": "The default message",
+  },
+  "foo.bar.biff": {
+    "defaultMessage": "Hello Nurse!",
+    "description": "Another message",
+  },
+  "inline-msg": {
+    "defaultMessage": "inline message",
+    "description": "inline description",
+  },
+  "trailing.ws": {
+    "defaultMessage": "Some whitespace",
+    "description": "Whitespace",
+  },
+}
+`;
+
 exports[`duplicated descriptor ids shows warning 1`] = `
 {
   "foo": {
diff --git a/packages/cli/integration-tests/extract/inFile.txt b/packages/cli/integration-tests/extract/inFile.txt
@@ -0,0 +1,2 @@
+defineMessages/actual.js
+duplicated/file1.tsx
diff --git a/packages/cli/integration-tests/extract/integration.test.ts b/packages/cli/integration-tests/extract/integration.test.ts
@@ -65,6 +65,19 @@ test('basic case: defineMessages -> out-file', async () => {
   ).toMatchSnapshot()
 }, 20000)
 
+test('basic case: inFile', async () => {
+  process.chdir(__dirname)
+  await expect(
+    exec(
+      `${BIN_PATH} extract --in-file inFile.txt --out-file ${ARTIFACT_PATH}/infile-actual.json`
+    )
+  ).resolves.toMatchSnapshot()
+
+  expect(
+    await readJSON(join(ARTIFACT_PATH, 'infile-actual.json'))
+  ).toMatchSnapshot()
+})
+
 test('basic case: defineMessages -> out-file with  --additional-function-names', async () => {
   process.chdir(__dirname)
   await expect(
diff --git a/website/docs/tooling/cli.md b/website/docs/tooling/cli.md
@@ -143,6 +143,10 @@ This is especially useful to convert from our extracted format to a TMS-specific
 
 See our [builtin formatters](https://github.com/formatjs/formatjs/tree/main/packages/cli-lib/src/formatters) for examples.
 
+### `--in-file [path]`
+
+The file containing list of file paths to extract. This is useful to bypass bash command line limit.
+
 ### `--out-file [path]`
 
 The target file path where the plugin will output an aggregated `.json` file of allthe translations from the `files` supplied. This flag will ignore `--messages-dir`

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+defineMessages/actual.js`
	`2`	`+duplicated/file1.tsx`