Merge pull request #672 from highcharts/enhancement/highcharts-npm-scripts

enhancement/highcharts-npm-scripts

Author: cvasseng

.env.sample

@@ -4,6 +4,7 @@ PUPPETEER_TEMP_DIR = ./tmp/
 # HIGHCHARTS CONFIG
 HIGHCHARTS_VERSION = latest
 HIGHCHARTS_CDN_URL = https://code.highcharts.com/
+HIGHCHARTS_USE_NPM = false
 HIGHCHARTS_CORE_SCRIPTS =
 HIGHCHARTS_MODULE_SCRIPTS =
 HIGHCHARTS_INDICATOR_SCRIPTS =

.prettierrc .prettierrc.json.prettierrc renamed to .prettierrc.json

@@ -1,3 +1,9 @@
+# 5.1.0
+
+_New Features:_
+
+- Added the `useNpm` option to load Highcharts scripts from the NPM package instead of the CDN.
+
 # 5.0.0
 
 _Breaking Changes:_

CHANGELOG.md

@@ -114,6 +114,7 @@ The format, along with its default values, is as follows (using the recommended
   "highcharts": {
     "version": "latest",
     "cdnURL": "https://code.highcharts.com/",
+    "useNpm": false,
     "coreScripts": [
       "highcharts",
       "highcharts-more",
@@ -307,6 +308,7 @@ These variables are set in your environment and take precedence over options fro
 
 - `HIGHCHARTS_VERSION`: Highcharts version to use (defaults to `latest`).
 - `HIGHCHARTS_CDN_URL`: Highcharts CDN URL of scripts to be used (defaults to `https://code.highcharts.com/`).
+- `HIGHCHARTS_USE_NPM`: The flag that determines whether to use Highcharts scripts from CDN or NPM package (defaults to `false`).
 - `HIGHCHARTS_CORE_SCRIPTS`: Highcharts core scripts to fetch (defaults to ``).
 - `HIGHCHARTS_MODULE_SCRIPTS`: Highcharts module scripts to fetch (defaults to ``).
 - `HIGHCHARTS_INDICATOR_SCRIPTS`: Highcharts indicator scripts to fetch (defaults to ``).
@@ -412,6 +414,7 @@ To supply command line arguments, add them as flags when running the application
 
 _Available options:_
 
+- `--useNpm`: The flag that determines whether to use Highcharts scripts from CDN or NPM package (defaults to `false`).
 - `--infile`: The input file should include a name and a type (**.json** or **.svg**) and must be a correctly formatted JSON or SVG file (defaults to `false`).
 - `--instr`: An input in a form of a stringified JSON or SVG file. Overrides the `--infile` option (defaults to `false`).
 - `--options`: An alias for the `--instr` option (defaults to `false`).
@@ -562,6 +565,10 @@ curl -H 'hc-auth: 12345' -X POST 127.0.0.1:7801/change_hc_version/10.3.3
 
 This is useful to e.g. upgrade to the latest HC version without downtime.
 
+IMPORTANT NOTE:
+
+This is not possible when using the Highcharts dependency package directly (by setting the `useNpm` to **true**).
+
 # Node.js Module
 
 Finally, the Export Server can also be used as a Node.js module to simplify integrations:
@@ -709,6 +716,10 @@ Samples and tests for every mentioned export method can be found in the `./sampl
 
 # Tips, Tricks & Notes
 
+## Note About Highcharts Version
+
+When `useNpm` is set to **true**, Highcharts uses the version specified in `package.json`. The `version` option or switching the Highcharts version on the server at runtime will have no effect. To change the version of local Highcharts scripts, update `package.json` directly.
+
 ## Note about Deprecated Options
 
 At some point during the transition process from the `PhantomJS` solution, certain options were deprecated. Here is a list of options that no longer work with the server based on `Puppeteer`:

README.md

@@ -17,15 +17,15 @@ See LICENSE file in root for details.
 // before starting the service
 
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
-import { join } from 'path';
+import { join, resolve, sep } from 'path';
 
 import { HttpsProxyAgent } from 'https-proxy-agent';
 
 import { getOptions } from './config.js';
 import { envs } from './envs.js';
 import { fetch } from './fetch.js';
 import { log } from './logger.js';
-import { __dirname } from './utils.js';
+import { __dirname, __highchartsDir } from './utils.js';
 
 import ExportError from './errors/ExportError.js';
 
@@ -52,12 +52,14 @@ export const extractVersion = (cache) => {
 
 /**
  * Extracts the Highcharts module name based on the scriptPath.
+ *
+ * @param {string} scriptPath - The path to the module.
+ *
+ * @returns {string} The name of a module.
  */
 export const extractModuleName = (scriptPath) => {
-  return scriptPath.replace(
-    /(.*)\/|(.*)modules\/|stock\/(.*)indicators\/|maps\/(.*)modules\//gi,
-    ''
-  );
+  // Normalize slashes, get part after the last '/' and remove .js extension
+  return scriptPath.replace(/\\/g, '/').split('/').pop().replace(/\.js$/i, '');
 };
 
 /**
@@ -102,8 +104,10 @@ export const saveConfigToManifest = async (config, fetchedModules) => {
  * to use for a request.
  * @param {Object} fetchedModules - An object which tracks which Highcharts
  * modules have been fetched.
- * @param {boolean} shouldThrowError - A flag to indicate if the error should be
- * thrown. This should be used only for the core scripts.
+ * @param {boolean} [useNpm=false] - A flag to indicate if the script should be
+ * get from NPM package or fetched from CDN. The default value is `false`.
+ * @param {boolean} [shouldThrowError=false] - A flag to indicate if the error
+ * should be thrown. This should be used only for the core scripts.
  *
  * @returns {Promise<string>} A Promise resolving to the text representation
  * of the fetched script.
@@ -115,36 +119,71 @@ export const fetchAndProcessScript = async (
   script,
   requestOptions,
   fetchedModules,
+  useNpm = false,
   shouldThrowError = false
 ) => {
-  // Get rid of the .js from the custom strings
-  if (script.endsWith('.js')) {
-    script = script.substring(0, script.length - 3);
+  let response;
+
+  // Add the missing .js to the strings
+  if (!script.endsWith('.js')) {
+    script = `${script}.js`;
   }
 
-  log(4, `[cache] Fetching script - ${script}.js`);
+  // Whether to use NPM package scripts or fetch it from CDN
+  if (useNpm) {
+    try {
+      // Log fetched script
+      log(
+        4,
+        `[cache] Fetching script from NPM - ${join('node_modules', 'highcharts', script)}`
+      );
+
+      // Sanitize and validate path
+      const resolvedScriptPath = resolve(__highchartsDir, script);
+      if (!resolvedScriptPath.startsWith(resolve(__highchartsDir) + sep)) {
+        throw new ExportError(
+          `[cache] Invalid script path detected for '${script}'. Directory traversal attempt or bad version input.`,
+          403
+        );
+      }
 
-  // Fetch the script
-  const response = await fetch(`${script}.js`, requestOptions);
+      // Fetch the script from NPM
+      response = readFileSync(resolvedScriptPath, 'utf8');
 
-  // If OK, return its text representation
-  if (response.statusCode === 200 && typeof response.text == 'string') {
-    if (fetchedModules) {
-      const moduleName = extractModuleName(script);
-      fetchedModules[moduleName] = 1;
+      // If OK, return its text representation
+      if (fetchedModules && response) {
+        fetchedModules[extractModuleName(script)] = 1;
+      }
+      return response;
+    } catch {
+      // Proceed
     }
+  } else {
+    // Log fetched script
+    log(4, `[cache] Fetching script from CDN - ${script}`);
 
-    return response.text;
+    // Fetch the script from CDN
+    response = await fetch(script, requestOptions);
+
+    // If OK, return its text representation
+    if (response.statusCode === 200 && typeof response.text == 'string') {
+      if (fetchedModules) {
+        fetchedModules[extractModuleName(script)] = 1;
+      }
+      return response.text;
+    }
   }
 
+  // Based on the `shouldThrowError` flag, decide how to serve error message
   if (shouldThrowError) {
     throw new ExportError(
-      `Could not fetch the ${script}.js. The script might not exist in the requested version (status code: ${response.statusCode}).`
-    ).setError(response);
+      `[cache] Could not fetch the mandatory ${script}. The script might not exist in the requested version.`,
+      404
+    );
   } else {
     log(
       2,
-      `[cache] Could not fetch the ${script}.js. The script might not exist in the requested version.`
+      `[cache] Could not fetch the ${script}. The script might not exist in the requested version.`
     );
   }
 
@@ -154,10 +193,7 @@ export const fetchAndProcessScript = async (
 /**
  * Fetches Highcharts scripts and customScripts from the given CDNs.
  *
- * @param {string} coreScripts - Array of Highcharts core scripts to fetch.
- * @param {string} moduleScripts - Array of Highcharts modules to fetch.
- * @param {string} customScripts - Array of custom script paths to fetch
- * (full URLs).
+ * @param {Object} highchartsOptions - Object containing all highcharts options.
  * @param {object} proxyOptions - Options for the proxy agent to use for
  * a request.
  * @param {object} fetchedModules - An object which tracks which Highcharts
@@ -166,12 +202,22 @@ export const fetchAndProcessScript = async (
  * @returns {Promise<string>} The fetched scripts content joined.
  */
 export const fetchScripts = async (
-  coreScripts,
-  moduleScripts,
-  customScripts,
+  highchartsOptions,
   proxyOptions,
   fetchedModules
 ) => {
+  const version = highchartsOptions.version;
+  const hcVersion = version === 'latest' || !version ? '' : `${version}/`;
+  const cdnURL = highchartsOptions.cdnURL || cache.cdnURL;
+
+  log(
+    3,
+    `[cache] Updating cache version to Highcharts: ${hcVersion || 'latest'}.`
+  );
+
+  // Whether to use NPM or CDN
+  const useNpm = highchartsOptions.useNpm;
+
   // Configure proxy if exists
   let proxyAgent;
   const { host, port, username, password } = proxyOptions;
@@ -199,26 +245,49 @@ export const fetchScripts = async (
       }
     : {};
 
-  const allFetchPromises = [
-    ...coreScripts.map((script) =>
-      fetchAndProcessScript(`${script}`, requestOptions, fetchedModules, true)
+  const fetchedScripts = await Promise.all([
+    ...highchartsOptions.coreScripts.map((c) =>
+      fetchAndProcessScript(
+        (useNpm && c) || `${cdnURL}${hcVersion}${c}`,
+        requestOptions,
+        fetchedModules,
+        useNpm,
+        true
+      )
     ),
-    ...moduleScripts.map((script) =>
-      fetchAndProcessScript(`${script}`, requestOptions, fetchedModules)
+    ...highchartsOptions.moduleScripts.map((m) =>
+      fetchAndProcessScript(
+        (useNpm && join('modules', m)) ||
+          (m === 'map'
+            ? `${cdnURL}maps/${hcVersion}modules/${m}`
+            : `${cdnURL}${hcVersion}modules/${m}`),
+        requestOptions,
+        fetchedModules,
+        useNpm
+      )
+    ),
+    ...highchartsOptions.indicatorScripts.map((i) =>
+      fetchAndProcessScript(
+        (useNpm && join('indicators', i)) ||
+          `${cdnURL}stock/${hcVersion}indicators/${i}`,
+        requestOptions,
+        fetchedModules,
+        useNpm
+      )
     ),
-    ...customScripts.map((script) =>
-      fetchAndProcessScript(`${script}`, requestOptions)
+    ...highchartsOptions.customScripts.map((c) =>
+      fetchAndProcessScript(`${c}`, requestOptions)
     )
-  ];
+  ]);
 
-  const fetchedScripts = await Promise.all(allFetchPromises);
   return fetchedScripts.join(';\n');
 };
 
 /**
  * Updates the local cache with Highcharts scripts and their versions.
  *
- * @param {Object} options - Object containing all options.
+ * @param {Object} highchartsOptions - Object containing all options from
+ * the highcharts section.
  * @param {string} sourcePath - The path to the source file in the cache.
  *
  * @returns {Promise<object>} A Promise resolving to an object representing
@@ -232,40 +301,22 @@ export const updateCache = async (
   proxyOptions,
   sourcePath
 ) => {
-  const version = highchartsOptions.version;
-  const hcVersion = version === 'latest' || !version ? '' : `${version}/`;
-  const cdnURL = highchartsOptions.cdnURL || cache.cdnURL;
-
-  log(
-    3,
-    `[cache] Updating cache version to Highcharts: ${hcVersion || 'latest'}.`
-  );
-
-  const fetchedModules = {};
   try {
+    const fetchedModules = {};
+
+    // Get sources
     cache.sources = await fetchScripts(
-      [
-        ...highchartsOptions.coreScripts.map((c) => `${cdnURL}${hcVersion}${c}`)
-      ],
-      [
-        ...highchartsOptions.moduleScripts.map((m) =>
-          m === 'map'
-            ? `${cdnURL}maps/${hcVersion}modules/${m}`
-            : `${cdnURL}${hcVersion}modules/${m}`
-        ),
-        ...highchartsOptions.indicatorScripts.map(
-          (i) => `${cdnURL}stock/${hcVersion}indicators/${i}`
-        )
-      ],
-      highchartsOptions.customScripts,
+      highchartsOptions,
       proxyOptions,
       fetchedModules
     );
 
+    // Get sources version
     cache.hcVersion = extractVersion(cache);
 
     // Save the fetched modules into caches' source JSON
     writeFileSync(sourcePath, cache.sources);
+
     return fetchedModules;
   } catch (error) {
     throw new ExportError(

dist/index.cjs

@@ -141,6 +141,7 @@ export const Config = z.object({
       })
     )
     .transform((value) => (value !== '' ? value : undefined)),
+  HIGHCHARTS_USE_NPM: v.boolean(),
   HIGHCHARTS_CORE_SCRIPTS: v.array(scriptsNames.core),
   HIGHCHARTS_MODULE_SCRIPTS: v.array(scriptsNames.modules),
   HIGHCHARTS_INDICATOR_SCRIPTS: v.array(scriptsNames.indicators),

dist/index.esm.js

@@ -45,7 +45,7 @@ export const scriptsNames = {
     'geoheatmap',
     'pyramid3d',
     'networkgraph',
-    'overlapping-datalabels',
+    // 'overlapping-datalabels',
     'pareto',
     'pattern-fill',
     'pictorial',
@@ -168,6 +168,12 @@ export const defaultConfig = {
       envLink: 'HIGHCHARTS_CDN_URL',
       description: 'The CDN URL for Highcharts scripts to be used.'
     },
+    useNpm: {
+      value: false,
+      type: 'boolean',
+      envLink: 'HIGHCHARTS_USE_NPM',
+      description: 'Flag to use Highcharts scripts from NPM package'
+    },
     coreScripts: {
       value: scriptsNames.core,
       type: 'string[]',
@@ -753,6 +759,12 @@ export const promptsConfig = {
       message: 'The URL of CDN',
       initial: defaultConfig.highcharts.cdnURL.value
     },
+    {
+      type: 'toggle',
+      name: 'useNpm',
+      message: 'Flag to use Highcharts scripts from NPM package',
+      initial: defaultConfig.highcharts.useNpm.value
+    },
     {
       type: 'multiselect',
       name: 'coreScripts',