loader: add experimental text and bytes import

lib/internal/modules/esm/assert.js

@@ -8,6 +8,7 @@ const {
   ObjectValues,
 } = primordials;
 const { validateString } = require('internal/validators');
+const { getOptionValue } = require('internal/options');
 
 const {
   ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE,
@@ -32,6 +33,12 @@ const formatTypeMap = {
   'wasm': kImplicitTypeAttribute, // It's unclear whether the HTML spec will require an type attribute or not for Wasm; see https://github.com/WebAssembly/esm-integration/issues/42
 };
 
+const experimentalFormatTypeMap = {
+  '__proto__': null,
+  'bytes': 'bytes',
+  'text': 'text',
+};
+
 /**
  * The HTML spec disallows the default type to be explicitly specified
  * (for now); so `import './file.js'` is okay but
@@ -42,6 +49,17 @@ const supportedTypeAttributes = ArrayPrototypeFilter(
   ObjectValues(formatTypeMap),
   (type) => type !== kImplicitTypeAttribute);
 
+function getExperimentalFormatType(format) {
+  const type = experimentalFormatTypeMap[format];
+  if (type === undefined) {
+    return undefined;
+  }
+  if (!getOptionValue('--experimental-raw-imports')) {
+    return undefined;
+  }
+  return type;
+}
+
 
 /**
  * Test a module's import attributes.
@@ -60,7 +78,7 @@ function validateAttributes(url, format,
       throw new ERR_IMPORT_ATTRIBUTE_UNSUPPORTED(keys[i], importAttributes[keys[i]], url);
     }
   }
-  const validType = formatTypeMap[format];
+  const validType = formatTypeMap[format] ?? getExperimentalFormatType(format);
 
   switch (validType) {
     case undefined:
@@ -101,7 +119,9 @@ function handleInvalidType(url, type) {
   validateString(type, 'type');
 
   // `type` might not have been one of the types we understand.
-  if (!ArrayPrototypeIncludes(supportedTypeAttributes, type)) {
+  if (!ArrayPrototypeIncludes(supportedTypeAttributes, type) &&
+      !(ObjectPrototypeHasOwnProperty(experimentalFormatTypeMap, type) &&
+        getOptionValue('--experimental-raw-imports'))) {
     throw new ERR_IMPORT_ATTRIBUTE_UNSUPPORTED('type', type, url);
   }
 

lib/internal/modules/esm/get_format.js

@@ -78,6 +78,25 @@ const protocolHandlers = {
   'node:'() { return 'builtin'; },
 };
 
+function getFormatFromImportAttributes(importAttributes) {
+  if (
+    !getOptionValue('--experimental-raw-imports') ||
+    !importAttributes ||
+    !ObjectPrototypeHasOwnProperty(importAttributes, 'type') ||
+    typeof importAttributes.type !== 'string'
+  ) {
+    return undefined;
+  }
+
+  switch (importAttributes.type) {
+    case 'text':
+    case 'bytes':
+      return importAttributes.type;
+    default:
+      return undefined;
+  }
+}
+
 /**
  * Determine whether the given ambiguous source contains CommonJS or ES module syntax.
  * @param {string | Buffer | undefined} [source]
@@ -238,10 +257,15 @@ function getFileProtocolModuleFormat(url, context = { __proto__: null }, ignoreE
 
 /**
  * @param {URL} url
- * @param {{parentURL: string}} context
+ * @param {{parentURL: string, importAttributes?: Record<string, string>}} context
  * @returns {Promise<string> | string | undefined} only works when enabled
  */
 function defaultGetFormatWithoutErrors(url, context) {
+  const format = getFormatFromImportAttributes(context?.importAttributes);
+  if (format !== undefined) {
+    return format;
+  }
+
   const protocol = url.protocol;
   if (!ObjectPrototypeHasOwnProperty(protocolHandlers, protocol)) {
     return null;
@@ -251,10 +275,15 @@ function defaultGetFormatWithoutErrors(url, context) {
 
 /**
  * @param {URL} url
- * @param {{parentURL: string}} context
+ * @param {{parentURL: string, importAttributes?: Record<string, string>}} context
  * @returns {Promise<string> | string | undefined} only works when enabled
  */
 function defaultGetFormat(url, context) {
+  const format = getFormatFromImportAttributes(context?.importAttributes);
+  if (format !== undefined) {
+    return format;
+  }
+
   const protocol = url.protocol;
   if (!ObjectPrototypeHasOwnProperty(protocolHandlers, protocol)) {
     return null;

lib/internal/modules/esm/load.js

@@ -91,7 +91,10 @@ function defaultLoad(url, context = kEmptyObject) {
 
     if (format == null) {
       // Now that we have the source for the module, run `defaultGetFormat` to detect its format.
-      format = defaultGetFormat(urlInstance, context);
+      format = defaultGetFormat(urlInstance, {
+        __proto__: context,
+        importAttributes,
+      });
 
       if (format === 'commonjs') {
         // For backward compatibility reasons, we need to discard the source in
@@ -155,7 +158,10 @@ function defaultLoadSync(url, context = kEmptyObject) {
     }
 
     // Now that we have the source for the module, run `defaultGetFormat` to detect its format.
-    format ??= defaultGetFormat(urlInstance, context);
+    format ??= defaultGetFormat(urlInstance, {
+      __proto__: context,
+      importAttributes,
+    });
 
     // For backward compatibility reasons, we need to let go through Module._load
     // again.

lib/internal/modules/esm/translators.js

@@ -642,3 +642,23 @@ translators.set('module-typescript', function(url, translateContext, parentURL)
   translateContext.source = stripTypeScriptModuleTypes(stringify(source), url);
   return FunctionPrototypeCall(translators.get('module'), this, url, translateContext, parentURL);
 });
+
+// Strategy for loading source as text.
+translators.set('text', function textStrategy(url, translateContext) {
+  let { source } = translateContext;
+  assertBufferSource(source, true, 'load');
+  source = stringify(source);
+  return new ModuleWrap(url, undefined, ['default'], function() {
+    this.setExport('default', source);
+  });
+});
+
+// Strategy for loading source as raw bytes.
+translators.set('bytes', function bytesStrategy(url, translateContext) {
+  let { source } = translateContext;
+  assertBufferSource(source, false, 'load');
+  source = new Uint8Array(Buffer.from(source));
+  return new ModuleWrap(url, undefined, ['default'], function() {
+    this.setExport('default', source);
+  });
+});

src/node_options.cc

@@ -632,6 +632,11 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
   AddAlias("--loader", "--experimental-loader");
   AddOption("--experimental-modules", "", NoOp{}, kAllowedInEnvvar);
   AddOption("--experimental-wasm-modules", "", NoOp{}, kAllowedInEnvvar);
+  AddOption("--experimental-raw-imports",
+            "experimental support for raw source imports with import "
+            "attributes",
+            &EnvironmentOptions::experimental_raw_imports,
+            kAllowedInEnvvar);
   AddOption("--experimental-import-meta-resolve",
             "experimental ES Module import.meta.resolve() parentURL support",
             &EnvironmentOptions::experimental_import_meta_resolve,

src/node_options.h

@@ -134,6 +134,7 @@ class EnvironmentOptions : public Options {
   std::string localstorage_file;
   bool experimental_global_navigator = true;
   bool experimental_global_web_crypto = true;
+  bool experimental_raw_imports = false;
   bool experimental_import_meta_resolve = false;
   std::string input_type;  // Value of --input-type
   bool entry_is_url = false;

test/es-module/test-esm-import-attributes-errors.js

@@ -26,6 +26,16 @@ async function test() {
     { code: 'ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE' }
   );
 
+  await assert.rejects(
+    import(jsModuleDataUrl, { with: { type: 'text' } }),
+    { code: 'ERR_IMPORT_ATTRIBUTE_UNSUPPORTED' }
+  );
+
+  await assert.rejects(
+    import(jsModuleDataUrl, { with: { type: 'bytes' } }),
+    { code: 'ERR_IMPORT_ATTRIBUTE_UNSUPPORTED' }
+  );
+
   await assert.rejects(
     import(jsModuleDataUrl, { with: { type: 'json', other: 'unsupported' } }),
     { code: 'ERR_IMPORT_ATTRIBUTE_UNSUPPORTED' }

test/es-module/test-esm-import-attributes-errors.mjs

@@ -21,6 +21,16 @@ await assert.rejects(
   { code: 'ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE' }
 );
 
+await assert.rejects(
+  import(jsModuleDataUrl, { with: { type: 'text' } }),
+  { code: 'ERR_IMPORT_ATTRIBUTE_UNSUPPORTED' }
+);
+
+await assert.rejects(
+  import(jsModuleDataUrl, { with: { type: 'bytes' } }),
+  { code: 'ERR_IMPORT_ATTRIBUTE_UNSUPPORTED' }
+);
+
 await assert.rejects(
   import(jsModuleDataUrl, { with: { type: 'json', other: 'unsupported' } }),
   { code: 'ERR_IMPORT_ATTRIBUTE_UNSUPPORTED' }

test/es-module/test-esm-import-experimental-raw-imports-attributes-validation.js

@@ -0,0 +1,31 @@
+// Flags: --expose-internals --experimental-raw-imports
+'use strict';
+require('../common');
+
+const assert = require('assert');
+
+const { validateAttributes } = require('internal/modules/esm/assert');
+
+const url = 'test://';
+
+// ... {type = 'text'}
+assert.ok(validateAttributes(url, 'text', { type: 'text' }));
+
+assert.throws(() => validateAttributes(url, 'text', {}), {
+  code: 'ERR_IMPORT_ATTRIBUTE_MISSING',
+});
+
+assert.throws(() => validateAttributes(url, 'module', { type: 'text' }), {
+  code: 'ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE',
+});
+
+// ... {type = 'bytes'}
+assert.ok(validateAttributes(url, 'bytes', { type: 'bytes' }));
+
+assert.throws(() => validateAttributes(url, 'bytes', {}), {
+  code: 'ERR_IMPORT_ATTRIBUTE_MISSING',
+});
+
+assert.throws(() => validateAttributes(url, 'module', { type: 'bytes' }), {
+  code: 'ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE',
+});

test/es-module/test-esm-import-experimental-raw-imports-disabled.mjs

@@ -0,0 +1,22 @@
+import '../common/index.mjs';
+import assert from 'assert';
+
+await assert.rejects(
+  import('../fixtures/file-to-read-without-bom.txt', { with: { type: 'text' } }),
+  { code: 'ERR_UNKNOWN_FILE_EXTENSION' },
+);
+
+await assert.rejects(
+  import('data:text/plain,hello%20world', { with: { type: 'text' } }),
+  { code: 'ERR_UNKNOWN_MODULE_FORMAT' },
+);
+
+await assert.rejects(
+  import('../fixtures/file-to-read-without-bom.txt', { with: { type: 'bytes' } }),
+  { code: 'ERR_UNKNOWN_FILE_EXTENSION' },
+);
+
+await assert.rejects(
+  import('data:text/plain,hello%20world', { with: { type: 'bytes' } }),
+  { code: 'ERR_UNKNOWN_MODULE_FORMAT' },
+);

test/es-module/test-esm-import-experimental-raw-imports.mjs

@@ -0,0 +1,77 @@
+// Flags: --experimental-raw-imports
+import '../common/index.mjs';
+import assert from 'assert';
+import { Buffer } from 'buffer';
+
+import staticText from '../fixtures/file-to-read-without-bom.txt' with { type: 'text' };
+import staticTextWithBOM from '../fixtures/file-to-read-with-bom.txt' with { type: 'text' };
+import staticBytes from '../fixtures/file-to-read-without-bom.txt' with { type: 'bytes' };
+
+const expectedText = 'abc\ndef\nghi\n';
+const expectedBytes = Buffer.from(expectedText);
+
+assert.strictEqual(staticText, expectedText);
+assert.strictEqual(staticTextWithBOM, expectedText);
+
+assert.ok(staticBytes instanceof Uint8Array);
+assert.deepStrictEqual(Buffer.from(staticBytes), expectedBytes);
+
+const dynamicText = await import('../fixtures/file-to-read-without-bom.txt', {
+  with: { type: 'text' },
+});
+assert.strictEqual(dynamicText.default, expectedText);
+
+const dynamicBytes = await import('../fixtures/file-to-read-without-bom.txt', {
+  with: { type: 'bytes' },
+});
+assert.deepStrictEqual(Buffer.from(dynamicBytes.default), expectedBytes);
+
+const dataText = await import('data:text/plain,hello%20world', {
+  with: { type: 'text' },
+});
+assert.strictEqual(dataText.default, 'hello world');
+
+const dataBytes = await import('data:text/plain,hello%20world', {
+  with: { type: 'bytes' },
+});
+assert.strictEqual(Buffer.from(dataBytes.default).toString(), 'hello world');
+
+const dataJsAsText = await import('data:text/javascript,export{}', {
+  with: { type: 'text' },
+});
+assert.strictEqual(dataJsAsText.default, 'export{}');
+
+const dataJsAsBytes = await import('data:text/javascript,export{}', {
+  with: { type: 'bytes' },
+});
+assert.strictEqual(Buffer.from(dataJsAsBytes.default).toString(), 'export{}');
+
+const dataInvalidUtf8 = await import('data:text/plain,%66%6f%80%6f', {
+  with: { type: 'text' },
+});
+assert.strictEqual(dataInvalidUtf8.default, 'fo\ufffdo');
+
+const jsAsText = await import('../fixtures/syntax/bad_syntax.js', {
+  with: { type: 'text' },
+});
+assert.match(jsAsText.default, /^var foo bar;/);
+
+const jsAsBytes = await import('../fixtures/syntax/bad_syntax.js', {
+  with: { type: 'bytes' },
+});
+assert.match(Buffer.from(jsAsBytes.default).toString(), /^var foo bar;/);
+
+const jsonAsText = await import('../fixtures/invalid.json', {
+  with: { type: 'text' },
+});
+assert.match(jsonAsText.default, /"im broken"/);
+
+await assert.rejects(
+  import('data:text/plain,hello%20world'),
+  { code: 'ERR_UNKNOWN_MODULE_FORMAT' },
+);
+
+await assert.rejects(
+  import('../fixtures/file-to-read-without-bom.txt'),
+  { code: 'ERR_UNKNOWN_FILE_EXTENSION' },
+);