feat: add getContents() for reading external markdown files into table descriptions pr fix

core/compilers.ts

@@ -56,6 +56,14 @@ export function compile(code: string, path: string): string {
       .replace(/\${/g, "\\${");
     return `exports.query = \`${escapedCode}\`;`;
   }
+  if (Path.fileExtension(path) === "md") {
+    const contents = code
+      .replace(/\r\n/g, "\n")
+      .replace(/\\/g, "\\\\")
+      .replace(/`/g, "\\`")
+      .replace(/\${/g, "\\${");
+    return `exports.contents = \`${contents}\`;`;
+  }
   return code;
 }
 

core/compilers_test.ts

@@ -92,5 +92,47 @@ suite("core/compilers", () => {
             const result = compile(code, path);
             expect(result).to.equal(code);
         });
+
+        test("compiles md to js contents export",() => {
+            const code = '# this table defines';
+            const path = "definitions/foo.md";
+            const result = compile(code, path);
+            expect(result).to.equal("exports.contents = `# this table defines`;");
+
+        })
+
+        test("escapes backslashes in md", () => {
+            const code = "select ''";
+            const path = "definitions/foo.md";
+            const result = compile(code, path);
+            expect(result).to.equal("exports.contents = `select ''`;");
+        });
+        test("escapes newlines in md", () => {
+            const code = `
+- item1
+- item2`;
+            const path = "definitions/foo.md";
+            const result = compile(code, path);
+            expect(result).to.equal("exports.contents = `\n- item1\n- item2`;");
+        });
+        test ("escapes template literals in md", () => {
+            const code = "select ${foo}";
+            const path = "definitions/foo.md";
+            const result = compile(code, path);
+            expect(result).to.equal("exports.contents = `select \\${foo}`;");
+        });
+        test("escapes backslashes and backticks in md", () => {
+            const code = "c = '\\'";
+            const path = "definitions/foo.md";
+            const result = compile(code, path);
+            expect(result).to.equal("exports.contents = `c = '\\\\'`;");
+        });
+        test("escapes backticks in md", () => {
+            const code = "select `a` from `b`";
+            const path = "definitions/foo.md";
+            const result = compile(code, path);
+            expect(result).to.equal("exports.contents = `select \\`a\\` from \\`b\\``;");
+        });
+
     });
 });

core/main.ts

@@ -232,7 +232,8 @@ function dataformCompile(compileRequest: dataform.ICompileExecutionRequest, sess
   globalAny.notebook = session.notebook.bind(session);
   globalAny.test = session.test.bind(session);
   globalAny.jitData = session.jitData.bind(session);
-
+  globalAny.getContents = session.getContents.bind(session);
+
   loadActionConfigs(session, compileRequest.compileConfig.filePaths);
 
   // Require all "definitions" files (attaching them to the session).

core/main_test.ts

@@ -1764,7 +1764,83 @@ dataform.jitData("key", {test: () => {}});
         ).to.deep.equal(["Unsupported value: () => {}"]);
       });
     });
+    suite("markdown in description", () => {
+      test("markdown contents added to description in sqlx", () => {
+          const projectDir = tmpDirFixture.createNewTmpDir();
+          fs.writeFileSync(
+            path.join(projectDir, "workflow_settings.yaml"),
+            VALID_WORKFLOW_SETTINGS_YAML
+          );
+          fs.mkdirSync(path.join(projectDir, "definitions"));
+          fs.writeFileSync(
+            path.join(projectDir, "definitions/descriptions.md"),
+            `# This table contains data about`
+          );
+          fs.writeFileSync(
+            path.join(projectDir, "definitions/table.sqlx"),
+            `config {
+            type: "table",
+            description: getContents('./descriptions.md'),
+          }
+            SELECT 1 AS test`
+          );
+
+          const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
 
+          expect(
+            result.compile.compiledGraph.tables[0].actionDescriptor.description
+          ).to.equal(`# This table contains data about`);
+
+      });
+
+       test("throws error for invalid missing markedown", () => {
+          const projectDir = tmpDirFixture.createNewTmpDir();
+          fs.writeFileSync(
+            path.join(projectDir, "workflow_settings.yaml"),
+            VALID_WORKFLOW_SETTINGS_YAML
+          );
+          fs.mkdirSync(path.join(projectDir, "definitions"));
+          fs.writeFileSync(
+            path.join(projectDir, "definitions/table.sqlx"),
+            `config {
+            type: "table",
+            description: getContents('./nonexistent.md'),
+          }
+            SELECT 1 AS test`
+          );
+
+          const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
+
+          expect(
+            result.compile.compiledGraph.graphErrors.compilationErrors[0].message
+          ).to.include("nonexistent.md");
+
+      });
+
+      test("throws error for file outisde of rootDir", () => {
+          const projectDir = tmpDirFixture.createNewTmpDir();
+          fs.writeFileSync(
+            path.join(projectDir, "workflow_settings.yaml"),
+            VALID_WORKFLOW_SETTINGS_YAML
+          );
+          fs.mkdirSync(path.join(projectDir, "definitions"));
+          fs.writeFileSync(
+            path.join(projectDir, "definitions/table.sqlx"),
+            `config {
+            type: "table",
+            description: getContents('../../description.md'),
+          }
+            SELECT 1 AS test`
+          );
+
+          const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
+
+          expect(
+            result.compile.compiledGraph.graphErrors.compilationErrors[0].message
+          ).to.include("outside the project directory");
+
+      });
+    });
     suite("invalid options", () => {
       [
         {

core/session.ts

@@ -16,12 +16,14 @@ import { Test } from "df/core/actions/test";
 import { View } from "df/core/actions/view";
 import { CompilationSql } from "df/core/compilation_sql";
 import { Contextable, IActionContext, ITableContext, Resolvable } from "df/core/contextables";
+import * as Path from "df/core/path";
 import { targetAsReadableString, targetStringifier } from "df/core/targets";
 import * as utils from "df/core/utils";
 import { ResolvableMap, toResolvable } from "df/core/utils";
 import { version as dataformCoreVersion } from "df/core/version";
 import { dataform, google } from "df/protos/ts";
 
+
 const DEFAULT_CONFIG = {
   defaultSchema: "dataform",
   assertionSchema: "dataform_assertions"
@@ -91,6 +93,29 @@ export class Session {
     return new CompilationSql(this.projectConfig, dataformCoreVersion);
   }
 
+  public getContents(filePath: string): string {
+    const callerFile = utils.getCallerFile(this.rootDir);
+    const callerDir = Path.dirName(callerFile);
+    const resolvedPath = Path.join(callerDir,filePath);
+    const absolutePath = Path.separator + Path.normalize(Path.join(this.rootDir, resolvedPath));
+
+    if (!absolutePath.startsWith(this.rootDir)){
+      throw new Error(`Cannot read "${filePath}": path resolves outside the project directory.`);
+    }
+
+    let module: any;
+    try {
+      module = utils.nativeRequire(absolutePath);
+    } catch {
+      throw new Error(`Cannot read "${filePath}": file not found.`);
+    }
+
+    if (!module || typeof module.contents !== "string") {
+      throw new Error (`Cannot read "${filePath}": only .md files are supported.`)
+    }
+    return module.contents;
+  }
+
   public sqlxAction(actionOptions: {
     // sqlxConfig has type any here because any object can be passed in from the compiler - the
     // structure of it is verified at later steps.

docs/reference/session.md

@@ -19,6 +19,7 @@ folder of a Dataform project.
 
 * [assert](_core_session_.session.md#assert)
 * [declare](_core_session_.session.md#declare)
+* [getContents](_core_session_.session.md#getcontents)
 * [notebook](_core_session_.session.md#notebook)
 * [operate](_core_session_.session.md#operate)
 * [publish](_core_session_.session.md#publish)
@@ -82,6 +83,34 @@ Name | Type |
 
 ___
 
+###  getContents
+
+▸ **getContents**(`filePath`: string): *string*
+
+Reads the contents of an external markdown file and returns it as a string. The path is resolved relative to the file that calls `getContents`.
+
+Available only in the `/definitions` directory.
+
+**Example:**
+
+```sqlx
+config {
+  type: "table",
+  description: getContents('./my_table_description.md')
+}
+SELECT ...
+```
+
+**Parameters:**
+
+Name | Type | Description |
+------ | ------ | ------ |
+`filePath` | string | Path to the file, relative to the calling file. |
+
+**Returns:** *string*
+
+___
+
 ###  notebook
 
 ▸ **notebook**(`config`: NotebookConfig): *[Notebook](_core_actions_notebook_.notebook.md)*

testing/run_core.ts

@@ -48,7 +48,7 @@ export class WorkflowSettingsTemplates {
   });
 }
 
-const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb"];
+const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb","md"];
 
 export function coreExecutionRequestFromPath(
   projectDir: string,