Added remote files + small fixes

Author: kevinwang5658

package.json

@@ -1,6 +1,6 @@
 {
   "name": "default",
-  "version": "0.15.1",
+  "version": "0.16.1",
   "description": "",
   "main": "dist/index.js",
   "scripts": {
@@ -22,51 +22,52 @@
   "dependencies": {
     "ajv": "^8.12.0",
     "ajv-formats": "^2.1.1",
-    "semver": "^7.6.0",
-    "codify-plugin-lib": "1.0.176",
-    "codify-schemas": "1.0.63",
     "chalk": "^5.3.0",
+    "codify-plugin-lib": "../codify-plugin-lib",
+    "codify-schemas": "1.0.63",
     "debug": "^4.3.4",
-    "plist": "^3.1.0",
     "lodash.isequal": "^4.5.0",
+    "nanoid": "^5.0.9",
+    "plist": "^3.1.0",
+    "semver": "^7.6.0",
     "strip-ansi": "^7.1.0",
-    "nanoid": "^5.0.9"
+    "trash": "^10.0.0"
   },
   "devDependencies": {
-    "rollup": "^4.12.0",
-    "@rollup/plugin-json": "^6.1.0",
-    "@rollup/plugin-typescript": "^11.1.6",
+    "@apidevtools/json-schema-ref-parser": "^11.7.2",
+    "@fastify/merge-json-schemas": "^0.2.0",
+    "@oclif/prettier-config": "^0.2.1",
+    "@oclif/test": "^3",
     "@rollup/plugin-commonjs": "^25.0.7",
+    "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
-    "@oclif/prettier-config": "^0.2.1",
-    "@oclif/test": "^3",
-    "@types/node": "^18",
-    "@types/semver": "^7.5.4",
-    "@types/mock-fs": "^4.13.4",
+    "@rollup/plugin-typescript": "^11.1.6",
     "@types/chalk": "^2.2.0",
     "@types/commander": "^2.12.2",
     "@types/debug": "4.1.12",
-    "@types/plist": "^3.0.5",
     "@types/lodash.isequal": "^4.5.8",
-    "codify-plugin-test": "0.0.51",
+    "@types/mock-fs": "^4.13.4",
+    "@types/node": "^18",
+    "@types/plist": "^3.0.5",
+    "@types/semver": "^7.5.4",
+    "codify-plugin-test": "0.0.52",
     "commander": "^12.1.0",
     "eslint": "^8.51.0",
     "eslint-config-oclif": "^5",
     "eslint-config-oclif-typescript": "^3",
+    "eslint-config-prettier": "^9.0.0",
     "glob": "^11.0.0",
-    "vitest": "^1.4.0",
+    "merge-json-schemas": "^1.0.0",
     "mock-fs": "^5.2.0",
+    "rollup": "^4.12.0",
     "shx": "^0.3.3",
     "ts-node": "^10.9.1",
     "tslib": "^2.6.2",
-    "typescript": "^5",
-    "eslint-config-prettier": "^9.0.0",
     "tsx": "^4.7.2",
-    "@fastify/merge-json-schemas": "^0.2.0",
-    "@apidevtools/json-schema-ref-parser": "^11.7.2",
-    "merge-json-schemas": "^1.0.0"
+    "typescript": "^5",
+    "vitest": "^1.4.0"
   },
   "engines": {
     "node": ">=18.0.0"

src/index.ts

@@ -8,12 +8,16 @@ import { AsdfLocalResource } from './resources/asdf/asdf-local.js';
 import { AsdfPluginResource } from './resources/asdf/asdf-plugin.js';
 import { AwsCliResource } from './resources/aws-cli/cli/aws-cli.js';
 import { AwsProfileResource } from './resources/aws-cli/profile/aws-profile.js';
+import { DockerResource } from './resources/docker/docker.js';
+import { RemoteFileResource } from './resources/file/remote-file.js';
 import { GitCloneResource } from './resources/git/clone/git-repository.js';
 import { GitResource } from './resources/git/git/git-resource.js';
 import { GitLfsResource } from './resources/git/lfs/git-lfs.js';
 import { WaitGithubSshKey } from './resources/git/wait-github-ssh-key/wait-github-ssh-key.js';
 import { HomebrewResource } from './resources/homebrew/homebrew.js';
 import { JenvResource } from './resources/java/jenv/jenv.js';
+import { MacportsResource } from './resources/macports/macports.js';
+import { Npm } from './resources/node/npm/npm.js';
 import { NvmResource } from './resources/node/nvm/nvm.js';
 import { Pnpm } from './resources/node/pnpm/pnpm.js';
 import { PgcliResource } from './resources/pgcli/pgcli.js';
@@ -24,7 +28,6 @@ import { VenvProject } from './resources/python/venv/venv-project.js';
 import { Virtualenv } from './resources/python/virtualenv/virtualenv.js';
 import { VirtualenvProject } from './resources/python/virtualenv/virtualenv-project.js';
 import { ActionResource } from './resources/scripting/action.js';
-import { FileResource } from './resources/scripting/file.js';
 import { AliasResource } from './resources/shell/alias/alias-resource.js';
 import { PathResource } from './resources/shell/path/path-resource.js';
 import { SshAddResource } from './resources/ssh/ssh-add.js';
@@ -33,9 +36,7 @@ import { SshKeyResource } from './resources/ssh/ssh-key.js';
 import { TerraformResource } from './resources/terraform/terraform.js';
 import { VscodeResource } from './resources/vscode/vscode.js';
 import { XcodeToolsResource } from './resources/xcode-tools/xcode-tools.js';
-import { MacportsResource } from './resources/macports/macports.js';
-import { Npm } from './resources/node/npm/npm.js';
-import { DockerResource } from './resources/docker/docker.js';
+import { FileResource } from './resources/file/file.js';
 
 runPlugin(Plugin.create(
   'default',
@@ -66,6 +67,7 @@ runPlugin(Plugin.create(
     new SshAddResource(),
     new ActionResource(),
     new FileResource(),
+    new RemoteFileResource(),
     new Virtualenv(),
     new VirtualenvProject(),
     new Pnpm(),

src/resources/scripting/file-schema.json src/resources/file/file-schema.jsonsrc/resources/scripting/file-schema.json renamed to src/resources/file/file-schema.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "$id": "https://www.codifycli.com/file.json",
+  "title": "File resource",
+  "description": "Manages a file.",
+  "type": "object",
+  "properties": {
+    "path": {
+      "type": "string",
+      "description": "The location of the file."
+    },
+    "remote": {
+      "type": "string",
+      "description": "The contents of the file."
+    },
+    "onlyCreate": {
+      "type": "boolean",
+      "description": "Forces the resource to only create the file if it doesn't exist but don't detect any content changes."
+    }
+  },
+  "required": ["path", "remote"],
+  "additionalProperties": false
+}

src/resources/scripting/file.ts src/resources/file/file.tssrc/resources/scripting/file.ts renamed to src/resources/file/file.ts

@@ -0,0 +1,162 @@
+import {
+  CodifyCliSender,
+  CreatePlan,
+  DestroyPlan,
+  ModifyPlan,
+  ParameterChange,
+  RefreshContext,
+  Resource,
+  ResourceSettings
+} from 'codify-plugin-lib';
+import { ResourceConfig } from 'codify-schemas';
+import { createHash } from 'node:crypto';
+import * as fsSync from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { finished } from 'node:stream/promises';
+import trash from 'trash';
+
+import { FileUtils } from '../../utils/file-utils.js';
+import schema from './remote-file-schema.json'
+
+export interface FileConfig extends ResourceConfig{
+  path: string;
+  remote?: string;
+  hash?: string;
+  onlyCreate: boolean;
+}
+
+export class RemoteFileResource extends Resource<FileConfig> {
+  getSettings(): ResourceSettings<FileConfig> {
+    return {
+      id: 'remote-file',
+      allowMultiple: true,
+      schema,
+      parameterSettings: {
+        path: { type: 'directory' },
+        remote: { type: 'string', canModify: true },
+        onlyCreate: { type: 'boolean', setting: true, default: false },
+        hash: { type: 'string', canModify: true },
+      },
+      transformation: {
+        to: async (input: Partial<FileConfig>) => {
+          if (this.isRemoteCodifyFile(input.remote!)) {
+            return this.getRemoteCodifyFileConfig(input)
+          }
+
+          return input;
+        },
+        from: async (input: Partial<FileConfig>) => input,
+      }
+    }
+  }
+
+  async refresh(parameters: Partial<FileConfig>, context: RefreshContext<FileConfig>): Promise<Partial<FileConfig> | null> {
+    if (!parameters.path) {
+      throw new Error('Path must be specified');
+    }
+
+    if (!(await FileUtils.exists(parameters.path))) {
+      return null;
+    }
+
+    const current = await fs.readFile(parameters.path, 'utf8');
+    const currentHash = createHash('md5').update(current).digest('hex');
+
+    return {
+      ...parameters,
+      hash: parameters.onlyCreate ? parameters.hash : currentHash,
+    }
+  }
+
+  async create(plan: CreatePlan<FileConfig>): Promise<void> {
+    return this.updateCodifyFile(plan.desiredConfig);
+  }
+  
+  async modify(pc: ParameterChange<FileConfig>, plan: ModifyPlan<FileConfig>): Promise<void> {
+    return this.updateCodifyFile(plan.desiredConfig);
+  }
+
+  destroy(plan: DestroyPlan<FileConfig>): Promise<void> {
+    return trash(plan.currentConfig.path);
+  }
+
+  private async updateCodifyFile(config: FileConfig) {
+    const { path: filePath, remote } = config;
+    const resolvedPath = path.resolve(filePath);
+
+    if (!(await FileUtils.dirExists(path.dirname(resolvedPath)))) {
+      await fs.mkdir(path.dirname(resolvedPath), { recursive: true });
+    }
+
+    if (this.isRemoteCodifyFile(remote!)) {
+      const { documentId, fileId } = this.extractCodifyFileInfo(remote!);
+      const fileStream = fsSync.createWriteStream(filePath, { flags: 'wx' });
+
+      const credentials = await CodifyCliSender.getCodifyCliCredentials();
+      const response = await fetch(`https://api.codifycli.com/v1/documents/${documentId}/file/${fileId}`, {
+        method: 'GET',
+        headers: {
+          'Authorization': `Bearer ${credentials}`,
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(`Unable to fetch file ${remote}, ${await response.text()}`);
+      }
+
+      console.log(`Updating file ${filePath} with contents from ${remote}`);
+      await finished(Readable.fromWeb(response.body as any).pipe(fileStream));
+    } else {
+      console.log(`Updating file ${filePath} with contents`);
+      await fs.writeFile(filePath, remote ?? '');
+    }
+
+    console.log(`Finished updating file ${filePath}`);
+  }
+
+  private isRemoteCodifyFile(contents: string) {
+    return contents?.startsWith('codify://');
+  }
+
+  private async getRemoteCodifyFileConfig(parameters: Partial<FileConfig>): Promise<Partial<FileConfig>> {
+    const { documentId, fileId } = this.extractCodifyFileInfo(parameters.remote!);
+
+    const credentials = await CodifyCliSender.getCodifyCliCredentials();
+    const response = await fetch((`https://api.codifycli.com/v1/documents/${documentId}/file/${fileId}/hash`), {
+      method: 'GET',
+      headers: {
+        'Authorization': `Bearer ${credentials}`,
+      },
+    });
+
+    if (!response.ok) {
+      return {
+        ...parameters,
+        hash: undefined,
+      }
+    }
+
+    const data = await response.json();
+
+    return {
+      ...parameters,
+      hash: data.hash,
+    };
+  }
+
+  private extractCodifyFileInfo(url: string) {
+    const regex = /codify:\/\/(.*):(.*)/
+
+    const [, group1, group2] = regex.exec(url) ?? [];
+    if (!group1 || !group2) {
+      throw new Error(`Invalid codify url ${url} for file`);
+    }
+
+    return {
+      documentId: group1,
+      fileId: group2,
+    }
+  }
+}

src/resources/file/remote-file-schema.json

@@ -1,4 +1,5 @@
 import { CreatePlan, DestroyPlan, getPty, Resource, ResourceSettings } from 'codify-plugin-lib';
+import { RefreshContext } from 'codify-plugin-lib/src/resource/resource.js';
 import { StringIndexedObject } from 'codify-schemas';
 
 import { SpawnStatus, codifySpawn } from '../../utils/codify-spawn.js';
@@ -26,13 +27,12 @@ export class ActionResource extends Resource<ActionConfig> {
     }
   }
 
-  async refresh(parameters: Partial<ActionConfig>): Promise<Partial<ActionConfig> | Partial<ActionConfig>[] | null> {
+  async refresh(parameters: Partial<ActionConfig>, context: RefreshContext<ActionConfig>): Promise<Partial<ActionConfig> | Partial<ActionConfig>[] | null> {
     const $ = getPty();
 
     // Always run if condition doesn't exist
-    // TODO: Remove hack. Right now we're returning null to simulate CREATE and a value for NO-OP
     if (!parameters.condition) {
-      return null;
+      return context.commandType === 'validationPlan' ? parameters : null;
     }
 
     const { condition, action, cwd } = parameters;

src/resources/file/remote-file.ts

@@ -60,15 +60,7 @@
             "type": "boolean",
             "description": "Specifies whether to use password authentication."
           }
-        },
-        "oneOf": [
-          {
-            "required": ["Host"]
-          },
-          {
-            "required": ["Match"]
-          }
-        ]
+        }
       }
     }
   },

src/resources/scripting/action.ts

@@ -0,0 +1,27 @@
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { PluginTester } from 'codify-plugin-test';
+import * as path from 'node:path';
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import { ResourceOperation } from 'codify-schemas';
+import os from 'node:os';
+
+describe('File integration tests', async () => {
+  const pluginPath = path.resolve('./src/index.ts');
+
+  it('Can download a file from codify cloud', { timeout: 300000 }, async () => {
+    await PluginTester.fullTest(pluginPath, [
+      {
+        type: 'remote-file',
+        path: './my_test_file',
+        remote: 'codify://29198d08-08fb-44aa-a014-c45690aa822b:favicon.svg',
+      }
+    ], {
+      validateApply: async () => {
+        expect(fs.existsSync('./my_test_file')).to.be.true;
+
+        console.log(fs.readFileSync('./my_test_file', 'utf8'));
+      }
+    });
+  })
+})