diff --git a/grunt/helpers.js b/grunt/helpers.js index a6690e451..d6a775157 100644 --- a/grunt/helpers.js +++ b/grunt/helpers.js @@ -3,7 +3,7 @@ const fs = require('fs-extra'); const path = require('path'); // extends grunt.file.expand with { order: cb(filePaths) } require('grunt-file-order'); -const Framework = require('./helpers/Framework'); +const { Framework } = require('adapt-project'); module.exports = function(grunt) { @@ -49,53 +49,6 @@ module.exports = function(grunt) { if (grunt.config('languages') !== '**') grunt.log.ok(`The following languages will be included in the build '${grunt.config('languages')}'`); }); - // privates - const generateIncludedRegExp = function() { - const includes = grunt.config('includes') || []; - const pluginTypes = exports.defaults.pluginTypes; - - // Return a more specific plugin regExp including src path. - const re = _.map(includes, function(plugin) { - return _.map(pluginTypes, function(type) { - // eslint-disable-next-line no-useless-escape - return exports.defaults.sourcedir + type + '\/' + plugin + '\/'; - }).join('|'); - }).join('|'); - // eslint-disable-next-line no-useless-escape - const core = exports.defaults.sourcedir + 'core\/'; - return new RegExp(core + '|' + re, 'i'); - }; - - const generateNestedIncludedRegExp = function() { - const includes = grunt.config('includes') || []; - const folderRegEx = 'less/plugins'; - - // Return a more specific plugin regExp including src path. - const re = _.map(includes, function(plugin) { - // eslint-disable-next-line no-useless-escape - return exports.defaults.sourcedir + '([^\/]*)\/([^\/]*)\/' + folderRegEx + '\/' + plugin + '\/'; - }).join('|'); - return new RegExp(re, 'i'); - }; - - const generateExcludedRegExp = function() { - const excludes = grunt.config('excludes') || []; - if (grunt.config('type') === 'production') { - const productionExcludes = grunt.config('productionExcludes') || []; - excludes.push(...productionExcludes); - } - const pluginTypes = exports.defaults.pluginTypes; - - // Return a more specific plugin regExp including src path. - const re = _.map(excludes, function(plugin) { - return _.map(pluginTypes, function(type) { - // eslint-disable-next-line no-useless-escape - return exports.defaults.sourcedir + type + '\/' + plugin + '\/'; - }).join('|'); - }).join('|'); - return new RegExp(re, 'i'); - }; - const generateScriptSafeRegExp = function() { const includes = grunt.config('scriptSafe') || []; let re = ''; @@ -116,27 +69,6 @@ module.exports = function(grunt) { } }; - // eslint-disable-next-line no-unused-vars - const includedProcess = function(content, filepath) { - if (!exports.isPathIncluded(filepath)) return ''; - else return content; - }; - - const getIncludedRegExp = function() { - const configValue = grunt.config('includedRegExp'); - return configValue || grunt.config('includedRegExp', generateIncludedRegExp()); - }; - - const getNestedIncludedRegExp = function() { - const configValue = grunt.config('nestedIncludedRegExp'); - return configValue || grunt.config('nestedIncludedRegExp', generateNestedIncludedRegExp()); - }; - - const getExcludedRegExp = function() { - const configValue = grunt.config('excludedRegExp'); - return configValue || grunt.config('excludedRegExp', generateExcludedRegExp()); - }; - const getScriptSafeRegExp = function() { const configValue = grunt.config('scriptSafeRegExp'); return configValue || grunt.config('scriptSafeRegExp', generateScriptSafeRegExp()); @@ -273,15 +205,20 @@ module.exports = function(grunt) { outputPath: data.outputdir, sourcePath: data.sourcedir, courseDir: data.coursedir, - includedFilter: exports.includedFilter, jsonext: data.jsonext, trackingIdType: data.trackingIdType, useOutputData: Boolean(grunt.option('outputdir')), + usePackageJSON: false, + schemaVersion: '0.1.0', + specifiedMenus: data.menu === '**' ? null : [data.menu], + specifiedThemes: data.theme === '**' ? null : [data.theme], log: grunt.log.ok, warn: grunt.log.error }); framework.load(); + exports.includedFilter = framework.includedFilter; + data.availableLanguageNames = []; try { data.availableLanguageNames = framework.getData().languageNames; @@ -314,44 +251,6 @@ module.exports = function(grunt) { return false; }, - isPathIncluded: function(pluginPath) { - pluginPath = pluginPath.replace(convertSlashes, '/'); - - const includes = grunt.config('includes'); - const excludes = grunt.config('excludes') || (grunt.config('type') === 'production' && grunt.config('productionExcludes')); - - // carry on as normal if no includes/excludes - if (!includes && !excludes) return true; - - // Very basic check to see if the file path string contains any - // of the included list of plugin string names. - const isIncluded = includes && pluginPath.search(getIncludedRegExp()) !== -1; - const isExcluded = excludes && pluginPath.search(getExcludedRegExp()) !== -1; - - // Exclude any plugins that don't match any part of the full file path string. - if (isExcluded || isIncluded === false) { - return false; - } - - // Check the LESS plugins folder exists. - // The LESS 'plugins' folder doesn't exist, so add the file, - // as the plugin has already been found in the previous check. - const nestedPluginsPath = !!pluginPath.match(/(?:.)+(?:\/less\/plugins)/g); - if (!nestedPluginsPath) { - return true; - } - - // The LESS 'plugins' folder exists, so check that any plugins in this folder are allowed. - const hasPluginSubDirectory = !!pluginPath.match(getNestedIncludedRegExp()); - if (hasPluginSubDirectory) { - return true; - } - - // File might be in the included plugin/less/plugins directory, - // but the naming convention or directory structure is not correct. - return false; - }, - isPluginScriptSafe: function(pluginPath) { pluginPath = pluginPath.replace(convertSlashes, '/'); @@ -363,10 +262,6 @@ module.exports = function(grunt) { }, - includedFilter: function(filepath) { - return exports.isPathIncluded(filepath); - }, - scriptSafeFilter: function(filepath) { return exports.isPluginScriptSafe(filepath); }, @@ -382,10 +277,13 @@ module.exports = function(grunt) { outputPath: buildConfig.outputdir, sourcePath: buildConfig.sourcedir, courseDir: buildConfig.coursedir, - includedFilter: exports.includedFilter, jsonext: buildConfig.jsonext, trackingIdType: buildConfig.trackingIdType, useOutputData, + usePackageJSON: false, + schemaVersion: '0.1.0', + specifiedMenus: buildConfig.menu === '**' ? null : [buildConfig.menu], + specifiedThemes: buildConfig.theme === '**' ? null : [buildConfig.theme], log: grunt.log.ok, warn: grunt.log.error }); diff --git a/grunt/helpers/Data.js b/grunt/helpers/Data.js deleted file mode 100644 index 51e136ee6..000000000 --- a/grunt/helpers/Data.js +++ /dev/null @@ -1,187 +0,0 @@ -const path = require('path'); -const fs = require('fs-extra'); -const globs = require('globs'); -const JSONFile = require('./lib/JSONFile'); -const Language = require('./data/Language'); - -/** - * @typedef {import('./Framework')} Framework - * @typedef {import('./lib/JSONFileItem')} JSONFileItem - */ - -/** - * This class represents the course folder. It contains references to the config.json, - * all languages, each language file and subsequently each language file item. - * It is filename agnostic, except for config.[jsonext], such that there are no - * hard references to the other file names, allowing any filename to be used with the - * appropriate [jsonext] file extension (usually txt or json). - * It assumes all language files are located at course/[langName]/*.[jsonext] and - * the config file is located at course/config.[jsonext]. - * It has _id and _parentId structure checking and _trackingId management included. - */ -class Data { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {string} options.sourcePath - * @param {string} options.courseDir - * @param {string} options.jsonext - * @param {string} options.trackingIdType - * @param {function} options.log - */ - constructor({ - framework = null, - sourcePath = null, - courseDir = null, - jsonext = 'json', - trackingIdType = 'block', - log = console.log - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {string} */ - this.sourcePath = sourcePath; - /** @type {string} */ - this.courseDir = courseDir; - /** @type {string} */ - this.jsonext = jsonext; - /** @type {string} */ - this.trackingIdType = trackingIdType; - /** @type {function} */ - this.log = log; - /** @type {JSONFile} */ - this.configFile = null; - /** @type {[Language]} */ - this.languages = null; - /** @type {string} */ - this.coursePath = path.join(this.sourcePath, this.courseDir).replace(/\\/g, '/'); - } - - /** @returns {Data} */ - load() { - this.languages = globs.sync(path.join(this.coursePath, '*/')).map(languagePath => { - const language = new Language({ - framework: this.framework, - languagePath, - courseDir: this.courseDir, - jsonext: this.jsonext, - trackingIdType: this.trackingIdType, - log: this.log - }); - language.load(); - return language; - }).filter(lang => lang.isValid); - this.configFile = new JSONFile({ - framework: this.framework, - path: path.join(this.coursePath, `config.${this.jsonext}`), - jsonext: this.jsonext - }); - this.configFile.load(); - return this; - } - - /** @type {boolean} */ - get hasChanged() { - return this.languages.some(language => language.hasChanged); - } - - /** @type {[string]} */ - get languageNames() { - return this.languages.map(language => language.name); - } - - /** - * Fetch a Language instance by name. - * @param {string} name - * @returns {Language} - */ - getLanguage(name) { - const language = this.languages.find(language => language.name === name); - if (!language) { - const err = new Error(`Cannot find language '${name}'.`); - err.number = 10004; - throw err; - } - return language; - } - - /** - * Returns a JSONFileItem representing the course/config.json file object. - * @returns {JSONFileItem} - * */ - getConfigFileItem() { - return this.configFile.firstFileItem; - } - - /** - * @param {string} fromName - * @param {string} toName - * @param {boolean} replace - * @returns {Language} - */ - copyLanguage(fromName, toName, replace = false) { - const fromLang = this.getLanguage(fromName); - const newPath = (`${fromLang.rootPath}/${toName}/`).replace(/\\/g, '/'); - - if (this.languageNames.includes(toName) && !replace) { - const err = new Error(`Folder already exists. ${newPath}`); - err.number = 10003; - throw err; - } - - let toLang; - if (this.languageNames.includes(toName)) { - toLang = this.getLanguage(toName); - } else { - toLang = new Language({ - framework: this.framework, - languagePath: newPath, - courseDir: this.courseDir, - jsonext: this.jsonext, - trackingIdType: this.trackingIdType - }); - this.languages.push(toLang); - } - - fs.mkdirpSync(newPath); - - fromLang.files.forEach(file => { - const pathParsed = path.parse(file.path.replace(/\\/g, '/')); - const newLocation = `${newPath}${pathParsed.name}${pathParsed.ext}`; - fs.removeSync(newLocation); - fs.copyFileSync(file.path, newLocation); - }); - - toLang.load(); - return toLang; - } - - /** @returns {Data} */ - checkIds() { - this.languages.forEach(lang => lang.checkIds()); - return this; - } - - /** @returns {Data} */ - addTrackingIds() { - this.languages.forEach(lang => lang.addTrackingIds()); - return this; - } - - /** @returns {Data} */ - removeTrackingIds() { - this.languages.forEach(lang => lang.removeTrackingIds()); - return this; - } - - /** @returns {Data} */ - save() { - this.configFile.save(); - this.languages.forEach(language => language.save()); - return this; - } - -} - -module.exports = Data; diff --git a/grunt/helpers/Framework.js b/grunt/helpers/Framework.js deleted file mode 100644 index b53ead49b..000000000 --- a/grunt/helpers/Framework.js +++ /dev/null @@ -1,208 +0,0 @@ -const path = require('path'); -const JSONFile = require('./lib/JSONFile'); -const Data = require('./Data'); -const Translate = require('./Translate'); -const Plugins = require('./Plugins'); -const Schemas = require('./Schemas'); - -/** - * @typedef {import('./lib/JSONFileItem')} JSONFileItem - */ - -/** - * The class represents an Adapt Framework root directory. It provides APIs for - * plugins, schemas, data and translations. - */ -class Framework { - - /** - * @param {Object} options - * @param {string} options.rootPath - * @param {string} options.outputPath - * @param {string} options.sourcePath - * @param {function} options.includedFilter - * @param {string} options.jsonext - * @param {string} options.trackingIdType, - * @param {boolean} options.useOutputData - * @param {function} options.log - * @param {function} options.warn - */ - constructor({ - rootPath = process.cwd(), - outputPath = path.join(rootPath, '/build/'), - sourcePath = path.join(rootPath, '/src/'), - courseDir = 'course', - includedFilter = function() { return true; }, - jsonext = 'json', - trackingIdType = 'block', - useOutputData = false, - log = console.log, - warn = console.warn - } = {}) { - /** @type {string} */ - this.rootPath = rootPath.replace(/\\/g, '/'); - /** @type {string} */ - this.outputPath = path.resolve(this.rootPath, outputPath).replace(/\\/g, '/').replace(/\/?$/, '/'); - /** @type {string} */ - this.sourcePath = path.resolve(this.rootPath, sourcePath).replace(/\\/g, '/').replace(/\/?$/, '/'); - /** @type {string} */ - this.courseDir = courseDir; - /** @type {function} */ - this.includedFilter = includedFilter; - /** @type {string} */ - this.jsonext = jsonext; - /** @type {string} */ - this.trackingIdType = trackingIdType; - /** @type {boolean} */ - this.useOutputData = useOutputData; - /** @type {function} */ - this.log = log; - /** @type {function} */ - this.warn = warn; - /** @type {JSONFile} */ - this.packageJSONFile = null; - } - - /** @returns {Framework} */ - load() { - this.packageJSONFile = new JSONFile({ - framework: this, - path: path.join(this.rootPath, 'package.json').replace(/\\/g, '/') - }); - this.packageJSONFile.load(); - return this; - } - - /** @returns {JSONFileItem} */ - getPackageJSONFileItem() { - return this.packageJSONFile.firstFileItem; - } - - /** @returns {string} */ - get version() { - return this.getPackageJSONFileItem().item.version; - } - - /** - * Returns a Data instance for either the src/course or build/course folder - * depending on the specification of the useOutputData property on either the - * function or the Framework instance. - * @returns {Data} - */ - getData({ - useOutputData = this.useOutputData, - performLoad = true - } = {}) { - const data = new Data({ - framework: this, - sourcePath: useOutputData ? this.outputPath : this.sourcePath, - courseDir: this.courseDir, - jsonext: this.jsonext, - trackingIdType: this.trackingIdType, - log: this.log - }); - if (performLoad) data.load(); - return data; - } - - /** @returns {Plugins} */ - getPlugins({ - includedFilter = this.includedFilter - } = {}) { - const plugins = new Plugins({ - framework: this.framework, - includedFilter, - sourcePath: this.sourcePath, - log: this.log, - warn: this.warn - }); - plugins.load(); - return plugins; - } - - /** @returns {Schemas} */ - getSchemas({ - includedFilter = this.includedFilter - } = {}) { - const schemas = new Schemas({ - framework: this, - includedFilter, - sourcePath: this.sourcePath, - log: this.log - }); - schemas.load(); - return schemas; - } - - /** @returns {Translate} */ - getTranslate({ - includedFilter = this.includedFilter, - masterLang = 'en', - targetLang = null, - format = 'csv', - csvDelimiter = ',', - shouldReplaceExisting = false, - languagePath = '', - isTest = false - } = {}) { - const translate = new Translate({ - framework: this, - includedFilter, - masterLang, - targetLang, - format, - csvDelimiter, - shouldReplaceExisting, - jsonext: this.jsonext, - sourcePath: this.sourcePath, - languagePath, - outputPath: this.outputPath, - courseDir: this.courseDir, - useOutputData: this.useOutputData, - isTest, - log: this.log, - warn: this.warn - }); - translate.load(); - return translate; - } - - /** @returns {Framework} */ - applyGlobalsDefaults({ - includedFilter = this.includedFilter, - useOutputData = this.useOutputData, - schemas = this.getSchemas({ - includedFilter - }), - data = this.getData(useOutputData) - } = {}) { - const courseSchema = schemas.getCourseSchema(); - data.languages.forEach(language => { - const { file, item: course } = language.getCourseFileItem(); - course._globals = courseSchema.applyDefaults(course._globals, '_globals'); - file.changed(); - }); - data.save(); - return this; - } - - /** @returns {Framework} */ - applyScreenSizeDefaults({ - includedFilter = this.includedFilter, - useOutputData = this.useOutputData, - schemas = this.getSchemas({ - includedFilter - }), - data = this.getData(useOutputData) - } = {}) { - const configSchema = schemas.getConfigSchema(); - const { file, item: config } = data.getConfigFileItem(); - config.screenSize = configSchema.applyDefaults(config.screenSize, 'screenSize'); - file.changed(); - data.save(); - return this; - } - -} - -module.exports = Framework; diff --git a/grunt/helpers/Plugins.js b/grunt/helpers/Plugins.js deleted file mode 100644 index 6ddd3cd6c..000000000 --- a/grunt/helpers/Plugins.js +++ /dev/null @@ -1,85 +0,0 @@ -const globs = require('globs'); -const Plugin = require('./plugins/Plugin'); - -/** - * @typedef {import('./Framework')} Framework - * @typedef {import('./lib/JSONFileItem')} JSONFileItem - */ - -/** - * Represents all of the plugins in the src/ folder. - */ -class Plugins { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {function} options.includedFilter - * @param {string} options.sourcePath - * @param {function} options.log - * @param {function} options.warn - */ - constructor({ - framework = null, - includedFilter = function() { return true; }, - sourcePath = process.cwd() + '/src/', - courseDir = 'course', - log = console.log, - warn = console.warn - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {function} */ - this.includedFilter = includedFilter; - /** @type {string} */ - this.sourcePath = sourcePath; - /** @type {string} */ - this.courseDir = courseDir; - /** @type {function} */ - this.log = log; - /** @type {function} */ - this.warn = warn; - /** @type {[Plugin]} */ - this.plugins = []; - } - - /** - * Returns the locations of all plugins in the src/ folder. - * @returns {[string]} - */ - get pluginLocations() { - return [ - `${this.sourcePath}core/`, - `${this.sourcePath}!(core|${this.courseDir})/*/` - ]; - } - - /** @returns {Plugins} */ - load() { - this.plugins = globs.sync(this.pluginLocations).map(sourcePath => { - if (!this.includedFilter(sourcePath)) { - return; - } - const plugin = new Plugin({ - framework: this.framework, - sourcePath, - log: this.log, - warn: this.warn - }); - plugin.load(); - return plugin; - }).filter(Boolean); - return this; - } - - /** @returns {JSONFileItem} */ - getAllPackageJSONFileItems() { - return this.plugins.reduce((items, plugin) => { - items.push(...plugin.packageJSONFile.fileItems); - return items; - }, []); - } - -} - -module.exports = Plugins; diff --git a/grunt/helpers/Schemas.js b/grunt/helpers/Schemas.js deleted file mode 100644 index ab5952f51..000000000 --- a/grunt/helpers/Schemas.js +++ /dev/null @@ -1,211 +0,0 @@ -const _ = require('lodash'); -const path = require('path'); -const fs = require('fs-extra'); -const globs = require('globs'); -const ExtensionSchema = require('./schema/ExtensionSchema'); -const ModelSchema = require('./schema/ModelSchema'); -const ModelSchemas = require('./schema/ModelSchemas'); -const Plugins = require('./Plugins'); - -/** - * @typedef {import('./Framework')} Framework - * @typedef {import('./Plugins')} Plugins - * @typedef {import('./plugins/Plugin')} Plugin - */ - -/** - * Represents all of the schemas in a course. - * @todo Work out how to do schema inheritance properly (i.e. component+accordion) - * @todo Stop deriving schema types (model/extension) from bower and/or folder paths - * @todo Stop deriving schema names from bower.json or filenames - * @todo Combining and applying multiple schemas for validation or defaults needs consideration - */ -class Schemas { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {function} options.includedFilter - * @param {Plugins} options.plugins - * @param {string} options.sourcePath - * @param {function} options.log - * @param {function} options.warn - */ - constructor({ - framework = null, - includedFilter = function() { return true; }, - plugins = null, - sourcePath = '', - log = console.log, - warn = console.warn - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {function} */ - this.includedFilter = includedFilter; - /** @type {string} */ - this.sourcePath = sourcePath.replace(/\\/g, '/'); - /** @type {Plugins} */ - this.plugins = plugins; - /** @type {[Schema]]} */ - this.schemas = null; - /** @type {function} */ - this.log = log; - /** @type {function} */ - this.warn = warn; - } - - /** @returns {Schemas} */ - load() { - - /** - * @param {Plugin} plugin - * @param {string} filePath - */ - const createSchema = (plugin, filePath) => { - const json = fs.readJSONSync(filePath); - const isExtensionSchema = Boolean(json.properties.pluginLocations); - const InferredSchemaClass = (isExtensionSchema ? ExtensionSchema : ModelSchema); - const inferredSchemaName = (plugin.name === 'core') ? - path.parse(filePath).name.split('.')[0] : // if core, get schema name from file name - isExtensionSchema ? - plugin.name : // assume schema name is plugin name - plugin.targetAttribute; // assume schema name is plugin._[type] value - return new InferredSchemaClass({ - name: inferredSchemaName, - plugin, - framework: this.framework, - filePath, - globalsType: plugin.type, - targetAttribute: plugin.targetAttribute - }); - }; - - this.plugins = new Plugins({ - framework: this.framework, - includedFilter: this.includedFilter, - sourcePath: this.sourcePath, - log: this.log, - warn: this.warn - }); - this.plugins.load(); - - this.schemas = []; - this.plugins.plugins.forEach(plugin => globs.sync(plugin.schemaLocations).forEach(filePath => { - const schema = createSchema(plugin, filePath); - schema.load(); - this.schemas.push(schema); - })); - - this.generateCourseGlobals(); - this.generateModelExtensions(); - - return this; - } - - /** - * Copy globals schema extensions from model/extension plugins to the course._globals - * schema. - * @returns {Schemas} - * @example - * courseModelSchema.properties._globals.properties._components.properties._accordion - */ - generateCourseGlobals() { - const courseSchema = this.getCourseSchema(); - this.schemas.forEach(schema => { - const globalsPart = schema.getCourseGlobalsPart(); - if (!globalsPart) { - return; - } - _.merge(courseSchema.json.properties._globals.properties, globalsPart); - }); - return this; - } - - /** - * Copy pluginLocations schema extensions from the extension plugins to the appropriate model schemas - * @returns {Schemas} - * @example - * courseModelSchema.properties._assessment - * articleModelSchema.properties._trickle - * blockModelSchema.properties._trickle - */ - generateModelExtensions() { - const extensionSchemas = this.schemas.filter(schema => schema instanceof ExtensionSchema); - extensionSchemas.forEach(schema => { - const extensionParts = schema.getModelExtensionParts(); - if (!extensionParts) { - return; - } - for (const modelName in extensionParts) { - const extensionPart = extensionParts[modelName]; - /** - * Check if the sub-schema part has any defined properties. - * A lot of extension schemas have empty objects with no properties. - */ - if (!extensionPart.properties) { - continue; - } - const modelSchema = this.getModelSchemaByName(modelName); - if (!modelSchema) { - const err = new Error(`Cannot add extensions to model which doesn't exits ${modelName}`); - err.number = 10012; - throw err; - } - /** - * Notice that the targetAttribute is not used here, we allow the extension schema - * to define its own _[targetAttribute] to extend any core model. - */ - modelSchema.json.properties = _.merge({}, modelSchema.json.properties, extensionPart.properties); - } - }); - return this; - } - - /** - * @param {string} schemaName - * @returns {ModelSchema} - */ - getModelSchemaByName(schemaName) { - const modelSchemas = this.schemas.filter(schema => schema instanceof ModelSchema); - return modelSchemas.find(({ name }) => name === schemaName); - } - - /** @returns {ModelSchema} */ - getCourseSchema() { - return this.getModelSchemaByName('course'); - } - - /** @returns {ModelSchema} */ - getConfigSchema() { - return this.getModelSchemaByName('config'); - } - - /** - * Uses a model JSON to derive the appropriate schemas for the model. - * @param {Object} json - * @returns {ModelSchemas} - */ - getSchemasForModelJSON(json) { - const schemas = []; - if (json._type) { - if (json._type === 'menu' || json._type === 'page') { - schemas.push(this.getModelSchemaByName('contentobject')); - } - schemas.push(this.getModelSchemaByName(json._type)); - } - if (json._component) { - schemas.push(this.getModelSchemaByName(json._component)); - } - if (json._model) { - schemas.push(this.getModelSchemaByName(json._model)); - } - return new ModelSchemas({ - framework: this.framework, - schemas: schemas.filter(Boolean) - }); - } - -} - -module.exports = Schemas; diff --git a/grunt/helpers/Translate.js b/grunt/helpers/Translate.js deleted file mode 100644 index 00b8b6ad3..000000000 --- a/grunt/helpers/Translate.js +++ /dev/null @@ -1,501 +0,0 @@ -const path = require('path'); -const _ = require('lodash'); -const fs = require('fs-extra'); -const csv = require('csv'); -const { XMLParser } = require('fast-xml-parser'); -const async = require('async'); -const globs = require('globs'); -const jschardet = require('jschardet'); -const iconv = require('iconv-lite'); -const Data = require('./Data'); -const Schemas = require('./Schemas'); - -/** - * @typedef {import('./Framework')} Framework - */ - -/** - * Pulls together schemas and data to enable both the export and import of data item attributes - * marked by the schemas as translatable. - */ -class Translate { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {function} options.includedFilter - * @param {string} options.masterLang - * @param {string} options.targetLang - * @param {string} options.format - * @param {string} options.csvDelimiter - * @param {boolean} options.shouldReplaceExisting - * @param {string} options.jsonext - * @param {string} options.sourcePath - * @param {string} options.languagePath - * @param {string} options.outputPath - * @param {boolean} options.isTest - * @param {function} options.log - */ - constructor({ - framework = null, - includedFilter = function() { return true; }, - masterLang = 'en', - targetLang = null, - format = 'csv', - csvDelimiter = null, - shouldReplaceExisting = false, - jsonext = 'json', - sourcePath = '', - languagePath = path.join(process.cwd(), 'languagefiles'), - outputPath = '', - courseDir = 'course', - useOutputData = false, - isTest = false, - log = console.log, - warn = console.warn - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {function} */ - this.includedFilter = includedFilter; - /** @type {string} */ - this.masterLang = masterLang; - /** @type {string} */ - this.targetLang = targetLang; - // format can be raw, json or csv - /** @type {string} */ - this.format = format; - /** @type {string} */ - this.csvDelimiter = csvDelimiter; - /** @type {boolean} */ - this.shouldReplaceExisting = shouldReplaceExisting; - /** @type {string} */ - this.jsonext = jsonext; - /** @type {string} */ - this.sourcePath = sourcePath.replace(/\\/g, '/'); - /** @type {string} */ - this.outputPath = outputPath.replace(/\\/g, '/'); - /** @type {string} */ - this.courseDir = courseDir; - /** @type {Framework} */ - this.useOutputData = useOutputData; - /** @type {string} */ - this.languagePath = languagePath.replace(/\\/g, '/'); - /** @type {Data} */ - this.data = null; - /** @type {boolean} */ - this.isTest = isTest; - /** @type {function} */ - this.log = log; - /** @type {function} */ - this.warn = warn; - } - - /** @returns {Translate} */ - load() { - this.data = new Data({ - framework: this.framework, - sourcePath: this.useOutputData ? this.outputPath : this.sourcePath, - courseDir: this.courseDir, - jsonext: this.jsonext, - trackingIdType: this.framework.trackingIdType, - log: this.log - }); - this.data.load(); - return this; - } - - /** - * Produces a single JSON file or a series of CSV files representing all of the - * files and file items in the data structure. - * @returns {Translate} - */ - async export() { - - const schemas = new Schemas({ framework: this.framework, includedFilter: this.includedFilter, sourcePath: this.sourcePath }); - schemas.load(); - - const exportTextData = []; - - // collection translatable texts - const language = this.data.getLanguage(this.masterLang); - language.getAllFileItems().forEach(({ file, item }) => { - - const applicableSchemas = schemas.getSchemasForModelJSON(item); - const translatablePaths = applicableSchemas.getTranslatablePaths(); - - function recursiveJSONProcess(data, level, path, lookupPath, id, file, component) { - if (level === 0) { - // at the root - id = data.hasOwnProperty('_id') ? data._id : null; - component = data.hasOwnProperty('_component') ? data._component : null; - } - if (Array.isArray(data)) { - for (let i = 0; i < data.length; i++) { - recursiveJSONProcess(data[i], level += 1, path + i + '/', lookupPath, id, file, component); - } - return; - } - if (typeof data === 'object') { - for (const attribute in data) { - recursiveJSONProcess(data[attribute], level += 1, path + attribute + '/', lookupPath + attribute + '/', id, file, component); - } - return; - } - if (data && translatablePaths.includes(lookupPath)) { - exportTextData.push({ - file, - id, - path, - value: data - }); - } - } - - const filename = path.parse(file.path).name.split('.')[0]; - recursiveJSONProcess(item, 0, '/', '/', null, filename, null); - - }); - - // maintain order with original translate tasks - const typeSortLevel = { - course: 1, - contentObjects: 2, - articles: 3, - blocks: 4, - components: 5 - }; - exportTextData.sort((a, b) => { - const typeSort = ((typeSortLevel[a.file] || 100) - (typeSortLevel[b.file] || 100)); - return typeSort || a.id.length - b.id.length || a.id.localeCompare(b.id); - }); - - // output based upon format options - const outputFolder = path.join(this.languagePath, this.masterLang); - fs.mkdirpSync(outputFolder); - - if (this.format === 'json' || this.format === 'raw') { - const filePath = path.join(outputFolder, 'export.json'); - this.log(`Exporting json to ${filePath}`); - fs.writeJSONSync(filePath, exportTextData, { spaces: 2 }); - return this; - } - - if (['xliff', 'xlf'].includes(this.format)) { - // create csv for each file - const outputGroupedByFile = exportTextData.reduce((prev, current) => { - if (!prev.hasOwnProperty(current.file)) { - prev[current.file] = []; - } - prev[current.file].push(current); - return prev; - }, {}); - - // xliff 2.0 - // const output = ` - // - // ${Object.entries(outputGroupedByFile).map(([fileName, entries]) => { - // return ` - // ${entries.map(item => { - // const value = /[<>&"'/]/.test(item.value) - // ? `` - // : item.value; - // return ` - // - // ${value} - // ${value} - // - // - // `; - // }).filter(Boolean).join('')} - // `; - // }).join('')}`; - - // xliff 1.2 - const output = ` - -${Object.entries(outputGroupedByFile).map(([fileName, entries]) => { - return ` -${entries.map(item => { - const value = /[<>&"'/]/.test(item.value) - ? `` - : item.value; - return ` - ${value} - ${value} - -`; - }).filter(Boolean).join('')} -`; - }).join('')}`; - const filePath = path.join(outputFolder, 'source.xlf'); - this.log(`Exporting xliff to ${filePath}`); - fs.writeFileSync(filePath, `${output}`); - return this; - } - - // create csv for each file - const outputGroupedByFile = exportTextData.reduce((prev, current) => { - if (!prev.hasOwnProperty(current.file)) { - prev[current.file] = []; - } - prev[current.file].push([`${current.file}/${current.id}${current.path}`, current.value]); - return prev; - }, {}); - - const csvOptions = { - quotedString: true, - delimiter: this.csvDelimiter || ',' - }; - - const fileNames = Object.keys(outputGroupedByFile); - await async.each(fileNames, (name, done) => { - csv.stringify(outputGroupedByFile[name], csvOptions, (error, output) => { - if (error) { - return done(new Error('Error saving CSV files.')); - } - const filePath = path.join(outputFolder, `${name}.csv`); - this.log(`Exporting csv to ${filePath}`); - fs.writeFileSync(filePath, `\ufeff${output}`); - done(null); - }); - }); - - return this; - } - - /** - * Imports a single JSON file or multiple CSV files to replace values in the - * existing data file items. - * @returns {Translate} - */ - async import() { - - if (this.isTest) { - this.log('!TEST IMPORT, not changing data.'); - } - - // check that a targetLang has been specified - if (!this.targetLang) { - const err = new Error('Target language option is missing. '); - err.number = 10001; - throw err; - } - - // check input folder exists - const inputFolder = path.join(this.languagePath, this.targetLang); - if (!fs.existsSync(inputFolder) || !fs.statSync(inputFolder).isDirectory()) { - const err = new Error(`Folder does not exist. ${inputFolder}`); - err.number = 10002; - throw err; - } - - // auto-detect format if not specified - let format = this.format; - if (!format) { - const filePaths = globs.sync([`${inputFolder}/*.*`]); - const uniqueFileExtensions = _.uniq(filePaths.map(filePath => path.extname(filePath).slice(1))); - if (uniqueFileExtensions.length !== 1) { - throw new Error(`Format autodetection failed, ${uniqueFileExtensions.length} file types found.`); - } - format = uniqueFileExtensions[0]; - switch (format) { - case 'xlf': - case 'xliff': - case 'csv': - case 'json': - this.log(`Format autodetected as ${format}`); - break; - default: - throw new Error(`Format of the language file is not supported: ${format}`); - } - } - - if (format === 'xliff') format = 'xlf'; - - // discover import files - const langFiles = globs.sync([`${inputFolder}/*.${format}`]); - if (langFiles.length === 0) { - throw new Error(`No languagefiles found to process in folder ${inputFolder}`); - } - - // copy master language files to target language directory if needed - if (this.targetLang !== this.masterLang) { - this.data.copyLanguage(this.masterLang, this.targetLang, this.shouldReplaceExisting); - } - - // get target language data - const targetLanguage = this.data.getLanguage(this.targetLang); - - if (this.targetLang === this.masterLang && !this.shouldReplaceExisting) { - const err = new Error(`Folder already exists. ${targetLanguage.path}`); - err.number = 10003; - throw err; - } - - // process import files - let importData; - switch (format) { - case 'json': - importData = fs.readJSONSync(langFiles[0]); - break; - case 'xliff': - case 'xlf': { - importData = []; - await async.each(langFiles, (filename, done) => { - const XMLData = fs.readFileSync(filename); - const parser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '' - }); - const xml = parser.parse(XMLData); - // xliff 2.0 - // for (const file of xml.xliff.file) { - // for (const unit of file.unit) { - // const [ id, ...path ] = unit.id.split('/'); - // importData.push({ - // file: file.id, - // id, - // path: path.filter(Boolean).join('/'), - // value: unit.segment.target['#text'] - // }); - // } - // } - // xliff 1.2 - for (const file of xml.xliff.file) { - for (const unit of file.body['trans-unit']) { - const [ id, ...path ] = unit.id.split('/'); - importData.push({ - file: file.original, - id, - path: path.filter(Boolean).join('/'), - value: unit.source - }); - } - } - done(); - }); - break; - } - case 'csv': - default: { - importData = []; - const lines = []; - await async.each(langFiles, (filename, done) => { - const fileBuffer = fs.readFileSync(filename, { - encoding: null - }); - const detected = jschardet.detect(fileBuffer); - let fileContent; - if (iconv.encodingExists(detected.encoding)) { - fileContent = iconv.decode(fileBuffer, detected.encoding); - this.log(`Encoding detected as ${detected.encoding} ${filename}`); - } else { - fileContent = iconv.decode(fileBuffer, 'utf8'); - this.log(`Encoding not detected used utf-8 ${filename}`); - } - let csvDelimiter = this.csvDelimiter; - if (!csvDelimiter) { - const firstLineMatches = fileContent.match(/^[^,;\t| \n\r]+\/"{0,1}[,;\t| ]{1}/); - if (firstLineMatches && firstLineMatches.length) { - const detectedDelimiter = firstLineMatches[0].slice(-1); - if (detectedDelimiter !== this.csvDelimiter) { - this.log(`Delimiter detected as ${detectedDelimiter} in ${filename}`); - csvDelimiter = detectedDelimiter; - } - } - } - if (!csvDelimiter) { - const err = new Error(`Could not detect csv delimiter ${targetLanguage.path}`); - err.number = 10014; - throw err; - } - const options = { - delimiter: csvDelimiter - }; - csv.parse(fileContent, options, (error, output) => { - if (error) { - return done(error); - } - let hasWarnedTruncated = false; - output.forEach(line => { - if (line.length < 2) { - throw new Error(`Too few columns detected: expected 2, found ${line.length} in ${filename}`); - } - if (line.length !== 2 && !hasWarnedTruncated) { - this.log(`Truncating, too many columns detected: expected 2, found extra ${line.length - 2} in ${filename}`); - hasWarnedTruncated = true; - } - line.length = 2; - }); - lines.push(...output); - done(null); - }); - }).then(() => { - lines.forEach(line => { - const [ file, id, ...path ] = line[0].split('/'); - importData.push({ - file, - id, - path: path.filter(Boolean).join('/'), - value: line[1] - }); - }); - }, err => { - throw new Error(`Error processing CSV files: ${err}`); - }); - break; - } - } - - // check import validity - const item = importData[0]; - const isValid = item.hasOwnProperty('file') && item.hasOwnProperty('id') && item.hasOwnProperty('path') && item.hasOwnProperty('value'); - if (!isValid) { - throw new Error('Sorry, the imported File is not valid'); - } - - // maintain output order with original translate tasks - // TODO: could probably improve this with read order rather than file order - const typeSortLevel = { - course: 1, - contentObjects: 2, - articles: 3, - blocks: 4, - components: 5 - }; - importData.sort((a, b) => { - const typeSort = ((typeSortLevel[a.file] || 100) - (typeSortLevel[b.file] || 100)); - return typeSort || a.id.length - b.id.length || a.id.localeCompare(b.id); - }); - - // update data - importData.forEach(data => { - const { file, item } = targetLanguage.getFileItemById(data.id); - const attributePath = data.path.split('/').filter(Boolean); - const currentValue = _.get(item, attributePath); - if (currentValue === data.value) { - // value is unchanged, skip - return; - } - this.log(`#${data.id}\t${attributePath.join('.')}`); - this.log(` '${currentValue}'`); - this.log(` '${data.value}'`); - _.set(item, attributePath, data.value); - file.changed(); - }); - if (!targetLanguage.hasChanged) { - this.warn('No changed were found, target and import are identical.'); - } - - // save data - if (!this.isTest) { - this.data.save(); - } - - return this; - } - -} - -module.exports = Translate; diff --git a/grunt/helpers/data/Language.js b/grunt/helpers/data/Language.js deleted file mode 100644 index 8c5ff1313..000000000 --- a/grunt/helpers/data/Language.js +++ /dev/null @@ -1,303 +0,0 @@ -const path = require('path'); -const fs = require('fs-extra'); -const globs = require('globs'); -const _ = require('lodash'); -const chalk = require('chalk'); -const LanguageFile = require('./LanguageFile'); - -/** - * @typedef {import('../Framework')} Framework - * @typedef {import('../lib/JSONFileItem')} JSONFileItem - */ - -/** - * Represents all of the json files and file item in a language directory - * at course/[lang]/*.[jsonext]. - * It is filename agnostic, such that there are no hard references to file names. - * It has _id and _parentId structure checking and _trackingId management included. - */ -class Language { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {string} options.languagePath - * @param {string} options.jsonext - * @param {string} options.trackingIdType, - * @param {function} options.log - */ - constructor({ - framework = null, - languagePath = '', - jsonext = 'json', - trackingIdType = 'block', - log = console.log - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {string} */ - this.jsonext = jsonext; - /** @type {string} */ - this.path = path.normalize(languagePath).replace(/\\/g, '/'); - /** @type {string} */ - this.name = this.path.split('/').filter(Boolean).pop(); - /** @type {string} */ - this.rootPath = this.path.split('/').filter(Boolean).slice(0, -1).join('/'); - /** @type {string} */ - this.manifestFileName = 'language_data_manifest.js'; - /** @type {string} */ - this.manifestPath = this.path + this.manifestFileName; - /** @type {string} */ - this.trackingIdType = trackingIdType; - /** @type {[LanguageFile]} */ - this.files = null; - /** @type {Object.} */ - this._itemIdIndex = null; - /** @type {[JSONFileItem]} */ - this.courseFileItem = null; - /** @type {function} */ - this.log = log; - } - - /** @returns {Language} */ - load() { - this.files = []; - this._itemIdIndex = {}; - this.courseFileItem = null; - - const dataFiles = globs.sync(path.join(this.path, '*.' + this.jsonext)).map((dataFilePath) => { - const relativePath = dataFilePath.slice(this.path.length); - return relativePath; - }).filter((dataFilePath) => { - const isManifest = (dataFilePath === this.manifestPath); - // Skip file if it is the Authoring Tool import/export asset manifest - const isAATAssetJSON = (dataFilePath === 'assets.json'); - return !isManifest && !isAATAssetJSON; - }); - - dataFiles.forEach(jsonFileName => { - const jsonFilePath = (this.path + jsonFileName).replace(/\\/g, '/'); - const file = new LanguageFile({ - framework: this.framework, - language: this, - jsonext: this.jsonext, - path: jsonFilePath, - data: null, - hasChanged: false - }); - file.load(); - this.files.push(file); - }); - - this._itemIdIndex = this.getAllFileItems().reduce((index, fileItem) => { - const { file, item } = fileItem; - if (item._id && index[item._id]) { - const err = new Error(`Duplicate ids ${item._id} in ${index[item._id].file.path} and ${file.path}`); - err.number = 10006; - throw err; - } else if (item._id) { - index[item._id] = fileItem; - } - if (item._type === 'course' && this.courseFileItem) { - const err = new Error(`Duplicate course items found, in ${index[item._id].file.path} and ${file.path}`); - err.number = 10007; - throw err; - } else if (item._type === 'course') { - this.courseFileItem = fileItem; - } - return index; - }, {}); - - return this; - } - - /** @type {boolean} */ - get isValid() { - return Boolean(this.courseFileItem); - } - - /** @type {boolean} */ - get hasChanged() { - return this.files.some(file => file.hasChanged); - } - - /** - * Produces a manifest file for the Framework data layer at course/lang/language_data_manifest.js. - * @returns {Language} - */ - saveManifest() { - const dataFiles = globs.sync(path.join(this.path, '*.' + this.jsonext)).map((dataFilePath) => { - const relativePath = dataFilePath.slice(this.path.length); - return relativePath; - }).filter((dataFilePath) => { - const isManifest = (dataFilePath === this.manifestPath); - // Skip file if it is the Authoring Tool import/export asset manifest - const isAATAssetJSON = (dataFilePath === 'assets.json'); - return !isManifest && !isAATAssetJSON; - }); - const hasNoDataFiles = !dataFiles.length; - if (hasNoDataFiles) { - const err = new Error(`No data files found in ${this.path}`); - err.number = 10008; - throw err; - } - fs.writeJSONSync(this.manifestPath, dataFiles, { spaces: 0 }); - return this; - } - - /** @returns {[JSONFileItem]} */ - getAllFileItems() { - return this.files.reduce((memo, file) => { - memo.push(...file.fileItems); - return memo; - }, []); - } - - /** @returns {JSONFileItem} */ - getCourseFileItem() { - if (!this.courseFileItem) { - const err = new Error(`Could not find course item for ${this.path}`); - err.number = 10009; - throw err; - } - return this.courseFileItem; - } - - /** - * @param {string} id - * @returns {JSONFileItem} - */ - getFileItemById(id) { - const fileItem = this._itemIdIndex[id]; - if (!fileItem) { - const err = new Error(`Could not find item for id ${id} in ${this.path}`); - err.number = 10010; - throw err; - } - return fileItem; - } - - /** @returns {Language} */ - checkIds() { - const items = this.getAllFileItems().map(({ item }) => item); - // Index and group - const idIndex = _.keyBy(items, '_id'); - const idGroups = _.groupBy(items, '_id'); - const parentIdGroups = _.groupBy(items, '_parentId'); - // Setup error collection arrays - let orphanedIds = {}; - let emptyIds = {}; - let duplicateIds = {}; - let missingIds = {}; - items.forEach((o) => { - const isCourseType = (o._type === 'course'); - const isComponentType = (o._type === 'component'); - if (idGroups[o._id].length > 1) { - duplicateIds[o._id] = true; // Id has more than one item - } - if (!isComponentType && !parentIdGroups[o._id]) { - emptyIds[o._id] = true; // Course has no children - } - if (!isCourseType && (!o._parentId || !idIndex[o._parentId])) { - orphanedIds[o._id] = o._type; // Item has no defined parent id or the parent id doesn't exist - } - if (!isCourseType && o._parentId && !idIndex[o._parentId]) { - missingIds[o._parentId] = o._type; // Referenced parent item does not exist - } - }); - const orphanedIdsArray = Object.keys(orphanedIds); - const missingIdsArray = Object.keys(missingIds); - emptyIds = Object.keys(emptyIds); - duplicateIds = Object.keys(duplicateIds); - // Output for each type of error - const hasErrored = orphanedIdsArray.length || emptyIds.length || duplicateIds.length || missingIdsArray.length; - if (orphanedIdsArray.length) { - const orphanedIdString = orphanedIdsArray.map((id) => `${id} (${orphanedIds[id]})`); - this.log(chalk.yellow(`Orphaned _ids: ${orphanedIdString.join(', ')}`)); - } - if (missingIdsArray.length) { - const missingIdString = missingIdsArray.map((id) => `${id} (${missingIds[id]})`); - this.log(chalk.yellow(`Missing _ids: ${missingIdString.join(', ')}`)); - } - if (emptyIds.length) { - this.log(chalk.yellow(`Empty _ids: ${emptyIds}`)); - } - if (duplicateIds.length) { - this.log(chalk.yellow(`Duplicate _ids: ${duplicateIds}`)); - } - // If any error has occured, stop processing. - if (hasErrored) { - const err = new Error('Oops, looks like you have some json errors.'); - err.number = 10011; - throw err; - } - this.log(`No issues found in course/${this.name}, your JSON is a-ok!`); - - return this; - } - - /** @returns {Language} */ - addTrackingIds() { - const { file, item: course } = this.getCourseFileItem(); - course._latestTrackingId = course._latestTrackingId || -1; - file.changed(); - - let wasAdded = false; - const trackingIdsSeen = []; - const fileItems = this.getAllFileItems().filter(fileItem => fileItem.item._type === this.trackingIdType); - fileItems.forEach(({ file, item }) => { - this.log(`${this.trackingIdType}: ${item._id}: ${item._trackingId !== undefined ? item._trackingId : 'not set'}`); - if (item._trackingId === undefined) { - item._trackingId = ++course._latestTrackingId; - file.changed(); - wasAdded = true; - this.log(`Adding tracking ID: ${item._trackingId} to ${this.trackingIdType} ${item._id}`); - } else { - if (trackingIdsSeen.indexOf(item._trackingId) > -1) { - item._trackingId = ++course._latestTrackingId; - file.changed(); - wasAdded = true; - this.log(`Warning: ${item._id} has the tracking ID ${item._trackingId} but this is already in use. Changing to ${course._latestTrackingId + 1}.`); - } else { - trackingIdsSeen.push(item._trackingId); - } - } - if (course._latestTrackingId < item._trackingId) { - course._latestTrackingId = item._trackingId; - } - }); - - this.save(); - this.log(`Tracking IDs ${wasAdded ? 'were added to' : 'are ok for'} course/${this.name}. The latest tracking ID is ${course._latestTrackingId}\n`); - - return this; - } - - /** @returns {Language} */ - removeTrackingIds() { - const { file, item: course } = this.getCourseFileItem(); - course._latestTrackingId = -1; - file.changed(); - - this.getAllFileItems().forEach(({ file, item }) => { - if (item._type !== this.trackingIdType) return; - delete item._trackingId; - file.changed(); - }); - - this.save(); - this.log(`Tracking IDs removed from course/${this.name}.`); - - return this; - } - - /** @returns {Language} */ - save() { - this.files.forEach(file => file.save()); - this.load(); - return this; - } - -} - -module.exports = Language; diff --git a/grunt/helpers/data/LanguageFile.js b/grunt/helpers/data/LanguageFile.js deleted file mode 100644 index c2d58dc1f..000000000 --- a/grunt/helpers/data/LanguageFile.js +++ /dev/null @@ -1,37 +0,0 @@ -const JSONFile = require('../lib/JSONFile'); - -/** - * @typedef {import('../Framework')} Framework - * @typedef {import('./Language')} Language - */ - -/** - * Represents any of the files at course/[lang]/*.[jsonext]. - */ -class LanguageFile extends JSONFile { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {Language} options.language - * @param {string} options.path - * @param {string} options.jsonext - * @param {Object} options.data - * @param {boolean} options.hasChanged - */ - constructor({ - framework = null, - language = null, - path = null, - jsonext = null, - data = null, - hasChanged = false - } = {}) { - super({ framework, path, jsonext, data, hasChanged }); - /** @type {Language} */ - this.language = language; - } - -} - -module.exports = LanguageFile; diff --git a/grunt/helpers/lib/JSONFile.js b/grunt/helpers/lib/JSONFile.js deleted file mode 100644 index 037dfa87e..000000000 --- a/grunt/helpers/lib/JSONFile.js +++ /dev/null @@ -1,106 +0,0 @@ -const fs = require('fs-extra'); -const JSONFileItem = require('./JSONFileItem'); - -/** - * @typedef {import('../Framework')} Framework - * @typedef {import('./JSONFile')} JSONFile - */ - -/** - * An abstraction for centralising the loading of JSON files, keeping track of - * sub-item changes and saving changed files. - */ -class JSONFile { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {string} options.path - * @param {Object|Array} options.data - * @param {boolean} options.hasChanged - */ - constructor({ - framework = null, - path = null, - jsonext = null, - data = null, - hasChanged = false - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {string} */ - this.path = path; - /** @type {string} */ - this.jsonext = jsonext; - /** @type {Object|Array} */ - this.data = data; - /** @type {boolean} */ - this.hasChanged = hasChanged; - /** @type {[JSONFileItem]} */ - this.fileItems = null; - } - - /** @returns {JSONFile} */ - load() { - this.fileItems = []; - - if (this.path) { - this.data = fs.readJSONSync(this.path); - } - - const addObject = (item, index = null) => { - this.fileItems.push(new JSONFileItem({ - framework: this.framework, - file: this, - item, - index - })); - }; - - if (this.data instanceof Array) { - this.data.forEach((item, index) => addObject(item, index)); - } else if (this.data instanceof Object) { - addObject(this.data, null); - } else { - const err = new Error(`Cannot load json file ${this.path} as it doesn't contain an Array or Object as its root`); - err.number = 10013; - throw err; - } - - return this; - } - - /** - * This is useful for files such as config.json or course.json which only have - * one item/object per file. - * @returns {JSONFileItem} - */ - get firstFileItem() { - return this.fileItems && this.fileItems[0]; - } - - /** - * Marks this file as having changed. This should be called after changing - * the fileItems contained in this instance. - * @returns {JSONFile} - */ - changed() { - this.hasChanged = true; - return this; - } - - /** - * Saves any fileItem changes to disk. - * @return {JSONFile} - */ - save() { - if (!this.hasChanged) { - return this; - } - fs.writeJSONSync(this.path, this.data, { spaces: 2 }); - return this; - } - -} - -module.exports = JSONFile; diff --git a/grunt/helpers/lib/JSONFileItem.js b/grunt/helpers/lib/JSONFileItem.js deleted file mode 100644 index 3e63c4ab4..000000000 --- a/grunt/helpers/lib/JSONFileItem.js +++ /dev/null @@ -1,29 +0,0 @@ -/** - * @typedef {import('../Framework')} Framework - * @typedef {import('./JSONFile')} JSONFile - */ - -/** - * An abstraction for carrying JSON sub-items with their file origins and locations. - */ -class JSONFileItem { - - constructor({ - framework = null, - file = null, - item = null, - index = null - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {JSONFile} */ - this.file = file; - /** @type {Object} */ - this.item = item; - /** @type {number} */ - this.index = index; - } - -} - -module.exports = JSONFileItem; diff --git a/grunt/helpers/plugins/Plugin.js b/grunt/helpers/plugins/Plugin.js deleted file mode 100644 index 44962fd0d..000000000 --- a/grunt/helpers/plugins/Plugin.js +++ /dev/null @@ -1,145 +0,0 @@ -const semver = require('semver'); -const globs = require('globs'); -const JSONFile = require('../lib/JSONFile'); - -/** - * Represents a single plugin location, bower.json, version, name and schema - * locations. - * @todo Should be able to define multiple schemas for all plugins in the AAT - * and in the Framework - * @todo Switch to package.json with the move to npm - * @todo Plugin type is inferred from the JSON. - * @todo Component _globals target attribute is inferred from the _component - */ -class Plugin { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {string} options.sourcePath - * @param {string} options.jsonext - * @param {function} options.log - */ - constructor({ - framework = null, - sourcePath = '', - log = console.log, - warn = console.warn - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {string} */ - this.sourcePath = sourcePath.replace(/\\/g, '/'); - /** @type {function} */ - this.log = log; - /** @type {function} */ - this.warn = warn; - /** @type {JSONFile} */ - this.packageJSONFile = null; - } - - /** @returns {Plugin} */ - load() { - const pathDerivedName = this.sourcePath.split('/').filter(Boolean).pop(); - const files = globs.sync(this.packageJSONLocations); - const firstFile = files[0]; - if (firstFile) { - // use the first package definition found (this will be bower.json / package.json) - this.packageJSONFile = new JSONFile({ - path: firstFile - }); - } else { - // no json found, make some up (this will usually be the core) - this.packageJSONFile = new JSONFile({ - path: null, - data: { - name: pathDerivedName - } - }); - } - this.packageJSONFile.load(); - const packageName = (this.packageJSONFile.firstFileItem.item.name); - if (packageName !== pathDerivedName) { - // assume path name is also plugin name, this shouldn't be necessary - this.warn(`Plugin folder name ${pathDerivedName} does not match package name ${packageName}.`); - } - if (this.requiredFramework && !this.isFrameworkCompatible) { - this.warn(`Required framework version (${this.requiredFramework}) for plugin ${packageName} not satisfied by current framework version (${this.framework.version}).`); - } - return this; - } - - /** - * Informs the Schemas API from where to fetch the schemas defined in this - * plugin. - * @returns {[string]} - */ - get schemaLocations() { - return [ - `${this.sourcePath}properties.schema`, - `${this.sourcePath}schema/*.schema` - ]; - } - - /** - * @returns {[string]} - */ - get packageJSONLocations() { - return [ - `${this.sourcePath}bower.json` - ]; - } - - /** @returns {string} */ - get name() { - return this.packageJSONFile.firstFileItem.item.name; - } - - /** @returns {string} */ - get version() { - return this.packageJSONFile.firstFileItem.item.version; - } - - /** @returns {string} */ - get requiredFramework() { - return this.packageJSONFile.firstFileItem.item.framework; - } - - /** @returns {boolean} */ - get isFrameworkCompatible() { - if (!this.framework || !this.framework.version) return true; - - return semver.satisfies(this.framework.version, this.requiredFramework); - } - - /** - * Returns the plugin type name. - * @returns {string} - */ - get type() { - if (this.name === 'core') { - return 'component'; - } - const config = this.packageJSONFile.firstFileItem.item; - const configKeys = Object.keys(config); - const typeKeyName = ['component', 'extension', 'menu', 'theme']; - const foundType = configKeys.find(key => typeKeyName.includes(key)); - if (!foundType) { - throw new Error(`Unknown plugin type for ${this.name}`); - } - return foundType; - } - - /** - * @returns {string} - */ - get targetAttribute() { - if (this.type === 'component') { - return this.packageJSONFile.firstFileItem.item.component; - } - return this.packageJSONFile.firstFileItem.item.targetAttribute; - } - -} - -module.exports = Plugin; diff --git a/grunt/helpers/schema/ExtensionSchema.js b/grunt/helpers/schema/ExtensionSchema.js deleted file mode 100644 index 1506c2617..000000000 --- a/grunt/helpers/schema/ExtensionSchema.js +++ /dev/null @@ -1,28 +0,0 @@ -const GlobalsSchema = require('./GlobalsSchema'); - -/** - * Represents a model extension schema. Currently these schemas are only able to - * extend config, course, contentobject, article, block and component models. - * - * @todo pluginLocations is not a good way of listing model extensions or detecting - * extension schemas. There should be a schema type (model/extension) and - * the extensions should be declared at the root of the schema. - * @todo pluginLocations[modelName] should extend any model (article, block, - * accordion, narrative) - */ -class ExtensionSchema extends GlobalsSchema { - - /** - * Returns the defined model extension sub-schemas listed at pluginLocations. - * @returns {Object|undefined} - */ - getModelExtensionParts() { - if (!this.json.properties.pluginLocations || !this.json.properties.pluginLocations.properties) { - return; - } - return this.json.properties.pluginLocations && this.json.properties.pluginLocations.properties; - } - -} - -module.exports = ExtensionSchema; diff --git a/grunt/helpers/schema/GlobalsSchema.js b/grunt/helpers/schema/GlobalsSchema.js deleted file mode 100644 index 716f4ddf8..000000000 --- a/grunt/helpers/schema/GlobalsSchema.js +++ /dev/null @@ -1,67 +0,0 @@ -const Schema = require('./Schema'); - -/** - * @typedef {import('./Framework')} Framework - * @typedef {import('../plugins/Plugin')} Plugin - */ - -/** - * Represents the globals properties listed in a model or extension schema. - * @todo _globals doesn't need to differentiate by plugin type, plugin name should suffice - * @todo We should drop all pluralisations, they're unnecessary and complicated. - */ -class GlobalsSchema extends Schema { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {string} options.name - * @param {Plugin} options.plugin - * @param {Object} options.json - * @param {string} options.filePath - * @param {string} options.globalsType - * @param {string} options.targetAttribute Attribute where this sub-schema will be injected into the course.json:_globals._[pluginType] object - */ - constructor({ - framework = null, - name = '', - plugin = null, - json = {}, - filePath = '', - globalsType = '', - targetAttribute = '' - } = {}) { - super({ framework, name, plugin, json, filePath, globalsType }); - // Add an underscore to the front of the targetAttribute if necessary - this.targetAttribute = (targetAttribute && targetAttribute[0] !== '_' ? '_' : '') + targetAttribute; - } - - /** - * Returns the sub-schema for the course.json:_globals object. - * @returns {Object|undefined} - */ - getCourseGlobalsPart() { - if (!this.json.globals) { - return; - } - /** - * pluralise location name if necessary (components, extensions) etc - */ - const shouldPluralise = ['component', 'extension'].includes(this.globalsType); - const globalsType = shouldPluralise ? this.globalsType + 's' : this.globalsType; - return { - [`_${globalsType}`]: { - type: 'object', - properties: { - [this.targetAttribute]: { - type: 'object', - properties: this.json.globals - } - } - } - }; - } - -} - -module.exports = GlobalsSchema; diff --git a/grunt/helpers/schema/ModelSchema.js b/grunt/helpers/schema/ModelSchema.js deleted file mode 100644 index 4bdcd7d82..000000000 --- a/grunt/helpers/schema/ModelSchema.js +++ /dev/null @@ -1,46 +0,0 @@ -const GlobalsSchema = require('./GlobalsSchema'); - -/** - * Represents an article, block, accordion or other type of model schema - */ -class ModelSchema extends GlobalsSchema { - - /** - * Create array of translatable attribute paths - * @returns {[string]} - */ - getTranslatablePaths() { - const paths = {}; - this.traverse('', ({ description, next }, attributePath) => { - switch (description.type) { - case 'object': - next(attributePath + description.name + '/'); - break; - case 'array': - if (!description.hasOwnProperty('items')) { - // handles 'inputType': 'List' edge-case - break; - } - if (description.items.type === 'object') { - next(attributePath + description.name + '/'); - } else { - next(attributePath); - } - break; - case 'string': - // check if attribute should be picked - const value = Boolean(description.translatable); - if (value === false) { - break; - } - // add value to store - paths[attributePath + description.name + '/'] = value; - break; - } - }, '/'); - return Object.keys(paths); - } - -} - -module.exports = ModelSchema; diff --git a/grunt/helpers/schema/ModelSchemas.js b/grunt/helpers/schema/ModelSchemas.js deleted file mode 100644 index b536d146a..000000000 --- a/grunt/helpers/schema/ModelSchemas.js +++ /dev/null @@ -1,43 +0,0 @@ -const _ = require('lodash'); - -/** - * @typedef {import('./Framework')} Framework - * @typedef {import('./ModelSchema')} ModelSchema - * @typedef {import('./Schema')} Schema - */ - -/** - * Encapsulates a collection of ModelSchema - * @todo Validation, maybe, if this set of files is used in the AAT - */ -class ModelSchemas { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {[ModelSchema]} options.schemas - */ - constructor({ - framework = null, - schemas = [] - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {[ModelSchema]} */ - this.schemas = schemas; - } - - /** - * Returns an array of translatable attribute paths derived from the schemas. - * @returns {[string]} - */ - getTranslatablePaths() { - return _.uniq(this.schemas.reduce((paths, modelSchema) => { - paths.push(...modelSchema.getTranslatablePaths()); - return paths; - }, [])); - } - -} - -module.exports = ModelSchemas; diff --git a/grunt/helpers/schema/Schema.js b/grunt/helpers/schema/Schema.js deleted file mode 100644 index 9bdeaab15..000000000 --- a/grunt/helpers/schema/Schema.js +++ /dev/null @@ -1,193 +0,0 @@ -const _ = require('lodash'); -const fs = require('fs-extra'); - -/** - * @typedef {import('./Framework')} Framework - * @typedef {import('./JSONFileItem')} JSONFileItem - * @typedef {import('../plugins/Plugin')} Plugin - */ - -class Schema { - - /** - * @param {Object} options - * @param {Framework} options.framework - * @param {string} options.name - * @param {Plugin} options.plugin - * @param {Object} options.json - * @param {string} options.filePath - * @param {string} options.globalsType - */ - constructor({ - framework = null, - name = '', - plugin = null, - json = null, - filePath = '', - globalsType = '' - } = {}) { - /** @type {Framework} */ - this.framework = framework; - /** @type {Plugin} */ - this.plugin = plugin; - /** @type {string} */ - this.name = name; - /** @type {Object} */ - this.json = json; - /** @type {string} */ - this.filePath = filePath; - /** @type {string} */ - this.globalsType = globalsType; - } - - /** @returns {Schema} */ - load() { - this.json = fs.readJSONSync(this.filePath); - return this; - } - - /** - * Walk through schema properties and an object's attributes calling an - * iterator function with the attributeName, attributeType and schema - * description along with the framework object, the current object attribute - * node and any other pass-through arguments. - * @param {string} schemaPath The attribute path from which to start in the schema - * @param {SchemaTraverseIterator} iterator - * @param {...any} args pass-through arguments - * @returns {Schema} - */ - traverse(schemaPath, iterator, ...args) { - let shouldStop = false; - const json = schemaPath ? _.get(this.json.properties, schemaPath) : this.json; - const recursiveSchemaNodeProperties = (properties, ...args) => { - let rtnValue = false; - // process properties - for (const attributeName in properties) { - let description = properties[attributeName]; - if (description.hasOwnProperty('editorOnly') || !description.hasOwnProperty('type')) { - // go to next attribute - continue; - } - description = { framework: this.framework, name: attributeName, ...description }; - // process current properties attribute - const returned = iterator({ - description, - next: (...args) => { - // continue with recursion if able - switch (description.type) { - case 'object': - return recursiveSchemaNodeProperties(description.properties, ...args); - case 'array': - if (description.items.type === 'object') { - return recursiveSchemaNodeProperties(description.items.properties, ...args); - } - const next = {}; - next[attributeName] = description.items; - return recursiveSchemaNodeProperties(next, ...args); - } - }, - stop: () => { - shouldStop = true; - } - }, ...args); - rtnValue = rtnValue || returned; - if (shouldStop) { - return; - } - } - }; - recursiveSchemaNodeProperties(json.properties, ...args); - return this; - } - - /** - * Applies schema defaults to the given object. - * @param {Object} output - * @param {string} schemaPath - * @param {Object} options - * @param {boolean} options.fillObjects Infer array or object default objects - * @returns {Schema} - */ - applyDefaults(output = {}, schemaPath, options = { fillObjects: true }) { - - function sortKeys(object) { - const keys = Object.keys(object).sort((a, b) => { - return a.localeCompare(b); - }); - keys.forEach(name => { - const value = object[name]; - delete object[name]; - object[name] = value; - }); - return object; - } - - this.traverse(schemaPath, ({ description, next }, output) => { - let hasChanged = false; - let haveChildenChanged = false; - let defaultValue; - - if (description === null || output === null) return; - - switch (description.type) { - case 'object': - defaultValue = description.hasOwnProperty('default') && options.fillObjects ? description.default : {}; - if (!output.hasOwnProperty(description.name)) { - output[description.name] = defaultValue; - hasChanged = true; - } - haveChildenChanged = next(output[description.name]); - if (haveChildenChanged) { - sortKeys(output[description.name]); - } - break; - case 'array': - defaultValue = description.hasOwnProperty('default') && options.fillObjects ? description.default : []; - if (!output.hasOwnProperty(description.name)) { - output[description.name] = defaultValue; - hasChanged = true; - } - haveChildenChanged = next(output[description.name]); - if (haveChildenChanged) { - sortKeys(output[description.name]); - } - break; - default: - defaultValue = description.default; - if (description.hasOwnProperty('default') && !output.hasOwnProperty(description.name)) { - output[description.name] = defaultValue; - hasChanged = true; - } - break; - } - return hasChanged; - }, output); - - return output; - } - -} - -/** - * @typedef SchemaNodeDescription - * @property {Framework} framework Schema properties node - * @property {string} name Attribute name - * @property {string} type Attribute type - * @property {boolean} editorOnly - */ - -/** - * @typedef SchemaTraverseIteratorParam0 - * @property {SchemaNodeDescription} description - * @property {function} next - * @property {function} stop - */ - -/** - * Iterator function for schema.traverse. - * @callback SchemaTraverseIterator - * @param {SchemaTraverseIteratorParam0} config - * @param {...any} args pass-through arguments - */ - -module.exports = Schema; diff --git a/package-lock.json b/package-lock.json index c5dc0d281..85599bb45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@types/backbone": "^1.4.14", "@types/jquery": "^3.5.16", "adapt-migrations": "^1.0.0", + "adapt-project": "^1.0.5", "async": "^3.2.2", "babel-plugin-transform-amd-to-es6": "^0.6.1", "babel-plugin-transform-react-templates": "^0.1.0", @@ -4358,6 +4359,24 @@ "node": ">= 10.0.0" } }, + "node_modules/adapt-project": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/adapt-project/-/adapt-project-1.0.6.tgz", + "integrity": "sha512-ena1I3X7/reLuSG8R1q98MBcVjz1noazxjWukQFnVneUCC4c+PpCOtwIc66jZjQsy7Wl3SKxJG6eOzjC6EQ+iw==", + "license": "GPL-3.0", + "dependencies": { + "async": "^3.2.2", + "chalk": "^2.4.1", + "csv": "^5.5.3", + "fast-xml-parser": "^4.5.0", + "fs-extra": "^8.1.0", + "globs": "^0.1.4", + "iconv-lite": "^0.6.3", + "jschardet": "^1.6.0", + "lodash": "^4.17.23", + "semver": "^7.6.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -19396,6 +19415,15 @@ } ] }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", diff --git a/package.json b/package.json index 33d6f3f65..6a534d29c 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@types/backbone": "^1.4.14", "@types/jquery": "^3.5.16", "adapt-migrations": "^1.0.0", + "adapt-project": "^1.0.5", "async": "^3.2.2", "babel-plugin-transform-amd-to-es6": "^0.6.1", "babel-plugin-transform-react-templates": "^0.1.0",