Merge pull request #1063 from marklogic/release/4.1.0

MLE-26608 Release MarkLogic Node Client API 4.1.0

Author: stevebio

.copyrightconfig

@@ -11,4 +11,4 @@ startyear: 2015
 #   - Dotfiles already skipped automatically
 # Enable by removing the leading '# ' from the next line and editing values.
 # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js
-filesexcluded: .github/*, README.md, Jenkinsfile, test-app/*, *.md, docker-compose.yaml, test-complete-app-mlDeploy/*, *.json
+filesexcluded: .github/*, README.md, Jenkinsfile, test-app/*, *.md, docker-compose.yaml, test-complete-app-mlDeploy/*, *.json, *.sh

.github/copilot-instructions.md

@@ -0,0 +1,10 @@
+When generating TypeScript definitions, follow this advice:
+
+All types will go into marklogic.d.ts.
+
+Determining the output type is difficult. You have to look at the module containing the 
+implementation code and look for an "outputTransform". The implementation of that function 
+should reveal what the user-facing function will return. 
+
+Please add "runtime" tests to the test-typescript directory. These tests should do 
+"smoke" tests that - critically - verify the output of each function. 

.gitignore

@@ -18,4 +18,7 @@ test-app/containerLogs
 test-complete-app/build
 test-complete-app/.gradle
 test-complete-app-mlDeploy/build
-test-complete-app-mlDeploy/.gradle
+test-complete-app-mlDeploy/.gradle
+
+# Compiled TypeScript test files
+test-typescript/*.js

CHANGELOG.md

@@ -1,10 +1,16 @@
 # CHANGELOG
 
+## 4.1.0
+
+- Added TypeScript typings for core client/document/connection APIs, with better autocomplete and compile-time checks to catch mistakes earlier.
+- Added support for transform parameters when writing transforms.
+- Performed dependency/security maintenance (including lodash and qs updates).
+
 ## 4.0.0
 
 This major release does not impact any client functionality, but rather raises the minimum required version of Node.js
 to 22 or higher. It also updates all dependencies to their latest version and removes unnecessary dependencies that
-were included in previous releases. 
+were included in previous releases.
 
 ## 3.7.1
 #### Bug Fix

CONTRIBUTING.md

@@ -58,6 +58,11 @@ or
 There are also tests in the `test-complete` folder. The setup for these is more complicated and can 
 be found in the `Jenkinsfile` file in this repository in the `runE2ETests` function.
 
+## Testing TypeScript support
+
+The 4.1.0 release will add TypeScript support for the client. To try this out locally and manually, 
+see the `README.md` file in the `./typescript-test-project` directory.
+
 ## Generating documentation
 
 After installing the project dependencies, you can build the reference documentation locally from the root
@@ -85,11 +90,11 @@ brace-expansion: "2.0.2"
 - Why needed: Prevents regex denial of service attacks
 - CVE/Issue: Related to minimatch vulnerabilities
 
-glob: "10.3.11"
-- Purpose: Fixes ReDoS and security issues in file globbing
-- Affects: mocha, gulp-mocha build tooling
-- Why needed: Older glob versions have pattern matching vulnerabilities
-- CVE/Issue: Multiple vulnerabilities in older glob versions
+glob: "12.0.0"
+- Purpose: Fixes command injection vulnerability in glob CLI
+- Affects: mocha, gulp-mocha, rimraf (via bunyan/mv)
+- Why needed: Versions 10.3.7-11.0.3 vulnerable to command injection via -c/--cmd flag
+- CVE/Issue: GHSA-5j98-mcp5-4vw2 - Command injection via shell:true execution
 
 glob-parent: "6.0.2"
 - Purpose: Fixes ReDoS in path parsing

Jenkinsfile

@@ -1,29 +1,29 @@
 @Library('shared-libraries') _
 
-def runTests() {
-  sh label: 'deploy-test-app-and-run-tests', script: '''
-		export JAVA_HOME=$JAVA_HOME_DIR
-		export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
-		export PATH=$JAVA_HOME/bin:${NODE_HOME_DIR}/bin:$PATH
+def runTests(excludeFragileTests) {
+  def excludeFlag = excludeFragileTests ? '--exclude "test-basic/documents-data-movement-*.js"' : ''
+
+  sh label: 'deploy-test-app-and-run-tests', script: """
+		export JAVA_HOME=\$JAVA_HOME_DIR
+		export GRADLE_USER_HOME=\$WORKSPACE/\$GRADLE_DIR
+		export PATH=\$JAVA_HOME/bin:\${NODE_HOME_DIR}/bin:\$PATH
 		cd node-client-api
 		node --version
 		npm --version
 		npm ci
 
     cd test-app
     ./gradlew -i mlWaitTillReady
-    sleep 3
-    ./gradlew -i mlWaitTillReady
     ./gradlew -i mlTestConnections
     ./gradlew -i mlDeploy
     ./gradlew -i -Penv=e2e mlLoadData mlLoadModules
 
 		cd ..
-		rm -rf $WORKSPACE/*.xml || true
-		./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic/ --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-reports.xml || true
+		rm -rf \$WORKSPACE/*.xml || true
+		./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic/ ${excludeFlag} --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/test-basic-reports.xml || true
 		./node_modules/.bin/gulp setupProxyTests || true
-		./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic-proxy/lib/**/*.js --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-proxy-reports.xml || true
-	'''
+		./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic-proxy/lib/**/*.js --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/test-basic-proxy-reports.xml || true
+	"""
 	junit '**/*.xml'
 }
 
@@ -55,7 +55,7 @@ def runAuditReport() {
 		cd node-client-api
 		npm ci
 		rm -rf $WORKSPACE/npm-audit-report.json || true
-		npm audit --audit-level=low --json > $WORKSPACE/npm-audit-report.json
+		npm audit --audit-level=moderate --json > $WORKSPACE/npm-audit-report.json
 	'''
 }
 
@@ -69,16 +69,38 @@ def runLint() {
 	'''
 }
 
-def runE2ETests() {
-  sh label: 'run-e2e-tests', script: '''
+def runTypeCheck() {
+  sh label: 'run-type-check', script: '''
+    export PATH=${NODE_HOME_DIR}/bin:$PATH
+    cd node-client-api
+    npm ci
+    npm run test:types
+	'''
+}
+
+def runTypeScriptTests() {
+  sh label: 'run-typescript-tests', script: '''
     export PATH=${NODE_HOME_DIR}/bin:$PATH
+    cd node-client-api
+    npm ci
+    npm run test:compile
+    ./node_modules/.bin/mocha --timeout 10000 test-typescript/*.js --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-typescript-reports.xml || true
+	'''
+  junit '**/*test-typescript-reports.xml'
+}
+
+def runE2ETests(excludeFragileTests) {
+  def excludeFlag = excludeFragileTests ? '--exclude "test-complete/nodejs-dmsdk*.js"' : ''
+
+  sh label: 'run-e2e-tests', script: """
+    export PATH=\${NODE_HOME_DIR}/bin:\$PATH
 		cd node-client-api
 		node --version
 		npm --version
 		npm ci
 
     echo "Running test-complete tests"
-    ./node_modules/.bin/mocha --no-parallel -R xunit --timeout 60000  test-complete/ --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-complete-results.xml  || true
+    ./node_modules/.bin/mocha --no-parallel -R xunit --timeout 60000  test-complete/ ${excludeFlag} --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/test-complete-results.xml  || true
     echo "Done with test-complete tests"
 
     cd test-complete-proxy
@@ -90,12 +112,12 @@ def runE2ETests() {
 		cp -R ml-modules/ ../test-complete
 		cd ../test-complete
 		../node_modules/.bin/mocha -R xunit --timeout 20000 nodejs-ds-setup-docs.js
-		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-required-params.js"  --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-required-params-results.xml || true
-		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-error-map.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-multipleWorker-results.xml || true
-		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-multipleWorker.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-multipleWorker-results.xml || true
-		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-transactions.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-transactions-results.js.xml || true
-		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-dynamic.js" --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/ds-dynamic-results.xml || true
-	'''
+		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-required-params.js"  --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/ds-required-params-results.xml || true
+		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-error-map.js" --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/ds-multipleWorker-results.xml || true
+		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-multipleWorker.js" --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/ds-multipleWorker-results.xml || true
+		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-transactions.js" --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/ds-transactions-results.js.xml || true
+		../node_modules/.bin/mocha -R xunit --timeout 20000 "nodejs-ds-dynamic.js" --reporter mocha-junit-reporter --reporter-options mochaFile=\$WORKSPACE/ds-dynamic-results.xml || true
+	"""
   junit '**/*.xml'
 }
 
@@ -108,6 +130,7 @@ pipeline {
 
   parameters {
     booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions')
+    string(name: 'MARKLOGIC_IMAGE_TAGS', defaultValue: 'marklogic-server-ubi:latest-11,marklogic-server-ubi:latest-12', description: 'Comma-delimited list of MarkLogic image tags including variant (e.g., marklogic-server-ubi:latest-11,marklogic-server-ubi-rootless:11.3.2). The registry/org (ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic) path will be prepended automatically.')
   }
 
   options {
@@ -130,9 +153,11 @@ pipeline {
       steps {
         runAuditReport()
         runLint()
+        runTypeCheck()
         runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12')
-        runTests()
-        runE2ETests()
+        runTests(true)
+        runTypeScriptTests()
+        runE2ETests(true)
       }
       post {
         always {
@@ -142,63 +167,27 @@ pipeline {
     }
 
     stage('regressions') {
-      parallel {
-
-        stage('runtests-11-nightly') {
-          when {
-            allOf {
-              branch 'develop'
-              expression { return params.regressions }
-            }
-          }
-          agent { label 'nodeclientpool' }
-          steps {
-            runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11')
-            runTests()
-            runE2ETests()
-          }
-          post {
-            always {
-              teardownAfterTests()
-            }
-          }
-        }
-
-        stage('runtests-12-nightly') {
-          when {
-            allOf {
-              branch 'develop'
-              expression { return params.regressions }
-            }
-          }
-          agent { label 'nodeclientpool' }
-          steps {
-            runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12')
-            runTests()
-            runE2ETests()
-          }
-          post {
-            always {
-              teardownAfterTests()
-            }
-          }
+      agent { label 'nodeclientpool' }
+      when {
+        allOf {
+          branch 'develop'
+          expression { return params.regressions }
         }
-
-        stage('runtests-10-nightly') {
-          when {
-            allOf {
-              branch 'develop'
-              expression { return params.regressions }
-            }
-          }
-          agent { label 'nodeclientpool' }
-          steps {
-            runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10')
-            runTests()
-            runE2ETests()
-          }
-          post {
-            always {
+      }
+      steps {
+        script {
+          def imageTags = params.MARKLOGIC_IMAGE_TAGS.split(',')
+          def imagePrefix = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/'
+
+          imageTags.each { tag ->
+            def fullImage = imagePrefix + tag.trim()
+
+            try {
+              runDockerCompose(fullImage)
+              runTests(false)
+              runTypeScriptTests()
+              runE2ETests(false)
+            } finally {
               teardownAfterTests()
             }
           }