From 26aa32dba1e0979ba35c122716d0e6ea4d739067 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 11:57:48 +0100 Subject: [PATCH 01/12] moved lint check into its own CI task --- .github/workflows/ci.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b831934e8..69630dd9ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,26 @@ jobs: run: npm ci - name: CI Self-Check run: npm run ci:check + check-lint: + name: Lint + timeout-minutes: 30 + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.NODE_VERSION }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Cache Node.js modules + uses: actions/cache@v2 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ matrix.NODE_VERSION }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ matrix.NODE_VERSION }}- + - name: Install dependencies + run: npm ci + - run: npm run lint check-mongo: strategy: matrix: @@ -106,8 +126,6 @@ jobs: ${{ runner.os }}-node-${{ matrix.NODE_VERSION }}- - name: Install dependencies run: npm ci - - if: ${{ matrix.name == 'Mongo 3.6.21' }} - run: npm run lint - run: npm run pretest - run: npm run coverage env: From 2608a84b405850474f8b0fd371a738bda1025af5 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 12:16:48 +0100 Subject: [PATCH 02/12] added PR creation --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69630dd9ff..0c6391556e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,13 @@ jobs: run: npm ci - name: CI Self-Check run: npm run ci:check + - name: Create PR to upgrade CI environments if needed + uses: peter-evans/create-pull-request@v3 + with: + title: Upgrade CI environments + body: This is an automated PR to upgrade package versions in CI environments for MongoDB, Node.js. + commit-message: Upgrading versions of MongoDB / Node.js + branch: ci/upgrade-ci-environments check-lint: name: Lint timeout-minutes: 30 @@ -99,6 +106,7 @@ jobs: NODE_VERSION: 15.8.0 name: ${{ matrix.name }} timeout-minutes: 30 + needs: check-ci runs-on: ubuntu-18.04 services: redis: @@ -147,6 +155,7 @@ jobs: POSTGRES_IMAGE: postgis/postgis:13-3.1 name: ${{ matrix.name }} timeout-minutes: 30 + needs: check-ci runs-on: ubuntu-18.04 services: redis: From bd5bf49e8e2dbe7657cee3c5d8129a42342bffc2 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 12:27:33 +0100 Subject: [PATCH 03/12] added PR base branch --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c6391556e..9d19f59bf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: body: This is an automated PR to upgrade package versions in CI environments for MongoDB, Node.js. commit-message: Upgrading versions of MongoDB / Node.js branch: ci/upgrade-ci-environments + base: master check-lint: name: Lint timeout-minutes: 30 From 03e24c09ff77c1bebcade7370980ffa33b5c1a96 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 12:37:53 +0100 Subject: [PATCH 04/12] refactored version component range operator --- resources/ci/CiVersionCheck.js | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/resources/ci/CiVersionCheck.js b/resources/ci/CiVersionCheck.js index 763d907355..0d2e026cd0 100644 --- a/resources/ci/CiVersionCheck.js +++ b/resources/ci/CiVersionCheck.js @@ -144,11 +144,7 @@ class CiVersionCheck { // ]; // Determine operator for range comparison - const operator = versionComponent == CiVersionCheck.versionComponents.major - ? '>=' - : versionComponent == CiVersionCheck.versionComponents.minor - ? '^' - : '~' + const operator = this._getOperatorForVersionComponent(versionComponent); // Get all untested versions const untestedVersions = releasedVersions.reduce((m, v) => { @@ -187,11 +183,7 @@ class CiVersionCheck { */ getNewerVersion(versions, version, versionComponent) { // Determine operator for range comparison - const operator = versionComponent == CiVersionCheck.versionComponents.major - ? '>=' - : versionComponent == CiVersionCheck.versionComponents.minor - ? '^' - : '~' + const operator = this._getOperatorForVersionComponent(versionComponent); const latest = semver.maxSatisfying(versions, `${operator}${version}`); return semver.gt(latest, version) ? latest : undefined; } @@ -209,6 +201,23 @@ class CiVersionCheck { } } + /** + * Returns the semver range operator that relates to the a version component. + * For example, the operator for the `patch` version component defines a + * range up to the highest patch version. + * @param {String} component The version component (`patch`, `minor`, + * `major`). + * @returns {String} The semver operator. + */ + _getOperatorForVersionComponent(component) { + // Determine operator for range comparison + return component == CiVersionCheck.versionComponents.major + ? '>=' + : component == CiVersionCheck.versionComponents.minor + ? '^' + : '~' + } + /** * Runs the check. */ From db226c3c50a520208819a084bd9ee5ba3ccc9814 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 14:32:13 +0100 Subject: [PATCH 05/12] added YAML update --- .github/workflows/ci.yml | 10 ++--- package-lock.json | 14 +++++-- package.json | 2 +- resources/ci/CiVersionCheck.js | 69 +++++++++++++++++++++++++++++++++- resources/ci/ciCheck.js | 2 + 5 files changed, 87 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d19f59bf4..6e24e7365b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: matrix: include: - name: Mongo 4.4, ReplicaSet, WiredTiger - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: replicaset MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 14.15.5 @@ -86,22 +86,22 @@ jobs: NODE_VERSION: 14.15.5 - name: Redis Cache PARSE_SERVER_TEST_CACHE: redis - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 14.15.5 - name: Node 10 - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 10.23.3 - name: Node 12 - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 12.20.2 - name: Node 15 - MONGODB_VERSION: 4.4.4 + MONGODB_VERSION: 4.4.3 MONGODB_TOPOLOGY: standalone MONGODB_STORAGE_ENGINE: wiredTiger NODE_VERSION: 15.8.0 diff --git a/package-lock.json b/package-lock.json index 01a2bd7794..be1db21b9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3997,6 +3997,14 @@ "parse-json": "^5.0.0", "path-type": "^4.0.0", "yaml": "^1.7.2" + }, + "dependencies": { + "yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "dev": true + } } }, "cross-env": { @@ -11915,9 +11923,9 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" }, "yaml": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", - "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "version": "2.0.0-3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.0.0-3.tgz", + "integrity": "sha512-gvtVaY+/mQ0OsXgaWy2Tf830JuXN7qEUYdXWsuiJVSkMRsBBQ90YVpQQofaURbhoA1xSbLBf7965oH6ddzNbBQ==", "dev": true }, "yargs": { diff --git a/package.json b/package.json index 753795d9ee..21480593a4 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "node-fetch": "2.6.1", "nyc": "15.1.0", "prettier": "2.0.5", - "yaml": "1.10.0" + "yaml": "2.0.0-3" }, "scripts": { "ci:check": "node ./resources/ci/ciCheck.js", diff --git a/resources/ci/CiVersionCheck.js b/resources/ci/CiVersionCheck.js index 0d2e026cd0..9e51a706dd 100644 --- a/resources/ci/CiVersionCheck.js +++ b/resources/ci/CiVersionCheck.js @@ -1,6 +1,7 @@ const core = require('@actions/core'); const semver = require('semver'); const yaml = require('yaml'); +const lodash = require('lodash'); const fs = require('fs').promises; /** @@ -43,6 +44,8 @@ class CiVersionCheck { * test against 2.0.0. * If the latest version component is `major` then the check would * fail and recommend an upgrade to version 2.0.0. + * @param {Boolean} [config.updateYaml=false] Is true if the YAML file + * should be updated and package versions should be bumped. */ constructor(config) { const { @@ -54,6 +57,7 @@ class CiVersionCheck { releasedVersions, ignoreReleasedVersions = [], latestComponent = CiVersionCheck.versionComponents.patch, + updateYaml = false, } = config; // Ensure required params are set @@ -64,6 +68,9 @@ class CiVersionCheck { ciEnvironmentsKeyPath, ciVersionKey, releasedVersions, + ignoreReleasedVersions, + latestComponent, + updateYaml, ].includes(undefined)) { throw 'invalid configuration'; } @@ -80,6 +87,7 @@ class CiVersionCheck { this.releasedVersions = releasedVersions; this.ignoreReleasedVersions = ignoreReleasedVersions; this.latestComponent = latestComponent; + this.updateYaml = updateYaml; } /** @@ -101,9 +109,10 @@ class CiVersionCheck { // Get CI workflow const ciYaml = await fs.readFile(this.yamlFilePath, 'utf-8'); const ci = yaml.parse(ciYaml); + this.ci = ci; // Extract package versions - let versions = this.ciEnvironmentsKeyPath.split('.').reduce((o,k) => o !== undefined ? o[k] : undefined, ci); + let versions = this._getKeyAtPath(ci, this.ciEnvironmentsKeyPath); versions = Object.entries(versions) .map(entry => entry[1]) .filter(entry => entry[this.ciVersionKey]); @@ -114,6 +123,16 @@ class CiVersionCheck { } } + /** + * Returns the value at a given key path. + * @param {Object} obj The object to traverse. + * @param {String} keyPath The path to the key to return. + * @returns {Any} The value at the given key path. + */ + _getKeyAtPath(obj, keyPath) { + return keyPath.split('.').reduce((o,k) => o !== undefined ? o[k] : undefined, obj); + } + /** * Returns the package versions which are missing in the CI environment. * @param {Array} releasedVersions The released versions; need to @@ -218,6 +237,40 @@ class CiVersionCheck { : '~' } + /** + * Updates a key in the CI YAML file with a given value. + * @param {Array} update The key to update. + * @param {String} update.old The old value to update. + * @param {String} update.new The new value to set. + */ + async _updateYaml(update) { + if (this.ci === undefined) { + throw 'YAML file has not been read.'; + } + + // Get environments + const newCi = lodash.cloneDeep(this.ci); + const envs = this._getKeyAtPath(newCi, this.ciEnvironmentsKeyPath); + + // Update keys + for (const key of Object.keys(update)) { + for (const oldValue of Object.keys(update[key])) { + const newValue = update[key][oldValue]; + + // Update value + for (const env of envs) { + if (env[key] == oldValue) { + env[key] = newValue; + } + } + } + } + + // Write to file + const newYaml = yaml.stringify(newCi); + await fs.writeFile(this.yamlFilePath, newYaml); + } + /** * Runs the check. */ @@ -242,6 +295,9 @@ class CiVersionCheck { // Is true if any of the checks failed let failed = false; + // The keys that should be updated + const keyUpdatesNeeded = {}; + // Check whether each tested version is the latest patch for (const test of tests) { const version = test[this.ciVersionKey]; @@ -258,6 +314,7 @@ class CiVersionCheck { if (newer) { console.log(`❌ CI environment '${test.name}' uses an old ${this.packageName} ${this.latestComponent} version ${version} instead of ${newer}.`); failed = true; + keyUpdatesNeeded[this.ciVersionKey] = Object.assign(keyUpdatesNeeded[this.ciVersionKey] || {}, { [version]: newer }); } else { console.log(`✅ CI environment '${test.name}' uses the latest ${this.packageName} ${this.latestComponent} version ${version}.`); } @@ -280,6 +337,16 @@ class CiVersionCheck { `file.\n\nℹ️ Additionally, there may be versions of ${this.packageName} that have reached their official end-of-life ` + `support date and should be removed from the CI, see ${this.packageSupportUrl}.` ); + + // If packages in YAML file should be updated + if (this.updateYaml && Object.keys(keyUpdatesNeeded).length > 0) { + try { + this._updateYaml(keyUpdatesNeeded); + } catch (e) { + console.log(`Failed to update ${this.packageName} versions in YAML file with error: ${e}`); + } + console.log(`\n🚀 Updated YAML file to use newer versions of ${this.packageName}. Check the Pull Request list for a PR.`); + } } } catch (e) { diff --git a/resources/ci/ciCheck.js b/resources/ci/ciCheck.js index 66ee6d9aa0..e855e69742 100644 --- a/resources/ci/ciCheck.js +++ b/resources/ci/ciCheck.js @@ -32,6 +32,7 @@ async function checkMongoDbVersions() { ciVersionKey: 'MONGODB_VERSION', releasedVersions, latestComponent: CiVersionCheck.versionComponents.path, + updateYaml: true, ignoreReleasedVersions: [ '<3.6.0', // These versions have reached their MongoDB end-of-life support date '~3.7.0', // This is a development release according to MongoDB support @@ -58,6 +59,7 @@ async function checkNodeVersions() { ciVersionKey: 'NODE_VERSION', releasedVersions, latestComponent: CiVersionCheck.versionComponents.minor, + updateYaml: true, ignoreReleasedVersions: [ '<10.0.0', // These versions have reached their end-of-life support date '>=11.0.0 <12.0.0', // These versions have reached their end-of-life support date From df42c6732f4081de34ed79d1dcf2aff985bb5371 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 14:55:23 +0100 Subject: [PATCH 06/12] added conditoinal PR creation --- resources/ci/CiVersionCheck.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/ci/CiVersionCheck.js b/resources/ci/CiVersionCheck.js index 9e51a706dd..0ddfa813ed 100644 --- a/resources/ci/CiVersionCheck.js +++ b/resources/ci/CiVersionCheck.js @@ -334,8 +334,8 @@ class CiVersionCheck { core.setFailed( `CI environments are not up-to-date with the latest ${this.packageName} versions.` + `\n\nCheck the error messages above and update the ${this.packageName} versions in the CI YAML ` + - `file.\n\nℹ️ Additionally, there may be versions of ${this.packageName} that have reached their official end-of-life ` + - `support date and should be removed from the CI, see ${this.packageSupportUrl}.` + `file. Additionally, check for versions of ${this.packageName} that have reached their official end-of-life ` + + `support date and may be removed from the CI, see ${this.packageSupportUrl}.` ); // If packages in YAML file should be updated From 06036734921fa2a8304cb8a3f95c5e39c21a9011 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 15:58:40 +0100 Subject: [PATCH 07/12] fix PR creation --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e24e7365b..fc073e44c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ jobs: - name: CI Self-Check run: npm run ci:check - name: Create PR to upgrade CI environments if needed + if: ${{ always() }} uses: peter-evans/create-pull-request@v3 with: title: Upgrade CI environments From 4ece4b5e9ee17f2a66db1a0d1c0a772efbfc2a31 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:28:31 +0100 Subject: [PATCH 08/12] fix PR creation --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc073e44c8..06282440f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,15 +31,16 @@ jobs: - name: Install dependencies run: npm ci - name: CI Self-Check + id: ci-check run: npm run ci:check - name: Create PR to upgrade CI environments if needed - if: ${{ always() }} + if: steps.ci-check.conclusion == 'failure' uses: peter-evans/create-pull-request@v3 with: title: Upgrade CI environments body: This is an automated PR to upgrade package versions in CI environments for MongoDB, Node.js. commit-message: Upgrading versions of MongoDB / Node.js - branch: ci/upgrade-ci-environments + branch: upgrade-ci-environments base: master check-lint: name: Lint From 662c5979034c6667ef3b737384e35cc3d3c20e5e Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:30:19 +0100 Subject: [PATCH 09/12] fix PR creation --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06282440f9..f4661f3de2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,17 +31,16 @@ jobs: - name: Install dependencies run: npm ci - name: CI Self-Check - id: ci-check run: npm run ci:check - name: Create PR to upgrade CI environments if needed - if: steps.ci-check.conclusion == 'failure' + if: ${{ always() }} uses: peter-evans/create-pull-request@v3 with: title: Upgrade CI environments body: This is an automated PR to upgrade package versions in CI environments for MongoDB, Node.js. commit-message: Upgrading versions of MongoDB / Node.js branch: upgrade-ci-environments - base: master + # base: master check-lint: name: Lint timeout-minutes: 30 From 3fdf7158a03d98be3501a7c679bd18c8ae128f54 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:32:40 +0100 Subject: [PATCH 10/12] fix PR creation --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4661f3de2..52618d6474 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: body: This is an automated PR to upgrade package versions in CI environments for MongoDB, Node.js. commit-message: Upgrading versions of MongoDB / Node.js branch: upgrade-ci-environments - # base: master + base: master check-lint: name: Lint timeout-minutes: 30 From 0d83e0be6ee58880fd3644a3130ab9b970873a68 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:51:38 +0100 Subject: [PATCH 11/12] fix PR creation --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52618d6474..0ba1072d36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,12 +32,13 @@ jobs: run: npm ci - name: CI Self-Check run: npm run ci:check + continue-on-error: true - name: Create PR to upgrade CI environments if needed if: ${{ always() }} uses: peter-evans/create-pull-request@v3 with: title: Upgrade CI environments - body: This is an automated PR to upgrade package versions in CI environments for MongoDB, Node.js. + body: This is an automated PR to upgrade package versions in CI environments. commit-message: Upgrading versions of MongoDB / Node.js branch: upgrade-ci-environments base: master From b19db818a3acaa870b27c8fb56653100fab03132 Mon Sep 17 00:00:00 2001 From: Manuel Trezza <5673677+mtrezza@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:53:59 +0100 Subject: [PATCH 12/12] fix PR creation --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ba1072d36..4fdfe46b03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,6 @@ jobs: run: npm ci - name: CI Self-Check run: npm run ci:check - continue-on-error: true - name: Create PR to upgrade CI environments if needed if: ${{ always() }} uses: peter-evans/create-pull-request@v3