diff --git a/.github/workflows/node-test.yml b/.github/workflows/node-test.yml new file mode 100644 index 000000000..883b18345 --- /dev/null +++ b/.github/workflows/node-test.yml @@ -0,0 +1,49 @@ +name: Node Test + +on: + pull_request: + types: [opened, synchronize] + branches: [main] + paths: + - "**/*.mjs" + - "**/*.js" + push: + branches: [main] + paths: + - "**/*.mjs" + - "**/*.js" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Run node tests + run: | + if [ "${{ github.event_name }}" = "push" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + npx c8 --reporter=lcov --report-dir=coverage node --test **/*.test.mjs + else + node --test **/*.test.mjs + fi + + - name: Upload coverage artifact + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: js-coverage + path: coverage/lcov.info diff --git a/infrastructure/setup-vpc-nat-efs.yml b/infrastructure/setup-vpc-nat-efs.yml index 052bd70c8..88b28ac70 100644 --- a/infrastructure/setup-vpc-nat-efs.yml +++ b/infrastructure/setup-vpc-nat-efs.yml @@ -442,21 +442,11 @@ Resources: export NPM_TOKEN="$NPM_TOKEN" fi $PKG_MANAGER install - # Pre-download MongoDB binary to EFS cache for MongoMemoryServer. - # Without this, MongoMemoryServer downloads ~100MB at test runtime, - # eating into Jest's 600s timeout on Lambda. By caching the binary on EFS - # during CodeBuild install, tests start instantly. - if node -e "require.resolve('mongodb-memory-server-core')" 2>/dev/null; then - echo "Pre-downloading MongoDB binary for MongoMemoryServer..." - export MONGOMS_DOWNLOAD_DIR="$EFS_DIR/.cache/mongodb-binaries" - node -e " - const pkg = require('./package.json'); - const v = (pkg.config && pkg.config.mongodbMemoryServer && pkg.config.mongodbMemoryServer.version) || ''; - if (v && parseInt(v.split('.')[0]) < 7) process.env.MONGOMS_DISTRO = 'ubuntu-22.04'; - require('mongodb-memory-server-core').MongoBinary.getPath() - .then(p => console.log('MongoDB binary cached at:', p)) - .catch(e => console.error('MongoDB download failed:', e.message)); - " || true + # Pre-download MongoDB binary to EFS cache if repo uses MongoMemoryServer. + # Script is copied to EFS by Lambda before starting CodeBuild. + MONGOD_SCRIPT="/mnt/efs/.scripts/download_mongod_binary.mjs" + if [ -f "$MONGOD_SCRIPT" ] && node -e "require.resolve('mongodb-memory-server-core')" 2>/dev/null; then + MONGOMS_DOWNLOAD_DIR="$EFS_DIR/.cache/mongodb-binaries" node "$MONGOD_SCRIPT" || true fi rm -f "$EFS_DIR/node_modules.tar.gz" tar -czf "$EFS_DIR/node_modules.tar.gz" -C "$LOCAL_DIR" node_modules diff --git a/services/aws/download_mongod_binary.mjs b/services/aws/download_mongod_binary.mjs new file mode 100644 index 000000000..2e788fc11 --- /dev/null +++ b/services/aws/download_mongod_binary.mjs @@ -0,0 +1,59 @@ +/** + * Pre-downloads the MongoDB binary for MongoMemoryServer so it's cached on EFS. + * Without this, MongoMemoryServer downloads ~100MB at test runtime, eating into + * Jest's 600s timeout on Lambda. + * + * Called by CodeBuild after `yarn/npm install`, with these env vars: + * MONGOMS_DOWNLOAD_DIR - EFS path to cache the binary (e.g. /mnt/efs/owner/repo/.cache/mongodb-binaries) + * + * Lambda runs on Amazon Linux 2023. MongoDB < 7.0 has no amazon2023 binary, + * so we override distro to ubuntu-22.04 (glibc-compatible). + * MongoDB 7.0+ has native amazon2023 binaries, so auto-detection works. + * + * Version detection priority: + * 1. config.mongodbMemoryServer.version in package.json + * 2. MONGOMS_VERSION in the test script command (e.g. "cross-env MONGOMS_VERSION=v7.0-latest jest") + * 3. Library default (if neither is set) + */ + +import { readFileSync } from "fs"; +import { join } from "path"; + +const pkg = JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8")); + +// Check config.mongodbMemoryServer.version first (e.g. {"config": {"mongodbMemoryServer": {"version": "6.0.14"}}}) +let version = pkg.config?.mongodbMemoryServer?.version; + +// Fall back to extracting MONGOMS_VERSION from the test script command +// e.g. "test": "cross-env MONGOMS_VERSION=v7.0-latest jest --coverage" +if (!version && pkg.scripts) { + for (const script of Object.values(pkg.scripts)) { + const match = script.match(/MONGOMS_VERSION=(\S+)/); + if (match) { + version = match[1]; + break; + } + } +} + +console.log("MongoMemoryServer version:", version || "not specified (using library default)"); + +if (version) { + process.env.MONGOMS_VERSION = version; + // Strip leading "v" for major version comparison (e.g. "v7.0-latest" -> "7") + const major = parseInt(version.replace(/^v/, "").split(".")[0], 10); + if (major < 7) { + process.env.MONGOMS_DISTRO = "ubuntu-22.04"; + console.log("Overriding distro to ubuntu-22.04 for MongoDB <7.0 Lambda compatibility"); + } +} + +// Dynamic import so version detection runs first (and is testable without this package installed) +const { MongoBinary } = await import("mongodb-memory-server-core"); + +MongoBinary.getPath() + .then((p) => console.log("MongoDB binary cached at:", p)) + .catch((e) => { + console.error("MongoDB binary download failed:", e.message); + process.exit(1); + }); diff --git a/services/aws/download_mongod_binary.test.mjs b/services/aws/download_mongod_binary.test.mjs new file mode 100644 index 000000000..e3d1f8360 --- /dev/null +++ b/services/aws/download_mongod_binary.test.mjs @@ -0,0 +1,148 @@ +/** + * Tests for download_mongod_binary.mjs version detection logic. + * Run with: node --test services/aws/download_mongod_binary.test.mjs + * + * These tests verify version extraction from real customer package.json configs + * WITHOUT requiring mongodb-memory-server-core to be installed. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { writeFileSync, mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { execFileSync } from "child_process"; + +/** + * Runs the download script against a mock package.json in a temp dir. + * The script will fail on the dynamic import("mongodb-memory-server-core") since it's + * not installed locally, but version detection runs before that and is visible in output. + */ +function runWithPackageJson(pkg) { + const tmpDir = join("/tmp", `mongod-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); + writeFileSync(join(tmpDir, "package.json"), JSON.stringify(pkg)); + + try { + const output = execFileSync( + "node", + [join(process.cwd(), "services/aws/download_mongod_binary.mjs")], + { cwd: tmpDir, encoding: "utf8", env: { ...process.env, MONGOMS_DOWNLOAD_DIR: "/tmp/mongod-test-cache" }, timeout: 5000 }, + ); + return { output, exitCode: 0 }; + } catch (e) { + return { output: (e.stdout || "") + (e.stderr || ""), exitCode: e.status }; + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +} + +describe("download_mongod_binary", () => { + // Real case: Foxquilt/foxcom-payment-backend + // - mongodb-memory-server ^7.4.0 in dependencies + // - MONGOMS_VERSION=v7.0-latest via cross-env in test script + it("foxcom-payment-backend: cross-env MONGOMS_VERSION in test script", () => { + const { output } = runWithPackageJson({ + name: "foxcom-payment-backend", + scripts: { + test: 'cross-env MONGOMS_VERSION=v7.0-latest jest --coverage --coverageReporters="lcov"', + }, + dependencies: { "mongodb-memory-server": "^7.4.0" }, + }); + assert.match(output, /MongoMemoryServer version: v7\.0-latest/); + assert.doesNotMatch(output, /ubuntu-22\.04/, "v7.0 should NOT override distro"); + }); + + // Real case: Foxquilt/foxcom-forms-backend + // - mongodb-memory-server 7.4.0 (exact, not caret) in dependencies + // - MONGOMS_VERSION=v7.0-latest directly (no cross-env prefix) + it("foxcom-forms-backend: MONGOMS_VERSION without cross-env", () => { + const { output } = runWithPackageJson({ + name: "foxcom-forms-backend", + scripts: { + test: "MONGOMS_VERSION=v7.0-latest NODE_OPTIONS='--max-old-space-size=6144' jest", + }, + dependencies: { "mongodb-memory-server": "7.4.0" }, + }); + assert.match(output, /MongoMemoryServer version: v7\.0-latest/); + assert.doesNotMatch(output, /ubuntu-22\.04/); + }); + + // Real case: Foxquilt/foxden-rating-quoting-backend + // - mongodb-memory-server 9.1.5 in dependencies + // - TZ=UTC before MONGOMS_VERSION in test script + it("foxden-rating-quoting-backend: TZ=UTC before MONGOMS_VERSION", () => { + const { output } = runWithPackageJson({ + name: "foxden-rating-quoting-backend", + scripts: { + test: "cross-env TZ=UTC MONGOMS_VERSION=v7.0-latest NODE_OPTIONS='--max-old-space-size=6144' jest", + }, + dependencies: { "mongodb-memory-server": "9.1.5" }, + }); + assert.match(output, /MongoMemoryServer version: v7\.0-latest/); + assert.doesNotMatch(output, /ubuntu-22\.04/); + }); + + // Real case: Foxquilt/foxden-tools + // - mongodb-memory-server ^9.1.7 in dependencies + // - yarn build && before cross-env MONGOMS_VERSION in test script + it("foxden-tools: yarn build && before MONGOMS_VERSION", () => { + const { output } = runWithPackageJson({ + name: "foxden-tools", + scripts: { + test: "yarn build && cross-env MONGOMS_VERSION=v7.0-latest jest", + }, + dependencies: { "mongodb-memory-server": "^9.1.7" }, + }); + assert.match(output, /MongoMemoryServer version: v7\.0-latest/); + assert.doesNotMatch(output, /ubuntu-22\.04/); + }); + + // Real case: Foxquilt/foxden-billing + // - mongodb-memory-server ^10.0.0 in dependencies + // - NO MONGOMS_VERSION in any script, no config section + it("foxden-billing: no version specified anywhere", () => { + const { output } = runWithPackageJson({ + name: "foxden-billing", + scripts: { test: "jest" }, + dependencies: { "mongodb-memory-server": "^10.0.0" }, + }); + assert.match(output, /not specified \(using library default\)/); + assert.doesNotMatch(output, /ubuntu-22\.04/); + }); + + // Real case: Foxquilt/foxden-shared-lib + // - mongodb-memory-server ^9.3.0 in dependencies + // - NO MONGOMS_VERSION in any script, no config section + it("foxden-shared-lib: no version specified anywhere", () => { + const { output } = runWithPackageJson({ + name: "foxden-shared-lib", + dependencies: { "mongodb-memory-server": "^9.3.0" }, + }); + assert.match(output, /not specified \(using library default\)/); + assert.doesNotMatch(output, /ubuntu-22\.04/); + }); + + // Hypothetical case: older MongoDB version in config section + // - config.mongodbMemoryServer.version = "6.0.14" + // - Should override distro to ubuntu-22.04 for Lambda compatibility + it("config section with MongoDB <7 overrides distro", () => { + const { output } = runWithPackageJson({ + name: "test-repo", + config: { mongodbMemoryServer: { version: "6.0.14" } }, + dependencies: { "mongodb-memory-server": "^9.0.0" }, + }); + assert.match(output, /MongoMemoryServer version: 6\.0\.14/); + assert.match(output, /ubuntu-22\.04/); + }); + + // config.mongodbMemoryServer.version takes priority over script env var + it("config section takes priority over test script env var", () => { + const { output } = runWithPackageJson({ + name: "test-repo", + config: { mongodbMemoryServer: { version: "6.0.14" } }, + scripts: { test: "cross-env MONGOMS_VERSION=v7.0-latest jest" }, + dependencies: { "mongodb-memory-server": "^9.0.0" }, + }); + assert.match(output, /MongoMemoryServer version: 6\.0\.14/); + }); +}); diff --git a/services/aws/run_install_via_codebuild.py b/services/aws/run_install_via_codebuild.py index aaea944a2..1b8d3df62 100644 --- a/services/aws/run_install_via_codebuild.py +++ b/services/aws/run_install_via_codebuild.py @@ -1,11 +1,23 @@ +import os +import shutil + from mypy_boto3_codebuild.type_defs import EnvironmentVariableTypeDef from constants.general import IS_PRD from services.aws.clients import codebuild_client +from services.node.detect_package_manager import PACKAGE_MANAGER_TO_LOCK_FILE from services.supabase.npm_tokens.get_npm_token import get_npm_token from utils.error.handle_exceptions import handle_exceptions from utils.logging.logging_config import logger +# Path to the JS script inside the Lambda container (copied from our repo via Dockerfile) +_MONGOD_SCRIPT_SRC = os.path.join( + os.path.dirname(__file__), "download_mongod_binary.mjs" +) + +# Shared location on EFS where CodeBuild can find the script +_MONGOD_SCRIPT_EFS = "/mnt/efs/.scripts/download_mongod_binary.mjs" + @handle_exceptions(default_return_value=None, raise_on_error=False) def run_install_via_codebuild( @@ -17,6 +29,16 @@ def run_install_via_codebuild( logger.info("codebuild: Skipping in non-prod environment") return None + # Copy the mongod download script to EFS so CodeBuild can run it + if pkg_manager in PACKAGE_MANAGER_TO_LOCK_FILE and os.path.isfile( + _MONGOD_SCRIPT_SRC + ): + os.makedirs(os.path.dirname(_MONGOD_SCRIPT_EFS), exist_ok=True) + shutil.copy2(_MONGOD_SCRIPT_SRC, _MONGOD_SCRIPT_EFS) + logger.info( + "codebuild: Copied mongod download script to %s", _MONGOD_SCRIPT_EFS + ) + env_overrides: list[EnvironmentVariableTypeDef] = [ {"name": "EFS_DIR", "value": efs_dir, "type": "PLAINTEXT"}, {"name": "PKG_MANAGER", "value": pkg_manager, "type": "PLAINTEXT"}, diff --git a/services/aws/test_run_install_via_codebuild.py b/services/aws/test_run_install_via_codebuild.py index c66023099..37bf929af 100644 --- a/services/aws/test_run_install_via_codebuild.py +++ b/services/aws/test_run_install_via_codebuild.py @@ -1,6 +1,8 @@ # pylint: disable=import-outside-toplevel from unittest.mock import patch +from services.node.detect_package_manager import PACKAGE_MANAGER_TO_LOCK_FILE + def test_run_install_via_codebuild_skips_non_prod(): with patch("services.aws.run_install_via_codebuild.IS_PRD", False): @@ -21,14 +23,20 @@ def test_run_install_via_codebuild_starts_build(): with patch( "services.aws.run_install_via_codebuild.get_npm_token" ) as mock_token: - mock_client.start_build.return_value = mock_response - mock_token.return_value = None + with patch( + "services.aws.run_install_via_codebuild._MONGOD_SCRIPT_SRC", + "/nonexistent", + ): + mock_client.start_build.return_value = mock_response + mock_token.return_value = None - from services.aws.run_install_via_codebuild import ( - run_install_via_codebuild, - ) + from services.aws.run_install_via_codebuild import ( + run_install_via_codebuild, + ) - result = run_install_via_codebuild("/mnt/efs/owner/repo", 123, "npm") + result = run_install_via_codebuild( + "/mnt/efs/owner/repo", 123, "npm" + ) mock_client.start_build.assert_called_once() call_args = mock_client.start_build.call_args @@ -57,14 +65,20 @@ def test_run_install_via_codebuild_includes_npm_token(): with patch( "services.aws.run_install_via_codebuild.get_npm_token" ) as mock_token: - mock_client.start_build.return_value = mock_response - mock_token.return_value = "npm_secret_token" + with patch( + "services.aws.run_install_via_codebuild._MONGOD_SCRIPT_SRC", + "/nonexistent", + ): + mock_client.start_build.return_value = mock_response + mock_token.return_value = "npm_secret_token" - from services.aws.run_install_via_codebuild import ( - run_install_via_codebuild, - ) + from services.aws.run_install_via_codebuild import ( + run_install_via_codebuild, + ) - result = run_install_via_codebuild("/mnt/efs/owner/repo", 123, "yarn") + result = run_install_via_codebuild( + "/mnt/efs/owner/repo", 123, "yarn" + ) call_args = mock_client.start_build.call_args env_vars = call_args.kwargs["environmentVariablesOverride"] @@ -74,3 +88,66 @@ def test_run_install_via_codebuild_includes_npm_token(): "type": "PLAINTEXT", } in env_vars assert result == "gitauto-package-install:build-456" + + +def test_run_install_via_codebuild_copies_mongod_script_for_node(tmp_path): + """Node.js package managers should copy the mongod download script to EFS.""" + mock_response = {"build": {"id": "build-789"}} + efs_script = str(tmp_path / ".scripts" / "download_mongod_binary.mjs") + + with patch("services.aws.run_install_via_codebuild.IS_PRD", True): + with patch( + "services.aws.run_install_via_codebuild.codebuild_client" + ) as mock_client: + with patch( + "services.aws.run_install_via_codebuild.get_npm_token", + return_value=None, + ): + with patch( + "services.aws.run_install_via_codebuild._MONGOD_SCRIPT_EFS", + efs_script, + ): + mock_client.start_build.return_value = mock_response + + from services.aws.run_install_via_codebuild import ( + run_install_via_codebuild, + ) + + for pkg_manager in PACKAGE_MANAGER_TO_LOCK_FILE: + run_install_via_codebuild( + "/mnt/efs/owner/repo", 123, pkg_manager + ) + + import os + + assert os.path.isfile(efs_script) + + +def test_run_install_via_codebuild_skips_mongod_script_for_composer(tmp_path): + """Composer (PHP) projects should NOT copy the mongod download script.""" + mock_response = {"build": {"id": "build-789"}} + efs_script = str(tmp_path / ".scripts" / "download_mongod_binary.mjs") + + with patch("services.aws.run_install_via_codebuild.IS_PRD", True): + with patch( + "services.aws.run_install_via_codebuild.codebuild_client" + ) as mock_client: + with patch( + "services.aws.run_install_via_codebuild.get_npm_token", + return_value=None, + ): + with patch( + "services.aws.run_install_via_codebuild._MONGOD_SCRIPT_EFS", + efs_script, + ): + mock_client.start_build.return_value = mock_response + + from services.aws.run_install_via_codebuild import ( + run_install_via_codebuild, + ) + + run_install_via_codebuild("/mnt/efs/owner/repo", 123, "composer") + + import os + + assert not os.path.isfile(efs_script)