Merge dev: Node.js scraper migration + CI fix #32

Merged
stephen merged 32 commits from dev into main 2026-02-08 20:35:29 -07:00
19 changed files with 7441 additions and 25 deletions
Showing only changes of commit 3509da8185 - Show all commits

View File

@ -12,15 +12,17 @@ env:
jobs: jobs:
# ============================================================ # ============================================================
# Lint Job - ESLint static analysis # CI Job - Lint + Test in a single job (one checkout, one npm ci)
# ============================================================ # ============================================================
lint: ci:
name: Run Linting name: Lint & Test
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
status: ${{ steps.lint.outcome }} lint_status: ${{ steps.lint.outcome }}
output: ${{ steps.lint.outputs.lint_output }} lint_output: ${{ steps.lint.outputs.lint_output }}
test_status: ${{ steps.test.outcome }}
test_output: ${{ steps.test.outputs.test_output }}
steps: steps:
- name: Checkout code - name: Checkout code
@ -47,54 +49,18 @@ jobs:
echo "EOF" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE exit $EXIT_CODE
# ============================================================
# Test Job - Runs first, blocks everything if tests fail
# ============================================================
test:
name: Run Tests
runs-on: ubuntu-latest
outputs:
status: ${{ steps.test.outcome }}
output: ${{ steps.test.outputs.test_output }}
services:
mongodb:
image: mongo:7
ports:
- 27018:27017
options: >-
--health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'npm'
- name: Install dependencies
run: npm ci
- name: Run tests - name: Run tests
id: test id: test
continue-on-error: true continue-on-error: true
run: | run: |
set +e set +e
OUTPUT=$(npm test -- --runInBand 2>&1) OUTPUT=$(npm test 2>&1)
EXIT_CODE=$? EXIT_CODE=$?
echo "test_output<<EOF" >> $GITHUB_OUTPUT echo "test_output<<EOF" >> $GITHUB_OUTPUT
echo "$OUTPUT" >> $GITHUB_OUTPUT echo "$OUTPUT" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT
exit $EXIT_CODE exit $EXIT_CODE
env: env:
MONGO_URI: mongodb://localhost:27018
JWT_SECRET: test-jwt-secret-for-ci JWT_SECRET: test-jwt-secret-for-ci
NODE_ENV: test NODE_ENV: test
@ -104,16 +70,16 @@ jobs:
notify: notify:
name: Send Webhook Notification name: Send Webhook Notification
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [lint, test] needs: [ci]
if: always() if: always()
steps: steps:
- name: Send results to n8n webhook - name: Send results to n8n webhook
env: env:
LINT_STATUS: ${{ needs.lint.result }} LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
TEST_STATUS: ${{ needs.test.result }} TEST_STATUS: ${{ needs.ci.outputs.test_status }}
RAW_LINT_OUTPUT: ${{ needs.lint.outputs.output }} RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
RAW_TEST_OUTPUT: ${{ needs.test.outputs.output }} RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
GH_REPO: ${{ github.repository }} GH_REPO: ${{ github.repository }}
GH_BRANCH: ${{ github.head_ref || github.ref_name }} GH_BRANCH: ${{ github.head_ref || github.ref_name }}
GH_SHA: ${{ github.sha }} GH_SHA: ${{ github.sha }}
@ -191,11 +157,11 @@ jobs:
--fail --silent --show-error --fail --silent --show-error
- name: Fail if lint or tests failed - name: Fail if lint or tests failed
if: needs.lint.result != 'success' || needs.test.result != 'success' if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
run: | run: |
echo "❌ Pipeline failed:" echo "❌ Pipeline failed:"
echo " Lint: ${{ needs.lint.result }}" echo " Lint: ${{ needs.ci.outputs.lint_status }}"
echo " Test: ${{ needs.test.result }}" echo " Test: ${{ needs.ci.outputs.test_status }}"
exit 1 exit 1
# ============================================================ # ============================================================
@ -228,7 +194,7 @@ jobs:
build: build:
name: Build & Push Image name: Build & Push Image
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [lint, test, scan-deps, notify] needs: [ci, scan-deps, notify]
if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
outputs: outputs: