Merge dev: Node.js scraper migration + CI fix (#32)

Co-authored-by: Stephen Minakian <stephenminakian@gmail.com>
Co-committed-by: Stephen Minakian <stephenminakian@gmail.com>

.github/workflows/deploy.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
   workflow_dispatch:
 
 env:
@@ -13,22 +12,17 @@ env:
 
 jobs:
   # ============================================================
-  # Test Job - Runs first, blocks everything if tests fail
+  # CI Job - Lint + Test in a single job (one checkout, one npm ci)
   # ============================================================
-  test:
-    name: Run Tests
+  ci:
+    name: Lint & Test
     runs-on: ubuntu-latest
 
-    services:
-      mongodb:
-        image: mongo:7
-        ports:
-          - 27018:27017
-        options: >-
-          --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
+    outputs:
+      lint_status: ${{ steps.lint.outcome }}
+      lint_output: ${{ steps.lint.outputs.lint_output }}
+      test_status: ${{ steps.test.outcome }}
+      test_output: ${{ steps.test.outputs.test_output }}
 
     steps:
       - name: Checkout code
@@ -43,13 +37,137 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      - name: Run ESLint
+        id: lint
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm run lint 2>&1)
+          EXIT_CODE=$?
+          # Truncate before writing to GITHUB_OUTPUT to prevent
+          # "argument list too long" in downstream jobs
+          echo "lint_output<<EOF" >> $GITHUB_OUTPUT
+          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
+
       - name: Run tests
-        run: npm test -- --runInBand
+        id: test
+        continue-on-error: true
+        run: |
+          set +e
+          OUTPUT=$(npm test -- --runInBand 2>&1)
+          EXIT_CODE=$?
+          # Truncate before writing to GITHUB_OUTPUT to prevent
+          # "argument list too long" in downstream jobs
+          echo "test_output<<EOF" >> $GITHUB_OUTPUT
+          echo "${OUTPUT:0:10000}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          exit $EXIT_CODE
         env:
-          MONGO_URI: mongodb://localhost:27018
           JWT_SECRET: test-jwt-secret-for-ci
           NODE_ENV: test
 
+  # ============================================================
+  # Notify Job - Send results to n8n webhook
+  # ============================================================
+  notify:
+    name: Send Webhook Notification
+    runs-on: ubuntu-latest
+    needs: [ci]
+    if: always()
+
+    steps:
+      - name: Send results to n8n webhook
+        env:
+          LINT_STATUS: ${{ needs.ci.outputs.lint_status }}
+          TEST_STATUS: ${{ needs.ci.outputs.test_status }}
+          RAW_LINT_OUTPUT: ${{ needs.ci.outputs.lint_output }}
+          RAW_TEST_OUTPUT: ${{ needs.ci.outputs.test_output }}
+          GH_REPO: ${{ github.repository }}
+          GH_BRANCH: ${{ github.head_ref || github.ref_name }}
+          GH_SHA: ${{ github.sha }}
+          GH_COMMIT_MSG: ${{ github.event.head_commit.message || github.event.pull_request.title || 'N/A' }}
+          GH_ACTOR: ${{ github.actor }}
+          GH_EVENT: ${{ github.event_name }}
+          GH_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
+          GH_RUN_ID: ${{ github.run_id }}
+          GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          WEBHOOK_URL: ${{ secrets.N8N_WEBHOOK_URL }}
+        run: |
+          # Determine overall status
+          if [ "$LINT_STATUS" = "success" ] && [ "$TEST_STATUS" = "success" ]; then
+            OVERALL_STATUS="success"
+          else
+            OVERALL_STATUS="failure"
+          fi
+
+          # Outputs are already truncated at the source (ci job)
+          LINT_OUTPUT="$RAW_LINT_OUTPUT"
+          TEST_OUTPUT="$RAW_TEST_OUTPUT"
+
+          # Build JSON payload
+          PAYLOAD=$(jq -n \
+            --arg repo "$GH_REPO" \
+            --arg branch "$GH_BRANCH" \
+            --arg commit "$GH_SHA" \
+            --arg commit_short "${GH_SHA:0:7}" \
+            --arg commit_message "$GH_COMMIT_MSG" \
+            --arg author "$GH_ACTOR" \
+            --arg event "$GH_EVENT" \
+            --arg pr_number "$GH_PR_NUMBER" \
+            --arg run_id "$GH_RUN_ID" \
+            --arg run_url "$GH_RUN_URL" \
+            --arg overall_status "$OVERALL_STATUS" \
+            --arg lint_status "$LINT_STATUS" \
+            --arg lint_output "$LINT_OUTPUT" \
+            --arg test_status "$TEST_STATUS" \
+            --arg test_output "$TEST_OUTPUT" \
+            --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            '{
+              repository: $repo,
+              branch: $branch,
+              pull_request: (if $pr_number != "" then { number: ($pr_number | tonumber) } else null end),
+              commit: {
+                sha: $commit,
+                short_sha: $commit_short,
+                message: $commit_message,
+                author: $author
+              },
+              event: $event,
+              run: {
+                id: $run_id,
+                url: $run_url
+              },
+              status: $overall_status,
+              results: {
+                lint: {
+                  status: $lint_status,
+                  output: (if $lint_status != "success" then $lint_output else null end)
+                },
+                test: {
+                  status: $test_status,
+                  output: (if $test_status != "success" then $test_output else null end)
+                }
+              },
+              timestamp: $timestamp
+            }')
+
+          # Send webhook
+          curl -X POST \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD" \
+            "$WEBHOOK_URL" \
+            --fail --silent --show-error
+
+      - name: Fail if lint or tests failed
+        if: needs.ci.outputs.lint_status != 'success' || needs.ci.outputs.test_status != 'success'
+        run: |
+          echo "❌ Pipeline failed:"
+          echo "   Lint: ${{ needs.ci.outputs.lint_status }}"
+          echo "   Test: ${{ needs.ci.outputs.test_status }}"
+          exit 1
+
   # ============================================================
   # Dependency Scan Job - Runs in parallel with tests
   # ============================================================
@@ -80,7 +198,7 @@ jobs:
   build:
     name: Build & Push Image
     runs-on: ubuntu-latest
-    needs: [test, scan-deps]
+    needs: [ci, scan-deps, notify]
     if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
 
     outputs:
@@ -146,12 +264,13 @@ jobs:
       # ============================================================
       # SBOM Generation with Syft
       # ============================================================
+      - name: Install Syft
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
+
       - name: Generate SBOM with Syft
-        uses: anchore/sbom-action@v0
-        with:
-          image: ${{ steps.set-tag.outputs.full_image }}
-          format: spdx-json
-          output-file: sbom.spdx.json
+        run: |
+          syft ${{ steps.set-tag.outputs.full_image }} -o spdx-json=sbom.spdx.json
 
       # ============================================================
       # Image Signing with Cosign