fix(ci): use docker container in integration-test

.github/workflows/integration-test-cron.yml

@@ -130,6 +130,7 @@ jobs:
           persist-credentials: false
 
       - name: Checkout viewer
+        if: ${{ matrix.server-versions != 'master'}}  # todo: update this when NC > 36 is master
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           repository: nextcloud/viewer
@@ -199,36 +200,45 @@ jobs:
           ./occ app:enable -vvv -f encryption
           ./occ encryption:enable
 
-      - name: Setup python 3.11
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-          cache-dependency-path: context_chat_backend/requirements.txt
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
 
-      - name: Install and init backend
+      - name: Build the context_chat_backend cpu image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6
+        with:
+          context: context_chat_backend
+          push: false
+          platforms: linux/amd64
+          # use local tag so image is not pulled from remote
+          tags: ghcr.io/ccb-cpu:local
+          target: runtime-cpu
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Start backend container
         run: |
           cd context_chat_backend
-          pip install --upgrade pip setuptools wheel
-          # use the cpu version of torch to not run out of space
-          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
-          sed -i '/torch(vision)?/d' requirements.txt
-          pip install -r requirements.txt
-          cp example.env .env
-          echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
-          python3 -u ./main_em.py > em_backend_logs 2>&1 &
-          echo $! > ../em_pid.txt       # Save the embedding server process ID (PID)
-          python3 -u ./main.py > backend_logs 2>&1 &
-          echo $! > ../pid.txt          # Save the process ID (PID)
-          sleep 60 # Wait for the backend to get ready
+          echo "NEXTCLOUD_URL=http://localhost:8080" >> example.env
+          cd ..
+          # create it up front so the bind mount is not root-owned-only and the log
+          # files stay readable from the runner after the job
+          mkdir -p context_chat_backend/persistent_storage/logs
+          chmod -R 777 context_chat_backend/persistent_storage
+          docker run --net host --name context_chat_backend \
+            --env-file context_chat_backend/example.env \
+            -v $(pwd)/context_chat_backend/persistent_storage:/app/persistent_storage \
+            -e APP_PERSISTENT_STORAGE=/app/persistent_storage \
+            -e EXTERNAL_DB="$CCB_DB_URL" \
+            -d ghcr.io/ccb-cpu:local
 
       - name: Register backend
         run: |
           set -x
           ./occ config:system:set debug --value true
           ./occ config:system:set loglevel --value 1
           ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
-          timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
+          timeout 180 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
           sleep 60 # Wait for the embedding server to get ready
 
       - name: Checkout documentation
@@ -353,17 +363,12 @@ jobs:
               echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
             fi
 
-            # Check if backend is still alive
-            ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0")
-            if [ "$ccb_alive" -eq 0 ]; then
-              echo "Error: Context Chat Backend process is not running. Exiting."
-              exit 1
-            fi
-
-            # Check if the embedding server is still alive
-            em_alive=$(ps -p $(cat em_pid.txt) -o cmd= | grep -c "main_em.py" || echo "0")
-            if [ "$em_alive" -eq 0 ]; then
-              echo "Error: Context Chat embedding server process is not running. Exiting."
+            # Check if backend is still alive; inspect prints exactly "true"/"false"
+            # and nothing at all when the container is gone
+            ccb_alive=$(docker inspect -f '{{.State.Running}}' context_chat_backend 2>/dev/null || true)
+            if [ "$ccb_alive" != "true" ]; then
+              echo "Error: Context Chat Backend container is not running. Exiting."
+              docker logs --tail 50 context_chat_backend 2>&1 || true
               exit 1
             fi
 
@@ -415,25 +420,22 @@ jobs:
         run: |
           cat data/context_chat.log
 
-      - name: Show CCB main app logs
+      - name: Show CCB container logs
         if: always()
         run: |
-          cat context_chat_backend/backend_logs || echo "No main backend logs"
+          # supervisord routes both the main app and the embedding server to the
+          # container's stdout/stderr, so this covers each of them
+          docker logs context_chat_backend || echo "No container logs"
 
       - name: Show CCB main app JSON logs
         if: always()
         run: |
-          tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
-
-      - name: Show CCB embedding server logs
-        if: always()
-        run: |
-          cat context_chat_backend/em_backend_logs || echo "No main backend logs"
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
 
       - name: Show CCB embedding server JSON logs
         if: always()
         run: |
-          tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
 
       - name: Final stats log
         if: always()

.github/workflows/integration-test-file-listener.yml

@@ -118,6 +118,7 @@ jobs:
           persist-credentials: false
 
       - name: Checkout viewer
+        if: ${{ matrix.server-versions != 'master'}}  # todo: update this when NC > 36 is master
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           repository: nextcloud/viewer
@@ -187,36 +188,45 @@ jobs:
           ./occ app:enable -vvv -f encryption
           ./occ encryption:enable
 
-      - name: Setup python 3.11
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-          cache-dependency-path: context_chat_backend/requirements.txt
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
 
-      - name: Install and init backend
+      - name: Build the context_chat_backend cpu image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6
+        with:
+          context: context_chat_backend
+          push: false
+          platforms: linux/amd64
+          # use local tag so image is not pulled from remote
+          tags: ghcr.io/ccb-cpu:local
+          target: runtime-cpu
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Start backend container
         run: |
           cd context_chat_backend
-          pip install --upgrade pip setuptools wheel
-          # use the cpu version of torch to not run out of space
-          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
-          sed -i '/torch(vision)?/d' requirements.txt
-          pip install -r requirements.txt
-          cp example.env .env
-          echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
-          python3 -u ./main_em.py > em_backend_logs 2>&1 &
-          echo $! > ../em_pid.txt       # Save the embedding server process ID (PID)
-          python3 -u ./main.py > backend_logs 2>&1 &
-          echo $! > ../pid.txt          # Save the process ID (PID)
-          sleep 60 # Wait for the backend to get ready
+          echo "NEXTCLOUD_URL=http://localhost:8080" >> example.env
+          cd ..
+          # create it up front so the bind mount is not root-owned-only and the log
+          # files stay readable from the runner after the job
+          mkdir -p context_chat_backend/persistent_storage/logs
+          chmod -R 777 context_chat_backend/persistent_storage
+          docker run --net host --name context_chat_backend \
+            --env-file context_chat_backend/example.env \
+            -v $(pwd)/context_chat_backend/persistent_storage:/app/persistent_storage \
+            -e APP_PERSISTENT_STORAGE=/app/persistent_storage \
+            -e EXTERNAL_DB="$CCB_DB_URL" \
+            -d ghcr.io/ccb-cpu:local
 
       - name: Register backend
         run: |
           set -x
           ./occ config:system:set debug --value true
           ./occ config:system:set loglevel --value 1
           ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
-          timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
+          timeout 180 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
           sleep 60 # Wait for the embedding server to get ready
 
       - name: Checkout documentation
@@ -344,17 +354,12 @@ jobs:
               echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
             fi
 
-            # Check if backend is still alive
-            ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0")
-            if [ "$ccb_alive" -eq 0 ]; then
-              echo "Error: Context Chat Backend process is not running. Exiting."
-              exit 1
-            fi
-
-            # Check if the embedding server is still alive
-            em_alive=$(ps -p $(cat em_pid.txt) -o cmd= | grep -c "main_em.py" || echo "0")
-            if [ "$em_alive" -eq 0 ]; then
-              echo "Error: Context Chat embedding server process is not running. Exiting."
+            # Check if backend is still alive; inspect prints exactly "true"/"false"
+            # and nothing at all when the container is gone
+            ccb_alive=$(docker inspect -f '{{.State.Running}}' context_chat_backend 2>/dev/null || true)
+            if [ "$ccb_alive" != "true" ]; then
+              echo "Error: Context Chat Backend container is not running. Exiting."
+              docker logs --tail 50 context_chat_backend 2>&1 || true
               exit 1
             fi
 
@@ -542,25 +547,22 @@ jobs:
         run: |
           cat data/context_chat.log
 
-      - name: Show CCB main app logs
+      - name: Show CCB container logs
         if: always()
         run: |
-          cat context_chat_backend/backend_logs || echo "No main backend logs"
+          # supervisord routes both the main app and the embedding server to the
+          # container's stdout/stderr, so this covers each of them
+          docker logs context_chat_backend || echo "No container logs"
 
       - name: Show CCB main app JSON logs
         if: always()
         run: |
-          tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
-
-      - name: Show CCB embedding server logs
-        if: always()
-        run: |
-          cat context_chat_backend/em_backend_logs || echo "No main backend logs"
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
 
       - name: Show CCB embedding server JSON logs
         if: always()
         run: |
-          tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
 
       - name: Final stats log
         if: always()