diff --git a/.github/workflows/integration-test-cron.yml b/.github/workflows/integration-test-cron.yml index d457e56..004e5c7 100644 --- a/.github/workflows/integration-test-cron.yml +++ b/.github/workflows/integration-test-cron.yml @@ -130,6 +130,7 @@ jobs: persist-credentials: false - name: Checkout viewer + if: ${{ matrix.server-versions != 'master'}} # todo: update this when NC > 36 is master uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: nextcloud/viewer @@ -199,28 +200,37 @@ jobs: ./occ app:enable -vvv -f encryption ./occ encryption:enable - - name: Setup python 3.11 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.11' - cache: 'pip' - cache-dependency-path: context_chat_backend/requirements.txt + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3 - - name: Install and init backend + - name: Build the context_chat_backend cpu image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6 + with: + context: context_chat_backend + push: false + platforms: linux/amd64 + # use local tag so image is not pulled from remote + tags: ghcr.io/ccb-cpu:local + target: runtime-cpu + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Start backend container run: | cd context_chat_backend - pip install --upgrade pip setuptools wheel - # use the cpu version of torch to not run out of space - pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - sed -i '/torch(vision)?/d' requirements.txt - pip install -r requirements.txt - cp example.env .env - echo "NEXTCLOUD_URL=http://localhost:8080" >> .env - python3 -u ./main_em.py > em_backend_logs 2>&1 & - echo $! > ../em_pid.txt # Save the embedding server process ID (PID) - python3 -u ./main.py > backend_logs 2>&1 & - echo $! > ../pid.txt # Save the process ID (PID) - sleep 60 # Wait for the backend to get ready + echo "NEXTCLOUD_URL=http://localhost:8080" >> example.env + cd .. + # create it up front so the bind mount is not root-owned-only and the log + # files stay readable from the runner after the job + mkdir -p context_chat_backend/persistent_storage/logs + chmod -R 777 context_chat_backend/persistent_storage + docker run --net host --name context_chat_backend \ + --env-file context_chat_backend/example.env \ + -v $(pwd)/context_chat_backend/persistent_storage:/app/persistent_storage \ + -e APP_PERSISTENT_STORAGE=/app/persistent_storage \ + -e EXTERNAL_DB="$CCB_DB_URL" \ + -d ghcr.io/ccb-cpu:local - name: Register backend run: | @@ -228,7 +238,7 @@ jobs: ./occ config:system:set debug --value true ./occ config:system:set loglevel --value 1 ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080 - timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish + timeout 180 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish sleep 60 # Wait for the embedding server to get ready - name: Checkout documentation @@ -353,17 +363,12 @@ jobs: echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold" fi - # Check if backend is still alive - ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0") - if [ "$ccb_alive" -eq 0 ]; then - echo "Error: Context Chat Backend process is not running. Exiting." - exit 1 - fi - - # Check if the embedding server is still alive - em_alive=$(ps -p $(cat em_pid.txt) -o cmd= | grep -c "main_em.py" || echo "0") - if [ "$em_alive" -eq 0 ]; then - echo "Error: Context Chat embedding server process is not running. Exiting." + # Check if backend is still alive; inspect prints exactly "true"/"false" + # and nothing at all when the container is gone + ccb_alive=$(docker inspect -f '{{.State.Running}}' context_chat_backend 2>/dev/null || true) + if [ "$ccb_alive" != "true" ]; then + echo "Error: Context Chat Backend container is not running. Exiting." + docker logs --tail 50 context_chat_backend 2>&1 || true exit 1 fi @@ -415,25 +420,22 @@ jobs: run: | cat data/context_chat.log - - name: Show CCB main app logs + - name: Show CCB container logs if: always() run: | - cat context_chat_backend/backend_logs || echo "No main backend logs" + # supervisord routes both the main app and the embedding server to the + # container's stdout/stderr, so this covers each of them + docker logs context_chat_backend || echo "No container logs" - name: Show CCB main app JSON logs if: always() run: | - tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory" - - - name: Show CCB embedding server logs - if: always() - run: | - cat context_chat_backend/em_backend_logs || echo "No main backend logs" + sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory" - name: Show CCB embedding server JSON logs if: always() run: | - tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory" + sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory" - name: Final stats log if: always() diff --git a/.github/workflows/integration-test-file-listener.yml b/.github/workflows/integration-test-file-listener.yml index 2b44524..3a5e2ac 100644 --- a/.github/workflows/integration-test-file-listener.yml +++ b/.github/workflows/integration-test-file-listener.yml @@ -118,6 +118,7 @@ jobs: persist-credentials: false - name: Checkout viewer + if: ${{ matrix.server-versions != 'master'}} # todo: update this when NC > 36 is master uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: nextcloud/viewer @@ -187,28 +188,37 @@ jobs: ./occ app:enable -vvv -f encryption ./occ encryption:enable - - name: Setup python 3.11 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.11' - cache: 'pip' - cache-dependency-path: context_chat_backend/requirements.txt + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3 - - name: Install and init backend + - name: Build the context_chat_backend cpu image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6 + with: + context: context_chat_backend + push: false + platforms: linux/amd64 + # use local tag so image is not pulled from remote + tags: ghcr.io/ccb-cpu:local + target: runtime-cpu + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Start backend container run: | cd context_chat_backend - pip install --upgrade pip setuptools wheel - # use the cpu version of torch to not run out of space - pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - sed -i '/torch(vision)?/d' requirements.txt - pip install -r requirements.txt - cp example.env .env - echo "NEXTCLOUD_URL=http://localhost:8080" >> .env - python3 -u ./main_em.py > em_backend_logs 2>&1 & - echo $! > ../em_pid.txt # Save the embedding server process ID (PID) - python3 -u ./main.py > backend_logs 2>&1 & - echo $! > ../pid.txt # Save the process ID (PID) - sleep 60 # Wait for the backend to get ready + echo "NEXTCLOUD_URL=http://localhost:8080" >> example.env + cd .. + # create it up front so the bind mount is not root-owned-only and the log + # files stay readable from the runner after the job + mkdir -p context_chat_backend/persistent_storage/logs + chmod -R 777 context_chat_backend/persistent_storage + docker run --net host --name context_chat_backend \ + --env-file context_chat_backend/example.env \ + -v $(pwd)/context_chat_backend/persistent_storage:/app/persistent_storage \ + -e APP_PERSISTENT_STORAGE=/app/persistent_storage \ + -e EXTERNAL_DB="$CCB_DB_URL" \ + -d ghcr.io/ccb-cpu:local - name: Register backend run: | @@ -216,7 +226,7 @@ jobs: ./occ config:system:set debug --value true ./occ config:system:set loglevel --value 1 ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080 - timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish + timeout 180 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish sleep 60 # Wait for the embedding server to get ready - name: Checkout documentation @@ -344,17 +354,12 @@ jobs: echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold" fi - # Check if backend is still alive - ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0") - if [ "$ccb_alive" -eq 0 ]; then - echo "Error: Context Chat Backend process is not running. Exiting." - exit 1 - fi - - # Check if the embedding server is still alive - em_alive=$(ps -p $(cat em_pid.txt) -o cmd= | grep -c "main_em.py" || echo "0") - if [ "$em_alive" -eq 0 ]; then - echo "Error: Context Chat embedding server process is not running. Exiting." + # Check if backend is still alive; inspect prints exactly "true"/"false" + # and nothing at all when the container is gone + ccb_alive=$(docker inspect -f '{{.State.Running}}' context_chat_backend 2>/dev/null || true) + if [ "$ccb_alive" != "true" ]; then + echo "Error: Context Chat Backend container is not running. Exiting." + docker logs --tail 50 context_chat_backend 2>&1 || true exit 1 fi @@ -542,25 +547,22 @@ jobs: run: | cat data/context_chat.log - - name: Show CCB main app logs + - name: Show CCB container logs if: always() run: | - cat context_chat_backend/backend_logs || echo "No main backend logs" + # supervisord routes both the main app and the embedding server to the + # container's stdout/stderr, so this covers each of them + docker logs context_chat_backend || echo "No container logs" - name: Show CCB main app JSON logs if: always() run: | - tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory" - - - name: Show CCB embedding server logs - if: always() - run: | - cat context_chat_backend/em_backend_logs || echo "No main backend logs" + sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory" - name: Show CCB embedding server JSON logs if: always() run: | - tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory" + sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory" - name: Final stats log if: always()