Add flower for monitoring celery (#132)

* Upgrade qdrant-client to v1.11.1

* Add flower for monitoring celery clusters

.env.example

@@ -63,3 +63,6 @@ DOCKER_IMAGE_FRONTEND=frontend
 
 # Qdrant
 QDRANT__SERVICE__API_KEY=changethis
+
+# Flower
+FLOWER_BASIC_AUTH=admin:changethis

.github/workflows/deploy-production.yml

@@ -23,6 +23,7 @@ jobs:
       EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }}
       POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+      FLOWER_BASIC_AUTH: ${{ secrets.FLOWER_BASIC_AUTH }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/deploy-staging.yml

@@ -23,6 +23,7 @@ jobs:
       EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }}
       POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+      FLOWER_BASIC_AUTH: ${{ secrets.FLOWER_BASIC_AUTH }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

backend/README.md

@@ -25,6 +25,8 @@ Adminer, database web administration: http://localhost:8080
 
 Traefik UI, to see how the routes are being handled by the proxy: http://localhost:8090
 
+Flower, administration of Celery tasks: http://localhost:5555
+
 **Note**: The first time you start your stack, it might take a minute for it to be ready. While the backend waits for the database to be ready and configures everything. You can check the logs to monitor it.
 
 To check the logs, run:

backend/pyproject.toml

@@ -45,7 +45,7 @@ types-psycopg2 = "^2.9.21.20240417"
 asyncpg-stubs = "^0.29.1"
 langchain-qdrant = "^0.1.1"
 fastembed = "^0.3.1"
-qdrant-client = "1.10.1"
+qdrant-client = "1.11.1"
 coverage = "^7.5.4"
 celery = "^5.4.0"
 redis = "^5.0.7"

copier.yml

@@ -59,6 +59,11 @@ sentry_dsn:
   help: The DSN for Sentry, if you are using it, you can set it later in .env
   default: ""
 
+flower_basic_auth:
+  type: str
+  help: The basic auth for Flower, you can set it later in .env
+  default: "admin:changethis"
+
 _exclude:
   # Global
   - .vscode

deployment.md

@@ -144,6 +144,7 @@ You can set several variables, like:
 * `POSTGRES_USER`: The Postgres user, you can leave the default.
 * `POSTGRES_DB`: The database name to use for this application. You can leave the default of `app`.
 * `SENTRY_DSN`: The DSN for Sentry, if you are using it.
+* `FLOWER_BASIC_AUTH`: The HTTP Basic Auth for Flower, like `admin:changethis`.
 
 ## GitHub Actions Environment Variables
 
@@ -293,6 +294,8 @@ Backend API base URL: `https://tribe.example.com/api/`
 
 Adminer: `https://adminer.tribe.example.com`
 
+Flower: `https://flower.tribe.example.com`
+
 ### Staging
 
 Frontend: `https://staging.tribe.example.com`
@@ -302,3 +305,5 @@ Backend API docs: `https://staging.tribe.example.com/docs`
 Backend API base URL: `https://staging.tribe.example.com/api/`
 
 Adminer: `https://staging.adminer.tribe.example.com`
+
+Flower: `https://staging.flower.tribe.example.com`

development.md

@@ -158,6 +158,8 @@ Adminer: http://localhost:8080
 
 Traefik UI: http://localhost:8090
 
+Flower: http://localhost:5555
+
 ### Development in localhost with a custom domain URLs
 
 Development URLs, for local development.
@@ -173,3 +175,5 @@ Automatic Alternative Docs (ReDoc): http://localhost.tribe.com/redoc
 Adminer: http://localhost.tribe.com:8080
 
 Traefik UI: http://localhost.tribe.com:8090
+
+Flower: http://localhost.tribe.com:5555

docker-compose.override.yml

@@ -78,6 +78,11 @@ services:
     volumes:
       - ./backend/:/app
 
+  flower:
+    restart: "no"
+    ports:
+      - "5555:5555"
+
   frontend:
     restart: "no"
     build:

docker-compose.yml

@@ -176,6 +176,33 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/0
 
+  flower:
+    image: mher/flower:2.0
+    restart: always
+    networks:
+      - traefik-public
+      - default
+    env_file:
+      - .env
+    environment:
+      - FLOWER_BASIC_AUTH=${FLOWER_BASIC_AUTH}
+    command:
+      - celery
+      - "--broker=redis://redis:6379/0"
+      - flower
+    labels:
+      - traefik.enable=true
+      - traefik.docker.network=traefik-public
+      - traefik.constraint-label=traefik-public
+      - traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.rule=Host(`flower.${DOMAIN?Variable not set}`)
+      - traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.entrypoints=http
+      - traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.middlewares=https-redirect
+      - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.rule=Host(`flower.${DOMAIN?Variable not set}`)
+      - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.entrypoints=https
+      - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls=true
+      - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls.certresolver=le
+      - traefik.http.services.${STACK_NAME?Variable not set}-flower.loadbalancer.server.port=5555
+
   frontend:
     image: "${DOCKER_IMAGE_FRONTEND?Variable not set}:${TAG-latest}"
     restart: always

local-deployment.md

@@ -35,6 +35,7 @@ Update the `.env` file with your custom configurations. At a minimum, you should
 - `FIRST_SUPERUSER_PASSWORD`
 - `POSTGRES_PASSWORD`
 - `OPENAI_API_KEY` (Choose your preferred model provider)
+- `FLOWER_BASIC_AUTH`
 
 For variables with a default value of `changethis`, generate secure values using the following command:
 ```bash
@@ -75,12 +76,13 @@ docker compose -f docker-compose.yml -f docker-compose.local.yml up
 
 Once the containers are running, you can access various services through the following URLs:
 
-- **Traefik UI**: [http://traefik.localhost/](http://traefik.localhost/)
 - **Frontend**: [http://localhost/](http://localhost/)
 - **Backend API Documentation**: [http://localhost/docs/](http://localhost/docs/)
 - **Backend API Base URL**: [http://localhost/api/](http://localhost/api/)
 - **Qdrant Dashboard**: [http://qdrant.localhost/dashboard](http://qdrant.localhost/dashboard)
 - **Adminer**: [http://adminer.localhost/](http://adminer.localhost/)
+- **Traefik UI**: [http://traefik.localhost/](http://traefik.localhost/)
+- **Flower**: [http://flower.localhost/](http://flower.localhost/)
 
 ## Troubleshooting
 - **Out of Memory**: If you are getting the `Worker (pid:14) was sent SIGKILL! Perhaps out of memory?` error, this is due to the number of processes started for the backend container consuming more memory than what is available. You can fix this by decreasing `MAX_WORKERS` in your `.env` file.