Feature/high availability

sf-docker/special/high-availability/README.md

@@ -0,0 +1,4 @@
+# SmartFace Large Scale Deployment
+
+This sample is intended to be deployed on a single host for demonstration purposes. When running on multiple hosts (either physical or virtual), changes listed bellow are required.
+

sf-docker/special/high-availability/minio/docker-compose.yml

@@ -0,0 +1,80 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:RELEASE.2023-04-13T03-08-07Z
+  command: server --console-address ":9001" http://minio{1...4}/data{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  # environment:
+    # MINIO_ROOT_USER: minioadmin
+    # MINIO_ROOT_PASSWORD: minioadmin
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - data1-1:/data1
+      - data1-2:/data2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - data2-1:/data1
+      - data2-2:/data2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - data3-1:/data1
+      - data3-2:/data2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - data4-1:/data1
+      - data4-2:/data2
+
+  minio-proxy:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  data1-1:
+  data1-2:
+  data2-1:
+  data2-2:
+  data3-1:
+  data3-2:
+  data4-1:
+  data4-2:
+
+networks:
+  default:
+    external:
+      name: HighAvailabilityClusterNetwork

sf-docker/special/high-availability/minio/nginx.conf

@@ -0,0 +1,106 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

sf-docker/special/high-availability/pgsql/docker-compose.yml

@@ -0,0 +1,30 @@
+version: '3'
+services:
+  pgsql:
+    image: "postgres:14.3"
+    container_name: pgsql
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_PASSWORD=Test1234
+    restart: unless-stopped
+    volumes:
+      - pgsqldata:/var/lib/postgresql/data
+      #- /var/lib/smartface/pgsql:/var/lib/postgresql/data
+
+  pgadmin:
+    image: "dpage/pgadmin4:6.10"
+    container_name: pgadmin
+    environment:
+      PGADMIN_DEFAULT_EMAIL: [email protected]
+      PGADMIN_DEFAULT_PASSWORD: Test1234
+    ports:
+      - "7070:80"
+
+volumes:
+  pgsqldata:
+
+networks:
+  default:
+    external:
+      name: HighAvailabilityClusterNetwork

sf-docker/special/high-availability/server1/.env

@@ -0,0 +1,47 @@
+# RMQ config
+RabbitMQ__Hostname=rmq1
+RabbitMQ__Username=guest
+RabbitMQ__Password=guest
+RabbitMQ__VirtualHost=/
+RabbitMQ__Port=5672
+RabbitMQ__UseSsl=false
+
+# Database config
+#Database__DbEngine=MsSql
+#ConnectionStrings__CoreDbContext=Server=mssql;Database=SmartFace;User ID=sa;Password=Test1234;TrustServerCertificate=true;
+Database__DbEngine=PgSql
+ConnectionStrings__CoreDbContext=Server=pgsql;Database=smartface;Username=postgres;Password=Test1234;Trust Server Certificate=true;
+
+# S3 config
+S3Bucket__Endpoint=http://minio-proxy:9000
+S3Bucket__BucketName=inno-smartface
+S3Bucket__AccessKey=minioadmin
+S3Bucket__SecretKey=minioadmin
+
+# Set true when a Jaeger tracing is required
+AppSettings__USE_JAEGER_APP_SETTINGS=false
+
+# Jaeger tracing endpoint. 'jaeger' is the name of included docker container.
+# If targeting outside SmartFace docker, change to remote URL
+JAEGER_AGENT_HOST=jaeger
+
+# API config - we use port 80 in docker and forward it as needed
+Hosting__Host=http://0.0.0.0
+Hosting__Port=80
+
+# we override the default generation of preview ports for camera in favor of static configuration
+CameraDefaults__PreviewPort=30000
+
+# Using NoSql database
+NoSqlDataStorageDisabled=false
+
+# Using template for Gstreamer pipeline
+GstPipelineTemplate=uridecodebin uri={0} source::latency=0 ! queue max-size-buffers=1 leaky=downstream ! nvvideoconvert ! video/x-raw, format=(string)BGRx ! videoconvert ! video/x-raw, format=(string)BGR ! appsink
+
+# Registry, must end with a forward slash `/`
+REGISTRY=registry.gitlab.com/innovatrics/smartface/
+
+# Version
+SF_VERSION=v5_4.18.1
+AC_VERSION=v5_1.8.0
+SFS_VERSION=v5_1.15.0

sf-docker/special/high-availability/server1/.env.sfac

@@ -0,0 +1,47 @@
+FilterConfiguration__FaceOrderConfiguration__Enabled=false
+FilterConfiguration__FaceOrderConfiguration__Order=1
+
+FilterConfiguration__OpeningDebounceConfiguration__OpeningDebounceEnabled=true
+FilterConfiguration__OpeningDebounceConfiguration__OpeningDebounceMs=4000
+
+FilterConfiguration__BlockingDebounceConfiguration__BlockingDebounceEnabled=true
+FilterConfiguration__BlockingDebounceConfiguration__BlockingDebounceMs=4000
+
+FilterConfiguration__ExclusiveCameraConfiguration__Enabled=true
+FilterConfiguration__ExclusiveCameraConfiguration__ExclusivityMs=5000
+
+FilterConfiguration__NotIdentifiedPersonConfiguration__Enabled=true
+FilterConfiguration__NotIdentifiedPersonConfiguration__RoamingLimitTimeMs=3000
+
+FilterConfiguration__BlacklistsConfiguration__Enabled=true
+FilterConfiguration__BlacklistsConfiguration__Blacklists__0=firt_black_list_id
+FilterConfiguration__BlacklistsConfiguration__Blacklists__1=second_black_list_id
+FilterConfiguration__BlacklistsConfiguration__Blacklists__2=third_black_list_id
+
+FilterConfiguration__FaceMaskConfiguration__Enabled=false
+FilterConfiguration__FaceMaskConfiguration__DenyingDebounceMs=4000
+
+FilterConfiguration__IntentionalAccessConfiguration__Enabled=false
+FilterConfiguration__IntentionalAccessConfiguration__AlwaysOpenForFaceAreaPercentLargerThan=7
+FilterConfiguration__IntentionalAccessConfiguration__RequiredFaceApproachingRatePercent=0.4
+
+FilterConfiguration__StreamGroupsConfiguration__Enabled=true
+FilterConfiguration__StreamGroupsConfiguration__GroupOpeningDebounceMs=3000
+
+FilterConfiguration__SpoofCheckConfiguration__Enabled=false
+FilterConfiguration__SpoofCheckConfiguration__DenyingDebounceMs=4000
+
+# Set true when a Jaeger tracing is required
+AppSettings__0__Key=USE_JAEGER_APP_SETTINGS
+AppSettings__0__Value=false
+
+AppSettings__1__Key=JAEGER_SAMPLER_TYPE
+AppSettings__1__Value=const
+
+AppSettings__2__Key=JAEGER_SAMPLER_PARAM
+AppSettings__2__Value=1
+
+# Jaeger tracing endpoint. 'jaeger' is the name of included docker container. 
+# If targeting outside SmartFace docker, change to remote URL
+AppSettings__3__Key=JAEGER_AGENT_HOST
+AppSettings__3__Value=jaeger