From 9a9dba16441931e822bb32aa726e190034744b2e Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Mon, 9 Dec 2024 17:03:02 +0530 Subject: [PATCH 01/14] enh: db schema change Signed-off-by: Anupam Kumar --- appinfo/routes.php | 1 + composer.lock | 381 +++++++++++------- lib/AppInfo/Application.php | 6 +- lib/BackgroundJobs/ActionJob.php | 117 ++++++ lib/BackgroundJobs/DeleteJob.php | 74 ---- lib/BackgroundJobs/IndexerJob.php | 98 +++-- .../InitialContentImportJob.php | 5 +- lib/BackgroundJobs/SchedulerJob.php | 9 +- lib/BackgroundJobs/StorageCrawlJob.php | 4 +- lib/BackgroundJobs/SubmitContentJob.php | 48 +-- lib/Command/Diagnostics.php | 2 +- lib/Command/ScanFiles.php | 2 +- lib/Controller/ConfigController.php | 3 +- lib/Controller/ProviderController.php | 1 + lib/Db/{QueueDelete.php => QueueAction.php} | 17 +- ...DeleteMapper.php => QueueActionMapper.php} | 20 +- lib/Db/QueueContentItem.php | 1 + lib/Db/QueueContentItemMapper.php | 1 + lib/Db/QueueFile.php | 5 +- lib/Db/QueueMapper.php | 8 +- lib/Event/ContentProviderRegisterEvent.php | 1 + lib/Listener/AppDisableListener.php | 7 +- lib/Listener/FileListener.php | 83 ++-- lib/Listener/UserDeletedListener.php | 42 ++ .../Version001000000Date20231102094721.php | 1 + .../Version1001Date20240130120627.php | 1 + .../Version2002Date20240619004215.php | 1 + .../Version4000Date20241108004215.php | 3 +- .../Version4000Date20241202141755.php | 34 ++ .../Version4000Date20241206135634.php | 51 +++ lib/Public/ContentItem.php | 1 + lib/Public/ContentManager.php | 112 ++++- lib/Public/IContentProvider.php | 1 + .../UpdateAccessOp.php} | 8 +- lib/Service/ActionService.php | 139 +++++++ lib/Service/DeleteService.php | 106 ----- lib/Service/LangRopeService.php | 86 +++- lib/Service/MetadataService.php | 7 +- lib/Service/ProviderConfigService.php | 3 +- lib/Service/QueueService.php | 10 +- lib/Service/ScanService.php | 24 +- lib/Service/StorageService.php | 8 +- lib/TaskProcessing/ContextChatProvider.php | 9 +- lib/Type/ActionType.php | 28 ++ lib/Type/ScopeType.php | 1 + lib/Type/Source.php | 5 +- lib/Type/UpdateAccessOp.php | 20 + tests/integration/ContentManagerTest.php | 11 +- .../integration/ProviderConfigServiceTest.php | 1 + 49 files changed, 1067 insertions(+), 540 deletions(-) create mode 100644 lib/BackgroundJobs/ActionJob.php delete mode 100644 lib/BackgroundJobs/DeleteJob.php rename lib/Db/{QueueDelete.php => QueueAction.php} (66%) rename lib/Db/{QueueDeleteMapper.php => QueueActionMapper.php} (73%) create mode 100644 lib/Listener/UserDeletedListener.php create mode 100644 lib/Migration/Version4000Date20241202141755.php create mode 100644 lib/Migration/Version4000Date20241206135634.php rename lib/{Type/DeleteContext.php => Public/UpdateAccessOp.php} (52%) create mode 100644 lib/Service/ActionService.php delete mode 100644 lib/Service/DeleteService.php create mode 100644 lib/Type/ActionType.php create mode 100644 lib/Type/UpdateAccessOp.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 58866f3..6f4835e 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -1,4 +1,5 @@ =8.1", - "phpunit/php-file-iterator": "^4.0", - "phpunit/php-text-template": "^3.0", - "sebastian/code-unit-reverse-lookup": "^3.0", - "sebastian/complexity": "^3.0", - "sebastian/environment": "^6.0", - "sebastian/lines-of-code": "^2.0", - "sebastian/version": "^4.0", - "theseer/tokenizer": "^1.2.0" + "phpunit/php-file-iterator": "^4.1.0", + "phpunit/php-text-template": "^3.0.1", + "sebastian/code-unit-reverse-lookup": "^3.0.0", + "sebastian/complexity": "^3.2.0", + "sebastian/environment": "^6.1.0", + "sebastian/lines-of-code": "^2.0.2", + "sebastian/version": "^4.0.1", + "theseer/tokenizer": "^1.2.3" }, "require-dev": { "phpunit/phpunit": "^10.1" @@ -420,7 +471,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "10.1-dev" + "dev-main": "10.1.x-dev" } }, "autoload": { @@ -449,7 +500,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.15" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.16" }, "funding": [ { @@ -457,7 +508,7 @@ "type": "github" } ], - "time": "2024-06-29T08:25:15+00:00" + "time": "2024-08-22T04:31:57+00:00" }, { "name": "phpunit/php-file-iterator", @@ -704,16 +755,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.27", + "version": "10.5.38", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "2425f713b2a5350568ccb1a2d3984841a23e83c5" + "reference": "a86773b9e887a67bc53efa9da9ad6e3f2498c132" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/2425f713b2a5350568ccb1a2d3984841a23e83c5", - "reference": "2425f713b2a5350568ccb1a2d3984841a23e83c5", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/a86773b9e887a67bc53efa9da9ad6e3f2498c132", + "reference": "a86773b9e887a67bc53efa9da9ad6e3f2498c132", "shasum": "" }, "require": { @@ -727,14 +778,14 @@ "phar-io/manifest": "^2.0.4", "phar-io/version": "^3.2.1", "php": ">=8.1", - "phpunit/php-code-coverage": "^10.1.15", + "phpunit/php-code-coverage": "^10.1.16", "phpunit/php-file-iterator": "^4.1.0", "phpunit/php-invoker": "^4.0.0", "phpunit/php-text-template": "^3.0.1", "phpunit/php-timer": "^6.0.0", "sebastian/cli-parser": "^2.0.1", "sebastian/code-unit": "^2.0.0", - "sebastian/comparator": "^5.0.1", + "sebastian/comparator": "^5.0.3", "sebastian/diff": "^5.1.1", "sebastian/environment": "^6.1.0", "sebastian/exporter": "^5.1.2", @@ -785,7 +836,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.27" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.38" }, "funding": [ { @@ -801,20 +852,20 @@ "type": "tidelift" } ], - "time": "2024-07-10T11:48:06+00:00" + "time": "2024-10-28T13:06:21+00:00" }, { "name": "psalm/phar", - "version": "5.25.0", + "version": "5.26.1", "source": { "type": "git", "url": "https://github.com/psalm/phar.git", - "reference": "d42708449bd2d99ec6509924332fd94263974b20" + "reference": "8a38e7ad04499a0ccd2c506fd1da6fc01fff4547" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/psalm/phar/zipball/d42708449bd2d99ec6509924332fd94263974b20", - "reference": "d42708449bd2d99ec6509924332fd94263974b20", + "url": "https://api.github.com/repos/psalm/phar/zipball/8a38e7ad04499a0ccd2c506fd1da6fc01fff4547", + "reference": "8a38e7ad04499a0ccd2c506fd1da6fc01fff4547", "shasum": "" }, "require": { @@ -834,9 +885,9 @@ "description": "Composer-based Psalm Phar", "support": { "issues": "https://github.com/psalm/phar/issues", - "source": "https://github.com/psalm/phar/tree/5.25.0" + "source": "https://github.com/psalm/phar/tree/5.26.1" }, - "time": "2024-06-19T20:02:02+00:00" + "time": "2024-09-09T16:22:43+00:00" }, { "name": "psr/clock", @@ -991,30 +1042,30 @@ }, { "name": "psr/log", - "version": "1.1.4", + "version": "3.0.2", "source": { "type": "git", "url": "https://github.com/php-fig/log.git", - "reference": "d49695b909c3b7628b6289db5479a1c204601f11" + "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/d49695b909c3b7628b6289db5479a1c204601f11", - "reference": "d49695b909c3b7628b6289db5479a1c204601f11", + "url": "https://api.github.com/repos/php-fig/log/zipball/f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", + "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", "shasum": "" }, "require": { - "php": ">=5.3.0" + "php": ">=8.0.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.1.x-dev" + "dev-master": "3.x-dev" } }, "autoload": { "psr-4": { - "Psr\\Log\\": "Psr/Log/" + "Psr\\Log\\": "src" } }, "notification-url": "https://packagist.org/downloads/", @@ -1035,9 +1086,9 @@ "psr-3" ], "support": { - "source": "https://github.com/php-fig/log/tree/1.1.4" + "source": "https://github.com/php-fig/log/tree/3.0.2" }, - "time": "2021-05-03T11:20:27+00:00" + "time": "2024-09-11T13:17:53+00:00" }, { "name": "roave/security-advisories", @@ -1045,23 +1096,23 @@ "source": { "type": "git", "url": "https://github.com/Roave/SecurityAdvisories.git", - "reference": "80e489d7669e5f7a4c56ca250489a0450d3d3219" + "reference": "dcb2bdb48e1d9b0b5b1c333b61f49772aee879ff" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Roave/SecurityAdvisories/zipball/80e489d7669e5f7a4c56ca250489a0450d3d3219", - "reference": "80e489d7669e5f7a4c56ca250489a0450d3d3219", + "url": "https://api.github.com/repos/Roave/SecurityAdvisories/zipball/dcb2bdb48e1d9b0b5b1c333b61f49772aee879ff", + "reference": "dcb2bdb48e1d9b0b5b1c333b61f49772aee879ff", "shasum": "" }, "conflict": { "3f/pygmentize": "<1.2", - "admidio/admidio": "<4.2.13", + "admidio/admidio": "<4.3.12", "adodb/adodb-php": "<=5.20.20|>=5.21,<=5.21.3", "aheinze/cockpit": "<2.2", - "aimeos/ai-admin-graphql": ">=2022.04.1,<2022.10.10|>=2023.04.1,<2023.10.6|>=2024.04.1,<2024.04.6", + "aimeos/ai-admin-graphql": ">=2022.04.1,<2022.10.10|>=2023.04.1,<2023.10.6|>=2024.04.1,<2024.07.2", "aimeos/ai-admin-jsonadm": "<2020.10.13|>=2021.04.1,<2021.10.6|>=2022.04.1,<2022.10.3|>=2023.04.1,<2023.10.4|==2024.04.1", "aimeos/ai-client-html": ">=2020.04.1,<2020.10.27|>=2021.04.1,<2021.10.22|>=2022.04.1,<2022.10.13|>=2023.04.1,<2023.10.15|>=2024.04.1,<2024.04.7", - "aimeos/ai-controller-frontend": "<2020.10.15|>=2021.04.1,<2021.10.8|>=2022.04.1,<2022.10.8|>=2023.04.1,<2023.10.9", + "aimeos/ai-controller-frontend": "<2020.10.15|>=2021.04.1,<2021.10.8|>=2022.04.1,<2022.10.8|>=2023.04.1,<2023.10.9|==2024.04.1", "aimeos/aimeos-core": ">=2022.04.1,<2022.10.17|>=2023.04.1,<2023.10.17|>=2024.04.1,<2024.04.7", "aimeos/aimeos-typo3": "<19.10.12|>=20,<20.10.5", "airesvsg/acf-to-rest-api": "<=3.1", @@ -1070,6 +1121,7 @@ "alextselegidis/easyappointments": "<1.5", "alterphp/easyadmin-extension-bundle": ">=1.2,<1.2.11|>=1.3,<1.3.1", "amazing/media2click": ">=1,<1.3.3", + "ameos/ameos_tarteaucitron": "<1.2.23", "amphp/artax": "<1.0.6|>=2,<2.0.6", "amphp/http": "<=1.7.2|>=2,<=2.1", "amphp/http-client": ">=4,<4.4", @@ -1088,13 +1140,14 @@ "athlon1600/php-proxy-app": "<=3", "austintoddj/canvas": "<=3.4.2", "auth0/wordpress": "<=4.6", - "automad/automad": "<=1.10.9", + "automad/automad": "<2.0.0.0-alpha5", "automattic/jetpack": "<9.8", "awesome-support/awesome-support": "<=6.0.7", "aws/aws-sdk-php": "<3.288.1", "azuracast/azuracast": "<0.18.3", - "backdrop/backdrop": "<1.24.2", + "backdrop/backdrop": "<1.27.3|>=1.28,<1.28.2", "backpack/crud": "<3.4.9", + "backpack/filemanager": "<2.0.2|>=3,<3.0.9", "bacula-web/bacula-web": "<8.0.0.0-RC2-dev", "badaso/core": "<2.7", "bagisto/bagisto": "<2.1", @@ -1102,13 +1155,13 @@ "barrelstrength/sprout-forms": "<3.9", "barryvdh/laravel-translation-manager": "<0.6.2", "barzahlen/barzahlen-php": "<2.0.1", - "baserproject/basercms": "<5.0.9", + "baserproject/basercms": "<=5.1.1", "bassjobsen/bootstrap-3-typeahead": ">4.0.2", "bbpress/bbpress": "<2.6.5", "bcosca/fatfree": "<3.7.2", "bedita/bedita": "<4", "bigfork/silverstripe-form-capture": ">=3,<3.1.1", - "billz/raspap-webgui": "<2.9.5", + "billz/raspap-webgui": "<=3.1.4", "bk2k/bootstrap-package": ">=7.1,<7.1.2|>=8,<8.0.8|>=9,<9.0.4|>=9.1,<9.1.3|>=10,<10.0.10|>=11,<11.0.3", "blueimp/jquery-file-upload": "==6.4.4", "bmarshall511/wordpress_zero_spam": "<5.2.13", @@ -1147,31 +1200,34 @@ "codeigniter4/shield": "<1.0.0.0-beta8", "codiad/codiad": "<=2.8.4", "composer/composer": "<1.10.27|>=2,<2.2.24|>=2.3,<2.7.7", - "concrete5/concrete5": "<9.2.8", + "concrete5/concrete5": "<9.3.4", "concrete5/core": "<8.5.8|>=9,<9.1", "contao-components/mediaelement": ">=2.14.2,<2.21.1", "contao/comments-bundle": ">=2,<4.13.40|>=5.0.0.0-RC1-dev,<5.3.4", - "contao/contao": ">=3,<3.5.37|>=4,<4.4.56|>=4.5,<4.9.40|>=4.10,<4.11.7|>=4.13,<4.13.21|>=5.1,<5.1.4", + "contao/contao": "<=5.4.1", "contao/core": "<3.5.39", - "contao/core-bundle": "<4.13.40|>=5,<5.3.4", + "contao/core-bundle": "<4.13.49|>=5,<5.3.15|>=5.4,<5.4.3", "contao/listing-bundle": ">=3,<=3.5.30|>=4,<4.4.8", "contao/managed-edition": "<=1.5", "corveda/phpsandbox": "<1.3.5", "cosenary/instagram": "<=2.3", - "craftcms/cms": "<4.6.2", + "craftcms/cms": "<=4.12.6.1|>=5,<=5.4.7.1", "croogo/croogo": "<4", "cuyz/valinor": "<0.12", + "czim/file-handling": "<1.5|>=2,<2.3", "czproject/git-php": "<4.0.3", + "damienharper/auditor-bundle": "<5.2.6", "dapphp/securimage": "<3.6.6", "darylldoyle/safe-svg": "<1.9.10", "datadog/dd-trace": ">=0.30,<0.30.2", "datatables/datatables": "<1.10.10", "david-garcia/phpwhois": "<=4.3.1", "dbrisinajumi/d2files": "<1", - "dcat/laravel-admin": "<=2.1.3.0-beta", + "dcat/laravel-admin": "<=2.1.3", "derhansen/fe_change_pwd": "<2.0.5|>=3,<3.0.3", "derhansen/sf_event_mgt": "<4.3.1|>=5,<5.1.1|>=7,<7.4", "desperado/xml-bundle": "<=0.1.7", + "dev-lancer/minecraft-motd-parser": "<=1.0.5", "devgroup/dotplant": "<2020.09.14-dev", "directmailteam/direct-mail": "<6.0.3|>=7,<7.0.3|>=8,<9.5.2", "doctrine/annotations": "<1.2.7", @@ -1186,8 +1242,9 @@ "dolibarr/dolibarr": "<19.0.2", "dompdf/dompdf": "<2.0.4", "doublethreedigital/guest-entries": "<3.1.2", - "drupal/core": ">=6,<6.38|>=7,<7.96|>=8,<10.1.8|>=10.2,<10.2.2", - "drupal/drupal": ">=5,<5.11|>=6,<6.38|>=7,<7.80|>=8,<8.9.16|>=9,<9.1.12|>=9.2,<9.2.4", + "drupal/core": ">=6,<6.38|>=7,<7.96|>=8,<10.2.10|>=10.3,<10.3.6|>=11,<11.0.5", + "drupal/core-recommended": ">=8,<10.2.9|>=10.3,<10.3.6|>=11,<11.0.5", + "drupal/drupal": ">=5,<5.11|>=6,<6.38|>=7,<7.80|>=8,<10.2.9|>=10.3,<10.3.6|>=11,<11.0.5", "duncanmcclean/guest-entries": "<3.1.2", "dweeves/magmi": "<=0.7.24", "ec-cube/ec-cube": "<2.4.4|>=2.11,<=2.17.1|>=3,<=3.0.18.0-patch4|>=4,<=4.1.2", @@ -1211,29 +1268,34 @@ "ezsystems/ezdemo-ls-extension": ">=5.4,<5.4.2.1-dev", "ezsystems/ezfind-ls": ">=5.3,<5.3.6.1-dev|>=5.4,<5.4.11.1-dev|>=2017.12,<2017.12.0.1-dev", "ezsystems/ezplatform": "<=1.13.6|>=2,<=2.5.24", - "ezsystems/ezplatform-admin-ui": ">=1.3,<1.3.5|>=1.4,<1.4.6|>=1.5,<1.5.29|>=2.3,<2.3.26", + "ezsystems/ezplatform-admin-ui": ">=1.3,<1.3.5|>=1.4,<1.4.6|>=1.5,<1.5.29|>=2.3,<2.3.26|>=3.3,<3.3.39", "ezsystems/ezplatform-admin-ui-assets": ">=4,<4.2.1|>=5,<5.0.1|>=5.1,<5.1.1", "ezsystems/ezplatform-graphql": ">=1.0.0.0-RC1-dev,<1.0.13|>=2.0.0.0-beta1,<2.3.12", + "ezsystems/ezplatform-http-cache": "<2.3.16", "ezsystems/ezplatform-kernel": "<1.2.5.1-dev|>=1.3,<1.3.35", "ezsystems/ezplatform-rest": ">=1.2,<=1.2.2|>=1.3,<1.3.8", - "ezsystems/ezplatform-richtext": ">=2.3,<2.3.7.1-dev", + "ezsystems/ezplatform-richtext": ">=2.3,<2.3.7.1-dev|>=3.3,<3.3.40", "ezsystems/ezplatform-solr-search-engine": ">=1.7,<1.7.12|>=2,<2.0.2|>=3.3,<3.3.15", "ezsystems/ezplatform-user": ">=1,<1.0.1", "ezsystems/ezpublish-kernel": "<6.13.8.2-dev|>=7,<7.5.31", "ezsystems/ezpublish-legacy": "<=2017.12.7.3|>=2018.6,<=2019.03.5.1", "ezsystems/platform-ui-assets-bundle": ">=4.2,<4.2.3", "ezsystems/repository-forms": ">=2.3,<2.3.2.1-dev|>=2.5,<2.5.15", - "ezyang/htmlpurifier": "<4.1.1", + "ezyang/htmlpurifier": "<=4.2", "facade/ignition": "<1.16.15|>=2,<2.4.2|>=2.5,<2.5.2", "facturascripts/facturascripts": "<=2022.08", "fastly/magento2": "<1.2.26", "feehi/cms": "<=2.1.1", "feehi/feehicms": "<=2.1.1", "fenom/fenom": "<=2.12.1", + "filament/actions": ">=3.2,<3.2.123", + "filament/infolists": ">=3,<3.2.115", + "filament/tables": ">=3,<3.2.115", "filegator/filegator": "<7.8", "filp/whoops": "<2.1.13", "fineuploader/php-traditional-server": "<=1.2.2", "firebase/php-jwt": "<6", + "fisharebest/webtrees": "<=2.1.18", "fixpunkt/fp-masterquiz": "<2.2.1|>=3,<3.5.2", "fixpunkt/fp-newsletter": "<1.1.1|>=2,<2.1.2|>=2.2,<3.2.6", "flarum/core": "<1.8.5", @@ -1256,19 +1318,19 @@ "friendsofsymfony/rest-bundle": ">=1.2,<1.2.2", "friendsofsymfony/user-bundle": ">=1,<1.3.5", "friendsofsymfony1/swiftmailer": ">=4,<5.4.13|>=6,<6.2.5", - "friendsofsymfony1/symfony1": ">=1.1,<1.15.19", + "friendsofsymfony1/symfony1": ">=1.1,<1.5.19", "friendsoftypo3/mediace": ">=7.6.2,<7.6.5", "friendsoftypo3/openid": ">=4.5,<4.5.31|>=4.7,<4.7.16|>=6,<6.0.11|>=6.1,<6.1.6", "froala/wysiwyg-editor": "<3.2.7|>=4.0.1,<=4.1.3", - "froxlor/froxlor": "<2.1.9", + "froxlor/froxlor": "<=2.2.0.0-RC3", "frozennode/administrator": "<=5.0.12", "fuel/core": "<1.8.1", - "funadmin/funadmin": "<=3.2|>=3.3.2,<=3.3.3", + "funadmin/funadmin": "<=5.0.2", "gaoming13/wechat-php-sdk": "<=1.10.2", "genix/cms": "<=1.1.11", "getformwork/formwork": "<1.13.1|==2.0.0.0-beta1", "getgrav/grav": "<1.7.46", - "getkirby/cms": "<4.1.1", + "getkirby/cms": "<=3.6.6.5|>=3.7,<=3.7.5.4|>=3.8,<=3.8.4.3|>=3.9,<=3.9.8.1|>=3.10,<=3.10.1|>=4,<=4.3", "getkirby/kirby": "<=2.5.12", "getkirby/panel": "<2.5.14", "getkirby/starterkit": "<=3.7.0.2", @@ -1294,10 +1356,12 @@ "hov/jobfair": "<1.0.13|>=2,<2.0.2", "httpsoft/http-message": "<1.0.12", "hyn/multi-tenant": ">=5.6,<5.7.2", - "ibexa/admin-ui": ">=4.2,<4.2.3", + "ibexa/admin-ui": ">=4.2,<4.2.3|>=4.6,<4.6.14", "ibexa/core": ">=4,<4.0.7|>=4.1,<4.1.4|>=4.2,<4.2.3|>=4.5,<4.5.6|>=4.6,<4.6.2", + "ibexa/fieldtype-richtext": ">=4.6,<4.6.10", "ibexa/graphql": ">=2.5,<2.5.31|>=3.3,<3.3.28|>=4.2,<4.2.3", - "ibexa/post-install": "<=1.0.4", + "ibexa/http-cache": ">=4.6,<4.6.14", + "ibexa/post-install": "<1.0.16|>=4.6,<4.6.14", "ibexa/solr": ">=4.5,<4.5.4", "ibexa/user": ">=4,<4.4.3", "icecoder/icecoder": "<=8.1", @@ -1314,9 +1378,11 @@ "in2code/femanager": "<5.5.3|>=6,<6.3.4|>=7,<7.2.3", "in2code/ipandlanguageredirect": "<5.1.2", "in2code/lux": "<17.6.1|>=18,<24.0.2", + "in2code/powermail": "<7.5.1|>=8,<8.5.1|>=9,<10.9.1|>=11,<12.4.1", "innologi/typo3-appointments": "<2.0.6", "intelliants/subrion": "<4.2.2", "inter-mediator/inter-mediator": "==5.5", + "ipl/web": "<0.10.1", "islandora/islandora": ">=2,<2.4.1", "ivankristianto/phpwhois": "<=4.3", "jackalope/jackalope-doctrine-dbal": "<1.7.4", @@ -1342,21 +1408,24 @@ "kelvinmo/simplexrd": "<3.1.1", "kevinpapst/kimai2": "<1.16.7", "khodakhah/nodcms": "<=3", - "kimai/kimai": "<2.16", + "kimai/kimai": "<=2.20.1", "kitodo/presentation": "<3.2.3|>=3.3,<3.3.4", "klaviyo/magento2-extension": ">=1,<3", "knplabs/knp-snappy": "<=1.4.2", "kohana/core": "<3.3.3", - "krayin/laravel-crm": "<1.2.2", + "krayin/laravel-crm": "<=1.3", "kreait/firebase-php": ">=3.2,<3.8.1", "kumbiaphp/kumbiapp": "<=1.1.1", "la-haute-societe/tcpdf": "<6.2.22", "laminas/laminas-diactoros": "<2.18.1|==2.19|==2.20|==2.21|==2.22|==2.23|>=2.24,<2.24.2|>=2.25,<2.25.2", "laminas/laminas-form": "<2.17.1|>=3,<3.0.2|>=3.1,<3.1.1", "laminas/laminas-http": "<2.14.2", + "lara-zeus/artemis": ">=1,<=1.0.6", + "lara-zeus/dynamic-dashboard": ">=3,<=3.0.1", "laravel/fortify": "<1.11.1", - "laravel/framework": "<6.20.44|>=7,<7.30.6|>=8,<8.75", + "laravel/framework": "<6.20.45|>=7,<7.30.7|>=8,<8.83.28|>=9,<9.52.17|>=10,<10.48.23|>=11,<11.31", "laravel/laravel": ">=5.4,<5.4.22", + "laravel/reverb": "<1.4", "laravel/socialite": ">=1,<2.0.10", "latte/latte": "<2.10.8", "lavalite/cms": "<=9|==10.1", @@ -1369,13 +1438,14 @@ "librenms/librenms": "<2017.08.18", "liftkit/database": "<2.13.2", "lightsaml/lightsaml": "<1.3.5", - "limesurvey/limesurvey": "<3.27.19", + "limesurvey/limesurvey": "<6.5.12", "livehelperchat/livehelperchat": "<=3.91", - "livewire/livewire": ">2.2.4,<2.2.6|>=3.3.5,<3.4.9", + "livewire/livewire": "<2.12.7|>=3.0.0.0-beta1,<3.5.2", "lms/routes": "<2.1.1", "localizationteam/l10nmgr": "<7.4|>=8,<8.7|>=9,<9.2", "luyadev/yii-helpers": "<1.2.1", - "magento/community-edition": "<2.4.5|==2.4.5|>=2.4.5.0-patch1,<2.4.5.0-patch8|==2.4.6|>=2.4.6.0-patch1,<2.4.6.0-patch6|==2.4.7", + "maestroerror/php-heic-to-jpg": "<1.0.5", + "magento/community-edition": "<2.4.5|==2.4.5|>=2.4.5.0-patch1,<2.4.5.0-patch10|==2.4.6|>=2.4.6.0-patch1,<2.4.6.0-patch8|>=2.4.7.0-beta1,<2.4.7.0-patch3", "magento/core": "<=1.9.4.5", "magento/magento1ce": "<1.9.4.3-dev", "magento/magento1ee": ">=1,<1.14.4.3-dev", @@ -1383,12 +1453,16 @@ "magneto/core": "<1.9.4.4-dev", "maikuolan/phpmussel": ">=1,<1.6", "mainwp/mainwp": "<=4.4.3.3", - "mantisbt/mantisbt": "<2.26.2", + "mantisbt/mantisbt": "<=2.26.3", "marcwillmann/turn": "<0.3.3", "matyhtf/framework": "<3.0.6", - "mautic/core": "<4.4.12|>=5.0.0.0-alpha,<5.0.4", + "mautic/core": "<4.4.13|>=5,<5.1.1", + "mautic/core-lib": ">=1.0.0.0-beta,<4.4.13|>=5.0.0.0-alpha,<5.1.1", + "maximebf/debugbar": "<1.19", "mdanter/ecc": "<2", - "mediawiki/core": "<1.36.2", + "mediawiki/abuse-filter": "<1.39.9|>=1.40,<1.41.3|>=1.42,<1.42.2", + "mediawiki/cargo": "<3.6.1", + "mediawiki/core": "<1.39.5|==1.40", "mediawiki/matomo": "<2.4.3", "mediawiki/semantic-media-wiki": "<4.0.2", "melisplatform/melis-asset-manager": "<5.0.1", @@ -1399,7 +1473,7 @@ "microsoft/microsoft-graph": ">=1.16,<1.109.1|>=2,<2.0.1", "microsoft/microsoft-graph-beta": "<2.0.1", "microsoft/microsoft-graph-core": "<2.0.2", - "microweber/microweber": "<=2.0.4", + "microweber/microweber": "<=2.0.16", "mikehaertl/php-shellcommand": "<1.6.1", "miniorange/miniorange-saml": "<1.4.3", "mittwald/typo3_forum": "<1.2.1", @@ -1408,7 +1482,7 @@ "mojo42/jirafeau": "<4.4", "mongodb/mongodb": ">=1,<1.9.2", "monolog/monolog": ">=1.8,<1.12", - "moodle/moodle": "<4.3.5|>=4.4.0.0-beta,<4.4.1", + "moodle/moodle": "<4.3.8|>=4.4,<4.4.4", "mos/cimage": "<0.7.19", "movim/moxl": ">=0.8,<=0.10", "movingbytes/social-network": "<=1.2.1", @@ -1421,6 +1495,7 @@ "munkireport/softwareupdate": "<1.6", "mustache/mustache": ">=2,<2.14.1", "namshi/jose": "<2.2", + "nategood/httpful": "<1", "neoan3-apps/template": "<1.1.1", "neorazorx/facturascripts": "<2022.04", "neos/flow": ">=1,<1.0.4|>=1.1,<1.1.1|>=2,<2.0.1|>=2.3,<2.3.16|>=3,<3.0.12|>=3.1,<3.1.10|>=3.2,<3.2.13|>=3.3,<3.3.13|>=4,<4.0.6", @@ -1443,7 +1518,7 @@ "nzo/url-encryptor-bundle": ">=4,<4.3.2|>=5,<5.0.1", "october/backend": "<1.1.2", "october/cms": "<1.0.469|==1.0.469|==1.0.471|==1.1.1", - "october/october": "<=3.4.4", + "october/october": "<=3.6.4", "october/rain": "<1.0.472|>=1.1,<1.1.2", "october/system": "<1.0.476|>=1.1,<1.1.12|>=2,<2.2.34|>=3,<3.5.15", "omeka/omeka-s": "<4.0.3", @@ -1452,16 +1527,17 @@ "open-web-analytics/open-web-analytics": "<1.7.4", "opencart/opencart": ">=0", "openid/php-openid": "<2.3", - "openmage/magento-lts": "<20.5", + "openmage/magento-lts": "<20.10.1", "opensolutions/vimbadmin": "<=3.0.15", "opensource-workshop/connect-cms": "<1.7.2|>=2,<2.3.2", - "orchid/platform": ">=9,<9.4.4|>=14.0.0.0-alpha4,<14.5", + "orchid/platform": ">=8,<14.43", "oro/calendar-bundle": ">=4.2,<=4.2.6|>=5,<=5.0.6|>=5.1,<5.1.1", "oro/commerce": ">=4.1,<5.0.11|>=5.1,<5.1.1", "oro/crm": ">=1.7,<1.7.4|>=3.1,<4.1.17|>=4.2,<4.2.7", "oro/crm-call-bundle": ">=4.2,<=4.2.5|>=5,<5.0.4|>=5.1,<5.1.1", "oro/customer-portal": ">=4.1,<=4.1.13|>=4.2,<=4.2.10|>=5,<=5.0.11|>=5.1,<=5.1.3", "oro/platform": ">=1.7,<1.7.4|>=3.1,<3.1.29|>=4.1,<4.1.17|>=4.2,<=4.2.10|>=5,<=5.0.12|>=5.1,<=5.1.3", + "oveleon/contao-cookiebar": "<1.16.3|>=2,<2.1.3", "oxid-esales/oxideshop-ce": "<4.5", "oxid-esales/paymorrow-module": ">=1,<1.0.2|>=2,<2.0.1", "packbackbooks/lti-1-3-php-library": "<5", @@ -1485,7 +1561,7 @@ "phenx/php-svg-lib": "<0.5.2", "php-censor/php-censor": "<2.0.13|>=2.1,<2.1.5", "php-mod/curl": "<2.3.2", - "phpbb/phpbb": "<3.2.10|>=3.3,<3.3.1", + "phpbb/phpbb": "<3.3.11", "phpems/phpems": ">=6,<=6.1.3", "phpfastcache/phpfastcache": "<6.1.5|>=7,<7.1.2|>=8,<8.0.7", "phpmailer/phpmailer": "<6.5", @@ -1493,8 +1569,8 @@ "phpmyadmin/phpmyadmin": "<5.2.1", "phpmyfaq/phpmyfaq": "<3.2.5|==3.2.5", "phpoffice/common": "<0.2.9", - "phpoffice/phpexcel": "<1.8", - "phpoffice/phpspreadsheet": "<1.16", + "phpoffice/phpexcel": "<1.8.1", + "phpoffice/phpspreadsheet": "<1.29.4|>=2,<2.1.3|>=2.2,<2.3.2|>=3.3,<3.4", "phpseclib/phpseclib": "<2.0.47|>=3,<3.0.36", "phpservermon/phpservermon": "<3.6", "phpsysinfo/phpsysinfo": "<3.4.3", @@ -1503,9 +1579,10 @@ "phpxmlrpc/extras": "<0.6.1", "phpxmlrpc/phpxmlrpc": "<4.9.2", "pi/pi": "<=2.5", - "pimcore/admin-ui-classic-bundle": "<=1.4.2", + "pimcore/admin-ui-classic-bundle": "<1.5.4", "pimcore/customer-management-framework-bundle": "<4.0.6", "pimcore/data-hub": "<1.2.4", + "pimcore/data-importer": "<1.8.9|>=1.9,<1.9.3", "pimcore/demo": "<10.3", "pimcore/ecommerce-framework-bundle": "<1.0.10", "pimcore/perspective-editor": "<1.5.1", @@ -1527,15 +1604,16 @@ "prestashop/ps_facetedsearch": "<3.4.1", "prestashop/ps_linklist": "<3.1", "privatebin/privatebin": "<1.4|>=1.5,<1.7.4", - "processwire/processwire": "<=3.0.210", + "processwire/processwire": "<=3.0.229", "propel/propel": ">=2.0.0.0-alpha1,<=2.0.0.0-alpha7", "propel/propel1": ">=1,<=1.7.1", - "pterodactyl/panel": "<1.11.6", + "pterodactyl/panel": "<1.11.8", "ptheofan/yii2-statemachine": ">=2.0.0.0-RC1-dev,<=2", "ptrofimov/beanstalk_console": "<1.7.14", "pubnub/pubnub": "<6.1", "pusher/pusher-php-server": "<2.2.1", "pwweb/laravel-core": "<=0.3.6.0-beta", + "pxlrbt/filament-excel": "<1.1.14|>=2.0.0.0-alpha,<2.3.3", "pyrocms/pyrocms": "<=3.9.1", "qcubed/qcubed": "<=3.1.1", "quickapps/cms": "<=2.0.0.0-beta2", @@ -1546,7 +1624,7 @@ "rap2hpoutre/laravel-log-viewer": "<0.13", "react/http": ">=0.7,<1.9", "really-simple-plugins/complianz-gdpr": "<6.4.2", - "redaxo/source": "<=5.15.1", + "redaxo/source": "<5.18", "remdex/livehelperchat": "<4.29", "reportico-web/reportico": "<=8.1", "rhukster/dom-sanitizer": "<1.0.7", @@ -1558,13 +1636,14 @@ "s-cart/s-cart": "<6.9", "sabberworm/php-css-parser": ">=1,<1.0.1|>=2,<2.0.1|>=3,<3.0.1|>=4,<4.0.1|>=5,<5.0.9|>=5.1,<5.1.3|>=5.2,<5.2.1|>=6,<6.0.2|>=7,<7.0.4|>=8,<8.0.1|>=8.1,<8.1.1|>=8.2,<8.2.1|>=8.3,<8.3.1", "sabre/dav": ">=1.6,<1.7.11|>=1.8,<1.8.9", + "samwilson/unlinked-wikibase": "<1.39.6|>=1.40,<1.40.2|>=1.41,<1.41.1", "scheb/two-factor-bundle": "<3.26|>=4,<4.11", "sensiolabs/connect": "<4.2.3", "serluck/phpwhois": "<=4.2.6", "sfroemken/url_redirect": "<=1.2.1", "sheng/yiicms": "<=1.2", - "shopware/core": "<6.5.8.8-dev|>=6.6.0.0-RC1-dev,<6.6.1", - "shopware/platform": "<6.5.8.8-dev|>=6.6.0.0-RC1-dev,<6.6.1", + "shopware/core": "<=6.5.8.12|>=6.6,<=6.6.5", + "shopware/platform": "<=6.5.8.12|>=6.6,<=6.6.5", "shopware/production": "<=6.3.5.2", "shopware/shopware": "<=5.7.17", "shopware/storefront": "<=6.4.8.1|>=6.5.8,<6.5.8.7-dev", @@ -1589,11 +1668,13 @@ "silverstripe/userforms": "<3|>=5,<5.4.2", "silverstripe/versioned-admin": ">=1,<1.11.1", "simple-updates/phpwhois": "<=1", - "simplesamlphp/saml2": "<1.10.6|>=2,<2.3.8|>=3,<3.1.4|==5.0.0.0-alpha12", + "simplesamlphp/saml2": "<4.6.14|==5.0.0.0-alpha12", + "simplesamlphp/saml2-legacy": "<4.6.14", "simplesamlphp/simplesamlphp": "<1.18.6", "simplesamlphp/simplesamlphp-module-infocard": "<1.0.1", "simplesamlphp/simplesamlphp-module-openid": "<1", "simplesamlphp/simplesamlphp-module-openidprovider": "<0.9", + "simplesamlphp/xml-common": "<1.20", "simplesamlphp/xml-security": "==1.6.11", "simplito/elliptic-php": "<1.0.6", "sitegeist/fluid-components": "<3.5", @@ -1602,24 +1683,26 @@ "slim/slim": "<2.6", "slub/slub-events": "<3.0.3", "smarty/smarty": "<4.5.3|>=5,<5.1.1", - "snipe/snipe-it": "<6.4.2", + "snipe/snipe-it": "<=7.0.13", "socalnick/scn-social-auth": "<1.15.2", "socialiteproviders/steam": "<1.1", "spatie/browsershot": "<3.57.4", "spatie/image-optimizer": "<1.7.3", + "spencer14420/sp-php-email-handler": "<1", "spipu/html2pdf": "<5.2.8", "spoon/library": "<1.4.1", "spoonity/tcpdf": "<6.2.22", "squizlabs/php_codesniffer": ">=1,<2.8.1|>=3,<3.0.1", "ssddanbrown/bookstack": "<24.05.1", - "statamic/cms": "<4.46|>=5.3,<5.6.2", + "starcitizentools/citizen-skin": ">=2.6.3,<2.31", + "statamic/cms": "<=5.16", "stormpath/sdk": "<9.9.99", - "studio-42/elfinder": "<2.1.62", + "studio-42/elfinder": "<=2.1.64", "studiomitte/friendlycaptcha": "<0.1.4", "subhh/libconnect": "<7.0.8|>=8,<8.1", "sukohi/surpass": "<1", "sulu/form-bundle": ">=2,<2.5.3", - "sulu/sulu": "<1.6.44|>=2,<2.4.17|>=2.5,<2.5.13", + "sulu/sulu": "<1.6.44|>=2,<2.5.21|>=2.6,<2.6.5", "sumocoders/framework-user-bundle": "<1.4", "superbig/craft-audit": "<3.0.2", "swag/paypal": "<5.4.4", @@ -1641,7 +1724,8 @@ "symfony/error-handler": ">=4.4,<4.4.4|>=5,<5.0.4", "symfony/form": ">=2.3,<2.3.35|>=2.4,<2.6.12|>=2.7,<2.7.50|>=2.8,<2.8.49|>=3,<3.4.20|>=4,<4.0.15|>=4.1,<4.1.9|>=4.2,<4.2.1", "symfony/framework-bundle": ">=2,<2.3.18|>=2.4,<2.4.8|>=2.5,<2.5.2|>=2.7,<2.7.51|>=2.8,<2.8.50|>=3,<3.4.26|>=4,<4.1.12|>=4.2,<4.2.7|>=5.3.14,<5.3.15|>=5.4.3,<5.4.4|>=6.0.3,<6.0.4", - "symfony/http-foundation": ">=2,<2.8.52|>=3,<3.4.35|>=4,<4.2.12|>=4.3,<4.3.8|>=4.4,<4.4.7|>=5,<5.0.7", + "symfony/http-client": ">=4.3,<5.4.47|>=6,<6.4.15|>=7,<7.1.8", + "symfony/http-foundation": "<5.4.46|>=6,<6.4.14|>=7,<7.1.7", "symfony/http-kernel": ">=2,<4.4.50|>=5,<5.4.20|>=6,<6.0.20|>=6.1,<6.1.12|>=6.2,<6.2.6", "symfony/intl": ">=2.7,<2.7.38|>=2.8,<2.8.31|>=3,<3.2.14|>=3.3,<3.3.13", "symfony/maker-bundle": ">=1.27,<1.29.2|>=1.30,<1.31.1", @@ -1649,20 +1733,22 @@ "symfony/phpunit-bridge": ">=2.8,<2.8.50|>=3,<3.4.26|>=4,<4.1.12|>=4.2,<4.2.7", "symfony/polyfill": ">=1,<1.10", "symfony/polyfill-php55": ">=1,<1.10", + "symfony/process": "<5.4.46|>=6,<6.4.14|>=7,<7.1.7", "symfony/proxy-manager-bridge": ">=2.7,<2.7.51|>=2.8,<2.8.50|>=3,<3.4.26|>=4,<4.1.12|>=4.2,<4.2.7", "symfony/routing": ">=2,<2.0.19", + "symfony/runtime": ">=5.3,<5.4.46|>=6,<6.4.14|>=7,<7.1.7", "symfony/security": ">=2,<2.7.51|>=2.8,<3.4.49|>=4,<4.4.24|>=5,<5.2.8", - "symfony/security-bundle": ">=2,<4.4.50|>=5,<5.4.20|>=6,<6.0.20|>=6.1,<6.1.12|>=6.2,<6.2.6", + "symfony/security-bundle": ">=2,<4.4.50|>=5,<5.4.20|>=6,<6.0.20|>=6.1,<6.1.12|>=6.2,<6.4.10|>=7,<7.0.10|>=7.1,<7.1.3", "symfony/security-core": ">=2.4,<2.6.13|>=2.7,<2.7.9|>=2.7.30,<2.7.32|>=2.8,<3.4.49|>=4,<4.4.24|>=5,<5.2.9", "symfony/security-csrf": ">=2.4,<2.7.48|>=2.8,<2.8.41|>=3,<3.3.17|>=3.4,<3.4.11|>=4,<4.0.11", "symfony/security-guard": ">=2.8,<3.4.48|>=4,<4.4.23|>=5,<5.2.8", - "symfony/security-http": ">=2.3,<2.3.41|>=2.4,<2.7.51|>=2.8,<2.8.50|>=3,<3.4.26|>=4,<4.2.12|>=4.3,<4.3.8|>=4.4,<4.4.7|>=5,<5.0.7|>=5.1,<5.2.8|>=5.3,<5.3.2|>=5.4,<5.4.31|>=6,<6.3.8", + "symfony/security-http": ">=2.3,<2.3.41|>=2.4,<2.7.51|>=2.8,<2.8.50|>=3,<3.4.26|>=4,<4.2.12|>=4.3,<4.3.8|>=4.4,<4.4.7|>=5,<5.0.7|>=5.1,<5.2.8|>=5.3,<5.4.47|>=6,<6.4.15|>=7,<7.1.8", "symfony/serializer": ">=2,<2.0.11|>=4.1,<4.4.35|>=5,<5.3.12", - "symfony/symfony": ">=2,<4.4.51|>=5,<5.4.31|>=6,<6.3.8", + "symfony/symfony": "<5.4.47|>=6,<6.4.15|>=7,<7.1.8", "symfony/translation": ">=2,<2.0.17", "symfony/twig-bridge": ">=2,<4.4.51|>=5,<5.4.31|>=6,<6.3.8", "symfony/ux-autocomplete": "<2.11.2", - "symfony/validator": ">=2,<2.0.24|>=2.1,<2.1.12|>=2.2,<2.2.5|>=2.3,<2.3.3", + "symfony/validator": "<5.4.43|>=6,<6.4.11|>=7,<7.1.4", "symfony/var-exporter": ">=4.2,<4.2.12|>=4.3,<4.3.8", "symfony/web-profiler-bundle": ">=2,<2.3.19|>=2.4,<2.4.9|>=2.5,<2.5.4", "symfony/webhook": ">=6.3,<6.3.8", @@ -1673,30 +1759,31 @@ "t3s/content-consent": "<1.0.3|>=2,<2.0.2", "tastyigniter/tastyigniter": "<3.3", "tcg/voyager": "<=1.4", - "tecnickcom/tcpdf": "<=6.7.4", + "tecnickcom/tcpdf": "<=6.7.5", "terminal42/contao-tablelookupwizard": "<3.3.5", "thelia/backoffice-default-template": ">=2.1,<2.1.2", "thelia/thelia": ">=2.1,<2.1.3", "theonedemon/phpwhois": "<=4.2.5", "thinkcmf/thinkcmf": "<6.0.8", - "thorsten/phpmyfaq": "<3.2.2", + "thorsten/phpmyfaq": "<4", "tikiwiki/tiki-manager": "<=17.1", "timber/timber": ">=0.16.6,<1.23.1|>=1.24,<1.24.1|>=2,<2.1", "tinymce/tinymce": "<7.2", "tinymighty/wiki-seo": "<1.2.2", "titon/framework": "<9.9.99", "tobiasbg/tablepress": "<=2.0.0.0-RC1", - "topthink/framework": "<6.0.17|>=6.1,<6.1.5|>=8,<8.0.4", + "topthink/framework": "<6.0.17|>=6.1,<=8.0.4", "topthink/think": "<=6.1.1", - "topthink/thinkphp": "<=3.2.3", + "topthink/thinkphp": "<=3.2.3|>=6.1.3,<=8.0.4", "torrentpier/torrentpier": "<=2.4.3", "tpwd/ke_search": "<4.0.3|>=4.1,<4.6.6|>=5,<5.0.2", - "tribalsystems/zenario": "<9.5.60602", + "tribalsystems/zenario": "<=9.7.61188", "truckersmp/phpwhois": "<=4.3.1", "ttskch/pagination-service-provider": "<1", - "twig/twig": "<1.44.7|>=2,<2.15.3|>=3,<3.4.3", + "twbs/bootstrap": "<=3.4.1|>=4,<=4.6.2", + "twig/twig": "<3.11.2|>=3.12,<3.14.1", "typo3/cms": "<9.5.29|>=10,<10.4.35|>=11,<11.5.23|>=12,<12.2", - "typo3/cms-backend": "<4.1.14|>=4.2,<4.2.15|>=4.3,<4.3.7|>=4.4,<4.4.4|>=7,<=7.6.50|>=8,<=8.7.39|>=9,<=9.5.24|>=10,<=10.4.13|>=11,<=11.1", + "typo3/cms-backend": "<4.1.14|>=4.2,<4.2.15|>=4.3,<4.3.7|>=4.4,<4.4.4|>=7,<=7.6.50|>=8,<=8.7.39|>=9,<=9.5.24|>=10,<10.4.46|>=11,<11.5.40|>=12,<12.4.21|>=13,<13.3.1", "typo3/cms-core": "<=8.7.56|>=9,<=9.5.47|>=10,<=10.4.44|>=11,<=11.5.36|>=12,<=12.4.14|>=13,<=13.1", "typo3/cms-extbase": "<6.2.24|>=7,<7.6.8|==8.1.1", "typo3/cms-fluid": "<4.3.4|>=4.4,<4.4.1", @@ -1713,6 +1800,7 @@ "ua-parser/uap-php": "<3.8", "uasoft-indonesia/badaso": "<=2.9.7", "unisharp/laravel-filemanager": "<2.6.4", + "unopim/unopim": "<0.1.5", "userfrosting/userfrosting": ">=0.3.1,<4.6.3", "usmanhalalit/pixie": "<1.0.3|>=2,<2.0.2", "uvdesk/community-skeleton": "<=1.1.1", @@ -1746,6 +1834,7 @@ "winter/wn-dusk-plugin": "<2.1", "winter/wn-system-module": "<1.2.4", "wintercms/winter": "<=1.2.3", + "wireui/wireui": "<1.19.3|>=2,<2.1.3", "woocommerce/woocommerce": "<6.6|>=8.8,<8.8.5|>=8.9,<8.9.3", "wp-cli/wp-cli": ">=0.12,<2.5", "wp-graphql/wp-graphql": "<=1.14.5", @@ -1757,7 +1846,7 @@ "xataface/xataface": "<3", "xpressengine/xpressengine": "<3.0.15", "yab/quarx": "<2.4.5", - "yeswiki/yeswiki": "<4.1", + "yeswiki/yeswiki": "<=4.4.4", "yetiforce/yetiforce-crm": "<=6.4", "yidashi/yii2cmf": "<=2", "yii2mod/yii2-cms": "<1.9.2", @@ -1848,7 +1937,7 @@ "type": "tidelift" } ], - "time": "2024-07-18T00:16:06+00:00" + "time": "2024-12-06T23:05:03+00:00" }, { "name": "sebastian/cli-parser", @@ -2020,16 +2109,16 @@ }, { "name": "sebastian/comparator", - "version": "5.0.1", + "version": "5.0.3", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "2db5010a484d53ebf536087a70b4a5423c102372" + "reference": "a18251eb0b7a2dcd2f7aa3d6078b18545ef0558e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/2db5010a484d53ebf536087a70b4a5423c102372", - "reference": "2db5010a484d53ebf536087a70b4a5423c102372", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/a18251eb0b7a2dcd2f7aa3d6078b18545ef0558e", + "reference": "a18251eb0b7a2dcd2f7aa3d6078b18545ef0558e", "shasum": "" }, "require": { @@ -2040,7 +2129,7 @@ "sebastian/exporter": "^5.0" }, "require-dev": { - "phpunit/phpunit": "^10.3" + "phpunit/phpunit": "^10.5" }, "type": "library", "extra": { @@ -2085,7 +2174,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", "security": "https://github.com/sebastianbergmann/comparator/security/policy", - "source": "https://github.com/sebastianbergmann/comparator/tree/5.0.1" + "source": "https://github.com/sebastianbergmann/comparator/tree/5.0.3" }, "funding": [ { @@ -2093,7 +2182,7 @@ "type": "github" } ], - "time": "2023-08-14T13:18:12+00:00" + "time": "2024-10-18T14:56:07+00:00" }, { "name": "sebastian/complexity", diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index f945e95..80d36e6 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -1,4 +1,5 @@ registerEventListener(NodeRemovedFromCache::class, FileListener::class); $context->registerEventListener(NodeWrittenEvent::class, FileListener::class); $context->registerEventListener(AppDisableEvent::class, AppDisableListener::class); + $context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class); $context->registerTaskProcessingTaskType(ContextChatTaskType::class); $context->registerTaskProcessingProvider(ContextChatProvider::class); diff --git a/lib/BackgroundJobs/ActionJob.php b/lib/BackgroundJobs/ActionJob.php new file mode 100644 index 0000000..a36d788 --- /dev/null +++ b/lib/BackgroundJobs/ActionJob.php @@ -0,0 +1,117 @@ + + * @copyright Anupam Kumar 2024 + */ + +declare(strict_types=1); +namespace OCA\ContextChat\BackgroundJobs; + +use OCA\ContextChat\Db\QueueActionMapper; +use OCA\ContextChat\Service\DiagnosticService; +use OCA\ContextChat\Service\LangRopeService; +use OCA\ContextChat\Type\ActionType; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; +use OCP\BackgroundJob\QueuedJob; +use Psr\Log\LoggerInterface; + +class ActionJob extends QueuedJob { + private const BATCH_SIZE = 100; + + public function __construct( + ITimeFactory $timeFactory, + private LangRopeService $networkService, + private QueueActionMapper $actionMapper, + private IJobList $jobList, + private LoggerInterface $logger, + private DiagnosticService $diagnosticService, + ) { + parent::__construct($timeFactory); + } + + protected function run($argument): void { + $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + $entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE); + + if (empty($entities)) { + return; + } + + foreach ($entities as $entity) { + $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + + switch ($entity->getType()) { + case ActionType::DELETE_SOURCE_IDS: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['sourceIds'])) { + $this->logger->warning('Invalid payload for DELETE_SOURCE_IDS action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->deleteSources($decoded['sourceIds']); + break; + + case ActionType::DELETE_PROVIDER_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['providerId'])) { + $this->logger->warning('Invalid payload for DELETE_PROVIDER_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->deleteProvider($decoded['providerId']); + break; + + case ActionType::DELETE_USER_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['userId'])) { + $this->logger->warning('Invalid payload for DELETE_USER_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->deleteUser($decoded['userId']); + break; + + case ActionType::UPDATE_ACCESS_SOURCE_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) { + $this->logger->warning('Invalid payload for UPDATE_ACCESS_SOURCE_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->updateAccess($decoded['op'], $decoded['userIds'], $decoded['sourceId']); + break; + + case ActionType::UPDATE_ACCESS_PROVIDER_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['providerId'])) { + $this->logger->warning('Invalid payload for UPDATE_ACCESS_PROVIDER_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->updateAccessProvider($decoded['op'], $decoded['userIds'], $decoded['providerId']); + break; + + case ActionType::UPDATE_ACCESS_DECL_SOURCE_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) { + $this->logger->warning('Invalid payload for UPDATE_ACCESS_DECL_SOURCE_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->updateAccessDeclarative($decoded['userIds'], $decoded['sourceId']); + break; + + default: + $this->logger->warning('Unknown action type', ['type' => $entity->getType()]); + } + } + + foreach ($entities as $entity) { + $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + $this->actionMapper->removeFromQueue($entity); + } + + $this->jobList->add(static::class); + } +} diff --git a/lib/BackgroundJobs/DeleteJob.php b/lib/BackgroundJobs/DeleteJob.php deleted file mode 100644 index b98e72c..0000000 --- a/lib/BackgroundJobs/DeleteJob.php +++ /dev/null @@ -1,74 +0,0 @@ - - * @copyright Anupam Kumar 2024 - */ - -declare(strict_types=1); -namespace OCA\ContextChat\BackgroundJobs; - -use OCA\ContextChat\Db\QueueDeleteMapper; -use OCA\ContextChat\Service\DeleteService; -use OCA\ContextChat\Service\DiagnosticService; -use OCA\ContextChat\Service\LangRopeService; -use OCA\ContextChat\Type\DeleteContext; -use OCP\AppFramework\Utility\ITimeFactory; -use OCP\BackgroundJob\IJobList; -use OCP\BackgroundJob\QueuedJob; -use Psr\Log\LoggerInterface; - -class DeleteJob extends QueuedJob { - private const BATCH_SIZE = 1000; - - public function __construct( - ITimeFactory $timeFactory, - private LangRopeService $networkService, - private QueueDeleteMapper $deleteMapper, - private DeleteService $deleteService, - private IJobList $jobList, - private LoggerInterface $logger, - private DiagnosticService $diagnosticService, - ) { - parent::__construct($timeFactory); - } - - protected function run($argument): void { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); - $entities = $this->deleteMapper->getFromQueue(static::BATCH_SIZE); - - if (empty($entities)) { - return; - } - - $bucket = $this->deleteService->bucketIntoTypes($entities); - - foreach ($bucket[DeleteContext::PROVIDER_ALL_USERS] as $providerKey) { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); - $this->networkService->deleteSourcesByProviderForAllUsers($providerKey); - } - - foreach ($bucket[DeleteContext::PROVIDER_ONE_USER] as $userId => $providerKeys) { - foreach ($providerKeys as $providerKey) { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); - $this->networkService->deleteSourcesByProvider($userId, $providerKey); - } - } - - foreach ($bucket[DeleteContext::SOURCE_ONE_USER] as $userId => $sourceIds) { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); - $this->networkService->deleteSources($userId, $sourceIds); - } - - foreach ($entities as $entity) { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); - $this->deleteMapper->removeFromQueue($entity); - } - - $this->jobList->add(static::class); - } -} diff --git a/lib/BackgroundJobs/IndexerJob.php b/lib/BackgroundJobs/IndexerJob.php index 16e892c..68d85aa 100644 --- a/lib/BackgroundJobs/IndexerJob.php +++ b/lib/BackgroundJobs/IndexerJob.php @@ -1,4 +1,5 @@ getMaxIndexingTime(); $startTime = time(); + $sources = []; + $allSourceIds = []; + $loadedSources = []; + $size = 0; + foreach ($files as $queueFile) { $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); if ($startTime + $maxTime < time()) { break; } + $file = current($this->rootFolder->getById($queueFile->getFileId())); if (!$file instanceof File) { continue; } + + $file_size = $file->getSize(); + if ($size + $file_size > Application::CC_MAX_SIZE || count($sources) >= Application::CC_MAX_FILES) { + $loadedSources = array_merge($loadedSources, $this->langRopeService->indexSources($sources)); + $sources = []; + $size = 0; + } + $userIds = $this->storageService->getUsersForFileId($queueFile->getFileId()); - foreach ($userIds as $userId) { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + + try { try { - try { - $fileHandle = $file->fopen('r'); - } catch (LockedException|NotPermittedException $e) { - $this->logger->error('Could not open file ' . $file->getPath() . ' for reading', ['exception' => $e]); - continue; - } - if (!is_resource($fileHandle)) { - $this->logger->warning('File handle for' . $file->getPath() . ' is not readable'); - continue; - } - $source = new Source( - $userId, - ProviderConfigService::getSourceId($file->getId()), - $file->getPath(), - $fileHandle, - $file->getMtime(), - $file->getMimeType(), - ProviderConfigService::getDefaultProviderKey(), - ); - } catch (InvalidPathException|NotFoundException $e) { - $this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]); - continue 2; + $fileHandle = $file->fopen('r'); + } catch (LockedException|NotPermittedException $e) { + $this->logger->error('Could not open file ' . $file->getPath() . ' for reading', ['exception' => $e]); + continue; } - $this->langRopeService->indexSources([$source]); - } - try { - $this->queue->removeFromQueue($queueFile); - } catch (Exception $e) { - $this->logger->error('Could not remove file from queue', ['exception' => $e]); + if (!is_resource($fileHandle)) { + $this->logger->warning('File handle for' . $file->getPath() . ' is not readable'); + continue; + } + + $sources[] = new Source( + $userIds, + ProviderConfigService::getSourceId($file->getId()), + $file->getPath(), + $fileHandle, + $file->getMtime(), + $file->getMimeType(), + ProviderConfigService::getDefaultProviderKey(), + ); + $allSourceIds[] = ProviderConfigService::getSourceId($file->getId()); + } catch (InvalidPathException|NotFoundException $e) { + $this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]); + continue; } } + + if (count($sources) > 0) { + $loadedSources = array_merge($loadedSources, $this->langRopeService->indexSources($sources)); + } + + $emptyInvalidSources = array_diff($allSourceIds, $loadedSources); + if (count($emptyInvalidSources) > 0) { + $this->logger->info('Invalid or empty sources that were not indexed', ['sourceIds' => $emptyInvalidSources]); + } + + try { + $this->queue->removeFromQueue($files); + } catch (Exception $e) { + $this->logger->error('Could not remove indexed files from queue', ['exception' => $e]); + } } } diff --git a/lib/BackgroundJobs/InitialContentImportJob.php b/lib/BackgroundJobs/InitialContentImportJob.php index b5e955d..0419d33 100644 --- a/lib/BackgroundJobs/InitialContentImportJob.php +++ b/lib/BackgroundJobs/InitialContentImportJob.php @@ -1,4 +1,5 @@ logger->warning('Could not run initial import for content provider', ['exception' => $e]); return; } if (!$this->appManager->isEnabledForUser($providerObj->getAppId())) { + $this->logger->info('App is not enabled for user, skipping content import', ['appId' => $providerObj->getAppId()]); return; } @@ -62,6 +64,7 @@ protected function run($argument): void { if (!isset($registeredProviders[$identifier]) || $registeredProviders[$identifier]['isInitiated'] ) { + $this->logger->info('Provider has already been initiated, skipping content import', ['provider' => $identifier]); return; } diff --git a/lib/BackgroundJobs/SchedulerJob.php b/lib/BackgroundJobs/SchedulerJob.php index dc92673..f55e43e 100644 --- a/lib/BackgroundJobs/SchedulerJob.php +++ b/lib/BackgroundJobs/SchedulerJob.php @@ -1,4 +1,5 @@ @@ -16,10 +17,12 @@ class SchedulerJob extends QueuedJob { - public function __construct(ITimeFactory $timeFactory, + public function __construct( + ITimeFactory $timeFactory, private LoggerInterface $logger, private IJobList $jobList, - private StorageService $storageService) { + private StorageService $storageService, + ) { parent::__construct($timeFactory); } @@ -28,7 +31,7 @@ public function __construct(ITimeFactory $timeFactory, */ protected function run($argument): void { foreach ($this->storageService->getMounts() as $mount) { - $this->logger->debug('Scheduling StorageCrawlJob storage_id='.$mount['storage_id'].' root_id='.$mount['root_id' ]); + $this->logger->debug('Scheduling StorageCrawlJob storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id' ]); $this->jobList->add(StorageCrawlJob::class, [ 'storage_id' => $mount['storage_id'], 'root_id' => $mount['root_id' ], diff --git a/lib/BackgroundJobs/StorageCrawlJob.php b/lib/BackgroundJobs/StorageCrawlJob.php index ab51945..c8c55db 100644 --- a/lib/BackgroundJobs/StorageCrawlJob.php +++ b/lib/BackgroundJobs/StorageCrawlJob.php @@ -1,4 +1,5 @@ @@ -20,7 +21,8 @@ class StorageCrawlJob extends QueuedJob { public const BATCH_SIZE = 2000; - public function __construct(ITimeFactory $timeFactory, + public function __construct( + ITimeFactory $timeFactory, private LoggerInterface $logger, private QueueService $queue, private IJobList $jobList, diff --git a/lib/BackgroundJobs/SubmitContentJob.php b/lib/BackgroundJobs/SubmitContentJob.php index 0fef02b..62afa8b 100644 --- a/lib/BackgroundJobs/SubmitContentJob.php +++ b/lib/BackgroundJobs/SubmitContentJob.php @@ -1,4 +1,5 @@ > */ - $bucketed = []; - foreach ($entities as $entity) { - foreach (explode(',', $entity->getUsers()) as $userId) { - if (!is_array($bucketed[$userId])) { - $bucketed[$userId] = []; - } - $bucketed[$userId][] = $entity; - } - } + $sources = array_map(function (QueueContentItem $item) { + $providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId()); + $sourceId = ProviderConfigService::getSourceId($item->getItemId(), $providerKey); + return new Source( + explode(',', $item->getUsers()), + $sourceId, + $item->getTitle(), + $item->getContent(), + $item->getLastModified()->getTimeStamp(), + $item->getDocumentType(), + $providerKey, + ); + }, $entities); - foreach ($bucketed as $userId => $entities) { - $sources = array_map(function (QueueContentItem $item) use ($userId) { - $providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId()); - $sourceId = ProviderConfigService::getSourceId($item->getItemId(), $providerKey); - return new Source( - $userId, - $sourceId, - $item->getTitle(), - $item->getContent(), - $item->getLastModified()->getTimeStamp(), - $item->getDocumentType(), - $providerKey, - ); - }, $entities); - - $this->service->indexSources($sources); - } + $loadedSources = $this->service->indexSources($sources); + $this->logger->info('Indexed sources for providers', [ + 'count' => count($loadedSources), + 'sources' => $loadedSources, + ]); foreach ($entities as $entity) { $this->mapper->removeFromQueue($entity); diff --git a/lib/Command/Diagnostics.php b/lib/Command/Diagnostics.php index 3dc4aa8..cf9d564 100644 --- a/lib/Command/Diagnostics.php +++ b/lib/Command/Diagnostics.php @@ -53,7 +53,7 @@ protected function execute(InputInterface $input, OutputInterface $output) { $output->write('last_seen=' . (new \DateTime('@' . $value))->format('Y-m-d H:i:s')); } } - $output->writeln(""); + $output->writeln(''); } if ($count === 0) { $output->writeln('No jobs running.'); diff --git a/lib/Command/ScanFiles.php b/lib/Command/ScanFiles.php index 23acbc1..607d5ae 100644 --- a/lib/Command/ScanFiles.php +++ b/lib/Command/ScanFiles.php @@ -23,7 +23,7 @@ class ScanFiles extends Command { public function __construct( - private ScanService $scanService + private ScanService $scanService, ) { parent::__construct(); } diff --git a/lib/Controller/ConfigController.php b/lib/Controller/ConfigController.php index 19bb9ad..7a0fa7a 100644 --- a/lib/Controller/ConfigController.php +++ b/lib/Controller/ConfigController.php @@ -1,4 +1,5 @@ * diff --git a/lib/Db/QueueDelete.php b/lib/Db/QueueAction.php similarity index 66% rename from lib/Db/QueueDelete.php rename to lib/Db/QueueAction.php index 4f998d8..e3b9fdd 100644 --- a/lib/Db/QueueDelete.php +++ b/lib/Db/QueueAction.php @@ -1,4 +1,5 @@ addType('id', 'integer'); $this->addType('type', 'string'); - $this->addType('userId', 'string'); $this->addType('payload', 'string'); } } diff --git a/lib/Db/QueueDeleteMapper.php b/lib/Db/QueueActionMapper.php similarity index 73% rename from lib/Db/QueueDeleteMapper.php rename to lib/Db/QueueActionMapper.php index 2532ba1..6573848 100644 --- a/lib/Db/QueueDeleteMapper.php +++ b/lib/Db/QueueActionMapper.php @@ -1,4 +1,5 @@ + * @template-extends QBMapper */ -class QueueDeleteMapper extends QBMapper { +class QueueActionMapper extends QBMapper { /** * @var IDBConnection $db */ protected $db; public function __construct(IDBConnection $db) { - parent::__construct($db, 'context_chat_delete_queue', QueueDelete::class); + parent::__construct($db, 'context_chat_action_queue', QueueAction::class); } /** * @param int $limit - * @return array + * @return array * @throws \OCP\DB\Exception */ public function getFromQueue(int $limit): array { $qb = $this->db->getQueryBuilder(); - $qb->select(QueueDelete::$columns) + $qb->select(QueueAction::$columns) ->from($this->getTableName()) ->setMaxResults($limit); @@ -44,11 +45,11 @@ public function getFromQueue(int $limit): array { } /** - * @param QueueDelete $item + * @param QueueAction $item * @return void * @throws \OCP\DB\Exception */ - public function removeFromQueue(QueueDelete $item): void { + public function removeFromQueue(QueueAction $item): void { $qb = $this->db->getQueryBuilder(); $qb->delete($this->getTableName()) ->where($qb->expr()->eq('id', $qb->createPositionalParameter($item->getId()))) @@ -56,16 +57,15 @@ public function removeFromQueue(QueueDelete $item): void { } /** - * @param QueueDelete $item + * @param QueueAction $item * @return void * @throws \OCP\DB\Exception */ - public function insertIntoQueue(QueueDelete $item): void { + public function insertIntoQueue(QueueAction $item): void { $qb = $this->db->getQueryBuilder(); $qb->insert($this->getTableName()) ->values([ 'type' => $qb->createPositionalParameter($item->getType(), IQueryBuilder::PARAM_STR), - 'user_id' => $qb->createPositionalParameter($item->getUserId(), IQueryBuilder::PARAM_STR), 'payload' => $qb->createPositionalParameter($item->getPayload(), IQueryBuilder::PARAM_STR), ]) ->executeStatement(); diff --git a/lib/Db/QueueContentItem.php b/lib/Db/QueueContentItem.php index 70b3cdb..c7ec43d 100644 --- a/lib/Db/QueueContentItem.php +++ b/lib/Db/QueueContentItem.php @@ -1,4 +1,5 @@ @@ -29,9 +30,9 @@ class QueueFile extends Entity { protected $rootId; protected $update; - /** @var string[] */ + /** @var string[] */ public static array $columns = ['id', 'file_id', 'storage_id', 'root_id', 'update']; - /** @var string[] */ + /** @var string[] */ public static array $fields = ['id', 'fileId', 'storageId', 'rootId', 'update']; public function __construct() { diff --git a/lib/Db/QueueMapper.php b/lib/Db/QueueMapper.php index 3f96053..0a7687b 100644 --- a/lib/Db/QueueMapper.php +++ b/lib/Db/QueueMapper.php @@ -1,4 +1,5 @@ @@ -45,14 +46,15 @@ public function getFromQueue(int $storageId, int $rootId, int $n) : array { } /** - * @param QueueFile $file + * @param QueueFile[] $files * @return void * @throws \OCP\DB\Exception */ - public function removeFromQueue(QueueFile $file) : void { + public function removeFromQueue(array $files) : void { + $fileIds = array_map(fn (QueueFile $file) => $file->getId(), $files); $qb = $this->db->getQueryBuilder(); $qb->delete($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createPositionalParameter($file->getId()))) + ->where($qb->expr()->in('id', $qb->createPositionalParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) ->executeStatement(); } diff --git a/lib/Event/ContentProviderRegisterEvent.php b/lib/Event/ContentProviderRegisterEvent.php index 0a41b33..d83e03a 100644 --- a/lib/Event/ContentProviderRegisterEvent.php +++ b/lib/Event/ContentProviderRegisterEvent.php @@ -1,4 +1,5 @@ providerConfig->removeProvider($appId, $providerId); - $this->deleteService->deleteSourcesByProviderForAllUsers($providerId); + $this->actionService->deleteProvider($providerId); } } } diff --git a/lib/Listener/FileListener.php b/lib/Listener/FileListener.php index 8356514..b1c6d24 100644 --- a/lib/Listener/FileListener.php +++ b/lib/Listener/FileListener.php @@ -1,4 +1,5 @@ @@ -9,7 +10,8 @@ use OCA\ContextChat\AppInfo\Application; use OCA\ContextChat\Db\QueueFile; -use OCA\ContextChat\Service\DeleteService; +use OCA\ContextChat\Public\UpdateAccessOp; +use OCA\ContextChat\Service\ActionService; use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Service\QueueService; use OCA\ContextChat\Service\StorageService; @@ -44,7 +46,8 @@ public function __construct( private StorageService $storageService, private IManager $shareManager, private IRootFolder $rootFolder, - private DeleteService $deleteService) { + private ActionService $actionService, + ) { } public function handle(Event $event): void { @@ -58,14 +61,23 @@ public function handle(Event $event): void { if ($event instanceof ShareCreatedEvent) { $share = $event->getShare(); - $ownerId = $share->getShareOwner(); $node = $share->getNode(); - $accessList = $this->shareManager->getAccessList($node, true, true); - /** - * @var string[] $userIds - */ - $userIds = array_keys($accessList['users']); + switch ($share->getShareType()) { + case \OCP\Share\IShare::TYPE_USER: + $userIds = [$share->getSharedWith()]; + break; + case \OCP\Share\IShare::TYPE_GROUP: + // todo: probably a group listener so when a user enters/leaves a group, we can update the access for all files shared with that group + $accessList = $this->shareManager->getAccessList($node, true, true); + /** + * @var string[] $userIds + */ + $userIds = array_keys($accessList['users']); + break; + default: + return; + } if ($node->getType() === FileInfo::TYPE_FOLDER) { $mount = $node->getMountPoint(); @@ -74,27 +86,22 @@ public function handle(Event $event): void { } $files = $this->storageService->getFilesInMount($mount->getNumericStorageId(), $node->getId(), 0, 0); foreach ($files as $fileId) { - foreach ($userIds as $userId) { - if ($userId === $ownerId) { - continue; - } - $file = current($this->rootFolder->getById($fileId)); - if (!$file instanceof File) { - continue; - } - $this->postInsert($file, false, true); - } - } - } else { - foreach ($userIds as $userId) { - if ($userId === $ownerId) { - continue; - } - if (!$node instanceof File) { + $file = current($this->rootFolder->getById($fileId)); + if (!$file instanceof File) { continue; } - $this->postInsert($node, false, true); + $this->actionService->updateAccess( + UpdateAccessOp::ALLOW, + $userIds, + ProviderConfigService::getSourceId($file->getId()), + ); } + } else { + $this->actionService->updateAccess( + UpdateAccessOp::ALLOW, + $userIds, + ProviderConfigService::getSourceId($node->getId()), + ); } } @@ -124,8 +131,12 @@ public function handle(Event $event): void { $fileRefs[] = ProviderConfigService::getSourceId($node->getId()); } - foreach ($userIds as $userId) { - $this->deleteService->deleteSources($userId, $fileRefs); + foreach ($fileRefs as $fileRef) { + $this->actionService->updateAccess( + UpdateAccessOp::DENY, + $userIds, + $fileRef, + ); } } else { if (!$this->allowedMimeType($node)) { @@ -133,9 +144,11 @@ public function handle(Event $event): void { } $fileRef = ProviderConfigService::getSourceId($node->getId()); - foreach ($userIds as $userId) { - $this->deleteService->deleteSources($userId, [$fileRef]); - } + $this->actionService->updateAccess( + UpdateAccessOp::DENY, + $userIds, + $fileRef, + ); } } @@ -196,10 +209,8 @@ public function postDelete(Node $node, bool $recurse = true): void { return; } - foreach ($this->storageService->getUsersForFileId($node->getId()) as $userId) { - $fileRef = ProviderConfigService::getSourceId($node->getId()); - $this->deleteService->deleteSources($userId, [$fileRef]); - } + $fileRef = ProviderConfigService::getSourceId($node->getId()); + $this->actionService->deleteSources($fileRef); } /** @@ -249,7 +260,7 @@ public function postInsert(Node $node, bool $recurse = true, bool $update = fals } } - private function allowedMimeType(File $file) { + private function allowedMimeType(File $file): bool { $mimeType = $file->getMimeType(); return in_array($mimeType, Application::MIMETYPES, true); } diff --git a/lib/Listener/UserDeletedListener.php b/lib/Listener/UserDeletedListener.php new file mode 100644 index 0000000..04c9f96 --- /dev/null +++ b/lib/Listener/UserDeletedListener.php @@ -0,0 +1,42 @@ + + * @copyright Anupam Kumar 2024 + */ + +declare(strict_types=1); + +namespace OCA\ContextChat\Listener; + +use OCA\ContextChat\Service\ActionService; +use OCA\ContextChat\Service\ProviderConfigService; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\User\Events\UserDeletedEvent; +use Psr\Log\LoggerInterface; + +/** + * @template-implements IEventListener + */ +class UserDeletedListener implements IEventListener { + public function __construct( + private ProviderConfigService $providerConfig, + private ActionService $actionService, + private LoggerInterface $logger, + ) { + } + + public function handle(Event $event): void { + if (!($event instanceof UserDeletedEvent)) { + return; + } + + $this->actionService->deleteUser($event->getUid()); + } +} diff --git a/lib/Migration/Version001000000Date20231102094721.php b/lib/Migration/Version001000000Date20231102094721.php index 709d736..3c9aec7 100644 --- a/lib/Migration/Version001000000Date20231102094721.php +++ b/lib/Migration/Version001000000Date20231102094721.php @@ -1,4 +1,5 @@ diff --git a/lib/Migration/Version1001Date20240130120627.php b/lib/Migration/Version1001Date20240130120627.php index dbc6b28..8a27f29 100644 --- a/lib/Migration/Version1001Date20240130120627.php +++ b/lib/Migration/Version1001Date20240130120627.php @@ -1,4 +1,5 @@ startProgress(10); - foreach([ + foreach ([ SchedulerJob::class, StorageCrawlJob::class, IndexerJob::class, diff --git a/lib/Migration/Version4000Date20241202141755.php b/lib/Migration/Version4000Date20241202141755.php new file mode 100644 index 0000000..fdfbf7c --- /dev/null +++ b/lib/Migration/Version4000Date20241202141755.php @@ -0,0 +1,34 @@ +hasTable('context_chat_delete_queue')) { + $schema->dropTable('context_chat_delete_queue'); + } + + return $schema; + } +} diff --git a/lib/Migration/Version4000Date20241206135634.php b/lib/Migration/Version4000Date20241206135634.php new file mode 100644 index 0000000..17c9da6 --- /dev/null +++ b/lib/Migration/Version4000Date20241206135634.php @@ -0,0 +1,51 @@ +hasTable('context_chat_action_queue')) { + $table = $schema->createTable('context_chat_action_queue'); + + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 64, + 'unsigned' => true, + ]); + $table->addColumn('type', Types::STRING, [ + 'notnull' => true, + 'length' => 128, + ]); + $table->addColumn('payload', Types::TEXT, [ + 'notnull' => false, + ]); + + $table->setPrimaryKey(['id'], 'ccc_action_queue_id'); + } + + return $schema; + } +} diff --git a/lib/Public/ContentItem.php b/lib/Public/ContentItem.php index a8c420e..190a89e 100644 --- a/lib/Public/ContentItem.php +++ b/lib/Public/ContentItem.php @@ -1,4 +1,5 @@ logger->warning('Could not find content provider by class name', ['classString' => $providerClass, 'exception' => $e]); return; } @@ -108,34 +109,113 @@ public function submitContent(string $appId, array $items): void { * @param string $appId * @param string $providerId * @param string $itemId - * @param array $users + * @param array $users * @return void - * @since 1.1.0 + * @deprecated 4.0.0 */ public function removeContentForUsers(string $appId, string $providerId, string $itemId, array $users): void { $this->collectAllContentProviders(); - foreach ($users as $userId) { - $this->deleteService->deleteSources($userId, [ - ProviderConfigService::getSourceId($itemId, ProviderConfigService::getConfigKey($appId, $providerId)) - ]); - } + $this->actionService->updateAccess( + UpdateAccessOp::DENY, + $users, + ProviderConfigService::getSourceId($itemId, ProviderConfigService::getConfigKey($appId, $providerId)), + ); } /** - * Remove all content items from the knowledge base of context chat for specified users + * Deny access to all content items for the given provider for specified users. + * If no user has access to the content items, it will be removed from the knowledge base. * * @param string $appId * @param string $providerId - * @param array $users + * @param array $users * @return void - * @since 1.1.0 + * @deprecated 4.0.0 */ public function removeAllContentForUsers(string $appId, string $providerId, array $users): void { $this->collectAllContentProviders(); + $this->actionService->updateAccessProvider( + UpdateAccessOp::DENY, + $users, + ProviderConfigService::getConfigKey($appId, $providerId), + ); + } - foreach ($users as $userId) { - $this->deleteService->deleteSourcesByProvider($userId, ProviderConfigService::getConfigKey($appId, $providerId)); - } + /** + * Update access for a content item for specified users. + * This modifies the access list for the content item, + * allowing or denying access to the specified users. + * If no user has access to the content item, it will be removed from the knowledge base. + * + * @param string $appId + * @param string $providerId + * @param string $itemId + * @param string $op + * @param array $userIds + * @return void + * @since 4.0.0 + */ + public function updateAccess(string $appId, string $providerId, string $itemId, string $op, array $userIds): void { + $this->collectAllContentProviders(); + + $this->actionService->updateAccess( + $op, + $userIds, + ProviderConfigService::getSourceId($itemId, ProviderConfigService::getConfigKey($appId, $providerId)), + ); + } + + /** + * Update access for a content item for specified users declaratively. + * This overwrites the access list for the content item, + * allowing only the specified users access to it. + * + * @param string $appId + * @param string $providerId + * @param string $itemId + * @param array $userIds + * @return void + * @since 4.0.0 + */ + public function updateAccessDeclarative(string $appId, string $providerId, string $itemId, array $userIds): void { + $this->collectAllContentProviders(); + + $this->actionService->updateAccessDeclSource( + $userIds, + ProviderConfigService::getSourceId($itemId, ProviderConfigService::getConfigKey($appId, $providerId)), + ); + } + + /** + * Delete all content items and access lists for a provider. + * This does not unregister the provider itself. + * + * @param string $appId + * @param string $providerId + * @return void + * @since 4.0.0 + */ + public function deleteProvider(string $appId, string $providerId): void { + $this->collectAllContentProviders(); + $this->actionService->deleteProvider(ProviderConfigService::getConfigKey($appId, $providerId)); + } + + /** + * Remove a content item from the knowledge base of context chat. + * + * @param string $appId + * @param string $providerId + * @param string[] $itemIds + * @return void + * @since 4.0.0 + */ + public function deleteContent(string $appId, string $providerId, string ...$itemIds): void { + $this->collectAllContentProviders(); + + $providerKey = ProviderConfigService::getConfigKey($appId, $providerId); + $this->actionService->deleteSources(...array_map(function (string $itemId) use ($providerKey) { + return ProviderConfigService::getSourceId($itemId, $providerKey); + }, $itemIds)); } } diff --git a/lib/Public/IContentProvider.php b/lib/Public/IContentProvider.php index 3aed8a2..3eb2c61 100644 --- a/lib/Public/IContentProvider.php +++ b/lib/Public/IContentProvider.php @@ -1,4 +1,5 @@ + * @copyright Anupam Kumar 2024 + */ + +declare(strict_types=1); + +namespace OCA\ContextChat\Service; + +use OCA\ContextChat\BackgroundJobs\ActionJob; +use OCA\ContextChat\Db\QueueAction; +use OCA\ContextChat\Db\QueueActionMapper; +use OCA\ContextChat\Type\ActionType; +use OCP\BackgroundJob\IJobList; +use Psr\Log\LoggerInterface; + +class ActionService { + private const BATCH_SIZE = 500; + + public function __construct( + private IJobList $jobList, + private QueueActionMapper $actionMapper, + private LoggerInterface $logger, + ) { + } + + /** + * @param string $type + * @param string $payload + * @return void + */ + private function scheduleAction(string $type, string $payload): void { + $item = new QueueAction(); + $item->setType($type); + $item->setPayload($payload); + + // do not catch DB exceptions + $this->actionMapper->insertIntoQueue($item); + + if (!$this->jobList->has(ActionJob::class, null)) { + $this->jobList->add(ActionJob::class, null); + } + } + + /** + * @param string[] $sourceIds + * @return void + */ + public function deleteSources(string ...$sourceIds): void { + // batch sourceIds into self::BATCH_SIZE chunks + $batches = array_chunk($sourceIds, self::BATCH_SIZE); + + foreach ($batches as $batch) { + $payload = json_encode($batch); + if ($payload === false) { + $this->logger->warning('Failed to json_encode sourceIds for deletion', ['sourceIds' => $batch]); + continue; + } + $this->scheduleAction(ActionType::DELETE_SOURCE_IDS, $payload); + } + } + + /** + * @param string $providerKey + * @return void + */ + public function deleteProvider(string $providerKey): void { + $payload = json_encode(['providerId' => $providerKey]); + if ($payload === false) { + $this->logger->warning('Failed to json_encode providerId for deletion', ['providerId' => $providerKey]); + return; + } + $this->scheduleAction(ActionType::DELETE_PROVIDER_ID, $payload); + } + + /** + * @param string $userId + * @return void + */ + public function deleteUser(string $userId): void { + $payload = json_encode(['userId' => $userId]); + if ($payload === false) { + $this->logger->warning('Failed to json_encode userId for deletion', ['userId' => $userId]); + return; + } + $this->scheduleAction(ActionType::DELETE_USER_ID, $payload); + } + + /** + * @param string $op + * @param string[] $userIds + * @param string $sourceId + * @return void + */ + public function updateAccess(string $op, array $userIds, string $sourceId): void { + $payload = json_encode(['op' => $op, 'userIds' => $userIds, 'sourceId' => $sourceId]); + if ($payload === false) { + $this->logger->warning('Failed to json_encode access update for source', ['op' => $op, 'sourceId' => $sourceId]); + return; + } + $this->scheduleAction(ActionType::UPDATE_ACCESS_SOURCE_ID, $payload); + } + + /** + * @param string $op + * @param string[] $userIds + * @param string $providerId + * @return void + */ + public function updateAccessProvider(string $op, array $userIds, string $providerId): void { + $payload = json_encode(['op' => $op, 'userIds' => $userIds, 'providerId' => $providerId]); + if ($payload === false) { + $this->logger->warning('Failed to json_encode access update for provider', ['op' => $op, 'providerId' => $providerId]); + return; + } + $this->scheduleAction(ActionType::UPDATE_ACCESS_PROVIDER_ID, $payload); + } + + /** + * @param string[] $userIds + * @param string $sourceId + * @return void + */ + public function updateAccessDeclSource(array $userIds, string $sourceId): void { + $payload = json_encode(['userIds' => $userIds, 'sourceId' => $sourceId]); + if ($payload === false) { + $this->logger->warning('Failed to json_encode access update declarative for source', ['sourceId' => $sourceId]); + return; + } + $this->scheduleAction(ActionType::UPDATE_ACCESS_DECL_SOURCE_ID, $payload); + } +} diff --git a/lib/Service/DeleteService.php b/lib/Service/DeleteService.php deleted file mode 100644 index 41f65fc..0000000 --- a/lib/Service/DeleteService.php +++ /dev/null @@ -1,106 +0,0 @@ - - * @copyright Anupam Kumar 2024 - */ - -declare(strict_types=1); - -namespace OCA\ContextChat\Service; - -use OCA\ContextChat\BackgroundJobs\DeleteJob; -use OCA\ContextChat\Db\QueueDelete; -use OCA\ContextChat\Db\QueueDeleteMapper; -use OCA\ContextChat\Type\DeleteContext; -use OCP\BackgroundJob\IJobList; - -class DeleteService { - public function __construct( - private IJobList $jobList, - private QueueDeleteMapper $deleteMapper, - ) { - } - - /** - * @param string $type - * @param string $payload - * @param string $userId - * @return void - */ - public function scheduleDelete(string $type, string $payload, string $userId = ''): void { - $item = new QueueDelete(); - $item->setType($type); - $item->setUserId($userId); - $item->setPayload($payload); - - // do not catch DB exceptions - $this->deleteMapper->insertIntoQueue($item); - - if (!$this->jobList->has(DeleteJob::class, null)) { - $this->jobList->add(DeleteJob::class, null); - } - } - - /** - * @param array $entities - * @return array|array>> - */ - public function bucketIntoTypes(array $entities): array { - $bucket = [ - DeleteContext::SOURCE_ONE_USER => [], - DeleteContext::PROVIDER_ONE_USER => [], - DeleteContext::PROVIDER_ALL_USERS => [], - ]; - - foreach ($entities as $entity) { - $type = $entity->getType(); - $userId = $entity->getUserId(); - $payload = $entity->getPayload(); - - if ($type === DeleteContext::PROVIDER_ALL_USERS) { - $bucket[DeleteContext::PROVIDER_ALL_USERS][] = $payload; - continue; - } - - if (!isset($bucket[$type][$userId])) { - $bucket[$type][$userId] = []; - } - $bucket[$type][$userId][] = $payload; - } - - return $bucket; - } - - /** - * @param string $providerKey - * @return void - */ - public function deleteSourcesByProviderForAllUsers(string $providerKey): void { - $this->scheduleDelete(DeleteContext::PROVIDER_ALL_USERS, $providerKey); - } - - /** - * @param string $userId - * @param string $providerKey - * @return void - */ - public function deleteSourcesByProvider(string $userId, string $providerKey): void { - $this->scheduleDelete(DeleteContext::PROVIDER_ONE_USER, $providerKey, $userId); - } - - /** - * @param string $userId - * @param string[] $sourceNames - * @return void - */ - public function deleteSources(string $userId, array $sourceNames): void { - foreach ($sourceNames as $source) { - $this->scheduleDelete(DeleteContext::SOURCE_ONE_USER, $source, $userId); - } - } -} diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php index 4604a1e..31d9121 100644 --- a/lib/Service/LangRopeService.php +++ b/lib/Service/LangRopeService.php @@ -1,4 +1,5 @@ $providerKey, + 'sourceIds' => $sourceIds, ]; - $this->requestToExApp('/deleteSourcesByProviderForAllUsers', 'POST', $params); + $this->requestToExApp('/deleteSources', 'POST', $params); } /** - * @param string $userId * @param string $providerKey * @return void */ - public function deleteSourcesByProvider(string $userId, string $providerKey): void { + public function deleteProvider(string $providerKey): void { $params = [ - 'userId' => $userId, 'providerKey' => $providerKey, ]; - $this->requestToExApp('/deleteSourcesByProvider', 'POST', $params); + $this->requestToExApp('/deleteProvider', 'POST', $params); } /** * @param string $userId - * @param string[] $sourceNames * @return void */ - public function deleteSources(string $userId, array $sourceNames): void { + public function deleteUser(string $userId): void { $params = [ 'userId' => $userId, - 'sourceNames' => $sourceNames, ]; - $this->requestToExApp('/deleteSources', 'POST', $params); + $this->requestToExApp('/deleteUser', 'POST', $params); } /** - * @param Source[] $sources + * @param string $op UpdateAccessOp::* + * @param string[] $userIds + * @param string $sourceId + * @return void + */ + public function updateAccess(string $op, array $userIds, string $sourceId): void { + $params = [ + 'op' => $op, + 'userIds' => $userIds, + 'sourceId' => $sourceId, + ]; + $this->requestToExApp('/updateAccess', 'POST', $params); + } + + /** + * @param string $op UpdateAccessOp::* + * @param string[] $userIds + * @param string $providerId * @return void + */ + public function updateAccessProvider(string $op, array $userIds, string $providerId): void { + $params = [ + 'op' => $op, + 'userIds' => $userIds, + 'providerId' => $providerId, + ]; + $this->requestToExApp('/updateAccessProvider', 'POST', $params); + } + + /** + * @param string[] $userIds + * @param string $sourceId + * @return void + */ + public function updateAccessDeclarative(array $userIds, string $sourceId): void { + $params = [ + 'userIds' => $userIds, + 'sourceId' => $sourceId, + ]; + $this->requestToExApp('/updateAccessDeclarative', 'POST', $params); + } + + /** + * @param Source[] $sources + * @return array * @throws RuntimeException */ - public function indexSources(array $sources): void { + public function indexSources(array $sources): array { if (count($sources) === 0) { - return; + return []; } $params = array_map(function (Source $source) { @@ -223,7 +263,7 @@ public function indexSources(array $sources): void { 'filename' => $source->reference, // eg. 'files__default: 555' 'contents' => $source->content, 'headers' => [ - 'userId' => $source->userId, + 'userIds' => implode(',', $source->userIds), 'title' => $source->title, 'type' => $source->type, 'modified' => $source->modified, @@ -232,7 +272,11 @@ public function indexSources(array $sources): void { ]; }, $sources); - $this->requestToExApp('/loadSources', 'PUT', $params, 'multipart/form-data'); + $response = $this->requestToExApp('/loadSources', 'PUT', $params, 'multipart/form-data'); + if (!isset($response['loaded_sources'])) { + return []; + } + return $response['loaded_sources']; } /** @@ -304,7 +348,7 @@ public function getWithPresentableSources(string $llmResponse, string ...$source } if (!isset($provider['classString'])) { - $this->logger->warning("Provider does not have a class string", ['provider' => $provider]); + $this->logger->warning('Provider does not have a class string', ['provider' => $provider]); continue; } @@ -313,7 +357,7 @@ public function getWithPresentableSources(string $llmResponse, string ...$source try { /** @var IContentProvider */ $providerObj = Server::get($classString); - } catch (ContainerExceptionInterface | NotFoundExceptionInterface $e) { + } catch (ContainerExceptionInterface|NotFoundExceptionInterface $e) { $this->logger->warning('Could not run initial import for content provider', ['exception' => $e]); continue; } diff --git a/lib/Service/MetadataService.php b/lib/Service/MetadataService.php index 9622286..1c85ec4 100644 --- a/lib/Service/MetadataService.php +++ b/lib/Service/MetadataService.php @@ -1,4 +1,5 @@ getItemUrl($this->getIdFromSource($source)); $provider['url'] = $url; $enrichedSources[] = $provider; - } catch (ContainerExceptionInterface | NotFoundExceptionInterface $e) { + } catch (ContainerExceptionInterface|NotFoundExceptionInterface $e) { $this->logger->warning('Could not find content provider by class name', ['classString' => $providerConfig['classString'], 'exception' => $e]); continue; } diff --git a/lib/Service/ProviderConfigService.php b/lib/Service/ProviderConfigService.php index 52d9f5c..b5ebf8a 100644 --- a/lib/Service/ProviderConfigService.php +++ b/lib/Service/ProviderConfigService.php @@ -1,4 +1,5 @@ queueMapper->removeFromQueue($queueFile); + public function removeFromQueue(array $files): void { + $this->queueMapper->removeFromQueue($files); } public function clearQueue(): void { diff --git a/lib/Service/ScanService.php b/lib/Service/ScanService.php index 0641a06..96b51a5 100644 --- a/lib/Service/ScanService.php +++ b/lib/Service/ScanService.php @@ -1,4 +1,5 @@ root->getUserFolder($userId)->get($directory); } - yield from ($this->scanDirectory($userId, $mimeTypeFilter, $userFolder)); + yield from ($this->scanDirectory($mimeTypeFilter, $userFolder)); return []; } /** - * @param string $userId * @param array $mimeTypeFilter * @param Folder $directory * @return \Generator */ - public function scanDirectory(string $userId, array $mimeTypeFilter, Folder $directory): \Generator { + public function scanDirectory(array $mimeTypeFilter, Folder $directory): \Generator { $sources = []; $size = 0; foreach ($directory->getDirectoryListing() as $node) { @@ -63,12 +64,12 @@ public function scanDirectory(string $userId, array $mimeTypeFilter, Folder $dir $node_size = $node->getSize(); if ($size + $node_size > Application::CC_MAX_SIZE || count($sources) >= Application::CC_MAX_FILES) { - $this->indexSources($sources); + $this->langRopeService->indexSources($sources); $sources = []; $size = 0; } - $source = $this->getSourceFromFile($userId, $mimeTypeFilter, $node); + $source = $this->getSourceFromFile($mimeTypeFilter, $node); if ($source === null) { continue; } @@ -82,19 +83,19 @@ public function scanDirectory(string $userId, array $mimeTypeFilter, Folder $dir } if (count($sources) > 0) { - $this->indexSources($sources); + $this->langRopeService->indexSources($sources); } foreach ($directory->getDirectoryListing() as $node) { if ($node instanceof Folder) { - yield from $this->scanDirectory($userId, $mimeTypeFilter, $node); + yield from $this->scanDirectory($mimeTypeFilter, $node); } } return []; } - public function getSourceFromFile(string $userId, array $mimeTypeFilter, File $node): Source | null { + public function getSourceFromFile(array $mimeTypeFilter, File $node): ?Source { if (!in_array($node->getMimeType(), $mimeTypeFilter)) { return null; } @@ -108,8 +109,9 @@ public function getSourceFromFile(string $userId, array $mimeTypeFilter, File $n $providerKey = ProviderConfigService::getDefaultProviderKey(); $sourceId = ProviderConfigService::getSourceId($node->getId()); + $userIds = $this->storageService->getUsersForFileId($node->getId()); return new Source( - $userId, + $userIds, $sourceId, $node->getPath(), $fileHandle, @@ -118,8 +120,4 @@ public function getSourceFromFile(string $userId, array $mimeTypeFilter, File $n $providerKey, ); } - - public function indexSources(array $sources): void { - $this->langRopeService->indexSources($sources); - } } diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index a34fb23..353cf3f 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -1,4 +1,5 @@ @@ -34,12 +35,13 @@ class StorageService { ]; public function __construct( - private IDBConnection $db, + private IDBConnection $db, private LoggerInterface $logger, - private SystemConfig $systemConfig, + private SystemConfig $systemConfig, private IMimeTypeLoader $mimeTypes, private IUserMountCache $userMountCache, - private IFilesMetadataManager $metadataManager) { + private IFilesMetadataManager $metadataManager, + ) { } /** diff --git a/lib/TaskProcessing/ContextChatProvider.php b/lib/TaskProcessing/ContextChatProvider.php index 62abeeb..a91e598 100644 --- a/lib/TaskProcessing/ContextChatProvider.php +++ b/lib/TaskProcessing/ContextChatProvider.php @@ -247,11 +247,14 @@ private function indexFiles(string $userId, string ...$scopeList): array { foreach ($filteredNodes as $node) { try { if ($node['node'] instanceof File) { - $source = $this->scanService->getSourceFromFile($userId, Application::MIMETYPES, $node['node']); - $this->scanService->indexSources([$source]); + $source = $this->scanService->getSourceFromFile(Application::MIMETYPES, $node['node']); + if ($source === null) { + continue; + } + $this->langRopeService->indexSources([$source]); $indexedSources[] = $node['scope']; } elseif ($node['node'] instanceof Folder) { - $fileSources = iterator_to_array($this->scanService->scanDirectory($userId, Application::MIMETYPES, $node['node'])); + $fileSources = iterator_to_array($this->scanService->scanDirectory(Application::MIMETYPES, $node['node'])); $indexedSources = array_merge( $indexedSources, array_map(fn (Source $source) => $source->reference, $fileSources), diff --git a/lib/Type/ActionType.php b/lib/Type/ActionType.php new file mode 100644 index 0000000..e539fa6 --- /dev/null +++ b/lib/Type/ActionType.php @@ -0,0 +1,28 @@ + + * @copyright Anupam Kumar 2024 + */ + +namespace OCA\ContextChat\Type; + +class ActionType { + // { sourceIds: array } + public const DELETE_SOURCE_IDS = 'delete_source_ids'; + // { providerId: string } + public const DELETE_PROVIDER_ID = 'delete_provider_id'; + // { userId: string } + public const DELETE_USER_ID = 'delete_user_id'; + // { op: string, userIds: array, sourceId: string } + public const UPDATE_ACCESS_SOURCE_ID = 'update_access_source_id'; + // { op: string, userIds: array, providerId: string } + public const UPDATE_ACCESS_PROVIDER_ID = 'update_access_provider_id'; + // { userIds: array, sourceId: string } + public const UPDATE_ACCESS_DECL_SOURCE_ID = 'update_access_decl_source_id'; +} diff --git a/lib/Type/ScopeType.php b/lib/Type/ScopeType.php index 50bebca..47a00e1 100644 --- a/lib/Type/ScopeType.php +++ b/lib/Type/ScopeType.php @@ -1,4 +1,5 @@ + * @copyright Anupam Kumar 2023 + */ + +declare(strict_types=1); + +namespace OCA\ContextChat\Type; + +class UpdateAccessOp { + public const ALLOW = 'allow'; + public const DENY = 'deny'; +} diff --git a/tests/integration/ContentManagerTest.php b/tests/integration/ContentManagerTest.php index 451bd57..02c0153 100644 --- a/tests/integration/ContentManagerTest.php +++ b/tests/integration/ContentManagerTest.php @@ -1,4 +1,5 @@ mapper = $this->createMock(QueueContentItemMapper::class); $this->providerConfig = $this->createMock(ProviderConfigService::class); - $this->deleteService = $this->createMock(DeleteService::class); + $this->actionService = $this->createMock(ActionService::class); // new dispatcher for each test $this->dispatcher = new SymfonyDispatcher(); @@ -97,7 +98,7 @@ public function setUp(): void { $this->jobList, $this->providerConfig, $this->mapper, - $this->deleteService, + $this->actionService, $this->logger, $this->eventDispatcher, ); diff --git a/tests/integration/ProviderConfigServiceTest.php b/tests/integration/ProviderConfigServiceTest.php index 6d8a851..8b8b342 100644 --- a/tests/integration/ProviderConfigServiceTest.php +++ b/tests/integration/ProviderConfigServiceTest.php @@ -1,4 +1,5 @@ Date: Mon, 9 Dec 2024 17:31:11 +0530 Subject: [PATCH 02/14] add updateAccessProvider to ContentManager Signed-off-by: Anupam Kumar --- lib/Public/ContentManager.php | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/Public/ContentManager.php b/lib/Public/ContentManager.php index 2090d88..4e9bbd9 100644 --- a/lib/Public/ContentManager.php +++ b/lib/Public/ContentManager.php @@ -151,7 +151,7 @@ public function removeAllContentForUsers(string $appId, string $providerId, arra * @param string $appId * @param string $providerId * @param string $itemId - * @param string $op + * @param string $op UpdateAccessOp::* * @param array $userIds * @return void * @since 4.0.0 @@ -166,6 +166,27 @@ public function updateAccess(string $appId, string $providerId, string $itemId, ); } + /** + * Update access for content items from the given provider for specified users. + * If no user has access to the content item, it will be removed from the knowledge base. + * + * @param string $appId + * @param string $providerId + * @param string $op UpdateAccessOp::* + * @param array $userIds + * @return void + * @since 4.0.0 + */ + public function updateAccessProvider(string $appId, string $providerId, string $op, array $userIds): void { + $this->collectAllContentProviders(); + + $this->actionService->updateAccessProvider( + $op, + $userIds, + ProviderConfigService::getConfigKey($appId, $providerId), + ); + } + /** * Update access for a content item for specified users declaratively. * This overwrites the access list for the content item, From 82ba1447ecec7073728bd7b891610f1718ef023a Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Mon, 9 Dec 2024 18:33:31 +0530 Subject: [PATCH 03/14] add action job again on exceptions Signed-off-by: Anupam Kumar --- lib/BackgroundJobs/ActionJob.php | 119 ++++++++++++++++--------------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/lib/BackgroundJobs/ActionJob.php b/lib/BackgroundJobs/ActionJob.php index a36d788..8ad74c7 100644 --- a/lib/BackgroundJobs/ActionJob.php +++ b/lib/BackgroundJobs/ActionJob.php @@ -44,72 +44,77 @@ protected function run($argument): void { return; } - foreach ($entities as $entity) { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); - - switch ($entity->getType()) { - case ActionType::DELETE_SOURCE_IDS: - $decoded = json_decode($entity->getPayload(), true); - if (!is_array($decoded) || !isset($decoded['sourceIds'])) { - $this->logger->warning('Invalid payload for DELETE_SOURCE_IDS action', ['payload' => $entity->getPayload()]); + try { + foreach ($entities as $entity) { + $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + + switch ($entity->getType()) { + case ActionType::DELETE_SOURCE_IDS: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['sourceIds'])) { + $this->logger->warning('Invalid payload for DELETE_SOURCE_IDS action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->deleteSources($decoded['sourceIds']); break; - } - $this->networkService->deleteSources($decoded['sourceIds']); - break; - - case ActionType::DELETE_PROVIDER_ID: - $decoded = json_decode($entity->getPayload(), true); - if (!is_array($decoded) || !isset($decoded['providerId'])) { - $this->logger->warning('Invalid payload for DELETE_PROVIDER_ID action', ['payload' => $entity->getPayload()]); + + case ActionType::DELETE_PROVIDER_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['providerId'])) { + $this->logger->warning('Invalid payload for DELETE_PROVIDER_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->deleteProvider($decoded['providerId']); break; - } - $this->networkService->deleteProvider($decoded['providerId']); - break; - - case ActionType::DELETE_USER_ID: - $decoded = json_decode($entity->getPayload(), true); - if (!is_array($decoded) || !isset($decoded['userId'])) { - $this->logger->warning('Invalid payload for DELETE_USER_ID action', ['payload' => $entity->getPayload()]); + + case ActionType::DELETE_USER_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['userId'])) { + $this->logger->warning('Invalid payload for DELETE_USER_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->deleteUser($decoded['userId']); break; - } - $this->networkService->deleteUser($decoded['userId']); - break; - - case ActionType::UPDATE_ACCESS_SOURCE_ID: - $decoded = json_decode($entity->getPayload(), true); - if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) { - $this->logger->warning('Invalid payload for UPDATE_ACCESS_SOURCE_ID action', ['payload' => $entity->getPayload()]); + + case ActionType::UPDATE_ACCESS_SOURCE_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) { + $this->logger->warning('Invalid payload for UPDATE_ACCESS_SOURCE_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->updateAccess($decoded['op'], $decoded['userIds'], $decoded['sourceId']); break; - } - $this->networkService->updateAccess($decoded['op'], $decoded['userIds'], $decoded['sourceId']); - break; - - case ActionType::UPDATE_ACCESS_PROVIDER_ID: - $decoded = json_decode($entity->getPayload(), true); - if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['providerId'])) { - $this->logger->warning('Invalid payload for UPDATE_ACCESS_PROVIDER_ID action', ['payload' => $entity->getPayload()]); + + case ActionType::UPDATE_ACCESS_PROVIDER_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['providerId'])) { + $this->logger->warning('Invalid payload for UPDATE_ACCESS_PROVIDER_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->updateAccessProvider($decoded['op'], $decoded['userIds'], $decoded['providerId']); break; - } - $this->networkService->updateAccessProvider($decoded['op'], $decoded['userIds'], $decoded['providerId']); - break; - - case ActionType::UPDATE_ACCESS_DECL_SOURCE_ID: - $decoded = json_decode($entity->getPayload(), true); - if (!is_array($decoded) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) { - $this->logger->warning('Invalid payload for UPDATE_ACCESS_DECL_SOURCE_ID action', ['payload' => $entity->getPayload()]); + + case ActionType::UPDATE_ACCESS_DECL_SOURCE_ID: + $decoded = json_decode($entity->getPayload(), true); + if (!is_array($decoded) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) { + $this->logger->warning('Invalid payload for UPDATE_ACCESS_DECL_SOURCE_ID action', ['payload' => $entity->getPayload()]); + break; + } + $this->networkService->updateAccessDeclarative($decoded['userIds'], $decoded['sourceId']); break; - } - $this->networkService->updateAccessDeclarative($decoded['userIds'], $decoded['sourceId']); - break; - default: - $this->logger->warning('Unknown action type', ['type' => $entity->getType()]); + default: + $this->logger->warning('Unknown action type', ['type' => $entity->getType()]); + } } - } - foreach ($entities as $entity) { - $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); - $this->actionMapper->removeFromQueue($entity); + foreach ($entities as $entity) { + $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + $this->actionMapper->removeFromQueue($entity); + } + } catch (\Throwable $e) { + $this->jobList->add(static::class); + throw $e; } $this->jobList->add(static::class); From 6a5ba616708065d9f92b337e486e135772813467 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Mon, 9 Dec 2024 18:33:51 +0530 Subject: [PATCH 04/14] network: only throw on 5xx errors Signed-off-by: Anupam Kumar --- lib/Service/LangRopeService.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php index 31d9121..214e0cc 100644 --- a/lib/Service/LangRopeService.php +++ b/lib/Service/LangRopeService.php @@ -160,12 +160,16 @@ private function requestToExApp( 'code' => $response->getStatusCode(), 'response' => $response->getBody(), ]); - throw new RuntimeException( - 'Error during request to Context Chat Backend (ExApp) with status code ' - . $response->getStatusCode() - . ': ' - . (isset($finalBody['error']) ? $finalBody['error'] : 'unknown error') - ); + + if (intval($response->getStatusCode() / 100) === 5) { + // only throw for 5xx errors + throw new RuntimeException( + 'Error during request to Context Chat Backend (ExApp) with status code ' + . $response->getStatusCode() + . ': ' + . (isset($finalBody['error']) ? $finalBody['error'] : 'unknown error') + ); + } } return $finalBody; From 0147d11da2e82780d83d2da741d98ab4afc0bda5 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Mon, 9 Dec 2024 18:58:22 +0530 Subject: [PATCH 05/14] fix action service deleteSources json payload Signed-off-by: Anupam Kumar --- lib/Service/ActionService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/ActionService.php b/lib/Service/ActionService.php index f99f554..e94fb35 100644 --- a/lib/Service/ActionService.php +++ b/lib/Service/ActionService.php @@ -58,7 +58,7 @@ public function deleteSources(string ...$sourceIds): void { $batches = array_chunk($sourceIds, self::BATCH_SIZE); foreach ($batches as $batch) { - $payload = json_encode($batch); + $payload = json_encode(['sourceIds' => $batch]); if ($payload === false) { $this->logger->warning('Failed to json_encode sourceIds for deletion', ['sourceIds' => $batch]); continue; From bb7907de0e0e5bf05d76000368a446eb3edb5798 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Mon, 9 Dec 2024 20:09:14 +0530 Subject: [PATCH 06/14] rename migration to be the latest version Signed-off-by: Anupam Kumar --- ...0241206135634.php => Version004000000Date20241206135634.php} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename lib/Migration/{Version4000Date20241206135634.php => Version004000000Date20241206135634.php} (94%) diff --git a/lib/Migration/Version4000Date20241206135634.php b/lib/Migration/Version004000000Date20241206135634.php similarity index 94% rename from lib/Migration/Version4000Date20241206135634.php rename to lib/Migration/Version004000000Date20241206135634.php index 17c9da6..c1222f3 100644 --- a/lib/Migration/Version4000Date20241206135634.php +++ b/lib/Migration/Version004000000Date20241206135634.php @@ -15,7 +15,7 @@ use OCP\Migration\IOutput; use OCP\Migration\SimpleMigrationStep; -class Version4000Date20241206135634 extends SimpleMigrationStep { +class Version004000000Date20241206135634 extends SimpleMigrationStep { /** * @param IOutput $output * @param Closure(): ISchemaWrapper $schemaClosure From ea772181d47171aed65b70b28540a0eec51300ac Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Tue, 10 Dec 2024 15:22:54 +0530 Subject: [PATCH 07/14] paper cuts Signed-off-by: Anupam Kumar --- lib/Public/ContentManager.php | 2 +- lib/Service/LangRopeService.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Public/ContentManager.php b/lib/Public/ContentManager.php index 4e9bbd9..d42dadf 100644 --- a/lib/Public/ContentManager.php +++ b/lib/Public/ContentManager.php @@ -231,7 +231,7 @@ public function deleteProvider(string $appId, string $providerId): void { * @return void * @since 4.0.0 */ - public function deleteContent(string $appId, string $providerId, string ...$itemIds): void { + public function deleteContent(string $appId, string $providerId, array $itemIds): void { $this->collectAllContentProviders(); $providerKey = ProviderConfigService::getConfigKey($appId, $providerId); diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php index 214e0cc..fe98d9f 100644 --- a/lib/Service/LangRopeService.php +++ b/lib/Service/LangRopeService.php @@ -156,15 +156,15 @@ private function requestToExApp( } if (intval($response->getStatusCode() / 100) !== 2) { - $this->logger->error('Error during request to Context Chat Backend (ExApp)', [ + $this->logger->error('Error received from Context Chat Backend (ExApp)', [ 'code' => $response->getStatusCode(), - 'response' => $response->getBody(), + 'response' => $finalBody, ]); if (intval($response->getStatusCode() / 100) === 5) { // only throw for 5xx errors throw new RuntimeException( - 'Error during request to Context Chat Backend (ExApp) with status code ' + 'Error received from Context Chat Backend (ExApp) with status code ' . $response->getStatusCode() . ': ' . (isset($finalBody['error']) ? $finalBody['error'] : 'unknown error') From 5fc3cabd42500201cf3e69b61854e3f08a0126a7 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Thu, 12 Dec 2024 00:57:29 +0530 Subject: [PATCH 08/14] fix allow files in NodeWrittenEvent Signed-off-by: Anupam Kumar --- lib/Listener/FileListener.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Listener/FileListener.php b/lib/Listener/FileListener.php index b1c6d24..25922d2 100644 --- a/lib/Listener/FileListener.php +++ b/lib/Listener/FileListener.php @@ -53,7 +53,7 @@ public function __construct( public function handle(Event $event): void { if ($event instanceof NodeWrittenEvent) { $node = $event->getNode(); - if ($node instanceof File) { + if (!$node instanceof File) { return; } $this->postInsert($node, false, true); From 89a2df2c7207554a0ce3dead2f00323c181c7d40 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Thu, 12 Dec 2024 01:06:23 +0530 Subject: [PATCH 09/14] fix: do not index files_versions|files_trashbin files Signed-off-by: Anupam Kumar --- lib/Listener/FileListener.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/Listener/FileListener.php b/lib/Listener/FileListener.php index 25922d2..e86a0f0 100644 --- a/lib/Listener/FileListener.php +++ b/lib/Listener/FileListener.php @@ -238,6 +238,10 @@ public function postInsert(Node $node, bool $recurse = true, bool $update = fals return; } + if (!$this->allowedPath($node)) { + return; + } + $queueFile = new QueueFile(); if ($node->getMountPoint()->getNumericStorageId() === null) { return; @@ -260,8 +264,13 @@ public function postInsert(Node $node, bool $recurse = true, bool $update = fals } } - private function allowedMimeType(File $file): bool { + private function allowedMimeType(Node $file): bool { $mimeType = $file->getMimeType(); return in_array($mimeType, Application::MIMETYPES, true); } + + private function allowedPath(Node $file): bool { + $path = $file->getPath(); + return !preg_match('/^\/.+\/(files_versions|files_trashbin)\/.+/', $path, $matches); + } } From a785c28f82b79688b72d62c783ce12a4c559f78a Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Thu, 12 Dec 2024 17:22:10 +0530 Subject: [PATCH 10/14] remove username and 'files' from source title Signed-off-by: Anupam Kumar --- lib/BackgroundJobs/IndexerJob.php | 2 +- lib/Service/ScanService.php | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/BackgroundJobs/IndexerJob.php b/lib/BackgroundJobs/IndexerJob.php index 68d85aa..f85796d 100644 --- a/lib/BackgroundJobs/IndexerJob.php +++ b/lib/BackgroundJobs/IndexerJob.php @@ -173,7 +173,7 @@ protected function index(array $files): void { $sources[] = new Source( $userIds, ProviderConfigService::getSourceId($file->getId()), - $file->getPath(), + substr($file->getInternalPath(), 6), // remove 'files/' prefix $fileHandle, $file->getMtime(), $file->getMimeType(), diff --git a/lib/Service/ScanService.php b/lib/Service/ScanService.php index 96b51a5..3d5b5db 100644 --- a/lib/Service/ScanService.php +++ b/lib/Service/ScanService.php @@ -110,10 +110,11 @@ public function getSourceFromFile(array $mimeTypeFilter, File $node): ?Source { $providerKey = ProviderConfigService::getDefaultProviderKey(); $sourceId = ProviderConfigService::getSourceId($node->getId()); $userIds = $this->storageService->getUsersForFileId($node->getId()); + $path = substr($node->getInternalPath(), 6); // remove 'files/' prefix return new Source( $userIds, $sourceId, - $node->getPath(), + $path, $fileHandle, $node->getMTime(), $node->getMimeType(), From 649a6bf969919aee6aed046af288b0e46c1cd466 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Thu, 12 Dec 2024 19:27:57 +0530 Subject: [PATCH 11/14] address review comments Signed-off-by: Anupam Kumar --- lib/Public/ContentManager.php | 2 ++ lib/Service/ActionService.php | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/Public/ContentManager.php b/lib/Public/ContentManager.php index d42dadf..96aa21b 100644 --- a/lib/Public/ContentManager.php +++ b/lib/Public/ContentManager.php @@ -111,6 +111,7 @@ public function submitContent(string $appId, array $items): void { * @param string $itemId * @param array $users * @return void + * @since 1.1.0 * @deprecated 4.0.0 */ public function removeContentForUsers(string $appId, string $providerId, string $itemId, array $users): void { @@ -131,6 +132,7 @@ public function removeContentForUsers(string $appId, string $providerId, string * @param string $providerId * @param array $users * @return void + * @since 1.1.0 * @deprecated 4.0.0 */ public function removeAllContentForUsers(string $appId, string $providerId, array $users): void { diff --git a/lib/Service/ActionService.php b/lib/Service/ActionService.php index e94fb35..4487695 100644 --- a/lib/Service/ActionService.php +++ b/lib/Service/ActionService.php @@ -18,6 +18,7 @@ use OCA\ContextChat\Db\QueueAction; use OCA\ContextChat\Db\QueueActionMapper; use OCA\ContextChat\Type\ActionType; +use OCA\ContextChat\Type\UpdateAccessOp; use OCP\BackgroundJob\IJobList; use Psr\Log\LoggerInterface; @@ -32,7 +33,7 @@ public function __construct( } /** - * @param string $type + * @param ActionType::* $type * @param string $payload * @return void */ @@ -94,7 +95,7 @@ public function deleteUser(string $userId): void { } /** - * @param string $op + * @param UpdateAccessOp::* $op * @param string[] $userIds * @param string $sourceId * @return void @@ -109,7 +110,7 @@ public function updateAccess(string $op, array $userIds, string $sourceId): void } /** - * @param string $op + * @param UpdateAccessOp::* $op * @param string[] $userIds * @param string $providerId * @return void From e04be83d7ba49c5e0e22fd521265674286ff3b07 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Thu, 12 Dec 2024 23:46:38 +0530 Subject: [PATCH 12/14] update share listener logic Signed-off-by: Anupam Kumar --- lib/AppInfo/Application.php | 5 +- lib/Listener/FileListener.php | 100 ------------------ lib/Listener/ShareListener.php | 178 +++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 102 deletions(-) create mode 100644 lib/Listener/ShareListener.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 80d36e6..2d4d776 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -12,6 +12,7 @@ use OCA\ContextChat\Listener\AppDisableListener; use OCA\ContextChat\Listener\FileListener; +use OCA\ContextChat\Listener\ShareListener; use OCA\ContextChat\Listener\UserDeletedListener; use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\TaskProcessing\ContextChatProvider; @@ -74,13 +75,13 @@ public function __construct(array $urlParams = []) { public function register(IRegistrationContext $context): void { $context->registerEventListener(BeforeNodeDeletedEvent::class, FileListener::class); $context->registerEventListener(NodeCreatedEvent::class, FileListener::class); - $context->registerEventListener(ShareCreatedEvent::class, FileListener::class); - $context->registerEventListener(ShareDeletedEvent::class, FileListener::class); $context->registerEventListener(CacheEntryInsertedEvent::class, FileListener::class); $context->registerEventListener(NodeRemovedFromCache::class, FileListener::class); $context->registerEventListener(NodeWrittenEvent::class, FileListener::class); $context->registerEventListener(AppDisableEvent::class, AppDisableListener::class); $context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class); + $context->registerEventListener(ShareCreatedEvent::class, ShareListener::class); + $context->registerEventListener(ShareDeletedEvent::class, ShareListener::class); $context->registerTaskProcessingTaskType(ContextChatTaskType::class); $context->registerTaskProcessingProvider(ContextChatProvider::class); diff --git a/lib/Listener/FileListener.php b/lib/Listener/FileListener.php index e86a0f0..91a8e11 100644 --- a/lib/Listener/FileListener.php +++ b/lib/Listener/FileListener.php @@ -10,11 +10,9 @@ use OCA\ContextChat\AppInfo\Application; use OCA\ContextChat\Db\QueueFile; -use OCA\ContextChat\Public\UpdateAccessOp; use OCA\ContextChat\Service\ActionService; use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Service\QueueService; -use OCA\ContextChat\Service\StorageService; use OCP\DB\Exception; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; @@ -30,9 +28,6 @@ use OCP\Files\IRootFolder; use OCP\Files\Node; use OCP\Files\NotFoundException; -use OCP\Share\Events\ShareCreatedEvent; -use OCP\Share\Events\ShareDeletedEvent; -use OCP\Share\IManager; use Psr\Log\LoggerInterface; /** @@ -43,8 +38,6 @@ class FileListener implements IEventListener { public function __construct( private LoggerInterface $logger, private QueueService $queue, - private StorageService $storageService, - private IManager $shareManager, private IRootFolder $rootFolder, private ActionService $actionService, ) { @@ -59,99 +52,6 @@ public function handle(Event $event): void { $this->postInsert($node, false, true); } - if ($event instanceof ShareCreatedEvent) { - $share = $event->getShare(); - $node = $share->getNode(); - - switch ($share->getShareType()) { - case \OCP\Share\IShare::TYPE_USER: - $userIds = [$share->getSharedWith()]; - break; - case \OCP\Share\IShare::TYPE_GROUP: - // todo: probably a group listener so when a user enters/leaves a group, we can update the access for all files shared with that group - $accessList = $this->shareManager->getAccessList($node, true, true); - /** - * @var string[] $userIds - */ - $userIds = array_keys($accessList['users']); - break; - default: - return; - } - - if ($node->getType() === FileInfo::TYPE_FOLDER) { - $mount = $node->getMountPoint(); - if ($mount->getNumericStorageId() === null) { - return; - } - $files = $this->storageService->getFilesInMount($mount->getNumericStorageId(), $node->getId(), 0, 0); - foreach ($files as $fileId) { - $file = current($this->rootFolder->getById($fileId)); - if (!$file instanceof File) { - continue; - } - $this->actionService->updateAccess( - UpdateAccessOp::ALLOW, - $userIds, - ProviderConfigService::getSourceId($file->getId()), - ); - } - } else { - $this->actionService->updateAccess( - UpdateAccessOp::ALLOW, - $userIds, - ProviderConfigService::getSourceId($node->getId()), - ); - } - } - - if ($event instanceof ShareDeletedEvent) { - $share = $event->getShare(); - $node = $share->getNode(); - - $accessList = $this->shareManager->getAccessList($node, true, true); - /** - * @var string[] $userIds - */ - $userIds = array_keys($accessList['users']); - - if ($node instanceof Folder) { - $mount = $node->getMountPoint(); - if ($mount->getNumericStorageId() === null) { - return; - } - $files = $this->storageService->getFilesInMount($mount->getNumericStorageId(), $node->getId(), 0, 0); - $fileRefs = []; - - foreach ($files as $fileId) { - $node = current($this->rootFolder->getById($fileId)); - if (!$node instanceof File) { - continue; - } - $fileRefs[] = ProviderConfigService::getSourceId($node->getId()); - } - - foreach ($fileRefs as $fileRef) { - $this->actionService->updateAccess( - UpdateAccessOp::DENY, - $userIds, - $fileRef, - ); - } - } else { - if (!$this->allowedMimeType($node)) { - return; - } - - $fileRef = ProviderConfigService::getSourceId($node->getId()); - $this->actionService->updateAccess( - UpdateAccessOp::DENY, - $userIds, - $fileRef, - ); - } - } - if ($event instanceof BeforeNodeDeletedEvent) { $this->postDelete($event->getNode(), false); return; diff --git a/lib/Listener/ShareListener.php b/lib/Listener/ShareListener.php new file mode 100644 index 0000000..1290a73 --- /dev/null +++ b/lib/Listener/ShareListener.php @@ -0,0 +1,178 @@ + + * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file. + */ +declare(strict_types=1); +namespace OCA\ContextChat\Listener; + +use OCA\ContextChat\AppInfo\Application; +use OCA\ContextChat\Public\UpdateAccessOp; +use OCA\ContextChat\Service\ActionService; +use OCA\ContextChat\Service\ProviderConfigService; +use OCA\ContextChat\Service\StorageService; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\File; +use OCP\Files\FileInfo; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\Node; +use OCP\IGroupManager; +use OCP\Share\Events\ShareCreatedEvent; +use OCP\Share\Events\ShareDeletedEvent; +use OCP\Share\IManager; +use Psr\Log\LoggerInterface; + +/** + * @template-implements IEventListener + */ +class ShareListener implements IEventListener { + + public function __construct( + private LoggerInterface $logger, + private StorageService $storageService, + private IManager $shareManager, + private IRootFolder $rootFolder, + private ActionService $actionService, + private IGroupManager $groupManager, + ) { + } + + public function handle(Event $event): void { + if ($event instanceof ShareCreatedEvent) { + $share = $event->getShare(); + $node = $share->getNode(); + + switch ($share->getShareType()) { + case \OCP\Share\IShare::TYPE_USER: + $userIds = [$share->getSharedWith()]; + break; + case \OCP\Share\IShare::TYPE_GROUP: + // todo: probably a group listener so when a user enters/leaves a group, we can update the access for all files shared with that group + $accessList = $this->shareManager->getAccessList($node, true, true); + /** + * @var string[] $userIds + */ + $userIds = array_keys($accessList['users']); + break; + default: + return; + } + + if ($node->getType() === FileInfo::TYPE_FOLDER) { + $mount = $node->getMountPoint(); + if ($mount->getNumericStorageId() === null) { + return; + } + $files = $this->storageService->getFilesInMount($mount->getNumericStorageId(), $node->getId(), 0, 0); + foreach ($files as $fileId) { + $file = current($this->rootFolder->getById($fileId)); + if (!$file instanceof File) { + continue; + } + $this->actionService->updateAccess( + UpdateAccessOp::ALLOW, + $userIds, + ProviderConfigService::getSourceId($file->getId()), + ); + } + } else { + $this->actionService->updateAccess( + UpdateAccessOp::ALLOW, + $userIds, + ProviderConfigService::getSourceId($node->getId()), + ); + } + } + + if ($event instanceof ShareDeletedEvent) { + $share = $event->getShare(); + $node = $share->getNode(); + + // fileUserIds list is not fully accurate and doesn't update until the user(s) + // in question logs in again, so we need to get the share access list + // and the user(s) from whom the file was unshared with to update the access list, + // keeping the access for the user(s) who still have access to the file through + // file mounts. + + switch ($share->getShareType()) { + case \OCP\Share\IShare::TYPE_USER: + $unsharedWith = [$share->getSharedWith()]; + break; + case \OCP\Share\IShare::TYPE_GROUP: + $unsharedWithGroup = $this->groupManager->get($share->getSharedWith()); + if ($unsharedWithGroup === null) { + $this->logger->warning('Could not find group with id ' . $share->getSharedWith()); + return; + } + $unsharedWith = array_keys($unsharedWithGroup->getUsers()); + break; + default: + return; + } + + $shareAccessList = $this->shareManager->getAccessList($node, true, true); + /** + * @var string[] $shareUserIds + */ + $shareUserIds = array_keys($shareAccessList['users']); + $fileUserIds = $this->storageService->getUsersForFileId($node->getId()); + + // the user(s) who have really lost access to the file and don't have access to it + // through any other shares + $reallyUnsharedWith = array_diff($unsharedWith, $shareUserIds); + + // the user(s) who have access to the file through file mounts, excluding the user(s) + // who have really lost access to the file and are present in $fileUserIds list + $realFileUserIds = array_diff($fileUserIds, $reallyUnsharedWith); + // merge the share and file lists to get the final list of user(s) who have access to the file + $userIds = array_unique(array_merge($realFileUserIds, $shareUserIds)); + + if ($node instanceof Folder) { + $mount = $node->getMountPoint(); + if ($mount->getNumericStorageId() === null) { + return; + } + $files = $this->storageService->getFilesInMount($mount->getNumericStorageId(), $node->getId(), 0, 0); + $files = []; + + foreach ($files as $fileId) { + $node = current($this->rootFolder->getById($fileId)); + if (!$node instanceof File) { + continue; + } + $files[] = $node; + } + + foreach ($files as $file) { + $owner = $file->getOwner()->getUID(); + $this->actionService->updateAccessDeclSource( + $userIds, + ProviderConfigService::getSourceId($file->getId()), + $owner, + ); + } + } else { + if (!$this->allowedMimeType($node)) { + return; + } + + $fileRef = ProviderConfigService::getSourceId($node->getId()); + $owner = $node->getOwner()->getUID(); + $this->actionService->updateAccessDeclSource( + $userIds, + $fileRef, + $owner, + ); + } + } + } + + private function allowedMimeType(Node $file): bool { + $mimeType = $file->getMimeType(); + return in_array($mimeType, Application::MIMETYPES, true); + } +} From 42f774a3fc47791479b7dd1222e98efd55577b16 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Fri, 13 Dec 2024 00:06:48 +0530 Subject: [PATCH 13/14] add locked files to the end of the index queue Signed-off-by: Anupam Kumar --- lib/BackgroundJobs/IndexerJob.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/BackgroundJobs/IndexerJob.php b/lib/BackgroundJobs/IndexerJob.php index f85796d..da5243a 100644 --- a/lib/BackgroundJobs/IndexerJob.php +++ b/lib/BackgroundJobs/IndexerJob.php @@ -135,6 +135,7 @@ protected function index(array $files): void { $sources = []; $allSourceIds = []; $loadedSources = []; + $retryQFiles = []; $size = 0; foreach ($files as $queueFile) { @@ -161,9 +162,13 @@ protected function index(array $files): void { try { try { $fileHandle = $file->fopen('r'); - } catch (LockedException|NotPermittedException $e) { + } catch (NotPermittedException $e) { $this->logger->error('Could not open file ' . $file->getPath() . ' for reading', ['exception' => $e]); continue; + } catch (LockedException $e) { + $retryQFiles[] = $queueFile; + $this->logger->info('File ' . $file->getPath() . ' is locked, could not read for indexing. Adding it to the next batch.'); + continue; } if (!is_resource($fileHandle)) { $this->logger->warning('File handle for' . $file->getPath() . ' is not readable'); @@ -197,6 +202,10 @@ protected function index(array $files): void { try { $this->queue->removeFromQueue($files); + // add files that were locked to the end of the queue + foreach ($retryQFiles as $queueFile) { + $this->queue->insertIntoQueue($queueFile); + } } catch (Exception $e) { $this->logger->error('Could not remove indexed files from queue', ['exception' => $e]); } From e5b7b8911e5c6a765589b27d265a6ea7e26980d2 Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Fri, 13 Dec 2024 00:43:13 +0530 Subject: [PATCH 14/14] fix remnants from expts Signed-off-by: Anupam Kumar --- lib/Listener/ShareListener.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/Listener/ShareListener.php b/lib/Listener/ShareListener.php index 1290a73..3d4b84d 100644 --- a/lib/Listener/ShareListener.php +++ b/lib/Listener/ShareListener.php @@ -51,7 +51,6 @@ public function handle(Event $event): void { $userIds = [$share->getSharedWith()]; break; case \OCP\Share\IShare::TYPE_GROUP: - // todo: probably a group listener so when a user enters/leaves a group, we can update the access for all files shared with that group $accessList = $this->shareManager->getAccessList($node, true, true); /** * @var string[] $userIds @@ -148,11 +147,9 @@ public function handle(Event $event): void { } foreach ($files as $file) { - $owner = $file->getOwner()->getUID(); $this->actionService->updateAccessDeclSource( $userIds, ProviderConfigService::getSourceId($file->getId()), - $owner, ); } } else { @@ -161,11 +158,9 @@ public function handle(Event $event): void { } $fileRef = ProviderConfigService::getSourceId($node->getId()); - $owner = $node->getOwner()->getUID(); $this->actionService->updateAccessDeclSource( $userIds, $fileRef, - $owner, ); } }