refactor and fix scoped context chat

- added routes: /providers, /default-provider-key - merge ScopedContextChatProvider to ContextChatProvider - make use of ProviderService everywhere, which extends ProviderConfigService - 'file' provider type is now files__default - handle mimetype array parsing error in `context_chat:scan` command Signed-off-by: Anupam Kumar <[email protected]>
nextcloud · Mar 21, 2024 · fa1c296 · fa1c296
1 parent 661ff4e
commit fa1c296
Show file tree

Hide file tree

Showing 20 changed files with 469 additions and 264 deletions.
diff --git a/appinfo/routes.php b/appinfo/routes.php
@@ -13,5 +13,8 @@
 	'routes' => [
 		['name' => 'config#setConfig', 'url' => '/config', 'verb' => 'PUT'],
 		['name' => 'config#setAdminConfig', 'url' => '/admin-config', 'verb' => 'PUT'],
+
+		['name' => 'provider#getProviders', 'url' => '/providers', 'verb' => 'GET'],
+		['name' => 'provider#getDefaultProviderKey', 'url' => '/default-provider-key', 'verb' => 'GET'],
 	],
 ];
diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php
@@ -11,9 +11,9 @@
 
 use OCA\ContextChat\Listener\AppDisableListener;
 use OCA\ContextChat\Listener\FileListener;
+use OCA\ContextChat\Service\ProviderConfigService;
 use OCA\ContextChat\TextProcessing\ContextChatProvider;
 use OCA\ContextChat\TextProcessing\FreePromptProvider;
-use OCA\ContextChat\TextProcessing\ScopedContextChatProvider;
 use OCP\App\Events\AppDisableEvent;
 use OCP\AppFramework\App;
 use OCP\AppFramework\Bootstrap\IBootContext;
@@ -24,6 +24,7 @@
 use OCP\Files\Events\Node\NodeCreatedEvent;
 use OCP\Files\Events\Node\NodeWrittenEvent;
 use OCP\Files\Events\NodeRemovedFromCache;
+use OCP\IConfig;
 use OCP\Share\Events\ShareCreatedEvent;
 use OCP\Share\Events\ShareDeletedEvent;
 
@@ -58,8 +59,13 @@ class Application extends App implements IBootstrap {
 		'text/org',
 	];
 
+	private IConfig $config;
+
 	public function __construct(array $urlParams = []) {
 		parent::__construct(self::APP_ID, $urlParams);
+
+		$container = $this->getContainer();
+		$this->config = $container->get(IConfig::class);
 	}
 
 	public function register(IRegistrationContext $context): void {
@@ -73,7 +79,10 @@ public function register(IRegistrationContext $context): void {
 		$context->registerEventListener(AppDisableEvent::class, AppDisableListener::class);
 		$context->registerTextProcessingProvider(ContextChatProvider::class);
 		$context->registerTextProcessingProvider(FreePromptProvider::class);
-		$context->registerTextProcessingProvider(ScopedContextChatProvider::class);
+
+		$providerConfigService = new ProviderConfigService($this->config);
+		/** @psalm-suppress ArgumentTypeCoercion, UndefinedClass  */
+		$providerConfigService->updateProvider('files', 'default', '', true);
 	}
 
 	public function boot(IBootContext $context): void {

diff --git a/lib/BackgroundJobs/IndexerJob.php b/lib/BackgroundJobs/IndexerJob.php
@@ -9,6 +9,7 @@
 
 use OCA\ContextChat\Db\QueueFile;
 use OCA\ContextChat\Service\LangRopeService;
+use OCA\ContextChat\Service\ProviderService;
 use OCA\ContextChat\Service\QueueService;
 use OCA\ContextChat\Service\StorageService;
 use OCA\ContextChat\Type\Source;
@@ -125,7 +126,15 @@ protected function index(array $files): void {
 			$userIds = $this->storageService->getUsersForFileId($queueFile->getFileId());
 			foreach ($userIds as $userId) {
 				try {
-					$source = new Source($userId, 'file: ' . $file->getId(), $file->getPath(), $fileHandle, $file->getMtime(), $file->getMimeType(), 'file');
+					$source = new Source(
+						$userId,
+						ProviderService::getSourceId($file->getId()),
+						$file->getPath(),
+						$fileHandle,
+						$file->getMtime(),
+						$file->getMimeType(),
+						ProviderService::getDefaultProviderKey(),
+					);
 				} catch (InvalidPathException|NotFoundException $e) {
 					$this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]);
 					continue 2;

diff --git a/lib/BackgroundJobs/InitialContentImportJob.php b/lib/BackgroundJobs/InitialContentImportJob.php
@@ -14,7 +14,7 @@
 namespace OCA\ContextChat\BackgroundJobs;
 
 use OCA\ContextChat\Public\IContentProvider;
-use OCA\ContextChat\Service\ProviderConfigService;
+use OCA\ContextChat\Service\ProviderService;
 use OCP\App\IAppManager;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\QueuedJob;
@@ -27,7 +27,7 @@
 class InitialContentImportJob extends QueuedJob {
 	public function __construct(
 		private IAppManager $appManager,
-		private ProviderConfigService $configService,
+		private ProviderService $providerService,
 		private LoggerInterface $logger,
 		private IUserManager $userMan,
 		ITimeFactory $timeFactory,
@@ -57,15 +57,15 @@ protected function run($argument): void {
 			return;
 		}
 
-		$registeredProviders = $this->configService->getProviders();
-		$identifier = ProviderConfigService::getConfigKey($providerObj->getAppId(), $providerObj->getId());
+		$registeredProviders = $this->providerService->getProviders();
+		$identifier = ProviderService::getConfigKey($providerObj->getAppId(), $providerObj->getId());
 		if (!isset($registeredProviders[$identifier])
 			|| $registeredProviders[$identifier]['isInitiated']
 		) {
 			return;
 		}
 
 		$providerObj->triggerInitialImport();
-		$this->configService->updateProvider($providerObj->getAppId(), $providerObj->getId(), $argument, true);
+		$this->providerService->updateProvider($providerObj->getAppId(), $providerObj->getId(), $argument, true);
 	}
 }
diff --git a/lib/BackgroundJobs/SubmitContentJob.php b/lib/BackgroundJobs/SubmitContentJob.php
@@ -16,7 +16,7 @@
 use OCA\ContextChat\Db\QueueContentItem;
 use OCA\ContextChat\Db\QueueContentItemMapper;
 use OCA\ContextChat\Service\LangRopeService;
-use OCA\ContextChat\Service\ProviderConfigService;
+use OCA\ContextChat\Service\ProviderService;
 use OCA\ContextChat\Type\Source;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
@@ -58,8 +58,8 @@ protected function run($argument): void {
 
 		foreach ($bucketed as $userId => $entities) {
 			$sources = array_map(function (QueueContentItem $item) use ($userId) {
-				$providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId());
-				$sourceId = $providerKey . ': ' . $item->getItemId();
+				$providerKey = ProviderService::getConfigKey($item->getAppId(), $item->getProviderId());
+				$sourceId = ProviderService::getSourceId($item->getItemId(), $providerKey);
 				return new Source(
 					$userId,
 					$sourceId,

diff --git a/lib/Command/Prompt.php b/lib/Command/Prompt.php
@@ -13,7 +13,6 @@
 namespace OCA\ContextChat\Command;
 
 use OCA\ContextChat\TextProcessing\ContextChatTaskType;
-use OCA\ContextChat\TextProcessing\ScopedContextChatTaskType;
 use OCA\ContextChat\Type\ScopeType;
 use OCP\TextProcessing\FreePromptTaskType;
 use OCP\TextProcessing\IManager;
@@ -80,26 +79,30 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 			throw new \InvalidArgumentException('Cannot use --context-sources with --context-provider');
 		}
 
-		if ($noContext) {
-			$task = new Task(FreePromptTaskType::class, $prompt, 'context_chat', $userId);
-		} elseif (!empty($contextSources)) {
-			$contextSources = preg_replace('/\s*,+\s*/', ',', $contextSources);
-			$contextSourcesArray = array_filter(explode(',', $contextSources), fn ($source) => !empty($source));
-			$task = new Task(ScopedContextChatTaskType::class, json_encode([
-				'scopeType' => ScopeType::SOURCE,
-				'scopeList' => $contextSourcesArray,
-				'prompt' => $prompt,
-			]), 'context_chat', $userId);
-		} elseif (!empty($contextProviders)) {
-			$contextProviders = preg_replace('/\s*,+\s*/', ',', $contextProviders);
-			$contextProvidersArray = array_filter(explode(',', $contextProviders), fn ($source) => !empty($source));
-			$task = new Task(ScopedContextChatTaskType::class, json_encode([
-				'scopeType' => ScopeType::PROVIDER,
-				'scopeList' => $contextProvidersArray,
-				'prompt' => $prompt,
-			]), 'context_chat', $userId);
-		} else {
-			$task = new Task(ContextChatTaskType::class, $prompt, 'context_chat', $userId);
+		try {
+			if ($noContext) {
+				$task = new Task(FreePromptTaskType::class, $prompt, 'context_chat', $userId);
+			} elseif (!empty($contextSources)) {
+				$contextSources = preg_replace('/\s*,+\s*/', ',', $contextSources);
+				$contextSourcesArray = array_filter(explode(',', $contextSources), fn ($source) => !empty($source));
+				$task = new Task(ContextChatTaskType::class, json_encode([
+					'scopeType' => ScopeType::SOURCE,
+					'scopeList' => $contextSourcesArray,
+					'prompt' => $prompt,
+				], JSON_THROW_ON_ERROR), 'context_chat', $userId);
+			} elseif (!empty($contextProviders)) {
+				$contextProviders = preg_replace('/\s*,+\s*/', ',', $contextProviders);
+				$contextProvidersArray = array_filter(explode(',', $contextProviders), fn ($source) => !empty($source));
+				$task = new Task(ContextChatTaskType::class, json_encode([
+					'scopeType' => ScopeType::PROVIDER,
+					'scopeList' => $contextProvidersArray,
+					'prompt' => $prompt,
+				], JSON_THROW_ON_ERROR), 'context_chat', $userId);
+			} else {
+				$task = new Task(ContextChatTaskType::class, json_encode([ 'prompt' => $prompt ], JSON_THROW_ON_ERROR), 'context_chat', $userId);
+			}
+		} catch (\JsonException $e) {
+			throw new \InvalidArgumentException('Invalid input, cannot encode JSON', intval($e->getCode()), $e);
 		}
 
 		$this->textProcessingManager->runTask($task);

diff --git a/lib/Command/ScanFiles.php b/lib/Command/ScanFiles.php
@@ -44,6 +44,11 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 			? explode(',', $input->getOption('mimetype'))
 			: Application::MIMETYPES;
 
+		if ($mimeTypeFilter === false) {
+			$output->writeln('Invalid mime type filter');
+			return 1;
+		}
+
 		$userId = $input->getArgument('user_id');
 		$scan = $this->scanService->scanUserFiles($userId, $mimeTypeFilter);
 		foreach ($scan as $s) {

diff --git a/lib/Controller/ProviderController.php b/lib/Controller/ProviderController.php
@@ -0,0 +1,58 @@
+<?php
+/**
+ * @copyright Copyright (c) 2024 Anupam Kumar <[email protected]>
+ *
+ * @author Anupam Kumar <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\ContextChat\Controller;
+
+use OCA\ContextChat\Service\ProviderService;
+use OCP\AppFramework\Controller;
+use OCP\AppFramework\Http\Attribute\NoAdminRequired;
+use OCP\AppFramework\Http\DataResponse;
+use OCP\IRequest;
+
+class ProviderController extends Controller {
+
+	public function __construct(
+		string $appName,
+		IRequest $request,
+		private ProviderService $providerService,
+	) {
+		parent::__construct($appName, $request);
+	}
+
+	/**
+	 * @return DataResponse
+	 */
+	#[NoAdminRequired]
+	public function getDefaultProviderKey(): DataResponse {
+		$providerKey = $this->providerService->getDefaultProviderKey();
+		return new DataResponse($providerKey);
+	}
+
+	/**
+	 * @return DataResponse
+	 */
+	#[NoAdminRequired]
+	public function getProviders(): DataResponse {
+		$providers = $this->providerService->getEnrichedProviders();
+		return new DataResponse($providers);
+	}
+}
diff --git a/lib/Listener/AppDisableListener.php b/lib/Listener/AppDisableListener.php
@@ -14,7 +14,7 @@
 namespace OCA\ContextChat\Listener;
 
 use OCA\ContextChat\Service\LangRopeService;
-use OCA\ContextChat\Service\ProviderConfigService;
+use OCA\ContextChat\Service\ProviderService;
 use OCP\App\Events\AppDisableEvent;
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
@@ -25,7 +25,7 @@
  */
 class AppDisableListener implements IEventListener {
 	public function __construct(
-		private ProviderConfigService $configService,
+		private ProviderService $providerService,
 		private LangRopeService $service,
 		private LoggerInterface $logger,
 	) {
@@ -36,7 +36,7 @@ public function handle(Event $event): void {
 			return;
 		}
 
-		foreach ($this->configService->getProviders() as $key => $values) {
+		foreach ($this->providerService->getProviders() as $key => $values) {
 			/** @var string[] */
 			$identifierValues = explode('__', $key, 2);
 
@@ -51,7 +51,7 @@ public function handle(Event $event): void {
 				continue;
 			}
 
-			$this->configService->removeProvider($appId, $providerId);
+			$this->providerService->removeProvider($appId, $providerId);
 			$this->service->deleteSourcesByProviderForAllUsers($providerId);
 		}
 	}

diff --git a/lib/Listener/FileListener.php b/lib/Listener/FileListener.php
@@ -10,6 +10,7 @@
 use OCA\ContextChat\AppInfo\Application;
 use OCA\ContextChat\Db\QueueFile;
 use OCA\ContextChat\Service\LangRopeService;
+use OCA\ContextChat\Service\ProviderService;
 use OCA\ContextChat\Service\QueueService;
 use OCA\ContextChat\Service\StorageService;
 use OCP\DB\Exception;
@@ -120,7 +121,7 @@ public function handle(Event $event): void {
 					if (!$node instanceof File) {
 						continue;
 					}
-					$fileRefs[] = 'file: ' . $node->getId();
+					$fileRefs[] = ProviderService::getSourceId($node->getId());
 				}
 
 				$this->langRopeService->deleteSources($userId, $fileRefs);
@@ -129,7 +130,7 @@ public function handle(Event $event): void {
 					return;
 				}
 
-				$fileRef = 'file: ' . $node->getId();
+				$fileRef = ProviderService::getSourceId($node->getId());
 				foreach ($userIds as $userId) {
 					$this->langRopeService->deleteSources($userId, [$fileRef]);
 				}
@@ -194,7 +195,7 @@ public function postDelete(Node $node, bool $recurse = true): void {
 		}
 
 		foreach ($this->storageService->getUsersForFileId($node->getId()) as $userId) {
-			$fileRef = 'file: ' . $node->getId();
+			$fileRef = ProviderService::getSourceId($node->getId());
 			$this->langRopeService->deleteSources($userId, [$fileRef]);
 		}
 	}

diff --git a/lib/Public/ContentManager.php b/lib/Public/ContentManager.php
@@ -16,7 +16,7 @@
 use OCA\ContextChat\Db\QueueContentItem;
 use OCA\ContextChat\Db\QueueContentItemMapper;
 use OCA\ContextChat\Service\LangRopeService;
-use OCA\ContextChat\Service\ProviderConfigService;
+use OCA\ContextChat\Service\ProviderService;
 use OCP\BackgroundJob\IJobList;
 use OCP\Server;
 use Psr\Container\ContainerExceptionInterface;
@@ -26,7 +26,7 @@
 class ContentManager {
 	public function __construct(
 		private IJobList $jobList,
-		private ProviderConfigService $configService,
+		private ProviderService $providerService,
 		private LangRopeService $service,
 		private QueueContentItemMapper $mapper,
 		private LoggerInterface $logger,
@@ -47,11 +47,11 @@ public function registerContentProvider(string $providerClass): void {
 			return;
 		}
 
-		if ($this->configService->hasProvider($providerObj->getAppId(), $providerObj->getId())) {
+		if ($this->providerService->hasProvider($providerObj->getAppId(), $providerObj->getId())) {
 			return;
 		}
 
-		$this->configService->updateProvider($providerObj->getAppId(), $providerObj->getId(), $providerClass);
+		$this->providerService->updateProvider($providerObj->getAppId(), $providerObj->getId(), $providerClass);
 
 		if (!$this->jobList->has(InitialContentImportJob::class, $providerClass)) {
 			$this->jobList->add(InitialContentImportJob::class, $providerClass);
@@ -98,7 +98,9 @@ public function submitContent(string $appId, array $items): void {
 	 */
 	public function removeContentForUsers(string $appId, string $providerId, string $itemId, array $users): void {
 		foreach ($users as $userId) {
-			$this->service->deleteSources($userId, [$this->configService->getConfigKey($appId, $providerId) . ": $itemId"]);
+			$this->service->deleteSources($userId, [
+				ProviderService::getSourceId($itemId, ProviderService::getConfigKey($appId, $providerId))
+			]);
 		}
 	}
 
@@ -112,7 +114,7 @@ public function removeContentForUsers(string $appId, string $providerId, string
 	 */
 	public function removeAllContentForUsers(string $appId, string $providerId, array $users): void {
 		foreach ($users as $userId) {
-			$this->service->deleteSourcesByProvider($userId, $this->configService->getConfigKey($appId, $providerId));
+			$this->service->deleteSourcesByProvider($userId, ProviderService::getConfigKey($appId, $providerId));
 		}
 	}
 }