refactor and fix scoped context chat

.github/workflows/integration-test.yml

@@ -20,7 +20,7 @@ concurrency:
 
 jobs:
   transcription:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       # do not stop on another job's failure
@@ -155,15 +155,18 @@ jobs:
       - name: Install and init backend
         run: |
           cd context_chat_backend
-          pip install --no-deps -r reqs.txt
+          pip install --no-deps -r requirements.txt
+          pip install --upgrade pip setuptools wheel
+          CMAKE_ARGS="-DLLAMA_OPENBLAS=on" pip install llama-cpp-python
           cp example.env .env
+          cp config.cpu.yaml config.yaml
           echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
           ./main.py &> backend_logs &
 
       - name: Register backend
         run: |
           ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
-          ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.1.1\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
+          ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.2.0\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
 
       - name: Scan files
         run: |

appinfo/routes.php

@@ -13,5 +13,8 @@
 	'routes' => [
 		['name' => 'config#setConfig', 'url' => '/config', 'verb' => 'PUT'],
 		['name' => 'config#setAdminConfig', 'url' => '/admin-config', 'verb' => 'PUT'],
+
+		['name' => 'provider#getProviders', 'url' => '/providers', 'verb' => 'GET'],
+		['name' => 'provider#getDefaultProviderKey', 'url' => '/default-provider-key', 'verb' => 'GET'],
 	],
 ];

lib/AppInfo/Application.php

@@ -11,9 +11,9 @@
 
 use OCA\ContextChat\Listener\AppDisableListener;
 use OCA\ContextChat\Listener\FileListener;
+use OCA\ContextChat\Service\ProviderConfigService;
 use OCA\ContextChat\TextProcessing\ContextChatProvider;
 use OCA\ContextChat\TextProcessing\FreePromptProvider;
-use OCA\ContextChat\TextProcessing\ScopedContextChatProvider;
 use OCP\App\Events\AppDisableEvent;
 use OCP\AppFramework\App;
 use OCP\AppFramework\Bootstrap\IBootContext;
@@ -24,6 +24,7 @@
 use OCP\Files\Events\Node\NodeCreatedEvent;
 use OCP\Files\Events\Node\NodeWrittenEvent;
 use OCP\Files\Events\NodeRemovedFromCache;
+use OCP\IConfig;
 use OCP\Share\Events\ShareCreatedEvent;
 use OCP\Share\Events\ShareDeletedEvent;
 
@@ -58,8 +59,13 @@ class Application extends App implements IBootstrap {
 		'text/org',
 	];
 
+	private IConfig $config;
+
 	public function __construct(array $urlParams = []) {
 		parent::__construct(self::APP_ID, $urlParams);
+
+		$container = $this->getContainer();
+		$this->config = $container->get(IConfig::class);
 	}
 
 	public function register(IRegistrationContext $context): void {
@@ -73,7 +79,10 @@ public function register(IRegistrationContext $context): void {
 		$context->registerEventListener(AppDisableEvent::class, AppDisableListener::class);
 		$context->registerTextProcessingProvider(ContextChatProvider::class);
 		$context->registerTextProcessingProvider(FreePromptProvider::class);
-		$context->registerTextProcessingProvider(ScopedContextChatProvider::class);
+
+		$providerConfigService = new ProviderConfigService($this->config);
+		/** @psalm-suppress ArgumentTypeCoercion, UndefinedClass  */
+		$providerConfigService->updateProvider('files', 'default', '', true);
 	}
 
 	public function boot(IBootContext $context): void {

lib/BackgroundJobs/IndexerJob.php

@@ -9,6 +9,7 @@
 
 use OCA\ContextChat\Db\QueueFile;
 use OCA\ContextChat\Service\LangRopeService;
+use OCA\ContextChat\Service\ProviderService;
 use OCA\ContextChat\Service\QueueService;
 use OCA\ContextChat\Service\StorageService;
 use OCA\ContextChat\Type\Source;
@@ -125,7 +126,15 @@ protected function index(array $files): void {
 			$userIds = $this->storageService->getUsersForFileId($queueFile->getFileId());
 			foreach ($userIds as $userId) {
 				try {
-					$source = new Source($userId, 'file: ' . $file->getId(), $file->getPath(), $fileHandle, $file->getMtime(), $file->getMimeType(), 'file');
+					$source = new Source(
+						$userId,
+						ProviderService::getSourceId($file->getId()),
+						$file->getPath(),
+						$fileHandle,
+						$file->getMtime(),
+						$file->getMimeType(),
+						ProviderService::getDefaultProviderKey(),
+					);
 				} catch (InvalidPathException|NotFoundException $e) {
 					$this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]);
 					continue 2;

lib/BackgroundJobs/InitialContentImportJob.php

@@ -14,7 +14,7 @@
 namespace OCA\ContextChat\BackgroundJobs;
 
 use OCA\ContextChat\Public\IContentProvider;
-use OCA\ContextChat\Service\ProviderConfigService;
+use OCA\ContextChat\Service\ProviderService;
 use OCP\App\IAppManager;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\QueuedJob;
@@ -27,7 +27,7 @@
 class InitialContentImportJob extends QueuedJob {
 	public function __construct(
 		private IAppManager $appManager,
-		private ProviderConfigService $configService,
+		private ProviderService $providerService,
 		private LoggerInterface $logger,
 		private IUserManager $userMan,
 		ITimeFactory $timeFactory,
@@ -57,15 +57,15 @@ protected function run($argument): void {
 			return;
 		}
 
-		$registeredProviders = $this->configService->getProviders();
-		$identifier = ProviderConfigService::getConfigKey($providerObj->getAppId(), $providerObj->getId());
+		$registeredProviders = $this->providerService->getProviders();
+		$identifier = ProviderService::getConfigKey($providerObj->getAppId(), $providerObj->getId());
 		if (!isset($registeredProviders[$identifier])
 			|| $registeredProviders[$identifier]['isInitiated']
 		) {
 			return;
 		}
 
 		$providerObj->triggerInitialImport();
-		$this->configService->updateProvider($providerObj->getAppId(), $providerObj->getId(), $argument, true);
+		$this->providerService->updateProvider($providerObj->getAppId(), $providerObj->getId(), $argument, true);
 	}
 }

lib/BackgroundJobs/SubmitContentJob.php

@@ -16,7 +16,7 @@
 use OCA\ContextChat\Db\QueueContentItem;
 use OCA\ContextChat\Db\QueueContentItemMapper;
 use OCA\ContextChat\Service\LangRopeService;
-use OCA\ContextChat\Service\ProviderConfigService;
+use OCA\ContextChat\Service\ProviderService;
 use OCA\ContextChat\Type\Source;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
@@ -58,8 +58,8 @@ protected function run($argument): void {
 
 		foreach ($bucketed as $userId => $entities) {
 			$sources = array_map(function (QueueContentItem $item) use ($userId) {
-				$providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId());
-				$sourceId = $providerKey . ': ' . $item->getItemId();
+				$providerKey = ProviderService::getConfigKey($item->getAppId(), $item->getProviderId());
+				$sourceId = ProviderService::getSourceId($item->getItemId(), $providerKey);
 				return new Source(
 					$userId,
 					$sourceId,

lib/Command/Prompt.php

@@ -13,7 +13,6 @@
 namespace OCA\ContextChat\Command;
 
 use OCA\ContextChat\TextProcessing\ContextChatTaskType;
-use OCA\ContextChat\TextProcessing\ScopedContextChatTaskType;
 use OCA\ContextChat\Type\ScopeType;
 use OCP\TextProcessing\FreePromptTaskType;
 use OCP\TextProcessing\IManager;
@@ -80,26 +79,30 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 			throw new \InvalidArgumentException('Cannot use --context-sources with --context-provider');
 		}
 
-		if ($noContext) {
-			$task = new Task(FreePromptTaskType::class, $prompt, 'context_chat', $userId);
-		} elseif (!empty($contextSources)) {
-			$contextSources = preg_replace('/\s*,+\s*/', ',', $contextSources);
-			$contextSourcesArray = array_filter(explode(',', $contextSources), fn ($source) => !empty($source));
-			$task = new Task(ScopedContextChatTaskType::class, json_encode([
-				'scopeType' => ScopeType::SOURCE,
-				'scopeList' => $contextSourcesArray,
-				'prompt' => $prompt,
-			]), 'context_chat', $userId);
-		} elseif (!empty($contextProviders)) {
-			$contextProviders = preg_replace('/\s*,+\s*/', ',', $contextProviders);
-			$contextProvidersArray = array_filter(explode(',', $contextProviders), fn ($source) => !empty($source));
-			$task = new Task(ScopedContextChatTaskType::class, json_encode([
-				'scopeType' => ScopeType::PROVIDER,
-				'scopeList' => $contextProvidersArray,
-				'prompt' => $prompt,
-			]), 'context_chat', $userId);
-		} else {
-			$task = new Task(ContextChatTaskType::class, $prompt, 'context_chat', $userId);
+		try {
+			if ($noContext) {
+				$task = new Task(FreePromptTaskType::class, $prompt, 'context_chat', $userId);
+			} elseif (!empty($contextSources)) {
+				$contextSources = preg_replace('/\s*,+\s*/', ',', $contextSources);
+				$contextSourcesArray = array_filter(explode(',', $contextSources), fn ($source) => !empty($source));
+				$task = new Task(ContextChatTaskType::class, json_encode([
+					'scopeType' => ScopeType::SOURCE,
+					'scopeList' => $contextSourcesArray,
+					'prompt' => $prompt,
+				], JSON_THROW_ON_ERROR), 'context_chat', $userId);
+			} elseif (!empty($contextProviders)) {
+				$contextProviders = preg_replace('/\s*,+\s*/', ',', $contextProviders);
+				$contextProvidersArray = array_filter(explode(',', $contextProviders), fn ($source) => !empty($source));
+				$task = new Task(ContextChatTaskType::class, json_encode([
+					'scopeType' => ScopeType::PROVIDER,
+					'scopeList' => $contextProvidersArray,
+					'prompt' => $prompt,
+				], JSON_THROW_ON_ERROR), 'context_chat', $userId);
+			} else {
+				$task = new Task(ContextChatTaskType::class, json_encode([ 'prompt' => $prompt ], JSON_THROW_ON_ERROR), 'context_chat', $userId);
+			}
+		} catch (\JsonException $e) {
+			throw new \InvalidArgumentException('Invalid input, cannot encode JSON', intval($e->getCode()), $e);
 		}
 
 		$this->textProcessingManager->runTask($task);

lib/Command/ScanFiles.php

@@ -44,6 +44,11 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 			? explode(',', $input->getOption('mimetype'))
 			: Application::MIMETYPES;
 
+		if ($mimeTypeFilter === false) {
+			$output->writeln('Invalid mime type filter');
+			return 1;
+		}
+
 		$userId = $input->getArgument('user_id');
 		$scan = $this->scanService->scanUserFiles($userId, $mimeTypeFilter);
 		foreach ($scan as $s) {

lib/Controller/ProviderController.php

@@ -0,0 +1,58 @@
+<?php
+/**
+ * @copyright Copyright (c) 2024 Anupam Kumar <[email protected]>
+ *
+ * @author Anupam Kumar <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\ContextChat\Controller;
+
+use OCA\ContextChat\Service\ProviderService;
+use OCP\AppFramework\Controller;
+use OCP\AppFramework\Http\Attribute\NoAdminRequired;
+use OCP\AppFramework\Http\DataResponse;
+use OCP\IRequest;
+
+class ProviderController extends Controller {
+
+	public function __construct(
+		string $appName,
+		IRequest $request,
+		private ProviderService $providerService,
+	) {
+		parent::__construct($appName, $request);
+	}
+
+	/**
+	 * @return DataResponse
+	 */
+	#[NoAdminRequired]
+	public function getDefaultProviderKey(): DataResponse {
+		$providerKey = $this->providerService->getDefaultProviderKey();
+		return new DataResponse($providerKey);
+	}
+
+	/**
+	 * @return DataResponse
+	 */
+	#[NoAdminRequired]
+	public function getProviders(): DataResponse {
+		$providers = $this->providerService->getEnrichedProviders();
+		return new DataResponse($providers);
+	}
+}

lib/Listener/AppDisableListener.php

@@ -14,7 +14,7 @@
 namespace OCA\ContextChat\Listener;
 
 use OCA\ContextChat\Service\LangRopeService;
-use OCA\ContextChat\Service\ProviderConfigService;
+use OCA\ContextChat\Service\ProviderService;
 use OCP\App\Events\AppDisableEvent;
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
@@ -25,7 +25,7 @@
  */
 class AppDisableListener implements IEventListener {
 	public function __construct(
-		private ProviderConfigService $configService,
+		private ProviderService $providerService,
 		private LangRopeService $service,
 		private LoggerInterface $logger,
 	) {
@@ -36,7 +36,7 @@ public function handle(Event $event): void {
 			return;
 		}
 
-		foreach ($this->configService->getProviders() as $key => $values) {
+		foreach ($this->providerService->getProviders() as $key => $values) {
 			/** @var string[] */
 			$identifierValues = explode('__', $key, 2);
 
@@ -51,7 +51,7 @@ public function handle(Event $event): void {
 				continue;
 			}
 
-			$this->configService->removeProvider($appId, $providerId);
+			$this->providerService->removeProvider($appId, $providerId);
 			$this->service->deleteSourcesByProviderForAllUsers($providerId);
 		}
 	}

lib/Listener/FileListener.php

@@ -10,6 +10,7 @@
 use OCA\ContextChat\AppInfo\Application;
 use OCA\ContextChat\Db\QueueFile;
 use OCA\ContextChat\Service\LangRopeService;
+use OCA\ContextChat\Service\ProviderService;
 use OCA\ContextChat\Service\QueueService;
 use OCA\ContextChat\Service\StorageService;
 use OCP\DB\Exception;
@@ -120,7 +121,7 @@ public function handle(Event $event): void {
 					if (!$node instanceof File) {
 						continue;
 					}
-					$fileRefs[] = 'file: ' . $node->getId();
+					$fileRefs[] = ProviderService::getSourceId($node->getId());
 				}
 
 				$this->langRopeService->deleteSources($userId, $fileRefs);
@@ -129,7 +130,7 @@ public function handle(Event $event): void {
 					return;
 				}
 
-				$fileRef = 'file: ' . $node->getId();
+				$fileRef = ProviderService::getSourceId($node->getId());
 				foreach ($userIds as $userId) {
 					$this->langRopeService->deleteSources($userId, [$fileRef]);
 				}
@@ -194,7 +195,7 @@ public function postDelete(Node $node, bool $recurse = true): void {
 		}
 
 		foreach ($this->storageService->getUsersForFileId($node->getId()) as $userId) {
-			$fileRef = 'file: ' . $node->getId();
+			$fileRef = ProviderService::getSourceId($node->getId());
 			$this->langRopeService->deleteSources($userId, [$fileRef]);
 		}
 	}