Merge pull request #14 from nextcloud/fix/metadata-search

fix: metadata search for provider

.github/workflows/integration-test.yml

@@ -142,23 +142,23 @@ jobs:
       - name: Prepare docs
         run: |
           cd data/admin/files/documentation
-          find ./ -depth -name "*.rst" -exec sh -c 'mv "$1" "${1%.rst}.txt"' _ {} \;
+          find ./admin_manual/ -depth -name "*.rst" -exec sh -c 'mv "$1" "${1%.rst}.txt"' _ {} \;
           git status
 
       - name: Setup python 3.11
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: context_chat_backend/reqs.txt
 
       - name: Install and init backend
         run: |
           cd context_chat_backend
           pip install --no-deps -r reqs.txt
           cp example.env .env
-          echo "DISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env
           echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
-          curl -L https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf
-          ./main.py &
+          ./main.py &> backend_logs &
 
       - name: Register backend
         run: |
@@ -167,14 +167,16 @@ jobs:
 
       - name: Scan files
         run: |
-          ./occ files:scan --all
+          ./occ files:scan admin
           ./occ context_chat:scan -m text/plain admin
 
       - name: Run prompt
         run: |
           ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
 
-      - name: Show log on failure
+      - name: Show logs
         if: always()
         run: |
           tail data/nextcloud.log
+          echo '--------------------------------------------------'
+          [ -f context_chat_backend/backend_logs ] && cat context_chat_backend/backend_logs || echo "No backend logs"

lib/BackgroundJobs/IndexerJob.php

@@ -125,7 +125,7 @@ protected function index(array $files): void {
 			$userIds = $this->storageService->getUsersForFileId($queueFile->getFileId());
 			foreach ($userIds as $userId) {
 				try {
-					$source = new Source($userId, 'file: ' . $file->getId(), $file->getPath(), $fileHandle, $file->getMtime(), $file->getMimeType());
+					$source = new Source($userId, 'file: ' . $file->getId(), $file->getPath(), $fileHandle, $file->getMtime(), $file->getMimeType(), 'file');
 				} catch (InvalidPathException|NotFoundException $e) {
 					$this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]);
 					continue 2;

lib/BackgroundJobs/SubmitContentJob.php

@@ -58,14 +58,16 @@ protected function run($argument): void {
 
 		foreach ($bucketed as $userId => $entities) {
 			$sources = array_map(function (QueueContentItem $item) use ($userId) {
-				$sourceId = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId()) . ': ' . $item->getItemId();
+				$providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId());
+				$sourceId = $providerKey . ': ' . $item->getItemId();
 				return new Source(
 					$userId,
 					$sourceId,
 					$item->getTitle(),
 					$item->getContent(),
 					$item->getLastModified()->getTimeStamp(),
 					$item->getDocumentType(),
+					$providerKey,
 				);
 			}, $entities);
 

lib/Listener/AppDisableListener.php

@@ -58,7 +58,7 @@ public function handle(Event $event): void {
 			}
 
 			$this->configService->removeProvider($appId, $providerId);
-			$this->service->deleteMatchingSources($this->userId, $key);
+			$this->service->deleteSourcesByProvider($this->userId, $key);
 		}
 	}
 }

lib/Public/ContentManager.php

@@ -112,7 +112,7 @@ public function removeContentForUsers(string $appId, string $providerId, string
 	 */
 	public function removeAllContentForUsers(string $appId, string $providerId, array $users): void {
 		foreach ($users as $userId) {
-			$this->service->deleteMatchingSources($userId, $this->configService->getConfigKey($appId, $providerId));
+			$this->service->deleteSourcesByProvider($userId, $this->configService->getConfigKey($appId, $providerId));
 		}
 	}
 }

lib/Service/LangRopeService.php

@@ -119,16 +119,16 @@ private function requestToExApp(
 
 	/**
 	 * @param string $userId
-	 * @param string $keyword Keyword to search for in the source names
+	 * @param string $providerKey
 	 * @return void
 	 */
-	public function deleteMatchingSources(string $userId, string $keyword): void {
+	public function deleteSourcesByProvider(string $userId, string $providerKey): void {
 		$params = [
 			'userId' => $userId,
-			'keyword' => $keyword,
+			'providerKey' => $providerKey,
 		];
 
-		$this->requestToExApp('/deleteMatchingSources', 'POST', $params);
+		$this->requestToExApp('/deleteSourcesByProvider', 'POST', $params);
 	}
 
 	/**
@@ -161,13 +161,14 @@ public function indexSources(array $sources): void {
 		$params = array_map(function (Source $source) {
 			return [
 				'name' => 'sources',
-				'filename' => $source->reference, // 'file: 555'
+				'filename' => $source->reference, // eg. 'file: 555'
 				'contents' => $source->content,
 				'headers' => [
 					'userId' => $source->userId,
 					'title' => $source->title,
 					'type' => $source->type,
 					'modified' => $source->modified,
+					'provider' => $source->provider, // eg. 'file'
 				],
 			];
 		}, $sources);

lib/Service/ScanService.php

@@ -74,6 +74,7 @@ public function scanDirectory(string $userId, array $mimeTypeFilter, Folder $dir
 					$fileHandle,
 					$node->getMTime(),
 					$node->getMimeType(),
+					'file'
 				);
 				$sources[] = $source;
 				$size += $node_size;

lib/Type/Source.php

@@ -19,6 +19,7 @@ public function __construct(
 		public mixed $content,
 		public int | string $modified,
 		public string $type,
+		public string $provider,
 	) {
 	}
 }