Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: metadata search for provider #14

Merged
merged 2 commits into from
Feb 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -142,23 +142,23 @@ jobs:
- name: Prepare docs
run: |
cd data/admin/files/documentation
find ./ -depth -name "*.rst" -exec sh -c 'mv "$1" "${1%.rst}.txt"' _ {} \;
find ./admin_manual/ -depth -name "*.rst" -exec sh -c 'mv "$1" "${1%.rst}.txt"' _ {} \;
git status
- name: Setup python 3.11
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
cache-dependency-path: context_chat_backend/reqs.txt

- name: Install and init backend
run: |
cd context_chat_backend
pip install --no-deps -r reqs.txt
cp example.env .env
echo "DISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env
echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
curl -L https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf
./main.py &
./main.py &> backend_logs &
- name: Register backend
run: |
Expand All @@ -167,14 +167,16 @@ jobs:
- name: Scan files
run: |
./occ files:scan --all
./occ files:scan admin
./occ context_chat:scan -m text/plain admin
- name: Run prompt
run: |
./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
- name: Show log on failure
- name: Show logs
if: always()
run: |
tail data/nextcloud.log
echo '--------------------------------------------------'
[ -f context_chat_backend/backend_logs ] && cat context_chat_backend/backend_logs || echo "No backend logs"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it actually failed there once, soo 😅

2 changes: 1 addition & 1 deletion lib/BackgroundJobs/IndexerJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ protected function index(array $files): void {
$userIds = $this->storageService->getUsersForFileId($queueFile->getFileId());
foreach ($userIds as $userId) {
try {
$source = new Source($userId, 'file: ' . $file->getId(), $file->getPath(), $fileHandle, $file->getMtime(), $file->getMimeType());
$source = new Source($userId, 'file: ' . $file->getId(), $file->getPath(), $fileHandle, $file->getMtime(), $file->getMimeType(), 'file');
} catch (InvalidPathException|NotFoundException $e) {
$this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]);
continue 2;
Expand Down
4 changes: 3 additions & 1 deletion lib/BackgroundJobs/SubmitContentJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,16 @@ protected function run($argument): void {

foreach ($bucketed as $userId => $entities) {
$sources = array_map(function (QueueContentItem $item) use ($userId) {
$sourceId = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId()) . ': ' . $item->getItemId();
$providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId());
$sourceId = $providerKey . ': ' . $item->getItemId();
return new Source(
$userId,
$sourceId,
$item->getTitle(),
$item->getContent(),
$item->getLastModified()->getTimeStamp(),
$item->getDocumentType(),
$providerKey,
);
}, $entities);

Expand Down
2 changes: 1 addition & 1 deletion lib/Listener/AppDisableListener.php
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public function handle(Event $event): void {
}

$this->configService->removeProvider($appId, $providerId);
$this->service->deleteMatchingSources($this->userId, $key);
$this->service->deleteSourcesByProvider($this->userId, $key);
}
}
}
2 changes: 1 addition & 1 deletion lib/Public/ContentManager.php
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public function removeContentForUsers(string $appId, string $providerId, string
*/
public function removeAllContentForUsers(string $appId, string $providerId, array $users): void {
foreach ($users as $userId) {
$this->service->deleteMatchingSources($userId, $this->configService->getConfigKey($appId, $providerId));
$this->service->deleteSourcesByProvider($userId, $this->configService->getConfigKey($appId, $providerId));
}
}
}
11 changes: 6 additions & 5 deletions lib/Service/LangRopeService.php
Original file line number Diff line number Diff line change
Expand Up @@ -119,16 +119,16 @@ private function requestToExApp(

/**
* @param string $userId
* @param string $keyword Keyword to search for in the source names
* @param string $providerKey
* @return void
*/
public function deleteMatchingSources(string $userId, string $keyword): void {
public function deleteSourcesByProvider(string $userId, string $providerKey): void {
$params = [
'userId' => $userId,
'keyword' => $keyword,
'providerKey' => $providerKey,
];

$this->requestToExApp('/deleteMatchingSources', 'POST', $params);
$this->requestToExApp('/deleteSourcesByProvider', 'POST', $params);
}

/**
Expand Down Expand Up @@ -161,13 +161,14 @@ public function indexSources(array $sources): void {
$params = array_map(function (Source $source) {
return [
'name' => 'sources',
'filename' => $source->reference, // 'file: 555'
'filename' => $source->reference, // eg. 'file: 555'
'contents' => $source->content,
'headers' => [
'userId' => $source->userId,
'title' => $source->title,
'type' => $source->type,
'modified' => $source->modified,
'provider' => $source->provider, // eg. 'file'
],
];
}, $sources);
Expand Down
1 change: 1 addition & 0 deletions lib/Service/ScanService.php
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public function scanDirectory(string $userId, array $mimeTypeFilter, Folder $dir
$fileHandle,
$node->getMTime(),
$node->getMimeType(),
'file'
);
$sources[] = $source;
$size += $node_size;
Expand Down
1 change: 1 addition & 0 deletions lib/Type/Source.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public function __construct(
public mixed $content,
public int | string $modified,
public string $type,
public string $provider,
) {
}
}
Loading