feat: remote engine management

core/src/browser/extensions/enginesManagement.ts

@@ -3,6 +3,7 @@
   Engines,
   EngineVariant,
   EngineReleased,
+  EngineConfig,
   DefaultEngineVariant,
 } from '../../types'
 import { BaseExtension, ExtensionTypeEnum } from '../extension'
@@ -14,7 +15,7 @@
  */
 export abstract class EngineManagementExtension extends BaseExtension {
   type(): ExtensionTypeEnum | undefined {
     return ExtensionTypeEnum.Engine
   }
 
   /**
@@ -55,8 +56,16 @@
    * @returns A Promise that resolves to intall of engine.
    */
   abstract installEngine(
-    name: InferenceEngine,
-    engineConfig: { variant: string; version?: string }
+    name: string,
+    engineConfig: EngineConfig
+  ): Promise<{ messages: string }>
+
+  /**
+   * Add a new remote engine
+   * @returns A Promise that resolves to intall of engine.
+   */
+  abstract addRemoteEngine(
+    engineConfig: EngineConfig
   ): Promise<{ messages: string }>
 
   /**
@@ -65,14 +74,16 @@
    */
   abstract uninstallEngine(
     name: InferenceEngine,
-    engineConfig: { variant: string; version: string }
+    engineConfig: EngineConfig
   ): Promise<{ messages: string }>
 
   /**
    * @param name - Inference engine name.
    * @returns A Promise that resolves to an object of default engine.
    */
-  abstract getDefaultEngineVariant(name: InferenceEngine): Promise<DefaultEngineVariant>
+  abstract getDefaultEngineVariant(
+    name: InferenceEngine
+  ): Promise<DefaultEngineVariant>
 
   /**
    * @body variant - string
@@ -81,11 +92,19 @@
    */
   abstract setDefaultEngineVariant(
     name: InferenceEngine,
-    engineConfig: { variant: string; version: string }
+    engineConfig: EngineConfig
   ): Promise<{ messages: string }>
 
   /**
    * @returns A Promise that resolves to update engine.
    */
-  abstract updateEngine(name: InferenceEngine): Promise<{ messages: string }>
+  abstract updateEngine(
+    name: InferenceEngine,
+    engineConfig?: EngineConfig
+  ): Promise<{ messages: string }>
+
+  /**
+   * @returns A Promise that resolves to an object of remote models list .
+   */
+  abstract getRemoteModels(name: InferenceEngine | string): Promise<any>
 }

core/src/node/helper/config.test.ts

@@ -1,28 +1,19 @@
-import { getEngineConfiguration } from './config';
-import { getAppConfigurations, defaultAppConfig } from './config';
-
-import { getJanExtensionsPath } from './config';
-import { getJanDataFolderPath } from './config';
-it('should return undefined for invalid engine ID', async () => {
-  const config = await getEngineConfiguration('invalid_engine');
-  expect(config).toBeUndefined();
-});
+import { getAppConfigurations, defaultAppConfig } from './config'
 
+import { getJanExtensionsPath, getJanDataFolderPath } from './config'
 
 it('should return default config when CI is e2e', () => {
-  process.env.CI = 'e2e';
-  const config = getAppConfigurations();
-  expect(config).toEqual(defaultAppConfig());
-});
-
+  process.env.CI = 'e2e'
+  const config = getAppConfigurations()
+  expect(config).toEqual(defaultAppConfig())
+})
 
 it('should return extensions path when retrieved successfully', () => {
-  const extensionsPath = getJanExtensionsPath();
-  expect(extensionsPath).not.toBeUndefined();
-});
-
+  const extensionsPath = getJanExtensionsPath()
+  expect(extensionsPath).not.toBeUndefined()
+})
 
 it('should return data folder path when retrieved successfully', () => {
-  const dataFolderPath = getJanDataFolderPath();
-  expect(dataFolderPath).not.toBeUndefined();
-});
+  const dataFolderPath = getJanDataFolderPath()
+  expect(dataFolderPath).not.toBeUndefined()
+})

core/src/node/helper/config.ts

@@ -1,8 +1,7 @@
-import { AppConfiguration, SettingComponentProps } from '../../types'
+import { AppConfiguration } from '../../types'
 import { join, resolve } from 'path'
 import fs from 'fs'
 import os from 'os'
-import childProcess from 'child_process'
 const configurationFileName = 'settings.json'
 
 /**
@@ -19,34 +18,44 @@
 
   if (!fs.existsSync(configurationFile)) {
     // create default app config if we don't have one
-    console.debug(`App config not found, creating default config at ${configurationFile}`)
+    console.debug(
+      `App config not found, creating default config at ${configurationFile}`
+    )
     fs.writeFileSync(configurationFile, JSON.stringify(appDefaultConfiguration))
     return appDefaultConfiguration
   }
 
   try {
     const appConfigurations: AppConfiguration = JSON.parse(
       fs.readFileSync(configurationFile, 'utf-8')
     )
     return appConfigurations
   } catch (err) {
-    console.error(`Failed to read app config, return default config instead! Err: ${err}`)
+    console.error(
+      `Failed to read app config, return default config instead! Err: ${err}`
+    )
     return defaultAppConfig()
   }
 }
 
 const getConfigurationFilePath = () =>
   join(
-    global.core?.appPath() || process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'],
+    global.core?.appPath() ||
+      process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'],
     configurationFileName
   )
 
-export const updateAppConfiguration = (configuration: AppConfiguration): Promise<void> => {
+export const updateAppConfiguration = (
+  configuration: AppConfiguration
+): Promise<void> => {
   const configurationFile = getConfigurationFilePath()
-  console.debug('updateAppConfiguration, configurationFile: ', configurationFile)
+  console.debug(
+    'updateAppConfiguration, configurationFile: ',
+    configurationFile
+  )
 
   fs.writeFileSync(configurationFile, JSON.stringify(configuration))
   return Promise.resolve()
 }
 
 /**
@@ -69,86 +78,6 @@
   return join(appConfigurations.data_folder, 'extensions')
 }
 
-/**
- * Utility function to physical cpu count
- *
- * @returns {number} The physical cpu count.
- */
-export const physicalCpuCount = async (): Promise<number> => {
-  const platform = os.platform()
-  try {
-    if (platform === 'linux') {
-      const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l')
-      return parseInt(output.trim(), 10)
-    } else if (platform === 'darwin') {
-      const output = await exec('sysctl -n hw.physicalcpu_max')
-      return parseInt(output.trim(), 10)
-    } else if (platform === 'win32') {
-      const output = await exec('WMIC CPU Get NumberOfCores')
-      return output
-        .split(os.EOL)
-        .map((line: string) => parseInt(line))
-        .filter((value: number) => !isNaN(value))
-        .reduce((sum: number, number: number) => sum + number, 1)
-    } else {
-      const cores = os.cpus().filter((cpu: any, index: number) => {
-        const hasHyperthreading = cpu.model.includes('Intel')
-        const isOdd = index % 2 === 1
-        return !hasHyperthreading || isOdd
-      })
-      return cores.length
-    }
-  } catch (err) {
-    console.warn('Failed to get physical CPU count', err)
-    // Divide by 2 to get rid of hyper threading
-    const coreCount = Math.ceil(os.cpus().length / 2)
-    console.debug('Using node API to get physical CPU count:', coreCount)
-    return coreCount
-  }
-}
-
-const exec = async (command: string): Promise<string> => {
-  return new Promise((resolve, reject) => {
-    childProcess.exec(command, { encoding: 'utf8' }, (error, stdout) => {
-      if (error) {
-        reject(error)
-      } else {
-        resolve(stdout)
-      }
-    })
-  })
-}
-
-// a hacky way to get the api key. we should comes up with a better
-// way to handle this
-export const getEngineConfiguration = async (engineId: string) => {
-  if (engineId !== 'openai' && engineId !== 'groq') return undefined
-
-  const settingDirectoryPath = join(
-    getJanDataFolderPath(),
-    'settings',
-    '@janhq',
-    engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension',
-    'settings.json'
-  )
-
-  const content = fs.readFileSync(settingDirectoryPath, 'utf-8')
-  const settings: SettingComponentProps[] = JSON.parse(content)
-  const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key'
-  const keySetting = settings.find((setting) => setting.key === apiKeyId)
-  let fullUrl = settings.find((setting) => setting.key === 'chat-completions-endpoint')
-    ?.controllerProps.value
-
-  let apiKey = keySetting?.controllerProps.value
-  if (typeof apiKey !== 'string') apiKey = ''
-  if (typeof fullUrl !== 'string') fullUrl = ''
-
-  return {
-    api_key: apiKey,
-    full_url: fullUrl,
-  }
-}
-
 /**
  * Default app configurations
  * App Data Folder default to Electron's userData
@@ -158,7 +87,10 @@
  */
 export const defaultAppConfig = (): AppConfiguration => {
   const { app } = require('electron')
-  const defaultJanDataFolder = join(app?.getPath('userData') ?? os?.homedir() ?? '', 'data')
+  const defaultJanDataFolder = join(
+    app?.getPath('userData') ?? os?.homedir() ?? '',
+    'data'
+  )
   return {
     data_folder:
       process.env.CI === 'e2e'

core/src/node/helper/resource.test.ts

@@ -1,15 +1,9 @@
-import { getSystemResourceInfo } from './resource';
+import { getSystemResourceInfo } from './resource'
 
 it('should return the correct system resource information with a valid CPU count', async () => {
-  const mockCpuCount = 4;
-  jest.spyOn(require('./config'), 'physicalCpuCount').mockResolvedValue(mockCpuCount);
-  const logSpy = jest.spyOn(require('./logger'), 'log').mockImplementation(() => {});
-
-  const result = await getSystemResourceInfo();
+  const result = await getSystemResourceInfo()
 
   expect(result).toEqual({
-    numCpuPhysicalCore: mockCpuCount,
     memAvailable: 0,
-  });
-  expect(logSpy).toHaveBeenCalledWith(`[CORTEX]::CPU information - ${mockCpuCount}`);
-});
+  })
+})

core/src/node/helper/resource.ts

@@ -1,13 +1,7 @@
 import { SystemResourceInfo } from '../../types'
-import { physicalCpuCount } from './config'
-import { log } from './logger'
 
 export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
-  const cpu = await physicalCpuCount()
-  log(`[CORTEX]::CPU information - ${cpu}`)
-
   return {
-    numCpuPhysicalCore: cpu,
     memAvailable: 0, // TODO: this should not be 0
   }
 }

core/src/types/engine/index.ts

@@ -1,7 +1,23 @@
 import { InferenceEngine } from '../../types'
 
 export type Engines = {
-  [key in InferenceEngine]: EngineVariant[]
+  [key in InferenceEngine]: (EngineVariant & EngineConfig)[]
+}
+
+export type EngineMetadata = {
+  get_models_url?: string
+  api_key_template?: string
+  transform_req?: {
+    chat_completions?: {
+      url?: string
+      template?: string
+    }
+  }
+  transform_resp?: {
+    chat_completions?: {
+      template?: string
+    }
+  }
 }
 
 export type EngineVariant = {
@@ -23,6 +39,16 @@ export type EngineReleased = {
   size: number
 }
 
+export type EngineConfig = {
+  engine?: string
+  version?: string
+  variant?: string
+  type?: string
+  url?: string
+  api_key?: string
+  metadata?: EngineMetadata
+}
+
 export enum EngineEvent {
   OnEngineUpdate = 'OnEngineUpdate',
 }

core/src/types/message/messageEntity.ts

@@ -32,9 +32,8 @@ export type ThreadMessage = {
   completed_at: number
   /** The additional metadata of this message. **/
   metadata?: Record<string, unknown>
-
+  /** Type of the message */
   type?: string
-
   /** The error code which explain what error type. Used in conjunction with MessageStatus.Error */
   error_code?: ErrorCode
 }
@@ -72,6 +71,10 @@ export type MessageRequest = {
   // TODO: deprecate threadId field
   thread?: Thread
 
+  /** Engine name to process */
+  engine?: string
+
+  /** Message type */
   type?: string
 }
 
@@ -147,7 +150,9 @@ export interface Attachment {
   /**
    * The tools to add this file to.
    */
-  tools?: Array<CodeInterpreterTool | Attachment.AssistantToolsFileSearchTypeOnly>
+  tools?: Array<
+    CodeInterpreterTool | Attachment.AssistantToolsFileSearchTypeOnly
+  >
 }
 
 export namespace Attachment {
@@ -166,5 +171,10 @@ export interface IncompleteDetails {
   /**
    * The reason the message is incomplete.
    */
-  reason: 'content_filter' | 'max_tokens' | 'run_cancelled' | 'run_expired' | 'run_failed'
+  reason:
+    | 'content_filter'
+    | 'max_tokens'
+    | 'run_cancelled'
+    | 'run_expired'
+    | 'run_failed'
 }

core/src/types/miscellaneous/systemResourceInfo.ts

@@ -1,5 +1,4 @@
 export type SystemResourceInfo = {
-  numCpuPhysicalCore: number
   memAvailable: number
 }