feat(EMS-4016): broker address lookup - data migration

database/exip.sql

@@ -154,7 +154,7 @@ CREATE TABLE `Application` (
 	`dealType` varchar(4) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'EXIP',
   `policyContact` varchar(191) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
   `nominatedLossPayee` varchar(191) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
-  `migratedV2toV3` tinyint(1) DEFAULT NULL,
+  `migratedTo` int DEFAULT NULL,
   PRIMARY KEY (`id`),
   KEY `Application_eligibility_idx` (`eligibility`),
   KEY `Application_referenceNumber_idx` (`referenceNumber`),

src/api/constants/application/versions/index.test.ts

@@ -30,6 +30,27 @@ describe('api/constants/application/versions', () => {
         BROKER_ADDRESS_AS_MULTIPLE_FIELDS: false,
         REQUESTED_CREDIT_LIMIT_REQUIRED: true,
       },
+      {
+        VERSION_NUMBER: '4',
+        OVER_500K_SUPPORT: true,
+        DEFAULT_FINAL_DESTINATION_KNOWN: null,
+        DEFAULT_NEED_PRE_CREDIT_PERIOD_COVER: null,
+        DEFAULT_CURRENCY: GBP,
+        BROKER_ADDRESS_AS_MULTIPLE_FIELDS: false,
+        REQUESTED_CREDIT_LIMIT_REQUIRED: true,
+        DECLARATIONS_MODERN_SLAVERY: true,
+      },
+      {
+        VERSION_NUMBER: '5',
+        OVER_500K_SUPPORT: true,
+        DEFAULT_FINAL_DESTINATION_KNOWN: null,
+        DEFAULT_NEED_PRE_CREDIT_PERIOD_COVER: null,
+        DEFAULT_CURRENCY: GBP,
+        BROKER_ADDRESS_AS_MULTIPLE_FIELDS: false,
+        REQUESTED_CREDIT_LIMIT_REQUIRED: true,
+        DECLARATIONS_MODERN_SLAVERY: true,
+        BROKER_ADDRESS_LOOKUP: true,
+      },
     ];
 
     expect(VERSIONS).toEqual(expected);

src/api/constants/application/versions/index.ts

@@ -26,6 +26,18 @@ const VERSION_3 = {
   REQUESTED_CREDIT_LIMIT_REQUIRED: true,
 } as ApplicationVersion;
 
+const VERSION_4: ApplicationVersion = {
+  ...VERSION_3,
+  VERSION_NUMBER: '4',
+  DECLARATIONS_MODERN_SLAVERY: true,
+};
+
+const VERSION_5: ApplicationVersion = {
+  ...VERSION_4,
+  VERSION_NUMBER: '5',
+  BROKER_ADDRESS_LOOKUP: true,
+};
+
 /**
  * VERSIONS
  * All possible application versions.
@@ -35,9 +47,11 @@ const VERSION_3 = {
  * - Version number 1: MVP, no support for applications over 500k.
  * - Version number 2: Support for applications over 500k.
  * - Version number 3: Design and content iterations. 1x new database field.
- * - Version number 4: Payments integration
+ * - Version number 4: New declaration - "Modern slavery".
+ * - Version number 5: Broker address lookup (Ordnance Survey integration)
+ * - Version number 6: Payments integration
  * @returns {Array<ApplicationVersion>} All application versions
  */
-const VERSIONS = [VERSION_1, VERSION_2, VERSION_3] as Array<ApplicationVersion>;
+const VERSIONS = [VERSION_1, VERSION_2, VERSION_3, VERSION_4, VERSION_5] as Array<ApplicationVersion>;
 
 export default VERSIONS;

src/api/constants/application/versions/latest.ts

@@ -9,8 +9,8 @@ import VERSIONS from '.';
  * - Version number 1: MVP - No support for applications over 500k.
  * - Version number 2: "No PDF" - Support for applications over 500k.
  * - Version number 3: "No PDF" design and content iterations. 1x new database field.
- * - Version number 4: File uploads
- * - Version number 5: Address lookup
+ * - Version number 4: New declaration - "Modern slavery".
+ * - Version number 5: Broker address lookup (Ordnance Survey integration)
  * - Version number 6: Payments integration
  * @returns {String} Latest application version number
  */

src/api/data-migration/execute-sql-query.ts

@@ -28,7 +28,11 @@ const executeSqlQuery = async ({ connection, query, loggingMessage }: ExecuteSql
       throw new Error(`🚨 Invalid connection passed to executeSqlQuery (${loggingMessage})`);
     }
 
-    const response = await connection.query(query);
+    const [response] = await connection.query(query);
+
+    const responseLogContext = ` ${loggingMessage} - `;
+
+    console.info('ℹ️ %s %s', responseLogContext, response.info);
 
     return response;
   } catch (error) {

src/api/data-migration/version-1-to-version-2/update-applications/update-application-migrated.ts

@@ -11,7 +11,7 @@ const { STATUS } = APPLICATION;
  * @returns {Promise<Array<object>>} executeSqlQuery response
  */
 const updateApplicationMigrated = (connection: Connection) => {
-  const loggingMessage = `Updating migratedV1toV2 FIELD to true in the application table`;
+  const loggingMessage = `Updating migratedV1toV2 FIELD to true in the Application table`;
 
   const query = `
     UPDATE Application SET migratedV1toV2 = 1 WHERE status = '${STATUS.IN_PROGRESS}'

src/api/data-migration/version-1-to-version-2/update-applications/update-application-version.ts

@@ -13,7 +13,7 @@ const { STATUS, LATEST_VERSION_NUMBER, VERSIONS } = APPLICATION;
 const updateApplicationVersion = (connection: Connection) => {
   const originalVersionNumber = VERSIONS[0].VERSION_NUMBER;
 
-  const loggingMessage = `Updating IN_PROGRESS FIELD VERSION from ${originalVersionNumber} to ${LATEST_VERSION_NUMBER} in the application table`;
+  const loggingMessage = `Updating IN_PROGRESS FIELD VERSION from ${originalVersionNumber} to ${LATEST_VERSION_NUMBER} in the Application table`;
 
   const query = `
     UPDATE Application SET version = '${LATEST_VERSION_NUMBER}' WHERE status = '${STATUS.IN_PROGRESS}'

src/api/data-migration/version-2-to-version-3/update-applications/update-application-migrated.ts

@@ -11,7 +11,7 @@ const { STATUS } = APPLICATION;
  * @returns {Promise<Array<object>>} executeSqlQuery response
  */
 const updateApplicationMigrated = (connection: Connection) => {
-  const loggingMessage = `Updating migratedV2toV3 FIELD to true in the application table`;
+  const loggingMessage = `Updating migratedV2toV3 FIELD to true in the Application table`;
 
   const query = `
     UPDATE Application SET migratedV2toV3 = 1 WHERE status = '${STATUS.IN_PROGRESS}'

src/api/data-migration/version-2-to-version-3/update-applications/update-application-version.ts

@@ -13,7 +13,7 @@ const { STATUS, LATEST_VERSION_NUMBER, VERSIONS } = APPLICATION;
 const updateApplicationVersion = (connection: Connection) => {
   const originalVersionNumber = VERSIONS[1].VERSION_NUMBER;
 
-  const loggingMessage = `Updating IN_PROGRESS FIELD VERSION from ${originalVersionNumber} to ${LATEST_VERSION_NUMBER} in the application table`;
+  const loggingMessage = `Updating IN_PROGRESS FIELD VERSION from ${originalVersionNumber} to ${LATEST_VERSION_NUMBER} in the Application table`;
 
   const query = `
     UPDATE Application SET version = '${LATEST_VERSION_NUMBER}' WHERE status = '${STATUS.IN_PROGRESS}'

src/api/data-migration/version-4-to-version-5/README.md

@@ -0,0 +1,39 @@
+# EXIP API - data migration - version 4 to version 5 :file_folder:
+
+This directory contains source code for migrating version 4 of EXIP data into the version 5 data structure.
+
+## In version 4, the following has changed
+
+- `Broker` table - 2x new fields - `buildingNumberOrName`, `isBasedInUk`.
+
+## Running Locally :computer:
+
+1. Ensure that your database has the version 4 data structure.
+2. In the API directory, execute `npm run data-migration`.
+
+The migration should successfully do the following:
+
+1. Connect to the database.
+2. Create a new `buildingNumberOrName` field in the broker table.
+3. Create a new `isBasedInUk` field in the broker table.
+4. Update the application version number.
+5. Update in progress application's to have a `migratedTo` value of 5.
+
+## How to ensure that data migration was successful
+
+1. All applications should be aligned with the version 5 data model (listed above)
+2. In the UI, all existing applications with a status of "in progress" can be progressed and successfully submitted.
+
+## What happens to applications that are in progress :microscope:
+
+Due to the nature of GraphQL and KeystoneJS - the version 4 and version 5 data models are essentially "out of sync".
+
+If we try to run the version 5 API, with version 4 data, things will not work.
+
+Similarly, if we migrate the database and the API is running on version 5 - any applications created with the version 4 data model need to be migrated to the new version 5 model - hence why we need migration logic.
+
+This means that if a user has completed e.g all questions in an application, but has not yet submitted - and then we migrate to version 5, the user's application will be migrated from version 4 to version 5. Because this migration is minimal, only one section will be marked as incomplete - the "Broker" section. Because the new "broker" fields are now required as part of version 5.
+
+This is an intentional behaviour, so that a user can continue to complete and submit an application. The alternative to this is to ask a user to start again, which is not recommended.
+
+If you have any specific questions or need further guidance related to this data migration or the API, please feel free to ask.

src/api/data-migration/version-4-to-version-5/index.ts

@@ -0,0 +1,30 @@
+import connectToDatabase from '../connect-to-database';
+import updateApplications from './update-applications';
+
+/**
+ * dataMigration
+ * Update all applications from the V4 data model/structure,
+ * to the V5 data model/structure.
+ * @returns {Function} process.exit()
+ */
+const dataMigration = async () => {
+  try {
+    console.info('🚀 Beginning data migration (v4 to v5)');
+
+    const connection = await connectToDatabase();
+
+    await updateApplications(connection);
+
+    console.info('✅ Applications successfully updated');
+
+    console.info('🎉 Migration complete. Exiting script');
+
+    process.exit();
+  } catch (error) {
+    console.error('🚨 Error with data migration %o', error);
+
+    throw new Error(`🚨 Error with data migration ${error}`);
+  }
+};
+
+dataMigration();

src/api/data-migration/version-4-to-version-5/update-applications/add-broker-fields.ts

@@ -0,0 +1,39 @@
+import { Connection } from 'mysql2/promise';
+import executeSqlQuery from '../../execute-sql-query';
+
+/**
+ * addBrokerFields
+ * Update the broker table to include 2x new fields:
+ * 1) buildingNumberOrName
+ * 2) isBasedInUk
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<object>>} executeSqlQuery response
+ */
+const addBrokerFields = async (connection: Connection) => {
+  const loggingMessage = 'Adding buildingNumberOrName and isBasedInUk fields to the Broker table';
+
+  console.info('✅ %s', loggingMessage);
+
+  try {
+    const promises = await Promise.all([
+      executeSqlQuery({
+        connection,
+        query: `ALTER TABLE Broker ADD buildingNumberOrName varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''`,
+        loggingMessage: 'Adding FIELD buildingNumberOrName to Broker table',
+      }),
+      executeSqlQuery({
+        connection,
+        query: `ALTER TABLE Broker ADD isBasedInUk tinyint(1) DEFAULT NULL`,
+        loggingMessage: 'Adding FIELD isBasedInUk to Broker table',
+      }),
+    ]);
+
+    return promises;
+  } catch (error) {
+    console.error('🚨 Error %s %o', loggingMessage, error);
+
+    throw new Error(`🚨 error ${loggingMessage} ${error}`);
+  }
+};
+
+export default addBrokerFields;

src/api/data-migration/version-4-to-version-5/update-applications/index.ts

@@ -0,0 +1,28 @@
+import { Connection } from 'mysql2/promise';
+import addBrokerFields from './add-broker-fields';
+import updateApplicationVersion from './update-application-version';
+import updateInProgressApplicationsMigratedTo from './update-application-migrated-fields';
+
+/**
+ * updateApplications
+ * Update applications from the v4 data model/structure, to the V5 data model/structure.
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<object>>} executeSqlQuery responses
+ */
+const updateApplications = async (connection: Connection) => {
+  const loggingMessage = 'Updating applications';
+
+  console.info('✅ %s', loggingMessage);
+
+  try {
+    const promises = await Promise.all([addBrokerFields(connection), updateApplicationVersion(connection), updateInProgressApplicationsMigratedTo(connection)]);
+
+    return promises;
+  } catch (error) {
+    console.error('🚨 Error %s %o', loggingMessage, error);
+
+    throw new Error(`🚨 error ${loggingMessage} ${error}`);
+  }
+};
+
+export default updateApplications;

src/api/data-migration/version-4-to-version-5/update-applications/update-application-migrated-fields.ts

@@ -0,0 +1,21 @@
+import { Connection } from 'mysql2/promise';
+import { APPLICATION } from '../../../constants';
+import executeSqlQuery from '../../execute-sql-query';
+
+const { LATEST_VERSION_NUMBER, STATUS } = APPLICATION;
+
+/**
+ * updateInProgressApplicationsMigratedTo
+ * Update IN_PROGRESS applications migratedTo fields to have a value of 4.
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<object>} executeSqlQuery response
+ */
+const updateInProgressApplicationsMigratedTo = (connection: Connection) => {
+  const loggingMessage = `Updating in progress application's migratedTo FIELD to 5 in the Application table`;
+
+  const query = `UPDATE Application SET migratedTo=${LATEST_VERSION_NUMBER} WHERE status = '${STATUS.IN_PROGRESS}'`;
+
+  return executeSqlQuery({ connection, query, loggingMessage });
+};
+
+export default updateInProgressApplicationsMigratedTo;

src/api/data-migration/version-4-to-version-5/update-applications/update-application-version.ts

@@ -0,0 +1,23 @@
+import { Connection } from 'mysql2/promise';
+import { APPLICATION } from '../../../constants';
+import executeSqlQuery from '../../execute-sql-query';
+
+const { STATUS, LATEST_VERSION_NUMBER, VERSIONS } = APPLICATION;
+
+/**
+ * updateApplicationVersion
+ * Update IN_PROGRESS applications from version 4 to version 5
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<object>>} executeSqlQuery response
+ */
+const updateApplicationVersion = (connection: Connection) => {
+  const originalVersionNumber = VERSIONS[2].VERSION_NUMBER;
+
+  const loggingMessage = `Updating IN_PROGRESS FIELD VERSION from ${originalVersionNumber} to ${LATEST_VERSION_NUMBER} in the Application table`;
+
+  const query = `UPDATE Application SET version = '${LATEST_VERSION_NUMBER}' WHERE status = '${STATUS.IN_PROGRESS}'`;
+
+  return executeSqlQuery({ connection, query, loggingMessage });
+};
+
+export default updateApplicationVersion;