Merge pull request #1864 from betagouv/1843-metabase-dump-fix

1849 metabase dump fix
betagouv · Nov 14, 2023 · 0dd546c · 0dd546c
2 parents 8815c65 + a0101e4
commit 0dd546c
Show file tree

Hide file tree

Showing 8 changed files with 76 additions and 38 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/api/migrations/20231113162302-log-add-user-id.js b/packages/api/migrations/20231113162302-log-add-user-id.js
@@ -0,0 +1,6 @@
+const updateLogUserId = require("../src/modules/stats/migrations/userId.migration");
+module.exports = {
+    async up(db) {
+        await updateLogUserId(db);
+    },
+};
diff --git a/packages/api/package.json b/packages/api/package.json
@@ -93,7 +93,7 @@
         "unzip": "^0.1.11",
         "unzipper": "^0.10.14",
         "winston": "^3.5.1",
-        "winston-mongodb": "^5.0.7",
+        "winston-mongodb": "^5.1.1",
         "xss": "^1.0.14"
     },
     "devDependencies": {

diff --git a/packages/api/src/middlewares/LogMiddleware.ts b/packages/api/src/middlewares/LogMiddleware.ts
@@ -1,8 +1,8 @@
 import winston from "winston";
 import expressWinston from "express-winston";
-import { client } from "../shared/MongoConnection";
-
 import "winston-mongodb";
+import { ObjectId } from "mongodb";
+import { client } from "../shared/MongoConnection";
 
 const LOGGER_SECRET_FIELDS = ["password", "token"];
 
@@ -49,6 +49,11 @@ export const expressLogger = () =>
         requestFilter: (req, propName) => {
             if (propName === "body" && typeof req[propName] === "object" && req[propName])
                 recursiveFilter(req[propName]);
+
+            // @ts-expect-error strange express-winston types
+            // we convert _id into string as a workaround to winston-mongodb bug that serializes them to {}
+            if (propName === "user" && req[propName]?._id) req[propName]._id = req[propName]._id.toString();
+
             return LOGGER_SECRET_FIELDS.includes(propName) ? "**********" : req[propName];
         },
         responseFilter: (req, propName) => {

diff --git a/packages/api/src/modules/dump/dump.service.ts b/packages/api/src/modules/dump/dump.service.ts
@@ -1,7 +1,7 @@
 import { ENV } from "../../configurations/env.conf";
 import configurationsService from "../configurations/configurations.service";
 import statsService from "../stats/stats.service";
-import userRgpdService from "../user/services/rgpd/user.rgpd.service";
+import userCrudService from "../user/services/crud/user.crud.service";
 import metabaseDumpRepo from "./repositories/metabase-dump.repository";
 
 export class DumpService {
@@ -38,7 +38,7 @@ export class DumpService {
 
         if (lastAssociationVisits.length) await metabaseDumpRepo.addVisits(lastAssociationVisits);
 
-        const users = await userRgpdService.findAnonymizedUsers();
+        const users = await userCrudService.find();
         console.log("users: ", users.length);
 
         if (users.length) await metabaseDumpRepo.upsertUsers(users);

diff --git a/packages/api/src/modules/stats/migrations/userId.migration.ts b/packages/api/src/modules/stats/migrations/userId.migration.ts
@@ -0,0 +1,42 @@
+import { Db } from "mongodb";
+
+export default async function updateLogUserId(db: Db) {
+    const collection = db.collection("log");
+    await collection
+        .aggregate([
+            {
+                $lookup: {
+                    from: "users",
+                    let: { logEmail: "$meta.req.user.email" },
+                    pipeline: [
+                        { $match: { $expr: { $eq: ["$$logEmail", "$email"] } } },
+                        { $project: { userId: { $convert: { input: "$_id", to: "string" } }, _id: 0 } },
+                        { $limit: 1 },
+                    ],
+                    as: "stringUserIds",
+                },
+            },
+            { $unwind: { path: "$stringUserIds", preserveNullAndEmptyArrays: true } },
+            {
+                $addFields: {
+                    "meta.req.user._id": "$stringUserIds.userId",
+                },
+            },
+            {
+                $project: {
+                    stringUserIds: 0,
+                },
+            },
+            { $out: "log" },
+        ])
+        .toArray();
+
+    await collection.updateMany(
+        { "meta.req.user.email": { $exists: false } },
+        {
+            $unset: {
+                "$meta.req.user": 1,
+            },
+        },
+    );
+}
diff --git a/packages/api/src/modules/stats/stats.service.ts b/packages/api/src/modules/stats/stats.service.ts
@@ -1,4 +1,4 @@
-import { WithId } from "mongodb";
+import { ObjectId, WithId } from "mongodb";
 import { UserCountByStatus } from "dto";
 import { firstDayOfPeriod, isValidDate, oneYearAfterPeriod } from "../../shared/helpers/DateHelper";
 import { BadRequestError } from "../../shared/errors/httpErrors";
@@ -297,7 +297,8 @@ class StatsService {
             if (log.meta.req?.body?.lastName) delete log.meta.req.body.lastName;
             if (log.meta.req?.body?.phoneNumber) delete log.meta.req.body.phoneNumber;
             if (log.meta.req?.user) {
-                log.meta.req.userId = log.meta.req.user._id?.toString(); // userId is needed for joins with another table
+                // userId is needed for joins with another table, but is saved as a string because of a dependency bug
+                log.meta.req.userId = new ObjectId(log.meta.req.user._id);
                 delete log.meta.req.user;
             }
 

diff --git a/packages/api/src/modules/user/services/rgpd/user.rgpd.service.ts b/packages/api/src/modules/user/services/rgpd/user.rgpd.service.ts
@@ -65,7 +65,6 @@ export class UserRgpdService {
         return users.map(user => {
             return {
                 ...user,
-                _id: user._id.toString(),
                 email: undefined,
                 firstName: undefined,
                 lastName: undefined,