feat(mrf): V4 response migration — PR 2/5 BE V4-native + V3 wire shim

apps/backend/src/app/modules/core/core.errors.ts

@@ -54,6 +54,7 @@ export enum ErrorCodes {
   SUBMISSION_PROCESSING = 100205,
   SUBMISSION_VALIDATE_FIELD = 100206,
   SUBMISSION_VALIDATE_FIELD_V3 = 100702,
+  SUBMISSION_VALIDATE_FIELD_V4 = 100703,
   SUBMISSION_SEND_EMAIL_CONFIRMATION = 100207,
   SUBMISSION_WRONG_RESPONSE_MODE = 100208,
   SUBMISSION_ATTACHMENT_TOO_LARGE = 100209,

apps/backend/src/app/modules/spcp/spcp.util.ts

@@ -1,3 +1,4 @@
+import type { FieldResponsesV4 } from '@opengovsg/formsg-sdk'
 import { BasicField, FieldResponsesV3, FormAuthType } from 'formsg-shared/types'
 import { hasProp } from 'formsg-shared/utils/has-prop'
 import {
@@ -143,6 +144,43 @@ export const createNdiResponsesV3FromRecord = (
   return responses
 }
 
+export const createNdiResponsesV4FromRecord = (
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  ndiResponses: Record<string, any>,
+): FieldResponsesV4 => {
+  const responses: FieldResponsesV4 = {}
+  const provenance = {}
+
+  Object.entries(ndiResponses).forEach(([key, value]) => {
+    const title = mapVerifiedKeyToSPCPTitle(key)
+
+    if (key.startsWith(VerifiedKeys.SpUinFin)) {
+      responses[title] = {
+        fieldType: BasicField.Nric,
+        answer: { value: value as string },
+        question: title,
+        provenance,
+      }
+    } else if (key.startsWith(VerifiedKeys.CpUen)) {
+      responses[title] = {
+        fieldType: BasicField.ShortText,
+        answer: { value: value as string },
+        question: title,
+        provenance,
+      }
+    } else if (key.startsWith(VerifiedKeys.CpUid)) {
+      responses[title] = {
+        fieldType: BasicField.Nric,
+        answer: { value: value as string },
+        question: title,
+        provenance,
+      }
+    }
+  })
+
+  return responses
+}
+
 export const isSPCPFieldTitle = (key: string): key is SPCPFieldTitle =>
   Object.values(SPCPFieldTitle).includes(key as SPCPFieldTitle)
 

apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.middleware.spec.ts

@@ -694,12 +694,14 @@ describe('Multirespondent Submission Middleware', () => {
         responses: {},
       })
 
-      jest.mocked(SpcpUtil.createNdiResponsesV3FromRecord).mockReturnValue({
+      jest.mocked(SpcpUtil.createNdiResponsesV4FromRecord).mockReturnValue({
         'SingPass Validated NRIC': {
           fieldType: BasicField.Nric,
-          answer: 'S9812379B',
+          answer: { value: 'S9812379B' },
+          question: 'SingPass Validated NRIC',
+          provenance: {},
         },
-      })
+      } as any)
 
       const mockNext = jest.fn()
 
@@ -722,7 +724,7 @@ describe('Multirespondent Submission Middleware', () => {
 
       // Assert
       expect(
-        jest.mocked(SpcpUtil.createNdiResponsesV3FromRecord),
+        jest.mocked(SpcpUtil.createNdiResponsesV4FromRecord),
       ).toHaveBeenCalled()
       expect(mockReq.formsg.encryptedPayload.responses).toHaveProperty(
         'SingPass Validated NRIC',
@@ -774,12 +776,14 @@ describe('Multirespondent Submission Middleware', () => {
         },
       })
 
-      jest.mocked(SpcpUtil.createNdiResponsesV3FromRecord).mockReturnValue({
+      jest.mocked(SpcpUtil.createNdiResponsesV4FromRecord).mockReturnValue({
         'SingPass Validated NRIC': {
           fieldType: BasicField.Nric,
-          answer: 'S1234567A',
+          answer: { value: 'S1234567A' },
+          question: 'SingPass Validated NRIC',
+          provenance: {},
         },
-      })
+      } as any)
 
       jest
         .mocked(VerifiedContentService.getVerifiedContent)
@@ -845,7 +849,7 @@ describe('Multirespondent Submission Middleware', () => {
   describe('encryptSubmission', () => {
     const MOCK_FORM_ID = new ObjectId().toHexString()
 
-    const MOCK_FORM = {
+    const MOCK_FORM_BASE = {
       _id: MOCK_FORM_ID,
       publicKey: 'mockPublicKey',
       form_fields: [
@@ -854,10 +858,15 @@ describe('Multirespondent Submission Middleware', () => {
     } as any
 
     const MOCK_RESPONSES = {
-      field1: { fieldType: BasicField.ShortText, answer: 'hello' },
+      field1: {
+        fieldType: BasicField.ShortText,
+        answer: 'hello',
+        question: 'Field 1',
+        provenance: {},
+      },
     }
 
-    const createMockEncryptReq = (isV4FlagOn: boolean) =>
+    const createMockEncryptReq = (hasWebhook: boolean) =>
       ({
         params: { formId: MOCK_FORM_ID },
         body: {
@@ -867,10 +876,15 @@ describe('Multirespondent Submission Middleware', () => {
           responseMetadata: {},
         },
         formsg: {
-          formDef: MOCK_FORM,
+          formDef: hasWebhook
+            ? {
+                ...MOCK_FORM_BASE,
+                webhook: { url: 'https://example.com/webhook' },
+              }
+            : MOCK_FORM_BASE,
         },
         growthbook: {
-          isOn: jest.fn().mockReturnValue(isV4FlagOn),
+          isOn: jest.fn().mockReturnValue(false),
           setAttributes: jest.fn().mockResolvedValue(undefined),
           getAttributes: jest.fn().mockReturnValue({}),
         },
@@ -905,21 +919,9 @@ describe('Multirespondent Submission Middleware', () => {
       mockDecrypt.mockReturnValue({ responses: MOCK_RESPONSES })
     })
 
-    it('should encrypt responses as V3 and set mrfVersion to 1 when feature flag is off', async () => {
-      const mockReq = createMockEncryptReq(false)
-      const mockNext = jest.fn()
-      const mockRes = createMockRes()
-
-      await encryptSubmission(mockReq, mockRes as any, mockNext)
-
-      expect(jest.mocked(adaptV3ToV4)).not.toHaveBeenCalled()
-      expect(mockReq.formsg.encryptedPayload.mrfVersion).toBe(1)
-      expect(mockNext).toHaveBeenCalled()
-    })
-
-    it('should adapt V3 responses to V4 format and set mrfVersion to 2 when feature flag is on', async () => {
-      jest.mocked(adaptV3ToV4).mockReturnValue({
-        field1: { answer: 'hello', question: 'Field 1', provenance: {} },
+    it('should encrypt responses as V3 and set mrfVersion to 1 when form has a webhook url', async () => {
+      jest.mocked(adaptV4ToV3).mockReturnValue({
+        field1: { fieldType: BasicField.ShortText, answer: 'hello' },
       } as any)
 
       const mockReq = createMockEncryptReq(true)
@@ -928,11 +930,8 @@ describe('Multirespondent Submission Middleware', () => {
 
       await encryptSubmission(mockReq, mockRes as any, mockNext)
 
-      expect(jest.mocked(adaptV3ToV4)).toHaveBeenCalledWith(
-        { field1: { fieldType: BasicField.ShortText, answer: 'hello' } },
-        { formFields: { field1: { question: 'Field 1' } }, provenance: {} },
-      )
-      expect(mockReq.formsg.encryptedPayload.mrfVersion).toBe(2)
+      expect(jest.mocked(adaptV4ToV3)).toHaveBeenCalledWith(MOCK_RESPONSES)
+      expect(mockReq.formsg.encryptedPayload.mrfVersion).toBe(1)
       expect(mockNext).toHaveBeenCalled()
     })
 
@@ -980,13 +979,13 @@ describe('Multirespondent Submission Middleware', () => {
       { step: 1, edit: [EDITABLE_FIELD_ID] }, // only field1 editable at step 1
     ]
 
-    // mrfVersion: 2 means responses were encrypted in V4 format
+    // mrfVersion: 1 means previous submission was encrypted in V3 format
     // workflowStep: 0 means the current incoming submission is at step 1
-    const MOCK_MRF_SUBMISSION_V2 = {
+    const MOCK_MRF_SUBMISSION_V1 = {
       form: MOCK_FORM_ID,
-      encryptedContent: 'v4-encrypted-content',
+      encryptedContent: 'v3-encrypted-content',
       version: 1,
-      mrfVersion: 2,
+      mrfVersion: 1,
       form_fields: SNAPSHOT_FORM_FIELDS,
       form_logics: [],
       workflow: SNAPSHOT_WORKFLOW,
@@ -999,29 +998,31 @@ describe('Multirespondent Submission Middleware', () => {
       getWebhookView: jest.fn(),
     } as any
 
-    // V4 decrypted responses (each entry has provenance)
-    const MOCK_V4_DECRYPTED_RESPONSES = {
+    // V3 decrypted responses returned by decryptFromSubmissionKey for a V3-encrypted previous submission
+    const MOCK_V3_DECRYPTED_RESPONSES = {
       [EDITABLE_FIELD_ID]: {
+        fieldType: BasicField.ShortText,
         answer: 'original',
-        question: 'Editable Field',
-        provenance: {},
       },
       [NON_EDITABLE_FIELD_ID]: {
+        fieldType: BasicField.ShortText,
         answer: 'locked-value',
-        question: 'Non-editable Field',
-        provenance: {},
       },
     }
 
-    // V3 responses produced by adaptV4ToV3
-    const MOCK_V3_CONVERTED_RESPONSES = {
+    // V4 responses produced by adaptV3ToV4 (V4 shape with provenance)
+    const MOCK_V4_ADAPTED_RESPONSES = {
       [EDITABLE_FIELD_ID]: {
         fieldType: BasicField.ShortText,
-        answer: 'original',
+        answer: { value: 'original' },
+        question: 'Editable Field',
+        provenance: {},
       },
       [NON_EDITABLE_FIELD_ID]: {
         fieldType: BasicField.ShortText,
-        answer: 'locked-value',
+        answer: { value: 'locked-value' },
+        question: 'Non-editable Field',
+        provenance: {},
       },
     }
 
@@ -1036,16 +1037,21 @@ describe('Multirespondent Submission Middleware', () => {
       ;(
         formsgSdk.cryptoV3.decryptFromSubmissionKey as jest.Mock
       ).mockReturnValue({
-        responses: MOCK_V4_DECRYPTED_RESPONSES,
+        responses: MOCK_V3_DECRYPTED_RESPONSES,
         verified: {},
         submissionSecretKey: '',
       })
 
-      jest.mocked(isFieldResponsesV4).mockReturnValue(true)
+      // Previous decrypted responses are V3-shaped, so isFieldResponsesV4 must return false
+      // to trigger the V3->V4 adaptation path
+      jest.mocked(isFieldResponsesV4).mockReturnValue(false)
 
+      jest.mocked(adaptV3ToV4).mockReturnValue(MOCK_V4_ADAPTED_RESPONSES as any)
+
+      // adaptV4ToV3 is still called once on req.body.responses for logic evaluation
       jest
         .mocked(adaptV4ToV3)
-        .mockReturnValue(MOCK_V3_CONVERTED_RESPONSES as any)
+        .mockReturnValue(MOCK_V3_DECRYPTED_RESPONSES as any)
 
       jest
         .mocked(LogicAdaptor.getVisibleFieldIdsV3)
@@ -1057,22 +1063,26 @@ describe('Multirespondent Submission Middleware', () => {
 
       jest
         .mocked(MrfUtils.validateMrfFieldResponses)
-        .mockReturnValue(ok(MOCK_V3_CONVERTED_RESPONSES) as any)
+        .mockReturnValue(ok(MOCK_V4_ADAPTED_RESPONSES) as any)
     })
 
-    it('should call adaptV4ToV3 and call next when previous mrfVersion is 2 and non-editable fields match', async () => {
+    it('should call adaptV3ToV4 and call next when previous mrfVersion is 1 and non-editable fields match', async () => {
       const mockReq = createMockReq({
         formId: MOCK_FORM_ID,
         submissionId: MOCK_SUBMISSION_ID,
       })
       mockReq.body.responses = {
         [EDITABLE_FIELD_ID]: {
           fieldType: BasicField.ShortText,
-          answer: 'updated',
+          answer: { value: 'updated' },
+          question: 'Editable Field',
+          provenance: {},
         },
         [NON_EDITABLE_FIELD_ID]: {
           fieldType: BasicField.ShortText,
-          answer: 'locked-value',
+          answer: { value: 'locked-value' },
+          question: 'Non-editable Field',
+          provenance: {},
         },
       }
       mockReq.body.submissionSecretKey = 'submission-secret-key'
@@ -1083,44 +1093,38 @@ describe('Multirespondent Submission Middleware', () => {
           form_logics: [],
           workflow: SNAPSHOT_WORKFLOW,
         },
-        mrfSubmission: MOCK_MRF_SUBMISSION_V2,
+        mrfSubmission: MOCK_MRF_SUBMISSION_V1,
       }
 
       const mockNext = jest.fn()
       const mockRes = createMockRes()
 
       await validateMultirespondentSubmission(mockReq, mockRes as any, mockNext)
 
-      expect(jest.mocked(adaptV4ToV3)).toHaveBeenCalledWith(
-        MOCK_V4_DECRYPTED_RESPONSES,
+      expect(jest.mocked(adaptV3ToV4)).toHaveBeenCalledWith(
+        MOCK_V3_DECRYPTED_RESPONSES,
+        { formFields: {}, provenance: {} },
       )
       expect(mockNext).toHaveBeenCalled()
     })
 
-    it('should reject submission when a non-editable field is tampered after V4-to-V3 conversion', async () => {
-      jest.mocked(adaptV4ToV3).mockReturnValue({
-        [EDITABLE_FIELD_ID]: {
-          fieldType: BasicField.ShortText,
-          answer: 'original',
-        },
-        [NON_EDITABLE_FIELD_ID]: {
-          fieldType: BasicField.ShortText,
-          answer: 'locked-value',
-        },
-      } as any)
-
+    it('should reject submission when a non-editable field is tampered after V3-to-V4 conversion', async () => {
       const mockReq = createMockReq({
         formId: MOCK_FORM_ID,
         submissionId: MOCK_SUBMISSION_ID,
       })
       mockReq.body.responses = {
         [EDITABLE_FIELD_ID]: {
           fieldType: BasicField.ShortText,
-          answer: 'updated',
+          answer: { value: 'updated' },
+          question: 'Editable Field',
+          provenance: {},
         },
         [NON_EDITABLE_FIELD_ID]: {
           fieldType: BasicField.ShortText,
-          answer: 'tampered', // differs from 'locked-value'
+          answer: { value: 'tampered' }, // differs from 'locked-value'
+          question: 'Non-editable Field',
+          provenance: {},
         },
       }
       mockReq.body.submissionSecretKey = 'submission-secret-key'
@@ -1131,16 +1135,17 @@ describe('Multirespondent Submission Middleware', () => {
           form_logics: [],
           workflow: SNAPSHOT_WORKFLOW,
         },
-        mrfSubmission: MOCK_MRF_SUBMISSION_V2,
+        mrfSubmission: MOCK_MRF_SUBMISSION_V1,
       }
 
       const mockNext = jest.fn()
       const mockRes = createMockRes()
 
       await validateMultirespondentSubmission(mockReq, mockRes as any, mockNext)
 
-      expect(jest.mocked(adaptV4ToV3)).toHaveBeenCalledWith(
-        MOCK_V4_DECRYPTED_RESPONSES,
+      expect(jest.mocked(adaptV3ToV4)).toHaveBeenCalledWith(
+        MOCK_V3_DECRYPTED_RESPONSES,
+        { formFields: {}, provenance: {} },
       )
       expect(mockNext).not.toHaveBeenCalled()
       expect(mockRes.status).toHaveBeenCalledWith(400)