Add UTF-8-safe base64 encoding for code blocks

packages/engine/src/lib/code-block/code-builder.ts

@@ -15,6 +15,7 @@ import fs, { rm } from 'node:fs/promises';
 import path from 'node:path';
 import { rollup } from 'rollup';
 import { ExecutionMode } from '../core/code/execution-mode';
+import { BASE64_POLYFILL_TYPES } from '../core/code/polyfills/base64-polyfill';
 import { CodeArtifact } from './code-artifact';
 
 const executionMode = system.get<ExecutionMode>(
@@ -140,6 +141,12 @@ const compileCode = async ({
     encoding: 'utf8',
     flag: 'w',
   });
+  // Declare the polyfilled globals (btoaUtf8/atobUtf8) so tsc can type-check
+  // code blocks that use them; the runtime definitions live in BASE64_POLYFILL.
+  await fs.writeFile(`${path}/openops-globals.d.ts`, BASE64_POLYFILL_TYPES, {
+    encoding: 'utf8',
+    flag: 'w',
+  });
   await fs.writeFile(`${path}/index.ts`, code, { encoding: 'utf8', flag: 'w' });
 
   await packageManager.exec({

packages/engine/src/lib/core/code/polyfills/base64-polyfill.ts

@@ -39,6 +39,12 @@ export const BASE64_POLYFILL = `(() => {
     };
   }
 
+  if (typeof globalThis.btoaUtf8 !== 'function') {
+    globalThis.btoaUtf8 = (str) => {
+      return btoa(encodeURIComponent(str).replace(/%([0-9A-F]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16))));
+    }
+  }
+
   if (typeof globalThis.atob !== 'function') {
     globalThis.atob = (input) => {
       let string = String(input).replace(/[\\t\\n\\f\\r ]+/g, '');
@@ -89,4 +95,21 @@ export const BASE64_POLYFILL = `(() => {
       return output;
     };
   }
+
+  if (typeof globalThis.atobUtf8 !== 'function') {
+    globalThis.atobUtf8 = (str) => {
+      return decodeURIComponent(Array.prototype.map.call(atob(str), (c) => '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2)).join(''));
+    }
+  }
 })();`;
+
+/**
+ * Ambient declarations for the custom globals the polyfill injects at runtime.
+ * `btoa`/`atob` are already typed via the `dom` lib, but `btoaUtf8`/`atobUtf8`
+ * are OpenOps-specific, so without this declaration `tsc` rejects code blocks
+ * that use them with `TS2304: Cannot find name 'btoaUtf8'`. This is written into
+ * each code block's compile directory by the code builder so the two stay in sync.
+ */
+export const BASE64_POLYFILL_TYPES = `declare function btoaUtf8(data: string): string;
+declare function atobUtf8(data: string): string;
+`;

packages/engine/test/core/code/v8-isolate-code-sandbox.test.ts

@@ -1,37 +1,66 @@
 import { v8IsolateCodeSandbox } from '../../../src/lib/core/code/v8-isolate-code-sandbox'
 
 const runScript = (script: string): Promise<unknown> =>
-    v8IsolateCodeSandbox.runScript({ script, scriptContext: {} })
+  v8IsolateCodeSandbox.runScript({ script, scriptContext: {} })
 
 describe('v8IsolateCodeSandbox base64 polyfill', () => {
-    it('exposes btoa that matches Latin1 base64 encoding', async () => {
-        const result = await runScript("btoa('hello world')")
-        expect(result).toBe(Buffer.from('hello world', 'latin1').toString('base64'))
-    })
-
-    it('exposes atob that decodes base64 back to the original string', async () => {
-        const encoded = Buffer.from('hello world', 'latin1').toString('base64')
-        const result = await runScript(`atob('${encoded}')`)
-        expect(result).toBe('hello world')
-    })
-
-    it('round-trips a value through btoa and atob', async () => {
-        const result = await runScript("atob(btoa('ServiceNow ticket #42'))")
-        expect(result).toBe('ServiceNow ticket #42')
-    })
-
-    it('decodes a base64-encoded ServiceNow batch response body', async () => {
-        const body = Buffer.from(
-            JSON.stringify({ result: { sys_id: 'abc123def456' } }),
-            'latin1',
-        ).toString('base64')
-        const result = await runScript(
-            `JSON.parse(atob('${body}')).result.sys_id`,
-        )
-        expect(result).toBe('abc123def456')
-    })
-
-    it('throws when btoa receives characters outside the Latin1 range', async () => {
-        await expect(runScript("btoa('日本語')")).rejects.toThrow(/Latin1 range/)
-    })
+  it('exposes btoa that matches Latin1 base64 encoding', async () => {
+    const result = await runScript("btoa('hello world')")
+    expect(result).toBe(Buffer.from('hello world', 'latin1').toString('base64'))
+  })
+
+  it('exposes atob that decodes base64 back to the original string', async () => {
+    const encoded = Buffer.from('hello world', 'latin1').toString('base64')
+    const result = await runScript(`atob('${encoded}')`)
+    expect(result).toBe('hello world')
+  })
+
+  it('round-trips a value through btoa and atob', async () => {
+    const result = await runScript("atob(btoa('ServiceNow ticket #42'))")
+    expect(result).toBe('ServiceNow ticket #42')
+  })
+
+  it('decodes a base64-encoded ServiceNow batch response body', async () => {
+    const body = Buffer.from(
+      JSON.stringify({ result: { sys_id: 'abc123def456' } }),
+      'latin1',
+    ).toString('base64')
+    const result = await runScript(
+      `JSON.parse(atob('${body}')).result.sys_id`,
+    )
+    expect(result).toBe('abc123def456')
+  })
+
+  it('throws when btoa receives characters outside the Latin1 range', async () => {
+    await expect(runScript("btoa('日本語')")).rejects.toThrow(/Latin1 range/)
+  })
+
+  it('exposes btoaUtf8 that encodes characters outside the Latin1 range', async () => {
+    const result = await runScript("btoaUtf8('日本語')")
+    expect(result).toBe(Buffer.from('日本語', 'utf8').toString('base64'))
+  })
+
+  it('exposes atobUtf8 that decodes utf8 base64 back to the original string', async () => {
+    const encoded = Buffer.from('日本語', 'utf8').toString('base64')
+    const result = await runScript(`atobUtf8('${encoded}')`)
+    expect(result).toBe('日本語')
+  })
+
+  it('round-trips a unicode value through btoaUtf8 and atobUtf8', async () => {
+    const result = await runScript(
+      "atobUtf8(btoaUtf8('Coût € — 日本語 🚀'))",
+    )
+    expect(result).toBe('Coût € — 日本語 🚀')
+  })
+
+  it('decodes a utf8 base64-encoded ServiceNow batch response body', async () => {
+    const body = Buffer.from(
+      JSON.stringify({ result: { sys_id: 'Coût-中文-🚀' } }),
+      'utf8',
+    ).toString('base64')
+    const result = await runScript(
+      `JSON.parse(atobUtf8('${body}')).result.sys_id`,
+    )
+    expect(result).toBe('Coût-中文-🚀')
+  })
 })