refactor: enhance high-entropy literal detection logic (#2)

Author: VisualBean

packages/core/src/detectors/high-entropy-literal.ts

@@ -26,6 +26,10 @@ const ENTROPY_THRESHOLD = 4.5; // bits/char
 const MAX_SOURCE_BYTES = 2_000_000; // skip files larger than 2 MB
 const MAX_FINDINGS_PER_FILE = 50; // protect the report from minified bundles
 
+const BASE64ISH_RE = /^[A-Za-z0-9+/=_-]+$/;
+const DATA_URI_BASE64_RE = /^data:[^,]{1,120};base64,[A-Za-z0-9+/=]+$/i;
+const ESCAPED_BYTES_RE = /^(?:\\x[0-9A-Fa-f]{2}|\\u[0-9A-Fa-f]{4}|\\[0-7]{3})+$/;
+
 /** Shannon entropy in bits/char. */
 function shannon(s: string): number {
   if (s.length === 0) return 0;
@@ -39,6 +43,17 @@ function shannon(s: string): number {
   return h;
 }
 
+function looksEncodedOrPacked(s: string): boolean {
+  if (DATA_URI_BASE64_RE.test(s)) return true;
+  if (ESCAPED_BYTES_RE.test(s)) return true;
+
+  // Ordinary prose, SQL, and structured log messages often cross the entropy
+  // threshold. Packed payloads are typically dense and whitespace-free.
+  if (/\s/.test(s)) return false;
+
+  return BASE64ISH_RE.test(s);
+}
+
 /** Convert a 0-based character offset to a 1-based line number. */
 function lineAt(source: string, offset: number): number {
   let line = 1;
@@ -70,6 +85,7 @@ export const highEntropyLiteral: Detector = {
 
       const body = match[2];
       if (!body || body.length < MIN_LEN) continue;
+      if (!looksEncodedOrPacked(body)) continue;
 
       const entropy = shannon(body);
       if (entropy < ENTROPY_THRESHOLD) continue;

packages/core/test/unit/layer-a.test.ts

@@ -64,6 +64,23 @@ describe("Layer A: high-entropy-literal", () => {
     expect(result).toNotFlag("obf.high-entropy-literal");
   });
 
+  it("does not flag ordinary C# SQL, exception, or log message strings", async () => {
+    const src = `
+command.CommandText = "SELECT * FROM pgmq.send(@queue_name, @message::jsonb)";
+throw new JsonException($"PGMQ message {reader.GetInt64(0)} from queue '{queueName}' deserialized to null.");
+logger.LogWarning(
+    "Failed to publish queued package event batch of {Count} events. First event: {EventType} for {Package}@{Version} (CorrelationId: {CorrelationId}); dropping batch",
+    count,
+    eventType,
+    packageId,
+    version,
+    correlationId);
+`;
+    const { input, fileResolver } = virtualFiles({ "src/PgmqWorker.cs": src });
+    const result = await scan(input, { fileResolver, ...silentOptions() });
+    expect(result).toNotFlag("obf.high-entropy-literal");
+  });
+
   it("does not flag short strings even if entropy is high", async () => {
     const { input, fileResolver } = virtualFiles({
       "src/short.js": `const k = "Zm9v";\n`, // 4 chars, well below the 64-char floor