Skip to content

Commit 44cd08d

Browse files
committed
refactor: enhance high-entropy literal detection logic
Introduces checks for base64ish strings within looksEncodedOrPacked, making the detector more robust against encoded payloads. Adds a new test case to verify that ordinary C# SQL, exception, and log message strings are correctly ignored by the high-entropy literal detector.
1 parent 30f573f commit 44cd08d

2 files changed

Lines changed: 33 additions & 0 deletions

File tree

packages/core/src/detectors/high-entropy-literal.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ const ENTROPY_THRESHOLD = 4.5; // bits/char
2626
const MAX_SOURCE_BYTES = 2_000_000; // skip files larger than 2 MB
2727
const MAX_FINDINGS_PER_FILE = 50; // protect the report from minified bundles
2828

29+
const BASE64ISH_RE = /^[A-Za-z0-9+/=_-]+$/;
30+
const DATA_URI_BASE64_RE = /^data:[^,]{1,120};base64,[A-Za-z0-9+/=]+$/i;
31+
const ESCAPED_BYTES_RE = /^(?:\\x[0-9A-Fa-f]{2}|\\u[0-9A-Fa-f]{4}|\\[0-7]{3})+$/;
32+
2933
/** Shannon entropy in bits/char. */
3034
function shannon(s: string): number {
3135
if (s.length === 0) return 0;
@@ -39,6 +43,17 @@ function shannon(s: string): number {
3943
return h;
4044
}
4145

46+
function looksEncodedOrPacked(s: string): boolean {
47+
if (DATA_URI_BASE64_RE.test(s)) return true;
48+
if (ESCAPED_BYTES_RE.test(s)) return true;
49+
50+
// Ordinary prose, SQL, and structured log messages often cross the entropy
51+
// threshold. Packed payloads are typically dense and whitespace-free.
52+
if (/\s/.test(s)) return false;
53+
54+
return BASE64ISH_RE.test(s);
55+
}
56+
4257
/** Convert a 0-based character offset to a 1-based line number. */
4358
function lineAt(source: string, offset: number): number {
4459
let line = 1;
@@ -70,6 +85,7 @@ export const highEntropyLiteral: Detector = {
7085

7186
const body = match[2];
7287
if (!body || body.length < MIN_LEN) continue;
88+
if (!looksEncodedOrPacked(body)) continue;
7389

7490
const entropy = shannon(body);
7591
if (entropy < ENTROPY_THRESHOLD) continue;

packages/core/test/unit/layer-a.test.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,23 @@ describe("Layer A: high-entropy-literal", () => {
6464
expect(result).toNotFlag("obf.high-entropy-literal");
6565
});
6666

67+
it("does not flag ordinary C# SQL, exception, or log message strings", async () => {
68+
const src = `
69+
command.CommandText = "SELECT * FROM pgmq.send(@queue_name, @message::jsonb)";
70+
throw new JsonException($"PGMQ message {reader.GetInt64(0)} from queue '{queueName}' deserialized to null.");
71+
logger.LogWarning(
72+
"Failed to publish queued package event batch of {Count} events. First event: {EventType} for {Package}@{Version} (CorrelationId: {CorrelationId}); dropping batch",
73+
count,
74+
eventType,
75+
packageId,
76+
version,
77+
correlationId);
78+
`;
79+
const { input, fileResolver } = virtualFiles({ "src/PgmqWorker.cs": src });
80+
const result = await scan(input, { fileResolver, ...silentOptions() });
81+
expect(result).toNotFlag("obf.high-entropy-literal");
82+
});
83+
6784
it("does not flag short strings even if entropy is high", async () => {
6885
const { input, fileResolver } = virtualFiles({
6986
"src/short.js": `const k = "Zm9v";\n`, // 4 chars, well below the 64-char floor

0 commit comments

Comments
 (0)