From f32e56e8bf46148c8f724aa58f02635bbdc1e38c Mon Sep 17 00:00:00 2001 From: Christopher Date: Sat, 7 Mar 2026 18:12:08 -0600 Subject: [PATCH 1/8] =?UTF-8?q?=F0=9F=94=A4=20Update=20Unicode=20scanner?= =?UTF-8?q?=20to=20v2026.03.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refactor: Expand common Unicode exclusions and improve documentation Update version to 2026.03.0 across all references. Enhance the --exclude-common flag to cover additional typography characters including soft hyphens, superscripts, subscripts, Roman numerals, combining diacritical marks, and the replacement character. These additions reduce false positives in documentation and markdown repositories while maintaining security against actual threats. Improve help text clarity for the --exclude-common option. --- check-for-unicode/run.sh | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index ea82511..ede0260 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -# Unicode Security Scanner v2025.11.0 AI+ +# Unicode Security Scanner v2026.03.0 AI+ # Detects dangerous Unicode characters that can be used in security attacks # Including Trojan Source attacks (CVE-2021-42574) and other invisible characters # Script configuration -VERSION="2025.11.0" +VERSION="2026.03.0" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Command-line options (defaults) @@ -53,7 +53,9 @@ OPTIONS: (comma-separated, e.g., "critical,high") --allowlist FILE Path to allowlist file (default: .unicode-allowlist) --exclude-emojis Exclude emoji characters and variation selectors (reduces false positives) - --exclude-common Exclude common Unicode like smart quotes, dashes (very permissive) + --exclude-common Exclude common Unicode typography: smart quotes, dashes, + ellipsis, superscripts, subscripts, Roman numerals, + combining accents (recommended for docs/markdown repos) --include-binary Include binary files (archives, images, executables, etc.) By default, only text files are scanned to avoid false positives @@ -202,6 +204,19 @@ is_common_unicode() { [[ "$unicode_code" =~ ^203[9A]$ ]] && return 0 # Per mille: U+2030 [[ "$unicode_code" == "2030" ]] && return 0 + # Soft Hyphen: U+00AD (invisible formatting hint, harmless in HTML/markdown) + [[ "$unicode_code" == "00AD" ]] && return 0 + # Superscript digits: U+00B2 (²), U+00B3 (³), U+00B9 (¹), U+2070-U+2079 + [[ "$unicode_code" =~ ^00B[239]$ ]] && return 0 + [[ "$unicode_code" =~ ^207[0-9]$ ]] && return 0 + # Subscript digits: U+2080-U+2084 + [[ "$unicode_code" =~ ^208[0-4]$ ]] && return 0 + # Roman numerals: U+2160-U+2179 (used in outlines, legal docs, lists) + [[ "$unicode_code" =~ ^21[67][0-9A-F]$ ]] && return 0 + # Combining diacritical marks (U+0300-U+030C) — used in accented Latin text (French, Spanish, etc.) + [[ "$unicode_code" =~ ^030[0-9A-C]$ ]] && return 0 + # Replacement Character U+FFFD — appears naturally from encoding tools/editors, not a security threat + [[ "$unicode_code" == "FFFD" ]] && return 0 return 1 } From ba7022858987039f3dc89d2d447f25bd09f46e07 Mon Sep 17 00:00:00 2001 From: Christopher Date: Sat, 7 Mar 2026 18:24:30 -0600 Subject: [PATCH 2/8] =?UTF-8?q?fix:=F0=9F=94=A7=20Correct=20Roman=20numera?= =?UTF-8?q?l=20range=20and=20remove=20false=20positives?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correct the Roman numeral Unicode range from U+2179 to U+217F to include the complete set of Roman numeral characters. Remove checks for combining diacritical marks (U+0300-U+030C) and the replacement character (U+FFFD) as these are not security threats and cause false positives in legitimate text processing. --- check-for-unicode/run.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index ede0260..3ecebe6 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -211,12 +211,8 @@ is_common_unicode() { [[ "$unicode_code" =~ ^207[0-9]$ ]] && return 0 # Subscript digits: U+2080-U+2084 [[ "$unicode_code" =~ ^208[0-4]$ ]] && return 0 - # Roman numerals: U+2160-U+2179 (used in outlines, legal docs, lists) + # Roman numerals: U+2160-U+217F (used in outlines, legal docs, lists) [[ "$unicode_code" =~ ^21[67][0-9A-F]$ ]] && return 0 - # Combining diacritical marks (U+0300-U+030C) — used in accented Latin text (French, Spanish, etc.) - [[ "$unicode_code" =~ ^030[0-9A-C]$ ]] && return 0 - # Replacement Character U+FFFD — appears naturally from encoding tools/editors, not a security threat - [[ "$unicode_code" == "FFFD" ]] && return 0 return 1 } From c18fdac6976496e0aafa7d8ae5fc34f330af3d3a Mon Sep 17 00:00:00 2001 From: Christopher Date: Sun, 8 Mar 2026 00:19:28 -0600 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=94=A7=20refactor:=20improve=20docs?= =?UTF-8?q?=20and=20version=20display?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update help text to clarify --exclude-common behavior and mention soft hyphen exclusion. Replace hardcoded version string with VERSION variable for dynamic version display in header output. --- check-for-unicode/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index 3ecebe6..749e80a 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -54,8 +54,8 @@ OPTIONS: --allowlist FILE Path to allowlist file (default: .unicode-allowlist) --exclude-emojis Exclude emoji characters and variation selectors (reduces false positives) --exclude-common Exclude common Unicode typography: smart quotes, dashes, - ellipsis, superscripts, subscripts, Roman numerals, - combining accents (recommended for docs/markdown repos) + ellipsis, soft hyphen, superscripts, subscripts, + Roman numerals (recommended for docs/markdown repos) --include-binary Include binary files (archives, images, executables, etc.) By default, only text files are scanned to avoid false positives @@ -72,7 +72,7 @@ EXAMPLES: $0 --quiet --json ./app/ > results.json # JSON output for CI $0 --severity critical,high ./ # Only show critical/high $0 --exclude-emojis ./ui/ # Skip emoji characters in UI code - $0 --exclude-common ./docs/ # Very permissive for documentation + $0 --exclude-common ./docs/ # Recommended for docs/markdown repos $0 --include-binary ./ # Scan all files including binaries $0 --allowlist .unicode-allowlist ./ # Use custom allowlist @@ -644,7 +644,7 @@ load_allowlist # Show header unless in quiet or JSON mode if [ "$QUIET_MODE" = false ] && [ "$JSON_OUTPUT" = false ]; then echo -e "\033[1;35m╔══════════════════════════════════════════════════════════════╗\033[0m" - echo -e "\033[1;35m║ Big Bear Unicode Security Scanner v2.1.1 AI+ ║\033[0m" + echo -e "\033[1;35m║ Big Bear Unicode Security Scanner v${VERSION} AI+ ║\033[0m" echo -e "\033[1;35m║ Detecting dangerous Unicode & AI injection attacks ║\033[0m" echo -e "\033[1;35m║ Please support me! ║\033[0m" echo -e "\033[1;35m║ https://ko-fi.com/bigbeartechworld ║\033[0m" From 765be03cd47ab54626f1586cfa78dfd5204528a1 Mon Sep 17 00:00:00 2001 From: Christopher Date: Sun, 8 Mar 2026 12:17:50 -0500 Subject: [PATCH 4/8] =?UTF-8?q?=F0=9F=94=A4=20docs:=20clarify=20exclude-co?= =?UTF-8?q?mmon=20flag=20behavior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update help text to clarify that --exclude-common also suppresses AI-confusion and homograph checks. Fix alignment of header banner text to center properly within the box borders. --- check-for-unicode/run.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index 749e80a..09837be 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -54,8 +54,9 @@ OPTIONS: --allowlist FILE Path to allowlist file (default: .unicode-allowlist) --exclude-emojis Exclude emoji characters and variation selectors (reduces false positives) --exclude-common Exclude common Unicode typography: smart quotes, dashes, - ellipsis, soft hyphen, superscripts, subscripts, - Roman numerals (recommended for docs/markdown repos) + ellipsis, soft hyphen, superscripts, subscripts, Roman + numerals (recommended for docs/markdown repos; note: + also suppresses some AI-confusion and homograph checks) --include-binary Include binary files (archives, images, executables, etc.) By default, only text files are scanned to avoid false positives @@ -644,8 +645,8 @@ load_allowlist # Show header unless in quiet or JSON mode if [ "$QUIET_MODE" = false ] && [ "$JSON_OUTPUT" = false ]; then echo -e "\033[1;35m╔══════════════════════════════════════════════════════════════╗\033[0m" - echo -e "\033[1;35m║ Big Bear Unicode Security Scanner v${VERSION} AI+ ║\033[0m" - echo -e "\033[1;35m║ Detecting dangerous Unicode & AI injection attacks ║\033[0m" + echo -e "\033[1;35m║ Big Bear Unicode Security Scanner v${VERSION} AI+ ║\033[0m" + echo -e "\033[1;35m║ Detecting dangerous Unicode & AI injection attacks ║\033[0m" echo -e "\033[1;35m║ Please support me! ║\033[0m" echo -e "\033[1;35m║ https://ko-fi.com/bigbeartechworld ║\033[0m" echo -e "\033[1;35m║ Thank you! ║\033[0m" From 0160904573157fa19912224d1cfd955fa8ab5067 Mon Sep 17 00:00:00 2001 From: Christopher Date: Sun, 8 Mar 2026 16:54:26 -0500 Subject: [PATCH 5/8] =?UTF-8?q?=F0=9F=94=A4=20refactor:=20improve=20Unicod?= =?UTF-8?q?e=20detection=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorganize help text for better readability by rewrapping lines at 72 characters. Remove soft hyphen from common Unicode exclusions and refine Roman numeral detection to exclude Latin-lookalike confusables (I, VI, X, v, x) while maintaining detection of other Roman numeral characters. --- check-for-unicode/run.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index 09837be..69d6ec4 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -54,9 +54,9 @@ OPTIONS: --allowlist FILE Path to allowlist file (default: .unicode-allowlist) --exclude-emojis Exclude emoji characters and variation selectors (reduces false positives) --exclude-common Exclude common Unicode typography: smart quotes, dashes, - ellipsis, soft hyphen, superscripts, subscripts, Roman - numerals (recommended for docs/markdown repos; note: - also suppresses some AI-confusion and homograph checks) + ellipsis, superscripts, subscripts, Roman numerals + (recommended for docs/markdown repos; note: also + suppresses some AI-confusion and homograph checks) --include-binary Include binary files (archives, images, executables, etc.) By default, only text files are scanned to avoid false positives @@ -205,15 +205,15 @@ is_common_unicode() { [[ "$unicode_code" =~ ^203[9A]$ ]] && return 0 # Per mille: U+2030 [[ "$unicode_code" == "2030" ]] && return 0 - # Soft Hyphen: U+00AD (invisible formatting hint, harmless in HTML/markdown) - [[ "$unicode_code" == "00AD" ]] && return 0 # Superscript digits: U+00B2 (²), U+00B3 (³), U+00B9 (¹), U+2070-U+2079 [[ "$unicode_code" =~ ^00B[239]$ ]] && return 0 [[ "$unicode_code" =~ ^207[0-9]$ ]] && return 0 # Subscript digits: U+2080-U+2084 [[ "$unicode_code" =~ ^208[0-4]$ ]] && return 0 - # Roman numerals: U+2160-U+217F (used in outlines, legal docs, lists) - [[ "$unicode_code" =~ ^21[67][0-9A-F]$ ]] && return 0 + # Roman numerals: U+2160-U+217F, excluding Latin-lookalike confusables + # (U+2160 I, U+2165 VI, U+2169 X, U+2174 v, U+2179 x remain detectable) + [[ "$unicode_code" =~ ^21[67][0-9A-F]$ ]] && \ + [[ ! "$unicode_code" =~ ^(2160|2165|2169|2174|2179)$ ]] && return 0 return 1 } From 4ca4a3eb0a2fcb0c3892ca2e6ee4a9b87f35eb9a Mon Sep 17 00:00:00 2001 From: Christopher Date: Sun, 8 Mar 2026 17:03:33 -0500 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=94=A4Expand=20Unicode=20exclusion=20?= =?UTF-8?q?patterns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand the --exclude-common option to cover additional common Unicode characters including common spaces, angle quotes, and per mille sign. Update documentation and add clarifying notes about superscript character coverage to reduce false positives in documentation and markdown repositories. --- check-for-unicode/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index 69d6ec4..4ee503e 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -54,7 +54,8 @@ OPTIONS: --allowlist FILE Path to allowlist file (default: .unicode-allowlist) --exclude-emojis Exclude emoji characters and variation selectors (reduces false positives) --exclude-common Exclude common Unicode typography: smart quotes, dashes, - ellipsis, superscripts, subscripts, Roman numerals + ellipsis, common spaces, angle quotes, per mille, + superscripts, subscripts, non-confusable Roman numerals (recommended for docs/markdown repos; note: also suppresses some AI-confusion and homograph checks) --include-binary Include binary files (archives, images, executables, etc.) @@ -206,6 +207,7 @@ is_common_unicode() { # Per mille: U+2030 [[ "$unicode_code" == "2030" ]] && return 0 # Superscript digits: U+00B2 (²), U+00B3 (³), U+00B9 (¹), U+2070-U+2079 + # Note: ^207[0-9]$ also covers U+2071 (ⁱ) and U+2073 which are not in harmful_patterns [[ "$unicode_code" =~ ^00B[239]$ ]] && return 0 [[ "$unicode_code" =~ ^207[0-9]$ ]] && return 0 # Subscript digits: U+2080-U+2084 From 1abf34775050e795111c5a8758878ab9b9f1d540 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 10 Mar 2026 22:00:39 -0500 Subject: [PATCH 7/8] =?UTF-8?q?fix:=20=F0=9F=94=A7=20Expand=20subscript=20?= =?UTF-8?q?digit=20range=20to=20U+2089?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the subscript digits Unicode range from U+2080-U+2084 to U+2080-U+2089 to include all subscript digits. Update the regex pattern from ^208[0-4]$ to ^208[0-9]$ to match the complete range of subscript digit characters. --- check-for-unicode/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index 4ee503e..258a448 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -210,8 +210,8 @@ is_common_unicode() { # Note: ^207[0-9]$ also covers U+2071 (ⁱ) and U+2073 which are not in harmful_patterns [[ "$unicode_code" =~ ^00B[239]$ ]] && return 0 [[ "$unicode_code" =~ ^207[0-9]$ ]] && return 0 - # Subscript digits: U+2080-U+2084 - [[ "$unicode_code" =~ ^208[0-4]$ ]] && return 0 + # Subscript digits: U+2080-U+2089 + [[ "$unicode_code" =~ ^208[0-9]$ ]] && return 0 # Roman numerals: U+2160-U+217F, excluding Latin-lookalike confusables # (U+2160 I, U+2165 VI, U+2169 X, U+2174 v, U+2179 x remain detectable) [[ "$unicode_code" =~ ^21[67][0-9A-F]$ ]] && \ From b93595d3cf6b5eff08c0254d24da9937d88eff08 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 10 Mar 2026 22:12:36 -0500 Subject: [PATCH 8/8] =?UTF-8?q?docs:=20=F0=9F=93=9D=20Add=20clarification?= =?UTF-8?q?=20on=20subscript=20digits=20range?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add explanatory comment to clarify that the subscript digits regex pattern ^208[0-9]$ covers Unicode range U+2085-U+2089, which are not currently included in the harmful_patterns list. This documents the intentional broader matching for future compatibility. --- check-for-unicode/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/check-for-unicode/run.sh b/check-for-unicode/run.sh index 258a448..c22c840 100755 --- a/check-for-unicode/run.sh +++ b/check-for-unicode/run.sh @@ -211,6 +211,7 @@ is_common_unicode() { [[ "$unicode_code" =~ ^00B[239]$ ]] && return 0 [[ "$unicode_code" =~ ^207[0-9]$ ]] && return 0 # Subscript digits: U+2080-U+2089 + # Note: ^208[0-9]$ also covers U+2085-U+2089 which are not currently in harmful_patterns [[ "$unicode_code" =~ ^208[0-9]$ ]] && return 0 # Roman numerals: U+2160-U+217F, excluding Latin-lookalike confusables # (U+2160 I, U+2165 VI, U+2169 X, U+2174 v, U+2179 x remain detectable)