//! Threat signature tables and pattern matchers. //! //! This module centralizes the static lookup tables used by the //! heuristics engine. Keeping them in one place makes them easy to //! audit, extend, and eventually wire up to an external threat-intel //! feed (e.g. a YARA rules file or a STIX/TAXII subscription). //! //! ## What lives here //! //! - [`PHISHING_TLDS`] — TLDs statistically overrepresented in //! phishing URLs. Sourced from public phishing reports //! (Spamhaus, PhishTank yearly summaries). //! - [`SUSPICIOUS_URL_KEYWORDS`] — path/host keywords that strongly //! indicate credential harvesting or fake login pages. //! - [`URL_SHORTENER_DOMAINS`] — shortener domains. Not malicious //! per se, but a common obfuscation layer for phishing links. //! - [`KNOWN_FILE_SIGNATURES`] — magic-byte signatures for executable //! and high-risk file formats (PE, ELF, Mach-O, OLE2, RTF, etc.). //! - [`SHELLCODE_PATTERNS`] — known shellcode prologue byte sequences //! (NOP sleds, syscall stubs, common encoders). //! - [`COMMON_PHISHING_BRANDS`] — brand names frequently spoofed in //! phishing URLs (microsoft, paypal, appleid, …). //! //! ## External threat-intel feeds //! //! Additional rules can be loaded at runtime via //! [`load_external_rules`]. Loaded rules are stored in a //! process-wide static and checked by every match function //! alongside the built-in tables. use std::path::Path; use std::sync::OnceLock; use serde::Deserialize; use crate::CorbelResult; /// TLDs statistically overrepresented in phishing URLs. /// /// Source: synthesized from public yearly phishing reports /// (Spamhaus, PhishTank, Interisle). This list is intentionally /// conservative — inclusion requires the TLD to appear in multiple /// reports as a top-10 phishing TLD. pub const PHISHING_TLDS: &[&str] = &[ // High-risk TLDs (cheap registration, low verification) ".zip", ".mov", ".xyz", ".top", ".click", ".link", ".rest", ".cyou", ".sbs", ".online", ".live", ".buzz", ".surf", ".monster", ".fit", ".loan", ".win", ".download", ".stream", ".review", ".men", ".work", ".racing", ".party", ".trade", ".science", ".kim", ".cricket", ".gq", ".cf", ".tk", ".ml", ".ga", // Country-code TLDs frequently abused for phishing ".ru", ".cn", ".su", ".country", ".kim", // Newer TLDs that have been flagged ".quest", ".bond", ".ha", ".cyou", ".quest", ".beauty", ]; /// URL path / host keywords that strongly suggest credential harvesting /// or fake login pages. Matched case-insensitively as substrings. pub const SUSPICIOUS_URL_KEYWORDS: &[&str] = &[ "login", "signin", "sign-in", "log-in", "verify", "verification", "account", "update", "confirm", "secure", "security", "wallet", "unlock", "recover", "reactivate", "validate", "activate", "webscr", "cmd=", "_session", "authorization", "authenticate", "reset", "password", "credential", "billing", "invoice", "support", "suspended", "limited", "alert", "warning", "urgent", "important-notice", "tax", "refund", "irs", "postbank", "amzn", "appleid", "icloud", "office365", ]; /// Common URL-shortener domains. Shortened URLs are not malicious /// per se, but they hide the real destination — we flag them as /// `Suspicious` so the operator can preview the destination before /// clicking. pub const URL_SHORTENER_DOMAINS: &[&str] = &[ "bit.ly", "t.co", "tinyurl.com", "goo.gl", "ow.ly", "is.gd", "buff.ly", "rebrand.ly", "cutt.ly", "shorturl.at", "tiny.cc", "rb.gy", "s.id", "v.gd", "qr.ae", "x.co", "shorte.st", "soo.gd", "lnkd.in", "po.st", "yourls.org", "bl.ink", "surl.li", "kutt.it", "urlzs.com", "shrtco.de", ]; /// Brand names frequently spoofed in phishing URLs. Used to detect /// homograph attacks (e.g. `micros0ft.com`, `paypa1.com`). /// /// This list intentionally includes BOTH canonical spellings ("microsoft") /// AND known homograph variants ("micros0ft" with zero instead of 'o'). /// The matcher uses a canonical-domain check to suppress benign /// matches: when a brand is mentioned, we look for the canonical /// spelling followed by a TLD; if found, we don't flag. pub const COMMON_PHISHING_BRANDS: &[&str] = &[ // Microsoft family "microsoft", "micros0ft", "micros0fte", "micr0soft", "msn", "windows", "wind0ws", "office", "0ffice", "outlook", "outl00k", "outl0ok", "live", "1ive", // PayPal "paypal", "paypa1", "paypaI", "paypa|", // Apple "apple", "app1e", "appie", "icloud", "ic1oud", "appleid", "app1eid", // Google "google", "g00gle", "goog1e", "gmail", "gmai", // Amazon "amazon", "amzn", "amaz0n", "a-m-a-z-o-n", // Social "facebook", "faceb00k", "facebo0k", "instagram", "instagrarn", "twitter", "tw1tter", "twtter", "linkedin", "1inkedin", // Streaming "netflix", "netf1ix", "spotify", "spot1fy", // Storage / SaaS "dropbox", "dr0pbox", "adobe", "ad0be", // Banking "bankofamerica", "bofa", "b0fa", "wellsfargo", "wellsfarg0", "chase", "citibank", "citi", "hsbc", "barclays", "santander", "unicredit", // Crypto "binance", "binanc3", "coinbase", "c0inbase", "metamask", "metam4sk", "ledger", "trezor", // Shipping "dhl", "fedex", "f3dex", "ups", "usps", "royalmail", // Gaming "steamcommunity", "steampowered", "epicgames", "playstation", "nintendo", "xbox", ]; /// Magic-byte signatures for executable and high-risk file formats. /// /// Each entry is (offset, magic_bytes, name). When a payload's bytes /// at `offset` match `magic_bytes`, the payload is considered /// executable / high-risk. pub const KNOWN_FILE_SIGNATURES: &[(usize, &[u8], &str)] = &[ // Windows PE (0, b"MZ", "pe"), // ELF (0, b"\x7FELF", "elf"), // Mach-O fat binary (0, b"\xCA\xFE\xBA\xBE", "mach-o-fat"), // Mach-O 64-bit (big-endian) (0, b"\xFE\xED\xFA\xCF", "mach-o-64-be"), // Mach-O 64-bit (little-endian) (0, b"\xCF\xFA\xED\xFE", "mach-o-64-le"), // Mach-O 32-bit (big-endian) (0, b"\xFE\xED\xFA\xFE", "mach-o-32-be"), // Mach-O 32-bit (little-endian) (0, b"\xCE\xFA\xED\xFE", "mach-o-32-le"), // OLE2 (Microsoft Office legacy, also used by some malware) (0, b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1", "ole2"), // RTF (often used as an obfuscation vector for CVE-2017-11882 etc.) (0, b"{\\rtf", "rtf"), // Java class file (0, b"\xCA\xFE\xBA\xBE", "java-class"), // Java JAR (zip, but flag if inside PDF) // (zip is too generic — we don't flag it without other signals) // Python bytecode (0, b"\x42\x0d\x0d\x0a", "python-bytecode"), // SWF (Flash — historically a huge attack surface) (0, b"FWS", "swf"), (0, b"CWS", "swf-compressed"), (0, b"ZWS", "swf-lzma"), // Windows Help file (.hlp) — old but still seen in attacks (0, b"?_\x03\x00", "winhelp"), // Windows shortcut (.lnk) — common payload in phishing docs (0, b"\x4c\x00\x00\x00\x01\x14\x02\x00", "lnk"), // HTA application (HTML Application — executes as a script) (0, b", name: String, } /// Processed external rules, organized by type for fast matching. #[derive(Debug, Clone, Default)] pub(crate) struct ExternalRulesData { /// Additional TLD strings. pub(crate) tlds: Vec, /// Additional suspicious URL keywords. pub(crate) keywords: Vec, /// Additional brand strings (may include homograph variants). pub(crate) brands: Vec, /// Additional file-signature entries: (offset, magic-bytes, name). pub(crate) signatures: Vec<(usize, Vec, String)>, /// Additional shellcode byte patterns. pub(crate) shellcode: Vec>, } /// Process-wide storage for externally loaded rules. /// /// Populated once by [`load_external_rules`] and then read /// (immutably) by every match function. static EXTERNAL_RULES_STORAGE: OnceLock = OnceLock::new(); /// Load external signature rules from a JSON file. /// /// The file must contain a JSON array of [`ExternalRule`] objects. /// Rules are sorted into type-specific buckets and stored in a /// process-wide static ([`EXTERNAL_RULES_STORAGE`]). Subsequent calls /// to the match functions (`match_file_signature`, `has_phishing_tld`, /// etc.) will check both the built-in tables and the external rules. /// /// # Errors /// /// Returns [`CorbelError::Io`] if the file cannot be read, or /// [`CorbelError::Serde`] if the JSON is malformed. pub fn load_external_rules(path: &Path) -> CorbelResult> { let data = std::fs::read_to_string(path)?; let rules: Vec = serde_json::from_str(&data)?; let mut storage = ExternalRulesData::default(); for rule in &rules { match rule.rule_type.as_str() { "tld-list" => { if let Some(arr) = rule.values.as_array() { for v in arr { if let Some(s) = v.as_str() { storage.tlds.push(s.to_string()); } } } } "keyword-list" => { if let Some(arr) = rule.values.as_array() { for v in arr { if let Some(s) = v.as_str() { storage.keywords.push(s.to_string()); } } } } "brand-list" => { if let Some(arr) = rule.values.as_array() { for v in arr { if let Some(s) = v.as_str() { storage.brands.push(s.to_string()); } } } } "signature-list" => { if let Some(arr) = rule.values.as_array() { for v in arr { if let Ok(sig) = serde_json::from_value::(v.clone()) { storage .signatures .push((sig.offset, sig.bytes, sig.name)); } } } } "shellcode-list" => { if let Some(arr) = rule.values.as_array() { for v in arr { if let Some(hex_str) = v.as_str() { // Hex-encoded string: "fc4883e4..." if let Ok(bytes) = hex::decode(hex_str) { if !bytes.is_empty() { storage.shellcode.push(bytes); } } } else if let Some(byte_arr) = v.as_array() { // Raw byte array: [0xfc, 0x48, ...] let bytes: Vec = byte_arr .iter() .filter_map(|b| b.as_u64().map(|n| n as u8)) .collect(); if !bytes.is_empty() { storage.shellcode.push(bytes); } } } } } _ => { // Unknown rule type — silently skip. } } } let _ = EXTERNAL_RULES_STORAGE.set(storage); Ok(rules) } /// Return a reference to the externally loaded rules storage, if any /// has been loaded via [`load_external_rules`]. /// /// This is `pub(crate)` because the return type (`ExternalRulesData`) /// is itself `pub(crate)` — exposing it publicly would leak a private /// type through the public API. #[must_use] pub(crate) fn get_external_rules_storage() -> Option<&'static ExternalRulesData> { EXTERNAL_RULES_STORAGE.get() } /// Check whether `bytes` starts with any known executable / high-risk /// file signature. /// /// Returns the signature name (e.g. `"pe"`, `"elf"`) if matched, so /// the caller can include it in the forensic report. /// /// Checks both the built-in table and any external rules loaded via /// [`load_external_rules`]. #[must_use] pub fn match_file_signature(bytes: &[u8]) -> Option<&'static str> { for (offset, magic, name) in KNOWN_FILE_SIGNATURES { if bytes.len() >= *offset + magic.len() { if &bytes[*offset..*offset + magic.len()] == *magic { return Some(name); } } } if let Some(ext) = EXTERNAL_RULES_STORAGE.get() { for (offset, magic, name) in &ext.signatures { if bytes.len() >= *offset + magic.len() { if &bytes[*offset..*offset + magic.len()] == magic.as_slice() { return Some(name); } } } } None } /// Check whether `bytes` contains any known shellcode prologue. /// /// Checks both the built-in table and any external rules loaded via /// [`load_external_rules`]. #[must_use] pub fn match_shellcode_pattern(bytes: &[u8]) -> Option<&'static [u8]> { for pattern in SHELLCODE_PATTERNS { if bytes.windows(pattern.len()).any(|w| w == *pattern) { return Some(pattern); } } if let Some(ext) = EXTERNAL_RULES_STORAGE.get() { for pattern in &ext.shellcode { if bytes.windows(pattern.len()).any(|w| w == pattern.as_slice()) { return Some(pattern.as_slice()); } } } None } /// Check whether `host` (lowercase, no scheme) is a known URL shortener. #[must_use] pub fn is_url_shortener(host: &str) -> bool { URL_SHORTENER_DOMAINS.iter().any(|d| host == *d || host.ends_with(&format!(".{d}"))) } /// Check whether `uri` mentions a commonly-phished brand with a /// non-canonical domain (homograph bait). /// /// Returns the matched brand name if found. /// /// Logic: /// 1. If a homograph variant (`micros0ft`, `paypa1`, ...) is found /// anywhere in the URI, it's always a phishing signal. /// 2. If a canonical spelling (`microsoft`, `paypal`, ...) is found, /// we check whether the host portion is ANY brand's canonical /// domain (e.g. `microsoft.com` for "microsoft"). If yes → benign. /// Otherwise, the brand is mentioned in a non-canonical context /// → suspicious. /// /// Checks both the built-in table and any external brands loaded via /// [`load_external_rules`]. #[must_use] pub fn match_phishing_brand(uri: &str) -> Option<&'static str> { let lower = uri.to_ascii_lowercase(); // Extract host (after scheme://, before path/query/fragment, sans port). let host = lower .split("://") .nth(1) .unwrap_or(&lower) .split('/') .next() .unwrap_or("") .split(':') .next() .unwrap_or(""); let is_homograph = |brand: &str| brand.chars().any(|c| !c.is_ascii_alphabetic()); // First pass: check for homograph variants — these are ALWAYS phishing. // Built-in brands. for brand in COMMON_PHISHING_BRANDS { if is_homograph(brand) && lower.contains(brand) { return Some(brand); } } // External brands. if let Some(ext) = EXTERNAL_RULES_STORAGE.get() { for brand in &ext.brands { if is_homograph(brand) && lower.contains(brand.as_str()) { return Some(brand); } } } // Second pass: check canonical spellings. If the host is ANY // brand's canonical domain, all canonical brand mentions are // treated as benign. This handles cases like `microsoft.com/windows` // (windows is a brand, but the host is microsoft's canonical domain). let host_is_canonical_for_builtin = COMMON_PHISHING_BRANDS .iter() .any(|brand| !is_homograph(brand) && is_canonical_brand_host(host, brand)); let host_is_canonical_for_external = EXTERNAL_RULES_STORAGE .get() .map(|ext| { ext.brands .iter() .any(|brand| !is_homograph(brand) && is_canonical_brand_host(host, brand)) }) .unwrap_or(false); if host_is_canonical_for_builtin || host_is_canonical_for_external { return None; } // Host is not a canonical brand domain — any canonical brand // mentioned in the URL is suspicious. // Built-in brands. for brand in COMMON_PHISHING_BRANDS { if !is_homograph(brand) && lower.contains(brand) { return Some(brand); } } // External brands. if let Some(ext) = EXTERNAL_RULES_STORAGE.get() { for brand in &ext.brands { if !is_homograph(brand) && lower.contains(brand.as_str()) { return Some(brand); } } } None } /// Check whether `host` is the canonical domain for `brand`. /// /// A host is canonical if it matches `.` or has `` /// as a dot-separated segment (e.g. `microsoft.com`, `login.microsoft.com`). /// The goal is to allow legitimate brand-owned domains while still /// flagging `login-microsoft.com` (which is NOT a Microsoft domain). fn is_canonical_brand_host(host: &str, brand: &str) -> bool { if host == brand { return true; } // Check if `.` is a prefix. let canonical_prefix = format!("{}.", brand); if host.starts_with(&canonical_prefix) { return true; } // Check if `` is a dot-separated segment (e.g. `login.microsoft.com`). host.split('.').any(|seg| seg == brand) } /// Check whether `uri`'s path/host contains any suspicious keyword. /// /// Checks both the built-in table and any external rules loaded via /// [`load_external_rules`]. #[must_use] pub fn match_suspicious_keyword(uri: &str) -> Option<&'static str> { let lower = uri.to_ascii_lowercase(); if let Some(kw) = SUSPICIOUS_URL_KEYWORDS .iter() .copied() .find(|kw| lower.contains(kw)) { return Some(kw); } if let Some(ext) = EXTERNAL_RULES_STORAGE.get() { if let Some(kw) = ext .keywords .iter() .find(|kw| lower.contains(kw.as_str())) { return Some(kw); } } None } /// Check whether the host part of `uri` ends with a known phishing TLD. /// /// Checks both the built-in table and any external rules loaded via /// [`load_external_rules`]. #[must_use] pub fn has_phishing_tld(uri: &str) -> Option<&'static str> { let lower = uri.to_ascii_lowercase(); // Extract host portion (after scheme://, before path/query/fragment). let host = lower .split("://") .nth(1) .unwrap_or(&lower) .split('/') .next() .unwrap_or(""); // Strip port. let host = host.split(':').next().unwrap_or(""); if let Some(tld) = PHISHING_TLDS.iter().copied().find(|tld| host.ends_with(tld)) { return Some(tld); } if let Some(ext) = EXTERNAL_RULES_STORAGE.get() { if let Some(tld) = ext.tlds.iter().find(|tld| host.ends_with(tld.as_str())) { return Some(tld); } } None } #[cfg(test)] mod tests { use super::*; #[test] fn detects_pe_signature() { assert_eq!(match_file_signature(b"MZ\x90\x00\x03"), Some("pe")); } #[test] fn detects_elf_signature() { assert_eq!(match_file_signature(b"\x7FELF\x02"), Some("elf")); } #[test] fn detects_ole2_signature() { assert_eq!( match_file_signature(b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1rest"), Some("ole2") ); } #[test] fn detects_rtf_signature() { assert_eq!(match_file_signature(b"{\\rtf1\\ansi..."), Some("rtf")); } #[test] fn detects_swf_signature() { assert_eq!(match_file_signature(b"FWS\x09"), Some("swf")); } #[test] fn detects_lnk_signature() { assert_eq!( match_file_signature(b"\x4c\x00\x00\x00\x01\x14\x02\x00rest"), Some("lnk") ); } #[test] fn detects_vba_macro_signature() { assert_eq!( match_file_signature(b"Attribute VB_Name = \"evil\""), Some("vba-macro") ); } #[test] fn detects_nop_sled_pattern() { let pattern = match_shellcode_pattern(&[0x90; 32]).unwrap(); assert_eq!(pattern[0], 0x90); } #[test] fn detects_metasploit_stager_pattern() { let payload = [0xFC, 0xE8, 0x82, 0x00, 0x00, 0x00, 0x60, 0x89]; assert!(match_shellcode_pattern(&payload).is_some()); } #[test] fn detects_url_shortener() { assert!(is_url_shortener("bit.ly")); assert!(is_url_shortener("sub.bit.ly")); assert!(!is_url_shortener("example.com")); } #[test] fn detects_phishing_brand_homograph() { // micros0ft.com (with zero instead of 'o') should match the // homograph variant directly. assert_eq!( match_phishing_brand("https://micros0ft.com/login"), Some("micros0ft") ); // paypa1.com (with one instead of 'l') should match. assert_eq!( match_phishing_brand("https://paypa1.com/signin"), Some("paypa1") ); } #[test] fn canonical_brand_domain_not_flagged() { // microsoft.com (canonical) should not be flagged. assert!(match_phishing_brand("https://microsoft.com/windows").is_none()); // paypal.com (canonical) should not be flagged. assert!(match_phishing_brand("https://paypal.com/home").is_none()); } #[test] fn canonical_brand_in_non_canonical_domain_is_flagged() { // microsoft mentioned in a non-canonical host → suspicious. assert_eq!( match_phishing_brand("https://login-microsoft.com/verify"), Some("microsoft") ); } #[test] fn detects_suspicious_keyword() { // Should return the first matching keyword — both "account" // and "verify" are in the list. Either is acceptable; check // that we get one of them. let result = match_suspicious_keyword("https://example.com/account/verify"); assert!(matches!(result, Some("account") | Some("verify"))); assert_eq!( match_suspicious_keyword("https://example.com/signin"), Some("signin") ); } #[test] fn detects_phishing_tld() { assert_eq!(has_phishing_tld("https://example.xyz"), Some(".xyz")); assert_eq!(has_phishing_tld("https://example.top/path"), Some(".top")); assert!(has_phishing_tld("https://example.com").is_none()); } #[test] fn detects_phishing_tld_with_port() { assert_eq!( has_phishing_tld("https://example.xyz:8080/path"), Some(".xyz") ); } }