//! Markdown parser. //! //! Markdown is the simplest of the three formats: pure text, no //! executable hooks. The only vectors we extract are hyperlinks //! (which the scanner will inspect for phishing patterns). use std::path::PathBuf; use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd}; use crate::core::config::Config; use crate::core::types::{ Document, DocumentFormat, DocumentMetadata, ExecutableVector, Location, TextContext, TextNode, VectorType, }; use crate::CorbelResult; /// Concrete [`DocumentParser`] for Markdown. pub struct MarkdownParser; impl super::DocumentParser for MarkdownParser { fn parse( bytes: &[u8], _format: DocumentFormat, source_path: Option, _config: &Config, ) -> CorbelResult { // Markdown is text; lossy-convert to UTF-8. let text = String::from_utf8_lossy(bytes).into_owned(); let sha256 = crate::sha256_hex(&text.as_bytes()); let size = bytes.len() as u64; let mut text_nodes = Vec::new(); let mut vectors = Vec::new(); // Track the current semantic context as we walk events. let mut ctx_stack: Vec = vec![TextContext::Paragraph]; let mut current_line: u32 = 1; let mut current_col: u32 = 0; // Buffer accumulating text under the current node, plus the // line/col where accumulation started. let mut buf = String::new(); let mut buf_start: Option<(u32, u32)> = None; let opts = Options::ENABLE_TABLES .union(Options::ENABLE_STRIKETHROUGH) .union(Options::ENABLE_TASKLISTS); let parser = Parser::new_ext(&text, opts); for event in parser { match event { Event::Start(tag) => { // If this is a link, capture its destination as a vector. if let Tag::Link { dest_url, .. } = &tag { vectors.push(ExecutableVector { location: Location::MarkdownLine { line: current_line, col: current_col, }, vector_type: VectorType::MarkdownHyperlink, raw_payload: dest_url.as_bytes().to_vec(), decoded_preview: Some(dest_url.to_string()), }); } let new_ctx = context_for_tag(&tag); ctx_stack.push(new_ctx); // For code blocks / headings we want to capture // a fresh text node, so flush any pending buf. let ctx_for_flush = *ctx_stack.last().unwrap_or(&TextContext::Paragraph); flush_buf(&mut buf, &mut buf_start, &ctx_for_flush, &mut text_nodes); } Event::End(tag_end) => { let ended_ctx = *ctx_stack.last().unwrap_or(&TextContext::Paragraph); flush_buf(&mut buf, &mut buf_start, &ended_ctx, &mut text_nodes); // Pop the context stack — but make sure we don't pop // the bottom Paragraph frame. if ctx_stack.len() > 1 { ctx_stack.pop(); } // Reference `tag_end` so the compiler doesn't warn // about unused variable (we don't need it for anything // else; the link vector was already captured on Start). let _ = tag_end; } Event::Text(t) => { if buf_start.is_none() { buf_start = Some((current_line, current_col)); } // Update line/col by counting newlines in the text. for ch in t.chars() { if ch == '\n' { current_line += 1; current_col = 0; } else { current_col += 1; } } buf.push_str(&t); } Event::Code(c) => { // Inline code span — emit as its own text node so the // scanner can see it even if no other text accumulated. text_nodes.push(TextNode { location: Location::MarkdownLine { line: current_line, col: current_col, }, context: TextContext::CodeSpan, content: c.into_string(), }); } Event::InlineHtml(h) => { // Inline HTML inside Markdown is suspicious by default — // emit as a text node with CodeSpan context so the // scanner can decide. text_nodes.push(TextNode { location: Location::MarkdownLine { line: current_line, col: current_col, }, context: TextContext::CodeSpan, content: h.into_string(), }); } Event::DisplayMath(m) | Event::InlineMath(m) => { text_nodes.push(TextNode { location: Location::MarkdownLine { line: current_line, col: current_col, }, context: TextContext::CodeSpan, content: m.into_string(), }); } Event::SoftBreak | Event::HardBreak => { current_line += 1; current_col = 0; if !buf.is_empty() { buf.push('\n'); } } Event::TaskListMarker(_) => { // Ignore — TaskListMarker doesn't carry text. } Event::FootnoteReference(f) => { text_nodes.push(TextNode { location: Location::MarkdownLine { line: current_line, col: current_col, }, context: TextContext::Paragraph, content: format!("[^{}]", f), }); } Event::Html(h) => { // Block-level HTML — emit as its own paragraph-context // text node so the scanner can inspect it. text_nodes.push(TextNode { location: Location::MarkdownLine { line: current_line, col: current_col, }, context: TextContext::Paragraph, content: h.into_string(), }); } Event::Rule => { // Horizontal rule — no text to capture. } } } // Flush any trailing buffered text. flush_buf( &mut buf, &mut buf_start, ctx_stack.last().unwrap_or(&TextContext::Paragraph), &mut text_nodes, ); // Extract metadata from the first heading. let metadata = extract_metadata(&text_nodes); Ok(Document { format: DocumentFormat::Markdown, source_path, raw_bytes: bytes.to_vec(), sha256, size, metadata, text_nodes, executable_vectors: vectors, }) } } fn context_for_tag(tag: &Tag) -> TextContext { match tag { Tag::Paragraph => TextContext::Paragraph, Tag::Heading { .. } => TextContext::Heading, Tag::CodeBlock(_) => TextContext::CodeBlock, Tag::Emphasis | Tag::Strong | Tag::Strikethrough => TextContext::Paragraph, Tag::Link { .. } => TextContext::Hyperlink, Tag::Image { .. } => TextContext::Paragraph, Tag::BlockQuote(_) => TextContext::BlockQuote, Tag::List(_) | Tag::Item => TextContext::Paragraph, _ => TextContext::Paragraph, } } fn flush_buf( buf: &mut String, buf_start: &mut Option<(u32, u32)>, ctx: &TextContext, nodes: &mut Vec, ) { if buf.is_empty() { return; } let (line, col) = buf_start.unwrap_or((1, 0)); nodes.push(TextNode { location: Location::MarkdownLine { line, col }, context: *ctx, content: std::mem::take(buf), }); *buf_start = None; } /// Best-effort extraction of document metadata from the first heading /// and optional `` comment. fn extract_metadata(nodes: &[TextNode]) -> DocumentMetadata { let mut meta = DocumentMetadata::default(); // First H1 heading becomes the title. if let Some(first_h1) = nodes .iter() .find(|n| n.context == TextContext::Heading) .map(|n| n.content.trim().to_string()) .filter(|s| !s.is_empty()) { meta.title = Some(first_h1); } meta } #[cfg(test)] mod tests { use super::*; use crate::parsers::DocumentParser; #[test] fn parses_simple_markdown() { let md = b"# Hello\n\nThis is a paragraph.\n\n## Subhead\n\nMore text."; let doc = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap(); assert_eq!(doc.format, DocumentFormat::Markdown); assert!(!doc.text_nodes.is_empty()); assert_eq!(doc.metadata.title.as_deref(), Some("Hello")); } #[test] fn captures_code_blocks() { let md = b"# Title\n\n```python\nimport os\nos.system('rm -rf /')\n```\n"; let doc = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap(); let code_nodes: Vec<_> = doc .text_nodes .iter() .filter(|n| n.context == TextContext::CodeBlock) .collect(); assert!(!code_nodes.is_empty(), "should have at least one code block node"); let combined: String = code_nodes.iter().map(|n| n.content.as_str()).collect(); assert!(combined.contains("os.system")); } #[test] fn captures_hyperlinks_as_vectors() { let md = b"# Title\n\n[click me](https://evil.example.com/path)\n"; let doc = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap(); assert_eq!(doc.executable_vectors.len(), 1); assert_eq!( doc.executable_vectors[0].vector_type, VectorType::MarkdownHyperlink ); assert_eq!( doc.executable_vectors[0].decoded_preview.as_deref(), Some("https://evil.example.com/path") ); } #[test] fn sha256_is_stable() { let md = b"# Hello\n"; let doc1 = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap(); let doc2 = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap(); assert_eq!(doc1.sha256, doc2.sha256); assert_eq!(doc1.sha256.len(), 64); } } // Suppress unused-import warning for `TagEnd` — we keep it imported so // the parser intent stays explicit, even though we currently rely on // context-stack pop rather than matching specific TagEnd variants. #[allow(unused_imports)] use TagEnd as _UnusedTagEnd;