corbel/src/parsers/md_parser.rs

//! Markdown parser.
//!
//! Markdown is the simplest of the three formats: pure text, no
//! executable hooks. The only vectors we extract are hyperlinks
//! (which the scanner will inspect for phishing patterns).

use std::path::PathBuf;

use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};

use crate::core::config::Config;
use crate::core::types::{
    Document, DocumentFormat, DocumentMetadata, ExecutableVector, Location, TextContext, TextNode,
    VectorType,
};
use crate::CorbelResult;

/// Concrete [`DocumentParser`] for Markdown.
pub struct MarkdownParser;

impl super::DocumentParser for MarkdownParser {
    fn parse(
        bytes: &[u8],
        _format: DocumentFormat,
        source_path: Option<PathBuf>,
        _config: &Config,
    ) -> CorbelResult<Document> {
        // Markdown is text; lossy-convert to UTF-8.
        let text = String::from_utf8_lossy(bytes).into_owned();

        let sha256 = crate::sha256_hex(&text.as_bytes());
        let size = bytes.len() as u64;

        let mut text_nodes = Vec::new();
        let mut vectors = Vec::new();

        // Track the current semantic context as we walk events.
        let mut ctx_stack: Vec<TextContext> = vec![TextContext::Paragraph];
        let mut current_line: u32 = 1;
        let mut current_col: u32 = 0;
        // Buffer accumulating text under the current node, plus the
        // line/col where accumulation started.
        let mut buf = String::new();
        let mut buf_start: Option<(u32, u32)> = None;

        let opts = Options::ENABLE_TABLES
            .union(Options::ENABLE_STRIKETHROUGH)
            .union(Options::ENABLE_TASKLISTS);

        let parser = Parser::new_ext(&text, opts);

        for event in parser {
            match event {
                Event::Start(tag) => {
                    // If this is a link, capture its destination as a vector.
                    if let Tag::Link { dest_url, .. } = &tag {
                        vectors.push(ExecutableVector {
                            location: Location::MarkdownLine {
                                line: current_line,
                                col: current_col,
                            },
                            vector_type: VectorType::MarkdownHyperlink,
                            raw_payload: dest_url.as_bytes().to_vec(),
                            decoded_preview: Some(dest_url.to_string()),
                        });
                    }

                    let new_ctx = context_for_tag(&tag);
                    ctx_stack.push(new_ctx);
                    // For code blocks / headings we want to capture
                    // a fresh text node, so flush any pending buf.
                    let ctx_for_flush = *ctx_stack.last().unwrap_or(&TextContext::Paragraph);
                    flush_buf(&mut buf, &mut buf_start, &ctx_for_flush, &mut text_nodes);
                }
                Event::End(tag_end) => {
                    let ended_ctx = *ctx_stack.last().unwrap_or(&TextContext::Paragraph);
                    flush_buf(&mut buf, &mut buf_start, &ended_ctx, &mut text_nodes);
                    // Pop the context stack — but make sure we don't pop
                    // the bottom Paragraph frame.
                    if ctx_stack.len() > 1 {
                        ctx_stack.pop();
                    }
                    // Reference `tag_end` so the compiler doesn't warn
                    // about unused variable (we don't need it for anything
                    // else; the link vector was already captured on Start).
                    let _ = tag_end;
                }
                Event::Text(t) => {
                    if buf_start.is_none() {
                        buf_start = Some((current_line, current_col));
                    }
                    // Update line/col by counting newlines in the text.
                    for ch in t.chars() {
                        if ch == '\n' {
                            current_line += 1;
                            current_col = 0;
                        } else {
                            current_col += 1;
                        }
                    }
                    buf.push_str(&t);
                }
                Event::Code(c) => {
                    // Inline code span — emit as its own text node so the
                    // scanner can see it even if no other text accumulated.
                    text_nodes.push(TextNode {
                        location: Location::MarkdownLine {
                            line: current_line,
                            col: current_col,
                        },
                        context: TextContext::CodeSpan,
                        content: c.into_string(),
                    });
                }
                Event::InlineHtml(h) => {
                    // Inline HTML inside Markdown is suspicious by default —
                    // emit as a text node with CodeSpan context so the
                    // scanner can decide.
                    text_nodes.push(TextNode {
                        location: Location::MarkdownLine {
                            line: current_line,
                            col: current_col,
                        },
                        context: TextContext::CodeSpan,
                        content: h.into_string(),
                    });
                }
                Event::DisplayMath(m) | Event::InlineMath(m) => {
                    text_nodes.push(TextNode {
                        location: Location::MarkdownLine {
                            line: current_line,
                            col: current_col,
                        },
                        context: TextContext::CodeSpan,
                        content: m.into_string(),
                    });
                }
                Event::SoftBreak | Event::HardBreak => {
                    current_line += 1;
                    current_col = 0;
                    if !buf.is_empty() {
                        buf.push('\n');
                    }
                }
                Event::TaskListMarker(_) => {
                    // Ignore — TaskListMarker doesn't carry text.
                }
                Event::FootnoteReference(f) => {
                    text_nodes.push(TextNode {
                        location: Location::MarkdownLine {
                            line: current_line,
                            col: current_col,
                        },
                        context: TextContext::Paragraph,
                        content: format!("[^{}]", f),
                    });
                }
                Event::Html(h) => {
                    // Block-level HTML — emit as its own paragraph-context
                    // text node so the scanner can inspect it.
                    text_nodes.push(TextNode {
                        location: Location::MarkdownLine {
                            line: current_line,
                            col: current_col,
                        },
                        context: TextContext::Paragraph,
                        content: h.into_string(),
                    });
                }
                Event::Rule => {
                    // Horizontal rule — no text to capture.
                }
            }
        }

        // Flush any trailing buffered text.
        flush_buf(
            &mut buf,
            &mut buf_start,
            ctx_stack.last().unwrap_or(&TextContext::Paragraph),
            &mut text_nodes,
        );

        // Extract metadata from the first heading.
        let metadata = extract_metadata(&text_nodes);

        Ok(Document {
            format: DocumentFormat::Markdown,
            source_path,
            raw_bytes: bytes.to_vec(),
            sha256,
            size,
            metadata,
            text_nodes,
            executable_vectors: vectors,
        })
    }
}

fn context_for_tag(tag: &Tag) -> TextContext {
    match tag {
        Tag::Paragraph => TextContext::Paragraph,
        Tag::Heading { .. } => TextContext::Heading,
        Tag::CodeBlock(_) => TextContext::CodeBlock,
        Tag::Emphasis | Tag::Strong | Tag::Strikethrough => TextContext::Paragraph,
        Tag::Link { .. } => TextContext::Hyperlink,
        Tag::Image { .. } => TextContext::Paragraph,
        Tag::BlockQuote(_) => TextContext::BlockQuote,
        Tag::List(_) | Tag::Item => TextContext::Paragraph,
        _ => TextContext::Paragraph,
    }
}

fn flush_buf(
    buf: &mut String,
    buf_start: &mut Option<(u32, u32)>,
    ctx: &TextContext,
    nodes: &mut Vec<TextNode>,
) {
    if buf.is_empty() {
        return;
    }
    let (line, col) = buf_start.unwrap_or((1, 0));
    nodes.push(TextNode {
        location: Location::MarkdownLine { line, col },
        context: *ctx,
        content: std::mem::take(buf),
    });
    *buf_start = None;
}

/// Best-effort extraction of document metadata from the first heading
/// and optional `<!-- corbel: ... -->` comment.
fn extract_metadata(nodes: &[TextNode]) -> DocumentMetadata {
    let mut meta = DocumentMetadata::default();

    // First H1 heading becomes the title.
    if let Some(first_h1) = nodes
        .iter()
        .find(|n| n.context == TextContext::Heading)
        .map(|n| n.content.trim().to_string())
        .filter(|s| !s.is_empty())
    {
        meta.title = Some(first_h1);
    }

    meta
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::parsers::DocumentParser;

    #[test]
    fn parses_simple_markdown() {
        let md = b"# Hello\n\nThis is a paragraph.\n\n## Subhead\n\nMore text.";
        let doc = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap();
        assert_eq!(doc.format, DocumentFormat::Markdown);
        assert!(!doc.text_nodes.is_empty());
        assert_eq!(doc.metadata.title.as_deref(), Some("Hello"));
    }

    #[test]
    fn captures_code_blocks() {
        let md = b"# Title\n\n```python\nimport os\nos.system('rm -rf /')\n```\n";
        let doc = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap();
        let code_nodes: Vec<_> = doc
            .text_nodes
            .iter()
            .filter(|n| n.context == TextContext::CodeBlock)
            .collect();
        assert!(!code_nodes.is_empty(), "should have at least one code block node");
        let combined: String = code_nodes.iter().map(|n| n.content.as_str()).collect();
        assert!(combined.contains("os.system"));
    }

    #[test]
    fn captures_hyperlinks_as_vectors() {
        let md = b"# Title\n\n[click me](https://evil.example.com/path)\n";
        let doc = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap();
        assert_eq!(doc.executable_vectors.len(), 1);
        assert_eq!(
            doc.executable_vectors[0].vector_type,
            VectorType::MarkdownHyperlink
        );
        assert_eq!(
            doc.executable_vectors[0].decoded_preview.as_deref(),
            Some("https://evil.example.com/path")
        );
    }

    #[test]
    fn sha256_is_stable() {
        let md = b"# Hello\n";
        let doc1 = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap();
        let doc2 = MarkdownParser::parse(md, DocumentFormat::Markdown, None, &Config::default()).unwrap();
        assert_eq!(doc1.sha256, doc2.sha256);
        assert_eq!(doc1.sha256.len(), 64);
    }
}

// Suppress unused-import warning for `TagEnd` — we keep it imported so
// the parser intent stays explicit, even though we currently rely on
// context-stack pop rather than matching specific TagEnd variants.
#[allow(unused_imports)]
use TagEnd as _UnusedTagEnd;