From 990176ac7c0adce58095678ead16a1df5fbea9cb Mon Sep 17 00:00:00 2001 From: M0N7Y5 <17201053+M0n7y5@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:47:58 +0200 Subject: [PATCH] feat(markdown): add a binary AST envelope to the native ABI (#2117) --- packages/markdown_parser/rust/src/binary.rs | 288 ++++++++++++++++++ packages/markdown_parser/rust/src/lib.rs | 1 + packages/markdown_parser/rust/src/native.rs | 30 ++ .../rust/tests/binary_envelope.rs | 61 ++++ 4 files changed, 380 insertions(+) create mode 100644 packages/markdown_parser/rust/src/binary.rs create mode 100644 packages/markdown_parser/rust/tests/binary_envelope.rs diff --git a/packages/markdown_parser/rust/src/binary.rs b/packages/markdown_parser/rust/src/binary.rs new file mode 100644 index 000000000..50a01ce13 --- /dev/null +++ b/packages/markdown_parser/rust/src/binary.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::ast::{ + AlertType, EmojiKind, GuildNavigationType, ListItem, MentionKind, Node, TableAlignment, + TimestampStyle, +}; + +pub const FORMAT_VERSION: u8 = 1; + +pub fn write_ast_binary(nodes: &[Node]) -> Vec { + let mut out = Vec::with_capacity(256); + out.push(FORMAT_VERSION); + write_nodes(&mut out, nodes); + out +} + +fn write_nodes(out: &mut Vec, nodes: &[Node]) { + write_varint(out, nodes.len() as u64); + for node in nodes { + write_node(out, node); + } +} + +fn write_node(out: &mut Vec, node: &Node) { + match node { + Node::Text { content } => { + out.push(0); + write_str(out, content); + } + Node::Blockquote { + children, + blank_lines, + } => { + out.push(1); + write_optional_varint(out, blank_lines.map(|value| value as u64)); + write_nodes(out, children); + } + Node::Strong { children } => { + out.push(2); + write_nodes(out, children); + } + Node::Emphasis { children } => { + out.push(3); + write_nodes(out, children); + } + Node::Underline { children } => { + out.push(4); + write_nodes(out, children); + } + Node::Strikethrough { children } => { + out.push(5); + write_nodes(out, children); + } + Node::Spoiler { children, is_block } => { + out.push(6); + out.push(match is_block { + None => 0, + Some(false) => 1, + Some(true) => 2, + }); + write_nodes(out, children); + } + Node::Heading { level, children } => { + out.push(7); + out.push(*level); + write_nodes(out, children); + } + Node::Subtext { children } => { + out.push(8); + write_nodes(out, children); + } + Node::List { ordered, items } => { + out.push(9); + out.push(u8::from(*ordered)); + write_varint(out, items.len() as u64); + for ListItem { children, ordinal } in items { + write_optional_varint(out, ordinal.map(|value| value as u64)); + write_nodes(out, children); + } + } + Node::CodeBlock { language, content } => { + out.push(10); + write_optional_str(out, language.as_deref()); + write_str(out, content); + } + Node::InlineCode { content } => { + out.push(11); + write_str(out, content); + } + Node::Sequence { children } => { + out.push(12); + write_nodes(out, children); + } + Node::Link { + text, + url, + escaped, + raw_url, + source, + } => { + out.push(13); + out.push(u8::from(*escaped) | (u8::from(text.is_some()) << 1)); + write_str(out, url); + write_str(out, raw_url); + write_str(out, source); + if let Some(text) = text { + write_node(out, text); + } + } + Node::Mention { kind } => { + out.push(14); + write_mention(out, kind); + } + Node::Timestamp { timestamp, style } => { + out.push(15); + write_varint(out, *timestamp); + out.push(timestamp_style_code(*style)); + } + Node::Emoji { kind } => { + out.push(16); + write_emoji(out, kind); + } + Node::Table { + header, + alignments, + rows, + } => { + out.push(17); + write_node(out, header); + write_varint(out, alignments.len() as u64); + for alignment in alignments { + out.push(alignment_code(*alignment)); + } + write_nodes(out, rows); + } + Node::TableRow { cells } => { + out.push(18); + write_nodes(out, cells); + } + Node::TableCell { children } => { + out.push(19); + write_nodes(out, children); + } + Node::Alert { + alert_type, + children, + } => { + out.push(20); + out.push(alert_code(*alert_type)); + write_nodes(out, children); + } + } +} + +fn write_mention(out: &mut Vec, kind: &MentionKind) { + match kind { + MentionKind::User { id } => { + out.push(0); + write_str(out, id); + } + MentionKind::Channel { id } => { + out.push(1); + write_str(out, id); + } + MentionKind::Role { id } => { + out.push(2); + write_str(out, id); + } + MentionKind::Command { + name, + subcommand_group, + subcommand, + id, + } => { + out.push(3); + write_str(out, name); + write_optional_str(out, subcommand_group.as_deref()); + write_optional_str(out, subcommand.as_deref()); + write_str(out, id); + } + MentionKind::GuildNavigation { + navigation_type, + id, + } => { + out.push(4); + out.push(match navigation_type { + GuildNavigationType::Customize => 0, + GuildNavigationType::Browse => 1, + GuildNavigationType::Guide => 2, + GuildNavigationType::LinkedRoles => 3, + }); + write_optional_str(out, id.as_deref()); + } + MentionKind::Everyone => out.push(5), + MentionKind::Here => out.push(6), + } +} + +fn write_emoji(out: &mut Vec, kind: &EmojiKind) { + match kind { + EmojiKind::Standard { + raw, + codepoints, + name, + } => { + out.push(0); + write_str(out, raw); + write_str(out, codepoints); + write_str(out, name); + } + EmojiKind::Custom { name, id, animated } => { + out.push(1); + out.push(u8::from(*animated)); + write_str(out, name); + write_str(out, id); + } + } +} + +fn timestamp_style_code(style: TimestampStyle) -> u8 { + match style { + TimestampStyle::ShortTime => 0, + TimestampStyle::LongTime => 1, + TimestampStyle::ShortDate => 2, + TimestampStyle::LongDate => 3, + TimestampStyle::ShortDateTime => 4, + TimestampStyle::LongDateTime => 5, + TimestampStyle::ShortDateShortTime => 6, + TimestampStyle::ShortDateMediumTime => 7, + TimestampStyle::RelativeTime => 8, + } +} + +fn alignment_code(alignment: TableAlignment) -> u8 { + match alignment { + TableAlignment::Left => 0, + TableAlignment::Center => 1, + TableAlignment::Right => 2, + TableAlignment::None => 3, + } +} + +fn alert_code(alert_type: AlertType) -> u8 { + match alert_type { + AlertType::Note => 0, + AlertType::Tip => 1, + AlertType::Important => 2, + AlertType::Warning => 3, + AlertType::Caution => 4, + } +} + +fn write_varint(out: &mut Vec, mut value: u64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +fn write_optional_varint(out: &mut Vec, value: Option) { + match value { + None => out.push(0), + Some(value) => { + out.push(1); + write_varint(out, value); + } + } +} + +fn write_str(out: &mut Vec, value: &str) { + write_varint(out, value.len() as u64); + out.extend_from_slice(value.as_bytes()); +} + +fn write_optional_str(out: &mut Vec, value: Option<&str>) { + match value { + None => out.push(0), + Some(value) => { + out.push(1); + write_str(out, value); + } + } +} diff --git a/packages/markdown_parser/rust/src/lib.rs b/packages/markdown_parser/rust/src/lib.rs index e4f149ec8..e7e8ccf96 100644 --- a/packages/markdown_parser/rust/src/lib.rs +++ b/packages/markdown_parser/rust/src/lib.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later pub mod ast; +pub mod binary; pub mod block; pub mod constants; pub mod emoji; diff --git a/packages/markdown_parser/rust/src/native.rs b/packages/markdown_parser/rust/src/native.rs index 672d5b1bc..2419eb78e 100644 --- a/packages/markdown_parser/rust/src/native.rs +++ b/packages/markdown_parser/rust/src/native.rs @@ -36,6 +36,36 @@ pub unsafe extern "C" fn fluxer_md_parse( } } +#[allow(clippy::missing_safety_doc)] +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fluxer_md_parse_binary( + input_ptr: *const u8, + input_len: usize, + flags: u32, + tsv_ptr: *const u8, + tsv_len: usize, + out: *mut FluxerMdBuffer, +) -> u32 { + let Ok(input) = std::str::from_utf8(unsafe { slice(input_ptr, input_len) }) else { + return unsafe { write_error(out, "invalid markdown input") }; + }; + let Ok(emoji_context) = std::str::from_utf8(unsafe { slice(tsv_ptr, tsv_len) }) else { + return unsafe { write_error(out, "invalid emoji context") }; + }; + let parsed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let context = crate::EmojiContext::parse(emoji_context); + let mut parser = crate::MarkdownParser::new(flags, context); + parser + .parse(input) + .map(|nodes| crate::binary::write_ast_binary(&nodes)) + })); + match parsed { + Ok(Ok(bytes)) => unsafe { write_data(out, bytes) }, + Ok(Err(_)) => unsafe { write_error(out, "markdown parse failed") }, + Err(_) => unsafe { write_error(out, "markdown parser panicked") }, + } +} + #[allow(clippy::missing_safety_doc)] #[unsafe(no_mangle)] pub unsafe extern "C" fn fluxer_md_buffer_free(out: *mut FluxerMdBuffer) { diff --git a/packages/markdown_parser/rust/tests/binary_envelope.rs b/packages/markdown_parser/rust/tests/binary_envelope.rs new file mode 100644 index 000000000..c16b8b17f --- /dev/null +++ b/packages/markdown_parser/rust/tests/binary_envelope.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use fluxer_markdown_parser::binary::{FORMAT_VERSION, write_ast_binary}; +use fluxer_markdown_parser::{EmojiContext, MarkdownParser, ParserFlags}; + +fn encode(input: &str) -> Vec { + let mut parser = MarkdownParser::new(ParserFlags::ALL, EmojiContext::parse("")); + let nodes = parser.parse(input).expect("parse should succeed"); + write_ast_binary(&nodes) +} + +#[test] +fn version_byte_is_stable() { + assert_eq!(FORMAT_VERSION, 1); +} + +#[test] +fn empty_input_is_version_and_zero_count() { + assert_eq!(encode(""), [1, 0]); +} + +#[test] +fn strong_text_golden_bytes() { + assert_eq!(encode("**b**"), [1, 1, 2, 1, 0, 1, b'b']); +} + +#[test] +fn timestamp_golden_bytes() { + assert_eq!( + encode(""), + [1, 1, 15, 0xD2, 0x85, 0xD8, 0xCC, 0x04, 8] + ); +} + +#[test] +fn ordered_list_golden_bytes() { + assert_eq!(encode("4. a"), [1, 1, 9, 1, 1, 1, 4, 1, 0, 1, b'a']); +} + +#[test] +fn link_golden_bytes() { + let url = b"https://e.com/a"; + let mut expected = vec![1, 1, 13, 2]; + for _ in 0..2 { + expected.push(url.len() as u8); + expected.extend_from_slice(url); + } + let source = b"[t](https://e.com/a)"; + expected.push(source.len() as u8); + expected.extend_from_slice(source); + expected.extend_from_slice(&[0, 1, b't']); + assert_eq!(encode("[t](https://e.com/a)"), expected); +} + +#[test] +fn spoiler_and_heading_golden_bytes() { + assert_eq!( + encode("# h\n||s||"), + [1, 2, 7, 1, 1, 0, 1, b'h', 6, 1, 1, 0, 1, b's'] + ); +}