Skip to content

feat(markdown): improve markdown parser and diagnostics #5292

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 32 commits into from
Closed
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
b712073
feat(markdown_parser): add blockquote block parsing support
afonsojramos Mar 7, 2025
6cfda97
feat(markdown_parser): add code block parsing support for indented an…
afonsojramos Mar 7, 2025
3e88015
feat(markdown_parser): add header block parsing support
afonsojramos Mar 7, 2025
421f9e8
feat(markdown_parser): add HTML block parsing support
afonsojramos Mar 7, 2025
339ba33
feat(markdown_parser): add list block parsing support
afonsojramos Mar 7, 2025
184aeec
feat(markdown_parser): add paragraph block parsing support
afonsojramos Mar 7, 2025
0ef00ca
feat(markdown_parser): add table block parsing support
afonsojramos Mar 7, 2025
d5cd19b
feat(markdown_parser): improve thematic break block parsing
afonsojramos Mar 7, 2025
f0758f3
feat(markdown_parser): add document parsing method
afonsojramos Mar 7, 2025
1d511ad
feat(markdown_parser): implement comprehensive document parsing with …
afonsojramos Mar 7, 2025
5c03604
feat(markdown_parser): add header validation and improve parsing robu…
afonsojramos Mar 7, 2025
7b4fbc6
test(markdown_parser): add test case for invalid markdown headers
afonsojramos Mar 7, 2025
7f822f2
test(markdown_parser): add test cases for blockquotes, comprehensive …
afonsojramos Mar 7, 2025
ba4f600
test(markdown_parser): refactor spec tests and add more flexible test…
afonsojramos Mar 7, 2025
381d45b
feat(markdown_parser): add support for additional Markdown syntax tok…
afonsojramos Mar 7, 2025
ec95111
feat(configuration): add Markdown configuration support
afonsojramos Mar 7, 2025
4a0fdf6
feat(cli): add Markdown linter configuration support
afonsojramos Mar 7, 2025
9b3bc39
Merge remote-tracking branch 'biome/main' into feat/markdown-parser
afonsojramos Mar 7, 2025
63d17b8
refactor(markdown_parser): improve parsing robustness for various Mar…
afonsojramos Mar 7, 2025
8991695
chore: run cargo fmt
afonsojramos Mar 7, 2025
833a088
chore: remove Markdown configuration
afonsojramos Mar 10, 2025
0222689
refactor(markdown_parser): remove test files and cleanup markdown con…
afonsojramos Mar 11, 2025
cf45339
feat(markdown_parser): improve thematic break parsing with more robus…
afonsojramos Mar 12, 2025
9a29b58
refactor(markdown_parser): improve blockquote parsing with enhanced l…
afonsojramos Mar 12, 2025
135c2b5
refactor(markdown_parser): improve header block parsing with simplifi…
afonsojramos Mar 12, 2025
18b9f0c
refactor(markdown_parser): improve lexer parsing for Markdown list ma…
afonsojramos Mar 12, 2025
804f2ba
refactor(markdown_parser): enhance parsing logic for Markdown documents
afonsojramos Mar 15, 2025
b45b44a
refactor(markdown_parser): enhance header block parsing with improved…
afonsojramos Mar 15, 2025
c9743c9
test(markdown_parser): add comprehensive tests for list marker parsing
afonsojramos Mar 15, 2025
e91ec37
refactor(markdown_parser): enhance list parsing logic with improved v…
afonsojramos Mar 15, 2025
952ca45
refactor(markdown_parser): streamline paragraph block parsing logic
afonsojramos Mar 15, 2025
0505548
refactor(markdown_parser): update snapshot files for blockquotes, he…
afonsojramos Mar 15, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions crates/biome_cli/src/commands/lint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use biome_configuration::css::CssLinterConfiguration;
use biome_configuration::graphql::GraphqlLinterConfiguration;
use biome_configuration::javascript::JsLinterConfiguration;
use biome_configuration::json::JsonLinterConfiguration;
use biome_configuration::markdown::MarkdownLinterConfiguration;
use biome_configuration::vcs::VcsConfiguration;
use biome_configuration::{Configuration, FilesConfiguration, LinterConfiguration};
use biome_console::Console;
Expand Down Expand Up @@ -37,6 +38,7 @@ pub(crate) struct LintCommandPayload {
pub(crate) json_linter: Option<JsonLinterConfiguration>,
pub(crate) css_linter: Option<CssLinterConfiguration>,
pub(crate) graphql_linter: Option<GraphqlLinterConfiguration>,
pub(crate) markdown_linter: Option<MarkdownLinterConfiguration>,
}

impl CommandRunner for LintCommandPayload {
Expand Down Expand Up @@ -83,12 +85,21 @@ impl CommandRunner for LintCommandPayload {
.get_or_insert_with(Default::default);
graphql.linter.merge_with(self.graphql_linter.clone());
}

if self.markdown_linter.is_some() {
let markdown = fs_configuration
.markdown
.get_or_insert_with(Default::default);
markdown.linter.merge_with(self.markdown_linter.clone());
}

if self.javascript_linter.is_some() {
let javascript = fs_configuration
.javascript
.get_or_insert_with(Default::default);
javascript.linter.merge_with(self.javascript_linter.clone());
}

if self.json_linter.is_some() {
let json = fs_configuration.json.get_or_insert_with(Default::default);
json.linter.merge_with(self.json_linter.clone());
Expand Down
7 changes: 6 additions & 1 deletion crates/biome_cli/src/commands/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use biome_configuration::formatter::FormatterEnabled;
use biome_configuration::graphql::{GraphqlFormatterConfiguration, GraphqlLinterConfiguration};
use biome_configuration::javascript::{JsFormatterConfiguration, JsLinterConfiguration};
use biome_configuration::json::{JsonFormatterConfiguration, JsonLinterConfiguration};
use biome_configuration::markdown::MarkdownLinterConfiguration;
use biome_configuration::vcs::VcsConfiguration;
use biome_configuration::{BiomeDiagnostic, Configuration};
use biome_configuration::{
Expand All @@ -20,7 +21,8 @@ use biome_configuration::{
formatter_configuration, graphql::graphql_formatter_configuration,
graphql::graphql_linter_configuration, javascript::js_formatter_configuration,
javascript::js_linter_configuration, json::json_formatter_configuration,
json::json_linter_configuration, linter_configuration, vcs::vcs_configuration,
json::json_linter_configuration, linter_configuration, markdown::markdown_linter_configuration,
vcs::vcs_configuration,
};
use biome_console::{Console, ConsoleExt, markup};
use biome_diagnostics::{Diagnostic, PrintDiagnostic, Severity};
Expand Down Expand Up @@ -208,6 +210,9 @@ pub enum BiomeCommand {
#[bpaf(external(graphql_linter_configuration), optional, hide_usage, hide)]
graphql_linter: Option<GraphqlLinterConfiguration>,

#[bpaf(external(markdown_linter_configuration), optional, hide_usage, hide)]
markdown_linter: Option<MarkdownLinterConfiguration>,

#[bpaf(external, hide_usage)]
cli_options: CliOptions,

Expand Down
2 changes: 2 additions & 0 deletions crates/biome_cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ impl<'app> CliSession<'app> {
javascript_linter,
json_linter,
graphql_linter,
markdown_linter,
} => run_command(
self,
&cli_options,
Expand All @@ -146,6 +147,7 @@ impl<'app> CliSession<'app> {
javascript_linter,
json_linter,
graphql_linter,
markdown_linter,
},
),
BiomeCommand::Ci {
Expand Down
7 changes: 7 additions & 0 deletions crates/biome_configuration/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ pub mod grit;
pub mod html;
pub mod javascript;
pub mod json;
pub mod markdown;
pub mod max_size;
mod overrides;
pub mod plugins;
Expand All @@ -30,6 +31,7 @@ use crate::graphql::{GraphqlFormatterConfiguration, GraphqlLinterConfiguration};
pub use crate::grit::{GritConfiguration, grit_configuration};
use crate::javascript::{JsFormatterConfiguration, JsLinterConfiguration};
use crate::json::{JsonFormatterConfiguration, JsonLinterConfiguration};
use crate::markdown::{MarkdownConfiguration, markdown_configuration};
use crate::max_size::MaxSize;
use crate::vcs::{VcsConfiguration, vcs_configuration};
pub use analyzer::{
Expand Down Expand Up @@ -151,6 +153,11 @@ pub struct Configuration {
#[serde(skip_serializing_if = "Option::is_none")]
pub html: Option<HtmlConfiguration>,

/// Specific configuration for the Markdown language
#[bpaf(external(markdown_configuration), optional)]
#[serde(skip_serializing_if = "Option::is_none")]
pub markdown: Option<MarkdownConfiguration>,

/// A list of granular patterns that should be applied only to a sub set of files
#[bpaf(hide, pure(Default::default()))]
#[serde(skip_serializing_if = "Option::is_none")]
Expand Down
108 changes: 108 additions & 0 deletions crates/biome_configuration/src/markdown.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
use crate::bool::Bool;
use biome_deserialize_macros::{Deserializable, Merge};
use biome_formatter::{
BracketSpacing, IndentStyle, IndentWidth, LineEnding, LineWidth, QuoteStyle,
};
use bpaf::Bpaf;
use serde::{Deserialize, Serialize};

/// Configuration for the Markdown language
#[derive(
Bpaf, Clone, Deserializable, Debug, Default, Deserialize, Eq, PartialEq, Merge, Serialize,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
pub struct MarkdownConfiguration {
/// Configuration for the Markdown formatter
#[bpaf(external(markdown_formatter_configuration), optional)]
#[serde(skip_serializing_if = "Option::is_none")]
pub formatter: Option<MarkdownFormatterConfiguration>,

/// Configuration for the Markdown linter
#[bpaf(external(markdown_linter_configuration), optional)]
#[serde(skip_serializing_if = "Option::is_none")]
pub linter: Option<MarkdownLinterConfiguration>,

/// Configuration for the Markdown parser
#[bpaf(external(markdown_parser_configuration), optional)]
#[serde(skip_serializing_if = "Option::is_none")]
pub parser: Option<MarkdownParserConfiguration>,
}

pub type MarkdownFormatterEnabled = Bool<true>;

/// Configuration for the Markdown formatter
#[derive(
Bpaf, Clone, Deserializable, Debug, Default, Deserialize, Eq, PartialEq, Merge, Serialize,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
pub struct MarkdownFormatterConfiguration {
/// Markdown formatter options
#[bpaf(long("markdown-formatter-enabled"), argument("true|false"))]
pub enabled: Option<MarkdownFormatterEnabled>,

/// The indent style applied to Markdown files.
#[bpaf(long("markdown-formatter-indent-style"), argument("tab|space"))]
pub indent_style: Option<IndentStyle>,

/// The size of the indentation applied to Markdown files. Default to 2.
#[bpaf(long("markdown-formatter-indent-width"), argument("NUMBER"))]
pub indent_width: Option<IndentWidth>,

/// The type of line ending applied to Markdown files.
#[bpaf(long("markdown-formatter-line-ending"), argument("lf|crlf|cr"))]
pub line_ending: Option<LineEnding>,

/// What's the max width of a line applied to Markdown files. Defaults to 80.
#[bpaf(long("markdown-formatter-line-width"), argument("NUMBER"))]
pub line_width: Option<LineWidth>,

/// The type of quotes used in Markdown code. Defaults to double.
#[bpaf(long("markdown-formatter-quote-style"), argument("double|single"))]
pub quote_style: Option<QuoteStyle>,

/// Whether to insert spaces around brackets in object literals. Defaults to true.
#[bpaf(long("bracket-spacing"), argument("true|false"))]
pub bracket_spacing: Option<BracketSpacing>,
}

impl MarkdownFormatterConfiguration {
pub fn is_enabled(&self) -> bool {
self.enabled.unwrap_or_default().into()
}
}

pub type MarkdownLinterEnabled = Bool<true>;

/// Configuration for the Markdown linter
#[derive(
Bpaf, Clone, Deserializable, Debug, Default, Deserialize, Eq, PartialEq, Merge, Serialize,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
pub struct MarkdownLinterConfiguration {
/// Control the linter for Markdown files.
#[bpaf(long("markdown-linter-enabled"), argument("true|false"))]
pub enabled: Option<MarkdownLinterEnabled>,
}

/// Configuration for the Markdown parser
#[derive(
Bpaf, Clone, Deserializable, Debug, Default, Deserialize, Eq, PartialEq, Merge, Serialize,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
pub struct MarkdownParserConfiguration {
/// Whether to parse GitHub Flavored Markdown extensions
#[bpaf(hide)]
#[serde(skip_serializing_if = "Option::is_none")]
pub gfm: Option<Bool<true>>,
}

#[test]
fn default_markdown_formatter() {
let markdown_configuration = MarkdownFormatterConfiguration::default();

assert!(markdown_configuration.is_enabled());
}
2 changes: 1 addition & 1 deletion crates/biome_markdown_factory/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ pub use crate::generated::MarkdownSyntaxFactory;
#[doc(hidden)]
pub use biome_markdown_syntax as syntax;

pub type DemoSyntaxTreeBuilder = TreeBuilder<'static, MarkdownLanguage, MarkdownSyntaxFactory>;
pub type MarkdownSyntaxTreeBuilder = TreeBuilder<'static, MarkdownLanguage, MarkdownSyntaxFactory>;

pub mod make;
122 changes: 122 additions & 0 deletions crates/biome_markdown_factory/src/make.rs
Original file line number Diff line number Diff line change
@@ -1 +1,123 @@
use biome_markdown_syntax::{MarkdownSyntaxKind, MarkdownSyntaxToken};

pub use crate::generated::node_factory::*;

/// Create a textual token
pub fn textual(text: &str) -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::MD_TEXTUAL_LITERAL, text, [], [])
}

/// Create a string token
pub fn string(text: &str) -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::MD_STRING_LITERAL, text, [], [])
}

/// Create a hash token for headers
pub fn hash() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::HASH, "#", [], [])
}

/// Create a backtick token
pub fn backtick() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::BACKTICK, "`", [], [])
}

/// Create a star token for emphasis
pub fn star() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::STAR, "*", [], [])
}

/// Create an underscore token for emphasis
pub fn underscore() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::UNDERSCORE, "_", [], [])
}

/// Create a left bracket token
pub fn l_brack() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::L_BRACK, "[", [], [])
}

/// Create a right bracket token
pub fn r_brack() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::R_BRACK, "]", [], [])
}

/// Create a left parenthesis token
pub fn l_paren() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::L_PAREN, "(", [], [])
}

/// Create a right parenthesis token
pub fn r_paren() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::R_PAREN, ")", [], [])
}

/// Create a bang token for images
pub fn bang() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::BANG, "!", [], [])
}

/// Create a minus token for thematic breaks
pub fn minus() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::MINUS, "-", [], [])
}

/// Create a thematic break token
pub fn thematic_break() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::MD_THEMATIC_BREAK_LITERAL, "---", [], [])
}

/// Create a newline token
pub fn newline() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::NEWLINE, "\n", [], [])
}

/// Create a whitespace token
pub fn whitespace(text: &str) -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::WHITESPACE, text, [], [])
}

/// Create a tab token
pub fn tab() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::TAB, "\t", [], [])
}

/// Create an indent chunk token for indented code blocks
pub fn indent_chunk() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::MD_INDENT_CHUNK_LITERAL, " ", [], [])
}

/// Create a hard line break token
pub fn hard_line_break() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::MD_HARD_LINE_LITERAL, " \n", [], [])
}

/// Create a greater than token for blockquotes
pub fn greater_than() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::R_ANGLE, ">", [], [])
}

/// Create a plus token for unordered lists
pub fn plus() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::PLUS, "+", [], [])
}

/// Create a digit token for ordered lists
pub fn digit(text: &str) -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::DIGIT, text, [], [])
}

/// Create a period token for ordered lists
pub fn period() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::PERIOD, ".", [], [])
}

/// Create a pipe token for tables
pub fn pipe() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::PIPE, "|", [], [])
}

/// Create a colon token for table alignment
pub fn colon() -> MarkdownSyntaxToken {
MarkdownSyntaxToken::new_detached(MarkdownSyntaxKind::COLON, ":", [], [])
}
Loading
Loading