Skip to content

Commit 2af3e2c

Browse files
RunDevelopmentRunDevelopment
authored
Markdown: Improved URL tokenization (#2678)
It now tokenizes all parts of a URL except for braces.
1 parent df0738e commit 2af3e2c

File tree

6 files changed

+123
-183
lines changed

6 files changed

+123
-183
lines changed

‎components/prism-markdown.js

+13-7
Original file line numberDiff line numberDiff line change
@@ -221,21 +221,27 @@
221221
// [example](http://example.com "Optional title")
222222
// [example][id]
223223
// [example] [id]
224-
pattern: createInline(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[(?:(?!\])<inner>)+\])/.source),
224+
pattern: createInline(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)|[ \t]?\[(?:(?!\])<inner>)+\])/.source),
225225
lookbehind: true,
226226
greedy: true,
227227
inside: {
228-
'variable': {
229-
pattern: /(\[)[^\]]+(?=\]$)/,
230-
lookbehind: true
231-
},
228+
'operator': /^!/,
232229
'content': {
233-
pattern: /(^!?\[)[^\]]+(?=\])/,
230+
pattern: /(^\[)[^\]]+(?=\])/,
234231
lookbehind: true,
235232
inside: {} // see below
236233
},
234+
'variable': {
235+
pattern: /(^\][ \t]?\[)[^\]]+(?=\]$)/,
236+
lookbehind: true
237+
},
238+
'url': {
239+
pattern: /(^\]\()[^\s)]+/,
240+
lookbehind: true
241+
},
237242
'string': {
238-
pattern: /"(?:\\.|[^"\\])*"(?=\)$)/
243+
pattern: /(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,
244+
lookbehind: true
239245
}
240246
}
241247
}

‎components/prism-markdown.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/languages/markdown/bold_feature.test

+23-44
Original file line numberDiff line numberDiff line change
@@ -24,41 +24,32 @@ not__bold__ __this__either
2424
[
2525
["bold", [
2626
["punctuation", "**"],
27-
["content", [
28-
"foobar"
29-
]],
27+
["content", ["foobar"]],
3028
["punctuation", "**"]
3129
]],
3230
["bold", [
3331
["punctuation", "**"],
34-
["content", [
35-
"foo\r\nbar"
36-
]],
32+
["content", ["foo\r\nbar"]],
3733
["punctuation", "**"]
3834
]],
3935
["bold", [
4036
["punctuation", "__"],
41-
["content", [
42-
"foobar"
43-
]],
37+
["content", ["foobar"]],
4438
["punctuation", "__"]
4539
]],
4640
["bold", [
4741
["punctuation", "__"],
48-
["content", [
49-
"foo\r\nbar"
50-
]],
42+
["content", ["foo\r\nbar"]],
5143
["punctuation", "__"]
5244
]],
45+
5346
["bold", [
5447
["punctuation", "__"],
5548
["content", [
5649
"foo ",
5750
["italic", [
5851
["punctuation", "*"],
59-
["content", [
60-
"bar"
61-
]],
52+
["content", ["bar"]],
6253
["punctuation", "*"]
6354
]],
6455
" baz"
@@ -71,9 +62,7 @@ not__bold__ __this__either
7162
"foo ",
7263
["italic", [
7364
["punctuation", "_"],
74-
["content", [
75-
"bar"
76-
]],
65+
["content", ["bar"]],
7766
["punctuation", "_"]
7867
]],
7968
" baz"
@@ -86,9 +75,7 @@ not__bold__ __this__either
8675
"foo ",
8776
["strike", [
8877
["punctuation", "~"],
89-
["content", [
90-
"bar"
91-
]],
78+
["content", ["bar"]],
9279
["punctuation", "~"]
9380
]],
9481
" baz"
@@ -101,9 +88,7 @@ not__bold__ __this__either
10188
"foo ",
10289
["strike", [
10390
["punctuation", "~~"],
104-
["content", [
105-
"bar"
106-
]],
91+
["content", ["bar"]],
10792
["punctuation", "~~"]
10893
]],
10994
" baz"
@@ -116,23 +101,22 @@ not__bold__ __this__either
116101
"foo",
117102
["url", [
118103
"[",
119-
["content", [
120-
"bar"
121-
]],
122-
"](baz)"
104+
["content", ["bar"]],
105+
"](",
106+
["url", "baz"],
107+
")"
123108
]]
124109
]],
125110
["punctuation", "__"]
126111
]],
112+
127113
["bold", [
128114
["punctuation", "**"],
129115
["content", [
130116
"foo ",
131117
["italic", [
132118
["punctuation", "*"],
133-
["content", [
134-
"bar"
135-
]],
119+
["content", ["bar"]],
136120
["punctuation", "*"]
137121
]],
138122
" baz"
@@ -145,9 +129,7 @@ not__bold__ __this__either
145129
"foo ",
146130
["italic", [
147131
["punctuation", "_"],
148-
["content", [
149-
"bar"
150-
]],
132+
["content", ["bar"]],
151133
["punctuation", "_"]
152134
]],
153135
" baz"
@@ -160,9 +142,7 @@ not__bold__ __this__either
160142
"foo ",
161143
["strike", [
162144
["punctuation", "~"],
163-
["content", [
164-
"bar"
165-
]],
145+
["content", ["bar"]],
166146
["punctuation", "~"]
167147
]],
168148
" baz"
@@ -175,9 +155,7 @@ not__bold__ __this__either
175155
"foo ",
176156
["strike", [
177157
["punctuation", "~~"],
178-
["content", [
179-
"bar"
180-
]],
158+
["content", ["bar"]],
181159
["punctuation", "~~"]
182160
]],
183161
" baz"
@@ -190,14 +168,15 @@ not__bold__ __this__either
190168
"foo",
191169
["url", [
192170
"[",
193-
["content", [
194-
"bar"
195-
]],
196-
"](baz)"
171+
["content", ["bar"]],
172+
"](",
173+
["url", "baz"],
174+
")"
197175
]]
198176
]],
199177
["punctuation", "**"]
200178
]],
179+
201180
"\r\n\r\nnot__bold__ __this__either"
202181
]
203182

‎tests/languages/markdown/italic_feature.test

+23-44
Original file line numberDiff line numberDiff line change
@@ -24,41 +24,32 @@ not_italic_ _this_either
2424
[
2525
["italic", [
2626
["punctuation", "*"],
27-
["content", [
28-
"foobar"
29-
]],
27+
["content", ["foobar"]],
3028
["punctuation", "*"]
3129
]],
3230
["italic", [
3331
["punctuation", "*"],
34-
["content", [
35-
"foo\r\nbar"
36-
]],
32+
["content", ["foo\r\nbar"]],
3733
["punctuation", "*"]
3834
]],
3935
["italic", [
4036
["punctuation", "_"],
41-
["content", [
42-
"foobar"
43-
]],
37+
["content", ["foobar"]],
4438
["punctuation", "_"]
4539
]],
4640
["italic", [
4741
["punctuation", "_"],
48-
["content", [
49-
"foo\r\nbar"
50-
]],
42+
["content", ["foo\r\nbar"]],
5143
["punctuation", "_"]
5244
]],
45+
5346
["italic", [
5447
["punctuation", "_"],
5548
["content", [
5649
"foo ",
5750
["bold", [
5851
["punctuation", "__"],
59-
["content", [
60-
"bar"
61-
]],
52+
["content", ["bar"]],
6253
["punctuation", "__"]
6354
]],
6455
" baz"
@@ -71,9 +62,7 @@ not_italic_ _this_either
7162
"foo ",
7263
["bold", [
7364
["punctuation", "**"],
74-
["content", [
75-
"bar"
76-
]],
65+
["content", ["bar"]],
7766
["punctuation", "**"]
7867
]],
7968
" baz"
@@ -86,9 +75,7 @@ not_italic_ _this_either
8675
"foo ",
8776
["strike", [
8877
["punctuation", "~"],
89-
["content", [
90-
"bar"
91-
]],
78+
["content", ["bar"]],
9279
["punctuation", "~"]
9380
]],
9481
" baz"
@@ -101,9 +88,7 @@ not_italic_ _this_either
10188
"foo ",
10289
["strike", [
10390
["punctuation", "~~"],
104-
["content", [
105-
"bar"
106-
]],
91+
["content", ["bar"]],
10792
["punctuation", "~~"]
10893
]],
10994
" baz"
@@ -116,23 +101,22 @@ not_italic_ _this_either
116101
"foo",
117102
["url", [
118103
"[",
119-
["content", [
120-
"bar"
121-
]],
122-
"](baz)"
104+
["content", ["bar"]],
105+
"](",
106+
["url", "baz"],
107+
")"
123108
]]
124109
]],
125110
["punctuation", "_"]
126111
]],
112+
127113
["italic", [
128114
["punctuation", "*"],
129115
["content", [
130116
"foo ",
131117
["bold", [
132118
["punctuation", "__"],
133-
["content", [
134-
"bar"
135-
]],
119+
["content", ["bar"]],
136120
["punctuation", "__"]
137121
]],
138122
" baz"
@@ -145,9 +129,7 @@ not_italic_ _this_either
145129
"foo ",
146130
["bold", [
147131
["punctuation", "**"],
148-
["content", [
149-
"bar"
150-
]],
132+
["content", ["bar"]],
151133
["punctuation", "**"]
152134
]],
153135
" baz"
@@ -160,9 +142,7 @@ not_italic_ _this_either
160142
"foo ",
161143
["strike", [
162144
["punctuation", "~"],
163-
["content", [
164-
"bar"
165-
]],
145+
["content", ["bar"]],
166146
["punctuation", "~"]
167147
]],
168148
" baz"
@@ -175,9 +155,7 @@ not_italic_ _this_either
175155
"foo ",
176156
["strike", [
177157
["punctuation", "~~"],
178-
["content", [
179-
"bar"
180-
]],
158+
["content", ["bar"]],
181159
["punctuation", "~~"]
182160
]],
183161
" baz"
@@ -190,14 +168,15 @@ not_italic_ _this_either
190168
"foo",
191169
["url", [
192170
"[",
193-
["content", [
194-
"bar"
195-
]],
196-
"](baz)"
171+
["content", ["bar"]],
172+
"](",
173+
["url", "baz"],
174+
")"
197175
]]
198176
]],
199177
["punctuation", "*"]
200178
]],
179+
201180
"\r\n\r\nnot_italic_ _this_either"
202181
]
203182

0 commit comments

Comments
 (0)