/
githubmirror
/
marked
Обзор
Документация
Войти
/
githubmirror
/
marked
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
test/unit/Lexer.test.js
2 161 строка
55 KB
Andrew Stelmach
fix: cache list indentation regexes (#3969)
19 май 2026, 08:13
Не верифицирован
19 май 2026, 08:13
a37983f
Код
Авторство
О чём код?
import { Lexer } from '../../lib/marked.esm.js'; import { describe, it } from 'node:test'; import assert from 'node:assert'; function expectTokens({ md, options, tokens = [], links = {}, log = false }) { const lexer = new Lexer(options); const actual = lexer.lex(md); const expected = tokens; expected.links = links; if (log) { console.log(JSON.stringify( actual, (k, v) => v === undefined ? null : v, 2, )); } assert.deepEqual(actual, expected); } function expectInlineTokens({ md, options, tokens, links = {} }) { const lexer = new Lexer(options); lexer.tokens.links = links; const outTokens = []; lexer.inlineTokens(md, outTokens); assert.deepEqual(outTokens, tokens); } describe('Lexer', () => { describe('rules', () => { it('should create indentation regexes for zero indent', () => { const { nextBulletRegex, hrRegex, fencesBeginRegex, headingBeginRegex, htmlBeginRegex, blockquoteBeginRegex, } = new Lexer().tokenizer.rules.other; assert.ok(nextBulletRegex(0).test('- item\n')); assert.ok(hrRegex(0).test('---\n')); assert.ok(fencesBeginRegex(0).test('```')); assert.ok(headingBeginRegex(0).test('# heading')); assert.ok(htmlBeginRegex(0).test('<div>')); assert.ok(blockquoteBeginRegex(0).test('> quote')); }); }); describe('paragraph', () => { it('space between paragraphs', () => { expectTokens({ md: 'paragraph 1\n\nparagraph 2', tokens: [ { type: 'paragraph', raw: 'paragraph 1', text: 'paragraph 1', tokens: [{ type: 'text', raw: 'paragraph 1', text: 'paragraph 1', escaped: false }], }, { type: 'space', raw: '\n\n' }, { type: 'paragraph', raw: 'paragraph 2', text: 'paragraph 2', tokens: [{ type: 'text', raw: 'paragraph 2', text: 'paragraph 2', escaped: false }], }, ], }); }); }); describe('code', () => { it('indented code', () => { expectTokens({ md: ' code', tokens: [ { type: 'code', raw: ' code', text: 'code', codeBlockStyle: 'indented' }, ], }); }); it('indented code with trailing blank lines', () => { expectTokens({ md: ' code\n \n\t\t\n \t\n\nnext', tokens: [ { type: 'code', raw: ' code', text: 'code', codeBlockStyle: 'indented' }, { type: 'space', raw: '\n \n\t\t\n \t\n\n' }, { type: 'paragraph', raw: 'next', text: 'next', tokens: [{ type: 'text', raw: 'next', text: 'next', escaped: false }], }, ], }); }); it('fenced code', () => { expectTokens({ md: '```\ncode\n```', tokens: [ { type: 'code', raw: '```\ncode\n```', text: 'code', lang: '' }, ], }); }); it('fenced code lang', () => { expectTokens({ md: '```text\ncode\n```', tokens: [ { type: 'code', raw: '```text\ncode\n```', text: 'code', lang: 'text' }, ], }); }); }); describe('headings', () => { it('depth', () => { expectTokens({ md: ` # heading 1 ## heading 2 ### heading 3 #### heading 4 ##### heading 5 ###### heading 6 lheading 1 ========== lheading 2 ---------- `, tokens: [ { type: 'space', raw: '\n', }, { type: 'heading', raw: '# heading 1', depth: 1, text: 'heading 1', tokens: [{ type: 'text', raw: 'heading 1', text: 'heading 1', escaped: false }], }, { type: 'space', raw: '\n\n', }, { type: 'heading', raw: '## heading 2', depth: 2, text: 'heading 2', tokens: [{ type: 'text', raw: 'heading 2', text: 'heading 2', escaped: false }], }, { type: 'space', raw: '\n\n', }, { type: 'heading', raw: '### heading 3', depth: 3, text: 'heading 3', tokens: [{ type: 'text', raw: 'heading 3', text: 'heading 3', escaped: false }], }, { type: 'space', raw: '\n\n', }, { type: 'heading', raw: '#### heading 4', depth: 4, text: 'heading 4', tokens: [{ type: 'text', raw: 'heading 4', text: 'heading 4', escaped: false }], }, { type: 'space', raw: '\n\n', }, { type: 'heading', raw: '##### heading 5', depth: 5, text: 'heading 5', tokens: [{ type: 'text', raw: 'heading 5', text: 'heading 5', escaped: false }], }, { type: 'space', raw: '\n\n', }, { type: 'heading', raw: '###### heading 6', depth: 6, text: 'heading 6', tokens: [{ type: 'text', raw: 'heading 6', text: 'heading 6', escaped: false }], }, { type: 'space', raw: '\n\n', }, { type: 'heading', raw: 'lheading 1\n==========', depth: 1, text: 'lheading 1', tokens: [{ type: 'text', raw: 'lheading 1', text: 'lheading 1', escaped: false }], }, { type: 'space', raw: '\n\n', }, { type: 'heading', raw: 'lheading 2\n----------\n', depth: 2, text: 'lheading 2', tokens: [{ type: 'text', raw: 'lheading 2', text: 'lheading 2', escaped: false }], }, ], }); }); it('lheading should trim trailing whitespace from text', () => { expectTokens({ md: 'heading with trailing spaces \n==============================', tokens: [{ type: 'heading', raw: 'heading with trailing spaces \n==============================', depth: 1, text: 'heading with trailing spaces', tokens: [{ type: 'text', raw: 'heading with trailing spaces', text: 'heading with trailing spaces', escaped: false }], }], }); }); it('should not be heading if depth > 6', () => { expectTokens({ md: '####### heading 7', tokens: [{ type: 'paragraph', raw: '####### heading 7', text: '####### heading 7', tokens: [{ type: 'text', raw: '####### heading 7', text: '####### heading 7', escaped: false }], }], }); }); it('should strip trailing hashes with preceding space', () => { expectTokens({ md: '## heading ##', tokens: [{ type: 'heading', raw: '## heading ##', depth: 2, text: 'heading', tokens: [{ type: 'text', raw: 'heading', text: 'heading', escaped: false }], }], }); }); it('should not strip trailing hashes without preceding space', () => { expectTokens({ md: '## heading##', tokens: [{ type: 'heading', raw: '## heading##', depth: 2, text: 'heading##', tokens: [{ type: 'text', raw: 'heading##', text: 'heading##', escaped: false }], }], }); }); it('should tokenize inline content inside heading', () => { expectTokens({ md: '## **bold** and _em_', tokens: [{ type: 'heading', raw: '## **bold** and _em_', depth: 2, text: '**bold** and _em_', tokens: [ { type: 'strong', raw: '**bold**', text: 'bold', tokens: [{ type: 'text', raw: 'bold', text: 'bold', escaped: false }], }, { type: 'text', raw: ' and ', text: ' and ', escaped: false }, { type: 'em', raw: '_em_', text: 'em', tokens: [{ type: 'text', raw: 'em', text: 'em', escaped: false }], }, ], }], }); }); it('heading immediately followed by paragraph without blank line', () => { expectTokens({ md: '# heading\nparagraph', tokens: [ { type: 'heading', raw: '# heading\n', depth: 1, text: 'heading', tokens: [{ type: 'text', raw: 'heading', text: 'heading', escaped: false }], }, { type: 'paragraph', raw: 'paragraph', text: 'paragraph', tokens: [{ type: 'text', raw: 'paragraph', text: 'paragraph', escaped: false }], }, ], }); }); }); describe('table', () => { it('pipe table', () => { expectTokens({ md: ` | a | b | |---|---| | 1 | 2 | `, tokens: [{ type: 'space', raw: '\n', }, { type: 'table', align: [null, null], raw: '| a | b |\n|---|---|\n| 1 | 2 |\n', header: [ { text: 'a', tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }], header: true, align: null, }, { text: 'b', tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }], header: true, align: null, }, ], rows: [ [ { text: '1', tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }], header: false, align: null, }, { text: '2', tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }], header: false, align: null, }, ], ], }], }); }); it('table after para', () => { expectTokens({ md: ` paragraph 1 | a | b | |---|---| | 1 | 2 | `, tokens: [{ type: 'space', raw: '\n', }, { type: 'paragraph', raw: 'paragraph 1\n', text: 'paragraph 1', tokens: [{ type: 'text', raw: 'paragraph 1', text: 'paragraph 1', escaped: false }], }, { type: 'table', align: [null, null], raw: '| a | b |\n|---|---|\n| 1 | 2 |\n', header: [ { text: 'a', tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }], header: true, align: null, }, { text: 'b', tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }], header: true, align: null, }, ], rows: [ [ { text: '1', tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }], header: false, align: null, }, { text: '2', tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }], header: false, align: null, }, ], ], }, ], }); }); it('align table', () => { expectTokens({ md: ` | a | b | c | |:--|:-:|--:| | 1 | 2 | 3 | `, tokens: [{ type: 'space', raw: '\n', }, { type: 'table', align: ['left', 'center', 'right'], raw: '| a | b | c |\n|:--|:-:|--:|\n| 1 | 2 | 3 |\n', header: [ { text: 'a', tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }], header: true, align: 'left', }, { text: 'b', tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }], header: true, align: 'center', }, { text: 'c', tokens: [{ type: 'text', raw: 'c', text: 'c', escaped: false }], header: true, align: 'right', }, ], rows: [ [ { text: '1', tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }], header: false, align: 'left', }, { text: '2', tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }], header: false, align: 'center', }, { text: '3', tokens: [{ type: 'text', raw: '3', text: '3', escaped: false }], header: false, align: 'right', }, ], ], }], }); }); it('no pipe table', () => { expectTokens({ md: ` a | b --|-- 1 | 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'table', align: [null, null], raw: 'a | b\n--|--\n1 | 2\n', header: [ { text: 'a', tokens: [{ type: 'text', raw: 'a', text: 'a', escaped: false }], header: true, align: null, }, { text: 'b', tokens: [{ type: 'text', raw: 'b', text: 'b', escaped: false }], header: true, align: null, }, ], rows: [ [ { text: '1', tokens: [{ type: 'text', raw: '1', text: '1', escaped: false }], header: false, align: null, }, { text: '2', tokens: [{ type: 'text', raw: '2', text: '2', escaped: false }], header: false, align: null, }, ], ], }], }); }); }); describe('hr', () => { it('hr', () => { expectTokens({ md: '---', tokens: [ { type: 'hr', raw: '---' }, ], }); }); }); describe('blockquote', () => { it('start, inner-tokens, end', () => { expectTokens({ md: '> blockquote', tokens: [ { type: 'blockquote', raw: '> blockquote', text: 'blockquote', tokens: [{ type: 'paragraph', raw: 'blockquote', text: 'blockquote', tokens: [ { type: 'text', raw: 'blockquote', text: 'blockquote', escaped: false }, ], }], }, ], }); }); it('trim newline in text', () => { expectTokens({ md: '> blockquote\n', tokens: [ { type: 'blockquote', raw: '> blockquote\n', text: 'blockquote', tokens: [{ type: 'paragraph', raw: 'blockquote', text: 'blockquote', tokens: [ { type: 'text', raw: 'blockquote', text: 'blockquote', escaped: false }, ], }], }, ], }); }); it('paragraph token in list', () => { expectTokens({ md: '- > blockquote', tokens: [ { type: 'list', raw: '- > blockquote', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- > blockquote', task: false, loose: false, text: '> blockquote', tokens: [ { type: 'blockquote', raw: '> blockquote', tokens: [ { type: 'paragraph', raw: 'blockquote', text: 'blockquote', tokens: [ { type: 'text', raw: 'blockquote', text: 'blockquote', escaped: false }, ], }, ], text: 'blockquote', }, ], }, ], }, ], }); }); }); describe('list', () => { it('unordered', () => { expectTokens({ md: ` - item 1 - item 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '- item 1\n- item 2\n', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- item 1\n', task: false, loose: false, text: 'item 1', tokens: [{ type: 'text', raw: 'item 1', text: 'item 1', tokens: [{ type: 'text', raw: 'item 1', text: 'item 1', escaped: false }], }], }, { type: 'list_item', raw: '- item 2', task: false, loose: false, text: 'item 2', tokens: [{ type: 'text', raw: 'item 2', text: 'item 2', tokens: [{ type: 'text', raw: 'item 2', text: 'item 2', escaped: false }], }], }, ], }, ], }); }); it('ordered', () => { expectTokens({ md: ` 1. item 1 2. item 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '1. item 1\n2. item 2\n', ordered: true, start: 1, loose: false, items: [ { type: 'list_item', raw: '1. item 1\n', task: false, loose: false, text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '2. item 2', task: false, loose: false, text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, ], }, ], }, ], }); }); it('ordered with parenthesis', () => { expectTokens({ md: ` 1) item 1 2) item 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '1) item 1\n2) item 2\n', ordered: true, start: 1, loose: false, items: [ { type: 'list_item', raw: '1) item 1\n', task: false, loose: false, text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '2) item 2', task: false, loose: false, text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, ], }, ], }, ], }); }); it('space after list', () => { expectTokens({ md: ` - item 1 - item 2 paragraph `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '- item 1\n- item 2', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- item 1\n', task: false, loose: false, text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '- item 2', task: false, loose: false, text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, ], }, ], }, { type: 'space', raw: '\n\n', }, { type: 'paragraph', raw: 'paragraph\n', text: 'paragraph', tokens: [ { type: 'text', raw: 'paragraph', text: 'paragraph', escaped: false, }, ], }, ], }); }); it('start', () => { expectTokens({ md: ` 2. item 1 3. item 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '2. item 1\n3. item 2\n', ordered: true, start: 2, loose: false, items: [ { type: 'list_item', raw: '2. item 1\n', task: false, loose: false, text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '3. item 2', task: false, loose: false, text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, ], }, ], }, ], }); }); it('loose', () => { expectTokens({ md: ` - item 1 - item 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '- item 1\n\n- item 2\n', ordered: false, start: '', loose: true, items: [ { type: 'list_item', raw: '- item 1\n\n', task: false, loose: true, text: 'item 1\n', tokens: [ { type: 'paragraph', raw: 'item 1\n', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '- item 2', task: false, loose: true, text: 'item 2', tokens: [ { type: 'paragraph', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, ], }, ], }, ], }); }); it('end loose', () => { expectTokens({ md: ` - item 1 - item 2 item 2a - item 3 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '- item 1\n- item 2\n\n item 2a\n- item 3\n', ordered: false, start: '', loose: true, items: [ { type: 'list_item', raw: '- item 1\n', task: false, loose: true, text: 'item 1', tokens: [ { type: 'paragraph', raw: 'item 1', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '- item 2\n\n item 2a\n', task: false, loose: true, text: 'item 2\n\nitem 2a', tokens: [ { type: 'paragraph', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, { type: 'space', raw: '\n\n', }, { type: 'paragraph', raw: 'item 2a', text: 'item 2a', tokens: [ { type: 'text', raw: 'item 2a', text: 'item 2a', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '- item 3', task: false, loose: true, text: 'item 3', tokens: [ { type: 'paragraph', raw: 'item 3', text: 'item 3', tokens: [ { type: 'text', raw: 'item 3', text: 'item 3', escaped: false, }, ], }, ], }, ], }, ], }); }); it('not loose with spaces', () => { expectTokens({ md: ` - item 1 - item 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '- item 1\n - item 2\n', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- item 1\n - item 2', task: false, loose: false, text: 'item 1\n- item 2', tokens: [ { type: 'text', raw: 'item 1\n', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, { type: 'list', raw: '- item 2', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- item 2', task: false, loose: false, text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, ], }, ], }, ], }, ], }, ], }); }); it('task', () => { expectTokens({ md: ` - [ ] item 1 - [x] item 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '- [ ] item 1\n- [x] item 2\n', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- [ ] item 1\n', task: true, checked: false, loose: false, text: 'item 1', tokens: [ { type: 'checkbox', raw: '[ ] ', checked: false, }, { type: 'text', raw: 'item 1', text: 'item 1', tokens: [ { type: 'text', raw: 'item 1', text: 'item 1', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '- [x] item 2', task: true, checked: true, loose: false, text: 'item 2', tokens: [ { type: 'checkbox', raw: '[x] ', checked: true, }, { type: 'text', raw: 'item 2', text: 'item 2', tokens: [ { type: 'text', raw: 'item 2', text: 'item 2', escaped: false, }, ], }, ], }, ], }, ], }); }); it('does not parse setext heading text as a task checkbox', () => { expectTokens({ md: '- [x] title\n ---', tokens: [ { type: 'list', raw: '- [x] title\n ---', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- [x] title\n ---', task: false, loose: false, text: '[x] title\n---', tokens: [ { type: 'heading', raw: '[x] title\n---', depth: 2, text: '[x] title', tokens: [ { type: 'text', raw: '[x] title', text: '[x] title', }, ], }, ], }, ], }, ], }); }); it('keeps setext heading task candidates loose when separated by blank lines', () => { expectTokens({ md: '- [x] title\n ---\n\n body\n- second', tokens: [ { type: 'list', raw: '- [x] title\n ---\n\n body\n- second', ordered: false, start: '', loose: true, items: [ { type: 'list_item', raw: '- [x] title\n ---\n\n body\n', task: false, loose: true, text: '[x] title\n---\n\nbody', tokens: [ { type: 'heading', raw: '[x] title\n---', depth: 2, text: '[x] title', tokens: [ { type: 'text', raw: '[x] title', text: '[x] title', }, ], }, { type: 'space', raw: '\n\n', }, { type: 'paragraph', raw: 'body', text: 'body', tokens: [ { type: 'text', raw: 'body', text: 'body', escaped: false, }, ], }, ], }, { type: 'list_item', raw: '- second', task: false, loose: true, text: 'second', tokens: [ { type: 'paragraph', raw: 'second', text: 'second', tokens: [ { type: 'text', raw: 'second', text: 'second', escaped: false, }, ], }, ], }, ], }, ], }); }); it('multiline', () => { expectTokens({ md: ` - line 1 line 2 `, tokens: [ { type: 'space', raw: '\n', }, { type: 'list', raw: '- line 1\n line 2\n', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- line 1\n line 2', task: false, loose: false, text: 'line 1\nline 2', tokens: [{ type: 'text', raw: 'line 1\nline 2', text: 'line 1\nline 2', tokens: [{ type: 'text', raw: 'line 1\nline 2', text: 'line 1\nline 2', escaped: false }], }], }, ], }, ], }); }); it('indented code after paragraph', () => { expectTokens({ md: ` - a - b `, tokens: [{ type: 'space', raw: '\n', }, { type: 'list', raw: '- a\n - b\n', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- a\n - b', task: false, loose: false, text: 'a\n - b', tokens: [ { type: 'text', raw: 'a\n - b', text: 'a\n- b', tokens: [ { type: 'text', raw: 'a\n- b', text: 'a\n- b', escaped: false }, ], }, ], }, ], }, ], }); }); it('def after paragraph', () => { expectTokens({ md: ` - hello [1]: hello `, tokens: [ { type: 'space', raw: '\n' }, { type: 'list', raw: '- hello\n[1]: hello\n', ordered: false, start: '', loose: false, items: [ { type: 'list_item', raw: '- hello\n[1]: hello', task: false, loose: false, text: 'hello\n[1]: hello', tokens: [ { type: 'text', raw: 'hello\n[1]: hello', text: 'hello\n[1]: hello', tokens: [ { type: 'text', raw: 'hello\n[1]: hello', text: 'hello\n[1]: hello', escaped: false }, ], }, ], }, ], }, ], }); }); }); describe('html', () => { it('div', () => { expectTokens({ md: '<div>html</div>', tokens: [ { type: 'html', raw: '<div>html</div>', pre: false, block: true, text: '<div>html</div>', }, ], }); }); it('pre', () => { expectTokens({ md: '<pre>html</pre>', tokens: [ { type: 'html', raw: '<pre>html</pre>', pre: true, block: true, text: '<pre>html</pre>', }, ], }); }); it('html with trailing blank lines', () => { expectTokens({ md: '<div>\n html\n</div>\n \n\t\n\nnext', tokens: [ { type: 'html', raw: '<div>\n html\n</div>', pre: false, block: true, text: '<div>\n html\n</div>', }, { type: 'space', raw: '\n \n\t\n\n', }, { type: 'paragraph', raw: 'next', text: 'next', tokens: [{ type: 'text', raw: 'next', text: 'next', escaped: false }], }, ], }); }); }); describe('def', () => { it('link', () => { expectTokens({ md: '[link]: https://example.com', links: { link: { href: 'https://example.com', title: undefined }, }, tokens: [ { type: 'def', raw: '[link]: https://example.com', tag: 'link', href: 'https://example.com', title: undefined, }, ], }); }); it('title', () => { expectTokens({ md: '[link]: https://example.com "title"', links: { link: { href: 'https://example.com', title: 'title' }, }, tokens: [ { type: 'def', raw: '[link]: https://example.com "title"', tag: 'link', href: 'https://example.com', title: 'title', }, ], }); }); }); describe('inline', () => { describe('inlineTokens', () => { it('escape', () => { expectInlineTokens({ md: '\\>', tokens: [ { type: 'escape', raw: '\\>', text: '>' }, ], }); }); it('escaped punctuation inside emphasis', () => { expectInlineTokens({ md: '**strong text\\[**\\]', tokens: [ { type: 'strong', raw: '**strong text\\[**', text: 'strong text\\[', tokens: [ { type: 'text', raw: 'strong text', text: 'strong text', escaped: false }, { type: 'escape', raw: '\\[', text: '[' }, ], }, { type: 'escape', raw: '\\]', text: ']' }, ], }); expectInlineTokens({ md: '_em\\<pha\\>sis_', tokens: [ { type: 'em', raw: '_em\\<pha\\>sis_', text: 'em\\<pha\\>sis', tokens: [ { type: 'text', raw: 'em', text: 'em', escaped: false }, { type: 'escape', raw: '\\<', text: '<' }, { type: 'text', raw: 'pha', text: 'pha', escaped: false }, { type: 'escape', raw: '\\>', text: '>' }, { type: 'text', raw: 'sis', text: 'sis', escaped: false }, ], }, ], }); }); it('html', () => { expectInlineTokens({ md: '<div>html</div>', tokens: [ { type: 'html', raw: '<div>', inLink: false, inRawBlock: false, block: false, text: '<div>' }, { type: 'text', raw: 'html', text: 'html', escaped: false }, { type: 'html', raw: '</div>', inLink: false, inRawBlock: false, block: false, text: '</div>' }, ], }); }); it('link', () => { expectInlineTokens({ md: '[link](https://example.com)', tokens: [ { type: 'link', raw: '[link](https://example.com)', href: 'https://example.com', title: null, text: 'link', tokens: [ { type: 'text', raw: 'link', text: 'link', escaped: false, }, ], }, ], }); }); it('link title', () => { expectInlineTokens({ md: '[link](https://example.com "title")', tokens: [ { type: 'link', raw: '[link](https://example.com "title")', href: 'https://example.com', title: 'title', text: 'link', tokens: [ { type: 'text', raw: 'link', text: 'link', escaped: false, }, ], }, ], }); }); it('image', () => { expectInlineTokens({ md: '', tokens: [ { type: 'image', raw: '', text: 'image', href: 'https://example.com/image.png', title: null, tokens: [{ type: 'text', raw: 'image', text: 'image', escaped: false, }], }, ], }); }); it('image title', () => { expectInlineTokens({ md: '', tokens: [ { type: 'image', raw: '', text: 'image', href: 'https://example.com/image.png', title: 'title', tokens: [{ type: 'text', raw: 'image', text: 'image', escaped: false, }], }, ], }); }); describe('reflink', () => { it('reflink', () => { expectInlineTokens({ md: '[link][]', links: { link: { href: 'https://example.com', title: 'title' }, }, tokens: [ { type: 'link', raw: '[link][]', href: 'https://example.com', title: 'title', text: 'link', tokens: [{ type: 'text', raw: 'link', text: 'link', escaped: false, }], }, ], }); }); it('nolink', () => { expectInlineTokens({ md: '[link]', links: { link: { href: 'https://example.com', title: 'title' }, }, tokens: [ { type: 'link', raw: '[link]', href: 'https://example.com', title: 'title', text: 'link', tokens: [{ type: 'text', raw: 'link', text: 'link', escaped: false, }], }, ], }); }); it('no def', () => { expectInlineTokens({ md: '[link]', tokens: [ { type: 'text', raw: '[link]', text: '[link]', }, ], }); }); }); it('strong', () => { expectInlineTokens({ md: '**strong**', tokens: [ { type: 'strong', raw: '**strong**', text: 'strong', tokens: [ { type: 'text', raw: 'strong', text: 'strong', escaped: false, }, ], }, ], }); }); it('em', () => { expectInlineTokens({ md: '*em*', tokens: [ { type: 'em', raw: '*em*', text: 'em', tokens: [ { type: 'text', raw: 'em', text: 'em', escaped: false, }, ], }, ], }); }); describe('codespan', () => { it('code', () => { expectInlineTokens({ md: '`code`', tokens: [ { type: 'codespan', raw: '`code`', text: 'code' }, ], }); }); it('only spaces not stripped', () => { expectInlineTokens({ md: '` `', tokens: [ { type: 'codespan', raw: '` `', text: ' ' }, ], }); }); it('beginning space only not stripped', () => { expectInlineTokens({ md: '` a`', tokens: [ { type: 'codespan', raw: '` a`', text: ' a' }, ], }); }); it('end space only not stripped', () => { expectInlineTokens({ md: '`a `', tokens: [ { type: 'codespan', raw: '`a `', text: 'a ' }, ], }); }); it('begin and end spaces are stripped', () => { expectInlineTokens({ md: '` a `', tokens: [ { type: 'codespan', raw: '` a `', text: 'a' }, ], }); }); it('begin and end newlines are stripped', () => { expectInlineTokens({ md: '`\na\n`', tokens: [ { type: 'codespan', raw: '`\na\n`', text: 'a' }, ], }); }); it('begin and end tabs are not stripped', () => { expectInlineTokens({ md: '`\ta\t`', tokens: [ { type: 'codespan', raw: '`\ta\t`', text: '\ta\t' }, ], }); }); it('begin and end newlines', () => { expectInlineTokens({ md: '`\na\n`', tokens: [ { type: 'codespan', raw: '`\na\n`', text: 'a' }, ], }); }); it('begin and end multiple spaces only one stripped', () => { expectInlineTokens({ md: '` a `', tokens: [ { type: 'codespan', raw: '` a `', text: ' a ' }, ], }); }); it('newline to space', () => { expectInlineTokens({ md: '`a\nb`', tokens: [ { type: 'codespan', raw: '`a\nb`', text: 'a b' }, ], }); }); }); it('br', () => { expectInlineTokens({ md: 'a\nb', options: { gfm: true, breaks: true }, tokens: [ { raw: 'a', text: 'a', type: 'text', escaped: false, }, { raw: '\n', type: 'br', }, { raw: 'b', text: 'b', type: 'text', escaped: false, }, ], }); }); it('del', () => { expectInlineTokens({ md: '~~del~~', tokens: [ { type: 'del', raw: '~~del~~', text: 'del', tokens: [ { type: 'text', raw: 'del', text: 'del', escaped: false, }, ], }, ], }); }); describe('url', () => { it('autolink', () => { expectInlineTokens({ md: '<https://example.com>', tokens: [ { type: 'link', raw: '<https://example.com>', text: 'https://example.com', href: 'https://example.com', tokens: [ { type: 'text', raw: 'https://example.com', text: 'https://example.com', }, ], }, ], }); }); it('autolink email', () => { expectInlineTokens({ md: '<test@example.com>', options: {}, tokens: [ { type: 'link', raw: '<test@example.com>', text: 'test@example.com', href: 'mailto:test@example.com', tokens: [ { type: 'text', raw: 'test@example.com', text: 'test@example.com', }, ], }, ], }); }); it('url', () => { expectInlineTokens({ md: 'https://example.com', tokens: [ { type: 'link', raw: 'https://example.com', text: 'https://example.com', href: 'https://example.com', tokens: [ { type: 'text', raw: 'https://example.com', text: 'https://example.com', }, ], }, ], }); }); it('url email', () => { expectInlineTokens({ md: 'test@example.com', options: { gfm: true }, tokens: [ { type: 'link', raw: 'test@example.com', text: 'test@example.com', href: 'mailto:test@example.com', tokens: [ { type: 'text', raw: 'test@example.com', text: 'test@example.com', }, ], }, ], }); }); }); it('text', () => { expectInlineTokens({ md: 'text', tokens: [ { type: 'text', raw: 'text', text: 'text', escaped: false, }, ], }); }); }); }); describe('multiple instances', () => { it('should produce identical tokens when multiple Lexer instances lex the same input (issue #3854)', () => { // Create the first instance and lex — this may mutate shared _defaults.tokenizer.lexer const lexer1 = new Lexer(); const tokensA = lexer1.lex('Test'); // Create a second instance after the first — previously tokensB.tokens would be empty // because lexer1's constructor overwrote _defaults.tokenizer.lexer to point to lexer1 const lexer2 = new Lexer(); const tokensB = lexer2.lex('Test'); assert.deepEqual(tokensA, tokensB); }); it('should produce identical tokens using static lex and instance lex (issue #3854)', () => { const lexer = new Lexer(); const tokensA = Lexer.lex('Test'); const tokensB = lexer.lex('Test'); assert.deepEqual(tokensA, tokensB); }); }); });