import { StackElementMetadata, INITIAL, Registry as Registry$1, Theme } from './textmate.mjs'; import { FontStyle } from './types.mjs'; function toArray(x) { return Array.isArray(x) ? x : [x]; } /** * Slipt a string into lines, each line preserves the line ending. */ function splitLines(code, preserveEnding = false) { const parts = code.split(/(\r?\n)/g); let index = 0; const lines = []; for (let i = 0; i < parts.length; i += 2) { const line = preserveEnding ? parts[i] + (parts[i + 1] || '') : parts[i]; lines.push([line, index]); index += parts[i].length; index += parts[i + 1]?.length || 0; } return lines; } /** * Check if the language is plaintext that is ignored by Shiki. * * Hard-coded plain text languages: `plaintext`, `txt`, `text`, `plain`. */ function isPlainLang(lang) { return !lang || ['plaintext', 'txt', 'text', 'plain'].includes(lang); } /** * Check if the language is specially handled or bypassed by Shiki. * * Hard-coded languages: `ansi` and plaintexts like `plaintext`, `txt`, `text`, `plain`. */ function isSpecialLang(lang) { return lang === 'ansi' || isPlainLang(lang); } /** * Check if the theme is specially handled or bypassed by Shiki. * * Hard-coded themes: `none`. */ function isNoneTheme(theme) { return theme === 'none'; } /** * Check if the theme is specially handled or bypassed by Shiki. * * Hard-coded themes: `none`. */ function isSpecialTheme(theme) { return isNoneTheme(theme); } /** * Utility to append class to a hast node * * If the `property.class` is a string, it will be splitted by space and converted to an array. */ function addClassToHast(node, className) { if (!className) return node; node.properties ||= {}; node.properties.class ||= []; if (typeof node.properties.class === 'string') node.properties.class = node.properties.class.split(/\s+/g); if (!Array.isArray(node.properties.class)) node.properties.class = []; const targets = Array.isArray(className) ? className : className.split(/\s+/g); for (const c of targets) { if (c && !node.properties.class.includes(c)) node.properties.class.push(c); } return node; } /** * Split a token into multiple tokens by given offsets. * * The offsets are relative to the token, and should be sorted. */ function splitToken(token, offsets) { let lastOffset = 0; const tokens = []; for (const offset of offsets) { if (offset > lastOffset) { tokens.push({ ...token, content: token.content.slice(lastOffset, offset), offset: token.offset + lastOffset, }); } lastOffset = offset; } if (lastOffset < token.content.length) { tokens.push({ ...token, content: token.content.slice(lastOffset), offset: token.offset + lastOffset, }); } return tokens; } /** * Split 2D tokens array by given breakpoints. */ function splitTokens(tokens, breakpoints) { const sorted = Array.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints)) .sort((a, b) => a - b); if (!sorted.length) return tokens; return tokens.map((line) => { return line.flatMap((token) => { const breakpointsInToken = sorted .filter(i => token.offset < i && i < token.offset + token.content.length) .map(i => i - token.offset) .sort((a, b) => a - b); if (!breakpointsInToken.length) return token; return splitToken(token, breakpointsInToken); }); }); } function resolveColorReplacements(theme, options) { const replacements = typeof theme === 'string' ? {} : { ...theme.colorReplacements }; const themeName = typeof theme === 'string' ? theme : theme.name; for (const [key, value] of Object.entries(options?.colorReplacements || {})) { if (typeof value === 'string') replacements[key] = value; else if (key === themeName) Object.assign(replacements, value); } return replacements; } function applyColorReplacements(color, replacements) { if (!color) return color; return replacements?.[color?.toLowerCase()] || color; } function getTokenStyleObject(token) { const styles = {}; if (token.color) styles.color = token.color; if (token.bgColor) styles['background-color'] = token.bgColor; if (token.fontStyle) { if (token.fontStyle & FontStyle.Italic) styles['font-style'] = 'italic'; if (token.fontStyle & FontStyle.Bold) styles['font-weight'] = 'bold'; if (token.fontStyle & FontStyle.Underline) styles['text-decoration'] = 'underline'; } return styles; } function stringifyTokenStyle(token) { return Object.entries(token).map(([key, value]) => `${key}:${value}`).join(';'); } /** * Creates a converter between index and position in a code block. */ function createPositionConverter(code) { const lines = splitLines(code, true).map(([line]) => line); function indexToPos(index) { let character = index; let line = 0; for (const lineText of lines) { if (character < lineText.length) break; character -= lineText.length; line++; } return { line, character }; } function posToIndex(line, character) { let index = 0; for (let i = 0; i < line; i++) index += lines[i].length; index += character; return index; } return { lines, indexToPos, posToIndex, }; } // src/colors.ts var namedColors = [ "black", "red", "green", "yellow", "blue", "magenta", "cyan", "white", "brightBlack", "brightRed", "brightGreen", "brightYellow", "brightBlue", "brightMagenta", "brightCyan", "brightWhite" ]; // src/decorations.ts var decorations = { 1: "bold", 2: "dim", 3: "italic", 4: "underline", 7: "reverse", 9: "strikethrough" }; // src/parser.ts function findSequence(value, position) { const nextEscape = value.indexOf("\x1B[", position); if (nextEscape !== -1) { const nextClose = value.indexOf("m", nextEscape); return { sequence: value.substring(nextEscape + 2, nextClose).split(";"), startPosition: nextEscape, position: nextClose + 1 }; } return { position: value.length }; } function parseColor(sequence, index) { let offset = 1; const colorMode = sequence[index + offset++]; let color; if (colorMode === "2") { const rgb = [ sequence[index + offset++], sequence[index + offset++], sequence[index + offset] ].map((x) => Number.parseInt(x)); if (rgb.length === 3 && !rgb.some((x) => Number.isNaN(x))) { color = { type: "rgb", rgb }; } } else if (colorMode === "5") { const colorIndex = Number.parseInt(sequence[index + offset]); if (!Number.isNaN(colorIndex)) { color = { type: "table", index: Number(colorIndex) }; } } return [offset, color]; } function parseSequence(sequence) { const commands = []; for (let i = 0; i < sequence.length; i++) { const code = sequence[i]; const codeInt = Number.parseInt(code); if (Number.isNaN(codeInt)) continue; if (codeInt === 0) { commands.push({ type: "resetAll" }); } else if (codeInt <= 9) { const decoration = decorations[codeInt]; if (decoration) { commands.push({ type: "setDecoration", value: decorations[codeInt] }); } } else if (codeInt <= 29) { const decoration = decorations[codeInt - 20]; if (decoration) { commands.push({ type: "resetDecoration", value: decoration }); } } else if (codeInt <= 37) { commands.push({ type: "setForegroundColor", value: { type: "named", name: namedColors[codeInt - 30] } }); } else if (codeInt === 38) { const [offset, color] = parseColor(sequence, i); if (color) { commands.push({ type: "setForegroundColor", value: color }); } i += offset; } else if (codeInt === 39) { commands.push({ type: "resetForegroundColor" }); } else if (codeInt <= 47) { commands.push({ type: "setBackgroundColor", value: { type: "named", name: namedColors[codeInt - 40] } }); } else if (codeInt === 48) { const [offset, color] = parseColor(sequence, i); if (color) { commands.push({ type: "setBackgroundColor", value: color }); } i += offset; } else if (codeInt === 49) { commands.push({ type: "resetBackgroundColor" }); } else if (codeInt >= 90 && codeInt <= 97) { commands.push({ type: "setForegroundColor", value: { type: "named", name: namedColors[codeInt - 90 + 8] } }); } else if (codeInt >= 100 && codeInt <= 107) { commands.push({ type: "setBackgroundColor", value: { type: "named", name: namedColors[codeInt - 100 + 8] } }); } } return commands; } function createAnsiSequenceParser() { let foreground = null; let background = null; let decorations2 = /* @__PURE__ */ new Set(); return { parse(value) { const tokens = []; let position = 0; do { const findResult = findSequence(value, position); const text = findResult.sequence ? value.substring(position, findResult.startPosition) : value.substring(position); if (text.length > 0) { tokens.push({ value: text, foreground, background, decorations: new Set(decorations2) }); } if (findResult.sequence) { const commands = parseSequence(findResult.sequence); for (const styleToken of commands) { if (styleToken.type === "resetAll") { foreground = null; background = null; decorations2.clear(); } else if (styleToken.type === "resetForegroundColor") { foreground = null; } else if (styleToken.type === "resetBackgroundColor") { background = null; } else if (styleToken.type === "resetDecoration") { decorations2.delete(styleToken.value); } } for (const styleToken of commands) { if (styleToken.type === "setForegroundColor") { foreground = styleToken.value; } else if (styleToken.type === "setBackgroundColor") { background = styleToken.value; } else if (styleToken.type === "setDecoration") { decorations2.add(styleToken.value); } } } position = findResult.position; } while (position < value.length); return tokens; } }; } // src/palette.ts var defaultNamedColorsMap = { black: "#000000", red: "#bb0000", green: "#00bb00", yellow: "#bbbb00", blue: "#0000bb", magenta: "#ff00ff", cyan: "#00bbbb", white: "#eeeeee", brightBlack: "#555555", brightRed: "#ff5555", brightGreen: "#00ff00", brightYellow: "#ffff55", brightBlue: "#5555ff", brightMagenta: "#ff55ff", brightCyan: "#55ffff", brightWhite: "#ffffff" }; function createColorPalette(namedColorsMap = defaultNamedColorsMap) { function namedColor(name) { return namedColorsMap[name]; } function rgbColor(rgb) { return `#${rgb.map((x) => Math.max(0, Math.min(x, 255)).toString(16).padStart(2, "0")).join("")}`; } let colorTable; function getColorTable() { if (colorTable) { return colorTable; } colorTable = []; for (let i = 0; i < namedColors.length; i++) { colorTable.push(namedColor(namedColors[i])); } let levels = [0, 95, 135, 175, 215, 255]; for (let r = 0; r < 6; r++) { for (let g = 0; g < 6; g++) { for (let b = 0; b < 6; b++) { colorTable.push(rgbColor([levels[r], levels[g], levels[b]])); } } } let level = 8; for (let i = 0; i < 24; i++, level += 10) { colorTable.push(rgbColor([level, level, level])); } return colorTable; } function tableColor(index) { return getColorTable()[index]; } function value(color) { switch (color.type) { case "named": return namedColor(color.name); case "rgb": return rgbColor(color.rgb); case "table": return tableColor(color.index); } } return { value }; } function tokenizeAnsiWithTheme(theme, fileContents, options) { const colorReplacements = resolveColorReplacements(theme, options); const lines = splitLines(fileContents); const colorPalette = createColorPalette(Object.fromEntries(namedColors.map(name => [ name, theme.colors?.[`terminal.ansi${name[0].toUpperCase()}${name.substring(1)}`], ]))); const parser = createAnsiSequenceParser(); return lines.map(line => parser.parse(line[0]).map((token) => { let color; let bgColor; if (token.decorations.has('reverse')) { color = token.background ? colorPalette.value(token.background) : theme.bg; bgColor = token.foreground ? colorPalette.value(token.foreground) : theme.fg; } else { color = token.foreground ? colorPalette.value(token.foreground) : theme.fg; bgColor = token.background ? colorPalette.value(token.background) : undefined; } color = applyColorReplacements(color, colorReplacements); bgColor = applyColorReplacements(bgColor, colorReplacements); if (token.decorations.has('dim')) color = dimColor(color); let fontStyle = FontStyle.None; if (token.decorations.has('bold')) fontStyle |= FontStyle.Bold; if (token.decorations.has('italic')) fontStyle |= FontStyle.Italic; if (token.decorations.has('underline')) fontStyle |= FontStyle.Underline; return { content: token.value, offset: line[1], // TODO: more accurate offset? might need to fork ansi-sequence-parser color, bgColor, fontStyle, }; })); } /** * Adds 50% alpha to a hex color string or the "-dim" postfix to a CSS variable */ function dimColor(color) { const hexMatch = color.match(/#([0-9a-f]{3})([0-9a-f]{3})?([0-9a-f]{2})?/); if (hexMatch) { if (hexMatch[3]) { // convert from #rrggbbaa to #rrggbb(aa/2) const alpha = Math.round(Number.parseInt(hexMatch[3], 16) / 2) .toString(16) .padStart(2, '0'); return `#${hexMatch[1]}${hexMatch[2]}${alpha}`; } else if (hexMatch[2]) { // convert from #rrggbb to #rrggbb80 return `#${hexMatch[1]}${hexMatch[2]}80`; } else { // convert from #rgb to #rrggbb80 return `#${Array.from(hexMatch[1]) .map(x => `${x}${x}`) .join('')}80`; } } const cssVarMatch = color.match(/var\((--[\w-]+-ansi-[\w-]+)\)/); if (cssVarMatch) return `var(${cssVarMatch[1]}-dim)`; return color; } /** * Code to tokens, with a simple theme. */ function codeToTokensBase(internal, code, options = {}) { const { lang = 'text', theme: themeName = internal.getLoadedThemes()[0], } = options; if (isPlainLang(lang) || isNoneTheme(themeName)) return splitLines(code).map(line => [{ content: line[0], offset: line[1] }]); const { theme, colorMap } = internal.setTheme(themeName); if (lang === 'ansi') return tokenizeAnsiWithTheme(theme, code, options); const _grammar = internal.getLanguage(lang); return tokenizeWithTheme(code, _grammar, theme, colorMap, options); } function tokenizeWithTheme(code, grammar, theme, colorMap, options) { const colorReplacements = resolveColorReplacements(theme, options); const { tokenizeMaxLineLength = 0, tokenizeTimeLimit = 500, } = options; const lines = splitLines(code); let ruleStack = INITIAL; let actual = []; const final = []; const themeSettingsSelectors = []; if (options.includeExplanation) { for (const setting of theme.settings) { let selectors; switch (typeof setting.scope) { case 'string': selectors = setting.scope.split(/,/).map(scope => scope.trim()); break; case 'object': selectors = setting.scope; break; default: continue; } themeSettingsSelectors.push({ settings: setting, selectors: selectors.map(selector => selector.split(/ /)), }); } } for (let i = 0, len = lines.length; i < len; i++) { const [line, lineOffset] = lines[i]; if (line === '') { actual = []; final.push([]); continue; } // Do not attempt to tokenize if the line length is longer than the `tokenizationMaxLineLength` if (tokenizeMaxLineLength > 0 && line.length >= tokenizeMaxLineLength) { actual = []; final.push([{ content: line, offset: lineOffset, color: '', fontStyle: 0, }]); continue; } let resultWithScopes; let tokensWithScopes; let tokensWithScopesIndex; if (options.includeExplanation) { resultWithScopes = grammar.tokenizeLine(line, ruleStack); tokensWithScopes = resultWithScopes.tokens; tokensWithScopesIndex = 0; } const result = grammar.tokenizeLine2(line, ruleStack, tokenizeTimeLimit); const tokensLength = result.tokens.length / 2; for (let j = 0; j < tokensLength; j++) { const startIndex = result.tokens[2 * j]; const nextStartIndex = j + 1 < tokensLength ? result.tokens[2 * j + 2] : line.length; if (startIndex === nextStartIndex) continue; const metadata = result.tokens[2 * j + 1]; const color = applyColorReplacements(colorMap[StackElementMetadata.getForeground(metadata)], colorReplacements); const fontStyle = StackElementMetadata.getFontStyle(metadata); const token = { content: line.substring(startIndex, nextStartIndex), offset: lineOffset + startIndex, color, fontStyle, }; if (options.includeExplanation) { token.explanation = []; let offset = 0; while (startIndex + offset < nextStartIndex) { const tokenWithScopes = tokensWithScopes[tokensWithScopesIndex]; const tokenWithScopesText = line.substring(tokenWithScopes.startIndex, tokenWithScopes.endIndex); offset += tokenWithScopesText.length; token.explanation.push({ content: tokenWithScopesText, scopes: explainThemeScopes(themeSettingsSelectors, tokenWithScopes.scopes), }); tokensWithScopesIndex += 1; } } actual.push(token); } final.push(actual); actual = []; ruleStack = result.ruleStack; } return final; } function explainThemeScopes(themeSelectors, scopes) { const result = []; for (let i = 0, len = scopes.length; i < len; i++) { const parentScopes = scopes.slice(0, i); const scope = scopes[i]; result[i] = { scopeName: scope, themeMatches: explainThemeScope(themeSelectors, scope, parentScopes), }; } return result; } function matchesOne(selector, scope) { return selector === scope || (scope.substring(0, selector.length) === selector && scope[selector.length] === '.'); } function matches(selectors, scope, parentScopes) { if (!matchesOne(selectors[selectors.length - 1], scope)) return false; let selectorParentIndex = selectors.length - 2; let parentIndex = parentScopes.length - 1; while (selectorParentIndex >= 0 && parentIndex >= 0) { if (matchesOne(selectors[selectorParentIndex], parentScopes[parentIndex])) selectorParentIndex -= 1; parentIndex -= 1; } if (selectorParentIndex === -1) return true; return false; } function explainThemeScope(themeSettingsSelectors, scope, parentScopes) { const result = []; for (const { selectors, settings } of themeSettingsSelectors) { for (const selectorPieces of selectors) { if (matches(selectorPieces, scope, parentScopes)) { result.push(settings); break; // continue to the next theme settings } } } return result; } /** * Get tokens with multiple themes */ function codeToTokensWithThemes(internal, code, options) { const themes = Object.entries(options.themes) .filter(i => i[1]) .map(i => ({ color: i[0], theme: i[1] })); const tokens = syncThemesTokenization(...themes.map(t => codeToTokensBase(internal, code, { ...options, theme: t.theme, }))); const mergedTokens = tokens[0] .map((line, lineIdx) => line .map((_token, tokenIdx) => { const mergedToken = { content: _token.content, variants: {}, offset: _token.offset, }; tokens.forEach((t, themeIdx) => { const { content: _, explanation: __, offset: ___, ...styles } = t[lineIdx][tokenIdx]; mergedToken.variants[themes[themeIdx].color] = styles; }); return mergedToken; })); return mergedTokens; } /** * Break tokens from multiple themes into same tokenization. * * For example, given two themes that tokenize `console.log("hello")` as: * * - `console . log (" hello ")` (6 tokens) * - `console .log ( "hello" )` (5 tokens) * * This function will return: * * - `console . log ( " hello " )` (8 tokens) * - `console . log ( " hello " )` (8 tokens) */ function syncThemesTokenization(...themes) { const outThemes = themes.map(() => []); const count = themes.length; for (let i = 0; i < themes[0].length; i++) { const lines = themes.map(t => t[i]); const outLines = outThemes.map(() => []); outThemes.forEach((t, i) => t.push(outLines[i])); const indexes = lines.map(() => 0); const current = lines.map(l => l[0]); while (current.every(t => t)) { const minLength = Math.min(...current.map(t => t.content.length)); for (let n = 0; n < count; n++) { const token = current[n]; if (token.content.length === minLength) { outLines[n].push(token); indexes[n] += 1; current[n] = lines[n][indexes[n]]; } else { outLines[n].push({ ...token, content: token.content.slice(0, minLength), }); current[n] = { ...token, content: token.content.slice(minLength), offset: token.offset + minLength, }; } } } } return outThemes; } class ShikiError extends Error { constructor(message) { super(message); this.name = 'ShikiError'; } } /** * High-level code-to-tokens API. * * It will use `codeToTokensWithThemes` or `codeToTokensBase` based on the options. */ function codeToTokens(internal, code, options) { let bg; let fg; let tokens; let themeName; let rootStyle; if ('themes' in options) { const { defaultColor = 'light', cssVariablePrefix = '--shiki-', } = options; const themes = Object.entries(options.themes) .filter(i => i[1]) .map(i => ({ color: i[0], theme: i[1] })) .sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0); if (themes.length === 0) throw new ShikiError('`themes` option must not be empty'); const themeTokens = codeToTokensWithThemes(internal, code, options); if (defaultColor && !themes.find(t => t.color === defaultColor)) throw new ShikiError(`\`themes\` option must contain the defaultColor key \`${defaultColor}\``); const themeRegs = themes.map(t => internal.getTheme(t.theme)); const themesOrder = themes.map(t => t.color); tokens = themeTokens .map(line => line.map(token => mergeToken(token, themesOrder, cssVariablePrefix, defaultColor))); const themeColorReplacements = themes.map(t => resolveColorReplacements(t.theme, options)); fg = themes.map((t, idx) => (idx === 0 && defaultColor ? '' : `${cssVariablePrefix + t.color}:`) + (applyColorReplacements(themeRegs[idx].fg, themeColorReplacements[idx]) || 'inherit')).join(';'); bg = themes.map((t, idx) => (idx === 0 && defaultColor ? '' : `${cssVariablePrefix + t.color}-bg:`) + (applyColorReplacements(themeRegs[idx].bg, themeColorReplacements[idx]) || 'inherit')).join(';'); themeName = `shiki-themes ${themeRegs.map(t => t.name).join(' ')}`; rootStyle = defaultColor ? undefined : [fg, bg].join(';'); } else if ('theme' in options) { const colorReplacements = resolveColorReplacements(options.theme, options.colorReplacements); tokens = codeToTokensBase(internal, code, options); const _theme = internal.getTheme(options.theme); bg = applyColorReplacements(_theme.bg, colorReplacements); fg = applyColorReplacements(_theme.fg, colorReplacements); themeName = _theme.name; } else { throw new ShikiError('Invalid options, either `theme` or `themes` must be provided'); } return { tokens, fg, bg, themeName, rootStyle, }; } function mergeToken(merged, variantsOrder, cssVariablePrefix, defaultColor) { const token = { content: merged.content, explanation: merged.explanation, offset: merged.offset, }; const styles = variantsOrder.map(t => getTokenStyleObject(merged.variants[t])); // Get all style keys, for themes that missing some style, we put `inherit` to override as needed const styleKeys = new Set(styles.flatMap(t => Object.keys(t))); const mergedStyles = styles.reduce((acc, cur, idx) => { for (const key of styleKeys) { const value = cur[key] || 'inherit'; if (idx === 0 && defaultColor) { acc[key] = value; } else { const keyName = key === 'color' ? '' : key === 'background-color' ? '-bg' : `-${key}`; const varKey = cssVariablePrefix + variantsOrder[idx] + (key === 'color' ? '' : keyName); if (acc[key]) acc[key] += `;${varKey}:${value}`; else acc[key] = `${varKey}:${value}`; } } return acc; }, {}); token.htmlStyle = defaultColor ? stringifyTokenStyle(mergedStyles) : Object.values(mergedStyles).join(';'); return token; } /** * A built-in transformer to add decorations to the highlighted code. */ function transformerDecorations() { const map = new WeakMap(); function getContext(shiki) { if (!map.has(shiki.meta)) { const converter = createPositionConverter(shiki.source); function normalizePosition(p) { if (typeof p === 'number') { return { ...converter.indexToPos(p), offset: p, }; } else { return { ...p, offset: converter.posToIndex(p.line, p.character), }; } } const decorations = (shiki.options.decorations || []) .map((d) => ({ ...d, start: normalizePosition(d.start), end: normalizePosition(d.end), })); verifyIntersections(decorations); map.set(shiki.meta, { decorations, converter, source: shiki.source, }); } return map.get(shiki.meta); } function verifyIntersections(items) { for (let i = 0; i < items.length; i++) { const foo = items[i]; if (foo.start.offset > foo.end.offset) throw new ShikiError(`Invalid decoration range: ${JSON.stringify(foo.start)} - ${JSON.stringify(foo.end)}`); for (let j = i + 1; j < items.length; j++) { const bar = items[j]; const isFooHasBarStart = foo.start.offset < bar.start.offset && bar.start.offset < foo.end.offset; const isFooHasBarEnd = foo.start.offset < bar.end.offset && bar.end.offset < foo.end.offset; const isBarHasFooStart = bar.start.offset < foo.start.offset && foo.start.offset < bar.end.offset; const isBarHasFooEnd = bar.start.offset < foo.end.offset && foo.end.offset < bar.end.offset; if (isFooHasBarStart || isFooHasBarEnd || isBarHasFooStart || isBarHasFooEnd) { if (isFooHasBarEnd && isFooHasBarEnd) continue; // nested if (isBarHasFooStart && isBarHasFooEnd) continue; // nested throw new ShikiError(`Decorations ${JSON.stringify(foo.start)} and ${JSON.stringify(bar.start)} intersect.`); } } } } return { name: 'shiki:decorations', tokens(tokens) { if (!this.options.decorations?.length) return; const ctx = getContext(this); const breakpoints = ctx.decorations.flatMap(d => [d.start.offset, d.end.offset]); const splitted = splitTokens(tokens, breakpoints); return splitted; }, code(codeEl) { if (!this.options.decorations?.length) return; const ctx = getContext(this); const lines = Array.from(codeEl.children).filter(i => i.type === 'element' && i.tagName === 'span'); if (lines.length !== ctx.converter.lines.length) throw new ShikiError(`Number of lines in code element (${lines.length}) does not match the number of lines in the source (${ctx.converter.lines.length}). Failed to apply decorations.`); function applyLineSection(line, start, end, decoration) { const lineEl = lines[line]; let text = ''; let startIndex = -1; let endIndex = -1; function stringify(el) { if (el.type === 'text') return el.value; if (el.type === 'element') return el.children.map(stringify).join(''); return ''; } if (start === 0) startIndex = 0; if (end === 0) endIndex = 0; if (end === Number.POSITIVE_INFINITY) endIndex = lineEl.children.length; if (startIndex === -1 || endIndex === -1) { for (let i = 0; i < lineEl.children.length; i++) { text += stringify(lineEl.children[i]); if (startIndex === -1 && text.length === start) startIndex = i + 1; if (endIndex === -1 && text.length === end) endIndex = i + 1; } } if (startIndex === -1) throw new ShikiError(`Failed to find start index for decoration ${JSON.stringify(decoration.start)}`); if (endIndex === -1) throw new ShikiError(`Failed to find end index for decoration ${JSON.stringify(decoration.end)}`); const children = lineEl.children.slice(startIndex, endIndex); // Full line decoration if (!decoration.alwaysWrap && children.length === lineEl.children.length) { applyDecoration(lineEl, decoration, 'line'); } // Single token decoration else if (!decoration.alwaysWrap && children.length === 1 && children[0].type === 'element') { applyDecoration(children[0], decoration, 'token'); } // Create a wrapper for the decoration else { const wrapper = { type: 'element', tagName: 'span', properties: {}, children, }; applyDecoration(wrapper, decoration, 'wrapper'); lineEl.children.splice(startIndex, children.length, wrapper); } } function applyLine(line, decoration) { lines[line] = applyDecoration(lines[line], decoration, 'line'); } function applyDecoration(el, decoration, type) { const properties = decoration.properties || {}; const transform = decoration.transform || (i => i); el.tagName = decoration.tagName || 'span'; el.properties = { ...el.properties, ...properties, class: el.properties.class, }; if (decoration.properties?.class) addClassToHast(el, decoration.properties.class); el = transform(el, type) || el; return el; } const lineApplies = []; // Apply decorations in reverse order so the nested ones get applied first. const sorted = ctx.decorations.sort((a, b) => b.start.offset - a.start.offset); for (const decoration of sorted) { const { start, end } = decoration; if (start.line === end.line) { applyLineSection(start.line, start.character, end.character, decoration); } else if (start.line < end.line) { applyLineSection(start.line, start.character, Number.POSITIVE_INFINITY, decoration); for (let i = start.line + 1; i < end.line; i++) lineApplies.unshift(() => applyLine(i, decoration)); applyLineSection(end.line, 0, end.character, decoration); } } lineApplies.forEach(i => i()); }, }; } const builtInTransformers = [ /* @__PURE__ */ transformerDecorations(), ]; function getTransformers(options) { return [ ...options.transformers || [], ...builtInTransformers, ]; } function codeToHast(internal, code, options, transformerContext = { meta: {}, options, codeToHast: (_code, _options) => codeToHast(internal, _code, _options), codeToTokens: (_code, _options) => codeToTokens(internal, _code, _options), }) { let input = code; for (const transformer of getTransformers(options)) input = transformer.preprocess?.call(transformerContext, input, options) || input; let { tokens, fg, bg, themeName, rootStyle, } = codeToTokens(internal, input, options); const { mergeWhitespaces = true, } = options; if (mergeWhitespaces === true) tokens = mergeWhitespaceTokens(tokens); else if (mergeWhitespaces === 'never') tokens = splitWhitespaceTokens(tokens); const contextSource = { ...transformerContext, get source() { return input; }, }; for (const transformer of getTransformers(options)) tokens = transformer.tokens?.call(contextSource, tokens) || tokens; return tokensToHast(tokens, { ...options, fg, bg, themeName, rootStyle, }, contextSource); } function tokensToHast(tokens, options, transformerContext) { const transformers = getTransformers(options); const lines = []; const root = { type: 'root', children: [], }; const { structure = 'classic', } = options; let preNode = { type: 'element', tagName: 'pre', properties: { class: `shiki ${options.themeName || ''}`, style: options.rootStyle || `background-color:${options.bg};color:${options.fg}`, tabindex: '0', ...Object.fromEntries(Array.from(Object.entries(options.meta || {})) .filter(([key]) => !key.startsWith('_'))), }, children: [], }; let codeNode = { type: 'element', tagName: 'code', properties: {}, children: lines, }; const lineNodes = []; const context = { ...transformerContext, structure, addClassToHast, get source() { return transformerContext.source; }, get tokens() { return tokens; }, get options() { return options; }, get root() { return root; }, get pre() { return preNode; }, get code() { return codeNode; }, get lines() { return lineNodes; }, }; tokens.forEach((line, idx) => { if (idx) { if (structure === 'inline') root.children.push({ type: 'element', tagName: 'br', properties: {}, children: [] }); else if (structure === 'classic') lines.push({ type: 'text', value: '\n' }); } let lineNode = { type: 'element', tagName: 'span', properties: { class: 'line' }, children: [], }; let col = 0; for (const token of line) { let tokenNode = { type: 'element', tagName: 'span', properties: {}, children: [{ type: 'text', value: token.content }], }; const style = token.htmlStyle || stringifyTokenStyle(getTokenStyleObject(token)); if (style) tokenNode.properties.style = style; for (const transformer of transformers) tokenNode = transformer?.span?.call(context, tokenNode, idx + 1, col, lineNode) || tokenNode; if (structure === 'inline') root.children.push(tokenNode); else if (structure === 'classic') lineNode.children.push(tokenNode); col += token.content.length; } if (structure === 'classic') { for (const transformer of transformers) lineNode = transformer?.line?.call(context, lineNode, idx + 1) || lineNode; lineNodes.push(lineNode); lines.push(lineNode); } }); if (structure === 'classic') { for (const transformer of transformers) codeNode = transformer?.code?.call(context, codeNode) || codeNode; preNode.children.push(codeNode); for (const transformer of transformers) preNode = transformer?.pre?.call(context, preNode) || preNode; root.children.push(preNode); } let result = root; for (const transformer of transformers) result = transformer?.root?.call(context, result) || result; return result; } function mergeWhitespaceTokens(tokens) { return tokens.map((line) => { const newLine = []; let carryOnContent = ''; let firstOffset = 0; line.forEach((token, idx) => { const isUnderline = token.fontStyle && token.fontStyle & FontStyle.Underline; const couldMerge = !isUnderline; if (couldMerge && token.content.match(/^\s+$/) && line[idx + 1]) { if (!firstOffset) firstOffset = token.offset; carryOnContent += token.content; } else { if (carryOnContent) { if (couldMerge) { newLine.push({ ...token, offset: firstOffset, content: carryOnContent + token.content, }); } else { newLine.push({ content: carryOnContent, offset: firstOffset, }, token); } firstOffset = 0; carryOnContent = ''; } else { newLine.push(token); } } }); return newLine; }); } function splitWhitespaceTokens(tokens) { return tokens.map((line) => { return line.flatMap((token) => { if (token.content.match(/^\s+$/)) return token; // eslint-disable-next-line regexp/no-super-linear-backtracking const match = token.content.match(/^(\s*)(.*?)(\s*)$/); if (!match) return token; const [, leading, content, trailing] = match; if (!leading && !trailing) return token; const expanded = [{ ...token, offset: token.offset + leading.length, content, }]; if (leading) { expanded.unshift({ content: leading, offset: token.offset, }); } if (trailing) { expanded.push({ content: trailing, offset: token.offset + leading.length + content.length, }); } return expanded; }); }); } /** * List of HTML void tag names. * * @type {Array} */ const htmlVoidElements = [ 'area', 'base', 'basefont', 'bgsound', 'br', 'col', 'command', 'embed', 'frame', 'hr', 'image', 'img', 'input', 'keygen', 'link', 'meta', 'param', 'source', 'track', 'wbr' ]; /** * @typedef {import('./info.js').Info} Info * @typedef {Record} Properties * @typedef {Record} Normal */ class Schema { /** * @constructor * @param {Properties} property * @param {Normal} normal * @param {string} [space] */ constructor(property, normal, space) { this.property = property; this.normal = normal; if (space) { this.space = space; } } } /** @type {Properties} */ Schema.prototype.property = {}; /** @type {Normal} */ Schema.prototype.normal = {}; /** @type {string|null} */ Schema.prototype.space = null; /** * @typedef {import('./schema.js').Properties} Properties * @typedef {import('./schema.js').Normal} Normal */ /** * @param {Schema[]} definitions * @param {string} [space] * @returns {Schema} */ function merge(definitions, space) { /** @type {Properties} */ const property = {}; /** @type {Normal} */ const normal = {}; let index = -1; while (++index < definitions.length) { Object.assign(property, definitions[index].property); Object.assign(normal, definitions[index].normal); } return new Schema(property, normal, space) } /** * @param {string} value * @returns {string} */ function normalize(value) { return value.toLowerCase() } class Info { /** * @constructor * @param {string} property * @param {string} attribute */ constructor(property, attribute) { /** @type {string} */ this.property = property; /** @type {string} */ this.attribute = attribute; } } /** @type {string|null} */ Info.prototype.space = null; Info.prototype.boolean = false; Info.prototype.booleanish = false; Info.prototype.overloadedBoolean = false; Info.prototype.number = false; Info.prototype.commaSeparated = false; Info.prototype.spaceSeparated = false; Info.prototype.commaOrSpaceSeparated = false; Info.prototype.mustUseProperty = false; Info.prototype.defined = false; let powers = 0; const boolean = increment(); const booleanish = increment(); const overloadedBoolean = increment(); const number = increment(); const spaceSeparated = increment(); const commaSeparated = increment(); const commaOrSpaceSeparated = increment(); function increment() { return 2 ** ++powers } var types = /*#__PURE__*/Object.freeze({ __proto__: null, boolean: boolean, booleanish: booleanish, commaOrSpaceSeparated: commaOrSpaceSeparated, commaSeparated: commaSeparated, number: number, overloadedBoolean: overloadedBoolean, spaceSeparated: spaceSeparated }); /** @type {Array} */ // @ts-expect-error: hush. const checks = Object.keys(types); class DefinedInfo extends Info { /** * @constructor * @param {string} property * @param {string} attribute * @param {number|null} [mask] * @param {string} [space] */ constructor(property, attribute, mask, space) { let index = -1; super(property, attribute); mark(this, 'space', space); if (typeof mask === 'number') { while (++index < checks.length) { const check = checks[index]; mark(this, checks[index], (mask & types[check]) === types[check]); } } } } DefinedInfo.prototype.defined = true; /** * @param {DefinedInfo} values * @param {string} key * @param {unknown} value */ function mark(values, key, value) { if (value) { // @ts-expect-error: assume `value` matches the expected value of `key`. values[key] = value; } } /** * @typedef {import('./schema.js').Properties} Properties * @typedef {import('./schema.js').Normal} Normal * * @typedef {Record} Attributes * * @typedef {Object} Definition * @property {Record} properties * @property {(attributes: Attributes, property: string) => string} transform * @property {string} [space] * @property {Attributes} [attributes] * @property {Array} [mustUseProperty] */ const own$3 = {}.hasOwnProperty; /** * @param {Definition} definition * @returns {Schema} */ function create(definition) { /** @type {Properties} */ const property = {}; /** @type {Normal} */ const normal = {}; /** @type {string} */ let prop; for (prop in definition.properties) { if (own$3.call(definition.properties, prop)) { const value = definition.properties[prop]; const info = new DefinedInfo( prop, definition.transform(definition.attributes || {}, prop), value, definition.space ); if ( definition.mustUseProperty && definition.mustUseProperty.includes(prop) ) { info.mustUseProperty = true; } property[prop] = info; normal[normalize(prop)] = prop; normal[normalize(info.attribute)] = prop; } } return new Schema(property, normal, definition.space) } const xlink = create({ space: 'xlink', transform(_, prop) { return 'xlink:' + prop.slice(5).toLowerCase() }, properties: { xLinkActuate: null, xLinkArcRole: null, xLinkHref: null, xLinkRole: null, xLinkShow: null, xLinkTitle: null, xLinkType: null } }); const xml = create({ space: 'xml', transform(_, prop) { return 'xml:' + prop.slice(3).toLowerCase() }, properties: {xmlLang: null, xmlBase: null, xmlSpace: null} }); /** * @param {Record} attributes * @param {string} attribute * @returns {string} */ function caseSensitiveTransform(attributes, attribute) { return attribute in attributes ? attributes[attribute] : attribute } /** * @param {Record} attributes * @param {string} property * @returns {string} */ function caseInsensitiveTransform(attributes, property) { return caseSensitiveTransform(attributes, property.toLowerCase()) } const xmlns = create({ space: 'xmlns', attributes: {xmlnsxlink: 'xmlns:xlink'}, transform: caseInsensitiveTransform, properties: {xmlns: null, xmlnsXLink: null} }); const aria = create({ transform(_, prop) { return prop === 'role' ? prop : 'aria-' + prop.slice(4).toLowerCase() }, properties: { ariaActiveDescendant: null, ariaAtomic: booleanish, ariaAutoComplete: null, ariaBusy: booleanish, ariaChecked: booleanish, ariaColCount: number, ariaColIndex: number, ariaColSpan: number, ariaControls: spaceSeparated, ariaCurrent: null, ariaDescribedBy: spaceSeparated, ariaDetails: null, ariaDisabled: booleanish, ariaDropEffect: spaceSeparated, ariaErrorMessage: null, ariaExpanded: booleanish, ariaFlowTo: spaceSeparated, ariaGrabbed: booleanish, ariaHasPopup: null, ariaHidden: booleanish, ariaInvalid: null, ariaKeyShortcuts: null, ariaLabel: null, ariaLabelledBy: spaceSeparated, ariaLevel: number, ariaLive: null, ariaModal: booleanish, ariaMultiLine: booleanish, ariaMultiSelectable: booleanish, ariaOrientation: null, ariaOwns: spaceSeparated, ariaPlaceholder: null, ariaPosInSet: number, ariaPressed: booleanish, ariaReadOnly: booleanish, ariaRelevant: null, ariaRequired: booleanish, ariaRoleDescription: spaceSeparated, ariaRowCount: number, ariaRowIndex: number, ariaRowSpan: number, ariaSelected: booleanish, ariaSetSize: number, ariaSort: null, ariaValueMax: number, ariaValueMin: number, ariaValueNow: number, ariaValueText: null, role: null } }); const html$3 = create({ space: 'html', attributes: { acceptcharset: 'accept-charset', classname: 'class', htmlfor: 'for', httpequiv: 'http-equiv' }, transform: caseInsensitiveTransform, mustUseProperty: ['checked', 'multiple', 'muted', 'selected'], properties: { // Standard Properties. abbr: null, accept: commaSeparated, acceptCharset: spaceSeparated, accessKey: spaceSeparated, action: null, allow: null, allowFullScreen: boolean, allowPaymentRequest: boolean, allowUserMedia: boolean, alt: null, as: null, async: boolean, autoCapitalize: null, autoComplete: spaceSeparated, autoFocus: boolean, autoPlay: boolean, blocking: spaceSeparated, capture: null, charSet: null, checked: boolean, cite: null, className: spaceSeparated, cols: number, colSpan: null, content: null, contentEditable: booleanish, controls: boolean, controlsList: spaceSeparated, coords: number | commaSeparated, crossOrigin: null, data: null, dateTime: null, decoding: null, default: boolean, defer: boolean, dir: null, dirName: null, disabled: boolean, download: overloadedBoolean, draggable: booleanish, encType: null, enterKeyHint: null, fetchPriority: null, form: null, formAction: null, formEncType: null, formMethod: null, formNoValidate: boolean, formTarget: null, headers: spaceSeparated, height: number, hidden: boolean, high: number, href: null, hrefLang: null, htmlFor: spaceSeparated, httpEquiv: spaceSeparated, id: null, imageSizes: null, imageSrcSet: null, inert: boolean, inputMode: null, integrity: null, is: null, isMap: boolean, itemId: null, itemProp: spaceSeparated, itemRef: spaceSeparated, itemScope: boolean, itemType: spaceSeparated, kind: null, label: null, lang: null, language: null, list: null, loading: null, loop: boolean, low: number, manifest: null, max: null, maxLength: number, media: null, method: null, min: null, minLength: number, multiple: boolean, muted: boolean, name: null, nonce: null, noModule: boolean, noValidate: boolean, onAbort: null, onAfterPrint: null, onAuxClick: null, onBeforeMatch: null, onBeforePrint: null, onBeforeToggle: null, onBeforeUnload: null, onBlur: null, onCancel: null, onCanPlay: null, onCanPlayThrough: null, onChange: null, onClick: null, onClose: null, onContextLost: null, onContextMenu: null, onContextRestored: null, onCopy: null, onCueChange: null, onCut: null, onDblClick: null, onDrag: null, onDragEnd: null, onDragEnter: null, onDragExit: null, onDragLeave: null, onDragOver: null, onDragStart: null, onDrop: null, onDurationChange: null, onEmptied: null, onEnded: null, onError: null, onFocus: null, onFormData: null, onHashChange: null, onInput: null, onInvalid: null, onKeyDown: null, onKeyPress: null, onKeyUp: null, onLanguageChange: null, onLoad: null, onLoadedData: null, onLoadedMetadata: null, onLoadEnd: null, onLoadStart: null, onMessage: null, onMessageError: null, onMouseDown: null, onMouseEnter: null, onMouseLeave: null, onMouseMove: null, onMouseOut: null, onMouseOver: null, onMouseUp: null, onOffline: null, onOnline: null, onPageHide: null, onPageShow: null, onPaste: null, onPause: null, onPlay: null, onPlaying: null, onPopState: null, onProgress: null, onRateChange: null, onRejectionHandled: null, onReset: null, onResize: null, onScroll: null, onScrollEnd: null, onSecurityPolicyViolation: null, onSeeked: null, onSeeking: null, onSelect: null, onSlotChange: null, onStalled: null, onStorage: null, onSubmit: null, onSuspend: null, onTimeUpdate: null, onToggle: null, onUnhandledRejection: null, onUnload: null, onVolumeChange: null, onWaiting: null, onWheel: null, open: boolean, optimum: number, pattern: null, ping: spaceSeparated, placeholder: null, playsInline: boolean, popover: null, popoverTarget: null, popoverTargetAction: null, poster: null, preload: null, readOnly: boolean, referrerPolicy: null, rel: spaceSeparated, required: boolean, reversed: boolean, rows: number, rowSpan: number, sandbox: spaceSeparated, scope: null, scoped: boolean, seamless: boolean, selected: boolean, shadowRootDelegatesFocus: boolean, shadowRootMode: null, shape: null, size: number, sizes: null, slot: null, span: number, spellCheck: booleanish, src: null, srcDoc: null, srcLang: null, srcSet: null, start: number, step: null, style: null, tabIndex: number, target: null, title: null, translate: null, type: null, typeMustMatch: boolean, useMap: null, value: booleanish, width: number, wrap: null, // Legacy. // See: https://html.spec.whatwg.org/#other-elements,-attributes-and-apis align: null, // Several. Use CSS `text-align` instead, aLink: null, // ``. Use CSS `a:active {color}` instead archive: spaceSeparated, // ``. List of URIs to archives axis: null, // `` and ``. Use `scope` on `` background: null, // ``. Use CSS `background-image` instead bgColor: null, // `` and table elements. Use CSS `background-color` instead border: number, // ``. Use CSS `border-width` instead, borderColor: null, // `
`. Use CSS `border-color` instead, bottomMargin: number, // `` cellPadding: null, // `
` cellSpacing: null, // `
` char: null, // Several table elements. When `align=char`, sets the character to align on charOff: null, // Several table elements. When `char`, offsets the alignment classId: null, // `` clear: null, // `
`. Use CSS `clear` instead code: null, // `` codeBase: null, // `` codeType: null, // `` color: null, // `` and `
`. Use CSS instead compact: boolean, // Lists. Use CSS to reduce space between items instead declare: boolean, // `` event: null, // `