From 74138a9a586c7af88e1b839dbae863b9333ca07e Mon Sep 17 00:00:00 2001 From: ocavue Date: Thu, 21 Dec 2023 02:54:40 +0800 Subject: [PATCH] refactor: migrate from shiki to shikiji (#2520) --- .dockerignore | 1 - .gitignore | 1 - README.md | 2 +- composables/{shiki.ts => shikiji.ts} | 37 +++---- composables/tiptap.ts | 4 +- composables/tiptap/shiki.ts | 129 ------------------------- composables/tiptap/shikiji-parser.ts | 20 ++++ composables/tiptap/shikiji.ts | 25 +++++ config/pwa.ts | 2 +- docs/content/1.guide/3.contributing.md | 2 +- mocks/prosemirror.ts | 4 +- nuxt.config.ts | 7 +- package.json | 4 +- pnpm-lock.yaml | 78 ++++++++++----- scripts/prepare.ts | 6 -- service-worker/sw.ts | 17 +--- tests/nuxt/content-rich.test.ts | 8 -- 17 files changed, 124 insertions(+), 223 deletions(-) rename composables/{shiki.ts => shikiji.ts} (61%) delete mode 100644 composables/tiptap/shiki.ts create mode 100644 composables/tiptap/shikiji-parser.ts create mode 100644 composables/tiptap/shikiji.ts diff --git a/.dockerignore b/.dockerignore index 46154788..90ce5d10 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,7 +11,6 @@ dist .netlify/ .eslintcache -public/shiki public/emojis *~ diff --git a/.gitignore b/.gitignore index a7cd793f..b61e1356 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ dist .eslintcache elk-translation-status.json -public/shiki public/emojis *~ diff --git a/README.md b/README.md index 802a00c2..e56c330d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ You can consult the [PWA documentation](https://docs.elk.zone/pwa) to learn more - [UnoCSS](https://uno.antfu.me/) - The instant on-demand atomic CSS engine - [Iconify](https://github.com/iconify/icon-sets#iconify-icon-sets-in-json-format) - Iconify icon sets in JSON format - [Masto.js](https://neet.github.io/masto.js) - Mastodon API client in TypeScript -- [shiki](https://shiki.matsu.io/) - A beautiful Syntax Highlighter +- [shikiji](https://shikiji.netlify.app/) - A beautiful and powerful syntax highlighter - [vite-plugin-pwa](https://github.com/vite-pwa/vite-plugin-pwa) - Prompt for update, Web Push Notifications and Web Share Target API ## 👨‍💻 Contributors diff --git a/composables/shiki.ts b/composables/shikiji.ts similarity index 61% rename from composables/shiki.ts rename to composables/shikiji.ts index d5ade98e..6a2476d5 100644 --- a/composables/shiki.ts +++ b/composables/shikiji.ts @@ -1,16 +1,15 @@ -import type { Highlighter, Lang } from 'shiki-es' +import { type Highlighter, type BuiltinLanguage as Lang } from 'shikiji' -const shiki = ref() +const highlighter = ref() const registeredLang = ref(new Map()) -let shikiImport: Promise | undefined +let shikijiImport: Promise | undefined export function useHighlighter(lang: Lang) { - if (!shikiImport) { - shikiImport = import('shiki-es') - .then(async (r) => { - r.setCDN('/shiki/') - shiki.value = await r.getHighlighter({ + if (!shikijiImport) { + shikijiImport = import('shikiji') + .then(async ({ getHighlighter }) => { + highlighter.value = await getHighlighter({ themes: [ 'vitesse-dark', 'vitesse-light', @@ -24,27 +23,27 @@ export function useHighlighter(lang: Lang) { }) } - if (!shiki.value) + if (!highlighter.value) return undefined if (!registeredLang.value.get(lang)) { - shiki.value.loadLanguage(lang) + highlighter.value.loadLanguage(lang) .then(() => { registeredLang.value.set(lang, true) }) .catch(() => { const fallbackLang = 'md' - shiki.value?.loadLanguage(fallbackLang).then(() => { + highlighter.value?.loadLanguage(fallbackLang).then(() => { registeredLang.value.set(fallbackLang, true) }) }) return undefined } - return shiki.value + return highlighter.value } -export function useShikiTheme() { +function useShikijiTheme() { return useColorMode().value === 'dark' ? 'vitesse-dark' : 'vitesse-light' } @@ -61,16 +60,12 @@ function escapeHtml(text: string) { } export function highlightCode(code: string, lang: Lang) { - const shiki = useHighlighter(lang) - if (!shiki) + const highlighter = useHighlighter(lang) + if (!highlighter) return escapeHtml(code) - return shiki.codeToHtml(code, { + return highlighter.codeToHtml(code, { lang, - theme: useShikiTheme(), + theme: useShikijiTheme(), }) } - -export function useShiki() { - return shiki -} diff --git a/composables/tiptap.ts b/composables/tiptap.ts index a9a80dac..557b27fd 100644 --- a/composables/tiptap.ts +++ b/composables/tiptap.ts @@ -14,7 +14,7 @@ import { Plugin } from 'prosemirror-state' import type { Ref } from 'vue' import { TiptapEmojiSuggestion, TiptapHashtagSuggestion, TiptapMentionSuggestion } from './tiptap/suggestion' -import { TiptapPluginCodeBlockShiki } from './tiptap/shiki' +import { TiptapPluginCodeBlockShikiji } from './tiptap/shikiji' import { TiptapPluginCustomEmoji } from './tiptap/custom-emoji' import { TiptapPluginEmoji } from './tiptap/emoji' @@ -70,7 +70,7 @@ export function useTiptap(options: UseTiptapOptions) { Placeholder.configure({ placeholder: () => placeholder.value!, }), - TiptapPluginCodeBlockShiki, + TiptapPluginCodeBlockShikiji, History.configure({ depth: 10, }), diff --git a/composables/tiptap/shiki.ts b/composables/tiptap/shiki.ts deleted file mode 100644 index fef20052..00000000 --- a/composables/tiptap/shiki.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { CodeBlockOptions } from '@tiptap/extension-code-block' -import CodeBlock from '@tiptap/extension-code-block' -import { VueNodeViewRenderer } from '@tiptap/vue-3' - -import { findChildren } from '@tiptap/core' -import type { Node as ProsemirrorNode } from 'prosemirror-model' -import { Plugin, PluginKey } from 'prosemirror-state' -import { Decoration, DecorationSet } from 'prosemirror-view' -import TiptapCodeBlock from '~/components/tiptap/TiptapCodeBlock.vue' - -export interface CodeBlockShikiOptions extends CodeBlockOptions { - defaultLanguage: string | null | undefined -} - -export const TiptapPluginCodeBlockShiki = CodeBlock.extend({ - addOptions() { - return { - ...this.parent?.(), - defaultLanguage: null, - } - }, - - addProseMirrorPlugins() { - return [ - ...this.parent?.() || [], - ProseMirrorShikiPlugin({ - name: this.name, - }), - ] - }, - - addNodeView() { - return VueNodeViewRenderer(TiptapCodeBlock) - }, -}) - -function getDecorations({ - doc, - name, -}: { doc: ProsemirrorNode; name: string }) { - const decorations: Decoration[] = [] - - findChildren(doc, node => node.type.name === name) - .forEach((block) => { - let from = block.pos + 1 - const language = block.node.attrs.language - - const shiki = useHighlighter(language) - - if (!shiki) - return - - const lines = shiki.codeToThemedTokens(block.node.textContent, language, useShikiTheme()) - - lines.forEach((line) => { - line.forEach((token) => { - const decoration = Decoration.inline(from, from + token.content.length, { - style: `color: ${token.color}`, - }) - - decorations.push(decoration) - from += token.content.length - }) - from += 1 - }) - }) - - return DecorationSet.create(doc, decorations) -} - -function ProseMirrorShikiPlugin({ name }: { name: string }) { - const plugin: Plugin = new Plugin({ - key: new PluginKey('shiki'), - - state: { - init: (_, { doc }) => getDecorations({ - doc, - name, - }), - apply: (transaction, decorationSet, oldState, newState) => { - const oldNodeName = oldState.selection.$head.parent.type.name - const newNodeName = newState.selection.$head.parent.type.name - const oldNodes = findChildren(oldState.doc, node => node.type.name === name) - const newNodes = findChildren(newState.doc, node => node.type.name === name) - - if ( - transaction.docChanged - // Apply decorations if: - && ( - // selection includes named node, - [oldNodeName, newNodeName].includes(name) - // OR transaction adds/removes named node, - || newNodes.length !== oldNodes.length - // OR transaction has changes that completely encapsulte a node - // (for example, a transaction that affects the entire document). - // Such transactions can happen during collab syncing via y-prosemirror, for example. - || transaction.steps.some((step) => { - // @ts-expect-error cast - return step.from !== undefined - // @ts-expect-error cast - && step.to !== undefined - && oldNodes.some((node) => { - // @ts-expect-error cast - return node.pos >= step.from - // @ts-expect-error cast - && node.pos + node.node.nodeSize <= step.to - }) - }) - ) - ) { - return getDecorations({ - doc: transaction.doc, - name, - }) - } - - return decorationSet.map(transaction.mapping, transaction.doc) - }, - }, - - props: { - decorations(state) { - return plugin.getState(state) - }, - }, - }) - - return plugin -} diff --git a/composables/tiptap/shikiji-parser.ts b/composables/tiptap/shikiji-parser.ts new file mode 100644 index 00000000..1072d0cc --- /dev/null +++ b/composables/tiptap/shikiji-parser.ts @@ -0,0 +1,20 @@ +import { type Parser, createParser } from 'prosemirror-highlight/shikiji' +import type { BuiltinLanguage } from 'shikiji/langs' + +let parser: Parser | undefined + +export const shikijiParser: Parser = (options) => { + const lang = options.language ?? 'text' + + // Register the language if it's not yet registered + const highlighter = useHighlighter(lang as BuiltinLanguage) + + // If the language is not loaded, we return an empty set of decorations + if (!highlighter) + return [] + + if (!parser) + parser = createParser(highlighter) + + return parser(options) +} diff --git a/composables/tiptap/shikiji.ts b/composables/tiptap/shikiji.ts new file mode 100644 index 00000000..30f3c536 --- /dev/null +++ b/composables/tiptap/shikiji.ts @@ -0,0 +1,25 @@ +import CodeBlock from '@tiptap/extension-code-block' +import { VueNodeViewRenderer } from '@tiptap/vue-3' + +import { createHighlightPlugin } from 'prosemirror-highlight' +import { shikijiParser } from './shikiji-parser' +import TiptapCodeBlock from '~/components/tiptap/TiptapCodeBlock.vue' + +export const TiptapPluginCodeBlockShikiji = CodeBlock.extend({ + addOptions() { + return { + ...this.parent?.(), + defaultLanguage: null, + } + }, + + addProseMirrorPlugins() { + return [ + createHighlightPlugin({ parser: shikijiParser, nodeTypes: ['codeBlock'] }), + ] + }, + + addNodeView() { + return VueNodeViewRenderer(TiptapCodeBlock) + }, +}) diff --git a/config/pwa.ts b/config/pwa.ts index c6ce1016..4ae5e35b 100644 --- a/config/pwa.ts +++ b/config/pwa.ts @@ -14,7 +14,7 @@ export const pwa: VitePWANuxtOptions = { manifest: false, injectManifest: { globPatterns: ['**/*.{js,json,css,html,txt,svg,png,ico,webp,woff,woff2,ttf,eot,otf,wasm}'], - globIgnores: ['emojis/**', 'shiki/**', 'manifest**.webmanifest'], + globIgnores: ['emojis/**', 'manifest**.webmanifest'], }, devOptions: { enabled: process.env.VITE_DEV_PWA === 'true', diff --git a/docs/content/1.guide/3.contributing.md b/docs/content/1.guide/3.contributing.md index 456e28b9..17db5d8a 100644 --- a/docs/content/1.guide/3.contributing.md +++ b/docs/content/1.guide/3.contributing.md @@ -49,5 +49,5 @@ nr test - [UnoCSS](https://uno.antfu.me/) - The instant on-demand atomic CSS engine - [Iconify](https://github.com/iconify/icon-sets#iconify-icon-sets-in-json-format) - Iconify icon sets in JSON format - [Masto.js](https://neet.github.io/masto.js) - Mastodon API client in TypeScript -- [shiki](https://shiki.matsu.io/) - A beautiful Syntax Highlighter +- [shikiji](https://shikiji.netlify.app/) - A beautiful and powerful syntax highlighter - [vite-plugin-pwa](https://github.com/vite-pwa/vite-plugin-pwa) - Prompt for update, Web Push Notifications and Web Share Target API diff --git a/mocks/prosemirror.ts b/mocks/prosemirror.ts index 84e65835..746ca252 100644 --- a/mocks/prosemirror.ts +++ b/mocks/prosemirror.ts @@ -2,7 +2,7 @@ import proxy from 'unenv/runtime/mock/proxy' export const Plugin = proxy export const PluginKey = proxy -export const Decoration = proxy -export const DecorationSet = proxy +export const createParser = proxy +export const createHighlightPlugin = proxy export { proxy as default } diff --git a/nuxt.config.ts b/nuxt.config.ts index 41cdd882..4d27c2eb 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -157,11 +157,6 @@ export default defineNuxtConfig({ maxAge: 24 * 60 * 60 * 365, // 1 year (versioned) baseURL: '/fonts', }, - { - dir: '~/public/shiki', - maxAge: 24 * 60 * 60 * 365, // 1 year, matching service worker - baseURL: '/shiki', - }, ], }, sourcemap: isDevelopment, @@ -179,7 +174,7 @@ export default defineNuxtConfig({ const alias = config.resolve!.alias as Record for (const dep of ['eventemitter3', 'isomorphic-ws']) alias[dep] = resolve('./mocks/class') - for (const dep of ['shiki-es', 'fuse.js']) + for (const dep of ['fuse.js']) alias[dep] = 'unenv/runtime/mock/proxy' const resolver = createResolver(import.meta.url) diff --git a/package.json b/package.json index cbefe476..439e3ede 100644 --- a/package.json +++ b/package.json @@ -83,9 +83,9 @@ "page-lifecycle": "^0.1.2", "pinia": "^2.1.4", "postcss-nested": "^6.0.1", + "prosemirror-highlight": "^0.3.3", "rollup-plugin-node-polyfills": "^0.2.1", - "shiki": "^0.14.3", - "shiki-es": "^0.2.0", + "shikiji": "^0.9.9", "simple-git": "^3.19.1", "slimeform": "^0.9.1", "stale-dep": "^0.7.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4040dafa..7c67cd8c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -176,15 +176,15 @@ importers: postcss-nested: specifier: ^6.0.1 version: 6.0.1(postcss@8.4.32) + prosemirror-highlight: + specifier: ^0.3.3 + version: 0.3.3(prosemirror-model@1.19.2)(prosemirror-state@1.4.3)(prosemirror-view@1.31.5)(shikiji@0.9.9) rollup-plugin-node-polyfills: specifier: ^0.2.1 version: 0.2.1 - shiki: - specifier: ^0.14.3 - version: 0.14.3 - shiki-es: - specifier: ^0.2.0 - version: 0.2.0 + shikiji: + specifier: ^0.9.9 + version: 0.9.9 simple-git: specifier: ^3.19.1 version: 3.21.0 @@ -6318,10 +6318,6 @@ packages: resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==} engines: {node: '>=12'} - /ansi-sequence-parser@1.1.0: - resolution: {integrity: sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==} - dev: false - /ansi-styles@3.2.1: resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} engines: {node: '>=4'} @@ -12095,6 +12091,47 @@ packages: prosemirror-view: 1.31.5 dev: false + /prosemirror-highlight@0.3.3(prosemirror-model@1.19.2)(prosemirror-state@1.4.3)(prosemirror-view@1.31.5)(shikiji@0.9.9): + resolution: {integrity: sha512-tOGyPvmRKZ49ubzKmFIwiwS7CNXlU9g/D4zZLaHGzXLVNVnBrmbDOajZ4eP0lylOAWPxZN+vrFZ9DwrtyikuoA==} + peerDependencies: + '@types/hast': ^3.0.0 + highlight.js: ^11.9.0 + lowlight: ^3.1.0 + prosemirror-model: ^1.19.3 + prosemirror-state: ^1.4.3 + prosemirror-transform: ^1.8.0 + prosemirror-view: ^1.32.4 + refractor: ^4.8.1 + shiki: ^0.14.6 + shikiji: ^0.8.0 || ^0.9.0 + peerDependenciesMeta: + '@types/hast': + optional: true + highlight.js: + optional: true + lowlight: + optional: true + prosemirror-model: + optional: true + prosemirror-state: + optional: true + prosemirror-transform: + optional: true + prosemirror-view: + optional: true + refractor: + optional: true + shiki: + optional: true + shikiji: + optional: true + dependencies: + prosemirror-model: 1.19.2 + prosemirror-state: 1.4.3 + prosemirror-view: 1.31.5 + shikiji: 0.9.9 + dev: false + /prosemirror-history@1.3.2: resolution: {integrity: sha512-/zm0XoU/N/+u7i5zepjmZAEnpvjDtzoPWW6VmKptcAnPadN/SStsBjMImdCEbb3seiNTpveziPTIrXQbHLtU1g==} dependencies: @@ -12887,17 +12924,14 @@ packages: resolution: {integrity: sha512-e+/aueHx0YeIEut6RXC6K8gSf0PykwZiHD7q7AHtpTW8Kd8TpFUIWqTwhAnrGjOyOMyrwv+syr5WPagMpDpVYQ==} dev: true - /shiki-es@0.2.0: - resolution: {integrity: sha512-RbRMD+IuJJseSZljDdne9ThrUYrwBwJR04FvN4VXpfsU3MNID5VJGHLAD5je/HGThCyEKNgH+nEkSFEWKD7C3Q==} + /shikiji-core@0.9.9: + resolution: {integrity: sha512-qu5Qq7Co6JIMY312J9Ek6WYjXieeyJT/fIqmkcjF4MdnMNlUnhSqPo8/42g5UdPgdyTCwijS7Nhg8DfLSLodkg==} dev: false - /shiki@0.14.3: - resolution: {integrity: sha512-U3S/a+b0KS+UkTyMjoNojvTgrBHjgp7L6ovhFVZsXmBGnVdQ4K4U9oK0z63w538S91ATngv1vXigHCSWOwnr+g==} + /shikiji@0.9.9: + resolution: {integrity: sha512-/S3unr/0mZTstNOuAmNDEufeimtqeQb8lXvPMLsYfDvqyfmG6334bO2xmDzD0kfxH2y8gnFgSWAJpdEzksmYXg==} dependencies: - ansi-sequence-parser: 1.1.0 - jsonc-parser: 3.2.0 - vscode-oniguruma: 1.7.0 - vscode-textmate: 8.0.0 + shikiji-core: 0.9.9 dev: false /side-channel@1.0.4: @@ -14708,14 +14742,6 @@ packages: dependencies: vscode-languageserver-protocol: 3.16.0 - /vscode-oniguruma@1.7.0: - resolution: {integrity: sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==} - dev: false - - /vscode-textmate@8.0.0: - resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==} - dev: false - /vscode-uri@3.0.7: resolution: {integrity: sha512-eOpPHogvorZRobNqJGhapa0JdwaxpjVvyBp0QIUMRMSf8ZAlqOdEquKuRmw9Qwu0qXtJIWqFtMkmvJjUZmMjVA==} diff --git a/scripts/prepare.ts b/scripts/prepare.ts index fc78ea61..2e6e2fbe 100644 --- a/scripts/prepare.ts +++ b/scripts/prepare.ts @@ -5,12 +5,6 @@ import { colorsMap } from './generate-themes' const dereference = process.platform === 'win32' ? true : undefined -await fs.copy('node_modules/shiki-es/dist/assets', 'public/shiki/', { - dereference, - filter: src => src === 'node_modules/shiki/' || src.includes('languages') || src.includes('dist'), -}) -await fs.copy('node_modules/theme-vitesse/themes', 'public/shiki/themes', { dereference }) -await fs.copy('node_modules/theme-vitesse/themes', 'node_modules/shiki/themes', { overwrite: true, dereference }) await fs.copy(`node_modules/${iconifyEmojiPackage}/icons`, `public/emojis/${emojiPrefix}`, { overwrite: true, dereference }) await fs.writeJSON('constants/themes.json', colorsMap, { spaces: 2, EOL: '\n' }) diff --git a/service-worker/sw.ts b/service-worker/sw.ts index 1c53ff21..d0b2816d 100644 --- a/service-worker/sw.ts +++ b/service-worker/sw.ts @@ -39,9 +39,7 @@ if (import.meta.env.PROD) { /^\/oauth\//, /^\/signin\//, /^\/web-share-target\//, - // exclude shiki: has its own cache - /^\/shiki\//, - // exclude shiki: has its own cache + // exclude emoji: has its own cache /^\/emojis\//, // exclude sw: if the user navigates to it, fallback to index.html /^\/sw.js$/, @@ -65,19 +63,6 @@ if (import.meta.env.PROD) { ], }), ) - // include shiki cache - registerRoute( - ({ sameOrigin, url }) => - sameOrigin && url.pathname.startsWith('/shiki/'), - new StaleWhileRevalidate({ - cacheName: 'elk-shiki', - plugins: [ - new CacheableResponsePlugin({ statuses: [200] }), - // 365 days max - new ExpirationPlugin({ purgeOnQuotaError: true, maxAgeSeconds: 60 * 60 * 24 * 365 }), - ], - }), - ) // include emoji icons registerRoute( ({ sameOrigin, request, url }) => diff --git a/tests/nuxt/content-rich.test.ts b/tests/nuxt/content-rich.test.ts index 5ffef951..a2cf433e 100644 --- a/tests/nuxt/content-rich.test.ts +++ b/tests/nuxt/content-rich.test.ts @@ -279,14 +279,6 @@ vi.mock('vue-router', async () => { } }) -vi.mock('shiki-es', async (importOriginal) => { - const mod = await importOriginal() - return { - ...(mod as any), - setCDN() {}, - } -}) - mockComponent('ContentMentionGroup', { setup(props, { slots }) { return () => h('mention-group', null, { default: () => slots?.default?.() })