| #!/usr/bin/env node |
|
|
| import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'fs'; |
| import { join, dirname, basename, extname } from 'path'; |
| import { fileURLToPath } from 'url'; |
| import matter from 'gray-matter'; |
| import fetch from 'node-fetch'; |
|
|
| const __filename = fileURLToPath(import.meta.url); |
| const __dirname = dirname(__filename); |
|
|
| |
| const DEFAULT_INPUT = join(__dirname, 'output'); |
| const DEFAULT_OUTPUT = join(__dirname, 'output'); |
| const STATIC_FRONTMATTER_PATH = join(__dirname, 'static', 'frontmatter.mdx'); |
|
|
| function parseArgs() { |
| const args = process.argv.slice(2); |
| const config = { |
| input: DEFAULT_INPUT, |
| output: DEFAULT_OUTPUT, |
| }; |
|
|
| for (const arg of args) { |
| if (arg.startsWith('--input=')) { |
| config.input = arg.substring('--input='.length); |
| } else if (arg.startsWith('--output=')) { |
| config.output = arg.substring('--output='.length); |
| } else if (arg === '--help' || arg === '-h') { |
| console.log(` |
| π Notion Markdown to MDX Converter |
| |
| Usage: |
| node mdx-converter.mjs [options] |
| |
| Options: |
| --input=PATH Input directory or file (default: ${DEFAULT_INPUT}) |
| --output=PATH Output directory (default: ${DEFAULT_OUTPUT}) |
| --help, -h Show this help |
| |
| Examples: |
| # Convert all markdown files in output directory |
| node mdx-converter.mjs |
| |
| # Convert specific file |
| node mdx-converter.mjs --input=article.md --output=converted/ |
| |
| # Convert directory |
| node mdx-converter.mjs --input=markdown-files/ --output=mdx-files/ |
| `); |
| process.exit(0); |
| } else if (!config.input) { |
| config.input = arg; |
| } else if (!config.output) { |
| config.output = arg; |
| } |
| } |
| return config; |
| } |
|
|
| |
| |
| |
| const usedComponents = new Set(); |
|
|
| |
| |
| |
| const imageImports = new Map(); |
|
|
| |
| |
| |
| const externalImagesToDownload = new Map(); |
|
|
| |
| |
| |
| |
| |
| function generateImageVarName(src) { |
| |
| const filename = src.split('/').pop().replace(/\.[^.]+$/, ''); |
| return filename.replace(/[^a-zA-Z0-9]/g, '_').replace(/^[0-9]/, 'img_$&'); |
| } |
|
|
| |
| |
| |
| |
| |
| function isExternalImageUrl(url) { |
| try { |
| const urlObj = new URL(url); |
| |
| return urlObj.protocol === 'http:' || urlObj.protocol === 'https:'; |
| } catch { |
| return false; |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| async function extractTwitterImageUrl(tweetUrl) { |
| try { |
| const response = await fetch(tweetUrl, { |
| headers: { |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' |
| } |
| }); |
|
|
| if (!response.ok) { |
| return null; |
| } |
|
|
| const html = await response.text(); |
|
|
| |
| const metaImageMatch = html.match(/<meta property="og:image" content="([^"]+)"/); |
| if (metaImageMatch) { |
| let imageUrl = metaImageMatch[1]; |
| |
| if (imageUrl.includes('?')) { |
| imageUrl = imageUrl.split('?')[0] + '?format=jpg&name=large'; |
| } |
| return imageUrl; |
| } |
|
|
| |
| const pbsMatch = html.match(/https:\/\/pbs\.twimg\.com\/media\/([^"?]+)/); |
| if (pbsMatch) { |
| return `https://pbs.twimg.com/media/${pbsMatch[1]}?format=jpg&name=large`; |
| } |
|
|
| return null; |
| } catch (error) { |
| console.log(` β οΈ Failed to extract Twitter image: ${error.message}`); |
| return null; |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| async function downloadExternalImage(imageUrl, outputDir) { |
| try { |
| console.log(` π Downloading external URL: ${imageUrl}`); |
|
|
| |
| if (!existsSync(outputDir)) { |
| mkdirSync(outputDir, { recursive: true }); |
| } |
|
|
| let actualImageUrl = imageUrl; |
|
|
| |
| if (imageUrl.includes('twitter.com/') || imageUrl.includes('x.com/')) { |
| console.log(` π¦ Detected Twitter/X URL, attempting to extract image...`); |
| const extractedUrl = await extractTwitterImageUrl(imageUrl); |
| if (extractedUrl) { |
| actualImageUrl = extractedUrl; |
| console.log(` β
Extracted image URL: ${extractedUrl}`); |
| } else { |
| console.log(` β οΈ Could not automatically extract image from Twitter/X`); |
| console.log(` π‘ Manual download required:`); |
| console.log(` 1. Open ${imageUrl} in your browser`); |
| console.log(` 2. Right-click on the image and "Save image as..."`); |
| console.log(` 3. Save it to: app/src/content/assets/image/`); |
| throw new Error('Twitter/X images require manual download'); |
| } |
| } |
|
|
| |
| const urlObj = new URL(actualImageUrl); |
| const pathname = urlObj.pathname; |
|
|
| |
| let extension = 'jpg'; |
| if (pathname.includes('.')) { |
| const urlExtension = pathname.split('.').pop().toLowerCase(); |
| if (['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'bmp', 'tiff'].includes(urlExtension)) { |
| extension = urlExtension; |
| } |
| } |
|
|
| |
| const filename = `external_${Date.now()}_${Math.random().toString(36).substr(2, 9)}.${extension}`; |
| const localPath = join(outputDir, filename); |
|
|
| |
| const response = await fetch(actualImageUrl, { |
| headers: { |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' |
| } |
| }); |
|
|
| if (!response.ok) { |
| throw new Error(`HTTP ${response.status}: ${response.statusText}`); |
| } |
|
|
| const buffer = await response.buffer(); |
|
|
| |
| if (buffer.length === 0) { |
| throw new Error('Empty response'); |
| } |
|
|
| |
| const contentType = response.headers.get('content-type'); |
| if (contentType && contentType.includes('text/html')) { |
| throw new Error('Downloaded content is HTML, not an image'); |
| } |
|
|
| |
| writeFileSync(localPath, buffer); |
|
|
| console.log(` β
Downloaded: ${filename} (${buffer.length} bytes)`); |
| return localPath; |
|
|
| } catch (error) { |
| console.log(` β Failed to download ${imageUrl}: ${error.message}`); |
| throw error; |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| async function processExternalImages(content, outputDir) { |
| console.log(' π Processing external images...'); |
|
|
| let processedCount = 0; |
| let downloadedCount = 0; |
|
|
| |
| const externalImageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; |
| let match; |
| const externalImages = new Map(); |
|
|
| |
| while ((match = externalImageRegex.exec(content)) !== null) { |
| const alt = match[1]; |
| const url = match[2]; |
|
|
| if (isExternalImageUrl(url)) { |
| externalImages.set(url, alt); |
| console.log(` π Found external image: ${url}`); |
| } |
| } |
|
|
| if (externalImages.size === 0) { |
| console.log(' βΉοΈ No external images found'); |
| return content; |
| } |
|
|
| |
| let processedContent = content; |
|
|
| for (const [url, alt] of externalImages) { |
| try { |
| |
| const localPath = await downloadExternalImage(url, outputDir); |
| const relativePath = `./assets/image/${basename(localPath)}`; |
|
|
| |
| processedContent = processedContent.replace( |
| new RegExp(`!\\[${alt.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\]\\(${url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\)`, 'g'), |
| `` |
| ); |
|
|
| downloadedCount++; |
| processedCount++; |
|
|
| } catch (error) { |
| console.log(` β οΈ Skipping external image due to download failure: ${url}`); |
| } |
| } |
|
|
| if (downloadedCount > 0) { |
| console.log(` β
Downloaded ${downloadedCount} external image(s)`); |
| } |
|
|
| return processedContent; |
| } |
|
|
| |
| |
| |
| |
| function detectAstroComponents(content) { |
| console.log(' π Detecting Astro components in content...'); |
|
|
| let detectedCount = 0; |
|
|
| |
| const knownComponents = [ |
| 'HtmlEmbed', 'Image', 'Note', 'Sidenote', 'Wide', 'FullWidth', |
| 'Accordion', 'Quote', 'Reference', 'Glossary', 'Stack', 'ThemeToggle', |
| 'RawHtml', 'HfUser' |
| ]; |
|
|
| |
| |
| const componentMatches = content.match(/<([A-Z][a-zA-Z0-9]*)\s*[^>]*\/?>/g); |
|
|
| if (componentMatches) { |
| for (const match of componentMatches) { |
| |
| const componentMatch = match.match(/<([A-Z][a-zA-Z0-9]*)/); |
| if (componentMatch) { |
| const componentName = componentMatch[1]; |
|
|
| |
| if (knownComponents.includes(componentName) && !usedComponents.has(componentName)) { |
| usedComponents.add(componentName); |
| detectedCount++; |
| console.log(` π¦ Found component: ${componentName}`); |
| } |
| } |
| } |
| } |
|
|
| if (detectedCount > 0) { |
| console.log(` β
Detected ${detectedCount} new Astro component(s)`); |
| } else { |
| console.log(` βΉοΈ No new Astro components detected`); |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| function addComponentImports(content) { |
| console.log(' π¦ Adding component and image imports...'); |
|
|
| let imports = []; |
|
|
| |
| if (usedComponents.size > 0) { |
| const componentImports = Array.from(usedComponents) |
| .map(component => `import ${component} from '../components/${component}.astro';`); |
| imports.push(...componentImports); |
| console.log(` β
Importing components: ${Array.from(usedComponents).join(', ')}`); |
| } |
|
|
| |
| if (imageImports.size > 0) { |
| const imageImportStatements = Array.from(imageImports.entries()) |
| .map(([src, varName]) => `import ${varName} from '${src}';`); |
| imports.push(...imageImportStatements); |
| console.log(` β
Importing ${imageImports.size} image(s)`); |
| } |
|
|
| if (imports.length === 0) { |
| console.log(' βΉοΈ No imports needed'); |
| return content; |
| } |
|
|
| const importBlock = imports.join('\n'); |
|
|
| |
| const frontmatterEnd = content.indexOf('---', 3) + 3; |
| if (frontmatterEnd > 2) { |
| return content.slice(0, frontmatterEnd) + '\n\n' + importBlock + '\n\n' + content.slice(frontmatterEnd); |
| } else { |
| |
| return importBlock + '\n\n' + content; |
| } |
| } |
|
|
|
|
| |
| |
| |
| |
| function loadStaticFrontmatter() { |
| try { |
| if (existsSync(STATIC_FRONTMATTER_PATH)) { |
| const staticContent = readFileSync(STATIC_FRONTMATTER_PATH, 'utf8'); |
| const { data } = matter(staticContent); |
| console.log(' β
Loaded static frontmatter from file'); |
| return data; |
| } |
| console.log(' βΉοΈ No static frontmatter file found'); |
| return {}; |
| } catch (error) { |
| console.log(` β οΈ Failed to load static frontmatter: ${error.message}`); |
| return {}; |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| async function ensureFrontmatter(content, pageId = null, notionToken = null) { |
| console.log(' π Ensuring proper frontmatter...'); |
|
|
| |
| const staticData = loadStaticFrontmatter(); |
|
|
| if (!content.startsWith('---')) { |
| |
| let baseData = { ...staticData }; |
|
|
| |
| if (!baseData.title) baseData.title = 'Article'; |
| if (!baseData.published) { |
| baseData.published = new Date().toLocaleDateString('en-US', { |
| year: 'numeric', |
| month: 'short', |
| day: '2-digit' |
| }); |
| } |
| if (baseData.tableOfContentsAutoCollapse === undefined) { |
| baseData.tableOfContentsAutoCollapse = true; |
| } |
|
|
| const frontmatter = matter.stringify('', baseData); |
| console.log(' β
Applied static frontmatter to content without frontmatter'); |
| return frontmatter + content; |
| } |
|
|
| |
| try { |
| const { data: existingData, content: body } = matter(content); |
|
|
| |
| const mergedData = { ...existingData, ...staticData }; |
|
|
| |
| if (!mergedData.title) mergedData.title = 'Article'; |
| if (!mergedData.published) { |
| mergedData.published = new Date().toLocaleDateString('en-US', { |
| year: 'numeric', |
| month: 'short', |
| day: '2-digit' |
| }); |
| } |
| if (mergedData.tableOfContentsAutoCollapse === undefined) { |
| mergedData.tableOfContentsAutoCollapse = true; |
| } |
|
|
| const enhancedContent = matter.stringify(body, mergedData); |
| console.log(' β
Merged static and existing frontmatter'); |
| return enhancedContent; |
| } catch (error) { |
| console.log(' β οΈ Could not parse frontmatter, keeping as is'); |
| return content; |
| } |
| } |
|
|
| |
| |
| |
| |
| function generateBasicFrontmatter() { |
| const currentDate = new Date().toLocaleDateString('en-US', { |
| year: 'numeric', |
| month: 'short', |
| day: '2-digit' |
| }); |
| return `--- |
| title: "Notion Article" |
| published: "${currentDate}" |
| tableOfContentsAutoCollapse: true |
| --- |
| |
| `; |
| } |
|
|
|
|
| |
| |
| |
| |
| |
| function isTableLine(line) { |
| const trimmed = line.trim(); |
| return trimmed.startsWith('|') && trimmed.endsWith('|'); |
| } |
|
|
| |
| |
| |
| |
| |
| function isListItem(line) { |
| const trimmed = line.trim(); |
| |
| return /^\s*[\*\-\+]\s/.test(trimmed) || /^\s*\d+\.\s/.test(trimmed); |
| } |
|
|
| |
| |
| |
| |
| |
| function addBlankLineAfterTablesAndLists(content) { |
| console.log(' π Adding blank lines after tables and lists...'); |
|
|
| let addedTableCount = 0; |
| let addedListCount = 0; |
| const lines = content.split('\n'); |
| const result = []; |
|
|
| for (let i = 0; i < lines.length; i++) { |
| result.push(lines[i]); |
|
|
| |
| if (isTableLine(lines[i])) { |
| |
| let isLastTableLine = false; |
|
|
| |
| if (i + 1 >= lines.length || |
| lines[i + 1].trim() === '' || |
| !isTableLine(lines[i + 1])) { |
|
|
| |
| let tableLineCount = 0; |
| for (let j = i; j >= 0 && isTableLine(lines[j]); j--) { |
| tableLineCount++; |
| } |
|
|
| |
| if (tableLineCount >= 2) { |
| isLastTableLine = true; |
| } |
| } |
|
|
| if (isLastTableLine) { |
| addedTableCount++; |
| result.push(''); |
| } |
| } |
| |
| else if (isListItem(lines[i])) { |
| |
| let isLastListItem = false; |
|
|
| |
| if (i + 1 >= lines.length || |
| lines[i + 1].trim() === '' || |
| !isListItem(lines[i + 1])) { |
| isLastListItem = true; |
| } |
|
|
| if (isLastListItem) { |
| addedListCount++; |
| result.push(''); |
| } |
| } |
| } |
|
|
| if (addedTableCount > 0 || addedListCount > 0) { |
| console.log(` β
Added blank line after ${addedTableCount} table(s) and ${addedListCount} list(s)`); |
| } else { |
| console.log(' βΉοΈ No tables or lists found to process'); |
| } |
|
|
| return result.join('\n'); |
| } |
|
|
| |
| |
| |
| |
| |
| function transformMarkdownImages(content) { |
| console.log(' πΌοΈ Transforming markdown images to Image components...'); |
|
|
| let transformedCount = 0; |
|
|
| |
| content = content.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, src) => { |
| transformedCount++; |
|
|
| |
| let cleanSrc = src; |
| if (src.startsWith('/media/')) { |
| cleanSrc = src.replace('/media/', './assets/image/'); |
| } |
|
|
| |
| const varName = generateImageVarName(cleanSrc); |
|
|
| |
| if (!imageImports.has(cleanSrc)) { |
| imageImports.set(cleanSrc, varName); |
| } |
|
|
| |
| const finalAlt = alt || src.split('/').pop().split('.')[0]; |
|
|
| return `<Image src={${varName}} alt="${finalAlt}" />`; |
| }); |
|
|
| if (transformedCount > 0) { |
| console.log(` β
Transformed ${transformedCount} markdown image(s) to Image components with imports`); |
| } else { |
| console.log(' βΉοΈ No markdown images found to transform'); |
| } |
|
|
| return content; |
| } |
|
|
| |
| |
| |
| |
| |
| function addSpacingAroundComponents(content) { |
| console.log(' π Adding spacing around Astro components...'); |
|
|
| let processedContent = content; |
| let spacingCount = 0; |
|
|
| |
| const knownComponents = [ |
| 'HtmlEmbed', 'Image', 'Note', 'Sidenote', 'Wide', 'FullWidth', |
| 'Accordion', 'Quote', 'Reference', 'Glossary', 'Stack', 'ThemeToggle', |
| 'RawHtml', 'HfUser', 'Figure' |
| ]; |
|
|
| |
| for (const component of knownComponents) { |
| |
| |
| const withContentPattern = new RegExp(`(<${component}[^>]*>)([\\s\\S]*?)(<\\/${component}>)`, 'g'); |
| processedContent = processedContent.replace(withContentPattern, (match, openTag, content, closeTag) => { |
| spacingCount++; |
| |
| |
| const trimmedContent = content.trim(); |
| return `\n\n${openTag}\n${trimmedContent}\n${closeTag}\n\n`; |
| }); |
|
|
| |
| const selfClosingPattern = new RegExp(`(<${component}[^>]*\\/?>)`, 'g'); |
| processedContent = processedContent.replace(selfClosingPattern, (match) => { |
| spacingCount++; |
| return `\n\n${match}\n\n`; |
| }); |
| } |
|
|
| |
| processedContent = processedContent.replace(/\n{3,}/g, '\n\n'); |
|
|
| if (spacingCount > 0) { |
| console.log(` β
Added spacing around ${spacingCount} component(s)`); |
| } else { |
| console.log(' βΉοΈ No components found to add spacing around'); |
| } |
|
|
| return processedContent; |
| } |
|
|
| |
| |
| |
| |
| |
| function fixSmartQuotes(content) { |
| console.log(' βοΈ Fixing smart quotes (curly quotes)...'); |
|
|
| let fixedCount = 0; |
| const originalContent = content; |
|
|
| |
| content = content.replace(/\u201C/g, '"'); |
|
|
| |
| content = content.replace(/\u201D/g, '"'); |
|
|
| |
| content = content.replace(/\u2018/g, "'"); |
|
|
| |
| content = content.replace(/\u2019/g, "'"); |
|
|
| |
| fixedCount = 0; |
| for (let i = 0; i < originalContent.length; i++) { |
| const char = originalContent[i]; |
| if (char === '\u201C' || char === '\u201D' || char === '\u2018' || char === '\u2019') { |
| fixedCount++; |
| } |
| } |
|
|
| if (fixedCount > 0) { |
| console.log(` β
Fixed ${fixedCount} smart quote(s)`); |
| } else { |
| console.log(' βΉοΈ No smart quotes found'); |
| } |
|
|
| return content; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| async function processMdxContent(content, pageId = null, notionToken = null, outputDir = null) { |
| console.log('π§ Processing for Astro MDX compatibility...'); |
|
|
| |
| usedComponents.clear(); |
| imageImports.clear(); |
| externalImagesToDownload.clear(); |
|
|
| let processedContent = content; |
|
|
| |
| processedContent = fixSmartQuotes(processedContent); |
|
|
| |
| if (outputDir) { |
| |
| const externalImagesDir = join(outputDir, 'external-images'); |
| processedContent = await processExternalImages(processedContent, externalImagesDir); |
| } |
|
|
| |
| processedContent = await ensureFrontmatter(processedContent, pageId, notionToken); |
|
|
| |
| processedContent = addBlankLineAfterTablesAndLists(processedContent); |
|
|
| |
| processedContent = transformMarkdownImages(processedContent); |
|
|
| |
| processedContent = addSpacingAroundComponents(processedContent); |
|
|
| |
| detectAstroComponents(processedContent); |
|
|
| |
| processedContent = addComponentImports(processedContent); |
|
|
| return processedContent; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| async function convertFileToMdx(inputFile, outputDir, pageId = null, notionToken = null) { |
| const filename = basename(inputFile, '.md'); |
| const outputFile = join(outputDir, `${filename}.mdx`); |
|
|
| console.log(`π Converting: ${basename(inputFile)} β ${basename(outputFile)}`); |
|
|
| try { |
| const markdownContent = readFileSync(inputFile, 'utf8'); |
| const mdxContent = await processMdxContent(markdownContent, pageId, notionToken, outputDir); |
| writeFileSync(outputFile, mdxContent); |
|
|
| console.log(` β
Converted: ${outputFile}`); |
|
|
| |
| const inputSize = Math.round(markdownContent.length / 1024); |
| const outputSize = Math.round(mdxContent.length / 1024); |
| console.log(` π Input: ${inputSize}KB β Output: ${outputSize}KB`); |
|
|
| } catch (error) { |
| console.error(` β Failed to convert ${inputFile}: ${error.message}`); |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| async function convertToMdx(inputPath, outputDir, pageId = null, notionToken = null) { |
| console.log('π Notion Markdown to Astro MDX Converter'); |
| console.log(`π Input: ${inputPath}`); |
| console.log(`π Output: ${outputDir}`); |
|
|
| |
| if (!existsSync(inputPath)) { |
| console.error(`β Input not found: ${inputPath}`); |
| process.exit(1); |
| } |
|
|
| try { |
| |
| if (!existsSync(outputDir)) { |
| mkdirSync(outputDir, { recursive: true }); |
| } |
|
|
| let filesToConvert = []; |
|
|
| if (statSync(inputPath).isDirectory()) { |
| |
| const files = readdirSync(inputPath); |
| filesToConvert = files |
| .filter(file => file.endsWith('.md') && !file.includes('.raw.md')) |
| .map(file => join(inputPath, file)); |
| } else if (inputPath.endsWith('.md')) { |
| |
| filesToConvert = [inputPath]; |
| } else { |
| console.error('β Input must be a .md file or directory containing .md files'); |
| process.exit(1); |
| } |
|
|
| if (filesToConvert.length === 0) { |
| console.log('βΉοΈ No .md files found to convert'); |
| return; |
| } |
|
|
| console.log(`π Found ${filesToConvert.length} file(s) to convert`); |
|
|
| |
| for (const file of filesToConvert) { |
| await convertFileToMdx(file, outputDir, pageId, notionToken); |
| } |
|
|
| console.log(`β
Conversion completed! ${filesToConvert.length} file(s) processed`); |
|
|
| } catch (error) { |
| console.error('β Conversion failed:', error.message); |
| process.exit(1); |
| } |
| } |
|
|
| export { convertToMdx }; |
|
|
| function main() { |
| const config = parseArgs(); |
| convertToMdx(config.input, config.output); |
| console.log('π MDX conversion completed!'); |
| } |
|
|
| if (import.meta.url === `file://${process.argv[1]}`) { |
| main(); |
| } |
|
|