Skip to content

Commit

Permalink
fix: fix unexpected latex formula escape
Browse files Browse the repository at this point in the history
  • Loading branch information
pionxzh committed Sep 8, 2024
1 parent 67af2bf commit 30f0b62
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 13 deletions.
47 changes: 34 additions & 13 deletions src/exporter/markdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ export async function exportAllToMarkdown(fileNameFormat: string, apiConversatio
return true
}

const LatexRegex = /(\s\$\$.+\$\$\s|\s\$.+\$\s|\\\[.+\\\]|\\\(.+\\\))|(^\$$[\S\s]+^\$$)|(^\$\$[\S\s]+^\$\$$)/gm

function conversationToMarkdown(conversation: ConversationResult, metaList?: ExportMeta[]) {
const { id, title, model, modelSlug, createTime, updateTime, conversationNodes } = conversation
const source = `${baseUrl}/c/${id}`
Expand Down Expand Up @@ -125,25 +127,44 @@ function conversationToMarkdown(conversation: ConversationResult, metaList?: Exp

const author = transformAuthor(message.author)

let postSteps: Array<(input: string) => string> = []
const postSteps: Array<(input: string) => string> = []
if (message.author.role === 'assistant') {
postSteps = [...postSteps, input => transformFootNotes(input, message.metadata)]
postSteps.push(input => transformFootNotes(input, message.metadata))
}
// Only message from assistant will be reformatted
if (message.author.role === 'assistant') {
postSteps = [...postSteps, (input) => {
postSteps.push((input) => {
// Replace mathematical formula annotation
input = input
.replace(/^\\\[(.+)\\\]$/gm, '$$$$$1$$$$')
.replace(/\\\[/g, '$')
.replace(/\\\]/g, '$')
.replace(/\\\(/g, '$')
.replace(/\\\)/g, '$')

const matches = input.match(LatexRegex)

// Skip code block as the following steps can potentially break the code
if (!(/```/.test(input))) {
// Replace mathematical formula annotation
input = input
.replace(/^\\\[(.+)\\\]$/gm, '$$$$$1$$$$')
.replace(/\\\[/g, '$')
.replace(/\\\]/g, '$')
.replace(/\\\(/g, '$')
.replace(/\\\)/g, '$')
const isCodeBlock = /```/.test(input)
if (!isCodeBlock && matches) {
let index = 0
input = input.replace(LatexRegex, () => {
// Replace it with `╬${index}╬` to avoid markdown processor ruin the formula
return `╬${index++}╬`
})
}
return toMarkdown(fromMarkdown(input))
}]

let transformed = toMarkdown(fromMarkdown(input))

if (!isCodeBlock && matches) {
// Replace `╬${index}╬` back to the original latex
transformed = transformed.replace(/╬(\d+)╬/g, (_, index) => {
return matches[+index]
})
}

return transformed
})
}
const postProcess = (input: string) => postSteps.reduce((acc, fn) => fn(acc), input)
const content = transformContent(message.content, message.metadata, postProcess)
Expand Down
20 changes: 20 additions & 0 deletions src/exporter/text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ export async function exportToText() {
return true
}

const LatexRegex = /(\s\$\$.+\$\$\s|\s\$.+\$\s|\\\[.+\\\]|\\\(.+\\\))|(^\$$[\S\s]+^\$$)|(^\$\$[\S\s]+^\$\$$)/gm

function transformMessage(message?: ConversationNodeMessage) {
if (!message || !message.content) return null

Expand All @@ -53,6 +55,16 @@ function transformMessage(message?: ConversationNodeMessage) {

const author = transformAuthor(message.author)
let content = transformContent(message.content, message.metadata)

const matches = content.match(LatexRegex)
if (matches) {
let index = 0
content = content.replace(LatexRegex, () => {
// Replace it with `╬${index}╬` to avoid markdown processor ruin the formula
return `╬${index++}╬`
})
}

if (message.author.role === 'assistant') {
content = transformFootNotes(content, message.metadata)
}
Expand All @@ -61,6 +73,14 @@ function transformMessage(message?: ConversationNodeMessage) {
if (message.author.role === 'assistant' && content) {
content = reformatContent(content)
}

if (matches) {
// Replace `╬${index}╬` back to the original latex
content = content.replace(/╬(\d+)╬/g, (_, index) => {
return matches[+index]
})
}

return `${author}:\n${content}`
}

Expand Down

0 comments on commit 30f0b62

Please sign in to comment.