6bf8bebf51
CI / Test and Build (push) Failing after 1s
CI / Migrate Dev DB (push) Has been skipped
CI / Migrate DB (push) Has been skipped
CodeQL / Analyze actions (push) Has been cancelled
CodeQL / Analyze javascript-typescript (push) Has been cancelled
CI / Detect Version (push) Has been cancelled
CI / Detect Desktop Changes (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/cron.Dockerfile, ubuntu-latest, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build AMD64 (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build AMD64 (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/cron.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/db.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/pii.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-4vcpu-ubuntu-2404-arm, ./docker/realtime.Dockerfile, ubuntu-24.04-arm, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (blacksmith-8vcpu-ubuntu-2404-arm, ./docker/app.Dockerfile, linux-arm64-8-core, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
Helm Chart / Lint, test, and validate chart (push) Has been cancelled
Helm Chart / Chart version bumped (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Build Dev ECR (blacksmith-8vcpu-ubuntu-2404, ./docker/app.Dockerfile, ECR_APP, linux-x64-8-core) (push) Has been cancelled
CI / Promote Images (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/cron) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-2vcpu-ubuntu-2404, ./docker/db.Dockerfile, ECR_MIGRATIONS, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/pii.Dockerfile, ECR_PII, ubuntu-latest) (push) Has been cancelled
CI / Build Dev ECR (blacksmith-4vcpu-ubuntu-2404, ./docker/realtime.Dockerfile, ECR_REALTIME, ubuntu-latest) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Check Desktop Signing Secrets (push) Has been cancelled
CI / Desktop Release (push) Has been cancelled
CI / Create Desktop Prerelease (push) Has been cancelled
CI / Desktop Prerelease Build (push) Has been cancelled
CI / Publish Desktop Prerelease (push) Has been cancelled
CI / Prune Desktop Prereleases (push) Has been cancelled
Helm Chart / Install on kind and run helm test (push) Has been cancelled
777 lines
25 KiB
TypeScript
777 lines
25 KiB
TypeScript
import { sha256Hex } from '@sim/security/hash'
|
|
import {
|
|
applySourceEdits,
|
|
CodePlaceholderCompileError,
|
|
createCodePlaceholderCompilationContext,
|
|
isOffsetInRanges,
|
|
type SourceEdit,
|
|
} from '@/lib/execution/code-placeholders/shared'
|
|
import type {
|
|
CodePlaceholderOccurrence,
|
|
CompiledCodePlaceholders,
|
|
InternalCompileCodePlaceholdersInput,
|
|
} from '@/lib/execution/code-placeholders/types'
|
|
|
|
interface PythonStringToken {
|
|
start: number
|
|
end: number
|
|
prefix: string
|
|
quoteLength: 1 | 3
|
|
bracketDepth: number
|
|
}
|
|
|
|
interface PythonLexResult {
|
|
strings: PythonStringToken[]
|
|
comments: Array<[number, number]>
|
|
}
|
|
|
|
function isIdentifierCharacter(character: string | undefined): boolean {
|
|
return character !== undefined && /[A-Za-z0-9_]/.test(character)
|
|
}
|
|
|
|
function findLastPythonKeyword(
|
|
value: string,
|
|
keyword: string,
|
|
isIgnored: (index: number) => boolean = () => false
|
|
): number {
|
|
let index = value.lastIndexOf(keyword)
|
|
while (index >= 0) {
|
|
const before = value[index - 1]
|
|
const after = value[index + keyword.length]
|
|
if (!isIdentifierCharacter(before) && !isIdentifierCharacter(after) && !isIgnored(index)) {
|
|
return index
|
|
}
|
|
index = value.lastIndexOf(keyword, index - 1)
|
|
}
|
|
return -1
|
|
}
|
|
|
|
function readPythonStringStart(
|
|
code: string,
|
|
index: number
|
|
): { prefix: string; quote: string } | null {
|
|
if (isIdentifierCharacter(code[index - 1])) return null
|
|
const match = /^(?:[rRuUbBfF]{0,3})(?:'''|"""|'|")/.exec(code.slice(index))
|
|
if (!match) return null
|
|
const quoteMatch = /('''|"""|'|")$/.exec(match[0])
|
|
if (!quoteMatch) return null
|
|
return { prefix: match[0].slice(0, -quoteMatch[1].length), quote: quoteMatch[1] }
|
|
}
|
|
|
|
function readPythonStringToken(
|
|
code: string,
|
|
index: number,
|
|
bracketDepth: number
|
|
): PythonStringToken | null {
|
|
const stringStart = readPythonStringStart(code, index)
|
|
if (!stringStart) return null
|
|
|
|
const start = index
|
|
const quoteLength = stringStart.quote.length as 1 | 3
|
|
const formatted = /f/i.test(stringStart.prefix)
|
|
let replacementDepth = 0
|
|
let replacementParentheses = 0
|
|
let replacementBrackets = 0
|
|
let inFormatSpec = false
|
|
let expressionComment = false
|
|
index += stringStart.prefix.length + quoteLength
|
|
while (index < code.length) {
|
|
const character = code[index]
|
|
if (replacementDepth > 0) {
|
|
if (expressionComment) {
|
|
if (character === '\n' || character === '\r') expressionComment = false
|
|
index += 1
|
|
continue
|
|
}
|
|
const nestedString = readPythonStringToken(code, index, replacementDepth)
|
|
if (nestedString) {
|
|
index = nestedString.end
|
|
continue
|
|
}
|
|
if (character === '#' && (!inFormatSpec || replacementDepth > 1)) {
|
|
expressionComment = true
|
|
index += 1
|
|
continue
|
|
}
|
|
if (character === '\\') {
|
|
index += Math.min(2, code.length - index)
|
|
continue
|
|
}
|
|
if (character === '(') replacementParentheses += 1
|
|
else if (character === ')' && replacementParentheses > 0) replacementParentheses -= 1
|
|
else if (character === '[') replacementBrackets += 1
|
|
else if (character === ']' && replacementBrackets > 0) replacementBrackets -= 1
|
|
else if (
|
|
character === ':' &&
|
|
replacementDepth === 1 &&
|
|
replacementParentheses === 0 &&
|
|
replacementBrackets === 0
|
|
) {
|
|
inFormatSpec = true
|
|
} else if (character === '{') replacementDepth += 1
|
|
else if (character === '}') {
|
|
replacementDepth -= 1
|
|
if (replacementDepth === 0) {
|
|
replacementParentheses = 0
|
|
replacementBrackets = 0
|
|
inFormatSpec = false
|
|
}
|
|
}
|
|
index += 1
|
|
continue
|
|
}
|
|
if (code[index] === '\\') {
|
|
index += Math.min(2, code.length - index)
|
|
continue
|
|
}
|
|
if (code.startsWith(stringStart.quote, index)) {
|
|
index += quoteLength
|
|
break
|
|
}
|
|
if (formatted && character === '{') {
|
|
if (code[index + 1] === '{') index += 2
|
|
else {
|
|
replacementDepth = 1
|
|
replacementParentheses = 0
|
|
replacementBrackets = 0
|
|
inFormatSpec = false
|
|
index += 1
|
|
}
|
|
continue
|
|
}
|
|
if (formatted && character === '}' && code[index + 1] === '}') {
|
|
index += 2
|
|
continue
|
|
}
|
|
if (quoteLength === 1 && character === '\n') break
|
|
index += 1
|
|
}
|
|
return { start, end: index, prefix: stringStart.prefix, quoteLength, bracketDepth }
|
|
}
|
|
|
|
function lexPython(code: string): PythonLexResult {
|
|
const strings: PythonStringToken[] = []
|
|
const comments: Array<[number, number]> = []
|
|
let bracketDepth = 0
|
|
for (let index = 0; index < code.length; ) {
|
|
if (code[index] === '#') {
|
|
const end = code.indexOf('\n', index)
|
|
const commentEnd = end === -1 ? code.length : end
|
|
comments.push([index, commentEnd])
|
|
index = commentEnd
|
|
continue
|
|
}
|
|
|
|
const stringToken = readPythonStringToken(code, index, bracketDepth)
|
|
if (!stringToken) {
|
|
if (code[index] === '(' || code[index] === '[' || code[index] === '{') {
|
|
bracketDepth += 1
|
|
} else if (
|
|
(code[index] === ')' || code[index] === ']' || code[index] === '}') &&
|
|
bracketDepth > 0
|
|
) {
|
|
bracketDepth -= 1
|
|
}
|
|
index += 1
|
|
continue
|
|
}
|
|
strings.push(stringToken)
|
|
index = stringToken.end
|
|
}
|
|
return { strings, comments }
|
|
}
|
|
|
|
function lexNestedFStringStrings(code: string, token: PythonStringToken): PythonStringToken[] {
|
|
if (!/f/i.test(token.prefix)) return []
|
|
|
|
const strings: PythonStringToken[] = []
|
|
const contentStart = token.start + token.prefix.length + token.quoteLength
|
|
const contentEnd = token.end - token.quoteLength
|
|
let depth = 0
|
|
let parentheses = 0
|
|
let brackets = 0
|
|
let inFormatSpec = false
|
|
let comment = false
|
|
for (let index = contentStart; index < contentEnd; ) {
|
|
const character = code[index]
|
|
if (comment) {
|
|
if (character === '\n' || character === '\r') comment = false
|
|
index += 1
|
|
continue
|
|
}
|
|
|
|
const nestedString =
|
|
depth > 0
|
|
? readPythonStringToken(code, index, parentheses + brackets + (depth > 0 ? 1 : 0))
|
|
: null
|
|
if (nestedString) {
|
|
strings.push(nestedString)
|
|
index = nestedString.end
|
|
continue
|
|
}
|
|
if (depth > 0 && (!inFormatSpec || depth > 1) && character === '#') {
|
|
comment = true
|
|
index += 1
|
|
continue
|
|
}
|
|
if (depth > 0 && character === '(') parentheses += 1
|
|
else if (depth > 0 && character === ')' && parentheses > 0) parentheses -= 1
|
|
else if (depth > 0 && character === '[') brackets += 1
|
|
else if (depth > 0 && character === ']' && brackets > 0) brackets -= 1
|
|
else if (depth === 1 && parentheses === 0 && brackets === 0 && character === ':') {
|
|
inFormatSpec = true
|
|
} else if (character === '{') {
|
|
if (depth === 0 && code[index + 1] === '{') index += 1
|
|
else depth += 1
|
|
} else if (character === '}') {
|
|
if (depth === 0 && code[index + 1] === '}') index += 1
|
|
else if (depth > 0) {
|
|
depth -= 1
|
|
if (depth === 0) inFormatSpec = false
|
|
}
|
|
}
|
|
index += 1
|
|
}
|
|
return strings
|
|
}
|
|
|
|
function createSentinel(code: string): string {
|
|
const sourceDigest = sha256Hex(code)
|
|
for (let attempt = 0; attempt < 32; attempt += 1) {
|
|
const digest = sha256Hex(`${sourceDigest}\0${attempt}`)
|
|
const sentinel = `__sim_placeholder_${digest}__`
|
|
if (!code.includes(sentinel)) return sentinel
|
|
}
|
|
throw new CodePlaceholderCompileError('Unable to allocate a collision-free Python marker')
|
|
}
|
|
|
|
function pythonTriviaGap(
|
|
code: string,
|
|
start: number,
|
|
end: number,
|
|
comments: ReadonlyArray<[number, number]>
|
|
): string {
|
|
let gap = code.slice(start, end)
|
|
for (const [commentStart, commentEnd] of comments) {
|
|
const overlapStart = Math.max(start, commentStart)
|
|
const overlapEnd = Math.min(end, commentEnd)
|
|
if (overlapStart >= overlapEnd) continue
|
|
const relativeStart = overlapStart - start
|
|
const relativeEnd = overlapEnd - start
|
|
gap = `${gap.slice(0, relativeStart)}${' '.repeat(relativeEnd - relativeStart)}${gap.slice(relativeEnd)}`
|
|
}
|
|
return gap
|
|
}
|
|
|
|
function groupAdjacentStrings(
|
|
code: string,
|
|
strings: PythonStringToken[],
|
|
comments: ReadonlyArray<[number, number]>
|
|
): PythonStringToken[][] {
|
|
const groups: PythonStringToken[][] = []
|
|
for (const token of strings) {
|
|
const current = groups.at(-1)
|
|
const previous = current?.at(-1)
|
|
const gap = previous ? pythonTriviaGap(code, previous.end, token.start, comments) : ''
|
|
const sameLogicalExpression =
|
|
!/[\r\n]/.test(gap) ||
|
|
(previous !== undefined && previous.bracketDepth > 0 && token.bracketDepth > 0) ||
|
|
/^[ \t]*(?:\\\r?\n[ \t]*)+$/.test(gap)
|
|
if (current && previous && /^\s*(?:\\\r?\n\s*)*$/.test(gap) && sameLogicalExpression) {
|
|
current.push(token)
|
|
} else {
|
|
groups.push([token])
|
|
}
|
|
}
|
|
return groups
|
|
}
|
|
|
|
type FStringPosition =
|
|
| 'literal'
|
|
| 'expression'
|
|
| 'format'
|
|
| 'conversion'
|
|
| 'nested-string'
|
|
| 'comment'
|
|
| 'debug'
|
|
|
|
function hasFStringDebugMarker(
|
|
code: string,
|
|
token: PythonStringToken,
|
|
start: number,
|
|
initialDepth: number,
|
|
initialParentheses: number,
|
|
initialBrackets: number
|
|
): boolean {
|
|
const contentEnd = token.end - token.quoteLength
|
|
let depth = initialDepth
|
|
let parentheses = initialParentheses
|
|
let brackets = initialBrackets
|
|
let nestedString: { delimiter: string } | null = null
|
|
let comment = false
|
|
|
|
for (let index = start; index < contentEnd; index += 1) {
|
|
const character = code[index]
|
|
if (comment) {
|
|
if (character === '\n' || character === '\r') comment = false
|
|
continue
|
|
}
|
|
if (nestedString) {
|
|
if (character === '\\') {
|
|
index += 1
|
|
} else if (code.startsWith(nestedString.delimiter, index)) {
|
|
index += nestedString.delimiter.length - 1
|
|
nestedString = null
|
|
}
|
|
continue
|
|
}
|
|
const stringStart = depth > 0 ? readPythonStringStart(code, index) : null
|
|
if (stringStart) {
|
|
nestedString = { delimiter: stringStart.quote }
|
|
index += stringStart.prefix.length + stringStart.quote.length - 1
|
|
continue
|
|
}
|
|
if (depth > 0 && character === '#') {
|
|
comment = true
|
|
continue
|
|
}
|
|
if (code.startsWith('{{', index)) {
|
|
const placeholderEnd = code.indexOf('}}', index + 2)
|
|
if (placeholderEnd !== -1) {
|
|
index = placeholderEnd + 1
|
|
continue
|
|
}
|
|
}
|
|
if (character === '(') parentheses += 1
|
|
else if (character === ')' && parentheses > 0) parentheses -= 1
|
|
else if (character === '[') brackets += 1
|
|
else if (character === ']' && brackets > 0) brackets -= 1
|
|
else if (character === '{') depth += 1
|
|
else if (character === '}') {
|
|
depth -= 1
|
|
if (depth <= 0) return false
|
|
} else if (
|
|
depth === 1 &&
|
|
parentheses === 0 &&
|
|
brackets === 0 &&
|
|
character === '=' &&
|
|
code[index - 1] !== '=' &&
|
|
code[index - 1] !== '!' &&
|
|
code[index - 1] !== '<' &&
|
|
code[index - 1] !== '>' &&
|
|
code[index - 1] !== ':' &&
|
|
code[index + 1] !== '='
|
|
) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
function getFStringPosition(
|
|
code: string,
|
|
token: PythonStringToken,
|
|
absoluteOffset: number
|
|
): FStringPosition {
|
|
if (!/f/i.test(token.prefix)) return 'literal'
|
|
const quoteLength = token.quoteLength
|
|
const contentStart = token.start + token.prefix.length + quoteLength
|
|
const contentEnd = token.end - quoteLength
|
|
let depth = 0
|
|
let parentheses = 0
|
|
let brackets = 0
|
|
let inFormatSpec = false
|
|
let inConversion = false
|
|
let nestedString: { delimiter: string } | null = null
|
|
let comment = false
|
|
for (let index = contentStart; index < contentEnd && index < absoluteOffset; index += 1) {
|
|
const character = code[index]
|
|
if (comment) {
|
|
if (character === '\n' || character === '\r') comment = false
|
|
continue
|
|
}
|
|
if (nestedString) {
|
|
if (character === '\\') {
|
|
index += 1
|
|
} else if (code.startsWith(nestedString.delimiter, index)) {
|
|
index += nestedString.delimiter.length - 1
|
|
nestedString = null
|
|
}
|
|
continue
|
|
}
|
|
const stringStart = depth > 0 ? readPythonStringStart(code, index) : null
|
|
if (stringStart) {
|
|
nestedString = { delimiter: stringStart.quote }
|
|
index += stringStart.prefix.length + stringStart.quote.length - 1
|
|
continue
|
|
}
|
|
if (depth > 0 && !inFormatSpec && character === '#') {
|
|
comment = true
|
|
continue
|
|
}
|
|
if (depth > 0 && character === '(') {
|
|
parentheses += 1
|
|
continue
|
|
}
|
|
if (depth > 0 && character === ')' && parentheses > 0) {
|
|
parentheses -= 1
|
|
continue
|
|
}
|
|
if (depth > 0 && character === '[') {
|
|
brackets += 1
|
|
continue
|
|
}
|
|
if (depth > 0 && character === ']' && brackets > 0) {
|
|
brackets -= 1
|
|
continue
|
|
}
|
|
if (
|
|
depth === 1 &&
|
|
parentheses === 0 &&
|
|
brackets === 0 &&
|
|
character === '!' &&
|
|
code[index + 1] !== '='
|
|
) {
|
|
inConversion = true
|
|
continue
|
|
}
|
|
if (depth === 1 && parentheses === 0 && brackets === 0 && character === ':') {
|
|
inFormatSpec = true
|
|
inConversion = false
|
|
continue
|
|
}
|
|
if (character === '{') {
|
|
if (depth === 0 && code[index + 1] === '{') {
|
|
index += 1
|
|
} else {
|
|
depth += 1
|
|
}
|
|
} else if (character === '}') {
|
|
if (depth === 0 && code[index + 1] === '}') index += 1
|
|
else if (depth > 0) {
|
|
depth -= 1
|
|
if (depth === 0) {
|
|
parentheses = 0
|
|
brackets = 0
|
|
inFormatSpec = false
|
|
inConversion = false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (comment) return 'comment'
|
|
if (nestedString) return 'nested-string'
|
|
if (depth === 0) return 'literal'
|
|
if (inFormatSpec && depth === 1) return 'format'
|
|
if (inConversion) return 'conversion'
|
|
if (hasFStringDebugMarker(code, token, absoluteOffset, depth, parentheses, brackets)) {
|
|
return 'debug'
|
|
}
|
|
return 'expression'
|
|
}
|
|
|
|
function buildSimultaneousInterpolation(
|
|
expression: string,
|
|
sentinel: string | undefined,
|
|
accessors: string[],
|
|
bytes: boolean
|
|
): string {
|
|
if (!sentinel || accessors.length === 0) return expression
|
|
|
|
const parts = '__sim_parts'
|
|
const sentinelLiteral = bytes ? `b${JSON.stringify(sentinel)}` : JSON.stringify(sentinel)
|
|
const values: string[] = [`${parts}[0]`]
|
|
for (const [index, accessor] of accessors.entries()) {
|
|
values.push(bytes ? `${accessor}.encode("utf-8")` : accessor, `${parts}[${index + 1}]`)
|
|
}
|
|
const empty = bytes ? 'b""' : '""'
|
|
const expectedParts = accessors.length + 1
|
|
return `(lambda ${parts}: ${empty}.join([${values.join(', ')}]) if ${parts}.__len__() == ${expectedParts} else {}["Sim placeholder interpolation mismatch"])((${expression}).split(${sentinelLiteral}))`
|
|
}
|
|
|
|
function pythonRuntimeValue(bindingName: string): string {
|
|
return `(${bindingName} if True else None)`
|
|
}
|
|
|
|
type PythonBarePlaceholderPosition = 'value' | 'attribute' | 'unsupported-name'
|
|
|
|
function classifyPythonBarePlaceholder(
|
|
code: string,
|
|
occurrence: CodePlaceholderOccurrence,
|
|
ignoredRanges: ReadonlyArray<[number, number]> = []
|
|
): PythonBarePlaceholderPosition {
|
|
const immediatelyPrevious = code[occurrence.start - 1]
|
|
const immediatelyNext = code[occurrence.end]
|
|
let previousIndex = occurrence.start - 1
|
|
while (previousIndex >= 0 && /[ \t\f]/.test(code[previousIndex])) previousIndex -= 1
|
|
const previous = code[previousIndex]
|
|
const attribute = previous === '.'
|
|
if (isIdentifierCharacter(immediatelyPrevious) || isIdentifierCharacter(immediatelyNext)) {
|
|
return 'unsupported-name'
|
|
}
|
|
|
|
const lineStart = code.lastIndexOf('\n', occurrence.start - 1) + 1
|
|
const nextNewline = code.indexOf('\n', occurrence.end)
|
|
const lineEnd = nextNewline === -1 ? code.length : nextNewline
|
|
const before = code.slice(lineStart, occurrence.start)
|
|
const after = code.slice(occurrence.end, lineEnd)
|
|
const isIgnoredBeforeOffset = (offset: number): boolean =>
|
|
isOffsetInRanges(lineStart + offset, ignoredRanges)
|
|
const parameterSegment = before.slice(
|
|
Math.max(before.lastIndexOf('('), before.lastIndexOf(',')) + 1
|
|
)
|
|
const inFunctionParameterName =
|
|
/(?:^|\s)(?:async\s+)?def\s+[A-Za-z_][A-Za-z0-9_]*\s*\([^)]*$/.test(before) &&
|
|
!/[=:]/.test(parameterSegment)
|
|
const lambdaStart = findLastPythonKeyword(before, 'lambda', isIgnoredBeforeOffset)
|
|
const inLambdaParameterName =
|
|
lambdaStart >= 0 &&
|
|
before.indexOf(':', lambdaStart) === -1 &&
|
|
!before
|
|
.slice(Math.max(lambdaStart + 'lambda'.length, before.lastIndexOf(',') + 1))
|
|
.includes('=')
|
|
const forMatch = /(?:^|\s)(?:async\s+)?for\s+([^\r\n]*)$/.exec(before)
|
|
const inForTarget = forMatch !== null && !/(?:^|\s)in\s/.test(forMatch[1])
|
|
const followedByAssignment =
|
|
/^\s*(?:=(?!=)|:=|\+=|-=|\*=|\/=|\/\/=|%=|@=|&=|\|=|\^=|>>=|<<=|\*\*=)/.test(after)
|
|
const caseClause = /^\s*case\s+([^\r\n]*)$/.exec(before)
|
|
const casePrefix = caseClause?.[1] ?? ''
|
|
const caseStartsWithAssignment =
|
|
/^(?:=(?!=)|:=|\+=|-=|\*=|\/=|\/\/=|%=|@=|&=|\|=|\^=|>>=|<<=|\*\*=)/.test(casePrefix)
|
|
const casePrefixOffset = before.length - casePrefix.length
|
|
const inCasePattern =
|
|
caseClause !== null &&
|
|
!caseStartsWithAssignment &&
|
|
findLastPythonKeyword(casePrefix, 'if', (offset) =>
|
|
isIgnoredBeforeOffset(casePrefixOffset + offset)
|
|
) === -1 &&
|
|
after.includes(':')
|
|
if (
|
|
/(?:^|\s)(?:async\s+)?(?:def|class|import|from|as|global|nonlocal|del)\s*$/.test(before) ||
|
|
/(?:^|\s)(?:async\s+)?for\s*$/.test(before) ||
|
|
/(?:^|\s)(?:as|lambda)\s*$/.test(before) ||
|
|
inFunctionParameterName ||
|
|
inLambdaParameterName ||
|
|
inForTarget ||
|
|
inCasePattern ||
|
|
followedByAssignment ||
|
|
(attribute && /(?:^|\s)del\s+[^\r\n]*\.\s*$/.test(before))
|
|
) {
|
|
return 'unsupported-name'
|
|
}
|
|
if (attribute) return 'attribute'
|
|
return 'value'
|
|
}
|
|
|
|
export async function compilePythonPlaceholders(
|
|
input: InternalCompileCodePlaceholdersInput
|
|
): Promise<CompiledCodePlaceholders> {
|
|
const context = createCodePlaceholderCompilationContext(input, { identifierSuffix: '__' })
|
|
if (context.occurrences.length === 0) return context.finish(input.code)
|
|
|
|
const lexed = lexPython(input.code)
|
|
const edits: SourceEdit[] = []
|
|
const consumed = new Set<CodePlaceholderOccurrence>()
|
|
let compilationSentinel: string | undefined
|
|
const getSentinel = (): string => {
|
|
if (!compilationSentinel) {
|
|
compilationSentinel = createSentinel(input.code)
|
|
context.registerInternalIdentifier(compilationSentinel)
|
|
}
|
|
return compilationSentinel
|
|
}
|
|
|
|
for (const group of groupAdjacentStrings(input.code, lexed.strings, lexed.comments)) {
|
|
const start = group[0].start
|
|
const end = group.at(-1)?.end ?? start
|
|
const items = context.occurrences.filter(
|
|
(occurrence) => occurrence.start >= start && occurrence.end <= end
|
|
)
|
|
if (items.length === 0) continue
|
|
|
|
let groupSource = input.code.slice(start, end)
|
|
const replacements: string[] = []
|
|
let groupSentinel: string | undefined
|
|
const sourceEdits: SourceEdit[] = []
|
|
const nestedStringGroups = groupAdjacentStrings(
|
|
input.code,
|
|
group.flatMap((token) => lexNestedFStringStrings(input.code, token)),
|
|
[]
|
|
)
|
|
const nestedReplacements = new Map<
|
|
PythonStringToken[],
|
|
Array<{ occurrence: CodePlaceholderOccurrence; accessor: string }>
|
|
>()
|
|
for (const occurrence of items) {
|
|
const token = group.find(
|
|
(candidate) => occurrence.start >= candidate.start && occurrence.end <= candidate.end
|
|
)
|
|
if (!token) continue
|
|
const fStringPosition = getFStringPosition(input.code, token, occurrence.start)
|
|
if (fStringPosition === 'comment') continue
|
|
if (!context.hasValue(occurrence.name)) continue
|
|
if (fStringPosition === 'nested-string') {
|
|
const nestedGroup = nestedStringGroups.find((candidate) =>
|
|
candidate.some(
|
|
(nestedToken) =>
|
|
occurrence.start >= nestedToken.start && occurrence.end <= nestedToken.end
|
|
)
|
|
)
|
|
const nestedToken = nestedGroup?.find(
|
|
(candidate) => occurrence.start >= candidate.start && occurrence.end <= candidate.end
|
|
)
|
|
if (
|
|
nestedGroup &&
|
|
nestedToken &&
|
|
getFStringPosition(input.code, nestedToken, occurrence.start) === 'literal'
|
|
) {
|
|
const resolved = context.resolve(occurrence)
|
|
if (!resolved) continue
|
|
const pending = nestedReplacements.get(nestedGroup) ?? []
|
|
pending.push({
|
|
occurrence,
|
|
accessor: pythonRuntimeValue(resolved.bindingName),
|
|
})
|
|
nestedReplacements.set(nestedGroup, pending)
|
|
continue
|
|
}
|
|
}
|
|
if (
|
|
fStringPosition === 'conversion' ||
|
|
fStringPosition === 'nested-string' ||
|
|
fStringPosition === 'debug'
|
|
) {
|
|
if (input.analysisOnly) {
|
|
context.resolveValue(occurrence)
|
|
consumed.add(occurrence)
|
|
continue
|
|
}
|
|
throw new CodePlaceholderCompileError(
|
|
`Variable placeholder "${occurrence.name}" is not supported in this f-string syntax position`,
|
|
input.code,
|
|
occurrence.start
|
|
)
|
|
}
|
|
const resolved = context.resolve(occurrence)
|
|
if (!resolved) continue
|
|
const accessor = pythonRuntimeValue(resolved.bindingName)
|
|
if (fStringPosition === 'expression' || fStringPosition === 'format') {
|
|
if (
|
|
fStringPosition === 'expression' &&
|
|
classifyPythonBarePlaceholder(
|
|
input.code,
|
|
occurrence,
|
|
nestedStringGroups
|
|
.flat()
|
|
.map(
|
|
({ start: nestedStart, end: nestedEnd }) =>
|
|
[nestedStart, nestedEnd] as [number, number]
|
|
)
|
|
) === 'unsupported-name'
|
|
) {
|
|
if (input.analysisOnly) {
|
|
context.resolveValue(occurrence)
|
|
consumed.add(occurrence)
|
|
continue
|
|
}
|
|
throw new CodePlaceholderCompileError(
|
|
`Variable placeholder "${occurrence.name}" is not supported in a Python name or assignment position`,
|
|
input.code,
|
|
occurrence.start
|
|
)
|
|
}
|
|
sourceEdits.push({
|
|
start: occurrence.start - start,
|
|
end: occurrence.end - start,
|
|
text: fStringPosition === 'format' ? `{${accessor}}` : accessor,
|
|
})
|
|
} else {
|
|
if (!groupSentinel) {
|
|
groupSentinel = getSentinel()
|
|
}
|
|
sourceEdits.push({
|
|
start: occurrence.start - start,
|
|
end: occurrence.end - start,
|
|
text: groupSentinel,
|
|
})
|
|
replacements.push(accessor)
|
|
}
|
|
consumed.add(occurrence)
|
|
}
|
|
for (const [nestedGroup, pending] of nestedReplacements) {
|
|
const nestedStart = nestedGroup[0].start
|
|
const nestedEnd = nestedGroup.at(-1)?.end ?? nestedStart
|
|
const nestedEdits: SourceEdit[] = []
|
|
const interpolationAccessors: string[] = []
|
|
const nestedSentinel = getSentinel()
|
|
for (const { occurrence, accessor } of pending) {
|
|
nestedEdits.push({
|
|
start: occurrence.start - nestedStart,
|
|
end: occurrence.end - nestedStart,
|
|
text: nestedSentinel,
|
|
})
|
|
interpolationAccessors.push(accessor)
|
|
consumed.add(occurrence)
|
|
}
|
|
const nestedSource = applySourceEdits(input.code.slice(nestedStart, nestedEnd), nestedEdits)
|
|
sourceEdits.push({
|
|
start: nestedStart - start,
|
|
end: nestedEnd - start,
|
|
text: buildSimultaneousInterpolation(
|
|
nestedSource,
|
|
nestedSentinel,
|
|
interpolationAccessors,
|
|
nestedGroup.every((nestedToken) => /b/i.test(nestedToken.prefix))
|
|
),
|
|
})
|
|
}
|
|
if (sourceEdits.length === 0) continue
|
|
groupSource = applySourceEdits(groupSource, sourceEdits)
|
|
const bytes = group.every((token) => /b/i.test(token.prefix))
|
|
edits.push({
|
|
start,
|
|
end,
|
|
text: buildSimultaneousInterpolation(groupSource, groupSentinel, replacements, bytes),
|
|
})
|
|
}
|
|
|
|
for (const occurrence of context.occurrences) {
|
|
if (consumed.has(occurrence) || isOffsetInRanges(occurrence.start, lexed.comments)) continue
|
|
if (
|
|
lexed.strings.some((token) => occurrence.start >= token.start && occurrence.end <= token.end)
|
|
) {
|
|
continue
|
|
}
|
|
if (!context.hasValue(occurrence.name)) continue
|
|
const position = classifyPythonBarePlaceholder(input.code, occurrence, [
|
|
...lexed.strings.map(({ start, end }) => [start, end] as [number, number]),
|
|
...lexed.comments,
|
|
])
|
|
if (position === 'unsupported-name') {
|
|
if (input.analysisOnly) {
|
|
context.resolveValue(occurrence)
|
|
consumed.add(occurrence)
|
|
continue
|
|
}
|
|
throw new CodePlaceholderCompileError(
|
|
`Variable placeholder "${occurrence.name}" is not supported in a Python name or assignment position`,
|
|
input.code,
|
|
occurrence.start
|
|
)
|
|
}
|
|
const resolved = context.resolve(occurrence)
|
|
if (!resolved) continue
|
|
edits.push({
|
|
start: occurrence.start,
|
|
end: occurrence.end,
|
|
text:
|
|
position === 'attribute'
|
|
? `__getattribute__(${pythonRuntimeValue(resolved.bindingName)})`
|
|
: pythonRuntimeValue(resolved.bindingName),
|
|
})
|
|
consumed.add(occurrence)
|
|
}
|
|
|
|
return context.finish(applySourceEdits(input.code, edits))
|
|
}
|