diff --git a/apps/cli/ai/block-content-policy.ts b/apps/cli/ai/block-content-policy.ts index 78621a45e8..5b7a6eb6c1 100644 --- a/apps/cli/ai/block-content-policy.ts +++ b/apps/cli/ai/block-content-policy.ts @@ -2,6 +2,7 @@ const BLOCK_COMMENT_PATTERN = //g; const SINGLE_SCRIPT_PATTERN = /^]*)?>[\s\S]*<\/script>\s*$/i; const SINGLE_SVG_PATTERN = /^]*)?>[\s\S]*<\/svg>\s*$/i; const INTERACTION_MARKUP_PATTERN = /<(marquee)\b/i; +const HTML_BLOCK_PATTERN = /([\s\S]*?)/g; const PLUGIN_RECOMMENDATION_HTML_BLOCK_POLICIES = [ { htmlPatterns: [ /<(form|input|select|textarea|fieldset)\b/i ], @@ -10,6 +11,18 @@ const PLUGIN_RECOMMENDATION_HTML_BLOCK_POLICIES = [ }, ]; +export interface HtmlBlockPolicyResult { + blockNumber: number; + line: number; + content: string; + issues: string[]; +} + +export interface HtmlBlockPolicyReport { + totalHtmlBlocks: number; + invalidHtmlBlocks: HtmlBlockPolicyResult[]; +} + export function getHtmlBlockPolicyIssues( content: string ): string[] { const html = content.replace( BLOCK_COMMENT_PATTERN, '' ).trim(); if ( ! html ) { @@ -35,3 +48,29 @@ export function getHtmlBlockPolicyIssues( content: string ): string[] { 'core/html contains markup that should use editable core blocks. Load the block-content skill and use core/group, core/columns, core/heading, core/paragraph, core/list, core/image, core/buttons, and theme CSS instead. Keep core/html only for inline SVG, interaction markup with no block equivalent (marquee, cursor), or a single script block.', ]; } + +export function validateHtmlBlockPolicy( content: string ): HtmlBlockPolicyReport { + const invalidHtmlBlocks: HtmlBlockPolicyResult[] = []; + let totalHtmlBlocks = 0; + + for ( const match of content.matchAll( HTML_BLOCK_PATTERN ) ) { + totalHtmlBlocks++; + const blockContent = match[ 1 ] ?? ''; + const issues = getHtmlBlockPolicyIssues( blockContent ); + if ( issues.length === 0 ) { + continue; + } + + invalidHtmlBlocks.push( { + blockNumber: totalHtmlBlocks, + line: content.slice( 0, match.index ?? 0 ).split( '\n' ).length, + content: blockContent.trim(), + issues, + } ); + } + + return { + totalHtmlBlocks, + invalidHtmlBlocks, + }; +} diff --git a/apps/cli/ai/block-validator.ts b/apps/cli/ai/block-validator.ts index ca4743b551..acf95f7b99 100644 --- a/apps/cli/ai/block-validator.ts +++ b/apps/cli/ai/block-validator.ts @@ -1,4 +1,3 @@ -import { getHtmlBlockPolicyIssues } from 'cli/ai/block-content-policy'; import { EditorPage } from 'cli/ai/browser-utils'; interface BlockValidationResult { @@ -9,7 +8,7 @@ interface BlockValidationResult { expectedContent?: string; } -export interface ValidationReport { +export interface ValidationReportBase { totalBlocks: number; validBlocks: number; invalidBlocks: number; @@ -17,6 +16,15 @@ export interface ValidationReport { error?: string; } +export interface BlockFixProposal { + fixedContent: string; + report: ValidationReportBase; +} + +export interface ValidationReport extends ValidationReportBase { + proposedFix?: BlockFixProposal; +} + // Cache one EditorPage per site URL so repeated validations reuse the same // browser tab instead of navigating and loading the block editor each time. const editorPages = new Map< string, EditorPage >(); @@ -48,22 +56,21 @@ export async function validateBlocks( try { const page = await editorPage.getPage(); - const report = await page.evaluate( ( html: string ) => { + const validation = await page.evaluate( ( html: string ) => { /* eslint-disable @typescript-eslint/no-explicit-any */ const wpBlocks = ( window as any ).wp?.blocks; if ( ! wpBlocks ) { return { - totalBlocks: 0, - validBlocks: 0, - invalidBlocks: 0, - results: [] as any[], - error: 'wp.blocks is not available on this page.', + report: { + totalBlocks: 0, + validBlocks: 0, + invalidBlocks: 0, + results: [] as any[], + error: 'wp.blocks is not available on this page.', + }, }; } - const blocks = wpBlocks.parse( html ); - const results: any[] = []; - function extractIssues( validationIssues: any[] ): string[] { const issues: string[] = []; for ( const issue of validationIssues ) { @@ -87,88 +94,152 @@ export async function validateBlocks( return issues; } - function validateRecursive( block: any ) { - if ( ! block.name ) { - return; - } - if ( block.name === 'core/freeform' || block.name === 'core/missing' ) { - return; - } + function validateContent( blockHtml: string ) { + const blocks = wpBlocks.parse( blockHtml ); + const results: any[] = []; - const blockType = wpBlocks.getBlockType( block.name ); - let issues: string[] = []; - let isValid = true; - let expectedContent: string | undefined; - - if ( ! blockType ) { - isValid = false; - issues.push( - `Block type "${ block.name }" is not registered. ` + - 'It may require a plugin that is not active.' - ); - } else { - // validateBlock performs the save-function comparison. - // Handle both return formats: - // WP 6.3+: { isValid: boolean, validationIssues: Array } - // Older: [boolean, Array] - const validation = wpBlocks.validateBlock( block, blockType ); - - if ( Array.isArray( validation ) ) { - isValid = validation[ 0 ]; - if ( ! isValid ) { - issues = extractIssues( validation[ 1 ] || [] ); - } + function validateRecursive( block: any ) { + if ( ! block.name ) { + return; + } + if ( block.name === 'core/freeform' || block.name === 'core/missing' ) { + return; + } + + const blockType = wpBlocks.getBlockType( block.name ); + let issues: string[] = []; + let isValid = true; + let expectedContent: string | undefined; + + if ( ! blockType ) { + isValid = false; + issues.push( + `Block type "${ block.name }" is not registered. ` + + 'It may require a plugin that is not active.' + ); } else { - isValid = validation.isValid; + // validateBlock performs the save-function comparison. + // Handle both return formats: + // WP 6.3+: { isValid: boolean, validationIssues: Array } + // Older: [boolean, Array] + const validationResult = wpBlocks.validateBlock( block, blockType ); + + if ( Array.isArray( validationResult ) ) { + isValid = validationResult[ 0 ]; + if ( ! isValid ) { + issues = extractIssues( validationResult[ 1 ] || [] ); + } + } else { + isValid = validationResult.isValid; + if ( ! isValid ) { + issues = extractIssues( validationResult.validationIssues || [] ); + } + } + if ( ! isValid ) { - issues = extractIssues( validation.validationIssues || [] ); + // Re-render expected HTML including inner blocks for comparison + try { + expectedContent = wpBlocks.getSaveContent( + blockType, + block.attributes, + block.innerBlocks + ); + } catch { + // If re-rendering fails, skip expected content + } } } - if ( ! isValid ) { - // Re-render expected HTML including inner blocks for comparison - try { - expectedContent = wpBlocks.getSaveContent( - blockType, - block.attributes, - block.innerBlocks - ); - } catch { - // If re-rendering fails, skip expected content + results.push( { + blockName: block.name, + isValid, + issues, + originalContent: block.originalContent || '', + expectedContent: isValid ? undefined : expectedContent, + } ); + + if ( block.innerBlocks ) { + for ( const inner of block.innerBlocks ) { + validateRecursive( inner ); } } } - results.push( { - blockName: block.name, - isValid, - issues, - originalContent: block.originalContent || '', - expectedContent: isValid ? undefined : expectedContent, - } ); - - if ( block.innerBlocks ) { - for ( const inner of block.innerBlocks ) { - validateRecursive( inner ); - } + for ( const block of blocks ) { + validateRecursive( block ); + } + + const validCount = results.filter( ( r: any ) => r.isValid ).length; + return { + blocks, + report: { + totalBlocks: results.length, + validBlocks: validCount, + invalidBlocks: results.length - validCount, + results, + }, + }; + } + + function normalizeBlock( block: any ): any { + if ( ! block.name || block.name === 'core/freeform' || block.name === 'core/missing' ) { + return block; } + + const blockType = wpBlocks.getBlockType( block.name ); + if ( ! blockType || typeof wpBlocks.createBlock !== 'function' ) { + return block; + } + + const fixedInnerBlocks = Array.isArray( block.innerBlocks ) + ? block.innerBlocks.map( normalizeBlock ) + : []; + + try { + return wpBlocks.createBlock( + block.name, + block.attributes, + fixedInnerBlocks.length > 0 ? fixedInnerBlocks : undefined + ); + } catch { + return block; + } + } + + const initial = validateContent( html ); + + if ( initial.report.invalidBlocks === 0 ) { + return { report: initial.report }; } - for ( const block of blocks ) { - validateRecursive( block ); + const fixedBlocks = initial.blocks.map( normalizeBlock ); + const fixedContent = wpBlocks.serialize( fixedBlocks ); + + if ( fixedContent === html ) { + return { report: initial.report }; } - const validCount = results.filter( ( r: any ) => r.isValid ).length; + const fixed = validateContent( fixedContent ); return { - totalBlocks: results.length, - validBlocks: validCount, - invalidBlocks: results.length - validCount, - results, + report: initial.report, + proposedFix: { + fixedContent, + report: fixed.report, + }, }; /* eslint-enable @typescript-eslint/no-explicit-any */ }, content ); - return applyBlockContentPolicy( report as ValidationReport ); + const report = validation.report as ValidationReport; + + if ( validation.proposedFix ) { + report.proposedFix = { + fixedContent: validation.proposedFix.fixedContent, + report: validation.proposedFix.report as ValidationReportBase, + }; + } + + return report; } catch ( error ) { // If navigation or evaluation failed, discard the cached page so the // next call gets a fresh one. @@ -187,33 +258,6 @@ export async function validateBlocks( } } -function applyBlockContentPolicy( report: ValidationReport ): ValidationReport { - const results = report.results.map( ( result ) => { - if ( result.blockName !== 'core/html' ) { - return result; - } - - const policyIssues = getHtmlBlockPolicyIssues( result.originalContent ); - if ( policyIssues.length === 0 ) { - return result; - } - - return { - ...result, - isValid: false, - issues: [ ...result.issues, ...policyIssues ], - }; - } ); - - const validBlocks = results.filter( ( result ) => result.isValid ).length; - return { - ...report, - results, - validBlocks, - invalidBlocks: results.length - validBlocks, - }; -} - /** Clean up all cached editor pages. */ export async function cleanupValidatorPages(): Promise< void > { for ( const ep of editorPages.values() ) { diff --git a/apps/cli/ai/content-diff.ts b/apps/cli/ai/content-diff.ts new file mode 100644 index 0000000000..9d6f15a550 --- /dev/null +++ b/apps/cli/ai/content-diff.ts @@ -0,0 +1,100 @@ +type DiffOperation = 'equal' | 'delete' | 'insert'; + +interface DiffLine { + type: DiffOperation; + line: string; +} + +function splitLines( content: string ): string[] { + if ( content.length === 0 ) { + return []; + } + return content.endsWith( '\n' ) ? content.slice( 0, -1 ).split( '\n' ) : content.split( '\n' ); +} + +function buildFullReplacementDiff( oldLines: string[], newLines: string[] ): DiffLine[] { + return [ + ...oldLines.map( ( line ) => ( { type: 'delete' as const, line } ) ), + ...newLines.map( ( line ) => ( { type: 'insert' as const, line } ) ), + ]; +} + +function buildDiffLines( oldLines: string[], newLines: string[] ): DiffLine[] { + const cellCount = oldLines.length * newLines.length; + if ( cellCount > 4_000_000 ) { + return buildFullReplacementDiff( oldLines, newLines ); + } + + const table = Array.from( { length: oldLines.length + 1 }, () => + new Array< number >( newLines.length + 1 ).fill( 0 ) + ); + + for ( let oldIndex = oldLines.length - 1; oldIndex >= 0; oldIndex-- ) { + for ( let newIndex = newLines.length - 1; newIndex >= 0; newIndex-- ) { + table[ oldIndex ][ newIndex ] = + oldLines[ oldIndex ] === newLines[ newIndex ] + ? table[ oldIndex + 1 ][ newIndex + 1 ] + 1 + : Math.max( table[ oldIndex + 1 ][ newIndex ], table[ oldIndex ][ newIndex + 1 ] ); + } + } + + const diffLines: DiffLine[] = []; + let oldIndex = 0; + let newIndex = 0; + + while ( oldIndex < oldLines.length && newIndex < newLines.length ) { + if ( oldLines[ oldIndex ] === newLines[ newIndex ] ) { + diffLines.push( { type: 'equal', line: oldLines[ oldIndex ] } ); + oldIndex++; + newIndex++; + } else if ( table[ oldIndex + 1 ][ newIndex ] >= table[ oldIndex ][ newIndex + 1 ] ) { + diffLines.push( { type: 'delete', line: oldLines[ oldIndex ] } ); + oldIndex++; + } else { + diffLines.push( { type: 'insert', line: newLines[ newIndex ] } ); + newIndex++; + } + } + + while ( oldIndex < oldLines.length ) { + diffLines.push( { type: 'delete', line: oldLines[ oldIndex ] } ); + oldIndex++; + } + while ( newIndex < newLines.length ) { + diffLines.push( { type: 'insert', line: newLines[ newIndex ] } ); + newIndex++; + } + + return diffLines; +} + +function countLinesByType( lines: DiffLine[], type: DiffOperation ): number { + return lines.filter( ( line ) => line.type === type ).length; +} + +export function createUnifiedDiff( + oldContent: string, + newContent: string, + oldLabel = 'original', + newLabel = 'fixed' +): string { + if ( oldContent === newContent ) { + return ''; + } + + const oldLines = splitLines( oldContent ); + const newLines = splitLines( newContent ); + const diffLines = buildDiffLines( oldLines, newLines ); + const oldCount = countLinesByType( diffLines, 'equal' ) + countLinesByType( diffLines, 'delete' ); + const newCount = countLinesByType( diffLines, 'equal' ) + countLinesByType( diffLines, 'insert' ); + + return [ + `--- ${ oldLabel }`, + `+++ ${ newLabel }`, + `@@ -1,${ oldCount } +1,${ newCount } @@`, + ...diffLines.map( ( diffLine ) => { + const prefix = diffLine.type === 'equal' ? ' ' : diffLine.type === 'delete' ? '-' : '+'; + return `${ prefix }${ diffLine.line }`; + } ), + ].join( '\n' ); +} diff --git a/apps/cli/ai/skills/block-content/SKILL.md b/apps/cli/ai/skills/block-content/SKILL.md index fe40a7da66..448e9e84c2 100644 --- a/apps/cli/ai/skills/block-content/SKILL.md +++ b/apps/cli/ai/skills/block-content/SKILL.md @@ -88,6 +88,6 @@ Do not use `--post_content-file=`. `wp_cli` runs inside the PHP-WASM ## Validation -- Run `validate_blocks` after every write or edit that creates or changes block content. -- If validation rewrites expected markup, treat the difference as structural. Classes added or removed by the validator can affect layout and styling. +- Run `validate_html_blocks` after every write or edit that creates or changes block content. If it reports invalid `core/html` blocks, rewrite only those blocks as editable core or plugin blocks, then call `validate_html_blocks` again before editor validation. +- Run `validate_and_fix_blocks` after `validate_html_blocks` passes. Call it with `filePath` whenever the content lives in a file; safe editor serialization fixes are applied directly to that file. If it says an auto-fix was applied, do not manually replace markup or call validation again unless you intentionally change block markup afterward. Use the diff only to inspect structural changes for CSS impact. Classes added or removed by the validator can affect layout and styling. - After fixing invalid blocks, take desktop and mobile screenshots and check that layout, spacing, and full-width sections still render as intended. diff --git a/apps/cli/ai/skills/plugin-recommendations/SKILL.md b/apps/cli/ai/skills/plugin-recommendations/SKILL.md index 40cda45c61..502d1ff786 100644 --- a/apps/cli/ai/skills/plugin-recommendations/SKILL.md +++ b/apps/cli/ai/skills/plugin-recommendations/SKILL.md @@ -34,7 +34,7 @@ wp_cli eval 'foreach (\WP_Block_Type_Registry::get_instance()->get_all_registere 4. If you expect a plugin block but it is missing, check whether the plugin uses modules or feature flags, then activate the relevant module. 5. Use the registered block in editable block markup. -6. Validate generated block markup with `validate_blocks`. +6. Validate generated block markup with `validate_html_blocks`, then `validate_and_fix_blocks` with `filePath` when the content lives in a file so safe editor fixes are applied automatically. ## Jetpack Forms @@ -134,4 +134,4 @@ Then activate the needed module: wp_cli jetpack module activate ``` -3. Use the registered block in page markup and validate with `validate_blocks`. +3. Use the registered block in page markup and validate with `validate_html_blocks`, then `validate_and_fix_blocks` with `filePath` when possible. diff --git a/apps/cli/ai/system-prompt.ts b/apps/cli/ai/system-prompt.ts index d8c6062ef5..eeff3e8935 100644 --- a/apps/cli/ai/system-prompt.ts +++ b/apps/cli/ai/system-prompt.ts @@ -98,7 +98,7 @@ IMPORTANT: For any generated content for the site, these three principles are ma - Gorgeous design: Load the \`visual-design\` skill for site creation, redesign, layout, style, CSS, typography, color, motion, or polish work. - Editable block content: Load the \`block-content\` skill before writing page, post, template, template-part, or other block markup. -- Valid blocks: Use validate_blocks every time to ensure that generated block markup is 100% valid. +- Valid blocks: Use validate_html_blocks first to catch invalid core/html usage, then use validate_and_fix_blocks to validate in the live editor. When called with filePath, validate_and_fix_blocks applies safe editor-serialization fixes directly to that file and returns a CSS-review diff. ## Workflow @@ -117,8 +117,8 @@ Then continue with: 1. **Get site details**: Use site_info to get the site path, URL, and credentials. 2. **Plan the design**: Before writing any code, review the site spec (from the \`site-spec\` skill) and load the \`visual-design\` skill to plan the visual direction: layout, colors, typography, and spacing. 3. **Write theme/plugin files**: For a brand new theme, call \`scaffold_theme\` first — it drops an unopinionated block-theme baseline (style.css with only the theme header, theme.json with appearanceTools only, functions.php with frontend + editor style enqueue, default templates and parts, empty assets/fonts and patterns dirs) and activates it by default. Then use Write and Edit to fill the scaffold (one part/template/file per turn). For plugins or for editing an existing theme, use Write and Edit directly under the site's wp-content/themes/ or wp-content/plugins/ directory. -4. **Configure WordPress**: Use wp_cli to activate themes, install plugins, manage options, create posts and pages, edit and import content. The site must be running. Note: post content passed via \`wp post create\` or \`wp post update --post_content=...\` need to be pre-validated for editability, follow the \`block-content\` skill, and be validated with validate_blocks. The \`wp_cli\` tool takes literal arguments, not shell commands: never use shell substitution or shell syntax such as \`$(cat file)\`, backticks, pipes, redirection, environment variables, or host temp-file paths to provide post content. Pass the literal content directly in \`--post_content=...\`, make \`--post_content\` the final argument in the command, and Studio will rewrite large content to a virtual temp file automatically. -5. **Check the misuse of HTML blocks**: Verify if HTML blocks were used outside the \`block-content\` skill's allowed cases. If they were, convert them to editable blocks and run block validation again. +4. **Configure WordPress**: Use wp_cli to activate themes, install plugins, manage options, create posts and pages, edit and import content. The site must be running. Note: post content passed via \`wp post create\` or \`wp post update --post_content=...\` need to be pre-validated for editability, follow the \`block-content\` skill, checked with validate_html_blocks, and validated/fixed with validate_and_fix_blocks. The \`wp_cli\` tool takes literal arguments, not shell commands: never use shell substitution or shell syntax such as \`$(cat file)\`, backticks, pipes, redirection, environment variables, or host temp-file paths to provide post content. Pass the literal content directly in \`--post_content=...\`, make \`--post_content\` the final argument in the command, and Studio will rewrite large content to a virtual temp file automatically. +5. **Check and fix block validity**: Run validate_html_blocks on block content first. If it reports invalid core/html blocks, rewrite only those blocks as editable core or plugin blocks and call validate_html_blocks again. Then call validate_and_fix_blocks with filePath whenever the content lives in a file. If validate_and_fix_blocks says an auto-fix was applied, the file already contains the fixed block content; do not manually replace markup or call validation again unless you intentionally change block markup afterward. Use the diff only to inspect class/nesting changes and update CSS selectors if needed. For inline content, use any returned fixed block content exactly as the replacement content. 6. **Check the result**: Use take_screenshot with \`viewport: "all"\` to capture the site's landing page on desktop and mobile in one call and verify the design visually on both viewports, check for wrong spacing, alignment, colors, contrast, borders, hover styles and other visual issues. Fix any issues found. Pay particular attention to the navigation menu and the CTA buttons. The design needs to match your original expectations. **Width check**: any section that was meant to be full-width (heroes, banners, edge-to-edge galleries, full-bleed footers) must visibly span the entire viewport in the desktop screenshot. If a "full-width" section only spans the content column (~700px at 1280px viewport), the block markup is missing \`align: "full"\` on the outer group or has a mismatched inner \`layout\` type. Fix in markup, not custom CSS. ## Working cadence @@ -145,7 +145,8 @@ For long CSS or page-content files (>~200 lines), load the \`block-content\` ski - preview_delete: Delete a hosted WordPress.com preview by hostname - wp_cli: Run WP-CLI commands on a running site - scaffold_theme: Scaffold a minimal block theme (style.css, theme.json, functions.php with frontend + editor enqueue, default templates and parts, empty assets/fonts and patterns dirs) into a site and activate it. Use as the first step when starting a new custom theme; the agent fills design-specific content afterwards. Block themes only. -- validate_blocks: Validate block content for correctness on a running site (runs each block through its save() function in a real browser). Requires a site name or path. Call after every file write/edit that contains block content. +- validate_html_blocks: Check core/html blocks for misuse. Call before live editor validation; rewrite reported invalid HTML blocks as editable core or plugin blocks and call this again until it passes. +- validate_and_fix_blocks: Validate block content in the running site's real block editor. With filePath, applies safe editor fixes directly to the file and returns a CSS-review diff. With inline content, returns exact fixed block content plus the diff. Requires a site name or path. Call after every file write/edit that contains block content. - take_screenshot: Take a full-page screenshot of a URL (supports desktop, mobile, or \`viewport: "all"\` for both). Use this to visually check the site after building it. - need_for_speed: Measure frontend performance metrics (TTFB, FCP, LCP, CLS, page weight, DOM size, JS/CSS/image/font asset breakdown) for a running site. Use this to identify performance bottlenecks and guide optimization. - rank_me_up: Run an on-page SEO audit (title/meta tags, headings, image alt text, OpenGraph/Twitter cards, JSON-LD structured data, robots.txt and sitemap.xml availability) for a running site. Use this to identify on-page SEO issues and guide fixes. diff --git a/apps/cli/ai/tests/block-content-policy.test.ts b/apps/cli/ai/tests/block-content-policy.test.ts index 9b88761919..bb91ccb71b 100644 --- a/apps/cli/ai/tests/block-content-policy.test.ts +++ b/apps/cli/ai/tests/block-content-policy.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { getHtmlBlockPolicyIssues } from 'cli/ai/block-content-policy'; +import { getHtmlBlockPolicyIssues, validateHtmlBlockPolicy } from 'cli/ai/block-content-policy'; describe( 'HTML block content policy', () => { it( 'allows inline SVG markup', () => { @@ -35,4 +35,15 @@ describe( 'HTML block content policy', () => { expect( issues ).toHaveLength( 1 ); expect( issues[ 0 ] ).toContain( 'should use editable core blocks' ); } ); + + it( 'returns invalid core/html blocks from full block content', () => { + const report = + validateHtmlBlockPolicy( ` +

Editable text

+

Wrapped text

` ); + + expect( report.totalHtmlBlocks ).toBe( 2 ); + expect( report.invalidHtmlBlocks ).toHaveLength( 1 ); + expect( report.invalidHtmlBlocks[ 0 ].content ).toContain( '
' ); + } ); } ); diff --git a/apps/cli/ai/tests/tools.test.ts b/apps/cli/ai/tests/tools.test.ts index f1092fe21b..475f87318c 100644 --- a/apps/cli/ai/tests/tools.test.ts +++ b/apps/cli/ai/tests/tools.test.ts @@ -3,6 +3,7 @@ import os from 'os'; import path from 'path'; import { SITE_RUNTIME_PLAYGROUND } from '@studio/common/lib/site-runtime'; import { vi } from 'vitest'; +import { validateBlocks } from 'cli/ai/block-validator'; import { getSharedBrowser } from 'cli/ai/browser-utils'; import { emitEvent } from 'cli/ai/json-events'; import { setLocalSiteSelectedCallback } from 'cli/ai/site-selection'; @@ -125,6 +126,37 @@ describe( 'Studio AI MCP tools', () => { tool: ReturnType< typeof resolveStudioToolDefinitions >[ number ], args: Record< string, unknown > ) => tool.execute( 'tool-call-1', args as never, new AbortController().signal, () => {} ); + const mockValidatedFix = ( fixedContent: string, blockName = 'core/paragraph' ) => { + vi.mocked( validateBlocks ).mockResolvedValue( { + totalBlocks: 1, + validBlocks: 0, + invalidBlocks: 1, + results: [ + { + blockName, + isValid: false, + issues: [], + originalContent: '', + }, + ], + proposedFix: { + fixedContent, + report: { + totalBlocks: 1, + validBlocks: 1, + invalidBlocks: 0, + results: [ + { + blockName, + isValid: true, + issues: [], + originalContent: '', + }, + ], + }, + }, + } ); + }; beforeEach( () => { vi.resetAllMocks(); @@ -152,6 +184,63 @@ describe( 'Studio AI MCP tools', () => { ); } ); + it( 'reports invalid core/html blocks', async () => { + const result = await getTool( 'validate_html_blocks' ).rawHandler( { + content: + '
', + } as never ); + + const text = getTextContent( result ); + expect( text ).toContain( 'HTML block policy: 1/1 core/html blocks invalid' ); + expect( text ).toContain( '
' ); + } ); + + it( 'returns fixed inline block content', async () => { + const originalContent = + '

Hello

'; + const fixedContent = + '\n

Hello

\n'; + mockValidatedFix( fixedContent ); + + const result = await getTool( 'validate_and_fix_blocks' ).rawHandler( { + nameOrPath: 'My Site', + content: originalContent, + } as never ); + + expect( validateBlocks ).toHaveBeenCalledWith( + originalContent, + expect.stringContaining( 'localhost:8888' ) + ); + const text = getTextContent( result ); + expect( text ).toContain( 'Fixed block content:' ); + expect( text ).toContain( fixedContent ); + } ); + + it( 'applies valid editor serialization fixes to files', async () => { + const tempDir = await mkdtemp( path.join( os.tmpdir(), 'studio-block-fix-' ) ); + const filePath = path.join( tempDir, 'page.html' ); + const originalContent = + '
'; + const fixedContent = + '\n
\n'; + await writeFile( filePath, originalContent ); + mockValidatedFix( fixedContent, 'core/separator' ); + + try { + const result = await getTool( 'validate_and_fix_blocks' ).rawHandler( { + nameOrPath: 'My Site', + filePath, + } as never ); + + await expect( readFile( filePath, 'utf8' ) ).resolves.toBe( fixedContent ); + const text = getTextContent( result ); + expect( text ).toContain( 'Auto-fix applied: 1/1 blocks valid' ); + expect( text ).not.toContain( 'Fixed block content:' ); + } finally { + await rm( tempDir, { recursive: true, force: true } ); + } + } ); + it( 'exposes the explicit presentation tool when chat artifacts are enabled', () => { const names = resolveStudioToolDefinitions().map( ( tool ) => tool.name ); expect( names ).not.toContain( 'show_artifact' ); diff --git a/apps/cli/ai/tools/index.ts b/apps/cli/ai/tools/index.ts index 968e06c240..1e281feccf 100644 --- a/apps/cli/ai/tools/index.ts +++ b/apps/cli/ai/tools/index.ts @@ -22,7 +22,8 @@ import { stopSiteTool } from './stop-site'; import { studioPresentTool } from './studio-present'; import { takeScreenshotTool } from './take-screenshot'; import { updatePreviewTool } from './update-preview'; -import { validateBlocksTool } from './validate-blocks'; +import { validateAndFixBlocksTool } from './validate-and-fix-blocks'; +import { validateHtmlBlocksTool } from './validate-html-blocks'; import { waitForAnnotationsTool } from './wait-for-annotations'; import { runWpCliTool } from './wp-cli'; import type { AnyStudioAgentTool } from './define-tool'; @@ -42,7 +43,8 @@ export const studioToolDefinitions: AnyStudioAgentTool[] = [ deletePreviewTool, runWpCliTool, scaffoldThemeTool, - validateBlocksTool, + validateHtmlBlocksTool, + validateAndFixBlocksTool, takeScreenshotTool, shareScreenshotTool, installTaxonomyScriptsTool, diff --git a/apps/cli/ai/tools/validate-and-fix-blocks.ts b/apps/cli/ai/tools/validate-and-fix-blocks.ts new file mode 100644 index 0000000000..7689831a86 --- /dev/null +++ b/apps/cli/ai/tools/validate-and-fix-blocks.ts @@ -0,0 +1,154 @@ +import { readFile, writeFile } from 'fs/promises'; +import { Type } from 'typebox'; +import { validateBlocks, type ValidationReportBase } from 'cli/ai/block-validator'; +import { createUnifiedDiff } from 'cli/ai/content-diff'; +import { getSiteUrl } from 'cli/lib/cli-config/sites'; +import { emitProgress } from 'cli/logger'; +import { defineTool } from './define-tool'; +import { resolveSite, textResult } from './utils'; + +function formatInvalidBlocks( report: ValidationReportBase ): string[] { + const lines: string[] = []; + for ( const result of report.results ) { + if ( ! result.isValid ) { + lines.push( ` - ${ result.blockName }` ); + for ( const issue of result.issues ) { + lines.push( ` ${ issue }` ); + } + if ( result.expectedContent !== undefined ) { + lines.push( ` Expected: ${ result.expectedContent }` ); + lines.push( ` Actual: ${ result.originalContent }` ); + } + } + } + return lines; +} + +function formatMarkdownFence( language: string, content: string ): string { + const longestBacktickRun = Math.max( + 0, + ...Array.from( content.matchAll( /`+/g ), ( match ) => match[ 0 ].length ) + ); + const fence = '`'.repeat( Math.max( 3, longestBacktickRun + 1 ) ); + return `${ fence }${ language }\n${ content }\n${ fence }`; +} + +export const validateAndFixBlocksTool = defineTool( + 'validate_and_fix_blocks', + "Validates WordPress block content in the site's real block editor. When filePath is provided, applies safe live-editor serialization fixes directly to the file and returns a diff for CSS impact review. For inline content, returns exact fixed block content plus the diff. The site must be running.", + { + nameOrPath: Type.String( { + description: 'The site name or file system path — the site must be running', + } ), + filePath: Type.Optional( + Type.String( { + description: 'Path to a file containing WordPress block content to validate and fix', + } ) + ), + content: Type.Optional( + Type.String( { + description: 'Raw WordPress block content (HTML with block comments) to validate and fix', + } ) + ), + }, + async ( args ) => { + try { + let blockContent: string; + let fileName = 'inline content'; + let shouldApplyFixToFile = false; + + if ( args.filePath ) { + blockContent = await readFile( args.filePath, 'utf-8' ); + fileName = args.filePath.split( '/' ).slice( -2 ).join( '/' ); + shouldApplyFixToFile = true; + } else if ( args.content !== undefined ) { + blockContent = args.content; + } else { + throw new Error( 'Either content or filePath must be provided.' ); + } + + emitProgress( `Validating and fixing blocks in ${ fileName }…` ); + + const site = await resolveSite( args.nameOrPath ); + const siteUrl = getSiteUrl( site ); + const report = await validateBlocks( blockContent, siteUrl ); + + if ( report.error ) { + emitProgress( `Validation failed for ${ fileName }: ${ report.error.slice( 0, 80 ) }` ); + throw new Error( `Block validation failed: ${ report.error }` ); + } + + if ( report.invalidBlocks === 0 ) { + emitProgress( `${ fileName }: all ${ report.totalBlocks } blocks valid` ); + return textResult( + [ + `Validation: ${ report.validBlocks }/${ report.totalBlocks } blocks valid`, + 'No editor serialization fixes needed.', + ].join( '\n' ) + ); + } + + const invalidNames = report.results + .filter( ( result ) => ! result.isValid ) + .map( ( result ) => result.blockName ) + .join( ', ' ); + emitProgress( `${ fileName }: ${ report.invalidBlocks } invalid (${ invalidNames })` ); + + const lines = [ + `Validation: ${ report.validBlocks }/${ report.totalBlocks } blocks valid`, + '', + 'Invalid blocks:', + ...formatInvalidBlocks( report ), + ]; + + if ( report.proposedFix ) { + const fixedReport = report.proposedFix.report; + if ( fixedReport.error ) { + lines.push( '', `Auto-fix proposal failed validation: ${ fixedReport.error }` ); + } else if ( fixedReport.invalidBlocks === 0 ) { + const fixedContent = report.proposedFix.fixedContent; + const diff = createUnifiedDiff( + blockContent, + fixedContent, + fileName, + `${ fileName } (editor-fixed)` + ); + if ( shouldApplyFixToFile && args.filePath ) { + await writeFile( args.filePath, fixedContent, 'utf-8' ); + emitProgress( `${ fileName }: editor serialization fix applied` ); + lines.push( + '', + `Auto-fix applied: ${ fixedReport.validBlocks }/${ fixedReport.totalBlocks } blocks valid after live-editor serialization.`, + `The fixed block content has already been written to ${ fileName }. Do not replace it manually. Use the diff only to review class/nesting changes and update CSS selectors if needed.` + ); + } else { + lines.push( + '', + `Auto-fix proposal: ${ fixedReport.validBlocks }/${ fixedReport.totalBlocks } blocks valid after live-editor serialization.`, + 'Use the fixed block content below as the replacement block content. Use the diff only to review class/nesting changes and update CSS selectors if needed.', + '', + 'Fixed block content:', + formatMarkdownFence( 'html', fixedContent ) + ); + } + lines.push( '', 'Diff for CSS review:', '```diff', diff, '```' ); + } else { + lines.push( + '', + `Auto-fix proposal still has ${ fixedReport.invalidBlocks } invalid block(s), so no trusted diff is returned.`, + 'Remaining invalid blocks:', + ...formatInvalidBlocks( fixedReport ) + ); + } + } else { + lines.push( '', 'No automatic editor serialization fix was available.' ); + } + + return textResult( lines.join( '\n' ) ); + } catch ( error ) { + throw new Error( + `Block validation failed: ${ error instanceof Error ? error.message : String( error ) }` + ); + } + } +); diff --git a/apps/cli/ai/tools/validate-blocks.ts b/apps/cli/ai/tools/validate-blocks.ts deleted file mode 100644 index f2ad4110b6..0000000000 --- a/apps/cli/ai/tools/validate-blocks.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { readFile } from 'fs/promises'; -import { Type } from 'typebox'; -import { validateBlocks, type ValidationReport } from 'cli/ai/block-validator'; -import { getSiteUrl } from 'cli/lib/cli-config/sites'; -import { emitProgress } from 'cli/logger'; -import { defineTool } from './define-tool'; -import { resolveSite, textResult } from './utils'; - -/** - * Render the invalid-block portion of a validation report as a list of - * indented lines suitable for the agent's tool result. Private to this - * module — only the validate_blocks tool consumes it. - */ -function formatInvalidBlocks( report: ValidationReport ): string[] { - const lines: string[] = []; - for ( const result of report.results ) { - if ( ! result.isValid ) { - lines.push( ` - ${ result.blockName }` ); - for ( const issue of result.issues ) { - lines.push( ` ${ issue }` ); - } - if ( result.expectedContent !== undefined ) { - lines.push( ` Expected: ${ result.expectedContent }` ); - lines.push( ` Actual: ${ result.originalContent }` ); - } - } - } - return lines; -} - -export const validateBlocksTool = defineTool( - 'validate_blocks', - "Validates WordPress block content by running each block through its save() function in the site's block editor (real browser). " + - 'The site must be running. Returns per-block validation results with expected HTML for invalid blocks.', - { - nameOrPath: Type.String( { - description: 'The site name or file system path — the site must be running', - } ), - filePath: Type.Optional( - Type.String( { - description: 'Path to a file containing WordPress block content to validate', - } ) - ), - content: Type.Optional( - Type.String( { - description: 'Raw WordPress block content (HTML with block comments) to validate', - } ) - ), - }, - async ( args ) => { - try { - let blockContent: string; - let fileName = 'inline content'; - - if ( args.filePath ) { - blockContent = await readFile( args.filePath, 'utf-8' ); - fileName = args.filePath.split( '/' ).slice( -2 ).join( '/' ); - } else if ( args.content ) { - blockContent = args.content; - } else { - throw new Error( 'Either content or filePath must be provided.' ); - } - - emitProgress( `Validating blocks in ${ fileName }…` ); - - const site = await resolveSite( args.nameOrPath ); - const siteUrl = getSiteUrl( site ); - const report = await validateBlocks( blockContent, siteUrl ); - - if ( report.error ) { - emitProgress( `Validation failed for ${ fileName }: ${ report.error.slice( 0, 80 ) }` ); - throw new Error( `Block validation failed: ${ report.error }` ); - } - - if ( report.invalidBlocks > 0 ) { - const invalidNames = report.results - .filter( ( r ) => ! r.isValid ) - .map( ( r ) => r.blockName ) - .join( ', ' ); - emitProgress( `${ fileName }: ${ report.invalidBlocks } invalid (${ invalidNames })` ); - } else { - emitProgress( `${ fileName }: all ${ report.totalBlocks } blocks valid` ); - } - - const lines = [ `Validation: ${ report.validBlocks }/${ report.totalBlocks } blocks valid` ]; - - if ( report.invalidBlocks > 0 ) { - lines.push( '', 'Invalid blocks:', ...formatInvalidBlocks( report ) ); - lines.push( - '', - 'Before fixing: each Expected/Actual diff is a structural change, not a literal text swap. Classes the validator adds or removes (has-X-color, alignwide, is-style-Y, wp-block-*-is-layout-flex) pull in or strip core CSS that drives layout, spacing, and color. Diff the markup explicitly, update any style.css selectors that target the old class or nesting in the same edit batch, preserve your intentional className hooks, then take a screenshot of desktop and mobile to verify the design did not drift.' - ); - } - - return textResult( lines.join( '\n' ) ); - } catch ( error ) { - throw new Error( - `Block validation failed: ${ error instanceof Error ? error.message : String( error ) }` - ); - } - } -); diff --git a/apps/cli/ai/tools/validate-html-blocks.ts b/apps/cli/ai/tools/validate-html-blocks.ts new file mode 100644 index 0000000000..4e696b0ab4 --- /dev/null +++ b/apps/cli/ai/tools/validate-html-blocks.ts @@ -0,0 +1,71 @@ +import { readFile } from 'fs/promises'; +import { Type } from 'typebox'; +import { validateHtmlBlockPolicy } from 'cli/ai/block-content-policy'; +import { emitProgress } from 'cli/logger'; +import { defineTool } from './define-tool'; +import { textResult } from './utils'; + +function formatPreview( content: string ): string { + const compact = content.replace( /\s+/g, ' ' ).trim(); + return compact.length > 500 ? compact.slice( 0, 500 ) + '…' : compact; +} + +export const validateHtmlBlocksTool = defineTool( + 'validate_html_blocks', + 'Checks core/html blocks for misuse before editor validation. Returns the invalid HTML blocks that should be rewritten as editable core or plugin blocks.', + { + filePath: Type.Optional( + Type.String( { + description: 'Path to a file containing WordPress block content to check', + } ) + ), + content: Type.Optional( + Type.String( { + description: 'Raw WordPress block content (HTML with block comments) to check', + } ) + ), + }, + async ( args ) => { + let blockContent: string; + let fileName = 'inline content'; + + if ( args.filePath ) { + blockContent = await readFile( args.filePath, 'utf-8' ); + fileName = args.filePath.split( '/' ).slice( -2 ).join( '/' ); + } else if ( args.content !== undefined ) { + blockContent = args.content; + } else { + throw new Error( 'Either content or filePath must be provided.' ); + } + + emitProgress( `Checking HTML blocks in ${ fileName }…` ); + + const report = validateHtmlBlockPolicy( blockContent ); + const lines = [ + `HTML block policy: ${ report.invalidHtmlBlocks.length }/${ report.totalHtmlBlocks } core/html blocks invalid`, + ]; + + if ( report.invalidHtmlBlocks.length === 0 ) { + lines.push( + report.totalHtmlBlocks === 0 + ? 'No core/html blocks found.' + : 'All core/html blocks are within the allowed policy.' + ); + return textResult( lines.join( '\n' ) ); + } + + lines.push( + '', + 'Invalid HTML blocks:', + ...report.invalidHtmlBlocks.flatMap( ( block ) => [ + ` - #${ block.blockNumber } line ${ block.line }`, + ...block.issues.map( ( issue ) => ` ${ issue }` ), + ` Content: ${ formatPreview( block.content ) }`, + ] ), + '', + 'Rewrite each invalid core/html block as editable core or plugin blocks, then call validate_html_blocks again before validate_and_fix_blocks.' + ); + + return textResult( lines.join( '\n' ) ); + } +); diff --git a/apps/cli/ai/ui.ts b/apps/cli/ai/ui.ts index 4a6df6144d..94cbde82e7 100644 --- a/apps/cli/ai/ui.ts +++ b/apps/cli/ai/ui.ts @@ -960,7 +960,7 @@ export class AiChatUI implements AiOutputAdapter { case 'preview_create': case 'preview_list': case 'preview_update': - case 'validate_blocks': { + case 'validate_and_fix_blocks': { const nameOrPath = toolInput?.nameOrPath; if ( typeof nameOrPath === 'string' ) { await this.selectLocalSiteFromTool( nameOrPath ); @@ -1919,7 +1919,8 @@ export class AiChatUI implements AiOutputAdapter { return; } - const maxLength = toolName === 'validate_blocks' ? 2000 : 500; + const maxLength = + toolName === 'validate_html_blocks' || toolName === 'validate_and_fix_blocks' ? 2000 : 500; const truncated = text.length > maxLength ? text.slice( 0, maxLength ) + '…' : text; const resultLines = truncated.split( '\n' ); this.addExpandablePreview( diff --git a/eval/promptfoo.config.yaml b/eval/promptfoo.config.yaml index 9b1ade359a..b051ca33d7 100644 --- a/eval/promptfoo.config.yaml +++ b/eval/promptfoo.config.yaml @@ -251,21 +251,36 @@ tests: const marker = output.split(/\r?\n/).map(l => l.trim()).find(l => l.startsWith('EVAL_RUNNER_RESULT_FILE=')); if (!marker) return { pass: false, score: 0, reason: `no result-file marker on stdout; got: ${output.slice(0, 200)}` }; const d = JSON.parse(readFileSync(marker.slice('EVAL_RUNNER_RESULT_FILE='.length), 'utf8')); - const validationResults = (d.toolResults || []).filter(r => r.toolName === 'validate_blocks'); + const htmlResults = (d.toolResults || []).filter(r => r.toolName === 'validate_html_blocks'); + if (htmlResults.length === 0) { + return { pass: false, score: 0, reason: 'validate_html_blocks was not called before editor validation' }; + } + const lastHtml = htmlResults[htmlResults.length - 1]; + const htmlText = lastHtml.text || ''; + if (lastHtml.isError || !/HTML block policy:\s+0\/\d+\s+core\/html blocks invalid/i.test(htmlText)) { + return { + pass: false, + score: 0, + reason: `final validate_html_blocks result was invalid or inconclusive: ${(htmlText || JSON.stringify(lastHtml)).slice(0, 500)}`, + }; + } + const validationResults = (d.toolResults || []).filter(r => r.toolName === 'validate_and_fix_blocks'); if (validationResults.length === 0) { - return { pass: false, score: 0, reason: 'validate_blocks was not called after building the page' }; + return { pass: false, score: 0, reason: 'validate_and_fix_blocks was not called after HTML policy validation' }; } const last = validationResults[validationResults.length - 1]; const text = last.text || ''; - const failed = last.isError || /Invalid blocks:/i.test(text) || !/Validation:\s+\d+\/\d+\s+blocks valid/i.test(text); + const failed = last.isError || + !/Validation:\s+\d+\/\d+\s+blocks valid/i.test(text) || + /Auto-fix proposal failed validation|Auto-fix proposal still has|No automatic editor serialization fix was available/i.test(text); if (failed) { return { pass: false, score: 0, - reason: `final validate_blocks result was invalid or inconclusive: ${(text || JSON.stringify(last)).slice(0, 500)}`, + reason: `final validate_and_fix_blocks result was invalid or inconclusive: ${(text || JSON.stringify(last)).slice(0, 500)}`, }; } - return { pass: true, score: 1, reason: `validate_blocks passed: ${(last.text || '').split(/\r?\n/)[0]}` }; + return { pass: true, score: 1, reason: `validation passed: ${(last.text || '').split(/\r?\n/)[0]}` }; }); - description: agent reaches for Jetpack on slideshow request (catch-all rule) diff --git a/tools/common/ai/tools.ts b/tools/common/ai/tools.ts index f30f357980..1807ed5a03 100644 --- a/tools/common/ai/tools.ts +++ b/tools/common/ai/tools.ts @@ -23,7 +23,8 @@ export function getToolDisplayName( name: string ): string { preview_delete: __( 'Delete preview' ), wp_cli: __( 'Run WP-CLI' ), scaffold_theme: __( 'Scaffold theme' ), - validate_blocks: __( 'Validate blocks' ), + validate_html_blocks: __( 'Check HTML blocks' ), + validate_and_fix_blocks: __( 'Validate and fix blocks' ), take_screenshot: __( 'Take screenshot' ), share_screenshot: __( 'Share screenshot' ), need_for_speed: __( 'Audit performance' ), @@ -83,7 +84,8 @@ export function getToolDetail( name: string, input?: Record< string, unknown > ) return typeof input.command === 'string' ? `wp ${ input.command }` : ''; case 'scaffold_theme': return typeof input.name === 'string' ? input.name : ''; - case 'validate_blocks': + case 'validate_html_blocks': + case 'validate_and_fix_blocks': if ( typeof input.filePath === 'string' ) { return input.filePath.split( '/' ).slice( -2 ).join( '/' ); }