chore: introduce verification tools (#951)

This commit is contained in:
Pavel Feldman
2025-08-25 18:08:07 -07:00
committed by GitHub
parent 7774ad93ca
commit 78298c3448
5 changed files with 680 additions and 4 deletions

2
config.d.ts vendored
View File

@@ -16,7 +16,7 @@
import type * as playwright from 'playwright';
export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf';
export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf' | 'verify';
export type Config = {
/**

View File

@@ -22,6 +22,7 @@ import files from './tools/files.js';
import form from './tools/form.js';
import install from './tools/install.js';
import keyboard from './tools/keyboard.js';
import mouse from './tools/mouse.js';
import navigate from './tools/navigate.js';
import network from './tools/network.js';
import pdf from './tools/pdf.js';
@@ -29,7 +30,7 @@ import snapshot from './tools/snapshot.js';
import tabs from './tools/tabs.js';
import screenshot from './tools/screenshot.js';
import wait from './tools/wait.js';
import mouse from './tools/mouse.js';
import verify from './tools/verify.js';
import type { Tool } from './tools/tool.js';
import type { FullConfig } from './config.js';
@@ -51,6 +52,7 @@ export const allTools: Tool<any>[] = [
...snapshot,
...tabs,
...wait,
...verify,
];
export function filteredTools(config: FullConfig) {

149
src/tools/verify.ts Normal file
View File

@@ -0,0 +1,149 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { z } from 'zod';
import { defineTabTool } from './tool.js';
import * as javascript from '../utils/codegen.js';
import { generateLocator } from './utils.js';
const verifyElement = defineTabTool({
capability: 'verify',
schema: {
name: 'browser_verify_element_visible',
title: 'Verify element visible',
description: 'Verify element is visible on the page',
inputSchema: z.object({
role: z.string().describe('ROLE of the element. Can be found in the snapshot like this: \`- {ROLE} "Accessible Name":\`'),
accessibleName: z.string().describe('ACCESSIBLE_NAME of the element. Can be found in the snapshot like this: \`- role "{ACCESSIBLE_NAME}"\`'),
}),
type: 'readOnly',
},
handle: async (tab, params, response) => {
const locator = tab.page.getByRole(params.role as any, { name: params.accessibleName });
if (await locator.count() === 0) {
response.addError(`Element with role "${params.role}" and accessible name "${params.accessibleName}" not found`);
return;
}
response.addCode(`await expect(page.getByRole(${javascript.escapeWithQuotes(params.role)}, { name: ${javascript.escapeWithQuotes(params.accessibleName)} })).toBeVisible();`);
response.addResult('Done');
},
});
const verifyText = defineTabTool({
capability: 'verify',
schema: {
name: 'browser_verify_text_visible',
title: 'Verify text visible',
description: `Verify text is visible on the page. Prefer ${verifyElement.schema.name} if possible.`,
inputSchema: z.object({
text: z.string().describe('TEXT to verify. Can be found in the snapshot like this: \`- role "Accessible Name": {TEXT}\` or like this: \`- text: {TEXT}\`'),
}),
type: 'readOnly',
},
handle: async (tab, params, response) => {
const locator = tab.page.getByText(params.text).filter({ visible: true });
if (await locator.count() === 0) {
response.addError('Text not found');
return;
}
response.addCode(`await expect(page.getByText(${javascript.escapeWithQuotes(params.text)})).toBeVisible();`);
response.addResult('Done');
},
});
const verifyList = defineTabTool({
capability: 'verify',
schema: {
name: 'browser_verify_list_visible',
title: 'Verify list visible',
description: 'Verify list is visible on the page',
inputSchema: z.object({
element: z.string().describe('Human-readable list description'),
ref: z.string().describe('Exact target element reference that points to the list'),
items: z.array(z.string()).describe('Items to verify'),
}),
type: 'readOnly',
},
handle: async (tab, params, response) => {
const locator = await tab.refLocator({ ref: params.ref, element: params.element });
const itemTexts: string[] = [];
for (const item of params.items) {
const itemLocator = locator.getByText(item);
if (await itemLocator.count() === 0) {
response.addError(`Item "${item}" not found`);
return;
}
itemTexts.push((await itemLocator.textContent())!);
}
const ariaSnapshot = `\`
- list:
${itemTexts.map(t => ` - text: ${javascript.escapeWithQuotes(t, '"')}`).join('\n')}
\``;
response.addCode(`await expect(page.locator('body')).toMatchAriaSnapshot(${ariaSnapshot});`);
response.addResult('Done');
},
});
const verifyValue = defineTabTool({
capability: 'verify',
schema: {
name: 'browser_verify_value',
title: 'Verify value',
description: 'Verify element value',
inputSchema: z.object({
type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the element'),
element: z.string().describe('Human-readable element description'),
ref: z.string().describe('Exact target element reference that points to the element'),
value: z.string().describe('Value to verify. For checkbox, use "true" or "false".'),
}),
type: 'readOnly',
},
handle: async (tab, params, response) => {
const locator = await tab.refLocator({ ref: params.ref, element: params.element });
const locatorSource = `page.${await generateLocator(locator)}`;
if (params.type === 'textbox' || params.type === 'slider' || params.type === 'combobox') {
const value = await locator.inputValue();
if (value !== params.value) {
response.addError(`Expected value "${params.value}", but got "${value}"`);
return;
}
response.addCode(`await expect(${locatorSource}).toHaveValue(${javascript.quote(params.value)});`);
} else if (params.type === 'checkbox' || params.type === 'radio') {
const value = await locator.isChecked();
if (value !== (params.value === 'true')) {
response.addError(`Expected value "${params.value}", but got "${value}"`);
return;
}
const matcher = value ? 'toBeChecked' : 'not.toBeChecked';
response.addCode(`await expect(${locatorSource}).${matcher}();`);
}
response.addResult('Done');
},
});
export default [
verifyElement,
verifyText,
verifyList,
verifyValue,
];

View File

@@ -31,6 +31,7 @@ import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js';
import type { Stream } from 'stream';
export type TestOptions = {
mcpArgs: string[] | undefined;
mcpBrowser: string | undefined;
mcpMode: 'docker' | undefined;
};
@@ -65,17 +66,19 @@ type WorkerFixtures = {
export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>({
mcpArgs: [undefined, { option: true }],
client: async ({ startClient }, use) => {
const { client } = await startClient();
await use(client);
},
startClient: async ({ mcpHeadless, mcpBrowser, mcpMode }, use, testInfo) => {
startClient: async ({ mcpHeadless, mcpBrowser, mcpMode, mcpArgs }, use, testInfo) => {
const configDir = path.dirname(test.info().config.configFile!);
const clients: Client[] = [];
await use(async options => {
const args: string[] = [];
const args: string[] = mcpArgs ?? [];
if (process.env.CI && process.platform === 'linux')
args.push('--no-sandbox');
if (mcpHeadless)

522
tests/verify.spec.ts Normal file
View File

@@ -0,0 +1,522 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect } from './fixtures.js';
test.use({ mcpArgs: ['--caps=verify'] });
test('browser_verify_element_visible', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<button>Submit</button>
<h1>Welcome</h1>
<div role="alert" aria-label="Success message"></div>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_element_visible',
arguments: {
role: 'button',
accessibleName: 'Submit',
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByRole('button', { name: 'Submit' })).toBeVisible();`,
});
expect(await client.callTool({
name: 'browser_verify_element_visible',
arguments: {
role: 'heading',
accessibleName: 'Welcome',
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByRole('heading', { name: 'Welcome' })).toBeVisible();`,
});
expect(await client.callTool({
name: 'browser_verify_element_visible',
arguments: {
role: 'alert',
accessibleName: 'Success message',
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByRole('alert', { name: 'Success message' })).toBeVisible();`,
});
});
test('browser_verify_element_visible (not found)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<button>Submit</button>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_element_visible',
arguments: {
role: 'button',
accessibleName: 'Cancel',
},
})).toHaveResponse({
isError: true,
result: 'Element with role "button" and accessible name "Cancel" not found',
});
});
test('browser_verify_text_visible', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<p>Hello world</p>
<div>Welcome to our site</div>
<span>Status: Active</span>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_text_visible',
arguments: {
text: 'Hello world',
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByText('Hello world')).toBeVisible();`,
});
expect(await client.callTool({
name: 'browser_verify_text_visible',
arguments: {
text: 'Welcome to our site',
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByText('Welcome to our site')).toBeVisible();`,
});
expect(await client.callTool({
name: 'browser_verify_text_visible',
arguments: {
text: 'Status: Active',
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByText('Status: Active')).toBeVisible();`,
});
});
test('browser_verify_text_visible (not found)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<p>Hello world</p>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_text_visible',
arguments: {
text: 'Goodbye world',
},
})).toHaveResponse({
isError: true,
result: 'Text not found',
});
});
test('browser_verify_text_visible (with quotes)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<p>She said "Hello world"</p>
<div>It's a beautiful day</div>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_text_visible',
arguments: {
text: 'She said "Hello world"',
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByText('She said "Hello world"')).toBeVisible();`,
});
expect(await client.callTool({
name: 'browser_verify_text_visible',
arguments: {
text: "It's a beautiful day",
},
})).toHaveResponse({
result: 'Done',
code: `await expect(page.getByText('It\\'s a beautiful day')).toBeVisible();`,
});
});
test('browser_verify_list_visible', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<ul>
<li>Apple</li>
<li>Banana</li>
<li>Cherry</li>
</ul>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_list_visible',
arguments: {
element: 'Fruit list',
ref: 'e2',
items: ['Apple', 'Banana', 'Cherry'],
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.locator('body')).toMatchAriaSnapshot(\`
- list:
- text: "Apple"
- text: "Banana"
- text: "Cherry"
\`);`),
});
});
test('browser_verify_list_visible (partial items)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<ul>
<li>Apple</li>
<li>Banana</li>
<li>Cherry</li>
<li>Date</li>
</ul>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_list_visible',
arguments: {
element: 'Fruit list',
ref: 'e2',
items: ['Apple', 'Cherry'],
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.locator('body')).toMatchAriaSnapshot(\`
- list:
- text: "Apple"
- text: "Cherry"
\`);`),
});
});
test('browser_verify_list_visible (item not found)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<ul>
<li>Apple</li>
<li>Banana</li>
</ul>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_list_visible',
arguments: {
element: 'Fruit list',
ref: 'e2',
items: ['Apple', 'Cherry'],
},
})).toHaveResponse({
isError: true,
result: 'Item "Cherry" not found',
});
});
test('browser_verify_value (textbox)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<input type="text" aria-label="Name" value="John Doe" />
<input type="email" aria-label="Email" value="john@example.com" />
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'textbox',
element: 'Name textbox',
ref: 'e3',
value: 'John Doe',
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.getByRole('textbox', { name: 'Name' })).toHaveValue('John Doe');`),
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'textbox',
element: 'Email textbox',
ref: 'e4',
value: 'john@example.com',
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.getByRole('textbox', { name: 'Email' })).toHaveValue('john@example.com');`),
});
});
test('browser_verify_value (textbox wrong value)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<input type="text" name="name" value="John Doe" />
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'textbox',
element: 'Name textbox',
ref: 'e3',
value: 'Jane Smith',
},
})).toHaveResponse({
isError: true,
result: 'Expected value "Jane Smith", but got "John Doe"',
});
});
test('browser_verify_value (checkbox checked)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<input type="checkbox" name="subscribe" checked />
<label for="subscribe">Subscribe to newsletter</label>
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'checkbox',
element: 'Subscribe checkbox',
ref: 'e3',
value: 'true',
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.getByRole('checkbox')).toBeChecked();`),
});
});
test('browser_verify_value (checkbox unchecked)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<input type="checkbox" name="subscribe" />
<label for="subscribe">Subscribe to newsletter</label>
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'checkbox',
element: 'Subscribe checkbox',
ref: 'e3',
value: 'false',
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.getByRole('checkbox')).not.toBeChecked();`),
});
});
test('browser_verify_value (checkbox wrong value)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<input type="checkbox" name="subscribe" checked />
<label for="subscribe">Subscribe to newsletter</label>
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'checkbox',
element: 'Subscribe checkbox',
ref: 'e3',
value: 'false',
},
})).toHaveResponse({
isError: true,
result: 'Expected value "false", but got "true"',
});
});
test('browser_verify_value (radio checked)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<label for="red">Red</label>
<input id="red" type="radio" name="color" value="red" checked />
<label for="blue">Blue</label>
<input id="blue" type="radio" name="color" value="blue" />
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'radio',
element: 'Color radio',
ref: 'e4',
value: 'true',
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.getByRole('radio', { name: 'Red' })).toBeChecked();`),
});
});
test('browser_verify_value (slider)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<input type="range" name="volume" min="0" max="100" value="75" />
<label>Volume</label>
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'slider',
element: 'Volume slider',
ref: 'e3',
value: '75',
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.getByRole('slider')).toHaveValue('75');`),
});
});
test('browser_verify_value (combobox)', async ({ client, server }) => {
server.setContent('/', `
<title>Test Page</title>
<form>
<select name="country">
<option>Choose a country</option>
<option selected>United States</option>
<option>United Kingdom</option>
</select>
</form>
`, 'text/html');
await client.callTool({
name: 'browser_navigate',
arguments: { url: server.PREFIX },
});
expect(await client.callTool({
name: 'browser_verify_value',
arguments: {
type: 'combobox',
element: 'Country select',
ref: 'e3',
value: 'United States',
},
})).toHaveResponse({
result: 'Done',
code: expect.stringContaining(`await expect(page.getByRole('combobox')).toHaveValue('United States');`),
});
});