diff --git a/packages/midscene/tests/ai/evaluate/__snapshots__/planning-input.test.ts.snap b/packages/midscene/tests/ai/evaluate/__snapshots__/planning-input.test.ts.snap new file mode 100644 index 00000000..59816e31 --- /dev/null +++ b/packages/midscene/tests/ai/evaluate/__snapshots__/planning-input.test.ts.snap @@ -0,0 +1,511 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`automation - planning input - openAI > input value 1`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value 2`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, + { + "param": { + "value": "Enter", + }, + "thought": undefined, + "type": "KeyboardPress", + }, +] +`; + +exports[`automation - planning input - openAI > input value 3`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value 4`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, + { + "param": { + "value": "Enter", + }, + "thought": undefined, + "type": "KeyboardPress", + }, +] +`; + +exports[`automation - planning input - openAI > input value 5`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value 6`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, + { + "param": { + "value": "Enter", + }, + "thought": undefined, + "type": "KeyboardPress", + }, +] +`; + +exports[`automation - planning input - openAI > input value 7`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value 8`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, + { + "param": { + "value": "Enter", + }, + "thought": undefined, + "type": "KeyboardPress", + }, +] +`; + +exports[`automation - planning input - openAI > input value 9`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value 10`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "learning english", + }, + "thought": undefined, + "type": "Input", + }, + { + "param": { + "value": "Enter", + }, + "thought": undefined, + "type": "KeyboardPress", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 1`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn English tomorrow", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 2`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn Skiing", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 3`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 4`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn English tomorrow", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 5`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn Skiing", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 6`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 7`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn English tomorrow", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 8`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn Skiing", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 9`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 10`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn English tomorrow", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 11`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn Skiing", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 12`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 13`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn English tomorrow", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 14`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn Skiing", + }, + "thought": undefined, + "type": "Input", + }, +] +`; + +exports[`automation - planning input - openAI > input value Add, delete, correct and check 15`] = ` +[ + { + "param": { + "prompt": "", + }, + "thought": undefined, + "type": "Locate", + }, + { + "param": { + "value": "Learn", + }, + "thought": undefined, + "type": "Input", + }, +] +`; diff --git a/packages/midscene/tests/ai/evaluate/__snapshots__/planning.test.ts.snap b/packages/midscene/tests/ai/evaluate/__snapshots__/planning.test.ts.snap new file mode 100644 index 00000000..1bc807ff --- /dev/null +++ b/packages/midscene/tests/ai/evaluate/__snapshots__/planning.test.ts.snap @@ -0,0 +1,13 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`automation - planning openAI > basic run 1`] = ` +{ + "timeMs": 3500, +} +`; + +exports[`automation - planning openAI > basic run 2`] = ` +{ + "value": "Enter", +} +`; diff --git a/packages/midscene/tests/ai/evaluate/test-suite/util.ts b/packages/midscene/tests/ai/evaluate/test-suite/util.ts index 82725c21..bfa5ec98 100644 --- a/packages/midscene/tests/ai/evaluate/test-suite/util.ts +++ b/packages/midscene/tests/ai/evaluate/test-suite/util.ts @@ -163,6 +163,6 @@ export async function getPageTestData(targetDir: string) { } export async function getPageDataOfTestName(testName: string) { - const targetDir = path.join(__dirname, `test-data/${testName}`); + const targetDir = path.join(__dirname, `../test-data/${testName}`); return await getPageTestData(targetDir); } diff --git a/packages/midscene/tests/ai/util.ts b/packages/midscene/tests/ai/util.ts index 4a690941..414fd187 100644 --- a/packages/midscene/tests/ai/util.ts +++ b/packages/midscene/tests/ai/util.ts @@ -16,4 +16,4 @@ export const modelList: Array<'openAI' | 'coze'> = useCozeModel('coze') ? ['openAI', 'coze'] : ['openAI']; -export const repeatTime = process.env.GITHUB_ACTIONS ? 2 : 4; +export const repeatTime = process.env.GITHUB_ACTIONS ? 2 : 1; diff --git a/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts b/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts index 936c3f12..194249cb 100644 --- a/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts @@ -21,8 +21,6 @@ test('ai report', async ({ page, ai, aiAssert }, testInfo) => { type: 'query', }, ); - const parseList = JSON.stringify(actionsList, null, 4); - expect(parseList).toMatchSnapshot(); await aiAssert( 'On the left taskbar, check whether the specific execution content of the right task is normal', ); diff --git a/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts-snapshots/ai-report-1-report.txt b/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts-snapshots/ai-report-1-report.txt deleted file mode 100644 index b14d493c..00000000 --- a/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts-snapshots/ai-report-1-report.txt +++ /dev/null @@ -1,66 +0,0 @@ -[ - { - "title": "Enter \"Learn\" in the task box, don't press enter", - "actions": [ - "Planning", - "Insight / Locate", - "Action / Input" - ] - }, - { - "title": "Add \"JS today\" to base on the existing content(important) of the task box, then press enter", - "actions": [ - "Planning", - "Insight / Locate", - "Action / Input", - "Action / KeyboardPress" - ] - }, - { - "title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create", - "actions": [ - "Planning", - "Insight / Locate", - "Action / Input", - "Action / KeyboardPress" - ] - }, - { - "title": "Enter \"Learning AI the day after tomorrow\" in the task box, then press Enter to create", - "actions": [ - "Planning", - "Insight / Locate", - "Action / Input", - "Action / KeyboardPress" - ] - }, - { - "title": "string[], tasks in the list", - "actions": [ - "Insight / Query" - ] - }, - { - "title": "Move your mouse over the second item in the task list", - "actions": [ - "Planning", - "Insight / Locate", - "Action / Hover" - ] - }, - { - "title": "Click the delete button to the right of the second task", - "actions": [ - "Planning", - "Insight / Locate", - "Action / Tap" - ] - }, - { - "title": "Click the check button to the left of the second task", - "actions": [ - "Planning", - "Insight / Locate" - ] - } -] \ No newline at end of file