alihan/browser-use-web-ui
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

stream function only

Browse Source
This commit is contained in:
katiue
2025-01-09 21:32:29 +07:00
parent 6227240a04
commit d466231034
10 changed files with 359 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
requirements.txt
View File

@@ -2,6 +2,5 @@ browser-use>=0.1.18
langchain-google-genai>=2.0.8
pyperclip
gradio
python-dotenv
argparse
langchain-ollama

16
src/agent/custom_agent.py
View File

@@ -85,6 +85,7 @@ class CustomAgent(Agent):
include_attributes=include_attributes,
max_error_length=max_error_length,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content,
)
self.add_infos = add_infos
self.message_manager = CustomMassageManager(
@@ -125,7 +126,9 @@ class CustomAgent(Agent):
f"🛠️ Action {i + 1}/{len(response.action)}: {action.model_dump_json(exclude_unset=True)}"
)
def update_step_info(self, model_output: CustomAgentOutput, step_info: CustomAgentStepInfo | None = None):
def update_step_info(
self, model_output: CustomAgentOutput, step_info: CustomAgentStepInfo = None
):
"""
update step info
"""
@@ -155,7 +158,7 @@ class CustomAgent(Agent):
parsed: AgentOutput = response['parsed']
# cut the number of actions to max_actions_per_step
parsed.action = parsed.action[: self.max_actions_per_step]
self._log_response(parsed) # type: ignore
self._log_response(parsed)
self.n_steps += 1
return parsed
@@ -164,7 +167,7 @@ class CustomAgent(Agent):
# and Manually parse the response. Temporarily solution for DeepSeek
ret = self.llm.invoke(input_messages)
if isinstance(ret.content, list):
parsed_json = json.loads(str(ret.content[0]).replace("```json", "").replace("```", ""))
parsed_json = json.loads(ret.content[0].replace("```json", "").replace("```", ""))
else:
parsed_json = json.loads(ret.content.replace("```json", "").replace("```", ""))
parsed: AgentOutput = self.AgentOutput(**parsed_json)
@@ -191,9 +194,8 @@ class CustomAgent(Agent):
self.message_manager.add_state_message(state, self._last_result, step_info)
input_messages = self.message_manager.get_messages()
model_output = await self.get_next_action(input_messages)
if step_info is not None:
self.update_step_info(model_output=CustomAgentOutput(**model_output.dict()), step_info=step_info)
logger.info(f'🧠 All Memory: {step_info.memory}')
self.update_step_info(model_output, step_info)
logger.info(f"🧠 All Memory: {step_info.memory}")
self._save_conversation(input_messages, model_output)
self.message_manager._remove_last_state_message() # we dont want the whole state in the chat history
self.message_manager.add_model_output(model_output)
@@ -280,4 +282,4 @@ class CustomAgent(Agent):
await self.browser_context.close()
if not self.injected_browser and self.browser:
await self.browser.close()
await self.browser.close()

40
src/agent/custom_massage_manager.py
View File

@@ -12,8 +12,7 @@ from typing import List, Optional, Type
from browser_use.agent.message_manager.service import MessageManager
from browser_use.agent.message_manager.views import MessageHistory
from browser_use.agent.prompts import SystemPrompt
from browser_use.agent.views import ActionResult
from .custom_views import CustomAgentStepInfo
from browser_use.agent.views import ActionResult, AgentStepInfo
from browser_use.browser.views import BrowserState
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import (
@@ -41,7 +40,6 @@ class CustomMassageManager(MessageManager):
max_actions_per_step: int = 10,
tool_call_in_content: bool = False,
):
self.tool_call_in_content = tool_call_in_content
super().__init__(
llm=llm,
task=task,
@@ -53,24 +51,22 @@ class CustomMassageManager(MessageManager):
include_attributes=include_attributes,
max_error_length=max_error_length,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content,
)
# Custom: Move Task info to state_message
self.history = MessageHistory()
self._add_message_with_tokens(self.system_prompt)
tool_calls = self._create_tool_calls()
example_tool_call = self._create_example_tool_call(tool_calls)
self._add_message_with_tokens(example_tool_call)
def _create_tool_calls(self):
return [
tool_calls = [
{
'name': 'AgentOutput',
'name': 'CustomAgentOutput',
'args': {
'current_state': {
'evaluation_previous_goal': 'Unknown - No previous actions to evaluate.',
'memory': '',
'next_goal': 'Obtain task from user',
'prev_action_evaluation': 'Unknown - No previous actions to evaluate.',
'important_contents': '',
'completed_contents': '',
'thought': 'Now Google is open. Need to type OpenAI to search.',
'summary': 'Type OpenAI to search.',
},
'action': [],
},
@@ -78,25 +74,25 @@ class CustomMassageManager(MessageManager):
'type': 'tool_call',
}
]
def _create_example_tool_call(self, tool_calls):
if self.tool_call_in_content:
# openai throws error if tool_calls are not responded -> move to content
return AIMessage(
example_tool_call = AIMessage(
content=f'{tool_calls}',
tool_calls=[],
)
else:
return AIMessage(
example_tool_call = AIMessage(
content=f'',
tool_calls=tool_calls,
)
self._add_message_with_tokens(example_tool_call)
def add_state_message(
self,
state: BrowserState,
result: Optional[List[ActionResult]] = None,
step_info: Optional[CustomAgentStepInfo] = None,
self,
state: BrowserState,
result: Optional[List[ActionResult]] = None,
step_info: Optional[AgentStepInfo] = None,
) -> None:
"""Add browser state as human message"""
@@ -109,7 +105,7 @@ class CustomMassageManager(MessageManager):
self._add_message_with_tokens(msg)
if r.error:
msg = HumanMessage(
content=str(r.error)[-self.max_error_length :]
content=str(r.error)[-self.max_error_length:]
)
self._add_message_with_tokens(msg)
result = None # if result in history, we dont want to add it again

25
src/agent/custom_prompts.py
View File

@@ -148,12 +148,12 @@ class CustomSystemPrompt(SystemPrompt):
class CustomAgentMessagePrompt:
def __init__(
self,
state: BrowserState,
result: Optional[List[ActionResult]] = None,
include_attributes: list[str] = [],
max_error_length: int = 400,
step_info: Optional[CustomAgentStepInfo] = None,
self,
state: BrowserState,
result: Optional[List[ActionResult]] = None,
include_attributes: list[str] = [],
max_error_length: int = 400,
step_info: Optional[CustomAgentStepInfo] = None,
):
self.state = state
self.result = result
@@ -162,19 +162,14 @@ class CustomAgentMessagePrompt:
self.step_info = step_info
def get_user_message(self) -> HumanMessage:
task = self.step_info.task if self.step_info else "No task provided"
add_infos = self.step_info.add_infos if self.step_info else "No hints provided"
memory = self.step_info.memory if self.step_info else "No memory available"
task_progress = self.step_info.task_progress if self.step_info else "No progress recorded"
state_description = f"""
1. Task: {task}
1. Task: {self.step_info.task}
2. Hints(Optional):
{add_infos}
{self.step_info.add_infos}
3. Memory:
{memory}
{self.step_info.memory}
4. Task Progress:
{task_progress}
{self.step_info.task_progress}
5. Current url: {self.state.url}
6. Available tabs:
{self.state.tabs}

2
src/agent/custom_views.py
View File

@@ -56,4 +56,4 @@ class CustomAgentOutput(AgentOutput):
Field(...),
), # Properly annotated field with no default
__module__=CustomAgentOutput.__module__,
)
)

82
src/browser/custom_browser.py
View File

@@ -2,89 +2,19 @@
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: custom_browser.py
# @FileName: browser.py
from browser_use.browser.browser import Browser
from browser_use.browser.context import BrowserContext, BrowserContextConfig
import logging
from playwright.async_api import Playwright, Browser as PlaywrightBrowser, async_playwright
from browser_use.browser.browser import Browser, BrowserConfig
from browser_use.browser.context import BrowserContextConfig, BrowserContext
from .custom_context import CustomBrowserContext
logger = logging.getLogger(__name__)
class CustomBrowser(Browser):
async def new_context(
self,
config: BrowserContextConfig = BrowserContextConfig(),
context=None,
context: CustomBrowserContext = None,
) -> BrowserContext:
"""Create a browser context with custom implementation"""
"""Create a browser context"""
return CustomBrowserContext(config=config, browser=self, context=context)
async def _init(self):
"""Initialize the browser session"""
playwright = await async_playwright().start()
browser = await self._setup_browser(playwright)
self.playwright = playwright
self.playwright_browser = browser
return self.playwright_browser
async def _setup_browser(self, playwright: Playwright) -> PlaywrightBrowser:
"""Sets up and returns a Playwright Browser instance with anti-detection measures."""
try:
disable_security_args = []
if self.config.disable_security:
disable_security_args = [
'--disable-web-security',
'--disable-site-isolation-trials',
'--disable-features=IsolateOrigins,site-per-process',
]
browser = await playwright.chromium.launch(
headless=self.config.headless,
args=[
'--no-sandbox',
'--disable-blink-features=AutomationControlled',
'--disable-infobars',
'--disable-background-timer-throttling',
'--disable-popup-blocking',
'--disable-backgrounding-occluded-windows',
'--disable-renderer-backgrounding',
'--disable-window-activation',
'--disable-focus-on-load',
'--no-first-run',
'--no-default-browser-check',
'--no-startup-window',
'--window-position=0,0',
]
+ disable_security_args
+ self.config.extra_chromium_args,
proxy=self.config.proxy,
)
return browser
except Exception as e:
logger.error(f'Failed to initialize Playwright browser: {str(e)}')
raise
async def get_playwright_browser(self) -> PlaywrightBrowser:
"""Get a browser context"""
if self.playwright_browser is None:
return await self._init()
return self.playwright_browser
async def close(self):
"""Close the browser instance"""
try:
if self.playwright_browser:
await self.playwright_browser.close()
if self.playwright:
await self.playwright.stop()
except Exception as e:
logger.debug(f'Failed to close browser properly: {e}')
finally:
self.playwright_browser = None
self.playwright = None

146
src/browser/custom_context.py
View File

@@ -3,100 +3,94 @@
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: custom_context.py
# @FileName: context.py
import json
import logging
import os
from typing import TYPE_CHECKING
from playwright.async_api import Browser as PlaywrightBrowser, Page, BrowserContext as PlaywrightContext
from browser_use.browser.browser import Browser
from browser_use.browser.context import BrowserContext, BrowserContextConfig
if TYPE_CHECKING:
from .custom_browser import CustomBrowser
from playwright.async_api import Browser as PlaywrightBrowser
logger = logging.getLogger(__name__)
class CustomBrowserContext(BrowserContext):
def __init__(
self,
browser: 'CustomBrowser', # Forward declaration for CustomBrowser
config: BrowserContextConfig = BrowserContextConfig(),
context: 'PlaywrightContext | None' = None
self,
browser: "Browser",
config: BrowserContextConfig = BrowserContextConfig(),
context: BrowserContext = None,
):
super().__init__(browser=browser, config=config) # Add proper inheritance
self._impl_context = context # Rename to avoid confusion
self._page = None
self.session = None # Add session attribute
super(CustomBrowserContext, self).__init__(browser=browser, config=config)
self.context = context
@property
def impl_context(self) -> PlaywrightContext:
"""Returns the underlying Playwright context implementation"""
if self._impl_context is None:
raise RuntimeError("Browser context has not been initialized")
return self._impl_context
async def _create_context(self, browser: PlaywrightBrowser):
"""Creates a new browser context with anti-detection measures and loads cookies if available."""
# If we have a context, return it directly
if self.context:
return self.context
if self.browser.config.chrome_instance_path and len(browser.contexts) > 0:
# Connect to existing Chrome instance instead of creating new one
context = browser.contexts[0]
else:
# Original code for creating new context
context = await browser.new_context(
viewport=self.config.browser_window_size,
no_viewport=False,
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36"
),
java_script_enabled=True,
bypass_csp=self.config.disable_security,
ignore_https_errors=self.config.disable_security,
record_video_dir=self.config.save_recording_path,
record_video_size=self.config.browser_window_size, # set record video size, same as windows size
)
async def _create_context(self, config: BrowserContextConfig | None = None):
"""Creates a new browser context"""
if self._impl_context:
return self._impl_context
if self.config.trace_path:
await context.tracing.start(screenshots=True, snapshots=True, sources=True)
# Get the Playwright browser from our custom browser
pw_browser = await self.browser.get_playwright_browser()
context_args = {
'viewport': self.config.browser_window_size,
'no_viewport': False,
'bypass_csp': self.config.disable_security,
'ignore_https_errors': self.config.disable_security
}
if self.config.save_recording_path:
context_args.update({
'record_video_dir': self.config.save_recording_path,
'record_video_size': self.config.browser_window_size
})
# Load cookies if they exist
if self.config.cookies_file and os.path.exists(self.config.cookies_file):
with open(self.config.cookies_file, "r") as f:
cookies = json.load(f)
logger.info(
f"Loaded {len(cookies)} cookies from {self.config.cookies_file}"
)
await context.add_cookies(cookies)
self._impl_context = await pw_browser.new_context(**context_args)
# Create an initial page
self._page = await self._impl_context.new_page()
await self._page.goto('about:blank') # Ensure page is ready
return self._impl_context
# Expose anti-detection scripts
await context.add_init_script(
"""
// Webdriver property
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
});
async def new_page(self) -> Page:
"""Creates and returns a new page in this context"""
context = await self._create_context()
return await context.new_page()
// Languages
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en']
});
async def __aenter__(self):
if not self._impl_context:
await self._create_context()
return self
// Plugins
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5]
});
async def __aexit__(self, *args):
if self._impl_context:
await self._impl_context.close()
self._impl_context = None
// Chrome runtime
window.chrome = { runtime: {} };
@property
def pages(self):
"""Returns list of pages in context"""
return self._impl_context.pages if self._impl_context else []
// Permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
"""
)
async def get_state(self, **kwargs):
if self._impl_context:
# pages() is a synchronous property, not an async method:
pages = self._impl_context.pages
if pages:
return await super().get_state(**kwargs)
return None
async def get_pages(self):
"""Get pages in a way that works"""
if not self._impl_context:
return []
# Again, pages() is a property:
return self._impl_context.pages
return context

2
src/controller/custom_controller.py
View File

@@ -30,4 +30,4 @@ class CustomController(Controller):
page = await browser.get_current_page()
await page.keyboard.type(text)
return ActionResult(extracted_content=text)
return ActionResult(extracted_content=text)

13
src/utils/utils.py
View File

@@ -7,7 +7,6 @@
import base64
import os
from pydantic import SecretStr
from langchain_anthropic import ChatAnthropic
from langchain_google_genai import ChatGoogleGenerativeAI
@@ -37,9 +36,7 @@ def get_llm_model(provider: str, **kwargs):
model_name=kwargs.get("model_name", "claude-3-5-sonnet-20240620"),
temperature=kwargs.get("temperature", 0.0),
base_url=base_url,
api_key=SecretStr(api_key or ""),
timeout=kwargs.get("timeout", 60),
stop=kwargs.get("stop", None),
api_key=api_key,
)
elif provider == "openai":
if not kwargs.get("base_url", ""):
@@ -56,7 +53,7 @@ def get_llm_model(provider: str, **kwargs):
model=kwargs.get("model_name", "gpt-4o"),
temperature=kwargs.get("temperature", 0.0),
base_url=base_url,
api_key=SecretStr(api_key or ""),
api_key=api_key,
)
elif provider == "deepseek":
if not kwargs.get("base_url", ""):
@@ -73,7 +70,7 @@ def get_llm_model(provider: str, **kwargs):
model=kwargs.get("model_name", "deepseek-chat"),
temperature=kwargs.get("temperature", 0.0),
base_url=base_url,
api_key=SecretStr(api_key or ""),
api_key=api_key,
)
elif provider == "gemini":
if not kwargs.get("api_key", ""):
@@ -83,7 +80,7 @@ def get_llm_model(provider: str, **kwargs):
return ChatGoogleGenerativeAI(
model=kwargs.get("model_name", "gemini-2.0-flash-exp"),
temperature=kwargs.get("temperature", 0.0),
api_key=SecretStr(api_key or ""),
google_api_key=api_key,
)
elif provider == "ollama":
return ChatOllama(
@@ -105,7 +102,7 @@ def get_llm_model(provider: str, **kwargs):
temperature=kwargs.get("temperature", 0.0),
api_version="2024-05-01-preview",
azure_endpoint=base_url,
api_key=SecretStr(api_key or ""),
api_key=api_key,
)
else:
raise ValueError(f"Unsupported provider: {provider}")

420
webui.py
View File

@@ -6,23 +6,28 @@
# @FileName: webui.py
import pdb
import glob
from dotenv import load_dotenv
load_dotenv()
import argparse
import gradio as gr
import os
import gradio as gr
import argparse
from gradio.themes import Base, Default, Soft, Monochrome, Glass, Origin, Citrus, Ocean
import asyncio
from playwright.async_api import async_playwright
import os, glob
from browser_use.agent.service import Agent
from browser_use.browser.browser import Browser, BrowserConfig
from browser_use.browser.context import (
BrowserContextConfig,
BrowserContextWindowSize,
)
from browser_use.agent.service import Agent
from src.browser.custom_browser import CustomBrowser
from src.controller.custom_controller import CustomController
from playwright.async_api import async_playwright
from src.agent.custom_agent import CustomAgent
from src.agent.custom_prompts import CustomSystemPrompt
from src.browser.custom_browser import CustomBrowser
@@ -30,9 +35,7 @@ from src.browser.custom_context import BrowserContextConfig
from src.controller.custom_controller import CustomController
from src.utils import utils
from src.utils.utils import update_model_dropdown
from src.utils.file_utils import get_latest_files
from src.utils.stream_utils import stream_browser_view, capture_screenshot
from src.utils.stream_utils import capture_screenshot
async def run_browser_agent(
agent_type,
@@ -47,15 +50,31 @@ async def run_browser_agent(
window_w,
window_h,
save_recording_path,
save_trace_path,
enable_recording,
task,
add_infos,
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content,
browser_context=None # Added optional argument
tool_call_in_content
):
# Disable recording if the checkbox is unchecked
if not enable_recording:
save_recording_path = None
# Ensure the recording directory exists if recording is enabled
if save_recording_path:
os.makedirs(save_recording_path, exist_ok=True)
# Get the list of existing videos before the agent runs
existing_videos = set()
if save_recording_path:
existing_videos = set(
glob.glob(os.path.join(save_recording_path, "*.[mM][pP]4"))
+ glob.glob(os.path.join(save_recording_path, "*.[wW][eE][bB][mM]"))
)
# Run the agent
llm = utils.get_llm_model(
provider=llm_provider,
@@ -65,22 +84,22 @@ async def run_browser_agent(
api_key=llm_api_key,
)
if agent_type == "org":
return await run_org_agent(
final_result, errors, model_actions, model_thoughts = await run_org_agent(
llm=llm,
headless=headless,
disable_security=disable_security,
window_w=window_w,
window_h=window_h,
save_recording_path=save_recording_path,
save_trace_path=save_trace_path,
task=task,
max_steps=max_steps,
use_vision=use_vision,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content,
browser_context=browser_context, # pass context,
tool_call_in_content=tool_call_in_content
)
elif agent_type == "custom":
return await run_custom_agent(
final_result, errors, model_actions, model_thoughts = await run_custom_agent(
llm=llm,
use_own_browser=use_own_browser,
headless=headless,
@@ -88,17 +107,30 @@ async def run_browser_agent(
window_w=window_w,
window_h=window_h,
save_recording_path=save_recording_path,
save_trace_path=save_trace_path,
task=task,
add_infos=add_infos,
max_steps=max_steps,
use_vision=use_vision,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content,
browser_context=browser_context, # pass context,
tool_call_in_content=tool_call_in_content
)
else:
raise ValueError(f"Invalid agent type: {agent_type}")
# Get the list of videos after the agent runs (if recording is enabled)
latest_video = None
if save_recording_path:
new_videos = set(
glob.glob(os.path.join(save_recording_path, "*.[mM][pP]4"))
+ glob.glob(os.path.join(save_recording_path, "*.[wW][eE][bB][mM]"))
)
if new_videos - existing_videos:
latest_video = list(new_videos - existing_videos)[0] # Get the first new video
return final_result, errors, model_actions, model_thoughts, latest_video
async def run_org_agent(
llm,
headless,
@@ -106,65 +138,48 @@ async def run_org_agent(
window_w,
window_h,
save_recording_path,
save_trace_path,
task,
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content,
browser_context=None, # receive context
tool_call_in_content
):
browser = None
if browser_context is None:
browser = Browser(
config=BrowserConfig(
headless=False, # Force non-headless for streaming
disable_security=disable_security,
extra_chromium_args=[f'--window-size={window_w},{window_h}'],
)
browser = Browser(
config=BrowserConfig(
headless=headless,
disable_security=disable_security,
extra_chromium_args=[f"--window-size={window_w},{window_h}"],
)
async with await browser.new_context(
config=BrowserContextConfig(
trace_path='./tmp/traces',
save_recording_path=save_recording_path if save_recording_path else None,
no_viewport=False,
browser_window_size=BrowserContextWindowSize(width=window_w, height=window_h),
)
) as browser_context_in:
agent = Agent(
task=task,
llm=llm,
use_vision=use_vision,
browser_context=browser_context_in,
)
async with await browser.new_context(
config=BrowserContextConfig(
trace_path=save_trace_path if save_trace_path else None,
save_recording_path=save_recording_path if save_recording_path else None,
no_viewport=False,
browser_window_size=BrowserContextWindowSize(
width=window_w, height=window_h
),
)
history = await agent.run(max_steps=max_steps)
final_result = history.final_result()
errors = history.errors()
model_actions = history.model_actions()
model_thoughts = history.model_thoughts()
recorded_files = get_latest_files(save_recording_path)
trace_file = get_latest_files(save_recording_path + "/../traces")
await browser.close()
return final_result, errors, model_actions, model_thoughts, recorded_files.get('.webm'), trace_file.get('.zip')
else:
# Reuse existing context
) as browser_context:
agent = Agent(
task=task,
llm=llm,
use_vision=use_vision,
max_actions_per_step=max_actions_per_step,
browser_context=browser_context,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content
)
history = await agent.run(max_steps=max_steps)
final_result = history.final_result()
errors = history.errors()
model_actions = history.model_actions()
model_thoughts = history.model_thoughts()
recorded_files = get_latest_files(save_recording_path)
trace_file = get_latest_files(save_recording_path + "/../traces")
return final_result, errors, model_actions, model_thoughts, recorded_files.get('.webm'), trace_file.get('.zip')
await browser.close()
return final_result, errors, model_actions, model_thoughts
async def run_custom_agent(
llm,
@@ -174,17 +189,17 @@ async def run_custom_agent(
window_w,
window_h,
save_recording_path,
save_trace_path,
task,
add_infos,
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content,
browser_context=None, # receive context
tool_call_in_content
):
controller = CustomController()
playwright = None
browser = None
browser_context_ = None
try:
if use_own_browser:
playwright = await async_playwright().start()
@@ -195,12 +210,12 @@ async def run_custom_agent(
chrome_exe = None
elif not os.path.exists(chrome_exe):
raise ValueError(f"Chrome executable not found at {chrome_exe}")
if chrome_use_data == "":
chrome_use_data = None
browser_context_ = await playwright.chromium.launch_persistent_context(
user_data_dir=chrome_use_data if chrome_use_data else "",
user_data_dir=chrome_use_data,
executable_path=chrome_exe,
no_viewport=False,
headless=headless, # 保持浏览器窗口可见
@@ -217,8 +232,26 @@ async def run_custom_agent(
else:
browser_context_ = None
if browser_context is not None:
# Reuse context
browser = CustomBrowser(
config=BrowserConfig(
headless=headless,
disable_security=disable_security,
extra_chromium_args=[f"--window-size={window_w},{window_h}"],
)
)
async with await browser.new_context(
config=BrowserContextConfig(
trace_path=save_trace_path if save_trace_path else None,
save_recording_path=save_recording_path
if save_recording_path
else None,
no_viewport=False,
browser_window_size=BrowserContextWindowSize(
width=window_w, height=window_h
),
),
context=browser_context_,
) as browser_context:
agent = CustomAgent(
task=task,
add_infos=add_infos,
@@ -231,50 +264,11 @@ async def run_custom_agent(
tool_call_in_content=tool_call_in_content
)
history = await agent.run(max_steps=max_steps)
final_result = history.final_result()
errors = history.errors()
model_actions = history.model_actions()
model_thoughts = history.model_thoughts()
recorded_files = get_latest_files(save_recording_path)
trace_file = get_latest_files(save_recording_path + "/../traces")
return final_result, errors, model_actions, model_thoughts, recorded_files.get('.webm'), trace_file.get('.zip')
else:
browser = CustomBrowser(
config=BrowserConfig(
headless=headless,
disable_security=disable_security,
extra_chromium_args=[f'--window-size={window_w},{window_h}'],
)
)
async with await browser.new_context(
config=BrowserContextConfig(
trace_path='./tmp/result_processing',
save_recording_path=save_recording_path if save_recording_path else None,
no_viewport=False,
browser_window_size=BrowserContextWindowSize(width=window_w, height=window_h),
),
context=browser_context_
) as browser_context_in:
agent = CustomAgent(
task=task,
add_infos=add_infos,
use_vision=use_vision,
llm=llm,
browser_context=browser_context_in,
controller=controller,
system_prompt_class=CustomSystemPrompt,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content
)
history = await agent.run(max_steps=max_steps)
final_result = history.final_result()
errors = history.errors()
model_actions = history.model_actions()
model_thoughts = history.model_thoughts()
recorded_files = get_latest_files(save_recording_path)
trace_file = get_latest_files(save_recording_path + "/../traces")
except Exception as e:
import traceback
@@ -284,8 +278,6 @@ async def run_custom_agent(
errors = str(e) + "\n" + traceback.format_exc()
model_actions = ""
model_thoughts = ""
recorded_files = {}
trace_file = {}
finally:
# 显式关闭持久化上下文
if browser_context_:
@@ -294,9 +286,20 @@ async def run_custom_agent(
# 关闭 Playwright 对象
if playwright:
await playwright.stop()
if browser:
await browser.close()
return final_result, errors, model_actions, model_thoughts, trace_file.get('.webm'), recorded_files.get('.zip')
await browser.close()
return final_result, errors, model_actions, model_thoughts
# Define the theme map globally
theme_map = {
"Default": Default(),
"Soft": Soft(),
"Monochrome": Monochrome(),
"Glass": Glass(),
"Origin": Origin(),
"Citrus": Citrus(),
"Ocean": Ocean(),
"Base": Base()
}
async def run_with_stream(
agent_type,
@@ -311,6 +314,8 @@ async def run_with_stream(
window_w,
window_h,
save_recording_path,
save_trace_path,
enable_recording,
task,
add_infos,
max_steps,
@@ -333,7 +338,7 @@ async def run_with_stream(
# Create a new browser context
async with await browser.new_context(
config=BrowserContextConfig(
trace_path="./tmp/traces",
trace_path=save_trace_path,
save_recording_path=save_recording_path,
no_viewport=False,
browser_window_size=BrowserContextWindowSize(
@@ -356,6 +361,8 @@ async def run_with_stream(
window_w,
window_h,
save_recording_path,
save_trace_path,
enable_recording,
task,
add_infos,
max_steps,
@@ -432,21 +439,6 @@ async def run_with_stream(
if browser:
await browser.close()
from gradio.themes import Citrus, Default, Glass, Monochrome, Ocean, Origin, Soft, Base
# Define the theme map globally
theme_map = {
"Default": Default(),
"Soft": Soft(),
"Monochrome": Monochrome(),
"Glass": Glass(),
"Origin": Origin(),
"Citrus": Citrus(),
"Ocean": Ocean(),
"Base": Base()
}
# Create the Gradio UI
def create_ui(theme_name="Ocean"):
css = """
.gradio-container {
@@ -465,8 +457,19 @@ def create_ui(theme_name="Ocean"):
}
"""
with gr.Blocks(title="Browser Use WebUI", theme=theme_map[theme_name], css=css) as demo:
# Header
js = """
function refresh() {
const url = new URL(window.location);
if (url.searchParams.get('__theme') !== 'dark') {
url.searchParams.set('__theme', 'dark');
window.location.href = url.href;
}
}
"""
with gr.Blocks(
title="Browser Use WebUI", theme=theme_map[theme_name], css=css, js=js
) as demo:
with gr.Row():
gr.Markdown(
"""
@@ -547,7 +550,7 @@ def create_ui(theme_name="Ocean"):
value=os.getenv(f"{llm_provider.value.upper()}_API_KEY", ""), # Default to .env value
info="Your API key (leave blank to use .env)"
)
with gr.TabItem("🌐 Browser Settings", id=3):
with gr.Group():
with gr.Row():
@@ -592,75 +595,128 @@ def create_ui(theme_name="Ocean"):
interactive=True, # Allow editing only if recording is enabled
)
save_trace_path = gr.Textbox(
label="Trace Path",
placeholder="e.g. ./tmp/traces",
value="./tmp/traces",
info="Path to save Agent traces",
interactive=True,
)
with gr.TabItem("🤖 Run Agent", id=4):
task = gr.Textbox(
label="Task Description",
lines=4,
placeholder="Enter your task here...",
value="go to google.com and type 'OpenAI' click search and give me the first url",
info="Describe what you want the agent to do",
)
add_infos = gr.Textbox(lines=3, label="Additional Information")
# Results
with gr.Tab("📊 Results"):
browser_view = gr.HTML(
value="<div>Waiting for browser session...</div>",
label="Live Browser View",
add_infos = gr.Textbox(
label="Additional Information",
lines=3,
placeholder="Add any helpful context or instructions...",
info="Optional hints to help the LLM complete the task",
)
final_result_output = gr.Textbox(label="Final Result", lines=3)
errors_output = gr.Textbox(label="Errors", lines=3)
model_actions_output = gr.Textbox(label="Model Actions", lines=3)
model_thoughts_output = gr.Textbox(label="Model Thoughts", lines=3)
recording_file = gr.Video(label="Latest Recording")
trace_file = gr.File(label="Trace File")
with gr.Row():
run_button = gr.Button("▶️ Run Agent", variant="primary")
# Button logic
with gr.Row():
run_button = gr.Button("▶️ Run Agent", variant="primary", scale=2)
stop_button = gr.Button("⏹️ Stop", variant="stop", scale=1)
with gr.TabItem("📊 Results", id=5):
recording_display = gr.Video(label="Latest Recording")
with gr.Group():
gr.Markdown("### Results")
with gr.Row():
with gr.Column():
final_result_output = gr.Textbox(
label="Final Result", lines=3, show_label=True
)
with gr.Column():
errors_output = gr.Textbox(
label="Errors", lines=3, show_label=True
)
with gr.Row():
with gr.Column():
model_actions_output = gr.Textbox(
label="Model Actions", lines=3, show_label=True
)
with gr.Column():
model_thoughts_output = gr.Textbox(
label="Model Thoughts", lines=3, show_label=True
)
with gr.TabItem("🎥 Recordings", id=6):
def list_recordings(save_recording_path):
if not os.path.exists(save_recording_path):
return []
# Get all video files
recordings = glob.glob(os.path.join(save_recording_path, "*.[mM][pP]4")) + glob.glob(os.path.join(save_recording_path, "*.[wW][eE][bB][mM]"))
# Sort recordings by creation time (oldest first)
recordings.sort(key=os.path.getctime)
# Add numbering to the recordings
numbered_recordings = []
for idx, recording in enumerate(recordings, start=1):
filename = os.path.basename(recording)
numbered_recordings.append((recording, f"{idx}. {filename}"))
return numbered_recordings
recordings_gallery = gr.Gallery(
label="Recordings",
value=list_recordings("./tmp/record_videos"),
columns=3,
height="auto",
object_fit="contain"
)
refresh_button = gr.Button("🔄 Refresh Recordings", variant="secondary")
refresh_button.click(
fn=list_recordings,
inputs=save_recording_path,
outputs=recordings_gallery
)
# Attach the callback to the LLM provider dropdown
llm_provider.change(
lambda provider, api_key, base_url: update_model_dropdown(provider, api_key, base_url),
inputs=[llm_provider, llm_api_key, llm_base_url],
outputs=llm_model_name
)
# Add this after defining the components
enable_recording.change(
lambda enabled: gr.update(interactive=enabled),
inputs=enable_recording,
outputs=save_recording_path
)
# Run button click handler
run_button.click(
fn=run_with_stream,
inputs=[
agent_type,
llm_provider,
llm_model_name,
llm_temperature,
llm_base_url,
llm_api_key,
use_own_browser,
headless,
disable_security,
window_w,
window_h,
save_recording_path,
task,
add_infos,
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content,
agent_type, llm_provider, llm_model_name, llm_temperature, llm_base_url, llm_api_key,
use_own_browser, headless, disable_security, window_w, window_h, save_recording_path, save_trace_path,
enable_recording, task, add_infos, max_steps, use_vision, max_actions_per_step, tool_call_in_content
],
outputs=[
browser_view,
final_result_output,
errors_output,
model_actions_output,
model_thoughts_output,
recording_file,
trace_file
],
queue=True,
outputs=[final_result_output, errors_output, model_actions_output, model_thoughts_output, recording_display],
)
return demo
if __name__ == "__main__":
import argparse
def main():
parser = argparse.ArgumentParser(description="Gradio UI for Browser Agent")
parser.add_argument("--ip", type=str, default="0.0.0.0", help="IP address to bind to")
parser.add_argument("--port", type=int, default=7860, help="Port to listen on")
parser.add_argument("--theme", type=str, default="Ocean", choices=theme_map.keys())
parser.add_argument("--ip", type=str, default="127.0.0.1", help="IP address to bind to")
parser.add_argument("--port", type=int, default=7788, help="Port to listen on")
parser.add_argument("--theme", type=str, default="Ocean", choices=theme_map.keys(), help="Theme to use for the UI")
parser.add_argument("--dark-mode", action="store_true", help="Enable dark mode")
args = parser.parse_args()
ui = create_ui(theme_name=args.theme)
ui.launch(server_name=args.ip, server_port=args.port, share=True)
demo = create_ui(theme_name=args.theme)
demo.launch(server_name=args.ip, server_port=args.port)
if __name__ == '__main__':
main()