alihan/browser-use-web-ui
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main'

Browse Source
This commit is contained in:
Måns Abrahamsson
2025-01-31 08:35:13 +01:00
parent 456988fef0 566bca77cf
commit 79b61a95fd
26 changed files with 736 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.env.example
View File

@@ -11,6 +11,8 @@ AZURE_OPENAI_API_KEY=
DEEPSEEK_ENDPOINT=https://api.deepseek.com
DEEPSEEK_API_KEY=
OLLAMA_ENDPOINT=http://localhost:11434
# Set to false to disable anonymized telemetry
ANONYMIZED_TELEMETRY=true
@@ -22,12 +24,16 @@ CHROME_PATH=
CHROME_USER_DATA=
CHROME_DEBUGGING_PORT=9222
CHROME_DEBUGGING_HOST=localhost
CHROME_PERSISTENT_SESSION=false # Set to true to keep browser open between AI tasks
# Set to true to keep browser open between AI tasks
CHROME_PERSISTENT_SESSION=false
# Display settings
RESOLUTION=1920x1080x24 # Format: WIDTHxHEIGHTxDEPTH
RESOLUTION_WIDTH=1920 # Width in pixels
RESOLUTION_HEIGHT=1080 # Height in pixels
# Format: WIDTHxHEIGHTxDEPTH
RESOLUTION=1920x1080x24
# Width in pixels
RESOLUTION_WIDTH=1920
# Height in pixels
RESOLUTION_HEIGHT=1080
# VNC settings
VNC_PASSWORD=youvncpassword

1
Dockerfile
View File

@@ -3,6 +3,7 @@ FROM python:3.11-slim
# Install system dependencies
RUN apt-get update && apt-get install -y \
wget \
netcat-traditional \
gnupg \
curl \
unzip \

2
README.md
View File

@@ -179,6 +179,6 @@ playwright install
```
## Changelog
- [x] **2025/01/26:** Thanks to @vvincent1234. Now browser-use-webui can combine with DeepSeek-r1 to engage in deep thinking!
- [x] **2025/01/10:** Thanks to @casistack. Now we have Docker Setup option and also Support keep browser open between tasks.[Video tutorial demo](https://github.com/browser-use/web-ui/issues/1#issuecomment-2582511750).
- [x] **2025/01/06:** Thanks to @richard-devbot. A New and Well-Designed WebUI is released. [Video tutorial demo](https://github.com/warmshao/browser-use-webui/issues/1#issuecomment-2573393113).

19
SECURITY.md Normal file
View File

@@ -0,0 +1,19 @@
## Reporting Security Issues
If you believe you have found a security vulnerability in browser-use, please report it through coordinated disclosure.
**Please do not report security vulnerabilities through the repository issues, discussions, or pull requests.**
Instead, please open a new [Github security advisory](https://github.com/browser-use/web-ui/security/advisories/new).
Please include as much of the information listed below as you can to help me better understand and resolve the issue:
* The type of issue (e.g., buffer overflow, SQL injection, or cross-site scripting)
* Full paths of source file(s) related to the manifestation of the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Any special configuration required to reproduce the issue
* Step-by-step instructions to reproduce the issue
* Proof-of-concept or exploit code (if possible)
* Impact of the issue, including how an attacker might exploit the issue
This information will help me triage your report more quickly.

1
docker-compose.yml
View File

@@ -1,5 +1,6 @@
services:
browser-use-webui:
platform: linux/amd64
build:
context: .
dockerfile: Dockerfile

8
requirements.txt
View File

@@ -1,7 +1,3 @@
browser-use==0.1.19
langchain-google-genai==2.0.8
browser-use==0.1.29
pyperclip==1.9.0
gradio==5.9.1
langchain-ollama==0.2.2
langchain-openai==0.2.14
langchain-mistralai==0.2.4
gradio==5.10.0

6
src/__init__.py
View File

@@ -1,6 +0,0 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: __init__.py.py

6
src/agent/__init__.py
View File

@@ -1,6 +0,0 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: __init__.py.py

442
src/agent/custom_agent.py
View File

@@ -1,19 +1,13 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: custom_agent.py
import json
import logging
import pdb
import traceback
from typing import Optional, Type
from typing import Optional, Type, List, Dict, Any, Callable
from PIL import Image, ImageDraw, ImageFont
import os
import base64
import io
import platform
from browser_use.agent.prompts import SystemPrompt
from browser_use.agent.service import Agent
from browser_use.agent.views import (
@@ -27,9 +21,9 @@ from browser_use.browser.context import BrowserContext
from browser_use.browser.views import BrowserStateHistory
from browser_use.controller.service import Controller
from browser_use.telemetry.views import (
AgentEndTelemetryEvent,
AgentRunTelemetryEvent,
AgentStepErrorTelemetryEvent,
AgentEndTelemetryEvent,
AgentRunTelemetryEvent,
AgentStepTelemetryEvent,
)
from browser_use.utils import time_execution_async
from langchain_core.language_models.chat_models import BaseChatModel
@@ -76,6 +70,11 @@ class CustomAgent(Agent):
max_actions_per_step: int = 10,
tool_call_in_content: bool = True,
agent_state: AgentState = None,
initial_actions: Optional[List[Dict[str, Dict[str, Any]]]] = None,
# Cloud Callbacks
register_new_step_callback: Callable[['BrowserState', 'AgentOutput', int], None] | None = None,
register_done_callback: Callable[['AgentHistoryList'], None] | None = None,
tool_calling_method: Optional[str] = 'auto',
):
super().__init__(
task=task,
@@ -94,8 +93,22 @@ class CustomAgent(Agent):
max_error_length=max_error_length,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content,
initial_actions=initial_actions,
register_new_step_callback=register_new_step_callback,
register_done_callback=register_done_callback,
tool_calling_method=tool_calling_method
)
if self.model_name in ["deepseek-reasoner"] or self.model_name.startswith("deepseek-r1"):
# deepseek-reasoner does not support function calling
self.use_deepseek_r1 = True
# deepseek-reasoner only support 64000 context
self.max_input_tokens = 64000
else:
self.use_deepseek_r1 = False
# custom new info
self.add_infos = add_infos
# agent_state for Stop
self.agent_state = agent_state
self.message_manager = CustomMassageManager(
llm=self.llm,
@@ -106,7 +119,7 @@ class CustomAgent(Agent):
include_attributes=self.include_attributes,
max_error_length=self.max_error_length,
max_actions_per_step=self.max_actions_per_step,
tool_call_in_content=tool_call_in_content,
use_deepseek_r1=self.use_deepseek_r1
)
def _setup_action_models(self) -> None:
@@ -127,7 +140,8 @@ class CustomAgent(Agent):
logger.info(f"{emoji} Eval: {response.current_state.prev_action_evaluation}")
logger.info(f"🧠 New Memory: {response.current_state.important_contents}")
logger.info(f"⏳ Task Progress: {response.current_state.completed_contents}")
logger.info(f"⏳ Task Progress: \n{response.current_state.task_progress}")
logger.info(f"📋 Future Plans: \n{response.current_state.future_plans}")
logger.info(f"🤔 Thought: {response.current_state.thought}")
logger.info(f"🎯 Summary: {response.current_state.summary}")
for i, action in enumerate(response.action):
@@ -153,42 +167,50 @@ class CustomAgent(Agent):
):
step_info.memory += important_contents + "\n"
completed_contents = model_output.current_state.completed_contents
if completed_contents and "None" not in completed_contents:
step_info.task_progress = completed_contents
task_progress = model_output.current_state.task_progress
if task_progress and "None" not in task_progress:
step_info.task_progress = task_progress
future_plans = model_output.current_state.future_plans
if future_plans and "None" not in future_plans:
step_info.future_plans = future_plans
@time_execution_async("--get_next_action")
async def get_next_action(self, input_messages: list[BaseMessage]) -> AgentOutput:
"""Get next action from LLM based on current state"""
try:
structured_llm = self.llm.with_structured_output(self.AgentOutput, include_raw=True)
response: dict[str, Any] = await structured_llm.ainvoke(input_messages) # type: ignore
parsed: AgentOutput = response['parsed']
# cut the number of actions to max_actions_per_step
parsed.action = parsed.action[: self.max_actions_per_step]
self._log_response(parsed)
self.n_steps += 1
return parsed
except Exception as e:
# If something goes wrong, try to invoke the LLM again without structured output,
# and Manually parse the response. Temporarily solution for DeepSeek
ret = self.llm.invoke(input_messages)
if isinstance(ret.content, list):
parsed_json = json.loads(ret.content[0].replace("```json", "").replace("```", ""))
if self.use_deepseek_r1:
merged_input_messages = self.message_manager.merge_successive_human_messages(input_messages)
ai_message = self.llm.invoke(merged_input_messages)
self.message_manager._add_message_with_tokens(ai_message)
logger.info(f"🤯 Start Deep Thinking: ")
logger.info(ai_message.reasoning_content)
logger.info(f"🤯 End Deep Thinking")
if isinstance(ai_message.content, list):
parsed_json = json.loads(ai_message.content[0].replace("```json", "").replace("```", ""))
else:
parsed_json = json.loads(ret.content.replace("```json", "").replace("```", ""))
parsed_json = json.loads(ai_message.content.replace("```json", "").replace("```", ""))
parsed: AgentOutput = self.AgentOutput(**parsed_json)
if parsed is None:
logger.debug(ai_message.content)
raise ValueError(f'Could not parse response.')
else:
ai_message = self.llm.invoke(input_messages)
self.message_manager._add_message_with_tokens(ai_message)
if isinstance(ai_message.content, list):
parsed_json = json.loads(ai_message.content[0].replace("```json", "").replace("```", ""))
else:
parsed_json = json.loads(ai_message.content.replace("```json", "").replace("```", ""))
parsed: AgentOutput = self.AgentOutput(**parsed_json)
if parsed is None:
logger.debug(ai_message.content)
raise ValueError(f'Could not parse response.')
# cut the number of actions to max_actions_per_step
parsed.action = parsed.action[: self.max_actions_per_step]
self._log_response(parsed)
self.n_steps += 1
# cut the number of actions to max_actions_per_step
parsed.action = parsed.action[: self.max_actions_per_step]
self._log_response(parsed)
self.n_steps += 1
return parsed
return parsed
@time_execution_async("--step")
async def step(self, step_info: Optional[CustomAgentStepInfo] = None) -> None:
@@ -202,16 +224,32 @@ class CustomAgent(Agent):
state = await self.browser_context.get_state(use_vision=self.use_vision)
self.message_manager.add_state_message(state, self._last_result, step_info)
input_messages = self.message_manager.get_messages()
model_output = await self.get_next_action(input_messages)
self.update_step_info(model_output, step_info)
logger.info(f"🧠 All Memory: {step_info.memory}")
self._save_conversation(input_messages, model_output)
self.message_manager._remove_last_state_message() # we dont want the whole state in the chat history
self.message_manager.add_model_output(model_output)
try:
model_output = await self.get_next_action(input_messages)
if self.register_new_step_callback:
self.register_new_step_callback(state, model_output, self.n_steps)
self.update_step_info(model_output, step_info)
logger.info(f"🧠 All Memory: \n{step_info.memory}")
self._save_conversation(input_messages, model_output)
# should we remove last state message? at least, deepseek-reasoner cannot remove
if self.model_name != "deepseek-reasoner":
self.message_manager._remove_last_state_message()
except Exception as e:
# model call failed, remove last state message from history
self.message_manager._remove_last_state_message()
raise e
result: list[ActionResult] = await self.controller.multi_act(
model_output.action, self.browser_context
)
if len(result) != len(model_output.action):
# I think something changes, such information should let LLM know
for ri in range(len(result), len(model_output.action)):
result.append(ActionResult(extracted_content=None,
include_in_memory=True,
error=f"{model_output.action[ri].model_dump_json(exclude_unset=True)} is Failed to execute. \
Something new appeared after action {model_output.action[len(result) - 1].model_dump_json(exclude_unset=True)}",
is_done=False))
self._last_result = result
if len(result) > 0 and result[-1].is_done:
@@ -220,34 +258,172 @@ class CustomAgent(Agent):
self.consecutive_failures = 0
except Exception as e:
result = self._handle_step_error(e)
result = await self._handle_step_error(e)
self._last_result = result
finally:
actions = [a.model_dump(exclude_unset=True) for a in model_output.action] if model_output else []
self.telemetry.capture(
AgentStepTelemetryEvent(
agent_id=self.agent_id,
step=self.n_steps,
actions=actions,
consecutive_failures=self.consecutive_failures,
step_error=[r.error for r in result if r.error] if result else ['No result'],
)
)
if not result:
return
for r in result:
if r.error:
self.telemetry.capture(
AgentStepErrorTelemetryEvent(
agent_id=self.agent_id,
error=r.error,
)
)
if state:
self._make_history_item(model_output, state, result)
async def run(self, max_steps: int = 100) -> AgentHistoryList:
"""Execute the task with maximum number of steps"""
try:
self._log_agent_run()
# Execute initial actions if provided
if self.initial_actions:
result = await self.controller.multi_act(self.initial_actions, self.browser_context, check_for_new_elements=False)
self._last_result = result
step_info = CustomAgentStepInfo(
task=self.task,
add_infos=self.add_infos,
step_number=1,
max_steps=max_steps,
memory="",
task_progress="",
future_plans=""
)
for step in range(max_steps):
# 1) Check if stop requested
if self.agent_state and self.agent_state.is_stop_requested():
logger.info("🛑 Stop requested by user")
self._create_stop_history_item()
break
# 2) Store last valid state before step
if self.browser_context and self.agent_state:
state = await self.browser_context.get_state(use_vision=self.use_vision)
self.agent_state.set_last_valid_state(state)
if self._too_many_failures():
break
# 3) Do the step
await self.step(step_info)
if self.history.is_done():
if (
self.validate_output and step < max_steps - 1
): # if last step, we dont need to validate
if not await self._validate_output():
continue
logger.info("✅ Task completed successfully")
break
else:
logger.info("❌ Failed to complete task in maximum steps")
return self.history
finally:
self.telemetry.capture(
AgentEndTelemetryEvent(
agent_id=self.agent_id,
success=self.history.is_done(),
steps=self.n_steps,
max_steps_reached=self.n_steps >= max_steps,
errors=self.history.errors(),
)
)
if not self.injected_browser_context:
await self.browser_context.close()
if not self.injected_browser and self.browser:
await self.browser.close()
if self.generate_gif:
output_path: str = 'agent_history.gif'
if isinstance(self.generate_gif, str):
output_path = self.generate_gif
self.create_history_gif(output_path=output_path)
def _create_stop_history_item(self):
"""Create a history item for when the agent is stopped."""
try:
# Attempt to retrieve the last valid state from agent_state
state = None
if self.agent_state:
last_state = self.agent_state.get_last_valid_state()
if last_state:
# Convert to BrowserStateHistory
state = BrowserStateHistory(
url=getattr(last_state, 'url', ""),
title=getattr(last_state, 'title', ""),
tabs=getattr(last_state, 'tabs', []),
interacted_element=[None],
screenshot=getattr(last_state, 'screenshot', None)
)
else:
state = self._create_empty_state()
else:
state = self._create_empty_state()
# Create a final item in the agent history indicating done
stop_history = AgentHistory(
model_output=None,
state=state,
result=[ActionResult(extracted_content=None, error=None, is_done=True)]
)
self.history.history.append(stop_history)
except Exception as e:
logger.error(f"Error creating stop history item: {e}")
# Create empty state as fallback
state = self._create_empty_state()
stop_history = AgentHistory(
model_output=None,
state=state,
result=[ActionResult(extracted_content=None, error=None, is_done=True)]
)
self.history.history.append(stop_history)
def _convert_to_browser_state_history(self, browser_state):
return BrowserStateHistory(
url=getattr(browser_state, 'url', ""),
title=getattr(browser_state, 'title', ""),
tabs=getattr(browser_state, 'tabs', []),
interacted_element=[None],
screenshot=getattr(browser_state, 'screenshot', None)
)
def _create_empty_state(self):
return BrowserStateHistory(
url="",
title="",
tabs=[],
interacted_element=[None],
screenshot=None
)
def create_history_gif(
self,
output_path: str = 'agent_history.gif',
duration: int = 3000,
show_goals: bool = True,
show_task: bool = True,
show_logo: bool = False,
font_size: int = 40,
title_font_size: int = 56,
goal_font_size: int = 44,
margin: int = 40,
line_spacing: float = 1.5,
self,
output_path: str = 'agent_history.gif',
duration: int = 3000,
show_goals: bool = True,
show_task: bool = True,
show_logo: bool = False,
font_size: int = 40,
title_font_size: int = 56,
goal_font_size: int = 44,
margin: int = 40,
line_spacing: float = 1.5,
) -> None:
"""Create a GIF from the agent's history with overlaid task and goal text."""
if not self.history.history:
@@ -268,10 +444,9 @@ class CustomAgent(Agent):
for font_name in font_options:
try:
import platform
if platform.system() == "Windows":
if platform.system() == 'Windows':
# Need to specify the abs font path on Windows
font_name = os.path.join(os.getenv("WIN_FONT_DIR", "C:\\Windows\\Fonts"), font_name + ".ttf")
font_name = os.path.join(os.getenv('WIN_FONT_DIR', 'C:\\Windows\\Fonts'), font_name + '.ttf')
regular_font = ImageFont.truetype(font_name, font_size)
title_font = ImageFont.truetype(font_name, title_font_size)
goal_font = ImageFont.truetype(font_name, goal_font_size)
@@ -348,133 +523,4 @@ class CustomAgent(Agent):
)
logger.info(f'Created GIF at {output_path}')
else:
logger.warning('No images found in history to create GIF')
async def run(self, max_steps: int = 100) -> AgentHistoryList:
"""Execute the task with maximum number of steps"""
try:
logger.info(f"🚀 Starting task: {self.task}")
self.telemetry.capture(
AgentRunTelemetryEvent(
agent_id=self.agent_id,
task=self.task,
)
)
step_info = CustomAgentStepInfo(
task=self.task,
add_infos=self.add_infos,
step_number=1,
max_steps=max_steps,
memory="",
task_progress="",
)
for step in range(max_steps):
# 1) Check if stop requested
if self.agent_state and self.agent_state.is_stop_requested():
logger.info("🛑 Stop requested by user")
self._create_stop_history_item()
break
# 2) Store last valid state before step
if self.browser_context and self.agent_state:
state = await self.browser_context.get_state(use_vision=self.use_vision)
self.agent_state.set_last_valid_state(state)
if self._too_many_failures():
break
# 3) Do the step
await self.step(step_info)
if self.history.is_done():
if (
self.validate_output and step < max_steps - 1
): # if last step, we dont need to validate
if not await self._validate_output():
continue
logger.info("✅ Task completed successfully")
break
else:
logger.info("❌ Failed to complete task in maximum steps")
return self.history
finally:
self.telemetry.capture(
AgentEndTelemetryEvent(
agent_id=self.agent_id,
task=self.task,
success=self.history.is_done(),
steps=len(self.history.history),
)
)
if not self.injected_browser_context:
await self.browser_context.close()
if not self.injected_browser and self.browser:
await self.browser.close()
if self.generate_gif:
self.create_history_gif()
def _create_stop_history_item(self):
"""Create a history item for when the agent is stopped."""
try:
# Attempt to retrieve the last valid state from agent_state
state = None
if self.agent_state:
last_state = self.agent_state.get_last_valid_state()
if last_state:
# Convert to BrowserStateHistory
state = BrowserStateHistory(
url=getattr(last_state, 'url', ""),
title=getattr(last_state, 'title', ""),
tabs=getattr(last_state, 'tabs', []),
interacted_element=[None],
screenshot=getattr(last_state, 'screenshot', None)
)
else:
state = self._create_empty_state()
else:
state = self._create_empty_state()
# Create a final item in the agent history indicating done
stop_history = AgentHistory(
model_output=None,
state=state,
result=[ActionResult(extracted_content=None, error=None, is_done=True)]
)
self.history.history.append(stop_history)
except Exception as e:
logger.error(f"Error creating stop history item: {e}")
# Create empty state as fallback
state = self._create_empty_state()
stop_history = AgentHistory(
model_output=None,
state=state,
result=[ActionResult(extracted_content=None, error=None, is_done=True)]
)
self.history.history.append(stop_history)
def _convert_to_browser_state_history(self, browser_state):
return BrowserStateHistory(
url=getattr(browser_state, 'url', ""),
title=getattr(browser_state, 'title', ""),
tabs=getattr(browser_state, 'tabs', []),
interacted_element=[None],
screenshot=getattr(browser_state, 'screenshot', None)
)
def _create_empty_state(self):
return BrowserStateHistory(
url="",
title="",
tabs=[],
interacted_element=[None],
screenshot=None
)
logger.warning('No images found in history to create GIF')

101
src/agent/custom_massage_manager.py
View File

@@ -1,9 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: custom_massage_manager.py
from __future__ import annotations
import logging
@@ -15,11 +9,16 @@ from browser_use.agent.prompts import SystemPrompt
from browser_use.agent.views import ActionResult, AgentStepInfo
from browser_use.browser.views import BrowserState
from langchain_core.language_models import BaseChatModel
from langchain_anthropic import ChatAnthropic
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import (
HumanMessage,
AIMessage
AIMessage,
BaseMessage,
HumanMessage,
ToolMessage
)
from langchain_openai import ChatOpenAI
from ..utils.llm import DeepSeekR1ChatOpenAI
from .custom_prompts import CustomAgentMessagePrompt
logger = logging.getLogger(__name__)
@@ -33,12 +32,13 @@ class CustomMassageManager(MessageManager):
action_descriptions: str,
system_prompt_class: Type[SystemPrompt],
max_input_tokens: int = 128000,
estimated_tokens_per_character: int = 3,
estimated_characters_per_token: int = 3,
image_tokens: int = 800,
include_attributes: list[str] = [],
max_error_length: int = 400,
max_actions_per_step: int = 10,
tool_call_in_content: bool = False,
message_context: Optional[str] = None,
use_deepseek_r1: bool = False
):
super().__init__(
llm=llm,
@@ -46,48 +46,32 @@ class CustomMassageManager(MessageManager):
action_descriptions=action_descriptions,
system_prompt_class=system_prompt_class,
max_input_tokens=max_input_tokens,
estimated_tokens_per_character=estimated_tokens_per_character,
estimated_characters_per_token=estimated_characters_per_token,
image_tokens=image_tokens,
include_attributes=include_attributes,
max_error_length=max_error_length,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content,
message_context=message_context
)
self.tool_id = 1
self.use_deepseek_r1 = use_deepseek_r1
# Custom: Move Task info to state_message
self.history = MessageHistory()
self._add_message_with_tokens(self.system_prompt)
tool_calls = [
{
'name': 'CustomAgentOutput',
'args': {
'current_state': {
'prev_action_evaluation': 'Unknown - No previous actions to evaluate.',
'important_contents': '',
'completed_contents': '',
'thought': 'Now Google is open. Need to type OpenAI to search.',
'summary': 'Type OpenAI to search.',
},
'action': [],
},
'id': '',
'type': 'tool_call',
}
]
if self.tool_call_in_content:
# openai throws error if tool_calls are not responded -> move to content
example_tool_call = AIMessage(
content=f'{tool_calls}',
tool_calls=[],
)
else:
example_tool_call = AIMessage(
content=f'',
tool_calls=tool_calls,
)
self._add_message_with_tokens(example_tool_call)
if self.message_context:
context_message = HumanMessage(content=self.message_context)
self._add_message_with_tokens(context_message)
def cut_messages(self):
"""Get current message list, potentially trimmed to max tokens"""
diff = self.history.total_tokens - self.max_input_tokens
min_message_len = 2 if self.message_context is not None else 1
while diff > 0 and len(self.history.messages) > min_message_len:
self.history.remove_message(min_message_len) # alway remove the oldest message
diff = self.history.total_tokens - self.max_input_tokens
def add_state_message(
self,
state: BrowserState,
@@ -95,21 +79,6 @@ class CustomMassageManager(MessageManager):
step_info: Optional[AgentStepInfo] = None,
) -> None:
"""Add browser state as human message"""
# if keep in memory, add to directly to history and add state without result
if result:
for r in result:
if r.include_in_memory:
if r.extracted_content:
msg = HumanMessage(content=str(r.extracted_content))
self._add_message_with_tokens(msg)
if r.error:
msg = HumanMessage(
content=str(r.error)[-self.max_error_length:]
)
self._add_message_with_tokens(msg)
result = None # if result in history, we dont want to add it again
# otherwise add state message and result to next message (which will not stay in memory)
state_message = CustomAgentMessagePrompt(
state,
@@ -119,3 +88,17 @@ class CustomMassageManager(MessageManager):
step_info=step_info,
).get_user_message()
self._add_message_with_tokens(state_message)
def _count_text_tokens(self, text: str) -> int:
if isinstance(self.llm, (ChatOpenAI, ChatAnthropic, DeepSeekR1ChatOpenAI)):
try:
tokens = self.llm.get_num_tokens(text)
except Exception:
tokens = (
len(text) // self.estimated_characters_per_token
) # Rough estimate if no tokenizer available
else:
tokens = (
len(text) // self.estimated_characters_per_token
) # Rough estimate if no tokenizer available
return tokens

118
src/agent/custom_prompts.py
View File

@@ -1,12 +1,7 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: custom_prompts.py
import pdb
from typing import List, Optional
from browser_use.agent.prompts import SystemPrompt
from browser_use.agent.prompts import SystemPrompt, AgentMessagePrompt
from browser_use.agent.views import ActionResult
from browser_use.browser.views import BrowserState
from langchain_core.messages import HumanMessage, SystemMessage
@@ -24,18 +19,14 @@ class CustomSystemPrompt(SystemPrompt):
{
"current_state": {
"prev_action_evaluation": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Ignore the action result. The website is the ground truth. Also mention if something unexpected happened like new suggestions in an input field. Shortly state why/why not. Note that the result you output must be consistent with the reasoning you output afterwards. If you consider it to be 'Failed,' you should reflect on this during your thought.",
"important_contents": "Output important contents closely related to user\'s instruction or task on the current page. If there is, please output the contents. If not, please output empty string ''.",
"completed_contents": "Update the input Task Progress. Completed contents is a general summary of the current contents that have been completed. Just summarize the contents that have been actually completed based on the current page and the history operations. Please list each completed item individually, such as: 1. Input username. 2. Input Password. 3. Click confirm button",
"thought": "Think about the requirements that have been completed in previous operations and the requirements that need to be completed in the next one operation. If the output of prev_action_evaluation is 'Failed', please reflect and output your reflection here. If you think you have entered the wrong page, consider to go back to the previous page in next action.",
"important_contents": "Output important contents closely related to user\'s instruction on the current page. If there is, please output the contents. If not, please output empty string ''.",
"task_progress": "Task Progress is a general summary of the current contents that have been completed. Just summarize the contents that have been actually completed based on the content at current step and the history operations. Please list each completed item individually, such as: 1. Input username. 2. Input Password. 3. Click confirm button. Please return string type not a list.",
"future_plans": "Based on the user's request and the current state, outline the remaining steps needed to complete the task. This should be a concise list of actions yet to be performed, such as: 1. Select a date. 2. Choose a specific time slot. 3. Confirm booking. Please return string type not a list.",
"thought": "Think about the requirements that have been completed in previous operations and the requirements that need to be completed in the next one operation. If your output of prev_action_evaluation is 'Failed', please reflect and output your reflection here.",
"summary": "Please generate a brief natural language description for the operation in next actions based on your Thought."
},
"action": [
{
"action_name": {
// action-specific parameters
}
},
// ... more actions in sequence
* actions in sequences, please refer to **Common action sequences**. Each output action MUST be formated as: \{action_name\: action_params\}*
]
}
@@ -48,7 +39,6 @@ class CustomSystemPrompt(SystemPrompt):
{"click_element": {"index": 3}}
]
- Navigation and extraction: [
{"open_new_tab": {}},
{"go_to_url": {"url": "https://example.com"}},
{"extract_page_content": {}}
]
@@ -70,6 +60,7 @@ class CustomSystemPrompt(SystemPrompt):
- Don't hallucinate actions.
- If the task requires specific information - make sure to include everything in the done function. This is what the user will see.
- If you are running out of steps (current step), think about speeding it up, and ALWAYS use the done action as the last action.
- Note that you must verify if you've truly fulfilled the user's request by examining the actual page content, not just by looking at the actions you output but also whether the action is executed successfully. Pay particular attention when errors occur during action execution.
6. VISUAL CONTEXT:
- When an image is provided, use it to understand the page layout
@@ -100,10 +91,9 @@ class CustomSystemPrompt(SystemPrompt):
1. Task: The user\'s instructions you need to complete.
2. Hints(Optional): Some hints to help you complete the user\'s instructions.
3. Memory: Important contents are recorded during historical operations for use in subsequent operations.
4. Task Progress: Up to the current page, the content you have completed can be understood as the progress of the task.
5. Current URL: The webpage you're currently on
6. Available Tabs: List of open browser tabs
7. Interactive Elements: List in the format:
4. Current URL: The webpage you're currently on
5. Available Tabs: List of open browser tabs
6. Interactive Elements: List in the format:
index[:]<element_type>element_text</element_type>
- index: Numeric identifier for interaction
- element_type: HTML element type (button, input, etc.)
@@ -131,7 +121,7 @@ class CustomSystemPrompt(SystemPrompt):
AGENT_PROMPT = f"""You are a precise browser automation agent that interacts with websites through structured commands. Your role is to:
1. Analyze the provided webpage elements and structure
2. Plan a sequence of actions to accomplish the given task
3. Respond with valid JSON containing your action sequence and state assessment
3. Your final result MUST be a valid JSON as the **RESPONSE FORMAT** described, containing your action sequence and state assessment, No need extra content to expalin.
Current date and time: {time_str}
@@ -146,7 +136,7 @@ class CustomSystemPrompt(SystemPrompt):
return SystemMessage(content=AGENT_PROMPT)
class CustomAgentMessagePrompt:
class CustomAgentMessagePrompt(AgentMessagePrompt):
def __init__(
self,
state: BrowserState,
@@ -155,38 +145,66 @@ class CustomAgentMessagePrompt:
max_error_length: int = 400,
step_info: Optional[CustomAgentStepInfo] = None,
):
self.state = state
self.result = result
self.max_error_length = max_error_length
self.include_attributes = include_attributes
self.step_info = step_info
super(CustomAgentMessagePrompt, self).__init__(state=state,
result=result,
include_attributes=include_attributes,
max_error_length=max_error_length,
step_info=step_info
)
def get_user_message(self) -> HumanMessage:
if self.step_info:
step_info_description = f'Current step: {self.step_info.step_number + 1}/{self.step_info.max_steps}'
else:
step_info_description = ''
elements_text = self.state.element_tree.clickable_elements_to_string(include_attributes=self.include_attributes)
has_content_above = (self.state.pixels_above or 0) > 0
has_content_below = (self.state.pixels_below or 0) > 0
if elements_text != '':
if has_content_above:
elements_text = (
f'... {self.state.pixels_above} pixels above - scroll or extract content to see more ...\n{elements_text}'
)
else:
elements_text = f'[Start of page]\n{elements_text}'
if has_content_below:
elements_text = (
f'{elements_text}\n... {self.state.pixels_below} pixels below - scroll or extract content to see more ...'
)
else:
elements_text = f'{elements_text}\n[End of page]'
else:
elements_text = 'empty page'
state_description = f"""
1. Task: {self.step_info.task}
2. Hints(Optional):
{self.step_info.add_infos}
3. Memory:
{self.step_info.memory}
4. Task Progress:
{self.step_info.task_progress}
5. Current url: {self.state.url}
6. Available tabs:
{self.state.tabs}
7. Interactive elements:
{self.state.element_tree.clickable_elements_to_string(include_attributes=self.include_attributes)}
"""
{step_info_description}
1. Task: {self.step_info.task}
2. Hints(Optional):
{self.step_info.add_infos}
3. Memory:
{self.step_info.memory}
4. Current url: {self.state.url}
5. Available tabs:
{self.state.tabs}
6. Interactive elements:
{elements_text}
"""
if self.result:
for i, result in enumerate(self.result):
if result.extracted_content:
state_description += f"\nResult of action {i + 1}/{len(self.result)}: {result.extracted_content}"
if result.error:
# only use last 300 characters of error
error = result.error[-self.max_error_length:]
state_description += (
f"\nError of action {i + 1}/{len(self.result)}: ...{error}"
)
if result.include_in_memory:
if result.extracted_content:
state_description += f"\nResult of previous action {i + 1}/{len(self.result)}: {result.extracted_content}"
if result.error:
# only use last 300 characters of error
error = result.error[-self.max_error_length:]
state_description += (
f"\nError of previous action {i + 1}/{len(self.result)}: ...{error}"
)
if self.state.screenshot:
# Format message for vision model
@@ -202,4 +220,4 @@ class CustomAgentMessagePrompt:
]
)
return HumanMessage(content=state_description)
return HumanMessage(content=state_description)

12
src/agent/custom_views.py
View File

@@ -1,9 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: custom_views.py
from dataclasses import dataclass
from typing import Type
@@ -20,6 +14,7 @@ class CustomAgentStepInfo:
add_infos: str
memory: str
task_progress: str
future_plans: str
class CustomAgentBrain(BaseModel):
@@ -27,7 +22,8 @@ class CustomAgentBrain(BaseModel):
prev_action_evaluation: str
important_contents: str
completed_contents: str
task_progress: str
future_plans: str
thought: str
summary: str
@@ -49,7 +45,7 @@ class CustomAgentOutput(AgentOutput):
) -> Type["CustomAgentOutput"]:
"""Extend actions with custom actions"""
return create_model(
"AgentOutput",
"CustomAgentOutput",
__base__=CustomAgentOutput,
action=(
list[custom_actions],

6
src/browser/__init__.py
View File

@@ -1,6 +0,0 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: __init__.py.py

30
src/browser/config.py
View File

@@ -1,30 +0,0 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/6
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: config.py
import os
from dataclasses import dataclass
from typing import Optional
@dataclass
class BrowserPersistenceConfig:
"""Configuration for browser persistence"""
persistent_session: bool = False
user_data_dir: Optional[str] = None
debugging_port: Optional[int] = None
debugging_host: Optional[str] = None
@classmethod
def from_env(cls) -> "BrowserPersistenceConfig":
"""Create config from environment variables"""
return cls(
persistent_session=os.getenv("CHROME_PERSISTENT_SESSION", "").lower()
== "true",
user_data_dir=os.getenv("CHROME_USER_DATA"),
debugging_port=int(os.getenv("CHROME_DEBUGGING_PORT", "9222")),
debugging_host=os.getenv("CHROME_DEBUGGING_HOST", "localhost"),
)

138
src/browser/custom_browser.py
View File

@@ -1,26 +1,19 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: browser.py
import asyncio
import pdb
from playwright.async_api import Browser as PlaywrightBrowser
from playwright.async_api import (
BrowserContext as PlaywrightBrowserContext,
BrowserContext as PlaywrightBrowserContext,
)
from playwright.async_api import (
Playwright,
async_playwright,
Playwright,
async_playwright,
)
from browser_use.browser.browser import Browser
from browser_use.browser.context import BrowserContext, BrowserContextConfig
from playwright.async_api import BrowserContext as PlaywrightBrowserContext
import logging
from .config import BrowserPersistenceConfig
from .custom_context import CustomBrowserContext
logger = logging.getLogger(__name__)
@@ -32,96 +25,57 @@ class CustomBrowser(Browser):
config: BrowserContextConfig = BrowserContextConfig()
) -> CustomBrowserContext:
return CustomBrowserContext(config=config, browser=self)
async def _setup_browser(self, playwright: Playwright) -> PlaywrightBrowser:
async def _setup_browser_with_instance(self, playwright: Playwright) -> PlaywrightBrowser:
"""Sets up and returns a Playwright Browser instance with anti-detection measures."""
if self.config.wss_url:
browser = await playwright.chromium.connect(self.config.wss_url)
return browser
elif self.config.chrome_instance_path:
import subprocess
if not self.config.chrome_instance_path:
raise ValueError('Chrome instance path is required')
import subprocess
import requests
import requests
try:
# Check if browser is already running
response = requests.get('http://localhost:9222/json/version', timeout=2)
if response.status_code == 200:
logger.info('Reusing existing Chrome instance')
browser = await playwright.chromium.connect_over_cdp(
endpoint_url='http://localhost:9222',
timeout=20000, # 20 second timeout for connection
)
return browser
except requests.ConnectionError:
logger.debug('No existing Chrome instance found, starting a new one')
# Start a new Chrome instance
subprocess.Popen(
[
self.config.chrome_instance_path,
'--remote-debugging-port=9222',
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
# Attempt to connect again after starting a new instance
for _ in range(10):
try:
response = requests.get('http://localhost:9222/json/version', timeout=2)
if response.status_code == 200:
break
except requests.ConnectionError:
pass
await asyncio.sleep(1)
try:
try:
# Check if browser is already running
response = requests.get('http://localhost:9222/json/version', timeout=2)
if response.status_code == 200:
logger.info('Reusing existing Chrome instance')
browser = await playwright.chromium.connect_over_cdp(
endpoint_url='http://localhost:9222',
timeout=20000, # 20 second timeout for connection
)
return browser
except Exception as e:
logger.error(f'Failed to start a new Chrome instance.: {str(e)}')
raise RuntimeError(
' To start chrome in Debug mode, you need to close all existing Chrome instances and try again otherwise we can not connect to the instance.'
)
except requests.ConnectionError:
logger.debug('No existing Chrome instance found, starting a new one')
else:
# Start a new Chrome instance
subprocess.Popen(
[
self.config.chrome_instance_path,
'--remote-debugging-port=9222',
] + self.config.extra_chromium_args,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
# try to connect first in case the browser have not started
for _ in range(10):
try:
disable_security_args = []
if self.config.disable_security:
disable_security_args = [
'--disable-web-security',
'--disable-site-isolation-trials',
'--disable-features=IsolateOrigins,site-per-process',
]
response = requests.get('http://localhost:9222/json/version', timeout=2)
if response.status_code == 200:
break
except requests.ConnectionError:
pass
await asyncio.sleep(1)
browser = await playwright.chromium.launch(
headless=self.config.headless,
args=[
'--no-sandbox',
'--disable-blink-features=AutomationControlled',
'--disable-infobars',
'--disable-background-timer-throttling',
'--disable-popup-blocking',
'--disable-backgrounding-occluded-windows',
'--disable-renderer-backgrounding',
'--disable-window-activation',
'--disable-focus-on-load',
'--no-first-run',
'--no-default-browser-check',
'--no-startup-window',
'--window-position=0,0',
# '--window-size=1280,1000',
]
+ disable_security_args
+ self.config.extra_chromium_args,
proxy=self.config.proxy,
)
return browser
except Exception as e:
logger.error(f'Failed to initialize Playwright browser: {str(e)}')
raise
# Attempt to connect again after starting a new instance
try:
browser = await playwright.chromium.connect_over_cdp(
endpoint_url='http://localhost:9222',
timeout=20000, # 20 second timeout for connection
)
return browser
except Exception as e:
logger.error(f'Failed to start a new Chrome instance.: {str(e)}')
raise RuntimeError(
' To start chrome in Debug mode, you need to close all existing Chrome instances and try again otherwise we can not connect to the instance.'
)

79
src/browser/custom_context.py
View File

@@ -1,10 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: context.py
import json
import logging
import os
@@ -14,7 +7,6 @@ from browser_use.browser.context import BrowserContext, BrowserContextConfig
from playwright.async_api import Browser as PlaywrightBrowser
from playwright.async_api import BrowserContext as PlaywrightBrowserContext
from .config import BrowserPersistenceConfig
logger = logging.getLogger(__name__)
@@ -24,73 +16,4 @@ class CustomBrowserContext(BrowserContext):
browser: "Browser",
config: BrowserContextConfig = BrowserContextConfig()
):
super(CustomBrowserContext, self).__init__(browser=browser, config=config)
async def _create_context(self, browser: PlaywrightBrowser) -> PlaywrightBrowserContext:
"""Creates a new browser context with anti-detection measures and loads cookies if available."""
# If we have a context, return it directly
# Check if we should use existing context for persistence
if self.browser.config.chrome_instance_path and len(browser.contexts) > 0:
# Connect to existing Chrome instance instead of creating new one
context = browser.contexts[0]
else:
# Original code for creating new context
context = await browser.new_context(
viewport=self.config.browser_window_size,
no_viewport=False,
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36"
),
java_script_enabled=True,
bypass_csp=self.config.disable_security,
ignore_https_errors=self.config.disable_security,
record_video_dir=self.config.save_recording_path,
record_video_size=self.config.browser_window_size,
)
if self.config.trace_path:
await context.tracing.start(screenshots=True, snapshots=True, sources=True)
# Load cookies if they exist
if self.config.cookies_file and os.path.exists(self.config.cookies_file):
with open(self.config.cookies_file, "r") as f:
cookies = json.load(f)
logger.info(
f"Loaded {len(cookies)} cookies from {self.config.cookies_file}"
)
await context.add_cookies(cookies)
# Expose anti-detection scripts
await context.add_init_script(
"""
// Webdriver property
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
});
// Languages
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en']
});
// Plugins
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5]
});
// Chrome runtime
window.chrome = { runtime: {} };
// Permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
"""
)
return context
super(CustomBrowserContext, self).__init__(browser=browser, config=config)

5
src/controller/__init__.py
View File

@@ -1,5 +0,0 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: __init__.py.py

14
src/controller/custom_controller.py
View File

@@ -1,18 +1,16 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: custom_action.py
import pyperclip
from typing import Optional, Type
from pydantic import BaseModel
from browser_use.agent.views import ActionResult
from browser_use.browser.context import BrowserContext
from browser_use.controller.service import Controller
class CustomController(Controller):
def __init__(self):
super().__init__()
def __init__(self, exclude_actions: list[str] = [],
output_model: Optional[Type[BaseModel]] = None
):
super().__init__(exclude_actions=exclude_actions, output_model=output_model)
self._register_custom_actions()
def _register_custom_actions(self):

6
src/utils/__init__.py
View File

@@ -1,6 +0,0 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: __init__.py.py

8
src/utils/default_config_settings.py
View File

@@ -11,13 +11,13 @@ def default_config():
"max_steps": 100,
"max_actions_per_step": 10,
"use_vision": True,
"tool_call_in_content": True,
"tool_calling_method": "auto",
"llm_provider": "openai",
"llm_model_name": "gpt-4o",
"llm_temperature": 1.0,
"llm_base_url": "",
"llm_api_key": "",
"use_own_browser": os.getenv("CHROME_PERSISTENT_SESSION", False),
"use_own_browser": os.getenv("CHROME_PERSISTENT_SESSION", "false").lower() == "true",
"keep_browser_open": False,
"headless": False,
"disable_security": True,
@@ -56,7 +56,7 @@ def save_current_config(*args):
"max_steps": args[1],
"max_actions_per_step": args[2],
"use_vision": args[3],
"tool_call_in_content": args[4],
"tool_calling_method": args[4],
"llm_provider": args[5],
"llm_model_name": args[6],
"llm_temperature": args[7],
@@ -86,7 +86,7 @@ def update_ui_from_config(config_file):
gr.update(value=loaded_config.get("max_steps", 100)),
gr.update(value=loaded_config.get("max_actions_per_step", 10)),
gr.update(value=loaded_config.get("use_vision", True)),
gr.update(value=loaded_config.get("tool_call_in_content", True)),
gr.update(value=loaded_config.get("tool_calling_method", True)),
gr.update(value=loaded_config.get("llm_provider", "openai")),
gr.update(value=loaded_config.get("llm_model_name", "gpt-4o")),
gr.update(value=loaded_config.get("llm_temperature", 1.0)),

136
src/utils/llm.py Normal file
View File

@@ -0,0 +1,136 @@
from openai import OpenAI
import pdb
from langchain_openai import ChatOpenAI
from langchain_core.globals import get_llm_cache
from langchain_core.language_models.base import (
BaseLanguageModel,
LangSmithParams,
LanguageModelInput,
)
from langchain_core.load import dumpd, dumps
from langchain_core.messages import (
AIMessage,
SystemMessage,
AnyMessage,
BaseMessage,
BaseMessageChunk,
HumanMessage,
convert_to_messages,
message_chunk_to_message,
)
from langchain_core.outputs import (
ChatGeneration,
ChatGenerationChunk,
ChatResult,
LLMResult,
RunInfo,
)
from langchain_ollama import ChatOllama
from langchain_core.output_parsers.base import OutputParserLike
from langchain_core.runnables import Runnable, RunnableConfig
from langchain_core.tools import BaseTool
from typing import (
TYPE_CHECKING,
Any,
Callable,
Literal,
Optional,
Union,
cast,
)
class DeepSeekR1ChatOpenAI(ChatOpenAI):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.client = OpenAI(
base_url=kwargs.get("base_url"),
api_key=kwargs.get("api_key")
)
async def ainvoke(
self,
input: LanguageModelInput,
config: Optional[RunnableConfig] = None,
*,
stop: Optional[list[str]] = None,
**kwargs: Any,
) -> AIMessage:
message_history = []
for input_ in input:
if isinstance(input_, SystemMessage):
message_history.append({"role": "system", "content": input_.content})
elif isinstance(input_, AIMessage):
message_history.append({"role": "assistant", "content": input_.content})
else:
message_history.append({"role": "user", "content": input_.content})
response = self.client.chat.completions.create(
model=self.model_name,
messages=messages
)
reasoning_content = response.choices[0].message.reasoning_content
content = response.choices[0].message.content
return AIMessage(content=content, reasoning_content=reasoning_content)
def invoke(
self,
input: LanguageModelInput,
config: Optional[RunnableConfig] = None,
*,
stop: Optional[list[str]] = None,
**kwargs: Any,
) -> AIMessage:
message_history = []
for input_ in input:
if isinstance(input_, SystemMessage):
message_history.append({"role": "system", "content": input_.content})
elif isinstance(input_, AIMessage):
message_history.append({"role": "assistant", "content": input_.content})
else:
message_history.append({"role": "user", "content": input_.content})
response = self.client.chat.completions.create(
model=self.model_name,
messages=message_history
)
reasoning_content = response.choices[0].message.reasoning_content
content = response.choices[0].message.content
return AIMessage(content=content, reasoning_content=reasoning_content)
class DeepSeekR1ChatOllama(ChatOllama):
async def ainvoke(
self,
input: LanguageModelInput,
config: Optional[RunnableConfig] = None,
*,
stop: Optional[list[str]] = None,
**kwargs: Any,
) -> AIMessage:
org_ai_message = await super().ainvoke(input=input)
org_content = org_ai_message.content
reasoning_content = org_content.split("</think>")[0].replace("<think>", "")
content = org_content.split("</think>")[1]
if "**JSON Response:**" in content:
content = content.split("**JSON Response:**")[-1]
return AIMessage(content=content, reasoning_content=reasoning_content)
def invoke(
self,
input: LanguageModelInput,
config: Optional[RunnableConfig] = None,
*,
stop: Optional[list[str]] = None,
**kwargs: Any,
) -> AIMessage:
org_ai_message = super().invoke(input=input)
org_content = org_ai_message.content
reasoning_content = org_content.split("</think>")[0].replace("<think>", "")
content = org_content.split("</think>")[1]
if "**JSON Response:**" in content:
content = content.split("**JSON Response:**")[-1]
return AIMessage(content=content, reasoning_content=reasoning_content)

57
src/utils/utils.py
View File

@@ -1,9 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: utils.py
import base64
import os
import time
@@ -17,6 +11,8 @@ from langchain_ollama import ChatOllama
from langchain_openai import AzureChatOpenAI, ChatOpenAI
import gradio as gr
from .llm import DeepSeekR1ChatOpenAI, DeepSeekR1ChatOllama
def get_llm_model(provider: str, **kwargs):
"""
获取LLM 模型
@@ -85,12 +81,20 @@ def get_llm_model(provider: str, **kwargs):
else:
api_key = kwargs.get("api_key")
return ChatOpenAI(
model=kwargs.get("model_name", "deepseek-chat"),
temperature=kwargs.get("temperature", 0.0),
base_url=base_url,
api_key=api_key,
)
if kwargs.get("model_name", "deepseek-chat") == "deepseek-reasoner":
return DeepSeekR1ChatOpenAI(
model=kwargs.get("model_name", "deepseek-reasoner"),
temperature=kwargs.get("temperature", 0.0),
base_url=base_url,
api_key=api_key,
)
else:
return ChatOpenAI(
model=kwargs.get("model_name", "deepseek-chat"),
temperature=kwargs.get("temperature", 0.0),
base_url=base_url,
api_key=api_key,
)
elif provider == "gemini":
if not kwargs.get("api_key", ""):
api_key = os.getenv("GOOGLE_API_KEY", "")
@@ -102,12 +106,25 @@ def get_llm_model(provider: str, **kwargs):
google_api_key=api_key,
)
elif provider == "ollama":
return ChatOllama(
model=kwargs.get("model_name", "qwen2.5:7b"),
temperature=kwargs.get("temperature", 0.0),
num_ctx=kwargs.get("num_ctx", 32000),
base_url=kwargs.get("base_url", "http://localhost:11434"),
)
if not kwargs.get("base_url", ""):
base_url = os.getenv("OLLAMA_ENDPOINT", "http://localhost:11434")
else:
base_url = kwargs.get("base_url")
if kwargs.get("model_name", "qwen2.5:7b").startswith("deepseek-r1"):
return DeepSeekR1ChatOllama(
model=kwargs.get("model_name", "deepseek-r1:7b"),
temperature=kwargs.get("temperature", 0.0),
num_ctx=kwargs.get("num_ctx", 32000),
base_url=kwargs.get("base_url", base_url),
)
else:
return ChatOllama(
model=kwargs.get("model_name", "qwen2.5:7b"),
temperature=kwargs.get("temperature", 0.0),
num_ctx=kwargs.get("num_ctx", 32000),
base_url=kwargs.get("base_url", base_url),
)
elif provider == "azure_openai":
if not kwargs.get("base_url", ""):
base_url = os.getenv("AZURE_OPENAI_ENDPOINT", "")
@@ -131,9 +148,9 @@ def get_llm_model(provider: str, **kwargs):
model_names = {
"anthropic": ["claude-3-5-sonnet-20240620", "claude-3-opus-20240229"],
"openai": ["gpt-4o", "gpt-4", "gpt-3.5-turbo"],
"deepseek": ["deepseek-chat"],
"deepseek": ["deepseek-chat", "deepseek-reasoner"],
"gemini": ["gemini-2.0-flash-exp", "gemini-2.0-flash-thinking-exp", "gemini-1.5-flash-latest", "gemini-1.5-flash-8b-latest", "gemini-2.0-flash-thinking-exp-1219" ],
"ollama": ["qwen2.5:7b", "llama2:7b"],
"ollama": ["qwen2.5:7b", "llama2:7b", "deepseek-r1:14b", "deepseek-r1:32b"],
"azure_openai": ["gpt-4o", "gpt-4", "gpt-3.5-turbo"],
"mistral": ["pixtral-large-latest", "mistral-large-latest", "mistral-small-latest", "ministral-8b-latest"]
}

41
tests/test_browser_use.py
View File

@@ -1,8 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @ProjectName: browser-use-webui
# @FileName: test_browser_use.py
import pdb
from dotenv import load_dotenv
@@ -45,7 +40,15 @@ async def test_browser_use_org():
window_w, window_h = 1920, 1080
use_vision = False
chrome_path = os.getenv("CHROME_PATH", None)
use_own_browser = False
if use_own_browser:
chrome_path = os.getenv("CHROME_PATH", None)
if chrome_path == "":
chrome_path = None
else:
chrome_path = None
tool_calling_method = "json_schema" # setting to json_schema when using ollma
browser = Browser(
config=BrowserConfig(
@@ -69,7 +72,8 @@ async def test_browser_use_org():
task="go to google.com and type 'OpenAI' click search and give me the first url",
llm=llm,
browser_context=browser_context,
use_vision=use_vision
use_vision=use_vision,
tool_calling_method=tool_calling_method
)
history: AgentHistoryList = await agent.run(max_steps=10)
@@ -247,22 +251,32 @@ async def test_browser_use_custom_v2():
# api_key=os.getenv("GOOGLE_API_KEY", "")
# )
# llm = utils.get_llm_model(
# provider="deepseek",
# model_name="deepseek-reasoner",
# temperature=0.8
# )
# llm = utils.get_llm_model(
# provider="deepseek",
# model_name="deepseek-chat",
# temperature=0.8
# )
llm = utils.get_llm_model(
provider="ollama", model_name="qwen2.5:7b", temperature=0.5
)
# llm = utils.get_llm_model(
# provider="ollama", model_name="qwen2.5:7b", temperature=0.5
# )
# llm = utils.get_llm_model(
# provider="ollama", model_name="deepseek-r1:14b", temperature=0.5
# )
controller = CustomController()
use_own_browser = False
disable_security = True
use_vision = False # Set to False when using DeepSeek
tool_call_in_content = True # Set to True when using Ollama
max_actions_per_step = 1
max_actions_per_step = 10
playwright = None
browser = None
browser_context = None
@@ -293,7 +307,7 @@ async def test_browser_use_custom_v2():
)
)
agent = CustomAgent(
task="go to google.com and type 'OpenAI' click search and give me the first url",
task="go to google.com and type 'Nvidia' click search and give me the first url",
add_infos="", # some hints for llm to complete the task
llm=llm,
browser=browser,
@@ -301,7 +315,6 @@ async def test_browser_use_custom_v2():
controller=controller,
system_prompt_class=CustomSystemPrompt,
use_vision=use_vision,
tool_call_in_content=tool_call_in_content,
max_actions_per_step=max_actions_per_step
)
history: AgentHistoryList = await agent.run(max_steps=10)

45
tests/test_llm_api.py
View File

@@ -1,9 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: test_llm_api.py
import os
import pdb
@@ -133,6 +127,33 @@ def test_deepseek_model():
ai_msg = llm.invoke([message])
print(ai_msg.content)
def test_deepseek_r1_model():
from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
from src.utils import utils
llm = utils.get_llm_model(
provider="deepseek",
model_name="deepseek-reasoner",
temperature=0.8,
base_url=os.getenv("DEEPSEEK_ENDPOINT", ""),
api_key=os.getenv("DEEPSEEK_API_KEY", "")
)
messages = []
sys_message = SystemMessage(
content=[{"type": "text", "text": "you are a helpful AI assistant"}]
)
messages.append(sys_message)
user_message = HumanMessage(
content=[
{"type": "text", "text": "9.11 and 9.8, which is greater?"}
]
)
messages.append(user_message)
ai_msg = llm.invoke(messages)
print(ai_msg.reasoning_content)
print(ai_msg.content)
print(llm.model_name)
pdb.set_trace()
def test_ollama_model():
from langchain_ollama import ChatOllama
@@ -140,6 +161,14 @@ def test_ollama_model():
llm = ChatOllama(model="qwen2.5:7b")
ai_msg = llm.invoke("Sing a ballad of LangChain.")
print(ai_msg.content)
def test_deepseek_r1_ollama_model():
from src.utils.llm import DeepSeekR1ChatOllama
llm = DeepSeekR1ChatOllama(model="deepseek-r1:14b")
ai_msg = llm.invoke("how many r in strawberry?")
print(ai_msg.content)
pdb.set_trace()
if __name__ == '__main__':
@@ -148,4 +177,6 @@ if __name__ == '__main__':
# test_azure_openai_model()
# test_deepseek_model()
# test_ollama_model()
test_mistral_model()
# test_deepseek_r1_model()
# test_deepseek_r1_ollama_model()
test_mistral_model()

6
tests/test_playwright.py
View File

@@ -1,9 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/2
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: test_playwright.py
import pdb
from dotenv import load_dotenv

101
webui.py
View File

@@ -1,10 +1,3 @@
# -*- coding: utf-8 -*-
# @Time : 2025/1/1
# @Author : wenshao
# @Email : wenshaoguo1026@gmail.com
# @Project : browser-use-webui
# @FileName: webui.py
import pdb
import logging
@@ -28,6 +21,7 @@ from browser_use.browser.context import (
BrowserContextConfig,
BrowserContextWindowSize,
)
from langchain_ollama import ChatOllama
from playwright.async_api import async_playwright
from src.utils.agent_state import AgentState
@@ -35,15 +29,12 @@ from src.utils import utils
from src.agent.custom_agent import CustomAgent
from src.browser.custom_browser import CustomBrowser
from src.agent.custom_prompts import CustomSystemPrompt
from src.browser.config import BrowserPersistenceConfig
from src.browser.custom_context import BrowserContextConfig, CustomBrowserContext
from src.controller.custom_controller import CustomController
from gradio.themes import Citrus, Default, Glass, Monochrome, Ocean, Origin, Soft, Base
from src.utils.default_config_settings import default_config, load_config_from_file, save_config_to_file, save_current_config, update_ui_from_config
from src.utils.utils import update_model_dropdown, get_latest_files, capture_screenshot
from dotenv import load_dotenv
load_dotenv()
# Global variables for persistence
_global_browser = None
@@ -101,7 +92,7 @@ async def run_browser_agent(
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content
tool_calling_method
):
global _global_agent_state
_global_agent_state.clear_stop() # Clear any previous stop requests
@@ -147,7 +138,7 @@ async def run_browser_agent(
max_steps=max_steps,
use_vision=use_vision,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content
tool_calling_method=tool_calling_method
)
elif agent_type == "custom":
final_result, errors, model_actions, model_thoughts, trace_file, history_file = await run_custom_agent(
@@ -166,7 +157,7 @@ async def run_browser_agent(
max_steps=max_steps,
use_vision=use_vision,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content
tool_calling_method=tool_calling_method
)
else:
raise ValueError(f"Invalid agent type: {agent_type}")
@@ -225,7 +216,7 @@ async def run_org_agent(
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content
tool_calling_method
):
try:
global _global_browser, _global_browser_context, _global_agent_state
@@ -261,7 +252,7 @@ async def run_org_agent(
),
)
)
agent = Agent(
task=task,
llm=llm,
@@ -269,7 +260,7 @@ async def run_org_agent(
browser=_global_browser,
browser_context=_global_browser_context,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content
tool_calling_method=tool_calling_method
)
history = await agent.run(max_steps=max_steps)
@@ -316,7 +307,7 @@ async def run_custom_agent(
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content
tool_calling_method
):
try:
global _global_browser, _global_browser_context, _global_agent_state
@@ -355,7 +346,7 @@ async def run_custom_agent(
),
)
)
# Create and run agent
agent = CustomAgent(
task=task,
@@ -367,8 +358,8 @@ async def run_custom_agent(
controller=controller,
system_prompt_class=CustomSystemPrompt,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content,
agent_state=_global_agent_state
agent_state=_global_agent_state,
tool_calling_method=tool_calling_method
)
history = await agent.run(max_steps=max_steps)
@@ -421,7 +412,7 @@ async def run_with_stream(
max_steps,
use_vision,
max_actions_per_step,
tool_call_in_content
tool_calling_method
):
global _global_agent_state
stream_vw = 80
@@ -449,7 +440,7 @@ async def run_with_stream(
max_steps=max_steps,
use_vision=use_vision,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content
tool_calling_method=tool_calling_method
)
# Add HTML content at the start of the result array
html_content = f"<h1 style='width:{stream_vw}vw; height:{stream_vh}vh'>Using browser...</h1>"
@@ -481,7 +472,7 @@ async def run_with_stream(
max_steps=max_steps,
use_vision=use_vision,
max_actions_per_step=max_actions_per_step,
tool_call_in_content=tool_call_in_content
tool_calling_method=tool_calling_method
)
)
@@ -638,32 +629,38 @@ def create_ui(config, theme_name="Ocean"):
value=config['agent_type'],
info="Select the type of agent to use",
)
max_steps = gr.Slider(
minimum=1,
maximum=200,
value=config['max_steps'],
step=1,
label="Max Run Steps",
info="Maximum number of steps the agent will take",
)
max_actions_per_step = gr.Slider(
minimum=1,
maximum=20,
value=config['max_actions_per_step'],
step=1,
label="Max Actions per Step",
info="Maximum number of actions the agent will take per step",
)
use_vision = gr.Checkbox(
label="Use Vision",
value=config['use_vision'],
info="Enable visual processing capabilities",
)
tool_call_in_content = gr.Checkbox(
label="Use Tool Calls in Content",
value=config['tool_call_in_content'],
info="Enable Tool Calls in content",
)
with gr.Column():
max_steps = gr.Slider(
minimum=1,
maximum=200,
value=config['max_steps'],
step=1,
label="Max Run Steps",
info="Maximum number of steps the agent will take",
)
max_actions_per_step = gr.Slider(
minimum=1,
maximum=20,
value=config['max_actions_per_step'],
step=1,
label="Max Actions per Step",
info="Maximum number of actions the agent will take per step",
)
with gr.Column():
use_vision = gr.Checkbox(
label="Use Vision",
value=config['use_vision'],
info="Enable visual processing capabilities",
)
tool_calling_method = gr.Dropdown(
label="Tool Calling Method",
value=config['tool_calling_method'],
interactive=True,
allow_custom_value=True, # Allow users to input custom model names
choices=["auto", "json_schema", "function_calling"],
info="Tool Calls Funtion Name",
visible=False
)
with gr.TabItem("🔧 LLM Configuration", id=2):
with gr.Group():
@@ -813,7 +810,7 @@ def create_ui(config, theme_name="Ocean"):
fn=update_ui_from_config,
inputs=[config_file_input],
outputs=[
agent_type, max_steps, max_actions_per_step, use_vision, tool_call_in_content,
agent_type, max_steps, max_actions_per_step, use_vision, tool_calling_method,
llm_provider, llm_model_name, llm_temperature, llm_base_url, llm_api_key,
use_own_browser, keep_browser_open, headless, disable_security, enable_recording,
window_w, window_h, save_recording_path, save_trace_path, save_agent_history_path,
@@ -824,7 +821,7 @@ def create_ui(config, theme_name="Ocean"):
save_config_button.click(
fn=save_current_config,
inputs=[
agent_type, max_steps, max_actions_per_step, use_vision, tool_call_in_content,
agent_type, max_steps, max_actions_per_step, use_vision, tool_calling_method,
llm_provider, llm_model_name, llm_temperature, llm_base_url, llm_api_key,
use_own_browser, keep_browser_open, headless, disable_security,
enable_recording, window_w, window_h, save_recording_path, save_trace_path,
@@ -876,7 +873,7 @@ def create_ui(config, theme_name="Ocean"):
agent_type, llm_provider, llm_model_name, llm_temperature, llm_base_url, llm_api_key,
use_own_browser, keep_browser_open, headless, disable_security, window_w, window_h,
save_recording_path, save_agent_history_path, save_trace_path, # Include the new path
enable_recording, task, add_infos, max_steps, use_vision, max_actions_per_step, tool_call_in_content
enable_recording, task, add_infos, max_steps, use_vision, max_actions_per_step, tool_calling_method
],
outputs=[
browser_view, # Browser view