-
-
-
+
+
| \n", - " | type | \n", - "bbox | \n", - "interactivity | \n", - "content | \n", - "ID | \n", - "
|---|---|---|---|---|---|
| 0 | \n", - "text | \n", - "[0.023970818147063255, 0.012974976561963558, 0... | \n", - "False | \n", - "AutoSave | \n", - "0 | \n", - "
| 1 | \n", - "text | \n", - "[0.0771235004067421, 0.011121409013867378, 0.1... | \n", - "False | \n", - "F 9~U | \n", - "1 | \n", - "
| 2 | \n", - "text | \n", - "[0.1500781625509262, 0.011121409013867378, 0.3... | \n", - "False | \n", - "Document 10.docx General* Last Modified: Jus... | \n", - "2 | \n", - "
| 3 | \n", - "text | \n", - "[0.4095883369445801, 0.011121409013867378, 0.4... | \n", - "False | \n", - "O Search | \n", - "3 | \n", - "
| 4 | \n", - "text | \n", - "[0.010943199507892132, 0.048192769289016724, 0... | \n", - "False | \n", - "File | \n", - "4 | \n", - "
| ... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "
| 146 | \n", - "icon | \n", - "[0.27768608927726746, 0.1485075205564499, 0.28... | \n", - "True | \n", - "Redo | \n", - "146 | \n", - "
| 147 | \n", - "icon | \n", - "[0.9438582062721252, 0.9580937027931213, 0.995... | \n", - "True | \n", - "Notifications. | \n", - "147 | \n", - "
| 148 | \n", - "icon | \n", - "[0.31950756907463074, 0.3229200839996338, 0.33... | \n", - "True | \n", - "minimizing a window. | \n", - "148 | \n", - "
| 149 | \n", - "icon | \n", - "[0.08737719058990479, 0.148496612906456, 0.095... | \n", - "True | \n", - "Redo | \n", - "149 | \n", - "
| 150 | \n", - "icon | \n", - "[0.7414734959602356, 0.000822930654976517, 0.7... | \n", - "True | \n", - "M0,0L9,0 4.5,5z | \n", - "150 | \n", - "
151 rows ร 5 columns
\n", - "| \n", + " | type | \n", + "bbox | \n", + "interactivity | \n", + "content | \n", + "ID | \n", + "
|---|---|---|---|---|---|
| 0 | \n", + "text | \n", + "[0.023970818147063255, 0.012974976561963558, 0... | \n", + "False | \n", + "AutoSave | \n", + "0 | \n", + "
| 1 | \n", + "text | \n", + "[0.0771235004067421, 0.011121409013867378, 0.1... | \n", + "False | \n", + "F 9~U | \n", + "1 | \n", + "
| 2 | \n", + "text | \n", + "[0.1500781625509262, 0.011121409013867378, 0.3... | \n", + "False | \n", + "Document 10.docx General* Last Modified: Jus... | \n", + "2 | \n", + "
| 3 | \n", + "text | \n", + "[0.4095883369445801, 0.011121409013867378, 0.4... | \n", + "False | \n", + "O Search | \n", + "3 | \n", + "
| 4 | \n", + "text | \n", + "[0.010943199507892132, 0.048192769289016724, 0... | \n", + "False | \n", + "File | \n", + "4 | \n", + "
| ... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "
| 146 | \n", + "icon | \n", + "[0.27768608927726746, 0.1485075205564499, 0.28... | \n", + "True | \n", + "Redo | \n", + "146 | \n", + "
| 147 | \n", + "icon | \n", + "[0.9438582062721252, 0.9580937027931213, 0.995... | \n", + "True | \n", + "Notifications. | \n", + "147 | \n", + "
| 148 | \n", + "icon | \n", + "[0.31950756907463074, 0.3229200839996338, 0.33... | \n", + "True | \n", + "minimizing a window. | \n", + "148 | \n", + "
| 149 | \n", + "icon | \n", + "[0.08737719058990479, 0.148496612906456, 0.095... | \n", + "True | \n", + "Redo | \n", + "149 | \n", + "
| 150 | \n", + "icon | \n", + "[0.7414734959602356, 0.000822930654976517, 0.7... | \n", + "True | \n", + "M0,0L9,0 4.5,5z | \n", + "150 | \n", + "
151 rows ร 5 columns
\n", + "
'
+
+ elif isinstance(message, BetaTextBlock) or isinstance(message, TextBlock):
+ return f"Analysis: {message.text}"
+ elif isinstance(message, BetaToolUseBlock) or isinstance(message, ToolUseBlock):
+ # return f"Tool Use: {message.name}\nInput: {message.input}"
+ return f"Next I will perform the following action: {message.input}"
+ else:
+ return message
+
+ def _truncate_string(s, max_length=500):
+ """Truncate long strings for concise printing."""
+ if isinstance(s, str) and len(s) > max_length:
+ return s[:max_length] + "..."
+ return s
+ # processing Anthropic messages
+ message = _render_message(message, hide_images)
+
+ if sender == "bot":
+ chatbot_state.append((None, message))
+ else:
+ chatbot_state.append((message, None))
+
+ # Create a concise version of the chatbot state for printing
+ concise_state = [(_truncate_string(user_msg), _truncate_string(bot_msg))
+ for user_msg, bot_msg in chatbot_state]
+ # print(f"chatbot_output_callback chatbot_state: {concise_state} (truncated)")
+
+def process_input(user_input, state):
+
+ setup_state(state)
+
+ # Append the user message to state["messages"]
+ if state["model"] == "gpt-4o + ShowUI" or state["model"] == "qwen2vl + ShowUI":
+ state["messages"].append(
+ {
+ "role": "user",
+ "content": [TextBlock(type="text", text=user_input)],
+ }
+ )
+ elif state["model"] == "claude-3-5-sonnet-20241022":
+ state["messages"].append(
+ {
+ "role": Sender.USER,
+ "content": [TextBlock(type="text", text=user_input)],
+ }
+ )
+ elif state["model"] == "omniparser + gpt-4o" or state["model"] == "omniparser + phi35v":
+ state["messages"].append(
+ {
+ "role": "user",
+ "content": [TextBlock(type="text", text=user_input)],
+ }
+ )
+
+ # Append the user's message to chatbot_messages with None for the assistant's reply
+ state['chatbot_messages'].append((user_input, None))
+ yield state['chatbot_messages'] # Yield to update the chatbot UI with the user's message
+
+ # Run sampling_loop_sync with the chatbot_output_callback
+ for loop_msg in sampling_loop_sync(
+ system_prompt_suffix=state["custom_system_prompt"],
+ model=state["model"],
+ provider=state["provider"],
+ messages=state["messages"],
+ output_callback=partial(chatbot_output_callback, chatbot_state=state['chatbot_messages'], hide_images=state["hide_images"]),
+ tool_output_callback=partial(_tool_output_callback, tool_state=state["tools"]),
+ api_response_callback=partial(_api_response_callback, response_state=state["responses"]),
+ api_key=state["api_key"],
+ only_n_most_recent_images=state["only_n_most_recent_images"],
+ selected_screen=state['selected_screen']
+ ):
+ if loop_msg is None:
+ yield state['chatbot_messages']
+ print("End of task. Close the loop.")
+ break
+
+ yield state['chatbot_messages'] # Yield the updated chatbot_messages to update the chatbot UI
+
+
+# with gr.Blocks(theme=gr.themes.Default()) as demo:
+with gr.Blocks(theme='YTheme/Minecraft') as demo:
+ state = gr.State({}) # Use Gradio's state management
+
+ setup_state(state.value) # Initialize the state
+
+ # Retrieve screen details
+ gr.Markdown("# OmniParser + โ๏ธ Demo")
+
+ if not os.getenv("HIDE_WARNING", False):
+ gr.Markdown(INTRO_TEXT)
+
+ with gr.Accordion("Settings", open=True):
+ with gr.Row():
+ with gr.Column():
+ model = gr.Dropdown(
+ label="Model",
+ choices=["omniparser + gpt-4o", "omniparser + phi35v", "claude-3-5-sonnet-20241022"],
+ value="omniparser + gpt-4o", # Set to one of the choices
+ interactive=True,
+ )
+ with gr.Column():
+ provider = gr.Dropdown(
+ label="API Provider",
+ choices=[option.value for option in APIProvider],
+ value="openai",
+ interactive=False,
+ )
+ with gr.Column():
+ api_key = gr.Textbox(
+ label="API Key",
+ type="password",
+ value=state.value.get("api_key", ""),
+ placeholder="Paste your API key here",
+ interactive=True,
+ )
+ with gr.Column():
+ custom_prompt = gr.Textbox(
+ label="System Prompt Suffix",
+ value="",
+ interactive=True,
+ )
+ with gr.Column():
+ screen_options, primary_index = get_screen_details()
+ SCREEN_NAMES = screen_options
+ SELECTED_SCREEN_INDEX = primary_index
+ screen_selector = gr.Dropdown(
+ label="Select Screen",
+ choices=screen_options,
+ value=screen_options[primary_index] if screen_options else None,
+ interactive=True,
+ )
+ with gr.Column():
+ only_n_images = gr.Slider(
+ label="N most recent screenshots",
+ minimum=0,
+ maximum=10,
+ step=1,
+ value=2,
+ interactive=True,
+ )
+ # hide_images = gr.Checkbox(label="Hide screenshots", value=False)
+
+ # Define the merged dictionary with task mappings
+ # merged_dict = json.load(open("examples/ootb_examples.json", "r"))
+ merged_dict = {}
+
+ def update_only_n_images(only_n_images_value, state):
+ state["only_n_most_recent_images"] = only_n_images_value
+
+ # Callback to update the second dropdown based on the first selection
+ def update_second_menu(selected_category):
+ return gr.update(choices=list(merged_dict.get(selected_category, {}).keys()))
+
+ # Callback to update the third dropdown based on the second selection
+ def update_third_menu(selected_category, selected_option):
+ return gr.update(choices=list(merged_dict.get(selected_category, {}).get(selected_option, {}).keys()))
+
+ # Callback to update the textbox based on the third selection
+ def update_textbox(selected_category, selected_option, selected_task):
+ task_data = merged_dict.get(selected_category, {}).get(selected_option, {}).get(selected_task, {})
+ prompt = task_data.get("prompt", "")
+ preview_image = task_data.get("initial_state", "")
+ task_hint = "Task Hint: " + task_data.get("hint", "")
+ return prompt, preview_image, task_hint
+
+ # Function to update the global variable when the dropdown changes
+ def update_selected_screen(selected_screen_name, state):
+ global SCREEN_NAMES
+ global SELECTED_SCREEN_INDEX
+ SELECTED_SCREEN_INDEX = SCREEN_NAMES.index(selected_screen_name)
+ print(f"Selected screen updated to: {SELECTED_SCREEN_INDEX}")
+ state['selected_screen'] = SELECTED_SCREEN_INDEX
+
+ def update_model(model_selection, state):
+ state["model"] = model_selection
+ print(f"Model updated to: {state['model']}")
+
+ if model_selection == "claude-3-5-sonnet-20241022":
+ # Provider can be any of the current choices except 'openai'
+ provider_choices = [option.value for option in APIProvider if option.value != "openai"]
+ provider_value = "anthropic" # Set default to 'anthropic'
+ provider_interactive = True
+ api_key_placeholder = "claude API key"
+ elif model_selection == "omniparser + gpt-4o" or model_selection == "omniparser + phi35v":
+ # Provider can be any of the current choices except 'openai'
+ provider_choices = ["openai"]
+ provider_value = "openai"
+ provider_interactive = False
+ api_key_placeholder = "openai API key"
+ else:
+ # Default case
+ provider_choices = [option.value for option in APIProvider]
+ provider_value = state.get("provider", provider_choices[0])
+ provider_interactive = True
+ api_key_placeholder = ""
+
+ # Update the provider in state
+ state["provider"] = provider_value
+
+ # Update api_key in state based on the provider
+ if provider_value == "openai":
+ state["api_key"] = state.get("openai_api_key", "")
+ elif provider_value == "anthropic":
+ state["api_key"] = state.get("anthropic_api_key", "")
+ elif provider_value == "qwen":
+ state["api_key"] = state.get("qwen_api_key", "")
+ else:
+ state["api_key"] = ""
+
+ # Use gr.update() instead of gr.Dropdown.update()
+ provider_update = gr.update(
+ choices=provider_choices,
+ value=provider_value,
+ interactive=provider_interactive
+ )
+
+ # Update the API Key textbox
+ api_key_update = gr.update(
+ placeholder=api_key_placeholder,
+ value=state["api_key"]
+ )
+
+ return provider_update, api_key_update
+
+ def update_api_key_placeholder(provider_value, model_selection):
+ if model_selection == "claude-3-5-sonnet-20241022":
+ if provider_value == "anthropic":
+ return gr.update(placeholder="anthropic API key")
+ elif provider_value == "bedrock":
+ return gr.update(placeholder="bedrock API key")
+ elif provider_value == "vertex":
+ return gr.update(placeholder="vertex API key")
+ else:
+ return gr.update(placeholder="")
+ elif model_selection == "gpt-4o + ShowUI":
+ return gr.update(placeholder="openai API key")
+ else:
+ return gr.update(placeholder="")
+
+ def update_system_prompt_suffix(system_prompt_suffix, state):
+ state["custom_system_prompt"] = system_prompt_suffix
+
+
+ api_key.change(fn=lambda key: save_to_storage(API_KEY_FILE, key), inputs=api_key)
+
+ with gr.Row():
+ # submit_button = gr.Button("Submit") # Add submit button
+ with gr.Column(scale=8):
+ chat_input = gr.Textbox(show_label=False, placeholder="Type a message to send to Computer Use OOTB...", container=False)
+ with gr.Column(scale=1, min_width=50):
+ submit_button = gr.Button(value="Send", variant="primary")
+
+ chatbot = gr.Chatbot(label="Chatbot History", autoscroll=True, height=580)
+
+ model.change(fn=update_model, inputs=[model, state], outputs=[provider, api_key])
+ provider.change(fn=update_api_key_placeholder, inputs=[provider, model], outputs=api_key)
+ screen_selector.change(fn=update_selected_screen, inputs=[screen_selector, state], outputs=None)
+ only_n_images.change(fn=update_only_n_images, inputs=[only_n_images, state], outputs=None)
+
+
+ # chat_input.submit(process_input, [chat_input, state], chatbot)
+ submit_button.click(process_input, [chat_input, state], chatbot)
+
+demo.launch(share=True, server_port=7861, server_name='0.0.0.0') # TODO: allowed_paths
diff --git a/demo/executor/anthropic_executor.py b/demo/executor/anthropic_executor.py
new file mode 100644
index 0000000..df61810
--- /dev/null
+++ b/demo/executor/anthropic_executor.py
@@ -0,0 +1,136 @@
+import asyncio
+from typing import Any, Dict, cast
+from collections.abc import Callable
+from anthropic.types.beta import (
+ BetaContentBlock,
+ BetaContentBlockParam,
+ BetaImageBlockParam,
+ BetaMessage,
+ BetaMessageParam,
+ BetaTextBlockParam,
+ BetaToolResultBlockParam,
+)
+from anthropic.types import TextBlock
+from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock
+from tools import BashTool, ComputerTool, EditTool, ToolCollection, ToolResult
+
+
+class AnthropicExecutor:
+ def __init__(
+ self,
+ output_callback: Callable[[BetaContentBlockParam], None],
+ tool_output_callback: Callable[[Any, str], None],
+ selected_screen: int = 0
+ ):
+ self.tool_collection = ToolCollection(
+ ComputerTool(selected_screen=selected_screen),
+ BashTool(),
+ EditTool(),
+ )
+ self.output_callback = output_callback
+ self.tool_output_callback = tool_output_callback
+
+ def __call__(self, response: BetaMessage, messages: list[BetaMessageParam]):
+ new_message = {
+ "role": "assistant",
+ "content": cast(list[BetaContentBlockParam], response.content),
+ }
+ if new_message not in messages:
+ messages.append(new_message)
+ else:
+ print("new_message already in messages, there are duplicates.")
+
+ tool_result_content: list[BetaToolResultBlockParam] = []
+ for content_block in cast(list[BetaContentBlock], response.content):
+
+ self.output_callback(content_block, sender="bot")
+ # Execute the tool
+ if content_block.type == "tool_use":
+ # Run the asynchronous tool execution in a synchronous context
+ result = asyncio.run(self.tool_collection.run(
+ name=content_block.name,
+ tool_input=cast(dict[str, Any], content_block.input),
+ ))
+
+ self.output_callback(result, sender="bot")
+
+ tool_result_content.append(
+ _make_api_tool_result(result, content_block.id)
+ )
+ self.tool_output_callback(result, content_block.id)
+
+ # Craft messages based on the content_block
+ # Note: to display the messages in the gradio, you should organize the messages in the following way (user message, bot message)
+
+ display_messages = _message_display_callback(messages)
+ # display_messages = []
+
+ # Send the messages to the gradio
+ for user_msg, bot_msg in display_messages:
+ # yield [user_msg, bot_msg], tool_result_content
+ yield [None, None], tool_result_content
+
+ if not tool_result_content:
+ return messages
+
+ return tool_result_content
+
+def _message_display_callback(messages):
+ display_messages = []
+ for msg in messages:
+ try:
+ if isinstance(msg["content"][0], TextBlock):
+ display_messages.append((msg["content"][0].text, None)) # User message
+ elif isinstance(msg["content"][0], BetaTextBlock):
+ display_messages.append((None, msg["content"][0].text)) # Bot message
+ elif isinstance(msg["content"][0], BetaToolUseBlock):
+ display_messages.append((None, f"Tool Use: {msg['content'][0].name}\nInput: {msg['content'][0].input}")) # Bot message
+ elif isinstance(msg["content"][0], Dict) and msg["content"][0]["content"][-1]["type"] == "image":
+ display_messages.append((None, f'
')) # Bot message
+ else:
+ print(msg["content"][0])
+ except Exception as e:
+ print("error", e)
+ pass
+ return display_messages
+
+def _make_api_tool_result(
+ result: ToolResult, tool_use_id: str
+) -> BetaToolResultBlockParam:
+ """Convert an agent ToolResult to an API ToolResultBlockParam."""
+ tool_result_content: list[BetaTextBlockParam | BetaImageBlockParam] | str = []
+ is_error = False
+ if result.error:
+ is_error = True
+ tool_result_content = _maybe_prepend_system_tool_result(result, result.error)
+ else:
+ if result.output:
+ tool_result_content.append(
+ {
+ "type": "text",
+ "text": _maybe_prepend_system_tool_result(result, result.output),
+ }
+ )
+ if result.base64_image:
+ tool_result_content.append(
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": result.base64_image,
+ },
+ }
+ )
+ return {
+ "type": "tool_result",
+ "content": tool_result_content,
+ "tool_use_id": tool_use_id,
+ "is_error": is_error,
+ }
+
+
+def _maybe_prepend_system_tool_result(result: ToolResult, result_text: str):
+ if result.system:
+ result_text = f")})
',
+ sender="bot")
+ yield None
+
+ output_callback(f"{colorful_text_vlm} sending action to {colorful_text_showui}:\n{next_action}", sender="bot")
+
+ actor_response = actor(messages=next_action)
+ yield actor_response
+
+ for message, tool_result_content in executor(actor_response, messages):
+ time.sleep(1)
+ yield message
+
+ # since showui executor has no feedback for now, we use "actor_response" to represent its response
+ # update messages for the next loop
+ messages.append({"role": "user",
+ "content": ["History plan:\n" + str(json.loads(vlm_response)) +
+ "\nHistory actions:\n" + str(actor_response["content"])]})
+ print(f"End of loop. Messages: {str(messages)[:100000]}. Total cost: $USD{planner.total_cost:.5f}")
\ No newline at end of file
diff --git a/demo/omniparser_agent/vlm_agent.py b/demo/omniparser_agent/vlm_agent.py
new file mode 100644
index 0000000..53ba46a
--- /dev/null
+++ b/demo/omniparser_agent/vlm_agent.py
@@ -0,0 +1,417 @@
+import json
+import asyncio
+import platform
+from collections.abc import Callable
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, cast, Dict, Callable
+import uuid
+import requests
+from PIL import Image, ImageDraw
+import base64
+from io import BytesIO
+
+from anthropic import Anthropic, AnthropicBedrock, AnthropicVertex, APIResponse
+from anthropic.types import TextBlock, ToolResultBlockParam
+from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock, BetaMessageParam, BetaUsage
+
+from tools.screen_capture import get_screenshot
+from gui_agent.llm_utils.oai import run_oai_interleaved, encode_image
+from gui_agent.llm_utils.qwen import run_qwen
+from gui_agent.llm_utils.llm_utils import extract_data
+from tools.colorful_text import colorful_text_showui, colorful_text_vlm
+
+
+SYSTEM_PROMPT = f"""\n''' + # screen_info += f'ID: {idx}, Text: {element["content"]}\n' + # elif element['type'] == 'icon': + # # screen_info += f'''
',
+ sender="bot")
+ self.output_callback(f'Set of Marks Screenshot for {colorful_text_vlm}:\n
', sender="bot")
+
+
+ if isinstance(planner_messages[-1], dict):
+ if not isinstance(planner_messages[-1]["content"], list):
+ planner_messages[-1]["content"] = [planner_messages[-1]["content"]]
+ planner_messages[-1]["content"].append(screenshot_path)
+ planner_messages[-1]["content"].append(som_screenshot_path)
+
+ print(f"Sending messages to VLMPlanner : {planner_messages}")
+
+ if "gpt" in self.model:
+ vlm_response, token_usage = run_oai_interleaved(
+ messages=planner_messages,
+ system=system,
+ llm=self.model,
+ api_key=self.api_key,
+ max_tokens=self.max_tokens,
+ temperature=0,
+ )
+ print(f"oai token usage: {token_usage}")
+ self.total_token_usage += token_usage
+ self.total_cost += (token_usage * 0.15 / 1000000) # https://openai.com/api/pricing/
+
+ elif "qwen" in self.model:
+ vlm_response, token_usage = run_qwen(
+ messages=planner_messages,
+ system=system,
+ llm=self.model,
+ api_key=self.api_key,
+ max_tokens=self.max_tokens,
+ temperature=0,
+ )
+ print(f"qwen token usage: {token_usage}")
+ self.total_token_usage += token_usage
+ self.total_cost += (token_usage * 0.02 / 7.25 / 1000) # 1USD=7.25CNY, https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api
+ elif "phi" in self.model:
+ pass # TODO
+ else:
+ raise ValueError(f"Model {self.model} not supported")
+
+ print(f"VLMPlanner response: {vlm_response}")
+
+ if self.print_usage:
+ print(f"VLMPlanner total token usage so far: {self.total_token_usage}. Total cost so far: $USD{self.total_cost:.5f}")
+
+ vlm_response_json = extract_data(vlm_response, "json")
+ vlm_response_json = json.loads(vlm_response_json)
+
+ # map "box_id" to "idx" in parsed_screen, and output the xy coordinate of bbox
+ # TODO add try except for the case when "box_id" is not in the response
+ # if 'Box ID' in vlm_response_json:
+ try:
+ bbox = parsed_screen["parsed_content_list"][int(vlm_response_json["Box ID"])]["bbox"]
+ vlm_response_json["coordinate"] = [int((bbox[0] + bbox[2]) / 2 * screen_width), int((bbox[1] + bbox[3]) / 2 * screen_height)]
+ # draw a circle on the screenshot image to indicate the action
+ self.draw_action(vlm_response_json, image_base64)
+ except:
+ print("No Box ID in the response.")
+
+
+ # vlm_plan_str = '\n'.join([f'{key}: {value}' for key, value in json.loads(response).items()])
+ vlm_plan_str = ""
+ for key, value in vlm_response_json.items():
+ if key == "Reasoning":
+ vlm_plan_str += f'{value}'
+ else:
+ vlm_plan_str += f'\n{key}: {value}'
+
+ # self.output_callback(f"{colorful_text_vlm}:\n{vlm_plan_str}", sender="bot")
+
+ # construct the response so that anthropicExcutor can execute the tool
+ analysis = BetaTextBlock(text=vlm_plan_str, type='text')
+ if 'coordinate' in vlm_response_json:
+ move_cursor_block = BetaToolUseBlock(id=f'toolu_{uuid.uuid4()}',
+ input={'action': 'mouse_move', 'coordinate': vlm_response_json["coordinate"]},
+ name='computer', type='tool_use')
+ response_content = [analysis, move_cursor_block]
+ else:
+ response_content = [analysis]
+ if vlm_response_json["Next Action"] == "type":
+ click_block = BetaToolUseBlock(id=f'toolu_{uuid.uuid4()}', input={'action': 'left_click'}, name='computer', type='tool_use')
+ sim_content_block = BetaToolUseBlock(id=f'toolu_{uuid.uuid4()}',
+ input={'action': vlm_response_json["Next Action"], 'text': vlm_response_json["value"]},
+ name='computer', type='tool_use')
+ response_content.extend([click_block, sim_content_block])
+ elif vlm_response_json["Next Action"] == "None":
+ print("Task paused/completed.")
+ else:
+ sim_content_block = BetaToolUseBlock(id=f'toolu_{uuid.uuid4()}',
+ input={'action': vlm_response_json["Next Action"]},
+ name='computer', type='tool_use')
+ response_content.append(sim_content_block)
+
+ response = BetaMessage(id=f'toolu_{uuid.uuid4()}', content=response_content, model='', role='assistant', type='message', stop_reason='tool_use', usage=BetaUsage(input_tokens=0, output_tokens=0))
+ return response, vlm_response_json
+
+
+ def _api_response_callback(self, response: APIResponse):
+ self.api_response_callback(response)
+
+
+ def reformat_messages(self, messages: list):
+ pass
+
+ def _get_system_prompt(self, screen_info: str = ""):
+ datetime_str = datetime.now().strftime("%A, %B %d, %Y")
+ os_name = platform.system()
+ return f"""
+You are using an {os_name} device.
+You are able to use a mouse and keyboard to interact with the computer based on the given task and screenshot.
+You can only interact with the desktop GUI (no terminal or application menu access).
+
+You may be given some history plan and actions, this is the response from the previous loop.
+You should carefully consider your plan base on the task, screenshot, and history actions.
+
+Here is the list of all detected bounding boxes by IDs on the screen and their description:{screen_info}
+
+Your available "Next Action" only include:
+- type: type a string of text.
+- left_click: Describe the ui element to be clicked.
+- enter: Press an enter key.
+- escape: Press an ESCAPE key.
+- hover: Describe the ui element to be hovered.
+- scroll: Scroll the screen, you must specify up or down.
+- press: Describe the ui element to be pressed.
+
+Based on the visual information from the screenshot image and the detected bounding boxes, please determine the next action, the Box ID you should operate on, and the value (if the action is 'type') in order to complete the task.
+
+Output format:
+```json
+{{
+ "Reasoning": str, # describe what is in the current screen, taking into account the history, then describe your step-by-step thoughts on how to achieve the task, choose one action from available actions at a time.
+ "Next Action": "action_type, action description" | "None" # one action at a time, describe it in short and precisely.
+ 'Box ID': n,
+ 'value': "xxx" # if the action is type, you should provide the text to type.
+}}
+```
+
+One Example:
+```json
+{{
+ "Reasoning": "The current screen shows google result of amazon, in previous action I have searched amazon on google. Then I need to click on the first search results to go to amazon.com.",
+ "Next Action": "left_click",
+ 'Box ID': m,
+}}
+```
+
+Another Example:
+```json
+{{
+ "Reasoning": "The current screen shows the front page of amazon. There is no previous action. Therefore I need to type "Apple watch" in the search bar.",
+ "Next Action": "type",
+ 'Box ID': n,
+ 'value': "Apple watch"
+}}
+```
+
+IMPORTANT NOTES:
+1. You should only give a single action at a time.
+2. You should give an analysis to the current screen, and reflect on what has been done by looking at the history, then describe your step-by-step thoughts on how to achieve the task.
+3. Attach the next action prediction in the "Next Action".
+4. You should not include other actions, such as keyboard shortcuts.
+5. When the task is completed, you should say "Next Action": "None" in the json field.
+"""
+ def draw_action(self, vlm_response_json, image_base64):
+ # draw a circle using the coordinate in parsed_screen['som_image_base64']
+ image_data = base64.b64decode(image_base64)
+ image = Image.open(BytesIO(image_data))
+
+ draw = ImageDraw.Draw(image)
+ x, y = vlm_response_json["coordinate"]
+ radius = 10
+ draw.ellipse((x - radius, y - radius, x + radius, y + radius), outline='red', width=3)
+ buffered = BytesIO()
+ image.save('demo.png')
+ image.save(buffered, format="PNG")
+ image_with_circle_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
+ self.output_callback(f'Action performed on the Screenshot (red circle), for {colorful_text_vlm}:\n
', sender="bot")
+
+
+def _keep_latest_images(messages):
+ for i in range(len(messages)-1):
+ if isinstance(messages[i]["content"], list):
+ for cnt in messages[i]["content"]:
+ if isinstance(cnt, str):
+ if cnt.endswith((".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif")):
+ messages[i]["content"].remove(cnt)
+ return messages
+
+
+def _maybe_filter_to_n_most_recent_images(
+ messages: list[BetaMessageParam],
+ images_to_keep: int,
+ min_removal_threshold: int = 10,
+):
+ """
+ With the assumption that images are screenshots that are of diminishing value as
+ the conversation progresses, remove all but the final `images_to_keep` tool_result
+ images in place, with a chunk of min_removal_threshold to reduce the amount we
+ break the implicit prompt cache.
+ """
+ if images_to_keep is None:
+ return messages
+
+ tool_result_blocks = cast(
+ list[ToolResultBlockParam],
+ [
+ item
+ for message in messages
+ for item in (
+ message["content"] if isinstance(message["content"], list) else []
+ )
+ if isinstance(item, dict) and item.get("type") == "tool_result"
+ ],
+ )
+
+ total_images = sum(
+ 1
+ for tool_result in tool_result_blocks
+ for content in tool_result.get("content", [])
+ if isinstance(content, dict) and content.get("type") == "image"
+ )
+
+ images_to_remove = total_images - images_to_keep
+ # for better cache behavior, we want to remove in chunks
+ images_to_remove -= images_to_remove % min_removal_threshold
+
+ for tool_result in tool_result_blocks:
+ if isinstance(tool_result.get("content"), list):
+ new_content = []
+ for content in tool_result.get("content", []):
+ if isinstance(content, dict) and content.get("type") == "image":
+ if images_to_remove > 0:
+ images_to_remove -= 1
+ continue
+ new_content.append(content)
+ tool_result["content"] = new_content
+
+
+def _message_filter_callback(messages):
+ filtered_list = []
+ try:
+ for msg in messages:
+ if msg.get('role') in ['user']:
+ if not isinstance(msg["content"], list):
+ msg["content"] = [msg["content"]]
+ if isinstance(msg["content"][0], TextBlock):
+ filtered_list.append(str(msg["content"][0].text)) # User message
+ elif isinstance(msg["content"][0], str):
+ filtered_list.append(msg["content"][0]) # User message
+ else:
+ print("[_message_filter_callback]: drop message", msg)
+ continue
+
+ # elif msg.get('role') in ['assistant']:
+ # if isinstance(msg["content"][0], TextBlock):
+ # msg["content"][0] = str(msg["content"][0].text)
+ # elif isinstance(msg["content"][0], BetaTextBlock):
+ # msg["content"][0] = str(msg["content"][0].text)
+ # elif isinstance(msg["content"][0], BetaToolUseBlock):
+ # msg["content"][0] = str(msg['content'][0].input)
+ # elif isinstance(msg["content"][0], Dict) and msg["content"][0]["content"][-1]["type"] == "image":
+ # msg["content"][0] = f''
+ # else:
+ # print("[_message_filter_callback]: drop message", msg)
+ # continue
+ # filtered_list.append(msg["content"][0]) # User message
+
+ else:
+ print("[_message_filter_callback]: drop message", msg)
+ continue
+
+ except Exception as e:
+ print("[_message_filter_callback]: error", e)
+
+ return filtered_list
\ No newline at end of file
diff --git a/demo/remote_request.py b/demo/remote_request.py
index 22f399a..d0bc1c3 100644
--- a/demo/remote_request.py
+++ b/demo/remote_request.py
@@ -1,78 +1,78 @@
-# uvicorn remote_request:app --host 0.0.0.0 --port 8000 --reload
-
-import sys
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from utils import get_som_labeled_img, check_ocr_box, get_caption_model_processor, get_yolo_model
-import torch
-from PIL import Image
-from typing import Dict, Tuple, List
-import base64
-
-
-config = {
- 'som_model_path': '../weights/icon_detect_v1_5/model_v1_5.pt',
- 'device': 'cpu',
- 'caption_model_name': 'florence2',
- 'caption_model_path': '../weights/icon_caption_florence',
- 'BOX_TRESHOLD': 0.05
-}
-
-
-class Omniparser(object):
- def __init__(self, config: Dict):
- self.config = config
- device = 'cuda' if torch.cuda.is_available() else 'cpu'
-
- self.som_model = get_yolo_model(model_path=config['som_model_path'])
- self.caption_model_processor = get_caption_model_processor(model_name=config['caption_model_name'], model_name_or_path=config['caption_model_path'], device=device)
- print('Omniparser initialized!!!')
-
- def parse(self, image_base64: str):
- image_path = '../imgs/demo_image.jpg'
- with open(image_path, "wb") as fh:
- fh.write(base64.b64decode(image_base64))
- print('Parsing image:', image_path)
-
- image = Image.open(image_path)
- print('image size:', image.size)
-
- box_overlay_ratio = max(image.size) / 3200
- draw_bbox_config = {
- 'text_scale': 0.8 * box_overlay_ratio,
- 'text_thickness': max(int(2 * box_overlay_ratio), 1),
- 'text_padding': max(int(3 * box_overlay_ratio), 1),
- 'thickness': max(int(3 * box_overlay_ratio), 1),
- }
- BOX_TRESHOLD = config['BOX_TRESHOLD']
-
- ocr_bbox_rslt, is_goal_filtered = check_ocr_box(image_path, display_img = False, output_bb_format='xyxy', goal_filtering=None, easyocr_args={'paragraph': False, 'text_threshold':0.8}, use_paddleocr=True)
- text, ocr_bbox = ocr_bbox_rslt
- dino_labled_img, label_coordinates, parsed_content_list = get_som_labeled_img(image_path, self.som_model, BOX_TRESHOLD = BOX_TRESHOLD, output_coord_in_ratio=True, ocr_bbox=ocr_bbox,draw_bbox_config=draw_bbox_config, caption_model_processor=self.caption_model_processor, ocr_text=text,use_local_semantics=True, iou_threshold=0.7, scale_img=False, batch_size=128)
- with open('../imgs/demo_image_som.jpg', "wb") as fh:
- fh.write(base64.b64decode(dino_labled_img))
-
- return dino_labled_img, parsed_content_list
-
-
-from fastapi import FastAPI
-from pydantic import BaseModel
-
-app = FastAPI()
-
-class Item(BaseModel):
- base64_image: str
- prompt: str
-
-Omniparser = Omniparser(config)
-
-@app.post("/send_text/")
-async def send_text(item: Item):
- print('start parsing...')
- import time
- start = time.time()
- dino_labled_img, parsed_content_list = Omniparser.parse(item.base64_image)
- latency = time.time() - start
- print('time:', latency)
+# uvicorn remote_request:app --host 0.0.0.0 --port 8000 --reload
+
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from utils import get_som_labeled_img, check_ocr_box, get_caption_model_processor, get_yolo_model
+import torch
+from PIL import Image
+from typing import Dict, Tuple, List
+import base64
+
+
+config = {
+ 'som_model_path': '../weights/icon_detect_v1_5/model_v1_5.pt',
+ 'device': 'cpu',
+ 'caption_model_name': 'florence2',
+ 'caption_model_path': '../weights/icon_caption_florence',
+ 'BOX_TRESHOLD': 0.05
+}
+
+
+class Omniparser(object):
+ def __init__(self, config: Dict):
+ self.config = config
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+ self.som_model = get_yolo_model(model_path=config['som_model_path'])
+ self.caption_model_processor = get_caption_model_processor(model_name=config['caption_model_name'], model_name_or_path=config['caption_model_path'], device=device)
+ print('Omniparser initialized!!!')
+
+ def parse(self, image_base64: str):
+ image_path = '../imgs/demo_image.jpg'
+ with open(image_path, "wb") as fh:
+ fh.write(base64.b64decode(image_base64))
+ print('Parsing image:', image_path)
+
+ image = Image.open(image_path)
+ print('image size:', image.size)
+
+ box_overlay_ratio = max(image.size) / 3200
+ draw_bbox_config = {
+ 'text_scale': 0.8 * box_overlay_ratio,
+ 'text_thickness': max(int(2 * box_overlay_ratio), 1),
+ 'text_padding': max(int(3 * box_overlay_ratio), 1),
+ 'thickness': max(int(3 * box_overlay_ratio), 1),
+ }
+ BOX_TRESHOLD = config['BOX_TRESHOLD']
+
+ ocr_bbox_rslt, is_goal_filtered = check_ocr_box(image_path, display_img = False, output_bb_format='xyxy', goal_filtering=None, easyocr_args={'paragraph': False, 'text_threshold':0.8}, use_paddleocr=True)
+ text, ocr_bbox = ocr_bbox_rslt
+ dino_labled_img, label_coordinates, parsed_content_list = get_som_labeled_img(image_path, self.som_model, BOX_TRESHOLD = BOX_TRESHOLD, output_coord_in_ratio=True, ocr_bbox=ocr_bbox,draw_bbox_config=draw_bbox_config, caption_model_processor=self.caption_model_processor, ocr_text=text,use_local_semantics=True, iou_threshold=0.7, scale_img=False, batch_size=128)
+ with open('../imgs/demo_image_som.jpg', "wb") as fh:
+ fh.write(base64.b64decode(dino_labled_img))
+
+ return dino_labled_img, parsed_content_list
+
+
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+app = FastAPI()
+
+class Item(BaseModel):
+ base64_image: str
+ prompt: str
+
+Omniparser = Omniparser(config)
+
+@app.post("/send_text/")
+async def send_text(item: Item):
+ print('start parsing...')
+ import time
+ start = time.time()
+ dino_labled_img, parsed_content_list = Omniparser.parse(item.base64_image)
+ latency = time.time() - start
+ print('time:', latency)
return {"som_image_base64": dino_labled_img, "parsed_content_list": parsed_content_list, 'latency': latency}
\ No newline at end of file
diff --git a/demo/tmp/outputs/screenshot.png b/demo/tmp/outputs/screenshot.png
new file mode 100644
index 0000000..eca7672
Binary files /dev/null and b/demo/tmp/outputs/screenshot.png differ
diff --git a/demo/tools/__init__.py b/demo/tools/__init__.py
new file mode 100644
index 0000000..e0bf3e2
--- /dev/null
+++ b/demo/tools/__init__.py
@@ -0,0 +1,16 @@
+from .base import CLIResult, ToolResult
+from .bash import BashTool
+from .collection import ToolCollection
+from .computer import ComputerTool
+from .edit import EditTool
+from .screen_capture import get_screenshot
+
+__ALL__ = [
+ BashTool,
+ CLIResult,
+ ComputerTool,
+ EditTool,
+ ToolCollection,
+ ToolResult,
+ get_screenshot,
+]
diff --git a/demo/tools/base.py b/demo/tools/base.py
new file mode 100644
index 0000000..1acf24a
--- /dev/null
+++ b/demo/tools/base.py
@@ -0,0 +1,69 @@
+from abc import ABCMeta, abstractmethod
+from dataclasses import dataclass, fields, replace
+from typing import Any
+
+from anthropic.types.beta import BetaToolUnionParam
+
+
+class BaseAnthropicTool(metaclass=ABCMeta):
+ """Abstract base class for Anthropic-defined tools."""
+
+ @abstractmethod
+ def __call__(self, **kwargs) -> Any:
+ """Executes the tool with the given arguments."""
+ ...
+
+ @abstractmethod
+ def to_params(
+ self,
+ ) -> BetaToolUnionParam:
+ raise NotImplementedError
+
+
+@dataclass(kw_only=True, frozen=True)
+class ToolResult:
+ """Represents the result of a tool execution."""
+
+ output: str | None = None
+ error: str | None = None
+ base64_image: str | None = None
+ system: str | None = None
+
+ def __bool__(self):
+ return any(getattr(self, field.name) for field in fields(self))
+
+ def __add__(self, other: "ToolResult"):
+ def combine_fields(
+ field: str | None, other_field: str | None, concatenate: bool = True
+ ):
+ if field and other_field:
+ if concatenate:
+ return field + other_field
+ raise ValueError("Cannot combine tool results")
+ return field or other_field
+
+ return ToolResult(
+ output=combine_fields(self.output, other.output),
+ error=combine_fields(self.error, other.error),
+ base64_image=combine_fields(self.base64_image, other.base64_image, False),
+ system=combine_fields(self.system, other.system),
+ )
+
+ def replace(self, **kwargs):
+ """Returns a new ToolResult with the given fields replaced."""
+ return replace(self, **kwargs)
+
+
+class CLIResult(ToolResult):
+ """A ToolResult that can be rendered as a CLI output."""
+
+
+class ToolFailure(ToolResult):
+ """A ToolResult that represents a failure."""
+
+
+class ToolError(Exception):
+ """Raised when a tool encounters an error."""
+
+ def __init__(self, message):
+ self.message = message
diff --git a/demo/tools/bash.py b/demo/tools/bash.py
new file mode 100644
index 0000000..56eb912
--- /dev/null
+++ b/demo/tools/bash.py
@@ -0,0 +1,136 @@
+import asyncio
+import os
+from typing import ClassVar, Literal
+
+from anthropic.types.beta import BetaToolBash20241022Param
+
+from .base import BaseAnthropicTool, CLIResult, ToolError, ToolResult
+
+
+class _BashSession:
+ """A session of a bash shell."""
+
+ _started: bool
+ _process: asyncio.subprocess.Process
+
+ command: str = "/bin/bash"
+ _output_delay: float = 0.2 # seconds
+ _timeout: float = 120.0 # seconds
+ _sentinel: str = "<
-
-
+
+