alihan/BrowserOS
1
0
Fork 0
mirror of https://github.com/browseros-ai/BrowserOS.git synced 2025-07-21 00:21:28 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
BrowserOS/patches/nxtscape/browserOS-API.patch
Nikhil Sonti d5a0e62d0c adding new patches
2025-07-17 17:57:18 -07:00

2742 lines
97 KiB
Diff
Raw Permalink Blame History

From fd332e09d563fe667f00c541a889d2a4b41dd200 Mon Sep 17 00:00:00 2001
From: Nikhil Sonti <nikhilsv92@gmail.com>
Date: Mon, 7 Jul 2025 15:38:54 -0700
Subject: [PATCH 1/2] browserOS API
---
chrome/browser/extensions/BUILD.gn | 8 +
.../api/browser_os/browser_os_api.cc | 760 ++++++++++++++++++
.../api/browser_os/browser_os_api.h | 196 +++++
.../api/browser_os/browser_os_api_helpers.cc | 63 ++
.../api/browser_os/browser_os_api_helpers.h | 44 +
.../api/browser_os/browser_os_api_utils.cc | 378 +++++++++
.../api/browser_os/browser_os_api_utils.h | 62 ++
.../browser_os_snapshot_processor.cc | 679 ++++++++++++++++
.../browser_os_snapshot_processor.h | 89 ++
.../chrome_extensions_browser_api_provider.cc | 8 +
.../common/extensions/api/_api_features.json | 28 +
.../extensions/api/_permission_features.json | 4 +
chrome/common/extensions/api/api_sources.gni | 1 +
chrome/common/extensions/api/browser_os.idl | 194 +++++
.../permissions/chrome_api_permissions.cc | 1 +
.../extension_function_histogram_value.h | 12 +
.../common/mojom/api_permission_id.mojom | 1 +
.../histograms/metadata/extensions/enums.xml | 10 +
18 files changed, 2538 insertions(+)
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_api.cc
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_api.h
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_api_helpers.cc
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_api_helpers.h
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_api_utils.cc
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_api_utils.h
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.cc
create mode 100644 chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.h
create mode 100644 chrome/common/extensions/api/browser_os.idl
diff --git a/chrome/browser/extensions/BUILD.gn b/chrome/browser/extensions/BUILD.gn
index d50ffdfbcce34..bbb5a16ccd5d0 100644
--- a/chrome/browser/extensions/BUILD.gn
+++ b/chrome/browser/extensions/BUILD.gn
@@ -516,6 +516,14 @@ source_set("extensions") {
"api/bookmark_manager_private/bookmark_manager_private_api.h",
"api/bookmarks/bookmarks_api.cc",
"api/bookmarks/bookmarks_api.h",
+ "api/browser_os/browser_os_api.cc",
+ "api/browser_os/browser_os_api.h",
+ "api/browser_os/browser_os_api_helpers.cc",
+ "api/browser_os/browser_os_api_helpers.h",
+ "api/browser_os/browser_os_api_utils.cc",
+ "api/browser_os/browser_os_api_utils.h",
+ "api/browser_os/browser_os_snapshot_processor.cc",
+ "api/browser_os/browser_os_snapshot_processor.h",
"api/chrome_device_permissions_prompt.h",
"api/chrome_extensions_api_client.cc",
"api/chrome_extensions_api_client.h",
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_api.cc b/chrome/browser/extensions/api/browser_os/browser_os_api.cc
new file mode 100644
index 0000000000000..c39e67d88e1de
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_api.cc
@@ -0,0 +1,760 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/extensions/api/browser_os/browser_os_api.h"
+
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "base/functional/bind.h"
+#include "base/json/json_writer.h"
+#include "base/strings/utf_string_conversions.h"
+#include "base/base64.h"
+#include "base/time/time.h"
+#include "base/values.h"
+#include "chrome/browser/extensions/api/browser_os/browser_os_api_helpers.h"
+#include "chrome/browser/extensions/api/browser_os/browser_os_api_utils.h"
+#include "chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.h"
+#include "chrome/browser/extensions/extension_tab_util.h"
+#include "chrome/browser/extensions/window_controller.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/browser_finder.h"
+#include "chrome/browser/ui/tabs/tab_strip_model.h"
+#include "chrome/common/extensions/api/browser_os.h"
+#include "content/browser/renderer_host/render_widget_host_impl.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_widget_host.h"
+#include "content/public/browser/render_widget_host_view.h"
+#include "content/public/browser/web_contents.h"
+#include "third_party/blink/public/common/input/web_input_event.h"
+#include "third_party/blink/public/common/input/web_mouse_event.h"
+#include "ui/accessibility/ax_action_data.h"
+#include "ui/accessibility/ax_enum_util.h"
+#include "ui/accessibility/ax_mode.h"
+#include "ui/accessibility/ax_node_data.h"
+#include "ui/accessibility/ax_role_properties.h"
+#include "ui/accessibility/ax_tree_update.h"
+#include "ui/base/ime/ime_text_span.h"
+#include "ui/events/base_event_utils.h"
+#include "ui/events/keycodes/dom/dom_code.h"
+#include "ui/events/keycodes/dom/dom_key.h"
+#include "ui/events/keycodes/keyboard_codes.h"
+#include "ui/gfx/geometry/point_f.h"
+#include "ui/gfx/geometry/rect.h"
+#include "ui/gfx/geometry/rect_f.h"
+#include "ui/gfx/range/range.h"
+#include "ui/gfx/codec/png_codec.h"
+#include "ui/gfx/image/image.h"
+#include "ui/snapshot/snapshot.h"
+
+namespace extensions {
+namespace api {
+
+// Static member initialization
+uint32_t BrowserOSGetInteractiveSnapshotFunction::next_snapshot_id_ = 1;
+
+// Constructor and destructor implementations
+BrowserOSGetInteractiveSnapshotFunction::BrowserOSGetInteractiveSnapshotFunction() = default;
+BrowserOSGetInteractiveSnapshotFunction::~BrowserOSGetInteractiveSnapshotFunction() = default;
+
+ExtensionFunction::ResponseAction BrowserOSGetAccessibilityTreeFunction::Run() {
+ std::optional<browser_os::GetAccessibilityTree::Params> params =
+ browser_os::GetAccessibilityTree::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+
+ // Enable accessibility if needed
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ // Request accessibility tree snapshot
+ // Use WebContents with extended properties to get a full tree
+ web_contents->RequestAXTreeSnapshot(
+ base::BindOnce(
+ &BrowserOSGetAccessibilityTreeFunction::OnAccessibilityTreeReceived,
+ this),
+ ui::AXMode(ui::AXMode::kWebContents | ui::AXMode::kExtendedProperties |
+ ui::AXMode::kInlineTextBoxes),
+ /* max_nodes= */ 0, // No limit
+ /* timeout= */ base::TimeDelta(),
+ content::WebContents::AXTreeSnapshotPolicy::kAll);
+
+ return RespondLater();
+}
+
+void BrowserOSGetAccessibilityTreeFunction::OnAccessibilityTreeReceived(
+ ui::AXTreeUpdate& tree_update) {
+ browser_os::AccessibilityTree result;
+ result.root_id = tree_update.root_id;
+
+ // Convert AX nodes to API format
+ base::Value::Dict nodes;
+ for (const auto& node_data : tree_update.nodes) {
+ browser_os::AccessibilityNode node;
+ node.id = node_data.id;
+ node.role = ui::ToString(node_data.role);
+
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kName)) {
+ node.name =
+ node_data.GetStringAttribute(ax::mojom::StringAttribute::kName);
+ }
+
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kValue)) {
+ node.value =
+ node_data.GetStringAttribute(ax::mojom::StringAttribute::kValue);
+ }
+
+ // Add child IDs
+ if (!node_data.child_ids.empty()) {
+ node.child_ids.emplace();
+ for (int32_t child_id : node_data.child_ids) {
+ node.child_ids->push_back(child_id);
+ }
+ }
+
+ // Add basic attributes
+ base::Value::Dict attributes;
+ if (node_data.HasBoolAttribute(ax::mojom::BoolAttribute::kSelected)) {
+ attributes.Set("selected",
+ node_data.GetBoolAttribute(ax::mojom::BoolAttribute::kSelected));
+ }
+ // TODO: Add focused attribute when available
+ if (node_data.HasIntAttribute(ax::mojom::IntAttribute::kHierarchicalLevel)) {
+ attributes.Set("level",
+ node_data.GetIntAttribute(ax::mojom::IntAttribute::kHierarchicalLevel));
+ }
+ if (!attributes.empty()) {
+ browser_os::AccessibilityNode::Attributes attr;
+ attr.additional_properties = std::move(attributes);
+ node.attributes = std::move(attr);
+ }
+
+ // Convert to dictionary
+ nodes.Set(base::NumberToString(node_data.id), node.ToValue());
+ }
+
+ result.nodes.additional_properties = std::move(nodes);
+
+ Respond(ArgumentList(
+ browser_os::GetAccessibilityTree::Results::Create(result)));
+}
+
+// Implementation of BrowserOSGetInteractiveSnapshotFunction
+
+ExtensionFunction::ResponseAction BrowserOSGetInteractiveSnapshotFunction::Run() {
+ std::optional<browser_os::GetInteractiveSnapshot::Params> params =
+ browser_os::GetInteractiveSnapshot::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+
+ // Note: We don't need to get scale factors here!
+ // The accessibility tree provides bounds in CSS pixels (logical pixels),
+ // which is the correct coordinate space for ForwardMouseEvent.
+ // The browser and renderer handle device pixel ratio conversion internally.
+
+ // Store tab ID for mapping
+ tab_id_ = tab_info->tab_id;
+
+ // Get viewport size
+ content::RenderWidgetHostView* rwhv = web_contents->GetRenderWidgetHostView();
+ if (rwhv) {
+ viewport_size_ = rwhv->GetVisibleViewportSize();
+ LOG(INFO) << "Viewport size: " << viewport_size_.ToString();
+ }
+
+ // Request accessibility tree snapshot
+ web_contents->RequestAXTreeSnapshot(
+ base::BindOnce(
+ &BrowserOSGetInteractiveSnapshotFunction::OnAccessibilityTreeReceived,
+ this),
+ ui::AXMode(ui::AXMode::kWebContents | ui::AXMode::kExtendedProperties |
+ ui::AXMode::kInlineTextBoxes),
+ /* max_nodes= */ 0, // No limit
+ /* timeout= */ base::TimeDelta(),
+ content::WebContents::AXTreeSnapshotPolicy::kAll);
+
+ return RespondLater();
+}
+
+void BrowserOSGetInteractiveSnapshotFunction::OnAccessibilityTreeReceived(
+ ui::AXTreeUpdate& tree_update) {
+ // Simple API layer - just delegates to the processor
+ SnapshotProcessor::ProcessAccessibilityTree(
+ tree_update,
+ tab_id_,
+ next_snapshot_id_++,
+ viewport_size_,
+ base::BindOnce(
+ &BrowserOSGetInteractiveSnapshotFunction::OnSnapshotProcessed,
+ base::WrapRefCounted(this)));
+}
+
+void BrowserOSGetInteractiveSnapshotFunction::OnSnapshotProcessed(
+ SnapshotProcessingResult result) {
+ Respond(ArgumentList(
+ browser_os::GetInteractiveSnapshot::Results::Create(result.snapshot)));
+}
+
+// Implementation of BrowserOSClickFunction
+
+ExtensionFunction::ResponseAction BrowserOSClickFunction::Run() {
+ std::optional<browser_os::Click::Params> params =
+ browser_os::Click::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+ int tab_id = tab_info->tab_id;
+
+ // Look up the AX node ID from our nodeId
+ auto tab_it = GetNodeIdMappings().find(tab_id);
+ if (tab_it == GetNodeIdMappings().end()) {
+ return RespondNow(Error("No snapshot data for this tab"));
+ }
+
+ auto node_it = tab_it->second.find(params->node_id);
+ if (node_it == tab_it->second.end()) {
+ return RespondNow(Error("Node ID not found"));
+ }
+
+ const NodeInfo& node_info = node_it->second;
+
+ // Calculate click point (center of the element)
+ gfx::PointF click_point(
+ node_info.bounds.x() + node_info.bounds.width() / 2.0f,
+ node_info.bounds.y() + node_info.bounds.height() / 2.0f);
+
+ // Perform the click
+ PerformClick(web_contents, click_point);
+
+ return RespondNow(NoArguments());
+}
+
+// Implementation of BrowserOSInputTextFunction
+
+ExtensionFunction::ResponseAction BrowserOSInputTextFunction::Run() {
+ std::optional<browser_os::InputText::Params> params =
+ browser_os::InputText::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+ int tab_id = tab_info->tab_id;
+
+ // Look up the AX node ID from our nodeId
+ auto tab_it = GetNodeIdMappings().find(tab_id);
+ if (tab_it == GetNodeIdMappings().end()) {
+ return RespondNow(Error("No snapshot data for this tab"));
+ }
+
+ auto node_it = tab_it->second.find(params->node_id);
+ if (node_it == tab_it->second.end()) {
+ return RespondNow(Error("Node ID not found"));
+ }
+
+ const NodeInfo& node_info = node_it->second;
+
+
+ // First, click on the element to focus it
+ gfx::PointF click_point(
+ node_info.bounds.x() + node_info.bounds.width() / 2.0f,
+ node_info.bounds.y() + node_info.bounds.height() / 2.0f);
+
+ PerformClick(web_contents, click_point);
+
+ // Get render widget host for text input
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh) {
+ return RespondNow(Error("No render widget host"));
+ }
+
+ // Convert text to UTF16
+ std::u16string text16 = base::UTF8ToUTF16(params->text);
+
+ // Add a small delay to ensure the element is focused after click
+ // Then send the text using ImeCommitText
+ base::SequencedTaskRunner::GetCurrentDefault()->PostDelayedTask(
+ FROM_HERE,
+ base::BindOnce(
+ [](content::RenderWidgetHost* rwh, const std::u16string& text) {
+ if (!rwh)
+ return;
+
+ content::RenderWidgetHostImpl* rwhi =
+ static_cast<content::RenderWidgetHostImpl*>(rwh);
+
+ // Ensure the widget has focus
+ rwhi->Focus();
+
+ // Try multiple approaches to input text
+ // 1. First try ImeSetComposition to simulate typing
+ rwhi->ImeSetComposition(text,
+ std::vector<ui::ImeTextSpan>(),
+ gfx::Range::InvalidRange(),
+ text.length(), // selection_start at end
+ text.length()); // selection_end at end
+
+ // 2. Then commit the text
+ rwhi->ImeCommitText(text,
+ std::vector<ui::ImeTextSpan>(),
+ gfx::Range::InvalidRange(),
+ 0); // relative_cursor_pos = 0 means after the text
+
+ // 3. Finish composing to ensure text is committed
+ rwhi->ImeFinishComposingText(false);
+
+ },
+ rwh, text16),
+ base::Milliseconds(100)); // Increase delay to 100ms for better focus handling
+
+
+ return RespondNow(NoArguments());
+}
+
+// Implementation of BrowserOSClearFunction
+
+ExtensionFunction::ResponseAction BrowserOSClearFunction::Run() {
+ std::optional<browser_os::Clear::Params> params =
+ browser_os::Clear::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+ int tab_id = tab_info->tab_id;
+
+ // Look up the AX node ID from our nodeId
+ auto tab_it = GetNodeIdMappings().find(tab_id);
+ if (tab_it == GetNodeIdMappings().end()) {
+ return RespondNow(Error("No snapshot data for this tab"));
+ }
+
+ auto node_it = tab_it->second.find(params->node_id);
+ if (node_it == tab_it->second.end()) {
+ return RespondNow(Error("Node ID not found"));
+ }
+
+ const NodeInfo& node_info = node_it->second;
+
+ // First, click on the element to focus it
+ gfx::PointF click_point(
+ node_info.bounds.x() + node_info.bounds.width() / 2.0f,
+ node_info.bounds.y() + node_info.bounds.height() / 2.0f);
+
+ PerformClick(web_contents, click_point);
+
+ // Get render widget host for keyboard events
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh) {
+ return RespondNow(Error("No render widget host"));
+ }
+
+ // Use JavaScript to clear the field, similar to how Puppeteer does it
+ rfh->ExecuteJavaScriptForTests(
+ u"(function() {"
+ u" var activeElement = document.activeElement;"
+ u" if (activeElement) {"
+ u" if (activeElement.value !== undefined) {"
+ u" activeElement.value = '';"
+ u" }"
+ u" if (activeElement.textContent !== undefined && activeElement.isContentEditable) {"
+ u" activeElement.textContent = '';"
+ u" }"
+ u" activeElement.dispatchEvent(new Event('input', {bubbles: true}));"
+ u" activeElement.dispatchEvent(new Event('change', {bubbles: true}));"
+ u" }"
+ u"})();",
+ base::NullCallback(),
+ /*honor_js_content_settings=*/false);
+
+ return RespondNow(NoArguments());
+}
+
+// Implementation of BrowserOSGetPageLoadStatusFunction
+
+ExtensionFunction::ResponseAction BrowserOSGetPageLoadStatusFunction::Run() {
+ std::optional<browser_os::GetPageLoadStatus::Params> params =
+ browser_os::GetPageLoadStatus::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+
+ // Get the primary main frame
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ // Build the status object
+ browser_os::PageLoadStatus status;
+
+ // Check if any resources are still loading
+ status.is_resources_loading = web_contents->IsLoading();
+
+ // Check if DOMContentLoaded has fired
+ status.is_dom_content_loaded = rfh->IsDOMContentLoaded();
+
+ // Check if onload has completed (all resources loaded)
+ status.is_page_complete = rfh->IsDocumentOnLoadCompletedInMainFrame();
+
+ return RespondNow(ArgumentList(
+ browser_os::GetPageLoadStatus::Results::Create(status)));
+}
+
+// Implementation of BrowserOSScrollUpFunction
+
+ExtensionFunction::ResponseAction BrowserOSScrollUpFunction::Run() {
+ std::optional<browser_os::ScrollUp::Params> params =
+ browser_os::ScrollUp::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+
+ // Get viewport height to scroll by approximately one page
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh) {
+ return RespondNow(Error("No render widget host"));
+ }
+
+ content::RenderWidgetHostView* rwhv = rwh->GetView();
+ if (!rwhv) {
+ return RespondNow(Error("No render widget host view"));
+ }
+
+ gfx::Rect viewport_bounds = rwhv->GetViewBounds();
+ int scroll_amount = viewport_bounds.height() * 0.9; // 90% of viewport height
+
+ // Perform scroll up (negative delta_y)
+ PerformScroll(web_contents, 0, -scroll_amount, true);
+
+ return RespondNow(NoArguments());
+}
+
+// Implementation of BrowserOSScrollDownFunction
+
+ExtensionFunction::ResponseAction BrowserOSScrollDownFunction::Run() {
+ std::optional<browser_os::ScrollDown::Params> params =
+ browser_os::ScrollDown::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+
+ // Get viewport height to scroll by approximately one page
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh) {
+ return RespondNow(Error("No render widget host"));
+ }
+
+ content::RenderWidgetHostView* rwhv = rwh->GetView();
+ if (!rwhv) {
+ return RespondNow(Error("No render widget host view"));
+ }
+
+ gfx::Rect viewport_bounds = rwhv->GetViewBounds();
+ int scroll_amount = viewport_bounds.height() * 0.9; // 90% of viewport height
+
+ // Perform scroll down (positive delta_y)
+ PerformScroll(web_contents, 0, scroll_amount, true);
+
+ return RespondNow(NoArguments());
+}
+
+// Implementation of BrowserOSScrollToNodeFunction
+
+ExtensionFunction::ResponseAction BrowserOSScrollToNodeFunction::Run() {
+ std::optional<browser_os::ScrollToNode::Params> params =
+ browser_os::ScrollToNode::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+ int tab_id = tab_info->tab_id;
+
+ // Look up the AX node ID from our nodeId
+ auto tab_it = GetNodeIdMappings().find(tab_id);
+ if (tab_it == GetNodeIdMappings().end()) {
+ return RespondNow(Error("No snapshot data for this tab"));
+ }
+
+ auto node_it = tab_it->second.find(params->node_id);
+ if (node_it == tab_it->second.end()) {
+ return RespondNow(Error("Node ID not found"));
+ }
+
+ const NodeInfo& node_info = node_it->second;
+
+ // Get viewport bounds to check if node is already in view
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh) {
+ return RespondNow(Error("No render widget host"));
+ }
+
+ content::RenderWidgetHostView* rwhv = rwh->GetView();
+ if (!rwhv) {
+ return RespondNow(Error("No render widget host view"));
+ }
+
+ gfx::Rect viewport_bounds = rwhv->GetViewBounds();
+
+ // Check if the node is already visible in the viewport
+ // We consider it visible if any part of it is within the viewport
+ bool is_in_view = false;
+ if (node_info.bounds.y() < viewport_bounds.height() &&
+ node_info.bounds.bottom() > 0 &&
+ node_info.bounds.x() < viewport_bounds.width() &&
+ node_info.bounds.right() > 0) {
+ is_in_view = true;
+ }
+
+ if (!is_in_view) {
+ // Use accessibility action to scroll
+ if (rfh) {
+ ui::AXActionData action_data;
+ action_data.action = ax::mojom::Action::kScrollToMakeVisible;
+ action_data.target_node_id = node_info.ax_node_id;
+ action_data.horizontal_scroll_alignment = ax::mojom::ScrollAlignment::kScrollAlignmentCenter;
+ action_data.vertical_scroll_alignment = ax::mojom::ScrollAlignment::kScrollAlignmentCenter;
+ action_data.scroll_behavior = ax::mojom::ScrollBehavior::kScrollIfVisible;
+
+ rfh->AccessibilityPerformAction(action_data);
+ }
+ }
+
+ return RespondNow(ArgumentList(
+ browser_os::ScrollToNode::Results::Create(!is_in_view)));
+}
+
+// Implementation of BrowserOSSendKeysFunction
+
+ExtensionFunction::ResponseAction BrowserOSSendKeysFunction::Run() {
+ std::optional<browser_os::SendKeys::Params> params =
+ browser_os::SendKeys::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+
+ // Validate the key - use a simple check instead of std::set to avoid exit-time destructor
+ const std::string& key = params->key;
+ bool is_supported = (key == "Enter" || key == "Delete" || key == "Backspace" ||
+ key == "Tab" || key == "Escape" || key == "ArrowUp" ||
+ key == "ArrowDown" || key == "ArrowLeft" || key == "ArrowRight" ||
+ key == "Home" || key == "End" || key == "PageUp" || key == "PageDown");
+
+ if (!is_supported) {
+ return RespondNow(Error("Unsupported key: " + params->key));
+ }
+
+ // Send the key
+ SendSpecialKey(web_contents, params->key);
+
+ return RespondNow(NoArguments());
+}
+
+// Implementation of BrowserOSCaptureScreenshotFunction
+
+ExtensionFunction::ResponseAction BrowserOSCaptureScreenshotFunction::Run() {
+ std::optional<browser_os::CaptureScreenshot::Params> params =
+ browser_os::CaptureScreenshot::Params::Create(args());
+ EXTENSION_FUNCTION_VALIDATE(params);
+
+ // Get the target tab
+ std::string error_message;
+ auto tab_info = GetTabFromOptionalId(params->tab_id, browser_context(),
+ include_incognito_information(),
+ &error_message);
+ if (!tab_info) {
+ return RespondNow(Error(error_message));
+ }
+
+ content::WebContents* web_contents = tab_info->web_contents;
+
+ // Get the render widget host view
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh) {
+ return RespondNow(Error("No render frame"));
+ }
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh) {
+ return RespondNow(Error("No render widget host"));
+ }
+
+ content::RenderWidgetHostView* rwhv = rwh->GetView();
+ if (!rwhv) {
+ return RespondNow(Error("No render widget host view"));
+ }
+
+ // Get the view bounds to determine the size
+ gfx::Rect view_bounds = rwhv->GetViewBounds();
+
+ // Create a reasonable thumbnail size (e.g., 256x256 or proportional)
+ const int kMaxThumbnailSize = 1024; // 512;//256;
+ gfx::Size thumbnail_size = view_bounds.size();
+
+ // Scale down proportionally
+ if (thumbnail_size.width() > kMaxThumbnailSize ||
+ thumbnail_size.height() > kMaxThumbnailSize) {
+ float scale = std::min(
+ static_cast<float>(kMaxThumbnailSize) / thumbnail_size.width(),
+ static_cast<float>(kMaxThumbnailSize) / thumbnail_size.height());
+ thumbnail_size = gfx::ScaleToFlooredSize(thumbnail_size, scale);
+ }
+
+ // For macOS, we need to use a different approach since GrabWindowSnapshot
+ // expects a window, not a view. Let's use CopyFromSurface instead.
+ content::RenderWidgetHostImpl* rwhi =
+ static_cast<content::RenderWidgetHostImpl*>(rwh);
+
+ // Request a copy of the surface
+ rwhi->GetView()->CopyFromSurface(
+ gfx::Rect(), // Empty rect means copy entire surface
+ thumbnail_size,
+ base::BindOnce(&BrowserOSCaptureScreenshotFunction::OnScreenshotCaptured,
+ this));
+
+ return RespondLater();
+}
+
+void BrowserOSCaptureScreenshotFunction::OnScreenshotCaptured(
+ const SkBitmap& bitmap) {
+ if (bitmap.empty()) {
+ Respond(Error("Failed to capture screenshot"));
+ return;
+ }
+
+ // Convert bitmap to PNG
+ auto png_data = gfx::PNGCodec::EncodeBGRASkBitmap(bitmap, false);
+ if (!png_data.has_value()) {
+ Respond(Error("Failed to encode screenshot"));
+ return;
+ }
+
+ // Convert to base64 data URL
+ std::string base64_data = base::Base64Encode(png_data.value());
+
+ std::string data_url = "data:image/png;base64," + base64_data;
+
+ Respond(ArgumentList(
+ browser_os::CaptureScreenshot::Results::Create(data_url)));
+}
+
+} // namespace api
+} // namespace extensions
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_api.h b/chrome/browser/extensions/api/browser_os/browser_os_api.h
new file mode 100644
index 0000000000000..58acc663f0170
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_api.h
@@ -0,0 +1,196 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_H_
+#define CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_H_
+
+#include <cstdint>
+
+#include "base/values.h"
+#include "chrome/browser/extensions/api/browser_os/browser_os_api_utils.h"
+#include "chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.h"
+#include "extensions/browser/extension_function.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+
+namespace content {
+class WebContents;
+}
+
+namespace ui {
+struct AXTreeUpdate;
+}
+
+namespace extensions {
+namespace api {
+
+
+class BrowserOSGetAccessibilityTreeFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.getAccessibilityTree",
+ BROWSER_OS_GETACCESSIBILITYTREE)
+
+ BrowserOSGetAccessibilityTreeFunction() = default;
+
+ protected:
+ ~BrowserOSGetAccessibilityTreeFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+
+ private:
+ void OnAccessibilityTreeReceived(ui::AXTreeUpdate& tree_update);
+};
+
+class BrowserOSGetInteractiveSnapshotFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.getInteractiveSnapshot",
+ BROWSER_OS_GETINTERACTIVESNAPSHOT)
+
+ BrowserOSGetInteractiveSnapshotFunction();
+
+ protected:
+ ~BrowserOSGetInteractiveSnapshotFunction() override;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+
+ private:
+ void OnAccessibilityTreeReceived(ui::AXTreeUpdate& tree_update);
+ void OnSnapshotProcessed(SnapshotProcessingResult result);
+
+ // Counter for snapshot IDs
+ static uint32_t next_snapshot_id_;
+
+ // Tab ID for storing mappings
+ int tab_id_ = -1;
+
+ // Viewport size for checking visibility
+ gfx::Size viewport_size_;
+};
+
+class BrowserOSClickFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.click", BROWSER_OS_CLICK)
+
+ BrowserOSClickFunction() = default;
+
+ protected:
+ ~BrowserOSClickFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSInputTextFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.inputText", BROWSER_OS_INPUTTEXT)
+
+ BrowserOSInputTextFunction() = default;
+
+ protected:
+ ~BrowserOSInputTextFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSClearFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.clear", BROWSER_OS_CLEAR)
+
+ BrowserOSClearFunction() = default;
+
+ protected:
+ ~BrowserOSClearFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSGetPageLoadStatusFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.getPageLoadStatus",
+ BROWSER_OS_GETPAGELOADSTATUS)
+
+ BrowserOSGetPageLoadStatusFunction() = default;
+
+ protected:
+ ~BrowserOSGetPageLoadStatusFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSScrollUpFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.scrollUp", BROWSER_OS_SCROLLUP)
+
+ BrowserOSScrollUpFunction() = default;
+
+ protected:
+ ~BrowserOSScrollUpFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSScrollDownFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.scrollDown", BROWSER_OS_SCROLLDOWN)
+
+ BrowserOSScrollDownFunction() = default;
+
+ protected:
+ ~BrowserOSScrollDownFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSScrollToNodeFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.scrollToNode", BROWSER_OS_SCROLLTONODE)
+
+ BrowserOSScrollToNodeFunction() = default;
+
+ protected:
+ ~BrowserOSScrollToNodeFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSSendKeysFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.sendKeys", BROWSER_OS_SENDKEYS)
+
+ BrowserOSSendKeysFunction() = default;
+
+ protected:
+ ~BrowserOSSendKeysFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+};
+
+class BrowserOSCaptureScreenshotFunction : public ExtensionFunction {
+ public:
+ DECLARE_EXTENSION_FUNCTION("browserOS.captureScreenshot", BROWSER_OS_CAPTURESCREENSHOT)
+
+ BrowserOSCaptureScreenshotFunction() = default;
+
+ protected:
+ ~BrowserOSCaptureScreenshotFunction() override = default;
+
+ // ExtensionFunction:
+ ResponseAction Run() override;
+
+ private:
+ void OnScreenshotCaptured(const SkBitmap& bitmap);
+};
+
+} // namespace api
+} // namespace extensions
+
+#endif // CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_H_
\ No newline at end of file
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_api_helpers.cc b/chrome/browser/extensions/api/browser_os/browser_os_api_helpers.cc
new file mode 100644
index 0000000000000..e66c5004440ae
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_api_helpers.cc
@@ -0,0 +1,63 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/extensions/api/browser_os/browser_os_api_helpers.h"
+
+#include "chrome/browser/extensions/extension_tab_util.h"
+#include "chrome/browser/extensions/window_controller.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/browser_finder.h"
+#include "chrome/browser/ui/tabs/tab_strip_model.h"
+#include "content/public/browser/web_contents.h"
+
+namespace extensions {
+namespace api {
+
+std::optional<TabInfo> GetTabFromOptionalId(
+ std::optional<int> tab_id_param,
+ content::BrowserContext* browser_context,
+ bool include_incognito_information,
+ std::string* error_message) {
+ content::WebContents* web_contents = nullptr;
+ int tab_id = -1;
+
+ if (tab_id_param) {
+ // Get specific tab by ID
+ WindowController* controller = nullptr;
+ int tab_index = -1;
+ if (!ExtensionTabUtil::GetTabById(*tab_id_param, browser_context,
+ include_incognito_information,
+ &controller, &web_contents,
+ &tab_index)) {
+ if (error_message) {
+ *error_message = "Tab not found";
+ }
+ return std::nullopt;
+ }
+ tab_id = *tab_id_param;
+ } else {
+ // Get active tab
+ Browser* browser = chrome::FindLastActive();
+ if (!browser) {
+ if (error_message) {
+ *error_message = "No active browser";
+ }
+ return std::nullopt;
+ }
+
+ web_contents = browser->tab_strip_model()->GetActiveWebContents();
+ if (!web_contents) {
+ if (error_message) {
+ *error_message = "No active tab";
+ }
+ return std::nullopt;
+ }
+ tab_id = ExtensionTabUtil::GetTabId(web_contents);
+ }
+
+ return TabInfo(web_contents, tab_id);
+}
+
+} // namespace api
+} // namespace extensions
\ No newline at end of file
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_api_helpers.h b/chrome/browser/extensions/api/browser_os/browser_os_api_helpers.h
new file mode 100644
index 0000000000000..142874b3d0374
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_api_helpers.h
@@ -0,0 +1,44 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_HELPERS_H_
+#define CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_HELPERS_H_
+
+#include <optional>
+#include <string>
+
+#include "base/memory/raw_ptr.h"
+
+namespace content {
+class BrowserContext;
+class WebContents;
+} // namespace content
+
+namespace extensions {
+
+class WindowController;
+
+namespace api {
+
+// Result structure for tab retrieval
+struct TabInfo {
+ raw_ptr<content::WebContents> web_contents;
+ int tab_id;
+
+ TabInfo(content::WebContents* wc, int id)
+ : web_contents(wc), tab_id(id) {}
+};
+
+// Helper to get WebContents and tab ID from optional tab_id parameter
+// Returns nullptr if tab is not found, with error message set
+std::optional<TabInfo> GetTabFromOptionalId(
+ std::optional<int> tab_id_param,
+ content::BrowserContext* browser_context,
+ bool include_incognito_information,
+ std::string* error_message);
+
+} // namespace api
+} // namespace extensions
+
+#endif // CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_HELPERS_H_
\ No newline at end of file
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_api_utils.cc b/chrome/browser/extensions/api/browser_os/browser_os_api_utils.cc
new file mode 100644
index 0000000000000..0d83f4da95ba9
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_api_utils.cc
@@ -0,0 +1,378 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/extensions/api/browser_os/browser_os_api_utils.h"
+
+#include "base/hash/hash.h"
+#include "base/no_destructor.h"
+#include "base/strings/string_number_conversions.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_widget_host.h"
+#include "content/public/browser/render_widget_host_view.h"
+#include "content/public/browser/web_contents.h"
+#include "components/input/native_web_keyboard_event.h"
+#include "third_party/blink/public/common/input/web_input_event.h"
+#include "third_party/blink/public/common/input/web_keyboard_event.h"
+#include "third_party/blink/public/common/input/web_mouse_event.h"
+#include "third_party/blink/public/common/input/web_mouse_wheel_event.h"
+#include "ui/accessibility/ax_role_properties.h"
+#include "ui/events/base_event_utils.h"
+#include "ui/events/keycodes/dom/dom_code.h"
+#include "ui/events/keycodes/dom/dom_key.h"
+#include "ui/events/keycodes/keyboard_codes.h"
+#include "ui/gfx/geometry/point_f.h"
+
+namespace extensions {
+namespace api {
+
+// Global node ID mappings storage
+// Use NoDestructor to avoid exit-time destructor
+std::unordered_map<int, std::unordered_map<uint32_t, NodeInfo>>&
+GetNodeIdMappings() {
+ static base::NoDestructor<std::unordered_map<int, std::unordered_map<uint32_t, NodeInfo>>>
+ g_node_id_mappings;
+ return *g_node_id_mappings;
+}
+
+// Helper to create and dispatch mouse events for clicking
+void PerformClick(content::WebContents* web_contents,
+ const gfx::PointF& point) {
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh)
+ return;
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh)
+ return;
+
+ content::RenderWidgetHostView* rwhv = rwh->GetView();
+ if (!rwhv)
+ return;
+
+ // Get viewport bounds for screen position calculation
+ gfx::Rect viewport_bounds = rwhv->GetViewBounds();
+ gfx::PointF viewport_origin(viewport_bounds.x(), viewport_bounds.y());
+
+ // The coordinates are already in widget space (CSS pixels)
+ gfx::PointF widget_point = point;
+
+ // Create mouse down event
+ blink::WebMouseEvent mouse_down;
+ mouse_down.SetType(blink::WebInputEvent::Type::kMouseDown);
+ mouse_down.button = blink::WebPointerProperties::Button::kLeft;
+ mouse_down.click_count = 1;
+ mouse_down.SetPositionInWidget(widget_point.x(), widget_point.y());
+ mouse_down.SetPositionInScreen(widget_point.x() + viewport_origin.x(),
+ widget_point.y() + viewport_origin.y());
+ mouse_down.SetTimeStamp(ui::EventTimeForNow());
+ mouse_down.SetModifiers(blink::WebInputEvent::kLeftButtonDown);
+
+ // Create mouse up event
+ blink::WebMouseEvent mouse_up;
+ mouse_up.SetType(blink::WebInputEvent::Type::kMouseUp);
+ mouse_up.button = blink::WebPointerProperties::Button::kLeft;
+ mouse_up.click_count = 1;
+ mouse_up.SetPositionInWidget(widget_point.x(), widget_point.y());
+ mouse_up.SetPositionInScreen(widget_point.x() + viewport_origin.x(),
+ widget_point.y() + viewport_origin.y());
+ mouse_up.SetTimeStamp(ui::EventTimeForNow());
+
+ // Send the events
+ rwh->ForwardMouseEvent(mouse_down);
+ rwh->ForwardMouseEvent(mouse_up);
+}
+
+// Helper to determine if a node is interactive (clickable/typeable/selectable)
+browser_os::InteractiveNodeType GetInteractiveNodeType(
+ const ui::AXNodeData& node_data) {
+
+ // Skip invisible or ignored nodes early
+ if (node_data.IsInvisibleOrIgnored()) {
+ return browser_os::InteractiveNodeType::kOther;
+ }
+
+ // Use built-in IsTextField() and related methods for typeable elements
+ if (node_data.IsTextField() ||
+ node_data.IsPasswordField() ||
+ node_data.IsAtomicTextField() ||
+ node_data.IsNonAtomicTextField() ||
+ node_data.IsSpinnerTextField()) {
+ return browser_os::InteractiveNodeType::kTypeable;
+ }
+
+ // Use built-in IsSelectable() for selectable elements
+ if (node_data.IsSelectable()) {
+ return browser_os::InteractiveNodeType::kSelectable;
+ }
+
+ // Use built-in IsClickable() method
+ if (node_data.IsClickable()) {
+ return browser_os::InteractiveNodeType::kClickable;
+ }
+
+ // Additional check for combobox and list options which might not be caught by IsSelectable
+ using Role = ax::mojom::Role;
+ if (node_data.role == Role::kComboBoxSelect ||
+ node_data.role == Role::kComboBoxMenuButton ||
+ node_data.role == Role::kComboBoxGrouping ||
+ node_data.role == Role::kListBox ||
+ node_data.role == Role::kListBoxOption ||
+ node_data.role == Role::kMenuListOption ||
+ node_data.role == Role::kMenuItem ||
+ node_data.role == Role::kMenuItemCheckBox ||
+ node_data.role == Role::kMenuItemRadio) {
+ return browser_os::InteractiveNodeType::kSelectable;
+ }
+
+ return browser_os::InteractiveNodeType::kOther;
+}
+
+// Helper to compute branch path hash for an AX node
+uint64_t ComputeBranchPath(const ui::AXNodeData& node_data,
+ const std::unordered_map<int32_t, ui::AXNodeData>& node_map,
+ std::unordered_map<int32_t, uint64_t>& path_cache) {
+ // Check cache first
+ auto it = path_cache.find(node_data.id);
+ if (it != path_cache.end()) {
+ return it->second;
+ }
+
+ // Build path from root to this node
+ std::vector<int32_t> path;
+ int32_t current_id = node_data.id;
+
+ while (current_id != 0) {
+ path.push_back(current_id);
+
+ // Find parent
+ bool found_parent = false;
+ for (const auto& [id, data] : node_map) {
+ if (std::find(data.child_ids.begin(), data.child_ids.end(), current_id) !=
+ data.child_ids.end()) {
+ current_id = id;
+ found_parent = true;
+ break;
+ }
+ }
+
+ if (!found_parent) {
+ break;
+ }
+ }
+
+ // Compute hash from path
+ std::string path_str;
+ for (auto path_it = path.rbegin(); path_it != path.rend(); ++path_it) {
+ path_str += base::NumberToString(*path_it) + "/";
+ }
+
+ uint64_t hash = base::PersistentHash(path_str);
+ path_cache[node_data.id] = hash;
+ return hash;
+}
+
+// Helper to get the HTML tag name from AX role
+std::string GetTagFromRole(ax::mojom::Role role) {
+ switch (role) {
+ case ax::mojom::Role::kButton:
+ return "button";
+ case ax::mojom::Role::kLink:
+ return "a";
+ case ax::mojom::Role::kTextField:
+ case ax::mojom::Role::kSearchBox:
+ return "input";
+ case ax::mojom::Role::kTextFieldWithComboBox:
+ return "input";
+ case ax::mojom::Role::kComboBoxSelect:
+ return "select";
+ case ax::mojom::Role::kCheckBox:
+ return "input";
+ case ax::mojom::Role::kRadioButton:
+ return "input";
+ case ax::mojom::Role::kImage:
+ return "img";
+ case ax::mojom::Role::kHeading:
+ return "h1"; // Could be h1-h6
+ case ax::mojom::Role::kParagraph:
+ return "p";
+ case ax::mojom::Role::kListItem:
+ return "li";
+ case ax::mojom::Role::kList:
+ return "ul";
+ case ax::mojom::Role::kForm:
+ return "form";
+ case ax::mojom::Role::kTable:
+ return "table";
+ default:
+ return "div";
+ }
+}
+
+// Helper to perform scroll actions using mouse wheel events
+void PerformScroll(content::WebContents* web_contents,
+ int delta_x,
+ int delta_y,
+ bool precise) {
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh)
+ return;
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh)
+ return;
+
+ content::RenderWidgetHostView* rwhv = rwh->GetView();
+ if (!rwhv)
+ return;
+
+ // Get viewport bounds and center point
+ gfx::Rect viewport_bounds = rwhv->GetViewBounds();
+ gfx::PointF center_point(viewport_bounds.width() / 2.0f,
+ viewport_bounds.height() / 2.0f);
+
+ // Create mouse wheel event
+ blink::WebMouseWheelEvent wheel_event;
+ wheel_event.SetType(blink::WebInputEvent::Type::kMouseWheel);
+ wheel_event.SetPositionInWidget(center_point.x(), center_point.y());
+ wheel_event.SetPositionInScreen(center_point.x() + viewport_bounds.x(),
+ center_point.y() + viewport_bounds.y());
+ wheel_event.SetTimeStamp(ui::EventTimeForNow());
+
+ // Set the scroll deltas
+ wheel_event.delta_x = delta_x;
+ wheel_event.delta_y = delta_y;
+
+ // Set wheel tick values (120 = one notch)
+ wheel_event.wheel_ticks_x = delta_x / 120.0f;
+ wheel_event.wheel_ticks_y = delta_y / 120.0f;
+
+ // Phase information for smooth scrolling
+ wheel_event.phase = blink::WebMouseWheelEvent::kPhaseBegan;
+
+ // Precise scrolling for touchpad, non-precise for mouse wheel
+ if (precise) {
+ // For precise scrolling, deltas are in pixels
+ wheel_event.delta_units = ui::ScrollGranularity::kScrollByPrecisePixel;
+ } else {
+ // For non-precise scrolling, deltas are in lines
+ wheel_event.delta_units = ui::ScrollGranularity::kScrollByLine;
+ }
+
+ // Send the wheel event
+ rwh->ForwardWheelEvent(wheel_event);
+
+ // Send phase ended event for smooth scrolling
+ wheel_event.phase = blink::WebMouseWheelEvent::kPhaseEnded;
+ wheel_event.delta_x = 0;
+ wheel_event.delta_y = 0;
+ wheel_event.wheel_ticks_x = 0;
+ wheel_event.wheel_ticks_y = 0;
+ rwh->ForwardWheelEvent(wheel_event);
+}
+
+// Helper to send special key events
+void SendSpecialKey(content::WebContents* web_contents,
+ const std::string& key) {
+ content::RenderFrameHost* rfh = web_contents->GetPrimaryMainFrame();
+ if (!rfh)
+ return;
+
+ content::RenderWidgetHost* rwh = rfh->GetRenderWidgetHost();
+ if (!rwh)
+ return;
+
+ // Map key names to Windows key codes and DOM codes/keys
+ ui::KeyboardCode windows_key_code;
+ ui::DomCode dom_code;
+ ui::DomKey dom_key;
+
+ // Use if-else chain to avoid static initialization
+ if (key == "Enter") {
+ windows_key_code = ui::VKEY_RETURN;
+ dom_code = ui::DomCode::ENTER;
+ dom_key = ui::DomKey::ENTER;
+ } else if (key == "Delete") {
+ windows_key_code = ui::VKEY_DELETE;
+ dom_code = ui::DomCode::DEL;
+ dom_key = ui::DomKey::DEL;
+ } else if (key == "Backspace") {
+ windows_key_code = ui::VKEY_BACK;
+ dom_code = ui::DomCode::BACKSPACE;
+ dom_key = ui::DomKey::BACKSPACE;
+ } else if (key == "Tab") {
+ windows_key_code = ui::VKEY_TAB;
+ dom_code = ui::DomCode::TAB;
+ dom_key = ui::DomKey::TAB;
+ } else if (key == "Escape") {
+ windows_key_code = ui::VKEY_ESCAPE;
+ dom_code = ui::DomCode::ESCAPE;
+ dom_key = ui::DomKey::ESCAPE;
+ } else if (key == "ArrowUp") {
+ windows_key_code = ui::VKEY_UP;
+ dom_code = ui::DomCode::ARROW_UP;
+ dom_key = ui::DomKey::ARROW_UP;
+ } else if (key == "ArrowDown") {
+ windows_key_code = ui::VKEY_DOWN;
+ dom_code = ui::DomCode::ARROW_DOWN;
+ dom_key = ui::DomKey::ARROW_DOWN;
+ } else if (key == "ArrowLeft") {
+ windows_key_code = ui::VKEY_LEFT;
+ dom_code = ui::DomCode::ARROW_LEFT;
+ dom_key = ui::DomKey::ARROW_LEFT;
+ } else if (key == "ArrowRight") {
+ windows_key_code = ui::VKEY_RIGHT;
+ dom_code = ui::DomCode::ARROW_RIGHT;
+ dom_key = ui::DomKey::ARROW_RIGHT;
+ } else if (key == "Home") {
+ windows_key_code = ui::VKEY_HOME;
+ dom_code = ui::DomCode::HOME;
+ dom_key = ui::DomKey::HOME;
+ } else if (key == "End") {
+ windows_key_code = ui::VKEY_END;
+ dom_code = ui::DomCode::END;
+ dom_key = ui::DomKey::END;
+ } else if (key == "PageUp") {
+ windows_key_code = ui::VKEY_PRIOR;
+ dom_code = ui::DomCode::PAGE_UP;
+ dom_key = ui::DomKey::PAGE_UP;
+ } else if (key == "PageDown") {
+ windows_key_code = ui::VKEY_NEXT;
+ dom_code = ui::DomCode::PAGE_DOWN;
+ dom_key = ui::DomKey::PAGE_DOWN;
+ } else {
+ return; // Unsupported key
+ }
+
+ // Create keyboard event
+ input::NativeWebKeyboardEvent key_down(
+ blink::WebInputEvent::Type::kKeyDown,
+ blink::WebInputEvent::kNoModifiers,
+ ui::EventTimeForNow());
+
+ key_down.windows_key_code = windows_key_code;
+ key_down.native_key_code = windows_key_code;
+ key_down.dom_code = static_cast<int>(dom_code);
+ key_down.dom_key = static_cast<int>(dom_key);
+
+ // Send key down
+ rwh->ForwardKeyboardEvent(key_down);
+
+ // For most keys, also send key up
+ if (key != "Tab") { // Tab usually doesn't need key up for focus change
+ input::NativeWebKeyboardEvent key_up(
+ blink::WebInputEvent::Type::kKeyUp,
+ blink::WebInputEvent::kNoModifiers,
+ ui::EventTimeForNow());
+
+ key_up.windows_key_code = windows_key_code;
+ key_up.native_key_code = windows_key_code;
+ key_up.dom_code = static_cast<int>(dom_code);
+ key_up.dom_key = static_cast<int>(dom_key);
+
+ rwh->ForwardKeyboardEvent(key_up);
+ }
+}
+
+} // namespace api
+} // namespace extensions
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_api_utils.h b/chrome/browser/extensions/api/browser_os/browser_os_api_utils.h
new file mode 100644
index 0000000000000..2070807877455
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_api_utils.h
@@ -0,0 +1,62 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_UTILS_H_
+#define CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_UTILS_H_
+
+#include <string>
+#include <unordered_map>
+
+#include "chrome/common/extensions/api/browser_os.h"
+#include "ui/accessibility/ax_node_data.h"
+#include "ui/gfx/geometry/rect_f.h"
+
+namespace content {
+class RenderWidgetHost;
+class WebContents;
+} // namespace content
+
+namespace extensions {
+namespace api {
+
+// Stores mapping information for a node
+struct NodeInfo {
+ int32_t ax_node_id;
+ gfx::RectF bounds; // Absolute bounds in CSS pixels
+};
+
+// Global node ID mappings storage
+std::unordered_map<int, std::unordered_map<uint32_t, NodeInfo>>&
+GetNodeIdMappings();
+
+// Helper to create and dispatch mouse events for clicking
+void PerformClick(content::WebContents* web_contents,
+ const gfx::PointF& point);
+
+// Helper to determine if a node is interactive (clickable/typable)
+browser_os::InteractiveNodeType GetInteractiveNodeType(
+ const ui::AXNodeData& node_data);
+
+// Helper to compute branch path hash for an AX node
+uint64_t ComputeBranchPath(const ui::AXNodeData& node_data,
+ const std::unordered_map<int32_t, ui::AXNodeData>& node_map,
+ std::unordered_map<int32_t, uint64_t>& path_cache);
+
+// Helper to get the HTML tag name from AX role
+std::string GetTagFromRole(ax::mojom::Role role);
+
+// Helper to perform scroll actions using mouse wheel events
+void PerformScroll(content::WebContents* web_contents,
+ int delta_x,
+ int delta_y,
+ bool precise = false);
+
+// Helper to send special key events
+void SendSpecialKey(content::WebContents* web_contents,
+ const std::string& key);
+
+} // namespace api
+} // namespace extensions
+
+#endif // CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_API_UTILS_H_
\ No newline at end of file
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.cc b/chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.cc
new file mode 100644
index 0000000000000..c84fe62d9e76d
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.cc
@@ -0,0 +1,679 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.h"
+
+#include <algorithm>
+#include <atomic>
+#include <cctype>
+#include <functional>
+#include <future>
+#include <memory>
+#include <queue>
+#include <sstream>
+#include <unordered_set>
+#include <utility>
+
+#include "base/functional/bind.h"
+#include "base/logging.h"
+#include "base/memory/raw_ptr.h"
+#include "base/memory/ref_counted.h"
+#include "base/strings/string_util.h"
+#include "base/task/thread_pool.h"
+#include "base/time/time.h"
+#include "chrome/browser/extensions/api/browser_os/browser_os_api_utils.h"
+#include "ui/accessibility/ax_enum_util.h"
+#include "ui/accessibility/ax_node_data.h"
+#include "ui/accessibility/ax_tree_id.h"
+#include "ui/accessibility/ax_tree_update.h"
+#include "ui/gfx/geometry/rect.h"
+#include "ui/gfx/geometry/rect_conversions.h"
+#include "ui/gfx/geometry/rect_f.h"
+#include "ui/gfx/geometry/transform.h"
+
+namespace extensions {
+namespace api {
+
+// Helper to compute absolute bounds from relative bounds by walking up the tree
+// If bounds_cache is provided, it will be used to cache computed bounds
+static gfx::RectF ComputeAbsoluteBoundsFromRelative(
+ const ui::AXNodeData& node_data,
+ const std::unordered_map<int32_t, ui::AXNodeData>& node_map,
+ std::unordered_map<int32_t, gfx::RectF>* bounds_cache = nullptr) {
+ // Check cache first if provided
+ if (bounds_cache) {
+ auto cache_it = bounds_cache->find(node_data.id);
+ if (cache_it != bounds_cache->end()) {
+ return cache_it->second;
+ }
+ }
+ // Compute absolute bounds by walking up the tree
+ gfx::RectF absolute_bounds = node_data.relative_bounds.bounds;
+ gfx::Transform accumulated_transform;
+
+ // Apply this node's transform if it has one
+ if (node_data.relative_bounds.transform) {
+ accumulated_transform = *node_data.relative_bounds.transform;
+ }
+
+ // Walk up the tree to compute absolute position
+ int32_t current_container_id = node_data.relative_bounds.offset_container_id;
+ int walk_depth = 0;
+
+ while (current_container_id >= 0 && walk_depth < 100) { // Prevent infinite loops
+ auto container_it = node_map.find(current_container_id);
+ if (container_it == node_map.end()) {
+ break;
+ }
+
+ const ui::AXNodeData& container = container_it->second;
+
+ // Offset by container's position
+ absolute_bounds.Offset(container.relative_bounds.bounds.x(),
+ container.relative_bounds.bounds.y());
+
+ // Apply container's transform if any
+ if (container.relative_bounds.transform) {
+ gfx::Transform container_transform = *container.relative_bounds.transform;
+ container_transform.PostConcat(accumulated_transform);
+ accumulated_transform = container_transform;
+ }
+
+ // Account for scroll offset if container has it
+ if (container.HasIntAttribute(ax::mojom::IntAttribute::kScrollX) ||
+ container.HasIntAttribute(ax::mojom::IntAttribute::kScrollY)) {
+ int scroll_x = container.GetIntAttribute(ax::mojom::IntAttribute::kScrollX);
+ int scroll_y = container.GetIntAttribute(ax::mojom::IntAttribute::kScrollY);
+ absolute_bounds.Offset(-scroll_x, -scroll_y);
+ }
+
+ // Move to next container
+ current_container_id = container.relative_bounds.offset_container_id;
+ walk_depth++;
+ }
+
+ // Apply accumulated transform
+ if (!accumulated_transform.IsIdentity()) {
+ absolute_bounds = accumulated_transform.MapRect(absolute_bounds);
+ }
+
+ // Store in cache if provided
+ if (bounds_cache) {
+ (*bounds_cache)[node_data.id] = absolute_bounds;
+ }
+
+ return absolute_bounds;
+}
+
+// ProcessedNode implementation
+SnapshotProcessor::ProcessedNode::ProcessedNode()
+ : node_data(nullptr), node_id(0) {}
+
+SnapshotProcessor::ProcessedNode::ProcessedNode(const ProcessedNode&) = default;
+SnapshotProcessor::ProcessedNode::ProcessedNode(ProcessedNode&&) = default;
+SnapshotProcessor::ProcessedNode&
+SnapshotProcessor::ProcessedNode::operator=(const ProcessedNode&) = default;
+SnapshotProcessor::ProcessedNode&
+SnapshotProcessor::ProcessedNode::operator=(ProcessedNode&&) = default;
+SnapshotProcessor::ProcessedNode::~ProcessedNode() = default;
+
+
+namespace {
+
+// Check if a node should create a section (container roles)
+[[maybe_unused]]
+bool IsContainer(ax::mojom::Role role) {
+ return role == ax::mojom::Role::kMain ||
+ role == ax::mojom::Role::kArticle ||
+ role == ax::mojom::Role::kSection ||
+ role == ax::mojom::Role::kNavigation ||
+ role == ax::mojom::Role::kForm ||
+ role == ax::mojom::Role::kDialog ||
+ role == ax::mojom::Role::kSearch ||
+ role == ax::mojom::Role::kRegion ||
+ role == ax::mojom::Role::kBanner || // header
+ role == ax::mojom::Role::kContentInfo || // footer
+ role == ax::mojom::Role::kComplementary || // aside
+ role == ax::mojom::Role::kHeading ||
+ role == ax::mojom::Role::kList ||
+ role == ax::mojom::Role::kListItem || // product cards
+ role == ax::mojom::Role::kGrid ||
+ role == ax::mojom::Role::kTable;
+}
+
+// Get a readable name for the section
+[[maybe_unused]]
+std::string GetSectionName(const ui::AXNodeData& node) {
+ std::string name;
+ if (node.HasStringAttribute(ax::mojom::StringAttribute::kName)) {
+ name = node.GetStringAttribute(ax::mojom::StringAttribute::kName);
+ }
+
+ // Add role description
+ std::string role_desc;
+ switch (node.role) {
+ case ax::mojom::Role::kNavigation:
+ role_desc = "navigation";
+ break;
+ case ax::mojom::Role::kMain:
+ role_desc = "main";
+ break;
+ case ax::mojom::Role::kBanner:
+ role_desc = "header";
+ break;
+ case ax::mojom::Role::kContentInfo:
+ role_desc = "footer";
+ break;
+ case ax::mojom::Role::kForm:
+ role_desc = "form";
+ break;
+ case ax::mojom::Role::kList:
+ role_desc = "list";
+ break;
+ case ax::mojom::Role::kListItem:
+ role_desc = "listitem";
+ break;
+ case ax::mojom::Role::kHeading:
+ role_desc = "heading";
+ break;
+ default:
+ role_desc = "section"; // Generic fallback
+ }
+
+ if (!name.empty()) {
+ return name + " (" + role_desc + ")";
+ } else {
+ // Capitalize first letter
+ if (!role_desc.empty()) {
+ role_desc[0] = std::toupper(role_desc[0]);
+ }
+ return role_desc;
+ }
+}
+
+// Helper to sanitize strings to ensure valid UTF-8 by keeping only printable ASCII
+std::string SanitizeStringForOutput(const std::string& input) {
+ std::string output;
+ output.reserve(input.size());
+
+ for (char c : input) {
+ // Only include printable ASCII and whitespace
+ if ((c >= 32 && c <= 126) || c == '\t' || c == '\n') {
+ output.push_back(c);
+ } else {
+ output.push_back(' '); // Replace non-printable with space
+ }
+ }
+
+ return output;
+}
+
+// Helper to determine if a node should be skipped for the interactive snapshot
+bool ShouldSkipNode(const ui::AXNodeData& node_data) {
+ // Skip invisible or ignored nodes
+ if (node_data.IsInvisibleOrIgnored()) {
+ return true;
+ }
+
+ // Get the interactive type and skip if it's not interactive
+ browser_os::InteractiveNodeType node_type = GetInteractiveNodeType(node_data);
+ if (node_type == browser_os::InteractiveNodeType::kOther) {
+ return true;
+ }
+
+ return false;
+}
+
+} // namespace
+
+// Internal structure for managing async processing
+struct SnapshotProcessor::ProcessingContext
+ : public base::RefCountedThreadSafe<ProcessingContext> {
+ browser_os::InteractiveSnapshot snapshot;
+ std::unordered_map<int32_t, ui::AXNodeData> node_map;
+ std::unordered_map<int32_t, int32_t> parent_map; // child_id -> parent_id
+ std::unordered_map<int32_t, std::vector<int32_t>> children_map; // parent_id -> child_ids
+ int tab_id;
+ base::TimeTicks start_time;
+ size_t total_nodes;
+ size_t processed_batches;
+ size_t total_batches;
+ gfx::Rect viewport_bounds;
+ base::OnceCallback<void(SnapshotProcessingResult)> callback;
+
+ private:
+ friend class base::RefCountedThreadSafe<ProcessingContext>;
+ ~ProcessingContext() = default;
+};
+
+// Helper to collect text from a node's subtree
+std::string CollectTextFromNode(
+ int32_t node_id,
+ const std::unordered_map<int32_t, ui::AXNodeData>& node_map,
+ int max_chars = 200) {
+
+ auto node_it = node_map.find(node_id);
+ if (node_it == node_map.end()) {
+ return "";
+ }
+
+ std::vector<std::string> text_parts;
+
+ // BFS to collect text from this node and its children
+ std::queue<int32_t> queue;
+ queue.push(node_id);
+ int chars_collected = 0;
+
+ while (!queue.empty() && chars_collected < max_chars) {
+ int32_t current_id = queue.front();
+ queue.pop();
+
+ auto current_it = node_map.find(current_id);
+ if (current_it == node_map.end()) continue;
+
+ const ui::AXNodeData& current = current_it->second;
+
+ // Collect text from this node
+ if (current.HasStringAttribute(ax::mojom::StringAttribute::kName)) {
+ std::string text = current.GetStringAttribute(ax::mojom::StringAttribute::kName);
+ text = std::string(base::TrimWhitespaceASCII(text, base::TRIM_ALL));
+ if (!text.empty()) {
+ std::string clean_text = SanitizeStringForOutput(text);
+ if (!clean_text.empty()) {
+ text_parts.push_back(clean_text);
+ chars_collected += clean_text.length();
+ }
+ }
+ }
+
+ // Add children to queue
+ for (int32_t child_id : current.child_ids) {
+ queue.push(child_id);
+ }
+ }
+
+ std::string result = base::JoinString(text_parts, " ");
+ if (result.length() > static_cast<size_t>(max_chars)) {
+ result = result.substr(0, max_chars - 3) + "...";
+ }
+ return result;
+}
+
+// Helper to build path using offset_container_id and return depth
+std::pair<std::string, int> BuildPathAndDepth(
+ int32_t node_id,
+ const std::unordered_map<int32_t, ui::AXNodeData>& node_map) {
+
+ std::vector<std::string> path_parts;
+ int32_t current_id = node_id;
+ int depth = 0;
+ const int max_depth = 10;
+
+ while (current_id >= 0 && depth < max_depth) {
+ auto node_it = node_map.find(current_id);
+ if (node_it == node_map.end()) break;
+
+ const ui::AXNodeData& node = node_it->second;
+
+ // Just append the role
+ path_parts.push_back(ui::ToString(node.role));
+
+ // Move to offset container
+ current_id = node.relative_bounds.offset_container_id;
+ depth++;
+ }
+
+ // Reverse to get top-down path
+ std::reverse(path_parts.begin(), path_parts.end());
+ return std::make_pair(base::JoinString(path_parts, " > "), depth);
+}
+
+// Helper to populate all attributes for a node
+void PopulateNodeAttributes(
+ const ui::AXNodeData& node_data,
+ std::unordered_map<std::string, std::string>& attributes) {
+
+ // Add role as string
+ attributes["role"] = ui::ToString(node_data.role);
+
+ // Add value attribute for inputs
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kValue)) {
+ std::string value = node_data.GetStringAttribute(ax::mojom::StringAttribute::kValue);
+ attributes["value"] = SanitizeStringForOutput(value);
+ }
+
+ // Add HTML tag if available
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kHtmlTag)) {
+ attributes["html-tag"] = node_data.GetStringAttribute(ax::mojom::StringAttribute::kHtmlTag);
+ }
+
+ // Add role description
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kRoleDescription)) {
+ std::string role_desc = node_data.GetStringAttribute(ax::mojom::StringAttribute::kRoleDescription);
+ attributes["role-description"] = SanitizeStringForOutput(role_desc);
+ }
+
+ // Add input type
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kInputType)) {
+ std::string input_type = node_data.GetStringAttribute(ax::mojom::StringAttribute::kInputType);
+ attributes["input-type"] = SanitizeStringForOutput(input_type);
+ }
+
+ // Add tooltip
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kTooltip)) {
+ std::string tooltip = node_data.GetStringAttribute(ax::mojom::StringAttribute::kTooltip);
+ attributes["tooltip"] = SanitizeStringForOutput(tooltip);
+ }
+
+ // Add placeholder for input fields
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kPlaceholder)) {
+ std::string placeholder = node_data.GetStringAttribute(ax::mojom::StringAttribute::kPlaceholder);
+ attributes["placeholder"] = SanitizeStringForOutput(placeholder);
+ }
+
+ // Add description for more context
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kDescription)) {
+ std::string description = node_data.GetStringAttribute(ax::mojom::StringAttribute::kDescription);
+ attributes["description"] = SanitizeStringForOutput(description);
+ }
+
+ // Add URL for links
+ // if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kUrl)) {
+ // std::string url = node_data.GetStringAttribute(ax::mojom::StringAttribute::kUrl);
+ // attributes["url"] = SanitizeStringForOutput(url);
+ // }
+
+ // Add checked state description
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kCheckedStateDescription)) {
+ std::string checked_desc = node_data.GetStringAttribute(ax::mojom::StringAttribute::kCheckedStateDescription);
+ attributes["checked-state"] = SanitizeStringForOutput(checked_desc);
+ }
+
+ // Add autocomplete hint
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kAutoComplete)) {
+ std::string autocomplete = node_data.GetStringAttribute(ax::mojom::StringAttribute::kAutoComplete);
+ attributes["autocomplete"] = SanitizeStringForOutput(autocomplete);
+ }
+
+ // Add HTML ID for form associations
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kHtmlId)) {
+ std::string html_id = node_data.GetStringAttribute(ax::mojom::StringAttribute::kHtmlId);
+ attributes["id"] = SanitizeStringForOutput(html_id);
+ }
+}
+
+// Process a batch of nodes
+std::vector<SnapshotProcessor::ProcessedNode> SnapshotProcessor::ProcessNodeBatch(
+ const std::vector<ui::AXNodeData>& nodes_to_process,
+ const std::unordered_map<int32_t, ui::AXNodeData>& node_map,
+ uint32_t start_node_id,
+ const gfx::Rect& doc_viewport_bounds) {
+ std::vector<ProcessedNode> results;
+ results.reserve(nodes_to_process.size());
+
+ // Local caches for this batch
+ std::unordered_map<int32_t, gfx::RectF> bounds_cache;
+ std::unordered_map<int32_t, uint64_t> path_cache;
+
+ uint32_t current_node_id = start_node_id;
+
+ for (const auto& node_data : nodes_to_process) {
+ // Skip invisible, ignored, or non-interactive elements
+ if (ShouldSkipNode(node_data)) {
+ continue;
+ }
+
+ // Double-check invisibility (already done in ShouldSkipNode, but being explicit)
+ if (node_data.IsInvisibleOrIgnored()) {
+ continue;
+ }
+
+ // Get the interactive node type
+ browser_os::InteractiveNodeType node_type = GetInteractiveNodeType(node_data);
+
+ ProcessedNode data;
+ data.node_data = &node_data;
+ data.node_id = current_node_id++;
+ data.node_type = node_type;
+
+ // Get accessible name
+ if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kName)) {
+ std::string name = node_data.GetStringAttribute(ax::mojom::StringAttribute::kName);
+ data.name = SanitizeStringForOutput(name);
+ }
+
+ // Compute absolute bounds with caching
+ data.absolute_bounds = ComputeAbsoluteBoundsFromRelative(
+ node_data, node_map, &bounds_cache);
+
+ // Populate all attributes using helper function
+ PopulateNodeAttributes(node_data, data.attributes);
+
+ // Add context from parent node
+ int32_t parent_id = node_data.relative_bounds.offset_container_id;
+ if (parent_id >= 0) {
+ std::string context = CollectTextFromNode(parent_id, node_map, 200);
+ if (!context.empty()) {
+ data.attributes["context"] = context;
+ }
+ }
+
+ // Add path and depth using offset_container_id chain
+ auto [path, depth] = BuildPathAndDepth(node_data.id, node_map);
+ if (!path.empty()) {
+ data.attributes["path"] = path;
+ }
+ data.attributes["depth"] = std::to_string(depth);
+
+ // Check if node is in viewport
+ bool in_viewport = false;
+ if (!doc_viewport_bounds.IsEmpty()) {
+ // Convert absolute bounds to integer rect for intersection test
+ gfx::Rect node_rect = gfx::ToEnclosingRect(data.absolute_bounds);
+ in_viewport = doc_viewport_bounds.Intersects(node_rect);
+ }
+ data.attributes["in_viewport"] = in_viewport ? "true" : "false";
+
+ results.push_back(std::move(data));
+ }
+
+ return results;
+}
+
+// Helper to handle batch processing results
+void SnapshotProcessor::OnBatchProcessed(
+ scoped_refptr<ProcessingContext> context,
+ std::vector<ProcessedNode> batch_results) {
+ // Process batch results
+ for (const auto& node_data : batch_results) {
+ // Store mapping from our nodeId to AX node ID and bounds
+ NodeInfo info;
+ info.ax_node_id = node_data.node_data->id;
+ info.bounds = node_data.absolute_bounds;
+ GetNodeIdMappings()[context->tab_id][node_data.node_id] = info;
+
+ // Log the mapping for debugging
+ VLOG(2) << "Node ID Mapping: Interactive nodeId=" << node_data.node_id
+ << " -> AX node ID=" << info.ax_node_id
+ << " (name: " << node_data.name << ")";
+
+ // Create interactive node
+ browser_os::InteractiveNode interactive_node;
+ interactive_node.node_id = node_data.node_id;
+ interactive_node.type = node_data.node_type;
+ interactive_node.name = node_data.name;
+
+ // Set the bounding rectangle
+ browser_os::Rect rect;
+ rect.x = node_data.absolute_bounds.x();
+ rect.y = node_data.absolute_bounds.y();
+ rect.width = node_data.absolute_bounds.width();
+ rect.height = node_data.absolute_bounds.height();
+ interactive_node.rect = std::move(rect);
+
+ // Create attributes dictionary by iterating over all key-value pairs
+ if (!node_data.attributes.empty()) {
+ browser_os::InteractiveNode::Attributes attributes;
+
+ // Iterate over all attributes and add them to the dictionary
+ for (const auto& [key, value] : node_data.attributes) {
+ attributes.additional_properties.Set(key, value);
+ }
+
+ interactive_node.attributes = std::move(attributes);
+ }
+
+ context->snapshot.elements.push_back(std::move(interactive_node));
+ }
+
+ context->processed_batches++;
+
+ // Check if all batches are complete
+ if (context->processed_batches == context->total_batches) {
+ // Sort elements by node_id to maintain consistent ordering
+ std::sort(context->snapshot.elements.begin(),
+ context->snapshot.elements.end(),
+ [](const browser_os::InteractiveNode& a,
+ const browser_os::InteractiveNode& b) {
+ return a.node_id < b.node_id;
+ });
+
+ // Leave hierarchical_structure empty for now as requested
+ context->snapshot.hierarchical_structure = "";
+
+ base::TimeDelta processing_time = base::TimeTicks::Now() - context->start_time;
+ LOG(INFO) << "[PERF] Interactive snapshot processed in "
+ << processing_time.InMilliseconds() << " ms"
+ << " (nodes: " << context->snapshot.elements.size() << ")";
+
+ // Set processing time in the snapshot
+ context->snapshot.processing_time_ms = processing_time.InMilliseconds();
+
+ SnapshotProcessingResult result;
+ result.snapshot = std::move(context->snapshot);
+ result.nodes_processed = context->total_nodes;
+ result.processing_time_ms = processing_time.InMilliseconds();
+
+ // Run callback (context will be deleted when last ref is released)
+ std::move(context->callback).Run(std::move(result));
+ }
+}
+
+// Main processing function
+void SnapshotProcessor::ProcessAccessibilityTree(
+ const ui::AXTreeUpdate& tree_update,
+ int tab_id,
+ uint32_t snapshot_id,
+ const gfx::Size& viewport_size,
+ base::OnceCallback<void(SnapshotProcessingResult)> callback) {
+ base::TimeTicks start_time = base::TimeTicks::Now();
+
+
+ // Build node ID map, parent map and children map for efficient lookup
+ std::unordered_map<int32_t, ui::AXNodeData> node_map;
+ std::unordered_map<int32_t, int32_t> parent_map;
+ std::unordered_map<int32_t, std::vector<int32_t>> children_map;
+
+ for (const auto& node : tree_update.nodes) {
+ node_map[node.id] = node;
+ // Build parent and children relationships
+ for (int32_t child_id : node.child_ids) {
+ parent_map[child_id] = node.id;
+ children_map[node.id].push_back(child_id);
+ }
+ }
+
+ // Prepare processing context using RefCounted
+ auto context = base::MakeRefCounted<ProcessingContext>();
+ context->snapshot.snapshot_id = snapshot_id;
+ context->snapshot.timestamp = base::Time::Now().InMillisecondsFSinceUnixEpoch();
+ context->tab_id = tab_id;
+ context->node_map = std::move(node_map);
+ context->parent_map = std::move(parent_map);
+ context->children_map = std::move(children_map);
+ context->start_time = start_time;
+
+ // Convert viewport size to document viewport bounds
+ // Find the root node and get its scroll offset
+ gfx::Rect doc_viewport_bounds;
+ if (!viewport_size.IsEmpty() && tree_update.has_tree_data && tree_update.root_id != 0) {
+ auto root_it = node_map.find(tree_update.root_id);
+ if (root_it != node_map.end()) {
+ const ui::AXNodeData& root_node = root_it->second;
+ int scroll_x = root_node.GetIntAttribute(ax::mojom::IntAttribute::kScrollX);
+ int scroll_y = root_node.GetIntAttribute(ax::mojom::IntAttribute::kScrollY);
+
+ // Create viewport in document coordinates
+ // Position is based on scroll offset, size is the visible viewport size
+ doc_viewport_bounds = gfx::Rect(scroll_x, scroll_y,
+ viewport_size.width(),
+ viewport_size.height());
+
+ LOG(INFO) << "Viewport size: " << viewport_size.ToString();
+ LOG(INFO) << "Root scroll offset: (" << scroll_x << ", " << scroll_y << ")";
+ LOG(INFO) << "Document viewport bounds: " << doc_viewport_bounds.ToString();
+ }
+ }
+
+ context->viewport_bounds = doc_viewport_bounds;
+ context->callback = std::move(callback);
+ context->processed_batches = 0;
+
+ // Clear previous mappings for this tab
+ GetNodeIdMappings()[tab_id].clear();
+
+ // Collect all nodes to process and filter
+ std::vector<ui::AXNodeData> nodes_to_process;
+ for (const auto& node : tree_update.nodes) {
+ // Skip invisible, ignored, or non-interactive nodes
+ if (ShouldSkipNode(node)) {
+ continue;
+ }
+ nodes_to_process.push_back(node);
+ }
+
+ context->total_nodes = nodes_to_process.size();
+
+ // Handle empty case
+ if (nodes_to_process.empty()) {
+ base::TimeDelta processing_time = base::TimeTicks::Now() - start_time;
+ context->snapshot.processing_time_ms = processing_time.InMilliseconds();
+
+ SnapshotProcessingResult result;
+ result.snapshot = std::move(context->snapshot);
+ result.nodes_processed = 0;
+ result.processing_time_ms = processing_time.InMilliseconds();
+ std::move(context->callback).Run(std::move(result));
+ return;
+ }
+
+ // Process nodes in batches using ThreadPool
+ const size_t batch_size = 100; // Process 100 nodes per batch
+ size_t num_batches = (nodes_to_process.size() + batch_size - 1) / batch_size;
+ context->total_batches = num_batches;
+
+ for (size_t i = 0; i < nodes_to_process.size(); i += batch_size) {
+ size_t end = std::min(i + batch_size, nodes_to_process.size());
+ std::vector<ui::AXNodeData> batch(
+ std::make_move_iterator(nodes_to_process.begin() + i),
+ std::make_move_iterator(nodes_to_process.begin() + end));
+ uint32_t start_node_id = i + 1; // Node IDs start at 1
+
+ // Post task to ThreadPool and handle result on UI thread
+ base::ThreadPool::PostTaskAndReplyWithResult(
+ FROM_HERE,
+ {base::TaskPriority::USER_VISIBLE},
+ base::BindOnce(&SnapshotProcessor::ProcessNodeBatch,
+ std::move(batch),
+ context->node_map,
+ start_node_id,
+ context->viewport_bounds),
+ base::BindOnce(&SnapshotProcessor::OnBatchProcessed,
+ context));
+ }
+}
+
+
+} // namespace api
+} // namespace extensions
diff --git a/chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.h b/chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.h
new file mode 100644
index 0000000000000..5e1114c40fe89
--- /dev/null
+++ b/chrome/browser/extensions/api/browser_os/browser_os_snapshot_processor.h
@@ -0,0 +1,89 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_SNAPSHOT_PROCESSOR_H_
+#define CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_SNAPSHOT_PROCESSOR_H_
+
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "base/functional/callback.h"
+#include "base/memory/raw_ptr.h"
+#include "chrome/common/extensions/api/browser_os.h"
+#include "ui/gfx/geometry/rect_f.h"
+
+namespace ui {
+struct AXNodeData;
+struct AXTreeUpdate;
+} // namespace ui
+
+namespace extensions {
+namespace api {
+
+// Result of snapshot processing
+struct SnapshotProcessingResult {
+ browser_os::InteractiveSnapshot snapshot;
+ int nodes_processed = 0;
+ int64_t processing_time_ms = 0;
+};
+
+// Processes accessibility trees into interactive snapshots with parallel processing
+class SnapshotProcessor {
+ public:
+ // Structure to hold data for a processed node
+ struct ProcessedNode {
+ ProcessedNode();
+ ProcessedNode(const ProcessedNode&);
+ ProcessedNode(ProcessedNode&&);
+ ProcessedNode& operator=(const ProcessedNode&);
+ ProcessedNode& operator=(ProcessedNode&&);
+ ~ProcessedNode();
+
+ raw_ptr<const ui::AXNodeData> node_data;
+ uint32_t node_id;
+ browser_os::InteractiveNodeType node_type;
+ std::string name;
+ gfx::RectF absolute_bounds;
+ // All attributes stored as key-value pairs
+ std::unordered_map<std::string, std::string> attributes;
+ };
+
+ SnapshotProcessor() = default;
+ ~SnapshotProcessor() = default;
+
+ // Main processing function - handles all threading internally
+ // This function processes the accessibility tree into an interactive snapshot
+ // using parallel processing on the thread pool.
+ static void ProcessAccessibilityTree(
+ const ui::AXTreeUpdate& tree_update,
+ int tab_id,
+ uint32_t snapshot_id,
+ const gfx::Size& viewport_size,
+ base::OnceCallback<void(SnapshotProcessingResult)> callback);
+
+ // Process a batch of nodes (exposed for testing)
+ static std::vector<ProcessedNode> ProcessNodeBatch(
+ const std::vector<ui::AXNodeData>& nodes_to_process,
+ const std::unordered_map<int32_t, ui::AXNodeData>& node_map,
+ uint32_t start_node_id,
+ const gfx::Rect& doc_viewport_bounds);
+
+ private:
+ // Internal processing context
+ struct ProcessingContext;
+
+ // Batch processing callback
+ static void OnBatchProcessed(scoped_refptr<ProcessingContext> context,
+ std::vector<ProcessedNode> batch_results);
+
+ SnapshotProcessor(const SnapshotProcessor&) = delete;
+ SnapshotProcessor& operator=(const SnapshotProcessor&) = delete;
+};
+
+} // namespace api
+} // namespace extensions
+
+#endif // CHROME_BROWSER_EXTENSIONS_API_BROWSER_OS_BROWSER_OS_SNAPSHOT_PROCESSOR_H_
\ No newline at end of file
diff --git a/chrome/browser/extensions/chrome_extensions_browser_api_provider.cc b/chrome/browser/extensions/chrome_extensions_browser_api_provider.cc
index 9c73fc6067b2f..6b3227c786686 100644
--- a/chrome/browser/extensions/chrome_extensions_browser_api_provider.cc
+++ b/chrome/browser/extensions/chrome_extensions_browser_api_provider.cc
@@ -4,6 +4,7 @@
#include "chrome/browser/extensions/chrome_extensions_browser_api_provider.h"
+#include "chrome/browser/extensions/api/browser_os/browser_os_api.h"
#include "chrome/browser/extensions/api/generated_api_registration.h"
#include "extensions/browser/extension_function_registry.h"
#include "extensions/buildflags/buildflags.h"
@@ -21,6 +22,13 @@ void ChromeExtensionsBrowserAPIProvider::RegisterExtensionFunctions(
// Commands
registry->RegisterFunction<GetAllCommandsFunction>();
+ // Browser OS API
+ registry->RegisterFunction<api::BrowserOSGetAccessibilityTreeFunction>();
+ registry->RegisterFunction<api::BrowserOSGetInteractiveSnapshotFunction>();
+ registry->RegisterFunction<api::BrowserOSClickFunction>();
+ registry->RegisterFunction<api::BrowserOSInputTextFunction>();
+ registry->RegisterFunction<api::BrowserOSClearFunction>();
+
// Generated APIs from Chrome.
api::ChromeGeneratedFunctionRegistry::RegisterAll(registry);
}
diff --git a/chrome/common/extensions/api/_api_features.json b/chrome/common/extensions/api/_api_features.json
index 846a910323217..e78e1125f61cd 100644
--- a/chrome/common/extensions/api/_api_features.json
+++ b/chrome/common/extensions/api/_api_features.json
@@ -179,6 +179,34 @@
],
"contexts": ["privileged_extension"]
}],
+ "browserOS": {
+ "dependencies": ["permission:browserOS"],
+ "contexts": ["privileged_extension"]
+ },
+ "browserOS.getAccessibilityTree": {
+ "dependencies": ["permission:browserOS"],
+ "contexts": ["privileged_extension"]
+ },
+ "browserOS.getInteractiveSnapshot": {
+ "dependencies": ["permission:browserOS"],
+ "contexts": ["privileged_extension"]
+ },
+ "browserOS.getPageStructure": {
+ "dependencies": ["permission:browserOS"],
+ "contexts": ["privileged_extension"]
+ },
+ "browserOS.click": {
+ "dependencies": ["permission:browserOS"],
+ "contexts": ["privileged_extension"]
+ },
+ "browserOS.inputText": {
+ "dependencies": ["permission:browserOS"],
+ "contexts": ["privileged_extension"]
+ },
+ "browserOS.clear": {
+ "dependencies": ["permission:browserOS"],
+ "contexts": ["privileged_extension"]
+ },
"browsingData": {
"dependencies": ["permission:browsingData"],
"contexts": ["privileged_extension"]
diff --git a/chrome/common/extensions/api/_permission_features.json b/chrome/common/extensions/api/_permission_features.json
index 93ae24f9a0972..01742801dc6b5 100644
--- a/chrome/common/extensions/api/_permission_features.json
+++ b/chrome/common/extensions/api/_permission_features.json
@@ -91,6 +91,10 @@
"extension_types": ["extension", "legacy_packaged_app", "platform_app"],
"location": "component"
},
+ "browserOS": {
+ "channel": "stable",
+ "extension_types": ["extension", "platform_app"]
+ },
"browsingData": {
"channel": "stable",
"extension_types": ["extension", "legacy_packaged_app"]
diff --git a/chrome/common/extensions/api/api_sources.gni b/chrome/common/extensions/api/api_sources.gni
index cb1b525b39038..c57c8b74785f0 100644
--- a/chrome/common/extensions/api/api_sources.gni
+++ b/chrome/common/extensions/api/api_sources.gni
@@ -48,6 +48,7 @@ if (enable_extensions) {
"bookmark_manager_private.json",
"bookmarks.json",
"braille_display_private.idl",
+ "browser_os.idl",
"chrome_web_view_internal.json",
"command_line_private.json",
"content_settings.json",
diff --git a/chrome/common/extensions/api/browser_os.idl b/chrome/common/extensions/api/browser_os.idl
new file mode 100644
index 0000000000000..f088a5d1041f0
--- /dev/null
+++ b/chrome/common/extensions/api/browser_os.idl
@@ -0,0 +1,194 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// browserOS API for accessing system-level browser functionality
+namespace browserOS {
+ dictionary AccessibilityNode {
+ long id;
+ DOMString role;
+ DOMString? name;
+ DOMString? value;
+ object? attributes;
+ long[]? childIds;
+ };
+
+ dictionary AccessibilityTree {
+ // The ID of the root node
+ long rootId;
+
+ // Map of node IDs to AccessibilityNode objects
+ object nodes;
+ };
+
+ // Interactive element types
+ enum InteractiveNodeType {
+ clickable,
+ typeable,
+ selectable,
+ other
+ };
+
+ // Rectangle bounds
+ dictionary Rect {
+ double x;
+ double y;
+ double width;
+ double height;
+ };
+
+ // Interactive node in the snapshot
+ dictionary InteractiveNode {
+ long nodeId;
+ InteractiveNodeType type;
+ DOMString? name;
+ // Bounding rectangle of the node
+ Rect? rect;
+ // Flexible attributes dictionary for extensibility
+ // Can include: tag, axValue, htmlTag, role, context, path, and any future attributes
+ object? attributes;
+ };
+
+ // Snapshot of interactive elements
+ dictionary InteractiveSnapshot {
+ long snapshotId;
+ double timestamp;
+ InteractiveNode[] elements;
+ // Hierarchical text representation with context
+ DOMString? hierarchicalStructure;
+ // Performance metrics
+ long processingTimeMs;
+ };
+
+ // Options for getInteractiveSnapshot
+ dictionary InteractiveSnapshotOptions {
+ boolean? viewportOnly;
+ };
+
+ // Page load status information
+ dictionary PageLoadStatus {
+ boolean isResourcesLoading;
+ boolean isDOMContentLoaded;
+ boolean isPageComplete;
+ };
+
+
+ callback GetAccessibilityTreeCallback = void(AccessibilityTree tree);
+ callback GetInteractiveSnapshotCallback = void(InteractiveSnapshot snapshot);
+ callback ClickCallback = void();
+ callback InputTextCallback = void();
+ callback ClearCallback = void();
+ callback GetPageLoadStatusCallback = void(PageLoadStatus status);
+ callback ScrollCallback = void();
+ callback ScrollToNodeCallback = void(boolean scrolled);
+ callback SendKeysCallback = void();
+ callback CaptureScreenshotCallback = void(DOMString dataUrl);
+
+ interface Functions {
+ // Gets the full accessibility tree for a tab
+ // |tabId|: The tab to get the accessibility tree for. Defaults to active tab.
+ // |callback|: Called with the accessibility tree data.
+ static void getAccessibilityTree(
+ optional long tabId,
+ GetAccessibilityTreeCallback callback);
+
+ // Gets a snapshot of interactive elements on the page
+ // |tabId|: The tab to get the snapshot for. Defaults to active tab.
+ // |options|: Options for the snapshot.
+ // |callback|: Called with the interactive snapshot data.
+ static void getInteractiveSnapshot(
+ optional long tabId,
+ optional InteractiveSnapshotOptions options,
+ GetInteractiveSnapshotCallback callback);
+
+
+ // Clicks on an element by its nodeId from the interactive snapshot
+ // |tabId|: The tab containing the element. Defaults to active tab.
+ // |nodeId|: The nodeId from the interactive snapshot.
+ // |callback|: Called when the click is complete.
+ static void click(
+ optional long tabId,
+ long nodeId,
+ ClickCallback callback);
+
+ // Inputs text into an element by its nodeId
+ // |tabId|: The tab containing the element. Defaults to active tab.
+ // |nodeId|: The nodeId from the interactive snapshot.
+ // |text|: The text to input.
+ // |callback|: Called when the input is complete.
+ static void inputText(
+ optional long tabId,
+ long nodeId,
+ DOMString text,
+ InputTextCallback callback);
+
+ // Clears the content of an input element by its nodeId
+ // |tabId|: The tab containing the element. Defaults to active tab.
+ // |nodeId|: The nodeId from the interactive snapshot.
+ // |callback|: Called when the clear is complete.
+ static void clear(
+ optional long tabId,
+ long nodeId,
+ ClearCallback callback);
+
+ // Gets the page load status for a tab
+ // |tabId|: The tab to check. Defaults to active tab.
+ // |callback|: Called with the page load status.
+ static void getPageLoadStatus(
+ optional long tabId,
+ GetPageLoadStatusCallback callback);
+
+ // Scrolls the page up by approximately one viewport height
+ // |tabId|: The tab to scroll. Defaults to active tab.
+ // |callback|: Called when the scroll is complete.
+ static void scrollUp(
+ optional long tabId,
+ ScrollCallback callback);
+
+ // Scrolls the page down by approximately one viewport height
+ // |tabId|: The tab to scroll. Defaults to active tab.
+ // |callback|: Called when the scroll is complete.
+ static void scrollDown(
+ optional long tabId,
+ ScrollCallback callback);
+
+ // Scrolls the page to bring the specified node into view
+ // |tabId|: The tab to scroll. Defaults to active tab.
+ // |nodeId|: The node ID from getInteractiveSnapshot to scroll to.
+ // |callback|: Called with whether scrolling was needed (false if already in view).
+ static void scrollToNode(
+ optional long tabId,
+ long nodeId,
+ ScrollToNodeCallback callback);
+
+ // Sends special key events to the active element in a tab
+ // |tabId|: The tab to send keys to. Defaults to active tab.
+ // |key|: The special key to send. Supported keys:
+ // - "Enter": Submit forms, activate buttons, insert line break
+ // - "Delete": Delete character after cursor
+ // - "Backspace": Delete character before cursor
+ // - "Tab": Move focus to next element
+ // - "Escape": Cancel operations, close dialogs
+ // - "ArrowUp": Move cursor/selection up
+ // - "ArrowDown": Move cursor/selection down
+ // - "ArrowLeft": Move cursor/selection left
+ // - "ArrowRight": Move cursor/selection right
+ // - "Home": Move to beginning of line/document
+ // - "End": Move to end of line/document
+ // - "PageUp": Scroll up one page
+ // - "PageDown": Scroll down one page
+ // |callback|: Called when the key has been sent.
+ static void sendKeys(
+ optional long tabId,
+ DOMString key,
+ SendKeysCallback callback);
+
+ // Captures a screenshot of the tab as a thumbnail
+ // |tabId|: The tab to capture. Defaults to active tab.
+ // |callback|: Called with the screenshot as a data URL.
+ static void captureScreenshot(
+ optional long tabId,
+ CaptureScreenshotCallback callback);
+ };
+};
+
diff --git a/chrome/common/extensions/permissions/chrome_api_permissions.cc b/chrome/common/extensions/permissions/chrome_api_permissions.cc
index 7eba27856109e..141f5f93d7213 100644
--- a/chrome/common/extensions/permissions/chrome_api_permissions.cc
+++ b/chrome/common/extensions/permissions/chrome_api_permissions.cc
@@ -69,6 +69,7 @@ constexpr APIPermissionInfo::InitInfo permissions_to_register[] = {
{APIPermissionID::kBookmark, "bookmarks"},
{APIPermissionID::kBrailleDisplayPrivate, "brailleDisplayPrivate",
APIPermissionInfo::kFlagCannotBeOptional},
+ {APIPermissionID::kBrowserOS, "browserOS"},
{APIPermissionID::kBrowsingData, "browsingData",
APIPermissionInfo::kFlagDoesNotRequireManagedSessionFullLoginWarning},
{APIPermissionID::kCertificateProvider, "certificateProvider",
diff --git a/extensions/browser/extension_function_histogram_value.h b/extensions/browser/extension_function_histogram_value.h
index daced4aed4d50..880d5f4812347 100644
--- a/extensions/browser/extension_function_histogram_value.h
+++ b/extensions/browser/extension_function_histogram_value.h
@@ -1997,6 +1997,18 @@ enum HistogramValue {
EXPERIMENTALACTOR_STARTTASK = 1934,
EXPERIMENTALACTOR_EXECUTEACTION = 1935,
EXPERIMENTALACTOR_STOPTASK = 1936,
+ BROWSER_OS_GETACCESSIBILITYTREE = 1937,
+ BROWSER_OS_GETINTERACTIVESNAPSHOT = 1938,
+ BROWSER_OS_CLICK = 1939,
+ BROWSER_OS_INPUTTEXT = 1940,
+ BROWSER_OS_CLEAR = 1941,
+ BROWSER_OS_GETPAGELOADSTATUS = 1942,
+ BROWSER_OS_SCROLLUP = 1943,
+ BROWSER_OS_SCROLLDOWN = 1944,
+ BROWSER_OS_SCROLLTONODE = 1945,
+ BROWSER_OS_SENDKEYS = 1946,
+ BROWSER_OS_GETPAGESTRUCTURE = 1947,
+ BROWSER_OS_CAPTURESCREENSHOT = 1948,
// Last entry: Add new entries above, then run:
// tools/metrics/histograms/update_extension_histograms.py
ENUM_BOUNDARY
diff --git a/extensions/common/mojom/api_permission_id.mojom b/extensions/common/mojom/api_permission_id.mojom
index 96be0882a86cb..6e99ceae68be8 100644
--- a/extensions/common/mojom/api_permission_id.mojom
+++ b/extensions/common/mojom/api_permission_id.mojom
@@ -287,6 +287,7 @@ enum APIPermissionID {
kExperimentalAiData = 260,
kOmniboxDirectInput = 261,
kExperimentalActor = 262,
+ kBrowserOS = 263,
// Add new entries at the end of the enum and be sure to update the
// "ExtensionPermission3" enum in
diff --git a/tools/metrics/histograms/metadata/extensions/enums.xml b/tools/metrics/histograms/metadata/extensions/enums.xml
index 3eea7f9d144a7..7af6e83dedf57 100644
--- a/tools/metrics/histograms/metadata/extensions/enums.xml
+++ b/tools/metrics/histograms/metadata/extensions/enums.xml
@@ -2819,6 +2819,16 @@ Called by update_extension_histograms.py.-->
<int value="1934" label="EXPERIMENTALACTOR_STARTTASK"/>
<int value="1935" label="EXPERIMENTALACTOR_EXECUTEACTION"/>
<int value="1936" label="EXPERIMENTALACTOR_STOPTASK"/>
+ <int value="1937" label="BROWSER_OS_GETACCESSIBILITYTREE"/>
+ <int value="1938" label="BROWSER_OS_GETINTERACTIVESNAPSHOT"/>
+ <int value="1939" label="BROWSER_OS_CLICK"/>
+ <int value="1940" label="BROWSER_OS_INPUTTEXT"/>
+ <int value="1941" label="BROWSER_OS_CLEAR"/>
+ <int value="1942" label="BROWSER_OS_GETPAGELOADSTATUS"/>
+ <int value="1943" label="BROWSER_OS_SCROLLUP"/>
+ <int value="1944" label="BROWSER_OS_SCROLLDOWN"/>
+ <int value="1945" label="BROWSER_OS_SCROLLTONODE"/>
+ <int value="1946" label="BROWSER_OS_SENDKEYS"/>
</enum>
<!-- LINT.ThenChange(//extensions/browser/extension_function_histogram_value.h:HistogramValue) -->
--
2.49.0
Reference in New Issue View Git Blame Copy Permalink