From b48d5935f21417ba7e5d4b8b1917495a264874e5 Mon Sep 17 00:00:00 2001 From: sameer srivastava Date: Sat, 1 Nov 2025 16:22:46 +0530 Subject: [PATCH 1/2] Implemented event overlay as test example --- .../src/notte_browser/session.py | 117 ++++++++++++++++++ tests/examples/Interaction-overlay.py | 66 ++++++++++ 2 files changed, 183 insertions(+) create mode 100644 tests/examples/Interaction-overlay.py diff --git a/packages/notte-browser/src/notte_browser/session.py b/packages/notte-browser/src/notte_browser/session.py index f8bba876b..4a40292f9 100644 --- a/packages/notte-browser/src/notte_browser/session.py +++ b/packages/notte-browser/src/notte_browser/session.py @@ -74,6 +74,9 @@ class NotteSession(AsyncResource, SyncResource): observe_max_retry_after_snapshot_update: ClassVar[int] = 2 nb_seconds_between_snapshots_check: ClassVar[int] = 10 + EXECUTION_HIGHLIGHT_DURATION_MS: ClassVar[int] = 450 + EXECUTION_HIGHLIGHT_FADE_MS: ClassVar[int] = 220 + EXECUTION_HIGHLIGHT_PADDING: ClassVar[int] = 6 @track_usage("local.session.create") def __init__( @@ -354,6 +357,119 @@ async def locate(self, action: BaseAction) -> Locator | None: return locator return None + async def _flash_execution_highlight(self, action: BaseAction | None) -> None: + if not isinstance(action, InteractionAction): + return + if not config.highlight_elements: + return + selectors = action.selectors + if selectors is None: + return + try: + locator = await locate_element(self.window.page, selectors) + except Exception as exc: + if config.verbose: + logger.debug( + "Highlight skipped: unable to locate element for action '%s': %s", + getattr(action, "id", ""), + exc, + ) + return + + try: + highlight_created = await locator.evaluate( + """ + (el, data) => { + if (!el) { return false; } + try { + if (typeof el.scrollIntoView === 'function') { + try { + el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); + } catch (_) { + el.scrollIntoView(); + } + } + const doc = el.ownerDocument || document; + const body = doc.body || doc.documentElement; + if (!body) { return false; } + + const rect = el.getBoundingClientRect(); + if (!rect || rect.width < 1 || rect.height < 1) { + return false; + } + + const left = rect.left - data.padding; + const top = rect.top - data.padding; + const width = rect.width + data.padding * 2; + const height = rect.height + data.padding * 2; + + const overlayId = "__notte_action_highlight"; + const previous = doc.getElementById(overlayId); + if (previous) { + previous.remove(); + } + + const overlay = doc.createElement("div"); + overlay.id = overlayId; + overlay.style.position = "fixed"; + overlay.style.left = `${left}px`; + overlay.style.top = `${top}px`; + overlay.style.width = `${width}px`; + overlay.style.height = `${height}px`; + overlay.style.borderRadius = "8px"; + overlay.style.border = `3px solid ${data.color}`; + overlay.style.boxShadow = `0 0 28px ${data.glow}`; + overlay.style.background = data.fill; + overlay.style.pointerEvents = "none"; + overlay.style.zIndex = "2147483646"; + overlay.style.opacity = "0"; + overlay.style.transition = `opacity ${data.fadeMs}ms ease-in-out`; + body.appendChild(overlay); + + requestAnimationFrame(() => { + overlay.style.opacity = "0.95"; + window.setTimeout(() => { + overlay.style.opacity = "0"; + }, data.durationMs); + window.setTimeout(() => { + if (overlay.parentNode) { + overlay.parentNode.removeChild(overlay); + } + }, data.durationMs + data.fadeMs); + }); + return true; + } catch (err) { + console.warn("Notte highlight failed", err); + return false; + } + } + """, + { + "color": "#ff9800", + "glow": "rgba(255, 152, 0, 0.35)", + "fill": "rgba(255, 152, 0, 0.18)", + "durationMs": self.EXECUTION_HIGHLIGHT_DURATION_MS, + "fadeMs": self.EXECUTION_HIGHLIGHT_FADE_MS, + "padding": self.EXECUTION_HIGHLIGHT_PADDING, + }, + ) + except Exception as exc: + if config.verbose: + logger.debug( + "Highlight skipped: evaluation failed for action '%s': %s", + getattr(action, "id", ""), + exc, + ) + return + + if highlight_created: + await asyncio.sleep(min(0.12, self.EXECUTION_HIGHLIGHT_DURATION_MS / 1000)) + elif config.verbose: + logger.debug( + "Highlight skipped: overlay not created for action '%s' (likely zero-size element)", + getattr(action, "id", ""), + ) + @overload async def aexecute(self, action: BaseAction, *, raise_on_failure: bool | None = None) -> ExecutionResult: ... @overload @@ -409,6 +525,7 @@ async def _aexecute_impl( # -------------------------------- resolved_action = await NodeResolutionPipe.forward(step_action, self._snapshot, verbose=config.verbose) + await self._flash_execution_highlight(resolved_action) if config.verbose: logger.info(f"🌌 starting execution of action '{resolved_action.type}' ...") # -------------------------------- diff --git a/tests/examples/Interaction-overlay.py b/tests/examples/Interaction-overlay.py new file mode 100644 index 000000000..a4f8a03e3 --- /dev/null +++ b/tests/examples/Interaction-overlay.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import inspect +import sys +from pathlib import Path + +from dotenv import load_dotenv + + +#There was a clash with the local session.py file and the virtual env file + +def _bootstrap_local_packages() -> None: + """Ensure we import local Notte packages instead of the virtualenv wheels.""" + + current = Path(__file__).resolve().parent + repo_root = current + for candidate in [current, *current.parents]: + if (candidate / "packages").is_dir() and (candidate / "pyproject.toml").is_file(): + repo_root = candidate + break + + packages_root = repo_root / "packages" + candidate_paths = [ + repo_root / "src", + packages_root / "notte-core" / "src", + packages_root / "notte-browser" / "src", + packages_root / "notte-agent" / "src", + packages_root / "notte-llm" / "src", + packages_root / "notte-sdk" / "src", + ] + + for path in candidate_paths: + if path.exists(): + path_str = str(path) + if path_str not in sys.path: + sys.path.insert(0, path_str) + + # Drop any previously imported notte modules so the interpreter reloads from the local paths + for name in list(sys.modules.keys()): + if name.startswith(("notte_browser", "notte_core", "notte_agent", "notte_llm", "notte_sdk", "notte")): + sys.modules.pop(name, None) + + +_bootstrap_local_packages() + +import notte + + +def main() -> None: + load_dotenv() + + with notte.Session(headless=False) as session: + agent = notte.Agent(session=session, reasoning_model="gemini/gemini-2.5-flash", max_steps=30) + response = agent.run(task="go to google and search for am and select the second dropdown suggestion") + + print(f"\n(session.py in use: {inspect.getfile(notte.Session)})") + # status_icon = "✅" if response.success else "❌" + # print("\n=== Agent Run Summary ===") + # print(f"{status_icon} Success: {response.success}") + # print(f"⌛ Duration: {response.duration_in_s:.1f}s | Steps: {len(response.steps)}") + # print(f"🧠 Answer: {response.answer}") + # print(f"\n(session.py in use: {inspect.getfile(notte.Session)})") + + +if __name__ == "__main__": + main() From dc2218c981d2eea02410d2679a15d79deb9d4296 Mon Sep 17 00:00:00 2001 From: Sameer Srivastava Date: Sun, 9 Nov 2025 19:41:28 +0530 Subject: [PATCH 2/2] Feat: Replay saving functionality added and readme polished. --- README.md | 4 ++- .../src/notte_browser/session.py | 34 ++++++++++++++++++- tests/examples/Interaction-overlay.py | 18 ++-------- tests/integration/sdk/test_steps.py | 2 +- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6a44b9632..6d93634a0 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ with notte.Session(headless=False) as session: response = agent.run(task="doom scroll cat memes on google images") ``` +> ⚙️ Before running the snippet, copy `.env.example` to `.env` and set at least one LLM provider key. + ### Using Python SDK (Recommended) We also provide an effortless API that hosts the browser sessions for you - and provide plenty of premium features. To run the agent you'll need to first sign up on the [Notte Console](https://console.notte.cc) and create a free Notte API key 🔑 @@ -70,7 +72,7 @@ with client.Session(open_viewer=True) as session: response = agent.run(task="doom scroll cat memes on google images") ``` -Our setup allows you to experiment locally, then drop-in replace the import and prefix `notte` objects with `cli` to switch to SDK and get hosted browser sessions plus access to premium features! +Our setup allows you to experiment locally, then drop-in replace the `notte` import with `notte_sdk` to switch to SDK and unlock hosted browser sessions and access to premium features. # Benchmarks diff --git a/packages/notte-browser/src/notte_browser/session.py b/packages/notte-browser/src/notte_browser/session.py index 4a40292f9..e1e2531a8 100644 --- a/packages/notte-browser/src/notte_browser/session.py +++ b/packages/notte-browser/src/notte_browser/session.py @@ -89,6 +89,7 @@ def __init__( tools: list[BaseTool] | None = None, window: BrowserWindow | None = None, keep_alive: bool = False, + save_replay_to: str | Path | None = None, **data: Unpack[SessionStartRequestDict], ) -> None: self._request: SessionStartRequest = SessionStartRequest.model_validate(data) @@ -110,6 +111,7 @@ def __init__( self._cookie_file: Path | None = Path(cookie_file) if cookie_file is not None else None self._keep_alive: bool = keep_alive self._keep_alive_msg: str = "🌌 Keep alive mode enabled, skipping session stop... Use `session.close()` to manually stop the session. Never `keep_alive=True` is production." + self._save_replay_to: Path | None = Path(save_replay_to) if save_replay_to is not None else None @track_usage("local.session.cookies.set") async def aset_cookies( @@ -155,8 +157,10 @@ async def astop(self) -> None: if self._keep_alive: logger.info(self._keep_alive_msg) return - await self.window.close() + window = self.window + await window.close() self._window = None + self._save_replay_if_requested() @override def start(self) -> None: @@ -220,6 +224,34 @@ def replay(self, screenshot_type: ScreenshotType | None = None) -> WebpReplay: screenshots = screenshots[1:] return ScreenshotReplay.from_bytes(screenshots).get(quality=90) # pyright: ignore [reportArgumentType] + def save_replay(self, output_file: str | Path, *, screenshot_type: ScreenshotType | None = None) -> Path: + """ + Persist the current session trajectory as a WebP animation. + + Args: + output_file: Target path that should end with `.webp`. + screenshot_type: Optional override for the screenshot type used in the replay. + + Returns: + Path to the saved replay file. + """ + replay = self.replay(screenshot_type=screenshot_type) + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + replay.save(str(output_path)) + return output_path + + def _save_replay_if_requested(self) -> None: + if self._save_replay_to is None: + return + try: + saved_path = self.save_replay(self._save_replay_to) + logger.info(f"🎬 Session replay saved to {saved_path}") + except ValueError as exc: + logger.warning(f"Skipping session replay save: {exc}") + except Exception as exc: # pragma: no cover - best effort logging + logger.exception(f"Failed to save session replay to {self._save_replay_to}: {exc}") + # ---------------------------- observe, step functions ---------------------------- async def _interaction_action_listing( diff --git a/tests/examples/Interaction-overlay.py b/tests/examples/Interaction-overlay.py index a4f8a03e3..f7003daca 100644 --- a/tests/examples/Interaction-overlay.py +++ b/tests/examples/Interaction-overlay.py @@ -1,12 +1,9 @@ from __future__ import annotations - -import inspect import sys from pathlib import Path from dotenv import load_dotenv - #There was a clash with the local session.py file and the virtual env file def _bootstrap_local_packages() -> None: @@ -48,19 +45,10 @@ def _bootstrap_local_packages() -> None: def main() -> None: load_dotenv() - - with notte.Session(headless=False) as session: + #Use save_replay_to in Session method to save the screenshots WebP file at the specified location. + with notte.Session(headless=False, save_replay_to=r".\replays\rp.webp") as session: agent = notte.Agent(session=session, reasoning_model="gemini/gemini-2.5-flash", max_steps=30) - response = agent.run(task="go to google and search for am and select the second dropdown suggestion") - - print(f"\n(session.py in use: {inspect.getfile(notte.Session)})") - # status_icon = "✅" if response.success else "❌" - # print("\n=== Agent Run Summary ===") - # print(f"{status_icon} Success: {response.success}") - # print(f"⌛ Duration: {response.duration_in_s:.1f}s | Steps: {len(response.steps)}") - # print(f"🧠 Answer: {response.answer}") - # print(f"\n(session.py in use: {inspect.getfile(notte.Session)})") - + response = agent.run(task="go to google and search for plujss and select the second dropdown suggestion") if __name__ == "__main__": main() diff --git a/tests/integration/sdk/test_steps.py b/tests/integration/sdk/test_steps.py index 2e756e51b..a98931f87 100644 --- a/tests/integration/sdk/test_steps.py +++ b/tests/integration/sdk/test_steps.py @@ -42,7 +42,7 @@ def test_new_steps(): last_action = session_steps[-2]["value"].get("action") assert last_action is not None, f"{session_steps[-2]} should have an action" assert last_action["type"] == "fill", f"{session_steps[-2]} should a fill action" - # shoudl be equal to the last agent step + # should be equal to the last agent step last_agent_action = agent_steps[-2]["value"].get("action") assert last_agent_action is not None, f"{agent_steps[-2]} should have an action" assert last_action == last_agent_action