Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ with notte.Session(headless=False) as session:
response = agent.run(task="doom scroll cat memes on google images")
```

> ⚙️ Before running the snippet, copy `.env.example` to `.env` and set at least one LLM provider key.

### Using Python SDK (Recommended)

We also provide an effortless API that hosts the browser sessions for you - and provide plenty of premium features. To run the agent you'll need to first sign up on the [Notte Console](https://console.notte.cc) and create a free Notte API key 🔑
Expand All @@ -70,7 +72,7 @@ with client.Session(open_viewer=True) as session:
response = agent.run(task="doom scroll cat memes on google images")
```

Our setup allows you to experiment locally, then drop-in replace the import and prefix `notte` objects with `cli` to switch to SDK and get hosted browser sessions plus access to premium features!
Our setup allows you to experiment locally, then drop-in replace the `notte` import with `notte_sdk` to switch to SDK and unlock hosted browser sessions and access to premium features.

# Benchmarks

Expand Down
151 changes: 150 additions & 1 deletion packages/notte-browser/src/notte_browser/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@
class NotteSession(AsyncResource, SyncResource):
observe_max_retry_after_snapshot_update: ClassVar[int] = 2
nb_seconds_between_snapshots_check: ClassVar[int] = 10
EXECUTION_HIGHLIGHT_DURATION_MS: ClassVar[int] = 450
EXECUTION_HIGHLIGHT_FADE_MS: ClassVar[int] = 220
EXECUTION_HIGHLIGHT_PADDING: ClassVar[int] = 6

@track_usage("local.session.create")
def __init__(
Expand All @@ -86,6 +89,7 @@ def __init__(
tools: list[BaseTool] | None = None,
window: BrowserWindow | None = None,
keep_alive: bool = False,
save_replay_to: str | Path | None = None,
**data: Unpack[SessionStartRequestDict],
) -> None:
self._request: SessionStartRequest = SessionStartRequest.model_validate(data)
Expand All @@ -107,6 +111,7 @@ def __init__(
self._cookie_file: Path | None = Path(cookie_file) if cookie_file is not None else None
self._keep_alive: bool = keep_alive
self._keep_alive_msg: str = "🌌 Keep alive mode enabled, skipping session stop... Use `session.close()` to manually stop the session. Never `keep_alive=True` is production."
self._save_replay_to: Path | None = Path(save_replay_to) if save_replay_to is not None else None

@track_usage("local.session.cookies.set")
async def aset_cookies(
Expand Down Expand Up @@ -152,8 +157,10 @@ async def astop(self) -> None:
if self._keep_alive:
logger.info(self._keep_alive_msg)
return
await self.window.close()
window = self.window
await window.close()
self._window = None
self._save_replay_if_requested()

@override
def start(self) -> None:
Expand Down Expand Up @@ -217,6 +224,34 @@ def replay(self, screenshot_type: ScreenshotType | None = None) -> WebpReplay:
screenshots = screenshots[1:]
return ScreenshotReplay.from_bytes(screenshots).get(quality=90) # pyright: ignore [reportArgumentType]

def save_replay(self, output_file: str | Path, *, screenshot_type: ScreenshotType | None = None) -> Path:
"""
Persist the current session trajectory as a WebP animation.

Args:
output_file: Target path that should end with `.webp`.
screenshot_type: Optional override for the screenshot type used in the replay.

Returns:
Path to the saved replay file.
"""
replay = self.replay(screenshot_type=screenshot_type)
output_path = Path(output_file)
output_path.parent.mkdir(parents=True, exist_ok=True)
replay.save(str(output_path))
return output_path

def _save_replay_if_requested(self) -> None:
if self._save_replay_to is None:
return
try:
saved_path = self.save_replay(self._save_replay_to)
logger.info(f"🎬 Session replay saved to {saved_path}")
except ValueError as exc:
logger.warning(f"Skipping session replay save: {exc}")
except Exception as exc: # pragma: no cover - best effort logging
logger.exception(f"Failed to save session replay to {self._save_replay_to}: {exc}")

# ---------------------------- observe, step functions ----------------------------

async def _interaction_action_listing(
Expand Down Expand Up @@ -354,6 +389,119 @@ async def locate(self, action: BaseAction) -> Locator | None:
return locator
return None

async def _flash_execution_highlight(self, action: BaseAction | None) -> None:
if not isinstance(action, InteractionAction):
return
if not config.highlight_elements:
return
selectors = action.selectors
if selectors is None:
return
try:
locator = await locate_element(self.window.page, selectors)
except Exception as exc:
if config.verbose:
logger.debug(
"Highlight skipped: unable to locate element for action '%s': %s",
getattr(action, "id", "<unknown>"),
exc,
)
return

try:
highlight_created = await locator.evaluate(
"""
(el, data) => {
if (!el) { return false; }
try {
if (typeof el.scrollIntoView === 'function') {
try {
el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
} catch (_) {
el.scrollIntoView();
}
}
const doc = el.ownerDocument || document;
const body = doc.body || doc.documentElement;
if (!body) { return false; }

const rect = el.getBoundingClientRect();
if (!rect || rect.width < 1 || rect.height < 1) {
return false;
}

const left = rect.left - data.padding;
const top = rect.top - data.padding;
const width = rect.width + data.padding * 2;
const height = rect.height + data.padding * 2;

const overlayId = "__notte_action_highlight";
const previous = doc.getElementById(overlayId);
if (previous) {
previous.remove();
}

const overlay = doc.createElement("div");
overlay.id = overlayId;
overlay.style.position = "fixed";
overlay.style.left = `${left}px`;
overlay.style.top = `${top}px`;
overlay.style.width = `${width}px`;
overlay.style.height = `${height}px`;
overlay.style.borderRadius = "8px";
overlay.style.border = `3px solid ${data.color}`;
overlay.style.boxShadow = `0 0 28px ${data.glow}`;
overlay.style.background = data.fill;
overlay.style.pointerEvents = "none";
overlay.style.zIndex = "2147483646";
overlay.style.opacity = "0";
overlay.style.transition = `opacity ${data.fadeMs}ms ease-in-out`;
body.appendChild(overlay);

requestAnimationFrame(() => {
overlay.style.opacity = "0.95";
window.setTimeout(() => {
overlay.style.opacity = "0";
}, data.durationMs);
window.setTimeout(() => {
if (overlay.parentNode) {
overlay.parentNode.removeChild(overlay);
}
}, data.durationMs + data.fadeMs);
});
return true;
} catch (err) {
console.warn("Notte highlight failed", err);
return false;
}
}
""",
{
"color": "#ff9800",
"glow": "rgba(255, 152, 0, 0.35)",
"fill": "rgba(255, 152, 0, 0.18)",
"durationMs": self.EXECUTION_HIGHLIGHT_DURATION_MS,
"fadeMs": self.EXECUTION_HIGHLIGHT_FADE_MS,
"padding": self.EXECUTION_HIGHLIGHT_PADDING,
},
)
except Exception as exc:
if config.verbose:
logger.debug(
"Highlight skipped: evaluation failed for action '%s': %s",
getattr(action, "id", "<unknown>"),
exc,
)
return

if highlight_created:
await asyncio.sleep(min(0.12, self.EXECUTION_HIGHLIGHT_DURATION_MS / 1000))
elif config.verbose:
logger.debug(
"Highlight skipped: overlay not created for action '%s' (likely zero-size element)",
getattr(action, "id", "<unknown>"),
)

@overload
async def aexecute(self, action: BaseAction, *, raise_on_failure: bool | None = None) -> ExecutionResult: ...
@overload
Expand Down Expand Up @@ -409,6 +557,7 @@ async def _aexecute_impl(
# --------------------------------

resolved_action = await NodeResolutionPipe.forward(step_action, self._snapshot, verbose=config.verbose)
await self._flash_execution_highlight(resolved_action)
if config.verbose:
logger.info(f"🌌 starting execution of action '{resolved_action.type}' ...")
# --------------------------------
Expand Down
54 changes: 54 additions & 0 deletions tests/examples/Interaction-overlay.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
from __future__ import annotations
import sys
from pathlib import Path

from dotenv import load_dotenv

#There was a clash with the local session.py file and the virtual env file

def _bootstrap_local_packages() -> None:
"""Ensure we import local Notte packages instead of the virtualenv wheels."""

current = Path(__file__).resolve().parent
repo_root = current
for candidate in [current, *current.parents]:
if (candidate / "packages").is_dir() and (candidate / "pyproject.toml").is_file():
repo_root = candidate
break

packages_root = repo_root / "packages"
candidate_paths = [
repo_root / "src",
packages_root / "notte-core" / "src",
packages_root / "notte-browser" / "src",
packages_root / "notte-agent" / "src",
packages_root / "notte-llm" / "src",
packages_root / "notte-sdk" / "src",
]

for path in candidate_paths:
if path.exists():
path_str = str(path)
if path_str not in sys.path:
sys.path.insert(0, path_str)

# Drop any previously imported notte modules so the interpreter reloads from the local paths
for name in list(sys.modules.keys()):
if name.startswith(("notte_browser", "notte_core", "notte_agent", "notte_llm", "notte_sdk", "notte")):
sys.modules.pop(name, None)


_bootstrap_local_packages()

import notte


def main() -> None:
load_dotenv()
#Use save_replay_to in Session method to save the screenshots WebP file at the specified location.
with notte.Session(headless=False, save_replay_to=r".\replays\rp.webp") as session:
agent = notte.Agent(session=session, reasoning_model="gemini/gemini-2.5-flash", max_steps=30)
response = agent.run(task="go to google and search for plujss and select the second dropdown suggestion")

if __name__ == "__main__":
main()
Comment on lines +46 to +54
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Remove unused variable assignment.

The response variable is assigned but never used. Either utilize it or remove the assignment to clean up the code.

Apply this diff to remove the unused assignment:

-        response = agent.run(task="go to google and search for plujss and select the second dropdown suggestion")
+        _ = agent.run(task="go to google and search for plujss and select the second dropdown suggestion")
🧰 Tools
🪛 Ruff (0.14.3)

51-51: Local variable response is assigned to but never used

Remove assignment to unused variable response

(F841)

🤖 Prompt for AI Agents
In tests/examples/Interaction-overlay.py around lines 46 to 54, the local
variable `response` is assigned the result of `agent.run(...)` but never used;
remove the unused assignment or make use of the returned value. Fix by either
deleting the `response =` assignment and calling `agent.run(...)` directly, or
replace it with a short use (for example logging/printing or passing it to an
assertion) so the value is consumed.

2 changes: 1 addition & 1 deletion tests/integration/sdk/test_steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def test_new_steps():
last_action = session_steps[-2]["value"].get("action")
assert last_action is not None, f"{session_steps[-2]} should have an action"
assert last_action["type"] == "fill", f"{session_steps[-2]} should a fill action"
# shoudl be equal to the last agent step
# should be equal to the last agent step
last_agent_action = agent_steps[-2]["value"].get("action")
assert last_agent_action is not None, f"{agent_steps[-2]} should have an action"
assert last_action == last_agent_action
Expand Down
Loading