Merge branch 'main' of github-personal.com:autoscrape-labs/pydoll

Author: thalissonvs

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 2.23.1 (2026-07-16)
+
+### Fix
+
+- **cloudflare**: improve checkbox click logic for faster failure handling
+- **cloudflare**: update checkbox selector and improve bypass logic
+- nested iframes
+- **mouse**: keep debug overlay visible across page navigations
+
 ## 2.23.0 (2026-05-22)
 
 ### Feat

cz.yaml

@@ -2,4 +2,4 @@
 commitizen:
   name: cz_conventional_commits
   tag_format: $version
-  version: 2.23.0
+  version: 2.23.1

pydoll/browser/requests/har_recorder.py

@@ -54,6 +54,8 @@
 
 _PYDOLL_CREATOR_NAME = 'pydoll'
 _HTTP_NOT_MODIFIED = 304
+_BODY_FETCH_ATTEMPTS = 5
+_BODY_FETCH_RETRY_DELAY = 0.1
 
 
 def _get_pydoll_version() -> str:
@@ -265,7 +267,8 @@ async def _finalize_entry(self, request_id: str) -> None:
         if not pending:
             return
 
-        body, base64_encoded = await self._fetch_response_body(request_id)
+        expects_body = pending.get('body_bytes', -1) > 0
+        body, base64_encoded = await self._fetch_response_body(request_id, expects_body)
         pending['response_body'] = body
         pending['response_body_base64'] = base64_encoded
 
@@ -302,20 +305,43 @@ def _flush_pending(self) -> None:
             self._entries.append(entry)
         logger.debug('HAR: flushed pending entries')
 
-    async def _fetch_response_body(self, request_id: str) -> tuple[str, bool]:
+    async def _fetch_response_body(
+        self, request_id: str, expects_body: bool = False
+    ) -> tuple[str, bool]:
         """Fetch the response body via Network.getResponseBody.
 
+        The DevTools body buffer is occasionally not ready the instant
+        loadingFinished fires (seen intermittently on Windows). When the
+        dataReceived events already told us a body exists (``expects_body``),
+        retry briefly on an errored or empty result instead of silently
+        recording an empty body.
+
         Returns:
             Tuple of (body_text, is_base64_encoded). Returns ('', False) on failure.
         """
-        try:
-            command = NetworkCommands.get_response_body(request_id)
-            response: GetResponseBodyResponse = await self._tab._execute_command(command)
-            body_result = response['result']
-            return body_result['body'], body_result['base64Encoded']
-        except Exception:
-            logger.debug('HAR: failed to fetch response body for %s', request_id)
-            return '', False
+        attempts = _BODY_FETCH_ATTEMPTS if expects_body else 1
+        for attempt in range(attempts):
+            try:
+                command = NetworkCommands.get_response_body(request_id)
+                response: GetResponseBodyResponse = await self._tab._execute_command(command)
+                body_result = response['result']
+                body, base64_encoded = body_result['body'], body_result['base64Encoded']
+                if body or not expects_body:
+                    return body, base64_encoded
+            except Exception:
+                logger.debug(
+                    'HAR: failed to fetch response body for %s (attempt %d/%d)',
+                    request_id,
+                    attempt + 1,
+                    attempts,
+                )
+            if attempt + 1 < attempts:
+                await asyncio.sleep(_BODY_FETCH_RETRY_DELAY)
+
+        logger.debug(
+            'HAR: response body unavailable for %s after %d attempt(s)', request_id, attempts
+        )
+        return '', False
 
     def _build_entry(self, pending: dict[str, Any]) -> HarEntry:
         """Build a HAR entry from accumulated pending data."""

pydoll/browser/tab.py

@@ -131,7 +131,7 @@
 
 _CLOUDFLARE_CHALLENGE_DOMAIN = 'challenges.cloudflare.com'
 _CLOUDFLARE_IFRAME_SELECTOR = f'iframe[src*="{_CLOUDFLARE_CHALLENGE_DOMAIN}"]'
-_CLOUDFLARE_CHECKBOX_SELECTOR = 'span.cb-i'
+_CLOUDFLARE_CHECKBOX_SELECTOR = 'input[type="checkbox"]'
 
 
 class Tab(FindElementsMixin):
@@ -1966,36 +1966,36 @@ def on_loaded(_: dict):
                 with contextlib.suppress(Exception):
                     await self.disable_page_events()
 
-    async def _find_cloudflare_shadow_root(self, timeout: float) -> ShadowRoot:
-        """Poll for the Cloudflare Turnstile shadow root.
+    async def _find_cloudflare_shadow_root(self) -> Optional[ShadowRoot]:
+        """Return the Cloudflare Turnstile shadow root if currently present.
 
-        Repeatedly calls ``find_shadow_roots(deep=False)`` and checks each
-        shadow root's ``inner_html`` for the Cloudflare challenge domain.
-
-        Args:
-            timeout: Maximum seconds to wait for the shadow root.
-
-        Returns:
-            The first ShadowRoot whose inner HTML contains
-            ``challenges.cloudflare.com``.
-
-        Raises:
-            WaitElementTimeout: If no matching shadow root is found within
-                *timeout* seconds.
+        Performs a single scan of the page's shadow roots and returns the first
+        one whose ``inner_html`` references ``challenges.cloudflare.com``, or
+        ``None`` when the challenge widget has not been injected yet.
         """
-        start_time = asyncio.get_event_loop().time()
-        while True:
-            shadow_roots = await self.find_shadow_roots(deep=False)
-            for sr in shadow_roots:
-                html = await sr.inner_html
-                if _CLOUDFLARE_CHALLENGE_DOMAIN in html:
-                    return sr
+        for shadow_root in await self.find_shadow_roots(deep=False):
+            with contextlib.suppress(Exception):
+                if _CLOUDFLARE_CHALLENGE_DOMAIN in await shadow_root.inner_html:
+                    return shadow_root
+        return None
 
-            if asyncio.get_event_loop().time() - start_time > timeout:
-                raise WaitElementTimeout(
-                    f'Timed out after {timeout}s waiting for Cloudflare Turnstile shadow root'
-                )
-            await asyncio.sleep(0.5)
+    @staticmethod
+    async def _click_cloudflare_checkbox(shadow_root: ShadowRoot) -> None:
+        """Traverse the Turnstile widget and click its verification checkbox.
+
+        Navigates shadow root -> challenge iframe -> body -> inner shadow root
+        and clicks the ``input[type="checkbox"]`` element. Every step fails
+        fast (``timeout=0``): any node captured here can go stale while
+        Cloudflare re-renders the iframe, and polling locally on a stale node
+        both wastes time and can let four sequential waits overrun the caller's
+        deadline. Failing fast lets ``_bypass_cloudflare`` restart the whole
+        traversal from the top on its next poll.
+        """
+        iframe = await shadow_root.query(_CLOUDFLARE_IFRAME_SELECTOR, timeout=0)
+        body = await iframe.find(tag_name='body', timeout=0)
+        inner_shadow = await body.get_shadow_root(timeout=0)
+        checkbox = await inner_shadow.query(_CLOUDFLARE_CHECKBOX_SELECTOR, timeout=0)
+        await checkbox.click()
 
     async def _bypass_cloudflare(
         self,
@@ -2004,21 +2004,31 @@ async def _bypass_cloudflare(
     ) -> None:
         """Attempt to bypass Cloudflare Turnstile captcha via shadow root traversal.
 
-        Traverses shadow roots to locate the Cloudflare iframe, navigates into
-        it, and clicks the actual checkbox element (``span.cb-i``).
+        Polls for the challenge widget and clicks its checkbox, retrying the
+        whole traversal until *time_to_wait_captcha* elapses. Retrying is
+        required because Cloudflare injects the widget after the load event and
+        re-renders the challenge iframe during its proof-of-work, which
+        invalidates any node captured mid-traversal.
         """
-        try:
-            timeout_int = int(time_to_wait_captcha)
-            shadow_root = await self._find_cloudflare_shadow_root(
-                timeout=time_to_wait_captcha,
-            )
-            iframe = await shadow_root.query(_CLOUDFLARE_IFRAME_SELECTOR, timeout=timeout_int)
-            body = await iframe.find(tag_name='body', timeout=timeout_int)
-            inner_shadow = await body.get_shadow_root(timeout=time_to_wait_captcha)
-            checkbox = await inner_shadow.query(_CLOUDFLARE_CHECKBOX_SELECTOR, timeout=timeout_int)
-            await checkbox.click()
-        except Exception as exc:
-            logger.error(f'Error in cloudflare bypass: {exc}')
+        loop = asyncio.get_event_loop()
+        deadline = loop.time() + time_to_wait_captcha
+        last_error: Optional[Exception] = None
+        while True:
+            try:
+                shadow_root = await self._find_cloudflare_shadow_root()
+                if shadow_root is not None:
+                    await self._click_cloudflare_checkbox(shadow_root)
+                    return
+            except Exception as exc:
+                last_error = exc
+                logger.debug(f'Cloudflare bypass attempt failed, retrying: {exc}')
+
+            if loop.time() >= deadline:
+                break
+            await asyncio.sleep(0.5)
+
+        if last_error is not None:
+            logger.error(f'Error in cloudflare bypass: {last_error}')
 
 
 class _DownloadHandle:

pydoll/interactions/iframe.py

@@ -74,13 +74,13 @@ async def resolve(self) -> IFrameContext:
 
         context = IFrameContext(frame_id=frame_id, document_url=document_url)
 
-        if session_handler and session_id:
-            context.session_handler = session_handler
-            context.session_id = session_id
-
         effective_handler = session_handler or base_handler
         effective_session_id = session_id or base_session_id
 
+        if effective_session_id:
+            context.session_handler = effective_handler
+            context.session_id = effective_session_id
+
         execution_context_id = await self._create_isolated_world_for_frame(
             frame_id, effective_handler, effective_session_id
         )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pydoll-python"
-version = "2.23.0"
+version = "2.23.1"
 description = "Pydoll is a library for automating chromium-based browsers without a WebDriver, offering realistic interactions."
 authors = ["Thalison Fernandes <thalissfernandes99@gmail.com>"]
 readme = "README.md"

tests/integration/pages/oopif/oopif_content.html

@@ -19,6 +19,10 @@ <h1 id="oopif-heading">Cross-Origin Content</h1>
     <iframe id="nested-iframe" src="oopif_nested.html"
             style="width:600px;height:200px;border:1px solid #666;"></iframe>
 
+    <!-- Nested data: URL iframe (opaque origin, no own OOPIF target) -->
+    <iframe id="data-iframe"
+            style="width:400px;height:120px;border:1px solid #333;"></iframe>
+
     <!-- Shadow root with elements and a nested iframe -->
     <div id="shadow-host"></div>
 
@@ -32,6 +36,16 @@ <h1 id="oopif-heading">Cross-Origin Content</h1>
             });
         })();
 
+        // Data: URL iframe (opaque origin) nested inside this OOPIF
+        (function() {
+            document.getElementById('data-iframe').src =
+                'data:text/html,' +
+                encodeURIComponent(
+                    '<h2 id="data-heading">Data Frame Content</h2>' +
+                    '<input id="data-input" type="text">'
+                );
+        })();
+
         // Shadow root containing text, button, and a nested iframe
         (function() {
             var host = document.getElementById('shadow-host');

tests/integration/test_nested_oopif_integration.py

@@ -38,6 +38,26 @@ def _wait_for_server(host: str, port: int, timeout: float = 5.0) -> None:
     raise RuntimeError(f'Server {host}:{port} not ready within {timeout}s')
 
 
+def _cross_site_main_url(port_a: int, port_b: int) -> str:
+    """Build a main-page URL that is a different *site* than the OOPIF.
+
+    ``oopif_main.html`` always points its child iframe at ``127.0.0.1:{port_b}``,
+    so serving the main page from ``localhost`` (a distinct registrable domain
+    for Chrome's site isolation) forces the child into a real out-of-process
+    iframe with its own target/session. Cross-*port* alone (``127.0.0.1`` on
+    both sides) is same-site and would not create an OOPIF.
+
+    Skips the test when ``localhost`` does not resolve to the loopback address
+    the server is bound to (e.g. IPv6-only ``localhost``).
+    """
+    try:
+        with socket.create_connection(('localhost', port_a), timeout=0.5):
+            pass
+    except OSError:
+        pytest.skip('localhost is not reachable on the IPv4 loopback server')
+    return f'http://localhost:{port_a}/oopif_main.html?port={port_b}'
+
+
 @pytest.fixture(scope='module')
 def cross_origin_servers():
     """Two HTTP servers on different ports -> different origins -> OOPIF."""
@@ -152,6 +172,75 @@ async def test_type_text_in_nested_iframe_inside_oopif(
             await wait_for_js_value(input_el, 'this.value', 'hello from nested oopif')
 
 
+class TestDataUrlIframeInsideOopif:
+    """Regression for nested cross-origin iframes: main -> OOPIF -> data: iframe.
+
+    Reproduces the reporter's scenario (nestedframes.netlify.app) where the
+    inner iframe uses a ``data:`` URL. A ``data:`` frame has an opaque origin
+    and stays inside the parent OOPIF's process, so it has no target of its
+    own. IFrameContextResolver therefore resolves no OOPIF session for it and
+    must fall back to the parent OOPIF session that created the isolated world;
+    before the fix it evaluated on the tab session and raised InvalidIFrame.
+    """
+
+    @pytest.mark.asyncio
+    async def test_find_element_in_data_iframe_inside_oopif(
+        self, ci_chrome_options, cross_origin_servers
+    ):
+        port_a, port_b = cross_origin_servers
+        url = _cross_site_main_url(port_a, port_b)
+
+        ci_chrome_options.add_argument('--site-per-process')
+        async with Chrome(options=ci_chrome_options) as browser:
+            tab = await browser.start()
+            await tab.go_to(url)
+
+            oopif = await tab.find(id='cross-origin-iframe', timeout=10)
+            data_iframe = await oopif.find(id='data-iframe', timeout=10)
+            assert data_iframe.is_iframe
+
+            heading = await data_iframe.find(id='data-heading', timeout=10)
+            assert await heading.text == 'Data Frame Content'
+
+    @pytest.mark.asyncio
+    async def test_find_body_in_data_iframe_inside_oopif(
+        self, ci_chrome_options, cross_origin_servers
+    ):
+        """The exact call from the bug report: find(tag_name='body')."""
+        port_a, port_b = cross_origin_servers
+        url = _cross_site_main_url(port_a, port_b)
+
+        ci_chrome_options.add_argument('--site-per-process')
+        async with Chrome(options=ci_chrome_options) as browser:
+            tab = await browser.start()
+            await tab.go_to(url)
+
+            oopif = await tab.find(id='cross-origin-iframe', timeout=10)
+            data_iframe = await oopif.find(id='data-iframe', timeout=10)
+
+            body = await data_iframe.find(tag_name='body', timeout=10)
+            assert 'Data Frame Content' in await body.text
+
+    @pytest.mark.asyncio
+    async def test_type_text_in_data_iframe_inside_oopif(
+        self, ci_chrome_options, cross_origin_servers
+    ):
+        port_a, port_b = cross_origin_servers
+        url = _cross_site_main_url(port_a, port_b)
+
+        ci_chrome_options.add_argument('--site-per-process')
+        async with Chrome(options=ci_chrome_options) as browser:
+            tab = await browser.start()
+            await tab.go_to(url)
+
+            oopif = await tab.find(id='cross-origin-iframe', timeout=10)
+            data_iframe = await oopif.find(id='data-iframe', timeout=10)
+
+            input_el = await data_iframe.find(id='data-input', timeout=10)
+            await input_el.type_text('typed into data frame')
+            await wait_for_js_value(input_el, 'this.value', 'typed into data frame')
+
+
 class TestShadowRootInsideOopif:
     """Discovering and interacting with shadow roots inside OOPIFs."""