blasthttp integration

.gitignore

@@ -3,3 +3,4 @@ __pycache__/
 /data/
 /neo4j/
 .DS_Store
+pytest_debug.log

AGENTS.md

@@ -146,6 +146,7 @@ Key helpers:
 | Helper | What it does |
 |--------|-------------|
 | `self.helpers.request(url)` | Make an HTTP request (with retries, SSL handling, etc.) |
+| `self.helpers.blasthttp` | Shared blasthttp client (rate-limited via `web.http_rate_limit` config) |
 | `self.helpers.resolve(host)` | DNS resolution |
 | `self.helpers.is_ip(s)` | Check if string is an IP |
 | `self.helpers.is_dns_name(s)` | Check if string is a hostname |
@@ -219,7 +220,7 @@ from .base import ModuleTestBase
 
 class TestMyModule(ModuleTestBase):
     async def setup_after_prep(self, module_test):
-        module_test.httpx_mock.add_response(
+        module_test.blasthttp_mock.add_response(
             url="https://api.example.com/lookup?domain=blacklanternsecurity.com",
             json={"emails": ["info@blacklanternsecurity.com"]},
         )
@@ -370,7 +371,7 @@ Whether to process seed events (the initial targets provided to the scan).
 Whether to accept "special" URLs (e.g. JavaScript files) that are not normally distributed to web modules.
 
 ```python
-# httpx.py - needs to process all URLs including special ones
+# http.py - needs to process all URLs including special ones
 accept_url_special = True
 ```
 
@@ -535,7 +536,7 @@ _preserve_graph = True
 Exclude this module from scan statistics. Used by output and report modules.
 
 ##### `_disable_auto_module_deps` (bool) -- default: `False`
-Prevent BBOT from automatically enabling dependency modules. For example, if your module watches `URL` events, BBOT normally auto-enables `httpx`. Set this to `True` to prevent that.
+Prevent BBOT from automatically enabling dependency modules. For example, if your module watches `URL` events, BBOT normally auto-enables `http`. Set this to `True` to prevent that.
 
 ---
 
@@ -875,7 +876,7 @@ class TestMyModule(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
 
     # Optional: override which modules are enabled
-    modules_overrides = ["httpx", "my_module"]
+    modules_overrides = ["http", "my_module"]
 
     # Optional: override config
     config_overrides = {"modules": {"my_module": {"some_option": True}}}
@@ -887,7 +888,7 @@ class TestMyModule(ModuleTestBase):
     async def setup_after_prep(self, module_test):
         """Called AFTER the scan is prepared. Modify modules, add mocks here."""
         # Mock an HTTP response
-        module_test.httpx_mock.add_response(
+        module_test.blasthttp_mock.add_response(
             url="https://api.example.com/lookup?domain=blacklanternsecurity.com",
             json={"results": ["sub.blacklanternsecurity.com"]},
         )
@@ -920,7 +921,7 @@ The test lifecycle runs:
 
 ### Test Utilities
 
-- **`module_test.httpx_mock`** - mock HTTP responses (from pytest-httpx)
+- **`module_test.blasthttp_mock`** - mock HTTP responses 
 - **`module_test.httpserver`** - real HTTP server on port 8888
 - **`module_test.httpserver_ssl`** - real HTTPS server on port 9999
 - **`module_test.mock_dns(data)`** - mock DNS responses
@@ -933,7 +934,7 @@ Real example -- `test_module_robots.py`:
 ```python
 class TestRobots(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["httpx", "robots"]
+    modules_overrides = ["http", "robots"]
     config_overrides = {"modules": {"robots": {"include_sitemap": True}}}
 
     async def setup_after_prep(self, module_test):

bbot/core/event/base.py

@@ -101,8 +101,8 @@ class BaseEvent:
             "parent": "OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7",
             "tags": ["in-scope", "distance-0", "dir", "status-301"],
             "http_title": "301 Moved Permanently",
-            "module": "httpx",
-            "module_sequence": "httpx"
+            "module": "http",
+            "module_sequence": "http"
         }
         ```
     """

bbot/core/helpers/command.py

@@ -1,6 +1,7 @@
 import os
 import asyncio
 import logging
+import contextlib
 import traceback
 from signal import SIGINT
 from subprocess import CompletedProcess, CalledProcessError, SubprocessError
@@ -157,7 +158,18 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, **
                     command_str = " ".join(command)
                     log.warning(f"Stderr for run_live({command_str}):\n\t{stderr}")
         finally:
-            proc_tracker.remove(proc)
+            proc_tracker.discard(proc)
+            # Kill the subprocess if it's still running (e.g. generator was cancelled/closed)
+            if proc.returncode is None:
+                with contextlib.suppress(Exception):
+                    proc.terminate()
+                try:
+                    await asyncio.wait_for(proc.wait(), timeout=5)
+                except (asyncio.TimeoutError, Exception):
+                    with contextlib.suppress(Exception):
+                        proc.kill()
+                if input_task is not None:
+                    input_task.cancel()
 
 
 async def _spawn_proc(self, *command, **kwargs):
@@ -270,7 +282,7 @@ def _prepare_command_kwargs(self, command, kwargs):
         >>> _prepare_command_kwargs(['ls', '-l'], {'sudo': True})
         (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})
     """
-    # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)
+    # limit = 100MB (this is needed for cases that are sending large JSON blobs over stdout)
     if "limit" not in kwargs:
         kwargs["limit"] = 1024 * 1024 * 100
     if "stdout" not in kwargs:

bbot/core/helpers/diff.py

@@ -148,7 +148,7 @@ def compare_headers(self, headers_1, headers_2):
             for x in list(ddiff[k]):
                 try:
                     header_value = str(x).split("'")[1]
-                except KeyError:
+                except (KeyError, IndexError):
                     continue
                 differing_headers.append(header_value)
         return differing_headers
@@ -233,9 +233,21 @@ async def compare(
                         if item in subject_response.text:
                             reflection = True
                             break
+        diff_reasons = await self.parent_helper.run_in_executor_cpu(
+            self._compare_sync,
+            subject_response,
+            subject,
+        )
+
+        if not diff_reasons:
+            return (True, [], reflection, subject_response)
+        else:
+            return (False, diff_reasons, reflection, subject_response)
+
+    def _compare_sync(self, subject_response, subject):
+        """CPU-bound comparison work offloaded from the event loop."""
         try:
             subject_json = xmltodict.parse(subject_response.text)
-
         except ExpatError:
             log.debug(f"Can't HTML parse for {subject.split('?')[0]}. Switching to text parsing as a backup")
             subject_json = subject_response.text.split("\n")
@@ -255,13 +267,9 @@ async def compare(
 
         if self.compare_body(self.baseline_json, subject_json) is False:
             log.debug("difference in HTML body, no match")
-
             diff_reasons.append("body")
 
-        if not diff_reasons:
-            return (True, [], reflection, subject_response)
-        else:
-            return (False, diff_reasons, reflection, subject_response)
+        return diff_reasons
 
     async def canary_check(self, url, mode, rounds=3):
         """

bbot/core/helpers/helper.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import multiprocessing as mp
 from functools import partial
-from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 
 from . import misc
 from .asn import ASNHelper
@@ -86,12 +86,14 @@ def __init__(self, preset):
         self.process_pool = ProcessPoolExecutor(max_workers=num_processes)
 
         self._cloud = None
+        self._blasthttp_client = None
 
         self.re = RegexHelper(self)
         self.yara = YaraHelper(self)
         self.simhash = SimHashHelper()
         self._dns = None
         self._web = None
+        self._asn = None
         self._cloudcheck = None
         self._asn = None
         self.config_aware_validators = self.validators.Validators(self)
@@ -117,6 +119,17 @@ def asn(self):
             self._asn = ASNHelper(self)
         return self._asn
 
+    @property
+    def blasthttp(self):
+        if self._blasthttp_client is None:
+            import blasthttp as _blasthttp
+
+            self._blasthttp_client = _blasthttp.BlastHTTP()
+            rate_limit = self.web_config.get("http_rate_limit", 0)
+            if rate_limit:
+                self._blasthttp_client.set_rate_limit(rate_limit)
+        return self._blasthttp_client
+
     @property
     def cloudcheck(self):
         if self._cloudcheck is None:
@@ -195,22 +208,38 @@ def loop(self):
         """
         if self._loop is None:
             self._loop = get_event_loop()
+            # only current caller is wafw00f (sync requests library)
+            self._io_executor = ThreadPoolExecutor(max_workers=max(8, (os.cpu_count() or 1) + 4))
+            self._cpu_executor = ThreadPoolExecutor(max_workers=max(8, os.cpu_count() or 4))
+            self._loop.set_default_executor(self._io_executor)
         return self._loop
 
-    def run_in_executor(self, callback, *args, **kwargs):
+    def run_in_executor_io(self, callback, *args, **kwargs):
         """
         Run a synchronous task in the event loop's default thread pool executor
 
         Examples:
             Execute callback:
-            >>> result = await self.helpers.run_in_executor(callback_fn, arg1, arg2)
+            >>> result = await self.helpers.run_in_executor_io(callback_fn, arg1, arg2)
+        """
+        callback = partial(callback, **kwargs)
+        return self.loop.run_in_executor(self._io_executor, callback, *args)
+
+    def run_in_executor_cpu(self, callback, *args, **kwargs):
+        """
+        Run short CPU-bound work that releases the GIL in a dedicated thread pool,
+        separate from I/O so it never queues behind long-running network calls.
+
+        Examples:
+            Execute callback:
+            >>> result = await self.helpers.run_in_executor_cpu(callback_fn, arg1, arg2)
         """
         callback = partial(callback, **kwargs)
-        return self.loop.run_in_executor(None, callback, *args)
+        return self.loop.run_in_executor(self._cpu_executor, callback, *args)
 
     def run_in_executor_mp(self, callback, *args, **kwargs):
         """
-        Same as run_in_executor() except with a process pool executor
+        Same as run_in_executor_io() except with a process pool executor
         Use only in cases where callback is CPU-bound
 
         Examples:

bbot/core/helpers/misc.py

@@ -1661,7 +1661,7 @@ def rm_rf(f, ignore_errors=False):
         f (str or Path): The directory path to delete.
 
     Examples:
-        >>> rm_rf("/tmp/httpx98323849")
+        >>> rm_rf("/tmp/bbot98323849")
     """
     import shutil
 
@@ -2718,7 +2718,7 @@ def clean_dns_record(record):
     """
     if not isinstance(record, str):
         record = str(record.to_text())
-    return str(record).rstrip(".").lower()
+    return str(record).strip("'\"").rstrip(".").lower()
 
 
 def truncate_filename(file_path, max_length=255):

bbot/core/helpers/names_generator.py

@@ -25,6 +25,7 @@
     "blazed",
     "bloodshot",
     "brown",
+    "cantankerous",
     "cheeky",
     "childish",
     "chiseled",
@@ -56,6 +57,7 @@
     "depressed",
     "deranged",
     "derogatory",
+    "derpy",
     "despicable",
     "devilish",
     "devious",
@@ -103,6 +105,7 @@
     "glutinous",
     "golden",
     "gothic",
+    "greasy",
     "grievous",
     "gummy",
     "hallucinogenic",
@@ -137,11 +140,14 @@
     "intoxicated",
     "inventive",
     "irritable",
+    "janky",
+    "lackadaisical",
     "large",
     "liquid",
     "loveable",
     "lovely",
     "lucid",
+    "lumpy",
     "malevolent",
     "malfunctioning",
     "malicious",
@@ -221,6 +227,7 @@
     "sinful",
     "sinister",
     "slippery",
+    "sloppy",
     "sly",
     "sneaky",
     "soft",
@@ -311,6 +318,7 @@
     "amir",
     "amy",
     "andrea",
+    "andres",
     "andrew",
     "angela",
     "ann",
@@ -345,6 +353,7 @@
     "brandon",
     "brandybuck",
     "brenda",
+    "brendan",
     "brian",
     "brianna",
     "brittany",
@@ -399,6 +408,7 @@
     "diana",
     "diane",
     "dobby",
+    "dominic",
     "donald",
     "donna",
     "dooku",