diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4224e041c..af93f6100 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ exclude: ^docs repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.2 + rev: v0.15.7 hooks: - id: ruff-check args: [--fix, --ignore, D] @@ -20,7 +20,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell stages: [commit, commit-msg] diff --git a/docs_rst/conf.py b/docs_rst/conf.py index b63217632..4fe2328aa 100644 --- a/docs_rst/conf.py +++ b/docs_rst/conf.py @@ -302,6 +302,7 @@ "python": ("https://docs.python.org/3", None), } + # AJ: a hack found online to get __init__ to show up in docs def skip(app, what, name, obj, skip, options): if name == "__init__": diff --git a/fireworks/core/launchpad.py b/fireworks/core/launchpad.py index 9cf9ce4dc..ae37a09c8 100644 --- a/fireworks/core/launchpad.py +++ b/fireworks/core/launchpad.py @@ -2026,14 +2026,16 @@ def recover_offline(self, launch_id, ignore_errors=False, print_errors=False): ) fw_id = launch["fw_id"] f = self.fireworks.find_one_and_update( - {"fw_id": fw_id}, {"$set": {"state": "RUNNING", "updated_on": datetime.datetime.now(datetime.timezone.utc)}} + {"fw_id": fw_id}, + {"$set": {"state": "RUNNING", "updated_on": datetime.datetime.now(datetime.timezone.utc)}}, ) if f: self._refresh_wf(fw_id) # update the updated_on self.offline_runs.update_one( - {"launch_id": launch_id}, {"$set": {"updated_on": datetime.datetime.now(datetime.timezone.utc).isoformat()}} + {"launch_id": launch_id}, + {"$set": {"updated_on": datetime.datetime.now(datetime.timezone.utc).isoformat()}}, ) return None diff --git a/fireworks/core/rocket.py b/fireworks/core/rocket.py index 0fbe56cc6..c5d7052cc 100644 --- a/fireworks/core/rocket.py +++ b/fireworks/core/rocket.py @@ -15,7 +15,7 @@ import shutil import traceback from threading import Event, Thread, current_thread -from typing import TYPE_CHECKING, IO +from typing import IO, TYPE_CHECKING from monty.io import zopen from monty.os.path import zpath diff --git a/fireworks/core/rocket_launcher.py b/fireworks/core/rocket_launcher.py index 0225bcff4..59652e336 100644 --- a/fireworks/core/rocket_launcher.py +++ b/fireworks/core/rocket_launcher.py @@ -32,8 +32,7 @@ def get_fworker(fworker): return my_fwkr -def launch_rocket(launchpad, fworker=None, fw_id=None, strm_lvl=STREAM_LOGLEVEL, - pdb_on_exception=False, err_file=None): +def launch_rocket(launchpad, fworker=None, fw_id=None, strm_lvl=STREAM_LOGLEVEL, pdb_on_exception=False, err_file=None): """Run a single rocket in the current directory. Args: @@ -107,9 +106,9 @@ def time_ok(): os.chdir(launcher_dir) if local_redirect: with redirect_local() as err_file: - rocket_ran = launch_rocket(launchpad, fworker, strm_lvl=strm_lvl, - pdb_on_exception=pdb_on_exception, - err_file=err_file[1]) + rocket_ran = launch_rocket( + launchpad, fworker, strm_lvl=strm_lvl, pdb_on_exception=pdb_on_exception, err_file=err_file[1] + ) else: rocket_ran = launch_rocket(launchpad, fworker, strm_lvl=strm_lvl, pdb_on_exception=pdb_on_exception) diff --git a/fireworks/core/tests/test_firework.py b/fireworks/core/tests/test_firework.py index 624ab1d35..040a152c6 100644 --- a/fireworks/core/tests/test_firework.py +++ b/fireworks/core/tests/test_firework.py @@ -64,7 +64,7 @@ class FiretaskPickleTest(unittest.TestCase): def setUp(self) -> None: self.task = PickleTask(test=0) self.pkl_task = pickle.dumps(self.task) - self.upkl_task = pickle.loads(self.pkl_task) # noqa: S301 + self.upkl_task = pickle.loads(self.pkl_task) def test_init(self) -> None: assert isinstance(self.upkl_task, PickleTask) diff --git a/fireworks/core/tests/test_launchpad.py b/fireworks/core/tests/test_launchpad.py index a16403e5d..25d6f5495 100644 --- a/fireworks/core/tests/test_launchpad.py +++ b/fireworks/core/tests/test_launchpad.py @@ -54,7 +54,7 @@ def _is_mongomock() -> bool: client = fireworks.fw_config.MongoClient() # Check if the client is from mongomock return "mongomock" in str(type(client).__module__) - except Exception: # noqa: BLE001 + except Exception: return False @@ -72,7 +72,7 @@ def setUpClass(cls) -> None: except OperationFailure: pass # User doesn't exist, that's fine client.not_the_admin_db.command({"createUser": "my-user", "pwd": "my-password", "roles": ["dbOwner"]}) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running or authentication not available") @classmethod @@ -81,12 +81,12 @@ def tearDownClass(cls) -> None: try: client = fireworks.fw_config.MongoClient() client.drop_database("not_the_admin_db") - except Exception: # noqa: BLE001, S110 + except Exception: pass # Database cleanup - OK to silently fail def test_no_admin_privileges_for_plebs(self) -> None: """Normal users can not authenticate against the admin db.""" - lp = LaunchPad(name="admin", username="my-user", password="my-password", authsource="admin") # noqa: S106 + lp = LaunchPad(name="admin", username="my-user", password="my-password", authsource="admin") with pytest.raises(OperationFailure): lp.db.collection.count_documents({}) @@ -97,7 +97,7 @@ def test_authenticating_to_users_db(self) -> None: lp = LaunchPad( name="not_the_admin_db", username="my-user", - password="my-password", # noqa: S106 + password="my-password", authsource="not_the_admin_db", ) lp.db.collection.count_documents({}) @@ -106,7 +106,7 @@ def test_authsource_inferred_from_db_name(self) -> None: """The default behavior is to authenticate against the db that the user is trying to access. """ - lp = LaunchPad(name="not_the_admin_db", username="my-user", password="my-password") # noqa: S106 + lp = LaunchPad(name="not_the_admin_db", username="my-user", password="my-password") lp.db.collection.count_documents({}) @@ -117,7 +117,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod @@ -222,7 +222,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod @@ -606,7 +606,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod @@ -719,7 +719,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod @@ -1018,7 +1018,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod @@ -1125,7 +1125,7 @@ def test_task_level_rerun_wrong_state(self) -> None: self.lp.rerun_fw(1, recover_launch="last") def test_task_level_rerun_no_recovery_info(self) -> None: - self.lp.add_wf(Firework(ScriptTask.from_str('echo'))) + self.lp.add_wf(Firework(ScriptTask.from_str("echo"))) launch_rocket(self.lp, self.fworker, fw_id=2) with pytest.raises(ValueError, match="No recovery info found in launch 1"): self.lp.rerun_fw(2, recover_launch="last") @@ -1143,7 +1143,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod @@ -1208,7 +1208,7 @@ def test_fix_db_inconsistencies_completed(self) -> None: assert "SlowAdditionTask" in child_fw.spec assert "WaitWFLockTask" not in child_fw.spec - self.lp._refresh_wf(fw_id=2) # noqa: SLF001 + self.lp._refresh_wf(fw_id=2) child_fw = self.lp.get_fw_by_id(3) @@ -1247,7 +1247,7 @@ def test_fix_db_inconsistencies_fizzled(self) -> None: assert "SlowAdditionTask" in child_fw.spec assert "WaitWFLockTask" not in child_fw.spec - self.lp._refresh_wf(fw_id=2) # noqa: SLF001 + self.lp._refresh_wf(fw_id=2) fast_fw = self.lp.get_fw_by_id(2) @@ -1262,7 +1262,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod @@ -1343,7 +1343,7 @@ def setUpClass(cls) -> None: try: cls.lp = LaunchPad(name=TEST_DB_NAME, strm_lvl="ERROR") cls.lp.reset(password=None, require_password=False) - except Exception: # noqa: BLE001 + except Exception: raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.") @classmethod diff --git a/fireworks/features/multi_launcher.py b/fireworks/features/multi_launcher.py index 913bec798..346351caf 100644 --- a/fireworks/features/multi_launcher.py +++ b/fireworks/features/multi_launcher.py @@ -223,7 +223,7 @@ def split_node_lists(num_jobs, total_node_list=None, ppn=24): return node_lists, sub_nproc_list -# TODO: why is loglvl a required parameter??? Also nlaunches and sleep_time could have a sensible default?? # noqa: E501, FIX002, TD002, TD003 +# TODO: why is loglvl a required parameter??? Also nlaunches and sleep_time could have a sensible default?? # noqa: E501 def launch_multiprocess( launchpad, fworker, diff --git a/fireworks/flask_site/app.py b/fireworks/flask_site/app.py index 0b780c5fd..76d57aa66 100644 --- a/fireworks/flask_site/app.py +++ b/fireworks/flask_site/app.py @@ -78,9 +78,10 @@ def _addq_WF(q): @app.template_filter("datetime") def datetime(value): import datetime as dt + try: date = dt.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f%z") - except ValueError: #backwards comptability + except ValueError: # backwards compatibility date = dt.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f") return date.strftime("%m/%d/%Y") diff --git a/fireworks/fw_config.py b/fireworks/fw_config.py index ebfcc815b..4aa8b2d22 100644 --- a/fireworks/fw_config.py +++ b/fireworks/fw_config.py @@ -9,6 +9,7 @@ import pymongo from monty.design_patterns import singleton from monty.serialization import dumpfn, loadfn + from fireworks.utilities.exceptions import FWConfigurationError __author__ = "Anubhav Jain" diff --git a/fireworks/tests/master_tests.py b/fireworks/tests/master_tests.py index ed1e12bc8..d2ea508a8 100644 --- a/fireworks/tests/master_tests.py +++ b/fireworks/tests/master_tests.py @@ -2,6 +2,7 @@ completed properly. """ +import datetime import unittest import pytest @@ -10,7 +11,7 @@ from fireworks.core.firework import Workflow from fireworks.user_objects.firetasks.script_task import ScriptTask from fireworks.user_objects.queue_adapters.common_adapter import CommonAdapter -from fireworks.utilities.fw_serializers import load_object +from fireworks.utilities.fw_serializers import load_object, reconstitute_dates __author__ = "Anubhav Jain" __copyright__ = "Copyright 2013, The Materials Project" @@ -99,3 +100,19 @@ def test_recursive_deserialize(self) -> None: "defuse_children": False, } FWAction.from_dict(my_dict) + + +@pytest.mark.parametrize( + ("input_str", "expected"), + [ + ("2000-02-01", "2000-02-01"), + ("2024-01-15", "2024-01-15"), + ("1970-01-01", "1970-01-01"), + ("not-a-date", "not-a-date"), + ("2014-10-14T00:56:27.758673", datetime.datetime(2014, 10, 14, 0, 56, 27, 758673)), + ("2024-03-15T08:30:00", datetime.datetime(2024, 3, 15, 8, 30)), + ], +) +def test_reconstitute_dates_preserves_date_only_strings(input_str: str, expected: str | datetime.datetime) -> None: + """Regression test for #570: date-only strings must not become datetime objects.""" + assert reconstitute_dates(input_str) == expected diff --git a/fireworks/utilities/filepad.py b/fireworks/utilities/filepad.py index 981e7b625..e92380929 100644 --- a/fireworks/utilities/filepad.py +++ b/fireworks/utilities/filepad.py @@ -6,7 +6,6 @@ import zlib from typing import TYPE_CHECKING -from uuid import uuid4 import gridfs from bson.objectid import ObjectId from monty.json import MSONable diff --git a/fireworks/utilities/fw_serializers.py b/fireworks/utilities/fw_serializers.py index ae0726e82..decb7c602 100644 --- a/fireworks/utilities/fw_serializers.py +++ b/fireworks/utilities/fw_serializers.py @@ -49,8 +49,8 @@ USER_PACKAGES, YAML_STYLE, ) +from fireworks.utilities.exceptions import FWFormatError, FWSerializationError from fireworks.utilities.fw_utilities import get_fw_logger -from fireworks.utilities.exceptions import FWSerializationError, FWFormatError __author__ = "Anubhav Jain" __copyright__ = "Copyright 2012, The Materials Project" @@ -134,13 +134,7 @@ def _recursive_load(obj): return [_recursive_load(v) for v in obj] if isinstance(obj, str): - try: - # convert String to datetime if really datetime - return reconstitute_dates(obj) - except Exception: - # convert unicode to ASCII if not really unicode - if obj == obj.encode("ascii", "ignore"): - return str(obj) + return reconstitute_dates(obj) return obj @@ -451,17 +445,19 @@ def reconstitute_dates(obj_dict): if isinstance(obj_dict, (list, tuple)): return [reconstitute_dates(v) for v in obj_dict] - if isinstance(obj_dict, str): - + if isinstance(obj_dict, str) and "T" in obj_dict: + # only attempt datetime parsing for strings containing 'T' (ISO 8601 + # date-time separator) to avoid converting date-only or version-like + # strings such as '2000-02-01' into datetime objects (see #570) for method, args in [ - (datetime.datetime.fromisoformat, tuple()), + (datetime.datetime.fromisoformat, ()), (datetime.datetime.strptime, ("%Y-%m-%dT%H:%M:%S.%f",)), - (datetime.datetime.strptime, ("%Y-%m-%dT%H:%M:%S", )), + (datetime.datetime.strptime, ("%Y-%m-%dT%H:%M:%S",)), ]: try: return method(obj_dict, *args) - except Exception: - pass + except (ValueError, TypeError): + continue return obj_dict diff --git a/fireworks/utilities/fw_utilities.py b/fireworks/utilities/fw_utilities.py index 654fcadc0..8b4bea19d 100644 --- a/fireworks/utilities/fw_utilities.py +++ b/fireworks/utilities/fw_utilities.py @@ -145,7 +145,7 @@ def get_path(): full_path = get_path() if os.path.exists(full_path): full_path = None - time.sleep(random.random() / 3 + 0.1) # noqa: S311 (pseudo-random sufficient for retry timing) + time.sleep(random.random() / 3 + 0.1) continue try: os.mkdir(full_path) @@ -252,10 +252,10 @@ def explicit_serialize(o): """Mark a class for explicit serialization by adding _fw_name attribute.""" module_name = o.__module__ if module_name == "__main__": - import __main__ # noqa: PLC0415 + import __main__ module_name = os.path.splitext(os.path.basename(__main__.__file__))[0] - o._fw_name = f"{{{{{module_name}.{o.__name__}}}}}" # noqa: SLF001 + o._fw_name = f"{{{{{module_name}.{o.__name__}}}}}" return o diff --git a/fireworks/utilities/visualize.py b/fireworks/utilities/visualize.py index 876292d8b..2ce88619d 100644 --- a/fireworks/utilities/visualize.py +++ b/fireworks/utilities/visualize.py @@ -46,7 +46,7 @@ def plot_wf( fontsize (int): font size for the node label. """ try: - import matplotlib.pyplot as plt # noqa: PLC0415 + import matplotlib.pyplot as plt except ImportError: raise SystemExit("Install matplotlib. Exiting.") diff --git a/pyproject.toml b/pyproject.toml index a07d71479..164f128c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,48 +5,71 @@ line-length = 120 [tool.ruff.lint] select = ["ALL"] ignore = [ + "A001", # variable shadowing a Python builtin + "A002", # argument shadowing a Python builtin + "ANN", + "ARG", # unused function/method arguments "B023", # Function definition does not bind loop variable "B028", # No explicit stacklevel keyword argument found "B904", # Within an except clause, raise exceptions with ... + "BLE001", # Do not catch blind exception: Exception "C408", # unnecessary-collection-call - "D102", # missing-docstring-in-public-method (temp ignore, re-enable later once we have docstrings) - "D205", # 1 blank line required between summary line and description - "D212", # Multi-line docstring summary should start at the first line - "PERF203", # try-except-in-loop - # "PERF401", # manual-list-comprehension (TODO fix these or wait for autofix) - "ANN", "C901", # function too complex "COM812", # missing trailing comma - "DTZ011", # datetime.today() used + "D1", # missing docstrings (module, class, method, function, package, magic, init) + "D205", # 1 blank line required between summary line and description + "D212", # Multi-line docstring summary should start at the first line + "D410", # missing blank line after section + "D411", # missing blank line before section + "D417", # missing argument descriptions in docstring + "DTZ", # datetime timezone-related warnings "EM101", "EM102", + "ERA001", # found commented-out code "FBT", + "FIX002", # Line contains TODO + "FIX004", # Line contains HACK "G004", + "INP001", # implicit namespace package "ISC001", + "N802", # Function name should be lowercase + "N806", # variable in function should be lowercase + "N816", # mixed case variable in global scope + "N818", # exception name should end in Error + "PERF203", # try-except-in-loop + "PLC0415", # import not at top of file "PLR", # pylint refactor "PLW2901", # Outer for loop variable overwritten by inner assignment target "PT013", # pytest-incorrect-pytest-import "PTH", # prefer pathlib over os.path - "PYI024", # collections-named-tuple (TODO should maybe fix these) + "PYI024", # collections-named-tuple "RUF012", # Disable checks for mutable class args. This is a non-problem. + "RUF013", # implicit Optional + "S101", # use of assert detected + "S103", # permissive file permissions + "S106", # possible hardcoded password + "S108", # probable insecure temp file usage + "S110", # try-except-pass detected "S113", # requests call without timeout + "S201", # use of Flask debug mode + "S301", # suspicious pickle usage + "S311", # suspicious non-cryptographic random usage + "S603", # subprocess call with shell=True "SIM105", # Use contextlib.suppress(OSError) instead of try-except-pass + "SLF001", # private member access + "T100", # debugger breakpoint "T201", # print statement found + "TD002", # Missing author in TODO + "TD003", # Missing issue link in TODO "TRY003", + "TRY300", # consider using else block + "TRY301", # raise within try ] pydocstyle.convention = "google" isort.split-on-trailing-comma = false [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] -"**/tests/*" = [ - "D100", # missing-module-docstring - "D101", # missing-class-docstring - "D102", # missing-docstring-in-public-method - "D103", # missing-function-docstring - "D107", # missing-init-docstring - "S101", # use of assert detected -] "tasks.py" = ["D"] "fw_tutorials/*" = ["D"]