From 0617ebe9a7c7aa137651656a8ed502f0ea4b8e2d Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Tue, 10 Mar 2026 21:17:16 +0100 Subject: [PATCH 1/9] Refactor model discovery and introduce tags. Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- README.pydantic.md | 14 +- .../pyproject.toml | 2 +- packages/overture-schema-annex/pyproject.toml | 2 +- .../overture-schema-base-theme/pyproject.toml | 14 +- .../pyproject.toml | 4 +- .../src/overture/schema/cli/commands.py | 162 +++++----- .../src/overture/schema/cli/types.py | 2 +- .../tests/test_resolve_types.py | 8 +- .../src/overture/schema/codegen/cli.py | 9 +- .../tests/codegen_test_support.py | 9 +- .../overture-schema-codegen/tests/conftest.py | 2 +- .../tests/test_integration_real_models.py | 2 +- packages/overture-schema-core/pyproject.toml | 5 + .../src/overture/schema/core/discovery.py | 134 --------- .../src/overture/schema/core/tag_providers.py | 88 ++++++ .../tests/test_approved_models.py | 11 + .../pyproject.toml | 8 +- .../pyproject.toml | 4 +- .../overture-schema-system/pyproject.toml | 3 + .../src/overture/schema/system/discovery.py | 279 ++++++++++++++++++ .../overture-schema-system/tests/test_tags.py | 37 +++ .../pyproject.toml | 6 +- .../src/overture/schema/__init__.py | 2 +- 23 files changed, 554 insertions(+), 253 deletions(-) delete mode 100644 packages/overture-schema-core/src/overture/schema/core/discovery.py create mode 100644 packages/overture-schema-core/src/overture/schema/core/tag_providers.py create mode 100644 packages/overture-schema-core/tests/test_approved_models.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery.py create mode 100644 packages/overture-schema-system/tests/test_tags.py diff --git a/README.pydantic.md b/README.pydantic.md index 0655c46d9..213426d43 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -151,26 +151,26 @@ Registration is done in the `[project.entry-points."overture.models"]` section: ```toml [project.entry-points."overture.models"] -"buildings.building" = "overture.schema.buildings.building.models:Building" -"buildings.building_part" = "overture.schema.buildings.building_part.models:BuildingPart" +building = "overture.schema.buildings.building.models:Building" +building_part = "overture.schema.buildings.building_part.models:BuildingPart" ``` The discovery system provides programmatic access to registered models: ```python -from overture.schema.core.discovery import discover_models, get_registered_model +from overture.schema.system.discovery import discover_models, get_registered_model # Discover all registered models all_models = discover_models() # Returns: # { -# ("buildings", "building"): BuildingModel, -# ("places", "place"): PlaceModel, +# ("building", "acme:Building", {"building_tag"}): BuildingModel, +# ("place", "acme:Place", {"place_tag"}): PlaceModel, # ... # } -# Get a specific model by theme and type -building_model = get_registered_model("buildings", "building") +# Get a specific model by type +building_model = get_registered_model("building") if building_model: # Use the model class building = building_model.model_validate(building_data) diff --git a/packages/overture-schema-addresses-theme/pyproject.toml b/packages/overture-schema-addresses-theme/pyproject.toml index e794b90d8..3c0a151f8 100644 --- a/packages/overture-schema-addresses-theme/pyproject.toml +++ b/packages/overture-schema-addresses-theme/pyproject.toml @@ -37,7 +37,7 @@ pythonpath = ["src"] testpaths = ["tests"] [project.entry-points."overture.models"] -"overture:addresses:address" = "overture.schema.addresses:Address" +address = "overture.schema.addresses:Address" [[examples.Address]] id = "416ab01c-d836-4c4f-aedc-2f30941ce94d" diff --git a/packages/overture-schema-annex/pyproject.toml b/packages/overture-schema-annex/pyproject.toml index 41e8f263e..a5c2e494b 100644 --- a/packages/overture-schema-annex/pyproject.toml +++ b/packages/overture-schema-annex/pyproject.toml @@ -29,7 +29,7 @@ path = "src/overture/schema/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"annex:sources" = "overture.schema.annex:Sources" +sources = "overture.schema.annex:Sources" [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/packages/overture-schema-base-theme/pyproject.toml b/packages/overture-schema-base-theme/pyproject.toml index aa9d3ba39..1375b92f6 100644 --- a/packages/overture-schema-base-theme/pyproject.toml +++ b/packages/overture-schema-base-theme/pyproject.toml @@ -35,13 +35,13 @@ path = "src/overture/schema/base/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:base:bathymetry" = "overture.schema.base:Bathymetry" -"overture:base:infrastructure" = "overture.schema.base:Infrastructure" -"overture:base:land" = "overture.schema.base:Land" -"overture:base:land_cover" = "overture.schema.base:LandCover" -"overture:base:land_use" = "overture.schema.base:LandUse" -"overture:base:water" = "overture.schema.base:Water" - +bathymetry = "overture.schema.base:Bathymetry" +infrastructure = "overture.schema.base:Infrastructure" +land = "overture.schema.base:Land" +land_cover = "overture.schema.base:LandCover" +land_use = "overture.schema.base:LandUse" +water = "overture.schema.base:Water" + [[examples.Bathymetry]] id = "5d40bd6c-db14-5492-b29f-5e25a59032bc" geometry = "MULTIPOLYGON (((-170.71296928 -76.744313428, -170.719841483 -76.757076376, -170.731061124 -76.761566192, -170.775652756 -76.76338726, -170.853616381 -76.76253958, -170.918562293 -76.755380155, -170.970490492 -76.741908984, -170.998699301 -76.729180777, -171.003188718 -76.717195533, -170.990421551 -76.703765214, -170.960397802 -76.68888982, -170.940748072 -76.674697941, -170.931472364 -76.661189576, -170.927114414 -76.637296658, -170.927674224 -76.603019188, -170.939335393 -76.574637428, -170.962097922 -76.552151379, -170.999015387 -76.535715361, -171.050087788 -76.525329373, -171.079133298 -76.50751024, -171.086151917 -76.482257963, -171.098653755 -76.462747286, -171.11663881 -76.448978211, -171.146691397 -76.437601179, -171.188811514 -76.428616191, -171.296181785 -76.4228609, -171.468802209 -76.420335306, -171.566055241 -76.41501101, -171.587940879 -76.406888013, -171.59004284 -76.387987744, -171.572361122 -76.358310204, -171.549343725 -76.334488281, -171.520990649 -76.316521976, -171.453759127 -76.301763636, -171.347649159 -76.290213262, -171.30597166 -76.267707269, -171.328726628 -76.234245658, -171.36676019 -76.195627518, -171.420072345 -76.151852851, -171.444766298 -76.12494912, -171.44084205 -76.114916326, -171.378107286 -76.099627787, -171.256562007 -76.079083503, -171.228218647 -76.058825682, -171.293077208 -76.038854322, -171.421365419 -76.023534207, -171.613083278 -76.012865337, -171.76411833 -75.99938969, -171.874470572 -75.983107266, -172.121928361 -75.958403596, -172.506491695 -75.925278679, -172.744527804 -75.899736153, -172.836036689 -75.88177602, -172.904681746 -75.862406785, -172.950462974 -75.841628448, -173.000855857 -75.830396498, -173.055860393 -75.828710933, -173.177561398 -75.810743709, -173.365958872 -75.776494827, -173.493573084 -75.759370386, -173.560404033 -75.759370386, -173.620925776 -75.77158365, -173.675138312 -75.796010178, -173.733786206 -75.808642966, -173.796869456 -75.809482015, -173.847216433 -75.805553449, -173.884827135 -75.79685727, -173.90475244 -75.789177124, -173.906992347 -75.782513013, -173.881736947 -75.76894365, -173.828986239 -75.748469035, -173.797974615 -75.732298475, -173.788702075 -75.72043197, -173.82491541 -75.701013882, -173.90661462 -75.674044211, -173.977087913 -75.656066882, -174.03633529 -75.647081894, -174.150190099 -75.643010485, -174.31865234 -75.643852655, -174.444433211 -75.652836726, -174.527532713 -75.669962696, -174.581709229 -75.687086831, -174.606962758 -75.704209131, -174.631095834 -75.708279163, -174.654108458 -75.699296928, -174.688637451 -75.699296928, -174.734682816 -75.708279163, -174.797846917 -75.708699866, -174.878129754 -75.700559037, -174.939903816 -75.70870181, -174.9831691 -75.733128185, -175.025841122 -75.746602837, -175.06791988 -75.749125768, -175.09922327 -75.755318987, -175.119751293 -75.765182495, -175.127900229 -75.775197415, -175.123670077 -75.785363749, -175.111718372 -75.791289392, -175.092045112 -75.792974345, -175.049907399 -75.780622976, -174.985305232 -75.754235285, -174.935355308 -75.74552996, -174.900057628 -75.754507001, -174.886060973 -75.766815613, -174.893365345 -75.782455795, -174.907537393 -75.791536245, -174.928577117 -75.794056963, -174.971105378 -75.818213107, -175.035122174 -75.864004677, -175.060941949 -75.892403254, -175.048564703 -75.903408839, -175.020469049 -75.909193043, -174.976654988 -75.909755867, -174.944760829 -75.90482541, -174.924786572 -75.894401673, -174.92111336 -75.881479168, -174.933741192 -75.866057897, -174.900484967 -75.857513625, -174.821344686 -75.855846351, -174.752433709 -75.839289534, -174.693752038 -75.807843172, -174.652894268 -75.780747792, -174.629860399 -75.758003392, -174.571227588 -75.745793709, -174.476995837 -75.744118743, -174.398722205 -75.751841803, -174.336406693 -75.768962888, -174.300477946 -75.783262828, -174.290935964 -75.794741623, -174.28812912 -75.812412878, -174.292057414 -75.836276591, -174.289237223 -75.852155302, -174.279668547 -75.860049012, -174.205113931 -75.879998026, -174.065573375 -75.912002343, -173.957779122 -75.924071248, -173.881731171 -75.916204739, -173.846521251 -75.926706189, -173.852149361 -75.955575598, -173.845408416 -75.979439305, -173.826298414 -75.99829731, -173.76424232 -76.018956172, -173.659240133 -76.041415889, -173.560434089 -76.057698465, -173.467824188 -76.067803901, -173.404678836 -76.077625909, -173.370998032 -76.087164489, -173.332530272 -76.106814524, -173.289275555 -76.136576014, -173.231864101 -76.154545405, -173.160295911 -76.1607227, -173.093917454 -76.17278471, -173.032728732 -76.190731436, -173.009710709 -76.205560908, -173.024863387 -76.217273124, -173.048718935 -76.225374126, -173.081277354 -76.229863912, -173.219658797 -76.237442552, -173.463863265 -76.248110046, -173.60352174 -76.25793895, -173.638634223 -76.266929265, -173.658723482 -76.274676093, -173.663789516 -76.281179435, -173.661403366 -76.289363255, -173.651565032 -76.299227554, -173.627282775 -76.313843189, -173.588556596 -76.33321016, -173.575369172 -76.355231445, -173.587720504 -76.379907046, -173.573965869 -76.402499893, -173.53410527 -76.423009985, -173.518376226 -76.437156259, -173.526778738 -76.444938715, -173.559015515 -76.446303683, -173.615086557 -76.441251162, -173.686785609 -76.421600788, -173.774112673 -76.387352563, -173.854573513 -76.372333877, -173.928168128 -76.37654473, -173.968906731 -76.383732772, -173.97678932 -76.393898005, -173.979325549 -76.410884215, -173.976515417 -76.434691403, -174.000646474 -76.454452818, -174.051718722 -76.470168462, -174.08231827 -76.482963711, -174.092445119 -76.492838563, -174.075053216 -76.514344245, -174.030142562 -76.547480757, -174.016669929 -76.575274601, -174.034635317 -76.597725777, -174.037021169 -76.62030279, -174.023827484 -76.64300564, -174.034634583 -76.661942018, -174.069442464 -76.677111923, -174.086843964 -76.690616859, -174.086839082 -76.702456825, -174.080513222 -76.712456309, -174.067866385 -76.72061531, -174.036259441 -76.725116584, -173.98569239 -76.725960131, -173.93723318 -76.720486558, -173.89088181 -76.708695864, -173.780274695 -76.695221211, -173.605411835 -76.6800626, -173.487930602 -76.662096294, -173.427830996 -76.641322294, -173.370307559 -76.630935294, -173.315360292 -76.630935294, -173.249406002 -76.637251344, -173.17244469 -76.649883444, -173.110795196 -76.653532162, -173.06445752 -76.648197497, -173.029349452 -76.637355272, -173.005470993 -76.621005486, -173.01753216 -76.605236858, -173.065532955 -76.590049388, -173.096548505 -76.576599032, -173.11057881 -76.564885791, -173.108053605 -76.552301955, -173.08897289 -76.538847523, -173.051362225 -76.527628807, -172.99522161 -76.518645807, -172.891534181 -76.516119525, -172.740299938 -76.52004996, -172.648684331 -76.524540794, -172.61668736 -76.529592027, -172.584268588 -76.541098757, -172.551428016 -76.559060982, -172.533042741 -76.576141146, -172.529112765 -76.592339249, -172.540195073 -76.604524646, -172.566289666 -76.612697339, -172.576243291 -76.621303431, -172.570055947 -76.630342924, -172.555183534 -76.636123529, -172.531626051 -76.638645245, -172.517040304 -76.643518276, -172.511426292 -76.650742621, -172.551848294 -76.672312544, -172.63830631 -76.708228042, -172.701431121 -76.728711408, -172.741222726 -76.733762641, -172.81460886 -76.72534004, -172.921589524 -76.703443605, -173.006960733 -76.697273314, -173.070722487 -76.706829166, -173.101615682 -76.719791531, -173.099640316 -76.736160408, -173.033958817 -76.759064999, -172.904571183 -76.788505304, -172.847033841 -76.810916113, -172.861346791 -76.826297424, -172.924787296 -76.856444925, -173.037355356 -76.901358615, -173.149640378 -76.935043659, -173.26164236 -76.957500057, -173.354942309 -76.968728255, -173.429540223 -76.968728255, -173.487771718 -76.964657535, -173.529636796 -76.956516094, -173.572768938 -76.955559014, -173.617168145 -76.961786296, -173.614655836 -76.97446809, -173.565232013 -76.993604396, -173.461502424 -77.006682128, -173.303467069 -77.013701287, -173.163373388 -77.02787859, -173.041221382 -77.049214037, -172.918094542 -77.059179951, -172.793992869 -77.057776334, -172.720418717 -77.044861043, -172.697372088 -77.020434079, -172.675885915 -77.003730799, -172.655960197 -76.994751205, -172.60882792 -76.987594764, -172.534489083 -76.982261476, -172.480072837 -76.983094424, -172.445579184 -76.990093609, -172.428332542 -76.998610734, -172.428332911 -77.008645799, -172.435068344 -77.018150822, -172.448538839 -77.027125803, -172.490777829 -77.039613708, -172.561785312 -77.055614535, -172.628175119 -77.080598263, -172.68994725 -77.114564892, -172.751818039 -77.133793765, -172.813787485 -77.138284883, -172.900229764 -77.131828165, -173.011144875 -77.114423613, -173.119679588 -77.128474884, -173.2258339 -77.17398198, -173.273849553 -77.202664633, -173.263726547 -77.214522842, -173.165895559 -77.239681117, -172.980356589 -77.278139457, -172.880291531 -77.312658914, -172.865700386 -77.343239487, -172.867667457 -77.371126102, -172.886192744 -77.39631876, -172.999732531 -77.429966955, -173.208286817 -77.472070689, -173.335454668 -77.509278677, -173.381236082 -77.541590921, -173.403703936 -77.570407724, -173.40285823 -77.595729086, -173.378288408 -77.634921, -173.329994472 -77.687983467, -173.241287742 -77.735563094, -173.112168219 -77.777659882, -173.054064387 -77.81089869, -173.066976248 -77.835279519, -173.063736051 -77.854657976, -173.044343797 -77.869034061, -172.890349983 -77.896435115, -172.60175461 -77.936861139, -172.376181212 -77.961986812, -172.213629791 -77.971812135, -172.023427102 -77.967320559, -171.805573145 -77.948512083, -171.581263004 -77.918894833, -171.350496677 -77.87846881, -171.217147208 -77.851799157, -171.181214596 -77.838885875, -171.160572341 -77.826074082, -171.155220441 -77.813363779, -171.178789134 -77.790158543, -171.231278422 -77.756458375, -171.27338337 -77.70988804, -171.305103978 -77.65044754, -171.293875473 -77.602346602, -171.239697854 -77.565585227, -171.168401509 -77.532887375, -171.079986438 -77.504253044, -171.028614514 -77.483042244, -171.014285737 -77.469254974, -171.016677114 -77.456576914, -171.035788644 -77.445008064, -171.086879845 -77.431646501, -171.169950715 -77.416492226, -171.216537864 -77.403175691, -171.226641293 -77.391696895, -171.228607057 -77.378968685, -171.222435157 -77.364991059, -171.168824693 -77.334840949, -171.067775664 -77.288518355, -171.000402018 -77.24121644, -170.966703754 -77.192935206, -170.894838531 -77.157002595, -170.784806349 -77.133418606, -170.725150821 -77.11627156, -170.715871945 -77.105561456, -170.710674146 -77.077210652, -170.709557424 -77.031219147, -170.697909144 -76.992502178, -170.675729304 -76.961059744, -170.654536164 -76.940848729, -170.634329723 -76.931869135, -170.581564681 -76.922044903, -170.496241038 -76.911376032, -170.429709562 -76.893409727, -170.381970254 -76.868145986, -170.285260999 -76.838950739, -170.139581798 -76.805823986, -170.061542334 -76.78431495, -170.051142608 -76.77442363, -170.076677284 -76.763148845, -170.138146365 -76.750490597, -170.192753568 -76.731526593, -170.240498896 -76.706256833, -170.315896371 -76.686462585, -170.418945993 -76.67214385, -170.498267121 -76.665405567, -170.553859754 -76.666247738, -170.609039198 -76.673409769, -170.663805452 -76.68689166, -170.695686968 -76.698414281, -170.704683743 -76.70797763, -170.710444514 -76.723277346, -170.71296928 -76.744313428), (-172.46185717 -77.485683162, -172.491725041 -77.49003391, -172.535448064 -77.490594163, -172.566986057 -77.488349711, -172.586339021 -77.483300552, -172.598540475 -77.476173053, -172.60359042 -77.466967216, -172.601627836 -77.458872071, -172.592652724 -77.451887618, -172.556765055 -77.448396429, -172.49396483 -77.448398503, -172.453726685 -77.452881992, -172.436050621 -77.461846897, -172.429868964 -77.468114837, -172.435181715 -77.47168581, -172.44584445 -77.477541919, -172.46185717 -77.485683162), (-172.812798475 -76.363628771, -172.855573928 -76.365453015, -172.885037626 -76.36040045, -172.90720433 -76.351027386, -172.92207404 -76.337333821, -172.9168827 -76.324750727, -172.89163031 -76.313278104, -172.862193885 -76.307261221, -172.828573425 -76.30670008, -172.792121028 -76.311189877, -172.752836694 -76.320730613, -172.732062811 -76.331770033, -172.729799379 -76.344308139, -172.756711267 -76.354927718, -172.812798475 -76.363628771), (-171.932998671 -76.183124002, -172.010021088 -76.180457336, -172.070931389 -76.166984091, -172.113033554 -76.150312062, -172.136327583 -76.130441248, -172.133522137 -76.111120124, -172.104617217 -76.092348689, -172.06028165 -76.080296327, -172.000515436 -76.074963039, -171.918725408 -76.076928027, -171.814911566 -76.086191292, -171.745182124 -76.097695899, -171.709537083 -76.111441849, -171.696346087 -76.126554541, -171.705609136 -76.143033974, -171.731004713 -76.156183802, -171.77253282 -76.166004024, -171.83986414 -76.174984091, -171.932998671 -76.183124002), (-173.16885937 -76.066345013, -173.199147981 -76.070696107, -173.23950163 -76.071257052, -173.269213382 -76.065813298, -173.288283234 -76.054364845, -173.2799961 -76.038973879, -173.244351978 -76.0196404, -173.207608446 -76.007588038, -173.169765504 -76.002816794, -173.139490241 -76.003094691, -173.116782658 -76.008421729, -173.104589039 -76.016938854, -173.102909386 -76.028646065, -173.111183172 -76.03940804, -173.129410398 -76.049224779, -173.148635798 -76.05820377, -173.16885937 -76.066345013)))" diff --git a/packages/overture-schema-buildings-theme/pyproject.toml b/packages/overture-schema-buildings-theme/pyproject.toml index 07e418faa..a9c20c2f2 100644 --- a/packages/overture-schema-buildings-theme/pyproject.toml +++ b/packages/overture-schema-buildings-theme/pyproject.toml @@ -35,8 +35,8 @@ path = "src/overture/schema/buildings/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:buildings:building" = "overture.schema.buildings:Building" -"overture:buildings:building_part" = "overture.schema.buildings:BuildingPart" +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" [[examples.Building]] id = "148f35b1-7bc1-4180-9280-10d39b13883b" diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 74c7cbae4..0c36007ba 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -14,21 +14,20 @@ import yaml from pydantic import BaseModel, Field, Tag, TypeAdapter, ValidationError from rich.console import Console +from rich.text import Text from yamlcore import CoreLoader # type: ignore from overture.schema.core import OvertureFeature -from overture.schema.core.discovery import ModelKey, discover_models +from overture.schema.system.discovery import ModelKey, discover_models, tags_by_key from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema -from .docstrings import get_model_docstring, get_theme_module_docstring from .error_formatting import ( format_validation_error, format_validation_errors_verbose, group_errors_by_discriminator, select_most_likely_errors, ) -from .output import rewrap from .type_analysis import StructuralTuple, get_item_index, introspect_union from .types import ErrorLocation, ModelDict, UnionType @@ -208,34 +207,40 @@ def resolve_types( ------- Model type suitable for passing to parse_feature """ - # Determine effective namespace - effective_namespace = "overture" if use_overture_types else namespace - # Discover models once with the appropriate namespace - all_models = discover_models(namespace=effective_namespace) + all_models = discover_models() # Filter models based on CLI options filtered_models: ModelDict = {} + if namespace and namespace != "overture": + filtered_models = { + key: model_class + for key, model_class in all_models.items() + if namespace in key.tags + } + if use_overture_types: - filtered_models = all_models + for key, model_class in all_models.items(): + if tags_by_key(key.tags, "overture:theme"): + filtered_models[key] = model_class elif theme_names and not type_names: # Theme-only mode: all types in specified themes for key, model_class in all_models.items(): - if key.theme in theme_names: + if next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: filtered_models[key] = model_class elif type_names and not theme_names: # Type-only mode: find matching types across all themes for key, model_class in all_models.items(): - if key.type in type_names: + if key.name in type_names and tags_by_key(key.tags, "overture:theme"): filtered_models[key] = model_class elif type_names and theme_names: # Both specified: find matching types within specified themes for key, model_class in all_models.items(): - if key.theme in theme_names and key.type in type_names: + if key.name in type_names and next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: filtered_models[key] = model_class else: @@ -766,49 +771,28 @@ def json_schema_command( raise click.UsageError(str(e)) from e -def dump_namespace( - theme_types: dict[str | None, list[tuple[ModelKey, type[BaseModel]]]], -) -> None: - """Print all themes and types for a namespace. - - Displays themes in alphabetical order with their types and docstrings. - Each type includes its model class name and description. - - Args - ---- - theme_types : dict[str | None, list[tuple[ModelKey, type[BaseModel]]]] - Dict mapping theme name to list of (ModelKey, model_class) tuples - """ - for theme in sorted(theme_types.keys(), key=lambda x: (x is None, x)): - if theme: - stdout.print( - f"[bold green underline]{theme.upper()}[/bold green underline]" - ) - - theme_docstring = get_theme_module_docstring(theme) - if theme_docstring: - stdout.print( - rewrap(theme_docstring, stdout, padding_right=4), style="dim" - ) - - stdout.print() - - # Add types to the tree - sorted_types = sorted(theme_types[theme], key=lambda x: x[0].type) - for key, model_class in sorted_types: - stdout.print( - f" [bright_black]→[/bright_black] [bold cyan]{key.type}[/bold cyan] [dim magenta]({key.entry_point})[/dim magenta]" - ) - docstring = get_model_docstring(model_class) - if docstring: - stdout.print( - rewrap(docstring, stdout, indent=4, padding_right=12), style="dim" - ) - stdout.print() - - @cli.command("list-types") -def list_types() -> None: +@click.option( + "--tag", + "tags", + multiple=True, + help="Filter types by tag (e.g., overture:theme=addresses)", +) +@click.option( + "--exclude-tag", + "excluded_tags", + multiple=True, + help="Filter types by tag (e.g., overture:theme=base)", +) +@click.option( + "--group-by", + help="Group types by tag prefix (e.g., 'overture:theme')", +) +def list_types( + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], + group_by: str | None +) -> None: r"""List all available types grouped by theme with descriptions. Displays all registered Overture Maps types organized by theme, @@ -821,35 +805,51 @@ def list_types() -> None: """ try: models = discover_models() + filters = [] + + if tags: + filters.append(lambda key: all(tag in key.tags for tag in tags)) + if excluded_tags: + filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + + if filters: + models = { + key: model + for key, model in models.items() + if all(f(key) for f in filters) + } + + if group_by: + grouped_models: dict[str, set[ModelKey]] = {} + + for key, model_class in models.items(): + if groups := tags_by_key(key.tags, group_by): + for group in groups: + grouped_models.setdefault(group, set()).add(key) + + padding = max((len(key.name) for keys in grouped_models.values() for key in keys), default=0) + 2 + + for group, keys in sorted(grouped_models.items()): + stdout.print(f"[green bold]{group_by}={group} ({len(keys)})[/green bold]") + for key in sorted(keys, key=lambda k: k.name): + model = Text() + model.append("→ ", style="bright_black") + model.append(key.name, style="bold cyan") + model.pad_right(max(1, padding - len(key.name))) + model.append_text(Text().append(" ".join(sorted(key.tags)))) + stdout.print(model) + stdout.print() + + else: + padding = max((len(key.name) for key in models.keys()), default=0) + 2 + + for key in sorted(models.keys(), key=lambda k: k.name): + model = Text() + model.append(key.name, style="bold cyan") + model.pad_right(max(1, padding - len(key.name))) + model.append_text(Text().append(" ".join(sorted(key.tags)))) + stdout.print(model) - # Group models by namespace and theme - namespaces: dict[ - str, dict[str | None, list[tuple[ModelKey, type[BaseModel]]]] - ] = {} - for key, model_class in models.items(): - if key.namespace not in namespaces: - namespaces[key.namespace] = {} - if key.theme not in namespaces[key.namespace]: - namespaces[key.namespace][key.theme] = [] - - namespaces[key.namespace][key.theme].append((key, model_class)) - - # display Overture themes first - if "overture" in namespaces: - stdout.print("[bold red]OVERTURE THEMES[/bold red]", justify="center") - stdout.print() - - dump_namespace(namespaces["overture"]) - - stdout.print("[bold red]ADDITIONAL TYPES[/bold red]", justify="center") - stdout.print() - - for namespace in sorted(namespaces.keys()): - if namespace == "overture": - continue - - stdout.print(f"[bold blue]{namespace.upper()}[/bold blue]") - dump_namespace(namespaces[namespace]) except Exception as e: click.echo(f"Error listing types: {e}", err=True) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/types.py b/packages/overture-schema-cli/src/overture/schema/cli/types.py index 1b5d4e44d..f438edf2f 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/types.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/types.py @@ -5,7 +5,7 @@ from pydantic import BaseModel from pydantic_core import ErrorDetails -from overture.schema.core.discovery import ModelKey +from overture.schema.system.discovery import ModelKey # Type alias for union types created from Pydantic models # This represents either a single model or a discriminated union of models diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 94231a1fe..19ce9592f 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -2,7 +2,7 @@ import pytest from overture.schema.cli.commands import resolve_types -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import tags_by_key class TestResolveTypes: @@ -125,8 +125,10 @@ def test_resolve_types_returns_expected_themes( expected_themes: set[str], ) -> None: """Test that resolve_types returns models from expected themes.""" - models = discover_models(namespace=namespace) - actual_themes = {key.theme for key in models.keys()} + from overture.schema.system.discovery import discover_models + + models = discover_models() + actual_themes = {next(iter(tags_by_key(key.tags, "overture:theme")),None) for key in models.keys()} # Check that we have at least the expected themes (may have more) assert expected_themes.issubset(actual_themes), ( diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index 0a24c7348..ced8932ac 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -6,7 +6,7 @@ import click -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models, tags_by_key from .extraction.model_extraction import extract_model from .extraction.specs import ( @@ -47,6 +47,11 @@ def _write_output( click.echo() # separate entries with a blank line in stdout mode +def _find_theme(tags: frozenset[str]) -> str | None: + """Find the theme tag in a set of tags, if any.""" + return next(iter(tags_by_key(tags, "overture:theme")),None) + + @click.group() def cli() -> None: """Overture Schema code generator. @@ -99,7 +104,7 @@ def generate( schema_root = compute_schema_root(module_paths) models = ( - {k: v for k, v in all_models.items() if k.theme in theme} + {k: v for k, v in all_models.items() if _find_theme(k.tags) in theme} if theme else all_models ) diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 64facf5a9..79881ed5b 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -25,7 +25,7 @@ is_model_class, ) from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models, tags_by_key from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import require_any_of @@ -301,6 +301,11 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec: return next(m for m in spec.members if m.name == name) +def find_theme(tags: frozenset[str]) -> str | None: + """Extract the theme from a set of tags, if present.""" + return next(iter(tags_by_key(tags, "overture:theme")),None) + + T = TypeVar("T") @@ -332,7 +337,7 @@ def flat_specs_from_discovery( """Build a flat list of ModelSpecs from discovery, with entry_point set.""" models = discover_models() if theme: - models = {k: v for k, v in models.items() if k.theme == theme} + models = {k: v for k, v in models.items() if find_theme(k.tags) == theme} result = [] for key, cls in models.items(): if not is_model_class(cls): diff --git a/packages/overture-schema-codegen/tests/conftest.py b/packages/overture-schema-codegen/tests/conftest.py index 8dce88bf5..d66cf72a3 100644 --- a/packages/overture-schema-codegen/tests/conftest.py +++ b/packages/overture-schema-codegen/tests/conftest.py @@ -14,7 +14,7 @@ render_geometry_from_values, render_primitives_from_specs, ) -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.system.primitive import GeometryType from pydantic import BaseModel diff --git a/packages/overture-schema-codegen/tests/test_integration_real_models.py b/packages/overture-schema-codegen/tests/test_integration_real_models.py index b4dd9419f..9ed20d112 100644 --- a/packages/overture-schema-codegen/tests/test_integration_real_models.py +++ b/packages/overture-schema-codegen/tests/test_integration_real_models.py @@ -20,7 +20,7 @@ from overture.schema.codegen.layout.module_layout import entry_point_class from overture.schema.codegen.markdown.pipeline import generate_markdown_pages from overture.schema.codegen.markdown.renderer import render_feature -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.transportation import Segment from overture.schema.transportation.segment.models import RoadSegment from pydantic import BaseModel diff --git a/packages/overture-schema-core/pyproject.toml b/packages/overture-schema-core/pyproject.toml index cff805739..ceee55a91 100644 --- a/packages/overture-schema-core/pyproject.toml +++ b/packages/overture-schema-core/pyproject.toml @@ -36,3 +36,8 @@ dev = [ "types-pyyaml>=6.0.12.20250516", "types-shapely>=2.1.0.20250710", ] + +[project.entry-points."overture.tag_providers"] +overture = "overture.schema.core.tag_providers:overture_provider" +authority = "overture.schema.core.tag_providers:authority_provider" +theme = "overture.schema.core.tag_providers:theme_provider" diff --git a/packages/overture-schema-core/src/overture/schema/core/discovery.py b/packages/overture-schema-core/src/overture/schema/core/discovery.py deleted file mode 100644 index b9290d29a..000000000 --- a/packages/overture-schema-core/src/overture/schema/core/discovery.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Model discovery system for Overture schema registry.""" - -import importlib.metadata -import logging -from dataclasses import dataclass - -from pydantic import BaseModel - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True, slots=True) -class ModelKey: - """Key identifying a registered model by namespace, theme, and type. - - Attributes - ---------- - namespace : str - The namespace (e.g., "overture", "annex") - theme : str | None - The theme name (e.g., "buildings", "places"), or None for non-themed models - type : str - The feature type (e.g., "building", "place") - entry_point : str - The entry point value in "module:Class" format - - """ - - namespace: str - theme: str | None - type: str - entry_point: str - - -def discover_models( - namespace: str | None = None, -) -> dict[ModelKey, type[BaseModel]]: - """Discover all registered Overture models via entry points. - - Parameters - ---------- - namespace : str | None, optional - Optional namespace filter. If provided, only models from this - namespace will be returned (e.g., "overture", "annex"). - - Returns - ------- - dict[ModelKey, type[BaseModel]] - Dict mapping ModelKey to model classes. - Theme will be None for entries without an explicit theme component. - - Notes - ----- - Entry point name format: - - Core themes: "overture::" - - Non-core (2-part): "annex:" (theme will be None) - - Non-core (3-part): "annex::" - - """ - models = {} - try: - for entry_point in importlib.metadata.entry_points(group="overture.models"): - # Parse namespace:theme:type or namespace:type from entry point name - parts = entry_point.name.split(":", 2) - - if len(parts) == 2: - # namespace:type format (no theme) - ns, feature_type = parts - theme = None - elif len(parts) == 3: - # namespace:theme:type format - ns, theme, feature_type = parts - else: - logger.warning( - "Invalid entry point format %s, expected namespace:theme:type or namespace:type", - entry_point.name, - ) - continue - - # Filter by namespace if specified - if namespace is not None and ns != namespace: - continue - - try: - model_class = entry_point.load() - key = ModelKey( - namespace=ns, - theme=theme, - type=feature_type, - entry_point=entry_point.value, - ) - models[key] = model_class - except Exception as e: - # Log warning but don't fail for individual models - logger.warning("Could not load model %s: %s", entry_point.name, e) - except Exception as e: - logger.warning("Could not discover entry points: %s", e) - - return models - - -def get_registered_model( - namespace: str, feature_type: str, theme: str | None = None -) -> type[BaseModel] | None: - """Get the Pydantic model for a namespace/theme/type combination. - - This uses setuptools entry points for registration. - - Parameters - ---------- - namespace : str - The namespace (e.g., "overture", "annex") - feature_type : str - The type name - theme : str | None, optional - The theme name (optional) - - Returns - ------- - type[BaseModel] | None - The model class if found, None otherwise. - - """ - # Check all discovered models for a match - models = discover_models(namespace=namespace) - # Need to find by namespace/theme/type, not exact key match - for key, model_class in models.items(): - if ( - key.namespace == namespace - and key.theme == theme - and key.type == feature_type - ): - return model_class - return None diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py new file mode 100644 index 000000000..de52abf93 --- /dev/null +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -0,0 +1,88 @@ +from typing import Any, get_origin, Annotated, get_args, Union, Literal + +from overture.schema.core import OvertureFeature +from pydantic import BaseModel + +from overture.schema.system.discovery import ModelKey + +APPROVED = { + "overture.schema.addresses:Address", + "overture.schema.base:Bathymetry", + "overture.schema.base:Infrastructure", + "overture.schema.base:Land", + "overture.schema.base:LandCover", + "overture.schema.base:LandUse", + "overture.schema.base:Water", + "overture.schema.buildings:Building", + "overture.schema.buildings:BuildingPart", + "overture.schema.divisions:Division", + "overture.schema.divisions:DivisionArea", + "overture.schema.divisions:DivisionBoundary", + "overture.schema.places:Place", + "overture.schema.transportation:Connector", + "overture.schema.transportation:Segment", +} + + +def authority_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + if _matches_manifest(key): + tags.add("overture:official") + return tags + + +def overture_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + if any(issubclass(tp, OvertureFeature) for tp in _reduce_types(model_class)): + tags.add("overture:feature") + return tags + + +def theme_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + for tp in _reduce_types(model_class): + if issubclass(tp, OvertureFeature): + tags.add( + "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] + ) + return tags + + +def _matches_manifest(key: ModelKey) -> bool: + if key.entry_point in APPROVED: + return True + return False + + +def _reduce_types(tp: Any) -> set[type]: + result: set[type] = set() + + def visit(t: Any) -> None: + origin = get_origin(t) + if origin is Annotated: + visit(get_args(t)[0]) + return + + if hasattr(t, "__supertype__"): + visit(t.__supertype__) + return + + origin = get_origin(t) + + if origin is Union: + for arg in get_args(t): + visit(arg) + return + + if origin is Literal: + for val in get_args(t): + result.add(type(val)) + return + + result.add(t) + + visit(tp) + return result diff --git a/packages/overture-schema-core/tests/test_approved_models.py b/packages/overture-schema-core/tests/test_approved_models.py new file mode 100644 index 000000000..6d7108610 --- /dev/null +++ b/packages/overture-schema-core/tests/test_approved_models.py @@ -0,0 +1,11 @@ +import pytest +from overture.schema.system.discovery import discover_models + + +def test_overture_feature_models_are_official() -> None: + models = discover_models() + for key in models: + if "overture:feature" in key.tags: + assert ( + "overture:official" in key.tags + ), f"Model {key.name} is missing 'overture:official' tag." diff --git a/packages/overture-schema-divisions-theme/pyproject.toml b/packages/overture-schema-divisions-theme/pyproject.toml index 0a1cd9e61..943985e21 100644 --- a/packages/overture-schema-divisions-theme/pyproject.toml +++ b/packages/overture-schema-divisions-theme/pyproject.toml @@ -34,10 +34,10 @@ path = "src/overture/schema/divisions/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:divisions:division" = "overture.schema.divisions:Division" -"overture:divisions:division_area" = "overture.schema.divisions:DivisionArea" -"overture:divisions:division_boundary" = "overture.schema.divisions:DivisionBoundary" - +division = "overture.schema.divisions:Division" +division_area = "overture.schema.divisions:DivisionArea" +division_boundary = "overture.schema.divisions:DivisionBoundary" + [[examples.Division]] id = "350e85f6-68ba-4114-9906-c2844815988b" geometry = "POINT (-175.2551522 -21.1353686)" diff --git a/packages/overture-schema-places-theme/pyproject.toml b/packages/overture-schema-places-theme/pyproject.toml index b374835be..f036bda65 100644 --- a/packages/overture-schema-places-theme/pyproject.toml +++ b/packages/overture-schema-places-theme/pyproject.toml @@ -35,8 +35,8 @@ path = "src/overture/schema/places/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:places:place" = "overture.schema.places:Place" - +place = "overture.schema.places:Place" + [[examples.Place]] id = "99003ee6-e75b-4dd6-8a8a-53a5a716c50d" geometry = "POINT (-150.46875 -79.1713346)" diff --git a/packages/overture-schema-system/pyproject.toml b/packages/overture-schema-system/pyproject.toml index 973effa8c..1f1d54e52 100644 --- a/packages/overture-schema-system/pyproject.toml +++ b/packages/overture-schema-system/pyproject.toml @@ -55,3 +55,6 @@ ignore = [ "C901", # too complex ] per-file-ignores = {"__init__.py" = ["F401"]} + +[project.entry-points."overture.tag_providers"] +feature = "overture.schema.system.discovery:feature_provider" \ No newline at end of file diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py new file mode 100644 index 000000000..5231c737d --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -0,0 +1,279 @@ +"""Model discovery system for Overture schema registry.""" + +import importlib.metadata +import logging +import re +from dataclasses import dataclass, replace +from typing import get_args, Literal, Union, get_origin, Annotated, Any, Callable + +from pydantic import BaseModel + +from overture.schema.system.feature import Feature + +logger = logging.getLogger(__name__) + + +RESERVED_TAGS: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "feature": {"overture-schema-system"}, +} +TAG = r"[a-z0-9][a-z0-9_-]*" +NAMESPACE_TAG = r"[a-z0-9]+:[a-z0-9]+(?:=[a-z0-9_.-]+)?" +TAG_RE = re.compile(rf"^(?:{TAG}|{NAMESPACE_TAG})$") + + +@dataclass(frozen=True, slots=True) +class ModelKey: + """Key identifying a registered model by namespace, theme, and type. + + Attributes + ---------- + name : str + The friendly name of the model, derived from the entry point key + entry_point : str + The entry point value in "module:Class" format + tags : frozenset[str] + A set of tags associated with the model, including both plain tags and structured tags + + """ + + name: str # friendly name from entry point key + entry_point : str # The entry point value in "module:Class" format + tags: frozenset[str] # plain and structured tags + + +@dataclass(frozen=True, slots=True) +class TagProviderKey: + """Key identifying a registered model by namespace, theme, and type. + + Attributes + ---------- + name : str + The friendly name of the model, derived from the entry point key + entry_point : str + The entry point value in "module:Class" format + + """ + + name: str # friendly name from entry point key + entry_point: str # entry point value (module:Class) + package_name: str # distribution package name + + +TagProvider = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] + + +def generate_tags( + model_class: type[BaseModel], + key: ModelKey, + providers: dict[TagProviderKey, TagProvider], +) -> set[str]: + tags: set[str] = set() + + for provider_key, provider in providers.items(): + try: + added_tags = provider(model_class, key, tags.copy()).difference(tags) + filtered_tags = _filter_tags(added_tags, provider_key.package_name) + tags.update(filtered_tags) + except Exception as e: + print( + f"Error in tag provider {provider.__name__} for model {key.name}: {e}" + ) + + return tags + + +def _filter_tags(tags: set[str], package: str) -> set[str]: + reserved_namespaces = tuple( + f"{namespace}:" + for namespace, reserved_packages in RESERVED_TAGS.items() + if package not in reserved_packages + ) + + return { + tag + for tag in tags + if TAG_RE.match(tag) and not tag.startswith(reserved_namespaces) + } + + +def discover_tag_providers( + tag_providers_group: str = "overture.tag_providers", +) -> dict[TagProviderKey, TagProvider]: + tag_providers = {} + + try: + for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): + + try: + tag_provider_class = tag_provider.load() + + key = TagProviderKey( + name=tag_provider.name, + entry_point=tag_provider.value, + package_name=getattr(tag_provider.dist, "name", ""), + ) + + tag_providers[key] = tag_provider_class + + except Exception as e: + # Log warning but don't fail for individual tag providers + logger.warning( + "Could not load tag provider %s: %s", tag_provider.name, e + ) + except Exception as e: + logger.warning("Could not discover entry points: %s", e) + + return tag_providers + + +def discover_models( + model_group: str = "overture.models", +) -> dict[ModelKey, type[BaseModel]]: + """Discover all registered Overture models via entry points. + + Parameters + ---------- + model_group: str + The entry point group to search for models (default: "overture.models") + + Returns + ------- + dict[ModelKey, type[BaseModel]] + Dict mapping ModelKey to model classes. + Theme will be None for entries without an explicit theme component. + """ + models = {} + tag_providers = discover_tag_providers() + + try: + for model in importlib.metadata.entry_points(group=model_group): + + try: + model_class = model.load() + + key = ModelKey( + name=model.name, + entry_point=model.value, + tags=( + frozenset(_distribution_tags(model.dist)) + if model.dist + else frozenset() + ), + ) + + try: + key = replace( + key, + tags=frozenset(generate_tags(model_class, key, tag_providers)), + ) + except Exception as e: + logger.warning( + "Could not resolve tags for model %s: %s", model.name, e + ) + + models[key] = model_class + + except Exception as e: + # Log warning but don't fail for individual models + logger.warning("Could not load model %s: %s", model.name, e) + except Exception as e: + logger.warning("Could not discover entry points: %s", e) + + return models + + +def _distribution_tags(dist: importlib.metadata.Distribution) -> set[str]: + """Extract tags from the distribution metadata.""" + tags = set() + if dist.name: + tags.add("dist:name=" + dist.name) + if dist.version: + tags.add("dist:version=" + dist.version) + + return tags + + +def get_registered_model(feature_type: str) -> type[BaseModel] | None: + """Get the Pydantic model for a type. + + This uses setuptools entry points for registration. + If multiple types share the same name, the first one encountered will be returned. + + Parameters + ---------- + feature_type : str + The type name + + Returns + ------- + type[BaseModel] | None + The first encountered model class if found, None otherwise. + + """ + # Check all discovered models for a match + models = discover_models() + # Need to find by type, not exact key match + for key, model_class in models.items(): + if key.name == feature_type: + return model_class + return None + + +def tags_by_key(tags: frozenset[str] | set[str], key: str) -> set[str]: + """Extract values for k/v tags with the given key. + + tags_by_key(frozenset({"overture:theme=buildings", "overture", "draft"}), "overture:theme") + -> {"buildings"} + """ + prefix = key + "=" + return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} + + +def tags_by_namespace(tags: frozenset[str] | set[str], namespace: str) -> set[str]: + """Extract tag bodies within a namespace. + + tags_by_namespace(frozenset({"system:extension", "overture"}), "system") + -> {"extension"} + """ + prefix = namespace + ":" + return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} + + +def feature_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + if any(issubclass(tp, Feature) for tp in _extract_types(model_class)): + tags.add("system:feature") + return tags + + +def _extract_types(tp: Any) -> set[type]: + result: set[type] = set() + + def visit(t: Any) -> None: + origin = get_origin(t) + if origin is Annotated: + visit(get_args(t)[0]) + return + + if hasattr(t, "__supertype__"): + visit(t.__supertype__) + return + + origin = get_origin(t) + + if origin is Union: + for arg in get_args(t): + visit(arg) + return + + if origin is Literal: + for val in get_args(t): + result.add(type(val)) + return + + result.add(t) + + visit(tp) + return result diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py new file mode 100644 index 000000000..0fc88f69b --- /dev/null +++ b/packages/overture-schema-system/tests/test_tags.py @@ -0,0 +1,37 @@ +from overture.schema.system.discovery import tags_by_key, tags_by_namespace + +def test_tags_by_key_returns_correct_values() -> None: + tags = frozenset({"overture:theme=buildings", "overture", "draft"}) + key = "overture:theme" + result = tags_by_key(tags, key) + assert result == {"buildings"} + +def test_tags_by_key_returns_empty_set_for_nonexistent_key() -> None: + tags = frozenset({"overture:theme=buildings", "overture", "draft"}) + key = "nonexistent:key" + result = tags_by_key(tags, key) + assert result == set() + +def test_tags_by_key_handles_empty_tags() -> None: + tags: frozenset[str] = frozenset() + key = "overture:theme" + result = tags_by_key(tags, key) + assert result == set() + +def test_tags_by_namespace_returns_correct_values() -> None: + tags = frozenset({"system:extension", "overture"}) + namespace = "system" + result = tags_by_namespace(tags, namespace) + assert result == {"extension"} + +def test_tags_by_namespace_returns_empty_set_for_nonexistent_namespace() -> None: + tags = frozenset({"system:extension", "overture"}) + namespace = "nonexistent" + result = tags_by_namespace(tags, namespace) + assert result == set() + +def test_tags_by_namespace_handles_empty_tags() -> None: + tags: frozenset[str] = frozenset() + namespace = "system" + result = tags_by_namespace(tags, namespace) + assert result == set() diff --git a/packages/overture-schema-transportation-theme/pyproject.toml b/packages/overture-schema-transportation-theme/pyproject.toml index 08811df39..228527d48 100644 --- a/packages/overture-schema-transportation-theme/pyproject.toml +++ b/packages/overture-schema-transportation-theme/pyproject.toml @@ -35,9 +35,9 @@ path = "src/overture/schema/transportation/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:transportation:connector" = "overture.schema.transportation:Connector" -"overture:transportation:segment" = "overture.schema.transportation:Segment" - +connector = "overture.schema.transportation:Connector" +segment = "overture.schema.transportation:Segment" + [[examples.Connector]] id = "39542bee-230f-4b91-b7e5-a9b58e0c59b1" geometry = "POINT (-176.5472979 -43.9679472)" diff --git a/packages/overture-schema/src/overture/schema/__init__.py b/packages/overture-schema/src/overture/schema/__init__.py index 1f75ea511..2f5a0b7a0 100644 --- a/packages/overture-schema/src/overture/schema/__init__.py +++ b/packages/overture-schema/src/overture/schema/__init__.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, Tag, TypeAdapter from overture.schema.core import OvertureFeature -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.system.feature import Feature From 29a1943e141e6ce8d7a69ff6c306ad4955b474a2 Mon Sep 17 00:00:00 2001 From: Roel Bollens <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:11:31 +0100 Subject: [PATCH 2/9] chore(system): use logger.warning instead of print Co-authored-by: Seth Fitzsimmons Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/system/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 5231c737d..56ec05f3d 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -76,7 +76,7 @@ def generate_tags( filtered_tags = _filter_tags(added_tags, provider_key.package_name) tags.update(filtered_tags) except Exception as e: - print( + logger.warning( f"Error in tag provider {provider.__name__} for model {key.name}: {e}" ) From ecf3796424d8d191c9e6e19978087057a81301c0 Mon Sep 17 00:00:00 2001 From: Roel Bollens <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:13:51 +0100 Subject: [PATCH 3/9] refactor(core): simplify using direct boolean return Co-authored-by: Seth Fitzsimmons Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/core/tag_providers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index de52abf93..52cf5e241 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -52,9 +52,7 @@ def theme_provider( def _matches_manifest(key: ModelKey) -> bool: - if key.entry_point in APPROVED: - return True - return False + return key.entry_point in APPROVED: def _reduce_types(tp: Any) -> set[type]: From 79c3a806196797cd24a7da71487f97fbe9af6d96 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:07:53 +0100 Subject: [PATCH 4/9] chore(system): update ModelKey docstring to reflect changed attributes Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/system/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 56ec05f3d..98f3402cc 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -24,7 +24,7 @@ @dataclass(frozen=True, slots=True) class ModelKey: - """Key identifying a registered model by namespace, theme, and type. + """Key identifying a registered model by name, entry point, and tags. Attributes ---------- From 097c9ca7e5cc1b5463017956b118caad73ec1c4e Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Sun, 15 Mar 2026 21:59:15 +0100 Subject: [PATCH 5/9] refactor(system,core): Removes deferred tag provider and corrects tag filtering logic - Removes overture tag provider (was deferred) - Simplified tags - Reserved tags instead of reserved namespaces - Fixes small issue introduced in earlier commit Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- packages/overture-schema-core/pyproject.toml | 1 - .../src/overture/schema/core/tag_providers.py | 13 ++------ .../src/overture/schema/system/discovery.py | 33 +++++-------------- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/packages/overture-schema-core/pyproject.toml b/packages/overture-schema-core/pyproject.toml index ceee55a91..183c9e352 100644 --- a/packages/overture-schema-core/pyproject.toml +++ b/packages/overture-schema-core/pyproject.toml @@ -38,6 +38,5 @@ dev = [ ] [project.entry-points."overture.tag_providers"] -overture = "overture.schema.core.tag_providers:overture_provider" authority = "overture.schema.core.tag_providers:authority_provider" theme = "overture.schema.core.tag_providers:theme_provider" diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 52cf5e241..08cfafca8 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -21,6 +21,7 @@ "overture.schema.places:Place", "overture.schema.transportation:Connector", "overture.schema.transportation:Segment", + "overture.schema.annex:Sources", } @@ -28,15 +29,7 @@ def authority_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: if _matches_manifest(key): - tags.add("overture:official") - return tags - - -def overture_provider( - model_class: type[BaseModel], key: ModelKey, tags: set[str] -) -> set[str]: - if any(issubclass(tp, OvertureFeature) for tp in _reduce_types(model_class)): - tags.add("overture:feature") + tags.add("overture") return tags @@ -52,7 +45,7 @@ def theme_provider( def _matches_manifest(key: ModelKey) -> bool: - return key.entry_point in APPROVED: + return key.entry_point in APPROVED def _reduce_types(tp: Any) -> set[type]: diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 98f3402cc..58097724e 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -73,7 +73,7 @@ def generate_tags( for provider_key, provider in providers.items(): try: added_tags = provider(model_class, key, tags.copy()).difference(tags) - filtered_tags = _filter_tags(added_tags, provider_key.package_name) + filtered_tags = _filter_tags(added_tags, provider_key) tags.update(filtered_tags) except Exception as e: logger.warning( @@ -83,17 +83,17 @@ def generate_tags( return tags -def _filter_tags(tags: set[str], package: str) -> set[str]: - reserved_namespaces = tuple( - f"{namespace}:" - for namespace, reserved_packages in RESERVED_TAGS.items() - if package not in reserved_packages +def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: + reserved_tags = tuple( + tag + for tag, dist in RESERVED_TAGS.items() + if provider.package_name not in dist ) return { tag for tag in tags - if TAG_RE.match(tag) and not tag.startswith(reserved_namespaces) + if TAG_RE.match(tag) and not tag in reserved_tags } @@ -155,11 +155,7 @@ def discover_models( key = ModelKey( name=model.name, entry_point=model.value, - tags=( - frozenset(_distribution_tags(model.dist)) - if model.dist - else frozenset() - ), + tags=frozenset(), ) try: @@ -183,17 +179,6 @@ def discover_models( return models -def _distribution_tags(dist: importlib.metadata.Distribution) -> set[str]: - """Extract tags from the distribution metadata.""" - tags = set() - if dist.name: - tags.add("dist:name=" + dist.name) - if dist.version: - tags.add("dist:version=" + dist.version) - - return tags - - def get_registered_model(feature_type: str) -> type[BaseModel] | None: """Get the Pydantic model for a type. @@ -244,7 +229,7 @@ def feature_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: if any(issubclass(tp, Feature) for tp in _extract_types(model_class)): - tags.add("system:feature") + tags.add("feature") return tags From 3e9b61bee4d8085a870cbff5a5bbeaefbb604150 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:58:32 +0100 Subject: [PATCH 6/9] chore(system, core): fixes linting/formatting issues Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/cli/commands.py | 31 +++++++++++++------ .../tests/test_resolve_types.py | 5 ++- .../src/overture/schema/codegen/cli.py | 2 +- .../tests/codegen_test_support.py | 2 +- .../src/overture/schema/core/tag_providers.py | 10 +++--- .../tests/test_approved_models.py | 7 ++--- .../src/overture/schema/system/discovery.py | 21 +++++-------- .../overture-schema-system/tests/test_tags.py | 6 ++++ 8 files changed, 48 insertions(+), 36 deletions(-) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 0c36007ba..33df762b1 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -228,7 +228,7 @@ def resolve_types( elif theme_names and not type_names: # Theme-only mode: all types in specified themes for key, model_class in all_models.items(): - if next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: + if next(iter(tags_by_key(key.tags, "overture:theme")), None) in theme_names: filtered_models[key] = model_class elif type_names and not theme_names: @@ -240,7 +240,11 @@ def resolve_types( elif type_names and theme_names: # Both specified: find matching types within specified themes for key, model_class in all_models.items(): - if key.name in type_names and next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: + if ( + key.name in type_names + and next(iter(tags_by_key(key.tags, "overture:theme")), None) + in theme_names + ): filtered_models[key] = model_class else: @@ -789,9 +793,7 @@ def json_schema_command( help="Group types by tag prefix (e.g., 'overture:theme')", ) def list_types( - tags: tuple[str, ...], - excluded_tags: tuple[str, ...], - group_by: str | None + tags: tuple[str, ...], excluded_tags: tuple[str, ...], group_by: str | None ) -> None: r"""List all available types grouped by theme with descriptions. @@ -810,7 +812,9 @@ def list_types( if tags: filters.append(lambda key: all(tag in key.tags for tag in tags)) if excluded_tags: - filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + filters.append( + lambda key: not any(tag in key.tags for tag in excluded_tags) + ) if filters: models = { @@ -822,15 +826,23 @@ def list_types( if group_by: grouped_models: dict[str, set[ModelKey]] = {} - for key, model_class in models.items(): + for key in models.keys(): if groups := tags_by_key(key.tags, group_by): for group in groups: grouped_models.setdefault(group, set()).add(key) - padding = max((len(key.name) for keys in grouped_models.values() for key in keys), default=0) + 2 + padding = ( + max( + (len(key.name) for keys in grouped_models.values() for key in keys), + default=0, + ) + + 2 + ) for group, keys in sorted(grouped_models.items()): - stdout.print(f"[green bold]{group_by}={group} ({len(keys)})[/green bold]") + stdout.print( + f"[green bold]{group_by}={group} ({len(keys)})[/green bold]" + ) for key in sorted(keys, key=lambda k: k.name): model = Text() model.append("→ ", style="bright_black") @@ -850,7 +862,6 @@ def list_types( model.append_text(Text().append(" ".join(sorted(key.tags)))) stdout.print(model) - except Exception as e: click.echo(f"Error listing types: {e}", err=True) diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 19ce9592f..976e17416 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -128,7 +128,10 @@ def test_resolve_types_returns_expected_themes( from overture.schema.system.discovery import discover_models models = discover_models() - actual_themes = {next(iter(tags_by_key(key.tags, "overture:theme")),None) for key in models.keys()} + actual_themes = { + next(iter(tags_by_key(key.tags, "overture:theme")), None) + for key in models.keys() + } # Check that we have at least the expected themes (may have more) assert expected_themes.issubset(actual_themes), ( diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index ced8932ac..8624bfc7a 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -49,7 +49,7 @@ def _write_output( def _find_theme(tags: frozenset[str]) -> str | None: """Find the theme tag in a set of tags, if any.""" - return next(iter(tags_by_key(tags, "overture:theme")),None) + return next(iter(tags_by_key(tags, "overture:theme")), None) @click.group() diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 79881ed5b..4ba7c0e3b 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -303,7 +303,7 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec: def find_theme(tags: frozenset[str]) -> str | None: """Extract the theme from a set of tags, if present.""" - return next(iter(tags_by_key(tags, "overture:theme")),None) + return next(iter(tags_by_key(tags, "overture:theme")), None) T = TypeVar("T") diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 08cfafca8..e4e9a72d6 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -1,8 +1,8 @@ -from typing import Any, get_origin, Annotated, get_args, Union, Literal +from typing import Annotated, Any, Literal, Union, get_args, get_origin -from overture.schema.core import OvertureFeature from pydantic import BaseModel +from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ModelKey APPROVED = { @@ -36,7 +36,7 @@ def authority_provider( def theme_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - for tp in _reduce_types(model_class): + for tp in _extract_types(model_class): if issubclass(tp, OvertureFeature): tags.add( "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] @@ -48,10 +48,10 @@ def _matches_manifest(key: ModelKey) -> bool: return key.entry_point in APPROVED -def _reduce_types(tp: Any) -> set[type]: +def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 result: set[type] = set() - def visit(t: Any) -> None: + def visit(t: Any) -> None: # noqa: ANN401 origin = get_origin(t) if origin is Annotated: visit(get_args(t)[0]) diff --git a/packages/overture-schema-core/tests/test_approved_models.py b/packages/overture-schema-core/tests/test_approved_models.py index 6d7108610..d7ccb9830 100644 --- a/packages/overture-schema-core/tests/test_approved_models.py +++ b/packages/overture-schema-core/tests/test_approved_models.py @@ -1,4 +1,3 @@ -import pytest from overture.schema.system.discovery import discover_models @@ -6,6 +5,6 @@ def test_overture_feature_models_are_official() -> None: models = discover_models() for key in models: if "overture:feature" in key.tags: - assert ( - "overture:official" in key.tags - ), f"Model {key.name} is missing 'overture:official' tag." + assert "overture:official" in key.tags, ( + f"Model {key.name} is missing 'overture:official' tag." + ) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 58097724e..1141f08f2 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -3,8 +3,9 @@ import importlib.metadata import logging import re +from collections.abc import Callable from dataclasses import dataclass, replace -from typing import get_args, Literal, Union, get_origin, Annotated, Any, Callable +from typing import Annotated, Any, Literal, Union, get_args, get_origin from pydantic import BaseModel @@ -38,7 +39,7 @@ class ModelKey: """ name: str # friendly name from entry point key - entry_point : str # The entry point value in "module:Class" format + entry_point: str # The entry point value in "module:Class" format tags: frozenset[str] # plain and structured tags @@ -85,16 +86,10 @@ def generate_tags( def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: reserved_tags = tuple( - tag - for tag, dist in RESERVED_TAGS.items() - if provider.package_name not in dist + tag for tag, dist in RESERVED_TAGS.items() if provider.package_name not in dist ) - return { - tag - for tag in tags - if TAG_RE.match(tag) and not tag in reserved_tags - } + return {tag for tag in tags if TAG_RE.match(tag) and tag not in reserved_tags} def discover_tag_providers( @@ -104,7 +99,6 @@ def discover_tag_providers( try: for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): - try: tag_provider_class = tag_provider.load() @@ -148,7 +142,6 @@ def discover_models( try: for model in importlib.metadata.entry_points(group=model_group): - try: model_class = model.load() @@ -233,10 +226,10 @@ def feature_provider( return tags -def _extract_types(tp: Any) -> set[type]: +def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 result: set[type] = set() - def visit(t: Any) -> None: + def visit(t: Any) -> None: # noqa: ANN401 origin = get_origin(t) if origin is Annotated: visit(get_args(t)[0]) diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index 0fc88f69b..c7fc28c9b 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -1,35 +1,41 @@ from overture.schema.system.discovery import tags_by_key, tags_by_namespace + def test_tags_by_key_returns_correct_values() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "overture:theme" result = tags_by_key(tags, key) assert result == {"buildings"} + def test_tags_by_key_returns_empty_set_for_nonexistent_key() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "nonexistent:key" result = tags_by_key(tags, key) assert result == set() + def test_tags_by_key_handles_empty_tags() -> None: tags: frozenset[str] = frozenset() key = "overture:theme" result = tags_by_key(tags, key) assert result == set() + def test_tags_by_namespace_returns_correct_values() -> None: tags = frozenset({"system:extension", "overture"}) namespace = "system" result = tags_by_namespace(tags, namespace) assert result == {"extension"} + def test_tags_by_namespace_returns_empty_set_for_nonexistent_namespace() -> None: tags = frozenset({"system:extension", "overture"}) namespace = "nonexistent" result = tags_by_namespace(tags, namespace) assert result == set() + def test_tags_by_namespace_handles_empty_tags() -> None: tags: frozenset[str] = frozenset() namespace = "system" From ab6a2748d130c4e25cfcf61d6e12265cdc4fb28b Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Tue, 17 Mar 2026 20:51:12 +0100 Subject: [PATCH 7/9] refactor(codegen, cli): replace theme filtering with tag filtering in CLI commands Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/cli/commands.py | 149 +++------- .../tests/test_cli_commands.py | 14 +- .../tests/test_cli_functions.py | 12 +- .../tests/test_error_formatting.py | 4 +- .../tests/test_resolve_types.py | 273 +++++++++--------- .../src/overture/schema/codegen/cli.py | 29 +- .../overture-schema-codegen/tests/test_cli.py | 34 ++- .../src/overture/schema/system/discovery.py | 23 ++ 8 files changed, 255 insertions(+), 283 deletions(-) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 33df762b1..3b298c130 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -18,7 +18,12 @@ from yamlcore import CoreLoader # type: ignore from overture.schema.core import OvertureFeature -from overture.schema.system.discovery import ModelKey, discover_models, tags_by_key +from overture.schema.system.discovery import ( + ModelKey, + discover_models, + filter_models, + tags_by_key, +) from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema @@ -189,72 +194,32 @@ def validate_features(data: list, model_type: UnionType) -> list[BaseModel]: def resolve_types( - use_overture_types: bool, - namespace: str | None, - theme_names: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], type_names: tuple[str, ...], ) -> UnionType: """Resolve CLI options into a model type suitable for parse_feature. Args ---- - use_overture_types: Boolean from --overture-types flag - namespace: Namespace to filter by (e.g., "overture", "annex") - theme_names: List of theme names from --theme option + tags: Tags to include (e.g., "feature", "overture:theme=buildings") + excluded_tags: Tags to exclude (e.g., "draft") type_names: List of type names from --type option Returns ------- Model type suitable for passing to parse_feature """ - # Discover models once with the appropriate namespace - all_models = discover_models() + # Discover models + models: ModelDict = discover_models() # Filter models based on CLI options - filtered_models: ModelDict = {} - - if namespace and namespace != "overture": - filtered_models = { - key: model_class - for key, model_class in all_models.items() - if namespace in key.tags - } - - if use_overture_types: - for key, model_class in all_models.items(): - if tags_by_key(key.tags, "overture:theme"): - filtered_models[key] = model_class - - elif theme_names and not type_names: - # Theme-only mode: all types in specified themes - for key, model_class in all_models.items(): - if next(iter(tags_by_key(key.tags, "overture:theme")), None) in theme_names: - filtered_models[key] = model_class - - elif type_names and not theme_names: - # Type-only mode: find matching types across all themes - for key, model_class in all_models.items(): - if key.name in type_names and tags_by_key(key.tags, "overture:theme"): - filtered_models[key] = model_class - - elif type_names and theme_names: - # Both specified: find matching types within specified themes - for key, model_class in all_models.items(): - if ( - key.name in type_names - and next(iter(tags_by_key(key.tags, "overture:theme")), None) - in theme_names - ): - filtered_models[key] = model_class - - else: - # No filters specified - use all models - filtered_models = all_models + models = filter_models(models, tags, excluded_tags, type_names) - if not filtered_models: + if not models: raise ValueError("No models found matching the specified criteria") - return create_union_type_from_models(filtered_models) + return create_union_type_from_models(models) def get_source_name(filename: Path) -> str: @@ -290,10 +255,10 @@ def cli() -> None: $ overture-schema list-types \b # Generate JSON schema - $ overture-schema json-schema --theme buildings + $ overture-schema json-schema --tag overture:theme=buildings \b # Validate specific types - $ overture-schema validate --theme buildings data.json + $ overture-schema validate --tag overture:theme=buildings data.json """ pass @@ -536,7 +501,7 @@ def handle_validation_error( style="yellow", ) stderr.print( - " • Consider validating each type separately with --theme or --type", + " • Consider validating each type separately with --tag or --type", style="dim", ) stderr.print() @@ -557,7 +522,7 @@ def handle_validation_error( style="yellow", ) stderr.print( - " • Specifying --theme or --type to narrow validation", style="dim" + " • Specifying --tag or --type to narrow validation", style="dim" ) stderr.print(" • Adding discriminator fields to clarify intent", style="dim") stderr.print() @@ -637,18 +602,16 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: @cli.command() @click.argument("filename", type=click.Path(path_type=Path), required=True) @click.option( - "--overture-types", - is_flag=True, - help="Validate against all official Overture types (excludes extensions)", -) -@click.option( - "--namespace", - help="Namespace to filter by (e.g., overture, annex)", + "--tag", + "tags", + multiple=True, + help="Tags to include (e.g., overture:theme=addresses)", ) @click.option( - "--theme", + "--exclude-tag", + "excluded_tags", multiple=True, - help="Theme to validate against (shorthand for all types in theme)", + help="Tags to exclude (e.g., overture:theme=base)", ) @click.option( "--type", @@ -664,9 +627,8 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: ) def validate( filename: Path, - overture_types: bool, - namespace: str | None, - theme: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], types: tuple[str, ...], show_fields: tuple[str, ...], ) -> None: @@ -684,17 +646,17 @@ def validate( $ overture-schema validate - < data.json \b # Validate only buildings - $ overture-schema validate --theme buildings data.json + $ overture-schema validate --tag overture:theme=buildings data.json \b # Validate specific type $ overture-schema validate --type building data.json \b # Official Overture types only - $ overture-schema validate --overture-types data.json + $ overture-schema validate --tag overture --tag feature data.json """ # Resolve model type first (errors here are ValueErrors, not ValidationErrors) try: - model_type = resolve_types(overture_types, namespace, theme, types) + model_type = resolve_types(tags, excluded_tags, types) except ValueError as e: handle_generic_error(e, filename, "value") return @@ -722,18 +684,16 @@ def validate( @cli.command("json-schema") @click.option( - "--overture-types", - is_flag=True, - help="Generate schema for all official Overture types (excludes extensions)", -) -@click.option( - "--namespace", - help="Namespace to filter by (e.g., overture, annex)", + "--tag", + "tags", + multiple=True, + help="Tags to include (e.g., overture:theme=addresses)", ) @click.option( - "--theme", + "--exclude-tag", + "excluded_tags", multiple=True, - help="Theme to generate schema for (shorthand for all types in theme)", + help="Tags to exclude (e.g., overture:theme=base)", ) @click.option( "--type", @@ -742,9 +702,8 @@ def validate( help="Specific type to generate schema for (e.g., building, segment)", ) def json_schema_command( - overture_types: bool, - namespace: str | None, - theme: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], types: tuple[str, ...], ) -> None: r"""Generate JSON schema for Overture Maps types. @@ -757,17 +716,17 @@ def json_schema_command( # All types $ overture-schema json-schema > schema.json \b - # Buildings theme - $ overture-schema json-schema --theme buildings + # Buildings theme by tag + $ overture-schema json-schema --tag overture:theme=buildings \b # Specific types $ overture-schema json-schema --type building \b # Official Overture types only - $ overture-schema json-schema --overture-types + $ overture-schema json-schema --tag overture --tag feature """ try: - model_type = resolve_types(overture_types, namespace, theme, types) + model_type = resolve_types(tags, excluded_tags, types) schema = json_schema(model_type) # Use plain print for JSON output to avoid Rich formatting print(json.dumps(schema, indent=2, sort_keys=True)) @@ -786,7 +745,7 @@ def json_schema_command( "--exclude-tag", "excluded_tags", multiple=True, - help="Filter types by tag (e.g., overture:theme=base)", + help="Exclude types by tag (e.g., overture:theme=base)", ) @click.option( "--group-by", @@ -797,8 +756,7 @@ def list_types( ) -> None: r"""List all available types grouped by theme with descriptions. - Displays all registered Overture Maps types organized by theme, - including model class names and docstrings. + Displays all registered Overture Maps types and can organized by grouping. \b Examples: @@ -807,21 +765,8 @@ def list_types( """ try: models = discover_models() - filters = [] - - if tags: - filters.append(lambda key: all(tag in key.tags for tag in tags)) - if excluded_tags: - filters.append( - lambda key: not any(tag in key.tags for tag in excluded_tags) - ) - if filters: - models = { - key: model - for key, model in models.items() - if all(f(key) for f in filters) - } + models = filter_models(models, tags=tags, excluded_tags=excluded_tags) if group_by: grouped_models: dict[str, set[ModelKey]] = {} diff --git a/packages/overture-schema-cli/tests/test_cli_commands.py b/packages/overture-schema-cli/tests/test_cli_commands.py index 7b6f3b42f..6c18d58ca 100644 --- a/packages/overture-schema-cli/tests/test_cli_commands.py +++ b/packages/overture-schema-cli/tests/test_cli_commands.py @@ -16,8 +16,8 @@ def test_list_types_command(self, cli_runner: CliRunner) -> None: """Test the list-types command.""" result = cli_runner.invoke(cli, ["list-types"]) assert result.exit_code == 0 - # Should show theme names - assert "BUILDINGS" in result.output or "buildings" in result.output + # Should show theme tag + assert "overture:theme=buildings" in result.output # Should show type names assert "building" in result.output @@ -33,7 +33,9 @@ class TestJsonSchemaCommand: def test_json_schema_generates_valid_output(self, cli_runner: CliRunner) -> None: """Test that json-schema command generates valid JSON.""" - result = cli_runner.invoke(cli, ["json-schema", "--theme", "buildings"]) + result = cli_runner.invoke( + cli, ["json-schema", "--tag", "overture:theme=buildings"] + ) assert result.exit_code == 0 # Should be valid JSON @@ -57,7 +59,7 @@ def test_validate_flat_format_input(self, cli_runner: CliRunner) -> None: flat_feature = build_feature(geojson_format=False) flat_json = json.dumps(flat_feature) result = cli_runner.invoke( - cli, ["validate", "--theme", "buildings", "-"], input=flat_json + cli, ["validate", "--tag", "overture:theme=buildings", "-"], input=flat_json ) assert result.exit_code == 0 assert "Successfully validated " in result.output @@ -222,7 +224,7 @@ def test_validate_with_nonexistent_filters_raises_error( # Try to validate with a nonexistent theme result = cli_runner.invoke( cli, - ["validate", "--theme", "nonexistent_theme", "-"], + ["validate", "--tag", "overture:theme=nonexistent_theme", "-"], input=building_feature_yaml_content, ) # UsageError exits with code 2 @@ -254,7 +256,7 @@ def test_validate_with_valid_theme_invalid_type_raises_error( # Try to validate buildings theme with a type that doesn't exist in that theme result = cli_runner.invoke( cli, - ["validate", "--theme", "buildings", "--type", "segment", "-"], + ["validate", "--tag", "overture:theme=buildings", "--type", "segment", "-"], input=building_feature_yaml_content, ) # UsageError exits with code 2 diff --git a/packages/overture-schema-cli/tests/test_cli_functions.py b/packages/overture-schema-cli/tests/test_cli_functions.py index 4218541f5..8c17ccd36 100644 --- a/packages/overture-schema-cli/tests/test_cli_functions.py +++ b/packages/overture-schema-cli/tests/test_cli_functions.py @@ -203,7 +203,7 @@ class TestPerformValidation: def test_perform_validation_raises_for_invalid_single_feature(self) -> None: """Test that perform_validation raises ValidationError for single invalid feature.""" data = build_feature(id=None) # Missing required 'id' - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -218,7 +218,7 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: id=None, coordinates=[[[2, 2], [3, 2], [3, 3], [2, 3], [2, 2]]] ) data = [feature1, feature2] - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -230,7 +230,7 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: def test_perform_validation_empty_list(self) -> None: """Test validating an empty list (edge case).""" data: list[dict[str, object]] = [] - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) # Should not raise perform_validation(data, model_type) @@ -238,7 +238,7 @@ def test_perform_validation_empty_list(self) -> None: def test_perform_validation_empty_feature_collection(self) -> None: """Test validating an empty FeatureCollection (edge case).""" data = {"type": "FeatureCollection", "features": []} - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) # Should not raise perform_validation(data, model_type) @@ -248,10 +248,10 @@ def test_perform_validation_with_different_themes(self) -> None: data = build_feature(theme="buildings", type="building") # Should work with buildings theme - buildings_type = resolve_types(False, None, ("buildings",), ()) + buildings_type = resolve_types(("overture:theme=buildings",), (), ()) perform_validation(data, buildings_type) # Should fail with wrong theme - places_type = resolve_types(False, None, ("places",), ()) + places_type = resolve_types(("overture:theme=places",), (), ()) with pytest.raises(ValidationError): perform_validation(data, places_type) diff --git a/packages/overture-schema-cli/tests/test_error_formatting.py b/packages/overture-schema-cli/tests/test_error_formatting.py index fa8ef1a5d..168d9d53c 100644 --- a/packages/overture-schema-cli/tests/test_error_formatting.py +++ b/packages/overture-schema-cli/tests/test_error_formatting.py @@ -40,7 +40,9 @@ def test_ambiguous_data_shows_most_likely_errors( version: 0 """) - result = cli_runner.invoke(cli, ["validate", "--theme", "buildings", filename]) + result = cli_runner.invoke( + cli, ["validate", "--tag", "overture:theme=buildings", filename] + ) assert result.exit_code == 1 diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 976e17416..cd2c04594 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -1,173 +1,172 @@ """Parametrized tests for resolve_types function.""" +from typing import get_args +from unittest.mock import Mock, patch + import pytest from overture.schema.cli.commands import resolve_types -from overture.schema.system.discovery import tags_by_key +from overture.schema.system.discovery import ModelKey +DISCOVER_MODELS = "overture.schema.cli.commands.discover_models" -class TestResolveTypes: - """Tests for the resolve_types function with various filter combinations.""" +# Mock model classes +class Place: + pass + + +class Segment: + pass + + +class Connector: + pass + + +class Building: + pass + + +class Sources: + pass + + +# Mock ModelKey instances +BUILDING_KEY = ModelKey( + name="building", + entry_point="mock:MyClass", + tags=frozenset({"feature", "overture", "overture:theme=buildings"}), +) +SEGMENT_KEY = ModelKey( + name="segment", + entry_point="mock:Segment", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +CONNECTOR_KEY = ModelKey( + name="connector", + entry_point="mock:Connector", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +PLACE_KEY = ModelKey( + name="place", + entry_point="mock:Place", + tags=frozenset({"feature", "overture", "overture:theme=places"}), +) +SOURCES_KEY = ModelKey( + name="sources", + entry_point="mock:Sources", + tags=frozenset({"overture"}), +) + +MOCK_MODELS = { + BUILDING_KEY: Building, + SEGMENT_KEY: Segment, + CONNECTOR_KEY: Connector, + PLACE_KEY: Place, + SOURCES_KEY: Sources, +} + + +class TestResolveTypes: @pytest.mark.parametrize( - "overture_types,namespace,theme_names,type_names,should_succeed", + "tags,excluded_tags,type_names,should_succeed", [ - # Test --overture-types flag - pytest.param(True, None, (), (), True, id="overture_types_only"), - pytest.param(False, "overture", (), (), True, id="overture_namespace"), - # Test theme filtering - pytest.param(False, None, ("buildings",), (), True, id="theme_buildings"), pytest.param( - False, None, ("transportation",), (), True, id="theme_transportation" + ("overture:theme=buildings",), (), (), True, id="tag_buildings" ), pytest.param( - False, None, ("buildings", "places"), (), True, id="multiple_themes" - ), - pytest.param(False, None, ("nonexistent",), (), False, id="invalid_theme"), - # Test type filtering - pytest.param(False, None, (), ("building",), True, id="type_building"), - pytest.param(False, None, (), ("segment",), True, id="type_segment"), - pytest.param( - False, None, (), ("building", "place"), True, id="multiple_types" + ("overture:theme=transportation",), + (), + (), + True, + id="tag_transportation", ), - pytest.param(False, None, (), ("nonexistent",), False, id="invalid_type"), - # Test combined theme + type filtering + pytest.param(("overture:theme=places",), (), (), True, id="tag_places"), + pytest.param(("nonexistent",), (), (), False, id="unknown_tag"), + pytest.param((), (), ("building",), True, id="type_building"), + pytest.param((), (), ("segment",), True, id="type_segment"), + pytest.param((), (), ("nonexistent",), False, id="invalid_type"), pytest.param( - False, - None, - ("buildings",), + ("overture:theme=buildings",), + (), ("building",), True, - id="theme_and_type_match", + id="tag_and_type_match", ), pytest.param( - False, - None, - ("buildings",), + ("overture:theme=buildings",), + (), ("segment",), False, - id="theme_and_type_mismatch", + id="tag_and_type_mismatch", ), pytest.param( - False, - None, - ("transportation",), - ("segment", "connector"), - True, - id="theme_with_multiple_types", - ), - # Test namespace combined with theme/type - pytest.param( - False, - "overture", - ("buildings",), + ("overture:theme=transportation",), (), + ("segment", "connector"), True, - id="namespace_with_theme", - ), - pytest.param( - False, - "overture", - (), - ("building",), - True, - id="namespace_with_type", - ), - pytest.param( - False, - "overture", - ("buildings",), - ("building",), - True, - id="namespace_with_theme_and_type", + id="tag_with_multiple_types", ), - # Test no filters (all models) - pytest.param(False, None, (), (), True, id="no_filters_all_models"), + pytest.param((), (), (), True, id="no_filters_all_models"), ], ) def test_resolve_types_combinations( self, - overture_types: bool, - namespace: str | None, - theme_names: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], type_names: tuple[str, ...], should_succeed: bool, ) -> None: - """Test resolve_types with various filter combinations.""" - if should_succeed: - model_type = resolve_types( - overture_types, namespace, theme_names, type_names - ) - assert model_type is not None - else: - with pytest.raises(ValueError, match="No models found"): - resolve_types(overture_types, namespace, theme_names, type_names) - - @pytest.mark.parametrize( - "namespace,expected_themes", - [ - pytest.param( - "overture", - { - "buildings", - "places", - "transportation", - "base", - "divisions", - "addresses", - }, - id="overture_namespace", - ), - ], - ) - def test_resolve_types_returns_expected_themes( - self, - namespace: str, - expected_themes: set[str], - ) -> None: - """Test that resolve_types returns models from expected themes.""" - from overture.schema.system.discovery import discover_models - - models = discover_models() - actual_themes = { - next(iter(tags_by_key(key.tags, "overture:theme")), None) - for key in models.keys() - } - - # Check that we have at least the expected themes (may have more) - assert expected_themes.issubset(actual_themes), ( - f"Missing expected themes. Expected {expected_themes}, got {actual_themes}" - ) - - -class TestResolveTypesEdgeCases: - """Tests for edge cases in resolve_types.""" + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + if should_succeed: + union = resolve_types(tags, excluded_tags, type_names) + assert union is not None + else: + with pytest.raises(ValueError, match="No models found"): + resolve_types(tags, excluded_tags, type_names) def test_resolve_types_case_sensitive(self) -> None: - """Test that theme and type names are case-sensitive.""" - # Lowercase should work - model_type = resolve_types(False, None, ("buildings",), ()) - assert model_type is not None - - # Uppercase should fail (themes are lowercase in registry) - with pytest.raises(ValueError, match="No models found"): - resolve_types(False, None, ("BUILDINGS",), ()) + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + # Lowercase should work + union = resolve_types((), (), ("building",)) + assert union is not None + # Uppercase should fail + with pytest.raises(ValueError, match="No models found"): + resolve_types((), (), ("BUILDING",)) def test_resolve_types_empty_result_error_message(self) -> None: - """Test that a helpful error message is shown when no models match.""" - with pytest.raises(ValueError) as exc_info: - resolve_types(False, None, ("nonexistent",), ("also_fake",)) - - assert "No models found" in str(exc_info.value) - - def test_resolve_types_namespace_isolation(self) -> None: - """Test that namespace filtering properly isolates models.""" - # Get all models (no namespace filter) - all_models_type = resolve_types(False, None, (), ()) - assert all_models_type is not None - - # Get only overture namespace - overture_type = resolve_types(False, "overture", (), ()) - assert overture_type is not None - - # Both should work, but they represent different sets of models - # (This test primarily ensures no exceptions are raised) + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + with pytest.raises(ValueError) as exc_info: + resolve_types(("nonexistent",), (), ("also_fake",)) + assert "No models found" in str(exc_info.value) + + def test_resolve_types_excluded_tags(self) -> None: + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + # Exclude 'overture:theme=buildings' tag + union = resolve_types((), ("overture:theme=buildings",), ()) + # Should not include Building model + assert not any(issubclass(model, Mock) for model in get_args(union)) + + def test_resolve_types_no_filters_returns_all(self) -> None: + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + union = resolve_types((), (), ()) + # Should include all mock models + assert all( + any(issubclass(model, t) for model in getattr(union, "__args__", [])) + for t in [Building, Segment, Connector, Place, Sources] + ) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index 8624bfc7a..801c55fb0 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -6,7 +6,7 @@ import click -from overture.schema.system.discovery import discover_models, tags_by_key +from overture.schema.system.discovery import discover_models, filter_models from .extraction.model_extraction import extract_model from .extraction.specs import ( @@ -47,11 +47,6 @@ def _write_output( click.echo() # separate entries with a blank line in stdout mode -def _find_theme(tags: frozenset[str]) -> str | None: - """Find the theme tag in a set of tags, if any.""" - return next(iter(tags_by_key(tags, "overture:theme")), None) - - @click.group() def cli() -> None: """Overture Schema code generator. @@ -81,9 +76,16 @@ def list_models() -> None: help="Output format", ) @click.option( - "--theme", + "--tag", + "tags", + multiple=True, + help="Tag(s) to include; repeatable (e.g., --tag feature --tag overture)", +) +@click.option( + "--exclude-tag", + "excluded_tags", multiple=True, - help="Filter to specific theme(s); repeatable (e.g., --theme buildings --theme places)", + help="Tag(s) to exclude; repeatable (e.g., --exclude-tag draft --exclude-tag overture:theme=base)", ) @click.option( "--output-dir", @@ -93,21 +95,18 @@ def list_models() -> None: ) def generate( output_format: str, - theme: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], output_dir: Path | None, ) -> None: """Generate code/docs from discovered models.""" all_models = discover_models() - # Schema root from ALL entry points (before theme filter). + # Schema root from ALL entry points (before tag filters). module_paths = [entry_point_module(k.entry_point) for k in all_models] schema_root = compute_schema_root(module_paths) - models = ( - {k: v for k, v in all_models.items() if _find_theme(k.tags) in theme} - if theme - else all_models - ) + models = filter_models(all_models, tags=tags, excluded_tags=excluded_tags) if output_dir: output_dir.mkdir(parents=True, exist_ok=True) diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index eecd45627..0ca7061e7 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -48,10 +48,11 @@ def test_generate_markdown_to_stdout(self, cli_runner: CliRunner) -> None: assert result.exit_code == 0 assert "# Building" in result.output or "# " in result.output - def test_generate_with_theme_filter(self, cli_runner: CliRunner) -> None: - """generate --theme should filter to specific theme.""" + def test_generate_with_tag_filter(self, cli_runner: CliRunner) -> None: + """generate --tag should filter to specific theme.""" result = cli_runner.invoke( - cli, ["generate", "--format", "markdown", "--theme", "buildings"] + cli, + ["generate", "--format", "markdown", "--tag", "overture:theme=buildings"], ) assert result.exit_code == 0 @@ -68,8 +69,8 @@ def test_generate_markdown_feature_at_theme_level( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -93,8 +94,8 @@ def test_feature_pages_have_sidebar_position( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -211,8 +212,8 @@ def test_generates_category_files( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -311,8 +312,8 @@ def test_generate_markdown_includes_enum_files( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -344,7 +345,8 @@ def spy(feature_specs: list, schema_root: str, output_dir: object) -> None: monkeypatch.setattr("overture.schema.codegen.cli._generate_markdown", spy) result = cli_runner.invoke( - cli, ["generate", "--format", "markdown", "--theme", "buildings"] + cli, + ["generate", "--format", "markdown", "--tag", "overture:theme=buildings"], ) assert result.exit_code == 0 @@ -381,8 +383,8 @@ def test_segment_appears_in_markdown_output( "generate", "--format", "markdown", - "--theme", - "transportation", + "--tag", + "overture:theme=transportation", "--output-dir", str(tmp_path), ], @@ -411,8 +413,8 @@ def test_used_by_sections_appear_in_markdown( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 1141f08f2..d59c9ba5f 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -172,6 +172,29 @@ def discover_models( return models +def filter_models( + models: dict[ModelKey, type[BaseModel]], + tags: tuple[str, ...] = (), + excluded_tags: tuple[str, ...] = (), + type_names: tuple[str, ...] = (), +) -> dict[ModelKey, type[BaseModel]]: + """Filter models to those that contain all required tags.""" + filters = [] + + if tags: + filters.append(lambda key: all(tag in key.tags for tag in tags)) + if excluded_tags: + filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + if type_names: + filters.append(lambda key: key.name in type_names) + + if filters: + models = { + key: model for key, model in models.items() if all(f(key) for f in filters) + } + return models + + def get_registered_model(feature_type: str) -> type[BaseModel] | None: """Get the Pydantic model for a type. From 039c90722d43e1a3b98f599937d02af44f3d30e7 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:32:28 +0100 Subject: [PATCH 8/9] refactor(system): Adds reserved namespaces and logging to tag filtering Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/core/tag_providers.py | 2 +- .../src/overture/schema/system/discovery.py | 86 +++++++++++-- .../tests/test_tag_providers.py | 117 ++++++++++++++++++ .../overture-schema-system/tests/test_tags.py | 79 +++++++++++- 4 files changed, 269 insertions(+), 15 deletions(-) create mode 100644 packages/overture-schema-system/tests/test_tag_providers.py diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index e4e9a72d6..4eec8761c 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -37,7 +37,7 @@ def theme_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: for tp in _extract_types(model_class): - if issubclass(tp, OvertureFeature): + if isinstance(tp, type) and issubclass(tp, OvertureFeature): tags.add( "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] ) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index d59c9ba5f..5a3eca2dd 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -5,7 +5,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, replace -from typing import Annotated, Any, Literal, Union, get_args, get_origin +from typing import Annotated, Any, Literal, TypeAlias, Union, get_args, get_origin from pydantic import BaseModel @@ -18,6 +18,11 @@ "overture": {"overture-schema-core"}, "feature": {"overture-schema-system"}, } +RESERVED_NAMESPACES: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "system": {"overture-schema-system"}, +} + TAG = r"[a-z0-9][a-z0-9_-]*" NAMESPACE_TAG = r"[a-z0-9]+:[a-z0-9]+(?:=[a-z0-9_.-]+)?" TAG_RE = re.compile(rf"^(?:{TAG}|{NAMESPACE_TAG})$") @@ -61,13 +66,19 @@ class TagProviderKey: package_name: str # distribution package name -TagProvider = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] +TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] + +ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] + +TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] + +ModelKeyFilter: TypeAlias = Callable[[ModelKey], bool] def generate_tags( model_class: type[BaseModel], key: ModelKey, - providers: dict[TagProviderKey, TagProvider], + providers: TagProviderDict, ) -> set[str]: tags: set[str] = set() @@ -85,16 +96,53 @@ def generate_tags( def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: - reserved_tags = tuple( - tag for tag, dist in RESERVED_TAGS.items() if provider.package_name not in dist - ) + """Filter tags, removing invalid, reserved, or namespace-restricted tags for a provider.""" + filtered_tags: set[str] = set() + reserved_tags: set[str] = { + tag for tag, pkgs in RESERVED_TAGS.items() if provider.package_name not in pkgs + } + reserved_namespaces: set[str] = { + ns + for ns, pkgs in RESERVED_NAMESPACES.items() + if provider.package_name not in pkgs + } + + for tag in tags: + # Validate tag format + if not TAG_RE.fullmatch(tag): + logger.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " + f"This tag does not match the required format." + ) + continue + + # Reserved tag check + if tag in reserved_tags: + allowed_pkgs = RESERVED_TAGS.get(tag, set()) + logger.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " + f"This tag can only be set by packages from: {allowed_pkgs}." + ) + continue + + # Reserved namespace check + tag_ns = namespace(tag) + if tag_ns and tag_ns in reserved_namespaces: + allowed_pkgs = RESERVED_NAMESPACES.get(tag_ns, set()) + logger.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " + f"This namespace can only be set by packages from: {allowed_pkgs}." + ) + continue - return {tag for tag in tags if TAG_RE.match(tag) and tag not in reserved_tags} + filtered_tags.add(tag) + + return filtered_tags def discover_tag_providers( tag_providers_group: str = "overture.tag_providers", -) -> dict[TagProviderKey, TagProvider]: +) -> TagProviderDict: tag_providers = {} try: @@ -123,7 +171,7 @@ def discover_tag_providers( def discover_models( model_group: str = "overture.models", -) -> dict[ModelKey, type[BaseModel]]: +) -> ModelDict: """Discover all registered Overture models via entry points. Parameters @@ -173,13 +221,13 @@ def discover_models( def filter_models( - models: dict[ModelKey, type[BaseModel]], + models: ModelDict, tags: tuple[str, ...] = (), excluded_tags: tuple[str, ...] = (), type_names: tuple[str, ...] = (), -) -> dict[ModelKey, type[BaseModel]]: +) -> ModelDict: """Filter models to those that contain all required tags.""" - filters = [] + filters: list[ModelKeyFilter] = [] if tags: filters.append(lambda key: all(tag in key.tags for tag in tags)) @@ -221,6 +269,15 @@ def get_registered_model(feature_type: str) -> type[BaseModel] | None: return None +def namespace(tag: str) -> str: + """Extract the namespace from a tag, or return an empty string if there is no namespace.""" + if not TAG_RE.fullmatch(tag): + raise ValueError(f"Invalid tag format: {tag}") + if ":" in tag: + return tag.split(":")[0] + return "" + + def tags_by_key(tags: frozenset[str] | set[str], key: str) -> set[str]: """Extract values for k/v tags with the given key. @@ -244,7 +301,10 @@ def tags_by_namespace(tags: frozenset[str] | set[str], namespace: str) -> set[st def feature_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - if any(issubclass(tp, Feature) for tp in _extract_types(model_class)): + if any( + isinstance(tp, type) and issubclass(tp, Feature) + for tp in _extract_types(model_class) + ): tags.add("feature") return tags diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py new file mode 100644 index 000000000..e1cfc3ca9 --- /dev/null +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -0,0 +1,117 @@ +import pytest +from pydantic import BaseModel + +from overture.schema.system.discovery import ( + ModelKey, + TagProviderKey, + _filter_tags, + feature_provider, +) +from overture.schema.system.feature import Feature + + +@pytest.fixture +def core_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="core", entry_point="core:Provider", package_name="overture-schema-core" + ) + + +@pytest.fixture +def system_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="system", + entry_point="system:Provider", + package_name="overture-schema-system", + ) + + +@pytest.fixture +def other_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="other", entry_point="other:Provider", package_name="other-package" + ) + + +@pytest.fixture +def feature() -> type[Feature]: + class SomeFeature(Feature): + pass + + return SomeFeature + + +@pytest.fixture +def not_a_feature() -> type[BaseModel]: + class NotAFeature(BaseModel): + pass + + return NotAFeature + + +def test_valid_tags(other_tag_provider: TagProviderKey) -> None: + tags = {"valid", "other:valid", "other:valid=true"} + filtered = _filter_tags(tags, other_tag_provider) + assert filtered == tags + + +def test_invalid_tag(other_tag_provider: TagProviderKey) -> None: + tags = {"InvalidTag"} + filtered = _filter_tags(tags, other_tag_provider) + assert filtered == set() + + +def test_reserved_tag(other_tag_provider: TagProviderKey) -> None: + tags = {"overture", "feature", "valid"} + filtered = _filter_tags(tags, other_tag_provider) + assert "valid" in filtered + assert "overture" not in filtered + assert "feature" not in filtered + + +def test_allowed_reserved_tag( + core_tag_provider: TagProviderKey, system_tag_provider: TagProviderKey +) -> None: + assert "overture" in _filter_tags({"overture"}, core_tag_provider) + assert "feature" in _filter_tags({"feature"}, system_tag_provider) + + +def test_reserved_namespace(other_tag_provider: TagProviderKey) -> None: + tags = {"overture:feature", "system:feature", "valid:tag"} + filtered = _filter_tags(tags, other_tag_provider) + assert "valid:tag" in filtered + assert "overture:feature" not in filtered + assert "system:feature" not in filtered + + +def test_allowed_reserved_namespace( + core_tag_provider: TagProviderKey, system_tag_provider: TagProviderKey +) -> None: + assert "overture:feature" in _filter_tags({"overture:feature"}, core_tag_provider) + assert "system:feature" in _filter_tags({"system:feature"}, system_tag_provider) + + +def test_empty_tags(other_tag_provider: TagProviderKey) -> None: + assert _filter_tags(set(), other_tag_provider) == set() + + +def test_mixed_tags(other_tag_provider: TagProviderKey) -> None: + tags = {"valid", "feature", "overture:feature", "InvalidTag"} + filtered = _filter_tags(tags, other_tag_provider) + assert filtered == {"valid"} + + +def test_feature_provider_adds_feature_tag(feature: type[Feature]) -> None: + key = ModelKey(name="feature", entry_point="system:Feature", tags=frozenset()) + result = feature_provider(feature, key, set()) + assert "feature" in result + + +def test_feature_provider_does_not_add_feature_tag( + not_a_feature: type[BaseModel], +) -> None: + key = ModelKey( + name="notafeature", entry_point="system:NotAFeature", tags=frozenset() + ) + result = feature_provider(not_a_feature, key, set()) + assert "feature" not in result diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index c7fc28c9b..d5efa90c2 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -1,4 +1,13 @@ -from overture.schema.system.discovery import tags_by_key, tags_by_namespace +import re +import unittest + +from overture.schema.system.discovery import ( + NAMESPACE_TAG, + TAG, + TAG_RE, + tags_by_key, + tags_by_namespace, +) def test_tags_by_key_returns_correct_values() -> None: @@ -41,3 +50,71 @@ def test_tags_by_namespace_handles_empty_tags() -> None: namespace = "system" result = tags_by_namespace(tags, namespace) assert result == set() + + +class TestSimpleTagRegex(unittest.TestCase): + def test_valid_simple_tags(self) -> None: + valid_tags = [ + "v", + "valid", + "valid1", + "valid_tag", + "valid-tag", + "0valid", + "42", + ] + for tag in valid_tags: + self.assertTrue(re.fullmatch(TAG, tag), f"Should match: {tag}") + self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + + def test_invalid_simple_tags(self) -> None: + invalid_tags = [ + "", + "_invalid", + "-invalid", + "Invalid", + "invalid!", + "invalid ", + "in.valid", + "3.14", + ] + for tag in invalid_tags: + self.assertFalse(re.fullmatch(TAG, tag), f"Should not match: {tag}") + self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}") + + +class TestNamespaceTagRegex(unittest.TestCase): + def test_valid_namespace_tags(self) -> None: + valid_tags = [ + "ns:predicate", + "ns:predicate1", + "ns:predicate=value", + "ns:predicate=value_0", + "ns:predicate=value-0", + "ns:predicate=value.0", + "ns:predicate=value_2-3.4", + "ns:predicate=42", + "ns:predicate=3.14", + ] + for tag in valid_tags: + self.assertTrue(re.fullmatch(NAMESPACE_TAG, tag), f"Should match: {tag}") + self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + + def test_invalid_namespace_tags(self) -> None: + invalid_tags = [ + "ns:", + ":predicate", + "ns:predicate=", + "ns:predicate=Value", + "ns:predicate=value ", + "ns:predicate=value!", + "ns:predicate=ns:value", + "ns:predicate=predicate=value", + "Ns:predicate", + "ns:Predicate", + ] + for tag in invalid_tags: + self.assertFalse( + re.fullmatch(NAMESPACE_TAG, tag), f"Should not match: {tag}" + ) + self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}") From e9eabf34fb86904637db299d5fa22bc49b58e2d0 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Wed, 8 Apr 2026 16:55:45 +0200 Subject: [PATCH 9/9] refactor(system): tighten discovery api and various small improvements Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- README.pydantic.md | 6 +- .../src/overture/schema/cli/__init__.py | 2 - .../src/overture/schema/cli/commands.py | 21 +- .../src/overture/schema/cli/types.py | 5 - .../tests/test_resolve_types.py | 4 +- packages/overture-schema-codegen/README.md | 2 +- .../tests/codegen_test_support.py | 7 +- .../src/overture/schema/core/tag_providers.py | 70 ++-- .../tests/test_approved_models.py | 4 +- .../overture-schema-system/pyproject.toml | 2 +- .../src/overture/schema/system/discovery.py | 340 ------------------ .../schema/system/discovery/__init__.py | 13 + .../schema/system/discovery/discovery.py | 257 +++++++++++++ .../schema/system/discovery/models.py | 39 ++ .../overture/schema/system/discovery/tag.py | 104 ++++++ .../schema/system/discovery/tag_providers.py | 31 ++ .../overture/schema/system/discovery/types.py | 13 + .../src/overture/schema/system/typing_util.py | 43 +++ .../tests/test_tag_providers.py | 9 +- .../overture-schema-system/tests/test_tags.py | 85 +++-- 20 files changed, 606 insertions(+), 451 deletions(-) delete mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/models.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/tag.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/types.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/typing_util.py diff --git a/README.pydantic.md b/README.pydantic.md index 213426d43..dc19d8472 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -164,9 +164,9 @@ from overture.schema.system.discovery import discover_models, get_registered_mod all_models = discover_models() # Returns: # { -# ("building", "acme:Building", {"building_tag"}): BuildingModel, -# ("place", "acme:Place", {"place_tag"}): PlaceModel, -# ... +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", tags=frozenset({"feature", "overture", "overture:theme=buildings"})): BuildingModel, +# ModelKey(name="place", entry_point="overture.schema.places:Place", tags=frozenset({"feature", "overture", "overture:theme=places"})): PlaceModel, +# ... # } # Get a specific model by type diff --git a/packages/overture-schema-cli/src/overture/schema/cli/__init__.py b/packages/overture-schema-cli/src/overture/schema/cli/__init__.py index 85045f0c0..8fd3e8bfd 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/__init__.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/__init__.py @@ -11,7 +11,6 @@ ) from .types import ( ErrorLocation, - ModelDict, UnionType, ValidationErrorDict, ) @@ -25,7 +24,6 @@ "perform_validation", "resolve_types", "ErrorLocation", - "ModelDict", "UnionType", "ValidationErrorDict", ] diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 3b298c130..3e196818d 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -22,8 +22,9 @@ ModelKey, discover_models, filter_models, - tags_by_key, ) +from overture.schema.system.discovery.tag import get_values_for_key +from overture.schema.system.discovery.types import ModelDict from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema @@ -34,7 +35,7 @@ select_most_likely_errors, ) from .type_analysis import StructuralTuple, get_item_index, introspect_union -from .types import ErrorLocation, ModelDict, UnionType +from .types import ErrorLocation, UnionType # Console instances for rich output stdout = Console(highlight=False) @@ -214,7 +215,9 @@ def resolve_types( models: ModelDict = discover_models() # Filter models based on CLI options - models = filter_models(models, tags, excluded_tags, type_names) + models = filter_models( + models, tags=tags, excluded_tags=excluded_tags, type_names=type_names + ) if not models: raise ValueError("No models found matching the specified criteria") @@ -749,14 +752,14 @@ def json_schema_command( ) @click.option( "--group-by", - help="Group types by tag prefix (e.g., 'overture:theme')", + help="Group types by tag key (e.g., 'overture:theme')", ) def list_types( tags: tuple[str, ...], excluded_tags: tuple[str, ...], group_by: str | None ) -> None: - r"""List all available types grouped by theme with descriptions. + r"""List all available types. - Displays all registered Overture Maps types and can organized by grouping. + Displays all registered models and can be organized by grouping. \b Examples: @@ -772,7 +775,7 @@ def list_types( grouped_models: dict[str, set[ModelKey]] = {} for key in models.keys(): - if groups := tags_by_key(key.tags, group_by): + if groups := get_values_for_key(key.tags, group_by): for group in groups: grouped_models.setdefault(group, set()).add(key) @@ -793,7 +796,7 @@ def list_types( model.append("→ ", style="bright_black") model.append(key.name, style="bold cyan") model.pad_right(max(1, padding - len(key.name))) - model.append_text(Text().append(" ".join(sorted(key.tags)))) + model.append(" ".join(sorted(key.tags))) stdout.print(model) stdout.print() @@ -804,7 +807,7 @@ def list_types( model = Text() model.append(key.name, style="bold cyan") model.pad_right(max(1, padding - len(key.name))) - model.append_text(Text().append(" ".join(sorted(key.tags)))) + model.append(" ".join(sorted(key.tags))) stdout.print(model) except Exception as e: diff --git a/packages/overture-schema-cli/src/overture/schema/cli/types.py b/packages/overture-schema-cli/src/overture/schema/cli/types.py index f438edf2f..f1394def8 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/types.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/types.py @@ -5,15 +5,10 @@ from pydantic import BaseModel from pydantic_core import ErrorDetails -from overture.schema.system.discovery import ModelKey - # Type alias for union types created from Pydantic models # This represents either a single model or a discriminated union of models UnionType: TypeAlias = type[BaseModel] | Any -# Dictionary mapping ModelKey to Pydantic model classes -ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] - # Pydantic validation error dictionary structure # In Pydantic v2, ValidationError.errors() returns list[ErrorDetails] ValidationErrorDict: TypeAlias = ErrorDetails diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index cd2c04594..42b4a3c07 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -1,7 +1,7 @@ """Parametrized tests for resolve_types function.""" from typing import get_args -from unittest.mock import Mock, patch +from unittest.mock import patch import pytest from overture.schema.cli.commands import resolve_types @@ -157,7 +157,7 @@ def test_resolve_types_excluded_tags(self) -> None: # Exclude 'overture:theme=buildings' tag union = resolve_types((), ("overture:theme=buildings",), ()) # Should not include Building model - assert not any(issubclass(model, Mock) for model in get_args(union)) + assert not any(issubclass(model, Building) for model in get_args(union)) def test_resolve_types_no_filters_returns_all(self) -> None: with patch( diff --git a/packages/overture-schema-codegen/README.md b/packages/overture-schema-codegen/README.md index 92a4d8fbe..e9f5b0fba 100644 --- a/packages/overture-schema-codegen/README.md +++ b/packages/overture-schema-codegen/README.md @@ -22,7 +22,7 @@ renderers, not extraction logic. overture-codegen generate --format markdown --output-dir docs/schema/reference # Generate for a single theme -overture-codegen generate --format markdown --theme buildings --output-dir out/ +overture-codegen generate --format markdown --tag overture:theme=buildings --output-dir out/ # List discovered models overture-codegen list diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 4ba7c0e3b..6260ab5b0 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -25,7 +25,8 @@ is_model_class, ) from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind -from overture.schema.system.discovery import discover_models, tags_by_key +from overture.schema.system.discovery import discover_models, filter_models +from overture.schema.system.discovery.tag import get_values_for_key from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import require_any_of @@ -303,7 +304,7 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec: def find_theme(tags: frozenset[str]) -> str | None: """Extract the theme from a set of tags, if present.""" - return next(iter(tags_by_key(tags, "overture:theme")), None) + return next(iter(get_values_for_key(tags, "overture:theme")), None) T = TypeVar("T") @@ -337,7 +338,7 @@ def flat_specs_from_discovery( """Build a flat list of ModelSpecs from discovery, with entry_point set.""" models = discover_models() if theme: - models = {k: v for k, v in models.items() if find_theme(k.tags) == theme} + models = filter_models(models, tags=(f"overture:theme={theme}",)) result = [] for key, cls in models.items(): if not is_model_class(cls): diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 4eec8761c..00cba5991 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -1,9 +1,10 @@ -from typing import Annotated, Any, Literal, Union, get_args, get_origin +from typing import get_args from pydantic import BaseModel from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ModelKey +from overture.schema.system.typing_util import collect_types APPROVED = { "overture.schema.addresses:Address", @@ -28,6 +29,22 @@ def authority_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: + """Add the ``"overture"`` tag if the model originates from an approved Overture package. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to inspect. + key : ModelKey + Key identifying the model. + tags : set[str] + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with ``"overture"`` added if applicable. + """ if _matches_manifest(key): tags.add("overture") return tags @@ -36,8 +53,24 @@ def authority_provider( def theme_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - for tp in _extract_types(model_class): - if isinstance(tp, type) and issubclass(tp, OvertureFeature): + """Add the ``"overture:theme={theme}"`` tag if the model is a subclass of OvertureFeature. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to inspect. + key : ModelKey + Key identifying the model. + tags : set[str] + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with ``"overture:theme={theme}"`` added if applicable. + """ + for tp in collect_types(model_class): + if issubclass(tp, OvertureFeature): tags.add( "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] ) @@ -46,34 +79,3 @@ def theme_provider( def _matches_manifest(key: ModelKey) -> bool: return key.entry_point in APPROVED - - -def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 - result: set[type] = set() - - def visit(t: Any) -> None: # noqa: ANN401 - origin = get_origin(t) - if origin is Annotated: - visit(get_args(t)[0]) - return - - if hasattr(t, "__supertype__"): - visit(t.__supertype__) - return - - origin = get_origin(t) - - if origin is Union: - for arg in get_args(t): - visit(arg) - return - - if origin is Literal: - for val in get_args(t): - result.add(type(val)) - return - - result.add(t) - - visit(tp) - return result diff --git a/packages/overture-schema-core/tests/test_approved_models.py b/packages/overture-schema-core/tests/test_approved_models.py index d7ccb9830..36f30d302 100644 --- a/packages/overture-schema-core/tests/test_approved_models.py +++ b/packages/overture-schema-core/tests/test_approved_models.py @@ -4,7 +4,7 @@ def test_overture_feature_models_are_official() -> None: models = discover_models() for key in models: - if "overture:feature" in key.tags: - assert "overture:official" in key.tags, ( + if "feature" in key.tags: + assert "overture" in key.tags, ( f"Model {key.name} is missing 'overture:official' tag." ) diff --git a/packages/overture-schema-system/pyproject.toml b/packages/overture-schema-system/pyproject.toml index 1f1d54e52..a00be2ffc 100644 --- a/packages/overture-schema-system/pyproject.toml +++ b/packages/overture-schema-system/pyproject.toml @@ -57,4 +57,4 @@ ignore = [ per-file-ignores = {"__init__.py" = ["F401"]} [project.entry-points."overture.tag_providers"] -feature = "overture.schema.system.discovery:feature_provider" \ No newline at end of file +feature = "overture.schema.system.discovery.tag_providers:feature_provider" diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py deleted file mode 100644 index 5a3eca2dd..000000000 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Model discovery system for Overture schema registry.""" - -import importlib.metadata -import logging -import re -from collections.abc import Callable -from dataclasses import dataclass, replace -from typing import Annotated, Any, Literal, TypeAlias, Union, get_args, get_origin - -from pydantic import BaseModel - -from overture.schema.system.feature import Feature - -logger = logging.getLogger(__name__) - - -RESERVED_TAGS: dict[str, set[str]] = { - "overture": {"overture-schema-core"}, - "feature": {"overture-schema-system"}, -} -RESERVED_NAMESPACES: dict[str, set[str]] = { - "overture": {"overture-schema-core"}, - "system": {"overture-schema-system"}, -} - -TAG = r"[a-z0-9][a-z0-9_-]*" -NAMESPACE_TAG = r"[a-z0-9]+:[a-z0-9]+(?:=[a-z0-9_.-]+)?" -TAG_RE = re.compile(rf"^(?:{TAG}|{NAMESPACE_TAG})$") - - -@dataclass(frozen=True, slots=True) -class ModelKey: - """Key identifying a registered model by name, entry point, and tags. - - Attributes - ---------- - name : str - The friendly name of the model, derived from the entry point key - entry_point : str - The entry point value in "module:Class" format - tags : frozenset[str] - A set of tags associated with the model, including both plain tags and structured tags - - """ - - name: str # friendly name from entry point key - entry_point: str # The entry point value in "module:Class" format - tags: frozenset[str] # plain and structured tags - - -@dataclass(frozen=True, slots=True) -class TagProviderKey: - """Key identifying a registered model by namespace, theme, and type. - - Attributes - ---------- - name : str - The friendly name of the model, derived from the entry point key - entry_point : str - The entry point value in "module:Class" format - - """ - - name: str # friendly name from entry point key - entry_point: str # entry point value (module:Class) - package_name: str # distribution package name - - -TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] - -ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] - -TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] - -ModelKeyFilter: TypeAlias = Callable[[ModelKey], bool] - - -def generate_tags( - model_class: type[BaseModel], - key: ModelKey, - providers: TagProviderDict, -) -> set[str]: - tags: set[str] = set() - - for provider_key, provider in providers.items(): - try: - added_tags = provider(model_class, key, tags.copy()).difference(tags) - filtered_tags = _filter_tags(added_tags, provider_key) - tags.update(filtered_tags) - except Exception as e: - logger.warning( - f"Error in tag provider {provider.__name__} for model {key.name}: {e}" - ) - - return tags - - -def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: - """Filter tags, removing invalid, reserved, or namespace-restricted tags for a provider.""" - filtered_tags: set[str] = set() - reserved_tags: set[str] = { - tag for tag, pkgs in RESERVED_TAGS.items() if provider.package_name not in pkgs - } - reserved_namespaces: set[str] = { - ns - for ns, pkgs in RESERVED_NAMESPACES.items() - if provider.package_name not in pkgs - } - - for tag in tags: - # Validate tag format - if not TAG_RE.fullmatch(tag): - logger.debug( - f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " - f"This tag does not match the required format." - ) - continue - - # Reserved tag check - if tag in reserved_tags: - allowed_pkgs = RESERVED_TAGS.get(tag, set()) - logger.debug( - f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " - f"This tag can only be set by packages from: {allowed_pkgs}." - ) - continue - - # Reserved namespace check - tag_ns = namespace(tag) - if tag_ns and tag_ns in reserved_namespaces: - allowed_pkgs = RESERVED_NAMESPACES.get(tag_ns, set()) - logger.debug( - f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " - f"This namespace can only be set by packages from: {allowed_pkgs}." - ) - continue - - filtered_tags.add(tag) - - return filtered_tags - - -def discover_tag_providers( - tag_providers_group: str = "overture.tag_providers", -) -> TagProviderDict: - tag_providers = {} - - try: - for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): - try: - tag_provider_class = tag_provider.load() - - key = TagProviderKey( - name=tag_provider.name, - entry_point=tag_provider.value, - package_name=getattr(tag_provider.dist, "name", ""), - ) - - tag_providers[key] = tag_provider_class - - except Exception as e: - # Log warning but don't fail for individual tag providers - logger.warning( - "Could not load tag provider %s: %s", tag_provider.name, e - ) - except Exception as e: - logger.warning("Could not discover entry points: %s", e) - - return tag_providers - - -def discover_models( - model_group: str = "overture.models", -) -> ModelDict: - """Discover all registered Overture models via entry points. - - Parameters - ---------- - model_group: str - The entry point group to search for models (default: "overture.models") - - Returns - ------- - dict[ModelKey, type[BaseModel]] - Dict mapping ModelKey to model classes. - Theme will be None for entries without an explicit theme component. - """ - models = {} - tag_providers = discover_tag_providers() - - try: - for model in importlib.metadata.entry_points(group=model_group): - try: - model_class = model.load() - - key = ModelKey( - name=model.name, - entry_point=model.value, - tags=frozenset(), - ) - - try: - key = replace( - key, - tags=frozenset(generate_tags(model_class, key, tag_providers)), - ) - except Exception as e: - logger.warning( - "Could not resolve tags for model %s: %s", model.name, e - ) - - models[key] = model_class - - except Exception as e: - # Log warning but don't fail for individual models - logger.warning("Could not load model %s: %s", model.name, e) - except Exception as e: - logger.warning("Could not discover entry points: %s", e) - - return models - - -def filter_models( - models: ModelDict, - tags: tuple[str, ...] = (), - excluded_tags: tuple[str, ...] = (), - type_names: tuple[str, ...] = (), -) -> ModelDict: - """Filter models to those that contain all required tags.""" - filters: list[ModelKeyFilter] = [] - - if tags: - filters.append(lambda key: all(tag in key.tags for tag in tags)) - if excluded_tags: - filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) - if type_names: - filters.append(lambda key: key.name in type_names) - - if filters: - models = { - key: model for key, model in models.items() if all(f(key) for f in filters) - } - return models - - -def get_registered_model(feature_type: str) -> type[BaseModel] | None: - """Get the Pydantic model for a type. - - This uses setuptools entry points for registration. - If multiple types share the same name, the first one encountered will be returned. - - Parameters - ---------- - feature_type : str - The type name - - Returns - ------- - type[BaseModel] | None - The first encountered model class if found, None otherwise. - - """ - # Check all discovered models for a match - models = discover_models() - # Need to find by type, not exact key match - for key, model_class in models.items(): - if key.name == feature_type: - return model_class - return None - - -def namespace(tag: str) -> str: - """Extract the namespace from a tag, or return an empty string if there is no namespace.""" - if not TAG_RE.fullmatch(tag): - raise ValueError(f"Invalid tag format: {tag}") - if ":" in tag: - return tag.split(":")[0] - return "" - - -def tags_by_key(tags: frozenset[str] | set[str], key: str) -> set[str]: - """Extract values for k/v tags with the given key. - - tags_by_key(frozenset({"overture:theme=buildings", "overture", "draft"}), "overture:theme") - -> {"buildings"} - """ - prefix = key + "=" - return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} - - -def tags_by_namespace(tags: frozenset[str] | set[str], namespace: str) -> set[str]: - """Extract tag bodies within a namespace. - - tags_by_namespace(frozenset({"system:extension", "overture"}), "system") - -> {"extension"} - """ - prefix = namespace + ":" - return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} - - -def feature_provider( - model_class: type[BaseModel], key: ModelKey, tags: set[str] -) -> set[str]: - if any( - isinstance(tp, type) and issubclass(tp, Feature) - for tp in _extract_types(model_class) - ): - tags.add("feature") - return tags - - -def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 - result: set[type] = set() - - def visit(t: Any) -> None: # noqa: ANN401 - origin = get_origin(t) - if origin is Annotated: - visit(get_args(t)[0]) - return - - if hasattr(t, "__supertype__"): - visit(t.__supertype__) - return - - origin = get_origin(t) - - if origin is Union: - for arg in get_args(t): - visit(arg) - return - - if origin is Literal: - for val in get_args(t): - result.add(type(val)) - return - - result.add(t) - - visit(tp) - return result diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py new file mode 100644 index 000000000..b7ccb1250 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py @@ -0,0 +1,13 @@ +from . import tag +from .discovery import discover_models, filter_models, get_registered_model +from .models import ModelKey +from .types import ModelDict + +__all__ = [ + "tag", + "ModelKey", + "ModelDict", + "discover_models", + "filter_models", + "get_registered_model", +] diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py new file mode 100644 index 000000000..f9cab6804 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -0,0 +1,257 @@ +"""Model discovery system for Overture schema registry.""" + +import importlib.metadata +import logging +from dataclasses import replace + +from pydantic import BaseModel + +from overture.schema.system.discovery.tag import ( + get_namespace, + is_valid_tag, +) +from overture.schema.system.discovery.types import ( + ModelDict, + ModelKey, + ModelKeyFilter, + TagProviderDict, + TagProviderKey, +) + +log = logging.getLogger(__name__) + +# Tags that are reserved and can only be set by specific packages. +_RESERVED_TAGS: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "feature": {"overture-schema-system"}, +} +# Namespaces that are reserved and can only be set by specific packages. +_RESERVED_NAMESPACES: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "system": {"overture-schema-system"}, +} + + +def generate_tags( + model_class: type[BaseModel], + key: ModelKey, + providers: TagProviderDict, +) -> set[str]: + """Generate tags for a model class using tag providers. + + Each provider is called in turn indeterministically; tags it adds are filtered for + validity and permission before being included. Provider errors are caught and + logged as warnings rather than propagated. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to generate tags for. + key : ModelKey + Key identifying the model. + providers : TagProviderDict + Tag providers to invoke. + + Returns + ------- + set[str] + Tags generated for the model. + """ + tags: set[str] = set() + for provider_key, provider in providers.items(): + try: + added_tags = provider(model_class, key, tags.copy()).difference(tags) + filtered_tags = _filter_tags(added_tags, provider_key) + tags.update(filtered_tags) + except Exception as e: + log.warning( + f"Error in tag provider {provider.__name__} for model {key.name}: {e}" + ) + return tags + + +def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: + """Filter tags that cannot be used by the provider, including invalid tags, + reserved tags, and tags using a reserved namespace. + + Parameters + ---------- + tags : set[str] + Tags to filter. + provider : TagProviderKey + Provider attempting to set the tags. + + Returns + ------- + set[str] + Permitted tags. + """ + filtered_tags: set[str] = set() + reserved_tags: set[str] = { + tag for tag, pkgs in _RESERVED_TAGS.items() if provider.package_name not in pkgs + } + reserved_namespaces: set[str] = { + ns + for ns, pkgs in _RESERVED_NAMESPACES.items() + if provider.package_name not in pkgs + } + for tag in tags: + if not is_valid_tag(tag): + log.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " + f"This tag does not match the required format." + ) + continue + if tag in reserved_tags: + allowed_pkgs = _RESERVED_TAGS.get(tag, set()) + log.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " + f"This tag can only be set by packages from: {allowed_pkgs}." + ) + continue + tag_ns = get_namespace(tag) + if tag_ns and tag_ns in reserved_namespaces: + allowed_pkgs = _RESERVED_NAMESPACES.get(tag_ns, set()) + log.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " + f"This namespace can only be set by packages from: {allowed_pkgs}." + ) + continue + filtered_tags.add(tag) + return filtered_tags + + +def discover_tag_providers( + tag_providers_group: str = "overture.tag_providers", +) -> TagProviderDict: + """Discover and load tag providers via entry points. + + Parameters + ---------- + tag_providers_group : str, optional + Entry point group to search (default: ``"overture.tag_providers"``). + + Returns + ------- + TagProviderDict + Discovered tag providers keyed by TagProviderKey. + """ + tag_providers = {} + try: + for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): + try: + tag_provider_class = tag_provider.load() + key = TagProviderKey( + name=tag_provider.name, + entry_point=tag_provider.value, + package_name=getattr(tag_provider.dist, "name", ""), + ) + tag_providers[key] = tag_provider_class + except Exception as e: + log.warning(f"Could not load tag provider {tag_provider.name}: {e}") + except Exception as e: + log.warning(f"Could not discover entry points: {e}") + return tag_providers + + +def discover_models( + model_group: str = "overture.models", +) -> ModelDict: + """Discover and load models via entry points, attaching tags from tag providers. + + Parameters + ---------- + model_group : str, optional + Entry point group to search (default: ``"overture.models"``). + + Returns + ------- + ModelDict + Discovered models keyed by ModelKey. + """ + models = {} + tag_providers = discover_tag_providers() + try: + for model in importlib.metadata.entry_points(group=model_group): + try: + model_class = model.load() + key = ModelKey( + name=model.name, + entry_point=model.value, + tags=frozenset(), + ) + try: + key = replace( + key, + tags=frozenset(generate_tags(model_class, key, tag_providers)), + ) + except Exception as e: + log.warning(f"Could not resolve tags for model {model.name}: {e}") + models[key] = model_class + except Exception as e: + log.warning(f"Could not load model {model.name}: {e}") + except Exception as e: + log.warning(f"Could not discover entry points: {e}") + return models + + +def filter_models( + models: ModelDict, + *, + tags: tuple[str, ...] = (), + excluded_tags: tuple[str, ...] = (), + type_names: tuple[str, ...] = (), +) -> ModelDict: + """Filter models by required tags, excluded tags, and/or type names. + + Parameters + ---------- + models : ModelDict + Models to filter. + tags : tuple[str, ...], optional + Tags that must all be present on the model. + excluded_tags : tuple[str, ...], optional + Tags that must not be present on the model. + type_names : tuple[str, ...], optional + Model names to include; all others are excluded. + + Returns + ------- + ModelDict + Filtered models. + """ + filters: list[ModelKeyFilter] = [] + if tags: + filters.append(lambda key: all(tag in key.tags for tag in tags)) + if excluded_tags: + filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + if type_names: + filters.append(lambda key: key.name in type_names) + if filters: + models = { + key: model for key, model in models.items() if all(f(key) for f in filters) + } + return models + + +def get_registered_model(model_name: str) -> type[BaseModel] | None: + """Get the model by name. + + Loads all models via entry points and returns the first with a matching name. + If multiple models share the same name, the first one encountered is returned. + + Parameters + ---------- + model_name : str + Model name to look up. + + Returns + ------- + type[BaseModel] or None + Model class if found, otherwise ``None``. + """ + models = discover_models() + for key, model_class in models.items(): + if key.name == model_name: + return model_class + return None diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/models.py b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py new file mode 100644 index 000000000..9c331b466 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py @@ -0,0 +1,39 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ModelKey: + """Key identifying a registered model by name, entry point, and tags. + + Attributes + ---------- + name : str + Friendly name derived from the entry point key. + entry_point : str + Entry point value in ``"module:Class"`` format. + tags : frozenset[str] + Tags associated with the model. + """ + + name: str + entry_point: str + tags: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class TagProviderKey: + """Key identifying a registered tag provider by name, entry point, and package. + + Attributes + ---------- + name : str + Friendly name derived from the entry point key. + entry_point : str + Entry point value in ``"module:function"`` format. + package_name : str + Package that provides this tag provider. + """ + + name: str + entry_point: str + package_name: str diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py new file mode 100644 index 000000000..9176b1998 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py @@ -0,0 +1,104 @@ +"""Tag format specification and utilities for Overture schema discovery. + +Tags follow the pattern ``[namespace:]predicate[=value]`` and come in three forms: + +- **Plain** — ``overture``, ``feature`` +- **Namespaced** — ``system:extension` +- **Key/value** — ``overture:theme=buildings`` + +``:`` signals ownership and reservation — only the owning package may set tags in a +given namespace. ``=`` signals a dimension with a discrete value. +One level of each: no nested colons, no multiple ``=`` signs. + +Tag matching is case-sensitive throughout. +""" + +import re + +PLAIN_TAG = r"[a-z0-9][a-z0-9_-]*" +NAMESPACE = PREDICATE = r"[a-z0-9][a-z0-9_.-]*" +VALUE = r"[a-zA-Z0-9_.-]+" +NAMESPACE_TAG = rf"{NAMESPACE}:{PREDICATE}(?:={VALUE})?" +TAG = re.compile(rf"^(?:{PLAIN_TAG}|{NAMESPACE_TAG})$") + + +def get_namespace(tag: str) -> str: + """Extract the namespace prefix from a namespaced tag. + + Parameters + ---------- + tag : str + A valid tag string. + + Returns + ------- + str + The namespace prefix if the tag is a namespaced tag, otherwise ``""``. + + Examples + -------- + >>> get_namespace("overture:theme=buildings") + 'overture' + """ + return tag.split(":")[0] if is_valid_tag(tag) and ":" in tag else "" + + +def get_values_for_key(tags: frozenset[str] | set[str], key: str) -> set[str]: + """Extract values from key/value namespaced tags matching the given key. + + Parameters + ---------- + tags : frozenset[str] or set[str] + Tags to search. + key : str + Key to match, e.g. ``"overture:theme"``. + + Returns + ------- + set[str] + Values of tags matching ``key=``. + + Examples + -------- + >>> get_values_for_key(frozenset({"overture:theme=buildings", "overture"}), "overture:theme") + {'buildings'} + """ + prefix = key + "=" + return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} + + +def is_valid_tag(tag: str) -> bool: + """Check whether a string is a valid tag. + + A valid tag is a plain tag, a namespaced tag, or a key/value tag: + + - **Plain**: ``[a-z0-9][a-z0-9_-]*`` — lowercase alphanumeric, hyphens, + underscores; no dots. + - **Namespace / predicate**: ``[a-z0-9][a-z0-9_.-]*`` — same but dots + are also allowed. + - **Key/value**: ``{namespace}:{predicate}=[a-zA-Z0-9_.-]+`` — namespace and predicate as + above; value is alphanumeric (upper and lower case), hyphens, underscores, or dots; + must be non-empty. + + Parameters + ---------- + tag : str + String to validate. + + Returns + ------- + bool + ``True`` if `tag` matches the required format. + + Examples + -------- + >>> is_valid_tag("feature") + True + >>> is_valid_tag("overture:theme=buildings") + True + >>> is_valid_tag("overture:theme=") + False + >>> is_valid_tag("Invalid") + False + """ + return bool(TAG.fullmatch(tag)) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py new file mode 100644 index 000000000..e336a3e2f --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py @@ -0,0 +1,31 @@ +"""Tag provider logic for Overture schema discovery system.""" + +from pydantic import BaseModel + +from overture.schema.system.discovery.types import ModelKey +from overture.schema.system.feature import Feature +from overture.schema.system.typing_util import collect_types + + +def feature_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + """Add the ``"feature"`` tag if the model is a subclass of Feature. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to inspect. + key : ModelKey + Key identifying the model. + tags : set[str] + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with ``"feature"`` added if applicable. + """ + if any(issubclass(tp, Feature) for tp in collect_types(model_class)): + tags.add("feature") + return tags diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py new file mode 100644 index 000000000..7ea168353 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -0,0 +1,13 @@ +"""Types and data classes for Overture schema discovery system.""" + +from collections.abc import Callable +from typing import TypeAlias + +from pydantic import BaseModel + +from .models import ModelKey, TagProviderKey + +TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] +ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] +TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] +ModelKeyFilter: TypeAlias = Callable[[ModelKey], bool] diff --git a/packages/overture-schema-system/src/overture/schema/system/typing_util.py b/packages/overture-schema-system/src/overture/schema/system/typing_util.py new file mode 100644 index 000000000..1d312bcf3 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/typing_util.py @@ -0,0 +1,43 @@ +"""Typing utilities for the Overture schema system.""" + +import types +from typing import Annotated, Any, Literal, Union, get_args, get_origin + + +def collect_types(tp: Any) -> set[type]: # noqa: ANN401 + """Collect all concrete types from a type annotation. + + Recursively unwraps ``Annotated``, ``NewType``, ``Union``/``X | Y``, and + ``Literal`` to collect the concrete types they contain. Only actual `type` + instances are returned. + + Parameters + ---------- + tp : Any + A type annotation to inspect. + + Returns + ------- + set[type] + All concrete types found within ``tp``. + + """ + result: set[type] = set() + + def _visit(t: Any) -> None: + origin = get_origin(t) + if origin is Annotated: + _visit(get_args(t)[0]) + elif hasattr(t, "__supertype__"): + _visit(t.__supertype__) + elif origin is Union or origin is types.UnionType: + for arg in get_args(t): + _visit(arg) + elif origin is Literal: + for val in get_args(t): + result.add(type(val)) + elif isinstance(t, type): + result.add(t) + + _visit(tp) + return result diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py index e1cfc3ca9..8c5bc06cc 100644 --- a/packages/overture-schema-system/tests/test_tag_providers.py +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -1,12 +1,9 @@ import pytest from pydantic import BaseModel -from overture.schema.system.discovery import ( - ModelKey, - TagProviderKey, - _filter_tags, - feature_provider, -) +from overture.schema.system.discovery.discovery import _filter_tags +from overture.schema.system.discovery.tag_providers import feature_provider +from overture.schema.system.discovery.types import ModelKey, TagProviderKey from overture.schema.system.feature import Feature diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index d5efa90c2..f203200c7 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -1,59 +1,38 @@ import re import unittest -from overture.schema.system.discovery import ( +from overture.schema.system.discovery.tag import ( NAMESPACE_TAG, + PLAIN_TAG, TAG, - TAG_RE, - tags_by_key, - tags_by_namespace, + get_values_for_key, + is_valid_tag, ) -def test_tags_by_key_returns_correct_values() -> None: +def test_get_values_for_key_returns_correct_values() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "overture:theme" - result = tags_by_key(tags, key) + result = get_values_for_key(tags, key) assert result == {"buildings"} -def test_tags_by_key_returns_empty_set_for_nonexistent_key() -> None: +def test_get_values_for_key_returns_empty_set_for_nonexistent_key() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "nonexistent:key" - result = tags_by_key(tags, key) + result = get_values_for_key(tags, key) assert result == set() -def test_tags_by_key_handles_empty_tags() -> None: +def test_get_values_for_key_handles_empty_tags() -> None: tags: frozenset[str] = frozenset() key = "overture:theme" - result = tags_by_key(tags, key) + result = get_values_for_key(tags, key) assert result == set() -def test_tags_by_namespace_returns_correct_values() -> None: - tags = frozenset({"system:extension", "overture"}) - namespace = "system" - result = tags_by_namespace(tags, namespace) - assert result == {"extension"} - - -def test_tags_by_namespace_returns_empty_set_for_nonexistent_namespace() -> None: - tags = frozenset({"system:extension", "overture"}) - namespace = "nonexistent" - result = tags_by_namespace(tags, namespace) - assert result == set() - - -def test_tags_by_namespace_handles_empty_tags() -> None: - tags: frozenset[str] = frozenset() - namespace = "system" - result = tags_by_namespace(tags, namespace) - assert result == set() - - -class TestSimpleTagRegex(unittest.TestCase): - def test_valid_simple_tags(self) -> None: +class TestPlainTagRegex(unittest.TestCase): + def test_valid_plain_tags(self) -> None: valid_tags = [ "v", "valid", @@ -64,10 +43,15 @@ def test_valid_simple_tags(self) -> None: "42", ] for tag in valid_tags: - self.assertTrue(re.fullmatch(TAG, tag), f"Should match: {tag}") - self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + self.assertTrue( + re.fullmatch(PLAIN_TAG, tag), f"PLAIN_TAG should match: {tag}" + ) + self.assertTrue(TAG.fullmatch(tag), f"TAG should match: {tag}") + self.assertTrue( + is_valid_tag(tag), f"is_valid_tag should return True for: {tag}" + ) - def test_invalid_simple_tags(self) -> None: + def test_invalid_plain_tags(self) -> None: invalid_tags = [ "", "_invalid", @@ -79,8 +63,13 @@ def test_invalid_simple_tags(self) -> None: "3.14", ] for tag in invalid_tags: - self.assertFalse(re.fullmatch(TAG, tag), f"Should not match: {tag}") - self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}") + self.assertFalse( + re.fullmatch(PLAIN_TAG, tag), f"PLAIN_TAG should not match: {tag}" + ) + self.assertFalse(TAG.fullmatch(tag), f"TAG should not match: {tag}") + self.assertFalse( + is_valid_tag(tag), f"is_valid_tag should return False for: {tag}" + ) class TestNamespaceTagRegex(unittest.TestCase): @@ -88,6 +77,7 @@ def test_valid_namespace_tags(self) -> None: valid_tags = [ "ns:predicate", "ns:predicate1", + "ns:predicate-1", "ns:predicate=value", "ns:predicate=value_0", "ns:predicate=value-0", @@ -95,17 +85,22 @@ def test_valid_namespace_tags(self) -> None: "ns:predicate=value_2-3.4", "ns:predicate=42", "ns:predicate=3.14", + "ns:predicate=Value", ] for tag in valid_tags: - self.assertTrue(re.fullmatch(NAMESPACE_TAG, tag), f"Should match: {tag}") - self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + self.assertTrue( + re.fullmatch(NAMESPACE_TAG, tag), f"NAMESPACE_TAG should match: {tag}" + ) + self.assertTrue(TAG.fullmatch(tag), f"TAG should match: {tag}") + self.assertTrue( + is_valid_tag(tag), f"is_valid_tag should return True for: {tag}" + ) def test_invalid_namespace_tags(self) -> None: invalid_tags = [ "ns:", ":predicate", "ns:predicate=", - "ns:predicate=Value", "ns:predicate=value ", "ns:predicate=value!", "ns:predicate=ns:value", @@ -115,6 +110,10 @@ def test_invalid_namespace_tags(self) -> None: ] for tag in invalid_tags: self.assertFalse( - re.fullmatch(NAMESPACE_TAG, tag), f"Should not match: {tag}" + re.fullmatch(NAMESPACE_TAG, tag), + f"NAMESPACE_TAG should not match: {tag}", + ) + self.assertFalse(TAG.fullmatch(tag), f"TAG should not match: {tag}") + self.assertFalse( + is_valid_tag(tag), f"is_valid_tag should return False for: {tag}" ) - self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}")