From 13a3ea17caad612d9f2ba7bae61222501fbdfdd5 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Tue, 10 Mar 2026 21:17:16 +0100 Subject: [PATCH 01/23] Refactor model discovery and introduce tags. Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- README.pydantic.md | 14 +- .../pyproject.toml | 2 +- packages/overture-schema-annex/pyproject.toml | 2 +- .../overture-schema-base-theme/pyproject.toml | 14 +- .../pyproject.toml | 4 +- .../src/overture/schema/cli/commands.py | 162 +++++----- .../src/overture/schema/cli/types.py | 2 +- .../tests/test_resolve_types.py | 8 +- .../src/overture/schema/codegen/cli.py | 9 +- .../tests/codegen_test_support.py | 9 +- .../overture-schema-codegen/tests/conftest.py | 2 +- .../tests/test_integration_real_models.py | 2 +- packages/overture-schema-core/pyproject.toml | 5 + .../src/overture/schema/core/discovery.py | 134 --------- .../src/overture/schema/core/tag_providers.py | 88 ++++++ .../tests/test_approved_models.py | 11 + .../pyproject.toml | 8 +- .../pyproject.toml | 4 +- .../overture-schema-system/pyproject.toml | 3 + .../src/overture/schema/system/discovery.py | 279 ++++++++++++++++++ .../overture-schema-system/tests/test_tags.py | 37 +++ .../pyproject.toml | 6 +- .../src/overture/schema/__init__.py | 2 +- 23 files changed, 554 insertions(+), 253 deletions(-) delete mode 100644 packages/overture-schema-core/src/overture/schema/core/discovery.py create mode 100644 packages/overture-schema-core/src/overture/schema/core/tag_providers.py create mode 100644 packages/overture-schema-core/tests/test_approved_models.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery.py create mode 100644 packages/overture-schema-system/tests/test_tags.py diff --git a/README.pydantic.md b/README.pydantic.md index 0655c46d9..213426d43 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -151,26 +151,26 @@ Registration is done in the `[project.entry-points."overture.models"]` section: ```toml [project.entry-points."overture.models"] -"buildings.building" = "overture.schema.buildings.building.models:Building" -"buildings.building_part" = "overture.schema.buildings.building_part.models:BuildingPart" +building = "overture.schema.buildings.building.models:Building" +building_part = "overture.schema.buildings.building_part.models:BuildingPart" ``` The discovery system provides programmatic access to registered models: ```python -from overture.schema.core.discovery import discover_models, get_registered_model +from overture.schema.system.discovery import discover_models, get_registered_model # Discover all registered models all_models = discover_models() # Returns: # { -# ("buildings", "building"): BuildingModel, -# ("places", "place"): PlaceModel, +# ("building", "acme:Building", {"building_tag"}): BuildingModel, +# ("place", "acme:Place", {"place_tag"}): PlaceModel, # ... # } -# Get a specific model by theme and type -building_model = get_registered_model("buildings", "building") +# Get a specific model by type +building_model = get_registered_model("building") if building_model: # Use the model class building = building_model.model_validate(building_data) diff --git a/packages/overture-schema-addresses-theme/pyproject.toml b/packages/overture-schema-addresses-theme/pyproject.toml index 3a3ae3127..0752149cc 100644 --- a/packages/overture-schema-addresses-theme/pyproject.toml +++ b/packages/overture-schema-addresses-theme/pyproject.toml @@ -37,7 +37,7 @@ pythonpath = ["src"] testpaths = ["tests"] [project.entry-points."overture.models"] -"overture:addresses:address" = "overture.schema.addresses:Address" +address = "overture.schema.addresses:Address" [[examples.Address]] id = "416ab01c-d836-4c4f-aedc-2f30941ce94d" diff --git a/packages/overture-schema-annex/pyproject.toml b/packages/overture-schema-annex/pyproject.toml index 9ef990a8d..de90cf8d1 100644 --- a/packages/overture-schema-annex/pyproject.toml +++ b/packages/overture-schema-annex/pyproject.toml @@ -29,7 +29,7 @@ path = "src/overture/schema/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"annex:sources" = "overture.schema.annex:Sources" +sources = "overture.schema.annex:Sources" [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/packages/overture-schema-base-theme/pyproject.toml b/packages/overture-schema-base-theme/pyproject.toml index fdf146adf..fd5a79cdf 100644 --- a/packages/overture-schema-base-theme/pyproject.toml +++ b/packages/overture-schema-base-theme/pyproject.toml @@ -35,13 +35,13 @@ path = "src/overture/schema/base/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:base:bathymetry" = "overture.schema.base:Bathymetry" -"overture:base:infrastructure" = "overture.schema.base:Infrastructure" -"overture:base:land" = "overture.schema.base:Land" -"overture:base:land_cover" = "overture.schema.base:LandCover" -"overture:base:land_use" = "overture.schema.base:LandUse" -"overture:base:water" = "overture.schema.base:Water" - +bathymetry = "overture.schema.base:Bathymetry" +infrastructure = "overture.schema.base:Infrastructure" +land = "overture.schema.base:Land" +land_cover = "overture.schema.base:LandCover" +land_use = "overture.schema.base:LandUse" +water = "overture.schema.base:Water" + [[examples.Bathymetry]] id = "5d40bd6c-db14-5492-b29f-5e25a59032bc" geometry = "MULTIPOLYGON (((-170.71296928 -76.744313428, -170.719841483 -76.757076376, -170.731061124 -76.761566192, -170.775652756 -76.76338726, -170.853616381 -76.76253958, -170.918562293 -76.755380155, -170.970490492 -76.741908984, -170.998699301 -76.729180777, -171.003188718 -76.717195533, -170.990421551 -76.703765214, -170.960397802 -76.68888982, -170.940748072 -76.674697941, -170.931472364 -76.661189576, -170.927114414 -76.637296658, -170.927674224 -76.603019188, -170.939335393 -76.574637428, -170.962097922 -76.552151379, -170.999015387 -76.535715361, -171.050087788 -76.525329373, -171.079133298 -76.50751024, -171.086151917 -76.482257963, -171.098653755 -76.462747286, -171.11663881 -76.448978211, -171.146691397 -76.437601179, -171.188811514 -76.428616191, -171.296181785 -76.4228609, -171.468802209 -76.420335306, -171.566055241 -76.41501101, -171.587940879 -76.406888013, -171.59004284 -76.387987744, -171.572361122 -76.358310204, -171.549343725 -76.334488281, -171.520990649 -76.316521976, -171.453759127 -76.301763636, -171.347649159 -76.290213262, -171.30597166 -76.267707269, -171.328726628 -76.234245658, -171.36676019 -76.195627518, -171.420072345 -76.151852851, -171.444766298 -76.12494912, -171.44084205 -76.114916326, -171.378107286 -76.099627787, -171.256562007 -76.079083503, -171.228218647 -76.058825682, -171.293077208 -76.038854322, -171.421365419 -76.023534207, -171.613083278 -76.012865337, -171.76411833 -75.99938969, -171.874470572 -75.983107266, -172.121928361 -75.958403596, -172.506491695 -75.925278679, -172.744527804 -75.899736153, -172.836036689 -75.88177602, -172.904681746 -75.862406785, -172.950462974 -75.841628448, -173.000855857 -75.830396498, -173.055860393 -75.828710933, -173.177561398 -75.810743709, -173.365958872 -75.776494827, -173.493573084 -75.759370386, -173.560404033 -75.759370386, -173.620925776 -75.77158365, -173.675138312 -75.796010178, -173.733786206 -75.808642966, -173.796869456 -75.809482015, -173.847216433 -75.805553449, -173.884827135 -75.79685727, -173.90475244 -75.789177124, -173.906992347 -75.782513013, -173.881736947 -75.76894365, -173.828986239 -75.748469035, -173.797974615 -75.732298475, -173.788702075 -75.72043197, -173.82491541 -75.701013882, -173.90661462 -75.674044211, -173.977087913 -75.656066882, -174.03633529 -75.647081894, -174.150190099 -75.643010485, -174.31865234 -75.643852655, -174.444433211 -75.652836726, -174.527532713 -75.669962696, -174.581709229 -75.687086831, -174.606962758 -75.704209131, -174.631095834 -75.708279163, -174.654108458 -75.699296928, -174.688637451 -75.699296928, -174.734682816 -75.708279163, -174.797846917 -75.708699866, -174.878129754 -75.700559037, -174.939903816 -75.70870181, -174.9831691 -75.733128185, -175.025841122 -75.746602837, -175.06791988 -75.749125768, -175.09922327 -75.755318987, -175.119751293 -75.765182495, -175.127900229 -75.775197415, -175.123670077 -75.785363749, -175.111718372 -75.791289392, -175.092045112 -75.792974345, -175.049907399 -75.780622976, -174.985305232 -75.754235285, -174.935355308 -75.74552996, -174.900057628 -75.754507001, -174.886060973 -75.766815613, -174.893365345 -75.782455795, -174.907537393 -75.791536245, -174.928577117 -75.794056963, -174.971105378 -75.818213107, -175.035122174 -75.864004677, -175.060941949 -75.892403254, -175.048564703 -75.903408839, -175.020469049 -75.909193043, -174.976654988 -75.909755867, -174.944760829 -75.90482541, -174.924786572 -75.894401673, -174.92111336 -75.881479168, -174.933741192 -75.866057897, -174.900484967 -75.857513625, -174.821344686 -75.855846351, -174.752433709 -75.839289534, -174.693752038 -75.807843172, -174.652894268 -75.780747792, -174.629860399 -75.758003392, -174.571227588 -75.745793709, -174.476995837 -75.744118743, -174.398722205 -75.751841803, -174.336406693 -75.768962888, -174.300477946 -75.783262828, -174.290935964 -75.794741623, -174.28812912 -75.812412878, -174.292057414 -75.836276591, -174.289237223 -75.852155302, -174.279668547 -75.860049012, -174.205113931 -75.879998026, -174.065573375 -75.912002343, -173.957779122 -75.924071248, -173.881731171 -75.916204739, -173.846521251 -75.926706189, -173.852149361 -75.955575598, -173.845408416 -75.979439305, -173.826298414 -75.99829731, -173.76424232 -76.018956172, -173.659240133 -76.041415889, -173.560434089 -76.057698465, -173.467824188 -76.067803901, -173.404678836 -76.077625909, -173.370998032 -76.087164489, -173.332530272 -76.106814524, -173.289275555 -76.136576014, -173.231864101 -76.154545405, -173.160295911 -76.1607227, -173.093917454 -76.17278471, -173.032728732 -76.190731436, -173.009710709 -76.205560908, -173.024863387 -76.217273124, -173.048718935 -76.225374126, -173.081277354 -76.229863912, -173.219658797 -76.237442552, -173.463863265 -76.248110046, -173.60352174 -76.25793895, -173.638634223 -76.266929265, -173.658723482 -76.274676093, -173.663789516 -76.281179435, -173.661403366 -76.289363255, -173.651565032 -76.299227554, -173.627282775 -76.313843189, -173.588556596 -76.33321016, -173.575369172 -76.355231445, -173.587720504 -76.379907046, -173.573965869 -76.402499893, -173.53410527 -76.423009985, -173.518376226 -76.437156259, -173.526778738 -76.444938715, -173.559015515 -76.446303683, -173.615086557 -76.441251162, -173.686785609 -76.421600788, -173.774112673 -76.387352563, -173.854573513 -76.372333877, -173.928168128 -76.37654473, -173.968906731 -76.383732772, -173.97678932 -76.393898005, -173.979325549 -76.410884215, -173.976515417 -76.434691403, -174.000646474 -76.454452818, -174.051718722 -76.470168462, -174.08231827 -76.482963711, -174.092445119 -76.492838563, -174.075053216 -76.514344245, -174.030142562 -76.547480757, -174.016669929 -76.575274601, -174.034635317 -76.597725777, -174.037021169 -76.62030279, -174.023827484 -76.64300564, -174.034634583 -76.661942018, -174.069442464 -76.677111923, -174.086843964 -76.690616859, -174.086839082 -76.702456825, -174.080513222 -76.712456309, -174.067866385 -76.72061531, -174.036259441 -76.725116584, -173.98569239 -76.725960131, -173.93723318 -76.720486558, -173.89088181 -76.708695864, -173.780274695 -76.695221211, -173.605411835 -76.6800626, -173.487930602 -76.662096294, -173.427830996 -76.641322294, -173.370307559 -76.630935294, -173.315360292 -76.630935294, -173.249406002 -76.637251344, -173.17244469 -76.649883444, -173.110795196 -76.653532162, -173.06445752 -76.648197497, -173.029349452 -76.637355272, -173.005470993 -76.621005486, -173.01753216 -76.605236858, -173.065532955 -76.590049388, -173.096548505 -76.576599032, -173.11057881 -76.564885791, -173.108053605 -76.552301955, -173.08897289 -76.538847523, -173.051362225 -76.527628807, -172.99522161 -76.518645807, -172.891534181 -76.516119525, -172.740299938 -76.52004996, -172.648684331 -76.524540794, -172.61668736 -76.529592027, -172.584268588 -76.541098757, -172.551428016 -76.559060982, -172.533042741 -76.576141146, -172.529112765 -76.592339249, -172.540195073 -76.604524646, -172.566289666 -76.612697339, -172.576243291 -76.621303431, -172.570055947 -76.630342924, -172.555183534 -76.636123529, -172.531626051 -76.638645245, -172.517040304 -76.643518276, -172.511426292 -76.650742621, -172.551848294 -76.672312544, -172.63830631 -76.708228042, -172.701431121 -76.728711408, -172.741222726 -76.733762641, -172.81460886 -76.72534004, -172.921589524 -76.703443605, -173.006960733 -76.697273314, -173.070722487 -76.706829166, -173.101615682 -76.719791531, -173.099640316 -76.736160408, -173.033958817 -76.759064999, -172.904571183 -76.788505304, -172.847033841 -76.810916113, -172.861346791 -76.826297424, -172.924787296 -76.856444925, -173.037355356 -76.901358615, -173.149640378 -76.935043659, -173.26164236 -76.957500057, -173.354942309 -76.968728255, -173.429540223 -76.968728255, -173.487771718 -76.964657535, -173.529636796 -76.956516094, -173.572768938 -76.955559014, -173.617168145 -76.961786296, -173.614655836 -76.97446809, -173.565232013 -76.993604396, -173.461502424 -77.006682128, -173.303467069 -77.013701287, -173.163373388 -77.02787859, -173.041221382 -77.049214037, -172.918094542 -77.059179951, -172.793992869 -77.057776334, -172.720418717 -77.044861043, -172.697372088 -77.020434079, -172.675885915 -77.003730799, -172.655960197 -76.994751205, -172.60882792 -76.987594764, -172.534489083 -76.982261476, -172.480072837 -76.983094424, -172.445579184 -76.990093609, -172.428332542 -76.998610734, -172.428332911 -77.008645799, -172.435068344 -77.018150822, -172.448538839 -77.027125803, -172.490777829 -77.039613708, -172.561785312 -77.055614535, -172.628175119 -77.080598263, -172.68994725 -77.114564892, -172.751818039 -77.133793765, -172.813787485 -77.138284883, -172.900229764 -77.131828165, -173.011144875 -77.114423613, -173.119679588 -77.128474884, -173.2258339 -77.17398198, -173.273849553 -77.202664633, -173.263726547 -77.214522842, -173.165895559 -77.239681117, -172.980356589 -77.278139457, -172.880291531 -77.312658914, -172.865700386 -77.343239487, -172.867667457 -77.371126102, -172.886192744 -77.39631876, -172.999732531 -77.429966955, -173.208286817 -77.472070689, -173.335454668 -77.509278677, -173.381236082 -77.541590921, -173.403703936 -77.570407724, -173.40285823 -77.595729086, -173.378288408 -77.634921, -173.329994472 -77.687983467, -173.241287742 -77.735563094, -173.112168219 -77.777659882, -173.054064387 -77.81089869, -173.066976248 -77.835279519, -173.063736051 -77.854657976, -173.044343797 -77.869034061, -172.890349983 -77.896435115, -172.60175461 -77.936861139, -172.376181212 -77.961986812, -172.213629791 -77.971812135, -172.023427102 -77.967320559, -171.805573145 -77.948512083, -171.581263004 -77.918894833, -171.350496677 -77.87846881, -171.217147208 -77.851799157, -171.181214596 -77.838885875, -171.160572341 -77.826074082, -171.155220441 -77.813363779, -171.178789134 -77.790158543, -171.231278422 -77.756458375, -171.27338337 -77.70988804, -171.305103978 -77.65044754, -171.293875473 -77.602346602, -171.239697854 -77.565585227, -171.168401509 -77.532887375, -171.079986438 -77.504253044, -171.028614514 -77.483042244, -171.014285737 -77.469254974, -171.016677114 -77.456576914, -171.035788644 -77.445008064, -171.086879845 -77.431646501, -171.169950715 -77.416492226, -171.216537864 -77.403175691, -171.226641293 -77.391696895, -171.228607057 -77.378968685, -171.222435157 -77.364991059, -171.168824693 -77.334840949, -171.067775664 -77.288518355, -171.000402018 -77.24121644, -170.966703754 -77.192935206, -170.894838531 -77.157002595, -170.784806349 -77.133418606, -170.725150821 -77.11627156, -170.715871945 -77.105561456, -170.710674146 -77.077210652, -170.709557424 -77.031219147, -170.697909144 -76.992502178, -170.675729304 -76.961059744, -170.654536164 -76.940848729, -170.634329723 -76.931869135, -170.581564681 -76.922044903, -170.496241038 -76.911376032, -170.429709562 -76.893409727, -170.381970254 -76.868145986, -170.285260999 -76.838950739, -170.139581798 -76.805823986, -170.061542334 -76.78431495, -170.051142608 -76.77442363, -170.076677284 -76.763148845, -170.138146365 -76.750490597, -170.192753568 -76.731526593, -170.240498896 -76.706256833, -170.315896371 -76.686462585, -170.418945993 -76.67214385, -170.498267121 -76.665405567, -170.553859754 -76.666247738, -170.609039198 -76.673409769, -170.663805452 -76.68689166, -170.695686968 -76.698414281, -170.704683743 -76.70797763, -170.710444514 -76.723277346, -170.71296928 -76.744313428), (-172.46185717 -77.485683162, -172.491725041 -77.49003391, -172.535448064 -77.490594163, -172.566986057 -77.488349711, -172.586339021 -77.483300552, -172.598540475 -77.476173053, -172.60359042 -77.466967216, -172.601627836 -77.458872071, -172.592652724 -77.451887618, -172.556765055 -77.448396429, -172.49396483 -77.448398503, -172.453726685 -77.452881992, -172.436050621 -77.461846897, -172.429868964 -77.468114837, -172.435181715 -77.47168581, -172.44584445 -77.477541919, -172.46185717 -77.485683162), (-172.812798475 -76.363628771, -172.855573928 -76.365453015, -172.885037626 -76.36040045, -172.90720433 -76.351027386, -172.92207404 -76.337333821, -172.9168827 -76.324750727, -172.89163031 -76.313278104, -172.862193885 -76.307261221, -172.828573425 -76.30670008, -172.792121028 -76.311189877, -172.752836694 -76.320730613, -172.732062811 -76.331770033, -172.729799379 -76.344308139, -172.756711267 -76.354927718, -172.812798475 -76.363628771), (-171.932998671 -76.183124002, -172.010021088 -76.180457336, -172.070931389 -76.166984091, -172.113033554 -76.150312062, -172.136327583 -76.130441248, -172.133522137 -76.111120124, -172.104617217 -76.092348689, -172.06028165 -76.080296327, -172.000515436 -76.074963039, -171.918725408 -76.076928027, -171.814911566 -76.086191292, -171.745182124 -76.097695899, -171.709537083 -76.111441849, -171.696346087 -76.126554541, -171.705609136 -76.143033974, -171.731004713 -76.156183802, -171.77253282 -76.166004024, -171.83986414 -76.174984091, -171.932998671 -76.183124002), (-173.16885937 -76.066345013, -173.199147981 -76.070696107, -173.23950163 -76.071257052, -173.269213382 -76.065813298, -173.288283234 -76.054364845, -173.2799961 -76.038973879, -173.244351978 -76.0196404, -173.207608446 -76.007588038, -173.169765504 -76.002816794, -173.139490241 -76.003094691, -173.116782658 -76.008421729, -173.104589039 -76.016938854, -173.102909386 -76.028646065, -173.111183172 -76.03940804, -173.129410398 -76.049224779, -173.148635798 -76.05820377, -173.16885937 -76.066345013)))" diff --git a/packages/overture-schema-buildings-theme/pyproject.toml b/packages/overture-schema-buildings-theme/pyproject.toml index 58665a55b..ec0f01d6b 100644 --- a/packages/overture-schema-buildings-theme/pyproject.toml +++ b/packages/overture-schema-buildings-theme/pyproject.toml @@ -35,8 +35,8 @@ path = "src/overture/schema/buildings/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:buildings:building" = "overture.schema.buildings:Building" -"overture:buildings:building_part" = "overture.schema.buildings:BuildingPart" +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" [[examples.Building]] id = "148f35b1-7bc1-4180-9280-10d39b13883b" diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 74c7cbae4..0c36007ba 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -14,21 +14,20 @@ import yaml from pydantic import BaseModel, Field, Tag, TypeAdapter, ValidationError from rich.console import Console +from rich.text import Text from yamlcore import CoreLoader # type: ignore from overture.schema.core import OvertureFeature -from overture.schema.core.discovery import ModelKey, discover_models +from overture.schema.system.discovery import ModelKey, discover_models, tags_by_key from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema -from .docstrings import get_model_docstring, get_theme_module_docstring from .error_formatting import ( format_validation_error, format_validation_errors_verbose, group_errors_by_discriminator, select_most_likely_errors, ) -from .output import rewrap from .type_analysis import StructuralTuple, get_item_index, introspect_union from .types import ErrorLocation, ModelDict, UnionType @@ -208,34 +207,40 @@ def resolve_types( ------- Model type suitable for passing to parse_feature """ - # Determine effective namespace - effective_namespace = "overture" if use_overture_types else namespace - # Discover models once with the appropriate namespace - all_models = discover_models(namespace=effective_namespace) + all_models = discover_models() # Filter models based on CLI options filtered_models: ModelDict = {} + if namespace and namespace != "overture": + filtered_models = { + key: model_class + for key, model_class in all_models.items() + if namespace in key.tags + } + if use_overture_types: - filtered_models = all_models + for key, model_class in all_models.items(): + if tags_by_key(key.tags, "overture:theme"): + filtered_models[key] = model_class elif theme_names and not type_names: # Theme-only mode: all types in specified themes for key, model_class in all_models.items(): - if key.theme in theme_names: + if next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: filtered_models[key] = model_class elif type_names and not theme_names: # Type-only mode: find matching types across all themes for key, model_class in all_models.items(): - if key.type in type_names: + if key.name in type_names and tags_by_key(key.tags, "overture:theme"): filtered_models[key] = model_class elif type_names and theme_names: # Both specified: find matching types within specified themes for key, model_class in all_models.items(): - if key.theme in theme_names and key.type in type_names: + if key.name in type_names and next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: filtered_models[key] = model_class else: @@ -766,49 +771,28 @@ def json_schema_command( raise click.UsageError(str(e)) from e -def dump_namespace( - theme_types: dict[str | None, list[tuple[ModelKey, type[BaseModel]]]], -) -> None: - """Print all themes and types for a namespace. - - Displays themes in alphabetical order with their types and docstrings. - Each type includes its model class name and description. - - Args - ---- - theme_types : dict[str | None, list[tuple[ModelKey, type[BaseModel]]]] - Dict mapping theme name to list of (ModelKey, model_class) tuples - """ - for theme in sorted(theme_types.keys(), key=lambda x: (x is None, x)): - if theme: - stdout.print( - f"[bold green underline]{theme.upper()}[/bold green underline]" - ) - - theme_docstring = get_theme_module_docstring(theme) - if theme_docstring: - stdout.print( - rewrap(theme_docstring, stdout, padding_right=4), style="dim" - ) - - stdout.print() - - # Add types to the tree - sorted_types = sorted(theme_types[theme], key=lambda x: x[0].type) - for key, model_class in sorted_types: - stdout.print( - f" [bright_black]→[/bright_black] [bold cyan]{key.type}[/bold cyan] [dim magenta]({key.entry_point})[/dim magenta]" - ) - docstring = get_model_docstring(model_class) - if docstring: - stdout.print( - rewrap(docstring, stdout, indent=4, padding_right=12), style="dim" - ) - stdout.print() - - @cli.command("list-types") -def list_types() -> None: +@click.option( + "--tag", + "tags", + multiple=True, + help="Filter types by tag (e.g., overture:theme=addresses)", +) +@click.option( + "--exclude-tag", + "excluded_tags", + multiple=True, + help="Filter types by tag (e.g., overture:theme=base)", +) +@click.option( + "--group-by", + help="Group types by tag prefix (e.g., 'overture:theme')", +) +def list_types( + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], + group_by: str | None +) -> None: r"""List all available types grouped by theme with descriptions. Displays all registered Overture Maps types organized by theme, @@ -821,35 +805,51 @@ def list_types() -> None: """ try: models = discover_models() + filters = [] + + if tags: + filters.append(lambda key: all(tag in key.tags for tag in tags)) + if excluded_tags: + filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + + if filters: + models = { + key: model + for key, model in models.items() + if all(f(key) for f in filters) + } + + if group_by: + grouped_models: dict[str, set[ModelKey]] = {} + + for key, model_class in models.items(): + if groups := tags_by_key(key.tags, group_by): + for group in groups: + grouped_models.setdefault(group, set()).add(key) + + padding = max((len(key.name) for keys in grouped_models.values() for key in keys), default=0) + 2 + + for group, keys in sorted(grouped_models.items()): + stdout.print(f"[green bold]{group_by}={group} ({len(keys)})[/green bold]") + for key in sorted(keys, key=lambda k: k.name): + model = Text() + model.append("→ ", style="bright_black") + model.append(key.name, style="bold cyan") + model.pad_right(max(1, padding - len(key.name))) + model.append_text(Text().append(" ".join(sorted(key.tags)))) + stdout.print(model) + stdout.print() + + else: + padding = max((len(key.name) for key in models.keys()), default=0) + 2 + + for key in sorted(models.keys(), key=lambda k: k.name): + model = Text() + model.append(key.name, style="bold cyan") + model.pad_right(max(1, padding - len(key.name))) + model.append_text(Text().append(" ".join(sorted(key.tags)))) + stdout.print(model) - # Group models by namespace and theme - namespaces: dict[ - str, dict[str | None, list[tuple[ModelKey, type[BaseModel]]]] - ] = {} - for key, model_class in models.items(): - if key.namespace not in namespaces: - namespaces[key.namespace] = {} - if key.theme not in namespaces[key.namespace]: - namespaces[key.namespace][key.theme] = [] - - namespaces[key.namespace][key.theme].append((key, model_class)) - - # display Overture themes first - if "overture" in namespaces: - stdout.print("[bold red]OVERTURE THEMES[/bold red]", justify="center") - stdout.print() - - dump_namespace(namespaces["overture"]) - - stdout.print("[bold red]ADDITIONAL TYPES[/bold red]", justify="center") - stdout.print() - - for namespace in sorted(namespaces.keys()): - if namespace == "overture": - continue - - stdout.print(f"[bold blue]{namespace.upper()}[/bold blue]") - dump_namespace(namespaces[namespace]) except Exception as e: click.echo(f"Error listing types: {e}", err=True) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/types.py b/packages/overture-schema-cli/src/overture/schema/cli/types.py index 1b5d4e44d..f438edf2f 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/types.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/types.py @@ -5,7 +5,7 @@ from pydantic import BaseModel from pydantic_core import ErrorDetails -from overture.schema.core.discovery import ModelKey +from overture.schema.system.discovery import ModelKey # Type alias for union types created from Pydantic models # This represents either a single model or a discriminated union of models diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 94231a1fe..19ce9592f 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -2,7 +2,7 @@ import pytest from overture.schema.cli.commands import resolve_types -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import tags_by_key class TestResolveTypes: @@ -125,8 +125,10 @@ def test_resolve_types_returns_expected_themes( expected_themes: set[str], ) -> None: """Test that resolve_types returns models from expected themes.""" - models = discover_models(namespace=namespace) - actual_themes = {key.theme for key in models.keys()} + from overture.schema.system.discovery import discover_models + + models = discover_models() + actual_themes = {next(iter(tags_by_key(key.tags, "overture:theme")),None) for key in models.keys()} # Check that we have at least the expected themes (may have more) assert expected_themes.issubset(actual_themes), ( diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index 0a24c7348..ced8932ac 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -6,7 +6,7 @@ import click -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models, tags_by_key from .extraction.model_extraction import extract_model from .extraction.specs import ( @@ -47,6 +47,11 @@ def _write_output( click.echo() # separate entries with a blank line in stdout mode +def _find_theme(tags: frozenset[str]) -> str | None: + """Find the theme tag in a set of tags, if any.""" + return next(iter(tags_by_key(tags, "overture:theme")),None) + + @click.group() def cli() -> None: """Overture Schema code generator. @@ -99,7 +104,7 @@ def generate( schema_root = compute_schema_root(module_paths) models = ( - {k: v for k, v in all_models.items() if k.theme in theme} + {k: v for k, v in all_models.items() if _find_theme(k.tags) in theme} if theme else all_models ) diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 64facf5a9..79881ed5b 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -25,7 +25,7 @@ is_model_class, ) from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models, tags_by_key from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import require_any_of @@ -301,6 +301,11 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec: return next(m for m in spec.members if m.name == name) +def find_theme(tags: frozenset[str]) -> str | None: + """Extract the theme from a set of tags, if present.""" + return next(iter(tags_by_key(tags, "overture:theme")),None) + + T = TypeVar("T") @@ -332,7 +337,7 @@ def flat_specs_from_discovery( """Build a flat list of ModelSpecs from discovery, with entry_point set.""" models = discover_models() if theme: - models = {k: v for k, v in models.items() if k.theme == theme} + models = {k: v for k, v in models.items() if find_theme(k.tags) == theme} result = [] for key, cls in models.items(): if not is_model_class(cls): diff --git a/packages/overture-schema-codegen/tests/conftest.py b/packages/overture-schema-codegen/tests/conftest.py index 8dce88bf5..d66cf72a3 100644 --- a/packages/overture-schema-codegen/tests/conftest.py +++ b/packages/overture-schema-codegen/tests/conftest.py @@ -14,7 +14,7 @@ render_geometry_from_values, render_primitives_from_specs, ) -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.system.primitive import GeometryType from pydantic import BaseModel diff --git a/packages/overture-schema-codegen/tests/test_integration_real_models.py b/packages/overture-schema-codegen/tests/test_integration_real_models.py index b4dd9419f..9ed20d112 100644 --- a/packages/overture-schema-codegen/tests/test_integration_real_models.py +++ b/packages/overture-schema-codegen/tests/test_integration_real_models.py @@ -20,7 +20,7 @@ from overture.schema.codegen.layout.module_layout import entry_point_class from overture.schema.codegen.markdown.pipeline import generate_markdown_pages from overture.schema.codegen.markdown.renderer import render_feature -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.transportation import Segment from overture.schema.transportation.segment.models import RoadSegment from pydantic import BaseModel diff --git a/packages/overture-schema-core/pyproject.toml b/packages/overture-schema-core/pyproject.toml index 010441cb1..8b5e7b918 100644 --- a/packages/overture-schema-core/pyproject.toml +++ b/packages/overture-schema-core/pyproject.toml @@ -36,3 +36,8 @@ dev = [ "types-pyyaml>=6.0.12.20250516", "types-shapely>=2.1.0.20250710", ] + +[project.entry-points."overture.tag_providers"] +overture = "overture.schema.core.tag_providers:overture_provider" +authority = "overture.schema.core.tag_providers:authority_provider" +theme = "overture.schema.core.tag_providers:theme_provider" diff --git a/packages/overture-schema-core/src/overture/schema/core/discovery.py b/packages/overture-schema-core/src/overture/schema/core/discovery.py deleted file mode 100644 index b9290d29a..000000000 --- a/packages/overture-schema-core/src/overture/schema/core/discovery.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Model discovery system for Overture schema registry.""" - -import importlib.metadata -import logging -from dataclasses import dataclass - -from pydantic import BaseModel - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True, slots=True) -class ModelKey: - """Key identifying a registered model by namespace, theme, and type. - - Attributes - ---------- - namespace : str - The namespace (e.g., "overture", "annex") - theme : str | None - The theme name (e.g., "buildings", "places"), or None for non-themed models - type : str - The feature type (e.g., "building", "place") - entry_point : str - The entry point value in "module:Class" format - - """ - - namespace: str - theme: str | None - type: str - entry_point: str - - -def discover_models( - namespace: str | None = None, -) -> dict[ModelKey, type[BaseModel]]: - """Discover all registered Overture models via entry points. - - Parameters - ---------- - namespace : str | None, optional - Optional namespace filter. If provided, only models from this - namespace will be returned (e.g., "overture", "annex"). - - Returns - ------- - dict[ModelKey, type[BaseModel]] - Dict mapping ModelKey to model classes. - Theme will be None for entries without an explicit theme component. - - Notes - ----- - Entry point name format: - - Core themes: "overture::" - - Non-core (2-part): "annex:" (theme will be None) - - Non-core (3-part): "annex::" - - """ - models = {} - try: - for entry_point in importlib.metadata.entry_points(group="overture.models"): - # Parse namespace:theme:type or namespace:type from entry point name - parts = entry_point.name.split(":", 2) - - if len(parts) == 2: - # namespace:type format (no theme) - ns, feature_type = parts - theme = None - elif len(parts) == 3: - # namespace:theme:type format - ns, theme, feature_type = parts - else: - logger.warning( - "Invalid entry point format %s, expected namespace:theme:type or namespace:type", - entry_point.name, - ) - continue - - # Filter by namespace if specified - if namespace is not None and ns != namespace: - continue - - try: - model_class = entry_point.load() - key = ModelKey( - namespace=ns, - theme=theme, - type=feature_type, - entry_point=entry_point.value, - ) - models[key] = model_class - except Exception as e: - # Log warning but don't fail for individual models - logger.warning("Could not load model %s: %s", entry_point.name, e) - except Exception as e: - logger.warning("Could not discover entry points: %s", e) - - return models - - -def get_registered_model( - namespace: str, feature_type: str, theme: str | None = None -) -> type[BaseModel] | None: - """Get the Pydantic model for a namespace/theme/type combination. - - This uses setuptools entry points for registration. - - Parameters - ---------- - namespace : str - The namespace (e.g., "overture", "annex") - feature_type : str - The type name - theme : str | None, optional - The theme name (optional) - - Returns - ------- - type[BaseModel] | None - The model class if found, None otherwise. - - """ - # Check all discovered models for a match - models = discover_models(namespace=namespace) - # Need to find by namespace/theme/type, not exact key match - for key, model_class in models.items(): - if ( - key.namespace == namespace - and key.theme == theme - and key.type == feature_type - ): - return model_class - return None diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py new file mode 100644 index 000000000..de52abf93 --- /dev/null +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -0,0 +1,88 @@ +from typing import Any, get_origin, Annotated, get_args, Union, Literal + +from overture.schema.core import OvertureFeature +from pydantic import BaseModel + +from overture.schema.system.discovery import ModelKey + +APPROVED = { + "overture.schema.addresses:Address", + "overture.schema.base:Bathymetry", + "overture.schema.base:Infrastructure", + "overture.schema.base:Land", + "overture.schema.base:LandCover", + "overture.schema.base:LandUse", + "overture.schema.base:Water", + "overture.schema.buildings:Building", + "overture.schema.buildings:BuildingPart", + "overture.schema.divisions:Division", + "overture.schema.divisions:DivisionArea", + "overture.schema.divisions:DivisionBoundary", + "overture.schema.places:Place", + "overture.schema.transportation:Connector", + "overture.schema.transportation:Segment", +} + + +def authority_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + if _matches_manifest(key): + tags.add("overture:official") + return tags + + +def overture_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + if any(issubclass(tp, OvertureFeature) for tp in _reduce_types(model_class)): + tags.add("overture:feature") + return tags + + +def theme_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + for tp in _reduce_types(model_class): + if issubclass(tp, OvertureFeature): + tags.add( + "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] + ) + return tags + + +def _matches_manifest(key: ModelKey) -> bool: + if key.entry_point in APPROVED: + return True + return False + + +def _reduce_types(tp: Any) -> set[type]: + result: set[type] = set() + + def visit(t: Any) -> None: + origin = get_origin(t) + if origin is Annotated: + visit(get_args(t)[0]) + return + + if hasattr(t, "__supertype__"): + visit(t.__supertype__) + return + + origin = get_origin(t) + + if origin is Union: + for arg in get_args(t): + visit(arg) + return + + if origin is Literal: + for val in get_args(t): + result.add(type(val)) + return + + result.add(t) + + visit(tp) + return result diff --git a/packages/overture-schema-core/tests/test_approved_models.py b/packages/overture-schema-core/tests/test_approved_models.py new file mode 100644 index 000000000..6d7108610 --- /dev/null +++ b/packages/overture-schema-core/tests/test_approved_models.py @@ -0,0 +1,11 @@ +import pytest +from overture.schema.system.discovery import discover_models + + +def test_overture_feature_models_are_official() -> None: + models = discover_models() + for key in models: + if "overture:feature" in key.tags: + assert ( + "overture:official" in key.tags + ), f"Model {key.name} is missing 'overture:official' tag." diff --git a/packages/overture-schema-divisions-theme/pyproject.toml b/packages/overture-schema-divisions-theme/pyproject.toml index 2fd1dac53..5d75161c4 100644 --- a/packages/overture-schema-divisions-theme/pyproject.toml +++ b/packages/overture-schema-divisions-theme/pyproject.toml @@ -34,10 +34,10 @@ path = "src/overture/schema/divisions/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:divisions:division" = "overture.schema.divisions:Division" -"overture:divisions:division_area" = "overture.schema.divisions:DivisionArea" -"overture:divisions:division_boundary" = "overture.schema.divisions:DivisionBoundary" - +division = "overture.schema.divisions:Division" +division_area = "overture.schema.divisions:DivisionArea" +division_boundary = "overture.schema.divisions:DivisionBoundary" + [[examples.Division]] id = "350e85f6-68ba-4114-9906-c2844815988b" geometry = "POINT (-175.2551522 -21.1353686)" diff --git a/packages/overture-schema-places-theme/pyproject.toml b/packages/overture-schema-places-theme/pyproject.toml index 48abbcbfa..0b1b68617 100644 --- a/packages/overture-schema-places-theme/pyproject.toml +++ b/packages/overture-schema-places-theme/pyproject.toml @@ -34,8 +34,8 @@ path = "src/overture/schema/places/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:places:place" = "overture.schema.places:Place" - +place = "overture.schema.places:Place" + [[examples.Place]] id = "99003ee6-e75b-4dd6-8a8a-53a5a716c50d" geometry = "POINT (-150.46875 -79.1713346)" diff --git a/packages/overture-schema-system/pyproject.toml b/packages/overture-schema-system/pyproject.toml index d81c649ce..f4771f04c 100644 --- a/packages/overture-schema-system/pyproject.toml +++ b/packages/overture-schema-system/pyproject.toml @@ -55,3 +55,6 @@ ignore = [ "C901", # too complex ] per-file-ignores = {"__init__.py" = ["F401"]} + +[project.entry-points."overture.tag_providers"] +feature = "overture.schema.system.discovery:feature_provider" \ No newline at end of file diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py new file mode 100644 index 000000000..5231c737d --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -0,0 +1,279 @@ +"""Model discovery system for Overture schema registry.""" + +import importlib.metadata +import logging +import re +from dataclasses import dataclass, replace +from typing import get_args, Literal, Union, get_origin, Annotated, Any, Callable + +from pydantic import BaseModel + +from overture.schema.system.feature import Feature + +logger = logging.getLogger(__name__) + + +RESERVED_TAGS: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "feature": {"overture-schema-system"}, +} +TAG = r"[a-z0-9][a-z0-9_-]*" +NAMESPACE_TAG = r"[a-z0-9]+:[a-z0-9]+(?:=[a-z0-9_.-]+)?" +TAG_RE = re.compile(rf"^(?:{TAG}|{NAMESPACE_TAG})$") + + +@dataclass(frozen=True, slots=True) +class ModelKey: + """Key identifying a registered model by namespace, theme, and type. + + Attributes + ---------- + name : str + The friendly name of the model, derived from the entry point key + entry_point : str + The entry point value in "module:Class" format + tags : frozenset[str] + A set of tags associated with the model, including both plain tags and structured tags + + """ + + name: str # friendly name from entry point key + entry_point : str # The entry point value in "module:Class" format + tags: frozenset[str] # plain and structured tags + + +@dataclass(frozen=True, slots=True) +class TagProviderKey: + """Key identifying a registered model by namespace, theme, and type. + + Attributes + ---------- + name : str + The friendly name of the model, derived from the entry point key + entry_point : str + The entry point value in "module:Class" format + + """ + + name: str # friendly name from entry point key + entry_point: str # entry point value (module:Class) + package_name: str # distribution package name + + +TagProvider = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] + + +def generate_tags( + model_class: type[BaseModel], + key: ModelKey, + providers: dict[TagProviderKey, TagProvider], +) -> set[str]: + tags: set[str] = set() + + for provider_key, provider in providers.items(): + try: + added_tags = provider(model_class, key, tags.copy()).difference(tags) + filtered_tags = _filter_tags(added_tags, provider_key.package_name) + tags.update(filtered_tags) + except Exception as e: + print( + f"Error in tag provider {provider.__name__} for model {key.name}: {e}" + ) + + return tags + + +def _filter_tags(tags: set[str], package: str) -> set[str]: + reserved_namespaces = tuple( + f"{namespace}:" + for namespace, reserved_packages in RESERVED_TAGS.items() + if package not in reserved_packages + ) + + return { + tag + for tag in tags + if TAG_RE.match(tag) and not tag.startswith(reserved_namespaces) + } + + +def discover_tag_providers( + tag_providers_group: str = "overture.tag_providers", +) -> dict[TagProviderKey, TagProvider]: + tag_providers = {} + + try: + for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): + + try: + tag_provider_class = tag_provider.load() + + key = TagProviderKey( + name=tag_provider.name, + entry_point=tag_provider.value, + package_name=getattr(tag_provider.dist, "name", ""), + ) + + tag_providers[key] = tag_provider_class + + except Exception as e: + # Log warning but don't fail for individual tag providers + logger.warning( + "Could not load tag provider %s: %s", tag_provider.name, e + ) + except Exception as e: + logger.warning("Could not discover entry points: %s", e) + + return tag_providers + + +def discover_models( + model_group: str = "overture.models", +) -> dict[ModelKey, type[BaseModel]]: + """Discover all registered Overture models via entry points. + + Parameters + ---------- + model_group: str + The entry point group to search for models (default: "overture.models") + + Returns + ------- + dict[ModelKey, type[BaseModel]] + Dict mapping ModelKey to model classes. + Theme will be None for entries without an explicit theme component. + """ + models = {} + tag_providers = discover_tag_providers() + + try: + for model in importlib.metadata.entry_points(group=model_group): + + try: + model_class = model.load() + + key = ModelKey( + name=model.name, + entry_point=model.value, + tags=( + frozenset(_distribution_tags(model.dist)) + if model.dist + else frozenset() + ), + ) + + try: + key = replace( + key, + tags=frozenset(generate_tags(model_class, key, tag_providers)), + ) + except Exception as e: + logger.warning( + "Could not resolve tags for model %s: %s", model.name, e + ) + + models[key] = model_class + + except Exception as e: + # Log warning but don't fail for individual models + logger.warning("Could not load model %s: %s", model.name, e) + except Exception as e: + logger.warning("Could not discover entry points: %s", e) + + return models + + +def _distribution_tags(dist: importlib.metadata.Distribution) -> set[str]: + """Extract tags from the distribution metadata.""" + tags = set() + if dist.name: + tags.add("dist:name=" + dist.name) + if dist.version: + tags.add("dist:version=" + dist.version) + + return tags + + +def get_registered_model(feature_type: str) -> type[BaseModel] | None: + """Get the Pydantic model for a type. + + This uses setuptools entry points for registration. + If multiple types share the same name, the first one encountered will be returned. + + Parameters + ---------- + feature_type : str + The type name + + Returns + ------- + type[BaseModel] | None + The first encountered model class if found, None otherwise. + + """ + # Check all discovered models for a match + models = discover_models() + # Need to find by type, not exact key match + for key, model_class in models.items(): + if key.name == feature_type: + return model_class + return None + + +def tags_by_key(tags: frozenset[str] | set[str], key: str) -> set[str]: + """Extract values for k/v tags with the given key. + + tags_by_key(frozenset({"overture:theme=buildings", "overture", "draft"}), "overture:theme") + -> {"buildings"} + """ + prefix = key + "=" + return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} + + +def tags_by_namespace(tags: frozenset[str] | set[str], namespace: str) -> set[str]: + """Extract tag bodies within a namespace. + + tags_by_namespace(frozenset({"system:extension", "overture"}), "system") + -> {"extension"} + """ + prefix = namespace + ":" + return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} + + +def feature_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + if any(issubclass(tp, Feature) for tp in _extract_types(model_class)): + tags.add("system:feature") + return tags + + +def _extract_types(tp: Any) -> set[type]: + result: set[type] = set() + + def visit(t: Any) -> None: + origin = get_origin(t) + if origin is Annotated: + visit(get_args(t)[0]) + return + + if hasattr(t, "__supertype__"): + visit(t.__supertype__) + return + + origin = get_origin(t) + + if origin is Union: + for arg in get_args(t): + visit(arg) + return + + if origin is Literal: + for val in get_args(t): + result.add(type(val)) + return + + result.add(t) + + visit(tp) + return result diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py new file mode 100644 index 000000000..0fc88f69b --- /dev/null +++ b/packages/overture-schema-system/tests/test_tags.py @@ -0,0 +1,37 @@ +from overture.schema.system.discovery import tags_by_key, tags_by_namespace + +def test_tags_by_key_returns_correct_values() -> None: + tags = frozenset({"overture:theme=buildings", "overture", "draft"}) + key = "overture:theme" + result = tags_by_key(tags, key) + assert result == {"buildings"} + +def test_tags_by_key_returns_empty_set_for_nonexistent_key() -> None: + tags = frozenset({"overture:theme=buildings", "overture", "draft"}) + key = "nonexistent:key" + result = tags_by_key(tags, key) + assert result == set() + +def test_tags_by_key_handles_empty_tags() -> None: + tags: frozenset[str] = frozenset() + key = "overture:theme" + result = tags_by_key(tags, key) + assert result == set() + +def test_tags_by_namespace_returns_correct_values() -> None: + tags = frozenset({"system:extension", "overture"}) + namespace = "system" + result = tags_by_namespace(tags, namespace) + assert result == {"extension"} + +def test_tags_by_namespace_returns_empty_set_for_nonexistent_namespace() -> None: + tags = frozenset({"system:extension", "overture"}) + namespace = "nonexistent" + result = tags_by_namespace(tags, namespace) + assert result == set() + +def test_tags_by_namespace_handles_empty_tags() -> None: + tags: frozenset[str] = frozenset() + namespace = "system" + result = tags_by_namespace(tags, namespace) + assert result == set() diff --git a/packages/overture-schema-transportation-theme/pyproject.toml b/packages/overture-schema-transportation-theme/pyproject.toml index 40db4638a..fe4eb8a47 100644 --- a/packages/overture-schema-transportation-theme/pyproject.toml +++ b/packages/overture-schema-transportation-theme/pyproject.toml @@ -35,9 +35,9 @@ path = "src/overture/schema/transportation/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:transportation:connector" = "overture.schema.transportation:Connector" -"overture:transportation:segment" = "overture.schema.transportation:Segment" - +connector = "overture.schema.transportation:Connector" +segment = "overture.schema.transportation:Segment" + [[examples.Connector]] id = "39542bee-230f-4b91-b7e5-a9b58e0c59b1" geometry = "POINT (-176.5472979 -43.9679472)" diff --git a/packages/overture-schema/src/overture/schema/__init__.py b/packages/overture-schema/src/overture/schema/__init__.py index 1f75ea511..2f5a0b7a0 100644 --- a/packages/overture-schema/src/overture/schema/__init__.py +++ b/packages/overture-schema/src/overture/schema/__init__.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, Tag, TypeAdapter from overture.schema.core import OvertureFeature -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.system.feature import Feature From 2ef051a11a0f5f06694488552451ac95bc057d22 Mon Sep 17 00:00:00 2001 From: Roel Bollens <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:11:31 +0100 Subject: [PATCH 02/23] chore(system): use logger.warning instead of print Co-authored-by: Seth Fitzsimmons Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/system/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 5231c737d..56ec05f3d 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -76,7 +76,7 @@ def generate_tags( filtered_tags = _filter_tags(added_tags, provider_key.package_name) tags.update(filtered_tags) except Exception as e: - print( + logger.warning( f"Error in tag provider {provider.__name__} for model {key.name}: {e}" ) From 5f4ec9b111209cb8594b766d93e728e357045fd5 Mon Sep 17 00:00:00 2001 From: Roel Bollens <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:13:51 +0100 Subject: [PATCH 03/23] refactor(core): simplify using direct boolean return Co-authored-by: Seth Fitzsimmons Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/core/tag_providers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index de52abf93..52cf5e241 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -52,9 +52,7 @@ def theme_provider( def _matches_manifest(key: ModelKey) -> bool: - if key.entry_point in APPROVED: - return True - return False + return key.entry_point in APPROVED: def _reduce_types(tp: Any) -> set[type]: From 52fe75150b084dde8efd6210e42e1ed065a1c3f8 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:07:53 +0100 Subject: [PATCH 04/23] chore(system): update ModelKey docstring to reflect changed attributes Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/system/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 56ec05f3d..98f3402cc 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -24,7 +24,7 @@ @dataclass(frozen=True, slots=True) class ModelKey: - """Key identifying a registered model by namespace, theme, and type. + """Key identifying a registered model by name, entry point, and tags. Attributes ---------- From fe2cb87c0de008175d6bc6f2760c4075bd06a4c5 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Sun, 15 Mar 2026 21:59:15 +0100 Subject: [PATCH 05/23] refactor(system,core): Removes deferred tag provider and corrects tag filtering logic - Removes overture tag provider (was deferred) - Simplified tags - Reserved tags instead of reserved namespaces - Fixes small issue introduced in earlier commit Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- packages/overture-schema-core/pyproject.toml | 1 - .../src/overture/schema/core/tag_providers.py | 13 ++------ .../src/overture/schema/system/discovery.py | 33 +++++-------------- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/packages/overture-schema-core/pyproject.toml b/packages/overture-schema-core/pyproject.toml index 8b5e7b918..87c579cde 100644 --- a/packages/overture-schema-core/pyproject.toml +++ b/packages/overture-schema-core/pyproject.toml @@ -38,6 +38,5 @@ dev = [ ] [project.entry-points."overture.tag_providers"] -overture = "overture.schema.core.tag_providers:overture_provider" authority = "overture.schema.core.tag_providers:authority_provider" theme = "overture.schema.core.tag_providers:theme_provider" diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 52cf5e241..08cfafca8 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -21,6 +21,7 @@ "overture.schema.places:Place", "overture.schema.transportation:Connector", "overture.schema.transportation:Segment", + "overture.schema.annex:Sources", } @@ -28,15 +29,7 @@ def authority_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: if _matches_manifest(key): - tags.add("overture:official") - return tags - - -def overture_provider( - model_class: type[BaseModel], key: ModelKey, tags: set[str] -) -> set[str]: - if any(issubclass(tp, OvertureFeature) for tp in _reduce_types(model_class)): - tags.add("overture:feature") + tags.add("overture") return tags @@ -52,7 +45,7 @@ def theme_provider( def _matches_manifest(key: ModelKey) -> bool: - return key.entry_point in APPROVED: + return key.entry_point in APPROVED def _reduce_types(tp: Any) -> set[type]: diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 98f3402cc..58097724e 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -73,7 +73,7 @@ def generate_tags( for provider_key, provider in providers.items(): try: added_tags = provider(model_class, key, tags.copy()).difference(tags) - filtered_tags = _filter_tags(added_tags, provider_key.package_name) + filtered_tags = _filter_tags(added_tags, provider_key) tags.update(filtered_tags) except Exception as e: logger.warning( @@ -83,17 +83,17 @@ def generate_tags( return tags -def _filter_tags(tags: set[str], package: str) -> set[str]: - reserved_namespaces = tuple( - f"{namespace}:" - for namespace, reserved_packages in RESERVED_TAGS.items() - if package not in reserved_packages +def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: + reserved_tags = tuple( + tag + for tag, dist in RESERVED_TAGS.items() + if provider.package_name not in dist ) return { tag for tag in tags - if TAG_RE.match(tag) and not tag.startswith(reserved_namespaces) + if TAG_RE.match(tag) and not tag in reserved_tags } @@ -155,11 +155,7 @@ def discover_models( key = ModelKey( name=model.name, entry_point=model.value, - tags=( - frozenset(_distribution_tags(model.dist)) - if model.dist - else frozenset() - ), + tags=frozenset(), ) try: @@ -183,17 +179,6 @@ def discover_models( return models -def _distribution_tags(dist: importlib.metadata.Distribution) -> set[str]: - """Extract tags from the distribution metadata.""" - tags = set() - if dist.name: - tags.add("dist:name=" + dist.name) - if dist.version: - tags.add("dist:version=" + dist.version) - - return tags - - def get_registered_model(feature_type: str) -> type[BaseModel] | None: """Get the Pydantic model for a type. @@ -244,7 +229,7 @@ def feature_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: if any(issubclass(tp, Feature) for tp in _extract_types(model_class)): - tags.add("system:feature") + tags.add("feature") return tags From f355be5813d4536b30e6d8bf7e89dde5b0d11a7b Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:58:32 +0100 Subject: [PATCH 06/23] chore(system, core): fixes linting/formatting issues Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/cli/commands.py | 31 +++++++++++++------ .../tests/test_resolve_types.py | 5 ++- .../src/overture/schema/codegen/cli.py | 2 +- .../tests/codegen_test_support.py | 2 +- .../src/overture/schema/core/tag_providers.py | 10 +++--- .../tests/test_approved_models.py | 7 ++--- .../src/overture/schema/system/discovery.py | 21 +++++-------- .../overture-schema-system/tests/test_tags.py | 6 ++++ 8 files changed, 48 insertions(+), 36 deletions(-) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 0c36007ba..33df762b1 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -228,7 +228,7 @@ def resolve_types( elif theme_names and not type_names: # Theme-only mode: all types in specified themes for key, model_class in all_models.items(): - if next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: + if next(iter(tags_by_key(key.tags, "overture:theme")), None) in theme_names: filtered_models[key] = model_class elif type_names and not theme_names: @@ -240,7 +240,11 @@ def resolve_types( elif type_names and theme_names: # Both specified: find matching types within specified themes for key, model_class in all_models.items(): - if key.name in type_names and next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names: + if ( + key.name in type_names + and next(iter(tags_by_key(key.tags, "overture:theme")), None) + in theme_names + ): filtered_models[key] = model_class else: @@ -789,9 +793,7 @@ def json_schema_command( help="Group types by tag prefix (e.g., 'overture:theme')", ) def list_types( - tags: tuple[str, ...], - excluded_tags: tuple[str, ...], - group_by: str | None + tags: tuple[str, ...], excluded_tags: tuple[str, ...], group_by: str | None ) -> None: r"""List all available types grouped by theme with descriptions. @@ -810,7 +812,9 @@ def list_types( if tags: filters.append(lambda key: all(tag in key.tags for tag in tags)) if excluded_tags: - filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + filters.append( + lambda key: not any(tag in key.tags for tag in excluded_tags) + ) if filters: models = { @@ -822,15 +826,23 @@ def list_types( if group_by: grouped_models: dict[str, set[ModelKey]] = {} - for key, model_class in models.items(): + for key in models.keys(): if groups := tags_by_key(key.tags, group_by): for group in groups: grouped_models.setdefault(group, set()).add(key) - padding = max((len(key.name) for keys in grouped_models.values() for key in keys), default=0) + 2 + padding = ( + max( + (len(key.name) for keys in grouped_models.values() for key in keys), + default=0, + ) + + 2 + ) for group, keys in sorted(grouped_models.items()): - stdout.print(f"[green bold]{group_by}={group} ({len(keys)})[/green bold]") + stdout.print( + f"[green bold]{group_by}={group} ({len(keys)})[/green bold]" + ) for key in sorted(keys, key=lambda k: k.name): model = Text() model.append("→ ", style="bright_black") @@ -850,7 +862,6 @@ def list_types( model.append_text(Text().append(" ".join(sorted(key.tags)))) stdout.print(model) - except Exception as e: click.echo(f"Error listing types: {e}", err=True) diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 19ce9592f..976e17416 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -128,7 +128,10 @@ def test_resolve_types_returns_expected_themes( from overture.schema.system.discovery import discover_models models = discover_models() - actual_themes = {next(iter(tags_by_key(key.tags, "overture:theme")),None) for key in models.keys()} + actual_themes = { + next(iter(tags_by_key(key.tags, "overture:theme")), None) + for key in models.keys() + } # Check that we have at least the expected themes (may have more) assert expected_themes.issubset(actual_themes), ( diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index ced8932ac..8624bfc7a 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -49,7 +49,7 @@ def _write_output( def _find_theme(tags: frozenset[str]) -> str | None: """Find the theme tag in a set of tags, if any.""" - return next(iter(tags_by_key(tags, "overture:theme")),None) + return next(iter(tags_by_key(tags, "overture:theme")), None) @click.group() diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 79881ed5b..4ba7c0e3b 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -303,7 +303,7 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec: def find_theme(tags: frozenset[str]) -> str | None: """Extract the theme from a set of tags, if present.""" - return next(iter(tags_by_key(tags, "overture:theme")),None) + return next(iter(tags_by_key(tags, "overture:theme")), None) T = TypeVar("T") diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 08cfafca8..e4e9a72d6 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -1,8 +1,8 @@ -from typing import Any, get_origin, Annotated, get_args, Union, Literal +from typing import Annotated, Any, Literal, Union, get_args, get_origin -from overture.schema.core import OvertureFeature from pydantic import BaseModel +from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ModelKey APPROVED = { @@ -36,7 +36,7 @@ def authority_provider( def theme_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - for tp in _reduce_types(model_class): + for tp in _extract_types(model_class): if issubclass(tp, OvertureFeature): tags.add( "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] @@ -48,10 +48,10 @@ def _matches_manifest(key: ModelKey) -> bool: return key.entry_point in APPROVED -def _reduce_types(tp: Any) -> set[type]: +def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 result: set[type] = set() - def visit(t: Any) -> None: + def visit(t: Any) -> None: # noqa: ANN401 origin = get_origin(t) if origin is Annotated: visit(get_args(t)[0]) diff --git a/packages/overture-schema-core/tests/test_approved_models.py b/packages/overture-schema-core/tests/test_approved_models.py index 6d7108610..d7ccb9830 100644 --- a/packages/overture-schema-core/tests/test_approved_models.py +++ b/packages/overture-schema-core/tests/test_approved_models.py @@ -1,4 +1,3 @@ -import pytest from overture.schema.system.discovery import discover_models @@ -6,6 +5,6 @@ def test_overture_feature_models_are_official() -> None: models = discover_models() for key in models: if "overture:feature" in key.tags: - assert ( - "overture:official" in key.tags - ), f"Model {key.name} is missing 'overture:official' tag." + assert "overture:official" in key.tags, ( + f"Model {key.name} is missing 'overture:official' tag." + ) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 58097724e..1141f08f2 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -3,8 +3,9 @@ import importlib.metadata import logging import re +from collections.abc import Callable from dataclasses import dataclass, replace -from typing import get_args, Literal, Union, get_origin, Annotated, Any, Callable +from typing import Annotated, Any, Literal, Union, get_args, get_origin from pydantic import BaseModel @@ -38,7 +39,7 @@ class ModelKey: """ name: str # friendly name from entry point key - entry_point : str # The entry point value in "module:Class" format + entry_point: str # The entry point value in "module:Class" format tags: frozenset[str] # plain and structured tags @@ -85,16 +86,10 @@ def generate_tags( def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: reserved_tags = tuple( - tag - for tag, dist in RESERVED_TAGS.items() - if provider.package_name not in dist + tag for tag, dist in RESERVED_TAGS.items() if provider.package_name not in dist ) - return { - tag - for tag in tags - if TAG_RE.match(tag) and not tag in reserved_tags - } + return {tag for tag in tags if TAG_RE.match(tag) and tag not in reserved_tags} def discover_tag_providers( @@ -104,7 +99,6 @@ def discover_tag_providers( try: for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): - try: tag_provider_class = tag_provider.load() @@ -148,7 +142,6 @@ def discover_models( try: for model in importlib.metadata.entry_points(group=model_group): - try: model_class = model.load() @@ -233,10 +226,10 @@ def feature_provider( return tags -def _extract_types(tp: Any) -> set[type]: +def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 result: set[type] = set() - def visit(t: Any) -> None: + def visit(t: Any) -> None: # noqa: ANN401 origin = get_origin(t) if origin is Annotated: visit(get_args(t)[0]) diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index 0fc88f69b..c7fc28c9b 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -1,35 +1,41 @@ from overture.schema.system.discovery import tags_by_key, tags_by_namespace + def test_tags_by_key_returns_correct_values() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "overture:theme" result = tags_by_key(tags, key) assert result == {"buildings"} + def test_tags_by_key_returns_empty_set_for_nonexistent_key() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "nonexistent:key" result = tags_by_key(tags, key) assert result == set() + def test_tags_by_key_handles_empty_tags() -> None: tags: frozenset[str] = frozenset() key = "overture:theme" result = tags_by_key(tags, key) assert result == set() + def test_tags_by_namespace_returns_correct_values() -> None: tags = frozenset({"system:extension", "overture"}) namespace = "system" result = tags_by_namespace(tags, namespace) assert result == {"extension"} + def test_tags_by_namespace_returns_empty_set_for_nonexistent_namespace() -> None: tags = frozenset({"system:extension", "overture"}) namespace = "nonexistent" result = tags_by_namespace(tags, namespace) assert result == set() + def test_tags_by_namespace_handles_empty_tags() -> None: tags: frozenset[str] = frozenset() namespace = "system" From 309f56481930845b8bb8f4f2775442c9e48ba3b9 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Tue, 17 Mar 2026 20:51:12 +0100 Subject: [PATCH 07/23] refactor(codegen, cli): replace theme filtering with tag filtering in CLI commands Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/cli/commands.py | 149 +++------- .../tests/test_cli_commands.py | 14 +- .../tests/test_cli_functions.py | 12 +- .../tests/test_error_formatting.py | 4 +- .../tests/test_resolve_types.py | 273 +++++++++--------- .../src/overture/schema/codegen/cli.py | 29 +- .../overture-schema-codegen/tests/test_cli.py | 34 ++- .../src/overture/schema/system/discovery.py | 23 ++ 8 files changed, 255 insertions(+), 283 deletions(-) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 33df762b1..3b298c130 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -18,7 +18,12 @@ from yamlcore import CoreLoader # type: ignore from overture.schema.core import OvertureFeature -from overture.schema.system.discovery import ModelKey, discover_models, tags_by_key +from overture.schema.system.discovery import ( + ModelKey, + discover_models, + filter_models, + tags_by_key, +) from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema @@ -189,72 +194,32 @@ def validate_features(data: list, model_type: UnionType) -> list[BaseModel]: def resolve_types( - use_overture_types: bool, - namespace: str | None, - theme_names: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], type_names: tuple[str, ...], ) -> UnionType: """Resolve CLI options into a model type suitable for parse_feature. Args ---- - use_overture_types: Boolean from --overture-types flag - namespace: Namespace to filter by (e.g., "overture", "annex") - theme_names: List of theme names from --theme option + tags: Tags to include (e.g., "feature", "overture:theme=buildings") + excluded_tags: Tags to exclude (e.g., "draft") type_names: List of type names from --type option Returns ------- Model type suitable for passing to parse_feature """ - # Discover models once with the appropriate namespace - all_models = discover_models() + # Discover models + models: ModelDict = discover_models() # Filter models based on CLI options - filtered_models: ModelDict = {} - - if namespace and namespace != "overture": - filtered_models = { - key: model_class - for key, model_class in all_models.items() - if namespace in key.tags - } - - if use_overture_types: - for key, model_class in all_models.items(): - if tags_by_key(key.tags, "overture:theme"): - filtered_models[key] = model_class - - elif theme_names and not type_names: - # Theme-only mode: all types in specified themes - for key, model_class in all_models.items(): - if next(iter(tags_by_key(key.tags, "overture:theme")), None) in theme_names: - filtered_models[key] = model_class - - elif type_names and not theme_names: - # Type-only mode: find matching types across all themes - for key, model_class in all_models.items(): - if key.name in type_names and tags_by_key(key.tags, "overture:theme"): - filtered_models[key] = model_class - - elif type_names and theme_names: - # Both specified: find matching types within specified themes - for key, model_class in all_models.items(): - if ( - key.name in type_names - and next(iter(tags_by_key(key.tags, "overture:theme")), None) - in theme_names - ): - filtered_models[key] = model_class - - else: - # No filters specified - use all models - filtered_models = all_models + models = filter_models(models, tags, excluded_tags, type_names) - if not filtered_models: + if not models: raise ValueError("No models found matching the specified criteria") - return create_union_type_from_models(filtered_models) + return create_union_type_from_models(models) def get_source_name(filename: Path) -> str: @@ -290,10 +255,10 @@ def cli() -> None: $ overture-schema list-types \b # Generate JSON schema - $ overture-schema json-schema --theme buildings + $ overture-schema json-schema --tag overture:theme=buildings \b # Validate specific types - $ overture-schema validate --theme buildings data.json + $ overture-schema validate --tag overture:theme=buildings data.json """ pass @@ -536,7 +501,7 @@ def handle_validation_error( style="yellow", ) stderr.print( - " • Consider validating each type separately with --theme or --type", + " • Consider validating each type separately with --tag or --type", style="dim", ) stderr.print() @@ -557,7 +522,7 @@ def handle_validation_error( style="yellow", ) stderr.print( - " • Specifying --theme or --type to narrow validation", style="dim" + " • Specifying --tag or --type to narrow validation", style="dim" ) stderr.print(" • Adding discriminator fields to clarify intent", style="dim") stderr.print() @@ -637,18 +602,16 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: @cli.command() @click.argument("filename", type=click.Path(path_type=Path), required=True) @click.option( - "--overture-types", - is_flag=True, - help="Validate against all official Overture types (excludes extensions)", -) -@click.option( - "--namespace", - help="Namespace to filter by (e.g., overture, annex)", + "--tag", + "tags", + multiple=True, + help="Tags to include (e.g., overture:theme=addresses)", ) @click.option( - "--theme", + "--exclude-tag", + "excluded_tags", multiple=True, - help="Theme to validate against (shorthand for all types in theme)", + help="Tags to exclude (e.g., overture:theme=base)", ) @click.option( "--type", @@ -664,9 +627,8 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: ) def validate( filename: Path, - overture_types: bool, - namespace: str | None, - theme: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], types: tuple[str, ...], show_fields: tuple[str, ...], ) -> None: @@ -684,17 +646,17 @@ def validate( $ overture-schema validate - < data.json \b # Validate only buildings - $ overture-schema validate --theme buildings data.json + $ overture-schema validate --tag overture:theme=buildings data.json \b # Validate specific type $ overture-schema validate --type building data.json \b # Official Overture types only - $ overture-schema validate --overture-types data.json + $ overture-schema validate --tag overture --tag feature data.json """ # Resolve model type first (errors here are ValueErrors, not ValidationErrors) try: - model_type = resolve_types(overture_types, namespace, theme, types) + model_type = resolve_types(tags, excluded_tags, types) except ValueError as e: handle_generic_error(e, filename, "value") return @@ -722,18 +684,16 @@ def validate( @cli.command("json-schema") @click.option( - "--overture-types", - is_flag=True, - help="Generate schema for all official Overture types (excludes extensions)", -) -@click.option( - "--namespace", - help="Namespace to filter by (e.g., overture, annex)", + "--tag", + "tags", + multiple=True, + help="Tags to include (e.g., overture:theme=addresses)", ) @click.option( - "--theme", + "--exclude-tag", + "excluded_tags", multiple=True, - help="Theme to generate schema for (shorthand for all types in theme)", + help="Tags to exclude (e.g., overture:theme=base)", ) @click.option( "--type", @@ -742,9 +702,8 @@ def validate( help="Specific type to generate schema for (e.g., building, segment)", ) def json_schema_command( - overture_types: bool, - namespace: str | None, - theme: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], types: tuple[str, ...], ) -> None: r"""Generate JSON schema for Overture Maps types. @@ -757,17 +716,17 @@ def json_schema_command( # All types $ overture-schema json-schema > schema.json \b - # Buildings theme - $ overture-schema json-schema --theme buildings + # Buildings theme by tag + $ overture-schema json-schema --tag overture:theme=buildings \b # Specific types $ overture-schema json-schema --type building \b # Official Overture types only - $ overture-schema json-schema --overture-types + $ overture-schema json-schema --tag overture --tag feature """ try: - model_type = resolve_types(overture_types, namespace, theme, types) + model_type = resolve_types(tags, excluded_tags, types) schema = json_schema(model_type) # Use plain print for JSON output to avoid Rich formatting print(json.dumps(schema, indent=2, sort_keys=True)) @@ -786,7 +745,7 @@ def json_schema_command( "--exclude-tag", "excluded_tags", multiple=True, - help="Filter types by tag (e.g., overture:theme=base)", + help="Exclude types by tag (e.g., overture:theme=base)", ) @click.option( "--group-by", @@ -797,8 +756,7 @@ def list_types( ) -> None: r"""List all available types grouped by theme with descriptions. - Displays all registered Overture Maps types organized by theme, - including model class names and docstrings. + Displays all registered Overture Maps types and can organized by grouping. \b Examples: @@ -807,21 +765,8 @@ def list_types( """ try: models = discover_models() - filters = [] - - if tags: - filters.append(lambda key: all(tag in key.tags for tag in tags)) - if excluded_tags: - filters.append( - lambda key: not any(tag in key.tags for tag in excluded_tags) - ) - if filters: - models = { - key: model - for key, model in models.items() - if all(f(key) for f in filters) - } + models = filter_models(models, tags=tags, excluded_tags=excluded_tags) if group_by: grouped_models: dict[str, set[ModelKey]] = {} diff --git a/packages/overture-schema-cli/tests/test_cli_commands.py b/packages/overture-schema-cli/tests/test_cli_commands.py index 7b6f3b42f..6c18d58ca 100644 --- a/packages/overture-schema-cli/tests/test_cli_commands.py +++ b/packages/overture-schema-cli/tests/test_cli_commands.py @@ -16,8 +16,8 @@ def test_list_types_command(self, cli_runner: CliRunner) -> None: """Test the list-types command.""" result = cli_runner.invoke(cli, ["list-types"]) assert result.exit_code == 0 - # Should show theme names - assert "BUILDINGS" in result.output or "buildings" in result.output + # Should show theme tag + assert "overture:theme=buildings" in result.output # Should show type names assert "building" in result.output @@ -33,7 +33,9 @@ class TestJsonSchemaCommand: def test_json_schema_generates_valid_output(self, cli_runner: CliRunner) -> None: """Test that json-schema command generates valid JSON.""" - result = cli_runner.invoke(cli, ["json-schema", "--theme", "buildings"]) + result = cli_runner.invoke( + cli, ["json-schema", "--tag", "overture:theme=buildings"] + ) assert result.exit_code == 0 # Should be valid JSON @@ -57,7 +59,7 @@ def test_validate_flat_format_input(self, cli_runner: CliRunner) -> None: flat_feature = build_feature(geojson_format=False) flat_json = json.dumps(flat_feature) result = cli_runner.invoke( - cli, ["validate", "--theme", "buildings", "-"], input=flat_json + cli, ["validate", "--tag", "overture:theme=buildings", "-"], input=flat_json ) assert result.exit_code == 0 assert "Successfully validated " in result.output @@ -222,7 +224,7 @@ def test_validate_with_nonexistent_filters_raises_error( # Try to validate with a nonexistent theme result = cli_runner.invoke( cli, - ["validate", "--theme", "nonexistent_theme", "-"], + ["validate", "--tag", "overture:theme=nonexistent_theme", "-"], input=building_feature_yaml_content, ) # UsageError exits with code 2 @@ -254,7 +256,7 @@ def test_validate_with_valid_theme_invalid_type_raises_error( # Try to validate buildings theme with a type that doesn't exist in that theme result = cli_runner.invoke( cli, - ["validate", "--theme", "buildings", "--type", "segment", "-"], + ["validate", "--tag", "overture:theme=buildings", "--type", "segment", "-"], input=building_feature_yaml_content, ) # UsageError exits with code 2 diff --git a/packages/overture-schema-cli/tests/test_cli_functions.py b/packages/overture-schema-cli/tests/test_cli_functions.py index 4218541f5..8c17ccd36 100644 --- a/packages/overture-schema-cli/tests/test_cli_functions.py +++ b/packages/overture-schema-cli/tests/test_cli_functions.py @@ -203,7 +203,7 @@ class TestPerformValidation: def test_perform_validation_raises_for_invalid_single_feature(self) -> None: """Test that perform_validation raises ValidationError for single invalid feature.""" data = build_feature(id=None) # Missing required 'id' - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -218,7 +218,7 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: id=None, coordinates=[[[2, 2], [3, 2], [3, 3], [2, 3], [2, 2]]] ) data = [feature1, feature2] - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -230,7 +230,7 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: def test_perform_validation_empty_list(self) -> None: """Test validating an empty list (edge case).""" data: list[dict[str, object]] = [] - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) # Should not raise perform_validation(data, model_type) @@ -238,7 +238,7 @@ def test_perform_validation_empty_list(self) -> None: def test_perform_validation_empty_feature_collection(self) -> None: """Test validating an empty FeatureCollection (edge case).""" data = {"type": "FeatureCollection", "features": []} - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types(("overture:theme=buildings",), (), ()) # Should not raise perform_validation(data, model_type) @@ -248,10 +248,10 @@ def test_perform_validation_with_different_themes(self) -> None: data = build_feature(theme="buildings", type="building") # Should work with buildings theme - buildings_type = resolve_types(False, None, ("buildings",), ()) + buildings_type = resolve_types(("overture:theme=buildings",), (), ()) perform_validation(data, buildings_type) # Should fail with wrong theme - places_type = resolve_types(False, None, ("places",), ()) + places_type = resolve_types(("overture:theme=places",), (), ()) with pytest.raises(ValidationError): perform_validation(data, places_type) diff --git a/packages/overture-schema-cli/tests/test_error_formatting.py b/packages/overture-schema-cli/tests/test_error_formatting.py index fa8ef1a5d..168d9d53c 100644 --- a/packages/overture-schema-cli/tests/test_error_formatting.py +++ b/packages/overture-schema-cli/tests/test_error_formatting.py @@ -40,7 +40,9 @@ def test_ambiguous_data_shows_most_likely_errors( version: 0 """) - result = cli_runner.invoke(cli, ["validate", "--theme", "buildings", filename]) + result = cli_runner.invoke( + cli, ["validate", "--tag", "overture:theme=buildings", filename] + ) assert result.exit_code == 1 diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 976e17416..cd2c04594 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -1,173 +1,172 @@ """Parametrized tests for resolve_types function.""" +from typing import get_args +from unittest.mock import Mock, patch + import pytest from overture.schema.cli.commands import resolve_types -from overture.schema.system.discovery import tags_by_key +from overture.schema.system.discovery import ModelKey +DISCOVER_MODELS = "overture.schema.cli.commands.discover_models" -class TestResolveTypes: - """Tests for the resolve_types function with various filter combinations.""" +# Mock model classes +class Place: + pass + + +class Segment: + pass + + +class Connector: + pass + + +class Building: + pass + + +class Sources: + pass + + +# Mock ModelKey instances +BUILDING_KEY = ModelKey( + name="building", + entry_point="mock:MyClass", + tags=frozenset({"feature", "overture", "overture:theme=buildings"}), +) +SEGMENT_KEY = ModelKey( + name="segment", + entry_point="mock:Segment", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +CONNECTOR_KEY = ModelKey( + name="connector", + entry_point="mock:Connector", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +PLACE_KEY = ModelKey( + name="place", + entry_point="mock:Place", + tags=frozenset({"feature", "overture", "overture:theme=places"}), +) +SOURCES_KEY = ModelKey( + name="sources", + entry_point="mock:Sources", + tags=frozenset({"overture"}), +) + +MOCK_MODELS = { + BUILDING_KEY: Building, + SEGMENT_KEY: Segment, + CONNECTOR_KEY: Connector, + PLACE_KEY: Place, + SOURCES_KEY: Sources, +} + + +class TestResolveTypes: @pytest.mark.parametrize( - "overture_types,namespace,theme_names,type_names,should_succeed", + "tags,excluded_tags,type_names,should_succeed", [ - # Test --overture-types flag - pytest.param(True, None, (), (), True, id="overture_types_only"), - pytest.param(False, "overture", (), (), True, id="overture_namespace"), - # Test theme filtering - pytest.param(False, None, ("buildings",), (), True, id="theme_buildings"), pytest.param( - False, None, ("transportation",), (), True, id="theme_transportation" + ("overture:theme=buildings",), (), (), True, id="tag_buildings" ), pytest.param( - False, None, ("buildings", "places"), (), True, id="multiple_themes" - ), - pytest.param(False, None, ("nonexistent",), (), False, id="invalid_theme"), - # Test type filtering - pytest.param(False, None, (), ("building",), True, id="type_building"), - pytest.param(False, None, (), ("segment",), True, id="type_segment"), - pytest.param( - False, None, (), ("building", "place"), True, id="multiple_types" + ("overture:theme=transportation",), + (), + (), + True, + id="tag_transportation", ), - pytest.param(False, None, (), ("nonexistent",), False, id="invalid_type"), - # Test combined theme + type filtering + pytest.param(("overture:theme=places",), (), (), True, id="tag_places"), + pytest.param(("nonexistent",), (), (), False, id="unknown_tag"), + pytest.param((), (), ("building",), True, id="type_building"), + pytest.param((), (), ("segment",), True, id="type_segment"), + pytest.param((), (), ("nonexistent",), False, id="invalid_type"), pytest.param( - False, - None, - ("buildings",), + ("overture:theme=buildings",), + (), ("building",), True, - id="theme_and_type_match", + id="tag_and_type_match", ), pytest.param( - False, - None, - ("buildings",), + ("overture:theme=buildings",), + (), ("segment",), False, - id="theme_and_type_mismatch", + id="tag_and_type_mismatch", ), pytest.param( - False, - None, - ("transportation",), - ("segment", "connector"), - True, - id="theme_with_multiple_types", - ), - # Test namespace combined with theme/type - pytest.param( - False, - "overture", - ("buildings",), + ("overture:theme=transportation",), (), + ("segment", "connector"), True, - id="namespace_with_theme", - ), - pytest.param( - False, - "overture", - (), - ("building",), - True, - id="namespace_with_type", - ), - pytest.param( - False, - "overture", - ("buildings",), - ("building",), - True, - id="namespace_with_theme_and_type", + id="tag_with_multiple_types", ), - # Test no filters (all models) - pytest.param(False, None, (), (), True, id="no_filters_all_models"), + pytest.param((), (), (), True, id="no_filters_all_models"), ], ) def test_resolve_types_combinations( self, - overture_types: bool, - namespace: str | None, - theme_names: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], type_names: tuple[str, ...], should_succeed: bool, ) -> None: - """Test resolve_types with various filter combinations.""" - if should_succeed: - model_type = resolve_types( - overture_types, namespace, theme_names, type_names - ) - assert model_type is not None - else: - with pytest.raises(ValueError, match="No models found"): - resolve_types(overture_types, namespace, theme_names, type_names) - - @pytest.mark.parametrize( - "namespace,expected_themes", - [ - pytest.param( - "overture", - { - "buildings", - "places", - "transportation", - "base", - "divisions", - "addresses", - }, - id="overture_namespace", - ), - ], - ) - def test_resolve_types_returns_expected_themes( - self, - namespace: str, - expected_themes: set[str], - ) -> None: - """Test that resolve_types returns models from expected themes.""" - from overture.schema.system.discovery import discover_models - - models = discover_models() - actual_themes = { - next(iter(tags_by_key(key.tags, "overture:theme")), None) - for key in models.keys() - } - - # Check that we have at least the expected themes (may have more) - assert expected_themes.issubset(actual_themes), ( - f"Missing expected themes. Expected {expected_themes}, got {actual_themes}" - ) - - -class TestResolveTypesEdgeCases: - """Tests for edge cases in resolve_types.""" + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + if should_succeed: + union = resolve_types(tags, excluded_tags, type_names) + assert union is not None + else: + with pytest.raises(ValueError, match="No models found"): + resolve_types(tags, excluded_tags, type_names) def test_resolve_types_case_sensitive(self) -> None: - """Test that theme and type names are case-sensitive.""" - # Lowercase should work - model_type = resolve_types(False, None, ("buildings",), ()) - assert model_type is not None - - # Uppercase should fail (themes are lowercase in registry) - with pytest.raises(ValueError, match="No models found"): - resolve_types(False, None, ("BUILDINGS",), ()) + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + # Lowercase should work + union = resolve_types((), (), ("building",)) + assert union is not None + # Uppercase should fail + with pytest.raises(ValueError, match="No models found"): + resolve_types((), (), ("BUILDING",)) def test_resolve_types_empty_result_error_message(self) -> None: - """Test that a helpful error message is shown when no models match.""" - with pytest.raises(ValueError) as exc_info: - resolve_types(False, None, ("nonexistent",), ("also_fake",)) - - assert "No models found" in str(exc_info.value) - - def test_resolve_types_namespace_isolation(self) -> None: - """Test that namespace filtering properly isolates models.""" - # Get all models (no namespace filter) - all_models_type = resolve_types(False, None, (), ()) - assert all_models_type is not None - - # Get only overture namespace - overture_type = resolve_types(False, "overture", (), ()) - assert overture_type is not None - - # Both should work, but they represent different sets of models - # (This test primarily ensures no exceptions are raised) + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + with pytest.raises(ValueError) as exc_info: + resolve_types(("nonexistent",), (), ("also_fake",)) + assert "No models found" in str(exc_info.value) + + def test_resolve_types_excluded_tags(self) -> None: + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + # Exclude 'overture:theme=buildings' tag + union = resolve_types((), ("overture:theme=buildings",), ()) + # Should not include Building model + assert not any(issubclass(model, Mock) for model in get_args(union)) + + def test_resolve_types_no_filters_returns_all(self) -> None: + with patch( + DISCOVER_MODELS, + return_value=MOCK_MODELS, + ): + union = resolve_types((), (), ()) + # Should include all mock models + assert all( + any(issubclass(model, t) for model in getattr(union, "__args__", [])) + for t in [Building, Segment, Connector, Place, Sources] + ) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index 8624bfc7a..801c55fb0 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -6,7 +6,7 @@ import click -from overture.schema.system.discovery import discover_models, tags_by_key +from overture.schema.system.discovery import discover_models, filter_models from .extraction.model_extraction import extract_model from .extraction.specs import ( @@ -47,11 +47,6 @@ def _write_output( click.echo() # separate entries with a blank line in stdout mode -def _find_theme(tags: frozenset[str]) -> str | None: - """Find the theme tag in a set of tags, if any.""" - return next(iter(tags_by_key(tags, "overture:theme")), None) - - @click.group() def cli() -> None: """Overture Schema code generator. @@ -81,9 +76,16 @@ def list_models() -> None: help="Output format", ) @click.option( - "--theme", + "--tag", + "tags", + multiple=True, + help="Tag(s) to include; repeatable (e.g., --tag feature --tag overture)", +) +@click.option( + "--exclude-tag", + "excluded_tags", multiple=True, - help="Filter to specific theme(s); repeatable (e.g., --theme buildings --theme places)", + help="Tag(s) to exclude; repeatable (e.g., --exclude-tag draft --exclude-tag overture:theme=base)", ) @click.option( "--output-dir", @@ -93,21 +95,18 @@ def list_models() -> None: ) def generate( output_format: str, - theme: tuple[str, ...], + tags: tuple[str, ...], + excluded_tags: tuple[str, ...], output_dir: Path | None, ) -> None: """Generate code/docs from discovered models.""" all_models = discover_models() - # Schema root from ALL entry points (before theme filter). + # Schema root from ALL entry points (before tag filters). module_paths = [entry_point_module(k.entry_point) for k in all_models] schema_root = compute_schema_root(module_paths) - models = ( - {k: v for k, v in all_models.items() if _find_theme(k.tags) in theme} - if theme - else all_models - ) + models = filter_models(all_models, tags=tags, excluded_tags=excluded_tags) if output_dir: output_dir.mkdir(parents=True, exist_ok=True) diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index eecd45627..0ca7061e7 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -48,10 +48,11 @@ def test_generate_markdown_to_stdout(self, cli_runner: CliRunner) -> None: assert result.exit_code == 0 assert "# Building" in result.output or "# " in result.output - def test_generate_with_theme_filter(self, cli_runner: CliRunner) -> None: - """generate --theme should filter to specific theme.""" + def test_generate_with_tag_filter(self, cli_runner: CliRunner) -> None: + """generate --tag should filter to specific theme.""" result = cli_runner.invoke( - cli, ["generate", "--format", "markdown", "--theme", "buildings"] + cli, + ["generate", "--format", "markdown", "--tag", "overture:theme=buildings"], ) assert result.exit_code == 0 @@ -68,8 +69,8 @@ def test_generate_markdown_feature_at_theme_level( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -93,8 +94,8 @@ def test_feature_pages_have_sidebar_position( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -211,8 +212,8 @@ def test_generates_category_files( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -311,8 +312,8 @@ def test_generate_markdown_includes_enum_files( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -344,7 +345,8 @@ def spy(feature_specs: list, schema_root: str, output_dir: object) -> None: monkeypatch.setattr("overture.schema.codegen.cli._generate_markdown", spy) result = cli_runner.invoke( - cli, ["generate", "--format", "markdown", "--theme", "buildings"] + cli, + ["generate", "--format", "markdown", "--tag", "overture:theme=buildings"], ) assert result.exit_code == 0 @@ -381,8 +383,8 @@ def test_segment_appears_in_markdown_output( "generate", "--format", "markdown", - "--theme", - "transportation", + "--tag", + "overture:theme=transportation", "--output-dir", str(tmp_path), ], @@ -411,8 +413,8 @@ def test_used_by_sections_appear_in_markdown( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index 1141f08f2..d59c9ba5f 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -172,6 +172,29 @@ def discover_models( return models +def filter_models( + models: dict[ModelKey, type[BaseModel]], + tags: tuple[str, ...] = (), + excluded_tags: tuple[str, ...] = (), + type_names: tuple[str, ...] = (), +) -> dict[ModelKey, type[BaseModel]]: + """Filter models to those that contain all required tags.""" + filters = [] + + if tags: + filters.append(lambda key: all(tag in key.tags for tag in tags)) + if excluded_tags: + filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + if type_names: + filters.append(lambda key: key.name in type_names) + + if filters: + models = { + key: model for key, model in models.items() if all(f(key) for f in filters) + } + return models + + def get_registered_model(feature_type: str) -> type[BaseModel] | None: """Get the Pydantic model for a type. From 77f3acd6a1cee029e53ccb15a9f7882a4991a853 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:32:28 +0100 Subject: [PATCH 08/23] refactor(system): Adds reserved namespaces and logging to tag filtering Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../src/overture/schema/core/tag_providers.py | 2 +- .../src/overture/schema/system/discovery.py | 86 +++++++++++-- .../tests/test_tag_providers.py | 117 ++++++++++++++++++ .../overture-schema-system/tests/test_tags.py | 79 +++++++++++- 4 files changed, 269 insertions(+), 15 deletions(-) create mode 100644 packages/overture-schema-system/tests/test_tag_providers.py diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index e4e9a72d6..4eec8761c 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -37,7 +37,7 @@ def theme_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: for tp in _extract_types(model_class): - if issubclass(tp, OvertureFeature): + if isinstance(tp, type) and issubclass(tp, OvertureFeature): tags.add( "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] ) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py index d59c9ba5f..5a3eca2dd 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery.py @@ -5,7 +5,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, replace -from typing import Annotated, Any, Literal, Union, get_args, get_origin +from typing import Annotated, Any, Literal, TypeAlias, Union, get_args, get_origin from pydantic import BaseModel @@ -18,6 +18,11 @@ "overture": {"overture-schema-core"}, "feature": {"overture-schema-system"}, } +RESERVED_NAMESPACES: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "system": {"overture-schema-system"}, +} + TAG = r"[a-z0-9][a-z0-9_-]*" NAMESPACE_TAG = r"[a-z0-9]+:[a-z0-9]+(?:=[a-z0-9_.-]+)?" TAG_RE = re.compile(rf"^(?:{TAG}|{NAMESPACE_TAG})$") @@ -61,13 +66,19 @@ class TagProviderKey: package_name: str # distribution package name -TagProvider = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] +TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] + +ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] + +TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] + +ModelKeyFilter: TypeAlias = Callable[[ModelKey], bool] def generate_tags( model_class: type[BaseModel], key: ModelKey, - providers: dict[TagProviderKey, TagProvider], + providers: TagProviderDict, ) -> set[str]: tags: set[str] = set() @@ -85,16 +96,53 @@ def generate_tags( def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: - reserved_tags = tuple( - tag for tag, dist in RESERVED_TAGS.items() if provider.package_name not in dist - ) + """Filter tags, removing invalid, reserved, or namespace-restricted tags for a provider.""" + filtered_tags: set[str] = set() + reserved_tags: set[str] = { + tag for tag, pkgs in RESERVED_TAGS.items() if provider.package_name not in pkgs + } + reserved_namespaces: set[str] = { + ns + for ns, pkgs in RESERVED_NAMESPACES.items() + if provider.package_name not in pkgs + } + + for tag in tags: + # Validate tag format + if not TAG_RE.fullmatch(tag): + logger.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " + f"This tag does not match the required format." + ) + continue + + # Reserved tag check + if tag in reserved_tags: + allowed_pkgs = RESERVED_TAGS.get(tag, set()) + logger.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " + f"This tag can only be set by packages from: {allowed_pkgs}." + ) + continue + + # Reserved namespace check + tag_ns = namespace(tag) + if tag_ns and tag_ns in reserved_namespaces: + allowed_pkgs = RESERVED_NAMESPACES.get(tag_ns, set()) + logger.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " + f"This namespace can only be set by packages from: {allowed_pkgs}." + ) + continue - return {tag for tag in tags if TAG_RE.match(tag) and tag not in reserved_tags} + filtered_tags.add(tag) + + return filtered_tags def discover_tag_providers( tag_providers_group: str = "overture.tag_providers", -) -> dict[TagProviderKey, TagProvider]: +) -> TagProviderDict: tag_providers = {} try: @@ -123,7 +171,7 @@ def discover_tag_providers( def discover_models( model_group: str = "overture.models", -) -> dict[ModelKey, type[BaseModel]]: +) -> ModelDict: """Discover all registered Overture models via entry points. Parameters @@ -173,13 +221,13 @@ def discover_models( def filter_models( - models: dict[ModelKey, type[BaseModel]], + models: ModelDict, tags: tuple[str, ...] = (), excluded_tags: tuple[str, ...] = (), type_names: tuple[str, ...] = (), -) -> dict[ModelKey, type[BaseModel]]: +) -> ModelDict: """Filter models to those that contain all required tags.""" - filters = [] + filters: list[ModelKeyFilter] = [] if tags: filters.append(lambda key: all(tag in key.tags for tag in tags)) @@ -221,6 +269,15 @@ def get_registered_model(feature_type: str) -> type[BaseModel] | None: return None +def namespace(tag: str) -> str: + """Extract the namespace from a tag, or return an empty string if there is no namespace.""" + if not TAG_RE.fullmatch(tag): + raise ValueError(f"Invalid tag format: {tag}") + if ":" in tag: + return tag.split(":")[0] + return "" + + def tags_by_key(tags: frozenset[str] | set[str], key: str) -> set[str]: """Extract values for k/v tags with the given key. @@ -244,7 +301,10 @@ def tags_by_namespace(tags: frozenset[str] | set[str], namespace: str) -> set[st def feature_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - if any(issubclass(tp, Feature) for tp in _extract_types(model_class)): + if any( + isinstance(tp, type) and issubclass(tp, Feature) + for tp in _extract_types(model_class) + ): tags.add("feature") return tags diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py new file mode 100644 index 000000000..e1cfc3ca9 --- /dev/null +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -0,0 +1,117 @@ +import pytest +from pydantic import BaseModel + +from overture.schema.system.discovery import ( + ModelKey, + TagProviderKey, + _filter_tags, + feature_provider, +) +from overture.schema.system.feature import Feature + + +@pytest.fixture +def core_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="core", entry_point="core:Provider", package_name="overture-schema-core" + ) + + +@pytest.fixture +def system_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="system", + entry_point="system:Provider", + package_name="overture-schema-system", + ) + + +@pytest.fixture +def other_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="other", entry_point="other:Provider", package_name="other-package" + ) + + +@pytest.fixture +def feature() -> type[Feature]: + class SomeFeature(Feature): + pass + + return SomeFeature + + +@pytest.fixture +def not_a_feature() -> type[BaseModel]: + class NotAFeature(BaseModel): + pass + + return NotAFeature + + +def test_valid_tags(other_tag_provider: TagProviderKey) -> None: + tags = {"valid", "other:valid", "other:valid=true"} + filtered = _filter_tags(tags, other_tag_provider) + assert filtered == tags + + +def test_invalid_tag(other_tag_provider: TagProviderKey) -> None: + tags = {"InvalidTag"} + filtered = _filter_tags(tags, other_tag_provider) + assert filtered == set() + + +def test_reserved_tag(other_tag_provider: TagProviderKey) -> None: + tags = {"overture", "feature", "valid"} + filtered = _filter_tags(tags, other_tag_provider) + assert "valid" in filtered + assert "overture" not in filtered + assert "feature" not in filtered + + +def test_allowed_reserved_tag( + core_tag_provider: TagProviderKey, system_tag_provider: TagProviderKey +) -> None: + assert "overture" in _filter_tags({"overture"}, core_tag_provider) + assert "feature" in _filter_tags({"feature"}, system_tag_provider) + + +def test_reserved_namespace(other_tag_provider: TagProviderKey) -> None: + tags = {"overture:feature", "system:feature", "valid:tag"} + filtered = _filter_tags(tags, other_tag_provider) + assert "valid:tag" in filtered + assert "overture:feature" not in filtered + assert "system:feature" not in filtered + + +def test_allowed_reserved_namespace( + core_tag_provider: TagProviderKey, system_tag_provider: TagProviderKey +) -> None: + assert "overture:feature" in _filter_tags({"overture:feature"}, core_tag_provider) + assert "system:feature" in _filter_tags({"system:feature"}, system_tag_provider) + + +def test_empty_tags(other_tag_provider: TagProviderKey) -> None: + assert _filter_tags(set(), other_tag_provider) == set() + + +def test_mixed_tags(other_tag_provider: TagProviderKey) -> None: + tags = {"valid", "feature", "overture:feature", "InvalidTag"} + filtered = _filter_tags(tags, other_tag_provider) + assert filtered == {"valid"} + + +def test_feature_provider_adds_feature_tag(feature: type[Feature]) -> None: + key = ModelKey(name="feature", entry_point="system:Feature", tags=frozenset()) + result = feature_provider(feature, key, set()) + assert "feature" in result + + +def test_feature_provider_does_not_add_feature_tag( + not_a_feature: type[BaseModel], +) -> None: + key = ModelKey( + name="notafeature", entry_point="system:NotAFeature", tags=frozenset() + ) + result = feature_provider(not_a_feature, key, set()) + assert "feature" not in result diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index c7fc28c9b..d5efa90c2 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -1,4 +1,13 @@ -from overture.schema.system.discovery import tags_by_key, tags_by_namespace +import re +import unittest + +from overture.schema.system.discovery import ( + NAMESPACE_TAG, + TAG, + TAG_RE, + tags_by_key, + tags_by_namespace, +) def test_tags_by_key_returns_correct_values() -> None: @@ -41,3 +50,71 @@ def test_tags_by_namespace_handles_empty_tags() -> None: namespace = "system" result = tags_by_namespace(tags, namespace) assert result == set() + + +class TestSimpleTagRegex(unittest.TestCase): + def test_valid_simple_tags(self) -> None: + valid_tags = [ + "v", + "valid", + "valid1", + "valid_tag", + "valid-tag", + "0valid", + "42", + ] + for tag in valid_tags: + self.assertTrue(re.fullmatch(TAG, tag), f"Should match: {tag}") + self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + + def test_invalid_simple_tags(self) -> None: + invalid_tags = [ + "", + "_invalid", + "-invalid", + "Invalid", + "invalid!", + "invalid ", + "in.valid", + "3.14", + ] + for tag in invalid_tags: + self.assertFalse(re.fullmatch(TAG, tag), f"Should not match: {tag}") + self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}") + + +class TestNamespaceTagRegex(unittest.TestCase): + def test_valid_namespace_tags(self) -> None: + valid_tags = [ + "ns:predicate", + "ns:predicate1", + "ns:predicate=value", + "ns:predicate=value_0", + "ns:predicate=value-0", + "ns:predicate=value.0", + "ns:predicate=value_2-3.4", + "ns:predicate=42", + "ns:predicate=3.14", + ] + for tag in valid_tags: + self.assertTrue(re.fullmatch(NAMESPACE_TAG, tag), f"Should match: {tag}") + self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + + def test_invalid_namespace_tags(self) -> None: + invalid_tags = [ + "ns:", + ":predicate", + "ns:predicate=", + "ns:predicate=Value", + "ns:predicate=value ", + "ns:predicate=value!", + "ns:predicate=ns:value", + "ns:predicate=predicate=value", + "Ns:predicate", + "ns:Predicate", + ] + for tag in invalid_tags: + self.assertFalse( + re.fullmatch(NAMESPACE_TAG, tag), f"Should not match: {tag}" + ) + self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}") From 7ae97ab0bc1501dd87c2ff413ff3a98462c9cd13 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Wed, 8 Apr 2026 16:55:45 +0200 Subject: [PATCH 09/23] refactor(system): tighten discovery api and various small improvements Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- README.pydantic.md | 6 +- .../src/overture/schema/cli/__init__.py | 2 - .../src/overture/schema/cli/commands.py | 21 +- .../src/overture/schema/cli/types.py | 5 - .../tests/test_resolve_types.py | 4 +- packages/overture-schema-codegen/README.md | 2 +- .../tests/codegen_test_support.py | 7 +- .../src/overture/schema/core/tag_providers.py | 70 ++-- .../tests/test_approved_models.py | 4 +- .../overture-schema-system/pyproject.toml | 2 +- .../src/overture/schema/system/discovery.py | 340 ------------------ .../schema/system/discovery/__init__.py | 13 + .../schema/system/discovery/discovery.py | 257 +++++++++++++ .../schema/system/discovery/models.py | 39 ++ .../overture/schema/system/discovery/tag.py | 104 ++++++ .../schema/system/discovery/tag_providers.py | 31 ++ .../overture/schema/system/discovery/types.py | 13 + .../src/overture/schema/system/typing_util.py | 43 +++ .../tests/test_tag_providers.py | 9 +- .../overture-schema-system/tests/test_tags.py | 85 +++-- 20 files changed, 606 insertions(+), 451 deletions(-) delete mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/models.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/tag.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/discovery/types.py create mode 100644 packages/overture-schema-system/src/overture/schema/system/typing_util.py diff --git a/README.pydantic.md b/README.pydantic.md index 213426d43..dc19d8472 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -164,9 +164,9 @@ from overture.schema.system.discovery import discover_models, get_registered_mod all_models = discover_models() # Returns: # { -# ("building", "acme:Building", {"building_tag"}): BuildingModel, -# ("place", "acme:Place", {"place_tag"}): PlaceModel, -# ... +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", tags=frozenset({"feature", "overture", "overture:theme=buildings"})): BuildingModel, +# ModelKey(name="place", entry_point="overture.schema.places:Place", tags=frozenset({"feature", "overture", "overture:theme=places"})): PlaceModel, +# ... # } # Get a specific model by type diff --git a/packages/overture-schema-cli/src/overture/schema/cli/__init__.py b/packages/overture-schema-cli/src/overture/schema/cli/__init__.py index 85045f0c0..8fd3e8bfd 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/__init__.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/__init__.py @@ -11,7 +11,6 @@ ) from .types import ( ErrorLocation, - ModelDict, UnionType, ValidationErrorDict, ) @@ -25,7 +24,6 @@ "perform_validation", "resolve_types", "ErrorLocation", - "ModelDict", "UnionType", "ValidationErrorDict", ] diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 3b298c130..3e196818d 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -22,8 +22,9 @@ ModelKey, discover_models, filter_models, - tags_by_key, ) +from overture.schema.system.discovery.tag import get_values_for_key +from overture.schema.system.discovery.types import ModelDict from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema @@ -34,7 +35,7 @@ select_most_likely_errors, ) from .type_analysis import StructuralTuple, get_item_index, introspect_union -from .types import ErrorLocation, ModelDict, UnionType +from .types import ErrorLocation, UnionType # Console instances for rich output stdout = Console(highlight=False) @@ -214,7 +215,9 @@ def resolve_types( models: ModelDict = discover_models() # Filter models based on CLI options - models = filter_models(models, tags, excluded_tags, type_names) + models = filter_models( + models, tags=tags, excluded_tags=excluded_tags, type_names=type_names + ) if not models: raise ValueError("No models found matching the specified criteria") @@ -749,14 +752,14 @@ def json_schema_command( ) @click.option( "--group-by", - help="Group types by tag prefix (e.g., 'overture:theme')", + help="Group types by tag key (e.g., 'overture:theme')", ) def list_types( tags: tuple[str, ...], excluded_tags: tuple[str, ...], group_by: str | None ) -> None: - r"""List all available types grouped by theme with descriptions. + r"""List all available types. - Displays all registered Overture Maps types and can organized by grouping. + Displays all registered models and can be organized by grouping. \b Examples: @@ -772,7 +775,7 @@ def list_types( grouped_models: dict[str, set[ModelKey]] = {} for key in models.keys(): - if groups := tags_by_key(key.tags, group_by): + if groups := get_values_for_key(key.tags, group_by): for group in groups: grouped_models.setdefault(group, set()).add(key) @@ -793,7 +796,7 @@ def list_types( model.append("→ ", style="bright_black") model.append(key.name, style="bold cyan") model.pad_right(max(1, padding - len(key.name))) - model.append_text(Text().append(" ".join(sorted(key.tags)))) + model.append(" ".join(sorted(key.tags))) stdout.print(model) stdout.print() @@ -804,7 +807,7 @@ def list_types( model = Text() model.append(key.name, style="bold cyan") model.pad_right(max(1, padding - len(key.name))) - model.append_text(Text().append(" ".join(sorted(key.tags)))) + model.append(" ".join(sorted(key.tags))) stdout.print(model) except Exception as e: diff --git a/packages/overture-schema-cli/src/overture/schema/cli/types.py b/packages/overture-schema-cli/src/overture/schema/cli/types.py index f438edf2f..f1394def8 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/types.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/types.py @@ -5,15 +5,10 @@ from pydantic import BaseModel from pydantic_core import ErrorDetails -from overture.schema.system.discovery import ModelKey - # Type alias for union types created from Pydantic models # This represents either a single model or a discriminated union of models UnionType: TypeAlias = type[BaseModel] | Any -# Dictionary mapping ModelKey to Pydantic model classes -ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] - # Pydantic validation error dictionary structure # In Pydantic v2, ValidationError.errors() returns list[ErrorDetails] ValidationErrorDict: TypeAlias = ErrorDetails diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index cd2c04594..42b4a3c07 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -1,7 +1,7 @@ """Parametrized tests for resolve_types function.""" from typing import get_args -from unittest.mock import Mock, patch +from unittest.mock import patch import pytest from overture.schema.cli.commands import resolve_types @@ -157,7 +157,7 @@ def test_resolve_types_excluded_tags(self) -> None: # Exclude 'overture:theme=buildings' tag union = resolve_types((), ("overture:theme=buildings",), ()) # Should not include Building model - assert not any(issubclass(model, Mock) for model in get_args(union)) + assert not any(issubclass(model, Building) for model in get_args(union)) def test_resolve_types_no_filters_returns_all(self) -> None: with patch( diff --git a/packages/overture-schema-codegen/README.md b/packages/overture-schema-codegen/README.md index 92a4d8fbe..e9f5b0fba 100644 --- a/packages/overture-schema-codegen/README.md +++ b/packages/overture-schema-codegen/README.md @@ -22,7 +22,7 @@ renderers, not extraction logic. overture-codegen generate --format markdown --output-dir docs/schema/reference # Generate for a single theme -overture-codegen generate --format markdown --theme buildings --output-dir out/ +overture-codegen generate --format markdown --tag overture:theme=buildings --output-dir out/ # List discovered models overture-codegen list diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 4ba7c0e3b..6260ab5b0 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -25,7 +25,8 @@ is_model_class, ) from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind -from overture.schema.system.discovery import discover_models, tags_by_key +from overture.schema.system.discovery import discover_models, filter_models +from overture.schema.system.discovery.tag import get_values_for_key from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import require_any_of @@ -303,7 +304,7 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec: def find_theme(tags: frozenset[str]) -> str | None: """Extract the theme from a set of tags, if present.""" - return next(iter(tags_by_key(tags, "overture:theme")), None) + return next(iter(get_values_for_key(tags, "overture:theme")), None) T = TypeVar("T") @@ -337,7 +338,7 @@ def flat_specs_from_discovery( """Build a flat list of ModelSpecs from discovery, with entry_point set.""" models = discover_models() if theme: - models = {k: v for k, v in models.items() if find_theme(k.tags) == theme} + models = filter_models(models, tags=(f"overture:theme={theme}",)) result = [] for key, cls in models.items(): if not is_model_class(cls): diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 4eec8761c..00cba5991 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -1,9 +1,10 @@ -from typing import Annotated, Any, Literal, Union, get_args, get_origin +from typing import get_args from pydantic import BaseModel from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ModelKey +from overture.schema.system.typing_util import collect_types APPROVED = { "overture.schema.addresses:Address", @@ -28,6 +29,22 @@ def authority_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: + """Add the ``"overture"`` tag if the model originates from an approved Overture package. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to inspect. + key : ModelKey + Key identifying the model. + tags : set[str] + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with ``"overture"`` added if applicable. + """ if _matches_manifest(key): tags.add("overture") return tags @@ -36,8 +53,24 @@ def authority_provider( def theme_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - for tp in _extract_types(model_class): - if isinstance(tp, type) and issubclass(tp, OvertureFeature): + """Add the ``"overture:theme={theme}"`` tag if the model is a subclass of OvertureFeature. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to inspect. + key : ModelKey + Key identifying the model. + tags : set[str] + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with ``"overture:theme={theme}"`` added if applicable. + """ + for tp in collect_types(model_class): + if issubclass(tp, OvertureFeature): tags.add( "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] ) @@ -46,34 +79,3 @@ def theme_provider( def _matches_manifest(key: ModelKey) -> bool: return key.entry_point in APPROVED - - -def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 - result: set[type] = set() - - def visit(t: Any) -> None: # noqa: ANN401 - origin = get_origin(t) - if origin is Annotated: - visit(get_args(t)[0]) - return - - if hasattr(t, "__supertype__"): - visit(t.__supertype__) - return - - origin = get_origin(t) - - if origin is Union: - for arg in get_args(t): - visit(arg) - return - - if origin is Literal: - for val in get_args(t): - result.add(type(val)) - return - - result.add(t) - - visit(tp) - return result diff --git a/packages/overture-schema-core/tests/test_approved_models.py b/packages/overture-schema-core/tests/test_approved_models.py index d7ccb9830..36f30d302 100644 --- a/packages/overture-schema-core/tests/test_approved_models.py +++ b/packages/overture-schema-core/tests/test_approved_models.py @@ -4,7 +4,7 @@ def test_overture_feature_models_are_official() -> None: models = discover_models() for key in models: - if "overture:feature" in key.tags: - assert "overture:official" in key.tags, ( + if "feature" in key.tags: + assert "overture" in key.tags, ( f"Model {key.name} is missing 'overture:official' tag." ) diff --git a/packages/overture-schema-system/pyproject.toml b/packages/overture-schema-system/pyproject.toml index f4771f04c..0646d7a0a 100644 --- a/packages/overture-schema-system/pyproject.toml +++ b/packages/overture-schema-system/pyproject.toml @@ -57,4 +57,4 @@ ignore = [ per-file-ignores = {"__init__.py" = ["F401"]} [project.entry-points."overture.tag_providers"] -feature = "overture.schema.system.discovery:feature_provider" \ No newline at end of file +feature = "overture.schema.system.discovery.tag_providers:feature_provider" diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery.py deleted file mode 100644 index 5a3eca2dd..000000000 --- a/packages/overture-schema-system/src/overture/schema/system/discovery.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Model discovery system for Overture schema registry.""" - -import importlib.metadata -import logging -import re -from collections.abc import Callable -from dataclasses import dataclass, replace -from typing import Annotated, Any, Literal, TypeAlias, Union, get_args, get_origin - -from pydantic import BaseModel - -from overture.schema.system.feature import Feature - -logger = logging.getLogger(__name__) - - -RESERVED_TAGS: dict[str, set[str]] = { - "overture": {"overture-schema-core"}, - "feature": {"overture-schema-system"}, -} -RESERVED_NAMESPACES: dict[str, set[str]] = { - "overture": {"overture-schema-core"}, - "system": {"overture-schema-system"}, -} - -TAG = r"[a-z0-9][a-z0-9_-]*" -NAMESPACE_TAG = r"[a-z0-9]+:[a-z0-9]+(?:=[a-z0-9_.-]+)?" -TAG_RE = re.compile(rf"^(?:{TAG}|{NAMESPACE_TAG})$") - - -@dataclass(frozen=True, slots=True) -class ModelKey: - """Key identifying a registered model by name, entry point, and tags. - - Attributes - ---------- - name : str - The friendly name of the model, derived from the entry point key - entry_point : str - The entry point value in "module:Class" format - tags : frozenset[str] - A set of tags associated with the model, including both plain tags and structured tags - - """ - - name: str # friendly name from entry point key - entry_point: str # The entry point value in "module:Class" format - tags: frozenset[str] # plain and structured tags - - -@dataclass(frozen=True, slots=True) -class TagProviderKey: - """Key identifying a registered model by namespace, theme, and type. - - Attributes - ---------- - name : str - The friendly name of the model, derived from the entry point key - entry_point : str - The entry point value in "module:Class" format - - """ - - name: str # friendly name from entry point key - entry_point: str # entry point value (module:Class) - package_name: str # distribution package name - - -TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] - -ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] - -TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] - -ModelKeyFilter: TypeAlias = Callable[[ModelKey], bool] - - -def generate_tags( - model_class: type[BaseModel], - key: ModelKey, - providers: TagProviderDict, -) -> set[str]: - tags: set[str] = set() - - for provider_key, provider in providers.items(): - try: - added_tags = provider(model_class, key, tags.copy()).difference(tags) - filtered_tags = _filter_tags(added_tags, provider_key) - tags.update(filtered_tags) - except Exception as e: - logger.warning( - f"Error in tag provider {provider.__name__} for model {key.name}: {e}" - ) - - return tags - - -def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: - """Filter tags, removing invalid, reserved, or namespace-restricted tags for a provider.""" - filtered_tags: set[str] = set() - reserved_tags: set[str] = { - tag for tag, pkgs in RESERVED_TAGS.items() if provider.package_name not in pkgs - } - reserved_namespaces: set[str] = { - ns - for ns, pkgs in RESERVED_NAMESPACES.items() - if provider.package_name not in pkgs - } - - for tag in tags: - # Validate tag format - if not TAG_RE.fullmatch(tag): - logger.debug( - f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " - f"This tag does not match the required format." - ) - continue - - # Reserved tag check - if tag in reserved_tags: - allowed_pkgs = RESERVED_TAGS.get(tag, set()) - logger.debug( - f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " - f"This tag can only be set by packages from: {allowed_pkgs}." - ) - continue - - # Reserved namespace check - tag_ns = namespace(tag) - if tag_ns and tag_ns in reserved_namespaces: - allowed_pkgs = RESERVED_NAMESPACES.get(tag_ns, set()) - logger.debug( - f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " - f"This namespace can only be set by packages from: {allowed_pkgs}." - ) - continue - - filtered_tags.add(tag) - - return filtered_tags - - -def discover_tag_providers( - tag_providers_group: str = "overture.tag_providers", -) -> TagProviderDict: - tag_providers = {} - - try: - for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): - try: - tag_provider_class = tag_provider.load() - - key = TagProviderKey( - name=tag_provider.name, - entry_point=tag_provider.value, - package_name=getattr(tag_provider.dist, "name", ""), - ) - - tag_providers[key] = tag_provider_class - - except Exception as e: - # Log warning but don't fail for individual tag providers - logger.warning( - "Could not load tag provider %s: %s", tag_provider.name, e - ) - except Exception as e: - logger.warning("Could not discover entry points: %s", e) - - return tag_providers - - -def discover_models( - model_group: str = "overture.models", -) -> ModelDict: - """Discover all registered Overture models via entry points. - - Parameters - ---------- - model_group: str - The entry point group to search for models (default: "overture.models") - - Returns - ------- - dict[ModelKey, type[BaseModel]] - Dict mapping ModelKey to model classes. - Theme will be None for entries without an explicit theme component. - """ - models = {} - tag_providers = discover_tag_providers() - - try: - for model in importlib.metadata.entry_points(group=model_group): - try: - model_class = model.load() - - key = ModelKey( - name=model.name, - entry_point=model.value, - tags=frozenset(), - ) - - try: - key = replace( - key, - tags=frozenset(generate_tags(model_class, key, tag_providers)), - ) - except Exception as e: - logger.warning( - "Could not resolve tags for model %s: %s", model.name, e - ) - - models[key] = model_class - - except Exception as e: - # Log warning but don't fail for individual models - logger.warning("Could not load model %s: %s", model.name, e) - except Exception as e: - logger.warning("Could not discover entry points: %s", e) - - return models - - -def filter_models( - models: ModelDict, - tags: tuple[str, ...] = (), - excluded_tags: tuple[str, ...] = (), - type_names: tuple[str, ...] = (), -) -> ModelDict: - """Filter models to those that contain all required tags.""" - filters: list[ModelKeyFilter] = [] - - if tags: - filters.append(lambda key: all(tag in key.tags for tag in tags)) - if excluded_tags: - filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) - if type_names: - filters.append(lambda key: key.name in type_names) - - if filters: - models = { - key: model for key, model in models.items() if all(f(key) for f in filters) - } - return models - - -def get_registered_model(feature_type: str) -> type[BaseModel] | None: - """Get the Pydantic model for a type. - - This uses setuptools entry points for registration. - If multiple types share the same name, the first one encountered will be returned. - - Parameters - ---------- - feature_type : str - The type name - - Returns - ------- - type[BaseModel] | None - The first encountered model class if found, None otherwise. - - """ - # Check all discovered models for a match - models = discover_models() - # Need to find by type, not exact key match - for key, model_class in models.items(): - if key.name == feature_type: - return model_class - return None - - -def namespace(tag: str) -> str: - """Extract the namespace from a tag, or return an empty string if there is no namespace.""" - if not TAG_RE.fullmatch(tag): - raise ValueError(f"Invalid tag format: {tag}") - if ":" in tag: - return tag.split(":")[0] - return "" - - -def tags_by_key(tags: frozenset[str] | set[str], key: str) -> set[str]: - """Extract values for k/v tags with the given key. - - tags_by_key(frozenset({"overture:theme=buildings", "overture", "draft"}), "overture:theme") - -> {"buildings"} - """ - prefix = key + "=" - return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} - - -def tags_by_namespace(tags: frozenset[str] | set[str], namespace: str) -> set[str]: - """Extract tag bodies within a namespace. - - tags_by_namespace(frozenset({"system:extension", "overture"}), "system") - -> {"extension"} - """ - prefix = namespace + ":" - return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} - - -def feature_provider( - model_class: type[BaseModel], key: ModelKey, tags: set[str] -) -> set[str]: - if any( - isinstance(tp, type) and issubclass(tp, Feature) - for tp in _extract_types(model_class) - ): - tags.add("feature") - return tags - - -def _extract_types(tp: Any) -> set[type]: # noqa: ANN401 - result: set[type] = set() - - def visit(t: Any) -> None: # noqa: ANN401 - origin = get_origin(t) - if origin is Annotated: - visit(get_args(t)[0]) - return - - if hasattr(t, "__supertype__"): - visit(t.__supertype__) - return - - origin = get_origin(t) - - if origin is Union: - for arg in get_args(t): - visit(arg) - return - - if origin is Literal: - for val in get_args(t): - result.add(type(val)) - return - - result.add(t) - - visit(tp) - return result diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py new file mode 100644 index 000000000..b7ccb1250 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py @@ -0,0 +1,13 @@ +from . import tag +from .discovery import discover_models, filter_models, get_registered_model +from .models import ModelKey +from .types import ModelDict + +__all__ = [ + "tag", + "ModelKey", + "ModelDict", + "discover_models", + "filter_models", + "get_registered_model", +] diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py new file mode 100644 index 000000000..f9cab6804 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -0,0 +1,257 @@ +"""Model discovery system for Overture schema registry.""" + +import importlib.metadata +import logging +from dataclasses import replace + +from pydantic import BaseModel + +from overture.schema.system.discovery.tag import ( + get_namespace, + is_valid_tag, +) +from overture.schema.system.discovery.types import ( + ModelDict, + ModelKey, + ModelKeyFilter, + TagProviderDict, + TagProviderKey, +) + +log = logging.getLogger(__name__) + +# Tags that are reserved and can only be set by specific packages. +_RESERVED_TAGS: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "feature": {"overture-schema-system"}, +} +# Namespaces that are reserved and can only be set by specific packages. +_RESERVED_NAMESPACES: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "system": {"overture-schema-system"}, +} + + +def generate_tags( + model_class: type[BaseModel], + key: ModelKey, + providers: TagProviderDict, +) -> set[str]: + """Generate tags for a model class using tag providers. + + Each provider is called in turn indeterministically; tags it adds are filtered for + validity and permission before being included. Provider errors are caught and + logged as warnings rather than propagated. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to generate tags for. + key : ModelKey + Key identifying the model. + providers : TagProviderDict + Tag providers to invoke. + + Returns + ------- + set[str] + Tags generated for the model. + """ + tags: set[str] = set() + for provider_key, provider in providers.items(): + try: + added_tags = provider(model_class, key, tags.copy()).difference(tags) + filtered_tags = _filter_tags(added_tags, provider_key) + tags.update(filtered_tags) + except Exception as e: + log.warning( + f"Error in tag provider {provider.__name__} for model {key.name}: {e}" + ) + return tags + + +def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: + """Filter tags that cannot be used by the provider, including invalid tags, + reserved tags, and tags using a reserved namespace. + + Parameters + ---------- + tags : set[str] + Tags to filter. + provider : TagProviderKey + Provider attempting to set the tags. + + Returns + ------- + set[str] + Permitted tags. + """ + filtered_tags: set[str] = set() + reserved_tags: set[str] = { + tag for tag, pkgs in _RESERVED_TAGS.items() if provider.package_name not in pkgs + } + reserved_namespaces: set[str] = { + ns + for ns, pkgs in _RESERVED_NAMESPACES.items() + if provider.package_name not in pkgs + } + for tag in tags: + if not is_valid_tag(tag): + log.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " + f"This tag does not match the required format." + ) + continue + if tag in reserved_tags: + allowed_pkgs = _RESERVED_TAGS.get(tag, set()) + log.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " + f"This tag can only be set by packages from: {allowed_pkgs}." + ) + continue + tag_ns = get_namespace(tag) + if tag_ns and tag_ns in reserved_namespaces: + allowed_pkgs = _RESERVED_NAMESPACES.get(tag_ns, set()) + log.debug( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " + f"This namespace can only be set by packages from: {allowed_pkgs}." + ) + continue + filtered_tags.add(tag) + return filtered_tags + + +def discover_tag_providers( + tag_providers_group: str = "overture.tag_providers", +) -> TagProviderDict: + """Discover and load tag providers via entry points. + + Parameters + ---------- + tag_providers_group : str, optional + Entry point group to search (default: ``"overture.tag_providers"``). + + Returns + ------- + TagProviderDict + Discovered tag providers keyed by TagProviderKey. + """ + tag_providers = {} + try: + for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): + try: + tag_provider_class = tag_provider.load() + key = TagProviderKey( + name=tag_provider.name, + entry_point=tag_provider.value, + package_name=getattr(tag_provider.dist, "name", ""), + ) + tag_providers[key] = tag_provider_class + except Exception as e: + log.warning(f"Could not load tag provider {tag_provider.name}: {e}") + except Exception as e: + log.warning(f"Could not discover entry points: {e}") + return tag_providers + + +def discover_models( + model_group: str = "overture.models", +) -> ModelDict: + """Discover and load models via entry points, attaching tags from tag providers. + + Parameters + ---------- + model_group : str, optional + Entry point group to search (default: ``"overture.models"``). + + Returns + ------- + ModelDict + Discovered models keyed by ModelKey. + """ + models = {} + tag_providers = discover_tag_providers() + try: + for model in importlib.metadata.entry_points(group=model_group): + try: + model_class = model.load() + key = ModelKey( + name=model.name, + entry_point=model.value, + tags=frozenset(), + ) + try: + key = replace( + key, + tags=frozenset(generate_tags(model_class, key, tag_providers)), + ) + except Exception as e: + log.warning(f"Could not resolve tags for model {model.name}: {e}") + models[key] = model_class + except Exception as e: + log.warning(f"Could not load model {model.name}: {e}") + except Exception as e: + log.warning(f"Could not discover entry points: {e}") + return models + + +def filter_models( + models: ModelDict, + *, + tags: tuple[str, ...] = (), + excluded_tags: tuple[str, ...] = (), + type_names: tuple[str, ...] = (), +) -> ModelDict: + """Filter models by required tags, excluded tags, and/or type names. + + Parameters + ---------- + models : ModelDict + Models to filter. + tags : tuple[str, ...], optional + Tags that must all be present on the model. + excluded_tags : tuple[str, ...], optional + Tags that must not be present on the model. + type_names : tuple[str, ...], optional + Model names to include; all others are excluded. + + Returns + ------- + ModelDict + Filtered models. + """ + filters: list[ModelKeyFilter] = [] + if tags: + filters.append(lambda key: all(tag in key.tags for tag in tags)) + if excluded_tags: + filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) + if type_names: + filters.append(lambda key: key.name in type_names) + if filters: + models = { + key: model for key, model in models.items() if all(f(key) for f in filters) + } + return models + + +def get_registered_model(model_name: str) -> type[BaseModel] | None: + """Get the model by name. + + Loads all models via entry points and returns the first with a matching name. + If multiple models share the same name, the first one encountered is returned. + + Parameters + ---------- + model_name : str + Model name to look up. + + Returns + ------- + type[BaseModel] or None + Model class if found, otherwise ``None``. + """ + models = discover_models() + for key, model_class in models.items(): + if key.name == model_name: + return model_class + return None diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/models.py b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py new file mode 100644 index 000000000..9c331b466 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py @@ -0,0 +1,39 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ModelKey: + """Key identifying a registered model by name, entry point, and tags. + + Attributes + ---------- + name : str + Friendly name derived from the entry point key. + entry_point : str + Entry point value in ``"module:Class"`` format. + tags : frozenset[str] + Tags associated with the model. + """ + + name: str + entry_point: str + tags: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class TagProviderKey: + """Key identifying a registered tag provider by name, entry point, and package. + + Attributes + ---------- + name : str + Friendly name derived from the entry point key. + entry_point : str + Entry point value in ``"module:function"`` format. + package_name : str + Package that provides this tag provider. + """ + + name: str + entry_point: str + package_name: str diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py new file mode 100644 index 000000000..9176b1998 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py @@ -0,0 +1,104 @@ +"""Tag format specification and utilities for Overture schema discovery. + +Tags follow the pattern ``[namespace:]predicate[=value]`` and come in three forms: + +- **Plain** — ``overture``, ``feature`` +- **Namespaced** — ``system:extension` +- **Key/value** — ``overture:theme=buildings`` + +``:`` signals ownership and reservation — only the owning package may set tags in a +given namespace. ``=`` signals a dimension with a discrete value. +One level of each: no nested colons, no multiple ``=`` signs. + +Tag matching is case-sensitive throughout. +""" + +import re + +PLAIN_TAG = r"[a-z0-9][a-z0-9_-]*" +NAMESPACE = PREDICATE = r"[a-z0-9][a-z0-9_.-]*" +VALUE = r"[a-zA-Z0-9_.-]+" +NAMESPACE_TAG = rf"{NAMESPACE}:{PREDICATE}(?:={VALUE})?" +TAG = re.compile(rf"^(?:{PLAIN_TAG}|{NAMESPACE_TAG})$") + + +def get_namespace(tag: str) -> str: + """Extract the namespace prefix from a namespaced tag. + + Parameters + ---------- + tag : str + A valid tag string. + + Returns + ------- + str + The namespace prefix if the tag is a namespaced tag, otherwise ``""``. + + Examples + -------- + >>> get_namespace("overture:theme=buildings") + 'overture' + """ + return tag.split(":")[0] if is_valid_tag(tag) and ":" in tag else "" + + +def get_values_for_key(tags: frozenset[str] | set[str], key: str) -> set[str]: + """Extract values from key/value namespaced tags matching the given key. + + Parameters + ---------- + tags : frozenset[str] or set[str] + Tags to search. + key : str + Key to match, e.g. ``"overture:theme"``. + + Returns + ------- + set[str] + Values of tags matching ``key=``. + + Examples + -------- + >>> get_values_for_key(frozenset({"overture:theme=buildings", "overture"}), "overture:theme") + {'buildings'} + """ + prefix = key + "=" + return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} + + +def is_valid_tag(tag: str) -> bool: + """Check whether a string is a valid tag. + + A valid tag is a plain tag, a namespaced tag, or a key/value tag: + + - **Plain**: ``[a-z0-9][a-z0-9_-]*`` — lowercase alphanumeric, hyphens, + underscores; no dots. + - **Namespace / predicate**: ``[a-z0-9][a-z0-9_.-]*`` — same but dots + are also allowed. + - **Key/value**: ``{namespace}:{predicate}=[a-zA-Z0-9_.-]+`` — namespace and predicate as + above; value is alphanumeric (upper and lower case), hyphens, underscores, or dots; + must be non-empty. + + Parameters + ---------- + tag : str + String to validate. + + Returns + ------- + bool + ``True`` if `tag` matches the required format. + + Examples + -------- + >>> is_valid_tag("feature") + True + >>> is_valid_tag("overture:theme=buildings") + True + >>> is_valid_tag("overture:theme=") + False + >>> is_valid_tag("Invalid") + False + """ + return bool(TAG.fullmatch(tag)) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py new file mode 100644 index 000000000..e336a3e2f --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py @@ -0,0 +1,31 @@ +"""Tag provider logic for Overture schema discovery system.""" + +from pydantic import BaseModel + +from overture.schema.system.discovery.types import ModelKey +from overture.schema.system.feature import Feature +from overture.schema.system.typing_util import collect_types + + +def feature_provider( + model_class: type[BaseModel], key: ModelKey, tags: set[str] +) -> set[str]: + """Add the ``"feature"`` tag if the model is a subclass of Feature. + + Parameters + ---------- + model_class : type[BaseModel] + Model class to inspect. + key : ModelKey + Key identifying the model. + tags : set[str] + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with ``"feature"`` added if applicable. + """ + if any(issubclass(tp, Feature) for tp in collect_types(model_class)): + tags.add("feature") + return tags diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py new file mode 100644 index 000000000..7ea168353 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -0,0 +1,13 @@ +"""Types and data classes for Overture schema discovery system.""" + +from collections.abc import Callable +from typing import TypeAlias + +from pydantic import BaseModel + +from .models import ModelKey, TagProviderKey + +TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] +ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] +TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] +ModelKeyFilter: TypeAlias = Callable[[ModelKey], bool] diff --git a/packages/overture-schema-system/src/overture/schema/system/typing_util.py b/packages/overture-schema-system/src/overture/schema/system/typing_util.py new file mode 100644 index 000000000..1d312bcf3 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/typing_util.py @@ -0,0 +1,43 @@ +"""Typing utilities for the Overture schema system.""" + +import types +from typing import Annotated, Any, Literal, Union, get_args, get_origin + + +def collect_types(tp: Any) -> set[type]: # noqa: ANN401 + """Collect all concrete types from a type annotation. + + Recursively unwraps ``Annotated``, ``NewType``, ``Union``/``X | Y``, and + ``Literal`` to collect the concrete types they contain. Only actual `type` + instances are returned. + + Parameters + ---------- + tp : Any + A type annotation to inspect. + + Returns + ------- + set[type] + All concrete types found within ``tp``. + + """ + result: set[type] = set() + + def _visit(t: Any) -> None: + origin = get_origin(t) + if origin is Annotated: + _visit(get_args(t)[0]) + elif hasattr(t, "__supertype__"): + _visit(t.__supertype__) + elif origin is Union or origin is types.UnionType: + for arg in get_args(t): + _visit(arg) + elif origin is Literal: + for val in get_args(t): + result.add(type(val)) + elif isinstance(t, type): + result.add(t) + + _visit(tp) + return result diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py index e1cfc3ca9..8c5bc06cc 100644 --- a/packages/overture-schema-system/tests/test_tag_providers.py +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -1,12 +1,9 @@ import pytest from pydantic import BaseModel -from overture.schema.system.discovery import ( - ModelKey, - TagProviderKey, - _filter_tags, - feature_provider, -) +from overture.schema.system.discovery.discovery import _filter_tags +from overture.schema.system.discovery.tag_providers import feature_provider +from overture.schema.system.discovery.types import ModelKey, TagProviderKey from overture.schema.system.feature import Feature diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index d5efa90c2..f203200c7 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -1,59 +1,38 @@ import re import unittest -from overture.schema.system.discovery import ( +from overture.schema.system.discovery.tag import ( NAMESPACE_TAG, + PLAIN_TAG, TAG, - TAG_RE, - tags_by_key, - tags_by_namespace, + get_values_for_key, + is_valid_tag, ) -def test_tags_by_key_returns_correct_values() -> None: +def test_get_values_for_key_returns_correct_values() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "overture:theme" - result = tags_by_key(tags, key) + result = get_values_for_key(tags, key) assert result == {"buildings"} -def test_tags_by_key_returns_empty_set_for_nonexistent_key() -> None: +def test_get_values_for_key_returns_empty_set_for_nonexistent_key() -> None: tags = frozenset({"overture:theme=buildings", "overture", "draft"}) key = "nonexistent:key" - result = tags_by_key(tags, key) + result = get_values_for_key(tags, key) assert result == set() -def test_tags_by_key_handles_empty_tags() -> None: +def test_get_values_for_key_handles_empty_tags() -> None: tags: frozenset[str] = frozenset() key = "overture:theme" - result = tags_by_key(tags, key) + result = get_values_for_key(tags, key) assert result == set() -def test_tags_by_namespace_returns_correct_values() -> None: - tags = frozenset({"system:extension", "overture"}) - namespace = "system" - result = tags_by_namespace(tags, namespace) - assert result == {"extension"} - - -def test_tags_by_namespace_returns_empty_set_for_nonexistent_namespace() -> None: - tags = frozenset({"system:extension", "overture"}) - namespace = "nonexistent" - result = tags_by_namespace(tags, namespace) - assert result == set() - - -def test_tags_by_namespace_handles_empty_tags() -> None: - tags: frozenset[str] = frozenset() - namespace = "system" - result = tags_by_namespace(tags, namespace) - assert result == set() - - -class TestSimpleTagRegex(unittest.TestCase): - def test_valid_simple_tags(self) -> None: +class TestPlainTagRegex(unittest.TestCase): + def test_valid_plain_tags(self) -> None: valid_tags = [ "v", "valid", @@ -64,10 +43,15 @@ def test_valid_simple_tags(self) -> None: "42", ] for tag in valid_tags: - self.assertTrue(re.fullmatch(TAG, tag), f"Should match: {tag}") - self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + self.assertTrue( + re.fullmatch(PLAIN_TAG, tag), f"PLAIN_TAG should match: {tag}" + ) + self.assertTrue(TAG.fullmatch(tag), f"TAG should match: {tag}") + self.assertTrue( + is_valid_tag(tag), f"is_valid_tag should return True for: {tag}" + ) - def test_invalid_simple_tags(self) -> None: + def test_invalid_plain_tags(self) -> None: invalid_tags = [ "", "_invalid", @@ -79,8 +63,13 @@ def test_invalid_simple_tags(self) -> None: "3.14", ] for tag in invalid_tags: - self.assertFalse(re.fullmatch(TAG, tag), f"Should not match: {tag}") - self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}") + self.assertFalse( + re.fullmatch(PLAIN_TAG, tag), f"PLAIN_TAG should not match: {tag}" + ) + self.assertFalse(TAG.fullmatch(tag), f"TAG should not match: {tag}") + self.assertFalse( + is_valid_tag(tag), f"is_valid_tag should return False for: {tag}" + ) class TestNamespaceTagRegex(unittest.TestCase): @@ -88,6 +77,7 @@ def test_valid_namespace_tags(self) -> None: valid_tags = [ "ns:predicate", "ns:predicate1", + "ns:predicate-1", "ns:predicate=value", "ns:predicate=value_0", "ns:predicate=value-0", @@ -95,17 +85,22 @@ def test_valid_namespace_tags(self) -> None: "ns:predicate=value_2-3.4", "ns:predicate=42", "ns:predicate=3.14", + "ns:predicate=Value", ] for tag in valid_tags: - self.assertTrue(re.fullmatch(NAMESPACE_TAG, tag), f"Should match: {tag}") - self.assertTrue(TAG_RE.fullmatch(tag), f"TAG_RE should match: {tag}") + self.assertTrue( + re.fullmatch(NAMESPACE_TAG, tag), f"NAMESPACE_TAG should match: {tag}" + ) + self.assertTrue(TAG.fullmatch(tag), f"TAG should match: {tag}") + self.assertTrue( + is_valid_tag(tag), f"is_valid_tag should return True for: {tag}" + ) def test_invalid_namespace_tags(self) -> None: invalid_tags = [ "ns:", ":predicate", "ns:predicate=", - "ns:predicate=Value", "ns:predicate=value ", "ns:predicate=value!", "ns:predicate=ns:value", @@ -115,6 +110,10 @@ def test_invalid_namespace_tags(self) -> None: ] for tag in invalid_tags: self.assertFalse( - re.fullmatch(NAMESPACE_TAG, tag), f"Should not match: {tag}" + re.fullmatch(NAMESPACE_TAG, tag), + f"NAMESPACE_TAG should not match: {tag}", + ) + self.assertFalse(TAG.fullmatch(tag), f"TAG should not match: {tag}") + self.assertFalse( + is_valid_tag(tag), f"is_valid_tag should return False for: {tag}" ) - self.assertFalse(TAG_RE.fullmatch(tag), f"TAG_RE should not match: {tag}") From cfad0665529f04d275c52e0bdc456fc208e82f16 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Tue, 28 Apr 2026 15:39:08 -0700 Subject: [PATCH 10/23] Simplify module path in entry point example Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Seth Fitzsimmons --- README.pydantic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.pydantic.md b/README.pydantic.md index dc19d8472..d966d92e7 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -151,8 +151,8 @@ Registration is done in the `[project.entry-points."overture.models"]` section: ```toml [project.entry-points."overture.models"] -building = "overture.schema.buildings.building.models:Building" -building_part = "overture.schema.buildings.building_part.models:BuildingPart" +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" ``` The discovery system provides programmatic access to registered models: From caaff4427f2331e1b4efb438738cadacda65e6ed Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 29 Apr 2026 15:39:05 -0700 Subject: [PATCH 11/23] feat(system,cli,codegen): tag combinator algebra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `filter_models` selects feature types from the registry through three combinators applied to the same tag grammar (plain `feature`, namespaced `system:extension`, or compound `overture:theme=buildings`): --tag OR defines scope (any-of) --filter AND narrows scope (all-of) --exclude OR-NOT subtracts (none-of) --type OR closed-list match on ModelKey.name (orthogonal) T = ⋃ tag predicates (absent → U) F = ⋂ filter predicates (absent → U) E = ⋃ exclude predicates (absent → ∅) result = (T ∩ F \ E) restricted to type_names if non-empty The mental model is procedural: --tag widens, --filter narrows, --exclude subtracts. Without --tag the scope is every registered model. An empty selector imposes no filtering. A `TagSelector` value object carries the three tag predicates: class TagSelector: include_any: tuple[str, ...] = () require_all: tuple[str, ...] = () exclude_any: tuple[str, ...] = () Field names encode the combinator (any-of / all-of / none-of), deliberately distinct from CLI flag names. Flags are user-facing affordances; field names are implementation-facing and self-document at the call site. `type_names` lives on `filter_models` as a keyword, not on `TagSelector`. It's a closed-list match on `ModelKey.name`, orthogonal to the tag predicate algebra. Isolating it makes `TagSelector`'s purpose statable in one sentence and confines a future fold-in of `--type` to a kwarg deletion that doesn't disturb `TagSelector`. User-facing help text frames flags as acting on feature types ("Include feature types with these tags — defines scope (OR; repeatable)"). Internal API docstrings keep "models" since they describe the Python class layer; "feature types" is the user-facing vocabulary for entry-point-registered top-level types, distinct from the Pydantic models used for nested fields. Signed-off-by: Seth Fitzsimmons --- .../src/overture/schema/cli/commands.py | 102 +++----- .../src/overture/schema/cli/tag_options.py | 66 +++++ .../tests/test_cli_commands.py | 43 +++ .../tests/test_cli_functions.py | 23 +- .../tests/test_resolve_types.py | 163 +++--------- .../overture-schema-codegen/pyproject.toml | 2 + .../src/overture/schema/codegen/cli.py | 24 +- .../tests/codegen_test_support.py | 10 +- .../overture-schema-codegen/tests/test_cli.py | 38 +++ .../src/overture/schema/core/tag_providers.py | 15 +- .../schema/system/discovery/__init__.py | 12 +- .../schema/system/discovery/discovery.py | 86 ++++-- .../schema/system/discovery/models.py | 4 +- .../overture/schema/system/discovery/tag.py | 28 +- .../schema/system/discovery/tag_providers.py | 4 +- .../overture/schema/system/discovery/types.py | 1 - .../schema/system/field_constraint/string.py | 8 +- .../src/overture/schema/system/typing_util.py | 6 +- .../tests/test_discovery_filter_models.py | 244 ++++++++++++++++++ .../tests/test_discovery_tag_selector.py | 44 ++++ uv.lock | 2 + 21 files changed, 645 insertions(+), 280 deletions(-) create mode 100644 packages/overture-schema-cli/src/overture/schema/cli/tag_options.py create mode 100644 packages/overture-schema-system/tests/test_discovery_filter_models.py create mode 100644 packages/overture-schema-system/tests/test_discovery_tag_selector.py diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 3e196818d..2590bf721 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -19,12 +19,13 @@ from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ( + ModelDict, ModelKey, + TagSelector, discover_models, filter_models, ) from overture.schema.system.discovery.tag import get_values_for_key -from overture.schema.system.discovery.types import ModelDict from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema @@ -34,6 +35,7 @@ group_errors_by_discriminator, select_most_likely_errors, ) +from .tag_options import build_selector, tag_selection_options from .type_analysis import StructuralTuple, get_item_index, introspect_union from .types import ErrorLocation, UnionType @@ -195,29 +197,13 @@ def validate_features(data: list, model_type: UnionType) -> list[BaseModel]: def resolve_types( - tags: tuple[str, ...], - excluded_tags: tuple[str, ...], - type_names: tuple[str, ...], + selector: TagSelector = TagSelector(), + *, + type_names: tuple[str, ...] = (), ) -> UnionType: - """Resolve CLI options into a model type suitable for parse_feature. - - Args - ---- - tags: Tags to include (e.g., "feature", "overture:theme=buildings") - excluded_tags: Tags to exclude (e.g., "draft") - type_names: List of type names from --type option - - Returns - ------- - Model type suitable for passing to parse_feature - """ - # Discover models - models: ModelDict = discover_models() - - # Filter models based on CLI options - models = filter_models( - models, tags=tags, excluded_tags=excluded_tags, type_names=type_names - ) + """Resolve a TagSelector + type-names into a Pydantic union type.""" + models = discover_models() + models = filter_models(models, selector, type_names=type_names) if not models: raise ValueError("No models found matching the specified criteria") @@ -500,11 +486,12 @@ def handle_validation_error( # Show heterogeneity warning if collection has mixed types if is_heterogeneous: stderr.print( - " ⚠ Heterogeneous collection: Data contains multiple feature types.", + " ⚠ Heterogeneous collection: Data contains multiple feature types. Consider:", style="yellow", ) stderr.print( - " • Consider validating each type separately with --tag or --type", + " • Validating each type separately with --tag, --filter, " + "--exclude, or --type", style="dim", ) stderr.print() @@ -604,18 +591,7 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: @cli.command() @click.argument("filename", type=click.Path(path_type=Path), required=True) -@click.option( - "--tag", - "tags", - multiple=True, - help="Tags to include (e.g., overture:theme=addresses)", -) -@click.option( - "--exclude-tag", - "excluded_tags", - multiple=True, - help="Tags to exclude (e.g., overture:theme=base)", -) +@tag_selection_options @click.option( "--type", "types", @@ -631,7 +607,8 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: def validate( filename: Path, tags: tuple[str, ...], - excluded_tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], types: tuple[str, ...], show_fields: tuple[str, ...], ) -> None: @@ -659,7 +636,9 @@ def validate( """ # Resolve model type first (errors here are ValueErrors, not ValidationErrors) try: - model_type = resolve_types(tags, excluded_tags, types) + model_type = resolve_types( + build_selector(tags, filters, excludes), type_names=types + ) except ValueError as e: handle_generic_error(e, filename, "value") return @@ -686,18 +665,7 @@ def validate( @cli.command("json-schema") -@click.option( - "--tag", - "tags", - multiple=True, - help="Tags to include (e.g., overture:theme=addresses)", -) -@click.option( - "--exclude-tag", - "excluded_tags", - multiple=True, - help="Tags to exclude (e.g., overture:theme=base)", -) +@tag_selection_options @click.option( "--type", "types", @@ -706,7 +674,8 @@ def validate( ) def json_schema_command( tags: tuple[str, ...], - excluded_tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], types: tuple[str, ...], ) -> None: r"""Generate JSON schema for Overture Maps types. @@ -729,7 +698,9 @@ def json_schema_command( $ overture-schema json-schema --tag overture --tag feature """ try: - model_type = resolve_types(tags, excluded_tags, types) + model_type = resolve_types( + build_selector(tags, filters, excludes), type_names=types + ) schema = json_schema(model_type) # Use plain print for JSON output to avoid Rich formatting print(json.dumps(schema, indent=2, sort_keys=True)) @@ -738,24 +709,18 @@ def json_schema_command( @cli.command("list-types") -@click.option( - "--tag", - "tags", - multiple=True, - help="Filter types by tag (e.g., overture:theme=addresses)", -) -@click.option( - "--exclude-tag", - "excluded_tags", - multiple=True, - help="Exclude types by tag (e.g., overture:theme=base)", -) +@tag_selection_options @click.option( "--group-by", - help="Group types by tag key (e.g., 'overture:theme')", + help="Group types by a key/value tag's key (e.g. 'overture:theme'). " + "Plain and namespaced tags have no value to group by and are " + "ignored here.", ) def list_types( - tags: tuple[str, ...], excluded_tags: tuple[str, ...], group_by: str | None + tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], + group_by: str | None, ) -> None: r"""List all available types. @@ -768,8 +733,7 @@ def list_types( """ try: models = discover_models() - - models = filter_models(models, tags=tags, excluded_tags=excluded_tags) + models = filter_models(models, build_selector(tags, filters, excludes)) if group_by: grouped_models: dict[str, set[ModelKey]] = {} diff --git a/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py new file mode 100644 index 000000000..3befb1143 --- /dev/null +++ b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py @@ -0,0 +1,66 @@ +"""Shared Click options for tag-based model selection.""" + +from collections.abc import Callable +from typing import TypeVar + +import click + +from overture.schema.system.discovery import TagSelector + +F = TypeVar("F", bound=Callable[..., object]) + +_TAG_SYNTAX_NOTE = ( + "Accepts plain tags (e.g. feature), namespaced tags " + "(e.g. overture:approved), or compound key/value tags " + "(e.g. overture:theme=buildings)." +) + + +def tag_selection_options(func: F) -> F: + """Decorate a Click command with --tag, --filter, and --exclude options. + + The decorated command receives `tags`, `filters`, and `excludes` + keyword arguments (each a `tuple[str, ...]`), suitable for passing to + `build_selector`. + """ + func = click.option( + "--exclude", + "excludes", + multiple=True, + help=( + "Exclude feature types with these tags — removes from scope (OR-NOT; " + f"repeatable). {_TAG_SYNTAX_NOTE}" + ), + )(func) + func = click.option( + "--filter", + "filters", + multiple=True, + help=( + "Require feature types to have these tags — narrows scope (AND; " + f"repeatable). {_TAG_SYNTAX_NOTE}" + ), + )(func) + func = click.option( + "--tag", + "tags", + multiple=True, + help=( + "Include feature types with these tags — defines scope (OR; repeatable). " + f"{_TAG_SYNTAX_NOTE}" + ), + )(func) + return func + + +def build_selector( + tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], +) -> TagSelector: + """Map `tag_selection_options` arguments to a `TagSelector`.""" + return TagSelector( + include_any=tags, + require_all=filters, + exclude_any=excludes, + ) diff --git a/packages/overture-schema-cli/tests/test_cli_commands.py b/packages/overture-schema-cli/tests/test_cli_commands.py index 6c18d58ca..e62df564f 100644 --- a/packages/overture-schema-cli/tests/test_cli_commands.py +++ b/packages/overture-schema-cli/tests/test_cli_commands.py @@ -368,3 +368,46 @@ def test_show_field_in_collection( # Should show id for both features assert "id=first" in stderr_output or "first" in stderr_output assert "id=second" in stderr_output or "second" in stderr_output + + +_TAG_COMBINATOR_FLAGS = ( + pytest.param("--filter", "feature", id="filter"), + pytest.param("--exclude", "overture:theme=places", id="exclude"), +) + + +@pytest.mark.parametrize(("flag", "value"), _TAG_COMBINATOR_FLAGS) +def test_validate_wires_tag_combinator_flag( + cli_runner: CliRunner, + building_feature_yaml: str, + flag: str, + value: str, +) -> None: + """validate wires --filter/--exclude through to the tag selector.""" + result = cli_runner.invoke( + cli, ["validate", building_feature_yaml, "--tag", "overture", flag, value] + ) + assert result.exit_code == 0 + + +@pytest.mark.parametrize(("flag", "value"), _TAG_COMBINATOR_FLAGS) +def test_json_schema_wires_tag_combinator_flag( + cli_runner: CliRunner, + flag: str, + value: str, +) -> None: + """json-schema wires --filter/--exclude through to the tag selector.""" + result = cli_runner.invoke(cli, ["json-schema", "--tag", "overture", flag, value]) + assert result.exit_code == 0 + assert result.output # non-empty JSON + + +@pytest.mark.parametrize(("flag", "value"), _TAG_COMBINATOR_FLAGS) +def test_list_types_wires_tag_combinator_flag( + cli_runner: CliRunner, + flag: str, + value: str, +) -> None: + """list-types wires --filter/--exclude through to the tag selector.""" + result = cli_runner.invoke(cli, ["list-types", "--tag", "overture", flag, value]) + assert result.exit_code == 0 diff --git a/packages/overture-schema-cli/tests/test_cli_functions.py b/packages/overture-schema-cli/tests/test_cli_functions.py index 8c17ccd36..5a3139720 100644 --- a/packages/overture-schema-cli/tests/test_cli_functions.py +++ b/packages/overture-schema-cli/tests/test_cli_functions.py @@ -9,6 +9,7 @@ from click.exceptions import UsageError from conftest import build_feature from overture.schema.cli.commands import load_input, perform_validation, resolve_types +from overture.schema.system.discovery import TagSelector from pydantic import ValidationError @@ -203,7 +204,9 @@ class TestPerformValidation: def test_perform_validation_raises_for_invalid_single_feature(self) -> None: """Test that perform_validation raises ValidationError for single invalid feature.""" data = build_feature(id=None) # Missing required 'id' - model_type = resolve_types(("overture:theme=buildings",), (), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -218,7 +221,9 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: id=None, coordinates=[[[2, 2], [3, 2], [3, 3], [2, 3], [2, 2]]] ) data = [feature1, feature2] - model_type = resolve_types(("overture:theme=buildings",), (), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -230,7 +235,9 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: def test_perform_validation_empty_list(self) -> None: """Test validating an empty list (edge case).""" data: list[dict[str, object]] = [] - model_type = resolve_types(("overture:theme=buildings",), (), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) # Should not raise perform_validation(data, model_type) @@ -238,7 +245,9 @@ def test_perform_validation_empty_list(self) -> None: def test_perform_validation_empty_feature_collection(self) -> None: """Test validating an empty FeatureCollection (edge case).""" data = {"type": "FeatureCollection", "features": []} - model_type = resolve_types(("overture:theme=buildings",), (), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) # Should not raise perform_validation(data, model_type) @@ -248,10 +257,12 @@ def test_perform_validation_with_different_themes(self) -> None: data = build_feature(theme="buildings", type="building") # Should work with buildings theme - buildings_type = resolve_types(("overture:theme=buildings",), (), ()) + buildings_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) perform_validation(data, buildings_type) # Should fail with wrong theme - places_type = resolve_types(("overture:theme=places",), (), ()) + places_type = resolve_types(TagSelector(include_any=("overture:theme=places",))) with pytest.raises(ValidationError): perform_validation(data, places_type) diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 42b4a3c07..55acfd762 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -1,11 +1,16 @@ -"""Parametrized tests for resolve_types function.""" +"""Tests for resolve_types — CLI glue between filter_models and union creation. +The combinator algebra of filter_models itself is covered in +`test_discovery_filter_models.py` in the system package. +""" + +from collections.abc import Iterator from typing import get_args from unittest.mock import patch import pytest from overture.schema.cli.commands import resolve_types -from overture.schema.system.discovery import ModelKey +from overture.schema.system.discovery import ModelKey, TagSelector DISCOVER_MODELS = "overture.schema.cli.commands.discover_models" @@ -19,22 +24,13 @@ class Segment: pass -class Connector: - pass - - class Building: pass -class Sources: - pass - - -# Mock ModelKey instances BUILDING_KEY = ModelKey( name="building", - entry_point="mock:MyClass", + entry_point="mock:Building", tags=frozenset({"feature", "overture", "overture:theme=buildings"}), ) SEGMENT_KEY = ModelKey( @@ -42,131 +38,44 @@ class Sources: entry_point="mock:Segment", tags=frozenset({"feature", "overture", "overture:theme=transportation"}), ) -CONNECTOR_KEY = ModelKey( - name="connector", - entry_point="mock:Connector", - tags=frozenset({"feature", "overture", "overture:theme=transportation"}), -) PLACE_KEY = ModelKey( name="place", entry_point="mock:Place", tags=frozenset({"feature", "overture", "overture:theme=places"}), ) -SOURCES_KEY = ModelKey( - name="sources", - entry_point="mock:Sources", - tags=frozenset({"overture"}), -) MOCK_MODELS = { BUILDING_KEY: Building, SEGMENT_KEY: Segment, - CONNECTOR_KEY: Connector, PLACE_KEY: Place, - SOURCES_KEY: Sources, } -class TestResolveTypes: - @pytest.mark.parametrize( - "tags,excluded_tags,type_names,should_succeed", - [ - pytest.param( - ("overture:theme=buildings",), (), (), True, id="tag_buildings" - ), - pytest.param( - ("overture:theme=transportation",), - (), - (), - True, - id="tag_transportation", - ), - pytest.param(("overture:theme=places",), (), (), True, id="tag_places"), - pytest.param(("nonexistent",), (), (), False, id="unknown_tag"), - pytest.param((), (), ("building",), True, id="type_building"), - pytest.param((), (), ("segment",), True, id="type_segment"), - pytest.param((), (), ("nonexistent",), False, id="invalid_type"), - pytest.param( - ("overture:theme=buildings",), - (), - ("building",), - True, - id="tag_and_type_match", - ), - pytest.param( - ("overture:theme=buildings",), - (), - ("segment",), - False, - id="tag_and_type_mismatch", - ), - pytest.param( - ("overture:theme=transportation",), - (), - ("segment", "connector"), - True, - id="tag_with_multiple_types", - ), - pytest.param((), (), (), True, id="no_filters_all_models"), - ], - ) - def test_resolve_types_combinations( - self, - tags: tuple[str, ...], - excluded_tags: tuple[str, ...], - type_names: tuple[str, ...], - should_succeed: bool, - ) -> None: - with patch( - DISCOVER_MODELS, - return_value=MOCK_MODELS, - ): - if should_succeed: - union = resolve_types(tags, excluded_tags, type_names) - assert union is not None - else: - with pytest.raises(ValueError, match="No models found"): - resolve_types(tags, excluded_tags, type_names) - - def test_resolve_types_case_sensitive(self) -> None: - with patch( - DISCOVER_MODELS, - return_value=MOCK_MODELS, - ): - # Lowercase should work - union = resolve_types((), (), ("building",)) - assert union is not None - # Uppercase should fail - with pytest.raises(ValueError, match="No models found"): - resolve_types((), (), ("BUILDING",)) - - def test_resolve_types_empty_result_error_message(self) -> None: - with patch( - DISCOVER_MODELS, - return_value=MOCK_MODELS, - ): - with pytest.raises(ValueError) as exc_info: - resolve_types(("nonexistent",), (), ("also_fake",)) - assert "No models found" in str(exc_info.value) - - def test_resolve_types_excluded_tags(self) -> None: - with patch( - DISCOVER_MODELS, - return_value=MOCK_MODELS, - ): - # Exclude 'overture:theme=buildings' tag - union = resolve_types((), ("overture:theme=buildings",), ()) - # Should not include Building model - assert not any(issubclass(model, Building) for model in get_args(union)) - - def test_resolve_types_no_filters_returns_all(self) -> None: - with patch( - DISCOVER_MODELS, - return_value=MOCK_MODELS, - ): - union = resolve_types((), (), ()) - # Should include all mock models - assert all( - any(issubclass(model, t) for model in getattr(union, "__args__", [])) - for t in [Building, Segment, Connector, Place, Sources] - ) +@pytest.fixture(autouse=True) +def _patched_discover_models() -> Iterator[None]: + with patch(DISCOVER_MODELS, return_value=MOCK_MODELS): + yield + + +def test_no_filters_returns_union_of_all() -> None: + union = resolve_types(TagSelector()) + assert set(get_args(union)) == {Building, Segment, Place} + + +def test_returns_type_when_filter_matches() -> None: + # Single match collapses to the bare class; multi-match yields a Union. + union = resolve_types(TagSelector(include_any=("overture:theme=transportation",))) + assert union is Segment + + +def test_empty_match_raises_value_error() -> None: + with pytest.raises(ValueError, match="No models found"): + resolve_types(TagSelector(include_any=("nonexistent",))) + + +def test_type_names_are_case_sensitive() -> None: + # Lowercase matches. + assert resolve_types(TagSelector(), type_names=("building",)) is Building + # Uppercase doesn't. + with pytest.raises(ValueError, match="No models found"): + resolve_types(TagSelector(), type_names=("BUILDING",)) diff --git a/packages/overture-schema-codegen/pyproject.toml b/packages/overture-schema-codegen/pyproject.toml index de42c5fb9..daa200a4f 100644 --- a/packages/overture-schema-codegen/pyproject.toml +++ b/packages/overture-schema-codegen/pyproject.toml @@ -6,6 +6,7 @@ requires = ["hatchling"] dependencies = [ "click>=8.0", "jinja2>=3.0", + "overture-schema-cli", "overture-schema-core", "overture-schema-system", "tomli>=2.0; python_version < '3.11'", @@ -19,6 +20,7 @@ name = "overture-schema-codegen" overture-codegen = "overture.schema.codegen.cli:main" [tool.uv.sources] +overture-schema-cli = { workspace = true } overture-schema-core = { workspace = true } overture-schema-system = { workspace = true } diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index 801c55fb0..279f22a84 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -6,7 +6,11 @@ import click -from overture.schema.system.discovery import discover_models, filter_models +from overture.schema.cli.tag_options import build_selector, tag_selection_options +from overture.schema.system.discovery import ( + discover_models, + filter_models, +) from .extraction.model_extraction import extract_model from .extraction.specs import ( @@ -75,18 +79,7 @@ def list_models() -> None: type=click.Choice(_OUTPUT_FORMATS), help="Output format", ) -@click.option( - "--tag", - "tags", - multiple=True, - help="Tag(s) to include; repeatable (e.g., --tag feature --tag overture)", -) -@click.option( - "--exclude-tag", - "excluded_tags", - multiple=True, - help="Tag(s) to exclude; repeatable (e.g., --exclude-tag draft --exclude-tag overture:theme=base)", -) +@tag_selection_options @click.option( "--output-dir", type=click.Path(path_type=Path), @@ -96,7 +89,8 @@ def list_models() -> None: def generate( output_format: str, tags: tuple[str, ...], - excluded_tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], output_dir: Path | None, ) -> None: """Generate code/docs from discovered models.""" @@ -106,7 +100,7 @@ def generate( module_paths = [entry_point_module(k.entry_point) for k in all_models] schema_root = compute_schema_root(module_paths) - models = filter_models(all_models, tags=tags, excluded_tags=excluded_tags) + models = filter_models(all_models, build_selector(tags, filters, excludes)) if output_dir: output_dir.mkdir(parents=True, exist_ok=True) diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 6260ab5b0..4c9be9c9b 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -25,7 +25,11 @@ is_model_class, ) from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind -from overture.schema.system.discovery import discover_models, filter_models +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) from overture.schema.system.discovery.tag import get_values_for_key from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint @@ -338,7 +342,9 @@ def flat_specs_from_discovery( """Build a flat list of ModelSpecs from discovery, with entry_point set.""" models = discover_models() if theme: - models = filter_models(models, tags=(f"overture:theme={theme}",)) + models = filter_models( + models, TagSelector(include_any=(f"overture:theme={theme}",)) + ) result = [] for key, cls in models.items(): if not is_model_class(cls): diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index 0ca7061e7..8ba4efaf9 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -59,6 +59,44 @@ def test_generate_with_tag_filter(self, cli_runner: CliRunner) -> None: assert "Building" in result.output assert "Place" not in result.output + def test_generate_accepts_filter_flag(self, cli_runner: CliRunner) -> None: + """generate accepts --filter without error. + + Selector algebra is covered in test_discovery_filter_models. + """ + result = cli_runner.invoke( + cli, + [ + "generate", + "--format", + "markdown", + "--tag", + "overture", + "--filter", + "feature", + ], + ) + assert result.exit_code == 0 + + def test_generate_accepts_exclude_flag(self, cli_runner: CliRunner) -> None: + """generate accepts --exclude without error. + + Selector algebra is covered in test_discovery_filter_models. + """ + result = cli_runner.invoke( + cli, + [ + "generate", + "--format", + "markdown", + "--tag", + "overture", + "--exclude", + "overture:theme=places", + ], + ) + assert result.exit_code == 0 + def test_generate_markdown_feature_at_theme_level( self, cli_runner: CliRunner, tmp_path: Path ) -> None: diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 00cba5991..8c9dc10b9 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -1,3 +1,10 @@ +"""Tag providers for the core Overture schema package. + +Each provider inspects a discovered model and returns the set of tags +that should be attached. Registered via the +`overture.tag_providers` entry-point group. +""" + from typing import get_args from pydantic import BaseModel @@ -29,7 +36,7 @@ def authority_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - """Add the ``"overture"`` tag if the model originates from an approved Overture package. + """Add the `"overture"` tag if the model originates from an approved Overture package. Parameters ---------- @@ -43,7 +50,7 @@ def authority_provider( Returns ------- set[str] - Updated tags, with ``"overture"`` added if applicable. + Updated tags, with `"overture"` added if applicable. """ if _matches_manifest(key): tags.add("overture") @@ -53,7 +60,7 @@ def authority_provider( def theme_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - """Add the ``"overture:theme={theme}"`` tag if the model is a subclass of OvertureFeature. + """Add the `"overture:theme={theme}"` tag if the model is a subclass of OvertureFeature. Parameters ---------- @@ -67,7 +74,7 @@ def theme_provider( Returns ------- set[str] - Updated tags, with ``"overture:theme={theme}"`` added if applicable. + Updated tags, with `"overture:theme={theme}"` added if applicable. """ for tp in collect_types(model_class): if issubclass(tp, OvertureFeature): diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py index b7ccb1250..272e9c682 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py @@ -1,13 +1,19 @@ from . import tag -from .discovery import discover_models, filter_models, get_registered_model +from .discovery import ( + TagSelector, + discover_models, + filter_models, + get_registered_model, +) from .models import ModelKey from .types import ModelDict __all__ = [ - "tag", - "ModelKey", "ModelDict", + "ModelKey", + "TagSelector", "discover_models", "filter_models", "get_registered_model", + "tag", ] diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index f9cab6804..aebb7a8ae 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -2,7 +2,7 @@ import importlib.metadata import logging -from dataclasses import replace +from dataclasses import dataclass, replace from pydantic import BaseModel @@ -13,7 +13,6 @@ from overture.schema.system.discovery.types import ( ModelDict, ModelKey, - ModelKeyFilter, TagProviderDict, TagProviderKey, ) @@ -129,7 +128,7 @@ def discover_tag_providers( Parameters ---------- tag_providers_group : str, optional - Entry point group to search (default: ``"overture.tag_providers"``). + Entry point group to search (default: `"overture.tag_providers"`). Returns ------- @@ -162,7 +161,7 @@ def discover_models( Parameters ---------- model_group : str, optional - Entry point group to search (default: ``"overture.models"``). + Entry point group to search (default: `"overture.models"`). Returns ------- @@ -195,43 +194,74 @@ def discover_models( return models +@dataclass(frozen=True, slots=True, kw_only=True) +class TagSelector: + """Three tag tuples consumed by `filter_models`. + + See `filter_models` for predicate semantics, including how + empty tuples are interpreted. + + Attributes + ---------- + include_any + Scope (OR) — tags that bring models into the result. + require_all + Narrow (AND) — tags every kept model must have. + exclude_any + Subtract (OR-NOT) — tags that drop a model from the result. + """ + + include_any: tuple[str, ...] = () + require_all: tuple[str, ...] = () + exclude_any: tuple[str, ...] = () + + def filter_models( models: ModelDict, + selector: TagSelector = TagSelector(), *, - tags: tuple[str, ...] = (), - excluded_tags: tuple[str, ...] = (), type_names: tuple[str, ...] = (), ) -> ModelDict: - """Filter models by required tags, excluded tags, and/or type names. + """Filter models by tag predicates and optional type-name match. + + Each tuple in `selector` is a predicate over `key.tags`; a model + is kept only if it satisfies every predicate. Empty tuples are + no-ops — empty `include_any` imposes no scope, empty + `require_all` imposes no narrowing, empty `exclude_any` drops + nothing — so an empty selector returns `models` unchanged. Parameters ---------- - models : ModelDict + models Models to filter. - tags : tuple[str, ...], optional - Tags that must all be present on the model. - excluded_tags : tuple[str, ...], optional - Tags that must not be present on the model. - type_names : tuple[str, ...], optional - Model names to include; all others are excluded. + selector + Tag predicates to apply. + type_names + If non-empty, only models whose `key.name` is in the list + are kept. Orthogonal to the tag predicate algebra. Returns ------- ModelDict - Filtered models. + Models satisfying every supplied predicate. """ - filters: list[ModelKeyFilter] = [] - if tags: - filters.append(lambda key: all(tag in key.tags for tag in tags)) - if excluded_tags: - filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags)) - if type_names: - filters.append(lambda key: key.name in type_names) - if filters: - models = { - key: model for key, model in models.items() if all(f(key) for f in filters) - } - return models + + def matches(key: ModelKey) -> bool: + if selector.include_any and not any( + t in key.tags for t in selector.include_any + ): + return False + if selector.require_all and not all( + t in key.tags for t in selector.require_all + ): + return False + if selector.exclude_any and any(t in key.tags for t in selector.exclude_any): + return False + if type_names and key.name not in type_names: + return False + return True + + return {k: m for k, m in models.items() if matches(k)} def get_registered_model(model_name: str) -> type[BaseModel] | None: @@ -248,7 +278,7 @@ def get_registered_model(model_name: str) -> type[BaseModel] | None: Returns ------- type[BaseModel] or None - Model class if found, otherwise ``None``. + Model class if found, otherwise `None`. """ models = discover_models() for key, model_class in models.items(): diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/models.py b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py index 9c331b466..4f2fbd0cc 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/models.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py @@ -10,7 +10,7 @@ class ModelKey: name : str Friendly name derived from the entry point key. entry_point : str - Entry point value in ``"module:Class"`` format. + Entry point value in `"module:Class"` format. tags : frozenset[str] Tags associated with the model. """ @@ -29,7 +29,7 @@ class TagProviderKey: name : str Friendly name derived from the entry point key. entry_point : str - Entry point value in ``"module:function"`` format. + Entry point value in `"module:function"` format. package_name : str Package that provides this tag provider. """ diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py index 9176b1998..0b440af2a 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py @@ -1,14 +1,14 @@ """Tag format specification and utilities for Overture schema discovery. -Tags follow the pattern ``[namespace:]predicate[=value]`` and come in three forms: +Tags follow the pattern `[namespace:]predicate[=value]` and come in three forms: -- **Plain** — ``overture``, ``feature`` -- **Namespaced** — ``system:extension` -- **Key/value** — ``overture:theme=buildings`` +- **Plain** — `overture`, `feature` +- **Namespaced** — `system:extension` +- **Key/value** — `overture:theme=buildings` -``:`` signals ownership and reservation — only the owning package may set tags in a -given namespace. ``=`` signals a dimension with a discrete value. -One level of each: no nested colons, no multiple ``=`` signs. +`:` signals ownership and reservation — only the owning package may set tags in a +given namespace. `=` signals a dimension with a discrete value. +One level of each: no nested colons, no multiple `=` signs. Tag matching is case-sensitive throughout. """ @@ -33,7 +33,7 @@ def get_namespace(tag: str) -> str: Returns ------- str - The namespace prefix if the tag is a namespaced tag, otherwise ``""``. + The namespace prefix if the tag is a namespaced tag, otherwise `""`. Examples -------- @@ -51,12 +51,12 @@ def get_values_for_key(tags: frozenset[str] | set[str], key: str) -> set[str]: tags : frozenset[str] or set[str] Tags to search. key : str - Key to match, e.g. ``"overture:theme"``. + Key to match, e.g. `"overture:theme"`. Returns ------- set[str] - Values of tags matching ``key=``. + Values of tags matching `key=`. Examples -------- @@ -72,11 +72,11 @@ def is_valid_tag(tag: str) -> bool: A valid tag is a plain tag, a namespaced tag, or a key/value tag: - - **Plain**: ``[a-z0-9][a-z0-9_-]*`` — lowercase alphanumeric, hyphens, + - **Plain**: `[a-z0-9][a-z0-9_-]*` — lowercase alphanumeric, hyphens, underscores; no dots. - - **Namespace / predicate**: ``[a-z0-9][a-z0-9_.-]*`` — same but dots + - **Namespace / predicate**: `[a-z0-9][a-z0-9_.-]*` — same but dots are also allowed. - - **Key/value**: ``{namespace}:{predicate}=[a-zA-Z0-9_.-]+`` — namespace and predicate as + - **Key/value**: `{namespace}:{predicate}=[a-zA-Z0-9_.-]+` — namespace and predicate as above; value is alphanumeric (upper and lower case), hyphens, underscores, or dots; must be non-empty. @@ -88,7 +88,7 @@ def is_valid_tag(tag: str) -> bool: Returns ------- bool - ``True`` if `tag` matches the required format. + `True` if `tag` matches the required format. Examples -------- diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py index e336a3e2f..0c036aa4f 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py @@ -10,7 +10,7 @@ def feature_provider( model_class: type[BaseModel], key: ModelKey, tags: set[str] ) -> set[str]: - """Add the ``"feature"`` tag if the model is a subclass of Feature. + """Add the `"feature"` tag if the model is a subclass of Feature. Parameters ---------- @@ -24,7 +24,7 @@ def feature_provider( Returns ------- set[str] - Updated tags, with ``"feature"`` added if applicable. + Updated tags, with `"feature"` added if applicable. """ if any(issubclass(tp, Feature) for tp in collect_types(model_class)): tags.add("feature") diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py index 7ea168353..6a52298e0 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -10,4 +10,3 @@ TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] -ModelKeyFilter: TypeAlias = Callable[[ModelKey], bool] diff --git a/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py b/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py index a057a3127..c8003f8a5 100644 --- a/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py +++ b/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py @@ -211,10 +211,10 @@ def __init__(self) -> None: class StrippedConstraint(PatternConstraint): r"""Allows only strings that have no leading/trailing whitespace. - Uses ``\Z`` (absolute end-of-string) instead of ``$`` because - Python's ``$`` matches before a trailing ``\n``. ECMA regex (used by - JSON Schema) treats ``$`` as absolute end-of-string, so the JSON - schema output swaps ``\Z`` back to ``$``. + Uses `\Z` (absolute end-of-string) instead of `$` because + Python's `$` matches before a trailing `\n`. ECMA regex (used by + JSON Schema) treats `$` as absolute end-of-string, so the JSON + schema output swaps `\Z` back to `$`. """ def __init__(self) -> None: diff --git a/packages/overture-schema-system/src/overture/schema/system/typing_util.py b/packages/overture-schema-system/src/overture/schema/system/typing_util.py index 1d312bcf3..4ab7e8718 100644 --- a/packages/overture-schema-system/src/overture/schema/system/typing_util.py +++ b/packages/overture-schema-system/src/overture/schema/system/typing_util.py @@ -7,8 +7,8 @@ def collect_types(tp: Any) -> set[type]: # noqa: ANN401 """Collect all concrete types from a type annotation. - Recursively unwraps ``Annotated``, ``NewType``, ``Union``/``X | Y``, and - ``Literal`` to collect the concrete types they contain. Only actual `type` + Recursively unwraps `Annotated`, `NewType`, `Union`/`X | Y`, and + `Literal` to collect the concrete types they contain. Only actual `type` instances are returned. Parameters @@ -19,7 +19,7 @@ def collect_types(tp: Any) -> set[type]: # noqa: ANN401 Returns ------- set[type] - All concrete types found within ``tp``. + All concrete types found within `tp`. """ result: set[type] = set() diff --git a/packages/overture-schema-system/tests/test_discovery_filter_models.py b/packages/overture-schema-system/tests/test_discovery_filter_models.py new file mode 100644 index 000000000..c1c051393 --- /dev/null +++ b/packages/overture-schema-system/tests/test_discovery_filter_models.py @@ -0,0 +1,244 @@ +"""Direct coverage of filter_models combinator algebra.""" + +from pydantic import BaseModel + +from overture.schema.system.discovery import ( + ModelDict, + ModelKey, + TagSelector, + filter_models, +) + + +# Mock model classes +class Building(BaseModel): + pass + + +class Segment(BaseModel): + pass + + +class Connector(BaseModel): + pass + + +class Place(BaseModel): + pass + + +class Sources(BaseModel): + pass + + +BUILDING_KEY = ModelKey( + name="building", + entry_point="mock:Building", + tags=frozenset({"feature", "overture", "overture:theme=buildings"}), +) +SEGMENT_KEY = ModelKey( + name="segment", + entry_point="mock:Segment", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +CONNECTOR_KEY = ModelKey( + name="connector", + entry_point="mock:Connector", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +PLACE_KEY = ModelKey( + name="place", + entry_point="mock:Place", + tags=frozenset({"feature", "overture", "overture:theme=places", "draft"}), +) +SOURCES_KEY = ModelKey( + name="sources", + entry_point="mock:Sources", + tags=frozenset({"overture"}), +) + +ALL_MODELS: ModelDict = { + BUILDING_KEY: Building, + SEGMENT_KEY: Segment, + CONNECTOR_KEY: Connector, + PLACE_KEY: Place, + SOURCES_KEY: Sources, +} + + +def names(models: ModelDict) -> set[str]: + """Return the set of model names in a ModelDict.""" + return {key.name for key in models} + + +class TestEmptySelector: + def test_empty_selector_returns_all(self) -> None: + result = filter_models(ALL_MODELS) + assert names(result) == names(ALL_MODELS) + + def test_empty_selector_explicit(self) -> None: + result = filter_models(ALL_MODELS, TagSelector()) + assert names(result) == names(ALL_MODELS) + + +class TestIncludeAny: + def test_single_tag(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(include_any=("overture:theme=buildings",)) + ) + assert names(result) == {"building"} + + def test_multi_tag_or(self) -> None: + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=buildings", + "overture:theme=transportation", + ) + ), + ) + assert names(result) == {"building", "segment", "connector"} + + def test_no_match(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(include_any=("overture:theme=nonexistent",)) + ) + assert result == {} + + def test_mixed_match(self) -> None: + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=("overture:theme=buildings", "overture:theme=nonexistent") + ), + ) + assert names(result) == {"building"} + + +class TestRequireAll: + def test_single_tag(self) -> None: + result = filter_models(ALL_MODELS, TagSelector(require_all=("feature",))) + assert names(result) == {"building", "segment", "connector", "place"} + + def test_multi_tag_and_match(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(require_all=("feature", "overture")) + ) + assert names(result) == {"building", "segment", "connector", "place"} + + def test_multi_tag_and_one_fails(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(require_all=("feature", "draft")) + ) + assert names(result) == {"place"} + + def test_no_match(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(require_all=("feature", "nonexistent")) + ) + assert result == {} + + +class TestExcludeAny: + def test_single_tag(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(exclude_any=("overture:theme=buildings",)) + ) + assert "building" not in names(result) + assert names(result) == {"segment", "connector", "place", "sources"} + + def test_multi_tag_or(self) -> None: + result = filter_models( + ALL_MODELS, + TagSelector( + exclude_any=( + "overture:theme=buildings", + "overture:theme=transportation", + ) + ), + ) + assert names(result) == {"place", "sources"} + + def test_no_match_keeps_all(self) -> None: + result = filter_models(ALL_MODELS, TagSelector(exclude_any=("nonexistent",))) + assert names(result) == names(ALL_MODELS) + + +class TestTypeNames: + def test_single(self) -> None: + result = filter_models(ALL_MODELS, type_names=("building",)) + assert names(result) == {"building"} + + def test_multiple(self) -> None: + result = filter_models(ALL_MODELS, type_names=("building", "place")) + assert names(result) == {"building", "place"} + + def test_none_match(self) -> None: + result = filter_models(ALL_MODELS, type_names=("nonexistent",)) + assert result == {} + + +class TestCrossCombinator: + def test_include_then_require(self) -> None: + # Scope to features (places, transportation), narrow to those + # also tagged "draft" → only place qualifies. + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=places", + "overture:theme=transportation", + ), + require_all=("draft",), + ), + ) + assert names(result) == {"place"} + + def test_include_then_exclude(self) -> None: + # Scope to all themed features, exclude buildings. + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=buildings", + "overture:theme=transportation", + "overture:theme=places", + ), + exclude_any=("overture:theme=buildings",), + ), + ) + assert names(result) == {"segment", "connector", "place"} + + def test_all_three_combinators_plus_type_names(self) -> None: + # Scope to features in either places or transportation, + # require feature tag, exclude drafts, restrict to segment by name. + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=places", + "overture:theme=transportation", + ), + require_all=("feature",), + exclude_any=("draft",), + ), + type_names=("segment",), + ) + assert names(result) == {"segment"} + + +class TestIdempotence: + def test_double_application(self) -> None: + selector = TagSelector( + include_any=("overture:theme=buildings", "overture:theme=places") + ) + once = filter_models(ALL_MODELS, selector) + twice = filter_models(once, selector) + assert names(once) == names(twice) + + +class TestInputInvariance: + def test_returns_new_dict(self) -> None: + result = filter_models(ALL_MODELS) + assert result is not ALL_MODELS diff --git a/packages/overture-schema-system/tests/test_discovery_tag_selector.py b/packages/overture-schema-system/tests/test_discovery_tag_selector.py new file mode 100644 index 000000000..3b0241d83 --- /dev/null +++ b/packages/overture-schema-system/tests/test_discovery_tag_selector.py @@ -0,0 +1,44 @@ +"""Tests for TagSelector dataclass.""" + +from dataclasses import FrozenInstanceError + +import pytest + +from overture.schema.system.discovery import TagSelector + + +class TestTagSelector: + def test_default_is_empty(self) -> None: + s = TagSelector() + assert s.include_any == () + assert s.require_all == () + assert s.exclude_any == () + + def test_construction_with_fields(self) -> None: + s = TagSelector( + include_any=("a", "b"), + require_all=("c",), + exclude_any=("d",), + ) + assert s.include_any == ("a", "b") + assert s.require_all == ("c",) + assert s.exclude_any == ("d",) + + def test_kw_only(self) -> None: + with pytest.raises(TypeError): + TagSelector(("a",)) # type: ignore[misc] + + def test_frozen(self) -> None: + s = TagSelector(include_any=("a",)) + with pytest.raises(FrozenInstanceError): + s.include_any = ("b",) # type: ignore[misc] + + def test_equality_by_value(self) -> None: + a = TagSelector(include_any=("x",)) + b = TagSelector(include_any=("x",)) + assert a == b + + def test_hashable(self) -> None: + s = TagSelector(include_any=("x",)) + d = {s: 1} + assert d[s] == 1 diff --git a/uv.lock b/uv.lock index 7a10a557a..ca70a5b86 100644 --- a/uv.lock +++ b/uv.lock @@ -833,6 +833,7 @@ source = { editable = "packages/overture-schema-codegen" } dependencies = [ { name = "click" }, { name = "jinja2" }, + { name = "overture-schema-cli" }, { name = "overture-schema-core" }, { name = "overture-schema-system" }, { name = "tomli", marker = "python_full_version < '3.11'" }, @@ -842,6 +843,7 @@ dependencies = [ requires-dist = [ { name = "click", specifier = ">=8.0" }, { name = "jinja2", specifier = ">=3.0" }, + { name = "overture-schema-cli", editable = "packages/overture-schema-cli" }, { name = "overture-schema-core", editable = "packages/overture-schema-core" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, From 12b1e66299f1209a3c8d1f4f593aff6ff3561786 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 29 Apr 2026 21:03:28 -0700 Subject: [PATCH 12/23] fix(system): harden tag provider error logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use provider_key.name (always a string) instead of provider.__name__, which raises AttributeError when a provider is a callable instance without __name__ — masking the original error inside the except block. Add exc_info=True to preserve the traceback in the warning. Signed-off-by: Seth Fitzsimmons --- .../src/overture/schema/system/discovery/discovery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index aebb7a8ae..ca08aea62 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -64,7 +64,8 @@ def generate_tags( tags.update(filtered_tags) except Exception as e: log.warning( - f"Error in tag provider {provider.__name__} for model {key.name}: {e}" + f"Error in tag provider {provider_key.name} for model {key.name}: {e}", + exc_info=True, ) return tags From 6a58214d2eee08bcb6fc2fe3f99d5e7f2c08a3be Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 29 Apr 2026 21:04:07 -0700 Subject: [PATCH 13/23] test(system): convert tag tests to pytest style Replace unittest.TestCase classes with module-level pytest functions parametrized over the tag lists. Per-tag parametrization isolates failures to the offending input instead of stopping at the first assertion in a loop. Signed-off-by: Seth Fitzsimmons --- .../overture-schema-system/tests/test_tags.py | 163 ++++++++---------- 1 file changed, 76 insertions(+), 87 deletions(-) diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index f203200c7..accc55dec 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -1,5 +1,6 @@ import re -import unittest + +import pytest from overture.schema.system.discovery.tag import ( NAMESPACE_TAG, @@ -31,89 +32,77 @@ def test_get_values_for_key_handles_empty_tags() -> None: assert result == set() -class TestPlainTagRegex(unittest.TestCase): - def test_valid_plain_tags(self) -> None: - valid_tags = [ - "v", - "valid", - "valid1", - "valid_tag", - "valid-tag", - "0valid", - "42", - ] - for tag in valid_tags: - self.assertTrue( - re.fullmatch(PLAIN_TAG, tag), f"PLAIN_TAG should match: {tag}" - ) - self.assertTrue(TAG.fullmatch(tag), f"TAG should match: {tag}") - self.assertTrue( - is_valid_tag(tag), f"is_valid_tag should return True for: {tag}" - ) - - def test_invalid_plain_tags(self) -> None: - invalid_tags = [ - "", - "_invalid", - "-invalid", - "Invalid", - "invalid!", - "invalid ", - "in.valid", - "3.14", - ] - for tag in invalid_tags: - self.assertFalse( - re.fullmatch(PLAIN_TAG, tag), f"PLAIN_TAG should not match: {tag}" - ) - self.assertFalse(TAG.fullmatch(tag), f"TAG should not match: {tag}") - self.assertFalse( - is_valid_tag(tag), f"is_valid_tag should return False for: {tag}" - ) - - -class TestNamespaceTagRegex(unittest.TestCase): - def test_valid_namespace_tags(self) -> None: - valid_tags = [ - "ns:predicate", - "ns:predicate1", - "ns:predicate-1", - "ns:predicate=value", - "ns:predicate=value_0", - "ns:predicate=value-0", - "ns:predicate=value.0", - "ns:predicate=value_2-3.4", - "ns:predicate=42", - "ns:predicate=3.14", - "ns:predicate=Value", - ] - for tag in valid_tags: - self.assertTrue( - re.fullmatch(NAMESPACE_TAG, tag), f"NAMESPACE_TAG should match: {tag}" - ) - self.assertTrue(TAG.fullmatch(tag), f"TAG should match: {tag}") - self.assertTrue( - is_valid_tag(tag), f"is_valid_tag should return True for: {tag}" - ) - - def test_invalid_namespace_tags(self) -> None: - invalid_tags = [ - "ns:", - ":predicate", - "ns:predicate=", - "ns:predicate=value ", - "ns:predicate=value!", - "ns:predicate=ns:value", - "ns:predicate=predicate=value", - "Ns:predicate", - "ns:Predicate", - ] - for tag in invalid_tags: - self.assertFalse( - re.fullmatch(NAMESPACE_TAG, tag), - f"NAMESPACE_TAG should not match: {tag}", - ) - self.assertFalse(TAG.fullmatch(tag), f"TAG should not match: {tag}") - self.assertFalse( - is_valid_tag(tag), f"is_valid_tag should return False for: {tag}" - ) +VALID_PLAIN_TAGS = [ + "v", + "valid", + "valid1", + "valid_tag", + "valid-tag", + "0valid", + "42", +] + +INVALID_PLAIN_TAGS = [ + "", + "_invalid", + "-invalid", + "Invalid", + "invalid!", + "invalid ", + "in.valid", + "3.14", +] + +VALID_NAMESPACE_TAGS = [ + "ns:predicate", + "ns:predicate1", + "ns:predicate-1", + "ns:predicate=value", + "ns:predicate=value_0", + "ns:predicate=value-0", + "ns:predicate=value.0", + "ns:predicate=value_2-3.4", + "ns:predicate=42", + "ns:predicate=3.14", + "ns:predicate=Value", +] + +INVALID_NAMESPACE_TAGS = [ + "ns:", + ":predicate", + "ns:predicate=", + "ns:predicate=value ", + "ns:predicate=value!", + "ns:predicate=ns:value", + "ns:predicate=predicate=value", + "Ns:predicate", + "ns:Predicate", +] + + +@pytest.mark.parametrize("tag", VALID_PLAIN_TAGS) +def test_valid_plain_tag(tag: str) -> None: + assert re.fullmatch(PLAIN_TAG, tag) + assert TAG.fullmatch(tag) + assert is_valid_tag(tag) + + +@pytest.mark.parametrize("tag", INVALID_PLAIN_TAGS) +def test_invalid_plain_tag(tag: str) -> None: + assert not re.fullmatch(PLAIN_TAG, tag) + assert not TAG.fullmatch(tag) + assert not is_valid_tag(tag) + + +@pytest.mark.parametrize("tag", VALID_NAMESPACE_TAGS) +def test_valid_namespace_tag(tag: str) -> None: + assert re.fullmatch(NAMESPACE_TAG, tag) + assert TAG.fullmatch(tag) + assert is_valid_tag(tag) + + +@pytest.mark.parametrize("tag", INVALID_NAMESPACE_TAGS) +def test_invalid_namespace_tag(tag: str) -> None: + assert not re.fullmatch(NAMESPACE_TAG, tag) + assert not TAG.fullmatch(tag) + assert not is_valid_tag(tag) From ef8c453001b687191336407201e84f4eee3a3d86 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 29 Apr 2026 21:09:20 -0700 Subject: [PATCH 14/23] docs(system): add module docstring to discovery models Fixes D100 reported by pydocstyle / make docformat. Signed-off-by: Seth Fitzsimmons --- .../src/overture/schema/system/discovery/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/models.py b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py index 4f2fbd0cc..26eebaf1a 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/models.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/models.py @@ -1,3 +1,5 @@ +"""Key types identifying registered models and tag providers.""" + from dataclasses import dataclass From b1f6c27a2a3167eb32dcc5c35598abf28724af60 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 29 Apr 2026 21:32:57 -0700 Subject: [PATCH 15/23] refactor(system): unify tag part pattern and tighten api Plain tags, namespaces, and predicates now share a single TAG_PART pattern: lowercase alphanumeric start followed by alphanumeric, hyphen, underscore, or dot. Values remain case-permissive. Drops the prior asymmetry where namespaces and predicates allowed dots but plain tags did not. Make generate_tags private (its sole caller is discover_models) and broaden TagProvider's return type to Iterable[str] so providers can yield, return lists, or return sets. Signed-off-by: Seth Fitzsimmons --- .../schema/system/discovery/discovery.py | 12 +++++------ .../overture/schema/system/discovery/tag.py | 20 +++++++++---------- .../overture/schema/system/discovery/types.py | 4 ++-- .../overture-schema-system/tests/test_tags.py | 14 ++++++++----- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index ca08aea62..0f95c57a3 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -31,7 +31,7 @@ } -def generate_tags( +def _generate_tags( model_class: type[BaseModel], key: ModelKey, providers: TagProviderDict, @@ -44,11 +44,11 @@ def generate_tags( Parameters ---------- - model_class : type[BaseModel] + model_class Model class to generate tags for. - key : ModelKey + key Key identifying the model. - providers : TagProviderDict + providers Tag providers to invoke. Returns @@ -59,7 +59,7 @@ def generate_tags( tags: set[str] = set() for provider_key, provider in providers.items(): try: - added_tags = provider(model_class, key, tags.copy()).difference(tags) + added_tags = set(provider(model_class, key, tags.copy())) - tags filtered_tags = _filter_tags(added_tags, provider_key) tags.update(filtered_tags) except Exception as e: @@ -183,7 +183,7 @@ def discover_models( try: key = replace( key, - tags=frozenset(generate_tags(model_class, key, tag_providers)), + tags=frozenset(_generate_tags(model_class, key, tag_providers)), ) except Exception as e: log.warning(f"Could not resolve tags for model {model.name}: {e}") diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py index 0b440af2a..5633ceeee 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py @@ -15,11 +15,10 @@ import re -PLAIN_TAG = r"[a-z0-9][a-z0-9_-]*" -NAMESPACE = PREDICATE = r"[a-z0-9][a-z0-9_.-]*" +TAG_PART = r"[a-z0-9][a-z0-9_.-]*" VALUE = r"[a-zA-Z0-9_.-]+" -NAMESPACE_TAG = rf"{NAMESPACE}:{PREDICATE}(?:={VALUE})?" -TAG = re.compile(rf"^(?:{PLAIN_TAG}|{NAMESPACE_TAG})$") +NAMESPACE_TAG = rf"{TAG_PART}:{TAG_PART}(?:={VALUE})?" +TAG = re.compile(rf"^(?:{TAG_PART}|{NAMESPACE_TAG})$") def get_namespace(tag: str) -> str: @@ -72,13 +71,12 @@ def is_valid_tag(tag: str) -> bool: A valid tag is a plain tag, a namespaced tag, or a key/value tag: - - **Plain**: `[a-z0-9][a-z0-9_-]*` — lowercase alphanumeric, hyphens, - underscores; no dots. - - **Namespace / predicate**: `[a-z0-9][a-z0-9_.-]*` — same but dots - are also allowed. - - **Key/value**: `{namespace}:{predicate}=[a-zA-Z0-9_.-]+` — namespace and predicate as - above; value is alphanumeric (upper and lower case), hyphens, underscores, or dots; - must be non-empty. + - **Plain / namespace / predicate**: `[a-z0-9][a-z0-9_.-]*` — + lowercase alphanumeric start, then alphanumeric, hyphens, + underscores, or dots. + - **Key/value**: `{namespace}:{predicate}=[a-zA-Z0-9_.-]+` — namespace and + predicate as above; value is alphanumeric (upper and lower case), + hyphens, underscores, or dots; must be non-empty. Parameters ---------- diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py index 6a52298e0..9e25b4839 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -1,12 +1,12 @@ """Types and data classes for Overture schema discovery system.""" -from collections.abc import Callable +from collections.abc import Callable, Iterable from typing import TypeAlias from pydantic import BaseModel from .models import ModelKey, TagProviderKey -TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], set[str]] +TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], Iterable[str]] ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py index accc55dec..111a532e4 100644 --- a/packages/overture-schema-system/tests/test_tags.py +++ b/packages/overture-schema-system/tests/test_tags.py @@ -4,8 +4,8 @@ from overture.schema.system.discovery.tag import ( NAMESPACE_TAG, - PLAIN_TAG, TAG, + TAG_PART, get_values_for_key, is_valid_tag, ) @@ -40,23 +40,27 @@ def test_get_values_for_key_handles_empty_tags() -> None: "valid-tag", "0valid", "42", + "in.valid", + "3.14", + "com.example", ] INVALID_PLAIN_TAGS = [ "", "_invalid", "-invalid", + ".invalid", "Invalid", "invalid!", "invalid ", - "in.valid", - "3.14", ] VALID_NAMESPACE_TAGS = [ "ns:predicate", "ns:predicate1", "ns:predicate-1", + "ns.dotted:predicate", + "ns:pred.icate", "ns:predicate=value", "ns:predicate=value_0", "ns:predicate=value-0", @@ -82,14 +86,14 @@ def test_get_values_for_key_handles_empty_tags() -> None: @pytest.mark.parametrize("tag", VALID_PLAIN_TAGS) def test_valid_plain_tag(tag: str) -> None: - assert re.fullmatch(PLAIN_TAG, tag) + assert re.fullmatch(TAG_PART, tag) assert TAG.fullmatch(tag) assert is_valid_tag(tag) @pytest.mark.parametrize("tag", INVALID_PLAIN_TAGS) def test_invalid_plain_tag(tag: str) -> None: - assert not re.fullmatch(PLAIN_TAG, tag) + assert not re.fullmatch(TAG_PART, tag) assert not TAG.fullmatch(tag) assert not is_valid_tag(tag) From a92e26875dc9fc8fd9947a40ffd855d2ac3465b1 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Thu, 30 Apr 2026 06:58:20 -0700 Subject: [PATCH 16/23] refactor(system,core): tighten tag provider contract and tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provider's first argument is the value loaded from an `overture.models` entry point. For discriminated-union features (e.g. `Segment`) that's `Annotated[Union[...], Field(...)]`, not `type[BaseModel]` — the prior signature was a lie. Widen `TagProvider` and the in-tree providers to accept `Any` and document the boundary in `discovery/types.py`. Strip `typing_util.collect_types` to the cases discovery actually meets today: `Annotated`, `Union`/`X | Y`, plain class. Drop the unreached `NewType` and `Literal` branches. Point at `overture-schema-codegen`'s `extraction/type_analyzer.py:analyze_type` as the more capable implementation, with consolidation across system, core, and cli flagged as future work. `theme_provider` extracts the theme via `_theme_literal`, which asserts that `theme` is a single-value `str` `Literal[...]` and raises `TypeError` otherwise. `_generate_tags` catches and logs at WARNING, so third-party model-definition bugs surface visibly without crashing discovery. Promote tag-rejection logging from DEBUG to WARNING so authorization failures (invalid tags, reserved tags, reserved namespaces) don't disappear silently in normal operation. Convert filter tests from direct `_filter_tags` calls to a fake `TagProvider` driven through `_generate_tags`. Tests now exercise provider invocation and merge wiring, not just the filter, and decouple from the private filter name. Provider-behavior tests still call the providers directly. Add discriminated-union coverage for both `feature_provider` and `theme_provider`, plus a `TypeError` case for a non-Literal `theme`. Signed-off-by: Seth Fitzsimmons --- .../src/overture/schema/core/tag_providers.py | 76 ++++++--- .../tests/test_core_tag_providers.py | 89 ++++++++++ .../schema/system/discovery/discovery.py | 6 +- .../schema/system/discovery/tag_providers.py | 22 ++- .../overture/schema/system/discovery/types.py | 10 +- .../src/overture/schema/system/typing_util.py | 31 ++-- .../tests/test_tag_providers.py | 158 ++++++++++++++---- 7 files changed, 309 insertions(+), 83 deletions(-) create mode 100644 packages/overture-schema-core/tests/test_core_tag_providers.py diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 8c9dc10b9..004aa7848 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -5,9 +5,7 @@ `overture.tag_providers` entry-point group. """ -from typing import get_args - -from pydantic import BaseModel +from typing import Any, Literal, get_args, get_origin from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ModelKey @@ -33,18 +31,17 @@ } -def authority_provider( - model_class: type[BaseModel], key: ModelKey, tags: set[str] -) -> set[str]: +def authority_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: # noqa: ANN401 """Add the `"overture"` tag if the model originates from an approved Overture package. Parameters ---------- - model_class : type[BaseModel] - Model class to inspect. - key : ModelKey + model_class + A class or discriminated-union type expression loaded from an + `overture.models` entry point. + key Key identifying the model. - tags : set[str] + tags Current tags; may be extended. Returns @@ -57,32 +54,69 @@ def authority_provider( return tags -def theme_provider( - model_class: type[BaseModel], key: ModelKey, tags: set[str] -) -> set[str]: - """Add the `"overture:theme={theme}"` tag if the model is a subclass of OvertureFeature. +def theme_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: # noqa: ANN401 + """Add `"overture:theme={theme}"` for each `OvertureFeature` referenced. + + Tags are attached to the entry point's `ModelKey`. For + discriminated-union features, the provider walks every concrete arm + via `collect_types` and reads each arm's `theme`; tags from all arms + accumulate on the union's `ModelKey`. Arms that share a theme + deduplicate to a single tag; arms with different themes contribute + multiple `overture:theme=X` tags to the same `ModelKey`. + + Each arm's `theme` field must be annotated as a single-value + `Literal[str]`; any other annotation is a model-definition bug and + raises `TypeError`. Parameters ---------- - model_class : type[BaseModel] - Model class to inspect. - key : ModelKey + model_class + A class or discriminated-union type expression loaded from an + `overture.models` entry point. + key Key identifying the model. - tags : set[str] + tags Current tags; may be extended. Returns ------- set[str] Updated tags, with `"overture:theme={theme}"` added if applicable. + + Raises + ------ + TypeError + If a referenced `OvertureFeature`'s `theme` is not a single-value + `Literal[str]`. """ for tp in collect_types(model_class): if issubclass(tp, OvertureFeature): - tags.add( - "overture:theme=" + get_args(tp.model_fields["theme"].annotation)[0] - ) + tags.add(f"overture:theme={_theme_literal(tp)}") return tags def _matches_manifest(key: ModelKey) -> bool: return key.entry_point in APPROVED + + +def _theme_literal(model_class: type[OvertureFeature]) -> str: + """Extract the literal `theme` value from an `OvertureFeature` subclass. + + Raises + ------ + TypeError + If `theme` is not annotated as a single-value `Literal`. + """ + annotation = model_class.model_fields["theme"].annotation + if get_origin(annotation) is not Literal: + raise TypeError( + f"{model_class.__name__}.theme must be annotated Literal[...]; " + f"got {annotation!r}" + ) + args = get_args(annotation) + if len(args) != 1 or not isinstance(args[0], str): + raise TypeError( + f"{model_class.__name__}.theme must be a single-value str Literal; " + f"got {annotation!r}" + ) + return args[0] diff --git a/packages/overture-schema-core/tests/test_core_tag_providers.py b/packages/overture-schema-core/tests/test_core_tag_providers.py new file mode 100644 index 000000000..3131501e3 --- /dev/null +++ b/packages/overture-schema-core/tests/test_core_tag_providers.py @@ -0,0 +1,89 @@ +"""Tests for core tag providers.""" + +from typing import Annotated, Literal + +import pytest +from overture.schema.core import OvertureFeature +from overture.schema.core.tag_providers import ( + APPROVED, + authority_provider, + theme_provider, +) +from overture.schema.system.discovery import ModelKey +from pydantic import BaseModel, Field, Tag + + +@pytest.fixture +def road() -> type[OvertureFeature]: + class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): + pass + + return Road + + +@pytest.fixture +def building() -> type[OvertureFeature]: + class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): + pass + + return Building + + +@pytest.fixture +def not_overture() -> type[BaseModel]: + class NotOverture(BaseModel): + pass + + return NotOverture + + +def _empty_key(name: str = "x", entry_point: str = "mod:X") -> ModelKey: + return ModelKey(name=name, entry_point=entry_point, tags=frozenset()) + + +def test_theme_provider_plain_class(building: type[OvertureFeature]) -> None: + tags = theme_provider(building, _empty_key(), set()) + assert tags == {"overture:theme=buildings"} + + +def test_theme_provider_discriminated_union() -> None: + class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): + pass + + class Rail(OvertureFeature[Literal["transportation"], Literal["rail"]]): + pass + + union = Annotated[ + Annotated[Road, Tag("road")] | Annotated[Rail, Tag("rail")], + Field(discriminator="type"), + ] + tags = theme_provider(union, _empty_key(), set()) + assert tags == {"overture:theme=transportation"} + + +def test_theme_provider_skips_non_overture(not_overture: type[BaseModel]) -> None: + tags = theme_provider(not_overture, _empty_key(), set()) + assert tags == set() + + +def test_theme_provider_raises_on_non_literal_theme() -> None: + class BadFeature(OvertureFeature): # type: ignore[type-arg] + # ThemeT defaults to str (its bound), not Literal — a third-party + # bug we want to surface. + pass + + with pytest.raises(TypeError, match="must be annotated Literal"): + theme_provider(BadFeature, _empty_key(), set()) + + +def test_authority_provider_approved(road: type[OvertureFeature]) -> None: + approved_entry = next(iter(APPROVED)) + key = ModelKey(name="x", entry_point=approved_entry, tags=frozenset()) + tags = authority_provider(road, key, set()) + assert "overture" in tags + + +def test_authority_provider_not_approved(road: type[OvertureFeature]) -> None: + key = ModelKey(name="x", entry_point="some.unapproved:Model", tags=frozenset()) + tags = authority_provider(road, key, set()) + assert "overture" not in tags diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index 0f95c57a3..5ebadef4f 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -97,14 +97,14 @@ def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: } for tag in tags: if not is_valid_tag(tag): - log.debug( + log.warning( f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " f"This tag does not match the required format." ) continue if tag in reserved_tags: allowed_pkgs = _RESERVED_TAGS.get(tag, set()) - log.debug( + log.warning( f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " f"This tag can only be set by packages from: {allowed_pkgs}." ) @@ -112,7 +112,7 @@ def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: tag_ns = get_namespace(tag) if tag_ns and tag_ns in reserved_namespaces: allowed_pkgs = _RESERVED_NAMESPACES.get(tag_ns, set()) - log.debug( + log.warning( f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " f"This namespace can only be set by packages from: {allowed_pkgs}." ) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py index 0c036aa4f..bb5456d09 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py @@ -1,24 +1,28 @@ """Tag provider logic for Overture schema discovery system.""" -from pydantic import BaseModel +from typing import Any from overture.schema.system.discovery.types import ModelKey from overture.schema.system.feature import Feature from overture.schema.system.typing_util import collect_types -def feature_provider( - model_class: type[BaseModel], key: ModelKey, tags: set[str] -) -> set[str]: - """Add the `"feature"` tag if the model is a subclass of Feature. +def feature_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: # noqa: ANN401 + """Add the `"feature"` tag if the entry point references a `Feature` subclass. + + Tags are attached to the entry point's `ModelKey`. For + discriminated-union features, the provider walks every concrete arm + via `collect_types`; if any arm is a `Feature` subclass, the tag is + added to the union's `ModelKey`. Parameters ---------- - model_class : type[BaseModel] - Model class to inspect. - key : ModelKey + model_class + A class or discriminated-union type expression loaded from an + `overture.models` entry point. + key Key identifying the model. - tags : set[str] + tags Current tags; may be extended. Returns diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py index 9e25b4839..bf96768c2 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -1,12 +1,18 @@ """Types and data classes for Overture schema discovery system.""" from collections.abc import Callable, Iterable -from typing import TypeAlias +from typing import Any, TypeAlias from pydantic import BaseModel from .models import ModelKey, TagProviderKey -TagProvider: TypeAlias = Callable[[type[BaseModel], ModelKey, set[str]], Iterable[str]] +# The first argument is the value loaded from an `overture.models` entry +# point. That is usually a `type[BaseModel]`, but discriminated-union +# features (e.g. `Segment`) load as `Annotated[Union[...], Field(...)]` +# expressions, which are not `type` objects. Providers should use +# `overture.schema.system.typing_util.collect_types` to walk to concrete +# classes. +TagProvider: TypeAlias = Callable[[Any, ModelKey, set[str]], Iterable[str]] ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] diff --git a/packages/overture-schema-system/src/overture/schema/system/typing_util.py b/packages/overture-schema-system/src/overture/schema/system/typing_util.py index 4ab7e8718..f532f5056 100644 --- a/packages/overture-schema-system/src/overture/schema/system/typing_util.py +++ b/packages/overture-schema-system/src/overture/schema/system/typing_util.py @@ -1,25 +1,35 @@ """Typing utilities for the Overture schema system.""" import types -from typing import Annotated, Any, Literal, Union, get_args, get_origin +from typing import Annotated, Any, Union, get_args, get_origin def collect_types(tp: Any) -> set[type]: # noqa: ANN401 - """Collect all concrete types from a type annotation. + """Collect concrete classes referenced by a type annotation. - Recursively unwraps `Annotated`, `NewType`, `Union`/`X | Y`, and - `Literal` to collect the concrete types they contain. Only actual `type` - instances are returned. + Unwraps `Annotated[X, ...]` and `Union[X, Y]` (including `X | Y`) to + find concrete `type` objects. Used by tag providers to walk + discriminated-union features (e.g. `Segment`) into their member + classes. + + Only handles the cases the discovery system encounters today. + `overture-schema-codegen` has a more capable + `analyze_type` (`extraction/type_analyzer.py`) that also unwraps + `NewType`, `Literal`, `list[...]`, `dict[K, V]`, and accumulates + constraints. A future work item is to consolidate this and the + similar logic in `overture-schema-cli` against that implementation. Parameters ---------- - tp : Any - A type annotation to inspect. + tp + A type annotation. Typically a class, an `Annotated[...]` + wrapper, or a discriminated union of classes. Returns ------- set[type] - All concrete types found within `tp`. + Concrete classes reachable through `Annotated` and `Union` + unwrapping. Other type expressions yield an empty set. """ result: set[type] = set() @@ -28,14 +38,9 @@ def _visit(t: Any) -> None: origin = get_origin(t) if origin is Annotated: _visit(get_args(t)[0]) - elif hasattr(t, "__supertype__"): - _visit(t.__supertype__) elif origin is Union or origin is types.UnionType: for arg in get_args(t): _visit(arg) - elif origin is Literal: - for val in get_args(t): - result.add(type(val)) elif isinstance(t, type): result.add(t) diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py index 8c5bc06cc..4ab95943f 100644 --- a/packages/overture-schema-system/tests/test_tag_providers.py +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -1,9 +1,15 @@ +from typing import Annotated, Any + import pytest -from pydantic import BaseModel +from pydantic import BaseModel, Field, Tag -from overture.schema.system.discovery.discovery import _filter_tags +from overture.schema.system.discovery.discovery import _generate_tags from overture.schema.system.discovery.tag_providers import feature_provider -from overture.schema.system.discovery.types import ModelKey, TagProviderKey +from overture.schema.system.discovery.types import ( + ModelKey, + TagProvider, + TagProviderKey, +) from overture.schema.system.feature import Feature @@ -46,56 +52,121 @@ class NotAFeature(BaseModel): return NotAFeature -def test_valid_tags(other_tag_provider: TagProviderKey) -> None: - tags = {"valid", "other:valid", "other:valid=true"} - filtered = _filter_tags(tags, other_tag_provider) - assert filtered == tags +@pytest.fixture +def any_key() -> ModelKey: + return ModelKey(name="x", entry_point="m:X", tags=frozenset()) + + +@pytest.fixture +def any_model() -> type[BaseModel]: + class M(BaseModel): + pass + + return M + + +def fake_provider(*tags: str) -> TagProvider: + """Provider that always returns the given tags, ignoring its inputs.""" + + def _provider(model_class: Any, key: ModelKey, current_tags: set[str]) -> set[str]: + return set(tags) + + return _provider + + +def test_valid_tags( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = { + other_tag_provider: fake_provider("valid", "other:valid", "other:valid=true") + } + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid", "other:valid", "other:valid=true"} -def test_invalid_tag(other_tag_provider: TagProviderKey) -> None: - tags = {"InvalidTag"} - filtered = _filter_tags(tags, other_tag_provider) - assert filtered == set() +def test_invalid_tag( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider("InvalidTag")} + result = _generate_tags(any_model, any_key, providers) + assert result == set() -def test_reserved_tag(other_tag_provider: TagProviderKey) -> None: - tags = {"overture", "feature", "valid"} - filtered = _filter_tags(tags, other_tag_provider) - assert "valid" in filtered - assert "overture" not in filtered - assert "feature" not in filtered +def test_reserved_tag( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider("overture", "feature", "valid")} + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid"} def test_allowed_reserved_tag( - core_tag_provider: TagProviderKey, system_tag_provider: TagProviderKey + core_tag_provider: TagProviderKey, + system_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], ) -> None: - assert "overture" in _filter_tags({"overture"}, core_tag_provider) - assert "feature" in _filter_tags({"feature"}, system_tag_provider) + core_providers = {core_tag_provider: fake_provider("overture")} + assert _generate_tags(any_model, any_key, core_providers) == {"overture"} + system_providers = {system_tag_provider: fake_provider("feature")} + assert _generate_tags(any_model, any_key, system_providers) == {"feature"} -def test_reserved_namespace(other_tag_provider: TagProviderKey) -> None: - tags = {"overture:feature", "system:feature", "valid:tag"} - filtered = _filter_tags(tags, other_tag_provider) - assert "valid:tag" in filtered - assert "overture:feature" not in filtered - assert "system:feature" not in filtered + +def test_reserved_namespace( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = { + other_tag_provider: fake_provider( + "overture:feature", "system:feature", "valid:tag" + ) + } + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid:tag"} def test_allowed_reserved_namespace( - core_tag_provider: TagProviderKey, system_tag_provider: TagProviderKey + core_tag_provider: TagProviderKey, + system_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], ) -> None: - assert "overture:feature" in _filter_tags({"overture:feature"}, core_tag_provider) - assert "system:feature" in _filter_tags({"system:feature"}, system_tag_provider) + core_providers = {core_tag_provider: fake_provider("overture:feature")} + assert _generate_tags(any_model, any_key, core_providers) == {"overture:feature"} + system_providers = {system_tag_provider: fake_provider("system:feature")} + assert _generate_tags(any_model, any_key, system_providers) == {"system:feature"} -def test_empty_tags(other_tag_provider: TagProviderKey) -> None: - assert _filter_tags(set(), other_tag_provider) == set() + +def test_empty_tags( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider()} + assert _generate_tags(any_model, any_key, providers) == set() -def test_mixed_tags(other_tag_provider: TagProviderKey) -> None: - tags = {"valid", "feature", "overture:feature", "InvalidTag"} - filtered = _filter_tags(tags, other_tag_provider) - assert filtered == {"valid"} +def test_mixed_tags( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = { + other_tag_provider: fake_provider( + "valid", "feature", "overture:feature", "InvalidTag" + ) + } + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid"} def test_feature_provider_adds_feature_tag(feature: type[Feature]) -> None: @@ -112,3 +183,20 @@ def test_feature_provider_does_not_add_feature_tag( ) result = feature_provider(not_a_feature, key, set()) assert "feature" not in result + + +def test_feature_provider_handles_discriminated_union() -> None: + # Mimics the shape of `Segment`: Annotated[Union[...], Field(discriminator=...)] + class ArmA(Feature): + pass + + class ArmB(BaseModel): + pass + + union = Annotated[ + Annotated[ArmA, Tag("a")] | Annotated[ArmB, Tag("b")], + Field(discriminator="type"), + ] + key = ModelKey(name="union", entry_point="mod:Union", tags=frozenset()) + result = feature_provider(union, key, set()) + assert "feature" in result From 7dbd389a316f1ae89b3a981e1f82b2ca69ab3a82 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Thu, 30 Apr 2026 11:19:37 -0700 Subject: [PATCH 17/23] docs(system,core): document tagging mechanism Add Discovery and Tagging sections to system's README, covering the overture.models / overture.tag_providers entry point groups, the tag format, provider contract, namespace and tag reservation, the built-in providers, and TagSelector-based filtering. Update core's README: replace the stale Discovery bullet (discovery has moved to system) with one describing the authority and theme tag providers core contributes. Signed-off-by: Seth Fitzsimmons --- packages/overture-schema-core/README.md | 2 +- packages/overture-schema-system/README.md | 102 ++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/packages/overture-schema-core/README.md b/packages/overture-schema-core/README.md index 58f5d3ee9..8955623f1 100644 --- a/packages/overture-schema-core/README.md +++ b/packages/overture-schema-core/README.md @@ -94,4 +94,4 @@ Rendering hints for map-making: `prominence` (1--100 significance scale), `min_z - **Types** -- domain-specific aliases built on system primitives: `ConfidenceScore` (0.0--1.0), `Level` (z-order), `FeatureVersion`. - **Units** -- measurement enumerations: `SpeedUnit`, `LengthUnit`, `WeightUnit`. -- **Discovery** -- entry-point-based model registry. Theme packages register models via `overture.models` entry points; `discover_models()` resolves them at runtime. +- **Tag providers** -- `authority` and `theme` providers for the discovery system in `overture-schema-system`. They tag `OvertureFeature`-derived models with `overture` (when manifest-approved) and `overture:theme={theme}`. diff --git a/packages/overture-schema-system/README.md b/packages/overture-schema-system/README.md index 238984eb0..e9a628932 100644 --- a/packages/overture-schema-system/README.md +++ b/packages/overture-schema-system/README.md @@ -109,6 +109,108 @@ class ParkBench(Identified): park_id: Annotated[Id, Reference(Relationship.BELONGS_TO, Park)] ``` +## Discovery + +Packages register models on the `overture.models` Python entry point group. Each entry maps a name to a class import path: + +```toml +[project.entry-points."overture.models"] +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" +``` + +`discover_models()` walks the group, loads each entry point, and returns a dict keyed by `ModelKey`. Consumers iterate over the result without knowing which package owns any given model -- the CLI and codegen tools both run discovery to assemble their working set. + +A `ModelKey` carries the entry point `name`, its `entry_point` value (`"module:Class"`), and a `frozenset[str]` of tags. [Tagging](#tagging) is how those tags get attached. + +## Tagging + +Tags classify discovered models. A package registers [tag providers](#providers) on `overture.tag_providers`; when `discover_models` runs, it asks every provider which tags apply to each model and attaches the resulting set to its `ModelKey`. Downstream tools read those tags -- the CLI's `--tag` filter, codegen's grouping logic, anything that reasons about a model without importing it. + +```python +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) + +models = discover_models() + +selected = filter_models( + models, + TagSelector(include_any=("feature",), exclude_any=("draft",)), +) +``` + +### Format + +Tags follow `[namespace:]predicate[=value]`: + +- **Plain** -- `feature`, `overture` +- **Namespaced** -- `system:extension` +- **Key/value** -- `overture:theme=buildings` + +`:` separates namespace from predicate -- one level only, no nested colons. `=` introduces a discrete value, one per tag. Predicate and namespace parts are lowercase alphanumeric (with `_`, `.`, `-`); values also accept uppercase. Matching is case-sensitive throughout. + +Helpers in `overture.schema.system.discovery.tag` parse structured tags: + +- `is_valid_tag(tag)` -- check whether a string matches the format +- `get_namespace(tag)` -- extract the namespace prefix, or `""` for a plain tag +- `get_values_for_key(tags, "overture:theme")` -- extract values from k/v tags with the given key + +### Providers + +A tag provider is a callable registered on the `overture.tag_providers` entry point group. Discovery passes it the model class and a copy of the tags accumulated so far; tags it adds are merged into the running set after passing the reservation checks below. + +```python +from overture.schema.system.discovery import ModelKey +from overture.schema.system.feature import Feature +from overture.schema.system.typing_util import collect_types + +def feature_provider(model_class, key: ModelKey, tags: set[str]) -> set[str]: + if any(issubclass(tp, Feature) for tp in collect_types(model_class)): + tags.add("feature") + return tags +``` + +```toml +[project.entry-points."overture.tag_providers"] +feature = "overture.schema.system.discovery.tag_providers:feature_provider" +``` + +Tags from one provider are visible to providers that run later, but execution order is unspecified -- a provider must not depend on tags added by another. Provider exceptions are caught, logged, and discarded; they do not abort discovery. + +The `model_class` argument is whatever the entry point loads. For most features that is a `type[BaseModel]`, but discriminated-union features (e.g. `Segment`) load as `Annotated[Union[...], Field(...)]` expressions, which are not `type` objects. Providers walk these with `collect_types` instead of calling `issubclass` directly. + +### Reservation + +Specific plain tags and namespaces are reserved for designated packages. For example: + +| Tag or namespace | Owning package | +|---|---| +| `feature` (tag) | `overture-schema-system` | +| `overture` (tag) | `overture-schema-core` | +| `system:` (namespace) | `overture-schema-system` | +| `overture:` (namespace) | `overture-schema-core` | + +When a provider attempts to set a reserved tag from an unauthorized package, discovery logs a warning and discards the tag. + +### Built-in Providers + +- **`feature`** (in `system`) -- adds `feature` if the entry point references a `Feature` subclass, walking discriminated unions via `collect_types`. +- **`authority`** (in `core`) -- adds `overture` if the model's entry point appears in the curated approval manifest (`APPROVED` in `overture.schema.core.tag_providers`). +- **`theme`** (in `core`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme. + +### Selecting Models by Tag + +`filter_models(models, selector)` applies `TagSelector` predicates against each `ModelKey.tags`: + +- `include_any` -- OR scope; at least one tag must match (empty: no scope filter) +- `require_all` -- AND narrowing; every tag must be present (empty: no narrowing) +- `exclude_any` -- OR-NOT subtraction; any match drops the model + +An empty selector returns the input unchanged. + ## Also Included - **Optionality** -- `Omitable[T]` models JSON Schema's "may be absent but not null" semantics, which Pydantic's `T | None` conflates with nullable. From 0824f57aeaf11e727ae76857f98672b84c954403 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Thu, 30 Apr 2026 11:47:23 -0700 Subject: [PATCH 18/23] refactor(system,core): lift collect_types into tag provider caller Tag providers now receive the concrete BaseModel subclasses for the entry point instead of the raw entry-point value. _generate_tags walks the model once via collect_types and passes the result to every provider, so providers can't forget to handle discriminated unions and the walk happens once per model rather than once per provider. The TagProvider type alias drops Any in favor of Iterable[type[BaseModel]], honestly typing what providers receive. The first arg of _generate_tags is annotated Any to match the entry-point loader, which yields union expressions that aren't type[BaseModel]. All three registered providers (feature_provider, authority_provider, theme_provider) update to the new signature; unit tests pass concrete classes directly while union-handling tests move to the _generate_tags integration boundary, where the walk now lives. Signed-off-by: Seth Fitzsimmons --- README.pydantic.md | 48 +++++++++++++++---- .../src/overture/schema/core/tag_providers.py | 41 +++++++++------- .../tests/test_core_tag_providers.py | 21 +++++--- packages/overture-schema-system/README.md | 17 ++++--- .../schema/system/discovery/discovery.py | 18 ++++--- .../schema/system/discovery/tag_providers.py | 26 +++++----- .../overture/schema/system/discovery/types.py | 16 +++---- .../tests/test_tag_providers.py | 24 +++++++--- 8 files changed, 140 insertions(+), 71 deletions(-) diff --git a/README.pydantic.md b/README.pydantic.md index d966d92e7..c5d46f55c 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -160,22 +160,52 @@ The discovery system provides programmatic access to registered models: ```python from overture.schema.system.discovery import discover_models, get_registered_model -# Discover all registered models +# Discover all registered models, keyed by ModelKey all_models = discover_models() -# Returns: -# { -# ModelKey(name="building", entry_point="overture.schema.buildings:Building", tags=frozenset({"feature", "overture", "overture:theme=buildings"})): BuildingModel, -# ModelKey(name="place", entry_point="overture.schema.places:Place", tags=frozenset({"feature", "overture", "overture:theme=places"})): PlaceModel, -# ... -# } -# Get a specific model by type +# Get a specific model by name building_model = get_registered_model("building") if building_model: - # Use the model class building = building_model.model_validate(building_data) ``` +### Tagging + +Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags +that classify the model orthogonally to its entry-point name -- whether the model +is a `Feature` subclass, which Overture theme it belongs to, which package shipped +it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags +to filter the working set without importing every model: + +```python +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) + +models = discover_models() +# { +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): BuildingModel, +# ModelKey(name="place", entry_point="overture.schema.places:Place", +# tags=frozenset({"feature", "overture", "overture:theme=places"})): PlaceModel, +# ... +# } + +buildings = filter_models( + models, + TagSelector(include_any=("overture:theme=buildings",)), +) +``` + +Tags are produced by *tag providers* registered on the `overture.tag_providers` +entry-point group. The `system` and `core` packages ship the built-in providers +(`feature`, `overture`, `overture:theme=*`); third parties can register their own +to attach custom tags during discovery. See the [`overture-schema-system` +README](packages/overture-schema-system/README.md#tagging) for tag format, +reserved namespaces, and provider authoring. + ## Development This project uses [uv](https://docs.astral.sh/uv/) for dependency management: diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index 004aa7848..fa3c629ad 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -5,11 +5,13 @@ `overture.tag_providers` entry-point group. """ -from typing import Any, Literal, get_args, get_origin +from collections.abc import Iterable +from typing import Literal, get_args, get_origin + +from pydantic import BaseModel from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ModelKey -from overture.schema.system.typing_util import collect_types APPROVED = { "overture.schema.addresses:Address", @@ -31,14 +33,18 @@ } -def authority_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: # noqa: ANN401 +def authority_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: """Add the `"overture"` tag if the model originates from an approved Overture package. Parameters ---------- - model_class - A class or discriminated-union type expression loaded from an - `overture.models` entry point. + types + Concrete `BaseModel` subclasses for the entry point. Unused — + approval is determined by the entry-point identity in `key`. key Key identifying the model. tags @@ -54,15 +60,18 @@ def authority_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[s return tags -def theme_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: # noqa: ANN401 +def theme_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: """Add `"overture:theme={theme}"` for each `OvertureFeature` referenced. Tags are attached to the entry point's `ModelKey`. For - discriminated-union features, the provider walks every concrete arm - via `collect_types` and reads each arm's `theme`; tags from all arms - accumulate on the union's `ModelKey`. Arms that share a theme - deduplicate to a single tag; arms with different themes contribute - multiple `overture:theme=X` tags to the same `ModelKey`. + discriminated-union features, every concrete arm contributes its + own `theme`; arms that share a theme deduplicate to a single tag, + and arms with different themes contribute multiple + `overture:theme=X` tags to the same `ModelKey`. Each arm's `theme` field must be annotated as a single-value `Literal[str]`; any other annotation is a model-definition bug and @@ -70,9 +79,9 @@ def theme_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: Parameters ---------- - model_class - A class or discriminated-union type expression loaded from an - `overture.models` entry point. + types + Concrete `BaseModel` subclasses for the entry point. For + discriminated-union features this is every arm. key Key identifying the model. tags @@ -89,7 +98,7 @@ def theme_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: If a referenced `OvertureFeature`'s `theme` is not a single-value `Literal[str]`. """ - for tp in collect_types(model_class): + for tp in types: if issubclass(tp, OvertureFeature): tags.add(f"overture:theme={_theme_literal(tp)}") return tags diff --git a/packages/overture-schema-core/tests/test_core_tag_providers.py b/packages/overture-schema-core/tests/test_core_tag_providers.py index 3131501e3..5be8fcf53 100644 --- a/packages/overture-schema-core/tests/test_core_tag_providers.py +++ b/packages/overture-schema-core/tests/test_core_tag_providers.py @@ -10,6 +10,8 @@ theme_provider, ) from overture.schema.system.discovery import ModelKey +from overture.schema.system.discovery.discovery import _generate_tags +from overture.schema.system.discovery.types import TagProviderDict, TagProviderKey from pydantic import BaseModel, Field, Tag @@ -42,11 +44,12 @@ def _empty_key(name: str = "x", entry_point: str = "mod:X") -> ModelKey: def test_theme_provider_plain_class(building: type[OvertureFeature]) -> None: - tags = theme_provider(building, _empty_key(), set()) + tags = theme_provider((building,), _empty_key(), set()) assert tags == {"overture:theme=buildings"} def test_theme_provider_discriminated_union() -> None: + # `_generate_tags` is responsible for walking the union to concrete arms. class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): pass @@ -57,12 +60,18 @@ class Rail(OvertureFeature[Literal["transportation"], Literal["rail"]]): Annotated[Road, Tag("road")] | Annotated[Rail, Tag("rail")], Field(discriminator="type"), ] - tags = theme_provider(union, _empty_key(), set()) + provider_key = TagProviderKey( + name="theme", + entry_point="core:theme_provider", + package_name="overture-schema-core", + ) + providers: TagProviderDict = {provider_key: theme_provider} + tags = _generate_tags(union, _empty_key(), providers) assert tags == {"overture:theme=transportation"} def test_theme_provider_skips_non_overture(not_overture: type[BaseModel]) -> None: - tags = theme_provider(not_overture, _empty_key(), set()) + tags = theme_provider((not_overture,), _empty_key(), set()) assert tags == set() @@ -73,17 +82,17 @@ class BadFeature(OvertureFeature): # type: ignore[type-arg] pass with pytest.raises(TypeError, match="must be annotated Literal"): - theme_provider(BadFeature, _empty_key(), set()) + theme_provider((BadFeature,), _empty_key(), set()) def test_authority_provider_approved(road: type[OvertureFeature]) -> None: approved_entry = next(iter(APPROVED)) key = ModelKey(name="x", entry_point=approved_entry, tags=frozenset()) - tags = authority_provider(road, key, set()) + tags = authority_provider((road,), key, set()) assert "overture" in tags def test_authority_provider_not_approved(road: type[OvertureFeature]) -> None: key = ModelKey(name="x", entry_point="some.unapproved:Model", tags=frozenset()) - tags = authority_provider(road, key, set()) + tags = authority_provider((road,), key, set()) assert "overture" not in tags diff --git a/packages/overture-schema-system/README.md b/packages/overture-schema-system/README.md index e9a628932..d5d93f65a 100644 --- a/packages/overture-schema-system/README.md +++ b/packages/overture-schema-system/README.md @@ -160,15 +160,20 @@ Helpers in `overture.schema.system.discovery.tag` parse structured tags: ### Providers -A tag provider is a callable registered on the `overture.tag_providers` entry point group. Discovery passes it the model class and a copy of the tags accumulated so far; tags it adds are merged into the running set after passing the reservation checks below. +A tag provider is a callable registered on the `overture.tag_providers` entry point group. Discovery passes it the concrete `BaseModel` subclasses for the entry point and a copy of the tags accumulated so far; tags it adds are merged into the running set after passing the reservation checks below. ```python +from collections.abc import Iterable +from pydantic import BaseModel from overture.schema.system.discovery import ModelKey from overture.schema.system.feature import Feature -from overture.schema.system.typing_util import collect_types -def feature_provider(model_class, key: ModelKey, tags: set[str]) -> set[str]: - if any(issubclass(tp, Feature) for tp in collect_types(model_class)): +def feature_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: + if any(issubclass(tp, Feature) for tp in types): tags.add("feature") return tags ``` @@ -180,7 +185,7 @@ feature = "overture.schema.system.discovery.tag_providers:feature_provider" Tags from one provider are visible to providers that run later, but execution order is unspecified -- a provider must not depend on tags added by another. Provider exceptions are caught, logged, and discarded; they do not abort discovery. -The `model_class` argument is whatever the entry point loads. For most features that is a `type[BaseModel]`, but discriminated-union features (e.g. `Segment`) load as `Annotated[Union[...], Field(...)]` expressions, which are not `type` objects. Providers walk these with `collect_types` instead of calling `issubclass` directly. +Discovery resolves the entry-point value to concrete classes before invoking providers. For class entries that yields a one-element iterable; for discriminated-union features (e.g. `Segment`, which loads as `Annotated[Union[...], Field(...)]`) it yields every arm. Providers therefore work uniformly with `issubclass` and never need to walk type expressions themselves. ### Reservation @@ -197,7 +202,7 @@ When a provider attempts to set a reserved tag from an unauthorized package, dis ### Built-in Providers -- **`feature`** (in `system`) -- adds `feature` if the entry point references a `Feature` subclass, walking discriminated unions via `collect_types`. +- **`feature`** (in `system`) -- adds `feature` if any concrete arm is a `Feature` subclass. - **`authority`** (in `core`) -- adds `overture` if the model's entry point appears in the curated approval manifest (`APPROVED` in `overture.schema.core.tag_providers`). - **`theme`** (in `core`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme. diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index 5ebadef4f..0ac082b0d 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -3,6 +3,7 @@ import importlib.metadata import logging from dataclasses import dataclass, replace +from typing import Any from pydantic import BaseModel @@ -16,6 +17,7 @@ TagProviderDict, TagProviderKey, ) +from overture.schema.system.typing_util import collect_types log = logging.getLogger(__name__) @@ -32,20 +34,23 @@ def _generate_tags( - model_class: type[BaseModel], + model_class: Any, # noqa: ANN401 key: ModelKey, providers: TagProviderDict, ) -> set[str]: """Generate tags for a model class using tag providers. - Each provider is called in turn indeterministically; tags it adds are filtered for - validity and permission before being included. Provider errors are caught and - logged as warnings rather than propagated. + The model is walked once via `collect_types` to find every concrete + `BaseModel` arm, and each provider is called with the result. Tags + a provider adds are filtered for validity and permission before + being included. Provider errors are caught and logged as warnings + rather than propagated. Parameters ---------- model_class - Model class to generate tags for. + Value loaded from an `overture.models` entry point — usually a + `type[BaseModel]`, or a discriminated-union expression. key Key identifying the model. providers @@ -56,10 +61,11 @@ def _generate_tags( set[str] Tags generated for the model. """ + types = collect_types(model_class) tags: set[str] = set() for provider_key, provider in providers.items(): try: - added_tags = set(provider(model_class, key, tags.copy())) - tags + added_tags = set(provider(types, key, tags.copy())) - tags filtered_tags = _filter_tags(added_tags, provider_key) tags.update(filtered_tags) except Exception as e: diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py index bb5456d09..46fc507d8 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py @@ -1,25 +1,25 @@ """Tag provider logic for Overture schema discovery system.""" -from typing import Any +from collections.abc import Iterable + +from pydantic import BaseModel from overture.schema.system.discovery.types import ModelKey from overture.schema.system.feature import Feature -from overture.schema.system.typing_util import collect_types - -def feature_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str]: # noqa: ANN401 - """Add the `"feature"` tag if the entry point references a `Feature` subclass. - Tags are attached to the entry point's `ModelKey`. For - discriminated-union features, the provider walks every concrete arm - via `collect_types`; if any arm is a `Feature` subclass, the tag is - added to the union's `ModelKey`. +def feature_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: + """Add the `"feature"` tag if any concrete type is a `Feature` subclass. Parameters ---------- - model_class - A class or discriminated-union type expression loaded from an - `overture.models` entry point. + types + Concrete `BaseModel` subclasses for the entry point. For + discriminated-union features this is every arm. key Key identifying the model. tags @@ -30,6 +30,6 @@ def feature_provider(model_class: Any, key: ModelKey, tags: set[str]) -> set[str set[str] Updated tags, with `"feature"` added if applicable. """ - if any(issubclass(tp, Feature) for tp in collect_types(model_class)): + if any(issubclass(tp, Feature) for tp in types): tags.add("feature") return tags diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py index bf96768c2..13b219c79 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -1,18 +1,18 @@ """Types and data classes for Overture schema discovery system.""" from collections.abc import Callable, Iterable -from typing import Any, TypeAlias +from typing import TypeAlias from pydantic import BaseModel from .models import ModelKey, TagProviderKey -# The first argument is the value loaded from an `overture.models` entry -# point. That is usually a `type[BaseModel]`, but discriminated-union -# features (e.g. `Segment`) load as `Annotated[Union[...], Field(...)]` -# expressions, which are not `type` objects. Providers should use -# `overture.schema.system.typing_util.collect_types` to walk to concrete -# classes. -TagProvider: TypeAlias = Callable[[Any, ModelKey, set[str]], Iterable[str]] +# Tag providers receive the concrete `BaseModel` subclasses for an entry +# point. For class entries this is a one-element iterable; for +# discriminated unions it is every arm collected by `collect_types`. +TagProvider: TypeAlias = Callable[ + [Iterable[type[BaseModel]], ModelKey, set[str]], + Iterable[str], +] ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py index 4ab95943f..5a75f3f3a 100644 --- a/packages/overture-schema-system/tests/test_tag_providers.py +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -1,4 +1,5 @@ -from typing import Annotated, Any +from collections.abc import Iterable +from typing import Annotated import pytest from pydantic import BaseModel, Field, Tag @@ -8,6 +9,7 @@ from overture.schema.system.discovery.types import ( ModelKey, TagProvider, + TagProviderDict, TagProviderKey, ) from overture.schema.system.feature import Feature @@ -68,7 +70,11 @@ class M(BaseModel): def fake_provider(*tags: str) -> TagProvider: """Provider that always returns the given tags, ignoring its inputs.""" - def _provider(model_class: Any, key: ModelKey, current_tags: set[str]) -> set[str]: + def _provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + current_tags: set[str], + ) -> set[str]: return set(tags) return _provider @@ -171,7 +177,7 @@ def test_mixed_tags( def test_feature_provider_adds_feature_tag(feature: type[Feature]) -> None: key = ModelKey(name="feature", entry_point="system:Feature", tags=frozenset()) - result = feature_provider(feature, key, set()) + result = feature_provider((feature,), key, set()) assert "feature" in result @@ -181,12 +187,15 @@ def test_feature_provider_does_not_add_feature_tag( key = ModelKey( name="notafeature", entry_point="system:NotAFeature", tags=frozenset() ) - result = feature_provider(not_a_feature, key, set()) + result = feature_provider((not_a_feature,), key, set()) assert "feature" not in result -def test_feature_provider_handles_discriminated_union() -> None: - # Mimics the shape of `Segment`: Annotated[Union[...], Field(discriminator=...)] +def test_feature_provider_handles_discriminated_union( + system_tag_provider: TagProviderKey, +) -> None: + # Mimics the shape of `Segment`: Annotated[Union[...], Field(discriminator=...)]. + # `_generate_tags` is responsible for walking the union to concrete arms. class ArmA(Feature): pass @@ -198,5 +207,6 @@ class ArmB(BaseModel): Field(discriminator="type"), ] key = ModelKey(name="union", entry_point="mod:Union", tags=frozenset()) - result = feature_provider(union, key, set()) + providers: TagProviderDict = {system_tag_provider: feature_provider} + result = _generate_tags(union, key, providers) assert "feature" in result From 4fed96432b27960998729a049e4584f7f32abefc Mon Sep 17 00:00:00 2001 From: Victor Schappert Date: Wed, 6 May 2026 08:53:04 -0700 Subject: [PATCH 19/23] Rename models.py to keys.py Per discussion in the coding sesh. Signed-off-by: Victor Schappert --- .../src/overture/schema/system/discovery/{models.py => keys.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/overture-schema-system/src/overture/schema/system/discovery/{models.py => keys.py} (100%) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/models.py b/packages/overture-schema-system/src/overture/schema/system/discovery/keys.py similarity index 100% rename from packages/overture-schema-system/src/overture/schema/system/discovery/models.py rename to packages/overture-schema-system/src/overture/schema/system/discovery/keys.py From 25c29e1a6415c1a453870c55aa3df19f2fd4c87d Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 6 May 2026 08:53:18 -0700 Subject: [PATCH 20/23] refactor(system): rename discovery models module to keys The module defines ModelKey and TagProviderKey -- key types, not domain models. Rename clarifies intent and avoids confusion with theme model modules elsewhere in the codebase. Signed-off-by: Seth Fitzsimmons --- .../src/overture/schema/system/discovery/__init__.py | 2 +- .../src/overture/schema/system/discovery/types.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py index 272e9c682..ed8af77ad 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py @@ -5,7 +5,7 @@ filter_models, get_registered_model, ) -from .models import ModelKey +from .keys import ModelKey from .types import ModelDict __all__ = [ diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py index 13b219c79..9bad9250b 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -5,7 +5,7 @@ from pydantic import BaseModel -from .models import ModelKey, TagProviderKey +from .keys import ModelKey, TagProviderKey # Tag providers receive the concrete `BaseModel` subclasses for an entry # point. For class entries this is a one-element iterable; for From 5b34971515ce7357af506a873652e6b752a4dc83 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Wed, 6 May 2026 18:31:03 +0200 Subject: [PATCH 21/23] remove(core): drop authority_provider Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- .../tests/test_cli_commands.py | 6 +-- packages/overture-schema-core/README.md | 2 +- packages/overture-schema-core/pyproject.toml | 3 +- .../src/overture/schema/core/tag_providers.py | 50 ------------------- .../tests/test_approved_models.py | 10 ---- .../tests/test_core_tag_providers.py | 15 ------ packages/overture-schema-system/README.md | 2 - 7 files changed, 5 insertions(+), 83 deletions(-) delete mode 100644 packages/overture-schema-core/tests/test_approved_models.py diff --git a/packages/overture-schema-cli/tests/test_cli_commands.py b/packages/overture-schema-cli/tests/test_cli_commands.py index e62df564f..3abc8d3a0 100644 --- a/packages/overture-schema-cli/tests/test_cli_commands.py +++ b/packages/overture-schema-cli/tests/test_cli_commands.py @@ -385,7 +385,7 @@ def test_validate_wires_tag_combinator_flag( ) -> None: """validate wires --filter/--exclude through to the tag selector.""" result = cli_runner.invoke( - cli, ["validate", building_feature_yaml, "--tag", "overture", flag, value] + cli, ["validate", building_feature_yaml, "--tag", "feature", flag, value] ) assert result.exit_code == 0 @@ -397,7 +397,7 @@ def test_json_schema_wires_tag_combinator_flag( value: str, ) -> None: """json-schema wires --filter/--exclude through to the tag selector.""" - result = cli_runner.invoke(cli, ["json-schema", "--tag", "overture", flag, value]) + result = cli_runner.invoke(cli, ["json-schema", "--tag", "feature", flag, value]) assert result.exit_code == 0 assert result.output # non-empty JSON @@ -409,5 +409,5 @@ def test_list_types_wires_tag_combinator_flag( value: str, ) -> None: """list-types wires --filter/--exclude through to the tag selector.""" - result = cli_runner.invoke(cli, ["list-types", "--tag", "overture", flag, value]) + result = cli_runner.invoke(cli, ["list-types", "--tag", "feature", flag, value]) assert result.exit_code == 0 diff --git a/packages/overture-schema-core/README.md b/packages/overture-schema-core/README.md index 8955623f1..cef61bc9a 100644 --- a/packages/overture-schema-core/README.md +++ b/packages/overture-schema-core/README.md @@ -94,4 +94,4 @@ Rendering hints for map-making: `prominence` (1--100 significance scale), `min_z - **Types** -- domain-specific aliases built on system primitives: `ConfidenceScore` (0.0--1.0), `Level` (z-order), `FeatureVersion`. - **Units** -- measurement enumerations: `SpeedUnit`, `LengthUnit`, `WeightUnit`. -- **Tag providers** -- `authority` and `theme` providers for the discovery system in `overture-schema-system`. They tag `OvertureFeature`-derived models with `overture` (when manifest-approved) and `overture:theme={theme}`. +- **Tag providers** -- `theme` provider for the discovery system in `overture-schema-system`. Tags `OvertureFeature`-derived models with `overture:theme={theme}`. diff --git a/packages/overture-schema-core/pyproject.toml b/packages/overture-schema-core/pyproject.toml index 87c579cde..d7d93d7c5 100644 --- a/packages/overture-schema-core/pyproject.toml +++ b/packages/overture-schema-core/pyproject.toml @@ -38,5 +38,4 @@ dev = [ ] [project.entry-points."overture.tag_providers"] -authority = "overture.schema.core.tag_providers:authority_provider" -theme = "overture.schema.core.tag_providers:theme_provider" +theme = "overture.schema.core.tag_providers:theme_provider" diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py index fa3c629ad..0c4b69b9b 100644 --- a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -13,52 +13,6 @@ from overture.schema.core import OvertureFeature from overture.schema.system.discovery import ModelKey -APPROVED = { - "overture.schema.addresses:Address", - "overture.schema.base:Bathymetry", - "overture.schema.base:Infrastructure", - "overture.schema.base:Land", - "overture.schema.base:LandCover", - "overture.schema.base:LandUse", - "overture.schema.base:Water", - "overture.schema.buildings:Building", - "overture.schema.buildings:BuildingPart", - "overture.schema.divisions:Division", - "overture.schema.divisions:DivisionArea", - "overture.schema.divisions:DivisionBoundary", - "overture.schema.places:Place", - "overture.schema.transportation:Connector", - "overture.schema.transportation:Segment", - "overture.schema.annex:Sources", -} - - -def authority_provider( - types: Iterable[type[BaseModel]], - key: ModelKey, - tags: set[str], -) -> set[str]: - """Add the `"overture"` tag if the model originates from an approved Overture package. - - Parameters - ---------- - types - Concrete `BaseModel` subclasses for the entry point. Unused — - approval is determined by the entry-point identity in `key`. - key - Key identifying the model. - tags - Current tags; may be extended. - - Returns - ------- - set[str] - Updated tags, with `"overture"` added if applicable. - """ - if _matches_manifest(key): - tags.add("overture") - return tags - def theme_provider( types: Iterable[type[BaseModel]], @@ -104,10 +58,6 @@ def theme_provider( return tags -def _matches_manifest(key: ModelKey) -> bool: - return key.entry_point in APPROVED - - def _theme_literal(model_class: type[OvertureFeature]) -> str: """Extract the literal `theme` value from an `OvertureFeature` subclass. diff --git a/packages/overture-schema-core/tests/test_approved_models.py b/packages/overture-schema-core/tests/test_approved_models.py deleted file mode 100644 index 36f30d302..000000000 --- a/packages/overture-schema-core/tests/test_approved_models.py +++ /dev/null @@ -1,10 +0,0 @@ -from overture.schema.system.discovery import discover_models - - -def test_overture_feature_models_are_official() -> None: - models = discover_models() - for key in models: - if "feature" in key.tags: - assert "overture" in key.tags, ( - f"Model {key.name} is missing 'overture:official' tag." - ) diff --git a/packages/overture-schema-core/tests/test_core_tag_providers.py b/packages/overture-schema-core/tests/test_core_tag_providers.py index 5be8fcf53..2fde65e5d 100644 --- a/packages/overture-schema-core/tests/test_core_tag_providers.py +++ b/packages/overture-schema-core/tests/test_core_tag_providers.py @@ -5,8 +5,6 @@ import pytest from overture.schema.core import OvertureFeature from overture.schema.core.tag_providers import ( - APPROVED, - authority_provider, theme_provider, ) from overture.schema.system.discovery import ModelKey @@ -83,16 +81,3 @@ class BadFeature(OvertureFeature): # type: ignore[type-arg] with pytest.raises(TypeError, match="must be annotated Literal"): theme_provider((BadFeature,), _empty_key(), set()) - - -def test_authority_provider_approved(road: type[OvertureFeature]) -> None: - approved_entry = next(iter(APPROVED)) - key = ModelKey(name="x", entry_point=approved_entry, tags=frozenset()) - tags = authority_provider((road,), key, set()) - assert "overture" in tags - - -def test_authority_provider_not_approved(road: type[OvertureFeature]) -> None: - key = ModelKey(name="x", entry_point="some.unapproved:Model", tags=frozenset()) - tags = authority_provider((road,), key, set()) - assert "overture" not in tags diff --git a/packages/overture-schema-system/README.md b/packages/overture-schema-system/README.md index d5d93f65a..d5d037cc6 100644 --- a/packages/overture-schema-system/README.md +++ b/packages/overture-schema-system/README.md @@ -194,7 +194,6 @@ Specific plain tags and namespaces are reserved for designated packages. For exa | Tag or namespace | Owning package | |---|---| | `feature` (tag) | `overture-schema-system` | -| `overture` (tag) | `overture-schema-core` | | `system:` (namespace) | `overture-schema-system` | | `overture:` (namespace) | `overture-schema-core` | @@ -203,7 +202,6 @@ When a provider attempts to set a reserved tag from an unauthorized package, dis ### Built-in Providers - **`feature`** (in `system`) -- adds `feature` if any concrete arm is a `Feature` subclass. -- **`authority`** (in `core`) -- adds `overture` if the model's entry point appears in the curated approval manifest (`APPROVED` in `overture.schema.core.tag_providers`). - **`theme`** (in `core`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme. ### Selecting Models by Tag From 8c20e2c69bec460def5e9555be81d3388e68c9d6 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 6 May 2026 09:57:59 -0700 Subject: [PATCH 22/23] chore(cli,codegen): bump click floor to 8.1 Click 8.1 introduced typed decorator returns that preserve the TypeVar in `tag_selection_options`, so the lowest-direct mypy job no longer reports `Callable[..., Any]` reassignments. The 8.0 floor predated this and only affected lowest-direct. Signed-off-by: Seth Fitzsimmons --- packages/overture-schema-cli/pyproject.toml | 2 +- packages/overture-schema-codegen/pyproject.toml | 2 +- uv.lock | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/overture-schema-cli/pyproject.toml b/packages/overture-schema-cli/pyproject.toml index 590121005..ff2e5b7ef 100644 --- a/packages/overture-schema-cli/pyproject.toml +++ b/packages/overture-schema-cli/pyproject.toml @@ -7,7 +7,7 @@ dependencies = [ "overture-schema-system", "pydantic>=2.12.0", "pyyaml>=6.0.2", - "click>=8.0", + "click>=8.1", "rich>=13.0", "yamlcore>=0.0.4", ] diff --git a/packages/overture-schema-codegen/pyproject.toml b/packages/overture-schema-codegen/pyproject.toml index daa200a4f..e6f55e127 100644 --- a/packages/overture-schema-codegen/pyproject.toml +++ b/packages/overture-schema-codegen/pyproject.toml @@ -4,7 +4,7 @@ requires = ["hatchling"] [project] dependencies = [ - "click>=8.0", + "click>=8.1", "jinja2>=3.0", "overture-schema-cli", "overture-schema-core", diff --git a/uv.lock b/uv.lock index ca70a5b86..2b1bf0be2 100644 --- a/uv.lock +++ b/uv.lock @@ -215,7 +215,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -811,7 +811,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "click", specifier = ">=8.0" }, + { name = "click", specifier = ">=8.1" }, { name = "overture-schema-core", editable = "packages/overture-schema-core" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, { name = "pydantic", specifier = ">=2.12.0" }, @@ -841,7 +841,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "click", specifier = ">=8.0" }, + { name = "click", specifier = ">=8.1" }, { name = "jinja2", specifier = ">=3.0" }, { name = "overture-schema-cli", editable = "packages/overture-schema-cli" }, { name = "overture-schema-core", editable = "packages/overture-schema-core" }, From e11a1c4b4b8d5cce240debfffd2586c8cfa90858 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Wed, 6 May 2026 20:59:15 +0200 Subject: [PATCH 23/23] chore(system,core,codegen): removes leftover references to overture plain tag that were missed in the authority_provider removal Signed-off-by: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> --- packages/overture-schema-codegen/tests/test_cli.py | 4 ++-- .../overture-schema-core/tests/test_core_tag_providers.py | 8 -------- .../src/overture/schema/system/discovery/discovery.py | 1 - .../src/overture/schema/system/discovery/tag.py | 2 +- .../overture-schema-system/tests/test_tag_providers.py | 6 +----- 5 files changed, 4 insertions(+), 17 deletions(-) diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index 8ba4efaf9..13957606e 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -71,7 +71,7 @@ def test_generate_accepts_filter_flag(self, cli_runner: CliRunner) -> None: "--format", "markdown", "--tag", - "overture", + "feature", "--filter", "feature", ], @@ -90,7 +90,7 @@ def test_generate_accepts_exclude_flag(self, cli_runner: CliRunner) -> None: "--format", "markdown", "--tag", - "overture", + "feature", "--exclude", "overture:theme=places", ], diff --git a/packages/overture-schema-core/tests/test_core_tag_providers.py b/packages/overture-schema-core/tests/test_core_tag_providers.py index 2fde65e5d..8c8b7dce3 100644 --- a/packages/overture-schema-core/tests/test_core_tag_providers.py +++ b/packages/overture-schema-core/tests/test_core_tag_providers.py @@ -13,14 +13,6 @@ from pydantic import BaseModel, Field, Tag -@pytest.fixture -def road() -> type[OvertureFeature]: - class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): - pass - - return Road - - @pytest.fixture def building() -> type[OvertureFeature]: class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index 0ac082b0d..bd271fa9e 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -23,7 +23,6 @@ # Tags that are reserved and can only be set by specific packages. _RESERVED_TAGS: dict[str, set[str]] = { - "overture": {"overture-schema-core"}, "feature": {"overture-schema-system"}, } # Namespaces that are reserved and can only be set by specific packages. diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py index 5633ceeee..d11fa8f3b 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py @@ -2,7 +2,7 @@ Tags follow the pattern `[namespace:]predicate[=value]` and come in three forms: -- **Plain** — `overture`, `feature` +- **Plain** — `feature` - **Namespaced** — `system:extension` - **Key/value** — `overture:theme=buildings` diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py index 5a75f3f3a..ef7bba7ec 100644 --- a/packages/overture-schema-system/tests/test_tag_providers.py +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -107,20 +107,16 @@ def test_reserved_tag( any_key: ModelKey, any_model: type[BaseModel], ) -> None: - providers = {other_tag_provider: fake_provider("overture", "feature", "valid")} + providers = {other_tag_provider: fake_provider("feature", "valid")} result = _generate_tags(any_model, any_key, providers) assert result == {"valid"} def test_allowed_reserved_tag( - core_tag_provider: TagProviderKey, system_tag_provider: TagProviderKey, any_key: ModelKey, any_model: type[BaseModel], ) -> None: - core_providers = {core_tag_provider: fake_provider("overture")} - assert _generate_tags(any_model, any_key, core_providers) == {"overture"} - system_providers = {system_tag_provider: fake_provider("feature")} assert _generate_tags(any_model, any_key, system_providers) == {"feature"}