diff --git a/README.pydantic.md b/README.pydantic.md index 0655c46d9..c5d46f55c 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -151,31 +151,61 @@ Registration is done in the `[project.entry-points."overture.models"]` section: ```toml [project.entry-points."overture.models"] -"buildings.building" = "overture.schema.buildings.building.models:Building" -"buildings.building_part" = "overture.schema.buildings.building_part.models:BuildingPart" +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" ``` The discovery system provides programmatic access to registered models: ```python -from overture.schema.core.discovery import discover_models, get_registered_model +from overture.schema.system.discovery import discover_models, get_registered_model -# Discover all registered models +# Discover all registered models, keyed by ModelKey all_models = discover_models() -# Returns: -# { -# ("buildings", "building"): BuildingModel, -# ("places", "place"): PlaceModel, -# ... -# } -# Get a specific model by theme and type -building_model = get_registered_model("buildings", "building") +# Get a specific model by name +building_model = get_registered_model("building") if building_model: - # Use the model class building = building_model.model_validate(building_data) ``` +### Tagging + +Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags +that classify the model orthogonally to its entry-point name -- whether the model +is a `Feature` subclass, which Overture theme it belongs to, which package shipped +it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags +to filter the working set without importing every model: + +```python +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) + +models = discover_models() +# { +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): BuildingModel, +# ModelKey(name="place", entry_point="overture.schema.places:Place", +# tags=frozenset({"feature", "overture", "overture:theme=places"})): PlaceModel, +# ... +# } + +buildings = filter_models( + models, + TagSelector(include_any=("overture:theme=buildings",)), +) +``` + +Tags are produced by *tag providers* registered on the `overture.tag_providers` +entry-point group. The `system` and `core` packages ship the built-in providers +(`feature`, `overture`, `overture:theme=*`); third parties can register their own +to attach custom tags during discovery. See the [`overture-schema-system` +README](packages/overture-schema-system/README.md#tagging) for tag format, +reserved namespaces, and provider authoring. + ## Development This project uses [uv](https://docs.astral.sh/uv/) for dependency management: diff --git a/packages/overture-schema-addresses-theme/pyproject.toml b/packages/overture-schema-addresses-theme/pyproject.toml index 3a3ae3127..0752149cc 100644 --- a/packages/overture-schema-addresses-theme/pyproject.toml +++ b/packages/overture-schema-addresses-theme/pyproject.toml @@ -37,7 +37,7 @@ pythonpath = ["src"] testpaths = ["tests"] [project.entry-points."overture.models"] -"overture:addresses:address" = "overture.schema.addresses:Address" +address = "overture.schema.addresses:Address" [[examples.Address]] id = "416ab01c-d836-4c4f-aedc-2f30941ce94d" diff --git a/packages/overture-schema-annex/pyproject.toml b/packages/overture-schema-annex/pyproject.toml index 9ef990a8d..de90cf8d1 100644 --- a/packages/overture-schema-annex/pyproject.toml +++ b/packages/overture-schema-annex/pyproject.toml @@ -29,7 +29,7 @@ path = "src/overture/schema/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"annex:sources" = "overture.schema.annex:Sources" +sources = "overture.schema.annex:Sources" [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/packages/overture-schema-base-theme/pyproject.toml b/packages/overture-schema-base-theme/pyproject.toml index fdf146adf..fd5a79cdf 100644 --- a/packages/overture-schema-base-theme/pyproject.toml +++ b/packages/overture-schema-base-theme/pyproject.toml @@ -35,13 +35,13 @@ path = "src/overture/schema/base/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:base:bathymetry" = "overture.schema.base:Bathymetry" -"overture:base:infrastructure" = "overture.schema.base:Infrastructure" -"overture:base:land" = "overture.schema.base:Land" -"overture:base:land_cover" = "overture.schema.base:LandCover" -"overture:base:land_use" = "overture.schema.base:LandUse" -"overture:base:water" = "overture.schema.base:Water" - +bathymetry = "overture.schema.base:Bathymetry" +infrastructure = "overture.schema.base:Infrastructure" +land = "overture.schema.base:Land" +land_cover = "overture.schema.base:LandCover" +land_use = "overture.schema.base:LandUse" +water = "overture.schema.base:Water" + [[examples.Bathymetry]] id = "5d40bd6c-db14-5492-b29f-5e25a59032bc" geometry = "MULTIPOLYGON (((-170.71296928 -76.744313428, -170.719841483 -76.757076376, -170.731061124 -76.761566192, -170.775652756 -76.76338726, -170.853616381 -76.76253958, -170.918562293 -76.755380155, -170.970490492 -76.741908984, -170.998699301 -76.729180777, -171.003188718 -76.717195533, -170.990421551 -76.703765214, -170.960397802 -76.68888982, -170.940748072 -76.674697941, -170.931472364 -76.661189576, -170.927114414 -76.637296658, -170.927674224 -76.603019188, -170.939335393 -76.574637428, -170.962097922 -76.552151379, -170.999015387 -76.535715361, -171.050087788 -76.525329373, -171.079133298 -76.50751024, -171.086151917 -76.482257963, -171.098653755 -76.462747286, -171.11663881 -76.448978211, -171.146691397 -76.437601179, -171.188811514 -76.428616191, -171.296181785 -76.4228609, -171.468802209 -76.420335306, -171.566055241 -76.41501101, -171.587940879 -76.406888013, -171.59004284 -76.387987744, -171.572361122 -76.358310204, -171.549343725 -76.334488281, -171.520990649 -76.316521976, -171.453759127 -76.301763636, -171.347649159 -76.290213262, -171.30597166 -76.267707269, -171.328726628 -76.234245658, -171.36676019 -76.195627518, -171.420072345 -76.151852851, -171.444766298 -76.12494912, -171.44084205 -76.114916326, -171.378107286 -76.099627787, -171.256562007 -76.079083503, -171.228218647 -76.058825682, -171.293077208 -76.038854322, -171.421365419 -76.023534207, -171.613083278 -76.012865337, -171.76411833 -75.99938969, -171.874470572 -75.983107266, -172.121928361 -75.958403596, -172.506491695 -75.925278679, -172.744527804 -75.899736153, -172.836036689 -75.88177602, -172.904681746 -75.862406785, -172.950462974 -75.841628448, -173.000855857 -75.830396498, -173.055860393 -75.828710933, -173.177561398 -75.810743709, -173.365958872 -75.776494827, -173.493573084 -75.759370386, -173.560404033 -75.759370386, -173.620925776 -75.77158365, -173.675138312 -75.796010178, -173.733786206 -75.808642966, -173.796869456 -75.809482015, -173.847216433 -75.805553449, -173.884827135 -75.79685727, -173.90475244 -75.789177124, -173.906992347 -75.782513013, -173.881736947 -75.76894365, -173.828986239 -75.748469035, -173.797974615 -75.732298475, -173.788702075 -75.72043197, -173.82491541 -75.701013882, -173.90661462 -75.674044211, -173.977087913 -75.656066882, -174.03633529 -75.647081894, -174.150190099 -75.643010485, -174.31865234 -75.643852655, -174.444433211 -75.652836726, -174.527532713 -75.669962696, -174.581709229 -75.687086831, -174.606962758 -75.704209131, -174.631095834 -75.708279163, -174.654108458 -75.699296928, -174.688637451 -75.699296928, -174.734682816 -75.708279163, -174.797846917 -75.708699866, -174.878129754 -75.700559037, -174.939903816 -75.70870181, -174.9831691 -75.733128185, -175.025841122 -75.746602837, -175.06791988 -75.749125768, -175.09922327 -75.755318987, -175.119751293 -75.765182495, -175.127900229 -75.775197415, -175.123670077 -75.785363749, -175.111718372 -75.791289392, -175.092045112 -75.792974345, -175.049907399 -75.780622976, -174.985305232 -75.754235285, -174.935355308 -75.74552996, -174.900057628 -75.754507001, -174.886060973 -75.766815613, -174.893365345 -75.782455795, -174.907537393 -75.791536245, -174.928577117 -75.794056963, -174.971105378 -75.818213107, -175.035122174 -75.864004677, -175.060941949 -75.892403254, -175.048564703 -75.903408839, -175.020469049 -75.909193043, -174.976654988 -75.909755867, -174.944760829 -75.90482541, -174.924786572 -75.894401673, -174.92111336 -75.881479168, -174.933741192 -75.866057897, -174.900484967 -75.857513625, -174.821344686 -75.855846351, -174.752433709 -75.839289534, -174.693752038 -75.807843172, -174.652894268 -75.780747792, -174.629860399 -75.758003392, -174.571227588 -75.745793709, -174.476995837 -75.744118743, -174.398722205 -75.751841803, -174.336406693 -75.768962888, -174.300477946 -75.783262828, -174.290935964 -75.794741623, -174.28812912 -75.812412878, -174.292057414 -75.836276591, -174.289237223 -75.852155302, -174.279668547 -75.860049012, -174.205113931 -75.879998026, -174.065573375 -75.912002343, -173.957779122 -75.924071248, -173.881731171 -75.916204739, -173.846521251 -75.926706189, -173.852149361 -75.955575598, -173.845408416 -75.979439305, -173.826298414 -75.99829731, -173.76424232 -76.018956172, -173.659240133 -76.041415889, -173.560434089 -76.057698465, -173.467824188 -76.067803901, -173.404678836 -76.077625909, -173.370998032 -76.087164489, -173.332530272 -76.106814524, -173.289275555 -76.136576014, -173.231864101 -76.154545405, -173.160295911 -76.1607227, -173.093917454 -76.17278471, -173.032728732 -76.190731436, -173.009710709 -76.205560908, -173.024863387 -76.217273124, -173.048718935 -76.225374126, -173.081277354 -76.229863912, -173.219658797 -76.237442552, -173.463863265 -76.248110046, -173.60352174 -76.25793895, -173.638634223 -76.266929265, -173.658723482 -76.274676093, -173.663789516 -76.281179435, -173.661403366 -76.289363255, -173.651565032 -76.299227554, -173.627282775 -76.313843189, -173.588556596 -76.33321016, -173.575369172 -76.355231445, -173.587720504 -76.379907046, -173.573965869 -76.402499893, -173.53410527 -76.423009985, -173.518376226 -76.437156259, -173.526778738 -76.444938715, -173.559015515 -76.446303683, -173.615086557 -76.441251162, -173.686785609 -76.421600788, -173.774112673 -76.387352563, -173.854573513 -76.372333877, -173.928168128 -76.37654473, -173.968906731 -76.383732772, -173.97678932 -76.393898005, -173.979325549 -76.410884215, -173.976515417 -76.434691403, -174.000646474 -76.454452818, -174.051718722 -76.470168462, -174.08231827 -76.482963711, -174.092445119 -76.492838563, -174.075053216 -76.514344245, -174.030142562 -76.547480757, -174.016669929 -76.575274601, -174.034635317 -76.597725777, -174.037021169 -76.62030279, -174.023827484 -76.64300564, -174.034634583 -76.661942018, -174.069442464 -76.677111923, -174.086843964 -76.690616859, -174.086839082 -76.702456825, -174.080513222 -76.712456309, -174.067866385 -76.72061531, -174.036259441 -76.725116584, -173.98569239 -76.725960131, -173.93723318 -76.720486558, -173.89088181 -76.708695864, -173.780274695 -76.695221211, -173.605411835 -76.6800626, -173.487930602 -76.662096294, -173.427830996 -76.641322294, -173.370307559 -76.630935294, -173.315360292 -76.630935294, -173.249406002 -76.637251344, -173.17244469 -76.649883444, -173.110795196 -76.653532162, -173.06445752 -76.648197497, -173.029349452 -76.637355272, -173.005470993 -76.621005486, -173.01753216 -76.605236858, -173.065532955 -76.590049388, -173.096548505 -76.576599032, -173.11057881 -76.564885791, -173.108053605 -76.552301955, -173.08897289 -76.538847523, -173.051362225 -76.527628807, -172.99522161 -76.518645807, -172.891534181 -76.516119525, -172.740299938 -76.52004996, -172.648684331 -76.524540794, -172.61668736 -76.529592027, -172.584268588 -76.541098757, -172.551428016 -76.559060982, -172.533042741 -76.576141146, -172.529112765 -76.592339249, -172.540195073 -76.604524646, -172.566289666 -76.612697339, -172.576243291 -76.621303431, -172.570055947 -76.630342924, -172.555183534 -76.636123529, -172.531626051 -76.638645245, -172.517040304 -76.643518276, -172.511426292 -76.650742621, -172.551848294 -76.672312544, -172.63830631 -76.708228042, -172.701431121 -76.728711408, -172.741222726 -76.733762641, -172.81460886 -76.72534004, -172.921589524 -76.703443605, -173.006960733 -76.697273314, -173.070722487 -76.706829166, -173.101615682 -76.719791531, -173.099640316 -76.736160408, -173.033958817 -76.759064999, -172.904571183 -76.788505304, -172.847033841 -76.810916113, -172.861346791 -76.826297424, -172.924787296 -76.856444925, -173.037355356 -76.901358615, -173.149640378 -76.935043659, -173.26164236 -76.957500057, -173.354942309 -76.968728255, -173.429540223 -76.968728255, -173.487771718 -76.964657535, -173.529636796 -76.956516094, -173.572768938 -76.955559014, -173.617168145 -76.961786296, -173.614655836 -76.97446809, -173.565232013 -76.993604396, -173.461502424 -77.006682128, -173.303467069 -77.013701287, -173.163373388 -77.02787859, -173.041221382 -77.049214037, -172.918094542 -77.059179951, -172.793992869 -77.057776334, -172.720418717 -77.044861043, -172.697372088 -77.020434079, -172.675885915 -77.003730799, -172.655960197 -76.994751205, -172.60882792 -76.987594764, -172.534489083 -76.982261476, -172.480072837 -76.983094424, -172.445579184 -76.990093609, -172.428332542 -76.998610734, -172.428332911 -77.008645799, -172.435068344 -77.018150822, -172.448538839 -77.027125803, -172.490777829 -77.039613708, -172.561785312 -77.055614535, -172.628175119 -77.080598263, -172.68994725 -77.114564892, -172.751818039 -77.133793765, -172.813787485 -77.138284883, -172.900229764 -77.131828165, -173.011144875 -77.114423613, -173.119679588 -77.128474884, -173.2258339 -77.17398198, -173.273849553 -77.202664633, -173.263726547 -77.214522842, -173.165895559 -77.239681117, -172.980356589 -77.278139457, -172.880291531 -77.312658914, -172.865700386 -77.343239487, -172.867667457 -77.371126102, -172.886192744 -77.39631876, -172.999732531 -77.429966955, -173.208286817 -77.472070689, -173.335454668 -77.509278677, -173.381236082 -77.541590921, -173.403703936 -77.570407724, -173.40285823 -77.595729086, -173.378288408 -77.634921, -173.329994472 -77.687983467, -173.241287742 -77.735563094, -173.112168219 -77.777659882, -173.054064387 -77.81089869, -173.066976248 -77.835279519, -173.063736051 -77.854657976, -173.044343797 -77.869034061, -172.890349983 -77.896435115, -172.60175461 -77.936861139, -172.376181212 -77.961986812, -172.213629791 -77.971812135, -172.023427102 -77.967320559, -171.805573145 -77.948512083, -171.581263004 -77.918894833, -171.350496677 -77.87846881, -171.217147208 -77.851799157, -171.181214596 -77.838885875, -171.160572341 -77.826074082, -171.155220441 -77.813363779, -171.178789134 -77.790158543, -171.231278422 -77.756458375, -171.27338337 -77.70988804, -171.305103978 -77.65044754, -171.293875473 -77.602346602, -171.239697854 -77.565585227, -171.168401509 -77.532887375, -171.079986438 -77.504253044, -171.028614514 -77.483042244, -171.014285737 -77.469254974, -171.016677114 -77.456576914, -171.035788644 -77.445008064, -171.086879845 -77.431646501, -171.169950715 -77.416492226, -171.216537864 -77.403175691, -171.226641293 -77.391696895, -171.228607057 -77.378968685, -171.222435157 -77.364991059, -171.168824693 -77.334840949, -171.067775664 -77.288518355, -171.000402018 -77.24121644, -170.966703754 -77.192935206, -170.894838531 -77.157002595, -170.784806349 -77.133418606, -170.725150821 -77.11627156, -170.715871945 -77.105561456, -170.710674146 -77.077210652, -170.709557424 -77.031219147, -170.697909144 -76.992502178, -170.675729304 -76.961059744, -170.654536164 -76.940848729, -170.634329723 -76.931869135, -170.581564681 -76.922044903, -170.496241038 -76.911376032, -170.429709562 -76.893409727, -170.381970254 -76.868145986, -170.285260999 -76.838950739, -170.139581798 -76.805823986, -170.061542334 -76.78431495, -170.051142608 -76.77442363, -170.076677284 -76.763148845, -170.138146365 -76.750490597, -170.192753568 -76.731526593, -170.240498896 -76.706256833, -170.315896371 -76.686462585, -170.418945993 -76.67214385, -170.498267121 -76.665405567, -170.553859754 -76.666247738, -170.609039198 -76.673409769, -170.663805452 -76.68689166, -170.695686968 -76.698414281, -170.704683743 -76.70797763, -170.710444514 -76.723277346, -170.71296928 -76.744313428), (-172.46185717 -77.485683162, -172.491725041 -77.49003391, -172.535448064 -77.490594163, -172.566986057 -77.488349711, -172.586339021 -77.483300552, -172.598540475 -77.476173053, -172.60359042 -77.466967216, -172.601627836 -77.458872071, -172.592652724 -77.451887618, -172.556765055 -77.448396429, -172.49396483 -77.448398503, -172.453726685 -77.452881992, -172.436050621 -77.461846897, -172.429868964 -77.468114837, -172.435181715 -77.47168581, -172.44584445 -77.477541919, -172.46185717 -77.485683162), (-172.812798475 -76.363628771, -172.855573928 -76.365453015, -172.885037626 -76.36040045, -172.90720433 -76.351027386, -172.92207404 -76.337333821, -172.9168827 -76.324750727, -172.89163031 -76.313278104, -172.862193885 -76.307261221, -172.828573425 -76.30670008, -172.792121028 -76.311189877, -172.752836694 -76.320730613, -172.732062811 -76.331770033, -172.729799379 -76.344308139, -172.756711267 -76.354927718, -172.812798475 -76.363628771), (-171.932998671 -76.183124002, -172.010021088 -76.180457336, -172.070931389 -76.166984091, -172.113033554 -76.150312062, -172.136327583 -76.130441248, -172.133522137 -76.111120124, -172.104617217 -76.092348689, -172.06028165 -76.080296327, -172.000515436 -76.074963039, -171.918725408 -76.076928027, -171.814911566 -76.086191292, -171.745182124 -76.097695899, -171.709537083 -76.111441849, -171.696346087 -76.126554541, -171.705609136 -76.143033974, -171.731004713 -76.156183802, -171.77253282 -76.166004024, -171.83986414 -76.174984091, -171.932998671 -76.183124002), (-173.16885937 -76.066345013, -173.199147981 -76.070696107, -173.23950163 -76.071257052, -173.269213382 -76.065813298, -173.288283234 -76.054364845, -173.2799961 -76.038973879, -173.244351978 -76.0196404, -173.207608446 -76.007588038, -173.169765504 -76.002816794, -173.139490241 -76.003094691, -173.116782658 -76.008421729, -173.104589039 -76.016938854, -173.102909386 -76.028646065, -173.111183172 -76.03940804, -173.129410398 -76.049224779, -173.148635798 -76.05820377, -173.16885937 -76.066345013)))" diff --git a/packages/overture-schema-buildings-theme/pyproject.toml b/packages/overture-schema-buildings-theme/pyproject.toml index 58665a55b..ec0f01d6b 100644 --- a/packages/overture-schema-buildings-theme/pyproject.toml +++ b/packages/overture-schema-buildings-theme/pyproject.toml @@ -35,8 +35,8 @@ path = "src/overture/schema/buildings/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:buildings:building" = "overture.schema.buildings:Building" -"overture:buildings:building_part" = "overture.schema.buildings:BuildingPart" +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" [[examples.Building]] id = "148f35b1-7bc1-4180-9280-10d39b13883b" diff --git a/packages/overture-schema-cli/pyproject.toml b/packages/overture-schema-cli/pyproject.toml index 590121005..ff2e5b7ef 100644 --- a/packages/overture-schema-cli/pyproject.toml +++ b/packages/overture-schema-cli/pyproject.toml @@ -7,7 +7,7 @@ dependencies = [ "overture-schema-system", "pydantic>=2.12.0", "pyyaml>=6.0.2", - "click>=8.0", + "click>=8.1", "rich>=13.0", "yamlcore>=0.0.4", ] diff --git a/packages/overture-schema-cli/src/overture/schema/cli/__init__.py b/packages/overture-schema-cli/src/overture/schema/cli/__init__.py index 85045f0c0..8fd3e8bfd 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/__init__.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/__init__.py @@ -11,7 +11,6 @@ ) from .types import ( ErrorLocation, - ModelDict, UnionType, ValidationErrorDict, ) @@ -25,7 +24,6 @@ "perform_validation", "resolve_types", "ErrorLocation", - "ModelDict", "UnionType", "ValidationErrorDict", ] diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 74c7cbae4..2590bf721 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -14,23 +14,30 @@ import yaml from pydantic import BaseModel, Field, Tag, TypeAdapter, ValidationError from rich.console import Console +from rich.text import Text from yamlcore import CoreLoader # type: ignore from overture.schema.core import OvertureFeature -from overture.schema.core.discovery import ModelKey, discover_models +from overture.schema.system.discovery import ( + ModelDict, + ModelKey, + TagSelector, + discover_models, + filter_models, +) +from overture.schema.system.discovery.tag import get_values_for_key from overture.schema.system.feature import Feature from overture.schema.system.json_schema import json_schema -from .docstrings import get_model_docstring, get_theme_module_docstring from .error_formatting import ( format_validation_error, format_validation_errors_verbose, group_errors_by_discriminator, select_most_likely_errors, ) -from .output import rewrap +from .tag_options import build_selector, tag_selection_options from .type_analysis import StructuralTuple, get_item_index, introspect_union -from .types import ErrorLocation, ModelDict, UnionType +from .types import ErrorLocation, UnionType # Console instances for rich output stdout = Console(highlight=False) @@ -190,62 +197,18 @@ def validate_features(data: list, model_type: UnionType) -> list[BaseModel]: def resolve_types( - use_overture_types: bool, - namespace: str | None, - theme_names: tuple[str, ...], - type_names: tuple[str, ...], + selector: TagSelector = TagSelector(), + *, + type_names: tuple[str, ...] = (), ) -> UnionType: - """Resolve CLI options into a model type suitable for parse_feature. - - Args - ---- - use_overture_types: Boolean from --overture-types flag - namespace: Namespace to filter by (e.g., "overture", "annex") - theme_names: List of theme names from --theme option - type_names: List of type names from --type option - - Returns - ------- - Model type suitable for passing to parse_feature - """ - # Determine effective namespace - effective_namespace = "overture" if use_overture_types else namespace - - # Discover models once with the appropriate namespace - all_models = discover_models(namespace=effective_namespace) - - # Filter models based on CLI options - filtered_models: ModelDict = {} - - if use_overture_types: - filtered_models = all_models - - elif theme_names and not type_names: - # Theme-only mode: all types in specified themes - for key, model_class in all_models.items(): - if key.theme in theme_names: - filtered_models[key] = model_class - - elif type_names and not theme_names: - # Type-only mode: find matching types across all themes - for key, model_class in all_models.items(): - if key.type in type_names: - filtered_models[key] = model_class - - elif type_names and theme_names: - # Both specified: find matching types within specified themes - for key, model_class in all_models.items(): - if key.theme in theme_names and key.type in type_names: - filtered_models[key] = model_class - - else: - # No filters specified - use all models - filtered_models = all_models + """Resolve a TagSelector + type-names into a Pydantic union type.""" + models = discover_models() + models = filter_models(models, selector, type_names=type_names) - if not filtered_models: + if not models: raise ValueError("No models found matching the specified criteria") - return create_union_type_from_models(filtered_models) + return create_union_type_from_models(models) def get_source_name(filename: Path) -> str: @@ -281,10 +244,10 @@ def cli() -> None: $ overture-schema list-types \b # Generate JSON schema - $ overture-schema json-schema --theme buildings + $ overture-schema json-schema --tag overture:theme=buildings \b # Validate specific types - $ overture-schema validate --theme buildings data.json + $ overture-schema validate --tag overture:theme=buildings data.json """ pass @@ -523,11 +486,12 @@ def handle_validation_error( # Show heterogeneity warning if collection has mixed types if is_heterogeneous: stderr.print( - " ⚠ Heterogeneous collection: Data contains multiple feature types.", + " ⚠ Heterogeneous collection: Data contains multiple feature types. Consider:", style="yellow", ) stderr.print( - " • Consider validating each type separately with --theme or --type", + " • Validating each type separately with --tag, --filter, " + "--exclude, or --type", style="dim", ) stderr.print() @@ -548,7 +512,7 @@ def handle_validation_error( style="yellow", ) stderr.print( - " • Specifying --theme or --type to narrow validation", style="dim" + " • Specifying --tag or --type to narrow validation", style="dim" ) stderr.print(" • Adding discriminator fields to clarify intent", style="dim") stderr.print() @@ -627,20 +591,7 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: @cli.command() @click.argument("filename", type=click.Path(path_type=Path), required=True) -@click.option( - "--overture-types", - is_flag=True, - help="Validate against all official Overture types (excludes extensions)", -) -@click.option( - "--namespace", - help="Namespace to filter by (e.g., overture, annex)", -) -@click.option( - "--theme", - multiple=True, - help="Theme to validate against (shorthand for all types in theme)", -) +@tag_selection_options @click.option( "--type", "types", @@ -655,9 +606,9 @@ def handle_generic_error(e: Exception, filename: Path, error_type: str) -> None: ) def validate( filename: Path, - overture_types: bool, - namespace: str | None, - theme: tuple[str, ...], + tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], types: tuple[str, ...], show_fields: tuple[str, ...], ) -> None: @@ -675,17 +626,19 @@ def validate( $ overture-schema validate - < data.json \b # Validate only buildings - $ overture-schema validate --theme buildings data.json + $ overture-schema validate --tag overture:theme=buildings data.json \b # Validate specific type $ overture-schema validate --type building data.json \b # Official Overture types only - $ overture-schema validate --overture-types data.json + $ overture-schema validate --tag overture --tag feature data.json """ # Resolve model type first (errors here are ValueErrors, not ValidationErrors) try: - model_type = resolve_types(overture_types, namespace, theme, types) + model_type = resolve_types( + build_selector(tags, filters, excludes), type_names=types + ) except ValueError as e: handle_generic_error(e, filename, "value") return @@ -712,20 +665,7 @@ def validate( @cli.command("json-schema") -@click.option( - "--overture-types", - is_flag=True, - help="Generate schema for all official Overture types (excludes extensions)", -) -@click.option( - "--namespace", - help="Namespace to filter by (e.g., overture, annex)", -) -@click.option( - "--theme", - multiple=True, - help="Theme to generate schema for (shorthand for all types in theme)", -) +@tag_selection_options @click.option( "--type", "types", @@ -733,9 +673,9 @@ def validate( help="Specific type to generate schema for (e.g., building, segment)", ) def json_schema_command( - overture_types: bool, - namespace: str | None, - theme: tuple[str, ...], + tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], types: tuple[str, ...], ) -> None: r"""Generate JSON schema for Overture Maps types. @@ -748,17 +688,19 @@ def json_schema_command( # All types $ overture-schema json-schema > schema.json \b - # Buildings theme - $ overture-schema json-schema --theme buildings + # Buildings theme by tag + $ overture-schema json-schema --tag overture:theme=buildings \b # Specific types $ overture-schema json-schema --type building \b # Official Overture types only - $ overture-schema json-schema --overture-types + $ overture-schema json-schema --tag overture --tag feature """ try: - model_type = resolve_types(overture_types, namespace, theme, types) + model_type = resolve_types( + build_selector(tags, filters, excludes), type_names=types + ) schema = json_schema(model_type) # Use plain print for JSON output to avoid Rich formatting print(json.dumps(schema, indent=2, sort_keys=True)) @@ -766,53 +708,23 @@ def json_schema_command( raise click.UsageError(str(e)) from e -def dump_namespace( - theme_types: dict[str | None, list[tuple[ModelKey, type[BaseModel]]]], -) -> None: - """Print all themes and types for a namespace. - - Displays themes in alphabetical order with their types and docstrings. - Each type includes its model class name and description. - - Args - ---- - theme_types : dict[str | None, list[tuple[ModelKey, type[BaseModel]]]] - Dict mapping theme name to list of (ModelKey, model_class) tuples - """ - for theme in sorted(theme_types.keys(), key=lambda x: (x is None, x)): - if theme: - stdout.print( - f"[bold green underline]{theme.upper()}[/bold green underline]" - ) - - theme_docstring = get_theme_module_docstring(theme) - if theme_docstring: - stdout.print( - rewrap(theme_docstring, stdout, padding_right=4), style="dim" - ) - - stdout.print() - - # Add types to the tree - sorted_types = sorted(theme_types[theme], key=lambda x: x[0].type) - for key, model_class in sorted_types: - stdout.print( - f" [bright_black]→[/bright_black] [bold cyan]{key.type}[/bold cyan] [dim magenta]({key.entry_point})[/dim magenta]" - ) - docstring = get_model_docstring(model_class) - if docstring: - stdout.print( - rewrap(docstring, stdout, indent=4, padding_right=12), style="dim" - ) - stdout.print() - - @cli.command("list-types") -def list_types() -> None: - r"""List all available types grouped by theme with descriptions. +@tag_selection_options +@click.option( + "--group-by", + help="Group types by a key/value tag's key (e.g. 'overture:theme'). " + "Plain and namespaced tags have no value to group by and are " + "ignored here.", +) +def list_types( + tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], + group_by: str | None, +) -> None: + r"""List all available types. - Displays all registered Overture Maps types organized by theme, - including model class names and docstrings. + Displays all registered models and can be organized by grouping. \b Examples: @@ -821,35 +733,46 @@ def list_types() -> None: """ try: models = discover_models() + models = filter_models(models, build_selector(tags, filters, excludes)) - # Group models by namespace and theme - namespaces: dict[ - str, dict[str | None, list[tuple[ModelKey, type[BaseModel]]]] - ] = {} - for key, model_class in models.items(): - if key.namespace not in namespaces: - namespaces[key.namespace] = {} - if key.theme not in namespaces[key.namespace]: - namespaces[key.namespace][key.theme] = [] + if group_by: + grouped_models: dict[str, set[ModelKey]] = {} - namespaces[key.namespace][key.theme].append((key, model_class)) + for key in models.keys(): + if groups := get_values_for_key(key.tags, group_by): + for group in groups: + grouped_models.setdefault(group, set()).add(key) - # display Overture themes first - if "overture" in namespaces: - stdout.print("[bold red]OVERTURE THEMES[/bold red]", justify="center") - stdout.print() - - dump_namespace(namespaces["overture"]) - - stdout.print("[bold red]ADDITIONAL TYPES[/bold red]", justify="center") - stdout.print() - - for namespace in sorted(namespaces.keys()): - if namespace == "overture": - continue + padding = ( + max( + (len(key.name) for keys in grouped_models.values() for key in keys), + default=0, + ) + + 2 + ) - stdout.print(f"[bold blue]{namespace.upper()}[/bold blue]") - dump_namespace(namespaces[namespace]) + for group, keys in sorted(grouped_models.items()): + stdout.print( + f"[green bold]{group_by}={group} ({len(keys)})[/green bold]" + ) + for key in sorted(keys, key=lambda k: k.name): + model = Text() + model.append("→ ", style="bright_black") + model.append(key.name, style="bold cyan") + model.pad_right(max(1, padding - len(key.name))) + model.append(" ".join(sorted(key.tags))) + stdout.print(model) + stdout.print() + + else: + padding = max((len(key.name) for key in models.keys()), default=0) + 2 + + for key in sorted(models.keys(), key=lambda k: k.name): + model = Text() + model.append(key.name, style="bold cyan") + model.pad_right(max(1, padding - len(key.name))) + model.append(" ".join(sorted(key.tags))) + stdout.print(model) except Exception as e: click.echo(f"Error listing types: {e}", err=True) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py new file mode 100644 index 000000000..3befb1143 --- /dev/null +++ b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py @@ -0,0 +1,66 @@ +"""Shared Click options for tag-based model selection.""" + +from collections.abc import Callable +from typing import TypeVar + +import click + +from overture.schema.system.discovery import TagSelector + +F = TypeVar("F", bound=Callable[..., object]) + +_TAG_SYNTAX_NOTE = ( + "Accepts plain tags (e.g. feature), namespaced tags " + "(e.g. overture:approved), or compound key/value tags " + "(e.g. overture:theme=buildings)." +) + + +def tag_selection_options(func: F) -> F: + """Decorate a Click command with --tag, --filter, and --exclude options. + + The decorated command receives `tags`, `filters`, and `excludes` + keyword arguments (each a `tuple[str, ...]`), suitable for passing to + `build_selector`. + """ + func = click.option( + "--exclude", + "excludes", + multiple=True, + help=( + "Exclude feature types with these tags — removes from scope (OR-NOT; " + f"repeatable). {_TAG_SYNTAX_NOTE}" + ), + )(func) + func = click.option( + "--filter", + "filters", + multiple=True, + help=( + "Require feature types to have these tags — narrows scope (AND; " + f"repeatable). {_TAG_SYNTAX_NOTE}" + ), + )(func) + func = click.option( + "--tag", + "tags", + multiple=True, + help=( + "Include feature types with these tags — defines scope (OR; repeatable). " + f"{_TAG_SYNTAX_NOTE}" + ), + )(func) + return func + + +def build_selector( + tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], +) -> TagSelector: + """Map `tag_selection_options` arguments to a `TagSelector`.""" + return TagSelector( + include_any=tags, + require_all=filters, + exclude_any=excludes, + ) diff --git a/packages/overture-schema-cli/src/overture/schema/cli/types.py b/packages/overture-schema-cli/src/overture/schema/cli/types.py index 1b5d4e44d..f1394def8 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/types.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/types.py @@ -5,15 +5,10 @@ from pydantic import BaseModel from pydantic_core import ErrorDetails -from overture.schema.core.discovery import ModelKey - # Type alias for union types created from Pydantic models # This represents either a single model or a discriminated union of models UnionType: TypeAlias = type[BaseModel] | Any -# Dictionary mapping ModelKey to Pydantic model classes -ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] - # Pydantic validation error dictionary structure # In Pydantic v2, ValidationError.errors() returns list[ErrorDetails] ValidationErrorDict: TypeAlias = ErrorDetails diff --git a/packages/overture-schema-cli/tests/test_cli_commands.py b/packages/overture-schema-cli/tests/test_cli_commands.py index 7b6f3b42f..3abc8d3a0 100644 --- a/packages/overture-schema-cli/tests/test_cli_commands.py +++ b/packages/overture-schema-cli/tests/test_cli_commands.py @@ -16,8 +16,8 @@ def test_list_types_command(self, cli_runner: CliRunner) -> None: """Test the list-types command.""" result = cli_runner.invoke(cli, ["list-types"]) assert result.exit_code == 0 - # Should show theme names - assert "BUILDINGS" in result.output or "buildings" in result.output + # Should show theme tag + assert "overture:theme=buildings" in result.output # Should show type names assert "building" in result.output @@ -33,7 +33,9 @@ class TestJsonSchemaCommand: def test_json_schema_generates_valid_output(self, cli_runner: CliRunner) -> None: """Test that json-schema command generates valid JSON.""" - result = cli_runner.invoke(cli, ["json-schema", "--theme", "buildings"]) + result = cli_runner.invoke( + cli, ["json-schema", "--tag", "overture:theme=buildings"] + ) assert result.exit_code == 0 # Should be valid JSON @@ -57,7 +59,7 @@ def test_validate_flat_format_input(self, cli_runner: CliRunner) -> None: flat_feature = build_feature(geojson_format=False) flat_json = json.dumps(flat_feature) result = cli_runner.invoke( - cli, ["validate", "--theme", "buildings", "-"], input=flat_json + cli, ["validate", "--tag", "overture:theme=buildings", "-"], input=flat_json ) assert result.exit_code == 0 assert "Successfully validated " in result.output @@ -222,7 +224,7 @@ def test_validate_with_nonexistent_filters_raises_error( # Try to validate with a nonexistent theme result = cli_runner.invoke( cli, - ["validate", "--theme", "nonexistent_theme", "-"], + ["validate", "--tag", "overture:theme=nonexistent_theme", "-"], input=building_feature_yaml_content, ) # UsageError exits with code 2 @@ -254,7 +256,7 @@ def test_validate_with_valid_theme_invalid_type_raises_error( # Try to validate buildings theme with a type that doesn't exist in that theme result = cli_runner.invoke( cli, - ["validate", "--theme", "buildings", "--type", "segment", "-"], + ["validate", "--tag", "overture:theme=buildings", "--type", "segment", "-"], input=building_feature_yaml_content, ) # UsageError exits with code 2 @@ -366,3 +368,46 @@ def test_show_field_in_collection( # Should show id for both features assert "id=first" in stderr_output or "first" in stderr_output assert "id=second" in stderr_output or "second" in stderr_output + + +_TAG_COMBINATOR_FLAGS = ( + pytest.param("--filter", "feature", id="filter"), + pytest.param("--exclude", "overture:theme=places", id="exclude"), +) + + +@pytest.mark.parametrize(("flag", "value"), _TAG_COMBINATOR_FLAGS) +def test_validate_wires_tag_combinator_flag( + cli_runner: CliRunner, + building_feature_yaml: str, + flag: str, + value: str, +) -> None: + """validate wires --filter/--exclude through to the tag selector.""" + result = cli_runner.invoke( + cli, ["validate", building_feature_yaml, "--tag", "feature", flag, value] + ) + assert result.exit_code == 0 + + +@pytest.mark.parametrize(("flag", "value"), _TAG_COMBINATOR_FLAGS) +def test_json_schema_wires_tag_combinator_flag( + cli_runner: CliRunner, + flag: str, + value: str, +) -> None: + """json-schema wires --filter/--exclude through to the tag selector.""" + result = cli_runner.invoke(cli, ["json-schema", "--tag", "feature", flag, value]) + assert result.exit_code == 0 + assert result.output # non-empty JSON + + +@pytest.mark.parametrize(("flag", "value"), _TAG_COMBINATOR_FLAGS) +def test_list_types_wires_tag_combinator_flag( + cli_runner: CliRunner, + flag: str, + value: str, +) -> None: + """list-types wires --filter/--exclude through to the tag selector.""" + result = cli_runner.invoke(cli, ["list-types", "--tag", "feature", flag, value]) + assert result.exit_code == 0 diff --git a/packages/overture-schema-cli/tests/test_cli_functions.py b/packages/overture-schema-cli/tests/test_cli_functions.py index 4218541f5..5a3139720 100644 --- a/packages/overture-schema-cli/tests/test_cli_functions.py +++ b/packages/overture-schema-cli/tests/test_cli_functions.py @@ -9,6 +9,7 @@ from click.exceptions import UsageError from conftest import build_feature from overture.schema.cli.commands import load_input, perform_validation, resolve_types +from overture.schema.system.discovery import TagSelector from pydantic import ValidationError @@ -203,7 +204,9 @@ class TestPerformValidation: def test_perform_validation_raises_for_invalid_single_feature(self) -> None: """Test that perform_validation raises ValidationError for single invalid feature.""" data = build_feature(id=None) # Missing required 'id' - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -218,7 +221,9 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: id=None, coordinates=[[[2, 2], [3, 2], [3, 3], [2, 3], [2, 2]]] ) data = [feature1, feature2] - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) with pytest.raises(ValidationError) as exc_info: perform_validation(data, model_type) @@ -230,7 +235,9 @@ def test_perform_validation_raises_for_invalid_list_item(self) -> None: def test_perform_validation_empty_list(self) -> None: """Test validating an empty list (edge case).""" data: list[dict[str, object]] = [] - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) # Should not raise perform_validation(data, model_type) @@ -238,7 +245,9 @@ def test_perform_validation_empty_list(self) -> None: def test_perform_validation_empty_feature_collection(self) -> None: """Test validating an empty FeatureCollection (edge case).""" data = {"type": "FeatureCollection", "features": []} - model_type = resolve_types(False, None, ("buildings",), ()) + model_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) # Should not raise perform_validation(data, model_type) @@ -248,10 +257,12 @@ def test_perform_validation_with_different_themes(self) -> None: data = build_feature(theme="buildings", type="building") # Should work with buildings theme - buildings_type = resolve_types(False, None, ("buildings",), ()) + buildings_type = resolve_types( + TagSelector(include_any=("overture:theme=buildings",)) + ) perform_validation(data, buildings_type) # Should fail with wrong theme - places_type = resolve_types(False, None, ("places",), ()) + places_type = resolve_types(TagSelector(include_any=("overture:theme=places",))) with pytest.raises(ValidationError): perform_validation(data, places_type) diff --git a/packages/overture-schema-cli/tests/test_error_formatting.py b/packages/overture-schema-cli/tests/test_error_formatting.py index fa8ef1a5d..168d9d53c 100644 --- a/packages/overture-schema-cli/tests/test_error_formatting.py +++ b/packages/overture-schema-cli/tests/test_error_formatting.py @@ -40,7 +40,9 @@ def test_ambiguous_data_shows_most_likely_errors( version: 0 """) - result = cli_runner.invoke(cli, ["validate", "--theme", "buildings", filename]) + result = cli_runner.invoke( + cli, ["validate", "--tag", "overture:theme=buildings", filename] + ) assert result.exit_code == 1 diff --git a/packages/overture-schema-cli/tests/test_resolve_types.py b/packages/overture-schema-cli/tests/test_resolve_types.py index 94231a1fe..55acfd762 100644 --- a/packages/overture-schema-cli/tests/test_resolve_types.py +++ b/packages/overture-schema-cli/tests/test_resolve_types.py @@ -1,168 +1,81 @@ -"""Parametrized tests for resolve_types function.""" +"""Tests for resolve_types — CLI glue between filter_models and union creation. + +The combinator algebra of filter_models itself is covered in +`test_discovery_filter_models.py` in the system package. +""" + +from collections.abc import Iterator +from typing import get_args +from unittest.mock import patch import pytest from overture.schema.cli.commands import resolve_types -from overture.schema.core.discovery import discover_models - - -class TestResolveTypes: - """Tests for the resolve_types function with various filter combinations.""" - - @pytest.mark.parametrize( - "overture_types,namespace,theme_names,type_names,should_succeed", - [ - # Test --overture-types flag - pytest.param(True, None, (), (), True, id="overture_types_only"), - pytest.param(False, "overture", (), (), True, id="overture_namespace"), - # Test theme filtering - pytest.param(False, None, ("buildings",), (), True, id="theme_buildings"), - pytest.param( - False, None, ("transportation",), (), True, id="theme_transportation" - ), - pytest.param( - False, None, ("buildings", "places"), (), True, id="multiple_themes" - ), - pytest.param(False, None, ("nonexistent",), (), False, id="invalid_theme"), - # Test type filtering - pytest.param(False, None, (), ("building",), True, id="type_building"), - pytest.param(False, None, (), ("segment",), True, id="type_segment"), - pytest.param( - False, None, (), ("building", "place"), True, id="multiple_types" - ), - pytest.param(False, None, (), ("nonexistent",), False, id="invalid_type"), - # Test combined theme + type filtering - pytest.param( - False, - None, - ("buildings",), - ("building",), - True, - id="theme_and_type_match", - ), - pytest.param( - False, - None, - ("buildings",), - ("segment",), - False, - id="theme_and_type_mismatch", - ), - pytest.param( - False, - None, - ("transportation",), - ("segment", "connector"), - True, - id="theme_with_multiple_types", - ), - # Test namespace combined with theme/type - pytest.param( - False, - "overture", - ("buildings",), - (), - True, - id="namespace_with_theme", - ), - pytest.param( - False, - "overture", - (), - ("building",), - True, - id="namespace_with_type", - ), - pytest.param( - False, - "overture", - ("buildings",), - ("building",), - True, - id="namespace_with_theme_and_type", - ), - # Test no filters (all models) - pytest.param(False, None, (), (), True, id="no_filters_all_models"), - ], - ) - def test_resolve_types_combinations( - self, - overture_types: bool, - namespace: str | None, - theme_names: tuple[str, ...], - type_names: tuple[str, ...], - should_succeed: bool, - ) -> None: - """Test resolve_types with various filter combinations.""" - if should_succeed: - model_type = resolve_types( - overture_types, namespace, theme_names, type_names - ) - assert model_type is not None - else: - with pytest.raises(ValueError, match="No models found"): - resolve_types(overture_types, namespace, theme_names, type_names) - - @pytest.mark.parametrize( - "namespace,expected_themes", - [ - pytest.param( - "overture", - { - "buildings", - "places", - "transportation", - "base", - "divisions", - "addresses", - }, - id="overture_namespace", - ), - ], - ) - def test_resolve_types_returns_expected_themes( - self, - namespace: str, - expected_themes: set[str], - ) -> None: - """Test that resolve_types returns models from expected themes.""" - models = discover_models(namespace=namespace) - actual_themes = {key.theme for key in models.keys()} - - # Check that we have at least the expected themes (may have more) - assert expected_themes.issubset(actual_themes), ( - f"Missing expected themes. Expected {expected_themes}, got {actual_themes}" - ) - - -class TestResolveTypesEdgeCases: - """Tests for edge cases in resolve_types.""" - - def test_resolve_types_case_sensitive(self) -> None: - """Test that theme and type names are case-sensitive.""" - # Lowercase should work - model_type = resolve_types(False, None, ("buildings",), ()) - assert model_type is not None - - # Uppercase should fail (themes are lowercase in registry) - with pytest.raises(ValueError, match="No models found"): - resolve_types(False, None, ("BUILDINGS",), ()) - - def test_resolve_types_empty_result_error_message(self) -> None: - """Test that a helpful error message is shown when no models match.""" - with pytest.raises(ValueError) as exc_info: - resolve_types(False, None, ("nonexistent",), ("also_fake",)) - - assert "No models found" in str(exc_info.value) - - def test_resolve_types_namespace_isolation(self) -> None: - """Test that namespace filtering properly isolates models.""" - # Get all models (no namespace filter) - all_models_type = resolve_types(False, None, (), ()) - assert all_models_type is not None - - # Get only overture namespace - overture_type = resolve_types(False, "overture", (), ()) - assert overture_type is not None - - # Both should work, but they represent different sets of models - # (This test primarily ensures no exceptions are raised) +from overture.schema.system.discovery import ModelKey, TagSelector + +DISCOVER_MODELS = "overture.schema.cli.commands.discover_models" + + +# Mock model classes +class Place: + pass + + +class Segment: + pass + + +class Building: + pass + + +BUILDING_KEY = ModelKey( + name="building", + entry_point="mock:Building", + tags=frozenset({"feature", "overture", "overture:theme=buildings"}), +) +SEGMENT_KEY = ModelKey( + name="segment", + entry_point="mock:Segment", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +PLACE_KEY = ModelKey( + name="place", + entry_point="mock:Place", + tags=frozenset({"feature", "overture", "overture:theme=places"}), +) + +MOCK_MODELS = { + BUILDING_KEY: Building, + SEGMENT_KEY: Segment, + PLACE_KEY: Place, +} + + +@pytest.fixture(autouse=True) +def _patched_discover_models() -> Iterator[None]: + with patch(DISCOVER_MODELS, return_value=MOCK_MODELS): + yield + + +def test_no_filters_returns_union_of_all() -> None: + union = resolve_types(TagSelector()) + assert set(get_args(union)) == {Building, Segment, Place} + + +def test_returns_type_when_filter_matches() -> None: + # Single match collapses to the bare class; multi-match yields a Union. + union = resolve_types(TagSelector(include_any=("overture:theme=transportation",))) + assert union is Segment + + +def test_empty_match_raises_value_error() -> None: + with pytest.raises(ValueError, match="No models found"): + resolve_types(TagSelector(include_any=("nonexistent",))) + + +def test_type_names_are_case_sensitive() -> None: + # Lowercase matches. + assert resolve_types(TagSelector(), type_names=("building",)) is Building + # Uppercase doesn't. + with pytest.raises(ValueError, match="No models found"): + resolve_types(TagSelector(), type_names=("BUILDING",)) diff --git a/packages/overture-schema-codegen/README.md b/packages/overture-schema-codegen/README.md index 92a4d8fbe..e9f5b0fba 100644 --- a/packages/overture-schema-codegen/README.md +++ b/packages/overture-schema-codegen/README.md @@ -22,7 +22,7 @@ renderers, not extraction logic. overture-codegen generate --format markdown --output-dir docs/schema/reference # Generate for a single theme -overture-codegen generate --format markdown --theme buildings --output-dir out/ +overture-codegen generate --format markdown --tag overture:theme=buildings --output-dir out/ # List discovered models overture-codegen list diff --git a/packages/overture-schema-codegen/pyproject.toml b/packages/overture-schema-codegen/pyproject.toml index de42c5fb9..e6f55e127 100644 --- a/packages/overture-schema-codegen/pyproject.toml +++ b/packages/overture-schema-codegen/pyproject.toml @@ -4,8 +4,9 @@ requires = ["hatchling"] [project] dependencies = [ - "click>=8.0", + "click>=8.1", "jinja2>=3.0", + "overture-schema-cli", "overture-schema-core", "overture-schema-system", "tomli>=2.0; python_version < '3.11'", @@ -19,6 +20,7 @@ name = "overture-schema-codegen" overture-codegen = "overture.schema.codegen.cli:main" [tool.uv.sources] +overture-schema-cli = { workspace = true } overture-schema-core = { workspace = true } overture-schema-system = { workspace = true } diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index 0a24c7348..279f22a84 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -6,7 +6,11 @@ import click -from overture.schema.core.discovery import discover_models +from overture.schema.cli.tag_options import build_selector, tag_selection_options +from overture.schema.system.discovery import ( + discover_models, + filter_models, +) from .extraction.model_extraction import extract_model from .extraction.specs import ( @@ -75,11 +79,7 @@ def list_models() -> None: type=click.Choice(_OUTPUT_FORMATS), help="Output format", ) -@click.option( - "--theme", - multiple=True, - help="Filter to specific theme(s); repeatable (e.g., --theme buildings --theme places)", -) +@tag_selection_options @click.option( "--output-dir", type=click.Path(path_type=Path), @@ -88,21 +88,19 @@ def list_models() -> None: ) def generate( output_format: str, - theme: tuple[str, ...], + tags: tuple[str, ...], + filters: tuple[str, ...], + excludes: tuple[str, ...], output_dir: Path | None, ) -> None: """Generate code/docs from discovered models.""" all_models = discover_models() - # Schema root from ALL entry points (before theme filter). + # Schema root from ALL entry points (before tag filters). module_paths = [entry_point_module(k.entry_point) for k in all_models] schema_root = compute_schema_root(module_paths) - models = ( - {k: v for k, v in all_models.items() if k.theme in theme} - if theme - else all_models - ) + models = filter_models(all_models, build_selector(tags, filters, excludes)) if output_dir: output_dir.mkdir(parents=True, exist_ok=True) diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 64facf5a9..4c9be9c9b 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -25,7 +25,12 @@ is_model_class, ) from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) +from overture.schema.system.discovery.tag import get_values_for_key from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import require_any_of @@ -301,6 +306,11 @@ def find_member(spec: EnumSpec, name: str) -> EnumMemberSpec: return next(m for m in spec.members if m.name == name) +def find_theme(tags: frozenset[str]) -> str | None: + """Extract the theme from a set of tags, if present.""" + return next(iter(get_values_for_key(tags, "overture:theme")), None) + + T = TypeVar("T") @@ -332,7 +342,9 @@ def flat_specs_from_discovery( """Build a flat list of ModelSpecs from discovery, with entry_point set.""" models = discover_models() if theme: - models = {k: v for k, v in models.items() if k.theme == theme} + models = filter_models( + models, TagSelector(include_any=(f"overture:theme={theme}",)) + ) result = [] for key, cls in models.items(): if not is_model_class(cls): diff --git a/packages/overture-schema-codegen/tests/conftest.py b/packages/overture-schema-codegen/tests/conftest.py index 8dce88bf5..d66cf72a3 100644 --- a/packages/overture-schema-codegen/tests/conftest.py +++ b/packages/overture-schema-codegen/tests/conftest.py @@ -14,7 +14,7 @@ render_geometry_from_values, render_primitives_from_specs, ) -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.system.primitive import GeometryType from pydantic import BaseModel diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index eecd45627..13957606e 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -48,16 +48,55 @@ def test_generate_markdown_to_stdout(self, cli_runner: CliRunner) -> None: assert result.exit_code == 0 assert "# Building" in result.output or "# " in result.output - def test_generate_with_theme_filter(self, cli_runner: CliRunner) -> None: - """generate --theme should filter to specific theme.""" + def test_generate_with_tag_filter(self, cli_runner: CliRunner) -> None: + """generate --tag should filter to specific theme.""" result = cli_runner.invoke( - cli, ["generate", "--format", "markdown", "--theme", "buildings"] + cli, + ["generate", "--format", "markdown", "--tag", "overture:theme=buildings"], ) assert result.exit_code == 0 assert "Building" in result.output assert "Place" not in result.output + def test_generate_accepts_filter_flag(self, cli_runner: CliRunner) -> None: + """generate accepts --filter without error. + + Selector algebra is covered in test_discovery_filter_models. + """ + result = cli_runner.invoke( + cli, + [ + "generate", + "--format", + "markdown", + "--tag", + "feature", + "--filter", + "feature", + ], + ) + assert result.exit_code == 0 + + def test_generate_accepts_exclude_flag(self, cli_runner: CliRunner) -> None: + """generate accepts --exclude without error. + + Selector algebra is covered in test_discovery_filter_models. + """ + result = cli_runner.invoke( + cli, + [ + "generate", + "--format", + "markdown", + "--tag", + "feature", + "--exclude", + "overture:theme=places", + ], + ) + assert result.exit_code == 0 + def test_generate_markdown_feature_at_theme_level( self, cli_runner: CliRunner, tmp_path: Path ) -> None: @@ -68,8 +107,8 @@ def test_generate_markdown_feature_at_theme_level( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -93,8 +132,8 @@ def test_feature_pages_have_sidebar_position( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -211,8 +250,8 @@ def test_generates_category_files( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -311,8 +350,8 @@ def test_generate_markdown_includes_enum_files( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], @@ -344,7 +383,8 @@ def spy(feature_specs: list, schema_root: str, output_dir: object) -> None: monkeypatch.setattr("overture.schema.codegen.cli._generate_markdown", spy) result = cli_runner.invoke( - cli, ["generate", "--format", "markdown", "--theme", "buildings"] + cli, + ["generate", "--format", "markdown", "--tag", "overture:theme=buildings"], ) assert result.exit_code == 0 @@ -381,8 +421,8 @@ def test_segment_appears_in_markdown_output( "generate", "--format", "markdown", - "--theme", - "transportation", + "--tag", + "overture:theme=transportation", "--output-dir", str(tmp_path), ], @@ -411,8 +451,8 @@ def test_used_by_sections_appear_in_markdown( "generate", "--format", "markdown", - "--theme", - "buildings", + "--tag", + "overture:theme=buildings", "--output-dir", str(tmp_path), ], diff --git a/packages/overture-schema-codegen/tests/test_integration_real_models.py b/packages/overture-schema-codegen/tests/test_integration_real_models.py index b4dd9419f..9ed20d112 100644 --- a/packages/overture-schema-codegen/tests/test_integration_real_models.py +++ b/packages/overture-schema-codegen/tests/test_integration_real_models.py @@ -20,7 +20,7 @@ from overture.schema.codegen.layout.module_layout import entry_point_class from overture.schema.codegen.markdown.pipeline import generate_markdown_pages from overture.schema.codegen.markdown.renderer import render_feature -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.transportation import Segment from overture.schema.transportation.segment.models import RoadSegment from pydantic import BaseModel diff --git a/packages/overture-schema-core/README.md b/packages/overture-schema-core/README.md index 58f5d3ee9..cef61bc9a 100644 --- a/packages/overture-schema-core/README.md +++ b/packages/overture-schema-core/README.md @@ -94,4 +94,4 @@ Rendering hints for map-making: `prominence` (1--100 significance scale), `min_z - **Types** -- domain-specific aliases built on system primitives: `ConfidenceScore` (0.0--1.0), `Level` (z-order), `FeatureVersion`. - **Units** -- measurement enumerations: `SpeedUnit`, `LengthUnit`, `WeightUnit`. -- **Discovery** -- entry-point-based model registry. Theme packages register models via `overture.models` entry points; `discover_models()` resolves them at runtime. +- **Tag providers** -- `theme` provider for the discovery system in `overture-schema-system`. Tags `OvertureFeature`-derived models with `overture:theme={theme}`. diff --git a/packages/overture-schema-core/pyproject.toml b/packages/overture-schema-core/pyproject.toml index 010441cb1..d7d93d7c5 100644 --- a/packages/overture-schema-core/pyproject.toml +++ b/packages/overture-schema-core/pyproject.toml @@ -36,3 +36,6 @@ dev = [ "types-pyyaml>=6.0.12.20250516", "types-shapely>=2.1.0.20250710", ] + +[project.entry-points."overture.tag_providers"] +theme = "overture.schema.core.tag_providers:theme_provider" diff --git a/packages/overture-schema-core/src/overture/schema/core/discovery.py b/packages/overture-schema-core/src/overture/schema/core/discovery.py deleted file mode 100644 index b9290d29a..000000000 --- a/packages/overture-schema-core/src/overture/schema/core/discovery.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Model discovery system for Overture schema registry.""" - -import importlib.metadata -import logging -from dataclasses import dataclass - -from pydantic import BaseModel - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True, slots=True) -class ModelKey: - """Key identifying a registered model by namespace, theme, and type. - - Attributes - ---------- - namespace : str - The namespace (e.g., "overture", "annex") - theme : str | None - The theme name (e.g., "buildings", "places"), or None for non-themed models - type : str - The feature type (e.g., "building", "place") - entry_point : str - The entry point value in "module:Class" format - - """ - - namespace: str - theme: str | None - type: str - entry_point: str - - -def discover_models( - namespace: str | None = None, -) -> dict[ModelKey, type[BaseModel]]: - """Discover all registered Overture models via entry points. - - Parameters - ---------- - namespace : str | None, optional - Optional namespace filter. If provided, only models from this - namespace will be returned (e.g., "overture", "annex"). - - Returns - ------- - dict[ModelKey, type[BaseModel]] - Dict mapping ModelKey to model classes. - Theme will be None for entries without an explicit theme component. - - Notes - ----- - Entry point name format: - - Core themes: "overture::" - - Non-core (2-part): "annex:" (theme will be None) - - Non-core (3-part): "annex::" - - """ - models = {} - try: - for entry_point in importlib.metadata.entry_points(group="overture.models"): - # Parse namespace:theme:type or namespace:type from entry point name - parts = entry_point.name.split(":", 2) - - if len(parts) == 2: - # namespace:type format (no theme) - ns, feature_type = parts - theme = None - elif len(parts) == 3: - # namespace:theme:type format - ns, theme, feature_type = parts - else: - logger.warning( - "Invalid entry point format %s, expected namespace:theme:type or namespace:type", - entry_point.name, - ) - continue - - # Filter by namespace if specified - if namespace is not None and ns != namespace: - continue - - try: - model_class = entry_point.load() - key = ModelKey( - namespace=ns, - theme=theme, - type=feature_type, - entry_point=entry_point.value, - ) - models[key] = model_class - except Exception as e: - # Log warning but don't fail for individual models - logger.warning("Could not load model %s: %s", entry_point.name, e) - except Exception as e: - logger.warning("Could not discover entry points: %s", e) - - return models - - -def get_registered_model( - namespace: str, feature_type: str, theme: str | None = None -) -> type[BaseModel] | None: - """Get the Pydantic model for a namespace/theme/type combination. - - This uses setuptools entry points for registration. - - Parameters - ---------- - namespace : str - The namespace (e.g., "overture", "annex") - feature_type : str - The type name - theme : str | None, optional - The theme name (optional) - - Returns - ------- - type[BaseModel] | None - The model class if found, None otherwise. - - """ - # Check all discovered models for a match - models = discover_models(namespace=namespace) - # Need to find by namespace/theme/type, not exact key match - for key, model_class in models.items(): - if ( - key.namespace == namespace - and key.theme == theme - and key.type == feature_type - ): - return model_class - return None diff --git a/packages/overture-schema-core/src/overture/schema/core/tag_providers.py b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py new file mode 100644 index 000000000..0c4b69b9b --- /dev/null +++ b/packages/overture-schema-core/src/overture/schema/core/tag_providers.py @@ -0,0 +1,81 @@ +"""Tag providers for the core Overture schema package. + +Each provider inspects a discovered model and returns the set of tags +that should be attached. Registered via the +`overture.tag_providers` entry-point group. +""" + +from collections.abc import Iterable +from typing import Literal, get_args, get_origin + +from pydantic import BaseModel + +from overture.schema.core import OvertureFeature +from overture.schema.system.discovery import ModelKey + + +def theme_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: + """Add `"overture:theme={theme}"` for each `OvertureFeature` referenced. + + Tags are attached to the entry point's `ModelKey`. For + discriminated-union features, every concrete arm contributes its + own `theme`; arms that share a theme deduplicate to a single tag, + and arms with different themes contribute multiple + `overture:theme=X` tags to the same `ModelKey`. + + Each arm's `theme` field must be annotated as a single-value + `Literal[str]`; any other annotation is a model-definition bug and + raises `TypeError`. + + Parameters + ---------- + types + Concrete `BaseModel` subclasses for the entry point. For + discriminated-union features this is every arm. + key + Key identifying the model. + tags + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with `"overture:theme={theme}"` added if applicable. + + Raises + ------ + TypeError + If a referenced `OvertureFeature`'s `theme` is not a single-value + `Literal[str]`. + """ + for tp in types: + if issubclass(tp, OvertureFeature): + tags.add(f"overture:theme={_theme_literal(tp)}") + return tags + + +def _theme_literal(model_class: type[OvertureFeature]) -> str: + """Extract the literal `theme` value from an `OvertureFeature` subclass. + + Raises + ------ + TypeError + If `theme` is not annotated as a single-value `Literal`. + """ + annotation = model_class.model_fields["theme"].annotation + if get_origin(annotation) is not Literal: + raise TypeError( + f"{model_class.__name__}.theme must be annotated Literal[...]; " + f"got {annotation!r}" + ) + args = get_args(annotation) + if len(args) != 1 or not isinstance(args[0], str): + raise TypeError( + f"{model_class.__name__}.theme must be a single-value str Literal; " + f"got {annotation!r}" + ) + return args[0] diff --git a/packages/overture-schema-core/tests/test_core_tag_providers.py b/packages/overture-schema-core/tests/test_core_tag_providers.py new file mode 100644 index 000000000..8c8b7dce3 --- /dev/null +++ b/packages/overture-schema-core/tests/test_core_tag_providers.py @@ -0,0 +1,75 @@ +"""Tests for core tag providers.""" + +from typing import Annotated, Literal + +import pytest +from overture.schema.core import OvertureFeature +from overture.schema.core.tag_providers import ( + theme_provider, +) +from overture.schema.system.discovery import ModelKey +from overture.schema.system.discovery.discovery import _generate_tags +from overture.schema.system.discovery.types import TagProviderDict, TagProviderKey +from pydantic import BaseModel, Field, Tag + + +@pytest.fixture +def building() -> type[OvertureFeature]: + class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): + pass + + return Building + + +@pytest.fixture +def not_overture() -> type[BaseModel]: + class NotOverture(BaseModel): + pass + + return NotOverture + + +def _empty_key(name: str = "x", entry_point: str = "mod:X") -> ModelKey: + return ModelKey(name=name, entry_point=entry_point, tags=frozenset()) + + +def test_theme_provider_plain_class(building: type[OvertureFeature]) -> None: + tags = theme_provider((building,), _empty_key(), set()) + assert tags == {"overture:theme=buildings"} + + +def test_theme_provider_discriminated_union() -> None: + # `_generate_tags` is responsible for walking the union to concrete arms. + class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): + pass + + class Rail(OvertureFeature[Literal["transportation"], Literal["rail"]]): + pass + + union = Annotated[ + Annotated[Road, Tag("road")] | Annotated[Rail, Tag("rail")], + Field(discriminator="type"), + ] + provider_key = TagProviderKey( + name="theme", + entry_point="core:theme_provider", + package_name="overture-schema-core", + ) + providers: TagProviderDict = {provider_key: theme_provider} + tags = _generate_tags(union, _empty_key(), providers) + assert tags == {"overture:theme=transportation"} + + +def test_theme_provider_skips_non_overture(not_overture: type[BaseModel]) -> None: + tags = theme_provider((not_overture,), _empty_key(), set()) + assert tags == set() + + +def test_theme_provider_raises_on_non_literal_theme() -> None: + class BadFeature(OvertureFeature): # type: ignore[type-arg] + # ThemeT defaults to str (its bound), not Literal — a third-party + # bug we want to surface. + pass + + with pytest.raises(TypeError, match="must be annotated Literal"): + theme_provider((BadFeature,), _empty_key(), set()) diff --git a/packages/overture-schema-divisions-theme/pyproject.toml b/packages/overture-schema-divisions-theme/pyproject.toml index 2fd1dac53..5d75161c4 100644 --- a/packages/overture-schema-divisions-theme/pyproject.toml +++ b/packages/overture-schema-divisions-theme/pyproject.toml @@ -34,10 +34,10 @@ path = "src/overture/schema/divisions/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:divisions:division" = "overture.schema.divisions:Division" -"overture:divisions:division_area" = "overture.schema.divisions:DivisionArea" -"overture:divisions:division_boundary" = "overture.schema.divisions:DivisionBoundary" - +division = "overture.schema.divisions:Division" +division_area = "overture.schema.divisions:DivisionArea" +division_boundary = "overture.schema.divisions:DivisionBoundary" + [[examples.Division]] id = "350e85f6-68ba-4114-9906-c2844815988b" geometry = "POINT (-175.2551522 -21.1353686)" diff --git a/packages/overture-schema-places-theme/pyproject.toml b/packages/overture-schema-places-theme/pyproject.toml index 48abbcbfa..0b1b68617 100644 --- a/packages/overture-schema-places-theme/pyproject.toml +++ b/packages/overture-schema-places-theme/pyproject.toml @@ -34,8 +34,8 @@ path = "src/overture/schema/places/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:places:place" = "overture.schema.places:Place" - +place = "overture.schema.places:Place" + [[examples.Place]] id = "99003ee6-e75b-4dd6-8a8a-53a5a716c50d" geometry = "POINT (-150.46875 -79.1713346)" diff --git a/packages/overture-schema-system/README.md b/packages/overture-schema-system/README.md index 238984eb0..d5d037cc6 100644 --- a/packages/overture-schema-system/README.md +++ b/packages/overture-schema-system/README.md @@ -109,6 +109,111 @@ class ParkBench(Identified): park_id: Annotated[Id, Reference(Relationship.BELONGS_TO, Park)] ``` +## Discovery + +Packages register models on the `overture.models` Python entry point group. Each entry maps a name to a class import path: + +```toml +[project.entry-points."overture.models"] +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" +``` + +`discover_models()` walks the group, loads each entry point, and returns a dict keyed by `ModelKey`. Consumers iterate over the result without knowing which package owns any given model -- the CLI and codegen tools both run discovery to assemble their working set. + +A `ModelKey` carries the entry point `name`, its `entry_point` value (`"module:Class"`), and a `frozenset[str]` of tags. [Tagging](#tagging) is how those tags get attached. + +## Tagging + +Tags classify discovered models. A package registers [tag providers](#providers) on `overture.tag_providers`; when `discover_models` runs, it asks every provider which tags apply to each model and attaches the resulting set to its `ModelKey`. Downstream tools read those tags -- the CLI's `--tag` filter, codegen's grouping logic, anything that reasons about a model without importing it. + +```python +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) + +models = discover_models() + +selected = filter_models( + models, + TagSelector(include_any=("feature",), exclude_any=("draft",)), +) +``` + +### Format + +Tags follow `[namespace:]predicate[=value]`: + +- **Plain** -- `feature`, `overture` +- **Namespaced** -- `system:extension` +- **Key/value** -- `overture:theme=buildings` + +`:` separates namespace from predicate -- one level only, no nested colons. `=` introduces a discrete value, one per tag. Predicate and namespace parts are lowercase alphanumeric (with `_`, `.`, `-`); values also accept uppercase. Matching is case-sensitive throughout. + +Helpers in `overture.schema.system.discovery.tag` parse structured tags: + +- `is_valid_tag(tag)` -- check whether a string matches the format +- `get_namespace(tag)` -- extract the namespace prefix, or `""` for a plain tag +- `get_values_for_key(tags, "overture:theme")` -- extract values from k/v tags with the given key + +### Providers + +A tag provider is a callable registered on the `overture.tag_providers` entry point group. Discovery passes it the concrete `BaseModel` subclasses for the entry point and a copy of the tags accumulated so far; tags it adds are merged into the running set after passing the reservation checks below. + +```python +from collections.abc import Iterable +from pydantic import BaseModel +from overture.schema.system.discovery import ModelKey +from overture.schema.system.feature import Feature + +def feature_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: + if any(issubclass(tp, Feature) for tp in types): + tags.add("feature") + return tags +``` + +```toml +[project.entry-points."overture.tag_providers"] +feature = "overture.schema.system.discovery.tag_providers:feature_provider" +``` + +Tags from one provider are visible to providers that run later, but execution order is unspecified -- a provider must not depend on tags added by another. Provider exceptions are caught, logged, and discarded; they do not abort discovery. + +Discovery resolves the entry-point value to concrete classes before invoking providers. For class entries that yields a one-element iterable; for discriminated-union features (e.g. `Segment`, which loads as `Annotated[Union[...], Field(...)]`) it yields every arm. Providers therefore work uniformly with `issubclass` and never need to walk type expressions themselves. + +### Reservation + +Specific plain tags and namespaces are reserved for designated packages. For example: + +| Tag or namespace | Owning package | +|---|---| +| `feature` (tag) | `overture-schema-system` | +| `system:` (namespace) | `overture-schema-system` | +| `overture:` (namespace) | `overture-schema-core` | + +When a provider attempts to set a reserved tag from an unauthorized package, discovery logs a warning and discards the tag. + +### Built-in Providers + +- **`feature`** (in `system`) -- adds `feature` if any concrete arm is a `Feature` subclass. +- **`theme`** (in `core`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme. + +### Selecting Models by Tag + +`filter_models(models, selector)` applies `TagSelector` predicates against each `ModelKey.tags`: + +- `include_any` -- OR scope; at least one tag must match (empty: no scope filter) +- `require_all` -- AND narrowing; every tag must be present (empty: no narrowing) +- `exclude_any` -- OR-NOT subtraction; any match drops the model + +An empty selector returns the input unchanged. + ## Also Included - **Optionality** -- `Omitable[T]` models JSON Schema's "may be absent but not null" semantics, which Pydantic's `T | None` conflates with nullable. diff --git a/packages/overture-schema-system/pyproject.toml b/packages/overture-schema-system/pyproject.toml index d81c649ce..0646d7a0a 100644 --- a/packages/overture-schema-system/pyproject.toml +++ b/packages/overture-schema-system/pyproject.toml @@ -55,3 +55,6 @@ ignore = [ "C901", # too complex ] per-file-ignores = {"__init__.py" = ["F401"]} + +[project.entry-points."overture.tag_providers"] +feature = "overture.schema.system.discovery.tag_providers:feature_provider" diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py new file mode 100644 index 000000000..ed8af77ad --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py @@ -0,0 +1,19 @@ +from . import tag +from .discovery import ( + TagSelector, + discover_models, + filter_models, + get_registered_model, +) +from .keys import ModelKey +from .types import ModelDict + +__all__ = [ + "ModelDict", + "ModelKey", + "TagSelector", + "discover_models", + "filter_models", + "get_registered_model", + "tag", +] diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py new file mode 100644 index 000000000..bd271fa9e --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -0,0 +1,293 @@ +"""Model discovery system for Overture schema registry.""" + +import importlib.metadata +import logging +from dataclasses import dataclass, replace +from typing import Any + +from pydantic import BaseModel + +from overture.schema.system.discovery.tag import ( + get_namespace, + is_valid_tag, +) +from overture.schema.system.discovery.types import ( + ModelDict, + ModelKey, + TagProviderDict, + TagProviderKey, +) +from overture.schema.system.typing_util import collect_types + +log = logging.getLogger(__name__) + +# Tags that are reserved and can only be set by specific packages. +_RESERVED_TAGS: dict[str, set[str]] = { + "feature": {"overture-schema-system"}, +} +# Namespaces that are reserved and can only be set by specific packages. +_RESERVED_NAMESPACES: dict[str, set[str]] = { + "overture": {"overture-schema-core"}, + "system": {"overture-schema-system"}, +} + + +def _generate_tags( + model_class: Any, # noqa: ANN401 + key: ModelKey, + providers: TagProviderDict, +) -> set[str]: + """Generate tags for a model class using tag providers. + + The model is walked once via `collect_types` to find every concrete + `BaseModel` arm, and each provider is called with the result. Tags + a provider adds are filtered for validity and permission before + being included. Provider errors are caught and logged as warnings + rather than propagated. + + Parameters + ---------- + model_class + Value loaded from an `overture.models` entry point — usually a + `type[BaseModel]`, or a discriminated-union expression. + key + Key identifying the model. + providers + Tag providers to invoke. + + Returns + ------- + set[str] + Tags generated for the model. + """ + types = collect_types(model_class) + tags: set[str] = set() + for provider_key, provider in providers.items(): + try: + added_tags = set(provider(types, key, tags.copy())) - tags + filtered_tags = _filter_tags(added_tags, provider_key) + tags.update(filtered_tags) + except Exception as e: + log.warning( + f"Error in tag provider {provider_key.name} for model {key.name}: {e}", + exc_info=True, + ) + return tags + + +def _filter_tags(tags: set[str], provider: TagProviderKey) -> set[str]: + """Filter tags that cannot be used by the provider, including invalid tags, + reserved tags, and tags using a reserved namespace. + + Parameters + ---------- + tags : set[str] + Tags to filter. + provider : TagProviderKey + Provider attempting to set the tags. + + Returns + ------- + set[str] + Permitted tags. + """ + filtered_tags: set[str] = set() + reserved_tags: set[str] = { + tag for tag, pkgs in _RESERVED_TAGS.items() if provider.package_name not in pkgs + } + reserved_namespaces: set[str] = { + ns + for ns, pkgs in _RESERVED_NAMESPACES.items() + if provider.package_name not in pkgs + } + for tag in tags: + if not is_valid_tag(tag): + log.warning( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set '{tag}' as tag. " + f"This tag does not match the required format." + ) + continue + if tag in reserved_tags: + allowed_pkgs = _RESERVED_TAGS.get(tag, set()) + log.warning( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set reserved tag '{tag}'. " + f"This tag can only be set by packages from: {allowed_pkgs}." + ) + continue + tag_ns = get_namespace(tag) + if tag_ns and tag_ns in reserved_namespaces: + allowed_pkgs = _RESERVED_NAMESPACES.get(tag_ns, set()) + log.warning( + f"Tag provider '{provider.name}' (package '{provider.package_name}') attempted to set tag '{tag}' in reserved namespace '{tag_ns}'. " + f"This namespace can only be set by packages from: {allowed_pkgs}." + ) + continue + filtered_tags.add(tag) + return filtered_tags + + +def discover_tag_providers( + tag_providers_group: str = "overture.tag_providers", +) -> TagProviderDict: + """Discover and load tag providers via entry points. + + Parameters + ---------- + tag_providers_group : str, optional + Entry point group to search (default: `"overture.tag_providers"`). + + Returns + ------- + TagProviderDict + Discovered tag providers keyed by TagProviderKey. + """ + tag_providers = {} + try: + for tag_provider in importlib.metadata.entry_points(group=tag_providers_group): + try: + tag_provider_class = tag_provider.load() + key = TagProviderKey( + name=tag_provider.name, + entry_point=tag_provider.value, + package_name=getattr(tag_provider.dist, "name", ""), + ) + tag_providers[key] = tag_provider_class + except Exception as e: + log.warning(f"Could not load tag provider {tag_provider.name}: {e}") + except Exception as e: + log.warning(f"Could not discover entry points: {e}") + return tag_providers + + +def discover_models( + model_group: str = "overture.models", +) -> ModelDict: + """Discover and load models via entry points, attaching tags from tag providers. + + Parameters + ---------- + model_group : str, optional + Entry point group to search (default: `"overture.models"`). + + Returns + ------- + ModelDict + Discovered models keyed by ModelKey. + """ + models = {} + tag_providers = discover_tag_providers() + try: + for model in importlib.metadata.entry_points(group=model_group): + try: + model_class = model.load() + key = ModelKey( + name=model.name, + entry_point=model.value, + tags=frozenset(), + ) + try: + key = replace( + key, + tags=frozenset(_generate_tags(model_class, key, tag_providers)), + ) + except Exception as e: + log.warning(f"Could not resolve tags for model {model.name}: {e}") + models[key] = model_class + except Exception as e: + log.warning(f"Could not load model {model.name}: {e}") + except Exception as e: + log.warning(f"Could not discover entry points: {e}") + return models + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TagSelector: + """Three tag tuples consumed by `filter_models`. + + See `filter_models` for predicate semantics, including how + empty tuples are interpreted. + + Attributes + ---------- + include_any + Scope (OR) — tags that bring models into the result. + require_all + Narrow (AND) — tags every kept model must have. + exclude_any + Subtract (OR-NOT) — tags that drop a model from the result. + """ + + include_any: tuple[str, ...] = () + require_all: tuple[str, ...] = () + exclude_any: tuple[str, ...] = () + + +def filter_models( + models: ModelDict, + selector: TagSelector = TagSelector(), + *, + type_names: tuple[str, ...] = (), +) -> ModelDict: + """Filter models by tag predicates and optional type-name match. + + Each tuple in `selector` is a predicate over `key.tags`; a model + is kept only if it satisfies every predicate. Empty tuples are + no-ops — empty `include_any` imposes no scope, empty + `require_all` imposes no narrowing, empty `exclude_any` drops + nothing — so an empty selector returns `models` unchanged. + + Parameters + ---------- + models + Models to filter. + selector + Tag predicates to apply. + type_names + If non-empty, only models whose `key.name` is in the list + are kept. Orthogonal to the tag predicate algebra. + + Returns + ------- + ModelDict + Models satisfying every supplied predicate. + """ + + def matches(key: ModelKey) -> bool: + if selector.include_any and not any( + t in key.tags for t in selector.include_any + ): + return False + if selector.require_all and not all( + t in key.tags for t in selector.require_all + ): + return False + if selector.exclude_any and any(t in key.tags for t in selector.exclude_any): + return False + if type_names and key.name not in type_names: + return False + return True + + return {k: m for k, m in models.items() if matches(k)} + + +def get_registered_model(model_name: str) -> type[BaseModel] | None: + """Get the model by name. + + Loads all models via entry points and returns the first with a matching name. + If multiple models share the same name, the first one encountered is returned. + + Parameters + ---------- + model_name : str + Model name to look up. + + Returns + ------- + type[BaseModel] or None + Model class if found, otherwise `None`. + """ + models = discover_models() + for key, model_class in models.items(): + if key.name == model_name: + return model_class + return None diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/keys.py b/packages/overture-schema-system/src/overture/schema/system/discovery/keys.py new file mode 100644 index 000000000..26eebaf1a --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/keys.py @@ -0,0 +1,41 @@ +"""Key types identifying registered models and tag providers.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ModelKey: + """Key identifying a registered model by name, entry point, and tags. + + Attributes + ---------- + name : str + Friendly name derived from the entry point key. + entry_point : str + Entry point value in `"module:Class"` format. + tags : frozenset[str] + Tags associated with the model. + """ + + name: str + entry_point: str + tags: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class TagProviderKey: + """Key identifying a registered tag provider by name, entry point, and package. + + Attributes + ---------- + name : str + Friendly name derived from the entry point key. + entry_point : str + Entry point value in `"module:function"` format. + package_name : str + Package that provides this tag provider. + """ + + name: str + entry_point: str + package_name: str diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py new file mode 100644 index 000000000..d11fa8f3b --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag.py @@ -0,0 +1,102 @@ +"""Tag format specification and utilities for Overture schema discovery. + +Tags follow the pattern `[namespace:]predicate[=value]` and come in three forms: + +- **Plain** — `feature` +- **Namespaced** — `system:extension` +- **Key/value** — `overture:theme=buildings` + +`:` signals ownership and reservation — only the owning package may set tags in a +given namespace. `=` signals a dimension with a discrete value. +One level of each: no nested colons, no multiple `=` signs. + +Tag matching is case-sensitive throughout. +""" + +import re + +TAG_PART = r"[a-z0-9][a-z0-9_.-]*" +VALUE = r"[a-zA-Z0-9_.-]+" +NAMESPACE_TAG = rf"{TAG_PART}:{TAG_PART}(?:={VALUE})?" +TAG = re.compile(rf"^(?:{TAG_PART}|{NAMESPACE_TAG})$") + + +def get_namespace(tag: str) -> str: + """Extract the namespace prefix from a namespaced tag. + + Parameters + ---------- + tag : str + A valid tag string. + + Returns + ------- + str + The namespace prefix if the tag is a namespaced tag, otherwise `""`. + + Examples + -------- + >>> get_namespace("overture:theme=buildings") + 'overture' + """ + return tag.split(":")[0] if is_valid_tag(tag) and ":" in tag else "" + + +def get_values_for_key(tags: frozenset[str] | set[str], key: str) -> set[str]: + """Extract values from key/value namespaced tags matching the given key. + + Parameters + ---------- + tags : frozenset[str] or set[str] + Tags to search. + key : str + Key to match, e.g. `"overture:theme"`. + + Returns + ------- + set[str] + Values of tags matching `key=`. + + Examples + -------- + >>> get_values_for_key(frozenset({"overture:theme=buildings", "overture"}), "overture:theme") + {'buildings'} + """ + prefix = key + "=" + return {tag[len(prefix) :] for tag in tags if tag.startswith(prefix)} + + +def is_valid_tag(tag: str) -> bool: + """Check whether a string is a valid tag. + + A valid tag is a plain tag, a namespaced tag, or a key/value tag: + + - **Plain / namespace / predicate**: `[a-z0-9][a-z0-9_.-]*` — + lowercase alphanumeric start, then alphanumeric, hyphens, + underscores, or dots. + - **Key/value**: `{namespace}:{predicate}=[a-zA-Z0-9_.-]+` — namespace and + predicate as above; value is alphanumeric (upper and lower case), + hyphens, underscores, or dots; must be non-empty. + + Parameters + ---------- + tag : str + String to validate. + + Returns + ------- + bool + `True` if `tag` matches the required format. + + Examples + -------- + >>> is_valid_tag("feature") + True + >>> is_valid_tag("overture:theme=buildings") + True + >>> is_valid_tag("overture:theme=") + False + >>> is_valid_tag("Invalid") + False + """ + return bool(TAG.fullmatch(tag)) diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py new file mode 100644 index 000000000..46fc507d8 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/tag_providers.py @@ -0,0 +1,35 @@ +"""Tag provider logic for Overture schema discovery system.""" + +from collections.abc import Iterable + +from pydantic import BaseModel + +from overture.schema.system.discovery.types import ModelKey +from overture.schema.system.feature import Feature + + +def feature_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: + """Add the `"feature"` tag if any concrete type is a `Feature` subclass. + + Parameters + ---------- + types + Concrete `BaseModel` subclasses for the entry point. For + discriminated-union features this is every arm. + key + Key identifying the model. + tags + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with `"feature"` added if applicable. + """ + if any(issubclass(tp, Feature) for tp in types): + tags.add("feature") + return tags diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/types.py b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py new file mode 100644 index 000000000..9bad9250b --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/types.py @@ -0,0 +1,18 @@ +"""Types and data classes for Overture schema discovery system.""" + +from collections.abc import Callable, Iterable +from typing import TypeAlias + +from pydantic import BaseModel + +from .keys import ModelKey, TagProviderKey + +# Tag providers receive the concrete `BaseModel` subclasses for an entry +# point. For class entries this is a one-element iterable; for +# discriminated unions it is every arm collected by `collect_types`. +TagProvider: TypeAlias = Callable[ + [Iterable[type[BaseModel]], ModelKey, set[str]], + Iterable[str], +] +ModelDict: TypeAlias = dict[ModelKey, type[BaseModel]] +TagProviderDict: TypeAlias = dict[TagProviderKey, TagProvider] diff --git a/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py b/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py index a057a3127..c8003f8a5 100644 --- a/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py +++ b/packages/overture-schema-system/src/overture/schema/system/field_constraint/string.py @@ -211,10 +211,10 @@ def __init__(self) -> None: class StrippedConstraint(PatternConstraint): r"""Allows only strings that have no leading/trailing whitespace. - Uses ``\Z`` (absolute end-of-string) instead of ``$`` because - Python's ``$`` matches before a trailing ``\n``. ECMA regex (used by - JSON Schema) treats ``$`` as absolute end-of-string, so the JSON - schema output swaps ``\Z`` back to ``$``. + Uses `\Z` (absolute end-of-string) instead of `$` because + Python's `$` matches before a trailing `\n`. ECMA regex (used by + JSON Schema) treats `$` as absolute end-of-string, so the JSON + schema output swaps `\Z` back to `$`. """ def __init__(self) -> None: diff --git a/packages/overture-schema-system/src/overture/schema/system/typing_util.py b/packages/overture-schema-system/src/overture/schema/system/typing_util.py new file mode 100644 index 000000000..f532f5056 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/typing_util.py @@ -0,0 +1,48 @@ +"""Typing utilities for the Overture schema system.""" + +import types +from typing import Annotated, Any, Union, get_args, get_origin + + +def collect_types(tp: Any) -> set[type]: # noqa: ANN401 + """Collect concrete classes referenced by a type annotation. + + Unwraps `Annotated[X, ...]` and `Union[X, Y]` (including `X | Y`) to + find concrete `type` objects. Used by tag providers to walk + discriminated-union features (e.g. `Segment`) into their member + classes. + + Only handles the cases the discovery system encounters today. + `overture-schema-codegen` has a more capable + `analyze_type` (`extraction/type_analyzer.py`) that also unwraps + `NewType`, `Literal`, `list[...]`, `dict[K, V]`, and accumulates + constraints. A future work item is to consolidate this and the + similar logic in `overture-schema-cli` against that implementation. + + Parameters + ---------- + tp + A type annotation. Typically a class, an `Annotated[...]` + wrapper, or a discriminated union of classes. + + Returns + ------- + set[type] + Concrete classes reachable through `Annotated` and `Union` + unwrapping. Other type expressions yield an empty set. + + """ + result: set[type] = set() + + def _visit(t: Any) -> None: + origin = get_origin(t) + if origin is Annotated: + _visit(get_args(t)[0]) + elif origin is Union or origin is types.UnionType: + for arg in get_args(t): + _visit(arg) + elif isinstance(t, type): + result.add(t) + + _visit(tp) + return result diff --git a/packages/overture-schema-system/tests/test_discovery_filter_models.py b/packages/overture-schema-system/tests/test_discovery_filter_models.py new file mode 100644 index 000000000..c1c051393 --- /dev/null +++ b/packages/overture-schema-system/tests/test_discovery_filter_models.py @@ -0,0 +1,244 @@ +"""Direct coverage of filter_models combinator algebra.""" + +from pydantic import BaseModel + +from overture.schema.system.discovery import ( + ModelDict, + ModelKey, + TagSelector, + filter_models, +) + + +# Mock model classes +class Building(BaseModel): + pass + + +class Segment(BaseModel): + pass + + +class Connector(BaseModel): + pass + + +class Place(BaseModel): + pass + + +class Sources(BaseModel): + pass + + +BUILDING_KEY = ModelKey( + name="building", + entry_point="mock:Building", + tags=frozenset({"feature", "overture", "overture:theme=buildings"}), +) +SEGMENT_KEY = ModelKey( + name="segment", + entry_point="mock:Segment", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +CONNECTOR_KEY = ModelKey( + name="connector", + entry_point="mock:Connector", + tags=frozenset({"feature", "overture", "overture:theme=transportation"}), +) +PLACE_KEY = ModelKey( + name="place", + entry_point="mock:Place", + tags=frozenset({"feature", "overture", "overture:theme=places", "draft"}), +) +SOURCES_KEY = ModelKey( + name="sources", + entry_point="mock:Sources", + tags=frozenset({"overture"}), +) + +ALL_MODELS: ModelDict = { + BUILDING_KEY: Building, + SEGMENT_KEY: Segment, + CONNECTOR_KEY: Connector, + PLACE_KEY: Place, + SOURCES_KEY: Sources, +} + + +def names(models: ModelDict) -> set[str]: + """Return the set of model names in a ModelDict.""" + return {key.name for key in models} + + +class TestEmptySelector: + def test_empty_selector_returns_all(self) -> None: + result = filter_models(ALL_MODELS) + assert names(result) == names(ALL_MODELS) + + def test_empty_selector_explicit(self) -> None: + result = filter_models(ALL_MODELS, TagSelector()) + assert names(result) == names(ALL_MODELS) + + +class TestIncludeAny: + def test_single_tag(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(include_any=("overture:theme=buildings",)) + ) + assert names(result) == {"building"} + + def test_multi_tag_or(self) -> None: + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=buildings", + "overture:theme=transportation", + ) + ), + ) + assert names(result) == {"building", "segment", "connector"} + + def test_no_match(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(include_any=("overture:theme=nonexistent",)) + ) + assert result == {} + + def test_mixed_match(self) -> None: + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=("overture:theme=buildings", "overture:theme=nonexistent") + ), + ) + assert names(result) == {"building"} + + +class TestRequireAll: + def test_single_tag(self) -> None: + result = filter_models(ALL_MODELS, TagSelector(require_all=("feature",))) + assert names(result) == {"building", "segment", "connector", "place"} + + def test_multi_tag_and_match(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(require_all=("feature", "overture")) + ) + assert names(result) == {"building", "segment", "connector", "place"} + + def test_multi_tag_and_one_fails(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(require_all=("feature", "draft")) + ) + assert names(result) == {"place"} + + def test_no_match(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(require_all=("feature", "nonexistent")) + ) + assert result == {} + + +class TestExcludeAny: + def test_single_tag(self) -> None: + result = filter_models( + ALL_MODELS, TagSelector(exclude_any=("overture:theme=buildings",)) + ) + assert "building" not in names(result) + assert names(result) == {"segment", "connector", "place", "sources"} + + def test_multi_tag_or(self) -> None: + result = filter_models( + ALL_MODELS, + TagSelector( + exclude_any=( + "overture:theme=buildings", + "overture:theme=transportation", + ) + ), + ) + assert names(result) == {"place", "sources"} + + def test_no_match_keeps_all(self) -> None: + result = filter_models(ALL_MODELS, TagSelector(exclude_any=("nonexistent",))) + assert names(result) == names(ALL_MODELS) + + +class TestTypeNames: + def test_single(self) -> None: + result = filter_models(ALL_MODELS, type_names=("building",)) + assert names(result) == {"building"} + + def test_multiple(self) -> None: + result = filter_models(ALL_MODELS, type_names=("building", "place")) + assert names(result) == {"building", "place"} + + def test_none_match(self) -> None: + result = filter_models(ALL_MODELS, type_names=("nonexistent",)) + assert result == {} + + +class TestCrossCombinator: + def test_include_then_require(self) -> None: + # Scope to features (places, transportation), narrow to those + # also tagged "draft" → only place qualifies. + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=places", + "overture:theme=transportation", + ), + require_all=("draft",), + ), + ) + assert names(result) == {"place"} + + def test_include_then_exclude(self) -> None: + # Scope to all themed features, exclude buildings. + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=buildings", + "overture:theme=transportation", + "overture:theme=places", + ), + exclude_any=("overture:theme=buildings",), + ), + ) + assert names(result) == {"segment", "connector", "place"} + + def test_all_three_combinators_plus_type_names(self) -> None: + # Scope to features in either places or transportation, + # require feature tag, exclude drafts, restrict to segment by name. + result = filter_models( + ALL_MODELS, + TagSelector( + include_any=( + "overture:theme=places", + "overture:theme=transportation", + ), + require_all=("feature",), + exclude_any=("draft",), + ), + type_names=("segment",), + ) + assert names(result) == {"segment"} + + +class TestIdempotence: + def test_double_application(self) -> None: + selector = TagSelector( + include_any=("overture:theme=buildings", "overture:theme=places") + ) + once = filter_models(ALL_MODELS, selector) + twice = filter_models(once, selector) + assert names(once) == names(twice) + + +class TestInputInvariance: + def test_returns_new_dict(self) -> None: + result = filter_models(ALL_MODELS) + assert result is not ALL_MODELS diff --git a/packages/overture-schema-system/tests/test_discovery_tag_selector.py b/packages/overture-schema-system/tests/test_discovery_tag_selector.py new file mode 100644 index 000000000..3b0241d83 --- /dev/null +++ b/packages/overture-schema-system/tests/test_discovery_tag_selector.py @@ -0,0 +1,44 @@ +"""Tests for TagSelector dataclass.""" + +from dataclasses import FrozenInstanceError + +import pytest + +from overture.schema.system.discovery import TagSelector + + +class TestTagSelector: + def test_default_is_empty(self) -> None: + s = TagSelector() + assert s.include_any == () + assert s.require_all == () + assert s.exclude_any == () + + def test_construction_with_fields(self) -> None: + s = TagSelector( + include_any=("a", "b"), + require_all=("c",), + exclude_any=("d",), + ) + assert s.include_any == ("a", "b") + assert s.require_all == ("c",) + assert s.exclude_any == ("d",) + + def test_kw_only(self) -> None: + with pytest.raises(TypeError): + TagSelector(("a",)) # type: ignore[misc] + + def test_frozen(self) -> None: + s = TagSelector(include_any=("a",)) + with pytest.raises(FrozenInstanceError): + s.include_any = ("b",) # type: ignore[misc] + + def test_equality_by_value(self) -> None: + a = TagSelector(include_any=("x",)) + b = TagSelector(include_any=("x",)) + assert a == b + + def test_hashable(self) -> None: + s = TagSelector(include_any=("x",)) + d = {s: 1} + assert d[s] == 1 diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py new file mode 100644 index 000000000..ef7bba7ec --- /dev/null +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -0,0 +1,208 @@ +from collections.abc import Iterable +from typing import Annotated + +import pytest +from pydantic import BaseModel, Field, Tag + +from overture.schema.system.discovery.discovery import _generate_tags +from overture.schema.system.discovery.tag_providers import feature_provider +from overture.schema.system.discovery.types import ( + ModelKey, + TagProvider, + TagProviderDict, + TagProviderKey, +) +from overture.schema.system.feature import Feature + + +@pytest.fixture +def core_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="core", entry_point="core:Provider", package_name="overture-schema-core" + ) + + +@pytest.fixture +def system_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="system", + entry_point="system:Provider", + package_name="overture-schema-system", + ) + + +@pytest.fixture +def other_tag_provider() -> TagProviderKey: + return TagProviderKey( + name="other", entry_point="other:Provider", package_name="other-package" + ) + + +@pytest.fixture +def feature() -> type[Feature]: + class SomeFeature(Feature): + pass + + return SomeFeature + + +@pytest.fixture +def not_a_feature() -> type[BaseModel]: + class NotAFeature(BaseModel): + pass + + return NotAFeature + + +@pytest.fixture +def any_key() -> ModelKey: + return ModelKey(name="x", entry_point="m:X", tags=frozenset()) + + +@pytest.fixture +def any_model() -> type[BaseModel]: + class M(BaseModel): + pass + + return M + + +def fake_provider(*tags: str) -> TagProvider: + """Provider that always returns the given tags, ignoring its inputs.""" + + def _provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + current_tags: set[str], + ) -> set[str]: + return set(tags) + + return _provider + + +def test_valid_tags( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = { + other_tag_provider: fake_provider("valid", "other:valid", "other:valid=true") + } + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid", "other:valid", "other:valid=true"} + + +def test_invalid_tag( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider("InvalidTag")} + result = _generate_tags(any_model, any_key, providers) + assert result == set() + + +def test_reserved_tag( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider("feature", "valid")} + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid"} + + +def test_allowed_reserved_tag( + system_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + system_providers = {system_tag_provider: fake_provider("feature")} + assert _generate_tags(any_model, any_key, system_providers) == {"feature"} + + +def test_reserved_namespace( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = { + other_tag_provider: fake_provider( + "overture:feature", "system:feature", "valid:tag" + ) + } + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid:tag"} + + +def test_allowed_reserved_namespace( + core_tag_provider: TagProviderKey, + system_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + core_providers = {core_tag_provider: fake_provider("overture:feature")} + assert _generate_tags(any_model, any_key, core_providers) == {"overture:feature"} + + system_providers = {system_tag_provider: fake_provider("system:feature")} + assert _generate_tags(any_model, any_key, system_providers) == {"system:feature"} + + +def test_empty_tags( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider()} + assert _generate_tags(any_model, any_key, providers) == set() + + +def test_mixed_tags( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = { + other_tag_provider: fake_provider( + "valid", "feature", "overture:feature", "InvalidTag" + ) + } + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid"} + + +def test_feature_provider_adds_feature_tag(feature: type[Feature]) -> None: + key = ModelKey(name="feature", entry_point="system:Feature", tags=frozenset()) + result = feature_provider((feature,), key, set()) + assert "feature" in result + + +def test_feature_provider_does_not_add_feature_tag( + not_a_feature: type[BaseModel], +) -> None: + key = ModelKey( + name="notafeature", entry_point="system:NotAFeature", tags=frozenset() + ) + result = feature_provider((not_a_feature,), key, set()) + assert "feature" not in result + + +def test_feature_provider_handles_discriminated_union( + system_tag_provider: TagProviderKey, +) -> None: + # Mimics the shape of `Segment`: Annotated[Union[...], Field(discriminator=...)]. + # `_generate_tags` is responsible for walking the union to concrete arms. + class ArmA(Feature): + pass + + class ArmB(BaseModel): + pass + + union = Annotated[ + Annotated[ArmA, Tag("a")] | Annotated[ArmB, Tag("b")], + Field(discriminator="type"), + ] + key = ModelKey(name="union", entry_point="mod:Union", tags=frozenset()) + providers: TagProviderDict = {system_tag_provider: feature_provider} + result = _generate_tags(union, key, providers) + assert "feature" in result diff --git a/packages/overture-schema-system/tests/test_tags.py b/packages/overture-schema-system/tests/test_tags.py new file mode 100644 index 000000000..111a532e4 --- /dev/null +++ b/packages/overture-schema-system/tests/test_tags.py @@ -0,0 +1,112 @@ +import re + +import pytest + +from overture.schema.system.discovery.tag import ( + NAMESPACE_TAG, + TAG, + TAG_PART, + get_values_for_key, + is_valid_tag, +) + + +def test_get_values_for_key_returns_correct_values() -> None: + tags = frozenset({"overture:theme=buildings", "overture", "draft"}) + key = "overture:theme" + result = get_values_for_key(tags, key) + assert result == {"buildings"} + + +def test_get_values_for_key_returns_empty_set_for_nonexistent_key() -> None: + tags = frozenset({"overture:theme=buildings", "overture", "draft"}) + key = "nonexistent:key" + result = get_values_for_key(tags, key) + assert result == set() + + +def test_get_values_for_key_handles_empty_tags() -> None: + tags: frozenset[str] = frozenset() + key = "overture:theme" + result = get_values_for_key(tags, key) + assert result == set() + + +VALID_PLAIN_TAGS = [ + "v", + "valid", + "valid1", + "valid_tag", + "valid-tag", + "0valid", + "42", + "in.valid", + "3.14", + "com.example", +] + +INVALID_PLAIN_TAGS = [ + "", + "_invalid", + "-invalid", + ".invalid", + "Invalid", + "invalid!", + "invalid ", +] + +VALID_NAMESPACE_TAGS = [ + "ns:predicate", + "ns:predicate1", + "ns:predicate-1", + "ns.dotted:predicate", + "ns:pred.icate", + "ns:predicate=value", + "ns:predicate=value_0", + "ns:predicate=value-0", + "ns:predicate=value.0", + "ns:predicate=value_2-3.4", + "ns:predicate=42", + "ns:predicate=3.14", + "ns:predicate=Value", +] + +INVALID_NAMESPACE_TAGS = [ + "ns:", + ":predicate", + "ns:predicate=", + "ns:predicate=value ", + "ns:predicate=value!", + "ns:predicate=ns:value", + "ns:predicate=predicate=value", + "Ns:predicate", + "ns:Predicate", +] + + +@pytest.mark.parametrize("tag", VALID_PLAIN_TAGS) +def test_valid_plain_tag(tag: str) -> None: + assert re.fullmatch(TAG_PART, tag) + assert TAG.fullmatch(tag) + assert is_valid_tag(tag) + + +@pytest.mark.parametrize("tag", INVALID_PLAIN_TAGS) +def test_invalid_plain_tag(tag: str) -> None: + assert not re.fullmatch(TAG_PART, tag) + assert not TAG.fullmatch(tag) + assert not is_valid_tag(tag) + + +@pytest.mark.parametrize("tag", VALID_NAMESPACE_TAGS) +def test_valid_namespace_tag(tag: str) -> None: + assert re.fullmatch(NAMESPACE_TAG, tag) + assert TAG.fullmatch(tag) + assert is_valid_tag(tag) + + +@pytest.mark.parametrize("tag", INVALID_NAMESPACE_TAGS) +def test_invalid_namespace_tag(tag: str) -> None: + assert not re.fullmatch(NAMESPACE_TAG, tag) + assert not TAG.fullmatch(tag) + assert not is_valid_tag(tag) diff --git a/packages/overture-schema-transportation-theme/pyproject.toml b/packages/overture-schema-transportation-theme/pyproject.toml index 40db4638a..fe4eb8a47 100644 --- a/packages/overture-schema-transportation-theme/pyproject.toml +++ b/packages/overture-schema-transportation-theme/pyproject.toml @@ -35,9 +35,9 @@ path = "src/overture/schema/transportation/__about__.py" packages = ["src/overture"] [project.entry-points."overture.models"] -"overture:transportation:connector" = "overture.schema.transportation:Connector" -"overture:transportation:segment" = "overture.schema.transportation:Segment" - +connector = "overture.schema.transportation:Connector" +segment = "overture.schema.transportation:Segment" + [[examples.Connector]] id = "39542bee-230f-4b91-b7e5-a9b58e0c59b1" geometry = "POINT (-176.5472979 -43.9679472)" diff --git a/packages/overture-schema/src/overture/schema/__init__.py b/packages/overture-schema/src/overture/schema/__init__.py index 1f75ea511..2f5a0b7a0 100644 --- a/packages/overture-schema/src/overture/schema/__init__.py +++ b/packages/overture-schema/src/overture/schema/__init__.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, Tag, TypeAdapter from overture.schema.core import OvertureFeature -from overture.schema.core.discovery import discover_models +from overture.schema.system.discovery import discover_models from overture.schema.system.feature import Feature diff --git a/uv.lock b/uv.lock index 7a10a557a..2b1bf0be2 100644 --- a/uv.lock +++ b/uv.lock @@ -215,7 +215,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -811,7 +811,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "click", specifier = ">=8.0" }, + { name = "click", specifier = ">=8.1" }, { name = "overture-schema-core", editable = "packages/overture-schema-core" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, { name = "pydantic", specifier = ">=2.12.0" }, @@ -833,6 +833,7 @@ source = { editable = "packages/overture-schema-codegen" } dependencies = [ { name = "click" }, { name = "jinja2" }, + { name = "overture-schema-cli" }, { name = "overture-schema-core" }, { name = "overture-schema-system" }, { name = "tomli", marker = "python_full_version < '3.11'" }, @@ -840,8 +841,9 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "click", specifier = ">=8.0" }, + { name = "click", specifier = ">=8.1" }, { name = "jinja2", specifier = ">=3.0" }, + { name = "overture-schema-cli", editable = "packages/overture-schema-cli" }, { name = "overture-schema-core", editable = "packages/overture-schema-core" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" },