From e2754a3abd53b6952f8dff8bf96ae4e33e59cb75 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Thu, 16 May 2024 17:06:21 +0200 Subject: [PATCH 1/6] unraveling proto --- jhack/main.py | 2 + jhack/utils/deployment_graph.py | 142 ++++++++++++++++++++++++++++++++ jhack/utils/integrate.py | 2 +- 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 jhack/utils/deployment_graph.py diff --git a/jhack/main.py b/jhack/main.py index cc3552f7..4e522059 100755 --- a/jhack/main.py +++ b/jhack/main.py @@ -60,6 +60,7 @@ def main(): from jhack.utils.tail_charms import tail_events from jhack.utils.unbork_juju import unbork_juju from jhack.utils.unleash import vanity + from jhack.utils.deployment_graph import unravel if "--" in sys.argv: sep = sys.argv.index("--") @@ -81,6 +82,7 @@ def main(): utils.command(name="crpc", no_args_is_help=True)(charm_rpc) utils.command(name="eval", no_args_is_help=True)(charm_eval) utils.command(name="script", no_args_is_help=True)(charm_script) + utils.command(name="unravel", no_args_is_help=True)(unravel) jinx = typer.Typer( name="jinx", diff --git a/jhack/utils/deployment_graph.py b/jhack/utils/deployment_graph.py new file mode 100644 index 00000000..d4b9cee0 --- /dev/null +++ b/jhack/utils/deployment_graph.py @@ -0,0 +1,142 @@ +import dataclasses +from typing import List, Dict + +import typer + +from jhack.helpers import juju_status, get_current_model + +from jhack.logger import logger as jhack_logger + +logger = jhack_logger.getChild("graph") + + +@dataclasses.dataclass(frozen=True) +class _App: + name: str + model: str + + # raw juju status | jq | .applications[name] + meta: Dict + + @property + def scale(self): + return len(self.meta["units"]) + + def __hash__(self): + return hash((self.model, self.name)) + + +@dataclasses.dataclass(frozen=True) +class _Relation: + remote_app: _App + endpoint: str + meta: Dict + endpoint_to: str + + @property + def interface(self): + return self.meta["interface"] + + +class Graph: + """Graph type.""" + + def __init__(self, graph: Dict[_App, List[_Relation]]): + self._graph = graph + + @staticmethod + def bootstrap(app_name: str, model_name: str = None) -> "Graph": + """Bootstrap a graph from a single starting app url. + + Example: + >>> Graph.bootstrap("microk8s-localhost:clite.alertmanager/0") + """ + if "/" in app_name: + logger.warning( + f"stripping unit ID suffix from {app_name}. Pass an app name instead." + ) + app_name = app_name.split("/")[0] + + print(f"Bootstrapping graph from root: {model_name}.{app_name}") + + model_status_cache = {} + + def get_status(model_name_): + if model_name_ not in model_status_cache: + model_status_cache[model_name_] = juju_status( + model=model_name_, json=True + ) + return model_status_cache[model_name_] + + def get_app(app_name_, model_name_, status=None): + status = status or get_status(model_name_) + app_meta = status["applications"][app_name_] + return _App(name=app_name_, model=model_name_, meta=app_meta) + + def walk(model_name_: str, app_name_: str, graph_=None): + status = get_status(model_name_) + app = get_app(app_name_, model_name_) + relations: List[_Relation] = [] + graph_[app] = relations + + offers_meta = status.get("application-endpoints", ()) + + for endpoint, bindings in app.meta["relations"].items(): + for binding in bindings: + remote_app_name = binding["related-application"] + + if remote_app_name in offers_meta: + # CMR + remote_app_meta = offers_meta[remote_app_name] + # url is in the form 'localhost-localhost:admin/gagent1.gagent' + remote_model_name = remote_app_meta["url"].split(".")[0] + remote_app = get_app(remote_app_name, remote_model_name) + else: + remote_model_name = model_name_ + remote_app = get_app(remote_app_name, model_name_) + + if remote_app not in graph_: + walk(remote_model_name, remote_app_name, graph_) + + rel = _Relation( + remote_app=remote_app, + endpoint=endpoint, + meta=binding, + endpoint_to="", # todo + ) + relations.append(rel) + + return graph_ + + graph = walk(model_name or get_current_model(), app_name, {}) + return Graph(graph) + + def plot(self): + print("GRAPH:") + for origin, destination in self._graph.items(): + print(f"\t{origin} --> {{") + for app in destination: + print(f"\t\t{app}") + print(f"\t}}") + + +def _map(app_name: str, model_name: str): + graph = Graph.bootstrap(app_name=app_name, model_name=model_name) + graph.plot() + + +def unravel( + app_name: str = typer.Argument( + ..., help="""The starting point of the graph expansion.""" + ), + model_name: str = typer.Option( + "-m", + "--model", + help="The model in which to find the app from which to start the unraveling.", + ), +): + _map(app_name=app_name, model_name=model_name) + + +if __name__ == "__main__": + Graph.bootstrap("loki").plot() diff --git a/jhack/utils/integrate.py b/jhack/utils/integrate.py index 7805641a..3266c866 100644 --- a/jhack/utils/integrate.py +++ b/jhack/utils/integrate.py @@ -602,7 +602,7 @@ def fmt_endpoint(model, app, endpoint): setup_scripts += [ f"juju offer{controller} {model}.{req}:{binding.requirer_endpoint}", - f"juju consume {controller_prefix}admin/{model}.{req}", + f"juju consume {controller_prefix}{model}.{req}", ] relate_scripts += [ f"juju relate {req}:{binding.requirer_endpoint} {prov}:{binding.provider_endpoint}", From 268fd64a126cba88850f3f270dc629422dfa320a Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Fri, 17 May 2024 13:09:01 +0200 Subject: [PATCH 2/6] graph nx plot wip --- jhack/utils/deployment_graph.py | 76 ++++++++++++++++++++++++++++++--- pyproject.toml | 3 ++ 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/jhack/utils/deployment_graph.py b/jhack/utils/deployment_graph.py index d4b9cee0..01a523b4 100644 --- a/jhack/utils/deployment_graph.py +++ b/jhack/utils/deployment_graph.py @@ -1,11 +1,15 @@ import dataclasses +import os from typing import List, Dict +import matplotlib import typer from jhack.helpers import juju_status, get_current_model from jhack.logger import logger as jhack_logger +import networkx as nx +import matplotlib.pyplot as plt logger = jhack_logger.getChild("graph") @@ -25,6 +29,10 @@ def scale(self): def __hash__(self): return hash((self.model, self.name)) + @property + def charm_name(self): + return self.meta["charm-name"] + @dataclasses.dataclass(frozen=True) class _Relation: @@ -34,7 +42,7 @@ class _Relation: endpoint_to: str @property - def interface(self): + def interface(self) -> str: return self.meta["interface"] @@ -44,6 +52,57 @@ class Graph: def __init__(self, graph: Dict[_App, List[_Relation]]): self._graph = graph + def plot_nx_graph(self, show_peers: bool = False): + # todo: if running from terminal, we need: + # os.environ["QT_QPA_PLATFORM"] = "offscreen" + + g = nx.Graph() + labels = {} + + for origin, relations in self._graph.items(): + for relation in relations: + a = origin.name + b = relation.remote_app.name + if a == b and not show_peers: + continue + + e = (a, b) + g.add_edge(*e) + + labels[e] = relation.interface + + has_multiple_models = len(set(x.model for x in self._graph)) > 1 + + if has_multiple_models: + return self._plot_model_graph(g, labels) + + pos = nx.spring_layout(g, seed="41424142") + nx.draw(g, pos=pos, with_labels=True, font_weight="bold") + nx.draw_networkx_edge_labels(g, pos=pos, edge_labels=labels) + plt.draw() + plt.show() + + def _plot_model_graph(self, g): + # Compute positions for the node clusters as if they were themselves nodes in a + # supergraph using a larger scale factor + supergraph = nx.cycle_graph(len(groups)) + superpos = nx.spring_layout(g, scale=50, seed=429) + + # Use the "supernode" positions as the center of each node cluster + centers = list(superpos.values()) + pos = {} + for center, comm in zip(centers, groups): + pos.update(nx.spring_layout(nx.subgraph(g, comm), center=center, seed=1430)) + + # Nodes colored by cluster + for nodes, clr in zip(groups, ("tab:blue", "tab:orange", "tab:green")): + nx.draw_networkx_nodes( + g, pos=pos, nodelist=nodes, node_color=clr, node_size=100 + ) + nx.draw_networkx_edges(g, pos=pos) + + plt.tight_layout() + @staticmethod def bootstrap(app_name: str, model_name: str = None) -> "Graph": """Bootstrap a graph from a single starting app url. @@ -113,16 +172,18 @@ def walk(model_name_: str, app_name_: str, graph_=None): def plot(self): print("GRAPH:") - for origin, destination in self._graph.items(): - print(f"\t{origin} --> {{") - for app in destination: - print(f"\t\t{app}") + for origin, relations in self._graph.items(): + print(f"\t{origin.name} ({origin.charm_name}) --> {{") + for relation in relations: + print( + f"\t\t{relation.endpoint} >> {relation.remote_app.name} ({relation.remote_app.charm_name})" + ) print(f"\t}}") def _map(app_name: str, model_name: str): graph = Graph.bootstrap(app_name=app_name, model_name=model_name) - graph.plot() + graph.plot_nx_graph() def unravel( @@ -130,6 +191,7 @@ def unravel( ..., help="""The starting point of the graph expansion.""" ), model_name: str = typer.Option( + None, "-m", "--model", help="The model in which to find the app from which to start the unraveling.", @@ -139,4 +201,4 @@ def unravel( if __name__ == "__main__": - Graph.bootstrap("loki").plot() + Graph.bootstrap("loki").plot_nx_graph() diff --git a/pyproject.toml b/pyproject.toml index b50579fd..8916c0c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,9 @@ dependencies = [ "requests-unixsocket(==0.3.0)", "asttokens", "astunparse", + "networkx(==3.1)", + "matplotlib(==3.7.5)", + "pyqt5(>=5.15.9)", "toml", ] classifiers = [ From 705c80ede052355ba3fd67f3e77e81a955c56c1e Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Fri, 25 Oct 2024 10:04:58 +0200 Subject: [PATCH 3/6] this-is-fine and deployment graph base --- jhack/main.py | 4 + jhack/utils/deployment_graph.py | 184 +++++++++++++++++--------------- jhack/utils/show_relation.py | 18 +++- jhack/utils/this_is_fine.py | 113 ++++++++++++++++++++ 4 files changed, 229 insertions(+), 90 deletions(-) create mode 100644 jhack/utils/this_is_fine.py diff --git a/jhack/main.py b/jhack/main.py index 4e522059..e6789d62 100755 --- a/jhack/main.py +++ b/jhack/main.py @@ -7,6 +7,7 @@ import typer + # this will make jhack find its modules if you call it directly (i.e. no symlinks) # aliases are OK sys.path.append(str(Path(os.path.realpath(__file__)).parent.parent)) @@ -48,6 +49,7 @@ def main(): list_events, purge_db, ) + from jhack.utils.this_is_fine import this_is_fine from jhack.utils.ffwd import fast_forward from jhack.utils.just_deploy_this import just_deploy_this from jhack.utils.list_endpoints import list_endpoints @@ -83,6 +85,7 @@ def main(): utils.command(name="eval", no_args_is_help=True)(charm_eval) utils.command(name="script", no_args_is_help=True)(charm_script) utils.command(name="unravel", no_args_is_help=True)(unravel) + utils.command(name="this-is-fine", no_args_is_help=True)(this_is_fine) jinx = typer.Typer( name="jinx", @@ -144,6 +147,7 @@ def main(): app.command(name="crpc", no_args_is_help=True)(charm_rpc) app.command(name="eval", no_args_is_help=True)(charm_eval) app.command(name="script", no_args_is_help=True)(charm_script) + app.command(name="unravel", no_args_is_help=True)(unravel) conf = typer.Typer( name="conf", diff --git a/jhack/utils/deployment_graph.py b/jhack/utils/deployment_graph.py index 01a523b4..24a56fad 100644 --- a/jhack/utils/deployment_graph.py +++ b/jhack/utils/deployment_graph.py @@ -1,15 +1,18 @@ import dataclasses -import os -from typing import List, Dict +from typing import List, Dict, Optional, Tuple -import matplotlib import typer -from jhack.helpers import juju_status, get_current_model +from jhack.helpers import juju_status, get_current_model, cached_juju_status from jhack.logger import logger as jhack_logger -import networkx as nx -import matplotlib.pyplot as plt + +from jhack.utils.show_relation import ( + RelationEndpointURL, + Relation, + gather_relation_databags, + AppRelationData, +) logger = jhack_logger.getChild("graph") @@ -34,89 +37,50 @@ def charm_name(self): return self.meta["charm-name"] -@dataclasses.dataclass(frozen=True) -class _Relation: - remote_app: _App - endpoint: str - meta: Dict - endpoint_to: str - - @property - def interface(self) -> str: - return self.meta["interface"] - - class Graph: """Graph type.""" - def __init__(self, graph: Dict[_App, List[_Relation]]): + def __init__(self, graph: Dict[_App, List[Relation]], model: str): + self._include_default_juju_keys = True self._graph = graph - - def plot_nx_graph(self, show_peers: bool = False): - # todo: if running from terminal, we need: - # os.environ["QT_QPA_PLATFORM"] = "offscreen" - - g = nx.Graph() - labels = {} - - for origin, relations in self._graph.items(): - for relation in relations: - a = origin.name - b = relation.remote_app.name - if a == b and not show_peers: - continue - - e = (a, b) - g.add_edge(*e) - - labels[e] = relation.interface - - has_multiple_models = len(set(x.model for x in self._graph)) > 1 - - if has_multiple_models: - return self._plot_model_graph(g, labels) - - pos = nx.spring_layout(g, seed="41424142") - nx.draw(g, pos=pos, with_labels=True, font_weight="bold") - nx.draw_networkx_edge_labels(g, pos=pos, edge_labels=labels) - plt.draw() - plt.show() - - def _plot_model_graph(self, g): - # Compute positions for the node clusters as if they were themselves nodes in a - # supergraph using a larger scale factor - supergraph = nx.cycle_graph(len(groups)) - superpos = nx.spring_layout(g, scale=50, seed=429) - - # Use the "supernode" positions as the center of each node cluster - centers = list(superpos.values()) - pos = {} - for center, comm in zip(centers, groups): - pos.update(nx.spring_layout(nx.subgraph(g, comm), center=center, seed=1430)) - - # Nodes colored by cluster - for nodes, clr in zip(groups, ("tab:blue", "tab:orange", "tab:green")): - nx.draw_networkx_nodes( - g, pos=pos, nodelist=nodes, node_color=clr, node_size=100 + self._model = model + self._relation_data: Dict[Relation, Tuple[AppRelationData, ...]] = {} + + def get_relation_data(self, relation: Relation): + if relation.id is None: + raise ValueError(relation) + + if not self._relation_data.get(relation): + self._relation_data[relation] = gather_relation_databags( + RelationEndpointURL(relation.requirer_endpoint), + RelationEndpointURL(relation.provider_endpoint), + relation, + model=self._model, + include_default_juju_keys=self._include_default_juju_keys, ) - nx.draw_networkx_edges(g, pos=pos) - plt.tight_layout() + return self._relation_data[relation] @staticmethod - def bootstrap(app_name: str, model_name: str = None) -> "Graph": - """Bootstrap a graph from a single starting app url. + def bootstrap(app_name: Optional[str] = None, model_name: str = None) -> "Graph": + """Bootstrap a graph. + + From a single starting app url, or all of them otherwise. Example: - >>> Graph.bootstrap("microk8s-localhost:clite.alertmanager/0") + >>> Graph.bootstrap("alertmanager/0", "microk8s-localhost:clite") + >>> Graph.bootstrap(model_name="microk8s-localhost:clite") """ - if "/" in app_name: - logger.warning( - f"stripping unit ID suffix from {app_name}. Pass an app name instead." - ) - app_name = app_name.split("/")[0] + if app_name: + if "/" in app_name: + logger.warning( + f"stripping unit ID suffix from {app_name}. Pass an app name instead." + ) + app_name = app_name.split("/")[0] - print(f"Bootstrapping graph from root: {model_name}.{app_name}") + print(f"Bootstrapping graph from root: {model_name}.{app_name}") + else: + print(f"Bootstrapping graph in model {model_name}") model_status_cache = {} @@ -132,12 +96,30 @@ def get_app(app_name_, model_name_, status=None): app_meta = status["applications"][app_name_] return _App(name=app_name_, model=model_name_, meta=app_meta) + visited: List[str] = [] + def walk(model_name_: str, app_name_: str, graph_=None): + if app_name_ in visited: + return + visited.append(app_name_) + status = get_status(model_name_) app = get_app(app_name_, model_name_) - relations: List[_Relation] = [] + relations: List[Relation] = [] graph_[app] = relations + def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): + for endpoint, relations_meta in meta["relations"].items(): + for relation in relations_meta: + if ( + relation["interface"] == interface + and relation["related-application"] == local_app_name + ): + return endpoint + raise RuntimeError( + f"could not find a remote endpoint bound to {local_app_name} over {interface}" + ) + offers_meta = status.get("application-endpoints", ()) for endpoint, bindings in app.meta["relations"].items(): @@ -153,22 +135,40 @@ def walk(model_name_: str, app_name_: str, graph_=None): else: remote_model_name = model_name_ remote_app = get_app(remote_app_name, model_name_) + remote_app_meta = remote_app.meta if remote_app not in graph_: walk(remote_model_name, remote_app_name, graph_) - rel = _Relation( - remote_app=remote_app, - endpoint=endpoint, - meta=binding, - endpoint_to="", # todo + remote_endpoint = _find_remote_endpoint( + remote_app_meta, binding["interface"], app_name_ + ) + rel = Relation( + provider=app_name_, + provider_endpoint=endpoint, + requirer=remote_app_name, + requirer_endpoint=remote_endpoint, # todo + interface=binding["interface"], + raw_type="regular", + id=_find_relation_id( + app_name_, remote_app_name, endpoint, remote_endpoint + ), # TODO, ) relations.append(rel) return graph_ - graph = walk(model_name or get_current_model(), app_name, {}) - return Graph(graph) + model_ = model_name or get_current_model() + + if not app_name: + graph = {} + for app_name in cached_juju_status(model=model_)["applications"]: + graph = walk(model_, app_name, graph) + + else: + graph = walk(model_, app_name, {}) + + return Graph(graph, model=model_) def plot(self): print("GRAPH:") @@ -176,14 +176,22 @@ def plot(self): print(f"\t{origin.name} ({origin.charm_name}) --> {{") for relation in relations: print( - f"\t\t{relation.endpoint} >> {relation.remote_app.name} ({relation.remote_app.charm_name})" + f"\t\t({relation.provider}) {relation.provider_endpoint} >> " + f"{relation.requirer_endpoint} ({relation.requirer})" ) + if reldata := self.get_relation_data(relation): + print( + f"\t\tRelation found: " + f"{reldata[0].url} --> " + f"{reldata[1].url if len(reldata) == 2 else ''} " + f"({relation.id})" + ) print(f"\t}}") def _map(app_name: str, model_name: str): graph = Graph.bootstrap(app_name=app_name, model_name=model_name) - graph.plot_nx_graph() + graph.plot() def unravel( @@ -201,4 +209,4 @@ def unravel( if __name__ == "__main__": - Graph.bootstrap("loki").plot_nx_graph() + Graph.bootstrap("traefik").plot() diff --git a/jhack/utils/show_relation.py b/jhack/utils/show_relation.py index 1da67ddc..7107dcaa 100644 --- a/jhack/utils/show_relation.py +++ b/jhack/utils/show_relation.py @@ -51,11 +51,25 @@ class Relation: requirer_endpoint: str interface: str raw_type: str + id: Optional[int] = None @property def type(self) -> RelationType: return RelationType(self.raw_type) + def __hash__(self): + return hash( + ( + self.id or 0, + self.raw_type, + self.interface, + self.provider, + self.provider_endpoint, + self.requirer, + self.requirer_endpoint, + ) + ) + class RelationEndpointURL(str): def __init__(self, s): @@ -754,7 +768,7 @@ def pl(condition, a="", b=""): return ep_url_1, ep_url_2, relation -def _gather_entities( +def gather_relation_databags( endpoint1: RelationEndpointURL, endpoint2: Optional[RelationEndpointURL], relation: Relation, @@ -812,7 +826,7 @@ async def render_relation( if endpoint1.app_name in saas or (endpoint2 and endpoint2.app_name in saas): relation.raw_type = "cross_model" - entities = _gather_entities( + entities = gather_relation_databags( endpoint1, endpoint2, relation, diff --git a/jhack/utils/this_is_fine.py b/jhack/utils/this_is_fine.py new file mode 100644 index 00000000..41fc843b --- /dev/null +++ b/jhack/utils/this_is_fine.py @@ -0,0 +1,113 @@ +import time +from subprocess import getoutput, CalledProcessError +from typing import List, Optional + +import typer + +from jhack.helpers import juju_status +from jhack.logger import logger as jhack_logger + +logger = jhack_logger.getChild("tif") + + +def _this_is_fine( + targets: List[str] = None, + model: str = None, + keep_alive: bool = False, + retry: bool = False, + dry_run: bool = False, + rate: float = 2, +): + print("starting whacker... (interrupt with Ctrl+C)") + + while True: + start = time.time() + status = juju_status(model=model, json=True) + units_in_error = [] + + for app_name, app_meta in status["applications"].items(): + included = not targets or app_name in targets + + for unit_name, unit_meta in app_meta["units"].items(): + if targets and unit_name in targets: + included = True + + if not included: + continue + + wl_status = unit_meta["workload-status"]["current"] + jj_status = unit_meta["juju-status"]["current"] + if wl_status == "error" and jj_status == "idle": + units_in_error.append(unit_name) + + if units_in_error: + print(f"apps {units_in_error} in error") + else: + logger.debug("no apps in error") + if not keep_alive: + return + + for unit in units_in_error: + _model = f" -m {model}" if model else "" + cmd = f"juju resolve{' --no-retry' if not retry else ''}{_model} {unit}" + if dry_run: + print(f"would tell {unit} to stop whining with {cmd}") + else: + if retry: + print(f"\t{unit} is giving it another shot") + else: + print(f"\t{unit} is now *fine*") + try: + getoutput(cmd) + except CalledProcessError: + logger.exception(f"{cmd} bugged out") + continue + + elapsed = start - time.time() + time.sleep(max(rate - elapsed, 0)) + + +def this_is_fine( + targets: Optional[List[str]] = typer.Argument( + None, + help="The app, apps or units that are now *fine*. If left blank, will include all apps and units.", + ), + model: str = typer.Option( + None, "-m", "--model", help="Model to which to apply this command." + ), + keep_alive: bool = typer.Option( + False, + "-k", + "--keep-alive", + is_flag=True, + help="Keep the process alive instead of exiting as soon as all units in error have been told that life is good.", + ), + retry: bool = typer.Option( + False, + "-r", + "--retry", + is_flag=True, + help="Whether juju should retry the last failed hook or not.", + ), + dry_run: bool = typer.Option( + False, + help="Don't actually do anything, just print out what would have been done.", + ), + rate: float = typer.Option( + 2, + help="Minimum sleep between successive runs, only applicable in keep-alive mode.", + ), +): + """This command tells all your units and applications that things are good.""" + return _this_is_fine( + targets=targets or [], + model=model, + keep_alive=keep_alive, + retry=retry, + dry_run=dry_run, + rate=rate, + ) + + +if __name__ == "__main__": + _this_is_fine(keep_alive=True) From 7869a29155524738756372233260b4d2d4c1bf24 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Fri, 8 Nov 2024 17:38:57 +0100 Subject: [PATCH 4/6] deployment graph --- jhack/utils/deployment_graph.py | 35 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/jhack/utils/deployment_graph.py b/jhack/utils/deployment_graph.py index 24a56fad..b6ad5efb 100644 --- a/jhack/utils/deployment_graph.py +++ b/jhack/utils/deployment_graph.py @@ -12,6 +12,8 @@ Relation, gather_relation_databags, AppRelationData, + get_relation_by_endpoint, + get_unit_info, ) logger = jhack_logger.getChild("graph") @@ -100,7 +102,7 @@ def get_app(app_name_, model_name_, status=None): def walk(model_name_: str, app_name_: str, graph_=None): if app_name_ in visited: - return + return graph_ visited.append(app_name_) status = get_status(model_name_) @@ -108,6 +110,16 @@ def walk(model_name_: str, app_name_: str, graph_=None): relations: List[Relation] = [] graph_[app] = relations + def _find_relation_id( + app_name__: str, + model_name__: str, + endpoint_: str, + remote_endpoint_: str, + remote_app_name_: str, + ): + # FIXME: show-unit does not give the data, how do we get it? + return 0 + def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): for endpoint, relations_meta in meta["relations"].items(): for relation in relations_meta: @@ -141,7 +153,7 @@ def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): walk(remote_model_name, remote_app_name, graph_) remote_endpoint = _find_remote_endpoint( - remote_app_meta, binding["interface"], app_name_ + remote_app_meta, app_name_, binding["interface"] ) rel = Relation( provider=app_name_, @@ -151,8 +163,12 @@ def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): interface=binding["interface"], raw_type="regular", id=_find_relation_id( - app_name_, remote_app_name, endpoint, remote_endpoint - ), # TODO, + app_name_, + model_name_, + endpoint, + remote_endpoint, + remote_app_name, + ), ) relations.append(rel) @@ -162,7 +178,8 @@ def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): if not app_name: graph = {} - for app_name in cached_juju_status(model=model_)["applications"]: + cached_status = cached_juju_status(model=model_, json=True) + for app_name in cached_status["applications"]: graph = walk(model_, app_name, graph) else: @@ -173,7 +190,7 @@ def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): def plot(self): print("GRAPH:") for origin, relations in self._graph.items(): - print(f"\t{origin.name} ({origin.charm_name}) --> {{") + print(f"\t{origin.name} ({origin.charm_name}) :: {{") for relation in relations: print( f"\t\t({relation.provider}) {relation.provider_endpoint} >> " @@ -195,8 +212,8 @@ def _map(app_name: str, model_name: str): def unravel( - app_name: str = typer.Argument( - ..., help="""The starting point of the graph expansion.""" + app_name: Optional[str] = typer.Argument( + None, help="""The starting point of the graph expansion.""" ), model_name: str = typer.Option( None, @@ -209,4 +226,4 @@ def unravel( if __name__ == "__main__": - Graph.bootstrap("traefik").plot() + Graph.bootstrap().plot() From d71cd7c3d9827b784f925a0180b4a29edf718c0f Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Fri, 8 Nov 2024 17:39:29 +0100 Subject: [PATCH 5/6] unravel --- jhack/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jhack/main.py b/jhack/main.py index aeec7cf6..93bdcf09 100755 --- a/jhack/main.py +++ b/jhack/main.py @@ -70,7 +70,7 @@ def devmode_only(command): from jhack.utils.tail_charms import tail_events from jhack.utils.tail_logs import tail_logs from jhack.utils.unbork_juju import unbork_juju - from jhack.utils.unleash import vanity + from jhack.utils.unleash import vanity, vanity_2 from jhack.utils.deployment_graph import unravel if "--" in sys.argv: @@ -85,7 +85,6 @@ def devmode_only(command): utils.command(name="record", no_args_is_help=True)(record) utils.command(name="ffwd")(fast_forward) utils.command(name="print-env")(print_env) - utils.command(name="unravel", no_args_is_help=True)(unravel) utils.command(name="this-is-fine", no_args_is_help=True)(devmode_only(this_is_fine)) utils.command(name="unbork-juju")(devmode_only(unbork_juju)) @@ -132,8 +131,10 @@ def devmode_only(command): app.command(name="tail")(tail_events) app.command(name="ffwd")(fast_forward) app.command(name="unleash", hidden=True)(vanity) + app.command(name="is", hidden=True)(vanity_2) app.command(name="jenv")(print_env) app.command(name="list-endpoints")(list_endpoints) + app.command(name="unravel")(unravel) # DEVMODE ONLY COMMANDS def _test_devmode(): From 60f2240ed7a23f1e663d3a649c91a731724903c4 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Fri, 22 Nov 2024 10:08:50 +0100 Subject: [PATCH 6/6] deployment graph WIP --- jhack/utils/deployment_graph.py | 99 ++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/jhack/utils/deployment_graph.py b/jhack/utils/deployment_graph.py index fc828852..0fbbf00c 100644 --- a/jhack/utils/deployment_graph.py +++ b/jhack/utils/deployment_graph.py @@ -1,5 +1,7 @@ import dataclasses -from typing import List, Dict, Optional, Tuple +import re +from code import interact +from typing import List, Dict, Optional, Tuple, Set import typer @@ -18,6 +20,14 @@ logger = jhack_logger.getChild("graph") +identifier = "[a-zA-Z-_0-9]" +RELATIONS_STATUS_RE = re.compile( + f"(?P{identifier}+):(?P{identifier}+)\s+" + f"(?P{identifier}+):(?P{identifier}+)\s+" + f"(?P{identifier}+)\s+(?P{identifier}+)" + f"[\s+(?P.+)]?" +) + @dataclasses.dataclass(frozen=True) class _App: @@ -54,8 +64,12 @@ def get_relation_data(self, relation: Relation): if not self._relation_data.get(relation): self._relation_data[relation] = gather_relation_databags( - RelationEndpointURL(relation.requirer_endpoint), - RelationEndpointURL(relation.provider_endpoint), + RelationEndpointURL( + f"{relation.requirer}:{relation.requirer_endpoint}" + ), + RelationEndpointURL( + f"{relation.provider}:{relation.provider_endpoint}" + ), relation, model=self._model, include_default_juju_keys=self._include_default_juju_keys, @@ -86,52 +100,34 @@ def bootstrap(app_name: Optional[str] = None, model_name: str = None) -> "Graph" model_status_cache = {} - def get_status(model_name_): - if model_name_ not in model_status_cache: - model_status_cache[model_name_] = juju_status( - model=model_name_, json=True + def get_status(model_name_, json: bool): + if (model_name_, json) not in model_status_cache: + model_status_cache[(model_name_, json)] = juju_status( + model=model_name_, json=json ) - return model_status_cache[model_name_] + return model_status_cache[(model_name_, json)] def get_app(app_name_, model_name_, status=None): - status = status or get_status(model_name_) + status = status or get_status(model_name_, json=True) app_meta = status["applications"][app_name_] return _App(name=app_name_, model=model_name_, meta=app_meta) - visited: List[str] = [] + visited_applications: Set[str] = set() + visited_relations: Set[Relation] = set() def walk(model_name_: str, app_name_: str, graph_=None): - if app_name_ in visited: + if app_name_ in visited_applications: return graph_ - visited.append(app_name_) - status = get_status(model_name_) + visited_applications.add(app_name_) + + model_status_raw = get_status(model_name_, json=False) + model_relations = RELATIONS_STATUS_RE.findall(model_status_raw) + app = get_app(app_name_, model_name_) relations: List[Relation] = [] graph_[app] = relations - def _find_relation_id( - app_name__: str, - model_name__: str, - endpoint_: str, - remote_endpoint_: str, - remote_app_name_: str, - ): - # FIXME: show-unit does not give the data, how do we get it? - return 0 - - def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): - for endpoint, relations_meta in meta["relations"].items(): - for relation in relations_meta: - if ( - relation["interface"] == interface - and relation["related-application"] == local_app_name - ): - return endpoint - raise RuntimeError( - f"could not find a remote endpoint bound to {local_app_name} over {interface}" - ) - offers_meta = status.get("application-endpoints", ()) for endpoint, bindings in app.meta["relations"].items(): @@ -155,13 +151,35 @@ def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): remote_endpoint = _find_remote_endpoint( remote_app_meta, app_name_, binding["interface"] ) + + if remote_model_name == model_name_: + # todo subordinate check + if remote_app_name == app_name_: + relation_type = "peer" + else: + relation_type = "regular" + else: + relation_type = "cross_model" + + # todo compare provider, provider-endpoint, requirer.... with the raw relation data + + rel_footprint = ( + app_name_, + endpoint, + remote_app_name, + remote_endpoint, + binding["interface"], + ) + if rel_footprint not in model_relations: + continue + rel = Relation( provider=app_name_, provider_endpoint=endpoint, requirer=remote_app_name, requirer_endpoint=remote_endpoint, # todo interface=binding["interface"], - raw_type="regular", + raw_type=relation_type, id=_find_relation_id( app_name_, model_name_, @@ -170,7 +188,12 @@ def _find_remote_endpoint(meta: dict, local_app_name: str, interface: str): remote_app_name, ), ) + + if rel in visited_relations: + continue + relations.append(rel) + visited_relations.add(rel) return graph_ @@ -198,7 +221,7 @@ def plot(self): ) if reldata := self.get_relation_data(relation): print( - f"\t\tRelation found: " + f"\t\t\tRelation found: " f"{reldata[0].url} --> " f"{reldata[1].url if len(reldata) == 2 else ''} " f"({relation.id})" @@ -227,4 +250,4 @@ def unravel( if __name__ == "__main__": # jhack unravel traefik/0 - Graph.bootstrap().plot() + Graph.bootstrap("tempo/0").plot()