diff --git a/concore_cli/README.md b/concore_cli/README.md index 55546c0..7264d14 100644 --- a/concore_cli/README.md +++ b/concore_cli/README.md @@ -62,12 +62,21 @@ Generates and optionally builds a workflow from a GraphML file. - `-o, --output ` - Output directory (default: out) - `-t, --type ` - Execution type: windows, posix, or docker (default: windows) - `--auto-build` - Automatically run build script after generation +- `--compose` - Generate `docker-compose.yml` (only valid with `--type docker`) **Example:** ```bash concore run workflow.graphml --source ./src --output ./build --auto-build ``` +Docker compose example: + +```bash +concore run workflow.graphml --source ./src --output ./out --type docker --compose +cd out +docker compose up +``` + ### `concore validate ` Validates a GraphML workflow file before running. diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 32061c0..4d7dd64 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -70,10 +70,23 @@ def init(name, template, interactive): @click.option( "--auto-build", is_flag=True, help="Automatically run build after generation" ) -def run(workflow_file, source, output, type, auto_build): +@click.option( + "--compose", + is_flag=True, + help="Generate docker-compose.yml in output directory (docker type only)", +) +def run(workflow_file, source, output, type, auto_build, compose): """Run a concore workflow""" try: - run_workflow(workflow_file, source, output, type, auto_build, console) + run_workflow( + workflow_file, + source, + output, + type, + auto_build, + console, + compose=compose, + ) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py index ad1c23c..de7bed2 100644 --- a/concore_cli/commands/run.py +++ b/concore_cli/commands/run.py @@ -1,5 +1,7 @@ -import sys +import re +import shlex import subprocess +import sys from pathlib import Path from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn @@ -15,7 +17,113 @@ def _find_mkconcore_path(): return None -def run_workflow(workflow_file, source, output, exec_type, auto_build, console): +def _yaml_quote(value): + return "'" + value.replace("'", "''") + "'" + + +def _parse_docker_run_line(line): + text = line.strip() + if not text or text.startswith("#"): + return None + + if text.endswith("&"): + text = text[:-1].strip() + + try: + tokens = shlex.split(text) + except ValueError: + return None + + if "run" not in tokens: + return None + + run_index = tokens.index("run") + args = tokens[run_index + 1 :] + + container_name = None + volumes = [] + image = None + + i = 0 + while i < len(args): + token = args[i] + if token.startswith("--name="): + container_name = token.split("=", 1)[1] + elif token == "--name" and i + 1 < len(args): + container_name = args[i + 1] + i += 1 + elif token in ("-v", "--volume") and i + 1 < len(args): + volumes.append(args[i + 1]) + i += 1 + elif token.startswith("--volume="): + volumes.append(token.split("=", 1)[1]) + elif token.startswith("-"): + pass + else: + image = token + break + i += 1 + + if not container_name or not image: + return None + + return { + "container_name": container_name, + "volumes": volumes, + "image": image, + } + + +def _write_docker_compose(output_path): + run_script = output_path / "run" + if not run_script.exists(): + return None + + services = [] + for line in run_script.read_text(encoding="utf-8").splitlines(): + parsed = _parse_docker_run_line(line) + if parsed is not None: + services.append(parsed) + + if not services: + return None + + compose_lines = ["services:"] + + for index, service in enumerate(services, start=1): + service_name = re.sub(r"[^A-Za-z0-9_.-]", "-", service["container_name"]).strip( + "-." + ) + if not service_name: + service_name = f"service-{index}" + elif not service_name[0].isalnum(): + service_name = f"service-{service_name}" + + compose_lines.append(f" {service_name}:") + compose_lines.append(f" image: {_yaml_quote(service['image'])}") + compose_lines.append( + f" container_name: {_yaml_quote(service['container_name'])}" + ) + if service["volumes"]: + compose_lines.append(" volumes:") + for volume_spec in service["volumes"]: + compose_lines.append(f" - {_yaml_quote(volume_spec)}") + + compose_lines.append("") + compose_path = output_path / "docker-compose.yml" + compose_path.write_text("\n".join(compose_lines), encoding="utf-8") + return compose_path + + +def run_workflow( + workflow_file, + source, + output, + exec_type, + auto_build, + console, + compose=False, +): workflow_path = Path(workflow_file).resolve() source_path = Path(source).resolve() output_path = Path(output).resolve() @@ -34,8 +142,13 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): console.print(f"[cyan]Source:[/cyan] {source_path}") console.print(f"[cyan]Output:[/cyan] {output_path}") console.print(f"[cyan]Type:[/cyan] {exec_type}") + if compose: + console.print("[cyan]Compose:[/cyan] enabled") console.print() + if compose and exec_type != "docker": + raise ValueError("--compose can only be used with --type docker") + mkconcore_path = _find_mkconcore_path() if mkconcore_path is None: raise FileNotFoundError( @@ -73,6 +186,18 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): console.print( f"[green]✓[/green] Workflow generated in [cyan]{output_path}[/cyan]" ) + + if compose: + compose_path = _write_docker_compose(output_path) + if compose_path is not None: + console.print( + f"[green]✓[/green] Compose file written to [cyan]{compose_path}[/cyan]" + ) + else: + console.print( + "[yellow]Warning:[/yellow] Could not generate docker-compose.yml from run script" + ) + try: metadata_path = write_study_metadata( output_path, @@ -128,6 +253,10 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): if e.stderr: console.print(e.stderr) + run_command = "docker compose up" if compose else "./run" + if exec_type == "windows": + run_command = "run.bat" + console.print() console.print( Panel.fit( @@ -135,7 +264,7 @@ def run_workflow(workflow_file, source, output, exec_type, auto_build, console): f"To run your workflow:\n" f" cd {output_path}\n" f" {'build.bat' if exec_type == 'windows' else './build'}\n" - f" {'run.bat' if exec_type == 'windows' else './run'}", + f" {run_command}", title="Next Steps", border_style="green", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 63cd0f2..4f8f57b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -233,6 +233,105 @@ def test_run_command_docker_subdir_source_build_paths(self): self.assertIn("cp ../src/subdir/script.iport concore.iport", build_script) self.assertIn("cd ..", build_script) + def test_run_command_compose_requires_docker_type(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "posix", + "--compose", + ], + ) + self.assertNotEqual(result.exit_code, 0) + self.assertIn( + "--compose can only be used with --type docker", result.output + ) + + def test_run_command_docker_compose_single_node(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + result = self.runner.invoke(cli, ["init", "test-project"]) + self.assertEqual(result.exit_code, 0) + + result = self.runner.invoke( + cli, + [ + "run", + "test-project/workflow.graphml", + "--source", + "test-project/src", + "--output", + "out", + "--type", + "docker", + "--compose", + ], + ) + self.assertEqual(result.exit_code, 0) + + compose_path = Path("out/docker-compose.yml") + self.assertTrue(compose_path.exists()) + compose_content = compose_path.read_text() + self.assertIn("services:", compose_content) + self.assertIn("container_name: 'N1'", compose_content) + self.assertIn("image: 'docker-script'", compose_content) + + metadata = json.loads(Path("out/STUDY.json").read_text()) + self.assertIn("docker-compose.yml", metadata["checksums"]) + + def test_run_command_docker_compose_multi_node(self): + with self.runner.isolated_filesystem(temp_dir=self.temp_dir): + Path("src").mkdir() + Path("src/common.py").write_text( + "import concore\n\ndef step():\n return None\n" + ) + + workflow = """ + + + + + A:common.py + B:common.py + C:common.py + 0x1000_AB + 0x1001_BC + + +""" + Path("workflow.graphml").write_text(workflow) + + result = self.runner.invoke( + cli, + [ + "run", + "workflow.graphml", + "--source", + "src", + "--output", + "out", + "--type", + "docker", + "--compose", + ], + ) + self.assertEqual(result.exit_code, 0) + + compose_content = Path("out/docker-compose.yml").read_text() + self.assertIn("container_name: 'A'", compose_content) + self.assertIn("container_name: 'B'", compose_content) + self.assertIn("container_name: 'C'", compose_content) + self.assertIn("image: 'docker-common'", compose_content) + def test_run_command_shared_source_specialization_merges_edge_params(self): with self.runner.isolated_filesystem(temp_dir=self.temp_dir): Path("src").mkdir()