diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 8d31b78..b0d3fe1 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -12,13 +12,7 @@ from rich import print from vinca import config -from vinca.distro import Distro -from vinca.main import ( - generate_outputs, - get_conda_subdir, - get_selected_packages, - read_vinca_yaml, -) +from vinca.main import get_conda_subdir, read_vinca_yaml from vinca.pipeline import batch_stages, get_all_ancestors, get_skip_existing from vinca.utils import ( NoAliasDumper, @@ -437,51 +431,53 @@ def build_win_pipeline( dump_for_gha(workflow, outfile) -def get_full_tree(): +def load_configuration(): + """Read vinca.yaml next to the recipes directory for the few settings the + workflow needs (distribution, pixi versions); the recipes themselves are + already generated, so the packages are not resolved again.""" parsed_args = config.parsed_args if parsed_args is None: - raise RuntimeError("Pipeline arguments must be parsed before generating a tree") - recipes_dir = parsed_args.dir - - vinca_yaml = os.path.join(os.path.dirname(recipes_dir), "vinca.yaml") - - temp_vinca_conf = read_vinca_yaml(vinca_yaml) - temp_vinca_conf["build_all"] = True - temp_vinca_conf["skip_built_packages"] = [] + raise RuntimeError( + "Pipeline arguments must be parsed before reading vinca.yaml" + ) + vinca_yaml = os.path.join(os.path.dirname(parsed_args.dir), "vinca.yaml") + vinca_conf = read_vinca_yaml(vinca_yaml) config.selected_platform = get_conda_subdir() + return vinca_conf - python_version = temp_vinca_conf.get("python_version", None) - distro = Distro( - temp_vinca_conf["ros_distro"], - python_version, - temp_vinca_conf["_snapshot"], - temp_vinca_conf["_additional_packages_snapshot"], - ) - - all_packages = get_selected_packages(distro, temp_vinca_conf) - temp_vinca_conf["_selected_pkgs"] = all_packages - all_outputs = generate_outputs(distro, temp_vinca_conf) - return all_outputs +def get_recipe_requirements(recipes): + """Host and run dependency names of each recipe, keyed by package name.""" + requirements = {} + for pkg in recipes: + if "outputs" in pkg: + req_section = pkg["outputs"][0]["requirements"] + else: + req_section = pkg.get("requirements") or {} + requirements[pkg["package"]["name"]] = extract_dependency_names( + (req_section.get("host") or []) + (req_section.get("run") or []) + ) + return requirements def main(): args = parse_command_line(sys.argv) - full_tree = get_full_tree() + load_configuration() setup_pixi_version = config.setup_pixi_version or DEFAULT_SETUP_PIXI_VERSION pixi_version = config.pixi_version or DEFAULT_PIXI_VERSION metas = [] - additional_recipes = [] if args.additional_recipes: - additional_recipes = add_additional_recipes(args) + # copied into args.dir, so they are picked up with the generated recipes + add_additional_recipes(args) if not os.path.exists(args.dir): print(f"{args.dir} not found. Not generating a pipeline.") - all_recipes = glob.glob(os.path.join(args.dir, "**", "*.yaml")) + # sorted: the stage and batch order follows the order the recipes are read in + all_recipes = sorted(glob.glob(os.path.join(args.dir, "**", "*.yaml"))) for f in all_recipes: with open(f, encoding="utf-8") as fi: metas.append(yaml.safe_load(fi.read())) @@ -489,20 +485,10 @@ def main(): platform = args.platform if len(metas) >= 1: - requirements = {} - - for pkg in full_tree + additional_recipes: - if "outputs" in pkg: - req_section = pkg["outputs"][0]["requirements"] - else: - req_section = pkg["requirements"] - requirements[pkg["package"]["name"]] = req_section.get( - "host", [] - ) + req_section.get("run", []) - - # Normalize direct and conditional requirements to package names. - for pkg_name, reqs in requirements.items(): - requirements[pkg_name] = extract_dependency_names(reqs) + # The stages only order the recipes being built, so their own requirements + # are enough: resolving every package of the distribution again (as + # generate-recipes already did) would only add packages that are not built. + requirements = get_recipe_requirements(metas) test_requirements = add_test_requirements(requirements, metas) diff --git a/vinca/test_generate_gha.py b/vinca/test_generate_gha.py index 3c6f5ca..1d4cb1f 100644 --- a/vinca/test_generate_gha.py +++ b/vinca/test_generate_gha.py @@ -1,11 +1,14 @@ """Tests for the GitHub Actions pipeline generation.""" +import sys + import pytest -from vinca import config +from vinca import config, generate_gha from vinca.generate_gha import ( build_unix_pipeline, build_win_pipeline, + get_recipe_requirements, get_setup_pixi_step, get_stage_name, ) @@ -122,6 +125,57 @@ def test_win_pipeline_requires_the_repository_build_script(tmp_path, monkeypatch ) +def test_recipe_requirements_are_host_and_run_names(): + recipe = { + "package": {"name": "ros2-b", "version": "1.0.0"}, + "requirements": { + "build": ["cmake"], + "host": ["ros2-a ==1.0.0", {"if": "linux", "then": ["ros2-linux-only"]}], + "run": ["ros2-a", "python"], + }, + } + + assert get_recipe_requirements([recipe]) == { + "ros2-b": ["ros2-a", "ros2-linux-only", "python"] + } + + +def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch): + yaml = pytest.importorskip("yaml") + monkeypatch.chdir(tmp_path) + (tmp_path / "vinca.yaml").write_text("{}\n") + recipes = { + "ros2-a": [], + # ros2-published is not in ./recipes (already built): no stage of its own + "ros2-b": ["ros2-a", "ros2-published"], + "ros2-c": ["ros2-b"], + } + for name, deps in recipes.items(): + (tmp_path / "recipes" / name).mkdir(parents=True) + recipe = { + "package": {"name": name, "version": "1.0.0"}, + "requirements": {"host": deps, "run": deps}, + } + (tmp_path / "recipes" / name / "recipe.yaml").write_text(yaml.safe_dump(recipe)) + # no resolution of the ROS distribution: only the recipes are read + monkeypatch.setattr(generate_gha, "load_configuration", lambda: None) + monkeypatch.setattr( + sys, + "argv", + # batch size 1: consecutive small stages are not merged into one job + ["vinca-gha", "-d", "./recipes", "-t", "branch", "-p", "linux-64", "-b", "1"], + ) + + generate_gha.main() + + jobs = yaml.safe_load((tmp_path / "linux.yml").read_text())["jobs"] + assert [job["steps"][-1]["env"]["CURRENT_RECIPES"] for job in jobs.values()] == [ + "ros2-a", + "ros2-b", + "ros2-c", + ] + + def test_only_later_stages_declare_needs(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / ".scripts").mkdir()