From 3cf745eee57a9c1f565236900faba0e26eaae4e7 Mon Sep 17 00:00:00 2001 From: Phillip Stephens Date: Mon, 18 Dec 2023 21:04:46 -0800 Subject: [PATCH] Update project.py, add recursive submodules to CI Former-commit-id: ae888d4b47eb28919777d66cc01a09e09a46ad36 --- .github/workflows/build.yml | 3 ++ configure.py | 2 +- tools/project.py | 105 +++++++++++++++++++++++------------- 3 files changed, 72 insertions(+), 38 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4403ad93..a353e4a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive - name: Git config run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Prepare diff --git a/configure.py b/configure.py index 8d28e4f8..e52eab2b 100755 --- a/configure.py +++ b/configure.py @@ -267,7 +267,7 @@ def MusyX(objects, mw_version="GC/1.3.2", debug=False, major=2, minor=0, patch=0 return { "lib": "musyx", "mw_version": mw_version, - "extern": "extern/musyx/src", + "src_dir": "extern/musyx/src", "host": False, "cflags": [ *cflags, diff --git a/tools/project.py b/tools/project.py index e9bd7808..8ccfde1a 100644 --- a/tools/project.py +++ b/tools/project.py @@ -57,13 +57,13 @@ class ProjectConfig: self.version = None # Version name self.warn_missing_config = False # Warn on missing unit configuration self.warn_missing_source = False # Warn on missing source file + self.rel_strip_partial = True # Generate PLFs with -strip_partial + self.rel_empty_file = None # Path to empty.c for generating empty RELs # Progress output and progress.json config self.progress_all = True # Include combined "all" category self.progress_modules = True # Include combined "modules" category - self.progress_each_module = ( - True # Include individual modules, disable for large numbers of modules - ) + self.progress_each_module = True # Include individual modules, disable for large numbers of modules # Progress fancy printing self.progress_use_fancy = False @@ -106,6 +106,7 @@ class Object: self.options = { "add_to_all": True, "cflags": None, + "extra_cflags": None, "mw_version": None, "shiftjis": True, "source": name, @@ -469,7 +470,12 @@ def generate_build_ninja(config, build_config): preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" - plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}" + plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}" + if self.entry: + plf_ldflags += f" -m {self.entry}" + # -strip_partial is only valid with -m + if config.rel_strip_partial: + plf_ldflags += " -strip_partial" if config.generate_map: preplf_map = map_path(preplf_path) preplf_ldflags += f" -map {preplf_map}" @@ -503,6 +509,12 @@ def generate_build_ninja(config, build_config): host_source_inputs = [] source_added = set() + def make_cflags_str(cflags): + if isinstance(cflags, list): + return " ".join(cflags) + else: + return cflags + def add_unit(build_obj, link_step): obj_path, obj_name = build_obj["object"], build_obj["name"] result = config.find_object(obj_name) @@ -514,32 +526,24 @@ def generate_build_ninja(config, build_config): lib, obj = result lib_name = lib["lib"] + src_dir = Path(lib.get("src_dir", config.src_dir)) options = obj.options completed = obj.completed - # For extern sources we need to use the specified source directory - if "extern" in lib.keys(): - lib_path = Path(lib["extern"]) - if not lib_path.exists(): - sys.exit(f"Specified extern path '{lib_path}' not found") - - unit_src_path = lib_path / options["source"] - else: - unit_src_path = config.src_dir / options["source"] + unit_src_path = src_dir / options["source"] if not unit_src_path.exists(): - if config.warn_missing_source: + if config.warn_missing_source or completed: print(f"Missing source file {unit_src_path}") link_step.add(obj_path) return mw_version = options["mw_version"] or lib["mw_version"] - cflags = options["cflags"] or lib["cflags"] - if type(cflags) is list: - cflags_str = " ".join(cflags) - else: - cflags_str = str(cflags) + cflags_str = make_cflags_str(options["cflags"] or lib["cflags"]) + if options["extra_cflags"] is not None: + extra_cflags_str = make_cflags_str(options["extra_cflags"]) + cflags_str += " " + extra_cflags_str used_compiler_versions.add(mw_version) base_object = Path(obj.name).with_suffix("") @@ -600,6 +604,18 @@ def generate_build_ninja(config, build_config): module_link_step = LinkStep(module) for unit in module["units"]: add_unit(unit, module_link_step) + # Add empty object to empty RELs + if len(module_link_step.inputs) == 0: + if not config.rel_empty_file: + sys.exit("ProjectConfig.rel_empty_file missing") + add_unit( + { + "object": None, + "name": config.rel_empty_file, + "autogenerated": True, + }, + module_link_step, + ) link_steps.append(module_link_step) n.newline() @@ -624,18 +640,41 @@ def generate_build_ninja(config, build_config): ### # Generate RELs ### - rel_outputs = list( - map( - lambda step: step.output(), - filter(lambda step: step.module_id != 0, link_steps), + generated_rels = [] + for link in build_config["links"]: + # Map module names to link steps + link_steps_local = list( + filter( + lambda step: step.name in link["modules"], + link_steps, + ) + ) + link_steps_local.sort(key=lambda step: step.module_id) + # RELs can be the output of multiple link steps, + # so we need to filter out duplicates + rels_to_generate = list( + filter( + lambda step: step.module_id != 0 + and not step.name in generated_rels, + link_steps_local, + ) + ) + if len(rels_to_generate) == 0: + continue + generated_rels.extend(map(lambda step: step.name, rels_to_generate)) + rel_outputs = list( + map( + lambda step: step.output(), + rels_to_generate, + ) ) - ) - if len(rel_outputs) > 0: n.comment("Generate RELs") n.build( outputs=path(rel_outputs), rule="makerel", - inputs=path(list(map(lambda step: step.partial_output(), link_steps))), + inputs=path( + list(map(lambda step: step.partial_output(), link_steps_local)) + ), implicit=path([dtk, config.config_path]), variables={"config": path(config.config_path)}, ) @@ -844,17 +883,9 @@ def generate_objdiff_config(config, build_config): return lib, obj = result - lib_name = lib["lib"] + src_dir = Path(lib.get("src_dir", config.src_dir)) - # For extern sources we need to use the specified source directory - if "extern" in lib.keys(): - lib_path = Path(lib["extern"]) - if not lib_path.exists(): - sys.exit(f"Specified extern path '{lib_path}' not found") - - unit_src_path = Path(lib["extern"]) / obj.options["source"] - else: - unit_src_path = config.src_dir / obj.options["source"] + unit_src_path = src_dir / obj.options["source"] if not unit_src_path.exists(): objdiff_config["units"].append(unit_config) @@ -975,7 +1006,7 @@ def calculate_progress(config): def print_category(unit): if unit is None: return - + code_frac = unit.code_frac() data_frac = unit.data_frac() print(