Update project.py, add recursive submodules to CI

Former-commit-id: ae888d4b47eb28919777d66cc01a09e09a46ad36

.github/workflows/build.yml

@@ -15,6 +15,9 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        submodules: recursive
     - name: Git config
       run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
     - name: Prepare

configure.py

@@ -267,7 +267,7 @@ def MusyX(objects, mw_version="GC/1.3.2", debug=False, major=2, minor=0, patch=0
     return {
         "lib": "musyx",
         "mw_version": mw_version,
-        "extern": "extern/musyx/src",
+        "src_dir": "extern/musyx/src",
         "host": False,
         "cflags": [
             *cflags,

tools/project.py

@@ -57,13 +57,13 @@ class ProjectConfig:
         self.version = None  # Version name
         self.warn_missing_config = False  # Warn on missing unit configuration
         self.warn_missing_source = False  # Warn on missing source file
+        self.rel_strip_partial = True  # Generate PLFs with -strip_partial
+        self.rel_empty_file = None  # Path to empty.c for generating empty RELs
 
         # Progress output and progress.json config
         self.progress_all = True  # Include combined "all" category
         self.progress_modules = True  # Include combined "modules" category
-        self.progress_each_module = (
+        self.progress_each_module = True  # Include individual modules, disable for large numbers of modules
-            True  # Include individual modules, disable for large numbers of modules
-        )
 
         # Progress fancy printing
         self.progress_use_fancy = False
@@ -106,6 +106,7 @@ class Object:
         self.options = {
             "add_to_all": True,
             "cflags": None,
+            "extra_cflags": None,
             "mw_version": None,
             "shiftjis": True,
             "source": name,
@@ -469,7 +470,12 @@ def generate_build_ninja(config, build_config):
                 preplf_path = build_path / self.name / f"{self.name}.preplf"
                 plf_path = build_path / self.name / f"{self.name}.plf"
                 preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r"
-                plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}"
+                plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}"
+                if self.entry:
+                    plf_ldflags += f" -m {self.entry}"
+                    # -strip_partial is only valid with -m
+                    if config.rel_strip_partial:
+                        plf_ldflags += " -strip_partial"
                 if config.generate_map:
                     preplf_map = map_path(preplf_path)
                     preplf_ldflags += f" -map {preplf_map}"
@@ -503,6 +509,12 @@ def generate_build_ninja(config, build_config):
         host_source_inputs = []
         source_added = set()
 
+        def make_cflags_str(cflags):
+            if isinstance(cflags, list):
+                return " ".join(cflags)
+            else:
+                return cflags
+
         def add_unit(build_obj, link_step):
             obj_path, obj_name = build_obj["object"], build_obj["name"]
             result = config.find_object(obj_name)
@@ -514,32 +526,24 @@ def generate_build_ninja(config, build_config):
 
             lib, obj = result
             lib_name = lib["lib"]
+            src_dir = Path(lib.get("src_dir", config.src_dir))
 
             options = obj.options
             completed = obj.completed
 
-            # For extern sources we need to use the specified source directory
+            unit_src_path = src_dir / options["source"]
-            if "extern" in lib.keys():
-                lib_path = Path(lib["extern"])
-                if not lib_path.exists():
-                    sys.exit(f"Specified extern path '{lib_path}' not found")
-            
-                unit_src_path = lib_path / options["source"]
-            else:
-                unit_src_path = config.src_dir / options["source"]
 
             if not unit_src_path.exists():
-                if config.warn_missing_source:
+                if config.warn_missing_source or completed:
                     print(f"Missing source file {unit_src_path}")
                 link_step.add(obj_path)
                 return
 
             mw_version = options["mw_version"] or lib["mw_version"]
-            cflags = options["cflags"] or lib["cflags"]
+            cflags_str = make_cflags_str(options["cflags"] or lib["cflags"])
-            if type(cflags) is list:
+            if options["extra_cflags"] is not None:
-                cflags_str = " ".join(cflags)
+                extra_cflags_str = make_cflags_str(options["extra_cflags"])
-            else:
+                cflags_str += " " + extra_cflags_str
-                cflags_str = str(cflags)
             used_compiler_versions.add(mw_version)
 
             base_object = Path(obj.name).with_suffix("")
@@ -600,6 +604,18 @@ def generate_build_ninja(config, build_config):
                 module_link_step = LinkStep(module)
                 for unit in module["units"]:
                     add_unit(unit, module_link_step)
+                # Add empty object to empty RELs
+                if len(module_link_step.inputs) == 0:
+                    if not config.rel_empty_file:
+                        sys.exit("ProjectConfig.rel_empty_file missing")
+                    add_unit(
+                        {
+                            "object": None,
+                            "name": config.rel_empty_file,
+                            "autogenerated": True,
+                        },
+                        module_link_step,
+                    )
                 link_steps.append(module_link_step)
         n.newline()
 
@@ -624,18 +640,41 @@ def generate_build_ninja(config, build_config):
         ###
         # Generate RELs
         ###
-        rel_outputs = list(
+        generated_rels = []
-            map(
+        for link in build_config["links"]:
-                lambda step: step.output(),
+            # Map module names to link steps
-                filter(lambda step: step.module_id != 0, link_steps),
+            link_steps_local = list(
+                filter(
+                    lambda step: step.name in link["modules"],
+                    link_steps,
+                )
+            )
+            link_steps_local.sort(key=lambda step: step.module_id)
+            # RELs can be the output of multiple link steps,
+            # so we need to filter out duplicates
+            rels_to_generate = list(
+                filter(
+                    lambda step: step.module_id != 0
+                    and not step.name in generated_rels,
+                    link_steps_local,
+                )
+            )
+            if len(rels_to_generate) == 0:
+                continue
+            generated_rels.extend(map(lambda step: step.name, rels_to_generate))
+            rel_outputs = list(
+                map(
+                    lambda step: step.output(),
+                    rels_to_generate,
+                )
             )
-        )
-        if len(rel_outputs) > 0:
             n.comment("Generate RELs")
             n.build(
                 outputs=path(rel_outputs),
                 rule="makerel",
-                inputs=path(list(map(lambda step: step.partial_output(), link_steps))),
+                inputs=path(
+                    list(map(lambda step: step.partial_output(), link_steps_local))
+                ),
                 implicit=path([dtk, config.config_path]),
                 variables={"config": path(config.config_path)},
             )
@@ -844,17 +883,9 @@ def generate_objdiff_config(config, build_config):
             return
 
         lib, obj = result
-        lib_name = lib["lib"]
+        src_dir = Path(lib.get("src_dir", config.src_dir))
 
-        # For extern sources we need to use the specified source directory
+        unit_src_path = src_dir / obj.options["source"]
-        if "extern" in lib.keys():
-            lib_path = Path(lib["extern"])
-            if not lib_path.exists():
-                sys.exit(f"Specified extern path '{lib_path}' not found")
-            
-            unit_src_path = Path(lib["extern"]) / obj.options["source"]
-        else:
-            unit_src_path = config.src_dir / obj.options["source"]
 
         if not unit_src_path.exists():
             objdiff_config["units"].append(unit_config)
@@ -975,7 +1006,7 @@ def calculate_progress(config):
     def print_category(unit):
         if unit is None:
             return
-        
+
         code_frac = unit.code_frac()
         data_frac = unit.data_frac()
         print(