Ninja: default pool depth for link.exe

mesonbuild/backend/ninjabackend.py

@@ -502,6 +502,7 @@ def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional[Inter
         self.created_llvm_ir_rule = PerMachine(False, False)
         self.rust_crates: T.Dict[str, RustCrate] = {}
         self.implicit_meson_outs: T.List[str] = []
+        self.pools: T.Dict[str, int] = {}
         self._uses_dyndeps = False
         self._generated_header_cache: T.Dict[str, T.List[FileOrString]] = {}
         # nvcc chokes on thin archives:
@@ -589,7 +590,7 @@ def detect_prefix(out):
                 match = matchre.match(line)
                 if match:
                     with open(tempfilename, 'ab') as binfile:
-                        binfile.write(b'msvc_deps_prefix = ' + match.group(1) + b'\n')
+                        binfile.write(b'msvc_deps_prefix = ' + match.group(1) + b'\n\n')
                     return open(tempfilename, 'a', encoding='utf-8')
             return None
 
@@ -601,6 +602,30 @@ def detect_prefix(out):
 
         raise MesonException(f'Could not determine vs dep dependency prefix string. output: {stderr} {stdout}')
 
+    def set_pool(self, name, depth) -> str:
+        self.pools[name] = depth
+        return 'pool = ' + name
+
+    def get_static_link_pool(self) -> T.Optional[str]:
+        backend_max_links = self.environment.coredata.optstore.get_value('backend_max_links')
+        if backend_max_links > 0:
+            return self.set_pool('link_pool', backend_max_links)
+        return None
+
+    def get_dynamic_link_pool(self, linker: T.DynamicLinker) -> T.Optional[str]:
+        backend_max_links = self.environment.coredata.optstore.get_value('backend_max_links')
+        if backend_max_links > 0:
+            return self.set_pool('link_pool', backend_max_links)
+
+        if linker is None:
+            return None
+        depth = linker.get_default_pool_depth()
+        if depth == 0:
+            return None
+
+        name = 'linker_{}_pool'.format(linker.get_id())
+        return self.set_pool(name, depth)
+
     def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None) -> T.Optional[T.Dict]:
         if vslite_ctx:
             # We don't yet have a use case where we'd expect to make use of this,
@@ -632,13 +657,6 @@ def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None)
             outfile.write('# Do not edit by hand.\n\n')
             outfile.write('ninja_required_version = 1.8.2\n\n')
 
-            num_pools = self.environment.coredata.optstore.get_value('backend_max_links')
-            if num_pools > 0:
-                outfile.write(f'''pool link_pool
-  depth = {num_pools}
-
-''')
-
         with self.detect_vs_dep_prefix(tempfilename) as outfile:
             self.generate_rules()
 
@@ -682,6 +700,7 @@ def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None)
             mlog.log_timestamp("Utils generated")
             self.generate_ending()
 
+            self.write_pools(outfile)
             self.write_rules(outfile)
             self.write_builds(outfile)
 
@@ -1412,6 +1431,13 @@ def write_builds(self, outfile: T.TextIO) -> None:
             b.write(outfile)
         mlog.log_timestamp("build.ninja generated")
 
+    def write_pools(self, outfile: T.TextIO) -> None:
+        for name, depth in self.pools.items():
+            outfile.write(f'''pool {name}
+  depth = {depth}
+
+''')
+
     def generate_phony(self) -> None:
         self.add_build_comment(NinjaComment('Phony build target, always out of date'))
         elem = NinjaBuildElement(self.all_outputs, 'PHONY', 'phony', '')
@@ -2305,7 +2331,6 @@ def _rsp_options(self, tool: T.Union['Compiler', 'StaticLinker', 'DynamicLinker'
         return options
 
     def generate_static_link_rules(self) -> None:
-        num_pools = self.environment.coredata.optstore.get_value('backend_max_links')
         if 'java' in self.environment.coredata.compilers.host:
             self.generate_java_link()
         for for_machine in MachineChoice:
@@ -2344,16 +2369,12 @@ def generate_static_link_rules(self) -> None:
                     ranlib = ['ranlib']
                 cmdlist.extend(['&&'] + ranlib + ['-c', '$out'])
             description = 'Linking static target $out'
-            if num_pools > 0:
-                pool = 'pool = link_pool'
-            else:
-                pool = None
+            pool = self.get_static_link_pool()
 
             options = self._rsp_options(static_linker)
             self.add_rule(NinjaRule(rule, cmdlist, args, description, **options, extra=pool))
 
     def generate_dynamic_link_rules(self) -> None:
-        num_pools = self.environment.coredata.optstore.get_value('backend_max_links')
         for for_machine in MachineChoice:
             complist = self.environment.coredata.compilers[for_machine]
             for langname, compiler in complist.items():
@@ -2363,10 +2384,7 @@ def generate_dynamic_link_rules(self) -> None:
                 command = compiler.get_linker_exelist()
                 args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_linker_output_args('$out'), Quoting.none) + ['$in', '$LINK_ARGS']
                 description = 'Linking target $out'
-                if num_pools > 0:
-                    pool = 'pool = link_pool'
-                else:
-                    pool = None
+                pool = self.get_dynamic_link_pool(compiler.linker)
 
                 options = self._rsp_options(compiler)
                 self.add_rule(NinjaRule(rule, command, args, description, **options, extra=pool))

mesonbuild/linkers/linkers.py

@@ -298,6 +298,13 @@ def get_command_to_archive_shlib(self) -> T.List[str]:
         #Only used by AIX.
         return []
 
+    def get_default_pool_depth(self) -> int:
+        # Returns the ideal number of concurrent invocations.
+        # Returning 0 means no limit, so the default concurrency
+        # value (tipically the number of logical processors) is
+        # used
+        return 0
+
 
 if T.TYPE_CHECKING:
     StaticLinkerBase = StaticLinker
@@ -1368,6 +1375,12 @@ def get_win_subsystem_args(self, value: str) -> T.List[str]:
     def fatal_warnings(self) -> T.List[str]:
         return ['-WX']
 
+    def get_default_pool_depth(self) -> int:
+        # MS link.exe is internally multithreaded and uses lots of memory.
+        # we might check the amount of physical memory here, but it's not
+        # clear if parallel invocations of the linker bring any advantage.
+        return 1
+
 
 class ClangClDynamicLinker(VisualStudioLikeLinkerMixin, DynamicLinker):