diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 5716ea29e351..0ae409fba622 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -502,6 +502,7 @@ def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional[Inter self.created_llvm_ir_rule = PerMachine(False, False) self.rust_crates: T.Dict[str, RustCrate] = {} self.implicit_meson_outs: T.List[str] = [] + self.pools: T.Dict[str, int] = {} self._uses_dyndeps = False self._generated_header_cache: T.Dict[str, T.List[FileOrString]] = {} # nvcc chokes on thin archives: @@ -589,7 +590,7 @@ def detect_prefix(out): match = matchre.match(line) if match: with open(tempfilename, 'ab') as binfile: - binfile.write(b'msvc_deps_prefix = ' + match.group(1) + b'\n') + binfile.write(b'msvc_deps_prefix = ' + match.group(1) + b'\n\n') return open(tempfilename, 'a', encoding='utf-8') return None @@ -601,6 +602,30 @@ def detect_prefix(out): raise MesonException(f'Could not determine vs dep dependency prefix string. output: {stderr} {stdout}') + def set_pool(self, name, depth) -> str: + self.pools[name] = depth + return 'pool = ' + name + + def get_static_link_pool(self) -> T.Optional[str]: + backend_max_links = self.environment.coredata.optstore.get_value('backend_max_links') + if backend_max_links > 0: + return self.set_pool('link_pool', backend_max_links) + return None + + def get_dynamic_link_pool(self, linker: T.DynamicLinker) -> T.Optional[str]: + backend_max_links = self.environment.coredata.optstore.get_value('backend_max_links') + if backend_max_links > 0: + return self.set_pool('link_pool', backend_max_links) + + if linker is None: + return None + depth = linker.get_default_pool_depth() + if depth == 0: + return None + + name = 'linker_{}_pool'.format(linker.get_id()) + return self.set_pool(name, depth) + def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None) -> T.Optional[T.Dict]: if vslite_ctx: # We don't yet have a use case where we'd expect to make use of this, @@ -632,13 +657,6 @@ def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None) outfile.write('# Do not edit by hand.\n\n') outfile.write('ninja_required_version = 1.8.2\n\n') - num_pools = self.environment.coredata.optstore.get_value('backend_max_links') - if num_pools > 0: - outfile.write(f'''pool link_pool - depth = {num_pools} - -''') - with self.detect_vs_dep_prefix(tempfilename) as outfile: self.generate_rules() @@ -682,6 +700,7 @@ def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None) mlog.log_timestamp("Utils generated") self.generate_ending() + self.write_pools(outfile) self.write_rules(outfile) self.write_builds(outfile) @@ -1412,6 +1431,13 @@ def write_builds(self, outfile: T.TextIO) -> None: b.write(outfile) mlog.log_timestamp("build.ninja generated") + def write_pools(self, outfile: T.TextIO) -> None: + for name, depth in self.pools.items(): + outfile.write(f'''pool {name} + depth = {depth} + +''') + def generate_phony(self) -> None: self.add_build_comment(NinjaComment('Phony build target, always out of date')) elem = NinjaBuildElement(self.all_outputs, 'PHONY', 'phony', '') @@ -2305,7 +2331,6 @@ def _rsp_options(self, tool: T.Union['Compiler', 'StaticLinker', 'DynamicLinker' return options def generate_static_link_rules(self) -> None: - num_pools = self.environment.coredata.optstore.get_value('backend_max_links') if 'java' in self.environment.coredata.compilers.host: self.generate_java_link() for for_machine in MachineChoice: @@ -2344,16 +2369,12 @@ def generate_static_link_rules(self) -> None: ranlib = ['ranlib'] cmdlist.extend(['&&'] + ranlib + ['-c', '$out']) description = 'Linking static target $out' - if num_pools > 0: - pool = 'pool = link_pool' - else: - pool = None + pool = self.get_static_link_pool() options = self._rsp_options(static_linker) self.add_rule(NinjaRule(rule, cmdlist, args, description, **options, extra=pool)) def generate_dynamic_link_rules(self) -> None: - num_pools = self.environment.coredata.optstore.get_value('backend_max_links') for for_machine in MachineChoice: complist = self.environment.coredata.compilers[for_machine] for langname, compiler in complist.items(): @@ -2363,10 +2384,7 @@ def generate_dynamic_link_rules(self) -> None: command = compiler.get_linker_exelist() args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_linker_output_args('$out'), Quoting.none) + ['$in', '$LINK_ARGS'] description = 'Linking target $out' - if num_pools > 0: - pool = 'pool = link_pool' - else: - pool = None + pool = self.get_dynamic_link_pool(compiler.linker) options = self._rsp_options(compiler) self.add_rule(NinjaRule(rule, command, args, description, **options, extra=pool)) diff --git a/mesonbuild/linkers/linkers.py b/mesonbuild/linkers/linkers.py index d0ffc56182ee..29ea20ffbb36 100644 --- a/mesonbuild/linkers/linkers.py +++ b/mesonbuild/linkers/linkers.py @@ -298,6 +298,13 @@ def get_command_to_archive_shlib(self) -> T.List[str]: #Only used by AIX. return [] + def get_default_pool_depth(self) -> int: + # Returns the ideal number of concurrent invocations. + # Returning 0 means no limit, so the default concurrency + # value (tipically the number of logical processors) is + # used + return 0 + if T.TYPE_CHECKING: StaticLinkerBase = StaticLinker @@ -1368,6 +1375,12 @@ def get_win_subsystem_args(self, value: str) -> T.List[str]: def fatal_warnings(self) -> T.List[str]: return ['-WX'] + def get_default_pool_depth(self) -> int: + # MS link.exe is internally multithreaded and uses lots of memory. + # we might check the amount of physical memory here, but it's not + # clear if parallel invocations of the linker bring any advantage. + return 1 + class ClangClDynamicLinker(VisualStudioLikeLinkerMixin, DynamicLinker):