diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 3c26b39c93fe..24c8d42d9e23 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -16,6 +16,7 @@ import pickle import re import subprocess +import subprocess as sp import typing as T from . import backends @@ -508,8 +509,11 @@ def __init__(self, build: T.Optional[build.Build]): # - https://github.com/mesonbuild/meson/pull/9453 # - https://github.com/mesonbuild/meson/issues/9479#issuecomment-953485040 self.allow_thin_archives = PerMachine[bool](True, True) + self._first_deps_dd_rule_generated = False + self._all_scan_sources = [] self.import_std: T.Optional[ImportStdInfo] = None + def create_phony_target(self, dummy_outfile: str, rulename: str, phony_infilename: str) -> NinjaBuildElement: ''' We need to use aliases for targets that might be used as directory @@ -654,6 +658,7 @@ def generate(self, capture: bool = False, vslite_ctx: T.Optional[T.Dict] = None) for t in ProgressBar(self.build.get_targets().values(), desc='Generating targets'): self.generate_target(t) + self.generate_global_dependency_scan_target() mlog.log_timestamp("Targets generated") self.add_build_comment(NinjaComment('Test rules')) self.generate_tests() @@ -1097,7 +1102,8 @@ def generate_target(self, target: T.Union[build.BuildTarget, build.CustomTarget, final_obj_list = self.generate_prelink(target, obj_list) else: final_obj_list = obj_list - + if self.should_use_dyndeps_for_target(target): + self._all_scan_sources.extend(compiled_sources) self.generate_dependency_scan_target(target, compiled_sources, source2object, fortran_order_deps) if isinstance(target, build.SharedLibrary): @@ -1128,6 +1134,8 @@ def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool: return True # Currently only the preview version of Visual Studio is supported. cpp = target.compilers['cpp'] + if cpp.get_id() == 'clang': + return True if cpp.get_id() != 'msvc': return False cppversion = self.get_target_option(target, OptionKey('cpp_std', @@ -1141,6 +1149,32 @@ def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool: return False return True + def generate_project_wide_cpp_scanner_rules(self) -> None: + rulename = 'depscanaccumulate' + if rulename in self.ruledict: + return + command = self.environment.get_build_command() + \ + ['--internal', 'depscanaccumulate'] + args = ['$in', 'deps.json', '$out'] + description = 'Scanning project for modules' + rule = NinjaRule(rulename, command, args, description) + self.add_rule(rule) + + rulename = 'cpp_module_precompile' + command = ['$COMPILER', '--precompile', '-x', 'c++-module', '$in', '-o', '$out'] + args = ['$ARGS'] + description = 'Precompiling C++ module $in' + rule = NinjaRule(rulename, command, args, description) + self.add_rule(rule) + + def generate_global_dependency_scan_target(self) -> None: + self._uses_dyndeps = True + self.generate_project_wide_cpp_scanner_rules() + rule_name = 'depscanaccumulate' + elem = NinjaBuildElement(self.all_outputs, "deps.dd", rule_name, "compile_commands.json") + elem.add_dep(self._all_scan_sources) + self.add_build(elem) + def generate_dependency_scan_target(self, target: build.BuildTarget, compiled_sources: T.List[str], source2object: T.Dict[str, str], @@ -1148,46 +1182,7 @@ def generate_dependency_scan_target(self, target: build.BuildTarget, if not self.should_use_dyndeps_for_target(target): return self._uses_dyndeps = True - json_file, depscan_file = self.get_dep_scan_file_for(target) - pickle_base = target.name + '.dat' - pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/') - pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/') - rule_name = 'depscan' - scan_sources = list(self.select_sources_to_scan(compiled_sources)) - - scaninfo = TargetDependencyScannerInfo( - self.get_target_private_dir(target), source2object, scan_sources) - - write = True - if os.path.exists(pickle_abs): - with open(pickle_abs, 'rb') as p: - old = pickle.load(p) - write = old != scaninfo - - if write: - with open(pickle_abs, 'wb') as p: - pickle.dump(scaninfo, p) - - elem = NinjaBuildElement(self.all_outputs, json_file, rule_name, pickle_file) - # A full dependency is required on all scanned sources, if any of them - # are updated we need to rescan, as they may have changed the modules - # they use or export. - for s in scan_sources: - elem.deps.add(s[0]) - elem.orderdeps.update(object_deps) - elem.add_item('name', target.name) - self.add_build(elem) - - infiles: T.Set[str] = set() - for t in target.get_all_linked_targets(): - if self.should_use_dyndeps_for_target(t): - infiles.add(self.get_dep_scan_file_for(t)[0]) - _, od = self.flatten_object_list(target) - infiles.update({self.get_dep_scan_file_for(t)[0] for t in od if t.uses_fortran()}) - - elem = NinjaBuildElement(self.all_outputs, depscan_file, 'depaccumulate', [json_file] + sorted(infiles)) - elem.add_item('name', target.name) - self.add_build(elem) + self._all_scan_sources.extend(compiled_sources) def select_sources_to_scan(self, compiled_sources: T.List[str], ) -> T.Iterable[T.Tuple[str, Literal['cpp', 'fortran']]]: @@ -2750,21 +2745,7 @@ def generate_scanner_rules(self) -> None: if rulename in self.ruledict: # Scanning command is the same for native and cross compilation. return - - command = self.environment.get_build_command() + \ - ['--internal', 'depscan'] - args = ['$picklefile', '$out', '$in'] - description = 'Scanning target $name for modules' - rule = NinjaRule(rulename, command, args, description) - self.add_rule(rule) - - rulename = 'depaccumulate' - command = self.environment.get_build_command() + \ - ['--internal', 'depaccumulate'] - args = ['$out', '$in'] - description = 'Generating dynamic dependency information for target $name' - rule = NinjaRule(rulename, command, args, description) - self.add_rule(rule) + self.generate_project_wide_cpp_scanner_rules() def generate_compile_rules(self) -> None: for for_machine in MachineChoice: @@ -2779,6 +2760,28 @@ def generate_compile_rules(self) -> None: for mode in compiler.get_modes(): self.generate_compile_rule_for(langname, mode) + def _get_cpp_module_output_name(self, src, compiler, target) -> T.Tuple[str, T.List[str]]: + extra_args = target.extra_args.get('cpp', []) + include_dirs = [] + for inc in target.include_dirs: + for inc_dir in inc.get_incdirs(): + include_dirs.append(f'-I{os.path.join(inc.get_curdir(), inc_dir)}') + flags = extra_args + include_dirs + abs_src = src.absolute_path(self.environment.get_source_dir(), self.environment.get_build_dir()) + cmd = ["clang-scan-deps", "-format=p1689", "--", + compiler.get_exelist()[0], "-std=c++26", abs_src] + flags + result = sp.run(cmd, capture_output=True) + if result.returncode != 0: + return os.path.splitext(os.path.basename(src.fname))[0] + ".pcm", [] + info = json.loads(result.stdout) + required_pcms = [] + for rule in info.get("rules", []): + for req in rule.get("requires", []): + required_pcms.append(req["logical-name"] + ".pcm") + for provides in rule.get("provides", []): + return provides["logical-name"] + ".pcm", required_pcms + return 'dummy', [] + def generate_generator_list_rules(self, target) -> None: # CustomTargets have already written their rules and # CustomTargetIndexes don't actually get generated, so write rules for @@ -3297,6 +3300,19 @@ def quote_make_target(targetName: str) -> str: result += c return result element.add_item('CUDA_ESCAPED_TARGET', quote_make_target(rel_obj)) + if self.should_use_dyndeps_for_target(target) and compiler.get_language() == 'cpp' and compiler.get_id() == 'clang': + mod_output, required_pcms = self._get_cpp_module_output_name(src, compiler, target) + if mod_output != 'dummy': + pcm_path = mod_output + precompile_elem = NinjaBuildElement(self.all_outputs, pcm_path, 'cpp_module_precompile', rel_src) + precompile_elem.add_item('COMPILER', compiler.get_exelist()[0]) + precompile_elem.add_item('ARGS', list(commands) + [f'-fprebuilt-module-path={self.environment.get_build_dir()}']) + for req in required_pcms: + precompile_elem.add_orderdep(req) + self.add_build(precompile_elem) + commands.extend([ + f'-fprebuilt-module-path={self.environment.get_build_dir()}' + ]) element.add_item('ARGS', commands) self.add_dependency_scanner_entries_to_element(target, compiler, element, src) @@ -3367,7 +3383,7 @@ def add_dependency_scanner_entries_to_element(self, target: build.BuildTarget, c extension = extension.lower() if not (extension in compilers.lang_suffixes['fortran'] or extension in compilers.lang_suffixes['cpp']): return - dep_scan_file = self.get_dep_scan_file_for(target)[1] + dep_scan_file = 'deps.dd' element.add_item('dyndep', dep_scan_file) element.add_orderdep(dep_scan_file) diff --git a/mesonbuild/compilers/cpp.py b/mesonbuild/compilers/cpp.py index 5d319f9939eb..a2be0b5b9480 100644 --- a/mesonbuild/compilers/cpp.py +++ b/mesonbuild/compilers/cpp.py @@ -179,6 +179,8 @@ def get_options(self) -> 'MutableKeyedOptionDictType': }) return opts + def get_cpp20_module_bmi_extension(self) -> str: + raise MesonException("Your compiler does not support 'import std' feature or it has not been implemented") class _StdCPPLibMixin(CompilerMixinBase): @@ -332,6 +334,8 @@ def get_cpp_modules_args(self) -> T.List[str]: # Although -fmodules-ts is removed in LLVM 17, we keep this in for compatibility with old compilers. return ['-fmodules', '-fmodules-ts'] + def get_cpp20_module_bmi_extension(self) -> str: + return '.pcm' class ArmLtdClangCPPCompiler(ClangCPPCompiler): diff --git a/mesonbuild/scripts/depscan.py b/mesonbuild/scripts/depscan.py index 25b557e4079e..28e91cce91d8 100644 --- a/mesonbuild/scripts/depscan.py +++ b/mesonbuild/scripts/depscan.py @@ -11,6 +11,9 @@ import pickle import re import typing as T +import subprocess as sp +import shutil +from mesonbuild.compilers import cpp if T.TYPE_CHECKING: from typing_extensions import Literal, TypedDict, NotRequired @@ -201,8 +204,34 @@ def scan(self) -> int: return 0 +class CppDependenciesScanner: + pass + +class ClangDependencyScanner(CppDependenciesScanner): + def __init__(self, compilation_db_file: str, json_output_file: str, dd_output_file: str = 'deps.dd'): + self.compilation_db_file = compilation_db_file + self.json_output_file = json_output_file + self.dd_output_file = dd_output_file + self.clang_scan_deps = os.path.join(os.path.dirname(shutil.which(cpp)), 'clang-scan-deps') + def scan(self) -> int: + try: + result = sp.run( + [self.clang_scan_deps, + "-format=p1689", + "-compilation-database", self.compilation_db_file], + capture_output=True, + check=True + ) + with open(self.json_output_file, 'wb') as f: + f.write(result.stdout) + dependencies_info = json.loads(result.stdout) + all_deps_per_objfile = self.generate_dependencies(dependencies_info["rules"]) + self.generate_dd_file(all_deps_per_objfile) + return 0 + except sp.SubprocessError: + return 1 + def run(args: T.List[str]) -> int: - assert len(args) == 2, 'got wrong number of arguments!' - outfile, pickle_file = args - scanner = DependencyScanner(pickle_file, outfile) + comp_db, json_output, dd_output = args + scanner = ClangDependencyScanner(comp_db, json_output, dd_output) return scanner.scan() diff --git a/mesonbuild/scripts/depscanaccumulate.py b/mesonbuild/scripts/depscanaccumulate.py new file mode 100644 index 000000000000..798c51538cb4 --- /dev/null +++ b/mesonbuild/scripts/depscanaccumulate.py @@ -0,0 +1,95 @@ +from collections import defaultdict +from dataclasses import dataclass +import json +import os +import subprocess as sp +import sys +import typing as T +import shutil + +ModuleName: T.TypeAlias = str +ObjectFile: T.TypeAlias = str + +@dataclass(frozen=True) +class ModuleProviderInfo: + logical_name: ModuleName + source_path: str + is_interface: bool = False + +class CppDependenciesScanner: + pass + +def normalize_filename(fname): + return fname.replace(':', '-') + +class DynDepRule: + def __init__(self, out: str, imp_outs: T.Optional[T.List[str]], imp_ins: T.List[str]): + self.output = [f'build {out}'] + if imp_outs: + imp_out_str = " ".join([normalize_filename(o) for o in imp_outs]) + self.output.append(f" | {imp_out_str}") + self.output.append(": dyndep") + if imp_ins: + imp_ins_str = " ".join([normalize_filename(inf) for inf in imp_ins]) + self.output.append(" | " + imp_ins_str) + self.output_str = "".join(self.output) + "\n" + def __str__(self): + return self.output_str + +class ClangDependencyScanner(CppDependenciesScanner): + def __init__(self, compilation_db_file: str, json_output_file: str, dd_output_file: str = 'deps.dd', cpp: str = 'clang++'): + self.compilation_db_file = compilation_db_file + self.json_output_file = json_output_file + self.dd_output_file = dd_output_file + self.clang_scan_deps = os.path.join(os.path.dirname(shutil.which(cpp) or cpp), 'clang-scan-deps') + + def scan(self) -> int: + try: + r = sp.run([self.clang_scan_deps, + "-format=p1689", + "-compilation-database", self.compilation_db_file], + capture_output=True) + if r.returncode != 0: + print(r.stderr) + raise sp.SubprocessError("Failed to run command") + with open(self.json_output_file, 'wb') as f: + f.write(r.stdout) + dependencies_info = json.loads(r.stdout) + all_deps_per_objfile = self.generate_dependencies(dependencies_info["rules"]) + self.generate_dd_file(all_deps_per_objfile) + return 0 + except sp.SubprocessError: + return 1 + + def generate_dd_file(self, deps_per_object_file): + with open(self.dd_output_file, "w") as f: + f.write('ninja_dyndep_version = 1\n') + for obj, reqprov in deps_per_object_file.items(): + requires, provides = reqprov + dd = DynDepRule(obj, None, + [r + '.pcm' for r in requires]) + f.write(str(dd)) + + def generate_dependencies(self, rules: T.List): + all_entries: T.Mapping[ObjectFile, T.Tuple[T.Set[ModuleName], T.Set[ModuleProviderInfo]]] = defaultdict(lambda: (set(), set())) + for r in rules: + obj_processed = r["primary-output"] + all_entries[obj_processed] = (set(), set()) + for req in r.get("requires", []): + all_entries[obj_processed][0].add(req["logical-name"]) + for prov in r.get("provides", []): + all_entries[obj_processed][1].add(ModuleProviderInfo( + logical_name=prov["logical-name"], + source_path=prov["source-path"], + is_interface=prov.get('is-interface', False))) + return all_entries + +def run(args: T.List[str]) -> int: + assert len(args) >= 3, 'Expected [cpp_compiler] arguments' + comp_db_path, json_output_path, dd_output = args[:3] + cpp = args[3] if len(args) > 3 else 'clang++' + scanner = ClangDependencyScanner(comp_db_path, json_output_path, dd_output, cpp) + return scanner.scan() + +if __name__ == '__main__': + run(sys.argv[1:])