Quelle  gn_processor.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import hashlib
import json
import os
import subprocess
import sys
import tempfile
from collections import defaultdict, deque
from concurrent.futures import ProcessPoolExecutor, as_completed
from copy import deepcopy
from pathlib import Path
from shutil import which

import mozpack.path as mozpath
from mozbuild.bootstrap import bootstrap_toolchain
from mozbuild.dirutils import mkdir
from mozbuild.frontend.sandbox import alphabetical_sorted
from mozfile import json as mozfile_json

license_header = """# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""

generated_header = """
  ### This moz.build was AUTOMATICALLY GENERATED from a GN config,  ###
  ### DO NOT edit it by hand.                                       ###
"""

# GN's substitution pattern for the response file path in action args.
RESPONSE_FILE_NAME_FLAG = "{{response_file_name}}"


class MozbuildWriter:
    def __init__(self, fh):
        self._fh = fh
        self.indent = ""
        self._indent_increment = 4

        # We need to correlate a small amount of state here to figure out which
        # library template to use ("Library()" or "GeckoSharedLibrary()")
        self._library_name = None
        self._shared_library = None

    def mb_serialize(self, v):
        if isinstance(v, list):
            if len(v) <= 1:
                return repr(v)
            # Pretty print a list
            raw = json.dumps(v, indent=self._indent_increment)
            # Add the indent of the current indentation level
            return raw.replace("\n", "\n" + self.indent)
        if isinstance(v, bool):
            return repr(v)
        return json.dumps(v)

    def finalize(self):
        if self._library_name:
            self.write("\n")
            if self._shared_library:
                self.write_ln(
                    f"GeckoSharedLibrary({self.mb_serialize(self._library_name)})"
                )
            else:
                self.write_ln(f"Library({self.mb_serialize(self._library_name)})")

    def write(self, content):
        self._fh.write(content)

    def write_ln(self, line):
        self.write(self.indent)
        self.write(line)
        self.write("\n")

    def write_attrs(self, context_attrs):
        for k in sorted(context_attrs.keys()):
            v = context_attrs[k]
            if isinstance(v, (list, set)):
                self.write_mozbuild_list(k, v)
            elif isinstance(v, dict):
                self.write_mozbuild_dict(k, v)
            else:
                self.write_mozbuild_value(k, v)

    def write_mozbuild_list(self, key, value):
        if value:
            if key == "GeneratedFiles":
                for generated_file in value:
                    self.write("\n")
                    self.write_ln("GeneratedFile(")
                    self.indent += " " * self._indent_increment
                    for o in generated_file["outputs"]:
                        self.write_ln(f"{self.mb_serialize(o)},")
                    for k, v in sorted(generated_file.items()):
                        if k == "outputs":
                            continue
                        self.write_ln(f"{k}={self.mb_serialize(v)},")
                    self.indent = self.indent[self._indent_increment :]
                    self.write_ln(")")
                return

            self.write("\n")
            self.write(self.indent + key)
            self.write(" += [\n    " + self.indent)
            self.write(
                (",\n    " + self.indent).join(
                    alphabetical_sorted(self.mb_serialize(v) for v in value)
                )
            )
            self.write("\n")
            self.write_ln("]")

    def write_mozbuild_value(self, key, value):
        if value:
            if key == "LIBRARY_NAME":
                self._library_name = value
            elif key == "FORCE_SHARED_LIB":
                self._shared_library = True
            else:
                self.write("\n")
                self.write_ln(f"{key} = {self.mb_serialize(value)}")
                self.write("\n")

    def write_mozbuild_dict(self, key, value):
        # Templates we need to use instead of certain values.
        replacements = (
            (
                ("COMPILE_FLAGS", '"WARNINGS_AS_ERRORS"', "[]"),
                "AllowCompilerWarnings()",
            ),
        )
        if value:
            self.write("\n")
            for k in sorted(value.keys()):
                v = value[k]
                subst_vals = key, self.mb_serialize(k), self.mb_serialize(v)
                wrote_ln = False
                for flags, tmpl in replacements:
                    if subst_vals == flags:
                        self.write_ln(tmpl)
                        wrote_ln = True

                if not wrote_ln:
                    self.write_ln(
                        f"{key}[{self.mb_serialize(k)}] = {self.mb_serialize(v)}"
                    )

    def write_condition(self, values):
        def mk_condition(k, v):
            if not v:
                return f'not CONFIG["{k}"]'
            return f'CONFIG["{k}"] == {self.mb_serialize(v)}'

        self.write("\n")
        self.write("if ")
        self.write(
            " and ".join(mk_condition(k, values[k]) for k in sorted(values.keys()))
        )
        self.write(":\n")
        self.indent += " " * self._indent_increment

    def terminate_condition(self):
        assert len(self.indent) >= self._indent_increment
        self.indent = self.indent[self._indent_increment :]


def select_gn_target(gn_target_config, target_os):
    if isinstance(gn_target_config, str):
        return gn_target_config

    if target_os in gn_target_config:
        return gn_target_config[target_os]

    if "*" in gn_target_config:
        return gn_target_config["*"]

    raise Exception(
        f'No gn_target configured for target_os="{target_os}". '
        'Expected a string or a dict containing "*" and target_os overrides.'
    )


def find_deps(all_targets, target):
    all_deps = set()
    queue = deque([target])
    while queue:
        item = queue.popleft()
        all_deps.add(item)
        for dep in all_targets[item]["deps"]:
            if dep not in all_deps:
                queue.append(dep)
    return all_deps


def filter_gn_config(path, gn_result, sandbox_vars, input_vars, gn_target):
    gen_path = path / "gen"
    # Translates the raw output of gn into just what we'll need to generate a
    # mozbuild configuration.
    gn_out = {
        "targets": {},
        "sandbox_vars": sandbox_vars,
        "gen_input_files": gn_result["build_settings"]["gen_input_files"],
    }

    cpus = {
        "arm64": "aarch64",
        "x64": "x86_64",
        "mipsel": "mips32",
        "mips64el": "mips64",
        "loong64": "loongarch64",
    }
    oses = {
        "android": "Android",
        "linux": "Linux",
        "mac": "Darwin",
        "openbsd": "OpenBSD",
        "win": "WINNT",
    }

    mozbuild_args = {
        "MOZ_DEBUG": "1" if input_vars.get("is_debug") else None,
        "OS_TARGET": oses[input_vars["target_os"]],
        "TARGET_CPU": cpus.get(input_vars["target_cpu"], input_vars["target_cpu"]),
    }
    if "ozone_platform_x11" in input_vars:
        mozbuild_args["MOZ_X11"] = "1" if input_vars["ozone_platform_x11"] else None

    gn_out["mozbuild_args"] = mozbuild_args
    all_deps = find_deps(gn_result["targets"], gn_target)

    for target_fullname in all_deps:
        raw_spec = gn_result["targets"][target_fullname]

        if raw_spec["type"] == "action":
            # Special handling for the action type to avoid putting empty
            # arrays of args, script and outputs on all other types in `spec`.
            spec = {}
            for spec_attr in (
                "type",
                "args",
                "inputs",
                "script",
                "outputs",
                "response_file_contents",
            ):
                spec[spec_attr] = raw_spec.get(spec_attr, [])
                if spec_attr == "outputs":
                    # Rebase outputs from an absolute path in the temp dir to a
                    # path relative to the target dir.
                    spec[spec_attr] = [
                        mozpath.relpath(d, path) for d in spec[spec_attr]
                    ]
            gn_out["targets"][target_fullname] = spec
            continue

        if raw_spec["type"] == "group":
            gn_out["targets"][target_fullname] = {
                "type": "group",
                "deps": raw_spec.get("deps", []),
            }
            continue

        # TODO: 'executable' will need to be handled here at some point as well.
        if raw_spec["type"] not in ("static_library", "shared_library", "source_set"):
            continue

        spec = {}
        for spec_attr in (
            "type",
            "sources",
            "defines",
            "include_dirs",
            "cflags",
            "cflags_c",
            "cflags_cc",
            "cflags_objc",
            "cflags_objcc",
            "deps",
            "libs",
            "output_name",
        ):
            spec[spec_attr] = raw_spec.get(spec_attr, [])
            if spec_attr == "defines":
                spec[spec_attr] = [
                    d
                    for d in spec[spec_attr]
                    if "CR_XCODE_VERSION" not in d
                    and "CR_SYSROOT_HASH" not in d
                    and "CR_SYSROOT_KEY" not in d
                    and "_FORTIFY_SOURCE" not in d
                ]
            if spec_attr == "include_dirs":
                # Rebase outputs from an absolute path in the temp dir to a path
                # relative to the target dir.
                spec[spec_attr] = [
                    d if gen_path != Path(d) else "!//gen" for d in spec[spec_attr]
                ]

        gn_out["targets"][target_fullname] = spec

    return gn_out


def process_gn_config(
    gn_config,
    gn_config_dir,
    topsrcdir,
    srcdir,
    non_unified_sources,
    sandbox_vars,
    mozilla_flags,
    mozilla_add_override_dir,
):
    # Translates a json gn config into attributes that can be used to write out
    # moz.build files for this configuration.

    # Much of this code is based on similar functionality in `gyp_reader.py`.

    mozbuild_attrs = {"mozbuild_args": gn_config.get("mozbuild_args", None), "dirs": {}}

    targets = gn_config["targets"]

    project_relsrcdir = mozpath.relpath(srcdir, topsrcdir)

    non_unified_sources = set([mozpath.normpath(s) for s in non_unified_sources])

    def target_info(fullname):
        path, name = fullname.split(":")
        # Stripping '//' gives us a path relative to the project root,
        # adding a suffix avoids name collisions with libraries already
        # in the tree (like "webrtc").
        return path.lstrip("//"), name + "_gn"

    def get_lib_name(target_name):
        if target_name.startswith("lib"):
            return target_name[3:]
        return target_name

    # Return list of library dependencies for a target, recursing through any group's dependencies.
    def find_lib_deps(fullname):
        libs = []
        seen_targets = set()

        def visit(dep):
            if dep in seen_targets:
                return
            seen_targets.add(dep)

            dep_spec = targets.get(dep)
            if not dep_spec:
                return

            dep_type = dep_spec["type"]

            if dep_type in ("static_library", "shared_library", "source_set"):
                _, name = target_info(dep)
                libs.append(get_lib_name(name))
                return

            if dep_type == "group":
                for child in dep_spec.get("deps", []):
                    visit(child)

        spec = targets[fullname]
        for dep in spec.get("deps", []):
            visit(dep)

        return libs

    def resolve_path(path):
        # GN will have resolved all these paths relative to the root of the
        # project indicated by "//".
        if path.startswith("//"):
            path = path[2:]
        if not path.startswith("/"):
            path = f"/{project_relsrcdir}/{path}"
        return path

    # Encodes a file path used in a response file contents as a mozbuild-style
    # path, e.g. prefixed with "/" to denote topsrcdir-relative, or "!/" to
    # denote objdir-relative.
    def encode_response_file_content_path(path):
        path = mozpath.normpath(mozpath.join(str(gn_config_dir), path))
        if mozpath.basedir(path, [str(topsrcdir)]):
            return "/" + mozpath.relpath(path, str(topsrcdir))
        if mozpath.basedir(path, [str(gn_config_dir)]):
            return "!/" + mozpath.join(
                project_relsrcdir, mozpath.relpath(path, str(gn_config_dir))
            )
        raise Exception(
            f'Path "{path}" is not under topsrcdir "{topsrcdir}" '
            f'or GN config dir "{gn_config_dir}"'
        )

    # Process all targets from the given gn project and its dependencies.
    for target_fullname, spec in targets.items():
        target_path, target_name = target_info(target_fullname)
        target_relsrcdir = mozpath.join(project_relsrcdir, target_path, target_name)
        context_attrs = {}
        extra_files = {}

        if spec["type"] in ("static_library", "shared_library", "source_set", "action"):
            # Remove leading 'lib' from the target_name if any, and use as
            # library name.
            context_attrs["LIBRARY_NAME"] = get_lib_name(target_name)
        elif spec["type"] == "group":
            # groups are only used to propagate deps, so should be skipped here.
            continue
        else:
            raise Exception(
                "The following GN target type is not currently "
                f'consumed by moz.build: "{spec["type"]}". It may need to be '
                "added, or you may need to re-run the "
                "`GnConfigGen` step."
            )

        if spec["type"] == "shared_library":
            context_attrs["FORCE_SHARED_LIB"] = True

            # Target names are suffixed with "_gn" to avoid collisions, and the
            # target name by default gets used as the library name. For shared
            # libraries we should instead use the `output_name` derived from GN
            # so that consumers can load the expected library names.
            output_name = spec.get("output_name")
            if output_name:
                # `output_name` may already be prefixed with "lib". Mozbuild,
                # however, will add a "lib" prefix to `SHARED_LIBRARY_NAME` on
                # non-Windows platforms. Remove the prefix here to avoid a
                # resulting "liblibX" library name.
                if gn_config["mozbuild_args"]["OS_TARGET"] != "WINNT":
                    output_name = get_lib_name(output_name)
                context_attrs["SHARED_LIBRARY_NAME"] = output_name

        if spec["type"] == "action" and "script" in spec:
            generated_files = []

            if spec.get("response_file_contents"):
                # GN's response_file_contents field allows passing an unlimited
                # amount of data to a script via a temporary file, avoiding
                # maximum command-line length limits.
                #
                # In the only case we currently need to support, the response
                # file contents is a list of files in the source tree, relative
                # to the action's cwd. At vendor time, we cannot know the
                # location of the source tree relative to the action's cwd
                # (located in the objdir), nor the absolute location of the
                # source tree, so we cannot write them directly into a response
                # file.
                #
                # Instead, we encode the paths as mozbuild-style paths (i.e.
                # "/"-prefixed denoting topsrcdir-relative, "!/"-prefixed
                # denoting topobjdir-relative) and write this to a .rsp.in
                # file alongside the generated moz.build files.
                #
                # We then emit a GeneratedFile that will at build time convert
                # the .rsp.in file into the response file expected by the
                # script: converting the mozbuild-style paths into concrete
                # paths now that these are known, and shell-quoting as specified
                # by GN's response file format.
                #
                # Finally, we emit another GeneratedFile to perform the action,
                # ensuring we substitute the path to the generated response file
                # in place of the {{response_file_name}} placeholder arg.
                #
                # Note that if we ever need to support response file contents
                # that are not lists of paths then some of this behaviour must
                # be made optional.

                if RESPONSE_FILE_NAME_FLAG not in spec.get("args", []):
                    raise Exception(
                        f"{target_fullname} has response_file_contents without "
                        f"{RESPONSE_FILE_NAME_FLAG} in args."
                    )

                input_response_file_contents = "\n".join(
                    encode_response_file_content_path(d)
                    for d in spec["response_file_contents"]
                )

                # Response file contents can vary by build configuration.
                # A hash of the contents is included in the filename so that
                # each distinct set of contents gets a unique .rsp.in file,
                # allowing multiple configurations to coexist in-tree.
                digest = hashlib.sha256(
                    input_response_file_contents.encode("utf-8")
                ).hexdigest()[:16]
                input_response_file = (
                    f"{mozpath.basename(spec['outputs'][0])}-{digest}.rsp.in"
                )
                extra_files[input_response_file] = input_response_file_contents

                output_response_file = resolve_path(f"{spec['outputs'][0]}.rsp")

                # Generate the response file from the .rsp.in file, resolving
                # the mozbuild-style paths to absolute paths at build time now
                # these are known.
                # Because the .rsp.in filename includes a hash of its contents
                # and may vary between configurations, this GeneratedFile entry
                # may differ between configurations. Where entries differ, the
                # moz.build writer will wrap them in conditional statements,
                # ensuring each configuration uses the correct .rsp.in file.
                generated_files.append({
                    "script": "/python/mozbuild/mozbuild/action/write_path_list.py",
                    "entry_point": "write_paths",
                    "outputs": [output_response_file],
                    "inputs": [input_response_file],
                })

                args = [
                    f"{spec['outputs'][0]}.rsp"
                    if arg == RESPONSE_FILE_NAME_FLAG
                    else arg
                    for arg in spec.get("args", [])
                ]
                flags = [
                    resolve_path(spec["script"]),
                    resolve_path(""),
                ] + args
                generated_files.append({
                    "script": "/python/mozbuild/mozbuild/action/file_generate_wrapper.py",
                    "entry_point": "action",
                    "outputs": [resolve_path(f) for f in spec["outputs"]],
                    "extra_deps": ["!" + output_response_file],
                    "flags": flags,
                })

            else:
                flags = [
                    resolve_path(spec["script"]),
                    resolve_path(""),
                ] + spec.get("args", [])
                generated_files.append({
                    "script": "/python/mozbuild/mozbuild/action/file_generate_wrapper.py",
                    "entry_point": "action",
                    "outputs": [resolve_path(f) for f in spec["outputs"]],
                    "flags": flags,
                })

            context_attrs["GeneratedFiles"] = generated_files

        sources = []
        unified_sources = []
        extensions = set()
        use_defines_in_asflags = False

        for f in [item.lstrip("//") for item in spec.get("sources", [])]:
            ext = mozpath.splitext(f)[-1]
            extensions.add(ext)
            src = f"{project_relsrcdir}/{f}"
            if ext in {".h", ".hpp", ".inc"}:
                continue
            elif ext == ".def":
                context_attrs["DEFFILE"] = f"/{src}"
            elif ext == ".rc":
                context_attrs["RCFILE"] = f"/{src}"
            elif ext != ".S" and src not in non_unified_sources:
                unified_sources.append(f"/{src}")
            else:
                sources.append(f"/{src}")
            # The Mozilla build system doesn't use DEFINES for building
            # ASFILES.
            if ext == ".s":
                use_defines_in_asflags = True

        context_attrs["SOURCES"] = sources
        context_attrs["UNIFIED_SOURCES"] = unified_sources

        context_attrs["DEFINES"] = {}
        for define in spec.get("defines", []):
            if "=" in define:
                name, value = define.split("=", 1)
                context_attrs["DEFINES"][name] = value
            else:
                context_attrs["DEFINES"][define] = True

        context_attrs["LOCAL_INCLUDES"] = []
        for include in spec.get("include_dirs", []):
            if include.startswith("!"):
                include = "!" + resolve_path(include[1:])
            else:
                include = resolve_path(include)
                # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
                resolved = mozpath.abspath(mozpath.join(topsrcdir, include[1:]))
                if not os.path.exists(resolved):
                    # GN files may refer to include dirs that are outside of the
                    # tree or we simply didn't vendor. Print a warning in this case.
                    if not resolved.endswith("gn-output/gen"):
                        print(
                            f"Included path: '{resolved}' does not exist, dropping include from GN "
                            "configuration.",
                            file=sys.stderr,
                        )
                    continue
            if include in context_attrs["LOCAL_INCLUDES"]:
                continue
            context_attrs["LOCAL_INCLUDES"] += [include]

        context_attrs["ASFLAGS"] = spec.get("asflags_mozilla", [])
        if use_defines_in_asflags and context_attrs["DEFINES"]:
            context_attrs["ASFLAGS"] += ["-D" + d for d in context_attrs["DEFINES"]]
        suffix_map = {
            ".c": ("CFLAGS", ["cflags", "cflags_c"]),
            ".cpp": ("CXXFLAGS", ["cflags", "cflags_cc"]),
            ".cc": ("CXXFLAGS", ["cflags", "cflags_cc"]),
            ".m": ("CMFLAGS", ["cflags", "cflags_objc"]),
            ".mm": ("CMMFLAGS", ["cflags", "cflags_objcc"]),
        }
        variables = (suffix_map[e] for e in extensions if e in suffix_map)
        for var, flag_keys in variables:
            flags = [
                _f for _k in flag_keys for _f in spec.get(_k, []) if _f in mozilla_flags
            ]
            for f in flags:
                # the result may be a string or a list.
                if isinstance(f, str):
                    context_attrs.setdefault(var, []).append(f)
                else:
                    context_attrs.setdefault(var, []).extend(f)

        if "FINAL_LIBRARY" not in sandbox_vars:
            context_attrs["USE_LIBS"] = find_lib_deps(target_fullname)

        context_attrs["OS_LIBS"] = []
        for lib in spec.get("libs", []):
            lib_name = os.path.splitext(lib)[0]
            if lib.endswith(".framework"):
                context_attrs["OS_LIBS"] += ["-framework " + lib_name]
            else:
                context_attrs["OS_LIBS"] += [lib_name]

        # Add some features to all contexts. Put here in case LOCAL_INCLUDES
        # order matters.
        if mozilla_add_override_dir != "":
            context_attrs["LOCAL_INCLUDES"] += [mozilla_add_override_dir]

        context_attrs["LOCAL_INCLUDES"] += [
            "!/ipc/ipdl/_ipdlheaders",
            "/ipc/chromium/src",
            "/tools/profiler/public",
        ]
        # These get set via VC project file settings for normal GYP builds.
        # TODO: Determine if these defines are needed for GN builds.
        if gn_config["mozbuild_args"]["OS_TARGET"] == "WINNT":
            context_attrs["DEFINES"]["UNICODE"] = True
            context_attrs["DEFINES"]["_UNICODE"] = True

        context_attrs["COMPILE_FLAGS"] = {"OS_INCLUDES": []}

        for key, value in sandbox_vars.items():
            if context_attrs.get(key) and isinstance(context_attrs[key], list):
                # If we have a key from sandbox_vars that's also been
                # populated here we use the value from sandbox_vars as our
                # basis rather than overriding outright.
                context_attrs[key] = value + context_attrs[key]
            elif context_attrs.get(key) and isinstance(context_attrs[key], dict):
                context_attrs[key].update(value)
            else:
                context_attrs[key] = value

        mozbuild_attrs["dirs"][target_relsrcdir] = (context_attrs, extra_files)

    return mozbuild_attrs


def find_common_attrs(config_attributes):
    # Returns the intersection of the given configs and prunes the inputs
    # to no longer contain these common attributes.

    common_attrs = deepcopy(config_attributes[0])

    def make_intersection(reference, input_attrs):
        # Modifies `reference` so that after calling this function it only
        # contains parts it had in common with in `input_attrs`.

        for k, input_value in input_attrs.items():
            # Anything in `input_attrs` must match what's already in
            # `reference`.
            common_value = reference.get(k)
            if common_value:
                if isinstance(input_value, list):
                    reference[k] = [
                        i
                        for i in common_value
                        if input_value.count(i) == common_value.count(i)
                    ]
                elif isinstance(input_value, dict):
                    reference[k] = {
                        key: value
                        for key, value in common_value.items()
                        if key in input_value and value == input_value[key]
                    }
                elif input_value != common_value:
                    del reference[k]
            elif k in reference:
                del reference[k]

        # Additionally, any keys in `reference` that aren't in `input_attrs`
        # must be deleted.
        for k in set(reference.keys()) - set(input_attrs.keys()):
            del reference[k]

    def make_difference(reference, input_attrs):
        # Modifies `input_attrs` so that after calling this function it contains
        # no parts it has in common with in `reference`.
        for k, input_value in list(input_attrs.items()):
            common_value = reference.get(k)
            if common_value:
                if isinstance(input_value, list):
                    input_attrs[k] = [
                        i
                        for i in input_value
                        if common_value.count(i) != input_value.count(i)
                    ]
                elif isinstance(input_value, dict):
                    input_attrs[k] = {
                        key: value
                        for key, value in input_value.items()
                        if key not in common_value
                    }
                else:
                    del input_attrs[k]

    for config_attr_set in config_attributes[1:]:
        make_intersection(common_attrs, config_attr_set)

    for config_attr_set in config_attributes:
        make_difference(common_attrs, config_attr_set)

    return common_attrs


def write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs):
    target_srcdir = mozpath.join(topsrcdir, relsrcdir)
    mkdir(target_srcdir)

    target_mozbuild = mozpath.join(target_srcdir, "moz.build")
    with open(target_mozbuild, "w") as fh:
        mb = MozbuildWriter(fh)
        mb.write(license_header)
        mb.write("\n")
        mb.write(generated_header)

        try:
            if relsrcdir in write_mozbuild_variables["INCLUDE_TK_CFLAGS_DIRS"]:
                mb.write('if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk":\n')
                mb.write('    CXXFLAGS += CONFIG["MOZ_GTK3_CFLAGS"]\n')
        except KeyError:
            pass
        try:
            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_GBM_HANDLING"]:
                mb.write('CXXFLAGS += CONFIG["MOZ_GBM_CFLAGS"]\n')
                mb.write('if not CONFIG["MOZ_SYSTEM_GBM"]:\n')
                mb.write('    LOCAL_INCLUDES += [ "/third_party/gbm/gbm/" ]\n')
        except KeyError:
            pass
        try:
            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"]:
                mb.write('CXXFLAGS += CONFIG["MOZ_LIBDRM_CFLAGS"]\n')
                mb.write('if not CONFIG["MOZ_SYSTEM_LIBDRM"]:\n')
                mb.write('    LOCAL_INCLUDES += [ "/third_party/drm/drm/",\n')
                mb.write('                        "/third_party/drm/drm/include/",\n')
                mb.write(
                    '                        "/third_party/drm/drm/include/libdrm" ]\n'
                )
        except KeyError:
            pass
        try:
            if (
                relsrcdir
                in write_mozbuild_variables["INCLUDE_SYSTEM_PIPEWIRE_HANDLING"]
            ):
                mb.write('CXXFLAGS += CONFIG["MOZ_PIPEWIRE_CFLAGS"]\n')
                mb.write('if not CONFIG["MOZ_SYSTEM_PIPEWIRE"]:\n')
                mb.write('    LOCAL_INCLUDES += [ "/third_party/pipewire/" ]\n')
        except KeyError:
            pass
        try:
            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"]:
                mb.write('if not CONFIG["MOZ_SYSTEM_LIBVPX"]:\n')
                mb.write('    LOCAL_INCLUDES += [ "/media/libvpx/libvpx/" ]\n')
                mb.write('    CXXFLAGS += CONFIG["MOZ_LIBVPX_CFLAGS"]\n')
        except KeyError:
            pass
        try:
            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"]:
                mb.write('if CONFIG["MOZ_SYSTEM_AV1"]:\n')
                mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_DAV1D_CFLAGS"]\n')
                mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_LIBAOM_CFLAGS"]\n')
        except KeyError:
            pass
        try:
            if relsrcdir in write_mozbuild_variables["INCLUDE_GECKO_ZLIB_HANDLING"]:
                mb.write('USE_LIBS += [ "zlib" ]\n')
        except KeyError:
            pass

        all_args = [args for args, _ in configs]

        # Start with attributes that will be a part of the mozconfig
        # for every configuration, then factor by other potentially useful
        # combinations.
        # FIXME: this is a time-bomb. See bug 1775202.
        for attrs in (
            (),
            ("MOZ_DEBUG",),
            ("OS_TARGET",),
            ("TARGET_CPU",),
            ("MOZ_DEBUG", "OS_TARGET"),
            ("OS_TARGET", "MOZ_X11"),
            ("OS_TARGET", "TARGET_CPU"),
            ("OS_TARGET", "TARGET_CPU", "MOZ_X11"),
            ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG"),
            ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG", "MOZ_X11"),
        ):
            conditions = set()
            for args in all_args:
                cond = tuple((k, args.get(k) or "") for k in attrs)
                conditions.add(cond)

            for cond in sorted(conditions):
                common_attrs = find_common_attrs([
                    attrs
                    for args, attrs in configs
                    if all((args.get(k) or "") == v for k, v in cond)
                ])
                if any(common_attrs.values()):
                    if cond:
                        mb.write_condition(dict(cond))
                    mb.write_attrs(common_attrs)
                    if cond:
                        mb.terminate_condition()

        mb.finalize()
    return target_mozbuild


def write_mozbuild_files(
    topsrcdir,
    srcdir,
    all_mozbuild_results,
    write_mozbuild_variables,
):
    print("Writing moz.build files")

    # Translate {config -> {dirs -> build info}} into
    #           {dirs -> [(config, build_info)]}
    configs_by_dir = defaultdict(list)
    extra_files_by_dir = defaultdict(dict)
    for config_attrs in all_mozbuild_results:
        mozbuild_args = config_attrs["mozbuild_args"]
        dirs = config_attrs["dirs"]
        for d, (build_data, dir_extra_files) in dirs.items():
            configs_by_dir[d].append((mozbuild_args, build_data))
            for filename, contents in dir_extra_files.items():
                existing = extra_files_by_dir[d].get(filename)
                if existing is not None and existing != contents:
                    raise Exception(f"Conflicting contents for {d}/{filename}")
                extra_files_by_dir[d][filename] = contents

    mozbuilds = set()
    extra_files = set()

    # threading this section did not produce noticeable speed gains
    for relsrcdir, configs in sorted(configs_by_dir.items()):
        mozbuilds.add(
            write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs)
        )

    for relsrcdir, files in sorted(extra_files_by_dir.items()):
        for filename, contents in sorted(files.items()):
            path = mozpath.join(topsrcdir, relsrcdir, filename)
            extra_files.add(path)
            with open(path, "w") as fh:
                fh.write(contents)

    # write the project moz.build file
    dirs_mozbuild = mozpath.join(srcdir, "moz.build")
    mozbuilds.add(dirs_mozbuild)
    with open(dirs_mozbuild, "w") as fh:
        mb = MozbuildWriter(fh)
        mb.write(license_header)
        mb.write("\n")
        mb.write(generated_header)

        # Not every srcdir is present for every config, which needs to be
        # reflected in the generated root moz.build.
        dirs_by_config = {
            tuple(v["mozbuild_args"].items()): set(v["dirs"].keys())
            for v in all_mozbuild_results
        }

        for attrs in (
            (),
            ("OS_TARGET",),
            ("OS_TARGET", "MOZ_DEBUG"),
            ("OS_TARGET", "TARGET_CPU"),
            ("OS_TARGET", "TARGET_CPU", "MOZ_X11"),
        ):
            conditions = set()
            for args in dirs_by_config.keys():
                cond = tuple((k, dict(args).get(k) or "") for k in attrs)
                conditions.add(cond)

            for cond in sorted(conditions):
                common_dirs = None
                for args, dir_set in dirs_by_config.items():
                    if all((dict(args).get(k) or "") == v for k, v in cond):
                        if common_dirs is None:
                            common_dirs = deepcopy(dir_set)
                        else:
                            common_dirs &= dir_set

                for args, dir_set in dirs_by_config.items():
                    if all(dict(args).get(k) == v for k, v in cond):
                        dir_set -= common_dirs

                if common_dirs:
                    if cond:
                        mb.write_condition(dict(cond))
                    mb.write_mozbuild_list("DIRS", [f"/{d}" for d in common_dirs])
                    if cond:
                        mb.terminate_condition()

    # Remove stale moz.builds and extra files.
    for root, dirs, files in os.walk(srcdir):
        if mozpath.normpath(root) == mozpath.normpath(srcdir):
            # No need to remove anything from the project's root srcdir
            continue
        if "moz.build" not in files:
            # This dir never contained moz.build or extra files, so we don't
            # want to remove any of the project's actual files.
            continue

        for filename in files:
            file = mozpath.join(root, filename)
            if file not in mozbuilds | extra_files:
                os.unlink(file)


def generate_gn_config(
    topsrcdir,
    build_root_dir,
    target_dir,
    gn_binary,
    input_variables,
    sandbox_variables,
    gn_target_config,
    moz_build_flag,
    non_unified_sources,
    mozilla_flags,
    mozilla_add_override_dir,
):
    def str_for_arg(v):
        if v in (True, False):
            return str(v).lower()
        return f'"{v}"'

    build_root_dir = topsrcdir / build_root_dir
    srcdir = build_root_dir / target_dir
    gn_target = select_gn_target(gn_target_config, input_variables["target_os"])

    input_variables = input_variables.copy()
    input_variables.update({
        f"{moz_build_flag}": True,
        "concurrent_links": 1,
        "action_pool_depth": 1,
    })

    if input_variables["target_os"] == "win":
        input_variables.update({
            "visual_studio_path": "/",
            "visual_studio_version": 2015,
            "wdk_path": "/",
            "windows_sdk_version": "n/a",
        })
    if input_variables["target_os"] == "mac":
        input_variables.update({
            "mac_sdk_path": "/",
        })

    gn_args = f"--args={' '.join([f'{k}={str_for_arg(v)}' for k, v in input_variables.items()])}"
    with tempfile.TemporaryDirectory() as tempdir:
        # On Mac, `tempdir` starts with /var which is a symlink to /private/var.
        # We resolve the symlinks in `tempdir` here so later usage with
        # relpath() does not lead to unexpected results, should it be used
        # together with another path that has symlinks resolved.
        resolved_tempdir = Path(tempdir).resolve()
        gen_args = [
            gn_binary,
            "gen",
            str(resolved_tempdir),
            gn_args,
            "--ide=json",
            "--root=./",  # must find the google build directory in this directory
            f"--dotfile={target_dir}/.gn",
        ]
        print(f'Running "{" ".join(gen_args)}"', file=sys.stderr)
        subprocess.check_call(gen_args, cwd=build_root_dir, stderr=subprocess.STDOUT)

        gn_config_file = resolved_tempdir / "project.json"
        with open(gn_config_file) as fh:
            raw_json = fh.read()
            raw_json = raw_json.replace(f"//{target_dir}/", "//")
            raw_json = raw_json.replace(f"//{target_dir}:", "//:")
            raw_json = raw_json.replace(f"gen/{target_dir}", "gen")
            gn_config = mozfile_json.loads(raw_json)
            gn_config = filter_gn_config(
                resolved_tempdir,
                gn_config,
                sandbox_variables,
                input_variables,
                gn_target,
            )
            mozbuild_result = process_gn_config(
                gn_config,
                resolved_tempdir,
                topsrcdir,
                srcdir,
                non_unified_sources,
                gn_config["sandbox_vars"],
                mozilla_flags,
                mozilla_add_override_dir,
            )
            return gn_config, mozbuild_result


def generate_gn_configs(topsrcdir, config):
    gn_binary = bootstrap_toolchain("gn/gn") or which("gn")
    if not gn_binary:
        raise Exception("The GN program must be present to generate GN configs.")

    vars_set = []
    for is_debug in (True, False):
        for target_os in ("android", "linux", "mac", "openbsd", "win"):
            target_cpus = ["x64"]
            if target_os in ("android", "linux", "mac", "win", "openbsd"):
                target_cpus.append("arm64")
            if target_os in ("android", "linux"):
                target_cpus.append("arm")
            if target_os in ("android", "linux", "win"):
                target_cpus.append("x86")
            if target_os in ("linux", "openbsd"):
                target_cpus.append("riscv64")
            if target_os == "linux":
                target_cpus.extend(["loong64", "ppc64", "mipsel", "mips64el"])
            for target_cpu in target_cpus:
                vars = {
                    "host_cpu": "x64",
                    "is_debug": is_debug,
                    "target_cpu": target_cpu,
                    "target_os": target_os,
                }

                config_args = config.get("gn_args", {})
                vars.update(config_args.get("*", {}))
                vars.update(config_args.get(target_os, {}))

                if target_os == "linux":
                    for enable_x11 in (True, False):
                        vars["ozone_platform_x11"] = enable_x11
                        vars_set.append(vars.copy())
                else:
                    if target_os == "openbsd":
                        vars["ozone_platform_x11"] = True
                    vars_set.append(vars)

    gn_configs = []
    NUM_WORKERS = 5
    with ProcessPoolExecutor(max_workers=NUM_WORKERS) as executor:
        # Submit tasks to the executor
        futures = {
            executor.submit(
                generate_gn_config,
                topsrcdir,
                config["build_root_dir"],
                config["target_dir"],
                gn_binary,
                vars,
                config["gn_sandbox_variables"],
                config["gn_target"],
                config["moz_build_flag"],
                config["non_unified_sources"],
                config["mozilla_flags"],
                config["mozilla_add_override_dir"],
            ): vars
            for vars in vars_set
        }

        # Process completed tasks as they finish
        error_generating_configs = False
        for future in as_completed(futures):
            try:
                gn_configs.append(future.result())
            except Exception as e:
                print(f"[Task] Task failed with exception: {e}")
                error_generating_configs = True

        if error_generating_configs:
            print("\nGenerating configs failed.  See errors above.\n", file=sys.stderr)
            sys.exit(1)

        print("All generation tasks have been processed.")

    return gn_configs


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("config", help="configuration in json format")
    args = parser.parse_args()

    with open(args.config) as fh:
        config = mozfile_json.load(fh)

    topsrcdir = Path(__file__).parent.parent.resolve()

    _, mozbuild_results = zip(*generate_gn_configs(topsrcdir, config))

    write_mozbuild_files(
        topsrcdir,
        topsrcdir / config["build_root_dir"] / config["target_dir"],
        mozbuild_results,
        config["write_mozbuild_variables"],
    )


if __name__ == "__main__":
    main()