Quelle openh264_build.py Sprache: Python

#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import glob
import os
import re
import subprocess
import sys

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

# import the guts
import mozharness
from mozharness.base.log import DEBUG, ERROR, FATAL
from mozharness.base.transfer import TransferMixin
from mozharness.base.vcs.vcsbase import VCSScript
from mozharness.mozilla.tooltool import TooltoolMixin

external_tools_path = os.path.join(
    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
    "external_tools",
)

class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin):
    all_actions = [
        "clobber",
        "get-tooltool",
        "checkout-sources",
        "build",
        "test",
        "package",
        "dump-symbols",
    ]

    default_actions = [
        "get-tooltool",
        "checkout-sources",
        "build",
        "package",
        "dump-symbols",
    ]

    config_options = [
        [
            ["--repo"],
            {
                "dest": "repo",
                "help": "OpenH264 repository to use",
                "default": "https://github.com/dminor/openh264.git",
            },
        ],
        [
            ["--rev"],
            {"dest": "revision", "help": "revision to checkout", "default": "master"},
        ],
        [
            ["--debug"],
            {
                "dest": "debug_build",
                "action": "store_true",
                "help": "Do a debug build",
            },
        ],
        [
            ["--arch"],
            {
                "dest": "arch",
                "help": "Arch type to use (x64, x86, arm, or aarch64)",
            },
        ],
        [
            ["--os"],
            {
                "dest": "operating_system",
                "help": "Specify the operating system to build for",
            },
        ],
        [
            ["--branch"],
            {
                "dest": "branch",
                "help": "dummy option",
            },
        ],
        [
            ["--build-pool"],
            {
                "dest": "build_pool",
                "help": "dummy option",
            },
        ],
    ]

    def __init__(
        self,
        require_config_file=False,
        config={},
        all_actions=all_actions,
        default_actions=default_actions,
    ):
        # Default configuration
        default_config = {
            "debug_build": False,
            "upload_ssh_key": "~/.ssh/ffxbld_rsa",
            "upload_ssh_user": "ffxbld",
            "upload_ssh_host": "upload.ffxbld.productdelivery.prod.mozaws.net",
            "upload_path_base": "/tmp/openh264",
        }
        default_config.update(config)

        VCSScript.__init__(
            self,
            config_options=self.config_options,
            require_config_file=require_config_file,
            config=default_config,
            all_actions=all_actions,
            default_actions=default_actions,
        )

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        dirs = super(OpenH264Build, self).query_abs_dirs()
        dirs["abs_upload_dir"] = os.path.join(dirs["abs_work_dir"], "upload")
        self.abs_dirs = dirs
        return self.abs_dirs

    def get_tooltool(self):
        c = self.config
        if not c.get("tooltool_manifest_file"):
            self.info("Skipping tooltool fetching since no tooltool manifest")
            return
        dirs = self.query_abs_dirs()
        self.mkdir_p(dirs["abs_work_dir"])
        manifest = os.path.join(
            dirs["abs_src_dir"],
            "testing",
            "mozharness",
            "configs",
            "openh264",
            "tooltool-manifests",
            c["tooltool_manifest_file"],
        )
        self.info("Getting tooltool files from manifest (%s)" % manifest)
        try:
            self.tooltool_fetch(
                manifest=manifest,
                output_dir=os.path.join(dirs["abs_work_dir"]),
                cache=c.get("tooltool_cache"),
            )
        except KeyError:
            self.error("missing a required key.")

    def query_package_name(self):
        if self.config["arch"] in ("x64", "aarch64"):
            bits = "64"
        else:
            bits = "32"
        version = self.config["revision"]

        if sys.platform in ("linux2", "linux"):
            if self.config.get("operating_system") == "android":
                return "openh264-android-{arch}-{version}.zip".format(
                    version=version, arch=self.config["arch"]
                )
            elif self.config.get("operating_system") == "darwin":
                suffix = ""
                if self.config["arch"] != "x64":
                    suffix = "-" + self.config["arch"]
                return "openh264-macosx{bits}{suffix}-{version}.zip".format(
                    version=version, bits=bits, suffix=suffix
                )
            elif self.config["arch"] == "aarch64":
                return "openh264-linux64-aarch64-{version}.zip".format(version=version)
            else:
                return "openh264-linux{bits}-{version}.zip".format(
                    version=version, bits=bits
                )
        elif sys.platform == "win32":
            if self.config["arch"] == "aarch64":
                return "openh264-win64-aarch64-{version}.zip".format(version=version)
            else:
                return "openh264-win{bits}-{version}.zip".format(
                    version=version, bits=bits
                )
        self.fatal("can't determine platform")

    def query_make_params(self):
        retval = []
        if self.config["debug_build"]:
            retval.append("BUILDTYPE=Debug")

        if self.config["arch"] in ("x64", "aarch64"):
            retval.append("ENABLE64BIT=Yes")
        else:
            retval.append("ENABLE64BIT=No")

        if self.config["arch"] == "x86":
            retval.append("ARCH=x86")
        elif self.config["arch"] == "x64":
            retval.append("ARCH=x86_64")
        elif self.config["arch"] == "aarch64":
            retval.append("ARCH=arm64")
        else:
            self.fatal("Unknown arch: {}".format(self.config["arch"]))

        if "operating_system" in self.config:
            retval.append("OS=%s" % self.config["operating_system"])
            if self.config["operating_system"] == "android":
                retval.append("TARGET=invalid")
                retval.append("NDKLEVEL=%s" % self.config["min_sdk"])
                retval.append("NDKROOT=%s/android-ndk" % os.environ["MOZ_FETCHES_DIR"])
                retval.append("NDK_TOOLCHAIN_VERSION=clang")
            if self.config["operating_system"] == "darwin":
                retval.append("OS=darwin")

        if self._is_windows():
            retval.append("OS=msvc")
            retval.append("CC=clang-cl")
            retval.append("CXX=clang-cl")
            if self.config["arch"] == "aarch64":
                retval.append("CXX_LINK_O=-nologo --target=aarch64-windows-msvc -Fe$@")
        else:
            retval.append("CC=clang")
            retval.append("CXX=clang++")

        return retval

    def query_upload_ssh_key(self):
        return self.config["upload_ssh_key"]

    def query_upload_ssh_host(self):
        return self.config["upload_ssh_host"]

    def query_upload_ssh_user(self):
        return self.config["upload_ssh_user"]

    def query_upload_ssh_path(self):
        return "%s/%s" % (self.config["upload_path_base"], self.config["revision"])

    def run_make(self, target, capture_output=False):
        make = (
            f"{os.environ['MOZ_FETCHES_DIR']}/mozmake/mozmake"
            if sys.platform == "win32"
            else "make"
        )
        cmd = [make, target] + self.query_make_params()
        dirs = self.query_abs_dirs()
        repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
        env = None
        if self.config.get("partial_env"):
            env = self.query_env(self.config["partial_env"])
        kwargs = dict(cwd=repo_dir, env=env)
        if capture_output:
            return self.get_output_from_command(cmd, **kwargs)
        else:
            return self.run_command(cmd, **kwargs)

    def _git_checkout(self, repo, repo_dir, rev):
        try:
            subprocess.run(["git", "clone", "-q", "--no-checkout", repo, repo_dir])
            subprocess.run(["git", "checkout", "-q", "-f", f"{rev}^0"], cwd=repo_dir)
        except Exception:
            self.rmtree(repo_dir)
            raise
        return True

    def checkout_sources(self):
        repo = self.config["repo"]
        rev = self.config["revision"]

        dirs = self.query_abs_dirs()
        repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")

        if self._is_windows():
            # We don't have git on our windows builders, so download a zip
            # package instead.
            path = repo.replace(".git", "/archive/") + rev + ".zip"
            self.download_file(path)
            self.unzip(rev + ".zip", dirs["abs_work_dir"])
            self.move(
                os.path.join(dirs["abs_work_dir"], "openh264-" + rev),
                os.path.join(dirs["abs_work_dir"], "openh264"),
            )

            # Retrieve in-tree version of gmp-api
            self.copytree(
                os.path.join(dirs["abs_src_dir"], "dom", "media", "gmp", "gmp-api"),
                os.path.join(repo_dir, "gmp-api"),
            )

            # We need gas-preprocessor.pl for arm64 builds
            if self.config["arch"] == "aarch64":
                openh264_dir = os.path.join(dirs["abs_work_dir"], "openh264")
                self.download_file(
                    (
                        "https://raw.githubusercontent.com/libav/"
                        "gas-preprocessor/c2bc63c96678d9739509e58"
                        "7aa30c94bdc0e636d/gas-preprocessor.pl"
                    ),
                    parent_dir=openh264_dir,
                )
                self.chmod(os.path.join(openh264_dir, "gas-preprocessor.pl"), 744)

                # gas-preprocessor.pl expects cpp to exist
                # os.symlink is not available on Windows until we switch to
                # Python 3.
                os.system(
                    "ln -s %s %s"
                    % (
                        os.path.join(
                            os.environ["MOZ_FETCHES_DIR"], "clang", "bin", "clang.exe"
                        ),
                        os.path.join(openh264_dir, "cpp"),
                    )
                )
            return 0

        self.retry(
            self._git_checkout,
            error_level=FATAL,
            error_message="Automation Error: couldn't clone repo",
            args=(repo, repo_dir, rev),
        )

        # Checkout gmp-api
        # TODO: Nothing here updates it yet, or enforces versions!
        if not os.path.exists(os.path.join(repo_dir, "gmp-api")):
            retval = self.run_make("gmp-bootstrap")
            if retval != 0:
                self.fatal("couldn't bootstrap gmp")
        else:
            self.info("skipping gmp bootstrap - we have it locally")

        # Checkout gtest
        # TODO: Requires svn!
        if not os.path.exists(os.path.join(repo_dir, "gtest")):
            retval = self.run_make("gtest-bootstrap")
            if retval != 0:
                self.fatal("couldn't bootstrap gtest")
        else:
            self.info("skipping gtest bootstrap - we have it locally")

        return retval

    def build(self):
        retval = self.run_make("plugin")
        if retval != 0:
            self.fatal("couldn't build plugin")

    def package(self):
        dirs = self.query_abs_dirs()
        srcdir = os.path.join(dirs["abs_work_dir"], "openh264")
        package_name = self.query_package_name()
        package_file = os.path.join(dirs["abs_work_dir"], package_name)
        if os.path.exists(package_file):
            os.unlink(package_file)
        to_package = []
        for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*")):
            if not re.search(
                r"(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)", f
            ):
                # Don't package unnecessary zip bloat
                # Blocks things like libgmpopenh264.2.dylib and libgmpopenh264.so.1
                self.log("Skipping packaging of {package}".format(package=f))
                continue
            to_package.append(os.path.basename(f))
        self.log("Packaging files %s" % to_package)
        cmd = ["zip", package_file] + to_package
        retval = self.run_command(cmd, cwd=srcdir)
        if retval != 0:
            self.fatal("couldn't make package")
        self.copy_to_upload_dir(
            package_file, dest=os.path.join(srcdir, "artifacts", package_name)
        )

        # Taskcluster expects this path to exist, but we don't use it
        # because our builds are private.
        path = os.path.join(
            self.query_abs_dirs()["abs_work_dir"], "..", "public", "build"
        )
        self.mkdir_p(path)

    def dump_symbols(self):
        dirs = self.query_abs_dirs()
        c = self.config
        srcdir = os.path.join(dirs["abs_work_dir"], "openh264")
        package_name = self.run_make("echo-plugin-name", capture_output=True)
        if not package_name:
            self.fatal("failure running make")
        zip_package_name = self.query_package_name()
        if not zip_package_name[-4:] == ".zip":
            self.fatal("Unexpected zip_package_name")
        symbol_package_name = "{base}.symbols.zip".format(base=zip_package_name[:-4])
        symbol_zip_path = os.path.join(srcdir, "artifacts", symbol_package_name)
        repo_dir = os.path.join(dirs["abs_work_dir"], "openh264")
        env = None
        if self.config.get("partial_env"):
            env = self.query_env(self.config["partial_env"])
        kwargs = dict(cwd=repo_dir, env=env)
        dump_syms = os.path.join(dirs["abs_work_dir"], c["dump_syms_binary"])
        self.chmod(dump_syms, 0o755)
        python = self.query_exe("python3")
        cmd = [
            python,
            os.path.join(external_tools_path, "packagesymbols.py"),
            "--symbol-zip",
            symbol_zip_path,
            dump_syms,
            os.path.join(srcdir, package_name),
        ]
        self.run_command(cmd, **kwargs)

    def test(self):
        retval = self.run_make("test")
        if retval != 0:
            self.fatal("test failures")

    def copy_to_upload_dir(
        self,
        target,
        dest=None,
        log_level=DEBUG,
        error_level=ERROR,
        compress=False,
        upload_dir=None,
    ):
        """Copy target file to upload_dir/dest.

        Potentially update a manifest in the future if we go that route.

        Currently only copies a single file; would be nice to allow for
        recursive copying; that would probably done by creating a helper
        _copy_file_to_upload_dir().
        """
        dest_filename_given = dest is not None
        if upload_dir is None:
            upload_dir = self.query_abs_dirs()["abs_upload_dir"]
        if dest is None:
            dest = os.path.basename(target)
        if dest.endswith("/"):
            dest_file = os.path.basename(target)
            dest_dir = os.path.join(upload_dir, dest)
            dest_filename_given = False
        else:
            dest_file = os.path.basename(dest)
            dest_dir = os.path.join(upload_dir, os.path.dirname(dest))
        if compress and not dest_filename_given:
            dest_file += ".gz"
        dest = os.path.join(dest_dir, dest_file)
        if not os.path.exists(target):
            self.log("%s doesn't exist!" % target, level=error_level)
            return None
        self.mkdir_p(dest_dir)
        self.copyfile(target, dest, log_level=log_level, compress=compress)
        if os.path.exists(dest):
            return dest
        else:
            self.log("%s doesn't exist after copy!" % dest, level=error_level)
            return None

# main {{{1
if __name__ == "__main__":
    myScript = OpenH264Build()
    myScript.run_and_exit()

quality98%

¤ Dauer der Verarbeitung: 0.26 Sekunden (vorverarbeitet) ¤

Wurzel

Suchen

Beweissystem der NASA

Beweissystem Isabelle

NIST Cobol Testsuite

Cephes Mathematical Library

Wiener Entwicklungsmethode

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.

Bemerkung:

Die farbliche Syntaxdarstellung ist noch experimentell.