Quelle runner.py Sprache: Python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Pure Python runner so we can execute perftest in the CI without
depending on a full mach toolchain, that is not fully available in
all worker environments.

This runner can be executed in two different ways:

- by calling run_tests() from the mach command
- by executing this module directly

When the module is executed directly, if the --on-try option is used,
it will fetch arguments from Tascluster's parameters, that were
populated via a local --push-to-try call.

The --push-to-try flow is:

- a user calls ./mach perftest --push-to-try --option1 --option2
- a new push to try commit is made and includes all options in its parameters
- a generic TC job triggers the perftest by calling this module with --on-try
- run_test() grabs the parameters artifact and converts them into args for
  perftest
"""
import json
import logging
import os
import shutil
import sys
from pathlib import Path

TASKCLUSTER = "TASK_ID" in os.environ.keys()
RUNNING_TESTS = "RUNNING_TESTS" in os.environ.keys()
HERE = Path(__file__).parent
SRC_ROOT = Path(HERE, "..", "..", "..").resolve()

# XXX need to make that for all systems flavors
if "SHELL" not in os.environ:
    os.environ["SHELL"] = "/bin/bash"

def _activate_virtualenvs(flavor):
    """Adds all available dependencies in the path.

    This is done so the runner can be used with no prior
    install in all execution environments.
    """

    # We need the "mach" module to access the logic to parse virtualenv
    # requirements. Since that depends on "packaging", we add that to the path too.
    # We need filelock for solving a virtualenv race condition
    sys.path[0:0] = [
        os.path.join(SRC_ROOT, module)
        for module in (
            os.path.join("python", "mach"),
            os.path.join("third_party", "python", "packaging"),
            os.path.join("third_party", "python", "filelock"),
        )
    ]

    from mach.site import (
        CommandSiteManager,
        ExternalPythonSite,
        MachSiteManager,
        SitePackagesSource,
        resolve_requirements,
    )
    from mach.util import get_state_dir, get_virtualenv_base_dir

    mach_site = MachSiteManager(
        str(SRC_ROOT),
        None,
        resolve_requirements(str(SRC_ROOT), "mach"),
        ExternalPythonSite(sys.executable),
        SitePackagesSource.NONE,
    )
    mach_site.activate()

    command_site_manager = CommandSiteManager.from_environment(
        str(SRC_ROOT),
        lambda: os.path.normpath(get_state_dir(True, topsrcdir=str(SRC_ROOT))),
        "common",
        get_virtualenv_base_dir(str(SRC_ROOT)),
    )

    command_site_manager.activate()

    if TASKCLUSTER:
        # In CI, the directory structure is different: xpcshell code is in
        # "$topsrcdir/xpcshell/" rather than "$topsrcdir/testing/xpcshell". The
        # same is true for mochitest. It also needs additional settings for some
        # dependencies.
        if flavor == "xpcshell":
            print("Setting up xpcshell python paths...")
            sys.path.append("xpcshell")
        elif flavor == "mochitest":
            print("Setting up mochitest python paths...")
            sys.path.append("mochitest")
            sys.path.append(str(Path("tools", "geckoprocesstypes_generator")))

def _create_artifacts_dir(kwargs, artifacts):
    from mozperftest.utils import create_path

    results_dir = kwargs.get("test_name")
    if results_dir is None:
        results_dir = "results"

    return create_path(artifacts / "artifacts" / kwargs["tool"] / results_dir)

def _save_params(kwargs, artifacts):
    with open(os.path.join(str(artifacts), "side-by-side-params.json"), "w") as file:
        json.dump(kwargs, file, indent=4)

def run_tests(mach_cmd, kwargs, client_args):
    """This tests runner can be used directly via main or via Mach.

    When the --on-try option is used, the test runner looks at the
    `PERFTEST_OPTIONS` environment variable that contains all options passed by
    the user via a ./mach perftest --push-to-try call.
    """
    on_try = kwargs.pop("on_try", False)

    # trying to get the arguments from the task params
    if on_try:
        try_options = json.loads(os.environ["PERFTEST_OPTIONS"])
        print("Loading options from $PERFTEST_OPTIONS")
        print(json.dumps(try_options, indent=4, sort_keys=True))
        kwargs.update(try_options)

    from mozperftest import MachEnvironment, Metadata
    from mozperftest.hooks import Hooks
    from mozperftest.script import ScriptInfo
    from mozperftest.utils import build_test_list

    hooks_file = kwargs.pop("hooks", None)
    hooks = Hooks(mach_cmd, hooks_file)
    verbose = kwargs.get("verbose", False)
    log_level = logging.DEBUG if verbose else logging.INFO

    # If we run through mach, we just  want to set the level
    # of the existing termminal handler.
    # Otherwise, we're adding it.
    if mach_cmd.log_manager.terminal_handler is not None:
        mach_cmd.log_manager.terminal_handler.level = log_level
    else:
        mach_cmd.log_manager.add_terminal_logging(level=log_level)
        mach_cmd.log_manager.enable_all_structured_loggers()
        mach_cmd.log_manager.enable_unstructured()

    try:
        # Only pass the virtualenv to the before_iterations hook
        # so that users can install test-specific packages if needed.
        mach_cmd.activate_virtualenv()
        kwargs["virtualenv"] = mach_cmd.virtualenv_manager
        hooks.run("before_iterations", kwargs)
        del kwargs["virtualenv"]

        tests, tmp_dir = build_test_list(kwargs["tests"])

        for test in tests:
            script = ScriptInfo(test)

            # update the arguments with options found in the script, if any
            args = script.update_args(**client_args)
            # XXX this should be the default pool for update_args
            for key, value in kwargs.items():
                if key not in args:
                    args[key] = value

            # update the hooks, or use a copy of the general one
            script_hooks = Hooks(mach_cmd, args.pop("hooks", hooks_file))

            flavor = args["flavor"]
            if flavor == "doc":
                print(script)
                continue

            for iteration in range(args.get("test_iterations", 1)):
                try:
                    env = MachEnvironment(mach_cmd, hooks=script_hooks, **args)
                    metadata = Metadata(mach_cmd, env, flavor, script)
                    script_hooks.run("before_runs", env)
                    try:
                        with env.frozen() as e:
                            e.run(metadata)
                    finally:
                        script_hooks.run("after_runs", env)
                finally:
                    if tmp_dir is not None:
                        shutil.rmtree(tmp_dir)
    finally:
        hooks.cleanup()

def run_tools(mach_cmd, kwargs):
    """This tools runner can be used directly via main or via Mach.

    **TODO**: Before adding any more tools, we need to split this logic out
    into a separate file that runs the tools and sets them up dynamically
    in a similar way to how we use layers.
    """
    from mozperftest.utils import ON_TRY, install_package

    mach_cmd.activate_virtualenv()
    install_package(
        mach_cmd.virtualenv_manager,
        "mozperftest-tools==0.3.2",
    )

    log_level = logging.INFO
    if mach_cmd.log_manager.terminal_handler is not None:
        mach_cmd.log_manager.terminal_handler.level = log_level
    else:
        mach_cmd.log_manager.add_terminal_logging(level=log_level)
        mach_cmd.log_manager.enable_all_structured_loggers()
        mach_cmd.log_manager.enable_unstructured()

    if ON_TRY:
        artifacts = Path(os.environ.get("MOZ_FETCHES_DIR"), "..").resolve()
        artifacts = _create_artifacts_dir(kwargs, artifacts)
    else:
        artifacts = _create_artifacts_dir(kwargs, SRC_ROOT)

    _save_params(kwargs, artifacts)

    # Run the requested tool
    from mozperftest.tools import TOOL_RUNNERS

    tool = kwargs.pop("tool")
    print(f"Running {tool} tool")

    TOOL_RUNNERS[tool](artifacts, kwargs)

def main(argv=sys.argv[1:]):
    """Used when the runner is directly called from the shell"""
    flavor = "desktop-browser"
    if "--flavor" in argv:
        flavor = argv[argv.index("--flavor") + 1]
    _activate_virtualenvs(flavor)

    from mach.logging import LoggingManager
    from mach.util import get_state_dir
    from mozbuild.base import MachCommandBase, MozbuildObject
    from mozbuild.mozconfig import MozconfigLoader

    from mozperftest import PerftestArgumentParser, PerftestToolsArgumentParser

    if os.getenv("PERF_FLAGS"):
        extra_args = []
        for extra_arg in os.getenv("PERF_FLAGS").split():
            extra_args.append(f"--{extra_arg}")
        argv.extend(extra_args)

    mozconfig = SRC_ROOT / "browser" / "config" / "mozconfig"
    if mozconfig.exists():
        os.environ["MOZCONFIG"] = str(mozconfig)

    if "--xpcshell-mozinfo" in argv:
        mozinfo = argv[argv.index("--xpcshell-mozinfo") + 1]
        topobjdir = Path(mozinfo).parent
    else:
        topobjdir = None

    config = MozbuildObject(
        str(SRC_ROOT),
        None,
        LoggingManager(),
        topobjdir=topobjdir,
        mozconfig=MozconfigLoader.AUTODETECT,
    )
    config.topdir = config.topsrcdir
    config.cwd = os.getcwd()
    config.state_dir = get_state_dir()

    # This monkey patch forces mozbuild to reuse
    # our configuration when it tries to re-create
    # it from the environment.
    def _here(*args, **kw):
        return config

    MozbuildObject.from_environment = _here

    mach_cmd = MachCommandBase(config)

    if "tools" in argv[0]:
        if len(argv) == 1:
            raise SystemExit("No tool specified, cannot continue parsing")
        PerftestToolsArgumentParser.tool = argv[1]
        perftools_parser = PerftestToolsArgumentParser()
        args = dict(vars(perftools_parser.parse_args(args=argv[2:])))
        args["tool"] = argv[1]
        run_tools(mach_cmd, args)
    else:
        perftest_parser = PerftestArgumentParser(description="vanilla perftest")
        args = dict(vars(perftest_parser.parse_args(args=argv)))
        user_args = perftest_parser.get_user_args(args)
        run_tests(mach_cmd, args, user_args)

if __name__ == "__main__":
    sys.exit(main())

quality91%

¤ Dauer der Verarbeitung: 0.5 Sekunden ¤

Wurzel

Suchen

Beweissystem der NASA

Beweissystem Isabelle

NIST Cobol Testsuite

Cephes Mathematical Library

Wiener Entwicklungsmethode

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.

Bemerkung:

Die farbliche Syntaxdarstellung ist noch experimentell.