# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/.
import argparse import copy import glob import multiprocessing import os import pathlib import re import subprocess import sys import tempfile from shutil import copyfile, rmtree
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor from six import string_types
import mozharness from mozharness.base.errors import PythonErrorList from mozharness.base.log import CRITICAL, DEBUG, ERROR, INFO, OutputParser from mozharness.base.python import Python3Virtualenv from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.automation import (
EXIT_STATUS_DICT,
TBPL_RETRY,
TBPL_SUCCESS,
TBPL_WORST_LEVEL_TUPLE,
) from mozharness.mozilla.testing.android import AndroidMixin from mozharness.mozilla.testing.codecoverage import (
CodeCoverageMixin,
code_coverage_config_options,
) from mozharness.mozilla.testing.errors import HarnessErrorList, TinderBoxPrintRe from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
external_tools_path = os.path.join(scripts_path, "external_tools")
here = os.path.abspath(os.path.dirname(__file__))
RaptorErrorList = (
PythonErrorList
+ HarnessErrorList
+ [
{"regex": re.compile(r"""run-as: Package '.*' is unknown"""), "level": DEBUG},
{"substr": r"""raptorDebug""", "level": DEBUG},
{ "regex": re.compile(r"""(?i)^raptor[a-z-]*( - )?( )?error(:)?"""), "level": ERROR,
},
{ "regex": re.compile(r"""(?i)^raptor[a-z-]*( - )?( )?critical(:)?"""), "level": CRITICAL,
},
{ "regex": re.compile(r"""No machine_name called '.*' can be found"""), "level": CRITICAL,
},
{ "substr": r"""No such file or directory: 'browser_output.txt'""", "level": CRITICAL, "explanation": "Most likely the browser failed to launch, or the test otherwise " "failed to start.",
},
]
)
# When running raptor locally, we can attempt to make use of # the users locally cached ffmpeg binary from from when the user # ran `./mach browsertime --setup`
FFMPEG_LOCAL_CACHE = { "mac": "ffmpeg-macos", "linux": "ffmpeg-4.4.1-i686-static", "win": "ffmpeg-4.4.1-full_build",
}
class Raptor(
TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Python3Virtualenv
): """
Install and run Raptor tests """
if self.run_local: # Get app from command-line args, passed in from mach, inside 'raptor_cmd_line_args' # Command-line args can be in two formats depending on how the user entered them # i.e. "--app=geckoview" or separate as "--app", "geckoview" so we have to # parse carefully. It's simplest to use `argparse` to parse partially.
self.app = "firefox" if"raptor_cmd_line_args"in self.config:
sub_parser = argparse.ArgumentParser() # It's not necessary to limit the allowed values: each value # will be parsed and verifed by raptor/raptor.py.
sub_parser.add_argument("--app", default=None, dest="app")
sub_parser.add_argument("-i", "--intent", default=None, dest="intent")
sub_parser.add_argument( "-a", "--activity", default=None, dest="activity"
)
# We'd prefer to use `parse_known_intermixed_args`, but that's # new in Python 3.7.
known, unknown = sub_parser.parse_known_args(
self.config["raptor_cmd_line_args"]
)
if known.app:
self.app = known.app if known.intent:
self.intent = known.intent if known.activity:
self.activity = known.activity else: # Raptor initiated in production via mozharness
self.test = self.config["test"]
self.app = self.config.get("app", "firefox")
self.binary_path = self.config.get("binary_path", None)
if self.app in ("refbrow", "fenix"):
self.app_name = self.binary_path
for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line.
value = self.config.get(details["dest"]) if value and arg notin self.config.get("raptor_cmd_line_args", []):
setattr(self, details["dest"], value)
# We accept some configuration options from the try commit message in the # format mozharness: <options>. Example try commit message: mozharness: # --geckoProfile try: <stuff> def query_gecko_profile_options(self):
gecko_results = [] # If gecko_profile is set, we add that to Raptor's options if self.gecko_profile:
gecko_results.append("--gecko-profile") if self.gecko_profile_interval:
gecko_results.extend(
["--gecko-profile-interval", str(self.gecko_profile_interval)]
) if self.gecko_profile_entries:
gecko_results.extend(
["--gecko-profile-entries", str(self.gecko_profile_entries)]
) if self.gecko_profile_features:
gecko_results.extend(
["--gecko-profile-features", self.gecko_profile_features]
) if self.gecko_profile_threads:
gecko_results.extend(
["--gecko-profile-threads", self.gecko_profile_threads]
) elif self.extra_profiler_run:
gecko_results.append("--extra-profiler-run") return gecko_results
def install_chrome_android(self): """Install Google Chrome for Android in production from tooltool""" if self.app != "chrome-m":
self.info("Google Chrome for Android not required") return if self.config.get("run_local"):
self.info( "Google Chrome for Android will not be installed " "from tooltool when running locally"
) return
self.info("Fetching and installing Google Chrome for Android")
self.device.shell_output("cmd package install-existing com.android.chrome")
self.info("Google Chrome for Android successfully installed")
def install_chromium_android(self): """Install custom Chromium-as-Release for Android from toolchain fetch""" if self.app != "cstm-car-m":
self.info("Chromium-as-Release for Android not required") return if self.config.get("run_local"):
self.info( "Chromium-as-Release for Android will not be installed " "when running locally"
) return
self.info("Installing Custom Chromium-as-Release for Android")
cstm_car_m_apk = pathlib.Path(
os.environ["MOZ_FETCHES_DIR"], "chromium", "apks", "ChromePublic.apk"
)
self.device.install_app(str(cstm_car_m_apk))
self.info("Custom Chromium-as-Release for Android successfully installed")
def download_chrome_android(self): # Fetch the APK
tmpdir = tempfile.mkdtemp()
self.tooltool_fetch(
os.path.join(
self.raptor_path, "raptor", "tooltool-manifests", "chrome-android", "chrome87.manifest",
),
output_dir=tmpdir,
)
files = os.listdir(tmpdir) if len(files) > 1: raise Exception( "Found more than one chrome APK file after tooltool download"
)
chromeapk = os.path.join(tmpdir, files[0])
# Disable verification and install the APK
self.device.shell_output("settings put global verifier_verify_adb_installs 0")
self.install_android_app(chromeapk, replace=True)
# Re-enable verification and delete the temporary directory
self.device.shell_output("settings put global verifier_verify_adb_installs 1")
rmtree(tmpdir)
def install_safari_technology_preview(self): """Ensure latest version of Safari TP binary is running in CI"""
if self.app != "safari-tp"or self.run_local: return
if self.app notin available_chromium_dists:
self.info("Google Chrome or Chromium distributions are not required.") return
if self.app == "chrome":
self.info("Chrome should be preinstalled.") if win in self.platform_name():
base_path = "C:\\%s\\Google\\Chrome\\Application\\chrome.exe"
self.chromium_dist_path = base_path % "Progra~1" ifnot os.path.exists(self.chromium_dist_path):
self.chromium_dist_path = base_path % "Progra~2" elif linux in self.platform_name():
self.chromium_dist_path = "/usr/bin/google-chrome" elif mac in self.platform_name():
self.chromium_dist_path = ( "/Applications/Google Chrome.app/""Contents/MacOS/Google Chrome"
) else:
self.error( "Chrome is not installed on the platform %s yet."
% self.platform_name()
)
if os.path.exists(self.chromium_dist_path):
self.info( "Google Chrome found in expected location %s"
% self.chromium_dist_path
) else:
self.error("Cannot find Google Chrome at %s" % self.chromium_dist_path)
return
chromium_dist = self.app
if self.config.get("run_local"):
self.info("Expecting %s to be pre-installed locally" % chromium_dist) return
# Get the APK location to be able to get the browser version # through mozversion if self.app in self.firefox_android_browsers andnot self.run_local:
kw_options["installerpath"] = self.installer_path
# If testing on Firefox, the binary path already came from mozharness/pro; # otherwise the binary path is forwarded from command-line arg (raptor_cmd_line_args).
kw_options["app"] = self.app if self.app == "firefox"or (
self.app in self.firefox_android_browsers andnot self.run_local
):
binary_path = self.binary_path or self.config.get("binary_path") ifnot binary_path:
self.fatal("Raptor requires a path to the binary.")
kw_options["binary"] = binary_path if self.app in self.firefox_android_browsers: # In production ensure we have correct app name, # i.e. fennec_aurora or fennec_release etc.
kw_options["binary"] = self.query_package_name()
self.info( "Set binary to %s instead of %s"
% (kw_options["binary"], binary_path)
) elif self.app == "safari"andnot self.run_local:
binary_path = "/Applications/Safari.app/Contents/MacOS/Safari"
kw_options["binary"] = binary_path elif self.app == "safari-tp"andnot self.run_local:
binary_path = "/Applications/Safari Technology Preview.app/Contents/MacOS/Safari Technology Preview"
kw_options["binary"] = binary_path # Custom Chromium-as-Release for Android elif self.app == "cstm-car-m":
kw_options["binary"] = "org.chromium.chrome" # Running on Chromium elifnot self.run_local: # When running locally we already set the Chromium binary above, in init. # In production, we already installed Chromium, so set the binary path # to our install.
kw_options["binary"] = self.chromium_dist_path or""
# Options overwritten from **kw if"test"in self.config:
kw_options["test"] = self.config["test"] if"binary"in self.config:
kw_options["binary"] = self.config["binary"] if self.symbols_path:
kw_options["symbolsPath"] = self.symbols_path if self.config.get("obj_path", None) isnotNone:
kw_options["obj-path"] = self.config["obj_path"] if self.config.get("mozbuild_path", None) isnotNone:
kw_options["mozbuild-path"] = self.config["mozbuild_path"] if self.test_url_params:
kw_options["test-url-params"] = self.test_url_params if self.config.get("device_name") isnotNone:
kw_options["device-name"] = self.config["device_name"] if self.config.get("activity") isnotNone:
kw_options["activity"] = self.config["activity"] if self.config.get("conditioned_profile") isnotNone:
kw_options["conditioned-profile"] = self.config["conditioned_profile"] if self.config.get("benchmark_repository"):
kw_options["benchmark_repository"] = self.config["benchmark_repository"] if self.config.get("benchmark_revision"):
kw_options["benchmark_revision"] = self.config["benchmark_revision"] if self.config.get("benchmark_repository"):
kw_options["benchmark_branch"] = self.config["benchmark_branch"]
kw_options.update(kw) if self.host:
kw_options["host"] = self.host # Configure profiling options
options.extend(self.query_gecko_profile_options()) # Extra arguments if args isnotNone:
options += args if os.getenv("PERF_FLAGS"): for option in os.getenv("PERF_FLAGS").split(): if"="in option:
eq_index = option.find("=")
kw_option, value = option[:eq_index], option[eq_index + 1 :]
kw_options[kw_option] = value else:
options.extend(["--" + option])
if self.config.get("run_local", False):
options.extend(["--run-local"]) if"raptor_cmd_line_args"in self.config:
options += self.config["raptor_cmd_line_args"] if self.config.get("code_coverage", False):
options.extend(["--code-coverage"]) if self.config.get("is_release_build", False):
options.extend(["--is-release-build"]) if self.config.get("live_sites", False):
options.extend(["--live-sites"]) if self.config.get("chimera", False):
options.extend(["--chimera"]) if self.config.get("disable_perf_tuning", False):
options.extend(["--disable-perf-tuning"]) if self.config.get("cold", False):
options.extend(["--cold"]) ifnot self.config.get("fission", True):
options.extend(["--disable-fission"]) if self.config.get("verbose", False):
options.extend(["--verbose"]) if self.config.get("extra_prefs"):
options.extend(
["--setpref={}".format(i) for i in self.config.get("extra_prefs")]
) if self.config.get("environment"):
options.extend(
["--setenv={}".format(i) for i in self.config.get("environment")]
) if self.config.get("enable_marionette_trace", False):
options.extend(["--enable-marionette-trace"]) if self.config.get("browser_cycles"):
options.extend(
["--browser-cycles={}".format(self.config.get("browser_cycles"))]
) if self.config.get("test_bytecode_cache", False):
options.extend(["--test-bytecode-cache"]) if self.config.get("collect_perfstats", False):
options.extend(["--collect-perfstats"]) if self.config.get("extra_summary_methods"):
options.extend(
[ "--extra-summary-methods={}".format(method) for method in self.config.get("extra_summary_methods")
]
) if self.config.get("page_timeout"):
options.extend([f"--page-timeout={self.page_timeout}"]) if self.config.get("post_startup_delay"):
options.extend(
[f"--post-startup-delay={self.config['post_startup_delay']}"]
) if (
self.config.get("screenshot_on_failure", False) or os.environ.get("MOZ_AUTOMATION", None) isnotNone
):
options.extend(["--screenshot-on-failure"]) if self.config.get("power_test", False):
options.extend(["--power-test"])
for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line
value = self.config.get(details["dest"]) if value isNoneor value != getattr(self, details["dest"], None): # Check for modifications done to the instance variables
value = getattr(self, details["dest"], None) if value and arg notin self.config.get("raptor_cmd_line_args", []): if isinstance(value, string_types):
options.extend([arg, os.path.expandvars(value)]) elif isinstance(value, (tuple, list)): for val in value:
options.extend([arg, val]) else:
options.extend([arg])
for key, value in kw_options.items():
options.extend(["--%s" % key, value])
return options
def populate_webroot(self): """Populate the production test machines' webroots"""
self.raptor_path = os.path.join(
self.query_abs_dirs()["abs_test_install_dir"], "raptor"
) if self.config.get("run_local"):
self.raptor_path = os.path.join(self.repo_path, "testing", "raptor")
def clobber(self): # Recreate the upload directory for storing the logcat collected # during APK installation.
super(Raptor, self).clobber()
upload_dir = self.query_abs_dirs()["abs_blob_upload_dir"] ifnot os.path.isdir(upload_dir):
self.mkdir_p(upload_dir)
def install_android_app(self, apk, replace=False): # Override AndroidMixin's install_android_app in order to capture # logcat during the installation. If the installation fails, # the logcat file will be left in the upload directory.
self.logcat_start() try:
super(Raptor, self).install_android_app(apk, replace=replace) finally:
self.logcat_stop()
def create_virtualenv(self, **kwargs): """VirtualenvMixin.create_virtualenv() assumes we're using
self.config['virtualenv_modules']. Since we're installing
raptor from its source, we have to wrap that method here.""" # If virtualenv already exists, just add to path and don't re-install. # We need it in-path to import jsonschema later when validating output for perfherder.
_virtualenv_path = self.config.get("virtualenv_path")
if self.clean:
rmtree(_virtualenv_path, ignore_errors=True)
_python_interp = self.query_exe("python") if"win"in self.platform_name() and os.path.exists(_python_interp):
multiprocessing.set_executable(_python_interp)
if self.run_local and os.path.exists(_virtualenv_path):
self.info("Virtualenv already exists, skipping creation") # ffmpeg exists outside of this virtual environment so # we re-add it to the platform environment on repeated # local runs of browsertime visual metric tests
self.setup_local_ffmpeg()
if os.path.exists(path_to_ffmpeg):
os.environ["PATH"] += os.pathsep + path_to_ffmpeg
self.browsertime_ffmpeg = path_to_ffmpeg
self.info( "Added local ffmpeg found at: %s to environment." % path_to_ffmpeg
) else: raise Exception( "No local ffmpeg binary found. Expected it to be here: %s"
% path_to_ffmpeg
)
def install(self): ifnot self.config.get("no_install", False): if self.app in self.firefox_android_browsers:
self.device.uninstall_app(self.binary_path)
# Check if the user supplied their own APK, and install # that instead
installer_path = pathlib.Path(
self.raptor_path, "raptor", "user_upload.apk"
) ifnot installer_path.exists():
installer_path = self.installer_path
# Get Raptor options
options = self.raptor_options(args=args, **kw)
# Python version check
python = self.query_python_path()
self.run_command([python, "--version"])
parser = RaptorOutputParser(
config=self.config, log_obj=self.log_obj, error_list=RaptorErrorList
)
env = {}
env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] ifnot self.run_local:
env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk()
env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"]
env["RUST_BACKTRACE"] = "full" ifnot os.path.isdir(env["MOZ_UPLOAD_DIR"]):
self.mkdir_p(env["MOZ_UPLOAD_DIR"])
env = self.query_env(partial_env=env, log_level=INFO) # adjust PYTHONPATH to be able to use raptor as a python package if"PYTHONPATH"in env:
env["PYTHONPATH"] = self.raptor_path + os.pathsep + env["PYTHONPATH"] else:
env["PYTHONPATH"] = self.raptor_path
# mitmproxy needs path to mozharness when installing the cert, and tooltool
env["SCRIPTSPATH"] = scripts_path
env["EXTERNALTOOLSPATH"] = external_tools_path
env["XPCSHELL_PATH"] = os.path.join(
self.query_abs_dirs()["abs_test_install_dir"], "bin", "xpcshell.exe"
) if os.path.exists(env["XPCSHELL_PATH"]) andnot self.run_local:
dest = os.path.join(
self.query_abs_dirs()["abs_work_dir"], "application", "firefox", "xpcshell.exe",
)
copyfile(env["XPCSHELL_PATH"], dest)
env["XPCSHELL_PATH"] = dest
# Needed to load unsigned Raptor WebExt on release builds if self.is_release_build:
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
if self.repo_path isnotNone:
env["MOZ_DEVELOPER_REPO_DIR"] = self.repo_path if self.obj_path isnotNone:
env["MOZ_DEVELOPER_OBJ_DIR"] = self.obj_path if self.mozbuild_path isnotNone:
env["MOZ_MOZBUILD_DIR"] = self.mozbuild_path
# Sets a timeout for how long Raptor should run without output
output_timeout = self.config.get("raptor_output_timeout", 3600) # Run Raptor tests
run_tests = os.path.join(self.raptor_path, "raptor", "raptor.py")
# Dynamically set the log level based on the raptor config for consistency # throughout the test
mozlog_opts = [f"--log-tbpl-level={self.config['log_level']}"]
if self.app in self.android_browsers:
self.logcat_stop()
if parser.minidump_output:
self.info("Looking at the minidump files for debugging purposes...") for item in parser.minidump_output:
self.run_command(["ls", "-l", item])
elifnot self.run_local: # Copy results to upload dir so they are included as an artifact
self.info("Copying Raptor results to upload dir:")
# Make individual perfherder data JSON's for each supporting data type for file in glob.glob(
os.path.join(self.query_abs_dirs()["abs_work_dir"], "*")
):
path, filename = os.path.split(file)
ifnot filename.startswith("raptor-"): continue
# filename is expected to contain a unique data name # i.e. raptor-os-baseline-power.json would result in # the data name os-baseline-power
data_name = "-".join(filename.split("-")[1:])
data_name = ".".join(data_name.split(".")[:-1])
src = os.path.join(
self.query_abs_dirs()["abs_work_dir"], "screenshots.html"
) if os.path.exists(src):
dest = os.path.join(env["MOZ_UPLOAD_DIR"], "screenshots.html")
self.info(str(dest))
self._artifact_perf_data(src, dest)
# Allow log failures to over-ride successful runs of the test harness and # give log failures priority, so that, for instance, log failures resulting # in TBPL_RETRY cause a retry rather than simply reporting an error. if parser.tbpl_status != TBPL_SUCCESS:
parser_status = EXIT_STATUS_DICT[parser.tbpl_status]
self.info( "return code %s changed to %s due to log output"
% (str(self.return_code), str(parser_status))
)
self.return_code = parser_status
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.
