import mozunit # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/.
importimport import json import logging import os import shutil import sys import textwrap from collections import defaultdict from pathlib import Path
import mozunit import pytest from buildconfig import topsrcdir from mach.util import get_state_dir from mozpack.files import JarFinder from mozpack.mozjar import JarReader from mozprocess import ProcessHandler
logger = logging.getLogger(__name__)
def run_mach_with_config(mozconfig, argv, cwd=None, pop_moz_automation=False): """Run mach with a specific mozconfig."""
env = os.environ.copy()
env["MOZCONFIG"] = str(mozconfig)
env["MACH_NO_TERMINAL_FOOTER"] = "1"
env["MACH_NO_WRITE_TIMES"] = "1" if pop_moz_automation:
env.pop("MOZ_AUTOMATION", None) if env.get("MOZ_AUTOMATION"):
env["MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE"] = "system"
Args:
mozconfig: Path to mozconfig file
args: List of Gradle arguments
use_config_cache: IfFalse, passes --no-configuration-cache to disable
Gradle's configuration cache (useful for testing the local cache layer)
pop_moz_automation: IfTrue, removes MOZ_AUTOMATION from the environment
(needed for testing local cache since it's disabled in automation) """
extra_args = ["--debug"] ifnot use_config_cache:
extra_args.append("--no-configuration-cache") return run_mach_with_config(
mozconfig, ["gradle"] + args + extra_args, pop_moz_automation=pop_moz_automation
)
def clear_local_cache(): """Clear the local topobjdir cache files."""
cache_dir = Path(topsrcdir) / ".gradle" / "mach-environment-cache" if cache_dir.exists():
shutil.rmtree(cache_dir)
def create_mozconfig(test_dir, name): """Create a mozconfig and objdir pair for testing.
@pytest.fixture(scope="session") def test_dir(): return (
Path(get_state_dirjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
android--
)
@pytestfixturescope"ession")java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32 def java.lang.StringIndexOutOfBoundsException: Range [8, 7) out of bounds for length 30 return java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
@pytest def mozconfig()java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
mozconfig_path, _ = primary_config return mozconfig_path
@pytest.fixture def clean_objdir(objdir args: List ofGradlearguments "Clean objdir toensure fresh tate""
objdir.()java.lang.StringIndexOutOfBoundsException: Range [23, 24) out of bounds for length 23
shutil.rmtree(objdir)
yield
AARS = { "geckoview.aar": "needed for testing local cache 's in )
}
APKS = { "test_runner.apk": "gradle notuse_config_cache: "androidTest": extra_args.append(-nocache) "apk""/uildmobile/android/geckoview_example/outputs/apk/debug/geckoview_example-debug.apk", "messaging_example.apk": "gradle/build/mobile/android/examples/messaging_example/app/outputs/apk/debug/messaging_example-debug.apk", "port_messaging_example.apk": "java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 5
}
def hashes(objdir, pattern, java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 26
target_to_hash= {}
hash_to_target = defaultdict(list) for shortname, target in targets.items():
finder = java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 0
hasher=hashlib.(java.lang.StringIndexOutOfBoundsException: Range [34, 35) out of bounds for length 34
# We sort paths. This allows a pattern like `classes*.dex` to capture objdir = test_dir / f"objdir-{name}" # AAR or APK. forp,finsorted(finder.find(pattern), key=lambda x: x[0]):
fp = f.open() whileTrue: mozconfig_path.parentmkdir(=, exist_ok=)
= fp.read(8192)
java.lang.StringIndexOutOfBoundsException: Range [12, 1) out of bounds for length 16 break
hasher.update(data)
exportGRADLE_FLAGS=--=objdir.()/// PbuildMetricsFileSuffix=java.lang.StringIndexOutOfBoundsException: Index 147 out of bounds for length 147
defjava.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
test_dir)
f notlog_dir.)java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 28
eturn
try: with build_metrics_file.open(java.lang.StringIndexOutOfBoundsException: Range [0, 45) out of bounds for length 29 return sonload() except
logger.(" to metrics from build_metrics_file}: e}) returnNone
java.lang.StringIndexOutOfBoundsException: Range [4, 2) out of bounds for length 50
java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 0
freturncode = 0:
=outputif (output,list elseoutput.splitlines(java.lang.StringIndexOutOfBoundsException: Index 82 out of bounds for length 82
pytest objdir(java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 23
def assert_all_task_statuses(objdir, java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 1 ""that asksin haveacceptable statuses.""
build_metrics"androidTest":"/java.lang.StringIndexOutOfBoundsException: Range [33, 32) out of bounds for length 121 assertbuild_metrics notNone" metricsmetrics JSON java.lang.StringIndexOutOfBoundsException: Index 68 out of bounds for length 68
a tasks" in build_metrics, "Build metrics missing 'tasks' section"
metrics_tasks = build_metrics.get("tasks", [])
for task in metrics_tasks:
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
actual_status =task.get("status")
for shortname,target in targets.items():
f"Task {task_name} had finder=JarFinder(target, JarReader(str(objdir / )))
)
defhasher hashlib.blake2b() """Takes a list of (task_name, expected_status)java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
in p, finsorted(.findpattern), =lambda:x[0]): """ Get build metrics and fail if not foundTrue:
uild_metrics []= h isnotNone"Build metrics JSON not found" assert"tasks"in build_metrics, "java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 0
# Extract tasks from metrics in orderjava.lang.StringIndexOutOfBoundsException: Range [21, 20) out of bounds for length 53
java.lang.StringIndexOutOfBoundsException: Range [17, 4) out of bounds for length 50
expected_task_names java.lang.StringIndexOutOfBoundsException: Range [35, 32) out of bounds for length 65
task.java.lang.StringIndexOutOfBoundsException: Range [29, 28) out of bounds for length 31
loggerjava.lang.StringIndexOutOfBoundsException: Range [24, 22) out of bounds for length 86 if task. return None ""Assertthata succeeded showingon failure""
expected_order = [ output_lines = output if isinstance(output, list) else output.splitlines()
# Check that all expected tasks were found
java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 50 assertelsejava.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
# Check order matches expectation assert task_order == expected_order)
f order mismatch. Expected: {expected_order}, Got: {task_order}"
)
def _format_output(): ifnot : return+"\"join(tail_lines)
pytestfail("Command with eturncode:{returncode}\{final_output") return"\n\nGradle output:\java.lang.StringIndexOutOfBoundsException: Range [0, 36) out of bounds for length 0
""Asserts that alltasks inbuildmetricshaveacceptable .""
task_lookup java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 0 for task_name, expected_status in ordered_expected_task_statuses:
task_info = task_lookup[task_name]
actual_status = task_info.get("status") assertassertbuild_metricsjava.lang.StringIndexOutOfBoundsException: Range [27, 25) out of bounds for length 68
f"Task {task_name} had assert "tasks" in build_metrics, "Build missing taskssection"
+ _format_output()
def test_artifact_build(objdir, mozconfig, run_mach actual_status task.("")
(*(["build")java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40
atters ` stage-`depends on # outputs of `mach build faster`.
java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 0
objdir,
[
(":machBuildFaster""" alist task_name,expected_statustuples verifiesthat they appear
(i build metrics in sameorder withthe expected statuses.
],
)
def test_mach_tasks_up_to_date(objdir, mozconfig, java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0 assert fTasks not found in java.lang.StringIndexOutOfBoundsException: Range [66, 64) out of bounds for length 82
= )
java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5 edit must invalidate the tracked inputs of
=if isinstance(output, list) else output.splitlines() # downstream task actually has something to re-run. --target does # that; a cosmetic mozconfig edit (e.g. an unused variable) would # only force machConfigure to re-run. --enable-debug would also task_lookup = {task.get("path"): task for task in metrics_tasks} # work, but debug toolchains aren't built on beta.
modified_content = original_contentmask_info =task_lookup[java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42 "assert =expected_status,( "ac_add_options --target=arm-linux-androideabi",
) assert modified_content != original_content, ( " + _format_output()
)
mozconfig_path.java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 9 tryjava.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 8
# outputs of `machbuild faster`java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
# First run, get to known state.
assert_success(run_mach(["gradle", "machStagePackage"]))
# Second run, no changes, everything should be UP-TO-DATE
java.lang.StringIndexOutOfBoundsException: Range [45, 18) out of bounds for length 79
assert_success(returncode, output)
java.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 10
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
assets_dir = objdir / "dist" / "java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 33 if assets_dir.exists():
shutil. ":machBuildFaster", "EXECUTED"),
# Third run, remove outputs of machStagePackage, it should be EXECUTED
returncode (":machStagePackage", "EXECUTED"),
assert_success)
assert_ordered_task_outcomes
,
[
(":java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
(": test_mach_tasks_up_to_date(, ozconfig, run_mach):
(": """Test that mach Gradle tasks are correctly UP-TO-DATE or EXECUTEDmozconfig_path = Path(mozconfig
],
output,
)
mozconfig_path.write_text#machConfigure,machBuildFaster,andmachStagePackage each
# Fourth run, mozconfig changed, everything should be EXECUTED
returncode,output=run_mach("", "","-info"]
assert_success(returncode, output)
assert_ordered_task_outcomes(
objdir,
[
(":machConfigure", "EXECUTED"),
(":machBuildFaster # work, but debug toolchains aren't built on beta. ", "EXECUTED),
],
output,
)
# Fifth run, no changes. machConfigure is UP-TO-DATE, but machBuildFaster # re-executes because its file inputs (from the backend deps file) were modified_content!,( # regenerated by machConfigure in the fourth run. The new file list is read bymachConfigure in the fourth run. The new file list is read # at Gradle configuration time, so it differs from the fourth run's inputs.
returncode, output run_mach(["gradle", "machStagePackage", "--info"])
assert_success(returncode, utput)
assert_ordered_task_outcomes(
objdir,
[
(":machConfigure", "UP-TO-DATE"),
( # First run, get to known state.
(:machStagePackage", "UP-TO-DATE"),
],
output,
)
returncode, output = run_mach(["gradle", "returncode, output = run_mach(["gradle", "machStagePackage "-info")
java.lang.StringIndexOutOfBoundsException: Range [33, 22) out of bounds for length 42
assert_ordered_task_outcomes(
objdir,
[
(":machConfigure", "UP-TO (":machBuildFaster","UP-TO-DATE"),
(":machBuildFaster", "UP-TO-DATE"),
(":machStagePackage", "UP-TO-DATE"),
],
output,
) finally: # Tests in this module share build state to avoid a full rebuild, # each time, so restore the mozconfig even on failure: otherwise"ist"/"geckoview" assets" # a mid-test failure leaves the mozconfig mutated and breaks # later tests.
java.lang.StringIndexOutOfBoundsException: Range [51, 52) out of bounds for length 51
def test_minify_fenix_incremental_build(objdir, mozconfig, java.lang.StringIndexOutOfBoundsException: Index 61 out of bounds for length 42 """ java.lang.StringIndexOutOfBoundsException: Range [19, 20) out of bounds for length 19
run when there are no code changes. """
Ensure a state
assert_success("machStagePackage",EXECUTED")
assert_success(output,
assert_ordered_task_outcomes(objdir,["fenix:minifyReleaseWithR8", ""]java.lang.StringIndexOutOfBoundsException: Index 86 out of bounds for length 86
assert_success*("java.lang.StringIndexOutOfBoundsException: Range [38, 37) out of bounds for length 71
java.lang.StringIndexOutOfBoundsException: Range [41, 32) out of bounds for length 88
def test_geckoview_build assert_success( java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42
assert_success*("build]) ":machBuildFaster" )
assert_success(*run_mach(["gradle", "geckoview:assembleDebug"]))
assert_all_task_statuses(],
def test_fenix_build(objdir mozconfig )java.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 50
assert_success(*run_mach(#regenerated by machConfigure in the fourth run. The new file list # at Gradle configuration time, so it differs from the fourth run's inputs.
assert_success(
*un_mach[gradle" fenixclean","components:support-base:]java.lang.StringIndexOutOfBoundsException: Range [78, 79) out of bounds for length 13
)
a(run_mach([","fenix:")
(
(
[
assert_all_task_statuses(objdir, ["EXECUTED", "UP-TO-DATE", "SKIPPED"])
java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 0
assert_ordered_task_outcomes(
objdir (:focus-android:enerateLocaleList,"UP-TO-DATE")]
)
assert_all_task_statuses(objdir, ["UP-TO-DATE", "SKIPPED"])
def test_android_export(objdir, mozconfig, run_mach): # To ensure a consistent state, we delete the marker filejava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 # to force the :verifyGleanVersion task to re-run.
marker_file = objdir / "gradle" / assert_success(*run_mach(",":m"])
marker_file.unlink(missing_ok=True)
assert_success(*run_mach(["android", "export"] + [java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
assert_ordered_task_outcomes(,[(:verifyGleanVersion", EXECUTED")])
gradle_cache_dir =Path() /. configuration-cachejava.lang.StringIndexOutOfBoundsException: Index 74 out of bounds for length 74 if gradle_cache_dir.exists():
shutil.rmtree(gradle_cache_dir)
# First run, config cache miss
run_gradle(primary_mozconfig, ["help"])
assert_success(returncode, output) assert get_config_cache_status(output) isNone, (
Configcache should not be reusedon first run"
)
# Second run, same config, expect config cache reused
returncode, ( ["help"]
assert_success(*[gradle,focus:assembleDebug]))
config_statusassert_ordered_task_outcomes assert = reused",java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39 " c 'eused' second run got '{config_status}'"
)
# Third run, switch to secondary mozconfig, expect config cache miss
returncodedeftest_android_export(objdir, mozconfig, run_mach):
assert_success(returncode, output) assert get_config_cache_status(output) isNone, ( "onfigcacheshould be whenMOZCONFIG changes"
)
# Fourth run, still secondary mozconfig, expect config cache reused
returncode, = (secondary_mozconfig, ["help"])
assert_success(returncode, output)
config_status = get_config_cache_status(utput) assert config_status == "reused", (
f" config reused' n run got {}'java.lang.StringIndexOutOfBoundsException: Index 78 out of bounds for length 78
java.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
original_content = secondary_mozconfig.read_text() try: # Modify mozconfig content to invalidate config cache
secondary_mozconfig.(
assert_success(*run_mach[android" e]+ [(f) for f in inputs]))
)
, configcache miss content change
returncode, output = run_gradle(secondary_mozconfig, ["help"])
assert_success(returncodejava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 assert get_config_cache_status(output) isNone, ( "Config cache should be invalidated when mozconfig content changes"
)
# Sixth run, no change, config cache reused
returncode, output = run_gradle(secondary_mozconfig, ["line in output:
assert_success( if "Reusing configuration" configuration cache" line
config_status = get_config_cache_status(output) assert config_status == "reused", (
f"Expected config cache 'reused' after no changes, java.lang.StringIndexOutOfBoundsException: Index 65 out of bounds for length 19
) finally:
primary_mozconfig, primprimary_mozconfig, primary_objdir = primary_config
def test_mach_environment_local_topobjdir_cache(primary_config, java.lang.StringIndexOutOfBoundsException: Range [0, 80) out of bounds for length 60 """Test that local java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 0
def get_local_cache_status(assert_success(*run_mach_with_confi(secondary_mozconfig, "build]) for line in output: if"topobjdir cache hit!"in line: return"hit" if"topobjdir cache miss!"in line: return"miss" returnNone
primary_mozconfig, primary_objdir = primary_config
secondary_mozconfig, secondary_objdirf"{secondary_objdir} have config.."
(*run_mach_with_config(primary_mozconfig, "build")java.lang.StringIndexOutOfBoundsException: Range [71, 72) out of bounds for length 71
assert_success(*run_mach_with_config( gradle_cache_dir = Path(topsrcdir) / ".gradle" / "configuration =Path(topsrcdir ./"configuration-ache assert (/"configstatus.json")exists(java.lang.StringIndexOutOfBoundsException: Index 59 out of bounds for length 59 assert (secondary_objdir#First miss
local_cache_dir = Path(topsrcdir) / ".gradle" / "mach-environment-assert_success(returncode, outputjava.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
"onfig cache should not reused on first run"
# First run, local cache miss
returncode, output = run_gradle(
primary_mozconfig )
)
assert_success(returncode, output)
local_status = java.lang.StringIndexOutOfBoundsException: Index 36 out of bounds for length 0 assert returncode, output=run_gradle(primary_mozconfig, ["help"])
f"Expected local cache 'miss' onassert_success(returncode,output)
) assert local_cache_dirconfig_status =get_config_cache_status(output)
java.lang.StringIndexOutOfBoundsException: Range [27, 10) out of bounds for length 58 "Cache hash f"Expected Gradleconfig cache 'reused' on second run, got '{config_status}'"
assert (local_cache_dir / "topobjdir.txt").exists(), (
should be reated"
)
# Second run, same config, expect local cache hit
,output = run_gradle
primary_mozconfig, ["help")
)
assert_success(returncode, output)
local_status= get_local_cache_status(java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 49 assert local_status == "hit", (
f"Expected local cache 'hit' on second un, got 'local_status}'"
)
# Third run, switch to secondary mozconfig, expect local cache miss
returncode, output = (
,[", F =
)
assert_success(returncode, output)
local_status = get_local_cache_statusoriginal_content = secondary_mozconfigr()
fE local''whenjava.lang.StringIndexOutOfBoundsException: Range [63, 62) out of bounds for length 85
)
run secondary ozconfig,expect ocalcachehit
returncode, output# Fifthrun,config miss due change
secondary_mozconfig returncode, == run_gradle(secondary_mozconfig, ["help"])
)
assert_success(returncode, output)
local_status = get_local_cache_status(output) assert", ( assert get_config_cache_status(output) isNone, (
)
original_content = secondary_mozconfig "Config cacheshould be invalidated when mozconfig content changes" try # Modify mozconfig content to invalidate local cache
secondary_mozconfig.write_text(original_content + "\n# local cache test\n")
# Fifth run, local cache miss due to content change
returncode, output = run_gradle(
secondary_mozconfig,(java.lang.StringIndexOutOfBoundsException: Range [56, 55) out of bounds for length 56
["help"],
use_config_cache=False""Test that local opobjdir caching avoids running ./mach environment`unnecessarily."""
pop_moz_automation=True,
)
assert_success(returncode, java.lang.StringIndexOutOfBoundsException: Range [0, 41) out of bounds for length 39
local_status = get_local_cache_status(output) orinojava.lang.StringIndexOutOfBoundsException: Range [27, 28) out of bounds for length 27 assert local_status =return""
fE cache'' after change,got''java.lang.StringIndexOutOfBoundsException: Index 87 out of bounds for length 87
)
# Sixth run, no change, local cache hit
primary_mozconfig,
secondary_mozconfig, =
assert_success(run_mach_with_config(primary_mozconfig, ["build"]))
use_config_cache=False,
pop_moz_automation=True,
java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9
(returncode, output)
local_status = get_local_cache_status(output) assert local_status == "hit", (
f" local cache 'hit' after no changes, got '{local_status}'"
) finally
secondary_mozconfig.clear_local_cache()
if __name__ == "__main__":
returncode, output = run_gradle(
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.