# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this, # file, You can obtain one at http://mozilla.org/MPL/2.0/.
import errno import json import os import stat import subprocess import sys import time from pathlib import Path from typing import Optional, Union
import requests from tqdm import tqdm
# We need the NDK version in multiple different places, and it's inconvenient # to pass down the NDK version to all relevant places, so we have this global # variable. from mozboot.bootstrap import MOZCONFIG_SUGGESTION_TEMPLATE
# We expect the emulator AVD definitions to be platform agnostic
LINUX_X86_64_ANDROID_AVD = "linux64-android-avd-x86_64-repack"
LINUX_ARM_ANDROID_AVD = "linux64-android-avd-arm-repack"
ANDROID_NDK_EXISTS = """
Looks like you have the correct version of the Android NDK installed at:
%s """
ANDROID_SDK_EXISTS = """
Looks like you have the Android SDK installed at:
%s
We will install all required Android packages. """
ANDROID_SDK_TOO_OLD = """
Looks like you have an outdated Android SDK installed at:
%s
I can't update outdated Android SDKs to have the required 'sdkmanager'
tool. Move it out of the way (or remove it entirely) and then run
bootstrap again. """
INSTALLING_ANDROID_PACKAGES = """
We are now installing the following Android packages:
%s
You may be prompted to agree to the Android license. You may see some of
output as packages are downloaded and installed. """
MOBILE_ANDROID_MOZCONFIG_TEMPLATE = """ # Build GeckoView/Firefox for Android:
ac_add_options --enable-project=mobile/android
# Targeting the following architecture. # For regular phones, no --target is needed. # For x86 emulators (and x86 devices, which are uncommon): # ac_add_options --target=i686 # For newer phones or Apple silicon # ac_add_options --target=aarch64 # For x86_64 emulators (and x86_64 devices, which are even less common): # ac_add_options --target=x86_64
{extra_lines} # Write build artifacts to:
mk_add_options MOZ_OBJDIR=./objdir-frontend """
class GetNdkVersionError(Exception): pass
def install_mobile_android_sdk_or_ndk(url, path: Path): """
Fetch an Android SDK or NDK from |url| and unpack it into the given |path|.
We use, and'requests' respects, https. We could also include SHAs for a
small improvement in the integrity guarantee we give. But this script is
bootstrapped over https anyway, so it's a really minor improvement.
We keep a cache of the downloaded artifacts, writing into |path|/mozboot.
We don't yet clean the cache; it's better to waste some disk space and not require a long re-download than to wipe the cache prematurely. """
download_path = path / "mozboot" try:
download_path.mkdir(parents=True) except OSError as e: if e.errno == errno.EEXIST and download_path.is_dir(): pass else: raise
if file_name.endswith(".tar.gz") or file_name.endswith(".tgz"):
cmd = ["tar", "zxf", str(download_file_path)] elif file_name.endswith(".tar.bz2"):
cmd = ["tar", "jxf", str(download_file_path)] elif file_name.endswith(".zip"):
cmd = ["unzip", "-q", str(download_file_path)] elif file_name.endswith(".bin"): # Execute the .bin file, which unpacks the content.
mode = os.stat(path).st_mode
download_file_path.chmod(mode | stat.S_IXUSR)
cmd = [str(download_file_path)] else: raise NotImplementedError(f"Don't know how to unpack file: {file_name}")
print(f"Unpacking {download_file_path}...")
with open(os.devnull, "w") as stdout: # These unpack commands produce a ton of output; ignore it. The # .bin files are 7z archives; there's no command line flag to quiet # output, so we use this hammer.
subprocess.check_call(cmd, stdout=stdout, cwd=str(path))
print(f"Unpacking {download_file_path}... DONE") # Now delete the archive
download_file_path.unlink()
def download(
url,
download_file_path: Path,
): with requests.Session() as session:
request = session.head(url, allow_redirects=True)
request.raise_for_status()
remote_file_size = int(request.headers["content-length"])
if download_file_path.is_file():
local_file_size = download_file_path.stat().st_size
def download_internal(
download_file_path: Path,
session,
url,
remote_file_size,
resume_from_byte_pos: int = None,
): """
Handles both a fresh SDK/NDK download, as well as resuming a partial one """ # "ab" will behave same as "wb" if file does not exist with open(download_file_path, "ab") as file: # 64 KB/s should be fine on even the slowest internet connections
chunk_size = 1024 * 64 # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range#directives
resume_header = (
{"Range": f"bytes={resume_from_byte_pos}-"} if resume_from_byte_pos elseNone
)
with tqdm(
total=int(remote_file_size),
unit="B",
unit_scale=True,
unit_divisor=1024,
desc=download_file_path.name,
initial=resume_from_byte_pos if resume_from_byte_pos else 0,
) as progress_bar: for chunk in request.iter_content(chunk_size):
file.write(chunk)
progress_bar.update(len(chunk))
def get_ndk_version(ndk_path: Union[str, Path]): """Given the path to the NDK, return the version as a 3-tuple of (major,
minor, human). """
ndk_path = Path(ndk_path) with open(ndk_path / "source.properties", "r") as f:
revision = [line for line in f if line.startswith("Pkg.Revision")] ifnot revision: raise GetNdkVersionError( "Cannot determine NDK version from source.properties"
) if len(revision) != 1: raise GetNdkVersionError("Too many Pkg.Revision lines in source.properties")
(_, version) = revision[0].split("=") ifnot version: raise GetNdkVersionError( "Unexpected Pkg.Revision line in source.properties"
)
(major, minor, revision) = version.strip().split(".") ifnot major ornot minor: raise GetNdkVersionError("Unexpected NDK version string: " + version)
# source.properties contains a $MAJOR.$MINOR.$PATCH revision number, # but the more common nomenclature that Google uses is alphanumeric # version strings like "r20" or "r19c". Convert the source.properties # notation into an alphanumeric string.
int_minor = int(minor)
alphas = "abcdefghijklmnop"
ascii_minor = alphas[int_minor] if int_minor > 0 else""
human = "r%s%s" % (major, ascii_minor) return (major, minor, human)
def ensure_android(
os_name,
os_arch,
artifact_mode=False,
ndk_only=False,
system_images_only=False,
emulator_only=False,
avd_manifest_path: Optional[Path] = None,
prewarm_avd=False,
no_interactive=False,
list_packages=False,
): """
Ensure the Android SDK (and NDK, if `artifact_mode` is falsy) are
installed. Ifnot, fetch and unpack the SDK and/or NDK from the
given URLs. Ensure the required Android SDK packages are
installed.
`os_name` can be 'linux', 'macosx'or'windows'. """ # The user may have an external Android SDK (in which case we # save them a lengthy download), or they may have already # completed the download. We unpack to # ~/.mozbuild/{android-sdk-$OS_NAME, android-ndk-$VER}.
mozbuild_path, sdk_path, ndk_path, avd_home_path = get_paths(os_name)
avd_manifest = None if avd_manifest_path isnotNone: with open(avd_manifest_path) as f:
avd_manifest = json.load(f) # Some AVDs cannot be prewarmed in CI because they cannot run on linux64 # (like the arm64 AVD). if"emulator_prewarm"in avd_manifest:
prewarm_avd = prewarm_avd and avd_manifest["emulator_prewarm"]
# We expect the |sdkmanager| tool to be at # ~/.mozbuild/android-sdk-$OS_NAME/tools/cmdline-tools/$CMDLINE_TOOLS_VERSION_STRING/bin/sdkmanager. # NOQA: E501
ensure_android_packages(
os_name,
os_arch,
sdkmanager_tool=sdkmanager_tool(sdk_path),
emulator_only=emulator_only,
system_images_only=system_images_only,
avd_manifest=avd_manifest,
no_interactive=no_interactive,
list_packages=list_packages,
)
def ensure_android_sdk_and_ndk(
mozbuild_path: Path,
os_name,
sdk_path: Path,
sdk_url,
ndk_path: Path,
ndk_url,
bundletool_url,
artifact_mode,
ndk_only,
emulator_only,
): """
Ensure the Android SDK and NDK are found at the given paths. Ifnot, fetch and unpack the SDK and/or NDK from the given URLs into
|mozbuild_path/{android-sdk-$OS_NAME,android-ndk-$VER}|. """
# It's not particularly bad to overwrite the NDK toolchain, but it does take # a while to unpack, so let's avoid the disk activity if possible. The SDK # may prompt about licensing, so we do this first. # Check for Android NDK only if we are not in artifact mode. ifnot artifact_mode andnot emulator_only:
install_ndk = True if ndk_path.is_dir(): try:
_, _, human = get_ndk_version(ndk_path) if human == NDK_VERSION:
print(ANDROID_NDK_EXISTS % ndk_path)
install_ndk = False except GetNdkVersionError: pass# Just do the install. if install_ndk: # The NDK archive unpacks into a top-level android-ndk-$VER directory.
install_mobile_android_sdk_or_ndk(ndk_url, mozbuild_path)
if ndk_only: return
# We don't want to blindly overwrite, since we use the # |sdkmanager| tool to install additional parts of the Android # toolchain. If we overwrite, we lose whatever Android packages # the user may have already installed. if sdkmanager_tool(sdk_path).is_file():
print(ANDROID_SDK_EXISTS % sdk_path) elif sdk_path.is_dir(): raise NotImplementedError(ANDROID_SDK_TOO_OLD % sdk_path) else: # The SDK archive used to include a top-level # android-sdk-$OS_NAME directory; it no longer does so. We # preserve the old convention to smooth detecting existing SDK # installations.
cmdline_tools_path = mozbuild_path / f"android-sdk-{os_name}" / "cmdline-tools"
install_mobile_android_sdk_or_ndk(sdk_url, cmdline_tools_path) # The tools package *really* wants to be in # <sdk>/cmdline-tools/$CMDLINE_TOOLS_VERSION_STRING
(cmdline_tools_path / "cmdline-tools").rename(
cmdline_tools_path / CMDLINE_TOOLS_VERSION_STRING
)
download(bundletool_url, mozbuild_path / "bundletool.jar")
def ensure_android_avd(
avdmanager_tool: Path,
adb_tool: Path,
emulator_tool: Path,
avd_home_path: Path,
sdk_path: Path,
no_interactive=False,
avd_manifest=None,
prewarm_avd=False,
): """
Use the given sdkmanager tool (like 'sdkmanager') to install required
Android packages. """ if avd_manifest isNone: return
avd_home_path.mkdir(parents=True, exist_ok=True) # The AVD needs this folder to boot, so make sure it exists here.
(sdk_path / "platforms").mkdir(parents=True, exist_ok=True)
if config_file_name.is_file(): with open(config_file_name, "a") as config: for key, value in avd_manifest["emulator_extra_config"].items():
config.write("%s=%s\n" % (key, value)) else: raise NotImplementedError(
f"Could not find config file at {config_file_name}, something went wrong"
) if prewarm_avd:
run_prewarm_avd(adb_tool, emulator_tool, env, avd_name, avd_manifest) # When running in headless mode, the emulator does not run the cleanup # step, and thus doesn't delete lock files. On some platforms, left-over # lock files can cause the emulator to not start, so we remove them here. for lock_file in ["hardware-qemu.ini.lock", "multiinstance.lock"]:
lock_file_path = avd_path / lock_file try:
lock_file_path.unlink()
print(f"Removed lock file {lock_file_path}") except OSError: # The lock file is not there, nothing to do. pass
def run_prewarm_avd(
adb_tool: Path,
emulator_tool: Path,
env,
avd_name,
avd_manifest,
): """
Ensures the emulator is fully booted to save time on future iterations. """
args = [str(emulator_tool), "-avd", avd_name] + avd_manifest["emulator_extra_args"]
booted = False for i in range(100):
boot_completed_cmd = [str(adb_tool), "shell", "getprop", "sys.boot_completed"]
completed_proc = subprocess.Popen(
boot_completed_cmd, env=env, stdout=subprocess.PIPE
) try:
out, err = completed_proc.communicate(timeout=30)
boot_completed = out.decode("UTF-8").strip()
print("sys.boot_completed = %s" % boot_completed)
time.sleep(30) if boot_completed == "1":
booted = True break except subprocess.TimeoutExpired: # Sometimes the adb command hangs, that's ok
print("sys.boot_completed = Timeout")
ifnot booted: raise NotImplementedError("Could not prewarm emulator")
# Wait until the emulator completely shuts down
subprocess.Popen([str(adb_tool), "emu", "kill"], env=env).wait()
proc.wait()
def ensure_android_packages(
os_name,
os_arch,
sdkmanager_tool: Path,
emulator_only=False,
system_images_only=False,
avd_manifest=None,
no_interactive=False,
list_packages=False,
): """
Use the given sdkmanager tool (like 'sdkmanager') to install required
Android packages. """
# This tries to install all the required Android packages. The user # may be prompted to agree to the Android license. if system_images_only:
packages_file_name = "android-system-images-packages.txt" elif emulator_only:
packages_file_name = "android-emulator-packages.txt" else:
packages_file_name = "android-packages.txt"
args = [str(sdkmanager_tool)] if os_name == "macosx"and os_arch == "arm64": # Support for Apple Silicon is still in nightly
args.append("--channel=3")
args.extend(packages)
# Flush outputs before running sdkmanager.
sys.stdout.flush()
sys.stderr.flush() # Emulate yes. For a discussion of passing input to check_output, # see https://stackoverflow.com/q/10103551.
yes = "\n".join(["y"] * 100).encode("UTF-8")
proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
proc.communicate(yes)
retcode = proc.poll() if retcode:
cmd = args[0]
e = subprocess.CalledProcessError(retcode, cmd) raise e if list_packages:
subprocess.check_call([str(sdkmanager_tool), "--list"])
if os_name == "macosx": # |mach bootstrap| uses 'macosx', but Google uses 'darwin'.
os_name = "darwin"
return"%s-%s-%s.zip" % (base_url, ver, os_name)
def main(argv): import optparse # No argparse, which is new in Python 2.7. import platform
parser = optparse.OptionParser()
parser.add_option( "-a", "--artifact-mode",
dest="artifact_mode",
action="store_true",
help="If true, install only the Android SDK (and not the Android NDK).",
)
parser.add_option( "--jdk-only",
dest="jdk_only",
action="store_true",
help="If true, install only the Java JDK.",
)
parser.add_option( "--ndk-only",
dest="ndk_only",
action="store_true",
help="If true, install only the Android NDK (and not the Android SDK).",
)
parser.add_option( "--system-images-only",
dest="system_images_only",
action="store_true",
help="If true, install only the system images for the AVDs.",
)
parser.add_option( "--no-interactive",
dest="no_interactive",
action="store_true",
help="Accept the Android SDK licenses without user interaction.",
)
parser.add_option( "--emulator-only",
dest="emulator_only",
action="store_true",
help="If true, install only the Android emulator (and not the SDK or NDK).",
)
parser.add_option( "--avd-manifest",
dest="avd_manifest_path",
help="If present, generate AVD from the manifest pointed by this argument.",
)
parser.add_option( "--prewarm-avd",
dest="prewarm_avd",
action="store_true",
help="If true, boot the AVD and wait until completed to speed up subsequent boots.",
)
parser.add_option( "--list-packages",
dest="list_packages",
action="store_true",
help="If true, list installed packages.",
)
options, _ = parser.parse_args(argv)
if options.artifact_mode and options.ndk_only: raise NotImplementedError("Use no options to install the NDK and the SDK.")
if options.artifact_mode and options.emulator_only: raise NotImplementedError("Use no options to install the SDK and emulators.")
# |./mach bootstrap| automatically creates a mozconfig file for you if it doesn't # exist. However, here, we don't know where the "topsrcdir" is, and it's not worth # pulling in CommandContext (and its dependencies) to find out. # So, instead, we'll politely ask users to create (or update) the file themselves.
suggestion = MOZCONFIG_SUGGESTION_TEMPLATE % ("$topsrcdir/mozconfig", mozconfig)
print("\n" + suggestion)
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung ist noch experimentell.