Eine aufbereitete Darstellung der Quelle

 
     
 
 
Anforderungen  |   Konzepte  |   Entwurf  |   Entwicklung  |   Qualitätssicherung  |   Lebenszyklus  |   Steuerung
 
 
 
 

Benutzer

Quelle  run_task.py

  Sprache: Python
 

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Support for running jobs that are invoked via the `run-task` script.
"""

import dataclasses
import functools
import os
from typing import Literal, Union
from typing import Optional as TOptional

from mozpack import path
from taskgraph.transforms.run.common import support_caches
from taskgraph.util.schema import (
    Schema,
    taskref_or_string_msgspec,
)
from taskgraph.util.yaml import load_yaml

from gecko_taskgraph.transforms.job import run_job_using
from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout


class RunTaskSchema(Schema, kw_only=True):
    using: Literal["run-task"]
    # Use the specified caches.
    use_caches: TOptional[Union[bool, list[str]]] = None
    # if true (the default), perform a checkout of gecko on the worker
    checkout: bool = True
    # Path to run command in. If a checkout is present, the path
    # to the checkout will be interpolated with the key `checkout`
    cwd: TOptional[str] = None
    # The sparse checkout profile to use. Value is the filename relative to
    # "sparse-profile-prefix" which defaults to "build/sparse-profiles/".
    sparse_profile: TOptional[Union[str, None]] = None
    # The relative path to the sparse profile.
    sparse_profile_prefix: TOptional[str] = None
    # Whether to use a shallow clone or not, default True (git only).
    shallow_clone: TOptional[bool] = None
    # if true, perform a checkout of a comm-central based branch inside the
    # gecko checkout
    comm_checkout: bool = False
    # The command arguments to pass to the `run-task` script, after the
    # checkout arguments.  If a list, it will be passed directly; otherwise
    # it will be included in a single argument to `bash -cx`.
    command: Union[list[taskref_or_string_msgspec], taskref_or_string_msgspec]
    # Base work directory used to set up the task.
    workdir: TOptional[str] = None
    # If not false, tooltool downloads will be enabled via relengAPIProxy
    # for either just public files, or all files. Only supported on
    # docker-worker.
    tooltool_downloads: Union[bool, Literal["public""internal"]] = False
    # Whether to run as root. (defaults to False)
    run_as_root: TOptional[bool] = None

    def __post_init__(self):
        super().__post_init__()
        if self.tooltool_downloads is True:
            raise ValueError(
                "tooltool-downloads must be False, 'public', or 'internal'"
            )


def common_setup(config, job, taskdesc, command):
    run = job["run"]
    run_cwd = run.get("cwd")
    if run["checkout"]:
        repo_configs = config.repo_configs
        if len(repo_configs) > 1 and run["checkout"is True:
            raise Exception("Must explicitly specify checkouts with multiple repos.")
        elif run["checkout"is not True:
            repo_configs = {
                repo: dataclasses.replace(repo_configs[repo], **config)
                for (repo, config) in run["checkout"].items()
            }

        gecko_path = support_vcs_checkout(config, job, taskdesc, repo_configs)
        command.append(f"--gecko-checkout={gecko_path}")
        if config.params["repository_type"] == "git" and run.get("shallow-clone"True):
            command.append("--gecko-shallow-clone")

        if run_cwd:
            run_cwd = path.normpath(run_cwd.format(checkout=gecko_path))

    elif run_cwd and "{checkout}" in run_cwd:
        raise Exception(
            "Found `{{checkout}}` interpolation in `cwd` for task {name} "
            "but the task doesn't have a checkout: {cwd}".format(
                cwd=run_cwd, name=job.get("name", job.get("label"))
            )
        )

    if config.params["repository_type"] == "hg" and run["sparse-profile"]:
        sparse_profile_prefix = run.pop(
            "sparse-profile-prefix""build/sparse-profiles"
        )
        sparse_profile_path = path.join(sparse_profile_prefix, run["sparse-profile"])
        command.append(f"--gecko-sparse-profile={sparse_profile_path}")

    if run_cwd:
        command.append(f"--task-cwd={run_cwd}")

    support_caches(config, job, taskdesc)
    taskdesc["worker"].setdefault("env", {})["MOZ_SCM_LEVEL"] = config.params["level"]


worker_defaults = {
    "checkout"True,
    "comm-checkout"False,
    "sparse-profile"None,
    "tooltool-downloads"False,
    "run-as-root"False,
}


load_yaml = functools.cache(load_yaml)


def script_url(config, script):
    if "MOZ_AUTOMATION" in os.environ and "TASK_ID" not in os.environ:
        raise Exception("TASK_ID must be defined to use run-task on generic-worker")
    task_id = os.environ.get("TASK_ID""<TASK_ID>")
    tc_url = "http://firefox-ci-tc.services.mozilla.com"
    return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"


@run_job_using(
    "docker-worker""run-task", schema=RunTaskSchema, defaults=worker_defaults
)
def docker_worker_run_task(config, job, taskdesc):
    run = job["run"]
    worker = taskdesc["worker"] = job["worker"]
    run_task_bin = (
        "run-task-git" if config.params["repository_type"] == "git" else "run-task-hg"
    )
    command = [f"/builds/worker/bin/{run_task_bin}"]
    common_setup(config, job, taskdesc, command)

    if run["tooltool-downloads"]:
        internal = run["tooltool-downloads"] == "internal"
        add_tooltool(config, job, taskdesc, internal=internal)

    run_command = run["command"]

    # dict is for the case of `{'task-reference': text_type}`.
    if isinstance(run_command, (str, dict)):
        run_command = ["bash""-cx", run_command]
    if run["comm-checkout"]:
        command.append(
            "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"])
        )
    if run["run-as-root"]:
        command.extend(("--user""root""--group""root"))
    command.append("--")
    command.extend(run_command)
    worker["command"] = command


@run_job_using(
    "generic-worker""run-task", schema=RunTaskSchema, defaults=worker_defaults
)
def generic_worker_run_task(config, job, taskdesc):
    run = job["run"]
    worker = taskdesc["worker"] = job["worker"]
    is_win = worker["os"] == "windows"
    is_mac = worker["os"] == "macosx"
    is_bitbar = worker["os"] == "linux-bitbar"
    is_lambda = worker["os"] == "linux-lambda"

    if run["tooltool-downloads"]:
        internal = run["tooltool-downloads"] == "internal"
        add_tooltool(config, job, taskdesc, internal=internal)

    if is_win:
        command = ["C:/mozilla-build/python3/python3.exe""run-task"]
    elif is_mac:
        command = ["/usr/local/bin/python3""run-task"]
    else:
        command = ["./run-task"]

    common_setup(config, job, taskdesc, command)

    worker.setdefault("mounts", [])
    run_task_bin = (
        "run-task-git" if config.params["repository_type"] == "git" else "run-task-hg"
    )
    worker["mounts"].append({
        "content": {
            "url": script_url(config, run_task_bin),
        },
        "file""./run-task",
    })

    if (
        job.get("fetches")
        or job.get("use-uv")
        or job.get("use-python""system") != "system"
    ):
        worker["mounts"].append({
            "content": {
                "url": script_url(config, "fetch-content"),
            },
            "file""./fetch-content",
        })
    if run.get("checkout"):
        worker["mounts"].append({
            "content": {
                "url": script_url(config, "robustcheckout.py"),
            },
            "file""./robustcheckout.py",
        })

    run_command = run["command"]

    # dict is for the case of `{'task-reference': text_type}`.
    if isinstance(run_command, (str, dict)):
        if is_win:
            if isinstance(run_command, dict):
                for k in run_command.keys():
                    run_command[k] = f'"{run_command[k]}"'
            else:
                run_command = f'"{run_command}"'
        run_command = ["bash""-cx", run_command]

    if run["comm-checkout"]:
        command.append(
            "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"])
        )

    if run["run-as-root"]:
        command.extend(("--user""root""--group""root"))
    command.append("--")
    if is_bitbar:
        # Use the bitbar wrapper script which sets up the device and adb
        # environment variables
        command.append("/builds/taskcluster/script.py")
    elif is_lambda:
        command.append("/home/ltuser/taskcluster/script.py")
    command.extend(run_command)

    if is_win:
        taskref = False
        for c in command:
            if isinstance(c, dict):
                taskref = True

        if taskref:
            cmd = []
            for c in command:
                if isinstance(c, dict):
                    for v in c.values():
                        cmd.append(v)
                else:
                    cmd.append(c)
            worker["command"] = [{"artifact-reference"" ".join(cmd)}]
        else:
            worker["command"] = [" ".join(command)]
    else:
        worker["command"] = [
            ["chmod""+x""run-task"],
            command,
        ]

Messung V0.5 in Prozent
C=94 H=93 G=93

¤ Dauer der Verarbeitung: 0.4 Sekunden  ¤

*© Formatika GbR, Deutschland






Wurzel

Suchen

PVS Prover

Isabelle Prover

NIST Cobol Testsuite

Cephes Mathematical Library

Vienna Development Method

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.

Bemerkung:

Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.






                                                                                                                                                                                                                                                                                                                                                                                                     


Neuigkeiten

     Aktuelles
     Motto des Tages

Open Source Software

     Quellcodebibliothek
     Eigene Quellcodes
     Fremde Quellcodes
     Suchen

Jenseits des Üblichen ....

Besucherstatistik

Besucherstatistik

Statistik
#Sources=277311
#Domains=752002