Anforderungen  |   Konzepte  |   Entwurf  |   Entwicklung  |   Qualitätssicherung  |   Lebenszyklus  |   Steuerung
 
 
 
 


Quelle  mach_commands.py   Sprache: Python

 
# -*- coding: utf-8 -*-

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.


import argparse
import html
import json
import logging
import os
import re
import textwrap
import webbrowser

# Command files like this are listed in build/mach_initialize.py in alphabetical
# order, but we need to access commands earlier in the sorted order to grab
# their arguments. Force them to load now.
import mozbuild.artifact_commands  # NOQA: F401
import mozbuild.build_commands  # NOQA: F401
import mozhttpd
from mach.base import FailedCommandError, MachError
from mach.decorators import Command, CommandArgument, SubCommand
from mach.registrar import Registrar
from mozbuild.base import BuildEnvironmentNotFoundException
from mozbuild.mozconfig import MozconfigLoader


# Use a decorator to copy command arguments off of the named command. Instead
# of a decorator, this could be straight code that edits eg
# MachCommands.build_shell._mach_command.arguments, but that looked uglier.
def inherit_command_args(command, subcommand=None):
    """Decorator for inheriting all command-line arguments from `mach build`.

    This should come earlier in the source file than @Command or @SubCommand,
    because it relies on that decorator having run first."""

    def inherited(func):
        handler = Registrar.command_handlers.get(command)
        if handler is not None and subcommand is not None:
            handler = handler.subcommand_handlers.get(subcommand)
        if handler is None:
            raise MachError(
                "{} command unknown or not yet loaded".format(
                    command if subcommand is None else command + " " + subcommand
                )
            )
        func._mach_command.arguments.extend(handler.arguments)
        return func

    return inherited


def state_dir():
    return os.environ.get("MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild"))


def tools_dir():
    if os.environ.get("MOZ_FETCHES_DIR"):
        # In automation, tools are provided by toolchain dependencies.
        return os.path.join(os.environ["HOME"], os.environ["MOZ_FETCHES_DIR"])

    # In development, `mach hazard bootstrap` installs the tools separately
    # to avoid colliding with the "main" compiler versions, which can
    # change separately (and the precompiled sixgill and compiler version
    # must match exactly).
    return os.path.join(state_dir(), "hazard-tools")


def sixgill_dir():
    return os.path.join(tools_dir(), "sixgill")


def gcc_dir():
    return os.path.join(tools_dir(), "gcc")


def script_dir(command_context):
    return os.path.join(command_context.topsrcdir, "js/src/devtools/rootAnalysis")


def get_work_dir(command_context, project, given):
    if given is not None:
        return given
    return os.path.join(command_context.topsrcdir, "haz-" + project)


def get_objdir(command_context, kwargs):
    project = kwargs["project"]
    objdir = kwargs["haz_objdir"]
    if objdir is None:
        objdir = os.environ.get("HAZ_OBJDIR")
    if objdir is None:
        objdir = os.path.join(command_context.topsrcdir, "obj-analyzed-" + project)
    return objdir


def ensure_dir_exists(dir):
    os.makedirs(dir, exist_ok=True)
    return dir


# Force the use of hazard-compatible installs of tools.
def setup_env_for_tools(env):
    gccbin = os.path.join(gcc_dir(), "bin")
    env["CC"] = os.path.join(gccbin, "gcc")
    env["CXX"] = os.path.join(gccbin, "g++")
    env["PATH"] = "{sixgill_dir}/usr/bin:{gccbin}:{PATH}".format(
        sixgill_dir=sixgill_dir(), gccbin=gccbin, PATH=env["PATH"]
    )


def setup_env_for_shell(env, shell):
    """Add JS shell directory to dynamic lib search path"""
    for var in ("LD_LIBRARY_PATH""DYLD_LIBRARY_PATH"):
        env[var] = ":".join(p for p in (env.get(var), os.path.dirname(shell)) if p)


@Command(
    "hazards",
    category="build",
    order="declaration",
    description="Commands for running the static analysis for GC rooting hazards",
)
def hazards(command_context):
    """Commands related to performing the GC rooting hazard analysis"""
    print("See `mach hazards --help` for a list of subcommands")


@inherit_command_args("artifact""toolchain")
@SubCommand(
    "hazards",
    "bootstrap",
    description="Install prerequisites for the hazard analysis",
)
def bootstrap(command_context, **kwargs):
    orig_dir = os.getcwd()
    os.chdir(ensure_dir_exists(tools_dir()))
    try:
        kwargs["from_build"] = ("linux64-gcc-sixgill""linux64-gcc-9")
        command_context._mach_context.commands.dispatch(
            "artifact", command_context._mach_context, subcommand="toolchain", **kwargs
        )
    finally:
        os.chdir(orig_dir)


CLOBBER_CHOICES = {"objdir""work""shell""all"}


@SubCommand("hazards""clobber", description="Clean up hazard-related files")
@CommandArgument("--project", default="browser", help="Build the given project.")
@CommandArgument("--application", dest="project", help="Build the given project.")
@CommandArgument("--haz-objdir", default=None, help="Hazard analysis objdir.")
@CommandArgument(
    "--work-dir", default=None, help="Directory for output and working files."
)
@CommandArgument(
    "what",
    default=["objdir""work"],
    nargs="*",
    help="Target to clobber, must be one of {{{}}} (default "
    "objdir and work).".format(", ".join(CLOBBER_CHOICES)),
)
def clobber(command_context, what, **kwargs):
    from mozbuild.controller.clobber import Clobberer

    what = set(what)
    if "all" in what:
        what.update(CLOBBER_CHOICES)
    invalid = what - CLOBBER_CHOICES
    if invalid:
        print(
            "Unknown clobber target(s): {}. Choose from {{{}}}".format(
                ", ".join(invalid), ", ".join(CLOBBER_CHOICES)
            )
        )
        return 1

    try:
        substs = command_context.substs
    except BuildEnvironmentNotFoundException:
        substs = {}

    if "objdir" in what:
        objdir = get_objdir(command_context, kwargs)
        print(f"removing {objdir}")
        Clobberer(command_context.topsrcdir, objdir, substs).remove_objdir(full=True)
    if "work" in what:
        project = kwargs["project"]
        work_dir = get_work_dir(command_context, project, kwargs["work_dir"])
        print(f"removing {work_dir}")
        Clobberer(command_context.topsrcdir, work_dir, substs).remove_objdir(full=True)
    if "shell" in what:
        objdir = os.path.join(command_context.topsrcdir, "obj-haz-shell")
        print(f"removing {objdir}")
        Clobberer(command_context.topsrcdir, objdir, substs).remove_objdir(full=True)


@inherit_command_args("build")
@SubCommand(
    "hazards""build-shell", description="Build a shell for the hazard analysis"
)
@CommandArgument(
    "--mozconfig",
    default=None,
    metavar="FILENAME",
    help="Build with the given mozconfig.",
)
def build_shell(command_context, **kwargs):
    """Build a JS shell to use to run the rooting hazard analysis."""
    # The JS shell requires some specific configuration settings to execute
    # the hazard analysis code, and configuration is done via mozconfig.
    # Subprocesses find MOZCONFIG in the environment, so we can't just
    # modify the settings in this process's loaded version. Pass it through
    # the environment.

    default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.haz_shell"
    mozconfig_path = (
        kwargs.pop("mozconfig"None)
        or os.environ.get("MOZCONFIG")
        or default_mozconfig
    )
    mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path)
    loader = MozconfigLoader(command_context.topsrcdir)
    mozconfig = loader.read_mozconfig(mozconfig_path)

    # Validate the mozconfig settings in case the user overrode the default.
    configure_args = mozconfig["configure_args"]
    if "--enable-ctypes" not in configure_args:
        raise FailedCommandError(
            "ctypes required in hazard JS shell, mozconfig=" + mozconfig_path
        )

    # Transmit the mozconfig location to build subprocesses.
    os.environ["MOZCONFIG"] = mozconfig_path

    setup_env_for_tools(os.environ)

    # Set a default objdir for the shell, for developer builds.
    os.environ.setdefault(
        "MOZ_OBJDIR", os.path.join(command_context.topsrcdir, "obj-haz-shell")
    )

    return command_context._mach_context.commands.dispatch(
        "build", command_context._mach_context, **kwargs
    )


def read_json_file(filename):
    with open(filename) as fh:
        return json.load(fh)


def ensure_shell(command_context, objdir):
    if objdir is None:
        objdir = os.path.join(command_context.topsrcdir, "obj-haz-shell")

    try:
        binaries = read_json_file(os.path.join(objdir, "binaries.json"))
        info = [b for b in binaries["programs"if b["program"] == "js"][0]
        return os.path.join(objdir, info["install_target"], "js")
    except (OSError, KeyError):
        raise FailedCommandError(
            """\
no shell found in %s -- must build the JS shell with `mach hazards build-shell` first"""
            % objdir
        )


def validate_mozconfig(command_context, kwargs):
    app = kwargs.pop("project")
    default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.%s" % app
    mozconfig_path = (
        kwargs.pop("mozconfig"None)
        or os.environ.get("MOZCONFIG")
        or default_mozconfig
    )
    mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path)

    loader = MozconfigLoader(command_context.topsrcdir)
    mozconfig = loader.read_mozconfig(mozconfig_path)
    configure_args = mozconfig["configure_args"]

    # Require an explicit --enable-project/application=APP (even if you just
    # want to build the default browser project.)
    if (
        "--enable-project=%s" % app not in configure_args
        and "--enable-application=%s" % app not in configure_args
    ):
        raise FailedCommandError(
            textwrap.dedent(
                f"""\
            mozconfig {mozconfig_path} builds wrong project.
            unset MOZCONFIG to use the default {default_mozconfig}\
            """
            )
        )

    if not any("--with-compiler-wrapper" in a for a in configure_args):
        raise FailedCommandError(
            "mozconfig must wrap compiles with --with-compiler-wrapper"
        )

    return mozconfig_path


@inherit_command_args("build")
@SubCommand(
    "hazards",
    "gather",
    description="Gather analysis data by compiling the given project",
)
@CommandArgument("--project", default="browser", help="Build the given project.")
@CommandArgument("--application", dest="project", help="Build the given project.")
@CommandArgument(
    "--haz-objdir", default=None, help="Write object files to this directory."
)
@CommandArgument(
    "--work-dir", default=None, help="Directory for output and working files."
)
def gather_hazard_data(command_context, **kwargs):
    """Gather analysis information by compiling the tree"""
    project = kwargs["project"]
    objdir = get_objdir(command_context, kwargs)

    work_dir = get_work_dir(command_context, project, kwargs["work_dir"])
    ensure_dir_exists(work_dir)
    with open(os.path.join(work_dir, "defaults.py"), "wt"as fh:
        data = textwrap.dedent(
            """\
            analysis_scriptdir = "{script_dir}"
            objdir = "{objdir}"
            source = "{srcdir}"
            sixgill = "{sixgill_dir}/usr/libexec/sixgill"
            sixgill_bin = "{sixgill_dir}/usr/bin"
        """
        ).format(
            script_dir=script_dir(command_context),
            objdir=objdir,
            srcdir=command_context.topsrcdir,
            sixgill_dir=sixgill_dir(),
            gcc_dir=gcc_dir(),
        )
        fh.write(data)

    buildscript = " ".join(
        [
            command_context.topsrcdir + "/mach hazards compile",
            *kwargs.get("what", []),
            "--job-size=3.0",  # Conservatively estimate 3GB/process
            "--project=" + project,
            "--haz-objdir=" + objdir,
        ]
    )
    args = [
        os.path.join(script_dir(command_context), "run_complete"),
        "--foreground",
        "--no-logs",
        "--build-root=" + objdir,
        "--wrap-dir=" + sixgill_dir() + "/usr/libexec/sixgill/scripts/wrap_gcc",
        "--work-dir=work",
        "-b",
        sixgill_dir() + "/usr/bin",
        "--buildcommand=" + buildscript,
        ".",
    ]

    return command_context.run_process(args=args, cwd=work_dir, pass_thru=True)


@inherit_command_args("build")
@SubCommand("hazards""compile", description=argparse.SUPPRESS)
@CommandArgument(
    "--mozconfig",
    default=None,
    metavar="FILENAME",
    help="Build with the given mozconfig.",
)
@CommandArgument("--project", default="browser", help="Build the given project.")
@CommandArgument("--application", dest="project", help="Build the given project.")
@CommandArgument(
    "--haz-objdir",
    default=os.environ.get("HAZ_OBJDIR"),
    help="Write object files to this directory.",
)
def inner_compile(command_context, **kwargs):
    """Build a source tree and gather analysis information while running
    under the influence of the analysis collection server."""

    env = os.environ

    # Check whether we are running underneath the manager (and therefore
    # have a server to talk to).
    if "XGILL_CONFIG" not in env:
        raise FailedCommandError(
            "no sixgill manager detected. `mach hazards compile` "
            + "should only be run from `mach hazards gather`"
        )

    mozconfig_path = validate_mozconfig(command_context, kwargs)

    # Communicate mozconfig to build subprocesses.
    env["MOZCONFIG"] = os.path.join(command_context.topsrcdir, mozconfig_path)

    # hazard mozconfigs need to find binaries in .mozbuild
    env["MOZBUILD_STATE_PATH"] = state_dir()

    # Suppress the gathering of sources, to save disk space and memory.
    env["XGILL_NO_SOURCE"] = "1"

    setup_env_for_tools(env)

    if "haz_objdir" in kwargs:
        env["MOZ_OBJDIR"] = kwargs.pop("haz_objdir")

    return command_context._mach_context.commands.dispatch(
        "build", command_context._mach_context, **kwargs
    )


@SubCommand(
    "hazards""analyze", description="Analyzed gathered data for rooting hazards"
)
@CommandArgument(
    "--project",
    default="browser",
    help="Analyze the output for the given project.",
)
@CommandArgument("--application", dest="project", help="Build the given project.")
@CommandArgument(
    "--shell-objdir",
    default=None,
    help="objdir containing the optimized JS shell for running the analysis.",
)
@CommandArgument(
    "--work-dir", default=None, help="Directory for output and working files."
)
@CommandArgument(
    "--jobs""-j", default=None, type=int, help="Number of parallel analyzers."
)
@CommandArgument(
    "--verbose",
    "-v",
    default=False,
    action="store_true",
    help="Display executed commands.",
)
@CommandArgument(
    "--from-stage",
    default=None,
    help="Stage to begin running at ('list' to see all).",
)
@CommandArgument(
    "extra",
    nargs=argparse.REMAINDER,
    default=(),
    help="Remaining non-optional arguments to analyze.py script",
)
def analyze(
    command_context,
    project,
    shell_objdir,
    work_dir,
    jobs,
    verbose,
    from_stage,
    extra,
):
    """Analyzed gathered data for rooting hazards"""

    shell = ensure_shell(command_context, shell_objdir)
    args = [
        os.path.join(script_dir(command_context), "analyze.py"),
        "--js",
        shell,
        *extra,
    ]

    if from_stage is None:
        pass
    elif from_stage == "list":
        args.append("--list")
    else:
        args.extend(["--first", from_stage])

    if jobs is not None:
        args.extend(["-j", jobs])

    if verbose:
        args.append("-v")

    setup_env_for_tools(os.environ)
    setup_env_for_shell(os.environ, shell)

    work_dir = get_work_dir(command_context, project, work_dir)
    return command_context.run_process(args=args, cwd=work_dir, pass_thru=True)


@SubCommand(
    "hazards",
    "self-test",
    description="Run a self-test to verify hazards are detected",
)
@CommandArgument(
    "--shell-objdir",
    default=None,
    help="objdir containing the optimized JS shell for running the analysis.",
)
@CommandArgument(
    "extra",
    nargs=argparse.REMAINDER,
    help="Remaining non-optional arguments to pass to run-test.py",
)
def self_test(command_context, shell_objdir, extra):
    """Analyzed gathered data for rooting hazards"""
    shell = ensure_shell(command_context, shell_objdir)
    args = [
        os.path.join(script_dir(command_context), "run-test.py"),
        "-v",
        "--js",
        shell,
        "--sixgill",
        os.path.join(tools_dir(), "sixgill"),
        "--gccdir",
        gcc_dir(),
    ]
    args.extend(extra)

    setup_env_for_tools(os.environ)
    setup_env_for_shell(os.environ, shell)

    return command_context.run_process(args=args, pass_thru=True)


def annotated_source(filename, query):
    """The index page has URLs of the format <http://.../path/to/source.cpp?L=m-n#m>.
    The `#m` part will be stripped off and used by the browser to jump to the correct line.
    The `?L=m-n` or `?L=m` parameter will be processed here on the server to highlight
    the given line range."""
    linequery = query.replace("L=""")
    if "-" in linequery:
        line0, line1 = linequery.split("-", 1)
    else:
        line0, line1 = linequery or "0", linequery or "0"
    line0 = int(line0)
    line1 = int(line1)

    fh = open(filename, "rt")

    out = "
"
    for lineno, line in enumerate(fh, 1):
        processed = f"{lineno}
        if line0 <= lineno and lineno <= line1:
            processed += " style='background: yellow'"
        processed += ">" + html.escape(line.rstrip()) + "\n"

        out += processed

    return out


@SubCommand(
    "hazards""view", description="Display a web page describing any hazards found"
)
@CommandArgument(
    "--project",
    default="browser",
    help="Analyze the output for the given project.",
)
@CommandArgument("--application", dest="project", help="Build the given project.")
@CommandArgument(
    "--haz-objdir", default=None, help="Write object files to this directory."
)
@CommandArgument(
    "--work-dir", default=None, help="Directory for output and working files."
)
@CommandArgument("--port", default=6006, help="Port of the web server")
@CommandArgument(
    "--serve-only",
    default=False,
    action="store_true",
    help="Serve only, do not navigate to page",
)
def view_hazards(command_context, project, haz_objdir, work_dir, port, serve_only):
    work_dir = get_work_dir(command_context, project, work_dir)
    haztop = os.path.basename(work_dir)
    if haz_objdir is None:
        haz_objdir = os.environ.get("HAZ_OBJDIR")
    if haz_objdir is None:
        haz_objdir = os.path.join(command_context.topsrcdir, "obj-analyzed-" + project)

    httpd = None

    def serve_source_file(request, path):
        info = {"req": path}

        def log(fmt, level=logging.INFO):
            return command_context.log(level, "view-hazards", info, fmt)

        if path in ("", f"{haztop}"):
            info["dest"] = f"/{haztop}/hazards.html"
            info["code"] = 301
            log("serve '{req}' -> {code} {dest}")
            return (info["code"], {"Location": info["dest"]}, "")

        # Allow files to be served from the source directory or the objdir.
        roots = (command_context.topsrcdir, haz_objdir)

        try:
            # Validate the path. Some source files have weird characters in their paths (eg "+"), but they
            # all start with an alphanumeric or underscore.
            command_context.log(
                logging.DEBUG, "view-hazards", {"path": path}, "Raw path: {path}"
            )
            path_component = r"\w[\w\-\.\+]*"
            if not re.match(f"({path_component}/)*{path_component}$", path):
                raise ValueError("invalid path")

            # Resolve the path to under one of the roots, and
            # ensure that the actual file really is underneath a root directory.
            for rootdir in roots:
                fullpath = os.path.join(rootdir, path)
                info["path"] = fullpath
                fullpath = os.path.realpath(fullpath)
                if os.path.isfile(fullpath):
                    # symlinks between roots are ok, but not symlinks outside of the roots.
                    tops = [
                        d
                        for d in roots
                        if fullpath.startswith(os.path.realpath(d) + "/")
                    ]
                    if len(tops) > 0:
                        break  # Found a file underneath a root.
            else:
                raise IOError("not found")

            html = annotated_source(fullpath, request.query)
            log("serve '{req}' -> 200 {path}")
            return (
                200,
                {"Content-type""text/html""Content-length": len(html)},
                html,
            )
        except (IOError, ValueError):
            log("serve '{req}' -> 404 {path}", logging.ERROR)
            return (
                404,
                {"Content-type""text/plain"},
                "We don't have that around here. Don't be asking for it.",
            )

    httpd = mozhttpd.MozHttpd(
        port=port,
        docroot=None,
        path_mappings={"/" + haztop: work_dir},
        urlhandlers=[
            # Treat everything not starting with /haz-browser/ (or /haz-js/)
            # as a source file to be processed. Everything else is served
            # as a plain file.
            {
                "method""GET",
                "path""/(?!haz-" + project + "/)(.*)",
                "function": serve_source_file,
            },
        ],
        log_requests=True,
    )

    # The mozhttpd request handler class eats log messages.
    httpd.handler_class.log_message = lambda self, format, *args: command_context.log(
        logging.INFO, "view-hazards", {}, format % args
    )

    print("Serving at %s:%s" % (httpd.host, httpd.port))

    httpd.start(block=False)
    url = httpd.get_url(f"/{haztop}/hazards.html")
    display_url = True
    if not serve_only:
        try:
            webbrowser.get().open_new_tab(url)
            display_url = False
        except Exception:
            pass
    if display_url:
        print("Please open %s in a browser." % url)

    print("Hit CTRL+c to stop server.")
    httpd.server.join()

94%


¤ Dauer der Verarbeitung: 0.18 Sekunden  (vorverarbeitet)  ¤

*© Formatika GbR, Deutschland






Wurzel

Suchen

Beweissystem der NASA

Beweissystem Isabelle

NIST Cobol Testsuite

Cephes Mathematical Library

Wiener Entwicklungsmethode

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.

Bemerkung:

Die farbliche Syntaxdarstellung ist noch experimentell.






                                                                                                                                                                                                                                                                                                                                                                                                     


Neuigkeiten

     Aktuelles
     Motto des Tages

Software

     Produkte
     Quellcodebibliothek

Aktivitäten

     Artikel über Sicherheit
     Anleitung zur Aktivierung von SSL

Muße

     Gedichte
     Musik
     Bilder

Jenseits des Üblichen ....

Besucherstatistik

Besucherstatistik

Monitoring

Montastic status badge