# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Runs the static rooting analysis """
import argparse import os import subprocess import sys from subprocess import Popen
try: from shlex import quote except ImportError: from pipes import quote
# Label a string as an output. class Output(str): pass
# Label a string as a pattern for multiple inputs. class MultiInput(str): pass
# Construct a new environment by merging in some settings needed for running the individual scripts. def env(config): # Add config['sixgill_bin'] to $PATH if not already there.
path = os.environ["PATH"].split(":") if dir := config.get("sixgill_bin"): if dir notin path:
path.insert(0, dir)
def fill(command, config):
filled = [] for s in command: try:
rep = s.format(**config) except KeyError:
print("Substitution failed: %s" % s)
filled = None break
if isinstance(s, Output):
filled.append(Output(rep)) elif isinstance(s, MultiInput):
N = int(config["jobs"]) for i in range(1, N + 1):
filled.append(rep.format(i=i, n=N)) else:
filled.append(rep)
if filled isNone: raise Exception("substitution failure")
return tuple(filled)
def print_command(job, config, env=None): # Display a command to run that has roughly the same effect as what was # actually run. The actual command uses temporary files that get renamed at # the end, and run some commands in parallel chunks. The printed command # will substitute in the actual output and run in a single chunk, so that # it is easier to cut & paste and add a --function flag for debugging.
cfg = dict(config, n=1, i=1, jobs=1)
cmd = job_command_with_final_output_names(job)
cmd = fill(cmd, cfg)
cmd = [quote(s) for s in cmd] if outfile := job.get("redirect-output"):
cmd.extend([">", quote(outfile.format(**cfg))]) if HOME := os.environ.get("HOME"):
cmd = [s.replace(HOME, "~") for s in cmd]
if env: # Try to keep the command as short as possible by only displaying # modified environment variable settings.
e = os.environ
changed = {key: value for key, value in env.items() if value != e.get(key)} if changed:
settings = [] for key, value in changed.items(): if key in e and e[key] in value: # Display modifications as V=prefix${V}suffix when # possible. This can make a huge different for $PATH.
start = value.index(e[key])
end = start + len(e[key])
setting = '%s="%s${%s}%s"' % (key, value[:start], key, value[end:]) else:
setting = '%s="%s"' % (key, value) if HOME:
setting = setting.replace(HOME, "$HOME")
settings.append(setting)
# Generator of (i, j, item) tuples corresponding to outputs: # - i is just the index of the yielded tuple (a la enumerate()) # - j is the index of the item in the command list # - item is command[j] def out_indexes(command):
i = 0 for j, fragment in enumerate(command): if isinstance(fragment, Output): yield (i, j, fragment)
i += 1
def job_command_with_final_output_names(job):
outfiles = job.get("outputs", [])
command = list(job["command"]) for i, j, name in out_indexes(job["command"]):
command[j] = outfiles[i] return command
def run_job(name, config):
job = JOBS[name]
outs = job.get("outputs") or job.get("redirect-output")
print("Running " + name + " to generate " + str(outs)) if"function"in job:
job["function"](config, job["redirect-output"]) return
N = int(config["jobs"]) if job.get("multi-output") else 1
config["n"] = N
jobs = {} for i in range(1, N + 1):
config["i"] = i
cmd = fill(job["command"], config)
info = spawn_command(cmd, job, name, config)
jobs[info["proc"].pid] = info
if config["verbose"] > 0:
print_command(job, config, env=env(config))
final_status = 0 while jobs:
pid, status = os.wait()
final_status = final_status or status
info = jobs[pid] del jobs[pid] if"redirect"in info:
info["redirect"].close()
# Rename the temporary files to their final names. for temp, final in info["rename_map"].items(): try: if config["verbose"] > 1:
print("Renaming %s -> %s" % (temp, final))
os.rename(temp, final) except OSError:
print("Error renaming %s -> %s" % (temp, final)) raise
if final_status != 0: raise Exception("job {} returned status {}".format(name, final_status))
# Replace the Outputs with temporary filenames, and record a mapping # from those temp names to their actual final names that will be used # if the command succeeds.
command = list(cmdspec) for i, j, raw_name in out_indexes(cmdspec):
[name] = fill([raw_name], config)
command[j] = "{}.tmp{}".format(name, config.get("i", ""))
rename_map[command[j]] = outfiles[i]
if config["verbose"] > 1:
print("Spawned process {}".format(info["proc"].pid))
return info
# Default to conservatively assuming 4GB/job. def max_parallel_jobs(job_size=4 * 2**30): """Return the max number of parallel jobs we can run without overfilling
memory, assuming heavyweight jobs."""
from_cores = int(subprocess.check_output(["nproc", "--ignore=1"]).strip())
mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
from_mem = round(mem_bytes / job_size) return min(from_cores, from_mem)
for default in defaults: try:
execfile(default, config) if args.verbose > 1:
print("Loaded %s" % default) except Exception: pass
# execfile() used config as the globals for running the # defaults.py script, and will have set a __builtins__ key as a side effect. del config["__builtins__"]
data = config.copy()
for k, v in vars(args).items(): if v isnotNone:
data[k] = v
if args.list: for step in steps:
job = JOBS[step]
outfiles = job.get("outputs") or job.get("redirect-output") if outfiles:
print( "%s\n ->%s %s"
% (step, "*"if job.get("multi-output") else"", outfiles)
) else:
print(step)
sys.exit(0)
for step in steps:
job = JOBS[step] if"redirect-output"in job:
data[step] = job["redirect-output"] elif"outputs"in job and"command"in job:
outfiles = job["outputs"]
num_outputs = 0 for i, j, name in out_indexes(job["command"]): # Trim the {curly brackets} off of the output keys.
data[name[1:-1]] = outfiles[i]
num_outputs += 1 assert (
len(outfiles) == num_outputs
), 'step "%s": mismatched number of output files (%d) and params (%d)' % (
step,
num_outputs,
len(outfiles),
) # NOQA: E501
if args.step: if args.first or args.last: raise Exception( "--first and --last cannot be used when a step argument is given"
)
steps = [args.step] else: if args.first:
steps = steps[steps.index(args.first) :] if args.last:
steps = steps[: steps.index(args.last) + 1]
for step in steps:
run_job(step, data)
¤ Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.0.4Bemerkung:
¤
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung ist noch experimentell.
