# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import sys
from collections
import defaultdict
import mozpack.path
as mozpath
from mach.decorators
import Command, CommandArgument, SubCommand
TOPSRCDIR = os.path.abspath(os.path.join(__file__,
"../../../../../"))
class InvalidPathException(Exception):
"""Represents an error due to an invalid path."""
@Command(
"mozbuild-reference",
category=
"build-dev",
description=
"View reference documentation on mozbuild files.",
virtualenv_name=
"docs",
)
@CommandArgument(
"symbol",
default=
None,
nargs=
"*",
help=
"Symbol to view help on. If not specified, all will be shown.",
)
@CommandArgument(
"--name-only",
"-n",
default=
False,
action=
"store_true",
help=
"Print symbol names only.",
)
def reference(command_context, symbol, name_only=
False):
import mozbuild.frontend.context
as m
from mozbuild.sphinx
import (
format_module,
function_reference,
special_reference,
variable_reference,
)
if name_only:
for s
in sorted(m.VARIABLES.keys()):
print(s)
for s
in sorted(m.FUNCTIONS.keys()):
print(s)
for s
in sorted(m.SPECIAL_VARIABLES.keys()):
print(s)
return 0
if len(symbol):
for s
in symbol:
if s
in m.VARIABLES:
for line
in variable_reference(s, *m.VARIABLES[s]):
print(line)
continue
elif s
in m.FUNCTIONS:
for line
in function_reference(s, *m.FUNCTIONS[s]):
print(line)
continue
elif s
in m.SPECIAL_VARIABLES:
for line
in special_reference(s, *m.SPECIAL_VARIABLES[s]):
print(line)
continue
print(
"Could not find symbol: %s" % s)
return 1
return 0
for line
in format_module(m):
print(line)
return 0
@Command(
"file-info", category=
"build-dev", description=
"Query for metadata about files."
)
def file_info(command_context):
"""Show files metadata derived from moz.build files.
moz.build files contain
"Files" sub-contexts
for declaring metadata
against file patterns. This command suite
is used to query that data.
"""
@SubCommand(
"file-info",
"bugzilla-component",
"Show Bugzilla component info for files listed.",
)
@CommandArgument(
"-r",
"--rev", help=
"Version control revision to look up info from")
@CommandArgument(
"--format",
choices={
"json",
"plain"},
default=
"plain",
help=
"Output format",
dest=
"fmt",
)
@CommandArgument(
"paths", nargs=
"+", help=
"Paths whose data to query")
def file_info_bugzilla(command_context, paths, rev=
None, fmt=
None):
"""Show Bugzilla component for a set of files.
Given a requested set of files (which can be specified using
wildcards), print the Bugzilla component
for each file.
"""
components = defaultdict(set)
try:
for p, m
in _get_files_info(command_context, paths, rev=rev).items():
components[m.get(
"BUG_COMPONENT")].add(p)
except InvalidPathException
as e:
print(e)
return 1
if fmt ==
"json":
data = {}
for component, files
in components.items():
if not component:
continue
for f
in files:
data[f] = [component.product, component.component]
json.dump(data, sys.stdout, sort_keys=
True, indent=2)
return
elif fmt ==
"plain":
comp_to_file = sorted(
(
(
"UNKNOWN"
if component
is None
else "%s :: %s" % (component.product, component.component)
),
sorted(files),
)
for component, files
in components.items()
)
for component, files
in comp_to_file:
print(component)
for f
in files:
print(
" %s" % f)
else:
print(
"unhandled output format: %s" % fmt)
return 1
@SubCommand(
"file-info",
"missing-bugzilla",
"Show files missing Bugzilla component info"
)
@CommandArgument(
"-r",
"--rev", help=
"Version control revision to look up info from")
@CommandArgument(
"--format",
choices={
"json",
"plain"},
dest=
"fmt",
default=
"plain",
help=
"Output format",
)
@CommandArgument(
"paths", nargs=
"+", help=
"Paths whose data to query")
def file_info_missing_bugzilla(command_context, paths, rev=
None, fmt=
None):
missing = set()
try:
for p, m
in _get_files_info(command_context, paths, rev=rev).items():
if "BUG_COMPONENT" not in m:
missing.add(p)
except InvalidPathException
as e:
print(e)
return 1
if fmt ==
"json":
json.dump({
"missing": sorted(missing)}, sys.stdout, indent=2)
return
elif fmt ==
"plain":
for f
in sorted(missing):
print(f)
else:
print(
"unhandled output format: %s" % fmt)
return 1
@SubCommand(
"file-info",
"bugzilla-automation",
"Perform Bugzilla metadata analysis as required for automation",
)
@CommandArgument(
"out_dir", help=
"Where to write files")
def bugzilla_automation(command_context, out_dir):
"""Analyze and validate Bugzilla metadata as required by automation.
This will write out JSON
and gzipped JSON files
for Bugzilla metadata.
The exit code will be non-0
if Bugzilla metadata fails validation.
"""
import gzip
missing_component = set()
seen_components = set()
component_by_path = {}
# TODO operate in VCS space. This requires teaching the VCS reader
# to understand wildcards and/or for the relative path issue in the
# VCS finder to be worked out.
for p, m
in sorted(_get_files_info(command_context, [
"**"]).items()):
if "BUG_COMPONENT" not in m:
missing_component.add(p)
print(
"FileToBugzillaMappingError: Missing Bugzilla component: "
"%s - Set the BUG_COMPONENT in the moz.build file to fix "
"the issue." % p
)
continue
c = m[
"BUG_COMPONENT"]
seen_components.add(c)
component_by_path[p] = [c.product, c.component]
print(
"Examined %d files" % len(component_by_path))
# We also have a normalized versions of the file to components mapping
# that requires far less storage space by eliminating redundant strings.
indexed_components = {
i: [c.product, c.component]
for i, c
in enumerate(sorted(seen_components))
}
components_index = {tuple(v): k
for k, v
in indexed_components.items()}
normalized_component = {
"components": indexed_components,
"paths": {}}
for p, c
in component_by_path.items():
d = normalized_component[
"paths"]
while "/" in p:
base, p = p.split(
"/", 1)
d = d.setdefault(base, {})
d[p] = components_index[tuple(c)]
if not os.path.exists(out_dir):
os.makedirs(out_dir)
components_json = os.path.join(out_dir,
"components.json")
print(
"Writing %s" % components_json)
with open(components_json,
"w")
as fh:
json.dump(component_by_path, fh, sort_keys=
True, indent=2)
missing_json = os.path.join(out_dir,
"missing.json")
print(
"Writing %s" % missing_json)
with open(missing_json,
"w")
as fh:
json.dump({
"missing": sorted(missing_component)}, fh, indent=2)
indexed_components_json = os.path.join(out_dir,
"components-normalized.json")
print(
"Writing %s" % indexed_components_json)
with open(indexed_components_json,
"w")
as fh:
# Don't indent so file is as small as possible.
json.dump(normalized_component, fh, sort_keys=
True)
# Write compressed versions of JSON files.
for p
in (components_json, indexed_components_json, missing_json):
gzip_path =
"%s.gz" % p
print(
"Writing %s" % gzip_path)
with open(p,
"rb")
as ifh, gzip.open(gzip_path,
"wb")
as ofh:
while True:
data = ifh.read(32768)
if not data:
break
ofh.write(data)
# Causes CI task to fail if files are missing Bugzilla annotation.
if missing_component:
return 1
def _get_files_info(command_context, paths, rev=
None):
reader = command_context.mozbuild_reader(config_mode=
"empty", vcs_revision=rev)
# Normalize to relative from topsrcdir.
relpaths = []
for p
in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [command_context.topsrcdir]):
raise InvalidPathException(
"path is outside topsrcdir: %s" % p)
relpaths.append(mozpath.relpath(a, command_context.topsrcdir))
# Expand wildcards.
# One variable is for ordering. The other for membership tests.
# (Membership testing on a list can be slow.)
allpaths = []
all_paths_set = set()
for p
in relpaths:
if "*" not in p:
if p
not in all_paths_set:
if not os.path.exists(mozpath.join(command_context.topsrcdir, p)):
print(
"(%s does not exist; ignoring)" % p, file=sys.stderr)
continue
all_paths_set.add(p)
allpaths.append(p)
continue
if rev:
raise InvalidPathException(
"cannot use wildcard in version control mode")
# finder is rooted at / for now.
# TODO bug 1171069 tracks changing to relative.
search = mozpath.join(command_context.topsrcdir, p)[1:]
for path, f
in reader.finder.find(search):
path = path[len(command_context.topsrcdir) :]
if path
not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
return reader.files_info(allpaths)
@SubCommand(
"file-info",
"schedules",
"Show the combined SCHEDULES for the files listed."
)
@CommandArgument(
"paths", nargs=
"+", help=
"Paths whose data to query")
def file_info_schedules(command_context, paths):
"""Show what is scheduled by the given files.
Given a requested set of files (which can be specified using
wildcards), print the total set of scheduled components.
"""
from mozbuild.frontend.reader
import BuildReader, EmptyConfig
config = EmptyConfig(TOPSRCDIR)
reader = BuildReader(config)
schedules = set()
for p, m
in reader.files_info(paths).items():
schedules |= set(m[
"SCHEDULES"].components)
print(
", ".join(schedules))
