# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''The 'grit build
' tool.
'''
from __future__
import print_function
import codecs
import filecmp
import getopt
import gzip
import os
import shutil
import sys
import six
from grit
import grd_reader
from grit
import shortcuts
from grit
import util
from grit.format
import minifier
from grit.node
import brotli_util
from grit.node
import include
from grit.node
import message
from grit.node
import structure
from grit.tool
import interface
# It would be cleaner to have each module register itself, but that would
# require importing all of them on every run of GRIT.
'''Map from ''
_format_modules = {
'android':
'android_xml',
'c_format':
'c_format',
'chrome_messages_json':
'chrome_messages_json',
'chrome_messages_json_gzip':
'chrome_messages_json',
'data_package':
'data_pack',
'policy_templates':
'policy_templates_json',
'rc_all':
'rc',
'rc_header':
'rc_header',
'rc_nontranslateable':
'rc',
'rc_translateable':
'rc',
'resource_file_map_source':
'resource_map',
'resource_map_header':
'resource_map',
'resource_map_source':
'resource_map',
}
def GetFormatter(type):
modulename =
'grit.format.' + _format_modules[type]
__import__(modulename)
module = sys.modules[modulename]
try:
return module.Format
except AttributeError:
return module.GetFormatter(type)
class RcBuilder(interface.Tool):
'''A tool that builds RC files and resource header files for compilation.
Usage: grit build [-o OUTPUTDIR] [-D NAME[=VAL]]*
All output options
for this tool are specified
in the input file (see
'grit help' for details on how to specify the input file - it
is a
global
option).
Options:
-a FILE
Assert that the given file
is an output. There can be
multiple
"-a" flags listed
for multiple outputs.
If a
"-a"
or "--assert-file-list" argument
is present, then the list
of asserted files must match the output files
or the tool
will fail. The use-case
is for the build system to maintain
separate lists of output files
and to catch errors
if the
build system
's list and the grit list are out-of-sync.
--assert-file-list Provide a file listing multiple asserted output files.
There
is one file name per line. This acts like specifying
each file
with "-a" on the command line, but without the
possibility of running into OS line-length limits
for very
long lists.
-o OUTPUTDIR Specify what directory output paths are relative to.
Defaults to the current directory.
-p FILE Specify a file containing a pre-determined mapping
from
resource names to resource ids which will be used to assign
resource ids to those resources. Resources
not found
in this
file will be assigned ids normally. The motivation
is to run
your app
's startup and have it dump the resources it loads,
and then
pass these via this flag. This will pack startup
resources together, thus reducing paging
while all other
resources are unperturbed. The file should have the format:
RESOURCE_ONE_NAME 123
RESOURCE_TWO_NAME 124
-D NAME[=VAL] Specify a C-preprocessor-like define NAME
with optional
value VAL (defaults to 1) which will be used to control
conditional inclusion of resources.
-E NAME=VALUE Set environment variable NAME to VALUE (within grit).
-f FIRSTIDSFILE Path to a python file that specifies the first id of
value to use
for resources. A non-empty value here will
override the value specified
in the <grit> node
's
first_ids_file.
-w WHITELISTFILE Path to a file containing the string names of the
resources to include. Anything
not listed
is dropped.
-t PLATFORM Specifies the platform the build
is targeting; defaults
to the value of sys.platform. The value provided via this
flag should match what sys.platform would report
for your
target platform; see grit.node.base.EvaluateCondition.
--whitelist-support
Generate code to support extracting a resource whitelist
from executables.
--write-only-new flag
If flag
is non-0, write output files to a temporary file
first,
and copy it to the real output only
if the new file
is different
from the old file. This allows some build
systems to realize that dependent build steps might be
unnecessary, at the cost of comparing the output data at
grit time.
--depend-on-stamp
If specified along
with --depfile
and --depdir, the depfile
generated will depend on a stampfile instead of the first
output
in the input .grd file.
--js-minifier A command to run the Javascript minifier.
If not set then
Javascript won
't be minified. The command should read the
original Javascript
from standard input,
and output the
minified Javascript to standard output. A non-zero exit
status will be taken
as indicating failure.
--css-minifier A command to run the CSS minifier.
If not set then CSS won
't
be minified. The command should read the original CSS
from
standard input,
and output the minified CSS to standard
output. A non-zero exit status will be taken
as indicating
failure.
--brotli The full path to the brotli executable generated by
third_party/brotli/BUILD.gn, required
if any entries use
compress=
"brotli".
Conditional inclusion of resources only affects the output of files which
control which resources get linked into a binary, e.g. it affects .rc files
meant
for compilation but it does
not affect resource header files (that define
IDs). This helps ensure that values of IDs stay the same, that all messages
are exported to translation interchange files (e.g. XMB files), etc.
'''
def ShortDescription(self):
return 'A tool that builds RC files for compilation.'
def Run(self, opts, args):
brotli_util.SetBrotliCommand(
None)
os.environ[
'cwd'] = os.getcwd()
self.output_directory =
'.'
first_ids_file =
None
predetermined_ids_file =
None
whitelist_filenames = []
assert_output_files = []
target_platform =
None
depfile =
None
depdir =
None
whitelist_support =
False
write_only_new =
False
depend_on_stamp =
False
js_minifier =
None
css_minifier =
None
replace_ellipsis =
True
(own_opts, args) = getopt.getopt(
args,
'a:p:o:D:E:f:w:t:',
(
'depdir=',
'depfile=',
'assert-file-list=',
'help',
'output-all-resource-defines',
'no-output-all-resource-defines',
'no-replace-ellipsis',
'depend-on-stamp',
'js-minifier=',
'css-minifier=',
'write-only-new=',
'whitelist-support',
'brotli='))
for (key, val)
in own_opts:
if key ==
'-a':
assert_output_files.append(val)
elif key ==
'--assert-file-list':
with open(val)
as f:
assert_output_files += f.read().splitlines()
elif key ==
'-o':
self.output_directory = val
elif key ==
'-D':
name, val = util.ParseDefine(val)
self.defines[name] = val
elif key ==
'-E':
(env_name, env_value) = val.split(
'=', 1)
os.environ[env_name] = env_value
elif key ==
'-f':
# TODO(joi@chromium.org): Remove this override once change
# lands in WebKit.grd to specify the first_ids_file in the
# .grd itself.
first_ids_file = val
elif key ==
'-w':
whitelist_filenames.append(val)
elif key ==
'--no-replace-ellipsis':
replace_ellipsis =
False
elif key ==
'-p':
predetermined_ids_file = val
elif key ==
'-t':
target_platform = val
elif key ==
'--depdir':
depdir = val
elif key ==
'--depfile':
depfile = val
elif key ==
'--write-only-new':
write_only_new = val !=
'0'
elif key ==
'--depend-on-stamp':
depend_on_stamp =
True
elif key ==
'--js-minifier':
js_minifier = val
elif key ==
'--css-minifier':
css_minifier = val
elif key ==
'--whitelist-support':
whitelist_support =
True
elif key ==
'--brotli':
brotli_util.SetBrotliCommand([os.path.abspath(val)])
elif key ==
'--help':
self.ShowUsage()
sys.exit(0)
if len(args):
print(
'This tool takes no tool-specific arguments.')
return 2
self.SetOptions(opts)
self.VerboseOut(
'Output directory: %s (absolute path: %s)\n' %
(self.output_directory,
os.path.abspath(self.output_directory)))
if whitelist_filenames:
self.whitelist_names = set()
for whitelist_filename
in whitelist_filenames:
self.VerboseOut(
'Using whitelist: %s\n' % whitelist_filename);
whitelist_contents = util.ReadFile(whitelist_filename,
'utf-8')
self.whitelist_names.update(whitelist_contents.strip().split(
'\n'))
if js_minifier:
minifier.SetJsMinifier(js_minifier)
if css_minifier:
minifier.SetCssMinifier(css_minifier)
self.write_only_new = write_only_new
self.res = grd_reader.Parse(opts.input,
debug=opts.extra_verbose,
first_ids_file=first_ids_file,
predetermined_ids_file=predetermined_ids_file,
defines=self.defines,
target_platform=target_platform)
# Set an output context so that conditionals can use defines during the
# gathering stage; we use a dummy language here since we are not outputting
# a specific language.
self.res.SetOutputLanguage(
'en')
self.res.SetWhitelistSupportEnabled(whitelist_support)
self.res.RunGatherers()
# Replace ... with the single-character version. http://crbug.com/621772
if replace_ellipsis:
for node
in self.res:
if isinstance(node, message.MessageNode):
node.SetReplaceEllipsis(
True)
self.Process()
if assert_output_files:
if not self.CheckAssertedOutputFiles(assert_output_files):
return 2
if depfile
and depdir:
self.GenerateDepfile(depfile, depdir, first_ids_file, depend_on_stamp)
return 0
def __init__(self, defines=
None):
# Default file-creation function is codecs.open(). Only done to allow
# overriding by unit test.
self.fo_create = codecs.open
# key/value pairs of C-preprocessor like defines that are used for
# conditional output of resources
self.defines = defines
or {}
# self.res is a fully-populated resource tree if Run()
# has been called, otherwise None.
self.res =
None
# The set of names that are whitelisted to actually be included in the
# output.
self.whitelist_names =
None
# Whether to compare outputs to their old contents before writing.
self.write_only_new =
False
@staticmethod
def AddWhitelistTags(start_node, whitelist_names):
# Walk the tree of nodes added attributes for the nodes that shouldn't
# be written into the target files (skip markers).
for node
in start_node:
# Same trick data_pack.py uses to see what nodes actually result in
# real items.
if (isinstance(node, include.IncludeNode)
or
isinstance(node, message.MessageNode)
or
isinstance(node, structure.StructureNode)):
text_ids = node.GetTextualIds()
# Mark the item to be skipped if it wasn't in the whitelist.
if text_ids
and text_ids[0]
not in whitelist_names:
node.SetWhitelistMarkedAsSkip(
True)
@staticmethod
def ProcessNode(node, output_node, outfile):
'''Processes a node in-order, calling its formatter before and after
recursing to its children.
Args:
node: grit.node.base.Node subclass
output_node: grit.node.io.OutputNode
outfile: open filehandle
'''
base_dir = util.dirname(output_node.GetOutputFilename())
formatter = GetFormatter(output_node.GetType())
formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir)
# NB: Formatters may be generators or return lists. The writelines API
# accepts iterables as a shortcut to calling write directly. That means
# you can pass strings (iteration yields characters), but not bytes (as
# iteration yields integers). Python 2 worked due to its quirks with
# bytes/string implementation, but Python 3 fails. It's also a bit more
# inefficient to call write once per character/byte. Handle all of this
# ourselves by calling write directly on strings/bytes before falling back
# to writelines.
if isinstance(formatted, (six.string_types, six.binary_type)):
outfile.write(formatted)
else:
outfile.writelines(formatted)
if output_node.GetType() ==
'data_package':
with open(output_node.GetOutputFilename() +
'.info',
'w')
as infofile:
if node.info:
# We terminate with a newline so that when these files are
# concatenated later we consistently terminate with a newline so
# consumers can account for terminating newlines.
infofile.writelines([
'\n'.join(node.info),
'\n'])
@staticmethod
def _EncodingForOutputType(output_type):
# Microsoft's RC compiler can only deal with single-byte or double-byte
# files (no UTF-8), so we make all RC files UTF-16 to support all
# character sets.
if output_type
in (
'rc_header',
'resource_file_map_source',
'resource_map_header',
'resource_map_source'):
return 'cp1252'
if output_type
in (
'android',
'c_format',
'plist',
'plist_strings',
'doc',
'json',
'android_policy',
'chrome_messages_json',
'chrome_messages_json_gzip',
'policy_templates'):
return 'utf_8'
# TODO(gfeher) modify here to set utf-8 encoding for admx/adml
return 'utf_16'
def Process(self):
for output
in self.res.GetOutputFiles():
output.output_filename = os.path.abspath(os.path.join(
self.output_directory, output.GetOutputFilename()))
# If there are whitelisted names, tag the tree once up front, this way
# while looping through the actual output, it is just an attribute check.
if self.whitelist_names:
self.AddWhitelistTags(self.res, self.whitelist_names)
for output
in self.res.GetOutputFiles():
self.VerboseOut(
'Creating %s...' % output.GetOutputFilename())
# Set the context, for conditional inclusion of resources
self.res.SetOutputLanguage(output.GetLanguage())
self.res.SetOutputContext(output.GetContext())
self.res.SetFallbackToDefaultLayout(output.GetFallbackToDefaultLayout())
self.res.SetDefines(self.defines)
# Assign IDs only once to ensure that all outputs use the same IDs.
if self.res.GetIdMap()
is None:
self.res.InitializeIds()
# Make the output directory if it doesn't exist.
self.MakeDirectoriesTo(output.GetOutputFilename())
# Write the results to a temporary file and only overwrite the original
# if the file changed. This avoids unnecessary rebuilds.
out_filename = output.GetOutputFilename()
tmp_filename = out_filename +
'.tmp'
tmpfile = self.fo_create(tmp_filename,
'wb')
output_type = output.GetType()
if output_type !=
'data_package':
encoding = self._EncodingForOutputType(output_type)
tmpfile = util.WrapOutputStream(tmpfile, encoding)
# Iterate in-order through entire resource tree, calling formatters on
# the entry into a node and on exit out of it.
with tmpfile:
self.ProcessNode(self.res, output, tmpfile)
if output_type ==
'chrome_messages_json_gzip':
gz_filename = tmp_filename +
'.gz'
with open(tmp_filename,
'rb')
as tmpfile, open(gz_filename,
'wb')
as f:
with gzip.GzipFile(filename=
'', mode=
'wb', fileobj=f, mtime=0)
as fgz:
shutil.copyfileobj(tmpfile, fgz)
os.remove(tmp_filename)
tmp_filename = gz_filename
# Now copy from the temp file back to the real output, but on Windows,
# only if the real output doesn't exist or the contents of the file
# changed. This prevents identical headers from being written and .cc
# files from recompiling (which is painful on Windows).
if not os.path.exists(out_filename):
os.rename(tmp_filename, out_filename)
else:
# CHROMIUM SPECIFIC CHANGE.
# This clashes with gyp + vstudio, which expect the output timestamp
# to change on a rebuild, even if nothing has changed, so only do
# it when opted in.
if not self.write_only_new:
write_file =
True
else:
files_match = filecmp.cmp(out_filename, tmp_filename)
write_file =
not files_match
if write_file:
shutil.copy2(tmp_filename, out_filename)
os.remove(tmp_filename)
self.VerboseOut(
' done.\n')
# Print warnings if there are any duplicate shortcuts.
warnings = shortcuts.GenerateDuplicateShortcutsWarnings(
self.res.UberClique(), self.res.GetTcProject())
if warnings:
print(
'\n'.join(warnings))
# Print out any fallback warnings, and missing translation errors, and
# exit with an error code if there are missing translations in a non-pseudo
# and non-official build.
warnings = (self.res.UberClique().MissingTranslationsReport().
encode(
'ascii',
'replace'))
if warnings:
self.VerboseOut(warnings)
if self.res.UberClique().HasMissingTranslations():
print(self.res.UberClique().missing_translations_)
sys.exit(-1)
def CheckAssertedOutputFiles(self, assert_output_files):
'''Checks that the asserted output files are specified in the given list.
Returns
true if the asserted files are present.
If they are
not, returns
False and prints the failure.
'''
# Compare the absolute path names, sorted.
asserted = sorted([os.path.abspath(i)
for i
in assert_output_files])
actual = sorted([
os.path.abspath(os.path.join(self.output_directory,
i.GetOutputFilename()))
for i
in self.res.GetOutputFiles()])
if asserted != actual:
missing = list(set(asserted) - set(actual))
extra = list(set(actual) - set(asserted))
error =
'''Asserted file list does not match.
Expected output files:
%s
Actual output files:
%s
Missing output files:
%s
Extra output files:
%s
'''
print(error % (
'\n'.join(asserted),
'\n'.join(actual),
'\n'.join(missing),
' \n'.join(extra)))
return False
return True
def GenerateDepfile(self, depfile, depdir, first_ids_file, depend_on_stamp):
'''Generate a depfile that contains the imlicit dependencies of the input
grd. The depfile will be
in the same format
as a makefile,
and will contain
references to files relative to |depdir|. It will be put
in |depfile|.
For example, supposing we have three files
in a directory src/
src/
blah.grd <- depends on input{1,2}.xtb
input1.xtb
input2.xtb
and we run
grit -i blah.grd -o ../out/gen \
--depdir ../out \
--depfile ../out/gen/blah.rd.d
from the directory src/ we will generate a depfile ../out/gen/blah.grd.d
that has the contents
gen/blah.h: ../src/input1.xtb ../src/input2.xtb
Where
"gen/blah.h" is the first output (Ninja expects the .d file to list
the first output
in cases where there
is more than one).
If the flag
--depend-on-stamp
is specified,
"gen/blah.rd.d.stamp" will be used that
is
'touched' whenever a new depfile
is generated.
Note that all paths
in the depfile are relative to ../out, the depdir.
'''
depfile = os.path.abspath(depfile)
depdir = os.path.abspath(depdir)
infiles = self.res.GetInputFiles()
# We want to trigger a rebuild if the first ids change.
if first_ids_file
is not None:
infiles.append(first_ids_file)
if (depend_on_stamp):
output_file = depfile +
".stamp"
# Touch the stamp file before generating the depfile.
with open(output_file,
'a'):
os.utime(output_file,
None)
else:
# Get the first output file relative to the depdir.
outputs = self.res.GetOutputFiles()
output_file = os.path.join(self.output_directory,
outputs[0].GetOutputFilename())
output_file = os.path.relpath(output_file, depdir)
# The path prefix to prepend to dependencies in the depfile.
prefix = os.path.relpath(os.getcwd(), depdir)
deps_text =
' '.join([os.path.join(prefix, i)
for i
in infiles])
depfile_contents = output_file +
': ' + deps_text
self.MakeDirectoriesTo(depfile)
outfile = self.fo_create(depfile,
'w', encoding=
'utf-8')
outfile.write(depfile_contents)
@staticmethod
def MakeDirectoriesTo(file):
'''Creates directories necessary to contain |file|.'''
dir = os.path.split(file)[0]
if not os.path.exists(dir):
os.makedirs(dir)
