#!/usr/bin/env python
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Find header files missing in GN.
This script gets all the header files
from ninja_deps, which
is from the
true
dependency generated by the compiler,
and report
if they don
't exist in GN.
"""
from __future__
import print_function
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from multiprocessing
import Process, Queue
SRC_DIR = os.path.abspath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
DEPOT_TOOLS_DIR = os.path.join(SRC_DIR,
'third_party',
'depot_tools')
def GetHeadersFromNinja(out_dir, skip_obj, q):
"""Return all the header files from ninja_deps"""
def NinjaSource():
cmd = [os.path.join(DEPOT_TOOLS_DIR,
'ninja'),
'-C', out_dir,
'-t',
'deps']
# A negative bufsize means to use the system default, which usually
# means fully buffered.
popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
for line
in iter(popen.stdout.readline,
''):
yield line.rstrip()
popen.stdout.close()
return_code = popen.wait()
if return_code:
raise subprocess.CalledProcessError(return_code, cmd)
ans, err = set(),
None
try:
ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
except Exception
as e:
err = str(e)
q.put((ans, err))
def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
"""Parse ninja output and get the header files"""
all_headers = {}
# Ninja always uses "/", even on Windows.
prefix =
'../../'
is_valid =
False
obj_file =
''
for line
in ninja_out:
if line.startswith(
' '):
if not is_valid:
continue
if line.endswith(
'.h')
or line.endswith(
'.hh'):
f = line.strip()
if f.startswith(prefix):
f = f[6:]
# Remove the '../../' prefix
# build/ only contains build-specific files like build_config.h
# and buildflag.h, and system header files, so they should be
# skipped.
if f.startswith(out_dir)
or f.startswith(
'out'):
continue
if not f.startswith(
'build'):
all_headers.setdefault(f, [])
if not skip_obj:
all_headers[f].append(obj_file)
else:
is_valid = line.endswith(
'(VALID)')
obj_file = line.split(
':')[0]
return all_headers
def GetHeadersFromGN(out_dir, q):
"""Return all the header files from GN"""
tmp =
None
ans, err = set(),
None
try:
# Argument |dir| is needed to make sure it's on the same drive on Windows.
# dir='' means dir='.', but doesn't introduce an unneeded prefix.
tmp = tempfile.mkdtemp(dir=
'')
shutil.copy2(os.path.join(out_dir,
'args.gn'),
os.path.join(tmp,
'args.gn'))
# Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
gn_exe =
'gn.bat' if sys.platform ==
'win32' else 'gn'
subprocess.check_call([
os.path.join(DEPOT_TOOLS_DIR, gn_exe),
'gen', tmp,
'--ide=json',
'-q'])
gn_json = json.load(open(os.path.join(tmp,
'project.json')))
ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
except Exception
as e:
err = str(e)
finally:
if tmp:
shutil.rmtree(tmp)
q.put((ans, err))
def ParseGNProjectJSON(gn, out_dir, tmp_out):
"""Parse GN output and get the header files"""
all_headers = set()
for _target, properties
in gn[
'targets'].iteritems():
sources = properties.get(
'sources', [])
public = properties.get(
'public', [])
# Exclude '"public": "*"'.
if type(public)
is list:
sources += public
for f
in sources:
if f.endswith(
'.h')
or f.endswith(
'.hh'):
if f.startswith(
'//'):
f = f[2:]
# Strip the '//' prefix.
if f.startswith(tmp_out):
f = out_dir + f[len(tmp_out):]
all_headers.add(f)
return all_headers
def GetDepsPrefixes(q):
"""Return all the folders controlled by DEPS file"""
prefixes, err = set(),
None
try:
gclient_exe =
'gclient.bat' if sys.platform ==
'win32' else 'gclient'
gclient_out = subprocess.check_output([
os.path.join(DEPOT_TOOLS_DIR, gclient_exe),
'recurse',
'--no-progress',
'-j1',
'python',
'-c',
'import os;print os.environ["GCLIENT_DEP_PATH"]'],
universal_newlines=
True)
for i
in gclient_out.split(
'\n'):
if i.startswith(
'src/'):
i = i[4:]
prefixes.add(i)
except Exception
as e:
err = str(e)
q.put((prefixes, err))
def IsBuildClean(out_dir):
cmd = [os.path.join(DEPOT_TOOLS_DIR,
'ninja'),
'-C', out_dir,
'-n']
try:
out = subprocess.check_output(cmd)
return 'no work to do.' in out
except Exception
as e:
print(e)
return False
def ParseWhiteList(whitelist):
out = set()
for line
in whitelist.split(
'\n'):
line = re.sub(r
'#.*', '', line).strip()
if line:
out.add(line)
return out
def FilterOutDepsedRepo(files, deps):
return {f
for f
in files
if not any(f.startswith(d)
for d
in deps)}
def GetNonExistingFiles(lst):
out = set()
for f
in lst:
if not os.path.isfile(f):
out.add(f)
return out
def main():
def DumpJson(data):
if args.json:
with open(args.json,
'w')
as f:
json.dump(data, f)
def PrintError(msg):
DumpJson([])
parser.error(msg)
parser = argparse.ArgumentParser(description=
'''
NOTE: Use ninja to build all targets
in OUT_DIR before running
this script.
''')
parser.add_argument(
'--out-dir', metavar=
'OUT_DIR', default=
'out/Release',
help=
'output directory of the build')
parser.add_argument(
'--json',
help=
'JSON output filename for missing headers')
parser.add_argument(
'--whitelist', help=
'file containing whitelist')
parser.add_argument(
'--skip-dirty-check', action=
'store_true',
help=
'skip checking whether the build is dirty')
parser.add_argument(
'--verbose', action=
'store_true',
help=
'print more diagnostic info')
args, _extras = parser.parse_known_args()
if not os.path.isdir(args.out_dir):
parser.error(
'OUT_DIR "%s" does not exist.' % args.out_dir)
if not args.skip_dirty_check
and not IsBuildClean(args.out_dir):
dirty_msg =
'OUT_DIR looks dirty. You need to build all there.'
if args.json:
# Assume running on the bots. Silently skip this step.
# This is possible because "analyze" step can be wrong due to
# underspecified header files. See crbug.com/725877
print(dirty_msg)
DumpJson([])
return 0
else:
# Assume running interactively.
parser.error(dirty_msg)
d_q = Queue()
d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir,
True, d_q,))
d_p.start()
gn_q = Queue()
gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
gn_p.start()
deps_q = Queue()
deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
deps_p.start()
d, d_err = d_q.get()
gn, gn_err = gn_q.get()
missing = set(d.keys()) - gn
nonexisting = GetNonExistingFiles(gn)
deps, deps_err = deps_q.get()
missing = FilterOutDepsedRepo(missing, deps)
nonexisting = FilterOutDepsedRepo(nonexisting, deps)
d_p.join()
gn_p.join()
deps_p.join()
if d_err:
PrintError(d_err)
if gn_err:
PrintError(gn_err)
if deps_err:
PrintError(deps_err)
if len(GetNonExistingFiles(d)) > 0:
print(
'Non-existing files in ninja deps:', GetNonExistingFiles(d))
PrintError(
'Found non-existing files in ninja deps. You should ' +
'build all in OUT_DIR.')
if len(d) == 0:
PrintError(
'OUT_DIR looks empty. You should build all there.')
if any(((
'/gen/' in i)
for i
in nonexisting)):
PrintError(
'OUT_DIR looks wrong. You should build all there.')
if args.whitelist:
whitelist = ParseWhiteList(open(args.whitelist).read())
missing -= whitelist
nonexisting -= whitelist
missing = sorted(missing)
nonexisting = sorted(nonexisting)
DumpJson(sorted(missing + nonexisting))
if len(missing) == 0
and len(nonexisting) == 0:
return 0
if len(missing) > 0:
print(
'\nThe following files should be included in gn files:')
for i
in missing:
print(i)
if len(nonexisting) > 0:
print(
'\nThe following non-existing files should be removed from gn files:')
for i
in nonexisting:
print(i)
if args.verbose:
# Only get detailed obj dependency here since it is slower.
GetHeadersFromNinja(args.out_dir,
False, d_q)
d, d_err = d_q.get()
print(
'\nDetailed dependency info:')
for f
in missing:
print(f)
for cc
in d[f]:
print(
' ', cc)
print(
'\nMissing headers sorted by number of affected object files:')
count = {k: len(v)
for (k, v)
in d.iteritems()}
for f
in sorted(count, key=count.get, reverse=
True):
if f
in missing:
print(count[f], f)
if args.json:
# Assume running on the bots. Temporarily return 0 before
# https://crbug.com/937847 is fixed.
return 0
return 1
if __name__ ==
'__main__':
sys.exit(main())
