Quelle  gen_sbom.py

# !/usr/bin/env python3
#
# Copyright (C) 2024 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Generate the SBOM of the current target product in SPDX format.
Usage example:
  gen_sbom.py --output_file out/soong/sbom/aosp_cf_x86_64_phone/sbom.spdx \
              --metadata out/soong/metadata/aosp_cf_x86_64_phone/metadata.db \
              --product_out out/target/vsoc_x86_64
              --soong_out out/soong
              --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \
              --product_mfr=Google
"""

import argparse
import compliance_metadata
import datetime
import google.protobuf.text_format as text_format
import hashlib
import os
import pathlib
import queue
import metadata_file_pb2
import sbom_data
import sbom_writers
import sys

# Package type
PKG_SOURCE = 'SOURCE'
PKG_UPSTREAM = 'UPSTREAM'
PKG_PREBUILT = 'PREBUILT'

# Security tag
NVD_CPE23 = 'NVD-CPE2.3:'

# Report
ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:'
ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:'
ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:'
ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:'
ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-existent installed files:'
ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP = 'No module found for static dependency files:'
INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:'

SOONG_PREBUILT_MODULE_TYPES = [
    'android_app_import',
    'android_library_import',
    'apex_set',
    'cc_prebuilt_binary',
    'cc_prebuilt_library',
    'cc_prebuilt_library_headers',
    'cc_prebuilt_library_shared',
    'cc_prebuilt_library_static',
    'cc_prebuilt_object',
    'dex_import',
    'java_import',
    'java_sdk_library_import',
    'java_system_modules_import',
    'libclang_rt_prebuilt_library_static',
    'libclang_rt_prebuilt_library_shared',
    'llvm_prebuilt_library_static',
    'ndk_prebuilt_object',
    'ndk_prebuilt_shared_stl',
    'nkd_prebuilt_static_stl',
    'prebuilt_apex',
    'prebuilt_bootclasspath_fragment',
    'prebuilt_dsp',
    'prebuilt_firmware',
    'prebuilt_kernel_modules',
    'prebuilt_rfsa',
    'prebuilt_root',
    'rust_prebuilt_dylib',
    'rust_prebuilt_library',
    'rust_prebuilt_rlib',
    'vndk_prebuilt_shared',
]

THIRD_PARTY_IDENTIFIER_TYPES = [
    # Types defined in metadata_file.proto
    'Git',
    'SVN',
    'Hg',
    'Darcs',
    'Piper',
    'VCS',
    'Archive',
    'PrebuiltByAlphabet',
    'LocalSource',
    'Other',
    # OSV ecosystems defined at https://ossf.github.io/osv-schema/#affectedpackage-field.
    'Go',
    'npm',
    'OSS-Fuzz',
    'PyPI',
    'RubyGems',
    'crates.io',
    'Hackage',
    'GHC',
    'Packagist',
    'Maven',
    'NuGet',
    'Linux',
    'Debian',
    'Alpine',
    'Hex',
    'Android',
    'GitHub Actions',
    'Pub',
    'ConanCenter',
    'Rocky Linux',
    'AlmaLinux',
    'Bitnami',
    'Photon OS',
    'CRAN',
    'Bioconductor',
    'SwiftURL'
]


def get_args():
  parser = argparse.ArgumentParser()
  parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')
  parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
  parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')
  parser.add_argument('--metadata', required=True, help='The metadata DB file path.')
  parser.add_argument('--product_out', required=True, help='The path of PRODUCT_OUT, e.g. out/target/product/vsoc_x86_64.')
  parser.add_argument('--soong_out', required=True, help='The path of Soong output directory, e.g. out/soong')
  parser.add_argument('--build_version', required=True, help='The build version.')
  parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
  parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
  parser.add_argument('--unbundled_module', help='Module name, e.g. com.google.android.adbd, for which SBOM is generated')

  return parser.parse_args()


def log(*info):
  if args.verbose:
    for i in info:
      print(i)


def new_package_id(package_name, type):
  return f'SPDXRef-{type}-{sbom_data.encode_for_spdxid(package_name)}'


def new_file_id(file_path):
  return f'SPDXRef-{sbom_data.encode_for_spdxid(file_path)}'


def new_license_id(license_name):
  return f'LicenseRef-{sbom_data.encode_for_spdxid(license_name)}'


def checksum(file_path):
  h = hashlib.sha1()
  if os.path.islink(file_path):
    h.update(os.readlink(file_path).encode('utf-8'))
  else:
    with open(file_path, 'rb') as f:
      h.update(f.read())
  return f'SHA1: {h.hexdigest()}'


def is_soong_prebuilt_module(file_metadata):
  module_type = file_metadata.get('soong_module_type')
  if module_type and module_type in SOONG_PREBUILT_MODULE_TYPES:
    return True

  base_type = file_metadata.get('soong_base_module_type')
  if base_type and base_type in SOONG_PREBUILT_MODULE_TYPES:
    return True

  if file_metadata.get('cipd_src'):
    return True

  return False


def is_source_package(file_metadata):
  module_path = file_metadata['module_path']
  is_source_package = False
  if args.unbundled_module and os.path.exists(module_path + '/METADATA'):
    # See b/272356272, change the logic of identifying source package for mainline modules for now.
    is_source_package = True
  return (module_path.startswith('external/') or is_source_package) and not is_prebuilt_package(file_metadata)


def is_prebuilt_package(file_metadata):
  module_path = file_metadata['module_path']
  if module_path:
    return (module_path.startswith('prebuilts/') or
            is_soong_prebuilt_module(file_metadata) or
            file_metadata.get('is_prebuilt_make_module'))

  kernel_module_copy_files = file_metadata['kernel_module_copy_files']
  if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
    return True

  return False


def get_source_package_info(file_metadata, metadata_file_path):
  """Return source package info exists in its METADATA file, currently including name, security tag
  and external SBOM reference.

  See go/android-spdx and go/android-sbom-gen for more details.
  """
  if not metadata_file_path:
    return file_metadata['module_path'], []

  metadata_proto = metadata_file_protos[metadata_file_path]
  external_refs = []
  for tag in metadata_proto.third_party.security.tag:
    if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()):
      external_refs.append(
          sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
                                       type=sbom_data.PackageExternalRefType.cpe23Type,
                                       locator=tag.removeprefix(NVD_CPE23)))
    elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()):
      external_refs.append(
          sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
                                       type=sbom_data.PackageExternalRefType.cpe22Type,
                                       locator=tag.removeprefix(NVD_CPE23)))

  if metadata_proto.name:
    return metadata_proto.name, external_refs
  else:
    return os.path.basename(metadata_file_path), external_refs  # return the directory name only as package name


def get_prebuilt_package_name(file_metadata, metadata_file_path):
  """Return name of a prebuilt package, which can be from the METADATA file, metadata file path,
  module path or kernel module's source path if the installed file is a kernel module.

  See go/android-spdx and go/android-sbom-gen for more details.
  """
  name = None
  if metadata_file_path:
    metadata_proto = metadata_file_protos[metadata_file_path]
    if metadata_proto.name:
      name = metadata_proto.name
    else:
      name = metadata_file_path
  elif file_metadata['module_path']:
    name = file_metadata['module_path']
  elif file_metadata['kernel_module_copy_files']:
    src_path = file_metadata['kernel_module_copy_files'].split(':')[0]
    name = os.path.dirname(src_path)

  return name.removeprefix('prebuilts/').replace('/', '-')


def get_metadata_file_path(file_metadata):
  """Search for METADATA file of a package and return its path."""
  metadata_path = ''
  if file_metadata['module_path']:
    metadata_path = file_metadata['module_path']
  elif file_metadata['kernel_module_copy_files']:
    metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0])

  while metadata_path and not os.path.exists(metadata_path + '/METADATA'):
    metadata_path = os.path.dirname(metadata_path)

  return metadata_path


def get_package_version(metadata_file_path, is_src_package, cipd_src=None, prebuilt_src_file=None):
  """Return a package's version."""
  if is_src_package:
    # Version is from METADATA file for source packages
    if not metadata_file_path:
      return None
    metadata_proto = metadata_file_protos[metadata_file_path]

    if metadata_proto.third_party.version:
      return metadata_proto.third_party.version
    for identifier in metadata_proto.third_party.identifier:
      if identifier.primary_source:
        return identifier.version
    if metadata_proto.third_party.identifier:
      return metadata_proto.third_party.identifier[0].version
  else:  # prebuilt packages
    # Version is from cipd_package or METADATA file for prebuilt packages
    if cipd_src:
      cipd_version = db.get_cipd_package_version(cipd_src)
      if cipd_version:
        return cipd_version

    if not metadata_file_path:
      return None
    metadata_proto = metadata_file_protos[metadata_file_path]
    if metadata_proto.third_party.version:
      return metadata_proto.third_party.version

    prebuilt_identifiers = []
    other_identifiers = []
    for identifier in metadata_proto.third_party.identifier:
      if identifier.type == 'PrebuiltByAlphabet':
        prebuilt_identifiers.append(identifier)
      else:
        other_identifiers.append(identifier)
      if identifier.primary_source:
        return identifier.version

    if prebuilt_identifiers:
      for identifier in prebuilt_identifiers:
        if (metadata_file_path + '/' + identifier.value) == prebuilt_src_file:
          return identifier.version
    elif other_identifiers:
      return other_identifiers[0].version

  return None

def get_package_homepage(metadata_file_path):
  """Return a package's homepage URL in its METADATA file."""
  if not metadata_file_path:
    return None
  metadata_proto = metadata_file_protos[metadata_file_path]
  if metadata_proto.third_party.homepage:
    return metadata_proto.third_party.homepage
  for url in metadata_proto.third_party.url:
    if url.type == metadata_file_pb2.URL.Type.HOMEPAGE:
      return url.value

  return None


def get_package_download_location(metadata_file_path):
  """Return a package's code repository URL in its METADATA file."""
  if not metadata_file_path:
    return None
  metadata_proto = metadata_file_protos[metadata_file_path]
  if metadata_proto.third_party.url:
    urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type)
    if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE:
      return urls[0].value
    elif len(urls) > 1:
      return urls[1].value

  return None


def get_license_text(license_files):
  license_text = ''
  for license_file in license_files:
    if args.debug:
      license_text += '#### Content from ' + license_file + '\n'
    else:
      license_text += pathlib.Path(license_file).read_text(errors='replace') + '\n\n'
  return license_text


def get_sbom_fragments(installed_file_metadata, metadata_file_path):
  """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT
  package, a UPSTREAM package and an external SBOM document reference if sbom_ref defined in its
  METADATA file.

  See go/android-spdx and go/android-sbom-gen for more details.
  """
  external_doc_ref = None
  packages = []
  relationships = []
  licenses = []

  # Info from METADATA file
  homepage = get_package_homepage(metadata_file_path)
  download_location = get_package_download_location(metadata_file_path)

  lics = db.get_package_licenses(installed_file_metadata['module_path'])
  if not lics:
    lics = db.get_package_licenses(metadata_file_path)

  if lics:
    for license_name, license_files in lics.items():
      if not license_files:
        continue
      license_id = new_license_id(license_name)
      if license_name not in licenses_text:
        licenses_text[license_name] = get_license_text(license_files.split(' '))
      licenses.append(sbom_data.License(id=license_id, name=license_name, text=licenses_text[license_name]))

  if is_source_package(installed_file_metadata):
    # Source fork packages
    version = get_package_version(metadata_file_path, True)
    name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path)
    source_package_id = new_package_id(name, PKG_SOURCE)
    source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version,
                                       download_location=sbom_data.VALUE_NONE,
                                       supplier='Organization: ' + args.product_mfr,
                                       external_refs=external_refs)

    upstream_package_id = new_package_id(name, PKG_UPSTREAM)
    upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version,
                                         supplier=(
                                             'Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION,
                                         download_location=download_location)
    packages += [source_package, upstream_package]
    relationships.append(sbom_data.Relationship(id1=source_package_id,
                                                relationship=sbom_data.RelationshipType.VARIANT_OF,
                                                id2=upstream_package_id))

    for license in licenses:
      source_package.declared_license_ids.append(license.id)
      upstream_package.declared_license_ids.append(license.id)

  elif is_prebuilt_package(installed_file_metadata):
    # Prebuilt fork packages
    version = get_package_version(metadata_file_path, False, installed_file_metadata.get('cipd_src', None), installed_file_metadata.get('prebuilt_src_file', None))
    name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path)
    prebuilt_package_id = new_package_id(name, PKG_PREBUILT)
    prebuilt_package = sbom_data.Package(id=prebuilt_package_id,
                                         name=name,
                                         download_location=sbom_data.VALUE_NONE,
                                         version=args.build_version,
                                         supplier='Organization: ' + args.product_mfr)

    upstream_package_id = new_package_id(name, PKG_UPSTREAM)
    upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version,
                                         supplier=(
                                             'Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION,
                                         download_location=download_location)
    packages += [prebuilt_package, upstream_package]
    relationships.append(sbom_data.Relationship(id1=prebuilt_package_id,
                                                relationship=sbom_data.RelationshipType.VARIANT_OF,
                                                id2=upstream_package_id))
    for license in licenses:
      prebuilt_package.declared_license_ids.append(license.id)
      upstream_package.declared_license_ids.append(license.id)

  if metadata_file_path:
    metadata_proto = metadata_file_protos[metadata_file_path]
    if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
      sbom_url = metadata_proto.third_party.sbom_ref.url
      sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
      upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
      if sbom_url and sbom_checksum and upstream_element_id:
        doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{sbom_data.encode_for_spdxid(name)}'
        external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id,
                                                               uri=sbom_url,
                                                               checksum=sbom_checksum)
        relationships.append(
            sbom_data.Relationship(id1=upstream_package_id,
                                   relationship=sbom_data.RelationshipType.VARIANT_OF,
                                   id2=doc_ref_id + ':' + upstream_element_id))

  return external_doc_ref, packages, relationships, licenses


def save_report(report_file_path, report):
  with open(report_file_path, 'w', encoding='utf-8') as report_file:
    for type, issues in report.items():
      report_file.write(type + '\n')
      for issue in issues:
        report_file.write('\t' + issue + '\n')
      report_file.write('\n')


# Validate the metadata generated by Make for installed files and report if there is no metadata.
def installed_file_has_metadata(installed_file_metadata, report):
  installed_file = installed_file_metadata['installed_file']
  module_path = installed_file_metadata['module_path']
  is_soong_module = installed_file_metadata['is_soong_module']
  product_copy_files = installed_file_metadata['product_copy_files']
  kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
  is_platform_generated = installed_file_metadata['is_platform_generated']

  if (not module_path and
      not is_soong_module and
      not product_copy_files and
      not kernel_module_copy_files and
      not is_platform_generated and
      not installed_file.endswith('.fsv_meta')):
    report[ISSUE_NO_METADATA].append(installed_file)
    return False

  return True


# Validate identifiers in a package's METADATA.
# 1) Only known identifier type is allowed
# 2) Only one identifier's primary_source can be true
def validate_package_metadata(metadata_file_path, package_metadata):
  primary_source_found = False
  for identifier in package_metadata.third_party.identifier:
    if identifier.type not in THIRD_PARTY_IDENTIFIER_TYPES:
      sys.exit(f'Unknown value of third_party.identifier.type in {metadata_file_path}/METADATA: {identifier.type}.')
    if primary_source_found and identifier.primary_source:
      sys.exit(
          f'Field "primary_source" is set to true in multiple third_party.identifier in {metadata_file_path}/METADATA.')
    primary_source_found = identifier.primary_source


def report_metadata_file(metadata_file_path, installed_file_metadata, report):
  if metadata_file_path:
    report[INFO_METADATA_FOUND_FOR_PACKAGE].append(
        'installed_file: {}, module_path: {}, METADATA file: {}'.format(
            installed_file_metadata['installed_file'],
            installed_file_metadata['module_path'],
            metadata_file_path + '/METADATA'))

    package_metadata = metadata_file_pb2.Metadata()
    with open(metadata_file_path + '/METADATA', 'rt') as f:
      text_format.Parse(f.read(), package_metadata)

    validate_package_metadata(metadata_file_path, package_metadata)

    if not metadata_file_path in metadata_file_protos:
      metadata_file_protos[metadata_file_path] = package_metadata
      if not package_metadata.name:
        report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"')

      if not package_metadata.third_party.version:
        report[ISSUE_METADATA_FILE_INCOMPLETE].append(
            f'{metadata_file_path}/METADATA does not has "third_party.version"')

      for tag in package_metadata.third_party.security.tag:
        if not tag.startswith(NVD_CPE23):
          report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append(
              f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA')
  else:
    report[ISSUE_NO_METADATA_FILE].append(
        "installed_file: {}, module_path: {}".format(
            installed_file_metadata['installed_file'], installed_file_metadata['module_path']))


# If a file is from a source fork or prebuilt fork package, add its package information to SBOM
def add_package_of_file(file_id, file_metadata, doc, report):
  metadata_file_path = get_metadata_file_path(file_metadata)
  report_metadata_file(metadata_file_path, file_metadata, report)

  external_doc_ref, pkgs, rels, licenses = get_sbom_fragments(file_metadata, metadata_file_path)
  if len(pkgs) > 0:
    if external_doc_ref:
      doc.add_external_ref(external_doc_ref)
    for p in pkgs:
      doc.add_package(p)
    for rel in rels:
      doc.add_relationship(rel)
    fork_package_id = pkgs[0].id  # The first package should be the source/prebuilt fork package
    doc.add_relationship(sbom_data.Relationship(id1=file_id,
                                                relationship=sbom_data.RelationshipType.GENERATED_FROM,
                                                id2=fork_package_id))
    for license in licenses:
      doc.add_license(license)


# Add STATIC_LINK relationship for static dependencies of a file
def add_static_deps_of_file(file_id, file_metadata, doc):
  if not file_metadata['static_dep_files'] and not file_metadata['whole_static_dep_files']:
    return
  static_dep_files = []
  if file_metadata['static_dep_files']:
    static_dep_files += file_metadata['static_dep_files'].split(' ')
  if file_metadata['whole_static_dep_files']:
    static_dep_files += file_metadata['whole_static_dep_files'].split(' ')

  for dep_file in static_dep_files:
    # Static libs are not shipped on devices, so names are derived from .intermediates paths.
    doc.add_relationship(sbom_data.Relationship(id1=file_id,
                                                relationship=sbom_data.RelationshipType.STATIC_LINK,
                                                id2=new_file_id(
                                                    dep_file.removeprefix(args.soong_out + '/.intermediates/'))))


def add_licenses_of_file(file_id, file_metadata, doc):
  lics = db.get_module_licenses(file_metadata.get('name', ''), file_metadata['module_path'])
  if lics:
    file = next(f for f in doc.files if file_id == f.id)
    for license_name, license_files in lics.items():
      if not license_files:
        continue
      license_id = new_license_id(license_name)
      file.concluded_license_ids.append(license_id)
      if license_name not in licenses_text:
        license_text = get_license_text(license_files.split(' '))
        licenses_text[license_name] = license_text

      doc.add_license(sbom_data.License(id=license_id, name=license_name, text=licenses_text[license_name]))


def get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report):
  # Find all transitive static dep files of all installed files
  q = queue.Queue()
  for installed_file_metadata in installed_files_metadata:
    if installed_file_metadata['static_dep_files']:
      for f in installed_file_metadata['static_dep_files'].split(' '):
        q.put(f)
    if installed_file_metadata['whole_static_dep_files']:
      for f in installed_file_metadata['whole_static_dep_files'].split(' '):
        q.put(f)

  all_static_dep_files = {}
  while not q.empty():
    dep_file = q.get()
    if dep_file in all_static_dep_files:
      # It has been processed
      continue

    all_static_dep_files[dep_file] = True
    soong_module = db.get_soong_module_of_built_file(dep_file)
    if not soong_module:
      # This should not happen, add to report[ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP]
      report[ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP].append(f)
      continue

    if soong_module['static_dep_files']:
      for f in soong_module['static_dep_files'].split(' '):
        if f not in all_static_dep_files:
          q.put(f)
    if soong_module['whole_static_dep_files']:
      for f in soong_module['whole_static_dep_files'].split(' '):
        if f not in all_static_dep_files:
          q.put(f)

  return sorted(all_static_dep_files.keys())

def get_license_of_product_copy_file(file_path):
  # Provides license info for known AOSP files used in PRODUCT_COPY_FILES.
  paths = {
      'device/sample/etc/',
      'frameworks/av/media/libeffects/data/',
      'frameworks/av/media/libstagefright/data/',
      'frameworks/av/services/audiopolicy/config/',
      'frameworks/base/config/',
      'frameworks/base/data/keyboards/',
      'frameworks/base/data/sounds/',
      'frameworks/native/data/etc/',
      'hardware/google/camera/devices/EmulatedCamera/hwl/configs/',
      'system/core/rootdir/etc/',
  }
  for p in paths:
    if file_path.startswith(p):
      return 'build/soong/licenses/LICENSE'

  return ''

def main():
  global args
  args = get_args()
  log('Args:', vars(args))

  global db
  db = compliance_metadata.MetadataDb(args.metadata)
  if args.debug:
    db.dump_debug_db(os.path.dirname(args.output_file) + '/compliance-metadata-debug.db')

  global metadata_file_protos
  metadata_file_protos = {}
  global licenses_text
  licenses_text = {}

  product_package_id = sbom_data.SPDXID_PRODUCT
  product_package_name = sbom_data.PACKAGE_NAME_PRODUCT
  product_package = sbom_data.Package(id=product_package_id,
                                      name=product_package_name,
                                      download_location=sbom_data.VALUE_NONE,
                                      version=args.build_version,
                                      supplier='Organization: ' + args.product_mfr,
                                      files_analyzed=True)
  doc_name = args.build_version
  if args.unbundled_module:
    doc_name = f'{args.build_version}/{args.unbundled_module}'
  doc = sbom_data.Document(name=doc_name,
                           namespace=f'https://www.google.com/sbom/spdx/android/{doc_name}',
                           creators=['Organization: ' + args.product_mfr],
                           describes=product_package_id)

  if not args.unbundled_module:
    doc.packages.append(product_package)

  platform_package = sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
                                       name=sbom_data.PACKAGE_NAME_PLATFORM,
                                       download_location=sbom_data.VALUE_NONE,
                                       version=args.build_version,
                                       supplier='Organization: ' + args.product_mfr,
                                       declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE])
  doc.packages.append(platform_package)

  # Report on some issues and information
  report = {
      ISSUE_NO_METADATA: [],
      ISSUE_NO_METADATA_FILE: [],
      ISSUE_METADATA_FILE_INCOMPLETE: [],
      ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [],
      ISSUE_INSTALLED_FILE_NOT_EXIST: [],
      ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP: [],
      INFO_METADATA_FOUND_FOR_PACKAGE: [],
  }

  if args.unbundled_module:
    installed_files_metadata = db.get_installed_files_of_module(args.unbundled_module)
  else:
    # Get installed files and corresponding make modules' metadata if an installed file is from a make module.
    installed_files_metadata = db.get_installed_files()

  # Find which Soong module an installed file is from and merge metadata from Make and Soong
  for installed_file_metadata in installed_files_metadata:
    soong_module = db.get_soong_module_of_installed_file(installed_file_metadata['installed_file'])
    if not soong_module and args.unbundled_module:
      soong_module = db.get_soong_module_of_built_file(installed_file_metadata['build_output_path'])
    if soong_module:
      # Merge soong metadata to make metadata
      installed_file_metadata.update(soong_module)
    else:
      # For make modules soong_module_type should be empty
      installed_file_metadata['soong_module_type'] = ''
      installed_file_metadata['static_dep_files'] = ''
      installed_file_metadata['whole_static_dep_files'] = ''

  # Scan the metadata and create the corresponding package and file records in SPDX
  include_static_deps = True
  for index, installed_file_metadata in enumerate(installed_files_metadata):
    installed_file = installed_file_metadata['installed_file']
    module_path = installed_file_metadata['module_path']
    product_copy_files = installed_file_metadata.get('product_copy_files')
    kernel_module_copy_files = installed_file_metadata.get('kernel_module_copy_files')
    is_platform_generated = installed_file_metadata.get('is_platform_generated')
    build_output_path = installed_file
    if args.unbundled_module:
      build_output_path = installed_file_metadata['build_output_path']
    installed_file = installed_file.removeprefix(args.product_out)

    if not args.unbundled_module and not installed_file_has_metadata(installed_file_metadata, report):
      continue
    if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
      report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
      continue

    file_id = new_file_id(installed_file)
    sha1 = checksum(build_output_path)
    f = sbom_data.File(id=file_id, name=installed_file, checksum=sha1)
    doc.files.append(f)
    product_package.file_ids.append(file_id)
    if args.unbundled_module:
      if index == 0:
        # The generated SBOM describes the APEX file which is the first record
        doc.describes = file_id
        # Do not include static deps in SBOM of APKs
        if installed_file.endswith('.apk'):
          include_static_deps = False
      else:
        # All other files are contained by the APEX file, so add a CONTAINS relationship for them
        doc.add_relationship(sbom_data.Relationship(id1=product_package.file_ids[0],
                                                    relationship=sbom_data.RelationshipType.CONTAINS,
                                                    id2=file_id))

    if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
      add_package_of_file(file_id, installed_file_metadata, doc, report)

    elif module_path or is_platform_generated:
      # File from PLATFORM package
      doc.add_relationship(sbom_data.Relationship(id1=file_id,
                                                  relationship=sbom_data.RelationshipType.GENERATED_FROM,
                                                  id2=sbom_data.SPDXID_PLATFORM))
      if is_platform_generated:
        f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]

    elif product_copy_files:
      # Format of product_copy_files: <source path>:<dest path>
      src_path = product_copy_files.split(':')[0]
      # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device,
      # so process them as files from PLATFORM package
      doc.add_relationship(sbom_data.Relationship(id1=file_id,
                                                  relationship=sbom_data.RelationshipType.GENERATED_FROM,
                                                  id2=sbom_data.SPDXID_PLATFORM))
      if not installed_file_metadata['license_text']:
        installed_file_metadata['license_text'] = get_license_of_product_copy_file(src_path)
      if installed_file_metadata['license_text']:
        if installed_file_metadata['license_text'] == 'build/soong/licenses/LICENSE':
          f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]

    elif installed_file.endswith('.fsv_meta'):
      doc.add_relationship(sbom_data.Relationship(id1=file_id,
                                                  relationship=sbom_data.RelationshipType.GENERATED_FROM,
                                                  id2=sbom_data.SPDXID_PLATFORM))
      f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]

    elif kernel_module_copy_files.startswith('ANDROID-GEN'):
      # For the four files generated for _dlkm, _ramdisk partitions
      doc.add_relationship(sbom_data.Relationship(id1=file_id,
                                                  relationship=sbom_data.RelationshipType.GENERATED_FROM,
                                                  id2=sbom_data.SPDXID_PLATFORM))

    # Process static dependencies of the installed file
    if include_static_deps:
      add_static_deps_of_file(file_id, installed_file_metadata, doc)

    # Add licenses of the installed file
    add_licenses_of_file(file_id, installed_file_metadata, doc)

  # Add all static library files to SBOM
  if include_static_deps:
    for dep_file in get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report):
      filepath = dep_file.removeprefix(args.soong_out + '/.intermediates/')
      file_id = new_file_id(filepath)
      # SHA1 of empty string. Sometimes .a files might not be built.
      sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709'
      if os.path.islink(dep_file) or os.path.isfile(dep_file):
        sha1 = checksum(dep_file)
      doc.files.append(sbom_data.File(id=file_id,
                                      name=filepath,
                                      checksum=sha1))
      file_metadata = {
          'installed_file': dep_file,
          'is_prebuilt_make_module': False
      }
      soong_module = db.get_soong_module_of_built_file(dep_file)
      if not soong_module:
        continue
      file_metadata.update(soong_module)
      if is_source_package(file_metadata) or is_prebuilt_package(file_metadata):
        add_package_of_file(file_id, file_metadata, doc, report)
      else:
        # Other static lib files are generated from the platform
        doc.add_relationship(sbom_data.Relationship(id1=file_id,
                                                    relationship=sbom_data.RelationshipType.GENERATED_FROM,
                                                    id2=sbom_data.SPDXID_PLATFORM))

      # Add relationships for static deps of static libraries
      add_static_deps_of_file(file_id, file_metadata, doc)

      # Add licenses of the static lib
      add_licenses_of_file(file_id, file_metadata, doc)

  # Save SBOM records to output file
  doc.generate_packages_verification_code()
  doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
  prefix = args.output_file
  if prefix.endswith('.spdx'):
    prefix = prefix.removesuffix('.spdx')
  elif prefix.endswith('.spdx.json'):
    prefix = prefix.removesuffix('.spdx.json')

  output_file = prefix + '.spdx'
  with open(output_file, 'w', encoding="utf-8") as file:
    sbom_writers.TagValueWriter.write(doc, file)
  if args.json:
    with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file:
      sbom_writers.JSONWriter.write(doc, file)

  save_report(prefix + '-gen-report.txt', report)


if __name__ == '__main__':
  main()