Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

# -*- coding: utf-8 -*-

# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

# See https://llvm.org/LICENSE.txt for license information.

# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

""" This module is responsible to capture the compiler invocation of any

build process. The result of that should be a compilation database.

This implementation is using the LD_PRELOAD or DYLD_INSERT_LIBRARIES

mechanisms provided by the dynamic linker. The related library is implemented

in C language and can be found under 'libear' directory.

The 'libear' library is capturing all child process creation and logging the

relevant information about it into separate files in a specified directory.

The parameter of this process is the output directory name, where the report

files shall be placed. This parameter is passed as an environment variable.

The module also implements compiler wrappers to intercept the compiler calls.

The module implements the build command execution and the post-processing of

the output files, which will condensates into a compilation database. """

import sys

import os

import os.path

import re

import itertools

import json

import glob

import logging

from libear import build_libear, TemporaryDirectory

from libscanbuild import command_entry_point, compiler_wrapper, \

    wrapper_environment, run_command, run_build

from libscanbuild import duplicate_check

from libscanbuild.compilation import split_command

from libscanbuild.arguments import parse_args_for_intercept_build

from libscanbuild.shell import encode, decode

__all__ = ['capture', 'intercept_build', 'intercept_compiler_wrapper']

GS = chr(0x1d)

RS = chr(0x1e)

US = chr(0x1f)

COMPILER_WRAPPER_CC = 'intercept-cc'

COMPILER_WRAPPER_CXX = 'intercept-c++'

TRACE_FILE_EXTENSION = '.cmd'  # same as in ear.c

WRAPPER_ONLY_PLATFORMS = frozenset({'win32', 'cygwin'})

@command_entry_point

def intercept_build():

    """ Entry point for 'intercept-build' command. """

    args = parse_args_for_intercept_build()

    return capture(args)

def capture(args):

    """ The entry point of build command interception. """

    def post_processing(commands):

        """ To make a compilation database, it needs to filter out commands

        which are not compiler calls. Needs to find the source file name

        from the arguments. And do shell escaping on the command.

        To support incremental builds, it is desired to read elements from

        an existing compilation database from a previous run. These elements

        shall be merged with the new elements. """

        # create entries from the current run

        current = itertools.chain.from_iterable(

            # creates a sequence of entry generators from an exec,

            format_entry(command) for command in commands)

        # read entries from previous run

        if 'append' in args and args.append and os.path.isfile(args.cdb):

            with open(args.cdb) as handle:

                previous = iter(json.load(handle))

        else:

            previous = iter([])

        # filter out duplicate entries from both

        duplicate = duplicate_check(entry_hash)

        return (entry

                for entry in itertools.chain(previous, current)

                if os.path.exists(entry['file']) and not duplicate(entry))

    with TemporaryDirectory(prefix='intercept-') as tmp_dir:

        # run the build command

        environment = setup_environment(args, tmp_dir)

        exit_code = run_build(args.build, env=environment)

        # read the intercepted exec calls

        exec_traces = itertools.chain.from_iterable(

            parse_exec_trace(os.path.join(tmp_dir, filename))

            for filename in sorted(glob.iglob(os.path.join(tmp_dir, '*.cmd'))))

        # do post processing

        entries = post_processing(exec_traces)

        # dump the compilation database

        with open(args.cdb, 'w+') as handle:

            json.dump(list(entries), handle, sort_keys=True, indent=4)

        return exit_code

def setup_environment(args, destination):

    """ Sets up the environment for the build command.

    It sets the required environment variables and execute the given command.

    The exec calls will be logged by the 'libear' preloaded library or by the

    'wrapper' programs. """

    c_compiler = args.cc if 'cc' in args else 'cc'

    cxx_compiler = args.cxx if 'cxx' in args else 'c++'

    libear_path = None if args.override_compiler or is_preload_disabled(

        sys.platform) else build_libear(c_compiler, destination)

    environment = dict(os.environ)

    environment.update({'INTERCEPT_BUILD_TARGET_DIR': destination})

    if not libear_path:

        logging.debug('intercept gonna use compiler wrappers')

        environment.update(wrapper_environment(args))

        environment.update({

            'CC': COMPILER_WRAPPER_CC,

            'CXX': COMPILER_WRAPPER_CXX

        })

    elif sys.platform == 'darwin':

        logging.debug('intercept gonna preload libear on OSX')

        environment.update({

            'DYLD_INSERT_LIBRARIES': libear_path,

            'DYLD_FORCE_FLAT_NAMESPACE': '1'

        })

    else:

        logging.debug('intercept gonna preload libear on UNIX')

        environment.update({'LD_PRELOAD': libear_path})

    return environment

@command_entry_point

def intercept_compiler_wrapper():

    """ Entry point for `intercept-cc` and `intercept-c++`. """

    return compiler_wrapper(intercept_compiler_wrapper_impl)

def intercept_compiler_wrapper_impl(_, execution):

    """ Implement intercept compiler wrapper functionality.

    It does generate execution report into target directory.

    The target directory name is from environment variables. """

    message_prefix = 'execution report might be incomplete: %s'

    target_dir = os.getenv('INTERCEPT_BUILD_TARGET_DIR')

    if not target_dir:

        logging.warning(message_prefix, 'missing target directory')

        return

    # write current execution info to the pid file

    try:

        target_file_name = str(os.getpid()) + TRACE_FILE_EXTENSION

        target_file = os.path.join(target_dir, target_file_name)

        logging.debug('writing execution report to: %s', target_file)

        write_exec_trace(target_file, execution)

    except IOError:

        logging.warning(message_prefix, 'io problem')

def write_exec_trace(filename, entry):

    """ Write execution report file.

    This method shall be sync with the execution report writer in interception

    library. The entry in the file is a JSON objects.

    :param filename:    path to the output execution trace file,

    :param entry:       the Execution object to append to that file. """

    with open(filename, 'ab') as handler:

        pid = str(entry.pid)

        command = US.join(entry.cmd) + US

        content = RS.join([pid, pid, 'wrapper', entry.cwd, command]) + GS

        handler.write(content.encode('utf-8'))

def parse_exec_trace(filename):

    """ Parse the file generated by the 'libear' preloaded library.

    Given filename points to a file which contains the basic report

    generated by the interception library or wrapper command. A single

    report file _might_ contain multiple process creation info. """

    logging.debug('parse exec trace file: %s', filename)

    with open(filename, 'r') as handler:

        content = handler.read()

        for group in filter(bool, content.split(GS)):

            records = group.split(RS)

            yield {

                'pid': records[0],

                'ppid': records[1],

                'function': records[2],

                'directory': records[3],

                'command': records[4].split(US)[:-1]

            }

def format_entry(exec_trace):

    """ Generate the desired fields for compilation database entries. """

    def abspath(cwd, name):

        """ Create normalized absolute path from input filename. """

        fullname = name if os.path.isabs(name) else os.path.join(cwd, name)

        return os.path.normpath(fullname)

    logging.debug('format this command: %s', exec_trace['command'])

    compilation = split_command(exec_trace['command'])

    if compilation:

        for source in compilation.files:

            compiler = 'c++' if compilation.compiler == 'c++' else 'cc'

            command = [compiler, '-c'] + compilation.flags + [source]

            logging.debug('formated as: %s', command)

            yield {

                'directory': exec_trace['directory'],

                'command': encode(command),

                'file': abspath(exec_trace['directory'], source)

            }

def is_preload_disabled(platform):

    """ Library-based interposition will fail silently if SIP is enabled,

    so this should be detected. You can detect whether SIP is enabled on

    Darwin by checking whether (1) there is a binary called 'csrutil' in

    the path and, if so, (2) whether the output of executing 'csrutil status'

    contains 'System Integrity Protection status: enabled'.

    :param platform: name of the platform (returned by sys.platform),

    :return: True if library preload will fail by the dynamic linker. """

    if platform in WRAPPER_ONLY_PLATFORMS:

        return True

    elif platform == 'darwin':

        command = ['csrutil', 'status']

        pattern = re.compile(r'System Integrity Protection status:\s+enabled')

        try:

            return any(pattern.match(line) for line in run_command(command))

        except:

            return False

    else:

        return False

def entry_hash(entry):

    """ Implement unique hash method for compilation database entries. """

    # For faster lookup in set filename is reverted

    filename = entry['file'][::-1]

    # For faster lookup in set directory is reverted

    directory = entry['directory'][::-1]

    # On OS X the 'cc' and 'c++' compilers are wrappers for

    # 'clang' therefore both call would be logged. To avoid

    # this the hash does not contain the first word of the

    # command.

    command = ' '.join(decode(entry['command'])[1:])

    return '<>'.join([filename, directory, command])