Subversion Repositories QNX 8.QNX8 LLVM/Clang compiler suite

#!/bin/sh

# LLVM/Clang toolchain cross-compilation script for QNX 8.0 by Pierre-Marie Baty <pm@pmbaty.com>

# NOTE TO SELF: DO NOT USE $0 AS THIS SCRIPT CAN BE RUN *OR* SOURCED! (see build-llvm.sh in the VM)

export QNXSDK_VERSION="8.0.0" # version of the QNX SDK in use, in <major>.<minor>.<revision> format

export QNXSDK_PATH="../qnx800" # relative location from the path of this script where to find the QNX platform SDK

export QNXSDK_HOSTPATH="host/linux/x86_64" # relative location in QNXSDK_PATH of the tree containing the build tools that are runnable on the build host

export QNXSDK_TARGETPATH="target/qnx" # relative location in QNXSDK_PATH of the tree containing the QNX8 system header files

export BUILD_DIR_NAME="llvm-build" # name of the directory on the build host's desktop where the LLVM sources will be built

export BUILD_TARGET_ARCH="x86_64" # CPU architecture to build LLVM for, either "x86_64" or "aarch64"

export LLVM_VERSION="17.0.6" # version of LLVM that will be built

export LLVM_SOURCES_FILE="llvmorg-${LLVM_VERSION}.tar.gz" # name of the file containing LLVM version LLVM_VERSION sources

export LLVM_SOURCES_URL="https://github.com/llvm/llvm-project/archive/refs/tags/${LLVM_SOURCES_FILE}" # download URL of LLVM_SOURCES_FILE

export LLVM_SOURCES_DIR="llvm-project-llvmorg-${LLVM_VERSION}" # name of directory created when extracting LLVM_SOURCES_FILE

# see where we are

export CURRENT_DIR="$(pwd)"

# verify we're a x86_64 Linux host

if [ ! "$(uname)" = "Linux" ] || [ ! "$(uname -m)" = "x86_64" ]; then

        echo ""

        echo "Error: this script requires a x86_64 Linux machine (possibly a virtual machine) as the build host." | fold -s -w 79

        echo ""

        exit 1

fi

# verify that we have the QNX platform SDK

if [ ! -d "${QNXSDK_PATH}/${QNXSDK_HOSTPATH}" ] || [ ! -d "${QNXSDK_PATH}/${QNXSDK_TARGETPATH}" ]; then

        echo ""

        echo "Error: the ${QNXSDK_PATH} path doesn't contain a QNX SDK. It must contain the 'host' and 'target' directories of the QNX SDP for the targeted version of QNX and the ${BUILD_TARGET_ARCH} platform. Please deploy these directories and try again." | fold -s -w 79

        echo ""

        exit 1

fi

# verify that we have wget, python3, cmake, gcc, g++ and ninja

if    ! wget    --version > /dev/null 2>&1 \

   || ! python3 --version > /dev/null 2>&1 \

   || ! cmake   --version > /dev/null 2>&1 \

   || ! gcc     --version > /dev/null 2>&1 \

   || ! g++     --version > /dev/null 2>&1 \

   || ! ninja   --version > /dev/null 2>&1; then

        echo ""

        echo "Error: this script requires at the very least the following tools installed:"

        echo "  wget"

        echo "  python3"

        echo "  cmake"

        echo "  gcc"

        echo "  g++"

        echo "  ninja"

        echo "Please install them (possibly as binary packages with apt-get) and try again."

        echo ""

        exit 1

fi

# verify that the symlinks are deployed in the SDK -- just test one of them

if [ ! -e "${QNXSDK_PATH}/${QNXSDK_TARGETPATH}/usr/include/readline.h" ]; then

        echo ""

        echo "Error: the toolchain platform-specific symbolic links have not been deployed in this QNX SDK. Please run" | fold -s -w 79

        echo "(on a POSIX machine:)"

        echo "  cd ${QNXSDK_PATH}" | fold -s -w 79

        echo "  find . -name symlinks.lst -exec ./symlinks.sh {} create \\; && printf 'present' > .symlinks-state" | fold -s -w 79

        echo "(else on a Windows machine:)"

        echo "  cd ${QNXSDK_PATH}" | fold -s -w 79

        echo "  host\\win64\\x86_64\\usr\\bin\\busybox.exe sh -c \"" \

               "find . -name symlinks.lst -exec ./symlinks.sh {} create \\; && printf 'present' > .symlinks-state" \

               "\"" | fold -s -w 79

        echo "Note that this step WILL take time on a Win32 machine, but is only done once." | fold -s -w 79

        echo ""

        exit 1

fi

# construct the target triple (actually a quadruple)

export TARGET_ARCH="${BUILD_TARGET_ARCH}"

test "${BUILD_TARGET_ARCH}" = "x86_64" && export TARGET_VENDOR="pc" || export TARGET_VENDOR="unknown"

export TARGET_KERNEL="nto"

export TARGET_SYSTEM="qnx${QNXSDK_VERSION}"

export TARGET_TRIPLE="${TARGET_ARCH}-${TARGET_VENDOR}-${TARGET_KERNEL}-${TARGET_SYSTEM}"

echo "Will build LLVM for ${TARGET_TRIPLE}"

# change to an immediately visible path, i.e. the user's desktop (failsafe to $HOME if xdg-user-dir is unavailable)

STAGING_PATH="$(xdg-user-dir DESKTOP 2>/dev/null || echo "${HOME}")"

cd "${STAGING_PATH}"

# create a symlink in /tmp that will lead to the QNX platform SDK so as to avoid spaces in paths if it doesn't exist already

# (this is totally prohibitive with the official QNX toolchain)

if [ ! -L /tmp/qnxsdk ] || [ ! "$(readlink /tmp/qnxsdk)" = "$(realpath "${CURRENT_DIR}/${QNXSDK_PATH}")" ]; then

        echo "Creating symlink to QNX toolchain in /tmp/qnxsdk..."

        rm -rf /tmp/qnxsdk 2>/dev/null

        ln -fs "$(realpath "${CURRENT_DIR}/${QNXSDK_PATH}")" /tmp/qnxsdk || exit 1

fi

# setup the environment

export QNX_HOST="/tmp/qnxsdk/${QNXSDK_HOSTPATH}"

export QNX_TARGET="/tmp/qnxsdk/${QNXSDK_TARGETPATH}"

export MAKEFLAGS="-I${QNX_TARGET}/usr/include"

export PATH="${QNX_HOST}/usr/bin:${PATH}"

# download the involved source packages and unpack them if not done yet

download_and_unpack_if_necessary()

{

        # helper function that downloads a sources tarball, extracts it and patches it if necessary

        # args: <sources dirname> <sources filename> <download URL> <patchset URL or empty string>

        if [ ! -d "${1}" ]; then

                if [ ! -f "${CURRENT_DIR}/${2}" ]; then

                        echo "Downloading ${1} sources from ${3}..."

                        wget -O "${CURRENT_DIR}/${2}" "${3}" || exit 1

                fi

                echo "Extracting ${1} sources..."

                cd "$(dirname "${1}")"

                if echo "${2}"|grep -q "\.tar\.bz2$"; then

                        tar xjf "${CURRENT_DIR}/${2}" || exit 1

                elif echo "${2}"|grep -q "\.tar\.xz$"; then

                        tar xJf "${CURRENT_DIR}/${2}" || exit 1

                elif echo "${2}"|grep -q "\.tar\.gz$"; then

                        tar xzf "${CURRENT_DIR}/${2}" || exit 1

                else

                        echo "Error: unsupported file extension. Please improve the download_and_unpack_if_necessary() shell function to support it."

                        exit 1

                fi

                if [ ! -d "${1}" ]; then

                        echo "Error: couldn't find ${1} in extracted sources."

                        exit 1

                fi

                if [ -n "${4}" ]; then

                        echo "Downloading ${1} patchset from ${4}..."

                        wget -O "${CURRENT_DIR}/${2}.patchset" "${4}" || exit 1

                        echo "Applying patchset..."

                        OLDDIR="$(pwd)"

                        cd "${1}"

                        patch -N -Z -p1 < "${CURRENT_DIR}/${2}.patchset" || exit 1

                        cd "${OLDDIR}"

                        unset OLDDIR

                fi

        fi

        return 0

}

download_and_unpack_if_necessary "${LLVM_SOURCES_DIR}" "${LLVM_SOURCES_FILE}" "${LLVM_SOURCES_URL}" || exit 1

# download the LLVM source package and unpack it if not done yet

if [ ! -d "${LLVM_SOURCES_DIR}" ]; then

        if [ ! -f "${CURRENT_DIR}/${LLVM_SOURCES_FILE}" ]; then

                echo "Downloading LLVM ${LLVM_VERSION} sources from LLVM GitHub..."

                wget "${LLVM_SOURCES_URL}" || exit 1

        fi

        echo "Extracting LLVM ${LLVM_VERSION} sources..."

        cd "$(dirname "${LLVM_SOURCES_DIR}")"

        tar xzf "${CURRENT_DIR}/${LLVM_SOURCES_FILE}" || exit 1

        if [ ! -d "${LLVM_SOURCES_DIR}" ]; then

                echo "Error: couldn't find ${LLVM_SOURCES_DIR} in extracted LLVM sources."

                exit 1

        fi

fi

# create the build directory

echo "Wiping out build directory..."

test -e "${BUILD_DIR_NAME}" && rm -rf "${BUILD_DIR_NAME}"

mkdir "${BUILD_DIR_NAME}" || exit 1

cd "${BUILD_DIR_NAME}" || exit 1

# print the build environment

echo "QNX_HOST=${QNX_HOST}"

echo "QNX_TARGET=${QNX_TARGET}"

echo "MAKEFLAGS=${MAKEFLAGS}"

# create the QNX CMake toolchain file

test -e "${TARGET_TRIPLE}.cmake" || echo '# '${TARGET_TRIPLE}' CMake toolchain file by Pierre-Marie Baty <pm@pmbaty.com>

SET(CMAKE_SYSTEM_NAME "QNX")

SET(CMAKE_SYSTEM_VERSION "'${QNXSDK_VERSION}'")

SET(QNX "1")

SET(QNXNTO "1")

SET(QNX_HOST "$ENV{QNX_HOST}")

SET(QNX_TARGET "$ENV{QNX_TARGET}")

SET(QNX_PROCESSOR "'${TARGET_ARCH}'")

SET(CMAKE_ASM_COMPILER "${QNX_HOST}/usr/bin/'${TARGET_TRIPLE}'-gcc")

SET(CMAKE_ASM_COMPILER_TARGET "gcc_nto${QNX_PROCESSOR}")

SET(CMAKE_C_COMPILER "${QNX_HOST}/usr/bin/'${TARGET_TRIPLE}'-gcc")

SET(CMAKE_C_COMPILER_TARGET "gcc_nto${QNX_PROCESSOR}")

SET(CMAKE_C_FLAGS_DEBUG "-g")

SET(CMAKE_C_FLAGS_MINSIZEREL "-Os -DNDEBUG")

SET(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")

SET(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g")

SET(CMAKE_C_FLAGS "-D_QNX_SOURCE=1 -I${QNX_TARGET}/usr/include/devs/include_'${TARGET_ARCH}' -I${QNX_TARGET}/usr/include/devs -DDONT_DEFINE_BSD -DDONT_DEFINE___FreeBSD_kernel__ -DDONT_DEFINE_FSCALE -DDONT_DEFINE_MACHINE")

SET(CMAKE_CXX_COMPILER "${QNX_HOST}/usr/bin/'${TARGET_TRIPLE}'-g++")

SET(CMAKE_CXX_COMPILER_TARGET "gcc_nto${QNX_PROCESSOR}")

SET(CMAKE_CXX_FLAGS_DEBUG "-g")

SET(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG")

SET(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")

SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")

SET(CMAKE_CXX_FLAGS "-D_QNX_SOURCE=1 -I${QNX_TARGET}/usr/include/devs/include_'${TARGET_ARCH}' -I${QNX_TARGET}/usr/include/devs -DDONT_DEFINE_BSD -DDONT_DEFINE___FreeBSD_kernel__ -DDONT_DEFINE_FSCALE -DDONT_DEFINE_MACHINE")

SET(CMAKE_LINKER "${QNX_HOST}/usr/bin/'${TARGET_TRIPLE}'-ld.bfd")

SET(CMAKE_SHARED_LINKER_FLAGS "-lsocket")

SET(CMAKE_EXE_LINKER_FLAGS "-lsocket")

SET(CMAKE_RANLIB "${QNX_HOST}/usr/bin/'${TARGET_TRIPLE}'-ranlib")

SET(CMAKE_NM "${QNX_HOST}/usr/bin/'${TARGET_TRIPLE}'-nm")

SET(CMAKE_AR "${QNX_HOST}/usr/bin/'${TARGET_TRIPLE}'-ar")

SET(CMAKE_FIND_ROOT_PATH "${QNX_TARGET}")

SET(CMAKE_FIND_ROOT_PATH_HOST_PROGRAM NEVER)

SET(CMAKE_FIND_ROOT_PATH_HOST_LIBRARY ONLY)

SET(CMAKE_FIND_ROOT_PATH_HOST_INCLUDE ONLY)

' > "${TARGET_TRIPLE}.cmake"

backup_and_patch_if_necessary()

{

        # handy function that patches a file in LLVM_SOURCES_DIR with a given sed replacement regex if necessary, creating backups

        # args: <file pathname> <grep_string_to_test_for_presence_of_patch> <sed regex>

        # test if already patched

        grep -q "${2}" "../${LLVM_SOURCES_DIR}/${1}" && return

        # tell what we're about to do

        echo "Patching ${1}..."

        # have a backup first

        test -f "${CURRENT_DIR}/$(echo "${1}"|tr '/' '.') [ORIGINAL]" || cp "../${LLVM_SOURCES_DIR}/${1}" "${CURRENT_DIR}/$(echo "${1}"|tr '/' '.') [ORIGINAL]" || exit 1

        # perform the patch

        sed -E "${3}" "${CURRENT_DIR}/$(echo "${1}"|tr '/' '.') [ORIGINAL]" > "${CURRENT_DIR}/$(echo "${1}"|tr '/' '.') [PATCHED]"

        # verify that we did it successfully

        if ! grep -q "${2}" "${CURRENT_DIR}/$(echo "${1}"|tr '/' '.') [PATCHED]"; then

                echo "Error: the file ${1} could not be patched. Please investigate and fix manually." | fold -s -w 79; exit 1

        fi

        # and put the patched file in place

        cp -f "${CURRENT_DIR}/$(echo "${1}"|tr '/' '.') [PATCHED]" "../${LLVM_SOURCES_DIR}/${1}" || exit 1

}

# patch llvm/Support/Unix/Path.inc if not done yet

# replace [defined(__FreeBSD_kernel__)] with [(defined(__FreeBSD_kernel__) || defined(__QNXNTO__))]

# RATIONALE: the concerned parts of the QNX platform SDK (/usr/include/devs/*) are actually largely based on FreeBSD code, to conveniently use the FreeBSD network stack.

# NONETHELESS, QNX *IS NOT* FreeBSD and "__FreeBSD_kernel__" *SHOULD NOT* be defined, else userland code may falsely assume a genuine FreeBSD include tree to be available,

# which leads to compilation errors caused by include files assumed to be there whereas they are in fact nowhere to be found (ex: /usr/include/sys/user.h, among others).

# The QNX8 platform SDK has thus been patched to NOT define __FreeBSD_kernel__ when the -DDONT_DEFINE___FreeBSD_kernel__ flag is passed, and the feature tests in Path.inc

# that branch into FreeBSD APIs (which ones *are* exposed in the QNX libc since QNX largely shares FreeBSD code) are about to be patched right now to accept __QNXNTO__ too.

# Additionally, support for statfs()/fstatfs() has been restored in the QNX8 platform SDK in /usr/include/devs/sys/mount.h where it was claimed but absent from the QNX8 libc.

# FIXME: these hacks should be moved elsewhere not to pollute the QNX8 platform SDK -- it *SHOULD* be possible to build without devs

backup_and_patch_if_necessary "llvm/lib/Support/Unix/Path.inc" __QNXNTO__ 's/defined\(__FreeBSD_kernel__\)/\(defined\(__FreeBSD_kernel__\) \|\| \defined\(__QNXNTO__\)\)/g'

# patch lldb/Source/Host/common/Host.cpp if not done yet

# replace [defined(SIGINFO)] with [(defined(SIGINFO) && !defined(__QNXNTO__))]

# RATIONALE: the QNX people defined SIGINFO to the value of SIGUSR1 for a mysterious reason, defeating the purpose of SIGUSR1

# which should be a User-Definable signal as mandated by POSIX. Consequently, userland code that enumerates POSIX signals

# hits twice the same value and all is left to solve this problem is to filter either one out. Since SIGINFO, contrarily to SIGUSR1,

# is an optional signal, this is the value that will be left out - even if on QNX nobody can actually use SIGUSR1's value.

backup_and_patch_if_necessary "lldb/source/Host/common/Host.cpp" __QNXNTO__ 's/defined\(SIGINFO\)/\(defined\(SIGINFO\) \&\& \!defined\(__QNXNTO__\)\)/g'

# FIXME: another patch is needed for LLDB. Basically, the ptrace() POSIX system call doesn't exist on QNX. Instead

# QNX use their own debug utility: pdebug. Note the /proc filesystem can be used to access a debuggee's virtual memory space.

# in lldb/source/Host/posix/ProcessLauncherPosixFork.cpp line 196:

#   if (ptrace(PT_TRACE_ME, 0, nullptr, 0) == -1) // <--- undefined: ptrace, PT_TRACE_ME

#    -D LLVM_ENABLE_PROJECTS="clang;lld;lldb" \

# patch compiler-rt/lib/builtins/enable_execute_stack.c if not done yet

# replace [#include "int_lib.h"] with [#ifdef __QNXNTO__\n#define _QNX_SOURCE 1\n#endif\n#include "int_lib.h"]

# RATIONALE: the QNX people decided to hide most of the UNIX APIs of their libc behind a mandatory _QNX_SOURCE definition to separate the Neutrino primitives from the rest maybe.

# This name does not "enforce" any standard at all (like _POSIX_SOURCE and similar names), it's just a convenient hack for the QNX people to hide stuff.

# To put things back on track and expose the POSIX interfaces of their libc to the world, _QNX_SOURCE has been added to the toolchain CMake file, but it happens that

# when building compiler-rt specifically, the CMAKE_C_FLAGS directive is ignored. Patching the source file seems like the best option to fix that.

backup_and_patch_if_necessary "compiler-rt/lib/builtins/enable_execute_stack.c" __QNXNTO__ 's/#include "int_lib.h"/#ifdef __QNXNTO__\n#define _QNX_SOURCE 1\n#endif\n#include "int_lib.h"/g'

# patch libunwind/src/RWMutex.hpp

# replace [#include <pthread.h>] with [#ifdef __QNXNTO__\n#define __EXT_QNX\n#define __EXT_POSIX1_200112\n#endif\n#include <pthread.h>]

# RATIONALE: the pthreads implementation in QNX 'steals' things from the POSIX.1 standard and hides them behind the __EXT_QNX macro,

# which is defined when either _QNX_SOURCE or __EXT without anything else is defined. This is not correct and low-level libraries

# such as libunwind that do not define either of those fail to build. Also note that a FreeBSD-compatible header was added to the QNX8

# platform SDK: /usr/include/link.h, which defines Elf types as Elf64/Elf32 types depending on the architecture and adds some glue.

backup_and_patch_if_necessary "libunwind/src/RWMutex.hpp" __QNXNTO__ 's/#include <pthread.h>/#ifdef __QNXNTO__\n#define __EXT_QNX\n#define __EXT_POSIX1_200112\n#endif\n#include <pthread.h>/g'

# patch libunwind/src/libunwind.cpp

# replace [#include <libunwind.h>] with [#ifdef __QNXNTO__\n#include <sys/link.h>\n#include <link.h>\n#endif\n#include <libunwind.h>]

# RATIONALE: the ELF type definitions for QNX are incomplete. They renamed everything to Elf32 and Elf64, and now the original Elf type names point to nowhere.

# Additionally, since QNX claims to be a "FreeBSD kernel", it must expose a /usr/include/link.h file too, and the workaround for the aforementioned problem is there.

# Consequently, load the QNX <sys/link.h> THEN our BSD glue <link.h> headers before compiling libunwind.cpp which needs common ELF definitions (and not Elf32/Elf64).

backup_and_patch_if_necessary "libunwind/src/libunwind.cpp" __QNXNTO__ 's,#include <libunwind.h>,#ifdef __QNXNTO__\n#include <sys/link.h>\n#include <link.h>\n#endif\n#include <libunwind.h>,g'

# now configure LLVM -- and use ccache

echo "Configuring LLVM build..."

export CCACHE_DIR="$(realpath "../${LLVM_SOURCES_DIR}/.ccache")"

cmake \

    -D CMAKE_TOOLCHAIN_FILE="${TARGET_TRIPLE}.cmake" \

    -D CMAKE_BUILD_TYPE="MinSizeRel" \

    -D CMAKE_INSTALL_PREFIX="/tmp/qnxsdk/host/qnx$(echo "${QNXSDK_VERSION}"|awk -F. '{print $1}')/${BUILD_TARGET_ARCH}" \

    -D CMAKE_STAGING_PREFIX="/usr/bin" \

    -D CMAKE_C_COMPILER_LAUNCHER="ccache" \

    -D CMAKE_CXX_COMPILER_LAUNCHER="ccache" \

    -D LLVM_HOST_TRIPLE="${TARGET_TRIPLE}" \

    -D LLVM_ENABLE_PROJECTS="clang;lld" \

    -D LLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \

    -D LLVM_TARGETS_TO_BUILD="AArch64;X86" \

    -D COMPILER_RT_BUILD_SANITIZERS="OFF" \

    -G Ninja \

    "../${LLVM_SOURCES_DIR}/llvm" || exit 1

# hijack the "bin" and "lib/clang" output directories and redirect them to the hypervisor's filesystem

test -e "${CURRENT_DIR}/llvm-build" && rm -rf "${CURRENT_DIR}/llvm-build"

mkdir -p "${CURRENT_DIR}/llvm-build"

test -d bin && mv bin "${CURRENT_DIR}/llvm-build/bin" || mkdir "${CURRENT_DIR}/llvm-build/bin"

ln -s "${CURRENT_DIR}/llvm-build/bin" bin

mkdir -p "${CURRENT_DIR}/llvm-build/lib"

test -d lib/clang && mv lib/clang "${CURRENT_DIR}/llvm-build/lib/clang" || mkdir "${CURRENT_DIR}/llvm-build/lib/clang"

ln -s "${CURRENT_DIR}/llvm-build/lib/clang" lib/clang

# and do the Lord's work. https://youtu.be/jcyYmCnkbEE

echo "Building LLVM..."

cmake --build . || exit 1

echo "Champagne, James. Champagne."

exit 0