DRIVE PX2 ROS CROSS COMPILATION

Autonomous Vehicles DRIVE PX 2 FAQ
1

SETTING UP THE HOST ENVIRONMENT
Use an Ubuntu 16.04-LTS host system. Ubuntu 14.04-LTS may also be used but this increase complexity, requiring use of apt-get to install third-party dependency libraries and headers.

Use Linaro GCC cross compiler toolchain release 5.4, available for free at the following website:

https://releases.linaro.org/components/toolchain/binaries/5.4-2017.05/aarch64-linux-gnu/gcc-linaro-5.4.1-2017.05-x86_64_aarch64-linux-gnu.tar.xz

PREPARING A CROSS-COMPILATION SYSROOT
ROS cross-compilation requires linking standard Linux and third-party headers/libraries, requiring a system root which satisfies all the dependencies. For best results, use the sample arm64 ubuntu-xenial (16.04 LTS) root file system as a system root for cross compilation provided along with the NVIDIA® DRIVE™ 5.0 Linux release.

To install the ubuntu-xenial arm64 root file system
Run the following .run files included with the DRIVE 5.0 Linux release:

drive-t186ref-linux--<build_id>-oss-minimal-pdk.run
drive-t186ref-linux--<build_id>-xenial-ubuntu-rootfs.run
drive-t186ref-linux--<build_id>-nv-minimal-pdk.run

After extracting, an ubuntu-xenial arm64 root file system is available at /drive-t186ref-linux/targetfs, referred to as in this document. Use this as a system root directory for cross-compilation.

Install arm-64 Debian dependencies with one of the following two procedures.

Installing arm64 Debian dependencies with
Qemu-chroot

This method involves Qemu based emulation of arm64 root files system on x86_64 host. This method can be used for both Ubuntu16.04-LTS as well as Ubuntu-14.04 based host systems.

To install dependencies with Qemu-chroot

  1. Install Qemu on the host system with the following command: $ sudo apt-get install qemu-user-static
  2. Copy qemu-aarch64-static into the arm-64 target file system with the following command:

    $ sudo cp /usr/bin/qemu-aarch64-static /usr/bin

  3. Copy resolv.conf from the host system for network access through the Qemu environment with the following command:

    $ sudo cp /etc/resolv.conf /etc

  4. Mount the required sysfs from the host system to the emulated arm64 file system with the following commands:

    $ sudo mount -o bind /dev /dev
    $ sudo mount -o bind /proc /proc
    $ sudo mount -o bind /sys /sys

  5. Enter the emulated file system shell with the following command:

    $ sudo LC_ALL=C chroot

    Enter the commands in the remaining steps in this shell.

  6. Enable the universe repository for installation of several ROS dependencies with the following commands:

    apt-add-repository universe

    apt-get update

  7. Install the following packages with apt-get: libboost-all-dev libtinyxml-dev liblz4-dev libbz2-dev libapr1 libaprutil1 libconsole-bridge-dev python-defusedxml python-rospkg python-catkin-pkg python-netifaces
  8. Exit from the emulated file system shell with the following command:

    exit

  9. Unmount chroot mount points with the following commands:

    $ sudo umount /sys
    $ sudo umount /proc
    $ sudo umount /dev

  10. Remove the Qemu binary, resol.conf, and other cached files, with the following commands:

    $ sudo rm /usr/bin/qemu-aarch64-static
    $ sudo rm /etc/resolv.conf
    $ sudo rm -rf /var/lib/apt/lists/*
    $ sudo rm -rf /dev/*
    $ sudo rm -rf /var/log/*
    $ sudo rm -rf /var/tmp/*
    $ sudo rm -rf /var/cache/apt/archives/.deb
    $ sudo rm -rf /tmp/

Installing dependencies as a multi-architecture package download
This method of installing arm64 Debian dependencies makes use of multi-architecture download support in Ubuntu for downloading packages for a CPU architecture other than the host CPU architecture.
This method is applicable for Ubuntu-16.04-LTS hosts only. Multi architecture support may have to be explicitly enabled at dpkg.

To install dependencies as a multi-architecture download

  1. Download the following Debian packages:

    icu-devtools:arm64
    libboost-all-dev:arm64
    libboost-atomic-dev:arm64
    libboost-atomic1.58-dev:arm64
    libboost-atomic1.58.0:arm64
    libboost-chrono-dev:arm64
    libboost-chrono1.58-dev:arm64
    libboost-chrono1.58.0:arm64
    libboost-context-dev:arm64
    libboost-context1.58-dev:arm64
    libboost-context1.58.0:arm64
    libboost-coroutine-dev:arm64
    libboost-coroutine1.58-dev:arm64
    libboost-coroutine1.58.0:arm64
    libboost-date-time-dev:arm64
    libboost-date-time1.58-dev:arm64
    libboost-dev:arm64
    libboost-exception-dev:arm64
    libboost-exception1.58-dev:arm64
    libboost-filesystem-dev:arm64
    libboost-filesystem1.58-dev:arm64
    libboost-graph-dev:arm64
    libboost-graph-parallel-dev:arm64
    libboost-graph-parallel1.58-dev:arm64
    libboost-graph-parallel1.58.0:arm64
    libboost-graph1.58-dev:arm64
    libboost-graph1.58.0:arm64
    libboost-iostreams-dev:arm64
    libboost-iostreams1.58-dev:arm64
    libboost-locale-dev:arm64
    libboost-locale1.58-dev:arm64
    libboost-locale1.58.0:arm64
    libboost-log-dev:arm64
    libboost-log1.58-dev:arm64
    libboost-log1.58.0:arm64
    libboost-math-dev:arm64
    libboost-math1.58-dev:arm64
    libboost-math1.58.0:arm64
    libboost-mpi-dev:arm64
    libboost-mpi-python-dev:arm64
    libboost-mpi-python1.58-dev:arm64
    libboost-mpi-python1.58.0:arm64
    libboost-mpi1.58-dev:arm64
    libboost-mpi1.58.0:arm64
    libboost-program-options-dev:arm64
    libboost-program-options1.58-dev:arm64
    libboost-program-options1.58.0:arm64
    libboost-python-dev:arm64
    libboost-python1.58-dev:arm64
    libboost-python1.58.0:arm64
    libboost-random-dev:arm64
    libboost-random1.58-dev:arm64
    libboost-random1.58.0:arm64
    libboost-regex-dev:arm64
    libboost-regex1.58-dev:arm64
    libboost-regex1.58.0:arm64
    libboost-serialization-dev:arm64
    libboost-serialization1.58-dev:arm64
    libboost-serialization1.58.0:arm64
    libboost-signals-dev:arm64
    libboost-signals1.58-dev:arm64
    libboost-signals1.58.0:arm64
    libboost-system-dev:arm64
    libboost-system1.58-dev:arm64
    libboost-test-dev:arm64
    libboost-test1.58-dev:arm64
    libboost-test1.58.0:arm64
    libboost-thread-dev:arm64
    libboost-thread1.58-dev:arm64
    libboost-thread1.58.0:arm64
    libboost-timer-dev:arm64
    libboost-timer1.58-dev:arm64
    libboost-timer1.58.0:arm64
    libboost-tools-dev:arm64
    libboost-wave-dev:arm64
    libboost-wave1.58-dev:arm64
    libboost-wave1.58.0:arm64
    libboost1.58-dev:arm64
    libboost1.58-tools-dev:arm64
    libexpat1-dev:arm64
    libhwloc-dev:arm64
    libhwloc-plugins:arm64
    libhwloc5 libibverbs-dev:arm64
    libibverbs1:arm64
    libicu-dev:arm64
    libltdl-dev:arm64
    libnuma-dev:arm64
    libopenmpi-dev:arm64
    libopenmpi1.10:arm64
    libpython-dev:arm64
    libpython2.7-dev:arm64
    mpi-default-bin:arm64
    mpi-default-dev:arm64
    ocl-icd-libopencl1:arm64
    openmpi-bin:arm64
    libconsole-bridge-dev:arm64
    libconsole-bridge0.2v5
    libtinyxml-dev:arm64
    libtinyxml2.6.2v5:arm64
    liblz4-1:arm64
    liblz4-dev:arm64
    libc6-dev:arm64
    libbz2-dev:arm64
    libbz2-1.0:arm64

  2. Extract the packages at the system root with a shell script similar to the following: for entry in ""/*.deb do sudo dpkg --extract $entry /drive-t186ref-linux/targetfs done
  3. Mount /drive-t186ref-linux/targetfs/lib/aarch64-linux-gnu to /lib/aarch64-linux-gnu to fix broken symlinks.

Fixing Broken Symlinks
Whether you installed dependencies with Qemu-chroot or as a multi-architecture package download, the sample xenial root file system might contain symlinks similar to the following:

/usr/lib/aarch64-linux-gnu/libXXXX.so → /lib/aarch64-linux-gnu/libXXXX.so.versio.no

These symlinks resolve on the arm64 platform but are broken on the host system. Attempts to link libXXXX.so are unsuccessful.

To correct broken symlinks
Temporarily fix these broken symlinks with overlays, using commands similar to the following:

$ sudo mkdir /lib/aarch64-linux-gnu
$ sudo mkdir /tmp/ros-cc-overlayfs
$ sudo mount -t overlay -o lowerdir=/lib/aarch64-linux-gnu,upperdir=/lib/aarch64-linux-gnu,workdir=/tmp/ros-cc-overlayfs overlay /lib/aarch64-linux-gnu

DOWNLOADING SOURCES
Download ROS Bare Bone source code can be downloaded from the upstream repositories using the python utilities python-rosinstall-generator and python-wstool.

To install python-rosinstall-generator and python-wstool

  1. Set up the host system to accept packages from ros.org with the following commands:

    $ sudo sh -c ‘echo \ “deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main” > /etc/apt/sources.list.d/ros-latest.list’
    $ sudo apt-get update
    (Line breaks are provided here for readability. There are two commands.)

  2. Install the python ROS utilities with the following command: $ sudo apt-get install python-rosinstall-generator python-wstool python-empy

To download the source code

  1. Create a directory for the source code with the following command:

    $ mkdir $HOME/ros-bare-bone-src && cd $HOME/ros-bare-bone-src

  2. Prepare to install the ROS bare-bones packages with the following command:

    $ rosinstall_generator ros_comm --rosdistro kinetic --deps --wet-only --tar > kinetic-ros_comm-wet.rosinstall

  3. Download the source with the following command:

    $ wstool init -j8 src kinetic-ros_comm-wet.rosinstall
    Source code is downloaded to $HOME/ros-bare-bone-src/src.

INVOKING THE CROSS COMPILER

ROS has a cmake-based build system called catkin. Cross-compilation uses catkin with a system root and toolchain downloaded and set up earlier in this document. To invoke a different cross-compiler, cmake requires a specific file CMAKE_TOOLCHAIN_FILE, specified as an argument of cmake.
More information about CMAKE_TOOLCHAIN_FILE is available at the following websites:

https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html
https://www.vtk.org/Wiki/CMake_Cross_Compiling

The following is a sample CMAKE_TOOLCHAIN_FILE for ROS cross compilation. Add additional flags and variables as needed.

set(CMAKE_SYSTEM_NAME Linux)

Specify the cross compiler

set(TOOLCHAIN <PATH_TO_LINARO_GCC_CROSS_COMPILER>)
set(CMAKE_CXX_COMPILER “${TOOLCHAIN}/bin/aarch64-linux-gnu-g++”)
set(CMAKE_C_COMPILER “${TOOLCHAIN}/bin/aarch64-linux-gnu-gcc”)

Targetfs path

set(ROS_SYSROOT “PATH_OF_SYSROOT”)

Library paths

set(LD_PATH “${ROS_SYSROOT}/usr/lib/aarch64-linux-gnu”)
set(LD_PATH_EXTRA “${ROS_SYSROOT}/lib/aarch64-linux-gnu”)

locate the target environment

set(CMAKE_FIND_ROOT_PATH ${ROS_SYSROOT})

Set compiler flags

set(CMAKE_SHARED_LINKER_FLAGS “–sysroot=${CMAKE_FIND_ROOT_PATH} -L${LD_PATH} -L${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} ${CMAKE_SHARED_LINKER_FLAGS}”)
set(CMAKE_MODULE_LINKER_FLAGS “–sysroot=${CMAKE_FIND_ROOT_PATH} -L${LD_PATH} -L${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} ${CMAKE_SHARED_LINKER_FLAGS}”)
set(CMAKE_EXE_LINKER_FLAGS “–sysroot=${CMAKE_FIND_ROOT_PATH} -L${LD_PATH} -L${LD_PATH_EXTRA} -Wl,-rpath,${LD_PATH} -Wl,-rpath,${LD_PATH_EXTRA} ${CMAKE_EXE_LINKER_FLAGS}”)
set(CMAKE_C_FLAGS “-fPIC --sysroot=${CMAKE_FIND_ROOT_PATH}” CACHE INTERNAL “” FORCE)
set(CMAKE_CXX_FLAGS “-fPIC --sysroot=${CMAKE_FIND_ROOT_PATH}” CACHE INTERNAL “” FORCE)

Search for programs only in the build host directories

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

Search for libraries and headers only in the target directories

set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

set system default include dir

include_directories(BEFORE SYSTEM ${ROS_SYSROOT}/…/include)

To cross-compile ROS

  1. Invoke cross-compilation with the following command:

    $ sudo ./ros-bare-bone-src/src/catkin/bin/catkin_make_isolated
    –install -DCMAKE_BUILD_TYPE=Release
    –source ros-bare-bone-src/src
    -DCMAKE_TOOLCHAIN_FILE=<path_to_toolchain-cmake-file>
    -DCATKIN_ENABLE_TESTING=false

    The resultant binaries are created in the directory from which catkin_make_isolated is invoked, in three directories: devel_isolated, build_isolated and install_isolated. Use the --devel-space, --build_space, and --install-space to specify non-default paths.
    Transfer the binaries to the arm64 platform with scp or similar.

  2. After cross-compilation is complete, revert the overlayfs mount with the following commands:

    $ sudo umount /lib /aarch64-linux-gnu
    $ sudo rm -rf /tmp/ros-cc-overlayfs

    Transfer the binaries to the arm64 platform with scp or a similar utility. Before executing cross compiled ROS binaries on the arm64 target, make sure the packages which were installed using Qemu or Ubuntu multi-architecture support are also installed on the target system.

1 Like