Cross-compiling with cmake for embedded systems QNX

shyney7 · May 4, 2020, 1:19pm

Hi,
I would like to use the C++ API for a RTOS (QNX). I’m using cmake 3.10.2 for the cross compilation.
To make the cross compilation possible I use a toolchain file that looks like this:

SET(CMAKE_SYSTEM_PROCESSOR "x86_64")
SET(CMAKE_SYSTEM_NAME QNX)

IF(CMAKE_HOST_LINUX)
  SET(HOST_EXECUTABLE_SUFFIX "")
ENDIF(CMAKE_HOST_LINUX)

FIND_PATH(QNX_HOST
  NAME usr/bin/qcc${HOST_EXECUTABLE_SUFFIX}
  PATHS $ENV{QNX_HOST}
  NO_CMAKE_PATH
  NO_CMAKE_ENVIRONMENT_PATH
)
  
FIND_PATH(QNX_TARGET
  NAME usr/include/errno.h
  PATHS $ENV{QNX_TARGET}
  NO_CMAKE_PATH
  NO_CMAKE_ENVIRONMENT_PATH
)

IF(CMAKE_HOST_LINUX)
  FIND_PATH(QNX_CONFIGURATION
    NAME /etc/qnx/bin/qnxactivate
    PATHS $ENV{QNX_CONFIGURATION}
    "$ENV{QNX_HOST}/usr/bin/qconfig"
    NO_CMAKE_PATH
    NO_CMAKE_ENVIRONMENT_PATH
 )
ENDIF(CMAKE_HOST_LINUX)

SET(ENV{QNX_HOST} ${QNX_HOST})
SET(ENV{QNX_TARGET} ${QNX_TARGET})
IF(CMAKE_HOST_LINUX)
  SET(ENV{QNX_CONFIGURATION} ${QNX_CONFIGURATION})
  SET(ENV{PATH} "$ENV{PATH};${QNX_HOST}/usr/bin")
ENDIF(CMAKE_HOST_LINUX)

SET(CMAKE_MAKE_PROGRAM "${QNX_HOST}/usr/bin/make${HOST_EXECUTABLE_SUFFIX}"    CACHE PATH "QNX Make Program")
SET(CMAKE_SH           "${QNX_HOST}/usr/bin/ksh${HOST_EXECUTABLE_SUFFIX}"      CACHE PATH "QNX shell Program")
SET(CMAKE_AR           "${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-ar${HOST_EXECUTABLE_SUFFIX}"      CACHE PATH "QNX ar Program")
SET(CMAKE_RANLIB       "${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-ranlib${HOST_EXECUTABLE_SUFFIX}"      CACHE PATH "QNX ranlib Program")
SET(CMAKE_NM           "${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-nm${HOST_EXECUTABLE_SUFFIX}"      CACHE PATH "QNX nm Program")
SET(CMAKE_OBJCOPY      "${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-objcopy${HOST_EXECUTABLE_SUFFIX}" CACHE PATH "QNX objcopy Program")
SET(CMAKE_OBJDUMP      "${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-objdump${HOST_EXECUTABLE_SUFFIX}" CACHE PATH "QNX objdump Program")
SET(CMAKE_LINKER       "${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-ld"     CACHE PATH "QNX Linker Program")
SET(CMAKE_STRIP        "${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-strip${HOST_EXECUTABLE_SUFFIX}"   CACHE PATH "QNX Strip Program")

SET(CMAKE_C_COMPILER ${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-gcc${HOST_EXECUTABLE_SUFFIX})
SET(CMAKE_C_FLAGS_DEBUG "-g")
SET(CMAKE_C_FLAGS_MINSIZEREL "-Os -DNDEBUG")
SET(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
SET(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g")

SET(CMAKE_CXX_COMPILER ${QNX_HOST}/usr/bin/nto${CMAKE_SYSTEM_PROCESSOR}-c++${HOST_EXECUTABLE_SUFFIX})
SET(CMAKE_CXX_FLAGS_DEBUG "-g")
SET(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG")
SET(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g")

SET(CMAKE_FIND_ROOT_PATH ${QNX_TARGET}) 
SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

If I compile a program without Pytorch everything works fine. But when I customize the CMakeLists.txt for Pytorch I get an error message when I run the following cmake command:

cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/qnx_7.0.0_linux_x86_64.cmake ..
-- The C compiler identification is GNU 5.4.0
-- The CXX compiler identification is GNU 5.4.0
-- Check for working C compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-gcc
-- Check for working C compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-gcc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Check for working CXX compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-c++
-- Check for working CXX compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Looking for pthread_create
-- Looking for pthread_create - found
-- Found Threads: TRUE  
-- Could NOT find torch (missing: TORCH_LIBRARY) 
CMake Error: The following variables are used in this project, but they are set to NOTFOUND.
Please set them or make sure they are set and tested correctly in the CMake files:
torch;torch_library;C10_LIBRARY
    linked by target "owndataset_tsp" in directory /home/marcel/Documents/pytroch_cpp_owndataset_tsp

-- Configuring incomplete, errors occurred!
See also "/home/marcel/Documents/pytroch_cpp_owndataset_tsp/build/CMakeFiles/CMakeOutput.log".

My CMakeLists.txt File looks like this:

cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR)

project(pytorch_cpp_owndataset)

list(APPEND CMAKE_PREFIX_PATH "/home/marcel/Documents/libs/libtorch1.5linux/libtorch")
set(CMAKE_MODULE_PATH "/home/marcel/Documents/libs/libtorch1.5linux/libtorch/share/cmake/Torch")

find_package(Torch REQUIRED)

if ("${CMAKE_SYSTEM_NAME}" STREQUAL QNX)
	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14")
endif()

include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)

link_directories(${CMAKE_BINARY_DIR}/bin)
set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/bin)

set(EXECUTABLE_NAME owndataset_tsp)

add_executable(${EXECUTABLE_NAME} main.cpp src/csv2vector.cpp src/owndata.cpp)

target_link_libraries(${EXECUTABLE_NAME} "${TORCH_LIBRARIES}")
set_property(TARGET ${EXECUTABLE_NAME} PROPERTY CXX_STANDARD 14)

I can’t explain why cmake can’t find the torch library. The path is correctly set in the CMakeLists.txt. Because if I run cmake without the toolchain file and compile for my host machine everything works.

I hope someone can help me with that.

Here is the CMakeOutput.log:

gist.github.com

https://gist.github.com/shyney7/02c1d6cb5ce9418b7bb8df2817cbf024

CMakeOutput.log

The target system is: QNX -  - x86_64
The host system is: Linux - 5.3.0-51-generic - x86_64
Compiling the C compiler identification source file "CMakeCCompilerId.c" succeeded.
Compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-gcc 
Build flags: 
Id flags:  

The output was:
0

This file has been truncated. show original

glaringlee · May 5, 2020, 3:51pm

@shyney7
Can you remove “set(CMAKE_MODULE_PATH” line and try again?
If it doesn’t work, I suggest to follow https://pytorch.org/cppdocs/installing.html to create a simple example and make it compiled and then merge the Camke lines into your project.

shyney7 · May 6, 2020, 6:43am

@glaringlee Thank you for your help! Unfortunately I have the same problem after removing the set(CMAKE_MODULE_PATH… command. This is how my CMakeLists.txt file looks like right now. I have created a simple test program from the cppdocs where I just create a Tensor and print it out with std::cout.

cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
project(libtorch_cross_test)
list(APPEND CMAKE_PREFIX_PATH "/home/marcel/Documents/libs/libtorch1.5linux/libtorch")
find_package(Torch REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")

add_executable(example-app main.cpp)
target_link_libraries(example-app "${TORCH_LIBRARIES}")
set_property(TARGET example-app PROPERTY CXX_STANDARD 14)

If I compile this without the cmake toolchain file for the cross compilation everything works just fine.
As soon as I add the cmake flag
-DCMAKE_TOOLCHAIN_FILE=…/cmake/qnx_7.0.0_linux_x86_64.cmake
to use my toolchain I get the same missing: TORCH_LIBRARY message.

cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/qnx_7.0.0_linux_x86_64.cmake ..
-- The C compiler identification is GNU 5.4.0
-- The CXX compiler identification is GNU 5.4.0
-- Check for working C compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-gcc
-- Check for working C compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-gcc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Check for working CXX compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-c++
-- Check for working CXX compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Looking for pthread_create
-- Looking for pthread_create - found
-- Found Threads: TRUE  
-- Could NOT find torch (missing: TORCH_LIBRARY) 
CMake Error: The following variables are used in this project, but they are set to NOTFOUND.
Please set them or make sure they are set and tested correctly in the CMake files:
torch;torch_library;C10_LIBRARY
    linked by target "example-app" in directory /home/marcel/Documents/libtorch_cross_test

-- Configuring incomplete, errors occurred!
See also "/home/marcel/Documents/libtorch_cross_test/build/CMakeFiles/CMakeOutput.log".

The QNX compiler (qcc) seems to be recognized just fine by cmake so I cant explain why this happens.

shyney7 · May 6, 2020, 7:52am

Ok I was able to fix it by myself by changing SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) to
SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)

With this change I was able to create the make file with cmake.
But now as soon i try to build the source file with the “make” command I get the following compiler errors:

gist.github.com

https://gist.github.com/shyney7/ca3e59186c2904024ab037df11213e57

Libtorch_compiler_error_QNX

marcel@ubuntu:~/Documents/libtorch_cross_test/build$ make
Scanning dependencies of target example-app
[ 50%] Building CXX object CMakeFiles/example-app.dir/main.cpp.o
In file included from /home/marcel/qnx700/target/qnx7/usr/include/c++/v1/algorithm:628:0,
                 from /home/marcel/qnx700/target/qnx7/usr/include/c++/v1/string:439,
                 from /home/marcel/qnx700/target/qnx7/usr/include/c++/v1/__locale:15,
                 from /home/marcel/qnx700/target/qnx7/usr/include/c++/v1/ios:216,
                 from /home/marcel/qnx700/target/qnx7/usr/include/c++/v1/iostream:38,
                 from /home/marcel/Documents/libtorch_cross_test/main.cpp:1:
/home/marcel/qnx700/target/qnx7/usr/include/c++/v1/memory: In instantiation of 'struct std::__1::allocator_traits<std::__1::allocator_traits<std::__1::allocator<std::__1::pair<c10::IValue, c10::IValue> > >::rebind_alloc<ska_ordered::detailv3::sherwood_v3_entry<std::__1::pair<c10::IValue, c10::IValue> > > >':

This file has been truncated. show original

glaringlee · May 6, 2020, 3:10pm

@shyney7
Once you tell cmake to do CMAKE_TOOLCHAIN_FILE, cmake enter into cross-compiling mode, your library search root will be set here:
SET(CMAKE_FIND_ROOT_PATH ${QNX_TARGET})
This will override your CMAKE_PREFIX_PATH.

I am not sure how many other things will be override if specifying toolchain file, you can try specify your torch root in toolchain file, something like SET(CMAKE_FIND_ROOT_PATH $(QNX_TARGET), $(TORCH_ROOT)). or you can put torch root into system root and then set SET(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH)

shyney7 · May 7, 2020, 11:02am

@glaringlee
As you can see from the second post I have already managed to generate the makefiles with cmake by adjusting the toolchain file with the following command:

SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)

Since I told cmake to search for libraries not only in the root of the target but also in the root of the host it was able to find the libtorch library. But the problem I have now is that the compiler shows error messages during the build process. Unfortunately I am not a programmer but a mechanical engineer working at a research lab of a german university trying to implement libtorch into production. So I am not sure what the compiler errors mean. I guess it is because of the stricter memory policies of QNX since it is an RTOS? But then I wonder why in the announcement videos where Libtorch and the C++ API were introduced, it was always said that the C++ API is specifically for production environments. The fact is that in production environments you mostly work with real-time operating systems like QNX or FreeRTOS.
Or is there a way to run Libtorch on an RTOS that I’m not aware of? The most important thing would be if at least pretrained models would run in realtime in the so-called online training.

glaringlee · May 7, 2020, 4:34pm

I think we were typing at the same time and you posted yours right before mine
I personally don’t think this error is due to RTOS strict memory policies.

This is a compile error, this is related to how you use the libtorch apis in main.cpp, to me, this is more like an implementation error.

Can you try just include several libtorch headers without doing anything in main.cpp, see whether you can compile it successfully?

shyney7 · May 8, 2020, 1:15pm

@glaringlee
My main.cpp just had the example code from the cppdocs. But now I’ve done as you suggested and just created a main.cpp with the torch.h header file like this:


#include <torch/torch.h>

int main() {

return 0;
}

But Im still getting the same compiler errors:

gist.github.com

https://gist.github.com/shyney7/6430c088cdeff1d120c7620aa9ab2289

Libtorch cross-compilation error QNX

marcel@ubuntu:~/Documents/libtorch_cross_test/build$ source ~/qnx700/qnxsdp-env.sh 
QNX_HOST=/home/marcel/qnx700/host/linux/x86_64
QNX_TARGET=/home/marcel/qnx700/target/qnx7
MAKEFLAGS=-I/home/marcel/qnx700/target/qnx7/usr/include
marcel@ubuntu:~/Documents/libtorch_cross_test/build$ make
[ 50%] Building CXX object CMakeFiles/example-app.dir/main.cpp.o
In file included from /home/marcel/Documents/libs/libtorch/include/c10/core/Allocator.h:4:0,
                 from /home/marcel/Documents/libs/libtorch/include/ATen/ATen.h:3,
                 from /home/marcel/Documents/libs/libtorch/include/torch/csrc/api/include/torch/types.h:3,
                 from /home/marcel/Documents/libs/libtorch/include/torch/csrc/api/include/torch/data/dataloader_options.h:4,

This file has been truncated. show original

I hope you can help me with that.

glaringlee · May 12, 2020, 7:17pm

@shyney7
It seems to me that your make file still breaks something in building chain. This is not caused by libtorch.
I am not familiar with QNX, from your error log, I can see it compiled c++ code using v1/memory instead of standard memory lib. libtorch is compiled with standard c++14 libs, we can not guarantee it compiled with other version libs. I checked QNX a little bit, v1/memory is used for llvm compiler infrastructure, while libtorch is compiled using gcc/g++. You should compile your code using the memory lib in 5.4.0 folder (same level as ‘v1’ folder).

Can you try this, I am not sure whether it will work though.
if ("${CMAKE_SYSTEM_NAME}" STREQUAL QNX)
set(CMAKE_CXX_FLAGS “${CMAKE_CXX_FLAGS} -std=gnu++14 -stdlib=libstdc++”)
endif()

And another thing is that, I think it would be better to add your libtorch path into CMAKE_FIND_ROOT_PATH and put back CMAKE_FIND_ROOT_PATH_MODE_PACKAGE from BOTH to ONLY.

shyney7 · May 13, 2020, 12:52pm

@glaringlee
Thank you so much for replying and trying to help! I really appreciate this!
This is also what I found in another repo on github while searching the web for similar errors.
So I have changed the CMakelists.txt file as follows:

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14 -stdlib=libstdc++ ${TORCH_CXX_FLAGS}")

But now I’m getting the following compiler errors:

github.com

shyney7/libtorch-cross-RTOS-test/blob/fa8f42964be0b4e194f402f8296909f0e438d470/Compilerlogs/comperr_with-stdlib=libstdc++_flag.md

```console
marcel@ubuntu:~/Documents/libtorch_cross_test/build$ cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/qnx_7.0.0_linux_x86_64.cmake ..
-- The C compiler identification is GNU 5.4.0
-- The CXX compiler identification is GNU 5.4.0
-- Check for working C compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-gcc
-- Check for working C compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-gcc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Check for working CXX compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-c++
-- Check for working CXX compiler: /home/marcel/qnx700/host/linux/x86_64/usr/bin/ntox86_64-c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Looking for pthread_create
-- Looking for pthread_create - found

This file has been truncated. show original

I have also opened an issue on github describing everything and with all the files from a repo I created.

github.com/pytorch/pytorch

Compiling errors when trying to cross-compile the C++ API for RTOS (QNX)

opened 09:12AM - 12 May 20 UTC

shyney7

module: build module: cpp triaged

I am currently trying to implement the Pytorch C++ API into production. RTOS are… indispensable in production environments so I'm trying to cross-compile with cmake for QNX Neutrino 7.0.0 (x86_64). However, when compiling a simple test program (empty main() function just including "#include <torch/torch.h>") I get the following compiler errors: https://github.com/shyney7/libtorch-cross-RTOS-test/blob/master/Compilerlogs/comperr-without-stdlib-flag.md I had to set the following command so that cmake finds the torch library in the host path. ` SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)` This is how my **cmake Toolchain File** looks like: https://github.com/shyney7/libtorch-cross-RTOS-test/blob/master/cmake/qnx_7.0.0_linux_x86_64.cmake This is my **CMakeLists.txt** file: https://github.com/shyney7/libtorch-cross-RTOS-test/blob/master/CMakeLists.txt Unfortunately I am not a computer scientist but a mechanical engineer working at a research lab of a german university trying to implement libtorch into production. So I am not sure what the compiler errors mean. Thats why I searched the web for similar errors and found an issue of another repo here: https://github.com/eProsima/Fast-RTPS/issues/300 There it is mentioned that it is also needed to prevent the qcc compiler from using the LLVM Compiler Infrastructure from v1/memory by using the `-stdlib=libstdc++` flag. I also tried this by changing my CMakeLists.txt file to use the following flags: `set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14 -stdlib=libstdc++ ${TORCH_CXX_FLAGS}")` But then I'm getting the following **compiler errors**: https://github.com/shyney7/libtorch-cross-RTOS-test/blob/master/Compilerlogs/comperr_with-stdlib%3Dlibstdc%2B%2B_flag.md In the announcement videos where Libtorch and the C++ API were introduced, it was always said that the C++ API is specifically for production environments. The fact is that in production environments you mostly work with real-time operating systems like QNX or FreeRTOS, so I'm sure that there is somehow a way to get libtorch working on QNX. Here is the repo with all the files: https://github.com/shyney7/libtorch-cross-RTOS-test Here is also a link to my first post on the discussion forums where I faced some issues with setting up the cmake toolchain file correctly: https://discuss.pytorch.org/t/cross-compiling-with-cmake-for-embedded-systems-qnx/79681 ## Environment - Libtorch Version : 1.5 Release - OS : cross-compiling on Ubuntu 18.04 (VM) for **QNX 7.0.0 (x84_64)** - How you installed PyTorch : libtorch 1.5 zip from pytorch.org (for linux tried with both Pre & cxx11 ABI) - Build command you used : `cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/qnx_7.0.0_linux_x86_64.cmake ..` `make` - GNU version: 5.4.0 - CUDA/cuDNN version: none - GPU models and configuration: none - cmake version: 3.10.2 cc @malfet @yf225 @glaringlee

I’ve tried this by moving the libtorch folder to the QNX Target Environment as stated here (QNX Docs):

/home/marcel/qnx700/target/qnx7/x86_64/lib/libtorch/

And changed the line in the CMakelists.txt as follows:
list(APPEND CMAKE_PREFIX_PATH "${QNX_TARGET}/x86_64/lib/libtorch")

But this change did not lead to a succesful build and I’m getting the same compiler errors as shown above.
Compiler Errors

glaringlee · May 13, 2020, 3:15pm

@shyney7
What’s your operating system? ubuntu? centos?
Did you installed libc and libc6-dev? if so can you upgrade glibc?
From your latest error, that is caused by missing libs, please check the document of your operating system see how to install them. If you are on the system above, try install libc and libc6-dev and upgrade them.

Thanks for filing an issue with us, I will assign that issue to proper people to take a look at, but again we are not QNX people, if you can show your error to a QNX engineer, they might give you a quicker answer, the reason is that libtorch is built using standard libraries while QNX is built using extensions, these errors to me is caused by QNX env.