Problems Generating C++ project with Pytorch and CMake with multiple sourcefiles

I have a file structure like this:

proj/ 
    includes/ 
        foo.h 
    src/ 
        foo.cpp 
    main.cpp

Here is my CMakeLists.txt for main.cpp:

    cmake_minimum_required(VERSION 3.18)
    project(proj)

    set(Torch_DIR /path/to/cmake/Torch)
    find_package(Torch REQUIRED)
    
    include_directories(${PROJECT_SOURCE_DIR}/includes ${TORCH_INCLUDE_DIR})
    add_subdirectory(includes)
    add_subdirectory(src)

    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")

    include_directories(includes) 
    include_directories(${OpenCV_INCLUDE_DIRS})
    include_directories(${TORCH_INCLUDE_DIR})

    add_executable(proj main.cpp)
    target_link_libraries(proj foo_lib ${OpenCV_LIBS} "${TORCH_LIBRARIES}")
    set_property(TARGET proj PROPERTY CXX_STANDARD 14)

Here is my CMakeLists.txt for includes:

    set(Torch_DIR /path/to/cmake/Torch)
    find_package(Torch REQUIRED)
    include_directories(${PROJECT_SOURCE_DIR}/includes ${TORCH_INCLUDE_DIR})

Here is my CMakeLists.txt for src:

    include_directories(${PROJECT_SOURCE_DIR}/includes)
    add_library(foo_lib foo.cpp)

Upon executing make I get:

    fatal error: torch/torch.h: No such file or directory
    1 | #include <torch/torch.h>

But it can link in main.cpp. So obviously my CMakeLists are not created correctly, so any advice on how to create these properly would be appreciated.