[SOLVED] Installing and Linking to Apache Arrow inside Cmake

Issue

I'm trying to build and link-to apache-arrow v9.0.0 inside my cmake project using the following section in my CMakeLists.txt file.

ExternalProject_Add(arrow
        URL "https://www.apache.org/dist/arrow/arrow-9.0.0/apache-arrow-9.0.0.tar.gz"
        SOURCE_SUBDIR cpp)
message(STATUS "arrow source dir: ${arrow_SOURCE_DIR}")
include_directories(${arrow_SOURCE_DIR}/cpp/src)

The compilation complains that the apache-arrow headers are missing

fatal error: 'arrow/array.h' file not found
#include <arrow/array.h>
         ^~~~~~~~~~~~~~~
1 error generated.

supported by the fact that the output of message(STATUS "arrow source dir: ${arrow_SOURCE_DIR}") is empty

-- arrow source dir: 

Another error seemingly related to the apache-arrow installation reported by cmake is that

CMake Error at cmake_modules/ThirdpartyToolchain.cmake:267 (find_package):
  Could not find a configuration file for package "xsimd" that is compatible
  with requested version "8.1.0".

  The following configuration files were considered but not accepted:

    /opt/homebrew/lib/cmake/xsimd/xsimdConfig.cmake, version: 9.0.1

Call Stack (most recent call first):
  cmake_modules/ThirdpartyToolchain.cmake:2245 (resolve_dependency)
  CMakeLists.txt:575 (include)

Of course, the traditional approach of installing apache-arrow externally with say brew install apache-arrow and using find_package works well enough, but I'd like something more cross-platform. One of the arrow devs had provided a link on how to properly use include_directories with ExternalProject_Add for an earlier question, but I guess that example is now outdated.

What's the recommended way of installing and then linking-to apache-arrow inside a cmake project using ExternalProject_Add?

Edit: Minimal Example

CMakeLists.txt

cmake_minimum_required(VERSION 3.24)
project(arrow_cmake)

set(CMAKE_CXX_STANDARD 23)

include(ExternalProject)

ExternalProject_Add(Arrow
        URL "https://www.apache.org/dist/arrow/arrow-9.0.0/apache-arrow-9.0.0.tar.gz"
        SOURCE_SUBDIR cpp
        CMAKE_ARGS "-Dxsimd_SOURCE=BUNDLED"
        )
add_executable(${PROJECT_NAME} main.cpp)
target_link_libraries(${PROJECT_NAME} arrow_shared)

main.cpp

#include <iostream>

#include <arrow/array.h> // not found!

int main() {
    std::cout << "Hello, World!" << std::endl;
    return 0;
}

Solution

Building arrow from sources in cmake took quite some doing. It's heavily influenced by this link.

cmake/arrow.cmake

# Build the Arrow C++ libraries.
function(build_arrow)
    set(one_value_args)
    set(multi_value_args)

    cmake_parse_arguments(ARG
            "${options}"
            "${one_value_args}"
            "${multi_value_args}"
            ${ARGN})
    if (ARG_UNPARSED_ARGUMENTS)
        message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
    endif ()

    # If Arrow needs to be built, the default location will be within the build tree.
    set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix")

    set(ARROW_SHARED_LIBRARY_DIR "${ARROW_PREFIX}/lib")

    set(ARROW_SHARED_LIB_FILENAME
            "${CMAKE_SHARED_LIBRARY_PREFIX}arrow${CMAKE_SHARED_LIBRARY_SUFFIX}")
    set(ARROW_SHARED_LIB "${ARROW_SHARED_LIBRARY_DIR}/${ARROW_SHARED_LIB_FILENAME}")
    set(PARQUET_SHARED_LIB_FILENAME
            "${CMAKE_SHARED_LIBRARY_PREFIX}parquet${CMAKE_SHARED_LIBRARY_SUFFIX}")
    set(PARQUET_SHARED_LIB "${ARROW_SHARED_LIBRARY_DIR}/${PARQUET_SHARED_LIB_FILENAME}")

    set(ARROW_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-build")
    set(ARROW_CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}"
            "-DCMAKE_INSTALL_LIBDIR=lib" "-Dxsimd_SOURCE=BUNDLED"
            "-DARROW_BUILD_STATIC=OFF" "-DARROW_PARQUET=ON"
            "-DARROW_WITH_UTF8PROC=OFF" "-DARROW_WITH_RE2=OFF"
            "-DARROW_FILESYSTEM=ON" "-DARROW_CSV=ON" "-DARROW_PYTHON=ON")
    set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")

    set(ARROW_BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}" "${PARQUET_SHARED_LIB}")

    include(ExternalProject)

    externalproject_add(arrow_ep
            URL https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0.tar.gz
            SOURCE_SUBDIR cpp
            BINARY_DIR "${ARROW_BINARY_DIR}"
            CMAKE_ARGS "${ARROW_CMAKE_ARGS}"
            BUILD_BYPRODUCTS "${ARROW_BUILD_BYPRODUCTS}")

    set(ARROW_LIBRARY_TARGET arrow_shared)
    set(PARQUET_LIBRARY_TARGET parquet_shared)

    file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
    add_library(${ARROW_LIBRARY_TARGET} SHARED IMPORTED)
    add_library(${PARQUET_LIBRARY_TARGET} SHARED IMPORTED)
    set_target_properties(${ARROW_LIBRARY_TARGET}
            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${ARROW_INCLUDE_DIR}
            IMPORTED_LOCATION ${ARROW_SHARED_LIB})
    set_target_properties(${PARQUET_LIBRARY_TARGET}
            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${ARROW_INCLUDE_DIR}
            IMPORTED_LOCATION ${PARQUET_SHARED_LIB})

    add_dependencies(${ARROW_LIBRARY_TARGET} arrow_ep)
endfunction()

Use it in your CMakeLists.txt file as

...
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
include(arrow)
build_arrow()

Answered By - marital_weeping

Answer Checked By - Willingham (WPSolving Volunteer)