How do I build a Python extension module with CMake?

Question

I'm trying to build a Python extension module with CMake and f2py. The module builds just fine, but setuptools can't find it.

My build directory looks like this:

cmake/modules/FindF2PY.cmake
cmake/modules/FindPythonExtensions.cmake
cmake/modules/UseF2PY.cmake
cmake/modules/FindNumPy.cmake
cmake/modules/targetLinkLibrariesWithDynamicLookup.cmake
setup.py
CMakeLists.txt
f2py_test/__init__.py
f2py_test.f90

f2py_test/init.py is just an empty file. The files within cmake/modules are copied from scikit-build.

setup.py is based on a blog post from Martino Pilia

from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
import os
import sys

class CMakeExtension(Extension):
    def __init__(self, name, cmake_lists_dir='.', **kwa):
        Extension.__init__(self, name, sources=[], **kwa)
        self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)

class cmake_build_ext(build_ext):
    def build_extensions(self):

        import subprocess

        # Ensure that CMake is present and working
        try:
            out = subprocess.check_output(['cmake', '--version'])
        except OSError:
            raise RuntimeError('Cannot find CMake executable')

        for ext in self.extensions:

            extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
            cfg = 'Debug' if os.environ.get('DISPTOOLS_DEBUG','OFF') == 'ON' else 'Release'

            cmake_args = [
                '-DCMAKE_BUILD_TYPE=%s' % cfg,
                # Ask CMake to place the resulting library in the directory
                # containing the extension
                '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir),
                # Other intermediate static libraries are placed in a
                # temporary build directory instead
                '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), self.build_temp),
                # Hint CMake to use the same Python executable that
                # is launching the build, prevents possible mismatching if
                # multiple versions of Python are installed
                '-DPYTHON_EXECUTABLE={}'.format(sys.executable),

            ]

            if not os.path.exists(self.build_temp):
                os.makedirs(self.build_temp)

            # Config
            subprocess.check_call(['cmake', ext.cmake_lists_dir] + cmake_args,
                                  cwd=self.build_temp)

            # Build
            subprocess.check_call(['cmake', '--build', '.', '--config', cfg],
                                  cwd=self.build_temp)

setup(
    name="f2py_test",
    version='0.0.1',
    packages=['f2py_test'],
    ext_modules=[CMakeExtension(name='f2py_test_')],
    cmdclass={'build_ext':cmake_build_ext},
)

CMakeLists.txt:

cmake_minimum_required(VERSION 3.10.2)

project(f2py_test)

set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake/modules/")

enable_language(Fortran)

find_package(F2PY)
find_package(PythonExtensions)

set(f2py_test_sources f2py_test.f90)

add_library(f2py_test ${f2py_test_sources})

function(add_f2py_target)
  set(options)
  set(singleValueArgs)
  set(multiValueArgs SOURCES DEPENDS)
  cmake_parse_arguments(
    PARSE_ARGV 1
    F2PY_TARGET "${options}" "${singleValueArgs}"
    "${multiValueArgs}"
    )

  set(F2PY_TARGET_MODULE_NAME ${ARGV0})

  set(generated_module_file ${CMAKE_CURRENT_BINARY_DIR}/${F2PY_TARGET_MODULE_NAME}${PYTHON_EXTENSION_MODULE_SUFFIX})

  message(${generated_module_file})
  
  set(f2py_module_sources_fullpath "")
  foreach(f ${F2PY_TARGET_SOURCES})
    list(APPEND f2py_module_sources_fullpath "${CMAKE_CURRENT_SOURCE_DIR}/${f}")
  endforeach()

  add_custom_target(${F2PY_TARGET_MODULE_NAME} ALL
    DEPENDS ${generated_module_file} ${generated_module_file}
    )

  if(F2PY_TARGET_DEPENDS)
    add_dependencies(${F2PY_TARGET_MODULE_NAME} ${F2PY_TARGET_DEPENDS})
  endif()

  if(APPLE)
    set(F2PY_ENV LDFLAGS='-undefined dynamic_lookup -bundle')
  else()
    set(F2PY_ENV LDFLAGS='$ENV{LDFLAGS} -shared')
  endif()

  add_custom_command(
    OUTPUT ${generated_module_file}
    DEPENDS ${F2PY_TARGET_SOURCES}
    COMMAND env ${F2PY_ENV} ${F2PY_EXECUTABLE} --quiet
    -m ${F2PY_TARGET_MODULE_NAME}
    -c ${f2py_module_sources_fullpath}
    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})

  set_target_properties(
    ${F2PY_TARGET}
    PROPERTIES
    PREFIX ""
    OUTPUT_NAME ${F2PY_TARGET_MODULE_NAME})

endfunction(add_f2py_target)

if(F2PY_FOUND)
  add_f2py_target(f2py_test_ SOURCES ${f2py_test_sources} DEPENDS f2py_test)
endif()

f2py_test.f90:

module mod_f2py_test
  implicit none
contains
  subroutine f2py_test(a,b,c)
    real(kind=8), intent(in)::a,b
    real(kind=8), intent(out)::c
  end subroutine f2py_test
end module mod_f2py_test

python setup.py develop invokes cmake to build the extension module, which I can see in ./build/temp.macosx-10.14-x86_64-3.8/f2py_test_.cpython-38-darwin.so. However, setuptools can't find the file and prints the message error: can't copy 'build/lib.macosx-10.14-x86_64-3.8/f2py_test_.cpython-38-darwin.so': doesn't exist or not a regular file.

How do I either 1) Tell CMake to install the extension module where setuptools expects it or 2) Tell setuptools where to find the extension module.

Answer 1

The directory where setuptools looks for the compiled module can be obtained by build_ext.get_ext_fullpath(ext.name). In the above code the resulting path is passed to CMake by setting the variable CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE.

Since f2py is invoked through a custom command, the extension module is not automatically copied to the output directory. This can be achieved by another call to add_custom_command:

  add_custom_command(TARGET "${F2PY_TARGET_MODULE_NAME}" POST_BUILD
    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${CMAKE_CURRENT_BINARY_DIR}/${generated_module_file}" "${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}/${generated_module_file}")