Issue
I have a C++ library built using CMake, and it uses data pulled from https://www.dnd5eapi.co/. To do this, I have a Python script that runs and pulls the data using requests.
import os
import json
import requests
if __name__ == "__main__":
pulls = ["backgrounds",
"classes",
"equipment-categories/armor",
"equipment-categories/weapon",
"races",
"subclasses"]
gets = {}
for item in pulls:
with requests.get("https://www.dnd5eapi.co/api/" + item) as jsonurl:
gets[item] = jsonurl.json()
for item in gets:
if not os.path.exists(item):
os.makedirs(item)
with open(item + "/" + item.split("/")[-1] + ".json", "w") as super_f:
super_f.write(json.dumps(gets[item]))
for it in gets[item]["equipment" if "equipment" in gets[item] else "results"]:
if "url" in it:
with requests.get("https://www.dnd5eapi.co" + it["url"]) as jsonurl:
with open(item + "/" + jsonurl.json()["index"] + ".json", "w") as f:
f.write(json.dumps(jsonurl.json()))
I already have confirmed it runs and grabs the data on more than one machine. To test my library, I use the GitHub CMake Workflow to automatically build and test the code. If the option is specified, I run the Python code from my CMakeLists.txt.
option(PULL_DATA "Pull the API data from 5e API" OFF)
if (PULL_DATA)
find_package(Python REQUIRED)
if (EXISTS "${CMAKE_SOURCE_DIR}/data/apipull.py")
message("-- Found API Script: ${CMAKE_SOURCE_DIR}/data/apipull.py")
execute_process(COMMAND ${Python_EXECUTABLE} apipull.py
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/data/
RESULT_VARIABLE outt
)
if(NOT ${outt} EQUAL 0)
message(FATAL_ERROR "-- API Data script did not complete successfully!")
endif ()
message("-- Script output: ${outt}")
message("-- API Script Finished")
else ()
message(FATAL_ERROR "-- Cannot find script to pull API data!")
endif ()
execute_process(COMMAND pip show requests
RESULT_VARIABLE ecode
OUTPUT_QUIET
)
if (NOT ${ecode} EQUAL 0)
message(FATAL_ERROR "-- Cannot find Python module \"requests\". Please install the requirements.txt or set INSTALL_REQ to true.")
endif ()
message("-- Found required Python modules")
endif ()
This works on my machine, and on the ubuntu-latest OS with GitHub Actions. The problem begins when I try to build on macos-latest as requests isn't automatically installed. To solve this, I set the Python version and installed any dependencies using the cmake.yml. The relevant portion is:
name: CMake
on: [push]
env:
# Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
BUILD_TYPE: Release
jobs:
build:
# The CMake configure and build commands are platform agnostic and should work equally
# well on Windows or Mac. You can convert this to a matrix build if you need
# cross-platform coverage.
# See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
python-version: [3.8]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
# Allows us to test a wide range of Python versions
# This is to ensure full spectrum testing
uses: actions/setup-python@v2
- name: Install dependencies
# This installs our dependencies so it can automatically be built in testing
# Normally the user would run Python to ensure the data exists
run: |
python -m pip install --upgrade pip
python -m pip install requests
I've also set it to 3.5-3.8 but nothing works. I've tried different configurations of using pip but nothing happens. I can confirm by viewing the logs that requests is installed
Requirement already satisfied: pip in /Users/runner/hostedtoolcache/Python/3.9.4/x64/lib/python3.9/site-packages (21.0.1)
Collecting pip
Downloading pip-21.1-py3-none-any.whl (1.5 MB)
Installing collected packages: pip
Attempting uninstall: pip
Found existing installation: pip 21.0.1
Uninstalling pip-21.0.1:
Successfully uninstalled pip-21.0.1
Successfully installed pip-21.1
WARNING: Value for scheme.headers does not match. Please report this to <https://github.com/pypa/pip/issues/9617>
distutils: /Users/runner/hostedtoolcache/Python/3.9.4/x64/include/python3.9/UNKNOWN
sysconfig: /Users/runner/hostedtoolcache/Python/3.9.4/x64/include/python3.9
WARNING: Additional context:
user = False
home = None
root = None
prefix = None
Collecting requests
Downloading requests-2.25.1-py2.py3-none-any.whl (61 kB)
Collecting chardet<5,>=3.0.2
Downloading chardet-4.0.0-py2.py3-none-any.whl (178 kB)
Collecting urllib3<1.27,>=1.21.1
Downloading urllib3-1.26.4-py2.py3-none-any.whl (153 kB)
Collecting idna<3,>=2.5
Downloading idna-2.10-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.12.5-py2.py3-none-any.whl (147 kB)
Installing collected packages: urllib3, idna, chardet, certifi, requests
WARNING: Value for scheme.headers does not match. Please report this to <https://github.com/pypa/pip/issues/9617>
distutils: /Users/runner/hostedtoolcache/Python/3.9.4/x64/include/python3.9/UNKNOWN
sysconfig: /Users/runner/hostedtoolcache/Python/3.9.4/x64/include/python3.9
WARNING: Additional context:
user = False
home = None
root = None
prefix = None
Successfully installed certifi-2020.12.5 chardet-4.0.0 idna-2.10 requests-2.25.1 urllib3-1.26.4
but I still get
-- The CXX compiler identification is AppleClang 12.0.0.12000032
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Found Python: /usr/local/Frameworks/Python.framework/Versions/3.9/bin/python3.9 (found version "3.9.4") found components: Interpreter
-- Found API Script: /Users/runner/work/libeztp/libeztp/data/apipull.py
Traceback (most recent call last):
File "/Users/runner/work/libeztp/libeztp/data/apipull.py", line 3, in <module>
import requests
ModuleNotFoundError: No module named 'requests'
CMake Error at CMakeLists.txt:51 (message):
-- API Data script did not complete successfully!
-- Configuring incomplete, errors occurred!
See also "/Users/runner/work/libeztp/libeztp/build/CMakeFiles/CMakeOutput.log".
Error: Process completed with exit code 1.
everytime. Does anyone have any help they can offer as to why this is happening? I'm legitimately confused as I see zero reason why the module isn't being imported.
Solution
It turns out that on MacOS with GitHub Actions, there are multiple installed python interpreters and CMake wasn't finding the system version. I discovered this by adding python -m pip list to my cmake.yml before running the configuration for CMake. This gave me the Python location that GitHub Actions was using: /Users/runner/hostedtoolcache/Python/3.9.4/x64, but CMake was finding an interpreter at /usr/local/Frameworks/Python.framework/Versions/3.9/bin/python3.9. So I changed the CMakeLists.txt from
execute_process(COMMAND ${Python_EXECUTABLE} apipull.py
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/data/
RESULT_VARIABLE outt
)
to
execute_process(COMMAND python apipull.py
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/data/
RESULT_VARIABLE outt
)
Everything runs perfectly now!
Answered By - Gabe Ron
