Python 3.12 support

lief-project · Oct 28, 2023 · 29f7e9d · 29f7e9d
1 parent 00a0015
commit 29f7e9d
Show file tree

Hide file tree

Showing 26 changed files with 232 additions and 244 deletions.
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
@@ -36,7 +36,7 @@ jobs:
            -e CCACHE_DIR=/ccache      \
            -v $GITHUB_WORKSPACE:/work \
            -v $HOME/.ccache:/ccache   \
-          --rm liefproject/android-${ARCH} bash /work/scripts/docker/android-${ARCH}.sh
+          --rm liefproject/android-${ARCH} bash /work/scripts/docker/android-sdk-${ARCH}
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/linux-aarch64.yml b/.github/workflows/linux-aarch64.yml
@@ -24,14 +24,19 @@ jobs:
         shell: bash
         run: |
           mkdir -p $RUNNER_TEMP/lief-install
-          docker run --name lief_sdk --rm         \
-            -e CCACHE_COMPRESS=1                  \
-            -e CCACHE_MAXSIZE=10.0G               \
-            -e CCACHE_DIR=/ccache                 \
-            -v $GITHUB_WORKSPACE:/work            \
-            -v $RUNNER_TEMP/lief-install:/install \
-            -v $HOME/.ccache:/ccache              \
-            liefproject/manylinux2014-aarch64 bash /work/scripts/docker/linux-aarch64.sh
+          mkdir -p $HOME/.ccache
+
+          chmod -R 777 $HOME/.ccache
+
+          docker run --name lief-sdk -u 1001:1001 --rm \
+            -e CCACHE_COMPRESS=1                       \
+            -e CCACHE_MAXSIZE=10.0G                    \
+            -e CCACHE_DIR=/ccache                      \
+            -v $GITHUB_WORKSPACE:/src                  \
+            -v $RUNNER_TEMP/lief-install:/install      \
+            -v $HOME/.ccache:/ccache                   \
+            liefproject/manylinux2014-aarch64          \
+            bash /src/scripts/docker/linux-sdk-aarch64
       - name: 'Upload SDK Artifact'
         uses: actions/upload-artifact@v3
         with:
@@ -49,7 +54,7 @@ jobs:
     needs: linux-sdk
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11']
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
     steps:
       - uses: actions/checkout@v3
         with:
@@ -76,18 +81,23 @@ jobs:
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
+          mkdir -p $HOME/.ccache
+          chmod -R 777 $HOME/.ccache
+
           PYTHON_VERSION_ALT=$(echo "${PYTHON_VERSION}" | sed 's/\.//') # Transform 3.8 -> 38
-          docker run --rm                                       \
-           -e CCACHE_COMPRESS=1                                 \
-           -e CCACHE_MAXSIZE=10.0G                              \
-           -e CCACHE_CPP2=1                                     \
-           -e CCACHE_DIR=/ccache                                \
-           -e PYTHON_VERSION=$PYTHON_VERSION_ALT                \
-           -e PYTHON_BINARY=/opt/python/cp${PYTHON_VERSION_ALT}-cp${PYTHON_VERSION_ALT}/bin/python${PYTHON_VERSION} \
-           -v $GITHUB_WORKSPACE:/work                           \
-           -v $HOME/.ccache:/ccache                             \
-           -v $RUNNER_TEMP/lief-install:/install                \
-           liefproject/manylinux2014-aarch64 bash /work/scripts/docker/manylinux2014-aarch64.sh
+          PYTHON_BINARY="/opt/python/cp${PYTHON_VERSION_ALT}-cp${PYTHON_VERSION_ALT}/bin/python${PYTHON_VERSION}"
+          docker run -u 1001:1001 --rm           \
+           -e CCACHE_COMPRESS=1                  \
+           -e CCACHE_MAXSIZE=10.0G               \
+           -e CCACHE_CPP2=1                      \
+           -e CCACHE_DIR=/ccache                 \
+           -e PYTHON_VERSION=$PYTHON_VERSION_ALT \
+           -e PYTHON_BINARY=$PYTHON_BINARY       \
+           -v $GITHUB_WORKSPACE:/src             \
+           -v $HOME/.ccache:/ccache              \
+           -v $RUNNER_TEMP/lief-install:/install \
+           liefproject/manylinux2014-aarch64     \
+           bash /src/scripts/docker/linux-py-aarch64
       - name: 'Artifact'
         uses: actions/upload-artifact@v3
         with:
@@ -124,6 +134,7 @@ jobs:
             -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY           \
             -e LIEF_S3_KEY=$LIEF_S3_KEY                       \
             -e LIEF_S3_SECRET=$LIEF_S3_SECRET                 \
-            liefproject/deploy python3 /src/.github/deploy.py /runner_tmp/artifacts
+            liefproject/deploy                                \
+            python3 /src/.github/deploy.py /runner_tmp/artifacts
 
 
diff --git a/.github/workflows/linux-x86-64.yml b/.github/workflows/linux-x86-64.yml
@@ -24,14 +24,18 @@ jobs:
         shell: bash
         run: |
           mkdir -p $RUNNER_TEMP/lief-install
-          docker run --name lief_sdk --rm                        \
-            -e CCACHE_COMPRESS=1                                 \
-            -e CCACHE_MAXSIZE=10.0G                              \
-            -e CCACHE_DIR=/ccache                                \
-            -v $GITHUB_WORKSPACE:/src                            \
-            -v $RUNNER_TEMP/lief-install:/install                       \
-            -v $HOME/.ccache:/ccache                             \
-            liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux_sdk.sh
+          mkdir -p $HOME/.ccache
+          chmod -R 777 $HOME/.ccache
+
+          docker run --name lief-sdk -u 1001:1001 --rm \
+            -e CCACHE_COMPRESS=1                       \
+            -e CCACHE_MAXSIZE=10.0G                    \
+            -e CCACHE_DIR=/ccache                      \
+            -v $GITHUB_WORKSPACE:/src                  \
+            -v $RUNNER_TEMP/lief-install:/install      \
+            -v $HOME/.ccache:/ccache                   \
+            liefproject/manylinux_2_28_x86_64          \
+            bash /src/scripts/docker/linux-sdk-x64
       - name: 'Upload SDK Artifact'
         uses: actions/upload-artifact@v3
         with:
@@ -67,23 +71,28 @@ jobs:
         shell: bash
         env:
           PYTHON_VERSION: 38
+          PYTHON_BINARY: "/opt/python/cp38-cp38/bin/python3.8"
         run: |
-          docker run --rm                                       \
-           -e CCACHE_COMPRESS=1                                 \
-           -e CCACHE_MAXSIZE=10.0G                              \
-           -e CCACHE_CPP2=1                                     \
-           -e CCACHE_DIR=/ccache                                \
-           -e PYTHON_VERSION=$PYTHON_VERSION                    \
-           -e PYTHON_BINARY=/opt/python/cp38-cp38/bin/python3.8 \
-           -v $GITHUB_WORKSPACE:/src                            \
-           -v $HOME/.ccache:/ccache                             \
-           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux_test.sh
+          mkdir -p $HOME/.ccache
+          chmod -R 777 $HOME/.ccache
+
+          docker run --rm -u 1001:1001              \
+           -e CCACHE_COMPRESS=1                     \
+           -e CCACHE_MAXSIZE=10.0G                  \
+           -e CCACHE_CPP2=1                         \
+           -e CCACHE_DIR=/ccache                    \
+           -e PYTHON_VERSION=$PYTHON_VERSION        \
+           -e PYTHON_BINARY=$PYTHON_BINARY          \
+           -v $GITHUB_WORKSPACE:/src                \
+           -v $HOME/.ccache:/ccache                 \
+           liefproject/manylinux_2_28_x86_64        \
+           bash /src/scripts/docker/test-linux-x64
   pywheels:
     runs-on: ubuntu-latest
     needs: linux-sdk
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
     steps:
       - uses: actions/checkout@v3
         with:
@@ -111,17 +120,23 @@ jobs:
           PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
           PYTHON_VERSION_ALT=$(echo "${PYTHON_VERSION}" | sed 's/\.//') # Transform 3.8 -> 38
-          docker run --rm                                       \
-           -e CCACHE_COMPRESS=1                                 \
-           -e CCACHE_MAXSIZE=10.0G                              \
-           -e CCACHE_CPP2=1                                     \
-           -e CCACHE_DIR=/ccache                                \
-           -e PYTHON_VERSION=$PYTHON_VERSION_ALT                    \
-           -e PYTHON_BINARY=/opt/python/cp${PYTHON_VERSION_ALT}-cp${PYTHON_VERSION_ALT}/bin/python${PYTHON_VERSION} \
-           -v $GITHUB_WORKSPACE:/src                            \
-           -v $HOME/.ccache:/ccache                             \
-           -v $RUNNER_TEMP/lief-install:/install                       \
-           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux.sh
+          PYTHON_BINARY=/opt/python/cp${PYTHON_VERSION_ALT}-cp${PYTHON_VERSION_ALT}/bin/python${PYTHON_VERSION}
+
+          mkdir -p $HOME/.ccache
+          chmod -R 777 $HOME/.ccache
+
+          docker run --rm -u 1001:1001           \
+           -e CCACHE_COMPRESS=1                  \
+           -e CCACHE_MAXSIZE=10.0G               \
+           -e CCACHE_CPP2=1                      \
+           -e CCACHE_DIR=/ccache                 \
+           -e PYTHON_VERSION=$PYTHON_VERSION_ALT \
+           -e PYTHON_BINARY=$PYTHON_BINARY       \
+           -v $GITHUB_WORKSPACE:/src             \
+           -v $HOME/.ccache:/ccache              \
+           -v $RUNNER_TEMP/lief-install:/install \
+           liefproject/manylinux_2_28_x86_64     \
+           bash /src/scripts/docker/linux-py-x64
       - name: 'Artifact'
         uses: actions/upload-artifact@v3
         with:
@@ -158,7 +173,8 @@ jobs:
             -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY           \
             -e LIEF_S3_KEY=$LIEF_S3_KEY                       \
             -e LIEF_S3_SECRET=$LIEF_S3_SECRET                 \
-            liefproject/deploy python3 /src/.github/deploy.py /runner_tmp/artifacts
+            liefproject/deploy                                \
+            python3 /src/.github/deploy.py /runner_tmp/artifacts
   documentation:
     needs: [deploy]
     runs-on: ubuntu-latest

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
@@ -119,7 +119,7 @@ jobs:
     needs: osx-sdk
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11']
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
         arch: ['x64', 'arm64']
       fail-fast: false
     steps:

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -50,7 +50,7 @@ jobs:
            -e PYTHON_VERSION=$PYTHON_VERSION \
            -e PYTHON_BINARY=/opt/python/cp39-cp39/bin/python3.9 \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux_test.sh
+           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/test-linux-x64
   windows:
     runs-on: windows-latest
     steps:

diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml
@@ -15,7 +15,7 @@ jobs:
            -e BUILD_DIR=/tmp/build_gcc \
            -e SRC_DIR=/src \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/sanity:latest bash /src/scripts/docker/gcc.sh
+           liefproject/sanity:latest bash /src/scripts/docker/test-gcc
   clang:
     runs-on: ubuntu-latest
     steps:
@@ -29,7 +29,7 @@ jobs:
            -e BUILD_DIR=/tmp/build_clang \
            -e SRC_DIR=/src \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/sanity:latest bash /src/scripts/docker/clang.sh
+           liefproject/sanity:latest bash /src/scripts/docker/test-clang
   asan:
     runs-on: ubuntu-latest
     steps:
@@ -41,4 +41,4 @@ jobs:
         run: |
           docker run --rm \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/sanity:latest bash /src/scripts/docker/asan_check.sh
+           liefproject/sanity:latest bash /src/scripts/docker/test-asan
diff --git a/.github/workflows/windows-all.yml b/.github/workflows/windows-all.yml
@@ -13,7 +13,7 @@ env:
 
 jobs:
   build-sdk:
-    runs-on: windows-2019
+    runs-on: windows-2022
     strategy:
       matrix:
         arch: ['x64', 'x86']
@@ -36,7 +36,7 @@ jobs:
           choco install sccache
       - uses: ilammy/msvc-dev-cmd@v1
         with:
-          toolset: "14.16"
+          vsversion: "2022"
       - name: Get Date
         id: get-date
         run: |
@@ -96,7 +96,7 @@ jobs:
           choco install sccache
       - uses: ilammy/msvc-dev-cmd@v1
         with:
-          toolset: "14.16"
+          vsversion: "2022"
       - name: Get Date
         id: get-date
         run: |
@@ -150,7 +150,7 @@ jobs:
     needs: build-sdk
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11', '3.12-dev']
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
         arch: ['x86', 'x64']
       fail-fast: false
     env:
@@ -172,7 +172,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel
       - uses: ilammy/msvc-dev-cmd@v1
         with:
-          toolset: "14.16"
+          vsversion: "2022"
       - name: Get Date
         id: get-date
         run: |

diff --git a/.github/workflows/windows-quick.yml b/.github/workflows/windows-quick.yml
@@ -9,11 +9,11 @@ on:
       - "**"
 
 env:
-  python-version: '3.9'
+  python-version: '3.12'
 
 jobs:
   quick-test:
-    runs-on: windows-2019
+    runs-on: windows-2022
     strategy:
       matrix:
         arch: [x64, x86]
@@ -37,7 +37,7 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
     - uses: ilammy/msvc-dev-cmd@v1
       with:
-        toolset: "14.16"
+        vsversion: "2022"
     - name: Get Date
       id: get-date
       run: |

diff --git a/api/python/setup.py b/api/python/setup.py
@@ -85,7 +85,7 @@ def get_lief_logging_opt(self):
     def get_python_opt(self):
         config = ["-DLIEF_PYTHON_API=on"]
         interpreter = Path(sys.executable)
-        base = sysconfig.get_config_var("base")
+        base = sysconfig.get_config_var("installed_base")
         if base is not None:
             config += [f"-DPython_ROOT_DIR={base}"]
 
@@ -377,7 +377,9 @@ def __init__(self, *args, **kwargs):
     def run(self):
         for ext in self.extensions:
             self.build_extension(ext)
-        self.copy_extensions_to_source()
+
+        if self.inplace:
+            self.copy_extensions_to_source()
 
     def _fix_platform(self):
         if sys.platform == "darwin":
@@ -491,9 +493,8 @@ class egg_info(_egg_info):
     long_description=long_description.read_text(),
     long_description_content_type="text/x-rst; charset=UTF-8",
     distclass=LiefDistribution,
-    scripts=['examples/elf_reader.py', 'examples/pe_reader.py', 'examples/macho_reader.py'],
     packages=["lief"],
-    package_data={"lief": ["py.typed", "*.pyi", "*.so", "*.pyd"]},
+    package_data={"lief": ["py.typed", "*.pyi", "*.pyd"]},
     ext_modules=[Module("lief._lief")],
     cmdclass=cmdclass,
     version=version

diff --git a/scripts/docker/android-arm.sh → scripts/docker/android-sdk-arm b/scripts/docker/android-arm.sh → scripts/docker/android-sdk-arm
@@ -11,27 +11,27 @@ git config --global --add safe.directory /work
 mkdir -p build/$ARCH_DIR/static-release && mkdir -p build/$ARCH_DIR/shared-release
 pushd build/$ARCH_DIR/shared-release
 
-cmake ../../.. -GNinja \
-  -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
-  -DCMAKE_C_FLAGS="$CFLAGS" \
+cmake ../../.. -GNinja                            \
+  -DCMAKE_CXX_FLAGS="$CXXFLAGS"                   \
+  -DCMAKE_C_FLAGS="$CFLAGS"                       \
   -DCMAKE_SHARED_LINKER_FLAGS="-static-libstdc++" \
-  -DCMAKE_LINK_WHAT_YOU_USE=on \
-  -DBUILD_SHARED_LIBS=on \
-  -DLIEF_PYTHON_API=off \
-  -DLIEF_INSTALL_COMPILED_EXAMPLES=off \
+  -DCMAKE_LINK_WHAT_YOU_USE=on                    \
+  -DBUILD_SHARED_LIBS=on                          \
+  -DLIEF_PYTHON_API=off                           \
+  -DLIEF_INSTALL_COMPILED_EXAMPLES=off            \
   -DCMAKE_BUILD_TYPE=Release
 
 ninja
 
 popd
 pushd build/$ARCH_DIR/static-release
 
-cmake ../../.. -GNinja \
-  -DCMAKE_LINK_WHAT_YOU_USE=on \
-  -DBUILD_SHARED_LIBS=off \
+cmake ../../.. -GNinja                         \
+  -DCMAKE_LINK_WHAT_YOU_USE=on                 \
+  -DBUILD_SHARED_LIBS=off                      \
   -DCMAKE_EXE_LINKER_FLAGS="-static-libstdc++" \
-  -DLIEF_PYTHON_API=off \
-  -DLIEF_INSTALL_COMPILED_EXAMPLES=on \
+  -DLIEF_PYTHON_API=off                        \
+  -DLIEF_INSTALL_COMPILED_EXAMPLES=on          \
   -DCMAKE_BUILD_TYPE=Release
 
 ninja