From e04b34b437a47390b7cd45dda53ab016a58f618b Mon Sep 17 00:00:00 2001 From: Swapnil Patel Date: Thu, 9 Jan 2025 22:20:03 -0800 Subject: [PATCH 1/6] Add llvmdev and llvmlite build GHA workflows (#8) * add llvmdev gha workflow * update build recipe and script * add llvmlite build workflow * update artifact naming * update llvmlite build using conda recipe * update llvmlite build using conda recipe * update llvmlite build using conda recipe * update llvmlite build using conda recipe * update llvmlite build using conda recipe * add libllvm15 to recipe meta.yaml * remove libllvm15 to recipe meta.yaml * switch to using llvmdev from numba ci channel * update llvmlite build * update llvmlite build * update llvmlite build * update llvmlite build * update llvmlite build * update llvmlite build * build with setuptools * remove llvmdev * add llvmdev * add win builds * add win builds * update win builds * update win builds * Add macOS build workflow for multiple Python versions --- .github/workflows/llvmdev_build.yml | 65 ++++++ .github/workflows/llvmlite_build.yml | 195 ++++++++++++++++++ buildscripts/manylinux/docker_run.sh | 2 +- conda-recipes/llvmdev_manylinux/bld.bat | 59 ++++++ .../llvmdev_manylinux/conda_build_config.yaml | 20 ++ conda-recipes/llvmdev_manylinux/meta.yaml | 34 +-- 6 files changed, 360 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/llvmdev_build.yml create mode 100644 .github/workflows/llvmlite_build.yml create mode 100644 conda-recipes/llvmdev_manylinux/bld.bat create mode 100644 conda-recipes/llvmdev_manylinux/conda_build_config.yaml diff --git a/.github/workflows/llvmdev_build.yml b/.github/workflows/llvmdev_build.yml new file mode 100644 index 000000000..944606a7d --- /dev/null +++ b/.github/workflows/llvmdev_build.yml @@ -0,0 +1,65 @@ +name: llvmdev-build + +on: + workflow_dispatch: + +jobs: + linux-64: + name: Build LLVMDEV (linux-64) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Build LLVMDEV + run: | + mkdir -p /home/runner/work/llvmlite/llvmlite/docker_output + chmod 777 /home/runner/work/llvmlite/llvmlite/docker_output + cd buildscripts/manylinux + ./docker_run_x64.sh build_llvmdev.sh "${MINICONDA_FILE}" + RECIPE_NAME=./conda-recipes/llvmdev_manylinux + echo "OUTPUT=$(conda build --output $RECIPE_NAME)" >> $GITHUB_ENV + + - name: Upload LLVMDEV artifacts + uses: actions/upload-artifact@v4 + with: + name: llvmdev-linux-x86_64 + path: /home/runner/work/llvmlite/llvmlite/docker_output/linux-64/*.tar.bz2 + retention-days: 1 + if-no-files-found: error + + win-64: + name: Build LLVMDEV (win-64) + runs-on: windows-2019 + steps: + - uses: actions/checkout@v4 + + - name: Install Miniconda + shell: pwsh + run: | + $wc = New-Object net.webclient + $wc.Downloadfile("https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe", "Miniconda3-latest-Windows-x86_64.exe") + Start-Process "Miniconda3-latest-Windows-x86_64.exe" "/S /D=C:\Miniconda3" -Wait + + - name: Setup Conda Environment + shell: bash + run: | + source /c/Miniconda3/Scripts/activate + conda create -n builder conda-build -y + + - name: Build + shell: bash + run: | + source /c/Miniconda3/Scripts/activate + conda activate builder + RECIPE_NAME=./conda-recipes/llvmdev_manylinux + conda build $RECIPE_NAME + echo "OUTPUT=$(conda build --output $RECIPE_NAME)" >> $GITHUB_ENV + + - name: Upload LLVMDEV artifacts + uses: actions/upload-artifact@v4 + with: + name: llvmdev-windows-x86_64 + path: ${{ env.OUTPUT }} + retention-days: 1 + if-no-files-found: error + diff --git a/.github/workflows/llvmlite_build.yml b/.github/workflows/llvmlite_build.yml new file mode 100644 index 000000000..183e1944a --- /dev/null +++ b/.github/workflows/llvmlite_build.yml @@ -0,0 +1,195 @@ +name: llvmlite-build + +on: + workflow_dispatch: + +jobs: + linux-build: + name: build_linux_64_py${{ matrix.python-version }}_npy1.11 + runs-on: ubuntu-24.04 + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Install Miniconda + run: | + bash buildscripts/incremental/install_miniconda.sh + + - name: Add Conda to PATH + run: | + echo "$HOME/miniconda3/bin" >> $GITHUB_PATH + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda init bash + + - name: Create Build Environment + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda create -n build_env \ + python=${{ matrix.python-version }} \ + setuptools \ + wheel \ + -y + conda activate build_env + conda install -y numba/label/ci::llvmdev --no-deps + + - name: build_linux_64_py${{ matrix.python-version }}_npy1.11 + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda activate build_env + export LLVMLITE_CXX_STATIC_LINK=1 + export LLVM_CONFIG="$CONDA_PREFIX/bin/llvm-config" + export LLVMLITE_SHARED=1 + + python setup.py clean + python setup.py bdist_wheel + + - name: test_linux_64_py${{ matrix.python-version }}_npyNone + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda activate build_env + python -m pip install dist/*.whl + python -m llvmlite.tests + + - name: upload_linux_64_py${{ matrix.python-version }} + uses: actions/upload-artifact@v4 + with: + name: llvmlite-linux-64-py${{ matrix.python-version }} + path: dist/*.whl + retention-days: 7 + if-no-files-found: error + + windows-build: + name: build_win_64_py${{ matrix.python-version }}_npy1.11 + runs-on: windows-2019 + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Install Visual Studio + uses: microsoft/setup-msbuild@v1.1 + + - name: Install Miniconda + shell: pwsh + run: | + $wc = New-Object net.webclient + $wc.Downloadfile("https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe", "Miniconda3-latest-Windows-x86_64.exe") + Start-Process "Miniconda3-latest-Windows-x86_64.exe" "/S /D=C:\Miniconda3" -Wait + + - name: Create Build Environment + shell: bash + run: | + source /c/Miniconda3/Scripts/activate + conda create -n build_env \ + python=${{ matrix.python-version }} \ + setuptools \ + wheel \ + cmake \ + libxml2 \ + -y + conda activate build_env + # Install LLVM with all dependencies + conda install -y -c numba/label/ci llvmdev + + - name: Set Environment Variables + shell: bash + run: | + echo "CMAKE_GENERATOR=Visual Studio 16 2019" >> $GITHUB_ENV + echo "CMAKE_GENERATOR_PLATFORM=x64" >> $GITHUB_ENV + echo "LLVM_CONFIG=C:/Miniconda3/envs/build_env/Library/bin/llvm-config" >> $GITHUB_ENV + echo "LLVM_DIR=C:/Miniconda3/envs/build_env/Library/lib/cmake/llvm" >> $GITHUB_ENV + echo "LLVM_SHARED=1" >> $GITHUB_ENV + echo "LIBXML2_ROOT=C:/Miniconda3/envs/build_env/Library" >> $GITHUB_ENV + + - name: build_win_64_py${{ matrix.python-version }}_npy1.11 + shell: cmd + run: | + call C:\Miniconda3\Scripts\activate.bat build_env + set DISTUTILS_USE_SDK=1 + set VS_VERSION=2019 + set VS_PLATFORM=x64 + python setup.py clean + python setup.py bdist_wheel + + - name: test_win_64_py${{ matrix.python-version }}_npyNone + shell: cmd + run: | + call C:\Miniconda3\Scripts\activate.bat build_env + python -m pip install dist\*.whl + python -m llvmlite.tests + + - name: upload_win_64_py${{ matrix.python-version }} + uses: actions/upload-artifact@v4 + with: + name: llvmlite-win-64-py${{ matrix.python-version }} + path: dist/*.whl + retention-days: 7 + if-no-files-found: error + + macos-build: + name: build_osx_64_py${{ matrix.python-version }}_npy1.11 + runs-on: macos-13 + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Install Miniconda + run: | + bash buildscripts/incremental/install_miniconda.sh + + - name: Add Conda to PATH + run: | + echo "$HOME/miniconda3/bin" >> $GITHUB_PATH + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda init bash + + - name: Create Build Environment + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda create -n build_env \ + python=${{ matrix.python-version }} \ + setuptools \ + wheel \ + -y + conda activate build_env + # Install LLVM with all dependencies for macOS + conda install -y -c numba/label/ci llvmdev + + - name: build_osx_64_py${{ matrix.python-version }}_npy1.11 + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda activate build_env + export MACOSX_DEPLOYMENT_TARGET=10.9 + export CONDA_BUILD_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk + export LLVM_CONFIG="$CONDA_PREFIX/bin/llvm-config" + export LLVMLITE_SHARED=1 + + python setup.py clean + python setup.py bdist_wheel + + - name: test_osx_64_py${{ matrix.python-version }}_npyNone + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda activate build_env + python -m pip install dist/*.whl + python -m llvmlite.tests + + - name: upload_osx_64_py${{ matrix.python-version }} + uses: actions/upload-artifact@v4 + with: + name: llvmlite-osx-64-py${{ matrix.python-version }} + path: dist/*.whl + retention-days: 7 + if-no-files-found: error + diff --git a/buildscripts/manylinux/docker_run.sh b/buildscripts/manylinux/docker_run.sh index e9f81f933..a7c8c16b6 100755 --- a/buildscripts/manylinux/docker_run.sh +++ b/buildscripts/manylinux/docker_run.sh @@ -18,4 +18,4 @@ echo "MINICONDA_FILE=$MINICONDA_FILE" # Ensure the latest docker image IMAGE_URI="quay.io/pypa/${MANYLINUX_IMAGE}:latest" docker pull $IMAGE_URI -docker run --rm -it -v $SRCDIR:/root/llvmlite $IMAGE_URI ${PRECMD} /root/llvmlite/buildscripts/manylinux/$1 ${MINICONDA_FILE} $2 +docker run --rm -t -v $SRCDIR:/root/llvmlite $IMAGE_URI ${PRECMD} /root/llvmlite/buildscripts/manylinux/$1 ${MINICONDA_FILE} $2 diff --git a/conda-recipes/llvmdev_manylinux/bld.bat b/conda-recipes/llvmdev_manylinux/bld.bat new file mode 100644 index 000000000..fbb04c166 --- /dev/null +++ b/conda-recipes/llvmdev_manylinux/bld.bat @@ -0,0 +1,59 @@ +REM base on https://github.com/AnacondaRecipes/llvmdev-feedstock/blob/master/recipe/bld.bat +echo on + +mkdir build +cd build + +REM remove GL flag for now +set "CXXFLAGS=-MD" +set "CC=cl.exe" +set "CXX=cl.exe" + +cmake -G "Ninja" ^ + -DCMAKE_BUILD_TYPE="Release" ^ + -DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX% ^ + -DCMAKE_INSTALL_PREFIX:PATH=%LIBRARY_PREFIX% ^ + -DLLVM_USE_INTEL_JITEVENTS=ON ^ + -DLLVM_ENABLE_LIBXML2=OFF ^ + -DLLVM_ENABLE_RTTI=ON ^ + -DLLVM_ENABLE_ZLIB=OFF ^ + -DLLVM_ENABLE_ZSTD=OFF ^ + -DLLVM_INCLUDE_BENCHMARKS=OFF ^ + -DLLVM_INCLUDE_DOCS=OFF ^ + -DLLVM_INCLUDE_EXAMPLES=OFF ^ + -DLLVM_INCLUDE_TESTS=ON ^ + -DLLVM_INCLUDE_UTILS=ON ^ + -DLLVM_INSTALL_UTILS=ON ^ + -DLLVM_UTILS_INSTALL_DIR=libexec\llvm ^ + -DLLVM_BUILD_LLVM_C_DYLIB=no ^ + -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly ^ + -DCMAKE_POLICY_DEFAULT_CMP0111=NEW ^ + -DLLVM_ENABLE_PROJECTS:STRING=lld;compiler-rt ^ + -DLLVM_ENABLE_ASSERTIONS=ON ^ + -DLLVM_ENABLE_DIA_SDK=OFF ^ + -DCOMPILER_RT_BUILD_BUILTINS=ON ^ + -DCOMPILER_RT_BUILTINS_HIDE_SYMBOLS=OFF ^ + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF ^ + -DCOMPILER_RT_BUILD_CRT=OFF ^ + -DCOMPILER_RT_BUILD_MEMPROF=OFF ^ + -DCOMPILER_RT_BUILD_PROFILE=OFF ^ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF ^ + -DCOMPILER_RT_BUILD_XRAY=OFF ^ + -DCOMPILER_RT_BUILD_GWP_ASAN=OFF ^ + -DCOMPILER_RT_BUILD_ORC=OFF ^ + -DCOMPILER_RT_INCLUDE_TESTS=OFF ^ + %SRC_DIR%/llvm +if %ERRORLEVEL% neq 0 exit 1 + +cmake --build . +if %ERRORLEVEL% neq 0 exit 1 + +cmake --build . --target install + +if %ERRORLEVEL% neq 0 exit 1 + +REM bin\opt -S -vector-library=SVML -mcpu=haswell -O3 %RECIPE_DIR%\numba-3016.ll | bin\FileCheck %RECIPE_DIR%\numba-3016.ll +REM if %ERRORLEVEL% neq 0 exit 1 + +cd ..\llvm\test +python ..\..\build\bin\llvm-lit.py -vv Transforms ExecutionEngine Analysis CodeGen/X86 diff --git a/conda-recipes/llvmdev_manylinux/conda_build_config.yaml b/conda-recipes/llvmdev_manylinux/conda_build_config.yaml new file mode 100644 index 000000000..078468e31 --- /dev/null +++ b/conda-recipes/llvmdev_manylinux/conda_build_config.yaml @@ -0,0 +1,20 @@ +# # Numba/llvmlite stack needs an older compiler for backwards compatability. +# c_compiler_version: # [linux] +# - 7 # [linux and (x86_64 or ppc64le)] +# - 11 # [linux and aarch64] + +# cxx_compiler_version: # [linux] +# - 7 # [linux and (x86_64 or ppc64le)] +# - 11 # [linux and aarch64] + +# fortran_compiler_version: # [linux] +# - 7 # [linux and (x86_64 or ppc64le)] +# - 11 # [linux and aarch64] + +c_compiler: # [win] + - vs2019 # [win] +cxx_compiler: # [win] + - vs2019 # [win] + +MACOSX_SDK_VERSION: # [osx and x86_64] + - 10.12 # [osx and x86_64] diff --git a/conda-recipes/llvmdev_manylinux/meta.yaml b/conda-recipes/llvmdev_manylinux/meta.yaml index cc7868d4c..2144bd950 100644 --- a/conda-recipes/llvmdev_manylinux/meta.yaml +++ b/conda-recipes/llvmdev_manylinux/meta.yaml @@ -2,7 +2,7 @@ {% set shortversion = "15.0" %} {% set version = "15.0.7" %} {% set sha256_llvm = "8b5fcb24b4128cf04df1b0b9410ce8b1a729cb3c544e6da885d234280dedeac6" %} -{% set build_number = "0" %} +{% set build_number = "1" %} package: name: llvmdev @@ -17,13 +17,18 @@ source: - ../llvm15-svml.patch - ../compiler-rt-cfi-startproc-war.patch - ../compiler-rt-macos-build.patch + # Patches from conda-forge needed for windows to build + # backport of zlib patches, can be dropped for vs15.0.3, see + # https://reviews.llvm.org/D135457 & https://reviews.llvm.org/D136065 + - ../llvmdev/patches/0002-CMake-Fix-Findzstd-module-for-shared-DLL-on-Windows.patch + - ../llvmdev/patches/no-windows-symlinks.patch build: number: {{ build_number }} string: "manylinux" script_env: - - CFLAGS - - CXXFLAGS + - CFLAGS [linux] + - CXXFLAGS [linux] - PY_VCRUNTIME_REDIST ignore_run_exports: # Is static-linked @@ -31,32 +36,33 @@ build: requirements: build: - # Do not use the compiler - # - {{ compiler('cxx') }} + # Do not use the compiler on linux + - {{ compiler('cxx') }} # [not linux] - cmake - ninja - python >=3 - # - libcxx # it is not defined{{ cxx_compiler_version }} # [osx] + - libcxx # it is not defined{{ cxx_compiler_version }} # [osx] - patch # [not win] - # - m2-patch # [win] + - m2-patch # [win] - git # [(linux and x86_64)] host: - #- libcxx # it is not defined{{ cxx_compiler_version }} # [osx] + - libcxx # it is not defined{{ cxx_compiler_version }} # [osx] - libffi # [unix] # # libxml2 supports a windows-only feature, see https://github.com/llvm/llvm-project/blob/llvmorg-17.0.6/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h # - libxml2 # [win] - - zlib + - zlib # [not win] test: - files: - - numba-3016.ll + # Unused test file + # files: + # - numba-3016.ll commands: - $PREFIX/bin/llvm-config --libs # [not win] - $PREFIX/bin/llc -version # [not win] - # - if not exist %LIBRARY_INC%\\llvm\\Pass.h exit 1 # [win] - # - if not exist %LIBRARY_LIB%\\LLVMSupport.lib exit 1 # [win] + - if not exist %LIBRARY_INC%\\llvm\\Pass.h exit 1 # [win] + - if not exist %LIBRARY_LIB%\\LLVMSupport.lib exit 1 # [win] - test -f $PREFIX/include/llvm/Pass.h # [unix] - test -f $PREFIX/lib/libLLVMSupport.a # [unix] @@ -65,7 +71,7 @@ test: # LLD tests - ld.lld --version # [unix] - # - lld-link /? # [win] + - lld-link /? # [win] about: home: http://llvm.org/ From d95864eb9f0e3e44db7935e184cc8ffa3616ea34 Mon Sep 17 00:00:00 2001 From: Swapnil Patel Date: Fri, 10 Jan 2025 23:08:36 -0800 Subject: [PATCH 2/6] update llvmdev builds (#9) --- .github/workflows/llvmdev_build.yml | 72 ++++++++++++++----- .github/workflows/llvmdev_manylinux_build.yml | 27 +++++++ 2 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/llvmdev_manylinux_build.yml diff --git a/.github/workflows/llvmdev_build.yml b/.github/workflows/llvmdev_build.yml index 944606a7d..138968fe0 100644 --- a/.github/workflows/llvmdev_build.yml +++ b/.github/workflows/llvmdev_build.yml @@ -9,22 +9,32 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - - name: Build LLVMDEV + + - name: Install Miniconda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "$HOME/miniconda/bin" >> $GITHUB_PATH + + - name: Setup Conda Environment run: | - mkdir -p /home/runner/work/llvmlite/llvmlite/docker_output - chmod 777 /home/runner/work/llvmlite/llvmlite/docker_output - cd buildscripts/manylinux - ./docker_run_x64.sh build_llvmdev.sh "${MINICONDA_FILE}" - RECIPE_NAME=./conda-recipes/llvmdev_manylinux + source $HOME/miniconda/bin/activate + conda create -n builder conda-build -y + + - name: Build + run: | + source $HOME/miniconda/bin/activate + conda activate builder + RECIPE_NAME=./conda-recipes/llvmdev + conda build $RECIPE_NAME echo "OUTPUT=$(conda build --output $RECIPE_NAME)" >> $GITHUB_ENV - name: Upload LLVMDEV artifacts uses: actions/upload-artifact@v4 with: name: llvmdev-linux-x86_64 - path: /home/runner/work/llvmlite/llvmlite/docker_output/linux-64/*.tar.bz2 - retention-days: 1 + path: ${{ env.OUTPUT }} + retention-days: 7 if-no-files-found: error win-64: @@ -39,27 +49,55 @@ jobs: $wc = New-Object net.webclient $wc.Downloadfile("https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe", "Miniconda3-latest-Windows-x86_64.exe") Start-Process "Miniconda3-latest-Windows-x86_64.exe" "/S /D=C:\Miniconda3" -Wait + # Initialize conda for PowerShell + & C:\Miniconda3\shell\condabin\conda-hook.ps1 + conda init powershell - name: Setup Conda Environment - shell: bash + shell: pwsh run: | - source /c/Miniconda3/Scripts/activate + # Reload the path + $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") conda create -n builder conda-build -y + - name: Install Build Dependencies + shell: pwsh + run: | + conda activate builder + conda install -y cmake ninja + - name: Build - shell: bash + shell: pwsh run: | - source /c/Miniconda3/Scripts/activate conda activate builder - RECIPE_NAME=./conda-recipes/llvmdev_manylinux - conda build $RECIPE_NAME - echo "OUTPUT=$(conda build --output $RECIPE_NAME)" >> $GITHUB_ENV + + $ErrorActionPreference = 'Stop' + $RecipeName = ".\conda-recipes\llvmdev" + + Write-Host "Starting conda build..." + conda build $RecipeName + if ($LASTEXITCODE -ne 0) { + throw "Conda build failed with exit code $LASTEXITCODE" + } + + Write-Host "Getting output path..." + $OUTPUT = conda build --output $RecipeName + if ($LASTEXITCODE -ne 0) { + throw "Failed to get conda build output path" + } + + Write-Host "Conda package will be at: $OUTPUT" + if (-not (Test-Path $OUTPUT)) { + throw "Expected output package does not exist: $OUTPUT" + } + + "OUTPUT=$OUTPUT" | Out-File -FilePath $env:GITHUB_ENV -Append - name: Upload LLVMDEV artifacts uses: actions/upload-artifact@v4 with: name: llvmdev-windows-x86_64 path: ${{ env.OUTPUT }} - retention-days: 1 + retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/llvmdev_manylinux_build.yml b/.github/workflows/llvmdev_manylinux_build.yml new file mode 100644 index 000000000..39b2ced3e --- /dev/null +++ b/.github/workflows/llvmdev_manylinux_build.yml @@ -0,0 +1,27 @@ +name: llvmdev-manylinux-build + +on: + workflow_dispatch: + +jobs: + linux-64: + name: Build LLVMDEV Manylinux (linux-64) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Build LLVMDEV Manylinux + run: | + mkdir -p /home/runner/work/llvmlite/llvmlite/docker_output + chmod 777 /home/runner/work/llvmlite/llvmlite/docker_output + cd buildscripts/manylinux + ./docker_run_x64.sh build_llvmdev.sh "${MINICONDA_FILE}" + + - name: Upload LLVMDEV Manylinux artifacts + uses: actions/upload-artifact@v4 + with: + name: llvmdev-manylinux-linux-x86_64 + path: /home/runner/work/llvmlite/llvmlite/docker_output/linux-64/*.tar.bz2 + retention-days: 7 + if-no-files-found: error + From a8047a5f3d1ade2beceaecf91d3bfe5c7c100167 Mon Sep 17 00:00:00 2001 From: Swapnil Patel Date: Sat, 11 Jan 2025 01:17:11 -0800 Subject: [PATCH 3/6] update llvmlite workflow to use test env (#10) --- .github/workflows/llvmlite_build.yml | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/llvmlite_build.yml b/.github/workflows/llvmlite_build.yml index 183e1944a..e18ef7dc1 100644 --- a/.github/workflows/llvmlite_build.yml +++ b/.github/workflows/llvmlite_build.yml @@ -47,10 +47,17 @@ jobs: python setup.py clean python setup.py bdist_wheel + - name: Create Test Environment + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda create -n test_env \ + python=${{ matrix.python-version }} \ + -y + - name: test_linux_64_py${{ matrix.python-version }}_npyNone run: | source "$HOME/miniconda3/etc/profile.d/conda.sh" - conda activate build_env + conda activate test_env python -m pip install dist/*.whl python -m llvmlite.tests @@ -118,10 +125,16 @@ jobs: python setup.py clean python setup.py bdist_wheel + - name: Create Test Environment + shell: cmd + run: | + call C:\Miniconda3\Scripts\activate.bat + conda create -n test_env python=${{ matrix.python-version }} -y + - name: test_win_64_py${{ matrix.python-version }}_npyNone shell: cmd run: | - call C:\Miniconda3\Scripts\activate.bat build_env + call C:\Miniconda3\Scripts\activate.bat test_env python -m pip install dist\*.whl python -m llvmlite.tests @@ -178,10 +191,24 @@ jobs: python setup.py clean python setup.py bdist_wheel + - name: Create Test Environment + run: | + source "$HOME/miniconda3/etc/profile.d/conda.sh" + conda create -n test_env \ + python=${{ matrix.python-version }} \ + -y + conda activate test_env + conda install -y -c numba/label/ci llvmdev + - name: test_osx_64_py${{ matrix.python-version }}_npyNone run: | source "$HOME/miniconda3/etc/profile.d/conda.sh" - conda activate build_env + conda activate test_env + export MACOSX_DEPLOYMENT_TARGET=10.9 + export CONDA_BUILD_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk + export LLVM_CONFIG="$CONDA_PREFIX/bin/llvm-config" + export LLVMLITE_SHARED=1 + python -m pip install dist/*.whl python -m llvmlite.tests From 7fdad613eafec067b32b005acabf3a30ef0d0e5b Mon Sep 17 00:00:00 2001 From: Swapnil Patel Date: Wed, 15 Jan 2025 22:28:59 +0530 Subject: [PATCH 4/6] Add osx 64-bit llvmdev build workflow (#11) --- .github/workflows/llvmdev_build.yml | 44 +++++++++++++++++++++++++++++ conda-recipes/llvmdev/build.sh | 14 +++++++++ conda-recipes/llvmdev/meta.yaml | 2 +- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/workflows/llvmdev_build.yml b/.github/workflows/llvmdev_build.yml index 138968fe0..545a8e642 100644 --- a/.github/workflows/llvmdev_build.yml +++ b/.github/workflows/llvmdev_build.yml @@ -101,3 +101,47 @@ jobs: retention-days: 7 if-no-files-found: error + osx-64: + name: Build LLVMDEV (osx-64) + runs-on: macos-13 + steps: + - uses: actions/checkout@v4 + + - name: Setup Xcode + run: | + sudo xcode-select --switch /Applications/Xcode_14.2.app + xcrun --show-sdk-path + + - name: Install Miniconda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "$HOME/miniconda/bin" >> $GITHUB_PATH + + - name: Setup Conda Environment + run: | + source $HOME/miniconda/bin/activate + conda create -n builder conda-build cmake ninja -y + + - name: Build + run: | + source $HOME/miniconda/bin/activate + conda activate builder + RECIPE_NAME=./conda-recipes/llvmdev + conda build $RECIPE_NAME --no-anaconda-upload + echo "OUTPUT=$(conda build --output $RECIPE_NAME)" >> $GITHUB_ENV + + - name: Upload LLVMDEV artifacts + uses: actions/upload-artifact@v4 + with: + name: llvmdev-osx-64 + path: ${{ env.OUTPUT }} + retention-days: 7 + if-no-files-found: error + + - name: Get Workflow Run ID + run: | + echo "Current workflow run ID: ${{ github.run_id }}" + echo "Use this ID when triggering llvmlite workflow" + + diff --git a/conda-recipes/llvmdev/build.sh b/conda-recipes/llvmdev/build.sh index e1adebd59..a249eefe5 100644 --- a/conda-recipes/llvmdev/build.sh +++ b/conda-recipes/llvmdev/build.sh @@ -8,6 +8,20 @@ set -x sed -i.bak "s/NOT APPLE AND ARG_SONAME/ARG_SONAME/g" llvm/cmake/modules/AddLLVM.cmake sed -i.bak "s/NOT APPLE AND NOT ARG_SONAME/NOT ARG_SONAME/g" llvm/cmake/modules/AddLLVM.cmake +# For macOS, use system clang instead of conda's toolchain + if [[ "$target_platform" == osx-* ]]; then + export CONDA_BUILD_SYSROOT=$(xcrun --show-sdk-path) + export SDKROOT=$(xcrun --show-sdk-path) + export MACOSX_DEPLOYMENT_TARGET=10.9 + export CC=$(xcrun -f clang) + export CXX=$(xcrun -f clang++) + export CFLAGS="-isysroot $SDKROOT" + export CXXFLAGS="-isysroot $SDKROOT" + export LDFLAGS="-isysroot $SDKROOT" + # Override conda's cmake args + CMAKE_ARGS="" + fi + mkdir build cd build diff --git a/conda-recipes/llvmdev/meta.yaml b/conda-recipes/llvmdev/meta.yaml index 928e4f61c..464593708 100644 --- a/conda-recipes/llvmdev/meta.yaml +++ b/conda-recipes/llvmdev/meta.yaml @@ -33,7 +33,7 @@ build: requirements: build: - - {{ compiler('cxx') }} + - {{ compiler('cxx') }} # [not osx] - cmake - ninja - python >=3 From 6e35eb47064cf562bf24bb39b2896423f892ec62 Mon Sep 17 00:00:00 2001 From: Swapnil Patel Date: Wed, 15 Jan 2025 23:11:22 +0530 Subject: [PATCH 5/6] add option to use llvmdev from workflow artifacts(#12) --- .github/workflows/llvmdev_build.yml | 11 ++++- .github/workflows/llvmlite_build.yml | 61 +++++++++++++++++++++++++--- conda-recipes/llvmdev/meta.yaml | 3 +- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/.github/workflows/llvmdev_build.yml b/.github/workflows/llvmdev_build.yml index 545a8e642..4884fe93c 100644 --- a/.github/workflows/llvmdev_build.yml +++ b/.github/workflows/llvmdev_build.yml @@ -37,6 +37,11 @@ jobs: retention-days: 7 if-no-files-found: error + - name: Get Workflow Run ID + run: | + echo "Current workflow run ID: ${{ github.run_id }}" + echo "Use this ID when triggering llvmlite workflow" + win-64: name: Build LLVMDEV (win-64) runs-on: windows-2019 @@ -101,6 +106,11 @@ jobs: retention-days: 7 if-no-files-found: error + - name: Get Workflow Run ID + run: | + echo "Current workflow run ID: ${{ github.run_id }}" + echo "Use this ID when triggering llvmlite workflow" + osx-64: name: Build LLVMDEV (osx-64) runs-on: macos-13 @@ -143,5 +153,4 @@ jobs: run: | echo "Current workflow run ID: ${{ github.run_id }}" echo "Use this ID when triggering llvmlite workflow" - diff --git a/.github/workflows/llvmlite_build.yml b/.github/workflows/llvmlite_build.yml index e18ef7dc1..63eca00ee 100644 --- a/.github/workflows/llvmlite_build.yml +++ b/.github/workflows/llvmlite_build.yml @@ -2,6 +2,10 @@ name: llvmlite-build on: workflow_dispatch: + inputs: + llvmdev_run_id: + description: 'LLVMDEV workflow run ID to use artifacts from' + type: string jobs: linux-build: @@ -15,6 +19,17 @@ jobs: steps: - uses: actions/checkout@v4 + # Download llvmdev artifact from specific workflow run + - name: Download LLVMDEV Artifact + if: ${{ inputs.llvmdev_run_id != '' }} + uses: actions/download-artifact@v4 + with: + name: llvmdev-linux-x86_64 + path: llvmdev-pkg + run-id: ${{ inputs.llvmdev_run_id }} + repository: swap357/llvmlite + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Miniconda run: | bash buildscripts/incremental/install_miniconda.sh @@ -34,7 +49,13 @@ jobs: wheel \ -y conda activate build_env - conda install -y numba/label/ci::llvmdev --no-deps + if [ "${{ inputs.llvmdev_run_id }}" != "" ]; then + # Install local llvmdev package if workflow ID provided + conda install -y ./llvmdev-pkg/*.tar.bz2 + else + # Install from numba channel if no workflow ID + conda install -y -c numba/label/ci llvmdev + fi - name: build_linux_64_py${{ matrix.python-version }}_npy1.11 run: | @@ -80,6 +101,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Download LLVMDEV Artifact + if: ${{ inputs.llvmdev_run_id != '' }} + uses: actions/download-artifact@v4 + with: + name: llvmdev-windows-x86_64 + path: llvmdev-pkg + run-id: ${{ inputs.llvmdev_run_id }} + repository: swap357/llvmlite + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Visual Studio uses: microsoft/setup-msbuild@v1.1 @@ -102,8 +133,13 @@ jobs: libxml2 \ -y conda activate build_env - # Install LLVM with all dependencies - conda install -y -c numba/label/ci llvmdev + if [ "${{ inputs.llvmdev_run_id }}" != "" ]; then + # Install local llvmdev package if workflow ID provided + conda install -y ./llvmdev-pkg/*.tar.bz2 + else + # Install from numba channel if no workflow ID + conda install -y -c numba/label/ci llvmdev + fi - name: Set Environment Variables shell: bash @@ -157,6 +193,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Download LLVMDEV Artifact + if: ${{ inputs.llvmdev_run_id != '' }} + uses: actions/download-artifact@v4 + with: + name: llvmdev-osx-64 + path: llvmdev-pkg + run-id: ${{ inputs.llvmdev_run_id }} + repository: swap357/llvmlite + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Miniconda run: | bash buildscripts/incremental/install_miniconda.sh @@ -176,8 +222,13 @@ jobs: wheel \ -y conda activate build_env - # Install LLVM with all dependencies for macOS - conda install -y -c numba/label/ci llvmdev + if [ "${{ inputs.llvmdev_run_id }}" != "" ]; then + # Install local llvmdev package if workflow ID provided + conda install -y ./llvmdev-pkg/*.tar.bz2 + else + # Install from numba channel if no workflow ID + conda install -y -c numba/label/ci llvmdev + fi - name: build_osx_64_py${{ matrix.python-version }}_npy1.11 run: | diff --git a/conda-recipes/llvmdev/meta.yaml b/conda-recipes/llvmdev/meta.yaml index 464593708..65d0e6202 100644 --- a/conda-recipes/llvmdev/meta.yaml +++ b/conda-recipes/llvmdev/meta.yaml @@ -28,12 +28,11 @@ build: script_env: - PY_VCRUNTIME_REDIST ignore_run_exports: - # Is static-linked - xar requirements: build: - - {{ compiler('cxx') }} # [not osx] + - {{ compiler('cxx') }} # [not osx] - cmake - ninja - python >=3 From f13ae3f9123170e217f0a75d6ec0d8e5e7c97769 Mon Sep 17 00:00:00 2001 From: Swapnil Patel Date: Fri, 17 Jan 2025 17:10:16 +0530 Subject: [PATCH 6/6] add llvmlite win builder workflow (#14) --- .github/workflows/llvmlite_win_builder.yml | 79 ++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/llvmlite_win_builder.yml diff --git a/.github/workflows/llvmlite_win_builder.yml b/.github/workflows/llvmlite_win_builder.yml new file mode 100644 index 000000000..e6d6102a5 --- /dev/null +++ b/.github/workflows/llvmlite_win_builder.yml @@ -0,0 +1,79 @@ +name: llvmlite-build + +on: + workflow_dispatch: + inputs: + llvmdev_run_id: + description: 'llvmdev workflow run ID (optional)' + required: false + type: string + +jobs: + windows: + name: windows + runs-on: windows-2019 + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Download LLVMDEV Artifact + if: ${{ inputs.llvmdev_run_id != '' }} + uses: actions/download-artifact@v4 + with: + name: llvmdev-windows-x86_64 + path: llvmdev-pkg + run-id: ${{ inputs.llvmdev_run_id }} + repository: ${{ github.repository }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install Visual Studio + uses: microsoft/setup-msbuild@v1.1 + + - name: Install Miniconda + shell: pwsh + run: | + $wc = New-Object net.webclient + $wc.Downloadfile("https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe", "Miniconda3-latest-Windows-x86_64.exe") + Start-Process "Miniconda3-latest-Windows-x86_64.exe" "/S /D=C:\Miniconda3" -Wait + + - name: Create Build Environment + shell: bash + run: | + source /c/Miniconda3/Scripts/activate + conda create -n build_env \ + python=${{ matrix.python-version }} \ + conda-build \ + setuptools \ + wheel \ + cmake \ + libxml2 \ + -y + conda activate build_env + if [ "${{ inputs.llvmdev_run_id }}" != "" ]; then + # Install local llvmdev package if workflow ID provided + conda install -y ./llvmdev-pkg/*.tar.bz2 + else + # Install from numba channel if no workflow ID + conda install -y -c numba/label/ci llvmdev + fi + + - name: Build Conda Package + shell: bash + run: | + source /c/Miniconda3/Scripts/activate + conda activate build_env + RECIPE_NAME=./conda-recipes/llvmlite + conda build $RECIPE_NAME + echo "CONDA_PKG=$(conda build --output $RECIPE_NAME)" >> $GITHUB_ENV + + - name: Upload Conda Package + uses: actions/upload-artifact@v4 + with: + name: llvmlite-win-64-py${{ matrix.python-version }} + path: ${{ env.CONDA_PKG }} + retention-days: 7 + if-no-files-found: error \ No newline at end of file