Spaces:
Running
Running
ci : use ninja and fix caching for windows-cublas (#2910)
Browse filesThis commit updates the windows-cublas job to use Ninja as the build
system instead of msbuild/msvc.
The motivation for this is that msbuild/mscv does not seem to handle
ccache/sccache well, for example it ignores the
`CMAKE_C_COMPILER_LAUNCHER` etc. variables. But using Ninja as the build
caching works and the build is initially the same speed as it is
currently (without caching) subsequently builds are much faster.
Refs: https://github.com/ggerganov/whisper.cpp/issues/2781
- .github/workflows/build.yml +15 -30
.github/workflows/build.yml
CHANGED
|
@@ -677,6 +677,11 @@ jobs:
|
|
| 677 |
- name: Clone repository
|
| 678 |
uses: actions/checkout@v4
|
| 679 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
- name: Install ccache
|
| 681 |
uses: hendrikmuhs/[email protected]
|
| 682 |
with:
|
|
@@ -684,12 +689,6 @@ jobs:
|
|
| 684 |
variant: sccache
|
| 685 |
evict-old-files: 1d
|
| 686 |
|
| 687 |
-
- name: Configure CUDA compilation cache
|
| 688 |
-
run: |
|
| 689 |
-
New-Item -Path "${{ github.workspace }}\cuda_cache" -ItemType Directory -Force
|
| 690 |
-
echo "CUDA_CACHE_PATH=${{ github.workspace }}\cuda_cache" | Out-File -FilePath $env:GITHUB_ENV -Append
|
| 691 |
-
echo "CUDA_CACHE_MAXSIZE=4294967296" | Out-File -FilePath $env:GITHUB_ENV -Append
|
| 692 |
-
|
| 693 |
- name: Install Cuda Toolkit 11.8.0
|
| 694 |
if: ${{ matrix.cuda-toolkit == '11.8.0' }}
|
| 695 |
run: |
|
|
@@ -781,7 +780,7 @@ jobs:
|
|
| 781 |
curl -O "$CUDA_DOWNLOAD/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-${CCCL_VER}-archive.zip"
|
| 782 |
|
| 783 |
# Extract all the downloaded files to the CUDA Toolkit directory
|
| 784 |
-
unzip '*.zip' -d $CUDA_TOOLKIT_DIR
|
| 785 |
|
| 786 |
# Copy all the extracted files to the main CUDA Toolkit directory
|
| 787 |
xcopy "$CUDA_TOOLKIT_DIR\cuda_cudart-windows-x86_64-${CUDART_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
|
|
@@ -803,14 +802,6 @@ jobs:
|
|
| 803 |
echo "CUDA_PATH=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
| 804 |
echo "CUDA_PATH_V12_2=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
| 805 |
|
| 806 |
-
- name: Cache CUDA compilation results
|
| 807 |
-
uses: actions/cache@v3
|
| 808 |
-
with:
|
| 809 |
-
path: ${{ github.workspace }}/cuda_cache
|
| 810 |
-
key: cuda-cache-${{ runner.os }}-${{ matrix.cuda-toolkit }}-${{ hashFiles('**/*.cu', '**/*.cuh') }}
|
| 811 |
-
restore-keys: |
|
| 812 |
-
cuda-cache-${{ runner.os }}-${{ matrix.cuda-toolkit }}-
|
| 813 |
-
|
| 814 |
- name: Add msbuild to PATH
|
| 815 |
uses: microsoft/setup-msbuild@v2
|
| 816 |
|
|
@@ -825,34 +816,28 @@ jobs:
|
|
| 825 |
echo "SDL2_DIR=${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" | Out-File -FilePath $env:GITHUB_ENV -Append
|
| 826 |
echo "${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" > SDL2_PATH.txt
|
| 827 |
|
| 828 |
-
- name:
|
|
|
|
|
|
|
|
|
|
| 829 |
shell: cmd
|
| 830 |
run: |
|
| 831 |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
|
| 832 |
-
|
| 833 |
-
cmake
|
|
|
|
| 834 |
-DCMAKE_BUILD_TYPE=${{ matrix.build }} ^
|
| 835 |
-DGGML_CUDA=${{ matrix.cublas }} ^
|
| 836 |
-DCMAKE_CUDA_ARCHITECTURES=all ^
|
| 837 |
-DWHISPER_SDL2=${{ matrix.sdl2 }} ^
|
| 838 |
-
-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache ^
|
| 839 |
-
-DCMAKE_C_COMPILER_LAUNCHER=sccache ^
|
| 840 |
-
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache ^
|
| 841 |
-
-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded ^
|
| 842 |
-
-DCMAKE_POLICY_CMP0141=NEW ^
|
| 843 |
-DSDL2_DIR="%SDL2_DIR%"
|
|
|
|
|
|
|
| 844 |
|
| 845 |
- name: Check sccache status after build
|
| 846 |
run: |
|
| 847 |
sccache --show-stats
|
| 848 |
|
| 849 |
-
- name: Build Project
|
| 850 |
-
shell: cmd
|
| 851 |
-
run: |
|
| 852 |
-
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
|
| 853 |
-
cd ./build
|
| 854 |
-
cmake --build . --config ${{ matrix.build }} -j %NUMBER_OF_PROCESSORS%
|
| 855 |
-
|
| 856 |
- name: Copy CUDA DLLs
|
| 857 |
run: |
|
| 858 |
Get-ChildItem "$env:CUDA_PATH\bin\" -Filter "*.dll" |
|
|
|
|
| 677 |
- name: Clone repository
|
| 678 |
uses: actions/checkout@v4
|
| 679 |
|
| 680 |
+
- name: Install Ninja
|
| 681 |
+
id: install_ninja
|
| 682 |
+
run: |
|
| 683 |
+
choco install ninja
|
| 684 |
+
|
| 685 |
- name: Install ccache
|
| 686 |
uses: hendrikmuhs/[email protected]
|
| 687 |
with:
|
|
|
|
| 689 |
variant: sccache
|
| 690 |
evict-old-files: 1d
|
| 691 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 692 |
- name: Install Cuda Toolkit 11.8.0
|
| 693 |
if: ${{ matrix.cuda-toolkit == '11.8.0' }}
|
| 694 |
run: |
|
|
|
|
| 780 |
curl -O "$CUDA_DOWNLOAD/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-${CCCL_VER}-archive.zip"
|
| 781 |
|
| 782 |
# Extract all the downloaded files to the CUDA Toolkit directory
|
| 783 |
+
unzip -q '*.zip' -d $CUDA_TOOLKIT_DIR
|
| 784 |
|
| 785 |
# Copy all the extracted files to the main CUDA Toolkit directory
|
| 786 |
xcopy "$CUDA_TOOLKIT_DIR\cuda_cudart-windows-x86_64-${CUDART_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
|
|
|
|
| 802 |
echo "CUDA_PATH=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
| 803 |
echo "CUDA_PATH_V12_2=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
| 804 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 805 |
- name: Add msbuild to PATH
|
| 806 |
uses: microsoft/setup-msbuild@v2
|
| 807 |
|
|
|
|
| 816 |
echo "SDL2_DIR=${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" | Out-File -FilePath $env:GITHUB_ENV -Append
|
| 817 |
echo "${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" > SDL2_PATH.txt
|
| 818 |
|
| 819 |
+
- name: Install cmake
|
| 820 |
+
run: choco install cmake
|
| 821 |
+
|
| 822 |
+
- name: Build Project
|
| 823 |
shell: cmd
|
| 824 |
run: |
|
| 825 |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
|
| 826 |
+
cmake --version
|
| 827 |
+
where cmake
|
| 828 |
+
cmake -S . -B build -G "Ninja Multi-Config" ^
|
| 829 |
-DCMAKE_BUILD_TYPE=${{ matrix.build }} ^
|
| 830 |
-DGGML_CUDA=${{ matrix.cublas }} ^
|
| 831 |
-DCMAKE_CUDA_ARCHITECTURES=all ^
|
| 832 |
-DWHISPER_SDL2=${{ matrix.sdl2 }} ^
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 833 |
-DSDL2_DIR="%SDL2_DIR%"
|
| 834 |
+
set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
|
| 835 |
+
cmake --build build --config ${{ matrix.build }} -j %NUMBER_OF_PROCESSORS%
|
| 836 |
|
| 837 |
- name: Check sccache status after build
|
| 838 |
run: |
|
| 839 |
sccache --show-stats
|
| 840 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 841 |
- name: Copy CUDA DLLs
|
| 842 |
run: |
|
| 843 |
Get-ChildItem "$env:CUDA_PATH\bin\" -Filter "*.dll" |
|