Merge pull request #5 from DangoSys/main

bump to latest version

Author: shirohasuki

.github/workflows/TestBuild.yml

@@ -2,32 +2,70 @@ name: test build process
 
 on:
   push:
-    branches: [ main, ci ]
+    branches: [ main, ci, ci-test ]
   pull_request:
-    branches: [ main, ci ]
+    branches: [ main, ci, ci-test ]
 
 jobs:
-  build:
-    runs-on: self-hosted
+  test:
+    name: Build (${{ matrix.arch }})
+    timeout-minutes: 360
+    continue-on-error: ${{ matrix.experimental }}
+
+    concurrency:
+      group: buddy-mlir-${{ matrix.arch }}
+      cancel-in-progress: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x64
+            experimental: false
+          - arch: riscv64
+            experimental: true
+
+    runs-on:
+      - self-hosted
+      - ${{ matrix.arch }}
+
     steps:
+      - name: Print target
+        run: |
+          echo "Building on ${{ matrix.arch }}"
+          echo "------ uname ------"
+          uname -a
+
       # 0. Install the Ninja build system.
       - name: Set up ninja
+        if: matrix.arch == 'x64'
         uses: seanmiddleditch/gha-setup-ninja@master
 
-      # 1. Checkout the main repository without fetching submodules initially
+      # 1A. Checkout the main repository without fetching submodules initially
       - name: Checkout main repository
+        if: matrix.arch == 'x64'
         uses: actions/checkout@v4
         with:
           submodules: 'false'
 
+      # 1B. first clone from local gitea then update metedata from github
+      - name: Checkout from local Gitea (riscv64)
+        if: matrix.arch == 'riscv64'
+        run: |
+          git clone --no-checkout https://community-ci.openruyi.cn/RuyiAI-Stack/buddy-mlir .
+          git remote add github https://github.com/buddy-compiler/buddy-mlir.git
+          git fetch github ${{ github.sha }}
+          git checkout ${{ github.sha }}
+
       # 2. Retrieve the commit ID of the LLVM submodule.
       - name: Get LLVM submodule commit
         id: llvm-submodule-commit
         run: |
           echo "commit=$(git submodule status llvm | awk '{print $1;}')" >> $GITHUB_OUTPUT
 
-      # 3. Cache the LLVM submodule source code.
-      - name: Cache LLVM source
+      # 3. Cache the LLVM submodule source code (only for x86).
+      - name: Cache LLVM source (x86)
+        if: matrix.arch == 'x64'
         id: cache-llvm-source
         uses: actions/cache@v4
         with:
@@ -36,15 +74,16 @@ jobs:
           restore-keys: |
             llvm-source-
 
-      # 4. If the cache is not found, pull the LLVM submodule.
+      # 4. If the cache is not found, pull the LLVM submodule. (x86, cache miss)
       - name: Checkout LLVM submodule
+        if: matrix.arch == 'x64' && steps.cache-llvm-source.outputs.cache-hit != 'true'
         run: |
           rm -rf llvm
           git submodule update --init --recursive llvm
-        if: steps.cache-llvm-source.outputs.cache-hit != 'true'
 
       # 5. Cache the LLVM build directory.
       - name: Cache LLVM build directory
+        if: matrix.arch == 'x64'
         id: cache-llvm-build-dir
         uses: actions/cache@v4
         with:
@@ -54,9 +93,9 @@ jobs:
             llvm-build-
 
       # 6. Verify llvm-build when cached; build LLVM when no cache or verification fails.
-      - name: Check LLVM cache
+      - name: Check LLVM cache (x86)
         id: check-llvm-cache
-        if: steps.cache-llvm-build-dir.outputs.cache-hit == 'true'
+        if: matrix.arch == 'x64' && (steps.cache-llvm-build-dir.outputs.cache-hit == 'true')
         run: |
           for conda in ~/miniconda3/bin/activate ~/miniforge3/bin/activate; do
             [ -f "$conda" ] && source "$conda" buddy && break
@@ -72,8 +111,9 @@ jobs:
           fi
         continue-on-error: true
 
-      - name: Configure and Build LLVM
-        if: steps.cache-llvm-build-dir.outputs.cache-hit != 'true' || steps.check-llvm-cache.outputs.need-rebuild == 'true'
+      - name: Configure and Build LLVM (x86)
+        if: matrix.arch == 'x64' &&
+          (steps.cache-llvm-build-dir.outputs.cache-hit != 'true' || steps.check-llvm-cache.outputs.need-rebuild == 'true')
         run: |
           for conda in ~/miniconda3/bin/activate ~/miniforge3/bin/activate; do
             [ -f "$conda" ] && source "$conda" buddy && break
@@ -94,8 +134,76 @@ jobs:
             -DPython3_EXECUTABLE=$(which python3)
           ninja check-clang check-mlir omp
 
-      # 7. Check buddy-mlir build.
-      - name: Check buddy-mlir build
+      - name: Enable ccache (riscv64)
+        if: matrix.arch == 'riscv64'
+        run: |
+          echo "CCACHE_DIR=/home/jenkins/.ccache" >> $GITHUB_ENV
+          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+          ccache -M 50G
+          echo "CCACHE_MAXSIZE=50G" >> $GITHUB_ENV
+
+      - name: Setup LLVM build (riscv64)
+        if: matrix.arch == 'riscv64'
+        id: prepare-llvm-build
+        run: |
+          LLVM_SRC=/home/jenkins/src/llvm-project
+          LLVM_BUILD_ROOT=/home/jenkins/llvm-cache
+          LLVM_COMMIT=$(git ls-tree HEAD llvm | awk '{print $3}')
+          LLVM_BUILD_DIR=$LLVM_BUILD_ROOT/$LLVM_COMMIT
+
+          mkdir -p $LLVM_BUILD_ROOT
+
+          # prepare source
+          if [ ! -d "$LLVM_SRC" ]; then
+            mkdir -p /home/jenkins/src
+            git clone https://community-ci.openruyi.cn/toolchain/llvm-project.git $LLVM_SRC
+          fi
+
+          git -C $LLVM_SRC fetch --all
+          git -C $LLVM_SRC reset --hard
+          git -C $LLVM_SRC clean -fdx
+          git -C $LLVM_SRC checkout $LLVM_COMMIT
+
+          echo "LLVM_SRC=$LLVM_SRC" >> $GITHUB_ENV
+          echo "LLVM_COMMIT=$LLVM_COMMIT" >> $GITHUB_ENV
+          echo "LLVM_BUILD_DIR=$LLVM_BUILD_DIR" >> $GITHUB_ENV
+
+          if [ -f "$LLVM_BUILD_DIR/build/CMakeCache.txt" ] &&
+            grep -q "$LLVM_SRC" "$LLVM_BUILD_DIR/build/CMakeCache.txt"; then
+            echo "need-rebuild=false" >> $GITHUB_OUTPUT
+          else
+            echo "need-rebuild=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Configure and Build LLVM (riscv64)
+        if: matrix.arch == 'riscv64'&&
+            steps.prepare-llvm-build.outputs.need-rebuild == 'true'
+        run: |
+          source /home/jenkins/venv/bin/activate
+          pip install -U pip setuptools wheel packaging
+          sed -i \
+            -e '1s/^/# /' \
+            requirements.txt
+          pip install -r requirements.txt
+          rm -rf $LLVM_BUILD_DIR
+          cmake -G Ninja -S $LLVM_SRC/llvm  -B $LLVM_BUILD_DIR/build \
+            -DLLVM_ENABLE_PROJECTS="mlir;clang;openmp" \
+            -DLLVM_TARGETS_TO_BUILD="host;RISCV" \
+            -DLLVM_ENABLE_ASSERTIONS=ON \
+            -DCMAKE_BUILD_TYPE=RELEASE \
+            -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
+            -DPython3_EXECUTABLE=$(which python3)
+          ninja -C $LLVM_BUILD_DIR/build check-clang check-mlir omp -j$(nproc) || true
+          deactivate
+
+      - name: Show ccache stats (riscv64)
+        if: matrix.arch == 'riscv64'
+        run: |
+          ccache -s
+
+      # 7A. Check buddy-mlir build (x86).
+      - name: Check buddy-mlir build (x86)
+        if: matrix.arch == 'x64'
         run: |
           for conda in ~/miniconda3/bin/activate ~/miniforge3/bin/activate; do
             [ -f "$conda" ] && source "$conda" buddy && break
@@ -113,3 +221,38 @@ jobs:
             -DPython3_EXECUTABLE=$(which python3)
           ninja
           ninja check-buddy
+
+       # 7B. Check buddy-mlir build (riscv64).
+      - name: Check buddy-mlir build (riscv64)
+        if: matrix.arch == 'riscv64'
+        run: |
+          source /home/jenkins/venv/bin/activate
+          echo "---- CHECK LOCAL SUBMODULE ----"
+          ls -l llvm || true
+          git submodule status || true
+          rm -rf build
+          mkdir build
+          cd build
+
+          #  It will fail to find source code if no -DLLVM_MAIN_SRC_DIR=$LLVM_SRC/llvm  -DMLIR_MAIN_SRC_DIR=$LLVM_SRC/mlir
+          cmake -G Ninja .. \
+            -DMLIR_DIR=$LLVM_BUILD_DIR/build/lib/cmake/mlir \
+            -DLLVM_DIR=$LLVM_BUILD_DIR/build/lib/cmake/llvm \
+            -DLLVM_MAIN_SRC_DIR=$LLVM_SRC/llvm \
+            -DMLIR_MAIN_SRC_DIR=$LLVM_SRC/mlir \
+            -DLLVM_ENABLE_ASSERTIONS=ON \
+            -DCMAKE_BUILD_TYPE=RELEASE \
+            -DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON \
+            -DPython3_EXECUTABLE=$(which python3)
+          ninja -j$(nproc)
+          ninja check-buddy -j$(nproc) || true
+          deactivate
+
+      - name: Cleanup old LLVM build cache (riscv64)
+        if: matrix.arch == 'riscv64'
+        run: |
+          CACHE_DIR=/home/jenkins/llvm-cache
+          KEEP=3
+
+          cd $CACHE_DIR
+          ls -1dt */ | tail -n +$((KEEP+1)) | xargs -r rm -rf

.github/workflows/build-manylinux.yml

@@ -0,0 +1,76 @@
+name: build-manylinux
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Release version tag (e.g., v0.1.0)"
+        required: true
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  wheel:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.10", "3.11", "3.12", "3.13"]
+        torch_version: ["2.8"]  # placeholder; not used in build, keeps matrix extensible
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Record LLVM submodule commit
+        id: llvm_rev
+        run: echo "sha=$(git -C llvm rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache LLVM build tree
+        id: cache-llvm-build
+        uses: actions/cache@v4
+        with:
+          path: llvm/build.docker
+          key: llvm-${{ runner.os }}-${{ matrix.python }}-${{ steps.llvm_rev.outputs.sha }}
+
+      - name: Build manylinux wheel in container
+        run: |
+          set -euo pipefail
+          PY_VER="${{ matrix.python }}"
+          PY_NODOT="${PY_VER//./}"
+          PY_TAG="cp${PY_NODOT}-cp${PY_NODOT}"
+          export TORCH_VERSION="${{ matrix.torch_version }}"
+          export LLVM_CACHE_HIT="${{ steps.cache-llvm-build.outputs.cache-hit }}"
+          ./scripts/release_wheel_manylinux.sh "${PY_TAG}"
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-py${{ matrix.python }}-torch${{ matrix.torch_version }}
+          path: build.docker/dist/*.whl
+
+  release:
+    needs: wheel
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download wheels
+        uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
+      - name: Upload assets
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ inputs.version }}
+          overwrite_files: true
+          files: dist/*manylinux*.whl
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pre-commit.yml

@@ -0,0 +1,21 @@
+name: pre-commit
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - uses: pre-commit/action@v3.0.1
+        with:
+          extra_args: --show-diff-on-failure --color=always