Add LLVM as submodule (and add GHA build scripts)

.github/workflows/ci_manylinux_x86_64_clang_llvm.yml

@@ -0,0 +1,118 @@
+name: CI - manylinux x86_64 clang LLVM
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+  merge_group:
+  push:
+    branches:
+      - main
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  manylinux_x64_clang_llvm:
+    name: Build LLVM and MLIR (linux)
+
+    runs-on: ubuntu-22.04
+
+    env:
+      CACHE_DIR: ${{ github.workspace }}/.container-cache
+      # either the PR number or `branch-N` where N always increments
+      CACHE_KEY: manylinux_x64_clang_llvm_${{ format('{0}-{1}', github.ref_name, github.run_number) }}
+
+    container:
+      image: quay.io/pypa/manylinux_2_28_x86_64
+
+    steps:
+
+      - name: "Set unified TZ"
+        uses: szenius/[email protected]
+        with:
+          # this is an arbitrary choice
+          timezoneLinux: "Asia/Singapore"
+          timezoneMacos: "Asia/Singapore"
+          timezoneWindows: "Singapore Standard Time"
+
+      - name: Enable cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.CACHE_DIR }}
+          key:  ${{ env.CACHE_KEY }}
+          restore-keys: manylinux_x64_clang_llvm_
+
+      - name: "Check out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+
+      - name: "Install OS deps"
+        run: |
+          dnf install -y epel-release 
+          dnf install -y sudo ncurses-compat-libs tmate
+
+      - name: "Setup compiler/toolchain"
+        uses: aminya/setup-cpp@v1
+        with:
+          compiler: llvm-18
+          cmake: true
+          ninja: true
+          ccache: true
+
+      - name: "Setup Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: "Python deps"
+        run: |
+          pip install -r third_party/llvm-project/mlir/python/requirements.txt
+
+      - name: "Build LLVM and MLIR"
+        run: |
+          export CCACHE_DIR="${{ env.CACHE_DIR }}"
+          export CCACHE_MAXSIZE=700M
+          export CCACHE_COMPILERCHECK="string:$(clang --version)"
+          export CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros
+          
+          export LLVM_BUILD_DIR="$PWD/llvm-build"
+          export LLVM_INSTALL_DIR="$PWD/llvm-install"
+          
+          build_tools/cmake/build_llvm.sh build_llvm
+          build_tools/cmake/build_llvm.sh build_mlir
+
+      - name: "Tar dist"
+        if: ${{ !cancelled() }}
+        run: |
+          pushd third_party/llvm-project && llvm_sha_short=$(git rev-parse --short HEAD) && popd
+          tar cf llvm-dist-linux-$llvm_sha_short.tar llvm-install
+
+      - name: "Upload dist"
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: linux_x86_64_llvm_packages
+          path: llvm-dist-*.tar
+          if-no-files-found: error
+
+      - name: "Save cache"
+        uses: actions/cache/save@v3
+        # if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }}
+        if: ${{ !cancelled() }}
+        with:
+          path: ${{ env.CACHE_DIR }}
+          key: ${{ env.CACHE_KEY }}
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/[email protected]
+        with:
+          limit-access-to-actor: true
+          install-dependencies: false

.gitignore

@@ -0,0 +1,34 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+.idea

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "third_party/llvm-project"]
+	path = third_party/llvm-project
+	url = https://github.com/llvm/llvm-project.git
+	ignore = dirty
+	shallow = true

build_tools/cmake/build_llvm.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+TD="$(cd $(dirname $0) && pwd)"
+REPO_ROOT="$(cd $TD/../.. && pwd)"
+
+LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${REPO_ROOT}/third_party/llvm-project}"
+LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${REPO_ROOT}/../llvm-build}"
+LLVM_INSTALL_DIR="${LLVM_INSTALL_DIR:-${REPO_ROOT}/../llvm-install}"
+
+LLVM_SOURCE_DIR="$(realpath -m "${LLVM_SOURCE_DIR}")"
+LLVM_BUILD_DIR="$(realpath -m "${LLVM_BUILD_DIR}")"
+LLVM_INSTALL_DIR="$(realpath -m "${LLVM_INSTALL_DIR}")"
+
+echo "Paths canonicalized as:"
+echo "LLVM_SOURCE_DIR=${LLVM_SOURCE_DIR}"
+echo "LLVM_BUILD_DIR=${LLVM_BUILD_DIR}"
+echo "LLVM_INSTALL_DIR=${LLVM_INSTALL_DIR}"
+
+command="$1"
+shift
+
+has_ccache=false
+if (command -v ccache &> /dev/null); then
+  has_ccache=true
+fi
+
+has_clang=false
+if (command -v clang &> /dev/null) && (command -v clang++ &> /dev/null); then
+  has_clang=true
+fi
+
+has_lld=false
+if (command -v lld &> /dev/null); then
+  has_lld=true
+fi
+
+python3_command=""
+if (command -v python3 &> /dev/null); then
+  python3_command="python3"
+elif (command -v python &> /dev/null); then
+  python3_command="python"
+fi
+
+set -euo pipefail
+
+print_toolchain_config() {
+  if $has_ccache; then
+    echo -n "-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache "
+  fi
+  if $has_clang; then
+    echo "-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
+  fi
+}
+
+do_build_llvm() {
+  echo "*********************** BUILDING LLVM *********************************"
+  main_build_dir="${LLVM_BUILD_DIR}/llvm"
+  main_install_dir="${LLVM_INSTALL_DIR}/llvm"
+  targets_to_build="${LLVM_TARGETS_TO_BUILD:-X86}"
+
+  cmake_options="-DLLVM_TARGETS_TO_BUILD='${targets_to_build}'"
+  cmake_options="${cmake_options} -DCMAKE_BUILD_TYPE=Release"
+  cmake_options="${cmake_options} -C $TD/llvm_config.cmake"
+  cmake_options="${cmake_options} -DCMAKE_INSTALL_PREFIX=${main_install_dir}"
+  cmake_options="${cmake_options} $(print_toolchain_config)"
+  if $has_lld; then
+    cmake_options="${cmake_options} -DLLVM_ENABLE_LLD=ON"
+  fi
+
+  echo "Source Directory: ${LLVM_SOURCE_DIR}"
+  echo "Build Directory: ${main_build_dir}"
+  echo "CMake Options: ${cmake_options}"
+  cmake -GNinja -S "${LLVM_SOURCE_DIR}/llvm" -B "${main_build_dir}" \
+    $cmake_options
+  cmake --build "${main_build_dir}" \
+    --target install-toolchain-distribution \
+    --target install-development-distribution
+}
+
+do_build_mlir() {
+  echo "*********************** BUILDING MLIR *********************************"
+  main_install_dir="${LLVM_INSTALL_DIR}/llvm"
+  mlir_build_dir="${LLVM_BUILD_DIR}/mlir"
+  mlir_install_dir="${LLVM_INSTALL_DIR}/mlir"
+
+  cmake_options="-DLLVM_DIR='${main_install_dir}/lib/cmake/llvm'"
+  cmake_options="${cmake_options} -DPython3_EXECUTABLE='$(which $python3_command)'"
+  # Note: Building the MLIR Python bindings requires the installation of
+  # dependencies as specified in `mlir/python/requirements.txt`, which among
+  # others include pybind11.
+  cmake_options="${cmake_options} -DMLIR_ENABLE_BINDINGS_PYTHON=ON"
+  cmake_options="${cmake_options} -DCMAKE_INSTALL_PREFIX=${mlir_install_dir}"
+  cmake_options="${cmake_options} -C $TD/mlir_config.cmake"
+  cmake_options="${cmake_options} $(print_toolchain_config)"
+  if $has_lld; then
+    cmake_options="${cmake_options} -DLLVM_ENABLE_LLD=ON"
+  fi
+
+  echo "Source Directory: ${LLVM_SOURCE_DIR}"
+  echo "Build Directory: ${mlir_build_dir}"
+  echo "CMake Options: ${cmake_options}"
+  cmake -GNinja -S "${LLVM_SOURCE_DIR}/mlir" -B "${mlir_build_dir}" \
+    $cmake_options
+  cmake --build "${mlir_build_dir}" \
+    --target install-mlirdevelopment-distribution
+}
+
+case "${command}" in
+  build_llvm)
+    do_build_llvm
+    ;;
+
+  build_mlir)
+    do_build_mlir
+    ;;
+
+  *)
+    echo "ERROR: Expected command of 'build_llvm', 'build_mlir' (got '${command}')"
+    exit 1
+    ;;
+esac