Add LLVM as submodule (and add GHA build scripts)

.github/workflows/ci_manylinux_x86_64_clang_llvm.yml

@@ -0,0 +1,105 @@
+name: CI - manylinux x86_64 clang LLVM
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+  merge_group:
+  push:
+    branches:
+      - main
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  manylinux_x64_clang_llvm:
+    name: Build LLVM and MLIR (linux)
+
+    runs-on: ubuntu-22.04
+
+    container:
+      image: quay.io/pypa/manylinux_2_28_x86_64
+      env:
+        CACHE_DIR: ${{ github.workspace }}/.container-cache
+        # either the PR number or `branch-N` where N always increments
+        CACHE_KEY: manylinux_x64_clang_llvm_${{ format('{0}-{1}', github.ref_name, github.run_number) }}
+
+    steps:
+      - name: "Set unified TZ"
+        uses: szenius/[email protected]
+        with:
+          # this is an arbitrary choice
+          timezoneLinux: "Asia/Singapore"
+          timezoneMacos: "Asia/Singapore"
+          timezoneWindows: "Singapore Standard Time"
+
+      - name: Enable cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.CACHE_DIR }}
+          key:  ${{ env.CACHE_KEY }}
+          restore-keys: manylinux_x64_clang_llvm_
+
+      - name: "Check out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+
+      - name: "Setup compiler/toolchain"
+        uses: aminya/[email protected]
+        with:
+          compiler: llvm-18
+          cmake: true
+          ninja: true
+          cache: ccache
+
+      - name: "Setup ccache"
+        run: |
+          echo "CCACHE_DIR=${{ env.CACHE_DIR }}" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=700M" >> $GITHUB_ENV
+          echo "CCACHE_COMPILERCHECK=string:$(clang --version)" >> $GITHUB_ENV
+          echo "CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros" >> $GITHUB_ENV
+
+      - name: "Setup Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: "Python deps"
+        run: |
+          pip install -r third_party/llvm-project/mlir/python/requirements.txt
+
+      - name: "Build LLVM and MLIR"
+        run: |
+          export LLVM_BUILD_DIR="$PWD/llvm-build"
+          export LLVM_INSTALL_DIR="$PWD/llvm-install"
+          build_tools/cmake/build_llvm.sh build_llvm
+          build_tools/cmake/build_llvm.sh build_mlir
+
+      - name: "Tar dist"
+        if: ${{ !cancelled() }}
+        run: |
+          pushd third_party/llvm-project && llvm_sha_short=$(git rev-parse --short HEAD) && popd
+          tar cf llvm-dist-linux-$llvm_sha_short.tar llvm-install
+
+      - name: "Upload dist"
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: linux_x86_64_llvm_packages
+          path: llvm-dist-*.tar
+          if-no-files-found: error
+
+      - name: "Save cache"
+        uses: actions/cache/save@v3
+        # if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }}
+        if: ${{ !cancelled() }}
+        with:
+          path: ${{ env.CACHE_DIR }}
+          key: ${{ env.CACHE_KEY }}

.gitignore

@@ -0,0 +1,34 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+.idea

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "third_party/llvm-project"]
+	path = third_party/llvm-project
+	url = https://github.com/llvm/llvm-project.git
+	ignore = dirty
+	shallow = true

build_tools/cmake/build_llvm.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+TD="$(cd $(dirname $0) && pwd)"
+REPO_ROOT="$(cd $TD/../.. && pwd)"
+
+LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${REPO_ROOT}/third_party/llvm-project}"
+LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${REPO_ROOT}/../llvm-build}"
+LLVM_INSTALL_DIR="${LLVM_INSTALL_DIR:-${REPO_ROOT}/../llvm-install}"
+
+LLVM_SOURCE_DIR="$(realpath -m "${LLVM_SOURCE_DIR}")"
+LLVM_BUILD_DIR="$(realpath -m "${LLVM_BUILD_DIR}")"
+LLVM_INSTALL_DIR="$(realpath -m "${LLVM_INSTALL_DIR}")"
+
+echo "Paths canonicalized as:"
+echo "LLVM_SOURCE_DIR=${LLVM_SOURCE_DIR}"
+echo "LLVM_BUILD_DIR=${LLVM_BUILD_DIR}"
+echo "LLVM_INSTALL_DIR=${LLVM_INSTALL_DIR}"
+
+command="$1"
+shift
+
+has_ccache=false
+if (command -v ccache &> /dev/null); then
+  has_ccache=true
+fi
+
+has_clang=false
+if (command -v clang &> /dev/null) && (command -v clang++ &> /dev/null); then
+  has_clang=true
+fi
+
+has_lld=false
+if (command -v lld &> /dev/null); then
+  has_lld=true
+fi
+
+python3_command=""
+if (command -v python3 &> /dev/null); then
+  python3_command="python3"
+elif (command -v python &> /dev/null); then
+  python3_command="python"
+fi
+
+set -euo pipefail
+
+print_toolchain_config() {
+  if $has_ccache; then
+    echo -n "-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache "
+  fi
+  if $has_clang; then
+    echo "-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
+  fi
+}
+
+do_build_llvm() {
+  echo "*********************** BUILDING LLVM *********************************"
+  main_build_dir="${LLVM_BUILD_DIR}/llvm"
+  main_install_dir="${LLVM_INSTALL_DIR}/llvm"
+  targets_to_build="${LLVM_TARGETS_TO_BUILD:-X86}"
+
+  cmake_options="-DLLVM_TARGETS_TO_BUILD='${targets_to_build}'"
+  cmake_options="${cmake_options} -DCMAKE_BUILD_TYPE=Release"
+  cmake_options="${cmake_options} -C $TD/llvm_config.cmake"
+  cmake_options="${cmake_options} -DCMAKE_INSTALL_PREFIX=${main_install_dir}"
+  cmake_options="${cmake_options} $(print_toolchain_config)"
+  if $has_lld; then
+    cmake_options="${cmake_options} -DLLVM_ENABLE_LLD=ON"
+  fi
+
+  echo "Source Directory: ${LLVM_SOURCE_DIR}"
+  echo "Build Directory: ${main_build_dir}"
+  echo "CMake Options: ${cmake_options}"
+  cmake -GNinja -S "${LLVM_SOURCE_DIR}/llvm" -B "${main_build_dir}" \
+    $cmake_options
+  cmake --build "${main_build_dir}" \
+    --target install-toolchain-distribution \
+    --target install-development-distribution
+}
+
+do_build_mlir() {
+  echo "*********************** BUILDING MLIR *********************************"
+  main_install_dir="${LLVM_INSTALL_DIR}/llvm"
+  mlir_build_dir="${LLVM_BUILD_DIR}/mlir"
+  mlir_install_dir="${LLVM_INSTALL_DIR}/mlir"
+
+  cmake_options="-DLLVM_DIR='${main_install_dir}/lib/cmake/llvm'"
+  cmake_options="${cmake_options} -DPython3_EXECUTABLE='$(which $python3_command)'"
+  # Note: Building the MLIR Python bindings requires the installation of
+  # dependencies as specified in `mlir/python/requirements.txt`, which among
+  # others include pybind11.
+  cmake_options="${cmake_options} -DMLIR_ENABLE_BINDINGS_PYTHON=ON"
+  cmake_options="${cmake_options} -DCMAKE_INSTALL_PREFIX=${mlir_install_dir}"
+  cmake_options="${cmake_options} -C $TD/mlir_config.cmake"
+  cmake_options="${cmake_options} $(print_toolchain_config)"
+  if $has_lld; then
+    cmake_options="${cmake_options} -DLLVM_ENABLE_LLD=ON"
+  fi
+
+  echo "Source Directory: ${LLVM_SOURCE_DIR}"
+  echo "Build Directory: ${mlir_build_dir}"
+  echo "CMake Options: ${cmake_options}"
+  cmake -GNinja -S "${LLVM_SOURCE_DIR}/mlir" -B "${mlir_build_dir}" \
+    $cmake_options
+  cmake --build "${mlir_build_dir}" \
+    --target install-mlirdevelopment-distribution
+}
+
+case "${command}" in
+  build_llvm)
+    do_build_llvm
+    ;;
+
+  build_mlir)
+    do_build_mlir
+    ;;
+
+  *)
+    echo "ERROR: Expected command of 'build_llvm', 'build_mlir' (got '${command}')"
+    exit 1
+    ;;
+esac

build_tools/cmake/llvm_cache.cmake

@@ -0,0 +1,129 @@
+### Components built ###
+
+set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld" CACHE STRING "")
+
+### Target architectures ###
+
+# Compiler target architectures
+set(LLVM_TARGETS_TO_BUILD "X86" CACHE STRING "")
+
+set(LLVM_ENABLE_RUNTIMES "compiler-rt" CACHE STRING "")
+
+# CMake builtin variables and modules are not available for this cache file
+# Gather directly build/host information
+execute_process(COMMAND "uname" "-m" OUTPUT_VARIABLE _UNAME_M)
+string(STRIP ${_UNAME_M} BUILD_MACHINE_ARCH)
+
+### Default settings for the toolchain ###
+
+# Use the LLVM components
+set(CLANG_DEFAULT_OBJCOPY llvm-objcopy CACHE STRING "")
+set(CLANG_DEFAULT_LINKER lld CACHE STRING "")
+
+set(CLANG_ENABLE_STATIC_ANALYZER ON CACHE BOOL "")
+set(LLVM_ENABLE_LIBCXX OFF CACHE BOOL "")
+set(LLVM_ENABLE_ZLIB ON CACHE BOOL "")
+
+### Disk size optimizations ###
+
+# All the tools will use libllvm shared library
+set(LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
+set(LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
+
+# When exceptions are disabled, unwind tables are large and useless
+set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
+
+# Mildly useful misc stuff (which might also be hard to cross-compile)
+set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "")
+set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "")
+set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
+set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")
+set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
+set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "")
+set(LLVM_FORCE_ENABLE_STATS ON CACHE BOOL "")
+
+# Do not store debug information by default.
+set(CMAKE_BUILD_TYPE Release CACHE STRING "")
+
+# Use the distributions below for the installation
+set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "")
+
+### Distributions ###
+
+set(LLVM_DISTRIBUTIONS
+      Toolchain
+      Development
+    CACHE STRING "")
+
+set(LLVM_TOOLCHAIN_TOOLS
+  llvm-addr2line
+  llvm-ar
+  llvm-cxxfilt
+  llvm-dis
+  llvm-dwarfdump
+  llvm-lib
+  llvm-link
+  llvm-mc
+  llvm-nm
+  llvm-objcopy
+  llvm-objdump
+  llvm-rc
+  llvm-ranlib
+  llvm-readelf
+  llvm-readobj
+  llvm-size
+  llvm-strip
+  llvm-symbolizer
+  llvm-xray
+  CACHE STRING "")
+
+set(LLVM_BUILD_UTILS ON CACHE BOOL "")
+set(LLVM_INSTALL_UTILS ON CACHE BOOL "")
+set(LLVM_TOOLCHAIN_UTILITIES
+    FileCheck
+    count
+    not
+    CACHE STRING "")
+
+set(LLVM_RUNTIME_DISTRIBUTION_COMPONENTS
+    CACHE STRING "")
+
+set(LLVM_Toolchain_DISTRIBUTION_COMPONENTS
+      builtins
+      runtimes
+      clang
+      clang-cpp
+      clang-format
+      clang-offload-bundler
+      clang-resource-headers
+      clang-tidy
+      clangd
+      libclang
+      lld
+      LLVM
+      LTO
+      ${LLVM_TOOLCHAIN_TOOLS}
+      ${LLVM_TOOLCHAIN_UTILITIES}
+    CACHE STRING "")
+
+set(LLVM_Development_DISTRIBUTION_COMPONENTS
+      # LLVM
+      cmake-exports
+      development-cmake-exports
+      toolchain-cmake-exports
+      llc
+      llvm-config
+      llvm-headers
+      llvm-libraries
+      opt
+      Remarks
+      # Clang
+      clang-cmake-exports
+      clang-development-cmake-exports
+      clang-toolchain-cmake-exports
+      clang-headers
+      clang-libraries
+      # LLD
+      lld-cmake-exports
+      lld-toolchain-cmake-exports
+    CACHE STRING "")