[Python] Re-work Python bindings using upstream improvements. (#1484)

This is mostly mechanical for us. Some notable changes:

* Require a unified build for Python in CMake, docs, and CI
* New CMake defaults for some variables in CMakeLists.txt
* Use the new CMake functions for Python bindings and PyCDE
* Update imports for generated Python and Python extension libraries
* Update PYTHONPATH to reflect the new location under unified builds

.github/workflows/nightlyIntegrationTests.yml

@@ -7,49 +7,13 @@ on:
     - cron: 0 12 * * *
 
 jobs:
-  # Build the LLVM submodule then cache it. Do not rebuild if hit in the
-  # cache.
-  build-llvm:
-    name: Build LLVM
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/circt/images/circt-integration-test:v5
-    steps:
-      # Clone the CIRCT repo and its submodules. Do shallow clone to save clone
-      # time.
-      - name: Get CIRCT
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-          submodules: "true"
-
-      # Extract the LLVM submodule hash for use in the cache key.
-      - name: Get LLVM Hash
-        id: get-llvm-hash
-        run: echo "::set-output name=hash::$(git rev-parse @:./llvm)"
-        shell: bash
-
-      # Try to fetch LLVM from the cache.
-      - name: Cache LLVM
-        id: cache-llvm
-        uses: actions/cache@v2
-        with:
-          path: llvm
-          key: ${{ runner.os }}-llvm-python-${{ steps.get-llvm-hash.outputs.hash }}
-
-      # Build LLVM if we didn't hit in the cache.
-      - name: Rebuild and Install LLVM
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
-        run: utils/build-llvm.sh build install Release -DMLIR_ENABLE_BINDINGS_PYTHON=ON
-
   # Build CIRCT and run its tests using a Docker container with all the
   # integration testing prerequisite installed.
   build-circt:
     name: Build and Test
-    needs: build-llvm
     runs-on: ubuntu-18.04
     container:
-      image: ghcr.io/circt/images/circt-integration-test:v5
+      image: ghcr.io/circt/images/circt-integration-test:v6-beta3
     strategy:
       matrix:
         build-assert: [ON, OFF]
@@ -60,14 +24,8 @@ jobs:
             cxx: clang++
           - cc: gcc
             cxx: g++
-        include:
-          - build-shared: ON
-            bindings-python: ON
 
     steps:
-      - name: Configure Environment
-        run: echo "$GITHUB_WORKSPACE/llvm/install/bin" >> $GITHUB_PATH
-
       # Clone the CIRCT repo and its submodules. Do shallow clone to save clone
       # time.
       - name: Get CIRCT
@@ -76,28 +34,11 @@ jobs:
           fetch-depth: 1
           submodules: true
 
-      # --------
+      # Install Ccache
-      # Restore LLVM from cache and build if it's not in there.
+      - name: Ccache for C++ compilation
-      # --------
+        uses: hendrikmuhs/ccache-action@v1
-
-      # Extract the LLVM submodule hash for use in the cache key.
-      - name: Get LLVM Hash
-        id: get-llvm-hash
-        run: echo "::set-output name=hash::$(git rev-parse @:./llvm)"
-        shell: bash
-
-      # Try to fetch LLVM from the cache.
-      - name: Cache LLVM
-        id: cache-llvm
-        uses: actions/cache@v2
         with:
-          path: llvm
+          key: ${{ matrix.compiler.cc }}-${{ matrix.build-type }}-${{ matrix.build-shared }}-${{ matrix.build-assert }}
-          key: ${{ runner.os }}-llvm-python-${{ steps.get-llvm-hash.outputs.hash }}
-
-      # Build LLVM if we didn't hit in the cache.
-      - name: Rebuild and Install LLVM
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
-        run: utils/build-llvm.sh build install Release -DMLIR_ENABLE_BINDINGS_PYTHON=ON
 
       # --------
       # Build and test CIRCT
@@ -110,30 +51,25 @@ jobs:
           BUILD_ASSERT: ${{ matrix.build-assert }}
           BUILD_SHARED: ${{ matrix.build-shared }}
           BUILD_TYPE: ${{ matrix.build-type }}
-          BINDINGS_PYTHON: ${{matrix.bindings-python}}
         run: |
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
           mkdir build && cd build
-          cmake .. \
+          cmake -GNinja ../llvm/llvm \
             -DBUILD_SHARED_LIBS=$BUILD_SHARED \
             -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-            -DLLVM_ENABLE_ASSERTIONS=$BUILD_ASSERT \
-            -DMLIR_DIR=../llvm/install/lib/cmake/mlir/ \
-            -DLLVM_DIR=../llvm/install/lib/cmake/llvm/ \
-            -DCMAKE_LINKER=lld \
             -DCMAKE_C_COMPILER=$CC \
             -DCMAKE_CXX_COMPILER=$CXX \
-            -DLLVM_EXTERNAL_LIT=`pwd`/../llvm/build/bin/llvm-lit \
+            -DLLVM_CCACHE_BUILD=ON \
-            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+            -DLLVM_ENABLE_ASSERTIONS=$BUILD_ASSERT \
-            -DCIRCT_BINDINGS_PYTHON_ENABLED=$BINDINGS_PYTHON
+            -DLLVM_ENABLE_PROJECTS=mlir \
-      - name: Build CIRCT
+            -DLLVM_EXTERNAL_PROJECTS=circt \
-        run: |
+            -DLLVM_EXTERNAL_CIRCT_SOURCE_DIR=.. \
-          cd build
+            -DLLVM_USE_LINKER=lld \
-          make -j$(nproc)
+            -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
+            -DCIRCT_BINDINGS_PYTHON_ENABLED=ON
       - name: Test CIRCT
         run: |
-          cd build
+          ninja -C build check-circt -j$(nproc)
-          make check-circt -j$(nproc)
       - name: Integration Test CIRCT
         run: |
-          cd build
+          ninja -C build check-circt-integration -j$(nproc)
-          make check-circt-integration

CMakeLists.txt

@@ -4,7 +4,7 @@
 ##
 ##===----------------------------------------------------------------------===//
 
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0068)
   cmake_policy(SET CMP0068 NEW)
@@ -19,9 +19,29 @@ if(POLICY CMP0077)
   cmake_policy(SET CMP0077 NEW)
 endif()
 
+# CMake library generation settings.
+set(BUILD_SHARED_LIBS OFF CACHE BOOL "Default to building a static mondo-lib")
+set(CMAKE_PLATFORM_NO_VERSIONED_SONAME ON CACHE BOOL
+  "Python soname linked libraries are bad")
+set(CMAKE_VISIBILITY_INLINES_HIDDEN ON CACHE BOOL "Hide inlines")
+
+# The -fvisibility=hidden option only works for static builds.
+if (NOT BUILD_SHARED_LIBS)
+  set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+else()
+  if (CMAKE_CXX_VISIBILITY_PRESET STREQUAL "hidden")
+    message(FATAL_ERROR "CMAKE_CXX_VISIBILITY_PRESET=hidden is incompatible \
+                         with BUILD_SHARED_LIBS.")
+  endif()
+endif()
+
 # If we are not building as a part of LLVM, build Circt as an
 # standalone project, using LLVM as an external library:
 if( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
+  if (CIRCT_BINDINGS_PYTHON_ENABLED)
+    message(FATAL_ERROR "CIRCT Python bindings require a unified build. \
+                         See docs/PythonBindings.md.")
+  endif()
 
 #-------------------------------------------------------------------------------
 # Project setup and globals
@@ -97,6 +117,7 @@ set(CIRCT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(CIRCT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 set(CIRCT_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/include )
 set(CIRCT_TOOLS_DIR ${CMAKE_BINARY_DIR}/bin)
+set(CIRCT_PYTHON_PACKAGES_DIR ${CIRCT_BINARY_DIR}/python_packages)
 
 list(APPEND CMAKE_MODULE_PATH "${MLIR_MAIN_SRC_DIR}/cmake/modules")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")

docs/PythonBindings.md

@@ -1,13 +1,19 @@
 # Using the Python Bindings
 
-If you are mainly interested in using CIRCT from Python scripts, you need to compile both LLVM/MLIR and CIRCT with Python bindings enabled. To do this, follow the steps in ["Setting this up"](GettingStarted.md#setting-this-up), adding the following options:
+If you are mainly interested in using CIRCT from Python scripts, you need to compile both LLVM/MLIR and CIRCT with Python bindings enabled. Furthermore, you must use a unified build, where LLVM/MLIR and CIRCT are compiled together in one step. To do this, you can use a single CMake invocation like this:
 
 ```
-# in the LLVM/MLIR build directory
+$ cd circt
-cmake [...] -DMLIR_ENABLE_BINDINGS_PYTHON=ON
+$ mkdir build
-
+$ cd build
-# in the CIRCT build directory
+$ cmake -G Ninja ../llvm/llvm \
-cmake [...] -DCIRCT_BINDINGS_PYTHON_ENABLED=ON
+    -DCMAKE_BUILD_TYPE=Debug \
+    -DLLVM_ENABLE_PROJECTS=mlir \
+    -DLLVM_ENABLE_ASSERTIONS=ON \
+    -DLLVM_EXTERNAL_PROJECTS=circt \
+    -DLLVM_EXTERNAL_CIRCT_SOURCE_DIR=.. \
+    -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
+    -DCIRCT_BINDINGS_PYTHON_ENABLED=ON
 ```
 
 Afterwards, use `ninja check-circt-integration` to ensure that the bindings work. (This will now additionally spin up a couple of Python scripts to test that they are accessible.)
@@ -17,7 +23,7 @@ Afterwards, use `ninja check-circt-integration` to ensure that the bindings work
 If you want to try the bindings fresh from the compiler without installation, you need to ensure Python can find the generated modules:
 
 ```
-export PYTHONPATH="$PWD/llvm/build/python:$PWD/build/python:$PYTHONPATH"
+export PYTHONPATH="$PWD/llvm/build/tools/circt/python_packages/circt_core"
 ```
 
 ## With Installation
@@ -40,9 +46,9 @@ with Context() as ctx, Location.unknown():
   i42 = IntegerType.get_signless(42)
   m = Module.create()
   with InsertionPoint(m.body):
-    @hw.HWModuleOp.from_py_func(i42, i42)
     def magic(a, b):
       return comb.XorOp(i42, [a, b]).result
+    hw.HWModuleOp(name="magic", body_builder=magic)
   print(m)
 ```
 

frontends/PyCDE/src/CMakeLists.txt

@@ -6,33 +6,13 @@
 #
 # ===-----------------------------------------------------------------------===//
 
-file(GLOB_RECURSE PY_SRC_FILES
+declare_mlir_python_sources(PyCDESources
-  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
-  "${CMAKE_CURRENT_SOURCE_DIR}/pycde/*.py")
+  SOURCES_GLOB pycde/*.py)
 
-add_custom_target(PyCDE ALL
+add_mlir_python_modules(PyCDE
-  DEPENDS ${PY_SRC_FILES}
+  ROOT_PREFIX "${CIRCT_PYTHON_PACKAGES_DIR}/pycde"
-)
+  INSTALL_PREFIX "python_packages/pycde"
-
+  DECLARED_SOURCES
-foreach(PY_SRC_FILE ${PY_SRC_FILES})
+    PyCDESources
-  set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/${PY_SRC_FILE}")
-  get_filename_component(PY_DEST_DIR "${PY_DEST_FILE}" DIRECTORY)
-  file(MAKE_DIRECTORY "${PY_DEST_DIR}")
-  add_custom_command(
-    TARGET PyCDE PRE_BUILD
-    COMMENT "Symlinking Python source ${PY_SRC_FILE} -> ${PY_DEST_FILE}"
-    DEPENDS "${PY_SRC_FILE}"
-    COMMAND "${CMAKE_COMMAND}" -E create_symlink
-        "${CMAKE_CURRENT_SOURCE_DIR}/${PY_SRC_FILE}" "${PY_DEST_FILE}"
-  )
-endforeach()
-
-# Note that we copy from the source tree just like for headers because
-# it will not be polluted with py_cache runtime artifacts (from testing and
-# such).
-install(
-  DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pycde
-  DESTINATION python
-  COMPONENT PyCDE
-  FILES_MATCHING PATTERN "*.py"
 )

frontends/PyCDE/test/CMakeLists.txt

@@ -9,14 +9,8 @@
 set(PyCDE_TEST_DEPENDS
   FileCheck count not
   PyCDE
-  CIRCTBindingsPython
+  CIRCTMLIRPythonModules
-  CIRCTBindingsPythonCombOps
+  CIRCTPythonModules
-  CIRCTBindingsPythonESIOps
-  CIRCTBindingsPythonHWOps
-  CIRCTBindingsPythonSVOps
-  CIRCTBindingsPythonSeqOps
-  CIRCTBindingsPythonSources
-  CIRCTBindingsPythonExtension
   )
 
 configure_lit_site_cfg(

frontends/PyCDE/test/lit.cfg.py

@@ -65,7 +65,7 @@ llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
 
 # Tweak the PYTHONPATH to include the binary dir.
 llvm_config.with_environment('PYTHONPATH', [
-    os.path.join(config.llvm_obj_root, 'python'),
+    os.path.join(config.circt_python_packages_dir, 'circt_core'),
-    os.path.join(config.circt_obj_root, 'python')
+    os.path.join(config.circt_python_packages_dir, 'pycde')
 ],
                              append_path=True)

frontends/PyCDE/test/lit.site.cfg.py.in

@@ -36,6 +36,7 @@ config.circt_obj_root = "@CIRCT_BINARY_DIR@"
 config.circt_tools_dir = "@CIRCT_TOOLS_DIR@"
 config.circt_include_dir = "@CIRCT_MAIN_INCLUDE_DIR@"
 config.circt_shlib_dir = "@LLVM_LIBRARY_OUTPUT_INTDIR@"
+config.circt_python_packages_dir = "@CIRCT_PYTHON_PACKAGES_DIR@"
 config.verilator_path = "@VERILATOR_PATH@"
 config.esi_cosim_path = "@ESI_COSIM_PATH@"
 config.timeout = "@CIRCT_INTEGRATION_TIMEOUT@"

integration_test/CMakeLists.txt

@@ -3,7 +3,7 @@ llvm_canonicalize_cmake_booleans(
   )
 
 set(CIRCT_INTEGRATION_TEST_DEPENDS
-  FileCheck count not
+  FileCheck count not split-file
   circt-opt
   circt-translate
   circt-rtl-sim
@@ -12,7 +12,8 @@ set(CIRCT_INTEGRATION_TEST_DEPENDS
 
 # If Python bindings are available to build then enable the tests.
 if(CIRCT_BINDINGS_PYTHON_ENABLED)
-  list(APPEND CIRCT_INTEGRATION_TEST_DEPENDS CIRCTBindingsPython)
+  list(APPEND CIRCT_INTEGRATION_TEST_DEPENDS CIRCTMLIRPythonModules)
+  list(APPEND CIRCT_INTEGRATION_TEST_DEPENDS CIRCTPythonModules)
 endif()
 
 # If ESI Cosim is available to build then enable its tests.

integration_test/lit.cfg.py

@@ -65,10 +65,9 @@ llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
 
 # Tweak the PYTHONPATH to include the binary dir.
 if config.bindings_python_enabled:
-  llvm_config.with_environment('PYTHONPATH', [
+  llvm_config.with_environment(
-      os.path.join(config.llvm_obj_root, 'python'),
+      'PYTHONPATH',
-      os.path.join(config.circt_obj_root, 'python')
+      [os.path.join(config.circt_python_packages_dir, 'circt_core')],
-  ],
       append_path=True)
 
 tool_dirs = [

integration_test/lit.site.cfg.py.in

@@ -36,6 +36,7 @@ config.circt_obj_root = "@CIRCT_BINARY_DIR@"
 config.circt_tools_dir = "@CIRCT_TOOLS_DIR@"
 config.circt_include_dir = "@CIRCT_MAIN_INCLUDE_DIR@"
 config.circt_shlib_dir = "@LLVM_LIBRARY_OUTPUT_INTDIR@"
+config.circt_python_packages_dir = "@CIRCT_PYTHON_PACKAGES_DIR@"
 config.verilator_path = "@VERILATOR_PATH@"
 config.esi_cosim_path = "@ESI_COSIM_PATH@"
 config.timeout = "@CIRCT_INTEGRATION_TIMEOUT@"

lib/Bindings/Python/CMakeLists.txt

@@ -3,21 +3,30 @@
 ################################################################################
 
 include(AddMLIRPython)
-add_custom_target(CIRCTBindingsPython)
 
 ################################################################################
-# Build native Python extension
+# Declare native Python extension
 ################################################################################
 
-add_mlir_python_extension(CIRCTBindingsPythonExtension _circt
+declare_mlir_python_sources(CIRCTBindingsPythonExtension)
-  INSTALL_DIR
+
-    python
+set(_depends LLVMSupport)
+
+# TODO: remove these when ESIModule only uses C-APIs.
+if (BUILD_SHARED_LIBS)
+  list(APPEND _depends MLIRIR)
+  list(APPEND _depends MLIRParser)
+endif()
+
+declare_mlir_python_extension(CIRCTBindingsPythonExtension.Core
+  MODULE_NAME _circt
+  ADD_TO_PARENT CIRCTBindingsPythonExtension
   SOURCES
     CIRCTModule.cpp
     ESIModule.cpp
     HWModule.cpp
     MSFTModule.cpp
-  LINK_LIBS
+  EMBED_CAPI_LINK_LIBS
     CIRCTCAPIComb
     CIRCTCAPIESI
     CIRCTCAPIMSFT
@@ -25,100 +34,112 @@ add_mlir_python_extension(CIRCTBindingsPythonExtension _circt
     CIRCTCAPISeq
     CIRCTCAPISV
     CIRCTCAPIExportVerilog
-    MLIRPythonCAPI
+  PRIVATE_LINK_LIBS
-)
+    ${_depends}
-add_dependencies(CIRCTBindingsPython CIRCTBindingsPythonExtension)
-
-################################################################################
-# Python source tree management
-################################################################################
-
-file(GLOB_RECURSE PY_SRC_FILES
-  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
-  "${CMAKE_CURRENT_SOURCE_DIR}/circt/*.py")
-
-add_custom_target(CIRCTBindingsPythonSources ALL
-  DEPENDS ${PY_SRC_FILES}
-)
-add_dependencies(CIRCTBindingsPython CIRCTBindingsPythonSources)
-
-foreach(PY_SRC_FILE ${PY_SRC_FILES})
-  set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/${PY_SRC_FILE}")
-  get_filename_component(PY_DEST_DIR "${PY_DEST_FILE}" DIRECTORY)
-  file(MAKE_DIRECTORY "${PY_DEST_DIR}")
-  add_custom_command(
-    TARGET CIRCTBindingsPythonSources PRE_BUILD
-    COMMENT "Symlinking Python source ${PY_SRC_FILE} -> ${PY_DEST_FILE}"
-    DEPENDS "${PY_SRC_FILE}"
-    COMMAND "${CMAKE_COMMAND}" -E create_symlink
-        "${CMAKE_CURRENT_SOURCE_DIR}/${PY_SRC_FILE}" "${PY_DEST_FILE}"
-  )
-endforeach()
-
-# The Python sources, including extensions, are in the circt.dialects namespace,
-# as expected. But, at the moment, the bindings are generated in the
-# mlir.dialects namespace, and look for Python extension modules there. To shim
-# this, symlink the extension modules in mlir.dialects.
-file(GLOB_RECURSE PY_EXT_FILES
-  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/circt/dialects"
-  "${CMAKE_CURRENT_SOURCE_DIR}/circt/*_ops_ext.py")
-foreach(PY_EXT_FILE ${PY_EXT_FILES})
-  set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/mlir/dialects/${PY_EXT_FILE}")
-  set(PY_SRC_FILE "${PROJECT_BINARY_DIR}/python/circt/dialects/${PY_EXT_FILE}")
-  add_custom_command(
-    TARGET CIRCTBindingsPythonSources POST_BUILD
-    COMMENT "Symlinking Python extension ${PY_SRC_FILE} -> ${PY_DEST_FILE}"
-    DEPENDS "${PY_SRC_FILE}"
-    COMMAND "${CMAKE_COMMAND}" -E create_symlink "${PY_SRC_FILE}" "${PY_DEST_FILE}"
-  )
-endforeach()
-
-# Note that we copy from the source tree just like for headers because
-# it will not be polluted with py_cache runtime artifacts (from testing and
-# such).
-install(
-  DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/circt
-  DESTINATION python
-  COMPONENT CIRCTBindingsPythonSources
-  FILES_MATCHING PATTERN "*.py"
-)
-
-# Dialect sources are generated. Install separately.
-# Note that __pycache__ directories may have been left by tests and other
-# executions. And __init__.py is handled as a regular source file.
-install(
-  DIRECTORY ${PROJECT_BINARY_DIR}/python/mlir/dialects
-  DESTINATION python/mlir
-  COMPONENT CIRCTBindingsPythonDialects
-  FILES_MATCHING PATTERN "*.py"
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "__init__.py" EXCLUDE
 )
 
 ################################################################################
-# Generate dialect-specific bindings.
+# Declare Python sources
 ################################################################################
-add_mlir_dialect_python_bindings(CIRCTBindingsPythonCombOps
+
-  TD_FILE CombOps.td
+declare_mlir_python_sources(CIRCTBindingsPythonSources
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
+  SOURCES
+    circt/__init__.py
+    circt/esi.py
+    circt/support.py
+    circt/dialects/_ods_common.py)
+
+################################################################################
+# Declare dialect-specific bindings.
+################################################################################
+declare_mlir_python_sources(CIRCTBindingsPythonSources.Dialects
+  ADD_TO_PARENT CIRCTBindingsPythonSources)
+
+declare_mlir_dialect_python_bindings(
+  ADD_TO_PARENT CIRCTBindingsPythonSources.Dialects
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
+  TD_FILE circt/dialects/CombOps.td
+  SOURCES
+    circt/dialects/comb.py
+    circt/dialects/_comb_ops_ext.py
   DIALECT_NAME comb)
-add_dependencies(CIRCTBindingsPythonSources CIRCTBindingsPythonCombOps)
 
-add_mlir_dialect_python_bindings(CIRCTBindingsPythonESIOps
+declare_mlir_dialect_python_bindings(
-  TD_FILE ESIOps.td
+  ADD_TO_PARENT CIRCTBindingsPythonSources.Dialects
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
+  TD_FILE circt/dialects/ESIOps.td
+  SOURCES circt/dialects/esi.py
   DIALECT_NAME esi)
-add_dependencies(CIRCTBindingsPythonSources CIRCTBindingsPythonESIOps)
 
-add_mlir_dialect_python_bindings(CIRCTBindingsPythonHWOps
+declare_mlir_dialect_python_bindings(
-  TD_FILE HWOps.td
+  ADD_TO_PARENT CIRCTBindingsPythonSources.Dialects
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
+  TD_FILE circt/dialects/HWOps.td
+  SOURCES
+    circt/dialects/hw.py
+    circt/dialects/_hw_ops_ext.py
   DIALECT_NAME hw)
-add_dependencies(CIRCTBindingsPythonSources CIRCTBindingsPythonHWOps)
 
-add_mlir_dialect_python_bindings(CIRCTBindingsPythonSeqOps
+declare_mlir_dialect_python_bindings(
-  TD_FILE SeqOps.td
+  ADD_TO_PARENT CIRCTBindingsPythonSources.Dialects
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
+  TD_FILE circt/dialects/SeqOps.td
+  SOURCES
+    circt/dialects/seq.py
+    circt/dialects/_seq_ops_ext.py
   DIALECT_NAME seq)
-add_dependencies(CIRCTBindingsPythonSources CIRCTBindingsPythonSeqOps)
 
-add_mlir_dialect_python_bindings(CIRCTBindingsPythonSVOps
+declare_mlir_dialect_python_bindings(
-  TD_FILE SVOps.td
+  ADD_TO_PARENT CIRCTBindingsPythonSources.Dialects
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
+  TD_FILE circt/dialects/SVOps.td
+  SOURCES circt/dialects/sv.py
   DIALECT_NAME sv)
-add_dependencies(CIRCTBindingsPythonSources CIRCTBindingsPythonSVOps)
+
+################################################################################
+# Build composite binaries
+################################################################################
+
+# Bundle our own, self-contained CAPI library with all of our deps.
+add_mlir_python_common_capi_library(CIRCTBindingsPythonCAPI
+  INSTALL_COMPONENT CIRCTBindingsPythonModules
+  INSTALL_DESTINATION python_packages/circt_core/mlir/_mlir_libs
+  # NOTE: When the MLIR API is relocated under circt, this would change to
+  # .../circt/_mlir_libs
+  OUTPUT_DIRECTORY "${CIRCT_PYTHON_PACKAGES_DIR}/circt_core/mlir/_mlir_libs"
+  RELATIVE_INSTALL_ROOT "../../../.."
+  DECLARED_SOURCES
+    # TODO: This can be chopped down significantly for size.
+    MLIRPythonSources
+    MLIRPythonExtension.AllPassesRegistration
+    CIRCTBindingsPythonSources
+    CIRCTBindingsPythonExtension
+)
+
+# Bundle the MLIR python sources into our package.
+# The MLIR API is position independent, so we explicitly output it to the mlir/
+# folder as a temporary measure. It will eventually migrate under the circt/
+# folder and be accessible under the unified "import circt..." namespace.
+add_mlir_python_modules(CIRCTMLIRPythonModules
+  ROOT_PREFIX "${CIRCT_PYTHON_PACKAGES_DIR}/circt_core/mlir"
+  INSTALL_PREFIX "python_packages/circt_core/mlir"
+  DECLARED_SOURCES
+    MLIRPythonSources
+    MLIRPythonExtension.AllPassesRegistration
+    # We need the circt extensions co-located with the MLIR extensions. When
+    # the namespace is unified, this moves to the below.
+    CIRCTBindingsPythonExtension
+  COMMON_CAPI_LINK_LIBS
+    CIRCTBindingsPythonCAPI
+)
+
+# Bundle the CIRCT python sources into our package.
+add_mlir_python_modules(CIRCTPythonModules
+  ROOT_PREFIX "${CIRCT_PYTHON_PACKAGES_DIR}/circt_core"
+  INSTALL_PREFIX "python_packages/circt_core"
+  DECLARED_SOURCES
+    CIRCTBindingsPythonSources
+  COMMON_CAPI_LINK_LIBS
+    CIRCTBindingsPythonCAPI
+)

lib/Bindings/Python/ESIModule.cpp

@@ -35,6 +35,7 @@ using namespace circt::esi;
 // The main entry point into the ESI Assembly API.
 //===----------------------------------------------------------------------===//
 
+/// TODO: move this to only use C-APIs.
 class System {
 public:
   /// Construct an ESI system. The Python bindings really want to own the MLIR

lib/Bindings/Python/circt/__init__.py

@@ -2,5 +2,4 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Simply a wrapper around the extension module of the same name.
+from mlir._mlir_libs._circt import *
-from _circt import *

lib/Bindings/Python/circt/dialects/comb.py

@@ -2,8 +2,7 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Generated tablegen dialects end up in the mlir.dialects package for now.
+from ._comb_ops_gen import *
-from mlir.dialects._comb_ops_gen import *
 
 from circt.support import NamedValueOpView
 

lib/Bindings/Python/circt/dialects/esi.py

@@ -2,5 +2,4 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Generated tablegen dialects end up in the mlir.dialects package for now.
+from ._esi_ops_gen import *
-from mlir.dialects._esi_ops_gen import *

lib/Bindings/Python/circt/dialects/hw.py

@@ -2,6 +2,5 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Generated tablegen dialects end up in the mlir.dialects package for now.
+from ._hw_ops_gen import *
-from mlir.dialects._hw_ops_gen import *
+from mlir._mlir_libs._circt._hw import *
-from _circt._hw import *

lib/Bindings/Python/circt/dialects/seq.py

@@ -2,8 +2,7 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Generated tablegen dialects end up in the mlir.dialects package for now.
+from ._seq_ops_gen import *
-from mlir.dialects._seq_ops_gen import *
 
 from .seq import CompRegOp
 

lib/Bindings/Python/circt/dialects/sv.py

@@ -2,5 +2,4 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Generated tablegen dialects end up in the mlir.dialects package for now.
+from ._sv_ops_gen import *
-from mlir.dialects._sv_ops_gen import *

lib/Bindings/Python/circt/esi.py

@@ -5,7 +5,7 @@
 from mlir.passmanager import PassManager
 import mlir.ir
 
-from _circt._esi import *
+from mlir._mlir_libs._circt._esi import *
 import circt
 from circt.dialects import hw
 from circt.dialects.esi import *

llvm

@@ -1 +1 @@
-Subproject commit 345ace026b6e5cdbc38d207291e4b399d72e62ee
+Subproject commit 310c9496d80961188e8d8f8ad306cdf44bd7541f