Skip to content

Commit

Permalink
[ESI Runtime] Replace Cap'nProto with gRPC (#7217)
Browse files Browse the repository at this point in the history
After spending a truly obnoxious amount of time fighting capnp and
libkj, we made the decision to switch to another RPC system. We're no
longer modeling and serializing message types in Capnp and we don't need
the performance which capnp/libkj RPC promises, so there's really no
need for the additional complexity. A slower system which is thread safe
should work fine.

This commit breaks the build in a pretty horrible way and is not
intended to be merged on its own. It simply breaks up the diff.
  • Loading branch information
teqdruid authored Jun 21, 2024
1 parent ac858d8 commit 88eeb26
Show file tree
Hide file tree
Showing 29 changed files with 998 additions and 1,303 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/nightlyIntegrationTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
# John and re-run the job.
runs-on: ["self-hosted", "1ES.Pool=1ES-CIRCT-builds", "linux"]
container:
image: ghcr.io/circt/images/circt-integration-test:v13.1
image: ghcr.io/circt/images/circt-integration-test:v15.0
volumes:
- /mnt:/__w/circt
strategy:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/shortIntegrationTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
# John and re-run the job.
runs-on: ["self-hosted", "1ES.Pool=1ES-CIRCT-builds", "linux"]
container:
image: ghcr.io/circt/images/circt-integration-test:v13.1
image: ghcr.io/circt/images/circt-integration-test:v15.0
volumes:
- /mnt:/__w/circt
strategy:
Expand Down
4 changes: 3 additions & 1 deletion docs/Dialects/ESI/cosim.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ variety of languages: the Cap'nProto website lists C++, C#, Erlang, Go,
Haskell, JavaScript, Ocaml, Python, and Rust as languages which support
messages and RPC!

Status: **prototype**
Status: **production**
Documentation status: **out of date** needs to be updated to reflect the new RPC
interface and library.

## Usage

Expand Down
2 changes: 1 addition & 1 deletion frontends/PyCDE/integration_test/esitester.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# RUN: rm -rf %t
# RUN: mkdir %t && cd %t
# RUN: %PYTHON% %s %t 2>&1
# RUN: esi-cosim.py -- esitester cosim env | FileCheck %s
# RUN: esi-cosim.py --source %t -- esitester cosim env wait | FileCheck %s

import pycde
from pycde import AppID, Clock, Module, Reset, generator
Expand Down
1 change: 1 addition & 0 deletions integration_test/Dialect/ESI/runtime/basic_mmio.sv
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// REQUIRES: esi-cosim, esi-runtime, rtl-sim
// XFAIL: *
// RUN: rm -rf %t && mkdir %t && cp %s %t
// RUN: esi-cosim.py --source %t --top top -- %python %s.py cosim env

Expand Down
79 changes: 28 additions & 51 deletions lib/Dialect/ESI/runtime/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ find_package(ZLIB REQUIRED)

# JSON parser for the manifest.
if (NOT TARGET nlohmann_json)
message(" -- ESI runtime pulling down json")
message("-- ESI runtime pulling down json")
FetchContent_Declare(json
GIT_REPOSITORY https://github.com/nlohmann/json.git
GIT_TAG v3.11.3
Expand Down Expand Up @@ -94,49 +94,37 @@ ENDIF(MSVC)

option(ESI_COSIM "Enable ESI cosimulation." ON)
if(ESI_COSIM)
# gRPC for cosimulation. Local install required.
option(GRPC_PATH "Location of gRPC install.")
if (${GRPC_PATH})
find_package(Protobuf REQUIRED CONFIG HINTS ${GRPC_PATH})
find_package(gRPC REQUIRED CONFIG HINTS ${GRPC_PATH})
else()
find_package(Protobuf REQUIRED CONFIG)
find_package(gRPC REQUIRED CONFIG)
endif()

add_subdirectory(cosim)
# Inform the runtime code that Cosimulation is enabled. Kinda hacky since all
# backends should only need to be linked in.
# TODO: Once the hack in the python bindings is remidied, remove this.
add_compile_definitions(ESI_COSIM)

# Try to find Cap'nProto. If the user has set CAPNP_PATH, use that.
if(DEFINED CAPNP_PATH)
set(ENV{PKG_CONFIG_PATH}
"${CAPNP_PATH}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}")
find_package(CapnProto CONFIG PATHS ${CAPNP_PATH})
else()
set(ENV{PKG_CONFIG_PATH}
"${CMAKE_CURRENT_SOURCE_DIR}/ext/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}")
find_package(CapnProto CONFIG PATHS "${CMAKE_CURRENT_SOURCE_DIR}/ext")
endif()

# If Cap'nProto has been found, generate the headers and definitions.
if(CapnProto_FOUND)
message("-- ESI cosim enabled")

message(STATUS "Found Cap'nProto at ${CapnProto_DIR}.")
add_subdirectory(cosim)

set(ESIRuntimeSources
${ESIRuntimeSources}
${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/backends/Cosim.cpp
)
set(ESIRuntimeBackendHeaders
${ESIRuntimeBackendHeaders}
${CMAKE_CURRENT_SOURCE_DIR}/cpp/include/esi/backends/Cosim.h
)
set(ESIRuntimeLinkLibraries
${ESIRuntimeLinkLibraries}
EsiCosimCapnp
)
set(ESIRuntimeIncludeDirs
${ESIRuntimeIncludeDirs}
${CMAKE_CURRENT_SOURCE_DIR}/cosim/include
)
else()
message(FATAL_ERROR "ESI cosimulation requires Cap'nProto. Either install
Cap'nProto or disable ESI cosim with -DESI_COSIM=OFF.")
endif()
set(ESIRuntimeSources
${ESIRuntimeSources}
${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/backends/Cosim.cpp
)
set(ESIRuntimeBackendHeaders
${ESIRuntimeBackendHeaders}
${CMAKE_CURRENT_SOURCE_DIR}/cpp/include/esi/backends/Cosim.h
)
set(ESIRuntimeLinkLibraries
${ESIRuntimeLinkLibraries}
EsiCosimGRPC
)
set(ESIRuntimeIncludeDirs
${ESIRuntimeIncludeDirs}
${CMAKE_CURRENT_SOURCE_DIR}/cosim/include
)
else()
message("-- ESI cosim disabled")
endif()
Expand Down Expand Up @@ -193,17 +181,6 @@ install(TARGETS ESIRuntime
DESTINATION lib
COMPONENT ESIRuntime
)
install(IMPORTED_RUNTIME_ARTIFACTS ESIRuntime
RUNTIME_DEPENDENCY_SET ESIRuntime_RUNTIME_DEPS
DESTINATION lib
COMPONENT ESIRuntime
)
install(RUNTIME_DEPENDENCY_SET ESIRuntime_RUNTIME_DEPS
DESTINATION lib
PRE_EXCLUDE_REGEXES .*
PRE_INCLUDE_REGEXES capnp kj
COMPONENT ESIRuntime
)
install(FILES ${ESIRuntimeHeaders}
DESTINATION include/esi
COMPONENT ESIRuntime-dev
Expand Down
65 changes: 24 additions & 41 deletions lib/Dialect/ESI/runtime/cosim/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,6 @@
##
##===----------------------------------------------------------------------===//

# Compile Capnp file.
add_definitions(${CAPNP_DEFINITIONS})
set(CAPNPC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/../cpp/include/backends)
include_directories(${CAPNPC_OUTPUT_DIR})
file(MAKE_DIRECTORY ${CAPNPC_OUTPUT_DIR})
capnp_generate_cpp(
COSIM_CAPNP_SRCS COSIM_CANPN_HDRS
"CosimDpi.capnp"
)

# Compile a library for ESI cosim capnp for both the API runtime backend and the
# cosim DPI server to use.
add_library(EsiCosimCapnp SHARED
${COSIM_CAPNP_HDRS}
${COSIM_CAPNP_SRCS}
${COSIM_SCHEMA_HDR}

lib/CapnpThreads.cpp
lib/Client.cpp
lib/Endpoint.cpp
lib/Server.cpp
)
target_include_directories(EsiCosimCapnp PUBLIC ${CAPNPC_OUTPUT_DIR})
target_include_directories(EsiCosimCapnp PUBLIC ${CAPNP_INCLUDE_DIRS})
target_include_directories(EsiCosimCapnp PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/../cpp/include)
target_link_libraries(EsiCosimCapnp PUBLIC
CapnProto::kj CapnProto::kj-async CapnProto::kj-gzip
CapnProto::capnp CapnProto::capnp-rpc
)
install(TARGETS EsiCosimCapnp
DESTINATION lib
COMPONENT ESIRuntime
)

set(cosim_collateral
Cosim_DpiPkg.sv
Cosim_Endpoint.sv
Expand Down Expand Up @@ -73,13 +38,31 @@ install(FILES
COMPONENT ESIRuntime
)

# Cap'nProto MUST be built with exceptions enabled.
if (MSVC)
target_compile_options(EsiCosimCapnp PRIVATE /EHsc)
else()
target_compile_options(EsiCosimCapnp PRIVATE -fexceptions)
endif()
add_library(EsiCosimGRPC OBJECT "${CMAKE_CURRENT_LIST_DIR}/cosim.proto")
target_link_libraries(EsiCosimGRPC PUBLIC protobuf::libprotobuf gRPC::grpc++)
set(PROTO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
target_include_directories(EsiCosimGRPC PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>")

protobuf_generate(
TARGET EsiCosimGRPC
PROTOC_OUT_DIR "${PROTO_BINARY_DIR}")
protobuf_generate(
TARGET EsiCosimGRPC
LANGUAGE grpc
GENERATE_EXTENSIONS .grpc.pb.h .grpc.pb.cc
PLUGIN "protoc-gen-grpc=\$<TARGET_FILE:gRPC::grpc_cpp_plugin>"
PROTOC_OUT_DIR "${PROTO_BINARY_DIR}")

include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include")

add_library(CosimRpcServer
lib/RpcServer.cpp
)
target_link_libraries(CosimRpcServer
PUBLIC
EsiCosimGRPC
ESIRuntime
)

add_subdirectory(cosim_dpi_server)
add_subdirectory(MtiPliStub)
60 changes: 0 additions & 60 deletions lib/Dialect/ESI/runtime/cosim/CosimDpi.capnp

This file was deleted.

2 changes: 1 addition & 1 deletion lib/Dialect/ESI/runtime/cosim/Cosim_DpiPkg.sv
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ import "DPI-C" sv2cCosimserverEpTryGet =

import "DPI-C" sv2cCosimserverSetManifest =
function void cosim_set_manifest(
input int unsigned esi_version,
input int signed esi_version,
input byte unsigned compressed_manifest[]
);

Expand Down
68 changes: 68 additions & 0 deletions lib/Dialect/ESI/runtime/cosim/cosim.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
//===- cosim.proto - ESI cosim RPC definitions ------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// The ESI cosimulation gRPC schema. If something (client or server) wants to
// talk to an ESI runtime, it mergely needs to implement this schema. If
// possible, however, it is encouraged to use the C++ esiruntime API as that is
// expected to be more portable and supports more than just cosim.
//
//===----------------------------------------------------------------------===//

syntax = "proto3";

package esi.cosim;

// Description of a channel that can be connected to by the client.
message ChannelDesc {
string name = 1;

enum Direction {
TO_SERVER = 0;
TO_CLIENT = 1;
}
Direction dir = 2;

string type = 3;
}

// List of channels that the client can connect to.
message ListOfChannels { repeated ChannelDesc channels = 1; }

// Empty message since gRPC only supports exactly one argument and return.
message VoidMessage {}

// The manifest package.
message Manifest {
int32 esi_version = 1;
bytes compressed_manifest = 2;
}

// An ESI message.
message Message { bytes data = 1; }

// An ESI message and the channel to which is should be directed.
message AddressedMessage {
string channel_name = 1;
Message message = 2;
}

// The server interface provided by the ESI cosim server.
service ChannelServer {
// Get the manifest embedded in the accelertor.
rpc GetManifest(VoidMessage) returns (Manifest) {}

// List the channels that the client can connect to.
rpc ListChannels(VoidMessage) returns (ListOfChannels) {}

// Send a message to the server.
rpc SendToServer(AddressedMessage) returns (VoidMessage) {}

// Connect to a client channel and return a stream of messages coming from
// that channel.
rpc ConnectToClientChannel(ChannelDesc) returns (stream Message) {}
}
10 changes: 8 additions & 2 deletions lib/Dialect/ESI/runtime/cosim/cosim_dpi_server/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,14 @@ set_target_properties(EsiCosimDpiServer
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
CXX_VISIBILITY_PRESET "default"
)
add_dependencies(EsiCosimDpiServer EsiCosimCapnp MtiPli)
target_link_libraries(EsiCosimDpiServer PRIVATE EsiCosimCapnp MtiPli)
add_dependencies(EsiCosimDpiServer ESIRuntime MtiPli)
target_link_libraries(EsiCosimDpiServer
PRIVATE
ESIRuntime
CosimRpcServer
MtiPli
)

install(TARGETS EsiCosimDpiServer
DESTINATION lib
COMPONENT ESIRuntime
Expand Down
Loading

0 comments on commit 88eeb26

Please sign in to comment.