Adds a C++ parser for our EVENT logs

.github/workflows/build-extras-cpp-eventanalyze.yml

@@ -0,0 +1,88 @@
+name: Build extras/analyze
+
+on:
+  workflow_dispatch:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: true
+        default: '3.8-dev'
+      ref:
+        description: 'The OptSched git ref to checkout to build'
+        required: true
+        default: 'master'
+      build_type:
+        description: 'CMAKE_BUILD_TYPE'
+        required: true
+        default: 'Release'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Install APT dependencies
+        run: |
+          # For parallel STL
+          sudo apt-get install libtbb-dev
+
+          # For latest C++ features
+          sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+          sudo apt-get update
+
+          sudo apt-get install g++-11
+
+          # For pyenv python
+          sudo apt-get install make build-essential libssl-dev zlib1g-dev \
+            libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+            libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+      - name: Install Python version
+        run: |
+          curl https://pyenv.run | bash
+          eval "$(pyenv init --path)"
+          echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV
+          echo "$HOME/.pyenv/bin" >> $GITHUB_PATH
+
+          export PYENV_ROOT="$HOME/.pyenv"
+          export PATH="$PYENV_ROOT/bin:$PATH"
+
+          pyenv install ${{ github.event.inputs.python-version }}
+          pyenv global ${{ github.event.inputs.python-version }}
+
+          python3 --version
+
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.ref }}
+
+      - name: Configure
+        run: |
+          eval "$(pyenv init --path)"
+
+          cmake -S extras/analyze -B build \
+            -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} \
+            -DPython_FIND_UNVERSIONED_NAMES=FIRST \
+            -DPYBIND11_FINDPYTHON=ON \
+            -DCMAKE_CXX_COMPILER=g++-11
+
+      - name: Build
+        run: |
+          eval "$(pyenv init --path)"
+
+          cmake --build build -j 2
+
+      - name: Bundle Shared Objects
+        run: |
+          cd build
+
+          # Copy the shared object dependencies of this Python module to the current directory
+          ldd eventanalyze.*.so | sed -E 's/^.*=> (\S+).*$|(\S+) .*$/\1/g' | xargs -I {} cp {} .
+          chmod +x lib*.so*
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: Python-${{ github.event.inputs.python-version }} ${{ github.event.inputs.build_type }} Module
+          path: build/*.so*
+          if-no-files-found: error

.gitignore

@@ -1,4 +1,4 @@
-/build
-/.vscode
+build/
+.vscode/
 __pycache__
 *.pyc

extras/analyze/CMakeLists.txt

@@ -0,0 +1,57 @@
+cmake_minimum_required(VERSION 3.20.3)
+
+project(EventAnalyze)
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+include(FetchContent)
+
+FetchContent_Declare(
+  mio
+  GIT_REPOSITORY https://github.com/mandreyel/mio.git
+  GIT_TAG        3f86a95c0784d73ce6815237ec33ed25f233b643
+)
+FetchContent_MakeAvailable(mio)
+
+FetchContent_Declare(
+  pybind11
+  GIT_REPOSITORY https://github.com/pybind/pybind11.git
+  GIT_TAG        v2.6.2
+)
+FetchContent_MakeAvailable(pybind11)
+
+FetchContent_Declare(
+  abseil
+  GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git
+  GIT_TAG        f39e6ad4753e06d4a0d6a9bf6310478757479984
+)
+set(BUILD_TESTING OFF)
+FetchContent_MakeAvailable(abseil)
+
+FetchContent_Declare(
+  FindTBB
+  GIT_REPOSITORY https://github.com/justusc/FindTBB.git
+  GIT_TAG        25ecdea817b3af4a26d74ddcd439642dbd706acb
+)
+FetchContent_GetProperties(FindTBB)
+if(NOT findtbb_POPULATED)
+  FetchContent_Populate(FindTBB)
+  list(APPEND CMAKE_MODULE_PATH "${findtbb_SOURCE_DIR}" ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+endif()
+
+find_package(TBB)
+if(TBB_FOUND)
+  add_definitions(-DHAS_TBB)
+  link_libraries(tbb)
+endif()
+
+if(CMAKE_CXX_COMPILER_ID STREQUAL GNU)
+  add_compile_options(-Wall -Wextra)
+endif()
+
+file(GLOB_RECURSE sources CONFIGURE_DEPENDS "src/*.cpp")
+pybind11_add_module(eventanalyze ${sources})
+target_include_directories(eventanalyze PUBLIC include)
+target_compile_features(eventanalyze PUBLIC cxx_std_20)
+target_link_libraries(eventanalyze PRIVATE mio::mio absl::base absl::flat_hash_map)

extras/analyze/include/parse.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "py.hpp"
+
+namespace ev {
+void defParse(pybind11::module &Mod);
+
+struct EventSchema;
+
+const EventSchema *getSchema(std::string_view Id);
+} // namespace ev

extras/analyze/include/py.hpp

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+#include <filesystem>
+#include <string>
+#include <string_view>
+
+#include <iostream>
+
+namespace pybind11::detail {
+template <> struct type_caster<std::filesystem::path> {
+public:
+  PYBIND11_TYPE_CASTER(std::filesystem::path, _("pathlib.Path | str"));
+
+  // Python -> C++
+  bool load(handle Src, bool) {
+    // If !isinstance(Src, str):
+    if (!PyUnicode_Check(Src.ptr())) {
+      object PyPath = module::import("pathlib").attr("Path");
+
+      if (!PyObject_IsInstance(Src.ptr(), PyPath.ptr()))
+        return false;
+    }
+    this->value = std::filesystem::path(std::string(str(Src)));
+    return true;
+  }
+
+  static handle cast(const std::filesystem::path &Path, return_value_policy,
+                     handle) {
+    object PyPath = module::import("pathlib").attr("Path");
+    return PyPath(str(Path.string()));
+  }
+};
+} // namespace pybind11::detail

extras/analyze/include/types.hpp

@@ -0,0 +1,133 @@
+#pragma once
+
+#include <cassert>
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <variant>
+#include <vector>
+
+#include "py.hpp"
+
+#include <absl/container/flat_hash_map.h>
+#include <absl/container/flat_hash_set.h>
+#include <mio/mmap.hpp>
+
+namespace ev {
+using Number = std::variant<std::int64_t, std::uint64_t, double>;
+
+struct EventId {
+  std::string_view Value;
+
+  bool operator==(const EventId &) const = default;
+};
+
+enum class Type {
+  Number,
+  String,
+  Bool,
+};
+
+union Value {
+  Number Num;
+  std::string_view Str;
+  bool Bool;
+};
+
+struct EventSchema {
+  EventId Id;
+  std::vector<std::string_view> Parameters;
+  std::vector<Type> ParamTypes;
+
+  bool operator==(const EventSchema &) const = default;
+};
+
+struct Event {
+  EventId Id;
+  std::vector<Value> Values;
+};
+
+inline EventId getId(EventId Id) { return Id; }
+
+// clang-format off
+template <typename T>
+requires requires(const T &It) {
+  { It.Id } -> std::convertible_to<EventId>;
+}
+EventId getId(const T &It) { return It.Id; }
+// clang-format on
+
+template <typename T> EventId getId(const std::vector<T> &Vec) {
+  assert(!Vec.empty());
+  return getId(Vec.front());
+}
+
+struct EventIdHash {
+  using is_transparent = void;
+
+  std::size_t operator()(std::string_view Id) const noexcept {
+    return std::hash<std::string_view>()(Id);
+  }
+
+  std::size_t operator()(EventId Id) const noexcept {
+    return (*this)(Id.Value);
+  }
+
+  template <typename T> std::size_t operator()(const T &It) const noexcept {
+    return (*this)(getId(It));
+  }
+};
+
+struct EventIdEq {
+  using is_transparent = void;
+
+  bool operator()(EventId Lhs, EventId Rhs) const { return Lhs == Rhs; }
+
+  template <typename T, typename U>
+  bool operator()(const T &Lhs, const U &Rhs) const {
+    return getId(Lhs) == getId(Rhs);
+  }
+};
+
+using BlockEventMap =
+    absl::flat_hash_set<std::vector<Event>, EventIdHash, EventIdEq>;
+
+struct Logs;
+struct Benchmark;
+
+struct Block {
+  std::string_view Name;
+  BlockEventMap Events;
+  std::string_view RawLog;
+
+  std::string UniqueId;
+
+  ev::Benchmark *Bench;
+
+  std::string File; // Which file was compiled for this block
+};
+
+struct Benchmark {
+  std::string Name;
+  std::vector<Block> Blocks;
+  std::string_view RawLog;
+
+  // Keep the memory around so that we can detect if the Logs object was
+  // destroyed, giving the Python user a good error message.
+  std::weak_ptr<ev::Logs> Logs;
+};
+
+struct Logs {
+  std::filesystem::path LogFile;
+  mio::mmap_source MMap;
+  std::string_view RawLog;
+  std::vector<std::shared_ptr<Benchmark>> Benchmarks;
+};
+
+void defTypes(pybind11::module &Mod);
+} // namespace ev

extras/analyze/src/module.cpp

@@ -0,0 +1,14 @@
+#include "parse.hpp"
+#include "py.hpp"
+#include "types.hpp"
+
+namespace py = pybind11;
+
+PYBIND11_MODULE(eventanalyze, Mod) {
+  Mod.doc() = "C++-accelerated event logging types and parser";
+
+  Mod.attr("VERSION") = std::tuple(1, 0, 0);
+
+  ev::defTypes(Mod);
+  ev::defParse(Mod);
+}