Recently, when using FlatBuffers as a serialization protocol for RPC, I encountered some problems.

  1. the data types supported by FlatBuffers are limited, and specific data types need to be converted manually during serialization/deserialization.
  2. the TableType generated by the FlatBuffers code is not easy to use, and it is error-prone to call the Builder manually for construction.
  3. the performance of FlatBuffers code-generated NativeType is worrying, and the advantage of no deserialization is lost when using this type.

After careful consideration , the current project does not need to support cross-language RPC calls , only need to deal with the serialization/deserialization in C++ ; ease of use is the focus of requirements , all kinds of custom data structures want to be easy to serialize/deserialize without manual conversion . After referring to some open source projects, we decided to abandon FlatBuffers and use C++ native data structures and macros to define type Schema and complete serialization/deserialization.

1. Static reflection

Static reflection is implemented by referring to the intrusive definition in the garbageslam/visit_struct project. Its core principle is the visible scope of functions and the automatic type conversion of derived classes to base classes. As an example.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#include <algorithm>
#include <iostream>
#include <string_view>
#include <tuple>
#include <type_traits>

namespace reflection {

template <class List, class T>
struct Append;
template <class... Ts, class T>
struct Append<std::tuple<Ts...>, T> {
  using type = std::tuple<Ts..., T>;
};

template <int N = 64>
struct Rank : Rank<N - 1> {};
template <>
struct Rank<0> {};

[[maybe_unused]] static std::tuple<> CollectField(Rank<0>);

struct Helper {
  template <class T>
  static auto getFieldInfo() -> decltype(T::CollectField(Rank<>{}));

  template <class T>
  using FieldInfoList = decltype(getFieldInfo<T>());

  template <class T>
  static constexpr size_t Size = std::tuple_size_v<FieldInfoList<T>>;

  template <class T, size_t I>
  using FieldInfo = std::tuple_element_t<I, FieldInfoList<T>>;
};

#define REFL_NOW decltype(CollectField(::reflection::Rank<>{}))
#define REFL_ADD(info)                                                      \
  static ::reflection::Append<REFL_NOW, decltype(info)>::type CollectField( \
      ::reflection::Rank<std::tuple_size_v<REFL_NOW> + 1>)

}  // namespace reflection

namespace thief {

template <class T>
struct Bridge {
#if defined(__GNUC__) && !defined(__clang__)
// Silence unnecessary warning
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wnon-template-friend"
#endif
  friend auto ADL(Bridge<T>);
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
};

template <class T, class U>
struct StealType {
  friend auto ADL(Bridge<T>) { return std::type_identity<U>{}; }
};

template <class Tag, class Store>
using steal = decltype(StealType<Tag, Store>{});

template <class Tag>
using retrieve = typename decltype(ADL(Bridge<Tag>{}))::type;

}  // namespace thief

template <size_t N>
struct NameWrapper {
  constexpr NameWrapper(const char (&str)[N]) { std::copy_n(str, N, string); }
  constexpr operator std::string_view() const { return {string, N - 1}; }
  char string[N];
};

template <NameWrapper Name, auto Getter>
struct FieldInfo {
  static constexpr std::string_view name = Name;
  static constexpr auto getter = Getter;
};

#define ADD_FIELD(NAME, DEFAULT)                                       \
 protected:                                                            \
  friend struct ::reflection::Helper;                                  \
  struct T##NAME : std::type_identity<decltype(DEFAULT)> {};           \
                                                                       \
 public:                                                               \
  T##NAME::type NAME = DEFAULT;                                        \
                                                                       \
 private:                                                              \
  constexpr auto T##NAME()                                             \
      ->::thief::steal<struct T##NAME, std::decay_t<decltype(*this)>>; \
  REFL_ADD((FieldInfo<#NAME, &thief::retrieve<struct T##NAME>::NAME>{}))

struct A {
  ADD_FIELD(a, int{});
  ADD_FIELD(b, short{});
  ADD_FIELD(c, std::string{});
};

static_assert(reflection::Helper::Size<A> == 3);
static_assert(reflection::Helper::FieldInfo<A, 0>::name == "a");
static_assert(reflection::Helper::FieldInfo<A, 1>::name == "b");
static_assert(reflection::Helper::FieldInfo<A, 2>::name == "c");

int main() {
  A a;
  a.a = 10;
  a.b = 20;
  a.c = "hello";

  std::apply(
      [&](auto&&... type) {
        ((std::cout << "name: " << type.name << ", value: " << a.*type.getter
                    << std::endl),
         ...);
      },
      ::reflection::Helper::FieldInfoList<A>{});
  return 0;
}

/*
 * name: a, value: 10
 * name: b, value: 20
 * name: c, value: hello
 */

2. Serialization

With static reflection, serialization becomes a manual task. The binary serialization approach in eyalz800/zpp_bits project is referenced here, while a set of logic for serializing to TOML types is also implemented using marzer/tomlplusplus.

Ref

  1. garbageslam/visit_struct
  2. eyalz800/zpp_bits
  3. marzer/tomlplusplus