hmbdc
simplify-high-performance-messaging-programming
SendTransportEngine.hpp
1 #include "hmbdc/Copyright.hpp"
2 #pragma once
3 #include "hmbdc/app/Logger.hpp"
4 #include "hmbdc/tips/tcpcast/SendServer.hpp"
5 #include "hmbdc/tips/tcpcast/Transport.hpp"
6 #include "hmbdc/tips/tcpcast/Messages.hpp"
7 #include "hmbdc/tips/udpcast/SendTransportEngine.hpp"
8 #include "hmbdc/tips/tcpcast/DefaultUserConfig.hpp"
9 #include "hmbdc//MetaUtils.hpp"
10 #include "hmbdc/time/Time.hpp"
11 #include "hmbdc/time/Rater.hpp"
12 #include "hmbdc/numeric/BitMath.hpp"
13 
14 #include <boost/circular_buffer.hpp>
15 #include <memory>
16 #include <tuple>
17 #include <regex>
18 #include <type_traits>
19 #include <mutex>
20 
21 namespace hmbdc { namespace tips { namespace tcpcast {
22 
23 namespace send_detail {
24 HMBDC_CLASS_HAS_DECLARE(hmbdc_net_queued_ts);
25 using pattern::MonoLockFreeBuffer;
26 /**
27  * @brief capture the transportation mechanism
28  *
29  */
31 : Transport {
32  /**
33  * @brief ctor
34  * @param cfg jason specifing the transport - see example, perf-tcpcast.cpp
35  * @param maxMessageSize max messafe size in bytes to be sent
36  */
38 
39  size_t subscribingPartyDetectedCount(uint16_t tag) const {
40  return outboundSubscriptions_.check(tag);
41  }
42 
43  size_t bufferedMessageCount() const {
44  return buffer_.remainingSize();
45  }
46 
47  template <MessageC Message>
48  void queue(Message&& msg) {
49  if (!minRecvToStart_ && !outboundSubscriptions_.check(msg.getTypeTag())) {
50  using M = typename std::decay<Message>::type;
51  if constexpr (std::is_base_of<app::hasMemoryAttachment, M>::value) {
52  msg.release();
53  }
54  return;
55  }
56  auto n = 1;
57  auto it = buffer_.claim(n);
58  queue(it, std::forward<Message>(msg));
59  buffer_.commit(it, n);
60  }
61 
62  template <MessageC Message>
63  bool tryQueue(Message&& msg) {
64  if (!minRecvToStart_ && !outboundSubscriptions_.check(msg.getTypeTag())) {
65  using M = typename std::decay<Message>::type;
66  if constexpr (std::is_base_of<app::hasMemoryAttachment, M>::value) {
67  msg.release();
68  }
69  return true; //don't try again
70  }
71  auto n = 1;
72  auto it = buffer_.tryClaim(n);
73  if (it) {
74  queue(it, std::forward<Message>(msg));
75  buffer_.commit(it, n);
76  return true;
77  }
78  return false;
79  }
80 
81  template <typename Message, typename ... Args>
82  void queueInPlace(Args&&... args) {
83  static_assert(!std::is_base_of<app::JustBytes, Message>::value
84  , "use queueJustBytes instead");
85  static_assert(std::is_trivially_destructible<Message>::value
86  , "cannot send message with dtor");
87  static_assert(!std::is_base_of<app::hasMemoryAttachment, Message>::value
89  , "hasMemoryAttachment has to the first base for Message");
90  auto s = buffer_.claim();
91  char* addr = static_cast<char*>(*s);
92  auto h = new (addr) TransportMessageHeader;
93  h->messagePayloadLen = sizeof(app::MessageWrap<Message>);
94  if (hmbdc_likely(sizeof(Message) <= maxMessageSize_)) {
95  auto wrap = new (addr + sizeof(TransportMessageHeader))
96  app::MessageWrap<Message>(std::forward<Args>(args)...);
97  h->flag = wrap->scratchpad().desc.flag;
98  } else {
99  HMBDC_THROW(std::out_of_range
100  , "maxMessageSize too small to hold a message");
101  }
102  buffer_.commit(s);
103  }
104 
105  void queueJustBytes(uint16_t tag, void const* bytes, size_t len
106  , app::hasMemoryAttachment* att) {
107  if (!minRecvToStart_ && !outboundSubscriptions_.check(tag)) {
108  if (att) {
109  att->release();
110  }
111  return;
112  }
113  auto s = buffer_.claim();
114  char* addr = static_cast<char*>(*s);
115  auto h = new (addr) TransportMessageHeader;
116  h->messagePayloadLen = len + sizeof(app::MessageHead);
117  if (hmbdc_likely(len <= maxMessageSize_)) {
118  auto wrap = new (addr + sizeof(TransportMessageHeader))
119  app:: MessageWrap<app::JustBytes>(tag, bytes, len, att);
120  h->flag = wrap->scratchpad().desc.flag;
121  } else {
122  HMBDC_THROW(std::out_of_range
123  , "maxMessageSize too small to hold a message");
124  }
125  buffer_.commit(s);
126  }
127 
128  void stop();
129 
130 protected:
131  size_t maxMessageSize_;
132  size_t minRecvToStart_;
133  MonoLockFreeBuffer buffer_;
134 
135  std::unique_ptr<udpcast::SendTransport> mcSendTransport_;
136  hmbdc::time::Rater rater_;
137  hmbdc::app::Config mcConfig_;
138  TypeTagSet outboundSubscriptions_;
139 
140 private:
141  template<typename M, typename ... Messages>
142  void queue(MonoLockFreeBuffer::iterator it, M&& m, Messages&&... msgs) {
143  using Message = typename std::decay<M>::type;
144  static_assert(std::is_trivially_destructible<Message>::value, "cannot send message with dtor");
145  static_assert(!std::is_base_of<app::hasMemoryAttachment, Message>::value
147  , "hasMemoryAttachment has to the first base for Message");
148 
149  auto s = *it;
150  char* addr = static_cast<char*>(s);
151  auto h = new (addr) TransportMessageHeader;
152  h->messagePayloadLen = sizeof(app::MessageWrap<Message>);
153  if (hmbdc_likely(sizeof(Message) <= maxMessageSize_)) {
154  auto wrap = new (addr + sizeof(TransportMessageHeader))
155  app::MessageWrap<Message>(std::forward<M>(m));
156  h->flag = wrap->scratchpad().desc.flag;
157  } else {
158  HMBDC_THROW(std::out_of_range, "maxMessageSize too small to hold a message when constructing SendTransportEngine");
159  }
160  if constexpr (has_hmbdc_net_queued_ts<Message>::value) {
161  h->template wrapped<Message>().hmbdc_net_queued_ts = hmbdc::time::SysTime::now();
162  }
163  queue(++it, std::forward<Messages>(msgs)...);
164  }
165 
166  void queue(MonoLockFreeBuffer::iterator it) {}
167 };
168 
173 , hmbdc::app::Client<SendTransportEngine> {
174  SendTransportEngine(hmbdc::app::Config const&, size_t);
175  using SendTransport::hmbdcName;
176  using SendTransport::schedSpec;
177 
178  template <MessageTupleC Messages, typename Node>
179  void advertiseFor(Node const& node, uint16_t mod, uint16_t res) {
180  std::scoped_lock<std::mutex> g(advertisedTypeTags_.lock);
181  advertisedTypeTags_.markPubsFor<Messages>(node, mod, res);
182  server_->advertisingMessages(advertisedTypeTags_);
183  }
184 
185  void rotate() {
186  this->checkTimers(hmbdc::time::SysTime::now());
188  }
189 
190  /*virtual*/
191  void invokedCb(size_t) HMBDC_RESTRICT override {
192  runOnce();
193  mcSendTransport_->runOnce(true);
194  }
195 
196  /*virtual*/
197  bool droppedCb() override {
198  stop();
199  stopped_ = true;
200  return true;
201  };
202 
203  /**
204  * @brief check how many recipient sessions are still active
205  * @return numeric_limits<size_t>::max() if the sending hasn't started due to
206  * minRecvToStart has not been met yet
207  */
208  size_t sessionsRemainingActive() const {
209  return server_->readySessionCount();
210  }
211 
212 private:
213  void cullSlow() {
214  auto seq = buffer_.readSeq();
215  if (seq == lastSeq_ && buffer_.isFull()) {
216  server_->killSlowestSession();
217  }
218  lastSeq_ = seq;
219  }
220 
221  void runOnce();
222 
223  size_t mtu_;
224  size_t maxSendBatch_;
225  bool waitForSlowReceivers_;
226  MonoLockFreeBuffer::iterator begin_;
227  ToSend toSend_;
228  TypeTagSetST advertisedTypeTags_;
229 
230  std::optional<SendServer> server_;
231  size_t lastSeq_;
232  bool stopped_;
233 };
234 } //send_detail
235 
236 using SendTransport = send_detail::SendTransport;
237 using SendTransportEngine = send_detail::SendTransportEngine;
238 }}}
239 
240 namespace hmbdc { namespace tips { namespace tcpcast {
241 
242 namespace send_detail {
243 using pattern::MonoLockFreeBuffer;
244 
245 inline
248  , size_t maxMessageSize)
249 : Transport((cfg.setAdditionalFallbackConfig(hmbdc::app::Config(DefaultUserConfig))
250  , cfg.resetSection("tx", false)))
251 , maxMessageSize_(maxMessageSize)
252 , minRecvToStart_(config_.getExt<size_t>("minRecvToStart"))
253 , buffer_(maxMessageSize + sizeof(TransportMessageHeader) + sizeof(app::MessageHead)
254  , config_.getExt<uint16_t>("outBufferSizePower2")
255  ?config_.getExt<uint16_t>("outBufferSizePower2")
256  :hmbdc::numeric::log2Upper(128ul * 1024ul / maxMessageSize)
257 )
258 , rater_(hmbdc::time::Duration::seconds(1u)
259  , config_.getExt<size_t>("sendBytesPerSec")
260  , config_.getExt<size_t>("sendBytesBurst")
261  , config_.getExt<size_t>("sendBytesBurst") != 0ul //no rate control by default
262 )
263 , mcConfig_(config_) {
264  mcConfig_.put("loopback", true); //always allow advertising to loopback, so other process
265  //on the same machine get them
266  mcConfig_.put("outBufferSizePower2", 3u);
267 
268  mcSendTransport_.reset(new udpcast::SendTransport(
269  mcConfig_, sizeof(app::MessageWrap<TypeTagSource>)));
270 }
271 
272 inline
273 void
274 SendTransport::
275 stop() {
276  buffer_.reset();
277 };
278 
279 inline
280 SendTransportEngine::
281 SendTransportEngine(hmbdc::app::Config const& cfg
282  , size_t maxMessageSize)
283 : SendTransport(cfg, maxMessageSize)
284 , hmbdc::time::ReoccuringTimer(
285  hmbdc::time::Duration::seconds(config_.getExt<size_t>("typeTagAdvertisePeriodSeconds")))
286 , mtu_(config_.getExt<size_t>("mtu") - 20 - 20) //20 bytes ip header and 20 bytes tcp header
287 , maxSendBatch_(config_.getExt<size_t>("maxSendBatch"))
288 , waitForSlowReceivers_(config_.getExt<bool>("waitForSlowReceivers"))
289 , lastSeq_(0)
290 , stopped_(false) {
291  toSend_.reserve(maxSendBatch_ * 2);
292  // inititialize begin_ to first item
293  MonoLockFreeBuffer::iterator end;
294  buffer_.peek(begin_, end, 0);
295 
296  server_.emplace(config_, buffer_.capacity(), outboundSubscriptions_);
297  setCallback(
298  [this](hmbdc::time::TimerManager& tm, hmbdc::time::SysTime const& now) {
299  std::unique_lock<std::mutex> g(advertisedTypeTags_.lock, std::try_to_lock);
300  if (!g.owns_lock()) return;
301  for (auto& ad : server_->advertisingMessages()) {
302  mcSendTransport_->queue(ad);
303  }
304  if (!waitForSlowReceivers_) cullSlow();
305  }
306  );
307 
308  schedule(hmbdc::time::SysTime::now(), *this);
309 }
310 
311 inline
312 void
313 SendTransportEngine::
314 runOnce() HMBDC_RESTRICT {
315  MonoLockFreeBuffer::iterator begin, end;
316  //only START to send when enough sessions are ready to receive or
317  //buffer is full
318  if (hmbdc_likely(!minRecvToStart_
319  || server_->readySessionCount() >= minRecvToStart_)) {
320  minRecvToStart_ = 0; //now started no going back
321  buffer_.peek(begin, end, maxSendBatch_);
322  } else {
323  buffer_.peek(begin, end, 0);
324  }
325 
326  auto it = begin_;
327  uint16_t currentTypeTag = 0;
328  size_t toSendByteSize = 0;
329 
330  while (it != end) {
331  void* ptr = *it;
332  auto item = static_cast<TransportMessageHeader*>(ptr);
333 
334  if (hmbdc_unlikely(!rater_.check(item->wireSize()))) break;
335  if (hmbdc_unlikely(!item->flag
336  && toSendByteSize + item->wireSize() > mtu_)) break;
337  it++;
338  if (hmbdc_unlikely(!currentTypeTag)) {
339  currentTypeTag = item->typeTag();
340  toSendByteSize = item->wireSize();
341  toSend_.push_back(iovec{ptr, item->wireSize()});
342  } else if (item->typeTag() == currentTypeTag) {
343  toSendByteSize += item->wireSize();
344  toSend_.push_back(iovec{ptr, item->wireSize()});
345  } else {
346  server_->queue(currentTypeTag
347  , move(toSend_)
348  , toSendByteSize
349  , it);
350  toSend_.clear();
351 
352  currentTypeTag = item->typeTag();
353  toSendByteSize = item->wireSize();
354  toSend_.push_back(iovec{ptr, item->wireSize()});
355  }
356 
357  if (hmbdc_unlikely(item->flag == app::hasMemoryAttachment::flag)) {
358  auto& a = item->wrapped<app::hasMemoryAttachment>();
359  toSend_.push_back(iovec{a.attachment, a.len});
360  toSendByteSize += a.len;
361  }
362  rater_.commit();
363  }
364 
365  if (toSend_.size()) {
366  server_->queue(currentTypeTag
367  , move(toSend_)
368  , toSendByteSize
369  , it);
370  toSend_.clear();
371  }
372  begin_ = it;
373  //use it here which is the maximum
374  auto newStart = server_->runOnce(begin, it);
375  if (begin != end) { //only need to do this when there r things read from buffer_
376  for (auto it = begin; it != newStart; ++it) {
377  void* ptr = *it;
378  auto item = static_cast<TransportMessageHeader*>(ptr);
379  if (hmbdc_unlikely(item->flag == app::hasMemoryAttachment::flag)) {
380  auto& a = item->wrapped<app::hasMemoryAttachment>();
381  a.release();
382  }
383  }
384  buffer_.wasteAfterPeek(begin, newStart - begin, true);
385  }
386 }
387 } //send_detail
388 }}}
389 
void invokedCb(size_t) HMBDC_RESTRICT override
this callback is called all the time (frequently) - the exact timing is after a batch of messages are...
Definition: SendTransportEngine.hpp:191
Definition: MonoLockFreeBuffer.hpp:16
size_t sessionsRemainingActive() const
check how many recipient sessions are still active
Definition: SendTransportEngine.hpp:208
class to hold an hmbdc configuration
Definition: Config.hpp:44
bool droppedCb() override
callback called after the Client is safely taken out of the Context
Definition: SendTransportEngine.hpp:197
SendTransport(hmbdc::app::Config, size_t)
ctor
Definition: SendTransportEngine.hpp:247
Definition: Timers.hpp:70
Definition: Transport.hpp:65
Definition: SendTransportEngine.hpp:169
Definition: Message.hpp:212
Definition: Time.hpp:14
Definition: TypeTagSet.hpp:138
Definition: Message.hpp:263
Config & put(Args &&... args)
forward the call to ptree&#39;s put but return Configure
Definition: Config.hpp:194
a Node is a thread of execution that can suscribe and receive Messages
Definition: Node.hpp:51
capture the transportation mechanism
Definition: SendTransportEngine.hpp:30
A Client represents a thread of execution/a task. The execution is managed by a Context. a Client object could participate in message dispatching as the receiver of specifed message types.
Definition: Client.hpp:128
Definition: TypeTagSet.hpp:198
Definition: Rater.hpp:11
if a specific hmbdc network transport (for example tcpcast, rmcast, and rnetmap) supports message wit...
Definition: Message.hpp:125
Definition: Base.hpp:12
void wasteAfterPeek(iterator, size_t, bool=false)
if size not matching - please refer to the impl for details
Definition: MonoLockFreeBuffer.hpp:160
Definition: Timers.hpp:117
Definition: LockFreeBufferMisc.hpp:89