hmbdc
simplify-high-performance-messaging-programming
SendTransportEngine.hpp
1 #include "hmbdc/Copyright.hpp"
2 #pragma once
3 #include "hmbdc/tips/udpcast/Transport.hpp"
4 #include "hmbdc/tips/udpcast/Messages.hpp"
5 #include "hmbdc/tips/udpcast/DefaultUserConfig.hpp"
6 #include "hmbdc/app/Client.hpp"
7 #include "hmbdc/comm/inet/Endpoint.hpp"
8 #include "hmbdc/pattern/MonoLockFreeBuffer.hpp"
9 #include "hmbdc/time/Time.hpp"
10 #include "hmbdc/time/Rater.hpp"
11 #include "hmbdc/numeric/BitMath.hpp"
12 
13 #include <regex>
14 #include <memory>
15 #include <iostream>
16 #include <mutex>
17 
18 namespace hmbdc { namespace tips { namespace udpcast {
19 
20 namespace sendtransportengine_detail {
21 HMBDC_CLASS_HAS_DECLARE(hmbdc_net_queued_ts);
23 
25 : Transport {
26  SendTransport(Config, size_t);
27  ~SendTransport();
28 
29  size_t bufferedMessageCount() const {
30  return buffer_.remainingSize();
31  }
32 
33  size_t subscribingPartyDetectedCount(uint16_t tag) const {
34  return 0; //not supported
35  }
36 
37  template <MessageTupleC Messages, typename Node>
38  void advertiseFor(Node const& node, uint16_t mod, uint16_t res) {
39  //only applies to connection oriented transport
40  }
41 
42  template <MessageC Message>
43  void queue(Message&& msg) {
44  auto n = 1;
45  auto it = buffer_.claim(n);
46  queue(it, std::forward<Message>(msg));
47  buffer_.commit(it, n);
48  }
49 
50  template <typename Message>
51  bool tryQueue(Message&& msg) {
52  auto n = 1;
53  auto it = buffer_.tryClaim(n);
54  if (it) {
55  queue(it, std::forward<Message>(msg));
56  buffer_.commit(it, n);
57  return true;
58  }
59  return false;
60  }
61 
62  void runOnce(bool alwaysPoll = false) HMBDC_RESTRICT {
63  resumeSend();
64  if (hmbdc_unlikely(alwaysPoll || !isFdReady())) {
65  app::utils::EpollTask::instance().poll();
66  }
67  }
68 
69  void stop();
70 
71 private:
72  size_t maxMessageSize_;
73  typename Buffer::iterator begin_, it_, end_;
74  Buffer buffer_;
75  hmbdc::time::Rater rater_;
76  size_t maxSendBatch_;
77 
78  iovec* toSendMsgs_;
79  size_t toSendMsgsHead_;
80  size_t toSendMsgsTail_;
81  mmsghdr* toSendPkts_;
82  size_t toSendPktsHead_;
83  size_t toSendPktsTail_;
84  std::vector<comm::inet::Endpoint> udpcastDests_;
85 
86  uint16_t
87  outBufferSizePower2();
88 
89  template<typename M, typename ... Messages>
90  void queue(typename Buffer::iterator it, M&& m, Messages&&... msgs) {
91  using Message = typename std::decay<M>::type;
92  static_assert(std::is_trivially_destructible<Message>::value, "cannot send message with dtor");
93  auto s = *it;
94  char* addr = static_cast<char*>(s);
95  auto h = reinterpret_cast<TransportMessageHeader*>(addr);
96  if (hmbdc_likely(sizeof(Message) <= maxMessageSize_)) {
97  new (addr + sizeof(TransportMessageHeader)) app::MessageWrap<Message>(std::forward<M>(m));
98  h->messagePayloadLen() = sizeof(app::MessageWrap<Message>);
99  } else {
100  HMBDC_THROW(std::out_of_range
101  , "maxMessageSize too small to hold a message when constructing SendTransportEngine");
102  }
103  if constexpr (has_hmbdc_net_queued_ts<Message>::value) {
104  h->template wrapped<Message>().hmbdc_net_queued_ts = hmbdc::time::SysTime::now();
105  }
106  queue(++it, std::forward<Messages>(msgs)...);
107  }
108 
109  void queue(typename Buffer::iterator it) {}
110  void resumeSend();
111 };
112 
115 , hmbdc::app::Client<SendTransportEngine> {
116  using SendTransport::SendTransport;
117  using SendTransport::hmbdcName;
118  using SendTransport::schedSpec;
119 
120  void rotate() {
122  }
123 
124  /*virtual*/
125  void invokedCb(size_t) HMBDC_RESTRICT override {
126  runOnce();
127  }
128  /*virtual*/ bool droppedCb() override {
129  stop();
130  return true;
131  };
132 
133  using Transport::hmbdcName;
134 };
135 
136 } //sendtransportengine_detail
137 using SendTransport = sendtransportengine_detail::SendTransport;
138 using SendTransportEngine = sendtransportengine_detail::SendTransportEngine;
139 }}}
140 
141 
142 
143 namespace hmbdc { namespace tips { namespace udpcast {
144 
145 namespace sendtransportengine_detail {
146 
147 inline
148 SendTransport::
149 SendTransport(Config cfg
150  , size_t maxMessageSize)
151 : Transport((cfg.setAdditionalFallbackConfig(Config(DefaultUserConfig))
152  , cfg.resetSection("tx", false)))
153 , maxMessageSize_(maxMessageSize)
154 , buffer_(maxMessageSize + sizeof(TransportMessageHeader) + sizeof(app::MessageHead), outBufferSizePower2())
155 , rater_(hmbdc::time::Duration::seconds(1u)
156  , config_.getExt<size_t>("sendBytesPerSec")
157  , config_.getExt<size_t>("sendBytesBurst")
158  , config_.getExt<size_t>("sendBytesBurst") != 0ul) //no rate control by default
159 , maxSendBatch_(config_.getExt<size_t>("maxSendBatch"))
160 , toSendMsgs_(new iovec[maxSendBatch_])
161 , toSendMsgsHead_(0)
162 , toSendMsgsTail_(0)
163 , toSendPkts_(new mmsghdr[maxSendBatch_])
164 , toSendPktsHead_(0)
165 , toSendPktsTail_(0) {
166  char loopch = config_.getExt<bool>("loopback")?1:0;
167  if (setsockopt(fd, IPPROTO_IP, IP_MULTICAST_LOOP, (char *)&loopch, sizeof(loopch)) < 0) {
168  HMBDC_THROW(std::runtime_error, "failed to set loopback=" << config_.getExt<bool>("loopback"));
169  }
170 
171  auto sz = config_.getExt<int>("udpSendBufferBytes");
172  if (sz) {
173  if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)) < 0) {
174  HMBDC_LOG_C("failed to set send buffer size=", sz);
175  }
176  }
177 
178  auto ttl = config_.getExt<int>("ttl");
179  if (ttl > 0 && setsockopt(fd, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) < 0) {
180  HMBDC_THROW(std::runtime_error, "failed to set set ttl=" << ttl);
181  }
182 
183  hmbdc::app::utils::EpollTask::instance().add(
184  hmbdc::app::utils::EpollTask::EPOLLOUT
185  | hmbdc::app::utils::EpollTask::EPOLLET, *this);
186  auto minHead = sizeof(app::MessageHead) + sizeof(TransportMessageHeader);
187  if (maxMessageSize_ + minHead > mtu_) {
188  HMBDC_THROW(std::out_of_range, "maxMessageSize needs <= " << mtu_ - minHead);
189  }
190  cfg(udpcastDests_, "udpcastDests");
191  if (udpcastDests_.size() == 0) {
192  HMBDC_THROW(std::out_of_range, "empty udpcastDests");
193  }
194  memset(toSendMsgs_, 0, sizeof(iovec) * maxSendBatch_);
195  memset(toSendPkts_, 0, sizeof(mmsghdr) * maxSendBatch_);
196  std::for_each(toSendPkts_, toSendPkts_ + maxSendBatch_
197  , [this](mmsghdr& pkt) {
198  pkt.msg_hdr.msg_name = &udpcastDests_.begin()->v;
199  pkt.msg_hdr.msg_namelen = sizeof(udpcastDests_.begin()->v);
200  });
201 }
202 
203 inline
204 SendTransport::
205 ~SendTransport() {
206  delete []toSendPkts_;
207  delete []toSendMsgs_;
208 }
209 
210 inline
211 void
212 SendTransport::
213 stop() {
214  buffer_.reset();
215 }
216 
217 inline
218 uint16_t
219 SendTransport::
220 outBufferSizePower2() {
221  auto res = config_.getExt<uint16_t>("outBufferSizePower2");
222  if (res) {
223  return res;
224  }
225  res =hmbdc::numeric::log2Upper(8ul * 1024ul / (8ul + maxMessageSize_));
226  HMBDC_LOG_N("auto set --outBufferSizePower2=", res);
227  return res;
228 }
229 
230 inline
231 void
232 SendTransport::
233 resumeSend() HMBDC_RESTRICT {
234  bool rateOk = true;
235  while (rateOk) {
236  if (it_ == end_) {
237  buffer_.wasteAfterPeek(begin_, end_ - begin_);
238  buffer_.peek(it_, end_, maxSendBatch_);
239  begin_ = it_;
240  }
241  //make a packet
242  size_t packetBytes = 0;
243  while (it_ != end_ && toSendMsgsTail_ < maxSendBatch_) {
244  void* ptr = *it_;
245  auto item = static_cast<TransportMessageHeader*>(ptr);
246  rateOk = rater_.check(item->wireSize());
247  if (hmbdc_unlikely(!rateOk)) {
248  break;
249  }
250  packetBytes += item->wireSize();
251  if (hmbdc_unlikely(packetBytes > mtu_)) {
252  toSendPkts_[toSendPktsTail_].msg_hdr.msg_iov = toSendMsgs_ + toSendMsgsHead_;
253  toSendPkts_[toSendPktsTail_++].msg_hdr.msg_iovlen
254  = toSendMsgsTail_ - toSendMsgsHead_;
255  toSendMsgsHead_ = toSendMsgsTail_;
256  packetBytes = item->wireSize();
257  }
258 
259  toSendMsgs_[toSendMsgsTail_].iov_base = ptr;
260  toSendMsgs_[toSendMsgsTail_++].iov_len = item->wireSize();
261  rater_.commit();
262  it_++;
263  }
264  if (packetBytes) {
265  //wrap up current packet
266  toSendPkts_[toSendPktsTail_].msg_hdr.msg_iov = toSendMsgs_ + toSendMsgsHead_;
267  toSendPkts_[toSendPktsTail_++].msg_hdr.msg_iovlen
268  = toSendMsgsTail_ - toSendMsgsHead_;
269  toSendMsgsHead_ = toSendMsgsTail_;
270  }
271 
272  auto toSendPktsCount = toSendPktsTail_ - toSendPktsHead_;
273  if (toSendPktsCount && isFdReady()) {
274  int l = 0;
275  l = sendmmsg(fd, toSendPkts_ + toSendPktsHead_, toSendPktsCount, MSG_NOSIGNAL|MSG_DONTWAIT);
276  if (hmbdc_unlikely(udpcastDests_.size() > 1)) {
277  for (auto i = 1u; hmbdc_unlikely(i < udpcastDests_.size()); ++i) {
278  auto& addr = udpcastDests_[i].v;
279  std::for_each(toSendPkts_ + toSendPktsHead_, toSendPkts_ + toSendPktsHead_ + toSendPktsCount
280  , [&addr](mmsghdr& pkt) {
281  pkt.msg_hdr.msg_name = &addr;
282  }
283  );
284  l = std::max(l, sendmmsg(fd, toSendPkts_ + toSendPktsHead_, toSendPktsCount, MSG_NOSIGNAL|MSG_DONTWAIT));
285  }
286  std::for_each(toSendPkts_ + toSendPktsHead_, toSendPkts_ + toSendPktsHead_ + toSendPktsCount
287  , [this](mmsghdr& pkt) {
288  pkt.msg_hdr.msg_name = &udpcastDests_[0].v;
289  }
290  );
291  }
292  if (hmbdc_likely(l > 0)) {
293  } else if (hmbdc_likely(l < 0)) {
294  if (hmbdc_unlikely(!checkErr())) {
295  HMBDC_LOG_C("sendmmsg failed errno=", errno);
296  }
297  return;
298  } else {
299  return; //try again later
300  }
301 
302  toSendPktsHead_ += l;
303  if (toSendPktsHead_ == toSendPktsTail_) {
304  toSendPktsHead_ = toSendPktsTail_ = 0;
305  toSendMsgsHead_ = toSendMsgsTail_ = 0;
306  }
307  } else {
308  //nothing to do now
309  return;
310  }
311  }
312 }
313 } //sendtransportengine_detail
314 }}}
T getExt(const path_type &param, bool throwIfMissing=true) const
get a value from the config
Definition: Config.hpp:238
Definition: MonoLockFreeBuffer.hpp:16
Definition: Transport.hpp:38
bool droppedCb() override
callback called after the Client is safely taken out of the Context
Definition: SendTransportEngine.hpp:128
class to hold an hmbdc configuration
Definition: Config.hpp:44
void invokedCb(size_t) HMBDC_RESTRICT override
this callback is called all the time (frequently) - the exact timing is after a batch of messages are...
Definition: SendTransportEngine.hpp:125
Definition: Message.hpp:263
a Node is a thread of execution that can suscribe and receive Messages
Definition: Node.hpp:51
A Client represents a thread of execution/a task. The execution is managed by a Context. a Client object could participate in message dispatching as the receiver of specifed message types.
Definition: Client.hpp:128
Definition: Rater.hpp:11
Definition: Base.hpp:12
void wasteAfterPeek(iterator, size_t, bool=false)
if size not matching - please refer to the impl for details
Definition: MonoLockFreeBuffer.hpp:160
Definition: LockFreeBufferMisc.hpp:89