1 #include "hmbdc/Copyright.hpp" 3 #include "hmbdc/tips/rnetmap/Transport.hpp" 4 #include "hmbdc/tips/rnetmap/BackupSendServer.hpp" 5 #include "hmbdc/tips/rnetmap/Messages.hpp" 6 #include "hmbdc/tips/rnetmap/NmSendTransport.hpp" 7 #include "hmbdc/tips/rnetmap/DefaultUserConfig.hpp" 8 #include "hmbdc/app/Base.hpp" 9 #include "hmbdc/MetaUtils.hpp" 10 #include "hmbdc/time/Time.hpp" 11 #include "hmbdc/time/Rater.hpp" 12 #include "hmbdc/numeric/BitMath.hpp" 17 #include <type_traits> 20 namespace hmbdc {
namespace tips {
namespace rnetmap {
22 namespace sendtransportengine_detail{
23 HMBDC_CLASS_HAS_DECLARE(hmbdc_net_queued_ts);
30 size_t bufferedMessageCount()
const {
31 return buffer_.remainingSize();
34 size_t subscribingPartyDetectedCount(uint16_t tag)
const {
35 return outboundSubscriptions_.check(tag);
38 template <
typename Message>
40 ,
typename std::decay<Message>::type>::value,
void>::type
41 queue(Message&& msg) {
42 if (!outboundSubscriptions_.check(msg.getTypeTag()))
return;
44 auto it = buffer_.claim(n);
45 queue(it, std::forward<Message>(msg));
46 buffer_.commit(it, n);
49 template <
typename MemoryAttachementMessage>
51 ,
typename std::decay<MemoryAttachementMessage>::type>::value,
void>::type
52 queue(MemoryAttachementMessage&& msg) {
53 if (!minRecvToStart_ && !outboundSubscriptions_.check(msg.getTypeTag())) {
57 using Message =
typename std::decay<MemoryAttachementMessage>::type;
58 if (hmbdc_unlikely(
sizeof(Message) > maxMessageSize_
61 HMBDC_THROW(std::out_of_range
62 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
66 size_t n = (msg.app::hasMemoryAttachment::len + maxMemorySegPayloadSize_ - 1ul) / maxMemorySegPayloadSize_ + 2;
67 if (hmbdc_unlikely(n > buffer_.capacity())) {
68 HMBDC_THROW(std::out_of_range
69 ,
"send engine buffer too small capacity=" << buffer_.capacity() <<
"<" << n);
71 auto it = buffer_.claim(n);
72 queueMemorySegTrain(it, n, std::forward<MemoryAttachementMessage>(msg));
73 buffer_.commit(it, n);
76 template <
typename Message>
78 ,
typename std::decay<Message>::type>::value,
bool>::type
79 tryQueue(Message&& msg) HMBDC_RESTRICT {
80 if (!minRecvToStart_ && !outboundSubscriptions_.check(msg.getTypeTag()))
return true;
82 auto it = buffer_.tryClaim(n);
84 queue(it, std::forward<Message>(msg));
85 buffer_.commit(it, n);
91 template <
typename Message,
typename ... Args>
92 void queueInPlace(Args&&... args) HMBDC_RESTRICT {
93 static_assert(std::is_trivially_destructible<Message>::value,
"cannot send message with dtor");
94 if (hmbdc_unlikely(
sizeof(Message) > maxMessageSize_)) {
95 HMBDC_THROW(std::out_of_range,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
97 auto s = buffer_.claim();
98 TransportMessageHeader::copyToInPlace<Message>(*s, std::forward<Args>(args)...)->setSeq(s.seq_);
102 void queueJustBytes(uint16_t tag,
void const* bytes,
size_t len
105 if (!minRecvToStart_ && !outboundSubscriptions_.check(tag))
return;
106 auto it = buffer_.claim();
108 char* addr =
static_cast<char*
>(s);
110 if (hmbdc_likely(len <= maxMessageSize_)) {
116 HMBDC_THROW(std::out_of_range
117 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
121 if (!minRecvToStart_ && !outboundSubscriptions_.check(tag)) {
125 if (hmbdc_unlikely(len > maxMessageSize_
128 HMBDC_THROW(std::out_of_range
129 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
133 size_t n = (att->len + maxMemorySegPayloadSize_ - 1ul) / maxMemorySegPayloadSize_ + 2;
134 if (hmbdc_unlikely(n > buffer_.capacity())) {
135 HMBDC_THROW(std::out_of_range
136 ,
"send engine buffer too small capacity=" << buffer_.capacity() <<
"<" << n);
138 auto it = buffer_.claim(n);
139 queueMemorySegTrain(it, n, tag, *att, len);
140 buffer_.commit(it, n);
145 hmbdc::app::utils::EpollTask::instance().poll();
146 if (hmbdc_unlikely(minRecvToStart_ != std::numeric_limits<size_t>::max()
147 && asyncBackupSendServer_.readySessionCount() >= minRecvToStart_)) {
148 minRecvToStart_ = std::numeric_limits<size_t>::max();
149 nmSendTransport_.startSend();
151 asyncBackupSendServer_.runOnce();
152 if (advertisedTypeTagsDirty_) {
153 std::unique_lock<std::mutex> g(advertisedTypeTags_.lock, std::try_to_lock);
155 nmSendTransport_.setAds(asyncBackupSendServer_.advertisingMessages());
156 advertisedTypeTagsDirty_ =
false;
159 nmSendTransport_.runOnce(asyncBackupSendServer_.sessionCount());
163 size_t sessionsRemainingActive()
const {
164 return asyncBackupSendServer_.readySessionCount();
167 template <MessageTupleC Messages,
typename Node>
168 void advertiseFor(
Node const& node, uint16_t mod, uint16_t res) {
169 std::scoped_lock<std::mutex> g(advertisedTypeTags_.lock);
170 advertisedTypeTags_.markPubsFor<Messages>(node, mod, res);
171 asyncBackupSendServer_.advertisingMessages(advertisedTypeTags_);
172 advertisedTypeTagsDirty_ =
true;
176 size_t maxMessageSize_;
182 size_t minRecvToStart_;
184 HMBDC_SEQ_TYPE lastBackupSeq_;
186 bool waitForSlowReceivers_;
187 uint16_t maxMemorySegPayloadSize_;
189 reliable::ToCleanupAttQueue toCleanupAttQueue_;
193 std::atomic<bool> advertisedTypeTagsDirty_ =
false;
197 outBufferSizePower2();
199 void queueMemorySegTrain(
typename Buffer::iterator it,
size_t n
203 void queueMemorySegTrain(
typename Buffer::iterator it,
size_t n, M&& m) {
204 using Message =
typename std::decay<M>::type;
207 char* addr =
static_cast<char*
>(s);
212 trainHead.inbandUnderlyingTypeTag = m.getTypeTag();
214 trainHead.segCount = n - 2;
216 h->setSeq(it++.seq_);
220 auto totalLen = (size_t)m.app::hasMemoryAttachment::len;
221 char* base = (
char*)m.app::hasMemoryAttachment::attachment;
222 decltype(totalLen) offset = 0;
223 while(totalLen > offset) {
225 char* addr =
static_cast<char*
>(s);
229 seg.seg = base + offset;
230 auto l = (uint16_t)std::min((
size_t)maxMemorySegPayloadSize_, totalLen - offset);
233 h->setSeq(it++.seq_);
241 char* addr =
static_cast<char*
>(s);
245 h->flag = app::hasMemoryAttachment::flag;
246 h->setSeq(it++.seq_);
251 template<
typename M,
typename ... Messages>
252 void queue(
typename Buffer::iterator it
253 , M&& m, Messages&&... msgs) {
254 using Message =
typename std::decay<M>::type;
255 static_assert(std::is_trivially_destructible<Message>::value,
"cannot send message with dtor");
257 char* addr =
static_cast<char*
>(s);
259 if (hmbdc_likely(
sizeof(Message) <= maxMessageSize_)) {
264 HMBDC_THROW(std::out_of_range
265 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
267 if constexpr (has_hmbdc_net_queued_ts<Message>::value) {
268 h->template wrapped<Message>().hmbdc_net_queued_ts = hmbdc::time::SysTime::now();
270 queue(++it, std::forward<Messages>(msgs)...);
273 void queue(Buffer::iterator it) {}
281 using SendTransport::SendTransport;
290 this->checkTimers(hmbdc::time::SysTime::now());
301 HMBDC_LOG_C(e.what());
310 using EngineTransport::hmbdcName;
312 std::tuple<char const*, int> schedSpec()
const {
313 return std::make_tuple(this->schedPolicy_.c_str(), this->schedPriority_);
318 using SendTransport = sendtransportengine_detail::SendTransport;
319 using SendTransportEngine = sendtransportengine_detail::SendTransportEngine;
322 namespace hmbdc {
namespace tips {
namespace rnetmap {
324 namespace sendtransportengine_detail{
329 ,
size_t maxMessageSize)
330 : EngineTransport((cfgIn.setAdditionalFallbackConfig(
hmbdc::app::Config(DefaultUserConfig))
331 , cfgIn.resetSection(
"tx", false)))
332 , maxMessageSize_(maxMessageSize)
333 , buffer_(maxMessageSize + sizeof(TransportMessageHeader) + sizeof(app::MessageHead)
334 , outBufferSizePower2())
335 , rater_(
hmbdc::time::Duration::seconds(1u)
336 , config_.getExt<size_t>(
"sendBytesPerSec")
337 , config_.getExt<size_t>(
"sendBytesBurst")
338 , config_.getExt<size_t>(
"sendBytesBurst") != 0ul)
339 , nmSendTransport_(config_, maxMessageSize, buffer_, rater_, toCleanupAttQueue_)
340 , minRecvToStart_(config_.getExt<size_t>(
"minRecvToStart"))
341 , typeTagAdTimer_(
hmbdc::time::Duration::seconds(config_.getExt<uint32_t>(
"typeTagAdvertisePeriodSeconds")))
342 , lastBackupSeq_(
std::numeric_limits<HMBDC_SEQ_TYPE>::max())
343 , flushTimer_(
hmbdc::time::Duration::microseconds(config_.getExt<uint32_t>(
"netRoundtripLatencyMicrosec")))
344 , waitForSlowReceivers_(config_.getExt<bool>(
"waitForSlowReceivers"))
345 , maxMemorySegPayloadSize_(
std::min(mtu_ - sizeof(TransportMessageHeader) - sizeof(app::MessageHead)
346 , TransportMessageHeader::maxPayloadSize() - sizeof(app::MessageHead)))
347 , asyncBackupSendServer_(config_, buffer_, rater_, toCleanupAttQueue_, outboundSubscriptions_)
349 auto sendBytesBurst = config_.
getExt<
size_t>(
"sendBytesBurst");
350 auto sendBytesBurstMin = maxMessageSize +
sizeof(TransportMessageHeader);
351 if (sendBytesBurst && sendBytesBurst < sendBytesBurstMin) {
352 HMBDC_THROW(std::out_of_range,
"sendBytesBurst needs to >= " << sendBytesBurstMin);
354 toCleanupAttQueue_.set_capacity(buffer_.capacity() + 10);
356 typeTagAdTimer_.setCallback(
358 nmSendTransport_.setAdPending();
359 if (!waitForSlowReceivers_) cullSlow();
362 schedule(hmbdc::time::SysTime::now(), typeTagAdTimer_);
364 flushTimer_.setCallback(
366 nmSendTransport_.setSeqAlertPending();
369 schedule(hmbdc::time::SysTime::now(), flushTimer_);
375 queueMemorySegTrain(
typename Buffer::iterator it,
size_t n
376 , uint16_t tag, app::hasMemoryAttachment
const& m,
size_t len) {
379 char* addr =
static_cast<char*
>(s);
380 auto h =
new (addr) TransportMessageHeader;
381 auto& trainHead = (
new (addr +
sizeof(TransportMessageHeader))
382 app::MessageWrap<app::StartMemorySegTrain>)->
template get<app::StartMemorySegTrain>();
383 trainHead.inbandUnderlyingTypeTag = tag;
384 trainHead.segCount = n - 2;
386 h->messagePayloadLen =
sizeof(app::MessageWrap<app::StartMemorySegTrain>);
387 h->setSeq(it++.seq_);
391 auto totalLen = (size_t)m.len;
392 char* base = (
char*)m.attachment;
393 decltype(totalLen) offset = 0;
394 while(totalLen > offset) {
396 char* addr =
static_cast<char*
>(s);
397 auto h =
new (addr) TransportMessageHeader;
398 auto& seg = (
new (addr +
sizeof(TransportMessageHeader)) app::MessageWrap<app::MemorySeg>)
399 ->
template get<app::MemorySeg>();
400 seg.seg = base + offset;
401 auto l = (uint16_t)std::min((
size_t)maxMemorySegPayloadSize_, totalLen - offset);
403 seg.inbandUnderlyingTypeTag = tag;
404 h->messagePayloadLen =
sizeof(app::MessageHead) + l;
405 h->setSeq(it++.seq_);
413 char* addr =
static_cast<char*
>(s);
414 auto h =
new (addr) TransportMessageHeader;
415 new (addr +
sizeof(TransportMessageHeader)) app::MessageWrap<app::JustBytes>(
417 h->messagePayloadLen =
sizeof(app::MessageHead) + len;
418 h->flag = app::hasMemoryAttachment::flag;
419 h->setSeq(it++.seq_);
427 asyncBackupSendServer_.stop();
436 outBufferSizePower2() {
437 auto res = config_.
getExt<uint16_t>(
"outBufferSizePower2");
441 res =hmbdc::numeric::log2Upper(512ul * 1024ul / (8ul + maxMessageSize_));
442 HMBDC_LOG_N(
"auto set --outBufferSizePower2=", res);
450 if (hmbdc_likely(minRecvToStart_ == std::numeric_limits<size_t>::max())) {
451 auto seq = buffer_.readSeq(1);
452 if (seq == lastBackupSeq_ && buffer_.isFull()) {
453 asyncBackupSendServer_.killSlowestSession();
455 lastBackupSeq_ = seq;
T getExt(const path_type ¶m, bool throwIfMissing=true) const
get a value from the config
Definition: Config.hpp:238
class to hold an hmbdc configuration
Definition: Config.hpp:44
void stoppedCb(std::exception const &e) override
callback called when this Client is taken out of message dispatching
Definition: SendTransportEngine.hpp:300
bool droppedCb() override
callback called after the Client is safely taken out of the Context
Definition: SendTransportEngine.hpp:305
void invokedCb(size_t) HMBDC_RESTRICT override
this callback is called all the time (frequently) - the exact timing is after a batch of messages are...
Definition: SendTransportEngine.hpp:295
Definition: TypedString.hpp:84
Definition: Timers.hpp:70
Definition: SendTransportEngine.hpp:25
Definition: SendTransportEngine.hpp:277
void rotate()
if the user choose no to have a Context to manage and run the engine this method can be called from a...
Definition: SendTransportEngine.hpp:289
Definition: Transport.hpp:37
Definition: NmSendTransport.hpp:38
Definition: Message.hpp:212
Definition: BackupSendServerT.hpp:73
Definition: Message.hpp:405
void schedule(SysTime fireAt, Timer &timer)
schedule the timer to start at a specific time
Definition: Timers.hpp:79
Definition: Message.hpp:263
a Node is a thread of execution that can suscribe and receive Messages
Definition: Node.hpp:51
A Client represents a thread of execution/a task. The execution is managed by a Context. a Client object could participate in message dispatching as the receiver of specifed message types.
Definition: Client.hpp:128
Definition: Message.hpp:418
if a specific hmbdc network transport (for example tcpcast, rmcast, and rnetmap) supports message wit...
Definition: Message.hpp:125
Definition: Timers.hpp:117