1 #include "hmbdc/Copyright.hpp" 3 #include "hmbdc/app/Client.hpp" 4 #include "hmbdc/app/Logger.hpp" 5 #include "hmbdc/tips/rmcast/Transport.hpp" 6 #include "hmbdc/tips/rmcast/BackupSendServer.hpp" 7 #include "hmbdc/tips/rmcast/Messages.hpp" 8 #include "hmbdc/tips/rmcast/McSendTransport.hpp" 9 #include "hmbdc/tips/rmcast/DefaultUserConfig.hpp" 11 #include "hmbdc/time/Time.hpp" 12 #include "hmbdc/time/Rater.hpp" 13 #include "hmbdc/numeric/BitMath.hpp" 14 #include "hmbdc/MetaUtils.hpp" 20 #include <type_traits> 22 namespace hmbdc {
namespace tips {
namespace rmcast {
24 namespace sendtransportengine_detail {
25 HMBDC_CLASS_HAS_DECLARE(hmbdc_net_queued_ts);
42 size_t bufferedMessageCount()
const {
43 return buffer_.remainingSize();
46 size_t subscribingPartyDetectedCount(uint16_t tag)
const {
47 return outboundSubscriptions_.check(tag);
50 template <
typename MemoryAttachementMessage>
52 ,
typename std::decay<MemoryAttachementMessage>::type>::value,
void>::type
53 queue(MemoryAttachementMessage&& msg) {
54 if (!minRecvToStart_ && !outboundSubscriptions_.check(msg.getTypeTag())) {
58 using Message =
typename std::decay<MemoryAttachementMessage>::type;
59 if (hmbdc_unlikely(
sizeof(Message) > maxMessageSize_
62 HMBDC_THROW(std::out_of_range
63 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
67 size_t n = (msg.app::hasMemoryAttachment::len + maxMemorySegPayloadSize_ - 1ul) / maxMemorySegPayloadSize_ + 2;
68 if (hmbdc_unlikely(n > buffer_.capacity())) {
69 HMBDC_THROW(std::out_of_range
70 ,
"send engine buffer too small capacity=" << buffer_.capacity() <<
"<" << n);
72 auto it = buffer_.claim(n);
73 queueMemorySegTrain(it, n, std::forward<MemoryAttachementMessage>(msg));
74 buffer_.commit(it, n);
78 template <MessageC Message>
80 ,
typename std::decay<Message>::type>::value,
void>::type
81 queue(Message&& msg) {
82 if (!outboundSubscriptions_.check(msg.getTypeTag()))
return;
84 auto it = buffer_.claim(n);
85 queue(it, std::forward<Message>(msg));
86 buffer_.commit(it, n);
89 template <MessageC Message>
91 ,
typename std::decay<Message>::type>::value,
bool>::type
92 tryQueueMessages(Message&& msg) {
93 if (!outboundSubscriptions_.check(msg.getTypeTag()))
return true;
95 auto it = buffer_.tryClaim(n);
97 queue(it, std::forward<Message>(msg));
98 buffer_.commit(it, n);
104 template <
typename Message,
typename ... Args>
105 void queueInPlace(Args&&... args) {
106 static_assert(std::is_trivially_destructible<Message>::value,
"cannot send message with dtor");
107 auto s = buffer_.claim();
108 char* addr =
static_cast<char*
>(*s);
110 if (hmbdc_likely(
sizeof(Message) <= maxMessageSize_)) {
115 HMBDC_THROW(std::out_of_range
116 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
121 void queueJustBytes(uint16_t tag,
void const* bytes,
size_t len
124 if (!outboundSubscriptions_.check(tag))
return;
125 auto it = buffer_.claim();
127 char* addr =
static_cast<char*
>(s);
129 if (hmbdc_likely(len <= maxMessageSize_)) {
134 HMBDC_THROW(std::out_of_range
135 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
139 if (!minRecvToStart_ && !outboundSubscriptions_.check(tag)) {
143 if (hmbdc_unlikely(len > maxMessageSize_
146 HMBDC_THROW(std::out_of_range
147 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
151 size_t n = (att->len + maxMemorySegPayloadSize_ - 1ul) / maxMemorySegPayloadSize_ + 2;
152 if (hmbdc_unlikely(n > buffer_.capacity())) {
153 HMBDC_THROW(std::out_of_range
154 ,
"send engine buffer too small capacity=" << buffer_.capacity() <<
"<" << n);
156 auto it = buffer_.claim(n);
157 queueMemorySegTrain(it, n, tag, *att, len);
158 buffer_.commit(it, n);
163 hmbdc::app::utils::EpollTask::instance().poll();
164 if (hmbdc_unlikely(minRecvToStart_ != std::numeric_limits<size_t>::max()
165 && asyncBackupSendServer_.readySessionCount() >= minRecvToStart_)) {
166 minRecvToStart_ = std::numeric_limits<size_t>::max();
167 mcSendTransport_.startSend();
169 asyncBackupSendServer_.runOnce();
170 if (advertisedTypeTagsDirty_) {
171 std::unique_lock<std::mutex> g(advertisedTypeTags_.lock, std::try_to_lock);
173 mcSendTransport_.setAds(asyncBackupSendServer_.advertisingMessages());
174 advertisedTypeTagsDirty_ =
false;
177 mcSendTransport_.runOnce(asyncBackupSendServer_.sessionCount());
183 size_t sessionsRemainingActive()
const {
184 return asyncBackupSendServer_.readySessionCount();
187 template <MessageTupleC Messages,
typename Node>
188 void advertiseFor(
Node const& node, uint16_t mod, uint16_t res) {
189 std::scoped_lock<std::mutex> g(advertisedTypeTags_.lock);
190 advertisedTypeTags_.markPubsFor<Messages>(node, mod, res);
191 asyncBackupSendServer_.advertisingMessages(advertisedTypeTags_);
192 advertisedTypeTagsDirty_ =
true;
196 size_t maxMessageSize_;
202 size_t minRecvToStart_;
204 HMBDC_SEQ_TYPE lastBackupSeq_;
206 uint16_t maxMemorySegPayloadSize_;
207 bool waitForSlowReceivers_;
208 reliable::ToCleanupAttQueue toCleanupAttQueue_;
212 std::atomic<bool> advertisedTypeTagsDirty_ =
false;
217 outBufferSizePower2();
220 void queueMemorySegTrain(
typename Buffer::iterator it,
size_t n, M&& m) {
221 using Message =
typename std::decay<M>::type;
224 char* addr =
static_cast<char*
>(s);
229 trainHead.inbandUnderlyingTypeTag = m.getTypeTag();
231 trainHead.segCount = n - 2;
233 h->setSeq(it++.seq_);
237 auto totalLen = (size_t)m.app::hasMemoryAttachment::len;
238 char* base = (
char*)m.app::hasMemoryAttachment::attachment;
239 decltype(totalLen) offset = 0;
240 while(totalLen > offset) {
242 char* addr =
static_cast<char*
>(s);
245 ->
template get<app::MemorySeg>();
246 seg.seg = base + offset;
247 seg.inbandUnderlyingTypeTag = m.getTypeTag();
248 auto l = (uint16_t)std::min((
size_t)maxMemorySegPayloadSize_, totalLen - offset);
251 h->setSeq(it++.seq_);
259 char* addr =
static_cast<char*
>(s);
263 h->flag = app::hasMemoryAttachment::flag;
264 h->setSeq(it++.seq_);
268 void queueMemorySegTrain(
typename Buffer::iterator it,
size_t n
271 template<
typename M,
typename ... Messages>
272 void queue(
typename Buffer::iterator it
273 , M&& m, Messages&&... msgs) {
274 using Message =
typename std::decay<M>::type;
275 static_assert(std::is_trivially_destructible<Message>::value,
"cannot send message with dtor");
277 char* addr =
static_cast<char*
>(s);
279 if (hmbdc_likely(
sizeof(Message) <= maxMessageSize_)) {
284 HMBDC_THROW(std::out_of_range
285 ,
"maxMessageSize too small to hold a message when constructing SendTransportEngine");
287 if constexpr (has_hmbdc_net_queued_ts<Message>::value) {
288 h->template wrapped<Message>().hmbdc_net_queued_ts = hmbdc::time::SysTime::now();
290 queue(++it, std::forward<Messages>(msgs)...);
293 void queue(Buffer::iterator it) {}
304 if (runLock_.try_lock()) {
305 this->checkTimers(hmbdc::time::SysTime::now());
328 using EngineTransport::hmbdcName;
330 std::tuple<char const*, int> schedSpec()
const {
331 return std::make_tuple(this->schedPolicy_.c_str(), this->schedPriority_);
335 this->runLock_.unlock();
343 using SendTransport = sendtransportengine_detail::SendTransport;
344 using SendTransportEngine = sendtransportengine_detail::SendTransportEngine;
347 namespace hmbdc {
namespace tips {
namespace rmcast {
349 namespace sendtransportengine_detail {
354 ,
size_t maxMessageSize)
356 , cfg.resetSection(
"tx", false)))
357 , maxMessageSize_(maxMessageSize)
359 , outBufferSizePower2())
360 , rater_(
hmbdc::time::Duration::seconds(1u)
361 , config_.getExt<size_t>(
"sendBytesPerSec")
362 , config_.getExt<size_t>(
"sendBytesBurst")
363 , config_.getExt<size_t>(
"sendBytesBurst") != 0ul)
364 , mcSendTransport_(config_, maxMessageSize, buffer_, rater_, toCleanupAttQueue_)
365 , minRecvToStart_(config_.getExt<size_t>(
"minRecvToStart"))
366 , typeTagAdTimer_(
hmbdc::time::Duration::seconds(config_.getExt<uint32_t>(
"typeTagAdvertisePeriodSeconds")))
367 , lastBackupSeq_(
std::numeric_limits<HMBDC_SEQ_TYPE>::max())
368 , flushTimer_(
hmbdc::time::Duration::microseconds(config_.getExt<uint32_t>(
"netRoundtripLatencyMicrosec")))
372 , waitForSlowReceivers_(config_.getExt<bool>(
"waitForSlowReceivers"))
373 , asyncBackupSendServer_(config_, buffer_, rater_, toCleanupAttQueue_, outboundSubscriptions_)
375 if (maxMessageSize_ > mtu_) {
376 HMBDC_THROW(std::out_of_range,
"mtu needs to >= " << maxMessageSize_);
379 if (maxMessageSize_ > TransportMessageHeader::maxPayloadSize()) {
380 HMBDC_THROW(std::out_of_range
381 ,
"maxMessageSize_ needs to <=" << TransportMessageHeader::maxPayloadSize());
383 auto sendBytesBurst = config_.
getExt<
size_t>(
"sendBytesBurst");
385 if (sendBytesBurst && sendBytesBurst < sendBytesBurstMin) {
386 HMBDC_THROW(std::out_of_range,
"sendBytesBurst needs to >= " << sendBytesBurstMin);
389 toCleanupAttQueue_.set_capacity(buffer_.capacity() + 10);
390 typeTagAdTimer_.setCallback(
392 mcSendTransport_.setAdPending();
393 if (!waitForSlowReceivers_) cullSlow();
396 schedule(hmbdc::time::SysTime::now(), typeTagAdTimer_);
398 flushTimer_.setCallback(
400 mcSendTransport_.setSeqAlertPending();
403 schedule(hmbdc::time::SysTime::now(), flushTimer_);
410 asyncBackupSendServer_.stop();
419 outBufferSizePower2() {
420 auto res = config_.
getExt<uint16_t>(
"outBufferSizePower2");
424 res =hmbdc::numeric::log2Upper(128ul * 1024ul / (8ul + maxMessageSize_));
425 HMBDC_LOG_N(
"auto set --outBufferSizePower2=", res);
432 queueMemorySegTrain(
typename Buffer::iterator it,
size_t n
436 char* addr =
static_cast<char*
>(s);
437 auto h =
new (addr) TransportMessageHeader;
438 auto& trainHead = (
new (addr +
sizeof(TransportMessageHeader))
440 trainHead.inbandUnderlyingTypeTag = tag;
441 trainHead.segCount = n - 2;
444 h->setSeq(it++.seq_);
448 auto totalLen = (size_t)m.len;
449 char* base = (
char*)m.attachment;
450 decltype(totalLen) offset = 0;
451 while(totalLen > offset) {
453 char* addr =
static_cast<char*
>(s);
454 auto h =
new (addr) TransportMessageHeader;
456 ->
template get<app::MemorySeg>();
457 seg.seg = base + offset;
458 auto l = (uint16_t)std::min((
size_t)maxMemorySegPayloadSize_, totalLen - offset);
460 seg.inbandUnderlyingTypeTag = tag;
462 h->setSeq(it++.seq_);
470 char* addr =
static_cast<char*
>(s);
471 auto h =
new (addr) TransportMessageHeader;
472 new (addr +
sizeof(TransportMessageHeader)) app::MessageWrap<app::JustBytes>(
474 h->messagePayloadLen =
sizeof(app::MessageHead) + len;
475 h->flag = app::hasMemoryAttachment::flag;
476 h->setSeq(it++.seq_);
484 if (hmbdc_likely(minRecvToStart_ == std::numeric_limits<size_t>::max())) {
485 auto seq = buffer_.readSeq(1);
486 if (seq == lastBackupSeq_ && buffer_.isFull()) {
487 asyncBackupSendServer_.killSlowestSession();
489 lastBackupSeq_ = seq;
T getExt(const path_type ¶m, bool throwIfMissing=true) const
get a value from the config
Definition: Config.hpp:238
class to hold an hmbdc configuration
Definition: Config.hpp:44
void invokedCb(size_t) HMBDC_RESTRICT override
this callback is called all the time (frequently) - the exact timing is after a batch of messages are...
Definition: SendTransportEngine.hpp:318
Definition: TypedString.hpp:84
Definition: SendTransportEngine.hpp:298
Definition: Timers.hpp:70
Definition: McSendTransport.hpp:25
void messageDispatchingStartedCb(size_t const *) override
called before any messages got dispatched - only once
Definition: SendTransportEngine.hpp:312
SendTransport(hmbdc::app::Config cfg, size_t maxMessageSize)
ctor
Definition: SendTransportEngine.hpp:353
Definition: Transport.hpp:39
Definition: Message.hpp:212
Definition: BackupSendServerT.hpp:73
Definition: Message.hpp:403
bool droppedCb() override
callback called after the Client is safely taken out of the Context
Definition: SendTransportEngine.hpp:323
void schedule(SysTime fireAt, Timer &timer)
schedule the timer to start at a specific time
Definition: Timers.hpp:79
Definition: Message.hpp:263
a Node is a thread of execution that can suscribe and receive Messages
Definition: Node.hpp:51
A Client represents a thread of execution/a task. The execution is managed by a Context. a Client object could participate in message dispatching as the receiver of specifed message types.
Definition: Client.hpp:128
Definition: Message.hpp:416
if a specific hmbdc network transport (for example tcpcast, rmcast, and rnetmap) supports message wit...
Definition: Message.hpp:125
capture the transportation mechanism
Definition: SendTransportEngine.hpp:31
Definition: Timers.hpp:117