hmbdc
simplify-high-performance-messaging-programming
McSendTransport.hpp
1 #include "hmbdc/Copyright.hpp"
2 #pragma once
3 #include "hmbdc/app/Base.hpp"
4 #include "hmbdc/tips/rmcast/Transport.hpp"
5 #include "hmbdc/tips/rmcast/Messages.hpp"
6 #include "hmbdc/tips/udpcast/Transport.hpp"
7 #include "hmbdc/app/utils/EpollTask.hpp"
8 #include "hmbdc/comm/inet/Endpoint.hpp"
9 #include "hmbdc/time/Time.hpp"
10 #include "hmbdc/time/Rater.hpp"
11 #include "hmbdc/pattern/LockFreeBufferT.hpp"
12 
13 
14 #include <boost/bind.hpp>
15 #include <memory>
16 
17 #include <iostream>
18 
19 namespace hmbdc { namespace tips { namespace rmcast {
20 
21 namespace mcsendtransport_detail {
22 
24 
26 : Transport {
27  using ptr = std::shared_ptr<McSendTransport>;
29  , size_t maxMessageSize
30  , Buffer& buffer
31  , hmbdc::time::Rater& rater
32  , ToCleanupAttQueue& toCleanupAttQueue)
33  : Transport(cfg)
34  , maxMessageSize_(maxMessageSize)
35  , buffer_(buffer)
36  , wasteSize_(0)
37  , mcFd_(cfg)
38  , mcAddr_(comm::inet::Endpoint(config_.getExt<std::string>("mcastAddr")
39  , config_.getExt<uint16_t>("mcastPort")).v)
40  , toSend_{0}
41  , rater_(rater)
42  , maxSendBatch_(config_.getExt<size_t>("maxSendBatch"))
43  , adPending_(false)
44  , seqAlertPending_(false)
45  , seqAlert_(nullptr)
46  , startSending_(false)
47  , toCleanupAttQueue_(toCleanupAttQueue) {
48  toSend_.msg_name = &mcAddr_;
49  toSend_.msg_namelen = sizeof(mcAddr_);
50  auto totalHead = sizeof(app::MessageHead) + sizeof(TransportMessageHeader);
51  if (maxMessageSize_ + totalHead > mtu_) {
52  HMBDC_THROW(std::out_of_range, "maxMessageSize need <= " << mtu_ - totalHead);
53  }
54 
55  auto addr = seqAlertBuf_;
56  auto h = reinterpret_cast<TransportMessageHeader*>(addr);
58  h->messagePayloadLen = sizeof(app::MessageWrap<SeqAlert>);
59  h->setSeq(std::numeric_limits<HMBDC_SEQ_TYPE>::max());
60  seqAlert_ = &(h->wrapped<SeqAlert>());
61 
62  toSendMsgs_.reserve((maxSendBatch_ + 2) * 2); //double for MemorySeg cases
63 
64  auto ttl = config_.getExt<int>("ttl");
65  if (ttl > 0 && setsockopt(mcFd_.fd, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) < 0) {
66  HMBDC_THROW(std::runtime_error, "failed to set set ttl=" << ttl << " errno=" << errno);
67  }
68  char loopch = config_.getExt<bool>("loopback")?1:0;
69  if (setsockopt(mcFd_.fd, IPPROTO_IP, IP_MULTICAST_LOOP, (char *)&loopch, sizeof(loopch)) < 0) {
70  HMBDC_THROW(std::runtime_error, "failed to set loopback=" << config_.getExt<bool>("loopback"));
71  }
72 
73  auto sz = config_.getExt<int>("udpSendBufferBytes");
74  if (sz) {
75  if (setsockopt(mcFd_.fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)) < 0) {
76  HMBDC_LOG_C("failed to set send buffer size=", sz);
77  }
78  }
79  hmbdc::app::utils::EpollTask::instance().add(
80  hmbdc::app::utils::EpollTask::EPOLLOUT
81  | hmbdc::app::utils::EpollTask::EPOLLET, mcFd_);
82  }
83 
84  ~McSendTransport(){
85  }
86 
87  void startSend(){
88  startSending_ = true;
89  }
90 
91  template <typename AdvertisingMessages>
92  void setAds(AdvertisingMessages const& ads) {
93  decltype(adBufs_) newAdBufs;
94  for (auto const& ad : ads) {
95  newAdBufs.emplace_back();
96  auto addr = newAdBufs.rbegin()->data();
97  auto h = new (addr) TransportMessageHeader;
98  new (addr + sizeof(TransportMessageHeader))
100  h->messagePayloadLen = sizeof(app::MessageWrap<TypeTagBackupSource>);
101  h->setSeq(std::numeric_limits<HMBDC_SEQ_TYPE>::max());
102  }
103  std::swap(adBufs_, newAdBufs);
104  }
105 
106  void setAdPending() {
107  adPending_ = true;
108  }
109 
110  void setSeqAlertPending() {
111  seqAlertPending_ = true;
112  }
113 
114 
115  void runOnce(size_t sessionCount) HMBDC_RESTRICT {
116  resumeSend(sessionCount);
117  }
118 
119  void stop(){
120  buffer_.reset(0);
121  }
122 
123 private:
125  size_t maxMessageSize_;
126  Buffer& HMBDC_RESTRICT buffer_;
127  size_t wasteSize_;
128 
129  udpcast::EpollFd mcFd_;
130  sockaddr_in mcAddr_;
131  msghdr toSend_;
132  hmbdc::time::Rater& HMBDC_RESTRICT rater_;
133  size_t maxSendBatch_;
134  std::vector<std::array<char, sizeof(TransportMessageHeader)
135  + sizeof(app::MessageWrap<TypeTagBackupSource>)>> adBufs_;
136  bool adPending_;
137  char seqAlertBuf_[sizeof(TransportMessageHeader) + sizeof(app::MessageWrap<SeqAlert>)];
138  bool seqAlertPending_;
139  SeqAlert* HMBDC_RESTRICT seqAlert_;
140  bool startSending_;
141  std::vector<iovec> toSendMsgs_;
142  ToCleanupAttQueue& toCleanupAttQueue_;
143 
144  void
145  resumeSend(size_t sessionCount) {
146  do {
147  if (hmbdc_likely(toSendMsgs_.size())) {
148  if (hmbdc_unlikely(!mcFd_.isFdReady())) return;
149 
150  toSend_.msg_iov = &(toSendMsgs_[0]);
151  toSend_.msg_iovlen = toSendMsgs_.size();
152  if (sendmsg(mcFd_.fd, &toSend_, MSG_DONTWAIT) < 0) {
153  if (!mcFd_.checkErr()) {
154  HMBDC_LOG_C("sendmmsg failed errno=", errno);
155  }
156  return;
157  } else {
158  buffer_.wasteAfterPeek(0, wasteSize_);
159  toSendMsgs_.clear();
160  wasteSize_ = 0;
161  }
162  }
163 
164  size_t batchBytes = 0;
165  if (hmbdc_unlikely(adPending_)) {
166  for (auto& adBuf : adBufs_) {
167  toSendMsgs_.push_back(iovec{(void*)adBuf.data(), adBuf.size()});
168  batchBytes += sizeof(adBuf);
169  }
170  adPending_ = false;
171  }
172 
173  if (hmbdc_likely(startSending_) && !toSendMsgs_.size()) {
174  Buffer::iterator begin, end;
175  buffer_.peek(0, begin, end, maxSendBatch_);
176  if (hmbdc_unlikely(!sessionCount)) {
177  buffer_.wasteAfterPeek(0u, end - begin);
178  begin = end;
179  }
180  auto it = begin;
181  while (it != end) {
182  void* ptr = *it;
183  auto item = static_cast<TransportMessageHeader*>(ptr);
184  if (hmbdc_unlikely(!rater_.check(item->wireSize()))) break;
185  batchBytes += item->wireSize();
186  if (hmbdc_unlikely(batchBytes > mtu_)) break;
187  if (hmbdc_unlikely(item->typeTag() == app::MemorySeg::typeTag)) {
188  toSendMsgs_.push_back(iovec{(void*)item->wireBytes(), item->wireSize() - item->wireSizeMemorySeg()});
189  toSendMsgs_.push_back(iovec{(void*)item->wireBytesMemorySeg(), item->wireSizeMemorySeg()});
190  } else {
191  if (hmbdc_unlikely(item->typeTag() == app::StartMemorySegTrain::typeTag)) {
192  auto& trainHead = item->template wrapped<app::StartMemorySegTrain>();
193  auto itActual = it; itActual.seq_ += trainHead.segCount + 1;
194  auto actual = static_cast<TransportMessageHeader*>(*itActual);
195  toCleanupAttQueue_.push_back(std::make_tuple(itActual.seq_
196  , &actual->template wrapped<app::hasMemoryAttachment>()
197  , trainHead.att.afterConsumedCleanupFunc));
198  }
199  toSendMsgs_.push_back(iovec{(void*)item->wireBytes(), item->wireSize()});
200  }
201  it++;
202 
203  seqAlert_->expectSeq = item->getSeq() + 1;
204  rater_.commit();
205  }
206  wasteSize_ = it - begin;
207  }
208  if (hmbdc_unlikely(seqAlertPending_)) {
209  if (!wasteSize_) {
210  toSendMsgs_.push_back(iovec{seqAlertBuf_, sizeof(seqAlertBuf_)});
211  batchBytes += sizeof(seqAlertBuf_);
212  } //else no need to send seq alert
213  seqAlertPending_ = false;
214  }
215  } while (hmbdc_likely(toSendMsgs_.size()));
216  }
217 };
218 
219 } //mcsendtransport_detail
221 }}}
T getExt(const path_type &param, bool throwIfMissing=true) const
get a value from the config
Definition: Config.hpp:238
class to hold an hmbdc configuration
Definition: Config.hpp:44
Definition: Endpoint.hpp:17
Definition: Message.hpp:212
Definition: Messages.hpp:163
Definition: Transport.hpp:21
Definition: Message.hpp:263
Definition: Rater.hpp:11
Definition: Base.hpp:12
Definition: LockFreeBufferMisc.hpp:89