hmbdc
simplify-high-performance-messaging-programming
BackupRecvSessionT.hpp
1 #include "hmbdc/Copyright.hpp"
2 #pragma once
3 #include "hmbdc/tips/reliable/TcpEpollFd.hpp"
4 #include "hmbdc/tips/reliable/AttBufferAdaptor.hpp"
5 #include "hmbdc/app/Message.hpp"
6 #include "hmbdc/app/Config.hpp"
7 #include "hmbdc/app/Logger.hpp"
8 #include "hmbdc/pattern/SeqArb.hpp"
9 #include "hmbdc/time/Time.hpp"
10 #include "hmbdc/comm/inet/Misc.hpp"
11 
12 #include <unordered_set>
13 #include <boost/lexical_cast.hpp>
14 
15 #include <functional>
16 #include <memory>
17 #include <utility>
18 #include <regex>
19 
20 #include <iostream>
21 
22 namespace hmbdc { namespace tips { namespace reliable {
23 
24 using Tags = std::unordered_set<uint16_t>;
25 
26 namespace backuprecvsessiont_detail {
27 template <typename TransportMessageHeader
28  , typename SessionStarted
29  , typename SessionDropped
30  , typename SeqAlert
31  , typename OutputBufferIn
32  , typename SendFrom
33  , typename AttachmentAllocator>
36  using ptr = std::shared_ptr<BackupRecvSessionT<TransportMessageHeader
37  , SessionStarted
38  , SessionDropped
39  , SeqAlert
40  , OutputBufferIn
41  , SendFrom
42  , AttachmentAllocator>>;
44  , SendFrom const& sendFrom
45  , TypeTagSet const& subscriptions
46  , OutputBufferIn& outputBuffer
47  , time::Duration recvReportDelay)
48  : hmbdc::time::ReoccuringTimer(recvReportDelay)
49  , sendFrom(sendFrom)
50  , config_(config)
51  , minSeq_(std::numeric_limits<HMBDC_SEQ_TYPE>::max())
52  , subscriptions_(subscriptions)
53  , writeFd_(config_)
54  , outputBuffer_(outputBuffer)
55  , bufSize_(config.getExt<size_t>("maxTcpReadBytes"))
56  , buf_((char*)memalign(SMP_CACHE_BYTES, bufSize_))
57  , bufCur_(buf_)
58  , filledLen_(0)
59  , seqArb_(std::numeric_limits<HMBDC_SEQ_TYPE>::max())
60  , initialized_(false)
61  , ready_(false)
62  , stopped_(false)
63  , recvBackupMessageCount_(0)
64  , gapPending_(false)
65  , gapPendingSeq_(0) {
66  setCallback(
67  [this](hmbdc::time::TimerManager& tm, hmbdc::time::SysTime const& now) {
68  sendGapReport(seqArb_.expectingSeq(), 0);
69  }
70  );
71  }
72 
73  virtual ~BackupRecvSessionT() {
74  ::free(buf_);
75  HMBDC_LOG_N("BackupRecvSession retired: ", id(), " lifetime recved:", recvBackupMessageCount_);
76  }
77 
78  void stop() {
79  outputBuffer_.putSome(sessionDropped_);
80  }
81 
82  void start(in_addr_t ip, uint16_t port) {
83  sockaddr_in remoteAddr = {0};
84  remoteAddr.sin_family = AF_INET;
85  remoteAddr.sin_addr.s_addr = ip;
86  remoteAddr.sin_port = htons(port);
87  if (connect(writeFd_.fd, (sockaddr*)&remoteAddr, sizeof(remoteAddr)) < 0) {
88  if (errno != EINPROGRESS) {
89  HMBDC_THROW(std::runtime_error, "connect fail, errno=" << errno);
90  }
91  }
92 
93  auto forRead = dup(writeFd_.fd);
94  if (forRead == -1) {
95  HMBDC_THROW(std::runtime_error, "dup failed errno=" << errno);
96  }
97 
98  readFd_.fd = forRead;
99  hmbdc::app::utils::EpollTask::instance().add(hmbdc::app::utils::EpollTask::EPOLLOUT
100  | hmbdc::app::utils::EpollTask::EPOLLET, writeFd_);
101  hmbdc::app::utils::EpollTask::instance().add(hmbdc::app::utils::EpollTask::EPOLLIN
102  | hmbdc::app::utils::EpollTask::EPOLLET, readFd_);
103  }
104 
105  bool runOnce() {
106  if (hmbdc_unlikely(stopped_)) return false;
107  if (hmbdc_unlikely(!initialized_ && writeFd_.isFdReady())) {
108  try {
109  initializeConn();
110  } catch (std::exception const& e) {
111  HMBDC_LOG_W(e.what());
112  return false;
113  } catch (...) {
114  HMBDC_LOG_C("unknown exception");
115  return false;
116  }
117  }
118  return doRead();
119  }
120 
121  void sendSubscribe(uint16_t t) {
122  char buf[70];
123  auto l = sprintf(buf, "+%d\t", t) - 1;
124  if (hmbdc_unlikely(send(writeFd_.fd, buf, l, MSG_NOSIGNAL) != l)) {
125  HMBDC_LOG_C("error when sending subscribe to ", id());
126  stopped_ = true;
127  }
128  }
129 
130  void sendUnsubscribe(uint16_t const& t) {
131  char buf[70];
132  auto l = sprintf(buf, "-%d\t", t) - 1;
133  if (hmbdc_unlikely(l != send(writeFd_.fd, buf, l, MSG_NOSIGNAL))) {
134  HMBDC_LOG_C("error when sending unsubscribe to ", id(), " errno=", errno);
135  stopped_ = true;
136  }
137  }
138 
139  void sendGapReport(HMBDC_SEQ_TYPE missSeq, size_t len) {
140  if (hmbdc_likely(!gapPending_ && missSeq != std::numeric_limits<HMBDC_SEQ_TYPE>::max())) {
141  char buf[70];
142  auto l = sprintf(buf, "=%" PRIu64 ",%zu\t", missSeq, len);
143 
144  if (hmbdc_unlikely(l != send(writeFd_.fd, buf, l, MSG_NOSIGNAL))) {
145  HMBDC_LOG_C("error when sending gap report to ", id(), " errno=", errno);
146  stopped_ = true;
147  }
148 
149  if (len) {
150  gapPendingSeq_ = missSeq + len - 1;
151  gapPending_ = true;
152  }
153  }
154  }
155 
156  char const* id() const {
157  return id_.c_str();
158  }
159 
160  int accept(TransportMessageHeader* h) {
161  auto seq = h->getSeq();
162  auto a = arb(0, seq, h);
163  if (a == 1) {
164  auto tag = h->typeTag();
165  if (tag == app::StartMemorySegTrain::typeTag) {
166  tag = h->template wrapped<app::StartMemorySegTrain>().inbandUnderlyingTypeTag;
167  }
168  if (subscriptions_.check(tag)) {
169  outputBuffer_.put(h);
170  }
171  }
172  return a;
173  }
174 
175  int arb(uint16_t part, HMBDC_SEQ_TYPE seq
176  , TransportMessageHeader* h) HMBDC_RESTRICT {
177  //UDP packet out of order case
178  if (hmbdc_unlikely(gapPending_ && part == 0)) return 0;
179  if (seq != std::numeric_limits<HMBDC_SEQ_TYPE>::max()) {
180  auto res = seqArb_(part, seq, [](size_t) {
181  //impossible to get here
182  });
183  if (res == 0) {
184  sendGapReport(seqArb_.expectingSeq(), seq - seqArb_.expectingSeq());
185  } else if (seq == gapPendingSeq_) {
186  gapPending_ = false;
187  }
188  return res;
189  } else if (h->typeTag() == SeqAlert::typeTag) {
190  auto& alert = h->template wrapped<SeqAlert>();
191  auto nextSeq = alert.expectSeq;
192 
193  if (seqArb_.expectingSeq() < nextSeq) {
194  sendGapReport(seqArb_.expectingSeq(), nextSeq - seqArb_.expectingSeq());
195  }
196  // HMBDC_LOG_D(std::this_thread::get_id(), "=-1");
197  return -1;
198  }
199  return 1;
200  }
201 
202  SendFrom const sendFrom;
203 private:
205  void initializeConn() {
206  int flags = fcntl(writeFd_.fd, F_GETFL, 0);
207  flags &= ~O_NONBLOCK;
208  if (fcntl(writeFd_.fd, F_SETFL, flags) < 0) {
209  HMBDC_THROW(std::runtime_error, "fcntl failed errno=" << errno);
210  }
211 
212  auto sz = config_.getExt<int>("tcpRecvBufferBytes");
213  if (sz) {
214  if (setsockopt(readFd_.fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)) < 0) {
215  HMBDC_LOG_C("failed to set send buffer size=", sz);
216  }
217  }
218 
219  auto flag = config_.getExt<bool>("nagling")?0:1;
220  if (setsockopt(writeFd_.fd, IPPROTO_TCP, TCP_NODELAY, (char*) &flag, sizeof(flag)) < 0) {
221  HMBDC_LOG_C("failed to set TCP_NODELAY, errno=", errno);
222  }
223 
224  auto addrPort = hmbdc::comm::inet::getPeerIpPort(writeFd_.fd);
225  id_ = addrPort.first + ":" + std::to_string(addrPort.second);
226  strncpy(sessionStarted_.payload.ip, id_.c_str()
227  , sizeof(sessionStarted_.payload.ip));
228  sessionStarted_.payload.ip[sizeof(sessionStarted_.payload.ip) - 1] = 0;
229  strncpy(sessionDropped_.payload.ip, id_.c_str()
230  , sizeof(sessionDropped_.payload.ip));
231  sessionDropped_.payload.ip[sizeof(sessionDropped_.payload.ip) - 1] = 0;
232  outputBuffer_.putSome(sessionStarted_);
233  sendSubscriptions();
234  initialized_ = true;
235  }
236 
237  void sendSubscriptions() {
238  std::ostringstream oss;
239  subscriptions_.exportTo([&oss](uint16_t tag, auto&& count) {
240  if (count) {
241  oss << '+' << tag << '\t';
242  }
243  });
244  auto s = oss.str() + "+\t"; //mark all sent with "+\t"
245  auto sz = size_t(send(writeFd_.fd, s.c_str(), s.size(), MSG_NOSIGNAL));
246  if (hmbdc_unlikely(sz != s.size())) {
247  HMBDC_LOG_C("error when sending subscriptions to ", id());
248  stopped_ = true;
249  }
250  }
251 
252  bool
253  doRead() HMBDC_RESTRICT {
254  if (hmbdc_unlikely(seqArb_.expectingSeq() == std::numeric_limits<HMBDC_SEQ_TYPE>::max()) &&
255  filledLen_ >= sizeof(HMBDC_SEQ_TYPE)) {
256  minSeq_ = *reinterpret_cast<HMBDC_SEQ_TYPE*>(bufCur_);
257  auto wireSize = sizeof(HMBDC_SEQ_TYPE);
258  bufCur_ += wireSize;
259  filledLen_ -= wireSize;
260  HMBDC_LOG_N("BackupRecvSession started ", id(), " with minSeq=", minSeq_);
261  seqArb_.expectingSeq() = minSeq_;
262  }
263  while (minSeq_ != std::numeric_limits<HMBDC_SEQ_TYPE>::max()
264  && filledLen_ >= sizeof(TransportMessageHeader)) {
265  auto h = reinterpret_cast<TransportMessageHeader*>(bufCur_);
266  auto wireSize = h->wireSize();
267 
268  if (hmbdc_likely(filledLen_ >= wireSize)) {
269  if (hmbdc_likely(h->messagePayloadLen //0 for flushing - no typeTag
270  && subscriptions_.check(h->typeTag()))) {
271  outputBuffer_.put(h);
272  }
273  bufCur_ += wireSize;
274  //memmove(buf_, buf_ + h->wireSize(), filledLen_ - h->wireSize());
275  filledLen_ -= wireSize;
276 
277  arb(1, seqArb_.expectingSeq(), h); //just punch the card
278  recvBackupMessageCount_++;
279  } else {
280  break;
281  }
282  }
283  memmove(buf_, bufCur_, filledLen_);
284  bufCur_ = buf_;
285 
286  if (readFd_.isFdReady()) {
287  auto l = recv(readFd_.fd, buf_ + filledLen_, bufSize_ - filledLen_
288  , MSG_NOSIGNAL|MSG_DONTWAIT);
289  if (hmbdc_unlikely(l < 0)) {
290  if (hmbdc_unlikely(!readFd_.checkErr())) {
291  HMBDC_LOG_C("recv failed errno=", errno);
292  return false;
293  }
294  return true;
295  } else if (hmbdc_unlikely(l == 0)) {
296  HMBDC_LOG_W("peer dropped:", id());
297  return false;
298  }
299  filledLen_ += l;
300  }
301 
302  return true;
303  }
304 
305  hmbdc::app::Config const& config_;
306  HMBDC_SEQ_TYPE minSeq_;
307  TypeTagSet const& subscriptions_;
309  TcpEpollFd writeFd_;
310  OutputBuffer outputBuffer_;
311  std::string id_;
314  size_t const bufSize_;
315  char* buf_;
316  char* bufCur_;
317  size_t filledLen_;
319  bool initialized_;
320  bool ready_;
321  bool stopped_;
322  size_t recvBackupMessageCount_;
323  bool gapPending_; //a gap is reported and not filled yet
324  HMBDC_SEQ_TYPE gapPendingSeq_;
325  uint16_t attTypeTag_;
326  void* attUserScratchpad_;
327 };
328 } //backuprecvsessiont_detail
329 
330 template <typename TransportMessageHeader
331  , typename SessionStarted
332  , typename SessionDropped
333  , typename SeqAlert
334  , typename OutputBuffer
335  , typename SendFrom
336  , typename AttachmentAllocator>
337 using BackupRecvSessionT =
339  TransportMessageHeader
340  , SessionStarted
341  , SessionDropped
342  , SeqAlert
343  , OutputBuffer
344  , SendFrom
345  , AttachmentAllocator
346  >;
347 }}}
T getExt(const path_type &param, bool throwIfMissing=true) const
get a value from the config
Definition: Config.hpp:238
Definition: TcpEpollFd.hpp:14
class to hold an hmbdc configuration
Definition: Config.hpp:44
Definition: Timers.hpp:70
Definition: EpollTask.hpp:87
Definition: Time.hpp:14
Definition: TypeTagSet.hpp:138
Definition: Time.hpp:134
Definition: Base.hpp:12
Definition: Timers.hpp:117