hmbdc
simplify-high-performance-messaging-programming
BackupSendServerT.hpp
1 #include "hmbdc/Copyright.hpp"
2 #pragma once
3 #include "hmbdc/tips/reliable/TcpEpollFd.hpp"
4 #include "hmbdc/tips/TypeTagSet.hpp"
5 #include "hmbdc/app/Logger.hpp"
6 #include "hmbdc/time/Time.hpp"
7 #include "hmbdc/time/Rater.hpp"
8 #include "hmbdc/pattern/LockFreeBufferT.hpp"
9 #include "hmbdc/comm/inet/Misc.hpp"
10 
11 #include <boost/circular_buffer.hpp>
12 #include <unordered_set>
13 #include <memory>
14 #include <tuple>
15 #include <utility>
16 #include <netinet/tcp.h>
17 
18 namespace hmbdc { namespace tips { namespace reliable {
19 
20 using ToCleanupAttQueue = boost::circular_buffer<std::tuple<
21  HMBDC_SEQ_TYPE
22  , app::hasMemoryAttachment*
23  , app::hasMemoryAttachment::AfterConsumedCleanupFunc
24  >
25 >;
26 
27 namespace backupsendservert_detail {
28 template <typename TypeTagBackupSource, typename BackupSendSession>
30  BackupSendServerT(app::Config const& cfg)
31  : config_(cfg)
32  , serverFd_(config_)
33  , serverAddr_(serverFd_.localAddr) {
34  if (listen(serverFd_.fd, 10) < 0) {
35  HMBDC_THROW(std::runtime_error, "failed to listen, errno=" << errno);
36  }
37  hmbdc::app::utils::EpollTask::instance().add(
38  hmbdc::app::utils::EpollTask::EPOLLIN
39  | hmbdc::app::utils::EpollTask::EPOLLET, serverFd_);
40  }
41 
42  void advertisingMessages(TypeTagSetST& tts) {
43  decltype(advertisingMessages_) newAds;
44  tts.exportTo([&](uint16_t tag, auto&& subCount) {
45  if (newAds.size() == 0
46  || !newAds.rbegin()->addTypeTag(tag)) {
47  newAds.emplace_back(serverFd_.localIp
48  , serverFd_.localPort
49  , time::Duration::microseconds(config_.getExt<uint32_t>("recvReportDelayMicrosec"))
50  , config_.getExt<bool>("loopback"));
51  newAds.rbegin()->addTypeTag(tag);
52  }
53  });
54  std::swap(advertisingMessages_, newAds);
55  for (auto& m : advertisingMessages_) {
56  HMBDC_LOG_N("adverise at ", m);
57  }
58  }
59 
60  auto const& advertisingMessages() const {
61  return advertisingMessages_;
62  }
63 
64 protected:
65  app::Config config_;
66  TcpEpollFd serverFd_;
67  sockaddr_in& serverAddr_;
68  std::vector<TypeTagBackupSource> advertisingMessages_;
69 };
70 
71 
72 template <typename TypeTagBackupSource, typename BackupSendSession>
74 : BackupSendServerT<TypeTagBackupSource, BackupSendSession> {
75  using Buffer = typename BackupSendSession::Buffer;
77  , Buffer& buffer
78  , time::Rater& rater
79  , ToCleanupAttQueue& toCleanupAttQueue
80  , TypeTagSet& outboundSubscriptions
82  , buffer_(buffer)
83  , rater_(rater)
84  , maxSendBatch_(cfg.getExt<uint32_t>("maxSendBatch"))
85  , toCleanupAttQueue_(toCleanupAttQueue)
86  , replayHistoryForNewRecv_(cfg.getExt<size_t>("replayHistoryForNewRecv"))
87  , outboundSubscriptions_(outboundSubscriptions) {
88  leastIt_.seq_ = std::numeric_limits<HMBDC_SEQ_TYPE>::max();
89  }
90 
91  virtual ~AsyncBackupSendServerT() = default;
92 
93  void runOnce() HMBDC_RESTRICT {
94  // utils::EpollTask::instance().poll();
95  doAccept();
96  typename Buffer::iterator leastIt;
97  auto newSeq_= buffer_.readSeq(0);
98  for (auto it = sessions_.begin(); it != sessions_.end();) {
99  if (hmbdc_unlikely(!(*it)->runOnce())) {
100  (*it)->stop();
101  sessions_.erase(it++);
102  } else {
103  if (newSeq_ > (*it)->bufItNext_.seq_) {
104  newSeq_ = (*it)->bufItNext_.seq_;
105  }
106  it++;
107  }
108  }
109  if (hmbdc_likely(replayHistoryForNewRecv_ < newSeq_)) {
110  newSeq_ -= replayHistoryForNewRecv_;
111  } else {
112  newSeq_ = 0;
113  }
114  if (leastIt_.seq_ != newSeq_) {
115  while (toCleanupAttQueue_.size()) {
116  auto it = toCleanupAttQueue_.begin();
117  if (newSeq_ > std::get<0>(*it)) {
118  auto f = std::get<2>(*it);
119  if (f) f(std::get<1>(*it));
120  toCleanupAttQueue_.pop_front();
121  } else {
122  break;
123  }
124  }
125  buffer_.catchUpTo(1, newSeq_);
126  }
127  leastIt_.seq_ = newSeq_;
128  }
129 
130  void stop() {
131  while (toCleanupAttQueue_.size()) {
132  auto it = toCleanupAttQueue_.begin();
133  auto f = std::get<2>(*it);
134  if (f) f(std::get<1>(*it));
135  toCleanupAttQueue_.pop_front();
136  }
137  }
138 
139  size_t sessionCount() const {
140  return sessions_.size();
141  }
142 
143  size_t readySessionCount() HMBDC_RESTRICT const {
144  size_t res = 0;
145  for (auto const& s : sessions_) {
146  if (s->ready()) res++;
147  }
148  return res;
149  }
150 
151  void killSlowestSession() {
152  if (sessions_.size() == 0) {
153  return;
154  }
155 
156  auto it = sessions_.begin();
157  auto leastSessIt = it;
158  typename Buffer::iterator leastIt = (*it++)->bufItNext_;
159  for (; it != sessions_.end(); it++) {
160  if ((*it)->bufItNext_ < leastIt) {
161  leastIt = (*it)->bufItNext_;
162  leastSessIt = it;
163  }
164  }
165  HMBDC_LOG_C((*leastSessIt)->id(), " too slow, dropping");
166  (*leastSessIt)->stop();
167  sessions_.erase(leastSessIt);
168  }
169 
170 private:
171  void doAccept() HMBDC_RESTRICT {
172  auto& serverFd_ = this->serverFd_;
173  auto& serverAddr_ = this->serverAddr_;
174  if (hmbdc_unlikely(serverFd_.isFdReady())) {
175  auto addrlen = sizeof(serverAddr_);
176  auto conn = accept(serverFd_.fd, (struct sockaddr *)&serverAddr_, (socklen_t*)&addrlen);
177  if (conn == -1) {
178  if (!serverFd_.checkErr()) {
179  HMBDC_LOG_C("accept failure, errno=", errno);
180  }
181  return;
182  }
183  auto sz = this->config_.template getExt<int>("tcpSendBufferBytes");
184  if (sz) {
185  if (setsockopt(conn, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)) < 0) {
186  HMBDC_LOG_C("failed to set send buffer size=", sz, " errno=", errno);
187  }
188  }
189  int flag = this->config_.template getExt<bool>("nagling")?0:1;
190  if (setsockopt(conn, IPPROTO_TCP, TCP_NODELAY, (char*) &flag, sizeof(flag)) < 0) {
191  HMBDC_LOG_C("failed to set TCP_NODELAY, errno=", errno);
192  }
193  try {
194  typename Buffer::iterator it;
195  buffer_.peek(1, it, it, 0);
196  auto s = std::make_shared<BackupSendSession>(conn
197  , buffer_
198  , rater_
199  , it
200  , maxSendBatch_
201  , outboundSubscriptions_);
202  s->start();
203  sessions_.insert(s);
204  } catch (std::exception const& e) {
205  HMBDC_LOG_C(e.what());
206  }
207  };
208  }
209 
210  using Sessions = std::unordered_set<typename BackupSendSession::ptr>;
211  Sessions sessions_;
212  Buffer& HMBDC_RESTRICT buffer_;
213  hmbdc::time::Rater& HMBDC_RESTRICT rater_;
214  typename Buffer::iterator leastIt_;
215  size_t maxSendBatch_;
216  ToCleanupAttQueue& toCleanupAttQueue_;
217  size_t replayHistoryForNewRecv_;
218  TypeTagSet& outboundSubscriptions_;
219 };
220 
221 } //backupsendservert_detail
222 
223 template <typename TypeTagBackupSource, typename BackupSendSession>
225 
226 }}}
T getExt(const path_type &param, bool throwIfMissing=true) const
get a value from the config
Definition: Config.hpp:238
Definition: TcpEpollFd.hpp:14
class to hold an hmbdc configuration
Definition: Config.hpp:44
Definition: TypeTagSet.hpp:138
Definition: TypeTagSet.hpp:198
Definition: Rater.hpp:11
Definition: Base.hpp:12
Definition: LockFreeBufferMisc.hpp:89