hmbdc
simplify-high-performance-messaging-programming
RecvSession.hpp
1 #include "hmbdc/Copyright.hpp"
2 #pragma once
3 #include "hmbdc/app/Logger.hpp"
4 #include "hmbdc/tips/tcpcast/Transport.hpp"
5 #include "hmbdc/tips/tcpcast/Messages.hpp"
6 #include "hmbdc/tips/TypeTagSet.hpp"
7 #include "hmbdc/comm/inet/Misc.hpp"
8 
9 
10 #include <boost/lexical_cast.hpp>
11 
12 #include <functional>
13 #include <memory>
14 #include <utility>
15 #include <regex>
16 
17 #include <iostream>
18 
19 namespace hmbdc { namespace tips { namespace tcpcast {
20 
21 namespace recvsession_detail {
22 
23 template <typename OutputBuffer, typename AttachmentAllocator>
24 struct RecvSession {
25  using ptr = std::shared_ptr<RecvSession<OutputBuffer, AttachmentAllocator>>;
26  using CleanupFunc = std::function<void()>;
27  RecvSession(hmbdc::app::Config const& config
28  , TypeTagSet const& subscriptions
29  , OutputBuffer& outputBuffer)
30  : config_(config)
31  , subscriptions_(subscriptions)
32  , writeFd_(config)
33  , outputBuffer_(outputBuffer)
34  , initialized_(false)
35  , stopped_(false)
36  , bufSize_(config.getExt<size_t>("maxTcpReadBytes"))
37  , buf_((char*)memalign(SMP_CACHE_BYTES, bufSize_))
38  , bufCur_(buf_)
39  , filledLen_(0)
40  , currTransportHeadFlag_(0)
41  , hasMemoryAttachmentItemSpace_(
42  new char[std::max(outputBuffer_.maxItemSize()
44  , hasMemoryAttachment_(&((hmbdc::app::MessageHead*)(hasMemoryAttachmentItemSpace_.get()))
46  }
47 
48  ~RecvSession() {
49  free(buf_);
50  hasMemoryAttachment_->release();
51  HMBDC_LOG_N("RecvSession retired: ", id());
52  }
53 
54  void start(in_addr_t ip, uint16_t port) {
55  sockaddr_in remoteAddr = {0};
56  remoteAddr.sin_family = AF_INET;
57  remoteAddr.sin_addr.s_addr = ip;
58  remoteAddr.sin_port = htons(port);
59  if (connect(writeFd_.fd, (sockaddr*)&remoteAddr, sizeof(remoteAddr)) < 0) {
60  if (errno != EINPROGRESS) {
61  HMBDC_THROW(std::runtime_error, "connect fail, errno=" << errno);
62  }
63  }
64 
65  auto forRead = dup(writeFd_.fd);
66  if (forRead == -1) {
67  HMBDC_THROW(std::runtime_error, "dup failed errno=" << errno);
68  }
69 
70  readFd_.fd = forRead;
71  hmbdc::app::utils::EpollTask::instance().add(
72  hmbdc::app::utils::EpollTask::EPOLLOUT
73  | hmbdc::app::utils::EpollTask::EPOLLET, writeFd_);
74  hmbdc::app::utils::EpollTask::instance().add(
75  hmbdc::app::utils::EpollTask::EPOLLIN
76  | hmbdc::app::utils::EpollTask::EPOLLET, readFd_);
77  }
78 
79  void stop() {
80  if (currTransportHeadFlag_ == hmbdc::app::hasMemoryAttachment::flag) {
81  currTransportHeadFlag_ = 0;
82  }
83  outputBuffer_.putSome(sessionDropped_);
84  }
85 
86  void heartbeat() {
87  if (!initialized_) return;
88  char const* hb = "+\t";
89  if (hmbdc_unlikely(2 != send(writeFd_.fd, hb, 2, MSG_NOSIGNAL))){
90  HMBDC_LOG_C("error when sending heartbeat to ", id(), " errno=", errno);
91  stopped_ = true;
92  }
93  }
94 
95  char const* id() const {
96  return id_.c_str();
97  }
98 
99  bool runOnce() {
100  if (hmbdc_unlikely(stopped_)) return false;
101  if (hmbdc_unlikely(!initialized_ && writeFd_.isFdReady())) {
102  try {
103  initializeConn();
104  } catch (std::exception const& e) {
105  HMBDC_LOG_W(e.what());
106  return false;
107  } catch (...) {
108  HMBDC_LOG_C("unknown exception");
109  return false;
110  }
111  }
112  return doRead();
113  }
114 
115 private:
116  void initializeConn() {
117  int flags = fcntl(writeFd_.fd, F_GETFL, 0);
118  flags &= ~O_NONBLOCK;
119  if (fcntl(writeFd_.fd, F_SETFL, flags) < 0) {
120  HMBDC_THROW(std::runtime_error, "fcntl failed errno=" << errno);
121  }
122 
123  auto sz = config_.getExt<int>("tcpRecvBufferBytes");
124  if (sz) {
125  if (setsockopt(readFd_.fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)) < 0) {
126  HMBDC_LOG_C("failed to set send buffer size=", sz);
127  }
128  }
129 
130  auto addrPort = hmbdc::comm::inet::getPeerIpPort(writeFd_.fd);
131  id_ = addrPort.first + ":" + std::to_string(addrPort.second);
132  strncpy(sessionStarted_.payload.ip, id_.c_str()
133  , sizeof(sessionStarted_.payload.ip));
134  sessionStarted_.payload.ip[sizeof(sessionStarted_.payload.ip) - 1] = 0;
135  strncpy(sessionDropped_.payload.ip, id_.c_str()
136  , sizeof(sessionDropped_.payload.ip));
137  sessionDropped_.payload.ip[sizeof(sessionDropped_.payload.ip) - 1] = 0;
138  outputBuffer_.putSome(sessionStarted_);
139  HMBDC_LOG_N("RecvSession started: ", id());
140  sendSubscriptions();
141  initialized_ = true;
142  }
143 
144  void sendSubscriptions() {
145  std::ostringstream oss;
146  subscriptions_.exportTo([&oss](uint16_t tag, auto&& count) {
147  if (count) {
148  oss << '+' << tag << '\t';
149  }
150  });
151  auto s = oss.str() + "+\t"; //mark all sent with "+\t"
152  auto sz = size_t(send(writeFd_.fd, s.c_str(), s.size(), MSG_NOSIGNAL));
153  if (hmbdc_unlikely(sz != s.size())) {
154  HMBDC_LOG_C("error when sending subscriptions to ", id());
155  stopped_ = true;
156  }
157  }
158 
159  bool doRead() {
160  do {
161  if (hmbdc_unlikely(currTransportHeadFlag_ == hmbdc::app::hasMemoryAttachment::flag
162  && filledLen_)) {
163  auto s = memoryAttachment_.write(bufCur_, filledLen_);
164  if (memoryAttachment_.writeDone()) {
165  memoryAttachment_.close();
166  currTransportHeadFlag_ = 0;
168  hasMemoryAttachmentItemSpace_.get(), outputBuffer_.maxItemSize());
169  }
170  bufCur_ += s;
171  filledLen_ -= s;
172  }
173  while (currTransportHeadFlag_ == 0
174  && filledLen_ >= sizeof(TransportMessageHeader)) {
175  auto h = reinterpret_cast<TransportMessageHeader*>(bufCur_);
176  auto wireSize = h->wireSize();
177  if (hmbdc_likely(filledLen_ >= wireSize)) {
178  if (hmbdc_likely(subscriptions_.check(h->typeTag()))) {
179  if (hmbdc_unlikely(h->flag == hmbdc::app::hasMemoryAttachment::flag)) {
180  currTransportHeadFlag_ = h->flag;
181  auto l = std::min<size_t>(outputBuffer_.maxItemSize(), h->messagePayloadLen);
182  memcpy(hasMemoryAttachmentItemSpace_.get(), h->payload(), l);
183  memoryAttachment_.open(h->typeTag(), hasMemoryAttachment_);
184  // hasMemoryAttachment_->attachment = memoryAttachment_.open(hasMemoryAttachment_->len);
185  // hasMemoryAttachment_->afterConsumedCleanupFunc
186  // = [](hmbdc::app::hasMemoryAttachment* a) {::free(a->attachment);a->attachment = nullptr;};
187  } else {
188  auto l = std::min<size_t>(outputBuffer_.maxItemSize(), h->messagePayloadLen);
189  outputBuffer_.put(h->payload(), l);
190  currTransportHeadFlag_ = 0;
191  }
192  }
193  bufCur_ += wireSize;
194  //memmove(buf_, buf_ + h->wireSize(), filledLen_ - h->wireSize());
195  filledLen_ -= wireSize;
196  } else {
197  break;
198  }
199  }
200  memmove(buf_, bufCur_, filledLen_);
201  bufCur_ = buf_;
202 
203  if (readFd_.isFdReady()) {
204  auto l = recv(readFd_.fd, buf_ + filledLen_, bufSize_ - filledLen_
205  , MSG_NOSIGNAL|MSG_DONTWAIT);
206  if (hmbdc_unlikely(l < 0)) {
207  if (hmbdc_unlikely(!readFd_.checkErr())) {
208  HMBDC_LOG_C("recv failed errno=", errno);
209  return false;
210  }
211  } else if (hmbdc_unlikely(l == 0)) {
212  HMBDC_LOG_W("peer dropped:", id());
213  return false;
214  } else {
215  filledLen_ += l;
216  }
217  } else {
218  return true; //has done what can be done
219  }
220  } while (filledLen_);
221  return true;
222  }
223 
224  hmbdc::app::Config const& config_;
225  TypeTagSet const& subscriptions_;
227  EpollFd writeFd_;
228  OutputBuffer& outputBuffer_;
229  std::string id_;
232  bool initialized_;
233  bool stopped_;
234  size_t const bufSize_;
235  char* buf_;
236  char* bufCur_;
237  size_t filledLen_;
238  uint8_t currTransportHeadFlag_;
239  std::unique_ptr<char[]> hasMemoryAttachmentItemSpace_;
240  hmbdc::app::hasMemoryAttachment* hasMemoryAttachment_;
241 
242  struct DownloadMemory {
244  : addr_(nullptr)
245  , fullLen_(0)
246  , len_(0) {
247  }
248 
249  void* open(uint16_t typeTag, app::hasMemoryAttachment* att) {
250  addr_ = (char*)alloc_(typeTag, att);
251  if (addr_) {
252  fullLen_ = att->len;
253  len_ = 0;
254  }
255  return addr_;
256  }
257 
258  bool writeDone() const {
259  return fullLen_ == len_;
260  }
261 
262  size_t write(void const* mem, size_t l) {
263  auto wl = std::min(l, fullLen_ - len_);
264  if (addr_) {
265  memcpy(addr_ + len_, mem, wl);
266  }
267  len_ += wl;
268  return wl;
269  }
270 
271  void close() {
272  addr_ = nullptr;
273  fullLen_ = 0;
274  len_ = 0;
275  }
276 
277  private:
278  AttachmentAllocator alloc_;
279  char* addr_;
280  size_t fullLen_;
281  size_t len_;
282  };
283  DownloadMemory memoryAttachment_;
284 };
285 } //recvsession_detail
286 template <typename OutputBuffer, typename AttachmentAllocator>
288 }}}
T getExt(const path_type &param, bool throwIfMissing=true) const
get a value from the config
Definition: Config.hpp:238
class to hold an hmbdc configuration
Definition: Config.hpp:44
Definition: EpollTask.hpp:87
Definition: Message.hpp:212
Definition: Transport.hpp:20
Definition: TypeTagSet.hpp:138
if a specific hmbdc network transport (for example tcpcast, rmcast, and rnetmap) supports message wit...
Definition: Message.hpp:125
Definition: Base.hpp:12