hmbdc
simplify-high-performance-messaging-programming
Allocators.hpp
1 #include "hmbdc/Copyright.hpp"
2 #pragma once
3 
4 #include "hmbdc/Compile.hpp"
5 #include "hmbdc/Exception.hpp"
6 
7 #include <boost/align/align.hpp>
8 #include <boost/interprocess/sync/file_lock.hpp>
9 
10 #include <stdexcept>
11 #include <string>
12 #include <iostream>
13 #include <utility>
14 #include <mutex>
15 
16 #include <sys/mman.h>
17 #include <fcntl.h> /* For O_* constants */
18 #include <malloc.h>
19 #include <unistd.h>
20 #include <sys/mman.h>
21 #include <sys/stat.h> /* For mode constants */
22 
23 namespace hmbdc { namespace os {
24 
25 /**
26  * @brief helping allocating object and its aggregated objects in a continouse memory
27  * @details the class working with this cannot have significant dtor (dtor that does more than
28  * just freeing memory)
29  */
31  enum {
32  fromHeap = false,
33  };
35  : cur_(NULL)
36  , size_(0)
37  , runCtor_(false)
38  {}
39 
40  BasePtrAllocator(void* base, size_t size, bool runCtor /* = true */)
41  : cur_((char*)base)
42  , size_(size)
43  , runCtor_(runCtor)
44  {}
45 
46  BasePtrAllocator(BasePtrAllocator const&) = delete;
47  BasePtrAllocator& operator = (BasePtrAllocator const&) = delete;
48  virtual ~BasePtrAllocator(){}
49 
50  template <typename T, typename ...Args>
51  T* allocate(size_t alignment, Args&&... args) {
52  auto cur = boost::alignment::align(alignment, sizeof(T), cur_, size_);
53  cur_ = ((char*)cur) + sizeof(T);
54  T *res;
55  if (runCtor_) {
56  res = ::new (cur)T(std::forward<Args>(args)...);
57  msync(cur, sizeof(T), MS_SYNC);
58  } else {
59  res = static_cast<T*>(cur);
60  }
61  return res;
62  }
63 
64  void * memalign(size_t alignment, size_t size) {
65  auto res = boost::alignment::align(alignment, size, cur_, size_);
66  cur_ = ((char*)cur_) + size_;
67  return res;
68  }
69 
70  template <typename T> void unallocate(T* ptr){
71  if (runCtor_) {
72  ptr->~T();
73  }
74  }
75  void free(void*){}
76 
77 protected:
78  void set(void* base, size_t size, bool runCtor = true) {
79  cur_= base;
80  size_ = size;
81  runCtor_ = runCtor;
82  }
83 
84 private:
85  void* cur_;
86  size_t size_;
87  bool runCtor_;
88 };
89 
90 /**
91  * @brief helping allocating object and its aggregated objects in a continouse shared memory
92  * @details the class working with this cannot have significant dtor (dtor that does more than
93  * just freeing memory)
94  */
97  ShmBasePtrAllocator(char const* name, size_t offset, size_t len
98  , int& ownership);
99 
100  ShmBasePtrAllocator(char const* name, size_t offset, size_t len, int&& ownership)
101  : ShmBasePtrAllocator(name, offset, len, ownership)
102  {}
103 
105 
106  template <typename T, typename ...Args>
107  T* allocate(size_t alignment, Args&&... args) {
108  std::lock_guard<decltype(lock_)> g(lock_);
109  auto res = BasePtrAllocator::allocate<T>(alignment
110  , std::forward<Args>(args)...);
111  return res;
112  }
113 
114  auto& fileLock() {
115  return lock_;
116  }
117 
118 private:
119  std::string name_;
120  size_t len_;
121  void* addr_;
122  bool own_;
123  boost::interprocess::file_lock lock_;
124 };
125 
126 /**
127  * @brief similar to ShmBasePtrAllocator but using dev memory
128  * @details typically PCIe BAR
129  */
132  DevMemBasePtrAllocator(char const* devName, size_t offset, size_t len
133  , int ownership);
135 
136  template <typename T, typename ...Args>
137  T* allocate(size_t alignment, Args&&... args) {
138  std::lock_guard<decltype(lock_)> g(lock_);
139  return BasePtrAllocator::allocate<T>(alignment
140  , std::forward<Args>(args)...);
141  }
142 
143  auto& fileLock() {
144  return lock_;
145  }
146 
147 private:
148  size_t len_;
149  void* addr_;
150  bool own_;
151  boost::interprocess::file_lock lock_;
152 };
153 
154 /**
155  * @brief the default vanilla allocate
156  */
158  enum {
159  fromHeap = true,
160  };
161  template <typename ...NoUses>
162  DefaultAllocator(NoUses&& ...){}
163 
164  template <typename T, typename ...Args>
165  T* allocate(size_t alignment, Args&&... args) {
166  // return new T(std::forward<Args>(args)...);
167  auto space = memalign(alignment, sizeof(T));
168  return new (space) T(std::forward<Args>(args)...);
169  }
170 
171  void* memalign(size_t alignment, size_t size) {
172  auto res = ::memalign(alignment, size);
173  if (!res) {
174  throw std::bad_alloc();
175  }
176  return res;
177  }
178 
179  template <typename T>
180  void
181  unallocate(T* ptr){
182  ptr->~T();
183  this->free(ptr);
184  // delete ptr;
185  }
186  void free(void* ptr){
187  ::free(ptr);
188  }
189  static DefaultAllocator& instance() {
190  static DefaultAllocator ins;
191  return ins;
192  }
193 };
194 
195 inline
196 ShmBasePtrAllocator::
197 ShmBasePtrAllocator(char const* name, size_t offset, size_t len, int& ownership)
199 , len_(len)
200 , addr_(NULL)
201 , own_(false) {
202  if (name[0] != '/') {
203  name_ = std::string("/") + name;
204  } else {
205  name_ = name;
206  }
207 
208  auto devName = std::string("/dev/shm") + name_;
209  size_t retry = 3;
210 
211  while (true) {
212  try {
213  int oflags = O_RDWR;
214  int fd = -1;
215  if (ownership > 0) {
216  shm_unlink(name_.c_str());
217  oflags |= O_CREAT | O_EXCL;
218  fd = shm_open(name, oflags, 0660);
219  if (fd < 0) {
220  HMBDC_THROW(std::runtime_error,
221  "error creating fresh ipc transport errno=" << errno);
222  }
223  } else if (ownership < 0) {
224  fd = shm_open(name, oflags, 0660);
225  if (fd < 0) {
226  HMBDC_THROW(std::runtime_error,
227  "error attaching ipc transport errno=" << errno);
228  }
229  } else {
230  oflags |= O_CREAT | O_EXCL;
231  fd = shm_open(name, oflags, 0660);
232  if (fd < 0 && errno == EEXIST) {
233  oflags = O_RDWR;
234  fd = shm_open(name, oflags, 0660);
235  if (fd < 0) {
236  HMBDC_THROW(std::runtime_error,
237  "error attching ipc transport errno=" << errno);
238  }
239  ownership = -1;
240  } else if (fd >= 0) {
241  ownership = 1;
242  } else {
243  HMBDC_THROW(std::runtime_error,
244  "error creating ipc transport errno=" << errno);
245  }
246  }
247 
248  if (fd >= 0) {
249  if (ownership > 0) {
250  own_ = true;
251  if (ftruncate(fd, len)) {
252  close(fd);
253  HMBDC_THROW(std::runtime_error,
254  "error when sizing ipc transport errno=" << errno);
255 
256  }
257  }
258  struct stat st;
259  stat(devName.c_str(), &st);
260  if (len != (size_t)st.st_size) {
261  close(fd);
262  HMBDC_THROW(std::runtime_error, "file not expected size - unmatch?");
263  }
264  void* base = mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_SHARED
265  , fd, offset);
266  if (base != MAP_FAILED) {
267  set(base, len, own_);
268  addr_ = base;
269  lock_ = boost::interprocess::file_lock(devName.c_str());
270  close(fd);
271  return;
272  }
273  }
274 
275  HMBDC_THROW(std::runtime_error
276  , "error when creating file errno=" << errno);
277  } catch (std::runtime_error const& e) {
278  if (--retry == 0) throw;
279  sleep(1);
280  }
281  }
282 }
283 
284 inline
285 ShmBasePtrAllocator::
286 ~ShmBasePtrAllocator() {
287  munmap(addr_, len_);
288  if (own_) {
289  shm_unlink(name_.c_str());
290  }
291 }
292 
293 inline
294 DevMemBasePtrAllocator::
295 DevMemBasePtrAllocator(char const* devName, size_t offset, size_t len, int ownership)
296 : BasePtrAllocator()
297 , len_(len)
298 , addr_(NULL){
299  if (offset % getpagesize() || len_ % getpagesize()) {
300  HMBDC_THROW(std::out_of_range, "address not at page boundary devName="
301  << devName << "@" << offset << ':' << len_);
302  }
303  if (ownership == 0) {
304  HMBDC_THROW(std::out_of_range, "undefined ownership devName="
305  << devName << "@" << offset << ':' << len_);
306  }
307  int oflags = O_RDWR | O_SYNC;
308  int fd = open(devName, oflags);
309  addr_ = (uint32_t*)mmap(NULL, len_, PROT_READ | PROT_WRITE, MAP_SHARED
310  , fd, offset);
311  if (addr_ == MAP_FAILED) {
312  HMBDC_THROW(std::runtime_error
313  , "cannot map RegRange " << devName << '@' << offset << ':' << len_);
314  }
315  own_ = ownership > 0;
316  set(addr_, len, own_);
317  lock_ = boost::interprocess::file_lock(devName);
318  close(fd);
319 }
320 
321 inline
322 DevMemBasePtrAllocator::
323 ~DevMemBasePtrAllocator() {
324  munmap(addr_, len_);
325 }
326 
327 }}
328 
the default vanilla allocate
Definition: Allocators.hpp:157
similar to ShmBasePtrAllocator but using dev memory
Definition: Allocators.hpp:130
helping allocating object and its aggregated objects in a continouse memory
Definition: Allocators.hpp:30
helping allocating object and its aggregated objects in a continouse shared memory ...
Definition: Allocators.hpp:95
Definition: Base.hpp:12