Realm
A distributed, event-based tasking library
Loading...
Searching...
No Matches
activemsg.h
Go to the documentation of this file.
1/*
2 * Copyright 2025 Stanford University, NVIDIA Corporation
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18// hopefully a more user-friendly C++ template wrapper for GASNet active
19// messages...
20
21#ifndef ACTIVEMSG_H
22#define ACTIVEMSG_H
23
24#include "realm/realm_config.h"
26#include "realm/mutex.h"
27#include "realm/serialize.h"
28#include "realm/nodeset.h"
29#include "realm/network.h"
30#include "realm/atomics.h"
31#include "realm/threads.h"
32#include "realm/bgwork.h"
33#include <atomic>
34#include <cstddef>
35#include <limits>
36#include <type_traits>
37#include <mutex>
38#include <vector>
39
40#include <optional>
41
42namespace Realm {
43
44 namespace Config {
45 // if true, the number and min/max/avg/stddev duration of handler per
46 // message type is recorded and printed
48
49 // the maximum time we're willing to spend on inline message
50 // handlers
51 extern long long max_inline_message_time;
52 }; // namespace Config
53
54 enum
55 {
56 PAYLOAD_NONE, // no payload in packet
57 PAYLOAD_KEEP, // use payload pointer, guaranteed to be stable
58 PAYLOAD_FREE, // take ownership of payload, free when done
59 PAYLOAD_COPY, // make a copy of the payload
60 PAYLOAD_SRCPTR, // payload has been copied to the src data pool
61 PAYLOAD_PENDING, // payload needs to be copied, but hasn't yet
62 PAYLOAD_KEEPREG, // use payload pointer, AND it's registered!
63 PAYLOAD_EMPTY, // message can have payload, but this one is 0 bytes
64 };
65
66 class ActiveMessageImpl;
67
68 template <typename T, size_t INLINE_STORAGE = 256>
70 public:
71 // constructs an INACTIVE message object - call init(...) as needed
73
74 // construct a new active message for either a single recipient or a mask
75 // of recipients
76 // in addition to the header struct (T), a message can include a variable
77 // payload which can be delivered to a particular destination address
78 ActiveMessage(NodeID _target, size_t _max_payload_size = 0);
79 ActiveMessage(NodeID _target, size_t _max_payload_size,
80 const RemoteAddress &_dest_payload_addr);
81 ActiveMessage(const Realm::NodeSet &_targets, size_t _max_payload_size = 0);
82
83 // providing the payload (as a 1D reference, which must be PAYLOAD_KEEP)
84 // up front can avoid a copy if the source location is directly accessible
85 // by the networking hardware
86 // Per the semantics of PAYLOAD_KEEP, you must keep the payload buffer alive
87 // and unmodified until LOCAL COMPLETION - NOT merely until commit() returns.
88 // A backend may hand the pointer straight to the network and return before it
89 // has been read. Use add_local_completion() to learn when the buffer is free,
90 // or use the (target, max_payload_size) constructor with add_payload(), which
91 // copies during commit().
92 ActiveMessage(NodeID _target, const void *_data, size_t _datalen);
93 ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen,
94 const RemoteAddress &_dest_payload_addr);
95 ActiveMessage(const Realm::NodeSet &_targets, const void *_data, size_t _datalen);
96 ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr,
97 size_t _bytes_per_line, size_t _lines, size_t _line_stride,
98 const RemoteAddress &_dest_payload_addr);
99
101
102 // a version of `init` for each constructor above
103 void init(NodeID _target, size_t _max_payload_size = 0);
104 void init(NodeID _target, size_t _max_payload_size,
105 const RemoteAddress &_dest_payload_addr);
106 void init(const Realm::NodeSet &_targets, size_t _max_payload_size = 0);
107 void init(NodeID _target, const void *_data, size_t _datalen);
108 void init(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen,
109 const RemoteAddress &_dest_payload_addr);
110 void init(const Realm::NodeSet &_targets, const void *_data, size_t _datalen);
111 void init(NodeID _target, const LocalAddress &_src_payload_addr,
112 size_t _bytes_per_line, size_t _lines, size_t _line_stride,
113 const RemoteAddress &_dest_payload_addr);
114
115 // large messages may need to be fragmented, so use cases that can
116 // handle the fragmentation at a higher level may want to know the
117 // largest size that is fragmentation-free - the answer can depend
118 // on whether the data is to be delivered to a known RemoteAddress
119 // and/or whether the source data location is known
120 // a call that sets `with_congestion` may get a smaller value (maybe
121 // even 0) if the path to the named target(s) is getting full
122 static size_t recommended_max_payload(NodeID target, bool with_congestion);
123 static size_t recommended_max_payload(const NodeSet &targets, bool with_congestion);
124 static size_t recommended_max_payload(NodeID target,
125 const RemoteAddress &dest_payload_addr,
126 bool with_congestion);
127 static size_t recommended_max_payload(NodeID target, const void *data,
128 size_t bytes_per_line, size_t lines,
129 size_t line_stride, bool with_congestion);
130 static size_t recommended_max_payload(const NodeSet &targets, const void *data,
131 size_t bytes_per_line, size_t lines,
132 size_t line_stride, bool with_congestion);
133 static size_t
134 recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr,
135 size_t bytes_per_line, size_t lines, size_t line_stride,
136 const RemoteAddress &dest_payload_addr, bool with_congestion);
137
138 // operator-> gives access to the header structure
139 T *operator->(void);
140 T &operator*(void);
141
142 // variable payload can be written to in three ways:
143 // (a) Realm-style serialization (currently eager)
144 template <typename T2>
145 bool operator<<(const T2 &to_append);
146 // (b) old memcpy-like behavior (using the various payload modes)
147 void add_payload(const void *data, size_t datalen, int payload_mode = PAYLOAD_COPY);
148 void add_payload(const void *data, size_t bytes_per_line, size_t lines,
149 size_t line_stride, int payload_mode = PAYLOAD_COPY);
150 // (c) request for a pointer to write into (writes must be completed before
151 // call to commit or cancel)
152 void *payload_ptr(size_t datalen);
153
154 // register callbacks to be called upon:
155 // a) local completion - i.e. all source data has been read and can now
156 // be safely overwritten
157 // b) remote completion - message has been received AND HANDLED by target
158 //
159 // callbacks need to be "lightweight" - for heavier work, the message
160 // handler on the target can send an explicit response message
161 template <typename CALLABLE>
162 void add_local_completion(const CALLABLE &callable);
163 template <typename CALLABLE>
164 void add_remote_completion(const CALLABLE &callable);
165
166 // every active message must eventually be commit()'ed or cancel()'ed
167 void commit(void);
168 void cancel(void);
169
170 protected:
174 alignas(alignof(T) > alignof(uint64_t) ? alignof(T) : alignof(uint64_t)) uint64_t
175 inline_capacity[INLINE_STORAGE / sizeof(uint64_t)];
176
177 private:
178 // chunked-mode state: only used when the requested payload exceeds the
179 // network's hard limit for a single message
180 size_t network_max_payload_{0}; // 0 = normal (non-chunked) mode
181 NodeSet chunk_targets_;
182 const void *chunk_src_data_{nullptr};
183 size_t chunk_src_datalen_{0};
184 std::vector<std::byte>
185 chunk_alloc_; // owned buffer for network-allocated chunked mode
186
187 // Completion callbacks registered before commit(), held here because chunked mode
188 // has no ActiveMessageImpl to hand them to until commit() builds the chunks.
189 std::vector<char> chunk_local_comp_;
190 std::vector<char> chunk_remote_comp_;
191
192 void init_chunked(NodeID _target, size_t _max_payload_size);
193 void init_chunked(const NodeSet &_targets, size_t _max_payload_size);
194 void init_chunked_data(NodeID _target, const void *_data, size_t _datalen);
195 void init_chunked_data(const NodeSet &_targets, const void *_data, size_t _datalen);
196 void commit_chunked(void);
197 // makes room for one more completion callback, growing by cloning rather than
198 // letting the vector memcpy polymorphic objects
199 static void *reserve_chunked_completion(std::vector<char> &buf, size_t bytes);
200 static void discard_chunked_completions(std::vector<char> &buf);
201
202 // so that commit_chunked() can hand a completion to the per-chunk message, which
203 // is a different instantiation of this same template
204 template <typename, size_t>
205 friend class ActiveMessage;
206 };
207
208 // type-erased wrappers for completion callbacks
210 public:
212 virtual void invoke() = 0;
213 virtual size_t size() const = 0;
214 virtual CompletionCallbackBase *clone_at(void *p) const = 0;
215
216 static const size_t ALIGNMENT = 8;
217
218 // helper functions for invoking/cloning/destroying a collection of callbacks
219 static void invoke_all(void *start, size_t bytes);
220 static void clone_all(void *dst, const void *src, size_t bytes);
221 static void destroy_all(void *start, size_t bytes);
222 };
223
224 template <typename CALLABLE>
226 public:
227 CompletionCallback(const CALLABLE &_callable);
228 virtual void invoke();
229 virtual size_t size() const;
230 virtual CompletionCallbackBase *clone_at(void *p) const;
231
232 protected:
233 CALLABLE callable;
234 };
235
236 // per-network active message implementations are mostly opaque, but a few
237 // fields are exposed to avoid virtual function calls
239 public:
241
242 // reserves space for a local/remote completion - caller will
243 // placement-new the completion at the provided address
244 virtual void *add_local_completion(size_t size) = 0;
245 virtual void *add_remote_completion(size_t size) = 0;
246
247 virtual void commit(size_t act_payload_size) = 0;
248 virtual void cancel() = 0;
249
253 };
254
256
259
261 void record(long long t_start, long long t_end);
262 };
263
265 uint32_t chunk_id{0};
266 uint32_t total_chunks{0};
267 uint64_t msg_id{0};
268 };
269
270 // Allocates the next chunked-message id for this process. Deliberately a free
271 // function rather than a member of ActiveMessage<T>: a function-local static inside
272 // a member of a class template gets one instance per instantiation, so each message
273 // type would restart the sequence at zero and hand out ids that collide with every
274 // other type's.
276
277 // Called when an ActiveMessage's payload is too large to send as one message and
278 // the network cannot carry it as fragments either - Network::max_payload_size()
279 // returned 0 for the fragment header (sizeof(WrappedWithFragInfo<T>)), which a
280 // backend does when that header exceeds what it can put on the wire. Reports the
281 // message type and sizes and aborts: failing at the first such send, on every
282 // network, beats a backend overrunning its header buffer later.
284 report_unfragmentable_message(unsigned short msgid, size_t header_size,
285 size_t wrapped_header_size, size_t payload_size);
286
287 // singleton class that can convert message type->ID and ID->handler
289 public:
292
293 typedef unsigned short MessageID;
294 typedef void (*MessageHandler)(NodeID sender, const void *header, const void *payload,
295 size_t payload_size, TimeLimit work_until);
296 typedef void (*MessageHandlerNoTimeout)(NodeID sender, const void *header,
297 const void *payload, size_t payload_size);
298 typedef bool (*MessageHandlerInline)(NodeID sender, const void *header,
299 const void *payload, size_t payload_size,
300 TimeLimit work_until);
301
302 template <typename T>
304
306
307 // number of registered handlers (and so one more than the largest MessageID) -
308 // networks with a narrower message ID field than MessageID need to check this
309 size_t num_handlers(void) const { return handlers.size(); }
310 void record_message_handler_call(MessageID id, long long t_start, long long t_end);
312
314
316
317 typedef unsigned TypeHash;
318
329
331
332 protected:
334
335 std::vector<HandlerEntry> handlers;
336 };
337
339
355
356 template <typename T, typename T2 = T>
358 public:
361
362 // when registering an active message handler, the following three methods
363 // are looked for in class T2
364 // (a) void handle_message(NodeID, const T&, const void *, size_t, TimeLimit)
365 // (b) void handle_message(NodeID, const T&, const void *, size_t)
366 // (c) bool handle_inline(NodeID, const T&, const void *, size_t, TimeLimit)
367 //
368 // at least one of (a) or (b) must be present, with (a) being preferred
369 //
370 // if (c) is present, it will be used to attempt inline handling of
371 // active messages as they arrive, with the following constraints:
372 // (i) the handler must not block on any mutexes (trylocks are ok)
373 // (ii) the handler must not perform dynamic memory allocation/frees
374 // (iii) the handler must try very hard to stay within the specified
375 // time limit
376 // if the inline handler is unable to satisfy these requirements, it should
377 // not attempt to handle the message, returning 'false' and letting it be
378 // queued as normal
379
380 // returns either the requested kind of handler or a null pointer if
381 // it doesn't exist
387
388 // this method does nothing, but can be called to force the instantiation
389 // of a handler registration object (needed when things are inside templates)
391 };
392
393 namespace ThreadLocal {
394 // this flag will be true when we are running a message handler
395 extern thread_local bool in_message_handler;
396 }; // namespace ThreadLocal
397
399 : public BackgroundWorkItem {
400 public:
401 IncomingMessageManager(int _nodes, int _dedicated_threads,
404
405 typedef uintptr_t CallbackData;
407
408 // adds an incoming message to the queue
409 // returns true if the call was handled immediately (in which case the
410 // callback, if present, will NOT be called), or false if the message
411 // will be processed later
412 // a callback is only ever invoked where queued messages are handled (a handler
413 // thread or do_work), never from within this call, so a backend may call it
414 // from a receive path where its own callback would not be legal
416 const void *hdr, size_t hdr_size, int hdr_mode,
417 const void *payload, size_t payload_size, int payload_mode,
418 CallbackFnptr callback_fnptr, CallbackData callback_data1,
419 CallbackData callback_data2, TimeLimit work_until);
420
421 void start_handler_threads(size_t stack_size);
422
423 // stalls caller until all incoming messages have been handled (and at
424 // least 'min_messages_handled' in total)
425 void drain_incoming_messages(size_t min_messages_handled);
426
427 void shutdown(void);
428
429 virtual bool do_work(TimeLimit work_until);
430
432
433 protected:
434 struct MessageBlock;
435
436 // A remote completion rides on one fragment (see ActiveMessage<T>'s chunked mode),
437 // but "the target has handled it" is only true once the whole message has been
438 // reassembled and run. A callback arriving on a fragment is therefore held here
439 // until then.
445
446 struct Message {
451 void *hdr;
452 size_t hdr_size;
454 void *payload;
459 // callbacks that arrived on earlier fragments of this message and could not be
460 // fired then - null for every message that was not reassembled
461 std::vector<DeferredCallback> *deferred_callbacks;
462 };
463
465 static MessageBlock *new_block(size_t _total_size);
466 static void free_block(MessageBlock *block);
467
468 void reset();
469
470 // called with message manager lock held
471 Message *append_message(size_t hdr_bytes_needed, size_t payload_bytes_needed);
472
473 // total bytes a message with these requirements would occupy in a freshly
474 // reset block - a request larger than the configured block size can never
475 // be satisfied by any block, no matter how many are tried
476 static size_t bytes_needed(size_t hdr_bytes_needed, size_t payload_bytes_needed);
477
478 // called _without_ message manager lock held
480
484 };
485
486 // fires and releases any callbacks held from this message's fragments
488
489 int get_messages(Message *&head, Message **&tail, bool wait);
490 bool return_messages(int sender, size_t num_handled, Message *head, Message **tail);
491
498 int *todo_list; // list of nodes with non-empty message lists
507 std::vector<Thread *> handler_threads;
512
513 // An in-flight reassembly is identified by (sender, message type, sender-chosen
514 // id). The message type is part of the key as defence in depth: ids are unique
515 // per sender process, but including the type means any future regression in id
516 // allocation fails loudly instead of silently interleaving two messages' bytes.
517 struct FragmentKey {
520 uint64_t msg_id;
521
522 bool operator==(const FragmentKey &rhs) const
523 {
524 return ((sender == rhs.sender) && (msgid == rhs.msgid) && (msg_id == rhs.msg_id));
525 }
526 };
527
529 std::size_t operator()(const FragmentKey &k) const
530 {
531 return (std::hash<NodeID>()(k.sender) ^ (std::hash<unsigned>()(k.msgid) << 1) ^
532 (std::hash<uint64_t>()(k.msg_id) << 2));
533 }
534 };
535
537 std::unique_ptr<FragmentedMessage> message;
538 std::vector<DeferredCallback> deferred;
539 };
540
541 std::unordered_map<FragmentKey, FragmentReassembly, FragmentKeyHash> frag_message;
542 };
543
544 template <typename UserHdr>
547 UserHdr user;
548
549 UserHdr *operator->() { return &user; }
550 const UserHdr *operator->() const { return &user; }
551 UserHdr &operator*() { return user; }
552 const UserHdr &operator*() const { return user; }
553 };
554
555 // trait to detect WrappedWithFragInfo types
556 template <typename T>
557 struct is_wrapped_with_frag_info : std::false_type {};
558 template <typename U>
559 struct is_wrapped_with_frag_info<WrappedWithFragInfo<U>> : std::true_type {};
560
561 // The largest fragment header, sizeof(WrappedWithFragInfo<T>), that every network
562 // must be able to send; a network with a tight header limit checks this against
563 // it where that limit is defined. ActiveMessageHandlerReg checks every type
564 // against it, so a header that grows too large fails the build, rather than
565 // aborting in report_unfragmentable_message the first time the type carries a
566 // large payload.
567 static constexpr size_t MAX_FRAGMENT_HEADER_BYTES = 56;
568
569 // Specialize to std::true_type to register a type whose fragment header exceeds
570 // MAX_FRAGMENT_HEADER_BYTES anyway. It can still be sent, but where the network
571 // cannot fit its fragment header a payload too large for one message aborts in
572 // report_unfragmentable_message. Only test types, which never travel over a real
573 // network, should need this.
574 template <typename T>
575 struct allow_unfragmentable_header : std::false_type {};
576
577} // namespace Realm
578
579#include "realm/activemsg.inl"
580
581#endif
Definition activemsg.h:340
const char * name
Definition activemsg.h:350
virtual ActiveMessageHandlerTable::MessageHandler get_handler(void) const =0
virtual ~ActiveMessageHandlerRegBase(void)
Definition activemsg.h:342
ActiveMessageHandlerRegBase * next_handler
Definition activemsg.h:352
virtual ActiveMessageHandlerTable::MessageHandlerNoTimeout get_handler_notimeout(void) const =0
std::optional< const FragmentInfo &(*)(const void *)> extract_frag_info
Definition activemsg.h:353
bool must_free
Definition activemsg.h:351
ActiveMessageHandlerTable::TypeHash hash
Definition activemsg.h:349
virtual ActiveMessageHandlerTable::MessageHandlerInline get_handler_inline(void) const =0
void force_instantiation(void)
Definition activemsg.h:390
virtual ActiveMessageHandlerTable::MessageHandlerInline get_handler_inline(void) const
virtual ActiveMessageHandlerTable::MessageHandler get_handler(void) const
virtual ActiveMessageHandlerTable::MessageHandlerNoTimeout get_handler_notimeout(void) const
Definition activemsg.h:288
const char * lookup_message_name(MessageID id)
bool(* MessageHandlerInline)(NodeID sender, const void *header, const void *payload, size_t payload_size, TimeLimit work_until)
Definition activemsg.h:298
unsigned short MessageID
Definition activemsg.h:293
MessageID lookup_message_id(void) const
void(* MessageHandlerNoTimeout)(NodeID sender, const void *header, const void *payload, size_t payload_size)
Definition activemsg.h:296
size_t num_handlers(void) const
Definition activemsg.h:309
void record_message_handler_call(MessageID id, long long t_start, long long t_end)
unsigned TypeHash
Definition activemsg.h:317
void(* MessageHandler)(NodeID sender, const void *header, const void *payload, size_t payload_size, TimeLimit work_until)
Definition activemsg.h:294
static void append_handler_reg(ActiveMessageHandlerRegBase *new_reg)
HandlerEntry * lookup_message_handler(MessageID id)
static ActiveMessageHandlerRegBase * pending_handlers
Definition activemsg.h:333
std::vector< HandlerEntry > handlers
Definition activemsg.h:335
Definition activemsg.h:238
virtual void commit(size_t act_payload_size)=0
virtual void * add_local_completion(size_t size)=0
void * header_base
Definition activemsg.h:250
virtual void * add_remote_completion(size_t size)=0
size_t payload_size
Definition activemsg.h:252
void * payload_base
Definition activemsg.h:251
virtual void cancel()=0
virtual ~ActiveMessageImpl()
Definition activemsg.h:240
Realm::Serialization::FixedBufferSerializer fbs
Definition activemsg.h:173
void init(NodeID _target, const LocalAddress &_src_payload_addr, size_t _bytes_per_line, size_t _lines, size_t _line_stride, const RemoteAddress &_dest_payload_addr)
static size_t recommended_max_payload(NodeID target, const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion)
ActiveMessage(const Realm::NodeSet &_targets, const void *_data, size_t _datalen)
ActiveMessageImpl * impl
Definition activemsg.h:171
ActiveMessage(NodeID _target, size_t _max_payload_size=0)
void init(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen, const RemoteAddress &_dest_payload_addr)
uint64_t inline_capacity[INLINE_STORAGE/sizeof(uint64_t)]
Definition activemsg.h:175
void * payload_ptr(size_t datalen)
void init(NodeID _target, const void *_data, size_t _datalen)
void add_payload(const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, int payload_mode=PAYLOAD_COPY)
void init(NodeID _target, size_t _max_payload_size=0)
ActiveMessage(const Realm::NodeSet &_targets, size_t _max_payload_size=0)
T * header
Definition activemsg.h:172
ActiveMessage(NodeID _target, const void *_data, size_t _datalen)
void init(const Realm::NodeSet &_targets, const void *_data, size_t _datalen)
bool operator<<(const T2 &to_append)
void add_payload(const void *data, size_t datalen, int payload_mode=PAYLOAD_COPY)
static size_t recommended_max_payload(NodeID target, bool with_congestion)
static size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion)
friend class ActiveMessage
Definition activemsg.h:205
static size_t recommended_max_payload(const NodeSet &targets, const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion)
ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, size_t _bytes_per_line, size_t _lines, size_t _line_stride, const RemoteAddress &_dest_payload_addr)
static size_t recommended_max_payload(const NodeSet &targets, bool with_congestion)
void add_local_completion(const CALLABLE &callable)
void add_remote_completion(const CALLABLE &callable)
static size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, size_t line_stride, const RemoteAddress &dest_payload_addr, bool with_congestion)
void init(const Realm::NodeSet &_targets, size_t _max_payload_size=0)
void init(NodeID _target, size_t _max_payload_size, const RemoteAddress &_dest_payload_addr)
ActiveMessage(NodeID _target, size_t _max_payload_size, const RemoteAddress &_dest_payload_addr)
ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen, const RemoteAddress &_dest_payload_addr)
BackgroundWorkManager * manager
Definition bgwork.h:150
BackgroundWorkItem(const std::string &_name)
Definition activemsg.h:209
static void destroy_all(void *start, size_t bytes)
static const size_t ALIGNMENT
Definition activemsg.h:216
virtual size_t size() const =0
static void clone_all(void *dst, const void *src, size_t bytes)
static void invoke_all(void *start, size_t bytes)
virtual CompletionCallbackBase * clone_at(void *p) const =0
virtual CompletionCallbackBase * clone_at(void *p) const
CALLABLE callable
Definition activemsg.h:233
CompletionCallback(const CALLABLE &_callable)
virtual size_t size() const
Definition threads.h:382
Definition threads.h:342
size_t total_messages_handled
Definition activemsg.h:503
int todo_oldest
Definition activemsg.h:499
void drain_incoming_messages(size_t min_messages_handled)
int todo_newest
Definition activemsg.h:499
size_t drain_min_count
Definition activemsg.h:502
Message ** heads
Definition activemsg.h:495
bool return_messages(int sender, size_t num_handled, Message *head, Message **tail)
MessageBlock * current_block
Definition activemsg.h:508
int handlers_active
Definition activemsg.h:500
uintptr_t CallbackData
Definition activemsg.h:405
atomic< bool > bgwork_requested
Definition activemsg.h:493
std::vector< Thread * > handler_threads
Definition activemsg.h:507
Mutex::CondVar drain_condvar
Definition activemsg.h:505
int dedicated_threads
Definition activemsg.h:492
Mutex::CondVar condvar
Definition activemsg.h:505
Message *** tails
Definition activemsg.h:496
CoreReservation * core_rsrv
Definition activemsg.h:506
int get_messages(Message *&head, Message **&tail, bool wait)
bool add_incoming_message(NodeID sender, ActiveMessageHandlerTable::MessageID msgid, const void *hdr, size_t hdr_size, int hdr_mode, const void *payload, size_t payload_size, int payload_mode, CallbackFnptr callback_fnptr, CallbackData callback_data1, CallbackData callback_data2, TimeLimit work_until)
void start_handler_threads(size_t stack_size)
int sleeper_count
Definition activemsg.h:492
bool * in_handler
Definition activemsg.h:497
static void invoke_deferred_callbacks(Message *msg)
void(* CallbackFnptr)(NodeID, CallbackData, CallbackData)
Definition activemsg.h:406
IncomingMessageManager(int _nodes, int _dedicated_threads, Realm::CoreReservationSet &crs)
size_t num_available_blocks
Definition activemsg.h:510
Mutex mutex
Definition activemsg.h:504
int nodes
Definition activemsg.h:492
int shutdown_flag
Definition activemsg.h:494
std::unordered_map< FragmentKey, FragmentReassembly, FragmentKeyHash > frag_message
Definition activemsg.h:541
size_t cfg_message_block_size
Definition activemsg.h:511
int * todo_list
Definition activemsg.h:498
MessageBlock * available_blocks
Definition activemsg.h:509
bool drain_pending
Definition activemsg.h:501
virtual bool do_work(TimeLimit work_until)
size_t cfg_max_available_blocks
Definition activemsg.h:511
Definition nodeset.h:117
Definition timers.h:129
UnfairCondVar CondVar
Definition mutex.h:227
Definition atomics.h:31
#define REALM_INTERNAL_API_EXTERNAL_LINKAGE
Definition compiler_support.h:226
Helper utility for reconstructing large ActiveMessages that were split into multiple network packets.
Definition activemsg.h:44
bool profile_activemsg_handlers
long long max_inline_message_time
Definition activemsg.h:393
Definition activemsg.h:42
ActiveMessageHandlerTable activemsg_handler_table
UnfairMutex Mutex
Definition mutex.h:68
int NodeID
Definition nodeset.h:40
@ PAYLOAD_EMPTY
Definition activemsg.h:63
@ PAYLOAD_KEEPREG
Definition activemsg.h:62
@ PAYLOAD_PENDING
Definition activemsg.h:61
@ PAYLOAD_SRCPTR
Definition activemsg.h:60
@ PAYLOAD_NONE
Definition activemsg.h:56
@ PAYLOAD_COPY
Definition activemsg.h:59
@ PAYLOAD_FREE
Definition activemsg.h:58
@ PAYLOAD_KEEP
Definition activemsg.h:57
REALM_INTERNAL_API_EXTERNAL_LINKAGE uint64_t next_chunk_message_id(NodeID node_id)
REALM_INTERNAL_API_EXTERNAL_LINKAGE void report_unfragmentable_message(unsigned short msgid, size_t header_size, size_t wrapped_header_size, size_t payload_size)
Definition activemsg.h:257
void record(long long t_start, long long t_end)
atomic< size_t > count
Definition activemsg.h:258
atomic< size_t > minval
Definition activemsg.h:258
atomic< size_t > maxval
Definition activemsg.h:258
atomic< size_t > sum
Definition activemsg.h:258
atomic< size_t > sum2
Definition activemsg.h:258
MessageHandlerNoTimeout handler_notimeout
Definition activemsg.h:323
std::optional< const FragmentInfo &(*)(const void *)> extract_frag_info
Definition activemsg.h:327
const char * name
Definition activemsg.h:321
ActiveMessageHandlerStats stats
Definition activemsg.h:325
TypeHash hash
Definition activemsg.h:320
MessageHandlerInline handler_inline
Definition activemsg.h:324
MessageHandler handler
Definition activemsg.h:322
Definition activemsg.h:264
uint32_t total_chunks
Definition activemsg.h:266
uint64_t msg_id
Definition activemsg.h:267
uint32_t chunk_id
Definition activemsg.h:265
CallbackData data1
Definition activemsg.h:442
CallbackFnptr fnptr
Definition activemsg.h:441
CallbackData data2
Definition activemsg.h:443
std::size_t operator()(const FragmentKey &k) const
Definition activemsg.h:529
bool operator==(const FragmentKey &rhs) const
Definition activemsg.h:522
ActiveMessageHandlerTable::MessageID msgid
Definition activemsg.h:519
NodeID sender
Definition activemsg.h:518
uint64_t msg_id
Definition activemsg.h:520
std::unique_ptr< FragmentedMessage > message
Definition activemsg.h:537
std::vector< DeferredCallback > deferred
Definition activemsg.h:538
size_t total_size
Definition activemsg.h:481
static MessageBlock * new_block(size_t _total_size)
static void free_block(MessageBlock *block)
void recycle_message(Message *msg, IncomingMessageManager *manager)
Message * append_message(size_t hdr_bytes_needed, size_t payload_bytes_needed)
atomic< unsigned > use_count
Definition activemsg.h:482
size_t size_used
Definition activemsg.h:481
static size_t bytes_needed(size_t hdr_bytes_needed, size_t payload_bytes_needed)
MessageBlock * next_free
Definition activemsg.h:483
Definition activemsg.h:446
CallbackFnptr callback_fnptr
Definition activemsg.h:457
CallbackData callback_data2
Definition activemsg.h:458
CallbackData callback_data1
Definition activemsg.h:458
void * payload
Definition activemsg.h:454
bool hdr_needs_free
Definition activemsg.h:453
void * hdr
Definition activemsg.h:451
MessageBlock * block
Definition activemsg.h:447
size_t payload_size
Definition activemsg.h:455
Message * next_msg
Definition activemsg.h:448
ActiveMessageHandlerTable::HandlerEntry * handler
Definition activemsg.h:450
size_t hdr_size
Definition activemsg.h:452
std::vector< DeferredCallback > * deferred_callbacks
Definition activemsg.h:461
NodeID sender
Definition activemsg.h:449
bool payload_needs_free
Definition activemsg.h:456
Definition network.h:58
Definition network.h:46
Definition activemsg.h:545
UserHdr * operator->()
Definition activemsg.h:549
UserHdr & operator*()
Definition activemsg.h:551
const UserHdr * operator->() const
Definition activemsg.h:550
FragmentInfo frag_info
Definition activemsg.h:546
UserHdr user
Definition activemsg.h:547
const UserHdr & operator*() const
Definition activemsg.h:552
Definition activemsg.h:575
Definition activemsg.h:557
NodeID src
Definition ucp_internal.h:1
unsigned short msgid
Definition ucp_internal.h:2