66 class ActiveMessageImpl;
68 template <
typename T,
size_t INLINE_STORAGE = 256>
97 size_t _bytes_per_line,
size_t _lines,
size_t _line_stride,
107 void init(
NodeID _target,
const void *_data,
size_t _datalen);
112 size_t _bytes_per_line,
size_t _lines,
size_t _line_stride,
126 bool with_congestion);
128 size_t bytes_per_line,
size_t lines,
129 size_t line_stride,
bool with_congestion);
131 size_t bytes_per_line,
size_t lines,
132 size_t line_stride,
bool with_congestion);
135 size_t bytes_per_line,
size_t lines,
size_t line_stride,
136 const RemoteAddress &dest_payload_addr,
bool with_congestion);
144 template <
typename T2>
148 void add_payload(
const void *data,
size_t bytes_per_line,
size_t lines,
161 template <
typename CALLABLE>
163 template <
typename CALLABLE>
174 alignas(
alignof(T) >
alignof(uint64_t) ?
alignof(T) :
alignof(uint64_t)) uint64_t
180 size_t network_max_payload_{0};
182 const void *chunk_src_data_{
nullptr};
183 size_t chunk_src_datalen_{0};
184 std::vector<std::byte>
189 std::vector<char> chunk_local_comp_;
190 std::vector<char> chunk_remote_comp_;
192 void init_chunked(
NodeID _target,
size_t _max_payload_size);
193 void init_chunked(
const NodeSet &_targets,
size_t _max_payload_size);
194 void init_chunked_data(
NodeID _target,
const void *_data,
size_t _datalen);
195 void init_chunked_data(
const NodeSet &_targets,
const void *_data,
size_t _datalen);
196 void commit_chunked(
void);
199 static void *reserve_chunked_completion(std::vector<char> &buf,
size_t bytes);
200 static void discard_chunked_completions(std::vector<char> &buf);
204 template <
typename,
size_t>
213 virtual size_t size()
const = 0;
224 template <
typename CALLABLE>
247 virtual void commit(
size_t act_payload_size) = 0;
261 void record(
long long t_start,
long long t_end);
285 size_t wrapped_header_size,
size_t payload_size);
295 size_t payload_size,
TimeLimit work_until);
297 const void *payload,
size_t payload_size);
299 const void *payload,
size_t payload_size,
302 template <
typename T>
356 template <
typename T,
typename T2 = T>
416 const void *hdr,
size_t hdr_size,
int hdr_mode,
417 const void *payload,
size_t payload_size,
int payload_mode,
476 static size_t bytes_needed(
size_t hdr_bytes_needed,
size_t payload_bytes_needed);
531 return (std::hash<NodeID>()(k.
sender) ^ (std::hash<unsigned>()(k.
msgid) << 1) ^
532 (std::hash<uint64_t>()(k.
msg_id) << 2));
541 std::unordered_map<FragmentKey, FragmentReassembly, FragmentKeyHash>
frag_message;
544 template <
typename UserHdr>
556 template <
typename T>
558 template <
typename U>
567 static constexpr size_t MAX_FRAGMENT_HEADER_BYTES = 56;
574 template <
typename T>
579#include "realm/activemsg.inl"
Definition activemsg.h:340
const char * name
Definition activemsg.h:350
virtual ActiveMessageHandlerTable::MessageHandler get_handler(void) const =0
virtual ~ActiveMessageHandlerRegBase(void)
Definition activemsg.h:342
ActiveMessageHandlerRegBase * next_handler
Definition activemsg.h:352
virtual ActiveMessageHandlerTable::MessageHandlerNoTimeout get_handler_notimeout(void) const =0
std::optional< const FragmentInfo &(*)(const void *)> extract_frag_info
Definition activemsg.h:353
bool must_free
Definition activemsg.h:351
ActiveMessageHandlerTable::TypeHash hash
Definition activemsg.h:349
virtual ActiveMessageHandlerTable::MessageHandlerInline get_handler_inline(void) const =0
void force_instantiation(void)
Definition activemsg.h:390
~ActiveMessageHandlerReg(void)
virtual ActiveMessageHandlerTable::MessageHandlerInline get_handler_inline(void) const
virtual ActiveMessageHandlerTable::MessageHandler get_handler(void) const
virtual ActiveMessageHandlerTable::MessageHandlerNoTimeout get_handler_notimeout(void) const
ActiveMessageHandlerReg(void)
Definition activemsg.h:288
const char * lookup_message_name(MessageID id)
bool(* MessageHandlerInline)(NodeID sender, const void *header, const void *payload, size_t payload_size, TimeLimit work_until)
Definition activemsg.h:298
unsigned short MessageID
Definition activemsg.h:293
MessageID lookup_message_id(void) const
void report_message_handler_stats()
void(* MessageHandlerNoTimeout)(NodeID sender, const void *header, const void *payload, size_t payload_size)
Definition activemsg.h:296
ActiveMessageHandlerTable(void)
~ActiveMessageHandlerTable(void)
void construct_handler_table(void)
size_t num_handlers(void) const
Definition activemsg.h:309
void record_message_handler_call(MessageID id, long long t_start, long long t_end)
unsigned TypeHash
Definition activemsg.h:317
void(* MessageHandler)(NodeID sender, const void *header, const void *payload, size_t payload_size, TimeLimit work_until)
Definition activemsg.h:294
static void append_handler_reg(ActiveMessageHandlerRegBase *new_reg)
HandlerEntry * lookup_message_handler(MessageID id)
static ActiveMessageHandlerRegBase * pending_handlers
Definition activemsg.h:333
std::vector< HandlerEntry > handlers
Definition activemsg.h:335
Definition activemsg.h:238
virtual void commit(size_t act_payload_size)=0
virtual void * add_local_completion(size_t size)=0
void * header_base
Definition activemsg.h:250
virtual void * add_remote_completion(size_t size)=0
size_t payload_size
Definition activemsg.h:252
void * payload_base
Definition activemsg.h:251
virtual ~ActiveMessageImpl()
Definition activemsg.h:240
Realm::Serialization::FixedBufferSerializer fbs
Definition activemsg.h:173
void init(NodeID _target, const LocalAddress &_src_payload_addr, size_t _bytes_per_line, size_t _lines, size_t _line_stride, const RemoteAddress &_dest_payload_addr)
static size_t recommended_max_payload(NodeID target, const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion)
ActiveMessage(const Realm::NodeSet &_targets, const void *_data, size_t _datalen)
ActiveMessageImpl * impl
Definition activemsg.h:171
ActiveMessage(NodeID _target, size_t _max_payload_size=0)
void init(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen, const RemoteAddress &_dest_payload_addr)
uint64_t inline_capacity[INLINE_STORAGE/sizeof(uint64_t)]
Definition activemsg.h:175
void * payload_ptr(size_t datalen)
void init(NodeID _target, const void *_data, size_t _datalen)
void add_payload(const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, int payload_mode=PAYLOAD_COPY)
void init(NodeID _target, size_t _max_payload_size=0)
ActiveMessage(const Realm::NodeSet &_targets, size_t _max_payload_size=0)
T * header
Definition activemsg.h:172
ActiveMessage(NodeID _target, const void *_data, size_t _datalen)
void init(const Realm::NodeSet &_targets, const void *_data, size_t _datalen)
bool operator<<(const T2 &to_append)
void add_payload(const void *data, size_t datalen, int payload_mode=PAYLOAD_COPY)
static size_t recommended_max_payload(NodeID target, bool with_congestion)
static size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion)
friend class ActiveMessage
Definition activemsg.h:205
static size_t recommended_max_payload(const NodeSet &targets, const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion)
ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, size_t _bytes_per_line, size_t _lines, size_t _line_stride, const RemoteAddress &_dest_payload_addr)
static size_t recommended_max_payload(const NodeSet &targets, bool with_congestion)
void add_local_completion(const CALLABLE &callable)
void add_remote_completion(const CALLABLE &callable)
static size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, size_t line_stride, const RemoteAddress &dest_payload_addr, bool with_congestion)
void init(const Realm::NodeSet &_targets, size_t _max_payload_size=0)
void init(NodeID _target, size_t _max_payload_size, const RemoteAddress &_dest_payload_addr)
ActiveMessage(NodeID _target, size_t _max_payload_size, const RemoteAddress &_dest_payload_addr)
ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen, const RemoteAddress &_dest_payload_addr)
BackgroundWorkManager * manager
Definition bgwork.h:150
BackgroundWorkItem(const std::string &_name)
Definition activemsg.h:209
virtual ~CompletionCallbackBase()
static void destroy_all(void *start, size_t bytes)
static const size_t ALIGNMENT
Definition activemsg.h:216
virtual size_t size() const =0
static void clone_all(void *dst, const void *src, size_t bytes)
static void invoke_all(void *start, size_t bytes)
virtual CompletionCallbackBase * clone_at(void *p) const =0
virtual CompletionCallbackBase * clone_at(void *p) const
CALLABLE callable
Definition activemsg.h:233
CompletionCallback(const CALLABLE &_callable)
virtual size_t size() const
size_t total_messages_handled
Definition activemsg.h:503
int todo_oldest
Definition activemsg.h:499
void drain_incoming_messages(size_t min_messages_handled)
int todo_newest
Definition activemsg.h:499
size_t drain_min_count
Definition activemsg.h:502
Message ** heads
Definition activemsg.h:495
bool return_messages(int sender, size_t num_handled, Message *head, Message **tail)
MessageBlock * current_block
Definition activemsg.h:508
int handlers_active
Definition activemsg.h:500
uintptr_t CallbackData
Definition activemsg.h:405
atomic< bool > bgwork_requested
Definition activemsg.h:493
std::vector< Thread * > handler_threads
Definition activemsg.h:507
Mutex::CondVar drain_condvar
Definition activemsg.h:505
int dedicated_threads
Definition activemsg.h:492
Mutex::CondVar condvar
Definition activemsg.h:505
Message *** tails
Definition activemsg.h:496
CoreReservation * core_rsrv
Definition activemsg.h:506
int get_messages(Message *&head, Message **&tail, bool wait)
bool add_incoming_message(NodeID sender, ActiveMessageHandlerTable::MessageID msgid, const void *hdr, size_t hdr_size, int hdr_mode, const void *payload, size_t payload_size, int payload_mode, CallbackFnptr callback_fnptr, CallbackData callback_data1, CallbackData callback_data2, TimeLimit work_until)
void start_handler_threads(size_t stack_size)
int sleeper_count
Definition activemsg.h:492
bool * in_handler
Definition activemsg.h:497
static void invoke_deferred_callbacks(Message *msg)
~IncomingMessageManager(void)
void(* CallbackFnptr)(NodeID, CallbackData, CallbackData)
Definition activemsg.h:406
IncomingMessageManager(int _nodes, int _dedicated_threads, Realm::CoreReservationSet &crs)
size_t num_available_blocks
Definition activemsg.h:510
Mutex mutex
Definition activemsg.h:504
int nodes
Definition activemsg.h:492
int shutdown_flag
Definition activemsg.h:494
std::unordered_map< FragmentKey, FragmentReassembly, FragmentKeyHash > frag_message
Definition activemsg.h:541
size_t cfg_message_block_size
Definition activemsg.h:511
int * todo_list
Definition activemsg.h:498
MessageBlock * available_blocks
Definition activemsg.h:509
bool drain_pending
Definition activemsg.h:501
virtual bool do_work(TimeLimit work_until)
size_t cfg_max_available_blocks
Definition activemsg.h:511
void handler_thread_loop(void)
Definition serialize.h:118
UnfairCondVar CondVar
Definition mutex.h:227
#define REALM_INTERNAL_API_EXTERNAL_LINKAGE
Definition compiler_support.h:226
Helper utility for reconstructing large ActiveMessages that were split into multiple network packets.
Definition activemsg.h:44
bool profile_activemsg_handlers
long long max_inline_message_time
Definition activemsg.h:393
Definition activemsg.h:42
ActiveMessageHandlerTable activemsg_handler_table
UnfairMutex Mutex
Definition mutex.h:68
int NodeID
Definition nodeset.h:40
@ PAYLOAD_EMPTY
Definition activemsg.h:63
@ PAYLOAD_KEEPREG
Definition activemsg.h:62
@ PAYLOAD_PENDING
Definition activemsg.h:61
@ PAYLOAD_SRCPTR
Definition activemsg.h:60
@ PAYLOAD_NONE
Definition activemsg.h:56
@ PAYLOAD_COPY
Definition activemsg.h:59
@ PAYLOAD_FREE
Definition activemsg.h:58
@ PAYLOAD_KEEP
Definition activemsg.h:57
REALM_INTERNAL_API_EXTERNAL_LINKAGE uint64_t next_chunk_message_id(NodeID node_id)
REALM_INTERNAL_API_EXTERNAL_LINKAGE void report_unfragmentable_message(unsigned short msgid, size_t header_size, size_t wrapped_header_size, size_t payload_size)
Definition activemsg.h:257
void record(long long t_start, long long t_end)
atomic< size_t > count
Definition activemsg.h:258
atomic< size_t > minval
Definition activemsg.h:258
ActiveMessageHandlerStats(void)
atomic< size_t > maxval
Definition activemsg.h:258
atomic< size_t > sum
Definition activemsg.h:258
atomic< size_t > sum2
Definition activemsg.h:258
Definition activemsg.h:319
MessageHandlerNoTimeout handler_notimeout
Definition activemsg.h:323
std::optional< const FragmentInfo &(*)(const void *)> extract_frag_info
Definition activemsg.h:327
const char * name
Definition activemsg.h:321
ActiveMessageHandlerStats stats
Definition activemsg.h:325
TypeHash hash
Definition activemsg.h:320
MessageHandlerInline handler_inline
Definition activemsg.h:324
MessageHandler handler
Definition activemsg.h:322
Definition activemsg.h:264
uint32_t total_chunks
Definition activemsg.h:266
uint64_t msg_id
Definition activemsg.h:267
uint32_t chunk_id
Definition activemsg.h:265
Definition activemsg.h:440
CallbackData data1
Definition activemsg.h:442
CallbackFnptr fnptr
Definition activemsg.h:441
CallbackData data2
Definition activemsg.h:443
Definition activemsg.h:528
std::size_t operator()(const FragmentKey &k) const
Definition activemsg.h:529
Definition activemsg.h:517
bool operator==(const FragmentKey &rhs) const
Definition activemsg.h:522
ActiveMessageHandlerTable::MessageID msgid
Definition activemsg.h:519
NodeID sender
Definition activemsg.h:518
uint64_t msg_id
Definition activemsg.h:520
Definition activemsg.h:536
std::unique_ptr< FragmentedMessage > message
Definition activemsg.h:537
std::vector< DeferredCallback > deferred
Definition activemsg.h:538
Definition activemsg.h:464
size_t total_size
Definition activemsg.h:481
static MessageBlock * new_block(size_t _total_size)
static void free_block(MessageBlock *block)
void recycle_message(Message *msg, IncomingMessageManager *manager)
Message * append_message(size_t hdr_bytes_needed, size_t payload_bytes_needed)
atomic< unsigned > use_count
Definition activemsg.h:482
size_t size_used
Definition activemsg.h:481
static size_t bytes_needed(size_t hdr_bytes_needed, size_t payload_bytes_needed)
MessageBlock * next_free
Definition activemsg.h:483
Definition activemsg.h:446
CallbackFnptr callback_fnptr
Definition activemsg.h:457
CallbackData callback_data2
Definition activemsg.h:458
CallbackData callback_data1
Definition activemsg.h:458
void * payload
Definition activemsg.h:454
bool hdr_needs_free
Definition activemsg.h:453
void * hdr
Definition activemsg.h:451
MessageBlock * block
Definition activemsg.h:447
size_t payload_size
Definition activemsg.h:455
Message * next_msg
Definition activemsg.h:448
ActiveMessageHandlerTable::HandlerEntry * handler
Definition activemsg.h:450
size_t hdr_size
Definition activemsg.h:452
std::vector< DeferredCallback > * deferred_callbacks
Definition activemsg.h:461
NodeID sender
Definition activemsg.h:449
bool payload_needs_free
Definition activemsg.h:456
Definition activemsg.h:545
UserHdr * operator->()
Definition activemsg.h:549
UserHdr & operator*()
Definition activemsg.h:551
const UserHdr * operator->() const
Definition activemsg.h:550
FragmentInfo frag_info
Definition activemsg.h:546
UserHdr user
Definition activemsg.h:547
const UserHdr & operator*() const
Definition activemsg.h:552
Definition activemsg.h:557
NodeID src
Definition ucp_internal.h:1
unsigned short msgid
Definition ucp_internal.h:2