Realm
A distributed, event-based tasking library
Loading...
Searching...
No Matches
gasnetex_internal.h
Go to the documentation of this file.
1/*
2 * Copyright 2025 Stanford University, NVIDIA Corporation
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18// GASNet-EX network module internals
19
20#ifndef GASNETEX_INTERNAL_H
21#define GASNETEX_INTERNAL_H
22
24
25#ifdef REALM_USE_GASNETEX_WRAPPER
26// Disable the definitions and importing of symbols we're not going to use
27#define GEX_NO_PROTOTYPES 1
28#endif
29
31
32#include "realm/bgwork.h"
33#include "realm/atomics.h"
34#include "realm/activemsg.h"
35#include "realm/lists.h"
36
37namespace Realm {
38
39#define REALM_GEX_MODULE_VERSION 20240301
40
42
43 // rdma pointers need to identify which endpoint they belong to
48
49 class OutbufManager;
50
52 public:
54
60
62
74
75 static const int MAX_PACKETS = 256;
76
77 bool pktbuf_reserve(size_t bytes, int &pktidx, uintptr_t &offset);
79 uintptr_t pktbuf_get_offset(int pktidx);
80 bool pktbuf_commit(int pktidx, PktType pkttype, bool update_realbuf);
81
82 protected:
83 friend class OutbufManager;
84 friend class XmitSrcDestPair;
85
86 void set_state(State new_state);
87
91 uintptr_t baseptr;
92 size_t size;
93
97
99
100 // pbuf reservatsions are NOT thread-safe - external serialization is used
101 atomic<int> pktbuf_total_packets; // unsynchronized read ok
103 size_t pktbuf_pkt_ends[MAX_PACKETS]; // stores where a packet _ends_
104
105 // pbuf commits are lock-free
108
109 // pbuf consumption is NOT thread-safe - one pusher at a time
113 };
114
116 public:
118 : md(_md)
119 {}
120
121 void operator()() const { md->dec_usecount(); }
122
123 protected:
125 };
126
128 public:
131
132 void init(size_t _outbuf_count, size_t _outbuf_size, uintptr_t _baseptr);
133
135 bool new_endpoint);
137
138 virtual bool do_work(TimeLimit work_until);
139
140 protected:
145 // we manage the copying over overflow bufs back to real outbufs - track
146 // both pending overflow bufs and reserved outbufs and request bgwork
147 // time when we've got at least one of each
149 // In order to detect the case of misconfigured objcounts given the
150 // number of endpoints we count how many different endpoints we
151 // need to allocate buffers for, if it is more than the total number
152 // of buffers then we fail immediately since we'll never succeed
153 // TODO: Remove this once we insert a slow insertion pathway
154 // from dynamically allocated output buffers that are not
155 // registered with GASNet
156 // https://github.com/StanfordLegion/realm/issues/239
161 };
162
164
166 public:
168
169 void *add_local_completion(size_t bytes, bool late_ok = false);
170 void *add_remote_completion(size_t bytes);
171
172 // marks ready and returns true if non-empty, else resets and returns false
173 bool mark_ready(unsigned exp_local, unsigned exp_remote);
174
177
178 // these two calls can be concurrent, which complicates the determination of
179 // who can free the entry
182
183 int index;
186 // these three bits need to be in a single atomic location
187 static const unsigned LOCAL_PENDING_BIT = 1;
188 static const unsigned REMOTE_PENDING_BIT = 2;
189 static const unsigned READY_BIT = 4;
193
194 static const size_t TOTAL_CAPACITY = 256;
198 Storage_aligned storage;
199 };
200
202 static const size_t LOG2_GROUPSIZE = 8; // 256 per group
203
205 };
206
208 public:
211
214
216 void invoke_completions(PendingCompletion *comp, bool do_local, bool do_remote);
217
219
221
222 // Completion indices share a 32-bit AM argument with the message ID and two flag
223 // bits, so the number of bits left for the index depends on how wide the message
224 // ID has to be (see MSGID_BITS in gasnetex_internal.cc). Limits this manager to
225 // 2^index_bits completions, and its soft limit to half of that. Must be called
226 // before any completion is allocated; returns false if index_bits cannot hold
227 // even one group of completions.
228 bool set_index_bits(unsigned index_bits);
229
230 protected:
231 // NOTE: we stuff completion IDs, message IDs and 2 more bits into a
232 // 32-bit word, so we're limited to 2^(30-12) = 256K completions with the usual
233 // 12-bit message IDs - fewer if set_index_bits() asks for a narrower index
234 static const size_t LOG2_MAXGROUPS = 10;
235 int max_groups; // groups we may allocate: at most 1 << LOG2_MAXGROUPS
236
237 // protects pops from the free list (to avoid A-B-A problem), but NOT pushes
240 Realm::atomic<int> num_groups; // number of groups currently allocated
243 size_t pending_soft_limit; // try to stall traffic above this threshold
244 };
245
246 template <typename T, unsigned CHUNK_SIZE>
248 public:
251
253 void free_obj(T *obj);
254
255 protected:
256 // we'll store a pointer back to the chunk header just past each object
257 // each object needs enough space for a pointer back to the chunk header
258 struct Chunk;
259 struct WithPtr {
260 typedef char Storage_unaligned[sizeof(T)];
262 union {
263 Storage_aligned raw_storage;
264 uintptr_t nextptr;
265 };
267 };
273
275 uintptr_t free_head;
276 uintptr_t *free_tail;
280 };
281
282 struct PendingPutHeader;
283
285 protected:
286 // should not be directly allocated
287 template <typename T, unsigned CHUNK_SIZE>
288 friend class ChunkedRecycler;
291
292 public:
294 {
296 STRAT_SHORT_IMMEDIATE, // AM short, attempt immediate
297 STRAT_SHORT_PBUF, // AM short, deferred in pktbuf
298 STRAT_MEDIUM_IMMEDIATE, // AM medium, attempt immediate
299 STRAT_MEDIUM_PBUF, // AM medium, header/data in pktbuf
300 STRAT_MEDIUM_MALLOCSRC, // AM medium, malloc'd temp source (HACK)
301 STRAT_MEDIUM_PREP, // AM medium, using NPAM
308 };
312 unsigned short msgid;
320 };
321
322 // we'll keep separate transmit queues for each src/dst pair
323 // (i.e src_ep_index, dst_rank, dst_ep_index), hopefully avoiding any
324 // head of line blocking if networking resources are exhausted on a given
325 // path
327 public:
329 gex_rank_t _tgt_rank, gex_ep_index_t _tgt_ep_index);
331
332 // indicates whether any packets are pending, including those that
333 // have not been committed yet - useful for quiescence detection but
334 // also for ensuring new packets don't jump the queue
335 bool has_packets_queued() const;
336
337 // a packet that is sent immediately is counted as if it was
338 // reserved and then sent in rapid succession
340
341 // reserves space in a pbuf for an outbound packet (allocating and
342 // enqueuing a pbuf if needed)
343 bool reserve_pbuf_inline(size_t hdr_bytes, size_t payload_bytes, bool overflow_ok,
344 OutbufMetadata *&pktbuf, int &pktidx, void *&hdr_base,
345 void *&payload_base);
346 bool reserve_pbuf_long_rget(size_t hdr_bytes, bool overflow_ok,
347 OutbufMetadata *&pktbuf, int &pktidx, void *&hdr_base);
348 bool reserve_pbuf_put(bool overflow_ok, OutbufMetadata *&pktbuf, int &pktidx);
349 void commit_pbuf_inline(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base,
350 gex_am_arg_t arg0, size_t act_payload_bytes);
351 void commit_pbuf_long(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base,
352 gex_am_arg_t arg0, const void *payload_base,
353 size_t payload_bytes, uintptr_t dest_addr,
354 OutbufMetadata *databuf);
355 void commit_pbuf_rget(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base,
356 gex_am_arg_t arg0, const void *payload_base,
357 size_t payload_bytes, uintptr_t dest_addr,
359 void commit_pbuf_put(OutbufMetadata *pktbuf, int pktidx, PendingPutHeader *put,
360 const void *payload_base, size_t payload_bytes,
361 uintptr_t dest_addr);
362 void cancel_pbuf(OutbufMetadata *pktbuf, int pktidx);
363
365
367
368 // adds the xpair to the injector ready list or the poller critical
369 // pair list as appropriate, eventually resulting in a call to
370 // push_packets - MUST NOT be called until the push_packets
371 // that resulted from the previous enqueue is done-ish (i.e. not going
372 // to push anything else)
373 void request_push(bool force_critical);
374
375 void push_packets(bool immediate_mode, TimeLimit work_until);
376
377 long long time_since_failure() const;
378
379 // cancels a pending push that was enqueued via request_push but never
380 // processed by push_packets - used during shutdown to balance the
381 // push_mutex_check lock
383
384 // used when cfg_am_limit is nonzero
386 void return_am_credits(int count);
387
393
395 const void *payload_base;
397 uintptr_t dest_addr;
398 union {
399 struct {
401 } l;
402 struct {
403 // rget needs to give both src and target ep index for data
405 } r;
406 };
407 };
408
409 struct PutMetadata {
410 const void *src_addr;
411 uintptr_t dest_addr;
414 };
415
416 protected:
417 friend class GASNetEXInternal;
418
419 bool reserve_pbuf_helper(size_t total_bytes, bool overflow_ok,
420 OutbufMetadata *&pktbuf, int &pktidx, uintptr_t &baseptr);
421 bool commit_pbuf_helper(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base,
422 uintptr_t &baseptr);
423
430 // we don't hold the mutex while pushing packets, but we need definitely
431 // don't want multiple threads trying to push for the same src/dst pair
440 // circular queue of pending completion replys
443 atomic<unsigned> comp_reply_count; // read without mutex
445 // TODO: track packets in flight to avoid clogging?
446 atomic<int> am_credits; // available end-to-end AM credits for this peer
447 };
448
449 class XmitSrc {
450 public:
451 XmitSrc(GASNetEXInternal *_internal, gex_ep_index_t _src_ep_index);
453
455
456 protected:
457 friend class GASNetEXInternal;
458
461
462 // we'll allocate XmitSrcDestPair's on demand - atomics allow nonblocking
463 // lookup
465
466 // TODO: track in flight work at src level?
467 };
468
469 struct PendingReverseGet;
470
509
510 // an injector tries to send packets, but is not allowed to stall -
511 // all requests must use GEX_FLAG_IMMEDIATE and any failure results in
512 // handing the xpair off to the poller (which is allowed to block eventually)
514 public:
516
518
519 // drains any xpairs remaining in the ready queue, cancelling their
520 // pending pushes - called during shutdown
522
523 // Count of items in ready_xpairs plus the number of workers that have
524 // popped an xpair and are currently inside push_packets. Used by
525 // GASNetEXInternal::sample_quiescence_state to feed
526 // QuiescenceState::queued_items; reporting the number of items rather
527 // than just their presence lets a draining queue surface as "progressing"
528 // across Mattern's rounds, and including active_workers keeps all
529 // in-flight push_packets visible so the quiescence check doesn't fire
530 // DONE mid-push.
531 size_t queue_size();
532
533 virtual bool do_work(TimeLimit work_until);
534
535 protected:
538 // Multiple background workers may execute do_work concurrently.
541 };
542
544 public:
546
549
551
552 // drains any xpairs remaining in the critical queue, cancelling their
553 // pending pushes - called during shutdown
555
557
558 // Count of items in critical_xpairs + pending_events plus 1 if a
559 // worker is currently inside do_work with items on its stack. See
560 // GASNetEXInjector::queue_size.
561 size_t queue_size();
562
563 virtual bool do_work(TimeLimit work_until);
564
565 // causes calling thread to wait for a full call to gasnet_AMPoll() to
566 // be performed by the poller
568
569 protected:
570 bool started = false;
573 atomic<bool> shutdown_flag; // set/cleared inside mutex, but tested outside
575 atomic<bool> pollwait_flag; // set/cleared inside mutex, but tested outside
577 atomic<bool> work_active; // set during do_work when items are on stack
580 };
581
583 public:
585
587
588 // Count of items in ready_events; returns 1 instead of 0 when
589 // has_work is set but ready_events has been swapped to a worker's
590 // local todo list. See GASNetEXInjector::queue_size.
591 size_t queue_size();
592
593 virtual bool do_work(TimeLimit work_until);
594
595 protected:
598 atomic<bool> has_work; // can be read without mutex
600 };
601
603 protected:
604 // should not be directly allocated
605 template <typename T, unsigned CHUNK_SIZE>
606 friend class ChunkedRecycler;
609
610 public:
614 static const size_t MAX_HDR_SIZE = 128;
615 size_t hdr_size;
616 unsigned char hdr_data[MAX_HDR_SIZE];
617 uintptr_t src_ptr, tgt_ptr;
622 };
623
624 // NOTE: ReverseGetter exists for now because we have to use RMAs instead
625 // of long AMs for transfers between segments bound to non-primordial
626 // endpoints (on the source and/or the target) - once GASNet supports
627 // AMs on all endpoints, this code should probably go away
628 class ReverseGetter;
629
631 protected:
632 // should not be directly allocated
633 template <typename T, unsigned CHUNK_SIZE>
634 friend class ChunkedRecycler;
637
638 public:
644 static const size_t MAX_HDR_SIZE = 128;
645 size_t hdr_size;
646 unsigned char hdr_data[MAX_HDR_SIZE];
647 uintptr_t src_ptr, tgt_ptr;
649 };
650
652 public:
654
655 void add_reverse_get(gex_rank_t srcrank, gex_ep_index_t src_ep_index,
656 gex_ep_index_t tgt_ep_index, gex_am_arg_t arg0, const void *hdr,
657 size_t hdr_bytes, uintptr_t src_ptr, uintptr_t tgt_ptr,
658 size_t payload_bytes);
659
660 // O(n) walk of the pending-rget list under the mutex. See
661 // GASNetEXInjector::queue_size.
662 size_t queue_size();
663
664 virtual bool do_work(TimeLimit work_until);
665
666 protected:
667 friend class GASNetEXEvent;
669
675 };
676
678 public:
680
682
683 bool init(int *argc, const char ***argv);
684 uintptr_t attach(size_t size);
685
686 bool attempt_binding(void *base, size_t size, NetworkSegmentInfo::MemoryType memtype,
688 gex_ep_index_t *ep_indexp);
690
691 void detach();
692
693 void get_shared_peers(Realm::NodeSet &shared_peers);
694 void barrier();
695 void broadcast(gex_rank_t root, const void *val_in, void *val_out, size_t bytes);
696 void gather(gex_rank_t root, const void *val_in, void *vals_out, size_t bytes);
697 void allgatherv(const char *val_in, size_t bytes, std::vector<char> &vals_out,
698 std::vector<size_t> &lengths);
699
701 void quiescence_allreduce_sum(const uint64_t *local_counts, uint64_t *total_counts,
702 size_t count);
703
706
707 size_t recommended_max_payload(gex_rank_t target, gex_ep_index_t target_ep_index,
708 bool with_congestion, size_t header_size,
709 uintptr_t dest_payload_addr);
710 size_t recommended_max_payload(gex_rank_t target, gex_ep_index_t target_ep_index,
711 const void *data, size_t bytes_per_line, size_t lines,
712 size_t line_stride, bool with_congestion,
713 size_t header_size, uintptr_t dest_payload_addr);
714 size_t recommended_max_payload(bool with_congestion, size_t header_size);
715
717 unsigned short msgid, void *&header_base,
718 size_t header_size, void *&payload_base,
719 size_t payload_size, uintptr_t dest_payload_addr);
720 void commit_message(PreparedMessage *msg, PendingCompletion *comp, void *header_base,
721 size_t header_size, void *payload_base, size_t payload_size);
723
724 gex_am_arg_t handle_short(gex_rank_t srcrank, gex_am_arg_t arg0, const void *hdr,
725 size_t hdr_bytes);
726 gex_am_arg_t handle_medium(gex_rank_t srcrank, gex_am_arg_t arg0, const void *hdr,
727 size_t hdr_bytes, const void *data, size_t data_bytes);
728 gex_am_arg_t handle_long(gex_rank_t srcrank, gex_am_arg_t arg0, const void *hdr,
729 size_t hdr_bytes, const void *data, size_t data_bytes);
730 void handle_reverse_get(gex_rank_t srcrank, gex_ep_index_t src_ep_index,
731 gex_ep_index_t tgt_ep_index, gex_am_arg_t arg0,
732 const void *hdr, size_t hdr_bytes, uintptr_t src_ptr,
733 uintptr_t tgt_ptr, size_t payload_bytes);
735 const void *data, size_t data_bytes, gex_am_arg_t *comps);
737 size_t nargs);
738
739 protected:
740 friend class ReverseGetter;
741 friend class XmitSrc;
742 friend class XmitSrcDestPair;
743 friend class GASNetEXEvent;
744 friend class GASNetEXPoller;
745 friend class GASNetEXCompleter;
746 friend class GASNetEXInjector;
747
748 // callbacks from IncomingMessageManager
749 static void short_message_complete(NodeID sender, uintptr_t objptr,
750 uintptr_t comp_info);
751 static void medium_message_complete(NodeID sender, uintptr_t objptr,
752 uintptr_t comp_info);
753 static void long_message_complete(NodeID sender, uintptr_t objptr,
754 uintptr_t comp_info);
755
757
761 // order in 'eps' should match GASNet's indexing
762 std::vector<gex_ep_opaque_t> eps;
767
775 struct SegmentInfoSorter;
776 // this list is sorted by address to enable quick address lookup
777 std::vector<SegmentInfo> segments_by_addr;
778
779 const SegmentInfo *find_segment(const void *srcptr) const;
780
781 std::vector<XmitSrc *> xmitsrcs;
782
789
790 // TODO: split counter into per-thread values to avoid contention?
792
793 // Monotonic count of items ever added to any of the four background-work
794 // queues (injector ready_xpairs, poller critical_xpairs/pending_events,
795 // completer ready_events, rgetter pending list). Sampled by
796 // sample_quiescence_state and reduced via SUM as
797 // QuiescenceState::events_added. Required because queued_items is a
798 // snapshot count that can be the same across rounds while items flow
799 // through (e.g., a put-completion event firing pops 1 from
800 // pending_events and adds 1 to ready_xpairs - net change 0 in
801 // queued_items, but real activity). Bumped at the same sites that call
802 // make_active() / push_back into a queue.
804
805 // manage a single open databuf for all endpoints
809
810 // allocator/managers for various objects we want to reuse
814 };
815
816}; // namespace Realm
817
818#endif
BackgroundWorkItem(const std::string &_name)
Definition gasnetex_internal.h:247
Mutex mutex
Definition gasnetex_internal.h:274
size_t max_alloc
Definition gasnetex_internal.h:279
atomic< size_t > cur_alloc
Definition gasnetex_internal.h:278
Chunk * chunks_head
Definition gasnetex_internal.h:277
uintptr_t free_head
Definition gasnetex_internal.h:275
size_t cur_capacity
Definition gasnetex_internal.h:279
uintptr_t * free_tail
Definition gasnetex_internal.h:276
static const size_t ALIGNMENT
Definition activemsg.h:216
Definition threads.h:428
GASNetEXCompleter(GASNetEXInternal *_internal)
void add_ready_events(GASNetEXEvent::EventList &newly_ready)
atomic< bool > has_work
Definition gasnetex_internal.h:598
Mutex mutex
Definition gasnetex_internal.h:597
GASNetEXInternal * internal
Definition gasnetex_internal.h:596
GASNetEXEvent::EventList ready_events
Definition gasnetex_internal.h:599
virtual bool do_work(TimeLimit work_until)
Definition gasnetex_internal.h:471
GASNetEXEvent & set_pktbuf(OutbufMetadata *_pktbuf)
GASNetEXEvent * leaf
Definition gasnetex_internal.h:507
~GASNetEXEvent()
Definition gasnetex_internal.h:477
REALM_PMTA_DEFN(GASNetEXEvent, IntrusiveListLink< GASNetEXEvent >, event_list_link)
GASNetEXEvent & set_leaf(GASNetEXEvent *_leaf)
GASNetEXEvent & set_put(PendingPutHeader *_put)
gex_ep_opaque_t event
Definition gasnetex_internal.h:501
PendingCompletion * local_comp
Definition gasnetex_internal.h:502
GASNetEXEvent & set_event(gex_event_opaque_t _event)
GASNetEXEvent & set_rget(PendingReverseGet *_rget)
GASNetEXEvent & set_databuf(OutbufMetadata *_databuf)
PendingPutHeader * put
Definition gasnetex_internal.h:506
IntrusiveListLink< GASNetEXEvent > event_list_link
Definition gasnetex_internal.h:494
OutbufMetadata * pktbuf
Definition gasnetex_internal.h:503
gex_event_opaque_t get_event() const
OutbufMetadata * databuf
Definition gasnetex_internal.h:504
void trigger(GASNetEXInternal *internal)
IntrusiveList< GASNetEXEvent, REALM_PMTA_USE(GASNetEXEvent, event_list_link), DummyLock > EventList
Definition gasnetex_internal.h:498
friend class ChunkedRecycler
Definition gasnetex_internal.h:475
GASNetEXEvent & set_local_comp(PendingCompletion *_local_comp)
PendingReverseGet * rget
Definition gasnetex_internal.h:505
void add_ready_xpair(XmitSrcDestPair *xpair)
GASNetEXInjector(GASNetEXInternal *_internal)
atomic< unsigned > active_workers
Definition gasnetex_internal.h:539
GASNetEXInternal * internal
Definition gasnetex_internal.h:536
virtual bool do_work(TimeLimit work_until)
XmitSrcDestPair::XmitPairList ready_xpairs
Definition gasnetex_internal.h:540
Mutex mutex
Definition gasnetex_internal.h:537
Definition gasnetex_internal.h:677
GASNetEXCompleter completer
Definition gasnetex_internal.h:785
RuntimeImpl * runtime
Definition gasnetex_internal.h:759
std::vector< XmitSrc * > xmitsrcs
Definition gasnetex_internal.h:781
atomic< uint64_t > total_packets_received
Definition gasnetex_internal.h:791
size_t recommended_max_payload(bool with_congestion, size_t header_size)
gex_rank_t prim_rank
Definition gasnetex_internal.h:764
gex_callback_handle_t gex_callback_handle
Definition gasnetex_internal.h:808
PreparedMessage * prepare_message(gex_rank_t target, gex_ep_index_t target_ep_index, unsigned short msgid, void *&header_base, size_t header_size, void *&payload_base, size_t payload_size, uintptr_t dest_payload_addr)
static void long_message_complete(NodeID sender, uintptr_t objptr, uintptr_t comp_info)
OutbufMetadata * databuf_md
Definition gasnetex_internal.h:807
size_t recommended_max_payload(gex_rank_t target, gex_ep_index_t target_ep_index, bool with_congestion, size_t header_size, uintptr_t dest_payload_addr)
GASNetEXPoller poller
Definition gasnetex_internal.h:783
void sample_quiescence_state(NetworkModule::QuiescenceState &state)
Mutex databuf_mutex
Definition gasnetex_internal.h:806
friend class XmitSrcDestPair
Definition gasnetex_internal.h:742
PendingCompletionManager compmgr
Definition gasnetex_internal.h:787
bool init(int *argc, const char ***argv)
void cancel_message(PreparedMessage *msg)
gex_tm_opaque_t prim_tm
Definition gasnetex_internal.h:763
uintptr_t attach(size_t size)
OutbufManager obmgr
Definition gasnetex_internal.h:788
size_t recommended_max_payload(gex_rank_t target, gex_ep_index_t target_ep_index, const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size, uintptr_t dest_payload_addr)
const SegmentInfo * find_segment(const void *srcptr) const
size_t handle_batch(gex_rank_t srcrank, gex_am_arg_t arg0, gex_am_arg_t cksum, const void *data, size_t data_bytes, gex_am_arg_t *comps)
friend class GASNetEXInjector
Definition gasnetex_internal.h:746
friend class XmitSrc
Definition gasnetex_internal.h:741
static void medium_message_complete(NodeID sender, uintptr_t objptr, uintptr_t comp_info)
PendingCompletion * early_local_completion(PendingCompletion *comp)
GASNetEXInternal(GASNetEXModule *_module, RuntimeImpl *_runtime)
std::vector< gex_ep_opaque_t > eps
Definition gasnetex_internal.h:762
PendingCompletion * get_available_comp()
gex_client_opaque_t client
Definition gasnetex_internal.h:760
ChunkedRecycler< PendingPutHeader, 32 > put_alloc
Definition gasnetex_internal.h:813
static void short_message_complete(NodeID sender, uintptr_t objptr, uintptr_t comp_info)
void get_shared_peers(Realm::NodeSet &shared_peers)
ReverseGetter rgetter
Definition gasnetex_internal.h:786
friend class ReverseGetter
Definition gasnetex_internal.h:740
size_t prim_segsize
Definition gasnetex_internal.h:766
friend class GASNetEXPoller
Definition gasnetex_internal.h:744
gex_am_arg_t handle_short(gex_rank_t srcrank, gex_am_arg_t arg0, const void *hdr, size_t hdr_bytes)
ChunkedRecycler< PreparedMessage, 32 > prep_alloc
Definition gasnetex_internal.h:812
void handle_completion_reply(gex_rank_t srcrank, const gex_am_arg_t *args, size_t nargs)
void commit_message(PreparedMessage *msg, PendingCompletion *comp, void *header_base, size_t header_size, void *payload_base, size_t payload_size)
friend class GASNetEXEvent
Definition gasnetex_internal.h:743
gex_am_arg_t handle_medium(gex_rank_t srcrank, gex_am_arg_t arg0, const void *hdr, size_t hdr_bytes, const void *data, size_t data_bytes)
gex_am_arg_t handle_long(gex_rank_t srcrank, gex_am_arg_t arg0, const void *hdr, size_t hdr_bytes, const void *data, size_t data_bytes)
void allgatherv(const char *val_in, size_t bytes, std::vector< char > &vals_out, std::vector< size_t > &lengths)
void broadcast(gex_rank_t root, const void *val_in, void *val_out, size_t bytes)
GASNetEXInjector injector
Definition gasnetex_internal.h:784
void gather(gex_rank_t root, const void *val_in, void *vals_out, size_t bytes)
ChunkedRecycler< GASNetEXEvent, 64 > event_alloc
Definition gasnetex_internal.h:811
atomic< uint64_t > total_events_added
Definition gasnetex_internal.h:803
void quiescence_allreduce_sum(const uint64_t *local_counts, uint64_t *total_counts, size_t count)
gex_segment_opaque_t prim_segment
Definition gasnetex_internal.h:765
gex_rank_t prim_size
Definition gasnetex_internal.h:764
GASNetEXModule * module
Definition gasnetex_internal.h:758
PendingCompletion * extract_arg0_local_comp(gex_am_arg_t &arg0)
void handle_reverse_get(gex_rank_t srcrank, gex_ep_index_t src_ep_index, gex_ep_index_t tgt_ep_index, gex_am_arg_t arg0, const void *hdr, size_t hdr_bytes, uintptr_t src_ptr, uintptr_t tgt_ptr, size_t payload_bytes)
std::vector< SegmentInfo > segments_by_addr
Definition gasnetex_internal.h:777
friend class GASNetEXCompleter
Definition gasnetex_internal.h:745
bool attempt_binding(void *base, size_t size, NetworkSegmentInfo::MemoryType memtype, NetworkSegmentInfo::MemoryTypeExtraData memextra, gex_ep_index_t *ep_indexp)
Definition gasnetex_module.h:29
GASNetEXPoller(GASNetEXInternal *_internal)
GASNetEXEvent::EventList pending_events
Definition gasnetex_internal.h:579
void add_pending_event(GASNetEXEvent *event)
virtual bool do_work(TimeLimit work_until)
Mutex mutex
Definition gasnetex_internal.h:572
Mutex::CondVar pollwait_cond
Definition gasnetex_internal.h:576
XmitSrcDestPair::XmitPairList critical_xpairs
Definition gasnetex_internal.h:578
void add_critical_xpair(XmitSrcDestPair *xpair)
atomic< bool > shutdown_flag
Definition gasnetex_internal.h:573
Mutex::CondVar shutdown_cond
Definition gasnetex_internal.h:574
bool started
Definition gasnetex_internal.h:570
GASNetEXInternal * internal
Definition gasnetex_internal.h:571
atomic< bool > work_active
Definition gasnetex_internal.h:577
atomic< bool > pollwait_flag
Definition gasnetex_internal.h:575
Definition lists.h:66
Definition mutex.h:80
Definition nodeset.h:117
Definition gasnetex_internal.h:127
OutbufMetadata * reserved_head
Definition gasnetex_internal.h:160
void init(size_t _outbuf_count, size_t _outbuf_size, uintptr_t _baseptr)
size_t num_endpoints
Definition gasnetex_internal.h:157
size_t outbuf_size
Definition gasnetex_internal.h:142
OutbufMetadata * overflow_head
Definition gasnetex_internal.h:158
OutbufMetadata * metadatas
Definition gasnetex_internal.h:141
OutbufMetadata * first_available
Definition gasnetex_internal.h:144
size_t num_overflow
Definition gasnetex_internal.h:148
OutbufMetadata ** overflow_tail
Definition gasnetex_internal.h:159
OutbufMetadata * alloc_outbuf(OutbufMetadata::State state, bool overflow_ok, bool new_endpoint)
void free_outbuf(OutbufMetadata *md)
virtual bool do_work(TimeLimit work_until)
size_t num_buffers
Definition gasnetex_internal.h:157
Mutex mutex
Definition gasnetex_internal.h:143
size_t num_reserved
Definition gasnetex_internal.h:148
Definition gasnetex_internal.h:51
OutbufManager * manager
Definition gasnetex_internal.h:89
bool is_overflow
Definition gasnetex_internal.h:94
size_t pktbuf_rsrv_offset
Definition gasnetex_internal.h:102
static const int MAX_PACKETS
Definition gasnetex_internal.h:75
bool pktbuf_reserve(size_t bytes, int &pktidx, uintptr_t &offset)
atomic< int > remain_count
Definition gasnetex_internal.h:98
friend class XmitSrcDestPair
Definition gasnetex_internal.h:84
State state
Definition gasnetex_internal.h:88
atomic< OutbufMetadata * > realbuf
Definition gasnetex_internal.h:96
OutbufMetadata * next_overflow
Definition gasnetex_internal.h:95
size_t pktbuf_sent_offset
Definition gasnetex_internal.h:111
atomic< int > pktbuf_total_packets
Definition gasnetex_internal.h:101
int pktbuf_use_count
Definition gasnetex_internal.h:112
PktType
Definition gasnetex_internal.h:64
@ PKTTYPE_LONG
Definition gasnetex_internal.h:68
@ PKTTYPE_RGET
Definition gasnetex_internal.h:69
@ PKTTYPE_INLINE
Definition gasnetex_internal.h:66
@ PKTTYPE_COPY_IN_PROGRESS
Definition gasnetex_internal.h:72
@ PKTTYPE_PUT
Definition gasnetex_internal.h:70
@ PKTTYPE_INVALID
Definition gasnetex_internal.h:65
@ PKTTYPE_INLINE_SHORT
Definition gasnetex_internal.h:67
@ PKTTYPE_CANCELLED
Definition gasnetex_internal.h:71
void set_state(State new_state)
bool pktbuf_commit(int pktidx, PktType pkttype, bool update_realbuf)
size_t size
Definition gasnetex_internal.h:92
atomic< PktType > pktbuf_pkt_types[MAX_PACKETS]
Definition gasnetex_internal.h:106
OutbufMetadata * nextbuf
Definition gasnetex_internal.h:90
uintptr_t pktbuf_get_offset(int pktidx)
int pktbuf_sent_packets
Definition gasnetex_internal.h:110
size_t pktbuf_pkt_ends[MAX_PACKETS]
Definition gasnetex_internal.h:103
friend class OutbufManager
Definition gasnetex_internal.h:83
State
Definition gasnetex_internal.h:56
@ STATE_PKTBUF
Definition gasnetex_internal.h:58
@ STATE_IDLE
Definition gasnetex_internal.h:57
atomic< int > pktbuf_ready_packets
Definition gasnetex_internal.h:107
uintptr_t baseptr
Definition gasnetex_internal.h:91
OutbufMetadata * md
Definition gasnetex_internal.h:124
OutbufUsecountDec(OutbufMetadata *_md)
Definition gasnetex_internal.h:117
void operator()() const
Definition gasnetex_internal.h:121
Definition gasnetex_internal.h:207
void invoke_completions(PendingCompletion *comp, bool do_local, bool do_remote)
PendingCompletion * get_available()
Realm::atomic< PendingCompletionGroup * > groups[1<< LOG2_MAXGROUPS]
Definition gasnetex_internal.h:241
size_t pending_soft_limit
Definition gasnetex_internal.h:243
Realm::atomic< PendingCompletion * > first_free
Definition gasnetex_internal.h:239
PendingCompletion * lookup_completion(int index)
Realm::Mutex mutex
Definition gasnetex_internal.h:238
void recycle_comp(PendingCompletion *comp)
int max_groups
Definition gasnetex_internal.h:235
Realm::atomic< int > num_groups
Definition gasnetex_internal.h:240
atomic< size_t > num_pending
Definition gasnetex_internal.h:242
bool set_index_bits(unsigned index_bits)
static const size_t LOG2_MAXGROUPS
Definition gasnetex_internal.h:234
bool over_pending_completion_soft_limit() const
Definition gasnetex_internal.h:165
REALM_ALIGNED_TYPE_CONST(Storage_aligned, Storage_unaligned, Realm::CompletionCallbackBase::ALIGNMENT)
void * add_local_completion(size_t bytes, bool late_ok=false)
int index
Definition gasnetex_internal.h:183
Storage_aligned storage
Definition gasnetex_internal.h:198
PendingCompletionManager * manager
Definition gasnetex_internal.h:185
static const size_t TOTAL_CAPACITY
Definition gasnetex_internal.h:194
char Storage_unaligned[TOTAL_CAPACITY]
Definition gasnetex_internal.h:195
static const unsigned LOCAL_PENDING_BIT
Definition gasnetex_internal.h:187
size_t remote_bytes
Definition gasnetex_internal.h:192
void * add_remote_completion(size_t bytes)
static const unsigned REMOTE_PENDING_BIT
Definition gasnetex_internal.h:188
Realm::atomic< unsigned > remote_left
Definition gasnetex_internal.h:191
Realm::atomic< unsigned > local_left
Definition gasnetex_internal.h:191
static const unsigned READY_BIT
Definition gasnetex_internal.h:189
Realm::atomic< unsigned > state
Definition gasnetex_internal.h:190
size_t local_bytes
Definition gasnetex_internal.h:192
PendingCompletion * next_free
Definition gasnetex_internal.h:184
bool mark_ready(unsigned exp_local, unsigned exp_remote)
Definition gasnetex_internal.h:651
void add_reverse_get(gex_rank_t srcrank, gex_ep_index_t src_ep_index, gex_ep_index_t tgt_ep_index, gex_am_arg_t arg0, const void *hdr, size_t hdr_bytes, uintptr_t src_ptr, uintptr_t tgt_ptr, size_t payload_bytes)
GASNetEXInternal * internal
Definition gasnetex_internal.h:670
PendingReverseGet ** tailp
Definition gasnetex_internal.h:673
Mutex mutex
Definition gasnetex_internal.h:671
ChunkedRecycler< PendingReverseGet, 8 > rget_alloc
Definition gasnetex_internal.h:674
PendingReverseGet * head
Definition gasnetex_internal.h:672
friend class GASNetEXEvent
Definition gasnetex_internal.h:667
virtual bool do_work(TimeLimit work_until)
void reverse_get_complete(PendingReverseGet *rget)
ReverseGetter(GASNetEXInternal *_internal)
Definition runtime_impl.h:271
Definition timers.h:129
UnfairCondVar CondVar
Definition mutex.h:227
Definition gasnetex_internal.h:326
IntrusiveListLink< XmitSrcDestPair > xpair_list_link
Definition gasnetex_internal.h:388
OutbufMetadata * cur_pbuf
Definition gasnetex_internal.h:434
void commit_pbuf_inline(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base, gex_am_arg_t arg0, size_t act_payload_bytes)
bool commit_pbuf_helper(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base, uintptr_t &baseptr)
void enqueue_put_header(PendingPutHeader *put)
atomic< unsigned > comp_reply_count
Definition gasnetex_internal.h:443
unsigned comp_reply_capacity
Definition gasnetex_internal.h:444
void commit_pbuf_put(OutbufMetadata *pktbuf, int pktidx, PendingPutHeader *put, const void *payload_base, size_t payload_bytes, uintptr_t dest_addr)
atomic< OutbufMetadata * > first_pbuf
Definition gasnetex_internal.h:433
GASNetEXInternal * internal
Definition gasnetex_internal.h:424
void request_push(bool force_critical)
bool reserve_pbuf_put(bool overflow_ok, OutbufMetadata *&pktbuf, int &pktidx)
unsigned comp_reply_wrptr
Definition gasnetex_internal.h:442
IntrusiveList< XmitSrcDestPair, REALM_PMTA_USE(XmitSrcDestPair, xpair_list_link), DummyLock > XmitPairList
Definition gasnetex_internal.h:392
friend class GASNetEXInternal
Definition gasnetex_internal.h:417
bool has_packets_queued() const
gex_ep_index_t src_ep_index
Definition gasnetex_internal.h:425
atomic< size_t > packets_reserved
Definition gasnetex_internal.h:428
REALM_PMTA_DEFN(XmitSrcDestPair, IntrusiveListLink< XmitSrcDestPair >, xpair_list_link)
Mutex mutex
Definition gasnetex_internal.h:429
long long time_since_failure() const
atomic< int > am_credits
Definition gasnetex_internal.h:446
atomic< unsigned > imm_fail_count
Definition gasnetex_internal.h:435
gex_am_arg_t * comp_reply_data
Definition gasnetex_internal.h:441
long long first_fail_time
Definition gasnetex_internal.h:437
void return_am_credits(int count)
atomic< size_t > packets_sent
Definition gasnetex_internal.h:428
bool reserve_pbuf_long_rget(size_t hdr_bytes, bool overflow_ok, OutbufMetadata *&pktbuf, int &pktidx, void *&hdr_base)
atomic< PendingPutHeader * > put_head
Definition gasnetex_internal.h:438
unsigned comp_reply_rdptr
Definition gasnetex_internal.h:442
void commit_pbuf_rget(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base, gex_am_arg_t arg0, const void *payload_base, size_t payload_bytes, uintptr_t dest_addr, gex_ep_index_t src_ep_index, gex_ep_index_t tgt_ep_index)
void enqueue_completion_reply(gex_am_arg_t comp_info)
MutexChecker push_mutex_check
Definition gasnetex_internal.h:432
void commit_pbuf_long(OutbufMetadata *pktbuf, int pktidx, const void *hdr_base, gex_am_arg_t arg0, const void *payload_base, size_t payload_bytes, uintptr_t dest_addr, OutbufMetadata *databuf)
bool reserve_pbuf_helper(size_t total_bytes, bool overflow_ok, OutbufMetadata *&pktbuf, int &pktidx, uintptr_t &baseptr)
gex_rank_t tgt_rank
Definition gasnetex_internal.h:426
atomic< PendingPutHeader * > * put_tailp
Definition gasnetex_internal.h:439
void cancel_pbuf(OutbufMetadata *pktbuf, int pktidx)
gex_ep_index_t tgt_ep_index
Definition gasnetex_internal.h:427
void push_packets(bool immediate_mode, TimeLimit work_until)
bool has_ready_packets
Definition gasnetex_internal.h:436
XmitSrcDestPair(GASNetEXInternal *_internal, gex_ep_index_t _src_ep_index, gex_rank_t _tgt_rank, gex_ep_index_t _tgt_ep_index)
bool reserve_pbuf_inline(size_t hdr_bytes, size_t payload_bytes, bool overflow_ok, OutbufMetadata *&pktbuf, int &pktidx, void *&hdr_base, void *&payload_base)
friend class GASNetEXInternal
Definition gasnetex_internal.h:457
XmitSrc(GASNetEXInternal *_internal, gex_ep_index_t _src_ep_index)
XmitSrcDestPair * lookup_pair(gex_rank_t tgt_rank, gex_ep_index_t tgt_ep_index)
gex_ep_index_t src_ep_index
Definition gasnetex_internal.h:460
atomic< XmitSrcDestPair * > * pairs
Definition gasnetex_internal.h:464
GASNetEXInternal * internal
Definition gasnetex_internal.h:459
Definition atomics.h:31
void * gex_tm_opaque_t
Definition gasnetex_wrapper.h:38
void * gex_event_opaque_t
Definition gasnetex_wrapper.h:42
int32_t gex_am_arg_t
Definition gasnetex_wrapper.h:33
uint16_t gex_ep_index_t
Definition gasnetex_wrapper.h:30
struct gex_wrapper_handle_s gex_wrapper_handle_t
Handle structure that contains the full API for the wrapper.
void * gex_am_src_desc_opaque_t
Definition gasnetex_wrapper.h:43
uint32_t gex_rank_t
Definition gasnetex_wrapper.h:29
void * gex_segment_opaque_t
Definition gasnetex_wrapper.h:39
void * gex_ep_opaque_t
Definition gasnetex_wrapper.h:37
struct gex_callback_handle_s gex_callback_handle_t
void * gex_client_opaque_t
Definition gasnetex_wrapper.h:36
#define REALM_PMTA_USE(structtype, name)
Definition lists.h:42
unsigned MemoryType
Definition network.h:391
uintptr_t MemoryTypeExtraData
Definition network.h:394
Definition activemsg.h:42
UnfairMutex Mutex
Definition mutex.h:68
int NodeID
Definition nodeset.h:40
gex_wrapper_handle_t gex_wrapper_handle
Definition gasnetex_internal.h:268
Chunk * next_chunk
Definition gasnetex_internal.h:270
atomic< unsigned > remaining_count
Definition gasnetex_internal.h:269
WithPtr elements[CHUNK_SIZE]
Definition gasnetex_internal.h:271
Definition gasnetex_internal.h:259
Storage_aligned raw_storage
Definition gasnetex_internal.h:263
Chunk * backptr
Definition gasnetex_internal.h:266
REALM_ALIGNED_TYPE_SAMEAS(Storage_aligned, Storage_unaligned, T)
char Storage_unaligned[sizeof(T)]
Definition gasnetex_internal.h:260
uintptr_t nextptr
Definition gasnetex_internal.h:264
Definition gasnetex_internal.h:768
uintptr_t base
Definition gasnetex_internal.h:769
NetworkSegmentInfo::MemoryTypeExtraData memextra
Definition gasnetex_internal.h:773
gex_segment_opaque_t segment
Definition gasnetex_internal.h:771
uintptr_t limit
Definition gasnetex_internal.h:769
NetworkSegmentInfo::MemoryType memtype
Definition gasnetex_internal.h:772
gex_ep_index_t ep_index
Definition gasnetex_internal.h:770
Definition gasnetex_internal.h:44
gex_ep_index_t ep_index
Definition gasnetex_internal.h:46
uintptr_t base
Definition gasnetex_internal.h:45
Definition gasnetex_internal.h:201
static const size_t LOG2_GROUPSIZE
Definition gasnetex_internal.h:202
PendingCompletion entries[1<< LOG2_GROUPSIZE]
Definition gasnetex_internal.h:204
Definition gasnetex_internal.h:602
gex_ep_index_t src_ep_index
Definition gasnetex_internal.h:612
PendingPutHeader()
Definition gasnetex_internal.h:607
uintptr_t tgt_ptr
Definition gasnetex_internal.h:617
gex_ep_index_t tgt_ep_index
Definition gasnetex_internal.h:612
gex_am_arg_t arg0
Definition gasnetex_internal.h:613
~PendingPutHeader()
Definition gasnetex_internal.h:608
size_t payload_bytes
Definition gasnetex_internal.h:618
static const size_t MAX_HDR_SIZE
Definition gasnetex_internal.h:614
size_t hdr_size
Definition gasnetex_internal.h:615
atomic< PendingPutHeader * > next_put
Definition gasnetex_internal.h:621
XmitSrcDestPair * xpair
Definition gasnetex_internal.h:620
unsigned char hdr_data[MAX_HDR_SIZE]
Definition gasnetex_internal.h:616
friend class ChunkedRecycler
Definition gasnetex_internal.h:606
uintptr_t src_ptr
Definition gasnetex_internal.h:617
gex_rank_t target
Definition gasnetex_internal.h:611
PendingCompletion * local_comp
Definition gasnetex_internal.h:619
Definition gasnetex_internal.h:630
ReverseGetter * rgetter
Definition gasnetex_internal.h:639
unsigned char hdr_data[MAX_HDR_SIZE]
Definition gasnetex_internal.h:646
PendingReverseGet()
Definition gasnetex_internal.h:635
static const size_t MAX_HDR_SIZE
Definition gasnetex_internal.h:644
gex_am_arg_t arg0
Definition gasnetex_internal.h:643
~PendingReverseGet()
Definition gasnetex_internal.h:636
gex_ep_index_t tgt_ep_index
Definition gasnetex_internal.h:642
size_t payload_bytes
Definition gasnetex_internal.h:648
size_t hdr_size
Definition gasnetex_internal.h:645
gex_ep_index_t src_ep_index
Definition gasnetex_internal.h:642
uintptr_t tgt_ptr
Definition gasnetex_internal.h:647
gex_rank_t srcrank
Definition gasnetex_internal.h:641
friend class ChunkedRecycler
Definition gasnetex_internal.h:634
PendingReverseGet * next_rget
Definition gasnetex_internal.h:640
uintptr_t src_ptr
Definition gasnetex_internal.h:647
Definition gasnetex_internal.h:284
gex_ep_index_t source_ep_index
Definition gasnetex_internal.h:311
PreparedMessage()
Definition gasnetex_internal.h:289
PendingPutHeader * put
Definition gasnetex_internal.h:319
OutbufMetadata * databuf
Definition gasnetex_internal.h:315
unsigned short msgid
Definition gasnetex_internal.h:312
void * temp_buffer
Definition gasnetex_internal.h:314
gex_am_src_desc_opaque_t srcdesc
Definition gasnetex_internal.h:318
Strategy
Definition gasnetex_internal.h:294
@ STRAT_MEDIUM_IMMEDIATE
Definition gasnetex_internal.h:298
@ STRAT_RGET_IMMEDIATE
Definition gasnetex_internal.h:304
@ STRAT_SHORT_PBUF
Definition gasnetex_internal.h:297
@ STRAT_PUT_PBUF
Definition gasnetex_internal.h:307
@ STRAT_LONG_IMMEDIATE
Definition gasnetex_internal.h:302
@ STRAT_LONG_PBUF
Definition gasnetex_internal.h:303
@ STRAT_SHORT_IMMEDIATE
Definition gasnetex_internal.h:296
@ STRAT_MEDIUM_PREP
Definition gasnetex_internal.h:301
@ STRAT_RGET_PBUF
Definition gasnetex_internal.h:305
@ STRAT_MEDIUM_MALLOCSRC
Definition gasnetex_internal.h:300
@ STRAT_MEDIUM_PBUF
Definition gasnetex_internal.h:299
@ STRAT_PUT_IMMEDIATE
Definition gasnetex_internal.h:306
@ STRAT_UNKNOWN
Definition gasnetex_internal.h:295
gex_ep_index_t target_ep_index
Definition gasnetex_internal.h:311
OutbufMetadata * pktbuf
Definition gasnetex_internal.h:316
Strategy strategy
Definition gasnetex_internal.h:309
uintptr_t dest_payload_addr
Definition gasnetex_internal.h:313
friend class ChunkedRecycler
Definition gasnetex_internal.h:288
int pktidx
Definition gasnetex_internal.h:317
~PreparedMessage()
Definition gasnetex_internal.h:290
gex_rank_t target
Definition gasnetex_internal.h:310
Definition gasnetex_internal.h:394
struct Realm::XmitSrcDestPair::LongRgetData::@231161274335142047267100147034003307003325101125::@215254133132341044306046217012041354200066360003 r
size_t payload_bytes
Definition gasnetex_internal.h:396
OutbufMetadata * databuf
Definition gasnetex_internal.h:400
const void * payload_base
Definition gasnetex_internal.h:395
struct Realm::XmitSrcDestPair::LongRgetData::@231161274335142047267100147034003307003325101125::@256270171352223205165123307265245130304071307362 l
gex_ep_index_t src_ep_index
Definition gasnetex_internal.h:404
gex_ep_index_t tgt_ep_index
Definition gasnetex_internal.h:404
uintptr_t dest_addr
Definition gasnetex_internal.h:397
Definition gasnetex_internal.h:409
const void * src_addr
Definition gasnetex_internal.h:410
uintptr_t dest_addr
Definition gasnetex_internal.h:411
size_t payload_bytes
Definition gasnetex_internal.h:412
PendingPutHeader * put
Definition gasnetex_internal.h:413
unsigned short msgid
Definition ucp_internal.h:2