Realm
A distributed, event-based tasking library
Loading...
Searching...
No Matches
machine_impl.h
Go to the documentation of this file.
1/*
2 * Copyright 2025 Stanford University, NVIDIA Corporation
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18// implementation for Realm Machine class
19
20#ifndef REALM_MACHINE_IMPL_H
21#define REALM_MACHINE_IMPL_H
22
23#include "realm/machine.h"
24#include "realm/network.h"
25#include "realm/mutex.h"
26#include "realm/atomics.h"
27
28#include <vector>
29#include <set>
30#include <optional>
31#include <memory>
32
33namespace Realm {
34
35 template <typename KT, typename AT>
39 bool add_affinity(KT key, const AT &aff, bool is_local);
40
41 std::map<KT, AT *> all;
42 std::map<KT, AT *> local;
43 std::map<KT, AT *> best;
44 };
45
55
68
70 MachineNodeInfo(int _node, RuntimeImpl *_runtime_impl);
76 bool add_process_info(const Machine::ProcessInfo &proc_info);
77
78 void update_kind_maps(void);
79
80 int node;
81
82 // The runtime_impl is not const because we calls get_processor_impl(), which is not
83 // const but the runtime_impl should not be modified in this class
85
87
88 std::map<Processor, MachineProcInfo *> procs;
89 std::map<Processor::Kind, std::map<Processor, MachineProcInfo *>> proc_by_kind;
90
91 std::map<Memory, MachineMemInfo *> mems;
92 std::map<Memory::Kind, std::map<Memory, MachineMemInfo *>> mem_by_kind;
93 };
94
96 public:
97 MachineImpl(RuntimeImpl *_runtime_impl);
99
101
102 void get_all_memories(std::set<Memory> &mset) const;
103 void get_all_processors(std::set<Processor> &pset) const;
104
105 void get_local_processors(std::set<Processor> &pset) const;
106 void get_local_processors_by_kind(std::set<Processor> &pset,
107 Processor::Kind kind) const;
108
109 // Return the set of memories visible from a processor
110 void get_visible_memories(Processor p, std::set<Memory> &mset, bool local_only) const;
111
112 // Return the set of memories visible from a memory
113 void get_visible_memories(Memory m, std::set<Memory> &mset, bool local_only) const;
114
115 // Return the set of processors which can all see a given memory
116 void get_shared_processors(Memory m, std::set<Processor> &pset,
117 bool local_only) const;
118
120
122 bool has_affinity(Memory m1, Memory m2, Machine::AffinityDetails *details = 0) const;
123
124 int get_proc_mem_affinity(std::vector<Machine::ProcessorMemoryAffinity> &result,
125 Processor restrict_proc = Processor::NO_PROC,
126 Memory restrict_memory = Memory::NO_MEMORY,
127 bool local_only = true) const;
128
129 int get_mem_mem_affinity(std::vector<Machine::MemoryMemoryAffinity> &result,
130 Memory restrict_mem1 = Memory::NO_MEMORY,
131 Memory restrict_mem2 = Memory::NO_MEMORY,
132 bool local_only = true) const;
133
134 void parse_node_announce_data(int node_id, const void *args, size_t arglen,
135 bool remote);
136
138 bool lock_held = false);
139
142
144
146
147 void add_process_info(int node_id, const Machine::ProcessInfo &process_info,
148 bool lock_held = false);
149
150 // guards nodeinfos and everything reachable from it. Readers: the
151 // accessors above and every machine walk in the query implementations
152 // below (which may not call the locking accessors while they hold it -
153 // the lock is not recursive - and use the *_unlocked variants instead).
154 // Writers: node announcements, affinity updates and the kind map
155 // rebuild. Reads vastly outnumber writes (which in practice all happen
156 // at startup), so this is a striped lock: readers never touch a shared
157 // cache line, and a writer is not starved by a steady stream of them.
159 std::vector<Machine::ProcessorMemoryAffinity> proc_mem_affinities;
160 std::set<Machine::MachineUpdateSubscriber *> subscribers;
161
162 std::map<int, MachineNodeInfo *> nodeinfos;
163
164 protected:
165 friend class ProcessorQueryImpl;
166 friend class MemoryQueryImpl;
167
168 // the bodies of the public accessors, for callers already holding 'mutex'
169 int
170 get_proc_mem_affinity_unlocked(std::vector<Machine::ProcessorMemoryAffinity> &result,
171 Processor restrict_proc, Memory restrict_memory,
172 bool local_only) const;
173 int get_mem_mem_affinity_unlocked(std::vector<Machine::MemoryMemoryAffinity> &result,
174 Memory restrict_mem1, Memory restrict_mem2,
175 bool local_only) const;
176
181
183 };
184
185 template <typename T, typename T2>
187 public:
188 virtual ~QueryPredicate(void){};
189
190 virtual QueryPredicate<T, T2> *clone(void) const = 0;
191
192 virtual bool matches_predicate(const MachineImpl *machine, T thing,
193 const T2 *info = 0) const = 0;
194 };
195
197
199 public:
200 ProcessorHasAffinityPredicate(Memory _memory, unsigned _min_bandwidth,
201 unsigned _max_latency);
202
203 virtual ProcQueryPredicate *clone(void) const;
204
205 virtual bool matches_predicate(const MachineImpl *machine, Processor thing,
206 const MachineProcInfo *info = 0) const;
207
208 protected:
211 unsigned max_latency;
212 };
213
214 namespace Config {
215 extern bool use_machine_query_cache;
216 };
217
224
225 // the number of values of Processor::Kind and Memory::Kind, for the arrays
226 // indexed by kind below
227#define REALM_COUNT_KIND(name, desc) +1
228 static constexpr unsigned NUM_PROCESSOR_KINDS =
230 static constexpr unsigned NUM_MEMORY_KINDS = 0 REALM_MEMORY_KINDS(REALM_COUNT_KIND);
231#undef REALM_COUNT_KIND
232
233 // The machine query cache.
234 //
235 // A query result is an immutable, sorted list stamped with the generation
236 // of the cache it was built for (the generation is bumped by every machine
237 // update). Lists live in snapshots: a snapshot holds the lists for one
238 // generation and is itself immutable once published. Readers take the
239 // current snapshot with one atomic load, so a query that the cache can
240 // serve writes no shared memory at all; this is what lets concurrent
241 // queries scale. A query remembers the list it last used (see
242 // ProcessorQueryImpl::attached), which is still valid while its generation
243 // is current, and each thread remembers where its last step through a list
244 // stopped, so iterating a cached list in order costs a few comparisons per
245 // step.
246 //
247 // A replaced snapshot is retired, not freed, until cleanup_query_caches()
248 // at runtime shutdown, because a query may still hold a pointer into it
249 // (attached_list() dereferences the pointer to compare generations). So
250 // every machine update that is followed by a query retains one snapshot of
251 // each kind, each a copy of the machine's lists. That is bounded today
252 // because the machine model only changes at startup. Snapshot builds are
253 // single-flight (QueryCacheSlot::get_or_build): an update costs one build
254 // per snapshot kind, not one per querying thread.
255 //
256 // Before the machine model may change at run time (elasticity), retired
257 // snapshots need reclaiming. What would have to change: a query would keep
258 // the generation of its attached list beside the pointer (one 64-bit atomic
259 // holding the generation and an index into a table of per-generation
260 // snapshot headers, say), so that staleness is decided without touching
261 // the snapshot; then only a call in progress can be inside a retired
262 // snapshot, and since calls are short and never block while reading one,
263 // an epoch or quiescent-state scheme (readers announce the generation they
264 // entered; a snapshot is freed once every reader has moved past it) can
265 // free retired snapshots safely. Until then the retention is by design.
266 template <typename T>
268 unsigned generation = 0;
269 std::vector<T> items; // sorted by id
270 };
271
272 // the lists of all things of each kind
273 template <typename T, unsigned NUM_KINDS>
275 unsigned generation = 0;
277 };
278
279 // the publication slot for one kind of snapshot
280 template <typename S>
282 public:
285 , builds(0)
286 {}
287 // the snapshots are freed by cleanup_query_caches() at runtime shutdown,
288 // not by this object's destructor: a process that exits without shutting
289 // Realm down may still have threads querying, and a hit on a leaked
290 // snapshot is harmless where a freed one is not. (The slot's own members
291 // are destroyed with the other statics at exit, like every Realm static;
292 // a thread still building a snapshot at that point is an application
293 // bug, as it was before.)
294
295 // the published snapshot if it is for 'generation', else null
296 const S *get(unsigned generation) const
297 {
298 const S *s = current.load_acquire();
299 return (s && (s->generation == generation)) ? s : nullptr;
300 }
301
302 // the snapshot for 'generation', built by 'build(generation)' if it is
303 // missing. Builders are serialized: after a machine update the first
304 // thread to miss builds, and the others wait for its snapshot instead of
305 // each building their own, so an update costs one build however many
306 // threads query. Returns null if the live generation has moved on from
307 // 'generation' (the caller then walks the machine), whether before the
308 // build started or while it ran. A build is attempted only for the live
309 // generation and generations only move forward, so each generation is
310 // built at most once. Lock order: build_mutex, then the machine's reader
311 // lock inside 'build', then 'mutex' in install(); machine writers take
312 // neither of this class's locks
313 template <typename BuildFn>
314 const S *get_or_build(unsigned generation, const atomic<unsigned int> &live,
315 BuildFn &&build)
316 {
318 const S *s = get(generation);
319 if(s)
320 return s; // built while we waited
321 if(live.load() != generation)
322 return nullptr;
323 builds.fetch_add(1);
324 S *built = build(generation);
325 // the live generation is read after the build (call arguments are not
326 // sequenced), so a change during the build is seen by install()
327 return install(built, live.load());
328 }
329
330 // publishes 'built' (taking ownership) and returns it - unless a snapshot
331 // for the same generation got there first, which is returned instead, or
332 // the cache generation has moved on from 'built', in which case null is
333 // returned (and the caller walks the machine)
334 const S *install(S *built, unsigned live_generation);
335
336 // frees every snapshot ever published (runtime shutdown)
337 void cleanup(void);
338
339 // how many snapshots have been built (for tests)
340 unsigned long build_count(void) const { return builds.load(); }
341
342 protected:
344 Mutex mutex; // serializes install and cleanup
345 std::vector<const S *> retired;
346 UnfairMutex build_mutex; // serializes builders (see get_or_build)
348 };
349
351 public:
352 ProcessorQueryImpl(const Machine &_machine);
353 ProcessorQueryImpl(const MachineImpl *_machine_impl);
354
357
358 // the processors of each kind with an affinity to each memory - any
359 // affinity in the model, local or not, exactly as the walk sees them. A
360 // kind and memory that are not in the map have no such processors (and
361 // are served the empty list), so a miss costs no more than a hit.
363 unsigned generation = 0;
364 std::map<Memory, CachedList> by_kind_and_mem[NUM_PROCESSOR_KINDS];
366 };
367
368 // the shared query cache, rebuilt lazily after the machine changes
372
373 protected:
374 // these things are refcounted and copied-on-write
377
378 public:
379 void add_reference(void);
381 // makes and returns if a copy if more than one reference is held
383
384 void restrict_to_node(int new_node_id);
387
390 size_t count_matches(void) const;
392
394 {
395 cached_mem = m;
396 if(predicates.size() == 1)
397 is_cached_mem = true;
398 else
399 is_cached_mem = false;
400 };
402 {
404 is_cached_mem = false;
405 };
407
408 public:
409 // Best affinity cost function structure
415
416 // Friend declarations for Machine::ProcessorQuery methods
419 int latency_weight);
420
421 protected:
428 std::vector<ProcQueryPredicate *> predicates;
429
430 // Best affinity cost function (not a predicate)
431 std::optional<BestAffinityCostFn> best_affinity_cost;
432
435
436 // the list this query last used: one in a shared snapshot, or one of
437 // the best-affinity lists it built itself. It is current while its
438 // generation matches cache_invalid_count. Copies of a query share this
439 // object and may read concurrently, so this is an atomic; the lists it
440 // points to are immutable and outlive it (see QueryCacheSlot)
442 // best-affinity results depend on the query's cost function, so they are
443 // built per query. Across machine updates every list is kept, because
444 // another copy may still be reading an older one. Refinement holds the
445 // only reference to the implementation and frees all its lists; otherwise
446 // they live until the query is destroyed. Between refinements a query
447 // builds one list per machine update it observes. The first list lives
448 // in the query itself, the rest on the heap; the optional is constructed
449 // only when a list is built, so a query that never needs one pays nothing
450 // for it
452 mutable std::optional<CachedList> first_own;
453 mutable std::vector<std::unique_ptr<CachedList>> own_lists;
454
455 // whether the shared cache can serve this query at all: one kind, at most
456 // one node, no best-affinity cost, and no predicate other than the one
457 // memory affinity the by-kind-and-memory lists cover; anything else walks
458 // the machine
460 {
462 !best_affinity_cost.has_value() && (predicates.empty() || is_cached_mem) &&
463 (unsigned(restricted_kind) < NUM_PROCESSOR_KINDS);
464 }
465 // the shared list that serves this query (attaching to it as a side
466 // effect), or null if the shared cache does not cover this query. The
467 // attached-list check (the hit path, which every cached call takes) and
468 // the coverage test (which every call of an uncovered query takes) are
469 // defined here so that they are inlined into the callers; the snapshot
470 // lookup, taken once per query per machine update, is out of line
471 const CachedList *cached_list() const
472 {
473 const CachedList *list = attached_list();
474 if(list)
475 return list;
476 return shared_cache_covers() ? cached_list_lookup() : nullptr;
477 }
478 // the rest of cached_list(), for a query the shared cache covers: its list
479 // in the current snapshot, attached
480 REALM_ATTR_NOINLINE(const CachedList *cached_list_lookup() const);
481 // the miss path of cached_list_lookup(): the snapshot for 'generation' is
482 // built (or waited for) and the query attached to its list
483 REALM_ATTR_NOINLINE(const CachedList *cached_list_miss(unsigned generation) const);
484 // the list for this query's kind and memory in an affinity snapshot
485 const CachedList *list_in(const AffinityCache *ac) const;
486 // builders of the shared snapshots for 'generation', run one at a time
487 // by QueryCacheSlot::get_or_build
488 KindCache *build_kind_cache(unsigned generation) const;
489 AffinityCache *build_affinity_cache(unsigned generation) const;
490 // the list of this query's best-affinity results, building it if needed
492 // next() over that list (see the definition)
493 REALM_ATTR_NOINLINE(Processor best_affinity_next(Processor after) const);
494 // builds the best-affinity results of this query into 'result'
495 void build_best_affinity_list(std::vector<Processor> &result) const;
496 // remembers 'list' as this query's current result
497 const CachedList *attach(const CachedList *list) const;
498 // this query's attached list if it is still current, else null
500 {
501 const CachedList *list = attached.load_acquire();
502 return (list && (list->generation == cache_invalid_count.load())) ? list : nullptr;
503 }
504 // drops the attached list - every change to the query's constraints does this
507 bool cached_query(Processor &pval, QueryType q) const;
508 bool cached_query(size_t &count) const;
509 };
510
512
514 public:
515 MemoryHasProcAffinityPredicate(Processor _proc, unsigned _min_bandwidth,
516 unsigned _max_latency);
517
518 virtual MemoryQueryPredicate *clone(void) const;
519
520 virtual bool matches_predicate(const MachineImpl *machine, Memory thing,
521 const MachineMemInfo *info = 0) const;
522
523 protected:
526 unsigned max_latency;
527 };
528
530 public:
531 MemoryHasMemAffinityPredicate(Memory _memory, unsigned _min_bandwidth,
532 unsigned _max_latency);
533
534 virtual MemoryQueryPredicate *clone(void) const;
535
536 virtual bool matches_predicate(const MachineImpl *machine, Memory thing,
537 const MachineMemInfo *info = 0) const;
538
539 protected:
542 unsigned max_latency;
543 };
544
546 public:
547 MemoryQueryImpl(const Machine &_machine);
548 MemoryQueryImpl(const MachineImpl *_machine_impl);
549
550 // see ProcessorQueryImpl
553
554 // the shared query cache: memories by kind (see ProcessorQueryImpl)
557
558 protected:
559 // these things are refcounted and copied-on-write
562
563 public:
564 void add_reference(void);
566 // makes and returns if a copy if more than one reference is held
568
569 void restrict_to_node(int new_node_id);
571 void restrict_by_capacity(size_t new_min_bytes);
573 // records the processor of a has_affinity_to(Processor) predicate when it
574 // is the query's only predicate, so matches can be enumerated from that
575 // processor's affinity map instead of every memory in the machine
577
578 Memory first_match(void) const;
580 size_t count_matches(void) const;
581 Memory random_match(void) const;
583 bool cached_query(Memory p, Memory &pval);
584 bool cached_query(Memory &pval, QueryType q) const;
585 bool cached_query(size_t &count) const;
586
587 public:
588 // Best affinity cost function structures
599
600 // Friend declarations for Machine::MemoryQuery methods
601 friend Machine::MemoryQuery &
603 int latency_weight);
604 friend Machine::MemoryQuery &
606 int latency_weight);
607
608 protected:
616 // see ProcessorQueryImpl
619 mutable std::optional<CachedList> first_own;
620 mutable std::vector<std::unique_ptr<CachedList>> own_lists;
621 std::vector<MemoryQueryPredicate *> predicates;
622 Processor affinity_proc; // see set_affinity_proc
623
624 // Best affinity cost functions (not predicates)
625 std::optional<BestProcAffinityCostFn> best_proc_affinity_cost;
626 std::optional<BestMemAffinityCostFn> best_mem_affinity_cost;
627
628 // see ProcessorQueryImpl
630 {
632 predicates.empty() && (restricted_min_capacity == 0) &&
633 !best_proc_affinity_cost.has_value() &&
634 !best_mem_affinity_cost.has_value() &&
635 (unsigned(restricted_kind) < NUM_MEMORY_KINDS);
636 }
637 const CachedList *cached_list() const
638 {
639 const CachedList *list = attached_list();
640 if(list)
641 return list;
642 return shared_cache_covers() ? cached_list_lookup() : nullptr;
643 }
644 REALM_ATTR_NOINLINE(const CachedList *cached_list_lookup() const);
645 REALM_ATTR_NOINLINE(const CachedList *cached_list_miss(unsigned generation) const);
646 KindCache *build_kind_cache(unsigned generation) const;
648 REALM_ATTR_NOINLINE(Memory best_affinity_next(Memory after) const);
649 void build_best_affinity_list(std::vector<Memory> &result) const;
650 const CachedList *attach(const CachedList *list) const;
652 {
653 const CachedList *list = attached.load_acquire();
654 return (list && (list->generation == cache_invalid_count.load())) ? list : nullptr;
655 }
657 // the affinity map of a processor the query is anchored to, or NULL
658 const std::map<Memory, Machine::ProcessorMemoryAffinity *> *
659 anchor_affinities(Processor p, bool local_only) const;
660 // the same-node outgoing affinity map of a memory, or NULL if it is not in
661 // the machine model (which "no memory" never is)
662 const std::map<Memory, Machine::MemoryMemoryAffinity *> *
664 // applies the node, kind, capacity and predicate restrictions to one memory
666 };
667
669 inline MachineImpl *get_machine(void) { return machine_singleton; }
670 extern void cleanup_query_caches();
671 // active messages
672
683
684}; // namespace Realm
685
686 // include "machine_impl.inl"
687
688#endif // ifndef REALM_MACHINE_IMPL_H
Definition mutex.h:384
Definition machine_impl.h:95
friend class MemoryQueryImpl
Definition machine_impl.h:166
friend class ProcessorQueryImpl
Definition machine_impl.h:165
void get_shared_processors(Memory m, std::set< Processor > &pset, bool local_only) const
bool has_affinity(Memory m1, Memory m2, Machine::AffinityDetails *details=0) const
StripedRWLock mutex
Definition machine_impl.h:158
MachineNodeInfo * get_nodeinfo(Memory m) const
MachineImpl(RuntimeImpl *_runtime_impl)
void parse_node_announce_data(int node_id, const void *args, size_t arglen, bool remote)
int get_mem_mem_affinity_unlocked(std::vector< Machine::MemoryMemoryAffinity > &result, Memory restrict_mem1, Memory restrict_mem2, bool local_only) const
RuntimeImpl * get_runtime_impl(void) const
void add_process_info(int node_id, const Machine::ProcessInfo &process_info, bool lock_held=false)
bool has_affinity(Processor p, Memory m, Machine::AffinityDetails *details=0) const
RuntimeImpl * runtime_impl
Definition machine_impl.h:182
MachineNodeInfo * get_nodeinfo(int node) const
int get_proc_mem_affinity_unlocked(std::vector< Machine::ProcessorMemoryAffinity > &result, Processor restrict_proc, Memory restrict_memory, bool local_only) const
void get_local_processors(std::set< Processor > &pset) const
void enumerate_mem_mem_affinities(void)
void remove_subscription(Machine::MachineUpdateSubscriber *subscriber)
std::map< int, MachineNodeInfo * > nodeinfos
Definition machine_impl.h:162
int get_proc_mem_affinity(std::vector< Machine::ProcessorMemoryAffinity > &result, Processor restrict_proc=Processor::NO_PROC, Memory restrict_memory=Memory::NO_MEMORY, bool local_only=true) const
void add_proc_mem_affinity(const Machine::ProcessorMemoryAffinity &pma, bool lock_held=false)
int get_mem_mem_affinity(std::vector< Machine::MemoryMemoryAffinity > &result, Memory restrict_mem1=Memory::NO_MEMORY, Memory restrict_mem2=Memory::NO_MEMORY, bool local_only=true) const
std::vector< Machine::ProcessorMemoryAffinity > proc_mem_affinities
Definition machine_impl.h:159
std::set< Machine::MachineUpdateSubscriber * > subscribers
Definition machine_impl.h:160
void get_all_memories(std::set< Memory > &mset) const
MachineNodeInfo * get_nodeinfo(Processor p) const
void invalidate_query_caches()
void get_local_processors_by_kind(std::set< Processor > &pset, Processor::Kind kind) const
void get_all_processors(std::set< Processor > &pset) const
void get_visible_memories(Processor p, std::set< Memory > &mset, bool local_only) const
bool get_process_info(Processor p, Machine::ProcessInfo *info) const
void add_subscription(Machine::MachineUpdateSubscriber *subscriber)
void update_kind_maps(void)
void get_visible_memories(Memory m, std::set< Memory > &mset, bool local_only) const
Definition machine.h:219
MemoryQuery & best_affinity_to(Processor p, int bandwidth_weight=1, int latency_weight=0)
Definition machine.h:159
ProcessorQuery & best_affinity_to(Memory m, int bandwidth_weight=1, int latency_weight=0)
Definition machine.h:32
realm_affinity_details_t AffinityDetails
Definition machine.h:55
Memory memory
Definition machine_impl.h:540
virtual bool matches_predicate(const MachineImpl *machine, Memory thing, const MachineMemInfo *info=0) const
MemoryHasMemAffinityPredicate(Memory _memory, unsigned _min_bandwidth, unsigned _max_latency)
virtual MemoryQueryPredicate * clone(void) const
unsigned max_latency
Definition machine_impl.h:542
unsigned min_bandwidth
Definition machine_impl.h:541
unsigned min_bandwidth
Definition machine_impl.h:525
MemoryHasProcAffinityPredicate(Processor _proc, unsigned _min_bandwidth, unsigned _max_latency)
Processor proc
Definition machine_impl.h:524
virtual bool matches_predicate(const MachineImpl *machine, Memory thing, const MachineMemInfo *info=0) const
unsigned max_latency
Definition machine_impl.h:526
virtual MemoryQueryPredicate * clone(void) const
bool is_restricted_node
Definition machine_impl.h:611
void restrict_by_capacity(size_t new_min_bytes)
Memory first_match(void) const
const CachedList * attach(const CachedList *list) const
std::vector< std::unique_ptr< CachedList > > own_lists
Definition machine_impl.h:620
REALM_ATTR_NOINLINE(const CachedList *cached_list_lookup() const)
atomic< const CachedList * > attached
Definition machine_impl.h:617
const CachedList * best_affinity_list() const
MemoryQueryImpl(const Machine &_machine)
const std::map< Memory, Machine::MemoryMemoryAffinity * > * local_mem_affinities(Memory m) const
bool cached_query(size_t &count) const
void add_predicate(MemoryQueryPredicate *pred)
int restricted_node_id
Definition machine_impl.h:612
bool shared_cache_covers() const
Definition machine_impl.h:629
Memory random_match(void) const
REALM_ATTR_NOINLINE(const CachedList *cached_list_miss(unsigned generation) const)
KindCache * build_kind_cache(unsigned generation) const
bool is_restricted_kind
Definition machine_impl.h:613
atomic< int > references
Definition machine_impl.h:609
void remove_reference(void)
size_t restricted_min_capacity
Definition machine_impl.h:615
std::vector< MemoryQueryPredicate * > predicates
Definition machine_impl.h:621
std::optional< BestMemAffinityCostFn > best_mem_affinity_cost
Definition machine_impl.h:626
void set_affinity_proc(Processor p)
Processor affinity_proc
Definition machine_impl.h:622
void restrict_to_kind(Memory::Kind new_kind)
bool cached_query(Memory p, Memory &pval)
void restrict_to_node(int new_node_id)
const std::map< Memory, Machine::ProcessorMemoryAffinity * > * anchor_affinities(Processor p, bool local_only) const
Memory cache_next(Memory after)
std::optional< CachedList > first_own
Definition machine_impl.h:619
static atomic< unsigned int > cache_invalid_count
Definition machine_impl.h:555
const MachineImpl * machine
Definition machine_impl.h:610
QueryKindCache< Memory, NUM_MEMORY_KINDS > KindCache
Definition machine_impl.h:552
MemoryQueryImpl(const MemoryQueryImpl &copy_from)
bool candidate_matches(Memory m) const
REALM_ATTR_NOINLINE(Memory best_affinity_next(Memory after) const)
const CachedList * cached_list() const
Definition machine_impl.h:637
std::optional< BestProcAffinityCostFn > best_proc_affinity_cost
Definition machine_impl.h:625
Memory next_match(Memory after) const
Memory::Kind restricted_kind
Definition machine_impl.h:614
static QueryCacheSlot< KindCache > kind_cache
Definition machine_impl.h:556
MemoryQueryImpl(const MachineImpl *_machine_impl)
bool cached_query(Memory &pval, QueryType q) const
QueryResultList< Memory > CachedList
Definition machine_impl.h:551
const CachedList * attached_list() const
Definition machine_impl.h:651
UnfairMutex build_mutex
Definition machine_impl.h:618
size_t count_matches(void) const
void build_best_affinity_list(std::vector< Memory > &result) const
MemoryQueryImpl * writeable_reference(void)
Definition memory.h:33
Kind
Definition memory.h:59
static const Memory NO_MEMORY
Definition memory.h:49
virtual bool matches_predicate(const MachineImpl *machine, Processor thing, const MachineProcInfo *info=0) const
ProcessorHasAffinityPredicate(Memory _memory, unsigned _min_bandwidth, unsigned _max_latency)
virtual ProcQueryPredicate * clone(void) const
unsigned min_bandwidth
Definition machine_impl.h:210
Memory memory
Definition machine_impl.h:209
unsigned max_latency
Definition machine_impl.h:211
int restricted_node_id
Definition machine_impl.h:425
bool is_restricted_kind
Definition machine_impl.h:426
QueryResultList< Processor > CachedList
Definition machine_impl.h:355
void set_cached_mem(Memory m)
Definition machine_impl.h:393
const CachedList * best_affinity_list() const
ProcessorQueryImpl(const MachineImpl *_machine_impl)
ProcessorQueryImpl(const ProcessorQueryImpl &copy_from)
Processor next_match(Processor after)
std::optional< BestAffinityCostFn > best_affinity_cost
Definition machine_impl.h:431
Processor first_match(void) const
void restrict_to_node(int new_node_id)
UnfairMutex build_mutex
Definition machine_impl.h:451
std::vector< ProcQueryPredicate * > predicates
Definition machine_impl.h:428
QueryKindCache< Processor, NUM_PROCESSOR_KINDS > KindCache
Definition machine_impl.h:356
const CachedList * cached_list() const
Definition machine_impl.h:471
const CachedList * list_in(const AffinityCache *ac) const
REALM_ATTR_NOINLINE(const CachedList *cached_list_lookup() const)
ProcessorQueryImpl * writeable_reference(void)
Processor cache_next(Processor after)
size_t count_matches(void) const
AffinityCache * build_affinity_cache(unsigned generation) const
void reset_cached_mem()
Definition machine_impl.h:401
KindCache * build_kind_cache(unsigned generation) const
void restrict_to_kind(Processor::Kind new_kind)
Processor::Kind restricted_kind
Definition machine_impl.h:427
static QueryCacheSlot< KindCache > kind_cache
Definition machine_impl.h:370
Memory cached_mem
Definition machine_impl.h:433
atomic< int > references
Definition machine_impl.h:422
REALM_ATTR_NOINLINE(const CachedList *cached_list_miss(unsigned generation) const)
bool cached_query(size_t &count) const
Processor random_match(void) const
const MachineImpl * machine
Definition machine_impl.h:423
REALM_ATTR_NOINLINE(Processor best_affinity_next(Processor after) const)
const CachedList * attach(const CachedList *list) const
void build_best_affinity_list(std::vector< Processor > &result) const
std::optional< CachedList > first_own
Definition machine_impl.h:452
bool is_restricted_node
Definition machine_impl.h:424
bool shared_cache_covers() const
Definition machine_impl.h:459
const CachedList * attached_list() const
Definition machine_impl.h:499
atomic< const CachedList * > attached
Definition machine_impl.h:441
ProcessorQueryImpl(const Machine &_machine)
void add_predicate(ProcQueryPredicate *pred)
bool is_cached_mem
Definition machine_impl.h:434
static atomic< unsigned int > cache_invalid_count
Definition machine_impl.h:369
bool cached_query(Processor &pval, QueryType q) const
std::vector< std::unique_ptr< CachedList > > own_lists
Definition machine_impl.h:453
static QueryCacheSlot< AffinityCache > affinity_cache
Definition machine_impl.h:371
bool cached_query(Processor p, Processor &pval)
Definition processor.h:37
Kind
Definition processor.h:65
static const Processor NO_PROC
Definition processor.h:54
Definition machine_impl.h:281
const S * install(S *built, unsigned live_generation)
Mutex mutex
Definition machine_impl.h:344
QueryCacheSlot(void)
Definition machine_impl.h:283
std::vector< const S * > retired
Definition machine_impl.h:345
atomic< const S * > current
Definition machine_impl.h:343
atomic< unsigned long > builds
Definition machine_impl.h:347
unsigned long build_count(void) const
Definition machine_impl.h:340
const S * get(unsigned generation) const
Definition machine_impl.h:296
UnfairMutex build_mutex
Definition machine_impl.h:346
const S * get_or_build(unsigned generation, const atomic< unsigned int > &live, BuildFn &&build)
Definition machine_impl.h:314
Definition machine_impl.h:186
virtual ~QueryPredicate(void)
Definition machine_impl.h:188
virtual QueryPredicate< T, T2 > * clone(void) const =0
virtual bool matches_predicate(const MachineImpl *machine, T thing, const T2 *info=0) const =0
Definition runtime_impl.h:271
Definition mutex.h:474
Definition mutex.h:223
Definition atomics.h:31
T load(void) const
#define REALM_COUNT_KIND(name, desc)
Definition machine_impl.h:227
Definition activemsg.h:44
bool use_machine_query_cache
Definition activemsg.h:42
QueryType
Definition machine_impl.h:219
@ QUERY_FIRST
Definition machine_impl.h:221
@ QUERY_RANDOM
Definition machine_impl.h:222
@ QUERY_NEXT
Definition machine_impl.h:220
UnfairMutex Mutex
Definition mutex.h:68
QueryPredicate< Processor, MachineProcInfo > ProcQueryPredicate
Definition machine_impl.h:196
MachineImpl * get_machine(void)
Definition machine_impl.h:669
NodeAnnounceTag
Definition machine_impl.h:674
@ NODE_ANNOUNCE_IB_MEM
Definition machine_impl.h:678
@ NODE_ANNOUNCE_PROCESS_INFO
Definition machine_impl.h:681
@ NODE_ANNOUNCE_MEM
Definition machine_impl.h:677
@ NODE_ANNOUNCE_DMA_CHANNEL
Definition machine_impl.h:680
@ NODE_ANNOUNCE_PROC
Definition machine_impl.h:676
@ NODE_ANNOUNCE_PMA
Definition machine_impl.h:679
@ NODE_ANNOUNCE_INVALID
Definition machine_impl.h:675
MachineImpl * machine_singleton
void cleanup_query_caches()
QueryPredicate< Memory, MachineMemInfo > MemoryQueryPredicate
Definition machine_impl.h:511
#define REALM_MEMORY_KINDS(__op__)
Definition realm_c.h:248
#define REALM_PROCESSOR_KINDS(__op__)
Definition realm_c.h:227
Definition machine_impl.h:36
std::map< KT, AT * > all
Definition machine_impl.h:41
std::map< KT, AT * > local
Definition machine_impl.h:42
std::map< KT, AT * > best
Definition machine_impl.h:43
bool add_affinity(KT key, const AT &aff, bool is_local)
Definition machine_impl.h:56
Memory m
Definition machine_impl.h:63
bool add_proc_mem_affinity(const Machine::ProcessorMemoryAffinity &pma)
MachineAffinityInfo< Memory, Machine::MemoryMemoryAffinity > mmas_out
Definition machine_impl.h:65
MachineAffinityInfo< Processor, Machine::ProcessorMemoryAffinity > pmas
Definition machine_impl.h:64
bool add_mem_mem_affinity(const Machine::MemoryMemoryAffinity &mma)
MachineMemInfo(Memory _m)
MachineAffinityInfo< Memory, Machine::MemoryMemoryAffinity > mmas_in
Definition machine_impl.h:66
Definition machine_impl.h:69
std::map< Processor, MachineProcInfo * > procs
Definition machine_impl.h:88
MachineNodeInfo(int _node, RuntimeImpl *_runtime_impl)
std::map< Memory, MachineMemInfo * > mems
Definition machine_impl.h:91
std::map< Processor::Kind, std::map< Processor, MachineProcInfo * > > proc_by_kind
Definition machine_impl.h:89
bool add_proc_mem_affinity(const Machine::ProcessorMemoryAffinity &pma)
bool add_process_info(const Machine::ProcessInfo &proc_info)
void update_kind_maps(void)
Machine::ProcessInfo * process_info
Definition machine_impl.h:86
RuntimeImpl * runtime_impl
Definition machine_impl.h:84
int node
Definition machine_impl.h:80
std::map< Memory::Kind, std::map< Memory, MachineMemInfo * > > mem_by_kind
Definition machine_impl.h:92
bool add_processor(Processor p)
bool add_mem_mem_affinity(const Machine::MemoryMemoryAffinity &mma)
bool add_memory(Memory m)
Definition machine_impl.h:46
MachineAffinityInfo< Memory, Machine::ProcessorMemoryAffinity > pmas
Definition machine_impl.h:53
Processor p
Definition machine_impl.h:52
bool add_proc_mem_affinity(const Machine::ProcessorMemoryAffinity &pma)
MachineProcInfo(Processor _p)
Definition machine.h:106
Definition machine.h:86
int bandwidth_weight
Definition machine_impl.h:596
Memory memory
Definition machine_impl.h:595
int latency_weight
Definition machine_impl.h:597
int latency_weight
Definition machine_impl.h:592
int bandwidth_weight
Definition machine_impl.h:591
Processor proc
Definition machine_impl.h:590
Definition machine_impl.h:362
CachedList none
Definition machine_impl.h:365
std::map< Memory, CachedList > by_kind_and_mem[NUM_PROCESSOR_KINDS]
Definition machine_impl.h:364
unsigned generation
Definition machine_impl.h:363
int bandwidth_weight
Definition machine_impl.h:412
Memory memory
Definition machine_impl.h:411
int latency_weight
Definition machine_impl.h:413
Definition machine_impl.h:274
QueryResultList< Processor > by_kind[NUM_KINDS]
Definition machine_impl.h:276
unsigned generation
Definition machine_impl.h:275
Definition machine_impl.h:267
unsigned generation
Definition machine_impl.h:268
std::vector< Processor > items
Definition machine_impl.h:269