4 #include <System/Mem.h>
6 #include <kern/kernel.h>
9 #include <arch/addrs.h>
11 #include <util/rbtree.h>
12 #include <util/list.h>
13 #include <util/lock.h>
14 #include <kernel/region.h>
15 #include <lowlevel/bitops.h>
18 // Used for allocating memory at boot time before the page allocator
19 // is running. Alignment must be a power of 2. Because nothing other
20 // than the kernel is guaranteed to be mapped from the beginning on
21 // all architectures, no generic code should use this until after
22 // Arch::arch_init() has run and set up physical memory mappings.
24 // This function may not be used after the page allocator has
25 // been initialized by architecture code.
27 // Architectures must provide Arch::next_free_bootmem initalized
28 // to the first free piece of bootmem.
30 static inline void *get_bootmem(size_t size, size_t align)
32 uintptr_t ret = (Arch::next_free_bootmem + align - 1) & ~(align - 1);
33 Arch::next_free_bootmem = ret + size;
34 return reinterpret_cast<void *>(ret);
37 typedef System::Mem::AddrSpace IAddrSpace;
38 typedef System::Mem::Mappable IMappable;
39 using System::Mem::Cacheable;
40 using System::Mem::Region;
41 using System::Mem::RegionWithOffset;
42 using System::Mem::AllocFlags;
43 using System::Mem::MapFlags;
44 using System::Mem::AccessFlags;
48 // This must be kept in sync with include/kern/generic-pte.h
51 // Readable, Writeable, and Executable are for permission only,
52 // not for implementing copy on write, swapping, etc.
62 // If set, then on a write access, the page is copied and this
63 // address space gets the new, anonymous version. The rmap list
64 // is then traversed; all downstream mappings will share the new
67 // For vareas that directly map something other than an address
68 // space, the action to be taken on a write fault is
74 // Do not allow the user to unmap or modify flags.
75 // Used for the shared user/kernel mappings.
79 #elif defined(BITFIELD_BE)
80 ulong pad:_LL_LONG_BYTES * 8 - 9;
91 #error Unspecified/unrecognized bitfield endianness
97 PTEFlags(ulong init) : raw(init)
111 using Arch::kvirt_to_phys;
112 using Arch::phys_to_kvirt;
117 typedef Util::RBTree<VirtualArea, Region, u64> VirtualAreaTree;
121 // This linked list keeps track of the virtual areas that map this
122 // mappable (this is not transitive; vareas that map a varea that
123 // maps this mappable are not on this list).
125 // OPT: rbtree keyed on mapped address range?
128 Lock::SpinLock mappings_lock;
133 virtual void get_size(u64 *size) = 0;
135 virtual void get_block_size(u64 *block_size)
137 *block_size = Arch::page_size;
140 // Register/unregister varea as mapping this mappable.
142 virtual void map(VirtualArea *varea);
143 virtual void unmap(VirtualArea *varea);
145 // Make the specified page available for mapping. This must be
146 // done before map() will succeed. It is possible (though
147 // unlikely) that the pages will be removed before map() is called,
148 // causing map() to return false. In such a case, pagein should be
149 // called again by the fault handler. If the mapping fails for
150 // other reasons (such as lack of permission, a hole in a stacked
151 // aspace, or an I/O error) then pagein() will throw a BadPageFault
154 virtual void pagein(u64 vaddr, PTEFlags reqflags) = 0;
156 // Returns the physical address and flags associated with a given
157 // virtual address. If flags.Valid is not set, then phys and all
158 // other flags are undefined, and pagein() should be retried.
159 // rmap_lock must be held.
161 virtual void get_mapping(u64 vaddr, u64 *phys, PTEFlags *flags) = 0;
163 #include <servers/mem/addrspace/Mem/Mappable.h>
179 // The red/black tree is used to find a region based on address.
181 // The linked list is kept in order and is used to iterate over
182 // vmas in a region (after looking up the starting point in the
183 // tree, unless the region is the entire address space).
185 VirtualAreaTree::Node rbtree_node;
186 Util::List list_node;
187 Util::List mappings_node;
192 // This is added to the virtual address to get the offset
193 // into the mappable.
198 return rbtree_node.value;
203 struct BadPageFault {
206 class ASpaceMappable : public Mappable {
209 static bool rec_pagein(AddrSpace *aspace, u64 vaddr,
213 ASpaceMappable (AddrSpace *ASPACE) : aspace(ASPACE)
218 void get_size(u64 *size);
221 virtual void pagein(u64 vaddr, PTEFlags reqflags);
222 virtual void get_mapping(u64 vaddr, u64 *phys, PTEFlags *flags);
224 friend class AddrSpace;
228 // OPT: Coalesce vareas when possible (except when setting flags to
229 // match surrounding vareas, as the flags are likely to change
230 // again if they've already changed).
232 // OPT: A subclass of AddrSpace that doesn't use
233 // VirtualArea::offset, but rather has its own virtual method that
234 // figures out offsets to the next level using its own data
235 // structures (such as filesystem block tables). This would avoid
236 // excessive vareas for fragmented files. Whether the excess of
237 // vareas is significant enough for this to be worthwhile remains
240 VirtualAreaTree varea_tree;
241 Util::List varea_list;
244 // This defines the start and end of the aspace; mappings outside
245 // this range may not be done, and will not be returned by
246 // get_free_region(). For process aspaces, this goes from
247 // Arch::user_start to Arch::user_end. For non-proc aspaces, this
250 Region aspace_region;
252 // Returns true if there is a mapped region that overlaps the given
253 // region. If there is a collision, then the first overlapping
254 // varea is returned in va. Otherwise, it returns the last mapped
255 // area before the region in va (if there are no areas, or the
256 // region is before the first area, then prev is NULL). The aspace
257 // lock must be held.
259 bool check_overlap(Region region, VirtualArea *&va);
261 // Finds a free region of the requested length and puts it in
262 // region. Returns true if an appropriate area is found. The prev
263 // pointer is as in check_overlap. The aspace lock must be held.
265 bool get_free_region(ulong len, Region ®ion, VirtualArea *&prev);
267 // This is the value after the last region returned by
268 // get_free_region. If there was an intervening unmap for a lower
269 // address, then it is set to that address instead.
271 u64 cached_free_region;
273 static u64 rec_unmap(AddrSpace *aspace, Region region,
274 PTEFlags reqflags, VirtualArea *va);
276 // If there are multiple virtual areas that cover the specified region,
277 // split them at the region's boundaries. The first varea in the region
278 // (if any) is returned. The aspace lock must be held.
280 VirtualArea *split_varea(Region region);
282 void break_copy_on_write(VirtualArea *va, u64 vaddr, u64 phys);
283 bool map(VirtualArea *va, u64 vaddr, PTEFlags reqflags);
289 #include <servers/mem/addrspace/Mem/AddrSpace.h>
291 ASpaceMappable mappable;
292 PageTable *page_table;
294 AddrSpace(PageTable *ptbl = NULL);
296 // Returns true if the fault was "good"; otherwise, the caller
297 // should dump regs. exec should only be used if the CPU
298 // implements per-page exec protection; otherwise, treat it
301 bool handle_fault(ulong addr, bool write, bool exec, bool user);
303 void get_mappable(IMappable *ma);
304 void clone(IAddrSpace *addrspace, u8 clone_is_real);
312 void map(IMappable ma, Region region, u64 *vstart, MapFlags mflags,
313 int map_type = map_user);
314 void unmap(Region region, bool from_kernel = false);
316 void set_mapflags(Region region, MapFlags mflags);
317 void get_mapflags(Region region, MapFlags *mflags, uint8_t *all_same);
318 void get_mapping(Region region, IMappable *ma, u64 *offset);
320 void get_page_size(u32 *page_size);
321 void get_min_align(u32 *min_align);
322 void get_size(u64 *size);
324 friend class ASpaceMappable;
327 extern Factory addr_space_factory, proc_addr_space_factory;
329 using ::System::RunTime::orbmm;
331 static inline bool page_aligned(u64 addr)
333 return !(addr & (u64)(Arch::page_size - 1));
336 static inline u64 page_align(u64 addr)
338 return addr & ~(u64)(Arch::page_size - 1);
341 // FIXME: Valid user addr? Paging holes?
342 static inline bool valid_addr(uint64_t addr)
344 if (sizeof(void *) == 8)
347 return (addr >> 32) == 0;