kernel/mem/addrspace.cc

   1 // mem/addrspace.cc -- System.Mem.AddrSpace
   2 //
   3 // OPT: Special AddrSpaces that only translate/export a linear block of
   4 // another AddrSpace, and don't have individual entries for every page.
   5 //
   6 // OPT: Special VAreas that use their own translation mechanism instead
   7 // of varea->offset, so that filesystem block tables (and similar things)
   8 // don't need to have a VArea per block.
   9 //
  10 // This software is copyright (c) 2006 Scott Wood <scott@buserror.net>.
  11 //
  12 // Permission is hereby granted, free of charge, to any person obtaining a copy of
  13 // this software and associated documentation files (the "Software"), to deal with
  14 // the Software without restriction, including without limitation the rights to
  15 // use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
  16 // of the Software, and to permit persons to whom the Software is furnished to do
  17 // so, subject to the following conditions:
  18 //
  19 //     * Redistributions of source code must retain the above copyright notice,
  20 //       this list of conditions and the following disclaimers.
  21 //
  22 //     * Redistributions in binary form must reproduce the above copyright notice,
  23 //       this list of conditions and the following disclaimers in the
  24 //       documentation and/or other materials provided with the distribution.
  25 //
  26 //     * The names of the Software's authors and/or contributors
  27 //       may not be used to endorse or promote products derived from
  28 //       this Software without specific prior written permission.
  29 //
  30 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  31 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
  32 // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  33 // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  34 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  35 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
  36 // SOFTWARE.
  37
  38 #include <kern/mem.h>
  39 #include <kern/paging.h>
  40 #include <kern/pagetable.h>
  41 #include <kern/pagealloc.h>
  42 #include <kern/generic-pte.h>
  43 #include <kern/compiler.h>
  44
  45 extern int roshared_start, roshared_page_end;
  46 extern int rwshared_start, rwshared_page_end;
  47
  48 namespace Mem {
  49         extern IMappable physmem, anonmem;
  50
  51         class AddrSpaceFactory {
  52         public:
  53                 #include <servers/mem/addrspace/Mem/AddrSpaceFactory.h>
  54
  55                 AddrSpaceFactory()
  56                 {
  57                         init_iface();
  58                 }
  59
  60                 void create(Object *obj)
  61                 {
  62                         *obj = static_cast<IAddrSpace>(*(new AddrSpace(false)));
  63                 }
  64         };
  65
  66         class ProcAddrSpaceFactory {
  67         public:
  68                 #include <servers/mem/addrspace/Mem/ProcAddrSpaceFactory.h>
  69
  70                 ProcAddrSpaceFactory()
  71                 {
  72                         init_iface();
  73                 }
  74
  75                 void create(Object *obj)
  76                 {
  77                         AddrSpace *as = new AddrSpace(true);
  78                         Region region;
  79                         MapFlags mf = 0;
  80                         u64 vstart;
  81
  82                         region.start = kvirt_to_phys(&roshared_start);
  83                         region.end = kvirt_to_phys(&roshared_page_end);
  84                         vstart = Arch::roshared_map;
  85                         mf.Fixed = 1;
  86                         mf.access_IDLNS_Read = 1;
  87                         mf.access_IDLNS_Exec = 1;
  88
  89                         as->map(physmem, region, &vstart, mf, AddrSpace::map_protected);
  90
  91                         region.start = kvirt_to_phys(&rwshared_start);
  92                         region.end = kvirt_to_phys(&rwshared_page_end);
  93                         vstart = Arch::rwshared_map;
  94                         mf.access_IDLNS_Exec = 0;
  95                         mf.access_IDLNS_Write = 1;
  96                         mf.CopyOnWrite = 1;
  97
  98                         as->map(physmem, region, &vstart, mf, AddrSpace::map_protected);
  99
 100                         // Leave the stack no-exec by default.
 101                         region.start = vstart = Arch::stack_bottom;
 102                         region.end = Arch::stack_top;
 103                         mf.CopyOnWrite = 0;
 104                         printf("vstart %llx\n", vstart);
 105                         as->map(anonmem, region, &vstart, mf);
 106
 107                         *obj = static_cast<IAddrSpace>(*(as));
 108                 }
 109         };
 110
 111         ProcAddrSpaceFactory real_proc_addrspace_factory;
 112         Factory proc_addr_space_factory = real_proc_addrspace_factory;
 113
 114         AddrSpaceFactory real_addrspace_factory;
 115         Factory addr_space_factory = real_addrspace_factory;
 116
 117         AddrSpace::AddrSpace(bool process) : mappable(this)
 118         {
 119                 init_iface();
 120                 is_process = process;
 121
 122                 // OPT: Allow optional use of the native PTE for stacked aspaces,
 123                 // either because the native PTE is 64-bit, or because it's an
 124                 // embedded system which does not need 64-bit storage.
 125
 126                 if (process)
 127                         page_table = new PageTableImpl<Arch::PTE>(true);
 128                 else
 129                         page_table = new PageTableImpl<GenPTE>(false);
 130
 131                 cached_free_region = Arch::user_start + Arch::page_size;
 132         }
 133
 134         // This should only be used once during bootup to initialize the
 135         // kernel's address space with a static initial page table.
 136
 137         AddrSpace::AddrSpace(void *ptbl_toplevel) : mappable(this)
 138         {
 139                 init_iface();
 140                 is_process = true;
 141                 page_table = new PageTableImpl<Arch::PTE>(ptbl_toplevel);
 142
 143                 // FIXME: should be kernel virtual space
 144                 cached_free_region = Arch::user_start + Arch::page_size;
 145         }
 146
 147         void AddrSpace::get_mappable(IMappable *ma)
 148         {
 149                 *ma = mappable;
 150         }
 151
 152         void AddrSpace::clone(IAddrSpace *addrspace, uint8_t clone_is_real)
 153         {
 154                 // FIXME: implement
 155                 *addrspace = NULL;
 156         }
 157
 158         bool AddrSpace::handle_fault(ulong vaddr, bool write, bool exec, bool user)
 159         {
 160                 if (lock.held_by_curthread())
 161                         return false;
 162
 163                 assert(!(write && exec));
 164                 PTEFlags reqflags;
 165
 166                 if (user)
 167                         reqflags.User = 1;
 168
 169                 if (write)
 170                         reqflags.Writeable = 1;
 171                 else if (exec)
 172                         reqflags.Executable = 1;
 173                 else
 174                         reqflags.Readable = 1;
 175
 176                 reqflags.Valid = 1;
 177
 178                 try {
 179                         mappable.pagein(page_align(vaddr), reqflags);
 180                 }
 181
 182                 catch (BadPageFault &bpf) {
 183                         // FIXME: retain info about nature of bpf
 184                         // to throw to user?
 185                         return false;
 186                 }
 187
 188                 return true;
 189         }
 190
 191         bool AddrSpace::check_overlap(Region region, VirtualArea *&va)
 192         {
 193                 if (region.end < region.start)
 194                         return true;
 195
 196                 va = varea_tree.find_nearest(region.start);
 197
 198                 if (!va)
 199                         return false;
 200
 201                 // If region.start is in an existing region, that region will
 202                 // be returned.
 203
 204                 if (region.end >= va->region().start &&
 205                     region.start <= va->region().end)
 206                         return true;
 207
 208                 // If it returns a region that's greater than region.start, and va
 209                 // itself does not overlap, then prev does not overlap (or else
 210                 // region.start would be in or before prev, and thus prev would
 211                 // have been returned).
 212
 213                 // If it returns a region that's less than region.start, we still
 214                 // need to check next, as region.end could be in (or beyond) that
 215                 // region.
 216
 217                 if (va->list_node.next != &varea_list) {
 218                         VirtualArea *next =
 219                                 va->list_node.next->listentry(VirtualArea, list_node);
 220
 221                         if (region.end >= next->region().start &&
 222                        region.start <= next->region().end)
 223                    {
 224                         va = next;
 225                                 return true;
 226                         }
 227                 }
 228
 229                 VirtualArea *prev;
 230
 231                 if (va->list_node.prev != &varea_list)
 232                         prev = va->list_node.prev->listentry(VirtualArea, list_node);
 233                 else
 234                         prev = NULL;
 235
 236                 if (region.start < va->region().start) {
 237                         assert(!prev || prev->region().end < region.start);
 238                         va = prev;
 239                 }
 240
 241                 return false;
 242         }
 243
 244         VirtualArea *AddrSpace::split_varea(Region region)
 245         {
 246                 VirtualArea *start, *mid, *end;
 247
 248                 // check_overlap is used rather than varea_tree.find,
 249                 // so that the first overlapping region can be returned,
 250                 // as most (if not all) callers will need this anyway.
 251
 252                 if (!check_overlap(region, start))
 253                         return NULL;
 254
 255                 assert(start);
 256                 assert(start->aspace == this);
 257                 assert(start->region().end >= region.start);
 258
 259                 if (start->region().start < region.start) {
 260                         // There is a varea that straddles region.start;
 261                         // create a new varea "mid" for the overlapping part.
 262
 263                         mid = new VirtualArea;
 264
 265                         mid->aspace = this;
 266                         mid->region().start = region.start;
 267
 268                         if (region.end > start->region().end)
 269                                 mid->region().end = start->region().end;
 270                         else
 271                                 mid->region().end = region.end;
 272
 273                         mid->flags = start->flags;
 274                         mid->ma = start->ma;
 275                         mid->offset = start->offset;
 276
 277                         if (start->region().end > region.end) {
 278                                 // The varea also straddles region.end; create a new
 279                                 // varea "end" for the other side of the region.
 280
 281                                 end = new VirtualArea;
 282
 283                                 end->aspace = this;
 284                                 end->region().start = region.end + 1;
 285                                 end->region().end = start->region().end;
 286
 287                                 end->flags = start->flags;
 288                                 end->ma = start->ma;
 289                                 end->offset = start->offset;
 290                         } else {
 291                                 end = NULL;
 292                         }
 293
 294                         start->region().end = region.start - 1;
 295
 296                         varea_tree.add(mid);
 297                         mid->ma->map(mid);
 298
 299                         if (end) {
 300                                 // Splits have already been done at both ends of the region,
 301                                 // so there's no need to look up the ending address.
 302
 303                                 varea_tree.add(end);
 304                                 mid->ma->map(end);
 305                                 return mid;
 306                         }
 307
 308                         start = mid;
 309                 }
 310
 311                 if (start->region().end == region.end)
 312                         return start;
 313
 314                 if (start->region().end > region.end)
 315                         end = start;
 316                 else {
 317                         end = varea_tree.find(region.end);
 318
 319                         if (!end)
 320                                 return start;
 321
 322                         assert(end->aspace == this);
 323                         assert(end->region().start <= region.end);
 324                         assert(end->region().end >= region.end);
 325
 326                         if (end->region().end == region.end)
 327                                 return start;
 328                 }
 329
 330                 assert(end->region().end > region.end);
 331
 332                 // There is a varea that straddles region.start;
 333                 // create a new varea "mid" for the overlapping part.
 334
 335                 mid = new VirtualArea;
 336
 337                 mid->aspace = this;
 338                 mid->region().start = region.start;
 339
 340                 mid->region().start = end->region().start;
 341                 mid->region().end = region.end;
 342
 343                 mid->flags = start->flags;
 344                 mid->ma = start->ma;
 345                 mid->offset = start->offset;
 346
 347                 end->region().start = region.end + 1;
 348
 349                 varea_tree.add(mid);
 350                 mid->ma->map(mid);
 351
 352                 return start;
 353         }
 354
 355         bool AddrSpace::get_free_region(ulong len, Region &region,
 356                                         VirtualArea *&prev)
 357         {
 358                 assert(page_aligned(len));
 359                 assert(cached_free_region);
 360
 361                 region.start = cached_free_region;
 362                 region.end = cached_free_region + len - 1;
 363
 364                 if (region.end <= Arch::user_end && !check_overlap(region, prev)) {
 365                         cached_free_region = region.end + 1;
 366                         return true;
 367                 }
 368
 369                 for (Util::List *node = &prev->list_node; node != &varea_list;
 370                      node = node->next)
 371                 {
 372                         VirtualArea *va = node->listentry(VirtualArea, list_node);
 373                         ulong end = Arch::user_end;
 374
 375                         if (node->next != &varea_list) {
 376                                 VirtualArea *next = node->next->listentry(VirtualArea, list_node);
 377                                 end = next->region().start - 1;
 378                         }
 379
 380                         assert(end > va->region().end);
 381
 382                         if (end - va->region().end >= len) {
 383                                 region.start = va->region().end + 1;
 384                                 region.end = region.start + len - 1;
 385
 386                                 assert(page_aligned(region.start));
 387                                 cached_free_region = region.end + 1;
 388                                 return true;
 389                         }
 390                 }
 391
 392                 if (cached_free_region != Arch::user_start + Arch::page_size) {
 393                         cached_free_region = Arch::user_start + Arch::page_size;
 394                         return get_free_region(len, region, prev);
 395                 }
 396
 397                 return false;
 398         }
 399
 400         // The "mapped" paramater is used to indicate whether the top-level
 401         // address space has had a mapping established.  If "mapped" is
 402         // false, but an exception is not thrown, then this method must
 403         // be called again to propagate the mapping along the aspace chain.
 404         //
 405         // FIXME: Between aspace locks, if aspace's mapping is revoked and
 406         // ma->aspace's mapping changes, a pagein could leak through and cause
 407         // a page load or a copy-on-write breaking.  This isn't a huge deal
 408         // (it doesn't affect the correctness of the code or give aspace
 409         // access to ma->aspace's new mapping), but it's unpleasant, and could
 410         // have an adverse impact on determinism.  If you have a real-time
 411         // application that can't tolerate the occasional spurious pagein or
 412         // copy-on-write breaking, then use an address space that hasn't
 413         // previously been exposed to recursive mappers.
 414
 415         bool ASpaceMappable::rec_pagein(AddrSpace *aspace, u64 vaddr,
 416                                         PTEFlags reqflags)
 417         {
 418                 bool mapped = true;
 419
 420                 // aspace->mappable.retain();
 421
 422                 while (true) {
 423                         Lock::DroppableAutoLock autolock(aspace->lock);
 424                         VirtualArea *va = aspace->varea_tree.find(vaddr);
 425
 426                         if (!va)
 427                                 throw BadPageFault();
 428
 429                         if ((va->flags & reqflags) != reqflags)
 430                                 throw BadPageFault();
 431
 432                         if (aspace->map(va, vaddr, reqflags))
 433                                 break;
 434
 435                         mapped = false;
 436                         Mappable *ma = va->ma;
 437                         vaddr += va->offset;
 438
 439                         // ma->retain();
 440                         autolock.unlock();
 441                         // aspace->mappable.release();
 442
 443                         if (!ma->is_aspace) {
 444                                 ma->pagein(vaddr, reqflags);
 445                                 // ma->release();
 446                                 break;
 447                         }
 448
 449                         aspace = static_cast<ASpaceMappable *>(ma)->aspace;
 450                 }
 451
 452                 return mapped;
 453         }
 454
 455         void ASpaceMappable::pagein(u64 vaddr, PTEFlags reqflags)
 456         {
 457                 while (!rec_pagein(aspace, vaddr, reqflags));
 458         }
 459
 460         void AddrSpace::break_copy_on_write(VirtualArea *va, u64 vaddr, u64 phys)
 461         {
 462                 assert(lock.held_by_curthread());
 463                 assert(rmap_lock.held_by_curthread());
 464
 465                 assert(va->flags.FaultOnWrite);
 466                 assert(va->aspace == this);
 467
 468                 Page *old_page = phys_to_page(phys);
 469
 470                 Region region = { vaddr, vaddr + Arch::page_size - 1 };
 471
 472                 // If this is the only reference to the page left, then
 473                 // nothing needs to be copied.  Just clear the COW condition.
 474                 if (is_phys_page(old_page) && old_page->get_refcount() == 1) {
 475                         PTEFlags mask, flags;
 476                         mask.FaultOnWrite = 1;
 477
 478                         page_table->set_flags(region, flags, mask);
 479                         return;
 480                 }
 481
 482                 Page *new_page = PageAlloc::alloc(1);
 483
 484                 // FIXME -- highmem
 485                 // OPT: It'd be better to do this without the rmap_lock held,
 486                 // especially if rmap_lock is global rather than per-physpage.
 487                 // I want to keep things simple for now and optimize later,
 488                 // though.
 489
 490                 memcpy(page_to_kvirt(new_page), phys_to_kvirt(phys),
 491                        Arch::page_size);
 492
 493                 page_table->rmap_table.break_copy_on_write(region.start, new_page);
 494                 new_page->release();
 495         }
 496
 497         void ASpaceMappable::get_mapping(u64 vaddr, u64 *phys, PTEFlags *flags)
 498         {
 499                 aspace->page_table->get_mapping(vaddr, phys, flags);
 500         }
 501
 502         bool AddrSpace::map(VirtualArea *va, u64 vaddr, PTEFlags reqflags)
 503         {
 504                 Lock::AutoLock autolock(rmap_lock);
 505                 assert(va->aspace == this);
 506
 507                 u64 phys;
 508                 PTEFlags flags;
 509                 va->ma->get_mapping(vaddr + va->offset, &phys, &flags);
 510
 511                 PTEFlags newflags = flags & va->flags;
 512                 newflags.FaultOnWrite = flags.FaultOnWrite | va->flags.FaultOnWrite;
 513
 514                 if (!newflags.Valid) {
 515                         assert(va->flags.Valid);
 516                         return false;
 517                 }
 518
 519                 if ((newflags & reqflags) != reqflags)
 520                         return false;
 521
 522                 u64 oldphys;
 523                 PTEFlags oldflags;
 524                 page_table->get_mapping(vaddr, &oldphys, &oldflags);
 525
 526                 if (oldflags.Valid &&
 527                     !(reqflags.Writeable && oldflags.FaultOnWrite))
 528                 {
 529                         // If the existing mapping is valid, don't try to map it again.
 530                         // The existing mapping was put there possibly by a race, but
 531                         // more likely because a FaultOnWrite was handled upstream.
 532                         //
 533                         // FaultOnWrite handling is the only type of mapping change that
 534                         // can be done directly; all others must change the varea and do
 535                         // an rmap invalidation instead.  FaultOnWrite is special
 536                         // because we don't want to split vareas for every page that
 537                         // gets its copy-on-write broken.
 538
 539                         assert((oldflags & reqflags) == reqflags);
 540                         assert(!va->flags.FaultOnWrite || oldphys == phys);
 541                         return true;
 542                 }
 543
 544                 if (reqflags.Writeable && oldflags.FaultOnWrite)
 545                 {
 546                         // The FaultOnWrite needs to be handled upstream.
 547                         if (!va->flags.FaultOnWrite)
 548                                 return false;
 549
 550                         va->aspace->break_copy_on_write(va, vaddr, phys);
 551                 } else {
 552                         assert(!oldflags.Valid);
 553                         PageTable *usptbl = NULL;
 554
 555                         if (va->ma->is_aspace) {
 556                                 ASpaceMappable *asma = static_cast<ASpaceMappable *>(va->ma);
 557                                 usptbl = asma->aspace->page_table;
 558                         }
 559
 560                         RMapTable::map(va, usptbl, vaddr, vaddr + va->offset);
 561
 562                         RegionWithOffset rwo;
 563                         rwo.start = vaddr;
 564                         rwo.end = vaddr + Arch::page_size - 1;
 565                         rwo.offset = phys;
 566
 567                         page_table->map(rwo, newflags);
 568                 }
 569
 570                 return true;
 571         }
 572
 573         void ASpaceMappable::get_size(u64 *size)
 574         {
 575                 aspace->get_size(size);
 576         }
 577
 578         void AddrSpace::map(IMappable ma, Region region, u64 *vstart,
 579                             MapFlags mflags, int map_type)
 580         {
 581                 // FIXME: check alignment for VIPT caches
 582                 // FIXME: Implement the "Replace" map flag
 583
 584                 if (mflags.Replace)
 585                         throw_idl(InvalidArgument, 3,
 586                                   countarray("Replace unimplemented"));
 587
 588                 Mappable *cma = Mappable::classptr(ma);
 589                 if (!cma) {
 590                         // The given IMappable does not refer to a Mappable
 591                         // of this kernel.
 592
 593                         throw_idl(InvalidArgument, 0, nullarray);
 594                 }
 595
 596                 bool fixed = mflags.Fixed;
 597
 598                 if (is_process)
 599                         mflags.Fixed = 1;
 600
 601                 if (!page_aligned(region.start))
 602                         throw_idl(InvalidArgument, 1, countarray("unaligned start"));
 603
 604                 if (!page_aligned(region.end + 1))
 605                         throw_idl(InvalidArgument, 1, countarray("unaligned end"));
 606
 607                 Lock::AutoLock autolock(lock);
 608                 Region vregion;
 609                 VirtualArea *prev;
 610
 611                 if (*vstart != System::Mem::AddrSpace_ns::unspecified_start) {
 612                         vregion.start = *vstart;
 613                         vregion.end = vregion.start + region.end - region.start;
 614
 615                         if (is_process) {
 616                                 if (!valid_addr(vregion.start))
 617                                         throw_idl(InvalidArgument, 2,
 618                                                   countarray("invalid virtual start"));
 619
 620                                 if (!valid_addr(vregion.end))
 621                                         throw_idl(InvalidArgument, 2,
 622                                                   countarray("invalid virtual end"));
 623                         }
 624
 625                         if (check_overlap(vregion, prev))
 626                                 *vstart = System::Mem::AddrSpace_ns::unspecified_start;
 627                 }
 628
 629                 if (*vstart == System::Mem::AddrSpace_ns::unspecified_start) {
 630                         if (fixed)
 631                                 throw_idl(ResourceBusy, 2, countarray("varea overlap"));
 632
 633                         if (!get_free_region(region.end - region.start + 1, vregion, prev))
 634                                 throw_idl(OutOfSpace, countarray("out of vspace"));
 635
 636                         *vstart = vregion.start;
 637                 }
 638
 639                 VirtualArea *newva = new VirtualArea;
 640                 newva->aspace = this;
 641                 newva->region() = vregion;
 642
 643                 newva->flags.Valid = 1;
 644                 newva->flags.User = map_type != map_kernel;
 645                 newva->flags.Readable = mflags.access_IDLNS_Read;
 646                 newva->flags.Writeable = mflags.access_IDLNS_Write;
 647                 newva->flags.Executable = mflags.access_IDLNS_Exec;
 648                 newva->flags.FaultOnWrite = mflags.CopyOnWrite;
 649                 newva->flags.Protected = map_type != map_user;
 650                 newva->ma = cma;
 651                 newva->offset = region.start - vregion.start;
 652
 653                 varea_tree.add(newva);
 654                 newva->ma->map(newva);
 655
 656                 if (prev) {
 657                         prev->list_node.add_front(&newva->list_node);
 658                 } else {
 659                         varea_list.add_front(&newva->list_node);
 660                 }
 661         }
 662
 663         void AddrSpace::unmap(Region region, bool from_kernel)
 664         {
 665                 u64 orig_start = region.start;
 666
 667                 while (region.start <= region.end) {
 668                         Lock::DroppableAutoLock autolock(lock);
 669                         VirtualArea *va;
 670
 671                         // If check_overlap returns false, then there are no vareas
 672                         // in the specified region, so there's nothing to unmap.
 673
 674                         if (!check_overlap(region, va))
 675                                 return;
 676
 677                         if (va->flags.Protected && !from_kernel) {
 678                                 region.start = va->list_node.next->
 679                                                listentry(VirtualArea, list_node)->region().start;
 680
 681                                 if (region.start <= orig_start)
 682                                         break;
 683
 684                                 continue;
 685                         }
 686
 687                         u64 va_end = va->region().end;
 688                         u64 next_start = 0;
 689
 690                         if (va_end > region.end) {
 691                                 u64 va_start = va->region().start;
 692                                 va->region().start = region.end + 1;
 693
 694                                 if (va_start < region.start) {
 695                                         VirtualArea *newva = new VirtualArea;
 696
 697                                         newva->aspace = this;
 698                                         newva->region().start = va_start;
 699                                         newva->region().end = region.start - 1;
 700
 701                                         newva->flags = va->flags;
 702                                         newva->ma = va->ma;
 703                                         newva->offset = va->offset;
 704
 705                                         varea_tree.add(newva);
 706                                         newva->ma->map(newva);
 707                                 }
 708
 709                                 VirtualArea *nextva =
 710                                         va->list_node.next->listentry(VirtualArea, list_node);
 711
 712                                 next_start = nextva->region().start;
 713                         } else if (va->region().start < region.start) {
 714                                 va->region().end = region.start - 1;
 715                         } else {
 716                                 varea_tree.del(va);
 717                                 va->ma->unmap(va);
 718                         }
 719
 720                         // This is done after the varea removal, so that new faults
 721                         // don't map things in again.
 722
 723                         // OPT: Skip RMap-based unmapping if nothing maps this aspace.
 724                         // OPT: Push this loop into the RMap code, allowing it to skip
 725                         // empty portions of the tables (as the pagetable code currently
 726                         // does).
 727
 728                         while (region.start <= va_end && region.start <= region.end) {
 729                                 page_table->rmap_table.unmap(region.start);
 730                                 region.start += Arch::page_size;
 731
 732                                 if (region.start <= orig_start)
 733                                         break;
 734                         }
 735
 736                         region.start = next_start;
 737
 738                         if (region.start <= orig_start)
 739                                 break;
 740                 }
 741         }
 742
 743         void AddrSpace::set_mapflags(Region region, MapFlags mflags)
 744         {
 745                 // FIXME: implement
 746                 // Find varea, split if necessary, propagate change to stacked aspaces
 747         }
 748
 749         void AddrSpace::get_mapflags(Region region, MapFlags *mflags, uint8_t *all_same)
 750         {
 751                 // FIXME: implement
 752         }
 753
 754         void AddrSpace::get_mapping(Region region, IMappable *ma, u64 *offset)
 755         {
 756                 // FIXME: implement
 757         }
 758
 759         void AddrSpace::get_page_size(u32 *page_size)
 760         {
 761                 *page_size = Arch::page_size;
 762         }
 763
 764         void AddrSpace::get_min_align(u32 *min_align)
 765         {
 766                 *min_align = Arch::page_mapping_min_align;
 767         }
 768
 769         void Mappable::map(VirtualArea *varea)
 770         {
 771                 mappings_lock.lock_irq();
 772                 mappings.add_back(&varea->mappings_node);
 773                 mappings_lock.unlock_irq();
 774         }
 775
 776         void Mappable::unmap(VirtualArea *varea)
 777         {
 778                 mappings_lock.lock_irq();
 779                 varea->mappings_node.del();
 780                 mappings_lock.unlock_irq();
 781         }
 782
 783         void PageTable::kill_pte(ulong vaddr, u64 paddr, bool dirty,
 784                                  bool valid, bool no_release)
 785         {
 786                 Page *oldpage = phys_to_page(paddr);
 787
 788                 if (!is_phys_page(oldpage))
 789                         oldpage = NULL;
 790
 791                 if (is_process && valid) {
 792                         Arch::invalidate_tlb_entry(vaddr);
 793
 794                         if (oldpage && dirty &&
 795                             !ll_test_and_set(&oldpage->flags, PageFlags::bits::Dirty))
 796                         {
 797                                 oldpage->retain();
 798                                 // Queue page for writeback
 799                         }
 800                 }
 801
 802                 if (!no_release && oldpage)
 803                         oldpage->release();
 804         }
 805
 806         // FIXME: Add a special PTE flag to indicate that PhysMem mappings
 807         // don't mess with page refcounts.
 808
 809         class PhysMem : public Mappable {
 810         public:
 811                 void get_size(u64 *size)
 812                 {
 813                         if (sizeof(long) == 8)
 814                                 *size = 1ULL << (64 - Arch::page_shift);
 815                         else
 816                                 *size = 1ULL << (32 - Arch::page_shift);
 817                 }
 818
 819                 void pagein(u64 vaddr, PTEFlags reqflags)
 820                 {
 821                         // Doesn't need to do anything yet, though it may later
 822                         // once high memory support is added.
 823                 }
 824
 825                 void get_mapping(u64 addr, u64 *phys, PTEFlags *flags)
 826                 {
 827                         *phys = addr;
 828                         *flags = 0;
 829                         flags->Valid = 1;
 830                         flags->Readable = 1;
 831                         flags->Writeable = 1;
 832                         flags->Executable = 1;
 833                         flags->User = 1;
 834                 }
 835         };
 836
 837         PhysMem real_physmem;
 838         IMappable physmem = real_physmem;
 839
 840         class AnonMem : public Mappable {
 841         public:
 842                 void get_size(u64 *size)
 843                 {
 844                         if (sizeof(long) == 8)
 845                                 *size = 1ULL << (64 - Arch::page_shift);
 846                         else
 847                                 *size = 1ULL << (32 - Arch::page_shift);
 848                 }
 849
 850                 void pagein(u64 vaddr, PTEFlags reqflags)
 851                 {
 852                         // Doesn't need to do anything yet, though it may later
 853                         // once high memory support is added.
 854                 }
 855
 856                 void get_mapping(u64 addr, u64 *phys, PTEFlags *flags)
 857                 {
 858                         Page *page = PageAlloc::alloc(1);
 859
 860                         // OPT: Only zero if it was asked for.
 861                         // OPT: Eventually, have separate pagelists for zeroed and
 862                         // unzeroed memory, and a low-priority background thread
 863                         // that zeroes pages and moves them to the zeroed list.
 864                         bzero(page_to_kvirt(page), Arch::page_size);
 865
 866                         *phys = page_to_phys(page);
 867                         *flags = 0;
 868                         flags->Valid = 1;
 869                         flags->Readable = 1;
 870                         flags->Writeable = 1;
 871                         flags->Executable = 1;
 872                         flags->User = 1;
 873                 }
 874         };
 875
 876         AnonMem real_anonmem;
 877         IMappable anonmem = real_anonmem;
 878 }
 879
 880 #include <servers/mem/addrspace/footer.cc>