499 lines
18 KiB
C++
499 lines
18 KiB
C++
|
|
/*
|
|
* Copyright (C) 2013 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
#include "rosalloc_space-inl.h"
|
|
|
|
#include "base/logging.h" // For VLOG.
|
|
#include "base/time_utils.h"
|
|
#include "base/utils.h"
|
|
#include "gc/accounting/card_table.h"
|
|
#include "gc/accounting/space_bitmap-inl.h"
|
|
#include "gc/heap.h"
|
|
#include "memory_tool_malloc_space-inl.h"
|
|
#include "mirror/class-inl.h"
|
|
#include "mirror/object-inl.h"
|
|
#include "runtime.h"
|
|
#include "scoped_thread_state_change-inl.h"
|
|
#include "thread.h"
|
|
#include "thread_list.h"
|
|
|
|
namespace art {
|
|
namespace gc {
|
|
namespace space {
|
|
|
|
static constexpr bool kPrefetchDuringRosAllocFreeList = false;
|
|
static constexpr size_t kPrefetchLookAhead = 8;
|
|
// Use this only for verification, it is not safe to use since the class of the object may have
|
|
// been freed.
|
|
static constexpr bool kVerifyFreedBytes = false;
|
|
|
|
// TODO: Fix
|
|
// template class MemoryToolMallocSpace<RosAllocSpace, allocator::RosAlloc*>;
|
|
|
|
RosAllocSpace::RosAllocSpace(MemMap&& mem_map,
|
|
size_t initial_size,
|
|
const std::string& name,
|
|
art::gc::allocator::RosAlloc* rosalloc,
|
|
uint8_t* begin,
|
|
uint8_t* end,
|
|
uint8_t* limit,
|
|
size_t growth_limit,
|
|
bool can_move_objects,
|
|
size_t starting_size,
|
|
bool low_memory_mode)
|
|
: MallocSpace(name,
|
|
std::move(mem_map),
|
|
begin,
|
|
end,
|
|
limit,
|
|
growth_limit,
|
|
true,
|
|
can_move_objects,
|
|
starting_size, initial_size),
|
|
rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) {
|
|
CHECK(rosalloc != nullptr);
|
|
}
|
|
|
|
RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap&& mem_map,
|
|
const std::string& name,
|
|
size_t starting_size,
|
|
size_t initial_size,
|
|
size_t growth_limit,
|
|
size_t capacity,
|
|
bool low_memory_mode,
|
|
bool can_move_objects) {
|
|
DCHECK(mem_map.IsValid());
|
|
|
|
bool running_on_memory_tool = Runtime::Current()->IsRunningOnMemoryTool();
|
|
|
|
allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map.Begin(),
|
|
starting_size,
|
|
initial_size,
|
|
capacity,
|
|
low_memory_mode,
|
|
running_on_memory_tool);
|
|
if (rosalloc == nullptr) {
|
|
LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")";
|
|
return nullptr;
|
|
}
|
|
|
|
// Protect memory beyond the starting size. MoreCore will add r/w permissions when necessory
|
|
uint8_t* end = mem_map.Begin() + starting_size;
|
|
if (capacity - starting_size > 0) {
|
|
CheckedCall(mprotect, name.c_str(), end, capacity - starting_size, PROT_NONE);
|
|
}
|
|
|
|
// Everything is set so record in immutable structure and leave
|
|
uint8_t* begin = mem_map.Begin();
|
|
// TODO: Fix RosAllocSpace to support ASan. There is currently some issues with
|
|
// AllocationSize caused by redzones. b/12944686
|
|
if (running_on_memory_tool) {
|
|
return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>(
|
|
std::move(mem_map),
|
|
initial_size,
|
|
name,
|
|
rosalloc,
|
|
begin,
|
|
end,
|
|
begin + capacity,
|
|
growth_limit,
|
|
can_move_objects,
|
|
starting_size,
|
|
low_memory_mode);
|
|
} else {
|
|
return new RosAllocSpace(std::move(mem_map),
|
|
initial_size,
|
|
name,
|
|
rosalloc,
|
|
begin,
|
|
end,
|
|
begin + capacity,
|
|
growth_limit,
|
|
can_move_objects,
|
|
starting_size,
|
|
low_memory_mode);
|
|
}
|
|
}
|
|
|
|
RosAllocSpace::~RosAllocSpace() {
|
|
delete rosalloc_;
|
|
}
|
|
|
|
RosAllocSpace* RosAllocSpace::Create(const std::string& name,
|
|
size_t initial_size,
|
|
size_t growth_limit,
|
|
size_t capacity,
|
|
bool low_memory_mode,
|
|
bool can_move_objects) {
|
|
uint64_t start_time = 0;
|
|
if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
|
|
start_time = NanoTime();
|
|
VLOG(startup) << "RosAllocSpace::Create entering " << name
|
|
<< " initial_size=" << PrettySize(initial_size)
|
|
<< " growth_limit=" << PrettySize(growth_limit)
|
|
<< " capacity=" << PrettySize(capacity);
|
|
}
|
|
|
|
// Memory we promise to rosalloc before it asks for morecore.
|
|
// Note: making this value large means that large allocations are unlikely to succeed as rosalloc
|
|
// will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
|
|
// size of the large allocation) will be greater than the footprint limit.
|
|
size_t starting_size = Heap::kDefaultStartingSize;
|
|
MemMap mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity);
|
|
if (!mem_map.IsValid()) {
|
|
LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size "
|
|
<< PrettySize(capacity);
|
|
return nullptr;
|
|
}
|
|
|
|
RosAllocSpace* space = CreateFromMemMap(std::move(mem_map),
|
|
name,
|
|
starting_size,
|
|
initial_size,
|
|
growth_limit,
|
|
capacity,
|
|
low_memory_mode,
|
|
can_move_objects);
|
|
// We start out with only the initial size possibly containing objects.
|
|
if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
|
|
LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time)
|
|
<< " ) " << *space;
|
|
}
|
|
return space;
|
|
}
|
|
|
|
allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start,
|
|
size_t initial_size,
|
|
size_t maximum_size, bool low_memory_mode,
|
|
bool running_on_memory_tool) {
|
|
// clear errno to allow PLOG on error
|
|
errno = 0;
|
|
// create rosalloc using our backing storage starting at begin and
|
|
// with a footprint of morecore_start. When morecore_start bytes of
|
|
// memory is exhaused morecore will be called.
|
|
allocator::RosAlloc* rosalloc = new art::gc::allocator::RosAlloc(
|
|
begin, morecore_start, maximum_size,
|
|
low_memory_mode ?
|
|
art::gc::allocator::RosAlloc::kPageReleaseModeAll :
|
|
art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd,
|
|
running_on_memory_tool);
|
|
if (rosalloc != nullptr) {
|
|
rosalloc->SetFootprintLimit(initial_size);
|
|
} else {
|
|
PLOG(ERROR) << "RosAlloc::Create failed";
|
|
}
|
|
return rosalloc;
|
|
}
|
|
|
|
mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes,
|
|
size_t* bytes_allocated, size_t* usable_size,
|
|
size_t* bytes_tl_bulk_allocated) {
|
|
mirror::Object* result;
|
|
{
|
|
MutexLock mu(self, lock_);
|
|
// Grow as much as possible within the space.
|
|
size_t max_allowed = Capacity();
|
|
rosalloc_->SetFootprintLimit(max_allowed);
|
|
// Try the allocation.
|
|
result = AllocCommon(self, num_bytes, bytes_allocated, usable_size,
|
|
bytes_tl_bulk_allocated);
|
|
// Shrink back down as small as possible.
|
|
size_t footprint = rosalloc_->Footprint();
|
|
rosalloc_->SetFootprintLimit(footprint);
|
|
}
|
|
// Note RosAlloc zeroes memory internally.
|
|
// Return the new allocation or null.
|
|
CHECK(!kDebugSpaces || result == nullptr || Contains(result));
|
|
return result;
|
|
}
|
|
|
|
MallocSpace* RosAllocSpace::CreateInstance(MemMap&& mem_map,
|
|
const std::string& name,
|
|
void* allocator,
|
|
uint8_t* begin,
|
|
uint8_t* end,
|
|
uint8_t* limit,
|
|
size_t growth_limit,
|
|
bool can_move_objects) {
|
|
if (Runtime::Current()->IsRunningOnMemoryTool()) {
|
|
return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>(
|
|
std::move(mem_map),
|
|
initial_size_,
|
|
name,
|
|
reinterpret_cast<allocator::RosAlloc*>(allocator),
|
|
begin,
|
|
end,
|
|
limit,
|
|
growth_limit,
|
|
can_move_objects,
|
|
starting_size_,
|
|
low_memory_mode_);
|
|
} else {
|
|
return new RosAllocSpace(std::move(mem_map),
|
|
initial_size_,
|
|
name,
|
|
reinterpret_cast<allocator::RosAlloc*>(allocator),
|
|
begin,
|
|
end,
|
|
limit,
|
|
growth_limit,
|
|
can_move_objects,
|
|
starting_size_,
|
|
low_memory_mode_);
|
|
}
|
|
}
|
|
|
|
size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) {
|
|
if (kDebugSpaces) {
|
|
CHECK(ptr != nullptr);
|
|
CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this;
|
|
}
|
|
if (kRecentFreeCount > 0) {
|
|
MutexLock mu(self, lock_);
|
|
RegisterRecentFree(ptr);
|
|
}
|
|
return rosalloc_->Free(self, ptr);
|
|
}
|
|
|
|
size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
|
|
DCHECK(ptrs != nullptr);
|
|
|
|
size_t verify_bytes = 0;
|
|
for (size_t i = 0; i < num_ptrs; i++) {
|
|
if (kPrefetchDuringRosAllocFreeList && i + kPrefetchLookAhead < num_ptrs) {
|
|
__builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead]));
|
|
}
|
|
if (kVerifyFreedBytes) {
|
|
verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr);
|
|
}
|
|
}
|
|
|
|
if (kRecentFreeCount > 0) {
|
|
MutexLock mu(self, lock_);
|
|
for (size_t i = 0; i < num_ptrs; i++) {
|
|
RegisterRecentFree(ptrs[i]);
|
|
}
|
|
}
|
|
|
|
if (kDebugSpaces) {
|
|
size_t num_broken_ptrs = 0;
|
|
for (size_t i = 0; i < num_ptrs; i++) {
|
|
if (!Contains(ptrs[i])) {
|
|
num_broken_ptrs++;
|
|
LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this;
|
|
} else {
|
|
size_t size = rosalloc_->UsableSize(ptrs[i]);
|
|
memset(ptrs[i], 0xEF, size);
|
|
}
|
|
}
|
|
CHECK_EQ(num_broken_ptrs, 0u);
|
|
}
|
|
|
|
const size_t bytes_freed = rosalloc_->BulkFree(self, reinterpret_cast<void**>(ptrs), num_ptrs);
|
|
if (kVerifyFreedBytes) {
|
|
CHECK_EQ(verify_bytes, bytes_freed);
|
|
}
|
|
return bytes_freed;
|
|
}
|
|
|
|
size_t RosAllocSpace::Trim() {
|
|
VLOG(heap) << "RosAllocSpace::Trim() ";
|
|
{
|
|
Thread* const self = Thread::Current();
|
|
// SOA required for Rosalloc::Trim() -> ArtRosAllocMoreCore() -> Heap::GetRosAllocSpace.
|
|
ScopedObjectAccess soa(self);
|
|
MutexLock mu(self, lock_);
|
|
// Trim to release memory at the end of the space.
|
|
rosalloc_->Trim();
|
|
}
|
|
// Attempt to release pages if it does not release all empty pages.
|
|
if (!rosalloc_->DoesReleaseAllPages()) {
|
|
return rosalloc_->ReleasePages();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
void RosAllocSpace::Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
|
|
void* arg) {
|
|
InspectAllRosAlloc(callback, arg, true);
|
|
}
|
|
|
|
size_t RosAllocSpace::GetFootprint() {
|
|
MutexLock mu(Thread::Current(), lock_);
|
|
return rosalloc_->Footprint();
|
|
}
|
|
|
|
size_t RosAllocSpace::GetFootprintLimit() {
|
|
MutexLock mu(Thread::Current(), lock_);
|
|
return rosalloc_->FootprintLimit();
|
|
}
|
|
|
|
void RosAllocSpace::SetFootprintLimit(size_t new_size) {
|
|
MutexLock mu(Thread::Current(), lock_);
|
|
VLOG(heap) << "RosAllocSpace::SetFootprintLimit " << PrettySize(new_size);
|
|
// Compare against the actual footprint, rather than the Size(), because the heap may not have
|
|
// grown all the way to the allowed size yet.
|
|
size_t current_space_size = rosalloc_->Footprint();
|
|
if (new_size < current_space_size) {
|
|
// Don't let the space grow any more.
|
|
new_size = current_space_size;
|
|
}
|
|
rosalloc_->SetFootprintLimit(new_size);
|
|
}
|
|
|
|
uint64_t RosAllocSpace::GetBytesAllocated() {
|
|
size_t bytes_allocated = 0;
|
|
InspectAllRosAlloc(art::gc::allocator::RosAlloc::BytesAllocatedCallback, &bytes_allocated, false);
|
|
return bytes_allocated;
|
|
}
|
|
|
|
uint64_t RosAllocSpace::GetObjectsAllocated() {
|
|
size_t objects_allocated = 0;
|
|
InspectAllRosAlloc(art::gc::allocator::RosAlloc::ObjectsAllocatedCallback, &objects_allocated, false);
|
|
return objects_allocated;
|
|
}
|
|
|
|
void RosAllocSpace::InspectAllRosAllocWithSuspendAll(
|
|
void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
|
|
void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
|
|
// TODO: NO_THREAD_SAFETY_ANALYSIS.
|
|
Thread* self = Thread::Current();
|
|
ScopedSuspendAll ssa(__FUNCTION__);
|
|
MutexLock mu(self, *Locks::runtime_shutdown_lock_);
|
|
MutexLock mu2(self, *Locks::thread_list_lock_);
|
|
rosalloc_->InspectAll(callback, arg);
|
|
if (do_null_callback_at_end) {
|
|
callback(nullptr, nullptr, 0, arg); // Indicate end of a space.
|
|
}
|
|
}
|
|
|
|
void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
|
|
void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
|
|
// TODO: NO_THREAD_SAFETY_ANALYSIS.
|
|
Thread* self = Thread::Current();
|
|
if (Locks::mutator_lock_->IsExclusiveHeld(self)) {
|
|
// The mutators are already suspended. For example, a call path
|
|
// from SignalCatcher::HandleSigQuit().
|
|
rosalloc_->InspectAll(callback, arg);
|
|
if (do_null_callback_at_end) {
|
|
callback(nullptr, nullptr, 0, arg); // Indicate end of a space.
|
|
}
|
|
} else if (Locks::mutator_lock_->IsSharedHeld(self)) {
|
|
// The mutators are not suspended yet and we have a shared access
|
|
// to the mutator lock. Temporarily release the shared access by
|
|
// transitioning to the suspend state, and suspend the mutators.
|
|
ScopedThreadSuspension sts(self, ThreadState::kSuspended);
|
|
InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
|
|
} else {
|
|
// The mutators are not suspended yet. Suspend the mutators.
|
|
InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
|
|
}
|
|
}
|
|
|
|
size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) {
|
|
return rosalloc_->RevokeThreadLocalRuns(thread);
|
|
}
|
|
|
|
size_t RosAllocSpace::RevokeAllThreadLocalBuffers() {
|
|
return rosalloc_->RevokeAllThreadLocalRuns();
|
|
}
|
|
|
|
void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
|
|
if (kIsDebugBuild) {
|
|
rosalloc_->AssertThreadLocalRunsAreRevoked(thread);
|
|
}
|
|
}
|
|
|
|
void RosAllocSpace::AssertAllThreadLocalBuffersAreRevoked() {
|
|
if (kIsDebugBuild) {
|
|
rosalloc_->AssertAllThreadLocalRunsAreRevoked();
|
|
}
|
|
}
|
|
|
|
void RosAllocSpace::Clear() {
|
|
size_t footprint_limit = GetFootprintLimit();
|
|
madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED);
|
|
live_bitmap_.Clear();
|
|
mark_bitmap_.Clear();
|
|
SetEnd(begin_ + starting_size_);
|
|
delete rosalloc_;
|
|
rosalloc_ = CreateRosAlloc(mem_map_.Begin(),
|
|
starting_size_,
|
|
initial_size_,
|
|
NonGrowthLimitCapacity(),
|
|
low_memory_mode_,
|
|
Runtime::Current()->IsRunningOnMemoryTool());
|
|
SetFootprintLimit(footprint_limit);
|
|
}
|
|
|
|
void RosAllocSpace::DumpStats(std::ostream& os) {
|
|
ScopedSuspendAll ssa(__FUNCTION__);
|
|
rosalloc_->DumpStats(os);
|
|
}
|
|
|
|
template<bool kMaybeIsRunningOnMemoryTool>
|
|
size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) {
|
|
// obj is a valid object. Use its class in the header to get the size.
|
|
// Don't use verification since the object may be dead if we are sweeping.
|
|
size_t size = obj->SizeOf<kVerifyNone>();
|
|
bool add_redzones = false;
|
|
if (kMaybeIsRunningOnMemoryTool) {
|
|
add_redzones = kRunningOnMemoryTool && kMemoryToolAddsRedzones;
|
|
if (add_redzones) {
|
|
size += 2 * kDefaultMemoryToolRedZoneBytes;
|
|
}
|
|
} else {
|
|
DCHECK(!kRunningOnMemoryTool);
|
|
}
|
|
size_t size_by_size = rosalloc_->UsableSize(size);
|
|
if (kIsDebugBuild) {
|
|
// On memory tool, the red zone has an impact...
|
|
const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj);
|
|
size_t size_by_ptr = rosalloc_->UsableSize(
|
|
obj_ptr - (add_redzones ? kDefaultMemoryToolRedZoneBytes : 0));
|
|
if (size_by_size != size_by_ptr) {
|
|
LOG(INFO) << "Found a bad sized obj of size " << size
|
|
<< " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec
|
|
<< " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr;
|
|
}
|
|
DCHECK_EQ(size_by_size, size_by_ptr);
|
|
}
|
|
if (usable_size != nullptr) {
|
|
*usable_size = size_by_size;
|
|
}
|
|
return size_by_size;
|
|
}
|
|
|
|
} // namespace space
|
|
|
|
namespace allocator {
|
|
|
|
// Callback from rosalloc when it needs to increase the footprint.
|
|
void* ArtRosAllocMoreCore(allocator::RosAlloc* rosalloc, intptr_t increment)
|
|
REQUIRES_SHARED(Locks::mutator_lock_) {
|
|
Heap* heap = Runtime::Current()->GetHeap();
|
|
art::gc::space::RosAllocSpace* rosalloc_space = heap->GetRosAllocSpace(rosalloc);
|
|
DCHECK(rosalloc_space != nullptr);
|
|
DCHECK_EQ(rosalloc_space->GetRosAlloc(), rosalloc);
|
|
return rosalloc_space->MoreCore(increment);
|
|
}
|
|
|
|
} // namespace allocator
|
|
|
|
} // namespace gc
|
|
} // namespace art
|