From 2e2e1f7cb36aeac0abb730a752fcd78cf91a380f Mon Sep 17 00:00:00 2001 From: Rob Arnold Date: Tue, 5 Jul 2011 22:44:22 -0700 Subject: [PATCH 1/2] Add realloc method to rust_kernel --- src/rt/rust_kernel.cpp | 5 +++++ src/rt/rust_kernel.h | 1 + 2 files changed, 6 insertions(+) diff --git a/src/rt/rust_kernel.cpp b/src/rt/rust_kernel.cpp index 85a6178ded7..b58a6a54cb1 100644 --- a/src/rt/rust_kernel.cpp +++ b/src/rt/rust_kernel.cpp @@ -206,6 +206,11 @@ rust_kernel::malloc(size_t size) { return _region->malloc(size); } +void * +rust_kernel::realloc(void *mem, size_t size) { + return _region->realloc(mem, size); +} + void rust_kernel::free(void *mem) { _region->free(mem); } diff --git a/src/rt/rust_kernel.h b/src/rt/rust_kernel.h index f61935c8c48..3ed82b22112 100644 --- a/src/rt/rust_kernel.h +++ b/src/rt/rust_kernel.h @@ -110,6 +110,7 @@ public: virtual ~rust_kernel(); void *malloc(size_t size); + void *realloc(void *mem, size_t size); void free(void *mem); // FIXME: this should go away From f6117173c9f26efdcf50cb664d006ea2bdf0d0cb Mon Sep 17 00:00:00 2001 From: Rob Arnold Date: Tue, 5 Jul 2011 22:55:41 -0700 Subject: [PATCH 2/2] Allocate rust_ivec buffers out of the kernel pool The duplication of upcalls is due to the fact that the runtime is shared between stage0/rustc and stage1/rustc. Once snapshots are updated, they should be de-duplicated. --- src/comp/back/upcall.rs | 15 +++++++-- src/comp/middle/trans.rs | 32 ++++++++++++++---- src/lib/ivec.rs | 9 ++--- src/rt/rust_builtin.cpp | 60 +++++++++++++++++++++++++++++++++ src/rt/rust_upcall.cpp | 72 ++++++++++++++++++++++++++++++++++++++++ src/rt/rustrt.def.in | 6 ++++ 6 files changed, 182 insertions(+), 12 deletions(-) diff --git a/src/comp/back/upcall.rs b/src/comp/back/upcall.rs index 5fad90ce031..db750fe7a6d 100644 --- a/src/comp/back/upcall.rs +++ b/src/comp/back/upcall.rs @@ -47,6 +47,8 @@ type upcalls = ValueRef exit, ValueRef malloc, ValueRef free, + ValueRef shared_malloc, + ValueRef shared_free, ValueRef mark, ValueRef new_str, ValueRef dup_str, @@ -56,7 +58,9 @@ type upcalls = ValueRef new_task, ValueRef start_task, ValueRef ivec_resize, - ValueRef ivec_spill); + ValueRef ivec_spill, + ValueRef ivec_resize_shared, + ValueRef ivec_spill_shared); fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls { fn decl(type_names tn, ModuleRef llmod, str name, vec[TypeRef] tys, @@ -97,6 +101,9 @@ fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls { malloc=d("malloc", [T_size_t(), T_ptr(T_tydesc(tn))], T_ptr(T_i8())), free=dv("free", [T_ptr(T_i8()), T_int()]), + shared_malloc=d("shared_malloc", + [T_size_t(), T_ptr(T_tydesc(tn))], T_ptr(T_i8())), + shared_free=dv("shared_free", [T_ptr(T_i8())]), mark=d("mark", [T_ptr(T_i8())], T_int()), new_str=d("new_str", [T_ptr(T_i8()), T_size_t()], T_ptr(T_str())), @@ -119,7 +126,11 @@ fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls { ivec_resize=d("ivec_resize", [T_ptr(T_opaque_ivec()), T_int()], T_void()), ivec_spill=d("ivec_spill", [T_ptr(T_opaque_ivec()), T_int()], - T_void())); + T_void()), + ivec_resize_shared=d("ivec_resize_shared", + [T_ptr(T_opaque_ivec()), T_int()], T_void()), + ivec_spill_shared=d("ivec_spill_shared", + [T_ptr(T_opaque_ivec()), T_int()], T_void())); } // // Local Variables: diff --git a/src/comp/middle/trans.rs b/src/comp/middle/trans.rs index e8139432ad2..6a7b5a412de 100644 --- a/src/comp/middle/trans.rs +++ b/src/comp/middle/trans.rs @@ -1150,6 +1150,12 @@ fn trans_non_gc_free(&@block_ctxt cx, ValueRef v) -> result { ret rslt(cx, C_int(0)); } +fn trans_shared_free(&@block_ctxt cx, ValueRef v) -> result { + cx.build.Call(cx.fcx.lcx.ccx.upcalls.shared_free, + [cx.fcx.lltaskptr, cx.build.PointerCast(v, T_ptr(T_i8()))]); + ret rslt(cx, C_int(0)); +} + fn find_scope_cx(&@block_ctxt cx) -> @block_ctxt { if (cx.kind != NON_SCOPE_BLOCK) { ret cx; } alt (cx.parent) { @@ -1578,6 +1584,18 @@ fn trans_raw_malloc(&@block_ctxt cx, TypeRef llptr_ty, ValueRef llsize) -> ret rslt(cx, cx.build.PointerCast(rval, llptr_ty)); } +// trans_shared_malloc: expects a type indicating which pointer type we want +// and a size indicating how much space we want malloc'd. +fn trans_shared_malloc(&@block_ctxt cx, TypeRef llptr_ty, ValueRef llsize) -> + result { + // FIXME: need a table to collect tydesc globals. + + auto tydesc = C_null(T_ptr(T_tydesc(cx.fcx.lcx.ccx.tn))); + auto rval = + cx.build.Call(cx.fcx.lcx.ccx.upcalls.shared_malloc, + [cx.fcx.lltaskptr, llsize, tydesc]); + ret rslt(cx, cx.build.PointerCast(rval, llptr_ty)); +} // trans_malloc_boxed: expects an unboxed type and returns a pointer to enough // space for something of that type, along with space for a reference count; @@ -2097,7 +2115,7 @@ fn maybe_free_ivec_heap_part(&@block_ctxt cx, ValueRef v0, ty::t unit_ty) -> auto m = maybe_on_heap_cx.build.InBoundsGEP(stub_ptr, v); maybe_on_heap_cx.build.Load(m) }; - auto after_free_cx = trans_non_gc_free(maybe_on_heap_cx, heap_ptr).bcx; + auto after_free_cx = trans_shared_free(maybe_on_heap_cx, heap_ptr).bcx; after_free_cx.build.Br(next_cx.llbb); ret rslt(next_cx, C_nil()); } @@ -3617,7 +3635,8 @@ mod ivec { { auto p = heap_resize_cx.build.PointerCast(v, T_ptr(T_opaque_ivec())); - heap_resize_cx.build.Call(cx.fcx.lcx.ccx.upcalls.ivec_resize, + auto upcall = cx.fcx.lcx.ccx.upcalls.ivec_resize_shared; + heap_resize_cx.build.Call(upcall, [cx.fcx.lltaskptr, p, new_heap_len]); } auto heap_ptr_resize = @@ -3657,7 +3676,8 @@ mod ivec { { auto p = stack_spill_cx.build.PointerCast(v, T_ptr(T_opaque_ivec())); - stack_spill_cx.build.Call(cx.fcx.lcx.ccx.upcalls.ivec_spill, + auto upcall = cx.fcx.lcx.ccx.upcalls.ivec_spill_shared; + stack_spill_cx.build.Call(upcall, [cx.fcx.lltaskptr, p, new_stack_len]); } auto spill_stub = @@ -3908,7 +3928,7 @@ mod ivec { heap_cx.build.InBoundsGEP(stub_ptr_heap, stub_a)); auto heap_sz = heap_cx.build.Add(llsize_of(llheappartty), lllen); - auto rs = trans_raw_malloc(heap_cx, T_ptr(llheappartty), heap_sz); + auto rs = trans_shared_malloc(heap_cx, T_ptr(llheappartty), heap_sz); auto heap_part = rs.val; heap_cx = rs.bcx; heap_cx.build.Store(heap_part, @@ -4045,7 +4065,7 @@ mod ivec { auto heap_part_sz = on_heap_cx.build.Add(alen, llsize_of(T_opaque_ivec_heap_part())); - auto rs = trans_raw_malloc(on_heap_cx, T_ptr(llheappartty), + auto rs = trans_shared_malloc(on_heap_cx, T_ptr(llheappartty), heap_part_sz); on_heap_cx = rs.bcx; auto new_heap_ptr = rs.val; @@ -6003,7 +6023,7 @@ fn trans_ivec(@block_ctxt bcx, &vec[@ast::expr] args, ast::node_id id) -> bcx.build.Store(lllen, bcx.build.InBoundsGEP(llstubptr, stub_a)); auto llheapsz = bcx.build.Add(llsize_of(llheapty), lllen); - auto rslt = trans_raw_malloc(bcx, T_ptr(llheapty), llheapsz); + auto rslt = trans_shared_malloc(bcx, T_ptr(llheapty), llheapsz); bcx = rslt.bcx; auto llheapptr = rslt.val; bcx.build.Store(llheapptr, diff --git a/src/lib/ivec.rs b/src/lib/ivec.rs index a9a5509dae9..d34b54d68e4 100644 --- a/src/lib/ivec.rs +++ b/src/lib/ivec.rs @@ -11,15 +11,16 @@ native "rust-intrinsic" mod rusti { } native "rust" mod rustrt { - fn ivec_reserve[T](&mutable T[mutable?] v, uint n); + fn ivec_reserve_shared[T](&mutable T[mutable?] v, uint n); fn ivec_on_heap[T](&T[] v) -> uint; fn ivec_to_ptr[T](&T[] v) -> *T; - fn ivec_copy_from_buf[T](&mutable T[mutable?] v, *T ptr, uint count); + fn ivec_copy_from_buf_shared[T](&mutable T[mutable?] v, + *T ptr, uint count); } /// Reserves space for `n` elements in the given vector. fn reserve[T](&mutable T[mutable?] v, uint n) { - rustrt::ivec_reserve(v, n); + rustrt::ivec_reserve_shared(v, n); } fn on_heap[T](&T[] v) -> bool { @@ -204,7 +205,7 @@ fn all[T](fn(&T)->bool f, &T[] v) -> bool { mod unsafe { fn copy_from_buf[T](&mutable T[] v, *T ptr, uint count) { - ret rustrt::ivec_copy_from_buf(v, ptr, count); + ret rustrt::ivec_copy_from_buf_shared(v, ptr, count); } } diff --git a/src/rt/rust_builtin.cpp b/src/rt/rust_builtin.cpp index 6438c549c55..872a317ca38 100644 --- a/src/rt/rust_builtin.cpp +++ b/src/rt/rust_builtin.cpp @@ -648,6 +648,37 @@ ivec_reserve(rust_task *task, type_desc *ty, rust_ivec *v, size_t n_elems) v->alloc = new_alloc; } +/** + * Preallocates the exact number of bytes in the given interior vector. + */ +extern "C" CDECL void +ivec_reserve_shared(rust_task *task, type_desc *ty, rust_ivec *v, + size_t n_elems) +{ + size_t new_alloc = n_elems * ty->size; + if (new_alloc <= v->alloc) + return; // Already big enough. + + rust_ivec_heap *heap_part; + if (v->fill || !v->payload.ptr) { + // On stack; spill to heap. + heap_part = (rust_ivec_heap *)task->kernel->malloc(new_alloc + + sizeof(size_t)); + heap_part->fill = v->fill; + memcpy(&heap_part->data, v->payload.data, v->fill); + + v->fill = 0; + v->payload.ptr = heap_part; + } else { + // On heap; resize. + heap_part = (rust_ivec_heap *)task->kernel->realloc(v->payload.ptr, + new_alloc + sizeof(size_t)); + v->payload.ptr = heap_part; + } + + v->alloc = new_alloc; +} + /** * Returns true if the given vector is on the heap and false if it's on the * stack. @@ -706,6 +737,35 @@ ivec_copy_from_buf(rust_task *task, type_desc *ty, rust_ivec *v, void *ptr, v->payload.ptr->fill = new_size; } +/** + * Copies elements in an unsafe buffer to the given interior vector. The + * vector must have size zero. + */ +extern "C" CDECL void +ivec_copy_from_buf_shared(rust_task *task, type_desc *ty, rust_ivec *v, + void *ptr, size_t count) +{ + size_t old_size = get_ivec_size(v); + if (old_size) { + task->fail(1); + return; + } + + ivec_reserve_shared(task, ty, v, count); + + size_t new_size = count * ty->size; + if (v->fill || !v->payload.ptr) { + // On stack. + memmove(v->payload.data, ptr, new_size); + v->fill = new_size; + return; + } + + // On heap. + memmove(v->payload.ptr->data, ptr, new_size); + v->payload.ptr->fill = new_size; +} + extern "C" CDECL void pin_task(rust_task *task) { task->pin(); diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp index d8ff9236aa6..a89c01ebdfb 100644 --- a/src/rt/rust_upcall.cpp +++ b/src/rt/rust_upcall.cpp @@ -289,6 +289,36 @@ upcall_free(rust_task *task, void* ptr, uintptr_t is_gc) { task->free(ptr, (bool) is_gc); } +extern "C" CDECL uintptr_t +upcall_shared_malloc(rust_task *task, size_t nbytes, type_desc *td) { + LOG_UPCALL_ENTRY(task); + scoped_lock with(task->kernel->scheduler_lock); + + LOG(task, mem, + "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR ")", + nbytes, td); + void *p = task->kernel->malloc(nbytes); + LOG(task, mem, + "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR + ") = 0x%" PRIxPTR, + nbytes, td, (uintptr_t)p); + return (uintptr_t) p; +} + +/** + * Called whenever an object's ref count drops to zero. + */ +extern "C" CDECL void +upcall_shared_free(rust_task *task, void* ptr) { + LOG_UPCALL_ENTRY(task); + scoped_lock with(task->kernel->scheduler_lock); + rust_scheduler *sched = task->sched; + DLOG(sched, mem, + "upcall shared_free(0x%" PRIxPTR")", + (uintptr_t)ptr); + task->kernel->free(ptr); +} + extern "C" CDECL uintptr_t upcall_mark(rust_task *task, void* ptr) { LOG_UPCALL_ENTRY(task); @@ -537,6 +567,7 @@ extern "C" CDECL void upcall_ivec_resize(rust_task *task, rust_ivec *v, size_t newsz) { + LOG_UPCALL_ENTRY(task); scoped_lock with(task->kernel->scheduler_lock); I(task->sched, !v->fill); @@ -556,6 +587,7 @@ extern "C" CDECL void upcall_ivec_spill(rust_task *task, rust_ivec *v, size_t newsz) { + LOG_UPCALL_ENTRY(task); scoped_lock with(task->kernel->scheduler_lock); size_t new_alloc = next_power_of_two(newsz); @@ -569,6 +601,46 @@ upcall_ivec_spill(rust_task *task, v->payload.ptr = heap_part; } +/** + * Resizes an interior vector that has been spilled to the heap. + */ +extern "C" CDECL void +upcall_ivec_resize_shared(rust_task *task, + rust_ivec *v, + size_t newsz) { + LOG_UPCALL_ENTRY(task); + scoped_lock with(task->kernel->scheduler_lock); + I(task->sched, !v->fill); + + size_t new_alloc = next_power_of_two(newsz); + rust_ivec_heap *new_heap_part = (rust_ivec_heap *) + task->kernel->realloc(v->payload.ptr, new_alloc + sizeof(size_t)); + + new_heap_part->fill = newsz; + v->alloc = new_alloc; + v->payload.ptr = new_heap_part; +} + +/** + * Spills an interior vector to the heap. + */ +extern "C" CDECL void +upcall_ivec_spill_shared(rust_task *task, + rust_ivec *v, + size_t newsz) { + LOG_UPCALL_ENTRY(task); + scoped_lock with(task->kernel->scheduler_lock); + size_t new_alloc = next_power_of_two(newsz); + + rust_ivec_heap *heap_part = (rust_ivec_heap *) + task->kernel->malloc(new_alloc + sizeof(size_t)); + heap_part->fill = newsz; + memcpy(&heap_part->data, v->payload.data, v->fill); + + v->fill = 0; + v->alloc = new_alloc; + v->payload.ptr = heap_part; +} // // Local Variables: // mode: C++ diff --git a/src/rt/rustrt.def.in b/src/rt/rustrt.def.in index 444faccef26..481936eee69 100644 --- a/src/rt/rustrt.def.in +++ b/src/rt/rustrt.def.in @@ -11,8 +11,10 @@ debug_tydesc do_gc get_time ivec_copy_from_buf +ivec_copy_from_buf_shared ivec_on_heap ivec_reserve +ivec_reserve_shared ivec_to_ptr last_os_error nano_time @@ -59,7 +61,9 @@ upcall_free upcall_get_type_desc upcall_grow_task upcall_ivec_resize +upcall_ivec_resize_shared upcall_ivec_spill +upcall_ivec_spill_shared upcall_kill upcall_log_double upcall_log_float @@ -74,6 +78,8 @@ upcall_new_task upcall_new_vec upcall_recv upcall_send +upcall_shared_malloc +upcall_shared_free upcall_sleep upcall_start_task upcall_trace_str