001
002
003
004
005
006
007
008
009
010
011
012
013
014 #include "ruby/ruby.h"
015 #include "ruby/st.h"
016 #include "ruby/re.h"
017 #include "ruby/io.h"
018 #include "ruby/thread.h"
019 #include "ruby/util.h"
020 #include "eval_intern.h"
021 #include "vm_core.h"
022 #include "internal.h"
023 #include "gc.h"
024 #include "constant.h"
025 #include "ruby_atomic.h"
026 #include "probes.h"
027 #include <stdio.h>
028 #include <setjmp.h>
029 #include <sys/types.h>
030 #include <assert.h>
031
032 #ifdef HAVE_SYS_TIME_H
033 #include <sys/time.h>
034 #endif
035
036 #ifdef HAVE_SYS_RESOURCE_H
037 #include <sys/resource.h>
038 #endif
039 #if defined(__native_client__) && defined(NACL_NEWLIB)
040 # include "nacl/resource.h"
041 # undef HAVE_POSIX_MEMALIGN
042 # undef HAVE_MEMALIGN
043
044 #endif
045
046 #if defined _WIN32 || defined __CYGWIN__
047 #include <windows.h>
048 #elif defined(HAVE_POSIX_MEMALIGN)
049 #elif defined(HAVE_MEMALIGN)
050 #include <malloc.h>
051 #endif
052
053 #ifdef HAVE_VALGRIND_MEMCHECK_H
054 # include <valgrind/memcheck.h>
055 # ifndef VALGRIND_MAKE_MEM_DEFINED
056 # define VALGRIND_MAKE_MEM_DEFINED(p, n) VALGRIND_MAKE_READABLE((p), (n))
057 # endif
058 # ifndef VALGRIND_MAKE_MEM_UNDEFINED
059 # define VALGRIND_MAKE_MEM_UNDEFINED(p, n) VALGRIND_MAKE_WRITABLE((p), (n))
060 # endif
061 #else
062 # define VALGRIND_MAKE_MEM_DEFINED(p, n) 0
063 # define VALGRIND_MAKE_MEM_UNDEFINED(p, n) 0
064 #endif
065
066 #define rb_setjmp(env) RUBY_SETJMP(env)
067 #define rb_jmp_buf rb_jmpbuf_t
068
069 #ifndef GC_MALLOC_LIMIT
070 #define GC_MALLOC_LIMIT 8000000
071 #endif
072 #define HEAP_MIN_SLOTS 10000
073 #define FREE_MIN 4096
074 #define HEAP_GROWTH_FACTOR 1.8
075
076 typedef struct {
077 unsigned int initial_malloc_limit;
078 unsigned int initial_heap_min_slots;
079 unsigned int initial_free_min;
080 double initial_growth_factor;
081 #if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
082 int gc_stress;
083 #endif
084 } ruby_gc_params_t;
085
086 static ruby_gc_params_t initial_params = {
087 GC_MALLOC_LIMIT,
088 HEAP_MIN_SLOTS,
089 FREE_MIN,
090 HEAP_GROWTH_FACTOR,
091 #if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
092 FALSE,
093 #endif
094 };
095
096 #define nomem_error GET_VM()->special_exceptions[ruby_error_nomemory]
097
098 #ifndef GC_PROFILE_MORE_DETAIL
099 #define GC_PROFILE_MORE_DETAIL 0
100 #endif
101 #ifndef GC_ENABLE_LAZY_SWEEP
102 #define GC_ENABLE_LAZY_SWEEP 1
103 #endif
104
105 typedef struct gc_profile_record {
106 double gc_time;
107 double gc_invoke_time;
108
109 size_t heap_total_objects;
110 size_t heap_use_size;
111 size_t heap_total_size;
112
113 int is_marked;
114
115 #if GC_PROFILE_MORE_DETAIL
116 double gc_mark_time;
117 double gc_sweep_time;
118
119 size_t heap_use_slots;
120 size_t heap_live_objects;
121 size_t heap_free_objects;
122
123 int have_finalize;
124
125 size_t allocate_increase;
126 size_t allocate_limit;
127 #endif
128 } gc_profile_record;
129
130 #if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN__)
131 #pragma pack(push, 1)
132 #endif
133
134 typedef struct RVALUE {
135 union {
136 struct {
137 VALUE flags;
138 struct RVALUE *next;
139 } free;
140 struct RBasic basic;
141 struct RObject object;
142 struct RClass klass;
143 struct RFloat flonum;
144 struct RString string;
145 struct RArray array;
146 struct RRegexp regexp;
147 struct RHash hash;
148 struct RData data;
149 struct RTypedData typeddata;
150 struct RStruct rstruct;
151 struct RBignum bignum;
152 struct RFile file;
153 struct RNode node;
154 struct RMatch match;
155 struct RRational rational;
156 struct RComplex complex;
157 } as;
158 #ifdef GC_DEBUG
159 const char *file;
160 int line;
161 #endif
162 } RVALUE;
163
164 #if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN__)
165 #pragma pack(pop)
166 #endif
167
168 struct heaps_slot {
169 struct heaps_header *header;
170 uintptr_t *bits;
171 RVALUE *freelist;
172 struct heaps_slot *next;
173 struct heaps_slot *prev;
174 struct heaps_slot *free_next;
175 };
176
177 struct heaps_header {
178 struct heaps_slot *base;
179 uintptr_t *bits;
180 RVALUE *start;
181 RVALUE *end;
182 size_t limit;
183 };
184
185 struct heaps_free_bitmap {
186 struct heaps_free_bitmap *next;
187 };
188
189 struct gc_list {
190 VALUE *varptr;
191 struct gc_list *next;
192 };
193
194 #define STACK_CHUNK_SIZE 500
195
196 typedef struct stack_chunk {
197 VALUE data[STACK_CHUNK_SIZE];
198 struct stack_chunk *next;
199 } stack_chunk_t;
200
201 typedef struct mark_stack {
202 stack_chunk_t *chunk;
203 stack_chunk_t *cache;
204 size_t index;
205 size_t limit;
206 size_t cache_size;
207 size_t unused_cache_size;
208 } mark_stack_t;
209
210 #ifndef CALC_EXACT_MALLOC_SIZE
211 #define CALC_EXACT_MALLOC_SIZE 0
212 #endif
213
214 typedef struct rb_objspace {
215 struct {
216 size_t limit;
217 size_t increase;
218 #if CALC_EXACT_MALLOC_SIZE
219 size_t allocated_size;
220 size_t allocations;
221 #endif
222 } malloc_params;
223 struct {
224 size_t increment;
225 struct heaps_slot *ptr;
226 struct heaps_slot *sweep_slots;
227 struct heaps_slot *free_slots;
228 struct heaps_header **sorted;
229 size_t length;
230 size_t used;
231 struct heaps_free_bitmap *free_bitmap;
232 RVALUE *range[2];
233 struct heaps_header *freed;
234 size_t marked_num;
235 size_t free_num;
236 size_t free_min;
237 size_t final_num;
238 size_t do_heap_free;
239 } heap;
240 struct {
241 int dont_gc;
242 int dont_lazy_sweep;
243 int during_gc;
244 rb_atomic_t finalizing;
245 } flags;
246 struct {
247 st_table *table;
248 RVALUE *deferred;
249 } final;
250 mark_stack_t mark_stack;
251 struct {
252 int run;
253 gc_profile_record *record;
254 size_t count;
255 size_t size;
256 double invoke_time;
257 } profile;
258 struct gc_list *global_list;
259 size_t count;
260 size_t total_allocated_object_num;
261 size_t total_freed_object_num;
262 int gc_stress;
263
264 struct mark_func_data_struct {
265 void *data;
266 void (*mark_func)(VALUE v, void *data);
267 } *mark_func_data;
268 } rb_objspace_t;
269
270 #if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
271 #define rb_objspace (*GET_VM()->objspace)
272 #define ruby_initial_gc_stress initial_params.gc_stress
273 int *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress;
274 #else
275 static rb_objspace_t rb_objspace = {{GC_MALLOC_LIMIT}};
276 int *ruby_initial_gc_stress_ptr = &rb_objspace.gc_stress;
277 #endif
278 #define malloc_limit objspace->malloc_params.limit
279 #define malloc_increase objspace->malloc_params.increase
280 #define heaps objspace->heap.ptr
281 #define heaps_length objspace->heap.length
282 #define heaps_used objspace->heap.used
283 #define lomem objspace->heap.range[0]
284 #define himem objspace->heap.range[1]
285 #define heaps_inc objspace->heap.increment
286 #define heaps_freed objspace->heap.freed
287 #define dont_gc objspace->flags.dont_gc
288 #define during_gc objspace->flags.during_gc
289 #define finalizing objspace->flags.finalizing
290 #define finalizer_table objspace->final.table
291 #define deferred_final_list objspace->final.deferred
292 #define global_List objspace->global_list
293 #define ruby_gc_stress objspace->gc_stress
294 #define initial_malloc_limit initial_params.initial_malloc_limit
295 #define initial_heap_min_slots initial_params.initial_heap_min_slots
296 #define initial_free_min initial_params.initial_free_min
297 #define initial_growth_factor initial_params.initial_growth_factor
298
299 #define is_lazy_sweeping(objspace) ((objspace)->heap.sweep_slots != 0)
300
301 #if SIZEOF_LONG == SIZEOF_VOIDP
302 # define nonspecial_obj_id(obj) (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG)
303 # define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG)
304 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
305 # define nonspecial_obj_id(obj) LL2NUM((SIGNED_VALUE)(obj) / 2)
306 # define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
307 ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
308 #else
309 # error not supported
310 #endif
311
312 #define RANY(o) ((RVALUE*)(o))
313 #define has_free_object (objspace->heap.free_slots && objspace->heap.free_slots->freelist)
314
315 #define HEAP_HEADER(p) ((struct heaps_header *)(p))
316 #define GET_HEAP_HEADER(x) (HEAP_HEADER((uintptr_t)(x) & ~(HEAP_ALIGN_MASK)))
317 #define GET_HEAP_SLOT(x) (GET_HEAP_HEADER(x)->base)
318 #define GET_HEAP_BITMAP(x) (GET_HEAP_HEADER(x)->bits)
319 #define NUM_IN_SLOT(p) (((uintptr_t)(p) & HEAP_ALIGN_MASK)/sizeof(RVALUE))
320 #define BITMAP_INDEX(p) (NUM_IN_SLOT(p) / (sizeof(uintptr_t) * CHAR_BIT))
321 #define BITMAP_OFFSET(p) (NUM_IN_SLOT(p) & ((sizeof(uintptr_t) * CHAR_BIT)-1))
322 #define MARKED_IN_BITMAP(bits, p) (bits[BITMAP_INDEX(p)] & ((uintptr_t)1 << BITMAP_OFFSET(p)))
323
324 #ifndef HEAP_ALIGN_LOG
325
326 #define HEAP_ALIGN_LOG 14
327 #endif
328
329 #define CEILDIV(i, mod) (((i) + (mod) - 1)/(mod))
330
331 enum {
332 HEAP_ALIGN = (1UL << HEAP_ALIGN_LOG),
333 HEAP_ALIGN_MASK = (~(~0UL << HEAP_ALIGN_LOG)),
334 REQUIRED_SIZE_BY_MALLOC = (sizeof(size_t) * 5),
335 HEAP_SIZE = (HEAP_ALIGN - REQUIRED_SIZE_BY_MALLOC),
336 HEAP_OBJ_LIMIT = (unsigned int)((HEAP_SIZE - sizeof(struct heaps_header))/sizeof(struct RVALUE)),
337 HEAP_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_SIZE, sizeof(struct RVALUE)), sizeof(uintptr_t) * CHAR_BIT)
338 };
339
340 int ruby_gc_debug_indent = 0;
341 VALUE rb_mGC;
342 extern st_table *rb_class_tbl;
343 int ruby_disable_gc_stress = 0;
344
345 static void rb_objspace_call_finalizer(rb_objspace_t *objspace);
346 static VALUE define_final0(VALUE obj, VALUE block);
347 VALUE rb_define_final(VALUE obj, VALUE block);
348 VALUE rb_undefine_final(VALUE obj);
349 static void run_final(rb_objspace_t *objspace, VALUE obj);
350 static void initial_expand_heap(rb_objspace_t *objspace);
351
352 static void negative_size_allocation_error(const char *);
353 static void *aligned_malloc(size_t, size_t);
354 static void aligned_free(void *);
355
356 static void init_mark_stack(mark_stack_t *stack);
357
358 static VALUE lazy_sweep_enable(void);
359 static int garbage_collect(rb_objspace_t *);
360 static int gc_prepare_free_objects(rb_objspace_t *);
361 static void mark_tbl(rb_objspace_t *, st_table *);
362 static void rest_sweep(rb_objspace_t *);
363 static void gc_mark_stacked_objects(rb_objspace_t *);
364
365 static double getrusage_time(void);
366 static inline void gc_prof_timer_start(rb_objspace_t *);
367 static inline void gc_prof_timer_stop(rb_objspace_t *, int);
368 static inline void gc_prof_mark_timer_start(rb_objspace_t *);
369 static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
370 static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
371 static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
372 static inline void gc_prof_set_malloc_info(rb_objspace_t *);
373
374
375
376
377
378
379 #if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
380 rb_objspace_t *
381 rb_objspace_alloc(void)
382 {
383 rb_objspace_t *objspace = malloc(sizeof(rb_objspace_t));
384 memset(objspace, 0, sizeof(*objspace));
385 malloc_limit = initial_malloc_limit;
386 ruby_gc_stress = ruby_initial_gc_stress;
387
388 return objspace;
389 }
390 #endif
391
392 #if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
393 static void free_stack_chunks(mark_stack_t *);
394
395 void
396 rb_objspace_free(rb_objspace_t *objspace)
397 {
398 rest_sweep(objspace);
399 if (objspace->profile.record) {
400 free(objspace->profile.record);
401 objspace->profile.record = 0;
402 }
403 if (global_List) {
404 struct gc_list *list, *next;
405 for (list = global_List; list; list = next) {
406 next = list->next;
407 xfree(list);
408 }
409 }
410 if (objspace->heap.free_bitmap) {
411 struct heaps_free_bitmap *list, *next;
412 for (list = objspace->heap.free_bitmap; list; list = next) {
413 next = list->next;
414 free(list);
415 }
416 }
417 if (objspace->heap.sorted) {
418 size_t i;
419 for (i = 0; i < heaps_used; ++i) {
420 free(objspace->heap.sorted[i]->bits);
421 aligned_free(objspace->heap.sorted[i]);
422 }
423 free(objspace->heap.sorted);
424 heaps_used = 0;
425 heaps = 0;
426 }
427 free_stack_chunks(&objspace->mark_stack);
428 free(objspace);
429 }
430 #endif
431
432 void
433 rb_global_variable(VALUE *var)
434 {
435 rb_gc_register_address(var);
436 }
437
438 static void
439 allocate_sorted_heaps(rb_objspace_t *objspace, size_t next_heaps_length)
440 {
441 struct heaps_header **p;
442 struct heaps_free_bitmap *bits;
443 size_t size, add, i;
444
445 size = next_heaps_length*sizeof(struct heaps_header *);
446 add = next_heaps_length - heaps_used;
447
448 if (heaps_used > 0) {
449 p = (struct heaps_header **)realloc(objspace->heap.sorted, size);
450 if (p) objspace->heap.sorted = p;
451 }
452 else {
453 p = objspace->heap.sorted = (struct heaps_header **)malloc(size);
454 }
455
456 if (p == 0) {
457 during_gc = 0;
458 rb_memerror();
459 }
460
461 for (i = 0; i < add; i++) {
462 bits = (struct heaps_free_bitmap *)malloc(HEAP_BITMAP_LIMIT * sizeof(uintptr_t));
463 if (bits == 0) {
464 during_gc = 0;
465 rb_memerror();
466 return;
467 }
468 bits->next = objspace->heap.free_bitmap;
469 objspace->heap.free_bitmap = bits;
470 }
471 }
472
473 static void
474 link_free_heap_slot(rb_objspace_t *objspace, struct heaps_slot *slot)
475 {
476 slot->free_next = objspace->heap.free_slots;
477 objspace->heap.free_slots = slot;
478 }
479
480 static void
481 unlink_free_heap_slot(rb_objspace_t *objspace, struct heaps_slot *slot)
482 {
483 objspace->heap.free_slots = slot->free_next;
484 slot->free_next = NULL;
485 }
486
487 static void
488 assign_heap_slot(rb_objspace_t *objspace)
489 {
490 RVALUE *p, *pend, *membase;
491 struct heaps_slot *slot;
492 size_t hi, lo, mid;
493 size_t objs;
494
495 objs = HEAP_OBJ_LIMIT;
496 p = (RVALUE*)aligned_malloc(HEAP_ALIGN, HEAP_SIZE);
497 if (p == 0) {
498 during_gc = 0;
499 rb_memerror();
500 }
501 slot = (struct heaps_slot *)malloc(sizeof(struct heaps_slot));
502 if (slot == 0) {
503 aligned_free(p);
504 during_gc = 0;
505 rb_memerror();
506 }
507 MEMZERO((void*)slot, struct heaps_slot, 1);
508
509 slot->next = heaps;
510 if (heaps) heaps->prev = slot;
511 heaps = slot;
512
513 membase = p;
514 p = (RVALUE*)((VALUE)p + sizeof(struct heaps_header));
515 if ((VALUE)p % sizeof(RVALUE) != 0) {
516 p = (RVALUE*)((VALUE)p + sizeof(RVALUE) - ((VALUE)p % sizeof(RVALUE)));
517 objs = (HEAP_SIZE - (size_t)((VALUE)p - (VALUE)membase))/sizeof(RVALUE);
518 }
519
520 lo = 0;
521 hi = heaps_used;
522 while (lo < hi) {
523 register RVALUE *mid_membase;
524 mid = (lo + hi) / 2;
525 mid_membase = (RVALUE *)objspace->heap.sorted[mid];
526 if (mid_membase < membase) {
527 lo = mid + 1;
528 }
529 else if (mid_membase > membase) {
530 hi = mid;
531 }
532 else {
533 rb_bug("same heap slot is allocated: %p at %"PRIuVALUE, (void *)membase, (VALUE)mid);
534 }
535 }
536 if (hi < heaps_used) {
537 MEMMOVE(&objspace->heap.sorted[hi+1], &objspace->heap.sorted[hi], struct heaps_header*, heaps_used - hi);
538 }
539 heaps->header = (struct heaps_header *)membase;
540 objspace->heap.sorted[hi] = heaps->header;
541 objspace->heap.sorted[hi]->start = p;
542 objspace->heap.sorted[hi]->end = (p + objs);
543 objspace->heap.sorted[hi]->base = heaps;
544 objspace->heap.sorted[hi]->limit = objs;
545 assert(objspace->heap.free_bitmap != NULL);
546 heaps->bits = (uintptr_t *)objspace->heap.free_bitmap;
547 objspace->heap.sorted[hi]->bits = (uintptr_t *)objspace->heap.free_bitmap;
548 objspace->heap.free_bitmap = objspace->heap.free_bitmap->next;
549 memset(heaps->bits, 0, HEAP_BITMAP_LIMIT * sizeof(uintptr_t));
550 pend = p + objs;
551 if (lomem == 0 || lomem > p) lomem = p;
552 if (himem < pend) himem = pend;
553 heaps_used++;
554
555 while (p < pend) {
556 p->as.free.flags = 0;
557 p->as.free.next = heaps->freelist;
558 heaps->freelist = p;
559 p++;
560 }
561 link_free_heap_slot(objspace, heaps);
562 }
563
564 static void
565 add_heap_slots(rb_objspace_t *objspace, size_t add)
566 {
567 size_t i;
568 size_t next_heaps_length;
569
570 next_heaps_length = heaps_used + add;
571
572 if (next_heaps_length > heaps_length) {
573 allocate_sorted_heaps(objspace, next_heaps_length);
574 heaps_length = next_heaps_length;
575 }
576
577 for (i = 0; i < add; i++) {
578 assign_heap_slot(objspace);
579 }
580 heaps_inc = 0;
581 }
582
583 static void
584 init_heap(rb_objspace_t *objspace)
585 {
586 add_heap_slots(objspace, HEAP_MIN_SLOTS / HEAP_OBJ_LIMIT);
587 init_mark_stack(&objspace->mark_stack);
588
589 #ifdef USE_SIGALTSTACK
590 {
591
592 rb_thread_t *th = GET_THREAD();
593 void *tmp = th->altstack;
594 th->altstack = malloc(rb_sigaltstack_size());
595 free(tmp);
596 }
597 #endif
598
599 objspace->profile.invoke_time = getrusage_time();
600 finalizer_table = st_init_numtable();
601 }
602
603 static void
604 initial_expand_heap(rb_objspace_t *objspace)
605 {
606 size_t min_size = initial_heap_min_slots / HEAP_OBJ_LIMIT;
607
608 if (min_size > heaps_used) {
609 add_heap_slots(objspace, min_size - heaps_used);
610 }
611 }
612
613 static void
614 set_heaps_increment(rb_objspace_t *objspace)
615 {
616 size_t next_heaps_length = (size_t)(heaps_used * initial_growth_factor);
617
618 if (next_heaps_length == heaps_used) {
619 next_heaps_length++;
620 }
621
622 heaps_inc = next_heaps_length - heaps_used;
623
624 if (next_heaps_length > heaps_length) {
625 allocate_sorted_heaps(objspace, next_heaps_length);
626 heaps_length = next_heaps_length;
627 }
628 }
629
630 static int
631 heaps_increment(rb_objspace_t *objspace)
632 {
633 if (heaps_inc > 0) {
634 assign_heap_slot(objspace);
635 heaps_inc--;
636 return TRUE;
637 }
638 return FALSE;
639 }
640
641 static VALUE
642 newobj(VALUE klass, VALUE flags)
643 {
644 rb_objspace_t *objspace = &rb_objspace;
645 VALUE obj;
646
647 if (UNLIKELY(during_gc)) {
648 dont_gc = 1;
649 during_gc = 0;
650 rb_bug("object allocation during garbage collection phase");
651 }
652
653 if (UNLIKELY(ruby_gc_stress && !ruby_disable_gc_stress)) {
654 if (!garbage_collect(objspace)) {
655 during_gc = 0;
656 rb_memerror();
657 }
658 }
659
660 if (UNLIKELY(!has_free_object)) {
661 if (!gc_prepare_free_objects(objspace)) {
662 during_gc = 0;
663 rb_memerror();
664 }
665 }
666
667 obj = (VALUE)objspace->heap.free_slots->freelist;
668 objspace->heap.free_slots->freelist = RANY(obj)->as.free.next;
669 if (objspace->heap.free_slots->freelist == NULL) {
670 unlink_free_heap_slot(objspace, objspace->heap.free_slots);
671 }
672
673 MEMZERO((void*)obj, RVALUE, 1);
674 #ifdef GC_DEBUG
675 RANY(obj)->file = rb_sourcefile();
676 RANY(obj)->line = rb_sourceline();
677 #endif
678 objspace->total_allocated_object_num++;
679
680 return obj;
681 }
682
683 VALUE
684 rb_newobj(void)
685 {
686 return newobj(0, T_NONE);
687 }
688
689 VALUE
690 rb_newobj_of(VALUE klass, VALUE flags)
691 {
692 VALUE obj;
693
694 obj = newobj(klass, flags);
695 OBJSETUP(obj, klass, flags);
696
697 return obj;
698 }
699
700 NODE*
701 rb_node_newnode(enum node_type type, VALUE a0, VALUE a1, VALUE a2)
702 {
703 NODE *n = (NODE*)rb_newobj();
704
705 n->flags |= T_NODE;
706 nd_set_type(n, type);
707
708 n->u1.value = a0;
709 n->u2.value = a1;
710 n->u3.value = a2;
711
712 return n;
713 }
714
715 VALUE
716 rb_data_object_alloc(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
717 {
718 NEWOBJ(data, struct RData);
719 if (klass) Check_Type(klass, T_CLASS);
720 OBJSETUP(data, klass, T_DATA);
721 data->data = datap;
722 data->dfree = dfree;
723 data->dmark = dmark;
724
725 return (VALUE)data;
726 }
727
728 VALUE
729 rb_data_typed_object_alloc(VALUE klass, void *datap, const rb_data_type_t *type)
730 {
731 NEWOBJ(data, struct RTypedData);
732
733 if (klass) Check_Type(klass, T_CLASS);
734
735 OBJSETUP(data, klass, T_DATA);
736
737 data->data = datap;
738 data->typed_flag = 1;
739 data->type = type;
740
741 return (VALUE)data;
742 }
743
744 size_t
745 rb_objspace_data_type_memsize(VALUE obj)
746 {
747 if (RTYPEDDATA_P(obj) && RTYPEDDATA_TYPE(obj)->function.dsize) {
748 return RTYPEDDATA_TYPE(obj)->function.dsize(RTYPEDDATA_DATA(obj));
749 }
750 else {
751 return 0;
752 }
753 }
754
755 const char *
756 rb_objspace_data_type_name(VALUE obj)
757 {
758 if (RTYPEDDATA_P(obj)) {
759 return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
760 }
761 else {
762 return 0;
763 }
764 }
765
766 static void gc_mark(rb_objspace_t *objspace, VALUE ptr);
767 static void gc_mark_children(rb_objspace_t *objspace, VALUE ptr);
768
769 static inline int
770 is_pointer_to_heap(rb_objspace_t *objspace, void *ptr)
771 {
772 register RVALUE *p = RANY(ptr);
773 register struct heaps_header *heap;
774 register size_t hi, lo, mid;
775
776 if (p < lomem || p > himem) return FALSE;
777 if ((VALUE)p % sizeof(RVALUE) != 0) return FALSE;
778
779
780 lo = 0;
781 hi = heaps_used;
782 while (lo < hi) {
783 mid = (lo + hi) / 2;
784 heap = objspace->heap.sorted[mid];
785 if (heap->start <= p) {
786 if (p < heap->end)
787 return TRUE;
788 lo = mid + 1;
789 }
790 else {
791 hi = mid;
792 }
793 }
794 return FALSE;
795 }
796
797 static int
798 free_method_entry_i(ID key, rb_method_entry_t *me, st_data_t data)
799 {
800 if (!me->mark) {
801 rb_free_method_entry(me);
802 }
803 return ST_CONTINUE;
804 }
805
806 void
807 rb_free_m_table(st_table *tbl)
808 {
809 st_foreach(tbl, free_method_entry_i, 0);
810 st_free_table(tbl);
811 }
812
813 static int
814 free_const_entry_i(ID key, rb_const_entry_t *ce, st_data_t data)
815 {
816 xfree(ce);
817 return ST_CONTINUE;
818 }
819
820 void
821 rb_free_const_table(st_table *tbl)
822 {
823 st_foreach(tbl, free_const_entry_i, 0);
824 st_free_table(tbl);
825 }
826
827 static int obj_free(rb_objspace_t *, VALUE);
828
829 static inline struct heaps_slot *
830 add_slot_local_freelist(rb_objspace_t *objspace, RVALUE *p)
831 {
832 struct heaps_slot *slot;
833
834 (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
835 p->as.free.flags = 0;
836 slot = GET_HEAP_SLOT(p);
837 p->as.free.next = slot->freelist;
838 slot->freelist = p;
839
840 return slot;
841 }
842
843 static void
844 unlink_heap_slot(rb_objspace_t *objspace, struct heaps_slot *slot)
845 {
846 if (slot->prev)
847 slot->prev->next = slot->next;
848 if (slot->next)
849 slot->next->prev = slot->prev;
850 if (heaps == slot)
851 heaps = slot->next;
852 if (objspace->heap.sweep_slots == slot)
853 objspace->heap.sweep_slots = slot->next;
854 slot->prev = NULL;
855 slot->next = NULL;
856 }
857
858 static void
859 free_unused_heaps(rb_objspace_t *objspace)
860 {
861 size_t i, j;
862 struct heaps_header *last = 0;
863
864 for (i = j = 1; j < heaps_used; i++) {
865 if (objspace->heap.sorted[i]->limit == 0) {
866 struct heaps_header* h = objspace->heap.sorted[i];
867 ((struct heaps_free_bitmap *)(h->bits))->next =
868 objspace->heap.free_bitmap;
869 objspace->heap.free_bitmap = (struct heaps_free_bitmap *)h->bits;
870 if (!last) {
871 last = objspace->heap.sorted[i];
872 }
873 else {
874 aligned_free(objspace->heap.sorted[i]);
875 }
876 heaps_used--;
877 }
878 else {
879 if (i != j) {
880 objspace->heap.sorted[j] = objspace->heap.sorted[i];
881 }
882 j++;
883 }
884 }
885 if (last) {
886 if (last < heaps_freed) {
887 aligned_free(heaps_freed);
888 heaps_freed = last;
889 }
890 else {
891 aligned_free(last);
892 }
893 }
894 }
895 static inline void
896 make_deferred(RVALUE *p)
897 {
898 p->as.basic.flags = (p->as.basic.flags & ~T_MASK) | T_ZOMBIE;
899 }
900
901 static inline void
902 make_io_deferred(RVALUE *p)
903 {
904 rb_io_t *fptr = p->as.file.fptr;
905 make_deferred(p);
906 p->as.data.dfree = (void (*)(void*))rb_io_fptr_finalize;
907 p->as.data.data = fptr;
908 }
909
910 static int
911 obj_free(rb_objspace_t *objspace, VALUE obj)
912 {
913 switch (BUILTIN_TYPE(obj)) {
914 case T_NIL:
915 case T_FIXNUM:
916 case T_TRUE:
917 case T_FALSE:
918 rb_bug("obj_free() called for broken object");
919 break;
920 }
921
922 if (FL_TEST(obj, FL_EXIVAR)) {
923 rb_free_generic_ivar((VALUE)obj);
924 FL_UNSET(obj, FL_EXIVAR);
925 }
926
927 switch (BUILTIN_TYPE(obj)) {
928 case T_OBJECT:
929 if (!(RANY(obj)->as.basic.flags & ROBJECT_EMBED) &&
930 RANY(obj)->as.object.as.heap.ivptr) {
931 xfree(RANY(obj)->as.object.as.heap.ivptr);
932 }
933 break;
934 case T_MODULE:
935 case T_CLASS:
936 rb_clear_cache_by_class((VALUE)obj);
937 if (RCLASS_M_TBL(obj)) {
938 rb_free_m_table(RCLASS_M_TBL(obj));
939 }
940 if (RCLASS_IV_TBL(obj)) {
941 st_free_table(RCLASS_IV_TBL(obj));
942 }
943 if (RCLASS_CONST_TBL(obj)) {
944 rb_free_const_table(RCLASS_CONST_TBL(obj));
945 }
946 if (RCLASS_IV_INDEX_TBL(obj)) {
947 st_free_table(RCLASS_IV_INDEX_TBL(obj));
948 }
949 xfree(RANY(obj)->as.klass.ptr);
950 break;
951 case T_STRING:
952 rb_str_free(obj);
953 break;
954 case T_ARRAY:
955 rb_ary_free(obj);
956 break;
957 case T_HASH:
958 if (RANY(obj)->as.hash.ntbl) {
959 st_free_table(RANY(obj)->as.hash.ntbl);
960 }
961 break;
962 case T_REGEXP:
963 if (RANY(obj)->as.regexp.ptr) {
964 onig_free(RANY(obj)->as.regexp.ptr);
965 }
966 break;
967 case T_DATA:
968 if (DATA_PTR(obj)) {
969 if (RTYPEDDATA_P(obj)) {
970 RDATA(obj)->dfree = RANY(obj)->as.typeddata.type->function.dfree;
971 }
972 if (RANY(obj)->as.data.dfree == (RUBY_DATA_FUNC)-1) {
973 xfree(DATA_PTR(obj));
974 }
975 else if (RANY(obj)->as.data.dfree) {
976 make_deferred(RANY(obj));
977 return 1;
978 }
979 }
980 break;
981 case T_MATCH:
982 if (RANY(obj)->as.match.rmatch) {
983 struct rmatch *rm = RANY(obj)->as.match.rmatch;
984 onig_region_free(&rm->regs, 0);
985 if (rm->char_offset)
986 xfree(rm->char_offset);
987 xfree(rm);
988 }
989 break;
990 case T_FILE:
991 if (RANY(obj)->as.file.fptr) {
992 make_io_deferred(RANY(obj));
993 return 1;
994 }
995 break;
996 case T_RATIONAL:
997 case T_COMPLEX:
998 break;
999 case T_ICLASS:
1000
1001 xfree(RANY(obj)->as.klass.ptr);
1002 break;
1003
1004 case T_FLOAT:
1005 break;
1006
1007 case T_BIGNUM:
1008 if (!(RBASIC(obj)->flags & RBIGNUM_EMBED_FLAG) && RBIGNUM_DIGITS(obj)) {
1009 xfree(RBIGNUM_DIGITS(obj));
1010 }
1011 break;
1012 case T_NODE:
1013 switch (nd_type(obj)) {
1014 case NODE_SCOPE:
1015 if (RANY(obj)->as.node.u1.tbl) {
1016 xfree(RANY(obj)->as.node.u1.tbl);
1017 }
1018 break;
1019 case NODE_ARGS:
1020 if (RANY(obj)->as.node.u3.args) {
1021 xfree(RANY(obj)->as.node.u3.args);
1022 }
1023 break;
1024 case NODE_ALLOCA:
1025 xfree(RANY(obj)->as.node.u1.node);
1026 break;
1027 }
1028 break;
1029
1030 case T_STRUCT:
1031 if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
1032 RANY(obj)->as.rstruct.as.heap.ptr) {
1033 xfree(RANY(obj)->as.rstruct.as.heap.ptr);
1034 }
1035 break;
1036
1037 default:
1038 rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
1039 BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
1040 }
1041
1042 return 0;
1043 }
1044
1045 void
1046 Init_heap(void)
1047 {
1048 init_heap(&rb_objspace);
1049 }
1050
1051 typedef int each_obj_callback(void *, void *, size_t, void *);
1052
1053 struct each_obj_args {
1054 each_obj_callback *callback;
1055 void *data;
1056 };
1057
1058 static VALUE
1059 objspace_each_objects(VALUE arg)
1060 {
1061 size_t i;
1062 RVALUE *membase = 0;
1063 RVALUE *pstart, *pend;
1064 rb_objspace_t *objspace = &rb_objspace;
1065 struct each_obj_args *args = (struct each_obj_args *)arg;
1066 volatile VALUE v;
1067
1068 i = 0;
1069 while (i < heaps_used) {
1070 while (0 < i && (uintptr_t)membase < (uintptr_t)objspace->heap.sorted[i-1])
1071 i--;
1072 while (i < heaps_used && (uintptr_t)objspace->heap.sorted[i] <= (uintptr_t)membase)
1073 i++;
1074 if (heaps_used <= i)
1075 break;
1076 membase = (RVALUE *)objspace->heap.sorted[i];
1077
1078 pstart = objspace->heap.sorted[i]->start;
1079 pend = pstart + objspace->heap.sorted[i]->limit;
1080
1081 for (; pstart != pend; pstart++) {
1082 if (pstart->as.basic.flags) {
1083 v = (VALUE)pstart;
1084 break;
1085 }
1086 }
1087 if (pstart != pend) {
1088 if ((*args->callback)(pstart, pend, sizeof(RVALUE), args->data)) {
1089 break;
1090 }
1091 }
1092 }
1093 RB_GC_GUARD(v);
1094
1095 return Qnil;
1096 }
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134 void
1135 rb_objspace_each_objects(each_obj_callback *callback, void *data)
1136 {
1137 struct each_obj_args args;
1138 rb_objspace_t *objspace = &rb_objspace;
1139
1140 rest_sweep(objspace);
1141 objspace->flags.dont_lazy_sweep = TRUE;
1142
1143 args.callback = callback;
1144 args.data = data;
1145 rb_ensure(objspace_each_objects, (VALUE)&args, lazy_sweep_enable, Qnil);
1146 }
1147
1148 struct os_each_struct {
1149 size_t num;
1150 VALUE of;
1151 };
1152
1153 static int
1154 internal_object_p(VALUE obj)
1155 {
1156 RVALUE *p = (RVALUE *)obj;
1157
1158 if (p->as.basic.flags) {
1159 switch (BUILTIN_TYPE(p)) {
1160 case T_NONE:
1161 case T_ICLASS:
1162 case T_NODE:
1163 case T_ZOMBIE:
1164 break;
1165 case T_CLASS:
1166 if (FL_TEST(p, FL_SINGLETON))
1167 break;
1168 default:
1169 if (!p->as.basic.klass) break;
1170 return 0;
1171 }
1172 }
1173 return 1;
1174 }
1175
1176 int
1177 rb_objspace_internal_object_p(VALUE obj)
1178 {
1179 return internal_object_p(obj);
1180 }
1181
1182 static int
1183 os_obj_of_i(void *vstart, void *vend, size_t stride, void *data)
1184 {
1185 struct os_each_struct *oes = (struct os_each_struct *)data;
1186 RVALUE *p = (RVALUE *)vstart, *pend = (RVALUE *)vend;
1187
1188 for (; p != pend; p++) {
1189 volatile VALUE v = (VALUE)p;
1190 if (!internal_object_p(v)) {
1191 if (!oes->of || rb_obj_is_kind_of(v, oes->of)) {
1192 rb_yield(v);
1193 oes->num++;
1194 }
1195 }
1196 }
1197
1198 return 0;
1199 }
1200
1201 static VALUE
1202 os_obj_of(VALUE of)
1203 {
1204 struct os_each_struct oes;
1205
1206 oes.num = 0;
1207 oes.of = of;
1208 rb_objspace_each_objects(os_obj_of_i, &oes);
1209 return SIZET2NUM(oes.num);
1210 }
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248 static VALUE
1249 os_each_obj(int argc, VALUE *argv, VALUE os)
1250 {
1251 VALUE of;
1252
1253 rb_secure(4);
1254 if (argc == 0) {
1255 of = 0;
1256 }
1257 else {
1258 rb_scan_args(argc, argv, "01", &of);
1259 }
1260 RETURN_ENUMERATOR(os, 1, &of);
1261 return os_obj_of(of);
1262 }
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272 static VALUE
1273 undefine_final(VALUE os, VALUE obj)
1274 {
1275 return rb_undefine_final(obj);
1276 }
1277
1278 VALUE
1279 rb_undefine_final(VALUE obj)
1280 {
1281 rb_objspace_t *objspace = &rb_objspace;
1282 st_data_t data = obj;
1283 rb_check_frozen(obj);
1284 st_delete(finalizer_table, &data, 0);
1285 FL_UNSET(obj, FL_FINALIZE);
1286 return obj;
1287 }
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298 static VALUE
1299 define_final(int argc, VALUE *argv, VALUE os)
1300 {
1301 VALUE obj, block;
1302
1303 rb_scan_args(argc, argv, "11", &obj, &block);
1304 rb_check_frozen(obj);
1305 if (argc == 1) {
1306 block = rb_block_proc();
1307 }
1308 else if (!rb_respond_to(block, rb_intern("call"))) {
1309 rb_raise(rb_eArgError, "wrong type argument %s (should be callable)",
1310 rb_obj_classname(block));
1311 }
1312
1313 return define_final0(obj, block);
1314 }
1315
1316 static VALUE
1317 define_final0(VALUE obj, VALUE block)
1318 {
1319 rb_objspace_t *objspace = &rb_objspace;
1320 VALUE table;
1321 st_data_t data;
1322
1323 if (!FL_ABLE(obj)) {
1324 rb_raise(rb_eArgError, "cannot define finalizer for %s",
1325 rb_obj_classname(obj));
1326 }
1327 RBASIC(obj)->flags |= FL_FINALIZE;
1328
1329 block = rb_ary_new3(2, INT2FIX(rb_safe_level()), block);
1330 OBJ_FREEZE(block);
1331
1332 if (st_lookup(finalizer_table, obj, &data)) {
1333 table = (VALUE)data;
1334 rb_ary_push(table, block);
1335 }
1336 else {
1337 table = rb_ary_new3(1, block);
1338 RBASIC(table)->klass = 0;
1339 st_add_direct(finalizer_table, obj, table);
1340 }
1341 return block;
1342 }
1343
1344 VALUE
1345 rb_define_final(VALUE obj, VALUE block)
1346 {
1347 rb_check_frozen(obj);
1348 if (!rb_respond_to(block, rb_intern("call"))) {
1349 rb_raise(rb_eArgError, "wrong type argument %s (should be callable)",
1350 rb_obj_classname(block));
1351 }
1352 return define_final0(obj, block);
1353 }
1354
1355 void
1356 rb_gc_copy_finalizer(VALUE dest, VALUE obj)
1357 {
1358 rb_objspace_t *objspace = &rb_objspace;
1359 VALUE table;
1360 st_data_t data;
1361
1362 if (!FL_TEST(obj, FL_FINALIZE)) return;
1363 if (st_lookup(finalizer_table, obj, &data)) {
1364 table = (VALUE)data;
1365 st_insert(finalizer_table, dest, table);
1366 }
1367 FL_SET(dest, FL_FINALIZE);
1368 }
1369
1370 static VALUE
1371 run_single_final(VALUE arg)
1372 {
1373 VALUE *args = (VALUE *)arg;
1374 rb_eval_cmd(args[0], args[1], (int)args[2]);
1375 return Qnil;
1376 }
1377
1378 static void
1379 run_finalizer(rb_objspace_t *objspace, VALUE obj, VALUE table)
1380 {
1381 long i;
1382 int status;
1383 VALUE args[3];
1384 VALUE objid = nonspecial_obj_id(obj);
1385
1386 if (RARRAY_LEN(table) > 0) {
1387 args[1] = rb_obj_freeze(rb_ary_new3(1, objid));
1388 }
1389 else {
1390 args[1] = 0;
1391 }
1392
1393 args[2] = (VALUE)rb_safe_level();
1394 for (i=0; i<RARRAY_LEN(table); i++) {
1395 VALUE final = RARRAY_PTR(table)[i];
1396 args[0] = RARRAY_PTR(final)[1];
1397 args[2] = FIX2INT(RARRAY_PTR(final)[0]);
1398 status = 0;
1399 rb_protect(run_single_final, (VALUE)args, &status);
1400 if (status)
1401 rb_set_errinfo(Qnil);
1402 }
1403 }
1404
1405 static void
1406 run_final(rb_objspace_t *objspace, VALUE obj)
1407 {
1408 RUBY_DATA_FUNC free_func = 0;
1409 st_data_t key, table;
1410
1411 objspace->heap.final_num--;
1412
1413 RBASIC(obj)->klass = 0;
1414
1415 if (RTYPEDDATA_P(obj)) {
1416 free_func = RTYPEDDATA_TYPE(obj)->function.dfree;
1417 }
1418 else {
1419 free_func = RDATA(obj)->dfree;
1420 }
1421 if (free_func) {
1422 (*free_func)(DATA_PTR(obj));
1423 }
1424
1425 key = (st_data_t)obj;
1426 if (st_delete(finalizer_table, &key, &table)) {
1427 run_finalizer(objspace, obj, (VALUE)table);
1428 }
1429 }
1430
1431 static void
1432 finalize_list(rb_objspace_t *objspace, RVALUE *p)
1433 {
1434 while (p) {
1435 RVALUE *tmp = p->as.free.next;
1436 run_final(objspace, (VALUE)p);
1437 objspace->total_freed_object_num++;
1438 if (!FL_TEST(p, FL_SINGLETON)) {
1439 add_slot_local_freelist(objspace, p);
1440 objspace->heap.free_num++;
1441 }
1442 else {
1443 struct heaps_slot *slot = (struct heaps_slot *)(VALUE)RDATA(p)->dmark;
1444 slot->header->limit--;
1445 }
1446 p = tmp;
1447 }
1448 }
1449
1450 static void
1451 finalize_deferred(rb_objspace_t *objspace)
1452 {
1453 RVALUE *p = deferred_final_list;
1454 deferred_final_list = 0;
1455
1456 if (p) {
1457 finalize_list(objspace, p);
1458 }
1459 }
1460
1461 void
1462 rb_gc_finalize_deferred(void)
1463 {
1464 rb_objspace_t *objspace = &rb_objspace;
1465 if (ATOMIC_EXCHANGE(finalizing, 1)) return;
1466 finalize_deferred(objspace);
1467 ATOMIC_SET(finalizing, 0);
1468 }
1469
1470 struct force_finalize_list {
1471 VALUE obj;
1472 VALUE table;
1473 struct force_finalize_list *next;
1474 };
1475
1476 static int
1477 force_chain_object(st_data_t key, st_data_t val, st_data_t arg)
1478 {
1479 struct force_finalize_list **prev = (struct force_finalize_list **)arg;
1480 struct force_finalize_list *curr = ALLOC(struct force_finalize_list);
1481 curr->obj = key;
1482 curr->table = val;
1483 curr->next = *prev;
1484 *prev = curr;
1485 return ST_CONTINUE;
1486 }
1487
1488 void
1489 rb_gc_call_finalizer_at_exit(void)
1490 {
1491 rb_objspace_call_finalizer(&rb_objspace);
1492 }
1493
1494 static void
1495 rb_objspace_call_finalizer(rb_objspace_t *objspace)
1496 {
1497 RVALUE *p, *pend;
1498 RVALUE *final_list = 0;
1499 size_t i;
1500
1501 rest_sweep(objspace);
1502
1503 if (ATOMIC_EXCHANGE(finalizing, 1)) return;
1504
1505
1506 finalize_deferred(objspace);
1507 assert(deferred_final_list == 0);
1508
1509
1510 while (finalizer_table->num_entries) {
1511 struct force_finalize_list *list = 0;
1512 st_foreach(finalizer_table, force_chain_object, (st_data_t)&list);
1513 while (list) {
1514 struct force_finalize_list *curr = list;
1515 st_data_t obj = (st_data_t)curr->obj;
1516 run_finalizer(objspace, curr->obj, curr->table);
1517 st_delete(finalizer_table, &obj, 0);
1518 list = curr->next;
1519 xfree(curr);
1520 }
1521 }
1522
1523
1524 during_gc++;
1525
1526
1527 for (i = 0; i < heaps_used; i++) {
1528 p = objspace->heap.sorted[i]->start; pend = p + objspace->heap.sorted[i]->limit;
1529 while (p < pend) {
1530 if (BUILTIN_TYPE(p) == T_DATA &&
1531 DATA_PTR(p) && RANY(p)->as.data.dfree &&
1532 !rb_obj_is_thread((VALUE)p) && !rb_obj_is_mutex((VALUE)p) &&
1533 !rb_obj_is_fiber((VALUE)p)) {
1534 p->as.free.flags = 0;
1535 if (RTYPEDDATA_P(p)) {
1536 RDATA(p)->dfree = RANY(p)->as.typeddata.type->function.dfree;
1537 }
1538 if (RANY(p)->as.data.dfree == (RUBY_DATA_FUNC)-1) {
1539 xfree(DATA_PTR(p));
1540 }
1541 else if (RANY(p)->as.data.dfree) {
1542 make_deferred(RANY(p));
1543 RANY(p)->as.free.next = final_list;
1544 final_list = p;
1545 }
1546 }
1547 else if (BUILTIN_TYPE(p) == T_FILE) {
1548 if (RANY(p)->as.file.fptr) {
1549 make_io_deferred(RANY(p));
1550 RANY(p)->as.free.next = final_list;
1551 final_list = p;
1552 }
1553 }
1554 p++;
1555 }
1556 }
1557 during_gc = 0;
1558 if (final_list) {
1559 finalize_list(objspace, final_list);
1560 }
1561
1562 st_free_table(finalizer_table);
1563 finalizer_table = 0;
1564 ATOMIC_SET(finalizing, 0);
1565 }
1566
1567 static inline int
1568 is_id_value(rb_objspace_t *objspace, VALUE ptr)
1569 {
1570 if (!is_pointer_to_heap(objspace, (void *)ptr)) return FALSE;
1571 if (BUILTIN_TYPE(ptr) > T_FIXNUM) return FALSE;
1572 if (BUILTIN_TYPE(ptr) == T_ICLASS) return FALSE;
1573 return TRUE;
1574 }
1575
1576 static inline int
1577 is_swept_object(rb_objspace_t *objspace, VALUE ptr)
1578 {
1579 struct heaps_slot *slot = objspace->heap.sweep_slots;
1580
1581 while (slot) {
1582 if ((VALUE)slot->header->start <= ptr && ptr < (VALUE)(slot->header->end))
1583 return FALSE;
1584 slot = slot->next;
1585 }
1586 return TRUE;
1587 }
1588
1589 static inline int
1590 is_dead_object(rb_objspace_t *objspace, VALUE ptr)
1591 {
1592 if (!is_lazy_sweeping(objspace) || MARKED_IN_BITMAP(GET_HEAP_BITMAP(ptr), ptr))
1593 return FALSE;
1594 if (!is_swept_object(objspace, ptr))
1595 return TRUE;
1596 return FALSE;
1597 }
1598
1599 static inline int
1600 is_live_object(rb_objspace_t *objspace, VALUE ptr)
1601 {
1602 if (BUILTIN_TYPE(ptr) == 0) return FALSE;
1603 if (RBASIC(ptr)->klass == 0) return FALSE;
1604 if (is_dead_object(objspace, ptr)) return FALSE;
1605 return TRUE;
1606 }
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621 static VALUE
1622 id2ref(VALUE obj, VALUE objid)
1623 {
1624 #if SIZEOF_LONG == SIZEOF_VOIDP
1625 #define NUM2PTR(x) NUM2ULONG(x)
1626 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
1627 #define NUM2PTR(x) NUM2ULL(x)
1628 #endif
1629 rb_objspace_t *objspace = &rb_objspace;
1630 VALUE ptr;
1631 void *p0;
1632
1633 rb_secure(4);
1634 ptr = NUM2PTR(objid);
1635 p0 = (void *)ptr;
1636
1637 if (ptr == Qtrue) return Qtrue;
1638 if (ptr == Qfalse) return Qfalse;
1639 if (ptr == Qnil) return Qnil;
1640 if (FIXNUM_P(ptr)) return (VALUE)ptr;
1641 if (FLONUM_P(ptr)) return (VALUE)ptr;
1642 ptr = obj_id_to_ref(objid);
1643
1644 if ((ptr % sizeof(RVALUE)) == (4 << 2)) {
1645 ID symid = ptr / sizeof(RVALUE);
1646 if (rb_id2name(symid) == 0)
1647 rb_raise(rb_eRangeError, "%p is not symbol id value", p0);
1648 return ID2SYM(symid);
1649 }
1650
1651 if (!is_id_value(objspace, ptr)) {
1652 rb_raise(rb_eRangeError, "%p is not id value", p0);
1653 }
1654 if (!is_live_object(objspace, ptr)) {
1655 rb_raise(rb_eRangeError, "%p is recycled object", p0);
1656 }
1657 return (VALUE)ptr;
1658 }
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694 VALUE
1695 rb_obj_id(VALUE obj)
1696 {
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725 if (SYMBOL_P(obj)) {
1726 return (SYM2ID(obj) * sizeof(RVALUE) + (4 << 2)) | FIXNUM_FLAG;
1727 }
1728 else if (FLONUM_P(obj)) {
1729 #if SIZEOF_LONG == SIZEOF_VOIDP
1730 return LONG2NUM((SIGNED_VALUE)obj);
1731 #else
1732 return LL2NUM((SIGNED_VALUE)obj);
1733 #endif
1734 }
1735 else if (SPECIAL_CONST_P(obj)) {
1736 return LONG2NUM((SIGNED_VALUE)obj);
1737 }
1738 return nonspecial_obj_id(obj);
1739 }
1740
1741 static int
1742 set_zero(st_data_t key, st_data_t val, st_data_t arg)
1743 {
1744 VALUE k = (VALUE)key;
1745 VALUE hash = (VALUE)arg;
1746 rb_hash_aset(hash, k, INT2FIX(0));
1747 return ST_CONTINUE;
1748 }
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775 static VALUE
1776 count_objects(int argc, VALUE *argv, VALUE os)
1777 {
1778 rb_objspace_t *objspace = &rb_objspace;
1779 size_t counts[T_MASK+1];
1780 size_t freed = 0;
1781 size_t total = 0;
1782 size_t i;
1783 VALUE hash;
1784
1785 if (rb_scan_args(argc, argv, "01", &hash) == 1) {
1786 if (!RB_TYPE_P(hash, T_HASH))
1787 rb_raise(rb_eTypeError, "non-hash given");
1788 }
1789
1790 for (i = 0; i <= T_MASK; i++) {
1791 counts[i] = 0;
1792 }
1793
1794 for (i = 0; i < heaps_used; i++) {
1795 RVALUE *p, *pend;
1796
1797 p = objspace->heap.sorted[i]->start; pend = p + objspace->heap.sorted[i]->limit;
1798 for (;p < pend; p++) {
1799 if (p->as.basic.flags) {
1800 counts[BUILTIN_TYPE(p)]++;
1801 }
1802 else {
1803 freed++;
1804 }
1805 }
1806 total += objspace->heap.sorted[i]->limit;
1807 }
1808
1809 if (hash == Qnil) {
1810 hash = rb_hash_new();
1811 }
1812 else if (!RHASH_EMPTY_P(hash)) {
1813 st_foreach(RHASH_TBL(hash), set_zero, hash);
1814 }
1815 rb_hash_aset(hash, ID2SYM(rb_intern("TOTAL")), SIZET2NUM(total));
1816 rb_hash_aset(hash, ID2SYM(rb_intern("FREE")), SIZET2NUM(freed));
1817
1818 for (i = 0; i <= T_MASK; i++) {
1819 VALUE type;
1820 switch (i) {
1821 #define COUNT_TYPE(t) case (t): type = ID2SYM(rb_intern(#t)); break;
1822 COUNT_TYPE(T_NONE);
1823 COUNT_TYPE(T_OBJECT);
1824 COUNT_TYPE(T_CLASS);
1825 COUNT_TYPE(T_MODULE);
1826 COUNT_TYPE(T_FLOAT);
1827 COUNT_TYPE(T_STRING);
1828 COUNT_TYPE(T_REGEXP);
1829 COUNT_TYPE(T_ARRAY);
1830 COUNT_TYPE(T_HASH);
1831 COUNT_TYPE(T_STRUCT);
1832 COUNT_TYPE(T_BIGNUM);
1833 COUNT_TYPE(T_FILE);
1834 COUNT_TYPE(T_DATA);
1835 COUNT_TYPE(T_MATCH);
1836 COUNT_TYPE(T_COMPLEX);
1837 COUNT_TYPE(T_RATIONAL);
1838 COUNT_TYPE(T_NIL);
1839 COUNT_TYPE(T_TRUE);
1840 COUNT_TYPE(T_FALSE);
1841 COUNT_TYPE(T_SYMBOL);
1842 COUNT_TYPE(T_FIXNUM);
1843 COUNT_TYPE(T_UNDEF);
1844 COUNT_TYPE(T_NODE);
1845 COUNT_TYPE(T_ICLASS);
1846 COUNT_TYPE(T_ZOMBIE);
1847 #undef COUNT_TYPE
1848 default: type = INT2NUM(i); break;
1849 }
1850 if (counts[i])
1851 rb_hash_aset(hash, type, SIZET2NUM(counts[i]));
1852 }
1853
1854 return hash;
1855 }
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865 static VALUE
1866 lazy_sweep_enable(void)
1867 {
1868 rb_objspace_t *objspace = &rb_objspace;
1869
1870 objspace->flags.dont_lazy_sweep = FALSE;
1871 return Qnil;
1872 }
1873
1874 static void
1875 gc_clear_slot_bits(struct heaps_slot *slot)
1876 {
1877 memset(slot->bits, 0, HEAP_BITMAP_LIMIT * sizeof(uintptr_t));
1878 }
1879
1880 static size_t
1881 objspace_live_num(rb_objspace_t *objspace)
1882 {
1883 return objspace->total_allocated_object_num - objspace->total_freed_object_num;
1884 }
1885
1886 static void
1887 slot_sweep(rb_objspace_t *objspace, struct heaps_slot *sweep_slot)
1888 {
1889 size_t empty_num = 0, freed_num = 0, final_num = 0;
1890 RVALUE *p, *pend;
1891 RVALUE *final = deferred_final_list;
1892 int deferred;
1893 uintptr_t *bits;
1894
1895 p = sweep_slot->header->start; pend = p + sweep_slot->header->limit;
1896 bits = GET_HEAP_BITMAP(p);
1897 while (p < pend) {
1898 if ((!(MARKED_IN_BITMAP(bits, p))) && BUILTIN_TYPE(p) != T_ZOMBIE) {
1899 if (p->as.basic.flags) {
1900 if ((deferred = obj_free(objspace, (VALUE)p)) ||
1901 (FL_TEST(p, FL_FINALIZE))) {
1902 if (!deferred) {
1903 p->as.free.flags = T_ZOMBIE;
1904 RDATA(p)->dfree = 0;
1905 }
1906 p->as.free.next = deferred_final_list;
1907 deferred_final_list = p;
1908 assert(BUILTIN_TYPE(p) == T_ZOMBIE);
1909 final_num++;
1910 }
1911 else {
1912 (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
1913 p->as.free.flags = 0;
1914 p->as.free.next = sweep_slot->freelist;
1915 sweep_slot->freelist = p;
1916 freed_num++;
1917 }
1918 }
1919 else {
1920 empty_num++;
1921 }
1922 }
1923 p++;
1924 }
1925 gc_clear_slot_bits(sweep_slot);
1926 if (final_num + freed_num + empty_num == sweep_slot->header->limit &&
1927 objspace->heap.free_num > objspace->heap.do_heap_free) {
1928 RVALUE *pp;
1929
1930 for (pp = deferred_final_list; pp != final; pp = pp->as.free.next) {
1931 RDATA(pp)->dmark = (void (*)(void *))(VALUE)sweep_slot;
1932 pp->as.free.flags |= FL_SINGLETON;
1933 }
1934 sweep_slot->header->limit = final_num;
1935 unlink_heap_slot(objspace, sweep_slot);
1936 }
1937 else {
1938 if (freed_num + empty_num > 0) {
1939 link_free_heap_slot(objspace, sweep_slot);
1940 }
1941 else {
1942 sweep_slot->free_next = NULL;
1943 }
1944 objspace->heap.free_num += freed_num + empty_num;
1945 }
1946 objspace->total_freed_object_num += freed_num;
1947 objspace->heap.final_num += final_num;
1948
1949 if (deferred_final_list && !finalizing) {
1950 rb_thread_t *th = GET_THREAD();
1951 if (th) {
1952 RUBY_VM_SET_FINALIZER_INTERRUPT(th);
1953 }
1954 }
1955 }
1956
1957 static int
1958 ready_to_gc(rb_objspace_t *objspace)
1959 {
1960 if (dont_gc || during_gc) {
1961 if (!has_free_object) {
1962 if (!heaps_increment(objspace)) {
1963 set_heaps_increment(objspace);
1964 heaps_increment(objspace);
1965 }
1966 }
1967 return FALSE;
1968 }
1969 return TRUE;
1970 }
1971
1972 static void
1973 before_gc_sweep(rb_objspace_t *objspace)
1974 {
1975 objspace->heap.do_heap_free = (size_t)((heaps_used * HEAP_OBJ_LIMIT) * 0.65);
1976 objspace->heap.free_min = (size_t)((heaps_used * HEAP_OBJ_LIMIT) * 0.2);
1977 if (objspace->heap.free_min < initial_free_min) {
1978 objspace->heap.free_min = initial_free_min;
1979 if (objspace->heap.do_heap_free < initial_free_min)
1980 objspace->heap.do_heap_free = initial_free_min;
1981 }
1982 objspace->heap.sweep_slots = heaps;
1983 objspace->heap.free_num = 0;
1984 objspace->heap.free_slots = NULL;
1985
1986
1987 if (GET_VM()->unlinked_method_entry_list) {
1988 rb_sweep_method_entry(GET_VM());
1989 }
1990 }
1991
1992 static void
1993 after_gc_sweep(rb_objspace_t *objspace)
1994 {
1995 size_t inc;
1996
1997 gc_prof_set_malloc_info(objspace);
1998 if (objspace->heap.free_num < objspace->heap.free_min) {
1999 set_heaps_increment(objspace);
2000 heaps_increment(objspace);
2001 }
2002
2003 inc = ATOMIC_SIZE_EXCHANGE(malloc_increase, 0);
2004 if (inc > malloc_limit) {
2005 malloc_limit +=
2006 (size_t)((inc - malloc_limit) * (double)objspace->heap.marked_num / (heaps_used * HEAP_OBJ_LIMIT));
2007 if (malloc_limit < initial_malloc_limit) malloc_limit = initial_malloc_limit;
2008 }
2009
2010 free_unused_heaps(objspace);
2011 }
2012
2013 static int
2014 lazy_sweep(rb_objspace_t *objspace)
2015 {
2016 struct heaps_slot *next;
2017
2018 heaps_increment(objspace);
2019 while (objspace->heap.sweep_slots) {
2020 next = objspace->heap.sweep_slots->next;
2021 slot_sweep(objspace, objspace->heap.sweep_slots);
2022 objspace->heap.sweep_slots = next;
2023 if (has_free_object) {
2024 during_gc = 0;
2025 return TRUE;
2026 }
2027 }
2028 return FALSE;
2029 }
2030
2031 static void
2032 rest_sweep(rb_objspace_t *objspace)
2033 {
2034 if (objspace->heap.sweep_slots) {
2035 while (objspace->heap.sweep_slots) {
2036 lazy_sweep(objspace);
2037 }
2038 after_gc_sweep(objspace);
2039 }
2040 }
2041
2042 static void gc_marks(rb_objspace_t *objspace);
2043
2044 static int
2045 gc_prepare_free_objects(rb_objspace_t *objspace)
2046 {
2047 int res;
2048
2049 if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_lazy_sweep) {
2050 if (heaps_increment(objspace)) {
2051 return TRUE;
2052 }
2053 else {
2054 return garbage_collect(objspace);
2055 }
2056 }
2057
2058
2059 if (!ready_to_gc(objspace)) return TRUE;
2060
2061 during_gc++;
2062 gc_prof_timer_start(objspace);
2063 gc_prof_sweep_timer_start(objspace);
2064
2065 if (objspace->heap.sweep_slots) {
2066 res = lazy_sweep(objspace);
2067 if (res) {
2068 gc_prof_sweep_timer_stop(objspace);
2069 gc_prof_set_malloc_info(objspace);
2070 gc_prof_timer_stop(objspace, Qfalse);
2071 return res;
2072 }
2073 after_gc_sweep(objspace);
2074 }
2075 else {
2076 if (heaps_increment(objspace)) {
2077 during_gc = 0;
2078 return TRUE;
2079 }
2080 }
2081
2082 gc_marks(objspace);
2083
2084 before_gc_sweep(objspace);
2085 if (objspace->heap.free_min > (heaps_used * HEAP_OBJ_LIMIT - objspace->heap.marked_num)) {
2086 set_heaps_increment(objspace);
2087 }
2088
2089 gc_prof_sweep_timer_start(objspace);
2090 if (!(res = lazy_sweep(objspace))) {
2091 after_gc_sweep(objspace);
2092 if (has_free_object) {
2093 res = TRUE;
2094 during_gc = 0;
2095 }
2096 }
2097 gc_prof_sweep_timer_stop(objspace);
2098
2099 gc_prof_timer_stop(objspace, Qtrue);
2100 return res;
2101 }
2102
2103 static void
2104 gc_sweep(rb_objspace_t *objspace)
2105 {
2106 struct heaps_slot *next;
2107
2108 before_gc_sweep(objspace);
2109
2110 while (objspace->heap.sweep_slots) {
2111 next = objspace->heap.sweep_slots->next;
2112 slot_sweep(objspace, objspace->heap.sweep_slots);
2113 objspace->heap.sweep_slots = next;
2114 }
2115
2116 after_gc_sweep(objspace);
2117
2118 during_gc = 0;
2119 }
2120
2121
2122
2123 static void push_mark_stack(mark_stack_t *, VALUE);
2124 static int pop_mark_stack(mark_stack_t *, VALUE *);
2125 static void shrink_stack_chunk_cache(mark_stack_t *stack);
2126
2127 static stack_chunk_t *
2128 stack_chunk_alloc(void)
2129 {
2130 stack_chunk_t *res;
2131
2132 res = malloc(sizeof(stack_chunk_t));
2133 if (!res)
2134 rb_memerror();
2135
2136 return res;
2137 }
2138
2139 static inline int
2140 is_mark_stask_empty(mark_stack_t *stack)
2141 {
2142 return stack->chunk == NULL;
2143 }
2144
2145 static void
2146 add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
2147 {
2148 chunk->next = stack->cache;
2149 stack->cache = chunk;
2150 stack->cache_size++;
2151 }
2152
2153 static void
2154 shrink_stack_chunk_cache(mark_stack_t *stack)
2155 {
2156 stack_chunk_t *chunk;
2157
2158 if (stack->unused_cache_size > (stack->cache_size/2)) {
2159 chunk = stack->cache;
2160 stack->cache = stack->cache->next;
2161 stack->cache_size--;
2162 free(chunk);
2163 }
2164 stack->unused_cache_size = stack->cache_size;
2165 }
2166
2167 static void
2168 push_mark_stack_chunk(mark_stack_t *stack)
2169 {
2170 stack_chunk_t *next;
2171
2172 assert(stack->index == stack->limit);
2173 if (stack->cache_size > 0) {
2174 next = stack->cache;
2175 stack->cache = stack->cache->next;
2176 stack->cache_size--;
2177 if (stack->unused_cache_size > stack->cache_size)
2178 stack->unused_cache_size = stack->cache_size;
2179 }
2180 else {
2181 next = stack_chunk_alloc();
2182 }
2183 next->next = stack->chunk;
2184 stack->chunk = next;
2185 stack->index = 0;
2186 }
2187
2188 static void
2189 pop_mark_stack_chunk(mark_stack_t *stack)
2190 {
2191 stack_chunk_t *prev;
2192
2193 prev = stack->chunk->next;
2194 assert(stack->index == 0);
2195 add_stack_chunk_cache(stack, stack->chunk);
2196 stack->chunk = prev;
2197 stack->index = stack->limit;
2198 }
2199
2200 #if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
2201 static void
2202 free_stack_chunks(mark_stack_t *stack)
2203 {
2204 stack_chunk_t *chunk = stack->chunk;
2205 stack_chunk_t *next = NULL;
2206
2207 while (chunk != NULL) {
2208 next = chunk->next;
2209 free(chunk);
2210 chunk = next;
2211 }
2212 }
2213 #endif
2214
2215 static void
2216 push_mark_stack(mark_stack_t *stack, VALUE data)
2217 {
2218 if (stack->index == stack->limit) {
2219 push_mark_stack_chunk(stack);
2220 }
2221 stack->chunk->data[stack->index++] = data;
2222 }
2223
2224 static int
2225 pop_mark_stack(mark_stack_t *stack, VALUE *data)
2226 {
2227 if (is_mark_stask_empty(stack)) {
2228 return FALSE;
2229 }
2230 if (stack->index == 1) {
2231 *data = stack->chunk->data[--stack->index];
2232 pop_mark_stack_chunk(stack);
2233 return TRUE;
2234 }
2235 *data = stack->chunk->data[--stack->index];
2236 return TRUE;
2237 }
2238
2239 static void
2240 init_mark_stack(mark_stack_t *stack)
2241 {
2242 int i;
2243
2244 push_mark_stack_chunk(stack);
2245 stack->limit = STACK_CHUNK_SIZE;
2246
2247 for (i=0; i < 4; i++) {
2248 add_stack_chunk_cache(stack, stack_chunk_alloc());
2249 }
2250 stack->unused_cache_size = stack->cache_size;
2251 }
2252
2253
2254
2255
2256 #define MARK_IN_BITMAP(bits, p) (bits[BITMAP_INDEX(p)] = bits[BITMAP_INDEX(p)] | ((uintptr_t)1 << BITMAP_OFFSET(p)))
2257
2258
2259 #ifdef __ia64
2260 #define SET_STACK_END (SET_MACHINE_STACK_END(&th->machine_stack_end), th->machine_register_stack_end = rb_ia64_bsp())
2261 #else
2262 #define SET_STACK_END SET_MACHINE_STACK_END(&th->machine_stack_end)
2263 #endif
2264
2265 #define STACK_START (th->machine_stack_start)
2266 #define STACK_END (th->machine_stack_end)
2267 #define STACK_LEVEL_MAX (th->machine_stack_maxsize/sizeof(VALUE))
2268
2269 #if STACK_GROW_DIRECTION < 0
2270 # define STACK_LENGTH (size_t)(STACK_START - STACK_END)
2271 #elif STACK_GROW_DIRECTION > 0
2272 # define STACK_LENGTH (size_t)(STACK_END - STACK_START + 1)
2273 #else
2274 # define STACK_LENGTH ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \
2275 : (size_t)(STACK_END - STACK_START + 1))
2276 #endif
2277 #if !STACK_GROW_DIRECTION
2278 int ruby_stack_grow_direction;
2279 int
2280 ruby_get_stack_grow_direction(volatile VALUE *addr)
2281 {
2282 VALUE *end;
2283 SET_MACHINE_STACK_END(&end);
2284
2285 if (end > addr) return ruby_stack_grow_direction = 1;
2286 return ruby_stack_grow_direction = -1;
2287 }
2288 #endif
2289
2290 size_t
2291 ruby_stack_length(VALUE **p)
2292 {
2293 rb_thread_t *th = GET_THREAD();
2294 SET_STACK_END;
2295 if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END);
2296 return STACK_LENGTH;
2297 }
2298
2299 #if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK))
2300 static int
2301 stack_check(int water_mark)
2302 {
2303 int ret;
2304 rb_thread_t *th = GET_THREAD();
2305 SET_STACK_END;
2306 ret = STACK_LENGTH > STACK_LEVEL_MAX - water_mark;
2307 #ifdef __ia64
2308 if (!ret) {
2309 ret = (VALUE*)rb_ia64_bsp() - th->machine_register_stack_start >
2310 th->machine_register_stack_maxsize/sizeof(VALUE) - water_mark;
2311 }
2312 #endif
2313 return ret;
2314 }
2315 #endif
2316
2317 #define STACKFRAME_FOR_CALL_CFUNC 512
2318
2319 int
2320 ruby_stack_check(void)
2321 {
2322 #if defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK)
2323 return 0;
2324 #else
2325 return stack_check(STACKFRAME_FOR_CALL_CFUNC);
2326 #endif
2327 }
2328
2329 static void
2330 mark_locations_array(rb_objspace_t *objspace, register VALUE *x, register long n)
2331 {
2332 VALUE v;
2333 while (n--) {
2334 v = *x;
2335 (void)VALGRIND_MAKE_MEM_DEFINED(&v, sizeof(v));
2336 if (is_pointer_to_heap(objspace, (void *)v)) {
2337 gc_mark(objspace, v);
2338 }
2339 x++;
2340 }
2341 }
2342
2343 static void
2344 gc_mark_locations(rb_objspace_t *objspace, VALUE *start, VALUE *end)
2345 {
2346 long n;
2347
2348 if (end <= start) return;
2349 n = end - start;
2350 mark_locations_array(objspace, start, n);
2351 }
2352
2353 void
2354 rb_gc_mark_locations(VALUE *start, VALUE *end)
2355 {
2356 gc_mark_locations(&rb_objspace, start, end);
2357 }
2358
2359 #define rb_gc_mark_locations(start, end) gc_mark_locations(objspace, (start), (end))
2360
2361 struct mark_tbl_arg {
2362 rb_objspace_t *objspace;
2363 };
2364
2365 static int
2366 mark_entry(st_data_t key, st_data_t value, st_data_t data)
2367 {
2368 struct mark_tbl_arg *arg = (void*)data;
2369 gc_mark(arg->objspace, (VALUE)value);
2370 return ST_CONTINUE;
2371 }
2372
2373 static void
2374 mark_tbl(rb_objspace_t *objspace, st_table *tbl)
2375 {
2376 struct mark_tbl_arg arg;
2377 if (!tbl || tbl->num_entries == 0) return;
2378 arg.objspace = objspace;
2379 st_foreach(tbl, mark_entry, (st_data_t)&arg);
2380 }
2381
2382 static int
2383 mark_key(st_data_t key, st_data_t value, st_data_t data)
2384 {
2385 struct mark_tbl_arg *arg = (void*)data;
2386 gc_mark(arg->objspace, (VALUE)key);
2387 return ST_CONTINUE;
2388 }
2389
2390 static void
2391 mark_set(rb_objspace_t *objspace, st_table *tbl)
2392 {
2393 struct mark_tbl_arg arg;
2394 if (!tbl) return;
2395 arg.objspace = objspace;
2396 st_foreach(tbl, mark_key, (st_data_t)&arg);
2397 }
2398
2399 void
2400 rb_mark_set(st_table *tbl)
2401 {
2402 mark_set(&rb_objspace, tbl);
2403 }
2404
2405 static int
2406 mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
2407 {
2408 struct mark_tbl_arg *arg = (void*)data;
2409 gc_mark(arg->objspace, (VALUE)key);
2410 gc_mark(arg->objspace, (VALUE)value);
2411 return ST_CONTINUE;
2412 }
2413
2414 static void
2415 mark_hash(rb_objspace_t *objspace, st_table *tbl)
2416 {
2417 struct mark_tbl_arg arg;
2418 if (!tbl) return;
2419 arg.objspace = objspace;
2420 st_foreach(tbl, mark_keyvalue, (st_data_t)&arg);
2421 }
2422
2423 void
2424 rb_mark_hash(st_table *tbl)
2425 {
2426 mark_hash(&rb_objspace, tbl);
2427 }
2428
2429 static void
2430 mark_method_entry(rb_objspace_t *objspace, const rb_method_entry_t *me)
2431 {
2432 const rb_method_definition_t *def = me->def;
2433
2434 gc_mark(objspace, me->klass);
2435 again:
2436 if (!def) return;
2437 switch (def->type) {
2438 case VM_METHOD_TYPE_ISEQ:
2439 gc_mark(objspace, def->body.iseq->self);
2440 break;
2441 case VM_METHOD_TYPE_BMETHOD:
2442 gc_mark(objspace, def->body.proc);
2443 break;
2444 case VM_METHOD_TYPE_ATTRSET:
2445 case VM_METHOD_TYPE_IVAR:
2446 gc_mark(objspace, def->body.attr.location);
2447 break;
2448 case VM_METHOD_TYPE_REFINED:
2449 if (def->body.orig_me) {
2450 def = def->body.orig_me->def;
2451 goto again;
2452 }
2453 break;
2454 default:
2455 break;
2456 }
2457 }
2458
2459 void
2460 rb_mark_method_entry(const rb_method_entry_t *me)
2461 {
2462 mark_method_entry(&rb_objspace, me);
2463 }
2464
2465 static int
2466 mark_method_entry_i(ID key, const rb_method_entry_t *me, st_data_t data)
2467 {
2468 struct mark_tbl_arg *arg = (void*)data;
2469 mark_method_entry(arg->objspace, me);
2470 return ST_CONTINUE;
2471 }
2472
2473 static void
2474 mark_m_tbl(rb_objspace_t *objspace, st_table *tbl)
2475 {
2476 struct mark_tbl_arg arg;
2477 if (!tbl) return;
2478 arg.objspace = objspace;
2479 st_foreach(tbl, mark_method_entry_i, (st_data_t)&arg);
2480 }
2481
2482 static int
2483 mark_const_entry_i(ID key, const rb_const_entry_t *ce, st_data_t data)
2484 {
2485 struct mark_tbl_arg *arg = (void*)data;
2486 gc_mark(arg->objspace, ce->value);
2487 gc_mark(arg->objspace, ce->file);
2488 return ST_CONTINUE;
2489 }
2490
2491 static void
2492 mark_const_tbl(rb_objspace_t *objspace, st_table *tbl)
2493 {
2494 struct mark_tbl_arg arg;
2495 if (!tbl) return;
2496 arg.objspace = objspace;
2497 st_foreach(tbl, mark_const_entry_i, (st_data_t)&arg);
2498 }
2499
2500 #if STACK_GROW_DIRECTION < 0
2501 #define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START)
2502 #elif STACK_GROW_DIRECTION > 0
2503 #define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix))
2504 #else
2505 #define GET_STACK_BOUNDS(start, end, appendix) \
2506 ((STACK_END < STACK_START) ? \
2507 ((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix)))
2508 #endif
2509
2510 #define numberof(array) (int)(sizeof(array) / sizeof((array)[0]))
2511
2512 static void
2513 mark_current_machine_context(rb_objspace_t *objspace, rb_thread_t *th)
2514 {
2515 union {
2516 rb_jmp_buf j;
2517 VALUE v[sizeof(rb_jmp_buf) / sizeof(VALUE)];
2518 } save_regs_gc_mark;
2519 VALUE *stack_start, *stack_end;
2520
2521 FLUSH_REGISTER_WINDOWS;
2522
2523 rb_setjmp(save_regs_gc_mark.j);
2524
2525 SET_STACK_END;
2526 GET_STACK_BOUNDS(stack_start, stack_end, 1);
2527
2528 mark_locations_array(objspace, save_regs_gc_mark.v, numberof(save_regs_gc_mark.v));
2529
2530 rb_gc_mark_locations(stack_start, stack_end);
2531 #ifdef __ia64
2532 rb_gc_mark_locations(th->machine_register_stack_start, th->machine_register_stack_end);
2533 #endif
2534 #if defined(__mc68000__)
2535 mark_locations_array(objspace, (VALUE*)((char*)STACK_END + 2),
2536 (STACK_START - STACK_END));
2537 #endif
2538 }
2539
2540 void
2541 rb_gc_mark_machine_stack(rb_thread_t *th)
2542 {
2543 rb_objspace_t *objspace = &rb_objspace;
2544 VALUE *stack_start, *stack_end;
2545
2546 GET_STACK_BOUNDS(stack_start, stack_end, 0);
2547 rb_gc_mark_locations(stack_start, stack_end);
2548 #ifdef __ia64
2549 rb_gc_mark_locations(th->machine_register_stack_start, th->machine_register_stack_end);
2550 #endif
2551 }
2552
2553 void
2554 rb_mark_tbl(st_table *tbl)
2555 {
2556 mark_tbl(&rb_objspace, tbl);
2557 }
2558
2559 void
2560 rb_gc_mark_maybe(VALUE obj)
2561 {
2562 if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
2563 gc_mark(&rb_objspace, obj);
2564 }
2565 }
2566
2567 static int
2568 gc_mark_ptr(rb_objspace_t *objspace, VALUE ptr)
2569 {
2570 register uintptr_t *bits = GET_HEAP_BITMAP(ptr);
2571 if (MARKED_IN_BITMAP(bits, ptr)) return 0;
2572 MARK_IN_BITMAP(bits, ptr);
2573 objspace->heap.marked_num++;
2574 return 1;
2575 }
2576
2577 static int
2578 markable_object_p(rb_objspace_t *objspace, VALUE ptr)
2579 {
2580 register RVALUE *obj = RANY(ptr);
2581
2582 if (rb_special_const_p(ptr)) return 0;
2583 if (obj->as.basic.flags == 0) return 0 ;
2584
2585 return 1;
2586 }
2587
2588 int
2589 rb_objspace_markable_object_p(VALUE obj)
2590 {
2591 return markable_object_p( 0, obj);
2592 }
2593
2594 static void
2595 gc_mark(rb_objspace_t *objspace, VALUE ptr)
2596 {
2597 if (!markable_object_p(objspace, ptr)) {
2598 return;
2599 }
2600
2601 if (LIKELY(objspace->mark_func_data == 0)) {
2602 if (!gc_mark_ptr(objspace, ptr)) return;
2603 push_mark_stack(&objspace->mark_stack, ptr);
2604 }
2605 else {
2606 objspace->mark_func_data->mark_func(ptr, objspace->mark_func_data->data);
2607 }
2608 }
2609
2610 void
2611 rb_gc_mark(VALUE ptr)
2612 {
2613 gc_mark(&rb_objspace, ptr);
2614 }
2615
2616 static void
2617 gc_mark_children(rb_objspace_t *objspace, VALUE ptr)
2618 {
2619 register RVALUE *obj = RANY(ptr);
2620
2621 goto marking;
2622
2623 again:
2624 if (LIKELY(objspace->mark_func_data == 0)) {
2625 obj = RANY(ptr);
2626 if (!markable_object_p(objspace, ptr)) return;
2627 if (!gc_mark_ptr(objspace, ptr)) return;
2628 }
2629 else {
2630 gc_mark(objspace, ptr);
2631 return;
2632 }
2633
2634 marking:
2635 if (FL_TEST(obj, FL_EXIVAR)) {
2636 rb_mark_generic_ivar(ptr);
2637 }
2638
2639 switch (BUILTIN_TYPE(obj)) {
2640 case T_NIL:
2641 case T_FIXNUM:
2642 rb_bug("rb_gc_mark() called for broken object");
2643 break;
2644
2645 case T_NODE:
2646 switch (nd_type(obj)) {
2647 case NODE_IF:
2648 case NODE_FOR:
2649 case NODE_ITER:
2650 case NODE_WHEN:
2651 case NODE_MASGN:
2652 case NODE_RESCUE:
2653 case NODE_RESBODY:
2654 case NODE_CLASS:
2655 case NODE_BLOCK_PASS:
2656 gc_mark(objspace, (VALUE)obj->as.node.u2.node);
2657
2658 case NODE_BLOCK:
2659 case NODE_ARRAY:
2660 case NODE_DSTR:
2661 case NODE_DXSTR:
2662 case NODE_DREGX:
2663 case NODE_DREGX_ONCE:
2664 case NODE_ENSURE:
2665 case NODE_CALL:
2666 case NODE_DEFS:
2667 case NODE_OP_ASGN1:
2668 gc_mark(objspace, (VALUE)obj->as.node.u1.node);
2669
2670 case NODE_SUPER:
2671 case NODE_FCALL:
2672 case NODE_DEFN:
2673 case NODE_ARGS_AUX:
2674 ptr = (VALUE)obj->as.node.u3.node;
2675 goto again;
2676
2677 case NODE_WHILE:
2678 case NODE_UNTIL:
2679 case NODE_AND:
2680 case NODE_OR:
2681 case NODE_CASE:
2682 case NODE_SCLASS:
2683 case NODE_DOT2:
2684 case NODE_DOT3:
2685 case NODE_FLIP2:
2686 case NODE_FLIP3:
2687 case NODE_MATCH2:
2688 case NODE_MATCH3:
2689 case NODE_OP_ASGN_OR:
2690 case NODE_OP_ASGN_AND:
2691 case NODE_MODULE:
2692 case NODE_ALIAS:
2693 case NODE_VALIAS:
2694 case NODE_ARGSCAT:
2695 gc_mark(objspace, (VALUE)obj->as.node.u1.node);
2696
2697 case NODE_GASGN:
2698 case NODE_LASGN:
2699 case NODE_DASGN:
2700 case NODE_DASGN_CURR:
2701 case NODE_IASGN:
2702 case NODE_IASGN2:
2703 case NODE_CVASGN:
2704 case NODE_COLON3:
2705 case NODE_OPT_N:
2706 case NODE_EVSTR:
2707 case NODE_UNDEF:
2708 case NODE_POSTEXE:
2709 ptr = (VALUE)obj->as.node.u2.node;
2710 goto again;
2711
2712 case NODE_HASH:
2713 case NODE_LIT:
2714 case NODE_STR:
2715 case NODE_XSTR:
2716 case NODE_DEFINED:
2717 case NODE_MATCH:
2718 case NODE_RETURN:
2719 case NODE_BREAK:
2720 case NODE_NEXT:
2721 case NODE_YIELD:
2722 case NODE_COLON2:
2723 case NODE_SPLAT:
2724 case NODE_TO_ARY:
2725 ptr = (VALUE)obj->as.node.u1.node;
2726 goto again;
2727
2728 case NODE_SCOPE:
2729 case NODE_CDECL:
2730 case NODE_OPT_ARG:
2731 gc_mark(objspace, (VALUE)obj->as.node.u3.node);
2732 ptr = (VALUE)obj->as.node.u2.node;
2733 goto again;
2734
2735 case NODE_ARGS:
2736 {
2737 struct rb_args_info *args = obj->as.node.u3.args;
2738 if (args) {
2739 if (args->pre_init) gc_mark(objspace, (VALUE)args->pre_init);
2740 if (args->post_init) gc_mark(objspace, (VALUE)args->post_init);
2741 if (args->opt_args) gc_mark(objspace, (VALUE)args->opt_args);
2742 if (args->kw_args) gc_mark(objspace, (VALUE)args->kw_args);
2743 if (args->kw_rest_arg) gc_mark(objspace, (VALUE)args->kw_rest_arg);
2744 }
2745 }
2746 ptr = (VALUE)obj->as.node.u2.node;
2747 goto again;
2748
2749 case NODE_ZARRAY:
2750 case NODE_ZSUPER:
2751 case NODE_VCALL:
2752 case NODE_GVAR:
2753 case NODE_LVAR:
2754 case NODE_DVAR:
2755 case NODE_IVAR:
2756 case NODE_CVAR:
2757 case NODE_NTH_REF:
2758 case NODE_BACK_REF:
2759 case NODE_REDO:
2760 case NODE_RETRY:
2761 case NODE_SELF:
2762 case NODE_NIL:
2763 case NODE_TRUE:
2764 case NODE_FALSE:
2765 case NODE_ERRINFO:
2766 case NODE_BLOCK_ARG:
2767 break;
2768 case NODE_ALLOCA:
2769 mark_locations_array(objspace,
2770 (VALUE*)obj->as.node.u1.value,
2771 obj->as.node.u3.cnt);
2772 gc_mark(objspace, (VALUE)obj->as.node.u2.node);
2773 break;
2774
2775 case NODE_CREF:
2776 gc_mark(objspace, obj->as.node.nd_refinements);
2777 gc_mark(objspace, (VALUE)obj->as.node.u1.node);
2778 ptr = (VALUE)obj->as.node.u3.node;
2779 goto again;
2780
2781 default:
2782 if (is_pointer_to_heap(objspace, obj->as.node.u1.node)) {
2783 gc_mark(objspace, (VALUE)obj->as.node.u1.node);
2784 }
2785 if (is_pointer_to_heap(objspace, obj->as.node.u2.node)) {
2786 gc_mark(objspace, (VALUE)obj->as.node.u2.node);
2787 }
2788 if (is_pointer_to_heap(objspace, obj->as.node.u3.node)) {
2789 gc_mark(objspace, (VALUE)obj->as.node.u3.node);
2790 }
2791 }
2792 return;
2793 }
2794
2795 gc_mark(objspace, obj->as.basic.klass);
2796 switch (BUILTIN_TYPE(obj)) {
2797 case T_ICLASS:
2798 case T_CLASS:
2799 case T_MODULE:
2800 mark_m_tbl(objspace, RCLASS_M_TBL(obj));
2801 if (!RCLASS_EXT(obj)) break;
2802 mark_tbl(objspace, RCLASS_IV_TBL(obj));
2803 mark_const_tbl(objspace, RCLASS_CONST_TBL(obj));
2804 ptr = RCLASS_SUPER(obj);
2805 goto again;
2806
2807 case T_ARRAY:
2808 if (FL_TEST(obj, ELTS_SHARED)) {
2809 ptr = obj->as.array.as.heap.aux.shared;
2810 goto again;
2811 }
2812 else {
2813 long i, len = RARRAY_LEN(obj);
2814 VALUE *ptr = RARRAY_PTR(obj);
2815 for (i=0; i < len; i++) {
2816 gc_mark(objspace, *ptr++);
2817 }
2818 }
2819 break;
2820
2821 case T_HASH:
2822 mark_hash(objspace, obj->as.hash.ntbl);
2823 ptr = obj->as.hash.ifnone;
2824 goto again;
2825
2826 case T_STRING:
2827 #define STR_ASSOC FL_USER3
2828 if (FL_TEST(obj, RSTRING_NOEMBED) && FL_ANY(obj, ELTS_SHARED|STR_ASSOC)) {
2829 ptr = obj->as.string.as.heap.aux.shared;
2830 goto again;
2831 }
2832 break;
2833
2834 case T_DATA:
2835 if (RTYPEDDATA_P(obj)) {
2836 RUBY_DATA_FUNC mark_func = obj->as.typeddata.type->function.dmark;
2837 if (mark_func) (*mark_func)(DATA_PTR(obj));
2838 }
2839 else {
2840 if (obj->as.data.dmark) (*obj->as.data.dmark)(DATA_PTR(obj));
2841 }
2842 break;
2843
2844 case T_OBJECT:
2845 {
2846 long i, len = ROBJECT_NUMIV(obj);
2847 VALUE *ptr = ROBJECT_IVPTR(obj);
2848 for (i = 0; i < len; i++) {
2849 gc_mark(objspace, *ptr++);
2850 }
2851 }
2852 break;
2853
2854 case T_FILE:
2855 if (obj->as.file.fptr) {
2856 gc_mark(objspace, obj->as.file.fptr->pathv);
2857 gc_mark(objspace, obj->as.file.fptr->tied_io_for_writing);
2858 gc_mark(objspace, obj->as.file.fptr->writeconv_asciicompat);
2859 gc_mark(objspace, obj->as.file.fptr->writeconv_pre_ecopts);
2860 gc_mark(objspace, obj->as.file.fptr->encs.ecopts);
2861 gc_mark(objspace, obj->as.file.fptr->write_lock);
2862 }
2863 break;
2864
2865 case T_REGEXP:
2866 ptr = obj->as.regexp.src;
2867 goto again;
2868
2869 case T_FLOAT:
2870 case T_BIGNUM:
2871 case T_ZOMBIE:
2872 break;
2873
2874 case T_MATCH:
2875 gc_mark(objspace, obj->as.match.regexp);
2876 if (obj->as.match.str) {
2877 ptr = obj->as.match.str;
2878 goto again;
2879 }
2880 break;
2881
2882 case T_RATIONAL:
2883 gc_mark(objspace, obj->as.rational.num);
2884 ptr = obj->as.rational.den;
2885 goto again;
2886
2887 case T_COMPLEX:
2888 gc_mark(objspace, obj->as.complex.real);
2889 ptr = obj->as.complex.imag;
2890 goto again;
2891
2892 case T_STRUCT:
2893 {
2894 long len = RSTRUCT_LEN(obj);
2895 VALUE *ptr = RSTRUCT_PTR(obj);
2896
2897 while (len--) {
2898 gc_mark(objspace, *ptr++);
2899 }
2900 }
2901 break;
2902
2903 default:
2904 rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
2905 BUILTIN_TYPE(obj), (void *)obj,
2906 is_pointer_to_heap(objspace, obj) ? "corrupted object" : "non object");
2907 }
2908 }
2909
2910 static void
2911 gc_mark_stacked_objects(rb_objspace_t *objspace)
2912 {
2913 mark_stack_t *mstack = &objspace->mark_stack;
2914 VALUE obj = 0;
2915
2916 if (!mstack->index) return;
2917 while (pop_mark_stack(mstack, &obj)) {
2918 gc_mark_children(objspace, obj);
2919 }
2920 shrink_stack_chunk_cache(mstack);
2921 }
2922
2923 static void
2924 gc_marks(rb_objspace_t *objspace)
2925 {
2926 struct gc_list *list;
2927 rb_thread_t *th = GET_THREAD();
2928 struct mark_func_data_struct *prev_mark_func_data;
2929
2930 prev_mark_func_data = objspace->mark_func_data;
2931 objspace->mark_func_data = 0;
2932
2933 gc_prof_mark_timer_start(objspace);
2934 objspace->heap.marked_num = 0;
2935 objspace->count++;
2936
2937 SET_STACK_END;
2938
2939 th->vm->self ? rb_gc_mark(th->vm->self) : rb_vm_mark(th->vm);
2940
2941 mark_tbl(objspace, finalizer_table);
2942 mark_current_machine_context(objspace, th);
2943
2944 rb_gc_mark_symbols();
2945 rb_gc_mark_encodings();
2946
2947
2948 for (list = global_List; list; list = list->next) {
2949 rb_gc_mark_maybe(*list->varptr);
2950 }
2951 rb_mark_end_proc();
2952 rb_gc_mark_global_tbl();
2953
2954 mark_tbl(objspace, rb_class_tbl);
2955
2956
2957 rb_mark_generic_ivar_tbl();
2958
2959 rb_gc_mark_parser();
2960
2961 rb_gc_mark_unlinked_live_method_entries(th->vm);
2962
2963
2964 gc_mark_stacked_objects(objspace);
2965
2966 gc_prof_mark_timer_stop(objspace);
2967
2968 objspace->mark_func_data = prev_mark_func_data;
2969 }
2970
2971
2972
2973 void
2974 rb_gc_force_recycle(VALUE p)
2975 {
2976 rb_objspace_t *objspace = &rb_objspace;
2977 struct heaps_slot *slot;
2978
2979 objspace->total_freed_object_num++;
2980 if (MARKED_IN_BITMAP(GET_HEAP_BITMAP(p), p)) {
2981 add_slot_local_freelist(objspace, (RVALUE *)p);
2982 }
2983 else {
2984 objspace->heap.free_num++;
2985 slot = add_slot_local_freelist(objspace, (RVALUE *)p);
2986 if (slot->free_next == NULL) {
2987 link_free_heap_slot(objspace, slot);
2988 }
2989 }
2990 }
2991
2992 void
2993 rb_gc_register_mark_object(VALUE obj)
2994 {
2995 VALUE ary = GET_THREAD()->vm->mark_object_ary;
2996 rb_ary_push(ary, obj);
2997 }
2998
2999 void
3000 rb_gc_register_address(VALUE *addr)
3001 {
3002 rb_objspace_t *objspace = &rb_objspace;
3003 struct gc_list *tmp;
3004
3005 tmp = ALLOC(struct gc_list);
3006 tmp->next = global_List;
3007 tmp->varptr = addr;
3008 global_List = tmp;
3009 }
3010
3011 void
3012 rb_gc_unregister_address(VALUE *addr)
3013 {
3014 rb_objspace_t *objspace = &rb_objspace;
3015 struct gc_list *tmp = global_List;
3016
3017 if (tmp->varptr == addr) {
3018 global_List = tmp->next;
3019 xfree(tmp);
3020 return;
3021 }
3022 while (tmp->next) {
3023 if (tmp->next->varptr == addr) {
3024 struct gc_list *t = tmp->next;
3025
3026 tmp->next = tmp->next->next;
3027 xfree(t);
3028 break;
3029 }
3030 tmp = tmp->next;
3031 }
3032 }
3033
3034 #define GC_NOTIFY 0
3035
3036 static int
3037 garbage_collect(rb_objspace_t *objspace)
3038 {
3039 if (GC_NOTIFY) printf("start garbage_collect()\n");
3040
3041 if (!heaps) {
3042 return FALSE;
3043 }
3044 if (!ready_to_gc(objspace)) {
3045 return TRUE;
3046 }
3047
3048 gc_prof_timer_start(objspace);
3049
3050 rest_sweep(objspace);
3051
3052 during_gc++;
3053 gc_marks(objspace);
3054
3055 gc_prof_sweep_timer_start(objspace);
3056 gc_sweep(objspace);
3057 gc_prof_sweep_timer_stop(objspace);
3058
3059 gc_prof_timer_stop(objspace, Qtrue);
3060 if (GC_NOTIFY) printf("end garbage_collect()\n");
3061 return TRUE;
3062 }
3063
3064 static void *
3065 gc_with_gvl(void *ptr)
3066 {
3067 return (void *)(VALUE)garbage_collect((rb_objspace_t *)ptr);
3068 }
3069
3070 static int
3071 garbage_collect_with_gvl(rb_objspace_t *objspace)
3072 {
3073 if (dont_gc) return TRUE;
3074 if (ruby_thread_has_gvl_p()) {
3075 return garbage_collect(objspace);
3076 }
3077 else {
3078 if (ruby_native_thread_p()) {
3079 return (int)(VALUE)rb_thread_call_with_gvl(gc_with_gvl, (void *)objspace);
3080 }
3081 else {
3082
3083 fprintf(stderr, "[FATAL] failed to allocate memory\n");
3084 exit(EXIT_FAILURE);
3085 }
3086 }
3087 }
3088
3089 int
3090 rb_garbage_collect(void)
3091 {
3092 return garbage_collect(&rb_objspace);
3093 }
3094
3095 #undef Init_stack
3096
3097 void
3098 Init_stack(volatile VALUE *addr)
3099 {
3100 ruby_init_stack(addr);
3101 }
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113 VALUE
3114 rb_gc_start(void)
3115 {
3116 rb_gc();
3117 return Qnil;
3118 }
3119
3120 void
3121 rb_gc(void)
3122 {
3123 rb_objspace_t *objspace = &rb_objspace;
3124 garbage_collect(objspace);
3125 if (!finalizing) finalize_deferred(objspace);
3126 free_unused_heaps(objspace);
3127 }
3128
3129 int
3130 rb_during_gc(void)
3131 {
3132 rb_objspace_t *objspace = &rb_objspace;
3133 return during_gc;
3134 }
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146 static VALUE
3147 gc_count(VALUE self)
3148 {
3149 return UINT2NUM(rb_objspace.count);
3150 }
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179 static VALUE
3180 gc_stat(int argc, VALUE *argv, VALUE self)
3181 {
3182 rb_objspace_t *objspace = &rb_objspace;
3183 VALUE hash;
3184 static VALUE sym_count;
3185 static VALUE sym_heap_used, sym_heap_length, sym_heap_increment;
3186 static VALUE sym_heap_live_num, sym_heap_free_num, sym_heap_final_num;
3187 static VALUE sym_total_allocated_object, sym_total_freed_object;
3188 if (sym_count == 0) {
3189 sym_count = ID2SYM(rb_intern_const("count"));
3190 sym_heap_used = ID2SYM(rb_intern_const("heap_used"));
3191 sym_heap_length = ID2SYM(rb_intern_const("heap_length"));
3192 sym_heap_increment = ID2SYM(rb_intern_const("heap_increment"));
3193 sym_heap_live_num = ID2SYM(rb_intern_const("heap_live_num"));
3194 sym_heap_free_num = ID2SYM(rb_intern_const("heap_free_num"));
3195 sym_heap_final_num = ID2SYM(rb_intern_const("heap_final_num"));
3196 sym_total_allocated_object = ID2SYM(rb_intern_const("total_allocated_object"));
3197 sym_total_freed_object = ID2SYM(rb_intern_const("total_freed_object"));
3198 }
3199
3200 if (rb_scan_args(argc, argv, "01", &hash) == 1) {
3201 if (!RB_TYPE_P(hash, T_HASH))
3202 rb_raise(rb_eTypeError, "non-hash given");
3203 }
3204
3205 if (hash == Qnil) {
3206 hash = rb_hash_new();
3207 }
3208
3209 rest_sweep(objspace);
3210
3211 rb_hash_aset(hash, sym_count, SIZET2NUM(objspace->count));
3212
3213 rb_hash_aset(hash, sym_heap_used, SIZET2NUM(objspace->heap.used));
3214 rb_hash_aset(hash, sym_heap_length, SIZET2NUM(objspace->heap.length));
3215 rb_hash_aset(hash, sym_heap_increment, SIZET2NUM(objspace->heap.increment));
3216 rb_hash_aset(hash, sym_heap_live_num, SIZET2NUM(objspace_live_num(objspace)));
3217 rb_hash_aset(hash, sym_heap_free_num, SIZET2NUM(objspace->heap.free_num));
3218 rb_hash_aset(hash, sym_heap_final_num, SIZET2NUM(objspace->heap.final_num));
3219 rb_hash_aset(hash, sym_total_allocated_object, SIZET2NUM(objspace->total_allocated_object_num));
3220 rb_hash_aset(hash, sym_total_freed_object, SIZET2NUM(objspace->total_freed_object_num));
3221
3222 return hash;
3223 }
3224
3225
3226
3227
3228
3229
3230
3231
3232 static VALUE
3233 gc_stress_get(VALUE self)
3234 {
3235 rb_objspace_t *objspace = &rb_objspace;
3236 return ruby_gc_stress ? Qtrue : Qfalse;
3237 }
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251 static VALUE
3252 gc_stress_set(VALUE self, VALUE flag)
3253 {
3254 rb_objspace_t *objspace = &rb_objspace;
3255 rb_secure(2);
3256 ruby_gc_stress = RTEST(flag);
3257 return flag;
3258 }
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273 VALUE
3274 rb_gc_enable(void)
3275 {
3276 rb_objspace_t *objspace = &rb_objspace;
3277 int old = dont_gc;
3278
3279 dont_gc = FALSE;
3280 return old ? Qtrue : Qfalse;
3281 }
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295 VALUE
3296 rb_gc_disable(void)
3297 {
3298 rb_objspace_t *objspace = &rb_objspace;
3299 int old = dont_gc;
3300
3301 dont_gc = TRUE;
3302 return old ? Qtrue : Qfalse;
3303 }
3304
3305 void
3306 rb_gc_set_params(void)
3307 {
3308 char *malloc_limit_ptr, *heap_min_slots_ptr, *free_min_ptr, *growth_factor_ptr;
3309
3310 if (rb_safe_level() > 0) return;
3311
3312 malloc_limit_ptr = getenv("RUBY_GC_MALLOC_LIMIT");
3313 if (malloc_limit_ptr != NULL) {
3314 int malloc_limit_i = atoi(malloc_limit_ptr);
3315 if (RTEST(ruby_verbose))
3316 fprintf(stderr, "malloc_limit=%d (%d)\n",
3317 malloc_limit_i, initial_malloc_limit);
3318 if (malloc_limit_i > 0) {
3319 initial_malloc_limit = malloc_limit_i;
3320 }
3321 }
3322
3323 heap_min_slots_ptr = getenv("RUBY_HEAP_MIN_SLOTS");
3324 if (heap_min_slots_ptr != NULL) {
3325 int heap_min_slots_i = atoi(heap_min_slots_ptr);
3326 if (RTEST(ruby_verbose))
3327 fprintf(stderr, "heap_min_slots=%d (%d)\n",
3328 heap_min_slots_i, initial_heap_min_slots);
3329 if (heap_min_slots_i > 0) {
3330 initial_heap_min_slots = heap_min_slots_i;
3331 initial_expand_heap(&rb_objspace);
3332 }
3333 }
3334
3335 growth_factor_ptr = getenv("RUBY_HEAP_SLOTS_GROWTH_FACTOR");
3336 if (growth_factor_ptr != NULL) {
3337 double growth_factor_f = strtod(growth_factor_ptr, NULL);
3338 if (RTEST(ruby_verbose))
3339 fprintf(stderr, "heap_slots_growth_factor=%f (%f)\n",
3340 growth_factor_f, initial_growth_factor);
3341 if (growth_factor_f > 1) {
3342 initial_growth_factor = growth_factor_f;
3343 }
3344 }
3345
3346 free_min_ptr = getenv("RUBY_FREE_MIN");
3347 if (free_min_ptr != NULL) {
3348 int free_min_i = atoi(free_min_ptr);
3349 if (RTEST(ruby_verbose))
3350 fprintf(stderr, "free_min=%d (%d)\n", free_min_i, initial_free_min);
3351 if (free_min_i > 0) {
3352 initial_free_min = free_min_i;
3353 }
3354 }
3355 }
3356
3357 void
3358 rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data)
3359 {
3360 rb_objspace_t *objspace = &rb_objspace;
3361
3362 if (markable_object_p(objspace, obj)) {
3363 struct mark_func_data_struct mfd;
3364 mfd.mark_func = func;
3365 mfd.data = data;
3366 objspace->mark_func_data = &mfd;
3367 gc_mark_children(objspace, obj);
3368 objspace->mark_func_data = 0;
3369 }
3370 }
3371
3372
3373
3374
3375
3376 static void vm_xfree(rb_objspace_t *objspace, void *ptr);
3377
3378 static void *
3379 negative_size_allocation_error_with_gvl(void *ptr)
3380 {
3381 rb_raise(rb_eNoMemError, "%s", (const char *)ptr);
3382 return 0;
3383 }
3384
3385 static void
3386 negative_size_allocation_error(const char *msg)
3387 {
3388 if (ruby_thread_has_gvl_p()) {
3389 rb_raise(rb_eNoMemError, "%s", msg);
3390 }
3391 else {
3392 if (ruby_native_thread_p()) {
3393 rb_thread_call_with_gvl(negative_size_allocation_error_with_gvl, (void *)msg);
3394 }
3395 else {
3396 fprintf(stderr, "[FATAL] %s\n", msg);
3397 exit(EXIT_FAILURE);
3398 }
3399 }
3400 }
3401
3402 static void *
3403 ruby_memerror_body(void *dummy)
3404 {
3405 rb_memerror();
3406 return 0;
3407 }
3408
3409 static void
3410 ruby_memerror(void)
3411 {
3412 if (ruby_thread_has_gvl_p()) {
3413 rb_memerror();
3414 }
3415 else {
3416 if (ruby_native_thread_p()) {
3417 rb_thread_call_with_gvl(ruby_memerror_body, 0);
3418 }
3419 else {
3420
3421 fprintf(stderr, "[FATAL] failed to allocate memory\n");
3422 exit(EXIT_FAILURE);
3423 }
3424 }
3425 }
3426
3427 void
3428 rb_memerror(void)
3429 {
3430 rb_thread_t *th = GET_THREAD();
3431 if (!nomem_error ||
3432 (rb_thread_raised_p(th, RAISED_NOMEMORY) && rb_safe_level() < 4)) {
3433 fprintf(stderr, "[FATAL] failed to allocate memory\n");
3434 exit(EXIT_FAILURE);
3435 }
3436 if (rb_thread_raised_p(th, RAISED_NOMEMORY)) {
3437 rb_thread_raised_clear(th);
3438 GET_THREAD()->errinfo = nomem_error;
3439 JUMP_TAG(TAG_RAISE);
3440 }
3441 rb_thread_raised_set(th, RAISED_NOMEMORY);
3442 rb_exc_raise(nomem_error);
3443 }
3444
3445 static void *
3446 aligned_malloc(size_t alignment, size_t size)
3447 {
3448 void *res;
3449
3450 #if defined __MINGW32__
3451 res = __mingw_aligned_malloc(size, alignment);
3452 #elif defined _WIN32 && !defined __CYGWIN__
3453 void *_aligned_malloc(size_t, size_t);
3454 res = _aligned_malloc(size, alignment);
3455 #elif defined(HAVE_POSIX_MEMALIGN)
3456 if (posix_memalign(&res, alignment, size) == 0) {
3457 return res;
3458 }
3459 else {
3460 return NULL;
3461 }
3462 #elif defined(HAVE_MEMALIGN)
3463 res = memalign(alignment, size);
3464 #else
3465 char* aligned;
3466 res = malloc(alignment + size + sizeof(void*));
3467 aligned = (char*)res + alignment + sizeof(void*);
3468 aligned -= ((VALUE)aligned & (alignment - 1));
3469 ((void**)aligned)[-1] = res;
3470 res = (void*)aligned;
3471 #endif
3472
3473 #if defined(_DEBUG) || defined(GC_DEBUG)
3474
3475 assert((alignment - 1) & alignment == 0);
3476 assert(alignment % sizeof(void*) == 0);
3477 #endif
3478 return res;
3479 }
3480
3481 static void
3482 aligned_free(void *ptr)
3483 {
3484 #if defined __MINGW32__
3485 __mingw_aligned_free(ptr);
3486 #elif defined _WIN32 && !defined __CYGWIN__
3487 _aligned_free(ptr);
3488 #elif defined(HAVE_MEMALIGN) || defined(HAVE_POSIX_MEMALIGN)
3489 free(ptr);
3490 #else
3491 free(((void**)ptr)[-1]);
3492 #endif
3493 }
3494
3495 static inline size_t
3496 vm_malloc_prepare(rb_objspace_t *objspace, size_t size)
3497 {
3498 if ((ssize_t)size < 0) {
3499 negative_size_allocation_error("negative allocation size (or too big)");
3500 }
3501 if (size == 0) size = 1;
3502
3503 #if CALC_EXACT_MALLOC_SIZE
3504 size += sizeof(size_t);
3505 #endif
3506
3507 if ((ruby_gc_stress && !ruby_disable_gc_stress) ||
3508 (malloc_increase+size) > malloc_limit) {
3509 garbage_collect_with_gvl(objspace);
3510 }
3511
3512 return size;
3513 }
3514
3515 static inline void *
3516 vm_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
3517 {
3518 ATOMIC_SIZE_ADD(malloc_increase, size);
3519
3520 #if CALC_EXACT_MALLOC_SIZE
3521 ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, size);
3522 ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
3523 ((size_t *)mem)[0] = size;
3524 mem = (size_t *)mem + 1;
3525 #endif
3526
3527 return mem;
3528 }
3529
3530 #define TRY_WITH_GC(alloc) do { \
3531 if (!(alloc) && \
3532 (!garbage_collect_with_gvl(objspace) || \
3533 !(alloc))) { \
3534 ruby_memerror(); \
3535 } \
3536 } while (0)
3537
3538 static void *
3539 vm_xmalloc(rb_objspace_t *objspace, size_t size)
3540 {
3541 void *mem;
3542
3543 size = vm_malloc_prepare(objspace, size);
3544 TRY_WITH_GC(mem = malloc(size));
3545 return vm_malloc_fixup(objspace, mem, size);
3546 }
3547
3548 static void *
3549 vm_xrealloc(rb_objspace_t *objspace, void *ptr, size_t size)
3550 {
3551 void *mem;
3552 #if CALC_EXACT_MALLOC_SIZE
3553 size_t oldsize;
3554 #endif
3555
3556 if ((ssize_t)size < 0) {
3557 negative_size_allocation_error("negative re-allocation size");
3558 }
3559
3560 if (!ptr) return vm_xmalloc(objspace, size);
3561
3562
3563
3564
3565
3566
3567 if (size == 0) {
3568 vm_xfree(objspace, ptr);
3569 return 0;
3570 }
3571 if (ruby_gc_stress && !ruby_disable_gc_stress)
3572 garbage_collect_with_gvl(objspace);
3573
3574 #if CALC_EXACT_MALLOC_SIZE
3575 size += sizeof(size_t);
3576 ptr = (size_t *)ptr - 1;
3577 oldsize = ((size_t *)ptr)[0];
3578 #endif
3579
3580 mem = realloc(ptr, size);
3581 if (!mem) {
3582 if (garbage_collect_with_gvl(objspace)) {
3583 mem = realloc(ptr, size);
3584 }
3585 if (!mem) {
3586 ruby_memerror();
3587 }
3588 }
3589 ATOMIC_SIZE_ADD(malloc_increase, size);
3590
3591 #if CALC_EXACT_MALLOC_SIZE
3592 ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, size - oldsize);
3593 ((size_t *)mem)[0] = size;
3594 mem = (size_t *)mem + 1;
3595 #endif
3596
3597 return mem;
3598 }
3599
3600 static void
3601 vm_xfree(rb_objspace_t *objspace, void *ptr)
3602 {
3603 #if CALC_EXACT_MALLOC_SIZE
3604 size_t size;
3605 ptr = ((size_t *)ptr) - 1;
3606 size = ((size_t*)ptr)[0];
3607 if (size) {
3608 ATOMIC_SIZE_SUB(objspace->malloc_params.allocated_size, size);
3609 ATOMIC_SIZE_DEC(objspace->malloc_params.allocations);
3610 }
3611 #endif
3612
3613 free(ptr);
3614 }
3615
3616 void *
3617 ruby_xmalloc(size_t size)
3618 {
3619 return vm_xmalloc(&rb_objspace, size);
3620 }
3621
3622 static inline size_t
3623 xmalloc2_size(size_t n, size_t size)
3624 {
3625 size_t len = size * n;
3626 if (n != 0 && size != len / n) {
3627 rb_raise(rb_eArgError, "malloc: possible integer overflow");
3628 }
3629 return len;
3630 }
3631
3632 void *
3633 ruby_xmalloc2(size_t n, size_t size)
3634 {
3635 return vm_xmalloc(&rb_objspace, xmalloc2_size(n, size));
3636 }
3637
3638 static void *
3639 vm_xcalloc(rb_objspace_t *objspace, size_t count, size_t elsize)
3640 {
3641 void *mem;
3642 size_t size;
3643
3644 size = xmalloc2_size(count, elsize);
3645 size = vm_malloc_prepare(objspace, size);
3646
3647 TRY_WITH_GC(mem = calloc(1, size));
3648 return vm_malloc_fixup(objspace, mem, size);
3649 }
3650
3651 void *
3652 ruby_xcalloc(size_t n, size_t size)
3653 {
3654 return vm_xcalloc(&rb_objspace, n, size);
3655 }
3656
3657 void *
3658 ruby_xrealloc(void *ptr, size_t size)
3659 {
3660 return vm_xrealloc(&rb_objspace, ptr, size);
3661 }
3662
3663 void *
3664 ruby_xrealloc2(void *ptr, size_t n, size_t size)
3665 {
3666 size_t len = size * n;
3667 if (n != 0 && size != len / n) {
3668 rb_raise(rb_eArgError, "realloc: possible integer overflow");
3669 }
3670 return ruby_xrealloc(ptr, len);
3671 }
3672
3673 void
3674 ruby_xfree(void *x)
3675 {
3676 if (x)
3677 vm_xfree(&rb_objspace, x);
3678 }
3679
3680
3681
3682
3683
3684 void *
3685 ruby_mimmalloc(size_t size)
3686 {
3687 void *mem;
3688 #if CALC_EXACT_MALLOC_SIZE
3689 size += sizeof(size_t);
3690 #endif
3691 mem = malloc(size);
3692 #if CALC_EXACT_MALLOC_SIZE
3693
3694 ((size_t *)mem)[0] = 0;
3695 mem = (size_t *)mem + 1;
3696 #endif
3697 return mem;
3698 }
3699
3700 #if CALC_EXACT_MALLOC_SIZE
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710 static VALUE
3711 gc_malloc_allocated_size(VALUE self)
3712 {
3713 return UINT2NUM(rb_objspace.malloc_params.allocated_size);
3714 }
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725 static VALUE
3726 gc_malloc_allocations(VALUE self)
3727 {
3728 return UINT2NUM(rb_objspace.malloc_params.allocations);
3729 }
3730 #endif
3731
3732
3733
3734
3735
3736 struct weakmap {
3737 st_table *obj2wmap;
3738 st_table *wmap2obj;
3739 VALUE final;
3740 };
3741
3742 static int
3743 wmap_mark_map(st_data_t key, st_data_t val, st_data_t arg)
3744 {
3745 gc_mark_ptr((rb_objspace_t *)arg, (VALUE)val);
3746 return ST_CONTINUE;
3747 }
3748
3749 static void
3750 wmap_mark(void *ptr)
3751 {
3752 struct weakmap *w = ptr;
3753 st_foreach(w->obj2wmap, wmap_mark_map, (st_data_t)&rb_objspace);
3754 rb_gc_mark(w->final);
3755 }
3756
3757 static int
3758 wmap_free_map(st_data_t key, st_data_t val, st_data_t arg)
3759 {
3760 rb_ary_resize((VALUE)val, 0);
3761 return ST_CONTINUE;
3762 }
3763
3764 static void
3765 wmap_free(void *ptr)
3766 {
3767 struct weakmap *w = ptr;
3768 st_foreach(w->obj2wmap, wmap_free_map, 0);
3769 st_free_table(w->obj2wmap);
3770 st_free_table(w->wmap2obj);
3771 }
3772
3773 size_t rb_ary_memsize(VALUE ary);
3774 static int
3775 wmap_memsize_map(st_data_t key, st_data_t val, st_data_t arg)
3776 {
3777 *(size_t *)arg += rb_ary_memsize((VALUE)val);
3778 return ST_CONTINUE;
3779 }
3780
3781 static size_t
3782 wmap_memsize(const void *ptr)
3783 {
3784 size_t size;
3785 const struct weakmap *w = ptr;
3786 if (!w) return 0;
3787 size = sizeof(*w);
3788 size += st_memsize(w->obj2wmap);
3789 size += st_memsize(w->wmap2obj);
3790 st_foreach(w->obj2wmap, wmap_memsize_map, (st_data_t)&size);
3791 return size;
3792 }
3793
3794 static const rb_data_type_t weakmap_type = {
3795 "weakmap",
3796 {
3797 wmap_mark,
3798 wmap_free,
3799 wmap_memsize,
3800 }
3801 };
3802
3803 static VALUE
3804 wmap_allocate(VALUE klass)
3805 {
3806 struct weakmap *w;
3807 VALUE obj = TypedData_Make_Struct(klass, struct weakmap, &weakmap_type, w);
3808 w->obj2wmap = st_init_numtable();
3809 w->wmap2obj = st_init_numtable();
3810 w->final = rb_obj_method(obj, ID2SYM(rb_intern("finalize")));
3811 return obj;
3812 }
3813
3814 static int
3815 wmap_final_func(st_data_t *key, st_data_t *value, st_data_t arg, int existing)
3816 {
3817 VALUE wmap, ary;
3818 if (!existing) return ST_STOP;
3819 wmap = (VALUE)arg, ary = (VALUE)*value;
3820 rb_ary_delete_same(ary, wmap);
3821 if (!RARRAY_LEN(ary)) return ST_DELETE;
3822 return ST_CONTINUE;
3823 }
3824
3825 static VALUE
3826 wmap_finalize(VALUE self, VALUE objid)
3827 {
3828 st_data_t orig, wmap, data;
3829 VALUE obj, rids;
3830 long i;
3831 struct weakmap *w;
3832
3833 TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
3834
3835 obj = obj_id_to_ref(objid);
3836
3837
3838 orig = (st_data_t)obj;
3839 if (st_delete(w->obj2wmap, &orig, &data)) {
3840 rids = (VALUE)data;
3841 for (i = 0; i < RARRAY_LEN(rids); ++i) {
3842 wmap = (st_data_t)RARRAY_PTR(rids)[i];
3843 st_delete(w->wmap2obj, &wmap, NULL);
3844 }
3845 }
3846
3847 wmap = (st_data_t)obj;
3848 if (st_delete(w->wmap2obj, &wmap, &orig)) {
3849 wmap = (st_data_t)obj;
3850 st_update(w->obj2wmap, orig, wmap_final_func, wmap);
3851 }
3852 return self;
3853 }
3854
3855
3856 static VALUE
3857 wmap_aset(VALUE self, VALUE wmap, VALUE orig)
3858 {
3859 st_data_t data;
3860 VALUE rids;
3861 struct weakmap *w;
3862
3863 TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
3864 rb_define_final(orig, w->final);
3865 rb_define_final(wmap, w->final);
3866 if (st_lookup(w->obj2wmap, (st_data_t)orig, &data)) {
3867 rids = (VALUE)data;
3868 }
3869 else {
3870 rids = rb_ary_tmp_new(1);
3871 st_insert(w->obj2wmap, (st_data_t)orig, (st_data_t)rids);
3872 }
3873 rb_ary_push(rids, wmap);
3874 st_insert(w->wmap2obj, (st_data_t)wmap, (st_data_t)orig);
3875 return nonspecial_obj_id(orig);
3876 }
3877
3878
3879 static VALUE
3880 wmap_aref(VALUE self, VALUE wmap)
3881 {
3882 st_data_t data;
3883 VALUE obj;
3884 struct weakmap *w;
3885 rb_objspace_t *objspace = &rb_objspace;
3886
3887 TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
3888 if (!st_lookup(w->wmap2obj, (st_data_t)wmap, &data)) return Qnil;
3889 obj = (VALUE)data;
3890 if (!is_id_value(objspace, obj)) return Qnil;
3891 if (!is_live_object(objspace, obj)) return Qnil;
3892 return obj;
3893 }
3894
3895
3896
3897
3898
3899
3900 static inline void gc_prof_set_heap_info(rb_objspace_t *, gc_profile_record *);
3901 #define GC_PROFILE_RECORD_DEFAULT_SIZE 100
3902
3903 static double
3904 getrusage_time(void)
3905 {
3906 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
3907 struct timespec ts;
3908
3909 if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts) == 0) {
3910 return ts.tv_sec + ts.tv_nsec * 1e-9;
3911 }
3912 return 0.0;
3913 #elif defined RUSAGE_SELF
3914 struct rusage usage;
3915 struct timeval time;
3916 getrusage(RUSAGE_SELF, &usage);
3917 time = usage.ru_utime;
3918 return time.tv_sec + time.tv_usec * 1e-6;
3919 #elif defined _WIN32
3920 FILETIME creation_time, exit_time, kernel_time, user_time;
3921 ULARGE_INTEGER ui;
3922 LONG_LONG q;
3923 double t;
3924
3925 if (GetProcessTimes(GetCurrentProcess(),
3926 &creation_time, &exit_time, &kernel_time, &user_time) == 0)
3927 {
3928 return 0.0;
3929 }
3930 memcpy(&ui, &user_time, sizeof(FILETIME));
3931 q = ui.QuadPart / 10L;
3932 t = (DWORD)(q % 1000000L) * 1e-6;
3933 q /= 1000000L;
3934 #ifdef __GNUC__
3935 t += q;
3936 #else
3937 t += (double)(DWORD)(q >> 16) * (1 << 16);
3938 t += (DWORD)q & ~(~0 << 16);
3939 #endif
3940 return t;
3941 #else
3942 return 0.0;
3943 #endif
3944 }
3945
3946 static inline void
3947 gc_prof_timer_start(rb_objspace_t *objspace)
3948 {
3949 if (objspace->profile.run) {
3950 size_t count = objspace->profile.count;
3951
3952 if (!objspace->profile.record) {
3953 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
3954 objspace->profile.record = malloc(sizeof(gc_profile_record) * objspace->profile.size);
3955 }
3956 if (count >= objspace->profile.size) {
3957 objspace->profile.size += 1000;
3958 objspace->profile.record = realloc(objspace->profile.record, sizeof(gc_profile_record) * objspace->profile.size);
3959 }
3960 if (!objspace->profile.record) {
3961 rb_bug("gc_profile malloc or realloc miss");
3962 }
3963 MEMZERO(&objspace->profile.record[count], gc_profile_record, 1);
3964 objspace->profile.record[count].gc_time = getrusage_time();
3965 objspace->profile.record[objspace->profile.count].gc_invoke_time =
3966 objspace->profile.record[count].gc_time - objspace->profile.invoke_time;
3967 }
3968 }
3969
3970 static inline void
3971 gc_prof_timer_stop(rb_objspace_t *objspace, int marked)
3972 {
3973 if (objspace->profile.run) {
3974 double gc_time = 0;
3975 size_t count = objspace->profile.count;
3976 gc_profile_record *record = &objspace->profile.record[count];
3977
3978 gc_time = getrusage_time() - record->gc_time;
3979 if (gc_time < 0) gc_time = 0;
3980 record->gc_time = gc_time;
3981 record->is_marked = !!(marked);
3982 gc_prof_set_heap_info(objspace, record);
3983 objspace->profile.count++;
3984 }
3985 }
3986
3987 #if !GC_PROFILE_MORE_DETAIL
3988
3989 static inline void
3990 gc_prof_mark_timer_start(rb_objspace_t *objspace)
3991 {
3992 if (RUBY_DTRACE_GC_MARK_BEGIN_ENABLED()) {
3993 RUBY_DTRACE_GC_MARK_BEGIN();
3994 }
3995 }
3996
3997 static inline void
3998 gc_prof_mark_timer_stop(rb_objspace_t *objspace)
3999 {
4000 if (RUBY_DTRACE_GC_MARK_END_ENABLED()) {
4001 RUBY_DTRACE_GC_MARK_END();
4002 }
4003 }
4004
4005 static inline void
4006 gc_prof_sweep_timer_start(rb_objspace_t *objspace)
4007 {
4008 if (RUBY_DTRACE_GC_SWEEP_BEGIN_ENABLED()) {
4009 RUBY_DTRACE_GC_SWEEP_BEGIN();
4010 }
4011 }
4012
4013 static inline void
4014 gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
4015 {
4016 if (RUBY_DTRACE_GC_SWEEP_END_ENABLED()) {
4017 RUBY_DTRACE_GC_SWEEP_END();
4018 }
4019 }
4020
4021 static inline void
4022 gc_prof_set_malloc_info(rb_objspace_t *objspace)
4023 {
4024 }
4025
4026 static inline void
4027 gc_prof_set_heap_info(rb_objspace_t *objspace, gc_profile_record *record)
4028 {
4029 size_t live = objspace_live_num(objspace);
4030 size_t total = heaps_used * HEAP_OBJ_LIMIT;
4031
4032 record->heap_total_objects = total;
4033 record->heap_use_size = live * sizeof(RVALUE);
4034 record->heap_total_size = total * sizeof(RVALUE);
4035 }
4036
4037 #else
4038
4039 static inline void
4040 gc_prof_mark_timer_start(rb_objspace_t *objspace)
4041 {
4042 if (RUBY_DTRACE_GC_MARK_BEGIN_ENABLED()) {
4043 RUBY_DTRACE_GC_MARK_BEGIN();
4044 }
4045 if (objspace->profile.run) {
4046 size_t count = objspace->profile.count;
4047
4048 objspace->profile.record[count].gc_mark_time = getrusage_time();
4049 }
4050 }
4051
4052 static inline void
4053 gc_prof_mark_timer_stop(rb_objspace_t *objspace)
4054 {
4055 if (RUBY_DTRACE_GC_MARK_END_ENABLED()) {
4056 RUBY_DTRACE_GC_MARK_END();
4057 }
4058 if (objspace->profile.run) {
4059 double mark_time = 0;
4060 size_t count = objspace->profile.count;
4061 gc_profile_record *record = &objspace->profile.record[count];
4062
4063 mark_time = getrusage_time() - record->gc_mark_time;
4064 if (mark_time < 0) mark_time = 0;
4065 record->gc_mark_time = mark_time;
4066 }
4067 }
4068
4069 static inline void
4070 gc_prof_sweep_timer_start(rb_objspace_t *objspace)
4071 {
4072 if (RUBY_DTRACE_GC_SWEEP_BEGIN_ENABLED()) {
4073 RUBY_DTRACE_GC_SWEEP_BEGIN();
4074 }
4075 if (objspace->profile.run) {
4076 size_t count = objspace->profile.count;
4077
4078 objspace->profile.record[count].gc_sweep_time = getrusage_time();
4079 }
4080 }
4081
4082 static inline void
4083 gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
4084 {
4085 if (RUBY_DTRACE_GC_SWEEP_END_ENABLED()) {
4086 RUBY_DTRACE_GC_SWEEP_END();
4087 }
4088 if (objspace->profile.run) {
4089 double sweep_time = 0;
4090 size_t count = objspace->profile.count;
4091 gc_profile_record *record = &objspace->profile.record[count];
4092
4093 sweep_time = getrusage_time() - record->gc_sweep_time;\
4094 if (sweep_time < 0) sweep_time = 0;\
4095 record->gc_sweep_time = sweep_time;
4096 }
4097 }
4098
4099 static inline void
4100 gc_prof_set_malloc_info(rb_objspace_t *objspace)
4101 {
4102 if (objspace->profile.run) {
4103 gc_profile_record *record = &objspace->profile.record[objspace->profile.count];
4104 if (record) {
4105 record->allocate_increase = malloc_increase;
4106 record->allocate_limit = malloc_limit;
4107 }
4108 }
4109 }
4110
4111 static inline void
4112 gc_prof_set_heap_info(rb_objspace_t *objspace, gc_profile_record *record)
4113 {
4114 size_t live = objspace->heap.live_num;
4115 size_t total = heaps_used * HEAP_OBJ_LIMIT;
4116
4117 record->heap_use_slots = heaps_used;
4118 record->heap_live_objects = live;
4119 record->heap_free_objects = total - live;
4120 record->heap_total_objects = total;
4121 record->have_finalize = deferred_final_list ? Qtrue : Qfalse;
4122 record->heap_use_size = live * sizeof(RVALUE);
4123 record->heap_total_size = total * sizeof(RVALUE);
4124 }
4125
4126 #endif
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137 static VALUE
4138 gc_profile_clear(void)
4139 {
4140 rb_objspace_t *objspace = &rb_objspace;
4141
4142 if (GC_PROFILE_RECORD_DEFAULT_SIZE * 2 < objspace->profile.size) {
4143 objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE * 2;
4144 objspace->profile.record = realloc(objspace->profile.record, sizeof(gc_profile_record) * objspace->profile.size);
4145 if (!objspace->profile.record) {
4146 rb_memerror();
4147 }
4148 }
4149 MEMZERO(objspace->profile.record, gc_profile_record, objspace->profile.size);
4150 objspace->profile.count = 0;
4151 return Qnil;
4152 }
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204 static VALUE
4205 gc_profile_record_get(void)
4206 {
4207 VALUE prof;
4208 VALUE gc_profile = rb_ary_new();
4209 size_t i;
4210 rb_objspace_t *objspace = (&rb_objspace);
4211
4212 if (!objspace->profile.run) {
4213 return Qnil;
4214 }
4215
4216 for (i =0; i < objspace->profile.count; i++) {
4217 prof = rb_hash_new();
4218 rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(objspace->profile.record[i].gc_time));
4219 rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(objspace->profile.record[i].gc_invoke_time));
4220 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(objspace->profile.record[i].heap_use_size));
4221 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(objspace->profile.record[i].heap_total_size));
4222 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(objspace->profile.record[i].heap_total_objects));
4223 rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), objspace->profile.record[i].is_marked);
4224 #if GC_PROFILE_MORE_DETAIL
4225 rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(objspace->profile.record[i].gc_mark_time));
4226 rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(objspace->profile.record[i].gc_sweep_time));
4227 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(objspace->profile.record[i].allocate_increase));
4228 rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(objspace->profile.record[i].allocate_limit));
4229 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SLOTS")), SIZET2NUM(objspace->profile.record[i].heap_use_slots));
4230 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(objspace->profile.record[i].heap_live_objects));
4231 rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(objspace->profile.record[i].heap_free_objects));
4232 rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), objspace->profile.record[i].have_finalize);
4233 #endif
4234 rb_ary_push(gc_profile, prof);
4235 }
4236
4237 return gc_profile;
4238 }
4239
4240 static void
4241 gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
4242 {
4243 rb_objspace_t *objspace = &rb_objspace;
4244 size_t count = objspace->profile.count;
4245
4246 if (objspace->profile.run && count) {
4247 int index = 1;
4248 size_t i;
4249 gc_profile_record r;
4250 append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->count));
4251 append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
4252 for (i = 0; i < count; i++) {
4253 r = objspace->profile.record[i];
4254 #if !GC_PROFILE_MORE_DETAIL
4255 if (r.is_marked) {
4256 #endif
4257 append(out, rb_sprintf("%5d %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
4258 index++, r.gc_invoke_time, r.heap_use_size,
4259 r.heap_total_size, r.heap_total_objects, r.gc_time*1000));
4260 #if !GC_PROFILE_MORE_DETAIL
4261 }
4262 #endif
4263 }
4264 #if GC_PROFILE_MORE_DETAIL
4265 append(out, rb_str_new_cstr("\n\n" \
4266 "More detail.\n" \
4267 "Index Allocate Increase Allocate Limit Use Slot Have Finalize Mark Time(ms) Sweep Time(ms)\n"));
4268 index = 1;
4269 for (i = 0; i < count; i++) {
4270 r = objspace->profile.record[i];
4271 append(out, rb_sprintf("%5d %17"PRIuSIZE" %17"PRIuSIZE" %9"PRIuSIZE" %14s %25.20f %25.20f\n",
4272 index++, r.allocate_increase, r.allocate_limit,
4273 r.heap_use_slots, (r.have_finalize ? "true" : "false"),
4274 r.gc_mark_time*1000, r.gc_sweep_time*1000));
4275 }
4276 #endif
4277 }
4278 }
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291 static VALUE
4292 gc_profile_result(void)
4293 {
4294 VALUE str = rb_str_buf_new(0);
4295 gc_profile_dump_on(str, rb_str_buf_append);
4296 return str;
4297 }
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308 static VALUE
4309 gc_profile_report(int argc, VALUE *argv, VALUE self)
4310 {
4311 VALUE out;
4312
4313 if (argc == 0) {
4314 out = rb_stdout;
4315 }
4316 else {
4317 rb_scan_args(argc, argv, "01", &out);
4318 }
4319 gc_profile_dump_on(out, rb_io_write);
4320
4321 return Qnil;
4322 }
4323
4324
4325
4326
4327
4328
4329
4330
4331 static VALUE
4332 gc_profile_total_time(VALUE self)
4333 {
4334 double time = 0;
4335 rb_objspace_t *objspace = &rb_objspace;
4336 size_t i;
4337
4338 if (objspace->profile.run && objspace->profile.count) {
4339 for (i = 0; i < objspace->profile.count; i++) {
4340 time += objspace->profile.record[i].gc_time;
4341 }
4342 }
4343 return DBL2NUM(time);
4344 }
4345
4346
4347
4348
4349
4350
4351
4352
4353 static VALUE
4354 gc_profile_enable_get(VALUE self)
4355 {
4356 rb_objspace_t *objspace = &rb_objspace;
4357 return objspace->profile.run ? Qtrue : Qfalse;
4358 }
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368 static VALUE
4369 gc_profile_enable(void)
4370 {
4371 rb_objspace_t *objspace = &rb_objspace;
4372
4373 objspace->profile.run = TRUE;
4374 return Qnil;
4375 }
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385 static VALUE
4386 gc_profile_disable(void)
4387 {
4388 rb_objspace_t *objspace = &rb_objspace;
4389
4390 objspace->profile.run = FALSE;
4391 return Qnil;
4392 }
4393
4394 #ifdef GC_DEBUG
4395
4396
4397
4398
4399
4400 void
4401 rb_gcdebug_print_obj_condition(VALUE obj)
4402 {
4403 rb_objspace_t *objspace = &rb_objspace;
4404
4405 if (is_pointer_to_heap(objspace, (void *)obj)) {
4406 fprintf(stderr, "pointer to heap?: true\n");
4407 }
4408 else {
4409 fprintf(stderr, "pointer to heap?: false\n");
4410 return;
4411 }
4412 fprintf(stderr, "marked?: %s\n",
4413 MARKED_IN_BITMAP(GET_HEAP_BITMAP(obj), obj) ? "true" : "false");
4414 if (is_lazy_sweeping(objspace)) {
4415 fprintf(stderr, "lazy sweeping?: true\n");
4416 fprintf(stderr, "swept?: %s\n",
4417 is_swept_object(objspace, obj) ? "done" : "not yet");
4418 }
4419 else {
4420 fprintf(stderr, "lazy sweeping?: false\n");
4421 }
4422 }
4423
4424 static VALUE
4425 gcdebug_sential(VALUE obj, VALUE name)
4426 {
4427 fprintf(stderr, "WARNING: object %s(%p) is inadvertently collected\n", (char *)name, (void *)obj);
4428 return Qnil;
4429 }
4430
4431 void
4432 rb_gcdebug_sentinel(VALUE obj, const char *name)
4433 {
4434 rb_define_final(obj, rb_proc_new(gcdebug_sential, (VALUE)name));
4435 }
4436 #endif
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507 void
4508 Init_GC(void)
4509 {
4510 VALUE rb_mObSpace;
4511 VALUE rb_mProfiler;
4512
4513 rb_mGC = rb_define_module("GC");
4514 rb_define_singleton_method(rb_mGC, "start", rb_gc_start, 0);
4515 rb_define_singleton_method(rb_mGC, "enable", rb_gc_enable, 0);
4516 rb_define_singleton_method(rb_mGC, "disable", rb_gc_disable, 0);
4517 rb_define_singleton_method(rb_mGC, "stress", gc_stress_get, 0);
4518 rb_define_singleton_method(rb_mGC, "stress=", gc_stress_set, 1);
4519 rb_define_singleton_method(rb_mGC, "count", gc_count, 0);
4520 rb_define_singleton_method(rb_mGC, "stat", gc_stat, -1);
4521 rb_define_method(rb_mGC, "garbage_collect", rb_gc_start, 0);
4522
4523 rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
4524 rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
4525 rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
4526 rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, 0);
4527 rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
4528 rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
4529 rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
4530 rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
4531 rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
4532
4533 rb_mObSpace = rb_define_module("ObjectSpace");
4534 rb_define_module_function(rb_mObSpace, "each_object", os_each_obj, -1);
4535 rb_define_module_function(rb_mObSpace, "garbage_collect", rb_gc_start, 0);
4536
4537 rb_define_module_function(rb_mObSpace, "define_finalizer", define_final, -1);
4538 rb_define_module_function(rb_mObSpace, "undefine_finalizer", undefine_final, 1);
4539
4540 rb_define_module_function(rb_mObSpace, "_id2ref", id2ref, 1);
4541
4542 nomem_error = rb_exc_new3(rb_eNoMemError,
4543 rb_obj_freeze(rb_str_new2("failed to allocate memory")));
4544 OBJ_TAINT(nomem_error);
4545 OBJ_FREEZE(nomem_error);
4546
4547 rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0);
4548 rb_define_method(rb_mKernel, "object_id", rb_obj_id, 0);
4549
4550 rb_define_module_function(rb_mObSpace, "count_objects", count_objects, -1);
4551
4552 {
4553 VALUE rb_cWeakMap = rb_define_class_under(rb_mObSpace, "WeakMap", rb_cObject);
4554 rb_define_alloc_func(rb_cWeakMap, wmap_allocate);
4555 rb_define_method(rb_cWeakMap, "[]=", wmap_aset, 2);
4556 rb_define_method(rb_cWeakMap, "[]", wmap_aref, 1);
4557 rb_define_private_method(rb_cWeakMap, "finalize", wmap_finalize, 1);
4558 }
4559
4560 #if CALC_EXACT_MALLOC_SIZE
4561 rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
4562 rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
4563 #endif
4564 }