Ruby  2.7.0p0(2019-12-25revision647ee6f091eafcce70ffb75ddf7e121e192ab217)
gc.c
Go to the documentation of this file.
1 /**********************************************************************
2 
3  gc.c -
4 
5  $Author$
6  created at: Tue Oct 5 09:44:46 JST 1993
7 
8  Copyright (C) 1993-2007 Yukihiro Matsumoto
9  Copyright (C) 2000 Network Applied Communication Laboratory, Inc.
10  Copyright (C) 2000 Information-technology Promotion Agency, Japan
11 
12 **********************************************************************/
13 
14 #define rb_data_object_alloc rb_data_object_alloc
15 #define rb_data_typed_object_alloc rb_data_typed_object_alloc
16 
17 #include "ruby/encoding.h"
18 #include "ruby/io.h"
19 #include "ruby/st.h"
20 #include "ruby/re.h"
21 #include "ruby/thread.h"
22 #include "ruby/util.h"
23 #include "ruby/debug.h"
24 #include "internal.h"
25 #include "eval_intern.h"
26 #include "vm_core.h"
27 #include "builtin.h"
28 #include "gc.h"
29 #include "constant.h"
30 #include "ruby_atomic.h"
31 #include "probes.h"
32 #include "id_table.h"
33 #include "symbol.h"
34 #include <stdio.h>
35 #include <stdarg.h>
36 #include <setjmp.h>
37 #include <sys/types.h>
38 #include "ruby_assert.h"
39 #include "debug_counter.h"
40 #include "transient_heap.h"
41 #include "mjit.h"
42 
43 #undef rb_data_object_wrap
44 
45 #ifndef HAVE_MALLOC_USABLE_SIZE
46 # ifdef _WIN32
47 # define HAVE_MALLOC_USABLE_SIZE
48 # define malloc_usable_size(a) _msize(a)
49 # elif defined HAVE_MALLOC_SIZE
50 # define HAVE_MALLOC_USABLE_SIZE
51 # define malloc_usable_size(a) malloc_size(a)
52 # endif
53 #endif
54 #ifdef HAVE_MALLOC_USABLE_SIZE
55 # ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
56 # include RUBY_ALTERNATIVE_MALLOC_HEADER
57 # elif HAVE_MALLOC_H
58 # include <malloc.h>
59 # elif defined(HAVE_MALLOC_NP_H)
60 # include <malloc_np.h>
61 # elif defined(HAVE_MALLOC_MALLOC_H)
62 # include <malloc/malloc.h>
63 # endif
64 #endif
65 
66 #ifdef HAVE_SYS_TIME_H
67 #include <sys/time.h>
68 #endif
69 
70 #ifdef HAVE_SYS_RESOURCE_H
71 #include <sys/resource.h>
72 #endif
73 
74 #if defined _WIN32 || defined __CYGWIN__
75 #include <windows.h>
76 #elif defined(HAVE_POSIX_MEMALIGN)
77 #elif defined(HAVE_MEMALIGN)
78 #include <malloc.h>
79 #endif
80 
81 #define rb_setjmp(env) RUBY_SETJMP(env)
82 #define rb_jmp_buf rb_jmpbuf_t
83 
84 #if defined(_MSC_VER) && defined(_WIN64)
85 #include <intrin.h>
86 #pragma intrinsic(_umul128)
87 #endif
88 
89 /* Expecting this struct to be eliminated by function inlinings */
90 struct optional {
91  bool left;
92  size_t right;
93 };
94 
95 static inline struct optional
96 size_mul_overflow(size_t x, size_t y)
97 {
98  bool p;
99  size_t z;
100 #if 0
101 
102 #elif defined(HAVE_BUILTIN___BUILTIN_MUL_OVERFLOW)
103  p = __builtin_mul_overflow(x, y, &z);
104 
105 #elif defined(DSIZE_T)
106  RB_GNUC_EXTENSION DSIZE_T dx = x;
107  RB_GNUC_EXTENSION DSIZE_T dy = y;
108  RB_GNUC_EXTENSION DSIZE_T dz = dx * dy;
109  p = dz > SIZE_MAX;
110  z = (size_t)dz;
111 
112 #elif defined(_MSC_VER) && defined(_WIN64)
113  unsigned __int64 dp;
114  unsigned __int64 dz = _umul128(x, y, &dp);
115  p = (bool)dp;
116  z = (size_t)dz;
117 
118 #else
119  /* https://wiki.sei.cmu.edu/confluence/display/c/INT30-C.+Ensure+that+unsigned+integer+operations+do+not+wrap */
120  p = (y != 0) && (x > SIZE_MAX / y);
121  z = x * y;
122 
123 #endif
124  return (struct optional) { p, z, };
125 }
126 
127 static inline struct optional
128 size_add_overflow(size_t x, size_t y)
129 {
130  size_t z;
131  bool p;
132 #if 0
133 
134 #elif defined(HAVE_BUILTIN___BUILTIN_ADD_OVERFLOW)
135  p = __builtin_add_overflow(x, y, &z);
136 
137 #elif defined(DSIZE_T)
138  RB_GNUC_EXTENSION DSIZE_T dx = x;
139  RB_GNUC_EXTENSION DSIZE_T dy = y;
140  RB_GNUC_EXTENSION DSIZE_T dz = dx + dy;
141  p = dz > SIZE_MAX;
142  z = (size_t)dz;
143 
144 #else
145  z = x + y;
146  p = z < y;
147 
148 #endif
149  return (struct optional) { p, z, };
150 }
151 
152 static inline struct optional
153 size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */
154 {
155  struct optional t = size_mul_overflow(x, y);
156  struct optional u = size_add_overflow(t.right, z);
157  return (struct optional) { t.left || u.left, u.right };
158 }
159 
160 static inline struct optional
161 size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
162 {
163  struct optional t = size_mul_overflow(x, y);
164  struct optional u = size_mul_overflow(z, w);
165  struct optional v = size_add_overflow(t.right, u.right);
166  return (struct optional) { t.left || u.left || v.left, v.right };
167 }
168 
169 PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3);
170 
171 static inline size_t
172 size_mul_or_raise(size_t x, size_t y, VALUE exc)
173 {
174  struct optional t = size_mul_overflow(x, y);
175  if (LIKELY(!t.left)) {
176  return t.right;
177  }
178  else if (rb_during_gc()) {
179  rb_memerror(); /* or...? */
180  }
181  else {
182  gc_raise(
183  exc,
184  "integer overflow: %"PRIuSIZE
185  " * %"PRIuSIZE
186  " > %"PRIuSIZE,
187  x, y, SIZE_MAX);
188  }
189 }
190 
191 size_t
192 rb_size_mul_or_raise(size_t x, size_t y, VALUE exc)
193 {
194  return size_mul_or_raise(x, y, exc);
195 }
196 
197 static inline size_t
198 size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
199 {
200  struct optional t = size_mul_add_overflow(x, y, z);
201  if (LIKELY(!t.left)) {
202  return t.right;
203  }
204  else if (rb_during_gc()) {
205  rb_memerror(); /* or...? */
206  }
207  else {
208  gc_raise(
209  exc,
210  "integer overflow: %"PRIuSIZE
211  " * %"PRIuSIZE
212  " + %"PRIuSIZE
213  " > %"PRIuSIZE,
214  x, y, z, SIZE_MAX);
215  }
216 }
217 
218 size_t
219 rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
220 {
221  return size_mul_add_or_raise(x, y, z, exc);
222 }
223 
224 static inline size_t
225 size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc)
226 {
227  struct optional t = size_mul_add_mul_overflow(x, y, z, w);
228  if (LIKELY(!t.left)) {
229  return t.right;
230  }
231  else if (rb_during_gc()) {
232  rb_memerror(); /* or...? */
233  }
234  else {
235  gc_raise(
236  exc,
237  "integer overflow: %"PRIdSIZE
238  " * %"PRIdSIZE
239  " + %"PRIdSIZE
240  " * %"PRIdSIZE
241  " > %"PRIdSIZE,
242  x, y, z, w, SIZE_MAX);
243  }
244 }
245 
246 #if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL
247 /* trick the compiler into thinking a external signal handler uses this */
249 volatile VALUE *
251 {
252  rb_gc_guarded_val = val;
253 
254  return ptr;
255 }
256 #endif
257 
258 #ifndef GC_HEAP_INIT_SLOTS
259 #define GC_HEAP_INIT_SLOTS 10000
260 #endif
261 #ifndef GC_HEAP_FREE_SLOTS
262 #define GC_HEAP_FREE_SLOTS 4096
263 #endif
264 #ifndef GC_HEAP_GROWTH_FACTOR
265 #define GC_HEAP_GROWTH_FACTOR 1.8
266 #endif
267 #ifndef GC_HEAP_GROWTH_MAX_SLOTS
268 #define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */
269 #endif
270 #ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
271 #define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
272 #endif
273 
274 #ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
275 #define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
276 #endif
277 #ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
278 #define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
279 #endif
280 #ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
281 #define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
282 #endif
283 
284 #ifndef GC_MALLOC_LIMIT_MIN
285 #define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
286 #endif
287 #ifndef GC_MALLOC_LIMIT_MAX
288 #define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
289 #endif
290 #ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
291 #define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
292 #endif
293 
294 #ifndef GC_OLDMALLOC_LIMIT_MIN
295 #define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
296 #endif
297 #ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
298 #define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
299 #endif
300 #ifndef GC_OLDMALLOC_LIMIT_MAX
301 #define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
302 #endif
303 
304 #ifndef PRINT_MEASURE_LINE
305 #define PRINT_MEASURE_LINE 0
306 #endif
307 #ifndef PRINT_ENTER_EXIT_TICK
308 #define PRINT_ENTER_EXIT_TICK 0
309 #endif
310 #ifndef PRINT_ROOT_TICKS
311 #define PRINT_ROOT_TICKS 0
312 #endif
313 
314 #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_MEASURE_LINE || PRINT_ROOT_TICKS)
315 #define TICK_TYPE 1
316 
317 typedef struct {
322 
327 
331 
335 
338 
339 static ruby_gc_params_t gc_params = {
344 
349 
353 
357 
358  FALSE,
359 };
360 
361 /* GC_DEBUG:
362  * enable to embed GC debugging information.
363  */
364 #ifndef GC_DEBUG
365 #define GC_DEBUG 0
366 #endif
367 
368 #if USE_RGENGC
369 /* RGENGC_DEBUG:
370  * 1: basic information
371  * 2: remember set operation
372  * 3: mark
373  * 4:
374  * 5: sweep
375  */
376 #ifndef RGENGC_DEBUG
377 #ifdef RUBY_DEVEL
378 #define RGENGC_DEBUG -1
379 #else
380 #define RGENGC_DEBUG 0
381 #endif
382 #endif
383 #if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
384 # define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
385 #else
386 # define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
387 #endif
389 
390 /* RGENGC_CHECK_MODE
391  * 0: disable all assertions
392  * 1: enable assertions (to debug RGenGC)
393  * 2: enable internal consistency check at each GC (for debugging)
394  * 3: enable internal consistency check at each GC steps (for debugging)
395  * 4: enable liveness check
396  * 5: show all references
397  */
398 #ifndef RGENGC_CHECK_MODE
399 #define RGENGC_CHECK_MODE 0
400 #endif
401 
402 // Note: using RUBY_ASSERT_WHEN() extend a macro in expr (info by nobu).
403 #define GC_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr)
404 
405 /* RGENGC_OLD_NEWOBJ_CHECK
406  * 0: disable all assertions
407  * >0: make a OLD object when new object creation.
408  *
409  * Make one OLD object per RGENGC_OLD_NEWOBJ_CHECK WB protected objects creation.
410  */
411 #ifndef RGENGC_OLD_NEWOBJ_CHECK
412 #define RGENGC_OLD_NEWOBJ_CHECK 0
413 #endif
414 
415 /* RGENGC_PROFILE
416  * 0: disable RGenGC profiling
417  * 1: enable profiling for basic information
418  * 2: enable profiling for each types
419  */
420 #ifndef RGENGC_PROFILE
421 #define RGENGC_PROFILE 0
422 #endif
423 
424 /* RGENGC_ESTIMATE_OLDMALLOC
425  * Enable/disable to estimate increase size of malloc'ed size by old objects.
426  * If estimation exceeds threshold, then will invoke full GC.
427  * 0: disable estimation.
428  * 1: enable estimation.
429  */
430 #ifndef RGENGC_ESTIMATE_OLDMALLOC
431 #define RGENGC_ESTIMATE_OLDMALLOC 1
432 #endif
433 
434 /* RGENGC_FORCE_MAJOR_GC
435  * Force major/full GC if this macro is not 0.
436  */
437 #ifndef RGENGC_FORCE_MAJOR_GC
438 #define RGENGC_FORCE_MAJOR_GC 0
439 #endif
440 
441 #else /* USE_RGENGC */
442 
443 #ifdef RGENGC_DEBUG
444 #undef RGENGC_DEBUG
445 #endif
446 #define RGENGC_DEBUG 0
447 #ifdef RGENGC_CHECK_MODE
448 #undef RGENGC_CHECK_MODE
449 #endif
450 #define RGENGC_CHECK_MODE 0
451 #define RGENGC_PROFILE 0
452 #define RGENGC_ESTIMATE_OLDMALLOC 0
453 #define RGENGC_FORCE_MAJOR_GC 0
454 
455 #endif /* USE_RGENGC */
456 
457 #ifndef GC_PROFILE_MORE_DETAIL
458 #define GC_PROFILE_MORE_DETAIL 0
459 #endif
460 #ifndef GC_PROFILE_DETAIL_MEMORY
461 #define GC_PROFILE_DETAIL_MEMORY 0
462 #endif
463 #ifndef GC_ENABLE_INCREMENTAL_MARK
464 #define GC_ENABLE_INCREMENTAL_MARK USE_RINCGC
465 #endif
466 #ifndef GC_ENABLE_LAZY_SWEEP
467 #define GC_ENABLE_LAZY_SWEEP 1
468 #endif
469 #ifndef CALC_EXACT_MALLOC_SIZE
470 #define CALC_EXACT_MALLOC_SIZE USE_GC_MALLOC_OBJ_INFO_DETAILS
471 #endif
472 #if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
473 #ifndef MALLOC_ALLOCATED_SIZE
474 #define MALLOC_ALLOCATED_SIZE 0
475 #endif
476 #else
477 #define MALLOC_ALLOCATED_SIZE 0
478 #endif
479 #ifndef MALLOC_ALLOCATED_SIZE_CHECK
480 #define MALLOC_ALLOCATED_SIZE_CHECK 0
481 #endif
482 
483 #ifndef GC_DEBUG_STRESS_TO_CLASS
484 #define GC_DEBUG_STRESS_TO_CLASS 0
485 #endif
486 
487 #ifndef RGENGC_OBJ_INFO
488 #define RGENGC_OBJ_INFO (RGENGC_DEBUG | RGENGC_CHECK_MODE)
489 #endif
490 
491 typedef enum {
492  GPR_FLAG_NONE = 0x000,
493  /* major reason */
498 #if RGENGC_ESTIMATE_OLDMALLOC
500 #endif
502 
503  /* gc reason */
507  GPR_FLAG_CAPI = 0x800,
508  GPR_FLAG_STRESS = 0x1000,
509 
510  /* others */
515 
520 
521 typedef struct gc_profile_record {
522  int flags;
523 
524  double gc_time;
526 
530 
531 #if GC_PROFILE_MORE_DETAIL
532  double gc_mark_time;
533  double gc_sweep_time;
534 
535  size_t heap_use_pages;
536  size_t heap_live_objects;
537  size_t heap_free_objects;
538 
539  size_t allocate_increase;
540  size_t allocate_limit;
541 
542  double prepare_time;
543  size_t removing_objects;
544  size_t empty_objects;
545 #if GC_PROFILE_DETAIL_MEMORY
546  long maxrss;
547  long minflt;
548  long majflt;
549 #endif
550 #endif
551 #if MALLOC_ALLOCATED_SIZE
552  size_t allocated_size;
553 #endif
554 
555 #if RGENGC_PROFILE > 0
556  size_t old_objects;
557  size_t remembered_normal_objects;
558  size_t remembered_shady_objects;
559 #endif
561 
562 #if defined(_MSC_VER) || defined(__CYGWIN__)
563 #pragma pack(push, 1) /* magic for reducing sizeof(RVALUE): 24 -> 20 */
564 #endif
565 
566 typedef struct RVALUE {
567  union {
568  struct {
569  VALUE flags; /* always 0 for freed obj */
570  struct RVALUE *next;
571  } free;
572  struct RMoved moved;
573  struct RBasic basic;
574  struct RObject object;
575  struct RClass klass;
576  struct RFloat flonum;
577  struct RString string;
578  struct RArray array;
579  struct RRegexp regexp;
580  struct RHash hash;
581  struct RData data;
583  struct RStruct rstruct;
584  struct RBignum bignum;
585  struct RFile file;
586  struct RMatch match;
589  union {
591  struct vm_svar svar;
593  struct vm_ifunc ifunc;
594  struct MEMO memo;
600  } imemo;
601  struct {
602  struct RBasic basic;
606  } values;
607  } as;
608 #if GC_DEBUG
609  const char *file;
610  int line;
611 #endif
612 } RVALUE;
613 
614 #if defined(_MSC_VER) || defined(__CYGWIN__)
615 #pragma pack(pop)
616 #endif
617 
619 enum {
620  BITS_SIZE = sizeof(bits_t),
622 };
623 #define popcount_bits rb_popcount_intptr
624 
626  struct heap_page *page;
627 };
628 
631  /* char gap[]; */
632  /* RVALUE values[]; */
633 };
634 
635 struct gc_list {
637  struct gc_list *next;
638 };
639 
640 #define STACK_CHUNK_SIZE 500
641 
642 typedef struct stack_chunk {
644  struct stack_chunk *next;
645 } stack_chunk_t;
646 
647 typedef struct mark_stack {
650  int index;
651  int limit;
652  size_t cache_size;
654 } mark_stack_t;
655 
656 typedef struct rb_heap_struct {
658 
661  struct list_head pages;
662  struct heap_page *sweeping_page; /* iterator for .pages */
663 #if GC_ENABLE_INCREMENTAL_MARK
665 #endif
666  size_t total_pages; /* total page count in a heap */
667  size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */
668 } rb_heap_t;
669 
670 enum gc_mode {
674 };
675 
676 typedef struct rb_objspace {
677  struct {
678  size_t limit;
679  size_t increase;
680 #if MALLOC_ALLOCATED_SIZE
681  size_t allocated_size;
682  size_t allocations;
683 #endif
684  } malloc_params;
685 
686  struct {
687  unsigned int mode : 2;
688  unsigned int immediate_sweep : 1;
689  unsigned int dont_gc : 1;
690  unsigned int dont_incremental : 1;
691  unsigned int during_gc : 1;
692  unsigned int during_compacting : 1;
693  unsigned int gc_stressful: 1;
694  unsigned int has_hook: 1;
695 #if USE_RGENGC
696  unsigned int during_minor_gc : 1;
697 #endif
698 #if GC_ENABLE_INCREMENTAL_MARK
699  unsigned int during_incremental_marking : 1;
700 #endif
701  } flags;
702 
706 
708  rb_heap_t tomb_heap; /* heap for zombies and ghosts */
709 
710  struct {
712  } atomic_flags;
713 
715  void *data;
716  void (*mark_func)(VALUE v, void *data);
717  } *mark_func_data;
718 
720  size_t marked_slots;
721 
722  struct {
723  struct heap_page **sorted;
729 
730  /* final */
731  size_t final_slots;
733  } heap_pages;
734 
736 
737  struct {
738  int run;
742  size_t next_index;
743  size_t size;
744 
745 #if GC_PROFILE_MORE_DETAIL
746  double prepare_time;
747 #endif
748  double invoke_time;
749 
750 #if USE_RGENGC
754 #if RGENGC_PROFILE > 0
755  size_t total_generated_normal_object_count;
756  size_t total_generated_shady_object_count;
757  size_t total_shade_operation_count;
758  size_t total_promoted_count;
759  size_t total_remembered_normal_object_count;
760  size_t total_remembered_shady_object_count;
761 
762 #if RGENGC_PROFILE >= 2
763  size_t generated_normal_object_count_types[RUBY_T_MASK];
764  size_t generated_shady_object_count_types[RUBY_T_MASK];
765  size_t shade_operation_count_types[RUBY_T_MASK];
766  size_t promoted_types[RUBY_T_MASK];
767  size_t remembered_normal_object_count_types[RUBY_T_MASK];
768  size_t remembered_shady_object_count_types[RUBY_T_MASK];
769 #endif
770 #endif /* RGENGC_PROFILE */
771 #endif /* USE_RGENGC */
772 
773  /* temporary profiling space */
777 
778  /* basic statistics */
779  size_t count;
783  } profile;
785 
787 
788 #if USE_RGENGC
789  struct {
795  size_t old_objects;
797 
798 #if RGENGC_ESTIMATE_OLDMALLOC
801 #endif
802 
803 #if RGENGC_CHECK_MODE >= 2
804  struct st_table *allrefs_table;
805  size_t error_count;
806 #endif
807  } rgengc;
808 
809  struct {
812  } rcompactor;
813 
814 #if GC_ENABLE_INCREMENTAL_MARK
815  struct {
816  size_t pooled_slots;
817  size_t step_slots;
818  } rincgc;
819 #endif
820 #endif /* USE_RGENGC */
821 
824 
825 #if GC_DEBUG_STRESS_TO_CLASS
827 #endif
828 } rb_objspace_t;
829 
830 
831 /* default tiny heap size: 16KB */
832 #define HEAP_PAGE_ALIGN_LOG 14
833 #define CEILDIV(i, mod) (((i) + (mod) - 1)/(mod))
834 enum {
837  REQUIRED_SIZE_BY_MALLOC = (sizeof(size_t) * 5),
839  HEAP_PAGE_OBJ_LIMIT = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header))/sizeof(struct RVALUE)),
842  HEAP_PAGE_BITMAP_PLANES = USE_RGENGC ? 4 : 1 /* RGENGC: mark, unprotected, uncollectible, marking */
843 };
844 
845 struct heap_page {
846  short total_slots;
847  short free_slots;
849  short final_slots;
850  struct {
851  unsigned int before_sweep : 1;
852  unsigned int has_remembered_objects : 1;
854  unsigned int in_tomb : 1;
855  } flags;
856 
861 
862 #if USE_RGENGC
864 #endif
865  /* the following three bitmaps are cleared at the beginning of full GC */
867 #if USE_RGENGC
870 #endif
871 
872  /* If set, the object is not movable */
874 };
875 
876 #define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
877 #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
878 #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
879 
880 #define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK)/sizeof(RVALUE))
881 #define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
882 #define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
883 #define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
884 
885 /* Bitmap Operations */
886 #define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
887 #define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
888 #define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
889 
890 /* getting bitmap */
891 #define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
892 #define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
893 #if USE_RGENGC
894 #define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
895 #define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
896 #define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
897 #endif
898 
899 /* Aliases */
900 #define rb_objspace (*rb_objspace_of(GET_VM()))
901 #define rb_objspace_of(vm) ((vm)->objspace)
902 
903 #define ruby_initial_gc_stress gc_params.gc_stress
904 
906 
907 #define malloc_limit objspace->malloc_params.limit
908 #define malloc_increase objspace->malloc_params.increase
909 #define malloc_allocated_size objspace->malloc_params.allocated_size
910 #define heap_pages_sorted objspace->heap_pages.sorted
911 #define heap_allocated_pages objspace->heap_pages.allocated_pages
912 #define heap_pages_sorted_length objspace->heap_pages.sorted_length
913 #define heap_pages_lomem objspace->heap_pages.range[0]
914 #define heap_pages_himem objspace->heap_pages.range[1]
915 #define heap_allocatable_pages objspace->heap_pages.allocatable_pages
916 #define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
917 #define heap_pages_final_slots objspace->heap_pages.final_slots
918 #define heap_pages_deferred_final objspace->heap_pages.deferred_final
919 #define heap_eden (&objspace->eden_heap)
920 #define heap_tomb (&objspace->tomb_heap)
921 #define dont_gc objspace->flags.dont_gc
922 #define during_gc objspace->flags.during_gc
923 #define finalizing objspace->atomic_flags.finalizing
924 #define finalizer_table objspace->finalizer_table
925 #define global_list objspace->global_list
926 #define ruby_gc_stressful objspace->flags.gc_stressful
927 #define ruby_gc_stress_mode objspace->gc_stress_mode
928 #if GC_DEBUG_STRESS_TO_CLASS
929 #define stress_to_class objspace->stress_to_class
930 #else
931 #define stress_to_class 0
932 #endif
933 
934 static inline enum gc_mode
935 gc_mode_verify(enum gc_mode mode)
936 {
937 #if RGENGC_CHECK_MODE > 0
938  switch (mode) {
939  case gc_mode_none:
940  case gc_mode_marking:
941  case gc_mode_sweeping:
942  break;
943  default:
944  rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
945  }
946 #endif
947  return mode;
948 }
949 
950 #define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
951 #define gc_mode_set(objspace, mode) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(mode))
952 
953 #define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
954 #define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
955 #if USE_RGENGC
956 #define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
957 #else
958 #define is_full_marking(objspace) TRUE
959 #endif
960 #if GC_ENABLE_INCREMENTAL_MARK
961 #define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
962 #else
963 #define is_incremental_marking(objspace) FALSE
964 #endif
965 #if GC_ENABLE_INCREMENTAL_MARK
966 #define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
967 #else
968 #define will_be_incremental_marking(objspace) FALSE
969 #endif
970 #define has_sweeping_pages(heap) ((heap)->sweeping_page != 0)
971 #define is_lazy_sweeping(heap) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(heap))
972 
973 #if SIZEOF_LONG == SIZEOF_VOIDP
974 # define nonspecial_obj_id(obj) (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG)
975 # define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
976 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
977 # define nonspecial_obj_id(obj) LL2NUM((SIGNED_VALUE)(obj) / 2)
978 # define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
979  ((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
980 #else
981 # error not supported
982 #endif
983 
984 #define RANY(o) ((RVALUE*)(o))
985 
986 struct RZombie {
987  struct RBasic basic;
989  void (*dfree)(void *);
990  void *data;
991 };
992 
993 #define RZOMBIE(o) ((struct RZombie *)(o))
994 
995 #define nomem_error GET_VM()->special_exceptions[ruby_error_nomemory]
996 
997 #if RUBY_MARK_FREE_DEBUG
998 int ruby_gc_debug_indent = 0;
999 #endif
1002 
1003 void rb_iseq_mark(const rb_iseq_t *iseq);
1005 void rb_iseq_free(const rb_iseq_t *iseq);
1006 size_t rb_iseq_memsize(const rb_iseq_t *iseq);
1007 void rb_vm_update_references(void *ptr);
1008 
1010 
1011 static VALUE define_final0(VALUE obj, VALUE block);
1012 
1013 NORETURN(static void negative_size_allocation_error(const char *));
1014 
1015 static void init_mark_stack(mark_stack_t *stack);
1016 
1017 static int ready_to_gc(rb_objspace_t *objspace);
1018 
1019 static int garbage_collect(rb_objspace_t *, int reason);
1020 
1021 static int gc_start(rb_objspace_t *objspace, int reason);
1022 static void gc_rest(rb_objspace_t *objspace);
1023 static inline void gc_enter(rb_objspace_t *objspace, const char *event);
1024 static inline void gc_exit(rb_objspace_t *objspace, const char *event);
1025 
1026 static void gc_marks(rb_objspace_t *objspace, int full_mark);
1027 static void gc_marks_start(rb_objspace_t *objspace, int full);
1028 static int gc_marks_finish(rb_objspace_t *objspace);
1029 static void gc_marks_rest(rb_objspace_t *objspace);
1030 static void gc_marks_step(rb_objspace_t *objspace, int slots);
1031 static void gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap);
1032 
1033 static void gc_sweep(rb_objspace_t *objspace);
1034 static void gc_sweep_start(rb_objspace_t *objspace);
1035 static void gc_sweep_finish(rb_objspace_t *objspace);
1036 static int gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap);
1037 static void gc_sweep_rest(rb_objspace_t *objspace);
1038 static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap);
1039 
1040 static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
1041 static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
1042 static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
1043 static void gc_mark_ptr(rb_objspace_t *objspace, VALUE ptr);
1044 NO_SANITIZE("memory", static void gc_mark_maybe(rb_objspace_t *objspace, VALUE ptr));
1045 static void gc_mark_children(rb_objspace_t *objspace, VALUE ptr);
1046 
1047 static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
1048 static int gc_mark_stacked_objects_all(rb_objspace_t *);
1049 static void gc_grey(rb_objspace_t *objspace, VALUE ptr);
1050 
1051 static inline int gc_mark_set(rb_objspace_t *objspace, VALUE obj);
1052 NO_SANITIZE("memory", static inline int is_pointer_to_heap(rb_objspace_t *objspace, void *ptr));
1053 
1054 static void push_mark_stack(mark_stack_t *, VALUE);
1055 static int pop_mark_stack(mark_stack_t *, VALUE *);
1056 static size_t mark_stack_size(mark_stack_t *stack);
1057 static void shrink_stack_chunk_cache(mark_stack_t *stack);
1058 
1059 static size_t obj_memsize_of(VALUE obj, int use_all_types);
1060 static void gc_verify_internal_consistency(rb_objspace_t *objspace);
1061 static int gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj);
1062 static int gc_verify_heap_pages(rb_objspace_t *objspace);
1063 
1064 static void gc_stress_set(rb_objspace_t *objspace, VALUE flag);
1065 
1066 static double getrusage_time(void);
1067 static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, int reason);
1068 static inline void gc_prof_timer_start(rb_objspace_t *);
1069 static inline void gc_prof_timer_stop(rb_objspace_t *);
1070 static inline void gc_prof_mark_timer_start(rb_objspace_t *);
1071 static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
1072 static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
1073 static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
1074 static inline void gc_prof_set_malloc_info(rb_objspace_t *);
1075 static inline void gc_prof_set_heap_info(rb_objspace_t *);
1076 
1077 #define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \
1078  if (gc_object_moved_p(_objspace, (VALUE)_thing)) { \
1079  *((_type *)(&_thing)) = (_type)RMOVED((_thing))->destination; \
1080  } \
1081 } while (0)
1082 
1083 #define UPDATE_IF_MOVED(_objspace, _thing) TYPED_UPDATE_IF_MOVED(_objspace, VALUE, _thing)
1084 
1085 #define gc_prof_record(objspace) (objspace)->profile.current_record
1086 #define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
1087 
1088 #ifdef HAVE_VA_ARGS_MACRO
1089 # define gc_report(level, objspace, ...) \
1090  if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
1091 #else
1092 # define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body
1093 #endif
1094 PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
1095 static const char *obj_info(VALUE obj);
1096 
1097 #define PUSH_MARK_FUNC_DATA(v) do { \
1098  struct mark_func_data_struct *prev_mark_func_data = objspace->mark_func_data; \
1099  objspace->mark_func_data = (v);
1100 
1101 #define POP_MARK_FUNC_DATA() objspace->mark_func_data = prev_mark_func_data;} while (0)
1102 
1103 /*
1104  * 1 - TSC (H/W Time Stamp Counter)
1105  * 2 - getrusage
1106  */
1107 #ifndef TICK_TYPE
1108 #define TICK_TYPE 1
1109 #endif
1110 
1111 #if USE_TICK_T
1112 
1113 #if TICK_TYPE == 1
1114 /* the following code is only for internal tuning. */
1115 
1116 /* Source code to use RDTSC is quoted and modified from
1117  * http://www.mcs.anl.gov/~kazutomo/rdtsc.html
1118  * written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
1119  */
1120 
1121 #if defined(__GNUC__) && defined(__i386__)
1122 typedef unsigned long long tick_t;
1123 #define PRItick "llu"
1124 static inline tick_t
1125 tick(void)
1126 {
1127  unsigned long long int x;
1128  __asm__ __volatile__ ("rdtsc" : "=A" (x));
1129  return x;
1130 }
1131 
1132 #elif defined(__GNUC__) && defined(__x86_64__)
1133 typedef unsigned long long tick_t;
1134 #define PRItick "llu"
1135 
1136 static __inline__ tick_t
1137 tick(void)
1138 {
1139  unsigned long hi, lo;
1140  __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
1141  return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
1142 }
1143 
1144 #elif defined(__powerpc64__) && GCC_VERSION_SINCE(4,8,0)
1145 typedef unsigned long long tick_t;
1146 #define PRItick "llu"
1147 
1148 static __inline__ tick_t
1149 tick(void)
1150 {
1151  unsigned long long val = __builtin_ppc_get_timebase();
1152  return val;
1153 }
1154 
1155 #elif defined(_WIN32) && defined(_MSC_VER)
1156 #include <intrin.h>
1157 typedef unsigned __int64 tick_t;
1158 #define PRItick "llu"
1159 
1160 static inline tick_t
1161 tick(void)
1162 {
1163  return __rdtsc();
1164 }
1165 
1166 #else /* use clock */
1167 typedef clock_t tick_t;
1168 #define PRItick "llu"
1169 
1170 static inline tick_t
1171 tick(void)
1172 {
1173  return clock();
1174 }
1175 #endif /* TSC */
1176 
1177 #elif TICK_TYPE == 2
1178 typedef double tick_t;
1179 #define PRItick "4.9f"
1180 
1181 static inline tick_t
1182 tick(void)
1183 {
1184  return getrusage_time();
1185 }
1186 #else /* TICK_TYPE */
1187 #error "choose tick type"
1188 #endif /* TICK_TYPE */
1189 
1190 #define MEASURE_LINE(expr) do { \
1191  volatile tick_t start_time = tick(); \
1192  volatile tick_t end_time; \
1193  expr; \
1194  end_time = tick(); \
1195  fprintf(stderr, "0\t%"PRItick"\t%s\n", end_time - start_time, #expr); \
1196 } while (0)
1197 
1198 #else /* USE_TICK_T */
1199 #define MEASURE_LINE(expr) expr
1200 #endif /* USE_TICK_T */
1201 
1202 #define FL_CHECK2(name, x, pred) \
1203  ((RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) ? \
1204  (rb_bug(name": SPECIAL_CONST (%p)", (void *)(x)), 0) : (pred))
1205 #define FL_TEST2(x,f) FL_CHECK2("FL_TEST2", x, FL_TEST_RAW((x),(f)) != 0)
1206 #define FL_SET2(x,f) FL_CHECK2("FL_SET2", x, RBASIC(x)->flags |= (f))
1207 #define FL_UNSET2(x,f) FL_CHECK2("FL_UNSET2", x, RBASIC(x)->flags &= ~(f))
1208 
1209 #define RVALUE_MARK_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
1210 #define RVALUE_PIN_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
1211 #define RVALUE_PAGE_MARKED(page, obj) MARKED_IN_BITMAP((page)->mark_bits, (obj))
1212 
1213 #if USE_RGENGC
1214 #define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
1215 #define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
1216 #define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
1217 
1218 #define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
1219 #define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
1220 #define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
1221 
1222 #define RVALUE_OLD_AGE 3
1223 #define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */
1224 
1225 static int rgengc_remembered(rb_objspace_t *objspace, VALUE obj);
1226 static int rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj);
1227 static int rgengc_remember(rb_objspace_t *objspace, VALUE obj);
1228 static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap);
1229 static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
1230 
1231 static inline int
1232 RVALUE_FLAGS_AGE(VALUE flags)
1233 {
1234  return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT);
1235 }
1236 
1237 #endif /* USE_RGENGC */
1238 
1239 static int
1240 check_rvalue_consistency_force(const VALUE obj, int terminate)
1241 {
1242  rb_objspace_t *objspace = &rb_objspace;
1243  int err = 0;
1244 
1245  if (SPECIAL_CONST_P(obj)) {
1246  fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
1247  err++;
1248  }
1249  else if (!is_pointer_to_heap(objspace, (void *)obj)) {
1250  /* check if it is in tomb_pages */
1251  struct heap_page *page = NULL;
1252  list_for_each(&heap_tomb->pages, page, page_node) {
1253  if (&page->start[0] <= (RVALUE *)obj &&
1254  (RVALUE *)obj < &page->start[page->total_slots]) {
1255  fprintf(stderr, "check_rvalue_consistency: %p is in a tomb_heap (%p).\n",
1256  (void *)obj, (void *)page);
1257  err++;
1258  goto skip;
1259  }
1260  }
1261  fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
1262  err++;
1263  skip:
1264  ;
1265  }
1266  else {
1267  const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1268  const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1269  const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0;
1270  const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0, remembered_bit = marking_bit;
1271  const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
1272 
1273  if (GET_HEAP_PAGE(obj)->flags.in_tomb) {
1274  fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", obj_info(obj));
1275  err++;
1276  }
1277  if (BUILTIN_TYPE(obj) == T_NONE) {
1278  fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", obj_info(obj));
1279  err++;
1280  }
1281  if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
1282  fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", obj_info(obj));
1283  err++;
1284  }
1285 
1286  obj_memsize_of((VALUE)obj, FALSE);
1287 
1288  /* check generation
1289  *
1290  * OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
1291  */
1292  if (age > 0 && wb_unprotected_bit) {
1293  fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", obj_info(obj), age);
1294  err++;
1295  }
1296 
1297  if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
1298  fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", obj_info(obj));
1299  err++;
1300  }
1301 
1302  if (!is_full_marking(objspace)) {
1303  if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
1304  fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
1305  obj_info(obj), age);
1306  err++;
1307  }
1308  if (remembered_bit && age != RVALUE_OLD_AGE) {
1309  fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
1310  obj_info(obj), age);
1311  err++;
1312  }
1313  }
1314 
1315  /*
1316  * check coloring
1317  *
1318  * marking:false marking:true
1319  * marked:false white *invalid*
1320  * marked:true black grey
1321  */
1322  if (is_incremental_marking(objspace) && marking_bit) {
1323  if (!is_marking(objspace) && !mark_bit) {
1324  fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", obj_info(obj));
1325  err++;
1326  }
1327  }
1328  }
1329 
1330  if (err > 0 && terminate) {
1331  rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
1332  }
1333 
1334  return err;
1335 }
1336 
1337 #if RGENGC_CHECK_MODE == 0
1338 static inline VALUE
1339 check_rvalue_consistency(const VALUE obj)
1340 {
1341  return obj;
1342 }
1343 #else
1344 static VALUE
1345 check_rvalue_consistency(const VALUE obj)
1346 {
1347  check_rvalue_consistency_force(obj, TRUE);
1348  return obj;
1349 }
1350 #endif
1351 
1352 static inline int
1353 gc_object_moved_p(rb_objspace_t * objspace, VALUE obj)
1354 {
1355  if (RB_SPECIAL_CONST_P(obj)) {
1356  return FALSE;
1357  }
1358  else {
1359  void *poisoned = asan_poisoned_object_p(obj);
1360  asan_unpoison_object(obj, false);
1361 
1362  int ret = BUILTIN_TYPE(obj) == T_MOVED;
1363  /* Re-poison slot if it's not the one we want */
1364  if (poisoned) {
1366  asan_poison_object(obj);
1367  }
1368  return ret;
1369  }
1370 }
1371 
1372 static inline int
1373 RVALUE_MARKED(VALUE obj)
1374 {
1375  check_rvalue_consistency(obj);
1376  return RVALUE_MARK_BITMAP(obj) != 0;
1377 }
1378 
1379 static inline int
1380 RVALUE_PINNED(VALUE obj)
1381 {
1382  check_rvalue_consistency(obj);
1383  return RVALUE_PIN_BITMAP(obj) != 0;
1384 }
1385 
1386 #if USE_RGENGC
1387 static inline int
1388 RVALUE_WB_UNPROTECTED(VALUE obj)
1389 {
1390  check_rvalue_consistency(obj);
1391  return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
1392 }
1393 
1394 static inline int
1395 RVALUE_MARKING(VALUE obj)
1396 {
1397  check_rvalue_consistency(obj);
1398  return RVALUE_MARKING_BITMAP(obj) != 0;
1399 }
1400 
1401 static inline int
1402 RVALUE_REMEMBERED(VALUE obj)
1403 {
1404  check_rvalue_consistency(obj);
1405  return RVALUE_MARKING_BITMAP(obj) != 0;
1406 }
1407 
1408 static inline int
1409 RVALUE_UNCOLLECTIBLE(VALUE obj)
1410 {
1411  check_rvalue_consistency(obj);
1412  return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
1413 }
1414 
1415 static inline int
1416 RVALUE_OLD_P_RAW(VALUE obj)
1417 {
1418  const VALUE promoted = FL_PROMOTED0 | FL_PROMOTED1;
1419  return (RBASIC(obj)->flags & promoted) == promoted;
1420 }
1421 
1422 static inline int
1423 RVALUE_OLD_P(VALUE obj)
1424 {
1425  check_rvalue_consistency(obj);
1426  return RVALUE_OLD_P_RAW(obj);
1427 }
1428 
1429 #if RGENGC_CHECK_MODE || GC_DEBUG
1430 static inline int
1431 RVALUE_AGE(VALUE obj)
1432 {
1433  check_rvalue_consistency(obj);
1434  return RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
1435 }
1436 #endif
1437 
1438 static inline void
1439 RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
1440 {
1442  objspace->rgengc.old_objects++;
1444 
1445 #if RGENGC_PROFILE >= 2
1446  objspace->profile.total_promoted_count++;
1447  objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
1448 #endif
1449 }
1450 
1451 static inline void
1452 RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
1453 {
1454  RB_DEBUG_COUNTER_INC(obj_promote);
1455  RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
1456 }
1457 
1458 static inline VALUE
1459 RVALUE_FLAGS_AGE_SET(VALUE flags, int age)
1460 {
1462  flags |= (age << RVALUE_AGE_SHIFT);
1463  return flags;
1464 }
1465 
1466 /* set age to age+1 */
1467 static inline void
1468 RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
1469 {
1470  VALUE flags = RBASIC(obj)->flags;
1471  int age = RVALUE_FLAGS_AGE(flags);
1472 
1473  if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
1474  rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj));
1475  }
1476 
1477  age++;
1478  RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(flags, age);
1479 
1480  if (age == RVALUE_OLD_AGE) {
1481  RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
1482  }
1483  check_rvalue_consistency(obj);
1484 }
1485 
1486 /* set age to RVALUE_OLD_AGE */
1487 static inline void
1488 RVALUE_AGE_SET_OLD(rb_objspace_t *objspace, VALUE obj)
1489 {
1490  check_rvalue_consistency(obj);
1491  GC_ASSERT(!RVALUE_OLD_P(obj));
1492 
1493  RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE);
1494  RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
1495 
1496  check_rvalue_consistency(obj);
1497 }
1498 
1499 /* set age to RVALUE_OLD_AGE - 1 */
1500 static inline void
1501 RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
1502 {
1503  check_rvalue_consistency(obj);
1504  GC_ASSERT(!RVALUE_OLD_P(obj));
1505 
1506  RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE - 1);
1507 
1508  check_rvalue_consistency(obj);
1509 }
1510 
1511 static inline void
1512 RVALUE_DEMOTE_RAW(rb_objspace_t *objspace, VALUE obj)
1513 {
1514  RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
1516 }
1517 
1518 static inline void
1519 RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
1520 {
1521  check_rvalue_consistency(obj);
1522  GC_ASSERT(RVALUE_OLD_P(obj));
1523 
1524  if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(obj)) {
1526  }
1527 
1528  RVALUE_DEMOTE_RAW(objspace, obj);
1529 
1530  if (RVALUE_MARKED(obj)) {
1531  objspace->rgengc.old_objects--;
1532  }
1533 
1534  check_rvalue_consistency(obj);
1535 }
1536 
1537 static inline void
1538 RVALUE_AGE_RESET_RAW(VALUE obj)
1539 {
1540  RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
1541 }
1542 
1543 static inline void
1544 RVALUE_AGE_RESET(VALUE obj)
1545 {
1546  check_rvalue_consistency(obj);
1547  GC_ASSERT(!RVALUE_OLD_P(obj));
1548 
1549  RVALUE_AGE_RESET_RAW(obj);
1550  check_rvalue_consistency(obj);
1551 }
1552 
1553 static inline int
1554 RVALUE_BLACK_P(VALUE obj)
1555 {
1556  return RVALUE_MARKED(obj) && !RVALUE_MARKING(obj);
1557 }
1558 
1559 #if 0
1560 static inline int
1561 RVALUE_GREY_P(VALUE obj)
1562 {
1563  return RVALUE_MARKED(obj) && RVALUE_MARKING(obj);
1564 }
1565 #endif
1566 
1567 static inline int
1568 RVALUE_WHITE_P(VALUE obj)
1569 {
1570  return RVALUE_MARKED(obj) == FALSE;
1571 }
1572 
1573 #endif /* USE_RGENGC */
1574 
1575 /*
1576  --------------------------- ObjectSpace -----------------------------
1577 */
1578 
1579 static inline void *
1580 calloc1(size_t n)
1581 {
1582  return calloc(1, n);
1583 }
1584 
1585 rb_objspace_t *
1587 {
1588  rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
1589  malloc_limit = gc_params.malloc_limit_min;
1590  list_head_init(&objspace->eden_heap.pages);
1591  list_head_init(&objspace->tomb_heap.pages);
1592 
1593  return objspace;
1594 }
1595 
1596 static void free_stack_chunks(mark_stack_t *);
1597 static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
1598 
1599 void
1601 {
1603  rb_bug("lazy sweeping underway when freeing object space");
1604 
1605  if (objspace->profile.records) {
1606  free(objspace->profile.records);
1607  objspace->profile.records = 0;
1608  }
1609 
1610  if (global_list) {
1611  struct gc_list *list, *next;
1612  for (list = global_list; list; list = next) {
1613  next = list->next;
1614  xfree(list);
1615  }
1616  }
1617  if (heap_pages_sorted) {
1618  size_t i;
1619  for (i = 0; i < heap_allocated_pages; ++i) {
1620  heap_page_free(objspace, heap_pages_sorted[i]);
1621  }
1625  heap_pages_lomem = 0;
1626  heap_pages_himem = 0;
1627 
1628  objspace->eden_heap.total_pages = 0;
1629  objspace->eden_heap.total_slots = 0;
1630  }
1631  st_free_table(objspace->id_to_obj_tbl);
1632  st_free_table(objspace->obj_to_id_tbl);
1633  free_stack_chunks(&objspace->mark_stack);
1634  free(objspace);
1635 }
1636 
1637 static void
1638 heap_pages_expand_sorted_to(rb_objspace_t *objspace, size_t next_length)
1639 {
1640  struct heap_page **sorted;
1641  size_t size = size_mul_or_raise(next_length, sizeof(struct heap_page *), rb_eRuntimeError);
1642 
1643  gc_report(3, objspace, "heap_pages_expand_sorted: next_length: %d, size: %d\n", (int)next_length, (int)size);
1644 
1645  if (heap_pages_sorted_length > 0) {
1646  sorted = (struct heap_page **)realloc(heap_pages_sorted, size);
1647  if (sorted) heap_pages_sorted = sorted;
1648  }
1649  else {
1650  sorted = heap_pages_sorted = (struct heap_page **)malloc(size);
1651  }
1652 
1653  if (sorted == 0) {
1654  rb_memerror();
1655  }
1656 
1657  heap_pages_sorted_length = next_length;
1658 }
1659 
1660 static void
1661 heap_pages_expand_sorted(rb_objspace_t *objspace)
1662 {
1663  /* usually heap_allocatable_pages + heap_eden->total_pages == heap_pages_sorted_length
1664  * because heap_allocatable_pages contains heap_tomb->total_pages (recycle heap_tomb pages).
1665  * however, if there are pages which do not have empty slots, then try to create new pages
1666  * so that the additional allocatable_pages counts (heap_tomb->total_pages) are added.
1667  */
1668  size_t next_length = heap_allocatable_pages;
1669  next_length += heap_eden->total_pages;
1670  next_length += heap_tomb->total_pages;
1671 
1672  if (next_length > heap_pages_sorted_length) {
1673  heap_pages_expand_sorted_to(objspace, next_length);
1674  }
1675 
1678 }
1679 
1680 static void
1681 heap_allocatable_pages_set(rb_objspace_t *objspace, size_t s)
1682 {
1684  heap_pages_expand_sorted(objspace);
1685 }
1686 
1687 
1688 static inline void
1689 heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
1690 {
1691  RVALUE *p = (RVALUE *)obj;
1692  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
1693 
1694  p->as.free.flags = 0;
1695  p->as.free.next = page->freelist;
1696  page->freelist = p;
1697  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
1698 
1699  if (RGENGC_CHECK_MODE &&
1700  /* obj should belong to page */
1701  !(&page->start[0] <= (RVALUE *)obj &&
1702  (RVALUE *)obj < &page->start[page->total_slots] &&
1703  obj % sizeof(RVALUE) == 0)) {
1704  rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)p);
1705  }
1706 
1707  asan_poison_object(obj);
1708 
1709  gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj);
1710 }
1711 
1712 static inline void
1713 heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
1714 {
1715  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
1716  GC_ASSERT(page->free_slots != 0);
1717  if (page->freelist) {
1718  page->free_next = heap->free_pages;
1719  heap->free_pages = page;
1720  }
1721  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
1722 }
1723 
1724 #if GC_ENABLE_INCREMENTAL_MARK
1725 static inline int
1726 heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
1727 {
1728  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
1729  if (page->freelist) {
1730  page->free_next = heap->pooled_pages;
1731  heap->pooled_pages = page;
1732  objspace->rincgc.pooled_slots += page->free_slots;
1733  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
1734 
1735  return TRUE;
1736  }
1737  else {
1738  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
1739 
1740  return FALSE;
1741  }
1742 }
1743 #endif
1744 
1745 static void
1746 heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
1747 {
1748  list_del(&page->page_node);
1749  heap->total_pages--;
1750  heap->total_slots -= page->total_slots;
1751 }
1752 
1753 static void rb_aligned_free(void *ptr);
1754 
1755 static void
1756 heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
1757 {
1759  objspace->profile.total_freed_pages++;
1760  rb_aligned_free(GET_PAGE_BODY(page->start));
1761  free(page);
1762 }
1763 
1764 static void
1765 heap_pages_free_unused_pages(rb_objspace_t *objspace)
1766 {
1767  size_t i, j;
1768 
1769  if (!list_empty(&heap_tomb->pages)) {
1770  for (i = j = 1; j < heap_allocated_pages; i++) {
1771  struct heap_page *page = heap_pages_sorted[i];
1772 
1773  if (page->flags.in_tomb && page->free_slots == page->total_slots) {
1774  heap_unlink_page(objspace, heap_tomb, page);
1775  heap_page_free(objspace, page);
1776  }
1777  else {
1778  if (i != j) {
1779  heap_pages_sorted[j] = page;
1780  }
1781  j++;
1782  }
1783  }
1785  }
1786 }
1787 
1788 static struct heap_page *
1789 heap_page_allocate(rb_objspace_t *objspace)
1790 {
1791  RVALUE *start, *end, *p;
1792  struct heap_page *page;
1793  struct heap_page_body *page_body = 0;
1794  size_t hi, lo, mid;
1795  int limit = HEAP_PAGE_OBJ_LIMIT;
1796 
1797  /* assign heap_page body (contains heap_page_header and RVALUEs) */
1799  if (page_body == 0) {
1800  rb_memerror();
1801  }
1802 
1803  /* assign heap_page entry */
1804  page = calloc1(sizeof(struct heap_page));
1805  if (page == 0) {
1806  rb_aligned_free(page_body);
1807  rb_memerror();
1808  }
1809 
1810  /* adjust obj_limit (object number available in this page) */
1811  start = (RVALUE*)((VALUE)page_body + sizeof(struct heap_page_header));
1812  if ((VALUE)start % sizeof(RVALUE) != 0) {
1813  int delta = (int)(sizeof(RVALUE) - ((VALUE)start % sizeof(RVALUE)));
1814  start = (RVALUE*)((VALUE)start + delta);
1815  limit = (HEAP_PAGE_SIZE - (int)((VALUE)start - (VALUE)page_body))/(int)sizeof(RVALUE);
1816  }
1817  end = start + limit;
1818 
1819  /* setup heap_pages_sorted */
1820  lo = 0;
1822  while (lo < hi) {
1823  struct heap_page *mid_page;
1824 
1825  mid = (lo + hi) / 2;
1826  mid_page = heap_pages_sorted[mid];
1827  if (mid_page->start < start) {
1828  lo = mid + 1;
1829  }
1830  else if (mid_page->start > start) {
1831  hi = mid;
1832  }
1833  else {
1834  rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
1835  }
1836  }
1837 
1838  if (hi < heap_allocated_pages) {
1840  }
1841 
1842  heap_pages_sorted[hi] = page;
1843 
1845 
1847  GC_ASSERT(heap_eden->total_pages + heap_tomb->total_pages == heap_allocated_pages - 1);
1849 
1850  objspace->profile.total_allocated_pages++;
1851 
1853  rb_bug("heap_page_allocate: allocated(%"PRIdSIZE") > sorted(%"PRIdSIZE")",
1855  }
1856 
1858  if (heap_pages_himem < end) heap_pages_himem = end;
1859 
1860  page->start = start;
1861  page->total_slots = limit;
1862  page_body->header.page = page;
1863 
1864  for (p = start; p != end; p++) {
1865  gc_report(3, objspace, "assign_heap_page: %p is added to freelist\n", (void *)p);
1866  heap_page_add_freeobj(objspace, page, (VALUE)p);
1867  }
1868  page->free_slots = limit;
1869 
1870  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
1871  return page;
1872 }
1873 
1874 static struct heap_page *
1875 heap_page_resurrect(rb_objspace_t *objspace)
1876 {
1877  struct heap_page *page = 0, *next;
1878 
1879  list_for_each_safe(&heap_tomb->pages, page, next, page_node) {
1880  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
1881  if (page->freelist != NULL) {
1882  heap_unlink_page(objspace, heap_tomb, page);
1883  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
1884  return page;
1885  }
1886  }
1887 
1888  return NULL;
1889 }
1890 
1891 static struct heap_page *
1892 heap_page_create(rb_objspace_t *objspace)
1893 {
1894  struct heap_page *page;
1895  const char *method = "recycle";
1896 
1898 
1899  page = heap_page_resurrect(objspace);
1900 
1901  if (page == NULL) {
1902  page = heap_page_allocate(objspace);
1903  method = "allocate";
1904  }
1905  if (0) fprintf(stderr, "heap_page_create: %s - %p, heap_allocated_pages: %d, heap_allocated_pages: %d, tomb->total_pages: %d\n",
1906  method, (void *)page, (int)heap_pages_sorted_length, (int)heap_allocated_pages, (int)heap_tomb->total_pages);
1907  return page;
1908 }
1909 
1910 static void
1911 heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
1912 {
1913  page->flags.in_tomb = (heap == heap_tomb);
1914  list_add(&heap->pages, &page->page_node);
1915  heap->total_pages++;
1916  heap->total_slots += page->total_slots;
1917 }
1918 
1919 static void
1920 heap_assign_page(rb_objspace_t *objspace, rb_heap_t *heap)
1921 {
1922  struct heap_page *page = heap_page_create(objspace);
1923  heap_add_page(objspace, heap, page);
1924  heap_add_freepage(heap, page);
1925 }
1926 
1927 static void
1928 heap_add_pages(rb_objspace_t *objspace, rb_heap_t *heap, size_t add)
1929 {
1930  size_t i;
1931 
1932  heap_allocatable_pages_set(objspace, add);
1933 
1934  for (i = 0; i < add; i++) {
1935  heap_assign_page(objspace, heap);
1936  }
1937 
1939 }
1940 
1941 static size_t
1942 heap_extend_pages(rb_objspace_t *objspace, size_t free_slots, size_t total_slots)
1943 {
1944  double goal_ratio = gc_params.heap_free_slots_goal_ratio;
1946  size_t next_used;
1947 
1948  if (goal_ratio == 0.0) {
1949  next_used = (size_t)(used * gc_params.growth_factor);
1950  }
1951  else {
1952  /* Find `f' where free_slots = f * total_slots * goal_ratio
1953  * => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
1954  */
1955  double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
1956 
1957  if (f > gc_params.growth_factor) f = gc_params.growth_factor;
1958  if (f < 1.0) f = 1.1;
1959 
1960  next_used = (size_t)(f * used);
1961 
1962  if (0) {
1963  fprintf(stderr,
1964  "free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
1965  " G(%1.2f), f(%1.2f),"
1966  " used(%8"PRIuSIZE") => next_used(%8"PRIuSIZE")\n",
1968  goal_ratio, f, used, next_used);
1969  }
1970  }
1971 
1972  if (gc_params.growth_max_slots > 0) {
1973  size_t max_used = (size_t)(used + gc_params.growth_max_slots/HEAP_PAGE_OBJ_LIMIT);
1974  if (next_used > max_used) next_used = max_used;
1975  }
1976 
1977  return next_used - used;
1978 }
1979 
1980 static void
1981 heap_set_increment(rb_objspace_t *objspace, size_t additional_pages)
1982 {
1983  size_t used = heap_eden->total_pages;
1984  size_t next_used_limit = used + additional_pages;
1985 
1986  if (next_used_limit == heap_allocated_pages) next_used_limit++;
1987 
1988  heap_allocatable_pages_set(objspace, next_used_limit - used);
1989 
1990  gc_report(1, objspace, "heap_set_increment: heap_allocatable_pages is %d\n", (int)heap_allocatable_pages);
1991 }
1992 
1993 static int
1994 heap_increment(rb_objspace_t *objspace, rb_heap_t *heap)
1995 {
1996  if (heap_allocatable_pages > 0) {
1997  gc_report(1, objspace, "heap_increment: heap_pages_sorted_length: %d, heap_pages_inc: %d, heap->total_pages: %d\n",
1999 
2002 
2003  heap_assign_page(objspace, heap);
2004  return TRUE;
2005  }
2006  return FALSE;
2007 }
2008 
2009 static void
2010 heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
2011 {
2012  GC_ASSERT(heap->free_pages == NULL);
2013 
2014  if (is_lazy_sweeping(heap)) {
2015  gc_sweep_continue(objspace, heap);
2016  }
2017  else if (is_incremental_marking(objspace)) {
2018  gc_marks_continue(objspace, heap);
2019  }
2020 
2021  if (heap->free_pages == NULL &&
2022  (will_be_incremental_marking(objspace) || heap_increment(objspace, heap) == FALSE) &&
2023  gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
2024  rb_memerror();
2025  }
2026 }
2027 
2028 static RVALUE *
2029 heap_get_freeobj_from_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap)
2030 {
2031  struct heap_page *page;
2032  RVALUE *p;
2033 
2034  while (heap->free_pages == NULL) {
2035  heap_prepare(objspace, heap);
2036  }
2037  page = heap->free_pages;
2038  heap->free_pages = page->free_next;
2039  heap->using_page = page;
2040 
2041  GC_ASSERT(page->free_slots != 0);
2042  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
2043  p = page->freelist;
2044  page->freelist = NULL;
2045  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
2046  page->free_slots = 0;
2047  asan_unpoison_object((VALUE)p, true);
2048  return p;
2049 }
2050 
2051 static inline VALUE
2052 heap_get_freeobj_head(rb_objspace_t *objspace, rb_heap_t *heap)
2053 {
2054  RVALUE *p = heap->freelist;
2055  if (LIKELY(p != NULL)) {
2056  heap->freelist = p->as.free.next;
2057  }
2058  asan_unpoison_object((VALUE)p, true);
2059  return (VALUE)p;
2060 }
2061 
2062 static inline VALUE
2063 heap_get_freeobj(rb_objspace_t *objspace, rb_heap_t *heap)
2064 {
2065  RVALUE *p = heap->freelist;
2066 
2067  while (1) {
2068  if (LIKELY(p != NULL)) {
2069  asan_unpoison_object((VALUE)p, true);
2070  heap->freelist = p->as.free.next;
2071  return (VALUE)p;
2072  }
2073  else {
2074  p = heap_get_freeobj_from_next_freepage(objspace, heap);
2075  }
2076  }
2077 }
2078 
2079 void
2081 {
2082  rb_objspace_t *objspace = &rb_objspace;
2083  objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
2084  objspace->flags.has_hook = (objspace->hook_events != 0);
2085 }
2086 
2087 static void
2088 gc_event_hook_body(rb_execution_context_t *ec, rb_objspace_t *objspace, const rb_event_flag_t event, VALUE data)
2089 {
2090  const VALUE *pc = ec->cfp->pc;
2091  if (pc && VM_FRAME_RUBYFRAME_P(ec->cfp)) {
2092  /* increment PC because source line is calculated with PC-1 */
2093  ec->cfp->pc++;
2094  }
2095  EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, data);
2096  ec->cfp->pc = pc;
2097 }
2098 
2099 #define gc_event_hook_available_p(objspace) ((objspace)->flags.has_hook)
2100 #define gc_event_hook_needed_p(objspace, event) ((objspace)->hook_events & (event))
2101 
2102 #define gc_event_hook(objspace, event, data) do { \
2103  if (UNLIKELY(gc_event_hook_needed_p(objspace, event))) { \
2104  gc_event_hook_body(GET_EC(), (objspace), (event), (data)); \
2105  } \
2106 } while (0)
2107 
2108 static inline VALUE
2109 newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, rb_objspace_t *objspace, VALUE obj)
2110 {
2111 #if !__has_feature(memory_sanitizer)
2113  GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
2114 #endif
2115 
2116  /* OBJSETUP */
2117  struct RVALUE buf = {
2118  .as = {
2119  .values = {
2120  .basic = {
2121  .flags = flags,
2122  .klass = klass,
2123  },
2124  .v1 = v1,
2125  .v2 = v2,
2126  .v3 = v3,
2127  },
2128  },
2129  };
2130  MEMCPY(RANY(obj), &buf, RVALUE, 1);
2131 
2132 #if RGENGC_CHECK_MODE
2133  GC_ASSERT(RVALUE_MARKED(obj) == FALSE);
2134  GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
2135  GC_ASSERT(RVALUE_OLD_P(obj) == FALSE);
2136  GC_ASSERT(RVALUE_WB_UNPROTECTED(obj) == FALSE);
2137 
2138  if (flags & FL_PROMOTED1) {
2139  if (RVALUE_AGE(obj) != 2) rb_bug("newobj: %s of age (%d) != 2.", obj_info(obj), RVALUE_AGE(obj));
2140  }
2141  else {
2142  if (RVALUE_AGE(obj) > 0) rb_bug("newobj: %s of age (%d) > 0.", obj_info(obj), RVALUE_AGE(obj));
2143  }
2144  if (rgengc_remembered(objspace, (VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj));
2145 #endif
2146 
2147 #if USE_RGENGC
2148  if (UNLIKELY(wb_protected == FALSE)) {
2150  }
2151 #endif
2152 
2153 #if RGENGC_PROFILE
2154  if (wb_protected) {
2155  objspace->profile.total_generated_normal_object_count++;
2156 #if RGENGC_PROFILE >= 2
2157  objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
2158 #endif
2159  }
2160  else {
2161  objspace->profile.total_generated_shady_object_count++;
2162 #if RGENGC_PROFILE >= 2
2163  objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
2164 #endif
2165  }
2166 #endif
2167 
2168 #if GC_DEBUG
2169  RANY(obj)->file = rb_source_location_cstr(&RANY(obj)->line);
2170  GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
2171 #endif
2172 
2173  objspace->total_allocated_objects++;
2174 
2175  gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
2176 
2177 #if RGENGC_OLD_NEWOBJ_CHECK > 0
2178  {
2179  static int newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
2180 
2181  if (!is_incremental_marking(objspace) &&
2182  flags & FL_WB_PROTECTED && /* do not promote WB unprotected objects */
2183  ! RB_TYPE_P(obj, T_ARRAY)) { /* array.c assumes that allocated objects are new */
2184  if (--newobj_cnt == 0) {
2185  newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
2186 
2187  gc_mark_set(objspace, obj);
2188  RVALUE_AGE_SET_OLD(objspace, obj);
2189 
2191  }
2192  }
2193  }
2194 #endif
2195  check_rvalue_consistency(obj);
2196  return obj;
2197 }
2198 
2199 static inline VALUE
2200 newobj_slowpath(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace, int wb_protected)
2201 {
2202  VALUE obj;
2203 
2205  if (during_gc) {
2206  dont_gc = 1;
2207  during_gc = 0;
2208  rb_bug("object allocation during garbage collection phase");
2209  }
2210 
2211  if (ruby_gc_stressful) {
2212  if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
2213  rb_memerror();
2214  }
2215  }
2216  }
2217 
2218  obj = heap_get_freeobj(objspace, heap_eden);
2219  newobj_init(klass, flags, v1, v2, v3, wb_protected, objspace, obj);
2221  return obj;
2222 }
2223 
2224 NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace));
2225 NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace));
2226 
2227 static VALUE
2228 newobj_slowpath_wb_protected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace)
2229 {
2230  return newobj_slowpath(klass, flags, v1, v2, v3, objspace, TRUE);
2231 }
2232 
2233 static VALUE
2234 newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace)
2235 {
2236  return newobj_slowpath(klass, flags, v1, v2, v3, objspace, FALSE);
2237 }
2238 
2239 static inline VALUE
2240 newobj_of(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected)
2241 {
2242  rb_objspace_t *objspace = &rb_objspace;
2243  VALUE obj;
2244 
2245  RB_DEBUG_COUNTER_INC(obj_newobj);
2246  (void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
2247 
2248 #if GC_DEBUG_STRESS_TO_CLASS
2249  if (UNLIKELY(stress_to_class)) {
2250  long i, cnt = RARRAY_LEN(stress_to_class);
2251  for (i = 0; i < cnt; ++i) {
2253  }
2254  }
2255 #endif
2256  if (!(during_gc ||
2258  gc_event_hook_available_p(objspace)) &&
2259  (obj = heap_get_freeobj_head(objspace, heap_eden)) != Qfalse) {
2260  return newobj_init(klass, flags, v1, v2, v3, wb_protected, objspace, obj);
2261  }
2262  else {
2263  RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
2264 
2265  return wb_protected ?
2266  newobj_slowpath_wb_protected(klass, flags, v1, v2, v3, objspace) :
2267  newobj_slowpath_wb_unprotected(klass, flags, v1, v2, v3, objspace);
2268  }
2269 }
2270 
2271 VALUE
2273 {
2274  GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
2275  return newobj_of(klass, flags, 0, 0, 0, FALSE);
2276 }
2277 
2278 VALUE
2280 {
2281  GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
2282  return newobj_of(klass, flags, 0, 0, 0, TRUE);
2283 }
2284 
2285 /* for compatibility */
2286 
2287 VALUE
2289 {
2290  return newobj_of(0, T_NONE, 0, 0, 0, FALSE);
2291 }
2292 
2293 VALUE
2295 {
2296  return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED);
2297 }
2298 
2299 #define UNEXPECTED_NODE(func) \
2300  rb_bug(#func"(): GC does not handle T_NODE 0x%x(%p) 0x%"PRIxVALUE, \
2301  BUILTIN_TYPE(obj), (void*)(obj), RBASIC(obj)->flags)
2302 
2303 #undef rb_imemo_new
2304 
2305 VALUE
2307 {
2308  VALUE flags = T_IMEMO | (type << FL_USHIFT);
2309  return newobj_of(v0, flags, v1, v2, v3, TRUE);
2310 }
2311 
2312 static VALUE
2313 rb_imemo_tmpbuf_new(VALUE v1, VALUE v2, VALUE v3, VALUE v0)
2314 {
2316  return newobj_of(v0, flags, v1, v2, v3, FALSE);
2317 }
2318 
2319 static VALUE
2320 rb_imemo_tmpbuf_auto_free_maybe_mark_buffer(void *buf, size_t cnt)
2321 {
2322  return rb_imemo_tmpbuf_new((VALUE)buf, 0, (VALUE)cnt, 0);
2323 }
2324 
2327 {
2328  return (rb_imemo_tmpbuf_t *)rb_imemo_tmpbuf_new((VALUE)buf, (VALUE)old_heap, (VALUE)cnt, 0);
2329 }
2330 
2331 static size_t
2332 imemo_memsize(VALUE obj)
2333 {
2334  size_t size = 0;
2335  switch (imemo_type(obj)) {
2336  case imemo_ment:
2337  size += sizeof(RANY(obj)->as.imemo.ment.def);
2338  break;
2339  case imemo_iseq:
2341  break;
2342  case imemo_env:
2343  size += RANY(obj)->as.imemo.env.env_size * sizeof(VALUE);
2344  break;
2345  case imemo_tmpbuf:
2346  size += RANY(obj)->as.imemo.alloc.cnt * sizeof(VALUE);
2347  break;
2348  case imemo_ast:
2349  size += rb_ast_memsize(&RANY(obj)->as.imemo.ast);
2350  break;
2351  case imemo_cref:
2352  case imemo_svar:
2353  case imemo_throw_data:
2354  case imemo_ifunc:
2355  case imemo_memo:
2356  case imemo_parser_strterm:
2357  break;
2358  default:
2359  /* unreachable */
2360  break;
2361  }
2362  return size;
2363 }
2364 
2365 #if IMEMO_DEBUG
2366 VALUE
2367 rb_imemo_new_debug(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0, const char *file, int line)
2368 {
2369  VALUE memo = rb_imemo_new(type, v1, v2, v3, v0);
2370  fprintf(stderr, "memo %p (type: %d) @ %s:%d\n", (void *)memo, imemo_type(memo), file, line);
2371  return memo;
2372 }
2373 #endif
2374 
2375 VALUE
2377 {
2378  if (klass) Check_Type(klass, T_CLASS);
2379  return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, FALSE);
2380 }
2381 
2382 #undef rb_data_object_alloc
2384  RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree),
2385  rb_data_object_wrap, (klass, datap, dmark, dfree))
2386 
2387 
2388 VALUE
2390 {
2391  VALUE obj = rb_data_object_wrap(klass, 0, dmark, dfree);
2392  DATA_PTR(obj) = xcalloc(1, size);
2393  return obj;
2394 }
2395 
2396 VALUE
2398 {
2399  if (klass) Check_Type(klass, T_CLASS);
2400  return newobj_of(klass, T_DATA, (VALUE)type, (VALUE)1, (VALUE)datap, type->flags & RUBY_FL_WB_PROTECTED);
2401 }
2402 
2403 #undef rb_data_typed_object_alloc
2405  const rb_data_type_t *type),
2407 
2408 VALUE
2410 {
2412  DATA_PTR(obj) = xcalloc(1, size);
2413  return obj;
2414 }
2415 
2416 size_t
2418 {
2419  if (RTYPEDDATA_P(obj)) {
2421  const void *ptr = RTYPEDDATA_DATA(obj);
2422  if (ptr && type->function.dsize) {
2423  return type->function.dsize(ptr);
2424  }
2425  }
2426  return 0;
2427 }
2428 
2429 const char *
2431 {
2432  if (RTYPEDDATA_P(obj)) {
2433  return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
2434  }
2435  else {
2436  return 0;
2437  }
2438 }
2439 
2440 PUREFUNC(static inline int is_pointer_to_heap(rb_objspace_t *objspace, void *ptr);)
2441 static inline int
2442 is_pointer_to_heap(rb_objspace_t *objspace, void *ptr)
2443 {
2444  register RVALUE *p = RANY(ptr);
2445  register struct heap_page *page;
2446  register size_t hi, lo, mid;
2447 
2448  RB_DEBUG_COUNTER_INC(gc_isptr_trial);
2449 
2450  if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
2451  RB_DEBUG_COUNTER_INC(gc_isptr_range);
2452 
2453  if ((VALUE)p % sizeof(RVALUE) != 0) return FALSE;
2454  RB_DEBUG_COUNTER_INC(gc_isptr_align);
2455 
2456  /* check if p looks like a pointer using bsearch*/
2457  lo = 0;
2459  while (lo < hi) {
2460  mid = (lo + hi) / 2;
2461  page = heap_pages_sorted[mid];
2462  if (page->start <= p) {
2463  if (p < page->start + page->total_slots) {
2464  RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
2465 
2466  if (page->flags.in_tomb) {
2467  return FALSE;
2468  }
2469  else {
2470  return TRUE;
2471  }
2472  }
2473  lo = mid + 1;
2474  }
2475  else {
2476  hi = mid;
2477  }
2478  }
2479  return FALSE;
2480 }
2481 
2482 static enum rb_id_table_iterator_result
2483 free_const_entry_i(VALUE value, void *data)
2484 {
2485  rb_const_entry_t *ce = (rb_const_entry_t *)value;
2486  xfree(ce);
2487  return ID_TABLE_CONTINUE;
2488 }
2489 
2490 void
2492 {
2493  rb_id_table_foreach_values(tbl, free_const_entry_i, 0);
2494  rb_id_table_free(tbl);
2495 }
2496 
2497 static inline void
2498 make_zombie(rb_objspace_t *objspace, VALUE obj, void (*dfree)(void *), void *data)
2499 {
2500  struct RZombie *zombie = RZOMBIE(obj);
2501  zombie->basic.flags = T_ZOMBIE | (zombie->basic.flags & FL_SEEN_OBJ_ID);
2502  zombie->dfree = dfree;
2503  zombie->data = data;
2504  zombie->next = heap_pages_deferred_final;
2505  heap_pages_deferred_final = (VALUE)zombie;
2506 }
2507 
2508 static inline void
2509 make_io_zombie(rb_objspace_t *objspace, VALUE obj)
2510 {
2511  rb_io_t *fptr = RANY(obj)->as.file.fptr;
2512  make_zombie(objspace, obj, (void (*)(void*))rb_io_fptr_finalize, fptr);
2513 }
2514 
2515 static void
2516 obj_free_object_id(rb_objspace_t *objspace, VALUE obj)
2517 {
2518  VALUE id;
2519 
2522 
2523  if (st_delete(objspace->obj_to_id_tbl, (st_data_t *)&obj, &id)) {
2524  GC_ASSERT(id);
2525  st_delete(objspace->id_to_obj_tbl, (st_data_t *)&id, NULL);
2526  }
2527  else {
2528  rb_bug("Object ID seen, but not in mapping table: %s\n", obj_info(obj));
2529  }
2530 }
2531 
2532 static int
2533 obj_free(rb_objspace_t *objspace, VALUE obj)
2534 {
2535  RB_DEBUG_COUNTER_INC(obj_free);
2536 
2538 
2539  switch (BUILTIN_TYPE(obj)) {
2540  case T_NIL:
2541  case T_FIXNUM:
2542  case T_TRUE:
2543  case T_FALSE:
2544  rb_bug("obj_free() called for broken object");
2545  break;
2546  }
2547 
2548  if (FL_TEST(obj, FL_EXIVAR)) {
2551  }
2552 
2554  obj_free_object_id(objspace, obj);
2555  }
2556 
2557 #if USE_RGENGC
2558  if (RVALUE_WB_UNPROTECTED(obj)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
2559 
2560 #if RGENGC_CHECK_MODE
2561 #define CHECK(x) if (x(obj) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", obj_info(obj))
2562  CHECK(RVALUE_WB_UNPROTECTED);
2563  CHECK(RVALUE_MARKED);
2564  CHECK(RVALUE_MARKING);
2565  CHECK(RVALUE_UNCOLLECTIBLE);
2566 #undef CHECK
2567 #endif
2568 #endif
2569 
2570  switch (BUILTIN_TYPE(obj)) {
2571  case T_OBJECT:
2572  if ((RANY(obj)->as.basic.flags & ROBJECT_EMBED) ||
2573  RANY(obj)->as.object.as.heap.ivptr == NULL) {
2574  RB_DEBUG_COUNTER_INC(obj_obj_embed);
2575  }
2576  else if (ROBJ_TRANSIENT_P(obj)) {
2577  RB_DEBUG_COUNTER_INC(obj_obj_transient);
2578  }
2579  else {
2580  xfree(RANY(obj)->as.object.as.heap.ivptr);
2581  RB_DEBUG_COUNTER_INC(obj_obj_ptr);
2582  }
2583  break;
2584  case T_MODULE:
2585  case T_CLASS:
2588  if (RCLASS_IV_TBL(obj)) {
2590  }
2591  if (RCLASS_CONST_TBL(obj)) {
2593  }
2594  if (RCLASS_IV_INDEX_TBL(obj)) {
2596  }
2597  if (RCLASS_EXT(obj)->subclasses) {
2598  if (BUILTIN_TYPE(obj) == T_MODULE) {
2600  }
2601  else {
2603  }
2604  RCLASS_EXT(obj)->subclasses = NULL;
2605  }
2608  if (RANY(obj)->as.klass.ptr)
2609  xfree(RANY(obj)->as.klass.ptr);
2610  RANY(obj)->as.klass.ptr = NULL;
2611 
2612  (void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE);
2613  (void)RB_DEBUG_COUNTER_INC_IF(obj_class_ptr, BUILTIN_TYPE(obj) == T_CLASS);
2614  break;
2615  case T_STRING:
2616  rb_str_free(obj);
2617  break;
2618  case T_ARRAY:
2619  rb_ary_free(obj);
2620  break;
2621  case T_HASH:
2622 #if USE_DEBUG_COUNTER
2623  switch RHASH_SIZE(obj) {
2624  case 0:
2625  RB_DEBUG_COUNTER_INC(obj_hash_empty);
2626  break;
2627  case 1:
2628  RB_DEBUG_COUNTER_INC(obj_hash_1);
2629  break;
2630  case 2:
2631  RB_DEBUG_COUNTER_INC(obj_hash_2);
2632  break;
2633  case 3:
2634  RB_DEBUG_COUNTER_INC(obj_hash_3);
2635  break;
2636  case 4:
2637  RB_DEBUG_COUNTER_INC(obj_hash_4);
2638  break;
2639  case 5:
2640  case 6:
2641  case 7:
2642  case 8:
2643  RB_DEBUG_COUNTER_INC(obj_hash_5_8);
2644  break;
2645  default:
2646  GC_ASSERT(RHASH_SIZE(obj) > 8);
2647  RB_DEBUG_COUNTER_INC(obj_hash_g8);
2648  }
2649 
2650  if (RHASH_AR_TABLE_P(obj)) {
2651  if (RHASH_AR_TABLE(obj) == NULL) {
2652  RB_DEBUG_COUNTER_INC(obj_hash_null);
2653  }
2654  else {
2655  RB_DEBUG_COUNTER_INC(obj_hash_ar);
2656  }
2657  }
2658  else {
2659  RB_DEBUG_COUNTER_INC(obj_hash_st);
2660  }
2661 #endif
2662  if (/* RHASH_AR_TABLE_P(obj) */ !FL_TEST_RAW(obj, RHASH_ST_TABLE_FLAG)) {
2663  struct ar_table_struct *tab = RHASH(obj)->as.ar;
2664 
2665  if (tab) {
2666  if (RHASH_TRANSIENT_P(obj)) {
2667  RB_DEBUG_COUNTER_INC(obj_hash_transient);
2668  }
2669  else {
2670  ruby_xfree(tab);
2671  }
2672  }
2673  }
2674  else {
2676  st_free_table(RHASH(obj)->as.st);
2677  }
2678  break;
2679  case T_REGEXP:
2680  if (RANY(obj)->as.regexp.ptr) {
2681  onig_free(RANY(obj)->as.regexp.ptr);
2682  RB_DEBUG_COUNTER_INC(obj_regexp_ptr);
2683  }
2684  break;
2685  case T_DATA:
2686  if (DATA_PTR(obj)) {
2687  int free_immediately = FALSE;
2688  void (*dfree)(void *);
2689  void *data = DATA_PTR(obj);
2690 
2691  if (RTYPEDDATA_P(obj)) {
2692  free_immediately = (RANY(obj)->as.typeddata.type->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0;
2693  dfree = RANY(obj)->as.typeddata.type->function.dfree;
2694  if (0 && free_immediately == 0) {
2695  /* to expose non-free-immediate T_DATA */
2696  fprintf(stderr, "not immediate -> %s\n", RANY(obj)->as.typeddata.type->wrap_struct_name);
2697  }
2698  }
2699  else {
2700  dfree = RANY(obj)->as.data.dfree;
2701  }
2702 
2703  if (dfree) {
2704  if (dfree == RUBY_DEFAULT_FREE) {
2705  xfree(data);
2706  RB_DEBUG_COUNTER_INC(obj_data_xfree);
2707  }
2708  else if (free_immediately) {
2709  (*dfree)(data);
2710  RB_DEBUG_COUNTER_INC(obj_data_imm_free);
2711  }
2712  else {
2713  make_zombie(objspace, obj, dfree, data);
2714  RB_DEBUG_COUNTER_INC(obj_data_zombie);
2715  return 1;
2716  }
2717  }
2718  else {
2719  RB_DEBUG_COUNTER_INC(obj_data_empty);
2720  }
2721  }
2722  break;
2723  case T_MATCH:
2724  if (RANY(obj)->as.match.rmatch) {
2725  struct rmatch *rm = RANY(obj)->as.match.rmatch;
2726 #if USE_DEBUG_COUNTER
2727  if (rm->regs.num_regs >= 8) {
2728  RB_DEBUG_COUNTER_INC(obj_match_ge8);
2729  }
2730  else if (rm->regs.num_regs >= 4) {
2731  RB_DEBUG_COUNTER_INC(obj_match_ge4);
2732  }
2733  else if (rm->regs.num_regs >= 1) {
2734  RB_DEBUG_COUNTER_INC(obj_match_under4);
2735  }
2736 #endif
2737  onig_region_free(&rm->regs, 0);
2738  if (rm->char_offset)
2739  xfree(rm->char_offset);
2740  xfree(rm);
2741 
2742  RB_DEBUG_COUNTER_INC(obj_match_ptr);
2743  }
2744  break;
2745  case T_FILE:
2746  if (RANY(obj)->as.file.fptr) {
2747  make_io_zombie(objspace, obj);
2748  RB_DEBUG_COUNTER_INC(obj_file_ptr);
2749  return 1;
2750  }
2751  break;
2752  case T_RATIONAL:
2753  RB_DEBUG_COUNTER_INC(obj_rational);
2754  break;
2755  case T_COMPLEX:
2756  RB_DEBUG_COUNTER_INC(obj_complex);
2757  break;
2758  case T_MOVED:
2759  break;
2760  case T_ICLASS:
2761  /* Basically , T_ICLASS shares table with the module */
2762  if (FL_TEST(obj, RICLASS_IS_ORIGIN)) {
2764  }
2765  if (RCLASS_CALLABLE_M_TBL(obj) != NULL) {
2767  }
2768  if (RCLASS_EXT(obj)->subclasses) {
2770  RCLASS_EXT(obj)->subclasses = NULL;
2771  }
2774  xfree(RANY(obj)->as.klass.ptr);
2775  RANY(obj)->as.klass.ptr = NULL;
2776 
2777  RB_DEBUG_COUNTER_INC(obj_iclass_ptr);
2778  break;
2779 
2780  case T_FLOAT:
2781  RB_DEBUG_COUNTER_INC(obj_float);
2782  break;
2783 
2784  case T_BIGNUM:
2785  if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
2787  RB_DEBUG_COUNTER_INC(obj_bignum_ptr);
2788  }
2789  else {
2790  RB_DEBUG_COUNTER_INC(obj_bignum_embed);
2791  }
2792  break;
2793 
2794  case T_NODE:
2795  UNEXPECTED_NODE(obj_free);
2796  break;
2797 
2798  case T_STRUCT:
2799  if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) ||
2800  RANY(obj)->as.rstruct.as.heap.ptr == NULL) {
2801  RB_DEBUG_COUNTER_INC(obj_struct_embed);
2802  }
2803  else if (RSTRUCT_TRANSIENT_P(obj)) {
2804  RB_DEBUG_COUNTER_INC(obj_struct_transient);
2805  }
2806  else {
2807  xfree((void *)RANY(obj)->as.rstruct.as.heap.ptr);
2808  RB_DEBUG_COUNTER_INC(obj_struct_ptr);
2809  }
2810  break;
2811 
2812  case T_SYMBOL:
2813  {
2815  RB_DEBUG_COUNTER_INC(obj_symbol);
2816  }
2817  break;
2818 
2819  case T_IMEMO:
2820  switch (imemo_type(obj)) {
2821  case imemo_ment:
2822  rb_free_method_entry(&RANY(obj)->as.imemo.ment);
2823  RB_DEBUG_COUNTER_INC(obj_imemo_ment);
2824  break;
2825  case imemo_iseq:
2826  rb_iseq_free(&RANY(obj)->as.imemo.iseq);
2827  RB_DEBUG_COUNTER_INC(obj_imemo_iseq);
2828  break;
2829  case imemo_env:
2830  GC_ASSERT(VM_ENV_ESCAPED_P(RANY(obj)->as.imemo.env.ep));
2831  xfree((VALUE *)RANY(obj)->as.imemo.env.env);
2832  RB_DEBUG_COUNTER_INC(obj_imemo_env);
2833  break;
2834  case imemo_tmpbuf:
2835  xfree(RANY(obj)->as.imemo.alloc.ptr);
2836  RB_DEBUG_COUNTER_INC(obj_imemo_tmpbuf);
2837  break;
2838  case imemo_ast:
2839  rb_ast_free(&RANY(obj)->as.imemo.ast);
2840  RB_DEBUG_COUNTER_INC(obj_imemo_ast);
2841  break;
2842  case imemo_cref:
2843  RB_DEBUG_COUNTER_INC(obj_imemo_cref);
2844  break;
2845  case imemo_svar:
2846  RB_DEBUG_COUNTER_INC(obj_imemo_svar);
2847  break;
2848  case imemo_throw_data:
2849  RB_DEBUG_COUNTER_INC(obj_imemo_throw_data);
2850  break;
2851  case imemo_ifunc:
2852  RB_DEBUG_COUNTER_INC(obj_imemo_ifunc);
2853  break;
2854  case imemo_memo:
2855  RB_DEBUG_COUNTER_INC(obj_imemo_memo);
2856  break;
2857  case imemo_parser_strterm:
2858  RB_DEBUG_COUNTER_INC(obj_imemo_parser_strterm);
2859  break;
2860  default:
2861  /* unreachable */
2862  break;
2863  }
2864  return 0;
2865 
2866  default:
2867  rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
2868  BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
2869  }
2870 
2871  if (FL_TEST(obj, FL_FINALIZE)) {
2872  make_zombie(objspace, obj, 0, 0);
2873  return 1;
2874  }
2875  else {
2876  return 0;
2877  }
2878 }
2879 
2880 
2881 #define OBJ_ID_INCREMENT (sizeof(RVALUE) / 2)
2882 #define OBJ_ID_INITIAL (OBJ_ID_INCREMENT * 2)
2883 
2884 static int
2885 object_id_cmp(st_data_t x, st_data_t y)
2886 {
2887  if (RB_TYPE_P(x, T_BIGNUM)) {
2888  return !rb_big_eql(x, y);
2889  } else {
2890  return x != y;
2891  }
2892 }
2893 
2894 static st_index_t
2895 object_id_hash(st_data_t n)
2896 {
2897  if (RB_TYPE_P(n, T_BIGNUM)) {
2898  return FIX2LONG(rb_big_hash(n));
2899  } else {
2900  return st_numhash(n);
2901  }
2902 }
2903 static const struct st_hash_type object_id_hash_type = {
2904  object_id_cmp,
2905  object_id_hash,
2906 };
2907 
2908 void
2910 {
2911  rb_objspace_t *objspace = &rb_objspace;
2912 
2913  objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL);
2914  objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
2915  objspace->obj_to_id_tbl = st_init_numtable();
2916 
2917 #if RGENGC_ESTIMATE_OLDMALLOC
2918  objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
2919 #endif
2920 
2921  heap_add_pages(objspace, heap_eden, gc_params.heap_init_slots / HEAP_PAGE_OBJ_LIMIT);
2922  init_mark_stack(&objspace->mark_stack);
2923 
2924  objspace->profile.invoke_time = getrusage_time();
2926 }
2927 
2928 void
2930 {
2931  rb_objspace_t *objspace = &rb_objspace;
2932 
2933  gc_stress_set(objspace, ruby_initial_gc_stress);
2934 }
2935 
2936 typedef int each_obj_callback(void *, void *, size_t, void *);
2937 
2938 static void objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data);
2939 static void objspace_reachable_objects_from_root(rb_objspace_t *, void (func)(const char *, VALUE, void *), void *);
2940 
2944  void *data;
2945 };
2946 
2947 static void
2948 objspace_each_objects_without_setup(rb_objspace_t *objspace, each_obj_callback *callback, void *data)
2949 {
2950  size_t i;
2951  struct heap_page *page;
2952  RVALUE *pstart = NULL, *pend;
2953 
2954  i = 0;
2955  while (i < heap_allocated_pages) {
2956  while (0 < i && pstart < heap_pages_sorted[i-1]->start) i--;
2957  while (i < heap_allocated_pages && heap_pages_sorted[i]->start <= pstart) i++;
2958  if (heap_allocated_pages <= i) break;
2959 
2960  page = heap_pages_sorted[i];
2961 
2962  pstart = page->start;
2963  pend = pstart + page->total_slots;
2964 
2965  if ((*callback)(pstart, pend, sizeof(RVALUE), data)) {
2966  break;
2967  }
2968  }
2969 }
2970 
2971 static VALUE
2972 objspace_each_objects_protected(VALUE arg)
2973 {
2974  struct each_obj_args *args = (struct each_obj_args *)arg;
2975  objspace_each_objects_without_setup(args->objspace, args->callback, args->data);
2976  return Qnil;
2977 }
2978 
2979 static VALUE
2980 incremental_enable(VALUE _)
2981 {
2983 
2985  return Qnil;
2986 }
2987 
2988 /*
2989  * rb_objspace_each_objects() is special C API to walk through
2990  * Ruby object space. This C API is too difficult to use it.
2991  * To be frank, you should not use it. Or you need to read the
2992  * source code of this function and understand what this function does.
2993  *
2994  * 'callback' will be called several times (the number of heap page,
2995  * at current implementation) with:
2996  * vstart: a pointer to the first living object of the heap_page.
2997  * vend: a pointer to next to the valid heap_page area.
2998  * stride: a distance to next VALUE.
2999  *
3000  * If callback() returns non-zero, the iteration will be stopped.
3001  *
3002  * This is a sample callback code to iterate liveness objects:
3003  *
3004  * int
3005  * sample_callback(void *vstart, void *vend, int stride, void *data) {
3006  * VALUE v = (VALUE)vstart;
3007  * for (; v != (VALUE)vend; v += stride) {
3008  * if (RBASIC(v)->flags) { // liveness check
3009  * // do something with live object 'v'
3010  * }
3011  * return 0; // continue to iteration
3012  * }
3013  *
3014  * Note: 'vstart' is not a top of heap_page. This point the first
3015  * living object to grasp at least one object to avoid GC issue.
3016  * This means that you can not walk through all Ruby object page
3017  * including freed object page.
3018  *
3019  * Note: On this implementation, 'stride' is same as sizeof(RVALUE).
3020  * However, there are possibilities to pass variable values with
3021  * 'stride' with some reasons. You must use stride instead of
3022  * use some constant value in the iteration.
3023  */
3024 void
3026 {
3027  objspace_each_objects(&rb_objspace, callback, data);
3028 }
3029 
3030 static void
3031 objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data)
3032 {
3033  int prev_dont_incremental = objspace->flags.dont_incremental;
3034 
3035  gc_rest(objspace);
3037 
3038  if (prev_dont_incremental) {
3039  objspace_each_objects_without_setup(objspace, callback, data);
3040  }
3041  else {
3042  struct each_obj_args args = {objspace, callback, data};
3043  rb_ensure(objspace_each_objects_protected, (VALUE)&args, incremental_enable, Qnil);
3044  }
3045 }
3046 
3047 void
3049 {
3050  objspace_each_objects_without_setup(&rb_objspace, callback, data);
3051 }
3052 
3054  size_t num;
3056 };
3057 
3058 static int
3059 internal_object_p(VALUE obj)
3060 {
3061  RVALUE *p = (RVALUE *)obj;
3063  asan_unpoison_object(obj, false);
3064  bool used_p = p->as.basic.flags;
3065 
3066  if (used_p) {
3067  switch (BUILTIN_TYPE(p)) {
3068  case T_NODE:
3069  UNEXPECTED_NODE(internal_object_p);
3070  break;
3071  case T_NONE:
3072  case T_MOVED:
3073  case T_IMEMO:
3074  case T_ICLASS:
3075  case T_ZOMBIE:
3076  break;
3077  case T_CLASS:
3078  if (!p->as.basic.klass) break;
3079  if (FL_TEST(obj, FL_SINGLETON)) {
3081  }
3082  return 0;
3083  default:
3084  if (!p->as.basic.klass) break;
3085  return 0;
3086  }
3087  }
3088  if (ptr || ! used_p) {
3089  asan_poison_object(obj);
3090  }
3091  return 1;
3092 }
3093 
3094 int
3096 {
3097  return internal_object_p(obj);
3098 }
3099 
3100 static int
3101 os_obj_of_i(void *vstart, void *vend, size_t stride, void *data)
3102 {
3103  struct os_each_struct *oes = (struct os_each_struct *)data;
3104  RVALUE *p = (RVALUE *)vstart, *pend = (RVALUE *)vend;
3105 
3106  for (; p != pend; p++) {
3107  volatile VALUE v = (VALUE)p;
3108  if (!internal_object_p(v)) {
3109  if (!oes->of || rb_obj_is_kind_of(v, oes->of)) {
3110  rb_yield(v);
3111  oes->num++;
3112  }
3113  }
3114  }
3115 
3116  return 0;
3117 }
3118 
3119 static VALUE
3120 os_obj_of(VALUE of)
3121 {
3122  struct os_each_struct oes;
3123 
3124  oes.num = 0;
3125  oes.of = of;
3126  rb_objspace_each_objects(os_obj_of_i, &oes);
3127  return SIZET2NUM(oes.num);
3128 }
3129 
3130 /*
3131  * call-seq:
3132  * ObjectSpace.each_object([module]) {|obj| ... } -> integer
3133  * ObjectSpace.each_object([module]) -> an_enumerator
3134  *
3135  * Calls the block once for each living, nonimmediate object in this
3136  * Ruby process. If <i>module</i> is specified, calls the block
3137  * for only those classes or modules that match (or are a subclass of)
3138  * <i>module</i>. Returns the number of objects found. Immediate
3139  * objects (<code>Fixnum</code>s, <code>Symbol</code>s
3140  * <code>true</code>, <code>false</code>, and <code>nil</code>) are
3141  * never returned. In the example below, #each_object returns both
3142  * the numbers we defined and several constants defined in the Math
3143  * module.
3144  *
3145  * If no block is given, an enumerator is returned instead.
3146  *
3147  * a = 102.7
3148  * b = 95 # Won't be returned
3149  * c = 12345678987654321
3150  * count = ObjectSpace.each_object(Numeric) {|x| p x }
3151  * puts "Total count: #{count}"
3152  *
3153  * <em>produces:</em>
3154  *
3155  * 12345678987654321
3156  * 102.7
3157  * 2.71828182845905
3158  * 3.14159265358979
3159  * 2.22044604925031e-16
3160  * 1.7976931348623157e+308
3161  * 2.2250738585072e-308
3162  * Total count: 7
3163  *
3164  */
3165 
3166 static VALUE
3167 os_each_obj(int argc, VALUE *argv, VALUE os)
3168 {
3169  VALUE of;
3170 
3171  of = (!rb_check_arity(argc, 0, 1) ? 0 : argv[0]);
3172  RETURN_ENUMERATOR(os, 1, &of);
3173  return os_obj_of(of);
3174 }
3175 
3176 /*
3177  * call-seq:
3178  * ObjectSpace.undefine_finalizer(obj)
3179  *
3180  * Removes all finalizers for <i>obj</i>.
3181  *
3182  */
3183 
3184 static VALUE
3185 undefine_final(VALUE os, VALUE obj)
3186 {
3187  return rb_undefine_finalizer(obj);
3188 }
3189 
3190 VALUE
3192 {
3193  rb_objspace_t *objspace = &rb_objspace;
3194  st_data_t data = obj;
3196  st_delete(finalizer_table, &data, 0);
3198  return obj;
3199 }
3200 
3201 static void
3202 should_be_callable(VALUE block)
3203 {
3204  if (!rb_obj_respond_to(block, idCall, TRUE)) {
3205  rb_raise(rb_eArgError, "wrong type argument %"PRIsVALUE" (should be callable)",
3206  rb_obj_class(block));
3207  }
3208 }
3209 
3210 static void
3211 should_be_finalizable(VALUE obj)
3212 {
3213  if (!FL_ABLE(obj)) {
3214  rb_raise(rb_eArgError, "cannot define finalizer for %s",
3216  }
3218 }
3219 
3220 /*
3221  * call-seq:
3222  * ObjectSpace.define_finalizer(obj, aProc=proc())
3223  *
3224  * Adds <i>aProc</i> as a finalizer, to be called after <i>obj</i>
3225  * was destroyed. The object ID of the <i>obj</i> will be passed
3226  * as an argument to <i>aProc</i>. If <i>aProc</i> is a lambda or
3227  * method, make sure it can be called with a single argument.
3228  *
3229  */
3230 
3231 static VALUE
3232 define_final(int argc, VALUE *argv, VALUE os)
3233 {
3234  VALUE obj, block;
3235 
3236  rb_scan_args(argc, argv, "11", &obj, &block);
3237  should_be_finalizable(obj);
3238  if (argc == 1) {
3239  block = rb_block_proc();
3240  }
3241  else {
3242  should_be_callable(block);
3243  }
3244 
3245  return define_final0(obj, block);
3246 }
3247 
3248 static VALUE
3249 define_final0(VALUE obj, VALUE block)
3250 {
3251  rb_objspace_t *objspace = &rb_objspace;
3252  VALUE table;
3253  st_data_t data;
3254 
3255  RBASIC(obj)->flags |= FL_FINALIZE;
3256 
3257  block = rb_ary_new3(2, INT2FIX(0), block);
3258  OBJ_FREEZE(block);
3259 
3260  if (st_lookup(finalizer_table, obj, &data)) {
3261  table = (VALUE)data;
3262 
3263  /* avoid duplicate block, table is usually small */
3264  {
3265  long len = RARRAY_LEN(table);
3266  long i;
3267 
3268  for (i = 0; i < len; i++) {
3269  VALUE recv = RARRAY_AREF(table, i);
3270  if (rb_funcall(recv, idEq, 1, block)) {
3271  return recv;
3272  }
3273  }
3274  }
3275 
3276  rb_ary_push(table, block);
3277  }
3278  else {
3279  table = rb_ary_new3(1, block);
3280  RBASIC_CLEAR_CLASS(table);
3282  }
3283  return block;
3284 }
3285 
3286 VALUE
3288 {
3289  should_be_finalizable(obj);
3290  should_be_callable(block);
3291  return define_final0(obj, block);
3292 }
3293 
3294 void
3296 {
3297  rb_objspace_t *objspace = &rb_objspace;
3298  VALUE table;
3299  st_data_t data;
3300 
3301  if (!FL_TEST(obj, FL_FINALIZE)) return;
3302  if (st_lookup(finalizer_table, obj, &data)) {
3303  table = (VALUE)data;
3304  st_insert(finalizer_table, dest, table);
3305  }
3306  FL_SET(dest, FL_FINALIZE);
3307 }
3308 
3309 static VALUE
3310 run_single_final(VALUE final, VALUE objid)
3311 {
3312  const VALUE cmd = RARRAY_AREF(final, 1);
3313  return rb_check_funcall(cmd, idCall, 1, &objid);
3314 }
3315 
3316 static void
3317 run_finalizer(rb_objspace_t *objspace, VALUE obj, VALUE table)
3318 {
3319  long i;
3320  enum ruby_tag_type state;
3321  volatile struct {
3322  VALUE errinfo;
3323  VALUE objid;
3325  long finished;
3326  } saved;
3327  rb_execution_context_t * volatile ec = GET_EC();
3328 #define RESTORE_FINALIZER() (\
3329  ec->cfp = saved.cfp, \
3330  rb_set_errinfo(saved.errinfo))
3331 
3332  saved.errinfo = rb_errinfo();
3333  saved.objid = rb_obj_id(obj);
3334  saved.cfp = ec->cfp;
3335  saved.finished = 0;
3336 
3337  EC_PUSH_TAG(ec);
3338  state = EC_EXEC_TAG();
3339  if (state != TAG_NONE) {
3340  ++saved.finished; /* skip failed finalizer */
3341  }
3342  for (i = saved.finished;
3343  RESTORE_FINALIZER(), i<RARRAY_LEN(table);
3344  saved.finished = ++i) {
3345  run_single_final(RARRAY_AREF(table, i), saved.objid);
3346  }
3347  EC_POP_TAG();
3348 #undef RESTORE_FINALIZER
3349 }
3350 
3351 static void
3352 run_final(rb_objspace_t *objspace, VALUE zombie)
3353 {
3354  st_data_t key, table;
3355 
3356  if (RZOMBIE(zombie)->dfree) {
3357  RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
3358  }
3359 
3360  key = (st_data_t)zombie;
3361  if (st_delete(finalizer_table, &key, &table)) {
3362  run_finalizer(objspace, zombie, (VALUE)table);
3363  }
3364 }
3365 
3366 static void
3367 finalize_list(rb_objspace_t *objspace, VALUE zombie)
3368 {
3369  while (zombie) {
3370  VALUE next_zombie;
3371  struct heap_page *page;
3372  asan_unpoison_object(zombie, false);
3373  next_zombie = RZOMBIE(zombie)->next;
3374  page = GET_HEAP_PAGE(zombie);
3375 
3376  run_final(objspace, zombie);
3377 
3378  GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
3379  if (FL_TEST(zombie, FL_SEEN_OBJ_ID)) {
3380  obj_free_object_id(objspace, zombie);
3381  }
3382 
3383  RZOMBIE(zombie)->basic.flags = 0;
3385  page->final_slots--;
3386  page->free_slots++;
3387  heap_page_add_freeobj(objspace, GET_HEAP_PAGE(zombie), zombie);
3388 
3389  objspace->profile.total_freed_objects++;
3390 
3391  zombie = next_zombie;
3392  }
3393 }
3394 
3395 static void
3396 finalize_deferred(rb_objspace_t *objspace)
3397 {
3398  VALUE zombie;
3399 
3400  while ((zombie = ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
3401  finalize_list(objspace, zombie);
3402  }
3403 }
3404 
3405 static void
3406 gc_finalize_deferred(void *dmy)
3407 {
3408  rb_objspace_t *objspace = dmy;
3409  if (ATOMIC_EXCHANGE(finalizing, 1)) return;
3410  finalize_deferred(objspace);
3411  ATOMIC_SET(finalizing, 0);
3412 }
3413 
3414 static void
3415 gc_finalize_deferred_register(rb_objspace_t *objspace)
3416 {
3417  if (rb_postponed_job_register_one(0, gc_finalize_deferred, objspace) == 0) {
3418  rb_bug("gc_finalize_deferred_register: can't register finalizer.");
3419  }
3420 }
3421 
3426 };
3427 
3428 static int
3429 force_chain_object(st_data_t key, st_data_t val, st_data_t arg)
3430 {
3431  struct force_finalize_list **prev = (struct force_finalize_list **)arg;
3432  struct force_finalize_list *curr = ALLOC(struct force_finalize_list);
3433  curr->obj = key;
3434  curr->table = val;
3435  curr->next = *prev;
3436  *prev = curr;
3437  return ST_CONTINUE;
3438 }
3439 
3440 void
3442 {
3443  RVALUE *p, *pend;
3444  size_t i;
3445 
3446 #if RGENGC_CHECK_MODE >= 2
3447  gc_verify_internal_consistency(objspace);
3448 #endif
3449  gc_rest(objspace);
3450 
3451  if (ATOMIC_EXCHANGE(finalizing, 1)) return;
3452 
3453  /* run finalizers */
3454  finalize_deferred(objspace);
3456 
3457  gc_rest(objspace);
3458  /* prohibit incremental GC */
3459  objspace->flags.dont_incremental = 1;
3460 
3461  /* force to run finalizer */
3462  while (finalizer_table->num_entries) {
3463  struct force_finalize_list *list = 0;
3464  st_foreach(finalizer_table, force_chain_object, (st_data_t)&list);
3465  while (list) {
3466  struct force_finalize_list *curr = list;
3467  st_data_t obj = (st_data_t)curr->obj;
3468  run_finalizer(objspace, curr->obj, curr->table);
3470  list = curr->next;
3471  xfree(curr);
3472  }
3473  }
3474 
3475  /* prohibit GC because force T_DATA finalizers can break an object graph consistency */
3476  dont_gc = 1;
3477 
3478  /* running data/file finalizers are part of garbage collection */
3479  gc_enter(objspace, "rb_objspace_call_finalizer");
3480 
3481  /* run data/file object's finalizers */
3482  for (i = 0; i < heap_allocated_pages; i++) {
3483  p = heap_pages_sorted[i]->start; pend = p + heap_pages_sorted[i]->total_slots;
3484  while (p < pend) {
3485  void *poisoned = asan_poisoned_object_p((VALUE)p);
3486  asan_unpoison_object((VALUE)p, false);
3487  switch (BUILTIN_TYPE(p)) {
3488  case T_DATA:
3489  if (!DATA_PTR(p) || !RANY(p)->as.data.dfree) break;
3490  if (rb_obj_is_thread((VALUE)p)) break;
3491  if (rb_obj_is_mutex((VALUE)p)) break;
3492  if (rb_obj_is_fiber((VALUE)p)) break;
3493  p->as.free.flags = 0;
3494  if (RTYPEDDATA_P(p)) {
3495  RDATA(p)->dfree = RANY(p)->as.typeddata.type->function.dfree;
3496  }
3497  if (RANY(p)->as.data.dfree == RUBY_DEFAULT_FREE) {
3498  xfree(DATA_PTR(p));
3499  }
3500  else if (RANY(p)->as.data.dfree) {
3501  make_zombie(objspace, (VALUE)p, RANY(p)->as.data.dfree, RANY(p)->as.data.data);
3502  }
3503  break;
3504  case T_FILE:
3505  if (RANY(p)->as.file.fptr) {
3506  make_io_zombie(objspace, (VALUE)p);
3507  }
3508  break;
3509  }
3510  if (poisoned) {
3511  GC_ASSERT(BUILTIN_TYPE(p) == T_NONE);
3512  asan_poison_object((VALUE)p);
3513  }
3514  p++;
3515  }
3516  }
3517 
3518  gc_exit(objspace, "rb_objspace_call_finalizer");
3519 
3521  finalize_list(objspace, heap_pages_deferred_final);
3522  }
3523 
3525  finalizer_table = 0;
3526  ATOMIC_SET(finalizing, 0);
3527 }
3528 
3529 PUREFUNC(static inline int is_id_value(rb_objspace_t *objspace, VALUE ptr));
3530 static inline int
3531 is_id_value(rb_objspace_t *objspace, VALUE ptr)
3532 {
3533  if (!is_pointer_to_heap(objspace, (void *)ptr)) return FALSE;
3534  if (BUILTIN_TYPE(ptr) > T_FIXNUM) return FALSE;
3535  if (BUILTIN_TYPE(ptr) == T_ICLASS) return FALSE;
3536  return TRUE;
3537 }
3538 
3539 static inline int
3540 heap_is_swept_object(rb_objspace_t *objspace, rb_heap_t *heap, VALUE ptr)
3541 {
3542  struct heap_page *page = GET_HEAP_PAGE(ptr);
3543  return page->flags.before_sweep ? FALSE : TRUE;
3544 }
3545 
3546 static inline int
3547 is_swept_object(rb_objspace_t *objspace, VALUE ptr)
3548 {
3549  if (heap_is_swept_object(objspace, heap_eden, ptr)) {
3550  return TRUE;
3551  }
3552  else {
3553  return FALSE;
3554  }
3555 }
3556 
3557 /* garbage objects will be collected soon. */
3558 static inline int
3559 is_garbage_object(rb_objspace_t *objspace, VALUE ptr)
3560 {
3561  if (!is_lazy_sweeping(heap_eden) ||
3562  is_swept_object(objspace, ptr) ||
3564 
3565  return FALSE;
3566  }
3567  else {
3568  return TRUE;
3569  }
3570 }
3571 
3572 static inline int
3573 is_live_object(rb_objspace_t *objspace, VALUE ptr)
3574 {
3575  switch (BUILTIN_TYPE(ptr)) {
3576  case T_NONE:
3577  case T_ZOMBIE:
3578  return FALSE;
3579  }
3580 
3581  if (!is_garbage_object(objspace, ptr)) {
3582  return TRUE;
3583  }
3584  else {
3585  return FALSE;
3586  }
3587 }
3588 
3589 static inline int
3590 is_markable_object(rb_objspace_t *objspace, VALUE obj)
3591 {
3592  if (rb_special_const_p(obj)) return FALSE; /* special const is not markable */
3593  check_rvalue_consistency(obj);
3594  return TRUE;
3595 }
3596 
3597 int
3599 {
3600  rb_objspace_t *objspace = &rb_objspace;
3601  return is_markable_object(objspace, obj) && is_live_object(objspace, obj);
3602 }
3603 
3604 int
3606 {
3607  rb_objspace_t *objspace = &rb_objspace;
3608  return is_garbage_object(objspace, obj);
3609 }
3610 
3611 static VALUE
3612 id2ref_obj_tbl(rb_objspace_t *objspace, VALUE objid)
3613 {
3614  VALUE orig;
3615  if (st_lookup(objspace->id_to_obj_tbl, objid, &orig)) {
3616  return orig;
3617  }
3618  else {
3619  return Qundef;
3620  }
3621 }
3622 
3623 /*
3624  * call-seq:
3625  * ObjectSpace._id2ref(object_id) -> an_object
3626  *
3627  * Converts an object id to a reference to the object. May not be
3628  * called on an object id passed as a parameter to a finalizer.
3629  *
3630  * s = "I am a string" #=> "I am a string"
3631  * r = ObjectSpace._id2ref(s.object_id) #=> "I am a string"
3632  * r == s #=> true
3633  *
3634  */
3635 
3636 static VALUE
3637 id2ref(VALUE objid)
3638 {
3639 #if SIZEOF_LONG == SIZEOF_VOIDP
3640 #define NUM2PTR(x) NUM2ULONG(x)
3641 #elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
3642 #define NUM2PTR(x) NUM2ULL(x)
3643 #endif
3644  rb_objspace_t *objspace = &rb_objspace;
3645  VALUE ptr;
3646  VALUE orig;
3647  void *p0;
3648 
3649  if (FIXNUM_P(objid) || rb_big_size(objid) <= SIZEOF_VOIDP) {
3650  ptr = NUM2PTR(objid);
3651  if (ptr == Qtrue) return Qtrue;
3652  if (ptr == Qfalse) return Qfalse;
3653  if (ptr == Qnil) return Qnil;
3654  if (FIXNUM_P(ptr)) return (VALUE)ptr;
3655  if (FLONUM_P(ptr)) return (VALUE)ptr;
3656 
3657  ptr = obj_id_to_ref(objid);
3658  if ((ptr % sizeof(RVALUE)) == (4 << 2)) {
3659  ID symid = ptr / sizeof(RVALUE);
3660  p0 = (void *)ptr;
3661  if (rb_id2str(symid) == 0)
3662  rb_raise(rb_eRangeError, "%p is not symbol id value", p0);
3663  return ID2SYM(symid);
3664  }
3665  }
3666 
3667  if ((orig = id2ref_obj_tbl(objspace, objid)) != Qundef &&
3668  is_live_object(objspace, orig)) {
3669  return orig;
3670  }
3671 
3672  if (rb_int_ge(objid, objspace->next_object_id)) {
3673  rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_int2str(objid, 10));
3674  } else {
3675  rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_int2str(objid, 10));
3676  }
3677 }
3678 
3679 static VALUE
3680 os_id2ref(VALUE os, VALUE objid)
3681 {
3682  return id2ref(objid);
3683 }
3684 
3685 static VALUE
3686 rb_find_object_id(VALUE obj, VALUE (*get_heap_object_id)(VALUE))
3687 {
3688  if (STATIC_SYM_P(obj)) {
3689  return (SYM2ID(obj) * sizeof(RVALUE) + (4 << 2)) | FIXNUM_FLAG;
3690  }
3691  else if (FLONUM_P(obj)) {
3692 #if SIZEOF_LONG == SIZEOF_VOIDP
3693  return LONG2NUM((SIGNED_VALUE)obj);
3694 #else
3695  return LL2NUM((SIGNED_VALUE)obj);
3696 #endif
3697  }
3698  else if (SPECIAL_CONST_P(obj)) {
3699  return LONG2NUM((SIGNED_VALUE)obj);
3700  }
3701 
3702  return get_heap_object_id(obj);
3703 }
3704 
3705 static VALUE
3706 cached_object_id(VALUE obj)
3707 {
3708  VALUE id;
3709  rb_objspace_t *objspace = &rb_objspace;
3710 
3711  if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &id)) {
3713  return id;
3714  }
3715  else {
3717 
3718  id = objspace->next_object_id;
3720 
3721  st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
3722  st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
3724 
3725  return id;
3726  }
3727 }
3728 
3729 static VALUE
3730 nonspecial_obj_id_(VALUE obj)
3731 {
3732  return nonspecial_obj_id(obj);
3733 }
3734 
3735 
3736 VALUE
3738 {
3739  return rb_find_object_id(obj, nonspecial_obj_id_);
3740 }
3741 
3742 /*
3743  * Document-method: __id__
3744  * Document-method: object_id
3745  *
3746  * call-seq:
3747  * obj.__id__ -> integer
3748  * obj.object_id -> integer
3749  *
3750  * Returns an integer identifier for +obj+.
3751  *
3752  * The same number will be returned on all calls to +object_id+ for a given
3753  * object, and no two active objects will share an id.
3754  *
3755  * Note: that some objects of builtin classes are reused for optimization.
3756  * This is the case for immediate values and frozen string literals.
3757  *
3758  * BasicObject implements +__id__+, Kernel implements +object_id+.
3759  *
3760  * Immediate values are not passed by reference but are passed by value:
3761  * +nil+, +true+, +false+, Fixnums, Symbols, and some Floats.
3762  *
3763  * Object.new.object_id == Object.new.object_id # => false
3764  * (21 * 2).object_id == (21 * 2).object_id # => true
3765  * "hello".object_id == "hello".object_id # => false
3766  * "hi".freeze.object_id == "hi".freeze.object_id # => true
3767  */
3768 
3769 VALUE
3771 {
3772  /*
3773  * 32-bit VALUE space
3774  * MSB ------------------------ LSB
3775  * false 00000000000000000000000000000000
3776  * true 00000000000000000000000000000010
3777  * nil 00000000000000000000000000000100
3778  * undef 00000000000000000000000000000110
3779  * symbol ssssssssssssssssssssssss00001110
3780  * object oooooooooooooooooooooooooooooo00 = 0 (mod sizeof(RVALUE))
3781  * fixnum fffffffffffffffffffffffffffffff1
3782  *
3783  * object_id space
3784  * LSB
3785  * false 00000000000000000000000000000000
3786  * true 00000000000000000000000000000010
3787  * nil 00000000000000000000000000000100
3788  * undef 00000000000000000000000000000110
3789  * symbol 000SSSSSSSSSSSSSSSSSSSSSSSSSSS0 S...S % A = 4 (S...S = s...s * A + 4)
3790  * object oooooooooooooooooooooooooooooo0 o...o % A = 0
3791  * fixnum fffffffffffffffffffffffffffffff1 bignum if required
3792  *
3793  * where A = sizeof(RVALUE)/4
3794  *
3795  * sizeof(RVALUE) is
3796  * 20 if 32-bit, double is 4-byte aligned
3797  * 24 if 32-bit, double is 8-byte aligned
3798  * 40 if 64-bit
3799  */
3800 
3801  return rb_find_object_id(obj, cached_object_id);
3802 }
3803 
3804 #include "regint.h"
3805 
3806 static size_t
3807 obj_memsize_of(VALUE obj, int use_all_types)
3808 {
3809  size_t size = 0;
3810 
3811  if (SPECIAL_CONST_P(obj)) {
3812  return 0;
3813  }
3814 
3815  if (FL_TEST(obj, FL_EXIVAR)) {
3817  }
3818 
3819  switch (BUILTIN_TYPE(obj)) {
3820  case T_OBJECT:
3821  if (!(RBASIC(obj)->flags & ROBJECT_EMBED) &&
3822  ROBJECT(obj)->as.heap.ivptr) {
3823  size += ROBJECT(obj)->as.heap.numiv * sizeof(VALUE);
3824  }
3825  break;
3826  case T_MODULE:
3827  case T_CLASS:
3828  if (RCLASS_EXT(obj)) {
3829  if (RCLASS_M_TBL(obj)) {
3831  }
3832  if (RCLASS_IV_TBL(obj)) {
3834  }
3835  if (RCLASS_IV_INDEX_TBL(obj)) {
3837  }
3838  if (RCLASS(obj)->ptr->iv_tbl) {
3839  size += st_memsize(RCLASS(obj)->ptr->iv_tbl);
3840  }
3841  if (RCLASS(obj)->ptr->const_tbl) {
3842  size += rb_id_table_memsize(RCLASS(obj)->ptr->const_tbl);
3843  }
3844  size += sizeof(rb_classext_t);
3845  }
3846  break;
3847  case T_ICLASS:
3848  if (FL_TEST(obj, RICLASS_IS_ORIGIN)) {
3849  if (RCLASS_M_TBL(obj)) {
3851  }
3852  }
3853  break;
3854  case T_STRING:
3855  size += rb_str_memsize(obj);
3856  break;
3857  case T_ARRAY:
3858  size += rb_ary_memsize(obj);
3859  break;
3860  case T_HASH:
3861  if (RHASH_AR_TABLE_P(obj)) {
3862  if (RHASH_AR_TABLE(obj) != NULL) {
3863  size_t rb_hash_ar_table_size();
3865  }
3866  }
3867  else {
3870  }
3871  break;
3872  case T_REGEXP:
3873  if (RREGEXP_PTR(obj)) {
3875  }
3876  break;
3877  case T_DATA:
3878  if (use_all_types) size += rb_objspace_data_type_memsize(obj);
3879  break;
3880  case T_MATCH:
3881  if (RMATCH(obj)->rmatch) {
3882  struct rmatch *rm = RMATCH(obj)->rmatch;
3883  size += onig_region_memsize(&rm->regs);
3884  size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
3885  size += sizeof(struct rmatch);
3886  }
3887  break;
3888  case T_FILE:
3889  if (RFILE(obj)->fptr) {
3890  size += rb_io_memsize(RFILE(obj)->fptr);
3891  }
3892  break;
3893  case T_RATIONAL:
3894  case T_COMPLEX:
3895  break;
3896  case T_IMEMO:
3897  size += imemo_memsize(obj);
3898  break;
3899 
3900  case T_FLOAT:
3901  case T_SYMBOL:
3902  break;
3903 
3904  case T_BIGNUM:
3905  if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
3906  size += BIGNUM_LEN(obj) * sizeof(BDIGIT);
3907  }
3908  break;
3909 
3910  case T_NODE:
3911  UNEXPECTED_NODE(obj_memsize_of);
3912  break;
3913 
3914  case T_STRUCT:
3915  if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
3916  RSTRUCT(obj)->as.heap.ptr) {
3917  size += sizeof(VALUE) * RSTRUCT_LEN(obj);
3918  }
3919  break;
3920 
3921  case T_ZOMBIE:
3922  case T_MOVED:
3923  break;
3924 
3925  default:
3926  rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
3927  BUILTIN_TYPE(obj), (void*)obj);
3928  }
3929 
3930  return size + sizeof(RVALUE);
3931 }
3932 
3933 size_t
3935 {
3936  return obj_memsize_of(obj, TRUE);
3937 }
3938 
3939 static int
3940 set_zero(st_data_t key, st_data_t val, st_data_t arg)
3941 {
3942  VALUE k = (VALUE)key;
3943  VALUE hash = (VALUE)arg;
3944  rb_hash_aset(hash, k, INT2FIX(0));
3945  return ST_CONTINUE;
3946 }
3947 
3948 static VALUE
3949 type_sym(size_t type)
3950 {
3951  switch (type) {
3952 #define COUNT_TYPE(t) case (t): return ID2SYM(rb_intern(#t)); break;
3953  COUNT_TYPE(T_NONE);
3961  COUNT_TYPE(T_HASH);
3964  COUNT_TYPE(T_FILE);
3965  COUNT_TYPE(T_DATA);
3969  COUNT_TYPE(T_NIL);
3970  COUNT_TYPE(T_TRUE);
3976  COUNT_TYPE(T_NODE);
3980 #undef COUNT_TYPE
3981  default: return INT2NUM(type); break;
3982  }
3983 }
3984 
3985 /*
3986  * call-seq:
3987  * ObjectSpace.count_objects([result_hash]) -> hash
3988  *
3989  * Counts all objects grouped by type.
3990  *
3991  * It returns a hash, such as:
3992  * {
3993  * :TOTAL=>10000,
3994  * :FREE=>3011,
3995  * :T_OBJECT=>6,
3996  * :T_CLASS=>404,
3997  * # ...
3998  * }
3999  *
4000  * The contents of the returned hash are implementation specific.
4001  * It may be changed in future.
4002  *
4003  * The keys starting with +:T_+ means live objects.
4004  * For example, +:T_ARRAY+ is the number of arrays.
4005  * +:FREE+ means object slots which is not used now.
4006  * +:TOTAL+ means sum of above.
4007  *
4008  * If the optional argument +result_hash+ is given,
4009  * it is overwritten and returned. This is intended to avoid probe effect.
4010  *
4011  * h = {}
4012  * ObjectSpace.count_objects(h)
4013  * puts h
4014  * # => { :TOTAL=>10000, :T_CLASS=>158280, :T_MODULE=>20672, :T_STRING=>527249 }
4015  *
4016  * This method is only expected to work on C Ruby.
4017  *
4018  */
4019 
4020 static VALUE
4021 count_objects(int argc, VALUE *argv, VALUE os)
4022 {
4023  rb_objspace_t *objspace = &rb_objspace;
4024  size_t counts[T_MASK+1];
4025  size_t freed = 0;
4026  size_t total = 0;
4027  size_t i;
4028  VALUE hash = Qnil;
4029 
4030  if (rb_check_arity(argc, 0, 1) == 1) {
4031  hash = argv[0];
4032  if (!RB_TYPE_P(hash, T_HASH))
4033  rb_raise(rb_eTypeError, "non-hash given");
4034  }
4035 
4036  for (i = 0; i <= T_MASK; i++) {
4037  counts[i] = 0;
4038  }
4039 
4040  for (i = 0; i < heap_allocated_pages; i++) {
4041  struct heap_page *page = heap_pages_sorted[i];
4042  RVALUE *p, *pend;
4043 
4044  p = page->start; pend = p + page->total_slots;
4045  for (;p < pend; p++) {
4046  void *poisoned = asan_poisoned_object_p((VALUE)p);
4047  asan_unpoison_object((VALUE)p, false);
4048  if (p->as.basic.flags) {
4049  counts[BUILTIN_TYPE(p)]++;
4050  }
4051  else {
4052  freed++;
4053  }
4054  if (poisoned) {
4056  asan_poison_object((VALUE)p);
4057  }
4058  }
4059  total += page->total_slots;
4060  }
4061 
4062  if (hash == Qnil) {
4063  hash = rb_hash_new();
4064  }
4065  else if (!RHASH_EMPTY_P(hash)) {
4066  rb_hash_stlike_foreach(hash, set_zero, hash);
4067  }
4068  rb_hash_aset(hash, ID2SYM(rb_intern("TOTAL")), SIZET2NUM(total));
4069  rb_hash_aset(hash, ID2SYM(rb_intern("FREE")), SIZET2NUM(freed));
4070 
4071  for (i = 0; i <= T_MASK; i++) {
4072  VALUE type = type_sym(i);
4073  if (counts[i])
4074  rb_hash_aset(hash, type, SIZET2NUM(counts[i]));
4075  }
4076 
4077  return hash;
4078 }
4079 
4080 /*
4081  ------------------------ Garbage Collection ------------------------
4082 */
4083 
4084 /* Sweeping */
4085 
4086 static size_t
4087 objspace_available_slots(rb_objspace_t *objspace)
4088 {
4089  return heap_eden->total_slots + heap_tomb->total_slots;
4090 }
4091 
4092 static size_t
4093 objspace_live_slots(rb_objspace_t *objspace)
4094 {
4096 }
4097 
4098 static size_t
4099 objspace_free_slots(rb_objspace_t *objspace)
4100 {
4101  return objspace_available_slots(objspace) - objspace_live_slots(objspace) - heap_pages_final_slots;
4102 }
4103 
4104 static void
4105 gc_setup_mark_bits(struct heap_page *page)
4106 {
4107 #if USE_RGENGC
4108  /* copy oldgen bitmap to mark bitmap */
4110 #else
4111  /* clear mark bitmap */
4112  memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
4113 #endif
4114 }
4115 
4116 static inline int
4117 gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page)
4118 {
4119  int i;
4120  int empty_slots = 0, freed_slots = 0, final_slots = 0;
4121  RVALUE *p, *pend,*offset;
4122  bits_t *bits, bitset;
4123 
4124  gc_report(2, objspace, "page_sweep: start.\n");
4125 
4126  sweep_page->flags.before_sweep = FALSE;
4127 
4128  p = sweep_page->start; pend = p + sweep_page->total_slots;
4129  offset = p - NUM_IN_PAGE(p);
4130  bits = sweep_page->mark_bits;
4131 
4132  /* create guard : fill 1 out-of-range */
4133  bits[BITMAP_INDEX(p)] |= BITMAP_BIT(p)-1;
4134  bits[BITMAP_INDEX(pend)] |= ~(BITMAP_BIT(pend) - 1);
4135 
4136  for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
4137  bitset = ~bits[i];
4138  if (bitset) {
4139  p = offset + i * BITS_BITLENGTH;
4140  do {
4141  asan_unpoison_object((VALUE)p, false);
4142  if (bitset & 1) {
4143  switch (BUILTIN_TYPE(p)) {
4144  default: { /* majority case */
4145  gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
4146 #if USE_RGENGC && RGENGC_CHECK_MODE
4147  if (!is_full_marking(objspace)) {
4148  if (RVALUE_OLD_P((VALUE)p)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
4149  if (rgengc_remembered_sweep(objspace, (VALUE)p)) rb_bug("page_sweep: %p - remembered.", (void *)p);
4150  }
4151 #endif
4152  if (obj_free(objspace, (VALUE)p)) {
4153  final_slots++;
4154  }
4155  else {
4156  (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
4157  heap_page_add_freeobj(objspace, sweep_page, (VALUE)p);
4158  gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info((VALUE)p));
4159  freed_slots++;
4160  asan_poison_object((VALUE)p);
4161  }
4162  break;
4163  }
4164 
4165  /* minor cases */
4166  case T_ZOMBIE:
4167  /* already counted */
4168  break;
4169  case T_NONE:
4170  empty_slots++; /* already freed */
4171  break;
4172  }
4173  }
4174  p++;
4175  bitset >>= 1;
4176  } while (bitset);
4177  }
4178  }
4179 
4180  gc_setup_mark_bits(sweep_page);
4181 
4182 #if GC_PROFILE_MORE_DETAIL
4183  if (gc_prof_enabled(objspace)) {
4184  gc_profile_record *record = gc_prof_record(objspace);
4185  record->removing_objects += final_slots + freed_slots;
4186  record->empty_objects += empty_slots;
4187  }
4188 #endif
4189  if (0) fprintf(stderr, "gc_page_sweep(%d): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
4190  (int)rb_gc_count(),
4191  (int)sweep_page->total_slots,
4192  freed_slots, empty_slots, final_slots);
4193 
4194  sweep_page->free_slots = freed_slots + empty_slots;
4195  objspace->profile.total_freed_objects += freed_slots;
4197  sweep_page->final_slots += final_slots;
4198 
4200  rb_thread_t *th = GET_THREAD();
4201  if (th) {
4202  gc_finalize_deferred_register(objspace);
4203  }
4204  }
4205 
4206  gc_report(2, objspace, "page_sweep: end.\n");
4207 
4208  return freed_slots + empty_slots;
4209 }
4210 
4211 /* allocate additional minimum page to work */
4212 static void
4213 gc_heap_prepare_minimum_pages(rb_objspace_t *objspace, rb_heap_t *heap)
4214 {
4215  if (!heap->free_pages && heap_increment(objspace, heap) == FALSE) {
4216  /* there is no free after page_sweep() */
4217  heap_set_increment(objspace, 1);
4218  if (!heap_increment(objspace, heap)) { /* can't allocate additional free objects */
4219  rb_memerror();
4220  }
4221  }
4222 }
4223 
4224 static const char *
4225 gc_mode_name(enum gc_mode mode)
4226 {
4227  switch (mode) {
4228  case gc_mode_none: return "none";
4229  case gc_mode_marking: return "marking";
4230  case gc_mode_sweeping: return "sweeping";
4231  default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode);
4232  }
4233 }
4234 
4235 static void
4236 gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
4237 {
4238 #if RGENGC_CHECK_MODE
4239  enum gc_mode prev_mode = gc_mode(objspace);
4240  switch (prev_mode) {
4241  case gc_mode_none: GC_ASSERT(mode == gc_mode_marking); break;
4242  case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping); break;
4243  case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none); break;
4244  }
4245 #endif
4246  if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode));
4247  gc_mode_set(objspace, mode);
4248 }
4249 
4250 static void
4251 gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
4252 {
4253  heap->sweeping_page = list_top(&heap->pages, struct heap_page, page_node);
4254  heap->free_pages = NULL;
4255 #if GC_ENABLE_INCREMENTAL_MARK
4256  heap->pooled_pages = NULL;
4257  objspace->rincgc.pooled_slots = 0;
4258 #endif
4259  if (heap->using_page) {
4260  struct heap_page *page = heap->using_page;
4261  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
4262 
4263  RVALUE **p = &page->freelist;
4264  while (*p) {
4265  p = &(*p)->as.free.next;
4266  }
4267  *p = heap->freelist;
4268  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
4269  heap->using_page = NULL;
4270  }
4271  heap->freelist = NULL;
4272 }
4273 
4274 #if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4
4275 __attribute__((noinline))
4276 #endif
4277 static void
4278 gc_sweep_start(rb_objspace_t *objspace)
4279 {
4280  gc_mode_transition(objspace, gc_mode_sweeping);
4281  gc_sweep_start_heap(objspace, heap_eden);
4282 }
4283 
4284 static void
4285 gc_sweep_finish(rb_objspace_t *objspace)
4286 {
4287  gc_report(1, objspace, "gc_sweep_finish\n");
4288 
4289  gc_prof_set_heap_info(objspace);
4290  heap_pages_free_unused_pages(objspace);
4291 
4292  /* if heap_pages has unused pages, then assign them to increment */
4293  if (heap_allocatable_pages < heap_tomb->total_pages) {
4294  heap_allocatable_pages_set(objspace, heap_tomb->total_pages);
4295  }
4296 
4298  gc_mode_transition(objspace, gc_mode_none);
4299 
4300 #if RGENGC_CHECK_MODE >= 2
4301  gc_verify_internal_consistency(objspace);
4302 #endif
4303 }
4304 
4305 static int
4306 gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
4307 {
4308  struct heap_page *sweep_page = heap->sweeping_page;
4309  int unlink_limit = 3;
4310 #if GC_ENABLE_INCREMENTAL_MARK
4311  int need_pool = will_be_incremental_marking(objspace) ? TRUE : FALSE;
4312 
4313  gc_report(2, objspace, "gc_sweep_step (need_pool: %d)\n", need_pool);
4314 #else
4315  gc_report(2, objspace, "gc_sweep_step\n");
4316 #endif
4317 
4318  if (sweep_page == NULL) return FALSE;
4319 
4320 #if GC_ENABLE_LAZY_SWEEP
4321  gc_prof_sweep_timer_start(objspace);
4322 #endif
4323 
4324  do {
4325  int free_slots = gc_page_sweep(objspace, heap, sweep_page);
4326  heap->sweeping_page = list_next(&heap->pages, sweep_page, page_node);
4327 
4328  if (sweep_page->final_slots + free_slots == sweep_page->total_slots &&
4330  unlink_limit > 0) {
4332  unlink_limit--;
4333  /* there are no living objects -> move this page to tomb heap */
4334  heap_unlink_page(objspace, heap, sweep_page);
4335  heap_add_page(objspace, heap_tomb, sweep_page);
4336  }
4337  else if (free_slots > 0) {
4338 #if GC_ENABLE_INCREMENTAL_MARK
4339  if (need_pool) {
4340  if (heap_add_poolpage(objspace, heap, sweep_page)) {
4341  need_pool = FALSE;
4342  }
4343  }
4344  else {
4345  heap_add_freepage(heap, sweep_page);
4346  break;
4347  }
4348 #else
4349  heap_add_freepage(heap, sweep_page);
4350  break;
4351 #endif
4352  }
4353  else {
4354  sweep_page->free_next = NULL;
4355  }
4356  } while ((sweep_page = heap->sweeping_page));
4357 
4358  if (!heap->sweeping_page) {
4359  gc_sweep_finish(objspace);
4360  }
4361 
4362 #if GC_ENABLE_LAZY_SWEEP
4363  gc_prof_sweep_timer_stop(objspace);
4364 #endif
4365 
4366  return heap->free_pages != NULL;
4367 }
4368 
4369 static void
4370 gc_sweep_rest(rb_objspace_t *objspace)
4371 {
4372  rb_heap_t *heap = heap_eden; /* lazy sweep only for eden */
4373 
4374  while (has_sweeping_pages(heap)) {
4375  gc_sweep_step(objspace, heap);
4376  }
4377 }
4378 
4379 static void
4380 gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap)
4381 {
4382  GC_ASSERT(dont_gc == FALSE);
4383  if (!GC_ENABLE_LAZY_SWEEP) return;
4384 
4385  gc_enter(objspace, "sweep_continue");
4386 #if USE_RGENGC
4387  if (objspace->rgengc.need_major_gc == GPR_FLAG_NONE && heap_increment(objspace, heap)) {
4388  gc_report(3, objspace, "gc_sweep_continue: success heap_increment().\n");
4389  }
4390 #endif
4391  gc_sweep_step(objspace, heap);
4392  gc_exit(objspace, "sweep_continue");
4393 }
4394 
4395 static void
4396 gc_sweep(rb_objspace_t *objspace)
4397 {
4398  const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
4399 
4400  gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
4401 
4402  if (immediate_sweep) {
4403 #if !GC_ENABLE_LAZY_SWEEP
4404  gc_prof_sweep_timer_start(objspace);
4405 #endif
4406  gc_sweep_start(objspace);
4407  gc_sweep_rest(objspace);
4408 #if !GC_ENABLE_LAZY_SWEEP
4409  gc_prof_sweep_timer_stop(objspace);
4410 #endif
4411  }
4412  else {
4413  struct heap_page *page = NULL;
4414  gc_sweep_start(objspace);
4415 
4416  list_for_each(&heap_eden->pages, page, page_node) {
4417  page->flags.before_sweep = TRUE;
4418  }
4419  gc_sweep_step(objspace, heap_eden);
4420  }
4421 
4422  gc_heap_prepare_minimum_pages(objspace, heap_eden);
4423 }
4424 
4425 /* Marking - Marking stack */
4426 
4427 static stack_chunk_t *
4428 stack_chunk_alloc(void)
4429 {
4430  stack_chunk_t *res;
4431 
4432  res = malloc(sizeof(stack_chunk_t));
4433  if (!res)
4434  rb_memerror();
4435 
4436  return res;
4437 }
4438 
4439 static inline int
4440 is_mark_stack_empty(mark_stack_t *stack)
4441 {
4442  return stack->chunk == NULL;
4443 }
4444 
4445 static size_t
4446 mark_stack_size(mark_stack_t *stack)
4447 {
4448  size_t size = stack->index;
4449  stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
4450 
4451  while (chunk) {
4452  size += stack->limit;
4453  chunk = chunk->next;
4454  }
4455  return size;
4456 }
4457 
4458 static void
4459 add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
4460 {
4461  chunk->next = stack->cache;
4462  stack->cache = chunk;
4463  stack->cache_size++;
4464 }
4465 
4466 static void
4467 shrink_stack_chunk_cache(mark_stack_t *stack)
4468 {
4469  stack_chunk_t *chunk;
4470 
4471  if (stack->unused_cache_size > (stack->cache_size/2)) {
4472  chunk = stack->cache;
4473  stack->cache = stack->cache->next;
4474  stack->cache_size--;
4475  free(chunk);
4476  }
4477  stack->unused_cache_size = stack->cache_size;
4478 }
4479 
4480 static void
4481 push_mark_stack_chunk(mark_stack_t *stack)
4482 {
4483  stack_chunk_t *next;
4484 
4485  GC_ASSERT(stack->index == stack->limit);
4486 
4487  if (stack->cache_size > 0) {
4488  next = stack->cache;
4489  stack->cache = stack->cache->next;
4490  stack->cache_size--;
4491  if (stack->unused_cache_size > stack->cache_size)
4492  stack->unused_cache_size = stack->cache_size;
4493  }
4494  else {
4495  next = stack_chunk_alloc();
4496  }
4497  next->next = stack->chunk;
4498  stack->chunk = next;
4499  stack->index = 0;
4500 }
4501 
4502 static void
4503 pop_mark_stack_chunk(mark_stack_t *stack)
4504 {
4505  stack_chunk_t *prev;
4506 
4507  prev = stack->chunk->next;
4508  GC_ASSERT(stack->index == 0);
4509  add_stack_chunk_cache(stack, stack->chunk);
4510  stack->chunk = prev;
4511  stack->index = stack->limit;
4512 }
4513 
4514 static void
4515 free_stack_chunks(mark_stack_t *stack)
4516 {
4517  stack_chunk_t *chunk = stack->chunk;
4518  stack_chunk_t *next = NULL;
4519 
4520  while (chunk != NULL) {
4521  next = chunk->next;
4522  free(chunk);
4523  chunk = next;
4524  }
4525 }
4526 
4527 static void
4528 push_mark_stack(mark_stack_t *stack, VALUE data)
4529 {
4530  if (stack->index == stack->limit) {
4531  push_mark_stack_chunk(stack);
4532  }
4533  stack->chunk->data[stack->index++] = data;
4534 }
4535 
4536 static int
4537 pop_mark_stack(mark_stack_t *stack, VALUE *data)
4538 {
4539  if (is_mark_stack_empty(stack)) {
4540  return FALSE;
4541  }
4542  if (stack->index == 1) {
4543  *data = stack->chunk->data[--stack->index];
4544  pop_mark_stack_chunk(stack);
4545  }
4546  else {
4547  *data = stack->chunk->data[--stack->index];
4548  }
4549  return TRUE;
4550 }
4551 
4552 #if GC_ENABLE_INCREMENTAL_MARK
4553 static int
4554 invalidate_mark_stack_chunk(stack_chunk_t *chunk, int limit, VALUE obj)
4555 {
4556  int i;
4557  for (i=0; i<limit; i++) {
4558  if (chunk->data[i] == obj) {
4559  chunk->data[i] = Qundef;
4560  return TRUE;
4561  }
4562  }
4563  return FALSE;
4564 }
4565 
4566 static void
4567 invalidate_mark_stack(mark_stack_t *stack, VALUE obj)
4568 {
4569  stack_chunk_t *chunk = stack->chunk;
4570  int limit = stack->index;
4571 
4572  while (chunk) {
4573  if (invalidate_mark_stack_chunk(chunk, limit, obj)) return;
4574  chunk = chunk->next;
4575  limit = stack->limit;
4576  }
4577  rb_bug("invalid_mark_stack: unreachable");
4578 }
4579 #endif
4580 
4581 static void
4582 init_mark_stack(mark_stack_t *stack)
4583 {
4584  int i;
4585 
4586  MEMZERO(stack, mark_stack_t, 1);
4587  stack->index = stack->limit = STACK_CHUNK_SIZE;
4588  stack->cache_size = 0;
4589 
4590  for (i=0; i < 4; i++) {
4591  add_stack_chunk_cache(stack, stack_chunk_alloc());
4592  }
4593  stack->unused_cache_size = stack->cache_size;
4594 }
4595 
4596 /* Marking */
4597 
4598 #define SET_STACK_END SET_MACHINE_STACK_END(&ec->machine.stack_end)
4599 
4600 #define STACK_START (ec->machine.stack_start)
4601 #define STACK_END (ec->machine.stack_end)
4602 #define STACK_LEVEL_MAX (ec->machine.stack_maxsize/sizeof(VALUE))
4603 
4604 #ifdef __EMSCRIPTEN__
4605 #undef STACK_GROW_DIRECTION
4606 #define STACK_GROW_DIRECTION 1
4607 #endif
4608 
4609 #if STACK_GROW_DIRECTION < 0
4610 # define STACK_LENGTH (size_t)(STACK_START - STACK_END)
4611 #elif STACK_GROW_DIRECTION > 0
4612 # define STACK_LENGTH (size_t)(STACK_END - STACK_START + 1)
4613 #else
4614 # define STACK_LENGTH ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \
4615  : (size_t)(STACK_END - STACK_START + 1))
4616 #endif
4617 #if !STACK_GROW_DIRECTION
4619 int
4621 {
4622  VALUE *end;
4623  SET_MACHINE_STACK_END(&end);
4624 
4625  if (end > addr) return ruby_stack_grow_direction = 1;
4626  return ruby_stack_grow_direction = -1;
4627 }
4628 #endif
4629 
4630 size_t
4632 {
4634  SET_STACK_END;
4635  if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END);
4636  return STACK_LENGTH;
4637 }
4638 
4639 #define PREVENT_STACK_OVERFLOW 1
4640 #ifndef PREVENT_STACK_OVERFLOW
4641 #if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK))
4642 # define PREVENT_STACK_OVERFLOW 1
4643 #else
4644 # define PREVENT_STACK_OVERFLOW 0
4645 #endif
4646 #endif
4647 #if PREVENT_STACK_OVERFLOW
4648 static int
4649 stack_check(rb_execution_context_t *ec, int water_mark)
4650 {
4651  SET_STACK_END;
4652 
4653  size_t length = STACK_LENGTH;
4654  size_t maximum_length = STACK_LEVEL_MAX - water_mark;
4655 
4656  return length > maximum_length;
4657 }
4658 #else
4659 #define stack_check(ec, water_mark) FALSE
4660 #endif
4661 
4662 #define STACKFRAME_FOR_CALL_CFUNC 2048
4663 
4666 {
4667  return stack_check(ec, STACKFRAME_FOR_CALL_CFUNC);
4668 }
4669 
4670 int
4672 {
4673  return stack_check(GET_EC(), STACKFRAME_FOR_CALL_CFUNC);
4674 }
4675 
4676 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void mark_locations_array(rb_objspace_t *objspace, register const VALUE *x, register long n));
4677 static void
4678 mark_locations_array(rb_objspace_t *objspace, register const VALUE *x, register long n)
4679 {
4680  VALUE v;
4681  while (n--) {
4682  v = *x;
4683  gc_mark_maybe(objspace, v);
4684  x++;
4685  }
4686 }
4687 
4688 static void
4689 gc_mark_locations(rb_objspace_t *objspace, const VALUE *start, const VALUE *end)
4690 {
4691  long n;
4692 
4693  if (end <= start) return;
4694  n = end - start;
4695  mark_locations_array(objspace, start, n);
4696 }
4697 
4698 void
4700 {
4701  gc_mark_locations(&rb_objspace, start, end);
4702 }
4703 
4704 static void
4705 gc_mark_values(rb_objspace_t *objspace, long n, const VALUE *values)
4706 {
4707  long i;
4708 
4709  for (i=0; i<n; i++) {
4710  gc_mark(objspace, values[i]);
4711  }
4712 }
4713 
4714 void
4715 rb_gc_mark_values(long n, const VALUE *values)
4716 {
4717  long i;
4718  rb_objspace_t *objspace = &rb_objspace;
4719 
4720  for (i=0; i<n; i++) {
4721  gc_mark_and_pin(objspace, values[i]);
4722  }
4723 }
4724 
4725 static void
4726 gc_mark_and_pin_stack_values(rb_objspace_t *objspace, long n, const VALUE *values)
4727 {
4728  long i;
4729 
4730  for (i=0; i<n; i++) {
4731  /* skip MOVED objects that are on the stack */
4732  if (is_markable_object(objspace, values[i]) && T_MOVED != BUILTIN_TYPE(values[i])) {
4733  gc_mark_and_pin(objspace, values[i]);
4734  }
4735  }
4736 }
4737 
4738 void
4739 rb_gc_mark_vm_stack_values(long n, const VALUE *values)
4740 {
4741  rb_objspace_t *objspace = &rb_objspace;
4742  gc_mark_and_pin_stack_values(objspace, n, values);
4743 }
4744 
4745 static int
4746 mark_value(st_data_t key, st_data_t value, st_data_t data)
4747 {
4748  rb_objspace_t *objspace = (rb_objspace_t *)data;
4749  gc_mark(objspace, (VALUE)value);
4750  return ST_CONTINUE;
4751 }
4752 
4753 static int
4754 mark_value_pin(st_data_t key, st_data_t value, st_data_t data)
4755 {
4756  rb_objspace_t *objspace = (rb_objspace_t *)data;
4757  gc_mark_and_pin(objspace, (VALUE)value);
4758  return ST_CONTINUE;
4759 }
4760 
4761 static void
4762 mark_tbl_no_pin(rb_objspace_t *objspace, st_table *tbl)
4763 {
4764  if (!tbl || tbl->num_entries == 0) return;
4765  st_foreach(tbl, mark_value, (st_data_t)objspace);
4766 }
4767 
4768 static void
4769 mark_tbl(rb_objspace_t *objspace, st_table *tbl)
4770 {
4771  if (!tbl || tbl->num_entries == 0) return;
4772  st_foreach(tbl, mark_value_pin, (st_data_t)objspace);
4773 }
4774 
4775 static int
4776 mark_key(st_data_t key, st_data_t value, st_data_t data)
4777 {
4778  rb_objspace_t *objspace = (rb_objspace_t *)data;
4779  gc_mark_and_pin(objspace, (VALUE)key);
4780  return ST_CONTINUE;
4781 }
4782 
4783 static void
4784 mark_set(rb_objspace_t *objspace, st_table *tbl)
4785 {
4786  if (!tbl) return;
4787  st_foreach(tbl, mark_key, (st_data_t)objspace);
4788 }
4789 
4790 static void
4791 mark_finalizer_tbl(rb_objspace_t *objspace, st_table *tbl)
4792 {
4793  if (!tbl) return;
4794  st_foreach(tbl, mark_value, (st_data_t)objspace);
4795 }
4796 
4797 void
4799 {
4800  mark_set(&rb_objspace, tbl);
4801 }
4802 
4803 static int
4804 mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
4805 {
4806  rb_objspace_t *objspace = (rb_objspace_t *)data;
4807 
4808  gc_mark(objspace, (VALUE)key);
4809  gc_mark(objspace, (VALUE)value);
4810  return ST_CONTINUE;
4811 }
4812 
4813 static int
4814 pin_key_pin_value(st_data_t key, st_data_t value, st_data_t data)
4815 {
4816  rb_objspace_t *objspace = (rb_objspace_t *)data;
4817 
4818  gc_mark_and_pin(objspace, (VALUE)key);
4819  gc_mark_and_pin(objspace, (VALUE)value);
4820  return ST_CONTINUE;
4821 }
4822 
4823 static int
4824 pin_key_mark_value(st_data_t key, st_data_t value, st_data_t data)
4825 {
4826  rb_objspace_t *objspace = (rb_objspace_t *)data;
4827 
4828  gc_mark_and_pin(objspace, (VALUE)key);
4829  gc_mark(objspace, (VALUE)value);
4830  return ST_CONTINUE;
4831 }
4832 
4833 static void
4834 mark_hash(rb_objspace_t *objspace, VALUE hash)
4835 {
4836  if (rb_hash_compare_by_id_p(hash)) {
4837  rb_hash_stlike_foreach(hash, pin_key_mark_value, (st_data_t)objspace);
4838  }
4839  else {
4840  rb_hash_stlike_foreach(hash, mark_keyvalue, (st_data_t)objspace);
4841  }
4842 
4843  if (RHASH_AR_TABLE_P(hash)) {
4844  if (objspace->mark_func_data == NULL && RHASH_TRANSIENT_P(hash)) {
4846  }
4847  }
4848  else {
4849  VM_ASSERT(!RHASH_TRANSIENT_P(hash));
4850  }
4851  gc_mark(objspace, RHASH(hash)->ifnone);
4852 }
4853 
4854 static void
4855 mark_st(rb_objspace_t *objspace, st_table *tbl)
4856 {
4857  if (!tbl) return;
4858  st_foreach(tbl, pin_key_pin_value, (st_data_t)objspace);
4859 }
4860 
4861 void
4863 {
4864  mark_st(&rb_objspace, tbl);
4865 }
4866 
4867 static void
4868 mark_method_entry(rb_objspace_t *objspace, const rb_method_entry_t *me)
4869 {
4870  const rb_method_definition_t *def = me->def;
4871 
4872  gc_mark(objspace, me->owner);
4873  gc_mark(objspace, me->defined_class);
4874 
4875  if (def) {
4876  switch (def->type) {
4877  case VM_METHOD_TYPE_ISEQ:
4878  if (def->body.iseq.iseqptr) gc_mark(objspace, (VALUE)def->body.iseq.iseqptr);
4879  gc_mark(objspace, (VALUE)def->body.iseq.cref);
4880  break;
4882  case VM_METHOD_TYPE_IVAR:
4883  gc_mark(objspace, def->body.attr.location);
4884  break;
4886  gc_mark(objspace, def->body.bmethod.proc);
4888  break;
4889  case VM_METHOD_TYPE_ALIAS:
4890  gc_mark(objspace, (VALUE)def->body.alias.original_me);
4891  return;
4893  gc_mark(objspace, (VALUE)def->body.refined.orig_me);
4894  gc_mark(objspace, (VALUE)def->body.refined.owner);
4895  break;
4896  case VM_METHOD_TYPE_CFUNC:
4897  case VM_METHOD_TYPE_ZSUPER:
4900  case VM_METHOD_TYPE_UNDEF:
4902  break;
4903  }
4904  }
4905 }
4906 
4907 static enum rb_id_table_iterator_result
4908 mark_method_entry_i(VALUE me, void *data)
4909 {
4910  rb_objspace_t *objspace = (rb_objspace_t *)data;
4911 
4912  gc_mark(objspace, me);
4913  return ID_TABLE_CONTINUE;
4914 }
4915 
4916 static void
4917 mark_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
4918 {
4919  if (tbl) {
4920  rb_id_table_foreach_values(tbl, mark_method_entry_i, objspace);
4921  }
4922 }
4923 
4924 static enum rb_id_table_iterator_result
4925 mark_const_entry_i(VALUE value, void *data)
4926 {
4927  const rb_const_entry_t *ce = (const rb_const_entry_t *)value;
4928  rb_objspace_t *objspace = data;
4929 
4930  gc_mark(objspace, ce->value);
4931  gc_mark(objspace, ce->file);
4932  return ID_TABLE_CONTINUE;
4933 }
4934 
4935 static void
4936 mark_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
4937 {
4938  if (!tbl) return;
4939  rb_id_table_foreach_values(tbl, mark_const_entry_i, objspace);
4940 }
4941 
4942 #if STACK_GROW_DIRECTION < 0
4943 #define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START)
4944 #elif STACK_GROW_DIRECTION > 0
4945 #define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix))
4946 #else
4947 #define GET_STACK_BOUNDS(start, end, appendix) \
4948  ((STACK_END < STACK_START) ? \
4949  ((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix)))
4950 #endif
4951 
4952 static void mark_stack_locations(rb_objspace_t *objspace, const rb_execution_context_t *ec,
4953  const VALUE *stack_start, const VALUE *stack_end);
4954 
4955 static void
4956 mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec)
4957 {
4958  union {
4959  rb_jmp_buf j;
4960  VALUE v[sizeof(rb_jmp_buf) / sizeof(VALUE)];
4961  } save_regs_gc_mark;
4962  VALUE *stack_start, *stack_end;
4963 
4965  memset(&save_regs_gc_mark, 0, sizeof(save_regs_gc_mark));
4966  /* This assumes that all registers are saved into the jmp_buf (and stack) */
4967  rb_setjmp(save_regs_gc_mark.j);
4968 
4969  /* SET_STACK_END must be called in this function because
4970  * the stack frame of this function may contain
4971  * callee save registers and they should be marked. */
4972  SET_STACK_END;
4973  GET_STACK_BOUNDS(stack_start, stack_end, 1);
4974 
4975  mark_locations_array(objspace, save_regs_gc_mark.v, numberof(save_regs_gc_mark.v));
4976 
4977  mark_stack_locations(objspace, ec, stack_start, stack_end);
4978 }
4979 
4980 void
4982 {
4983  rb_objspace_t *objspace = &rb_objspace;
4984  VALUE *stack_start, *stack_end;
4985 
4986  GET_STACK_BOUNDS(stack_start, stack_end, 0);
4987  mark_stack_locations(objspace, ec, stack_start, stack_end);
4988 }
4989 
4990 static void
4991 mark_stack_locations(rb_objspace_t *objspace, const rb_execution_context_t *ec,
4992  const VALUE *stack_start, const VALUE *stack_end)
4993 {
4994 
4995  gc_mark_locations(objspace, stack_start, stack_end);
4996 
4997 #if defined(__mc68000__)
4998  gc_mark_locations(objspace,
4999  (VALUE*)((char*)stack_start + 2),
5000  (VALUE*)((char*)stack_end - 2));
5001 #endif
5002 }
5003 
5004 void
5006 {
5007  mark_tbl(&rb_objspace, tbl);
5008 }
5009 
5010 void
5012 {
5013  mark_tbl_no_pin(&rb_objspace, tbl);
5014 }
5015 
5016 static void
5017 gc_mark_maybe(rb_objspace_t *objspace, VALUE obj)
5018 {
5019  (void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
5020 
5021  if (is_pointer_to_heap(objspace, (void *)obj)) {
5022  void *ptr = __asan_region_is_poisoned((void *)obj, SIZEOF_VALUE);
5023  asan_unpoison_object(obj, false);
5024 
5025  /* Garbage can live on the stack, so do not mark or pin */
5026  switch (BUILTIN_TYPE(obj)) {
5027  case T_MOVED:
5028  case T_ZOMBIE:
5029  case T_NONE:
5030  break;
5031  default:
5032  gc_mark_and_pin(objspace, obj);
5033  break;
5034  }
5035 
5036  if (ptr) {
5038  asan_poison_object(obj);
5039  }
5040  }
5041 }
5042 
5043 void
5045 {
5046  gc_mark_maybe(&rb_objspace, obj);
5047 }
5048 
5049 static inline int
5050 gc_mark_set(rb_objspace_t *objspace, VALUE obj)
5051 {
5052  if (RVALUE_MARKED(obj)) return 0;
5054  return 1;
5055 }
5056 
5057 #if USE_RGENGC
5058 static int
5059 gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
5060 {
5061  struct heap_page *page = GET_HEAP_PAGE(obj);
5063 
5068 
5069 #if RGENGC_PROFILE > 0
5070  objspace->profile.total_remembered_shady_object_count++;
5071 #if RGENGC_PROFILE >= 2
5072  objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
5073 #endif
5074 #endif
5075  return TRUE;
5076  }
5077  else {
5078  return FALSE;
5079  }
5080 }
5081 #endif
5082 
5083 static void
5084 rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
5085 {
5086 #if USE_RGENGC
5087  const VALUE old_parent = objspace->rgengc.parent_object;
5088 
5089  if (old_parent) { /* parent object is old */
5090  if (RVALUE_WB_UNPROTECTED(obj)) {
5091  if (gc_remember_unprotected(objspace, obj)) {
5092  gc_report(2, objspace, "relation: (O->S) %s -> %s\n", obj_info(old_parent), obj_info(obj));
5093  }
5094  }
5095  else {
5096  if (!RVALUE_OLD_P(obj)) {
5097  if (RVALUE_MARKED(obj)) {
5098  /* An object pointed from an OLD object should be OLD. */
5099  gc_report(2, objspace, "relation: (O->unmarked Y) %s -> %s\n", obj_info(old_parent), obj_info(obj));
5100  RVALUE_AGE_SET_OLD(objspace, obj);
5101  if (is_incremental_marking(objspace)) {
5102  if (!RVALUE_MARKING(obj)) {
5103  gc_grey(objspace, obj);
5104  }
5105  }
5106  else {
5107  rgengc_remember(objspace, obj);
5108  }
5109  }
5110  else {
5111  gc_report(2, objspace, "relation: (O->Y) %s -> %s\n", obj_info(old_parent), obj_info(obj));
5112  RVALUE_AGE_SET_CANDIDATE(objspace, obj);
5113  }
5114  }
5115  }
5116  }
5117 
5118  GC_ASSERT(old_parent == objspace->rgengc.parent_object);
5119 #endif
5120 }
5121 
5122 static void
5123 gc_grey(rb_objspace_t *objspace, VALUE obj)
5124 {
5125 #if RGENGC_CHECK_MODE
5126  if (RVALUE_MARKED(obj) == FALSE) rb_bug("gc_grey: %s is not marked.", obj_info(obj));
5127  if (RVALUE_MARKING(obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", obj_info(obj));
5128 #endif
5129 
5130 #if GC_ENABLE_INCREMENTAL_MARK
5131  if (is_incremental_marking(objspace)) {
5133  }
5134 #endif
5135 
5136  push_mark_stack(&objspace->mark_stack, obj);
5137 }
5138 
5139 static void
5140 gc_aging(rb_objspace_t *objspace, VALUE obj)
5141 {
5142 #if USE_RGENGC
5143  struct heap_page *page = GET_HEAP_PAGE(obj);
5144 
5145  GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
5146  check_rvalue_consistency(obj);
5147 
5148  if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
5149  if (!RVALUE_OLD_P(obj)) {
5150  gc_report(3, objspace, "gc_aging: YOUNG: %s\n", obj_info(obj));
5151  RVALUE_AGE_INC(objspace, obj);
5152  }
5153  else if (is_full_marking(objspace)) {
5155  RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
5156  }
5157  }
5158  check_rvalue_consistency(obj);
5159 #endif /* USE_RGENGC */
5160 
5161  objspace->marked_slots++;
5162 }
5163 
5164 NOINLINE(static void gc_mark_ptr(rb_objspace_t *objspace, VALUE obj));
5165 
5166 static void
5167 gc_mark_ptr(rb_objspace_t *objspace, VALUE obj)
5168 {
5169  if (LIKELY(objspace->mark_func_data == NULL)) {
5170  rgengc_check_relation(objspace, obj);
5171  if (!gc_mark_set(objspace, obj)) return; /* already marked */
5172  if (RB_TYPE_P(obj, T_NONE)) rb_bug("try to mark T_NONE object"); /* check here will help debugging */
5173  gc_aging(objspace, obj);
5174  gc_grey(objspace, obj);
5175  }
5176  else {
5177  objspace->mark_func_data->mark_func(obj, objspace->mark_func_data->data);
5178  }
5179 }
5180 
5181 static inline void
5182 gc_pin(rb_objspace_t *objspace, VALUE obj)
5183 {
5184  GC_ASSERT(is_markable_object(objspace, obj));
5185  if (UNLIKELY(objspace->flags.during_compacting)) {
5187  }
5188 }
5189 
5190 static inline void
5191 gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj)
5192 {
5193  if (!is_markable_object(objspace, obj)) return;
5194  gc_pin(objspace, obj);
5195  gc_mark_ptr(objspace, obj);
5196 }
5197 
5198 static inline void
5199 gc_mark(rb_objspace_t *objspace, VALUE obj)
5200 {
5201  if (!is_markable_object(objspace, obj)) return;
5202  gc_mark_ptr(objspace, obj);
5203 }
5204 
5205 void
5207 {
5208  gc_mark(&rb_objspace, ptr);
5209 }
5210 
5211 void
5213 {
5214  gc_mark_and_pin(&rb_objspace, ptr);
5215 }
5216 
5217 /* CAUTION: THIS FUNCTION ENABLE *ONLY BEFORE* SWEEPING.
5218  * This function is only for GC_END_MARK timing.
5219  */
5220 
5221 int
5223 {
5224  return RVALUE_MARKED(obj) ? TRUE : FALSE;
5225 }
5226 
5227 static inline void
5228 gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
5229 {
5230 #if USE_RGENGC
5231  if (RVALUE_OLD_P(obj)) {
5232  objspace->rgengc.parent_object = obj;
5233  }
5234  else {
5235  objspace->rgengc.parent_object = Qfalse;
5236  }
5237 #endif
5238 }
5239 
5240 static void
5241 gc_mark_imemo(rb_objspace_t *objspace, VALUE obj)
5242 {
5243  switch (imemo_type(obj)) {
5244  case imemo_env:
5245  {
5246  const rb_env_t *env = (const rb_env_t *)obj;
5247  GC_ASSERT(VM_ENV_ESCAPED_P(env->ep));
5248  gc_mark_values(objspace, (long)env->env_size, env->env);
5249  VM_ENV_FLAGS_SET(env->ep, VM_ENV_FLAG_WB_REQUIRED);
5250  gc_mark(objspace, (VALUE)rb_vm_env_prev_env(env));
5251  gc_mark(objspace, (VALUE)env->iseq);
5252  }
5253  return;
5254  case imemo_cref:
5255  gc_mark(objspace, RANY(obj)->as.imemo.cref.klass);
5256  gc_mark(objspace, (VALUE)RANY(obj)->as.imemo.cref.next);
5257  gc_mark(objspace, RANY(obj)->as.imemo.cref.refinements);
5258  return;
5259  case imemo_svar:
5260  gc_mark(objspace, RANY(obj)->as.imemo.svar.cref_or_me);
5261  gc_mark(objspace, RANY(obj)->as.imemo.svar.lastline);
5262  gc_mark(objspace, RANY(obj)->as.imemo.svar.backref);
5263  gc_mark(objspace, RANY(obj)->as.imemo.svar.others);
5264  return;
5265  case imemo_throw_data:
5266  gc_mark(objspace, RANY(obj)->as.imemo.throw_data.throw_obj);
5267  return;
5268  case imemo_ifunc:
5269  gc_mark_maybe(objspace, (VALUE)RANY(obj)->as.imemo.ifunc.data);
5270  return;
5271  case imemo_memo:
5272  gc_mark(objspace, RANY(obj)->as.imemo.memo.v1);
5273  gc_mark(objspace, RANY(obj)->as.imemo.memo.v2);
5274  gc_mark_maybe(objspace, RANY(obj)->as.imemo.memo.u3.value);
5275  return;
5276  case imemo_ment:
5277  mark_method_entry(objspace, &RANY(obj)->as.imemo.ment);
5278  return;
5279  case imemo_iseq:
5281  return;
5282  case imemo_tmpbuf:
5283  {
5284  const rb_imemo_tmpbuf_t *m = &RANY(obj)->as.imemo.alloc;
5285  do {
5286  rb_gc_mark_locations(m->ptr, m->ptr + m->cnt);
5287  } while ((m = m->next) != NULL);
5288  }
5289  return;
5290  case imemo_ast:
5291  rb_ast_mark(&RANY(obj)->as.imemo.ast);
5292  return;
5293  case imemo_parser_strterm:
5295  return;
5296 #if VM_CHECK_MODE > 0
5297  default:
5298  VM_UNREACHABLE(gc_mark_imemo);
5299 #endif
5300  }
5301 }
5302 
5303 static void
5304 gc_mark_children(rb_objspace_t *objspace, VALUE obj)
5305 {
5306  register RVALUE *any = RANY(obj);
5307  gc_mark_set_parent(objspace, obj);
5308 
5309  if (FL_TEST(obj, FL_EXIVAR)) {
5311  }
5312 
5313  switch (BUILTIN_TYPE(obj)) {
5314  case T_FLOAT:
5315  case T_BIGNUM:
5316  case T_SYMBOL:
5317  /* Not immediates, but does not have references and singleton
5318  * class */
5319  return;
5320 
5321  case T_NIL:
5322  case T_FIXNUM:
5323  rb_bug("rb_gc_mark() called for broken object");
5324  break;
5325 
5326  case T_NODE:
5328  break;
5329 
5330  case T_IMEMO:
5331  gc_mark_imemo(objspace, obj);
5332  return;
5333  }
5334 
5335  gc_mark(objspace, any->as.basic.klass);
5336 
5337  switch (BUILTIN_TYPE(obj)) {
5338  case T_CLASS:
5339  case T_MODULE:
5340  if (RCLASS_SUPER(obj)) {
5341  gc_mark(objspace, RCLASS_SUPER(obj));
5342  }
5343  if (!RCLASS_EXT(obj)) break;
5344  mark_m_tbl(objspace, RCLASS_M_TBL(obj));
5345  mark_tbl_no_pin(objspace, RCLASS_IV_TBL(obj));
5346  mark_const_tbl(objspace, RCLASS_CONST_TBL(obj));
5347  break;
5348 
5349  case T_ICLASS:
5350  if (FL_TEST(obj, RICLASS_IS_ORIGIN)) {
5351  mark_m_tbl(objspace, RCLASS_M_TBL(obj));
5352  }
5353  if (RCLASS_SUPER(obj)) {
5354  gc_mark(objspace, RCLASS_SUPER(obj));
5355  }
5356  if (!RCLASS_EXT(obj)) break;
5357  mark_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
5358  break;
5359 
5360  case T_ARRAY:
5361  if (FL_TEST(obj, ELTS_SHARED)) {
5362  VALUE root = any->as.array.as.heap.aux.shared_root;
5363  gc_mark(objspace, root);
5364  }
5365  else {
5366  long i, len = RARRAY_LEN(obj);
5368  for (i=0; i < len; i++) {
5369  gc_mark(objspace, ptr[i]);
5370  }
5371 
5372  if (objspace->mark_func_data == NULL) {
5376  }
5377  }
5378  }
5379  break;
5380 
5381  case T_HASH:
5382  mark_hash(objspace, obj);
5383  break;
5384 
5385  case T_STRING:
5386  if (STR_SHARED_P(obj)) {
5387  gc_mark(objspace, any->as.string.as.heap.aux.shared);
5388  }
5389  break;
5390 
5391  case T_DATA:
5392  {
5393  void *const ptr = DATA_PTR(obj);
5394  if (ptr) {
5395  RUBY_DATA_FUNC mark_func = RTYPEDDATA_P(obj) ?
5396  any->as.typeddata.type->function.dmark :
5397  any->as.data.dmark;
5398  if (mark_func) (*mark_func)(ptr);
5399  }
5400  }
5401  break;
5402 
5403  case T_OBJECT:
5404  {
5405  const VALUE * const ptr = ROBJECT_IVPTR(obj);
5406 
5407  if (ptr) {
5409  for (i = 0; i < len; i++) {
5410  gc_mark(objspace, ptr[i]);
5411  }
5412 
5413  if (objspace->mark_func_data == NULL &&
5414  ROBJ_TRANSIENT_P(obj)) {
5416  }
5417  }
5418  }
5419  break;
5420 
5421  case T_FILE:
5422  if (any->as.file.fptr) {
5423  gc_mark(objspace, any->as.file.fptr->pathv);
5424  gc_mark(objspace, any->as.file.fptr->tied_io_for_writing);
5425  gc_mark(objspace, any->as.file.fptr->writeconv_asciicompat);
5426  gc_mark(objspace, any->as.file.fptr->writeconv_pre_ecopts);
5427  gc_mark(objspace, any->as.file.fptr->encs.ecopts);
5428  gc_mark(objspace, any->as.file.fptr->write_lock);
5429  }
5430  break;
5431 
5432  case T_REGEXP:
5433  gc_mark(objspace, any->as.regexp.src);
5434  break;
5435 
5436  case T_MATCH:
5437  gc_mark(objspace, any->as.match.regexp);
5438  if (any->as.match.str) {
5439  gc_mark(objspace, any->as.match.str);
5440  }
5441  break;
5442 
5443  case T_RATIONAL:
5444  gc_mark(objspace, any->as.rational.num);
5445  gc_mark(objspace, any->as.rational.den);
5446  break;
5447 
5448  case T_COMPLEX:
5449  gc_mark(objspace, any->as.complex.real);
5450  gc_mark(objspace, any->as.complex.imag);
5451  break;
5452 
5453  case T_STRUCT:
5454  {
5455  long i;
5456  const long len = RSTRUCT_LEN(obj);
5457  const VALUE * const ptr = RSTRUCT_CONST_PTR(obj);
5458 
5459  for (i=0; i<len; i++) {
5460  gc_mark(objspace, ptr[i]);
5461  }
5462 
5463  if (objspace->mark_func_data == NULL &&
5466  }
5467  }
5468  break;
5469 
5470  default:
5471 #if GC_DEBUG
5473 #endif
5474  if (BUILTIN_TYPE(obj) == T_MOVED) rb_bug("rb_gc_mark(): %p is T_MOVED", (void *)obj);
5475  if (BUILTIN_TYPE(obj) == T_NONE) rb_bug("rb_gc_mark(): %p is T_NONE", (void *)obj);
5476  if (BUILTIN_TYPE(obj) == T_ZOMBIE) rb_bug("rb_gc_mark(): %p is T_ZOMBIE", (void *)obj);
5477  rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
5478  BUILTIN_TYPE(obj), (void *)any,
5479  is_pointer_to_heap(objspace, any) ? "corrupted object" : "non object");
5480  }
5481 }
5482 
5487 static inline int
5488 gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count)
5489 {
5490  mark_stack_t *mstack = &objspace->mark_stack;
5491  VALUE obj;
5492 #if GC_ENABLE_INCREMENTAL_MARK
5493  size_t marked_slots_at_the_beginning = objspace->marked_slots;
5494  size_t popped_count = 0;
5495 #endif
5496 
5497  while (pop_mark_stack(mstack, &obj)) {
5498  if (obj == Qundef) continue; /* skip */
5499 
5500  if (RGENGC_CHECK_MODE && !RVALUE_MARKED(obj)) {
5501  rb_bug("gc_mark_stacked_objects: %s is not marked.", obj_info(obj));
5502  }
5503  gc_mark_children(objspace, obj);
5504 
5505 #if GC_ENABLE_INCREMENTAL_MARK
5506  if (incremental) {
5507  if (RGENGC_CHECK_MODE && !RVALUE_MARKING(obj)) {
5508  rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0");
5509  }
5511  popped_count++;
5512 
5513  if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) {
5514  break;
5515  }
5516  }
5517  else {
5518  /* just ignore marking bits */
5519  }
5520 #endif
5521  }
5522 
5523  if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
5524 
5525  if (is_mark_stack_empty(mstack)) {
5526  shrink_stack_chunk_cache(mstack);
5527  return TRUE;
5528  }
5529  else {
5530  return FALSE;
5531  }
5532 }
5533 
5534 static int
5535 gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count)
5536 {
5537  return gc_mark_stacked_objects(objspace, TRUE, count);
5538 }
5539 
5540 static int
5541 gc_mark_stacked_objects_all(rb_objspace_t *objspace)
5542 {
5543  return gc_mark_stacked_objects(objspace, FALSE, 0);
5544 }
5545 
5546 #if PRINT_ROOT_TICKS
5547 #define MAX_TICKS 0x100
5548 static tick_t mark_ticks[MAX_TICKS];
5549 static const char *mark_ticks_categories[MAX_TICKS];
5550 
5551 static void
5552 show_mark_ticks(void)
5553 {
5554  int i;
5555  fprintf(stderr, "mark ticks result:\n");
5556  for (i=0; i<MAX_TICKS; i++) {
5557  const char *category = mark_ticks_categories[i];
5558  if (category) {
5559  fprintf(stderr, "%s\t%8lu\n", category, (unsigned long)mark_ticks[i]);
5560  }
5561  else {
5562  break;
5563  }
5564  }
5565 }
5566 
5567 #endif /* PRINT_ROOT_TICKS */
5568 
5569 static void
5570 gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
5571 {
5572  struct gc_list *list;
5574  rb_vm_t *vm = rb_ec_vm_ptr(ec);
5575 
5576 #if PRINT_ROOT_TICKS
5577  tick_t start_tick = tick();
5578  int tick_count = 0;
5579  const char *prev_category = 0;
5580 
5581  if (mark_ticks_categories[0] == 0) {
5582  atexit(show_mark_ticks);
5583  }
5584 #endif
5585 
5586  if (categoryp) *categoryp = "xxx";
5587 
5588 #if USE_RGENGC
5589  objspace->rgengc.parent_object = Qfalse;
5590 #endif
5591 
5592 #if PRINT_ROOT_TICKS
5593 #define MARK_CHECKPOINT_PRINT_TICK(category) do { \
5594  if (prev_category) { \
5595  tick_t t = tick(); \
5596  mark_ticks[tick_count] = t - start_tick; \
5597  mark_ticks_categories[tick_count] = prev_category; \
5598  tick_count++; \
5599  } \
5600  prev_category = category; \
5601  start_tick = tick(); \
5602 } while (0)
5603 #else /* PRINT_ROOT_TICKS */
5604 #define MARK_CHECKPOINT_PRINT_TICK(category)
5605 #endif
5606 
5607 #define MARK_CHECKPOINT(category) do { \
5608  if (categoryp) *categoryp = category; \
5609  MARK_CHECKPOINT_PRINT_TICK(category); \
5610 } while (0)
5611 
5612  MARK_CHECKPOINT("vm");
5613  SET_STACK_END;
5614  rb_vm_mark(vm);
5615  if (vm->self) gc_mark(objspace, vm->self);
5616 
5617  MARK_CHECKPOINT("finalizers");
5618  mark_finalizer_tbl(objspace, finalizer_table);
5619 
5620  MARK_CHECKPOINT("machine_context");
5621  mark_current_machine_context(objspace, ec);
5622 
5623  /* mark protected global variables */
5624  MARK_CHECKPOINT("global_list");
5625  for (list = global_list; list; list = list->next) {
5626  gc_mark_maybe(objspace, *list->varptr);
5627  }
5628 
5629  MARK_CHECKPOINT("end_proc");
5630  rb_mark_end_proc();
5631 
5632  MARK_CHECKPOINT("global_tbl");
5634 
5635  MARK_CHECKPOINT("object_id");
5636  rb_gc_mark(objspace->next_object_id);
5637  mark_tbl_no_pin(objspace, objspace->obj_to_id_tbl); /* Only mark ids */
5638 
5640 
5641  MARK_CHECKPOINT("finish");
5642 #undef MARK_CHECKPOINT
5643 }
5644 
5645 #if RGENGC_CHECK_MODE >= 4
5646 
5647 #define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
5648 #define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
5649 #define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
5650 
5651 struct reflist {
5652  VALUE *list;
5653  int pos;
5654  int size;
5655 };
5656 
5657 static struct reflist *
5658 reflist_create(VALUE obj)
5659 {
5660  struct reflist *refs = xmalloc(sizeof(struct reflist));
5661  refs->size = 1;
5662  refs->list = ALLOC_N(VALUE, refs->size);
5663  refs->list[0] = obj;
5664  refs->pos = 1;
5665  return refs;
5666 }
5667 
5668 static void
5669 reflist_destruct(struct reflist *refs)
5670 {
5671  xfree(refs->list);
5672  xfree(refs);
5673 }
5674 
5675 static void
5676 reflist_add(struct reflist *refs, VALUE obj)
5677 {
5678  if (refs->pos == refs->size) {
5679  refs->size *= 2;
5680  SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2);
5681  }
5682 
5683  refs->list[refs->pos++] = obj;
5684 }
5685 
5686 static void
5687 reflist_dump(struct reflist *refs)
5688 {
5689  int i;
5690  for (i=0; i<refs->pos; i++) {
5691  VALUE obj = refs->list[i];
5692  if (IS_ROOTSIG(obj)) { /* root */
5693  fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj));
5694  }
5695  else {
5696  fprintf(stderr, "<%s>", obj_info(obj));
5697  }
5698  if (i+1 < refs->pos) fprintf(stderr, ", ");
5699  }
5700 }
5701 
5702 static int
5703 reflist_referred_from_machine_context(struct reflist *refs)
5704 {
5705  int i;
5706  for (i=0; i<refs->pos; i++) {
5707  VALUE obj = refs->list[i];
5708  if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1;
5709  }
5710  return 0;
5711 }
5712 
5713 struct allrefs {
5714  rb_objspace_t *objspace;
5715  /* a -> obj1
5716  * b -> obj1
5717  * c -> obj1
5718  * c -> obj2
5719  * d -> obj3
5720  * #=> {obj1 => [a, b, c], obj2 => [c, d]}
5721  */
5722  struct st_table *references;
5723  const char *category;
5724  VALUE root_obj;
5726 };
5727 
5728 static int
5729 allrefs_add(struct allrefs *data, VALUE obj)
5730 {
5731  struct reflist *refs;
5732 
5733  if (st_lookup(data->references, obj, (st_data_t *)&refs)) {
5734  reflist_add(refs, data->root_obj);
5735  return 0;
5736  }
5737  else {
5738  refs = reflist_create(data->root_obj);
5739  st_insert(data->references, obj, (st_data_t)refs);
5740  return 1;
5741  }
5742 }
5743 
5744 static void
5745 allrefs_i(VALUE obj, void *ptr)
5746 {
5747  struct allrefs *data = (struct allrefs *)ptr;
5748 
5749  if (allrefs_add(data, obj)) {
5750  push_mark_stack(&data->mark_stack, obj);
5751  }
5752 }
5753 
5754 static void
5755 allrefs_roots_i(VALUE obj, void *ptr)
5756 {
5757  struct allrefs *data = (struct allrefs *)ptr;
5758  if (strlen(data->category) == 0) rb_bug("!!!");
5759  data->root_obj = MAKE_ROOTSIG(data->category);
5760 
5761  if (allrefs_add(data, obj)) {
5762  push_mark_stack(&data->mark_stack, obj);
5763  }
5764 }
5765 
5766 static st_table *
5767 objspace_allrefs(rb_objspace_t *objspace)
5768 {
5769  struct allrefs data;
5770  struct mark_func_data_struct mfd;
5771  VALUE obj;
5772  int prev_dont_gc = dont_gc;
5773  dont_gc = TRUE;
5774 
5775  data.objspace = objspace;
5776  data.references = st_init_numtable();
5777  init_mark_stack(&data.mark_stack);
5778 
5779  mfd.mark_func = allrefs_roots_i;
5780  mfd.data = &data;
5781 
5782  /* traverse root objects */
5783  PUSH_MARK_FUNC_DATA(&mfd);
5784  objspace->mark_func_data = &mfd;
5785  gc_mark_roots(objspace, &data.category);
5787 
5788  /* traverse rest objects reachable from root objects */
5789  while (pop_mark_stack(&data.mark_stack, &obj)) {
5790  rb_objspace_reachable_objects_from(data.root_obj = obj, allrefs_i, &data);
5791  }
5792  free_stack_chunks(&data.mark_stack);
5793 
5794  dont_gc = prev_dont_gc;
5795  return data.references;
5796 }
5797 
5798 static int
5799 objspace_allrefs_destruct_i(st_data_t key, st_data_t value, void *ptr)
5800 {
5801  struct reflist *refs = (struct reflist *)value;
5802  reflist_destruct(refs);
5803  return ST_CONTINUE;
5804 }
5805 
5806 static void
5807 objspace_allrefs_destruct(struct st_table *refs)
5808 {
5809  st_foreach(refs, objspace_allrefs_destruct_i, 0);
5810  st_free_table(refs);
5811 }
5812 
5813 #if RGENGC_CHECK_MODE >= 5
5814 static int
5815 allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
5816 {
5817  VALUE obj = (VALUE)k;
5818  struct reflist *refs = (struct reflist *)v;
5819  fprintf(stderr, "[allrefs_dump_i] %s <- ", obj_info(obj));
5820  reflist_dump(refs);
5821  fprintf(stderr, "\n");
5822  return ST_CONTINUE;
5823 }
5824 
5825 static void
5826 allrefs_dump(rb_objspace_t *objspace)
5827 {
5828  fprintf(stderr, "[all refs] (size: %d)\n", (int)objspace->rgengc.allrefs_table->num_entries);
5829  st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
5830 }
5831 #endif
5832 
5833 static int
5834 gc_check_after_marks_i(st_data_t k, st_data_t v, void *ptr)
5835 {
5836  VALUE obj = k;
5837  struct reflist *refs = (struct reflist *)v;
5838  rb_objspace_t *objspace = (rb_objspace_t *)ptr;
5839 
5840  /* object should be marked or oldgen */
5842  fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", obj_info(obj));
5843  fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj);
5844  reflist_dump(refs);
5845 
5846  if (reflist_referred_from_machine_context(refs)) {
5847  fprintf(stderr, " (marked from machine stack).\n");
5848  /* marked from machine context can be false positive */
5849  }
5850  else {
5851  objspace->rgengc.error_count++;
5852  fprintf(stderr, "\n");
5853  }
5854  }
5855  return ST_CONTINUE;
5856 }
5857 
5858 static void
5859 gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
5860 {
5861  size_t saved_malloc_increase = objspace->malloc_params.increase;
5862 #if RGENGC_ESTIMATE_OLDMALLOC
5863  size_t saved_oldmalloc_increase = objspace->rgengc.oldmalloc_increase;
5864 #endif
5865  VALUE already_disabled = rb_gc_disable();
5866 
5867  objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
5868 
5869  if (checker_func) {
5870  st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace);
5871  }
5872 
5873  if (objspace->rgengc.error_count > 0) {
5874 #if RGENGC_CHECK_MODE >= 5
5875  allrefs_dump(objspace);
5876 #endif
5877  if (checker_name) rb_bug("%s: GC has problem.", checker_name);
5878  }
5879 
5880  objspace_allrefs_destruct(objspace->rgengc.allrefs_table);
5881  objspace->rgengc.allrefs_table = 0;
5882 
5883  if (already_disabled == Qfalse) rb_gc_enable();
5884  objspace->malloc_params.increase = saved_malloc_increase;
5885 #if RGENGC_ESTIMATE_OLDMALLOC
5886  objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase;
5887 #endif
5888 }
5889 #endif /* RGENGC_CHECK_MODE >= 4 */
5890 
5896 
5897 #if USE_RGENGC
5901 #endif
5902 };
5903 
5904 #if USE_RGENGC
5905 static void
5906 check_generation_i(const VALUE child, void *ptr)
5907 {
5909  const VALUE parent = data->parent;
5910 
5911  if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(parent));
5912 
5913  if (!RVALUE_OLD_P(child)) {
5914  if (!RVALUE_REMEMBERED(parent) &&
5915  !RVALUE_REMEMBERED(child) &&
5916  !RVALUE_UNCOLLECTIBLE(child)) {
5917  fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", obj_info(parent), obj_info(child));
5918  data->err_count++;
5919  }
5920  }
5921 }
5922 
5923 static void
5924 check_color_i(const VALUE child, void *ptr)
5925 {
5927  const VALUE parent = data->parent;
5928 
5929  if (!RVALUE_WB_UNPROTECTED(parent) && RVALUE_WHITE_P(child)) {
5930  fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
5931  obj_info(parent), obj_info(child));
5932  data->err_count++;
5933  }
5934 }
5935 #endif
5936 
5937 static void
5938 check_children_i(const VALUE child, void *ptr)
5939 {
5941  if (check_rvalue_consistency_force(child, FALSE) != 0) {
5942  fprintf(stderr, "check_children_i: %s has error (referenced from %s)",
5943  obj_info(child), obj_info(data->parent));
5944  rb_print_backtrace(); /* C backtrace will help to debug */
5945 
5946  data->err_count++;
5947  }
5948 }
5949 
5950 static int
5951 verify_internal_consistency_i(void *page_start, void *page_end, size_t stride, void *ptr)
5952 {
5954  VALUE obj;
5955  rb_objspace_t *objspace = data->objspace;
5956 
5957  for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) {
5958  void *poisoned = asan_poisoned_object_p(obj);
5959  asan_unpoison_object(obj, false);
5960 
5961  if (is_live_object(objspace, obj)) {
5962  /* count objects */
5963  data->live_object_count++;
5964  data->parent = obj;
5965 
5966  /* Normally, we don't expect T_MOVED objects to be in the heap.
5967  * But they can stay alive on the stack, */
5968  if (!gc_object_moved_p(objspace, obj)) {
5969  /* moved slots don't have children */
5970  rb_objspace_reachable_objects_from(obj, check_children_i, (void *)data);
5971  }
5972 
5973 #if USE_RGENGC
5974  /* check health of children */
5975  if (RVALUE_OLD_P(obj)) data->old_object_count++;
5976  if (RVALUE_WB_UNPROTECTED(obj) && RVALUE_UNCOLLECTIBLE(obj)) data->remembered_shady_count++;
5977 
5978  if (!is_marking(objspace) && RVALUE_OLD_P(obj)) {
5979  /* reachable objects from an oldgen object should be old or (young with remember) */
5980  data->parent = obj;
5981  rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data);
5982  }
5983 
5985  if (RVALUE_BLACK_P(obj)) {
5986  /* reachable objects from black objects should be black or grey objects */
5987  data->parent = obj;
5988  rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data);
5989  }
5990  }
5991 #endif
5992  }
5993  else {
5994  if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
5995  GC_ASSERT((RBASIC(obj)->flags & ~FL_SEEN_OBJ_ID) == T_ZOMBIE);
5996  data->zombie_object_count++;
5997  }
5998  }
5999  if (poisoned) {
6001  asan_poison_object(obj);
6002  }
6003  }
6004 
6005  return 0;
6006 }
6007 
6008 static int
6009 gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
6010 {
6011 #if USE_RGENGC
6012  int i;
6013  unsigned int has_remembered_shady = FALSE;
6014  unsigned int has_remembered_old = FALSE;
6015  int remembered_old_objects = 0;
6016  int free_objects = 0;
6017  int zombie_objects = 0;
6018 
6019  for (i=0; i<page->total_slots; i++) {
6020  VALUE val = (VALUE)&page->start[i];
6021  void *poisoned = asan_poisoned_object_p(val);
6022  asan_unpoison_object(val, false);
6023 
6024  if (RBASIC(val) == 0) free_objects++;
6025  if (BUILTIN_TYPE(val) == T_ZOMBIE) zombie_objects++;
6026  if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
6027  has_remembered_shady = TRUE;
6028  }
6029  if (RVALUE_PAGE_MARKING(page, val)) {
6030  has_remembered_old = TRUE;
6031  remembered_old_objects++;
6032  }
6033 
6034  if (poisoned) {
6035  GC_ASSERT(BUILTIN_TYPE(val) == T_NONE);
6036  asan_poison_object(val);
6037  }
6038  }
6039 
6041  page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
6042 
6043  for (i=0; i<page->total_slots; i++) {
6044  VALUE val = (VALUE)&page->start[i];
6045  if (RVALUE_PAGE_MARKING(page, val)) {
6046  fprintf(stderr, "marking -> %s\n", obj_info(val));
6047  }
6048  }
6049  rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
6050  (void *)page, remembered_old_objects, obj ? obj_info(obj) : "");
6051  }
6052 
6053  if (page->flags.has_uncollectible_shady_objects == FALSE && has_remembered_shady == TRUE) {
6054  rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
6055  (void *)page, obj ? obj_info(obj) : "");
6056  }
6057 
6058  if (0) {
6059  /* free_slots may not equal to free_objects */
6060  if (page->free_slots != free_objects) {
6061  rb_bug("page %p's free_slots should be %d, but %d\n", (void *)page, (int)page->free_slots, free_objects);
6062  }
6063  }
6064  if (page->final_slots != zombie_objects) {
6065  rb_bug("page %p's final_slots should be %d, but %d\n", (void *)page, (int)page->final_slots, zombie_objects);
6066  }
6067 
6068  return remembered_old_objects;
6069 #else
6070  return 0;
6071 #endif
6072 }
6073 
6074 static int
6075 gc_verify_heap_pages_(rb_objspace_t *objspace, struct list_head *head)
6076 {
6077  int remembered_old_objects = 0;
6078  struct heap_page *page = 0;
6079 
6080  list_for_each(head, page, page_node) {
6081  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
6082  RVALUE *p = page->freelist;
6083  while (p) {
6084  RVALUE *prev = p;
6085  asan_unpoison_object((VALUE)p, false);
6086  if (BUILTIN_TYPE(p) != T_NONE) {
6087  fprintf(stderr, "freelist slot expected to be T_NONE but was: %s\n", obj_info((VALUE)p));
6088  }
6089  p = p->as.free.next;
6090  asan_poison_object((VALUE)prev);
6091  }
6092  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
6093 
6094  if (page->flags.has_remembered_objects == FALSE) {
6095  remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse);
6096  }
6097  }
6098 
6099  return remembered_old_objects;
6100 }
6101 
6102 static int
6103 gc_verify_heap_pages(rb_objspace_t *objspace)
6104 {
6105  int remembered_old_objects = 0;
6106  remembered_old_objects += gc_verify_heap_pages_(objspace, &heap_eden->pages);
6107  remembered_old_objects += gc_verify_heap_pages_(objspace, &heap_tomb->pages);
6108  return remembered_old_objects;
6109 }
6110 
6111 /*
6112  * call-seq:
6113  * GC.verify_internal_consistency -> nil
6114  *
6115  * Verify internal consistency.
6116  *
6117  * This method is implementation specific.
6118  * Now this method checks generational consistency
6119  * if RGenGC is supported.
6120  */
6121 static VALUE
6122 gc_verify_internal_consistency_m(VALUE dummy)
6123 {
6124  gc_verify_internal_consistency(&rb_objspace);
6125 
6126  return Qnil;
6127 }
6128 
6129 static void
6130 gc_verify_internal_consistency(rb_objspace_t *objspace)
6131 {
6132  struct verify_internal_consistency_struct data = {0};
6133 
6134  data.objspace = objspace;
6135  gc_report(5, objspace, "gc_verify_internal_consistency: start\n");
6136 
6137  /* check relations */
6138 
6139  objspace_each_objects_without_setup(objspace, verify_internal_consistency_i, &data);
6140 
6141  if (data.err_count != 0) {
6142 #if RGENGC_CHECK_MODE >= 5
6143  objspace->rgengc.error_count = data.err_count;
6144  gc_marks_check(objspace, NULL, NULL);
6145  allrefs_dump(objspace);
6146 #endif
6147  rb_bug("gc_verify_internal_consistency: found internal inconsistency.");
6148  }
6149 
6150  /* check heap_page status */
6151  gc_verify_heap_pages(objspace);
6152 
6153  /* check counters */
6154 
6156  if (objspace_live_slots(objspace) != data.live_object_count) {
6157  fprintf(stderr, "heap_pages_final_slots: %d, objspace->profile.total_freed_objects: %d\n",
6159  rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".", objspace_live_slots(objspace), data.live_object_count);
6160  }
6161  }
6162 
6163 #if USE_RGENGC
6164  if (!is_marking(objspace)) {
6166  rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".", objspace->rgengc.old_objects, data.old_object_count);
6167  }
6169  rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".", objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
6170  }
6171  }
6172 #endif
6173 
6174  if (!finalizing) {
6175  size_t list_count = 0;
6176 
6177  {
6179  while (z) {
6180  list_count++;
6181  z = RZOMBIE(z)->next;
6182  }
6183  }
6184 
6186  heap_pages_final_slots != list_count) {
6187 
6188  rb_bug("inconsistent finalizing object count:\n"
6189  " expect %"PRIuSIZE"\n"
6190  " but %"PRIuSIZE" zombies\n"
6191  " heap_pages_deferred_final list has %"PRIuSIZE" items.",
6193  data.zombie_object_count,
6194  list_count);
6195  }
6196  }
6197 
6198  gc_report(5, objspace, "gc_verify_internal_consistency: OK\n");
6199 }
6200 
6201 void
6203 {
6204  gc_verify_internal_consistency(&rb_objspace);
6205 }
6206 
6207 static VALUE
6208 gc_verify_transient_heap_internal_consistency(VALUE dmy)
6209 {
6211  return Qnil;
6212 }
6213 
6214 /* marks */
6215 
6216 static void
6217 gc_marks_start(rb_objspace_t *objspace, int full_mark)
6218 {
6219  /* start marking */
6220  gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor");
6221  gc_mode_transition(objspace, gc_mode_marking);
6222 
6223 #if USE_RGENGC
6224  if (full_mark) {
6225 #if GC_ENABLE_INCREMENTAL_MARK
6227 
6228  if (0) fprintf(stderr, "objspace->marked_slots: %d, objspace->rincgc.pooled_page_num: %d, objspace->rincgc.step_slots: %d, \n",
6230 #endif
6236  objspace->marked_slots = 0;
6237  rgengc_mark_and_rememberset_clear(objspace, heap_eden);
6238  }
6239  else {
6242  objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */
6244  rgengc_rememberset_mark(objspace, heap_eden);
6245  }
6246 #endif
6247 
6248  gc_mark_roots(objspace, NULL);
6249 
6250  gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %d\n", full_mark ? "full" : "minor", (int)mark_stack_size(&objspace->mark_stack));
6251 }
6252 
6253 #if GC_ENABLE_INCREMENTAL_MARK
6254 static void
6255 gc_marks_wb_unprotected_objects(rb_objspace_t *objspace)
6256 {
6257  struct heap_page *page = 0;
6258 
6259  list_for_each(&heap_eden->pages, page, page_node) {
6260  bits_t *mark_bits = page->mark_bits;
6261  bits_t *wbun_bits = page->wb_unprotected_bits;
6262  RVALUE *p = page->start;
6263  RVALUE *offset = p - NUM_IN_PAGE(p);
6264  size_t j;
6265 
6266  for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
6267  bits_t bits = mark_bits[j] & wbun_bits[j];
6268 
6269  if (bits) {
6270  p = offset + j * BITS_BITLENGTH;
6271 
6272  do {
6273  if (bits & 1) {
6274  gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", obj_info((VALUE)p));
6275  GC_ASSERT(RVALUE_WB_UNPROTECTED((VALUE)p));
6276  GC_ASSERT(RVALUE_MARKED((VALUE)p));
6277  gc_mark_children(objspace, (VALUE)p);
6278  }
6279  p++;
6280  bits >>= 1;
6281  } while (bits);
6282  }
6283  }
6284  }
6285 
6286  gc_mark_stacked_objects_all(objspace);
6287 }
6288 
6289 static struct heap_page *
6290 heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
6291 {
6292  struct heap_page *page = heap->pooled_pages;
6293 
6294  if (page) {
6295  heap->pooled_pages = page->free_next;
6296  heap_add_freepage(heap, page);
6297  }
6298 
6299  return page;
6300 }
6301 #endif
6302 
6303 static int
6304 gc_marks_finish(rb_objspace_t *objspace)
6305 {
6306 #if GC_ENABLE_INCREMENTAL_MARK
6307  /* finish incremental GC */
6308  if (is_incremental_marking(objspace)) {
6309  if (heap_eden->pooled_pages) {
6310  heap_move_pooled_pages_to_free_pages(heap_eden);
6311  gc_report(1, objspace, "gc_marks_finish: pooled pages are exists. retry.\n");
6312  return FALSE; /* continue marking phase */
6313  }
6314 
6315  if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) {
6316  rb_bug("gc_marks_finish: mark stack is not empty (%d).", (int)mark_stack_size(&objspace->mark_stack));
6317  }
6318 
6319  gc_mark_roots(objspace, 0);
6320 
6321  if (is_mark_stack_empty(&objspace->mark_stack) == FALSE) {
6322  gc_report(1, objspace, "gc_marks_finish: not empty (%d). retry.\n", (int)mark_stack_size(&objspace->mark_stack));
6323  return FALSE;
6324  }
6325 
6326 #if RGENGC_CHECK_MODE >= 2
6327  if (gc_verify_heap_pages(objspace) != 0) {
6328  rb_bug("gc_marks_finish (incremental): there are remembered old objects.");
6329  }
6330 #endif
6331 
6333  /* check children of all marked wb-unprotected objects */
6334  gc_marks_wb_unprotected_objects(objspace);
6335  }
6336 #endif /* GC_ENABLE_INCREMENTAL_MARK */
6337 
6338 #if RGENGC_CHECK_MODE >= 2
6339  gc_verify_internal_consistency(objspace);
6340 #endif
6341 
6342 #if USE_RGENGC
6343  if (is_full_marking(objspace)) {
6344  /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
6345  const double r = gc_params.oldobject_limit_factor;
6347  objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
6348  }
6349 #endif
6350 
6351 #if RGENGC_CHECK_MODE >= 4
6352  gc_marks_check(objspace, gc_check_after_marks_i, "after_marks");
6353 #endif
6354 
6355  {
6356  /* decide full GC is needed or not */
6357  rb_heap_t *heap = heap_eden;
6359  size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */
6360  size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
6361  size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
6362  int full_marking = is_full_marking(objspace);
6363 
6364  GC_ASSERT(heap->total_slots >= objspace->marked_slots);
6365 
6366  /* setup free-able page counts */
6367  if (max_free_slots < gc_params.heap_init_slots) max_free_slots = gc_params.heap_init_slots;
6368 
6369  if (sweep_slots > max_free_slots) {
6370  heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
6371  }
6372  else {
6374  }
6375 
6376  /* check free_min */
6377  if (min_free_slots < gc_params.heap_free_slots) min_free_slots = gc_params.heap_free_slots;
6378 
6379 #if USE_RGENGC
6380  if (sweep_slots < min_free_slots) {
6381  if (!full_marking) {
6382  if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
6383  full_marking = TRUE;
6384  /* do not update last_major_gc, because full marking is not done. */
6385  goto increment;
6386  }
6387  else {
6388  gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n");
6390  }
6391  }
6392  else {
6393  increment:
6394  gc_report(1, objspace, "gc_marks_finish: heap_set_increment!!\n");
6395  heap_set_increment(objspace, heap_extend_pages(objspace, sweep_slots, total_slots));
6396  heap_increment(objspace, heap);
6397  }
6398  }
6399 
6400  if (full_marking) {
6401  /* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
6402  const double r = gc_params.oldobject_limit_factor;
6404  objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
6405  }
6406 
6409  }
6410  if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) {
6412  }
6413  if (RGENGC_FORCE_MAJOR_GC) {
6415  }
6416 
6417  gc_report(1, objspace, "gc_marks_finish (marks %d objects, old %d objects, total %d slots, sweep %d slots, increment: %d, next GC: %s)\n",
6418  (int)objspace->marked_slots, (int)objspace->rgengc.old_objects, (int)heap->total_slots, (int)sweep_slots, (int)heap_allocatable_pages,
6419  objspace->rgengc.need_major_gc ? "major" : "minor");
6420 #else /* USE_RGENGC */
6421  if (sweep_slots < min_free_slots) {
6422  gc_report(1, objspace, "gc_marks_finish: heap_set_increment!!\n");
6423  heap_set_increment(objspace, heap_extend_pages(objspace, sweep_slot, total_slot));
6424  heap_increment(objspace, heap);
6425  }
6426 #endif
6427  }
6428 
6430 
6432 
6433  return TRUE;
6434 }
6435 
6436 static void
6437 gc_marks_step(rb_objspace_t *objspace, int slots)
6438 {
6439 #if GC_ENABLE_INCREMENTAL_MARK
6440  GC_ASSERT(is_marking(objspace));
6441 
6442  if (gc_mark_stacked_objects_incremental(objspace, slots)) {
6443  if (gc_marks_finish(objspace)) {
6444  /* finish */
6445  gc_sweep(objspace);
6446  }
6447  }
6448  if (0) fprintf(stderr, "objspace->marked_slots: %d\n", (int)objspace->marked_slots);
6449 #endif
6450 }
6451 
6452 static void
6453 gc_marks_rest(rb_objspace_t *objspace)
6454 {
6455  gc_report(1, objspace, "gc_marks_rest\n");
6456 
6457 #if GC_ENABLE_INCREMENTAL_MARK
6458  heap_eden->pooled_pages = NULL;
6459 #endif
6460 
6461  if (is_incremental_marking(objspace)) {
6462  do {
6463  while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE);
6464  } while (gc_marks_finish(objspace) == FALSE);
6465  }
6466  else {
6467  gc_mark_stacked_objects_all(objspace);
6468  gc_marks_finish(objspace);
6469  }
6470 
6471  /* move to sweep */
6472  gc_sweep(objspace);
6473 }
6474 
6475 static void
6476 gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap)
6477 {
6478  GC_ASSERT(dont_gc == FALSE);
6479 #if GC_ENABLE_INCREMENTAL_MARK
6480 
6481  gc_enter(objspace, "marks_continue");
6482 
6484  {
6485  int slots = 0;
6486  const char *from;
6487 
6488  if (heap->pooled_pages) {
6489  while (heap->pooled_pages && slots < HEAP_PAGE_OBJ_LIMIT) {
6490  struct heap_page *page = heap_move_pooled_pages_to_free_pages(heap);
6491  slots += page->free_slots;
6492  }
6493  from = "pooled-pages";
6494  }
6495  else if (heap_increment(objspace, heap)) {
6496  slots = heap->free_pages->free_slots;
6497  from = "incremented-pages";
6498  }
6499 
6500  if (slots > 0) {
6501  gc_report(2, objspace, "gc_marks_continue: provide %d slots from %s.\n", slots, from);
6502  gc_marks_step(objspace, (int)objspace->rincgc.step_slots);
6503  }
6504  else {
6505  gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %d).\n", (int)mark_stack_size(&objspace->mark_stack));
6506  gc_marks_rest(objspace);
6507  }
6508  }
6510 
6511  gc_exit(objspace, "marks_continue");
6512 #endif
6513 }
6514 
6515 static void
6516 gc_marks(rb_objspace_t *objspace, int full_mark)
6517 {
6518  gc_prof_mark_timer_start(objspace);
6519 
6521  {
6522  /* setup marking */
6523 
6524 #if USE_RGENGC
6525  gc_marks_start(objspace, full_mark);
6526  if (!is_incremental_marking(objspace)) {
6527  gc_marks_rest(objspace);
6528  }
6529 
6530 #if RGENGC_PROFILE > 0
6531  if (gc_prof_record(objspace)) {
6532  gc_profile_record *record = gc_prof_record(objspace);
6533  record->old_objects = objspace->rgengc.old_objects;
6534  }
6535 #endif
6536 
6537 #else /* USE_RGENGC */
6538  gc_marks_start(objspace, TRUE);
6539  gc_marks_rest(objspace);
6540 #endif
6541  }
6543  gc_prof_mark_timer_stop(objspace);
6544 }
6545 
6546 /* RGENGC */
6547 
6548 static void
6549 gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...)
6550 {
6551  if (level <= RGENGC_DEBUG) {
6552  char buf[1024];
6553  FILE *out = stderr;
6554  va_list args;
6555  const char *status = " ";
6556 
6557 #if USE_RGENGC
6558  if (during_gc) {
6559  status = is_full_marking(objspace) ? "+" : "-";
6560  }
6561  else {
6562  if (is_lazy_sweeping(heap_eden)) {
6563  status = "S";
6564  }
6565  if (is_incremental_marking(objspace)) {
6566  status = "M";
6567  }
6568  }
6569 #endif
6570 
6571  va_start(args, fmt);
6572  vsnprintf(buf, 1024, fmt, args);
6573  va_end(args);
6574 
6575  fprintf(out, "%s|", status);
6576  fputs(buf, out);
6577  }
6578 }
6579 
6580 #if USE_RGENGC
6581 
6582 /* bit operations */
6583 
6584 static int
6585 rgengc_remembersetbits_get(rb_objspace_t *objspace, VALUE obj)
6586 {
6587  return RVALUE_REMEMBERED(obj);
6588 }
6589 
6590 static int
6591 rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj)
6592 {
6593  struct heap_page *page = GET_HEAP_PAGE(obj);
6594  bits_t *bits = &page->marking_bits[0];
6595 
6596  GC_ASSERT(!is_incremental_marking(objspace));
6597 
6598  if (MARKED_IN_BITMAP(bits, obj)) {
6599  return FALSE;
6600  }
6601  else {
6603  MARK_IN_BITMAP(bits, obj);
6604  return TRUE;
6605  }
6606 }
6607 
6608 /* wb, etc */
6609 
6610 /* return FALSE if already remembered */
6611 static int
6612 rgengc_remember(rb_objspace_t *objspace, VALUE obj)
6613 {
6614  gc_report(6, objspace, "rgengc_remember: %s %s\n", obj_info(obj),
6615  rgengc_remembersetbits_get(objspace, obj) ? "was already remembered" : "is remembered now");
6616 
6617  check_rvalue_consistency(obj);
6618 
6619  if (RGENGC_CHECK_MODE) {
6620  if (RVALUE_WB_UNPROTECTED(obj)) rb_bug("rgengc_remember: %s is not wb protected.", obj_info(obj));
6621  }
6622 
6623 #if RGENGC_PROFILE > 0
6624  if (!rgengc_remembered(objspace, obj)) {
6625  if (RVALUE_WB_UNPROTECTED(obj) == 0) {
6626  objspace->profile.total_remembered_normal_object_count++;
6627 #if RGENGC_PROFILE >= 2
6628  objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++;
6629 #endif
6630  }
6631  }
6632 #endif /* RGENGC_PROFILE > 0 */
6633 
6634  return rgengc_remembersetbits_set(objspace, obj);
6635 }
6636 
6637 static int
6638 rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj)
6639 {
6640  int result = rgengc_remembersetbits_get(objspace, obj);
6641  check_rvalue_consistency(obj);
6642  return result;
6643 }
6644 
6645 static int
6646 rgengc_remembered(rb_objspace_t *objspace, VALUE obj)
6647 {
6648  gc_report(6, objspace, "rgengc_remembered: %s\n", obj_info(obj));
6649  return rgengc_remembered_sweep(objspace, obj);
6650 }
6651 
6652 #ifndef PROFILE_REMEMBERSET_MARK
6653 #define PROFILE_REMEMBERSET_MARK 0
6654 #endif
6655 
6656 static void
6657 rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
6658 {
6659  size_t j;
6660  struct heap_page *page = 0;
6661 #if PROFILE_REMEMBERSET_MARK
6662  int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
6663 #endif
6664  gc_report(1, objspace, "rgengc_rememberset_mark: start\n");
6665 
6666  list_for_each(&heap->pages, page, page_node) {
6668  RVALUE *p = page->start;
6669  RVALUE *offset = p - NUM_IN_PAGE(p);
6670  bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
6671  bits_t *marking_bits = page->marking_bits;
6674 #if PROFILE_REMEMBERSET_MARK
6675  if (page->flags.has_remembered_objects && page->flags.has_uncollectible_shady_objects) has_both++;
6676  else if (page->flags.has_remembered_objects) has_old++;
6677  else if (page->flags.has_uncollectible_shady_objects) has_shady++;
6678 #endif
6679  for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
6680  bits[j] = marking_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]);
6681  marking_bits[j] = 0;
6682  }
6684 
6685  for (j=0; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
6686  bitset = bits[j];
6687 
6688  if (bitset) {
6689  p = offset + j * BITS_BITLENGTH;
6690 
6691  do {
6692  if (bitset & 1) {
6693  VALUE obj = (VALUE)p;
6694  gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", obj_info(obj));
6695  GC_ASSERT(RVALUE_UNCOLLECTIBLE(obj));
6696  GC_ASSERT(RVALUE_OLD_P(obj) || RVALUE_WB_UNPROTECTED(obj));
6697 
6698  gc_mark_children(objspace, obj);
6699  }
6700  p++;
6701  bitset >>= 1;
6702  } while (bitset);
6703  }
6704  }
6705  }
6706 #if PROFILE_REMEMBERSET_MARK
6707  else {
6708  skip++;
6709  }
6710 #endif
6711  }
6712 
6713 #if PROFILE_REMEMBERSET_MARK
6714  fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
6715 #endif
6716  gc_report(1, objspace, "rgengc_rememberset_mark: finished\n");
6717 }
6718 
6719 static void
6720 rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap)
6721 {
6722  struct heap_page *page = 0;
6723 
6724  list_for_each(&heap->pages, page, page_node) {
6725  memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6727  memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6728  memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
6731  }
6732 }
6733 
6734 /* RGENGC: APIs */
6735 
6736 NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace));
6737 
6738 static void
6739 gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
6740 {
6741  if (RGENGC_CHECK_MODE) {
6742  if (!RVALUE_OLD_P(a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", obj_info(a));
6743  if ( RVALUE_OLD_P(b)) rb_bug("gc_writebarrier_generational: %s is an old object.", obj_info(b));
6744  if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", obj_info(a), obj_info(b));
6745  }
6746 
6747 #if 1
6748  /* mark `a' and remember (default behavior) */
6749  if (!rgengc_remembered(objspace, a)) {
6750  rgengc_remember(objspace, a);
6751  gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", obj_info(a), obj_info(b));
6752  }
6753 #else
6754  /* mark `b' and remember */
6756  if (RVALUE_WB_UNPROTECTED(b)) {
6757  gc_remember_unprotected(objspace, b);
6758  }
6759  else {
6760  RVALUE_AGE_SET_OLD(objspace, b);
6761  rgengc_remember(objspace, b);
6762  }
6763 
6764  gc_report(1, objspace, "gc_writebarrier_generational: %s -> %s (remembered)\n", obj_info(a), obj_info(b));
6765 #endif
6766 
6767  check_rvalue_consistency(a);
6768  check_rvalue_consistency(b);
6769 }
6770 
6771 #if GC_ENABLE_INCREMENTAL_MARK
6772 static void
6773 gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
6774 {
6775  gc_mark_set_parent(objspace, parent);
6776  rgengc_check_relation(objspace, obj);
6777  if (gc_mark_set(objspace, obj) == FALSE) return;
6778  gc_aging(objspace, obj);
6779  gc_grey(objspace, obj);
6780 }
6781 
6782 NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
6783 
6784 static void
6785 gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
6786 {
6787  gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, obj_info(b));
6788 
6789  if (RVALUE_BLACK_P(a)) {
6790  if (RVALUE_WHITE_P(b)) {
6791  if (!RVALUE_WB_UNPROTECTED(a)) {
6792  gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, obj_info(b));
6793  gc_mark_from(objspace, b, a);
6794  }
6795  }
6796  else if (RVALUE_OLD_P(a) && !RVALUE_OLD_P(b)) {
6797  if (!RVALUE_WB_UNPROTECTED(b)) {
6798  gc_report(1, objspace, "gc_writebarrier_incremental: [GN] %p -> %s\n", (void *)a, obj_info(b));
6799  RVALUE_AGE_SET_OLD(objspace, b);
6800 
6801  if (RVALUE_BLACK_P(b)) {
6802  gc_grey(objspace, b);
6803  }
6804  }
6805  else {
6806  gc_report(1, objspace, "gc_writebarrier_incremental: [LL] %p -> %s\n", (void *)a, obj_info(b));
6807  gc_remember_unprotected(objspace, b);
6808  }
6809  }
6810  }
6811 }
6812 #else
6813 #define gc_writebarrier_incremental(a, b, objspace)
6814 #endif
6815 
6816 void
6818 {
6819  rb_objspace_t *objspace = &rb_objspace;
6820 
6821  if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const");
6822  if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const");
6823 
6824  if (!is_incremental_marking(objspace)) {
6825  if (!RVALUE_OLD_P(a) || RVALUE_OLD_P(b)) {
6826  return;
6827  }
6828  else {
6829  gc_writebarrier_generational(a, b, objspace);
6830  }
6831  }
6832  else { /* slow path */
6833  gc_writebarrier_incremental(a, b, objspace);
6834  }
6835 }
6836 
6837 void
6839 {
6840  if (RVALUE_WB_UNPROTECTED(obj)) {
6841  return;
6842  }
6843  else {
6844  rb_objspace_t *objspace = &rb_objspace;
6845 
6846  gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", obj_info(obj),
6847  rgengc_remembered(objspace, obj) ? " (already remembered)" : "");
6848 
6849  if (RVALUE_OLD_P(obj)) {
6850  gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", obj_info(obj));
6851  RVALUE_DEMOTE(objspace, obj);
6852  gc_mark_set(objspace, obj);
6853  gc_remember_unprotected(objspace, obj);
6854 
6855 #if RGENGC_PROFILE
6856  objspace->profile.total_shade_operation_count++;
6857 #if RGENGC_PROFILE >= 2
6858  objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++;
6859 #endif /* RGENGC_PROFILE >= 2 */
6860 #endif /* RGENGC_PROFILE */
6861  }
6862  else {
6863  RVALUE_AGE_RESET(obj);
6864  }
6865 
6866  RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
6868  }
6869 }
6870 
6871 /*
6872  * remember `obj' if needed.
6873  */
6874 MJIT_FUNC_EXPORTED void
6876 {
6877  rb_objspace_t *objspace = &rb_objspace;
6878 
6879  gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", obj_info(obj));
6880 
6881  if (is_incremental_marking(objspace)) {
6882  if (RVALUE_BLACK_P(obj)) {
6883  gc_grey(objspace, obj);
6884  }
6885  }
6886  else {
6887  if (RVALUE_OLD_P(obj)) {
6888  rgengc_remember(objspace, obj);
6889  }
6890  }
6891 }
6892 
6893 static st_table *rgengc_unprotect_logging_table;
6894 
6895 static int
6896 rgengc_unprotect_logging_exit_func_i(st_data_t key, st_data_t val, st_data_t arg)
6897 {
6898  fprintf(stderr, "%s\t%d\n", (char *)key, (int)val);
6899  return ST_CONTINUE;
6900 }
6901 
6902 static void
6903 rgengc_unprotect_logging_exit_func(void)
6904 {
6905  st_foreach(rgengc_unprotect_logging_table, rgengc_unprotect_logging_exit_func_i, 0);
6906 }
6907 
6908 void
6909 rb_gc_unprotect_logging(void *objptr, const char *filename, int line)
6910 {
6911  VALUE obj = (VALUE)objptr;
6912 
6913  if (rgengc_unprotect_logging_table == 0) {
6914  rgengc_unprotect_logging_table = st_init_strtable();
6915  atexit(rgengc_unprotect_logging_exit_func);
6916  }
6917 
6918  if (RVALUE_WB_UNPROTECTED(obj) == 0) {
6919  char buff[0x100];
6920  st_data_t cnt = 1;
6921  char *ptr = buff;
6922 
6923  snprintf(ptr, 0x100 - 1, "%s|%s:%d", obj_info(obj), filename, line);
6924 
6925  if (st_lookup(rgengc_unprotect_logging_table, (st_data_t)ptr, &cnt)) {
6926  cnt++;
6927  }
6928  else {
6929  ptr = (strdup)(buff);
6930  if (!ptr) rb_memerror();
6931  }
6932  st_insert(rgengc_unprotect_logging_table, (st_data_t)ptr, cnt);
6933  }
6934 }
6935 #endif /* USE_RGENGC */
6936 
6937 void
6939 {
6940 #if USE_RGENGC
6941  rb_objspace_t *objspace = &rb_objspace;
6942 
6943  if (RVALUE_WB_UNPROTECTED(obj) && !RVALUE_WB_UNPROTECTED(dest)) {
6944  if (!RVALUE_OLD_P(dest)) {
6946  RVALUE_AGE_RESET_RAW(dest);
6947  }
6948  else {
6949  RVALUE_DEMOTE(objspace, dest);
6950  }
6951  }
6952 
6953  check_rvalue_consistency(dest);
6954 #endif
6955 }
6956 
6957 /* RGENGC analysis information */
6958 
6959 VALUE
6961 {
6962 #if USE_RGENGC
6963  return RVALUE_WB_UNPROTECTED(obj) ? Qfalse : Qtrue;
6964 #else
6965  return Qfalse;
6966 #endif
6967 }
6968 
6969 VALUE
6971 {
6972  return OBJ_PROMOTED(obj) ? Qtrue : Qfalse;
6973 }
6974 
6975 size_t
6977 {
6978  size_t n = 0;
6979  static ID ID_marked;
6980 #if USE_RGENGC
6981  static ID ID_wb_protected, ID_old, ID_marking, ID_uncollectible, ID_pinned;
6982 #endif
6983 
6984  if (!ID_marked) {
6985 #define I(s) ID_##s = rb_intern(#s);
6986  I(marked);
6987 #if USE_RGENGC
6988  I(wb_protected);
6989  I(old);
6990  I(marking);
6991  I(uncollectible);
6992  I(pinned);
6993 #endif
6994 #undef I
6995  }
6996 
6997 #if USE_RGENGC
6998  if (RVALUE_WB_UNPROTECTED(obj) == 0 && n<max) flags[n++] = ID_wb_protected;
6999  if (RVALUE_OLD_P(obj) && n<max) flags[n++] = ID_old;
7000  if (RVALUE_UNCOLLECTIBLE(obj) && n<max) flags[n++] = ID_uncollectible;
7001  if (MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj) && n<max) flags[n++] = ID_marking;
7002 #endif
7003  if (MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) && n<max) flags[n++] = ID_marked;
7004  if (MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) && n<max) flags[n++] = ID_pinned;
7005  return n;
7006 }
7007 
7008 /* GC */
7009 
7010 void
7012 {
7013  rb_objspace_t *objspace = &rb_objspace;
7014 
7015 #if USE_RGENGC
7016  int is_old = RVALUE_OLD_P(obj);
7017 
7018  gc_report(2, objspace, "rb_gc_force_recycle: %s\n", obj_info(obj));
7019 
7020  if (is_old) {
7021  if (RVALUE_MARKED(obj)) {
7022  objspace->rgengc.old_objects--;
7023  }
7024  }
7027 
7028 #if GC_ENABLE_INCREMENTAL_MARK
7029  if (is_incremental_marking(objspace)) {
7031  invalidate_mark_stack(&objspace->mark_stack, obj);
7033  }
7035  }
7036  else {
7037 #endif
7038  if (is_old || !GET_HEAP_PAGE(obj)->flags.before_sweep) {
7040  }
7042 #if GC_ENABLE_INCREMENTAL_MARK
7043  }
7044 #endif
7045 #endif
7046 
7047  objspace->profile.total_freed_objects++;
7048 
7049  heap_page_add_freeobj(objspace, GET_HEAP_PAGE(obj), obj);
7050 
7051  /* Disable counting swept_slots because there are no meaning.
7052  * if (!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(p), p)) {
7053  * objspace->heap.swept_slots++;
7054  * }
7055  */
7056 }
7057 
7058 #ifndef MARK_OBJECT_ARY_BUCKET_SIZE
7059 #define MARK_OBJECT_ARY_BUCKET_SIZE 1024
7060 #endif
7061 
7062 void
7064 {
7065  VALUE ary_ary = GET_VM()->mark_object_ary;
7066  VALUE ary = rb_ary_last(0, 0, ary_ary);
7067 
7068  if (ary == Qnil || RARRAY_LEN(ary) >= MARK_OBJECT_ARY_BUCKET_SIZE) {
7070  rb_ary_push(ary_ary, ary);
7071  }
7072 
7073  rb_ary_push(ary, obj);
7074 }
7075 
7076 void
7078 {
7079  rb_objspace_t *objspace = &rb_objspace;
7080  struct gc_list *tmp;
7081 
7082  tmp = ALLOC(struct gc_list);
7083  tmp->next = global_list;
7084  tmp->varptr = addr;
7085  global_list = tmp;
7086 }
7087 
7088 void
7090 {
7091  rb_objspace_t *objspace = &rb_objspace;
7092  struct gc_list *tmp = global_list;
7093 
7094  if (tmp->varptr == addr) {
7095  global_list = tmp->next;
7096  xfree(tmp);
7097  return;
7098  }
7099  while (tmp->next) {
7100  if (tmp->next->varptr == addr) {
7101  struct gc_list *t = tmp->next;
7102 
7103  tmp->next = tmp->next->next;
7104  xfree(t);
7105  break;
7106  }
7107  tmp = tmp->next;
7108  }
7109 }
7110 
7111 void
7113 {
7115 }
7116 
7117 #define GC_NOTIFY 0
7118 
7119 enum {
7124 };
7125 
7126 #define gc_stress_full_mark_after_malloc_p() \
7127  (FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
7128 
7129 static void
7130 heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap)
7131 {
7132  if (!heap->freelist && !heap->free_pages) {
7133  if (!heap_increment(objspace, heap)) {
7134  heap_set_increment(objspace, 1);
7135  heap_increment(objspace, heap);
7136  }
7137  }
7138 }
7139 
7140 static int
7141 ready_to_gc(rb_objspace_t *objspace)
7142 {
7143  if (dont_gc || during_gc || ruby_disable_gc) {
7144  heap_ready_to_gc(objspace, heap_eden);
7145  return FALSE;
7146  }
7147  else {
7148  return TRUE;
7149  }
7150 }
7151 
7152 static void
7153 gc_reset_malloc_info(rb_objspace_t *objspace)
7154 {
7155  gc_prof_set_malloc_info(objspace);
7156  {
7157  size_t inc = ATOMIC_SIZE_EXCHANGE(malloc_increase, 0);
7158  size_t old_limit = malloc_limit;
7159 
7160  if (inc > malloc_limit) {
7161  malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
7162  if (malloc_limit > gc_params.malloc_limit_max) {
7163  malloc_limit = gc_params.malloc_limit_max;
7164  }
7165  }
7166  else {
7167  malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */
7168  if (malloc_limit < gc_params.malloc_limit_min) {
7169  malloc_limit = gc_params.malloc_limit_min;
7170  }
7171  }
7172 
7173  if (0) {
7174  if (old_limit != malloc_limit) {
7175  fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n",
7176  rb_gc_count(), old_limit, malloc_limit);
7177  }
7178  else {
7179  fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n",
7181  }
7182  }
7183  }
7184 
7185  /* reset oldmalloc info */
7186 #if RGENGC_ESTIMATE_OLDMALLOC
7187  if (!is_full_marking(objspace)) {
7188  if (objspace->rgengc.oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
7190  objspace->rgengc.oldmalloc_increase_limit =
7192 
7193  if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
7194  objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
7195  }
7196  }
7197 
7198  if (0) fprintf(stderr, "%d\t%d\t%u\t%u\t%d\n",
7199  (int)rb_gc_count(),
7200  (int)objspace->rgengc.need_major_gc,
7201  (unsigned int)objspace->rgengc.oldmalloc_increase,
7202  (unsigned int)objspace->rgengc.oldmalloc_increase_limit,
7203  (unsigned int)gc_params.oldmalloc_limit_max);
7204  }
7205  else {
7206  /* major GC */
7207  objspace->rgengc.oldmalloc_increase = 0;
7208 
7209  if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
7210  objspace->rgengc.oldmalloc_increase_limit =
7211  (size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
7212  if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
7213  objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
7214  }
7215  }
7216  }
7217 #endif
7218 }
7219 
7220 static int
7221 garbage_collect(rb_objspace_t *objspace, int reason)
7222 {
7223 #if GC_PROFILE_MORE_DETAIL
7224  objspace->profile.prepare_time = getrusage_time();
7225 #endif
7226 
7227  gc_rest(objspace);
7228 
7229 #if GC_PROFILE_MORE_DETAIL
7230  objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time;
7231 #endif
7232 
7233  return gc_start(objspace, reason);
7234 }
7235 
7236 static int
7237 gc_start(rb_objspace_t *objspace, int reason)
7238 {
7239  unsigned int do_full_mark = !!((unsigned)reason & GPR_FLAG_FULL_MARK);
7240  unsigned int immediate_mark = (unsigned)reason & GPR_FLAG_IMMEDIATE_MARK;
7241 
7242  /* reason may be clobbered, later, so keep set immediate_sweep here */
7243  objspace->flags.immediate_sweep = !!((unsigned)reason & GPR_FLAG_IMMEDIATE_SWEEP);
7244 
7245  if (!heap_allocated_pages) return FALSE; /* heap is not ready */
7246  if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
7247 
7248  GC_ASSERT(gc_mode(objspace) == gc_mode_none);
7250  GC_ASSERT(!is_incremental_marking(objspace));
7251 #if RGENGC_CHECK_MODE >= 2
7252  gc_verify_internal_consistency(objspace);
7253 #endif
7254 
7255  gc_enter(objspace, "gc_start");
7256 
7257  if (ruby_gc_stressful) {
7259 
7260  if ((flag & (1<<gc_stress_no_major)) == 0) {
7261  do_full_mark = TRUE;
7262  }
7263 
7264  objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
7265  }
7266  else {
7267 #if USE_RGENGC
7268  if (objspace->rgengc.need_major_gc) {
7269  reason |= objspace->rgengc.need_major_gc;
7270  do_full_mark = TRUE;
7271  }
7272  else if (RGENGC_FORCE_MAJOR_GC) {
7273  reason = GPR_FLAG_MAJOR_BY_FORCE;
7274  do_full_mark = TRUE;
7275  }
7276 
7277  objspace->rgengc.need_major_gc = GPR_FLAG_NONE;
7278 #endif
7279  }
7280 
7281  if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
7282  reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */
7283  }
7284 
7285 #if GC_ENABLE_INCREMENTAL_MARK
7286  if (!GC_ENABLE_INCREMENTAL_MARK || objspace->flags.dont_incremental || immediate_mark) {
7288  }
7289  else {
7290  objspace->flags.during_incremental_marking = do_full_mark;
7291  }
7292 #endif
7293 
7294  if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) {
7295  objspace->flags.immediate_sweep = TRUE;
7296  }
7297 
7298  if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
7299 
7300  gc_report(1, objspace, "gc_start(reason: %d) => %u, %d, %d\n",
7301  reason,
7302  do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
7303 
7304 #if USE_DEBUG_COUNTER
7305  RB_DEBUG_COUNTER_INC(gc_count);
7306 
7307  if (reason & GPR_FLAG_MAJOR_MASK) {
7308  (void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
7309  (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
7310  (void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
7311  (void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
7312 #if RGENGC_ESTIMATE_OLDMALLOC
7313  (void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
7314 #endif
7315  }
7316  else {
7317  (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
7318  (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
7319  (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
7320  (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
7321  (void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
7322  }
7323 #endif
7324 
7325  objspace->profile.count++;
7326  objspace->profile.latest_gc_info = reason;
7329  gc_prof_setup_new_record(objspace, reason);
7330  gc_reset_malloc_info(objspace);
7331  rb_transient_heap_start_marking(do_full_mark);
7332 
7333  gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */);
7335 
7336  gc_prof_timer_start(objspace);
7337  {
7338  gc_marks(objspace, do_full_mark);
7339  }
7340  gc_prof_timer_stop(objspace);
7341 
7342  gc_exit(objspace, "gc_start");
7343  return TRUE;
7344 }
7345 
7346 static void
7347 gc_rest(rb_objspace_t *objspace)
7348 {
7349  int marking = is_incremental_marking(objspace);
7350  int sweeping = is_lazy_sweeping(heap_eden);
7351 
7352  if (marking || sweeping) {
7353  gc_enter(objspace, "gc_rest");
7354 
7355  if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace);
7356 
7357  if (is_incremental_marking(objspace)) {
7359  gc_marks_rest(objspace);
7361  }
7362  if (is_lazy_sweeping(heap_eden)) {
7363  gc_sweep_rest(objspace);
7364  }
7365  gc_exit(objspace, "gc_rest");
7366  }
7367 }
7368 
7371  int reason;
7372 };
7373 
7374 static void
7375 gc_current_status_fill(rb_objspace_t *objspace, char *buff)
7376 {
7377  int i = 0;
7378  if (is_marking(objspace)) {
7379  buff[i++] = 'M';
7380 #if USE_RGENGC
7381  if (is_full_marking(objspace)) buff[i++] = 'F';
7382 #if GC_ENABLE_INCREMENTAL_MARK
7383  if (is_incremental_marking(objspace)) buff[i++] = 'I';
7384 #endif
7385 #endif
7386  }
7387  else if (is_sweeping(objspace)) {
7388  buff[i++] = 'S';
7389  if (is_lazy_sweeping(heap_eden)) buff[i++] = 'L';
7390  }
7391  else {
7392  buff[i++] = 'N';
7393  }
7394  buff[i] = '\0';
7395 }
7396 
7397 static const char *
7398 gc_current_status(rb_objspace_t *objspace)
7399 {
7400  static char buff[0x10];
7401  gc_current_status_fill(objspace, buff);
7402  return buff;
7403 }
7404 
7405 #if PRINT_ENTER_EXIT_TICK
7406 
7407 static tick_t last_exit_tick;
7408 static tick_t enter_tick;
7409 static int enter_count = 0;
7410 static char last_gc_status[0x10];
7411 
7412 static inline void
7413 gc_record(rb_objspace_t *objspace, int direction, const char *event)
7414 {
7415  if (direction == 0) { /* enter */
7416  enter_count++;
7417  enter_tick = tick();
7418  gc_current_status_fill(objspace, last_gc_status);
7419  }
7420  else { /* exit */
7421  tick_t exit_tick = tick();
7422  char current_gc_status[0x10];
7423  gc_current_status_fill(objspace, current_gc_status);
7424 #if 1
7425  /* [last mutator time] [gc time] [event] */
7426  fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
7427  enter_tick - last_exit_tick,
7428  exit_tick - enter_tick,
7429  event,
7430  last_gc_status, current_gc_status,
7431  (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
7432  last_exit_tick = exit_tick;
7433 #else
7434  /* [enter_tick] [gc time] [event] */
7435  fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
7436  enter_tick,
7437  exit_tick - enter_tick,
7438  event,
7439  last_gc_status, current_gc_status,
7440  (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
7441 #endif
7442  }
7443 }
7444 #else /* PRINT_ENTER_EXIT_TICK */
7445 static inline void
7446 gc_record(rb_objspace_t *objspace, int direction, const char *event)
7447 {
7448  /* null */
7449 }
7450 #endif /* PRINT_ENTER_EXIT_TICK */
7451 
7452 static inline void
7453 gc_enter(rb_objspace_t *objspace, const char *event)
7454 {
7455  GC_ASSERT(during_gc == 0);
7456  if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
7457 
7459 
7460  during_gc = TRUE;
7461  gc_report(1, objspace, "gc_enter: %s [%s]\n", event, gc_current_status(objspace));
7462  gc_record(objspace, 0, event);
7463  gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_ENTER, 0); /* TODO: which parameter should be passed? */
7464 }
7465 
7466 static inline void
7467 gc_exit(rb_objspace_t *objspace, const char *event)
7468 {
7469  GC_ASSERT(during_gc != 0);
7470 
7471  gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_EXIT, 0); /* TODO: which parameter should be passsed? */
7472  gc_record(objspace, 1, event);
7473  gc_report(1, objspace, "gc_exit: %s [%s]\n", event, gc_current_status(objspace));
7474  during_gc = FALSE;
7475 
7477 }
7478 
7479 static void *
7480 gc_with_gvl(void *ptr)
7481 {
7482  struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr;
7483  return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
7484 }
7485 
7486 static int
7487 garbage_collect_with_gvl(rb_objspace_t *objspace, int reason)
7488 {
7489  if (dont_gc) return TRUE;
7490  if (ruby_thread_has_gvl_p()) {
7491  return garbage_collect(objspace, reason);
7492  }
7493  else {
7494  if (ruby_native_thread_p()) {
7495  struct objspace_and_reason oar;
7496  oar.objspace = objspace;
7497  oar.reason = reason;
7498  return (int)(VALUE)rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar);
7499  }
7500  else {
7501  /* no ruby thread */
7502  fprintf(stderr, "[FATAL] failed to allocate memory\n");
7503  exit(EXIT_FAILURE);
7504  }
7505  }
7506 }
7507 
7508 static VALUE
7509 gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep)
7510 {
7512  int reason = GPR_FLAG_FULL_MARK |
7516 
7517  if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
7518  if (!RTEST(immediate_mark)) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
7519  if (!RTEST(immediate_sweep)) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
7520 
7521  garbage_collect(objspace, reason);
7522  gc_finalize_deferred(objspace);
7523 
7524  return Qnil;
7525 }
7526 
7527 static int
7528 gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
7529 {
7530  if (SPECIAL_CONST_P(obj)) {
7531  return FALSE;
7532  }
7533 
7534  switch (BUILTIN_TYPE(obj)) {
7535  case T_NONE:
7536  case T_NIL:
7537  case T_MOVED:
7538  case T_ZOMBIE:
7539  return FALSE;
7540  break;
7541  case T_SYMBOL:
7542  if (DYNAMIC_SYM_P(obj) && (RSYMBOL(obj)->id & ~ID_SCOPE_MASK)) {
7543  return FALSE;
7544  }
7545  /* fall through */
7546  case T_STRING:
7547  case T_OBJECT:
7548  case T_FLOAT:
7549  case T_IMEMO:
7550  case T_ARRAY:
7551  case T_BIGNUM:
7552  case T_ICLASS:
7553  case T_MODULE:
7554  case T_REGEXP:
7555  case T_DATA:
7556  case T_MATCH:
7557  case T_STRUCT:
7558  case T_HASH:
7559  case T_FILE:
7560  case T_COMPLEX:
7561  case T_RATIONAL:
7562  case T_NODE:
7563  case T_CLASS:
7564  if (FL_TEST(obj, FL_FINALIZE)) {
7566  return FALSE;
7567  }
7568  }
7569  return !RVALUE_PINNED(obj);
7570  break;
7571 
7572  default:
7573  rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
7574  break;
7575  }
7576 
7577  return FALSE;
7578 }
7579 
7580 static VALUE
7581 gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, VALUE moved_list)
7582 {
7583  int marked;
7584  int wb_unprotected;
7585  int uncollectible;
7586  int marking;
7587  RVALUE *dest = (RVALUE *)free;
7588  RVALUE *src = (RVALUE *)scan;
7589 
7590  gc_report(4, objspace, "Moving object: %p -> %p\n", (void*)scan, (void *)free);
7591 
7592  GC_ASSERT(BUILTIN_TYPE(scan) != T_NONE);
7594 
7595  /* Save off bits for current object. */
7597  wb_unprotected = RVALUE_WB_UNPROTECTED((VALUE)src);
7598  uncollectible = RVALUE_UNCOLLECTIBLE((VALUE)src);
7599  marking = RVALUE_MARKING((VALUE)src);
7600 
7602 
7603  /* Clear bits for eventual T_MOVED */
7608 
7609  if (FL_TEST(src, FL_EXIVAR)) {
7610  rb_mv_generic_ivar((VALUE)src, (VALUE)dest);
7611  }
7612 
7613  VALUE id;
7614 
7615  /* If the source object's object_id has been seen, we need to update
7616  * the object to object id mapping. */
7617  if (st_lookup(objspace->obj_to_id_tbl, (VALUE)src, &id)) {
7618  gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest);
7620  st_insert(objspace->obj_to_id_tbl, (VALUE)dest, id);
7621  }
7622 
7623  /* Move the object */
7624  memcpy(dest, src, sizeof(RVALUE));
7625  memset(src, 0, sizeof(RVALUE));
7626 
7627  /* Set bits for object in new location */
7628  if (marking) {
7630  }
7631  else {
7633  }
7634 
7635  if (marked) {
7637  }
7638  else {
7640  }
7641 
7642  if (wb_unprotected) {
7644  }
7645  else {
7647  }
7648 
7649  if (uncollectible) {
7651  }
7652  else {
7654  }
7655 
7656  /* Assign forwarding address */
7657  src->as.moved.flags = T_MOVED;
7658  src->as.moved.destination = (VALUE)dest;
7659  src->as.moved.next = moved_list;
7660  GC_ASSERT(BUILTIN_TYPE((VALUE)dest) != T_NONE);
7661 
7662  return (VALUE)src;
7663 }
7664 
7665 struct heap_cursor {
7667  size_t index;
7668  struct heap_page *page;
7670 };
7671 
7672 static void
7673 advance_cursor(struct heap_cursor *free, struct heap_page **page_list)
7674 {
7675  if (free->slot == free->page->start + free->page->total_slots - 1) {
7676  free->index++;
7677  free->page = page_list[free->index];
7678  free->slot = free->page->start;
7679  }
7680  else {
7681  free->slot++;
7682  }
7683 }
7684 
7685 static void
7686 retreat_cursor(struct heap_cursor *scan, struct heap_page **page_list)
7687 {
7688  if (scan->slot == scan->page->start) {
7689  scan->index--;
7690  scan->page = page_list[scan->index];
7691  scan->slot = scan->page->start + scan->page->total_slots - 1;
7692  }
7693  else {
7694  scan->slot--;
7695  }
7696 }
7697 
7698 static int
7699 not_met(struct heap_cursor *free, struct heap_cursor *scan)
7700 {
7701  if (free->index < scan->index)
7702  return 1;
7703 
7704  if (free->index > scan->index)
7705  return 0;
7706 
7707  return free->slot < scan->slot;
7708 }
7709 
7710 static void
7711 init_cursors(rb_objspace_t *objspace, struct heap_cursor *free, struct heap_cursor *scan, struct heap_page **page_list)
7712 {
7713  struct heap_page *page;
7714  size_t total_pages = heap_eden->total_pages;
7715  page = page_list[0];
7716 
7717  free->index = 0;
7718  free->page = page;
7719  free->slot = page->start;
7720  free->objspace = objspace;
7721 
7722  page = page_list[total_pages - 1];
7723  scan->index = total_pages - 1;
7724  scan->page = page;
7725  scan->slot = page->start + page->total_slots - 1;
7726  scan->objspace = objspace;
7727 }
7728 
7729 static int
7730 count_pinned(struct heap_page *page)
7731 {
7732  int pinned = 0;
7733  int i;
7734 
7735  for (i = 0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
7736  pinned += popcount_bits(page->pinned_bits[i]);
7737  }
7738 
7739  return pinned;
7740 }
7741 
7742 static int
7743 compare_pinned(const void *left, const void *right, void *dummy)
7744 {
7745  struct heap_page *left_page;
7746  struct heap_page *right_page;
7747 
7748  left_page = *(struct heap_page * const *)left;
7749  right_page = *(struct heap_page * const *)right;
7750 
7751  return right_page->pinned_slots - left_page->pinned_slots;
7752 }
7753 
7754 static int
7755 compare_free_slots(const void *left, const void *right, void *dummy)
7756 {
7757  struct heap_page *left_page;
7758  struct heap_page *right_page;
7759 
7760  left_page = *(struct heap_page * const *)left;
7761  right_page = *(struct heap_page * const *)right;
7762 
7763  return right_page->free_slots - left_page->free_slots;
7764 }
7765 
7766 typedef int page_compare_func_t(const void *, const void *, void *);
7767 
7768 static struct heap_page **
7769 allocate_page_list(rb_objspace_t *objspace, page_compare_func_t *comparator)
7770 {
7771  size_t total_pages = heap_eden->total_pages;
7772  size_t size = size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
7773  struct heap_page *page = 0, **page_list = malloc(size);
7774  int i = 0;
7775 
7776  list_for_each(&heap_eden->pages, page, page_node) {
7777  page_list[i++] = page;
7778  page->pinned_slots = count_pinned(page);
7779  GC_ASSERT(page != NULL);
7780  }
7781  GC_ASSERT(total_pages > 0);
7782  GC_ASSERT((size_t)i == total_pages);
7783 
7784  ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), comparator, NULL);
7785 
7786  return page_list;
7787 }
7788 
7789 static VALUE
7790 gc_compact_heap(rb_objspace_t *objspace, page_compare_func_t *comparator)
7791 {
7792  struct heap_cursor free_cursor;
7793  struct heap_cursor scan_cursor;
7794  struct heap_page **page_list;
7795  VALUE moved_list;
7796 
7797  moved_list = Qfalse;
7798  memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
7799  memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
7800 
7801  page_list = allocate_page_list(objspace, comparator);
7802 
7803  init_cursors(objspace, &free_cursor, &scan_cursor, page_list);
7804 
7805  /* Two finger algorithm */
7806  while (not_met(&free_cursor, &scan_cursor)) {
7807  /* Free cursor movement */
7808 
7809  /* Unpoison free_cursor slot */
7810  void *free_slot_poison = asan_poisoned_object_p((VALUE)free_cursor.slot);
7811  asan_unpoison_object((VALUE)free_cursor.slot, false);
7812 
7813  while (BUILTIN_TYPE(free_cursor.slot) != T_NONE && not_met(&free_cursor, &scan_cursor)) {
7814  /* Re-poison slot if it's not the one we want */
7815  if (free_slot_poison) {
7816  GC_ASSERT(BUILTIN_TYPE(free_cursor.slot) == T_NONE);
7817  asan_poison_object((VALUE)free_cursor.slot);
7818  }
7819 
7820  advance_cursor(&free_cursor, page_list);
7821 
7822  /* Unpoison free_cursor slot */
7823  free_slot_poison = asan_poisoned_object_p((VALUE)free_cursor.slot);
7824  asan_unpoison_object((VALUE)free_cursor.slot, false);
7825  }
7826 
7827  /* Unpoison scan_cursor slot */
7828  void *scan_slot_poison = asan_poisoned_object_p((VALUE)scan_cursor.slot);
7829  asan_unpoison_object((VALUE)scan_cursor.slot, false);
7830 
7831  /* Scan cursor movement */
7832  objspace->rcompactor.considered_count_table[BUILTIN_TYPE((VALUE)scan_cursor.slot)]++;
7833 
7834  while (!gc_is_moveable_obj(objspace, (VALUE)scan_cursor.slot) && not_met(&free_cursor, &scan_cursor)) {
7835 
7836  /* Re-poison slot if it's not the one we want */
7837  if (scan_slot_poison) {
7838  GC_ASSERT(BUILTIN_TYPE(scan_cursor.slot) == T_NONE);
7839  asan_poison_object((VALUE)scan_cursor.slot);
7840  }
7841 
7842  retreat_cursor(&scan_cursor, page_list);
7843 
7844  /* Unpoison scan_cursor slot */
7845  scan_slot_poison = asan_poisoned_object_p((VALUE)scan_cursor.slot);
7846  asan_unpoison_object((VALUE)scan_cursor.slot, false);
7847 
7848  objspace->rcompactor.considered_count_table[BUILTIN_TYPE((VALUE)scan_cursor.slot)]++;
7849  }
7850 
7851  if (not_met(&free_cursor, &scan_cursor)) {
7852  objspace->rcompactor.moved_count_table[BUILTIN_TYPE((VALUE)scan_cursor.slot)]++;
7853 
7854  GC_ASSERT(BUILTIN_TYPE(free_cursor.slot) == T_NONE);
7855  GC_ASSERT(BUILTIN_TYPE(scan_cursor.slot) != T_NONE);
7856  GC_ASSERT(BUILTIN_TYPE(scan_cursor.slot) != T_MOVED);
7857 
7858  moved_list = gc_move(objspace, (VALUE)scan_cursor.slot, (VALUE)free_cursor.slot, moved_list);
7859 
7860  GC_ASSERT(BUILTIN_TYPE(free_cursor.slot) != T_MOVED);
7861  GC_ASSERT(BUILTIN_TYPE(free_cursor.slot) != T_NONE);
7862  GC_ASSERT(BUILTIN_TYPE(scan_cursor.slot) == T_MOVED);
7863 
7864  advance_cursor(&free_cursor, page_list);
7865  retreat_cursor(&scan_cursor, page_list);
7866  }
7867  }
7868  free(page_list);
7869 
7870  return moved_list;
7871 }
7872 
7873 static void
7874 gc_ref_update_array(rb_objspace_t * objspace, VALUE v)
7875 {
7876  long i, len;
7877 
7878  if (FL_TEST(v, ELTS_SHARED))
7879  return;
7880 
7881  len = RARRAY_LEN(v);
7882  if (len > 0) {
7884  for (i = 0; i < len; i++) {
7885  UPDATE_IF_MOVED(objspace, ptr[i]);
7886  }
7887  }
7888 }
7889 
7890 static void
7891 gc_ref_update_object(rb_objspace_t * objspace, VALUE v)
7892 {
7893  VALUE *ptr = ROBJECT_IVPTR(v);
7894 
7895  if (ptr) {
7897  for (i = 0; i < len; i++) {
7898  UPDATE_IF_MOVED(objspace, ptr[i]);
7899  }
7900  }
7901 }
7902 
7903 static int
7904 hash_replace_ref(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
7905 {
7906  rb_objspace_t *objspace = (rb_objspace_t *)argp;
7907 
7908  if (gc_object_moved_p(objspace, (VALUE)*key)) {
7909  *key = rb_gc_location((VALUE)*key);
7910  }
7911 
7912  if (gc_object_moved_p(objspace, (VALUE)*value)) {
7913  *value = rb_gc_location((VALUE)*value);
7914  }
7915 
7916  return ST_CONTINUE;
7917 }
7918 
7919 static int
7920 hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error)
7921 {
7922  rb_objspace_t *objspace;
7923 
7924  objspace = (rb_objspace_t *)argp;
7925 
7926  if (gc_object_moved_p(objspace, (VALUE)key)) {
7927  return ST_REPLACE;
7928  }
7929 
7930  if (gc_object_moved_p(objspace, (VALUE)value)) {
7931  return ST_REPLACE;
7932  }
7933  return ST_CONTINUE;
7934 }
7935 
7936 static int
7937 hash_replace_ref_value(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
7938 {
7939  rb_objspace_t *objspace = (rb_objspace_t *)argp;
7940 
7941  if (gc_object_moved_p(objspace, (VALUE)*value)) {
7942  *value = rb_gc_location((VALUE)*value);
7943  }
7944 
7945  return ST_CONTINUE;
7946 }
7947 
7948 static int
7949 hash_foreach_replace_value(st_data_t key, st_data_t value, st_data_t argp, int error)
7950 {
7951  rb_objspace_t *objspace;
7952 
7953  objspace = (rb_objspace_t *)argp;
7954 
7955  if (gc_object_moved_p(objspace, (VALUE)value)) {
7956  return ST_REPLACE;
7957  }
7958  return ST_CONTINUE;
7959 }
7960 
7961 static void
7962 gc_update_tbl_refs(rb_objspace_t * objspace, st_table *tbl)
7963 {
7964  if (!tbl || tbl->num_entries == 0) return;
7965 
7966  if (st_foreach_with_replace(tbl, hash_foreach_replace_value, hash_replace_ref_value, (st_data_t)objspace)) {
7967  rb_raise(rb_eRuntimeError, "hash modified during iteration");
7968  }
7969 }
7970 
7971 static void
7972 gc_update_table_refs(rb_objspace_t * objspace, st_table *tbl)
7973 {
7974  if (!tbl || tbl->num_entries == 0) return;
7975 
7976  if (st_foreach_with_replace(tbl, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace)) {
7977  rb_raise(rb_eRuntimeError, "hash modified during iteration");
7978  }
7979 }
7980 
7981 /* Update MOVED references in an st_table */
7982 void
7984 {
7985  rb_objspace_t *objspace = &rb_objspace;
7986  gc_update_table_refs(objspace, ptr);
7987 }
7988 
7989 static void
7990 gc_ref_update_hash(rb_objspace_t * objspace, VALUE v)
7991 {
7992  rb_hash_stlike_foreach_with_replace(v, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace);
7993 }
7994 
7995 static void
7996 gc_ref_update_method_entry(rb_objspace_t *objspace, rb_method_entry_t *me)
7997 {
7998  rb_method_definition_t *def = me->def;
7999 
8000  UPDATE_IF_MOVED(objspace, me->owner);
8001  UPDATE_IF_MOVED(objspace, me->defined_class);
8002 
8003  if (def) {
8004  switch (def->type) {
8005  case VM_METHOD_TYPE_ISEQ:
8006  if (def->body.iseq.iseqptr) {
8007  TYPED_UPDATE_IF_MOVED(objspace, rb_iseq_t *, def->body.iseq.iseqptr);
8008  }
8009  TYPED_UPDATE_IF_MOVED(objspace, rb_cref_t *, def->body.iseq.cref);
8010  break;
8012  case VM_METHOD_TYPE_IVAR:
8013  UPDATE_IF_MOVED(objspace, def->body.attr.location);
8014  break;
8016  UPDATE_IF_MOVED(objspace, def->body.bmethod.proc);
8017  break;
8018  case VM_METHOD_TYPE_ALIAS:
8020  return;
8023  UPDATE_IF_MOVED(objspace, def->body.refined.owner);
8024  break;
8025  case VM_METHOD_TYPE_CFUNC:
8026  case VM_METHOD_TYPE_ZSUPER:
8029  case VM_METHOD_TYPE_UNDEF:
8031  break;
8032  }
8033  }
8034 }
8035 
8036 static void
8037 gc_update_values(rb_objspace_t *objspace, long n, VALUE *values)
8038 {
8039  long i;
8040 
8041  for (i=0; i<n; i++) {
8042  UPDATE_IF_MOVED(objspace, values[i]);
8043  }
8044 }
8045 
8046 static void
8047 gc_ref_update_imemo(rb_objspace_t *objspace, VALUE obj)
8048 {
8049  switch (imemo_type(obj)) {
8050  case imemo_env:
8051  {
8052  rb_env_t *env = (rb_env_t *)obj;
8053  TYPED_UPDATE_IF_MOVED(objspace, rb_iseq_t *, env->iseq);
8054  UPDATE_IF_MOVED(objspace, env->ep[VM_ENV_DATA_INDEX_ENV]);
8055  gc_update_values(objspace, (long)env->env_size, (VALUE *)env->env);
8056  }
8057  break;
8058  case imemo_cref:
8059  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.cref.klass);
8060  TYPED_UPDATE_IF_MOVED(objspace, struct rb_cref_struct *, RANY(obj)->as.imemo.cref.next);
8061  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.cref.refinements);
8062  break;
8063  case imemo_svar:
8064  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.cref_or_me);
8065  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.lastline);
8066  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.backref);
8067  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.others);
8068  break;
8069  case imemo_throw_data:
8070  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.throw_data.throw_obj);
8071  break;
8072  case imemo_ifunc:
8073  break;
8074  case imemo_memo:
8075  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.memo.v1);
8076  UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.memo.v2);
8077  break;
8078  case imemo_ment:
8079  gc_ref_update_method_entry(objspace, &RANY(obj)->as.imemo.ment);
8080  break;
8081  case imemo_iseq:
8083  break;
8084  case imemo_ast:
8086  break;
8087  case imemo_parser_strterm:
8088  case imemo_tmpbuf:
8089  break;
8090  default:
8091  rb_bug("not reachable %d", imemo_type(obj));
8092  break;
8093  }
8094 }
8095 
8096 static enum rb_id_table_iterator_result
8097 check_id_table_move(ID id, VALUE value, void *data)
8098 {
8099  rb_objspace_t *objspace = (rb_objspace_t *)data;
8100 
8101  if (gc_object_moved_p(objspace, (VALUE)value)) {
8102  return ID_TABLE_REPLACE;
8103  }
8104 
8105  return ID_TABLE_CONTINUE;
8106 }
8107 
8108 /* Returns the new location of an object, if it moved. Otherwise returns
8109  * the existing location. */
8110 VALUE
8112 {
8113 
8114  VALUE destination;
8115 
8116  if (!SPECIAL_CONST_P((void *)value)) {
8117  void *poisoned = asan_poisoned_object_p(value);
8118  asan_unpoison_object(value, false);
8119 
8120  if (BUILTIN_TYPE(value) == T_MOVED) {
8121  destination = (VALUE)RMOVED(value)->destination;
8122  GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE);
8123  }
8124  else {
8125  destination = value;
8126  }
8127 
8128  /* Re-poison slot if it's not the one we want */
8129  if (poisoned) {
8130  GC_ASSERT(BUILTIN_TYPE(value) == T_NONE);
8131  asan_poison_object(value);
8132  }
8133  }
8134  else {
8135  destination = value;
8136  }
8137 
8138  return destination;
8139 }
8140 
8141 static enum rb_id_table_iterator_result
8142 update_id_table(ID *key, VALUE * value, void *data, int existing)
8143 {
8144  rb_objspace_t *objspace = (rb_objspace_t *)data;
8145 
8146  if (gc_object_moved_p(objspace, (VALUE)*value)) {
8147  *value = rb_gc_location((VALUE)*value);
8148  }
8149 
8150  return ID_TABLE_CONTINUE;
8151 }
8152 
8153 static void
8154 update_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
8155 {
8156  if (tbl) {
8157  rb_id_table_foreach_with_replace(tbl, check_id_table_move, update_id_table, objspace);
8158  }
8159 }
8160 
8161 static enum rb_id_table_iterator_result
8162 update_const_table(VALUE value, void *data)
8163 {
8164  rb_const_entry_t *ce = (rb_const_entry_t *)value;
8165  rb_objspace_t * objspace = (rb_objspace_t *)data;
8166 
8167  if (gc_object_moved_p(objspace, ce->value)) {
8168  ce->value = rb_gc_location(ce->value);
8169  }
8170 
8171  if (gc_object_moved_p(objspace, ce->file)) {
8172  ce->file = rb_gc_location(ce->file);
8173  }
8174 
8175  return ID_TABLE_CONTINUE;
8176 }
8177 
8178 static void
8179 update_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
8180 {
8181  if (!tbl) return;
8182  rb_id_table_foreach_values(tbl, update_const_table, objspace);
8183 }
8184 
8185 static void
8186 update_subclass_entries(rb_objspace_t *objspace, rb_subclass_entry_t *entry)
8187 {
8188  while (entry) {
8189  UPDATE_IF_MOVED(objspace, entry->klass);
8190  entry = entry->next;
8191  }
8192 }
8193 
8194 static void
8195 update_class_ext(rb_objspace_t *objspace, rb_classext_t *ext)
8196 {
8197  UPDATE_IF_MOVED(objspace, ext->origin_);
8198  UPDATE_IF_MOVED(objspace, ext->refined_class);
8199  update_subclass_entries(objspace, ext->subclasses);
8200 }
8201 
8202 static void
8203 gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
8204 {
8205  RVALUE *any = RANY(obj);
8206 
8207  gc_report(4, objspace, "update-refs: %p ->", (void *)obj);
8208 
8209  switch (BUILTIN_TYPE(obj)) {
8210  case T_CLASS:
8211  case T_MODULE:
8212  if (RCLASS_SUPER((VALUE)obj)) {
8213  UPDATE_IF_MOVED(objspace, RCLASS(obj)->super);
8214  }
8215  if (!RCLASS_EXT(obj)) break;
8216  update_m_tbl(objspace, RCLASS_M_TBL(obj));
8217  gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj));
8218  update_class_ext(objspace, RCLASS_EXT(obj));
8219  update_const_tbl(objspace, RCLASS_CONST_TBL(obj));
8220  break;
8221 
8222  case T_ICLASS:
8223  if (FL_TEST(obj, RICLASS_IS_ORIGIN)) {
8224  update_m_tbl(objspace, RCLASS_M_TBL(obj));
8225  }
8226  if (RCLASS_SUPER((VALUE)obj)) {
8227  UPDATE_IF_MOVED(objspace, RCLASS(obj)->super);
8228  }
8229  if (!RCLASS_EXT(obj)) break;
8230  if (RCLASS_IV_TBL(obj)) {
8231  gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj));
8232  }
8233  update_class_ext(objspace, RCLASS_EXT(obj));
8234  update_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
8235  break;
8236 
8237  case T_IMEMO:
8238  gc_ref_update_imemo(objspace, obj);
8239  return;
8240 
8241  case T_NIL:
8242  case T_FIXNUM:
8243  case T_NODE:
8244  case T_MOVED:
8245  case T_NONE:
8246  /* These can't move */
8247  return;
8248 
8249  case T_ARRAY:
8250  if (FL_TEST(obj, ELTS_SHARED)) {
8251  UPDATE_IF_MOVED(objspace, any->as.array.as.heap.aux.shared_root);
8252  }
8253  else {
8254  gc_ref_update_array(objspace, obj);
8255  }
8256  break;
8257 
8258  case T_HASH:
8259  gc_ref_update_hash(objspace, obj);
8260  UPDATE_IF_MOVED(objspace, any->as.hash.ifnone);
8261  break;
8262 
8263  case T_STRING:
8264  if (STR_SHARED_P(obj)) {
8265  UPDATE_IF_MOVED(objspace, any->as.string.as.heap.aux.shared);
8266  }
8267  break;
8268 
8269  case T_DATA:
8270  /* Call the compaction callback, if it exists */
8271  {
8272  void *const ptr = DATA_PTR(obj);
8273  if (ptr) {
8274  if (RTYPEDDATA_P(obj)) {
8275  RUBY_DATA_FUNC compact_func = any->as.typeddata.type->function.dcompact;
8276  if (compact_func) (*compact_func)(ptr);
8277  }
8278  }
8279  }
8280  break;
8281 
8282  case T_OBJECT:
8283  gc_ref_update_object(objspace, obj);
8284  break;
8285 
8286  case T_FILE:
8287  if (any->as.file.fptr) {
8288  UPDATE_IF_MOVED(objspace, any->as.file.fptr->pathv);
8289  UPDATE_IF_MOVED(objspace, any->as.file.fptr->tied_io_for_writing);
8291  UPDATE_IF_MOVED(objspace, any->as.file.fptr->writeconv_pre_ecopts);
8292  UPDATE_IF_MOVED(objspace, any->as.file.fptr->encs.ecopts);
8293  UPDATE_IF_MOVED(objspace, any->as.file.fptr->write_lock);
8294  }
8295  break;
8296  case T_REGEXP:
8297  UPDATE_IF_MOVED(objspace, any->as.regexp.src);
8298  break;
8299 
8300  case T_SYMBOL:
8301  if (DYNAMIC_SYM_P((VALUE)any)) {
8302  UPDATE_IF_MOVED(objspace, RSYMBOL(any)->fstr);
8303  }
8304  break;
8305 
8306  case T_FLOAT:
8307  case T_BIGNUM:
8308  break;
8309 
8310  case T_MATCH:
8311  UPDATE_IF_MOVED(objspace, any->as.match.regexp);
8312 
8313  if (any->as.match.str) {
8314  UPDATE_IF_MOVED(objspace, any->as.match.str);
8315  }
8316  break;
8317 
8318  case T_RATIONAL:
8319  UPDATE_IF_MOVED(objspace, any->as.rational.num);
8320  UPDATE_IF_MOVED(objspace, any->as.rational.den);
8321  break;
8322 
8323  case T_COMPLEX:
8324  UPDATE_IF_MOVED(objspace, any->as.complex.real);
8325  UPDATE_IF_MOVED(objspace, any->as.complex.imag);
8326 
8327  break;
8328 
8329  case T_STRUCT:
8330  {
8331  long i, len = RSTRUCT_LEN(obj);
8333 
8334  for (i = 0; i < len; i++) {
8335  UPDATE_IF_MOVED(objspace, ptr[i]);
8336  }
8337  }
8338  break;
8339  default:
8340 #if GC_DEBUG
8343  rb_bug("unreachable");
8344 #endif
8345  break;
8346 
8347  }
8348 
8349  UPDATE_IF_MOVED(objspace, RBASIC(obj)->klass);
8350 
8351  gc_report(4, objspace, "update-refs: %p <-", (void *)obj);
8352 }
8353 
8354 static int
8355 gc_ref_update(void *vstart, void *vend, size_t stride, void * data)
8356 {
8357  rb_objspace_t * objspace;
8358  struct heap_page *page;
8359  short free_slots = 0;
8360 
8361  VALUE v = (VALUE)vstart;
8362  objspace = (rb_objspace_t *)data;
8363  page = GET_HEAP_PAGE(v);
8364  asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
8365  page->freelist = NULL;
8366  asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
8369 
8370  /* For each object on the page */
8371  for (; v != (VALUE)vend; v += stride) {
8372  if (!SPECIAL_CONST_P(v)) {
8373  void *poisoned = asan_poisoned_object_p(v);
8374  asan_unpoison_object(v, false);
8375 
8376  switch (BUILTIN_TYPE(v)) {
8377  case T_NONE:
8378  heap_page_add_freeobj(objspace, page, v);
8379  free_slots++;
8380  break;
8381  case T_MOVED:
8382  break;
8383  case T_ZOMBIE:
8384  break;
8385  default:
8386  if (RVALUE_WB_UNPROTECTED(v)) {
8388  }
8389  if (RVALUE_PAGE_MARKING(page, v)) {
8391  }
8392  gc_update_object_references(objspace, v);
8393  }
8394 
8395  if (poisoned) {
8397  asan_poison_object(v);
8398  }
8399  }
8400  }
8401 
8402  page->free_slots = free_slots;
8403  return 0;
8404 }
8405 
8407 #define global_symbols ruby_global_symbols
8408 
8409 static void
8410 gc_update_references(rb_objspace_t * objspace)
8411 {
8413  rb_vm_t *vm = rb_ec_vm_ptr(ec);
8414 
8415  objspace_each_objects_without_setup(objspace, gc_ref_update, objspace);
8419  global_symbols.dsymbol_fstr_hash = rb_gc_location(global_symbols.dsymbol_fstr_hash);
8420  gc_update_tbl_refs(objspace, objspace->obj_to_id_tbl);
8421  gc_update_table_refs(objspace, objspace->id_to_obj_tbl);
8422  gc_update_table_refs(objspace, global_symbols.str_sym);
8423  gc_update_table_refs(objspace, finalizer_table);
8424 }
8425 
8426 static VALUE type_sym(size_t type);
8427 
8428 static VALUE
8429 gc_compact_stats(rb_objspace_t *objspace)
8430 {
8431  size_t i;
8432  VALUE h = rb_hash_new();
8433  VALUE considered = rb_hash_new();
8434  VALUE moved = rb_hash_new();
8435 
8436  for (i=0; i<T_MASK; i++) {
8437  rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i]));
8438  }
8439 
8440  for (i=0; i<T_MASK; i++) {
8441  rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i]));
8442  }
8443 
8444  rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered);
8445  rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved);
8446 
8447  return h;
8448 }
8449 
8450 static void gc_compact_after_gc(rb_objspace_t *objspace, int use_toward_empty, int use_double_pages, int use_verifier);
8451 
8452 static void
8453 gc_compact(rb_objspace_t *objspace, int use_toward_empty, int use_double_pages, int use_verifier)
8454 {
8455 
8456  objspace->flags.during_compacting = TRUE;
8457  {
8458  /* pin objects referenced by maybe pointers */
8459  garbage_collect(objspace, GPR_DEFAULT_REASON);
8460  /* compact */
8461  gc_compact_after_gc(objspace, use_toward_empty, use_double_pages, use_verifier);
8462  }
8463  objspace->flags.during_compacting = FALSE;
8464 }
8465 
8466 static VALUE
8467 rb_gc_compact(rb_execution_context_t *ec, VALUE self)
8468 {
8469  rb_objspace_t *objspace = &rb_objspace;
8470  if (dont_gc) return Qnil;
8471 
8472  gc_compact(objspace, FALSE, FALSE, FALSE);
8473  return gc_compact_stats(objspace);
8474 }
8475 
8476 static void
8477 root_obj_check_moved_i(const char *category, VALUE obj, void *data)
8478 {
8479  if (gc_object_moved_p(&rb_objspace, obj)) {
8480  rb_bug("ROOT %s points to MOVED: %p -> %s\n", category, (void *)obj, obj_info(rb_gc_location(obj)));
8481  }
8482 }
8483 
8484 static void
8485 reachable_object_check_moved_i(VALUE ref, void *data)
8486 {
8487  VALUE parent = (VALUE)data;
8488  if (gc_object_moved_p(&rb_objspace, ref)) {
8489  rb_bug("Object %s points to MOVED: %p -> %s\n", obj_info(parent), (void *)ref, obj_info(rb_gc_location(ref)));
8490  }
8491 }
8492 
8493 static int
8494 heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
8495 {
8496  VALUE v = (VALUE)vstart;
8497  for (; v != (VALUE)vend; v += stride) {
8498  if (gc_object_moved_p(&rb_objspace, v)) {
8499  /* Moved object still on the heap, something may have a reference. */
8500  }
8501  else {
8502  void *poisoned = asan_poisoned_object_p(v);
8503  asan_unpoison_object(v, false);
8504 
8505  switch (BUILTIN_TYPE(v)) {
8506  case T_NONE:
8507  case T_ZOMBIE:
8508  break;
8509  default:
8510  rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v);
8511  }
8512 
8513  if (poisoned) {
8515  asan_poison_object(v);
8516  }
8517  }
8518  }
8519 
8520  return 0;
8521 }
8522 
8523 static VALUE
8524 gc_check_references_for_moved(rb_objspace_t *objspace)
8525 {
8526  objspace_reachable_objects_from_root(objspace, root_obj_check_moved_i, NULL);
8527  objspace_each_objects(objspace, heap_check_moved_i, NULL);
8528  return Qnil;
8529 }
8530 
8531 static void
8532 gc_compact_after_gc(rb_objspace_t *objspace, int use_toward_empty, int use_double_pages, int use_verifier)
8533 {
8534  if (0) fprintf(stderr, "gc_compact_after_gc: %d,%d,%d\n", use_toward_empty, use_double_pages, use_verifier);
8535 
8536  mjit_gc_start_hook(); // prevent MJIT from running while moving pointers related to ISeq
8537 
8538  objspace->profile.compact_count++;
8539 
8540  if (use_verifier) {
8541  gc_verify_internal_consistency(objspace);
8542  }
8543 
8544  if (use_double_pages) {
8545  /* Double heap size */
8546  heap_add_pages(objspace, heap_eden, heap_allocated_pages);
8547  }
8548 
8549  VALUE moved_list_head;
8550  VALUE disabled = rb_gc_disable();
8551 
8552  if (use_toward_empty) {
8553  moved_list_head = gc_compact_heap(objspace, compare_free_slots);
8554  }
8555  else {
8556  moved_list_head = gc_compact_heap(objspace, compare_pinned);
8557  }
8558  heap_eden->freelist = NULL;
8559 
8560  gc_update_references(objspace);
8561  if (!RTEST(disabled)) rb_gc_enable();
8562 
8563  if (use_verifier) {
8564  gc_check_references_for_moved(objspace);
8565  }
8566 
8569  heap_eden->free_pages = NULL;
8570  heap_eden->using_page = NULL;
8571 
8572  /* For each moved slot */
8573  while (moved_list_head) {
8574  VALUE next_moved;
8575  struct heap_page *page;
8576 
8577  page = GET_HEAP_PAGE(moved_list_head);
8578  next_moved = RMOVED(moved_list_head)->next;
8579 
8580  /* clear the memory for that moved slot */
8581  RMOVED(moved_list_head)->flags = 0;
8582  RMOVED(moved_list_head)->destination = 0;
8583  RMOVED(moved_list_head)->next = 0;
8584  page->free_slots++;
8585  heap_page_add_freeobj(objspace, page, moved_list_head);
8586 
8587  if (page->free_slots == page->total_slots && heap_pages_freeable_pages > 0) {
8589  heap_unlink_page(objspace, heap_eden, page);
8590  heap_add_page(objspace, heap_tomb, page);
8591  }
8592  objspace->profile.total_freed_objects++;
8593  moved_list_head = next_moved;
8594  }
8595 
8596  /* Add any eden pages with free slots back to the free pages list */
8597  struct heap_page *page = NULL;
8598  list_for_each(&heap_eden->pages, page, page_node) {
8599  if (page->free_slots > 0) {
8600  heap_add_freepage(heap_eden, page);
8601  } else {
8602  page->free_next = NULL;
8603  }
8604  }
8605 
8606  /* Set up "using_page" if we have any pages with free slots */
8607  if (heap_eden->free_pages) {
8608  heap_eden->using_page = heap_eden->free_pages;
8609  heap_eden->free_pages = heap_eden->free_pages->free_next;
8610  }
8611 
8612  if (use_verifier) {
8613  gc_verify_internal_consistency(objspace);
8614  }
8615 
8616  mjit_gc_exit_hook(); // unlock MJIT here, because `rb_gc()` calls `mjit_gc_start_hook()` again.
8617 }
8618 
8619 /*
8620  * call-seq:
8621  * GC.verify_compaction_references -> nil
8622  *
8623  * Verify compaction reference consistency.
8624  *
8625  * This method is implementation specific. During compaction, objects that
8626  * were moved are replaced with T_MOVED objects. No object should have a
8627  * reference to a T_MOVED object after compaction.
8628  *
8629  * This function doubles the heap to ensure room to move all objects,
8630  * compacts the heap to make sure everything moves, updates all references,
8631  * then performs a full GC. If any object contains a reference to a T_MOVED
8632  * object, that object should be pushed on the mark stack, and will
8633  * make a SEGV.
8634  */
8635 static VALUE
8636 gc_verify_compaction_references(int argc, VALUE *argv, VALUE mod)
8637 {
8638  rb_objspace_t *objspace = &rb_objspace;
8639  int use_toward_empty = FALSE;
8640  int use_double_pages = FALSE;
8641 
8642  if (dont_gc) return Qnil;
8643 
8644  VALUE opt = Qnil;
8645  static ID keyword_ids[2];
8646  VALUE kwvals[2];
8647 
8648  kwvals[1] = Qtrue;
8649 
8650  rb_scan_args(argc, argv, "0:", &opt);
8651 
8652  if (!NIL_P(opt)) {
8653  if (!keyword_ids[0]) {
8654  keyword_ids[0] = rb_intern("toward");
8655  keyword_ids[1] = rb_intern("double_heap");
8656  }
8657 
8658  rb_get_kwargs(opt, keyword_ids, 0, 2, kwvals);
8659  if (rb_intern("empty") == rb_sym2id(kwvals[0])) {
8660  use_toward_empty = TRUE;
8661  }
8662  if (kwvals[1] != Qundef && RTEST(kwvals[1])) {
8663  use_double_pages = TRUE;
8664  }
8665  }
8666 
8667  gc_compact(objspace, use_toward_empty, use_double_pages, TRUE);
8668  return gc_compact_stats(objspace);
8669 }
8670 
8671 VALUE
8673 {
8674  rb_gc();
8675  return Qnil;
8676 }
8677 
8678 void
8679 rb_gc(void)
8680 {
8681  rb_objspace_t *objspace = &rb_objspace;
8682  int reason = GPR_DEFAULT_REASON;
8683  garbage_collect(objspace, reason);
8684 }
8685 
8686 int
8688 {
8689  rb_objspace_t *objspace = &rb_objspace;
8690  return during_gc;
8691 }
8692 
8693 #if RGENGC_PROFILE >= 2
8694 
8695 static const char *type_name(int type, VALUE obj);
8696 
8697 static void
8698 gc_count_add_each_types(VALUE hash, const char *name, const size_t *types)
8699 {
8701  int i;
8702  for (i=0; i<T_MASK; i++) {
8703  const char *type = type_name(i, 0);
8705  }
8706  rb_hash_aset(hash, ID2SYM(rb_intern(name)), result);
8707 }
8708 #endif
8709 
8710 size_t
8712 {
8713  return rb_objspace.profile.count;
8714 }
8715 
8716 static VALUE
8717 gc_count(rb_execution_context_t *ec, VALUE self)
8718 {
8719  return SIZET2NUM(rb_gc_count());
8720 }
8721 
8722 static VALUE
8723 gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const int orig_flags)
8724 {
8725  static VALUE sym_major_by = Qnil, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state;
8726  static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
8727 #if RGENGC_ESTIMATE_OLDMALLOC
8728  static VALUE sym_oldmalloc;
8729 #endif
8730  static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
8731  static VALUE sym_none, sym_marking, sym_sweeping;
8732  VALUE hash = Qnil, key = Qnil;
8733  VALUE major_by;
8734  VALUE flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
8735 
8736  if (SYMBOL_P(hash_or_key)) {
8737  key = hash_or_key;
8738  }
8739  else if (RB_TYPE_P(hash_or_key, T_HASH)) {
8740  hash = hash_or_key;
8741  }
8742  else {
8743  rb_raise(rb_eTypeError, "non-hash or symbol given");
8744  }
8745 
8746  if (sym_major_by == Qnil) {
8747 #define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
8748  S(major_by);
8749  S(gc_by);
8750  S(immediate_sweep);
8751  S(have_finalizer);
8752  S(state);
8753 
8754  S(stress);
8755  S(nofree);
8756  S(oldgen);
8757  S(shady);
8758  S(force);
8759 #if RGENGC_ESTIMATE_OLDMALLOC
8760  S(oldmalloc);
8761 #endif
8762  S(newobj);
8763  S(malloc);
8764  S(method);
8765  S(capi);
8766 
8767  S(none);
8768  S(marking);
8769  S(sweeping);
8770 #undef S
8771  }
8772 
8773 #define SET(name, attr) \
8774  if (key == sym_##name) \
8775  return (attr); \
8776  else if (hash != Qnil) \
8777  rb_hash_aset(hash, sym_##name, (attr));
8778 
8779  major_by =
8780  (flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
8781  (flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
8782  (flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
8783  (flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
8784 #if RGENGC_ESTIMATE_OLDMALLOC
8785  (flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
8786 #endif
8787  Qnil;
8788  SET(major_by, major_by);
8789 
8790  SET(gc_by,
8791  (flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
8792  (flags & GPR_FLAG_MALLOC) ? sym_malloc :
8793  (flags & GPR_FLAG_METHOD) ? sym_method :
8794  (flags & GPR_FLAG_CAPI) ? sym_capi :
8795  (flags & GPR_FLAG_STRESS) ? sym_stress :
8796  Qnil
8797  );
8798 
8799  SET(have_finalizer, (flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
8800  SET(immediate_sweep, (flags & GPR_FLAG_IMMEDIATE_SWEEP) ? Qtrue : Qfalse);
8801 
8802  if (orig_flags == 0) {
8803  SET(state, gc_mode(objspace) == gc_mode_none ? sym_none :
8804  gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
8805  }
8806 #undef SET
8807 
8808  if (!NIL_P(key)) {/* matched key should return above */
8809  rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
8810  }
8811 
8812  return hash;
8813 }
8814 
8815 VALUE
8817 {
8818  rb_objspace_t *objspace = &rb_objspace;
8819  return gc_info_decode(objspace, key, 0);
8820 }
8821 
8822 static VALUE
8823 gc_latest_gc_info(rb_execution_context_t *ec, VALUE self, VALUE arg)
8824 {
8825  rb_objspace_t *objspace = &rb_objspace;
8826 
8827  if (NIL_P(arg)) {
8828  arg = rb_hash_new();
8829  }
8830  else if (!SYMBOL_P(arg) && !RB_TYPE_P(arg, T_HASH)) {
8831  rb_raise(rb_eTypeError, "non-hash or symbol given");
8832  }
8833 
8834  return gc_info_decode(objspace, arg, 0);
8835 }
8836 
8855 #if USE_RGENGC
8863 #if RGENGC_ESTIMATE_OLDMALLOC
8866 #endif
8867 #if RGENGC_PROFILE
8868  gc_stat_sym_total_generated_normal_object_count,
8869  gc_stat_sym_total_generated_shady_object_count,
8870  gc_stat_sym_total_shade_operation_count,
8871  gc_stat_sym_total_promoted_count,
8872  gc_stat_sym_total_remembered_normal_object_count,
8873  gc_stat_sym_total_remembered_shady_object_count,
8874 #endif
8875 #endif
8877 };
8878 
8889 #if USE_RGENGC
8894 #endif
8899 #if RGENGC_ESTIMATE_OLDMALLOC
8902 #endif
8904 };
8905 
8906 static VALUE gc_stat_symbols[gc_stat_sym_last];
8907 static VALUE gc_stat_compat_symbols[gc_stat_compat_sym_last];
8908 static VALUE gc_stat_compat_table;
8909 
8910 static void
8911 setup_gc_stat_symbols(void)
8912 {
8913  if (gc_stat_symbols[0] == 0) {
8914 #define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
8915  S(count);
8917  S(heap_sorted_length);
8919  S(heap_available_slots);
8920  S(heap_live_slots);
8921  S(heap_free_slots);
8922  S(heap_final_slots);
8923  S(heap_marked_slots);
8924  S(heap_eden_pages);
8925  S(heap_tomb_pages);
8926  S(total_allocated_pages);
8927  S(total_freed_pages);
8928  S(total_allocated_objects);
8929  S(total_freed_objects);
8930  S(malloc_increase_bytes);
8931  S(malloc_increase_bytes_limit);
8932 #if USE_RGENGC
8933  S(minor_gc_count);
8934  S(major_gc_count);
8935  S(compact_count);
8936  S(remembered_wb_unprotected_objects);
8937  S(remembered_wb_unprotected_objects_limit);
8938  S(old_objects);
8939  S(old_objects_limit);
8940 #if RGENGC_ESTIMATE_OLDMALLOC
8941  S(oldmalloc_increase_bytes);
8942  S(oldmalloc_increase_bytes_limit);
8943 #endif
8944 #if RGENGC_PROFILE
8945  S(total_generated_normal_object_count);
8946  S(total_generated_shady_object_count);
8947  S(total_shade_operation_count);
8948  S(total_promoted_count);
8949  S(total_remembered_normal_object_count);
8950  S(total_remembered_shady_object_count);
8951 #endif /* RGENGC_PROFILE */
8952 #endif /* USE_RGENGC */
8953 #undef S
8954 #define S(s) gc_stat_compat_symbols[gc_stat_compat_sym_##s] = ID2SYM(rb_intern_const(#s))
8955  S(gc_stat_heap_used);
8956  S(heap_eden_page_length);
8957  S(heap_tomb_page_length);
8958  S(heap_increment);
8959  S(heap_length);
8960  S(heap_live_slot);
8961  S(heap_free_slot);
8962  S(heap_final_slot);
8963  S(heap_swept_slot);
8964 #if USE_RGEGC
8965  S(remembered_shady_object);
8966  S(remembered_shady_object_limit);
8967  S(old_object);
8968  S(old_object_limit);
8969 #endif
8970  S(total_allocated_object);
8971  S(total_freed_object);
8972  S(malloc_increase);
8973  S(malloc_limit);
8974 #if RGENGC_ESTIMATE_OLDMALLOC
8975  S(oldmalloc_increase);
8976  S(oldmalloc_limit);
8977 #endif
8978 #undef S
8979 
8980  {
8981  VALUE table = gc_stat_compat_table = rb_hash_new();
8982  rb_obj_hide(table);
8984 
8985  /* compatibility layer for Ruby 2.1 */
8986 #define OLD_SYM(s) gc_stat_compat_symbols[gc_stat_compat_sym_##s]
8987 #define NEW_SYM(s) gc_stat_symbols[gc_stat_sym_##s]
8988  rb_hash_aset(table, OLD_SYM(gc_stat_heap_used), NEW_SYM(heap_allocated_pages));
8989  rb_hash_aset(table, OLD_SYM(heap_eden_page_length), NEW_SYM(heap_eden_pages));
8990  rb_hash_aset(table, OLD_SYM(heap_tomb_page_length), NEW_SYM(heap_tomb_pages));
8991  rb_hash_aset(table, OLD_SYM(heap_increment), NEW_SYM(heap_allocatable_pages));
8992  rb_hash_aset(table, OLD_SYM(heap_length), NEW_SYM(heap_sorted_length));
8993  rb_hash_aset(table, OLD_SYM(heap_live_slot), NEW_SYM(heap_live_slots));
8994  rb_hash_aset(table, OLD_SYM(heap_free_slot), NEW_SYM(heap_free_slots));
8995  rb_hash_aset(table, OLD_SYM(heap_final_slot), NEW_SYM(heap_final_slots));
8996 #if USE_RGEGC
8997  rb_hash_aset(table, OLD_SYM(remembered_shady_object), NEW_SYM(remembered_wb_unprotected_objects));
8998  rb_hash_aset(table, OLD_SYM(remembered_shady_object_limit), NEW_SYM(remembered_wb_unprotected_objects_limit));
8999  rb_hash_aset(table, OLD_SYM(old_object), NEW_SYM(old_objects));
9000  rb_hash_aset(table, OLD_SYM(old_object_limit), NEW_SYM(old_objects_limit));
9001 #endif
9002  rb_hash_aset(table, OLD_SYM(total_allocated_object), NEW_SYM(total_allocated_objects));
9003  rb_hash_aset(table, OLD_SYM(total_freed_object), NEW_SYM(total_freed_objects));
9004  rb_hash_aset(table, OLD_SYM(malloc_increase), NEW_SYM(malloc_increase_bytes));
9005  rb_hash_aset(table, OLD_SYM(malloc_limit), NEW_SYM(malloc_increase_bytes_limit));
9006 #if RGENGC_ESTIMATE_OLDMALLOC
9007  rb_hash_aset(table, OLD_SYM(oldmalloc_increase), NEW_SYM(oldmalloc_increase_bytes));
9008  rb_hash_aset(table, OLD_SYM(oldmalloc_limit), NEW_SYM(oldmalloc_increase_bytes_limit));
9009 #endif
9010 #undef OLD_SYM
9011 #undef NEW_SYM
9012  rb_obj_freeze(table);
9013  }
9014  }
9015 }
9016 
9017 static VALUE
9018 compat_key(VALUE key)
9019 {
9020  VALUE new_key = rb_hash_lookup(gc_stat_compat_table, key);
9021 
9022  if (!NIL_P(new_key)) {
9023  static int warned = 0;
9024  if (warned == 0) {
9025  rb_warn("GC.stat keys were changed from Ruby 2.1. "
9026  "In this case, you refer to obsolete `%"PRIsVALUE"' (new key is `%"PRIsVALUE"'). "
9027  "Please check <https://bugs.ruby-lang.org/issues/9924> for more information.",
9028  key, new_key);
9029  warned = 1;
9030  }
9031  }
9032 
9033  return new_key;
9034 }
9035 
9036 static VALUE
9037 default_proc_for_compat_func(RB_BLOCK_CALL_FUNC_ARGLIST(hash, _))
9038 {
9039  VALUE key, new_key;
9040 
9041  Check_Type(hash, T_HASH);
9042  rb_check_arity(argc, 2, 2);
9043  key = argv[1];
9044 
9045  if ((new_key = compat_key(key)) != Qnil) {
9046  return rb_hash_lookup(hash, new_key);
9047  }
9048 
9049  return Qnil;
9050 }
9051 
9052 static size_t
9053 gc_stat_internal(VALUE hash_or_sym)
9054 {
9055  rb_objspace_t *objspace = &rb_objspace;
9056  VALUE hash = Qnil, key = Qnil;
9057 
9058  setup_gc_stat_symbols();
9059 
9060  if (RB_TYPE_P(hash_or_sym, T_HASH)) {
9061  hash = hash_or_sym;
9062 
9063  if (NIL_P(RHASH_IFNONE(hash))) {
9064  static VALUE default_proc_for_compat = 0;
9065  if (default_proc_for_compat == 0) { /* TODO: it should be */
9066  default_proc_for_compat = rb_proc_new(default_proc_for_compat_func, Qnil);
9067  rb_gc_register_mark_object(default_proc_for_compat);
9068  }
9069  rb_hash_set_default_proc(hash, default_proc_for_compat);
9070  }
9071  }
9072  else if (SYMBOL_P(hash_or_sym)) {
9073  key = hash_or_sym;
9074  }
9075  else {
9076  rb_raise(rb_eTypeError, "non-hash or symbol argument");
9077  }
9078 
9079 #define SET(name, attr) \
9080  if (key == gc_stat_symbols[gc_stat_sym_##name]) \
9081  return attr; \
9082  else if (hash != Qnil) \
9083  rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
9084 
9085  again:
9086  SET(count, objspace->profile.count);
9087 
9088  /* implementation dependent counters */
9090  SET(heap_sorted_length, heap_pages_sorted_length);
9092  SET(heap_available_slots, objspace_available_slots(objspace));
9093  SET(heap_live_slots, objspace_live_slots(objspace));
9094  SET(heap_free_slots, objspace_free_slots(objspace));
9095  SET(heap_final_slots, heap_pages_final_slots);
9096  SET(heap_marked_slots, objspace->marked_slots);
9097  SET(heap_eden_pages, heap_eden->total_pages);
9098  SET(heap_tomb_pages, heap_tomb->total_pages);
9099  SET(total_allocated_pages, objspace->profile.total_allocated_pages);
9100  SET(total_freed_pages, objspace->profile.total_freed_pages);
9101  SET(total_allocated_objects, objspace->total_allocated_objects);
9102  SET(total_freed_objects, objspace->profile.total_freed_objects);
9103  SET(malloc_increase_bytes, malloc_increase);
9104  SET(malloc_increase_bytes_limit, malloc_limit);
9105 #if USE_RGENGC
9106  SET(minor_gc_count, objspace->profile.minor_gc_count);
9107  SET(major_gc_count, objspace->profile.major_gc_count);
9108  SET(compact_count, objspace->profile.compact_count);
9109  SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
9110  SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
9111  SET(old_objects, objspace->rgengc.old_objects);
9112  SET(old_objects_limit, objspace->rgengc.old_objects_limit);
9113 #if RGENGC_ESTIMATE_OLDMALLOC
9114  SET(oldmalloc_increase_bytes, objspace->rgengc.oldmalloc_increase);
9115  SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
9116 #endif
9117 
9118 #if RGENGC_PROFILE
9119  SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
9120  SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
9121  SET(total_shade_operation_count, objspace->profile.total_shade_operation_count);
9122  SET(total_promoted_count, objspace->profile.total_promoted_count);
9123  SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count);
9124  SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
9125 #endif /* RGENGC_PROFILE */
9126 #endif /* USE_RGENGC */
9127 #undef SET
9128 
9129  if (!NIL_P(key)) { /* matched key should return above */
9130  VALUE new_key;
9131  if ((new_key = compat_key(key)) != Qnil) {
9132  key = new_key;
9133  goto again;
9134  }
9135  rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
9136  }
9137 
9138 #if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
9139  if (hash != Qnil) {
9140  gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types);
9141  gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types);
9142  gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types);
9143  gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types);
9144  gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types);
9145  gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types);
9146  }
9147 #endif
9148 
9149  return 0;
9150 }
9151 
9152 static VALUE
9153 gc_stat(rb_execution_context_t *ec, VALUE self, VALUE arg) // arg is (nil || hash || symbol)
9154 {
9155  if (NIL_P(arg)) {
9156  arg = rb_hash_new();
9157  }
9158  else if (SYMBOL_P(arg)) {
9159  size_t value = gc_stat_internal(arg);
9160  return SIZET2NUM(value);
9161  }
9162  else if (RB_TYPE_P(arg, T_HASH)) {
9163  // ok
9164  }
9165  else {
9166  rb_raise(rb_eTypeError, "non-hash or symbol given");
9167  }
9168 
9169  gc_stat_internal(arg);
9170  return arg;
9171 }
9172 
9173 size_t
9175 {
9176  if (SYMBOL_P(key)) {
9177  size_t value = gc_stat_internal(key);
9178  return value;
9179  }
9180  else {
9181  gc_stat_internal(key);
9182  return 0;
9183  }
9184 }
9185 
9186 static VALUE
9187 gc_stress_get(rb_execution_context_t *ec, VALUE self)
9188 {
9189  rb_objspace_t *objspace = &rb_objspace;
9190  return ruby_gc_stress_mode;
9191 }
9192 
9193 static void
9194 gc_stress_set(rb_objspace_t *objspace, VALUE flag)
9195 {
9196  objspace->flags.gc_stressful = RTEST(flag);
9197  objspace->gc_stress_mode = flag;
9198 }
9199 
9200 static VALUE
9201 gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag)
9202 {
9203  rb_objspace_t *objspace = &rb_objspace;
9204  gc_stress_set(objspace, flag);
9205  return flag;
9206 }
9207 
9208 VALUE
9210 {
9211  rb_objspace_t *objspace = &rb_objspace;
9212  int old = dont_gc;
9213 
9214  dont_gc = FALSE;
9215  return old ? Qtrue : Qfalse;
9216 }
9217 
9218 static VALUE
9219 gc_enable(rb_execution_context_t *ec, VALUE _)
9220 {
9221  return rb_gc_enable();
9222 }
9223 
9224 VALUE
9226 {
9227  rb_objspace_t *objspace = &rb_objspace;
9228  int old = dont_gc;
9229  dont_gc = TRUE;
9230  return old ? Qtrue : Qfalse;
9231 }
9232 
9233 
9234 VALUE
9236 {
9237  rb_objspace_t *objspace = &rb_objspace;
9238  gc_rest(objspace);
9239  return rb_gc_disable_no_rest();
9240 }
9241 
9242 static VALUE
9243 gc_disable(rb_execution_context_t *ec, VALUE _)
9244 {
9245  return rb_gc_disable();
9246 }
9247 
9248 static int
9249 get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
9250 {
9251  char *ptr = getenv(name);
9252  ssize_t val;
9253 
9254  if (ptr != NULL && *ptr) {
9255  size_t unit = 0;
9256  char *end;
9257 #if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
9258  val = strtoll(ptr, &end, 0);
9259 #else
9260  val = strtol(ptr, &end, 0);
9261 #endif
9262  switch (*end) {
9263  case 'k': case 'K':
9264  unit = 1024;
9265  ++end;
9266  break;
9267  case 'm': case 'M':
9268  unit = 1024*1024;
9269  ++end;
9270  break;
9271  case 'g': case 'G':
9272  unit = 1024*1024*1024;
9273  ++end;
9274  break;
9275  }
9276  while (*end && isspace((unsigned char)*end)) end++;
9277  if (*end) {
9278  if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
9279  return 0;
9280  }
9281  if (unit > 0) {
9282  if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
9283  if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr);
9284  return 0;
9285  }
9286  val *= unit;
9287  }
9288  if (val > 0 && (size_t)val > lower_bound) {
9289  if (RTEST(ruby_verbose)) {
9290  fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value);
9291  }
9292  *default_value = (size_t)val;
9293  return 1;
9294  }
9295  else {
9296  if (RTEST(ruby_verbose)) {
9297  fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n",
9298  name, val, *default_value, lower_bound);
9299  }
9300  return 0;
9301  }
9302  }
9303  return 0;
9304 }
9305 
9306 static int
9307 get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero)
9308 {
9309  char *ptr = getenv(name);
9310  double val;
9311 
9312  if (ptr != NULL && *ptr) {
9313  char *end;
9314  val = strtod(ptr, &end);
9315  if (!*ptr || *end) {
9316  if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
9317  return 0;
9318  }
9319 
9320  if (accept_zero && val == 0.0) {
9321  goto accept;
9322  }
9323  else if (val <= lower_bound) {
9324  if (RTEST(ruby_verbose)) {
9325  fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
9326  name, val, *default_value, lower_bound);
9327  }
9328  }
9329  else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */
9330  val > upper_bound) {
9331  if (RTEST(ruby_verbose)) {
9332  fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
9333  name, val, *default_value, upper_bound);
9334  }
9335  }
9336  else {
9337  accept:
9338  if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value);
9339  *default_value = val;
9340  return 1;
9341  }
9342  }
9343  return 0;
9344 }
9345 
9346 static void
9347 gc_set_initial_pages(void)
9348 {
9349  size_t min_pages;
9350  rb_objspace_t *objspace = &rb_objspace;
9351 
9352  min_pages = gc_params.heap_init_slots / HEAP_PAGE_OBJ_LIMIT;
9353  if (min_pages > heap_eden->total_pages) {
9354  heap_add_pages(objspace, heap_eden, min_pages - heap_eden->total_pages);
9355  }
9356 }
9357 
9358 /*
9359  * GC tuning environment variables
9360  *
9361  * * RUBY_GC_HEAP_INIT_SLOTS
9362  * - Initial allocation slots.
9363  * * RUBY_GC_HEAP_FREE_SLOTS
9364  * - Prepare at least this amount of slots after GC.
9365  * - Allocate slots if there are not enough slots.
9366  * * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
9367  * - Allocate slots by this factor.
9368  * - (next slots number) = (current slots number) * (this factor)
9369  * * RUBY_GC_HEAP_GROWTH_MAX_SLOTS (new from 2.1)
9370  * - Allocation rate is limited to this number of slots.
9371  * * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
9372  * - Allocate additional pages when the number of free slots is
9373  * lower than the value (total_slots * (this ratio)).
9374  * * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4)
9375  * - Allocate slots to satisfy this formula:
9376  * free_slots = total_slots * goal_ratio
9377  * - In other words, prepare (total_slots * goal_ratio) free slots.
9378  * - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
9379  * * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4)
9380  * - Allow to free pages when the number of free slots is
9381  * greater than the value (total_slots * (this ratio)).
9382  * * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1)
9383  * - Do full GC when the number of old objects is more than R * N
9384  * where R is this factor and
9385  * N is the number of old objects just after last full GC.
9386  *
9387  * * obsolete
9388  * * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
9389  * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1)
9390  *
9391  * * RUBY_GC_MALLOC_LIMIT
9392  * * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
9393  * * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
9394  *
9395  * * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1)
9396  * * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1)
9397  * * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
9398  */
9399 
9400 void
9402 {
9403  /* RUBY_GC_HEAP_FREE_SLOTS */
9404  if (get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0)) {
9405  /* ok */
9406  }
9407  else if (get_envparam_size("RUBY_FREE_MIN", &gc_params.heap_free_slots, 0)) {
9408  rb_warn("RUBY_FREE_MIN is obsolete. Use RUBY_GC_HEAP_FREE_SLOTS instead.");
9409  }
9410 
9411  /* RUBY_GC_HEAP_INIT_SLOTS */
9412  if (get_envparam_size("RUBY_GC_HEAP_INIT_SLOTS", &gc_params.heap_init_slots, 0)) {
9413  gc_set_initial_pages();
9414  }
9415  else if (get_envparam_size("RUBY_HEAP_MIN_SLOTS", &gc_params.heap_init_slots, 0)) {
9416  rb_warn("RUBY_HEAP_MIN_SLOTS is obsolete. Use RUBY_GC_HEAP_INIT_SLOTS instead.");
9417  gc_set_initial_pages();
9418  }
9419 
9420  get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
9421  get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0);
9422  get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
9423  0.0, 1.0, FALSE);
9424  get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
9425  gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
9426  get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
9428  get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
9429 
9430  get_envparam_size ("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0);
9431  get_envparam_size ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
9432  if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */
9433  gc_params.malloc_limit_max = SIZE_MAX;
9434  }
9435  get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
9436 
9437 #if RGENGC_ESTIMATE_OLDMALLOC
9438  if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
9439  rb_objspace_t *objspace = &rb_objspace;
9440  objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
9441  }
9442  get_envparam_size ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
9443  get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
9444 #endif
9445 }
9446 
9447 void
9448 rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data)
9449 {
9450  rb_objspace_t *objspace = &rb_objspace;
9451 
9452  if (is_markable_object(objspace, obj)) {
9453  struct mark_func_data_struct mfd;
9454  mfd.mark_func = func;
9455  mfd.data = data;
9456  PUSH_MARK_FUNC_DATA(&mfd);
9457  gc_mark_children(objspace, obj);
9459  }
9460 }
9461 
9463  const char *category;
9464  void (*func)(const char *category, VALUE, void *);
9465  void *data;
9466 };
9467 
9468 static void
9469 root_objects_from(VALUE obj, void *ptr)
9470 {
9471  const struct root_objects_data *data = (struct root_objects_data *)ptr;
9472  (*data->func)(data->category, obj, data->data);
9473 }
9474 
9475 void
9476 rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data)
9477 {
9478  rb_objspace_t *objspace = &rb_objspace;
9479  objspace_reachable_objects_from_root(objspace, func, passing_data);
9480 }
9481 
9482 static void
9483 objspace_reachable_objects_from_root(rb_objspace_t *objspace, void (func)(const char *category, VALUE, void *), void *passing_data)
9484 {
9485  struct root_objects_data data;
9486  struct mark_func_data_struct mfd;
9487 
9488  data.func = func;
9489  data.data = passing_data;
9490 
9491  mfd.mark_func = root_objects_from;
9492  mfd.data = &data;
9493 
9494  PUSH_MARK_FUNC_DATA(&mfd);
9495  gc_mark_roots(objspace, &data.category);
9497 }
9498 
9499 /*
9500  ------------------------ Extended allocator ------------------------
9501 */
9502 
9505  const char *fmt;
9507 };
9508 
9509 static void *
9510 gc_vraise(void *ptr)
9511 {
9512  struct gc_raise_tag *argv = ptr;
9513  rb_vraise(argv->exc, argv->fmt, *argv->ap);
9515 }
9516 
9517 static void
9518 gc_raise(VALUE exc, const char *fmt, ...)
9519 {
9520  va_list ap;
9521  va_start(ap, fmt);
9522  struct gc_raise_tag argv = {
9523  exc, fmt, &ap,
9524  };
9525 
9526  if (ruby_thread_has_gvl_p()) {
9527  gc_vraise(&argv);
9528  UNREACHABLE;
9529  }
9530  else if (ruby_native_thread_p()) {
9531  rb_thread_call_with_gvl(gc_vraise, &argv);
9532  UNREACHABLE;
9533  }
9534  else {
9535  /* Not in a ruby thread */
9536  fprintf(stderr, "%s", "[FATAL] ");
9537  vfprintf(stderr, fmt, ap);
9538  abort();
9539  }
9540 
9541  va_end(ap);
9542 }
9543 
9544 static void objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t size);
9545 
9546 static void
9547 negative_size_allocation_error(const char *msg)
9548 {
9549  gc_raise(rb_eNoMemError, "%s", msg);
9550 }
9551 
9552 static void *
9553 ruby_memerror_body(void *dummy)
9554 {
9555  rb_memerror();
9556  return 0;
9557 }
9558 
9559 static void
9560 ruby_memerror(void)
9561 {
9562  if (ruby_thread_has_gvl_p()) {
9563  rb_memerror();
9564  }
9565  else {
9566  if (ruby_native_thread_p()) {
9567  rb_thread_call_with_gvl(ruby_memerror_body, 0);
9568  }
9569  else {
9570  /* no ruby thread */
9571  fprintf(stderr, "[FATAL] failed to allocate memory\n");
9572  exit(EXIT_FAILURE);
9573  }
9574  }
9575 }
9576 
9577 void
9579 {
9581  rb_objspace_t *objspace = rb_objspace_of(rb_ec_vm_ptr(ec));
9582  VALUE exc;
9583 
9584  if (0) {
9585  // Print out pid, sleep, so you can attach debugger to see what went wrong:
9586  fprintf(stderr, "rb_memerror pid=%"PRI_PIDT_PREFIX"d\n", getpid());
9587  sleep(60);
9588  }
9589 
9590  if (during_gc) gc_exit(objspace, "rb_memerror");
9591 
9592  exc = nomem_error;
9593  if (!exc ||
9595  fprintf(stderr, "[FATAL] failed to allocate memory\n");
9596  exit(EXIT_FAILURE);
9597  }
9598  if (rb_ec_raised_p(ec, RAISED_NOMEMORY)) {
9599  rb_ec_raised_clear(ec);
9600  }
9601  else {
9604  }
9605  ec->errinfo = exc;
9606  EC_JUMP_TAG(ec, TAG_RAISE);
9607 }
9608 
9609 void *
9610 rb_aligned_malloc(size_t alignment, size_t size)
9611 {
9612  void *res;
9613 
9614 #if defined __MINGW32__
9615  res = __mingw_aligned_malloc(size, alignment);
9616 #elif defined _WIN32
9617  void *_aligned_malloc(size_t, size_t);
9618  res = _aligned_malloc(size, alignment);
9619 #elif defined(HAVE_POSIX_MEMALIGN)
9620  if (posix_memalign(&res, alignment, size) == 0) {
9621  return res;
9622  }
9623  else {
9624  return NULL;
9625  }
9626 #elif defined(HAVE_MEMALIGN)
9627  res = memalign(alignment, size);
9628 #else
9629  char* aligned;
9630  res = malloc(alignment + size + sizeof(void*));
9631  aligned = (char*)res + alignment + sizeof(void*);
9632  aligned -= ((VALUE)aligned & (alignment - 1));
9633  ((void**)aligned)[-1] = res;
9634  res = (void*)aligned;
9635 #endif
9636 
9637  /* alignment must be a power of 2 */
9638  GC_ASSERT(((alignment - 1) & alignment) == 0);
9639  GC_ASSERT(alignment % sizeof(void*) == 0);
9640  return res;
9641 }
9642 
9643 static void
9644 rb_aligned_free(void *ptr)
9645 {
9646 #if defined __MINGW32__
9647  __mingw_aligned_free(ptr);
9648 #elif defined _WIN32
9649  _aligned_free(ptr);
9650 #elif defined(HAVE_MEMALIGN) || defined(HAVE_POSIX_MEMALIGN)
9651  free(ptr);
9652 #else
9653  free(((void**)ptr)[-1]);
9654 #endif
9655 }
9656 
9657 static inline size_t
9658 objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint)
9659 {
9660 #ifdef HAVE_MALLOC_USABLE_SIZE
9661  return malloc_usable_size(ptr);
9662 #else
9663  return hint;
9664 #endif
9665 }
9666 
9671 };
9672 
9673 static inline void
9674 atomic_sub_nounderflow(size_t *var, size_t sub)
9675 {
9676  if (sub == 0) return;
9677 
9678  while (1) {
9679  size_t val = *var;
9680  if (val < sub) sub = val;
9681  if (ATOMIC_SIZE_CAS(*var, val, val-sub) == val) break;
9682  }
9683 }
9684 
9685 static void
9686 objspace_malloc_gc_stress(rb_objspace_t *objspace)
9687 {
9691 
9693  reason |= GPR_FLAG_FULL_MARK;
9694  }
9695  garbage_collect_with_gvl(objspace, reason);
9696  }
9697 }
9698 
9699 static void
9700 objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
9701 {
9702  if (new_size > old_size) {
9703  ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size);
9704 #if RGENGC_ESTIMATE_OLDMALLOC
9705  ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size);
9706 #endif
9707  }
9708  else {
9709  atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
9710 #if RGENGC_ESTIMATE_OLDMALLOC
9711  atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size);
9712 #endif
9713  }
9714 
9715  if (type == MEMOP_TYPE_MALLOC) {
9716  retry:
9719  gc_rest(objspace); /* gc_rest can reduce malloc_increase */
9720  goto retry;
9721  }
9722  garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
9723  }
9724  }
9725 
9726 #if MALLOC_ALLOCATED_SIZE
9727  if (new_size >= old_size) {
9728  ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size);
9729  }
9730  else {
9731  size_t dec_size = old_size - new_size;
9732  size_t allocated_size = objspace->malloc_params.allocated_size;
9733 
9734 #if MALLOC_ALLOCATED_SIZE_CHECK
9735  if (allocated_size < dec_size) {
9736  rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size.");
9737  }
9738 #endif
9739  atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size);
9740  }
9741 
9742  if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %d, old_size: %d\n",
9743  mem,
9744  type == MEMOP_TYPE_MALLOC ? "malloc" :
9745  type == MEMOP_TYPE_FREE ? "free " :
9746  type == MEMOP_TYPE_REALLOC ? "realloc": "error",
9747  (int)new_size, (int)old_size);
9748 
9749  switch (type) {
9750  case MEMOP_TYPE_MALLOC:
9751  ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
9752  break;
9753  case MEMOP_TYPE_FREE:
9754  {
9755  size_t allocations = objspace->malloc_params.allocations;
9756  if (allocations > 0) {
9757  atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1);
9758  }
9759 #if MALLOC_ALLOCATED_SIZE_CHECK
9760  else {
9761  GC_ASSERT(objspace->malloc_params.allocations > 0);
9762  }
9763 #endif
9764  }
9765  break;
9766  case MEMOP_TYPE_REALLOC: /* ignore */ break;
9767  }
9768 #endif
9769 }
9770 
9771 struct malloc_obj_info { /* 4 words */
9772  size_t size;
9773 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
9774  size_t gen;
9775  const char *file;
9776  size_t line;
9777 #endif
9778 };
9779 
9780 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
9781 const char *ruby_malloc_info_file;
9782 int ruby_malloc_info_line;
9783 #endif
9784 
9785 static inline size_t
9786 objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
9787 {
9788  if (size == 0) size = 1;
9789 
9790 #if CALC_EXACT_MALLOC_SIZE
9791  size += sizeof(struct malloc_obj_info);
9792 #endif
9793 
9794  return size;
9795 }
9796 
9797 static inline void *
9798 objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
9799 {
9800  size = objspace_malloc_size(objspace, mem, size);
9801  objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC);
9802 
9803 #if CALC_EXACT_MALLOC_SIZE
9804  {
9805  struct malloc_obj_info *info = mem;
9806  info->size = size;
9807 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
9808  info->gen = objspace->profile.count;
9809  info->file = ruby_malloc_info_file;
9810  info->line = info->file ? ruby_malloc_info_line : 0;
9811 #else
9812  info->file = NULL;
9813 #endif
9814  mem = info + 1;
9815  }
9816 #endif
9817 
9818  return mem;
9819 }
9820 
9821 #define TRY_WITH_GC(alloc) do { \
9822  objspace_malloc_gc_stress(objspace); \
9823  if (!(alloc) && \
9824  (!garbage_collect_with_gvl(objspace, GPR_FLAG_FULL_MARK | \
9825  GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP | \
9826  GPR_FLAG_MALLOC) || \
9827  !(alloc))) { \
9828  ruby_memerror(); \
9829  } \
9830  } while (0)
9831 
9832 /* these shouldn't be called directly.
9833  * objspace_* functinos do not check allocation size.
9834  */
9835 static void *
9836 objspace_xmalloc0(rb_objspace_t *objspace, size_t size)
9837 {
9838  void *mem;
9839 
9840  size = objspace_malloc_prepare(objspace, size);
9841  TRY_WITH_GC(mem = malloc(size));
9842  RB_DEBUG_COUNTER_INC(heap_xmalloc);
9843  return objspace_malloc_fixup(objspace, mem, size);
9844 }
9845 
9846 static inline size_t
9847 xmalloc2_size(const size_t count, const size_t elsize)
9848 {
9849  return size_mul_or_raise(count, elsize, rb_eArgError);
9850 }
9851 
9852 static void *
9853 objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t old_size)
9854 {
9855  void *mem;
9856 
9857  if (!ptr) return objspace_xmalloc0(objspace, new_size);
9858 
9859  /*
9860  * The behavior of realloc(ptr, 0) is implementation defined.
9861  * Therefore we don't use realloc(ptr, 0) for portability reason.
9862  * see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
9863  */
9864  if (new_size == 0) {
9865  if ((mem = objspace_xmalloc0(objspace, 0)) != NULL) {
9866  /*
9867  * - OpenBSD's malloc(3) man page says that when 0 is passed, it
9868  * returns a non-NULL pointer to an access-protected memory page.
9869  * The returned pointer cannot be read / written at all, but
9870  * still be a valid argument of free().
9871  *
9872  * https://man.openbsd.org/malloc.3
9873  *
9874  * - Linux's malloc(3) man page says that it _might_ perhaps return
9875  * a non-NULL pointer when its argument is 0. That return value
9876  * is safe (and is expected) to be passed to free().
9877  *
9878  * http://man7.org/linux/man-pages/man3/malloc.3.html
9879  *
9880  * - As I read the implementation jemalloc's malloc() returns fully
9881  * normal 16 bytes memory region when its argument is 0.
9882  *
9883  * - As I read the implementation musl libc's malloc() returns
9884  * fully normal 32 bytes memory region when its argument is 0.
9885  *
9886  * - Other malloc implementations can also return non-NULL.
9887  */
9888  objspace_xfree(objspace, ptr, old_size);
9889  return mem;
9890  }
9891  else {
9892  /*
9893  * It is dangerous to return NULL here, because that could lead to
9894  * RCE. Fallback to 1 byte instead of zero.
9895  *
9896  * https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932
9897  */
9898  new_size = 1;
9899  }
9900  }
9901 
9902 #if CALC_EXACT_MALLOC_SIZE
9903  {
9904  struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
9905  new_size += sizeof(struct malloc_obj_info);
9906  ptr = info;
9907  old_size = info->size;
9908  }
9909 #endif
9910 
9911  old_size = objspace_malloc_size(objspace, ptr, old_size);
9912  TRY_WITH_GC(mem = realloc(ptr, new_size));
9913  new_size = objspace_malloc_size(objspace, mem, new_size);
9914 
9915 #if CALC_EXACT_MALLOC_SIZE
9916  {
9917  struct malloc_obj_info *info = mem;
9918  info->size = new_size;
9919  mem = info + 1;
9920  }
9921 #endif
9922 
9923  objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
9924 
9925  RB_DEBUG_COUNTER_INC(heap_xrealloc);
9926  return mem;
9927 }
9928 
9929 #if CALC_EXACT_MALLOC_SIZE && USE_GC_MALLOC_OBJ_INFO_DETAILS
9930 
9931 #define MALLOC_INFO_GEN_SIZE 100
9932 #define MALLOC_INFO_SIZE_SIZE 10
9933 static size_t malloc_info_gen_cnt[MALLOC_INFO_GEN_SIZE];
9934 static size_t malloc_info_gen_size[MALLOC_INFO_GEN_SIZE];
9935 static size_t malloc_info_size[MALLOC_INFO_SIZE_SIZE+1];
9936 static st_table *malloc_info_file_table;
9937 
9938 static int
9939 mmalloc_info_file_i(st_data_t key, st_data_t val, st_data_t dmy)
9940 {
9941  const char *file = (void *)key;
9942  const size_t *data = (void *)val;
9943 
9944  fprintf(stderr, "%s\t%d\t%d\n", file, (int)data[0], (int)data[1]);
9945 
9946  return ST_CONTINUE;
9947 }
9948 
9949 __attribute__((destructor))
9950 void
9952 {
9953  int i;
9954 
9955  fprintf(stderr, "* malloc_info gen statistics\n");
9956  for (i=0; i<MALLOC_INFO_GEN_SIZE; i++) {
9957  if (i == MALLOC_INFO_GEN_SIZE-1) {
9958  fprintf(stderr, "more\t%d\t%d\n", (int)malloc_info_gen_cnt[i], (int)malloc_info_gen_size[i]);
9959  }
9960  else {
9961  fprintf(stderr, "%d\t%d\t%d\n", i, (int)malloc_info_gen_cnt[i], (int)malloc_info_gen_size[i]);
9962  }
9963  }
9964 
9965  fprintf(stderr, "* malloc_info size statistics\n");
9966  for (i=0; i<MALLOC_INFO_SIZE_SIZE; i++) {
9967  int s = 16 << i;
9968  fprintf(stderr, "%d\t%d\n", (int)s, (int)malloc_info_size[i]);
9969  }
9970  fprintf(stderr, "more\t%d\n", (int)malloc_info_size[i]);
9971 
9972  if (malloc_info_file_table) {
9973  fprintf(stderr, "* malloc_info file statistics\n");
9974  st_foreach(malloc_info_file_table, mmalloc_info_file_i, 0);
9975  }
9976 }
9977 #else
9978 void
9980 {
9981 }
9982 #endif
9983 
9984 static void
9985 objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t old_size)
9986 {
9987  if (!ptr) {
9988  /*
9989  * ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since
9990  * its first version. We would better follow.
9991  */
9992  return;
9993  }
9994 #if CALC_EXACT_MALLOC_SIZE
9995  struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
9996  ptr = info;
9997  old_size = info->size;
9998 
9999 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
10000  {
10001  int gen = (int)(objspace->profile.count - info->gen);
10002  int gen_index = gen >= MALLOC_INFO_GEN_SIZE ? MALLOC_INFO_GEN_SIZE-1 : gen;
10003  int i;
10004 
10005  malloc_info_gen_cnt[gen_index]++;
10006  malloc_info_gen_size[gen_index] += info->size;
10007 
10008  for (i=0; i<MALLOC_INFO_SIZE_SIZE; i++) {
10009  size_t s = 16 << i;
10010  if (info->size <= s) {
10011  malloc_info_size[i]++;
10012  goto found;
10013  }
10014  }
10015  malloc_info_size[i]++;
10016  found:;
10017 
10018  {
10019  st_data_t key = (st_data_t)info->file;
10020  size_t *data;
10021 
10022  if (malloc_info_file_table == NULL) {
10023  malloc_info_file_table = st_init_numtable_with_size(1024);
10024  }
10025  if (st_lookup(malloc_info_file_table, key, (st_data_t *)&data)) {
10026  /* hit */
10027  }
10028  else {
10029  data = malloc(xmalloc2_size(2, sizeof(size_t)));
10030  if (data == NULL) rb_bug("objspace_xfree: can not allocate memory");
10031  data[0] = data[1] = 0;
10032  st_insert(malloc_info_file_table, key, (st_data_t)data);
10033  }
10034  data[0] ++;
10035  data[1] += info->size;
10036  };
10037 #if 0 /* verbose output */
10038  if (gen >= 2) {
10039  if (info->file) {
10040  fprintf(stderr, "free - size:%d, gen:%d, pos: %s:%d\n", (int)info->size, gen, info->file, (int)info->line);
10041  }
10042  else {
10043  fprintf(stderr, "free - size:%d, gen:%d\n", (int)info->size, gen);
10044  }
10045  }
10046 #endif
10047  }
10048 #endif
10049 #endif
10050  old_size = objspace_malloc_size(objspace, ptr, old_size);
10051 
10052  free(ptr);
10053  RB_DEBUG_COUNTER_INC(heap_xfree);
10054 
10055  objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE);
10056 }
10057 
10058 static void *
10059 ruby_xmalloc0(size_t size)
10060 {
10061  return objspace_xmalloc0(&rb_objspace, size);
10062 }
10063 
10064 void *
10066 {
10067  if ((ssize_t)size < 0) {
10068  negative_size_allocation_error("too large allocation size");
10069  }
10070  return ruby_xmalloc0(size);
10071 }
10072 
10073 void
10074 ruby_malloc_size_overflow(size_t count, size_t elsize)
10075 {
10077  "malloc: possible integer overflow (%"PRIuSIZE"*%"PRIuSIZE")",
10078  count, elsize);
10079 }
10080 
10081 void *
10082 ruby_xmalloc2_body(size_t n, size_t size)
10083 {
10084  return objspace_xmalloc0(&rb_objspace, xmalloc2_size(n, size));
10085 }
10086 
10087 static void *
10088 objspace_xcalloc(rb_objspace_t *objspace, size_t size)
10089 {
10090  void *mem;
10091 
10092  size = objspace_malloc_prepare(objspace, size);
10093  TRY_WITH_GC(mem = calloc1(size));
10094  return objspace_malloc_fixup(objspace, mem, size);
10095 }
10096 
10097 void *
10098 ruby_xcalloc_body(size_t n, size_t size)
10099 {
10100  return objspace_xcalloc(&rb_objspace, xmalloc2_size(n, size));
10101 }
10102 
10103 #ifdef ruby_sized_xrealloc
10104 #undef ruby_sized_xrealloc
10105 #endif
10106 void *
10107 ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size)
10108 {
10109  if ((ssize_t)new_size < 0) {
10110  negative_size_allocation_error("too large allocation size");
10111  }
10112 
10113  return objspace_xrealloc(&rb_objspace, ptr, new_size, old_size);
10114 }
10115 
10116 void *
10117 ruby_xrealloc_body(void *ptr, size_t new_size)
10118 {
10119  return ruby_sized_xrealloc(ptr, new_size, 0);
10120 }
10121 
10122 #ifdef ruby_sized_xrealloc2
10123 #undef ruby_sized_xrealloc2
10124 #endif
10125 void *
10126 ruby_sized_xrealloc2(void *ptr, size_t n, size_t size, size_t old_n)
10127 {
10128  size_t len = xmalloc2_size(n, size);
10129  return objspace_xrealloc(&rb_objspace, ptr, len, old_n * size);
10130 }
10131 
10132 void *
10133 ruby_xrealloc2_body(void *ptr, size_t n, size_t size)
10134 {
10135  return ruby_sized_xrealloc2(ptr, n, size, 0);
10136 }
10137 
10138 #ifdef ruby_sized_xfree
10139 #undef ruby_sized_xfree
10140 #endif
10141 void
10142 ruby_sized_xfree(void *x, size_t size)
10143 {
10144  if (x) {
10145  objspace_xfree(&rb_objspace, x, size);
10146  }
10147 }
10148 
10149 void
10150 ruby_xfree(void *x)
10151 {
10152  ruby_sized_xfree(x, 0);
10153 }
10154 
10155 void *
10156 rb_xmalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
10157 {
10158  size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
10159  return ruby_xmalloc(w);
10160 }
10161 
10162 void *
10163 rb_xrealloc_mul_add(const void *p, size_t x, size_t y, size_t z) /* x * y + z */
10164 {
10165  size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
10166  return ruby_xrealloc((void *)p, w);
10167 }
10168 
10169 void *
10170 rb_xmalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
10171 {
10172  size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
10173  return ruby_xmalloc(u);
10174 }
10175 
10176 void *
10177 rb_xcalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
10178 {
10179  size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
10180  return ruby_xcalloc(u, 1);
10181 }
10182 
10183 /* Mimic ruby_xmalloc, but need not rb_objspace.
10184  * should return pointer suitable for ruby_xfree
10185  */
10186 void *
10188 {
10189  void *mem;
10190 #if CALC_EXACT_MALLOC_SIZE
10191  size += sizeof(struct malloc_obj_info);
10192 #endif
10193  mem = malloc(size);
10194 #if CALC_EXACT_MALLOC_SIZE
10195  if (!mem) {
10196  return NULL;
10197  }
10198  else
10199  /* set 0 for consistency of allocated_size/allocations */
10200  {
10201  struct malloc_obj_info *info = mem;
10202  info->size = 0;
10203 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
10204  info->gen = 0;
10205  info->file = NULL;
10206  info->line = 0;
10207 #else
10208  info->file = NULL;
10209 #endif
10210  mem = info + 1;
10211  }
10212 #endif
10213  return mem;
10214 }
10215 
10216 void
10218 {
10219 #if CALC_EXACT_MALLOC_SIZE
10220  struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
10221  ptr = info;
10222 #endif
10223  free(ptr);
10224 }
10225 
10226 void *
10227 rb_alloc_tmp_buffer_with_count(volatile VALUE *store, size_t size, size_t cnt)
10228 {
10229  void *ptr;
10230  VALUE imemo;
10231  rb_imemo_tmpbuf_t *tmpbuf;
10232 
10233  /* Keep the order; allocate an empty imemo first then xmalloc, to
10234  * get rid of potential memory leak */
10235  imemo = rb_imemo_tmpbuf_auto_free_maybe_mark_buffer(NULL, 0);
10236  *store = imemo;
10237  ptr = ruby_xmalloc0(size);
10238  tmpbuf = (rb_imemo_tmpbuf_t *)imemo;
10239  tmpbuf->ptr = ptr;
10240  tmpbuf->cnt = cnt;
10241  return ptr;
10242 }
10243 
10244 void *
10245 rb_alloc_tmp_buffer(volatile VALUE *store, long len)
10246 {
10247  long cnt;
10248 
10249  if (len < 0 || (cnt = (long)roomof(len, sizeof(VALUE))) < 0) {
10250  rb_raise(rb_eArgError, "negative buffer size (or size too big)");
10251  }
10252 
10253  return rb_alloc_tmp_buffer_with_count(store, len, cnt);
10254 }
10255 
10256 void
10257 rb_free_tmp_buffer(volatile VALUE *store)
10258 {
10260  if (s) {
10261  void *ptr = ATOMIC_PTR_EXCHANGE(s->ptr, 0);
10262  s->cnt = 0;
10263  ruby_xfree(ptr);
10264  }
10265 }
10266 
10267 #if MALLOC_ALLOCATED_SIZE
10268 /*
10269  * call-seq:
10270  * GC.malloc_allocated_size -> Integer
10271  *
10272  * Returns the size of memory allocated by malloc().
10273  *
10274  * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
10275  */
10276 
10277 static VALUE
10278 gc_malloc_allocated_size(VALUE self)
10279 {
10280  return UINT2NUM(rb_objspace.malloc_params.allocated_size);
10281 }
10282 
10283 /*
10284  * call-seq:
10285  * GC.malloc_allocations -> Integer
10286  *
10287  * Returns the number of malloc() allocations.
10288  *
10289  * Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
10290  */
10291 
10292 static VALUE
10293 gc_malloc_allocations(VALUE self)
10294 {
10295  return UINT2NUM(rb_objspace.malloc_params.allocations);
10296 }
10297 #endif
10298 
10299 void
10301 {
10302  rb_objspace_t *objspace = &rb_objspace;
10303  if (diff > 0) {
10304  objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
10305  }
10306  else if (diff < 0) {
10307  objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
10308  }
10309 }
10310 
10311 /*
10312  ------------------------------ WeakMap ------------------------------
10313 */
10314 
10315 struct weakmap {
10316  st_table *obj2wmap; /* obj -> [ref,...] */
10317  st_table *wmap2obj; /* ref -> obj */
10318  VALUE final;
10319 };
10320 
10321 #define WMAP_DELETE_DEAD_OBJECT_IN_MARK 0
10322 
10323 #if WMAP_DELETE_DEAD_OBJECT_IN_MARK
10324 static int
10325 wmap_mark_map(st_data_t key, st_data_t val, st_data_t arg)
10326 {
10327  rb_objspace_t *objspace = (rb_objspace_t *)arg;
10328  VALUE obj = (VALUE)val;
10329  if (!is_live_object(objspace, obj)) return ST_DELETE;
10330  return ST_CONTINUE;
10331 }
10332 #endif
10333 
10334 static void
10335 wmap_compact(void *ptr)
10336 {
10337  struct weakmap *w = ptr;
10340  w->final = rb_gc_location(w->final);
10341 }
10342 
10343 static void
10344 wmap_mark(void *ptr)
10345 {
10346  struct weakmap *w = ptr;
10347 #if WMAP_DELETE_DEAD_OBJECT_IN_MARK
10348  if (w->obj2wmap) st_foreach(w->obj2wmap, wmap_mark_map, (st_data_t)&rb_objspace);
10349 #endif
10351 }
10352 
10353 static int
10354 wmap_free_map(st_data_t key, st_data_t val, st_data_t arg)
10355 {
10356  VALUE *ptr = (VALUE *)val;
10357  ruby_sized_xfree(ptr, (ptr[0] + 1) * sizeof(VALUE));
10358  return ST_CONTINUE;
10359 }
10360 
10361 static void
10362 wmap_free(void *ptr)
10363 {
10364  struct weakmap *w = ptr;
10365  st_foreach(w->obj2wmap, wmap_free_map, 0);
10366  st_free_table(w->obj2wmap);
10367  st_free_table(w->wmap2obj);
10368 }
10369 
10370 static int
10371 wmap_memsize_map(st_data_t key, st_data_t val, st_data_t arg)
10372 {
10373  VALUE *ptr = (VALUE *)val;
10374  *(size_t *)arg += (ptr[0] + 1) * sizeof(VALUE);
10375  return ST_CONTINUE;
10376 }
10377 
10378 static size_t
10379 wmap_memsize(const void *ptr)
10380 {
10381  size_t size;
10382  const struct weakmap *w = ptr;
10383  size = sizeof(*w);
10384  size += st_memsize(w->obj2wmap);
10385  size += st_memsize(w->wmap2obj);
10386  st_foreach(w->obj2wmap, wmap_memsize_map, (st_data_t)&size);
10387  return size;
10388 }
10389 
10390 static const rb_data_type_t weakmap_type = {
10391  "weakmap",
10392  {
10393  wmap_mark,
10394  wmap_free,
10395  wmap_memsize,
10396  wmap_compact,
10397  },
10399 };
10400 
10401 extern const struct st_hash_type rb_hashtype_ident;
10402 static VALUE wmap_finalize(RB_BLOCK_CALL_FUNC_ARGLIST(objid, self));
10403 
10404 static VALUE
10405 wmap_allocate(VALUE klass)
10406 {
10407  struct weakmap *w;
10408  VALUE obj = TypedData_Make_Struct(klass, struct weakmap, &weakmap_type, w);
10411  w->final = rb_func_lambda_new(wmap_finalize, obj, 1, 1);
10412  return obj;
10413 }
10414 
10415 static int
10416 wmap_live_p(rb_objspace_t *objspace, VALUE obj)
10417 {
10418  if (!FL_ABLE(obj)) return TRUE;
10419  if (!is_id_value(objspace, obj)) return FALSE;
10420  if (!is_live_object(objspace, obj)) return FALSE;
10421  return TRUE;
10422 }
10423 
10424 static int
10425 wmap_final_func(st_data_t *key, st_data_t *value, st_data_t arg, int existing)
10426 {
10427  VALUE wmap, *ptr, size, i, j;
10428  if (!existing) return ST_STOP;
10429  wmap = (VALUE)arg, ptr = (VALUE *)*value;
10430  for (i = j = 1, size = ptr[0]; i <= size; ++i) {
10431  if (ptr[i] != wmap) {
10432  ptr[j++] = ptr[i];
10433  }
10434  }
10435  if (j == 1) {
10436  ruby_sized_xfree(ptr, i * sizeof(VALUE));
10437  return ST_DELETE;
10438  }
10439  if (j < i) {
10440  SIZED_REALLOC_N(ptr, VALUE, j + 1, i);
10441  ptr[0] = j;
10442  *value = (st_data_t)ptr;
10443  }
10444  return ST_CONTINUE;
10445 }
10446 
10447 /* :nodoc: */
10448 static VALUE
10449 wmap_finalize(RB_BLOCK_CALL_FUNC_ARGLIST(objid, self))
10450 {
10451  st_data_t orig, wmap, data;
10452  VALUE obj, *rids, i, size;
10453  struct weakmap *w;
10454 
10455  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10456  /* Get reference from object id. */
10457  if ((obj = id2ref_obj_tbl(&rb_objspace, objid)) == Qundef) {
10458  rb_bug("wmap_finalize: objid is not found.");
10459  }
10460 
10461  /* obj is original referenced object and/or weak reference. */
10462  orig = (st_data_t)obj;
10463  if (st_delete(w->obj2wmap, &orig, &data)) {
10464  rids = (VALUE *)data;
10465  size = *rids++;
10466  for (i = 0; i < size; ++i) {
10467  wmap = (st_data_t)rids[i];
10468  st_delete(w->wmap2obj, &wmap, NULL);
10469  }
10470  ruby_sized_xfree((VALUE *)data, (size + 1) * sizeof(VALUE));
10471  }
10472 
10473  wmap = (st_data_t)obj;
10474  if (st_delete(w->wmap2obj, &wmap, &orig)) {
10475  wmap = (st_data_t)obj;
10476  st_update(w->obj2wmap, orig, wmap_final_func, wmap);
10477  }
10478  return self;
10479 }
10480 
10484 };
10485 
10486 static int
10487 wmap_inspect_i(st_data_t key, st_data_t val, st_data_t arg)
10488 {
10489  VALUE str = (VALUE)arg;
10490  VALUE k = (VALUE)key, v = (VALUE)val;
10491 
10492  if (RSTRING_PTR(str)[0] == '#') {
10493  rb_str_cat2(str, ", ");
10494  }
10495  else {
10496  rb_str_cat2(str, ": ");
10497  RSTRING_PTR(str)[0] = '#';
10498  }
10499  k = SPECIAL_CONST_P(k) ? rb_inspect(k) : rb_any_to_s(k);
10500  rb_str_append(str, k);
10501  rb_str_cat2(str, " => ");
10503  rb_str_append(str, v);
10504 
10505  return ST_CONTINUE;
10506 }
10507 
10508 static VALUE
10509 wmap_inspect(VALUE self)
10510 {
10511  VALUE str;
10512  VALUE c = rb_class_name(CLASS_OF(self));
10513  struct weakmap *w;
10514 
10515  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10516  str = rb_sprintf("-<%"PRIsVALUE":%p", c, (void *)self);
10517  if (w->wmap2obj) {
10518  st_foreach(w->wmap2obj, wmap_inspect_i, str);
10519  }
10520  RSTRING_PTR(str)[0] = '#';
10521  rb_str_cat2(str, ">");
10522  return str;
10523 }
10524 
10525 static int
10526 wmap_each_i(st_data_t key, st_data_t val, st_data_t arg)
10527 {
10528  rb_objspace_t *objspace = (rb_objspace_t *)arg;
10529  VALUE obj = (VALUE)val;
10530  if (wmap_live_p(objspace, obj)) {
10531  rb_yield_values(2, (VALUE)key, obj);
10532  }
10533  return ST_CONTINUE;
10534 }
10535 
10536 /* Iterates over keys and objects in a weakly referenced object */
10537 static VALUE
10538 wmap_each(VALUE self)
10539 {
10540  struct weakmap *w;
10541  rb_objspace_t *objspace = &rb_objspace;
10542 
10543  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10544  st_foreach(w->wmap2obj, wmap_each_i, (st_data_t)objspace);
10545  return self;
10546 }
10547 
10548 static int
10549 wmap_each_key_i(st_data_t key, st_data_t val, st_data_t arg)
10550 {
10551  rb_objspace_t *objspace = (rb_objspace_t *)arg;
10552  VALUE obj = (VALUE)val;
10553  if (wmap_live_p(objspace, obj)) {
10554  rb_yield((VALUE)key);
10555  }
10556  return ST_CONTINUE;
10557 }
10558 
10559 /* Iterates over keys and objects in a weakly referenced object */
10560 static VALUE
10561 wmap_each_key(VALUE self)
10562 {
10563  struct weakmap *w;
10564  rb_objspace_t *objspace = &rb_objspace;
10565 
10566  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10567  st_foreach(w->wmap2obj, wmap_each_key_i, (st_data_t)objspace);
10568  return self;
10569 }
10570 
10571 static int
10572 wmap_each_value_i(st_data_t key, st_data_t val, st_data_t arg)
10573 {
10574  rb_objspace_t *objspace = (rb_objspace_t *)arg;
10575  VALUE obj = (VALUE)val;
10576  if (wmap_live_p(objspace, obj)) {
10577  rb_yield(obj);
10578  }
10579  return ST_CONTINUE;
10580 }
10581 
10582 /* Iterates over keys and objects in a weakly referenced object */
10583 static VALUE
10584 wmap_each_value(VALUE self)
10585 {
10586  struct weakmap *w;
10587  rb_objspace_t *objspace = &rb_objspace;
10588 
10589  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10590  st_foreach(w->wmap2obj, wmap_each_value_i, (st_data_t)objspace);
10591  return self;
10592 }
10593 
10594 static int
10595 wmap_keys_i(st_data_t key, st_data_t val, st_data_t arg)
10596 {
10597  struct wmap_iter_arg *argp = (struct wmap_iter_arg *)arg;
10598  rb_objspace_t *objspace = argp->objspace;
10599  VALUE ary = argp->value;
10600  VALUE obj = (VALUE)val;
10601  if (wmap_live_p(objspace, obj)) {
10602  rb_ary_push(ary, (VALUE)key);
10603  }
10604  return ST_CONTINUE;
10605 }
10606 
10607 /* Iterates over keys and objects in a weakly referenced object */
10608 static VALUE
10609 wmap_keys(VALUE self)
10610 {
10611  struct weakmap *w;
10612  struct wmap_iter_arg args;
10613 
10614  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10615  args.objspace = &rb_objspace;
10616  args.value = rb_ary_new();
10617  st_foreach(w->wmap2obj, wmap_keys_i, (st_data_t)&args);
10618  return args.value;
10619 }
10620 
10621 static int
10622 wmap_values_i(st_data_t key, st_data_t val, st_data_t arg)
10623 {
10624  struct wmap_iter_arg *argp = (struct wmap_iter_arg *)arg;
10625  rb_objspace_t *objspace = argp->objspace;
10626  VALUE ary = argp->value;
10627  VALUE obj = (VALUE)val;
10628  if (wmap_live_p(objspace, obj)) {
10629  rb_ary_push(ary, obj);
10630  }
10631  return ST_CONTINUE;
10632 }
10633 
10634 /* Iterates over values and objects in a weakly referenced object */
10635 static VALUE
10636 wmap_values(VALUE self)
10637 {
10638  struct weakmap *w;
10639  struct wmap_iter_arg args;
10640 
10641  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10642  args.objspace = &rb_objspace;
10643  args.value = rb_ary_new();
10644  st_foreach(w->wmap2obj, wmap_values_i, (st_data_t)&args);
10645  return args.value;
10646 }
10647 
10648 static int
10649 wmap_aset_update(st_data_t *key, st_data_t *val, st_data_t arg, int existing)
10650 {
10651  VALUE size, *ptr, *optr;
10652  if (existing) {
10653  size = (ptr = optr = (VALUE *)*val)[0];
10654  ++size;
10655  SIZED_REALLOC_N(ptr, VALUE, size + 1, size);
10656  }
10657  else {
10658  optr = 0;
10659  size = 1;
10660  ptr = ruby_xmalloc0(2 * sizeof(VALUE));
10661  }
10662  ptr[0] = size;
10663  ptr[size] = (VALUE)arg;
10664  if (ptr == optr) return ST_STOP;
10665  *val = (st_data_t)ptr;
10666  return ST_CONTINUE;
10667 }
10668 
10669 /* Creates a weak reference from the given key to the given value */
10670 static VALUE
10671 wmap_aset(VALUE self, VALUE wmap, VALUE orig)
10672 {
10673  struct weakmap *w;
10674 
10675  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10676  if (FL_ABLE(orig)) {
10677  define_final0(orig, w->final);
10678  }
10679  if (FL_ABLE(wmap)) {
10680  define_final0(wmap, w->final);
10681  }
10682 
10683  st_update(w->obj2wmap, (st_data_t)orig, wmap_aset_update, wmap);
10684  st_insert(w->wmap2obj, (st_data_t)wmap, (st_data_t)orig);
10685  return nonspecial_obj_id(orig);
10686 }
10687 
10688 /* Retrieves a weakly referenced object with the given key */
10689 static VALUE
10690 wmap_aref(VALUE self, VALUE wmap)
10691 {
10692  st_data_t data;
10693  VALUE obj;
10694  struct weakmap *w;
10695  rb_objspace_t *objspace = &rb_objspace;
10696 
10697  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10698  if (!st_lookup(w->wmap2obj, (st_data_t)wmap, &data)) return Qnil;
10699  obj = (VALUE)data;
10700  if (!wmap_live_p(objspace, obj)) return Qnil;
10701  return obj;
10702 }
10703 
10704 /* Returns +true+ if +key+ is registered */
10705 static VALUE
10706 wmap_has_key(VALUE self, VALUE key)
10707 {
10708  return NIL_P(wmap_aref(self, key)) ? Qfalse : Qtrue;
10709 }
10710 
10711 /* Returns the number of referenced objects */
10712 static VALUE
10713 wmap_size(VALUE self)
10714 {
10715  struct weakmap *w;
10716  st_index_t n;
10717 
10718  TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
10719  n = w->wmap2obj->num_entries;
10720 #if SIZEOF_ST_INDEX_T <= SIZEOF_LONG
10721  return ULONG2NUM(n);
10722 #else
10723  return ULL2NUM(n);
10724 #endif
10725 }
10726 
10727 /*
10728  ------------------------------ GC profiler ------------------------------
10729 */
10730 
10731 #define GC_PROFILE_RECORD_DEFAULT_SIZE 100
10732 
10733 /* return sec in user time */
10734 static double
10735 getrusage_time(void)
10736 {
10737 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
10738  {
10739  static int try_clock_gettime = 1;
10740  struct timespec ts;
10741  if (try_clock_gettime && clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts) == 0) {
10742  return ts.tv_sec + ts.tv_nsec * 1e-9;
10743  }
10744  else {
10745  try_clock_gettime = 0;
10746  }
10747  }
10748 #endif
10749 
10750 #ifdef RUSAGE_SELF
10751  {
10752  struct rusage usage;
10753  struct timeval time;
10754  if (getrusage(RUSAGE_SELF, &usage) == 0) {
10755  time = usage.ru_utime;
10756  return time.tv_sec + time.tv_usec * 1e-6;
10757  }
10758  }
10759 #endif
10760 
10761 #ifdef _WIN32
10762  {
10763  FILETIME creation_time, exit_time, kernel_time, user_time;
10764  ULARGE_INTEGER ui;
10765  LONG_LONG q;
10766  double t;
10767 
10768  if (GetProcessTimes(GetCurrentProcess(),
10769  &creation_time, &exit_time, &kernel_time, &user_time) != 0) {
10770  memcpy(&ui, &user_time, sizeof(FILETIME));
10771  q = ui.QuadPart / 10L;
10772  t = (DWORD)(q % 1000000L) * 1e-6;
10773  q /= 1000000L;
10774 #ifdef __GNUC__
10775  t += q;
10776 #else
10777  t += (double)(DWORD)(q >> 16) * (1 << 16);
10778  t += (DWORD)q & ~(~0 << 16);
10779 #endif
10780  return t;
10781  }
10782  }
10783 #endif
10784 
10785  return 0.0;
10786 }
10787 
10788 static inline void
10789 gc_prof_setup_new_record(rb_objspace_t *objspace, int reason)
10790 {
10791  if (objspace->profile.run) {
10792  size_t index = objspace->profile.next_index;
10793  gc_profile_record *record;
10794 
10795  /* create new record */
10796  objspace->profile.next_index++;
10797 
10798  if (!objspace->profile.records) {
10800  objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
10801  }
10802  if (index >= objspace->profile.size) {
10803  void *ptr;
10804  objspace->profile.size += 1000;
10805  ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
10806  if (!ptr) rb_memerror();
10807  objspace->profile.records = ptr;
10808  }
10809  if (!objspace->profile.records) {
10810  rb_bug("gc_profile malloc or realloc miss");
10811  }
10812  record = objspace->profile.current_record = &objspace->profile.records[objspace->profile.next_index - 1];
10813  MEMZERO(record, gc_profile_record, 1);
10814 
10815  /* setup before-GC parameter */
10816  record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
10817 #if MALLOC_ALLOCATED_SIZE
10818  record->allocated_size = malloc_allocated_size;
10819 #endif
10820 #if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
10821 #ifdef RUSAGE_SELF
10822  {
10823  struct rusage usage;
10824  if (getrusage(RUSAGE_SELF, &usage) == 0) {
10825  record->maxrss = usage.ru_maxrss;
10826  record->minflt = usage.ru_minflt;
10827  record->majflt = usage.ru_majflt;
10828  }
10829  }
10830 #endif
10831 #endif
10832  }
10833 }
10834 
10835 static inline void
10836 gc_prof_timer_start(rb_objspace_t *objspace)
10837 {
10838  if (gc_prof_enabled(objspace)) {
10839  gc_profile_record *record = gc_prof_record(objspace);
10840 #if GC_PROFILE_MORE_DETAIL
10841  record->prepare_time = objspace->profile.prepare_time;
10842 #endif
10843  record->gc_time = 0;
10844  record->gc_invoke_time = getrusage_time();
10845  }
10846 }
10847 
10848 static double
10849 elapsed_time_from(double time)
10850 {
10851  double now = getrusage_time();
10852  if (now > time) {
10853  return now - time;
10854  }
10855  else {
10856  return 0;
10857  }
10858 }
10859 
10860 static inline void
10861 gc_prof_timer_stop(rb_objspace_t *objspace)
10862 {
10863  if (gc_prof_enabled(objspace)) {
10864  gc_profile_record *record = gc_prof_record(objspace);
10865  record->gc_time = elapsed_time_from(record->gc_invoke_time);
10866  record->gc_invoke_time -= objspace->profile.invoke_time;
10867  }
10868 }
10869 
10870 #define RUBY_DTRACE_GC_HOOK(name) \
10871  do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_##name();} while (0)
10872 static inline void
10873 gc_prof_mark_timer_start(rb_objspace_t *objspace)
10874 {
10875  RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
10876 #if GC_PROFILE_MORE_DETAIL
10877  if (gc_prof_enabled(objspace)) {
10878  gc_prof_record(objspace)->gc_mark_time = getrusage_time();
10879  }
10880 #endif
10881 }
10882 
10883 static inline void
10884 gc_prof_mark_timer_stop(rb_objspace_t *objspace)
10885 {
10886  RUBY_DTRACE_GC_HOOK(MARK_END);
10887 #if GC_PROFILE_MORE_DETAIL
10888  if (gc_prof_enabled(objspace)) {
10889  gc_profile_record *record = gc_prof_record(objspace);
10890  record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
10891  }
10892 #endif
10893 }
10894 
10895 static inline void
10896 gc_prof_sweep_timer_start(rb_objspace_t *objspace)
10897 {
10898  RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
10899  if (gc_prof_enabled(objspace)) {
10900  gc_profile_record *record = gc_prof_record(objspace);
10901 
10902  if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
10903  objspace->profile.gc_sweep_start_time = getrusage_time();
10904  }
10905  }
10906 }
10907 
10908 static inline void
10909 gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
10910 {
10911  RUBY_DTRACE_GC_HOOK(SWEEP_END);
10912 
10913  if (gc_prof_enabled(objspace)) {
10914  double sweep_time;
10915  gc_profile_record *record = gc_prof_record(objspace);
10916 
10917  if (record->gc_time > 0) {
10918  sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
10919  /* need to accumulate GC time for lazy sweep after gc() */
10920  record->gc_time += sweep_time;
10921  }
10922  else if (GC_PROFILE_MORE_DETAIL) {
10923  sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
10924  }
10925 
10926 #if GC_PROFILE_MORE_DETAIL
10927  record->gc_sweep_time += sweep_time;
10929 #endif
10931  }
10932 }
10933 
10934 static inline void
10935 gc_prof_set_malloc_info(rb_objspace_t *objspace)
10936 {
10937 #if GC_PROFILE_MORE_DETAIL
10938  if (gc_prof_enabled(objspace)) {
10939  gc_profile_record *record = gc_prof_record(objspace);
10940  record->allocate_increase = malloc_increase;
10941  record->allocate_limit = malloc_limit;
10942  }
10943 #endif
10944 }
10945 
10946 static inline void
10947 gc_prof_set_heap_info(rb_objspace_t *objspace)
10948 {
10949  if (gc_prof_enabled(objspace)) {
10950  gc_profile_record *record = gc_prof_record(objspace);
10951  size_t live = objspace->profile.total_allocated_objects_at_gc_start - objspace->profile.total_freed_objects;
10952  size_t total = objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT;
10953 
10954 #if GC_PROFILE_MORE_DETAIL
10955  record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
10956  record->heap_live_objects = live;
10957  record->heap_free_objects = total - live;
10958 #endif
10959 
10960  record->heap_total_objects = total;
10961  record->heap_use_size = live * sizeof(RVALUE);
10962  record->heap_total_size = total * sizeof(RVALUE);
10963  }
10964 }
10965 
10966 /*
10967  * call-seq:
10968  * GC::Profiler.clear -> nil
10969  *
10970  * Clears the GC profiler data.
10971  *
10972  */
10973 
10974 static VALUE
10975 gc_profile_clear(VALUE _)
10976 {
10977  rb_objspace_t *objspace = &rb_objspace;
10978  void *p = objspace->profile.records;
10979  objspace->profile.records = NULL;
10980  objspace->profile.size = 0;
10981  objspace->profile.next_index = 0;
10982  objspace->profile.current_record = 0;
10983  if (p) {
10984  free(p);
10985  }
10986  return Qnil;
10987 }
10988 
10989 /*
10990  * call-seq:
10991  * GC::Profiler.raw_data -> [Hash, ...]
10992  *
10993  * Returns an Array of individual raw profile data Hashes ordered
10994  * from earliest to latest by +:GC_INVOKE_TIME+.
10995  *
10996  * For example:
10997  *
10998  * [
10999  * {
11000  * :GC_TIME=>1.3000000000000858e-05,
11001  * :GC_INVOKE_TIME=>0.010634999999999999,
11002  * :HEAP_USE_SIZE=>289640,
11003  * :HEAP_TOTAL_SIZE=>588960,
11004  * :HEAP_TOTAL_OBJECTS=>14724,
11005  * :GC_IS_MARKED=>false
11006  * },
11007  * # ...
11008  * ]
11009  *
11010  * The keys mean:
11011  *
11012  * +:GC_TIME+::
11013  * Time elapsed in seconds for this GC run
11014  * +:GC_INVOKE_TIME+::
11015  * Time elapsed in seconds from startup to when the GC was invoked
11016  * +:HEAP_USE_SIZE+::
11017  * Total bytes of heap used
11018  * +:HEAP_TOTAL_SIZE+::
11019  * Total size of heap in bytes
11020  * +:HEAP_TOTAL_OBJECTS+::
11021  * Total number of objects
11022  * +:GC_IS_MARKED+::
11023  * Returns +true+ if the GC is in mark phase
11024  *
11025  * If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access
11026  * to the following hash keys:
11027  *
11028  * +:GC_MARK_TIME+::
11029  * +:GC_SWEEP_TIME+::
11030  * +:ALLOCATE_INCREASE+::
11031  * +:ALLOCATE_LIMIT+::
11032  * +:HEAP_USE_PAGES+::
11033  * +:HEAP_LIVE_OBJECTS+::
11034  * +:HEAP_FREE_OBJECTS+::
11035  * +:HAVE_FINALIZE+::
11036  *
11037  */
11038 
11039 static VALUE
11040 gc_profile_record_get(VALUE _)
11041 {
11042  VALUE prof;
11043  VALUE gc_profile = rb_ary_new();
11044  size_t i;
11045  rb_objspace_t *objspace = (&rb_objspace);
11046 
11047  if (!objspace->profile.run) {
11048  return Qnil;
11049  }
11050 
11051  for (i =0; i < objspace->profile.next_index; i++) {
11052  gc_profile_record *record = &objspace->profile.records[i];
11053 
11054  prof = rb_hash_new();
11055  rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(0, rb_hash_new(), record->flags));
11056  rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time));
11057  rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time));
11058  rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
11059  rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
11060  rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
11061  rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
11062 #if GC_PROFILE_MORE_DETAIL
11063  rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
11064  rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time));
11065  rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase));
11066  rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit));
11067  rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages));
11068  rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects));
11069  rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects));
11070 
11071  rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects));
11072  rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects));
11073 
11074  rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), (record->flags & GPR_FLAG_HAVE_FINALIZE) ? Qtrue : Qfalse);
11075 #endif
11076 
11077 #if RGENGC_PROFILE > 0
11078  rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects));
11079  rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects));
11080  rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects));
11081 #endif
11082  rb_ary_push(gc_profile, prof);
11083  }
11084 
11085  return gc_profile;
11086 }
11087 
11088 #if GC_PROFILE_MORE_DETAIL
11089 #define MAJOR_REASON_MAX 0x10
11090 
11091 static char *
11092 gc_profile_dump_major_reason(int flags, char *buff)
11093 {
11094  int reason = flags & GPR_FLAG_MAJOR_MASK;
11095  int i = 0;
11096 
11097  if (reason == GPR_FLAG_NONE) {
11098  buff[0] = '-';
11099  buff[1] = 0;
11100  }
11101  else {
11102 #define C(x, s) \
11103  if (reason & GPR_FLAG_MAJOR_BY_##x) { \
11104  buff[i++] = #x[0]; \
11105  if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
11106  buff[i] = 0; \
11107  }
11108  C(NOFREE, N);
11109  C(OLDGEN, O);
11110  C(SHADY, S);
11111 #if RGENGC_ESTIMATE_OLDMALLOC
11112  C(OLDMALLOC, M);
11113 #endif
11114 #undef C
11115  }
11116  return buff;
11117 }
11118 #endif
11119 
11120 static void
11121 gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
11122 {
11123  rb_objspace_t *objspace = &rb_objspace;
11124  size_t count = objspace->profile.next_index;
11125 #ifdef MAJOR_REASON_MAX
11126  char reason_str[MAJOR_REASON_MAX];
11127 #endif
11128 
11129  if (objspace->profile.run && count /* > 1 */) {
11130  size_t i;
11131  const gc_profile_record *record;
11132 
11133  append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count));
11134  append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
11135 
11136  for (i = 0; i < count; i++) {
11137  record = &objspace->profile.records[i];
11138  append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
11139  i+1, record->gc_invoke_time, record->heap_use_size,
11140  record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
11141  }
11142 
11143 #if GC_PROFILE_MORE_DETAIL
11144  append(out, rb_str_new_cstr("\n\n" \
11145  "More detail.\n" \
11146  "Prepare Time = Previously GC's rest sweep time\n"
11147  "Index Flags Allocate Inc. Allocate Limit"
11149  " Allocated Size"
11150 #endif
11151  " Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
11152 #if RGENGC_PROFILE
11153  " OldgenObj RemNormObj RemShadObj"
11154 #endif
11156  " MaxRSS(KB) MinorFLT MajorFLT"
11157 #endif
11158  "\n"));
11159 
11160  for (i = 0; i < count; i++) {
11161  record = &objspace->profile.records[i];
11162  append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE
11164  " %15"PRIuSIZE
11165 #endif
11166  " %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
11167 #if RGENGC_PROFILE
11168  "%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
11169 #endif
11171  "%11ld %8ld %8ld"
11172 #endif
11173 
11174  "\n",
11175  i+1,
11176  gc_profile_dump_major_reason(record->flags, reason_str),
11177  (record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.',
11178  (record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" :
11179  (record->flags & GPR_FLAG_MALLOC) ? "MALLOC" :
11180  (record->flags & GPR_FLAG_METHOD) ? "METHOD" :
11181  (record->flags & GPR_FLAG_CAPI) ? "CAPI__" : "??????",
11182  (record->flags & GPR_FLAG_STRESS) ? '!' : ' ',
11183  record->allocate_increase, record->allocate_limit,
11185  record->allocated_size,
11186 #endif
11187  record->heap_use_pages,
11188  record->gc_mark_time*1000,
11189  record->gc_sweep_time*1000,
11190  record->prepare_time*1000,
11191 
11192  record->heap_live_objects,
11193  record->heap_free_objects,
11194  record->removing_objects,
11195  record->empty_objects
11196 #if RGENGC_PROFILE
11197  ,
11198  record->old_objects,
11199  record->remembered_normal_objects,
11200  record->remembered_shady_objects
11201 #endif
11203  ,
11204  record->maxrss / 1024,
11205  record->minflt,
11206  record->majflt
11207 #endif
11208 
11209  ));
11210  }
11211 #endif
11212  }
11213 }
11214 
11215 /*
11216  * call-seq:
11217  * GC::Profiler.result -> String
11218  *
11219  * Returns a profile data report such as:
11220  *
11221  * GC 1 invokes.
11222  * Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC time(ms)
11223  * 1 0.012 159240 212940 10647 0.00000000000001530000
11224  */
11225 
11226 static VALUE
11227 gc_profile_result(VALUE _)
11228 {
11229  VALUE str = rb_str_buf_new(0);
11230  gc_profile_dump_on(str, rb_str_buf_append);
11231  return str;
11232 }
11233 
11234 /*
11235  * call-seq:
11236  * GC::Profiler.report
11237  * GC::Profiler.report(io)
11238  *
11239  * Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object.
11240  *
11241  */
11242 
11243 static VALUE
11244 gc_profile_report(int argc, VALUE *argv, VALUE self)
11245 {
11246  VALUE out;
11247 
11248  out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]);
11249  gc_profile_dump_on(out, rb_io_write);
11250 
11251  return Qnil;
11252 }
11253 
11254 /*
11255  * call-seq:
11256  * GC::Profiler.total_time -> float
11257  *
11258  * The total time used for garbage collection in seconds
11259  */
11260 
11261 static VALUE
11262 gc_profile_total_time(VALUE self)
11263 {
11264  double time = 0;
11265  rb_objspace_t *objspace = &rb_objspace;
11266 
11267  if (objspace->profile.run && objspace->profile.next_index > 0) {
11268  size_t i;
11269  size_t count = objspace->profile.next_index;
11270 
11271  for (i = 0; i < count; i++) {
11272  time += objspace->profile.records[i].gc_time;
11273  }
11274  }
11275  return DBL2NUM(time);
11276 }
11277 
11278 /*
11279  * call-seq:
11280  * GC::Profiler.enabled? -> true or false
11281  *
11282  * The current status of GC profile mode.
11283  */
11284 
11285 static VALUE
11286 gc_profile_enable_get(VALUE self)
11287 {
11288  rb_objspace_t *objspace = &rb_objspace;
11289  return objspace->profile.run ? Qtrue : Qfalse;
11290 }
11291 
11292 /*
11293  * call-seq:
11294  * GC::Profiler.enable -> nil
11295  *
11296  * Starts the GC profiler.
11297  *
11298  */
11299 
11300 static VALUE
11301 gc_profile_enable(VALUE _)
11302 {
11303  rb_objspace_t *objspace = &rb_objspace;
11304  objspace->profile.run = TRUE;
11305  objspace->profile.current_record = 0;
11306  return Qnil;
11307 }
11308 
11309 /*
11310  * call-seq:
11311  * GC::Profiler.disable -> nil
11312  *
11313  * Stops the GC profiler.
11314  *
11315  */
11316 
11317 static VALUE
11318 gc_profile_disable(VALUE _)
11319 {
11320  rb_objspace_t *objspace = &rb_objspace;
11321 
11322  objspace->profile.run = FALSE;
11323  objspace->profile.current_record = 0;
11324  return Qnil;
11325 }
11326 
11327 /*
11328  ------------------------------ DEBUG ------------------------------
11329 */
11330 
11331 static const char *
11332 type_name(int type, VALUE obj)
11333 {
11334  switch (type) {
11335 #define TYPE_NAME(t) case (t): return #t;
11336  TYPE_NAME(T_NONE);
11338  TYPE_NAME(T_CLASS);
11340  TYPE_NAME(T_FLOAT);
11343  TYPE_NAME(T_ARRAY);
11344  TYPE_NAME(T_HASH);
11347  TYPE_NAME(T_FILE);
11348  TYPE_NAME(T_MATCH);
11351  TYPE_NAME(T_NIL);
11352  TYPE_NAME(T_TRUE);
11353  TYPE_NAME(T_FALSE);
11356  TYPE_NAME(T_UNDEF);
11357  TYPE_NAME(T_IMEMO);
11359  TYPE_NAME(T_MOVED);
11361  case T_DATA:
11364  }
11365  return "T_DATA";
11366 #undef TYPE_NAME
11367  }
11368  return "unknown";
11369 }
11370 
11371 static const char *
11372 obj_type_name(VALUE obj)
11373 {
11374  return type_name(TYPE(obj), obj);
11375 }
11376 
11377 const char *
11379 {
11380  switch (type) {
11381  case VM_METHOD_TYPE_ISEQ: return "iseq";
11382  case VM_METHOD_TYPE_ATTRSET: return "attrest";
11383  case VM_METHOD_TYPE_IVAR: return "ivar";
11384  case VM_METHOD_TYPE_BMETHOD: return "bmethod";
11385  case VM_METHOD_TYPE_ALIAS: return "alias";
11386  case VM_METHOD_TYPE_REFINED: return "refined";
11387  case VM_METHOD_TYPE_CFUNC: return "cfunc";
11388  case VM_METHOD_TYPE_ZSUPER: return "zsuper";
11389  case VM_METHOD_TYPE_MISSING: return "missing";
11390  case VM_METHOD_TYPE_OPTIMIZED: return "optimized";
11391  case VM_METHOD_TYPE_UNDEF: return "undef";
11392  case VM_METHOD_TYPE_NOTIMPLEMENTED: return "notimplemented";
11393  }
11394  rb_bug("rb_method_type_name: unreachable (type: %d)", type);
11395 }
11396 
11397 /* from array.c */
11398 # define ARY_SHARED_P(ary) \
11399  (GC_ASSERT(!FL_TEST((ary), ELTS_SHARED) || !FL_TEST((ary), RARRAY_EMBED_FLAG)), \
11400  FL_TEST((ary),ELTS_SHARED)!=0)
11401 # define ARY_EMBED_P(ary) \
11402  (GC_ASSERT(!FL_TEST((ary), ELTS_SHARED) || !FL_TEST((ary), RARRAY_EMBED_FLAG)), \
11403  FL_TEST((ary), RARRAY_EMBED_FLAG)!=0)
11404 
11405 static void
11406 rb_raw_iseq_info(char *buff, const int buff_size, const rb_iseq_t *iseq)
11407 {
11408  if (buff_size > 0 && iseq->body && iseq->body->location.label && !RB_TYPE_P(iseq->body->location.pathobj, T_MOVED)) {
11411  snprintf(buff, buff_size, " %s@%s:%d",
11413  RSTRING_PTR(path),
11414  n ? FIX2INT(n) : 0 );
11415  }
11416 }
11417 
11418 const char *
11419 rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
11420 {
11421  int pos = 0;
11422 
11423 #define BUFF_ARGS buff + pos, buff_size - pos
11424 #define APPENDF(f) if ((pos += snprintf f) >= buff_size) goto end
11425  if (SPECIAL_CONST_P(obj)) {
11426  APPENDF((BUFF_ARGS, "%s", obj_type_name(obj)));
11427 
11428  if (FIXNUM_P(obj)) {
11429  APPENDF((BUFF_ARGS, " %ld", FIX2LONG(obj)));
11430  }
11431  else if (SYMBOL_P(obj)) {
11432  APPENDF((BUFF_ARGS, " %s", rb_id2name(SYM2ID(obj))));
11433  }
11434  }
11435  else {
11436 #define TF(c) ((c) != 0 ? "true" : "false")
11437 #define C(c, s) ((c) != 0 ? (s) : " ")
11438  const int type = BUILTIN_TYPE(obj);
11439 #if USE_RGENGC
11440  const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
11441 
11442  if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
11443  APPENDF((BUFF_ARGS, "%p [%d%s%s%s%s%s] %s ",
11444  (void *)obj, age,
11446  C(RVALUE_MARK_BITMAP(obj), "M"),
11447  C(RVALUE_PIN_BITMAP(obj), "P"),
11448  C(RVALUE_MARKING_BITMAP(obj), "R"),
11450  obj_type_name(obj)));
11451  }
11452  else {
11453  /* fake */
11454  APPENDF((BUFF_ARGS, "%p [%dXXXX] %s",
11455  (void *)obj, age,
11456  obj_type_name(obj)));
11457  }
11458 #else
11459  APPENDF((BUFF_ARGS, "%p [%s] %s",
11460  (void *)obj,
11461  C(RVALUE_MARK_BITMAP(obj), "M"),
11462  obj_type_name(obj)));
11463 #endif
11464 
11465  if (internal_object_p(obj)) {
11466  /* ignore */
11467  }
11468  else if (RBASIC(obj)->klass == 0) {
11469  APPENDF((BUFF_ARGS, "(temporary internal)"));
11470  }
11471  else {
11472  if (RTEST(RBASIC(obj)->klass)) {
11473  VALUE class_path = rb_class_path_cached(RBASIC(obj)->klass);
11474  if (!NIL_P(class_path)) {
11475  APPENDF((BUFF_ARGS, "(%s)", RSTRING_PTR(class_path)));
11476  }
11477  }
11478  }
11479 
11480 #if GC_DEBUG
11481  APPENDF((BUFF_ARGS, "@%s:%d", RANY(obj)->file, RANY(obj)->line));
11482 #endif
11483 
11484  switch (type) {
11485  case T_NODE:
11487  break;
11488  case T_ARRAY:
11489  if (FL_TEST(obj, ELTS_SHARED)) {
11490  APPENDF((BUFF_ARGS, "shared -> %s",
11491  rb_obj_info(RARRAY(obj)->as.heap.aux.shared_root)));
11492  }
11493  else if (FL_TEST(obj, RARRAY_EMBED_FLAG)) {
11494  APPENDF((BUFF_ARGS, "[%s%s] len: %d (embed)",
11495  C(ARY_EMBED_P(obj), "E"),
11496  C(ARY_SHARED_P(obj), "S"),
11497  (int)RARRAY_LEN(obj)));
11498  }
11499  else {
11500  APPENDF((BUFF_ARGS, "[%s%s%s] len: %d, capa:%d ptr:%p",
11501  C(ARY_EMBED_P(obj), "E"),
11502  C(ARY_SHARED_P(obj), "S"),
11503  C(RARRAY_TRANSIENT_P(obj), "T"),
11504  (int)RARRAY_LEN(obj),
11505  ARY_EMBED_P(obj) ? -1 : (int)RARRAY(obj)->as.heap.aux.capa,
11506  (void *)RARRAY_CONST_PTR_TRANSIENT(obj)));
11507  }
11508  break;
11509  case T_STRING: {
11510  APPENDF((BUFF_ARGS, "%s", RSTRING_PTR(obj)));
11511  break;
11512  }
11513  case T_MOVED: {
11514  APPENDF((BUFF_ARGS, "-> %p", (void*)rb_gc_location(obj)));
11515  break;
11516  }
11517  case T_HASH: {
11518  APPENDF((BUFF_ARGS, "[%c%c] %d",
11519  RHASH_AR_TABLE_P(obj) ? 'A' : 'S',
11520  RHASH_TRANSIENT_P(obj) ? 'T' : ' ',
11521  (int)RHASH_SIZE(obj)));
11522  break;
11523  }
11524  case T_CLASS:
11525  case T_MODULE:
11526  {
11527  VALUE class_path = rb_class_path_cached(obj);
11528  if (!NIL_P(class_path)) {
11529  APPENDF((BUFF_ARGS, "%s", RSTRING_PTR(class_path)));
11530  }
11531  break;
11532  }
11533  case T_ICLASS:
11534  {
11535  VALUE class_path = rb_class_path_cached(RBASIC_CLASS(obj));
11536  if (!NIL_P(class_path)) {
11537  APPENDF((BUFF_ARGS, "src:%s", RSTRING_PTR(class_path)));
11538  }
11539  break;
11540  }
11541  case T_OBJECT:
11542  {
11544 
11545  if (RANY(obj)->as.basic.flags & ROBJECT_EMBED) {
11546  APPENDF((BUFF_ARGS, "(embed) len:%d", len));
11547  }
11548  else {
11549  VALUE *ptr = ROBJECT_IVPTR(obj);
11550  APPENDF((BUFF_ARGS, "len:%d ptr:%p", len, (void *)ptr));
11551  }
11552  }
11553  break;
11554  case T_DATA: {
11555  const struct rb_block *block;
11556  const rb_iseq_t *iseq;
11557  if (rb_obj_is_proc(obj) &&
11558  (block = vm_proc_block(obj)) != NULL &&
11559  (vm_block_type(block) == block_type_iseq) &&
11560  (iseq = vm_block_iseq(block)) != NULL) {
11561  rb_raw_iseq_info(BUFF_ARGS, iseq);
11562  }
11563  else {
11564  const char * const type_name = rb_objspace_data_type_name(obj);
11565  if (type_name) {
11566  APPENDF((BUFF_ARGS, "%s", type_name));
11567  }
11568  }
11569  break;
11570  }
11571  case T_IMEMO: {
11572  const char *imemo_name = "\0";
11573  switch (imemo_type(obj)) {
11574 #define IMEMO_NAME(x) case imemo_##x: imemo_name = #x; break;
11575  IMEMO_NAME(env);
11576  IMEMO_NAME(cref);
11577  IMEMO_NAME(svar);
11578  IMEMO_NAME(throw_data);
11579  IMEMO_NAME(ifunc);
11580  IMEMO_NAME(memo);
11581  IMEMO_NAME(ment);
11582  IMEMO_NAME(iseq);
11583  IMEMO_NAME(tmpbuf);
11584  IMEMO_NAME(ast);
11585  IMEMO_NAME(parser_strterm);
11586 #undef IMEMO_NAME
11587  default: UNREACHABLE;
11588  }
11589  APPENDF((BUFF_ARGS, "/%s", imemo_name));
11590 
11591  switch (imemo_type(obj)) {
11592  case imemo_ment: {
11593  const rb_method_entry_t *me = &RANY(obj)->as.imemo.ment;
11594  if (me->def) {
11595  APPENDF((BUFF_ARGS, "(called_id: %s, type: %s, alias: %d, owner: %s, defined_class: %s)",
11598  me->def->alias_count,
11599  obj_info(me->owner),
11600  obj_info(me->defined_class)));
11601  }
11602  else {
11603  APPENDF((BUFF_ARGS, "%s", rb_id2name(me->called_id)));
11604  }
11605  break;
11606  }
11607  case imemo_iseq: {
11608  const rb_iseq_t *iseq = (const rb_iseq_t *)obj;
11609  rb_raw_iseq_info(BUFF_ARGS, iseq);
11610  break;
11611  }
11612  default:
11613  break;
11614  }
11615  }
11616  default:
11617  break;
11618  }
11619 #undef TF
11620 #undef C
11621  }
11622  end:
11623  return buff;
11624 #undef APPENDF
11625 #undef BUFF_ARGS
11626 }
11627 
11628 #if RGENGC_OBJ_INFO
11629 #define OBJ_INFO_BUFFERS_NUM 10
11630 #define OBJ_INFO_BUFFERS_SIZE 0x100
11631 static int obj_info_buffers_index = 0;
11632 static char obj_info_buffers[OBJ_INFO_BUFFERS_NUM][OBJ_INFO_BUFFERS_SIZE];
11633 
11634 static const char *
11635 obj_info(VALUE obj)
11636 {
11637  const int index = obj_info_buffers_index++;
11638  char *const buff = &obj_info_buffers[index][0];
11639 
11640  if (obj_info_buffers_index >= OBJ_INFO_BUFFERS_NUM) {
11641  obj_info_buffers_index = 0;
11642  }
11643 
11644  return rb_raw_obj_info(buff, OBJ_INFO_BUFFERS_SIZE, obj);
11645 }
11646 #else
11647 static const char *
11648 obj_info(VALUE obj)
11649 {
11650  return obj_type_name(obj);
11651 }
11652 #endif
11653 
11654 MJIT_FUNC_EXPORTED const char *
11656 {
11657  return obj_info(obj);
11658 }
11659 
11660 void
11662 {
11663  char buff[0x100];
11664  fprintf(stderr, "rb_obj_info_dump: %s\n", rb_raw_obj_info(buff, 0x100, obj));
11665 }
11666 
11667 void
11668 rb_obj_info_dump_loc(VALUE obj, const char *file, int line, const char *func)
11669 {
11670  char buff[0x100];
11671  fprintf(stderr, "<OBJ_INFO:%s@%s:%d> %s\n", func, file, line, rb_raw_obj_info(buff, 0x100, obj));
11672 }
11673 
11674 #if GC_DEBUG
11675 
11676 void
11678 {
11679  rb_objspace_t *objspace = &rb_objspace;
11680 
11681  fprintf(stderr, "created at: %s:%d\n", RANY(obj)->file, RANY(obj)->line);
11682 
11683  if (BUILTIN_TYPE(obj) == T_MOVED) {
11684  fprintf(stderr, "moved?: true\n");
11685  }
11686  else {
11687  fprintf(stderr, "moved?: false\n");
11688  }
11689  if (is_pointer_to_heap(objspace, (void *)obj)) {
11690  fprintf(stderr, "pointer to heap?: true\n");
11691  }
11692  else {
11693  fprintf(stderr, "pointer to heap?: false\n");
11694  return;
11695  }
11696 
11697  fprintf(stderr, "marked? : %s\n", MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) ? "true" : "false");
11698  fprintf(stderr, "pinned? : %s\n", MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) ? "true" : "false");
11699 #if USE_RGENGC
11700  fprintf(stderr, "age? : %d\n", RVALUE_AGE(obj));
11701  fprintf(stderr, "old? : %s\n", RVALUE_OLD_P(obj) ? "true" : "false");
11702  fprintf(stderr, "WB-protected?: %s\n", RVALUE_WB_UNPROTECTED(obj) ? "false" : "true");
11703  fprintf(stderr, "remembered? : %s\n", RVALUE_REMEMBERED(obj) ? "true" : "false");
11704 #endif
11705 
11706  if (is_lazy_sweeping(heap_eden)) {
11707  fprintf(stderr, "lazy sweeping?: true\n");
11708  fprintf(stderr, "swept?: %s\n", is_swept_object(objspace, obj) ? "done" : "not yet");
11709  }
11710  else {
11711  fprintf(stderr, "lazy sweeping?: false\n");
11712  }
11713 }
11714 
11715 static VALUE
11716 gcdebug_sentinel(RB_BLOCK_CALL_FUNC_ARGLIST(obj, name))
11717 {
11718  fprintf(stderr, "WARNING: object %s(%p) is inadvertently collected\n", (char *)name, (void *)obj);
11719  return Qnil;
11720 }
11721 
11722 void
11723 rb_gcdebug_sentinel(VALUE obj, const char *name)
11724 {
11725  rb_define_finalizer(obj, rb_proc_new(gcdebug_sentinel, (VALUE)name));
11726 }
11727 
11728 #endif /* GC_DEBUG */
11729 
11730 #if GC_DEBUG_STRESS_TO_CLASS
11731 /*
11732  * call-seq:
11733  * GC.add_stress_to_class(class[, ...])
11734  *
11735  * Raises NoMemoryError when allocating an instance of the given classes.
11736  *
11737  */
11738 static VALUE
11739 rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self)
11740 {
11741  rb_objspace_t *objspace = &rb_objspace;
11742 
11743  if (!stress_to_class) {
11745  }
11747  return self;
11748 }
11749 
11750 /*
11751  * call-seq:
11752  * GC.remove_stress_to_class(class[, ...])
11753  *
11754  * No longer raises NoMemoryError when allocating an instance of the
11755  * given classes.
11756  *
11757  */
11758 static VALUE
11759 rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self)
11760 {
11761  rb_objspace_t *objspace = &rb_objspace;
11762  int i;
11763 
11764  if (stress_to_class) {
11765  for (i = 0; i < argc; ++i) {
11767  }
11768  if (RARRAY_LEN(stress_to_class) == 0) {
11769  stress_to_class = 0;
11770  }
11771  }
11772  return Qnil;
11773 }
11774 #endif
11775 
11776 /*
11777  * Document-module: ObjectSpace
11778  *
11779  * The ObjectSpace module contains a number of routines
11780  * that interact with the garbage collection facility and allow you to
11781  * traverse all living objects with an iterator.
11782  *
11783  * ObjectSpace also provides support for object finalizers, procs that will be
11784  * called when a specific object is about to be destroyed by garbage
11785  * collection.
11786  *
11787  * require 'objspace'
11788  *
11789  * a = "A"
11790  * b = "B"
11791  *
11792  * ObjectSpace.define_finalizer(a, proc {|id| puts "Finalizer one on #{id}" })
11793  * ObjectSpace.define_finalizer(b, proc {|id| puts "Finalizer two on #{id}" })
11794  *
11795  * _produces:_
11796  *
11797  * Finalizer two on 537763470
11798  * Finalizer one on 537763480
11799  */
11800 
11801 /*
11802  * Document-class: ObjectSpace::WeakMap
11803  *
11804  * An ObjectSpace::WeakMap object holds references to
11805  * any objects, but those objects can get garbage collected.
11806  *
11807  * This class is mostly used internally by WeakRef, please use
11808  * +lib/weakref.rb+ for the public interface.
11809  */
11810 
11811 /* Document-class: GC::Profiler
11812  *
11813  * The GC profiler provides access to information on GC runs including time,
11814  * length and object space size.
11815  *
11816  * Example:
11817  *
11818  * GC::Profiler.enable
11819  *
11820  * require 'rdoc/rdoc'
11821  *
11822  * GC::Profiler.report
11823  *
11824  * GC::Profiler.disable
11825  *
11826  * See also GC.count, GC.malloc_allocated_size and GC.malloc_allocations
11827  */
11828 
11829 #include "gc.rbinc"
11830 
11831 void
11832 Init_GC(void)
11833 {
11834 #undef rb_intern
11835  VALUE rb_mObjSpace;
11836  VALUE rb_mProfiler;
11837  VALUE gc_constants;
11838 
11839  rb_mGC = rb_define_module("GC");
11840  load_gc();
11841 
11842  gc_constants = rb_hash_new();
11843  rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(RVALUE)));
11844  rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));
11845  rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
11846  rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_PLANES")), SIZET2NUM(HEAP_PAGE_BITMAP_PLANES));
11847  OBJ_FREEZE(gc_constants);
11848  /* internal constants */
11849  rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
11850 
11851  rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
11852  rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
11853  rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
11854  rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, 0);
11855  rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
11856  rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
11857  rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
11858  rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
11859  rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
11860 
11861  rb_mObjSpace = rb_define_module("ObjectSpace");
11862 
11863  rb_define_module_function(rb_mObjSpace, "each_object", os_each_obj, -1);
11864 
11865  rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1);
11866  rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1);
11867 
11868  rb_define_module_function(rb_mObjSpace, "_id2ref", os_id2ref, 1);
11869 
11871 
11873  rb_define_method(rb_mKernel, "object_id", rb_obj_id, 0);
11874 
11875  rb_define_module_function(rb_mObjSpace, "count_objects", count_objects, -1);
11876 
11877  {
11878  VALUE rb_cWeakMap = rb_define_class_under(rb_mObjSpace, "WeakMap", rb_cObject);
11879  rb_define_alloc_func(rb_cWeakMap, wmap_allocate);
11880  rb_define_method(rb_cWeakMap, "[]=", wmap_aset, 2);
11881  rb_define_method(rb_cWeakMap, "[]", wmap_aref, 1);
11882  rb_define_method(rb_cWeakMap, "include?", wmap_has_key, 1);
11883  rb_define_method(rb_cWeakMap, "member?", wmap_has_key, 1);
11884  rb_define_method(rb_cWeakMap, "key?", wmap_has_key, 1);
11885  rb_define_method(rb_cWeakMap, "inspect", wmap_inspect, 0);
11886  rb_define_method(rb_cWeakMap, "each", wmap_each, 0);
11887  rb_define_method(rb_cWeakMap, "each_pair", wmap_each, 0);
11888  rb_define_method(rb_cWeakMap, "each_key", wmap_each_key, 0);
11889  rb_define_method(rb_cWeakMap, "each_value", wmap_each_value, 0);
11890  rb_define_method(rb_cWeakMap, "keys", wmap_keys, 0);
11891  rb_define_method(rb_cWeakMap, "values", wmap_values, 0);
11892  rb_define_method(rb_cWeakMap, "size", wmap_size, 0);
11893  rb_define_method(rb_cWeakMap, "length", wmap_size, 0);
11894  rb_include_module(rb_cWeakMap, rb_mEnumerable);
11895  }
11896 
11897  /* internal methods */
11898  rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
11899  rb_define_singleton_method(rb_mGC, "verify_compaction_references", gc_verify_compaction_references, -1);
11900  rb_define_singleton_method(rb_mGC, "verify_transient_heap_internal_consistency", gc_verify_transient_heap_internal_consistency, 0);
11901 #if MALLOC_ALLOCATED_SIZE
11902  rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
11903  rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
11904 #endif
11905 
11906 #if GC_DEBUG_STRESS_TO_CLASS
11907  rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
11908  rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
11909 #endif
11910 
11911  {
11912  VALUE opts;
11913  /* GC build options */
11914  rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new());
11915 #define OPT(o) if (o) rb_ary_push(opts, rb_fstring_lit(#o))
11916  OPT(GC_DEBUG);
11917  OPT(USE_RGENGC);
11918  OPT(RGENGC_DEBUG);
11928 #undef OPT
11929  OBJ_FREEZE(opts);
11930  }
11931 }
11932 
11933 #ifdef ruby_xmalloc
11934 #undef ruby_xmalloc
11935 #endif
11936 #ifdef ruby_xmalloc2
11937 #undef ruby_xmalloc2
11938 #endif
11939 #ifdef ruby_xcalloc
11940 #undef ruby_xcalloc
11941 #endif
11942 #ifdef ruby_xrealloc
11943 #undef ruby_xrealloc
11944 #endif
11945 #ifdef ruby_xrealloc2
11946 #undef ruby_xrealloc2
11947 #endif
11948 
11949 void *
11951 {
11952 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11953  ruby_malloc_info_file = __FILE__;
11954  ruby_malloc_info_line = __LINE__;
11955 #endif
11956  return ruby_xmalloc_body(size);
11957 }
11958 
11959 void *
11960 ruby_xmalloc2(size_t n, size_t size)
11961 {
11962 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11963  ruby_malloc_info_file = __FILE__;
11964  ruby_malloc_info_line = __LINE__;
11965 #endif
11966  return ruby_xmalloc2_body(n, size);
11967 }
11968 
11969 void *
11970 ruby_xcalloc(size_t n, size_t size)
11971 {
11972 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11973  ruby_malloc_info_file = __FILE__;
11974  ruby_malloc_info_line = __LINE__;
11975 #endif
11976  return ruby_xcalloc_body(n, size);
11977 }
11978 
11979 void *
11980 ruby_xrealloc(void *ptr, size_t new_size)
11981 {
11982 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11983  ruby_malloc_info_file = __FILE__;
11984  ruby_malloc_info_line = __LINE__;
11985 #endif
11986  return ruby_xrealloc_body(ptr, new_size);
11987 }
11988 
11989 void *
11990 ruby_xrealloc2(void *ptr, size_t n, size_t new_size)
11991 {
11992 #if USE_GC_MALLOC_OBJ_INFO_DETAILS
11993  ruby_malloc_info_file = __FILE__;
11994  ruby_malloc_info_line = __LINE__;
11995 #endif
11996  return ruby_xrealloc2_body(ptr, n, new_size);
11997 }
rb_yield_values
#define rb_yield_values(argc,...)
Definition: rb_mjit_min_header-2.7.0.h:6584
RMatch::regexp
VALUE regexp
Definition: re.h:47
rb_objspace::dont_gc
unsigned int dont_gc
Definition: gc.c:689
rb_objspace::atomic_flags
struct rb_objspace::@84 atomic_flags
FLONUM_P
#define FLONUM_P(x)
Definition: ruby.h:430
global_symbols
#define global_symbols
Definition: gc.c:8407
gc_stat_sym_heap_available_slots
@ gc_stat_sym_heap_available_slots
Definition: gc.c:8842
__attribute__
unsigned int UINT8 __attribute__((__mode__(__QI__)))
Definition: ffi_common.h:110
rb_objspace::mark_func_data_struct::mark_func
void(* mark_func)(VALUE v, void *data)
Definition: gc.c:716
gc_stat_sym_total_allocated_pages
@ gc_stat_sym_total_allocated_pages
Definition: gc.c:8849
rb_io_t::writeconv_pre_ecopts
VALUE writeconv_pre_ecopts
Definition: io.h:99
ATOMIC_VALUE_EXCHANGE
#define ATOMIC_VALUE_EXCHANGE(var, val)
Definition: ruby_atomic.h:216
rb_subclass_entry::next
rb_subclass_entry_t * next
Definition: internal.h:1000
gc_stat_compat_sym_malloc_limit
@ gc_stat_compat_sym_malloc_limit
Definition: gc.c:8898
rb_objspace::count
size_t count
Definition: gc.c:779
RARRAY_TRANSIENT_P
#define RARRAY_TRANSIENT_P(ary)
Definition: ruby.h:1076
rb_get_kwargs
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Definition: class.c:1886
FL_FINALIZE
#define FL_FINALIZE
Definition: ruby.h:1282
rmatch::regs
struct re_registers regs
Definition: re.h:37
heap_eden
#define heap_eden
Definition: gc.c:919
rb_big_eql
VALUE rb_big_eql(VALUE x, VALUE y)
Definition: bignum.c:5544
nonspecial_obj_id
#define nonspecial_obj_id(obj)
Definition: gc.c:974
UNLIKELY
#define UNLIKELY(x)
Definition: ffi_common.h:126
OBJ_ID_INCREMENT
#define OBJ_ID_INCREMENT
Definition: gc.c:2881
ID
unsigned long ID
Definition: ruby.h:103
gc_stat_sym_oldmalloc_increase_bytes_limit
@ gc_stat_sym_oldmalloc_increase_bytes_limit
Definition: gc.c:8865
MEMOP_TYPE_MALLOC
@ MEMOP_TYPE_MALLOC
Definition: gc.c:9668
rb_check_funcall
VALUE rb_check_funcall(VALUE, ID, int, const VALUE *)
Definition: vm_eval.c:505
ruby_xfree
void ruby_xfree(void *x)
Definition: gc.c:10150
GC_MALLOC_LIMIT_GROWTH_FACTOR
#define GC_MALLOC_LIMIT_GROWTH_FACTOR
Definition: gc.c:291
T_FALSE
#define T_FALSE
Definition: ruby.h:537
ruby::backward::cxxanyargs::rb_proc_new
VALUE rb_proc_new(type *q, VALUE w)
Creates a rb_cProc instance.
Definition: cxxanyargs.hpp:324
BIGNUM_DIGITS
#define BIGNUM_DIGITS(b)
Definition: internal.h:780
MEMOP_TYPE_FREE
@ MEMOP_TYPE_FREE
Definition: gc.c:9669
list_add
#define list_add(h, n)
Definition: rb_mjit_min_header-2.7.0.h:9004
ruby_stack_length
size_t ruby_stack_length(VALUE **p)
Definition: gc.c:4631
ruby_gc_stress_mode
#define ruby_gc_stress_mode
Definition: gc.c:927
STACK_END
#define STACK_END
Definition: gc.c:4601
COUNT_TYPE
#define COUNT_TYPE(t)
rb_objspace::id_to_obj_tbl
st_table * id_to_obj_tbl
Definition: gc.c:822
rb_raw_obj_info
const char * rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
Definition: gc.c:11419
rb_id2name
const char * rb_id2name(ID)
Definition: symbol.c:801
void
void
Definition: rb_mjit_min_header-2.7.0.h:13273
rb_method_bmethod_struct::proc
VALUE proc
Definition: method.h:152
constant.h
bits_t
uintptr_t bits_t
Definition: gc.c:618
gc_stat_sym_heap_free_slots
@ gc_stat_sym_heap_free_slots
Definition: gc.c:8844
STATIC_SYM_P
#define STATIC_SYM_P(x)
Definition: ruby.h:411
TypedData_Make_Struct
#define TypedData_Make_Struct(klass, type, data_type, sval)
Definition: ruby.h:1244
idEq
@ idEq
Definition: id.h:96
Check_Type
#define Check_Type(v, t)
Definition: ruby.h:595
gc_stat_compat_sym_heap_final_slot
@ gc_stat_compat_sym_heap_final_slot
Definition: gc.c:8887
RGENGC_PROFILE
#define RGENGC_PROFILE
Definition: gc.c:421
RVALUE::free
struct RVALUE::@78::@79 free
rb_xcalloc_mul_add_mul
void * rb_xcalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w)
Definition: gc.c:10177
TRUE
#define TRUE
Definition: nkf.h:175
rb_transient_heap_mark
void rb_transient_heap_mark(VALUE obj, const void *ptr)
Definition: transient_heap.c:529
MALLOC_ALLOCATED_SIZE_CHECK
#define MALLOC_ALLOCATED_SIZE_CHECK
Definition: gc.c:480
st_foreach_with_replace
int st_foreach_with_replace(st_table *tab, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg)
Definition: st.c:1700
rb_gc_count
size_t rb_gc_count(void)
Definition: gc.c:8711
GPR_FLAG_IMMEDIATE_MARK
@ GPR_FLAG_IMMEDIATE_MARK
Definition: gc.c:513
RFILE
#define RFILE(obj)
Definition: ruby.h:1276
rb_memory_id
VALUE rb_memory_id(VALUE obj)
Definition: gc.c:3737
GPR_FLAG_CAPI
@ GPR_FLAG_CAPI
Definition: gc.c:507
T_FLOAT
#define T_FLOAT
Definition: ruby.h:527
STACK_UPPER
#define STACK_UPPER(x, a, b)
Definition: gc.h:83
RB_DEBUG_COUNTER_INC_IF
#define RB_DEBUG_COUNTER_INC_IF(type, cond)
Definition: debug_counter.h:377
rb_mGC
VALUE rb_mGC
Definition: gc.c:1000
ruby_mimmalloc
void * ruby_mimmalloc(size_t size)
Definition: gc.c:10187
xcalloc
#define xcalloc
Definition: defines.h:213
rb_include_module
void rb_include_module(VALUE klass, VALUE module)
Definition: class.c:869
GPR_FLAG_HAVE_FINALIZE
@ GPR_FLAG_HAVE_FINALIZE
Definition: gc.c:512
RVALUE::v3
VALUE v3
Definition: gc.c:605
rb_mark_generic_ivar
void rb_mark_generic_ivar(VALUE)
Definition: variable.c:973
rb_objspace_call_finalizer
void rb_objspace_call_finalizer(rb_objspace_t *objspace)
Definition: gc.c:3441
strtod
#define strtod(s, e)
Definition: util.h:76
RTypedData::type
const rb_data_type_t * type
Definition: ruby.h:1170
LL2NUM
#define LL2NUM(v)
Definition: rb_mjit_min_header-2.7.0.h:4240
rb_obj_hide
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
Definition: object.c:78
rb_objspace::oldmalloc_increase
size_t oldmalloc_increase
Definition: gc.c:799
RUBY_FL_WB_PROTECTED
@ RUBY_FL_WB_PROTECTED
Definition: ruby.h:842
rb_objspace::dont_incremental
unsigned int dont_incremental
Definition: gc.c:690
rb_ec_raised_clear
#define rb_ec_raised_clear(ec)
Definition: eval_intern.h:261
rb_objspace::total_freed_pages
size_t total_freed_pages
Definition: gc.c:782
rb_objspace::oldmalloc_increase_limit
size_t oldmalloc_increase_limit
Definition: gc.c:800
RVALUE::string
struct RString string
Definition: gc.c:577
rb_objspace_data_type_name
const char * rb_objspace_data_type_name(VALUE obj)
Definition: gc.c:2430
RZombie::basic
struct RBasic basic
Definition: gc.c:987
rb_objspace::total_allocated_objects
size_t total_allocated_objects
Definition: gc.c:704
ruby_gc_params_t::heap_init_slots
size_t heap_init_slots
Definition: gc.c:318
RArray::heap
struct RArray::@97::@98 heap
rb_objspace::malloc_params
struct rb_objspace::@82 malloc_params
force_finalize_list::table
VALUE table
Definition: gc.c:3424
gc_stat_sym_heap_marked_slots
@ gc_stat_sym_heap_marked_slots
Definition: gc.c:8846
GC_PROFILE_DETAIL_MEMORY
#define GC_PROFILE_DETAIL_MEMORY
Definition: gc.c:461
rb_io_t::write_lock
VALUE write_lock
Definition: io.h:101
id
const int id
Definition: nkf.c:209
onig_memsize
size_t onig_memsize(const regex_t *reg)
Definition: regcomp.c:5654
RZOMBIE
#define RZOMBIE(o)
Definition: gc.c:993
st_table::num_entries
st_index_t num_entries
Definition: st.h:86
rb_objspace::during_minor_gc
unsigned int during_minor_gc
Definition: gc.c:696
VM_METHOD_TYPE_REFINED
@ VM_METHOD_TYPE_REFINED
refinement
Definition: method.h:113
vsnprintf
int int vsnprintf(char *__restrict, size_t, const char *__restrict, __gnuc_va_list) __attribute__((__format__(__printf__
rb_objspace::considered_count_table
size_t considered_count_table[T_MASK]
Definition: gc.c:810
env
#define env
FIX2INT
#define FIX2INT(x)
Definition: ruby.h:717
clock_t
unsigned long clock_t
Definition: rb_mjit_min_header-2.7.0.h:1303
RVALUE::flags
VALUE flags
Definition: gc.c:569
rb_iseq_struct
Definition: vm_core.h:456
rb_hash_new
VALUE rb_hash_new(void)
Definition: hash.c:1501
RFloat
Definition: internal.h:798
ruby_gc_stressful
#define ruby_gc_stressful
Definition: gc.c:926
ATOMIC_PTR_EXCHANGE
#define ATOMIC_PTR_EXCHANGE(var, val)
Definition: ruby_atomic.h:186
RVALUE::bignum
struct RBignum bignum
Definition: gc.c:584
rb_gc_register_mark_object
void rb_gc_register_mark_object(VALUE obj)
Definition: gc.c:7063
RVALUE::env
rb_env_t env
Definition: gc.c:597
path
VALUE path
Definition: rb_mjit_min_header-2.7.0.h:7351
ruby_malloc_size_overflow
void ruby_malloc_size_overflow(size_t count, size_t elsize)
Definition: gc.c:10074
rb_define_module_under
VALUE rb_define_module_under(VALUE outer, const char *name)
Definition: class.c:797
gc_mode
#define gc_mode(objspace)
Definition: gc.c:950
mark_stack::chunk
stack_chunk_t * chunk
Definition: gc.c:648
rb_classext_struct::subclasses
rb_subclass_entry_t * subclasses
Definition: internal.h:1028
rb_objspace::next_object_id
VALUE next_object_id
Definition: gc.c:705
RVALUE::complex
struct RComplex complex
Definition: gc.c:588
VM_METHOD_TYPE_OPTIMIZED
@ VM_METHOD_TYPE_OPTIMIZED
Kernel::send, Proc::call, etc.
Definition: method.h:111
rb_objspace::records
gc_profile_record * records
Definition: gc.c:740
rb_str_buf_new
VALUE rb_str_buf_new(long)
Definition: string.c:1315
rb_warn
void rb_warn(const char *fmt,...)
Definition: error.c:313
rb_postponed_job_register_one
int rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data)
Definition: vm_trace.c:1608
memset
void * memset(void *, int, size_t)
is_incremental_marking
#define is_incremental_marking(objspace)
Definition: gc.c:961
RVALUE::values
struct RVALUE::@78::@81 values
ruby_stack_grow_direction
int ruby_stack_grow_direction
Definition: gc.c:4618
rb_gc_update_tbl_refs
void rb_gc_update_tbl_refs(st_table *ptr)
Definition: gc.c:7983
imemo_memo
@ imemo_memo
Definition: internal.h:1138
RVALUE_PAGE_MARKING
#define RVALUE_PAGE_MARKING(page, obj)
Definition: gc.c:1220
rb_data_type_struct::dmark
void(* dmark)(void *)
Definition: ruby.h:1151
rb_method_definition_struct::attr
rb_method_attr_t attr
Definition: method.h:171
size_t
long unsigned int size_t
Definition: rb_mjit_min_header-2.7.0.h:666
rb_func_lambda_new
VALUE rb_func_lambda_new(rb_block_call_func_t func, VALUE val, int min_argc, int max_argc)
Definition: proc.c:735
gc_stat_compat_sym_old_object
@ gc_stat_compat_sym_old_object
Definition: gc.c:8892
weakmap::wmap2obj
st_table * wmap2obj
Definition: gc.c:10317
CALC_EXACT_MALLOC_SIZE
#define CALC_EXACT_MALLOC_SIZE
Definition: gc.c:470
rb_singleton_class_internal_p
int rb_singleton_class_internal_p(VALUE sklass)
Definition: class.c:455
gc.h
BITMAP_INDEX
#define BITMAP_INDEX(p)
Definition: gc.c:881
HEAP_PAGE_ALIGN
@ HEAP_PAGE_ALIGN
Definition: gc.c:835
rb_ary_free
void rb_ary_free(VALUE ary)
Definition: array.c:786
RBASIC_CLEAR_CLASS
#define RBASIC_CLEAR_CLASS(obj)
Definition: internal.h:1981
rb_method_iseq_struct::cref
rb_cref_t * cref
class reference, should be marked
Definition: method.h:128
ST_STOP
@ ST_STOP
Definition: st.h:99
gc_stress_no_immediate_sweep
@ gc_stress_no_immediate_sweep
Definition: gc.c:7121
rb_scan_args
#define rb_scan_args(argc, argvp, fmt,...)
Definition: rb_mjit_min_header-2.7.0.h:6372
rb_objspace::heap_pages
struct rb_objspace::@85 heap_pages
RRational::num
VALUE num
Definition: internal.h:790
rb_gc_force_recycle
void rb_gc_force_recycle(VALUE obj)
Definition: gc.c:7011
INT2FIX
#define INT2FIX(i)
Definition: ruby.h:263
RVALUE::rstruct
struct RStruct rstruct
Definition: gc.c:583
T_MASK
#define T_MASK
Definition: md5.c:131
ruby_gc_params_t::oldobject_limit_factor
double oldobject_limit_factor
Definition: gc.c:326
GPR_FLAG_MAJOR_BY_NOFREE
@ GPR_FLAG_MAJOR_BY_NOFREE
Definition: gc.c:494
st_is_member
#define st_is_member(table, key)
Definition: st.h:97
snprintf
int snprintf(char *__restrict, size_t, const char *__restrict,...) __attribute__((__format__(__printf__
gc_stat_compat_sym_heap_free_slot
@ gc_stat_compat_sym_heap_free_slot
Definition: gc.c:8886
gc_report
#define gc_report
Definition: gc.c:1092
rb_during_gc
int rb_during_gc(void)
Definition: gc.c:8687
RObject
Definition: ruby.h:922
heap_page::final_slots
short final_slots
Definition: gc.c:849
rb_heap_struct::pages
struct list_head pages
Definition: gc.c:661
rb_mark_tbl_no_pin
void rb_mark_tbl_no_pin(st_table *tbl)
Definition: gc.c:5011
PRIxVALUE
#define PRIxVALUE
Definition: ruby.h:164
ruby_xmalloc
void * ruby_xmalloc(size_t size)
Definition: gc.c:11950
gc_stat_sym_total_allocated_objects
@ gc_stat_sym_total_allocated_objects
Definition: gc.c:8851
rb_int2str
VALUE rb_int2str(VALUE num, int base)
Definition: numeric.c:3562
RHash::ifnone
const VALUE ifnone
Definition: internal.h:893
mjit_remove_class_serial
void mjit_remove_class_serial(rb_serial_t class_serial)
gc_profile_record::heap_total_size
size_t heap_total_size
Definition: gc.c:529
rb_class_remove_from_module_subclasses
void rb_class_remove_from_module_subclasses(VALUE klass)
Definition: class.c:94
RSTRING_PTR
#define RSTRING_PTR(str)
Definition: ruby.h:1009
rb_objspace::during_incremental_marking
unsigned int during_incremental_marking
Definition: gc.c:699
re.h
time
time_t time(time_t *_timer)
rb_gc_mark_locations
void rb_gc_mark_locations(const VALUE *start, const VALUE *end)
Definition: gc.c:4699
SIZE_MAX
#define SIZE_MAX
Definition: ruby.h:307
GET_HEAP_MARK_BITS
#define GET_HEAP_MARK_BITS(x)
Definition: gc.c:891
STACKFRAME_FOR_CALL_CFUNC
#define STACKFRAME_FOR_CALL_CFUNC
Definition: gc.c:4662
rb_callable_method_entry_struct::owner
const VALUE owner
Definition: method.h:64
RTYPEDDATA_TYPE
#define RTYPEDDATA_TYPE(v)
Definition: ruby.h:1178
rb_gc_verify_internal_consistency
void rb_gc_verify_internal_consistency(void)
Definition: gc.c:6202
gc_stat_compat_sym_malloc_increase
@ gc_stat_compat_sym_malloc_increase
Definition: gc.c:8897
os_each_struct::num
size_t num
Definition: gc.c:3054
EXEC_EVENT_HOOK
#define EXEC_EVENT_HOOK(ec_, flag_, self_, id_, called_id_, klass_, data_)
Definition: vm_core.h:1935
LONG_LONG
#define LONG_LONG
Definition: rb_mjit_min_header-2.7.0.h:3939
rb_io_t::pathv
VALUE pathv
Definition: io.h:72
ruby_sized_xrealloc
void * ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size)
Definition: gc.c:10107
malloc_obj_info::size
size_t size
Definition: gc.c:9772
rb_strterm_mark
void rb_strterm_mark(VALUE obj)
Definition: ripper.c:765
rb_objspace::mark_func_data_struct::data
void * data
Definition: gc.c:715
rb_objspace
Definition: gc.c:676
GPR_FLAG_NEWOBJ
@ GPR_FLAG_NEWOBJ
Definition: gc.c:504
FLUSH_REGISTER_WINDOWS
#define FLUSH_REGISTER_WINDOWS
Definition: defines.h:431
GET_HEAP_WB_UNPROTECTED_BITS
#define GET_HEAP_WB_UNPROTECTED_BITS(x)
Definition: gc.c:895
mark_stack
Definition: gc.c:647
BITS_SIZE
@ BITS_SIZE
Definition: gc.c:620
st_init_numtable
st_table * st_init_numtable(void)
Definition: st.c:653
PUSH_MARK_FUNC_DATA
#define PUSH_MARK_FUNC_DATA(v)
Definition: gc.c:1097
ruby_gc_params_t::malloc_limit_min
size_t malloc_limit_min
Definition: gc.c:328
objspace_and_reason::reason
int reason
Definition: gc.c:7371
HEAP_PAGE_OBJ_LIMIT
@ HEAP_PAGE_OBJ_LIMIT
Definition: gc.c:839
RUBY_INTERNAL_EVENT_FREEOBJ
#define RUBY_INTERNAL_EVENT_FREEOBJ
Definition: ruby.h:2269
ruby_xrealloc_body
void * ruby_xrealloc_body(void *ptr, size_t new_size)
Definition: gc.c:10117
heap_page::wb_unprotected_bits
bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT]
Definition: gc.c:863
gc_stat_compat_sym_heap_swept_slot
@ gc_stat_compat_sym_heap_swept_slot
Definition: gc.c:8888
VALGRIND_MAKE_MEM_DEFINED
#define VALGRIND_MAKE_MEM_DEFINED(p, n)
Definition: zlib.c:24
rb_objspace::has_hook
unsigned int has_hook
Definition: gc.c:694
rb_objspace_of
#define rb_objspace_of(vm)
Definition: gc.c:901
VALUE
unsigned long VALUE
Definition: ruby.h:102
index
int index
Definition: rb_mjit_min_header-2.7.0.h:11246
M
#define M
Definition: mt19937.c:53
GET_VM
#define GET_VM()
Definition: vm_core.h:1764
RVALUE_MARKING_BITMAP
#define RVALUE_MARKING_BITMAP(obj)
Definition: gc.c:1216
rb_eArgError
VALUE rb_eArgError
Definition: error.c:923
each_obj_args
Definition: gc.c:2941
rb_clear_method_cache_by_class
void rb_clear_method_cache_by_class(VALUE)
Definition: vm_method.c:93
list_del
#define list_del(n)
Definition: rb_mjit_min_header-2.7.0.h:9041
encoding.h
OPT
#define OPT(o)
ruby_verbose
#define ruby_verbose
Definition: ruby.h:1925
rb_intern
#define rb_intern(str)
malloc
void * malloc(size_t) __attribute__((__malloc__)) __attribute__((__warn_unused_result__)) __attribute__((__alloc_size__(1)))
gc_stress_max
@ gc_stress_max
Definition: gc.c:7123
os_each_struct::of
VALUE of
Definition: gc.c:3055
RUBY_DATA_FUNC
void(* RUBY_DATA_FUNC)(void *)
Definition: ruby.h:1184
st_delete
int st_delete(st_table *tab, st_data_t *key, st_data_t *value)
Definition: st.c:1418
C
#define C(c, s)
RB_TYPE_P
#define RB_TYPE_P(obj, type)
Definition: ruby.h:560
rb_ary_memsize
RUBY_FUNC_EXPORTED size_t rb_ary_memsize(VALUE ary)
Definition: array.c:816
heap_allocatable_pages
#define heap_allocatable_pages
Definition: gc.c:915
RVALUE_PAGE_WB_UNPROTECTED
#define RVALUE_PAGE_WB_UNPROTECTED(page, obj)
Definition: gc.c:1218
NORETURN
NORETURN(static void negative_size_allocation_error(const char *))
rb_xmalloc_mul_add
void * rb_xmalloc_mul_add(size_t x, size_t y, size_t z)
Definition: gc.c:10156
RMatch
Definition: re.h:43
rb_str_memsize
size_t rb_str_memsize(VALUE)
Definition: string.c:1371
rb_gc_location
VALUE rb_gc_location(VALUE value)
Definition: gc.c:8111
TYPE
#define TYPE(x)
Definition: ruby.h:554
rb_const_entry_struct::value
VALUE value
Definition: constant.h:34
st_add_direct
void st_add_direct(st_table *tab, st_data_t key, st_data_t value)
Definition: st.c:1251
rb_aligned_malloc
void * rb_aligned_malloc(size_t alignment, size_t size)
Definition: gc.c:9610
imemo_iseq
@ imemo_iseq
Definition: internal.h:1140
RUBY_INTERNAL_EVENT_GC_ENTER
#define RUBY_INTERNAL_EVENT_GC_ENTER
Definition: ruby.h:2273
gc_stat_compat_sym_heap_live_slot
@ gc_stat_compat_sym_heap_live_slot
Definition: gc.c:8885
rb_method_definition_struct::type
rb_method_type_t type
Definition: rb_mjit_min_header-2.7.0.h:8877
imemo_env
@ imemo_env
Definition: internal.h:1133
weakmap
Definition: gc.c:10315
rb_objspace::pooled_slots
size_t pooled_slots
Definition: gc.c:816
RVALUE::flonum
struct RFloat flonum
Definition: gc.c:576
int
__inline__ int
Definition: rb_mjit_min_header-2.7.0.h:2839
nomem_error
#define nomem_error
Definition: gc.c:995
GET_STACK_BOUNDS
#define GET_STACK_BOUNDS(start, end, appendix)
Definition: gc.c:4947
RRational
Definition: internal.h:788
rb_objspace::gc_sweep_start_time
double gc_sweep_start_time
Definition: gc.c:774
rb_define_module
VALUE rb_define_module(const char *name)
Definition: class.c:772
rb_id_table_iterator_result
rb_id_table_iterator_result
Definition: id_table.h:8
old
VALUE ID VALUE old
Definition: rb_mjit_min_header-2.7.0.h:16133
Init_GC
void Init_GC(void)
Definition: gc.c:11832
ruby_rgengc_debug
int ruby_rgengc_debug
Definition: gc.c:388
SIGNED_VALUE
#define SIGNED_VALUE
Definition: ruby.h:104
rb_ast_mark
void rb_ast_mark(rb_ast_t *ast)
Definition: node.c:1342
rb_objspace
#define rb_objspace
Definition: gc.c:900
rb_iseq_path
VALUE rb_iseq_path(const rb_iseq_t *iseq)
Definition: iseq.c:1027
rb_heap_struct::total_pages
size_t total_pages
Definition: gc.c:666
rb_data_object_wrap
VALUE rb_data_object_wrap(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
Definition: gc.c:2376
ID_SCOPE_MASK
#define ID_SCOPE_MASK
Definition: id.h:32
I
#define I(s)
getenv
#define getenv(name)
Definition: win32.c:73
EC_JUMP_TAG
#define EC_JUMP_TAG(ec, st)
Definition: eval_intern.h:184
RSYMBOL
#define RSYMBOL(obj)
Definition: symbol.h:33
rb_int_ge
VALUE rb_int_ge(VALUE x, VALUE y)
Definition: numeric.c:4292
during_gc
#define during_gc
Definition: gc.c:922
rb_objspace::mode
unsigned int mode
Definition: gc.c:687
UINT2NUM
#define UINT2NUM(x)
Definition: ruby.h:1610
rb_objspace::heap_used_at_gc_start
size_t heap_used_at_gc_start
Definition: gc.c:776
rb_free_generic_ivar
void rb_free_generic_ivar(VALUE)
Definition: variable.c:993
FL_SEEN_OBJ_ID
#define FL_SEEN_OBJ_ID
Definition: ruby.h:1285
wmap_iter_arg::value
VALUE value
Definition: gc.c:10483
GC_ENABLE_INCREMENTAL_MARK
#define GC_ENABLE_INCREMENTAL_MARK
Definition: gc.c:464
rb_gc_disable
VALUE rb_gc_disable(void)
Definition: gc.c:9235
rb_iseq_constant_body::location
rb_iseq_location_t location
Definition: vm_core.h:399
rb_inspect
VALUE rb_inspect(VALUE)
Convenient wrapper of Object::inspect.
Definition: object.c:551
DWORD
IUnknown DWORD
Definition: win32ole.c:33
rb_objspace::profile
struct rb_objspace::@86 profile
IMEMO_NAME
#define IMEMO_NAME(x)
rb_id_table
Definition: id_table.c:40
ruby_stack_check
int ruby_stack_check(void)
Definition: gc.c:4671
rb_mark_end_proc
void rb_mark_end_proc(void)
Definition: eval_jump.c:78
rb_obj_rgengc_promoted_p
VALUE rb_obj_rgengc_promoted_p(VALUE obj)
Definition: gc.c:6970
rb_method_definition_struct::refined
rb_method_refined_t refined
Definition: method.h:173
heap_page::start
RVALUE * start
Definition: gc.c:858
verify_internal_consistency_struct::err_count
int err_count
Definition: gc.c:5893
gc_stress_no_major
@ gc_stress_no_major
Definition: gc.c:7120
rb_iseq_location_struct::first_lineno
VALUE first_lineno
Definition: vm_core.h:276
rb_objspace::obj_to_id_tbl
st_table * obj_to_id_tbl
Definition: gc.c:823
rb_str_cat2
#define rb_str_cat2
Definition: intern.h:912
STACK_LEVEL_MAX
#define STACK_LEVEL_MAX
Definition: gc.c:4602
rb_objspace::allocatable_pages
size_t allocatable_pages
Definition: gc.c:725
gc_stat_compat_sym_old_object_limit
@ gc_stat_compat_sym_old_object_limit
Definition: gc.c:8893
rb_class_detach_module_subclasses
void rb_class_detach_module_subclasses(VALUE klass)
Definition: class.c:145
BITS_BITLENGTH
@ BITS_BITLENGTH
Definition: gc.c:621
rb_method_refined_struct::orig_me
struct rb_method_entry_struct * orig_me
Definition: method.h:147
rmatch::char_offset
struct rmatch_offset * char_offset
Definition: re.h:39
CEILDIV
#define CEILDIV(i, mod)
Definition: gc.c:833
DYNAMIC_SYM_P
#define DYNAMIC_SYM_P(x)
Definition: ruby.h:412
Qundef
#define Qundef
Definition: ruby.h:470
RVALUE::memo
struct MEMO memo
Definition: gc.c:594
heap_cursor::page
struct heap_page * page
Definition: gc.c:7668
imemo
union @11::@13 imemo
rb_define_singleton_method
void rb_define_singleton_method(VALUE obj, const char *name, VALUE(*func)(ANYARGS), int argc)
Defines a singleton method for obj.
Definition: class.c:1755
T_RATIONAL
#define T_RATIONAL
Definition: ruby.h:541
CHAR_BIT
#define CHAR_BIT
Definition: ruby.h:227
RHASH_ST_TABLE_FLAG
@ RHASH_ST_TABLE_FLAG
Definition: internal.h:820
EXIT_FAILURE
#define EXIT_FAILURE
Definition: eval_intern.h:32
heap_pages_final_slots
#define heap_pages_final_slots
Definition: gc.c:917
gc_profile_record::heap_use_size
size_t heap_use_size
Definition: gc.c:528
rb_define_method
void rb_define_method(VALUE klass, const char *name, VALUE(*func)(ANYARGS), int argc)
Definition: class.c:1551
GET_EC
#define GET_EC()
Definition: vm_core.h:1766
RUBY_DTRACE_GC_HOOK
#define RUBY_DTRACE_GC_HOOK(name)
Definition: gc.c:10870
RVALUE::throw_data
struct vm_throw_data throw_data
Definition: gc.c:592
INT2NUM
#define INT2NUM(x)
Definition: ruby.h:1609
MALLOC_ALLOCATED_SIZE
#define MALLOC_ALLOCATED_SIZE
Definition: gc.c:477
ptr
struct RIMemo * ptr
Definition: debug.c:74
RVALUE::ment
struct rb_method_entry_struct ment
Definition: gc.c:595
vm_ifunc
IFUNC (Internal FUNCtion)
Definition: internal.h:1215
heap_page_body::header
struct heap_page_header header
Definition: gc.c:630
STACK_START
#define STACK_START
Definition: gc.c:4600
rb_obj_id
VALUE rb_obj_id(VALUE obj)
Definition: gc.c:3770
T_DATA
#define T_DATA
Definition: ruby.h:538
rb_objspace_reachable_objects_from
void rb_objspace_reachable_objects_from(VALUE obj, void(func)(VALUE, void *), void *data)
Definition: gc.c:9448
rb_method_iseq_struct::iseqptr
rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition: method.h:127
ruby_xmalloc_body
void * ruby_xmalloc_body(size_t size)
Definition: gc.c:10065
has_sweeping_pages
#define has_sweeping_pages(heap)
Definition: gc.c:970
Qfalse
#define Qfalse
Definition: ruby.h:467
idCall
@ idCall
Definition: rb_mjit_min_header-2.7.0.h:8727
weakmap::obj2wmap
st_table * obj2wmap
Definition: gc.c:10316
clock
clock_t clock(void)
is_marking
#define is_marking(objspace)
Definition: gc.c:953
gc_stat_compat_sym_remembered_shady_object
@ gc_stat_compat_sym_remembered_shady_object
Definition: gc.c:8890
uintptr_t
unsigned int uintptr_t
Definition: win32.h:106
rb_stdout
RUBY_EXTERN VALUE rb_stdout
Definition: ruby.h:2090
DBL2NUM
#define DBL2NUM(dbl)
Definition: ruby.h:967
__asan_region_is_poisoned
#define __asan_region_is_poisoned(x, y)
Definition: internal.h:110
GC_HEAP_FREE_SLOTS_GOAL_RATIO
#define GC_HEAP_FREE_SLOTS_GOAL_RATIO
Definition: gc.c:278
ssize_t
_ssize_t ssize_t
Definition: rb_mjit_min_header-2.7.0.h:1329
ruby_gc_params_t
Definition: gc.c:317
RVALUE::ast
rb_ast_t ast
Definition: gc.c:599
wmap_iter_arg
Definition: gc.c:10481
rb_objspace::minor_gc_count
size_t minor_gc_count
Definition: gc.c:751
RVALUE_WB_UNPROTECTED_BITMAP
#define RVALUE_WB_UNPROTECTED_BITMAP(obj)
Definition: gc.c:1214
rb_method_definition_struct::alias_count
int alias_count
Definition: method.h:165
rb_objspace::major_gc_count
size_t major_gc_count
Definition: gc.c:752
DSIZE_T
#define DSIZE_T
Definition: rb_mjit_min_header-2.7.0.h:5045
heap_page::flags
struct heap_page::@90 flags
RMATCH
#define RMATCH(obj)
Definition: re.h:50
rb_id2str
#define rb_id2str(id)
Definition: vm_backtrace.c:30
T_NODE
#define T_NODE
Definition: ruby.h:545
each_obj_args::data
void * data
Definition: gc.c:2944
rb_objspace_alloc
rb_objspace_t * rb_objspace_alloc(void)
Definition: gc.c:1586
dp
#define dp(v)
Definition: vm_debug.h:21
SPECIAL_CONST_P
#define SPECIAL_CONST_P(x)
Definition: ruby.h:1313
rb_ary_new3
#define rb_ary_new3
Definition: intern.h:104
st.h
NULL
#define NULL
Definition: _sdbm.c:101
exit
void exit(int __status) __attribute__((__noreturn__))
RVALUE::object
struct RObject object
Definition: gc.c:574
T_COMPLEX
#define T_COMPLEX
Definition: ruby.h:542
heap_cursor::objspace
rb_objspace_t * objspace
Definition: gc.c:7669
gc_list
Definition: gc.c:635
rb_print_backtrace
void rb_print_backtrace(void)
Definition: vm_dump.c:750
ST_DELETE
@ ST_DELETE
Definition: st.h:99
gc_raise_tag::exc
VALUE exc
Definition: gc.c:9504
uint32_t
unsigned int uint32_t
Definition: sha2.h:101
heap_page::before_sweep
unsigned int before_sweep
Definition: gc.c:851
imemo_svar
@ imemo_svar
special variable
Definition: internal.h:1135
gc_stat_sym_count
@ gc_stat_sym_count
Definition: gc.c:8838
rb_gc_mark_vm_stack_values
void rb_gc_mark_vm_stack_values(long n, const VALUE *values)
Definition: gc.c:4739
FL_TEST
#define FL_TEST(x, f)
Definition: ruby.h:1353
rb_class_remove_from_super_subclasses
void rb_class_remove_from_super_subclasses(VALUE klass)
Definition: class.c:76
RVALUE
Definition: gc.c:566
FL_WB_PROTECTED
#define FL_WB_PROTECTED
Definition: ruby.h:1279
HEAP_PAGE_ALIGN_LOG
#define HEAP_PAGE_ALIGN_LOG
Definition: gc.c:832
PRIsVALUE
#define PRIsVALUE
Definition: ruby.h:166
HEAP_PAGE_BITMAP_SIZE
@ HEAP_PAGE_BITMAP_SIZE
Definition: gc.c:841
PRINTF_ARGS
PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char *,...)), 2, 3)
stack_chunk::next
struct stack_chunk * next
Definition: gc.c:644
gc_stat_sym_total_freed_objects
@ gc_stat_sym_total_freed_objects
Definition: gc.c:8852
heap_pages_sorted_length
#define heap_pages_sorted_length
Definition: gc.c:912
rb_obj_respond_to
int rb_obj_respond_to(VALUE, ID, int)
Definition: vm_method.c:2180
RUBY_T_MASK
@ RUBY_T_MASK
Definition: ruby.h:518
RVALUE_PAGE_UNCOLLECTIBLE
#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj)
Definition: gc.c:1219
vfprintf
int int int int int int vfprintf(FILE *__restrict, const char *__restrict, __gnuc_va_list) __attribute__((__format__(__printf__
heap_pages_sorted
#define heap_pages_sorted
Definition: gc.c:910
BUFF_ARGS
#define BUFF_ARGS
FL_SET
#define FL_SET(x, f)
Definition: ruby.h:1359
GC_ENABLE_LAZY_SWEEP
#define GC_ENABLE_LAZY_SWEEP
Definition: gc.c:467
st_insert
int st_insert(st_table *tab, st_data_t key, st_data_t value)
Definition: st.c:1171
FIX2LONG
#define FIX2LONG(x)
Definition: ruby.h:394
ID2SYM
#define ID2SYM(x)
Definition: ruby.h:414
error
const rb_iseq_t const char * error
Definition: rb_mjit_min_header-2.7.0.h:13506
popcount_bits
#define popcount_bits
Definition: gc.c:623
heap_cursor::index
size_t index
Definition: gc.c:7667
rb_method_definition_struct::bmethod
rb_method_bmethod_t bmethod
Definition: method.h:174
strlen
size_t strlen(const char *)
TYPED_UPDATE_IF_MOVED
#define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing)
Definition: gc.c:1077
OBJ_FREEZE
#define OBJ_FREEZE(x)
Definition: ruby.h:1377
rb_gc_enable
VALUE rb_gc_enable(void)
Definition: gc.c:9209
T_SYMBOL
#define T_SYMBOL
Definition: ruby.h:540
T_OBJECT
#define T_OBJECT
Definition: ruby.h:523
rb_free_tmp_buffer
void rb_free_tmp_buffer(volatile VALUE *store)
Definition: gc.c:10257
root_objects_data::data
void * data
Definition: gc.c:9465
rb_alloc_tmp_buffer
void * rb_alloc_tmp_buffer(volatile VALUE *store, long len)
Definition: gc.c:10245
VM_METHOD_TYPE_IVAR
@ VM_METHOD_TYPE_IVAR
attr_reader or attr_accessor
Definition: method.h:105
rb_objspace::rincgc
struct rb_objspace::@89 rincgc
rb_hash_set_default_proc
VALUE rb_hash_set_default_proc(VALUE hash, VALUE proc)
Definition: hash.c:2169
rb_mark_set
void rb_mark_set(st_table *tbl)
Definition: gc.c:4798
gc_stress_full_mark_after_malloc_p
#define gc_stress_full_mark_after_malloc_p()
Definition: gc.c:7126
atexit
int atexit(void(*__func)(void))
rb_iseq_update_references
void rb_iseq_update_references(rb_iseq_t *iseq)
Definition: iseq.c:221
RAISED_NOMEMORY
@ RAISED_NOMEMORY
Definition: eval_intern.h:256
VM_ASSERT
#define VM_ASSERT(expr)
Definition: vm_core.h:56
ATOMIC_SET
#define ATOMIC_SET(var, val)
Definition: ruby_atomic.h:131
rb_big_hash
VALUE rb_big_hash(VALUE x)
Definition: bignum.c:6726
ruby_gc_params_t::gc_stress
VALUE gc_stress
Definition: gc.c:336
rb_free_method_entry
void rb_free_method_entry(const rb_method_entry_t *me)
Definition: vm_method.c:174
L
#define L(x)
Definition: asm.h:125
imemo_throw_data
@ imemo_throw_data
Definition: internal.h:1136
MEMO
MEMO.
Definition: internal.h:1278
rb_imemo_new
VALUE rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
Definition: gc.c:2306
RB_BLOCK_CALL_FUNC_ARGLIST
#define RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, callback_arg)
Definition: ruby.h:1964
HEAP_PAGE_BITMAP_PLANES
@ HEAP_PAGE_BITMAP_PLANES
Definition: gc.c:842
rb_check_arity
#define rb_check_arity
Definition: intern.h:347
rb_iseq_free
void rb_iseq_free(const rb_iseq_t *iseq)
Definition: iseq.c:89
rb_objspace::current_record
gc_profile_record * current_record
Definition: gc.c:741
RTypedData
Definition: ruby.h:1168
timespec::tv_nsec
long tv_nsec
Definition: missing.h:62
add
#define add(x, y)
Definition: date_strftime.c:23
onig_region_free
ONIG_EXTERN void onig_region_free(OnigRegion *region, int free_self)
Definition: regexec.c:343
rb_hash_new_with_size
MJIT_FUNC_EXPORTED VALUE rb_hash_new_with_size(st_index_t size)
Definition: hash.c:1507
ULL2NUM
#define ULL2NUM(v)
Definition: rb_mjit_min_header-2.7.0.h:4242
pc
rb_control_frame_t const VALUE * pc
Definition: rb_mjit_min_header-2.7.0.h:16923
ruby_initial_gc_stress_ptr
VALUE * ruby_initial_gc_stress_ptr
Definition: gc.c:905
VM_METHOD_TYPE_UNDEF
@ VM_METHOD_TYPE_UNDEF
Definition: method.h:109
gc_stat_compat_sym
gc_stat_compat_sym
Definition: gc.c:8879
v
int VALUE v
Definition: rb_mjit_min_header-2.7.0.h:12332
heap_cursor
Definition: gc.c:7665
GET_HEAP_PINNED_BITS
#define GET_HEAP_PINNED_BITS(x)
Definition: gc.c:892
ALLOC_N
#define ALLOC_N(type, n)
Definition: ruby.h:1663
RVALUE
struct RVALUE RVALUE
GET_HEAP_UNCOLLECTIBLE_BITS
#define GET_HEAP_UNCOLLECTIBLE_BITS(x)
Definition: gc.c:894
rb_vm_register_special_exception
#define rb_vm_register_special_exception(sp, e, m)
Definition: vm_core.h:1726
imemo_ast
@ imemo_ast
Definition: internal.h:1142
rb_gc_writebarrier
void rb_gc_writebarrier(VALUE a, VALUE b)
Definition: gc.c:6817
RData::dmark
void(* dmark)(void *)
Definition: ruby.h:1141
rb_transient_heap_update_references
void rb_transient_heap_update_references(void)
Definition: transient_heap.c:853
STR_SHARED_P
#define STR_SHARED_P(s)
Definition: internal.h:2158
ruby_global_symbols
rb_symbols_t ruby_global_symbols
Definition: symbol.c:66
rb_iseq_mark
void rb_iseq_mark(const rb_iseq_t *iseq)
Definition: iseq.c:287
gc_profile_record
struct gc_profile_record gc_profile_record
is_full_marking
#define is_full_marking(objspace)
Definition: gc.c:956
GC_HEAP_FREE_SLOTS_MIN_RATIO
#define GC_HEAP_FREE_SLOTS_MIN_RATIO
Definition: gc.c:275
ruby_gc_params_t::oldmalloc_limit_max
size_t oldmalloc_limit_max
Definition: gc.c:333
rb_raise
void rb_raise(VALUE exc, const char *fmt,...)
Definition: error.c:2669
each_obj_args::objspace
rb_objspace_t * objspace
Definition: gc.c:2942
each_obj_args::callback
each_obj_callback * callback
Definition: gc.c:2943
verify_internal_consistency_struct::zombie_object_count
size_t zombie_object_count
Definition: gc.c:5895
rb_imemo_tmpbuf_struct::cnt
size_t cnt
Definition: internal.h:1236
force_finalize_list
Definition: gc.c:3422
RESTORE_FINALIZER
#define RESTORE_FINALIZER()
GPR_FLAG_MAJOR_BY_OLDGEN
@ GPR_FLAG_MAJOR_BY_OLDGEN
Definition: gc.c:495
GC_HEAP_INIT_SLOTS
#define GC_HEAP_INIT_SLOTS
Definition: gc.c:259
gc_stat_compat_sym_last
@ gc_stat_compat_sym_last
Definition: gc.c:8903
T_FILE
#define T_FILE
Definition: ruby.h:534
rb_execution_context_struct::cfp
rb_control_frame_t * cfp
Definition: vm_core.h:847
rb_eRangeError
VALUE rb_eRangeError
Definition: error.c:926
rb_obj_gc_flags
size_t rb_obj_gc_flags(VALUE obj, ID *flags, size_t max)
Definition: gc.c:6976
ruby_gc_params_t::heap_free_slots
size_t heap_free_slots
Definition: gc.c:319
FL_PROMOTED0
#define FL_PROMOTED0
Definition: ruby.h:1280
rb_objspace::mark_stack
mark_stack_t mark_stack
Definition: gc.c:719
RVALUE::as
union RVALUE::@78 as
imemo_parser_strterm
@ imemo_parser_strterm
Definition: internal.h:1143
rb_callable_method_entry_struct::called_id
ID called_id
Definition: method.h:63
RUBY_DEFAULT_FREE
#define RUBY_DEFAULT_FREE
Definition: ruby.h:1201
gc_stat_sym_heap_sorted_length
@ gc_stat_sym_heap_sorted_length
Definition: gc.c:8840
if
if((ID)(DISPID) nameid !=nameid)
Definition: win32ole.c:357
LONG2NUM
#define LONG2NUM(x)
Definition: ruby.h:1644
heap_pages_freeable_pages
#define heap_pages_freeable_pages
Definition: gc.c:916
rb_heap_struct::sweeping_page
struct heap_page * sweeping_page
Definition: gc.c:662
obj
const VALUE VALUE obj
Definition: rb_mjit_min_header-2.7.0.h:5742
ATOMIC_SIZE_EXCHANGE
#define ATOMIC_SIZE_EXCHANGE(var, val)
Definition: ruby_atomic.h:140
ruby_gc_params_t::malloc_limit_growth_factor
double malloc_limit_growth_factor
Definition: gc.c:330
heap_page::page_node
struct list_node page_node
Definition: gc.c:860
rb_xrealloc_mul_add
void * rb_xrealloc_mul_add(const void *p, size_t x, size_t y, size_t z)
Definition: gc.c:10163
rb_objspace_t
struct rb_objspace rb_objspace_t
rb_obj_class
VALUE rb_obj_class(VALUE)
Equivalent to Object#class in Ruby.
Definition: object.c:217
rb_method_definition_struct::body
union rb_method_definition_struct::@118 body
onig_free
ONIG_EXTERN void onig_free(OnigRegex)
ELTS_SHARED
#define ELTS_SHARED
Definition: ruby.h:970
rb_objspace::deferred_final
VALUE deferred_final
Definition: gc.c:732
probes.h
GC_MALLOC_LIMIT_MIN
#define GC_MALLOC_LIMIT_MIN
Definition: gc.c:285
SIZEOF_VALUE
#define SIZEOF_VALUE
Definition: ruby.h:105
OBJ_ID_INITIAL
#define OBJ_ID_INITIAL
Definition: gc.c:2882
RGENGC_ESTIMATE_OLDMALLOC
#define RGENGC_ESTIMATE_OLDMALLOC
Definition: gc.c:431
rb_obj_is_proc
VALUE rb_obj_is_proc(VALUE)
Definition: proc.c:152
rb_obj_info
const MJIT_FUNC_EXPORTED char * rb_obj_info(VALUE obj)
Definition: gc.c:11655
realloc
void * realloc(void *, size_t) __attribute__((__warn_unused_result__)) __attribute__((__alloc_size__(2)))
RICLASS_IS_ORIGIN
#define RICLASS_IS_ORIGIN
Definition: internal.h:1085
rb_data_typed_object_zalloc
VALUE rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type)
PRIuSIZE
#define PRIuSIZE
Definition: ruby.h:208
gc_stat_sym_remembered_wb_unprotected_objects
@ gc_stat_sym_remembered_wb_unprotected_objects
Definition: gc.c:8859
rb_vraise
void rb_vraise(VALUE exc, const char *fmt, va_list ap)
Definition: error.c:2663
T_ICLASS
#define T_ICLASS
Definition: ruby.h:525
rb_transient_heap_finish_marking
void rb_transient_heap_finish_marking(void)
Definition: transient_heap.c:916
ULONG2NUM
#define ULONG2NUM(x)
Definition: ruby.h:1645
memcpy
void * memcpy(void *__restrict, const void *__restrict, size_t)
GC_PROFILE_RECORD_DEFAULT_SIZE
#define GC_PROFILE_RECORD_DEFAULT_SIZE
Definition: gc.c:10731
FIXNUM_FLAG
#define FIXNUM_FLAG
Definition: ruby.h:472
rb_ast_struct
Definition: node.h:399
gc_profile_record::heap_total_objects
size_t heap_total_objects
Definition: gc.c:527
MARK_OBJECT_ARY_BUCKET_SIZE
#define MARK_OBJECT_ARY_BUCKET_SIZE
Definition: gc.c:7059
rb_method_type_name
const char * rb_method_type_name(rb_method_type_t type)
Definition: gc.c:11378
rb_objspace::marked_slots
size_t marked_slots
Definition: gc.c:720
imemo_type
imemo_type
Definition: internal.h:1132
rb_objspace_markable_object_p
int rb_objspace_markable_object_p(VALUE obj)
Definition: gc.c:3598
mark_stack::unused_cache_size
size_t unused_cache_size
Definition: gc.c:653
rb_method_attr_struct::location
VALUE location
Definition: method.h:139
vm_svar
SVAR (Special VARiable)
Definition: internal.h:1181
RVALUE::moved
struct RMoved moved
Definition: gc.c:572
OLD_SYM
#define OLD_SYM(s)
ST_REPLACE
@ ST_REPLACE
Definition: st.h:99
RVALUE::iseq
const rb_iseq_t iseq
Definition: gc.c:596
rb_objspace_internal_object_p
int rb_objspace_internal_object_p(VALUE obj)
Definition: gc.c:3095
RCLASS_IV_INDEX_TBL
#define RCLASS_IV_INDEX_TBL(c)
Definition: internal.h:1074
calloc
void * calloc(size_t, size_t) __attribute__((__malloc__)) __attribute__((__warn_unused_result__)) __attribute__((__alloc_size__(1
double
double
Definition: rb_mjit_min_header-2.7.0.h:5923
rb_objspace::step_slots
size_t step_slots
Definition: gc.c:817
rb_gc_mark_maybe
void rb_gc_mark_maybe(VALUE obj)
Definition: gc.c:5044
RFile::fptr
struct rb_io_t * fptr
Definition: ruby.h:1136
VM_ENV_DATA_INDEX_ENV
#define VM_ENV_DATA_INDEX_ENV
Definition: vm_core.h:1196
gc_stat_sym_old_objects
@ gc_stat_sym_old_objects
Definition: gc.c:8861
gc_stat_sym_heap_live_slots
@ gc_stat_sym_heap_live_slots
Definition: gc.c:8843
DATA_PTR
#define DATA_PTR(dta)
Definition: ruby.h:1175
rb_gc_unregister_address
void rb_gc_unregister_address(VALUE *addr)
Definition: gc.c:7089
fprintf
int fprintf(FILE *__restrict, const char *__restrict,...) __attribute__((__format__(__printf__
TYPE_NAME
#define TYPE_NAME(t)
verify_internal_consistency_struct::remembered_shady_count
size_t remembered_shady_count
Definition: gc.c:5900
ruby_error_nomemory
@ ruby_error_nomemory
Definition: vm_core.h:508
TRY_WITH_GC
#define TRY_WITH_GC(alloc)
Definition: gc.c:9821
rb_malloc_info_show_results
void rb_malloc_info_show_results(void)
Definition: gc.c:9979
VM_UNREACHABLE
#define VM_UNREACHABLE(func)
Definition: vm_core.h:57
rb_heap_struct
Definition: gc.c:656
LIKELY
#define LIKELY(x)
Definition: ffi_common.h:125
EC_POP_TAG
#define EC_POP_TAG()
Definition: eval_intern.h:137
verify_internal_consistency_struct::old_object_count
size_t old_object_count
Definition: gc.c:5899
timespec::tv_sec
time_t tv_sec
Definition: missing.h:61
rb_imemo_tmpbuf_struct::ptr
VALUE * ptr
Definition: internal.h:1234
rb_objspace::moved_count_table
size_t moved_count_table[T_MASK]
Definition: gc.c:811
rb_cBasicObject
RUBY_EXTERN VALUE rb_cBasicObject
Definition: ruby.h:2009
FL_ABLE
#define FL_ABLE(x)
Definition: ruby.h:1351
MARKED_IN_BITMAP
#define MARKED_IN_BITMAP(bits, p)
Definition: gc.c:886
rb_objspace::size
size_t size
Definition: gc.c:743
gc_stat_compat_sym_oldmalloc_limit
@ gc_stat_compat_sym_oldmalloc_limit
Definition: gc.c:8901
RCLASS_M_TBL
#define RCLASS_M_TBL(c)
Definition: internal.h:1069
mjit_gc_exit_hook
void mjit_gc_exit_hook(void)
rb_check_frozen
#define rb_check_frozen(obj)
Definition: intern.h:319
rb_callable_method_entry_struct::defined_class
const VALUE defined_class
Definition: method.h:61
RSTRUCT_TRANSIENT_P
#define RSTRUCT_TRANSIENT_P(st)
Definition: internal.h:933
rb_objspace::run
int run
Definition: gc.c:738
RHASH_SIZE
#define RHASH_SIZE(hsh)
Definition: fbuffer.h:8
rb_method_definition_struct::alias
rb_method_alias_t alias
Definition: method.h:172
GPR_FLAG_IMMEDIATE_SWEEP
@ GPR_FLAG_IMMEDIATE_SWEEP
Definition: gc.c:511
rb_objspace::tomb_heap
rb_heap_t tomb_heap
Definition: gc.c:708
RCLASS_CONST_TBL
#define RCLASS_CONST_TBL(c)
Definition: internal.h:1067
ruby_gc_params_t::heap_free_slots_goal_ratio
double heap_free_slots_goal_ratio
Definition: gc.c:324
heap_page::has_remembered_objects
unsigned int has_remembered_objects
Definition: gc.c:852
gc_stat_compat_sym_remembered_shady_object_limit
@ gc_stat_compat_sym_remembered_shady_object_limit
Definition: gc.c:8891
rb_obj_is_thread
VALUE rb_obj_is_thread(VALUE obj)
Definition: vm.c:2655
RVALUE::hash
struct RHash hash
Definition: gc.c:580
UNEXPECTED_NODE
#define UNEXPECTED_NODE(func)
Definition: gc.c:2299
RRegexp
Definition: ruby.h:1112
SET
#define SET(name, attr)
st_init_numtable_with_size
st_table * st_init_numtable_with_size(st_index_t size)
Definition: st.c:660
ATOMIC_SIZE_CAS
#define ATOMIC_SIZE_CAS(var, oldval, val)
Definition: ruby_atomic.h:163
rb_hash_compare_by_id_p
MJIT_FUNC_EXPORTED VALUE rb_hash_compare_by_id_p(VALUE hash)
Definition: hash.c:4192
bool
#define bool
Definition: stdbool.h:13
mark_stack::cache
stack_chunk_t * cache
Definition: gc.c:649
list_head
Definition: rb_mjit_min_header-2.7.0.h:8975
ruby_xrealloc
void * ruby_xrealloc(void *ptr, size_t new_size)
Definition: gc.c:11980
rb_iseq_location_struct::label
VALUE label
Definition: vm_core.h:275
gc_stat_sym
gc_stat_sym
Definition: gc.c:8837
T_FIXNUM
#define T_FIXNUM
Definition: ruby.h:535
malloc_limit
#define malloc_limit
Definition: gc.c:907
gc_stat_compat_sym_total_freed_object
@ gc_stat_compat_sym_total_freed_object
Definition: gc.c:8896
GPR_FLAG_FULL_MARK
@ GPR_FLAG_FULL_MARK
Definition: gc.c:514
RVALUE::alloc
struct rb_imemo_tmpbuf_struct alloc
Definition: gc.c:598
USE_RGENGC
#define USE_RGENGC
Definition: ruby.h:791
rb_ary_cat
VALUE rb_ary_cat(VALUE ary, const VALUE *argv, long len)
Definition: array.c:1208
rb_objspace::need_major_gc
int need_major_gc
Definition: gc.c:791
st_foreach_callback_func
int st_foreach_callback_func(st_data_t, st_data_t, st_data_t)
Definition: st.h:137
weakmap::final
VALUE final
Definition: gc.c:10318
APPENDF
#define APPENDF(f)
RSTRUCT_EMBED_LEN_MASK
#define RSTRUCT_EMBED_LEN_MASK
Definition: internal.h:920
rb_io_memsize
RUBY_FUNC_EXPORTED size_t rb_io_memsize(const rb_io_t *fptr)
Definition: io.c:4760
rb_ary_tmp_new
VALUE rb_ary_tmp_new(long capa)
Definition: array.c:768
rb_hash_ar_table_size
size_t rb_hash_ar_table_size(void)
Definition: hash.c:355
i
uint32_t i
Definition: rb_mjit_min_header-2.7.0.h:5464
FL_EXIVAR
#define FL_EXIVAR
Definition: ruby.h:1286
list_for_each
#define list_for_each(h, i, member)
Definition: rb_mjit_min_header-2.7.0.h:9094
rb_ast_memsize
size_t rb_ast_memsize(const rb_ast_t *ast)
Definition: node.c:1375
RHASH_EMPTY_P
#define RHASH_EMPTY_P(h)
Definition: ruby.h:1131
gc_list::next
struct gc_list * next
Definition: gc.c:637
RFile
Definition: ruby.h:1134
RZombie
Definition: gc.c:986
rb_clear_constant_cache
void rb_clear_constant_cache(void)
Definition: vm_method.c:87
heap_page::free_slots
short free_slots
Definition: gc.c:847
st_data_t
RUBY_SYMBOL_EXPORT_BEGIN typedef unsigned long st_data_t
Definition: st.h:22
st_numhash
st_index_t st_numhash(st_data_t n)
Definition: st.c:2176
GPR_DEFAULT_REASON
@ GPR_DEFAULT_REASON
Definition: gc.c:516
force_finalize_list::obj
VALUE obj
Definition: gc.c:3423
VM_METHOD_TYPE_CFUNC
@ VM_METHOD_TYPE_CFUNC
C method.
Definition: method.h:103
heap_pages_deferred_final
#define heap_pages_deferred_final
Definition: gc.c:918
ID_TABLE_REPLACE
@ ID_TABLE_REPLACE
Definition: id_table.h:12
GPR_FLAG_METHOD
@ GPR_FLAG_METHOD
Definition: gc.c:506
rb_jmp_buf
#define rb_jmp_buf
Definition: gc.c:82
rb_ast_update_references
void rb_ast_update_references(rb_ast_t *ast)
Definition: node.c:1332
st_init_strtable
st_table * st_init_strtable(void)
Definition: st.c:668
RCLASS
#define RCLASS(obj)
Definition: ruby.h:1269
rb_execution_context_struct::errinfo
VALUE errinfo
Definition: vm_core.h:875
obj_id_to_ref
#define obj_id_to_ref(objid)
Definition: gc.c:975
INT_MAX
#define INT_MAX
Definition: rb_mjit_min_header-2.7.0.h:4052
ruby_xcalloc_body
void * ruby_xcalloc_body(size_t n, size_t size)
Definition: gc.c:10098
rb_ary_last
VALUE rb_ary_last(int argc, const VALUE *argv, VALUE ary)
Definition: array.c:1677
rb_imemo_tmpbuf_struct::next
struct rb_imemo_tmpbuf_struct * next
Definition: internal.h:1235
HEAP_PAGE_ALIGN_MASK
@ HEAP_PAGE_ALIGN_MASK
Definition: gc.c:836
heap_pages_lomem
#define heap_pages_lomem
Definition: gc.c:913
T_REGEXP
#define T_REGEXP
Definition: ruby.h:529
long
#define long
Definition: rb_mjit_min_header-2.7.0.h:2880
RVALUE_OLD_AGE
#define RVALUE_OLD_AGE
Definition: gc.c:1222
VM_METHOD_TYPE_NOTIMPLEMENTED
@ VM_METHOD_TYPE_NOTIMPLEMENTED
Definition: method.h:110
rb_gc_writebarrier_unprotect
void rb_gc_writebarrier_unprotect(VALUE obj)
Definition: gc.c:6838
rb_objspace::next_index
size_t next_index
Definition: gc.c:742
MARK_IN_BITMAP
#define MARK_IN_BITMAP(bits, p)
Definition: gc.c:887
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void mark_locations_array(rb_objspace_t *objspace, register const VALUE *x, register long n))
FL_UNSET
#define FL_UNSET(x, f)
Definition: ruby.h:1361
GPR_FLAG_MALLOC
@ GPR_FLAG_MALLOC
Definition: gc.c:505
rb_cref_struct
CREF (Class REFerence)
Definition: method.h:41
mjit.h
RCLASS_SERIAL
#define RCLASS_SERIAL(c)
Definition: internal.h:1078
rb_hash_lookup
VALUE rb_hash_lookup(VALUE hash, VALUE key)
Definition: hash.c:1990
rb_errinfo
VALUE rb_errinfo(void)
The current exception in the current thread.
Definition: eval.c:1881
rb_objspace::range
RVALUE * range[2]
Definition: gc.c:727
rb_ary_push
VALUE rb_ary_push(VALUE ary, VALUE item)
Definition: array.c:1195
ruby_xcalloc
void * ruby_xcalloc(size_t n, size_t size)
Definition: gc.c:11970
GET_HEAP_PAGE
#define GET_HEAP_PAGE(x)
Definition: gc.c:878
st_index_t
st_data_t st_index_t
Definition: st.h:50
RComplex::real
VALUE real
Definition: internal.h:807
st_hash_type
Definition: st.h:61
vm_throw_data
THROW_DATA.
Definition: internal.h:1193
VM_METHOD_TYPE_BMETHOD
@ VM_METHOD_TYPE_BMETHOD
Definition: method.h:106
EC_EXEC_TAG
#define EC_EXEC_TAG()
Definition: eval_intern.h:181
rb_env_t
Definition: vm_core.h:1055
va_end
#define va_end(v)
Definition: rb_mjit_min_header-2.7.0.h:3979
objspace_and_reason::objspace
rb_objspace_t * objspace
Definition: gc.c:7370
cnt
rb_atomic_t cnt[RUBY_NSIG]
Definition: signal.c:503
ruby_sized_xrealloc2
void * ruby_sized_xrealloc2(void *ptr, size_t n, size_t size, size_t old_n)
Definition: gc.c:10126
transient_heap.h
rb_obj_freeze
VALUE rb_obj_freeze(VALUE)
Make the object unmodifiable.
Definition: object.c:1080
rb_method_definition_struct::iseq
rb_method_iseq_t iseq
Definition: method.h:169
list_empty
#define list_empty(h)
Definition: rb_mjit_min_header-2.7.0.h:9030
RHASH_IFNONE
#define RHASH_IFNONE(h)
Definition: ruby.h:1129
rb_transient_heap_promote
void rb_transient_heap_promote(VALUE obj)
Definition: transient_heap.c:640
onig_region_memsize
size_t onig_region_memsize(const OnigRegion *regs)
Definition: regcomp.c:5669
rb_classext_struct::refined_class
const VALUE refined_class
Definition: internal.h:1040
vm_core.h
stderr
#define stderr
Definition: rb_mjit_min_header-2.7.0.h:1485
RCLASS_CALLABLE_M_TBL
#define RCLASS_CALLABLE_M_TBL(c)
Definition: internal.h:1073
GPR_FLAG_MAJOR_BY_SHADY
@ GPR_FLAG_MAJOR_BY_SHADY
Definition: gc.c:496
CHECK
#define CHECK(sub)
Definition: compile.c:448
rb_source_location_cstr
const char * rb_source_location_cstr(int *pline)
Definition: vm.c:1376
ruby_xmalloc2_body
void * ruby_xmalloc2_body(size_t n, size_t size)
Definition: gc.c:10082
rb_eTypeError
VALUE rb_eTypeError
Definition: error.c:922
gc_stat_sym_remembered_wb_unprotected_objects_limit
@ gc_stat_sym_remembered_wb_unprotected_objects_limit
Definition: gc.c:8860
rb_gc_mark_machine_stack
void rb_gc_mark_machine_stack(const rb_execution_context_t *ec)
Definition: gc.c:4981
RBASIC_CLASS
#define RBASIC_CLASS(obj)
Definition: ruby.h:906
SIZED_REALLOC_N
#define SIZED_REALLOC_N(var, type, n, old_n)
Definition: internal.h:1657
RRational::den
VALUE den
Definition: internal.h:791
rb_obj_is_mutex
VALUE rb_obj_is_mutex(VALUE obj)
Definition: thread_sync.c:131
rb_heap_t
struct rb_heap_struct rb_heap_t
ALLOC
#define ALLOC(type)
Definition: ruby.h:1664
STACK_CHUNK_SIZE
#define STACK_CHUNK_SIZE
Definition: gc.c:640
T_CLASS
#define T_CLASS
Definition: ruby.h:524
rb_method_alias_struct::original_me
struct rb_method_entry_struct * original_me
Definition: method.h:143
T_MATCH
#define T_MATCH
Definition: ruby.h:539
rb_free_const_table
void rb_free_const_table(struct rb_id_table *tbl)
Definition: gc.c:2491
RARRAY_EMBED_FLAG
@ RARRAY_EMBED_FLAG
Definition: ruby.h:1029
rb_eRuntimeError
VALUE rb_eRuntimeError
Definition: error.c:920
rb_gc_mark_global_tbl
void rb_gc_mark_global_tbl(void)
Definition: variable.c:434
rb_objspace::immediate_sweep
unsigned int immediate_sweep
Definition: gc.c:688
mjit_gc_start_hook
void mjit_gc_start_hook(void)
gc_mode
gc_mode
Definition: gc.c:670
rb_imemo_tmpbuf_parser_heap
rb_imemo_tmpbuf_t * rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt)
Definition: gc.c:2326
rb_objspace::total_freed_objects
size_t total_freed_objects
Definition: gc.c:780
rb_size_mul_or_raise
size_t rb_size_mul_or_raise(size_t x, size_t y, VALUE exc)
Definition: gc.c:192
rb_objspace::gc_stressful
unsigned int gc_stressful
Definition: gc.c:693
rb_gc_copy_finalizer
void rb_gc_copy_finalizer(VALUE dest, VALUE obj)
Definition: gc.c:3295
RETURN_ENUMERATOR
#define RETURN_ENUMERATOR(obj, argc, argv)
Definition: intern.h:279
mod
#define mod(x, y)
Definition: date_strftime.c:28
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition: ruby.h:1101
rb_control_frame_struct
Definition: vm_core.h:760
RHASH_TRANSIENT_P
#define RHASH_TRANSIENT_P(hash)
Definition: internal.h:870
rb_mv_generic_ivar
void rb_mv_generic_ivar(VALUE src, VALUE dst)
Definition: variable.c:983
rb_newobj_of
VALUE rb_newobj_of(VALUE klass, VALUE flags)
Definition: gc.c:2294
FL_USHIFT
#define FL_USHIFT
Definition: ruby.h:1289
ROBJECT_IVPTR
#define ROBJECT_IVPTR(o)
Definition: ruby.h:937
gc_stat_sym_heap_final_slots
@ gc_stat_sym_heap_final_slots
Definition: gc.c:8845
RTYPEDDATA_DATA
#define RTYPEDDATA_DATA(v)
Definition: ruby.h:1179
heap_page::total_slots
short total_slots
Definition: gc.c:846
symbol.h
size
int size
Definition: encoding.c:58
gc_stat_compat_sym_oldmalloc_increase
@ gc_stat_compat_sym_oldmalloc_increase
Definition: gc.c:8900
FALSE
#define FALSE
Definition: nkf.h:174
rb_vm_mark
void rb_vm_mark(void *ptr)
Definition: vm.c:2243
ruby_gc_params_t::oldmalloc_limit_growth_factor
double oldmalloc_limit_growth_factor
Definition: gc.c:334
FIXNUM_P
#define FIXNUM_P(f)
Definition: ruby.h:396
optional::right
size_t right
Definition: gc.c:92
GET_HEAP_MARKING_BITS
#define GET_HEAP_MARKING_BITS(x)
Definition: gc.c:896
RString
Definition: ruby.h:988
rb_gc_adjust_memory_usage
void rb_gc_adjust_memory_usage(ssize_t diff)
Definition: gc.c:10300
rb_ast_free
void rb_ast_free(rb_ast_t *ast)
Definition: node.c:1354
rb_objspace_marked_object_p
int rb_objspace_marked_object_p(VALUE obj)
Definition: gc.c:5222
RCLASS_SUPER
#define RCLASS_SUPER(c)
Definition: classext.h:16
rb_gc
void rb_gc(void)
Definition: gc.c:8679
rb_gc_mark_movable
void rb_gc_mark_movable(VALUE ptr)
Definition: gc.c:5206
gc_stat_sym_malloc_increase_bytes
@ gc_stat_sym_malloc_increase_bytes
Definition: gc.c:8853
rb_objspace_free
void rb_objspace_free(rb_objspace_t *objspace)
Definition: gc.c:1600
gc_stat_sym_heap_tomb_pages
@ gc_stat_sym_heap_tomb_pages
Definition: gc.c:8848
rb_id_table_memsize
size_t rb_id_table_memsize(const struct rb_id_table *tbl)
Definition: id_table.c:123
POP_MARK_FUNC_DATA
#define POP_MARK_FUNC_DATA()
Definition: gc.c:1101
VM_METHOD_TYPE_ZSUPER
@ VM_METHOD_TYPE_ZSUPER
Definition: method.h:107
ruby_xmalloc2
void * ruby_xmalloc2(size_t n, size_t size)
Definition: gc.c:11960
rb_objspace::latest_gc_info
int latest_gc_info
Definition: gc.c:739
verify_internal_consistency_struct::objspace
rb_objspace_t * objspace
Definition: gc.c:5892
rb_gc_writebarrier_remember
MJIT_FUNC_EXPORTED void rb_gc_writebarrier_remember(VALUE obj)
Definition: gc.c:6875
rb_obj_is_fiber
VALUE rb_obj_is_fiber(VALUE obj)
Definition: cont.c:1041
mark_stack::cache_size
size_t cache_size
Definition: gc.c:652
rb_gc_free_dsymbol
void rb_gc_free_dsymbol(VALUE)
Definition: symbol.c:678
list
struct rb_encoding_entry * list
Definition: encoding.c:56
MEMZERO
#define MEMZERO(p, type, n)
Definition: ruby.h:1752
GPR_FLAG_NONE
@ GPR_FLAG_NONE
Definition: gc.c:492
gc_raise_tag::fmt
const char * fmt
Definition: gc.c:9505
rb_gc_start
VALUE rb_gc_start(void)
Definition: gc.c:8672
PRI_PIDT_PREFIX
#define PRI_PIDT_PREFIX
Definition: rb_mjit_min_header-2.7.0.h:103
rb_gc_register_address
void rb_gc_register_address(VALUE *addr)
Definition: gc.c:7077
rb_str_new_cstr
#define rb_str_new_cstr(str)
Definition: rb_mjit_min_header-2.7.0.h:6117
gc_stat_sym_heap_eden_pages
@ gc_stat_sym_heap_eden_pages
Definition: gc.c:8847
RGENGC_OLD_NEWOBJ_CHECK
#define RGENGC_OLD_NEWOBJ_CHECK
Definition: gc.c:412
getpid
pid_t getpid(void)
RHASH
#define RHASH(obj)
Definition: internal.h:859
rb_iseq_location_struct::pathobj
VALUE pathobj
Definition: vm_core.h:273
RVALUE::cref
rb_cref_t cref
Definition: gc.c:590
rb_gc_stat
size_t rb_gc_stat(VALUE key)
Definition: gc.c:9174
RArray
Definition: ruby.h:1048
NOINLINE
NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace))
T_NONE
#define T_NONE
Definition: ruby.h:521
N
#define N
Definition: lgamma_r.c:20
heap_page
Definition: gc.c:845
ROBJECT
#define ROBJECT(obj)
Definition: ruby.h:1268
rb_data_object_zalloc
VALUE rb_data_object_zalloc(VALUE, size_t, RUBY_DATA_FUNC, RUBY_DATA_FUNC)
rb_objspace::sorted
struct heap_page ** sorted
Definition: gc.c:723
rb_objspace::eden_heap
rb_heap_t eden_heap
Definition: gc.c:707
rb_class_path_cached
VALUE rb_class_path_cached(VALUE)
Definition: variable.c:162
rb_data_type_struct::dcompact
void(* dcompact)(void *)
Definition: ruby.h:1154
rb_ec_raised_set
#define rb_ec_raised_set(ec, f)
Definition: eval_intern.h:258
me
const rb_callable_method_entry_t * me
Definition: rb_mjit_min_header-2.7.0.h:13226
ruby_xrealloc2
void * ruby_xrealloc2(void *ptr, size_t n, size_t new_size)
Definition: gc.c:11990
st_update
int st_update(st_table *tab, st_data_t key, st_update_callback_func *func, st_data_t arg)
Definition: st.c:1510
RGENGC_DEBUG
#define RGENGC_DEBUG
Definition: gc.c:380
NEW_SYM
#define NEW_SYM(s)
heap_page::free_next
struct heap_page * free_next
Definition: gc.c:857
key
key
Definition: openssl_missing.h:181
T_HASH
#define T_HASH
Definition: ruby.h:531
RCLASS_IV_TBL
#define RCLASS_IV_TBL(c)
Definition: internal.h:1066
REQUIRED_SIZE_BY_MALLOC
@ REQUIRED_SIZE_BY_MALLOC
Definition: gc.c:837
heap_page::in_tomb
unsigned int in_tomb
Definition: gc.c:854
list_next
#define list_next(h, i, member)
Definition: rb_mjit_min_header-2.7.0.h:9098
rb_objspace::allocated_pages
size_t allocated_pages
Definition: gc.c:724
rb_objspace::rcompactor
struct rb_objspace::@88 rcompactor
RHASH_ST_TABLE_P
#define RHASH_ST_TABLE_P(h)
Definition: internal.h:861
gc_profile_record_flag
gc_profile_record_flag
Definition: gc.c:491
SET_STACK_END
#define SET_STACK_END
Definition: gc.c:4598
rb_heap_struct::freelist
RVALUE * freelist
Definition: gc.c:657
rb_global_variable
void rb_global_variable(VALUE *var)
Definition: gc.c:7112
RCLASS_EXT
#define RCLASS_EXT(c)
Definition: classext.h:15
rb_copy_wb_protected_attribute
void rb_copy_wb_protected_attribute(VALUE dest, VALUE obj)
Definition: gc.c:6938
RUBY_INTERNAL_EVENT_OBJSPACE_MASK
#define RUBY_INTERNAL_EVENT_OBJSPACE_MASK
Definition: ruby.h:2275
RHash
Definition: internal.h:887
CLASS_OF
#define CLASS_OF(v)
Definition: ruby.h:484
src
__inline__ const void *__restrict src
Definition: rb_mjit_min_header-2.7.0.h:2836
rb_str_buf_append
VALUE rb_str_buf_append(VALUE, VALUE)
Definition: string.c:2950
fmt
const VALUE int int int int int int VALUE char * fmt
Definition: rb_mjit_min_header-2.7.0.h:6462
rb_io_t::writeconv_asciicompat
VALUE writeconv_asciicompat
Definition: io.h:96
strcmp
int strcmp(const char *, const char *)
T_MODULE
#define T_MODULE
Definition: ruby.h:526
ruby_mimfree
void ruby_mimfree(void *ptr)
Definition: gc.c:10217
force_finalize_list::next
struct force_finalize_list * next
Definition: gc.c:3425
rb_gcdebug_print_obj_condition
void rb_gcdebug_print_obj_condition(VALUE obj)
RString::as
union RString::@94 as
RVALUE::imemo
union RVALUE::@78::@80 imemo
RString::heap
struct RString::@94::@95 heap
RVALUE::ifunc
struct vm_ifunc ifunc
Definition: gc.c:593
RVALUE::v2
VALUE v2
Definition: gc.c:604
gc_stat_sym_minor_gc_count
@ gc_stat_sym_minor_gc_count
Definition: gc.c:8856
heap_allocated_pages
#define heap_allocated_pages
Definition: gc.c:911
gc_event_hook_available_p
#define gc_event_hook_available_p(objspace)
Definition: gc.c:2099
RMoved
Definition: internal.h:908
TAG_RAISE
#define TAG_RAISE
Definition: vm_core.h:203
RARRAY_LEN
#define RARRAY_LEN(a)
Definition: ruby.h:1070
RUBY_INTERNAL_EVENT_GC_EXIT
#define RUBY_INTERNAL_EVENT_GC_EXIT
Definition: ruby.h:2274
st_foreach
int st_foreach(st_table *tab, st_foreach_callback_func *func, st_data_t arg)
Definition: st.c:1718
RHASH_AR_TABLE
#define RHASH_AR_TABLE(hash)
Definition: internal.h:855
ruby_get_stack_grow_direction
int ruby_get_stack_grow_direction(volatile VALUE *addr)
Definition: gc.c:4620
imemo_ment
@ imemo_ment
Definition: internal.h:1139
rb_define_module_function
void rb_define_module_function(VALUE module, const char *name, VALUE(*func)(ANYARGS), int argc)
Defines a module function for module.
Definition: class.c:1771
rb_method_bmethod_struct::hooks
struct rb_hook_list_struct * hooks
Definition: method.h:153
rb_objspace::finalizer_table
st_table * finalizer_table
Definition: gc.c:735
RSTRUCT_LEN
#define RSTRUCT_LEN(st)
Definition: ruby.h:1255
verify_internal_consistency_struct::parent
VALUE parent
Definition: gc.c:5898
rb_id_table_free
void rb_id_table_free(struct rb_id_table *tbl)
Definition: id_table.c:102
UPDATE_IF_MOVED
#define UPDATE_IF_MOVED(_objspace, _thing)
Definition: gc.c:1083
optional::left
bool left
Definition: gc.c:91
ruby_gc_params_t::oldmalloc_limit_min
size_t oldmalloc_limit_min
Definition: gc.c:332
rb_method_type_t
rb_method_type_t
Definition: method.h:101
rb_hook_list_mark
void rb_hook_list_mark(rb_hook_list_t *hooks)
Definition: vm_trace.c:53
FL_TEST_RAW
#define FL_TEST_RAW(x, f)
Definition: ruby.h:1352
rb_method_refined_struct::owner
VALUE owner
Definition: method.h:148
rb_objspace::during_compacting
unsigned int during_compacting
Definition: gc.c:692
heap_page::mark_bits
bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT]
Definition: gc.c:866
ATOMIC_SIZE_ADD
#define ATOMIC_SIZE_ADD(var, val)
Definition: ruby_atomic.h:138
rb_cObject
RUBY_EXTERN VALUE rb_cObject
Definition: ruby.h:2010
rb_event_flag_t
uint32_t rb_event_flag_t
Definition: ruby.h:2278
GC_HEAP_GROWTH_FACTOR
#define GC_HEAP_GROWTH_FACTOR
Definition: gc.c:265
verify_internal_consistency_struct
Definition: gc.c:5891
is_lazy_sweeping
#define is_lazy_sweeping(heap)
Definition: gc.c:971
buf
unsigned char buf[MIME_BUF_SIZE]
Definition: nkf.c:4322
n
const char size_t n
Definition: rb_mjit_min_header-2.7.0.h:5456
RVALUE_AGE_SHIFT
#define RVALUE_AGE_SHIFT
Definition: gc.c:1223
rb_classext_t
struct rb_classext_struct rb_classext_t
Definition: internal.h:1045
T_BIGNUM
#define T_BIGNUM
Definition: ruby.h:533
gc_mode_none
@ gc_mode_none
Definition: gc.c:671
TypedData_Get_Struct
#define TypedData_Get_Struct(obj, type, data_type, sval)
Definition: ruby.h:1252
rb_funcall
#define rb_funcall(recv, mid, argc,...)
Definition: rb_mjit_min_header-2.7.0.h:6585
rb_data_typed_object_wrap
VALUE rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
Definition: gc.c:2397
rb_objspace_each_objects
void rb_objspace_each_objects(each_obj_callback *callback, void *data)
Definition: gc.c:3025
root_objects_data::category
const char * category
Definition: gc.c:9463
rb_objspace_data_type_memsize
size_t rb_objspace_data_type_memsize(VALUE obj)
Definition: gc.c:2417
rb_str_append
VALUE rb_str_append(VALUE, VALUE)
Definition: string.c:2965
VM_METHOD_TYPE_MISSING
@ VM_METHOD_TYPE_MISSING
wrapper for method_missing(id)
Definition: method.h:112
rb_bug
void rb_bug(const char *fmt,...)
Definition: error.c:634
rb_control_frame_struct::self
VALUE self
Definition: vm_core.h:764
RVALUE::match
struct RMatch match
Definition: gc.c:586
rb_objspace_reachable_objects_from_root
void rb_objspace_reachable_objects_from_root(void(func)(const char *category, VALUE, void *), void *passing_data)
Definition: gc.c:9476
gc_stat_compat_sym_heap_increment
@ gc_stat_compat_sym_heap_increment
Definition: gc.c:8883
rb_io_t::rb_io_enc_t::ecopts
VALUE ecopts
Definition: io.h:89
rmatch::char_offset_num_allocated
int char_offset_num_allocated
Definition: re.h:40
GC_ASSERT
#define GC_ASSERT(expr)
Definition: gc.c:403
internal.h
dont_gc
#define dont_gc
Definition: gc.c:921
gc_mode_set
#define gc_mode_set(objspace, mode)
Definition: gc.c:951
rb_objspace::last_major_gc
size_t last_major_gc
Definition: gc.c:792
T_ARRAY
#define T_ARRAY
Definition: ruby.h:530
rb_objspace::increase
size_t increase
Definition: gc.c:679
arg
VALUE arg
Definition: rb_mjit_min_header-2.7.0.h:5601
stack_chunk::data
VALUE data[STACK_CHUNK_SIZE]
Definition: gc.c:643
rb_mKernel
RUBY_EXTERN VALUE rb_mKernel
Definition: ruby.h:1998
ROBJ_TRANSIENT_P
#define ROBJ_TRANSIENT_P(obj)
Definition: internal.h:2256
rb_gc_guarded_val
volatile VALUE rb_gc_guarded_val
Definition: gc.c:248
abort
void abort(void) __attribute__((__noreturn__))
argv
char ** argv
Definition: ruby.c:223
f
#define f
gc_stat_sym_major_gc_count
@ gc_stat_sym_major_gc_count
Definition: gc.c:8857
RHASH_ST_TABLE
#define RHASH_ST_TABLE(hash)
Definition: internal.h:856
objspace_and_reason
Definition: gc.c:7369
gc_stat_compat_sym_heap_length
@ gc_stat_compat_sym_heap_length
Definition: gc.c:8884
os_each_struct
Definition: gc.c:3053
ST_CONTINUE
@ ST_CONTINUE
Definition: st.h:99
RBignum
Definition: internal.h:749
rb_heap_struct::total_slots
size_t total_slots
Definition: gc.c:667
rb_setjmp
#define rb_setjmp(env)
Definition: gc.c:81
RVALUE::rational
struct RRational rational
Definition: gc.c:587
RVALUE::v1
VALUE v1
Definition: gc.c:603
st_init_table
st_table * st_init_table(const struct st_hash_type *type)
Definition: st.c:645
rb_data_typed_object_alloc
#define rb_data_typed_object_alloc
Definition: gc.c:15
strdup
char * strdup(const char *) __attribute__((__malloc__)) __attribute__((__warn_unused_result__))
xmalloc
#define xmalloc
Definition: defines.h:211
ruby_thread_has_gvl_p
int ruby_thread_has_gvl_p(void)
Definition: thread.c:1705
rb_objspace::hook_events
rb_event_flag_t hook_events
Definition: gc.c:703
UNREACHABLE
#define UNREACHABLE
Definition: ruby.h:63
VM_METHOD_TYPE_ATTRSET
@ VM_METHOD_TYPE_ATTRSET
attr_writer or attr_accessor
Definition: method.h:104
GET_PAGE_BODY
#define GET_PAGE_BODY(x)
Definition: gc.c:876
rb_sprintf
VALUE rb_sprintf(const char *format,...)
Definition: sprintf.c:1197
ARY_SHARED_P
#define ARY_SHARED_P(ary)
Definition: gc.c:11398
rb_objspace_garbage_object_p
int rb_objspace_garbage_object_p(VALUE obj)
Definition: gc.c:3605
gc_profile_record::flags
int flags
Definition: gc.c:522
rb_mark_hash
void rb_mark_hash(st_table *tbl)
Definition: gc.c:4862
rb_wb_protected_newobj_of
VALUE rb_wb_protected_newobj_of(VALUE klass, VALUE flags)
Definition: gc.c:2279
rb_gc_guarded_ptr_val
volatile VALUE * rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val)
Definition: gc.c:250
klass
VALUE klass
Definition: rb_mjit_min_header-2.7.0.h:13254
rb_hashtype_ident
const struct st_hash_type rb_hashtype_ident
Definition: hash.c:322
rb_id_table_foreach_with_replace
void rb_id_table_foreach_with_replace(struct rb_id_table *tbl, rb_id_table_foreach_func_t *func, rb_id_table_update_callback_func_t *replace, void *data)
Definition: id_table.c:270
RARRAY
#define RARRAY(obj)
Definition: ruby.h:1273
rb_xmalloc_mul_add_mul
void * rb_xmalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w)
Definition: gc.c:10170
st_data_t
unsigned long st_data_t
Definition: rb_mjit_min_header-2.7.0.h:5363
gc_event_hook
#define gc_event_hook(objspace, event, data)
Definition: gc.c:2102
GC_HEAP_FREE_SLOTS_MAX_RATIO
#define GC_HEAP_FREE_SLOTS_MAX_RATIO
Definition: gc.c:281
GC_OLDMALLOC_LIMIT_MIN
#define GC_OLDMALLOC_LIMIT_MIN
Definition: gc.c:295
T_NIL
#define T_NIL
Definition: ruby.h:522
GPR_FLAG_MAJOR_BY_FORCE
@ GPR_FLAG_MAJOR_BY_FORCE
Definition: gc.c:497
rb_subclass_entry::klass
VALUE klass
Definition: internal.h:999
heap_page_body
Definition: gc.c:629
rb_objspace::old_objects_limit
size_t old_objects_limit
Definition: gc.c:796
ruby_gc_params_t::growth_factor
double growth_factor
Definition: gc.c:320
BDIGIT
#define BDIGIT
Definition: bigdecimal.h:48
timeval
Definition: missing.h:53
rb_objspace::total_allocated_pages
size_t total_allocated_pages
Definition: gc.c:781
global_list
#define global_list
Definition: gc.c:925
lo
#define lo
Definition: siphash.c:21
gc_stat_sym_heap_allocatable_pages
@ gc_stat_sym_heap_allocatable_pages
Definition: gc.c:8841
RVALUE::next
struct RVALUE * next
Definition: gc.c:570
T_MOVED
#define T_MOVED
Definition: ruby.h:547
str
char str[HTML_ESCAPE_MAX_LEN+1]
Definition: escape.c:18
GET_THREAD
#define GET_THREAD()
Definition: vm_core.h:1765
GPR_FLAG_MAJOR_MASK
@ GPR_FLAG_MAJOR_MASK
Definition: gc.c:501
stack_chunk
Definition: gc.c:642
mark_stack::limit
int limit
Definition: gc.c:651
ruby_disable_gc
int ruby_disable_gc
Definition: gc.c:1001
NO_SANITIZE
NO_SANITIZE("memory", static void gc_mark_maybe(rb_objspace_t *objspace, VALUE ptr))
gc_stat_sym_compact_count
@ gc_stat_sym_compact_count
Definition: gc.c:8858
RUBY_TYPED_FREE_IMMEDIATELY
#define RUBY_TYPED_FREE_IMMEDIATELY
Definition: ruby.h:1207
ruby_native_thread_p
int ruby_native_thread_p(void)
Definition: thread.c:5277
heap_pages_himem
#define heap_pages_himem
Definition: gc.c:914
debug_counter.h
T_ZOMBIE
#define T_ZOMBIE
Definition: ruby.h:546
MEMCPY
#define MEMCPY(p1, p2, type, n)
Definition: ruby.h:1753
MARK_CHECKPOINT
#define MARK_CHECKPOINT(category)
PRIuVALUE
#define PRIuVALUE
Definition: ruby.h:163
RZombie::next
VALUE next
Definition: gc.c:988
types
enum imemo_type types
Definition: debug.c:72
GPR_FLAG_MAJOR_BY_OLDMALLOC
@ GPR_FLAG_MAJOR_BY_OLDMALLOC
Definition: gc.c:499
stress_to_class
#define stress_to_class
Definition: gc.c:931
SIZEOF_VOIDP
#define SIZEOF_VOIDP
Definition: rb_mjit_min_header-2.7.0.h:90
Init_heap
void Init_heap(void)
Definition: gc.c:2909
posix_memalign
int posix_memalign(void **, size_t, size_t) __attribute__((__nonnull__(1))) __attribute__((__warn_unused_result__))
rb_hash_aset
VALUE rb_hash_aset(VALUE hash, VALUE key, VALUE val)
Definition: hash.c:2779
ruby_gc_params_t::malloc_limit_max
size_t malloc_limit_max
Definition: gc.c:329
RComplex
Definition: internal.h:805
ATOMIC_EXCHANGE
#define ATOMIC_EXCHANGE(var, val)
Definition: ruby_atomic.h:135
RRegexp::src
const VALUE src
Definition: ruby.h:1115
gc_prof_enabled
#define gc_prof_enabled(objspace)
Definition: gc.c:1086
RVALUE::array
struct RArray array
Definition: gc.c:578
clock_gettime
int clock_gettime(clockid_t, struct timespec *)
Definition: win32.c:4612
rb_obj_info_dump
void rb_obj_info_dump(VALUE obj)
Definition: gc.c:11661
heap_tomb
#define heap_tomb
Definition: gc.c:920
NIL_P
#define NIL_P(v)
Definition: ruby.h:482
strtol
long strtol(const char *__restrict __n, char **__restrict __end_PTR, int __base)
RDATA
#define RDATA(obj)
Definition: ruby.h:1274
RVALUE_PIN_BITMAP
#define RVALUE_PIN_BITMAP(obj)
Definition: gc.c:1210
PRIdSIZE
#define PRIdSIZE
Definition: ruby.h:205
TAG_NONE
#define TAG_NONE
Definition: vm_core.h:197
RUBY_INTERNAL_EVENT_NEWOBJ
#define RUBY_INTERNAL_EVENT_NEWOBJ
Definition: ruby.h:2268
rb_objspace::uncollectible_wb_unprotected_objects_limit
size_t uncollectible_wb_unprotected_objects_limit
Definition: gc.c:794
RGENGC_CHECK_MODE
#define RGENGC_CHECK_MODE
Definition: gc.c:399
gc_stat_compat_sym_heap_eden_page_length
@ gc_stat_compat_sym_heap_eden_page_length
Definition: gc.c:8881
BIGNUM_EMBED_FLAG
#define BIGNUM_EMBED_FLAG
Definition: internal.h:769
gc_stress_full_mark_after_malloc
@ gc_stress_full_mark_after_malloc
Definition: gc.c:7122
io.h
Init_gc_stress
void Init_gc_stress(void)
Definition: gc.c:2929
rb_objspace_set_event_hook
void rb_objspace_set_event_hook(const rb_event_flag_t event)
Definition: gc.c:2080
BITMAP_BIT
#define BITMAP_BIT(p)
Definition: gc.c:883
argc
int argc
Definition: ruby.c:222
malloc_increase
#define malloc_increase
Definition: gc.c:908
VM_ENV_FLAG_WB_REQUIRED
@ VM_ENV_FLAG_WB_REQUIRED
Definition: vm_core.h:1188
VM_METHOD_TYPE_ISEQ
@ VM_METHOD_TYPE_ISEQ
Ruby method.
Definition: method.h:102
rb_objspace::total_allocated_objects_at_gc_start
size_t total_allocated_objects_at_gc_start
Definition: gc.c:775
rb_objspace::freeable_pages
size_t freeable_pages
Definition: gc.c:728
GC_OLDMALLOC_LIMIT_MAX
#define GC_OLDMALLOC_LIMIT_MAX
Definition: gc.c:301
regint.h
T_IMEMO
#define T_IMEMO
Definition: ruby.h:543
GC_MALLOC_LIMIT_MAX
#define GC_MALLOC_LIMIT_MAX
Definition: gc.c:288
rb_obj_classname
const char * rb_obj_classname(VALUE)
Definition: variable.c:289
memop_type
memop_type
Definition: gc.c:9667
rb_iseq_memsize
size_t rb_iseq_memsize(const rb_iseq_t *iseq)
Definition: iseq.c:373
rb_vm_struct::self
VALUE self
Definition: vm_core.h:577
mark_stack::index
int index
Definition: gc.c:650
roomof
#define roomof(x, y)
Definition: internal.h:1298
rb_obj_memsize_of
size_t rb_obj_memsize_of(VALUE obj)
Definition: gc.c:3934
ruby_gc_params_t::growth_max_slots
size_t growth_max_slots
Definition: gc.c:321
list_node
Definition: rb_mjit_min_header-2.7.0.h:8971
GC_DEBUG
#define GC_DEBUG
Definition: gc.c:365
rb_define_const
void rb_define_const(VALUE, const char *, VALUE)
Definition: variable.c:2880
free
#define free(x)
Definition: dln.c:52
err
int err
Definition: win32.c:135
will_be_incremental_marking
#define will_be_incremental_marking(objspace)
Definition: gc.c:966
GPR_FLAG_STRESS
@ GPR_FLAG_STRESS
Definition: gc.c:508
rb_objspace::flags
struct rb_objspace::@83 flags
is_sweeping
#define is_sweeping(objspace)
Definition: gc.c:954
gc_stat_sym_malloc_increase_bytes_limit
@ gc_stat_sym_malloc_increase_bytes_limit
Definition: gc.c:8854
STACK_LENGTH
#define STACK_LENGTH
Definition: gc.c:4614
rb_io_write
VALUE rb_io_write(VALUE, VALUE)
Definition: io.c:1804
verify_internal_consistency_struct::live_object_count
size_t live_object_count
Definition: gc.c:5894
gc_raise_tag::ap
va_list * ap
Definition: gc.c:9506
__asm__
#define __asm__
Definition: Context.c:12
rb_data_type_struct
Definition: ruby.h:1148
BUILTIN_TYPE
#define BUILTIN_TYPE(x)
Definition: ruby.h:551
gc_mode_marking
@ gc_mode_marking
Definition: gc.c:672
rb_vm_struct
Definition: vm_core.h:576
xfree
#define xfree
Definition: defines.h:216
heap_page::pinned_slots
short pinned_slots
Definition: gc.c:848
gc_stat_sym_oldmalloc_increase_bytes
@ gc_stat_sym_oldmalloc_increase_bytes
Definition: gc.c:8864
RVALUE::regexp
struct RRegexp regexp
Definition: gc.c:579
memalign
void * memalign(size_t, size_t)
RZombie::data
void * data
Definition: gc.c:990
RUBY_INTERNAL_EVENT_GC_END_MARK
#define RUBY_INTERNAL_EVENT_GC_END_MARK
Definition: ruby.h:2271
RMOVED
#define RMOVED(obj)
Definition: ruby.h:1266
ARY_EMBED_P
#define ARY_EMBED_P(ary)
Definition: gc.c:11401
MEMOP_TYPE_REALLOC
@ MEMOP_TYPE_REALLOC
Definition: gc.c:9670
SET_MACHINE_STACK_END
#define SET_MACHINE_STACK_END(p)
Definition: gc.h:13
RBASIC
#define RBASIC(obj)
Definition: ruby.h:1267
PUREFUNC
PUREFUNC(static inline int is_id_value(rb_objspace_t *objspace, VALUE ptr))
CLEAR_IN_BITMAP
#define CLEAR_IN_BITMAP(bits, p)
Definition: gc.c:888
root_objects_data
Definition: gc.c:9462
ruby_gc_params_t::heap_free_slots_max_ratio
double heap_free_slots_max_ratio
Definition: gc.c:325
rb_obj_info_dump_loc
void rb_obj_info_dump_loc(VALUE obj, const char *file, int line, const char *func)
Definition: gc.c:11668
rb_sym2id
ID rb_sym2id(VALUE)
Definition: symbol.c:748
root_objects_data::func
void(* func)(const char *category, VALUE, void *)
Definition: gc.c:9464
rb_method_definition_struct
Definition: method.h:163
HEAP_PAGE_SIZE
@ HEAP_PAGE_SIZE
Definition: gc.c:838
rb_gc_mark
void rb_gc_mark(VALUE ptr)
Definition: gc.c:5212
gc_stat_compat_sym_total_allocated_object
@ gc_stat_compat_sym_total_allocated_object
Definition: gc.c:8895
GC_HEAP_GROWTH_MAX_SLOTS
#define GC_HEAP_GROWTH_MAX_SLOTS
Definition: gc.c:268
imemo_cref
@ imemo_cref
class reference
Definition: internal.h:1134
rb_objspace_each_objects_without_setup
void rb_objspace_each_objects_without_setup(each_obj_callback *callback, void *data)
Definition: gc.c:3048
sleep
unsigned sleep(unsigned int __seconds)
rb_gc_mark_values
void rb_gc_mark_values(long n, const VALUE *values)
Definition: gc.c:4715
MJIT_FUNC_EXPORTED
#define MJIT_FUNC_EXPORTED
Definition: defines.h:396
rb_undefine_finalizer
VALUE rb_undefine_finalizer(VALUE obj)
Definition: gc.c:3191
rb_size_mul_add_or_raise
size_t rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
Definition: gc.c:219
_
#define _(args)
Definition: dln.h:28
EC_PUSH_TAG
#define EC_PUSH_TAG(ec)
Definition: eval_intern.h:130
rb_objspace::limit
size_t limit
Definition: gc.c:678
ar_table_struct
Definition: hash.c:349
count
int count
Definition: encoding.c:57
st_memsize
size_t st_memsize(const st_table *tab)
Definition: st.c:719
rb_callable_method_entry_struct::def
struct rb_method_definition_struct *const def
Definition: method.h:62
Qtrue
#define Qtrue
Definition: ruby.h:468
heap_page_header
Definition: gc.c:625
rb_obj_rgengc_writebarrier_protected_p
VALUE rb_obj_rgengc_writebarrier_protected_p(VALUE obj)
Definition: gc.c:6960
S
#define S(s)
rb_objspace::global_list
struct gc_list * global_list
Definition: gc.c:784
NUM_IN_PAGE
#define NUM_IN_PAGE(p)
Definition: gc.c:880
rb_class_name
VALUE rb_class_name(VALUE)
Definition: variable.c:274
rb_io_fptr_finalize
#define rb_io_fptr_finalize
Definition: internal.h:1728
re_registers::num_regs
int num_regs
Definition: onigmo.h:718
gc_stat_sym_heap_allocated_pages
@ gc_stat_sym_heap_allocated_pages
Definition: gc.c:8839
RArray::as
union RArray::@97 as
FL_PROMOTED1
#define FL_PROMOTED1
Definition: ruby.h:1281
RANY
#define RANY(o)
Definition: gc.c:984
len
uint8_t len
Definition: escape.c:17
gc_prof_record
#define gc_prof_record(objspace)
Definition: gc.c:1085
SYMBOL_P
#define SYMBOL_P(x)
Definition: ruby.h:413
RB_DEBUG_COUNTER_INC
#define RB_DEBUG_COUNTER_INC(type)
Definition: debug_counter.h:375
rb_heap_struct::using_page
struct heap_page * using_page
Definition: gc.c:660
RVALUE::svar
struct vm_svar svar
Definition: gc.c:591
rb_const_entry_struct::file
VALUE file
Definition: constant.h:35
rb_memerror
void rb_memerror(void)
Definition: gc.c:9578
GC_PROFILE_MORE_DETAIL
#define GC_PROFILE_MORE_DETAIL
Definition: gc.c:458
RVALUE::typeddata
struct RTypedData typeddata
Definition: gc.c:582
rb_method_entry_struct
Definition: method.h:51
rb_iseq_struct::body
struct rb_iseq_constant_body * body
Definition: vm_core.h:460
rb_mark_tbl
void rb_mark_tbl(st_table *tbl)
Definition: gc.c:5005
rb_subclass_entry
Definition: internal.h:998
RStruct
Definition: internal.h:942
rb_control_frame_struct::pc
const VALUE * pc
Definition: vm_core.h:761
MEMMOVE
#define MEMMOVE(p1, p2, type, n)
Definition: ruby.h:1754
rb_gc_unprotect_logging
void rb_gc_unprotect_logging(void *objptr, const char *filename, int line)
Definition: gc.c:6909
rb_data_type_struct::function
struct rb_data_type_struct::@100 function
gc_stat_compat_sym_heap_tomb_page_length
@ gc_stat_compat_sym_heap_tomb_page_length
Definition: gc.c:8882
timespec
Definition: missing.h:60
gc_stat_sym_old_objects_limit
@ gc_stat_sym_old_objects_limit
Definition: gc.c:8862
ruby_atomic.h
rb_io_t::tied_io_for_writing
VALUE tied_io_for_writing
Definition: io.h:77
heap_page::freelist
RVALUE * freelist
Definition: gc.c:859
va_start
#define va_start(v, l)
Definition: rb_mjit_min_header-2.7.0.h:3978
RTYPEDDATA_P
#define RTYPEDDATA_P(v)
Definition: ruby.h:1177
rb_define_finalizer
VALUE rb_define_finalizer(VALUE obj, VALUE block)
Definition: gc.c:3287
RVALUE::basic
struct RBasic basic
Definition: gc.c:573
rb_str_free
void rb_str_free(VALUE)
Definition: string.c:1349
rb_objspace::rgengc
struct rb_objspace::@87 rgengc
rb_alloc_tmp_buffer_with_count
void * rb_alloc_tmp_buffer_with_count(volatile VALUE *store, size_t size, size_t cnt)
Definition: gc.c:10227
rb_objspace::mark_func_data_struct
Definition: gc.c:714
OBJ_PROMOTED
#define OBJ_PROMOTED(x)
Definition: ruby.h:1494
ROBJECT_NUMIV
#define ROBJECT_NUMIV(o)
Definition: ruby.h:933
rb_class_detach_subclasses
void rb_class_detach_subclasses(VALUE klass)
Definition: class.c:133
T_STRING
#define T_STRING
Definition: ruby.h:528
gc_raise_tag
Definition: gc.c:9503
imemo_tmpbuf
@ imemo_tmpbuf
Definition: internal.h:1141
finalizing
#define finalizing
Definition: gc.c:923
rb_hash_stlike_foreach_with_replace
int rb_hash_stlike_foreach_with_replace(VALUE hash, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg)
Definition: hash.c:1431
rb_atomic_t
int rb_atomic_t
Definition: ruby_atomic.h:124
RZombie::dfree
void(* dfree)(void *)
Definition: gc.c:989
rb_block
Definition: vm_core.h:751
FL_SINGLETON
#define FL_SINGLETON
Definition: ruby.h:1278
heap_page_header::page
struct heap_page * page
Definition: gc.c:626
RMatch::str
VALUE str
Definition: re.h:45
rb_define_class_under
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
Definition: class.c:698
rb_sym2str
VALUE rb_sym2str(VALUE)
Definition: symbol.c:784
SIZET2NUM
#define SIZET2NUM(v)
Definition: ruby.h:295
va_list
__gnuc_va_list va_list
Definition: rb_mjit_min_header-2.7.0.h:836
rb_objspace::gc_stress_mode
VALUE gc_stress_mode
Definition: gc.c:786
rb_id_table_foreach_values
void rb_id_table_foreach_values(struct rb_id_table *tbl, rb_id_table_foreach_values_func_t *func, void *data)
Definition: id_table.c:311
iseq
const rb_iseq_t * iseq
Definition: rb_mjit_min_header-2.7.0.h:13504
rb_generic_ivar_memsize
size_t rb_generic_ivar_memsize(VALUE)
Definition: variable.c:1010
rb_yield
VALUE rb_yield(VALUE)
Definition: vm_eval.c:1237
RVALUE::klass
struct RClass klass
Definition: gc.c:575
rb_hash_stlike_foreach
int rb_hash_stlike_foreach(VALUE hash, st_foreach_callback_func *func, st_data_t arg)
Definition: hash.c:1420
rb_gc_disable_no_rest
VALUE rb_gc_disable_no_rest(void)
Definition: gc.c:9225
eval_intern.h
gc_list::varptr
VALUE * varptr
Definition: gc.c:636
ruby_initial_gc_stress
#define ruby_initial_gc_stress
Definition: gc.c:903
rb_objspace::mark_func_data
struct rb_objspace::mark_func_data_struct * mark_func_data
rb_ensure
VALUE rb_ensure(VALUE(*b_proc)(VALUE), VALUE data1, VALUE(*e_proc)(VALUE), VALUE data2)
An equivalent to ensure clause.
Definition: eval.c:1114
rb_io_t::encs
struct rb_io_t::rb_io_enc_t encs
RComplex::imag
VALUE imag
Definition: internal.h:808
RARRAY_CONST_PTR_TRANSIENT
#define RARRAY_CONST_PTR_TRANSIENT(a)
Definition: ruby.h:1073
ruby_vm_special_exception_copy
MJIT_STATIC VALUE ruby_vm_special_exception_copy(VALUE)
Definition: rb_mjit_min_header-2.7.0.h:12219
rb_ary_new
VALUE rb_ary_new(void)
Definition: array.c:723
fputs
int fputs(const char *__restrict, FILE *__restrict)
gc_stat_sym_last
@ gc_stat_sym_last
Definition: gc.c:8876
ruby_xrealloc2_body
void * ruby_xrealloc2_body(void *ptr, size_t n, size_t size)
Definition: gc.c:10133
rb_symbols_t
Definition: symbol.h:61
rb_vm_update_references
void rb_vm_update_references(void *ptr)
Definition: vm.c:2234
builtin.h
heap_page::has_uncollectible_shady_objects
unsigned int has_uncollectible_shady_objects
Definition: gc.c:853
ROBJECT_EMBED
@ ROBJECT_EMBED
Definition: ruby.h:917
rb_imemo_tmpbuf_struct
Definition: internal.h:1231
Qnil
#define Qnil
Definition: ruby.h:469
T_STRUCT
#define T_STRUCT
Definition: ruby.h:532
mark_stack_t
struct mark_stack mark_stack_t
GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
Definition: gc.c:298
gc_profile_record::gc_invoke_time
double gc_invoke_time
Definition: gc.c:525
CLOCK_PROCESS_CPUTIME_ID
#define CLOCK_PROCESS_CPUTIME_ID
Definition: rb_mjit_min_header-2.7.0.h:2372
rb_big_size
size_t rb_big_size(VALUE big)
Definition: bignum.c:6778
exc
const rb_iseq_t const VALUE exc
Definition: rb_mjit_min_header-2.7.0.h:13504
rb_objspace::final_slots
size_t final_slots
Definition: gc.c:731
heap_cursor::slot
RVALUE * slot
Definition: gc.c:7666
heap_page::marking_bits
bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT]
Definition: gc.c:869
gc_stat_sym_total_freed_pages
@ gc_stat_sym_total_freed_pages
Definition: gc.c:8850
RVALUE_UNCOLLECTIBLE_BITMAP
#define RVALUE_UNCOLLECTIBLE_BITMAP(obj)
Definition: gc.c:1215
thread.h
rmatch_offset
Definition: re.h:31
unsigned
#define unsigned
Definition: rb_mjit_min_header-2.7.0.h:2875
rb_mEnumerable
VALUE rb_mEnumerable
Definition: enum.c:20
ruby_qsort
void ruby_qsort(void *, const size_t, const size_t, int(*)(const void *, const void *, void *), void *)
h
size_t st_index_t h
Definition: rb_mjit_min_header-2.7.0.h:5462
st_lookup
int st_lookup(st_table *tab, st_data_t key, st_data_t *value)
Definition: st.c:1101
util.h
O
#define O(member)
strtoll
long long strtoll(const char *__restrict __n, char **__restrict __end_PTR, int __base)
BIGNUM_LEN
#define BIGNUM_LEN(b)
Definition: internal.h:774
stack_chunk_t
struct stack_chunk stack_chunk_t
rb_io_t
Definition: io.h:66
page_compare_func_t
int page_compare_func_t(const void *, const void *, void *)
Definition: gc.c:7766
GC_HEAP_FREE_SLOTS
#define GC_HEAP_FREE_SLOTS
Definition: gc.c:262
gc_mode_sweeping
@ gc_mode_sweeping
Definition: gc.c:673
rb_classext_struct
Definition: internal.h:1020
rb_int_plus
VALUE rb_int_plus(VALUE x, VALUE y)
Definition: numeric.c:3610
RUBY_INTERNAL_EVENT_GC_START
#define RUBY_INTERNAL_EVENT_GC_START
Definition: ruby.h:2270
rb_gc_latest_gc_info
VALUE rb_gc_latest_gc_info(VALUE key)
Definition: gc.c:8816
gc_profile_record
Definition: gc.c:521
numberof
#define numberof(array)
Definition: etc.c:618
rb_thread_struct
Definition: vm_core.h:910
rb_ary_delete_same
void rb_ary_delete_same(VALUE ary, VALUE item)
Definition: array.c:3396
UNREACHABLE_RETURN
#define UNREACHABLE_RETURN(val)
Definition: ruby.h:59
rb_objspace::finalizing
rb_atomic_t finalizing
Definition: gc.c:711
rb_classext_struct::origin_
const VALUE origin_
Definition: internal.h:1039
rb_objspace::uncollectible_wb_unprotected_objects
size_t uncollectible_wb_unprotected_objects
Definition: gc.c:793
st_free_table
void st_free_table(st_table *tab)
Definition: st.c:709
rb_eNoMemError
VALUE rb_eNoMemError
Definition: error.c:933
wmap_iter_arg::objspace
rb_objspace_t * objspace
Definition: gc.c:10482
list_for_each_safe
#define list_for_each_safe(h, i, nxt, member)
Definition: rb_mjit_min_header-2.7.0.h:9097
st_table
Definition: st.h:79
rb_transient_heap_verify
void rb_transient_heap_verify(void)
Definition: transient_heap.c:219
RHASH_AR_TABLE_P
#define RHASH_AR_TABLE_P(hash)
Definition: internal.h:854
block_type_iseq
@ block_type_iseq
Definition: vm_core.h:745
rb_ec_raised_p
#define rb_ec_raised_p(ec, f)
Definition: eval_intern.h:260
rb_objspace::old_objects
size_t old_objects
Definition: gc.c:795
hi
#define hi
Definition: siphash.c:22
RVALUE::file
struct RFile file
Definition: gc.c:585
RData
Definition: ruby.h:1139
ruby_assert.h
rb_objspace::sorted_length
size_t sorted_length
Definition: gc.c:726
gc_profile_record::gc_time
double gc_time
Definition: gc.c:524
malloc_allocated_size
#define malloc_allocated_size
Definition: gc.c:909
rb_any_to_s
VALUE rb_any_to_s(VALUE)
Default implementation of #to_s.
Definition: object.c:527
RUBY_INTERNAL_EVENT_GC_END_SWEEP
#define RUBY_INTERNAL_EVENT_GC_END_SWEEP
Definition: ruby.h:2272
sub
#define sub(x, y)
Definition: date_strftime.c:24
ID_TABLE_CONTINUE
@ ID_TABLE_CONTINUE
Definition: id_table.h:9
rb_heap_struct::free_pages
struct heap_page * free_pages
Definition: gc.c:659
GC_HEAP_OLDOBJECT_LIMIT_FACTOR
#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR
Definition: gc.c:271
rb_data_object_alloc
#define rb_data_object_alloc
Definition: gc.c:14
rb_const_entry_struct
Definition: constant.h:31
each_obj_callback
int each_obj_callback(void *, void *, size_t, void *)
Definition: gc.c:2936
id_table.h
finalizer_table
#define finalizer_table
Definition: gc.c:924
RREGEXP_PTR
#define RREGEXP_PTR(r)
Definition: ruby.h:1118
rb_newobj
VALUE rb_newobj(void)
Definition: gc.c:2288
rb_objspace::invoke_time
double invoke_time
Definition: gc.c:748
gc_stat_compat_sym_gc_stat_heap_used
@ gc_stat_compat_sym_gc_stat_heap_used
Definition: gc.c:8880
T_TRUE
#define T_TRUE
Definition: ruby.h:536
rb_obj_is_kind_of
VALUE rb_obj_is_kind_of(VALUE, VALUE)
Determines if obj is a kind of c.
Definition: object.c:692
ruby_gc_set_params
void ruby_gc_set_params(void)
Definition: gc.c:9401
malloc_obj_info
Definition: gc.c:9771
ruby_tag_type
ruby_tag_type
Definition: vm_core.h:184
rb_define_alloc_func
void rb_define_alloc_func(VALUE, rb_alloc_func_t)
VALGRIND_MAKE_MEM_UNDEFINED
#define VALGRIND_MAKE_MEM_UNDEFINED(p, n)
Definition: zlib.c:25
RUBY_ALIAS_FUNCTION
RUBY_ALIAS_FUNCTION(rb_data_object_alloc(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree), rb_data_object_wrap,(klass, datap, dmark, dfree))
Definition: gc.c:2383
RTEST
#define RTEST(v)
Definition: ruby.h:481
imemo_ifunc
@ imemo_ifunc
iterator function
Definition: internal.h:1137
rb_wb_unprotected_newobj_of
VALUE rb_wb_unprotected_newobj_of(VALUE klass, VALUE flags)
Definition: gc.c:2272
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition: cxxanyargs.hpp:39
debug.h
RB_SPECIAL_CONST_P
#define RB_SPECIAL_CONST_P(x)
Definition: ruby.h:1312
RB_GNUC_EXTENSION
#define RB_GNUC_EXTENSION
Definition: defines.h:121
VM_METHOD_TYPE_ALIAS
@ VM_METHOD_TYPE_ALIAS
Definition: method.h:108
heap_page::uncollectible_bits
bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT]
Definition: gc.c:868
SYM2ID
#define SYM2ID(x)
Definition: ruby.h:415
rb_vm_env_prev_env
const rb_env_t * rb_vm_env_prev_env(const rb_env_t *env)
Definition: vm.c:796
rb_special_const_p
#define rb_special_const_p(obj)
Definition: rb_mjit_min_header-2.7.0.h:5357
T_UNDEF
#define T_UNDEF
Definition: ruby.h:544
rmatch
Definition: re.h:36
list_top
#define list_top(h, type, member)
Definition: rb_mjit_min_header-2.7.0.h:9070
optional
Definition: gc.c:90
ATOMIC_SIZE_INC
#define ATOMIC_SIZE_INC(var)
Definition: ruby_atomic.h:151
__sFILE
Definition: vsnprintf.c:169
RSTRUCT_CONST_PTR
#define RSTRUCT_CONST_PTR(st)
Definition: internal.h:962
heap_page::pinned_bits
bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT]
Definition: gc.c:873
rb_ec_stack_check
MJIT_FUNC_EXPORTED int rb_ec_stack_check(rb_execution_context_t *ec)
Definition: gc.c:4665
rb_thread_call_with_gvl
RUBY_SYMBOL_EXPORT_BEGIN void * rb_thread_call_with_gvl(void *(*func)(void *), void *data1)
Definition: thread.c:1662
rb_heap_struct::pooled_pages
struct heap_page * pooled_pages
Definition: gc.c:664
HEAP_PAGE_BITMAP_LIMIT
@ HEAP_PAGE_BITMAP_LIMIT
Definition: gc.c:840
RSTRUCT
#define RSTRUCT(obj)
Definition: internal.h:966
rb_objspace::compact_count
size_t compact_count
Definition: gc.c:753
rb_objspace::during_gc
unsigned int during_gc
Definition: gc.c:691
RGENGC_FORCE_MAJOR_GC
#define RGENGC_FORCE_MAJOR_GC
Definition: gc.c:438
RClass
Definition: internal.h:1048
cfp
rb_control_frame_t * cfp
Definition: rb_mjit_min_header-2.7.0.h:14544
rb_transient_heap_start_marking
void rb_transient_heap_start_marking(int full_marking)
Definition: transient_heap.c:868
RVALUE::data
struct RData data
Definition: gc.c:581
ruby_sized_xfree
void ruby_sized_xfree(void *x, size_t size)
Definition: gc.c:10142
rb_objspace::parent_object
VALUE parent_object
Definition: gc.c:790
name
const char * name
Definition: nkf.c:208
NUM2PTR
#define NUM2PTR(x)
RVALUE_MARK_BITMAP
#define RVALUE_MARK_BITMAP(obj)
Definition: gc.c:1209
rb_execution_context_struct
Definition: vm_core.h:843
ruby_gc_params_t::heap_free_slots_min_ratio
double heap_free_slots_min_ratio
Definition: gc.c:323
rb_block_proc
VALUE rb_block_proc(void)
Definition: proc.c:837