diff options
Diffstat (limited to 'src/alloc.c')
| -rw-r--r-- | src/alloc.c | 945 |
1 files changed, 658 insertions, 287 deletions
diff --git a/src/alloc.c b/src/alloc.c index b02f13e911f..2ffd2415447 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -34,7 +34,6 @@ along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>. */ #include "dispextern.h" #include "intervals.h" #include "puresize.h" -#include "sheap.h" #include "sysstdio.h" #include "systime.h" #include "character.h" @@ -50,6 +49,10 @@ along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>. */ #include TERM_HEADER #endif /* HAVE_WINDOW_SYSTEM */ +#if defined HAVE_ANDROID && !defined ANDROID_STUBIFY +#include "sfntfont.h" +#endif + #ifdef HAVE_TREE_SITTER #include "treesit.h" #endif @@ -80,6 +83,37 @@ along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>. */ #include <valgrind/memcheck.h> #endif +/* AddressSanitizer exposes additional functions for manually marking + memory as poisoned/unpoisoned. When ASan is enabled and the needed + header is available, memory is poisoned when: + + * An ablock is freed (lisp_align_free), or ablocks are initially + allocated (lisp_align_malloc). + * An interval_block is initially allocated (make_interval). + * A dead INTERVAL is put on the interval free list + (sweep_intervals). + * A sdata is marked as dead (sweep_strings, pin_string). + * An sblock is initially allocated (allocate_string_data). + * A string_block is initially allocated (allocate_string). + * A dead string is put on string_free_list (sweep_strings). + * A float_block is initially allocated (make_float). + * A dead float is put on float_free_list. + * A cons_block is initially allocated (Fcons). + * A dead cons is put on cons_free_list (sweep_cons). + * A dead vector is put on vector_free_list (setup_on_free_list), + or a new vector block is allocated (allocate_vector_from_block). + Accordingly, objects reused from the free list are unpoisoned. + + This feature can be disabled with the run-time flag + `allow_user_poisoning' set to zero. */ +#if ADDRESS_SANITIZER && defined HAVE_SANITIZER_ASAN_INTERFACE_H \ + && !defined GC_ASAN_POISON_OBJECTS +# define GC_ASAN_POISON_OBJECTS 1 +# include <sanitizer/asan_interface.h> +#else +# define GC_ASAN_POISON_OBJECTS 0 +#endif + /* GC_CHECK_MARKED_OBJECTS means do sanity checks on allocated objects. We turn that on by default when ENABLE_CHECKING is defined; define GC_CHECK_MARKED_OBJECTS to zero to disable. */ @@ -325,8 +359,16 @@ static struct gcstat object_ct total_floats, total_free_floats; object_ct total_intervals, total_free_intervals; object_ct total_buffers; + + /* Size of the ancillary arrays of live hash-table and obarray objects. + The objects themselves are not included (counted as vectors above). */ + byte_ct total_hash_table_bytes; } gcstat; +/* Total size of ancillary arrays of all allocated hash-table and obarray + objects, both dead and alive. This number is always kept up-to-date. */ +static ptrdiff_t hash_table_allocated_bytes = 0; + /* Points to memory space allocated as "spare", to be freed if we run out of memory. We keep one large block, four cons-blocks, and two string blocks. */ @@ -378,31 +420,6 @@ static EMACS_INT gc_threshold; const char *pending_malloc_warning; -/* Pointer sanity only on request. FIXME: Code depending on - SUSPICIOUS_OBJECT_CHECKING is obsolete; remove it entirely. */ -#ifdef ENABLE_CHECKING -#define SUSPICIOUS_OBJECT_CHECKING 1 -#endif - -#ifdef SUSPICIOUS_OBJECT_CHECKING -struct suspicious_free_record -{ - void *suspicious_object; - void *backtrace[128]; -}; -static void *suspicious_objects[32]; -static int suspicious_object_index; -struct suspicious_free_record suspicious_free_history[64] EXTERNALLY_VISIBLE; -static int suspicious_free_history_index; -/* Find the first currently-monitored suspicious pointer in range - [begin,end) or NULL if no such pointer exists. */ -static void *find_suspicious_object_in_range (void *begin, void *end); -static void detect_suspicious_free (void *ptr); -#else -# define find_suspicious_object_in_range(begin, end) ((void *) NULL) -# define detect_suspicious_free(ptr) ((void) 0) -#endif - /* Maximum amount of C stack to save when a GC happens. */ #ifndef MAX_SAVE_STACK @@ -832,7 +849,7 @@ xnmalloc (ptrdiff_t nitems, ptrdiff_t item_size) { eassert (0 <= nitems && 0 < item_size); ptrdiff_t nbytes; - if (INT_MULTIPLY_WRAPV (nitems, item_size, &nbytes) || SIZE_MAX < nbytes) + if (ckd_mul (&nbytes, nitems, item_size) || SIZE_MAX < nbytes) memory_full (SIZE_MAX); return xmalloc (nbytes); } @@ -846,7 +863,7 @@ xnrealloc (void *pa, ptrdiff_t nitems, ptrdiff_t item_size) { eassert (0 <= nitems && 0 < item_size); ptrdiff_t nbytes; - if (INT_MULTIPLY_WRAPV (nitems, item_size, &nbytes) || SIZE_MAX < nbytes) + if (ckd_mul (&nbytes, nitems, item_size) || SIZE_MAX < nbytes) memory_full (SIZE_MAX); return xrealloc (pa, nbytes); } @@ -893,13 +910,13 @@ xpalloc (void *pa, ptrdiff_t *nitems, ptrdiff_t nitems_incr_min, NITEMS_MAX, and what the C language can represent safely. */ ptrdiff_t n, nbytes; - if (INT_ADD_WRAPV (n0, n0 >> 1, &n)) + if (ckd_add (&n, n0, n0 >> 1)) n = PTRDIFF_MAX; if (0 <= nitems_max && nitems_max < n) n = nitems_max; ptrdiff_t adjusted_nbytes - = ((INT_MULTIPLY_WRAPV (n, item_size, &nbytes) || SIZE_MAX < nbytes) + = ((ckd_mul (&nbytes, n, item_size) || SIZE_MAX < nbytes) ? min (PTRDIFF_MAX, SIZE_MAX) : nbytes < DEFAULT_MXFAST ? DEFAULT_MXFAST : 0); if (adjusted_nbytes) @@ -911,9 +928,9 @@ xpalloc (void *pa, ptrdiff_t *nitems, ptrdiff_t nitems_incr_min, if (! pa) *nitems = 0; if (n - n0 < nitems_incr_min - && (INT_ADD_WRAPV (n0, nitems_incr_min, &n) + && (ckd_add (&n, n0, nitems_incr_min) || (0 <= nitems_max && nitems_max < n) - || INT_MULTIPLY_WRAPV (n, item_size, &nbytes))) + || ckd_mul (&nbytes, n, item_size))) memory_full (SIZE_MAX); pa = xrealloc (pa, nbytes); *nitems = n; @@ -1052,7 +1069,11 @@ lisp_free (void *block) BLOCK_BYTES and guarantees they are aligned on a BLOCK_ALIGN boundary. */ /* Byte alignment of storage blocks. */ -#define BLOCK_ALIGN (1 << 10) +#ifdef HAVE_UNEXEC +# define BLOCK_ALIGN (1 << 10) +#else /* !HAVE_UNEXEC */ +# define BLOCK_ALIGN (1 << 15) +#endif verify (POWER_OF_2 (BLOCK_ALIGN)); /* Use aligned_alloc if it or a simple substitute is available. @@ -1157,6 +1178,16 @@ struct ablocks (1 & (intptr_t) ABLOCKS_BUSY (abase) ? abase : ((void **) (abase))[-1]) #endif +#if GC_ASAN_POISON_OBJECTS +# define ASAN_POISON_ABLOCK(b) \ + __asan_poison_memory_region (&(b)->x, sizeof ((b)->x)) +# define ASAN_UNPOISON_ABLOCK(b) \ + __asan_unpoison_memory_region (&(b)->x, sizeof ((b)->x)) +#else +# define ASAN_POISON_ABLOCK(b) ((void) 0) +# define ASAN_UNPOISON_ABLOCK(b) ((void) 0) +#endif + /* The list of free ablock. */ static struct ablock *free_ablock; @@ -1235,6 +1266,7 @@ lisp_align_malloc (size_t nbytes, enum mem_type type) { abase->blocks[i].abase = abase; abase->blocks[i].x.next_free = free_ablock; + ASAN_POISON_ABLOCK (&abase->blocks[i]); free_ablock = &abase->blocks[i]; } intptr_t ialigned = aligned; @@ -1247,6 +1279,7 @@ lisp_align_malloc (size_t nbytes, enum mem_type type) eassert ((intptr_t) ABLOCKS_BUSY (abase) == aligned); } + ASAN_UNPOISON_ABLOCK (free_ablock); abase = ABLOCK_ABASE (free_ablock); ABLOCKS_BUSY (abase) = (struct ablocks *) (2 + (intptr_t) ABLOCKS_BUSY (abase)); @@ -1278,6 +1311,7 @@ lisp_align_free (void *block) #endif /* Put on free list. */ ablock->x.next_free = free_ablock; + ASAN_POISON_ABLOCK (ablock); free_ablock = ablock; /* Update busy count. */ intptr_t busy = (intptr_t) ABLOCKS_BUSY (abase) - 2; @@ -1290,9 +1324,12 @@ lisp_align_free (void *block) bool aligned = busy; struct ablock **tem = &free_ablock; struct ablock *atop = &abase->blocks[aligned ? ABLOCKS_SIZE : ABLOCKS_SIZE - 1]; - while (*tem) { +#if GC_ASAN_POISON_OBJECTS + __asan_unpoison_memory_region (&(*tem)->x, + sizeof ((*tem)->x)); +#endif if (*tem >= (struct ablock *) abase && *tem < atop) { i++; @@ -1421,6 +1458,24 @@ static int interval_block_index = INTERVAL_BLOCK_SIZE; static INTERVAL interval_free_list; +#if GC_ASAN_POISON_OBJECTS +# define ASAN_POISON_INTERVAL_BLOCK(b) \ + __asan_poison_memory_region ((b)->intervals, \ + sizeof ((b)->intervals)) +# define ASAN_UNPOISON_INTERVAL_BLOCK(b) \ + __asan_unpoison_memory_region ((b)->intervals, \ + sizeof ((b)->intervals)) +# define ASAN_POISON_INTERVAL(i) \ + __asan_poison_memory_region (i, sizeof *(i)) +# define ASAN_UNPOISON_INTERVAL(i) \ + __asan_unpoison_memory_region (i, sizeof *(i)) +#else +# define ASAN_POISON_INTERVAL_BLOCK(b) ((void) 0) +# define ASAN_UNPOISON_INTERVAL_BLOCK(b) ((void) 0) +# define ASAN_POISON_INTERVAL(i) ((void) 0) +# define ASAN_UNPOISON_INTERVAL(i) ((void) 0) +#endif + /* Return a new interval. */ INTERVAL @@ -1433,6 +1488,7 @@ make_interval (void) if (interval_free_list) { val = interval_free_list; + ASAN_UNPOISON_INTERVAL (val); interval_free_list = INTERVAL_PARENT (interval_free_list); } else @@ -1443,10 +1499,12 @@ make_interval (void) = lisp_malloc (sizeof *newi, false, MEM_TYPE_NON_LISP); newi->next = interval_block; + ASAN_POISON_INTERVAL_BLOCK (newi); interval_block = newi; interval_block_index = 0; } val = &interval_block->intervals[interval_block_index++]; + ASAN_UNPOISON_INTERVAL (val); } MALLOC_UNBLOCK_INPUT; @@ -1687,6 +1745,41 @@ init_strings (void) staticpro (&empty_multibyte_string); } +#if GC_ASAN_POISON_OBJECTS +/* Prepare s for denoting a free sdata struct, i.e, poison all bytes + in the flexible array member, except the first SDATA_OFFSET bytes. + This is only effective for strings of size n where n > sdata_size(n). + */ +# define ASAN_PREPARE_DEAD_SDATA(s, size) \ + do { \ + __asan_poison_memory_region (s, sdata_size (size)); \ + __asan_unpoison_memory_region (&(s)->string, \ + sizeof (struct Lisp_String *)); \ + __asan_unpoison_memory_region (&SDATA_NBYTES (s), \ + sizeof SDATA_NBYTES (s)); \ + } while (false) +/* Prepare s for storing string data for NBYTES bytes. */ +# define ASAN_PREPARE_LIVE_SDATA(s, nbytes) \ + __asan_unpoison_memory_region (s, sdata_size (nbytes)) +# define ASAN_POISON_SBLOCK_DATA(b, size) \ + __asan_poison_memory_region ((b)->data, size) +# define ASAN_POISON_STRING_BLOCK(b) \ + __asan_poison_memory_region ((b)->strings, STRING_BLOCK_SIZE) +# define ASAN_UNPOISON_STRING_BLOCK(b) \ + __asan_unpoison_memory_region ((b)->strings, STRING_BLOCK_SIZE) +# define ASAN_POISON_STRING(s) \ + __asan_poison_memory_region (s, sizeof *(s)) +# define ASAN_UNPOISON_STRING(s) \ + __asan_unpoison_memory_region (s, sizeof *(s)) +#else +# define ASAN_PREPARE_DEAD_SDATA(s, size) ((void) 0) +# define ASAN_PREPARE_LIVE_SDATA(s, nbytes) ((void) 0) +# define ASAN_POISON_SBLOCK_DATA(b, size) ((void) 0) +# define ASAN_POISON_STRING_BLOCK(b) ((void) 0) +# define ASAN_UNPOISON_STRING_BLOCK(b) ((void) 0) +# define ASAN_POISON_STRING(s) ((void) 0) +# define ASAN_UNPOISON_STRING(s) ((void) 0) +#endif #ifdef GC_CHECK_STRING_BYTES @@ -1805,12 +1898,14 @@ allocate_string (void) NEXT_FREE_LISP_STRING (s) = string_free_list; string_free_list = s; } + ASAN_POISON_STRING_BLOCK (b); } check_string_free_list (); /* Pop a Lisp_String off the free-list. */ s = string_free_list; + ASAN_UNPOISON_STRING (s); string_free_list = NEXT_FREE_LISP_STRING (s); MALLOC_UNBLOCK_INPUT; @@ -1870,6 +1965,7 @@ allocate_string_data (struct Lisp_String *s, #endif b = lisp_malloc (size + GC_STRING_EXTRA, clearit, MEM_TYPE_NON_LISP); + ASAN_POISON_SBLOCK_DATA (b, size); #ifdef DOUG_LEA_MALLOC if (!mmap_lisp_allowed_p ()) @@ -1891,6 +1987,8 @@ allocate_string_data (struct Lisp_String *s, { /* Not enough room in the current sblock. */ b = lisp_malloc (SBLOCK_SIZE, false, MEM_TYPE_NON_LISP); + ASAN_POISON_SBLOCK_DATA (b, SBLOCK_SIZE); + data = b->data; b->next = NULL; b->next_free = data; @@ -1903,10 +2001,19 @@ allocate_string_data (struct Lisp_String *s, } data = b->next_free; + if (clearit) - memset (SDATA_DATA (data), 0, nbytes); + { +#if GC_ASAN_POISON_OBJECTS + /* We are accessing SDATA_DATA (data) before it gets + * normally unpoisoned, so do it manually. */ + __asan_unpoison_memory_region (SDATA_DATA (data), nbytes); +#endif + memset (SDATA_DATA (data), 0, nbytes); + } } + ASAN_PREPARE_LIVE_SDATA (data, nbytes); data->string = s; b->next_free = (sdata *) ((char *) data + needed + GC_STRING_EXTRA); eassert ((uintptr_t) b->next_free % alignof (sdata) == 0); @@ -1998,12 +2105,16 @@ sweep_strings (void) int i, nfree = 0; struct Lisp_String *free_list_before = string_free_list; + ASAN_UNPOISON_STRING_BLOCK (b); + next = b->next; for (i = 0; i < STRING_BLOCK_SIZE; ++i) { struct Lisp_String *s = b->strings + i; + ASAN_UNPOISON_STRING (s); + if (s->u.s.data) { /* String was not on free-list before. */ @@ -2040,6 +2151,8 @@ sweep_strings (void) /* Put the string on the free-list. */ NEXT_FREE_LISP_STRING (s) = string_free_list; + ASAN_POISON_STRING (s); + ASAN_PREPARE_DEAD_SDATA (data, SDATA_NBYTES (data)); string_free_list = s; ++nfree; } @@ -2048,6 +2161,8 @@ sweep_strings (void) { /* S was on the free-list before. Put it there again. */ NEXT_FREE_LISP_STRING (s) = string_free_list; + ASAN_POISON_STRING (s); + string_free_list = s; ++nfree; } @@ -2174,6 +2289,7 @@ compact_small_strings (void) if (from != to) { eassert (tb != b || to < from); + ASAN_PREPARE_LIVE_SDATA (to, nbytes); memmove (to, from, size + GC_STRING_EXTRA); to->string->u.s.data = SDATA_DATA (to); } @@ -2245,7 +2361,7 @@ a multibyte string even if INIT is an ASCII character. */) ptrdiff_t len = CHAR_STRING (c, str); EMACS_INT string_len = XFIXNUM (length); - if (INT_MULTIPLY_WRAPV (len, string_len, &nbytes)) + if (ckd_mul (&nbytes, len, string_len)) string_overflow (); val = make_clear_multibyte_string (string_len, nbytes, clearit); if (!clearit) @@ -2525,6 +2641,7 @@ pin_string (Lisp_Object string) memcpy (s->u.s.data, data, size); old_sdata->string = NULL; SDATA_NBYTES (old_sdata) = size; + ASAN_PREPARE_DEAD_SDATA (old_sdata, size); } s->u.s.size_byte = -3; } @@ -2574,13 +2691,31 @@ struct float_block }; #define XFLOAT_MARKED_P(fptr) \ - GETMARKBIT (FLOAT_BLOCK (fptr), FLOAT_INDEX ((fptr))) + GETMARKBIT (FLOAT_BLOCK (fptr), FLOAT_INDEX (fptr)) #define XFLOAT_MARK(fptr) \ - SETMARKBIT (FLOAT_BLOCK (fptr), FLOAT_INDEX ((fptr))) + SETMARKBIT (FLOAT_BLOCK (fptr), FLOAT_INDEX (fptr)) #define XFLOAT_UNMARK(fptr) \ - UNSETMARKBIT (FLOAT_BLOCK (fptr), FLOAT_INDEX ((fptr))) + UNSETMARKBIT (FLOAT_BLOCK (fptr), FLOAT_INDEX (fptr)) + +#if GC_ASAN_POISON_OBJECTS +# define ASAN_POISON_FLOAT_BLOCK(fblk) \ + __asan_poison_memory_region ((fblk)->floats, \ + sizeof ((fblk)->floats)) +# define ASAN_UNPOISON_FLOAT_BLOCK(fblk) \ + __asan_unpoison_memory_region ((fblk)->floats, \ + sizeof ((fblk)->floats)) +# define ASAN_POISON_FLOAT(p) \ + __asan_poison_memory_region (p, sizeof (struct Lisp_Float)) +# define ASAN_UNPOISON_FLOAT(p) \ + __asan_unpoison_memory_region (p, sizeof (struct Lisp_Float)) +#else +# define ASAN_POISON_FLOAT_BLOCK(fblk) ((void) 0) +# define ASAN_UNPOISON_FLOAT_BLOCK(fblk) ((void) 0) +# define ASAN_POISON_FLOAT(p) ((void) 0) +# define ASAN_UNPOISON_FLOAT(p) ((void) 0) +#endif /* Current float_block. */ @@ -2606,6 +2741,7 @@ make_float (double float_value) if (float_free_list) { XSETFLOAT (val, float_free_list); + ASAN_UNPOISON_FLOAT (float_free_list); float_free_list = float_free_list->u.chain; } else @@ -2616,9 +2752,11 @@ make_float (double float_value) = lisp_align_malloc (sizeof *new, MEM_TYPE_FLOAT); new->next = float_block; memset (new->gcmarkbits, 0, sizeof new->gcmarkbits); + ASAN_POISON_FLOAT_BLOCK (new); float_block = new; float_block_index = 0; } + ASAN_UNPOISON_FLOAT (&float_block->floats[float_block_index]); XSETFLOAT (val, &float_block->floats[float_block_index]); float_block_index++; } @@ -2665,13 +2803,13 @@ struct cons_block }; #define XCONS_MARKED_P(fptr) \ - GETMARKBIT (CONS_BLOCK (fptr), CONS_INDEX ((fptr))) + GETMARKBIT (CONS_BLOCK (fptr), CONS_INDEX (fptr)) #define XMARK_CONS(fptr) \ - SETMARKBIT (CONS_BLOCK (fptr), CONS_INDEX ((fptr))) + SETMARKBIT (CONS_BLOCK (fptr), CONS_INDEX (fptr)) #define XUNMARK_CONS(fptr) \ - UNSETMARKBIT (CONS_BLOCK (fptr), CONS_INDEX ((fptr))) + UNSETMARKBIT (CONS_BLOCK (fptr), CONS_INDEX (fptr)) /* Minimum number of bytes of consing since GC before next GC, when memory is full. */ @@ -2690,6 +2828,19 @@ static int cons_block_index = CONS_BLOCK_SIZE; static struct Lisp_Cons *cons_free_list; +#if GC_ASAN_POISON_OBJECTS +# define ASAN_POISON_CONS_BLOCK(b) \ + __asan_poison_memory_region ((b)->conses, sizeof ((b)->conses)) +# define ASAN_POISON_CONS(p) \ + __asan_poison_memory_region (p, sizeof (struct Lisp_Cons)) +# define ASAN_UNPOISON_CONS(p) \ + __asan_unpoison_memory_region (p, sizeof (struct Lisp_Cons)) +#else +# define ASAN_POISON_CONS_BLOCK(b) ((void) 0) +# define ASAN_POISON_CONS(p) ((void) 0) +# define ASAN_UNPOISON_CONS(p) ((void) 0) +#endif + /* Explicitly free a cons cell by putting it on the free-list. */ void @@ -2700,6 +2851,7 @@ free_cons (struct Lisp_Cons *ptr) cons_free_list = ptr; ptrdiff_t nbytes = sizeof *ptr; tally_consing (-nbytes); + ASAN_POISON_CONS (ptr); } DEFUN ("cons", Fcons, Scons, 2, 2, 0, @@ -2712,6 +2864,7 @@ DEFUN ("cons", Fcons, Scons, 2, 2, 0, if (cons_free_list) { + ASAN_UNPOISON_CONS (cons_free_list); XSETCONS (val, cons_free_list); cons_free_list = cons_free_list->u.s.u.chain; } @@ -2722,10 +2875,12 @@ DEFUN ("cons", Fcons, Scons, 2, 2, 0, struct cons_block *new = lisp_align_malloc (sizeof *new, MEM_TYPE_CONS); memset (new->gcmarkbits, 0, sizeof new->gcmarkbits); + ASAN_POISON_CONS_BLOCK (new); new->next = cons_block; cons_block = new; cons_block_index = 0; } + ASAN_UNPOISON_CONS (&cons_block->conses[cons_block_index]); XSETCONS (val, &cons_block->conses[cons_block_index]); cons_block_index++; } @@ -2881,9 +3036,8 @@ enum { VECTOR_BLOCK_SIZE = 4096 }; /* Vector size requests are a multiple of this. */ enum { roundup_size = COMMON_MULTIPLE (LISP_ALIGNMENT, word_size) }; -/* Verify assumptions described above. */ +/* Verify assumption described above. */ verify (VECTOR_BLOCK_SIZE % roundup_size == 0); -verify (VECTOR_BLOCK_SIZE <= (1 << PSEUDOVECTOR_SIZE_BITS)); /* Round up X to nearest mult-of-ROUNDUP_SIZE --- use at compile time. */ #define vroundup_ct(x) ROUNDUP (x, roundup_size) @@ -2894,6 +3048,11 @@ verify (VECTOR_BLOCK_SIZE <= (1 << PSEUDOVECTOR_SIZE_BITS)); enum {VECTOR_BLOCK_BYTES = VECTOR_BLOCK_SIZE - vroundup_ct (sizeof (void *))}; +/* The current code expects to be able to represent an unused block by + a single PVEC_FREE object, whose size is limited by the header word. + (Of course we could use multiple such objects.) */ +verify (VECTOR_BLOCK_BYTES <= (word_size << PSEUDOVECTOR_REST_BITS)); + /* Size of the minimal vector allocated from block. */ enum { VBLOCK_BYTES_MIN = vroundup_ct (header_size + sizeof (Lisp_Object)) }; @@ -2903,10 +3062,10 @@ enum { VBLOCK_BYTES_MIN = vroundup_ct (header_size + sizeof (Lisp_Object)) }; enum { VBLOCK_BYTES_MAX = vroundup_ct ((VECTOR_BLOCK_BYTES / 2) - word_size) }; /* We maintain one free list for each possible block-allocated - vector size, and this is the number of free lists we have. */ - -enum { VECTOR_MAX_FREE_LIST_INDEX = - (VECTOR_BLOCK_BYTES - VBLOCK_BYTES_MIN) / roundup_size + 1 }; + vector size, one for blocks one word bigger, + and one for all free vectors larger than that. */ +enum { VECTOR_FREE_LIST_ARRAY_SIZE = + (VBLOCK_BYTES_MAX - VBLOCK_BYTES_MIN) / roundup_size + 1 + 2 }; /* Common shortcut to advance vector pointer over a block data. */ @@ -2968,9 +3127,20 @@ struct vector_block static struct vector_block *vector_blocks; /* Vector free lists, where NTH item points to a chain of free - vectors of the same NBYTES size, so NTH == VINDEX (NBYTES). */ + vectors of the same NBYTES size, so NTH == VINDEX (NBYTES), + except for the last element which may contain larger vectors. + + I.e., for each vector V in vector_free_lists[I] the following holds: + - V has type PVEC_FREE + - V's total size in bytes, BS(V) = PVSIZE(V) * word_size + header_size + - For I < VECTOR_FREE_LIST_ARRAY_SIZE-1, VINDEX(BS(V)) = I + - For I = VECTOR_FREE_LIST_ARRAY_SIZE-1, VINDEX(BS(V)) ≥ I */ +static struct Lisp_Vector *vector_free_lists[VECTOR_FREE_LIST_ARRAY_SIZE]; -static struct Lisp_Vector *vector_free_lists[VECTOR_MAX_FREE_LIST_INDEX]; +/* Index to the bucket in vector_free_lists into which we last inserted + or split a free vector. We use this as a heuristic telling us where + to start looking for free vectors when the exact-size bucket is empty. */ +static ptrdiff_t last_inserted_vector_free_idx = VECTOR_FREE_LIST_ARRAY_SIZE; /* Singly-linked list of large vectors. */ @@ -2980,6 +3150,19 @@ static struct large_vector *large_vectors; Lisp_Object zero_vector; +#if GC_ASAN_POISON_OBJECTS +# define ASAN_POISON_VECTOR_CONTENTS(v, bytes) \ + __asan_poison_memory_region ((v)->contents, bytes) +# define ASAN_UNPOISON_VECTOR_CONTENTS(v, bytes) \ + __asan_unpoison_memory_region ((v)->contents, bytes) +# define ASAN_UNPOISON_VECTOR_BLOCK(b) \ + __asan_unpoison_memory_region ((b)->data, sizeof (b)->data) +#else +# define ASAN_POISON_VECTOR_CONTENTS(v, bytes) ((void) 0) +# define ASAN_UNPOISON_VECTOR_CONTENTS(v, bytes) ((void) 0) +# define ASAN_UNPOISON_VECTOR_BLOCK(b) ((void) 0) +#endif + /* Common shortcut to setup vector on a free list. */ static void @@ -2990,9 +3173,12 @@ setup_on_free_list (struct Lisp_Vector *v, ptrdiff_t nbytes) XSETPVECTYPESIZE (v, PVEC_FREE, 0, nwords); eassert (nbytes % roundup_size == 0); ptrdiff_t vindex = VINDEX (nbytes); - eassert (vindex < VECTOR_MAX_FREE_LIST_INDEX); + /* Anything too large goes into the last slot (overflow bin). */ + vindex = min(vindex, VECTOR_FREE_LIST_ARRAY_SIZE - 1); set_next_vector (v, vector_free_lists[vindex]); + ASAN_POISON_VECTOR_CONTENTS (v, nbytes - header_size); vector_free_lists[vindex] = v; + last_inserted_vector_free_idx = vindex; } /* Get a new vector block. */ @@ -3021,6 +3207,17 @@ init_vectors (void) staticpro (&zero_vector); } +/* Memory footprint in bytes of a pseudovector other than a bool-vector. */ +static ptrdiff_t +pseudovector_nbytes (const union vectorlike_header *hdr) +{ + eassert (!PSEUDOVECTOR_TYPEP (hdr, PVEC_BOOL_VECTOR)); + ptrdiff_t nwords = ((hdr->size & PSEUDOVECTOR_SIZE_MASK) + + ((hdr->size & PSEUDOVECTOR_REST_MASK) + >> PSEUDOVECTOR_SIZE_BITS)); + return vroundup (header_size + word_size * nwords); +} + /* Allocate vector from a vector block. */ static struct Lisp_Vector * @@ -3039,6 +3236,7 @@ allocate_vector_from_block (ptrdiff_t nbytes) if (vector_free_lists[index]) { vector = vector_free_lists[index]; + ASAN_UNPOISON_VECTOR_CONTENTS (vector, nbytes - header_size); vector_free_lists[index] = next_vector (vector); return vector; } @@ -3046,18 +3244,27 @@ allocate_vector_from_block (ptrdiff_t nbytes) /* Next, check free lists containing larger vectors. Since we will split the result, we should have remaining space large enough to use for one-slot vector at least. */ - for (index = VINDEX (nbytes + VBLOCK_BYTES_MIN); - index < VECTOR_MAX_FREE_LIST_INDEX; index++) + for (index = max (VINDEX (nbytes + VBLOCK_BYTES_MIN), + last_inserted_vector_free_idx); + index < VECTOR_FREE_LIST_ARRAY_SIZE; index++) if (vector_free_lists[index]) { /* This vector is larger than requested. */ vector = vector_free_lists[index]; + size_t vector_nbytes = pseudovector_nbytes (&vector->header); + eassert (vector_nbytes > nbytes); + ASAN_UNPOISON_VECTOR_CONTENTS (vector, nbytes - header_size); vector_free_lists[index] = next_vector (vector); /* Excess bytes are used for the smaller vector, which should be set on an appropriate free list. */ - restbytes = index * roundup_size + VBLOCK_BYTES_MIN - nbytes; + restbytes = vector_nbytes - nbytes; eassert (restbytes % roundup_size == 0); +#if GC_ASAN_POISON_OBJECTS + /* Ensure that accessing excess bytes does not trigger ASan. */ + __asan_unpoison_memory_region (ADVANCE (vector, nbytes), + restbytes); +#endif setup_on_free_list (ADVANCE (vector, nbytes), restbytes); return vector; } @@ -3105,9 +3312,7 @@ vectorlike_nbytes (const union vectorlike_header *hdr) nwords = (boolvec_bytes - header_size + word_size - 1) / word_size; } else - nwords = ((size & PSEUDOVECTOR_SIZE_MASK) - + ((size & PSEUDOVECTOR_REST_MASK) - >> PSEUDOVECTOR_SIZE_BITS)); + return pseudovector_nbytes (hdr); } else nwords = size; @@ -3129,85 +3334,159 @@ vectorlike_nbytes (const union vectorlike_header *hdr) static void cleanup_vector (struct Lisp_Vector *vector) { - detect_suspicious_free (vector); - - if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_BIGNUM)) - mpz_clear (PSEUDOVEC_STRUCT (vector, Lisp_Bignum)->value); - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_OVERLAY)) - { - struct Lisp_Overlay *ol = PSEUDOVEC_STRUCT (vector, Lisp_Overlay); - xfree (ol->interval); - } - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_FINALIZER)) - unchain_finalizer (PSEUDOVEC_STRUCT (vector, Lisp_Finalizer)); - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_FONT)) + if ((vector->header.size & PSEUDOVECTOR_FLAG) == 0) + return; /* nothing more to do for plain vectors */ + switch (PSEUDOVECTOR_TYPE (vector)) { - if ((vector->header.size & PSEUDOVECTOR_SIZE_MASK) == FONT_OBJECT_MAX) - { - struct font *font = PSEUDOVEC_STRUCT (vector, font); - struct font_driver const *drv = font->driver; + case PVEC_BIGNUM: + mpz_clear (PSEUDOVEC_STRUCT (vector, Lisp_Bignum)->value); + break; + case PVEC_OVERLAY: + { + struct Lisp_Overlay *ol = PSEUDOVEC_STRUCT (vector, Lisp_Overlay); + xfree (ol->interval); + } + break; + case PVEC_FINALIZER: + unchain_finalizer (PSEUDOVEC_STRUCT (vector, Lisp_Finalizer)); + break; + case PVEC_FONT: + { + if ((vector->header.size & PSEUDOVECTOR_SIZE_MASK) == FONT_OBJECT_MAX) + { + struct font *font = PSEUDOVEC_STRUCT (vector, font); + struct font_driver const *drv = font->driver; - /* The font driver might sometimes be NULL, e.g. if Emacs was - interrupted before it had time to set it up. */ - if (drv) - { - /* Attempt to catch subtle bugs like Bug#16140. */ - eassert (valid_font_driver (drv)); - drv->close_font (font); - } - } - } - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_THREAD)) - finalize_one_thread (PSEUDOVEC_STRUCT (vector, thread_state)); - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_MUTEX)) - finalize_one_mutex (PSEUDOVEC_STRUCT (vector, Lisp_Mutex)); - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_CONDVAR)) - finalize_one_condvar (PSEUDOVEC_STRUCT (vector, Lisp_CondVar)); - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_MARKER)) - { + /* The font driver might sometimes be NULL, e.g. if Emacs was + interrupted before it had time to set it up. */ + if (drv) + { + /* Attempt to catch subtle bugs like Bug#16140. */ + eassert (valid_font_driver (drv)); + drv->close_font (font); + } + } + +#if defined HAVE_ANDROID && !defined ANDROID_STUBIFY + /* The Android font driver needs the ability to associate extra + information with font entities. */ + if (((vector->header.size & PSEUDOVECTOR_SIZE_MASK) + == FONT_ENTITY_MAX) + && PSEUDOVEC_STRUCT (vector, font_entity)->is_android) + android_finalize_font_entity (PSEUDOVEC_STRUCT (vector, font_entity)); +#endif + } + break; + case PVEC_THREAD: + finalize_one_thread (PSEUDOVEC_STRUCT (vector, thread_state)); + break; + case PVEC_MUTEX: + finalize_one_mutex (PSEUDOVEC_STRUCT (vector, Lisp_Mutex)); + break; + case PVEC_CONDVAR: + finalize_one_condvar (PSEUDOVEC_STRUCT (vector, Lisp_CondVar)); + break; + case PVEC_MARKER: /* sweep_buffer should already have unchained this from its buffer. */ eassert (! PSEUDOVEC_STRUCT (vector, Lisp_Marker)->buffer); - } - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_USER_PTR)) - { - struct Lisp_User_Ptr *uptr = PSEUDOVEC_STRUCT (vector, Lisp_User_Ptr); - if (uptr->finalizer) - uptr->finalizer (uptr->p); - } + break; + case PVEC_USER_PTR: + { + struct Lisp_User_Ptr *uptr = PSEUDOVEC_STRUCT (vector, Lisp_User_Ptr); + if (uptr->finalizer) + uptr->finalizer (uptr->p); + } + break; + case PVEC_TS_PARSER: +#ifdef HAVE_TREE_SITTER + treesit_delete_parser (PSEUDOVEC_STRUCT (vector, Lisp_TS_Parser)); +#endif + break; + case PVEC_TS_COMPILED_QUERY: #ifdef HAVE_TREE_SITTER - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_TS_PARSER)) - treesit_delete_parser (PSEUDOVEC_STRUCT (vector, Lisp_TS_Parser)); - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_TS_COMPILED_QUERY)) - treesit_delete_query (PSEUDOVEC_STRUCT (vector, Lisp_TS_Query)); + treesit_delete_query (PSEUDOVEC_STRUCT (vector, Lisp_TS_Query)); #endif + break; + case PVEC_MODULE_FUNCTION: #ifdef HAVE_MODULES - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_MODULE_FUNCTION)) - { - ATTRIBUTE_MAY_ALIAS struct Lisp_Module_Function *function - = (struct Lisp_Module_Function *) vector; - module_finalize_function (function); - } + { + ATTRIBUTE_MAY_ALIAS struct Lisp_Module_Function *function + = (struct Lisp_Module_Function *) vector; + module_finalize_function (function); + } #endif + break; + case PVEC_NATIVE_COMP_UNIT: #ifdef HAVE_NATIVE_COMP - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_NATIVE_COMP_UNIT)) - { - struct Lisp_Native_Comp_Unit *cu = - PSEUDOVEC_STRUCT (vector, Lisp_Native_Comp_Unit); - unload_comp_unit (cu); - } - else if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_SUBR)) - { - struct Lisp_Subr *subr = - PSEUDOVEC_STRUCT (vector, Lisp_Subr); - if (!NILP (subr->native_comp_u)) - { - /* FIXME Alternative and non invasive solution to this - cast? */ - xfree ((char *)subr->symbol_name); - xfree (subr->native_c_name); - } - } + { + struct Lisp_Native_Comp_Unit *cu = + PSEUDOVEC_STRUCT (vector, Lisp_Native_Comp_Unit); + unload_comp_unit (cu); + } +#endif + break; + case PVEC_SUBR: +#ifdef HAVE_NATIVE_COMP + { + struct Lisp_Subr *subr = PSEUDOVEC_STRUCT (vector, Lisp_Subr); + if (!NILP (subr->native_comp_u)) + { + /* FIXME Alternative and non invasive solution to this cast? */ + xfree ((char *)subr->symbol_name); + xfree (subr->native_c_name); + } + } #endif + break; + case PVEC_HASH_TABLE: + { + struct Lisp_Hash_Table *h = PSEUDOVEC_STRUCT (vector, Lisp_Hash_Table); + if (h->table_size > 0) + { + eassert (h->index_bits > 0); + xfree (h->index); + xfree (h->key_and_value); + xfree (h->next); + xfree (h->hash); + ptrdiff_t bytes = (h->table_size * (2 * sizeof *h->key_and_value + + sizeof *h->hash + + sizeof *h->next) + + hash_table_index_size (h) * sizeof *h->index); + hash_table_allocated_bytes -= bytes; + } + } + break; + case PVEC_OBARRAY: + { + struct Lisp_Obarray *o = PSEUDOVEC_STRUCT (vector, Lisp_Obarray); + xfree (o->buckets); + ptrdiff_t bytes = obarray_size (o) * sizeof *o->buckets; + hash_table_allocated_bytes -= bytes; + } + break; + /* Keep the switch exhaustive. */ + case PVEC_NORMAL_VECTOR: + case PVEC_FREE: + case PVEC_SYMBOL_WITH_POS: + case PVEC_MISC_PTR: + case PVEC_PROCESS: + case PVEC_FRAME: + case PVEC_WINDOW: + case PVEC_BOOL_VECTOR: + case PVEC_BUFFER: + case PVEC_TERMINAL: + case PVEC_WINDOW_CONFIGURATION: + case PVEC_OTHER: + case PVEC_XWIDGET: + case PVEC_XWIDGET_VIEW: + case PVEC_TS_NODE: + case PVEC_SQLITE: + case PVEC_COMPILED: + case PVEC_CHAR_TABLE: + case PVEC_SUB_CHAR_TABLE: + case PVEC_RECORD: + break; + } } /* Reclaim space used by unmarked vectors. */ @@ -3223,6 +3502,7 @@ sweep_vectors (void) gcstat.total_vectors = 0; gcstat.total_vector_slots = gcstat.total_free_vector_slots = 0; memset (vector_free_lists, 0, sizeof (vector_free_lists)); + last_inserted_vector_free_idx = VECTOR_FREE_LIST_ARRAY_SIZE; /* Looking through vector blocks. */ @@ -3233,6 +3513,7 @@ sweep_vectors (void) for (vector = (struct Lisp_Vector *) block->data; VECTOR_IN_BLOCK (vector, block); vector = next) { + ASAN_UNPOISON_VECTOR_BLOCK (block); if (XVECTOR_MARKED_P (vector)) { XUNMARK_VECTOR (vector); @@ -3306,6 +3587,8 @@ sweep_vectors (void) lisp_free (lv); } } + + gcstat.total_hash_table_bytes = hash_table_allocated_bytes; } /* Maximum number of elements in a vector. This is a macro so that it @@ -3355,9 +3638,6 @@ allocate_vectorlike (ptrdiff_t len, bool clearit) mallopt (M_MMAP_MAX, MMAP_MAX_AREAS); #endif - if (find_suspicious_object_in_range (p, (char *) p + nbytes)) - emacs_abort (); - tally_consing (nbytes); vector_cells_consed += len; @@ -3409,7 +3689,7 @@ allocate_pseudovector (int memlen, int lisplen, enum { size_max = (1 << PSEUDOVECTOR_SIZE_BITS) - 1 }; enum { rest_max = (1 << PSEUDOVECTOR_REST_BITS) - 1 }; verify (size_max + rest_max <= VECTOR_ELTS_MAX); - eassert (0 <= tag && tag <= PVEC_FONT); + eassert (0 <= tag && tag <= PVEC_TAG_MAX); eassert (0 <= lisplen && lisplen <= zerolen && zerolen <= memlen); eassert (lisplen <= size_max); eassert (memlen <= size_max + rest_max); @@ -3609,6 +3889,23 @@ struct symbol_block struct symbol_block *next; }; +#if GC_ASAN_POISON_OBJECTS +# define ASAN_POISON_SYMBOL_BLOCK(s) \ + __asan_poison_memory_region ((s)->symbols, sizeof ((s)->symbols)) +# define ASAN_UNPOISON_SYMBOL_BLOCK(s) \ + __asan_unpoison_memory_region ((s)->symbols, sizeof ((s)->symbols)) +# define ASAN_POISON_SYMBOL(sym) \ + __asan_poison_memory_region (sym, sizeof *(sym)) +# define ASAN_UNPOISON_SYMBOL(sym) \ + __asan_unpoison_memory_region (sym, sizeof *(sym)) + +#else +# define ASAN_POISON_SYMBOL_BLOCK(s) ((void) 0) +# define ASAN_UNPOISON_SYMBOL_BLOCK(s) ((void) 0) +# define ASAN_POISON_SYMBOL(sym) ((void) 0) +# define ASAN_UNPOISON_SYMBOL(sym) ((void) 0) +#endif + /* Current symbol block and index of first unused Lisp_Symbol structure in it. */ @@ -3662,6 +3959,7 @@ Its value is void, and its function definition and property list are nil. */) if (symbol_free_list) { + ASAN_UNPOISON_SYMBOL (symbol_free_list); XSETSYMBOL (val, symbol_free_list); symbol_free_list = symbol_free_list->u.s.next; } @@ -3671,10 +3969,13 @@ Its value is void, and its function definition and property list are nil. */) { struct symbol_block *new = lisp_malloc (sizeof *new, false, MEM_TYPE_SYMBOL); + ASAN_POISON_SYMBOL_BLOCK (new); new->next = symbol_block; symbol_block = new; symbol_block_index = 0; } + + ASAN_UNPOISON_SYMBOL (&symbol_block->symbols[symbol_block_index]); XSETSYMBOL (val, &symbol_block->symbols[symbol_block_index]); symbol_block_index++; } @@ -4562,6 +4863,11 @@ static struct Lisp_String * live_string_holding (struct mem_node *m, void *p) { eassert (m->type == MEM_TYPE_STRING); +#if GC_ASAN_POISON_OBJECTS + if (__asan_address_is_poisoned (p)) + return NULL; +#endif + struct string_block *b = m->start; char *cp = p; ptrdiff_t offset = cp - (char *) &b->strings[0]; @@ -4578,6 +4884,10 @@ live_string_holding (struct mem_node *m, void *p) || off == offsetof (struct Lisp_String, u.s.data)) { struct Lisp_String *s = p = cp -= off; +#if GC_ASAN_POISON_OBJECTS + if (__asan_region_is_poisoned (s, sizeof (*s))) + return NULL; +#endif if (s->u.s.data) return s; } @@ -4599,6 +4909,11 @@ static struct Lisp_Cons * live_cons_holding (struct mem_node *m, void *p) { eassert (m->type == MEM_TYPE_CONS); +#if GC_ASAN_POISON_OBJECTS + if (__asan_address_is_poisoned (p)) + return NULL; +#endif + struct cons_block *b = m->start; char *cp = p; ptrdiff_t offset = cp - (char *) &b->conses[0]; @@ -4616,6 +4931,10 @@ live_cons_holding (struct mem_node *m, void *p) || off == offsetof (struct Lisp_Cons, u.s.u.cdr)) { struct Lisp_Cons *s = p = cp -= off; +#if GC_ASAN_POISON_OBJECTS + if (__asan_region_is_poisoned (s, sizeof (*s))) + return NULL; +#endif if (!deadp (s->u.s.car)) return s; } @@ -4638,6 +4957,10 @@ static struct Lisp_Symbol * live_symbol_holding (struct mem_node *m, void *p) { eassert (m->type == MEM_TYPE_SYMBOL); +#if GC_ASAN_POISON_OBJECTS + if (__asan_address_is_poisoned (p)) + return NULL; +#endif struct symbol_block *b = m->start; char *cp = p; ptrdiff_t offset = cp - (char *) &b->symbols[0]; @@ -4663,6 +4986,10 @@ live_symbol_holding (struct mem_node *m, void *p) || off == offsetof (struct Lisp_Symbol, u.s.next)) { struct Lisp_Symbol *s = p = cp -= off; +#if GC_ASAN_POISON_OBJECTS + if (__asan_region_is_poisoned (s, sizeof (*s))) + return NULL; +#endif if (!deadp (s->u.s.function)) return s; } @@ -4685,6 +5012,11 @@ static struct Lisp_Float * live_float_holding (struct mem_node *m, void *p) { eassert (m->type == MEM_TYPE_FLOAT); +#if GC_ASAN_POISON_OBJECTS + if (__asan_address_is_poisoned (p)) + return NULL; +#endif + struct float_block *b = m->start; char *cp = p; ptrdiff_t offset = cp - (char *) &b->floats[0]; @@ -4699,8 +5031,12 @@ live_float_holding (struct mem_node *m, void *p) && (b != float_block || offset / sizeof b->floats[0] < float_block_index)) { - p = cp - off; - return p; + struct Lisp_Float *f = (struct Lisp_Float *) (cp - off); +#if GC_ASAN_POISON_OBJECTS + if (__asan_region_is_poisoned (f, sizeof (*f))) + return NULL; +#endif + return f; } } return NULL; @@ -4979,7 +5315,7 @@ mark_memory (void const *start, void const *end) a Lisp_Object might be split into registers saved into non-adjacent words and P might be the low-order word's value. */ intptr_t ip; - INT_ADD_WRAPV ((intptr_t) p, (intptr_t) lispsym, &ip); + ckd_add (&ip, (intptr_t) p, (intptr_t) lispsym); mark_maybe_pointer ((void *) ip, true); } } @@ -5091,15 +5427,6 @@ typedef union #endif } stacktop_sentry; -/* Yield an address close enough to the top of the stack that the - garbage collector need not scan above it. Callers should be - declared NO_INLINE. */ -#ifdef HAVE___BUILTIN_FRAME_ADDRESS -# define NEAR_STACK_TOP(addr) ((void) (addr), __builtin_frame_address (0)) -#else -# define NEAR_STACK_TOP(addr) (addr) -#endif - /* Set *P to the address of the top of the stack. This must be a macro, not a function, so that it is executed in the caller's environment. It is not inside a do-while so that its storage @@ -5314,6 +5641,29 @@ valid_lisp_object_p (Lisp_Object obj) return 0; } +/* Like xmalloc, but makes allocation count toward the total consing + and hash table or obarray usage. + Return NULL for a zero-sized allocation. */ +void * +hash_table_alloc_bytes (ptrdiff_t nbytes) +{ + if (nbytes == 0) + return NULL; + tally_consing (nbytes); + hash_table_allocated_bytes += nbytes; + return xmalloc (nbytes); +} + +/* Like xfree, but makes allocation count toward the total consing. */ +void +hash_table_free_bytes (void *p, ptrdiff_t nbytes) +{ + tally_consing (-nbytes); + hash_table_allocated_bytes -= nbytes; + xfree (p); +} + + /*********************************************************************** Pure Storage Management ***********************************************************************/ @@ -5407,6 +5757,22 @@ find_string_data_in_pure (const char *data, ptrdiff_t nbytes) if (pure_bytes_used_non_lisp <= nbytes) return NULL; + /* The Android GCC generates code like: + + 0xa539e755 <+52>: lea 0x430(%esp),%esi +=> 0xa539e75c <+59>: movdqa %xmm0,0x0(%ebp) + 0xa539e761 <+64>: add $0x10,%ebp + + but data is not aligned appropriately, so a GP fault results. */ + +#if defined __i386__ \ + && defined HAVE_ANDROID \ + && !defined ANDROID_STUBIFY \ + && !defined (__clang__) + if ((intptr_t) data & 15) + return NULL; +#endif + /* Set up the Boyer-Moore table. */ skip = nbytes + 1; for (i = 0; i < 256; i++) @@ -5579,30 +5945,35 @@ make_pure_vector (ptrdiff_t len) static struct Lisp_Hash_Table * purecopy_hash_table (struct Lisp_Hash_Table *table) { - eassert (NILP (table->weak)); + eassert (table->weakness == Weak_None); eassert (table->purecopy); struct Lisp_Hash_Table *pure = pure_alloc (sizeof *pure, Lisp_Vectorlike); - struct hash_table_test pure_test = table->test; - - /* Purecopy the hash table test. */ - pure_test.name = purecopy (table->test.name); - pure_test.user_hash_function = purecopy (table->test.user_hash_function); - pure_test.user_cmp_function = purecopy (table->test.user_cmp_function); - - pure->header = table->header; - pure->weak = purecopy (Qnil); - pure->hash = purecopy (table->hash); - pure->next = purecopy (table->next); - pure->index = purecopy (table->index); - pure->count = table->count; - pure->next_free = table->next_free; - pure->purecopy = table->purecopy; - eassert (!pure->mutable); - pure->rehash_threshold = table->rehash_threshold; - pure->rehash_size = table->rehash_size; - pure->key_and_value = purecopy (table->key_and_value); - pure->test = pure_test; + *pure = *table; + pure->mutable = false; + + if (table->table_size > 0) + { + ptrdiff_t hash_bytes = table->table_size * sizeof *table->hash; + pure->hash = pure_alloc (hash_bytes, -(int)sizeof *table->hash); + memcpy (pure->hash, table->hash, hash_bytes); + + ptrdiff_t next_bytes = table->table_size * sizeof *table->next; + pure->next = pure_alloc (next_bytes, -(int)sizeof *table->next); + memcpy (pure->next, table->next, next_bytes); + + ptrdiff_t nvalues = table->table_size * 2; + ptrdiff_t kv_bytes = nvalues * sizeof *table->key_and_value; + pure->key_and_value = pure_alloc (kv_bytes, + -(int)sizeof *table->key_and_value); + for (ptrdiff_t i = 0; i < nvalues; i++) + pure->key_and_value[i] = purecopy (table->key_and_value[i]); + + ptrdiff_t index_bytes = hash_table_index_size (table) + * sizeof *table->index; + pure->index = pure_alloc (index_bytes, -(int)sizeof *table->index); + memcpy (pure->index, table->index, index_bytes); + } return pure; } @@ -5662,7 +6033,7 @@ purecopy (Lisp_Object obj) /* Do not purify hash tables which haven't been defined with :purecopy as non-nil or are weak - they aren't guaranteed to not change. */ - if (!NILP (table->weak) || !table->purecopy) + if (table->weakness != Weak_None || !table->purecopy) { /* Instead, add the hash table to the list of pinned objects, so that it will be marked during GC. */ @@ -5673,8 +6044,7 @@ purecopy (Lisp_Object obj) return obj; /* Don't hash cons it. */ } - struct Lisp_Hash_Table *h = purecopy_hash_table (table); - XSET_HASH_TABLE (obj, h); + obj = make_lisp_hash_table (purecopy_hash_table (table)); } else if (COMPILEDP (obj) || VECTORP (obj) || RECORDP (obj)) { @@ -5786,6 +6156,7 @@ total_bytes_of_live_objects (void) tot += object_bytes (gcstat.total_floats, sizeof (struct Lisp_Float)); tot += object_bytes (gcstat.total_intervals, sizeof (struct interval)); tot += object_bytes (gcstat.total_strings, sizeof (struct Lisp_String)); + tot += gcstat.total_hash_table_bytes; return tot; } @@ -5917,16 +6288,44 @@ mark_pinned_objects (void) mark_object (pobj->object); } +#if defined HAVE_ANDROID && !defined (__clang__) + +/* The Android gcc is broken and needs the following version of + make_lisp_symbol. Otherwise a mysterious ICE pops up. */ + +#define make_lisp_symbol android_make_lisp_symbol + +static Lisp_Object +android_make_lisp_symbol (struct Lisp_Symbol *sym) +{ + intptr_t symoffset; + + symoffset = (intptr_t) sym; + ckd_sub (&symoffset, symoffset, (intptr_t) &lispsym); + + { + Lisp_Object a = TAG_PTR_INITIALLY (Lisp_Symbol, symoffset); + return a; + } +} + +#endif + static void mark_pinned_symbols (void) { struct symbol_block *sblk; - int lim = (symbol_block_pinned == symbol_block - ? symbol_block_index : SYMBOL_BLOCK_SIZE); + int lim; + struct Lisp_Symbol *sym, *end; + + if (symbol_block_pinned == symbol_block) + lim = symbol_block_index; + else + lim = SYMBOL_BLOCK_SIZE; for (sblk = symbol_block_pinned; sblk; sblk = sblk->next) { - struct Lisp_Symbol *sym = sblk->symbols, *end = sym + lim; + sym = sblk->symbols, end = sym + lim; for (; sym < end; ++sym) if (sym->u.s.pinned) mark_object (make_lisp_symbol (sym)); @@ -6204,6 +6603,9 @@ garbage_collect (void) mark_terminals (); mark_kboards (); mark_threads (); + mark_charset (); + mark_composite (); + mark_profiler (); #ifdef HAVE_PGTK mark_pgtkterm (); #endif @@ -6222,15 +6624,24 @@ garbage_collect (void) #ifdef HAVE_X_WINDOWS mark_xterm (); + mark_xselect (); +#endif + +#ifdef HAVE_ANDROID + mark_androidterm (); +#ifndef ANDROID_STUBIFY + mark_sfntfont (); +#endif #endif #ifdef HAVE_NS mark_nsterm (); #endif + mark_fns (); /* Everything is now marked, except for the data in font caches, undo lists, and finalizers. The first two are compacted by - removing an items which aren't reachable otherwise. */ + removing any items which aren't reachable otherwise. */ compact_font_caches (); @@ -6291,13 +6702,6 @@ garbage_collect (void) image_prune_animation_caches (false); #endif - if (!NILP (Vpost_gc_hook)) - { - specpdl_ref gc_count = inhibit_garbage_collection (); - safe_run_hooks (Qpost_gc_hook); - unbind_to (gc_count, Qnil); - } - /* Accumulate statistics. */ if (FLOATP (Vgc_elapsed)) { @@ -6316,6 +6720,13 @@ garbage_collect (void) if (tot_after < tot_before) malloc_probe (min (tot_before - tot_after, SIZE_MAX)); } + + if (!NILP (Vpost_gc_hook)) + { + specpdl_ref gc_count = inhibit_garbage_collection (); + safe_run_hooks (Qpost_gc_hook); + unbind_to (gc_count, Qnil); + } } DEFUN ("garbage-collect", Fgarbage_collect, Sgarbage_collect, 0, 0, "", @@ -6555,7 +6966,7 @@ mark_buffer (struct buffer *buffer) if (!BUFFER_LIVE_P (buffer)) mark_object (BVAR (buffer, undo_list)); - if (buffer->overlays) + if (!itree_empty_p (buffer->overlays)) mark_overlays (buffer->overlays->root); /* If this is an indirect buffer, mark its base buffer. */ @@ -6587,20 +6998,6 @@ mark_face_cache (struct face_cache *c) } } -NO_INLINE /* To reduce stack depth in mark_object. */ -static void -mark_localized_symbol (struct Lisp_Symbol *ptr) -{ - struct Lisp_Buffer_Local_Value *blv = SYMBOL_BLV (ptr); - Lisp_Object where = blv->where; - /* If the value is set up for a killed buffer restore its global binding. */ - if ((BUFFERP (where) && !BUFFER_LIVE_P (XBUFFER (where)))) - swap_in_global_binding (ptr); - mark_object (blv->where); - mark_object (blv->valcell); - mark_object (blv->defcell); -} - /* Remove killed buffers or items whose car is a killed buffer from LIST, and mark other items. Return changed LIST, which is marked. */ @@ -6632,6 +7029,11 @@ static void mark_frame (struct Lisp_Vector *ptr) { struct frame *f = (struct frame *) ptr; +#ifdef HAVE_TEXT_CONVERSION + struct text_conversion_action *tem; +#endif + + mark_vectorlike (&ptr->header); mark_face_cache (f->face_cache); #ifdef HAVE_WINDOW_SYSTEM @@ -6643,6 +7045,15 @@ mark_frame (struct Lisp_Vector *ptr) mark_vectorlike (&font->header); } #endif + +#ifdef HAVE_TEXT_CONVERSION + mark_object (f->conversion.compose_region_start); + mark_object (f->conversion.compose_region_end); + mark_object (f->conversion.compose_region_overlay); + + for (tem = f->conversion.actions; tem; tem = tem->next) + mark_object (tem->data); +#endif } static void @@ -6891,26 +7302,32 @@ process_mark_stack (ptrdiff_t base_sp) case PVEC_HASH_TABLE: { struct Lisp_Hash_Table *h = (struct Lisp_Hash_Table *)ptr; - ptrdiff_t size = ptr->header.size & PSEUDOVECTOR_SIZE_MASK; set_vector_marked (ptr); - mark_stack_push_values (ptr->contents, size); - mark_stack_push_value (h->test.name); - mark_stack_push_value (h->test.user_hash_function); - mark_stack_push_value (h->test.user_cmp_function); - if (NILP (h->weak)) - mark_stack_push_value (h->key_and_value); + if (h->weakness == Weak_None) + /* The values pushed here may include + HASH_UNUSED_ENTRY_KEY, which this function must + cope with. */ + mark_stack_push_values (h->key_and_value, + 2 * h->table_size); else { - /* For weak tables, mark only the vector and not its + /* For weak tables, don't mark the contents --- that's what makes it weak. */ eassert (h->next_weak == NULL); h->next_weak = weak_hash_tables; weak_hash_tables = h; - set_vector_marked (XVECTOR (h->key_and_value)); } break; } + case PVEC_OBARRAY: + { + struct Lisp_Obarray *o = (struct Lisp_Obarray *)ptr; + set_vector_marked (ptr); + mark_stack_push_values (o->buckets, obarray_size (o)); + break; + } + case PVEC_CHAR_TABLE: case PVEC_SUB_CHAR_TABLE: mark_char_table (ptr, (enum pvec_type) pvectype); @@ -6988,7 +7405,17 @@ process_mark_stack (ptrdiff_t base_sp) break; } case SYMBOL_LOCALIZED: - mark_localized_symbol (ptr); + { + struct Lisp_Buffer_Local_Value *blv = SYMBOL_BLV (ptr); + Lisp_Object where = blv->where; + /* If the value is set up for a killed buffer, + restore its global binding. */ + if (BUFFERP (where) && !BUFFER_LIVE_P (XBUFFER (where))) + swap_in_global_binding (ptr); + mark_stack_push_value (blv->where); + mark_stack_push_value (blv->valcell); + mark_stack_push_value (blv->defcell); + } break; case SYMBOL_FORWARDED: /* If the value is forwarded to a buffer or keyboard field, @@ -7032,14 +7459,19 @@ process_mark_stack (ptrdiff_t base_sp) } case Lisp_Float: - CHECK_ALLOCATED_AND_LIVE (live_float_p, MEM_TYPE_FLOAT); - /* Do not mark floats stored in a dump image: these floats are - "cold" and do not have mark bits. */ - if (pdumper_object_p (XFLOAT (obj))) - eassert (pdumper_cold_object_p (XFLOAT (obj))); - else if (!XFLOAT_MARKED_P (XFLOAT (obj))) - XFLOAT_MARK (XFLOAT (obj)); - break; + { + struct Lisp_Float *f = XFLOAT (obj); + if (!f) + break; /* for HASH_UNUSED_ENTRY_KEY */ + CHECK_ALLOCATED_AND_LIVE (live_float_p, MEM_TYPE_FLOAT); + /* Do not mark floats stored in a dump image: these floats are + "cold" and do not have mark bits. */ + if (pdumper_object_p (f)) + eassert (pdumper_cold_object_p (f)); + else if (!XFLOAT_MARKED_P (f)) + XFLOAT_MARK (f); + break; + } case_Lisp_Int: break; @@ -7181,11 +7613,13 @@ sweep_conses (void) struct Lisp_Cons *acons = &cblk->conses[pos]; if (!XCONS_MARKED_P (acons)) { + ASAN_UNPOISON_CONS (&cblk->conses[pos]); this_free++; cblk->conses[pos].u.s.u.chain = cons_free_list; cons_free_list = &cblk->conses[pos]; cons_free_list->u.s.car = dead_object (); - } + ASAN_POISON_CONS (&cblk->conses[pos]); + } else { num_used++; @@ -7203,6 +7637,7 @@ sweep_conses (void) { *cprev = cblk->next; /* Unhook from the free list. */ + ASAN_UNPOISON_CONS (&cblk->conses[0]); cons_free_list = cblk->conses[0].u.s.u.chain; lisp_align_free (cblk); } @@ -7229,6 +7664,7 @@ sweep_floats (void) for (struct float_block *fblk; (fblk = *fprev); ) { int this_free = 0; + ASAN_UNPOISON_FLOAT_BLOCK (fblk); for (int i = 0; i < lim; i++) { struct Lisp_Float *afloat = &fblk->floats[i]; @@ -7236,6 +7672,7 @@ sweep_floats (void) { this_free++; fblk->floats[i].u.chain = float_free_list; + ASAN_POISON_FLOAT (&fblk->floats[i]); float_free_list = &fblk->floats[i]; } else @@ -7252,7 +7689,8 @@ sweep_floats (void) { *fprev = fblk->next; /* Unhook from the free list. */ - float_free_list = fblk->floats[0].u.chain; + ASAN_UNPOISON_FLOAT (&fblk->floats[0]); + float_free_list = fblk->floats[0].u.chain; lisp_align_free (fblk); } else @@ -7278,13 +7716,14 @@ sweep_intervals (void) for (struct interval_block *iblk; (iblk = *iprev); ) { int this_free = 0; - + ASAN_UNPOISON_INTERVAL_BLOCK (iblk); for (int i = 0; i < lim; i++) { if (!iblk->intervals[i].gcmarkbit) { set_interval_parent (&iblk->intervals[i], interval_free_list); interval_free_list = &iblk->intervals[i]; + ASAN_POISON_INTERVAL (&iblk->intervals[i]); this_free++; } else @@ -7301,6 +7740,7 @@ sweep_intervals (void) { *iprev = iblk->next; /* Unhook from the free list. */ + ASAN_UNPOISON_INTERVAL (&iblk->intervals[0]); interval_free_list = INTERVAL_PARENT (&iblk->intervals[0]); lisp_free (iblk); } @@ -7330,6 +7770,8 @@ sweep_symbols (void) for (sblk = symbol_block; sblk; sblk = *sprev) { + ASAN_UNPOISON_SYMBOL_BLOCK (sblk); + int this_free = 0; struct Lisp_Symbol *sym = sblk->symbols; struct Lisp_Symbol *end = sym + lim; @@ -7351,7 +7793,8 @@ sweep_symbols (void) sym->u.s.next = symbol_free_list; symbol_free_list = sym; symbol_free_list->u.s.function = dead_object (); - ++this_free; + ASAN_POISON_SYMBOL (sym); + ++this_free; } else { @@ -7370,6 +7813,7 @@ sweep_symbols (void) { *sprev = sblk->next; /* Unhook from the free list. */ + ASAN_UNPOISON_SYMBOL (&sblk->symbols[0]); symbol_free_list = sblk->symbols[0].u.s.next; lisp_free (sblk); } @@ -7623,78 +8067,6 @@ which_symbols (Lisp_Object obj, EMACS_INT find_max) return unbind_to (gc_count, found); } -#ifdef SUSPICIOUS_OBJECT_CHECKING - -static void * -find_suspicious_object_in_range (void *begin, void *end) -{ - char *begin_a = begin; - char *end_a = end; - int i; - - for (i = 0; i < ARRAYELTS (suspicious_objects); ++i) - { - char *suspicious_object = suspicious_objects[i]; - if (begin_a <= suspicious_object && suspicious_object < end_a) - return suspicious_object; - } - - return NULL; -} - -static void -note_suspicious_free (void *ptr) -{ - struct suspicious_free_record *rec; - - rec = &suspicious_free_history[suspicious_free_history_index++]; - if (suspicious_free_history_index == - ARRAYELTS (suspicious_free_history)) - { - suspicious_free_history_index = 0; - } - - memset (rec, 0, sizeof (*rec)); - rec->suspicious_object = ptr; - backtrace (&rec->backtrace[0], ARRAYELTS (rec->backtrace)); -} - -static void -detect_suspicious_free (void *ptr) -{ - int i; - - eassert (ptr != NULL); - - for (i = 0; i < ARRAYELTS (suspicious_objects); ++i) - if (suspicious_objects[i] == ptr) - { - note_suspicious_free (ptr); - suspicious_objects[i] = NULL; - } -} - -#endif /* SUSPICIOUS_OBJECT_CHECKING */ - -DEFUN ("suspicious-object", Fsuspicious_object, Ssuspicious_object, 1, 1, 0, - doc: /* Return OBJ, maybe marking it for extra scrutiny. -If Emacs is compiled with suspicious object checking, capture -a stack trace when OBJ is freed in order to help track down -garbage collection bugs. Otherwise, do nothing and return OBJ. */) - (Lisp_Object obj) -{ -#ifdef SUSPICIOUS_OBJECT_CHECKING - /* Right now, we care only about vectors. */ - if (VECTORLIKEP (obj)) - { - suspicious_objects[suspicious_object_index++] = XVECTOR (obj); - if (suspicious_object_index == ARRAYELTS (suspicious_objects)) - suspicious_object_index = 0; - } -#endif - return obj; -} - #ifdef ENABLE_CHECKING bool suppress_checking; @@ -7926,7 +8298,6 @@ N should be nonnegative. */); #ifdef HAVE_MALLOC_TRIM defsubr (&Smalloc_trim); #endif - defsubr (&Ssuspicious_object); Lisp_Object watcher; |