1 : /* -*- Mode: C; tab-width: 8; c-basic-offset: 8; indent-tabs-mode: t -*- */
2 : /* vim:set softtabstop=8 shiftwidth=8 noet: */
3 : /*-
4 : * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
5 : * All rights reserved.
6 : *
7 : * Redistribution and use in source and binary forms, with or without
8 : * modification, are permitted provided that the following conditions
9 : * are met:
10 : * 1. Redistributions of source code must retain the above copyright
11 : * notice(s), this list of conditions and the following disclaimer as
12 : * the first lines of this file unmodified other than the possible
13 : * addition of one or more copyright notices.
14 : * 2. Redistributions in binary form must reproduce the above copyright
15 : * notice(s), this list of conditions and the following disclaimer in
16 : * the documentation and/or other materials provided with the
17 : * distribution.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
20 : * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 : * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
26 : * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 : * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
28 : * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 : * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : *******************************************************************************
32 : *
33 : * This allocator implementation is designed to provide scalable performance
34 : * for multi-threaded programs on multi-processor systems. The following
35 : * features are included for this purpose:
36 : *
37 : * + Multiple arenas are used if there are multiple CPUs, which reduces lock
38 : * contention and cache sloshing.
39 : *
40 : * + Cache line sharing between arenas is avoided for internal data
41 : * structures.
42 : *
43 : * + Memory is managed in chunks and runs (chunks can be split into runs),
44 : * rather than as individual pages. This provides a constant-time
45 : * mechanism for associating allocations with particular arenas.
46 : *
47 : * Allocation requests are rounded up to the nearest size class, and no record
48 : * of the original request size is maintained. Allocations are broken into
49 : * categories according to size class. Assuming runtime defaults, 4 kB pages
50 : * and a 16 byte quantum on a 32-bit system, the size classes in each category
51 : * are as follows:
52 : *
53 : * |=====================================|
54 : * | Category | Subcategory | Size |
55 : * |=====================================|
56 : * | Small | Tiny | 2 |
57 : * | | | 4 |
58 : * | | | 8 |
59 : * | |----------------+---------|
60 : * | | Quantum-spaced | 16 |
61 : * | | | 32 |
62 : * | | | 48 |
63 : * | | | ... |
64 : * | | | 480 |
65 : * | | | 496 |
66 : * | | | 512 |
67 : * | |----------------+---------|
68 : * | | Sub-page | 1 kB |
69 : * | | | 2 kB |
70 : * |=====================================|
71 : * | Large | 4 kB |
72 : * | | 8 kB |
73 : * | | 12 kB |
74 : * | | ... |
75 : * | | 1012 kB |
76 : * | | 1016 kB |
77 : * | | 1020 kB |
78 : * |=====================================|
79 : * | Huge | 1 MB |
80 : * | | 2 MB |
81 : * | | 3 MB |
82 : * | | ... |
83 : * |=====================================|
84 : *
85 : * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an
86 : * allocation on Linux or Mac. So on 32-bit *nix, the smallest bucket size is
87 : * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes.
88 : *
89 : * A different mechanism is used for each category:
90 : *
91 : * Small : Each size class is segregated into its own set of runs. Each run
92 : * maintains a bitmap of which regions are free/allocated.
93 : *
94 : * Large : Each allocation is backed by a dedicated run. Metadata are stored
95 : * in the associated arena chunk header maps.
96 : *
97 : * Huge : Each allocation is backed by a dedicated contiguous set of chunks.
98 : * Metadata are stored in a separate red-black tree.
99 : *
100 : *******************************************************************************
101 : */
102 :
103 : #ifdef MOZ_MEMORY_ANDROID
104 : #define NO_TLS
105 : #define _pthread_self() pthread_self()
106 : #endif
107 :
108 : /*
109 : * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the
110 : * operating system. If we release 1MB of live pages with MADV_DONTNEED, our
111 : * RSS will decrease by 1MB (almost) immediately.
112 : *
113 : * On Mac, we use madvise(MADV_FREE). Unlike MADV_DONTNEED on Linux, MADV_FREE
114 : * on Mac doesn't cause the OS to release the specified pages immediately; the
115 : * OS keeps them in our process until the machine comes under memory pressure.
116 : *
117 : * It's therefore difficult to measure the process's RSS on Mac, since, in the
118 : * absence of memory pressure, the contribution from the heap to RSS will not
119 : * decrease due to our madvise calls.
120 : *
121 : * We therefore define MALLOC_DOUBLE_PURGE on Mac. This causes jemalloc to
122 : * track which pages have been MADV_FREE'd. You can then call
123 : * jemalloc_purge_freed_pages(), which will force the OS to release those
124 : * MADV_FREE'd pages, making the process's RSS reflect its true memory usage.
125 : *
126 : * The jemalloc_purge_freed_pages definition in jemalloc.h needs to be
127 : * adjusted if MALLOC_DOUBLE_PURGE is ever enabled on Linux.
128 : */
129 : #ifdef MOZ_MEMORY_DARWIN
130 : #define MALLOC_DOUBLE_PURGE
131 : #endif
132 :
133 : /*
134 : * MALLOC_PRODUCTION disables assertions and statistics gathering. It also
135 : * defaults the A and J runtime options to off. These settings are appropriate
136 : * for production systems.
137 : */
138 : #ifndef MOZ_MEMORY_DEBUG
139 : # define MALLOC_PRODUCTION
140 : #endif
141 :
142 : /*
143 : * Use only one arena by default. Mozilla does not currently make extensive
144 : * use of concurrent allocation, so the increased fragmentation associated with
145 : * multiple arenas is not warranted.
146 : */
147 : #define MOZ_MEMORY_NARENAS_DEFAULT_ONE
148 :
149 : /*
150 : * MALLOC_STATS enables statistics calculation, and is required for
151 : * jemalloc_stats().
152 : */
153 : #define MALLOC_STATS
154 :
155 : #ifndef MALLOC_PRODUCTION
156 : /*
157 : * MALLOC_DEBUG enables assertions and other sanity checks, and disables
158 : * inline functions.
159 : */
160 : # define MALLOC_DEBUG
161 :
162 : /* Memory filling (junk/zero). */
163 : # define MALLOC_FILL
164 :
165 : /* Allocation tracing. */
166 : # ifndef MOZ_MEMORY_WINDOWS
167 : # define MALLOC_UTRACE
168 : # endif
169 :
170 : /* Support optional abort() on OOM. */
171 : # define MALLOC_XMALLOC
172 :
173 : /* Support SYSV semantics. */
174 : # define MALLOC_SYSV
175 : #endif
176 :
177 : /*
178 : * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
179 : * validation. There are many possible errors that validation does not even
180 : * attempt to detect.
181 : */
182 : #define MALLOC_VALIDATE
183 :
184 : /* Embed no-op macros that support memory allocation tracking via valgrind. */
185 : #ifdef MOZ_VALGRIND
186 : # define MALLOC_VALGRIND
187 : #endif
188 : #ifdef MALLOC_VALGRIND
189 : # include <valgrind/valgrind.h>
190 : #else
191 : # define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
192 : # define VALGRIND_FREELIKE_BLOCK(addr, rzB)
193 : #endif
194 :
195 : /*
196 : * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
197 : * re-balances arena load if exponentially averaged contention exceeds a
198 : * certain threshold.
199 : */
200 : /* #define MALLOC_BALANCE */
201 :
202 : /*
203 : * MALLOC_PAGEFILE causes all mmap()ed memory to be backed by temporary
204 : * files, so that if a chunk is mapped, it is guaranteed to be swappable.
205 : * This avoids asynchronous OOM failures that are due to VM over-commit.
206 : */
207 : /* #define MALLOC_PAGEFILE */
208 :
209 : #ifdef MALLOC_PAGEFILE
210 : /* Write size when initializing a page file. */
211 : # define MALLOC_PAGEFILE_WRITE_SIZE 512
212 : #endif
213 :
214 : #if defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
215 : #define _GNU_SOURCE /* For mremap(2). */
216 : #define issetugid() 0
217 : #if 0 /* Enable in order to test decommit code on Linux. */
218 : # define MALLOC_DECOMMIT
219 : #endif
220 : #endif
221 :
222 : #include <sys/types.h>
223 :
224 : #include <errno.h>
225 : #include <stdlib.h>
226 : #include <limits.h>
227 : #include <stdarg.h>
228 : #include <stdio.h>
229 : #include <string.h>
230 :
231 : #ifdef MOZ_MEMORY_WINDOWS
232 :
233 : /* Some defines from the CRT internal headers that we need here. */
234 : #define _CRT_SPINCOUNT 5000
235 : #define __crtInitCritSecAndSpinCount InitializeCriticalSectionAndSpinCount
236 : #include <io.h>
237 : #include <windows.h>
238 :
239 : #pragma warning( disable: 4267 4996 4146 )
240 :
241 : #define bool BOOL
242 : #define false FALSE
243 : #define true TRUE
244 : #define inline __inline
245 : #define SIZE_T_MAX SIZE_MAX
246 : #define STDERR_FILENO 2
247 : #define PATH_MAX MAX_PATH
248 : #define vsnprintf _vsnprintf
249 :
250 : #ifndef NO_TLS
251 : static unsigned long tlsIndex = 0xffffffff;
252 : #endif
253 :
254 : #define __thread
255 : #define _pthread_self() __threadid()
256 : #define issetugid() 0
257 :
258 : /* use MSVC intrinsics */
259 : #pragma intrinsic(_BitScanForward)
260 : static __forceinline int
261 : ffs(int x)
262 : {
263 : unsigned long i;
264 :
265 : if (_BitScanForward(&i, x) != 0)
266 : return (i + 1);
267 :
268 : return (0);
269 : }
270 :
271 : /* Implement getenv without using malloc */
272 : static char mozillaMallocOptionsBuf[64];
273 :
274 : #define getenv xgetenv
275 : static char *
276 : getenv(const char *name)
277 : {
278 :
279 : if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
280 : sizeof(mozillaMallocOptionsBuf)) > 0)
281 : return (mozillaMallocOptionsBuf);
282 :
283 : return (NULL);
284 : }
285 :
286 : typedef unsigned char uint8_t;
287 : typedef unsigned uint32_t;
288 : typedef unsigned long long uint64_t;
289 : typedef unsigned long long uintmax_t;
290 : #if defined(MOZ_MEMORY_SIZEOF_PTR_2POW) && (MOZ_MEMORY_SIZEOF_PTR_2POW == 3)
291 : typedef long long ssize_t;
292 : #else
293 : typedef long ssize_t;
294 : #endif
295 :
296 : #define MALLOC_DECOMMIT
297 : #endif
298 :
299 : #ifndef MOZ_MEMORY_WINDOWS
300 : #ifndef MOZ_MEMORY_SOLARIS
301 : #include <sys/cdefs.h>
302 : #endif
303 : #ifndef __DECONST
304 : # define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
305 : #endif
306 : #ifndef MOZ_MEMORY
307 : __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
308 : #include "libc_private.h"
309 : #ifdef MALLOC_DEBUG
310 : # define _LOCK_DEBUG
311 : #endif
312 : #include "spinlock.h"
313 : #include "namespace.h"
314 : #endif
315 : #include <sys/mman.h>
316 : #ifndef MADV_FREE
317 : # define MADV_FREE MADV_DONTNEED
318 : #endif
319 : #ifndef MAP_NOSYNC
320 : # define MAP_NOSYNC 0
321 : #endif
322 : #include <sys/param.h>
323 : #ifndef MOZ_MEMORY
324 : #include <sys/stddef.h>
325 : #endif
326 : #include <sys/time.h>
327 : #include <sys/types.h>
328 : #if !defined(MOZ_MEMORY_SOLARIS) && !defined(MOZ_MEMORY_ANDROID)
329 : #include <sys/sysctl.h>
330 : #endif
331 : #include <sys/uio.h>
332 : #ifndef MOZ_MEMORY
333 : #include <sys/ktrace.h> /* Must come after several other sys/ includes. */
334 :
335 : #include <machine/atomic.h>
336 : #include <machine/cpufunc.h>
337 : #include <machine/vmparam.h>
338 : #endif
339 :
340 : #include <errno.h>
341 : #include <limits.h>
342 : #ifndef SIZE_T_MAX
343 : # define SIZE_T_MAX SIZE_MAX
344 : #endif
345 : #include <pthread.h>
346 : #ifdef MOZ_MEMORY_DARWIN
347 : #define _pthread_self pthread_self
348 : #define _pthread_mutex_init pthread_mutex_init
349 : #define _pthread_mutex_trylock pthread_mutex_trylock
350 : #define _pthread_mutex_lock pthread_mutex_lock
351 : #define _pthread_mutex_unlock pthread_mutex_unlock
352 : #endif
353 : #include <sched.h>
354 : #include <stdarg.h>
355 : #include <stdio.h>
356 : #include <stdbool.h>
357 : #include <stdint.h>
358 : #include <stdlib.h>
359 : #include <string.h>
360 : #ifndef MOZ_MEMORY_DARWIN
361 : #include <strings.h>
362 : #endif
363 : #include <unistd.h>
364 :
365 : #ifdef MOZ_MEMORY_DARWIN
366 : #include <libkern/OSAtomic.h>
367 : #include <mach/mach_error.h>
368 : #include <mach/mach_init.h>
369 : #include <mach/vm_map.h>
370 : #include <malloc/malloc.h>
371 : #endif
372 :
373 : #ifndef MOZ_MEMORY
374 : #include "un-namespace.h"
375 : #endif
376 :
377 : #endif
378 :
379 : #include "jemalloc_types.h"
380 : #include "linkedlist.h"
381 :
382 : /* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that
383 : * happen to override mmap() and call dlsym() from their overridden
384 : * mmap(). The problem is that dlsym() calls malloc(), and this ends
385 : * up in a dead lock in jemalloc.
386 : * On these systems, we prefer to directly use the system call.
387 : * We do that for Linux systems and kfreebsd with GNU userland.
388 : * Note sanity checks are not done (alignment of offset, ...) because
389 : * the uses of mmap are pretty limited, in jemalloc.
390 : *
391 : * On Alpha, glibc has a bug that prevents syscall() to work for system
392 : * calls with 6 arguments
393 : */
394 : #if (defined(MOZ_MEMORY_LINUX) && !defined(__alpha__)) || \
395 : (defined(MOZ_MEMORY_BSD) && defined(__GLIBC__))
396 : #include <sys/syscall.h>
397 : #if defined(SYS_mmap) || defined(SYS_mmap2)
398 : static inline
399 64774 : void *_mmap(void *addr, size_t length, int prot, int flags,
400 : int fd, off_t offset)
401 : {
402 : /* S390 only passes one argument to the mmap system call, which is a
403 : * pointer to a structure containing the arguments */
404 : #ifdef __s390__
405 : struct {
406 : void *addr;
407 : size_t length;
408 : int prot;
409 : int flags;
410 : int fd;
411 : off_t offset;
412 : } args = { addr, length, prot, flags, fd, offset };
413 : return (void *) syscall(SYS_mmap, &args);
414 : #else
415 : #ifdef SYS_mmap2
416 64774 : return (void *) syscall(SYS_mmap2, addr, length, prot, flags,
417 : fd, offset >> 12);
418 : #else
419 : return (void *) syscall(SYS_mmap, addr, length, prot, flags,
420 : fd, offset);
421 : #endif
422 : #endif
423 : }
424 : #define mmap _mmap
425 : #define munmap(a, l) syscall(SYS_munmap, a, l)
426 : #endif
427 : #endif
428 :
429 : #ifdef MOZ_MEMORY_DARWIN
430 : static const bool isthreaded = true;
431 : #endif
432 :
433 : #if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
434 : #define JEMALLOC_USES_MAP_ALIGN /* Required on Solaris 10. Might improve performance elsewhere. */
435 : #endif
436 :
437 : #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
438 :
439 : #ifdef MOZ_MEMORY_WINDOWS
440 : /* MSVC++ does not support C99 variable-length arrays. */
441 : # define RB_NO_C99_VARARRAYS
442 : #endif
443 : #include "rb.h"
444 :
445 : #ifdef MALLOC_DEBUG
446 : /* Disable inlining to make debugging easier. */
447 : #ifdef inline
448 : #undef inline
449 : #endif
450 :
451 : # define inline
452 : #endif
453 :
454 : /* Size of stack-allocated buffer passed to strerror_r(). */
455 : #define STRERROR_BUF 64
456 :
457 : /* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */
458 : # define QUANTUM_2POW_MIN 4
459 : #ifdef MOZ_MEMORY_SIZEOF_PTR_2POW
460 : # define SIZEOF_PTR_2POW MOZ_MEMORY_SIZEOF_PTR_2POW
461 : #else
462 : # define SIZEOF_PTR_2POW 2
463 : #endif
464 : #define PIC
465 : #ifndef MOZ_MEMORY_DARWIN
466 : static const bool isthreaded = true;
467 : #else
468 : # define NO_TLS
469 : #endif
470 : #if 0
471 : #ifdef __i386__
472 : # define QUANTUM_2POW_MIN 4
473 : # define SIZEOF_PTR_2POW 2
474 : # define CPU_SPINWAIT __asm__ volatile("pause")
475 : #endif
476 : #ifdef __ia64__
477 : # define QUANTUM_2POW_MIN 4
478 : # define SIZEOF_PTR_2POW 3
479 : #endif
480 : #ifdef __alpha__
481 : # define QUANTUM_2POW_MIN 4
482 : # define SIZEOF_PTR_2POW 3
483 : # define NO_TLS
484 : #endif
485 : #ifdef __sparc64__
486 : # define QUANTUM_2POW_MIN 4
487 : # define SIZEOF_PTR_2POW 3
488 : # define NO_TLS
489 : #endif
490 : #ifdef __amd64__
491 : # define QUANTUM_2POW_MIN 4
492 : # define SIZEOF_PTR_2POW 3
493 : # define CPU_SPINWAIT __asm__ volatile("pause")
494 : #endif
495 : #ifdef __arm__
496 : # define QUANTUM_2POW_MIN 3
497 : # define SIZEOF_PTR_2POW 2
498 : # define NO_TLS
499 : #endif
500 : #ifdef __mips__
501 : # define QUANTUM_2POW_MIN 3
502 : # define SIZEOF_PTR_2POW 2
503 : # define NO_TLS
504 : #endif
505 : #ifdef __powerpc__
506 : # define QUANTUM_2POW_MIN 4
507 : # define SIZEOF_PTR_2POW 2
508 : #endif
509 : #endif
510 :
511 : #define SIZEOF_PTR (1U << SIZEOF_PTR_2POW)
512 :
513 : /* sizeof(int) == (1U << SIZEOF_INT_2POW). */
514 : #ifndef SIZEOF_INT_2POW
515 : # define SIZEOF_INT_2POW 2
516 : #endif
517 :
518 : /* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
519 : #if (!defined(PIC) && !defined(NO_TLS))
520 : # define NO_TLS
521 : #endif
522 :
523 : #ifdef NO_TLS
524 : /* MALLOC_BALANCE requires TLS. */
525 : # ifdef MALLOC_BALANCE
526 : # undef MALLOC_BALANCE
527 : # endif
528 : #endif
529 :
530 : /*
531 : * Size and alignment of memory chunks that are allocated by the OS's virtual
532 : * memory system.
533 : */
534 : #define CHUNK_2POW_DEFAULT 20
535 : /* Maximum number of dirty pages per arena. */
536 : #define DIRTY_MAX_DEFAULT (1U << 10)
537 :
538 : /*
539 : * Maximum size of L1 cache line. This is used to avoid cache line aliasing,
540 : * so over-estimates are okay (up to a point), but under-estimates will
541 : * negatively affect performance.
542 : */
543 : #define CACHELINE_2POW 6
544 : #define CACHELINE ((size_t)(1U << CACHELINE_2POW))
545 :
546 : /*
547 : * Smallest size class to support. On Linux and Mac, even malloc(1) must
548 : * reserve a word's worth of memory (see Mozilla bug 691003).
549 : */
550 : #ifdef MOZ_MEMORY_WINDOWS
551 : #define TINY_MIN_2POW 1
552 : #else
553 : #define TINY_MIN_2POW (sizeof(void*) == 8 ? 3 : 2)
554 : #endif
555 :
556 : /*
557 : * Maximum size class that is a multiple of the quantum, but not (necessarily)
558 : * a power of 2. Above this size, allocations are rounded up to the nearest
559 : * power of 2.
560 : */
561 : #define SMALL_MAX_2POW_DEFAULT 9
562 : #define SMALL_MAX_DEFAULT (1U << SMALL_MAX_2POW_DEFAULT)
563 :
564 : /*
565 : * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized
566 : * as small as possible such that this setting is still honored, without
567 : * violating other constraints. The goal is to make runs as small as possible
568 : * without exceeding a per run external fragmentation threshold.
569 : *
570 : * We use binary fixed point math for overhead computations, where the binary
571 : * point is implicitly RUN_BFP bits to the left.
572 : *
573 : * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
574 : * honored for some/all object sizes, since there is one bit of header overhead
575 : * per object (plus a constant). This constraint is relaxed (ignored) for runs
576 : * that are so small that the per-region overhead is greater than:
577 : *
578 : * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
579 : */
580 : #define RUN_BFP 12
581 : /* \/ Implicit binary fixed point. */
582 : #define RUN_MAX_OVRHD 0x0000003dU
583 : #define RUN_MAX_OVRHD_RELAX 0x00001800U
584 :
585 : /* Put a cap on small object run size. This overrides RUN_MAX_OVRHD. */
586 : #define RUN_MAX_SMALL_2POW 15
587 : #define RUN_MAX_SMALL (1U << RUN_MAX_SMALL_2POW)
588 :
589 : /*
590 : * Hyper-threaded CPUs may need a special instruction inside spin loops in
591 : * order to yield to another virtual CPU. If no such instruction is defined
592 : * above, make CPU_SPINWAIT a no-op.
593 : */
594 : #ifndef CPU_SPINWAIT
595 : # define CPU_SPINWAIT
596 : #endif
597 :
598 : /*
599 : * Adaptive spinning must eventually switch to blocking, in order to avoid the
600 : * potential for priority inversion deadlock. Backing off past a certain point
601 : * can actually waste time.
602 : */
603 : #define SPIN_LIMIT_2POW 11
604 :
605 : /*
606 : * Conversion from spinning to blocking is expensive; we use (1U <<
607 : * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
608 : * worst-case spinning.
609 : */
610 : #define BLOCK_COST_2POW 4
611 :
612 : #ifdef MALLOC_BALANCE
613 : /*
614 : * We use an exponential moving average to track recent lock contention,
615 : * where the size of the history window is N, and alpha=2/(N+1).
616 : *
617 : * Due to integer math rounding, very small values here can cause
618 : * substantial degradation in accuracy, thus making the moving average decay
619 : * faster than it would with precise calculation.
620 : */
621 : # define BALANCE_ALPHA_INV_2POW 9
622 :
623 : /*
624 : * Threshold value for the exponential moving contention average at which to
625 : * re-assign a thread.
626 : */
627 : # define BALANCE_THRESHOLD_DEFAULT (1U << (SPIN_LIMIT_2POW-4))
628 : #endif
629 :
630 : /******************************************************************************/
631 :
632 : /* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */
633 : #if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE)
634 : #error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive.
635 : #endif
636 :
637 : /*
638 : * Mutexes based on spinlocks. We can't use normal pthread spinlocks in all
639 : * places, because they require malloc()ed memory, which causes bootstrapping
640 : * issues in some cases.
641 : */
642 : #if defined(MOZ_MEMORY_WINDOWS)
643 : #define malloc_mutex_t CRITICAL_SECTION
644 : #define malloc_spinlock_t CRITICAL_SECTION
645 : #elif defined(MOZ_MEMORY_DARWIN)
646 : typedef struct {
647 : OSSpinLock lock;
648 : } malloc_mutex_t;
649 : typedef struct {
650 : OSSpinLock lock;
651 : } malloc_spinlock_t;
652 : #elif defined(MOZ_MEMORY)
653 : typedef pthread_mutex_t malloc_mutex_t;
654 : typedef pthread_mutex_t malloc_spinlock_t;
655 : #else
656 : /* XXX these should #ifdef these for freebsd (and linux?) only */
657 : typedef struct {
658 : spinlock_t lock;
659 : } malloc_mutex_t;
660 : typedef malloc_spinlock_t malloc_mutex_t;
661 : #endif
662 :
663 : /* Set to true once the allocator has been initialized. */
664 : static bool malloc_initialized = false;
665 :
666 : #if defined(MOZ_MEMORY_WINDOWS)
667 : /* No init lock for Windows. */
668 : #elif defined(MOZ_MEMORY_DARWIN)
669 : static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
670 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
671 : static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
672 : #elif defined(MOZ_MEMORY)
673 : static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
674 : #else
675 : static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
676 : #endif
677 :
678 : /******************************************************************************/
679 : /*
680 : * Statistics data structures.
681 : */
682 :
683 : #ifdef MALLOC_STATS
684 :
685 : typedef struct malloc_bin_stats_s malloc_bin_stats_t;
686 : struct malloc_bin_stats_s {
687 : /*
688 : * Number of allocation requests that corresponded to the size of this
689 : * bin.
690 : */
691 : uint64_t nrequests;
692 :
693 : /* Total number of runs created for this bin's size class. */
694 : uint64_t nruns;
695 :
696 : /*
697 : * Total number of runs reused by extracting them from the runs tree for
698 : * this bin's size class.
699 : */
700 : uint64_t reruns;
701 :
702 : /* High-water mark for this bin. */
703 : unsigned long highruns;
704 :
705 : /* Current number of runs in this bin. */
706 : unsigned long curruns;
707 : };
708 :
709 : typedef struct arena_stats_s arena_stats_t;
710 : struct arena_stats_s {
711 : /* Number of bytes currently mapped. */
712 : size_t mapped;
713 :
714 : /*
715 : * Total number of purge sweeps, total number of madvise calls made,
716 : * and total pages purged in order to keep dirty unused memory under
717 : * control.
718 : */
719 : uint64_t npurge;
720 : uint64_t nmadvise;
721 : uint64_t purged;
722 : #ifdef MALLOC_DECOMMIT
723 : /*
724 : * Total number of decommit/commit operations, and total number of
725 : * pages decommitted.
726 : */
727 : uint64_t ndecommit;
728 : uint64_t ncommit;
729 : uint64_t decommitted;
730 : #endif
731 :
732 : /* Current number of committed pages. */
733 : size_t committed;
734 :
735 : /* Per-size-category statistics. */
736 : size_t allocated_small;
737 : uint64_t nmalloc_small;
738 : uint64_t ndalloc_small;
739 :
740 : size_t allocated_large;
741 : uint64_t nmalloc_large;
742 : uint64_t ndalloc_large;
743 :
744 : #ifdef MALLOC_BALANCE
745 : /* Number of times this arena reassigned a thread due to contention. */
746 : uint64_t nbalance;
747 : #endif
748 : };
749 :
750 : typedef struct chunk_stats_s chunk_stats_t;
751 : struct chunk_stats_s {
752 : /* Number of chunks that were allocated. */
753 : uint64_t nchunks;
754 :
755 : /* High-water mark for number of chunks allocated. */
756 : unsigned long highchunks;
757 :
758 : /*
759 : * Current number of chunks allocated. This value isn't maintained for
760 : * any other purpose, so keep track of it in order to be able to set
761 : * highchunks.
762 : */
763 : unsigned long curchunks;
764 : };
765 :
766 : #endif /* #ifdef MALLOC_STATS */
767 :
768 : /******************************************************************************/
769 : /*
770 : * Extent data structures.
771 : */
772 :
773 : /* Tree of extents. */
774 : typedef struct extent_node_s extent_node_t;
775 : struct extent_node_s {
776 : /* Linkage for the size/address-ordered tree. */
777 : rb_node(extent_node_t) link_szad;
778 :
779 : /* Linkage for the address-ordered tree. */
780 : rb_node(extent_node_t) link_ad;
781 :
782 : /* Pointer to the extent that this tree node is responsible for. */
783 : void *addr;
784 :
785 : /* Total region size. */
786 : size_t size;
787 : };
788 : typedef rb_tree(extent_node_t) extent_tree_t;
789 :
790 : /******************************************************************************/
791 : /*
792 : * Radix tree data structures.
793 : */
794 :
795 : #ifdef MALLOC_VALIDATE
796 : /*
797 : * Size of each radix tree node (must be a power of 2). This impacts tree
798 : * depth.
799 : */
800 : # if (SIZEOF_PTR == 4)
801 : # define MALLOC_RTREE_NODESIZE (1U << 14)
802 : # else
803 : # define MALLOC_RTREE_NODESIZE CACHELINE
804 : # endif
805 :
806 : typedef struct malloc_rtree_s malloc_rtree_t;
807 : struct malloc_rtree_s {
808 : malloc_spinlock_t lock;
809 : void **root;
810 : unsigned height;
811 : unsigned level2bits[1]; /* Dynamically sized. */
812 : };
813 : #endif
814 :
815 : /******************************************************************************/
816 : /*
817 : * Arena data structures.
818 : */
819 :
820 : typedef struct arena_s arena_t;
821 : typedef struct arena_bin_s arena_bin_t;
822 :
823 : /* Each element of the chunk map corresponds to one page within the chunk. */
824 : typedef struct arena_chunk_map_s arena_chunk_map_t;
825 : struct arena_chunk_map_s {
826 : /*
827 : * Linkage for run trees. There are two disjoint uses:
828 : *
829 : * 1) arena_t's runs_avail tree.
830 : * 2) arena_run_t conceptually uses this linkage for in-use non-full
831 : * runs, rather than directly embedding linkage.
832 : */
833 : rb_node(arena_chunk_map_t) link;
834 :
835 : /*
836 : * Run address (or size) and various flags are stored together. The bit
837 : * layout looks like (assuming 32-bit system):
838 : *
839 : * ???????? ???????? ????---- -mckdzla
840 : *
841 : * ? : Unallocated: Run address for first/last pages, unset for internal
842 : * pages.
843 : * Small: Run address.
844 : * Large: Run size for first page, unset for trailing pages.
845 : * - : Unused.
846 : * m : MADV_FREE/MADV_DONTNEED'ed?
847 : * c : decommitted?
848 : * k : key?
849 : * d : dirty?
850 : * z : zeroed?
851 : * l : large?
852 : * a : allocated?
853 : *
854 : * Following are example bit patterns for the three types of runs.
855 : *
856 : * r : run address
857 : * s : run size
858 : * x : don't care
859 : * - : 0
860 : * [cdzla] : bit set
861 : *
862 : * Unallocated:
863 : * ssssssss ssssssss ssss---- --c-----
864 : * xxxxxxxx xxxxxxxx xxxx---- ----d---
865 : * ssssssss ssssssss ssss---- -----z--
866 : *
867 : * Small:
868 : * rrrrrrrr rrrrrrrr rrrr---- -------a
869 : * rrrrrrrr rrrrrrrr rrrr---- -------a
870 : * rrrrrrrr rrrrrrrr rrrr---- -------a
871 : *
872 : * Large:
873 : * ssssssss ssssssss ssss---- ------la
874 : * -------- -------- -------- ------la
875 : * -------- -------- -------- ------la
876 : */
877 : size_t bits;
878 :
879 : /* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether
880 : * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined.
881 : *
882 : * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be
883 : * re-committed with pages_commit() before it may be touched. If
884 : * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined.
885 : *
886 : * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which
887 : * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with
888 : * CHUNK_MAP_MADVISED.
889 : *
890 : * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is
891 : * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED.
892 : * When it's finally freed with jemalloc_purge_freed_pages, the page is marked
893 : * as CHUNK_MAP_DECOMMITTED.
894 : */
895 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
896 : #define CHUNK_MAP_MADVISED ((size_t)0x40U)
897 : #define CHUNK_MAP_DECOMMITTED ((size_t)0x20U)
898 : #define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED)
899 : #endif
900 : #define CHUNK_MAP_KEY ((size_t)0x10U)
901 : #define CHUNK_MAP_DIRTY ((size_t)0x08U)
902 : #define CHUNK_MAP_ZEROED ((size_t)0x04U)
903 : #define CHUNK_MAP_LARGE ((size_t)0x02U)
904 : #define CHUNK_MAP_ALLOCATED ((size_t)0x01U)
905 : };
906 : typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
907 : typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
908 :
909 : /* Arena chunk header. */
910 : typedef struct arena_chunk_s arena_chunk_t;
911 : struct arena_chunk_s {
912 : /* Arena that owns the chunk. */
913 : arena_t *arena;
914 :
915 : /* Linkage for the arena's chunks_dirty tree. */
916 : rb_node(arena_chunk_t) link_dirty;
917 :
918 : #ifdef MALLOC_DOUBLE_PURGE
919 : /* If we're double-purging, we maintain a linked list of chunks which
920 : * have pages which have been madvise(MADV_FREE)'d but not explicitly
921 : * purged.
922 : *
923 : * We're currently lazy and don't remove a chunk from this list when
924 : * all its madvised pages are recommitted. */
925 : LinkedList chunks_madvised_elem;
926 : #endif
927 :
928 : /* Number of dirty pages. */
929 : size_t ndirty;
930 :
931 : /* Map of pages within chunk that keeps track of free/large/small. */
932 : arena_chunk_map_t map[1]; /* Dynamically sized. */
933 : };
934 : typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
935 :
936 : typedef struct arena_run_s arena_run_t;
937 : struct arena_run_s {
938 : #ifdef MALLOC_DEBUG
939 : uint32_t magic;
940 : # define ARENA_RUN_MAGIC 0x384adf93
941 : #endif
942 :
943 : /* Bin this run is associated with. */
944 : arena_bin_t *bin;
945 :
946 : /* Index of first element that might have a free region. */
947 : unsigned regs_minelm;
948 :
949 : /* Number of free regions in run. */
950 : unsigned nfree;
951 :
952 : /* Bitmask of in-use regions (0: in use, 1: free). */
953 : unsigned regs_mask[1]; /* Dynamically sized. */
954 : };
955 :
956 : struct arena_bin_s {
957 : /*
958 : * Current run being used to service allocations of this bin's size
959 : * class.
960 : */
961 : arena_run_t *runcur;
962 :
963 : /*
964 : * Tree of non-full runs. This tree is used when looking for an
965 : * existing run when runcur is no longer usable. We choose the
966 : * non-full run that is lowest in memory; this policy tends to keep
967 : * objects packed well, and it can also help reduce the number of
968 : * almost-empty chunks.
969 : */
970 : arena_run_tree_t runs;
971 :
972 : /* Size of regions in a run for this bin's size class. */
973 : size_t reg_size;
974 :
975 : /* Total size of a run for this bin's size class. */
976 : size_t run_size;
977 :
978 : /* Total number of regions in a run for this bin's size class. */
979 : uint32_t nregs;
980 :
981 : /* Number of elements in a run's regs_mask for this bin's size class. */
982 : uint32_t regs_mask_nelms;
983 :
984 : /* Offset of first region in a run for this bin's size class. */
985 : uint32_t reg0_offset;
986 :
987 : #ifdef MALLOC_STATS
988 : /* Bin statistics. */
989 : malloc_bin_stats_t stats;
990 : #endif
991 : };
992 :
993 : struct arena_s {
994 : #ifdef MALLOC_DEBUG
995 : uint32_t magic;
996 : # define ARENA_MAGIC 0x947d3d24
997 : #endif
998 :
999 : /* All operations on this arena require that lock be locked. */
1000 : #ifdef MOZ_MEMORY
1001 : malloc_spinlock_t lock;
1002 : #else
1003 : pthread_mutex_t lock;
1004 : #endif
1005 :
1006 : #ifdef MALLOC_STATS
1007 : arena_stats_t stats;
1008 : #endif
1009 :
1010 : /* Tree of dirty-page-containing chunks this arena manages. */
1011 : arena_chunk_tree_t chunks_dirty;
1012 :
1013 : #ifdef MALLOC_DOUBLE_PURGE
1014 : /* Head of a linked list of MADV_FREE'd-page-containing chunks this
1015 : * arena manages. */
1016 : LinkedList chunks_madvised;
1017 : #endif
1018 :
1019 : /*
1020 : * In order to avoid rapid chunk allocation/deallocation when an arena
1021 : * oscillates right on the cusp of needing a new chunk, cache the most
1022 : * recently freed chunk. The spare is left in the arena's chunk trees
1023 : * until it is deleted.
1024 : *
1025 : * There is one spare chunk per arena, rather than one spare total, in
1026 : * order to avoid interactions between multiple threads that could make
1027 : * a single spare inadequate.
1028 : */
1029 : arena_chunk_t *spare;
1030 :
1031 : /*
1032 : * Current count of pages within unused runs that are potentially
1033 : * dirty, and for which madvise(... MADV_FREE) has not been called. By
1034 : * tracking this, we can institute a limit on how much dirty unused
1035 : * memory is mapped for each arena.
1036 : */
1037 : size_t ndirty;
1038 :
1039 : /*
1040 : * Size/address-ordered tree of this arena's available runs. This tree
1041 : * is used for first-best-fit run allocation.
1042 : */
1043 : arena_avail_tree_t runs_avail;
1044 :
1045 : #ifdef MALLOC_BALANCE
1046 : /*
1047 : * The arena load balancing machinery needs to keep track of how much
1048 : * lock contention there is. This value is exponentially averaged.
1049 : */
1050 : uint32_t contention;
1051 : #endif
1052 :
1053 : /*
1054 : * bins is used to store rings of free regions of the following sizes,
1055 : * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
1056 : *
1057 : * bins[i] | size |
1058 : * --------+------+
1059 : * 0 | 2 |
1060 : * 1 | 4 |
1061 : * 2 | 8 |
1062 : * --------+------+
1063 : * 3 | 16 |
1064 : * 4 | 32 |
1065 : * 5 | 48 |
1066 : * 6 | 64 |
1067 : * : :
1068 : * : :
1069 : * 33 | 496 |
1070 : * 34 | 512 |
1071 : * --------+------+
1072 : * 35 | 1024 |
1073 : * 36 | 2048 |
1074 : * --------+------+
1075 : */
1076 : arena_bin_t bins[1]; /* Dynamically sized. */
1077 : };
1078 :
1079 : /******************************************************************************/
1080 : /*
1081 : * Data.
1082 : */
1083 :
1084 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1085 : /* Number of CPUs. */
1086 : static unsigned ncpus;
1087 : #endif
1088 :
1089 : /*
1090 : * When MALLOC_STATIC_SIZES is defined most of the parameters
1091 : * controlling the malloc behavior are defined as compile-time constants
1092 : * for best performance and cannot be altered at runtime.
1093 : */
1094 : #define MALLOC_STATIC_SIZES 1
1095 :
1096 : #ifdef MALLOC_STATIC_SIZES
1097 :
1098 : /*
1099 : * VM page size. It must divide the runtime CPU page size or the code
1100 : * will abort.
1101 : */
1102 : #define pagesize_2pow ((size_t) 12)
1103 : #define pagesize ((size_t) 1 << pagesize_2pow)
1104 : #define pagesize_mask (pagesize - 1)
1105 :
1106 : /* Various quantum-related settings. */
1107 :
1108 : #define QUANTUM_DEFAULT ((size_t) 1 << QUANTUM_2POW_MIN)
1109 : static const size_t quantum = QUANTUM_DEFAULT;
1110 : static const size_t quantum_mask = QUANTUM_DEFAULT - 1;
1111 :
1112 : /* Various bin-related settings. */
1113 :
1114 : static const size_t small_min = (QUANTUM_DEFAULT >> 1) + 1;
1115 : static const size_t small_max = (size_t) SMALL_MAX_DEFAULT;
1116 :
1117 : /* Max size class for bins. */
1118 : static const size_t bin_maxclass = pagesize >> 1;
1119 :
1120 : /* Number of (2^n)-spaced tiny bins. */
1121 : static const unsigned ntbins = (unsigned)
1122 : (QUANTUM_2POW_MIN - TINY_MIN_2POW);
1123 :
1124 : /* Number of quantum-spaced bins. */
1125 : static const unsigned nqbins = (unsigned)
1126 : (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
1127 :
1128 : /* Number of (2^n)-spaced sub-page bins. */
1129 : static const unsigned nsbins = (unsigned)
1130 : (pagesize_2pow -
1131 : SMALL_MAX_2POW_DEFAULT - 1);
1132 :
1133 : #else /* !MALLOC_STATIC_SIZES */
1134 :
1135 : /* VM page size. */
1136 : static size_t pagesize;
1137 : static size_t pagesize_mask;
1138 : static size_t pagesize_2pow;
1139 :
1140 : /* Various bin-related settings. */
1141 : static size_t bin_maxclass; /* Max size class for bins. */
1142 : static unsigned ntbins; /* Number of (2^n)-spaced tiny bins. */
1143 : static unsigned nqbins; /* Number of quantum-spaced bins. */
1144 : static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */
1145 : static size_t small_min;
1146 : static size_t small_max;
1147 :
1148 : /* Various quantum-related settings. */
1149 : static size_t quantum;
1150 : static size_t quantum_mask; /* (quantum - 1). */
1151 :
1152 : #endif
1153 :
1154 : /* Various chunk-related settings. */
1155 :
1156 : /*
1157 : * Compute the header size such that it is large enough to contain the page map
1158 : * and enough nodes for the worst case: one node per non-header page plus one
1159 : * extra for situations where we briefly have one more node allocated than we
1160 : * will need.
1161 : */
1162 : #define calculate_arena_header_size() \
1163 : (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1))
1164 :
1165 : #define calculate_arena_header_pages() \
1166 : ((calculate_arena_header_size() >> pagesize_2pow) + \
1167 : ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0))
1168 :
1169 : /* Max size class for arenas. */
1170 : #define calculate_arena_maxclass() \
1171 : (chunksize - (arena_chunk_header_npages << pagesize_2pow))
1172 :
1173 : #ifdef MALLOC_STATIC_SIZES
1174 : #define CHUNKSIZE_DEFAULT ((size_t) 1 << CHUNK_2POW_DEFAULT)
1175 : static const size_t chunksize = CHUNKSIZE_DEFAULT;
1176 : static const size_t chunksize_mask =CHUNKSIZE_DEFAULT - 1;
1177 : static const size_t chunk_npages = CHUNKSIZE_DEFAULT >> pagesize_2pow;
1178 : #define arena_chunk_header_npages calculate_arena_header_pages()
1179 : #define arena_maxclass calculate_arena_maxclass()
1180 : #else
1181 : static size_t chunksize;
1182 : static size_t chunksize_mask; /* (chunksize - 1). */
1183 : static size_t chunk_npages;
1184 : static size_t arena_chunk_header_npages;
1185 : static size_t arena_maxclass; /* Max size class for arenas. */
1186 : #endif
1187 :
1188 : /********/
1189 : /*
1190 : * Chunks.
1191 : */
1192 :
1193 : #ifdef MALLOC_VALIDATE
1194 : static malloc_rtree_t *chunk_rtree;
1195 : #endif
1196 :
1197 : /* Protects chunk-related data structures. */
1198 : static malloc_mutex_t huge_mtx;
1199 :
1200 : /* Tree of chunks that are stand-alone huge allocations. */
1201 : static extent_tree_t huge;
1202 :
1203 : #ifdef MALLOC_STATS
1204 : /* Huge allocation statistics. */
1205 : static uint64_t huge_nmalloc;
1206 : static uint64_t huge_ndalloc;
1207 : static size_t huge_allocated;
1208 : #endif
1209 :
1210 : #ifdef MALLOC_PAGEFILE
1211 : static char pagefile_templ[PATH_MAX];
1212 : #endif
1213 :
1214 : /****************************/
1215 : /*
1216 : * base (internal allocation).
1217 : */
1218 :
1219 : /*
1220 : * Current pages that are being used for internal memory allocations. These
1221 : * pages are carved up in cacheline-size quanta, so that there is no chance of
1222 : * false cache line sharing.
1223 : */
1224 : static void *base_pages;
1225 : static void *base_next_addr;
1226 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1227 : static void *base_next_decommitted;
1228 : #endif
1229 : static void *base_past_addr; /* Addr immediately past base_pages. */
1230 : static extent_node_t *base_nodes;
1231 : static malloc_mutex_t base_mtx;
1232 : #ifdef MALLOC_STATS
1233 : static size_t base_mapped;
1234 : static size_t base_committed;
1235 : #endif
1236 :
1237 : /********/
1238 : /*
1239 : * Arenas.
1240 : */
1241 :
1242 : /*
1243 : * Arenas that are used to service external requests. Not all elements of the
1244 : * arenas array are necessarily used; arenas are created lazily as needed.
1245 : */
1246 : static arena_t **arenas;
1247 : static unsigned narenas;
1248 : #ifndef NO_TLS
1249 : # ifdef MALLOC_BALANCE
1250 : static unsigned narenas_2pow;
1251 : # else
1252 : static unsigned next_arena;
1253 : # endif
1254 : #endif
1255 : #ifdef MOZ_MEMORY
1256 : static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
1257 : #else
1258 : static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
1259 : #endif
1260 :
1261 : #ifndef NO_TLS
1262 : /*
1263 : * Map of pthread_self() --> arenas[???], used for selecting an arena to use
1264 : * for allocations.
1265 : */
1266 : #ifndef MOZ_MEMORY_WINDOWS
1267 : static __thread arena_t *arenas_map;
1268 : #endif
1269 : #endif
1270 :
1271 : #ifdef MALLOC_STATS
1272 : /* Chunk statistics. */
1273 : static chunk_stats_t stats_chunks;
1274 : #endif
1275 :
1276 : /*******************************/
1277 : /*
1278 : * Runtime configuration options.
1279 : */
1280 : const char *_malloc_options;
1281 :
1282 : #ifndef MALLOC_PRODUCTION
1283 : static bool opt_abort = true;
1284 : #ifdef MALLOC_FILL
1285 : static bool opt_junk = true;
1286 : #endif
1287 : #else
1288 : static bool opt_abort = false;
1289 : #ifdef MALLOC_FILL
1290 : static bool opt_junk = false;
1291 : #endif
1292 : #endif
1293 : static size_t opt_dirty_max = DIRTY_MAX_DEFAULT;
1294 : #ifdef MALLOC_BALANCE
1295 : static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
1296 : #endif
1297 : static bool opt_print_stats = false;
1298 : #ifdef MALLOC_STATIC_SIZES
1299 : #define opt_quantum_2pow QUANTUM_2POW_MIN
1300 : #define opt_small_max_2pow SMALL_MAX_2POW_DEFAULT
1301 : #define opt_chunk_2pow CHUNK_2POW_DEFAULT
1302 : #else
1303 : static size_t opt_quantum_2pow = QUANTUM_2POW_MIN;
1304 : static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
1305 : static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT;
1306 : #endif
1307 : #ifdef MALLOC_PAGEFILE
1308 : static bool opt_pagefile = false;
1309 : #endif
1310 : #ifdef MALLOC_UTRACE
1311 : static bool opt_utrace = false;
1312 : #endif
1313 : #ifdef MALLOC_SYSV
1314 : static bool opt_sysv = false;
1315 : #endif
1316 : #ifdef MALLOC_XMALLOC
1317 : static bool opt_xmalloc = false;
1318 : #endif
1319 : #ifdef MALLOC_FILL
1320 : static bool opt_zero = false;
1321 : #endif
1322 : static int opt_narenas_lshift = 0;
1323 :
1324 : #ifdef MALLOC_UTRACE
1325 : typedef struct {
1326 : void *p;
1327 : size_t s;
1328 : void *r;
1329 : } malloc_utrace_t;
1330 :
1331 : #define UTRACE(a, b, c) \
1332 : if (opt_utrace) { \
1333 : malloc_utrace_t ut; \
1334 : ut.p = (a); \
1335 : ut.s = (b); \
1336 : ut.r = (c); \
1337 : utrace(&ut, sizeof(ut)); \
1338 : }
1339 : #else
1340 : #define UTRACE(a, b, c)
1341 : #endif
1342 :
1343 : /******************************************************************************/
1344 : /*
1345 : * Begin function prototypes for non-inline static functions.
1346 : */
1347 :
1348 : static char *umax2s(uintmax_t x, unsigned base, char *s);
1349 : static bool malloc_mutex_init(malloc_mutex_t *mutex);
1350 : static bool malloc_spin_init(malloc_spinlock_t *lock);
1351 : static void wrtmessage(const char *p1, const char *p2, const char *p3,
1352 : const char *p4);
1353 : #ifdef MALLOC_STATS
1354 : #ifdef MOZ_MEMORY_DARWIN
1355 : /* Avoid namespace collision with OS X's malloc APIs. */
1356 : #define malloc_printf moz_malloc_printf
1357 : #endif
1358 : static void malloc_printf(const char *format, ...);
1359 : #endif
1360 : static bool base_pages_alloc_mmap(size_t minsize);
1361 : static bool base_pages_alloc(size_t minsize);
1362 : static void *base_alloc(size_t size);
1363 : static void *base_calloc(size_t number, size_t size);
1364 : static extent_node_t *base_node_alloc(void);
1365 : static void base_node_dealloc(extent_node_t *node);
1366 : #ifdef MALLOC_STATS
1367 : static void stats_print(arena_t *arena);
1368 : #endif
1369 : static void *pages_map(void *addr, size_t size, int pfd);
1370 : static void pages_unmap(void *addr, size_t size);
1371 : static void *chunk_alloc_mmap(size_t size, bool pagefile);
1372 : #ifdef MALLOC_PAGEFILE
1373 : static int pagefile_init(size_t size);
1374 : static void pagefile_close(int pfd);
1375 : #endif
1376 : static void *chunk_alloc(size_t size, bool zero, bool pagefile);
1377 : static void chunk_dealloc_mmap(void *chunk, size_t size);
1378 : static void chunk_dealloc(void *chunk, size_t size);
1379 : #ifndef NO_TLS
1380 : static arena_t *choose_arena_hard(void);
1381 : #endif
1382 : static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
1383 : bool large, bool zero);
1384 : static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
1385 : static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
1386 : static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
1387 : size_t size, bool large, bool zero);
1388 : static void arena_purge(arena_t *arena);
1389 : static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
1390 : static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
1391 : arena_run_t *run, size_t oldsize, size_t newsize);
1392 : static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
1393 : arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
1394 : static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
1395 : static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
1396 : static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
1397 : #ifdef MALLOC_BALANCE
1398 : static void arena_lock_balance_hard(arena_t *arena);
1399 : #endif
1400 : static void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
1401 : static void *arena_palloc(arena_t *arena, size_t alignment, size_t size,
1402 : size_t alloc_size);
1403 : static size_t arena_salloc(const void *ptr);
1404 : static void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
1405 : void *ptr);
1406 : static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
1407 : void *ptr, size_t size, size_t oldsize);
1408 : static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
1409 : void *ptr, size_t size, size_t oldsize);
1410 : static bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
1411 : static void *arena_ralloc(void *ptr, size_t size, size_t oldsize);
1412 : static bool arena_new(arena_t *arena);
1413 : static arena_t *arenas_extend(unsigned ind);
1414 : static void *huge_malloc(size_t size, bool zero);
1415 : static void *huge_palloc(size_t alignment, size_t size);
1416 : static void *huge_ralloc(void *ptr, size_t size, size_t oldsize);
1417 : static void huge_dalloc(void *ptr);
1418 : static void malloc_print_stats(void);
1419 : #ifndef MOZ_MEMORY_WINDOWS
1420 : static
1421 : #endif
1422 : bool malloc_init_hard(void);
1423 :
1424 : static void _malloc_prefork(void);
1425 : static void _malloc_postfork(void);
1426 :
1427 : #ifdef MOZ_MEMORY_DARWIN
1428 : /*
1429 : * MALLOC_ZONE_T_NOTE
1430 : *
1431 : * On Darwin, we hook into the memory allocator using a malloc_zone_t struct.
1432 : * We must be very careful around this struct because of different behaviour on
1433 : * different versions of OSX.
1434 : *
1435 : * Each of OSX 10.5, 10.6 and 10.7 use different versions of the struct
1436 : * (with version numbers 3, 6 and 8 respectively). The binary we use on each of
1437 : * these platforms will not necessarily be built using the correct SDK [1].
1438 : * This means we need to statically know the correct struct size to use on all
1439 : * OSX releases, and have a fallback for unknown future versions. The struct
1440 : * sizes defined in osx_zone_types.h.
1441 : *
1442 : * For OSX 10.8 and later, we may expect the malloc_zone_t struct to change
1443 : * again, and need to dynamically account for this. By simply leaving
1444 : * malloc_zone_t alone, we don't quite deal with the problem, because there
1445 : * remain calls to jemalloc through the mozalloc interface. We check this
1446 : * dynamically on each allocation, using the CHECK_DARWIN macro and
1447 : * osx_use_jemalloc.
1448 : *
1449 : *
1450 : * [1] Mozilla is built as a universal binary on Mac, supporting i386 and
1451 : * x86_64. The i386 target is built using the 10.5 SDK, even if it runs on
1452 : * 10.6. The x86_64 target is built using the 10.6 SDK, even if it runs on
1453 : * 10.7 or later, or 10.5.
1454 : *
1455 : * FIXME:
1456 : * When later versions of OSX come out (10.8 and up), we need to check their
1457 : * malloc_zone_t versions. If they're greater than 8, we need a new version
1458 : * of malloc_zone_t adapted into osx_zone_types.h.
1459 : */
1460 :
1461 : #include "osx_zone_types.h"
1462 :
1463 : #define LEOPARD_MALLOC_ZONE_T_VERSION 3
1464 : #define SNOW_LEOPARD_MALLOC_ZONE_T_VERSION 6
1465 : #define LION_MALLOC_ZONE_T_VERSION 8
1466 :
1467 : static bool osx_use_jemalloc = false;
1468 :
1469 :
1470 : static lion_malloc_zone l_szone;
1471 : static malloc_zone_t * szone = (malloc_zone_t*)(&l_szone);
1472 :
1473 : static lion_malloc_introspection l_ozone_introspect;
1474 : static malloc_introspection_t * const ozone_introspect =
1475 : (malloc_introspection_t*)(&l_ozone_introspect);
1476 : static void szone2ozone(malloc_zone_t *zone, size_t size);
1477 : static size_t zone_version_size(int version);
1478 : #endif
1479 :
1480 : /*
1481 : * End function prototypes.
1482 : */
1483 : /******************************************************************************/
1484 :
1485 : /*
1486 : * umax2s() provides minimal integer printing functionality, which is
1487 : * especially useful for situations where allocation in vsnprintf() calls would
1488 : * potentially cause deadlock.
1489 : */
1490 : #define UMAX2S_BUFSIZE 65
1491 : char *
1492 0 : umax2s(uintmax_t x, unsigned base, char *s)
1493 : {
1494 : unsigned i;
1495 :
1496 0 : i = UMAX2S_BUFSIZE - 1;
1497 0 : s[i] = '\0';
1498 0 : switch (base) {
1499 : case 10:
1500 : do {
1501 0 : i--;
1502 0 : s[i] = "0123456789"[x % 10];
1503 0 : x /= 10;
1504 0 : } while (x > 0);
1505 0 : break;
1506 : case 16:
1507 : do {
1508 0 : i--;
1509 0 : s[i] = "0123456789abcdef"[x & 0xf];
1510 0 : x >>= 4;
1511 0 : } while (x > 0);
1512 0 : break;
1513 : default:
1514 : do {
1515 0 : i--;
1516 0 : s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % base];
1517 0 : x /= base;
1518 0 : } while (x > 0);
1519 : }
1520 :
1521 0 : return (&s[i]);
1522 : }
1523 :
1524 : static void
1525 0 : wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
1526 : {
1527 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
1528 : #define _write write
1529 : #endif
1530 0 : _write(STDERR_FILENO, p1, (unsigned int) strlen(p1));
1531 0 : _write(STDERR_FILENO, p2, (unsigned int) strlen(p2));
1532 0 : _write(STDERR_FILENO, p3, (unsigned int) strlen(p3));
1533 0 : _write(STDERR_FILENO, p4, (unsigned int) strlen(p4));
1534 0 : }
1535 :
1536 : #define _malloc_message malloc_message
1537 :
1538 : void (*_malloc_message)(const char *p1, const char *p2, const char *p3,
1539 : const char *p4) = wrtmessage;
1540 :
1541 : #ifdef MALLOC_DEBUG
1542 : # define assert(e) do { \
1543 : if (!(e)) { \
1544 : char line_buf[UMAX2S_BUFSIZE]; \
1545 : _malloc_message(__FILE__, ":", umax2s(__LINE__, 10, \
1546 : line_buf), ": Failed assertion: "); \
1547 : _malloc_message("\"", #e, "\"\n", ""); \
1548 : abort(); \
1549 : } \
1550 : } while (0)
1551 : #else
1552 : #define assert(e)
1553 : #endif
1554 :
1555 : /******************************************************************************/
1556 : /*
1557 : * Begin mutex. We can't use normal pthread mutexes in all places, because
1558 : * they require malloc()ed memory, which causes bootstrapping issues in some
1559 : * cases.
1560 : */
1561 :
1562 : static bool
1563 93345 : malloc_mutex_init(malloc_mutex_t *mutex)
1564 : {
1565 : #if defined(MOZ_MEMORY_WINDOWS)
1566 : if (isthreaded)
1567 : if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
1568 : return (true);
1569 : #elif defined(MOZ_MEMORY_DARWIN)
1570 : mutex->lock = OS_SPINLOCK_INIT;
1571 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1572 : pthread_mutexattr_t attr;
1573 93345 : if (pthread_mutexattr_init(&attr) != 0)
1574 0 : return (true);
1575 93345 : pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1576 93345 : if (pthread_mutex_init(mutex, &attr) != 0) {
1577 0 : pthread_mutexattr_destroy(&attr);
1578 0 : return (true);
1579 : }
1580 93345 : pthread_mutexattr_destroy(&attr);
1581 : #elif defined(MOZ_MEMORY)
1582 : if (pthread_mutex_init(mutex, NULL) != 0)
1583 : return (true);
1584 : #else
1585 : static const spinlock_t lock = _SPINLOCK_INITIALIZER;
1586 :
1587 : mutex->lock = lock;
1588 : #endif
1589 93345 : return (false);
1590 : }
1591 :
1592 : static inline void
1593 90845182 : malloc_mutex_lock(malloc_mutex_t *mutex)
1594 : {
1595 :
1596 : #if defined(MOZ_MEMORY_WINDOWS)
1597 : EnterCriticalSection(mutex);
1598 : #elif defined(MOZ_MEMORY_DARWIN)
1599 : OSSpinLockLock(&mutex->lock);
1600 : #elif defined(MOZ_MEMORY)
1601 90845182 : pthread_mutex_lock(mutex);
1602 : #else
1603 : if (isthreaded)
1604 : _SPINLOCK(&mutex->lock);
1605 : #endif
1606 90845224 : }
1607 :
1608 : static inline void
1609 90845224 : malloc_mutex_unlock(malloc_mutex_t *mutex)
1610 : {
1611 :
1612 : #if defined(MOZ_MEMORY_WINDOWS)
1613 : LeaveCriticalSection(mutex);
1614 : #elif defined(MOZ_MEMORY_DARWIN)
1615 : OSSpinLockUnlock(&mutex->lock);
1616 : #elif defined(MOZ_MEMORY)
1617 90845224 : pthread_mutex_unlock(mutex);
1618 : #else
1619 : if (isthreaded)
1620 : _SPINUNLOCK(&mutex->lock);
1621 : #endif
1622 90845222 : }
1623 :
1624 : static bool
1625 0 : malloc_spin_init(malloc_spinlock_t *lock)
1626 : {
1627 : #if defined(MOZ_MEMORY_WINDOWS)
1628 : if (isthreaded)
1629 : if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
1630 : return (true);
1631 : #elif defined(MOZ_MEMORY_DARWIN)
1632 : lock->lock = OS_SPINLOCK_INIT;
1633 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1634 : pthread_mutexattr_t attr;
1635 0 : if (pthread_mutexattr_init(&attr) != 0)
1636 0 : return (true);
1637 0 : pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1638 0 : if (pthread_mutex_init(lock, &attr) != 0) {
1639 0 : pthread_mutexattr_destroy(&attr);
1640 0 : return (true);
1641 : }
1642 0 : pthread_mutexattr_destroy(&attr);
1643 : #elif defined(MOZ_MEMORY)
1644 : if (pthread_mutex_init(lock, NULL) != 0)
1645 : return (true);
1646 : #else
1647 : lock->lock = _SPINLOCK_INITIALIZER;
1648 : #endif
1649 0 : return (false);
1650 : }
1651 :
1652 : static inline void
1653 0 : malloc_spin_lock(malloc_spinlock_t *lock)
1654 : {
1655 :
1656 : #if defined(MOZ_MEMORY_WINDOWS)
1657 : EnterCriticalSection(lock);
1658 : #elif defined(MOZ_MEMORY_DARWIN)
1659 : OSSpinLockLock(&lock->lock);
1660 : #elif defined(MOZ_MEMORY)
1661 0 : pthread_mutex_lock(lock);
1662 : #else
1663 : if (isthreaded)
1664 : _SPINLOCK(&lock->lock);
1665 : #endif
1666 0 : }
1667 :
1668 : static inline void
1669 0 : malloc_spin_unlock(malloc_spinlock_t *lock)
1670 : {
1671 : #if defined(MOZ_MEMORY_WINDOWS)
1672 : LeaveCriticalSection(lock);
1673 : #elif defined(MOZ_MEMORY_DARWIN)
1674 : OSSpinLockUnlock(&lock->lock);
1675 : #elif defined(MOZ_MEMORY)
1676 0 : pthread_mutex_unlock(lock);
1677 : #else
1678 : if (isthreaded)
1679 : _SPINUNLOCK(&lock->lock);
1680 : #endif
1681 0 : }
1682 :
1683 : /*
1684 : * End mutex.
1685 : */
1686 : /******************************************************************************/
1687 : /*
1688 : * Begin spin lock. Spin locks here are actually adaptive mutexes that block
1689 : * after a period of spinning, because unbounded spinning would allow for
1690 : * priority inversion.
1691 : */
1692 :
1693 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
1694 : # define malloc_spin_init malloc_mutex_init
1695 : # define malloc_spin_lock malloc_mutex_lock
1696 : # define malloc_spin_unlock malloc_mutex_unlock
1697 : #endif
1698 :
1699 : #ifndef MOZ_MEMORY
1700 : /*
1701 : * We use an unpublished interface to initialize pthread mutexes with an
1702 : * allocation callback, in order to avoid infinite recursion.
1703 : */
1704 : int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
1705 : void *(calloc_cb)(size_t, size_t));
1706 :
1707 : __weak_reference(_pthread_mutex_init_calloc_cb_stub,
1708 : _pthread_mutex_init_calloc_cb);
1709 :
1710 : int
1711 : _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
1712 : void *(calloc_cb)(size_t, size_t))
1713 : {
1714 :
1715 : return (0);
1716 : }
1717 :
1718 : static bool
1719 : malloc_spin_init(pthread_mutex_t *lock)
1720 : {
1721 :
1722 : if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
1723 : return (true);
1724 :
1725 : return (false);
1726 : }
1727 :
1728 : static inline unsigned
1729 : malloc_spin_lock(pthread_mutex_t *lock)
1730 : {
1731 : unsigned ret = 0;
1732 :
1733 : if (isthreaded) {
1734 : if (_pthread_mutex_trylock(lock) != 0) {
1735 : unsigned i;
1736 : volatile unsigned j;
1737 :
1738 : /* Exponentially back off. */
1739 : for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
1740 : for (j = 0; j < (1U << i); j++)
1741 : ret++;
1742 :
1743 : CPU_SPINWAIT;
1744 : if (_pthread_mutex_trylock(lock) == 0)
1745 : return (ret);
1746 : }
1747 :
1748 : /*
1749 : * Spinning failed. Block until the lock becomes
1750 : * available, in order to avoid indefinite priority
1751 : * inversion.
1752 : */
1753 : _pthread_mutex_lock(lock);
1754 : assert((ret << BLOCK_COST_2POW) != 0);
1755 : return (ret << BLOCK_COST_2POW);
1756 : }
1757 : }
1758 :
1759 : return (ret);
1760 : }
1761 :
1762 : static inline void
1763 : malloc_spin_unlock(pthread_mutex_t *lock)
1764 : {
1765 :
1766 : if (isthreaded)
1767 : _pthread_mutex_unlock(lock);
1768 : }
1769 : #endif
1770 :
1771 : /*
1772 : * End spin lock.
1773 : */
1774 : /******************************************************************************/
1775 : /*
1776 : * Begin Utility functions/macros.
1777 : */
1778 :
1779 : /* Return the chunk address for allocation address a. */
1780 : #define CHUNK_ADDR2BASE(a) \
1781 : ((void *)((uintptr_t)(a) & ~chunksize_mask))
1782 :
1783 : /* Return the chunk offset of address a. */
1784 : #define CHUNK_ADDR2OFFSET(a) \
1785 : ((size_t)((uintptr_t)(a) & chunksize_mask))
1786 :
1787 : /* Return the smallest chunk multiple that is >= s. */
1788 : #define CHUNK_CEILING(s) \
1789 : (((s) + chunksize_mask) & ~chunksize_mask)
1790 :
1791 : /* Return the smallest cacheline multiple that is >= s. */
1792 : #define CACHELINE_CEILING(s) \
1793 : (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
1794 :
1795 : /* Return the smallest quantum multiple that is >= a. */
1796 : #define QUANTUM_CEILING(a) \
1797 : (((a) + quantum_mask) & ~quantum_mask)
1798 :
1799 : /* Return the smallest pagesize multiple that is >= s. */
1800 : #define PAGE_CEILING(s) \
1801 : (((s) + pagesize_mask) & ~pagesize_mask)
1802 :
1803 : /* Compute the smallest power of 2 that is >= x. */
1804 : static inline size_t
1805 14496575 : pow2_ceil(size_t x)
1806 : {
1807 :
1808 14496575 : x--;
1809 14496575 : x |= x >> 1;
1810 14496575 : x |= x >> 2;
1811 14496575 : x |= x >> 4;
1812 14496575 : x |= x >> 8;
1813 14496575 : x |= x >> 16;
1814 : #if (SIZEOF_PTR == 8)
1815 : x |= x >> 32;
1816 : #endif
1817 14496575 : x++;
1818 14496575 : return (x);
1819 : }
1820 :
1821 : #ifdef MALLOC_BALANCE
1822 : /*
1823 : * Use a simple linear congruential pseudo-random number generator:
1824 : *
1825 : * prn(y) = (a*x + c) % m
1826 : *
1827 : * where the following constants ensure maximal period:
1828 : *
1829 : * a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
1830 : * c == Odd number (relatively prime to 2^n).
1831 : * m == 2^32
1832 : *
1833 : * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
1834 : *
1835 : * This choice of m has the disadvantage that the quality of the bits is
1836 : * proportional to bit position. For example. the lowest bit has a cycle of 2,
1837 : * the next has a cycle of 4, etc. For this reason, we prefer to use the upper
1838 : * bits.
1839 : */
1840 : # define PRN_DEFINE(suffix, var, a, c) \
1841 : static inline void \
1842 : sprn_##suffix(uint32_t seed) \
1843 : { \
1844 : var = seed; \
1845 : } \
1846 : \
1847 : static inline uint32_t \
1848 : prn_##suffix(uint32_t lg_range) \
1849 : { \
1850 : uint32_t ret, x; \
1851 : \
1852 : assert(lg_range > 0); \
1853 : assert(lg_range <= 32); \
1854 : \
1855 : x = (var * (a)) + (c); \
1856 : var = x; \
1857 : ret = x >> (32 - lg_range); \
1858 : \
1859 : return (ret); \
1860 : }
1861 : # define SPRN(suffix, seed) sprn_##suffix(seed)
1862 : # define PRN(suffix, lg_range) prn_##suffix(lg_range)
1863 : #endif
1864 :
1865 : #ifdef MALLOC_BALANCE
1866 : /* Define the PRNG used for arena assignment. */
1867 : static __thread uint32_t balance_x;
1868 : PRN_DEFINE(balance, balance_x, 1297, 1301)
1869 : #endif
1870 :
1871 : #ifdef MALLOC_UTRACE
1872 : static int
1873 0 : utrace(const void *addr, size_t len)
1874 : {
1875 0 : malloc_utrace_t *ut = (malloc_utrace_t *)addr;
1876 : char buf_a[UMAX2S_BUFSIZE];
1877 : char buf_b[UMAX2S_BUFSIZE];
1878 :
1879 0 : assert(len == sizeof(malloc_utrace_t));
1880 :
1881 0 : if (ut->p == NULL && ut->s == 0 && ut->r == NULL) {
1882 0 : _malloc_message(
1883 0 : umax2s(getpid(), 10, buf_a),
1884 : " x USER malloc_init()\n", "", "");
1885 0 : } else if (ut->p == NULL && ut->r != NULL) {
1886 0 : _malloc_message(
1887 0 : umax2s(getpid(), 10, buf_a),
1888 : " x USER 0x",
1889 0 : umax2s((uintptr_t)ut->r, 16, buf_b),
1890 : " = malloc(");
1891 0 : _malloc_message(
1892 0 : umax2s(ut->s, 10, buf_a),
1893 : ")\n", "", "");
1894 0 : } else if (ut->p != NULL && ut->r != NULL) {
1895 0 : _malloc_message(
1896 0 : umax2s(getpid(), 10, buf_a),
1897 : " x USER 0x",
1898 0 : umax2s((uintptr_t)ut->r, 16, buf_b),
1899 : " = realloc(0x");
1900 0 : _malloc_message(
1901 0 : umax2s((uintptr_t)ut->p, 16, buf_a),
1902 : ", ",
1903 0 : umax2s(ut->s, 10, buf_b),
1904 : ")\n");
1905 : } else {
1906 0 : _malloc_message(
1907 0 : umax2s(getpid(), 10, buf_a),
1908 : " x USER free(0x",
1909 0 : umax2s((uintptr_t)ut->p, 16, buf_b),
1910 : ")\n");
1911 : }
1912 :
1913 0 : return (0);
1914 : }
1915 : #endif
1916 :
1917 : static inline const char *
1918 0 : _getprogname(void)
1919 : {
1920 :
1921 0 : return ("<jemalloc>");
1922 : }
1923 :
1924 : #ifdef MALLOC_STATS
1925 : /*
1926 : * Print to stderr in such a way as to (hopefully) avoid memory allocation.
1927 : */
1928 : static void
1929 0 : malloc_printf(const char *format, ...)
1930 : {
1931 : char buf[4096];
1932 : va_list ap;
1933 :
1934 0 : va_start(ap, format);
1935 0 : vsnprintf(buf, sizeof(buf), format, ap);
1936 0 : va_end(ap);
1937 0 : _malloc_message(buf, "", "", "");
1938 0 : }
1939 : #endif
1940 :
1941 : /******************************************************************************/
1942 :
1943 : static inline void
1944 0 : pages_decommit(void *addr, size_t size)
1945 : {
1946 :
1947 : #ifdef MOZ_MEMORY_WINDOWS
1948 : VirtualFree(addr, size, MEM_DECOMMIT);
1949 : #else
1950 0 : if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
1951 : 0) == MAP_FAILED)
1952 0 : abort();
1953 : #endif
1954 0 : }
1955 :
1956 : static inline void
1957 0 : pages_commit(void *addr, size_t size)
1958 : {
1959 :
1960 : # ifdef MOZ_MEMORY_WINDOWS
1961 : VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
1962 : # else
1963 0 : if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
1964 : MAP_ANON, -1, 0) == MAP_FAILED)
1965 0 : abort();
1966 : # endif
1967 0 : }
1968 :
1969 : static bool
1970 18669 : base_pages_alloc_mmap(size_t minsize)
1971 : {
1972 : bool ret;
1973 : size_t csize;
1974 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1975 : size_t pminsize;
1976 : #endif
1977 : int pfd;
1978 :
1979 18669 : assert(minsize != 0);
1980 18669 : csize = CHUNK_CEILING(minsize);
1981 : #ifdef MALLOC_PAGEFILE
1982 : if (opt_pagefile) {
1983 : pfd = pagefile_init(csize);
1984 : if (pfd == -1)
1985 : return (true);
1986 : } else
1987 : #endif
1988 18669 : pfd = -1;
1989 18669 : base_pages = pages_map(NULL, csize, pfd);
1990 18669 : if (base_pages == NULL) {
1991 0 : ret = true;
1992 0 : goto RETURN;
1993 : }
1994 18669 : base_next_addr = base_pages;
1995 18669 : base_past_addr = (void *)((uintptr_t)base_pages + csize);
1996 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1997 : /*
1998 : * Leave enough pages for minsize committed, since otherwise they would
1999 : * have to be immediately recommitted.
2000 : */
2001 18669 : pminsize = PAGE_CEILING(minsize);
2002 18669 : base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
2003 : # if defined(MALLOC_DECOMMIT)
2004 : if (pminsize < csize)
2005 : pages_decommit(base_next_decommitted, csize - pminsize);
2006 : # endif
2007 : # ifdef MALLOC_STATS
2008 18669 : base_mapped += csize;
2009 18669 : base_committed += pminsize;
2010 : # endif
2011 : #endif
2012 :
2013 18669 : ret = false;
2014 : RETURN:
2015 : #ifdef MALLOC_PAGEFILE
2016 : if (pfd != -1)
2017 : pagefile_close(pfd);
2018 : #endif
2019 18669 : return (false);
2020 : }
2021 :
2022 : static bool
2023 18669 : base_pages_alloc(size_t minsize)
2024 : {
2025 :
2026 18669 : if (base_pages_alloc_mmap(minsize) == false)
2027 18669 : return (false);
2028 :
2029 0 : return (true);
2030 : }
2031 :
2032 : static void *
2033 75275 : base_alloc(size_t size)
2034 : {
2035 : void *ret;
2036 : size_t csize;
2037 :
2038 : /* Round size up to nearest multiple of the cacheline size. */
2039 75275 : csize = CACHELINE_CEILING(size);
2040 :
2041 75275 : malloc_mutex_lock(&base_mtx);
2042 : /* Make sure there's enough space for the allocation. */
2043 75275 : if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
2044 18669 : if (base_pages_alloc(csize)) {
2045 0 : malloc_mutex_unlock(&base_mtx);
2046 0 : return (NULL);
2047 : }
2048 : }
2049 : /* Allocate. */
2050 75275 : ret = base_next_addr;
2051 75275 : base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
2052 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
2053 : /* Make sure enough pages are committed for the new allocation. */
2054 75275 : if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
2055 18669 : void *pbase_next_addr =
2056 18669 : (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
2057 :
2058 : # ifdef MALLOC_DECOMMIT
2059 : pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
2060 : (uintptr_t)base_next_decommitted);
2061 : # endif
2062 18669 : base_next_decommitted = pbase_next_addr;
2063 : # ifdef MALLOC_STATS
2064 37338 : base_committed += (uintptr_t)pbase_next_addr -
2065 18669 : (uintptr_t)base_next_decommitted;
2066 : # endif
2067 : }
2068 : #endif
2069 75275 : malloc_mutex_unlock(&base_mtx);
2070 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
2071 :
2072 75275 : return (ret);
2073 : }
2074 :
2075 : static void *
2076 37338 : base_calloc(size_t number, size_t size)
2077 : {
2078 : void *ret;
2079 :
2080 37338 : ret = base_alloc(number * size);
2081 : #ifdef MALLOC_VALGRIND
2082 : if (ret != NULL) {
2083 : VALGRIND_FREELIKE_BLOCK(ret, 0);
2084 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, true);
2085 : }
2086 : #endif
2087 37338 : memset(ret, 0, number * size);
2088 :
2089 37338 : return (ret);
2090 : }
2091 :
2092 : static extent_node_t *
2093 4554 : base_node_alloc(void)
2094 : {
2095 : extent_node_t *ret;
2096 :
2097 4554 : malloc_mutex_lock(&base_mtx);
2098 4554 : if (base_nodes != NULL) {
2099 3955 : ret = base_nodes;
2100 3955 : base_nodes = *(extent_node_t **)ret;
2101 : VALGRIND_FREELIKE_BLOCK(ret, 0);
2102 : VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(extent_node_t), 0, false);
2103 3955 : malloc_mutex_unlock(&base_mtx);
2104 : } else {
2105 599 : malloc_mutex_unlock(&base_mtx);
2106 599 : ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
2107 : }
2108 :
2109 4554 : return (ret);
2110 : }
2111 :
2112 : static void
2113 4554 : base_node_dealloc(extent_node_t *node)
2114 : {
2115 :
2116 4554 : malloc_mutex_lock(&base_mtx);
2117 : VALGRIND_FREELIKE_BLOCK(node, 0);
2118 : VALGRIND_MALLOCLIKE_BLOCK(node, sizeof(extent_node_t *), 0, false);
2119 4554 : *(extent_node_t **)node = base_nodes;
2120 4554 : base_nodes = node;
2121 4554 : malloc_mutex_unlock(&base_mtx);
2122 4554 : }
2123 :
2124 : /******************************************************************************/
2125 :
2126 : #ifdef MALLOC_STATS
2127 : static void
2128 0 : stats_print(arena_t *arena)
2129 : {
2130 : unsigned i, gap_start;
2131 :
2132 : #ifdef MOZ_MEMORY_WINDOWS
2133 : malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
2134 : " %I64u madvise%s, %I64u page%s purged\n",
2135 : arena->ndirty, arena->ndirty == 1 ? "" : "s",
2136 : arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
2137 : arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
2138 : arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
2139 : # ifdef MALLOC_DECOMMIT
2140 : malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
2141 : " %I64u page%s decommitted\n",
2142 : arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
2143 : arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
2144 : arena->stats.decommitted,
2145 : (arena->stats.decommitted == 1) ? "" : "s");
2146 : # endif
2147 :
2148 : malloc_printf(" allocated nmalloc ndalloc\n");
2149 : malloc_printf("small: %12Iu %12I64u %12I64u\n",
2150 : arena->stats.allocated_small, arena->stats.nmalloc_small,
2151 : arena->stats.ndalloc_small);
2152 : malloc_printf("large: %12Iu %12I64u %12I64u\n",
2153 : arena->stats.allocated_large, arena->stats.nmalloc_large,
2154 : arena->stats.ndalloc_large);
2155 : malloc_printf("total: %12Iu %12I64u %12I64u\n",
2156 : arena->stats.allocated_small + arena->stats.allocated_large,
2157 : arena->stats.nmalloc_small + arena->stats.nmalloc_large,
2158 : arena->stats.ndalloc_small + arena->stats.ndalloc_large);
2159 : malloc_printf("mapped: %12Iu\n", arena->stats.mapped);
2160 : #else
2161 0 : malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
2162 : " %llu madvise%s, %llu page%s purged\n",
2163 0 : arena->ndirty, arena->ndirty == 1 ? "" : "s",
2164 0 : arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
2165 0 : arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
2166 0 : arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
2167 : # ifdef MALLOC_DECOMMIT
2168 : malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
2169 : " %llu page%s decommitted\n",
2170 : arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
2171 : arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
2172 : arena->stats.decommitted,
2173 : (arena->stats.decommitted == 1) ? "" : "s");
2174 : # endif
2175 :
2176 0 : malloc_printf(" allocated nmalloc ndalloc\n");
2177 0 : malloc_printf("small: %12zu %12llu %12llu\n",
2178 : arena->stats.allocated_small, arena->stats.nmalloc_small,
2179 : arena->stats.ndalloc_small);
2180 0 : malloc_printf("large: %12zu %12llu %12llu\n",
2181 : arena->stats.allocated_large, arena->stats.nmalloc_large,
2182 : arena->stats.ndalloc_large);
2183 0 : malloc_printf("total: %12zu %12llu %12llu\n",
2184 0 : arena->stats.allocated_small + arena->stats.allocated_large,
2185 0 : arena->stats.nmalloc_small + arena->stats.nmalloc_large,
2186 0 : arena->stats.ndalloc_small + arena->stats.ndalloc_large);
2187 0 : malloc_printf("mapped: %12zu\n", arena->stats.mapped);
2188 : #endif
2189 0 : malloc_printf("bins: bin size regs pgs requests newruns"
2190 : " reruns maxruns curruns\n");
2191 0 : for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
2192 0 : if (arena->bins[i].stats.nrequests == 0) {
2193 0 : if (gap_start == UINT_MAX)
2194 0 : gap_start = i;
2195 : } else {
2196 0 : if (gap_start != UINT_MAX) {
2197 0 : if (i > gap_start + 1) {
2198 : /* Gap of more than one size class. */
2199 0 : malloc_printf("[%u..%u]\n",
2200 : gap_start, i - 1);
2201 : } else {
2202 : /* Gap of one size class. */
2203 0 : malloc_printf("[%u]\n", gap_start);
2204 : }
2205 0 : gap_start = UINT_MAX;
2206 : }
2207 0 : malloc_printf(
2208 : #if defined(MOZ_MEMORY_WINDOWS)
2209 : "%13u %1s %4u %4u %3u %9I64u %9I64u"
2210 : " %9I64u %7u %7u\n",
2211 : #else
2212 : "%13u %1s %4u %4u %3u %9llu %9llu"
2213 : " %9llu %7lu %7lu\n",
2214 : #endif
2215 : i,
2216 0 : i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
2217 : arena->bins[i].reg_size,
2218 : arena->bins[i].nregs,
2219 0 : arena->bins[i].run_size >> pagesize_2pow,
2220 : arena->bins[i].stats.nrequests,
2221 : arena->bins[i].stats.nruns,
2222 : arena->bins[i].stats.reruns,
2223 : arena->bins[i].stats.highruns,
2224 : arena->bins[i].stats.curruns);
2225 : }
2226 : }
2227 0 : if (gap_start != UINT_MAX) {
2228 0 : if (i > gap_start + 1) {
2229 : /* Gap of more than one size class. */
2230 0 : malloc_printf("[%u..%u]\n", gap_start, i - 1);
2231 : } else {
2232 : /* Gap of one size class. */
2233 0 : malloc_printf("[%u]\n", gap_start);
2234 : }
2235 : }
2236 0 : }
2237 : #endif
2238 :
2239 : /*
2240 : * End Utility functions/macros.
2241 : */
2242 : /******************************************************************************/
2243 : /*
2244 : * Begin extent tree code.
2245 : */
2246 :
2247 : static inline int
2248 0 : extent_szad_comp(extent_node_t *a, extent_node_t *b)
2249 : {
2250 : int ret;
2251 0 : size_t a_size = a->size;
2252 0 : size_t b_size = b->size;
2253 :
2254 0 : ret = (a_size > b_size) - (a_size < b_size);
2255 0 : if (ret == 0) {
2256 0 : uintptr_t a_addr = (uintptr_t)a->addr;
2257 0 : uintptr_t b_addr = (uintptr_t)b->addr;
2258 :
2259 0 : ret = (a_addr > b_addr) - (a_addr < b_addr);
2260 : }
2261 :
2262 0 : return (ret);
2263 : }
2264 :
2265 : /* Wrap red-black tree macros in functions. */
2266 0 : rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
2267 0 : link_szad, extent_szad_comp)
2268 :
2269 : static inline int
2270 28261 : extent_ad_comp(extent_node_t *a, extent_node_t *b)
2271 : {
2272 28261 : uintptr_t a_addr = (uintptr_t)a->addr;
2273 28261 : uintptr_t b_addr = (uintptr_t)b->addr;
2274 :
2275 28261 : return ((a_addr > b_addr) - (a_addr < b_addr));
2276 : }
2277 :
2278 : /* Wrap red-black tree macros in functions. */
2279 33897 : rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
2280 33897 : extent_ad_comp)
2281 :
2282 : /*
2283 : * End extent tree code.
2284 : */
2285 : /******************************************************************************/
2286 : /*
2287 : * Begin chunk management functions.
2288 : */
2289 :
2290 : #ifdef MOZ_MEMORY_WINDOWS
2291 :
2292 : static void *
2293 : pages_map(void *addr, size_t size, int pfd)
2294 : {
2295 : void *ret = NULL;
2296 : ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
2297 : PAGE_READWRITE);
2298 : return (ret);
2299 : }
2300 :
2301 : static void
2302 : pages_unmap(void *addr, size_t size)
2303 : {
2304 : if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
2305 : _malloc_message(_getprogname(),
2306 : ": (malloc) Error in VirtualFree()\n", "", "");
2307 : if (opt_abort)
2308 : abort();
2309 : }
2310 : }
2311 : #else
2312 : #ifdef JEMALLOC_USES_MAP_ALIGN
2313 : static void *
2314 : pages_map_align(size_t size, int pfd, size_t alignment)
2315 : {
2316 : void *ret;
2317 :
2318 : /*
2319 : * We don't use MAP_FIXED here, because it can cause the *replacement*
2320 : * of existing mappings, and we only want to create new mappings.
2321 : */
2322 : #ifdef MALLOC_PAGEFILE
2323 : if (pfd != -1) {
2324 : ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2325 : MAP_NOSYNC | MAP_ALIGN, pfd, 0);
2326 : } else
2327 : #endif
2328 : {
2329 : ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2330 : MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
2331 : }
2332 : assert(ret != NULL);
2333 :
2334 : if (ret == MAP_FAILED)
2335 : ret = NULL;
2336 : return (ret);
2337 : }
2338 : #endif
2339 :
2340 : static void *
2341 64774 : pages_map(void *addr, size_t size, int pfd)
2342 : {
2343 : void *ret;
2344 : #if defined(__ia64__)
2345 : /*
2346 : * The JS engine assumes that all allocated pointers have their high 17 bits clear,
2347 : * which ia64's mmap doesn't support directly. However, we can emulate it by passing
2348 : * mmap an "addr" parameter with those bits clear. The mmap will return that address,
2349 : * or the nearest available memory above that address, providing a near-guarantee
2350 : * that those bits are clear. If they are not, we return NULL below to indicate
2351 : * out-of-memory.
2352 : *
2353 : * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual
2354 : * address space.
2355 : *
2356 : * See Bug 589735 for more information.
2357 : */
2358 : bool check_placement = true;
2359 : if (addr == NULL) {
2360 : addr = (void*)0x0000070000000000;
2361 : check_placement = false;
2362 : }
2363 : #endif
2364 :
2365 : /*
2366 : * We don't use MAP_FIXED here, because it can cause the *replacement*
2367 : * of existing mappings, and we only want to create new mappings.
2368 : */
2369 : #ifdef MALLOC_PAGEFILE
2370 : if (pfd != -1) {
2371 : ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2372 : MAP_NOSYNC, pfd, 0);
2373 : } else
2374 : #endif
2375 : {
2376 64774 : ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2377 : MAP_ANON, -1, 0);
2378 : }
2379 64774 : assert(ret != NULL);
2380 :
2381 64774 : if (ret == MAP_FAILED) {
2382 18 : ret = NULL;
2383 : }
2384 : #if defined(__ia64__)
2385 : /*
2386 : * If the allocated memory doesn't have its upper 17 bits clear, consider it
2387 : * as out of memory.
2388 : */
2389 : else if ((long long)ret & 0xffff800000000000) {
2390 : munmap(ret, size);
2391 : ret = NULL;
2392 : }
2393 : /* If the caller requested a specific memory location, verify that's what mmap returned. */
2394 : else if (check_placement && ret != addr) {
2395 : #else
2396 64756 : else if (addr != NULL && ret != addr) {
2397 : #endif
2398 : /*
2399 : * We succeeded in mapping memory, but not in the right place.
2400 : */
2401 0 : if (munmap(ret, size) == -1) {
2402 : char buf[STRERROR_BUF];
2403 :
2404 0 : strerror_r(errno, buf, sizeof(buf));
2405 0 : _malloc_message(_getprogname(),
2406 : ": (malloc) Error in munmap(): ", buf, "\n");
2407 0 : if (opt_abort)
2408 0 : abort();
2409 : }
2410 0 : ret = NULL;
2411 : }
2412 :
2413 : #if defined(__ia64__)
2414 : assert(ret == NULL || (!check_placement && ret != NULL)
2415 : || (check_placement && ret == addr));
2416 : #else
2417 64774 : assert(ret == NULL || (addr == NULL && ret != addr)
2418 : || (addr != NULL && ret == addr));
2419 : #endif
2420 64774 : return (ret);
2421 : }
2422 :
2423 : static void
2424 27462 : pages_unmap(void *addr, size_t size)
2425 : {
2426 :
2427 27462 : if (munmap(addr, size) == -1) {
2428 : char buf[STRERROR_BUF];
2429 :
2430 0 : strerror_r(errno, buf, sizeof(buf));
2431 0 : _malloc_message(_getprogname(),
2432 : ": (malloc) Error in munmap(): ", buf, "\n");
2433 0 : if (opt_abort)
2434 0 : abort();
2435 : }
2436 27462 : }
2437 : #endif
2438 :
2439 : #ifdef MOZ_MEMORY_DARWIN
2440 : #define VM_COPY_MIN (pagesize << 5)
2441 : static inline void
2442 : pages_copy(void *dest, const void *src, size_t n)
2443 : {
2444 :
2445 : assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
2446 : assert(n >= VM_COPY_MIN);
2447 : assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
2448 :
2449 : vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
2450 : (vm_address_t)dest);
2451 : }
2452 : #endif
2453 :
2454 : #ifdef MALLOC_VALIDATE
2455 : static inline malloc_rtree_t *
2456 18669 : malloc_rtree_new(unsigned bits)
2457 : {
2458 : malloc_rtree_t *ret;
2459 : unsigned bits_per_level, height, i;
2460 :
2461 37338 : bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
2462 18669 : sizeof(void *)))) - 1;
2463 18669 : height = bits / bits_per_level;
2464 18669 : if (height * bits_per_level != bits)
2465 0 : height++;
2466 18669 : assert(height * bits_per_level >= bits);
2467 :
2468 18669 : ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) +
2469 : (sizeof(unsigned) * (height - 1)));
2470 18669 : if (ret == NULL)
2471 0 : return (NULL);
2472 :
2473 18669 : malloc_spin_init(&ret->lock);
2474 18669 : ret->height = height;
2475 18669 : if (bits_per_level * height > bits)
2476 0 : ret->level2bits[0] = bits % bits_per_level;
2477 : else
2478 18669 : ret->level2bits[0] = bits_per_level;
2479 18669 : for (i = 1; i < height; i++)
2480 0 : ret->level2bits[i] = bits_per_level;
2481 :
2482 18669 : ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
2483 18669 : if (ret->root == NULL) {
2484 : /*
2485 : * We leak the rtree here, since there's no generic base
2486 : * deallocation.
2487 : */
2488 0 : return (NULL);
2489 : }
2490 :
2491 18669 : return (ret);
2492 : }
2493 :
2494 : #define MALLOC_RTREE_GET_GENERATE(f) \
2495 : /* The least significant bits of the key are ignored. */ \
2496 : static inline void * \
2497 : f(malloc_rtree_t *rtree, uintptr_t key) \
2498 : { \
2499 : void *ret; \
2500 : uintptr_t subkey; \
2501 : unsigned i, lshift, height, bits; \
2502 : void **node, **child; \
2503 : \
2504 : MALLOC_RTREE_LOCK(&rtree->lock); \
2505 : for (i = lshift = 0, height = rtree->height, node = rtree->root;\
2506 : i < height - 1; \
2507 : i++, lshift += bits, node = child) { \
2508 : bits = rtree->level2bits[i]; \
2509 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
2510 : child = (void**)node[subkey]; \
2511 : if (child == NULL) { \
2512 : MALLOC_RTREE_UNLOCK(&rtree->lock); \
2513 : return (NULL); \
2514 : } \
2515 : } \
2516 : \
2517 : /* \
2518 : * node is a leaf, so it contains values rather than node \
2519 : * pointers. \
2520 : */ \
2521 : bits = rtree->level2bits[i]; \
2522 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
2523 : ret = node[subkey]; \
2524 : MALLOC_RTREE_UNLOCK(&rtree->lock); \
2525 : \
2526 : MALLOC_RTREE_GET_VALIDATE \
2527 : return (ret); \
2528 : }
2529 :
2530 : #ifdef MALLOC_DEBUG
2531 : # define MALLOC_RTREE_LOCK(l) malloc_spin_lock(l)
2532 : # define MALLOC_RTREE_UNLOCK(l) malloc_spin_unlock(l)
2533 : # define MALLOC_RTREE_GET_VALIDATE
2534 0 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get_locked)
2535 : # undef MALLOC_RTREE_LOCK
2536 : # undef MALLOC_RTREE_UNLOCK
2537 : # undef MALLOC_RTREE_GET_VALIDATE
2538 : #endif
2539 :
2540 : #define MALLOC_RTREE_LOCK(l)
2541 : #define MALLOC_RTREE_UNLOCK(l)
2542 : #ifdef MALLOC_DEBUG
2543 : /*
2544 : * Suppose that it were possible for a jemalloc-allocated chunk to be
2545 : * munmap()ped, followed by a different allocator in another thread re-using
2546 : * overlapping virtual memory, all without invalidating the cached rtree
2547 : * value. The result would be a false positive (the rtree would claim that
2548 : * jemalloc owns memory that it had actually discarded). I don't think this
2549 : * scenario is possible, but the following assertion is a prudent sanity
2550 : * check.
2551 : */
2552 : # define MALLOC_RTREE_GET_VALIDATE \
2553 : assert(malloc_rtree_get_locked(rtree, key) == ret);
2554 : #else
2555 : # define MALLOC_RTREE_GET_VALIDATE
2556 : #endif
2557 0 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)
2558 : #undef MALLOC_RTREE_LOCK
2559 : #undef MALLOC_RTREE_UNLOCK
2560 : #undef MALLOC_RTREE_GET_VALIDATE
2561 :
2562 : static inline bool
2563 33224 : malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
2564 : {
2565 : uintptr_t subkey;
2566 : unsigned i, lshift, height, bits;
2567 : void **node, **child;
2568 :
2569 33224 : malloc_spin_lock(&rtree->lock);
2570 66448 : for (i = lshift = 0, height = rtree->height, node = rtree->root;
2571 33224 : i < height - 1;
2572 0 : i++, lshift += bits, node = child) {
2573 0 : bits = rtree->level2bits[i];
2574 0 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
2575 0 : child = (void**)node[subkey];
2576 0 : if (child == NULL) {
2577 0 : child = (void**)base_calloc(1, sizeof(void *) <<
2578 0 : rtree->level2bits[i+1]);
2579 0 : if (child == NULL) {
2580 0 : malloc_spin_unlock(&rtree->lock);
2581 0 : return (true);
2582 : }
2583 0 : node[subkey] = child;
2584 : }
2585 : }
2586 :
2587 : /* node is a leaf, so it contains values rather than node pointers. */
2588 33224 : bits = rtree->level2bits[i];
2589 33224 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
2590 33224 : node[subkey] = val;
2591 33224 : malloc_spin_unlock(&rtree->lock);
2592 :
2593 33224 : return (false);
2594 : }
2595 : #endif
2596 :
2597 : #if defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)
2598 :
2599 : /* Allocate an aligned chunk while maintaining a 1:1 correspondence between
2600 : * mmap and unmap calls. This is important on Windows, but not elsewhere. */
2601 : static void *
2602 : chunk_alloc_mmap(size_t size, bool pagefile)
2603 : {
2604 : void *ret;
2605 : #ifndef JEMALLOC_USES_MAP_ALIGN
2606 : size_t offset;
2607 : #endif
2608 : int pfd;
2609 :
2610 : #ifdef MALLOC_PAGEFILE
2611 : if (opt_pagefile && pagefile) {
2612 : pfd = pagefile_init(size);
2613 : if (pfd == -1)
2614 : return (NULL);
2615 : } else
2616 : #endif
2617 : pfd = -1;
2618 :
2619 : #ifdef JEMALLOC_USES_MAP_ALIGN
2620 : ret = pages_map_align(size, pfd, chunksize);
2621 : #else
2622 : ret = pages_map(NULL, size, pfd);
2623 : if (ret == NULL)
2624 : goto RETURN;
2625 :
2626 : offset = CHUNK_ADDR2OFFSET(ret);
2627 : if (offset != 0) {
2628 : /* Deallocate, then try to allocate at (ret + size - offset). */
2629 : pages_unmap(ret, size);
2630 : ret = pages_map((void *)((uintptr_t)ret + size - offset), size,
2631 : pfd);
2632 : while (ret == NULL) {
2633 : /*
2634 : * Over-allocate in order to map a memory region that
2635 : * is definitely large enough.
2636 : */
2637 : ret = pages_map(NULL, size + chunksize, -1);
2638 : if (ret == NULL)
2639 : goto RETURN;
2640 : /*
2641 : * Deallocate, then allocate the correct size, within
2642 : * the over-sized mapping.
2643 : */
2644 : offset = CHUNK_ADDR2OFFSET(ret);
2645 : pages_unmap(ret, size + chunksize);
2646 : if (offset == 0)
2647 : ret = pages_map(ret, size, pfd);
2648 : else {
2649 : ret = pages_map((void *)((uintptr_t)ret +
2650 : chunksize - offset), size, pfd);
2651 : }
2652 : /*
2653 : * Failure here indicates a race with another thread, so
2654 : * try again.
2655 : */
2656 : }
2657 : }
2658 : RETURN:
2659 : #endif
2660 : #ifdef MALLOC_PAGEFILE
2661 : if (pfd != -1)
2662 : pagefile_close(pfd);
2663 : #endif
2664 : #ifdef MALLOC_STATS
2665 : if (ret != NULL)
2666 : stats_chunks.nchunks += (size / chunksize);
2667 : #endif
2668 : return (ret);
2669 : }
2670 :
2671 : #else /* ! (defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)) */
2672 :
2673 : /*
2674 : * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
2675 : * potentially avoid some system calls.
2676 : */
2677 : #ifndef NO_TLS
2678 : static __thread bool mmap_unaligned_tls __attribute__((tls_model("initial-exec")));
2679 : #define MMAP_UNALIGNED_GET() mmap_unaligned_tls
2680 : #define MMAP_UNALIGNED_SET(v) do { \
2681 : mmap_unaligned_tls = (v); \
2682 : } while (0)
2683 : #else
2684 : #define NEEDS_PTHREAD_MMAP_UNALIGNED_TSD
2685 : static pthread_key_t mmap_unaligned_tsd;
2686 : #define MMAP_UNALIGNED_GET() ((bool)pthread_getspecific(mmap_unaligned_tsd))
2687 : #define MMAP_UNALIGNED_SET(v) do { \
2688 : pthread_setspecific(mmap_unaligned_tsd, (void *)(v)); \
2689 : } while (0)
2690 : #endif
2691 :
2692 : /* chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked from upstream
2693 : * jemalloc 2.2.3 to fix Mozilla bug 694896, enable jemalloc on Mac 10.7. */
2694 :
2695 : static void *
2696 2961 : chunk_alloc_mmap_slow(size_t size, bool unaligned)
2697 : {
2698 : void *ret;
2699 : size_t offset;
2700 :
2701 : /* Beware size_t wrap-around. */
2702 2961 : if (size + chunksize <= size)
2703 0 : return (NULL);
2704 :
2705 2961 : ret = pages_map(NULL, size + chunksize, -1);
2706 2961 : if (ret == NULL)
2707 0 : return (NULL);
2708 :
2709 : /* Clean up unneeded leading/trailing space. */
2710 2961 : offset = CHUNK_ADDR2OFFSET(ret);
2711 2961 : if (offset != 0) {
2712 : /* Note that mmap() returned an unaligned mapping. */
2713 10 : unaligned = true;
2714 :
2715 : /* Leading space. */
2716 10 : pages_unmap(ret, chunksize - offset);
2717 :
2718 10 : ret = (void *)((uintptr_t)ret +
2719 10 : (chunksize - offset));
2720 :
2721 : /* Trailing space. */
2722 10 : pages_unmap((void *)((uintptr_t)ret + size),
2723 : offset);
2724 : } else {
2725 : /* Trailing space only. */
2726 2951 : pages_unmap((void *)((uintptr_t)ret + size),
2727 : chunksize);
2728 : }
2729 :
2730 : /*
2731 : * If mmap() returned an aligned mapping, reset mmap_unaligned so that
2732 : * the next chunk_alloc_mmap() execution tries the fast allocation
2733 : * method.
2734 : */
2735 2961 : if (unaligned == false)
2736 2951 : MMAP_UNALIGNED_SET(false);
2737 :
2738 2961 : return (ret);
2739 : }
2740 :
2741 : static void *
2742 27428 : chunk_alloc_mmap(size_t size, bool pagefile)
2743 : {
2744 : void *ret;
2745 :
2746 : /*
2747 : * Ideally, there would be a way to specify alignment to mmap() (like
2748 : * NetBSD has), but in the absence of such a feature, we have to work
2749 : * hard to efficiently create aligned mappings. The reliable, but
2750 : * slow method is to create a mapping that is over-sized, then trim the
2751 : * excess. However, that always results in at least one call to
2752 : * pages_unmap().
2753 : *
2754 : * A more optimistic approach is to try mapping precisely the right
2755 : * amount, then try to append another mapping if alignment is off. In
2756 : * practice, this works out well as long as the application is not
2757 : * interleaving mappings via direct mmap() calls. If we do run into a
2758 : * situation where there is an interleaved mapping and we are unable to
2759 : * extend an unaligned mapping, our best option is to switch to the
2760 : * slow method until mmap() returns another aligned mapping. This will
2761 : * tend to leave a gap in the memory map that is too small to cause
2762 : * later problems for the optimistic method.
2763 : *
2764 : * Another possible confounding factor is address space layout
2765 : * randomization (ASLR), which causes mmap(2) to disregard the
2766 : * requested address. mmap_unaligned tracks whether the previous
2767 : * chunk_alloc_mmap() execution received any unaligned or relocated
2768 : * mappings, and if so, the current execution will immediately fall
2769 : * back to the slow method. However, we keep track of whether the fast
2770 : * method would have succeeded, and if so, we make a note to try the
2771 : * fast method next time.
2772 : */
2773 :
2774 27428 : if (MMAP_UNALIGNED_GET() == false) {
2775 : size_t offset;
2776 :
2777 24467 : ret = pages_map(NULL, size, -1);
2778 24467 : if (ret == NULL)
2779 18 : return (NULL);
2780 :
2781 24449 : offset = CHUNK_ADDR2OFFSET(ret);
2782 24449 : if (offset != 0) {
2783 18677 : MMAP_UNALIGNED_SET(true);
2784 : /* Try to extend chunk boundary. */
2785 18677 : if (pages_map((void *)((uintptr_t)ret + size),
2786 : chunksize - offset, -1) == NULL) {
2787 : /*
2788 : * Extension failed. Clean up, then revert to
2789 : * the reliable-but-expensive method.
2790 : */
2791 0 : pages_unmap(ret, size);
2792 0 : ret = chunk_alloc_mmap_slow(size, true);
2793 : } else {
2794 : /* Clean up unneeded leading space. */
2795 18677 : pages_unmap(ret, chunksize - offset);
2796 18677 : ret = (void *)((uintptr_t)ret + (chunksize -
2797 : offset));
2798 : }
2799 : }
2800 : } else
2801 2961 : ret = chunk_alloc_mmap_slow(size, false);
2802 :
2803 27410 : return (ret);
2804 : }
2805 :
2806 : #endif /* defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE) */
2807 :
2808 : #ifdef MALLOC_PAGEFILE
2809 : static int
2810 : pagefile_init(size_t size)
2811 : {
2812 : int ret;
2813 : size_t i;
2814 : char pagefile_path[PATH_MAX];
2815 : char zbuf[MALLOC_PAGEFILE_WRITE_SIZE];
2816 :
2817 : /*
2818 : * Create a temporary file, then immediately unlink it so that it will
2819 : * not persist.
2820 : */
2821 : strcpy(pagefile_path, pagefile_templ);
2822 : ret = mkstemp(pagefile_path);
2823 : if (ret == -1)
2824 : return (ret);
2825 : if (unlink(pagefile_path)) {
2826 : char buf[STRERROR_BUF];
2827 :
2828 : strerror_r(errno, buf, sizeof(buf));
2829 : _malloc_message(_getprogname(), ": (malloc) Error in unlink(\"",
2830 : pagefile_path, "\"):");
2831 : _malloc_message(buf, "\n", "", "");
2832 : if (opt_abort)
2833 : abort();
2834 : }
2835 :
2836 : /*
2837 : * Write sequential zeroes to the file in order to assure that disk
2838 : * space is committed, with minimal fragmentation. It would be
2839 : * sufficient to write one zero per disk block, but that potentially
2840 : * results in more system calls, for no real gain.
2841 : */
2842 : memset(zbuf, 0, sizeof(zbuf));
2843 : for (i = 0; i < size; i += sizeof(zbuf)) {
2844 : if (write(ret, zbuf, sizeof(zbuf)) != sizeof(zbuf)) {
2845 : if (errno != ENOSPC) {
2846 : char buf[STRERROR_BUF];
2847 :
2848 : strerror_r(errno, buf, sizeof(buf));
2849 : _malloc_message(_getprogname(),
2850 : ": (malloc) Error in write(): ", buf, "\n");
2851 : if (opt_abort)
2852 : abort();
2853 : }
2854 : pagefile_close(ret);
2855 : return (-1);
2856 : }
2857 : }
2858 :
2859 : return (ret);
2860 : }
2861 :
2862 : static void
2863 : pagefile_close(int pfd)
2864 : {
2865 :
2866 : if (close(pfd)) {
2867 : char buf[STRERROR_BUF];
2868 :
2869 : strerror_r(errno, buf, sizeof(buf));
2870 : _malloc_message(_getprogname(),
2871 : ": (malloc) Error in close(): ", buf, "\n");
2872 : if (opt_abort)
2873 : abort();
2874 : }
2875 : }
2876 : #endif
2877 :
2878 : static void *
2879 27428 : chunk_alloc(size_t size, bool zero, bool pagefile)
2880 : {
2881 : void *ret;
2882 :
2883 27428 : assert(size != 0);
2884 27428 : assert((size & chunksize_mask) == 0);
2885 :
2886 27428 : ret = chunk_alloc_mmap(size, pagefile);
2887 27428 : if (ret != NULL) {
2888 27410 : goto RETURN;
2889 : }
2890 :
2891 : /* All strategies for allocation failed. */
2892 18 : ret = NULL;
2893 : RETURN:
2894 : #ifdef MALLOC_STATS
2895 27428 : if (ret != NULL)
2896 27410 : stats_chunks.curchunks += (size / chunksize);
2897 27428 : if (stats_chunks.curchunks > stats_chunks.highchunks)
2898 23539 : stats_chunks.highchunks = stats_chunks.curchunks;
2899 : #endif
2900 :
2901 : #ifdef MALLOC_VALIDATE
2902 27428 : if (ret != NULL) {
2903 27410 : if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
2904 0 : chunk_dealloc(ret, size);
2905 0 : return (NULL);
2906 : }
2907 : }
2908 : #endif
2909 :
2910 27428 : assert(CHUNK_ADDR2BASE(ret) == ret);
2911 27428 : return (ret);
2912 : }
2913 :
2914 : static void
2915 5814 : chunk_dealloc_mmap(void *chunk, size_t size)
2916 : {
2917 :
2918 5814 : pages_unmap(chunk, size);
2919 5814 : }
2920 :
2921 : static void
2922 5814 : chunk_dealloc(void *chunk, size_t size)
2923 : {
2924 :
2925 5814 : assert(chunk != NULL);
2926 5814 : assert(CHUNK_ADDR2BASE(chunk) == chunk);
2927 5814 : assert(size != 0);
2928 5814 : assert((size & chunksize_mask) == 0);
2929 :
2930 : #ifdef MALLOC_STATS
2931 5814 : stats_chunks.curchunks -= (size / chunksize);
2932 : #endif
2933 : #ifdef MALLOC_VALIDATE
2934 5814 : malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
2935 : #endif
2936 :
2937 5814 : chunk_dealloc_mmap(chunk, size);
2938 5814 : }
2939 :
2940 : /*
2941 : * End chunk management functions.
2942 : */
2943 : /******************************************************************************/
2944 : /*
2945 : * Begin arena.
2946 : */
2947 :
2948 : /*
2949 : * Choose an arena based on a per-thread value (fast-path code, calls slow-path
2950 : * code if necessary).
2951 : */
2952 : static inline arena_t *
2953 46372702 : choose_arena(void)
2954 : {
2955 : arena_t *ret;
2956 :
2957 : /*
2958 : * We can only use TLS if this is a PIC library, since for the static
2959 : * library version, libc's malloc is used by TLS allocation, which
2960 : * introduces a bootstrapping issue.
2961 : */
2962 : #ifndef NO_TLS
2963 46372702 : if (isthreaded == false) {
2964 : /* Avoid the overhead of TLS for single-threaded operation. */
2965 0 : return (arenas[0]);
2966 : }
2967 :
2968 : # ifdef MOZ_MEMORY_WINDOWS
2969 : ret = (arena_t*)TlsGetValue(tlsIndex);
2970 : # else
2971 46372702 : ret = arenas_map;
2972 : # endif
2973 :
2974 46372702 : if (ret == NULL) {
2975 5 : ret = choose_arena_hard();
2976 5 : assert(ret != NULL);
2977 : }
2978 : #else
2979 : if (isthreaded && narenas > 1) {
2980 : unsigned long ind;
2981 :
2982 : /*
2983 : * Hash _pthread_self() to one of the arenas. There is a prime
2984 : * number of arenas, so this has a reasonable chance of
2985 : * working. Even so, the hashing can be easily thwarted by
2986 : * inconvenient _pthread_self() values. Without specific
2987 : * knowledge of how _pthread_self() calculates values, we can't
2988 : * easily do much better than this.
2989 : */
2990 : ind = (unsigned long) _pthread_self() % narenas;
2991 :
2992 : /*
2993 : * Optimistially assume that arenas[ind] has been initialized.
2994 : * At worst, we find out that some other thread has already
2995 : * done so, after acquiring the lock in preparation. Note that
2996 : * this lazy locking also has the effect of lazily forcing
2997 : * cache coherency; without the lock acquisition, there's no
2998 : * guarantee that modification of arenas[ind] by another thread
2999 : * would be seen on this CPU for an arbitrary amount of time.
3000 : *
3001 : * In general, this approach to modifying a synchronized value
3002 : * isn't a good idea, but in this case we only ever modify the
3003 : * value once, so things work out well.
3004 : */
3005 : ret = arenas[ind];
3006 : if (ret == NULL) {
3007 : /*
3008 : * Avoid races with another thread that may have already
3009 : * initialized arenas[ind].
3010 : */
3011 : malloc_spin_lock(&arenas_lock);
3012 : if (arenas[ind] == NULL)
3013 : ret = arenas_extend((unsigned)ind);
3014 : else
3015 : ret = arenas[ind];
3016 : malloc_spin_unlock(&arenas_lock);
3017 : }
3018 : } else
3019 : ret = arenas[0];
3020 : #endif
3021 :
3022 46372702 : assert(ret != NULL);
3023 46372702 : return (ret);
3024 : }
3025 :
3026 : #ifndef NO_TLS
3027 : /*
3028 : * Choose an arena based on a per-thread value (slow-path code only, called
3029 : * only by choose_arena()).
3030 : */
3031 : static arena_t *
3032 5 : choose_arena_hard(void)
3033 : {
3034 : arena_t *ret;
3035 :
3036 5 : assert(isthreaded);
3037 :
3038 : #ifdef MALLOC_BALANCE
3039 : /* Seed the PRNG used for arena load balancing. */
3040 : SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
3041 : #endif
3042 :
3043 5 : if (narenas > 1) {
3044 : #ifdef MALLOC_BALANCE
3045 : unsigned ind;
3046 :
3047 : ind = PRN(balance, narenas_2pow);
3048 : if ((ret = arenas[ind]) == NULL) {
3049 : malloc_spin_lock(&arenas_lock);
3050 : if ((ret = arenas[ind]) == NULL)
3051 : ret = arenas_extend(ind);
3052 : malloc_spin_unlock(&arenas_lock);
3053 : }
3054 : #else
3055 0 : malloc_spin_lock(&arenas_lock);
3056 0 : if ((ret = arenas[next_arena]) == NULL)
3057 0 : ret = arenas_extend(next_arena);
3058 0 : next_arena = (next_arena + 1) % narenas;
3059 0 : malloc_spin_unlock(&arenas_lock);
3060 : #endif
3061 : } else
3062 5 : ret = arenas[0];
3063 :
3064 : #ifdef MOZ_MEMORY_WINDOWS
3065 : TlsSetValue(tlsIndex, ret);
3066 : #else
3067 5 : arenas_map = ret;
3068 : #endif
3069 :
3070 5 : return (ret);
3071 : }
3072 : #endif
3073 :
3074 : static inline int
3075 194091 : arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
3076 : {
3077 194091 : uintptr_t a_chunk = (uintptr_t)a;
3078 194091 : uintptr_t b_chunk = (uintptr_t)b;
3079 :
3080 194091 : assert(a != NULL);
3081 194091 : assert(b != NULL);
3082 :
3083 194091 : return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
3084 : }
3085 :
3086 : /* Wrap red-black tree macros in functions. */
3087 313285 : rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
3088 313285 : arena_chunk_t, link_dirty, arena_chunk_comp)
3089 :
3090 : static inline int
3091 1479607 : arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
3092 : {
3093 1479607 : uintptr_t a_mapelm = (uintptr_t)a;
3094 1479607 : uintptr_t b_mapelm = (uintptr_t)b;
3095 :
3096 1479607 : assert(a != NULL);
3097 1479607 : assert(b != NULL);
3098 :
3099 1479607 : return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
3100 : }
3101 :
3102 : /* Wrap red-black tree macros in functions. */
3103 4897235 : rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
3104 4897235 : arena_run_comp)
3105 :
3106 : static inline int
3107 27108048 : arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
3108 : {
3109 : int ret;
3110 27108048 : size_t a_size = a->bits & ~pagesize_mask;
3111 27108048 : size_t b_size = b->bits & ~pagesize_mask;
3112 :
3113 27108048 : ret = (a_size > b_size) - (a_size < b_size);
3114 27108048 : if (ret == 0) {
3115 : uintptr_t a_mapelm, b_mapelm;
3116 :
3117 7525083 : if ((a->bits & CHUNK_MAP_KEY) == 0)
3118 7407268 : a_mapelm = (uintptr_t)a;
3119 : else {
3120 : /*
3121 : * Treat keys as though they are lower than anything
3122 : * else.
3123 : */
3124 117815 : a_mapelm = 0;
3125 : }
3126 7525083 : b_mapelm = (uintptr_t)b;
3127 :
3128 7525083 : ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
3129 : }
3130 :
3131 27108048 : return (ret);
3132 : }
3133 :
3134 : /* Wrap red-black tree macros in functions. */
3135 17699161 : rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
3136 17699161 : arena_avail_comp)
3137 :
3138 : static inline void *
3139 45735071 : arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
3140 : {
3141 : void *ret;
3142 : unsigned i, mask, bit, regind;
3143 :
3144 45735071 : assert(run->magic == ARENA_RUN_MAGIC);
3145 45735071 : assert(run->regs_minelm < bin->regs_mask_nelms);
3146 :
3147 : /*
3148 : * Move the first check outside the loop, so that run->regs_minelm can
3149 : * be updated unconditionally, without the possibility of updating it
3150 : * multiple times.
3151 : */
3152 45735071 : i = run->regs_minelm;
3153 45735071 : mask = run->regs_mask[i];
3154 45735071 : if (mask != 0) {
3155 : /* Usable allocation found. */
3156 44908552 : bit = ffs((int)mask) - 1;
3157 :
3158 44908552 : regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
3159 44908552 : assert(regind < bin->nregs);
3160 89817104 : ret = (void *)(((uintptr_t)run) + bin->reg0_offset
3161 44908552 : + (bin->reg_size * regind));
3162 :
3163 : /* Clear bit. */
3164 44908552 : mask ^= (1U << bit);
3165 44908552 : run->regs_mask[i] = mask;
3166 :
3167 44908552 : return (ret);
3168 : }
3169 :
3170 979239 : for (i++; i < bin->regs_mask_nelms; i++) {
3171 979239 : mask = run->regs_mask[i];
3172 979239 : if (mask != 0) {
3173 : /* Usable allocation found. */
3174 826519 : bit = ffs((int)mask) - 1;
3175 :
3176 826519 : regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
3177 826519 : assert(regind < bin->nregs);
3178 1653038 : ret = (void *)(((uintptr_t)run) + bin->reg0_offset
3179 826519 : + (bin->reg_size * regind));
3180 :
3181 : /* Clear bit. */
3182 826519 : mask ^= (1U << bit);
3183 826519 : run->regs_mask[i] = mask;
3184 :
3185 : /*
3186 : * Make a note that nothing before this element
3187 : * contains a free region.
3188 : */
3189 826519 : run->regs_minelm = i; /* Low payoff: + (mask == 0); */
3190 :
3191 826519 : return (ret);
3192 : }
3193 : }
3194 : /* Not reached. */
3195 0 : assert(0);
3196 : return (NULL);
3197 : }
3198 :
3199 : static inline void
3200 43659702 : arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
3201 : {
3202 : /*
3203 : * To divide by a number D that is not a power of two we multiply
3204 : * by (2^21 / D) and then right shift by 21 positions.
3205 : *
3206 : * X / D
3207 : *
3208 : * becomes
3209 : *
3210 : * (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
3211 : */
3212 : #define SIZE_INV_SHIFT 21
3213 : #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
3214 : static const unsigned size_invs[] = {
3215 : SIZE_INV(3),
3216 : SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
3217 : SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
3218 : SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
3219 : SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
3220 : SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
3221 : SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
3222 : SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
3223 : #if (QUANTUM_2POW_MIN < 4)
3224 : ,
3225 : SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
3226 : SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
3227 : SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
3228 : SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
3229 : SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
3230 : SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
3231 : SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
3232 : SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
3233 : #endif
3234 : };
3235 : unsigned diff, regind, elm, bit;
3236 :
3237 43659702 : assert(run->magic == ARENA_RUN_MAGIC);
3238 : assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
3239 : >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
3240 :
3241 : /*
3242 : * Avoid doing division with a variable divisor if possible. Using
3243 : * actual division here can reduce allocator throughput by over 20%!
3244 : */
3245 43659702 : diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
3246 43659702 : if ((size & (size - 1)) == 0) {
3247 : /*
3248 : * log2_table allows fast division of a power of two in the
3249 : * [1..128] range.
3250 : *
3251 : * (x / divisor) becomes (x >> log2_table[divisor - 1]).
3252 : */
3253 : static const unsigned char log2_table[] = {
3254 : 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
3255 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
3256 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3257 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
3258 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3259 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3260 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3261 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
3262 : };
3263 :
3264 34733586 : if (size <= 128)
3265 28967596 : regind = (diff >> log2_table[size - 1]);
3266 5765990 : else if (size <= 32768)
3267 5765990 : regind = diff >> (8 + log2_table[(size >> 8) - 1]);
3268 : else {
3269 : /*
3270 : * The run size is too large for us to use the lookup
3271 : * table. Use real division.
3272 : */
3273 0 : regind = diff / size;
3274 : }
3275 8926116 : } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
3276 : << QUANTUM_2POW_MIN) + 2) {
3277 8896621 : regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
3278 8896621 : regind >>= SIZE_INV_SHIFT;
3279 : } else {
3280 : /*
3281 : * size_invs isn't large enough to handle this size class, so
3282 : * calculate regind using actual division. This only happens
3283 : * if the user increases small_max via the 'S' runtime
3284 : * configuration option.
3285 : */
3286 29495 : regind = diff / size;
3287 : };
3288 43659702 : assert(diff == regind * size);
3289 43659702 : assert(regind < bin->nregs);
3290 :
3291 43659702 : elm = regind >> (SIZEOF_INT_2POW + 3);
3292 43659702 : if (elm < run->regs_minelm)
3293 624208 : run->regs_minelm = elm;
3294 43659702 : bit = regind - (elm << (SIZEOF_INT_2POW + 3));
3295 43659702 : assert((run->regs_mask[elm] & (1U << bit)) == 0);
3296 43659702 : run->regs_mask[elm] |= (1U << bit);
3297 : #undef SIZE_INV
3298 : #undef SIZE_INV_SHIFT
3299 43659702 : }
3300 :
3301 : static void
3302 3698730 : arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
3303 : bool zero)
3304 : {
3305 : arena_chunk_t *chunk;
3306 : size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
3307 :
3308 3698730 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
3309 3698730 : old_ndirty = chunk->ndirty;
3310 3698730 : run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
3311 : >> pagesize_2pow);
3312 3698730 : total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
3313 : pagesize_2pow;
3314 3698730 : need_pages = (size >> pagesize_2pow);
3315 3698730 : assert(need_pages > 0);
3316 3698730 : assert(need_pages <= total_pages);
3317 3698730 : rem_pages = total_pages - need_pages;
3318 :
3319 3698730 : arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
3320 :
3321 : /* Keep track of trailing unused pages for later use. */
3322 3698730 : if (rem_pages > 0) {
3323 7185680 : chunk->map[run_ind+need_pages].bits = (rem_pages <<
3324 3592840 : pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
3325 : pagesize_mask);
3326 7185680 : chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
3327 3592840 : pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
3328 : pagesize_mask);
3329 3592840 : arena_avail_tree_insert(&arena->runs_avail,
3330 3592840 : &chunk->map[run_ind+need_pages]);
3331 : }
3332 :
3333 21780038 : for (i = 0; i < need_pages; i++) {
3334 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
3335 : /*
3336 : * Commit decommitted pages if necessary. If a decommitted
3337 : * page is encountered, commit all needed adjacent decommitted
3338 : * pages in one operation, in order to reduce system call
3339 : * overhead.
3340 : */
3341 18081308 : if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) {
3342 : size_t j;
3343 :
3344 : /*
3345 : * Advance i+j to just past the index of the last page
3346 : * to commit. Clear CHUNK_MAP_DECOMMITTED and
3347 : * CHUNK_MAP_MADVISED along the way.
3348 : */
3349 11216558 : for (j = 0; i + j < need_pages && (chunk->map[run_ind +
3350 9221752 : i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) {
3351 : /* DECOMMITTED and MADVISED are mutually exclusive. */
3352 4610876 : assert(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED &&
3353 : chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED));
3354 :
3355 4610876 : chunk->map[run_ind + i + j].bits &=
3356 : ~CHUNK_MAP_MADVISED_OR_DECOMMITTED;
3357 : }
3358 :
3359 : # ifdef MALLOC_DECOMMIT
3360 : pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
3361 : << pagesize_2pow)), (j << pagesize_2pow));
3362 : # ifdef MALLOC_STATS
3363 : arena->stats.ncommit++;
3364 : # endif
3365 : # endif
3366 :
3367 : # ifdef MALLOC_STATS
3368 997403 : arena->stats.committed += j;
3369 : # endif
3370 :
3371 : # ifndef MALLOC_DECOMMIT
3372 : }
3373 : # else
3374 : } else /* No need to zero since commit zeros. */
3375 : # endif
3376 :
3377 : #endif
3378 :
3379 : /* Zero if necessary. */
3380 18081308 : if (zero) {
3381 790138 : if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
3382 : == 0) {
3383 : VALGRIND_MALLOCLIKE_BLOCK((void *)((uintptr_t)
3384 : chunk + ((run_ind + i) << pagesize_2pow)),
3385 : pagesize, 0, false);
3386 669225 : memset((void *)((uintptr_t)chunk + ((run_ind
3387 669225 : + i) << pagesize_2pow)), 0, pagesize);
3388 : VALGRIND_FREELIKE_BLOCK((void *)((uintptr_t)
3389 : chunk + ((run_ind + i) << pagesize_2pow)),
3390 : 0);
3391 : /* CHUNK_MAP_ZEROED is cleared below. */
3392 : }
3393 : }
3394 :
3395 : /* Update dirty page accounting. */
3396 18081308 : if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
3397 13470432 : chunk->ndirty--;
3398 13470432 : arena->ndirty--;
3399 : /* CHUNK_MAP_DIRTY is cleared below. */
3400 : }
3401 :
3402 : /* Initialize the chunk map. */
3403 18081308 : if (large) {
3404 4539996 : chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
3405 : | CHUNK_MAP_ALLOCATED;
3406 : } else {
3407 27082624 : chunk->map[run_ind + i].bits = (size_t)run
3408 13541312 : | CHUNK_MAP_ALLOCATED;
3409 : }
3410 : }
3411 :
3412 : /*
3413 : * Set the run size only in the first element for large runs. This is
3414 : * primarily a debugging aid, since the lack of size info for trailing
3415 : * pages only matters if the application tries to operate on an
3416 : * interior pointer.
3417 : */
3418 3698730 : if (large)
3419 646749 : chunk->map[run_ind].bits |= size;
3420 :
3421 3698730 : if (chunk->ndirty == 0 && old_ndirty > 0)
3422 133997 : arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
3423 3698730 : }
3424 :
3425 : static void
3426 22874 : arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
3427 : {
3428 : arena_run_t *run;
3429 : size_t i;
3430 :
3431 : VALGRIND_MALLOCLIKE_BLOCK(chunk, (arena_chunk_header_npages <<
3432 : pagesize_2pow), 0, false);
3433 : #ifdef MALLOC_STATS
3434 22874 : arena->stats.mapped += chunksize;
3435 : #endif
3436 :
3437 22874 : chunk->arena = arena;
3438 :
3439 : /*
3440 : * Claim that no pages are in use, since the header is merely overhead.
3441 : */
3442 22874 : chunk->ndirty = 0;
3443 :
3444 : /* Initialize the map to contain one maximal free untouched run. */
3445 22874 : run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
3446 : pagesize_2pow));
3447 45748 : for (i = 0; i < arena_chunk_header_npages; i++)
3448 22874 : chunk->map[i].bits = 0;
3449 22874 : chunk->map[i].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3450 5809996 : for (i++; i < chunk_npages-1; i++) {
3451 5787122 : chunk->map[i].bits = CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3452 : }
3453 22874 : chunk->map[chunk_npages-1].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3454 :
3455 : #ifdef MALLOC_DECOMMIT
3456 : /*
3457 : * Start out decommitted, in order to force a closer correspondence
3458 : * between dirty pages and committed untouched pages.
3459 : */
3460 : pages_decommit(run, arena_maxclass);
3461 : # ifdef MALLOC_STATS
3462 : arena->stats.ndecommit++;
3463 : arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
3464 : # endif
3465 : #endif
3466 : #ifdef MALLOC_STATS
3467 22874 : arena->stats.committed += arena_chunk_header_npages;
3468 : #endif
3469 :
3470 : /* Insert the run into the runs_avail tree. */
3471 22874 : arena_avail_tree_insert(&arena->runs_avail,
3472 22874 : &chunk->map[arena_chunk_header_npages]);
3473 :
3474 : #ifdef MALLOC_DOUBLE_PURGE
3475 : LinkedList_Init(&chunk->chunks_madvised_elem);
3476 : #endif
3477 22874 : }
3478 :
3479 : static void
3480 30235 : arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
3481 : {
3482 :
3483 30235 : if (arena->spare != NULL) {
3484 1278 : if (arena->spare->ndirty > 0) {
3485 2392 : arena_chunk_tree_dirty_remove(
3486 1196 : &chunk->arena->chunks_dirty, arena->spare);
3487 1196 : arena->ndirty -= arena->spare->ndirty;
3488 : #ifdef MALLOC_STATS
3489 1196 : arena->stats.committed -= arena->spare->ndirty;
3490 : #endif
3491 : }
3492 :
3493 : #ifdef MALLOC_DOUBLE_PURGE
3494 : /* This is safe to do even if arena->spare is not in the list. */
3495 : LinkedList_Remove(&arena->spare->chunks_madvised_elem);
3496 : #endif
3497 :
3498 : VALGRIND_FREELIKE_BLOCK(arena->spare, 0);
3499 1278 : chunk_dealloc((void *)arena->spare, chunksize);
3500 : #ifdef MALLOC_STATS
3501 1278 : arena->stats.mapped -= chunksize;
3502 1278 : arena->stats.committed -= arena_chunk_header_npages;
3503 : #endif
3504 : }
3505 :
3506 : /*
3507 : * Remove run from runs_avail, so that the arena does not use it.
3508 : * Dirty page flushing only uses the chunks_dirty tree, so leaving this
3509 : * chunk in the chunks_* trees is sufficient for that purpose.
3510 : */
3511 30235 : arena_avail_tree_remove(&arena->runs_avail,
3512 30235 : &chunk->map[arena_chunk_header_npages]);
3513 :
3514 30235 : arena->spare = chunk;
3515 30235 : }
3516 :
3517 : static arena_run_t *
3518 3689612 : arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
3519 : bool zero)
3520 : {
3521 : arena_run_t *run;
3522 : arena_chunk_map_t *mapelm, key;
3523 :
3524 3689612 : assert(size <= arena_maxclass);
3525 3689612 : assert((size & pagesize_mask) == 0);
3526 :
3527 : /* Search the arena's chunks for the lowest best fit. */
3528 3689612 : key.bits = size | CHUNK_MAP_KEY;
3529 3689612 : mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
3530 3689612 : if (mapelm != NULL) {
3531 3640705 : arena_chunk_t *chunk =
3532 3640705 : (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
3533 7281410 : size_t pageind = ((uintptr_t)mapelm -
3534 3640705 : (uintptr_t)chunk->map) /
3535 : sizeof(arena_chunk_map_t);
3536 :
3537 3640705 : run = (arena_run_t *)((uintptr_t)chunk + (pageind
3538 3640705 : << pagesize_2pow));
3539 3640705 : arena_run_split(arena, run, size, large, zero);
3540 3640705 : return (run);
3541 : }
3542 :
3543 48907 : if (arena->spare != NULL) {
3544 : /* Use the spare. */
3545 26033 : arena_chunk_t *chunk = arena->spare;
3546 26033 : arena->spare = NULL;
3547 26033 : run = (arena_run_t *)((uintptr_t)chunk +
3548 26033 : (arena_chunk_header_npages << pagesize_2pow));
3549 : /* Insert the run into the runs_avail tree. */
3550 26033 : arena_avail_tree_insert(&arena->runs_avail,
3551 26033 : &chunk->map[arena_chunk_header_npages]);
3552 26033 : arena_run_split(arena, run, size, large, zero);
3553 26033 : return (run);
3554 : }
3555 :
3556 : /*
3557 : * No usable runs. Create a new chunk from which to allocate
3558 : * the run.
3559 : */
3560 : {
3561 22874 : arena_chunk_t *chunk = (arena_chunk_t *)
3562 22874 : chunk_alloc(chunksize, true, true);
3563 22874 : if (chunk == NULL)
3564 0 : return (NULL);
3565 :
3566 22874 : arena_chunk_init(arena, chunk);
3567 22874 : run = (arena_run_t *)((uintptr_t)chunk +
3568 22874 : (arena_chunk_header_npages << pagesize_2pow));
3569 : }
3570 : /* Update page map. */
3571 22874 : arena_run_split(arena, run, size, large, zero);
3572 22874 : return (run);
3573 : }
3574 :
3575 : static void
3576 165 : arena_purge(arena_t *arena)
3577 : {
3578 : arena_chunk_t *chunk;
3579 : size_t i, npages;
3580 : #ifdef MALLOC_DEBUG
3581 165 : size_t ndirty = 0;
3582 1896 : rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
3583 : chunk) {
3584 1566 : ndirty += chunk->ndirty;
3585 1566 : } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
3586 165 : assert(ndirty == arena->ndirty);
3587 : #endif
3588 165 : assert(arena->ndirty > opt_dirty_max);
3589 :
3590 : #ifdef MALLOC_STATS
3591 165 : arena->stats.npurge++;
3592 : #endif
3593 :
3594 : /*
3595 : * Iterate downward through chunks until enough dirty memory has been
3596 : * purged. Terminate as soon as possible in order to minimize the
3597 : * number of system calls, even if a chunk has only been partially
3598 : * purged.
3599 : */
3600 1244 : while (arena->ndirty > (opt_dirty_max >> 1)) {
3601 : #ifdef MALLOC_DOUBLE_PURGE
3602 : bool madvised = false;
3603 : #endif
3604 914 : chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
3605 914 : assert(chunk != NULL);
3606 :
3607 73094 : for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
3608 72345 : assert(i >= arena_chunk_header_npages);
3609 :
3610 72345 : if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
3611 : #ifdef MALLOC_DECOMMIT
3612 : const size_t free_operation = CHUNK_MAP_DECOMMITTED;
3613 : #else
3614 1770 : const size_t free_operation = CHUNK_MAP_MADVISED;
3615 : #endif
3616 1770 : assert((chunk->map[i].bits &
3617 : CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
3618 1770 : chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
3619 : /* Find adjacent dirty run(s). */
3620 97438 : for (npages = 1;
3621 190931 : i > arena_chunk_header_npages &&
3622 95263 : (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY);
3623 93898 : npages++) {
3624 93898 : i--;
3625 93898 : assert((chunk->map[i].bits &
3626 : CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
3627 93898 : chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
3628 : }
3629 1770 : chunk->ndirty -= npages;
3630 1770 : arena->ndirty -= npages;
3631 :
3632 : #ifdef MALLOC_DECOMMIT
3633 : pages_decommit((void *)((uintptr_t)
3634 : chunk + (i << pagesize_2pow)),
3635 : (npages << pagesize_2pow));
3636 : # ifdef MALLOC_STATS
3637 : arena->stats.ndecommit++;
3638 : arena->stats.decommitted += npages;
3639 : # endif
3640 : #endif
3641 : #ifdef MALLOC_STATS
3642 1770 : arena->stats.committed -= npages;
3643 : #endif
3644 :
3645 : #ifndef MALLOC_DECOMMIT
3646 1770 : madvise((void *)((uintptr_t)chunk + (i <<
3647 : pagesize_2pow)), (npages << pagesize_2pow),
3648 : MADV_FREE);
3649 : # ifdef MALLOC_DOUBLE_PURGE
3650 : madvised = true;
3651 : # endif
3652 : #endif
3653 : #ifdef MALLOC_STATS
3654 1770 : arena->stats.nmadvise++;
3655 1770 : arena->stats.purged += npages;
3656 : #endif
3657 1770 : if (arena->ndirty <= (opt_dirty_max >> 1))
3658 165 : break;
3659 : }
3660 : }
3661 :
3662 914 : if (chunk->ndirty == 0) {
3663 860 : arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
3664 : chunk);
3665 : }
3666 : #ifdef MALLOC_DOUBLE_PURGE
3667 : if (madvised) {
3668 : /* The chunk might already be in the list, but this
3669 : * makes sure it's at the front. */
3670 : LinkedList_Remove(&chunk->chunks_madvised_elem);
3671 : LinkedList_InsertHead(&arena->chunks_madvised, &chunk->chunks_madvised_elem);
3672 : }
3673 : #endif
3674 : }
3675 165 : }
3676 :
3677 : static void
3678 3372371 : arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
3679 : {
3680 : arena_chunk_t *chunk;
3681 : size_t size, run_ind, run_pages;
3682 :
3683 3372371 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
3684 3372371 : run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
3685 : >> pagesize_2pow);
3686 3372371 : assert(run_ind >= arena_chunk_header_npages);
3687 3372371 : assert(run_ind < chunk_npages);
3688 3372371 : if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
3689 637739 : size = chunk->map[run_ind].bits & ~pagesize_mask;
3690 : else
3691 2734632 : size = run->bin->run_size;
3692 3372371 : run_pages = (size >> pagesize_2pow);
3693 :
3694 : /* Mark pages as unallocated in the chunk map. */
3695 3372371 : if (dirty) {
3696 : size_t i;
3697 :
3698 20482931 : for (i = 0; i < run_pages; i++) {
3699 17110560 : assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
3700 : == 0);
3701 17110560 : chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
3702 : }
3703 :
3704 3372371 : if (chunk->ndirty == 0) {
3705 157649 : arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
3706 : chunk);
3707 : }
3708 3372371 : chunk->ndirty += run_pages;
3709 3372371 : arena->ndirty += run_pages;
3710 : } else {
3711 : size_t i;
3712 :
3713 0 : for (i = 0; i < run_pages; i++) {
3714 0 : chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
3715 : CHUNK_MAP_ALLOCATED);
3716 : }
3717 : }
3718 3372371 : chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3719 : pagesize_mask);
3720 6744742 : chunk->map[run_ind+run_pages-1].bits = size |
3721 3372371 : (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3722 :
3723 : /* Try to coalesce forward. */
3724 6731503 : if (run_ind + run_pages < chunk_npages &&
3725 3359132 : (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
3726 2801107 : size_t nrun_size = chunk->map[run_ind+run_pages].bits &
3727 : ~pagesize_mask;
3728 :
3729 : /*
3730 : * Remove successor from runs_avail; the coalesced run is
3731 : * inserted later.
3732 : */
3733 2801107 : arena_avail_tree_remove(&arena->runs_avail,
3734 2801107 : &chunk->map[run_ind+run_pages]);
3735 :
3736 2801107 : size += nrun_size;
3737 2801107 : run_pages = size >> pagesize_2pow;
3738 :
3739 2801107 : assert((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
3740 : == nrun_size);
3741 2801107 : chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3742 : pagesize_mask);
3743 5602214 : chunk->map[run_ind+run_pages-1].bits = size |
3744 2801107 : (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3745 : }
3746 :
3747 : /* Try to coalesce backward. */
3748 3372371 : if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
3749 : CHUNK_MAP_ALLOCATED) == 0) {
3750 446690 : size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
3751 :
3752 446690 : run_ind -= prun_size >> pagesize_2pow;
3753 :
3754 : /*
3755 : * Remove predecessor from runs_avail; the coalesced run is
3756 : * inserted later.
3757 : */
3758 446690 : arena_avail_tree_remove(&arena->runs_avail,
3759 446690 : &chunk->map[run_ind]);
3760 :
3761 446690 : size += prun_size;
3762 446690 : run_pages = size >> pagesize_2pow;
3763 :
3764 446690 : assert((chunk->map[run_ind].bits & ~pagesize_mask) ==
3765 : prun_size);
3766 446690 : chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3767 : pagesize_mask);
3768 893380 : chunk->map[run_ind+run_pages-1].bits = size |
3769 446690 : (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3770 : }
3771 :
3772 : /* Insert into runs_avail, now that coalescing is complete. */
3773 3372371 : arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
3774 :
3775 : /* Deallocate chunk if it is now completely unused. */
3776 6744742 : if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
3777 3372371 : CHUNK_MAP_ALLOCATED)) == arena_maxclass)
3778 30235 : arena_chunk_dealloc(arena, chunk);
3779 :
3780 : /* Enforce opt_dirty_max. */
3781 3372371 : if (arena->ndirty > opt_dirty_max)
3782 165 : arena_purge(arena);
3783 3372371 : }
3784 :
3785 : static void
3786 0 : arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
3787 : size_t oldsize, size_t newsize)
3788 : {
3789 0 : size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
3790 0 : size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
3791 :
3792 0 : assert(oldsize > newsize);
3793 :
3794 : /*
3795 : * Update the chunk map so that arena_run_dalloc() can treat the
3796 : * leading run as separately allocated.
3797 : */
3798 0 : chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
3799 : CHUNK_MAP_ALLOCATED;
3800 0 : chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
3801 : CHUNK_MAP_ALLOCATED;
3802 :
3803 0 : arena_run_dalloc(arena, run, false);
3804 0 : }
3805 :
3806 : static void
3807 117 : arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
3808 : size_t oldsize, size_t newsize, bool dirty)
3809 : {
3810 117 : size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
3811 117 : size_t npages = newsize >> pagesize_2pow;
3812 :
3813 117 : assert(oldsize > newsize);
3814 :
3815 : /*
3816 : * Update the chunk map so that arena_run_dalloc() can treat the
3817 : * trailing run as separately allocated.
3818 : */
3819 117 : chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
3820 : CHUNK_MAP_ALLOCATED;
3821 234 : chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
3822 117 : | CHUNK_MAP_ALLOCATED;
3823 :
3824 117 : arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
3825 : dirty);
3826 117 : }
3827 :
3828 : static arena_run_t *
3829 3182856 : arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
3830 : {
3831 : arena_chunk_map_t *mapelm;
3832 : arena_run_t *run;
3833 : unsigned i, remainder;
3834 :
3835 : /* Look for a usable run. */
3836 3182856 : mapelm = arena_run_tree_first(&bin->runs);
3837 3182856 : if (mapelm != NULL) {
3838 : /* run is guaranteed to have available space. */
3839 130875 : arena_run_tree_remove(&bin->runs, mapelm);
3840 130875 : run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
3841 : #ifdef MALLOC_STATS
3842 130875 : bin->stats.reruns++;
3843 : #endif
3844 130875 : return (run);
3845 : }
3846 : /* No existing runs have any space available. */
3847 :
3848 : /* Allocate a new run. */
3849 3051981 : run = arena_run_alloc(arena, bin, bin->run_size, false, false);
3850 3051981 : if (run == NULL)
3851 0 : return (NULL);
3852 : /*
3853 : * Don't initialize if a race in arena_run_alloc() allowed an existing
3854 : * run to become usable.
3855 : */
3856 3051981 : if (run == bin->runcur)
3857 0 : return (run);
3858 :
3859 : VALGRIND_MALLOCLIKE_BLOCK(run, sizeof(arena_run_t) + (sizeof(unsigned) *
3860 : (bin->regs_mask_nelms - 1)), 0, false);
3861 :
3862 : /* Initialize run internals. */
3863 3051981 : run->bin = bin;
3864 :
3865 7742566 : for (i = 0; i < bin->regs_mask_nelms - 1; i++)
3866 4690585 : run->regs_mask[i] = UINT_MAX;
3867 3051981 : remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
3868 3051981 : if (remainder == 0)
3869 0 : run->regs_mask[i] = UINT_MAX;
3870 : else {
3871 : /* The last element has spare bits that need to be unset. */
3872 3051981 : run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
3873 3051981 : - remainder));
3874 : }
3875 :
3876 3051981 : run->regs_minelm = 0;
3877 :
3878 3051981 : run->nfree = bin->nregs;
3879 : #ifdef MALLOC_DEBUG
3880 3051981 : run->magic = ARENA_RUN_MAGIC;
3881 : #endif
3882 :
3883 : #ifdef MALLOC_STATS
3884 3051981 : bin->stats.nruns++;
3885 3051981 : bin->stats.curruns++;
3886 3051981 : if (bin->stats.curruns > bin->stats.highruns)
3887 678835 : bin->stats.highruns = bin->stats.curruns;
3888 : #endif
3889 3051981 : return (run);
3890 : }
3891 :
3892 : /* bin->runcur must have space available before this function is called. */
3893 : static inline void *
3894 45735071 : arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
3895 : {
3896 : void *ret;
3897 :
3898 45735071 : assert(run->magic == ARENA_RUN_MAGIC);
3899 45735071 : assert(run->nfree > 0);
3900 :
3901 45735071 : ret = arena_run_reg_alloc(run, bin);
3902 45735071 : assert(ret != NULL);
3903 45735071 : run->nfree--;
3904 :
3905 45735071 : return (ret);
3906 : }
3907 :
3908 : /* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
3909 : static void *
3910 3182856 : arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
3911 : {
3912 :
3913 3182856 : bin->runcur = arena_bin_nonfull_run_get(arena, bin);
3914 3182856 : if (bin->runcur == NULL)
3915 0 : return (NULL);
3916 3182856 : assert(bin->runcur->magic == ARENA_RUN_MAGIC);
3917 3182856 : assert(bin->runcur->nfree > 0);
3918 :
3919 3182856 : return (arena_bin_malloc_easy(arena, bin, bin->runcur));
3920 : }
3921 :
3922 : /*
3923 : * Calculate bin->run_size such that it meets the following constraints:
3924 : *
3925 : * *) bin->run_size >= min_run_size
3926 : * *) bin->run_size <= arena_maxclass
3927 : * *) bin->run_size <= RUN_MAX_SMALL
3928 : * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
3929 : *
3930 : * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
3931 : * also calculated here, since these settings are all interdependent.
3932 : */
3933 : static size_t
3934 672084 : arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
3935 : {
3936 : size_t try_run_size, good_run_size;
3937 : unsigned good_nregs, good_mask_nelms, good_reg0_offset;
3938 : unsigned try_nregs, try_mask_nelms, try_reg0_offset;
3939 :
3940 672084 : assert(min_run_size >= pagesize);
3941 672084 : assert(min_run_size <= arena_maxclass);
3942 672084 : assert(min_run_size <= RUN_MAX_SMALL);
3943 :
3944 : /*
3945 : * Calculate known-valid settings before entering the run_size
3946 : * expansion loop, so that the first part of the loop always copies
3947 : * valid settings.
3948 : *
3949 : * The do..while loop iteratively reduces the number of regions until
3950 : * the run header and the regions no longer overlap. A closed formula
3951 : * would be quite messy, since there is an interdependency between the
3952 : * header's mask length and the number of regions.
3953 : */
3954 672084 : try_run_size = min_run_size;
3955 672084 : try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
3956 : + 1; /* Counter-act try_nregs-- in loop. */
3957 : do {
3958 1381506 : try_nregs--;
3959 2763012 : try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
3960 1381506 : ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
3961 1381506 : try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
3962 1381506 : } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
3963 1381506 : > try_reg0_offset);
3964 :
3965 : /* run_size expansion loop. */
3966 : do {
3967 : /*
3968 : * Copy valid settings before trying more aggressive settings.
3969 : */
3970 802767 : good_run_size = try_run_size;
3971 802767 : good_nregs = try_nregs;
3972 802767 : good_mask_nelms = try_mask_nelms;
3973 802767 : good_reg0_offset = try_reg0_offset;
3974 :
3975 : /* Try more aggressive settings. */
3976 802767 : try_run_size += pagesize;
3977 1605534 : try_nregs = ((try_run_size - sizeof(arena_run_t)) /
3978 802767 : bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
3979 : do {
3980 2314956 : try_nregs--;
3981 4629912 : try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
3982 2314956 : ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
3983 : 1 : 0);
3984 2314956 : try_reg0_offset = try_run_size - (try_nregs *
3985 2314956 : bin->reg_size);
3986 2314956 : } while (sizeof(arena_run_t) + (sizeof(unsigned) *
3987 2314956 : (try_mask_nelms - 1)) > try_reg0_offset);
3988 1605534 : } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL
3989 746760 : && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
3990 1512189 : && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
3991 :
3992 672084 : assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
3993 : <= good_reg0_offset);
3994 672084 : assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
3995 :
3996 : /* Copy final settings. */
3997 672084 : bin->run_size = good_run_size;
3998 672084 : bin->nregs = good_nregs;
3999 672084 : bin->regs_mask_nelms = good_mask_nelms;
4000 672084 : bin->reg0_offset = good_reg0_offset;
4001 :
4002 672084 : return (good_run_size);
4003 : }
4004 :
4005 : #ifdef MALLOC_BALANCE
4006 : static inline void
4007 : arena_lock_balance(arena_t *arena)
4008 : {
4009 : unsigned contention;
4010 :
4011 : contention = malloc_spin_lock(&arena->lock);
4012 : if (narenas > 1) {
4013 : /*
4014 : * Calculate the exponentially averaged contention for this
4015 : * arena. Due to integer math always rounding down, this value
4016 : * decays somewhat faster then normal.
4017 : */
4018 : arena->contention = (((uint64_t)arena->contention
4019 : * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
4020 : + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
4021 : if (arena->contention >= opt_balance_threshold)
4022 : arena_lock_balance_hard(arena);
4023 : }
4024 : }
4025 :
4026 : static void
4027 : arena_lock_balance_hard(arena_t *arena)
4028 : {
4029 : uint32_t ind;
4030 :
4031 : arena->contention = 0;
4032 : #ifdef MALLOC_STATS
4033 : arena->stats.nbalance++;
4034 : #endif
4035 : ind = PRN(balance, narenas_2pow);
4036 : if (arenas[ind] != NULL) {
4037 : #ifdef MOZ_MEMORY_WINDOWS
4038 : TlsSetValue(tlsIndex, arenas[ind]);
4039 : #else
4040 : arenas_map = arenas[ind];
4041 : #endif
4042 : } else {
4043 : malloc_spin_lock(&arenas_lock);
4044 : if (arenas[ind] != NULL) {
4045 : #ifdef MOZ_MEMORY_WINDOWS
4046 : TlsSetValue(tlsIndex, arenas[ind]);
4047 : #else
4048 : arenas_map = arenas[ind];
4049 : #endif
4050 : } else {
4051 : #ifdef MOZ_MEMORY_WINDOWS
4052 : TlsSetValue(tlsIndex, arenas_extend(ind));
4053 : #else
4054 : arenas_map = arenas_extend(ind);
4055 : #endif
4056 : }
4057 : malloc_spin_unlock(&arenas_lock);
4058 : }
4059 : }
4060 : #endif
4061 :
4062 : static inline void *
4063 45735071 : arena_malloc_small(arena_t *arena, size_t size, bool zero)
4064 : {
4065 : void *ret;
4066 : arena_bin_t *bin;
4067 : arena_run_t *run;
4068 :
4069 45735071 : if (size < small_min) {
4070 : /* Tiny. */
4071 11588699 : size = pow2_ceil(size);
4072 11588699 : bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
4073 : 1)))];
4074 : #if (!defined(NDEBUG) || defined(MALLOC_STATS))
4075 : /*
4076 : * Bin calculation is always correct, but we may need
4077 : * to fix size for the purposes of assertions and/or
4078 : * stats accuracy.
4079 : */
4080 11588699 : if (size < (1U << TINY_MIN_2POW))
4081 39317 : size = (1U << TINY_MIN_2POW);
4082 : #endif
4083 34146372 : } else if (size <= small_max) {
4084 : /* Quantum-spaced. */
4085 32199741 : size = QUANTUM_CEILING(size);
4086 32199741 : bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
4087 : - 1];
4088 : } else {
4089 : /* Sub-page. */
4090 1946631 : size = pow2_ceil(size);
4091 3893262 : bin = &arena->bins[ntbins + nqbins
4092 1946631 : + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
4093 : }
4094 45735071 : assert(size == bin->reg_size);
4095 :
4096 : #ifdef MALLOC_BALANCE
4097 : arena_lock_balance(arena);
4098 : #else
4099 45735071 : malloc_spin_lock(&arena->lock);
4100 : #endif
4101 45735071 : if ((run = bin->runcur) != NULL && run->nfree > 0)
4102 42552215 : ret = arena_bin_malloc_easy(arena, bin, run);
4103 : else
4104 3182856 : ret = arena_bin_malloc_hard(arena, bin);
4105 :
4106 45735071 : if (ret == NULL) {
4107 0 : malloc_spin_unlock(&arena->lock);
4108 0 : return (NULL);
4109 : }
4110 :
4111 : #ifdef MALLOC_STATS
4112 45735071 : bin->stats.nrequests++;
4113 45735071 : arena->stats.nmalloc_small++;
4114 45735071 : arena->stats.allocated_small += size;
4115 : #endif
4116 45735071 : malloc_spin_unlock(&arena->lock);
4117 :
4118 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
4119 45735071 : if (zero == false) {
4120 : #ifdef MALLOC_FILL
4121 43672870 : if (opt_junk)
4122 43672870 : memset(ret, 0xa5, size);
4123 0 : else if (opt_zero)
4124 0 : memset(ret, 0, size);
4125 : #endif
4126 : } else
4127 2062201 : memset(ret, 0, size);
4128 :
4129 45735071 : return (ret);
4130 : }
4131 :
4132 : static void *
4133 637631 : arena_malloc_large(arena_t *arena, size_t size, bool zero)
4134 : {
4135 : void *ret;
4136 :
4137 : /* Large allocation. */
4138 637631 : size = PAGE_CEILING(size);
4139 : #ifdef MALLOC_BALANCE
4140 : arena_lock_balance(arena);
4141 : #else
4142 637631 : malloc_spin_lock(&arena->lock);
4143 : #endif
4144 637631 : ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
4145 637631 : if (ret == NULL) {
4146 0 : malloc_spin_unlock(&arena->lock);
4147 0 : return (NULL);
4148 : }
4149 : #ifdef MALLOC_STATS
4150 637631 : arena->stats.nmalloc_large++;
4151 637631 : arena->stats.allocated_large += size;
4152 : #endif
4153 637631 : malloc_spin_unlock(&arena->lock);
4154 :
4155 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
4156 637631 : if (zero == false) {
4157 : #ifdef MALLOC_FILL
4158 443449 : if (opt_junk)
4159 443449 : memset(ret, 0xa5, size);
4160 0 : else if (opt_zero)
4161 0 : memset(ret, 0, size);
4162 : #endif
4163 : }
4164 :
4165 637631 : return (ret);
4166 : }
4167 :
4168 : static inline void *
4169 46372702 : arena_malloc(arena_t *arena, size_t size, bool zero)
4170 : {
4171 :
4172 46372702 : assert(arena != NULL);
4173 46372702 : assert(arena->magic == ARENA_MAGIC);
4174 46372702 : assert(size != 0);
4175 46372702 : assert(QUANTUM_CEILING(size) <= arena_maxclass);
4176 :
4177 46372702 : if (size <= bin_maxclass) {
4178 45735071 : return (arena_malloc_small(arena, size, zero));
4179 : } else
4180 637631 : return (arena_malloc_large(arena, size, zero));
4181 : }
4182 :
4183 : static inline void *
4184 41858280 : imalloc(size_t size)
4185 : {
4186 :
4187 41858280 : assert(size != 0);
4188 :
4189 41858280 : if (size <= arena_maxclass)
4190 41855450 : return (arena_malloc(choose_arena(), size, false));
4191 : else
4192 2830 : return (huge_malloc(size, false));
4193 : }
4194 :
4195 : static inline void *
4196 2256419 : icalloc(size_t size)
4197 : {
4198 :
4199 2256419 : if (size <= arena_maxclass)
4200 2256383 : return (arena_malloc(choose_arena(), size, true));
4201 : else
4202 36 : return (huge_malloc(size, true));
4203 : }
4204 :
4205 : /* Only handles large allocations that require more than page alignment. */
4206 : static void *
4207 0 : arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
4208 : {
4209 : void *ret;
4210 : size_t offset;
4211 : arena_chunk_t *chunk;
4212 :
4213 0 : assert((size & pagesize_mask) == 0);
4214 0 : assert((alignment & pagesize_mask) == 0);
4215 :
4216 : #ifdef MALLOC_BALANCE
4217 : arena_lock_balance(arena);
4218 : #else
4219 0 : malloc_spin_lock(&arena->lock);
4220 : #endif
4221 0 : ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
4222 0 : if (ret == NULL) {
4223 0 : malloc_spin_unlock(&arena->lock);
4224 0 : return (NULL);
4225 : }
4226 :
4227 0 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
4228 :
4229 0 : offset = (uintptr_t)ret & (alignment - 1);
4230 0 : assert((offset & pagesize_mask) == 0);
4231 0 : assert(offset < alloc_size);
4232 0 : if (offset == 0)
4233 0 : arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
4234 : else {
4235 : size_t leadsize, trailsize;
4236 :
4237 0 : leadsize = alignment - offset;
4238 0 : if (leadsize > 0) {
4239 0 : arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
4240 : alloc_size - leadsize);
4241 0 : ret = (void *)((uintptr_t)ret + leadsize);
4242 : }
4243 :
4244 0 : trailsize = alloc_size - leadsize - size;
4245 0 : if (trailsize != 0) {
4246 : /* Trim trailing space. */
4247 0 : assert(trailsize < alloc_size);
4248 0 : arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
4249 : size, false);
4250 : }
4251 : }
4252 :
4253 : #ifdef MALLOC_STATS
4254 0 : arena->stats.nmalloc_large++;
4255 0 : arena->stats.allocated_large += size;
4256 : #endif
4257 0 : malloc_spin_unlock(&arena->lock);
4258 :
4259 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
4260 : #ifdef MALLOC_FILL
4261 0 : if (opt_junk)
4262 0 : memset(ret, 0xa5, size);
4263 0 : else if (opt_zero)
4264 0 : memset(ret, 0, size);
4265 : #endif
4266 0 : return (ret);
4267 : }
4268 :
4269 : static inline void *
4270 0 : ipalloc(size_t alignment, size_t size)
4271 : {
4272 : void *ret;
4273 : size_t ceil_size;
4274 :
4275 : /*
4276 : * Round size up to the nearest multiple of alignment.
4277 : *
4278 : * This done, we can take advantage of the fact that for each small
4279 : * size class, every object is aligned at the smallest power of two
4280 : * that is non-zero in the base two representation of the size. For
4281 : * example:
4282 : *
4283 : * Size | Base 2 | Minimum alignment
4284 : * -----+----------+------------------
4285 : * 96 | 1100000 | 32
4286 : * 144 | 10100000 | 32
4287 : * 192 | 11000000 | 64
4288 : *
4289 : * Depending on runtime settings, it is possible that arena_malloc()
4290 : * will further round up to a power of two, but that never causes
4291 : * correctness issues.
4292 : */
4293 0 : ceil_size = (size + (alignment - 1)) & (-alignment);
4294 : /*
4295 : * (ceil_size < size) protects against the combination of maximal
4296 : * alignment and size greater than maximal alignment.
4297 : */
4298 0 : if (ceil_size < size) {
4299 : /* size_t overflow. */
4300 0 : return (NULL);
4301 : }
4302 :
4303 0 : if (ceil_size <= pagesize || (alignment <= pagesize
4304 0 : && ceil_size <= arena_maxclass))
4305 0 : ret = arena_malloc(choose_arena(), ceil_size, false);
4306 : else {
4307 : size_t run_size;
4308 :
4309 : /*
4310 : * We can't achieve sub-page alignment, so round up alignment
4311 : * permanently; it makes later calculations simpler.
4312 : */
4313 0 : alignment = PAGE_CEILING(alignment);
4314 0 : ceil_size = PAGE_CEILING(size);
4315 : /*
4316 : * (ceil_size < size) protects against very large sizes within
4317 : * pagesize of SIZE_T_MAX.
4318 : *
4319 : * (ceil_size + alignment < ceil_size) protects against the
4320 : * combination of maximal alignment and ceil_size large enough
4321 : * to cause overflow. This is similar to the first overflow
4322 : * check above, but it needs to be repeated due to the new
4323 : * ceil_size value, which may now be *equal* to maximal
4324 : * alignment, whereas before we only detected overflow if the
4325 : * original size was *greater* than maximal alignment.
4326 : */
4327 0 : if (ceil_size < size || ceil_size + alignment < ceil_size) {
4328 : /* size_t overflow. */
4329 0 : return (NULL);
4330 : }
4331 :
4332 : /*
4333 : * Calculate the size of the over-size run that arena_palloc()
4334 : * would need to allocate in order to guarantee the alignment.
4335 : */
4336 0 : if (ceil_size >= alignment)
4337 0 : run_size = ceil_size + alignment - pagesize;
4338 : else {
4339 : /*
4340 : * It is possible that (alignment << 1) will cause
4341 : * overflow, but it doesn't matter because we also
4342 : * subtract pagesize, which in the case of overflow
4343 : * leaves us with a very large run_size. That causes
4344 : * the first conditional below to fail, which means
4345 : * that the bogus run_size value never gets used for
4346 : * anything important.
4347 : */
4348 0 : run_size = (alignment << 1) - pagesize;
4349 : }
4350 :
4351 0 : if (run_size <= arena_maxclass) {
4352 0 : ret = arena_palloc(choose_arena(), alignment, ceil_size,
4353 : run_size);
4354 0 : } else if (alignment <= chunksize)
4355 0 : ret = huge_malloc(ceil_size, false);
4356 : else
4357 0 : ret = huge_palloc(alignment, ceil_size);
4358 : }
4359 :
4360 0 : assert(((uintptr_t)ret & (alignment - 1)) == 0);
4361 0 : return (ret);
4362 : }
4363 :
4364 : /* Return the size of the allocation pointed to by ptr. */
4365 : static size_t
4366 2300043 : arena_salloc(const void *ptr)
4367 : {
4368 : size_t ret;
4369 : arena_chunk_t *chunk;
4370 : size_t pageind, mapbits;
4371 :
4372 2300043 : assert(ptr != NULL);
4373 2300043 : assert(CHUNK_ADDR2BASE(ptr) != ptr);
4374 :
4375 2300043 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4376 2300043 : pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
4377 2300043 : mapbits = chunk->map[pageind].bits;
4378 2300043 : assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
4379 2300043 : if ((mapbits & CHUNK_MAP_LARGE) == 0) {
4380 2271520 : arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
4381 2271520 : assert(run->magic == ARENA_RUN_MAGIC);
4382 2271520 : ret = run->bin->reg_size;
4383 : } else {
4384 28523 : ret = mapbits & ~pagesize_mask;
4385 28523 : assert(ret != 0);
4386 : }
4387 :
4388 2300043 : return (ret);
4389 : }
4390 :
4391 : #if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
4392 : /*
4393 : * Validate ptr before assuming that it points to an allocation. Currently,
4394 : * the following validation is performed:
4395 : *
4396 : * + Check that ptr is not NULL.
4397 : *
4398 : * + Check that ptr lies within a mapped chunk.
4399 : */
4400 : static inline size_t
4401 0 : isalloc_validate(const void *ptr)
4402 : {
4403 : arena_chunk_t *chunk;
4404 :
4405 0 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4406 0 : if (chunk == NULL)
4407 0 : return (0);
4408 :
4409 0 : if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
4410 0 : return (0);
4411 :
4412 0 : if (chunk != ptr) {
4413 0 : assert(chunk->arena->magic == ARENA_MAGIC);
4414 0 : return (arena_salloc(ptr));
4415 : } else {
4416 : size_t ret;
4417 : extent_node_t *node;
4418 : extent_node_t key;
4419 :
4420 : /* Chunk. */
4421 0 : key.addr = (void *)chunk;
4422 0 : malloc_mutex_lock(&huge_mtx);
4423 0 : node = extent_tree_ad_search(&huge, &key);
4424 0 : if (node != NULL)
4425 0 : ret = node->size;
4426 : else
4427 0 : ret = 0;
4428 0 : malloc_mutex_unlock(&huge_mtx);
4429 0 : return (ret);
4430 : }
4431 : }
4432 : #endif
4433 :
4434 : static inline size_t
4435 2301663 : isalloc(const void *ptr)
4436 : {
4437 : size_t ret;
4438 : arena_chunk_t *chunk;
4439 :
4440 2301663 : assert(ptr != NULL);
4441 :
4442 2301663 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4443 2301663 : if (chunk != ptr) {
4444 : /* Region. */
4445 2300043 : assert(chunk->arena->magic == ARENA_MAGIC);
4446 :
4447 2300043 : ret = arena_salloc(ptr);
4448 : } else {
4449 : extent_node_t *node, key;
4450 :
4451 : /* Chunk (huge allocation). */
4452 :
4453 1620 : malloc_mutex_lock(&huge_mtx);
4454 :
4455 : /* Extract from tree of huge allocations. */
4456 1620 : key.addr = __DECONST(void *, ptr);
4457 1620 : node = extent_tree_ad_search(&huge, &key);
4458 1620 : assert(node != NULL);
4459 :
4460 1620 : ret = node->size;
4461 :
4462 1620 : malloc_mutex_unlock(&huge_mtx);
4463 : }
4464 :
4465 2301663 : return (ret);
4466 : }
4467 :
4468 : static inline void
4469 43659702 : arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4470 : arena_chunk_map_t *mapelm)
4471 : {
4472 : arena_run_t *run;
4473 : arena_bin_t *bin;
4474 : size_t size;
4475 :
4476 43659702 : run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
4477 43659702 : assert(run->magic == ARENA_RUN_MAGIC);
4478 43659702 : bin = run->bin;
4479 43659702 : size = bin->reg_size;
4480 :
4481 : #ifdef MALLOC_FILL
4482 43659702 : if (opt_junk)
4483 43659702 : memset(ptr, 0x5a, size);
4484 : #endif
4485 :
4486 43659702 : arena_run_reg_dalloc(run, bin, ptr, size);
4487 43659702 : run->nfree++;
4488 :
4489 43659702 : if (run->nfree == bin->nregs) {
4490 : /* Deallocate run. */
4491 2734632 : if (run == bin->runcur)
4492 2572218 : bin->runcur = NULL;
4493 162414 : else if (bin->nregs != 1) {
4494 324828 : size_t run_pageind = (((uintptr_t)run -
4495 162414 : (uintptr_t)chunk)) >> pagesize_2pow;
4496 162414 : arena_chunk_map_t *run_mapelm =
4497 162414 : &chunk->map[run_pageind];
4498 : /*
4499 : * This block's conditional is necessary because if the
4500 : * run only contains one region, then it never gets
4501 : * inserted into the non-full runs tree.
4502 : */
4503 162414 : assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
4504 : run_mapelm);
4505 162414 : arena_run_tree_remove(&bin->runs, run_mapelm);
4506 : }
4507 : #ifdef MALLOC_DEBUG
4508 2734632 : run->magic = 0;
4509 : #endif
4510 : VALGRIND_FREELIKE_BLOCK(run, 0);
4511 2734632 : arena_run_dalloc(arena, run, true);
4512 : #ifdef MALLOC_STATS
4513 2734632 : bin->stats.curruns--;
4514 : #endif
4515 40925070 : } else if (run->nfree == 1 && run != bin->runcur) {
4516 : /*
4517 : * Make sure that bin->runcur always refers to the lowest
4518 : * non-full run, if one exists.
4519 : */
4520 300701 : if (bin->runcur == NULL)
4521 5398 : bin->runcur = run;
4522 295303 : else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
4523 : /* Switch runcur. */
4524 212684 : if (bin->runcur->nfree > 0) {
4525 210677 : arena_chunk_t *runcur_chunk =
4526 210677 : (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
4527 210677 : size_t runcur_pageind =
4528 421354 : (((uintptr_t)bin->runcur -
4529 210677 : (uintptr_t)runcur_chunk)) >> pagesize_2pow;
4530 210677 : arena_chunk_map_t *runcur_mapelm =
4531 210677 : &runcur_chunk->map[runcur_pageind];
4532 :
4533 : /* Insert runcur. */
4534 210677 : assert(arena_run_tree_search(&bin->runs,
4535 : runcur_mapelm) == NULL);
4536 210677 : arena_run_tree_insert(&bin->runs,
4537 : runcur_mapelm);
4538 : }
4539 212684 : bin->runcur = run;
4540 : } else {
4541 165238 : size_t run_pageind = (((uintptr_t)run -
4542 82619 : (uintptr_t)chunk)) >> pagesize_2pow;
4543 82619 : arena_chunk_map_t *run_mapelm =
4544 82619 : &chunk->map[run_pageind];
4545 :
4546 82619 : assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
4547 : NULL);
4548 82619 : arena_run_tree_insert(&bin->runs, run_mapelm);
4549 : }
4550 : }
4551 : #ifdef MALLOC_STATS
4552 43659702 : arena->stats.allocated_small -= size;
4553 43659702 : arena->stats.ndalloc_small++;
4554 : #endif
4555 43659702 : }
4556 :
4557 : static void
4558 637622 : arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
4559 : {
4560 : /* Large allocation. */
4561 637622 : malloc_spin_lock(&arena->lock);
4562 :
4563 : #ifdef MALLOC_FILL
4564 : #ifndef MALLOC_STATS
4565 : if (opt_junk)
4566 : #endif
4567 : #endif
4568 : {
4569 637622 : size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
4570 : pagesize_2pow;
4571 637622 : size_t size = chunk->map[pageind].bits & ~pagesize_mask;
4572 :
4573 : #ifdef MALLOC_FILL
4574 : #ifdef MALLOC_STATS
4575 637622 : if (opt_junk)
4576 : #endif
4577 637622 : memset(ptr, 0x5a, size);
4578 : #endif
4579 : #ifdef MALLOC_STATS
4580 637622 : arena->stats.allocated_large -= size;
4581 : #endif
4582 : }
4583 : #ifdef MALLOC_STATS
4584 637622 : arena->stats.ndalloc_large++;
4585 : #endif
4586 :
4587 637622 : arena_run_dalloc(arena, (arena_run_t *)ptr, true);
4588 637622 : malloc_spin_unlock(&arena->lock);
4589 637622 : }
4590 :
4591 : static inline void
4592 44297318 : arena_dalloc(void *ptr, size_t offset)
4593 : {
4594 : arena_chunk_t *chunk;
4595 : arena_t *arena;
4596 : size_t pageind;
4597 : arena_chunk_map_t *mapelm;
4598 :
4599 44297318 : assert(ptr != NULL);
4600 44297318 : assert(offset != 0);
4601 44297318 : assert(CHUNK_ADDR2OFFSET(ptr) == offset);
4602 :
4603 44297318 : chunk = (arena_chunk_t *) ((uintptr_t)ptr - offset);
4604 44297318 : arena = chunk->arena;
4605 44297318 : assert(arena != NULL);
4606 44297318 : assert(arena->magic == ARENA_MAGIC);
4607 :
4608 44297318 : pageind = offset >> pagesize_2pow;
4609 44297318 : mapelm = &chunk->map[pageind];
4610 44297318 : assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
4611 44297318 : if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
4612 : /* Small allocation. */
4613 43659696 : malloc_spin_lock(&arena->lock);
4614 43659702 : arena_dalloc_small(arena, chunk, ptr, mapelm);
4615 43659702 : malloc_spin_unlock(&arena->lock);
4616 : } else
4617 637622 : arena_dalloc_large(arena, chunk, ptr);
4618 : VALGRIND_FREELIKE_BLOCK(ptr, 0);
4619 44297323 : }
4620 :
4621 : static inline void
4622 2262557 : idalloc(void *ptr)
4623 : {
4624 : size_t offset;
4625 :
4626 2262557 : assert(ptr != NULL);
4627 :
4628 2262557 : offset = CHUNK_ADDR2OFFSET(ptr);
4629 2262557 : if (offset != 0)
4630 2260937 : arena_dalloc(ptr, offset);
4631 : else
4632 1620 : huge_dalloc(ptr);
4633 2262557 : }
4634 :
4635 : static void
4636 117 : arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4637 : size_t size, size_t oldsize)
4638 : {
4639 :
4640 117 : assert(size < oldsize);
4641 :
4642 : /*
4643 : * Shrink the run, and make trailing pages available for other
4644 : * allocations.
4645 : */
4646 : #ifdef MALLOC_BALANCE
4647 : arena_lock_balance(arena);
4648 : #else
4649 117 : malloc_spin_lock(&arena->lock);
4650 : #endif
4651 117 : arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
4652 : true);
4653 : #ifdef MALLOC_STATS
4654 117 : arena->stats.allocated_large -= oldsize - size;
4655 : #endif
4656 117 : malloc_spin_unlock(&arena->lock);
4657 117 : }
4658 :
4659 : static bool
4660 28113 : arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4661 : size_t size, size_t oldsize)
4662 : {
4663 28113 : size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
4664 28113 : size_t npages = oldsize >> pagesize_2pow;
4665 :
4666 28113 : assert(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
4667 :
4668 : /* Try to extend the run. */
4669 28113 : assert(size > oldsize);
4670 : #ifdef MALLOC_BALANCE
4671 : arena_lock_balance(arena);
4672 : #else
4673 28113 : malloc_spin_lock(&arena->lock);
4674 : #endif
4675 51977 : if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
4676 44938 : & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
4677 10537 : ~pagesize_mask) >= size - oldsize) {
4678 : /*
4679 : * The next run is available and sufficiently large. Split the
4680 : * following run, then merge the first part with the existing
4681 : * allocation.
4682 : */
4683 18236 : arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
4684 9118 : ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
4685 : false);
4686 :
4687 9118 : chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
4688 : CHUNK_MAP_ALLOCATED;
4689 9118 : chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
4690 : CHUNK_MAP_ALLOCATED;
4691 :
4692 : #ifdef MALLOC_STATS
4693 9118 : arena->stats.allocated_large += size - oldsize;
4694 : #endif
4695 9118 : malloc_spin_unlock(&arena->lock);
4696 9118 : return (false);
4697 : }
4698 18995 : malloc_spin_unlock(&arena->lock);
4699 :
4700 18995 : return (true);
4701 : }
4702 :
4703 : /*
4704 : * Try to resize a large allocation, in order to avoid copying. This will
4705 : * always fail if growing an object, and the following run is already in use.
4706 : */
4707 : static bool
4708 28338 : arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
4709 : {
4710 : size_t psize;
4711 :
4712 28338 : psize = PAGE_CEILING(size);
4713 28338 : if (psize == oldsize) {
4714 : /* Same size class. */
4715 : #ifdef MALLOC_FILL
4716 108 : if (opt_junk && size < oldsize) {
4717 108 : memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize -
4718 : size);
4719 : }
4720 : #endif
4721 108 : return (false);
4722 : } else {
4723 : arena_chunk_t *chunk;
4724 : arena_t *arena;
4725 :
4726 28230 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4727 28230 : arena = chunk->arena;
4728 28230 : assert(arena->magic == ARENA_MAGIC);
4729 :
4730 28230 : if (psize < oldsize) {
4731 : #ifdef MALLOC_FILL
4732 : /* Fill before shrinking in order avoid a race. */
4733 117 : if (opt_junk) {
4734 117 : memset((void *)((uintptr_t)ptr + size), 0x5a,
4735 : oldsize - size);
4736 : }
4737 : #endif
4738 117 : arena_ralloc_large_shrink(arena, chunk, ptr, psize,
4739 : oldsize);
4740 117 : return (false);
4741 : } else {
4742 28113 : bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
4743 : psize, oldsize);
4744 : #ifdef MALLOC_FILL
4745 28113 : if (ret == false && opt_zero) {
4746 0 : memset((void *)((uintptr_t)ptr + oldsize), 0,
4747 : size - oldsize);
4748 : }
4749 : #endif
4750 28113 : return (ret);
4751 : }
4752 : }
4753 : }
4754 :
4755 : static void *
4756 2299975 : arena_ralloc(void *ptr, size_t size, size_t oldsize)
4757 : {
4758 : void *ret;
4759 : size_t copysize;
4760 :
4761 : /* Try to avoid moving the allocation. */
4762 2299975 : if (size < small_min) {
4763 337330 : if (oldsize < small_min &&
4764 83165 : ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
4765 83165 : == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
4766 9324 : goto IN_PLACE; /* Same size class. */
4767 2045810 : } else if (size <= small_max) {
4768 3339456 : if (oldsize >= small_min && oldsize <= small_max &&
4769 1653543 : (QUANTUM_CEILING(size) >> opt_quantum_2pow)
4770 1653543 : == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
4771 18432 : goto IN_PLACE; /* Same size class. */
4772 359897 : } else if (size <= bin_maxclass) {
4773 393045 : if (oldsize > small_max && oldsize <= bin_maxclass &&
4774 89419 : pow2_ceil(size) == pow2_ceil(oldsize))
4775 2007 : goto IN_PLACE; /* Same size class. */
4776 56271 : } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
4777 28338 : assert(size > bin_maxclass);
4778 28338 : if (arena_ralloc_large(ptr, size, oldsize) == false)
4779 9343 : return (ptr);
4780 : }
4781 :
4782 : /*
4783 : * If we get here, then size and oldsize are different enough that we
4784 : * need to move the object. In that case, fall back to allocating new
4785 : * space and copying.
4786 : */
4787 2260869 : ret = arena_malloc(choose_arena(), size, false);
4788 2260869 : if (ret == NULL)
4789 0 : return (NULL);
4790 :
4791 : /* Junk/zero-filling were already done by arena_malloc(). */
4792 2260869 : copysize = (size < oldsize) ? size : oldsize;
4793 : #ifdef VM_COPY_MIN
4794 : if (copysize >= VM_COPY_MIN)
4795 : pages_copy(ret, ptr, copysize);
4796 : else
4797 : #endif
4798 2260869 : memcpy(ret, ptr, copysize);
4799 2260869 : idalloc(ptr);
4800 2260869 : return (ret);
4801 : IN_PLACE:
4802 : #ifdef MALLOC_FILL
4803 29763 : if (opt_junk && size < oldsize)
4804 20583 : memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
4805 9180 : else if (opt_zero && size > oldsize)
4806 0 : memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
4807 : #endif
4808 29763 : return (ptr);
4809 : }
4810 :
4811 : static inline void *
4812 2301663 : iralloc(void *ptr, size_t size)
4813 : {
4814 : size_t oldsize;
4815 :
4816 2301663 : assert(ptr != NULL);
4817 2301663 : assert(size != 0);
4818 :
4819 2301663 : oldsize = isalloc(ptr);
4820 :
4821 : #ifndef MALLOC_VALGRIND
4822 2301663 : if (size <= arena_maxclass)
4823 2299975 : return (arena_ralloc(ptr, size, oldsize));
4824 : else
4825 1688 : return (huge_ralloc(ptr, size, oldsize));
4826 : #else
4827 : /*
4828 : * Valgrind does not provide a public interface for modifying an
4829 : * existing allocation, so use malloc/memcpy/free instead.
4830 : */
4831 : {
4832 : void *ret = imalloc(size);
4833 : if (ret != NULL) {
4834 : if (oldsize < size)
4835 : memcpy(ret, ptr, oldsize);
4836 : else
4837 : memcpy(ret, ptr, size);
4838 : idalloc(ptr);
4839 : }
4840 : return (ret);
4841 : }
4842 : #endif
4843 : }
4844 :
4845 : static bool
4846 18669 : arena_new(arena_t *arena)
4847 : {
4848 : unsigned i;
4849 : arena_bin_t *bin;
4850 : size_t pow2_size, prev_run_size;
4851 :
4852 18669 : if (malloc_spin_init(&arena->lock))
4853 0 : return (true);
4854 :
4855 : #ifdef MALLOC_STATS
4856 18669 : memset(&arena->stats, 0, sizeof(arena_stats_t));
4857 : #endif
4858 :
4859 : /* Initialize chunks. */
4860 18669 : arena_chunk_tree_dirty_new(&arena->chunks_dirty);
4861 : #ifdef MALLOC_DOUBLE_PURGE
4862 : LinkedList_Init(&arena->chunks_madvised);
4863 : #endif
4864 18669 : arena->spare = NULL;
4865 :
4866 18669 : arena->ndirty = 0;
4867 :
4868 18669 : arena_avail_tree_new(&arena->runs_avail);
4869 :
4870 : #ifdef MALLOC_BALANCE
4871 : arena->contention = 0;
4872 : #endif
4873 :
4874 : /* Initialize bins. */
4875 18669 : prev_run_size = pagesize;
4876 :
4877 : /* (2^n)-spaced tiny bins. */
4878 56007 : for (i = 0; i < ntbins; i++) {
4879 37338 : bin = &arena->bins[i];
4880 37338 : bin->runcur = NULL;
4881 37338 : arena_run_tree_new(&bin->runs);
4882 :
4883 37338 : bin->reg_size = (1U << (TINY_MIN_2POW + i));
4884 :
4885 37338 : prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4886 :
4887 : #ifdef MALLOC_STATS
4888 37338 : memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
4889 : #endif
4890 : }
4891 :
4892 : /* Quantum-spaced bins. */
4893 616077 : for (; i < ntbins + nqbins; i++) {
4894 597408 : bin = &arena->bins[i];
4895 597408 : bin->runcur = NULL;
4896 597408 : arena_run_tree_new(&bin->runs);
4897 :
4898 597408 : bin->reg_size = quantum * (i - ntbins + 1);
4899 :
4900 597408 : pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
4901 597408 : prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4902 :
4903 : #ifdef MALLOC_STATS
4904 597408 : memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
4905 : #endif
4906 : }
4907 :
4908 : /* (2^n)-spaced sub-page bins. */
4909 56007 : for (; i < ntbins + nqbins + nsbins; i++) {
4910 37338 : bin = &arena->bins[i];
4911 37338 : bin->runcur = NULL;
4912 37338 : arena_run_tree_new(&bin->runs);
4913 :
4914 37338 : bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
4915 :
4916 37338 : prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4917 :
4918 : #ifdef MALLOC_STATS
4919 37338 : memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
4920 : #endif
4921 : }
4922 :
4923 : #ifdef MALLOC_DEBUG
4924 18669 : arena->magic = ARENA_MAGIC;
4925 : #endif
4926 :
4927 18669 : return (false);
4928 : }
4929 :
4930 : /* Create a new arena and insert it into the arenas array at index ind. */
4931 : static arena_t *
4932 18669 : arenas_extend(unsigned ind)
4933 : {
4934 : arena_t *ret;
4935 :
4936 : /* Allocate enough space for trailing bins. */
4937 18669 : ret = (arena_t *)base_alloc(sizeof(arena_t)
4938 18669 : + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
4939 18669 : if (ret != NULL && arena_new(ret) == false) {
4940 18669 : arenas[ind] = ret;
4941 18669 : return (ret);
4942 : }
4943 : /* Only reached if there is an OOM error. */
4944 :
4945 : /*
4946 : * OOM here is quite inconvenient to propagate, since dealing with it
4947 : * would require a check for failure in the fast path. Instead, punt
4948 : * by using arenas[0]. In practice, this is an extremely unlikely
4949 : * failure.
4950 : */
4951 0 : _malloc_message(_getprogname(),
4952 : ": (malloc) Error initializing arena\n", "", "");
4953 0 : if (opt_abort)
4954 0 : abort();
4955 :
4956 0 : return (arenas[0]);
4957 : }
4958 :
4959 : /*
4960 : * End arena.
4961 : */
4962 : /******************************************************************************/
4963 : /*
4964 : * Begin general internal functions.
4965 : */
4966 :
4967 : static void *
4968 4554 : huge_malloc(size_t size, bool zero)
4969 : {
4970 : void *ret;
4971 : size_t csize;
4972 : size_t psize;
4973 : extent_node_t *node;
4974 :
4975 : /* Allocate one or more contiguous chunks for this request. */
4976 :
4977 4554 : csize = CHUNK_CEILING(size);
4978 4554 : if (csize == 0) {
4979 : /* size is large enough to cause size_t wrap-around. */
4980 0 : return (NULL);
4981 : }
4982 :
4983 : /* Allocate an extent node with which to track the chunk. */
4984 4554 : node = base_node_alloc();
4985 4554 : if (node == NULL)
4986 0 : return (NULL);
4987 :
4988 4554 : ret = chunk_alloc(csize, zero, true);
4989 4554 : if (ret == NULL) {
4990 18 : base_node_dealloc(node);
4991 18 : return (NULL);
4992 : }
4993 :
4994 : /* Insert node into huge. */
4995 4536 : node->addr = ret;
4996 4536 : psize = PAGE_CEILING(size);
4997 4536 : node->size = psize;
4998 :
4999 4536 : malloc_mutex_lock(&huge_mtx);
5000 4536 : extent_tree_ad_insert(&huge, node);
5001 : #ifdef MALLOC_STATS
5002 4536 : huge_nmalloc++;
5003 :
5004 : /* Although we allocated space for csize bytes, we indicate that we've
5005 : * allocated only psize bytes.
5006 : *
5007 : * If DECOMMIT is defined, this is a reasonable thing to do, since
5008 : * we'll explicitly decommit the bytes in excess of psize.
5009 : *
5010 : * If DECOMMIT is not defined, then we're relying on the OS to be lazy
5011 : * about how it allocates physical pages to mappings. If we never
5012 : * touch the pages in excess of psize, the OS won't allocate a physical
5013 : * page, and we won't use more than psize bytes of physical memory.
5014 : *
5015 : * A correct program will only touch memory in excess of how much it
5016 : * requested if it first calls malloc_usable_size and finds out how
5017 : * much space it has to play with. But because we set node->size =
5018 : * psize above, malloc_usable_size will return psize, not csize, and
5019 : * the program will (hopefully) never touch bytes in excess of psize.
5020 : * Thus those bytes won't take up space in physical memory, and we can
5021 : * reasonably claim we never "allocated" them in the first place. */
5022 4536 : huge_allocated += psize;
5023 : #endif
5024 4536 : malloc_mutex_unlock(&huge_mtx);
5025 :
5026 : #ifdef MALLOC_DECOMMIT
5027 : if (csize - psize > 0)
5028 : pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
5029 : #endif
5030 :
5031 : #ifdef MALLOC_DECOMMIT
5032 : VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, zero);
5033 : #else
5034 : VALGRIND_MALLOCLIKE_BLOCK(ret, csize, 0, zero);
5035 : #endif
5036 :
5037 : #ifdef MALLOC_FILL
5038 4536 : if (zero == false) {
5039 4518 : if (opt_junk)
5040 : # ifdef MALLOC_DECOMMIT
5041 : memset(ret, 0xa5, psize);
5042 : # else
5043 4518 : memset(ret, 0xa5, csize);
5044 : # endif
5045 0 : else if (opt_zero)
5046 : # ifdef MALLOC_DECOMMIT
5047 : memset(ret, 0, psize);
5048 : # else
5049 0 : memset(ret, 0, csize);
5050 : # endif
5051 : }
5052 : #endif
5053 :
5054 4536 : return (ret);
5055 : }
5056 :
5057 : /* Only handles large allocations that require more than chunk alignment. */
5058 : static void *
5059 0 : huge_palloc(size_t alignment, size_t size)
5060 : {
5061 : void *ret;
5062 : size_t alloc_size, chunk_size, offset;
5063 : size_t psize;
5064 : extent_node_t *node;
5065 : int pfd;
5066 :
5067 : /*
5068 : * This allocation requires alignment that is even larger than chunk
5069 : * alignment. This means that huge_malloc() isn't good enough.
5070 : *
5071 : * Allocate almost twice as many chunks as are demanded by the size or
5072 : * alignment, in order to assure the alignment can be achieved, then
5073 : * unmap leading and trailing chunks.
5074 : */
5075 0 : assert(alignment >= chunksize);
5076 :
5077 0 : chunk_size = CHUNK_CEILING(size);
5078 :
5079 0 : if (size >= alignment)
5080 0 : alloc_size = chunk_size + alignment - chunksize;
5081 : else
5082 0 : alloc_size = (alignment << 1) - chunksize;
5083 :
5084 : /* Allocate an extent node with which to track the chunk. */
5085 0 : node = base_node_alloc();
5086 0 : if (node == NULL)
5087 0 : return (NULL);
5088 :
5089 : /*
5090 : * Windows requires that there be a 1:1 mapping between VM
5091 : * allocation/deallocation operations. Therefore, take care here to
5092 : * acquire the final result via one mapping operation.
5093 : *
5094 : * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
5095 : * since it reduces the number of page files.
5096 : */
5097 : #ifdef MALLOC_PAGEFILE
5098 : if (opt_pagefile) {
5099 : pfd = pagefile_init(size);
5100 : if (pfd == -1)
5101 : return (NULL);
5102 : } else
5103 : #endif
5104 0 : pfd = -1;
5105 : #ifdef JEMALLOC_USES_MAP_ALIGN
5106 : ret = pages_map_align(chunk_size, pfd, alignment);
5107 : #else
5108 : do {
5109 : void *over;
5110 :
5111 0 : over = chunk_alloc(alloc_size, false, false);
5112 0 : if (over == NULL) {
5113 0 : base_node_dealloc(node);
5114 0 : ret = NULL;
5115 0 : goto RETURN;
5116 : }
5117 :
5118 0 : offset = (uintptr_t)over & (alignment - 1);
5119 0 : assert((offset & chunksize_mask) == 0);
5120 0 : assert(offset < alloc_size);
5121 0 : ret = (void *)((uintptr_t)over + offset);
5122 0 : chunk_dealloc(over, alloc_size);
5123 0 : ret = pages_map(ret, chunk_size, pfd);
5124 : /*
5125 : * Failure here indicates a race with another thread, so try
5126 : * again.
5127 : */
5128 0 : } while (ret == NULL);
5129 : #endif
5130 : /* Insert node into huge. */
5131 0 : node->addr = ret;
5132 0 : psize = PAGE_CEILING(size);
5133 0 : node->size = psize;
5134 :
5135 0 : malloc_mutex_lock(&huge_mtx);
5136 0 : extent_tree_ad_insert(&huge, node);
5137 : #ifdef MALLOC_STATS
5138 0 : huge_nmalloc++;
5139 : /* See note in huge_alloc() for why huge_allocated += psize is correct
5140 : * here even when DECOMMIT is not defined. */
5141 0 : huge_allocated += psize;
5142 : #endif
5143 0 : malloc_mutex_unlock(&huge_mtx);
5144 :
5145 : #ifdef MALLOC_DECOMMIT
5146 : if (chunk_size - psize > 0) {
5147 : pages_decommit((void *)((uintptr_t)ret + psize),
5148 : chunk_size - psize);
5149 : }
5150 : #endif
5151 :
5152 : #ifdef MALLOC_DECOMMIT
5153 : VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, false);
5154 : #else
5155 : VALGRIND_MALLOCLIKE_BLOCK(ret, chunk_size, 0, false);
5156 : #endif
5157 :
5158 : #ifdef MALLOC_FILL
5159 0 : if (opt_junk)
5160 : # ifdef MALLOC_DECOMMIT
5161 : memset(ret, 0xa5, psize);
5162 : # else
5163 0 : memset(ret, 0xa5, chunk_size);
5164 : # endif
5165 0 : else if (opt_zero)
5166 : # ifdef MALLOC_DECOMMIT
5167 : memset(ret, 0, psize);
5168 : # else
5169 0 : memset(ret, 0, chunk_size);
5170 : # endif
5171 : #endif
5172 :
5173 : RETURN:
5174 : #ifdef MALLOC_PAGEFILE
5175 : if (pfd != -1)
5176 : pagefile_close(pfd);
5177 : #endif
5178 0 : return (ret);
5179 : }
5180 :
5181 : static void *
5182 1688 : huge_ralloc(void *ptr, size_t size, size_t oldsize)
5183 : {
5184 : void *ret;
5185 : size_t copysize;
5186 :
5187 : /* Avoid moving the allocation if the size class would not change. */
5188 :
5189 3191 : if (oldsize > arena_maxclass &&
5190 1503 : CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
5191 0 : size_t psize = PAGE_CEILING(size);
5192 : #ifdef MALLOC_FILL
5193 0 : if (opt_junk && size < oldsize) {
5194 0 : memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
5195 : - size);
5196 : }
5197 : #endif
5198 : #ifdef MALLOC_DECOMMIT
5199 : if (psize < oldsize) {
5200 : extent_node_t *node, key;
5201 :
5202 : pages_decommit((void *)((uintptr_t)ptr + psize),
5203 : oldsize - psize);
5204 :
5205 : /* Update recorded size. */
5206 : malloc_mutex_lock(&huge_mtx);
5207 : key.addr = __DECONST(void *, ptr);
5208 : node = extent_tree_ad_search(&huge, &key);
5209 : assert(node != NULL);
5210 : assert(node->size == oldsize);
5211 : # ifdef MALLOC_STATS
5212 : huge_allocated -= oldsize - psize;
5213 : # endif
5214 : node->size = psize;
5215 : malloc_mutex_unlock(&huge_mtx);
5216 : } else if (psize > oldsize) {
5217 : pages_commit((void *)((uintptr_t)ptr + oldsize),
5218 : psize - oldsize);
5219 : }
5220 : #endif
5221 :
5222 : /* Although we don't have to commit or decommit anything if
5223 : * DECOMMIT is not defined and the size class didn't change, we
5224 : * do need to update the recorded size if the size increased,
5225 : * so malloc_usable_size doesn't return a value smaller than
5226 : * what was requested via realloc(). */
5227 :
5228 0 : if (psize > oldsize) {
5229 : /* Update recorded size. */
5230 : extent_node_t *node, key;
5231 0 : malloc_mutex_lock(&huge_mtx);
5232 0 : key.addr = __DECONST(void *, ptr);
5233 0 : node = extent_tree_ad_search(&huge, &key);
5234 0 : assert(node != NULL);
5235 0 : assert(node->size == oldsize);
5236 : # ifdef MALLOC_STATS
5237 0 : huge_allocated += psize - oldsize;
5238 : # endif
5239 0 : node->size = psize;
5240 0 : malloc_mutex_unlock(&huge_mtx);
5241 : }
5242 :
5243 : #ifdef MALLOC_FILL
5244 0 : if (opt_zero && size > oldsize) {
5245 0 : memset((void *)((uintptr_t)ptr + oldsize), 0, size
5246 : - oldsize);
5247 : }
5248 : #endif
5249 0 : return (ptr);
5250 : }
5251 :
5252 : /*
5253 : * If we get here, then size and oldsize are different enough that we
5254 : * need to use a different size class. In that case, fall back to
5255 : * allocating new space and copying.
5256 : */
5257 1688 : ret = huge_malloc(size, false);
5258 1688 : if (ret == NULL)
5259 0 : return (NULL);
5260 :
5261 1688 : copysize = (size < oldsize) ? size : oldsize;
5262 : #ifdef VM_COPY_MIN
5263 : if (copysize >= VM_COPY_MIN)
5264 : pages_copy(ret, ptr, copysize);
5265 : else
5266 : #endif
5267 1688 : memcpy(ret, ptr, copysize);
5268 1688 : idalloc(ptr);
5269 1688 : return (ret);
5270 : }
5271 :
5272 : static void
5273 4536 : huge_dalloc(void *ptr)
5274 : {
5275 : extent_node_t *node, key;
5276 :
5277 4536 : malloc_mutex_lock(&huge_mtx);
5278 :
5279 : /* Extract from tree of huge allocations. */
5280 4536 : key.addr = ptr;
5281 4536 : node = extent_tree_ad_search(&huge, &key);
5282 4536 : assert(node != NULL);
5283 4536 : assert(node->addr == ptr);
5284 4536 : extent_tree_ad_remove(&huge, node);
5285 :
5286 : #ifdef MALLOC_STATS
5287 4536 : huge_ndalloc++;
5288 4536 : huge_allocated -= node->size;
5289 : #endif
5290 :
5291 4536 : malloc_mutex_unlock(&huge_mtx);
5292 :
5293 : /* Unmap chunk. */
5294 : #ifdef MALLOC_FILL
5295 4536 : if (opt_junk)
5296 4536 : memset(node->addr, 0x5a, node->size);
5297 : #endif
5298 4536 : chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
5299 : VALGRIND_FREELIKE_BLOCK(node->addr, 0);
5300 :
5301 4536 : base_node_dealloc(node);
5302 4536 : }
5303 :
5304 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5305 : #ifdef MOZ_MEMORY_BSD
5306 : static inline unsigned
5307 : malloc_ncpus(void)
5308 : {
5309 : unsigned ret;
5310 : int mib[2];
5311 : size_t len;
5312 :
5313 : mib[0] = CTL_HW;
5314 : mib[1] = HW_NCPU;
5315 : len = sizeof(ret);
5316 : if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
5317 : /* Error. */
5318 : return (1);
5319 : }
5320 :
5321 : return (ret);
5322 : }
5323 : #elif (defined(MOZ_MEMORY_LINUX))
5324 : #include <fcntl.h>
5325 :
5326 : static inline unsigned
5327 : malloc_ncpus(void)
5328 : {
5329 : unsigned ret;
5330 : int fd, nread, column;
5331 : char buf[1024];
5332 : static const char matchstr[] = "processor\t:";
5333 : int i;
5334 :
5335 : /*
5336 : * sysconf(3) would be the preferred method for determining the number
5337 : * of CPUs, but it uses malloc internally, which causes untennable
5338 : * recursion during malloc initialization.
5339 : */
5340 : fd = open("/proc/cpuinfo", O_RDONLY);
5341 : if (fd == -1)
5342 : return (1); /* Error. */
5343 : /*
5344 : * Count the number of occurrences of matchstr at the beginnings of
5345 : * lines. This treats hyperthreaded CPUs as multiple processors.
5346 : */
5347 : column = 0;
5348 : ret = 0;
5349 : while (true) {
5350 : nread = read(fd, &buf, sizeof(buf));
5351 : if (nread <= 0)
5352 : break; /* EOF or error. */
5353 : for (i = 0;i < nread;i++) {
5354 : char c = buf[i];
5355 : if (c == '\n')
5356 : column = 0;
5357 : else if (column != -1) {
5358 : if (c == matchstr[column]) {
5359 : column++;
5360 : if (column == sizeof(matchstr) - 1) {
5361 : column = -1;
5362 : ret++;
5363 : }
5364 : } else
5365 : column = -1;
5366 : }
5367 : }
5368 : }
5369 :
5370 : if (ret == 0)
5371 : ret = 1; /* Something went wrong in the parser. */
5372 : close(fd);
5373 :
5374 : return (ret);
5375 : }
5376 : #elif (defined(MOZ_MEMORY_DARWIN))
5377 : #include <mach/mach_init.h>
5378 : #include <mach/mach_host.h>
5379 :
5380 : static inline unsigned
5381 : malloc_ncpus(void)
5382 : {
5383 : kern_return_t error;
5384 : natural_t n;
5385 : processor_info_array_t pinfo;
5386 : mach_msg_type_number_t pinfocnt;
5387 :
5388 : error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
5389 : &n, &pinfo, &pinfocnt);
5390 : if (error != KERN_SUCCESS)
5391 : return (1); /* Error. */
5392 : else
5393 : return (n);
5394 : }
5395 : #elif (defined(MOZ_MEMORY_SOLARIS))
5396 :
5397 : static inline unsigned
5398 : malloc_ncpus(void)
5399 : {
5400 : return sysconf(_SC_NPROCESSORS_ONLN);
5401 : }
5402 : #else
5403 : static inline unsigned
5404 : malloc_ncpus(void)
5405 : {
5406 :
5407 : /*
5408 : * We lack a way to determine the number of CPUs on this platform, so
5409 : * assume 1 CPU.
5410 : */
5411 : return (1);
5412 : }
5413 : #endif
5414 : #endif
5415 :
5416 : static void
5417 0 : malloc_print_stats(void)
5418 : {
5419 :
5420 0 : if (opt_print_stats) {
5421 : char s[UMAX2S_BUFSIZE];
5422 0 : _malloc_message("___ Begin malloc statistics ___\n", "", "",
5423 : "");
5424 0 : _malloc_message("Assertions ",
5425 : #ifdef NDEBUG
5426 : "disabled",
5427 : #else
5428 : "enabled",
5429 : #endif
5430 : "\n", "");
5431 0 : _malloc_message("Boolean MALLOC_OPTIONS: ",
5432 : opt_abort ? "A" : "a", "", "");
5433 : #ifdef MALLOC_FILL
5434 0 : _malloc_message(opt_junk ? "J" : "j", "", "", "");
5435 : #endif
5436 : #ifdef MALLOC_PAGEFILE
5437 : _malloc_message(opt_pagefile ? "o" : "O", "", "", "");
5438 : #endif
5439 0 : _malloc_message("P", "", "", "");
5440 : #ifdef MALLOC_UTRACE
5441 0 : _malloc_message(opt_utrace ? "U" : "u", "", "", "");
5442 : #endif
5443 : #ifdef MALLOC_SYSV
5444 0 : _malloc_message(opt_sysv ? "V" : "v", "", "", "");
5445 : #endif
5446 : #ifdef MALLOC_XMALLOC
5447 0 : _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
5448 : #endif
5449 : #ifdef MALLOC_FILL
5450 0 : _malloc_message(opt_zero ? "Z" : "z", "", "", "");
5451 : #endif
5452 0 : _malloc_message("\n", "", "", "");
5453 :
5454 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5455 : _malloc_message("CPUs: ", umax2s(ncpus, 10, s), "\n", "");
5456 : #endif
5457 0 : _malloc_message("Max arenas: ", umax2s(narenas, 10, s), "\n",
5458 : "");
5459 : #ifdef MALLOC_BALANCE
5460 : _malloc_message("Arena balance threshold: ",
5461 : umax2s(opt_balance_threshold, 10, s), "\n", "");
5462 : #endif
5463 0 : _malloc_message("Pointer size: ", umax2s(sizeof(void *), 10, s),
5464 : "\n", "");
5465 0 : _malloc_message("Quantum size: ", umax2s(quantum, 10, s), "\n",
5466 : "");
5467 0 : _malloc_message("Max small size: ", umax2s(small_max, 10, s),
5468 : "\n", "");
5469 0 : _malloc_message("Max dirty pages per arena: ",
5470 0 : umax2s(opt_dirty_max, 10, s), "\n", "");
5471 :
5472 0 : _malloc_message("Chunk size: ", umax2s(chunksize, 10, s), "",
5473 : "");
5474 0 : _malloc_message(" (2^", umax2s(opt_chunk_2pow, 10, s), ")\n",
5475 : "");
5476 :
5477 : #ifdef MALLOC_STATS
5478 : {
5479 : size_t allocated, mapped;
5480 : #ifdef MALLOC_BALANCE
5481 : uint64_t nbalance = 0;
5482 : #endif
5483 : unsigned i;
5484 : arena_t *arena;
5485 :
5486 : /* Calculate and print allocated/mapped stats. */
5487 :
5488 : /* arenas. */
5489 0 : for (i = 0, allocated = 0; i < narenas; i++) {
5490 0 : if (arenas[i] != NULL) {
5491 0 : malloc_spin_lock(&arenas[i]->lock);
5492 0 : allocated +=
5493 0 : arenas[i]->stats.allocated_small;
5494 0 : allocated +=
5495 0 : arenas[i]->stats.allocated_large;
5496 : #ifdef MALLOC_BALANCE
5497 : nbalance += arenas[i]->stats.nbalance;
5498 : #endif
5499 0 : malloc_spin_unlock(&arenas[i]->lock);
5500 : }
5501 : }
5502 :
5503 : /* huge/base. */
5504 0 : malloc_mutex_lock(&huge_mtx);
5505 0 : allocated += huge_allocated;
5506 0 : mapped = stats_chunks.curchunks * chunksize;
5507 0 : malloc_mutex_unlock(&huge_mtx);
5508 :
5509 0 : malloc_mutex_lock(&base_mtx);
5510 0 : mapped += base_mapped;
5511 0 : malloc_mutex_unlock(&base_mtx);
5512 :
5513 : #ifdef MOZ_MEMORY_WINDOWS
5514 : malloc_printf("Allocated: %lu, mapped: %lu\n",
5515 : allocated, mapped);
5516 : #else
5517 0 : malloc_printf("Allocated: %zu, mapped: %zu\n",
5518 : allocated, mapped);
5519 : #endif
5520 :
5521 : #ifdef MALLOC_BALANCE
5522 : malloc_printf("Arena balance reassignments: %llu\n",
5523 : nbalance);
5524 : #endif
5525 :
5526 : /* Print chunk stats. */
5527 : {
5528 : chunk_stats_t chunks_stats;
5529 :
5530 0 : malloc_mutex_lock(&huge_mtx);
5531 0 : chunks_stats = stats_chunks;
5532 0 : malloc_mutex_unlock(&huge_mtx);
5533 :
5534 0 : malloc_printf("chunks: nchunks "
5535 : "highchunks curchunks\n");
5536 0 : malloc_printf(" %13llu%13lu%13lu\n",
5537 : chunks_stats.nchunks,
5538 : chunks_stats.highchunks,
5539 : chunks_stats.curchunks);
5540 : }
5541 :
5542 : /* Print chunk stats. */
5543 0 : malloc_printf(
5544 : "huge: nmalloc ndalloc allocated\n");
5545 : #ifdef MOZ_MEMORY_WINDOWS
5546 : malloc_printf(" %12llu %12llu %12lu\n",
5547 : huge_nmalloc, huge_ndalloc, huge_allocated);
5548 : #else
5549 0 : malloc_printf(" %12llu %12llu %12zu\n",
5550 : huge_nmalloc, huge_ndalloc, huge_allocated);
5551 : #endif
5552 : /* Print stats for each arena. */
5553 0 : for (i = 0; i < narenas; i++) {
5554 0 : arena = arenas[i];
5555 0 : if (arena != NULL) {
5556 0 : malloc_printf(
5557 : "\narenas[%u]:\n", i);
5558 0 : malloc_spin_lock(&arena->lock);
5559 0 : stats_print(arena);
5560 0 : malloc_spin_unlock(&arena->lock);
5561 : }
5562 : }
5563 : }
5564 : #endif /* #ifdef MALLOC_STATS */
5565 0 : _malloc_message("--- End malloc statistics ---\n", "", "", "");
5566 : }
5567 0 : }
5568 :
5569 : /*
5570 : * FreeBSD's pthreads implementation calls malloc(3), so the malloc
5571 : * implementation has to take pains to avoid infinite recursion during
5572 : * initialization.
5573 : */
5574 : #if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN))
5575 : #define malloc_init() false
5576 : #else
5577 : static inline bool
5578 44114699 : malloc_init(void)
5579 : {
5580 :
5581 44114699 : if (malloc_initialized == false)
5582 18669 : return (malloc_init_hard());
5583 :
5584 44096030 : return (false);
5585 : }
5586 : #endif
5587 :
5588 : #if !defined(MOZ_MEMORY_WINDOWS)
5589 : static
5590 : #endif
5591 : bool
5592 18669 : malloc_init_hard(void)
5593 : {
5594 : unsigned i;
5595 : char buf[PATH_MAX + 1];
5596 : const char *opts;
5597 : long result;
5598 : #ifndef MOZ_MEMORY_WINDOWS
5599 : int linklen;
5600 : #endif
5601 : #ifdef MOZ_MEMORY_DARWIN
5602 : malloc_zone_t* default_zone;
5603 : #endif
5604 :
5605 : #ifndef MOZ_MEMORY_WINDOWS
5606 18669 : malloc_mutex_lock(&init_lock);
5607 : #endif
5608 :
5609 18669 : if (malloc_initialized) {
5610 : /*
5611 : * Another thread initialized the allocator before this one
5612 : * acquired init_lock.
5613 : */
5614 : #ifndef MOZ_MEMORY_WINDOWS
5615 0 : malloc_mutex_unlock(&init_lock);
5616 : #endif
5617 0 : return (false);
5618 : }
5619 :
5620 : #ifdef MOZ_MEMORY_WINDOWS
5621 : /* get a thread local storage index */
5622 : tlsIndex = TlsAlloc();
5623 : #endif
5624 :
5625 : /* Get page size and number of CPUs */
5626 : #ifdef MOZ_MEMORY_WINDOWS
5627 : {
5628 : SYSTEM_INFO info;
5629 :
5630 : GetSystemInfo(&info);
5631 : result = info.dwPageSize;
5632 :
5633 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5634 : ncpus = info.dwNumberOfProcessors;
5635 : #endif
5636 : }
5637 : #else
5638 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5639 : ncpus = malloc_ncpus();
5640 : #endif
5641 :
5642 18669 : result = sysconf(_SC_PAGESIZE);
5643 18669 : assert(result != -1);
5644 : #endif
5645 :
5646 : /* We assume that the page size is a power of 2. */
5647 18669 : assert(((result - 1) & result) == 0);
5648 : #ifdef MALLOC_STATIC_SIZES
5649 18669 : if (pagesize % (size_t) result) {
5650 0 : _malloc_message(_getprogname(),
5651 : "Compile-time page size does not divide the runtime one.\n",
5652 : "", "");
5653 0 : abort();
5654 : }
5655 : #else
5656 : pagesize = (size_t) result;
5657 : pagesize_mask = (size_t) result - 1;
5658 : pagesize_2pow = ffs((int)result) - 1;
5659 : #endif
5660 :
5661 : #ifdef MALLOC_PAGEFILE
5662 : /*
5663 : * Determine where to create page files. It is insufficient to
5664 : * unconditionally use P_tmpdir (typically "/tmp"), since for some
5665 : * operating systems /tmp is a separate filesystem that is rather small.
5666 : * Therefore prefer, in order, the following locations:
5667 : *
5668 : * 1) MALLOC_TMPDIR
5669 : * 2) TMPDIR
5670 : * 3) P_tmpdir
5671 : */
5672 : {
5673 : char *s;
5674 : size_t slen;
5675 : static const char suffix[] = "/jemalloc.XXXXXX";
5676 :
5677 : if ((s = getenv("MALLOC_TMPDIR")) == NULL && (s =
5678 : getenv("TMPDIR")) == NULL)
5679 : s = P_tmpdir;
5680 : slen = strlen(s);
5681 : if (slen + sizeof(suffix) > sizeof(pagefile_templ)) {
5682 : _malloc_message(_getprogname(),
5683 : ": (malloc) Page file path too long\n",
5684 : "", "");
5685 : abort();
5686 : }
5687 : memcpy(pagefile_templ, s, slen);
5688 : memcpy(&pagefile_templ[slen], suffix, sizeof(suffix));
5689 : }
5690 : #endif
5691 :
5692 74676 : for (i = 0; i < 3; i++) {
5693 : unsigned j;
5694 :
5695 : /* Get runtime configuration. */
5696 56007 : switch (i) {
5697 : case 0:
5698 : #ifndef MOZ_MEMORY_WINDOWS
5699 18669 : if ((linklen = readlink("/etc/malloc.conf", buf,
5700 : sizeof(buf) - 1)) != -1) {
5701 : /*
5702 : * Use the contents of the "/etc/malloc.conf"
5703 : * symbolic link's name.
5704 : */
5705 0 : buf[linklen] = '\0';
5706 0 : opts = buf;
5707 : } else
5708 : #endif
5709 : {
5710 : /* No configuration specified. */
5711 18669 : buf[0] = '\0';
5712 18669 : opts = buf;
5713 : }
5714 18669 : break;
5715 : case 1:
5716 18669 : if (issetugid() == 0 && (opts =
5717 : getenv("MALLOC_OPTIONS")) != NULL) {
5718 : /*
5719 : * Do nothing; opts is already initialized to
5720 : * the value of the MALLOC_OPTIONS environment
5721 : * variable.
5722 : */
5723 : } else {
5724 : /* No configuration specified. */
5725 18669 : buf[0] = '\0';
5726 18669 : opts = buf;
5727 : }
5728 18669 : break;
5729 : case 2:
5730 18669 : if (_malloc_options != NULL) {
5731 : /*
5732 : * Use options that were compiled into the
5733 : * program.
5734 : */
5735 0 : opts = _malloc_options;
5736 : } else {
5737 : /* No configuration specified. */
5738 18669 : buf[0] = '\0';
5739 18669 : opts = buf;
5740 : }
5741 18669 : break;
5742 : default:
5743 : /* NOTREACHED */
5744 0 : buf[0] = '\0';
5745 0 : opts = buf;
5746 0 : assert(false);
5747 : }
5748 :
5749 56007 : for (j = 0; opts[j] != '\0'; j++) {
5750 : unsigned k, nreps;
5751 : bool nseen;
5752 :
5753 : /* Parse repetition count, if any. */
5754 0 : for (nreps = 0, nseen = false;; j++, nseen = true) {
5755 0 : switch (opts[j]) {
5756 : case '0': case '1': case '2': case '3':
5757 : case '4': case '5': case '6': case '7':
5758 : case '8': case '9':
5759 0 : nreps *= 10;
5760 0 : nreps += opts[j] - '0';
5761 : break;
5762 : default:
5763 0 : goto MALLOC_OUT;
5764 : }
5765 0 : }
5766 : MALLOC_OUT:
5767 0 : if (nseen == false)
5768 0 : nreps = 1;
5769 :
5770 0 : for (k = 0; k < nreps; k++) {
5771 0 : switch (opts[j]) {
5772 : case 'a':
5773 0 : opt_abort = false;
5774 0 : break;
5775 : case 'A':
5776 0 : opt_abort = true;
5777 0 : break;
5778 : case 'b':
5779 : #ifdef MALLOC_BALANCE
5780 : opt_balance_threshold >>= 1;
5781 : #endif
5782 0 : break;
5783 : case 'B':
5784 : #ifdef MALLOC_BALANCE
5785 : if (opt_balance_threshold == 0)
5786 : opt_balance_threshold = 1;
5787 : else if ((opt_balance_threshold << 1)
5788 : > opt_balance_threshold)
5789 : opt_balance_threshold <<= 1;
5790 : #endif
5791 0 : break;
5792 : case 'f':
5793 0 : opt_dirty_max >>= 1;
5794 0 : break;
5795 : case 'F':
5796 0 : if (opt_dirty_max == 0)
5797 0 : opt_dirty_max = 1;
5798 0 : else if ((opt_dirty_max << 1) != 0)
5799 0 : opt_dirty_max <<= 1;
5800 0 : break;
5801 : #ifdef MALLOC_FILL
5802 : case 'j':
5803 0 : opt_junk = false;
5804 0 : break;
5805 : case 'J':
5806 0 : opt_junk = true;
5807 0 : break;
5808 : #endif
5809 : #ifndef MALLOC_STATIC_SIZES
5810 : case 'k':
5811 : /*
5812 : * Chunks always require at least one
5813 : * header page, so chunks can never be
5814 : * smaller than two pages.
5815 : */
5816 : if (opt_chunk_2pow > pagesize_2pow + 1)
5817 : opt_chunk_2pow--;
5818 : break;
5819 : case 'K':
5820 : if (opt_chunk_2pow + 1 <
5821 : (sizeof(size_t) << 3))
5822 : opt_chunk_2pow++;
5823 : break;
5824 : #endif
5825 : case 'n':
5826 0 : opt_narenas_lshift--;
5827 0 : break;
5828 : case 'N':
5829 0 : opt_narenas_lshift++;
5830 0 : break;
5831 : #ifdef MALLOC_PAGEFILE
5832 : case 'o':
5833 : /* Do not over-commit. */
5834 : opt_pagefile = true;
5835 : break;
5836 : case 'O':
5837 : /* Allow over-commit. */
5838 : opt_pagefile = false;
5839 : break;
5840 : #endif
5841 : case 'p':
5842 0 : opt_print_stats = false;
5843 0 : break;
5844 : case 'P':
5845 0 : opt_print_stats = true;
5846 0 : break;
5847 : #ifndef MALLOC_STATIC_SIZES
5848 : case 'q':
5849 : if (opt_quantum_2pow > QUANTUM_2POW_MIN)
5850 : opt_quantum_2pow--;
5851 : break;
5852 : case 'Q':
5853 : if (opt_quantum_2pow < pagesize_2pow -
5854 : 1)
5855 : opt_quantum_2pow++;
5856 : break;
5857 : case 's':
5858 : if (opt_small_max_2pow >
5859 : QUANTUM_2POW_MIN)
5860 : opt_small_max_2pow--;
5861 : break;
5862 : case 'S':
5863 : if (opt_small_max_2pow < pagesize_2pow
5864 : - 1)
5865 : opt_small_max_2pow++;
5866 : break;
5867 : #endif
5868 : #ifdef MALLOC_UTRACE
5869 : case 'u':
5870 0 : opt_utrace = false;
5871 0 : break;
5872 : case 'U':
5873 0 : opt_utrace = true;
5874 0 : break;
5875 : #endif
5876 : #ifdef MALLOC_SYSV
5877 : case 'v':
5878 0 : opt_sysv = false;
5879 0 : break;
5880 : case 'V':
5881 0 : opt_sysv = true;
5882 0 : break;
5883 : #endif
5884 : #ifdef MALLOC_XMALLOC
5885 : case 'x':
5886 0 : opt_xmalloc = false;
5887 0 : break;
5888 : case 'X':
5889 0 : opt_xmalloc = true;
5890 0 : break;
5891 : #endif
5892 : #ifdef MALLOC_FILL
5893 : case 'z':
5894 0 : opt_zero = false;
5895 0 : break;
5896 : case 'Z':
5897 0 : opt_zero = true;
5898 0 : break;
5899 : #endif
5900 : default: {
5901 : char cbuf[2];
5902 :
5903 0 : cbuf[0] = opts[j];
5904 0 : cbuf[1] = '\0';
5905 0 : _malloc_message(_getprogname(),
5906 : ": (malloc) Unsupported character "
5907 : "in malloc options: '", cbuf,
5908 : "'\n");
5909 : }
5910 : }
5911 : }
5912 : }
5913 : }
5914 :
5915 : /* Take care to call atexit() only once. */
5916 18669 : if (opt_print_stats) {
5917 : #ifndef MOZ_MEMORY_WINDOWS
5918 : /* Print statistics at exit. */
5919 0 : atexit(malloc_print_stats);
5920 : #endif
5921 : }
5922 :
5923 : #if !defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN)
5924 : /* Prevent potential deadlock on malloc locks after fork. */
5925 18669 : pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
5926 : #endif
5927 :
5928 : #ifndef MALLOC_STATIC_SIZES
5929 : /* Set variables according to the value of opt_small_max_2pow. */
5930 : if (opt_small_max_2pow < opt_quantum_2pow)
5931 : opt_small_max_2pow = opt_quantum_2pow;
5932 : small_max = (1U << opt_small_max_2pow);
5933 :
5934 : /* Set bin-related variables. */
5935 : bin_maxclass = (pagesize >> 1);
5936 : assert(opt_quantum_2pow >= TINY_MIN_2POW);
5937 : ntbins = opt_quantum_2pow - TINY_MIN_2POW;
5938 : assert(ntbins <= opt_quantum_2pow);
5939 : nqbins = (small_max >> opt_quantum_2pow);
5940 : nsbins = pagesize_2pow - opt_small_max_2pow - 1;
5941 :
5942 : /* Set variables according to the value of opt_quantum_2pow. */
5943 : quantum = (1U << opt_quantum_2pow);
5944 : quantum_mask = quantum - 1;
5945 : if (ntbins > 0)
5946 : small_min = (quantum >> 1) + 1;
5947 : else
5948 : small_min = 1;
5949 : assert(small_min <= quantum);
5950 :
5951 : /* Set variables according to the value of opt_chunk_2pow. */
5952 : chunksize = (1LU << opt_chunk_2pow);
5953 : chunksize_mask = chunksize - 1;
5954 : chunk_npages = (chunksize >> pagesize_2pow);
5955 :
5956 : arena_chunk_header_npages = calculate_arena_header_pages();
5957 : arena_maxclass = calculate_arena_maxclass();
5958 : #endif
5959 :
5960 : #ifdef JEMALLOC_USES_MAP_ALIGN
5961 : /*
5962 : * When using MAP_ALIGN, the alignment parameter must be a power of two
5963 : * multiple of the system pagesize, or mmap will fail.
5964 : */
5965 : assert((chunksize % pagesize) == 0);
5966 : assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
5967 : #endif
5968 :
5969 18669 : UTRACE(0, 0, 0);
5970 :
5971 : #ifdef MALLOC_STATS
5972 18669 : memset(&stats_chunks, 0, sizeof(chunk_stats_t));
5973 : #endif
5974 :
5975 : /* Various sanity checks that regard configuration. */
5976 18669 : assert(quantum >= sizeof(void *));
5977 18669 : assert(quantum <= pagesize);
5978 18669 : assert(chunksize >= pagesize);
5979 18669 : assert(quantum * 4 <= chunksize);
5980 :
5981 : /* Initialize chunks data. */
5982 18669 : malloc_mutex_init(&huge_mtx);
5983 18669 : extent_tree_ad_new(&huge);
5984 : #ifdef MALLOC_STATS
5985 18669 : huge_nmalloc = 0;
5986 18669 : huge_ndalloc = 0;
5987 18669 : huge_allocated = 0;
5988 : #endif
5989 :
5990 : /* Initialize base allocation data structures. */
5991 : #ifdef MALLOC_STATS
5992 18669 : base_mapped = 0;
5993 18669 : base_committed = 0;
5994 : #endif
5995 18669 : base_nodes = NULL;
5996 18669 : malloc_mutex_init(&base_mtx);
5997 :
5998 : #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5999 18669 : narenas = 1;
6000 : #else
6001 : if (ncpus > 1) {
6002 : /*
6003 : * For SMP systems, create four times as many arenas as there
6004 : * are CPUs by default.
6005 : */
6006 : opt_narenas_lshift += 2;
6007 : }
6008 :
6009 : /* Determine how many arenas to use. */
6010 : narenas = ncpus;
6011 : #endif
6012 18669 : if (opt_narenas_lshift > 0) {
6013 0 : if ((narenas << opt_narenas_lshift) > narenas)
6014 0 : narenas <<= opt_narenas_lshift;
6015 : /*
6016 : * Make sure not to exceed the limits of what base_alloc() can
6017 : * handle.
6018 : */
6019 0 : if (narenas * sizeof(arena_t *) > chunksize)
6020 0 : narenas = chunksize / sizeof(arena_t *);
6021 18669 : } else if (opt_narenas_lshift < 0) {
6022 0 : if ((narenas >> -opt_narenas_lshift) < narenas)
6023 0 : narenas >>= -opt_narenas_lshift;
6024 : /* Make sure there is at least one arena. */
6025 0 : if (narenas == 0)
6026 0 : narenas = 1;
6027 : }
6028 : #ifdef MALLOC_BALANCE
6029 : assert(narenas != 0);
6030 : for (narenas_2pow = 0;
6031 : (narenas >> (narenas_2pow + 1)) != 0;
6032 : narenas_2pow++);
6033 : #endif
6034 :
6035 : #ifdef NO_TLS
6036 : if (narenas > 1) {
6037 : static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
6038 : 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
6039 : 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
6040 : 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
6041 : 223, 227, 229, 233, 239, 241, 251, 257, 263};
6042 : unsigned nprimes, parenas;
6043 :
6044 : /*
6045 : * Pick a prime number of hash arenas that is more than narenas
6046 : * so that direct hashing of pthread_self() pointers tends to
6047 : * spread allocations evenly among the arenas.
6048 : */
6049 : assert((narenas & 1) == 0); /* narenas must be even. */
6050 : nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
6051 : parenas = primes[nprimes - 1]; /* In case not enough primes. */
6052 : for (i = 1; i < nprimes; i++) {
6053 : if (primes[i] > narenas) {
6054 : parenas = primes[i];
6055 : break;
6056 : }
6057 : }
6058 : narenas = parenas;
6059 : }
6060 : #endif
6061 :
6062 : #ifndef NO_TLS
6063 : # ifndef MALLOC_BALANCE
6064 18669 : next_arena = 0;
6065 : # endif
6066 : #endif
6067 :
6068 : /* Allocate and initialize arenas. */
6069 18669 : arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
6070 18669 : if (arenas == NULL) {
6071 : #ifndef MOZ_MEMORY_WINDOWS
6072 0 : malloc_mutex_unlock(&init_lock);
6073 : #endif
6074 0 : return (true);
6075 : }
6076 : /*
6077 : * Zero the array. In practice, this should always be pre-zeroed,
6078 : * since it was just mmap()ed, but let's be sure.
6079 : */
6080 18669 : memset(arenas, 0, sizeof(arena_t *) * narenas);
6081 :
6082 : /*
6083 : * Initialize one arena here. The rest are lazily created in
6084 : * choose_arena_hard().
6085 : */
6086 18669 : arenas_extend(0);
6087 18669 : if (arenas[0] == NULL) {
6088 : #ifndef MOZ_MEMORY_WINDOWS
6089 0 : malloc_mutex_unlock(&init_lock);
6090 : #endif
6091 0 : return (true);
6092 : }
6093 : #ifndef NO_TLS
6094 : /*
6095 : * Assign the initial arena to the initial thread, in order to avoid
6096 : * spurious creation of an extra arena if the application switches to
6097 : * threaded mode.
6098 : */
6099 : #ifdef MOZ_MEMORY_WINDOWS
6100 : TlsSetValue(tlsIndex, arenas[0]);
6101 : #else
6102 18669 : arenas_map = arenas[0];
6103 : #endif
6104 : #endif
6105 :
6106 : /*
6107 : * Seed here for the initial thread, since choose_arena_hard() is only
6108 : * called for other threads. The seed value doesn't really matter.
6109 : */
6110 : #ifdef MALLOC_BALANCE
6111 : SPRN(balance, 42);
6112 : #endif
6113 :
6114 18669 : malloc_spin_init(&arenas_lock);
6115 :
6116 : #ifdef MALLOC_VALIDATE
6117 18669 : chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
6118 18669 : if (chunk_rtree == NULL)
6119 0 : return (true);
6120 : #endif
6121 :
6122 18669 : malloc_initialized = true;
6123 :
6124 : #if defined(NEEDS_PTHREAD_MMAP_UNALIGNED_TSD)
6125 : if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
6126 : malloc_printf("<jemalloc>: Error in pthread_key_create()\n");
6127 : }
6128 : #endif
6129 :
6130 : #ifdef MOZ_MEMORY_DARWIN
6131 : /*
6132 : * Overwrite the default memory allocator to use jemalloc everywhere.
6133 : */
6134 : default_zone = malloc_default_zone();
6135 :
6136 : /*
6137 : * We only use jemalloc with MacOS 10.6 and 10.7. jemalloc is disabled
6138 : * on 32-bit builds (10.5 and 32-bit 10.6) due to bug 702250, an
6139 : * apparent MacOS bug. In fact, this code isn't even compiled on
6140 : * 32-bit builds.
6141 : *
6142 : * We'll have to update our code to work with newer versions, because
6143 : * the malloc zone layout is likely to change.
6144 : */
6145 :
6146 : osx_use_jemalloc = (default_zone->version == SNOW_LEOPARD_MALLOC_ZONE_T_VERSION ||
6147 : default_zone->version == LION_MALLOC_ZONE_T_VERSION);
6148 :
6149 : /* Allow us dynamically turn off jemalloc for testing. */
6150 : if (getenv("NO_MAC_JEMALLOC")) {
6151 : osx_use_jemalloc = false;
6152 : #ifdef __i386__
6153 : malloc_printf("Warning: NO_MAC_JEMALLOC has no effect on "
6154 : "i386 machines (such as this one).\n");
6155 : #endif
6156 : }
6157 :
6158 : if (osx_use_jemalloc) {
6159 : /*
6160 : * Convert the default szone to an "overlay zone" that is capable
6161 : * of deallocating szone-allocated objects, but allocating new
6162 : * objects from jemalloc.
6163 : */
6164 : size_t size = zone_version_size(default_zone->version);
6165 : szone2ozone(default_zone, size);
6166 : }
6167 : else {
6168 : szone = default_zone;
6169 : }
6170 : #endif
6171 :
6172 : #ifndef MOZ_MEMORY_WINDOWS
6173 18669 : malloc_mutex_unlock(&init_lock);
6174 : #endif
6175 18669 : return (false);
6176 : }
6177 :
6178 : /* XXX Why not just expose malloc_print_stats()? */
6179 : #ifdef MOZ_MEMORY_WINDOWS
6180 : void
6181 : malloc_shutdown()
6182 : {
6183 :
6184 : malloc_print_stats();
6185 : }
6186 : #endif
6187 :
6188 : /*
6189 : * End general internal functions.
6190 : */
6191 : /******************************************************************************/
6192 : /*
6193 : * Begin malloc(3)-compatible functions.
6194 : */
6195 :
6196 : /*
6197 : * Mangle standard interfaces, in order to avoid linking problems.
6198 : */
6199 : #if defined(MOZ_MEMORY_DARWIN) || defined(MOZ_MEMORY_WINDOWS) || \
6200 : defined(MOZ_MEMORY_ANDROID)
6201 :
6202 : #ifdef MOZ_MEMORY_ANDROID
6203 : /*
6204 : * On Android, we use __wrap_* instead of je_* to accomodate with the
6205 : * linker's --wrap option we use. That option prefixes the function
6206 : * names it is given with __wrap_.
6207 : */
6208 : #define wrap(a) __wrap_ ## a
6209 : #else
6210 : #define wrap(a) je_ ## a
6211 : #endif
6212 :
6213 : #define malloc(a) wrap(malloc)(a)
6214 : #define memalign(a, b) wrap(memalign)(a, b)
6215 : #define posix_memalign(a, b, c) wrap(posix_memalign)(a, b, c)
6216 : #define valloc(a) wrap(valloc)(a)
6217 : #define calloc(a, b) wrap(calloc)(a, b)
6218 : #define realloc(a, b) wrap(realloc)(a, b)
6219 : #define free(a) wrap(free)(a)
6220 : #define malloc_usable_size(a) wrap(malloc_usable_size)(a)
6221 : #endif
6222 :
6223 : /*
6224 : * Even though we compile with MOZ_MEMORY, we may have to dynamically decide
6225 : * not to use jemalloc, as discussed above. However, we call jemalloc
6226 : * functions directly from mozalloc. Since it's pretty dangerous to mix the
6227 : * allocators, we need to call the OSX allocators from the functions below,
6228 : * when osx_use_jemalloc is not (dynamically) set.
6229 : *
6230 : * Note that we assume jemalloc is enabled on i386. This is safe because the
6231 : * only i386 versions of MacOS are 10.5 and 10.6, which we support. We have to
6232 : * do this because madvise isn't in the malloc zone struct for 10.5.
6233 : *
6234 : * This means that NO_MAC_JEMALLOC doesn't work on i386.
6235 : */
6236 : #if defined(MOZ_MEMORY_DARWIN) && !defined(__i386__)
6237 : #define DARWIN_ONLY(A) if (!osx_use_jemalloc) { A; }
6238 : #else
6239 : #define DARWIN_ONLY(A)
6240 : #endif
6241 :
6242 : void *
6243 : malloc(size_t size)
6244 : {
6245 : void *ret;
6246 :
6247 : DARWIN_ONLY(return (szone->malloc)(szone, size));
6248 :
6249 37094556 : if (malloc_init()) {
6250 0 : ret = NULL;
6251 0 : goto RETURN;
6252 : }
6253 :
6254 37094556 : if (size == 0) {
6255 : #ifdef MALLOC_SYSV
6256 22 : if (opt_sysv == false)
6257 : #endif
6258 22 : size = 1;
6259 : #ifdef MALLOC_SYSV
6260 : else {
6261 0 : ret = NULL;
6262 0 : goto RETURN;
6263 : }
6264 : #endif
6265 : }
6266 :
6267 37094556 : ret = imalloc(size);
6268 :
6269 : RETURN:
6270 37094556 : if (ret == NULL) {
6271 : #ifdef MALLOC_XMALLOC
6272 0 : if (opt_xmalloc) {
6273 0 : _malloc_message(_getprogname(),
6274 : ": (malloc) Error in malloc(): out of memory\n", "",
6275 : "");
6276 0 : abort();
6277 : }
6278 : #endif
6279 0 : errno = ENOMEM;
6280 : }
6281 :
6282 37094556 : UTRACE(0, size, ret);
6283 37094556 : return (ret);
6284 : }
6285 :
6286 : /*
6287 : * In ELF systems the default visibility allows symbols to be preempted at
6288 : * runtime. This in turn prevents the uses of memalign in this file from being
6289 : * optimized. What we do in here is define two aliasing symbols (they point to
6290 : * the same code): memalign and memalign_internal. The internal version has
6291 : * hidden visibility and is used in every reference from this file.
6292 : *
6293 : * For more information on this technique, see section 2.2.7 (Avoid Using
6294 : * Exported Symbols) in http://www.akkadia.org/drepper/dsohowto.pdf.
6295 : */
6296 :
6297 : #if defined(__GNUC__) && !defined(MOZ_MEMORY_DARWIN)
6298 : #define MOZ_MEMORY_ELF
6299 : #endif
6300 :
6301 : #ifdef MOZ_MEMORY_SOLARIS
6302 : # ifdef __SUNPRO_C
6303 : void *
6304 : memalign(size_t alignment, size_t size);
6305 : #pragma no_inline(memalign)
6306 : # elif (defined(__GNUC__))
6307 : __attribute__((noinline))
6308 : # endif
6309 : #else
6310 : #if (defined(MOZ_MEMORY_ELF))
6311 : __attribute__((visibility ("hidden")))
6312 : #endif
6313 : #endif
6314 :
6315 :
6316 : #ifdef MOZ_MEMORY_ELF
6317 : #define MEMALIGN memalign_internal
6318 : #else
6319 : #define MEMALIGN memalign
6320 : #endif
6321 :
6322 : void *
6323 0 : MEMALIGN(size_t alignment, size_t size)
6324 : {
6325 : void *ret;
6326 :
6327 : DARWIN_ONLY(return (szone->memalign)(szone, alignment, size));
6328 :
6329 0 : assert(((alignment - 1) & alignment) == 0);
6330 :
6331 0 : if (malloc_init()) {
6332 0 : ret = NULL;
6333 0 : goto RETURN;
6334 : }
6335 :
6336 0 : if (size == 0) {
6337 : #ifdef MALLOC_SYSV
6338 0 : if (opt_sysv == false)
6339 : #endif
6340 0 : size = 1;
6341 : #ifdef MALLOC_SYSV
6342 : else {
6343 0 : ret = NULL;
6344 0 : goto RETURN;
6345 : }
6346 : #endif
6347 : }
6348 :
6349 0 : alignment = alignment < sizeof(void*) ? sizeof(void*) : alignment;
6350 0 : ret = ipalloc(alignment, size);
6351 :
6352 : RETURN:
6353 : #ifdef MALLOC_XMALLOC
6354 0 : if (opt_xmalloc && ret == NULL) {
6355 0 : _malloc_message(_getprogname(),
6356 : ": (malloc) Error in memalign(): out of memory\n", "", "");
6357 0 : abort();
6358 : }
6359 : #endif
6360 0 : UTRACE(0, size, ret);
6361 0 : return (ret);
6362 : }
6363 :
6364 : #ifdef MOZ_MEMORY_ELF
6365 : extern void *
6366 : memalign(size_t alignment, size_t size) __attribute__((alias ("memalign_internal"), visibility ("default")));
6367 : #endif
6368 :
6369 : int
6370 : posix_memalign(void **memptr, size_t alignment, size_t size)
6371 : {
6372 : void *result;
6373 :
6374 : /* Make sure that alignment is a large enough power of 2. */
6375 0 : if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
6376 : #ifdef MALLOC_XMALLOC
6377 0 : if (opt_xmalloc) {
6378 0 : _malloc_message(_getprogname(),
6379 : ": (malloc) Error in posix_memalign(): "
6380 : "invalid alignment\n", "", "");
6381 0 : abort();
6382 : }
6383 : #endif
6384 0 : return (EINVAL);
6385 : }
6386 :
6387 : /* The 0-->1 size promotion is done in the memalign() call below */
6388 :
6389 0 : result = MEMALIGN(alignment, size);
6390 :
6391 0 : if (result == NULL)
6392 0 : return (ENOMEM);
6393 :
6394 0 : *memptr = result;
6395 0 : return (0);
6396 : }
6397 :
6398 : void *
6399 : valloc(size_t size)
6400 : {
6401 0 : return (MEMALIGN(pagesize, size));
6402 : }
6403 :
6404 : void *
6405 : calloc(size_t num, size_t size)
6406 : {
6407 : void *ret;
6408 : size_t num_size;
6409 :
6410 : DARWIN_ONLY(return (szone->calloc)(szone, num, size));
6411 :
6412 2256419 : if (malloc_init()) {
6413 0 : num_size = 0;
6414 0 : ret = NULL;
6415 0 : goto RETURN;
6416 : }
6417 :
6418 2256419 : num_size = num * size;
6419 2256419 : if (num_size == 0) {
6420 : #ifdef MALLOC_SYSV
6421 0 : if ((opt_sysv == false) && ((num == 0) || (size == 0)))
6422 : #endif
6423 0 : num_size = 1;
6424 : #ifdef MALLOC_SYSV
6425 : else {
6426 0 : ret = NULL;
6427 0 : goto RETURN;
6428 : }
6429 : #endif
6430 : /*
6431 : * Try to avoid division here. We know that it isn't possible to
6432 : * overflow during multiplication if neither operand uses any of the
6433 : * most significant half of the bits in a size_t.
6434 : */
6435 2256419 : } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
6436 551 : && (num_size / size != num)) {
6437 : /* size_t overflow. */
6438 0 : ret = NULL;
6439 0 : goto RETURN;
6440 : }
6441 :
6442 2256419 : ret = icalloc(num_size);
6443 :
6444 : RETURN:
6445 2256419 : if (ret == NULL) {
6446 : #ifdef MALLOC_XMALLOC
6447 18 : if (opt_xmalloc) {
6448 0 : _malloc_message(_getprogname(),
6449 : ": (malloc) Error in calloc(): out of memory\n", "",
6450 : "");
6451 0 : abort();
6452 : }
6453 : #endif
6454 18 : errno = ENOMEM;
6455 : }
6456 :
6457 2256419 : UTRACE(0, num_size, ret);
6458 2256419 : return (ret);
6459 : }
6460 :
6461 : void *
6462 : realloc(void *ptr, size_t size)
6463 : {
6464 : void *ret;
6465 :
6466 : DARWIN_ONLY(return (szone->realloc)(szone, ptr, size));
6467 :
6468 7065387 : if (size == 0) {
6469 : #ifdef MALLOC_SYSV
6470 0 : if (opt_sysv == false)
6471 : #endif
6472 0 : size = 1;
6473 : #ifdef MALLOC_SYSV
6474 : else {
6475 0 : if (ptr != NULL)
6476 0 : idalloc(ptr);
6477 0 : ret = NULL;
6478 0 : goto RETURN;
6479 : }
6480 : #endif
6481 : }
6482 :
6483 7065387 : if (ptr != NULL) {
6484 2301663 : assert(malloc_initialized);
6485 :
6486 2301663 : ret = iralloc(ptr, size);
6487 :
6488 2301663 : if (ret == NULL) {
6489 : #ifdef MALLOC_XMALLOC
6490 0 : if (opt_xmalloc) {
6491 0 : _malloc_message(_getprogname(),
6492 : ": (malloc) Error in realloc(): out of "
6493 : "memory\n", "", "");
6494 0 : abort();
6495 : }
6496 : #endif
6497 0 : errno = ENOMEM;
6498 : }
6499 : } else {
6500 4763724 : if (malloc_init())
6501 0 : ret = NULL;
6502 : else
6503 4763724 : ret = imalloc(size);
6504 :
6505 4763724 : if (ret == NULL) {
6506 : #ifdef MALLOC_XMALLOC
6507 0 : if (opt_xmalloc) {
6508 0 : _malloc_message(_getprogname(),
6509 : ": (malloc) Error in realloc(): out of "
6510 : "memory\n", "", "");
6511 0 : abort();
6512 : }
6513 : #endif
6514 0 : errno = ENOMEM;
6515 : }
6516 : }
6517 :
6518 : #ifdef MALLOC_SYSV
6519 : RETURN:
6520 : #endif
6521 7065387 : UTRACE(ptr, size, ret);
6522 7065387 : return (ret);
6523 : }
6524 :
6525 : void
6526 : free(void *ptr)
6527 : {
6528 : size_t offset;
6529 :
6530 : DARWIN_ONLY((szone->free)(szone, ptr); return);
6531 :
6532 50933286 : UTRACE(ptr, 0, 0);
6533 :
6534 : /*
6535 : * A version of idalloc that checks for NULL pointer but only for
6536 : * huge allocations assuming that CHUNK_ADDR2OFFSET(NULL) == 0.
6537 : */
6538 : assert(CHUNK_ADDR2OFFSET(NULL) == 0);
6539 50933289 : offset = CHUNK_ADDR2OFFSET(ptr);
6540 50933289 : if (offset != 0)
6541 42036375 : arena_dalloc(ptr, offset);
6542 8896914 : else if (ptr != NULL)
6543 2916 : huge_dalloc(ptr);
6544 50933300 : }
6545 :
6546 : /*
6547 : * End malloc(3)-compatible functions.
6548 : */
6549 : /******************************************************************************/
6550 : /*
6551 : * Begin non-standard functions.
6552 : */
6553 :
6554 : /* This was added by Mozilla for use by SQLite. */
6555 : #ifdef MOZ_MEMORY_DARWIN
6556 : static
6557 : #endif
6558 : size_t
6559 0 : je_malloc_good_size(size_t size)
6560 : {
6561 : /*
6562 : * This duplicates the logic in imalloc(), arena_malloc() and
6563 : * arena_malloc_small().
6564 : */
6565 0 : if (size < small_min) {
6566 : /* Small (tiny). */
6567 0 : size = pow2_ceil(size);
6568 : /*
6569 : * We omit the #ifdefs from arena_malloc_small() --
6570 : * it can be inaccurate with its size in some cases, but this
6571 : * function must be accurate.
6572 : */
6573 0 : if (size < (1U << TINY_MIN_2POW))
6574 0 : size = (1U << TINY_MIN_2POW);
6575 0 : } else if (size <= small_max) {
6576 : /* Small (quantum-spaced). */
6577 0 : size = QUANTUM_CEILING(size);
6578 0 : } else if (size <= bin_maxclass) {
6579 : /* Small (sub-page). */
6580 0 : size = pow2_ceil(size);
6581 0 : } else if (size <= arena_maxclass) {
6582 : /* Large. */
6583 0 : size = PAGE_CEILING(size);
6584 : } else {
6585 : /*
6586 : * Huge. We use PAGE_CEILING to get psize, instead of using
6587 : * CHUNK_CEILING to get csize. This ensures that this
6588 : * malloc_usable_size(malloc(n)) always matches
6589 : * je_malloc_good_size(n).
6590 : */
6591 0 : size = PAGE_CEILING(size);
6592 : }
6593 0 : return size;
6594 : }
6595 :
6596 :
6597 : #ifdef MOZ_MEMORY_ANDROID
6598 : size_t
6599 : malloc_usable_size(void *ptr)
6600 : #else
6601 : size_t
6602 0 : malloc_usable_size(const void *ptr)
6603 : #endif
6604 : {
6605 : DARWIN_ONLY(return (szone->size)(szone, ptr));
6606 :
6607 : #ifdef MALLOC_VALIDATE
6608 0 : return (isalloc_validate(ptr));
6609 : #else
6610 : assert(ptr != NULL);
6611 :
6612 : return (isalloc(ptr));
6613 : #endif
6614 : }
6615 :
6616 : void
6617 0 : jemalloc_stats(jemalloc_stats_t *stats)
6618 : {
6619 : size_t i;
6620 :
6621 0 : assert(stats != NULL);
6622 :
6623 : /*
6624 : * Gather runtime settings.
6625 : */
6626 0 : stats->opt_abort = opt_abort;
6627 0 : stats->opt_junk =
6628 : #ifdef MALLOC_FILL
6629 : opt_junk ? true :
6630 : #endif
6631 : false;
6632 0 : stats->opt_utrace =
6633 : #ifdef MALLOC_UTRACE
6634 : opt_utrace ? true :
6635 : #endif
6636 : false;
6637 0 : stats->opt_sysv =
6638 : #ifdef MALLOC_SYSV
6639 : opt_sysv ? true :
6640 : #endif
6641 : false;
6642 0 : stats->opt_xmalloc =
6643 : #ifdef MALLOC_XMALLOC
6644 : opt_xmalloc ? true :
6645 : #endif
6646 : false;
6647 0 : stats->opt_zero =
6648 : #ifdef MALLOC_FILL
6649 : opt_zero ? true :
6650 : #endif
6651 : false;
6652 0 : stats->narenas = narenas;
6653 0 : stats->balance_threshold =
6654 : #ifdef MALLOC_BALANCE
6655 : opt_balance_threshold
6656 : #else
6657 : SIZE_T_MAX
6658 : #endif
6659 : ;
6660 0 : stats->quantum = quantum;
6661 0 : stats->small_max = small_max;
6662 0 : stats->large_max = arena_maxclass;
6663 0 : stats->chunksize = chunksize;
6664 0 : stats->dirty_max = opt_dirty_max;
6665 :
6666 : /*
6667 : * Gather current memory usage statistics.
6668 : */
6669 0 : stats->mapped = 0;
6670 0 : stats->committed = 0;
6671 0 : stats->allocated = 0;
6672 0 : stats->dirty = 0;
6673 :
6674 : /* Get huge mapped/allocated. */
6675 0 : malloc_mutex_lock(&huge_mtx);
6676 0 : stats->mapped += stats_chunks.curchunks * chunksize;
6677 0 : stats->committed += huge_allocated;
6678 0 : stats->allocated += huge_allocated;
6679 0 : malloc_mutex_unlock(&huge_mtx);
6680 :
6681 : /* Get base mapped. */
6682 0 : malloc_mutex_lock(&base_mtx);
6683 0 : stats->mapped += base_mapped;
6684 0 : assert(base_committed <= base_mapped);
6685 0 : stats->committed += base_committed;
6686 0 : malloc_mutex_unlock(&base_mtx);
6687 :
6688 : /* Iterate over arenas and their chunks. */
6689 0 : for (i = 0; i < narenas; i++) {
6690 0 : arena_t *arena = arenas[i];
6691 0 : if (arena != NULL) {
6692 0 : malloc_spin_lock(&arena->lock);
6693 0 : stats->allocated += arena->stats.allocated_small;
6694 0 : stats->allocated += arena->stats.allocated_large;
6695 0 : stats->committed += (arena->stats.committed <<
6696 : pagesize_2pow);
6697 0 : stats->dirty += (arena->ndirty << pagesize_2pow);
6698 0 : malloc_spin_unlock(&arena->lock);
6699 : }
6700 : }
6701 :
6702 0 : assert(stats->mapped >= stats->committed);
6703 0 : assert(stats->committed >= stats->allocated);
6704 0 : }
6705 :
6706 : #ifdef MALLOC_DOUBLE_PURGE
6707 :
6708 : /* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */
6709 : static void
6710 : hard_purge_chunk(arena_chunk_t *chunk)
6711 : {
6712 : /* See similar logic in arena_purge(). */
6713 :
6714 : size_t i;
6715 : for (i = arena_chunk_header_npages; i < chunk_npages; i++) {
6716 : /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */
6717 : size_t npages;
6718 : for (npages = 0;
6719 : chunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages;
6720 : npages++) {
6721 : /* Turn off the chunk's MADV_FREED bit and turn on its
6722 : * DECOMMITTED bit. */
6723 : assert(!(chunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED));
6724 : chunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED;
6725 : }
6726 :
6727 : /* We could use mincore to find out which pages are actually
6728 : * present, but it's not clear that's better. */
6729 : if (npages > 0) {
6730 : pages_decommit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
6731 : pages_commit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
6732 : }
6733 : i += npages;
6734 : }
6735 : }
6736 :
6737 : /* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */
6738 : static void
6739 : hard_purge_arena(arena_t *arena)
6740 : {
6741 : malloc_spin_lock(&arena->lock);
6742 :
6743 : while (!LinkedList_IsEmpty(&arena->chunks_madvised)) {
6744 : LinkedList* next = arena->chunks_madvised.next;
6745 : arena_chunk_t *chunk =
6746 : LinkedList_Get(arena->chunks_madvised.next,
6747 : arena_chunk_t, chunks_madvised_elem);
6748 : hard_purge_chunk(chunk);
6749 : LinkedList_Remove(&chunk->chunks_madvised_elem);
6750 : }
6751 :
6752 : malloc_spin_unlock(&arena->lock);
6753 : }
6754 :
6755 : void
6756 : jemalloc_purge_freed_pages()
6757 : {
6758 : size_t i;
6759 : for (i = 0; i < narenas; i++) {
6760 : arena_t *arena = arenas[i];
6761 : if (arena != NULL)
6762 : hard_purge_arena(arena);
6763 : }
6764 : }
6765 :
6766 : #else /* !defined MALLOC_DOUBLE_PURGE */
6767 :
6768 : void
6769 0 : jemalloc_purge_freed_pages()
6770 : {
6771 : /* Do nothing. */
6772 0 : }
6773 :
6774 : #endif /* defined MALLOC_DOUBLE_PURGE */
6775 :
6776 :
6777 :
6778 : #ifdef MOZ_MEMORY_WINDOWS
6779 : void*
6780 : _recalloc(void *ptr, size_t count, size_t size)
6781 : {
6782 : size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
6783 : size_t newsize = count * size;
6784 :
6785 : /*
6786 : * In order for all trailing bytes to be zeroed, the caller needs to
6787 : * use calloc(), followed by recalloc(). However, the current calloc()
6788 : * implementation only zeros the bytes requested, so if recalloc() is
6789 : * to work 100% correctly, calloc() will need to change to zero
6790 : * trailing bytes.
6791 : */
6792 :
6793 : ptr = realloc(ptr, newsize);
6794 : if (ptr != NULL && oldsize < newsize) {
6795 : memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
6796 : oldsize);
6797 : }
6798 :
6799 : return ptr;
6800 : }
6801 :
6802 : /*
6803 : * This impl of _expand doesn't ever actually expand or shrink blocks: it
6804 : * simply replies that you may continue using a shrunk block.
6805 : */
6806 : void*
6807 : _expand(void *ptr, size_t newsize)
6808 : {
6809 : if (isalloc(ptr) >= newsize)
6810 : return ptr;
6811 :
6812 : return NULL;
6813 : }
6814 :
6815 : size_t
6816 : _msize(const void *ptr)
6817 : {
6818 :
6819 : return malloc_usable_size(ptr);
6820 : }
6821 : #endif
6822 :
6823 : /*
6824 : * End non-standard functions.
6825 : */
6826 : /******************************************************************************/
6827 : /*
6828 : * Begin library-private functions, used by threading libraries for protection
6829 : * of malloc during fork(). These functions are only called if the program is
6830 : * running in threaded mode, so there is no need to check whether the program
6831 : * is threaded here.
6832 : */
6833 :
6834 : static void
6835 0 : _malloc_prefork(void)
6836 : {
6837 : unsigned i;
6838 :
6839 : /* Acquire all mutexes in a safe order. */
6840 :
6841 0 : malloc_spin_lock(&arenas_lock);
6842 0 : for (i = 0; i < narenas; i++) {
6843 0 : if (arenas[i] != NULL)
6844 0 : malloc_spin_lock(&arenas[i]->lock);
6845 : }
6846 :
6847 0 : malloc_mutex_lock(&base_mtx);
6848 :
6849 0 : malloc_mutex_lock(&huge_mtx);
6850 0 : }
6851 :
6852 : static void
6853 0 : _malloc_postfork(void)
6854 : {
6855 : unsigned i;
6856 :
6857 : /* Release all mutexes, now that fork() has completed. */
6858 :
6859 0 : malloc_mutex_unlock(&huge_mtx);
6860 :
6861 0 : malloc_mutex_unlock(&base_mtx);
6862 :
6863 0 : for (i = 0; i < narenas; i++) {
6864 0 : if (arenas[i] != NULL)
6865 0 : malloc_spin_unlock(&arenas[i]->lock);
6866 : }
6867 0 : malloc_spin_unlock(&arenas_lock);
6868 0 : }
6869 :
6870 : /*
6871 : * End library-private functions.
6872 : */
6873 : /******************************************************************************/
6874 :
6875 : #ifdef HAVE_DLOPEN
6876 : # include <dlfcn.h>
6877 : #endif
6878 :
6879 : #ifdef MOZ_MEMORY_DARWIN
6880 :
6881 : static void *
6882 : zone_malloc(malloc_zone_t *zone, size_t size)
6883 : {
6884 :
6885 : return (malloc(size));
6886 : }
6887 :
6888 : static void *
6889 : zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
6890 : {
6891 :
6892 : return (calloc(num, size));
6893 : }
6894 :
6895 : static void *
6896 : zone_valloc(malloc_zone_t *zone, size_t size)
6897 : {
6898 : void *ret = NULL; /* Assignment avoids useless compiler warning. */
6899 :
6900 : posix_memalign(&ret, pagesize, size);
6901 :
6902 : return (ret);
6903 : }
6904 :
6905 : static void *
6906 : zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
6907 : {
6908 : return (memalign(alignment, size));
6909 : }
6910 :
6911 : static void *
6912 : zone_destroy(malloc_zone_t *zone)
6913 : {
6914 :
6915 : /* This function should never be called. */
6916 : assert(false);
6917 : return (NULL);
6918 : }
6919 :
6920 : static size_t
6921 : zone_good_size(malloc_zone_t *zone, size_t size)
6922 : {
6923 : return je_malloc_good_size(size);
6924 : }
6925 :
6926 : static size_t
6927 : ozone_size(malloc_zone_t *zone, void *ptr)
6928 : {
6929 : size_t ret = isalloc_validate(ptr);
6930 : if (ret == 0)
6931 : ret = szone->size(zone, ptr);
6932 :
6933 : return ret;
6934 : }
6935 :
6936 : static void
6937 : ozone_free(malloc_zone_t *zone, void *ptr)
6938 : {
6939 : if (isalloc_validate(ptr) != 0)
6940 : free(ptr);
6941 : else {
6942 : size_t size = szone->size(zone, ptr);
6943 : if (size != 0)
6944 : (szone->free)(zone, ptr);
6945 : /* Otherwise we leak. */
6946 : }
6947 : }
6948 :
6949 : static void *
6950 : ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
6951 : {
6952 : size_t oldsize;
6953 : if (ptr == NULL)
6954 : return (malloc(size));
6955 :
6956 : oldsize = isalloc_validate(ptr);
6957 : if (oldsize != 0)
6958 : return (realloc(ptr, size));
6959 : else {
6960 : oldsize = szone->size(zone, ptr);
6961 : if (oldsize == 0)
6962 : return (malloc(size));
6963 : else {
6964 : void *ret = malloc(size);
6965 : if (ret != NULL) {
6966 : memcpy(ret, ptr, (oldsize < size) ? oldsize :
6967 : size);
6968 : (szone->free)(zone, ptr);
6969 : }
6970 : return (ret);
6971 : }
6972 : }
6973 : }
6974 :
6975 : static unsigned
6976 : ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
6977 : unsigned num_requested)
6978 : {
6979 : /* Don't bother implementing this interface, since it isn't required. */
6980 : return 0;
6981 : }
6982 :
6983 : static void
6984 : ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
6985 : {
6986 : unsigned i;
6987 :
6988 : for (i = 0; i < num; i++)
6989 : ozone_free(zone, to_be_freed[i]);
6990 : }
6991 :
6992 : static void
6993 : ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
6994 : {
6995 : if (isalloc_validate(ptr) != 0) {
6996 : assert(isalloc_validate(ptr) == size);
6997 : free(ptr);
6998 : } else {
6999 : assert(size == szone->size(zone, ptr));
7000 : l_szone.m16(zone, ptr, size);
7001 : }
7002 : }
7003 :
7004 : static void
7005 : ozone_force_lock(malloc_zone_t *zone)
7006 : {
7007 : /* jemalloc locking is taken care of by the normal jemalloc zone. */
7008 : szone->introspect->force_lock(zone);
7009 : }
7010 :
7011 : static void
7012 : ozone_force_unlock(malloc_zone_t *zone)
7013 : {
7014 : /* jemalloc locking is taken care of by the normal jemalloc zone. */
7015 : szone->introspect->force_unlock(zone);
7016 : }
7017 :
7018 : static size_t
7019 : zone_version_size(int version)
7020 : {
7021 : switch (version)
7022 : {
7023 : case SNOW_LEOPARD_MALLOC_ZONE_T_VERSION:
7024 : return sizeof(snow_leopard_malloc_zone);
7025 : case LEOPARD_MALLOC_ZONE_T_VERSION:
7026 : return sizeof(leopard_malloc_zone);
7027 : default:
7028 : case LION_MALLOC_ZONE_T_VERSION:
7029 : return sizeof(lion_malloc_zone);
7030 : }
7031 : }
7032 :
7033 : /*
7034 : * Overlay the default scalable zone (szone) such that existing allocations are
7035 : * drained, and further allocations come from jemalloc. This is necessary
7036 : * because Core Foundation directly accesses and uses the szone before the
7037 : * jemalloc library is even loaded.
7038 : */
7039 : static void
7040 : szone2ozone(malloc_zone_t *default_zone, size_t size)
7041 : {
7042 : lion_malloc_zone *l_zone;
7043 : assert(malloc_initialized);
7044 :
7045 : /*
7046 : * Stash a copy of the original szone so that we can call its
7047 : * functions as needed. Note that internally, the szone stores its
7048 : * bookkeeping data structures immediately following the malloc_zone_t
7049 : * header, so when calling szone functions, we need to pass a pointer to
7050 : * the original zone structure.
7051 : */
7052 : memcpy(szone, default_zone, size);
7053 :
7054 : /* OSX 10.7 allocates the default zone in protected memory. */
7055 : if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
7056 : void* start_of_page = (void*)((size_t)(default_zone) & ~pagesize_mask);
7057 : mprotect (start_of_page, size, PROT_READ | PROT_WRITE);
7058 : }
7059 :
7060 : default_zone->size = (void *)ozone_size;
7061 : default_zone->malloc = (void *)zone_malloc;
7062 : default_zone->calloc = (void *)zone_calloc;
7063 : default_zone->valloc = (void *)zone_valloc;
7064 : default_zone->free = (void *)ozone_free;
7065 : default_zone->realloc = (void *)ozone_realloc;
7066 : default_zone->destroy = (void *)zone_destroy;
7067 : default_zone->batch_malloc = NULL;
7068 : default_zone->batch_free = ozone_batch_free;
7069 : default_zone->introspect = ozone_introspect;
7070 :
7071 : /* Don't modify default_zone->zone_name; Mac libc may rely on the name
7072 : * being unchanged. See Mozilla bug 694896. */
7073 :
7074 : ozone_introspect->enumerator = NULL;
7075 : ozone_introspect->good_size = (void *)zone_good_size;
7076 : ozone_introspect->check = NULL;
7077 : ozone_introspect->print = NULL;
7078 : ozone_introspect->log = NULL;
7079 : ozone_introspect->force_lock = (void *)ozone_force_lock;
7080 : ozone_introspect->force_unlock = (void *)ozone_force_unlock;
7081 : ozone_introspect->statistics = NULL;
7082 :
7083 : /* Platform-dependent structs */
7084 : l_zone = (lion_malloc_zone*)(default_zone);
7085 :
7086 : if (default_zone->version >= SNOW_LEOPARD_MALLOC_ZONE_T_VERSION) {
7087 : l_zone->m15 = (void (*)())zone_memalign;
7088 : l_zone->m16 = (void (*)())ozone_free_definite_size;
7089 : l_ozone_introspect.m9 = NULL;
7090 : }
7091 :
7092 : if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
7093 : l_zone->m17 = NULL;
7094 : l_ozone_introspect.m10 = NULL;
7095 : l_ozone_introspect.m11 = NULL;
7096 : l_ozone_introspect.m12 = NULL;
7097 : l_ozone_introspect.m13 = NULL;
7098 : }
7099 : }
7100 :
7101 : __attribute__((constructor))
7102 : void
7103 : jemalloc_darwin_init(void)
7104 : {
7105 : if (malloc_init_hard())
7106 : abort();
7107 : }
7108 :
7109 : #elif defined(__GLIBC__) && !defined(__UCLIBC__)
7110 : /*
7111 : * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
7112 : * to inconsistently reference libc's malloc(3)-compatible functions
7113 : * (bug 493541).
7114 : *
7115 : * These definitions interpose hooks in glibc. The functions are actually
7116 : * passed an extra argument for the caller return address, which will be
7117 : * ignored.
7118 : */
7119 : void (*__free_hook)(void *ptr) = free;
7120 : void *(*__malloc_hook)(size_t size) = malloc;
7121 : void *(*__realloc_hook)(void *ptr, size_t size) = realloc;
7122 : void *(*__memalign_hook)(size_t alignment, size_t size) = MEMALIGN;
7123 :
7124 : #elif defined(RTLD_DEEPBIND)
7125 : /*
7126 : * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
7127 : * implementations permit similar inconsistencies? Should STV_SINGLETON
7128 : * visibility be used for interposition where available?
7129 : */
7130 : # error "Interposing malloc is unsafe on this system without libc malloc hooks."
7131 : #endif
7132 :
7133 : #ifdef MOZ_MEMORY_WINDOWS
7134 : /*
7135 : * In the new style jemalloc integration jemalloc is built as a separate
7136 : * shared library. Since we're no longer hooking into the CRT binary,
7137 : * we need to initialize the heap at the first opportunity we get.
7138 : * DLL_PROCESS_ATTACH in DllMain is that opportunity.
7139 : */
7140 : BOOL APIENTRY DllMain(HINSTANCE hModule,
7141 : DWORD reason,
7142 : LPVOID lpReserved)
7143 : {
7144 : switch (reason) {
7145 : case DLL_PROCESS_ATTACH:
7146 : /* Don't force the system to page DllMain back in every time
7147 : * we create/destroy a thread */
7148 : DisableThreadLibraryCalls(hModule);
7149 : /* Initialize the heap */
7150 : malloc_init_hard();
7151 : break;
7152 :
7153 : case DLL_PROCESS_DETACH:
7154 : break;
7155 :
7156 : }
7157 :
7158 : return TRUE;
7159 : }
7160 :
7161 : /*
7162 : * There's a fun allocator mismatch in (at least) the VS 2010 CRT
7163 : * (see the giant comment in this directory's Makefile.in
7164 : * that gets redirected here to avoid a crash on shutdown.
7165 : */
7166 : void
7167 : je_dumb_free_thunk(void *ptr)
7168 : {
7169 : return; /* shutdown leaks that we don't care about */
7170 : }
7171 :
7172 : #include <wchar.h>
7173 :
7174 : /*
7175 : * We also need to provide our own impl of wcsdup so that we don't ask
7176 : * the CRT for memory from its heap (which will then be unfreeable).
7177 : */
7178 : wchar_t *je_wcsdup(const wchar_t *src)
7179 : {
7180 : size_t len = wcslen(src);
7181 : wchar_t* dst = (wchar_t*)je_malloc((len + 1) * sizeof(wchar_t));
7182 : if(dst)
7183 : wcsncpy(dst, src, len + 1);
7184 : return dst;
7185 : }
7186 :
7187 : #endif
|