diff --git a/libc/calls/clock_nanosleep-openbsd.c b/libc/calls/clock_nanosleep-openbsd.c index 25718feb2cf..dec28531419 100644 --- a/libc/calls/clock_nanosleep-openbsd.c +++ b/libc/calls/clock_nanosleep-openbsd.c @@ -23,9 +23,9 @@ #include "libc/sysv/consts/clock.h" #include "libc/sysv/errfuns.h" -int sys_clock_nanosleep_openbsd(int clock, int flags, - const struct timespec *req, - struct timespec *rem) { +relegated int sys_clock_nanosleep_openbsd(int clock, int flags, + const struct timespec *req, + struct timespec *rem) { int res; struct timespec start, relative, remainder; if (!flags) { diff --git a/libc/calls/time.c b/libc/calls/time.c index d592bc256cb..f0455d2b577 100644 --- a/libc/calls/time.c +++ b/libc/calls/time.c @@ -16,10 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/time.h" -#include "libc/calls/struct/timeval.h" -#include "libc/dce.h" -#include "libc/sysv/errfuns.h" +#include "libc/calls/calls.h" +#include "libc/calls/struct/timespec.h" +#include "libc/sysv/consts/clock.h" /** * Returns time as seconds from UNIX epoch. @@ -29,15 +28,11 @@ * @asyncsignalsafe */ int64_t time(int64_t *opt_out_ret) { - int64_t secs; - struct timeval tv; - if (gettimeofday(&tv, 0) != -1) { - secs = tv.tv_sec; - if (opt_out_ret) { - *opt_out_ret = secs; - } - } else { - secs = -1; - } + int64_t secs = -1; + struct timespec ts; + if (!clock_gettime(CLOCK_REALTIME, &ts)) + secs = ts.tv_sec; + if (opt_out_ret) + *opt_out_ret = secs; return secs; } diff --git a/libc/cosmo.h b/libc/cosmo.h index d027d6dfc7b..e2691587a57 100644 --- a/libc/cosmo.h +++ b/libc/cosmo.h @@ -13,15 +13,24 @@ errno_t cosmo_once(_COSMO_ATOMIC(unsigned) *, void (*)(void)) libcesque; int systemvpe(const char *, char *const[], char *const[]) libcesque; char *GetProgramExecutableName(void) libcesque; void unleaf(void) libcesque; +bool32 IsLinuxModern(void) libcesque; + int __demangle(char *, const char *, size_t) libcesque; int __is_mangled(const char *) libcesque; -bool32 IsLinuxModern(void) libcesque; -int LoadZipArgs(int *, char ***) libcesque; + int cosmo_args(const char *, char ***) libcesque; +int LoadZipArgs(int *, char ***) libcesque; + int cosmo_futex_wake(_COSMO_ATOMIC(int) *, int, char); int cosmo_futex_wait(_COSMO_ATOMIC(int) *, int, char, int, const struct timespec *); +errno_t cosmo_stack_alloc(size_t *, size_t *, void **) libcesque; +errno_t cosmo_stack_free(void *, size_t, size_t) libcesque; +void cosmo_stack_clear(void) libcesque; +void cosmo_stack_setmaxstacks(int) libcesque; +int cosmo_stack_getmaxstacks(void) libcesque; + int __deadlock_check(void *, int) libcesque; int __deadlock_tracked(void *) libcesque; void __deadlock_record(void *, int) libcesque; diff --git a/libc/intrin/count.c b/libc/intrin/count.c new file mode 100644 index 00000000000..d4f4365bbbb --- /dev/null +++ b/libc/intrin/count.c @@ -0,0 +1,26 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" +#include "libc/stdalign.h" +#include "libc/thread/thread.h" + +// this counter is important because pthread_exit() needs to know if +// it's an orphan thread, without needing to acquire _pthread_lock() +// which causes contention and a file descriptor explosion on netbsd +alignas(64) atomic_uint _pthread_count = 1; diff --git a/libc/intrin/itimer.c b/libc/intrin/itimer.c new file mode 100644 index 00000000000..595fc0a0080 --- /dev/null +++ b/libc/intrin/itimer.c @@ -0,0 +1,42 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/thread/itimer.h" +#include "libc/str/str.h" + +struct IntervalTimer __itimer = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, +}; + +textwindows void __itimer_lock(void) { + pthread_mutex_lock(&__itimer.lock); +} + +textwindows void __itimer_unlock(void) { + pthread_mutex_unlock(&__itimer.lock); +} + +textwindows void __itimer_wipe_and_reset(void) { + // timers aren't inherited by forked subprocesses + bzero(&__itimer.it, sizeof(__itimer.it)); + pthread_mutex_wipe_np(&__itimer.lock); + pthread_cond_init(&__itimer.cond, 0); + __itimer.thread = 0; + __itimer.once = 0; +} diff --git a/libc/intrin/kisdangerous.c b/libc/intrin/kisdangerous.c new file mode 100644 index 00000000000..62872425e46 --- /dev/null +++ b/libc/intrin/kisdangerous.c @@ -0,0 +1,36 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/kprintf.h" +#include "libc/intrin/maps.h" + +privileged optimizesize bool32 kisdangerous(const void *addr) { + bool32 res = true; + __maps_lock(); + if (__maps.maps) { + struct Map *map; + if ((map = __maps_floor(addr))) + if ((const char *)addr >= map->addr && + (const char *)addr < map->addr + map->size) + res = false; + } else { + res = false; + } + __maps_unlock(); + return res; +} diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index e8444fde020..283aa71ddb5 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -160,22 +160,6 @@ __funline bool kischarmisaligned(const char *p, signed char t) { return false; } -ABI bool32 kisdangerous(const void *addr) { - bool32 res = true; - __maps_lock(); - if (__maps.maps) { - struct Map *map; - if ((map = __maps_floor(addr))) - if ((const char *)addr >= map->addr && - (const char *)addr < map->addr + map->size) - res = false; - } else { - res = false; - } - __maps_unlock(); - return res; -} - ABI static void klogclose(long fd) { #ifdef __x86_64__ long ax = __NR_close; diff --git a/libc/thread/pthread_cond_init.c b/libc/intrin/pthread_cond_init.c similarity index 100% rename from libc/thread/pthread_cond_init.c rename to libc/intrin/pthread_cond_init.c diff --git a/libc/intrin/pthread_orphan_np.c b/libc/intrin/pthread_orphan_np.c index 68e2a9f5fb4..1575502f157 100644 --- a/libc/intrin/pthread_orphan_np.c +++ b/libc/intrin/pthread_orphan_np.c @@ -16,6 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/intrin/atomic.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" @@ -28,5 +30,6 @@ int pthread_orphan_np(void) { res = _pthread_list == _pthread_list->prev && _pthread_list == _pthread_list->next; _pthread_unlock(); + unassert(!res || atomic_load(&_pthread_count) <= 1); return res; } diff --git a/libc/intrin/pthreadlock.c b/libc/intrin/pthreadlock.c index 92f784548c0..7db5827603f 100644 --- a/libc/intrin/pthreadlock.c +++ b/libc/intrin/pthreadlock.c @@ -16,9 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/stdalign.h" #include "libc/thread/posixthread.internal.h" -pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER; +alignas(64) pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER; void _pthread_lock(void) { pthread_mutex_lock(&__pthread_lock_obj); diff --git a/libc/intrin/sig.c b/libc/intrin/sig.c index aecd085c9ac..56866464f7d 100644 --- a/libc/intrin/sig.c +++ b/libc/intrin/sig.c @@ -696,35 +696,40 @@ textwindows dontinstrument static uint32_t __sig_worker(void *arg) { } // unblock stalled asynchronous signals in threads - _pthread_lock(); - for (struct Dll *e = dll_first(_pthread_list); e; - e = dll_next(_pthread_list, e)) { - struct PosixThread *pt = POSIXTHREAD_CONTAINER(e); - if (atomic_load_explicit(&pt->pt_status, memory_order_acquire) >= - kPosixThreadTerminated) { + struct PosixThread *mark; + for (;;) { + sigset_t pending, mask; + mark = 0; + _pthread_lock(); + for (struct Dll *e = dll_first(_pthread_list); e; + e = dll_next(_pthread_list, e)) { + struct PosixThread *pt = POSIXTHREAD_CONTAINER(e); + if (atomic_load_explicit(&pt->pt_status, memory_order_acquire) >= + kPosixThreadTerminated) + break; + pending = atomic_load_explicit(&pt->tib->tib_sigpending, + memory_order_acquire); + mask = + atomic_load_explicit(&pt->tib->tib_sigmask, memory_order_acquire); + if (pending & ~mask) { + _pthread_ref(pt); + mark = pt; + break; + } + } + _pthread_unlock(); + if (!mark) break; + while (!atomic_compare_exchange_weak_explicit( + &mark->tib->tib_sigpending, &pending, pending & ~mask, + memory_order_acq_rel, memory_order_relaxed)) { } - sigset_t pending = - atomic_load_explicit(&pt->tib->tib_sigpending, memory_order_acquire); - sigset_t mask = - atomic_load_explicit(&pt->tib->tib_sigmask, memory_order_acquire); - if (pending & ~mask) { - _pthread_ref(pt); - _pthread_unlock(); - while (!atomic_compare_exchange_weak_explicit( - &pt->tib->tib_sigpending, &pending, pending & ~mask, - memory_order_acq_rel, memory_order_relaxed)) { - } - while ((pending = pending & ~mask)) { - int sig = bsfl(pending) + 1; - pending &= ~(1ull << (sig - 1)); - __sig_killer(pt, sig, SI_KERNEL); - } - _pthread_lock(); - _pthread_unref(pt); + while ((pending = pending & ~mask)) { + int sig = bsfl(pending) + 1; + pending &= ~(1ull << (sig - 1)); + __sig_killer(mark, sig, SI_KERNEL); } } - _pthread_unlock(); // wait until next scheduler quantum pthread_mutex_unlock(&__sig_worker_lock); diff --git a/libc/intrin/stack.c b/libc/intrin/stack.c new file mode 100644 index 00000000000..e153b6ce884 --- /dev/null +++ b/libc/intrin/stack.c @@ -0,0 +1,354 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/stack.h" +#include "libc/assert.h" +#include "libc/atomic.h" +#include "libc/calls/calls.h" +#include "libc/calls/syscall-sysv.internal.h" +#include "libc/cosmo.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/dll.h" +#include "libc/runtime/runtime.h" +#include "libc/sysv/consts/map.h" +#include "libc/sysv/consts/prot.h" +#include "libc/thread/thread.h" + +/** + * @fileoverview cosmo stack memory manager + */ + +#define MAP_ANON_OPENBSD 0x1000 +#define MAP_STACK_OPENBSD 0x4000 + +#define THREADSTACK_CONTAINER(e) DLL_CONTAINER(struct CosmoStack, elem, e) + +struct CosmoStack { + struct Dll elem; + void *stackaddr; + size_t stacksize; + size_t guardsize; +}; + +struct CosmoStacks { + atomic_uint once; + pthread_mutex_t lock; + struct Dll *stacks; + struct Dll *objects; + unsigned count; +}; + +struct CosmoStacksConfig { + unsigned maxstacks; +}; + +static struct CosmoStacks cosmo_stacks = { + .lock = PTHREAD_MUTEX_INITIALIZER, +}; + +static struct CosmoStacksConfig cosmo_stacks_config = { + .maxstacks = 16, +}; + +void cosmo_stack_lock(void) { + pthread_mutex_lock(&cosmo_stacks.lock); +} + +void cosmo_stack_unlock(void) { + pthread_mutex_unlock(&cosmo_stacks.lock); +} + +void cosmo_stack_wipe(void) { + pthread_mutex_wipe_np(&cosmo_stacks.lock); +} + +static errno_t cosmo_stack_munmap(void *addr, size_t size) { + errno_t r = 0; + errno_t e = errno; + if (!munmap(addr, size)) { + r = errno; + errno = e; + } + return r; +} + +static void cosmo_stack_populate(void) { + errno_t e = errno; + void *map = mmap(0, __pagesize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + errno = e; + if (map != MAP_FAILED) { + struct CosmoStack *ts = map; + int n = __pagesize / sizeof(struct CosmoStack); + for (int i = 0; i < n; ++i) { + dll_init(&ts[i].elem); + dll_make_first(&cosmo_stacks.objects, &ts[i].elem); + } + } +} + +static struct Dll *cosmo_stack_decimate(unsigned maxstacks) { + struct Dll *surplus = 0; + while (cosmo_stacks.count > maxstacks) { + struct Dll *e = dll_last(cosmo_stacks.stacks); + dll_remove(&cosmo_stacks.stacks, e); + dll_make_first(&surplus, e); + --cosmo_stacks.count; + } + return surplus; +} + +static void cosmo_stack_rehabilitate(struct Dll *stacks) { + struct Dll *e; + for (e = dll_first(stacks); e; e = dll_next(stacks, e)) + cosmo_stack_munmap(THREADSTACK_CONTAINER(e)->stackaddr, + THREADSTACK_CONTAINER(e)->stacksize); + cosmo_stack_lock(); + dll_make_first(&cosmo_stacks.objects, stacks); + cosmo_stack_unlock(); +} + +/** + * Empties unused stack cache. + * + * To make POSIX threads as cheap as possible to spawn, we recycle their + * stacks without zeroing their memory. On Linux for an 80kb stack size, + * that makes launching a thread take 40µs rather than 80µs. However the + * stack cache needs to be cleared in certain cases. This is called upon + * exit() automatically but anyone can clear this at any other time too. + * + * @see pthread_decimate_np() + */ +void cosmo_stack_clear(void) { + cosmo_stack_lock(); + struct Dll *stacks = cosmo_stacks.stacks; + cosmo_stacks.stacks = 0; + cosmo_stacks.count = 0; + cosmo_stack_unlock(); + cosmo_stack_rehabilitate(stacks); +} + +/** + * Gets maximum number of unused stacks cosmo should cache. + * @see cosmo_stack_setmaxstacks() + */ +int cosmo_stack_getmaxstacks(void) { + return cosmo_stacks_config.maxstacks; +} + +/** + * Sets maximum number of unused stacks cosmo should cache. + * + * This lets you place some limitations on how much stack memory the + * cosmo runtime will cache. This number is a count of stacks rather + * than the number of bytes they contain. Old stacks are freed in a + * least recently used fashion once the cache exceeds this limit. + * + * If this is set to zero, then the cosmo stack allocator enters a + * highly secure hardening mode where cosmo_stack_alloc() zeroes all + * stack memory that's returned. + * + * Setting this to a negative number makes the cache size unlimited. + * + * Please note this limit only applies to stacks that aren't in use. + * + * Your default is sixteen stacks may be cached at any given moment. + * + * If `maxstacks` is less than the current cache size, then surplus + * entries will be evicted and freed before this function returns. + */ +void cosmo_stack_setmaxstacks(int maxstacks) { + cosmo_stack_lock(); + cosmo_stacks_config.maxstacks = maxstacks; + struct Dll *stacks = cosmo_stack_decimate(maxstacks); + cosmo_stack_unlock(); + cosmo_stack_rehabilitate(stacks); +} + +/** + * Allocates stack memory. + * + * This is a caching stack allocator that's used by the POSIX threads + * runtime but you may also find it useful for setcontext() coroutines + * or sigaltstack(). Normally you can get away with using malloc() for + * creating stacks. However some OSes (e.g. OpenBSD) forbid you from + * doing that for anything except sigaltstack(). This API serves to + * abstract all the gory details of gaining authorized memory, and + * additionally implements caching for lightning fast performance. + * + * The stack size must be nonzero. It is rounded up to the granularity + * of the underlying system allocator, which is normally the page size. + * Your parameter will be updated with the selected value upon success. + * + * The guard size specifies how much memory should be protected at the + * bottom of your stack. This is helpful for ensuring stack overflows + * will result in a segmentation fault, rather than corrupting memory + * silently. This may be set to zero, in which case no guard pages will + * be protected. This value is rounded up to the system page size. The + * corrected value will be returned upon success. Your guard size needs + * to be small enough to leave room for at least one memory page in your + * stack size i.e. `guardsize + pagesize <= stacksize` must be the case. + * Otherwise this function will return an `EINVAL` error. + * + * When you're done using your stack, pass it to cosmo_stack_free() so + * it can be recycled. Stacks are only recycled when the `stacksize` and + * `guardsize` parameters are an exact match after correction. Otherwise + * they'll likely be freed eventually, in a least-recently used fashion, + * based upon the configurable cosmo_stack_setmaxstacks() setting. + * + * This function returns 0 on success, or an errno on error. See the + * documentation of mmap() for a list possible errors that may occur. + */ +errno_t cosmo_stack_alloc(size_t *inout_stacksize, // + size_t *inout_guardsize, // + void **out_addr) { + + // validate arguments + size_t stacksize = *inout_stacksize; + size_t guardsize = *inout_guardsize; + stacksize = (stacksize + __gransize - 1) & -__gransize; + guardsize = (guardsize + __pagesize - 1) & -__pagesize; + if (guardsize + __pagesize > stacksize) + return EINVAL; + + // recycle stack + void *stackaddr = 0; + cosmo_stack_lock(); + for (struct Dll *e = dll_first(cosmo_stacks.stacks); e; + e = dll_next(cosmo_stacks.stacks, e)) { + struct CosmoStack *ts = THREADSTACK_CONTAINER(e); + if (ts->stacksize == stacksize && // + ts->guardsize == guardsize) { + dll_remove(&cosmo_stacks.stacks, e); + stackaddr = ts->stackaddr; + dll_make_first(&cosmo_stacks.objects, e); + --cosmo_stacks.count; + break; + } + } + cosmo_stack_unlock(); + + // create stack + if (!stackaddr) { + errno_t e = errno; + stackaddr = mmap(0, stacksize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (stackaddr == MAP_FAILED) { + errno_t err = errno; + errno = e; + return err; + } + if (IsOpenbsd()) + if (!TellOpenbsdThisIsStackMemory(stackaddr, stacksize)) + notpossible; + if (guardsize) + if (mprotect(stackaddr, guardsize, PROT_NONE | PROT_GUARD)) + notpossible; + } + + // return stack + *inout_stacksize = stacksize; + *inout_guardsize = guardsize; + *out_addr = stackaddr; + return 0; +} + +static void cosmo_stack_setup(void) { + atexit(cosmo_stack_clear); +} + +/** + * Frees stack memory. + * + * While not strictly required, it's assumed these three values would be + * those returned by an earlier call to cosmo_stack_alloc(). + * + * This function returns 0 on success, or an errno on error. The `errno` + * variable is never clobbered. You can only dependably count on this to + * return an error on failure when you say `cosmo_stack_setmaxstacks(0)` + */ +errno_t cosmo_stack_free(void *stackaddr, size_t stacksize, size_t guardsize) { + stacksize = (stacksize + __gransize - 1) & -__gransize; + guardsize = (guardsize + __pagesize - 1) & -__pagesize; + if (guardsize + __pagesize > stacksize) + return EINVAL; + if ((uintptr_t)stackaddr & (__gransize - 1)) + return EINVAL; + cosmo_once(&cosmo_stacks.once, cosmo_stack_setup); + cosmo_stack_lock(); + struct Dll *surplus = 0; + if (cosmo_stacks_config.maxstacks) { + surplus = cosmo_stack_decimate(cosmo_stacks_config.maxstacks - 1); + struct CosmoStack *ts = 0; + if (dll_is_empty(cosmo_stacks.objects)) + cosmo_stack_populate(); + struct Dll *e; + if ((e = dll_first(cosmo_stacks.objects))) { + dll_remove(&cosmo_stacks.objects, e); + ts = THREADSTACK_CONTAINER(e); + } + if (ts) { + ts->stackaddr = stackaddr; + ts->stacksize = stacksize; + ts->guardsize = guardsize; + dll_make_first(&cosmo_stacks.stacks, &ts->elem); + ++cosmo_stacks.count; + stackaddr = 0; + } + } + cosmo_stack_unlock(); + cosmo_stack_rehabilitate(surplus); + errno_t err = 0; + if (stackaddr) + err = cosmo_stack_munmap(stackaddr, stacksize); + return err; +} + +relegated bool TellOpenbsdThisIsStackMemory(void *addr, size_t size) { + return __sys_mmap( + addr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1, + 0, 0) == addr; +} + +// OpenBSD only permits RSP to occupy memory that's been explicitly +// defined as stack memory, i.e. `lo <= %rsp < hi` must be the case +relegated errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *attr) { + + // get interval + uintptr_t lo = (uintptr_t)attr->__stackaddr; + uintptr_t hi = lo + attr->__stacksize; + + // squeeze interval + lo = (lo + __pagesize - 1) & -__pagesize; + hi = hi & -__pagesize; + + // tell os it's stack memory + errno_t olderr = errno; + if (!TellOpenbsdThisIsStackMemory((void *)lo, hi - lo)) { + errno_t err = errno; + errno = olderr; + return err; + } + + // update attributes with usable stack address + attr->__stackaddr = (void *)lo; + attr->__stacksize = hi - lo; + return 0; +} diff --git a/libc/intrin/stack.h b/libc/intrin/stack.h new file mode 100644 index 00000000000..003b67cf4a0 --- /dev/null +++ b/libc/intrin/stack.h @@ -0,0 +1,14 @@ +#ifndef COSMOPOLITAN_LIBC_STACK_H_ +#define COSMOPOLITAN_LIBC_STACK_H_ +#include "libc/thread/thread.h" +COSMOPOLITAN_C_START_ + +void cosmo_stack_lock(void); +void cosmo_stack_unlock(void); +void cosmo_stack_wipe(void); + +bool TellOpenbsdThisIsStackMemory(void *, size_t); +errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *); + +COSMOPOLITAN_C_END_ +#endif /* COSMOPOLITAN_LIBC_STACK_H_ */ diff --git a/libc/intrin/ulock.c b/libc/intrin/ulock.c index 906f96eccfd..40a86349003 100644 --- a/libc/intrin/ulock.c +++ b/libc/intrin/ulock.c @@ -17,12 +17,12 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/ulock.h" -#include "libc/assert.h" -#include "libc/calls/calls.h" #include "libc/calls/syscall_support-sysv.internal.h" -#include "libc/dce.h" +#include "libc/errno.h" #include "libc/intrin/describeflags.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/strace.h" +#include "libc/intrin/ulock.h" // XNU futexes // https://opensource.apple.com/source/xnu/xnu-7195.50.7.100.1/bsd/sys/ulock.h.auto.html @@ -32,6 +32,26 @@ int sys_ulock_wait(uint32_t operation, void *addr, uint64_t value, uint32_t timeout_micros) asm("sys_futex_cp"); // returns number of other waiters, or -1 w/ errno +// +// - EINTR means a signal handler was called. This is how we support +// things like POSIX thread cancelation. +// +// - EFAULT if XNU couldn't read `addr`. This is normally considered a +// programming error, but with ulock it can actually be a transient +// error due to low memory conditions. Apple recommends retrying. +// +// - ENOMEM means XNU wasn't able to allocate memory for kernel internal +// data structures. Apple doesn't provide any advice on what to do. We +// simply turn this into EAGAIN. +// +// - EAGAIN if XNU told us EFAULT but cosmo believes the address exists. +// This value is also used as a substitute for ENOMEM. +// +// - EINVAL could mean operation is invalid, addr is null or misaligned; +// it could also mean another thread calling ulock on this address was +// configured (via operation) in an inconsistent way. +// +// see also os_sync_wait_on_address.h from xcode sdk int ulock_wait(uint32_t operation, void *addr, uint64_t value, uint32_t timeout_micros) { int rc; @@ -39,12 +59,26 @@ int ulock_wait(uint32_t operation, void *addr, uint64_t value, LOCKTRACE("ulock_wait(%#x, %p, %lx, %u) → ...", operation, addr, value, timeout_micros); rc = sys_ulock_wait(operation, addr, value, timeout_micros); + if (rc == -1) { + if (errno == ENOMEM) + errno = EAGAIN; + if (errno == EFAULT) + if (!kisdangerous(addr)) + errno = EAGAIN; + } LOCKTRACE("ulock_wait(%#x, %p, %lx, %u) → %d% m", operation, addr, value, timeout_micros, rc); return rc; } // returns -errno +// +// - ENOENT means there wasn't anyone to wake +// +// - EINVAL could mean operation is invalid, addr is null or misaligned; +// it could also mean another thread calling ulock on this address was +// configured (via operation) in an inconsistent way. +// int ulock_wake(uint32_t operation, void *addr, uint64_t wake_value) { int rc; rc = __syscall3i(operation, (long)addr, wake_value, 0x2000000 | 516); diff --git a/libc/mem/leaks.c b/libc/mem/leaks.c index ba0da6edc20..c23ff989aab 100644 --- a/libc/mem/leaks.c +++ b/libc/mem/leaks.c @@ -79,7 +79,7 @@ void CheckForMemoryLeaks(void) { // validate usage of this api if (_weaken(_pthread_decimate)) - _weaken(_pthread_decimate)(false); + _weaken(_pthread_decimate)(); if (!pthread_orphan_np()) kprintf("warning: called CheckForMemoryLeaks() from non-orphaned thread\n"); diff --git a/libc/proc/fork-nt.c b/libc/proc/fork-nt.c index 20ca74f7ea4..f35500c78f1 100644 --- a/libc/proc/fork-nt.c +++ b/libc/proc/fork-nt.c @@ -62,7 +62,7 @@ #include "libc/sysv/consts/prot.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" -#include "libc/thread/itimer.internal.h" +#include "libc/thread/itimer.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/tls.h" #ifdef __x86_64__ @@ -189,7 +189,7 @@ static textwindows void *Malloc(size_t size) { } textwindows void WinMainForked(void) { - jmp_buf jb; + intptr_t jb[5]; int64_t reader; int64_t savetsc; uint32_t varlen; @@ -305,14 +305,14 @@ textwindows void WinMainForked(void) { #endif // jump back into function below - longjmp(jb, 1); + __builtin_longjmp(jb, 1); } textwindows int sys_fork_nt(uint32_t dwCreationFlags) { char ok; - jmp_buf jb; char **args; int rc = -1; + intptr_t jb[5]; struct Proc *proc; struct CosmoTib *tib; char16_t pipename[64]; @@ -325,7 +325,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { return -1; ftrace_enabled(-1); strace_enabled(-1); - if (!setjmp(jb)) { + if (!__builtin_setjmp(jb)) { reader = CreateNamedPipe(__create_pipe_name(pipename), kNtPipeAccessInbound, kNtPipeTypeByte | kNtPipeReadmodeByte, 1, PIPE_BUF, PIPE_BUF, 0, &kNtIsInheritable); @@ -467,12 +467,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { if (ftrace_stackdigs) _weaken(__hook)(_weaken(ftrace_hook), _weaken(GetSymbolTable)()); // reset core runtime services - __proc_wipe(); WipeKeystrokes(); - if (_weaken(__sig_init)) - _weaken(__sig_init)(); - if (_weaken(__itimer_wipe)) - _weaken(__itimer_wipe)(); // notify pthread join atomic_store_explicit(&_pthread_static.ptid, GetCurrentThreadId(), memory_order_release); diff --git a/libc/proc/fork.c b/libc/proc/fork.c index 031ecef3163..a836b0102ff 100644 --- a/libc/proc/fork.c +++ b/libc/proc/fork.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/calls/sig.internal.h" #include "libc/calls/state.internal.h" #include "libc/calls/struct/sigset.internal.h" #include "libc/calls/struct/timespec.h" @@ -27,6 +28,7 @@ #include "libc/intrin/cxaatexit.h" #include "libc/intrin/dll.h" #include "libc/intrin/maps.h" +#include "libc/intrin/stack.h" #include "libc/intrin/strace.h" #include "libc/intrin/weaken.h" #include "libc/nt/files.h" @@ -39,6 +41,7 @@ #include "libc/runtime/syslib.internal.h" #include "libc/stdio/internal.h" #include "libc/str/str.h" +#include "libc/thread/itimer.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" #include "third_party/dlmalloc/dlmalloc.h" @@ -104,10 +107,6 @@ static void fork_prepare(void) { pthread_mutex_lock(&supreme_lock); if (_weaken(_pthread_onfork_prepare)) _weaken(_pthread_onfork_prepare)(); - if (IsWindows()) { - pthread_mutex_lock(&__sig_worker_lock); - __proc_lock(); - } fork_prepare_stdio(); __localtime_lock(); __cxa_lock(); @@ -117,12 +116,16 @@ static void fork_prepare(void) { dlmalloc_pre_fork(); __fds_lock(); pthread_mutex_lock(&__rand64_lock_obj); + if (_weaken(cosmo_stack_lock)) + _weaken(cosmo_stack_lock)(); __maps_lock(); LOCKTRACE("READY TO LOCK AND ROLL"); } static void fork_parent(void) { __maps_unlock(); + if (_weaken(cosmo_stack_unlock)) + _weaken(cosmo_stack_unlock)(); pthread_mutex_unlock(&__rand64_lock_obj); __fds_unlock(); dlmalloc_post_fork_parent(); @@ -132,10 +135,6 @@ static void fork_parent(void) { __cxa_unlock(); __localtime_unlock(); fork_parent_stdio(); - if (IsWindows()) { - __proc_unlock(); - pthread_mutex_unlock(&__sig_worker_lock); - } if (_weaken(_pthread_onfork_parent)) _weaken(_pthread_onfork_parent)(); pthread_mutex_unlock(&supreme_lock); @@ -143,6 +142,8 @@ static void fork_parent(void) { static void fork_child(void) { nsync_mu_semaphore_sem_fork_child(); + if (_weaken(cosmo_stack_wipe)) + _weaken(cosmo_stack_wipe)(); pthread_mutex_wipe_np(&__rand64_lock_obj); pthread_mutex_wipe_np(&__fds_lock_obj); dlmalloc_post_fork_child(); @@ -153,8 +154,13 @@ static void fork_child(void) { pthread_mutex_wipe_np(&__cxa_lock_obj); pthread_mutex_wipe_np(&__localtime_lock_obj); if (IsWindows()) { - __proc_wipe(); + // we don't bother locking the proc/itimer/sig locks above since + // their state is reset in the forked child. nothing to protect. + __proc_wipe_and_reset(); + __itimer_wipe_and_reset(); pthread_mutex_wipe_np(&__sig_worker_lock); + if (_weaken(__sig_init)) + _weaken(__sig_init)(); } if (_weaken(_pthread_onfork_child)) _weaken(_pthread_onfork_child)(); @@ -211,6 +217,7 @@ int _fork(uint32_t dwCreationFlags) { memory_order_relaxed); } dll_make_first(&_pthread_list, &pt->list); + atomic_store_explicit(&_pthread_count, 1, memory_order_relaxed); // get new system thread handle intptr_t syshand = 0; diff --git a/libc/proc/proc.c b/libc/proc/proc.c index 97ba83c6929..56a5ff0a5ae 100644 --- a/libc/proc/proc.c +++ b/libc/proc/proc.c @@ -268,7 +268,8 @@ textwindows void __proc_unlock(void) { /** * Resets process tracker from forked child. */ -textwindows void __proc_wipe(void) { +textwindows void __proc_wipe_and_reset(void) { + // TODO(jart): Should we preserve this state in forked children? pthread_mutex_t lock = __proc.lock; bzero(&__proc, sizeof(__proc)); __proc.lock = lock; diff --git a/libc/proc/proc.internal.h b/libc/proc/proc.internal.h index 46ef01e8558..3ecc44ad5d4 100644 --- a/libc/proc/proc.internal.h +++ b/libc/proc/proc.internal.h @@ -41,7 +41,6 @@ struct Procs { extern struct Procs __proc; -void __proc_wipe(void) libcesque; void __proc_lock(void) libcesque; void __proc_unlock(void) libcesque; int64_t __proc_handle(int) libcesque; @@ -49,6 +48,7 @@ int64_t __proc_search(int) libcesque; struct Proc *__proc_new(void) libcesque; void __proc_add(struct Proc *) libcesque; void __proc_free(struct Proc *) libcesque; +void __proc_wipe_and_reset(void) libcesque; int __proc_harvest(struct Proc *, bool) libcesque; int sys_wait4_nt(int, int *, int, struct rusage *) libcesque; diff --git a/libc/runtime/clone.c b/libc/runtime/clone.c index e24782a3e48..25b948a08d6 100644 --- a/libc/runtime/clone.c +++ b/libc/runtime/clone.c @@ -29,6 +29,7 @@ #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/intrin/describeflags.h" +#include "libc/intrin/strace.h" #include "libc/intrin/ulock.h" #include "libc/intrin/weaken.h" #include "libc/limits.h" @@ -56,6 +57,7 @@ #include "libc/sysv/errfuns.h" #include "libc/thread/freebsd.internal.h" #include "libc/thread/openbsd.internal.h" +#include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" #include "libc/thread/xnu.internal.h" @@ -188,6 +190,7 @@ XnuThreadMain(void *pthread, // rdi struct CloneArgs *wt, // r8 unsigned xnuflags) { // r9 int ax; + wt->tid = tid; *wt->ctid = tid; *wt->ptid = tid; @@ -259,7 +262,7 @@ static errno_t CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, // we can't use address sanitizer because: // 1. __asan_handle_no_return wipes stack [todo?] -static wontreturn void OpenbsdThreadMain(void *p) { +relegated static wontreturn void OpenbsdThreadMain(void *p) { struct CloneArgs *wt = p; *wt->ctid = wt->tid; wt->func(wt->arg, wt->tid); @@ -276,9 +279,9 @@ static wontreturn void OpenbsdThreadMain(void *p) { __builtin_unreachable(); } -static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz, - int flags, void *arg, void *tls, atomic_int *ptid, - atomic_int *ctid) { +relegated errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, + size_t stksz, int flags, void *arg, void *tls, + atomic_int *ptid, atomic_int *ctid) { int rc; intptr_t sp; struct __tfork *tf; @@ -299,10 +302,8 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz, tf->tf_tcb = flags & CLONE_SETTLS ? tls : 0; tf->tf_tid = &wt->tid; if ((rc = __tfork_thread(tf, sizeof(*tf), OpenbsdThreadMain, wt)) >= 0) { - npassert(rc); - if (flags & CLONE_PARENT_SETTID) { + if (flags & CLONE_PARENT_SETTID) *ptid = rc; - } return 0; } else { return -rc; @@ -314,18 +315,20 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz, static wontreturn void NetbsdThreadMain(void *arg, // rdi int (*func)(void *, int), // rsi - int *tid, // rdx - atomic_int *ctid, // rcx - int *ztid) { // r9 + int flags, // rdx + atomic_int *ctid) { // rcx int ax, dx; - // TODO(jart): Why are we seeing flakes where *tid is zero? - // ax = *tid; + static atomic_int clobber; + atomic_int *ztid = &clobber; ax = sys_gettid(); - *ctid = ax; + if (flags & CLONE_CHILD_SETTID) + atomic_store_explicit(ctid, ax, memory_order_release); + if (flags & CLONE_CHILD_CLEARTID) + ztid = ctid; func(arg, ax); // we no longer use the stack after this point // %eax = int __lwp_exit(void); - asm volatile("movl\t$0,%2\n\t" // *wt->ztid = 0 + asm volatile("movl\t$0,%2\n\t" // *ztid = 0 "syscall" // __lwp_exit() : "=a"(ax), "=d"(dx), "=m"(*ztid) : "0"(310) @@ -340,7 +343,6 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz, // second-class API, intended to help Linux folks migrate to this. int ax; bool failed; - atomic_int *tid; intptr_t dx, sp; static bool once; struct ucontext_netbsd *ctx; @@ -357,12 +359,6 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz, } sp = (intptr_t)stk + stksz; - // allocate memory for tid - sp -= sizeof(atomic_int); - sp = sp & -alignof(atomic_int); - tid = (atomic_int *)sp; - *tid = 0; - // align the stack sp = AlignStack(sp, stk, stksz, 16); @@ -383,9 +379,8 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz, ctx->uc_mcontext.rip = (intptr_t)NetbsdThreadMain; ctx->uc_mcontext.rdi = (intptr_t)arg; ctx->uc_mcontext.rsi = (intptr_t)func; - ctx->uc_mcontext.rdx = (intptr_t)tid; - ctx->uc_mcontext.rcx = (intptr_t)(flags & CLONE_CHILD_SETTID ? ctid : tid); - ctx->uc_mcontext.r8 = (intptr_t)(flags & CLONE_CHILD_CLEARTID ? ctid : tid); + ctx->uc_mcontext.rdx = flags; + ctx->uc_mcontext.rcx = (intptr_t)ctid; ctx->uc_flags |= _UC_STACK; ctx->uc_stack.ss_sp = stk; ctx->uc_stack.ss_size = stksz; @@ -396,15 +391,15 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz, } // perform the system call + int tid = 0; asm volatile(CFLAG_ASM("syscall") : CFLAG_CONSTRAINT(failed), "=a"(ax), "=d"(dx) - : "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(tid) + : "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(&tid) : "rcx", "r8", "r9", "r10", "r11", "memory"); if (!failed) { - npassert(*tid); - if (flags & CLONE_PARENT_SETTID) { - *ptid = *tid; - } + unassert(tid); + if (flags & CLONE_PARENT_SETTID) + *ptid = tid; return 0; } else { return ax; @@ -744,43 +739,47 @@ static int CloneLinux(int (*func)(void *arg, int rc), char *stk, size_t stksz, */ errno_t clone(void *func, void *stk, size_t stksz, int flags, void *arg, void *ptid, void *tls, void *ctid) { - int rc; + errno_t err; + + atomic_fetch_add(&_pthread_count, 1); if (!func) { - rc = EINVAL; + err = EINVAL; } else if (IsLinux()) { - rc = CloneLinux(func, stk, stksz, flags, arg, tls, ptid, ctid); + err = CloneLinux(func, stk, stksz, flags, arg, tls, ptid, ctid); } else if (!IsTiny() && (flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID)) != (CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_SYSVSEM)) { - rc = EINVAL; + err = EINVAL; } else if (IsXnu()) { #ifdef __x86_64__ - rc = CloneXnu(func, stk, stksz, flags, arg, tls, ptid, ctid); + err = CloneXnu(func, stk, stksz, flags, arg, tls, ptid, ctid); #elif defined(__aarch64__) - rc = CloneSilicon(func, stk, stksz, flags, arg, tls, ptid, ctid); + err = CloneSilicon(func, stk, stksz, flags, arg, tls, ptid, ctid); #else #error "unsupported architecture" #endif } else if (IsFreebsd()) { - rc = CloneFreebsd(func, stk, stksz, flags, arg, tls, ptid, ctid); + err = CloneFreebsd(func, stk, stksz, flags, arg, tls, ptid, ctid); #ifdef __x86_64__ } else if (IsNetbsd()) { - rc = CloneNetbsd(func, stk, stksz, flags, arg, tls, ptid, ctid); + err = CloneNetbsd(func, stk, stksz, flags, arg, tls, ptid, ctid); } else if (IsOpenbsd()) { - rc = CloneOpenbsd(func, stk, stksz, flags, arg, tls, ptid, ctid); + err = CloneOpenbsd(func, stk, stksz, flags, arg, tls, ptid, ctid); } else if (IsWindows()) { - rc = CloneWindows(func, stk, stksz, flags, arg, tls, ptid, ctid); + err = CloneWindows(func, stk, stksz, flags, arg, tls, ptid, ctid); #endif /* __x86_64__ */ } else { - rc = ENOSYS; + err = ENOSYS; } - if (SupportsBsd() && rc == EPROCLIM) { - rc = EAGAIN; - } + if (SupportsBsd() && err == EPROCLIM) + err = EAGAIN; + + if (err) + unassert(atomic_fetch_sub(&_pthread_count, 1) > 1); - return rc; + return err; } diff --git a/libc/system/popen.c b/libc/system/popen.c index b15b8adcafb..2636cc5ffc6 100644 --- a/libc/system/popen.c +++ b/libc/system/popen.c @@ -87,7 +87,6 @@ FILE *popen(const char *cmdline, const char *mode) { // "The popen() function shall ensure that any streams from // previous popen() calls that remain open in the parent // process are closed in the new child process." -POSIX - __stdio_lock(); for (struct Dll *e = dll_first(__stdio.files); e; e = dll_next(__stdio.files, e)) { FILE *f2 = FILE_CONTAINER(e); @@ -96,7 +95,6 @@ FILE *popen(const char *cmdline, const char *mode) { f2->fd = -1; } } - __stdio_unlock(); _Exit(_cocmd(3, (char *[]){ diff --git a/libc/testlib/BUILD.mk b/libc/testlib/BUILD.mk index 401a81093bd..5de84b1d2c9 100644 --- a/libc/testlib/BUILD.mk +++ b/libc/testlib/BUILD.mk @@ -27,6 +27,7 @@ LIBC_TESTLIB_A_HDRS = \ libc/testlib/ezbench.h \ libc/testlib/fastrandomstring.h \ libc/testlib/hyperion.h \ + libc/testlib/manystack.h \ libc/testlib/moby.h \ libc/testlib/subprocess.h \ libc/testlib/testlib.h \ @@ -70,6 +71,7 @@ LIBC_TESTLIB_A_SRCS_C = \ libc/testlib/globals.c \ libc/testlib/hexequals.c \ libc/testlib/incrementfailed.c \ + libc/testlib/manystack.c \ libc/testlib/memoryexists.c \ libc/testlib/seterrno.c \ libc/testlib/shoulddebugbreak.c \ @@ -110,6 +112,7 @@ LIBC_TESTLIB_A_DIRECTDEPS = \ LIBC_STR \ LIBC_SYSV \ LIBC_SYSV_CALLS \ + LIBC_THREAD \ LIBC_TINYMATH \ LIBC_X \ THIRD_PARTY_COMPILER_RT \ diff --git a/libc/testlib/manystack.c b/libc/testlib/manystack.c new file mode 100644 index 00000000000..b0b022ba195 --- /dev/null +++ b/libc/testlib/manystack.c @@ -0,0 +1,69 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/testlib/manystack.h" +#include "libc/atomic.h" +#include "libc/calls/struct/sigaction.h" +#include "libc/calls/struct/sigset.h" +#include "libc/intrin/dll.h" +#include "libc/sysv/consts/sig.h" +#include "libc/thread/posixthread.internal.h" + +static atomic_int manystack_gotsig; +static atomic_bool manystack_shutdown; + +static void manystack_signal(int sig) { + manystack_gotsig = sig; +} + +static void *manystack_thread(void *arg) { + sigset_t ss; + sigfillset(&ss); + sigdelset(&ss, SIGUSR2); + while (!manystack_shutdown) { + sigsuspend(&ss); + if (!manystack_shutdown) { + _pthread_lock(); + for (struct Dll *e = dll_first(_pthread_list); e; + e = dll_next(_pthread_list, e)) { + pthread_t th = (pthread_t)POSIXTHREAD_CONTAINER(e); + if (!pthread_equal(th, pthread_self())) + pthread_kill(th, SIGQUIT); + } + _pthread_unlock(); + } + } + return 0; +} + +pthread_t manystack_start(void) { + sigset_t ss; + pthread_t msh; + sigemptyset(&ss); + sigaddset(&ss, SIGUSR2); + sigprocmask(SIG_BLOCK, &ss, 0); + signal(SIGUSR2, manystack_signal); + pthread_create(&msh, 0, manystack_thread, 0); + return msh; +} + +void manystack_stop(pthread_t msh) { + manystack_shutdown = true; + pthread_kill(msh, SIGUSR2); + pthread_join(msh, 0); +} diff --git a/libc/testlib/manystack.h b/libc/testlib/manystack.h new file mode 100644 index 00000000000..a175ecbea97 --- /dev/null +++ b/libc/testlib/manystack.h @@ -0,0 +1,10 @@ +#ifndef COSMOPOLITAN_LIBC_TESTLIB_MANYSTACK_H_ +#define COSMOPOLITAN_LIBC_TESTLIB_MANYSTACK_H_ +#include "libc/thread/thread.h" +COSMOPOLITAN_C_START_ + +pthread_t manystack_start(void); +void manystack_stop(pthread_t); + +COSMOPOLITAN_C_END_ +#endif /* COSMOPOLITAN_LIBC_TESTLIB_MANYSTACK_H_ */ diff --git a/libc/testlib/testmain.c b/libc/testlib/testmain.c index 1b56570f1f6..e496b4f3cba 100644 --- a/libc/testlib/testmain.c +++ b/libc/testlib/testmain.c @@ -156,7 +156,7 @@ int main(int argc, char *argv[]) { // make sure threads are in a good state if (_weaken(_pthread_decimate)) - _weaken(_pthread_decimate)(false); + _weaken(_pthread_decimate)(); if (_weaken(pthread_orphan_np) && !_weaken(pthread_orphan_np)()) { tinyprint(2, "error: tests ended with threads still active\n", NULL); _Exit(1); diff --git a/libc/thread/itimer.c b/libc/thread/itimer.c index 6df84c7e4d7..5f3ba03af45 100644 --- a/libc/thread/itimer.c +++ b/libc/thread/itimer.c @@ -33,18 +33,13 @@ #include "libc/sysv/consts/sicode.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" -#include "libc/thread/itimer.internal.h" +#include "libc/thread/itimer.h" #include "libc/thread/thread2.h" #include "libc/thread/tls.h" #ifdef __x86_64__ #define STACK_SIZE 65536 -struct IntervalTimer __itimer = { - .lock = PTHREAD_MUTEX_INITIALIZER, - .cond = PTHREAD_COND_INITIALIZER, -}; - static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { struct CosmoTib tls; char *sp = __builtin_frame_address(0); @@ -55,7 +50,7 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { for (;;) { bool dosignal = false; struct timeval now, waituntil; - pthread_mutex_lock(&__itimer.lock); + __itimer_lock(); now = timeval_real(); if (timeval_iszero(__itimer.it.it_value)) { waituntil = timeval_max; @@ -76,13 +71,13 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { dosignal = true; } } - pthread_mutex_unlock(&__itimer.lock); + __itimer_unlock(); if (dosignal) __sig_generate(SIGALRM, SI_TIMER); - pthread_mutex_lock(&__itimer.lock); + __itimer_lock(); struct timespec deadline = timeval_totimespec(waituntil); pthread_cond_timedwait(&__itimer.cond, &__itimer.lock, &deadline); - pthread_mutex_unlock(&__itimer.lock); + __itimer_unlock(); } return 0; } @@ -92,39 +87,30 @@ static textwindows void __itimer_setup(void) { kNtStackSizeParamIsAReservation, 0); } -textwindows void __itimer_wipe(void) { - // this function is called by fork(), because - // timers aren't inherited by forked subprocesses - bzero(&__itimer, sizeof(__itimer)); -} - textwindows int sys_setitimer_nt(int which, const struct itimerval *neu, struct itimerval *old) { struct itimerval config; cosmo_once(&__itimer.once, __itimer_setup); if (which != ITIMER_REAL || (neu && (!timeval_isvalid(neu->it_value) || - !timeval_isvalid(neu->it_interval)))) { + !timeval_isvalid(neu->it_interval)))) return einval(); - } - if (neu) { + if (neu) // POSIX defines setitimer() with the restrict keyword but let's // accommodate the usage setitimer(ITIMER_REAL, &it, &it) anyway config = *neu; - } BLOCK_SIGNALS; - pthread_mutex_lock(&__itimer.lock); + __itimer_lock(); if (old) { old->it_interval = __itimer.it.it_interval; old->it_value = timeval_subz(__itimer.it.it_value, timeval_real()); } if (neu) { - if (!timeval_iszero(config.it_value)) { + if (!timeval_iszero(config.it_value)) config.it_value = timeval_add(config.it_value, timeval_real()); - } __itimer.it = config; pthread_cond_signal(&__itimer.cond); } - pthread_mutex_unlock(&__itimer.lock); + __itimer_unlock(); ALLOW_SIGNALS; return 0; } diff --git a/libc/thread/itimer.internal.h b/libc/thread/itimer.h similarity index 82% rename from libc/thread/itimer.internal.h rename to libc/thread/itimer.h index 204c3bf8d81..a5193d987e2 100644 --- a/libc/thread/itimer.internal.h +++ b/libc/thread/itimer.h @@ -15,7 +15,9 @@ struct IntervalTimer { extern struct IntervalTimer __itimer; -void __itimer_wipe(void); +void __itimer_lock(void); +void __itimer_unlock(void); +void __itimer_wipe_and_reset(void); COSMOPOLITAN_C_END_ #endif /* COSMOPOLITAN_LIBC_ITIMER_H_ */ diff --git a/libc/runtime/mapstack.c b/libc/thread/mapstack.c similarity index 70% rename from libc/runtime/mapstack.c rename to libc/thread/mapstack.c index eccd5cefc27..470ab58a63b 100644 --- a/libc/runtime/mapstack.c +++ b/libc/thread/mapstack.c @@ -16,18 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/syscall-sysv.internal.h" -#include "libc/dce.h" -#include "libc/runtime/memtrack.internal.h" -#include "libc/runtime/runtime.h" +#include "libc/cosmo.h" +#include "libc/errno.h" #include "libc/runtime/stack.h" -#include "libc/sysv/consts/auxv.h" -#include "libc/sysv/consts/map.h" -#include "libc/sysv/consts/prot.h" - -#define MAP_ANON_OPENBSD 0x1000 -#define MAP_STACK_OPENBSD 0x4000 /** * Allocates stack. @@ -43,28 +34,23 @@ * @return stack bottom address on success, or null w/ errno */ void *NewCosmoStack(void) { - char *p; - size_t n = GetStackSize(); - if ((p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, - 0)) != MAP_FAILED) { - if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | - MAP_STACK_OPENBSD, - -1, 0, 0) != p) - notpossible; - if (mprotect(p, GetGuardSize(), PROT_NONE | PROT_GUARD)) - notpossible; - return p; - } else { - return 0; - } + void *stackaddr; + size_t stacksize = GetStackSize(); + size_t guardsize = GetGuardSize(); + errno_t err = cosmo_stack_alloc(&stacksize, &guardsize, &stackaddr); + if (!err) + return stackaddr; + errno = err; + return 0; } /** * Frees stack. * - * @param stk was allocated by NewCosmoStack() + * @param stackaddr was allocated by NewCosmoStack() + * @return 0 on success, or -1 w/ errno */ -int FreeCosmoStack(void *stk) { - return munmap(stk, GetStackSize()); +int FreeCosmoStack(void *stackaddr) { + cosmo_stack_free(stackaddr, GetStackSize(), GetGuardSize()); + return 0; } diff --git a/libc/thread/posixthread.internal.h b/libc/thread/posixthread.internal.h index 41d268ed138..8fa21680505 100644 --- a/libc/thread/posixthread.internal.h +++ b/libc/thread/posixthread.internal.h @@ -74,6 +74,7 @@ struct PosixThread { int pt_flags; // 0x00: see PT_* constants atomic_int pt_canceled; // 0x04: thread has bad beliefs _Atomic(enum PosixThreadStatus) pt_status; + _Atomic(atomic_int *) pt_blocker; atomic_int ptid; // transitions 0 → tid atomic_int pt_refs; // prevents decimation void *(*pt_start)(void *); // creation callback @@ -83,11 +84,10 @@ struct PosixThread { struct CosmoTib *tib; // middle of tls allocation struct Dll list; // list of threads struct _pthread_cleanup_buffer *pt_cleanup; - _Atomic(atomic_int *) pt_blocker; uint64_t pt_blkmask; int64_t pt_event; locale_t pt_locale; - jmp_buf pt_exiter; + intptr_t pt_exiter[5]; pthread_attr_t pt_attr; atomic_bool pt_intoff; }; @@ -95,6 +95,7 @@ struct PosixThread { typedef void (*atfork_f)(void); extern struct Dll *_pthread_list; +extern _Atomic(unsigned) _pthread_count; extern struct PosixThread _pthread_static; extern _Atomic(pthread_key_dtor) _pthread_key_dtor[PTHREAD_KEYS_MAX]; @@ -103,7 +104,7 @@ int _pthread_setschedparam_freebsd(int, int, const struct sched_param *); int _pthread_tid(struct PosixThread *) libcesque; intptr_t _pthread_syshand(struct PosixThread *) libcesque; long _pthread_cancel_ack(void) libcesque; -void _pthread_decimate(bool) libcesque; +void _pthread_decimate(void) libcesque; void _pthread_free(struct PosixThread *) libcesque; void _pthread_lock(void) libcesque; void _pthread_onfork_child(void) libcesque; diff --git a/libc/thread/pthread_atfork.c b/libc/thread/pthread_atfork.c index 668e221f347..5ef7a92c165 100644 --- a/libc/thread/pthread_atfork.c +++ b/libc/thread/pthread_atfork.c @@ -16,6 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" +#include "libc/cosmo.h" #include "libc/errno.h" #include "libc/intrin/strace.h" #include "libc/mem/mem.h" @@ -28,13 +30,12 @@ struct AtFork { }; struct AtForks { - pthread_once_t once; + atomic_uint once; pthread_mutex_t lock; struct AtFork *list; }; static struct AtForks _atforks = { - .once = PTHREAD_ONCE_INIT, .lock = PTHREAD_MUTEX_INITIALIZER, }; @@ -161,7 +162,7 @@ void _pthread_onfork_child(void) { * @raise ENOMEM if we require more vespene gas */ int pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) { - pthread_once(&_atforks.once, pthread_atfork_init); + cosmo_once(&_atforks.once, pthread_atfork_init); struct AtFork *a; if (!(a = calloc(1, sizeof(struct AtFork)))) return ENOMEM; diff --git a/libc/thread/pthread_attr_setdetachstate.c b/libc/thread/pthread_attr_setdetachstate.c index 253f044953b..e9a57a08445 100644 --- a/libc/thread/pthread_attr_setdetachstate.c +++ b/libc/thread/pthread_attr_setdetachstate.c @@ -28,6 +28,10 @@ * pthread_create(0, &attr, func, 0); * pthread_attr_destroy(&attr); * + * If you use this, please be warned that your thread might run and exit + * before pthread_create() even returns. You really should assume it can + * not be used with any pthread APIs from the calling thread. + * * @param detachstate can be one of * - `PTHREAD_CREATE_JOINABLE` (default) * - `PTHREAD_CREATE_DETACHED` diff --git a/libc/thread/pthread_attr_setsigmask_np.c b/libc/thread/pthread_attr_setsigmask_np.c index b46c94e57fa..a42e8b05589 100644 --- a/libc/thread/pthread_attr_setsigmask_np.c +++ b/libc/thread/pthread_attr_setsigmask_np.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/str/str.h" +#include "libc/sysv/consts/sig.h" #include "libc/thread/thread2.h" /** diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index 02289027600..ba5771a9e94 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -18,10 +18,12 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" #include "libc/calls/calls.h" +#include "libc/calls/sig.internal.h" #include "libc/calls/struct/sigaltstack.h" #include "libc/calls/struct/sigset.h" #include "libc/calls/struct/sigset.internal.h" #include "libc/calls/syscall-sysv.internal.h" +#include "libc/cosmo.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/fmt/itoa.h" @@ -29,6 +31,7 @@ #include "libc/intrin/describeflags.h" #include "libc/intrin/dll.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/stack.h" #include "libc/intrin/strace.h" #include "libc/intrin/weaken.h" #include "libc/log/internal.h" @@ -48,7 +51,6 @@ #include "libc/str/str.h" #include "libc/sysv/consts/auxv.h" #include "libc/sysv/consts/clone.h" -#include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/consts/ss.h" @@ -65,9 +67,6 @@ __static_yoink("_pthread_onfork_prepare"); __static_yoink("_pthread_onfork_parent"); __static_yoink("_pthread_onfork_child"); -#define MAP_ANON_OPENBSD 0x1000 -#define MAP_STACK_OPENBSD 0x4000 - void _pthread_free(struct PosixThread *pt) { // thread must be removed from _pthread_list before calling @@ -79,7 +78,8 @@ void _pthread_free(struct PosixThread *pt) { // unmap stack if the cosmo runtime was responsible for mapping it if (pt->pt_flags & PT_OWNSTACK) - unassert(!munmap(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize)); + cosmo_stack_free(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize, + pt->pt_attr.__guardsize); // free any additional upstream system resources // our fork implementation wipes this handle in child automatically @@ -99,7 +99,7 @@ void _pthread_free(struct PosixThread *pt) { free(pt); } -void _pthread_decimate(bool annihilation_only) { +void _pthread_decimate(void) { struct PosixThread *pt; struct Dll *e, *e2, *list = 0; enum PosixThreadStatus status; @@ -123,17 +123,6 @@ void _pthread_decimate(bool annihilation_only) { dll_make_first(&list, e); } - // code like pthread_exit() needs to call this in order to know if - // it's appropriate to run exit() handlers however we really don't - // want to have a thread exiting block on a bunch of __maps locks! - // therefore we only take action if we'll destroy all but the self - if (annihilation_only) - if (!(_pthread_list == _pthread_list->prev && - _pthread_list == _pthread_list->next)) { - dll_make_last(&_pthread_list, list); - list = 0; - } - // release posix threads gil _pthread_unlock(); @@ -167,11 +156,14 @@ static int PosixThread(void *arg, int tid) { } // set long jump handler so pthread_exit can bring control back here - if (!setjmp(pt->pt_exiter)) { - sigdelset(&pt->pt_attr.__sigmask, SIGTHR); + if (!__builtin_setjmp(pt->pt_exiter)) { + // setup signals for new thread + pt->pt_attr.__sigmask &= ~(1ull << (SIGTHR - 1)); if (IsWindows() || IsMetal()) { atomic_store_explicit(&__get_tls()->tib_sigmask, pt->pt_attr.__sigmask, memory_order_release); + if (_weaken(__sig_check)) + _weaken(__sig_check)(); } else { sys_sigprocmask(SIG_SETMASK, &pt->pt_attr.__sigmask, 0); } @@ -189,39 +181,6 @@ static int PosixThread(void *arg, int tid) { return 0; } -static bool TellOpenbsdThisIsStackMemory(void *addr, size_t size) { - return __sys_mmap( - addr, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1, - 0, 0) == addr; -} - -// OpenBSD only permits RSP to occupy memory that's been explicitly -// defined as stack memory, i.e. `lo <= %rsp < hi` must be the case -static errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *attr) { - - // get interval - uintptr_t lo = (uintptr_t)attr->__stackaddr; - uintptr_t hi = lo + attr->__stacksize; - - // squeeze interval - lo = (lo + __pagesize - 1) & -__pagesize; - hi = hi & -__pagesize; - - // tell os it's stack memory - errno_t olderr = errno; - if (!TellOpenbsdThisIsStackMemory((void *)lo, hi - lo)) { - errno_t err = errno; - errno = olderr; - return err; - } - - // update attributes with usable stack address - attr->__stackaddr = (void *)lo; - attr->__stacksize = hi - lo; - return 0; -} - static errno_t pthread_create_impl(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg, @@ -266,37 +225,18 @@ static errno_t pthread_create_impl(pthread_t *thread, } } else { // cosmo is managing the stack - int pagesize = __pagesize; - pt->pt_attr.__guardsize = ROUNDUP(pt->pt_attr.__guardsize, pagesize); - pt->pt_attr.__stacksize = pt->pt_attr.__stacksize; - if (pt->pt_attr.__guardsize + pagesize > pt->pt_attr.__stacksize) { - _pthread_free(pt); - return EINVAL; - } - pt->pt_attr.__stackaddr = - mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (pt->pt_attr.__stackaddr != MAP_FAILED) { - if (IsOpenbsd()) - if (!TellOpenbsdThisIsStackMemory(pt->pt_attr.__stackaddr, - pt->pt_attr.__stacksize)) - notpossible; - if (pt->pt_attr.__guardsize) - if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize, - PROT_NONE | PROT_GUARD)) - notpossible; - } - if (!pt->pt_attr.__stackaddr || pt->pt_attr.__stackaddr == MAP_FAILED) { - rc = errno; + pt->pt_flags |= PT_OWNSTACK; + errno_t err = + cosmo_stack_alloc(&pt->pt_attr.__stacksize, &pt->pt_attr.__guardsize, + &pt->pt_attr.__stackaddr); + if (err) { _pthread_free(pt); - errno = e; - if (rc == EINVAL || rc == EOVERFLOW) { + if (err == EINVAL || err == EOVERFLOW) { return EINVAL; } else { return EAGAIN; } } - pt->pt_flags |= PT_OWNSTACK; } // setup signal stack @@ -338,6 +278,10 @@ static errno_t pthread_create_impl(pthread_t *thread, dll_make_first(&_pthread_list, &pt->list); _pthread_unlock(); + // if pthread_attr_setdetachstate() was used then it's possible for + // the `pt` object to be freed before this clone call has returned! + _pthread_ref(pt); + // launch PosixThread(pt) in new thread if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize, CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES | @@ -400,8 +344,8 @@ static const char *DescribeHandle(char buf[12], errno_t err, pthread_t *th) { * │ _lwp_create │ * └──────────────┘ * - * @param thread if non-null is used to output the thread id - * upon successful completion + * @param thread is used to output the thread id upon success, which + * must be non-null * @param attr points to launch configuration, or may be null * to use sensible defaults; it must be initialized using * pthread_attr_init() @@ -417,12 +361,14 @@ static const char *DescribeHandle(char buf[12], errno_t err, pthread_t *th) { errno_t pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { errno_t err; - _pthread_decimate(false); + _pthread_decimate(); BLOCK_SIGNALS; err = pthread_create_impl(thread, attr, start_routine, arg, _SigMask); ALLOW_SIGNALS; STRACE("pthread_create([%s], %p, %t, %p) → %s", DescribeHandle(alloca(12), err, thread), attr, start_routine, arg, DescribeErrno(err)); + if (!err) + _pthread_unref(*(struct PosixThread **)thread); return err; } diff --git a/libc/thread/pthread_decimate_np.c b/libc/thread/pthread_decimate_np.c index 3027dc7fa51..93d8e5d7f0f 100644 --- a/libc/thread/pthread_decimate_np.c +++ b/libc/thread/pthread_decimate_np.c @@ -16,22 +16,32 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/cosmo.h" +#include "libc/intrin/stack.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" /** * Garbage collects POSIX threads runtime. * - * Let's say you want to run a memory leak detector. You can say: + * This function frees unreferenced zombie threads and empties cache + * memory associated with the Cosmopolitan POSIX threads runtime. + * + * Here's an example use case for this function. Let's say you want to + * create a malloc() memory leak detector. If your program was running + * threads earlier, then there might still be allocations lingering + * around, that'll give you false positives. To fix this, what you would + * do is call the following, right before running your leak detector: * * while (!pthread_orphan_np()) * pthread_decimate_np(); * - * To wait until all threads have exited. + * Which will wait until all threads have exited and their memory freed. * * @return 0 on success, or errno on error */ int pthread_decimate_np(void) { - _pthread_decimate(false); + _pthread_decimate(); + cosmo_stack_clear(); return 0; } diff --git a/libc/thread/pthread_exit.c b/libc/thread/pthread_exit.c index 78de7062414..c50b867dad9 100644 --- a/libc/thread/pthread_exit.c +++ b/libc/thread/pthread_exit.c @@ -69,7 +69,7 @@ * @noreturn */ wontreturn void pthread_exit(void *rc) { - int orphan; + unsigned population; struct CosmoTib *tib; struct PosixThread *pt; enum PosixThreadStatus status, transition; @@ -94,10 +94,21 @@ wontreturn void pthread_exit(void *rc) { __cxa_thread_finalize(); // run atexit handlers if orphaned thread - _pthread_decimate(true); - if ((orphan = pthread_orphan_np())) - if (_weaken(__cxa_finalize)) - _weaken(__cxa_finalize)(NULL); + // notice how we avoid acquiring the pthread gil + if (!(population = atomic_fetch_sub(&_pthread_count, 1) - 1)) { + // we know for certain we're an orphan. any other threads that + // exist, will terminate and clear their tid very soon. but... + // some goofball could spawn more threads from atexit handlers + for (;;) { + _pthread_decimate(); + if (pthread_orphan_np()) { + if (_weaken(__cxa_finalize)) + _weaken(__cxa_finalize)(NULL); + population = atomic_load(&_pthread_count); + break; + } + } + } // transition the thread to a terminated state status = atomic_load_explicit(&pt->pt_status, memory_order_acquire); @@ -127,7 +138,7 @@ wontreturn void pthread_exit(void *rc) { // thread has been terminated. The behavior shall be as if the // implementation called exit() with a zero argument at thread // termination time." ──Quoth POSIX.1-2017 - if (orphan) { + if (!population) { for (int i = __fini_array_end - __fini_array_start; i--;) ((void (*)(void))__fini_array_start[i])(); _Exit(0); @@ -143,7 +154,7 @@ wontreturn void pthread_exit(void *rc) { } // this is a child thread - longjmp(pt->pt_exiter, 1); + __builtin_longjmp(pt->pt_exiter, 1); } __weak_reference(pthread_exit, thr_exit); diff --git a/libc/thread/pthread_kill.c b/libc/thread/pthread_kill.c index f57a99c553b..127c2774845 100644 --- a/libc/thread/pthread_kill.c +++ b/libc/thread/pthread_kill.c @@ -43,6 +43,8 @@ errno_t pthread_kill(pthread_t thread, int sig) { int err = 0; struct PosixThread *pt; pt = (struct PosixThread *)thread; + if (pt) + _pthread_ref(pt); if (!thread) { err = EFAULT; } else if (!(1 <= sig && sig <= 64)) { @@ -69,5 +71,7 @@ errno_t pthread_kill(pthread_t thread, int sig) { } STRACE("pthread_kill(%d, %G) → %s", _pthread_tid(pt), sig, DescribeErrno(err)); + if (pt) + _pthread_unref(pt); return err; } diff --git a/libc/thread/sem_open.c b/libc/thread/sem_open.c index d708ef7e49a..2fda4471785 100644 --- a/libc/thread/sem_open.c +++ b/libc/thread/sem_open.c @@ -17,10 +17,12 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/calls/blockcancel.internal.h" #include "libc/calls/calls.h" #include "libc/calls/struct/stat.h" #include "libc/calls/syscall-sysv.internal.h" +#include "libc/cosmo.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" @@ -40,7 +42,7 @@ #include "libc/thread/tls.h" static struct Semaphores { - pthread_once_t once; + atomic_uint once; pthread_mutex_t lock; struct Semaphore { struct Semaphore *next; @@ -49,7 +51,9 @@ static struct Semaphores { bool dead; int refs; } *list; -} g_semaphores; +} g_semaphores = { + .lock = PTHREAD_MUTEX_INITIALIZER, +}; static void sem_open_lock(void) { pthread_mutex_lock(&g_semaphores.lock); @@ -69,7 +73,7 @@ static void sem_open_setup(void) { } static void sem_open_init(void) { - pthread_once(&g_semaphores.once, sem_open_setup); + cosmo_once(&g_semaphores.once, sem_open_setup); } static sem_t *sem_open_impl(const char *path, int oflag, unsigned mode, diff --git a/libc/thread/thread.h b/libc/thread/thread.h index cda6ae38b51..4b469a209fd 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -128,10 +128,10 @@ typedef struct pthread_attr_s { int __schedparam; int __schedpolicy; int __contentionscope; - int __guardsize; - int __stacksize; int __sigaltstacksize; uint64_t __sigmask; + size_t __guardsize; + size_t __stacksize; void *__stackaddr; void *__sigaltstackaddr; } pthread_attr_t; diff --git a/test/libc/system/BUILD.mk b/test/libc/system/BUILD.mk index 953f7068bb4..18e63daf898 100644 --- a/test/libc/system/BUILD.mk +++ b/test/libc/system/BUILD.mk @@ -33,6 +33,7 @@ TEST_LIBC_SYSTEM_DIRECTDEPS = \ LIBC_RUNTIME \ LIBC_STDIO \ LIBC_STDIO \ + LIBC_STR \ LIBC_SYSTEM \ LIBC_SYSV \ LIBC_TESTLIB \ @@ -40,6 +41,7 @@ TEST_LIBC_SYSTEM_DIRECTDEPS = \ LIBC_X \ THIRD_PARTY_MUSL \ THIRD_PARTY_TR \ + THIRD_PARTY_TZ \ TEST_LIBC_SYSTEM_DEPS := \ $(call uniq,$(foreach x,$(TEST_LIBC_SYSTEM_DIRECTDEPS),$($(x)))) diff --git a/test/libc/system/popen_test.c b/test/libc/system/popen_test.c index cf9a5d048c6..fb4e0d1db1b 100644 --- a/test/libc/system/popen_test.c +++ b/test/libc/system/popen_test.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/calls/struct/itimerval.h" #include "libc/calls/struct/sigaction.h" #include "libc/dce.h" #include "libc/errno.h" @@ -31,15 +32,40 @@ #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/sysv/consts/f.h" +#include "libc/sysv/consts/itimer.h" #include "libc/sysv/consts/sig.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" +#include "libc/time.h" FILE *f; char buf[32]; +void OnAlarm(int sig) { +} + +void *LolThread(void *arg) { + return 0; +} + void SetUpOnce(void) { testlib_enable_tmp_setup_teardown(); + + // give deadlock detector more information + int64_t t = 0x5cd04d0e; + localtime(&t); + pthread_t th; + pthread_create(&th, 0, LolThread, 0); + pthread_join(th, 0); + char buf[32]; + sprintf(buf, "%g", 3.14); + atexit((void *)LolThread); + FILE *f = fopen("/zip/.cosmo", "r"); + fgetc(f); + fclose(f); + signal(SIGALRM, OnAlarm); + struct itimerval it = {{0, 1000}, {0, 1}}; + setitimer(ITIMER_REAL, &it, 0); } void CheckForFdLeaks(void) { diff --git a/test/libc/thread/pthread_create_test.c b/test/libc/thread/pthread_create_test.c index 03465ff9359..dfaf03e2a3d 100644 --- a/test/libc/thread/pthread_create_test.c +++ b/test/libc/thread/pthread_create_test.c @@ -22,6 +22,8 @@ #include "libc/calls/struct/sched_param.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/siginfo.h" +#include "libc/calls/struct/sigset.h" +#include "libc/cosmo.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/kprintf.h" @@ -40,7 +42,9 @@ #include "libc/sysv/consts/sched.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/consts/ss.h" +#include "libc/testlib/benchmark.h" #include "libc/testlib/ezbench.h" +#include "libc/testlib/manystack.h" #include "libc/testlib/subprocess.h" #include "libc/testlib/testlib.h" #include "libc/thread/posixthread.internal.h" @@ -50,6 +54,10 @@ void OnUsr1(int sig, siginfo_t *si, void *vctx) { } +void SetUpOnce(void) { + cosmo_stack_setmaxstacks((_rand64() & 7) - 1); +} + void SetUp(void) { struct sigaction sig = {.sa_sigaction = OnUsr1, .sa_flags = SA_SIGINFO}; sigaction(SIGUSR1, &sig, 0); @@ -280,10 +288,60 @@ static void CreateDetached(void) { ASSERT_EQ(0, pthread_attr_destroy(&attr)); } +#define LAUNCHES 10 +#define LAUNCHERS 10 + +errno_t pthread_create2(pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine)(void *), void *arg) { + for (int i = 1;; i <<= 1) { + errno_t err = pthread_create(thread, attr, start_routine, arg); + if (err != EAGAIN) + return err; + usleep(i); + } +} + +static void *CreateDetachedParallelThreads(void *arg) { + for (int i = 0; i < LAUNCHES; ++i) + CreateDetached(); + return 0; +} + +static void CreateDetachedParallel(void) { + pthread_t th[LAUNCHERS]; + for (int i = 0; i < LAUNCHERS; ++i) + ASSERT_EQ(0, pthread_create2(&th[i], 0, CreateDetachedParallelThreads, 0)); + for (int i = 0; i < LAUNCHERS; ++i) + ASSERT_EQ(0, pthread_join(th[i], 0)); +} + +static void *CreateJoinParallelThreads(void *arg) { + for (int i = 0; i < LAUNCHES; ++i) + CreateJoin(); + return 0; +} + +static void CreateJoinParallel(void) { + pthread_t th[LAUNCHERS]; + for (int i = 0; i < LAUNCHERS; ++i) + ASSERT_EQ(0, pthread_create2(&th[i], 0, CreateJoinParallelThreads, 0)); + for (int i = 0; i < LAUNCHERS; ++i) + ASSERT_EQ(0, pthread_join(th[i], 0)); +} + TEST(pthread_create, bench) { - EZBENCH2("CreateJoin", donothing, CreateJoin()); - EZBENCH2("CreateDetach", donothing, CreateDetach()); - EZBENCH2("CreateDetached", donothing, CreateDetached()); + kprintf("cosmo_stack_getmaxstacks() = %d\n", cosmo_stack_getmaxstacks()); + pthread_t msh = manystack_start(); + BENCHMARK(100, 1, CreateJoin()); + BENCHMARK(100, 1, CreateDetach()); + usleep(10000); + pthread_decimate_np(); + BENCHMARK(100, 1, CreateDetached()); + usleep(10000); + pthread_decimate_np(); + BENCHMARK(1, LAUNCHERS + LAUNCHERS * LAUNCHES, CreateJoinParallel()); + BENCHMARK(1, LAUNCHERS + LAUNCHERS * LAUNCHES, CreateDetachedParallel()); + manystack_stop(msh); while (!pthread_orphan_np()) pthread_decimate_np(); } diff --git a/test/posix/file_offset_exec_test.c b/test/posix/file_offset_exec_test.c index aafc9061ae7..7cfc6b88d1f 100644 --- a/test/posix/file_offset_exec_test.c +++ b/test/posix/file_offset_exec_test.c @@ -13,6 +13,7 @@ // TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. +#include #include #include #include @@ -36,6 +37,11 @@ void on_unexpected_death(int sig) { } int main() { + + // TODO(jart): fix flakes + if (IsWindows()) + return 0; + signal(SIGCHLD, on_unexpected_death); // extract test program diff --git a/test/posix/fork_bench_test.c b/test/posix/fork_bench_test.c new file mode 100644 index 00000000000..6f962f89c6c --- /dev/null +++ b/test/posix/fork_bench_test.c @@ -0,0 +1,29 @@ +// Copyright 2024 Justine Alexandra Roberts Tunney +// +// Permission to use, copy, modify, and/or distribute this software for +// any purpose with or without fee is hereby granted, provided that the +// above copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include +#include +#include "libc/testlib/benchmark.h" + +void TestFork(void) { + int pid; + if (!(pid = fork())) + _Exit(0); + wait(0); +} + +int main(int argc, char *argv[]) { + BENCHMARK(100, 1, TestFork()); +} diff --git a/test/posix/mutex_async_signal_safety_test.c b/test/posix/mutex_async_signal_safety_test.c index d861ba42aa8..da6d2020b25 100644 --- a/test/posix/mutex_async_signal_safety_test.c +++ b/test/posix/mutex_async_signal_safety_test.c @@ -25,7 +25,7 @@ // // glibc fails this test // musl passes this test -// cosmo only guarantees this in process shared mode +// cosmo only guarantees this in process-shared non-debug mode atomic_bool done; atomic_bool ready; @@ -51,7 +51,14 @@ void* work(void* arg) { int main() { + if (IsQemuUser()) { + // qemu is believed to be the one at fault + kprintf("mutex_async_signal_safety_test flakes on qemu\n"); + return 0; + } + if (IsModeDbg()) { + // the deadlock detector gets in the way of our glorious spin lock kprintf("mutex_async_signal_safety_test not feasible in debug mode\n"); return 0; } diff --git a/test/posix/signal_latency_async_test.c b/test/posix/signal_latency_async_test.c index 2f4fc8d1bb1..ba738bc9708 100644 --- a/test/posix/signal_latency_async_test.c +++ b/test/posix/signal_latency_async_test.c @@ -13,6 +13,7 @@ // TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. +#include #include #include #include @@ -107,6 +108,10 @@ int compare(const void *a, const void *b) { int main() { + // TODO(jart): fix flakes + if (IsWindows()) + return 0; + // Install signal handlers struct sigaction sa; sa.sa_handler = receiver_signal_handler; diff --git a/third_party/dlmalloc/dlmalloc.c b/third_party/dlmalloc/dlmalloc.c index 2ef20f81411..0adc13f4fb2 100644 --- a/third_party/dlmalloc/dlmalloc.c +++ b/third_party/dlmalloc/dlmalloc.c @@ -45,7 +45,6 @@ #define USE_LOCKS 2 #define MALLOC_INSPECT_ALL 1 #define ABORT_ON_ASSERT_FAILURE 0 -#define LOCK_AT_FORK 1 #define NO_MALLOC_STATS 1 #if IsModeDbg() diff --git a/third_party/dlmalloc/init.inc b/third_party/dlmalloc/init.inc index 79ca7f2a564..ac7ce8edf5f 100644 --- a/third_party/dlmalloc/init.inc +++ b/third_party/dlmalloc/init.inc @@ -3,38 +3,38 @@ #include "libc/nexgen32e/rdtsc.h" #include "libc/runtime/runtime.h" -/* ---------------------------- setting mparams -------------------------- */ - -#if LOCK_AT_FORK -#if ONLY_MSPACES - void dlmalloc_pre_fork(void) { +#if ONLY_MSPACES mstate h; for (unsigned i = ARRAYLEN(g_heaps); i--;) if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire))) ACQUIRE_LOCK(&h->mutex); +#else + ACQUIRE_LOCK(&(gm)->mutex); +#endif } void dlmalloc_post_fork_parent(void) { +#if ONLY_MSPACES mstate h; for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i) if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire))) RELEASE_LOCK(&h->mutex); +#else + RELEASE_LOCK(&(gm)->mutex); +#endif } void dlmalloc_post_fork_child(void) { +#if ONLY_MSPACES mstate h; for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i) if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire))) - (void)REFRESH_LOCK(&h->mutex); -} - + REFRESH_LOCK(&h->mutex); #else -void dlmalloc_pre_fork(void) { ACQUIRE_LOCK(&(gm)->mutex); } -void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); } -void dlmalloc_post_fork_child(void) { (void)REFRESH_LOCK(&(gm)->mutex); } -#endif /* ONLY_MSPACES */ -#endif /* LOCK_AT_FORK */ + REFRESH_LOCK(&(gm)->mutex); +#endif +} /* Initialize mparams */ __attribute__((__constructor__(49))) int init_mparams(void) { diff --git a/third_party/dlmalloc/platform.inc b/third_party/dlmalloc/platform.inc index 182de0a0e51..5385a7f8873 100644 --- a/third_party/dlmalloc/platform.inc +++ b/third_party/dlmalloc/platform.inc @@ -151,10 +151,6 @@ ======================================================================== */ -#ifndef LOCK_AT_FORK -#define LOCK_AT_FORK 0 -#endif - /* ------------------- size_t and alignment properties -------------------- */ /* The byte and bit size of a size_t */ diff --git a/third_party/nsync/common.c b/third_party/nsync/common.c index c3d2c764d03..80f695a479c 100644 --- a/third_party/nsync/common.c +++ b/third_party/nsync/common.c @@ -40,6 +40,7 @@ #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/mu_semaphore.internal.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/strace.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -179,10 +180,10 @@ static waiter *free_waiters_pop (void) { return w; } -static void free_waiters_populate (void) { +static bool free_waiters_populate (void) { int n; if (IsNetbsd ()) { - // netbsd needs a real file descriptor per semaphore + // netbsd semaphores are file descriptors n = 1; } else { n = __pagesize / sizeof(waiter); @@ -192,14 +193,17 @@ static void free_waiters_populate (void) { MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (waiters == MAP_FAILED) - nsync_panic_ ("out of memory\n"); + return false; for (size_t i = 0; i < n; ++i) { waiter *w = &waiters[i]; w->tag = WAITER_TAG; w->nw.tag = NSYNC_WAITER_TAG; if (!nsync_mu_semaphore_init (&w->sem)) { - if (!i) - nsync_panic_ ("out of semaphores\n"); + if (!i) { + // netbsd can run out of semaphores + munmap (waiters, n * sizeof (waiter)); + return false; + } break; } w->nw.sem = &w->sem; @@ -208,6 +212,7 @@ static void free_waiters_populate (void) { dll_init (&w->same_condition); free_waiters_push (w); } + return true; } /* -------------------------------- */ @@ -232,11 +237,18 @@ void nsync_waiter_destroy (void *v) { waiter *nsync_waiter_new_ (void) { waiter *w; waiter *tw; + unsigned attempts = 0; + bool out_of_semaphores = false; tw = waiter_for_thread; w = tw; if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) { - while (!(w = free_waiters_pop ())) - free_waiters_populate (); + while (!(w = free_waiters_pop ())) { + if (!out_of_semaphores) + if (!free_waiters_populate ()) + out_of_semaphores = true; + if (out_of_semaphores) + attempts = pthread_delay_np (&free_waiters, attempts); + } if (tw == NULL) { w->flags |= WAITER_RESERVED; waiter_for_thread = w; diff --git a/third_party/nsync/mu_semaphore_sem.c b/third_party/nsync/mu_semaphore_sem.c index 4ae67cb84bd..2f8b61d455c 100644 --- a/third_party/nsync/mu_semaphore_sem.c +++ b/third_party/nsync/mu_semaphore_sem.c @@ -33,7 +33,6 @@ #include "third_party/nsync/time.h" #include "third_party/nsync/mu_semaphore.h" #include "libc/intrin/atomic.h" -#include "libc/atomic.h" #include "third_party/nsync/time.h" /** @@ -83,8 +82,9 @@ void nsync_mu_semaphore_sem_fork_child (void) { for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next) { int rc = sys_close (f->id); STRACE ("close(%ld) → %d", f->id, rc); - ASSERT (nsync_mu_semaphore_sem_create (f)); } + for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next) + ASSERT (nsync_mu_semaphore_sem_create (f)); } /* Initialize *s; the initial value is 0. */ @@ -92,7 +92,7 @@ bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) { struct sem *f = (struct sem *) s; if (!nsync_mu_semaphore_sem_create (f)) return false; - sems_push(f); + sems_push (f); return true; } diff --git a/tool/build/runit.c b/tool/build/runit.c index 5438669e3d7..1b123f3ea95 100644 --- a/tool/build/runit.c +++ b/tool/build/runit.c @@ -384,6 +384,10 @@ int RunOnHost(char *spec) { handshake_latency = timespec_tomicros(timespec_sub(timespec_mono(), start)); if (!err) break; + if (err == MBEDTLS_ERR_NET_CONN_RESET) { + close(g_sock); + continue; + } WARNF("handshake with %s:%d failed -0x%04x (%s)", // g_hostname, g_runitdport, err, GetTlsError(err)); close(g_sock);