diff options
| author | riastradh <riastradh@NetBSD.org> | 2018-08-27 13:33:59 +0000 |
|---|---|---|
| committer | riastradh <riastradh@NetBSD.org> | 2018-08-27 13:33:59 +0000 |
| commit | e0e9bea68fd71c656ba5d40506356cd00a2f33b9 (patch) | |
| tree | 4b06eba61cb74875c46e09fa2376b0d6e9b7bd8a /sys | |
| parent | e7d46ecb9428a40b975ce19378f2fca9f688370d (diff) | |
Draft implementation of the Linux reservation and fence APIs.
Diffstat (limited to 'sys')
| -rw-r--r-- | sys/external/bsd/drm2/include/linux/fence.h | 29 | ||||
| -rw-r--r-- | sys/external/bsd/drm2/include/linux/reservation.h | 86 | ||||
| -rw-r--r-- | sys/external/bsd/drm2/linux/files.drmkms_linux | 4 | ||||
| -rw-r--r-- | sys/external/bsd/drm2/linux/linux_fence.c | 537 | ||||
| -rw-r--r-- | sys/external/bsd/drm2/linux/linux_module.c | 7 | ||||
| -rw-r--r-- | sys/external/bsd/drm2/linux/linux_reservation.c | 638 |
6 files changed, 1232 insertions, 69 deletions
diff --git a/sys/external/bsd/drm2/include/linux/fence.h b/sys/external/bsd/drm2/include/linux/fence.h index 08e6a02ed08..f8219c9f597 100644 --- a/sys/external/bsd/drm2/include/linux/fence.h +++ b/sys/external/bsd/drm2/include/linux/fence.h @@ -1,4 +1,4 @@ -/* $NetBSD: fence.h,v 1.10 2018/08/27 07:53:05 riastradh Exp $ */ +/* $NetBSD: fence.h,v 1.11 2018/08/27 13:33:59 riastradh Exp $ */ /*- * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -33,9 +33,13 @@ #define _LINUX_FENCE_H_ #include <sys/types.h> +#include <sys/condvar.h> +#include <sys/kernel.h> +#include <sys/queue.h> -#include <linux/mutex.h> +#include <linux/kref.h> #include <linux/rcupdate.h> +#include <linux/spinlock.h> struct fence_cb; @@ -46,10 +50,15 @@ struct fence { unsigned context; unsigned seqno; const struct fence_ops *ops; + + TAILQ_HEAD(, fence_cb) f_callbacks; + kcondvar_t f_cv; + struct rcu_head f_rcu; }; -#define FENCE_FLAG_SIGNALED_BIT __BIT(0) -#define FENCE_FLAG_USER_BITS __BIT(1) +#define FENCE_FLAG_ENABLE_SIGNAL_BIT 0 +#define FENCE_FLAG_SIGNALED_BIT 1 +#define FENCE_FLAG_USER_BITS 2 struct fence_ops { const char *(*get_driver_name)(struct fence *); @@ -63,14 +72,19 @@ struct fence_ops { typedef void (*fence_func_t)(struct fence *, struct fence_cb *); struct fence_cb { - fence_func_t fcb_func; + fence_func_t fcb_func; + TAILQ_ENTRY(fence_cb) fcb_entry; + bool fcb_onqueue; }; #define fence_add_callback linux_fence_add_callback #define fence_context_alloc linux_fence_context_alloc +#define fence_destroy linux_fence_destroy #define fence_enable_sw_signaling linux_fence_enable_sw_signaling #define fence_get linux_fence_get #define fence_init linux_fence_init +#define fence_is_signaled linux_fence_is_signaled +#define fence_is_signaled_locked linux_fence_is_signaled_locked #define fence_put linux_fence_put #define fence_remove_callback linux_fence_remove_callback #define fence_signal linux_fence_signal @@ -80,6 +94,7 @@ struct fence_cb { void fence_init(struct fence *, const struct fence_ops *, spinlock_t *, unsigned, unsigned); +void fence_destroy(struct fence *); void fence_free(struct fence *); unsigned @@ -87,6 +102,8 @@ unsigned struct fence * fence_get(struct fence *); +struct fence * + fence_get_rcu(struct fence *); void fence_put(struct fence *); int fence_add_callback(struct fence *, struct fence_cb *, fence_func_t); @@ -99,7 +116,7 @@ int fence_signal(struct fence *); int fence_signal_locked(struct fence *); long fence_default_wait(struct fence *, bool, long); long fence_wait(struct fence *, bool); -long fence_wait_timeout(struct fence *, bool, int); +long fence_wait_timeout(struct fence *, bool, long); static inline void FENCE_TRACE(struct fence *f, const char *fmt, ...) diff --git a/sys/external/bsd/drm2/include/linux/reservation.h b/sys/external/bsd/drm2/include/linux/reservation.h index b74c70be708..53761b23dc1 100644 --- a/sys/external/bsd/drm2/include/linux/reservation.h +++ b/sys/external/bsd/drm2/include/linux/reservation.h @@ -1,7 +1,7 @@ -/* $NetBSD: reservation.h,v 1.4 2018/08/27 07:34:41 riastradh Exp $ */ +/* $NetBSD: reservation.h,v 1.5 2018/08/27 13:33:59 riastradh Exp $ */ /*- - * Copyright (c) 2014 The NetBSD Foundation, Inc. + * Copyright (c) 2018 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation @@ -29,89 +29,61 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _LINUX_RESERVATION_H_ -#define _LINUX_RESERVATION_H_ - -#include <sys/atomic.h> +#ifndef _LINUX_RESERVATION_H_ +#define _LINUX_RESERVATION_H_ #include <linux/rcupdate.h> #include <linux/ww_mutex.h> struct fence; -extern struct ww_class reservation_ww_class; - struct reservation_object { struct ww_mutex lock; - struct reservation_object_list __rcu *robj_objlist; - struct fence __rcu *robj_fence_excl; + unsigned robj_version; + struct fence __rcu *robj_fence; + struct reservation_object_list __rcu *robj_list; + struct reservation_object_list __rcu *robj_prealloc; + uint32_t robj_nreserved; }; struct reservation_object_list { + struct rcu_head rol_rcu; + uint32_t shared_count; + uint32_t shared_max; struct fence __rcu *shared[]; }; #define reservation_object_add_excl_fence linux_reservation_object_add_excl_fence #define reservation_object_add_shared_fence linux_reservation_object_add_shared_fence +#define reservation_object_fini linux_reservation_object_fini +#define reservation_object_get_excl linux_reservation_object_get_excl +#define reservation_object_get_list linux_reservation_object_get_list +#define reservation_object_held linux_reservation_object_held +#define reservation_object_init linux_reservation_object_init #define reservation_object_reserve_shared linux_reservation_object_reserve_shared #define reservation_object_test_signaled_rcu linux_reservation_object_test_signaled_rcu #define reservation_object_wait_timeout_rcu linux_reservation_object_wait_timeout_rcu +#define reservation_ww_class linux_reservation_ww_class + +extern struct ww_class reservation_ww_class; +void reservation_object_init(struct reservation_object *); +void reservation_object_fini(struct reservation_object *); +bool reservation_object_held(struct reservation_object *); +struct fence * + reservation_object_get_excl(struct reservation_object *); +struct reservation_object_list * + reservation_object_get_list(struct reservation_object *); +int reservation_object_reserve_shared(struct reservation_object *); void reservation_object_add_excl_fence(struct reservation_object *, struct fence *); void reservation_object_add_shared_fence(struct reservation_object *, struct fence *); -int reservation_object_reserve_shared(struct reservation_object *); bool reservation_object_test_signaled_rcu(struct reservation_object *, bool); long reservation_object_wait_timeout_rcu(struct reservation_object *, bool, bool, unsigned long); -static inline void -reservation_object_init(struct reservation_object *reservation) -{ - - ww_mutex_init(&reservation->lock, &reservation_ww_class); -} - -static inline void -reservation_object_fini(struct reservation_object *reservation) -{ - - ww_mutex_destroy(&reservation->lock); -} - -static inline bool -reservation_object_held(struct reservation_object *reservation) -{ - - return ww_mutex_is_locked(&reservation->lock); -} - -static inline struct fence * -reservation_object_get_excl(struct reservation_object *reservation) -{ - struct fence *fence; - - KASSERT(reservation_object_held(reservation)); - fence = reservation->robj_fence_excl; - membar_datadep_consumer(); - - return fence; -} - -static inline struct reservation_object_list * -reservation_object_get_list(struct reservation_object *reservation) -{ - struct reservation_object_list *objlist; - - KASSERT(reservation_object_held(reservation)); - objlist = reservation->robj_objlist; - membar_datadep_consumer(); - - return objlist; -} - -#endif /* _LINUX_RESERVATION_H_ */ +#endif /* _LINUX_RESERVATION_H_ */ diff --git a/sys/external/bsd/drm2/linux/files.drmkms_linux b/sys/external/bsd/drm2/linux/files.drmkms_linux index 0cd6669b56d..1e5cfe3ff4d 100644 --- a/sys/external/bsd/drm2/linux/files.drmkms_linux +++ b/sys/external/bsd/drm2/linux/files.drmkms_linux @@ -1,4 +1,4 @@ -# $NetBSD: files.drmkms_linux,v 1.12 2018/08/27 13:31:37 riastradh Exp $ +# $NetBSD: files.drmkms_linux,v 1.13 2018/08/27 13:33:59 riastradh Exp $ define drmkms_linux: i2cexec, i2c_bitbang @@ -6,11 +6,13 @@ makeoptions drmkms_linux CPPFLAGS+="-I$S/external/bsd/drm2/include" makeoptions drmkms_linux CPPFLAGS+="-I$S/external/bsd/common/include" file external/bsd/drm2/linux/linux_dmi.c drmkms_linux +file external/bsd/drm2/linux/linux_fence.c drmkms_linux file external/bsd/drm2/linux/linux_i2c.c drmkms_linux file external/bsd/drm2/linux/linux_idr.c drmkms_linux file external/bsd/drm2/linux/linux_kmap.c drmkms_linux file external/bsd/drm2/linux/linux_list_sort.c drmkms_linux file external/bsd/drm2/linux/linux_module.c drmkms_linux file external/bsd/drm2/linux/linux_rcu.c drmkms_linux +file external/bsd/drm2/linux/linux_reservation.c drmkms_linux file external/bsd/drm2/linux/linux_writecomb.c drmkms_linux file external/bsd/drm2/linux/linux_ww_mutex.c drmkms_linux diff --git a/sys/external/bsd/drm2/linux/linux_fence.c b/sys/external/bsd/drm2/linux/linux_fence.c new file mode 100644 index 00000000000..72d93bba432 --- /dev/null +++ b/sys/external/bsd/drm2/linux/linux_fence.c @@ -0,0 +1,537 @@ +/* $NetBSD: linux_fence.c,v 1.1 2018/08/27 13:33:59 riastradh Exp $ */ + +/*- + * Copyright (c) 2018 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Taylor R. Campbell. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__KERNEL_RCSID(0, "$NetBSD: linux_fence.c,v 1.1 2018/08/27 13:33:59 riastradh Exp $"); + +#include <sys/atomic.h> +#include <sys/condvar.h> +#include <sys/queue.h> + +#include <linux/errno.h> +#include <linux/kref.h> +#include <linux/fence.h> +#include <linux/sched.h> +#include <linux/spinlock.h> + +/* + * fence_init(fence, ops, lock, context, seqno) + * + * Initialize fence. Caller should call fence_destroy when done, + * after all references have been released. + */ +void +fence_init(struct fence *fence, const struct fence_ops *ops, spinlock_t *lock, + unsigned context, unsigned seqno) +{ + + kref_init(&fence->refcount); + fence->lock = lock; + fence->flags = 0; + fence->context = context; + fence->seqno = seqno; + fence->ops = ops; + TAILQ_INIT(&fence->f_callbacks); + cv_init(&fence->f_cv, "fence"); +} + +/* + * fence_destroy(fence) + * + * Clean up memory initialized with fence_init. This is meant to + * be used after a fence release callback. + */ +void +fence_destroy(struct fence *fence) +{ + + KASSERT(TAILQ_EMPTY(&fence->f_callbacks)); + cv_destroy(&fence->f_cv); +} + +static void +fence_free_cb(struct rcu_head *rcu) +{ + struct fence *fence = container_of(rcu, struct fence, f_rcu); + + fence_destroy(fence); + kfree(fence); +} + +/* + * fence_free(fence) + * + * Schedule fence to be destroyed and then freed with kfree after + * any pending RCU read sections on all CPUs have completed. + * Caller must guarantee all references have been released. This + * is meant to be used after a fence release callback. + * + * NOTE: Callers assume kfree will be used. We don't even use + * kmalloc to allocate these -- caller is expected to allocate + * memory with kmalloc to be initialized with fence_init. + */ +void +fence_free(struct fence *fence) +{ + + call_rcu(&fence->f_rcu, &fence_free_cb); +} + +/* + * fence_context_alloc(n) + * + * Return the first of a contiguous sequence of unique + * identifiers, at least until the system wraps around. + */ +unsigned +fence_context_alloc(unsigned n) +{ + static volatile unsigned next_context = 0; + + return atomic_add_int_nv(&next_context, n) - n; +} + +/* + * fence_get(fence) + * + * Acquire a reference to fence. The fence must not be being + * destroyed. Return the fence. + */ +struct fence * +fence_get(struct fence *fence) +{ + + kref_get(&fence->refcount); + return fence; +} + +/* + * fence_get_rcu(fence) + * + * Attempt to acquire a reference to a fence that may be about to + * be destroyed, during a read section. Return the fence on + * success, or NULL on failure. + */ +struct fence * +fence_get_rcu(struct fence *fence) +{ + + if (!kref_get_unless_zero(&fence->refcount)) + return NULL; + return fence; +} + +static void +fence_release(struct kref *refcount) +{ + struct fence *fence = container_of(refcount, struct fence, refcount); + + (*fence->ops->release)(fence); +} + +/* + * fence_put(fence) + * + * Release a reference to fence. If this was the last one, call + * the fence's release callback. + */ +void +fence_put(struct fence *fence) +{ + + kref_put(&fence->refcount, &fence_release); +} + +/* + * fence_ensure_signal_enabled(fence) + * + * Internal subroutine. If the fence was already signalled, + * return -ENOENT. Otherwise, if the enable signalling callback + * has not been called yet, call it. If fails, signal the fence + * and return -ENOENT. If it succeeds, or if it had already been + * called, return zero to indicate success. + * + * Caller must hold the fence's lock. + */ +static int +fence_ensure_signal_enabled(struct fence *fence) +{ + + KASSERT(spin_is_locked(fence->lock)); + + /* If the fence was already signalled, fail with -ENOENT. */ + if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT)) + return -ENOENT; + + /* If the enable signalling callback has been called, success. */ + if (fence->flags & (1u << FENCE_FLAG_ENABLE_SIGNAL_BIT)) + return 0; + + /* Otherwise, note that we've called it and call it. */ + fence->flags |= 1u << FENCE_FLAG_ENABLE_SIGNAL_BIT; + if (!(*fence->ops->enable_signaling)(fence)) { + /* If it failed, signal and return -ENOENT. */ + fence_signal_locked(fence); + return -ENOENT; + } + + /* Success! */ + return 0; +} + +/* + * fence_add_callback(fence, fcb, fn) + * + * If fence has been signalled, return -ENOENT. If the enable + * signalling callback hasn't been called yet, call it; if it + * fails, return -ENOENT. Otherwise, arrange to call fn(fence, + * fcb) when it is signalled, and return 0. + * + * The fence uses memory allocated by the caller in fcb from the + * time of fence_add_callback either to the time of + * fence_remove_callback, or just before calling fn. + */ +int +fence_add_callback(struct fence *fence, struct fence_cb *fcb, fence_func_t fn) +{ + int ret; + + /* Optimistically try to skip the lock if it's already signalled. */ + if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT)) { + ret = -ENOENT; + goto out0; + } + + /* Acquire the lock. */ + spin_lock(fence->lock); + + /* Ensure signalling is enabled, or fail if we can't. */ + ret = fence_ensure_signal_enabled(fence); + if (ret) + goto out1; + + /* Insert the callback. */ + fcb->fcb_func = fn; + TAILQ_INSERT_TAIL(&fence->f_callbacks, fcb, fcb_entry); + fcb->fcb_onqueue = true; + + /* Release the lock and we're done. */ +out1: spin_unlock(fence->lock); +out0: return ret; +} + +/* + * fence_remove_callback(fence, fcb) + * + * Remove the callback fcb from fence. Return true if it was + * removed from the list, or false if it had already run and so + * was no longer queued anyway. Caller must have already called + * fence_add_callback(fence, fcb). + */ +bool +fence_remove_callback(struct fence *fence, struct fence_cb *fcb) +{ + bool onqueue; + + spin_lock(fence->lock); + onqueue = fcb->fcb_onqueue; + if (onqueue) { + TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry); + fcb->fcb_onqueue = false; + } + spin_unlock(fence->lock); + + return onqueue; +} + +/* + * fence_enable_sw_signaling(fence) + * + * If it hasn't been called yet and the fence hasn't been + * signalled yet, call the fence's enable_sw_signaling callback. + * If when that happens, the callback indicates failure by + * returning false, signal the fence. + */ +void +fence_enable_sw_signaling(struct fence *fence) +{ + + spin_lock(fence->lock); + (void)fence_ensure_signal_enabled(fence); + spin_unlock(fence->lock); +} + +/* + * fence_is_signaled(fence) + * + * Test whether the fence has been signalled. If it has been + * signalled by fence_signal(_locked), return true. If the + * signalled callback returns true indicating that some implicit + * external condition has changed, call the callbacks as if with + * fence_signal. + */ +bool +fence_is_signaled(struct fence *fence) +{ + bool signaled; + + spin_lock(fence->lock); + signaled = fence_is_signaled_locked(fence); + spin_unlock(fence->lock); + + return signaled; +} + +/* + * fence_is_signaled_locked(fence) + * + * Test whether the fence has been signalled. Like + * fence_is_signaleed, but caller already holds the fence's lock. + */ +bool +fence_is_signaled_locked(struct fence *fence) +{ + + KASSERT(spin_is_locked(fence->lock)); + + /* Check whether we already set the signalled bit. */ + if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT)) + return true; + + /* If there's a signalled callback, test it. */ + if (fence->ops->signaled) { + if ((*fence->ops->signaled)(fence)) { + /* + * It's been signalled implicitly by some + * external phenomonen. Act as though someone + * has called fence_signal. + */ + fence_signal_locked(fence); + return true; + } + } + + return false; +} + +/* + * fence_signal(fence) + * + * Signal the fence. If it has already been signalled, return + * -EINVAL. If it has not been signalled, call the enable + * signalling callback if it hasn't been called yet, and remove + * each registered callback from the queue and call it; then + * return 0. + */ +int +fence_signal(struct fence *fence) +{ + int ret; + + spin_lock(fence->lock); + ret = fence_signal_locked(fence); + spin_unlock(fence->lock); + + return ret; +} + +/* + * fence_signal_locked(fence) + * + * Signal the fence. Like fence_signal, but caller already holds + * the fence's lock. + */ +int +fence_signal_locked(struct fence *fence) +{ + struct fence_cb *fcb, *next; + + KASSERT(spin_is_locked(fence->lock)); + + /* If it's been signalled, fail; otherwise set the signalled bit. */ + if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT)) + return -EINVAL; + fence->flags |= 1u << FENCE_FLAG_SIGNALED_BIT; + + /* Wake waiters. */ + cv_broadcast(&fence->f_cv); + + /* Remove and call the callbacks. */ + TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) { + TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry); + fcb->fcb_onqueue = false; + (*fcb->fcb_func)(fence, fcb); + } + + /* Success! */ + return 0; +} + +/* + * fence_wait_timeout(fence, intr, timeout) + * + * Wait until fence is signalled; or until interrupt, if intr is + * true; or until timeout, if positive. Return -ERESTARTSYS if + * interrupted, negative error code on any other error, zero on + * timeout, or positive number of ticks remaining if the fence is + * signalled before the timeout. Works by calling the fence wait + * callback. + * + * The timeout must be nonnegative and less than + * MAX_SCHEDULE_TIMEOUT. + */ +long +fence_wait_timeout(struct fence *fence, bool intr, long timeout) +{ + + KASSERT(timeout >= 0); + KASSERT(timeout < MAX_SCHEDULE_TIMEOUT); + return (*fence->ops->wait)(fence, intr, timeout); +} + +/* + * fence_wait_timeout(fence, intr, timeout) + * + * Wait until fence is signalled; or until interrupt, if intr is + * true. Return -ERESTARTSYS if interrupted, negative error code + * on any other error, zero on sucess. Works by calling the fence + * wait callback with MAX_SCHEDULE_TIMEOUT. + */ +long +fence_wait(struct fence *fence, bool intr) +{ + long ret; + + ret = (*fence->ops->wait)(fence, intr, MAX_SCHEDULE_TIMEOUT); + KASSERT(ret != 0); + + return (ret < 0 ? ret : 0); +} + +/* + * fence_default_wait(fence, intr, timeout) + * + * Default implementation of fence wait callback using a condition + * variable. If the fence is already signalled, return timeout, + * or 0 if no timeout. If the enable signalling callback hasn't + * been called, call it, and if it fails, act as if the fence had + * been signalled. Otherwise, wait on the internal condvar. If + * timeout is MAX_SCHEDULE_TIMEOUT, treat it as no timeout. + */ +long +fence_default_wait(struct fence *fence, bool intr, long timeout) +{ + int starttime = 0, now = 0, deadline = 0; /* XXXGCC */ + kmutex_t *lock = &fence->lock->sl_lock; + long ret = 0; + + KASSERT(timeout >= 0); + KASSERT(timeout <= MAX_SCHEDULE_TIMEOUT); + + /* Optimistically try to skip the lock if it's already signalled. */ + if (fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT)) + return (timeout < MAX_SCHEDULE_TIMEOUT ? timeout : 0); + + /* Acquire the lock. */ + spin_lock(fence->lock); + + /* Ensure signalling is enabled, or fail if we can't. */ + ret = fence_ensure_signal_enabled(fence); + if (ret) + goto out; + + /* Find out what our deadline is so we can handle spurious wakeup. */ + if (timeout < MAX_SCHEDULE_TIMEOUT) { + now = hardclock_ticks; + starttime = now; + deadline = starttime + timeout; + } + + /* Wait until the signalled bit is set. */ + while (!(fence->flags & (1u << FENCE_FLAG_SIGNALED_BIT))) { + /* + * If there's a timeout and we've passed the deadline, + * give up. + */ + if (timeout < MAX_SCHEDULE_TIMEOUT) { + now = hardclock_ticks; + if (deadline <= now) + break; + } + if (intr) { + if (timeout < MAX_SCHEDULE_TIMEOUT) { + ret = -cv_timedwait_sig(&fence->f_cv, lock, + deadline - now); + } else { + ret = -cv_wait_sig(&fence->f_cv, lock); + } + } else { + if (timeout < MAX_SCHEDULE_TIMEOUT) { + ret = -cv_timedwait(&fence->f_cv, lock, + deadline - now); + } else { + cv_wait(&fence->f_cv, lock); + ret = 0; + } + } + /* If the wait failed, give up. */ + if (ret) + break; + } + +out: + /* All done. Release the lock. */ + spin_unlock(fence->lock); + + /* If cv_timedwait gave up, return 0 meaning timeout. */ + if (ret == -EWOULDBLOCK) + return 0; + + /* If there was a timeout and the deadline passed, return 0. */ + if (timeout < MAX_SCHEDULE_TIMEOUT) { + if (deadline <= now) + return 0; + } + + /* If we were interrupted, return -ERESTARTSYS. */ + if (ret == -EINTR || ret == -ERESTART) + return -ERESTARTSYS; + + /* If there was any other kind of error, fail. */ + if (ret) + return ret; + + /* + * Success! Return the number of ticks left, at least 1, or 0 + * if no timeout. + */ + return (timeout < MAX_SCHEDULE_TIMEOUT ? MIN(deadline - now, 1) : 0); +} diff --git a/sys/external/bsd/drm2/linux/linux_module.c b/sys/external/bsd/drm2/linux/linux_module.c index cccdd98566e..398f6f28c83 100644 --- a/sys/external/bsd/drm2/linux/linux_module.c +++ b/sys/external/bsd/drm2/linux/linux_module.c @@ -1,4 +1,4 @@ -/* $NetBSD: linux_module.c,v 1.7 2018/08/27 13:31:37 riastradh Exp $ */ +/* $NetBSD: linux_module.c,v 1.8 2018/08/27 13:33:59 riastradh Exp $ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: linux_module.c,v 1.7 2018/08/27 13:31:37 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: linux_module.c,v 1.8 2018/08/27 13:33:59 riastradh Exp $"); #include <sys/module.h> #ifndef _MODULE @@ -42,13 +42,10 @@ __KERNEL_RCSID(0, "$NetBSD: linux_module.c,v 1.7 2018/08/27 13:31:37 riastradh E #include <linux/io.h> #include <linux/mutex.h> #include <linux/rcupdate.h> -#include <linux/reservation.h> #include <linux/workqueue.h> MODULE(MODULE_CLASS_MISC, drmkms_linux, "i2cexec"); -DEFINE_WW_CLASS(reservation_ww_class __cacheline_aligned); - static int linux_init(void) { diff --git a/sys/external/bsd/drm2/linux/linux_reservation.c b/sys/external/bsd/drm2/linux/linux_reservation.c new file mode 100644 index 00000000000..8d166830177 --- /dev/null +++ b/sys/external/bsd/drm2/linux/linux_reservation.c @@ -0,0 +1,638 @@ +/* $NetBSD: linux_reservation.c,v 1.1 2018/08/27 13:33:59 riastradh Exp $ */ + +/*- + * Copyright (c) 2018 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Taylor R. Campbell. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__KERNEL_RCSID(0, "$NetBSD: linux_reservation.c,v 1.1 2018/08/27 13:33:59 riastradh Exp $"); + +#include <linux/fence.h> +#include <linux/reservation.h> + +DEFINE_WW_CLASS(reservation_ww_class __cacheline_aligned); + +static struct reservation_object_list * +objlist_tryalloc(uint32_t n) +{ + struct reservation_object_list *list; + + list = kmem_alloc(offsetof(typeof(*list), shared[n]), KM_NOSLEEP); + if (list == NULL) + return NULL; + list->shared_max = n; + + return list; +} + +static void +objlist_free(struct reservation_object_list *list) +{ + uint32_t n = list->shared_max; + + kmem_free(list, offsetof(typeof(*list), shared[n])); +} + +static void +objlist_free_cb(struct rcu_head *rcu) +{ + struct reservation_object_list *list = container_of(rcu, + struct reservation_object_list, rol_rcu); + + objlist_free(list); +} + +static void +objlist_defer_free(struct reservation_object_list *list) +{ + + call_rcu(&list->rol_rcu, objlist_free_cb); +} + +/* + * reservation_object_init(robj) + * + * Initialize a reservation object. Caller must later destroy it + * with reservation_object_fini. + */ +void +reservation_object_init(struct reservation_object *robj) +{ + + ww_mutex_init(&robj->lock, &reservation_ww_class); + robj->robj_version = 0; + robj->robj_fence = NULL; + robj->robj_list = NULL; + robj->robj_prealloc = NULL; + robj->robj_nreserved = 0; +} + +/* + * reservation_object_fini(robj) + * + * Destroy a reservation object, freeing any memory that had been + * allocated for it. Caller must have exclusive access to it. + */ +void +reservation_object_fini(struct reservation_object *robj) +{ + unsigned i; + + if (robj->robj_prealloc) + objlist_free(robj->robj_prealloc); + if (robj->robj_list) { + for (i = 0; i < robj->robj_list->shared_count; i++) + fence_put(robj->robj_list->shared[i]); + objlist_free(robj->robj_list); + } + if (robj->robj_fence) + fence_put(robj->robj_fence); + ww_mutex_destroy(&robj->lock); +} + +/* + * reservation_object_held(roj) + * + * True if robj is locked. + */ +bool +reservation_object_held(struct reservation_object *robj) +{ + + return ww_mutex_is_locked(&robj->lock); +} + +/* + * reservation_object_get_excl(robj) + * + * Return a pointer to the exclusive fence of the reservation + * object robj. + * + * Caller must have robj locked. + */ +struct fence * +reservation_object_get_excl(struct reservation_object *robj) +{ + + KASSERT(reservation_object_held(robj)); + return robj->robj_fence; +} + +/* + * reservation_object_get_list(robj) + * + * Return a pointer to the shared fence list of the reservation + * object robj. + * + * Caller must have robj locked. + */ +struct reservation_object_list * +reservation_object_get_list(struct reservation_object *robj) +{ + + KASSERT(reservation_object_held(robj)); + return robj->robj_list; +} + +/* + * reservation_object_reserve_shared(robj) + * + * Reserve space in robj to add a shared fence. + * + * Caller must have robj locked. + * + * Internally, we start with room for four entries and double + * whenever we run out. This is not guaranteed. + */ +int +reservation_object_reserve_shared(struct reservation_object *robj) +{ + struct reservation_object_list *list, *prealloc; + uint32_t n, nalloc; + + KASSERT(reservation_object_held(robj)); + + list = robj->robj_list; + prealloc = robj->robj_prealloc; + + /* If there's too many objects reserved already, fail. */ + if (robj->robj_nreserved == UINT32_MAX) + return -ENOMEM; + + /* Bump the number of objects reserved. */ + n = ++robj->robj_nreserved; + + /* If there's enough room in the existing list, stop here. */ + if (list != NULL && n < list->shared_max) + return 0; + + /* If not, maybe there's a preallocated list ready. */ + if (prealloc != NULL) { + /* If there's enough room in it, stop here. */ + if (n < prealloc->shared_max) + return 0; + + /* Try to double its capacity. */ + nalloc = n > UINT32_MAX/2 ? UINT32_MAX : 2*n; + prealloc = objlist_tryalloc(nalloc); + if (prealloc == NULL) + return -ENOMEM; + + /* Swap the new preallocated list and free the old one. */ + objlist_free(robj->robj_prealloc); + robj->robj_prealloc = prealloc; + } else { + /* Start with some spare. */ + nalloc = n > UINT32_MAX/2 ? UINT32_MAX : MAX(2*n, 4); + prealloc = objlist_tryalloc(nalloc); + if (prealloc == NULL) + return -ENOMEM; + /* Save the new preallocated list. */ + robj->robj_prealloc = prealloc; + } + + /* Success! */ + return 0; +} + +struct reservation_object_write_ticket { + unsigned version; +}; + +/* + * reservation_object_write_begin(robj, ticket) + * + * Begin an atomic batch of writes to robj, and initialize opaque + * ticket for it. The ticket must be passed to + * reservation_object_write_commit to commit the writes. + * + * Caller must have robj locked. + */ +static void +reservation_object_write_begin(struct reservation_object *robj, + struct reservation_object_write_ticket *ticket) +{ + + KASSERT(reservation_object_held(robj)); + + ticket->version = robj->robj_version |= 1; + membar_producer(); +} + +/* + * reservation_object_write_commit(robj, ticket) + * + * Commit an atomic batch of writes to robj begun with the call to + * reservation_object_write_begin that returned ticket. + * + * Caller must have robj locked. + */ +static void +reservation_object_write_commit(struct reservation_object *robj, + struct reservation_object_write_ticket *ticket) +{ + + KASSERT(reservation_object_held(robj)); + KASSERT(ticket->version == robj->robj_version); + KASSERT((ticket->version & 1) == 1); + + membar_producer(); + robj->robj_version = ticket->version + 1; +} + +struct reservation_object_read_ticket { + unsigned version; +}; + +/* + * reservation_object_read_begin(robj, ticket) + * + * Begin a read section, and initialize opaque ticket for it. The + * ticket must be passed to reservation_object_read_exit, and the + * caller must be prepared to retry reading if it fails. + */ +static void +reservation_object_read_begin(struct reservation_object *robj, + struct reservation_object_read_ticket *ticket) +{ + + ticket->version = robj->robj_version; + membar_consumer(); +} + +/* + * reservation_object_read_valid(robj, ticket) + * + * Test whether the read sections are valid. Return true on + * success, or false on failure if the read ticket has been + * invalidated. + */ +static bool +reservation_object_read_valid(struct reservation_object *robj, + struct reservation_object_read_ticket *ticket) +{ + + membar_consumer(); + return ticket->version == robj->robj_version; +} + +/* + * reservation_object_add_excl_fence(robj, fence) + * + * Replace robj's exclusive fence, if any, by fence, and empty its + * list of shared fences. The old exclusive fence and all old + * shared fences are released. + * + * Caller must have robj locked. + */ +void +reservation_object_add_excl_fence(struct reservation_object *robj, + struct fence *fence) +{ + struct fence *old_fence = robj->robj_fence; + struct reservation_object_list *old_list = robj->robj_list; + uint32_t old_shared_count; + struct reservation_object_write_ticket ticket; + + KASSERT(reservation_object_held(robj)); + + /* If there are any shared fences, remember how many. */ + if (old_list) + old_shared_count = old_list->shared_count; + + /* Begin an update. */ + reservation_object_write_begin(robj, &ticket); + + /* Replace the fence and zero the shared count. */ + robj->robj_fence = fence; + if (old_list) + old_list->shared_count = 0; + + /* Commit the update. */ + reservation_object_write_commit(robj, &ticket); + + /* Release the old exclusive fence, if any. */ + if (old_fence) + fence_put(old_fence); + + /* Release any old shared fences. */ + if (old_list) { + while (old_shared_count--) + fence_put(old_list->shared[old_shared_count]); + } +} + +/* + * reservation_object_add_shared_fence(robj, fence) + * + * Add a shared fence to robj. If any fence was already added + * with the same context number, release it and replace it by this + * one. + * + * Caller must have robj locked, and must have preceded with a + * call to reservation_object_reserve_shared for each shared fence + * added. + */ +void +reservation_object_add_shared_fence(struct reservation_object *robj, + struct fence *fence) +{ + struct reservation_object_list *list = robj->robj_list; + struct reservation_object_list *prealloc = robj->robj_prealloc; + struct reservation_object_write_ticket ticket; + struct fence *replace = NULL; + uint32_t i; + + KASSERT(reservation_object_held(robj)); + + /* Check for a preallocated replacement list. */ + if (prealloc == NULL) { + KASSERT(list->shared_count < list->shared_max); + + /* Begin an update. */ + reservation_object_write_begin(robj, &ticket); + + /* Find a fence with the same context number. */ + for (i = 0; i < list->shared_count; i++) { + if (list->shared[i]->context == fence->context) { + replace = list->shared[i]; + list->shared[i] = fence; + break; + } + } + + /* If we didn't find one, add it at the end. */ + if (i == list->shared_count) + list->shared[list->shared_count++] = fence; + + /* Commit the update. */ + reservation_object_write_commit(robj, &ticket); + } else { + KASSERT(list->shared_count < prealloc->shared_max); + + /* + * Copy the fences over, but replace if we find one + * with the same context number. + */ + for (i = 0; i < list->shared_count; i++) { + if (replace == NULL && + list->shared[i]->context == fence->context) { + replace = list->shared[i]; + prealloc->shared[i] = fence; + } else { + prealloc->shared[i] = list->shared[i]; + } + } + prealloc->shared_count = list->shared_count; + + /* If we didn't find one, add it at the end. */ + if (replace == NULL) + prealloc->shared[prealloc->shared_count++] = fence; + + /* Now ready to replace the list. Begin an update. */ + reservation_object_write_begin(robj, &ticket); + + /* Replace the list. */ + robj->robj_list = prealloc; + robj->robj_prealloc = NULL; + + /* Commit the update. */ + reservation_object_write_commit(robj, &ticket); + + /* + * Free the old list when it is convenient. (We are + * not in a position at this point to sleep waiting for + * activity on all CPUs.) + */ + objlist_defer_free(list); + } + + /* Release a fence if we replaced it. */ + if (replace) + fence_put(replace); +} + +/* + * reservation_object_test_signaled_rcu(robj, shared) + * + * If shared is true, test whether all of the shared fences are + * signalled, or if there are none, test whether the exclusive + * fence is signalled. If shared is false, test only whether the + * exclusive fence is signalled. + * + * XXX Why does this _not_ test the exclusive fence if shared is + * true only if there are no shared fences? This makes no sense. + */ +bool +reservation_object_test_signaled_rcu(struct reservation_object *robj, + bool shared) +{ + struct reservation_object_read_ticket ticket; + struct reservation_object_list *list; + struct fence *fence; + uint32_t i, shared_count; + bool signaled = true; + +top: + /* Enter an RCU read section and get a read ticket. */ + rcu_read_lock(); + reservation_object_read_begin(robj, &ticket); + + /* If shared is requested and there is a shared list, test it. */ + if (shared && (list = robj->robj_list) != NULL) { + /* Make sure the content of the list has been published. */ + membar_datadep_consumer(); + + /* Find out how long it is. */ + shared_count = list->shared_count; + + /* + * Make sure we saw a consistent snapshot of the list + * pointer and length. + */ + if (!reservation_object_read_valid(robj, &ticket)) + goto restart; + + /* + * For each fence, if it is going away, restart. + * Otherwise, acquire a reference to it to test whether + * it is signalled. Stop if we find any that is not + * signalled. + */ + for (i = 0; i < shared_count; i++) { + fence = fence_get_rcu(list->shared[i]); + if (fence == NULL) + goto restart; + signaled &= fence_is_signaled(fence); + fence_put(fence); + if (!signaled) + goto out; + } + } + + /* If there is an exclusive fence, test it. */ + if ((fence = robj->robj_fence) != NULL) { + /* Make sure the content of the fence has been published. */ + membar_datadep_consumer(); + + /* + * Make sure we saw a consistent snapshot of the fence. + * + * XXX I'm not actually sure this is necessary since + * pointer writes are supposed to be atomic. + */ + if (!reservation_object_read_valid(robj, &ticket)) + goto restart; + + /* + * If it is going away, restart. Otherwise, acquire a + * reference to it to test whether it is signalled. + */ + if ((fence = fence_get_rcu(fence)) == NULL) + goto restart; + signaled &= fence_is_signaled(fence); + fence_put(fence); + if (!signaled) + goto out; + } + +out: rcu_read_unlock(); + return signaled; + +restart: + rcu_read_unlock(); + goto top; +} + +/* + * reservation_object_wait_timeout_rcu(robj, shared, intr, timeout) + * + * If shared is true, wait for all of the shared fences to be + * signalled, or if there are none, wait for the exclusive fence + * to be signalled. If shared is false, wait only for the + * exclusive fence to be signalled. If timeout is zero, don't + * wait, only test. + * + * XXX Why does this _not_ wait for the exclusive fence if shared + * is true only if there are no shared fences? This makes no + * sense. + */ +long +reservation_object_wait_timeout_rcu(struct reservation_object *robj, + bool shared, bool intr, unsigned long timeout) +{ + struct reservation_object_read_ticket ticket; + struct reservation_object_list *list; + struct fence *fence; + uint32_t i, shared_count; + + if (timeout == 0) + return reservation_object_test_signaled_rcu(robj, shared); + +top: + /* Enter an RCU read section and get a read ticket. */ + rcu_read_lock(); + reservation_object_read_begin(robj, &ticket); + + /* If shared is requested and there is a shared list, wait on it. */ + if (shared && (list = robj->robj_list) != NULL) { + /* Make sure the content of the list has been published. */ + membar_datadep_consumer(); + + /* Find out how long it is. */ + shared_count = list->shared_count; + + /* + * Make sure we saw a consistent snapshot of the list + * pointer and length. + */ + if (!reservation_object_read_valid(robj, &ticket)) + goto restart; + + /* + * For each fence, if it is going away, restart. + * Otherwise, acquire a reference to it to test whether + * it is signalled. Stop and wait if we find any that + * is not signalled. + */ + for (i = 0; i < shared_count; i++) { + fence = fence_get_rcu(list->shared[i]); + if (fence == NULL) + goto restart; + if (!fence_is_signaled(fence)) + goto wait; + fence_put(fence); + } + } + + /* If there is an exclusive fence, test it. */ + if ((fence = robj->robj_fence) != NULL) { + /* Make sure the content of the fence has been published. */ + membar_datadep_consumer(); + + /* + * Make sure we saw a consistent snapshot of the fence. + * + * XXX I'm not actually sure this is necessary since + * pointer writes are supposed to be atomic. + */ + if (!reservation_object_read_valid(robj, &ticket)) + goto restart; + + /* + * If it is going away, restart. Otherwise, acquire a + * reference to it to test whether it is signalled. If + * not, wait for it. + */ + if ((fence = fence_get_rcu(fence)) == NULL) + goto restart; + if (!fence_is_signaled(fence)) + goto wait; + fence_put(fence); + } + + /* Success! Return the number of ticks left. */ + rcu_read_unlock(); + return timeout; + +restart: + rcu_read_unlock(); + goto top; + +wait: + /* + * Exit the RCU read section and wait for it. If we time out + * or fail, bail. Otherwise, go back to the top. + */ + KASSERT(fence != NULL); + rcu_read_unlock(); + timeout = fence_wait_timeout(fence, intr, timeout); + if (timeout <= 0) + return timeout; + goto top; +} |
