From 5352ebf735064373dc2dd96b39c0fbf347db0095 Mon Sep 17 00:00:00 2001 From: Siddh Raman Pant Date: Mon, 27 Jun 2022 13:21:48 +0530 Subject: [PATCH 01/46] tools/testing/crypto: Use vzalloc instead of vmalloc+memset This fixes the corresponding coccinelle warning. Signed-off-by: Siddh Raman Pant Signed-off-by: Harald Freudenberger Link: https://lore.kernel.org/r/20220627075148.140705-1-code@siddh.me [agordeev@linux.ibm.com added Link] Signed-off-by: Alexander Gordeev --- tools/testing/crypto/chacha20-s390/test-cipher.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/testing/crypto/chacha20-s390/test-cipher.c b/tools/testing/crypto/chacha20-s390/test-cipher.c index 34e8b855266f..8141d45df51a 100644 --- a/tools/testing/crypto/chacha20-s390/test-cipher.c +++ b/tools/testing/crypto/chacha20-s390/test-cipher.c @@ -252,29 +252,26 @@ static int __init chacha_s390_test_init(void) memset(plain, 'a', data_size); get_random_bytes(plain, (data_size > 256 ? 256 : data_size)); - cipher_generic = vmalloc(data_size); + cipher_generic = vzalloc(data_size); if (!cipher_generic) { pr_info("could not allocate cipher_generic buffer\n"); ret = -2; goto out; } - memset(cipher_generic, 0, data_size); - cipher_s390 = vmalloc(data_size); + cipher_s390 = vzalloc(data_size); if (!cipher_s390) { pr_info("could not allocate cipher_s390 buffer\n"); ret = -2; goto out; } - memset(cipher_s390, 0, data_size); - revert = vmalloc(data_size); + revert = vzalloc(data_size); if (!revert) { pr_info("could not allocate revert buffer\n"); ret = -2; goto out; } - memset(revert, 0, data_size); if (debug) print_hex_dump(KERN_INFO, "src: ", DUMP_PREFIX_OFFSET, From 034921cdea1c4c491735c8eb344634c7b899d930 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Thu, 28 Jan 2021 15:53:07 -0500 Subject: [PATCH 02/46] s390/vfio-ap: use new AP bus interface to search for queue devices This patch refactors the vfio_ap device driver to use the AP bus's ap_get_qdev() function to retrieve the vfio_ap_queue struct containing information about a queue that is bound to the vfio_ap device driver. The bus's ap_get_qdev() function retrieves the queue device from a hashtable keyed by APQN. This is much more efficient than looping over the list of devices attached to the AP bus by several orders of magnitude. Signed-off-by: Tony Krowiak Reviewed-by: Halil Pasic Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index a7d2a95796d3..634e23ae8912 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -30,13 +30,6 @@ static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev); static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); static const struct vfio_device_ops vfio_ap_matrix_dev_ops; -static int match_apqn(struct device *dev, const void *data) -{ - struct vfio_ap_queue *q = dev_get_drvdata(dev); - - return (q->apqn == *(int *)(data)) ? 1 : 0; -} - /** * vfio_ap_get_queue - retrieve a queue with a specific APQN from a list * @matrix_mdev: the associated mediated matrix @@ -1286,15 +1279,17 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) static struct vfio_ap_queue *vfio_ap_find_queue(int apqn) { - struct device *dev; + struct ap_queue *queue; struct vfio_ap_queue *q = NULL; - dev = driver_find_device(&matrix_dev->vfio_ap_drv->driver, NULL, - &apqn, match_apqn); - if (dev) { - q = dev_get_drvdata(dev); - put_device(dev); - } + queue = ap_get_qdev(apqn); + if (!queue) + return NULL; + + if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver) + q = dev_get_drvdata(&queue->ap_dev.device); + + put_device(&queue->ap_dev.device); return q; } From 260f3ea141382386e97611e7c2029bc013088ab1 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Wed, 14 Oct 2020 15:34:49 -0400 Subject: [PATCH 03/46] s390/vfio-ap: move probe and remove callbacks to vfio_ap_ops.c Let's move the probe and remove callbacks into the vfio_ap_ops.c file to keep all code related to managing queues in a single file. This way, all functions related to queue management can be removed from the vfio_ap_private.h header file defining the public interfaces for the vfio_ap device driver. Signed-off-by: Tony Krowiak Reviewed-by: Halil Pasic Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_drv.c | 118 +------------------------- drivers/s390/crypto/vfio_ap_ops.c | 99 ++++++++++++++++++++- drivers/s390/crypto/vfio_ap_private.h | 5 +- 3 files changed, 102 insertions(+), 120 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index 4ac9c6521ec1..1ff6e3dbbffe 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -18,9 +18,6 @@ #define VFIO_AP_ROOT_NAME "vfio_ap" #define VFIO_AP_DEV_NAME "matrix" -#define AP_QUEUE_ASSIGNED "assigned" -#define AP_QUEUE_UNASSIGNED "unassigned" -#define AP_QUEUE_IN_USE "in use" MODULE_AUTHOR("IBM Corporation"); MODULE_DESCRIPTION("VFIO AP device driver, Copyright IBM Corp. 2018"); @@ -46,120 +43,9 @@ static struct ap_device_id ap_queue_ids[] = { { /* end of sibling */ }, }; -static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) -{ - struct ap_matrix_mdev *matrix_mdev; - unsigned long apid = AP_QID_CARD(q->apqn); - unsigned long apqi = AP_QID_QUEUE(q->apqn); - - list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { - if (test_bit_inv(apid, matrix_mdev->matrix.apm) && - test_bit_inv(apqi, matrix_mdev->matrix.aqm)) - return matrix_mdev; - } - - return NULL; -} - -static ssize_t status_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - ssize_t nchars = 0; - struct vfio_ap_queue *q; - struct ap_matrix_mdev *matrix_mdev; - struct ap_device *apdev = to_ap_dev(dev); - - mutex_lock(&matrix_dev->lock); - q = dev_get_drvdata(&apdev->device); - matrix_mdev = vfio_ap_mdev_for_queue(q); - - if (matrix_mdev) { - if (matrix_mdev->kvm) - nchars = scnprintf(buf, PAGE_SIZE, "%s\n", - AP_QUEUE_IN_USE); - else - nchars = scnprintf(buf, PAGE_SIZE, "%s\n", - AP_QUEUE_ASSIGNED); - } else { - nchars = scnprintf(buf, PAGE_SIZE, "%s\n", - AP_QUEUE_UNASSIGNED); - } - - mutex_unlock(&matrix_dev->lock); - - return nchars; -} - -static DEVICE_ATTR_RO(status); - -static struct attribute *vfio_queue_attrs[] = { - &dev_attr_status.attr, - NULL, -}; - -static const struct attribute_group vfio_queue_attr_group = { - .attrs = vfio_queue_attrs, -}; - -/** - * vfio_ap_queue_dev_probe: Allocate a vfio_ap_queue structure and associate it - * with the device as driver_data. - * - * @apdev: the AP device being probed - * - * Return: returns 0 if the probe succeeded; otherwise, returns an error if - * storage could not be allocated for a vfio_ap_queue object or the - * sysfs 'status' attribute could not be created for the queue device. - */ -static int vfio_ap_queue_dev_probe(struct ap_device *apdev) -{ - int ret; - struct vfio_ap_queue *q; - - q = kzalloc(sizeof(*q), GFP_KERNEL); - if (!q) - return -ENOMEM; - - mutex_lock(&matrix_dev->lock); - dev_set_drvdata(&apdev->device, q); - q->apqn = to_ap_queue(&apdev->device)->qid; - q->saved_isc = VFIO_AP_ISC_INVALID; - - ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); - if (ret) { - dev_set_drvdata(&apdev->device, NULL); - kfree(q); - } - - mutex_unlock(&matrix_dev->lock); - - return ret; -} - -/** - * vfio_ap_queue_dev_remove: Free the associated vfio_ap_queue structure. - * - * @apdev: the AP device being removed - * - * Takes the matrix lock to avoid actions on this device while doing the remove. - */ -static void vfio_ap_queue_dev_remove(struct ap_device *apdev) -{ - struct vfio_ap_queue *q; - - mutex_lock(&matrix_dev->lock); - sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); - q = dev_get_drvdata(&apdev->device); - vfio_ap_mdev_reset_queue(q, 1); - dev_set_drvdata(&apdev->device, NULL); - kfree(q); - mutex_unlock(&matrix_dev->lock); -} - static struct ap_driver vfio_ap_drv = { - .probe = vfio_ap_queue_dev_probe, - .remove = vfio_ap_queue_dev_remove, + .probe = vfio_ap_mdev_probe_queue, + .remove = vfio_ap_mdev_remove_queue, .ids = ap_queue_ids, }; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 634e23ae8912..32d276c85079 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -26,6 +26,10 @@ #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough" #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device" +#define AP_QUEUE_ASSIGNED "assigned" +#define AP_QUEUE_UNASSIGNED "unassigned" +#define AP_QUEUE_IN_USE "in use" + static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev); static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); static const struct vfio_device_ops vfio_ap_matrix_dev_ops; @@ -1294,8 +1298,8 @@ static struct vfio_ap_queue *vfio_ap_find_queue(int apqn) return q; } -int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, - unsigned int retry) +static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, + unsigned int retry) { struct ap_queue_status status; int ret; @@ -1452,6 +1456,62 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, return ret; } +static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) +{ + struct ap_matrix_mdev *matrix_mdev; + unsigned long apid = AP_QID_CARD(q->apqn); + unsigned long apqi = AP_QID_QUEUE(q->apqn); + + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + if (test_bit_inv(apid, matrix_mdev->matrix.apm) && + test_bit_inv(apqi, matrix_mdev->matrix.aqm)) + return matrix_mdev; + } + + return NULL; +} + +static ssize_t status_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + ssize_t nchars = 0; + struct vfio_ap_queue *q; + struct ap_matrix_mdev *matrix_mdev; + struct ap_device *apdev = to_ap_dev(dev); + + mutex_lock(&matrix_dev->lock); + q = dev_get_drvdata(&apdev->device); + matrix_mdev = vfio_ap_mdev_for_queue(q); + + if (matrix_mdev) { + if (matrix_mdev->kvm) + nchars = scnprintf(buf, PAGE_SIZE, "%s\n", + AP_QUEUE_IN_USE); + else + nchars = scnprintf(buf, PAGE_SIZE, "%s\n", + AP_QUEUE_ASSIGNED); + } else { + nchars = scnprintf(buf, PAGE_SIZE, "%s\n", + AP_QUEUE_UNASSIGNED); + } + + mutex_unlock(&matrix_dev->lock); + + return nchars; +} + +static DEVICE_ATTR_RO(status); + +static struct attribute *vfio_queue_attrs[] = { + &dev_attr_status.attr, + NULL, +}; + +static const struct attribute_group vfio_queue_attr_group = { + .attrs = vfio_queue_attrs, +}; + static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { .open_device = vfio_ap_mdev_open_device, .close_device = vfio_ap_mdev_close_device, @@ -1495,3 +1555,38 @@ void vfio_ap_mdev_unregister(void) mdev_unregister_device(&matrix_dev->device); mdev_unregister_driver(&vfio_ap_matrix_driver); } + +int vfio_ap_mdev_probe_queue(struct ap_device *apdev) +{ + int ret; + struct vfio_ap_queue *q; + + ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); + if (ret) + return ret; + + q = kzalloc(sizeof(*q), GFP_KERNEL); + if (!q) + return -ENOMEM; + + mutex_lock(&matrix_dev->lock); + q->apqn = to_ap_queue(&apdev->device)->qid; + q->saved_isc = VFIO_AP_ISC_INVALID; + dev_set_drvdata(&apdev->device, q); + mutex_unlock(&matrix_dev->lock); + + return 0; +} + +void vfio_ap_mdev_remove_queue(struct ap_device *apdev) +{ + struct vfio_ap_queue *q; + + mutex_lock(&matrix_dev->lock); + sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); + q = dev_get_drvdata(&apdev->device); + vfio_ap_mdev_reset_queue(q, 1); + dev_set_drvdata(&apdev->device, NULL); + kfree(q); + mutex_unlock(&matrix_dev->lock); +} diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index a26efd804d0d..96655f2eb732 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -116,7 +116,8 @@ struct vfio_ap_queue { int vfio_ap_mdev_register(void); void vfio_ap_mdev_unregister(void); -int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, - unsigned int retry); + +int vfio_ap_mdev_probe_queue(struct ap_device *queue); +void vfio_ap_mdev_remove_queue(struct ap_device *queue); #endif /* _VFIO_AP_PRIVATE_H_ */ From 11cb2419fafe67f5ab965e2958475bce43c1529a Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Mon, 1 Feb 2021 11:18:09 -0500 Subject: [PATCH 04/46] s390/vfio-ap: manage link between queue struct and matrix mdev Let's create links between each queue device bound to the vfio_ap device driver and the matrix mdev to which the queue's APQN is assigned. The idea is to facilitate efficient retrieval of the objects representing the queue devices and matrix mdevs as well as to verify that a queue assigned to a matrix mdev is bound to the driver. The links will be created as follows: * When the queue device is probed, if its APQN is assigned to a matrix mdev, the structures representing the queue device and the matrix mdev will be linked. * When an adapter or domain is assigned to a matrix mdev, for each new APQN assigned that references a queue device bound to the vfio_ap device driver, the structures representing the queue device and the matrix mdev will be linked. The links will be removed as follows: * When the queue device is removed, if its APQN is assigned to a matrix mdev, the link from the structure representing the matrix mdev to the structure representing the queue will be removed. Since the storage allocated for the vfio_ap_queue will be freed, there is no need to remove the link to the matrix_mdev to which the queue's APQN is assigned. * When an adapter or domain is unassigned from a matrix mdev, for each APQN unassigned that references a queue device bound to the vfio_ap device driver, the structures representing the queue device and the matrix mdev will be unlinked. * When an mdev is removed, the link from any queues assigned to the mdev to the mdev will be removed. Signed-off-by: Tony Krowiak Reviewed-by: Halil Pasic Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 191 +++++++++++++++++++++----- drivers/s390/crypto/vfio_ap_private.h | 14 ++ 2 files changed, 168 insertions(+), 37 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 32d276c85079..9bed11aa0ea6 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -35,32 +35,27 @@ static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); static const struct vfio_device_ops vfio_ap_matrix_dev_ops; /** - * vfio_ap_get_queue - retrieve a queue with a specific APQN from a list - * @matrix_mdev: the associated mediated matrix - * @apqn: The queue APQN + * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a + * hash table of queues assigned to a matrix mdev + * @matrix_mdev: the matrix mdev + * @apqn: The APQN of a queue device * - * Retrieve a queue with a specific APQN from the list of the - * devices of the vfio_ap_drv. - * Verify that the APID and the APQI are set in the matrix. - * - * Return: the pointer to the associated vfio_ap_queue + * Return: the pointer to the vfio_ap_queue struct representing the queue or + * NULL if the queue is not assigned to @matrix_mdev */ -static struct vfio_ap_queue *vfio_ap_get_queue( +static struct vfio_ap_queue *vfio_ap_mdev_get_queue( struct ap_matrix_mdev *matrix_mdev, int apqn) { struct vfio_ap_queue *q; - if (!test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm)) - return NULL; - if (!test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) - return NULL; + hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode, + apqn) { + if (q && q->apqn == apqn) + return q; + } - q = vfio_ap_find_queue(apqn); - if (q) - q->matrix_mdev = matrix_mdev; - - return q; + return NULL; } /** @@ -177,7 +172,6 @@ static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q) status.response_code); end_free: vfio_ap_free_aqic_resources(q); - q->matrix_mdev = NULL; return status; } @@ -422,7 +416,7 @@ static int handle_pqap(struct kvm_vcpu *vcpu) goto out_unlock; } - q = vfio_ap_get_queue(matrix_mdev, apqn); + q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); if (!q) { VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n", __func__, AP_QID_CARD(apqn), @@ -472,6 +466,8 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) matrix_mdev->mdev = mdev; vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); matrix_mdev->pqap_hook = handle_pqap; + hash_init(matrix_mdev->qtable.queues); + dev_set_drvdata(&mdev->dev, matrix_mdev); mutex_lock(&matrix_dev->lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); mutex_unlock(&matrix_dev->lock); @@ -493,6 +489,55 @@ err_dec_available: return ret; } +static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev, + struct vfio_ap_queue *q) +{ + if (q) { + q->matrix_mdev = matrix_mdev; + hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn); + } +} + +static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn) +{ + struct vfio_ap_queue *q; + + q = vfio_ap_find_queue(apqn); + vfio_ap_mdev_link_queue(matrix_mdev, q); +} + +static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q) +{ + hash_del(&q->mdev_qnode); +} + +static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q) +{ + q->matrix_mdev = NULL; +} + +static void vfio_ap_mdev_unlink_queue(struct vfio_ap_queue *q) +{ + vfio_ap_unlink_queue_fr_mdev(q); + vfio_ap_unlink_mdev_fr_queue(q); +} + +static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev) +{ + struct vfio_ap_queue *q; + unsigned long apid, apqi; + + for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { + for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, + AP_DOMAINS) { + q = vfio_ap_mdev_get_queue(matrix_mdev, + AP_MKQID(apid, apqi)); + if (q) + q->matrix_mdev = NULL; + } + } +} + static void vfio_ap_mdev_remove(struct mdev_device *mdev) { struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev); @@ -501,6 +546,7 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) mutex_lock(&matrix_dev->lock); vfio_ap_mdev_reset_queues(matrix_mdev); + vfio_ap_mdev_unlink_fr_queues(matrix_mdev); list_del(&matrix_mdev->node); mutex_unlock(&matrix_dev->lock); vfio_uninit_group_dev(&matrix_mdev->vdev); @@ -709,6 +755,16 @@ static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *matrix_mdev) return 0; } +static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid) +{ + unsigned long apqi; + + for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) + vfio_ap_mdev_link_apqn(matrix_mdev, + AP_MKQID(apid, apqi)); +} + /** * assign_adapter_store - parses the APID from @buf and sets the * corresponding bit in the mediated matrix device's APM @@ -779,6 +835,7 @@ static ssize_t assign_adapter_store(struct device *dev, if (ret) goto share_err; + vfio_ap_mdev_link_adapter(matrix_mdev, apid); ret = count; goto done; @@ -791,6 +848,20 @@ done: } static DEVICE_ATTR_WO(assign_adapter); +static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid) +{ + unsigned long apqi; + struct vfio_ap_queue *q; + + for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) { + q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); + + if (q) + vfio_ap_mdev_unlink_queue(q); + } +} + /** * unassign_adapter_store - parses the APID from @buf and clears the * corresponding bit in the mediated matrix device's APM @@ -832,6 +903,7 @@ static ssize_t unassign_adapter_store(struct device *dev, } clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); + vfio_ap_mdev_unlink_adapter(matrix_mdev, apid); ret = count; done: mutex_unlock(&matrix_dev->lock); @@ -859,6 +931,16 @@ vfio_ap_mdev_verify_queues_reserved_for_apqi(struct ap_matrix_mdev *matrix_mdev, return 0; } +static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev, + unsigned long apqi) +{ + unsigned long apid; + + for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) + vfio_ap_mdev_link_apqn(matrix_mdev, + AP_MKQID(apid, apqi)); +} + /** * assign_domain_store - parses the APQI from @buf and sets the * corresponding bit in the mediated matrix device's AQM @@ -924,6 +1006,7 @@ static ssize_t assign_domain_store(struct device *dev, if (ret) goto share_err; + vfio_ap_mdev_link_domain(matrix_mdev, apqi); ret = count; goto done; @@ -936,6 +1019,19 @@ done: } static DEVICE_ATTR_WO(assign_domain); +static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, + unsigned long apqi) +{ + unsigned long apid; + struct vfio_ap_queue *q; + + for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { + q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); + + if (q) + vfio_ap_mdev_unlink_queue(q); + } +} /** * unassign_domain_store - parses the APQI from @buf and clears the @@ -978,6 +1074,7 @@ static ssize_t unassign_domain_store(struct device *dev, } clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); + vfio_ap_mdev_unlink_domain(matrix_mdev, apqi); ret = count; done: @@ -1351,25 +1448,18 @@ free_resources: static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev) { - int ret; - int rc = 0; - unsigned long apid, apqi; + int ret, loop_cursor, rc = 0; struct vfio_ap_queue *q; - for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, - matrix_mdev->matrix.apm_max + 1) { - for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, - matrix_mdev->matrix.aqm_max + 1) { - q = vfio_ap_find_queue(AP_MKQID(apid, apqi)); - ret = vfio_ap_mdev_reset_queue(q, 1); - /* - * Regardless whether a queue turns out to be busy, or - * is not operational, we need to continue resetting - * the remaining queues. - */ - if (ret) - rc = ret; - } + hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) { + ret = vfio_ap_mdev_reset_queue(q, 1); + /* + * Regardless whether a queue turns out to be busy, or + * is not operational, we need to continue resetting + * the remaining queues. + */ + if (ret) + rc = ret; } return rc; @@ -1556,6 +1646,28 @@ void vfio_ap_mdev_unregister(void) mdev_unregister_driver(&vfio_ap_matrix_driver); } +/* + * vfio_ap_queue_link_mdev + * + * @q: The queue to link with the matrix mdev. + * + * Links @q with the matrix mdev to which the queue's APQN is assigned. + */ +static void vfio_ap_queue_link_mdev(struct vfio_ap_queue *q) +{ + unsigned long apid = AP_QID_CARD(q->apqn); + unsigned long apqi = AP_QID_QUEUE(q->apqn); + struct ap_matrix_mdev *matrix_mdev; + + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + if (test_bit_inv(apid, matrix_mdev->matrix.apm) && + test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { + vfio_ap_mdev_link_queue(matrix_mdev, q); + break; + } + } +} + int vfio_ap_mdev_probe_queue(struct ap_device *apdev) { int ret; @@ -1572,6 +1684,7 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) mutex_lock(&matrix_dev->lock); q->apqn = to_ap_queue(&apdev->device)->qid; q->saved_isc = VFIO_AP_ISC_INVALID; + vfio_ap_queue_link_mdev(q); dev_set_drvdata(&apdev->device, q); mutex_unlock(&matrix_dev->lock); @@ -1585,6 +1698,10 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) mutex_lock(&matrix_dev->lock); sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); q = dev_get_drvdata(&apdev->device); + + if (q->matrix_mdev) + vfio_ap_unlink_queue_fr_mdev(q); + vfio_ap_mdev_reset_queue(q, 1); dev_set_drvdata(&apdev->device, NULL); kfree(q); diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 96655f2eb732..44d2eeb795a0 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "ap_bus.h" @@ -74,6 +75,15 @@ struct ap_matrix { DECLARE_BITMAP(adm, 256); }; +/** + * struct ap_queue_table - a table of queue objects. + * + * @queues: a hashtable of queues (struct vfio_ap_queue). + */ +struct ap_queue_table { + DECLARE_HASHTABLE(queues, 8); +}; + /** * struct ap_matrix_mdev - Contains the data associated with a matrix mediated * device. @@ -87,6 +97,7 @@ struct ap_matrix { * @pqap_hook: the function pointer to the interception handler for the * PQAP(AQIC) instruction. * @mdev: the mediated device + * @qtable: table of queues (struct vfio_ap_queue) assigned to the mdev */ struct ap_matrix_mdev { struct vfio_device vdev; @@ -96,6 +107,7 @@ struct ap_matrix_mdev { struct kvm *kvm; crypto_hook pqap_hook; struct mdev_device *mdev; + struct ap_queue_table qtable; }; /** @@ -105,6 +117,7 @@ struct ap_matrix_mdev { * @saved_pfn: the guest PFN pinned for the guest * @apqn: the APQN of the AP queue device * @saved_isc: the guest ISC registered with the GIB interface + * @mdev_qnode: allows the vfio_ap_queue struct to be added to a hashtable */ struct vfio_ap_queue { struct ap_matrix_mdev *matrix_mdev; @@ -112,6 +125,7 @@ struct vfio_ap_queue { int apqn; #define VFIO_AP_ISC_INVALID 0xff unsigned char saved_isc; + struct hlist_node mdev_qnode; }; int vfio_ap_mdev_register(void); From 49b0109fb399b78a424d76485194202f290bdece Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Wed, 11 Dec 2019 15:35:54 -0500 Subject: [PATCH 05/46] s390/vfio-ap: introduce shadow APCB The APCB is a field within the CRYCB that provides the AP configuration to a KVM guest. Let's introduce a shadow copy of the KVM guest's APCB and maintain it for the lifespan of the guest. The shadow APCB serves the following purposes: 1. The shadow APCB can be maintained even when the mediated device is not currently in use by a KVM guest. Since the mediated device's AP configuration is filtered to ensure that no AP queues are passed through to the KVM guest that are not bound to the vfio_ap device driver or available to the host, the mediated device's AP configuration may differ from the guest's. Having a shadow of a guest's APCB allows us to provide a sysfs interface to view the guest's APCB even if the mediated device is not currently passed through to a KVM guest. This can aid in problem determination when the guest is unexpectedly missing AP resources. 2. If filtering was done in-place for the real APCB, the guest could pick up a transient state. Doing the filtering on a shadow and transferring the AP configuration to the real APCB after the guest is started or when AP resources are assigned to or unassigned from the mediated device, or when the host configuration changes, the guest's AP configuration will never be in a transient state. Signed-off-by: Tony Krowiak Reviewed-by: Halil Pasic Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 10 ++++++---- drivers/s390/crypto/vfio_ap_private.h | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 9bed11aa0ea6..5cc4e1c31aae 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -466,6 +466,7 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) matrix_mdev->mdev = mdev; vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); matrix_mdev->pqap_hook = handle_pqap; + vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); hash_init(matrix_mdev->qtable.queues); dev_set_drvdata(&mdev->dev, matrix_mdev); mutex_lock(&matrix_dev->lock); @@ -1308,10 +1309,11 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; - kvm_arch_crypto_set_masks(kvm, - matrix_mdev->matrix.apm, - matrix_mdev->matrix.aqm, - matrix_mdev->matrix.adm); + memcpy(&matrix_mdev->shadow_apcb, &matrix_mdev->matrix, + sizeof(struct ap_matrix)); + kvm_arch_crypto_set_masks(kvm, matrix_mdev->shadow_apcb.apm, + matrix_mdev->shadow_apcb.aqm, + matrix_mdev->shadow_apcb.adm); mutex_unlock(&kvm->lock); mutex_unlock(&matrix_dev->lock); diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 44d2eeb795a0..acb3f9d22025 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -91,6 +91,7 @@ struct ap_queue_table { * @node: allows the ap_matrix_mdev struct to be added to a list * @matrix: the adapters, usage domains and control domains assigned to the * mediated matrix device. + * @shadow_apcb: the shadow copy of the APCB field of the KVM guest's CRYCB * @iommu_notifier: notifier block used for specifying callback function for * handling the VFIO_IOMMU_NOTIFY_DMA_UNMAP even * @kvm: the struct holding guest's state @@ -103,6 +104,7 @@ struct ap_matrix_mdev { struct vfio_device vdev; struct list_head node; struct ap_matrix matrix; + struct ap_matrix shadow_apcb; struct notifier_block iommu_notifier; struct kvm *kvm; crypto_hook pqap_hook; From 48cae940c31d2407d860d87c41d5f9871c0521db Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Wed, 11 Aug 2021 16:23:05 -0400 Subject: [PATCH 06/46] s390/vfio-ap: refresh guest's APCB by filtering AP resources assigned to mdev Refresh the guest's APCB by filtering the APQNs and control domain numbers assigned to the matrix mdev. Filtering of APQNs: ----------------- APQNs that do not reference an AP queue device bound to the vfio_ap device driver must be filtered from the APQNs assigned to the matrix mdev before they can be assigned to the guest's APCB. Given that the APQNs are configured in the guest's APCB as a matrix of APIDs (adapters) and APQIs (domains), it is not possible to filter an individual APQN. For example, suppose the matrix of APQNs is structured as follows: APIDs 3 4 5 0 (3,0) (4,0) (5,0) APQIs 1 (3,1) (4,1) (5,1) 2 (3,2) (4,2) (5,2) Now suppose APQN (4,1) does not reference a queue device bound to the vfio_ap device driver. If we filter APID 4, the APQNs (4,0), (4,1) and (4,2) will be removed. Similarly, if we filter domain 1, APQNs (3,1), (4,1) and (5,1) will be removed. To resolve this dilemma, the choice was made to filter the APID - in this case 4 - from the guest's APCB. The reason for this design decision is because the APID references an AP adapter which is a real hardware device that can be physically installed, removed, enabled or disabled; whereas, a domain is a partition within the adapter. It therefore better reflects reality to remove the APID from the guest's APCB. Filtering of control domains: ---------------------------- Any control domains that are not assigned to the host's AP configuration will be filtered from those assigned to the matrix mdev before assigning them to the guest's APCB. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 101 +++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 5cc4e1c31aae..c9439ba2ff85 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -447,6 +447,68 @@ static void vfio_ap_matrix_init(struct ap_config_info *info, matrix->adm_max = info->apxa ? info->Nd : 15; } +static void vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) +{ + bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm, + (unsigned long *)matrix_dev->info.adm, AP_DOMAINS); +} + +/* + * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev + * to ensure no queue devices are passed through to + * the guest that are not bound to the vfio_ap + * device driver. + * + * @matrix_mdev: the matrix mdev whose matrix is to be filtered. + * + * Note: If an APQN referencing a queue device that is not bound to the vfio_ap + * driver, its APID will be filtered from the guest's APCB. The matrix + * structure precludes filtering an individual APQN, so its APID will be + * filtered. + */ +static void vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, + struct ap_matrix_mdev *matrix_mdev) +{ + int ret; + unsigned long apid, apqi, apqn; + + ret = ap_qci(&matrix_dev->info); + if (ret) + return; + + vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); + + /* + * Copy the adapters, domains and control domains to the shadow_apcb + * from the matrix mdev, but only those that are assigned to the host's + * AP configuration. + */ + bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm, + (unsigned long *)matrix_dev->info.apm, AP_DEVICES); + bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm, + (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS); + + for_each_set_bit_inv(apid, apm, AP_DEVICES) { + for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { + /* + * If the APQN is not bound to the vfio_ap device + * driver, then we can't assign it to the guest's + * AP configuration. The AP architecture won't + * allow filtering of a single APQN, so let's filter + * the APID since an adapter represents a physical + * hardware device. + */ + apqn = AP_MKQID(apid, apqi); + + if (!vfio_ap_mdev_get_queue(matrix_mdev, apqn)) { + clear_bit_inv(apid, + matrix_mdev->shadow_apcb.apm); + break; + } + } + } +} + static int vfio_ap_mdev_probe(struct mdev_device *mdev) { struct ap_matrix_mdev *matrix_mdev; @@ -802,6 +864,8 @@ static ssize_t assign_adapter_store(struct device *dev, { int ret; unsigned long apid; + DECLARE_BITMAP(apm_delta, AP_DEVICES); + struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); mutex_lock(&matrix_dev->lock); @@ -837,6 +901,10 @@ static ssize_t assign_adapter_store(struct device *dev, goto share_err; vfio_ap_mdev_link_adapter(matrix_mdev, apid); + memset(apm_delta, 0, sizeof(apm_delta)); + set_bit_inv(apid, apm_delta); + vfio_ap_mdev_filter_matrix(apm_delta, + matrix_mdev->matrix.aqm, matrix_mdev); ret = count; goto done; @@ -905,6 +973,10 @@ static ssize_t unassign_adapter_store(struct device *dev, clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); vfio_ap_mdev_unlink_adapter(matrix_mdev, apid); + + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) + clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); + ret = count; done: mutex_unlock(&matrix_dev->lock); @@ -978,6 +1050,7 @@ static ssize_t assign_domain_store(struct device *dev, { int ret; unsigned long apqi; + DECLARE_BITMAP(aqm_delta, AP_DOMAINS); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); unsigned long max_apqi = matrix_mdev->matrix.aqm_max; @@ -1008,6 +1081,10 @@ static ssize_t assign_domain_store(struct device *dev, goto share_err; vfio_ap_mdev_link_domain(matrix_mdev, apqi); + memset(aqm_delta, 0, sizeof(aqm_delta)); + set_bit_inv(apqi, aqm_delta); + vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta, + matrix_mdev); ret = count; goto done; @@ -1076,6 +1153,10 @@ static ssize_t unassign_domain_store(struct device *dev, clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); vfio_ap_mdev_unlink_domain(matrix_mdev, apqi); + + if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) + clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); + ret = count; done: @@ -1129,6 +1210,7 @@ static ssize_t assign_control_domain_store(struct device *dev, * number of control domains that can be assigned. */ set_bit_inv(id, matrix_mdev->matrix.adm); + vfio_ap_mdev_filter_cdoms(matrix_mdev); ret = count; done: mutex_unlock(&matrix_dev->lock); @@ -1176,6 +1258,10 @@ static ssize_t unassign_control_domain_store(struct device *dev, } clear_bit_inv(domid, matrix_mdev->matrix.adm); + + if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) + clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm); + ret = count; done: mutex_unlock(&matrix_dev->lock); @@ -1309,8 +1395,6 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; - memcpy(&matrix_mdev->shadow_apcb, &matrix_mdev->matrix, - sizeof(struct ap_matrix)); kvm_arch_crypto_set_masks(kvm, matrix_mdev->shadow_apcb.apm, matrix_mdev->shadow_apcb.aqm, matrix_mdev->shadow_apcb.adm); @@ -1687,6 +1771,11 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) q->apqn = to_ap_queue(&apdev->device)->qid; q->saved_isc = VFIO_AP_ISC_INVALID; vfio_ap_queue_link_mdev(q); + if (q->matrix_mdev) { + vfio_ap_mdev_filter_matrix(q->matrix_mdev->matrix.apm, + q->matrix_mdev->matrix.aqm, + q->matrix_mdev); + } dev_set_drvdata(&apdev->device, q); mutex_unlock(&matrix_dev->lock); @@ -1695,15 +1784,21 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) void vfio_ap_mdev_remove_queue(struct ap_device *apdev) { + unsigned long apid; struct vfio_ap_queue *q; mutex_lock(&matrix_dev->lock); sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); q = dev_get_drvdata(&apdev->device); - if (q->matrix_mdev) + if (q->matrix_mdev) { vfio_ap_unlink_queue_fr_mdev(q); + apid = AP_QID_CARD(q->apqn); + if (test_bit_inv(apid, q->matrix_mdev->shadow_apcb.apm)) + clear_bit_inv(apid, q->matrix_mdev->shadow_apcb.apm); + } + vfio_ap_mdev_reset_queue(q, 1); dev_set_drvdata(&apdev->device, NULL); kfree(q); From e2126a73746ec6d5f519edf5bf33d76347a51ee9 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Mon, 14 Dec 2020 21:14:54 -0500 Subject: [PATCH 07/46] s390/vfio-ap: allow assignment of unavailable AP queues to mdev device The current implementation does not allow assignment of an AP adapter or domain to an mdev device if each APQN resulting from the assignment does not reference an AP queue device that is bound to the vfio_ap device driver. This patch allows assignment of AP resources to the matrix mdev as long as the APQNs resulting from the assignment: 1. Are not reserved by the AP BUS for use by the zcrypt device drivers. 2. Are not assigned to another matrix mdev. The rationale behind this is that the AP architecture does not preclude assignment of APQNs to an AP configuration profile that are not available to the system. Signed-off-by: Tony Krowiak Reviewed-by: Halil Pasic Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 224 +++++++----------------------- 1 file changed, 53 insertions(+), 171 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index c9439ba2ff85..f1f24b359a85 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -660,141 +660,48 @@ static struct attribute_group *vfio_ap_mdev_type_groups[] = { NULL, }; -struct vfio_ap_queue_reserved { - unsigned long *apid; - unsigned long *apqi; - bool reserved; -}; +#define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \ + "already assigned to %s" -/** - * vfio_ap_has_queue - determines if the AP queue containing the target in @data - * - * @dev: an AP queue device - * @data: a struct vfio_ap_queue_reserved reference - * - * Flags whether the AP queue device (@dev) has a queue ID containing the APQN, - * apid or apqi specified in @data: - * - * - If @data contains both an apid and apqi value, then @data will be flagged - * as reserved if the APID and APQI fields for the AP queue device matches - * - * - If @data contains only an apid value, @data will be flagged as - * reserved if the APID field in the AP queue device matches - * - * - If @data contains only an apqi value, @data will be flagged as - * reserved if the APQI field in the AP queue device matches - * - * Return: 0 to indicate the input to function succeeded. Returns -EINVAL if - * @data does not contain either an apid or apqi. - */ -static int vfio_ap_has_queue(struct device *dev, void *data) +static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev, + unsigned long *apm, + unsigned long *aqm) { - struct vfio_ap_queue_reserved *qres = data; - struct ap_queue *ap_queue = to_ap_queue(dev); - ap_qid_t qid; - unsigned long id; + unsigned long apid, apqi; + const struct device *dev = mdev_dev(matrix_mdev->mdev); + const char *mdev_name = dev_name(dev); - if (qres->apid && qres->apqi) { - qid = AP_MKQID(*qres->apid, *qres->apqi); - if (qid == ap_queue->qid) - qres->reserved = true; - } else if (qres->apid && !qres->apqi) { - id = AP_QID_CARD(ap_queue->qid); - if (id == *qres->apid) - qres->reserved = true; - } else if (!qres->apid && qres->apqi) { - id = AP_QID_QUEUE(ap_queue->qid); - if (id == *qres->apqi) - qres->reserved = true; - } else { - return -EINVAL; - } - - return 0; + for_each_set_bit_inv(apid, apm, AP_DEVICES) + for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) + dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name); } /** - * vfio_ap_verify_queue_reserved - verifies that the AP queue containing - * @apid or @aqpi is reserved + * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs * - * @apid: an AP adapter ID - * @apqi: an AP queue index + * @mdev_apm: mask indicating the APIDs of the APQNs to be verified + * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified * - * Verifies that the AP queue with @apid/@apqi is reserved by the VFIO AP device - * driver according to the following rules: - * - * - If both @apid and @apqi are not NULL, then there must be an AP queue - * device bound to the vfio_ap driver with the APQN identified by @apid and - * @apqi - * - * - If only @apid is not NULL, then there must be an AP queue device bound - * to the vfio_ap driver with an APQN containing @apid - * - * - If only @apqi is not NULL, then there must be an AP queue device bound - * to the vfio_ap driver with an APQN containing @apqi - * - * Return: 0 if the AP queue is reserved; otherwise, returns -EADDRNOTAVAIL. - */ -static int vfio_ap_verify_queue_reserved(unsigned long *apid, - unsigned long *apqi) -{ - int ret; - struct vfio_ap_queue_reserved qres; - - qres.apid = apid; - qres.apqi = apqi; - qres.reserved = false; - - ret = driver_for_each_device(&matrix_dev->vfio_ap_drv->driver, NULL, - &qres, vfio_ap_has_queue); - if (ret) - return ret; - - if (qres.reserved) - return 0; - - return -EADDRNOTAVAIL; -} - -static int -vfio_ap_mdev_verify_queues_reserved_for_apid(struct ap_matrix_mdev *matrix_mdev, - unsigned long apid) -{ - int ret; - unsigned long apqi; - unsigned long nbits = matrix_mdev->matrix.aqm_max + 1; - - if (find_first_bit_inv(matrix_mdev->matrix.aqm, nbits) >= nbits) - return vfio_ap_verify_queue_reserved(&apid, NULL); - - for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, nbits) { - ret = vfio_ap_verify_queue_reserved(&apid, &apqi); - if (ret) - return ret; - } - - return 0; -} - -/** - * vfio_ap_mdev_verify_no_sharing - verifies that the AP matrix is not configured - * - * @matrix_mdev: the mediated matrix device - * - * Verifies that the APQNs derived from the cross product of the AP adapter IDs - * and AP queue indexes comprising the AP matrix are not configured for another + * Verifies that each APQN derived from the Cartesian product of a bitmap of + * AP adapter IDs and AP queue indexes is not configured for any matrix * mediated device. AP queue sharing is not allowed. * - * Return: 0 if the APQNs are not shared; otherwise returns -EADDRINUSE. + * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE. */ -static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *matrix_mdev) +static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, + unsigned long *mdev_aqm) { - struct ap_matrix_mdev *lstdev; + struct ap_matrix_mdev *matrix_mdev; DECLARE_BITMAP(apm, AP_DEVICES); DECLARE_BITMAP(aqm, AP_DOMAINS); - list_for_each_entry(lstdev, &matrix_dev->mdev_list, node) { - if (matrix_mdev == lstdev) + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + /* + * If the input apm and aqm are fields of the matrix_mdev + * object, then move on to the next matrix_mdev. + */ + if (mdev_apm == matrix_mdev->matrix.apm && + mdev_aqm == matrix_mdev->matrix.aqm) continue; memset(apm, 0, sizeof(apm)); @@ -804,20 +711,32 @@ static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *matrix_mdev) * We work on full longs, as we can only exclude the leftover * bits in non-inverse order. The leftover is all zeros. */ - if (!bitmap_and(apm, matrix_mdev->matrix.apm, - lstdev->matrix.apm, AP_DEVICES)) + if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm, + AP_DEVICES)) continue; - if (!bitmap_and(aqm, matrix_mdev->matrix.aqm, - lstdev->matrix.aqm, AP_DOMAINS)) + if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm, + AP_DOMAINS)) continue; + vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm); + return -EADDRINUSE; } return 0; } +static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev) +{ + if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm, + matrix_mdev->matrix.aqm)) + return -EADDRNOTAVAIL; + + return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm, + matrix_mdev->matrix.aqm); +} + static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev, unsigned long apid) { @@ -885,20 +804,13 @@ static ssize_t assign_adapter_store(struct device *dev, goto done; } - /* - * Set the bit in the AP mask (APM) corresponding to the AP adapter - * number (APID). The bits in the mask, from most significant to least - * significant bit, correspond to APIDs 0-255. - */ - ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid); - if (ret) - goto done; - set_bit_inv(apid, matrix_mdev->matrix.apm); - ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev); - if (ret) - goto share_err; + ret = vfio_ap_mdev_validate_masks(matrix_mdev); + if (ret) { + clear_bit_inv(apid, matrix_mdev->matrix.apm); + goto done; + } vfio_ap_mdev_link_adapter(matrix_mdev, apid); memset(apm_delta, 0, sizeof(apm_delta)); @@ -906,10 +818,6 @@ static ssize_t assign_adapter_store(struct device *dev, vfio_ap_mdev_filter_matrix(apm_delta, matrix_mdev->matrix.aqm, matrix_mdev); ret = count; - goto done; - -share_err: - clear_bit_inv(apid, matrix_mdev->matrix.apm); done: mutex_unlock(&matrix_dev->lock); @@ -984,26 +892,6 @@ done: } static DEVICE_ATTR_WO(unassign_adapter); -static int -vfio_ap_mdev_verify_queues_reserved_for_apqi(struct ap_matrix_mdev *matrix_mdev, - unsigned long apqi) -{ - int ret; - unsigned long apid; - unsigned long nbits = matrix_mdev->matrix.apm_max + 1; - - if (find_first_bit_inv(matrix_mdev->matrix.apm, nbits) >= nbits) - return vfio_ap_verify_queue_reserved(NULL, &apqi); - - for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, nbits) { - ret = vfio_ap_verify_queue_reserved(&apid, &apqi); - if (ret) - return ret; - } - - return 0; -} - static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev, unsigned long apqi) { @@ -1070,15 +958,13 @@ static ssize_t assign_domain_store(struct device *dev, goto done; } - ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi); - if (ret) - goto done; - set_bit_inv(apqi, matrix_mdev->matrix.aqm); - ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev); - if (ret) - goto share_err; + ret = vfio_ap_mdev_validate_masks(matrix_mdev); + if (ret) { + clear_bit_inv(apqi, matrix_mdev->matrix.aqm); + goto done; + } vfio_ap_mdev_link_domain(matrix_mdev, apqi); memset(aqm_delta, 0, sizeof(aqm_delta)); @@ -1086,10 +972,6 @@ static ssize_t assign_domain_store(struct device *dev, vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta, matrix_mdev); ret = count; - goto done; - -share_err: - clear_bit_inv(apqi, matrix_mdev->matrix.aqm); done: mutex_unlock(&matrix_dev->lock); From d0786556caa1a9b63a7bee321546c6cac1e84e96 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Wed, 16 Mar 2022 12:23:12 -0400 Subject: [PATCH 08/46] s390/vfio-ap: rename matrix_dev->lock mutex to matrix_dev->mdevs_lock The matrix_dev->lock mutex is being renamed to matrix_dev->mdevs_lock to better reflect its purpose, which is to control access to the state of the mediated devices under the control of the vfio_ap device driver. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_drv.c | 2 +- drivers/s390/crypto/vfio_ap_ops.c | 76 ++++++++++++++------------- drivers/s390/crypto/vfio_ap_private.h | 4 +- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index 1ff6e3dbbffe..ed162732b139 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -98,7 +98,7 @@ static int vfio_ap_matrix_dev_create(void) goto matrix_alloc_err; } - mutex_init(&matrix_dev->lock); + mutex_init(&matrix_dev->mdevs_lock); INIT_LIST_HEAD(&matrix_dev->mdev_list); dev_set_name(&matrix_dev->device, "%s", VFIO_AP_DEV_NAME); diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index f1f24b359a85..6f091cd602d3 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -397,10 +397,12 @@ static int handle_pqap(struct kvm_vcpu *vcpu) return -EOPNOTSUPP; } - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); + if (!vcpu->kvm->arch.crypto.pqap_hook) { VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n", __func__, apqn); + goto out_unlock; } @@ -435,7 +437,7 @@ static int handle_pqap(struct kvm_vcpu *vcpu) out_unlock: memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus)); vcpu->run->s.regs.gprs[1] >>= 32; - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return 0; } @@ -531,9 +533,9 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); hash_init(matrix_mdev->qtable.queues); dev_set_drvdata(&mdev->dev, matrix_mdev); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); if (ret) @@ -542,9 +544,9 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) return 0; err_list: - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); list_del(&matrix_mdev->node); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); vfio_uninit_group_dev(&matrix_mdev->vdev); kfree(matrix_mdev); err_dec_available: @@ -607,11 +609,11 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) vfio_unregister_group_dev(&matrix_mdev->vdev); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); vfio_ap_mdev_reset_queues(matrix_mdev); vfio_ap_mdev_unlink_fr_queues(matrix_mdev); list_del(&matrix_mdev->node); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); vfio_uninit_group_dev(&matrix_mdev->vdev); kfree(matrix_mdev); atomic_inc(&matrix_dev->available_instances); @@ -787,7 +789,7 @@ static ssize_t assign_adapter_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); /* If the KVM guest is running, disallow assignment of adapter */ if (matrix_mdev->kvm) { @@ -819,7 +821,7 @@ static ssize_t assign_adapter_store(struct device *dev, matrix_mdev->matrix.aqm, matrix_mdev); ret = count; done: - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return ret; } @@ -862,7 +864,7 @@ static ssize_t unassign_adapter_store(struct device *dev, unsigned long apid; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); /* If the KVM guest is running, disallow unassignment of adapter */ if (matrix_mdev->kvm) { @@ -887,7 +889,7 @@ static ssize_t unassign_adapter_store(struct device *dev, ret = count; done: - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return ret; } static DEVICE_ATTR_WO(unassign_adapter); @@ -942,7 +944,7 @@ static ssize_t assign_domain_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); unsigned long max_apqi = matrix_mdev->matrix.aqm_max; - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); /* If the KVM guest is running, disallow assignment of domain */ if (matrix_mdev->kvm) { @@ -973,7 +975,7 @@ static ssize_t assign_domain_store(struct device *dev, matrix_mdev); ret = count; done: - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return ret; } @@ -1016,7 +1018,7 @@ static ssize_t unassign_domain_store(struct device *dev, unsigned long apqi; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); /* If the KVM guest is running, disallow unassignment of domain */ if (matrix_mdev->kvm) { @@ -1042,7 +1044,7 @@ static ssize_t unassign_domain_store(struct device *dev, ret = count; done: - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return ret; } static DEVICE_ATTR_WO(unassign_domain); @@ -1069,7 +1071,7 @@ static ssize_t assign_control_domain_store(struct device *dev, unsigned long id; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); /* If the KVM guest is running, disallow assignment of control domain */ if (matrix_mdev->kvm) { @@ -1095,7 +1097,7 @@ static ssize_t assign_control_domain_store(struct device *dev, vfio_ap_mdev_filter_cdoms(matrix_mdev); ret = count; done: - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return ret; } static DEVICE_ATTR_WO(assign_control_domain); @@ -1123,7 +1125,7 @@ static ssize_t unassign_control_domain_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); unsigned long max_domid = matrix_mdev->matrix.adm_max; - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); /* If a KVM guest is running, disallow unassignment of control domain */ if (matrix_mdev->kvm) { @@ -1146,7 +1148,7 @@ static ssize_t unassign_control_domain_store(struct device *dev, ret = count; done: - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return ret; } static DEVICE_ATTR_WO(unassign_control_domain); @@ -1162,13 +1164,13 @@ static ssize_t control_domains_show(struct device *dev, struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); unsigned long max_domid = matrix_mdev->matrix.adm_max; - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) { n = sprintf(bufpos, "%04lx\n", id); bufpos += n; nchars += n; } - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return nchars; } @@ -1191,7 +1193,7 @@ static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, apid1 = find_first_bit_inv(matrix_mdev->matrix.apm, napm_bits); apqi1 = find_first_bit_inv(matrix_mdev->matrix.aqm, naqm_bits); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) { for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) { @@ -1217,7 +1219,7 @@ static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, } } - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return nchars; } @@ -1265,12 +1267,12 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, up_write(&kvm->arch.crypto.pqap_hook_rwsem); mutex_lock(&kvm->lock); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); list_for_each_entry(m, &matrix_dev->mdev_list, node) { if (m != matrix_mdev && m->kvm == kvm) { mutex_unlock(&kvm->lock); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return -EPERM; } } @@ -1282,7 +1284,7 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, matrix_mdev->shadow_apcb.adm); mutex_unlock(&kvm->lock); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); } return 0; @@ -1334,7 +1336,7 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) up_write(&kvm->arch.crypto.pqap_hook_rwsem); mutex_lock(&kvm->lock); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); kvm_arch_crypto_clear_masks(kvm); vfio_ap_mdev_reset_queues(matrix_mdev); @@ -1342,7 +1344,7 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) matrix_mdev->kvm = NULL; mutex_unlock(&kvm->lock); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); } } @@ -1497,7 +1499,7 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, container_of(vdev, struct ap_matrix_mdev, vdev); int ret; - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); switch (cmd) { case VFIO_DEVICE_GET_INFO: ret = vfio_ap_mdev_get_device_info(arg); @@ -1509,7 +1511,7 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, ret = -EOPNOTSUPP; break; } - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return ret; } @@ -1538,7 +1540,7 @@ static ssize_t status_show(struct device *dev, struct ap_matrix_mdev *matrix_mdev; struct ap_device *apdev = to_ap_dev(dev); - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); q = dev_get_drvdata(&apdev->device); matrix_mdev = vfio_ap_mdev_for_queue(q); @@ -1554,7 +1556,7 @@ static ssize_t status_show(struct device *dev, AP_QUEUE_UNASSIGNED); } - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return nchars; } @@ -1649,7 +1651,7 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) if (!q) return -ENOMEM; - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); q->apqn = to_ap_queue(&apdev->device)->qid; q->saved_isc = VFIO_AP_ISC_INVALID; vfio_ap_queue_link_mdev(q); @@ -1659,7 +1661,7 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) q->matrix_mdev); } dev_set_drvdata(&apdev->device, q); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); return 0; } @@ -1669,7 +1671,7 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) unsigned long apid; struct vfio_ap_queue *q; - mutex_lock(&matrix_dev->lock); + mutex_lock(&matrix_dev->mdevs_lock); sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); q = dev_get_drvdata(&apdev->device); @@ -1684,5 +1686,5 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) vfio_ap_mdev_reset_queue(q, 1); dev_set_drvdata(&apdev->device, NULL); kfree(q); - mutex_unlock(&matrix_dev->lock); + mutex_unlock(&matrix_dev->mdevs_lock); } diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index acb3f9d22025..cb4a626b4b0f 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -33,7 +33,7 @@ * @available_instances: number of mediated matrix devices that can be created * @info: the struct containing the output from the PQAP(QCI) instruction * @mdev_list: the list of mediated matrix devices created - * @lock: mutex for locking the AP matrix device. This lock will be + * @mdevs_lock: mutex for locking the AP matrix device. This lock will be * taken every time we fiddle with state managed by the vfio_ap * driver, be it using @mdev_list or writing the state of a * single ap_matrix_mdev device. It's quite coarse but we don't @@ -45,7 +45,7 @@ struct ap_matrix_dev { atomic_t available_instances; struct ap_config_info info; struct list_head mdev_list; - struct mutex lock; + struct mutex mdevs_lock; /* serializes access to each ap_matrix_mdev */ struct ap_driver *vfio_ap_drv; }; From 21195eb038bcb4c21daef761e8b18f3cc9f22020 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Thu, 23 Sep 2021 22:21:58 -0400 Subject: [PATCH 09/46] s390/vfio-ap: introduce new mutex to control access to the KVM pointer The vfio_ap device driver registers for notification when the pointer to the KVM object for a guest is set. Recall that the KVM lock (kvm->lock) mutex must be taken outside of the matrix_dev->lock mutex to prevent the reporting by lockdep of a circular locking dependency (a.k.a., a lockdep splat): * see commit 0cc00c8d4050 ("Fix circular lockdep when setting/clearing crypto masks") * see commit 86956e70761b ("replace open coded locks for VFIO_GROUP_NOTIFY_SET_KVM notification") With the introduction of support for hot plugging/unplugging AP devices passed through to a KVM guest, a new guests_lock mutex is introduced to ensure the proper locking order is maintained: struct ap_matrix_dev { ... struct mutex guests_lock; ... } The matrix_dev->guests_lock controls access to the matrix_mdev instances that hold the state for AP devices that have been passed through to a KVM guest. This lock must be held to control access to the KVM pointer (matrix_mdev->kvm) while the vfio_ap device driver is using it to plug/unplug AP devices passed through to the KVM guest. Keep in mind, the proper locking order must be maintained whenever dynamically updating a KVM guest's APCB to plug/unplug adapters, domains and control domains: 1. matrix_dev->guests_lock: required to use the KVM pointer - stored in a struct ap_matrix_mdev instance - to update a KVM guest's APCB 2. matrix_mdev->kvm->lock: required to update a guest's APCB 3. matrix_dev->mdevs_lock: required to access data stored in a struct ap_matrix_mdev instance. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_drv.c | 1 + drivers/s390/crypto/vfio_ap_private.h | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index ed162732b139..db8ca7bb3696 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -100,6 +100,7 @@ static int vfio_ap_matrix_dev_create(void) mutex_init(&matrix_dev->mdevs_lock); INIT_LIST_HEAD(&matrix_dev->mdev_list); + mutex_init(&matrix_dev->guests_lock); dev_set_name(&matrix_dev->device, "%s", VFIO_AP_DEV_NAME); matrix_dev->device.parent = root_device; diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index cb4a626b4b0f..97734b599583 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -39,6 +39,11 @@ * single ap_matrix_mdev device. It's quite coarse but we don't * expect much contention. * @vfio_ap_drv: the vfio_ap device driver + * @guests_lock: mutex for controlling access to a guest that is using AP + * devices passed through by the vfio_ap device driver. This lock + * will be taken when the AP devices are plugged into or unplugged + * from a guest, and when an ap_matrix_mdev device is added to or + * removed from @mdev_list or the list is iterated. */ struct ap_matrix_dev { struct device device; @@ -47,6 +52,7 @@ struct ap_matrix_dev { struct list_head mdev_list; struct mutex mdevs_lock; /* serializes access to each ap_matrix_mdev */ struct ap_driver *vfio_ap_drv; + struct mutex guests_lock; /* serializes access to each KVM guest */ }; extern struct ap_matrix_dev *matrix_dev; From b84eb8e05057f6f329e28f2563668340ea14184b Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Wed, 16 Mar 2022 17:46:38 -0400 Subject: [PATCH 10/46] s390/vfio-ap: use proper locking order when setting/clearing KVM pointer The group notifier that handles the VFIO_GROUP_NOTIFY_SET_KVM event must use the required locks in proper locking order to dynamically update the guest's APCB. The proper locking order is: 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM guest's APCB. 2. matrix_mdev->kvm->lock: required to update a KVM guest's APCB. 3. matrix_dev->mdevs_lock: required to store or access the data stored in a struct ap_matrix_mdev instance. Two macros are introduced to acquire and release the locks in the proper order. These macros are now used by the group notifier functions. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 58 +++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 6f091cd602d3..20f779dd8ab9 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -34,6 +34,49 @@ static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev); static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); static const struct vfio_device_ops vfio_ap_matrix_dev_ops; +/** + * get_update_locks_for_kvm: Acquire the locks required to dynamically update a + * KVM guest's APCB in the proper order. + * + * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. + * + * The proper locking order is: + * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM + * guest's APCB. + * 2. kvm->lock: required to update a guest's APCB + * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev + * + * Note: If @kvm is NULL, the KVM lock will not be taken. + */ +static inline void get_update_locks_for_kvm(struct kvm *kvm) +{ + mutex_lock(&matrix_dev->guests_lock); + if (kvm) + mutex_lock(&kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); +} + +/** + * release_update_locks_for_kvm: Release the locks used to dynamically update a + * KVM guest's APCB in the proper order. + * + * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB. + * + * The proper unlocking order is: + * 1. matrix_dev->mdevs_lock + * 2. kvm->lock + * 3. matrix_dev->guests_lock + * + * Note: If @kvm is NULL, the KVM lock will not be released. + */ +static inline void release_update_locks_for_kvm(struct kvm *kvm) +{ + mutex_unlock(&matrix_dev->mdevs_lock); + if (kvm) + mutex_unlock(&kvm->lock); + mutex_unlock(&matrix_dev->guests_lock); +} + /** * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a * hash table of queues assigned to a matrix mdev @@ -1266,13 +1309,11 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; up_write(&kvm->arch.crypto.pqap_hook_rwsem); - mutex_lock(&kvm->lock); - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_kvm(kvm); list_for_each_entry(m, &matrix_dev->mdev_list, node) { if (m != matrix_mdev && m->kvm == kvm) { - mutex_unlock(&kvm->lock); - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_kvm(kvm); return -EPERM; } } @@ -1283,8 +1324,7 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, matrix_mdev->shadow_apcb.aqm, matrix_mdev->shadow_apcb.adm); - mutex_unlock(&kvm->lock); - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_kvm(kvm); } return 0; @@ -1335,16 +1375,14 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) kvm->arch.crypto.pqap_hook = NULL; up_write(&kvm->arch.crypto.pqap_hook_rwsem); - mutex_lock(&kvm->lock); - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_kvm(kvm); kvm_arch_crypto_clear_masks(kvm); vfio_ap_mdev_reset_queues(matrix_mdev); kvm_put_kvm(kvm); matrix_mdev->kvm = NULL; - mutex_unlock(&kvm->lock); - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_kvm(kvm); } } From 8ee13ad99337b0b6920e732d87c692d577335f15 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Wed, 16 Mar 2022 22:21:38 -0400 Subject: [PATCH 11/46] s390/vfio-ap: prepare for dynamic update of guest's APCB on assign/unassign The functions backing the matrix mdev's sysfs attribute interfaces to assign/unassign adapters, domains and control domains must take and release the locks required to perform a dynamic update of a guest's APCB in the proper order. The proper order for taking the locks is: matrix_dev->guests_lock => kvm->lock => matrix_dev->mdevs_lock The proper order for releasing the locks is: matrix_dev->mdevs_lock => kvm->lock => matrix_dev->guests_lock Two new macros are introduced for this purpose: One to take the locks and the other to release the locks. These macros will be used by the assignment/unassignment functions to prepare for dynamic update of the KVM guest's APCB. Signed-off-by: Tony Krowiak Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 71 +++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 20f779dd8ab9..dcd36d427b86 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -77,6 +77,53 @@ static inline void release_update_locks_for_kvm(struct kvm *kvm) mutex_unlock(&matrix_dev->guests_lock); } +/** + * get_update_locks_for_mdev: Acquire the locks required to dynamically update a + * KVM guest's APCB in the proper order. + * + * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP + * configuration data to use to update a KVM guest's APCB. + * + * The proper locking order is: + * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM + * guest's APCB. + * 2. matrix_mdev->kvm->lock: required to update a guest's APCB + * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev + * + * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM + * lock will not be taken. + */ +static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) +{ + mutex_lock(&matrix_dev->guests_lock); + if (matrix_mdev && matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); +} + +/** + * release_update_locks_for_mdev: Release the locks used to dynamically update a + * KVM guest's APCB in the proper order. + * + * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP + * configuration data to use to update a KVM guest's APCB. + * + * The proper unlocking order is: + * 1. matrix_dev->mdevs_lock + * 2. matrix_mdev->kvm->lock + * 3. matrix_dev->guests_lock + * + * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM + * lock will not be released. + */ +static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev) +{ + mutex_unlock(&matrix_dev->mdevs_lock); + if (matrix_mdev && matrix_mdev->kvm) + mutex_unlock(&matrix_mdev->kvm->lock); + mutex_unlock(&matrix_dev->guests_lock); +} + /** * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a * hash table of queues assigned to a matrix mdev @@ -832,7 +879,7 @@ static ssize_t assign_adapter_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_mdev(matrix_mdev); /* If the KVM guest is running, disallow assignment of adapter */ if (matrix_mdev->kvm) { @@ -864,7 +911,7 @@ static ssize_t assign_adapter_store(struct device *dev, matrix_mdev->matrix.aqm, matrix_mdev); ret = count; done: - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); return ret; } @@ -907,7 +954,7 @@ static ssize_t unassign_adapter_store(struct device *dev, unsigned long apid; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_mdev(matrix_mdev); /* If the KVM guest is running, disallow unassignment of adapter */ if (matrix_mdev->kvm) { @@ -932,7 +979,7 @@ static ssize_t unassign_adapter_store(struct device *dev, ret = count; done: - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); return ret; } static DEVICE_ATTR_WO(unassign_adapter); @@ -987,7 +1034,7 @@ static ssize_t assign_domain_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); unsigned long max_apqi = matrix_mdev->matrix.aqm_max; - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_mdev(matrix_mdev); /* If the KVM guest is running, disallow assignment of domain */ if (matrix_mdev->kvm) { @@ -1018,7 +1065,7 @@ static ssize_t assign_domain_store(struct device *dev, matrix_mdev); ret = count; done: - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); return ret; } @@ -1061,7 +1108,7 @@ static ssize_t unassign_domain_store(struct device *dev, unsigned long apqi; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_mdev(matrix_mdev); /* If the KVM guest is running, disallow unassignment of domain */ if (matrix_mdev->kvm) { @@ -1087,7 +1134,7 @@ static ssize_t unassign_domain_store(struct device *dev, ret = count; done: - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); return ret; } static DEVICE_ATTR_WO(unassign_domain); @@ -1114,7 +1161,7 @@ static ssize_t assign_control_domain_store(struct device *dev, unsigned long id; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_mdev(matrix_mdev); /* If the KVM guest is running, disallow assignment of control domain */ if (matrix_mdev->kvm) { @@ -1140,7 +1187,7 @@ static ssize_t assign_control_domain_store(struct device *dev, vfio_ap_mdev_filter_cdoms(matrix_mdev); ret = count; done: - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); return ret; } static DEVICE_ATTR_WO(assign_control_domain); @@ -1168,7 +1215,7 @@ static ssize_t unassign_control_domain_store(struct device *dev, struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); unsigned long max_domid = matrix_mdev->matrix.adm_max; - mutex_lock(&matrix_dev->mdevs_lock); + get_update_locks_for_mdev(matrix_mdev); /* If a KVM guest is running, disallow unassignment of control domain */ if (matrix_mdev->kvm) { @@ -1191,7 +1238,7 @@ static ssize_t unassign_control_domain_store(struct device *dev, ret = count; done: - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); return ret; } static DEVICE_ATTR_WO(unassign_control_domain); From 2c1ee8983aa377fef4f725162f4f3aa5f9575d4a Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Thu, 17 Mar 2022 12:19:04 -0400 Subject: [PATCH 12/46] s390/vfio-ap: prepare for dynamic update of guest's APCB on queue probe/remove The callback functions for probing and removing a queue device must take and release the locks required to perform a dynamic update of a guest's APCB in the proper order. The proper order for taking the locks is: matrix_dev->guests_lock => kvm->lock => matrix_dev->mdevs_lock The proper order for releasing the locks is: matrix_dev->mdevs_lock => kvm->lock => matrix_dev->guests_lock A new helper function is introduced to be used by the probe callback to acquire the required locks. Since the probe callback only has access to a queue device when it is called, the helper function will find the ap_matrix_mdev object to which the queue device's APQN is assigned and return it so the KVM guest to which the mdev is attached can be dynamically updated. Note that in order to find the ap_matrix_mdev (matrix_mdev) object, it is necessary to search the matrix_dev->mdev_list. This presents a locking order dilemma because the matrix_dev->mdevs_lock can't be taken to protect against changes to the list while searching for the matrix_mdev to which a queue device's APQN is assigned. This is due to the fact that the proper locking order requires that the matrix_dev->mdevs_lock be taken after both the matrix_mdev->kvm->lock and the matrix_dev->mdevs_lock. Consequently, the matrix_dev->guests_lock will be used to protect against removal of a matrix_mdev object from the list while a queue device is being probed. This necessitates changes to the mdev probe/remove callback functions to take the matrix_dev->guests_lock prior to removing a matrix_mdev object from the list. A new macro is also introduced to acquire the locks required to dynamically update the guest's APCB in the proper order when a queue device is removed. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 124 ++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 39 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index dcd36d427b86..fd0f0f241197 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -124,6 +124,72 @@ static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_m mutex_unlock(&matrix_dev->guests_lock); } +/** + * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and + * acquire the locks required to update the APCB of + * the KVM guest to which the mdev is attached. + * + * @apqn: the APQN of a queue device. + * + * The proper locking order is: + * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM + * guest's APCB. + * 2. matrix_mdev->kvm->lock: required to update a guest's APCB + * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev + * + * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock + * will not be taken. + * + * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn + * is not assigned to an ap_matrix_mdev. + */ +static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn) +{ + struct ap_matrix_mdev *matrix_mdev; + + mutex_lock(&matrix_dev->guests_lock); + + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) && + test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) { + if (matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); + + mutex_lock(&matrix_dev->mdevs_lock); + + return matrix_mdev; + } + } + + mutex_lock(&matrix_dev->mdevs_lock); + + return NULL; +} + +/** + * get_update_locks_for_queue: get the locks required to update the APCB of the + * KVM guest to which the matrix mdev linked to a + * vfio_ap_queue object is attached. + * + * @q: a pointer to a vfio_ap_queue object. + * + * The proper locking order is: + * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a + * KVM guest's APCB. + * 2. q->matrix_mdev->kvm->lock: required to update a guest's APCB + * 3. matrix_dev->mdevs_lock: required to access data stored in matrix_mdev + * + * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock + * will not be taken. + */ +static inline void get_update_locks_for_queue(struct vfio_ap_queue *q) +{ + mutex_lock(&matrix_dev->guests_lock); + if (q->matrix_mdev && q->matrix_mdev->kvm) + mutex_lock(&q->matrix_mdev->kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); +} + /** * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a * hash table of queues assigned to a matrix mdev @@ -622,21 +688,17 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) matrix_mdev->pqap_hook = handle_pqap; vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); hash_init(matrix_mdev->qtable.queues); - dev_set_drvdata(&mdev->dev, matrix_mdev); - mutex_lock(&matrix_dev->mdevs_lock); - list_add(&matrix_mdev->node, &matrix_dev->mdev_list); - mutex_unlock(&matrix_dev->mdevs_lock); ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); if (ret) goto err_list; dev_set_drvdata(&mdev->dev, matrix_mdev); + mutex_lock(&matrix_dev->mdevs_lock); + list_add(&matrix_mdev->node, &matrix_dev->mdev_list); + mutex_unlock(&matrix_dev->mdevs_lock); return 0; err_list: - mutex_lock(&matrix_dev->mdevs_lock); - list_del(&matrix_mdev->node); - mutex_unlock(&matrix_dev->mdevs_lock); vfio_uninit_group_dev(&matrix_mdev->vdev); kfree(matrix_mdev); err_dec_available: @@ -699,11 +761,13 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) vfio_unregister_group_dev(&matrix_mdev->vdev); + mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); vfio_ap_mdev_reset_queues(matrix_mdev); vfio_ap_mdev_unlink_fr_queues(matrix_mdev); list_del(&matrix_mdev->node); mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); vfio_uninit_group_dev(&matrix_mdev->vdev); kfree(matrix_mdev); atomic_inc(&matrix_dev->available_instances); @@ -1701,32 +1765,11 @@ void vfio_ap_mdev_unregister(void) mdev_unregister_driver(&vfio_ap_matrix_driver); } -/* - * vfio_ap_queue_link_mdev - * - * @q: The queue to link with the matrix mdev. - * - * Links @q with the matrix mdev to which the queue's APQN is assigned. - */ -static void vfio_ap_queue_link_mdev(struct vfio_ap_queue *q) -{ - unsigned long apid = AP_QID_CARD(q->apqn); - unsigned long apqi = AP_QID_QUEUE(q->apqn); - struct ap_matrix_mdev *matrix_mdev; - - list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { - if (test_bit_inv(apid, matrix_mdev->matrix.apm) && - test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { - vfio_ap_mdev_link_queue(matrix_mdev, q); - break; - } - } -} - int vfio_ap_mdev_probe_queue(struct ap_device *apdev) { int ret; struct vfio_ap_queue *q; + struct ap_matrix_mdev *matrix_mdev; ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); if (ret) @@ -1736,17 +1779,18 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) if (!q) return -ENOMEM; - mutex_lock(&matrix_dev->mdevs_lock); q->apqn = to_ap_queue(&apdev->device)->qid; q->saved_isc = VFIO_AP_ISC_INVALID; - vfio_ap_queue_link_mdev(q); - if (q->matrix_mdev) { - vfio_ap_mdev_filter_matrix(q->matrix_mdev->matrix.apm, - q->matrix_mdev->matrix.aqm, - q->matrix_mdev); + matrix_mdev = get_update_locks_by_apqn(q->apqn); + + if (matrix_mdev) { + vfio_ap_mdev_link_queue(matrix_mdev, q); + vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, + matrix_mdev->matrix.aqm, + matrix_mdev); } dev_set_drvdata(&apdev->device, q); - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); return 0; } @@ -1755,12 +1799,14 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) { unsigned long apid; struct vfio_ap_queue *q; + struct ap_matrix_mdev *matrix_mdev; - mutex_lock(&matrix_dev->mdevs_lock); sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); q = dev_get_drvdata(&apdev->device); + get_update_locks_for_queue(q); + matrix_mdev = q->matrix_mdev; - if (q->matrix_mdev) { + if (matrix_mdev) { vfio_ap_unlink_queue_fr_mdev(q); apid = AP_QID_CARD(q->apqn); @@ -1771,5 +1817,5 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) vfio_ap_mdev_reset_queue(q, 1); dev_set_drvdata(&apdev->device, NULL); kfree(q); - mutex_unlock(&matrix_dev->mdevs_lock); + release_update_locks_for_mdev(matrix_mdev); } From 51dc562af09dfb1a0d3cc806f64f0d72704647b5 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Mon, 31 Jan 2022 18:35:47 -0500 Subject: [PATCH 13/46] s390/vfio-ap: allow hot plug/unplug of AP devices when assigned/unassigned Let's hot plug an adapter, domain or control domain into the guest when it is assigned to a matrix mdev that is attached to a KVM guest. Likewise, let's hot unplug an adapter, domain or control domain from the guest when it is unassigned from a matrix_mdev that is attached to a KVM guest. Whenever an assignment or unassignment of an adapter, domain or control domain is performed, the APQNs and control domains assigned to the matrix mdev will be filtered and assigned to the AP control block (APCB) that supplies the AP configuration to the guest so that no adapter, domain or control domain that is not in the host's AP configuration nor any APQN that does not reference a queue device bound to the vfio_ap device driver is assigned. After updating the APCB, if the mdev is in use by a KVM guest, it is hot plugged into the guest to dynamically provide access to the adapters, domains and control domains provided via the newly refreshed APCB. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 112 +++++++++++++++--------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index fd0f0f241197..561e68f69955 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -605,10 +605,25 @@ static void vfio_ap_matrix_init(struct ap_config_info *info, matrix->adm_max = info->apxa ? info->Nd : 15; } -static void vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) +static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev) { + if (matrix_mdev->kvm) + kvm_arch_crypto_set_masks(matrix_mdev->kvm, + matrix_mdev->shadow_apcb.apm, + matrix_mdev->shadow_apcb.aqm, + matrix_mdev->shadow_apcb.adm); +} + +static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) +{ + DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS); + + bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS); bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm, (unsigned long *)matrix_dev->info.adm, AP_DOMAINS); + + return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, + AP_DOMAINS); } /* @@ -623,17 +638,24 @@ static void vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) * driver, its APID will be filtered from the guest's APCB. The matrix * structure precludes filtering an individual APQN, so its APID will be * filtered. + * + * Return: a boolean value indicating whether the KVM guest's APCB was changed + * by the filtering or not. */ -static void vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, +static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, struct ap_matrix_mdev *matrix_mdev) { int ret; unsigned long apid, apqi, apqn; + DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); + DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); ret = ap_qci(&matrix_dev->info); if (ret) - return; + return false; + bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); + bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); /* @@ -665,6 +687,11 @@ static void vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, } } } + + return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, + AP_DEVICES) || + !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, + AP_DOMAINS); } static int vfio_ap_mdev_probe(struct mdev_device *mdev) @@ -940,17 +967,10 @@ static ssize_t assign_adapter_store(struct device *dev, int ret; unsigned long apid; DECLARE_BITMAP(apm_delta, AP_DEVICES); - struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); get_update_locks_for_mdev(matrix_mdev); - /* If the KVM guest is running, disallow assignment of adapter */ - if (matrix_mdev->kvm) { - ret = -EBUSY; - goto done; - } - ret = kstrtoul(buf, 0, &apid); if (ret) goto done; @@ -971,8 +991,11 @@ static ssize_t assign_adapter_store(struct device *dev, vfio_ap_mdev_link_adapter(matrix_mdev, apid); memset(apm_delta, 0, sizeof(apm_delta)); set_bit_inv(apid, apm_delta); - vfio_ap_mdev_filter_matrix(apm_delta, - matrix_mdev->matrix.aqm, matrix_mdev); + + if (vfio_ap_mdev_filter_matrix(apm_delta, + matrix_mdev->matrix.aqm, matrix_mdev)) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + ret = count; done: release_update_locks_for_mdev(matrix_mdev); @@ -1020,12 +1043,6 @@ static ssize_t unassign_adapter_store(struct device *dev, get_update_locks_for_mdev(matrix_mdev); - /* If the KVM guest is running, disallow unassignment of adapter */ - if (matrix_mdev->kvm) { - ret = -EBUSY; - goto done; - } - ret = kstrtoul(buf, 0, &apid); if (ret) goto done; @@ -1038,8 +1055,10 @@ static ssize_t unassign_adapter_store(struct device *dev, clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); vfio_ap_mdev_unlink_adapter(matrix_mdev, apid); - if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + } ret = count; done: @@ -1096,20 +1115,14 @@ static ssize_t assign_domain_store(struct device *dev, unsigned long apqi; DECLARE_BITMAP(aqm_delta, AP_DOMAINS); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - unsigned long max_apqi = matrix_mdev->matrix.aqm_max; get_update_locks_for_mdev(matrix_mdev); - /* If the KVM guest is running, disallow assignment of domain */ - if (matrix_mdev->kvm) { - ret = -EBUSY; - goto done; - } - ret = kstrtoul(buf, 0, &apqi); if (ret) goto done; - if (apqi > max_apqi) { + + if (apqi > matrix_mdev->matrix.aqm_max) { ret = -ENODEV; goto done; } @@ -1125,8 +1138,11 @@ static ssize_t assign_domain_store(struct device *dev, vfio_ap_mdev_link_domain(matrix_mdev, apqi); memset(aqm_delta, 0, sizeof(aqm_delta)); set_bit_inv(apqi, aqm_delta); - vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta, - matrix_mdev); + + if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta, + matrix_mdev)) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + ret = count; done: release_update_locks_for_mdev(matrix_mdev); @@ -1174,12 +1190,6 @@ static ssize_t unassign_domain_store(struct device *dev, get_update_locks_for_mdev(matrix_mdev); - /* If the KVM guest is running, disallow unassignment of domain */ - if (matrix_mdev->kvm) { - ret = -EBUSY; - goto done; - } - ret = kstrtoul(buf, 0, &apqi); if (ret) goto done; @@ -1192,8 +1202,10 @@ static ssize_t unassign_domain_store(struct device *dev, clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); vfio_ap_mdev_unlink_domain(matrix_mdev, apqi); - if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) + if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + } ret = count; @@ -1227,12 +1239,6 @@ static ssize_t assign_control_domain_store(struct device *dev, get_update_locks_for_mdev(matrix_mdev); - /* If the KVM guest is running, disallow assignment of control domain */ - if (matrix_mdev->kvm) { - ret = -EBUSY; - goto done; - } - ret = kstrtoul(buf, 0, &id); if (ret) goto done; @@ -1248,7 +1254,9 @@ static ssize_t assign_control_domain_store(struct device *dev, * number of control domains that can be assigned. */ set_bit_inv(id, matrix_mdev->matrix.adm); - vfio_ap_mdev_filter_cdoms(matrix_mdev); + if (vfio_ap_mdev_filter_cdoms(matrix_mdev)) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + ret = count; done: release_update_locks_for_mdev(matrix_mdev); @@ -1277,28 +1285,24 @@ static ssize_t unassign_control_domain_store(struct device *dev, int ret; unsigned long domid; struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - unsigned long max_domid = matrix_mdev->matrix.adm_max; get_update_locks_for_mdev(matrix_mdev); - /* If a KVM guest is running, disallow unassignment of control domain */ - if (matrix_mdev->kvm) { - ret = -EBUSY; - goto done; - } - ret = kstrtoul(buf, 0, &domid); if (ret) goto done; - if (domid > max_domid) { + + if (domid > matrix_mdev->matrix.adm_max) { ret = -ENODEV; goto done; } clear_bit_inv(domid, matrix_mdev->matrix.adm); - if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) + if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) { clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm); + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + } ret = count; done: @@ -1431,9 +1435,7 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; - kvm_arch_crypto_set_masks(kvm, matrix_mdev->shadow_apcb.apm, - matrix_mdev->shadow_apcb.aqm, - matrix_mdev->shadow_apcb.adm); + vfio_ap_mdev_update_guest_apcb(matrix_mdev); release_update_locks_for_kvm(kvm); } From 09d31ff78793ff007ec3b0a26000fcf29ad4928b Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Tue, 1 Feb 2022 10:52:29 -0500 Subject: [PATCH 14/46] s390/vfio-ap: hot plug/unplug of AP devices when probed/removed When an AP queue device is probed or removed, if the mediated device is attached to a KVM guest, the mediated device's adapter, domain and control domain bitmaps must be filtered to update the guest's APCB and if any changes are detected, the guest's APCB must then be hot plugged into the guest to reflect those changes to the guest. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 561e68f69955..8346ad4f1942 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1787,9 +1787,11 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) if (matrix_mdev) { vfio_ap_mdev_link_queue(matrix_mdev, q); - vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, - matrix_mdev->matrix.aqm, - matrix_mdev); + + if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, + matrix_mdev->matrix.aqm, + matrix_mdev)) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); } dev_set_drvdata(&apdev->device, q); release_update_locks_for_mdev(matrix_mdev); @@ -1799,7 +1801,7 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) void vfio_ap_mdev_remove_queue(struct ap_device *apdev) { - unsigned long apid; + unsigned long apid, apqi; struct vfio_ap_queue *q; struct ap_matrix_mdev *matrix_mdev; @@ -1812,8 +1814,17 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) vfio_ap_unlink_queue_fr_mdev(q); apid = AP_QID_CARD(q->apqn); - if (test_bit_inv(apid, q->matrix_mdev->shadow_apcb.apm)) - clear_bit_inv(apid, q->matrix_mdev->shadow_apcb.apm); + apqi = AP_QID_QUEUE(q->apqn); + + /* + * If the queue is assigned to the guest's APCB, then remove + * the adapter's APID from the APCB and hot it into the guest. + */ + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && + test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { + clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + } } vfio_ap_mdev_reset_queue(q, 1); From 70aeefe574cbf86528528832c615cae2701f2cf3 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Wed, 25 Aug 2021 15:00:25 -0400 Subject: [PATCH 15/46] s390/vfio-ap: reset queues after adapter/domain unassignment When an adapter or domain is unassigned from an mdev attached to a KVM guest, one or more of the guest's queues may get dynamically removed. Since the removed queues could get re-assigned to another mdev, they need to be reset. So, when an adapter or domain is unassigned from the mdev, the queues that are removed from the guest's AP configuration (APCB) will be reset. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 152 +++++++++++++++++++------- drivers/s390/crypto/vfio_ap_private.h | 2 + 2 files changed, 114 insertions(+), 40 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 8346ad4f1942..906bb8c7c4a1 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -30,9 +30,10 @@ #define AP_QUEUE_UNASSIGNED "unassigned" #define AP_QUEUE_IN_USE "in use" -static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev); +static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable); static struct vfio_ap_queue *vfio_ap_find_queue(int apqn); static const struct vfio_device_ops vfio_ap_matrix_dev_ops; +static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, unsigned int retry); /** * get_update_locks_for_kvm: Acquire the locks required to dynamically update a @@ -649,6 +650,7 @@ static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, unsigned long apid, apqi, apqn; DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); + struct vfio_ap_queue *q; ret = ap_qci(&matrix_dev->info); if (ret) @@ -679,8 +681,8 @@ static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, * hardware device. */ apqn = AP_MKQID(apid, apqi); - - if (!vfio_ap_mdev_get_queue(matrix_mdev, apqn)) { + q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); + if (!q || q->reset_rc) { clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); break; @@ -760,12 +762,6 @@ static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q) q->matrix_mdev = NULL; } -static void vfio_ap_mdev_unlink_queue(struct vfio_ap_queue *q) -{ - vfio_ap_unlink_queue_fr_mdev(q); - vfio_ap_unlink_mdev_fr_queue(q); -} - static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev) { struct vfio_ap_queue *q; @@ -790,7 +786,7 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); - vfio_ap_mdev_reset_queues(matrix_mdev); + vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); vfio_ap_mdev_unlink_fr_queues(matrix_mdev); list_del(&matrix_mdev->node); mutex_unlock(&matrix_dev->mdevs_lock); @@ -1004,20 +1000,72 @@ done: } static DEVICE_ATTR_WO(assign_adapter); +static struct vfio_ap_queue +*vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid, unsigned long apqi) +{ + struct vfio_ap_queue *q = NULL; + + q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); + /* If the queue is assigned to the matrix mdev, unlink it. */ + if (q) + vfio_ap_unlink_queue_fr_mdev(q); + + return q; +} + +/** + * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned + * adapter from the matrix mdev to which the + * adapter was assigned. + * @matrix_mdev: the matrix mediated device to which the adapter was assigned. + * @apid: the APID of the unassigned adapter. + * @qtable: table for storing queues associated with unassigned adapter. + */ static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev, - unsigned long apid) + unsigned long apid, + struct ap_queue_table *qtable) { unsigned long apqi; struct vfio_ap_queue *q; for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) { - q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); + q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); - if (q) - vfio_ap_mdev_unlink_queue(q); + if (q && qtable) { + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && + test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) + hash_add(qtable->queues, &q->mdev_qnode, + q->apqn); + } } } +static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev, + unsigned long apid) +{ + int loop_cursor; + struct vfio_ap_queue *q; + struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); + + hash_init(qtable->queues); + vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable); + + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { + clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + } + + vfio_ap_mdev_reset_queues(qtable); + + hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { + vfio_ap_unlink_mdev_fr_queue(q); + hash_del(&q->mdev_qnode); + } + + kfree(qtable); +} + /** * unassign_adapter_store - parses the APID from @buf and clears the * corresponding bit in the mediated matrix device's APM @@ -1053,13 +1101,7 @@ static ssize_t unassign_adapter_store(struct device *dev, } clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); - vfio_ap_mdev_unlink_adapter(matrix_mdev, apid); - - if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) { - clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); - vfio_ap_mdev_update_guest_apcb(matrix_mdev); - } - + vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid); ret = count; done: release_update_locks_for_mdev(matrix_mdev); @@ -1152,19 +1194,49 @@ done: static DEVICE_ATTR_WO(assign_domain); static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev, - unsigned long apqi) + unsigned long apqi, + struct ap_queue_table *qtable) { unsigned long apid; struct vfio_ap_queue *q; for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) { - q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi)); + q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi); - if (q) - vfio_ap_mdev_unlink_queue(q); + if (q && qtable) { + if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) && + test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) + hash_add(qtable->queues, &q->mdev_qnode, + q->apqn); + } } } +static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev, + unsigned long apqi) +{ + int loop_cursor; + struct vfio_ap_queue *q; + struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL); + + hash_init(qtable->queues); + vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable); + + if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { + clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + } + + vfio_ap_mdev_reset_queues(qtable); + + hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { + vfio_ap_unlink_mdev_fr_queue(q); + hash_del(&q->mdev_qnode); + } + + kfree(qtable); +} + /** * unassign_domain_store - parses the APQI from @buf and clears the * corresponding bit in the mediated matrix device's AQM @@ -1200,13 +1272,7 @@ static ssize_t unassign_domain_store(struct device *dev, } clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); - vfio_ap_mdev_unlink_domain(matrix_mdev, apqi); - - if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) { - clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm); - vfio_ap_mdev_update_guest_apcb(matrix_mdev); - } - + vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi); ret = count; done: @@ -1491,7 +1557,7 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) get_update_locks_for_kvm(kvm); kvm_arch_crypto_clear_masks(kvm); - vfio_ap_mdev_reset_queues(matrix_mdev); + vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); kvm_put_kvm(kvm); matrix_mdev->kvm = NULL; @@ -1525,9 +1591,9 @@ static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, if (!q) return 0; - retry_zapq: status = ap_zapq(q->apqn); + q->reset_rc = status.response_code; switch (status.response_code) { case AP_RESPONSE_NORMAL: ret = 0; @@ -1542,12 +1608,17 @@ retry_zapq: case AP_RESPONSE_Q_NOT_AVAIL: case AP_RESPONSE_DECONFIGURED: case AP_RESPONSE_CHECKSTOPPED: - WARN_ON_ONCE(status.irq_enabled); + WARN_ONCE(status.irq_enabled, + "PQAP/ZAPQ for %02x.%04x failed with rc=%u while IRQ enabled", + AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn), + status.response_code); ret = -EBUSY; goto free_resources; default: /* things are really broken, give up */ - WARN(true, "PQAP/ZAPQ completed with invalid rc (%x)\n", + WARN(true, + "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n", + AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn), status.response_code); return -EIO; } @@ -1559,7 +1630,8 @@ retry_zapq: msleep(20); status = ap_tapq(q->apqn, NULL); } - WARN_ON_ONCE(retry2 <= 0); + WARN_ONCE(retry2 <= 0, "unable to verify reset of queue %02x.%04x", + AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn)); free_resources: vfio_ap_free_aqic_resources(q); @@ -1567,12 +1639,12 @@ free_resources: return ret; } -static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev) +static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable) { int ret, loop_cursor, rc = 0; struct vfio_ap_queue *q; - hash_for_each(matrix_mdev->qtable.queues, loop_cursor, q, mdev_qnode) { + hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) { ret = vfio_ap_mdev_reset_queue(q, 1); /* * Regardless whether a queue turns out to be busy, or @@ -1656,7 +1728,7 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev, ret = vfio_ap_mdev_get_device_info(arg); break; case VFIO_DEVICE_RESET: - ret = vfio_ap_mdev_reset_queues(matrix_mdev); + ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable); break; default: ret = -EOPNOTSUPP; diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 97734b599583..dd81b3da0235 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -126,6 +126,7 @@ struct ap_matrix_mdev { * @apqn: the APQN of the AP queue device * @saved_isc: the guest ISC registered with the GIB interface * @mdev_qnode: allows the vfio_ap_queue struct to be added to a hashtable + * @reset_rc: the status response code from the last reset of the queue */ struct vfio_ap_queue { struct ap_matrix_mdev *matrix_mdev; @@ -134,6 +135,7 @@ struct vfio_ap_queue { #define VFIO_AP_ISC_INVALID 0xff unsigned char saved_isc; struct hlist_node mdev_qnode; + unsigned int reset_rc; }; int vfio_ap_mdev_register(void); From 3f85d1df26a9d9b0d1a5237c580e085c41d6bf50 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Thu, 30 Sep 2021 11:43:38 -0400 Subject: [PATCH 16/46] s390/vfio-ap: implement in-use callback for vfio_ap driver Let's implement the callback to indicate when an APQN is in use by the vfio_ap device driver. The callback is invoked whenever a change to the apmask or aqmask would result in one or more queue devices being removed from the driver. The vfio_ap device driver will indicate a resource is in use if the APQN of any of the queue devices to be removed are assigned to any of the matrix mdevs under the driver's control. There is potential for a deadlock condition between the matrix_dev->guests_lock used to lock the guest during assignment of adapters and domains and the ap_perms_mutex locked by the AP bus when changes are made to the sysfs apmask/aqmask attributes. The AP Perms lock controls access to the objects that store the adapter numbers (ap_perms) and domain numbers (aq_perms) for the sysfs /sys/bus/ap/apmask and /sys/bus/ap/aqmask attributes. These attributes identify which queues are reserved for the zcrypt default device drivers. Before allowing a bit to be removed from either mask, the AP bus must check with the vfio_ap device driver to verify that none of the queues are assigned to any of its mediated devices. The apmask/aqmask attributes can be written or read at any time from userspace, so care must be taken to prevent a deadlock with asynchronous operations that might be taking place in the vfio_ap device driver. For example, consider the following: 1. A system administrator assigns an adapter to a mediated device under the control of the vfio_ap device driver. The driver will need to first take the matrix_dev->guests_lock to potentially hot plug the adapter into the KVM guest. 2. At the same time, a system administrator sets a bit in the sysfs /sys/bus/ap/ap_mask attribute. To complete the operation, the AP bus must: a. Take the ap_perms_mutex lock to update the object storing the values for the /sys/bus/ap/ap_mask attribute. b. Call the vfio_ap device driver's in-use callback to verify that the queues now being reserved for the default zcrypt drivers are not assigned to a mediated device owned by the vfio_ap device driver. To do the verification, the in-use callback function takes the matrix_dev->guests_lock, but has to wait because it is already held by the operation in 1 above. 3. The vfio_ap device driver calls an AP bus function to verify that the new queues resulting from the assignment of the adapter in step 1 are not reserved for the default zcrypt device driver. This AP bus function tries to take the ap_perms_mutex lock but gets stuck waiting for the waiting for the lock due to step 2a above. Consequently, we have the following deadlock situation: matrix_dev->guests_lock locked (1) ap_perms_mutex lock locked (2a) Waiting for matrix_dev->gusts_lock (2b) which is currently held (1) Waiting for ap_perms_mutex lock (3) which is currently held (2a) To prevent this deadlock scenario, the function called in step 3 will no longer take the ap_perms_mutex lock and require the caller to take the lock. The lock will be the first taken by the adapter/domain assignment functions in the vfio_ap device driver to maintain the proper locking order. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/ap_bus.c | 31 +++++++++++---- drivers/s390/crypto/vfio_ap_drv.c | 1 + drivers/s390/crypto/vfio_ap_ops.c | 54 +++++++++++++++++++++++++++ drivers/s390/crypto/vfio_ap_private.h | 2 + 4 files changed, 80 insertions(+), 8 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 5c13d2079d96..7d5261113181 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -835,6 +835,17 @@ static void ap_bus_revise_bindings(void) bus_for_each_dev(&ap_bus_type, NULL, NULL, __ap_revise_reserved); } +/** + * ap_owned_by_def_drv: indicates whether an AP adapter is reserved for the + * default host driver or not. + * @card: the APID of the adapter card to check + * @queue: the APQI of the queue to check + * + * Note: the ap_perms_mutex must be locked by the caller of this function. + * + * Return: an int specifying whether the AP adapter is reserved for the host (1) + * or not (0). + */ int ap_owned_by_def_drv(int card, int queue) { int rc = 0; @@ -842,25 +853,31 @@ int ap_owned_by_def_drv(int card, int queue) if (card < 0 || card >= AP_DEVICES || queue < 0 || queue >= AP_DOMAINS) return -EINVAL; - mutex_lock(&ap_perms_mutex); - if (test_bit_inv(card, ap_perms.apm) && test_bit_inv(queue, ap_perms.aqm)) rc = 1; - mutex_unlock(&ap_perms_mutex); - return rc; } EXPORT_SYMBOL(ap_owned_by_def_drv); +/** + * ap_apqn_in_matrix_owned_by_def_drv: indicates whether every APQN contained in + * a set is reserved for the host drivers + * or not. + * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check + * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check + * + * Note: the ap_perms_mutex must be locked by the caller of this function. + * + * Return: an int specifying whether each APQN is reserved for the host (1) or + * not (0) + */ int ap_apqn_in_matrix_owned_by_def_drv(unsigned long *apm, unsigned long *aqm) { int card, queue, rc = 0; - mutex_lock(&ap_perms_mutex); - for (card = 0; !rc && card < AP_DEVICES; card++) if (test_bit_inv(card, apm) && test_bit_inv(card, ap_perms.apm)) @@ -869,8 +886,6 @@ int ap_apqn_in_matrix_owned_by_def_drv(unsigned long *apm, test_bit_inv(queue, ap_perms.aqm)) rc = 1; - mutex_unlock(&ap_perms_mutex); - return rc; } EXPORT_SYMBOL(ap_apqn_in_matrix_owned_by_def_drv); diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index db8ca7bb3696..2572fb0f0f54 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -46,6 +46,7 @@ static struct ap_device_id ap_queue_ids[] = { static struct ap_driver vfio_ap_drv = { .probe = vfio_ap_mdev_probe_queue, .remove = vfio_ap_mdev_remove_queue, + .in_use = vfio_ap_mdev_resource_in_use, .ids = ap_queue_ids, }; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 906bb8c7c4a1..4a864fc5b788 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -906,6 +906,21 @@ static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm, return 0; } +/** + * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are + * not reserved for the default zcrypt driver and + * are not assigned to another mdev. + * + * @matrix_mdev: the mdev to which the APQNs being validated are assigned. + * + * Return: One of the following values: + * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function, + * most likely -EBUSY indicating the ap_perms_mutex lock is already held. + * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the + * zcrypt default driver. + * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev + * o A zero indicating validation succeeded. + */ static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev) { if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm, @@ -955,6 +970,10 @@ static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev, * An APQN derived from the cross product of the APID being assigned * and the APQIs previously assigned is being used by another mediated * matrix device + * + * 5. -EAGAIN + * A lock required to validate the mdev's AP configuration could not + * be obtained. */ static ssize_t assign_adapter_store(struct device *dev, struct device_attribute *attr, @@ -965,6 +984,7 @@ static ssize_t assign_adapter_store(struct device *dev, DECLARE_BITMAP(apm_delta, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + mutex_lock(&ap_perms_mutex); get_update_locks_for_mdev(matrix_mdev); ret = kstrtoul(buf, 0, &apid); @@ -995,6 +1015,7 @@ static ssize_t assign_adapter_store(struct device *dev, ret = count; done: release_update_locks_for_mdev(matrix_mdev); + mutex_unlock(&ap_perms_mutex); return ret; } @@ -1148,6 +1169,10 @@ static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev, * An APQN derived from the cross product of the APQI being assigned * and the APIDs previously assigned is being used by another mediated * matrix device + * + * 5. -EAGAIN + * The lock required to validate the mdev's AP configuration could not + * be obtained. */ static ssize_t assign_domain_store(struct device *dev, struct device_attribute *attr, @@ -1158,6 +1183,7 @@ static ssize_t assign_domain_store(struct device *dev, DECLARE_BITMAP(aqm_delta, AP_DOMAINS); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + mutex_lock(&ap_perms_mutex); get_update_locks_for_mdev(matrix_mdev); ret = kstrtoul(buf, 0, &apqi); @@ -1188,6 +1214,7 @@ static ssize_t assign_domain_store(struct device *dev, ret = count; done: release_update_locks_for_mdev(matrix_mdev); + mutex_unlock(&ap_perms_mutex); return ret; } @@ -1904,3 +1931,30 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) kfree(q); release_update_locks_for_mdev(matrix_mdev); } + +/** + * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is + * assigned to a mediated device under the control + * of the vfio_ap device driver. + * + * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check. + * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check. + * + * Return: + * * -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are + * assigned to a mediated device under the control of the vfio_ap + * device driver. + * * Otherwise, return 0. + */ +int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) +{ + int ret; + + mutex_lock(&matrix_dev->guests_lock); + mutex_lock(&matrix_dev->mdevs_lock); + ret = vfio_ap_mdev_verify_no_sharing(apm, aqm); + mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); + + return ret; +} diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index dd81b3da0235..6726aa7caaf9 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -144,4 +144,6 @@ void vfio_ap_mdev_unregister(void); int vfio_ap_mdev_probe_queue(struct ap_device *queue); void vfio_ap_mdev_remove_queue(struct ap_device *queue); +int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm); + #endif /* _VFIO_AP_PRIVATE_H_ */ From f7f795c54d7117d06d002e06f9fca5baec3f293d Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Tue, 10 Sep 2019 11:17:45 -0400 Subject: [PATCH 17/46] s390/vfio-ap: sysfs attribute to display the guest's matrix The matrix of adapters and domains configured in a guest's APCB may differ from the matrix of adapters and domains assigned to the matrix mdev, so this patch introduces a sysfs attribute to display the matrix of adapters and domains that are or will be assigned to the APCB of a guest that is or will be using the matrix mdev. For a matrix mdev denoted by $uuid, the guest matrix can be displayed as follows: cat /sys/devices/vfio_ap/matrix/$uuid/guest_matrix Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_ops.c | 48 ++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 4a864fc5b788..6d722de44e8d 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1427,28 +1427,24 @@ static ssize_t control_domains_show(struct device *dev, } static DEVICE_ATTR_RO(control_domains); -static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, - char *buf) +static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf) { - struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); char *bufpos = buf; unsigned long apid; unsigned long apqi; unsigned long apid1; unsigned long apqi1; - unsigned long napm_bits = matrix_mdev->matrix.apm_max + 1; - unsigned long naqm_bits = matrix_mdev->matrix.aqm_max + 1; + unsigned long napm_bits = matrix->apm_max + 1; + unsigned long naqm_bits = matrix->aqm_max + 1; int nchars = 0; int n; - apid1 = find_first_bit_inv(matrix_mdev->matrix.apm, napm_bits); - apqi1 = find_first_bit_inv(matrix_mdev->matrix.aqm, naqm_bits); - - mutex_lock(&matrix_dev->mdevs_lock); + apid1 = find_first_bit_inv(matrix->apm, napm_bits); + apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits); if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) { - for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) { - for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, + for_each_set_bit_inv(apid, matrix->apm, napm_bits) { + for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) { n = sprintf(bufpos, "%02lx.%04lx\n", apid, apqi); @@ -1457,25 +1453,50 @@ static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, } } } else if (apid1 < napm_bits) { - for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) { + for_each_set_bit_inv(apid, matrix->apm, napm_bits) { n = sprintf(bufpos, "%02lx.\n", apid); bufpos += n; nchars += n; } } else if (apqi1 < naqm_bits) { - for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, naqm_bits) { + for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) { n = sprintf(bufpos, ".%04lx\n", apqi); bufpos += n; nchars += n; } } + return nchars; +} + +static ssize_t matrix_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + ssize_t nchars; + struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + + mutex_lock(&matrix_dev->mdevs_lock); + nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf); mutex_unlock(&matrix_dev->mdevs_lock); return nchars; } static DEVICE_ATTR_RO(matrix); +static ssize_t guest_matrix_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t nchars; + struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); + + mutex_lock(&matrix_dev->mdevs_lock); + nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf); + mutex_unlock(&matrix_dev->mdevs_lock); + + return nchars; +} +static DEVICE_ATTR_RO(guest_matrix); + static struct attribute *vfio_ap_mdev_attrs[] = { &dev_attr_assign_adapter.attr, &dev_attr_unassign_adapter.attr, @@ -1485,6 +1506,7 @@ static struct attribute *vfio_ap_mdev_attrs[] = { &dev_attr_unassign_control_domain.attr, &dev_attr_control_domains.attr, &dev_attr_matrix.attr, + &dev_attr_guest_matrix.attr, NULL, }; From eeb386aeb5b7c8d2dae6a3ba49255d8a97803182 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Mon, 4 Apr 2022 09:40:10 -0400 Subject: [PATCH 18/46] s390/vfio-ap: handle config changed and scan complete notification This patch implements two new AP driver callbacks: void (*on_config_changed)(struct ap_config_info *new_config_info, struct ap_config_info *old_config_info); void (*on_scan_complete)(struct ap_config_info *new_config_info, struct ap_config_info *old_config_info); The on_config_changed callback is invoked at the start of the AP bus scan function when it determines that the host AP configuration information has changed since the previous scan. The vfio_ap device driver registers a callback function for this callback that performs the following operations: 1. Unplugs the adapters, domains and control domains removed from the host's AP configuration from the guests to which they are assigned in a single operation. 2. Stores bitmaps identifying the adapters, domains and control domains added to the host's AP configuration with the structure representing the mediated device. When the vfio_ap device driver's probe callback is subsequently invoked, the probe function will recognize that the queue is being probed due to a change in the host's AP configuration and the plugging of the queue into the guest will be bypassed. The on_scan_complete callback is invoked after the ap bus scan is completed if the host AP configuration data has changed. The vfio_ap device driver registers a callback function for this callback that hot plugs each queue and control domain added to the AP configuration for each guest using them in a single hot plug operation. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- drivers/s390/crypto/vfio_ap_drv.c | 2 + drivers/s390/crypto/vfio_ap_ops.c | 341 +++++++++++++++++++++++++- drivers/s390/crypto/vfio_ap_private.h | 12 + 3 files changed, 350 insertions(+), 5 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index 2572fb0f0f54..f43cfeabd2cc 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -47,6 +47,8 @@ static struct ap_driver vfio_ap_drv = { .probe = vfio_ap_mdev_probe_queue, .remove = vfio_ap_mdev_remove_queue, .in_use = vfio_ap_mdev_resource_in_use, + .on_config_changed = vfio_ap_on_cfg_changed, + .on_scan_complete = vfio_ap_on_scan_complete, .ids = ap_queue_ids, }; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 6d722de44e8d..729a44d8955f 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -646,16 +646,11 @@ static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, struct ap_matrix_mdev *matrix_mdev) { - int ret; unsigned long apid, apqi, apqn; DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); struct vfio_ap_queue *q; - ret = ap_qci(&matrix_dev->info); - if (ret) - return false; - bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); @@ -1980,3 +1975,339 @@ int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) return ret; } + +/** + * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control + * domains that have been removed from the host's + * AP configuration from a guest. + * + * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest. + * @aprem: the adapters that have been removed from the host's AP configuration + * @aqrem: the domains that have been removed from the host's AP configuration + * @cdrem: the control domains that have been removed from the host's AP + * configuration. + */ +static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev, + unsigned long *aprem, + unsigned long *aqrem, + unsigned long *cdrem) +{ + int do_hotplug = 0; + + if (!bitmap_empty(aprem, AP_DEVICES)) { + do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm, + matrix_mdev->shadow_apcb.apm, + aprem, AP_DEVICES); + } + + if (!bitmap_empty(aqrem, AP_DOMAINS)) { + do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm, + matrix_mdev->shadow_apcb.aqm, + aqrem, AP_DEVICES); + } + + if (!bitmap_empty(cdrem, AP_DOMAINS)) + do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm, + matrix_mdev->shadow_apcb.adm, + cdrem, AP_DOMAINS); + + if (do_hotplug) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); +} + +/** + * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters, + * domains and control domains that have been removed + * from the host AP configuration and unplugs them + * from those guests. + * + * @ap_remove: bitmap specifying which adapters have been removed from the host + * config. + * @aq_remove: bitmap specifying which domains have been removed from the host + * config. + * @cd_remove: bitmap specifying which control domains have been removed from + * the host config. + */ +static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, + unsigned long *aq_remove, + unsigned long *cd_remove) +{ + struct ap_matrix_mdev *matrix_mdev; + DECLARE_BITMAP(aprem, AP_DEVICES); + DECLARE_BITMAP(aqrem, AP_DOMAINS); + DECLARE_BITMAP(cdrem, AP_DOMAINS); + int do_remove = 0; + + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + mutex_lock(&matrix_mdev->kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); + + do_remove |= bitmap_and(aprem, ap_remove, + matrix_mdev->matrix.apm, + AP_DEVICES); + do_remove |= bitmap_and(aqrem, aq_remove, + matrix_mdev->matrix.aqm, + AP_DOMAINS); + do_remove |= bitmap_andnot(cdrem, cd_remove, + matrix_mdev->matrix.adm, + AP_DOMAINS); + + if (do_remove) + vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem, + cdrem); + + mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_mdev->kvm->lock); + } +} + +/** + * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and + * control domains from the host AP configuration + * by unplugging them from the guests that are + * using them. + * @cur_config_info: the current host AP configuration information + * @prev_config_info: the previous host AP configuration information + */ +static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info, + struct ap_config_info *prev_config_info) +{ + int do_remove; + DECLARE_BITMAP(aprem, AP_DEVICES); + DECLARE_BITMAP(aqrem, AP_DOMAINS); + DECLARE_BITMAP(cdrem, AP_DOMAINS); + + do_remove = bitmap_andnot(aprem, + (unsigned long *)prev_config_info->apm, + (unsigned long *)cur_config_info->apm, + AP_DEVICES); + do_remove |= bitmap_andnot(aqrem, + (unsigned long *)prev_config_info->aqm, + (unsigned long *)cur_config_info->aqm, + AP_DEVICES); + do_remove |= bitmap_andnot(cdrem, + (unsigned long *)prev_config_info->adm, + (unsigned long *)cur_config_info->adm, + AP_DEVICES); + + if (do_remove) + vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem); +} + +/** + * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that + * are older than AP type 10 (CEX4). + * @apm: a bitmap of the APIDs to examine + * @aqm: a bitmap of the APQIs of the queues to query for the AP type. + */ +static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm) +{ + bool apid_cleared; + struct ap_queue_status status; + unsigned long apid, apqi, info; + int qtype, qtype_mask = 0xff000000; + + for_each_set_bit_inv(apid, apm, AP_DEVICES) { + apid_cleared = false; + + for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) { + status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info); + switch (status.response_code) { + /* + * According to the architecture in each case + * below, the queue's info should be filled. + */ + case AP_RESPONSE_NORMAL: + case AP_RESPONSE_RESET_IN_PROGRESS: + case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: + case AP_RESPONSE_BUSY: + qtype = info & qtype_mask; + + /* + * The vfio_ap device driver only + * supports CEX4 and newer adapters, so + * remove the APID if the adapter is + * older than a CEX4. + */ + if (qtype < AP_DEVICE_TYPE_CEX4) { + clear_bit_inv(apid, apm); + apid_cleared = true; + } + + break; + + default: + /* + * If we don't know the adapter type, + * clear its APID since it can't be + * determined whether the vfio_ap + * device driver supports it. + */ + clear_bit_inv(apid, apm); + apid_cleared = true; + break; + } + + /* + * If we've already cleared the APID from the apm, there + * is no need to continue examining the remainin AP + * queues to determine the type of the adapter. + */ + if (apid_cleared) + continue; + } + } +} + +/** + * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and + * control domains that have been added to the host's + * AP configuration for each matrix mdev to which they + * are assigned. + * + * @apm_add: a bitmap specifying the adapters that have been added to the AP + * configuration. + * @aqm_add: a bitmap specifying the domains that have been added to the AP + * configuration. + * @adm_add: a bitmap specifying the control domains that have been added to the + * AP configuration. + */ +static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, + unsigned long *adm_add) +{ + struct ap_matrix_mdev *matrix_mdev; + + if (list_empty(&matrix_dev->mdev_list)) + return; + + vfio_ap_filter_apid_by_qtype(apm_add, aqm_add); + + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + bitmap_and(matrix_mdev->apm_add, + matrix_mdev->matrix.apm, apm_add, AP_DEVICES); + bitmap_and(matrix_mdev->aqm_add, + matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS); + bitmap_and(matrix_mdev->adm_add, + matrix_mdev->matrix.adm, adm_add, AP_DEVICES); + } +} + +/** + * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and + * control domains to the host AP configuration + * by updating the bitmaps that specify what adapters, + * domains and control domains have been added so they + * can be hot plugged into the guest when the AP bus + * scan completes (see vfio_ap_on_scan_complete + * function). + * @cur_config_info: the current AP configuration information + * @prev_config_info: the previous AP configuration information + */ +static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info, + struct ap_config_info *prev_config_info) +{ + bool do_add; + DECLARE_BITMAP(apm_add, AP_DEVICES); + DECLARE_BITMAP(aqm_add, AP_DOMAINS); + DECLARE_BITMAP(adm_add, AP_DOMAINS); + + do_add = bitmap_andnot(apm_add, + (unsigned long *)cur_config_info->apm, + (unsigned long *)prev_config_info->apm, + AP_DEVICES); + do_add |= bitmap_andnot(aqm_add, + (unsigned long *)cur_config_info->aqm, + (unsigned long *)prev_config_info->aqm, + AP_DOMAINS); + do_add |= bitmap_andnot(adm_add, + (unsigned long *)cur_config_info->adm, + (unsigned long *)prev_config_info->adm, + AP_DOMAINS); + + if (do_add) + vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add); +} + +/** + * vfio_ap_on_cfg_changed - handles notification of changes to the host AP + * configuration. + * + * @cur_cfg_info: the current host AP configuration + * @prev_cfg_info: the previous host AP configuration + */ +void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, + struct ap_config_info *prev_cfg_info) +{ + if (!cur_cfg_info || !prev_cfg_info) + return; + + mutex_lock(&matrix_dev->guests_lock); + + vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info); + vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info); + memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info)); + + mutex_unlock(&matrix_dev->guests_lock); +} + +static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) +{ + bool do_hotplug = false; + int filter_domains = 0; + int filter_adapters = 0; + DECLARE_BITMAP(apm, AP_DEVICES); + DECLARE_BITMAP(aqm, AP_DOMAINS); + + mutex_lock(&matrix_mdev->kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); + + filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm, + matrix_mdev->apm_add, AP_DEVICES); + filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm, + matrix_mdev->aqm_add, AP_DOMAINS); + + if (filter_adapters && filter_domains) + do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev); + else if (filter_adapters) + do_hotplug |= + vfio_ap_mdev_filter_matrix(apm, + matrix_mdev->shadow_apcb.aqm, + matrix_mdev); + else + do_hotplug |= + vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm, + aqm, matrix_mdev); + + if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add, + AP_DOMAINS)) + do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev); + + if (do_hotplug) + vfio_ap_mdev_update_guest_apcb(matrix_mdev); + + mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_mdev->kvm->lock); +} + +void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, + struct ap_config_info *old_config_info) +{ + struct ap_matrix_mdev *matrix_mdev; + + mutex_lock(&matrix_dev->guests_lock); + + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) && + bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) && + bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS)) + continue; + + vfio_ap_mdev_hot_plug_cfg(matrix_mdev); + bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES); + bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS); + bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS); + } + + mutex_unlock(&matrix_dev->guests_lock); +} diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 6726aa7caaf9..2d03871597bf 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -105,6 +105,10 @@ struct ap_queue_table { * PQAP(AQIC) instruction. * @mdev: the mediated device * @qtable: table of queues (struct vfio_ap_queue) assigned to the mdev + * @apm_add: bitmap of APIDs added to the host's AP configuration + * @aqm_add: bitmap of APQIs added to the host's AP configuration + * @adm_add: bitmap of control domain numbers added to the host's AP + * configuration */ struct ap_matrix_mdev { struct vfio_device vdev; @@ -116,6 +120,9 @@ struct ap_matrix_mdev { crypto_hook pqap_hook; struct mdev_device *mdev; struct ap_queue_table qtable; + DECLARE_BITMAP(apm_add, AP_DEVICES); + DECLARE_BITMAP(aqm_add, AP_DOMAINS); + DECLARE_BITMAP(adm_add, AP_DOMAINS); }; /** @@ -146,4 +153,9 @@ void vfio_ap_mdev_remove_queue(struct ap_device *queue); int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm); +void vfio_ap_on_cfg_changed(struct ap_config_info *new_config_info, + struct ap_config_info *old_config_info); +void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, + struct ap_config_info *old_config_info); + #endif /* _VFIO_AP_PRIVATE_H_ */ From cb269e0aba7c3faef65a3b5d3a5be8b053f191a0 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Mon, 17 Aug 2020 15:19:44 -0400 Subject: [PATCH 19/46] s390/vfio-ap: update docs to include dynamic config support Update the documentation in vfio-ap.rst to include information about the AP dynamic configuration support (e.g., hot plug of adapters, domains and control domains via the matrix mediated device's sysfs assignment attributes). This patch also makes a few minor tweaks to make corrections and clarifications. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- Documentation/s390/vfio-ap.rst | 494 +++++++++++++++++++++++---------- 1 file changed, 347 insertions(+), 147 deletions(-) diff --git a/Documentation/s390/vfio-ap.rst b/Documentation/s390/vfio-ap.rst index f57ae621f33e..f4b8748ab9a8 100644 --- a/Documentation/s390/vfio-ap.rst +++ b/Documentation/s390/vfio-ap.rst @@ -123,27 +123,24 @@ Let's now take a look at how AP instructions executed on a guest are interpreted by the hardware. A satellite control block called the Crypto Control Block (CRYCB) is attached to -our main hardware virtualization control block. The CRYCB contains three fields -to identify the adapters, usage domains and control domains assigned to the KVM -guest: +our main hardware virtualization control block. The CRYCB contains an AP Control +Block (APCB) that has three fields to identify the adapters, usage domains and +control domains assigned to the KVM guest: * The AP Mask (APM) field is a bit mask that identifies the AP adapters assigned - to the KVM guest. Each bit in the mask, from left to right (i.e. from most - significant to least significant bit in big endian order), corresponds to + to the KVM guest. Each bit in the mask, from left to right, corresponds to an APID from 0-255. If a bit is set, the corresponding adapter is valid for use by the KVM guest. * The AP Queue Mask (AQM) field is a bit mask identifying the AP usage domains - assigned to the KVM guest. Each bit in the mask, from left to right (i.e. from - most significant to least significant bit in big endian order), corresponds to - an AP queue index (APQI) from 0-255. If a bit is set, the corresponding queue - is valid for use by the KVM guest. + assigned to the KVM guest. Each bit in the mask, from left to right, + corresponds to an AP queue index (APQI) from 0-255. If a bit is set, the + corresponding queue is valid for use by the KVM guest. * The AP Domain Mask field is a bit mask that identifies the AP control domains assigned to the KVM guest. The ADM bit mask controls which domains can be changed by an AP command-request message sent to a usage domain from the - guest. Each bit in the mask, from left to right (i.e. from most significant to - least significant bit in big endian order), corresponds to a domain from + guest. Each bit in the mask, from left to right, corresponds to a domain from 0-255. If a bit is set, the corresponding domain can be modified by an AP command-request message sent to a usage domain. @@ -151,10 +148,10 @@ If you recall from the description of an AP Queue, AP instructions include an APQN to identify the AP queue to which an AP command-request message is to be sent (NQAP and PQAP instructions), or from which a command-reply message is to be received (DQAP instruction). The validity of an APQN is defined by the matrix -calculated from the APM and AQM; it is the cross product of all assigned adapter -numbers (APM) with all assigned queue indexes (AQM). For example, if adapters 1 -and 2 and usage domains 5 and 6 are assigned to a guest, the APQNs (1,5), (1,6), -(2,5) and (2,6) will be valid for the guest. +calculated from the APM and AQM; it is the Cartesian product of all assigned +adapter numbers (APM) with all assigned queue indexes (AQM). For example, if +adapters 1 and 2 and usage domains 5 and 6 are assigned to a guest, the APQNs +(1,5), (1,6), (2,5) and (2,6) will be valid for the guest. The APQNs can provide secure key functionality - i.e., a private key is stored on the adapter card for each of its domains - so each APQN must be assigned to @@ -192,7 +189,7 @@ The design introduces three new objects: 1. AP matrix device 2. VFIO AP device driver (vfio_ap.ko) -3. VFIO AP mediated matrix pass-through device +3. VFIO AP mediated pass-through device The VFIO AP device driver ------------------------- @@ -200,12 +197,13 @@ The VFIO AP (vfio_ap) device driver serves the following purposes: 1. Provides the interfaces to secure APQNs for exclusive use of KVM guests. -2. Sets up the VFIO mediated device interfaces to manage a mediated matrix +2. Sets up the VFIO mediated device interfaces to manage a vfio_ap mediated device and creates the sysfs interfaces for assigning adapters, usage domains, and control domains comprising the matrix for a KVM guest. -3. Configures the APM, AQM and ADM in the CRYCB referenced by a KVM guest's - SIE state description to grant the guest access to a matrix of AP devices +3. Configures the APM, AQM and ADM in the APCB contained in the CRYCB referenced + by a KVM guest's SIE state description to grant the guest access to a matrix + of AP devices Reserve APQNs for exclusive use of KVM guests --------------------------------------------- @@ -235,10 +233,10 @@ reserved:: | | 8 probe | | +--------^---------+ +--^--^------------+ 6 edit | | | - apmask | +-----------------------------+ | 9 mdev create + apmask | +-----------------------------+ | 11 mdev create aqmask | | 1 modprobe | +--------+-----+---+ +----------------+-+ +----------------+ - | | | |8 create | mediated | + | | | |10 create| mediated | | admin | | VFIO device core |---------> matrix | | + | | | device | +------+-+---------+ +--------^---------+ +--------^-------+ @@ -246,14 +244,14 @@ reserved:: | | 9 create vfio_ap-passthrough | | | +------------------------------+ | +-------------------------------------------------------------+ - 10 assign adapter/domain/control domain + 12 assign adapter/domain/control domain The process for reserving an AP queue for use by a KVM guest is: 1. The administrator loads the vfio_ap device driver 2. The vfio-ap driver during its initialization will register a single 'matrix' device with the device core. This will serve as the parent device for - all mediated matrix devices used to configure an AP matrix for a guest. + all vfio_ap mediated devices used to configure an AP matrix for a guest. 3. The /sys/devices/vfio_ap/matrix device is created by the device core 4. The vfio_ap device driver will register with the AP bus for AP queue devices of type 10 and higher (CEX4 and newer). The driver will provide the vfio_ap @@ -269,24 +267,24 @@ The process for reserving an AP queue for use by a KVM guest is: default zcrypt cex4queue driver. 8. The AP bus probes the vfio_ap device driver to bind the queues reserved for it. -9. The administrator creates a passthrough type mediated matrix device to be +9. The administrator creates a passthrough type vfio_ap mediated device to be used by a guest 10. The administrator assigns the adapters, usage domains and control domains to be exclusively used by a guest. Set up the VFIO mediated device interfaces ------------------------------------------ -The VFIO AP device driver utilizes the common interface of the VFIO mediated +The VFIO AP device driver utilizes the common interfaces of the VFIO mediated device core driver to: -* Register an AP mediated bus driver to add a mediated matrix device to and +* Register an AP mediated bus driver to add a vfio_ap mediated device to and remove it from a VFIO group. -* Create and destroy a mediated matrix device -* Add a mediated matrix device to and remove it from the AP mediated bus driver -* Add a mediated matrix device to and remove it from an IOMMU group +* Create and destroy a vfio_ap mediated device +* Add a vfio_ap mediated device to and remove it from the AP mediated bus driver +* Add a vfio_ap mediated device to and remove it from an IOMMU group The following high-level block diagram shows the main components and interfaces -of the VFIO AP mediated matrix device driver:: +of the VFIO AP mediated device driver:: +-------------+ | | @@ -343,7 +341,7 @@ matrix device. * device_api: the mediated device type's API * available_instances: - the number of mediated matrix passthrough devices + the number of vfio_ap mediated passthrough devices that can be created * device_api: specifies the VFIO API @@ -351,29 +349,37 @@ matrix device. This attribute group identifies the user-defined sysfs attributes of the mediated device. When a device is registered with the VFIO mediated device framework, the sysfs attribute files identified in the 'mdev_attr_groups' - structure will be created in the mediated matrix device's directory. The - sysfs attributes for a mediated matrix device are: + structure will be created in the vfio_ap mediated device's directory. The + sysfs attributes for a vfio_ap mediated device are: assign_adapter / unassign_adapter: Write-only attributes for assigning/unassigning an AP adapter to/from the - mediated matrix device. To assign/unassign an adapter, the APID of the - adapter is echoed to the respective attribute file. + vfio_ap mediated device. To assign/unassign an adapter, the APID of the + adapter is echoed into the respective attribute file. assign_domain / unassign_domain: Write-only attributes for assigning/unassigning an AP usage domain to/from - the mediated matrix device. To assign/unassign a domain, the domain - number of the usage domain is echoed to the respective attribute + the vfio_ap mediated device. To assign/unassign a domain, the domain + number of the usage domain is echoed into the respective attribute file. matrix: - A read-only file for displaying the APQNs derived from the cross product - of the adapter and domain numbers assigned to the mediated matrix device. + A read-only file for displaying the APQNs derived from the Cartesian + product of the adapter and domain numbers assigned to the vfio_ap mediated + device. + guest_matrix: + A read-only file for displaying the APQNs derived from the Cartesian + product of the adapter and domain numbers assigned to the APM and AQM + fields respectively of the KVM guest's CRYCB. This may differ from the + the APQNs assigned to the vfio_ap mediated device if any APQN does not + reference a queue device bound to the vfio_ap device driver (i.e., the + queue is not in the host's AP configuration). assign_control_domain / unassign_control_domain: Write-only attributes for assigning/unassigning an AP control domain - to/from the mediated matrix device. To assign/unassign a control domain, - the ID of the domain to be assigned/unassigned is echoed to the respective - attribute file. + to/from the vfio_ap mediated device. To assign/unassign a control domain, + the ID of the domain to be assigned/unassigned is echoed into the + respective attribute file. control_domains: A read-only file for displaying the control domain numbers assigned to the - mediated matrix device. + vfio_ap mediated device. * functions: @@ -383,45 +389,75 @@ matrix device. * Store the reference to the KVM structure for the guest using the mdev * Store the AP matrix configuration for the adapters, domains, and control domains assigned via the corresponding sysfs attributes files + * Store the AP matrix configuration for the adapters, domains and control + domains available to a guest. A guest may not be provided access to APQNs + referencing queue devices that do not exist, or are not bound to the + vfio_ap device driver. remove: - deallocates the mediated matrix device's ap_matrix_mdev structure. This will - be allowed only if a running guest is not using the mdev. + deallocates the vfio_ap mediated device's ap_matrix_mdev structure. + This will be allowed only if a running guest is not using the mdev. * callback interfaces - open: + open_device: The vfio_ap driver uses this callback to register a - VFIO_GROUP_NOTIFY_SET_KVM notifier callback function for the mdev matrix - device. The open is invoked when QEMU connects the VFIO iommu group - for the mdev matrix device to the MDEV bus. Access to the KVM structure used - to configure the KVM guest is provided via this callback. The KVM structure, - is used to configure the guest's access to the AP matrix defined via the - mediated matrix device's sysfs attribute files. - release: - unregisters the VFIO_GROUP_NOTIFY_SET_KVM notifier callback function for the - mdev matrix device and deconfigures the guest's AP matrix. + VFIO_GROUP_NOTIFY_SET_KVM notifier callback function for the matrix mdev + devices. The open_device callback is invoked by userspace to connect the + VFIO iommu group for the matrix mdev device to the MDEV bus. Access to the + KVM structure used to configure the KVM guest is provided via this callback. + The KVM structure, is used to configure the guest's access to the AP matrix + defined via the vfio_ap mediated device's sysfs attribute files. -Configure the APM, AQM and ADM in the CRYCB -------------------------------------------- -Configuring the AP matrix for a KVM guest will be performed when the + close_device: + unregisters the VFIO_GROUP_NOTIFY_SET_KVM notifier callback function for the + matrix mdev device and deconfigures the guest's AP matrix. + + ioctl: + this callback handles the VFIO_DEVICE_GET_INFO and VFIO_DEVICE_RESET ioctls + defined by the vfio framework. + +Configure the guest's AP resources +---------------------------------- +Configuring the AP resources for a KVM guest will be performed when the VFIO_GROUP_NOTIFY_SET_KVM notifier callback is invoked. The notifier -function is called when QEMU connects to KVM. The guest's AP matrix is -configured via it's CRYCB by: +function is called when userspace connects to KVM. The guest's AP resources are +configured via it's APCB by: * Setting the bits in the APM corresponding to the APIDs assigned to the - mediated matrix device via its 'assign_adapter' interface. + vfio_ap mediated device via its 'assign_adapter' interface. * Setting the bits in the AQM corresponding to the domains assigned to the - mediated matrix device via its 'assign_domain' interface. + vfio_ap mediated device via its 'assign_domain' interface. * Setting the bits in the ADM corresponding to the domain dIDs assigned to the - mediated matrix device via its 'assign_control_domains' interface. + vfio_ap mediated device via its 'assign_control_domains' interface. + +The linux device model precludes passing a device through to a KVM guest that +is not bound to the device driver facilitating its pass-through. Consequently, +an APQN that does not reference a queue device bound to the vfio_ap device +driver will not be assigned to a KVM guest's matrix. The AP architecture, +however, does not provide a means to filter individual APQNs from the guest's +matrix, so the adapters, domains and control domains assigned to vfio_ap +mediated device via its sysfs 'assign_adapter', 'assign_domain' and +'assign_control_domain' interfaces will be filtered before providing the AP +configuration to a guest: + +* The APIDs of the adapters, the APQIs of the domains and the domain numbers of + the control domains assigned to the matrix mdev that are not also assigned to + the host's AP configuration will be filtered. + +* Each APQN derived from the Cartesian product of the APIDs and APQIs assigned + to the vfio_ap mdev is examined and if any one of them does not reference a + queue device bound to the vfio_ap device driver, the adapter will not be + plugged into the guest (i.e., the bit corresponding to its APID will not be + set in the APM of the guest's APCB). The CPU model features for AP ----------------------------- -The AP stack relies on the presence of the AP instructions as well as two -facilities: The AP Facilities Test (APFT) facility; and the AP Query -Configuration Information (QCI) facility. These features/facilities are made -available to a KVM guest via the following CPU model features: +The AP stack relies on the presence of the AP instructions as well as three +facilities: The AP Facilities Test (APFT) facility; the AP Query +Configuration Information (QCI) facility; and the AP Queue Interruption Control +facility. These features/facilities are made available to a KVM guest via the +following CPU model features: 1. ap: Indicates whether the AP instructions are installed on the guest. This feature will be enabled by KVM only if the AP instructions are installed @@ -435,24 +471,28 @@ available to a KVM guest via the following CPU model features: can be made available to the guest only if it is available on the host (i.e., facility bit 12 is set). +4. apqi: Indicates AP Queue Interruption Control faclity is available on the + guest. This facility can be made available to the guest only if it is + available on the host (i.e., facility bit 65 is set). + Note: If the user chooses to specify a CPU model different than the 'host' model to QEMU, the CPU model features and facilities need to be turned on explicitly; for example:: - /usr/bin/qemu-system-s390x ... -cpu z13,ap=on,apqci=on,apft=on + /usr/bin/qemu-system-s390x ... -cpu z13,ap=on,apqci=on,apft=on,apqi=on A guest can be precluded from using AP features/facilities by turning them off explicitly; for example:: - /usr/bin/qemu-system-s390x ... -cpu host,ap=off,apqci=off,apft=off + /usr/bin/qemu-system-s390x ... -cpu host,ap=off,apqci=off,apft=off,apqi=off Note: If the APFT facility is turned off (apft=off) for the guest, the guest -will not see any AP devices. The zcrypt device drivers that register for type 10 -and newer AP devices - i.e., the cex4card and cex4queue device drivers - need -the APFT facility to ascertain the facilities installed on a given AP device. If -the APFT facility is not installed on the guest, then the probe of device -drivers will fail since only type 10 and newer devices can be configured for -guest use. +will not see any AP devices. The zcrypt device drivers on the guest that +register for type 10 and newer AP devices - i.e., the cex4card and cex4queue +device drivers - need the APFT facility to ascertain the facilities installed on +a given AP device. If the APFT facility is not installed on the guest, then no +adapter or domain devices will get created by the AP bus running on the +guest because only type 10 and newer devices can be configured for guest use. Example ======= @@ -471,7 +511,7 @@ CARD.DOMAIN TYPE MODE 05.00ab CEX5C CCA-Coproc 06 CEX5A Accelerator 06.0004 CEX5A Accelerator -06.00ab CEX5C CCA-Coproc +06.00ab CEX5A Accelerator =========== ===== ============ Guest2 @@ -479,9 +519,9 @@ Guest2 =========== ===== ============ CARD.DOMAIN TYPE MODE =========== ===== ============ -05 CEX5A Accelerator -05.0047 CEX5A Accelerator -05.00ff CEX5A Accelerator +05 CEX5C CCA-Coproc +05.0047 CEX5C CCA-Coproc +05.00ff CEX5C CCA-Coproc =========== ===== ============ Guest3 @@ -529,40 +569,56 @@ These are the steps: 2. Secure the AP queues to be used by the three guests so that the host can not access them. To secure them, there are two sysfs files that specify - bitmasks marking a subset of the APQN range as 'usable by the default AP - queue device drivers' or 'not usable by the default device drivers' and thus - available for use by the vfio_ap device driver'. The location of the sysfs - files containing the masks are:: + bitmasks marking a subset of the APQN range as usable only by the default AP + queue device drivers. All remaining APQNs are available for use by + any other device driver. The vfio_ap device driver is currently the only + non-default device driver. The location of the sysfs files containing the + masks are:: /sys/bus/ap/apmask /sys/bus/ap/aqmask The 'apmask' is a 256-bit mask that identifies a set of AP adapter IDs - (APID). Each bit in the mask, from left to right (i.e., from most significant - to least significant bit in big endian order), corresponds to an APID from - 0-255. If a bit is set, the APID is marked as usable only by the default AP - queue device drivers; otherwise, the APID is usable by the vfio_ap - device driver. + (APID). Each bit in the mask, from left to right, corresponds to an APID from + 0-255. If a bit is set, the APID belongs to the subset of APQNs marked as + available only to the default AP queue device drivers. The 'aqmask' is a 256-bit mask that identifies a set of AP queue indexes - (APQI). Each bit in the mask, from left to right (i.e., from most significant - to least significant bit in big endian order), corresponds to an APQI from - 0-255. If a bit is set, the APQI is marked as usable only by the default AP - queue device drivers; otherwise, the APQI is usable by the vfio_ap device - driver. + (APQI). Each bit in the mask, from left to right, corresponds to an APQI from + 0-255. If a bit is set, the APQI belongs to the subset of APQNs marked as + available only to the default AP queue device drivers. - Take, for example, the following mask:: + The Cartesian product of the APIDs corresponding to the bits set in the + apmask and the APQIs corresponding to the bits set in the aqmask comprise + the subset of APQNs that can be used only by the host default device drivers. + All other APQNs are available to the non-default device drivers such as the + vfio_ap driver. - 0x7dffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + Take, for example, the following masks:: - It indicates: + apmask: + 0x7d00000000000000000000000000000000000000000000000000000000000000 - 1, 2, 3, 4, 5, and 7-255 belong to the default drivers' pool, and 0 and 6 - belong to the vfio_ap device driver's pool. + aqmask: + 0x8000000000000000000000000000000000000000000000000000000000000000 + + The masks indicate: + + * Adapters 1, 2, 3, 4, 5, and 7 are available for use by the host default + device drivers. + + * Domain 0 is available for use by the host default device drivers + + * The subset of APQNs available for use only by the default host device + drivers are: + + (1,0), (2,0), (3,0), (4.0), (5,0) and (7,0) + + * All other APQNs are available for use by the non-default device drivers. The APQN of each AP queue device assigned to the linux host is checked by the - AP bus against the set of APQNs derived from the cross product of APIDs - and APQIs marked as usable only by the default AP queue device drivers. If a + AP bus against the set of APQNs derived from the Cartesian product of APIDs + and APQIs marked as available to the default AP queue device drivers. If a match is detected, only the default AP queue device drivers will be probed; otherwise, the vfio_ap device driver will be probed. @@ -579,8 +635,7 @@ These are the steps: 0x4100000000000000000000000000000000000000000000000000000000000000 - Keep in mind that the mask reads from left to right (i.e., most - significant to least significant bit in big endian order), so the mask + Keep in mind that the mask reads from left to right, so the mask above identifies device numbers 1 and 7 (01000001). If the string is longer than the mask, the operation is terminated with @@ -626,11 +681,22 @@ These are the steps: default drivers pool: adapter 0-15, domain 1 alternate drivers pool: adapter 16-255, domains 0, 2-255 + Note ***: + Changing a mask such that one or more APQNs will be taken from a vfio_ap + mediated device (see below) will fail with an error (EBUSY). A message + is logged to the kernel ring buffer which can be viewed with the 'dmesg' + command. The output identifies each APQN flagged as 'in use' and identifies + the vfio_ap mediated device to which it is assigned; for example: + + Userspace may not re-assign queue 05.0054 already assigned to 62177883-f1bb-47f0-914d-32a22e3a8804 + Userspace may not re-assign queue 04.0054 already assigned to cef03c3c-903d-4ecc-9a83-40694cb8aee4 + Securing the APQNs for our example ---------------------------------- To secure the AP queues 05.0004, 05.0047, 05.00ab, 05.00ff, 06.0004, 06.0047, 06.00ab, and 06.00ff for use by the vfio_ap device driver, the corresponding - APQNs can either be removed from the default masks:: + APQNs can be removed from the default masks using either of the following + commands:: echo -5,-6 > /sys/bus/ap/apmask @@ -683,7 +749,7 @@ Securing the APQNs for our example /sys/devices/vfio_ap/matrix/ --- [mdev_supported_types] - ------ [vfio_ap-passthrough] (passthrough mediated matrix device type) + ------ [vfio_ap-passthrough] (passthrough vfio_ap mediated device type) --------- create --------- [devices] @@ -734,6 +800,9 @@ Securing the APQNs for our example ----------------unassign_control_domain ----------------unassign_domain + Note *****: The vfio_ap mdevs do not persist across reboots unless the + mdevctl tool is used to create and persist them. + 4. The administrator now needs to configure the matrixes for the mediated devices $uuid1 (for Guest1), $uuid2 (for Guest2) and $uuid3 (for Guest3). @@ -755,6 +824,10 @@ Securing the APQNs for our example cat matrix + To display the matrix that is or will be assigned to Guest1:: + + cat guest_matrix + This is how the matrix is configured for Guest2:: echo 5 > assign_adapter @@ -774,17 +847,24 @@ Securing the APQNs for our example higher than the maximum is specified, the operation will terminate with an error (ENODEV). - * All APQNs that can be derived from the adapter ID and the IDs of - the previously assigned domains must be bound to the vfio_ap device - driver. If no domains have yet been assigned, then there must be at least - one APQN with the specified APID bound to the vfio_ap driver. If no such - APQNs are bound to the driver, the operation will terminate with an - error (EADDRNOTAVAIL). + Note: The maximum adapter number can be obtained via the sysfs + /sys/bus/ap/ap_max_adapter_id attribute file. - No APQN that can be derived from the adapter ID and the IDs of the - previously assigned domains can be assigned to another mediated matrix - device. If an APQN is assigned to another mediated matrix device, the - operation will terminate with an error (EADDRINUSE). + * Each APQN derived from the Cartesian product of the APID of the adapter + being assigned and the APQIs of the domains previously assigned: + + - Must only be available to the vfio_ap device driver as specified in the + sysfs /sys/bus/ap/apmask and /sys/bus/ap/aqmask attribute files. If even + one APQN is reserved for use by the host device driver, the operation + will terminate with an error (EADDRNOTAVAIL). + + - Must NOT be assigned to another vfio_ap mediated device. If even one APQN + is assigned to another vfio_ap mediated device, the operation will + terminate with an error (EBUSY). + + - Must NOT be assigned while the sysfs /sys/bus/ap/apmask and + sys/bus/ap/aqmask attribute files are being edited or the operation may + terminate with an error (EBUSY). In order to successfully assign a domain: @@ -793,41 +873,50 @@ Securing the APQNs for our example higher than the maximum is specified, the operation will terminate with an error (ENODEV). - * All APQNs that can be derived from the domain ID and the IDs of - the previously assigned adapters must be bound to the vfio_ap device - driver. If no domains have yet been assigned, then there must be at least - one APQN with the specified APQI bound to the vfio_ap driver. If no such - APQNs are bound to the driver, the operation will terminate with an - error (EADDRNOTAVAIL). + Note: The maximum domain number can be obtained via the sysfs + /sys/bus/ap/ap_max_domain_id attribute file. - No APQN that can be derived from the domain ID and the IDs of the - previously assigned adapters can be assigned to another mediated matrix - device. If an APQN is assigned to another mediated matrix device, the - operation will terminate with an error (EADDRINUSE). + * Each APQN derived from the Cartesian product of the APQI of the domain + being assigned and the APIDs of the adapters previously assigned: - In order to successfully assign a control domain, the domain number - specified must represent a value from 0 up to the maximum domain number - configured for the system. If a control domain number higher than the maximum - is specified, the operation will terminate with an error (ENODEV). + - Must only be available to the vfio_ap device driver as specified in the + sysfs /sys/bus/ap/apmask and /sys/bus/ap/aqmask attribute files. If even + one APQN is reserved for use by the host device driver, the operation + will terminate with an error (EADDRNOTAVAIL). + + - Must NOT be assigned to another vfio_ap mediated device. If even one APQN + is assigned to another vfio_ap mediated device, the operation will + terminate with an error (EBUSY). + + - Must NOT be assigned while the sysfs /sys/bus/ap/apmask and + sys/bus/ap/aqmask attribute files are being edited or the operation may + terminate with an error (EBUSY). + + In order to successfully assign a control domain: + + * The domain number specified must represent a value from 0 up to the maximum + domain number configured for the system. If a control domain number higher + than the maximum is specified, the operation will terminate with an + error (ENODEV). 5. Start Guest1:: - /usr/bin/qemu-system-s390x ... -cpu host,ap=on,apqci=on,apft=on \ + /usr/bin/qemu-system-s390x ... -cpu host,ap=on,apqci=on,apft=on,apqi=on \ -device vfio-ap,sysfsdev=/sys/devices/vfio_ap/matrix/$uuid1 ... 7. Start Guest2:: - /usr/bin/qemu-system-s390x ... -cpu host,ap=on,apqci=on,apft=on \ + /usr/bin/qemu-system-s390x ... -cpu host,ap=on,apqci=on,apft=on,apqi=on \ -device vfio-ap,sysfsdev=/sys/devices/vfio_ap/matrix/$uuid2 ... 7. Start Guest3:: - /usr/bin/qemu-system-s390x ... -cpu host,ap=on,apqci=on,apft=on \ + /usr/bin/qemu-system-s390x ... -cpu host,ap=on,apqci=on,apft=on,apqi=on \ -device vfio-ap,sysfsdev=/sys/devices/vfio_ap/matrix/$uuid3 ... -When the guest is shut down, the mediated matrix devices may be removed. +When the guest is shut down, the vfio_ap mediated devices may be removed. -Using our example again, to remove the mediated matrix device $uuid1:: +Using our example again, to remove the vfio_ap mediated device $uuid1:: /sys/devices/vfio_ap/matrix/ --- [mdev_supported_types] @@ -840,26 +929,137 @@ Using our example again, to remove the mediated matrix device $uuid1:: echo 1 > remove -This will remove all of the mdev matrix device's sysfs structures including -the mdev device itself. To recreate and reconfigure the mdev matrix device, +This will remove all of the matrix mdev device's sysfs structures including +the mdev device itself. To recreate and reconfigure the matrix mdev device, all of the steps starting with step 3 will have to be performed again. Note -that the remove will fail if a guest using the mdev is still running. +that the remove will fail if a guest using the vfio_ap mdev is still running. -It is not necessary to remove an mdev matrix device, but one may want to +It is not necessary to remove a vfio_ap mdev, but one may want to remove it if no guest will use it during the remaining lifetime of the linux -host. If the mdev matrix device is removed, one may want to also reconfigure +host. If the vfio_ap mdev is removed, one may want to also reconfigure the pool of adapters and queues reserved for use by the default drivers. +Hot plug/unplug support: +================ +An adapter, domain or control domain may be hot plugged into a running KVM +guest by assigning it to the vfio_ap mediated device being used by the guest if +the following conditions are met: + +* The adapter, domain or control domain must also be assigned to the host's + AP configuration. + +* Each APQN derived from the Cartesian product comprised of the APID of the + adapter being assigned and the APQIs of the domains assigned must reference a + queue device bound to the vfio_ap device driver. + +* To hot plug a domain, each APQN derived from the Cartesian product + comprised of the APQI of the domain being assigned and the APIDs of the + adapters assigned must reference a queue device bound to the vfio_ap device + driver. + +An adapter, domain or control domain may be hot unplugged from a running KVM +guest by unassigning it from the vfio_ap mediated device being used by the +guest. + +Over-provisioning of AP queues for a KVM guest: +============================================== +Over-provisioning is defined herein as the assignment of adapters or domains to +a vfio_ap mediated device that do not reference AP devices in the host's AP +configuration. The idea here is that when the adapter or domain becomes +available, it will be automatically hot-plugged into the KVM guest using +the vfio_ap mediated device to which it is assigned as long as each new APQN +resulting from plugging it in references a queue device bound to the vfio_ap +device driver. + Limitations =========== -* The KVM/kernel interfaces do not provide a way to prevent restoring an APQN - to the default drivers pool of a queue that is still assigned to a mediated - device in use by a guest. It is incumbent upon the administrator to - ensure there is no mediated device in use by a guest to which the APQN is - assigned lest the host be given access to the private data of the AP queue - device such as a private key configured specifically for the guest. +Live guest migration is not supported for guests using AP devices without +intervention by a system administrator. Before a KVM guest can be migrated, +the vfio_ap mediated device must be removed. Unfortunately, it can not be +removed manually (i.e., echo 1 > /sys/devices/vfio_ap/matrix/$UUID/remove) while +the mdev is in use by a KVM guest. If the guest is being emulated by QEMU, +its mdev can be hot unplugged from the guest in one of two ways: -* Dynamically modifying the AP matrix for a running guest (which would amount to - hot(un)plug of AP devices for the guest) is currently not supported +1. If the KVM guest was started with libvirt, you can hot unplug the mdev via + the following commands: -* Live guest migration is not supported for guests using AP devices. + virsh detach-device + + For example, to hot unplug mdev 62177883-f1bb-47f0-914d-32a22e3a8804 from + the guest named 'my-guest': + + virsh detach-device my-guest ~/config/my-guest-hostdev.xml + + The contents of my-guest-hostdev.xml: + + + +
+ + + + + virsh qemu-monitor-command --hmp "device-del " + + For example, to hot unplug the vfio_ap mediated device identified on the + qemu command line with 'id=hostdev0' from the guest named 'my-guest': + + virsh qemu-monitor-command my-guest --hmp "device_del hostdev0" + +2. A vfio_ap mediated device can be hot unplugged by attaching the qemu monitor + to the guest and using the following qemu monitor command: + + (QEMU) device-del id= + + For example, to hot unplug the vfio_ap mediated device that was specified + on the qemu command line with 'id=hostdev0' when the guest was started: + + (QEMU) device-del id=hostdev0 + +After live migration of the KVM guest completes, an AP configuration can be +restored to the KVM guest by hot plugging a vfio_ap mediated device on the target +system into the guest in one of two ways: + +1. If the KVM guest was started with libvirt, you can hot plug a matrix mediated + device into the guest via the following virsh commands: + + virsh attach-device + + For example, to hot plug mdev 62177883-f1bb-47f0-914d-32a22e3a8804 into + the guest named 'my-guest': + + virsh attach-device my-guest ~/config/my-guest-hostdev.xml + + The contents of my-guest-hostdev.xml: + + + +
+ + + + + virsh qemu-monitor-command --hmp \ + "device_add vfio-ap,sysfsdev=,id=" + + For example, to hot plug the vfio_ap mediated device + 62177883-f1bb-47f0-914d-32a22e3a8804 into the guest named 'my-guest' with + device-id hostdev0: + + virsh qemu-monitor-command my-guest --hmp \ + "device_add vfio-ap,\ + sysfsdev=/sys/devices/vfio_ap/matrix/62177883-f1bb-47f0-914d-32a22e3a8804,\ + id=hostdev0" + +2. A vfio_ap mediated device can be hot plugged by attaching the qemu monitor + to the guest and using the following qemu monitor command: + + (qemu) device_add "vfio-ap,sysfsdev=,id=" + + For example, to plug the vfio_ap mediated device + 62177883-f1bb-47f0-914d-32a22e3a8804 into the guest with the device-id + hostdev0: + + (QEMU) device-add "vfio-ap,\ + sysfsdev=/sys/devices/vfio_ap/matrix/62177883-f1bb-47f0-914d-32a22e3a8804,\ + id=hostdev0" From e32d3827f3d5b2844d041066b89843599c3e82b9 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Fri, 4 Feb 2022 13:16:45 -0500 Subject: [PATCH 20/46] s390/Docs: new doc describing lock usage by the vfio_ap device driver Introduces a new document describing the locks used by the vfio_ap device driver and how to use them so as to avoid lockdep reports and deadlock situations. Signed-off-by: Tony Krowiak Reviewed-by: Jason J. Herne Signed-off-by: Alexander Gordeev --- Documentation/s390/vfio-ap-locking.rst | 105 +++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 Documentation/s390/vfio-ap-locking.rst diff --git a/Documentation/s390/vfio-ap-locking.rst b/Documentation/s390/vfio-ap-locking.rst new file mode 100644 index 000000000000..c4e1eeec79a0 --- /dev/null +++ b/Documentation/s390/vfio-ap-locking.rst @@ -0,0 +1,105 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================== +VFIO AP Locks Overview +====================== +This document describes the locks that are pertinent to the secure operation +of the vfio_ap device driver. Throughout this document, the following variables +will be used to denote instances of the structures herein described: + +struct ap_matrix_dev *matrix_dev; +struct ap_matrix_mdev *matrix_mdev; +struct kvm *kvm; + +The Matrix Devices Lock (drivers/s390/crypto/vfio_ap_private.h) +-------------------------------------------------------------- + +struct ap_matrix_dev { + ... + struct list_head mdev_list; + struct mutex mdevs_lock; + ... +} + +The Matrix Devices Lock (matrix_dev->mdevs_lock) is implemented as a global +mutex contained within the single object of struct ap_matrix_dev. This lock +controls access to all fields contained within each matrix_mdev +(matrix_dev->mdev_list). This lock must be held while reading from, writing to +or using the data from a field contained within a matrix_mdev instance +representing one of the vfio_ap device driver's mediated devices. + +The KVM Lock (include/linux/kvm_host.h) +--------------------------------------- + +struct kvm { + ... + struct mutex lock; + ... +} + +The KVM Lock (kvm->lock) controls access to the state data for a KVM guest. This +lock must be held by the vfio_ap device driver while one or more AP adapters, +domains or control domains are being plugged into or unplugged from the guest. + +The KVM pointer is stored in the in the matrix_mdev instance +(matrix_mdev->kvm = kvm) containing the state of the mediated device that has +been attached to the KVM guest. + +The Guests Lock (drivers/s390/crypto/vfio_ap_private.h) +----------------------------------------------------------- + +struct ap_matrix_dev { + ... + struct list_head mdev_list; + struct mutex guests_lock; + ... +} + +The Guests Lock (matrix_dev->guests_lock) controls access to the +matrix_mdev instances (matrix_dev->mdev_list) that represent mediated devices +that hold the state for the mediated devices that have been attached to a +KVM guest. This lock must be held: + +1. To control access to the KVM pointer (matrix_mdev->kvm) while the vfio_ap + device driver is using it to plug/unplug AP devices passed through to the KVM + guest. + +2. To add matrix_mdev instances to or remove them from matrix_dev->mdev_list. + This is necessary to ensure the proper locking order when the list is perused + to find an ap_matrix_mdev instance for the purpose of plugging/unplugging + AP devices passed through to a KVM guest. + + For example, when a queue device is removed from the vfio_ap device driver, + if the adapter is passed through to a KVM guest, it will have to be + unplugged. In order to figure out whether the adapter is passed through, + the matrix_mdev object to which the queue is assigned will have to be + found. The KVM pointer (matrix_mdev->kvm) can then be used to determine if + the mediated device is passed through (matrix_mdev->kvm != NULL) and if so, + to unplug the adapter. + +It is not necessary to take the Guests Lock to access the KVM pointer if the +pointer is not used to plug/unplug devices passed through to the KVM guest; +however, in this case, the Matrix Devices Lock (matrix_dev->mdevs_lock) must be +held in order to access the KVM pointer since it is set and cleared under the +protection of the Matrix Devices Lock. A case in point is the function that +handles interception of the PQAP(AQIC) instruction sub-function. This handler +needs to access the KVM pointer only for the purposes of setting or clearing IRQ +resources, so only the matrix_dev->mdevs_lock needs to be held. + +The PQAP Hook Lock (arch/s390/include/asm/kvm_host.h) +----------------------------------------------------- + +typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); + +struct kvm_s390_crypto { + ... + struct rw_semaphore pqap_hook_rwsem; + crypto_hook *pqap_hook; + ... +}; + +The PQAP Hook Lock is a r/w semaphore that controls access to the function +pointer of the handler (*kvm->arch.crypto.pqap_hook) to invoke when the +PQAP(AQIC) instruction sub-function is intercepted by the host. The lock must be +held in write mode when pqap_hook value is set, and in read mode when the +pqap_hook function is called. From 693714b900cea5417d7b3a48891b961d40fb5c55 Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Mon, 14 Feb 2022 19:25:40 -0500 Subject: [PATCH 21/46] MAINTAINERS: pick up all vfio_ap docs for VFIO AP maintainers A new document, Documentation/s390/vfio-ap-locking.rst was added. Make sure the new document is picked up for the VFIO AP maintainers by using a wildcard: Documentation/s390/vfio-ap*. Signed-off-by: Tony Krowiak Signed-off-by: Alexander Gordeev --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 3cf9842d9233..fbe417746e22 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17453,7 +17453,7 @@ M: Jason Herne L: linux-s390@vger.kernel.org S: Supported W: http://www.ibm.com/developerworks/linux/linux390/ -F: Documentation/s390/vfio-ap.rst +F: Documentation/s390/vfio-ap* F: drivers/s390/crypto/vfio_ap* S390 VFIO-CCW DRIVER From 0a5f9b382c6131381f9f2ed64ae6fdd994328d0d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 13 Jul 2022 14:56:42 +0200 Subject: [PATCH 22/46] s390/cpufeature: rework to allow more than only hwcap bits Rework cpufeature implementation to allow for various cpu feature indications, which is not only limited to hwcap bits. This is achieved by adding a sequential list of cpu feature numbers, where each of them is mapped to an entry which indicates what this number is about. Each entry contains a type member, which indicates what feature name space to look into (e.g. hwcap, or cpu facility). If wanted this allows also to automatically load modules only in e.g. z/VM configurations. Signed-off-by: Heiko Carstens Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Reviewed-by: Hendrik Brueckner Link: https://lore.kernel.org/r/20220713125644.16121-2-seiden@linux.ibm.com Signed-off-by: Alexander Gordeev --- arch/s390/crypto/aes_s390.c | 2 +- arch/s390/crypto/chacha-glue.c | 2 +- arch/s390/crypto/crc32-vx.c | 2 +- arch/s390/crypto/des_s390.c | 2 +- arch/s390/crypto/ghash_s390.c | 2 +- arch/s390/crypto/prng.c | 2 +- arch/s390/crypto/sha1_s390.c | 2 +- arch/s390/crypto/sha256_s390.c | 2 +- arch/s390/crypto/sha3_256_s390.c | 2 +- arch/s390/crypto/sha3_512_s390.c | 2 +- arch/s390/crypto/sha512_s390.c | 2 +- arch/s390/include/asm/cpufeature.h | 22 +++++----------- arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/cpufeature.c | 42 ++++++++++++++++++++++++++++++ arch/s390/kernel/processor.c | 10 ------- drivers/char/hw_random/s390-trng.c | 2 +- drivers/s390/crypto/pkey_api.c | 2 +- 17 files changed, 63 insertions(+), 39 deletions(-) create mode 100644 arch/s390/kernel/cpufeature.c diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 1023e9d43d44..526c3f40f6a2 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -1049,7 +1049,7 @@ out_err: return ret; } -module_cpu_feature_match(MSA, aes_s390_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, aes_s390_init); module_exit(aes_s390_fini); MODULE_ALIAS_CRYPTO("aes-all"); diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index 2ec51f339cec..7752bd314558 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -121,7 +121,7 @@ static void __exit chacha_mod_fini(void) crypto_unregister_skciphers(chacha_algs, ARRAY_SIZE(chacha_algs)); } -module_cpu_feature_match(VXRS, chacha_mod_init); +module_cpu_feature_match(S390_CPU_FEATURE_VXRS, chacha_mod_init); module_exit(chacha_mod_fini); MODULE_DESCRIPTION("ChaCha20 stream cipher"); diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index fafecad20752..017143e9cef7 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -298,7 +298,7 @@ static void __exit crc_vx_mod_exit(void) crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs)); } -module_cpu_feature_match(VXRS, crc_vx_mod_init); +module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init); module_exit(crc_vx_mod_exit); MODULE_AUTHOR("Hendrik Brueckner "); diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index e013088b5115..8e75b83a5ddc 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -492,7 +492,7 @@ out_err: return ret; } -module_cpu_feature_match(MSA, des_s390_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, des_s390_init); module_exit(des_s390_exit); MODULE_ALIAS_CRYPTO("des"); diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c index 6b07a2f1ce8a..0800a2a5799f 100644 --- a/arch/s390/crypto/ghash_s390.c +++ b/arch/s390/crypto/ghash_s390.c @@ -145,7 +145,7 @@ static void __exit ghash_mod_exit(void) crypto_unregister_shash(&ghash_alg); } -module_cpu_feature_match(MSA, ghash_mod_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, ghash_mod_init); module_exit(ghash_mod_exit); MODULE_ALIAS_CRYPTO("ghash"); diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index ae382bafc772..a077087bc6cc 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -907,5 +907,5 @@ static void __exit prng_exit(void) } } -module_cpu_feature_match(MSA, prng_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, prng_init); module_exit(prng_exit); diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index a3fabf310a38..bc3a22704e09 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -95,7 +95,7 @@ static void __exit sha1_s390_fini(void) crypto_unregister_shash(&alg); } -module_cpu_feature_match(MSA, sha1_s390_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, sha1_s390_init); module_exit(sha1_s390_fini); MODULE_ALIAS_CRYPTO("sha1"); diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index 24983f175676..6f1ccdf93d3e 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -134,7 +134,7 @@ static void __exit sha256_s390_fini(void) crypto_unregister_shash(&sha256_alg); } -module_cpu_feature_match(MSA, sha256_s390_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, sha256_s390_init); module_exit(sha256_s390_fini); MODULE_ALIAS_CRYPTO("sha256"); diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index 30ac49b635bf..e1350e033a32 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -137,7 +137,7 @@ static void __exit sha3_256_s390_fini(void) crypto_unregister_shash(&sha3_256_alg); } -module_cpu_feature_match(MSA, sha3_256_s390_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, sha3_256_s390_init); module_exit(sha3_256_s390_fini); MODULE_ALIAS_CRYPTO("sha3-256"); diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index e70d50f7620f..06c142ed9bb1 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -147,7 +147,7 @@ static void __exit fini(void) crypto_unregister_shash(&sha3_384_alg); } -module_cpu_feature_match(MSA, init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, init); module_exit(fini); MODULE_LICENSE("GPL"); diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 43ce4956df73..04f11c407763 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -142,7 +142,7 @@ static void __exit fini(void) crypto_unregister_shash(&sha384_alg); } -module_cpu_feature_match(MSA, init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, init); module_exit(fini); MODULE_LICENSE("GPL"); diff --git a/arch/s390/include/asm/cpufeature.h b/arch/s390/include/asm/cpufeature.h index 14cfd48d598e..771caf5281e5 100644 --- a/arch/s390/include/asm/cpufeature.h +++ b/arch/s390/include/asm/cpufeature.h @@ -2,28 +2,20 @@ /* * Module interface for CPU features * - * Copyright IBM Corp. 2015 + * Copyright IBM Corp. 2015, 2022 * Author(s): Hendrik Brueckner */ #ifndef __ASM_S390_CPUFEATURE_H #define __ASM_S390_CPUFEATURE_H -#include +enum { + S390_CPU_FEATURE_MSA, + S390_CPU_FEATURE_VXRS, + MAX_CPU_FEATURES +}; -/* Hardware features on Linux on z Systems are indicated by facility bits that - * are mapped to the so-called machine flags. Particular machine flags are - * then used to define ELF hardware capabilities; most notably hardware flags - * that are essential for user space / glibc. - * - * Restrict the set of exposed CPU features to ELF hardware capabilities for - * now. Additional machine flags can be indicated by values larger than - * MAX_ELF_HWCAP_FEATURES. - */ -#define MAX_ELF_HWCAP_FEATURES (8 * sizeof(elf_hwcap)) -#define MAX_CPU_FEATURES MAX_ELF_HWCAP_FEATURES - -#define cpu_feature(feat) ilog2(HWCAP_ ## feat) +#define cpu_feature(feature) (feature) int cpu_have_feature(unsigned int nr); diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 27d6b3c7aa06..3cbfa9fddd9a 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -35,7 +35,7 @@ CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls obj-y := traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o -obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o +obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o cpufeature.o obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o diff --git a/arch/s390/kernel/cpufeature.c b/arch/s390/kernel/cpufeature.c new file mode 100644 index 000000000000..89c41830894d --- /dev/null +++ b/arch/s390/kernel/cpufeature.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 2022 + */ + +#include +#include +#include + +enum { + TYPE_HWCAP, +}; + +struct s390_cpu_feature { + unsigned int type : 4; + unsigned int num : 28; +}; + +static struct s390_cpu_feature s390_cpu_features[MAX_CPU_FEATURES] = { + [S390_CPU_FEATURE_MSA] = {.type = TYPE_HWCAP, .num = HWCAP_NR_MSA}, + [S390_CPU_FEATURE_VXRS] = {.type = TYPE_HWCAP, .num = HWCAP_NR_VXRS}, +}; + +/* + * cpu_have_feature - Test CPU features on module initialization + */ +int cpu_have_feature(unsigned int num) +{ + struct s390_cpu_feature *feature; + + if (WARN_ON_ONCE(num >= MAX_CPU_FEATURES)) + return 0; + feature = &s390_cpu_features[num]; + switch (feature->type) { + case TYPE_HWCAP: + return !!(elf_hwcap & BIT(feature->num)); + default: + WARN_ON_ONCE(1); + return 0; + } +} +EXPORT_SYMBOL(cpu_have_feature); diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index aa0e0e7fc773..a194611ba88c 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -8,7 +8,6 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include -#include #include #include #include @@ -96,15 +95,6 @@ void cpu_init(void) enter_lazy_tlb(&init_mm, current); } -/* - * cpu_have_feature - Test CPU features on module initialization - */ -int cpu_have_feature(unsigned int num) -{ - return elf_hwcap & (1UL << num); -} -EXPORT_SYMBOL(cpu_have_feature); - static void show_facilities(struct seq_file *m) { unsigned int bit; diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index 2beaa35c0d74..12fbac0ed8ca 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -261,5 +261,5 @@ static void __exit trng_exit(void) trng_debug_exit(); } -module_cpu_feature_match(MSA, trng_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, trng_init); module_exit(trng_exit); diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 7329caa7d467..5a05d1cdfec2 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -2115,5 +2115,5 @@ static void __exit pkey_exit(void) pkey_debug_exit(); } -module_cpu_feature_match(MSA, pkey_init); +module_cpu_feature_match(S390_CPU_FEATURE_MSA, pkey_init); module_exit(pkey_exit); From e2f39c9f547c88efd208f67a9ecacc1d78bf30a4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 13 Jul 2022 14:56:43 +0200 Subject: [PATCH 23/46] s390/cpufeature: allow for facility bits Allow for facility bits to be used in cpu features. Signed-off-by: Heiko Carstens Reviewed-by: Claudio Imbrenda Reviewed-by: Steffen Eiden Link: https://lore.kernel.org/r/20220713125644.16121-3-seiden@linux.ibm.com Signed-off-by: Alexander Gordeev --- arch/s390/kernel/cpufeature.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/s390/kernel/cpufeature.c b/arch/s390/kernel/cpufeature.c index 89c41830894d..8b5072b335f4 100644 --- a/arch/s390/kernel/cpufeature.c +++ b/arch/s390/kernel/cpufeature.c @@ -9,6 +9,7 @@ enum { TYPE_HWCAP, + TYPE_FACILITY, }; struct s390_cpu_feature { @@ -34,6 +35,8 @@ int cpu_have_feature(unsigned int num) switch (feature->type) { case TYPE_HWCAP: return !!(elf_hwcap & BIT(feature->num)); + case TYPE_FACILITY: + return test_facility(feature->num); default: WARN_ON_ONCE(1); return 0; From 5fcd0d8ae23a3d26f21bc191c32283a5ba0bdaf9 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 13 Jul 2022 14:56:44 +0200 Subject: [PATCH 24/46] s390/uvdevice: autoload module based on CPU facility Make sure the uvdevice driver will be automatically loaded when facility 158 is available. Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Link: https://lore.kernel.org/r/20220713125644.16121-4-seiden@linux.ibm.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/cpufeature.h | 1 + arch/s390/kernel/cpufeature.c | 1 + drivers/s390/char/uvdevice.c | 5 ++--- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/s390/include/asm/cpufeature.h b/arch/s390/include/asm/cpufeature.h index 771caf5281e5..931204613753 100644 --- a/arch/s390/include/asm/cpufeature.h +++ b/arch/s390/include/asm/cpufeature.h @@ -12,6 +12,7 @@ enum { S390_CPU_FEATURE_MSA, S390_CPU_FEATURE_VXRS, + S390_CPU_FEATURE_UV, MAX_CPU_FEATURES }; diff --git a/arch/s390/kernel/cpufeature.c b/arch/s390/kernel/cpufeature.c index 8b5072b335f4..1b2ae42a0c15 100644 --- a/arch/s390/kernel/cpufeature.c +++ b/arch/s390/kernel/cpufeature.c @@ -20,6 +20,7 @@ struct s390_cpu_feature { static struct s390_cpu_feature s390_cpu_features[MAX_CPU_FEATURES] = { [S390_CPU_FEATURE_MSA] = {.type = TYPE_HWCAP, .num = HWCAP_NR_MSA}, [S390_CPU_FEATURE_VXRS] = {.type = TYPE_HWCAP, .num = HWCAP_NR_VXRS}, + [S390_CPU_FEATURE_UV] = {.type = TYPE_FACILITY, .num = 158}, }; /* diff --git a/drivers/s390/char/uvdevice.c b/drivers/s390/char/uvdevice.c index 66505d7166a6..1d40457c7b10 100644 --- a/drivers/s390/char/uvdevice.c +++ b/drivers/s390/char/uvdevice.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -244,12 +245,10 @@ static void __exit uvio_dev_exit(void) static int __init uvio_dev_init(void) { - if (!test_facility(158)) - return -ENXIO; return misc_register(&uvio_dev_miscdev); } -module_init(uvio_dev_init); +module_cpu_feature_match(S390_CPU_FEATURE_UV, uvio_dev_init); module_exit(uvio_dev_exit); MODULE_AUTHOR("IBM Corporation"); From 7190d84966b34de3892cd4eb8698a2229ceb8d82 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Fri, 15 Jul 2022 06:43:32 +0200 Subject: [PATCH 25/46] s390/mm: remove unused tprot() function Since commit 461e0da7ddbb ("s390: remove broken hibernate / power management support") there are no users of tprot() left. Remove the function itself as well. Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/mmu.h | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index 82aae78e1315..bc254bce48fa 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -42,18 +42,4 @@ typedef struct { .context.pgtable_list = LIST_HEAD_INIT(name.context.pgtable_list), \ .context.gmap_list = LIST_HEAD_INIT(name.context.gmap_list), -static inline int tprot(unsigned long addr) -{ - int rc = -EFAULT; - - asm volatile( - " tprot 0(%1),0\n" - "0: ipm %0\n" - " srl %0,28\n" - "1:\n" - EX_TABLE(0b,1b) - : "+d" (rc) : "a" (addr) : "cc"); - return rc; -} - #endif From 86caa4b678956bc19567256dd8422f3978620897 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Fri, 15 Jul 2022 12:02:33 +0200 Subject: [PATCH 26/46] s390/crash: remove redundant panic() on save area allocation failure Make save_area_alloc() return classic NULL on allocation failure. The only caller smp_save_dump_cpus() does check the return value already and panics if NULL is returned. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/crash_dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 28124d0fa1d5..117f13b3d153 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -63,7 +63,7 @@ struct save_area * __init save_area_alloc(bool is_boot_cpu) sa = memblock_alloc(sizeof(*sa), 8); if (!sa) - panic("Failed to allocate save area\n"); + return NULL; if (is_boot_cpu) list_add(&sa->list, &dump_save_areas); From f6749da17a34eb08c9665f072ce7c812ff68aad2 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Fri, 15 Jul 2022 12:59:33 +0200 Subject: [PATCH 27/46] s390/crash: fix incorrect number of bytes to copy to user space The number of bytes in a chunk is correctly calculated, but instead the total number of bytes is passed to copy_to_user_real() function. Reported-by: Matthew Wilcox Fixes: df9694c7975f ("s390/dump: streamline oldmem copy functions") Signed-off-by: Alexander Gordeev --- arch/s390/kernel/crash_dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 117f13b3d153..0efee5c49b1e 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -199,7 +199,7 @@ static int copy_oldmem_user(void __user *dst, unsigned long src, size_t count) } else { len = count; } - rc = copy_to_user_real(dst, src, count); + rc = copy_to_user_real(dst, src, len); if (rc) return rc; } From 9ffed254d938c9e99eb7761c7f739294c84e0367 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 19 Jul 2022 07:16:33 +0200 Subject: [PATCH 28/46] s390/zcore: fix race when reading from hardware system area Memory buffer used for reading out data from hardware system area is not protected against concurrent access. Reported-by: Matthew Wilcox Fixes: 411ed3225733 ("[S390] zfcpdump support.") Acked-by: Heiko Carstens Tested-by: Alexander Egorenkov Link: https://lore.kernel.org/r/e68137f0f9a0d2558f37becc20af18e2939934f6.1658206891.git.agordeev@linux.ibm.com Signed-off-by: Alexander Gordeev --- drivers/s390/char/zcore.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index 516783ba950f..92b32ce645b9 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -50,6 +50,7 @@ static struct dentry *zcore_reipl_file; static struct dentry *zcore_hsa_file; static struct ipl_parameter_block *zcore_ipl_block; +static DEFINE_MUTEX(hsa_buf_mutex); static char hsa_buf[PAGE_SIZE] __aligned(PAGE_SIZE); /* @@ -66,19 +67,24 @@ int memcpy_hsa_user(void __user *dest, unsigned long src, size_t count) if (!hsa_available) return -ENODATA; + mutex_lock(&hsa_buf_mutex); while (count) { if (sclp_sdias_copy(hsa_buf, src / PAGE_SIZE + 2, 1)) { TRACE("sclp_sdias_copy() failed\n"); + mutex_unlock(&hsa_buf_mutex); return -EIO; } offset = src % PAGE_SIZE; bytes = min(PAGE_SIZE - offset, count); - if (copy_to_user(dest, hsa_buf + offset, bytes)) + if (copy_to_user(dest, hsa_buf + offset, bytes)) { + mutex_unlock(&hsa_buf_mutex); return -EFAULT; + } src += bytes; dest += bytes; count -= bytes; } + mutex_unlock(&hsa_buf_mutex); return 0; } @@ -96,9 +102,11 @@ int memcpy_hsa_kernel(void *dest, unsigned long src, size_t count) if (!hsa_available) return -ENODATA; + mutex_lock(&hsa_buf_mutex); while (count) { if (sclp_sdias_copy(hsa_buf, src / PAGE_SIZE + 2, 1)) { TRACE("sclp_sdias_copy() failed\n"); + mutex_unlock(&hsa_buf_mutex); return -EIO; } offset = src % PAGE_SIZE; @@ -108,6 +116,7 @@ int memcpy_hsa_kernel(void *dest, unsigned long src, size_t count) dest += bytes; count -= bytes; } + mutex_unlock(&hsa_buf_mutex); return 0; } From d6da67378198e4caa37404f87851659553b936b9 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 19 Jul 2022 07:16:34 +0200 Subject: [PATCH 29/46] s390/crash: move copy_to_user_real() to crash_dump.c Function copy_to_user_real() does not really belong to maccess.c. It is only used for copying oldmem to user space, so let's move it to the friends. Acked-by: Heiko Carstens Tested-by: Alexander Egorenkov Link: https://lore.kernel.org/r/e8de968d40202d87caa09aef12e9c67ec23a1c1a.1658206891.git.agordeev@linux.ibm.com Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/uaccess.h | 1 - arch/s390/kernel/crash_dump.c | 26 ++++++++++++++++++++++++++ arch/s390/mm/maccess.c | 26 -------------------------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index f4511e21d646..2a067315fe59 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -285,7 +285,6 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo return __clear_user(to, n); } -int copy_to_user_real(void __user *dest, unsigned long src, unsigned long count); void *s390_kernel_write(void *dst, const void *src, size_t size); int __noreturn __put_kernel_bad(void); diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 0efee5c49b1e..8d7332d4444c 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -173,6 +173,32 @@ int copy_oldmem_kernel(void *dst, unsigned long src, size_t count) return 0; } +/* + * Copy memory from kernel (real) to user (virtual) + */ +static int copy_to_user_real(void __user *dest, unsigned long src, unsigned long count) +{ + int offs = 0, size, rc; + char *buf; + + buf = (char *)__get_free_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + rc = -EFAULT; + while (offs < count) { + size = min(PAGE_SIZE, count - offs); + if (memcpy_real(buf, src + offs, size)) + goto out; + if (copy_to_user(dest + offs, buf, size)) + goto out; + offs += size; + } + rc = 0; +out: + free_page((unsigned long)buf); + return rc; +} + /* * Copy memory of the old, dumped system to a user space virtual address */ diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index 421efa46946b..d6d84e02f35a 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -171,32 +171,6 @@ void memcpy_absolute(void *dest, void *src, size_t count) arch_local_irq_restore(flags); } -/* - * Copy memory from kernel (real) to user (virtual) - */ -int copy_to_user_real(void __user *dest, unsigned long src, unsigned long count) -{ - int offs = 0, size, rc; - char *buf; - - buf = (char *) __get_free_page(GFP_KERNEL); - if (!buf) - return -ENOMEM; - rc = -EFAULT; - while (offs < count) { - size = min(PAGE_SIZE, count - offs); - if (memcpy_real(buf, src + offs, size)) - goto out; - if (copy_to_user(dest + offs, buf, size)) - goto out; - offs += size; - } - rc = 0; -out: - free_page((unsigned long) buf); - return rc; -} - /* * Check if physical address is within prefix or zero page */ From 6d2e5a4a13da2e91a46a37cd00d23dba8613ad7b Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 19 Jul 2022 07:16:35 +0200 Subject: [PATCH 30/46] s390/crash: use static swap buffer for copy_to_user_real() Currently a temporary page-size buffer is allocated for copying oldmem to user space. That limits copy_to_user_real() operation only to stages when virtual memory is available and still makes it possible to fail while the system is being dumped. Instead of reallocating single page on each copy_oldmem_page() iteration use a statically allocated buffer. This also paves the way for a further memcpy_real() rework where no swap buffer is needed altogether. Acked-by: Heiko Carstens Tested-by: Alexander Egorenkov Link: https://lore.kernel.org/r/334ed359680c4d45dd32feb104909f610312ef0f.1658206891.git.agordeev@linux.ibm.com Signed-off-by: Alexander Gordeev --- arch/s390/kernel/crash_dump.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 8d7332d4444c..1662f1d81abe 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -53,6 +53,8 @@ struct save_area { }; static LIST_HEAD(dump_save_areas); +static DEFINE_MUTEX(memcpy_real_mutex); +static char memcpy_real_buf[PAGE_SIZE]; /* * Allocate a save area @@ -178,25 +180,21 @@ int copy_oldmem_kernel(void *dst, unsigned long src, size_t count) */ static int copy_to_user_real(void __user *dest, unsigned long src, unsigned long count) { - int offs = 0, size, rc; - char *buf; + unsigned long offs = 0, size; - buf = (char *)__get_free_page(GFP_KERNEL); - if (!buf) - return -ENOMEM; - rc = -EFAULT; + mutex_lock(&memcpy_real_mutex); while (offs < count) { size = min(PAGE_SIZE, count - offs); - if (memcpy_real(buf, src + offs, size)) - goto out; - if (copy_to_user(dest + offs, buf, size)) - goto out; + if (memcpy_real(memcpy_real_buf, src + offs, size)) + break; + if (copy_to_user(dest + offs, memcpy_real_buf, size)) + break; offs += size; } - rc = 0; -out: - free_page((unsigned long)buf); - return rc; + mutex_unlock(&memcpy_real_mutex); + if (offs < count) + return -EFAULT; + return 0; } /* From ebbc9570169147740aa39aee1d61b4cc5a631644 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 19 Jul 2022 07:16:36 +0200 Subject: [PATCH 31/46] s390/crash: support multi-segment iterators Make it possible to handle not only single-, but also multi- segment iterators in copy_oldmem_iter() callback. Change the semantics of called functions to match the iterator model - instead of an error code the exact number of bytes copied is returned. The swap page used to copy data to user space is adopted for kernel space too. That does not bring any performance impact. Suggested-by: Matthew Wilcox Fixes: cc02e6e21aa5 ("s390/crash: add missing iterator advance in copy_oldmem_page()") Acked-by: Heiko Carstens Tested-by: Alexander Egorenkov Link: https://lore.kernel.org/r/5af6da3a0bffe48a90b0b7139ecf6a818b2d18e8.1658206891.git.agordeev@linux.ibm.com Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/os_info.h | 17 ++++- arch/s390/include/asm/sclp.h | 4 +- arch/s390/kernel/crash_dump.c | 128 ++++++-------------------------- drivers/s390/char/zcore.c | 58 ++++++--------- 4 files changed, 63 insertions(+), 144 deletions(-) diff --git a/arch/s390/include/asm/os_info.h b/arch/s390/include/asm/os_info.h index 147a8d547ef9..85248d8fee0c 100644 --- a/arch/s390/include/asm/os_info.h +++ b/arch/s390/include/asm/os_info.h @@ -8,6 +8,8 @@ #ifndef _ASM_S390_OS_INFO_H #define _ASM_S390_OS_INFO_H +#include + #define OS_INFO_VERSION_MAJOR 1 #define OS_INFO_VERSION_MINOR 1 #define OS_INFO_MAGIC 0x4f53494e464f535aULL /* OSINFOSZ */ @@ -39,7 +41,20 @@ u32 os_info_csum(struct os_info *os_info); #ifdef CONFIG_CRASH_DUMP void *os_info_old_entry(int nr, unsigned long *size); -int copy_oldmem_kernel(void *dst, unsigned long src, size_t count); +size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count); + +static inline int copy_oldmem_kernel(void *dst, unsigned long src, size_t count) +{ + struct iov_iter iter; + struct kvec kvec; + + kvec.iov_base = dst; + kvec.iov_len = count; + iov_iter_kvec(&iter, WRITE, &kvec, 1, count); + if (copy_oldmem_iter(&iter, src, count) < count) + return -EFAULT; + return 0; +} #else static inline void *os_info_old_entry(int nr, unsigned long *size) { diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 236b34b75ddb..24ee532fab84 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -17,6 +17,7 @@ #define EXT_SCCB_READ_CPU (3 * PAGE_SIZE) #ifndef __ASSEMBLY__ +#include #include #include @@ -142,8 +143,7 @@ int sclp_pci_deconfigure(u32 fid); int sclp_ap_configure(u32 apid); int sclp_ap_deconfigure(u32 apid); int sclp_pci_report(struct zpci_report_error_header *report, u32 fh, u32 fid); -int memcpy_hsa_kernel(void *dest, unsigned long src, size_t count); -int memcpy_hsa_user(void __user *dest, unsigned long src, size_t count); +size_t memcpy_hsa_iter(struct iov_iter *iter, unsigned long src, size_t count); void sclp_ocf_cpc_name_copy(char *dst); static inline int sclp_get_core_info(struct sclp_core_info *info, int early) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 1662f1d81abe..bad8f47fc5d6 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -116,102 +116,35 @@ void __init save_area_add_vxrs(struct save_area *sa, __vector128 *vxrs) memcpy(sa->vxrs_high, vxrs + 16, 16 * sizeof(__vector128)); } -/* - * Return physical address for virtual address - */ -static inline void *load_real_addr(void *addr) +static size_t copy_to_iter_real(struct iov_iter *iter, unsigned long src, size_t count) { - unsigned long real_addr; - - asm volatile( - " lra %0,0(%1)\n" - " jz 0f\n" - " la %0,0\n" - "0:" - : "=a" (real_addr) : "a" (addr) : "cc"); - return (void *)real_addr; -} - -/* - * Copy memory of the old, dumped system to a kernel space virtual address - */ -int copy_oldmem_kernel(void *dst, unsigned long src, size_t count) -{ - unsigned long len; - void *ra; - int rc; - - while (count) { - if (!oldmem_data.start && src < sclp.hsa_size) { - /* Copy from zfcp/nvme dump HSA area */ - len = min(count, sclp.hsa_size - src); - rc = memcpy_hsa_kernel(dst, src, len); - if (rc) - return rc; - } else { - /* Check for swapped kdump oldmem areas */ - if (oldmem_data.start && src - oldmem_data.start < oldmem_data.size) { - src -= oldmem_data.start; - len = min(count, oldmem_data.size - src); - } else if (oldmem_data.start && src < oldmem_data.size) { - len = min(count, oldmem_data.size - src); - src += oldmem_data.start; - } else { - len = count; - } - if (is_vmalloc_or_module_addr(dst)) { - ra = load_real_addr(dst); - len = min(PAGE_SIZE - offset_in_page(ra), len); - } else { - ra = dst; - } - if (memcpy_real(ra, src, len)) - return -EFAULT; - } - dst += len; - src += len; - count -= len; - } - return 0; -} - -/* - * Copy memory from kernel (real) to user (virtual) - */ -static int copy_to_user_real(void __user *dest, unsigned long src, unsigned long count) -{ - unsigned long offs = 0, size; + size_t len, copied, res = 0; mutex_lock(&memcpy_real_mutex); - while (offs < count) { - size = min(PAGE_SIZE, count - offs); - if (memcpy_real(memcpy_real_buf, src + offs, size)) + while (count) { + len = min(PAGE_SIZE, count); + if (memcpy_real(memcpy_real_buf, src, len)) break; - if (copy_to_user(dest + offs, memcpy_real_buf, size)) + copied = copy_to_iter(memcpy_real_buf, len, iter); + count -= copied; + src += copied; + res += copied; + if (copied < len) break; - offs += size; } mutex_unlock(&memcpy_real_mutex); - if (offs < count) - return -EFAULT; - return 0; + return res; } -/* - * Copy memory of the old, dumped system to a user space virtual address - */ -static int copy_oldmem_user(void __user *dst, unsigned long src, size_t count) +size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count) { - unsigned long len; - int rc; + size_t len, copied, res = 0; while (count) { if (!oldmem_data.start && src < sclp.hsa_size) { /* Copy from zfcp/nvme dump HSA area */ len = min(count, sclp.hsa_size - src); - rc = memcpy_hsa_user(dst, src, len); - if (rc) - return rc; + copied = memcpy_hsa_iter(iter, src, len); } else { /* Check for swapped kdump oldmem areas */ if (oldmem_data.start && src - oldmem_data.start < oldmem_data.size) { @@ -223,15 +156,15 @@ static int copy_oldmem_user(void __user *dst, unsigned long src, size_t count) } else { len = count; } - rc = copy_to_user_real(dst, src, len); - if (rc) - return rc; + copied = copy_to_iter_real(iter, src, len); } - dst += len; - src += len; - count -= len; + count -= copied; + src += copied; + res += copied; + if (copied < len) + break; } - return 0; + return res; } /* @@ -241,26 +174,9 @@ ssize_t copy_oldmem_page(struct iov_iter *iter, unsigned long pfn, size_t csize, unsigned long offset) { unsigned long src; - int rc; - if (!(iter_is_iovec(iter) || iov_iter_is_kvec(iter))) - return -EINVAL; - /* Multi-segment iterators are not supported */ - if (iter->nr_segs > 1) - return -EINVAL; - if (!csize) - return 0; src = pfn_to_phys(pfn) + offset; - - /* XXX: pass the iov_iter down to a common function */ - if (iter_is_iovec(iter)) - rc = copy_oldmem_user(iter->iov->iov_base, src, csize); - else - rc = copy_oldmem_kernel(iter->kvec->iov_base, src, csize); - if (rc < 0) - return rc; - iov_iter_advance(iter, csize); - return csize; + return copy_oldmem_iter(iter, src, csize); } /* diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index 92b32ce645b9..f6da215ccf9f 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -54,38 +55,37 @@ static DEFINE_MUTEX(hsa_buf_mutex); static char hsa_buf[PAGE_SIZE] __aligned(PAGE_SIZE); /* - * Copy memory from HSA to user memory (not reentrant): + * Copy memory from HSA to iterator (not reentrant): * - * @dest: User buffer where memory should be copied to + * @iter: Iterator where memory should be copied to * @src: Start address within HSA where data should be copied * @count: Size of buffer, which should be copied */ -int memcpy_hsa_user(void __user *dest, unsigned long src, size_t count) +size_t memcpy_hsa_iter(struct iov_iter *iter, unsigned long src, size_t count) { - unsigned long offset, bytes; + size_t bytes, copied, res = 0; + unsigned long offset; if (!hsa_available) - return -ENODATA; + return 0; mutex_lock(&hsa_buf_mutex); while (count) { if (sclp_sdias_copy(hsa_buf, src / PAGE_SIZE + 2, 1)) { TRACE("sclp_sdias_copy() failed\n"); - mutex_unlock(&hsa_buf_mutex); - return -EIO; + break; } offset = src % PAGE_SIZE; bytes = min(PAGE_SIZE - offset, count); - if (copy_to_user(dest, hsa_buf + offset, bytes)) { - mutex_unlock(&hsa_buf_mutex); - return -EFAULT; - } - src += bytes; - dest += bytes; - count -= bytes; + copied = copy_to_iter(hsa_buf + offset, bytes, iter); + count -= copied; + src += copied; + res += copied; + if (copied < bytes) + break; } mutex_unlock(&hsa_buf_mutex); - return 0; + return res; } /* @@ -95,28 +95,16 @@ int memcpy_hsa_user(void __user *dest, unsigned long src, size_t count) * @src: Start address within HSA where data should be copied * @count: Size of buffer, which should be copied */ -int memcpy_hsa_kernel(void *dest, unsigned long src, size_t count) +static inline int memcpy_hsa_kernel(void *dst, unsigned long src, size_t count) { - unsigned long offset, bytes; + struct iov_iter iter; + struct kvec kvec; - if (!hsa_available) - return -ENODATA; - - mutex_lock(&hsa_buf_mutex); - while (count) { - if (sclp_sdias_copy(hsa_buf, src / PAGE_SIZE + 2, 1)) { - TRACE("sclp_sdias_copy() failed\n"); - mutex_unlock(&hsa_buf_mutex); - return -EIO; - } - offset = src % PAGE_SIZE; - bytes = min(PAGE_SIZE - offset, count); - memcpy(dest, hsa_buf + offset, bytes); - src += bytes; - dest += bytes; - count -= bytes; - } - mutex_unlock(&hsa_buf_mutex); + kvec.iov_base = dst; + kvec.iov_len = count; + iov_iter_kvec(&iter, WRITE, &kvec, 1, count); + if (memcpy_hsa_iter(&iter, src, count) < count) + return -EIO; return 0; } From d384690c6a2c8e6d5d09d4333f0c4b987d3c7049 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Thu, 21 Jul 2022 15:29:05 +0200 Subject: [PATCH 32/46] s390/docs: fix warnings for vfio_ap driver lock usage doc Fix multiple warnings produced by make htmldocs Fixes: e32d3827f3d5 ("s390/Docs: new doc describing lock usage by the vfio_ap device driver") Signed-off-by: Alexander Gordeev --- Documentation/s390/index.rst | 1 + Documentation/s390/vfio-ap-locking.rst | 68 +++++++++++++++----------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/Documentation/s390/index.rst b/Documentation/s390/index.rst index b10ca9192557..73c79bf586fd 100644 --- a/Documentation/s390/index.rst +++ b/Documentation/s390/index.rst @@ -12,6 +12,7 @@ s390 Architecture qeth s390dbf vfio-ap + vfio-ap-locking vfio-ccw zfcpdump common_io diff --git a/Documentation/s390/vfio-ap-locking.rst b/Documentation/s390/vfio-ap-locking.rst index c4e1eeec79a0..0dfcdb562e21 100644 --- a/Documentation/s390/vfio-ap-locking.rst +++ b/Documentation/s390/vfio-ap-locking.rst @@ -7,19 +7,23 @@ This document describes the locks that are pertinent to the secure operation of the vfio_ap device driver. Throughout this document, the following variables will be used to denote instances of the structures herein described: -struct ap_matrix_dev *matrix_dev; -struct ap_matrix_mdev *matrix_mdev; -struct kvm *kvm; +.. code-block:: c + + struct ap_matrix_dev *matrix_dev; + struct ap_matrix_mdev *matrix_mdev; + struct kvm *kvm; The Matrix Devices Lock (drivers/s390/crypto/vfio_ap_private.h) --------------------------------------------------------------- +--------------------------------------------------------------- -struct ap_matrix_dev { - ... - struct list_head mdev_list; - struct mutex mdevs_lock; - ... -} +.. code-block:: c + + struct ap_matrix_dev { + ... + struct list_head mdev_list; + struct mutex mdevs_lock; + ... + } The Matrix Devices Lock (matrix_dev->mdevs_lock) is implemented as a global mutex contained within the single object of struct ap_matrix_dev. This lock @@ -31,11 +35,13 @@ representing one of the vfio_ap device driver's mediated devices. The KVM Lock (include/linux/kvm_host.h) --------------------------------------- -struct kvm { - ... - struct mutex lock; - ... -} +.. code-block:: c + + struct kvm { + ... + struct mutex lock; + ... + } The KVM Lock (kvm->lock) controls access to the state data for a KVM guest. This lock must be held by the vfio_ap device driver while one or more AP adapters, @@ -48,12 +54,14 @@ been attached to the KVM guest. The Guests Lock (drivers/s390/crypto/vfio_ap_private.h) ----------------------------------------------------------- -struct ap_matrix_dev { - ... - struct list_head mdev_list; - struct mutex guests_lock; - ... -} +.. code-block:: c + + struct ap_matrix_dev { + ... + struct list_head mdev_list; + struct mutex guests_lock; + ... + } The Guests Lock (matrix_dev->guests_lock) controls access to the matrix_mdev instances (matrix_dev->mdev_list) that represent mediated devices @@ -89,17 +97,19 @@ resources, so only the matrix_dev->mdevs_lock needs to be held. The PQAP Hook Lock (arch/s390/include/asm/kvm_host.h) ----------------------------------------------------- -typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); +.. code-block:: c -struct kvm_s390_crypto { - ... - struct rw_semaphore pqap_hook_rwsem; - crypto_hook *pqap_hook; - ... -}; + typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); + + struct kvm_s390_crypto { + ... + struct rw_semaphore pqap_hook_rwsem; + crypto_hook *pqap_hook; + ... + }; The PQAP Hook Lock is a r/w semaphore that controls access to the function -pointer of the handler (*kvm->arch.crypto.pqap_hook) to invoke when the +pointer of the handler ``(*kvm->arch.crypto.pqap_hook)`` to invoke when the PQAP(AQIC) instruction sub-function is intercepted by the host. The lock must be held in write mode when pqap_hook value is set, and in read mode when the pqap_hook function is called. From 9aa938ddce81c045f627468b395714768945a90f Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Thu, 21 Jul 2022 15:59:36 +0200 Subject: [PATCH 33/46] s390/docs: fix warnings for vfio_ap driver doc Fix multiple warnings produced by make htmldocs Fixes: cb269e0aba7c ("s390/vfio-ap: update docs to include dynamic config support") Signed-off-by: Alexander Gordeev --- Documentation/s390/vfio-ap.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Documentation/s390/vfio-ap.rst b/Documentation/s390/vfio-ap.rst index f4b8748ab9a8..61a0a3c6c7b4 100644 --- a/Documentation/s390/vfio-ap.rst +++ b/Documentation/s390/vfio-ap.rst @@ -681,7 +681,7 @@ These are the steps: default drivers pool: adapter 0-15, domain 1 alternate drivers pool: adapter 16-255, domains 0, 2-255 - Note ***: + **Note:** Changing a mask such that one or more APQNs will be taken from a vfio_ap mediated device (see below) will fail with an error (EBUSY). A message is logged to the kernel ring buffer which can be viewed with the 'dmesg' @@ -940,7 +940,7 @@ host. If the vfio_ap mdev is removed, one may want to also reconfigure the pool of adapters and queues reserved for use by the default drivers. Hot plug/unplug support: -================ +======================== An adapter, domain or control domain may be hot plugged into a running KVM guest by assigning it to the vfio_ap mediated device being used by the guest if the following conditions are met: @@ -962,7 +962,7 @@ guest by unassigning it from the vfio_ap mediated device being used by the guest. Over-provisioning of AP queues for a KVM guest: -============================================== +=============================================== Over-provisioning is defined herein as the assignment of adapters or domains to a vfio_ap mediated device that do not reference AP devices in the host's AP configuration. The idea here is that when the adapter or domain becomes @@ -992,6 +992,8 @@ its mdev can be hot unplugged from the guest in one of two ways: The contents of my-guest-hostdev.xml: +.. code-block:: xml +
@@ -1004,6 +1006,8 @@ its mdev can be hot unplugged from the guest in one of two ways: For example, to hot unplug the vfio_ap mediated device identified on the qemu command line with 'id=hostdev0' from the guest named 'my-guest': +.. code-block:: sh + virsh qemu-monitor-command my-guest --hmp "device_del hostdev0" 2. A vfio_ap mediated device can be hot unplugged by attaching the qemu monitor @@ -1032,6 +1036,8 @@ system into the guest in one of two ways: The contents of my-guest-hostdev.xml: +.. code-block:: xml +
From 57c3ae8e44bba158c3f7064a3f9628eddfd5c276 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 14 Jul 2022 19:00:10 -0700 Subject: [PATCH 34/46] s390/hmcdrv: fix Kconfig "its" grammar Use the possessive "its" instead of the contraction "it's" where appropriate. Signed-off-by: Randy Dunlap Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: linux-s390@vger.kernel.org Link: https://lore.kernel.org/r/20220715020010.12678-1-rdunlap@infradead.org Signed-off-by: Alexander Gordeev --- drivers/s390/char/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/char/Kconfig b/drivers/s390/char/Kconfig index 57f41efb8043..7d1749b0d378 100644 --- a/drivers/s390/char/Kconfig +++ b/drivers/s390/char/Kconfig @@ -89,7 +89,7 @@ config HMC_DRV Management Console (HMC) drive CD/DVD-ROM. It is available as a module, called 'hmcdrv', and also as kernel built-in. There is one optional parameter for this module: cachesize=N, which modifies the - transfer cache size from it's default value 0.5MB to N bytes. If N + transfer cache size from its default value 0.5MB to N bytes. If N is zero, then no caching is performed. config SCLP_OFB From fc7fab3f9156df73cd5f2705f001451265453d34 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 15 Jul 2022 13:38:38 +0800 Subject: [PATCH 35/46] s390/tape: fix comment typo Remove duplicated `that' in a comment Signed-off-by: Jason Wang Link: https://lore.kernel.org/r/20220715053838.5005-1-wangborong@cdjrlc.com [agordeev@linux.ibm.com rephrased commit message] Signed-off-by: Alexander Gordeev --- drivers/s390/char/tape_34xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/char/tape_34xx.c b/drivers/s390/char/tape_34xx.c index 38cc1565d6ae..751945fb6793 100644 --- a/drivers/s390/char/tape_34xx.c +++ b/drivers/s390/char/tape_34xx.c @@ -548,7 +548,7 @@ tape_34xx_unit_check(struct tape_device *device, struct tape_request *request, case 0x2e: /* * Not capable. This indicates either that the drive fails - * reading the format id mark or that that format specified + * reading the format id mark or that format specified * is not supported by the drive. */ dev_warn (&device->cdev->dev, "The tape unit cannot process " From 6f5c672d17f583b081e283927f5040f726c54598 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 20 Jul 2022 07:24:03 +0200 Subject: [PATCH 36/46] s390/smp: enforce lowcore protection on CPU restart As result of commit 915fea04f932 ("s390/smp: enable DAT before CPU restart callback is called") the low-address protection bit gets mistakenly unset in control register 0 save area of the absolute zero memory. That area is used when manual PSW restart happened to hit an offline CPU. In this case the low-address protection for that CPU will be dropped. Reviewed-by: Heiko Carstens Fixes: 915fea04f932 ("s390/smp: enable DAT before CPU restart callback is called") Signed-off-by: Alexander Gordeev --- arch/s390/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 8d91eccc0963..792ba69032d5 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -508,8 +508,8 @@ static void __init setup_lowcore_dat_on(void) S390_lowcore.svc_new_psw.mask |= PSW_MASK_DAT; S390_lowcore.program_new_psw.mask |= PSW_MASK_DAT; S390_lowcore.io_new_psw.mask |= PSW_MASK_DAT; - __ctl_store(S390_lowcore.cregs_save_area, 0, 15); __ctl_set_bit(0, 28); + __ctl_store(S390_lowcore.cregs_save_area, 0, 15); put_abs_lowcore(restart_flags, RESTART_FLAG_CTLREGS); put_abs_lowcore(program_new_psw, lc->program_new_psw); for (cr = 0; cr < ARRAY_SIZE(lc->cregs_save_area); cr++) From 57ad19bcdefb1c65b0a90330b7b29ce658ef1a76 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Thu, 26 May 2022 07:57:36 +0200 Subject: [PATCH 37/46] s390/boot: cleanup adjust_to_uv_max() function Uncouple input and output arguments by making the latter the function return value. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/boot/startup.c | 10 ++++++---- arch/s390/boot/uv.c | 5 +++-- arch/s390/boot/uv.h | 7 +++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 863e6bcaa5a1..bc48fe82d949 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -152,6 +152,7 @@ static void setup_kernel_memory_layout(void) unsigned long vmemmap_start; unsigned long rte_size; unsigned long pages; + unsigned long vmax; pages = ident_map_size / PAGE_SIZE; /* vmemmap contains a multiple of PAGES_PER_SECTION struct pages */ @@ -163,10 +164,10 @@ static void setup_kernel_memory_layout(void) vmalloc_size > _REGION2_SIZE || vmemmap_start + vmemmap_size + vmalloc_size + MODULES_LEN > _REGION2_SIZE) { - MODULES_END = _REGION1_SIZE; + vmax = _REGION1_SIZE; rte_size = _REGION2_SIZE; } else { - MODULES_END = _REGION2_SIZE; + vmax = _REGION2_SIZE; rte_size = _REGION3_SIZE; } /* @@ -174,11 +175,12 @@ static void setup_kernel_memory_layout(void) * secure storage limit, so that any vmalloc allocation * we do could be used to back secure guest storage. */ - adjust_to_uv_max(&MODULES_END); + vmax = adjust_to_uv_max(vmax); #ifdef CONFIG_KASAN /* force vmalloc and modules below kasan shadow */ - MODULES_END = min(MODULES_END, KASAN_SHADOW_START); + vmax = min(vmax, KASAN_SHADOW_START); #endif + MODULES_END = vmax; MODULES_VADDR = MODULES_END - MODULES_LEN; VMALLOC_END = MODULES_VADDR; diff --git a/arch/s390/boot/uv.c b/arch/s390/boot/uv.c index e6be155ab2e5..d13ce34ab9f1 100644 --- a/arch/s390/boot/uv.c +++ b/arch/s390/boot/uv.c @@ -51,10 +51,11 @@ void uv_query_info(void) } #if IS_ENABLED(CONFIG_KVM) -void adjust_to_uv_max(unsigned long *vmax) +unsigned long adjust_to_uv_max(unsigned long limit) { if (is_prot_virt_host() && uv_info.max_sec_stor_addr) - *vmax = min_t(unsigned long, *vmax, uv_info.max_sec_stor_addr); + limit = min_t(unsigned long, limit, uv_info.max_sec_stor_addr); + return limit; } static int is_prot_virt_host_capable(void) diff --git a/arch/s390/boot/uv.h b/arch/s390/boot/uv.h index 690ce019af5a..0f3070856f8d 100644 --- a/arch/s390/boot/uv.h +++ b/arch/s390/boot/uv.h @@ -3,10 +3,13 @@ #define BOOT_UV_H #if IS_ENABLED(CONFIG_KVM) -void adjust_to_uv_max(unsigned long *vmax); +unsigned long adjust_to_uv_max(unsigned long limit); void sanitize_prot_virt_host(void); #else -static inline void adjust_to_uv_max(unsigned long *vmax) {} +static inline unsigned long adjust_to_uv_max(unsigned long limit) +{ + return limit; +} static inline void sanitize_prot_virt_host(void) {} #endif From 2e2493c675dfdb75d8aa1bc6ae436b9d7cc4abfa Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Fri, 22 Jul 2022 20:34:05 +0200 Subject: [PATCH 38/46] s390/setup: rearrange absolute lowcore initialization Make the absolute lowcore assignments immediately follow the boot CPU lowcore same member assignments. This way readability improves when reading from up to down, with no out of order mcck stack allocation in-between. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/setup.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 792ba69032d5..8f483132901e 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -474,19 +474,18 @@ static void __init setup_lowcore_dat_off(void) lc->restart_data = 0; lc->restart_source = -1U; - mcck_stack = (unsigned long)memblock_alloc(THREAD_SIZE, THREAD_SIZE); - if (!mcck_stack) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, THREAD_SIZE, THREAD_SIZE); - lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET; - - /* Setup absolute zero lowcore */ put_abs_lowcore(restart_stack, lc->restart_stack); put_abs_lowcore(restart_fn, lc->restart_fn); put_abs_lowcore(restart_data, lc->restart_data); put_abs_lowcore(restart_source, lc->restart_source); put_abs_lowcore(restart_psw, lc->restart_psw); + mcck_stack = (unsigned long)memblock_alloc(THREAD_SIZE, THREAD_SIZE); + if (!mcck_stack) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, THREAD_SIZE, THREAD_SIZE); + lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET; + lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; arch_spin_lock_setup(0); From 7d06fed77b7d8fc9f6cc41b4e3f2823d32532ad8 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 20 Jul 2022 08:22:01 +0200 Subject: [PATCH 39/46] s390/smp: rework absolute lowcore access Temporary unsetting of the prefix page in memcpy_absolute() routine poses a risk of executing code path with unexpectedly disabled prefix page. This rework avoids the prefix page uninstalling and disabling of normal and machine check interrupts when accessing the absolute zero memory. Although memcpy_absolute() routine can access the whole memory, it is only used to update the absolute zero lowcore. This rework therefore introduces a new mechanism for the absolute zero lowcore access and scraps memcpy_absolute() routine for good. Instead, an area is reserved in the virtual memory that is used for the absolute lowcore access only. That area holds an array of 8KB virtual mappings - one per CPU. Whenever a CPU is brought online, the corresponding item is mapped to the real address of the previously installed prefix page. The absolute zero lowcore access works like this: a CPU calls the new primitive get_abs_lowcore() to obtain its 8KB mapping as a pointer to the struct lowcore. Virtual address references to that pointer get translated to the real addresses of the prefix page, which in turn gets swapped with the absolute zero memory addresses due to prefixing. Once the pointer is not needed it must be released with put_abs_lowcore() primitive: struct lowcore *abs_lc; unsigned long flags; abs_lc = get_abs_lowcore(&flags); abs_lc->... = ...; put_abs_lowcore(abs_lc, flags); To ensure the described mechanism works large segment- and region- table entries must be avoided for the 8KB mappings. Failure to do so results in usage of Region-Frame Absolute Address (RFAA) or Segment-Frame Absolute Address (SFAA) large page fields. In that case absolute addresses would be used to address the prefix page instead of the real ones and the prefixing would get bypassed. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/boot/startup.c | 5 +- arch/s390/include/asm/abs_lowcore.h | 17 ++++++ arch/s390/include/asm/pgtable.h | 2 + arch/s390/include/asm/processor.h | 15 ----- arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/abs_lowcore.c | 88 +++++++++++++++++++++++++++++ arch/s390/kernel/ipl.c | 9 ++- arch/s390/kernel/machine_kexec.c | 8 ++- arch/s390/kernel/os_info.c | 9 ++- arch/s390/kernel/setup.c | 34 ++++++----- arch/s390/kernel/smp.c | 34 ++++++++--- arch/s390/mm/init.c | 2 +- arch/s390/mm/maccess.c | 67 ++++++++++------------ arch/s390/mm/vmem.c | 85 ++++++++++++++++++++++++++++ 14 files changed, 294 insertions(+), 83 deletions(-) create mode 100644 arch/s390/include/asm/abs_lowcore.h create mode 100644 arch/s390/kernel/abs_lowcore.c diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index bc48fe82d949..41b7af7a9365 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -10,11 +10,13 @@ #include #include #include +#include #include "decompressor.h" #include "boot.h" #include "uv.h" unsigned long __bootdata_preserved(__kaslr_offset); +unsigned long __bootdata_preserved(__abs_lowcore); unsigned long __bootdata(__amode31_base); unsigned long __bootdata_preserved(VMALLOC_START); unsigned long __bootdata_preserved(VMALLOC_END); @@ -180,7 +182,8 @@ static void setup_kernel_memory_layout(void) /* force vmalloc and modules below kasan shadow */ vmax = min(vmax, KASAN_SHADOW_START); #endif - MODULES_END = vmax; + __abs_lowcore = round_down(vmax - ABS_LOWCORE_MAP_SIZE, sizeof(struct lowcore)); + MODULES_END = round_down(__abs_lowcore, _SEGMENT_SIZE); MODULES_VADDR = MODULES_END - MODULES_LEN; VMALLOC_END = MODULES_VADDR; diff --git a/arch/s390/include/asm/abs_lowcore.h b/arch/s390/include/asm/abs_lowcore.h new file mode 100644 index 000000000000..bdef8d24d237 --- /dev/null +++ b/arch/s390/include/asm/abs_lowcore.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_ABS_LOWCORE_H +#define _ASM_S390_ABS_LOWCORE_H + +#include + +#define ABS_LOWCORE_MAP_SIZE (NR_CPUS * sizeof(struct lowcore)) + +extern unsigned long __abs_lowcore; +extern bool abs_lowcore_mapped; + +struct lowcore *get_abs_lowcore(unsigned long *flags); +void put_abs_lowcore(struct lowcore *lc, unsigned long flags); +int abs_lowcore_map(int cpu, struct lowcore *lc); +void abs_lowcore_unmap(int cpu); + +#endif /* _ASM_ABS_S390_LOWCORE_H */ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index a397b072a580..82506ebd544b 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1781,6 +1781,8 @@ static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset) extern int vmem_add_mapping(unsigned long start, unsigned long size); extern void vmem_remove_mapping(unsigned long start, unsigned long size); +extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot); +extern void vmem_unmap_4k_page(unsigned long addr); extern int s390_enable_sie(void); extern int s390_enable_skey(void); extern void s390_reset_cmma(struct mm_struct *mm); diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index bd66f8e34949..93677ae89e7e 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -307,21 +307,6 @@ static __always_inline void __noreturn disabled_wait(void) #define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL extern int memcpy_real(void *, unsigned long, size_t); -extern void memcpy_absolute(void *, void *, size_t); - -#define put_abs_lowcore(member, x) do { \ - unsigned long __abs_address = offsetof(struct lowcore, member); \ - __typeof__(((struct lowcore *)0)->member) __tmp = (x); \ - \ - memcpy_absolute(__va(__abs_address), &__tmp, sizeof(__tmp)); \ -} while (0) - -#define get_abs_lowcore(x, member) do { \ - unsigned long __abs_address = offsetof(struct lowcore, member); \ - __typeof__(((struct lowcore *)0)->member) *__ptr = &(x); \ - \ - memcpy_absolute(__ptr, __va(__abs_address), sizeof(*__ptr)); \ -} while (0) extern int s390_isolate_bp(void); extern int s390_isolate_bp_guest(void); diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 3cbfa9fddd9a..45e4b2f41e05 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -40,7 +40,7 @@ obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o -obj-y += smp.o text_amode31.o stacktrace.o +obj-y += smp.o text_amode31.o stacktrace.o abs_lowcore.o extra-y += head64.o vmlinux.lds diff --git a/arch/s390/kernel/abs_lowcore.c b/arch/s390/kernel/abs_lowcore.c new file mode 100644 index 000000000000..dc9f0ecd4695 --- /dev/null +++ b/arch/s390/kernel/abs_lowcore.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#define ABS_LOWCORE_UNMAPPED 1 +#define ABS_LOWCORE_LAP_ON 2 +#define ABS_LOWCORE_IRQS_ON 4 + +unsigned long __bootdata_preserved(__abs_lowcore); +bool __ro_after_init abs_lowcore_mapped; + +int abs_lowcore_map(int cpu, struct lowcore *lc) +{ + unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore)); + unsigned long phys = __pa(lc); + int rc, i; + + for (i = 0; i < LC_PAGES; i++) { + rc = vmem_map_4k_page(addr, phys, PAGE_KERNEL); + if (rc) { + for (--i; i >= 0; i--) { + addr -= PAGE_SIZE; + vmem_unmap_4k_page(addr); + } + return rc; + } + addr += PAGE_SIZE; + phys += PAGE_SIZE; + } + return 0; +} + +void abs_lowcore_unmap(int cpu) +{ + unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore)); + int i; + + for (i = 0; i < LC_PAGES; i++) { + vmem_unmap_4k_page(addr); + addr += PAGE_SIZE; + } +} + +struct lowcore *get_abs_lowcore(unsigned long *flags) +{ + unsigned long irq_flags; + union ctlreg0 cr0; + int cpu; + + *flags = 0; + cpu = get_cpu(); + if (abs_lowcore_mapped) { + return ((struct lowcore *)__abs_lowcore) + cpu; + } else { + if (cpu != 0) + panic("Invalid unmapped absolute lowcore access\n"); + local_irq_save(irq_flags); + if (!irqs_disabled_flags(irq_flags)) + *flags |= ABS_LOWCORE_IRQS_ON; + __ctl_store(cr0.val, 0, 0); + if (cr0.lap) { + *flags |= ABS_LOWCORE_LAP_ON; + __ctl_clear_bit(0, 28); + } + *flags |= ABS_LOWCORE_UNMAPPED; + return lowcore_ptr[0]; + } +} + +void put_abs_lowcore(struct lowcore *lc, unsigned long flags) +{ + if (abs_lowcore_mapped) { + if (flags) + panic("Invalid mapped absolute lowcore release\n"); + } else { + if (smp_processor_id() != 0) + panic("Invalid mapped absolute lowcore access\n"); + if (!(flags & ABS_LOWCORE_UNMAPPED)) + panic("Invalid unmapped absolute lowcore release\n"); + if (flags & ABS_LOWCORE_LAP_ON) + __ctl_set_bit(0, 28); + if (flags & ABS_LOWCORE_IRQS_ON) + local_irq_enable(); + } + put_cpu(); +} diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 1cc85b8ff42e..325cbf69ebbd 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1642,12 +1643,16 @@ static struct shutdown_action __refdata dump_action = { static void dump_reipl_run(struct shutdown_trigger *trigger) { unsigned long ipib = (unsigned long) reipl_block_actual; + struct lowcore *abs_lc; + unsigned long flags; unsigned int csum; csum = (__force unsigned int) csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); - put_abs_lowcore(ipib, ipib); - put_abs_lowcore(ipib_checksum, csum); + abs_lc = get_abs_lowcore(&flags); + abs_lc->ipib = ipib; + abs_lc->ipib_checksum = csum; + put_abs_lowcore(abs_lc, flags); dump_run(trigger); } diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index ab761c008f98..4579b42286d5 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -222,13 +223,18 @@ void machine_kexec_cleanup(struct kimage *image) void arch_crash_save_vmcoreinfo(void) { + struct lowcore *abs_lc; + unsigned long flags; + VMCOREINFO_SYMBOL(lowcore_ptr); VMCOREINFO_SYMBOL(high_memory); VMCOREINFO_LENGTH(lowcore_ptr, NR_CPUS); vmcoreinfo_append_str("SAMODE31=%lx\n", __samode31); vmcoreinfo_append_str("EAMODE31=%lx\n", __eamode31); vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); - put_abs_lowcore(vmcore_info, paddr_vmcoreinfo_note()); + abs_lc = get_abs_lowcore(&flags); + abs_lc->vmcore_info = paddr_vmcoreinfo_note(); + put_abs_lowcore(abs_lc, flags); } void machine_shutdown(void) diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 1acc2e05d70f..506ccb74d2d0 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include @@ -57,13 +57,16 @@ void os_info_entry_add(int nr, void *ptr, u64 size) */ void __init os_info_init(void) { - void *ptr = &os_info; + struct lowcore *abs_lc; + unsigned long flags; os_info.version_major = OS_INFO_VERSION_MAJOR; os_info.version_minor = OS_INFO_VERSION_MINOR; os_info.magic = OS_INFO_MAGIC; os_info.csum = os_info_csum(&os_info); - put_abs_lowcore(os_info, __pa(ptr)); + abs_lc = get_abs_lowcore(&flags); + abs_lc->os_info = __pa(&os_info); + put_abs_lowcore(abs_lc, flags); } #ifdef CONFIG_CRASH_DUMP diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 8f483132901e..91139a16a44f 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -58,7 +58,7 @@ #include #include #include -#include +#include #include #include #include @@ -411,8 +411,9 @@ void __init arch_call_rest_init(void) static void __init setup_lowcore_dat_off(void) { unsigned long int_psw_mask = PSW_KERNEL_BITS; + struct lowcore *abs_lc, *lc; unsigned long mcck_stack; - struct lowcore *lc; + unsigned long flags; if (IS_ENABLED(CONFIG_KASAN)) int_psw_mask |= PSW_MASK_DAT; @@ -474,11 +475,13 @@ static void __init setup_lowcore_dat_off(void) lc->restart_data = 0; lc->restart_source = -1U; - put_abs_lowcore(restart_stack, lc->restart_stack); - put_abs_lowcore(restart_fn, lc->restart_fn); - put_abs_lowcore(restart_data, lc->restart_data); - put_abs_lowcore(restart_source, lc->restart_source); - put_abs_lowcore(restart_psw, lc->restart_psw); + abs_lc = get_abs_lowcore(&flags); + abs_lc->restart_stack = lc->restart_stack; + abs_lc->restart_fn = lc->restart_fn; + abs_lc->restart_data = lc->restart_data; + abs_lc->restart_source = lc->restart_source; + abs_lc->restart_psw = lc->restart_psw; + put_abs_lowcore(abs_lc, flags); mcck_stack = (unsigned long)memblock_alloc(THREAD_SIZE, THREAD_SIZE); if (!mcck_stack) @@ -499,8 +502,8 @@ static void __init setup_lowcore_dat_off(void) static void __init setup_lowcore_dat_on(void) { - struct lowcore *lc = lowcore_ptr[0]; - int cr; + struct lowcore *abs_lc; + unsigned long flags; __ctl_clear_bit(0, 28); S390_lowcore.external_new_psw.mask |= PSW_MASK_DAT; @@ -509,10 +512,15 @@ static void __init setup_lowcore_dat_on(void) S390_lowcore.io_new_psw.mask |= PSW_MASK_DAT; __ctl_set_bit(0, 28); __ctl_store(S390_lowcore.cregs_save_area, 0, 15); - put_abs_lowcore(restart_flags, RESTART_FLAG_CTLREGS); - put_abs_lowcore(program_new_psw, lc->program_new_psw); - for (cr = 0; cr < ARRAY_SIZE(lc->cregs_save_area); cr++) - put_abs_lowcore(cregs_save_area[cr], lc->cregs_save_area[cr]); + abs_lc = get_abs_lowcore(&flags); + abs_lc->restart_flags = RESTART_FLAG_CTLREGS; + abs_lc->program_new_psw = S390_lowcore.program_new_psw; + memcpy(abs_lc->cregs_save_area, S390_lowcore.cregs_save_area, + sizeof(abs_lc->cregs_save_area)); + put_abs_lowcore(abs_lc, flags); + if (abs_lowcore_map(0, lowcore_ptr[0])) + panic("Couldn't setup absolute lowcore"); + abs_lowcore_mapped = true; } static struct resource code_resource = { diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 30c91d565933..40876d809ea6 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -45,7 +45,7 @@ #include #include #include -#include +#include #include #include #include @@ -212,10 +212,14 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) lc->preempt_count = PREEMPT_DISABLED; if (nmi_alloc_mcesa(&lc->mcesad)) goto out; + if (abs_lowcore_map(cpu, lc)) + goto out_mcesa; lowcore_ptr[cpu] = lc; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc)); return 0; +out_mcesa: + nmi_free_mcesa(&lc->mcesad); out: stack_free(mcck_stack); stack_free(async_stack); @@ -237,6 +241,7 @@ static void pcpu_free_lowcore(struct pcpu *pcpu) mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0); lowcore_ptr[cpu] = NULL; + abs_lowcore_unmap(cpu); nmi_free_mcesa(&lc->mcesad); stack_free(async_stack); stack_free(mcck_stack); @@ -315,9 +320,12 @@ static void pcpu_delegate(struct pcpu *pcpu, pcpu_delegate_fn *func, void *data, unsigned long stack) { - struct lowcore *lc = lowcore_ptr[pcpu - pcpu_devices]; - unsigned int source_cpu = stap(); + struct lowcore *lc, *abs_lc; + unsigned int source_cpu; + unsigned long flags; + lc = lowcore_ptr[pcpu - pcpu_devices]; + source_cpu = stap(); __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT); if (pcpu->address == source_cpu) { call_on_stack(2, stack, void, __pcpu_delegate, @@ -332,10 +340,12 @@ static void pcpu_delegate(struct pcpu *pcpu, lc->restart_data = (unsigned long)data; lc->restart_source = source_cpu; } else { - put_abs_lowcore(restart_stack, stack); - put_abs_lowcore(restart_fn, (unsigned long)func); - put_abs_lowcore(restart_data, (unsigned long)data); - put_abs_lowcore(restart_source, source_cpu); + abs_lc = get_abs_lowcore(&flags); + abs_lc->restart_stack = stack; + abs_lc->restart_fn = (unsigned long)func; + abs_lc->restart_data = (unsigned long)data; + abs_lc->restart_source = source_cpu; + put_abs_lowcore(abs_lc, flags); } __bpon(); asm volatile( @@ -581,6 +591,8 @@ static DEFINE_SPINLOCK(ctl_lock); void smp_ctl_set_clear_bit(int cr, int bit, bool set) { struct ec_creg_mask_parms parms = { .cr = cr, }; + struct lowcore *abs_lc; + unsigned long flags; u64 ctlreg; if (set) { @@ -591,9 +603,11 @@ void smp_ctl_set_clear_bit(int cr, int bit, bool set) parms.andval = ~(1UL << bit); } spin_lock(&ctl_lock); - get_abs_lowcore(ctlreg, cregs_save_area[cr]); + abs_lc = get_abs_lowcore(&flags); + ctlreg = abs_lc->cregs_save_area[cr]; ctlreg = (ctlreg & parms.andval) | parms.orval; - put_abs_lowcore(cregs_save_area[cr], ctlreg); + abs_lc->cregs_save_area[cr] = ctlreg; + put_abs_lowcore(abs_lc, flags); spin_unlock(&ctl_lock); on_each_cpu(smp_ctl_bit_callback, &parms, 1); } @@ -1281,6 +1295,8 @@ static int __init smp_reinit_ipl_cpu(void) __ctl_clear_bit(0, 28); /* disable lowcore protection */ S390_lowcore.mcesad = mcesad; __ctl_load(cr0, 0, 0); + if (abs_lowcore_map(0, lc)) + panic("Couldn't remap absolute lowcore"); lowcore_ptr[0] = lc; local_mcck_enable(); local_irq_restore(flags); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 6a0ac00d5a42..7b6873ac99d1 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index d6d84e02f35a..b8451ddbb3d6 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -15,6 +15,7 @@ #include #include #include +#include #include static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) @@ -148,46 +149,20 @@ int memcpy_real(void *dest, unsigned long src, size_t count) } /* - * Copy memory in absolute mode (kernel to kernel) + * Find CPU that owns swapped prefix page */ -void memcpy_absolute(void *dest, void *src, size_t count) -{ - unsigned long cr0, flags, prefix; - - flags = arch_local_irq_save(); - __ctl_store(cr0, 0, 0); - __ctl_clear_bit(0, 28); /* disable lowcore protection */ - prefix = store_prefix(); - if (prefix) { - local_mcck_disable(); - set_prefix(0); - memcpy(dest, src, count); - set_prefix(prefix); - local_mcck_enable(); - } else { - memcpy(dest, src, count); - } - __ctl_load(cr0, 0, 0); - arch_local_irq_restore(flags); -} - -/* - * Check if physical address is within prefix or zero page - */ -static int is_swapped(phys_addr_t addr) +static int get_swapped_owner(phys_addr_t addr) { phys_addr_t lc; int cpu; - if (addr < sizeof(struct lowcore)) - return 1; for_each_online_cpu(cpu) { lc = virt_to_phys(lowcore_ptr[cpu]); if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) continue; - return 1; + return cpu; } - return 0; + return -1; } /* @@ -200,17 +175,35 @@ void *xlate_dev_mem_ptr(phys_addr_t addr) { void *ptr = phys_to_virt(addr); void *bounce = ptr; + struct lowcore *abs_lc; + unsigned long flags; unsigned long size; + int this_cpu, cpu; cpus_read_lock(); - preempt_disable(); - if (is_swapped(addr)) { - size = PAGE_SIZE - (addr & ~PAGE_MASK); - bounce = (void *) __get_free_page(GFP_ATOMIC); - if (bounce) - memcpy_absolute(bounce, ptr, size); + this_cpu = get_cpu(); + if (addr >= sizeof(struct lowcore)) { + cpu = get_swapped_owner(addr); + if (cpu < 0) + goto out; } - preempt_enable(); + bounce = (void *)__get_free_page(GFP_ATOMIC); + if (!bounce) + goto out; + size = PAGE_SIZE - (addr & ~PAGE_MASK); + if (addr < sizeof(struct lowcore)) { + abs_lc = get_abs_lowcore(&flags); + ptr = (void *)abs_lc + addr; + memcpy(bounce, ptr, size); + put_abs_lowcore(abs_lc, flags); + } else if (cpu == this_cpu) { + ptr = (void *)(addr - virt_to_phys(lowcore_ptr[cpu])); + memcpy(bounce, ptr, size); + } else { + memcpy(bounce, ptr, size); + } +out: + put_cpu(); cpus_read_unlock(); return bounce; } diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index c2583f921ca8..203ba2bfea59 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -560,6 +560,91 @@ int vmem_add_mapping(unsigned long start, unsigned long size) return ret; } +/* + * Allocate new or return existing page-table entry, but do not map it + * to any physical address. If missing, allocate segment- and region- + * table entries along. Meeting a large segment- or region-table entry + * while traversing is an error, since the function is expected to be + * called against virtual regions reserverd for 4KB mappings only. + */ +static pte_t *vmem_get_alloc_pte(unsigned long addr) +{ + pte_t *ptep = NULL; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = pgd_offset_k(addr); + if (pgd_none(*pgd)) { + p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); + if (!p4d) + goto out; + pgd_populate(&init_mm, pgd, p4d); + } + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); + if (!pud) + goto out; + p4d_populate(&init_mm, p4d, pud); + } + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); + if (!pmd) + goto out; + pud_populate(&init_mm, pud, pmd); + } else if (WARN_ON_ONCE(pud_large(*pud))) { + goto out; + } + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + pte = vmem_pte_alloc(); + if (!pte) + goto out; + pmd_populate(&init_mm, pmd, pte); + } else if (WARN_ON_ONCE(pmd_large(*pmd))) { + goto out; + } + ptep = pte_offset_kernel(pmd, addr); +out: + return ptep; +} + +int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot) +{ + pte_t *ptep, pte; + int rc = 0; + + if (!IS_ALIGNED(addr, PAGE_SIZE)) + return -EINVAL; + mutex_lock(&vmem_mutex); + ptep = vmem_get_alloc_pte(addr); + if (!ptep) { + rc = -ENOMEM; + goto out; + } + __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); + pte = mk_pte_phys(phys, prot); + set_pte(ptep, pte); +out: + mutex_unlock(&vmem_mutex); + return rc; +} + +void vmem_unmap_4k_page(unsigned long addr) +{ + pte_t *ptep; + + mutex_lock(&vmem_mutex); + ptep = virt_to_kpte(addr); + __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); + pte_clear(&init_mm, addr, ptep); + mutex_unlock(&vmem_mutex); +} + /* * map whole physical memory to virtual memory (identity mapping) * we reserve enough space in the vmalloc area for vmemmap to hotplug From e409b7f19172a3c154de62de4baf32a2c25a375a Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 20 Jul 2022 08:32:13 +0200 Subject: [PATCH 40/46] s390/smp,ptdump: add absolute lowcore markers Add "Lowcore Area Start" and "Lowcore Area End" markers that fence pages where absolute lowcore resides. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/mm/dump_pagetables.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 9f9af5298dd6..393d6ef4589e 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,8 @@ enum address_markers_idx { VMALLOC_END_NR, MODULES_NR, MODULES_END_NR, + ABS_LOWCORE_NR, + ABS_LOWCORE_END_NR, }; static struct addr_marker address_markers[] = { @@ -62,6 +65,8 @@ static struct addr_marker address_markers[] = { [VMALLOC_END_NR] = {0, "vmalloc Area End"}, [MODULES_NR] = {0, "Modules Area Start"}, [MODULES_END_NR] = {0, "Modules Area End"}, + [ABS_LOWCORE_NR] = {0, "Lowcore Area Start"}, + [ABS_LOWCORE_END_NR] = {0, "Lowcore Area End"}, { -1, NULL } }; @@ -278,6 +283,8 @@ static int pt_dump_init(void) address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; address_markers[MODULES_NR].start_address = MODULES_VADDR; address_markers[MODULES_END_NR].start_address = MODULES_END; + address_markers[ABS_LOWCORE_NR].start_address = __abs_lowcore; + address_markers[ABS_LOWCORE_END_NR].start_address = __abs_lowcore + ABS_LOWCORE_MAP_SIZE; address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; address_markers[VMEMMAP_END_NR].start_address = (unsigned long)vmemmap + vmemmap_size; address_markers[VMALLOC_NR].start_address = VMALLOC_START; From a0c0c44e9aa2b5da876467083c359b368f3ce95e Mon Sep 17 00:00:00 2001 From: Janosch Frank Date: Wed, 20 Jul 2022 12:43:28 +0000 Subject: [PATCH 41/46] s390: add ELF note type for encrypted CPU state of a PV VCPU The type NT_S390_PV_CPU_DATA note contains the encrypted CPU state of a PV VCPU. It's only relevant in dumps of s390 PV VMs and can't be decrypted without a second block of encrypted data which provides key parts. Therefore we only reserve the note type here. The zgetdump tool from the s390-tools package can, together with a Customer Communication Key, be used to convert a PV VM dump into a normal VM dump. zgetdump will decrypt the CPU data and overwrite the other respective notes to make the data accessible for crash and other debugging tools. Signed-off-by: Janosch Frank Acked-by: Heiko Carstens [agordeev@linux.ibm.com changed desctiption] Signed-off-by: Alexander Gordeev --- include/uapi/linux/elf.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/elf.h b/include/uapi/linux/elf.h index 2b9f5e9985e5..c7b056af9ef0 100644 --- a/include/uapi/linux/elf.h +++ b/include/uapi/linux/elf.h @@ -420,6 +420,7 @@ typedef struct elf64_shdr { #define NT_S390_GS_CB 0x30b /* s390 guarded storage registers */ #define NT_S390_GS_BC 0x30c /* s390 guarded storage broadcast control block */ #define NT_S390_RI_CB 0x30d /* s390 runtime instrumentation */ +#define NT_S390_PV_CPU_DATA 0x30e /* s390 protvirt cpu dump data */ #define NT_ARM_VFP 0x400 /* ARM VFP/NEON registers */ #define NT_ARM_TLS 0x401 /* ARM TLS register */ #define NT_ARM_HW_BREAK 0x402 /* ARM hardware breakpoint registers */ From 520763a3272abf0ebf161c748bdbc8dad275521c Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Tue, 26 Jul 2022 09:43:53 +0200 Subject: [PATCH 42/46] s390/nmi: use irqentry_nmi_enter()/irqentry_nmi_exit() With generic entry in place switch the nmi handler to use the generic entry helper functions. Signed-off-by: Sven Schnelle Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/nmi.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 53ed3884fe64..60ac66aab163 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -397,11 +398,12 @@ int notrace s390_do_machine_check(struct pt_regs *regs) static unsigned long long last_ipd; struct mcck_struct *mcck; unsigned long long tmp; + irqentry_state_t irq_state; union mci mci; unsigned long mcck_dam_code; int mcck_pending = 0; - nmi_enter(); + irq_state = irqentry_nmi_enter(regs); if (user_mode(regs)) update_timer_mcck(); @@ -504,14 +506,14 @@ int notrace s390_do_machine_check(struct pt_regs *regs) clear_cpu_flag(CIF_MCCK_GUEST); if (user_mode(regs) && mcck_pending) { - nmi_exit(); + irqentry_nmi_exit(regs, irq_state); return 1; } if (mcck_pending) schedule_mcck_handler(); - nmi_exit(); + irqentry_nmi_exit(regs, irq_state); return 0; } NOKPROBE_SYMBOL(s390_do_machine_check); From ded466e1806686794b403ebf031133bbaca76bb2 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Tue, 26 Jul 2022 18:57:59 +0200 Subject: [PATCH 43/46] s390/unwind: fix fgraph return address recovery When HAVE_FUNCTION_GRAPH_RET_ADDR_PTR is defined, the return address to the fgraph caller is recovered by tagging it along with the stack pointer of ftrace stack. This makes the stack unwinding more reliable. When the fgraph return address is modified to return_to_handler, ftrace_graph_ret_addr tries to restore it to the original value using tagged stack pointer. Fix this by passing tagged sp to ftrace_graph_ret_addr. Fixes: d81675b60d09 ("s390/unwind: recover kretprobe modified return address in stacktrace") Cc: # 5.18 Reviewed-by: Vasily Gorbik Signed-off-by: Sumanth Korikkar Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/unwind.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/include/asm/unwind.h b/arch/s390/include/asm/unwind.h index 0bf06f1682d8..02462e7100c1 100644 --- a/arch/s390/include/asm/unwind.h +++ b/arch/s390/include/asm/unwind.h @@ -47,7 +47,7 @@ struct unwind_state { static inline unsigned long unwind_recover_ret_addr(struct unwind_state *state, unsigned long ip) { - ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, NULL); + ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, (void *)state->sp); if (is_kretprobe_trampoline(ip)) ip = kretprobe_find_ret_addr(state->task, (void *)state->sp, &state->kr_cur); return ip; From 3fb39cb7c5145a10e1a0221c057e92fb8855efbb Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Sat, 6 Aug 2022 09:13:28 +0200 Subject: [PATCH 44/46] Revert "s390/smp,ptdump: add absolute lowcore markers" This reverts commit e409b7f19172a3c154de62de4baf32a2c25a375a. Commit 7d06fed77b7d ("s390/smp: rework absolute lowcore access") introduced mutex lock with interrupts disabled. This commit is a follow-up that needs to be reverted as well. Signed-off-by: Alexander Gordeev --- arch/s390/mm/dump_pagetables.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 393d6ef4589e..9f9af5298dd6 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -40,8 +39,6 @@ enum address_markers_idx { VMALLOC_END_NR, MODULES_NR, MODULES_END_NR, - ABS_LOWCORE_NR, - ABS_LOWCORE_END_NR, }; static struct addr_marker address_markers[] = { @@ -65,8 +62,6 @@ static struct addr_marker address_markers[] = { [VMALLOC_END_NR] = {0, "vmalloc Area End"}, [MODULES_NR] = {0, "Modules Area Start"}, [MODULES_END_NR] = {0, "Modules Area End"}, - [ABS_LOWCORE_NR] = {0, "Lowcore Area Start"}, - [ABS_LOWCORE_END_NR] = {0, "Lowcore Area End"}, { -1, NULL } }; @@ -283,8 +278,6 @@ static int pt_dump_init(void) address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; address_markers[MODULES_NR].start_address = MODULES_VADDR; address_markers[MODULES_END_NR].start_address = MODULES_END; - address_markers[ABS_LOWCORE_NR].start_address = __abs_lowcore; - address_markers[ABS_LOWCORE_END_NR].start_address = __abs_lowcore + ABS_LOWCORE_MAP_SIZE; address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; address_markers[VMEMMAP_END_NR].start_address = (unsigned long)vmemmap + vmemmap_size; address_markers[VMALLOC_NR].start_address = VMALLOC_START; From 5e441f61f509617a3f57fcb156b7aa2870cc8752 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Sat, 6 Aug 2022 09:24:07 +0200 Subject: [PATCH 45/46] Revert "s390/smp: rework absolute lowcore access" This reverts commit 7d06fed77b7d8fc9f6cc41b4e3f2823d32532ad8. This introduced vmem_mutex locking from vmem_map_4k_page() function called from smp_reinit_ipl_cpu() with interrupts disabled. While it is a pre-SMP early initcall no other CPUs running in parallel nor other code taking vmem_mutex on this boot stage - it still needs to be fixed. Signed-off-by: Alexander Gordeev --- arch/s390/boot/startup.c | 5 +- arch/s390/include/asm/abs_lowcore.h | 17 ------ arch/s390/include/asm/pgtable.h | 2 - arch/s390/include/asm/processor.h | 15 +++++ arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/abs_lowcore.c | 88 ----------------------------- arch/s390/kernel/ipl.c | 9 +-- arch/s390/kernel/machine_kexec.c | 8 +-- arch/s390/kernel/os_info.c | 9 +-- arch/s390/kernel/setup.c | 34 +++++------ arch/s390/kernel/smp.c | 34 +++-------- arch/s390/mm/init.c | 2 +- arch/s390/mm/maccess.c | 67 ++++++++++++---------- arch/s390/mm/vmem.c | 85 ---------------------------- 14 files changed, 83 insertions(+), 294 deletions(-) delete mode 100644 arch/s390/include/asm/abs_lowcore.h delete mode 100644 arch/s390/kernel/abs_lowcore.c diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 41b7af7a9365..bc48fe82d949 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -10,13 +10,11 @@ #include #include #include -#include #include "decompressor.h" #include "boot.h" #include "uv.h" unsigned long __bootdata_preserved(__kaslr_offset); -unsigned long __bootdata_preserved(__abs_lowcore); unsigned long __bootdata(__amode31_base); unsigned long __bootdata_preserved(VMALLOC_START); unsigned long __bootdata_preserved(VMALLOC_END); @@ -182,8 +180,7 @@ static void setup_kernel_memory_layout(void) /* force vmalloc and modules below kasan shadow */ vmax = min(vmax, KASAN_SHADOW_START); #endif - __abs_lowcore = round_down(vmax - ABS_LOWCORE_MAP_SIZE, sizeof(struct lowcore)); - MODULES_END = round_down(__abs_lowcore, _SEGMENT_SIZE); + MODULES_END = vmax; MODULES_VADDR = MODULES_END - MODULES_LEN; VMALLOC_END = MODULES_VADDR; diff --git a/arch/s390/include/asm/abs_lowcore.h b/arch/s390/include/asm/abs_lowcore.h deleted file mode 100644 index bdef8d24d237..000000000000 --- a/arch/s390/include/asm/abs_lowcore.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_S390_ABS_LOWCORE_H -#define _ASM_S390_ABS_LOWCORE_H - -#include - -#define ABS_LOWCORE_MAP_SIZE (NR_CPUS * sizeof(struct lowcore)) - -extern unsigned long __abs_lowcore; -extern bool abs_lowcore_mapped; - -struct lowcore *get_abs_lowcore(unsigned long *flags); -void put_abs_lowcore(struct lowcore *lc, unsigned long flags); -int abs_lowcore_map(int cpu, struct lowcore *lc); -void abs_lowcore_unmap(int cpu); - -#endif /* _ASM_ABS_S390_LOWCORE_H */ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 82506ebd544b..a397b072a580 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1781,8 +1781,6 @@ static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset) extern int vmem_add_mapping(unsigned long start, unsigned long size); extern void vmem_remove_mapping(unsigned long start, unsigned long size); -extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot); -extern void vmem_unmap_4k_page(unsigned long addr); extern int s390_enable_sie(void); extern int s390_enable_skey(void); extern void s390_reset_cmma(struct mm_struct *mm); diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 93677ae89e7e..bd66f8e34949 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -307,6 +307,21 @@ static __always_inline void __noreturn disabled_wait(void) #define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL extern int memcpy_real(void *, unsigned long, size_t); +extern void memcpy_absolute(void *, void *, size_t); + +#define put_abs_lowcore(member, x) do { \ + unsigned long __abs_address = offsetof(struct lowcore, member); \ + __typeof__(((struct lowcore *)0)->member) __tmp = (x); \ + \ + memcpy_absolute(__va(__abs_address), &__tmp, sizeof(__tmp)); \ +} while (0) + +#define get_abs_lowcore(x, member) do { \ + unsigned long __abs_address = offsetof(struct lowcore, member); \ + __typeof__(((struct lowcore *)0)->member) *__ptr = &(x); \ + \ + memcpy_absolute(__ptr, __va(__abs_address), sizeof(*__ptr)); \ +} while (0) extern int s390_isolate_bp(void); extern int s390_isolate_bp_guest(void); diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 45e4b2f41e05..3cbfa9fddd9a 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -40,7 +40,7 @@ obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o -obj-y += smp.o text_amode31.o stacktrace.o abs_lowcore.o +obj-y += smp.o text_amode31.o stacktrace.o extra-y += head64.o vmlinux.lds diff --git a/arch/s390/kernel/abs_lowcore.c b/arch/s390/kernel/abs_lowcore.c deleted file mode 100644 index dc9f0ecd4695..000000000000 --- a/arch/s390/kernel/abs_lowcore.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include - -#define ABS_LOWCORE_UNMAPPED 1 -#define ABS_LOWCORE_LAP_ON 2 -#define ABS_LOWCORE_IRQS_ON 4 - -unsigned long __bootdata_preserved(__abs_lowcore); -bool __ro_after_init abs_lowcore_mapped; - -int abs_lowcore_map(int cpu, struct lowcore *lc) -{ - unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore)); - unsigned long phys = __pa(lc); - int rc, i; - - for (i = 0; i < LC_PAGES; i++) { - rc = vmem_map_4k_page(addr, phys, PAGE_KERNEL); - if (rc) { - for (--i; i >= 0; i--) { - addr -= PAGE_SIZE; - vmem_unmap_4k_page(addr); - } - return rc; - } - addr += PAGE_SIZE; - phys += PAGE_SIZE; - } - return 0; -} - -void abs_lowcore_unmap(int cpu) -{ - unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore)); - int i; - - for (i = 0; i < LC_PAGES; i++) { - vmem_unmap_4k_page(addr); - addr += PAGE_SIZE; - } -} - -struct lowcore *get_abs_lowcore(unsigned long *flags) -{ - unsigned long irq_flags; - union ctlreg0 cr0; - int cpu; - - *flags = 0; - cpu = get_cpu(); - if (abs_lowcore_mapped) { - return ((struct lowcore *)__abs_lowcore) + cpu; - } else { - if (cpu != 0) - panic("Invalid unmapped absolute lowcore access\n"); - local_irq_save(irq_flags); - if (!irqs_disabled_flags(irq_flags)) - *flags |= ABS_LOWCORE_IRQS_ON; - __ctl_store(cr0.val, 0, 0); - if (cr0.lap) { - *flags |= ABS_LOWCORE_LAP_ON; - __ctl_clear_bit(0, 28); - } - *flags |= ABS_LOWCORE_UNMAPPED; - return lowcore_ptr[0]; - } -} - -void put_abs_lowcore(struct lowcore *lc, unsigned long flags) -{ - if (abs_lowcore_mapped) { - if (flags) - panic("Invalid mapped absolute lowcore release\n"); - } else { - if (smp_processor_id() != 0) - panic("Invalid mapped absolute lowcore access\n"); - if (!(flags & ABS_LOWCORE_UNMAPPED)) - panic("Invalid unmapped absolute lowcore release\n"); - if (flags & ABS_LOWCORE_LAP_ON) - __ctl_set_bit(0, 28); - if (flags & ABS_LOWCORE_IRQS_ON) - local_irq_enable(); - } - put_cpu(); -} diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 325cbf69ebbd..1cc85b8ff42e 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -1643,16 +1642,12 @@ static struct shutdown_action __refdata dump_action = { static void dump_reipl_run(struct shutdown_trigger *trigger) { unsigned long ipib = (unsigned long) reipl_block_actual; - struct lowcore *abs_lc; - unsigned long flags; unsigned int csum; csum = (__force unsigned int) csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); - abs_lc = get_abs_lowcore(&flags); - abs_lc->ipib = ipib; - abs_lc->ipib_checksum = csum; - put_abs_lowcore(abs_lc, flags); + put_abs_lowcore(ipib, ipib); + put_abs_lowcore(ipib_checksum, csum); dump_run(trigger); } diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 4579b42286d5..ab761c008f98 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -223,18 +222,13 @@ void machine_kexec_cleanup(struct kimage *image) void arch_crash_save_vmcoreinfo(void) { - struct lowcore *abs_lc; - unsigned long flags; - VMCOREINFO_SYMBOL(lowcore_ptr); VMCOREINFO_SYMBOL(high_memory); VMCOREINFO_LENGTH(lowcore_ptr, NR_CPUS); vmcoreinfo_append_str("SAMODE31=%lx\n", __samode31); vmcoreinfo_append_str("EAMODE31=%lx\n", __eamode31); vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); - abs_lc = get_abs_lowcore(&flags); - abs_lc->vmcore_info = paddr_vmcoreinfo_note(); - put_abs_lowcore(abs_lc, flags); + put_abs_lowcore(vmcore_info, paddr_vmcoreinfo_note()); } void machine_shutdown(void) diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 506ccb74d2d0..1acc2e05d70f 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include @@ -57,16 +57,13 @@ void os_info_entry_add(int nr, void *ptr, u64 size) */ void __init os_info_init(void) { - struct lowcore *abs_lc; - unsigned long flags; + void *ptr = &os_info; os_info.version_major = OS_INFO_VERSION_MAJOR; os_info.version_minor = OS_INFO_VERSION_MINOR; os_info.magic = OS_INFO_MAGIC; os_info.csum = os_info_csum(&os_info); - abs_lc = get_abs_lowcore(&flags); - abs_lc->os_info = __pa(&os_info); - put_abs_lowcore(abs_lc, flags); + put_abs_lowcore(os_info, __pa(ptr)); } #ifdef CONFIG_CRASH_DUMP diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 91139a16a44f..8f483132901e 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -58,7 +58,7 @@ #include #include #include -#include +#include #include #include #include @@ -411,9 +411,8 @@ void __init arch_call_rest_init(void) static void __init setup_lowcore_dat_off(void) { unsigned long int_psw_mask = PSW_KERNEL_BITS; - struct lowcore *abs_lc, *lc; unsigned long mcck_stack; - unsigned long flags; + struct lowcore *lc; if (IS_ENABLED(CONFIG_KASAN)) int_psw_mask |= PSW_MASK_DAT; @@ -475,13 +474,11 @@ static void __init setup_lowcore_dat_off(void) lc->restart_data = 0; lc->restart_source = -1U; - abs_lc = get_abs_lowcore(&flags); - abs_lc->restart_stack = lc->restart_stack; - abs_lc->restart_fn = lc->restart_fn; - abs_lc->restart_data = lc->restart_data; - abs_lc->restart_source = lc->restart_source; - abs_lc->restart_psw = lc->restart_psw; - put_abs_lowcore(abs_lc, flags); + put_abs_lowcore(restart_stack, lc->restart_stack); + put_abs_lowcore(restart_fn, lc->restart_fn); + put_abs_lowcore(restart_data, lc->restart_data); + put_abs_lowcore(restart_source, lc->restart_source); + put_abs_lowcore(restart_psw, lc->restart_psw); mcck_stack = (unsigned long)memblock_alloc(THREAD_SIZE, THREAD_SIZE); if (!mcck_stack) @@ -502,8 +499,8 @@ static void __init setup_lowcore_dat_off(void) static void __init setup_lowcore_dat_on(void) { - struct lowcore *abs_lc; - unsigned long flags; + struct lowcore *lc = lowcore_ptr[0]; + int cr; __ctl_clear_bit(0, 28); S390_lowcore.external_new_psw.mask |= PSW_MASK_DAT; @@ -512,15 +509,10 @@ static void __init setup_lowcore_dat_on(void) S390_lowcore.io_new_psw.mask |= PSW_MASK_DAT; __ctl_set_bit(0, 28); __ctl_store(S390_lowcore.cregs_save_area, 0, 15); - abs_lc = get_abs_lowcore(&flags); - abs_lc->restart_flags = RESTART_FLAG_CTLREGS; - abs_lc->program_new_psw = S390_lowcore.program_new_psw; - memcpy(abs_lc->cregs_save_area, S390_lowcore.cregs_save_area, - sizeof(abs_lc->cregs_save_area)); - put_abs_lowcore(abs_lc, flags); - if (abs_lowcore_map(0, lowcore_ptr[0])) - panic("Couldn't setup absolute lowcore"); - abs_lowcore_mapped = true; + put_abs_lowcore(restart_flags, RESTART_FLAG_CTLREGS); + put_abs_lowcore(program_new_psw, lc->program_new_psw); + for (cr = 0; cr < ARRAY_SIZE(lc->cregs_save_area); cr++) + put_abs_lowcore(cregs_save_area[cr], lc->cregs_save_area[cr]); } static struct resource code_resource = { diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 40876d809ea6..30c91d565933 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -45,7 +45,7 @@ #include #include #include -#include +#include #include #include #include @@ -212,14 +212,10 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) lc->preempt_count = PREEMPT_DISABLED; if (nmi_alloc_mcesa(&lc->mcesad)) goto out; - if (abs_lowcore_map(cpu, lc)) - goto out_mcesa; lowcore_ptr[cpu] = lc; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc)); return 0; -out_mcesa: - nmi_free_mcesa(&lc->mcesad); out: stack_free(mcck_stack); stack_free(async_stack); @@ -241,7 +237,6 @@ static void pcpu_free_lowcore(struct pcpu *pcpu) mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0); lowcore_ptr[cpu] = NULL; - abs_lowcore_unmap(cpu); nmi_free_mcesa(&lc->mcesad); stack_free(async_stack); stack_free(mcck_stack); @@ -320,12 +315,9 @@ static void pcpu_delegate(struct pcpu *pcpu, pcpu_delegate_fn *func, void *data, unsigned long stack) { - struct lowcore *lc, *abs_lc; - unsigned int source_cpu; - unsigned long flags; + struct lowcore *lc = lowcore_ptr[pcpu - pcpu_devices]; + unsigned int source_cpu = stap(); - lc = lowcore_ptr[pcpu - pcpu_devices]; - source_cpu = stap(); __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT); if (pcpu->address == source_cpu) { call_on_stack(2, stack, void, __pcpu_delegate, @@ -340,12 +332,10 @@ static void pcpu_delegate(struct pcpu *pcpu, lc->restart_data = (unsigned long)data; lc->restart_source = source_cpu; } else { - abs_lc = get_abs_lowcore(&flags); - abs_lc->restart_stack = stack; - abs_lc->restart_fn = (unsigned long)func; - abs_lc->restart_data = (unsigned long)data; - abs_lc->restart_source = source_cpu; - put_abs_lowcore(abs_lc, flags); + put_abs_lowcore(restart_stack, stack); + put_abs_lowcore(restart_fn, (unsigned long)func); + put_abs_lowcore(restart_data, (unsigned long)data); + put_abs_lowcore(restart_source, source_cpu); } __bpon(); asm volatile( @@ -591,8 +581,6 @@ static DEFINE_SPINLOCK(ctl_lock); void smp_ctl_set_clear_bit(int cr, int bit, bool set) { struct ec_creg_mask_parms parms = { .cr = cr, }; - struct lowcore *abs_lc; - unsigned long flags; u64 ctlreg; if (set) { @@ -603,11 +591,9 @@ void smp_ctl_set_clear_bit(int cr, int bit, bool set) parms.andval = ~(1UL << bit); } spin_lock(&ctl_lock); - abs_lc = get_abs_lowcore(&flags); - ctlreg = abs_lc->cregs_save_area[cr]; + get_abs_lowcore(ctlreg, cregs_save_area[cr]); ctlreg = (ctlreg & parms.andval) | parms.orval; - abs_lc->cregs_save_area[cr] = ctlreg; - put_abs_lowcore(abs_lc, flags); + put_abs_lowcore(cregs_save_area[cr], ctlreg); spin_unlock(&ctl_lock); on_each_cpu(smp_ctl_bit_callback, &parms, 1); } @@ -1295,8 +1281,6 @@ static int __init smp_reinit_ipl_cpu(void) __ctl_clear_bit(0, 28); /* disable lowcore protection */ S390_lowcore.mcesad = mcesad; __ctl_load(cr0, 0, 0); - if (abs_lowcore_map(0, lc)) - panic("Couldn't remap absolute lowcore"); lowcore_ptr[0] = lc; local_mcck_enable(); local_irq_restore(flags); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 7b6873ac99d1..6a0ac00d5a42 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index b8451ddbb3d6..d6d84e02f35a 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -15,7 +15,6 @@ #include #include #include -#include #include static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) @@ -149,20 +148,46 @@ int memcpy_real(void *dest, unsigned long src, size_t count) } /* - * Find CPU that owns swapped prefix page + * Copy memory in absolute mode (kernel to kernel) */ -static int get_swapped_owner(phys_addr_t addr) +void memcpy_absolute(void *dest, void *src, size_t count) +{ + unsigned long cr0, flags, prefix; + + flags = arch_local_irq_save(); + __ctl_store(cr0, 0, 0); + __ctl_clear_bit(0, 28); /* disable lowcore protection */ + prefix = store_prefix(); + if (prefix) { + local_mcck_disable(); + set_prefix(0); + memcpy(dest, src, count); + set_prefix(prefix); + local_mcck_enable(); + } else { + memcpy(dest, src, count); + } + __ctl_load(cr0, 0, 0); + arch_local_irq_restore(flags); +} + +/* + * Check if physical address is within prefix or zero page + */ +static int is_swapped(phys_addr_t addr) { phys_addr_t lc; int cpu; + if (addr < sizeof(struct lowcore)) + return 1; for_each_online_cpu(cpu) { lc = virt_to_phys(lowcore_ptr[cpu]); if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) continue; - return cpu; + return 1; } - return -1; + return 0; } /* @@ -175,35 +200,17 @@ void *xlate_dev_mem_ptr(phys_addr_t addr) { void *ptr = phys_to_virt(addr); void *bounce = ptr; - struct lowcore *abs_lc; - unsigned long flags; unsigned long size; - int this_cpu, cpu; cpus_read_lock(); - this_cpu = get_cpu(); - if (addr >= sizeof(struct lowcore)) { - cpu = get_swapped_owner(addr); - if (cpu < 0) - goto out; + preempt_disable(); + if (is_swapped(addr)) { + size = PAGE_SIZE - (addr & ~PAGE_MASK); + bounce = (void *) __get_free_page(GFP_ATOMIC); + if (bounce) + memcpy_absolute(bounce, ptr, size); } - bounce = (void *)__get_free_page(GFP_ATOMIC); - if (!bounce) - goto out; - size = PAGE_SIZE - (addr & ~PAGE_MASK); - if (addr < sizeof(struct lowcore)) { - abs_lc = get_abs_lowcore(&flags); - ptr = (void *)abs_lc + addr; - memcpy(bounce, ptr, size); - put_abs_lowcore(abs_lc, flags); - } else if (cpu == this_cpu) { - ptr = (void *)(addr - virt_to_phys(lowcore_ptr[cpu])); - memcpy(bounce, ptr, size); - } else { - memcpy(bounce, ptr, size); - } -out: - put_cpu(); + preempt_enable(); cpus_read_unlock(); return bounce; } diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 203ba2bfea59..c2583f921ca8 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -560,91 +560,6 @@ int vmem_add_mapping(unsigned long start, unsigned long size) return ret; } -/* - * Allocate new or return existing page-table entry, but do not map it - * to any physical address. If missing, allocate segment- and region- - * table entries along. Meeting a large segment- or region-table entry - * while traversing is an error, since the function is expected to be - * called against virtual regions reserverd for 4KB mappings only. - */ -static pte_t *vmem_get_alloc_pte(unsigned long addr) -{ - pte_t *ptep = NULL; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - - pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { - p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); - if (!p4d) - goto out; - pgd_populate(&init_mm, pgd, p4d); - } - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); - if (!pud) - goto out; - p4d_populate(&init_mm, p4d, pud); - } - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); - if (!pmd) - goto out; - pud_populate(&init_mm, pud, pmd); - } else if (WARN_ON_ONCE(pud_large(*pud))) { - goto out; - } - pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) { - pte = vmem_pte_alloc(); - if (!pte) - goto out; - pmd_populate(&init_mm, pmd, pte); - } else if (WARN_ON_ONCE(pmd_large(*pmd))) { - goto out; - } - ptep = pte_offset_kernel(pmd, addr); -out: - return ptep; -} - -int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot) -{ - pte_t *ptep, pte; - int rc = 0; - - if (!IS_ALIGNED(addr, PAGE_SIZE)) - return -EINVAL; - mutex_lock(&vmem_mutex); - ptep = vmem_get_alloc_pte(addr); - if (!ptep) { - rc = -ENOMEM; - goto out; - } - __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); - pte = mk_pte_phys(phys, prot); - set_pte(ptep, pte); -out: - mutex_unlock(&vmem_mutex); - return rc; -} - -void vmem_unmap_4k_page(unsigned long addr) -{ - pte_t *ptep; - - mutex_lock(&vmem_mutex); - ptep = virt_to_kpte(addr); - __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); - pte_clear(&init_mm, addr, ptep); - mutex_unlock(&vmem_mutex); -} - /* * map whole physical memory to virtual memory (identity mapping) * we reserve enough space in the vmalloc area for vmemmap to hotplug From 953503751a426413ea8aee2299ae3ee971b70d9b Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Sat, 6 Aug 2022 09:29:46 +0200 Subject: [PATCH 46/46] Revert "s390/smp: enforce lowcore protection on CPU restart" This reverts commit 6f5c672d17f583b081e283927f5040f726c54598. This breaks normal crash dump when CPU0 is offline. Signed-off-by: Alexander Gordeev --- arch/s390/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 8f483132901e..c5d295c5e728 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -507,8 +507,8 @@ static void __init setup_lowcore_dat_on(void) S390_lowcore.svc_new_psw.mask |= PSW_MASK_DAT; S390_lowcore.program_new_psw.mask |= PSW_MASK_DAT; S390_lowcore.io_new_psw.mask |= PSW_MASK_DAT; - __ctl_set_bit(0, 28); __ctl_store(S390_lowcore.cregs_save_area, 0, 15); + __ctl_set_bit(0, 28); put_abs_lowcore(restart_flags, RESTART_FLAG_CTLREGS); put_abs_lowcore(program_new_psw, lc->program_new_psw); for (cr = 0; cr < ARRAY_SIZE(lc->cregs_save_area); cr++)