diff options
51 files changed, 2153 insertions, 1258 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index 36055f14c5..4c05ff8bba 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2532,6 +2532,12 @@ M: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> S: Maintained F: scripts/simplebench/ +Transactions helper +M: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> +S: Maintained +F: include/qemu/transactions.h +F: util/transactions.c + QAPI M: Markus Armbruster <armbru@redhat.com> M: Michael Roth <michael.roth@amd.com> diff --git a/block.c b/block.c index c5b887cec1..874c22c43e 100644 --- a/block.c +++ b/block.c @@ -2,6 +2,7 @@ * QEMU System Emulator block driver * * Copyright (c) 2003 Fabrice Bellard + * Copyright (c) 2020 Virtuozzo International GmbH. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -82,6 +83,25 @@ static BlockDriverState *bdrv_open_inherit(const char *filename, BdrvChildRole child_role, Error **errp); +static void bdrv_replace_child_noperm(BdrvChild *child, + BlockDriverState *new_bs); +static int bdrv_attach_child_noperm(BlockDriverState *parent_bs, + BlockDriverState *child_bs, + const char *child_name, + const BdrvChildClass *child_class, + BdrvChildRole child_role, + BdrvChild **child, + Transaction *tran, + Error **errp); +static void bdrv_remove_filter_or_cow_child(BlockDriverState *bs, + Transaction *tran); + +static int bdrv_reopen_prepare(BDRVReopenState *reopen_state, + BlockReopenQueue *queue, + Transaction *set_backings_tran, Error **errp); +static void bdrv_reopen_commit(BDRVReopenState *reopen_state); +static void bdrv_reopen_abort(BDRVReopenState *reopen_state); + /* If non-zero, use only whitelisted block drivers */ static int use_bdrv_whitelist; @@ -1394,6 +1414,13 @@ static int bdrv_child_cb_update_filename(BdrvChild *c, BlockDriverState *base, return 0; } +static AioContext *bdrv_child_cb_get_parent_aio_context(BdrvChild *c) +{ + BlockDriverState *bs = c->opaque; + + return bdrv_get_aio_context(bs); +} + const BdrvChildClass child_of_bds = { .parent_is_bds = true, .get_parent_desc = bdrv_child_get_parent_desc, @@ -1407,8 +1434,14 @@ const BdrvChildClass child_of_bds = { .can_set_aio_ctx = bdrv_child_cb_can_set_aio_ctx, .set_aio_ctx = bdrv_child_cb_set_aio_ctx, .update_filename = bdrv_child_cb_update_filename, + .get_parent_aio_context = bdrv_child_cb_get_parent_aio_context, }; +AioContext *bdrv_child_get_parent_aio_context(BdrvChild *c) +{ + return c->klass->get_parent_aio_context(c); +} + static int bdrv_open_flags(BlockDriverState *bs, int flags) { int open_flags = flags; @@ -1547,7 +1580,7 @@ static int bdrv_open_driver(BlockDriverState *bs, BlockDriver *drv, return ret; } - bdrv_refresh_limits(bs, &local_err); + bdrv_refresh_limits(bs, NULL, &local_err); if (local_err) { error_propagate(errp, local_err); return -EINVAL; @@ -1955,12 +1988,6 @@ static int bdrv_fill_options(QDict **options, const char *filename, return 0; } -static int bdrv_child_check_perm(BdrvChild *c, BlockReopenQueue *q, - uint64_t perm, uint64_t shared, - GSList *ignore_children, Error **errp); -static void bdrv_child_abort_perm_update(BdrvChild *c); -static void bdrv_child_set_perm(BdrvChild *c); - typedef struct BlockReopenQueueEntry { bool prepared; bool perms_checked; @@ -2008,6 +2035,57 @@ bool bdrv_is_writable(BlockDriverState *bs) return bdrv_is_writable_after_reopen(bs, NULL); } +static char *bdrv_child_user_desc(BdrvChild *c) +{ + if (c->klass->get_parent_desc) { + return c->klass->get_parent_desc(c); + } + + return g_strdup("another user"); +} + +static bool bdrv_a_allow_b(BdrvChild *a, BdrvChild *b, Error **errp) +{ + g_autofree char *user = NULL; + g_autofree char *perm_names = NULL; + + if ((b->perm & a->shared_perm) == b->perm) { + return true; + } + + perm_names = bdrv_perm_names(b->perm & ~a->shared_perm); + user = bdrv_child_user_desc(a); + error_setg(errp, "Conflicts with use by %s as '%s', which does not " + "allow '%s' on %s", + user, a->name, perm_names, bdrv_get_node_name(b->bs)); + + return false; +} + +static bool bdrv_parent_perms_conflict(BlockDriverState *bs, Error **errp) +{ + BdrvChild *a, *b; + + /* + * During the loop we'll look at each pair twice. That's correct because + * bdrv_a_allow_b() is asymmetric and we should check each pair in both + * directions. + */ + QLIST_FOREACH(a, &bs->parents, next_parent) { + QLIST_FOREACH(b, &bs->parents, next_parent) { + if (a == b) { + continue; + } + + if (!bdrv_a_allow_b(a, b, errp)) { + return true; + } + } + } + + return false; +} + static void bdrv_child_perm(BlockDriverState *bs, BlockDriverState *child_bs, BdrvChild *c, BdrvChildRole role, BlockReopenQueue *reopen_queue, @@ -2025,22 +2103,186 @@ static void bdrv_child_perm(BlockDriverState *bs, BlockDriverState *child_bs, } /* - * Check whether permissions on this node can be changed in a way that - * @cumulative_perms and @cumulative_shared_perms are the new cumulative - * permissions of all its parents. This involves checking whether all necessary - * permission changes to child nodes can be performed. + * Adds the whole subtree of @bs (including @bs itself) to the @list (except for + * nodes that are already in the @list, of course) so that final list is + * topologically sorted. Return the result (GSList @list object is updated, so + * don't use old reference after function call). + * + * On function start @list must be already topologically sorted and for any node + * in the @list the whole subtree of the node must be in the @list as well. The + * simplest way to satisfy this criteria: use only result of + * bdrv_topological_dfs() or NULL as @list parameter. + */ +static GSList *bdrv_topological_dfs(GSList *list, GHashTable *found, + BlockDriverState *bs) +{ + BdrvChild *child; + g_autoptr(GHashTable) local_found = NULL; + + if (!found) { + assert(!list); + found = local_found = g_hash_table_new(NULL, NULL); + } + + if (g_hash_table_contains(found, bs)) { + return list; + } + g_hash_table_add(found, bs); + + QLIST_FOREACH(child, &bs->children, next) { + list = bdrv_topological_dfs(list, found, child->bs); + } + + return g_slist_prepend(list, bs); +} + +typedef struct BdrvChildSetPermState { + BdrvChild *child; + uint64_t old_perm; + uint64_t old_shared_perm; +} BdrvChildSetPermState; + +static void bdrv_child_set_perm_abort(void *opaque) +{ + BdrvChildSetPermState *s = opaque; + + s->child->perm = s->old_perm; + s->child->shared_perm = s->old_shared_perm; +} + +static TransactionActionDrv bdrv_child_set_pem_drv = { + .abort = bdrv_child_set_perm_abort, + .clean = g_free, +}; + +static void bdrv_child_set_perm(BdrvChild *c, uint64_t perm, + uint64_t shared, Transaction *tran) +{ + BdrvChildSetPermState *s = g_new(BdrvChildSetPermState, 1); + + *s = (BdrvChildSetPermState) { + .child = c, + .old_perm = c->perm, + .old_shared_perm = c->shared_perm, + }; + + c->perm = perm; + c->shared_perm = shared; + + tran_add(tran, &bdrv_child_set_pem_drv, s); +} + +static void bdrv_drv_set_perm_commit(void *opaque) +{ + BlockDriverState *bs = opaque; + uint64_t cumulative_perms, cumulative_shared_perms; + + if (bs->drv->bdrv_set_perm) { + bdrv_get_cumulative_perm(bs, &cumulative_perms, + &cumulative_shared_perms); + bs->drv->bdrv_set_perm(bs, cumulative_perms, cumulative_shared_perms); + } +} + +static void bdrv_drv_set_perm_abort(void *opaque) +{ + BlockDriverState *bs = opaque; + + if (bs->drv->bdrv_abort_perm_update) { + bs->drv->bdrv_abort_perm_update(bs); + } +} + +TransactionActionDrv bdrv_drv_set_perm_drv = { + .abort = bdrv_drv_set_perm_abort, + .commit = bdrv_drv_set_perm_commit, +}; + +static int bdrv_drv_set_perm(BlockDriverState *bs, uint64_t perm, + uint64_t shared_perm, Transaction *tran, + Error **errp) +{ + if (!bs->drv) { + return 0; + } + + if (bs->drv->bdrv_check_perm) { + int ret = bs->drv->bdrv_check_perm(bs, perm, shared_perm, errp); + if (ret < 0) { + return ret; + } + } + + if (tran) { + tran_add(tran, &bdrv_drv_set_perm_drv, bs); + } + + return 0; +} + +typedef struct BdrvReplaceChildState { + BdrvChild *child; + BlockDriverState *old_bs; +} BdrvReplaceChildState; + +static void bdrv_replace_child_commit(void *opaque) +{ + BdrvReplaceChildState *s = opaque; + + bdrv_unref(s->old_bs); +} + +static void bdrv_replace_child_abort(void *opaque) +{ + BdrvReplaceChildState *s = opaque; + BlockDriverState *new_bs = s->child->bs; + + /* old_bs reference is transparently moved from @s to @s->child */ + bdrv_replace_child_noperm(s->child, s->old_bs); + bdrv_unref(new_bs); +} + +static TransactionActionDrv bdrv_replace_child_drv = { + .commit = bdrv_replace_child_commit, + .abort = bdrv_replace_child_abort, + .clean = g_free, +}; + +/* + * bdrv_replace_child * - * A call to this function must always be followed by a call to bdrv_set_perm() - * or bdrv_abort_perm_update(). + * Note: real unref of old_bs is done only on commit. */ -static int bdrv_check_perm(BlockDriverState *bs, BlockReopenQueue *q, - uint64_t cumulative_perms, - uint64_t cumulative_shared_perms, - GSList *ignore_children, Error **errp) +static void bdrv_replace_child(BdrvChild *child, BlockDriverState *new_bs, + Transaction *tran) +{ + BdrvReplaceChildState *s = g_new(BdrvReplaceChildState, 1); + *s = (BdrvReplaceChildState) { + .child = child, + .old_bs = child->bs, + }; + tran_add(tran, &bdrv_replace_child_drv, s); + + if (new_bs) { + bdrv_ref(new_bs); + } + bdrv_replace_child_noperm(child, new_bs); + /* old_bs reference is transparently moved from @child to @s */ +} + +/* + * Refresh permissions in @bs subtree. The function is intended to be called + * after some graph modification that was done without permission update. + */ +static int bdrv_node_refresh_perm(BlockDriverState *bs, BlockReopenQueue *q, + Transaction *tran, Error **errp) { BlockDriver *drv = bs->drv; BdrvChild *c; int ret; + uint64_t cumulative_perms, cumulative_shared_perms; + + bdrv_get_cumulative_perm(bs, &cumulative_perms, &cumulative_shared_perms); /* Write permissions never work with read-only images */ if ((cumulative_perms & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) && @@ -2049,15 +2291,8 @@ static int bdrv_check_perm(BlockDriverState *bs, BlockReopenQueue *q, if (!bdrv_is_writable_after_reopen(bs, NULL)) { error_setg(errp, "Block node is read-only"); } else { - uint64_t current_perms, current_shared; - bdrv_get_cumulative_perm(bs, ¤t_perms, ¤t_shared); - if (current_perms & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { - error_setg(errp, "Cannot make block node read-only, there is " - "a writer on it"); - } else { - error_setg(errp, "Cannot make block node read-only and create " - "a writer on it"); - } + error_setg(errp, "Read-only block node '%s' cannot support " + "read-write users", bdrv_get_node_name(bs)); } return -EPERM; @@ -2084,12 +2319,10 @@ static int bdrv_check_perm(BlockDriverState *bs, BlockReopenQueue *q, return 0; } - if (drv->bdrv_check_perm) { - ret = drv->bdrv_check_perm(bs, cumulative_perms, - cumulative_shared_perms, errp); - if (ret < 0) { - return ret; - } + ret = bdrv_drv_set_perm(bs, cumulative_perms, cumulative_shared_perms, tran, + errp); + if (ret < 0) { + return ret; } /* Drivers that never have children can omit .bdrv_child_perm() */ @@ -2105,68 +2338,32 @@ static int bdrv_check_perm(BlockDriverState *bs, BlockReopenQueue *q, bdrv_child_perm(bs, c->bs, c, c->role, q, cumulative_perms, cumulative_shared_perms, &cur_perm, &cur_shared); - ret = bdrv_child_check_perm(c, q, cur_perm, cur_shared, ignore_children, - errp); - if (ret < 0) { - return ret; - } + bdrv_child_set_perm(c, cur_perm, cur_shared, tran); } return 0; } -/* - * Notifies drivers that after a previous bdrv_check_perm() call, the - * permission update is not performed and any preparations made for it (e.g. - * taken file locks) need to be undone. - * - * This function recursively notifies all child nodes. - */ -static void bdrv_abort_perm_update(BlockDriverState *bs) +static int bdrv_list_refresh_perms(GSList *list, BlockReopenQueue *q, + Transaction *tran, Error **errp) { - BlockDriver *drv = bs->drv; - BdrvChild *c; - - if (!drv) { - return; - } - - if (drv->bdrv_abort_perm_update) { - drv->bdrv_abort_perm_update(bs); - } - - QLIST_FOREACH(c, &bs->children, next) { - bdrv_child_abort_perm_update(c); - } -} - -static void bdrv_set_perm(BlockDriverState *bs) -{ - uint64_t cumulative_perms, cumulative_shared_perms; - BlockDriver *drv = bs->drv; - BdrvChild *c; - - if (!drv) { - return; - } + int ret; + BlockDriverState *bs; - bdrv_get_cumulative_perm(bs, &cumulative_perms, &cumulative_shared_perms); + for ( ; list; list = list->next) { + bs = list->data; - /* Update this node */ - if (drv->bdrv_set_perm) { - drv->bdrv_set_perm(bs, cumulative_perms, cumulative_shared_perms); - } + if (bdrv_parent_perms_conflict(bs, errp)) { + return -EINVAL; + } - /* Drivers that never have children can omit .bdrv_child_perm() */ - if (!drv->bdrv_child_perm) { - assert(QLIST_EMPTY(&bs->children)); - return; + ret = bdrv_node_refresh_perm(bs, q, tran, errp); + if (ret < 0) { + return ret; + } } - /* Update all children */ - QLIST_FOREACH(c, &bs->children, next) { - bdrv_child_set_perm(c); - } + return 0; } void bdrv_get_cumulative_perm(BlockDriverState *bs, uint64_t *perm, @@ -2185,15 +2382,6 @@ void bdrv_get_cumulative_perm(BlockDriverState *bs, uint64_t *perm, *shared_perm = cumulative_shared_perms; } -static char *bdrv_child_user_desc(BdrvChild *c) -{ - if (c->klass->get_parent_desc) { - return c->klass->get_parent_desc(c); - } - - return g_strdup("another user"); -} - char *bdrv_perm_names(uint64_t perm) { struct perm_name { @@ -2223,140 +2411,33 @@ char *bdrv_perm_names(uint64_t perm) return g_string_free(result, FALSE); } -/* - * Checks whether a new reference to @bs can be added if the new user requires - * @new_used_perm/@new_shared_perm as its permissions. If @ignore_children is - * set, the BdrvChild objects in this list are ignored in the calculations; - * this allows checking permission updates for an existing reference. - * - * Needs to be followed by a call to either bdrv_set_perm() or - * bdrv_abort_perm_update(). */ -static int bdrv_check_update_perm(BlockDriverState *bs, BlockReopenQueue *q, - uint64_t new_used_perm, - uint64_t new_shared_perm, - GSList *ignore_children, - Error **errp) -{ - BdrvChild *c; - uint64_t cumulative_perms = new_used_perm; - uint64_t cumulative_shared_perms = new_shared_perm; - - - /* There is no reason why anyone couldn't tolerate write_unchanged */ - assert(new_shared_perm & BLK_PERM_WRITE_UNCHANGED); - - QLIST_FOREACH(c, &bs->parents, next_parent) { - if (g_slist_find(ignore_children, c)) { - continue; - } - - if ((new_used_perm & c->shared_perm) != new_used_perm) { - char *user = bdrv_child_user_desc(c); - char *perm_names = bdrv_perm_names(new_used_perm & ~c->shared_perm); - - error_setg(errp, "Conflicts with use by %s as '%s', which does not " - "allow '%s' on %s", - user, c->name, perm_names, bdrv_get_node_name(c->bs)); - g_free(user); - g_free(perm_names); - return -EPERM; - } - - if ((c->perm & new_shared_perm) != c->perm) { - char *user = bdrv_child_user_desc(c); - char *perm_names = bdrv_perm_names(c->perm & ~new_shared_perm); - - error_setg(errp, "Conflicts with use by %s as '%s', which uses " - "'%s' on %s", - user, c->name, perm_names, bdrv_get_node_name(c->bs)); - g_free(user); - g_free(perm_names); - return -EPERM; - } - - cumulative_perms |= c->perm; - cumulative_shared_perms &= c->shared_perm; - } - - return bdrv_check_perm(bs, q, cumulative_perms, cumulative_shared_perms, - ignore_children, errp); -} - -/* Needs to be followed by a call to either bdrv_child_set_perm() or - * bdrv_child_abort_perm_update(). */ -static int bdrv_child_check_perm(BdrvChild *c, BlockReopenQueue *q, - uint64_t perm, uint64_t shared, - GSList *ignore_children, Error **errp) -{ - int ret; - - ignore_children = g_slist_prepend(g_slist_copy(ignore_children), c); - ret = bdrv_check_update_perm(c->bs, q, perm, shared, ignore_children, errp); - g_slist_free(ignore_children); - - if (ret < 0) { - return ret; - } - - if (!c->has_backup_perm) { - c->has_backup_perm = true; - c->backup_perm = c->perm; - c->backup_shared_perm = c->shared_perm; - } - /* - * Note: it's OK if c->has_backup_perm was already set, as we can find the - * same child twice during check_perm procedure - */ - - c->perm = perm; - c->shared_perm = shared; - - return 0; -} - -static void bdrv_child_set_perm(BdrvChild *c) -{ - c->has_backup_perm = false; - - bdrv_set_perm(c->bs); -} - -static void bdrv_child_abort_perm_update(BdrvChild *c) -{ - if (c->has_backup_perm) { - c->perm = c->backup_perm; - c->shared_perm = c->backup_shared_perm; - c->has_backup_perm = false; - } - - bdrv_abort_perm_update(c->bs); -} static int bdrv_refresh_perms(BlockDriverState *bs, Error **errp) { int ret; - uint64_t perm, shared_perm; + Transaction *tran = tran_new(); + g_autoptr(GSList) list = bdrv_topological_dfs(NULL, NULL, bs); - bdrv_get_cumulative_perm(bs, &perm, &shared_perm); - ret = bdrv_check_perm(bs, NULL, perm, shared_perm, NULL, errp); - if (ret < 0) { - bdrv_abort_perm_update(bs); - return ret; - } - bdrv_set_perm(bs); + ret = bdrv_list_refresh_perms(list, NULL, tran, errp); + tran_finalize(tran, ret); - return 0; + return ret; } int bdrv_child_try_set_perm(BdrvChild *c, uint64_t perm, uint64_t shared, Error **errp) { Error *local_err = NULL; + Transaction *tran = tran_new(); int ret; - ret = bdrv_child_check_perm(c, NULL, perm, shared, NULL, &local_err); + bdrv_child_set_perm(c, perm, shared, tran); + + ret = bdrv_refresh_perms(c->bs, &local_err); + + tran_finalize(tran, ret); + if (ret < 0) { - bdrv_child_abort_perm_update(c); if ((perm & ~c->perm) || (c->shared_perm & ~shared)) { /* tighten permissions */ error_propagate(errp, local_err); @@ -2370,12 +2451,9 @@ int bdrv_child_try_set_perm(BdrvChild *c, uint64_t perm, uint64_t shared, error_free(local_err); ret = 0; } - return ret; } - bdrv_child_set_perm(c); - - return 0; + return ret; } int bdrv_child_refresh_perms(BlockDriverState *bs, BdrvChild *c, Error **errp) @@ -2627,37 +2705,177 @@ static void bdrv_replace_child_noperm(BdrvChild *child, } } +static void bdrv_child_free(void *opaque) +{ + BdrvChild *c = opaque; + + g_free(c->name); + g_free(c); +} + +static void bdrv_remove_empty_child(BdrvChild *child) +{ + assert(!child->bs); + QLIST_SAFE_REMOVE(child, next); + bdrv_child_free(child); +} + +typedef struct BdrvAttachChildCommonState { + BdrvChild **child; + AioContext *old_parent_ctx; + AioContext *old_child_ctx; +} BdrvAttachChildCommonState; + +static void bdrv_attach_child_common_abort(void *opaque) +{ + BdrvAttachChildCommonState *s = opaque; + BdrvChild *child = *s->child; + BlockDriverState *bs = child->bs; + + bdrv_replace_child_noperm(child, NULL); + + if (bdrv_get_aio_context(bs) != s->old_child_ctx) { + bdrv_try_set_aio_context(bs, s->old_child_ctx, &error_abort); + } + + if (bdrv_child_get_parent_aio_context(child) != s->old_parent_ctx) { + GSList *ignore = g_slist_prepend(NULL, child); + + child->klass->can_set_aio_ctx(child, s->old_parent_ctx, &ignore, + &error_abort); + g_slist_free(ignore); + ignore = g_slist_prepend(NULL, child); + child->klass->set_aio_ctx(child, s->old_parent_ctx, &ignore); + + g_slist_free(ignore); + } + + bdrv_unref(bs); + bdrv_remove_empty_child(child); + *s->child = NULL; +} + +static TransactionActionDrv bdrv_attach_child_common_drv = { + .abort = bdrv_attach_child_common_abort, + .clean = g_free, +}; + /* - * Updates @child to change its reference to point to @new_bs, including - * checking and applying the necessary permission updates both to the old node - * and to @new_bs. - * - * NULL is passed as @new_bs for removing the reference before freeing @child. - * - * If @new_bs is not NULL, bdrv_check_perm() must be called beforehand, as this - * function uses bdrv_set_perm() to update the permissions according to the new - * reference that @new_bs gets. - * - * Callers must ensure that child->frozen is false. + * Common part of attaching bdrv child to bs or to blk or to job */ -static void bdrv_replace_child(BdrvChild *child, BlockDriverState *new_bs) +static int bdrv_attach_child_common(BlockDriverState *child_bs, + const char *child_name, + const BdrvChildClass *child_class, + BdrvChildRole child_role, + uint64_t perm, uint64_t shared_perm, + void *opaque, BdrvChild **child, + Transaction *tran, Error **errp) +{ + BdrvChild *new_child; + AioContext *parent_ctx; + AioContext *child_ctx = bdrv_get_aio_context(child_bs); + + assert(child); + assert(*child == NULL); + + new_child = g_new(BdrvChild, 1); + *new_child = (BdrvChild) { + .bs = NULL, + .name = g_strdup(child_name), + .klass = child_class, + .role = child_role, + .perm = perm, + .shared_perm = shared_perm, + .opaque = opaque, + }; + + /* + * If the AioContexts don't match, first try to move the subtree of + * child_bs into the AioContext of the new parent. If this doesn't work, + * try moving the parent into the AioContext of child_bs instead. + */ + parent_ctx = bdrv_child_get_parent_aio_context(new_child); + if (child_ctx != parent_ctx) { + Error *local_err = NULL; + int ret = bdrv_try_set_aio_context(child_bs, parent_ctx, &local_err); + + if (ret < 0 && child_class->can_set_aio_ctx) { + GSList *ignore = g_slist_prepend(NULL, new_child); + if (child_class->can_set_aio_ctx(new_child, child_ctx, &ignore, + NULL)) + { + error_free(local_err); + ret = 0; + g_slist_free(ignore); + ignore = g_slist_prepend(NULL, new_child); + child_class->set_aio_ctx(new_child, child_ctx, &ignore); + } + g_slist_free(ignore); + } + + if (ret < 0) { + error_propagate(errp, local_err); + bdrv_remove_empty_child(new_child); + return ret; + } + } + + bdrv_ref(child_bs); + bdrv_replace_child_noperm(new_child, child_bs); + + *child = new_child; + + BdrvAttachChildCommonState *s = g_new(BdrvAttachChildCommonState, 1); + *s = (BdrvAttachChildCommonState) { + .child = child, + .old_parent_ctx = parent_ctx, + .old_child_ctx = child_ctx, + }; + tran_add(tran, &bdrv_attach_child_common_drv, s); + + return 0; +} + +static int bdrv_attach_child_noperm(BlockDriverState *parent_bs, + BlockDriverState *child_bs, + const char *child_name, + const BdrvChildClass *child_class, + BdrvChildRole child_role, + BdrvChild **child, + Transaction *tran, + Error **errp) { - BlockDriverState *old_bs = child->bs; + int ret; + uint64_t perm, shared_perm; - /* Asserts that child->frozen == false */ - bdrv_replace_child_noperm(child, new_bs); + assert(parent_bs->drv); + bdrv_get_cumulative_perm(parent_bs, &perm, &shared_perm); + bdrv_child_perm(parent_bs, child_bs, NULL, child_role, NULL, + perm, shared_perm, &perm, &shared_perm); + + ret = bdrv_attach_child_common(child_bs, child_name, child_class, + child_role, perm, shared_perm, parent_bs, + child, tran, errp); + if (ret < 0) { + return ret; + } + + QLIST_INSERT_HEAD(&parent_bs->children, *child, next); /* - * Start with the new node's permissions. If @new_bs is a (direct - * or indirect) child of @old_bs, we must complete the permission - * update on @new_bs before we loosen the restrictions on @old_bs. - * Otherwise, bdrv_check_perm() on @old_bs would re-initiate - * updating the permissions of @new_bs, and thus not purely loosen - * restrictions. + * child is removed in bdrv_attach_child_common_abort(), so don't care to + * abort this change separately. */ - if (new_bs) { - bdrv_set_perm(new_bs); - } + + return 0; +} + +static void bdrv_detach_child(BdrvChild *child) +{ + BlockDriverState *old_bs = child->bs; + + bdrv_replace_child_noperm(child, NULL); + bdrv_remove_empty_child(child); if (old_bs) { /* @@ -2667,8 +2885,10 @@ static void bdrv_replace_child(BdrvChild *child, BlockDriverState *new_bs) */ bdrv_refresh_perms(old_bs, NULL); - /* When the parent requiring a non-default AioContext is removed, the - * node moves back to the main AioContext */ + /* + * When the parent requiring a non-default AioContext is removed, the + * node moves back to the main AioContext + */ bdrv_try_set_aio_context(old_bs, qemu_get_aio_context(), NULL); } } @@ -2687,61 +2907,25 @@ BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs, const char *child_name, const BdrvChildClass *child_class, BdrvChildRole child_role, - AioContext *ctx, uint64_t perm, uint64_t shared_perm, void *opaque, Error **errp) { - BdrvChild *child; - Error *local_err = NULL; int ret; + BdrvChild *child = NULL; + Transaction *tran = tran_new(); - ret = bdrv_check_update_perm(child_bs, NULL, perm, shared_perm, NULL, errp); + ret = bdrv_attach_child_common(child_bs, child_name, child_class, + child_role, perm, shared_perm, opaque, + &child, tran, errp); if (ret < 0) { - bdrv_abort_perm_update(child_bs); bdrv_unref(child_bs); return NULL; } - child = g_new(BdrvChild, 1); - *child = (BdrvChild) { - .bs = NULL, - .name = g_strdup(child_name), - .klass = child_class, - .role = child_role, - .perm = perm, - .shared_perm = shared_perm, - .opaque = opaque, - }; - - /* If the AioContexts don't match, first try to move the subtree of - * child_bs into the AioContext of the new parent. If this doesn't work, - * try moving the parent into the AioContext of child_bs instead. */ - if (bdrv_get_aio_context(child_bs) != ctx) { - ret = bdrv_try_set_aio_context(child_bs, ctx, &local_err); - if (ret < 0 && child_class->can_set_aio_ctx) { - GSList *ignore = g_slist_prepend(NULL, child); - ctx = bdrv_get_aio_context(child_bs); - if (child_class->can_set_aio_ctx(child, ctx, &ignore, NULL)) { - error_free(local_err); - ret = 0; - g_slist_free(ignore); - ignore = g_slist_prepend(NULL, child); - child_class->set_aio_ctx(child, ctx, &ignore); - } - g_slist_free(ignore); - } - if (ret < 0) { - error_propagate(errp, local_err); - g_free(child); - bdrv_abort_perm_update(child_bs); - bdrv_unref(child_bs); - return NULL; - } - } - - /* This performs the matching bdrv_set_perm() for the above check. */ - bdrv_replace_child(child, child_bs); + ret = bdrv_refresh_perms(child_bs, errp); + tran_finalize(tran, ret); + bdrv_unref(child_bs); return child; } @@ -2763,34 +2947,27 @@ BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs, BdrvChildRole child_role, Error **errp) { - BdrvChild *child; - uint64_t perm, shared_perm; - - bdrv_get_cumulative_perm(parent_bs, &perm, &shared_perm); - - assert(parent_bs->drv); - bdrv_child_perm(parent_bs, child_bs, NULL, child_role, NULL, - perm, shared_perm, &perm, &shared_perm); + int ret; + BdrvChild *child = NULL; + Transaction *tran = tran_new(); - child = bdrv_root_attach_child(child_bs, child_name, child_class, - child_role, bdrv_get_aio_context(parent_bs), - perm, shared_perm, parent_bs, errp); - if (child == NULL) { - return NULL; + ret = bdrv_attach_child_noperm(parent_bs, child_bs, child_name, child_class, + child_role, &child, tran, errp); + if (ret < 0) { + goto out; } - QLIST_INSERT_HEAD(&parent_bs->children, child, next); - return child; -} + ret = bdrv_refresh_perms(parent_bs, errp); + if (ret < 0) { + goto out; + } -static void bdrv_detach_child(BdrvChild *child) -{ - QLIST_SAFE_REMOVE(child, next); +out: + tran_finalize(tran, ret); - bdrv_replace_child(child, NULL); + bdrv_unref(child_bs); - g_free(child->name); - g_free(child); + return child; } /* Callers must ensure that child->frozen is false. */ @@ -2803,11 +2980,49 @@ void bdrv_root_unref_child(BdrvChild *child) bdrv_unref(child_bs); } +typedef struct BdrvSetInheritsFrom { + BlockDriverState *bs; + BlockDriverState *old_inherits_from; +} BdrvSetInheritsFrom; + +static void bdrv_set_inherits_from_abort(void *opaque) +{ + BdrvSetInheritsFrom *s = opaque; + + s->bs->inherits_from = s->old_inherits_from; +} + +static TransactionActionDrv bdrv_set_inherits_from_drv = { + .abort = bdrv_set_inherits_from_abort, + .clean = g_free, +}; + +/* @tran is allowed to be NULL. In this case no rollback is possible */ +static void bdrv_set_inherits_from(BlockDriverState *bs, + BlockDriverState *new_inherits_from, + Transaction *tran) +{ + if (tran) { + BdrvSetInheritsFrom *s = g_new(BdrvSetInheritsFrom, 1); + + *s = (BdrvSetInheritsFrom) { + .bs = bs, + .old_inherits_from = bs->inherits_from, + }; + + tran_add(tran, &bdrv_set_inherits_from_drv, s); + } + + bs->inherits_from = new_inherits_from; +} + /** * Clear all inherits_from pointers from children and grandchildren of * @root that point to @root, where necessary. + * @tran is allowed to be NULL. In this case no rollback is possible */ -static void bdrv_unset_inherits_from(BlockDriverState *root, BdrvChild *child) +static void bdrv_unset_inherits_from(BlockDriverState *root, BdrvChild *child, + Transaction *tran) { BdrvChild *c; @@ -2822,12 +3037,12 @@ static void bdrv_unset_inherits_from(BlockDriverState *root, BdrvChild *child) } } if (c == NULL) { - child->bs->inherits_from = NULL; + bdrv_set_inherits_from(child->bs, NULL, tran); } } QLIST_FOREACH(c, &child->bs->children, next) { - bdrv_unset_inherits_from(root, c); + bdrv_unset_inherits_from(root, c, tran); } } @@ -2838,7 +3053,7 @@ void bdrv_unref_child(BlockDriverState *parent, BdrvChild *child) return; } - bdrv_unset_inherits_from(parent, child); + bdrv_unset_inherits_from(parent, child, NULL); bdrv_root_unref_child(child); } @@ -2883,8 +3098,9 @@ static BdrvChildRole bdrv_backing_role(BlockDriverState *bs) * Sets the bs->backing link of a BDS. A new reference is created; callers * which don't need their own reference any more must call bdrv_unref(). */ -int bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd, - Error **errp) +static int bdrv_set_backing_noperm(BlockDriverState *bs, + BlockDriverState *backing_hd, + Transaction *tran, Error **errp) { int ret = 0; bool update_inherits_from = bdrv_chain_contains(bs, backing_hd) && @@ -2894,36 +3110,53 @@ int bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd, return -EPERM; } - if (backing_hd) { - bdrv_ref(backing_hd); - } - if (bs->backing) { /* Cannot be frozen, we checked that above */ - bdrv_unref_child(bs, bs->backing); - bs->backing = NULL; + bdrv_unset_inherits_from(bs, bs->backing, tran); + bdrv_remove_filter_or_cow_child(bs, tran); } if (!backing_hd) { goto out; } - bs->backing = bdrv_attach_child(bs, backing_hd, "backing", &child_of_bds, - bdrv_backing_role(bs), errp); - if (!bs->backing) { - ret = -EPERM; - goto out; + ret = bdrv_attach_child_noperm(bs, backing_hd, "backing", + &child_of_bds, bdrv_backing_role(bs), + &bs->backing, tran, errp); + if (ret < 0) { + return ret; } - /* If backing_hd was already part of bs's backing chain, and + + /* + * If backing_hd was already part of bs's backing chain, and * inherits_from pointed recursively to bs then let's update it to - * point directly to bs (else it will become NULL). */ + * point directly to bs (else it will become NULL). + */ if (update_inherits_from) { - backing_hd->inherits_from = bs; + bdrv_set_inherits_from(backing_hd, bs, tran); + } + +out: + bdrv_refresh_limits(bs, tran, NULL); + + return 0; +} + +int bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd, + Error **errp) +{ + int ret; + Transaction *tran = tran_new(); + + ret = bdrv_set_backing_noperm(bs, backing_hd, tran, errp); + if (ret < 0) { + goto out; } + ret = bdrv_refresh_perms(bs, errp); out: - bdrv_refresh_limits(bs, NULL); + tran_finalize(tran, ret); return ret; } @@ -3213,11 +3446,6 @@ static BlockDriverState *bdrv_append_temp_snapshot(BlockDriverState *bs, goto out; } - /* bdrv_append() consumes a strong reference to bs_snapshot - * (i.e. it will call bdrv_unref() on it) even on error, so in - * order to be able to return one, we have to increase - * bs_snapshot's refcount here */ - bdrv_ref(bs_snapshot); ret = bdrv_append(bs_snapshot, bs, errp); if (ret < 0) { bs_snapshot = NULL; @@ -3729,10 +3957,6 @@ static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue, bs_entry->state.explicit_options = explicit_options; bs_entry->state.flags = flags; - /* This needs to be overwritten in bdrv_reopen_prepare() */ - bs_entry->state.perm = UINT64_MAX; - bs_entry->state.shared_perm = 0; - /* * If keep_old_opts is false then it means that unspecified * options must be reset to their original value. We don't allow @@ -3817,38 +4041,49 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp) { int ret = -1; BlockReopenQueueEntry *bs_entry, *next; + Transaction *tran = tran_new(); + g_autoptr(GHashTable) found = NULL; + g_autoptr(GSList) refresh_list = NULL; assert(bs_queue != NULL); QTAILQ_FOREACH(bs_entry, bs_queue, entry) { - assert(bs_entry->state.bs->quiesce_counter > 0); - if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, errp)) { + ret = bdrv_flush(bs_entry->state.bs); + if (ret < 0) { + error_setg_errno(errp, -ret, "Error flushing drive"); goto cleanup; } + } + + QTAILQ_FOREACH(bs_entry, bs_queue, entry) { + assert(bs_entry->state.bs->quiesce_counter > 0); + ret = bdrv_reopen_prepare(&bs_entry->state, bs_queue, tran, errp); + if (ret < 0) { + goto abort; + } bs_entry->prepared = true; } + found = g_hash_table_new(NULL, NULL); QTAILQ_FOREACH(bs_entry, bs_queue, entry) { BDRVReopenState *state = &bs_entry->state; - ret = bdrv_check_perm(state->bs, bs_queue, state->perm, - state->shared_perm, NULL, errp); - if (ret < 0) { - goto cleanup_perm; - } - /* Check if new_backing_bs would accept the new permissions */ - if (state->replace_backing_bs && state->new_backing_bs) { - uint64_t nperm, nshared; - bdrv_child_perm(state->bs, state->new_backing_bs, - NULL, bdrv_backing_role(state->bs), - bs_queue, state->perm, state->shared_perm, - &nperm, &nshared); - ret = bdrv_check_update_perm(state->new_backing_bs, NULL, - nperm, nshared, NULL, errp); - if (ret < 0) { - goto cleanup_perm; - } + + refresh_list = bdrv_topological_dfs(refresh_list, found, state->bs); + if (state->old_backing_bs) { + refresh_list = bdrv_topological_dfs(refresh_list, found, + state->old_backing_bs); } - bs_entry->perms_checked = true; + } + + /* + * Note that file-posix driver rely on permission update done during reopen + * (even if no permission changed), because it wants "new" permissions for + * reconfiguring the fd and that's why it does it in raw_check_perm(), not + * in raw_reopen_prepare() which is called with "old" permissions. + */ + ret = bdrv_list_refresh_perms(refresh_list, bs_queue, tran, errp); + if (ret < 0) { + goto abort; } /* @@ -3864,51 +4099,31 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp) bdrv_reopen_commit(&bs_entry->state); } - ret = 0; -cleanup_perm: - QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) { - BDRVReopenState *state = &bs_entry->state; - - if (!bs_entry->perms_checked) { - continue; - } - - if (ret == 0) { - uint64_t perm, shared; + tran_commit(tran); - bdrv_get_cumulative_perm(state->bs, &perm, &shared); - assert(perm == state->perm); - assert(shared == state->shared_perm); + QTAILQ_FOREACH_REVERSE(bs_entry, bs_queue, entry) { + BlockDriverState *bs = bs_entry->state.bs; - bdrv_set_perm(state->bs); - } else { - bdrv_abort_perm_update(state->bs); - if (state->replace_backing_bs && state->new_backing_bs) { - bdrv_abort_perm_update(state->new_backing_bs); - } + if (bs->drv->bdrv_reopen_commit_post) { + bs->drv->bdrv_reopen_commit_post(&bs_entry->state); } } - if (ret == 0) { - QTAILQ_FOREACH_REVERSE(bs_entry, bs_queue, entry) { - BlockDriverState *bs = bs_entry->state.bs; + ret = 0; + goto cleanup; - if (bs->drv->bdrv_reopen_commit_post) - bs->drv->bdrv_reopen_commit_post(&bs_entry->state); +abort: + tran_abort(tran); + QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) { + if (bs_entry->prepared) { + bdrv_reopen_abort(&bs_entry->state); } + qobject_unref(bs_entry->state.explicit_options); + qobject_unref(bs_entry->state.options); } + cleanup: QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) { - if (ret) { - if (bs_entry->prepared) { - bdrv_reopen_abort(&bs_entry->state); - } - qobject_unref(bs_entry->state.explicit_options); - qobject_unref(bs_entry->state.options); - } - if (bs_entry->state.new_backing_bs) { - bdrv_unref(bs_entry->state.new_backing_bs); - } g_free(bs_entry); } g_free(bs_queue); @@ -3933,53 +4148,6 @@ int bdrv_reopen_set_read_only(BlockDriverState *bs, bool read_only, return ret; } -static BlockReopenQueueEntry *find_parent_in_reopen_queue(BlockReopenQueue *q, - BdrvChild *c) -{ - BlockReopenQueueEntry *entry; - - QTAILQ_FOREACH(entry, q, entry) { - BlockDriverState *bs = entry->state.bs; - BdrvChild *child; - - QLIST_FOREACH(child, &bs->children, next) { - if (child == c) { - return entry; - } - } - } - - return NULL; -} - -static void bdrv_reopen_perm(BlockReopenQueue *q, BlockDriverState *bs, - uint64_t *perm, uint64_t *shared) -{ - BdrvChild *c; - BlockReopenQueueEntry *parent; - uint64_t cumulative_perms = 0; - uint64_t cumulative_shared_perms = BLK_PERM_ALL; - - QLIST_FOREACH(c, &bs->parents, next_parent) { - parent = find_parent_in_reopen_queue(q, c); - if (!parent) { - cumulative_perms |= c->perm; - cumulative_shared_perms &= c->shared_perm; - } else { - uint64_t nperm, nshared; - - bdrv_child_perm(parent->state.bs, bs, c, c->role, q, - parent->state.perm, parent->state.shared_perm, - &nperm, &nshared); - - cumulative_perms |= nperm; - cumulative_shared_perms &= nshared; - } - } - *perm = cumulative_perms; - *shared = cumulative_shared_perms; -} - static bool bdrv_reopen_can_attach(BlockDriverState *parent, BdrvChild *child, BlockDriverState *new_child, @@ -4021,6 +4189,7 @@ static bool bdrv_reopen_can_attach(BlockDriverState *parent, * Return 0 on success, otherwise return < 0 and set @errp. */ static int bdrv_reopen_parse_backing(BDRVReopenState *reopen_state, + Transaction *set_backings_tran, Error **errp) { BlockDriverState *bs = reopen_state->bs; @@ -4097,6 +4266,8 @@ static int bdrv_reopen_parse_backing(BDRVReopenState *reopen_state, /* If we want to replace the backing file we need some extra checks */ if (new_backing_bs != bdrv_filter_or_cow_bs(overlay_bs)) { + int ret; + /* Check for implicit nodes between bs and its backing file */ if (bs != overlay_bs) { error_setg(errp, "Cannot change backing link if '%s' has " @@ -4117,9 +4288,11 @@ static int bdrv_reopen_parse_backing(BDRVReopenState *reopen_state, return -EPERM; } reopen_state->replace_backing_bs = true; - if (new_backing_bs) { - bdrv_ref(new_backing_bs); - reopen_state->new_backing_bs = new_backing_bs; + reopen_state->old_backing_bs = bs->backing ? bs->backing->bs : NULL; + ret = bdrv_set_backing_noperm(bs, new_backing_bs, set_backings_tran, + errp); + if (ret < 0) { + return ret; } } @@ -4143,8 +4316,9 @@ static int bdrv_reopen_parse_backing(BDRVReopenState *reopen_state, * commit() for any other BDS that have been left in a prepare() state * */ -int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue, - Error **errp) +static int bdrv_reopen_prepare(BDRVReopenState *reopen_state, + BlockReopenQueue *queue, + Transaction *set_backings_tran, Error **errp) { int ret = -1; int old_flags; @@ -4211,16 +4385,6 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue, goto error; } - /* Calculate required permissions after reopening */ - bdrv_reopen_perm(queue, reopen_state->bs, - &reopen_state->perm, &reopen_state->shared_perm); - - ret = bdrv_flush(reopen_state->bs); - if (ret) { - error_setg_errno(errp, -ret, "Error flushing drive"); - goto error; - } - if (drv->bdrv_reopen_prepare) { /* * If a driver-specific option is missing, it means that we @@ -4274,7 +4438,7 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue, * either a reference to an existing node (using its node name) * or NULL to simply detach the current backing file. */ - ret = bdrv_reopen_parse_backing(reopen_state, errp); + ret = bdrv_reopen_parse_backing(reopen_state, set_backings_tran, errp); if (ret < 0) { goto error; } @@ -4359,7 +4523,7 @@ error: * makes them final by swapping the staging BlockDriverState contents into * the active BlockDriverState contents. */ -void bdrv_reopen_commit(BDRVReopenState *reopen_state) +static void bdrv_reopen_commit(BDRVReopenState *reopen_state) { BlockDriver *drv; BlockDriverState *bs; @@ -4396,30 +4560,14 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state) qdict_del(bs->explicit_options, child->name); qdict_del(bs->options, child->name); } - - /* - * Change the backing file if a new one was specified. We do this - * after updating bs->options, so bdrv_refresh_filename() (called - * from bdrv_set_backing_hd()) has the new values. - */ - if (reopen_state->replace_backing_bs) { - BlockDriverState *old_backing_bs = child_bs(bs->backing); - assert(!old_backing_bs || !old_backing_bs->implicit); - /* Abort the permission update on the backing bs we're detaching */ - if (old_backing_bs) { - bdrv_abort_perm_update(old_backing_bs); - } - bdrv_set_backing_hd(bs, reopen_state->new_backing_bs, &error_abort); - } - - bdrv_refresh_limits(bs, NULL); + bdrv_refresh_limits(bs, NULL, NULL); } /* * Abort the reopen, and delete and free the staged changes in * reopen_state */ -void bdrv_reopen_abort(BDRVReopenState *reopen_state) +static void bdrv_reopen_abort(BDRVReopenState *reopen_state) { BlockDriver *drv; @@ -4585,78 +4733,176 @@ static bool should_update_child(BdrvChild *c, BlockDriverState *to) return ret; } +typedef struct BdrvRemoveFilterOrCowChild { + BdrvChild *child; + bool is_backing; +} BdrvRemoveFilterOrCowChild; + +static void bdrv_remove_filter_or_cow_child_abort(void *opaque) +{ + BdrvRemoveFilterOrCowChild *s = opaque; + BlockDriverState *parent_bs = s->child->opaque; + + QLIST_INSERT_HEAD(&parent_bs->children, s->child, next); + if (s->is_backing) { + parent_bs->backing = s->child; + } else { + parent_bs->file = s->child; + } + + /* + * We don't have to restore child->bs here to undo bdrv_replace_child() + * because that function is transactionable and it registered own completion + * entries in @tran, so .abort() for bdrv_replace_child_safe() will be + * called automatically. + */ +} + +static void bdrv_remove_filter_or_cow_child_commit(void *opaque) +{ + BdrvRemoveFilterOrCowChild *s = opaque; + + bdrv_child_free(s->child); +} + +static TransactionActionDrv bdrv_remove_filter_or_cow_child_drv = { + .abort = bdrv_remove_filter_or_cow_child_abort, + .commit = bdrv_remove_filter_or_cow_child_commit, + .clean = g_free, +}; + /* - * With auto_skip=true bdrv_replace_node_common skips updating from parents - * if it creates a parent-child relation loop or if parent is block-job. - * - * With auto_skip=false the error is returned if from has a parent which should - * not be updated. + * A function to remove backing-chain child of @bs if exists: cow child for + * format nodes (always .backing) and filter child for filters (may be .file or + * .backing) */ -static int bdrv_replace_node_common(BlockDriverState *from, - BlockDriverState *to, - bool auto_skip, Error **errp) +static void bdrv_remove_filter_or_cow_child(BlockDriverState *bs, + Transaction *tran) { - BdrvChild *c, *next; - GSList *list = NULL, *p; - uint64_t perm = 0, shared = BLK_PERM_ALL; - int ret; + BdrvRemoveFilterOrCowChild *s; + BdrvChild *child = bdrv_filter_or_cow_child(bs); - /* Make sure that @from doesn't go away until we have successfully attached - * all of its parents to @to. */ - bdrv_ref(from); + if (!child) { + return; + } - assert(qemu_get_current_aio_context() == qemu_get_aio_context()); - assert(bdrv_get_aio_context(from) == bdrv_get_aio_context(to)); - bdrv_drained_begin(from); + if (child->bs) { + bdrv_replace_child(child, NULL, tran); + } + + s = g_new(BdrvRemoveFilterOrCowChild, 1); + *s = (BdrvRemoveFilterOrCowChild) { + .child = child, + .is_backing = (child == bs->backing), + }; + tran_add(tran, &bdrv_remove_filter_or_cow_child_drv, s); + + QLIST_SAFE_REMOVE(child, next); + if (s->is_backing) { + bs->backing = NULL; + } else { + bs->file = NULL; + } +} + +static int bdrv_replace_node_noperm(BlockDriverState *from, + BlockDriverState *to, + bool auto_skip, Transaction *tran, + Error **errp) +{ + BdrvChild *c, *next; - /* Put all parents into @list and calculate their cumulative permissions */ QLIST_FOREACH_SAFE(c, &from->parents, next_parent, next) { assert(c->bs == from); if (!should_update_child(c, to)) { if (auto_skip) { continue; } - ret = -EINVAL; error_setg(errp, "Should not change '%s' link to '%s'", c->name, from->node_name); - goto out; + return -EINVAL; } if (c->frozen) { - ret = -EPERM; error_setg(errp, "Cannot change '%s' link to '%s'", c->name, from->node_name); - goto out; + return -EPERM; } - list = g_slist_prepend(list, c); - perm |= c->perm; - shared &= c->shared_perm; + bdrv_replace_child(c, to, tran); } - /* Check whether the required permissions can be granted on @to, ignoring - * all BdrvChild in @list so that they can't block themselves. */ - ret = bdrv_check_update_perm(to, NULL, perm, shared, list, errp); + return 0; +} + +/* + * With auto_skip=true bdrv_replace_node_common skips updating from parents + * if it creates a parent-child relation loop or if parent is block-job. + * + * With auto_skip=false the error is returned if from has a parent which should + * not be updated. + * + * With @detach_subchain=true @to must be in a backing chain of @from. In this + * case backing link of the cow-parent of @to is removed. + */ +static int bdrv_replace_node_common(BlockDriverState *from, + BlockDriverState *to, + bool auto_skip, bool detach_subchain, + Error **errp) +{ + Transaction *tran = tran_new(); + g_autoptr(GHashTable) found = NULL; + g_autoptr(GSList) refresh_list = NULL; + BlockDriverState *to_cow_parent; + int ret; + + if (detach_subchain) { + assert(bdrv_chain_contains(from, to)); + assert(from != to); + for (to_cow_parent = from; + bdrv_filter_or_cow_bs(to_cow_parent) != to; + to_cow_parent = bdrv_filter_or_cow_bs(to_cow_parent)) + { + ; + } + } + + /* Make sure that @from doesn't go away until we have successfully attached + * all of its parents to @to. */ + bdrv_ref(from); + + assert(qemu_get_current_aio_context() == qemu_get_aio_context()); + assert(bdrv_get_aio_context(from) == bdrv_get_aio_context(to)); + bdrv_drained_begin(from); + + /* + * Do the replacement without permission update. + * Replacement may influence the permissions, we should calculate new + * permissions based on new graph. If we fail, we'll roll-back the + * replacement. + */ + ret = bdrv_replace_node_noperm(from, to, auto_skip, tran, errp); if (ret < 0) { - bdrv_abort_perm_update(to); goto out; } - /* Now actually perform the change. We performed the permission check for - * all elements of @list at once, so set the permissions all at once at the - * very end. */ - for (p = list; p != NULL; p = p->next) { - c = p->data; - - bdrv_ref(to); - bdrv_replace_child_noperm(c, to); - bdrv_unref(from); + if (detach_subchain) { + bdrv_remove_filter_or_cow_child(to_cow_parent, tran); } - bdrv_set_perm(to); + found = g_hash_table_new(NULL, NULL); + + refresh_list = bdrv_topological_dfs(refresh_list, found, to); + refresh_list = bdrv_topological_dfs(refresh_list, found, from); + + ret = bdrv_list_refresh_perms(refresh_list, NULL, tran, errp); + if (ret < 0) { + goto out; + } ret = 0; out: - g_slist_free(list); + tran_finalize(tran, ret); + bdrv_drained_end(from); bdrv_unref(from); @@ -4666,7 +4912,13 @@ out: int bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, Error **errp) { - return bdrv_replace_node_common(from, to, true, errp); + return bdrv_replace_node_common(from, to, true, false, errp); +} + +int bdrv_drop_filter(BlockDriverState *bs, Error **errp) +{ + return bdrv_replace_node_common(bs, bdrv_filter_or_cow_bs(bs), true, true, + errp); } /* @@ -4676,37 +4928,36 @@ int bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, * This will modify the BlockDriverState fields, and swap contents * between bs_new and bs_top. Both bs_new and bs_top are modified. * - * bs_new must not be attached to a BlockBackend. + * bs_new must not be attached to a BlockBackend and must not have backing + * child. * * This function does not create any image files. - * - * bdrv_append() takes ownership of a bs_new reference and unrefs it because - * that's what the callers commonly need. bs_new will be referenced by the old - * parents of bs_top after bdrv_append() returns. If the caller needs to keep a - * reference of its own, it must call bdrv_ref(). */ int bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top, Error **errp) { - int ret = bdrv_set_backing_hd(bs_new, bs_top, errp); + int ret; + Transaction *tran = tran_new(); + + assert(!bs_new->backing); + + ret = bdrv_attach_child_noperm(bs_new, bs_top, "backing", + &child_of_bds, bdrv_backing_role(bs_new), + &bs_new->backing, tran, errp); if (ret < 0) { goto out; } - ret = bdrv_replace_node(bs_top, bs_new, errp); + ret = bdrv_replace_node_noperm(bs_top, bs_new, true, tran, errp); if (ret < 0) { - bdrv_set_backing_hd(bs_new, NULL, &error_abort); goto out; } - ret = 0; - + ret = bdrv_refresh_perms(bs_new, errp); out: - /* - * bs_new is now referenced by its new parents, we don't need the - * additional reference any more. - */ - bdrv_unref(bs_new); + tran_finalize(tran, ret); + + bdrv_refresh_limits(bs_top, NULL, NULL); return ret; } @@ -5002,7 +5253,17 @@ int bdrv_drop_intermediate(BlockDriverState *top, BlockDriverState *base, updated_children = g_slist_prepend(updated_children, c); } - bdrv_replace_node_common(top, base, false, &local_err); + /* + * It seems correct to pass detach_subchain=true here, but it triggers + * one more yet not fixed bug, when due to nested aio_poll loop we switch to + * another drained section, which modify the graph (for example, removing + * the child, which we keep in updated_children list). So, it's a TODO. + * + * Note, bug triggered if pass detach_subchain=true here and run + * test-bdrv-drain. test_drop_intermediate_poll() test-case will crash. + * That's a FIXME. + */ + bdrv_replace_node_common(top, base, false, false, &local_err); if (local_err) { error_report_err(local_err); goto exit; diff --git a/block/backup-top.c b/block/backup-top.c index 589e8b651d..425e3778be 100644 --- a/block/backup-top.c +++ b/block/backup-top.c @@ -37,7 +37,6 @@ typedef struct BDRVBackupTopState { BlockCopyState *bcs; BdrvChild *target; - bool active; int64_t cluster_size; } BDRVBackupTopState; @@ -45,12 +44,6 @@ static coroutine_fn int backup_top_co_preadv( BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags) { - BDRVBackupTopState *s = bs->opaque; - - if (!s->active) { - return -EIO; - } - return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags); } @@ -60,10 +53,6 @@ static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset, BDRVBackupTopState *s = bs->opaque; uint64_t off, end; - if (!s->active) { - return -EIO; - } - if (flags & BDRV_REQ_WRITE_UNCHANGED) { return 0; } @@ -137,21 +126,6 @@ static void backup_top_child_perm(BlockDriverState *bs, BdrvChild *c, uint64_t perm, uint64_t shared, uint64_t *nperm, uint64_t *nshared) { - BDRVBackupTopState *s = bs->opaque; - - if (!s->active) { - /* - * The filter node may be in process of bdrv_append(), which firstly do - * bdrv_set_backing_hd() and then bdrv_replace_node(). This means that - * we can't unshare BLK_PERM_WRITE during bdrv_append() operation. So, - * let's require nothing during bdrv_append() and refresh permissions - * after it (see bdrv_backup_top_append()). - */ - *nperm = 0; - *nshared = BLK_PERM_ALL; - return; - } - if (!(role & BDRV_CHILD_FILTERED)) { /* * Target child @@ -234,7 +208,6 @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, bdrv_drained_begin(source); - bdrv_ref(top); ret = bdrv_append(top, source, errp); if (ret < 0) { error_prepend(errp, "Cannot append backup-top filter: "); @@ -242,17 +215,6 @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, } appended = true; - /* - * bdrv_append() finished successfully, now we can require permissions - * we want. - */ - state->active = true; - ret = bdrv_child_refresh_perms(top, top->backing, errp); - if (ret < 0) { - error_prepend(errp, "Cannot set permissions for backup-top filter: "); - goto fail; - } - state->cluster_size = cluster_size; state->bcs = block_copy_state_new(top->backing, state->target, cluster_size, perf->use_copy_range, @@ -269,7 +231,6 @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, fail: if (appended) { - state->active = false; bdrv_backup_top_drop(top); } else { bdrv_unref(top); @@ -284,16 +245,9 @@ void bdrv_backup_top_drop(BlockDriverState *bs) { BDRVBackupTopState *s = bs->opaque; - bdrv_drained_begin(bs); + bdrv_drop_filter(bs, &error_abort); block_copy_state_free(s->bcs); - s->active = false; - bdrv_child_refresh_perms(bs, bs->backing, &error_abort); - bdrv_replace_node(bs, bs->backing->bs, &error_abort); - bdrv_set_backing_hd(bs, NULL, &error_abort); - - bdrv_drained_end(bs); - bdrv_unref(bs); } diff --git a/block/block-backend.c b/block/block-backend.c index 413af51f3b..6fca9853e1 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -298,6 +298,13 @@ static void blk_root_detach(BdrvChild *child) } } +static AioContext *blk_root_get_parent_aio_context(BdrvChild *c) +{ + BlockBackend *blk = c->opaque; + + return blk_get_aio_context(blk); +} + static const BdrvChildClass child_root = { .inherit_options = blk_root_inherit_options, @@ -318,6 +325,8 @@ static const BdrvChildClass child_root = { .can_set_aio_ctx = blk_root_can_set_aio_ctx, .set_aio_ctx = blk_root_set_aio_ctx, + + .get_parent_aio_context = blk_root_get_parent_aio_context, }; /* @@ -398,15 +407,19 @@ BlockBackend *blk_new_open(const char *filename, const char *reference, BlockBackend *blk; BlockDriverState *bs; uint64_t perm = 0; + uint64_t shared = BLK_PERM_ALL; - /* blk_new_open() is mainly used in .bdrv_create implementations and the - * tools where sharing isn't a concern because the BDS stays private, so we - * just request permission according to the flags. + /* + * blk_new_open() is mainly used in .bdrv_create implementations and the + * tools where sharing isn't a major concern because the BDS stays private + * and the file is generally not supposed to be used by a second process, + * so we just request permission according to the flags. * * The exceptions are xen_disk and blockdev_init(); in these cases, the * caller of blk_new_open() doesn't make use of the permissions, but they * shouldn't hurt either. We can still share everything here because the - * guest devices will add their own blockers if they can't share. */ + * guest devices will add their own blockers if they can't share. + */ if ((flags & BDRV_O_NO_IO) == 0) { perm |= BLK_PERM_CONSISTENT_READ; if (flags & BDRV_O_RDWR) { @@ -416,8 +429,11 @@ BlockBackend *blk_new_open(const char *filename, const char *reference, if (flags & BDRV_O_RESIZE) { perm |= BLK_PERM_RESIZE; } + if (flags & BDRV_O_NO_SHARE) { + shared = BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED; + } - blk = blk_new(qemu_get_aio_context(), perm, BLK_PERM_ALL); + blk = blk_new(qemu_get_aio_context(), perm, shared); bs = bdrv_open(filename, reference, options, flags, errp); if (!bs) { blk_unref(blk); @@ -426,7 +442,7 @@ BlockBackend *blk_new_open(const char *filename, const char *reference, blk->root = bdrv_root_attach_child(bs, "root", &child_root, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, - blk->ctx, perm, BLK_PERM_ALL, blk, errp); + perm, shared, blk, errp); if (!blk->root) { blk_unref(blk); return NULL; @@ -840,7 +856,7 @@ int blk_insert_bs(BlockBackend *blk, BlockDriverState *bs, Error **errp) bdrv_ref(bs); blk->root = bdrv_root_attach_child(bs, "root", &child_root, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, - blk->ctx, blk->perm, blk->shared_perm, + blk->perm, blk->shared_perm, blk, errp); if (blk->root == NULL) { return -EPERM; diff --git a/block/commit.c b/block/commit.c index dd9ba87349..b89bb20b75 100644 --- a/block/commit.c +++ b/block/commit.c @@ -312,6 +312,7 @@ void commit_start(const char *job_id, BlockDriverState *bs, commit_top_bs->total_sectors = top->total_sectors; ret = bdrv_append(commit_top_bs, top, errp); + bdrv_unref(commit_top_bs); /* referenced by new parents or failed */ if (ret < 0) { commit_top_bs = NULL; goto fail; diff --git a/block/file-posix.c b/block/file-posix.c index 20e14f8e96..10b71d9a13 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -175,7 +175,6 @@ typedef struct BDRVRawState { } BDRVRawState; typedef struct BDRVRawReopenState { - int fd; int open_flags; bool drop_cache; bool check_cache_dropped; @@ -1075,7 +1074,6 @@ static int raw_reopen_prepare(BDRVReopenState *state, BDRVRawReopenState *rs; QemuOpts *opts; int ret; - Error *local_err = NULL; assert(state != NULL); assert(state->bs != NULL); @@ -1101,32 +1099,18 @@ static int raw_reopen_prepare(BDRVReopenState *state, * bdrv_reopen_prepare() will detect changes and complain. */ qemu_opts_to_qdict(opts, state->options); - rs->fd = raw_reconfigure_getfd(state->bs, state->flags, &rs->open_flags, - state->perm, true, &local_err); - if (local_err) { - error_propagate(errp, local_err); - ret = -1; - goto out; - } - - /* Fail already reopen_prepare() if we can't get a working O_DIRECT - * alignment with the new fd. */ - if (rs->fd != -1) { - raw_probe_alignment(state->bs, rs->fd, &local_err); - if (local_err) { - error_propagate(errp, local_err); - ret = -EINVAL; - goto out_fd; - } - } + /* + * As part of reopen prepare we also want to create new fd by + * raw_reconfigure_getfd(). But it wants updated "perm", when in + * bdrv_reopen_multiple() .bdrv_reopen_prepare() callback called prior to + * permission update. Happily, permission update is always a part (a seprate + * stage) of bdrv_reopen_multiple() so we can rely on this fact and + * reconfigure fd in raw_check_perm(). + */ s->reopen_state = state; ret = 0; -out_fd: - if (ret < 0) { - qemu_close(rs->fd); - rs->fd = -1; - } + out: qemu_opts_del(opts); return ret; @@ -1140,10 +1124,6 @@ static void raw_reopen_commit(BDRVReopenState *state) s->drop_cache = rs->drop_cache; s->check_cache_dropped = rs->check_cache_dropped; s->open_flags = rs->open_flags; - - qemu_close(s->fd); - s->fd = rs->fd; - g_free(state->opaque); state->opaque = NULL; @@ -1162,10 +1142,6 @@ static void raw_reopen_abort(BDRVReopenState *state) return; } - if (rs->fd >= 0) { - qemu_close(rs->fd); - rs->fd = -1; - } g_free(state->opaque); state->opaque = NULL; @@ -3073,39 +3049,30 @@ static int raw_check_perm(BlockDriverState *bs, uint64_t perm, uint64_t shared, Error **errp) { BDRVRawState *s = bs->opaque; - BDRVRawReopenState *rs = NULL; + int input_flags = s->reopen_state ? s->reopen_state->flags : bs->open_flags; int open_flags; int ret; - if (s->perm_change_fd) { + /* We may need a new fd if auto-read-only switches the mode */ + ret = raw_reconfigure_getfd(bs, input_flags, &open_flags, perm, + false, errp); + if (ret < 0) { + return ret; + } else if (ret != s->fd) { + Error *local_err = NULL; + /* - * In the context of reopen, this function may be called several times - * (directly and recursively while change permissions of the parent). - * This is even true for children that don't inherit from the original - * reopen node, so s->reopen_state is not set. - * - * Ignore all but the first call. + * Fail already check_perm() if we can't get a working O_DIRECT + * alignment with the new fd. */ - return 0; - } - - if (s->reopen_state) { - /* We already have a new file descriptor to set permissions for */ - assert(s->reopen_state->perm == perm); - assert(s->reopen_state->shared_perm == shared); - rs = s->reopen_state->opaque; - s->perm_change_fd = rs->fd; - s->perm_change_flags = rs->open_flags; - } else { - /* We may need a new fd if auto-read-only switches the mode */ - ret = raw_reconfigure_getfd(bs, bs->open_flags, &open_flags, perm, - false, errp); - if (ret < 0) { - return ret; - } else if (ret != s->fd) { - s->perm_change_fd = ret; - s->perm_change_flags = open_flags; + raw_probe_alignment(bs, ret, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return -EINVAL; } + + s->perm_change_fd = ret; + s->perm_change_flags = open_flags; } /* Prepare permissions on old fd to avoid conflicts between old and new, @@ -3127,7 +3094,7 @@ static int raw_check_perm(BlockDriverState *bs, uint64_t perm, uint64_t shared, return 0; fail: - if (s->perm_change_fd && !s->reopen_state) { + if (s->perm_change_fd) { qemu_close(s->perm_change_fd); } s->perm_change_fd = 0; @@ -3158,7 +3125,7 @@ static void raw_abort_perm_update(BlockDriverState *bs) /* For reopen, .bdrv_reopen_abort is called afterwards and will close * the file descriptor. */ - if (s->perm_change_fd && !s->reopen_state) { + if (s->perm_change_fd) { qemu_close(s->perm_change_fd); } s->perm_change_fd = 0; diff --git a/block/io.c b/block/io.c index ca2dca3007..35b6c56efc 100644 --- a/block/io.c +++ b/block/io.c @@ -133,13 +133,40 @@ static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); } -void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) +typedef struct BdrvRefreshLimitsState { + BlockDriverState *bs; + BlockLimits old_bl; +} BdrvRefreshLimitsState; + +static void bdrv_refresh_limits_abort(void *opaque) +{ + BdrvRefreshLimitsState *s = opaque; + + s->bs->bl = s->old_bl; +} + +static TransactionActionDrv bdrv_refresh_limits_drv = { + .abort = bdrv_refresh_limits_abort, + .clean = g_free, +}; + +/* @tran is allowed to be NULL, in this case no rollback is possible. */ +void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp) { ERRP_GUARD(); BlockDriver *drv = bs->drv; BdrvChild *c; bool have_limits; + if (tran) { + BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1); + *s = (BdrvRefreshLimitsState) { + .bs = bs, + .old_bl = bs->bl, + }; + tran_add(tran, &bdrv_refresh_limits_drv, s); + } + memset(&bs->bl, 0, sizeof(bs->bl)); if (!drv) { @@ -156,7 +183,7 @@ void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) QLIST_FOREACH(c, &bs->children, next) { if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) { - bdrv_refresh_limits(c->bs, errp); + bdrv_refresh_limits(c->bs, tran, errp); if (*errp) { return; } diff --git a/block/mirror.c b/block/mirror.c index 5a71bd8bbc..840b8e8c15 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -1630,9 +1630,6 @@ static BlockJob *mirror_start_job( bs_opaque->is_commit = target_is_backing; - /* bdrv_append takes ownership of the mirror_top_bs reference, need to keep - * it alive until block_job_create() succeeds even if bs has no parent. */ - bdrv_ref(mirror_top_bs); bdrv_drained_begin(bs); ret = bdrv_append(mirror_top_bs, bs, errp); bdrv_drained_end(bs); diff --git a/blockdev.c b/blockdev.c index a57590aae4..834c2304a1 100644 --- a/blockdev.c +++ b/blockdev.c @@ -1576,10 +1576,6 @@ static void external_snapshot_prepare(BlkActionState *common, goto out; } - /* This removes our old bs and adds the new bs. This is an operation that - * can fail, so we need to do it in .prepare; undoing it for abort is - * always possible. */ - bdrv_ref(state->new_bs); ret = bdrv_append(state->new_bs, state->old_bs, errp); if (ret < 0) { goto out; diff --git a/blockjob.c b/blockjob.c index 207e8c7fd9..2fe1d788ba 100644 --- a/blockjob.c +++ b/blockjob.c @@ -163,6 +163,13 @@ static void child_job_set_aio_ctx(BdrvChild *c, AioContext *ctx, job->job.aio_context = ctx; } +static AioContext *child_job_get_parent_aio_context(BdrvChild *c) +{ + BlockJob *job = c->opaque; + + return job->job.aio_context; +} + static const BdrvChildClass child_job = { .get_parent_desc = child_job_get_parent_desc, .drained_begin = child_job_drained_begin, @@ -171,6 +178,7 @@ static const BdrvChildClass child_job = { .can_set_aio_ctx = child_job_can_set_aio_ctx, .set_aio_ctx = child_job_set_aio_ctx, .stay_at_node = true, + .get_parent_aio_context = child_job_get_parent_aio_context, }; void block_job_remove_all_bdrv(BlockJob *job) @@ -221,8 +229,7 @@ int block_job_add_bdrv(BlockJob *job, const char *name, BlockDriverState *bs, if (need_context_ops && job->job.aio_context != qemu_get_aio_context()) { aio_context_release(job->job.aio_context); } - c = bdrv_root_attach_child(bs, name, &child_job, 0, - job->job.aio_context, perm, shared_perm, job, + c = bdrv_root_attach_child(bs, name, &child_job, 0, perm, shared_perm, job, errp); if (need_context_ops && job->job.aio_context != qemu_get_aio_context()) { aio_context_acquire(job->job.aio_context); diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c index 8705612535..228dc54b0b 100644 --- a/hw/arm/smmuv3.c +++ b/hw/arm/smmuv3.c @@ -259,8 +259,9 @@ static void smmuv3_init_regs(SMMUv3State *s) s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1); s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1); - /* 4K and 64K granule support */ + /* 4K, 16K and 64K granule support */ s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1); + s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN16K, 1); s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1); s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS); /* 44 bits */ @@ -503,7 +504,8 @@ static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event) tg = CD_TG(cd, i); tt->granule_sz = tg2granule(tg, i); - if ((tt->granule_sz != 12 && tt->granule_sz != 16) || CD_ENDI(cd)) { + if ((tt->granule_sz != 12 && tt->granule_sz != 14 && + tt->granule_sz != 16) || CD_ENDI(cd)) { goto bad_cd; } diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 9f01d9041b..fee696fb0e 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -2757,10 +2757,15 @@ static void machvirt_machine_init(void) } type_init(machvirt_machine_init); +static void virt_machine_6_1_options(MachineClass *mc) +{ +} +DEFINE_VIRT_MACHINE_AS_LATEST(6, 1) + static void virt_machine_6_0_options(MachineClass *mc) { } -DEFINE_VIRT_MACHINE_AS_LATEST(6, 0) +DEFINE_VIRT_MACHINE(6, 0) static void virt_machine_5_2_options(MachineClass *mc) { diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c index 0b5b9d44cd..f5e9682703 100644 --- a/hw/block/vhost-user-blk.c +++ b/hw/block/vhost-user-blk.c @@ -467,6 +467,11 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp) error_setg(errp, "vhost-user-blk: queue size must be non-zero"); return; } + if (s->queue_size > VIRTQUEUE_MAX_SIZE) { + error_setg(errp, "vhost-user-blk: queue size must not exceed %d", + VIRTQUEUE_MAX_SIZE); + return; + } if (!vhost_user_init(&s->vhost_user, &s->chardev, errp)) { return; diff --git a/hw/core/machine.c b/hw/core/machine.c index 40def78183..0f5ce43d0c 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -36,6 +36,11 @@ #include "hw/virtio/virtio.h" #include "hw/virtio/virtio-pci.h" +GlobalProperty hw_compat_6_0[] = { + { "gpex-pcihost", "allow-unmapped-accesses", "false" }, +}; +const size_t hw_compat_6_0_len = G_N_ELEMENTS(hw_compat_6_0); + GlobalProperty hw_compat_5_2[] = { { "ICH9-LPC", "smm-compat", "on"}, { "PIIX4_PM", "smm-compat", "on"}, diff --git a/hw/i386/pc.c b/hw/i386/pc.c index 8a84b25a03..364816efc9 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -96,6 +96,9 @@ #include "trace.h" #include CONFIG_DEVICES +GlobalProperty pc_compat_6_0[] = {}; +const size_t pc_compat_6_0_len = G_N_ELEMENTS(pc_compat_6_0); + GlobalProperty pc_compat_5_2[] = { { "ICH9-LPC", "x-smi-cpu-hotunplug", "off" }, }; diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c index 46cc951073..4e8edffeaf 100644 --- a/hw/i386/pc_piix.c +++ b/hw/i386/pc_piix.c @@ -415,7 +415,7 @@ static void pc_i440fx_machine_options(MachineClass *m) machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE); } -static void pc_i440fx_6_0_machine_options(MachineClass *m) +static void pc_i440fx_6_1_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_i440fx_machine_options(m); @@ -424,6 +424,18 @@ static void pc_i440fx_6_0_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_I440FX_MACHINE(v6_1, "pc-i440fx-6.1", NULL, + pc_i440fx_6_1_machine_options); + +static void pc_i440fx_6_0_machine_options(MachineClass *m) +{ + pc_i440fx_6_1_machine_options(m); + m->alias = NULL; + m->is_default = false; + compat_props_add(m->compat_props, hw_compat_6_0, hw_compat_6_0_len); + compat_props_add(m->compat_props, pc_compat_6_0, pc_compat_6_0_len); +} + DEFINE_I440FX_MACHINE(v6_0, "pc-i440fx-6.0", NULL, pc_i440fx_6_0_machine_options); diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c index 53450190f5..458ed41c65 100644 --- a/hw/i386/pc_q35.c +++ b/hw/i386/pc_q35.c @@ -345,7 +345,7 @@ static void pc_q35_machine_options(MachineClass *m) m->max_cpus = 288; } -static void pc_q35_6_0_machine_options(MachineClass *m) +static void pc_q35_6_1_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_q35_machine_options(m); @@ -353,6 +353,17 @@ static void pc_q35_6_0_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_Q35_MACHINE(v6_1, "pc-q35-6.1", NULL, + pc_q35_6_1_machine_options); + +static void pc_q35_6_0_machine_options(MachineClass *m) +{ + pc_q35_6_1_machine_options(m); + m->alias = NULL; + compat_props_add(m->compat_props, hw_compat_6_0, hw_compat_6_0_len); + compat_props_add(m->compat_props, pc_compat_6_0, pc_compat_6_0_len); +} + DEFINE_Q35_MACHINE(v6_0, "pc-q35-6.0", NULL, pc_q35_6_0_machine_options); diff --git a/hw/pci-host/gpex.c b/hw/pci-host/gpex.c index 2bdbe7b456..a6752fac5e 100644 --- a/hw/pci-host/gpex.c +++ b/hw/pci-host/gpex.c @@ -83,12 +83,51 @@ static void gpex_host_realize(DeviceState *dev, Error **errp) int i; pcie_host_mmcfg_init(pex, PCIE_MMCFG_SIZE_MAX); + sysbus_init_mmio(sbd, &pex->mmio); + + /* + * Note that the MemoryRegions io_mmio and io_ioport that we pass + * to pci_register_root_bus() are not the same as the + * MemoryRegions io_mmio_window and io_ioport_window that we + * expose as SysBus MRs. The difference is in the behaviour of + * accesses to addresses where no PCI device has been mapped. + * + * io_mmio and io_ioport are the underlying PCI view of the PCI + * address space, and when a PCI device does a bus master access + * to a bad address this is reported back to it as a transaction + * failure. + * + * io_mmio_window and io_ioport_window implement "unmapped + * addresses read as -1 and ignore writes"; this is traditional + * x86 PC behaviour, which is not mandated by the PCI spec proper + * but expected by much PCI-using guest software, including Linux. + * + * In the interests of not being unnecessarily surprising, we + * implement it in the gpex PCI host controller, by providing the + * _window MRs, which are containers with io ops that implement + * the 'background' behaviour and which hold the real PCI MRs as + * subregions. + */ memory_region_init(&s->io_mmio, OBJECT(s), "gpex_mmio", UINT64_MAX); memory_region_init(&s->io_ioport, OBJECT(s), "gpex_ioport", 64 * 1024); - sysbus_init_mmio(sbd, &pex->mmio); - sysbus_init_mmio(sbd, &s->io_mmio); - sysbus_init_mmio(sbd, &s->io_ioport); + if (s->allow_unmapped_accesses) { + memory_region_init_io(&s->io_mmio_window, OBJECT(s), + &unassigned_io_ops, OBJECT(s), + "gpex_mmio_window", UINT64_MAX); + memory_region_init_io(&s->io_ioport_window, OBJECT(s), + &unassigned_io_ops, OBJECT(s), + "gpex_ioport_window", 64 * 1024); + + memory_region_add_subregion(&s->io_mmio_window, 0, &s->io_mmio); + memory_region_add_subregion(&s->io_ioport_window, 0, &s->io_ioport); + sysbus_init_mmio(sbd, &s->io_mmio_window); + sysbus_init_mmio(sbd, &s->io_ioport_window); + } else { + sysbus_init_mmio(sbd, &s->io_mmio); + sysbus_init_mmio(sbd, &s->io_ioport); + } + for (i = 0; i < GPEX_NUM_IRQS; i++) { sysbus_init_irq(sbd, &s->irq[i]); s->irq_num[i] = -1; @@ -108,6 +147,16 @@ static const char *gpex_host_root_bus_path(PCIHostState *host_bridge, return "0000:00"; } +static Property gpex_host_properties[] = { + /* + * Permit CPU accesses to unmapped areas of the PIO and MMIO windows + * (discarding writes and returning -1 for reads) rather than aborting. + */ + DEFINE_PROP_BOOL("allow-unmapped-accesses", GPEXHost, + allow_unmapped_accesses, true), + DEFINE_PROP_END_OF_LIST(), +}; + static void gpex_host_class_init(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); @@ -117,6 +166,7 @@ static void gpex_host_class_init(ObjectClass *klass, void *data) dc->realize = gpex_host_realize; set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); dc->fw_name = "pci"; + device_class_set_props(dc, gpex_host_properties); } static void gpex_host_initfn(Object *obj) diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index e4be00b732..529ff056dd 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -4605,14 +4605,25 @@ static void spapr_machine_latest_class_options(MachineClass *mc) type_init(spapr_machine_register_##suffix) /* + * pseries-6.1 + */ +static void spapr_machine_6_1_class_options(MachineClass *mc) +{ + /* Defaults for the latest behaviour inherited from the base class */ +} + +DEFINE_SPAPR_MACHINE(6_1, "6.1", true); + +/* * pseries-6.0 */ static void spapr_machine_6_0_class_options(MachineClass *mc) { - /* Defaults for the latest behaviour inherited from the base class */ + spapr_machine_6_1_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_6_0, hw_compat_6_0_len); } -DEFINE_SPAPR_MACHINE(6_0, "6.0", true); +DEFINE_SPAPR_MACHINE(6_0, "6.0", false); /* * pseries-5.2 diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index 2972b607f3..56b52d2d30 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -795,14 +795,26 @@ bool css_migration_enabled(void) } \ type_init(ccw_machine_register_##suffix) +static void ccw_machine_6_1_instance_options(MachineState *machine) +{ +} + +static void ccw_machine_6_1_class_options(MachineClass *mc) +{ +} +DEFINE_CCW_MACHINE(6_1, "6.1", true); + static void ccw_machine_6_0_instance_options(MachineState *machine) { + ccw_machine_6_1_instance_options(machine); } static void ccw_machine_6_0_class_options(MachineClass *mc) { + ccw_machine_6_1_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_6_0, hw_compat_6_0_len); } -DEFINE_CCW_MACHINE(6_0, "6.0", true); +DEFINE_CCW_MACHINE(6_0, "6.0", false); static void ccw_machine_5_2_instance_options(MachineState *machine) { diff --git a/include/block/block.h b/include/block/block.h index b3f6e509d4..82185965ff 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -9,6 +9,7 @@ #include "block/dirty-bitmap.h" #include "block/blockjob.h" #include "qemu/hbitmap.h" +#include "qemu/transactions.h" /* * generated_co_wrapper @@ -101,6 +102,7 @@ typedef struct HDGeometry { uint32_t cylinders; } HDGeometry; +#define BDRV_O_NO_SHARE 0x0001 /* don't share permissions */ #define BDRV_O_RDWR 0x0002 #define BDRV_O_RESIZE 0x0004 /* request permission for resizing the node */ #define BDRV_O_SNAPSHOT 0x0008 /* open the file read only and save writes in a snapshot */ @@ -207,8 +209,7 @@ typedef struct BDRVReopenState { BlockdevDetectZeroesOptions detect_zeroes; bool backing_missing; bool replace_backing_bs; /* new_backing_bs is ignored if this is false */ - BlockDriverState *new_backing_bs; /* If NULL then detach the current bs */ - uint64_t perm, shared_perm; + BlockDriverState *old_backing_bs; /* keep pointer for permissions update */ QDict *options; QDict *explicit_options; void *opaque; @@ -362,6 +363,7 @@ int bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, Error **errp); BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options, int flags, Error **errp); +int bdrv_drop_filter(BlockDriverState *bs, Error **errp); int bdrv_parse_aio(const char *mode, int *flags); int bdrv_parse_cache_mode(const char *mode, int *flags, bool *writethrough); @@ -387,10 +389,6 @@ BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue, int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp); int bdrv_reopen_set_read_only(BlockDriverState *bs, bool read_only, Error **errp); -int bdrv_reopen_prepare(BDRVReopenState *reopen_state, - BlockReopenQueue *queue, Error **errp); -void bdrv_reopen_commit(BDRVReopenState *reopen_state); -void bdrv_reopen_abort(BDRVReopenState *reopen_state); int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, int64_t bytes, BdrvRequestFlags flags); int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags); @@ -424,7 +422,7 @@ int64_t bdrv_get_allocated_file_size(BlockDriverState *bs); BlockMeasureInfo *bdrv_measure(BlockDriver *drv, QemuOpts *opts, BlockDriverState *in_bs, Error **errp); void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr); -void bdrv_refresh_limits(BlockDriverState *bs, Error **errp); +void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp); int bdrv_commit(BlockDriverState *bs); int bdrv_make_empty(BdrvChild *c, Error **errp); int bdrv_change_backing_file(BlockDriverState *bs, const char *backing_file, @@ -702,6 +700,8 @@ bool bdrv_child_can_set_aio_context(BdrvChild *c, AioContext *ctx, GSList **ignore, Error **errp); bool bdrv_can_set_aio_context(BlockDriverState *bs, AioContext *ctx, GSList **ignore, Error **errp); +AioContext *bdrv_child_get_parent_aio_context(BdrvChild *c); + int bdrv_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz); int bdrv_probe_geometry(BlockDriverState *bs, HDGeometry *geo); diff --git a/include/block/block_int.h b/include/block/block_int.h index 88e4111939..c823f5b1b3 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -789,6 +789,8 @@ struct BdrvChildClass { bool (*can_set_aio_ctx)(BdrvChild *child, AioContext *ctx, GSList **ignore, Error **errp); void (*set_aio_ctx)(BdrvChild *child, AioContext *ctx, GSList **ignore); + + AioContext *(*get_parent_aio_context)(BdrvChild *child); }; extern const BdrvChildClass child_of_bds; @@ -811,11 +813,6 @@ struct BdrvChild { */ uint64_t shared_perm; - /* backup of permissions during permission update procedure */ - bool has_backup_perm; - uint64_t backup_perm; - uint64_t backup_shared_perm; - /* * This link is frozen: the child can neither be replaced nor * detached from the parent. @@ -1306,7 +1303,6 @@ BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs, const char *child_name, const BdrvChildClass *child_class, BdrvChildRole child_role, - AioContext *ctx, uint64_t perm, uint64_t shared_perm, void *opaque, Error **errp); void bdrv_root_unref_child(BdrvChild *child); diff --git a/include/hw/boards.h b/include/hw/boards.h index ad6c8fd537..3d55d2bd62 100644 --- a/include/hw/boards.h +++ b/include/hw/boards.h @@ -353,6 +353,9 @@ struct MachineState { } \ type_init(machine_initfn##_register_types) +extern GlobalProperty hw_compat_6_0[]; +extern const size_t hw_compat_6_0_len; + extern GlobalProperty hw_compat_5_2[]; extern const size_t hw_compat_5_2_len; diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h index dcf060b791..1522a3359a 100644 --- a/include/hw/i386/pc.h +++ b/include/hw/i386/pc.h @@ -197,6 +197,9 @@ bool pc_system_ovmf_table_find(const char *entry, uint8_t **data, void pc_madt_cpu_entry(AcpiDeviceIf *adev, int uid, const CPUArchIdList *apic_ids, GArray *entry); +extern GlobalProperty pc_compat_6_0[]; +extern const size_t pc_compat_6_0_len; + extern GlobalProperty pc_compat_5_2[]; extern const size_t pc_compat_5_2_len; diff --git a/include/hw/pci-host/gpex.h b/include/hw/pci-host/gpex.h index d48a020a95..fcf8b63820 100644 --- a/include/hw/pci-host/gpex.h +++ b/include/hw/pci-host/gpex.h @@ -49,8 +49,12 @@ struct GPEXHost { MemoryRegion io_ioport; MemoryRegion io_mmio; + MemoryRegion io_ioport_window; + MemoryRegion io_mmio_window; qemu_irq irq[GPEX_NUM_IRQS]; int irq_num[GPEX_NUM_IRQS]; + + bool allow_unmapped_accesses; }; struct GPEXConfig { diff --git a/include/qemu/transactions.h b/include/qemu/transactions.h new file mode 100644 index 0000000000..92c5965235 --- /dev/null +++ b/include/qemu/transactions.h @@ -0,0 +1,63 @@ +/* + * Simple transactions API + * + * Copyright (c) 2021 Virtuozzo International GmbH. + * + * Author: + * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * = Generic transaction API = + * + * The intended usage is the following: you create "prepare" functions, which + * represents the actions. They will usually have Transaction* argument, and + * call tran_add() to register finalization callbacks. For finalization + * callbacks, prepare corresponding TransactionActionDrv structures. + * + * Then, when you need to make a transaction, create an empty Transaction by + * tran_create(), call your "prepare" functions on it, and finally call + * tran_abort() or tran_commit() to finalize the transaction by corresponding + * finalization actions in reverse order. + */ + +#ifndef QEMU_TRANSACTIONS_H +#define QEMU_TRANSACTIONS_H + +#include <gmodule.h> + +typedef struct TransactionActionDrv { + void (*abort)(void *opaque); + void (*commit)(void *opaque); + void (*clean)(void *opaque); +} TransactionActionDrv; + +typedef struct Transaction Transaction; + +Transaction *tran_new(void); +void tran_add(Transaction *tran, TransactionActionDrv *drv, void *opaque); +void tran_abort(Transaction *tran); +void tran_commit(Transaction *tran); + +static inline void tran_finalize(Transaction *tran, int ret) +{ + if (ret < 0) { + tran_abort(tran); + } else { + tran_commit(tran); + } +} + +#endif /* QEMU_TRANSACTIONS_H */ diff --git a/qemu-img.c b/qemu-img.c index babb5573ab..a5993682aa 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -2146,7 +2146,7 @@ static void set_rate_limit(BlockBackend *blk, int64_t rate_limit) static int img_convert(int argc, char **argv) { - int c, bs_i, flags, src_flags = 0; + int c, bs_i, flags, src_flags = BDRV_O_NO_SHARE; const char *fmt = NULL, *out_fmt = NULL, *cache = "unsafe", *src_cache = BDRV_DEFAULT_CACHE, *out_baseimg = NULL, *out_filename, *out_baseimg_param, *snapshot_name = NULL; diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 193a49ec7f..616b393253 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -225,6 +225,11 @@ typedef struct ARMPACKey { } ARMPACKey; #endif +/* See the commentary above the TBFLAG field definitions. */ +typedef struct CPUARMTBFlags { + uint32_t flags; + target_ulong flags2; +} CPUARMTBFlags; typedef struct CPUARMState { /* Regs for current mode. */ @@ -253,7 +258,7 @@ typedef struct CPUARMState { uint32_t aarch64; /* 1 if CPU is in aarch64 state; inverse of PSTATE.nRW */ /* Cached TBFLAGS state. See below for which bits are included. */ - uint32_t hflags; + CPUARMTBFlags hflags; /* Frequently accessed CPSR bits are stored separately for efficiency. This contains all the other bits. Use cpsr_{read,write} to access @@ -3377,74 +3382,82 @@ typedef ARMCPU ArchCPU; #include "exec/cpu-all.h" /* - * Bit usage in the TB flags field: bit 31 indicates whether we are - * in 32 or 64 bit mode. The meaning of the other bits depends on that. - * We put flags which are shared between 32 and 64 bit mode at the top - * of the word, and flags which apply to only one mode at the bottom. + * We have more than 32-bits worth of state per TB, so we split the data + * between tb->flags and tb->cs_base, which is otherwise unused for ARM. + * We collect these two parts in CPUARMTBFlags where they are named + * flags and flags2 respectively. * - * 31 20 18 14 9 0 - * +--------------+-----+-----+----------+--------------+ - * | | | TBFLAG_A32 | | - * | | +-----+----------+ TBFLAG_AM32 | - * | TBFLAG_ANY | |TBFLAG_M32| | - * | +-----------+----------+--------------| - * | | TBFLAG_A64 | - * +--------------+-------------------------------------+ - * 31 20 0 + * The flags that are shared between all execution modes, TBFLAG_ANY, + * are stored in flags. The flags that are specific to a given mode + * are stores in flags2. Since cs_base is sized on the configured + * address size, flags2 always has 64-bits for A64, and a minimum of + * 32-bits for A32 and M32. + * + * The bits for 32-bit A-profile and M-profile partially overlap: + * + * 31 23 11 10 0 + * +-------------+----------+----------------+ + * | | | TBFLAG_A32 | + * | TBFLAG_AM32 | +-----+----------+ + * | | |TBFLAG_M32| + * +-------------+----------------+----------+ + * 31 23 5 4 0 * * Unless otherwise noted, these bits are cached in env->hflags. */ -FIELD(TBFLAG_ANY, AARCH64_STATE, 31, 1) -FIELD(TBFLAG_ANY, SS_ACTIVE, 30, 1) -FIELD(TBFLAG_ANY, PSTATE_SS, 29, 1) /* Not cached. */ -FIELD(TBFLAG_ANY, BE_DATA, 28, 1) -FIELD(TBFLAG_ANY, MMUIDX, 24, 4) +FIELD(TBFLAG_ANY, AARCH64_STATE, 0, 1) +FIELD(TBFLAG_ANY, SS_ACTIVE, 1, 1) +FIELD(TBFLAG_ANY, PSTATE__SS, 2, 1) /* Not cached. */ +FIELD(TBFLAG_ANY, BE_DATA, 3, 1) +FIELD(TBFLAG_ANY, MMUIDX, 4, 4) /* Target EL if we take a floating-point-disabled exception */ -FIELD(TBFLAG_ANY, FPEXC_EL, 22, 2) +FIELD(TBFLAG_ANY, FPEXC_EL, 8, 2) /* For A-profile only, target EL for debug exceptions. */ -FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 20, 2) +FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 10, 2) +/* Memory operations require alignment: SCTLR_ELx.A or CCR.UNALIGN_TRP */ +FIELD(TBFLAG_ANY, ALIGN_MEM, 12, 1) /* * Bit usage when in AArch32 state, both A- and M-profile. */ -FIELD(TBFLAG_AM32, CONDEXEC, 0, 8) /* Not cached. */ -FIELD(TBFLAG_AM32, THUMB, 8, 1) /* Not cached. */ +FIELD(TBFLAG_AM32, CONDEXEC, 24, 8) /* Not cached. */ +FIELD(TBFLAG_AM32, THUMB, 23, 1) /* Not cached. */ /* * Bit usage when in AArch32 state, for A-profile only. */ -FIELD(TBFLAG_A32, VECLEN, 9, 3) /* Not cached. */ -FIELD(TBFLAG_A32, VECSTRIDE, 12, 2) /* Not cached. */ +FIELD(TBFLAG_A32, VECLEN, 0, 3) /* Not cached. */ +FIELD(TBFLAG_A32, VECSTRIDE, 3, 2) /* Not cached. */ /* * We store the bottom two bits of the CPAR as TB flags and handle * checks on the other bits at runtime. This shares the same bits as * VECSTRIDE, which is OK as no XScale CPU has VFP. * Not cached, because VECLEN+VECSTRIDE are not cached. */ -FIELD(TBFLAG_A32, XSCALE_CPAR, 12, 2) -FIELD(TBFLAG_A32, VFPEN, 14, 1) /* Partially cached, minus FPEXC. */ -FIELD(TBFLAG_A32, SCTLR_B, 15, 1) -FIELD(TBFLAG_A32, HSTR_ACTIVE, 16, 1) +FIELD(TBFLAG_A32, XSCALE_CPAR, 5, 2) +FIELD(TBFLAG_A32, VFPEN, 7, 1) /* Partially cached, minus FPEXC. */ +FIELD(TBFLAG_A32, SCTLR__B, 8, 1) /* Cannot overlap with SCTLR_B */ +FIELD(TBFLAG_A32, HSTR_ACTIVE, 9, 1) /* * Indicates whether cp register reads and writes by guest code should access * the secure or nonsecure bank of banked registers; note that this is not * the same thing as the current security state of the processor! */ -FIELD(TBFLAG_A32, NS, 17, 1) +FIELD(TBFLAG_A32, NS, 10, 1) /* * Bit usage when in AArch32 state, for M-profile only. */ /* Handler (ie not Thread) mode */ -FIELD(TBFLAG_M32, HANDLER, 9, 1) +FIELD(TBFLAG_M32, HANDLER, 0, 1) /* Whether we should generate stack-limit checks */ -FIELD(TBFLAG_M32, STACKCHECK, 10, 1) +FIELD(TBFLAG_M32, STACKCHECK, 1, 1) /* Set if FPCCR.LSPACT is set */ -FIELD(TBFLAG_M32, LSPACT, 11, 1) /* Not cached. */ +FIELD(TBFLAG_M32, LSPACT, 2, 1) /* Not cached. */ /* Set if we must create a new FP context */ -FIELD(TBFLAG_M32, NEW_FP_CTXT_NEEDED, 12, 1) /* Not cached. */ +FIELD(TBFLAG_M32, NEW_FP_CTXT_NEEDED, 3, 1) /* Not cached. */ /* Set if FPCCR.S does not match current security state */ -FIELD(TBFLAG_M32, FPCCR_S_WRONG, 13, 1) /* Not cached. */ +FIELD(TBFLAG_M32, FPCCR_S_WRONG, 4, 1) /* Not cached. */ /* * Bit usage when in AArch64 state @@ -3462,6 +3475,26 @@ FIELD(TBFLAG_A64, TCMA, 16, 2) FIELD(TBFLAG_A64, MTE_ACTIVE, 18, 1) FIELD(TBFLAG_A64, MTE0_ACTIVE, 19, 1) +/* + * Helpers for using the above. + */ +#define DP_TBFLAG_ANY(DST, WHICH, VAL) \ + (DST.flags = FIELD_DP32(DST.flags, TBFLAG_ANY, WHICH, VAL)) +#define DP_TBFLAG_A64(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_A64, WHICH, VAL)) +#define DP_TBFLAG_A32(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_A32, WHICH, VAL)) +#define DP_TBFLAG_M32(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_M32, WHICH, VAL)) +#define DP_TBFLAG_AM32(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_AM32, WHICH, VAL)) + +#define EX_TBFLAG_ANY(IN, WHICH) FIELD_EX32(IN.flags, TBFLAG_ANY, WHICH) +#define EX_TBFLAG_A64(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_A64, WHICH) +#define EX_TBFLAG_A32(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_A32, WHICH) +#define EX_TBFLAG_M32(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_M32, WHICH) +#define EX_TBFLAG_AM32(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_AM32, WHICH) + /** * cpu_mmu_index: * @env: The cpu environment @@ -3472,7 +3505,7 @@ FIELD(TBFLAG_A64, MTE0_ACTIVE, 19, 1) */ static inline int cpu_mmu_index(CPUARMState *env, bool ifetch) { - return FIELD_EX32(env->hflags, TBFLAG_ANY, MMUIDX); + return EX_TBFLAG_ANY(env->hflags, MMUIDX); } static inline bool bswap_code(bool sctlr_b) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index 061c8ff846..9cc3b066e2 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -1020,7 +1020,7 @@ void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc) * the hflags rebuild, since we can pull the composite TBII field * from there. */ - tbii = FIELD_EX32(env->hflags, TBFLAG_A64, TBII); + tbii = EX_TBFLAG_A64(env->hflags, TBII); if ((tbii >> extract64(new_pc, 55, 1)) & 1) { /* TBI is enabled. */ int core_mmu_idx = cpu_mmu_index(env, false); diff --git a/target/arm/helper-a64.h b/target/arm/helper-a64.h index c139fa81f9..7b706571bb 100644 --- a/target/arm/helper-a64.h +++ b/target/arm/helper-a64.h @@ -104,8 +104,7 @@ DEF_HELPER_FLAGS_3(autdb, TCG_CALL_NO_WG, i64, env, i64, i64) DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64) DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64) -DEF_HELPER_FLAGS_3(mte_check1, TCG_CALL_NO_WG, i64, env, i32, i64) -DEF_HELPER_FLAGS_3(mte_checkN, TCG_CALL_NO_WG, i64, env, i32, i64) +DEF_HELPER_FLAGS_3(mte_check, TCG_CALL_NO_WG, i64, env, i32, i64) DEF_HELPER_FLAGS_3(mte_check_zva, TCG_CALL_NO_WG, i64, env, i32, i64) DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64) DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32) diff --git a/target/arm/helper.c b/target/arm/helper.c index d9220be7c5..9b1b98705f 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -12984,42 +12984,49 @@ ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) } #endif -static uint32_t rebuild_hflags_common(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx, uint32_t flags) +static CPUARMTBFlags rebuild_hflags_common(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx, + CPUARMTBFlags flags) { - flags = FIELD_DP32(flags, TBFLAG_ANY, FPEXC_EL, fp_el); - flags = FIELD_DP32(flags, TBFLAG_ANY, MMUIDX, - arm_to_core_mmu_idx(mmu_idx)); + DP_TBFLAG_ANY(flags, FPEXC_EL, fp_el); + DP_TBFLAG_ANY(flags, MMUIDX, arm_to_core_mmu_idx(mmu_idx)); if (arm_singlestep_active(env)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, SS_ACTIVE, 1); + DP_TBFLAG_ANY(flags, SS_ACTIVE, 1); } return flags; } -static uint32_t rebuild_hflags_common_32(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx, uint32_t flags) +static CPUARMTBFlags rebuild_hflags_common_32(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx, + CPUARMTBFlags flags) { bool sctlr_b = arm_sctlr_b(env); if (sctlr_b) { - flags = FIELD_DP32(flags, TBFLAG_A32, SCTLR_B, 1); + DP_TBFLAG_A32(flags, SCTLR__B, 1); } if (arm_cpu_data_is_big_endian_a32(env, sctlr_b)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1); + DP_TBFLAG_ANY(flags, BE_DATA, 1); } - flags = FIELD_DP32(flags, TBFLAG_A32, NS, !access_secure_reg(env)); + DP_TBFLAG_A32(flags, NS, !access_secure_reg(env)); return rebuild_hflags_common(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx) +static CPUARMTBFlags rebuild_hflags_m32(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx) { - uint32_t flags = 0; + CPUARMTBFlags flags = {}; + uint32_t ccr = env->v7m.ccr[env->v7m.secure]; + + /* Without HaveMainExt, CCR.UNALIGN_TRP is RES1. */ + if (ccr & R_V7M_CCR_UNALIGN_TRP_MASK) { + DP_TBFLAG_ANY(flags, ALIGN_MEM, 1); + } if (arm_v7m_is_handler_mode(env)) { - flags = FIELD_DP32(flags, TBFLAG_M32, HANDLER, 1); + DP_TBFLAG_M32(flags, HANDLER, 1); } /* @@ -13029,56 +13036,60 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, */ if (arm_feature(env, ARM_FEATURE_V8) && !((mmu_idx & ARM_MMU_IDX_M_NEGPRI) && - (env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKOFHFNMIGN_MASK))) { - flags = FIELD_DP32(flags, TBFLAG_M32, STACKCHECK, 1); + (ccr & R_V7M_CCR_STKOFHFNMIGN_MASK))) { + DP_TBFLAG_M32(flags, STACKCHECK, 1); } return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_aprofile(CPUARMState *env) +static CPUARMTBFlags rebuild_hflags_aprofile(CPUARMState *env) { - int flags = 0; + CPUARMTBFlags flags = {}; - flags = FIELD_DP32(flags, TBFLAG_ANY, DEBUG_TARGET_EL, - arm_debug_target_el(env)); + DP_TBFLAG_ANY(flags, DEBUG_TARGET_EL, arm_debug_target_el(env)); return flags; } -static uint32_t rebuild_hflags_a32(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx) +static CPUARMTBFlags rebuild_hflags_a32(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx) { - uint32_t flags = rebuild_hflags_aprofile(env); + CPUARMTBFlags flags = rebuild_hflags_aprofile(env); + int el = arm_current_el(env); + + if (arm_sctlr(env, el) & SCTLR_A) { + DP_TBFLAG_ANY(flags, ALIGN_MEM, 1); + } if (arm_el_is_aa64(env, 1)) { - flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1); + DP_TBFLAG_A32(flags, VFPEN, 1); } - if (arm_current_el(env) < 2 && env->cp15.hstr_el2 && + if (el < 2 && env->cp15.hstr_el2 && (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { - flags = FIELD_DP32(flags, TBFLAG_A32, HSTR_ACTIVE, 1); + DP_TBFLAG_A32(flags, HSTR_ACTIVE, 1); } return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, - ARMMMUIdx mmu_idx) +static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, + ARMMMUIdx mmu_idx) { - uint32_t flags = rebuild_hflags_aprofile(env); + CPUARMTBFlags flags = rebuild_hflags_aprofile(env); ARMMMUIdx stage1 = stage_1_mmu_idx(mmu_idx); uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; uint64_t sctlr; int tbii, tbid; - flags = FIELD_DP32(flags, TBFLAG_ANY, AARCH64_STATE, 1); + DP_TBFLAG_ANY(flags, AARCH64_STATE, 1); /* Get control bits for tagged addresses. */ tbid = aa64_va_parameter_tbi(tcr, mmu_idx); tbii = tbid & ~aa64_va_parameter_tbid(tcr, mmu_idx); - flags = FIELD_DP32(flags, TBFLAG_A64, TBII, tbii); - flags = FIELD_DP32(flags, TBFLAG_A64, TBID, tbid); + DP_TBFLAG_A64(flags, TBII, tbii); + DP_TBFLAG_A64(flags, TBID, tbid); if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { int sve_el = sve_exception_el(env, el); @@ -13093,14 +13104,18 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, } else { zcr_len = sve_zcr_len_for_el(env, el); } - flags = FIELD_DP32(flags, TBFLAG_A64, SVEEXC_EL, sve_el); - flags = FIELD_DP32(flags, TBFLAG_A64, ZCR_LEN, zcr_len); + DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); + DP_TBFLAG_A64(flags, ZCR_LEN, zcr_len); } sctlr = regime_sctlr(env, stage1); + if (sctlr & SCTLR_A) { + DP_TBFLAG_ANY(flags, ALIGN_MEM, 1); + } + if (arm_cpu_data_is_big_endian_a64(el, sctlr)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1); + DP_TBFLAG_ANY(flags, BE_DATA, 1); } if (cpu_isar_feature(aa64_pauth, env_archcpu(env))) { @@ -13111,14 +13126,14 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, * The decision of which action to take is left to a helper. */ if (sctlr & (SCTLR_EnIA | SCTLR_EnIB | SCTLR_EnDA | SCTLR_EnDB)) { - flags = FIELD_DP32(flags, TBFLAG_A64, PAUTH_ACTIVE, 1); + DP_TBFLAG_A64(flags, PAUTH_ACTIVE, 1); } } if (cpu_isar_feature(aa64_bti, env_archcpu(env))) { /* Note that SCTLR_EL[23].BT == SCTLR_BT1. */ if (sctlr & (el == 0 ? SCTLR_BT0 : SCTLR_BT1)) { - flags = FIELD_DP32(flags, TBFLAG_A64, BT, 1); + DP_TBFLAG_A64(flags, BT, 1); } } @@ -13130,7 +13145,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, case ARMMMUIdx_SE10_1: case ARMMMUIdx_SE10_1_PAN: /* TODO: ARMv8.3-NV */ - flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + DP_TBFLAG_A64(flags, UNPRIV, 1); break; case ARMMMUIdx_E20_2: case ARMMMUIdx_E20_2_PAN: @@ -13141,7 +13156,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, * gated by HCR_EL2.<E2H,TGE> == '11', and so is LDTR. */ if (env->cp15.hcr_el2 & HCR_TGE) { - flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + DP_TBFLAG_A64(flags, UNPRIV, 1); } break; default: @@ -13159,30 +13174,29 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, * 4) If no Allocation Tag Access, then all accesses are Unchecked. */ if (allocation_tag_access_enabled(env, el, sctlr)) { - flags = FIELD_DP32(flags, TBFLAG_A64, ATA, 1); + DP_TBFLAG_A64(flags, ATA, 1); if (tbid && !(env->pstate & PSTATE_TCO) && (sctlr & (el == 0 ? SCTLR_TCF0 : SCTLR_TCF))) { - flags = FIELD_DP32(flags, TBFLAG_A64, MTE_ACTIVE, 1); + DP_TBFLAG_A64(flags, MTE_ACTIVE, 1); } } /* And again for unprivileged accesses, if required. */ - if (FIELD_EX32(flags, TBFLAG_A64, UNPRIV) + if (EX_TBFLAG_A64(flags, UNPRIV) && tbid && !(env->pstate & PSTATE_TCO) && (sctlr & SCTLR_TCF0) && allocation_tag_access_enabled(env, 0, sctlr)) { - flags = FIELD_DP32(flags, TBFLAG_A64, MTE0_ACTIVE, 1); + DP_TBFLAG_A64(flags, MTE0_ACTIVE, 1); } /* Cache TCMA as well as TBI. */ - flags = FIELD_DP32(flags, TBFLAG_A64, TCMA, - aa64_va_parameter_tcma(tcr, mmu_idx)); + DP_TBFLAG_A64(flags, TCMA, aa64_va_parameter_tcma(tcr, mmu_idx)); } return rebuild_hflags_common(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_internal(CPUARMState *env) +static CPUARMTBFlags rebuild_hflags_internal(CPUARMState *env) { int el = arm_current_el(env); int fp_el = fp_exception_el(env, el); @@ -13211,6 +13225,7 @@ void HELPER(rebuild_hflags_m32_newel)(CPUARMState *env) int el = arm_current_el(env); int fp_el = fp_exception_el(env, el); ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el); + env->hflags = rebuild_hflags_m32(env, fp_el, mmu_idx); } @@ -13253,12 +13268,14 @@ void HELPER(rebuild_hflags_a64)(CPUARMState *env, int el) static inline void assert_hflags_rebuild_correctly(CPUARMState *env) { #ifdef CONFIG_DEBUG_TCG - uint32_t env_flags_current = env->hflags; - uint32_t env_flags_rebuilt = rebuild_hflags_internal(env); - - if (unlikely(env_flags_current != env_flags_rebuilt)) { - fprintf(stderr, "TCG hflags mismatch (current:0x%08x rebuilt:0x%08x)\n", - env_flags_current, env_flags_rebuilt); + CPUARMTBFlags c = env->hflags; + CPUARMTBFlags r = rebuild_hflags_internal(env); + + if (unlikely(c.flags != r.flags || c.flags2 != r.flags2)) { + fprintf(stderr, "TCG hflags mismatch " + "(current:(0x%08x,0x" TARGET_FMT_lx ")" + " rebuilt:(0x%08x,0x" TARGET_FMT_lx ")\n", + c.flags, c.flags2, r.flags, r.flags2); abort(); } #endif @@ -13267,15 +13284,15 @@ static inline void assert_hflags_rebuild_correctly(CPUARMState *env) void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, target_ulong *cs_base, uint32_t *pflags) { - uint32_t flags = env->hflags; + CPUARMTBFlags flags; - *cs_base = 0; assert_hflags_rebuild_correctly(env); + flags = env->hflags; - if (FIELD_EX32(flags, TBFLAG_ANY, AARCH64_STATE)) { + if (EX_TBFLAG_ANY(flags, AARCH64_STATE)) { *pc = env->pc; if (cpu_isar_feature(aa64_bti, env_archcpu(env))) { - flags = FIELD_DP32(flags, TBFLAG_A64, BTYPE, env->btype); + DP_TBFLAG_A64(flags, BTYPE, env->btype); } } else { *pc = env->regs[15]; @@ -13284,7 +13301,7 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, if (arm_feature(env, ARM_FEATURE_M_SECURITY) && FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S) != env->v7m.secure) { - flags = FIELD_DP32(flags, TBFLAG_M32, FPCCR_S_WRONG, 1); + DP_TBFLAG_M32(flags, FPCCR_S_WRONG, 1); } if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) && @@ -13296,12 +13313,12 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * active FP context; we must create a new FP context before * executing any FP insn. */ - flags = FIELD_DP32(flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 1); + DP_TBFLAG_M32(flags, NEW_FP_CTXT_NEEDED, 1); } bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK; if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) { - flags = FIELD_DP32(flags, TBFLAG_M32, LSPACT, 1); + DP_TBFLAG_M32(flags, LSPACT, 1); } } else { /* @@ -13309,21 +13326,18 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * Note that VECLEN+VECSTRIDE are RES0 for M-profile. */ if (arm_feature(env, ARM_FEATURE_XSCALE)) { - flags = FIELD_DP32(flags, TBFLAG_A32, - XSCALE_CPAR, env->cp15.c15_cpar); + DP_TBFLAG_A32(flags, XSCALE_CPAR, env->cp15.c15_cpar); } else { - flags = FIELD_DP32(flags, TBFLAG_A32, VECLEN, - env->vfp.vec_len); - flags = FIELD_DP32(flags, TBFLAG_A32, VECSTRIDE, - env->vfp.vec_stride); + DP_TBFLAG_A32(flags, VECLEN, env->vfp.vec_len); + DP_TBFLAG_A32(flags, VECSTRIDE, env->vfp.vec_stride); } if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30)) { - flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1); + DP_TBFLAG_A32(flags, VFPEN, 1); } } - flags = FIELD_DP32(flags, TBFLAG_AM32, THUMB, env->thumb); - flags = FIELD_DP32(flags, TBFLAG_AM32, CONDEXEC, env->condexec_bits); + DP_TBFLAG_AM32(flags, THUMB, env->thumb); + DP_TBFLAG_AM32(flags, CONDEXEC, env->condexec_bits); } /* @@ -13333,14 +13347,14 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * 0 x Inactive (the TB flag for SS is always 0) * 1 0 Active-pending * 1 1 Active-not-pending - * SS_ACTIVE is set in hflags; PSTATE_SS is computed every TB. + * SS_ACTIVE is set in hflags; PSTATE__SS is computed every TB. */ - if (FIELD_EX32(flags, TBFLAG_ANY, SS_ACTIVE) && - (env->pstate & PSTATE_SS)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, PSTATE_SS, 1); + if (EX_TBFLAG_ANY(flags, SS_ACTIVE) && (env->pstate & PSTATE_SS)) { + DP_TBFLAG_ANY(flags, PSTATE__SS, 1); } - *pflags = flags; + *pflags = flags.flags; + *cs_base = flags.flags2; } #ifdef TARGET_AARCH64 diff --git a/target/arm/internals.h b/target/arm/internals.h index f11bd32696..886db56b58 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -26,6 +26,7 @@ #define TARGET_ARM_INTERNALS_H #include "hw/registerfields.h" +#include "tcg/tcg-gvec-desc.h" #include "syndrome.h" /* register banks for CPU modes */ @@ -1142,14 +1143,10 @@ FIELD(MTEDESC, MIDX, 0, 4) FIELD(MTEDESC, TBI, 4, 2) FIELD(MTEDESC, TCMA, 6, 2) FIELD(MTEDESC, WRITE, 8, 1) -FIELD(MTEDESC, ESIZE, 9, 5) -FIELD(MTEDESC, TSIZE, 14, 10) /* mte_checkN only */ - -bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr); -uint64_t mte_check1(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra); -uint64_t mte_checkN(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra); +FIELD(MTEDESC, SIZEM1, 9, SIMD_DATA_BITS - 9) /* size - 1 */ + +bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr); +uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr, uintptr_t ra); static inline int allocation_tag_from_addr(uint64_t ptr) { diff --git a/target/arm/mte_helper.c b/target/arm/mte_helper.c index 8be17e1b70..a6fccc6e69 100644 --- a/target/arm/mte_helper.c +++ b/target/arm/mte_helper.c @@ -121,7 +121,7 @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, * exception for inaccessible pages, and resolves the virtual address * into the softmmu tlb. * - * When RA == 0, this is for mte_probe1. The page is expected to be + * When RA == 0, this is for mte_probe. The page is expected to be * valid. Indicate to probe_access_flags no-fault, then assert that * we received a valid page. */ @@ -617,80 +617,6 @@ static void mte_check_fail(CPUARMState *env, uint32_t desc, } } -/* - * Perform an MTE checked access for a single logical or atomic access. - */ -static bool mte_probe1_int(CPUARMState *env, uint32_t desc, uint64_t ptr, - uintptr_t ra, int bit55) -{ - int mem_tag, mmu_idx, ptr_tag, size; - MMUAccessType type; - uint8_t *mem; - - ptr_tag = allocation_tag_from_addr(ptr); - - if (tcma_check(desc, bit55, ptr_tag)) { - return true; - } - - mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); - type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD; - size = FIELD_EX32(desc, MTEDESC, ESIZE); - - mem = allocation_tag_mem(env, mmu_idx, ptr, type, size, - MMU_DATA_LOAD, 1, ra); - if (!mem) { - return true; - } - - mem_tag = load_tag1(ptr, mem); - return ptr_tag == mem_tag; -} - -/* - * No-fault version of mte_check1, to be used by SVE for MemSingleNF. - * Returns false if the access is Checked and the check failed. This - * is only intended to probe the tag -- the validity of the page must - * be checked beforehand. - */ -bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr) -{ - int bit55 = extract64(ptr, 55, 1); - - /* If TBI is disabled, the access is unchecked. */ - if (unlikely(!tbi_check(desc, bit55))) { - return true; - } - - return mte_probe1_int(env, desc, ptr, 0, bit55); -} - -uint64_t mte_check1(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra) -{ - int bit55 = extract64(ptr, 55, 1); - - /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */ - if (unlikely(!tbi_check(desc, bit55))) { - return ptr; - } - - if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) { - mte_check_fail(env, desc, ptr, ra); - } - - return useronly_clean_ptr(ptr); -} - -uint64_t HELPER(mte_check1)(CPUARMState *env, uint32_t desc, uint64_t ptr) -{ - return mte_check1(env, desc, ptr, GETPC()); -} - -/* - * Perform an MTE checked access for multiple logical accesses. - */ - /** * checkN: * @tag: tag memory to test @@ -753,59 +679,70 @@ static int checkN(uint8_t *mem, int odd, int cmp, int count) return n; } -uint64_t mte_checkN(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra) +/** + * mte_probe_int() - helper for mte_probe and mte_check + * @env: CPU environment + * @desc: MTEDESC descriptor + * @ptr: virtual address of the base of the access + * @fault: return virtual address of the first check failure + * + * Internal routine for both mte_probe and mte_check. + * Return zero on failure, filling in *fault. + * Return negative on trivial success for tbi disabled. + * Return positive on success with tbi enabled. + */ +static int mte_probe_int(CPUARMState *env, uint32_t desc, uint64_t ptr, + uintptr_t ra, uint64_t *fault) { int mmu_idx, ptr_tag, bit55; - uint64_t ptr_last, ptr_end, prev_page, next_page; - uint64_t tag_first, tag_end; - uint64_t tag_byte_first, tag_byte_end; - uint32_t esize, total, tag_count, tag_size, n, c; + uint64_t ptr_last, prev_page, next_page; + uint64_t tag_first, tag_last; + uint64_t tag_byte_first, tag_byte_last; + uint32_t sizem1, tag_count, tag_size, n, c; uint8_t *mem1, *mem2; MMUAccessType type; bit55 = extract64(ptr, 55, 1); + *fault = ptr; /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */ if (unlikely(!tbi_check(desc, bit55))) { - return ptr; + return -1; } ptr_tag = allocation_tag_from_addr(ptr); if (tcma_check(desc, bit55, ptr_tag)) { - goto done; + return 1; } mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD; - esize = FIELD_EX32(desc, MTEDESC, ESIZE); - total = FIELD_EX32(desc, MTEDESC, TSIZE); + sizem1 = FIELD_EX32(desc, MTEDESC, SIZEM1); - /* Find the addr of the end of the access, and of the last element. */ - ptr_end = ptr + total; - ptr_last = ptr_end - esize; + /* Find the addr of the end of the access */ + ptr_last = ptr + sizem1; /* Round the bounds to the tag granule, and compute the number of tags. */ tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE); - tag_end = QEMU_ALIGN_UP(ptr_last, TAG_GRANULE); - tag_count = (tag_end - tag_first) / TAG_GRANULE; + tag_last = QEMU_ALIGN_DOWN(ptr_last, TAG_GRANULE); + tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1; /* Round the bounds to twice the tag granule, and compute the bytes. */ tag_byte_first = QEMU_ALIGN_DOWN(ptr, 2 * TAG_GRANULE); - tag_byte_end = QEMU_ALIGN_UP(ptr_last, 2 * TAG_GRANULE); + tag_byte_last = QEMU_ALIGN_DOWN(ptr_last, 2 * TAG_GRANULE); /* Locate the page boundaries. */ prev_page = ptr & TARGET_PAGE_MASK; next_page = prev_page + TARGET_PAGE_SIZE; - if (likely(tag_end - prev_page <= TARGET_PAGE_SIZE)) { + if (likely(tag_last - prev_page <= TARGET_PAGE_SIZE)) { /* Memory access stays on one page. */ - tag_size = (tag_byte_end - tag_byte_first) / (2 * TAG_GRANULE); - mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, total, + tag_size = ((tag_byte_last - tag_byte_first) / (2 * TAG_GRANULE)) + 1; + mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, sizem1 + 1, MMU_DATA_LOAD, tag_size, ra); if (!mem1) { - goto done; + return 1; } /* Perform all of the comparisons. */ n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count); @@ -815,9 +752,9 @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc, mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr, MMU_DATA_LOAD, tag_size, ra); - tag_size = (tag_byte_end - next_page) / (2 * TAG_GRANULE); + tag_size = ((tag_byte_last - next_page) / (2 * TAG_GRANULE)) + 1; mem2 = allocation_tag_mem(env, mmu_idx, next_page, type, - ptr_end - next_page, + ptr_last - next_page + 1, MMU_DATA_LOAD, tag_size, ra); /* @@ -831,31 +768,57 @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc, } if (n == c) { if (!mem2) { - goto done; + return 1; } n += checkN(mem2, 0, ptr_tag, tag_count - c); } } + if (likely(n == tag_count)) { + return 1; + } + /* - * If we failed, we know which granule. Compute the element that - * is first in that granule, and signal failure on that element. + * If we failed, we know which granule. For the first granule, the + * failure address is @ptr, the first byte accessed. Otherwise the + * failure address is the first byte of the nth granule. */ - if (unlikely(n < tag_count)) { - uint64_t fail_ofs; - - fail_ofs = tag_first + n * TAG_GRANULE - ptr; - fail_ofs = ROUND_UP(fail_ofs, esize); - mte_check_fail(env, desc, ptr + fail_ofs, ra); + if (n > 0) { + *fault = tag_first + n * TAG_GRANULE; } + return 0; +} - done: +uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr, uintptr_t ra) +{ + uint64_t fault; + int ret = mte_probe_int(env, desc, ptr, ra, &fault); + + if (unlikely(ret == 0)) { + mte_check_fail(env, desc, fault, ra); + } else if (ret < 0) { + return ptr; + } return useronly_clean_ptr(ptr); } -uint64_t HELPER(mte_checkN)(CPUARMState *env, uint32_t desc, uint64_t ptr) +uint64_t HELPER(mte_check)(CPUARMState *env, uint32_t desc, uint64_t ptr) { - return mte_checkN(env, desc, ptr, GETPC()); + return mte_check(env, desc, ptr, GETPC()); +} + +/* + * No-fault version of mte_check, to be used by SVE for MemSingleNF. + * Returns false if the access is Checked and the check failed. This + * is only intended to probe the tag -- the validity of the page must + * be checked beforehand. + */ +bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr) +{ + uint64_t fault; + int ret = mte_probe_int(env, desc, ptr, 0, &fault); + + return ret != 0; } /* diff --git a/target/arm/neon-ls.decode b/target/arm/neon-ls.decode index c17f5019e3..0a2a0e15db 100644 --- a/target/arm/neon-ls.decode +++ b/target/arm/neon-ls.decode @@ -46,7 +46,7 @@ VLD_all_lanes 1111 0100 1 . 1 0 rn:4 .... 11 n:2 size:2 t:1 a:1 rm:4 \ VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 00 n:2 reg_idx:3 align:1 rm:4 \ vd=%vd_dp size=0 stride=1 -VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 01 n:2 reg_idx:2 align:2 rm:4 \ +VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 01 n:2 reg_idx:2 . align:1 rm:4 \ vd=%vd_dp size=1 stride=%imm1_5_p1 -VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 10 n:2 reg_idx:1 align:3 rm:4 \ +VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 10 n:2 reg_idx:1 . align:2 rm:4 \ vd=%vd_dp size=2 stride=%imm1_6_p1 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index fd6c58f96a..c068dfa0d5 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -4382,13 +4382,9 @@ static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, #endif } -typedef uint64_t mte_check_fn(CPUARMState *, uint32_t, uint64_t, uintptr_t); - -static inline QEMU_ALWAYS_INLINE -void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, int esize, - int msize, uint32_t mtedesc, uintptr_t ra, - mte_check_fn *check) +static void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, int esize, + int msize, uint32_t mtedesc, uintptr_t ra) { intptr_t mem_off, reg_off, reg_last; @@ -4405,7 +4401,7 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, uint64_t pg = vg[reg_off >> 6]; do { if ((pg >> (reg_off & 63)) & 1) { - check(env, mtedesc, addr, ra); + mte_check(env, mtedesc, addr, ra); } reg_off += esize; mem_off += msize; @@ -4422,7 +4418,7 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, uint64_t pg = vg[reg_off >> 6]; do { if ((pg >> (reg_off & 63)) & 1) { - check(env, mtedesc, addr, ra); + mte_check(env, mtedesc, addr, ra); } reg_off += esize; mem_off += msize; @@ -4431,30 +4427,6 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, } } -typedef void sve_cont_ldst_mte_check_fn(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, uint32_t mtedesc, - uintptr_t ra); - -static void sve_cont_ldst_mte_check1(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, uint32_t mtedesc, - uintptr_t ra) -{ - sve_cont_ldst_mte_check_int(info, env, vg, addr, esize, msize, - mtedesc, ra, mte_check1); -} - -static void sve_cont_ldst_mte_checkN(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, uint32_t mtedesc, - uintptr_t ra) -{ - sve_cont_ldst_mte_check_int(info, env, vg, addr, esize, msize, - mtedesc, ra, mte_checkN); -} - - /* * Common helper for all contiguous 1,2,3,4-register predicated stores. */ @@ -4463,8 +4435,7 @@ void sve_ldN_r(CPUARMState *env, uint64_t *vg, const target_ulong addr, uint32_t desc, const uintptr_t retaddr, const int esz, const int msz, const int N, uint32_t mtedesc, sve_ldst1_host_fn *host_fn, - sve_ldst1_tlb_fn *tlb_fn, - sve_cont_ldst_mte_check_fn *mte_check_fn) + sve_ldst1_tlb_fn *tlb_fn) { const unsigned rd = simd_data(desc); const intptr_t reg_max = simd_oprsz(desc); @@ -4493,9 +4464,9 @@ void sve_ldN_r(CPUARMState *env, uint64_t *vg, const target_ulong addr, * Handle mte checks for all active elements. * Since TBI must be set for MTE, !mtedesc => !mte_active. */ - if (mte_check_fn && mtedesc) { - mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz, - mtedesc, retaddr); + if (mtedesc) { + sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz, + mtedesc, retaddr); } flags = info.page[0].flags | info.page[1].flags; @@ -4621,8 +4592,7 @@ void sve_ldN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr, mtedesc = 0; } - sve_ldN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn, - N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN); + sve_ldN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn); } #define DO_LD1_1(NAME, ESZ) \ @@ -4630,7 +4600,7 @@ void HELPER(sve_##NAME##_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, 1, 0, \ - sve_##NAME##_host, sve_##NAME##_tlb, NULL); \ + sve_##NAME##_host, sve_##NAME##_tlb); \ } \ void HELPER(sve_##NAME##_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -4644,22 +4614,22 @@ void HELPER(sve_##NAME##_le_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \ - sve_##NAME##_le_host, sve_##NAME##_le_tlb, NULL); \ + sve_##NAME##_le_host, sve_##NAME##_le_tlb); \ } \ void HELPER(sve_##NAME##_be_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \ - sve_##NAME##_be_host, sve_##NAME##_be_tlb, NULL); \ + sve_##NAME##_be_host, sve_##NAME##_be_tlb); \ } \ void HELPER(sve_##NAME##_le_r_mte)(CPUARMState *env, void *vg, \ - target_ulong addr, uint32_t desc) \ + target_ulong addr, uint32_t desc) \ { \ sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, \ sve_##NAME##_le_host, sve_##NAME##_le_tlb); \ } \ void HELPER(sve_##NAME##_be_r_mte)(CPUARMState *env, void *vg, \ - target_ulong addr, uint32_t desc) \ + target_ulong addr, uint32_t desc) \ { \ sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, \ sve_##NAME##_be_host, sve_##NAME##_be_tlb); \ @@ -4693,7 +4663,7 @@ void HELPER(sve_ld##N##bb_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), MO_8, MO_8, N, 0, \ - sve_ld1bb_host, sve_ld1bb_tlb, NULL); \ + sve_ld1bb_host, sve_ld1bb_tlb); \ } \ void HELPER(sve_ld##N##bb_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -4707,13 +4677,13 @@ void HELPER(sve_ld##N##SUFF##_le_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \ - sve_ld1##SUFF##_le_host, sve_ld1##SUFF##_le_tlb, NULL); \ + sve_ld1##SUFF##_le_host, sve_ld1##SUFF##_le_tlb); \ } \ void HELPER(sve_ld##N##SUFF##_be_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \ - sve_ld1##SUFF##_be_host, sve_ld1##SUFF##_be_tlb, NULL); \ + sve_ld1##SUFF##_be_host, sve_ld1##SUFF##_be_tlb); \ } \ void HELPER(sve_ld##N##SUFF##_le_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -4826,7 +4796,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, if (fault == FAULT_FIRST) { /* Trapping mte check for the first-fault element. */ if (mtedesc) { - mte_check1(env, mtedesc, addr + mem_off, retaddr); + mte_check(env, mtedesc, addr + mem_off, retaddr); } /* @@ -4869,7 +4839,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, /* Watchpoint hit, see below. */ goto do_fault; } - if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) { + if (mtedesc && !mte_probe(env, mtedesc, addr + mem_off)) { goto do_fault; } /* @@ -4919,7 +4889,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, & BP_MEM_READ)) { goto do_fault; } - if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) { + if (mtedesc && !mte_probe(env, mtedesc, addr + mem_off)) { goto do_fault; } host_fn(vd, reg_off, host + mem_off); @@ -5090,8 +5060,7 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc, const uintptr_t retaddr, const int esz, const int msz, const int N, uint32_t mtedesc, sve_ldst1_host_fn *host_fn, - sve_ldst1_tlb_fn *tlb_fn, - sve_cont_ldst_mte_check_fn *mte_check_fn) + sve_ldst1_tlb_fn *tlb_fn) { const unsigned rd = simd_data(desc); const intptr_t reg_max = simd_oprsz(desc); @@ -5117,9 +5086,9 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, * Handle mte checks for all active elements. * Since TBI must be set for MTE, !mtedesc => !mte_active. */ - if (mte_check_fn && mtedesc) { - mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz, - mtedesc, retaddr); + if (mtedesc) { + sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz, + mtedesc, retaddr); } flags = info.page[0].flags | info.page[1].flags; @@ -5233,8 +5202,7 @@ void sve_stN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr, mtedesc = 0; } - sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn, - N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN); + sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn); } #define DO_STN_1(N, NAME, ESZ) \ @@ -5242,7 +5210,7 @@ void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, 0, \ - sve_st1##NAME##_host, sve_st1##NAME##_tlb, NULL); \ + sve_st1##NAME##_host, sve_st1##NAME##_tlb); \ } \ void HELPER(sve_st##N##NAME##_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -5256,13 +5224,13 @@ void HELPER(sve_st##N##NAME##_le_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \ - sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb, NULL); \ + sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb); \ } \ void HELPER(sve_st##N##NAME##_be_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \ - sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb, NULL); \ + sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb); \ } \ void HELPER(sve_st##N##NAME##_le_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -5373,7 +5341,7 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, info.attrs, BP_MEM_READ, retaddr); } if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } host_fn(&scratch, reg_off, info.host); } else { @@ -5386,7 +5354,7 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, BP_MEM_READ, retaddr); } if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } tlb_fn(env, &scratch, reg_off, addr, retaddr); } @@ -5552,7 +5520,7 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, */ addr = base + (off_fn(vm, reg_off) << scale); if (mtedesc) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } tlb_fn(env, vd, reg_off, addr, retaddr); @@ -5588,7 +5556,7 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, } if (mtedesc && arm_tlb_mte_tagged(&info.attrs) && - !mte_probe1(env, mtedesc, addr)) { + !mte_probe(env, mtedesc, addr)) { goto fault; } @@ -5773,7 +5741,7 @@ void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, } if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } } i += 1; diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index 0b42e53500..95897e63af 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -272,11 +272,11 @@ static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr, desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_size); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << log2_size) - 1); tcg_desc = tcg_const_i32(desc); ret = new_tmp_a64(s); - gen_helper_mte_check1(ret, cpu_env, tcg_desc, addr); + gen_helper_mte_check(ret, cpu_env, tcg_desc, addr); tcg_temp_free_i32(tcg_desc); return ret; @@ -295,9 +295,9 @@ TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write, * For MTE, check multiple logical sequential accesses. */ TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write, - bool tag_checked, int log2_esize, int total_size) + bool tag_checked, int size) { - if (tag_checked && s->mte_active[0] && total_size != (1 << log2_esize)) { + if (tag_checked && s->mte_active[0]) { TCGv_i32 tcg_desc; TCGv_i64 ret; int desc = 0; @@ -306,17 +306,16 @@ TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write, desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_esize); - desc = FIELD_DP32(desc, MTEDESC, TSIZE, total_size); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, size - 1); tcg_desc = tcg_const_i32(desc); ret = new_tmp_a64(s); - gen_helper_mte_checkN(ret, cpu_env, tcg_desc, addr); + gen_helper_mte_check(ret, cpu_env, tcg_desc, addr); tcg_temp_free_i32(tcg_desc); return ret; } - return gen_mte_check1(s, addr, is_write, tag_checked, log2_esize); + return clean_data_tbi(s, addr); } typedef struct DisasCompare64 { @@ -887,19 +886,19 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) * Store from GPR register to memory. */ static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source, - TCGv_i64 tcg_addr, int size, int memidx, + TCGv_i64 tcg_addr, MemOp memop, int memidx, bool iss_valid, unsigned int iss_srt, bool iss_sf, bool iss_ar) { - g_assert(size <= 3); - tcg_gen_qemu_st_i64(source, tcg_addr, memidx, s->be_data + size); + memop = finalize_memop(s, memop); + tcg_gen_qemu_st_i64(source, tcg_addr, memidx, memop); if (iss_valid) { uint32_t syn; syn = syn_data_abort_with_iss(0, - size, + (memop & MO_SIZE), false, iss_srt, iss_sf, @@ -910,37 +909,28 @@ static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source, } static void do_gpr_st(DisasContext *s, TCGv_i64 source, - TCGv_i64 tcg_addr, int size, + TCGv_i64 tcg_addr, MemOp memop, bool iss_valid, unsigned int iss_srt, bool iss_sf, bool iss_ar) { - do_gpr_st_memidx(s, source, tcg_addr, size, get_mem_index(s), + do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s), iss_valid, iss_srt, iss_sf, iss_ar); } /* * Load from memory to GPR register */ -static void do_gpr_ld_memidx(DisasContext *s, - TCGv_i64 dest, TCGv_i64 tcg_addr, - int size, bool is_signed, - bool extend, int memidx, +static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr, + MemOp memop, bool extend, int memidx, bool iss_valid, unsigned int iss_srt, bool iss_sf, bool iss_ar) { - MemOp memop = s->be_data + size; - - g_assert(size <= 3); - - if (is_signed) { - memop += MO_SIGN; - } - + memop = finalize_memop(s, memop); tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop); - if (extend && is_signed) { - g_assert(size < 3); + if (extend && (memop & MO_SIGN)) { + g_assert((memop & MO_SIZE) <= MO_32); tcg_gen_ext32u_i64(dest, dest); } @@ -948,8 +938,8 @@ static void do_gpr_ld_memidx(DisasContext *s, uint32_t syn; syn = syn_data_abort_with_iss(0, - size, - is_signed, + (memop & MO_SIZE), + (memop & MO_SIGN) != 0, iss_srt, iss_sf, iss_ar, @@ -958,14 +948,12 @@ static void do_gpr_ld_memidx(DisasContext *s, } } -static void do_gpr_ld(DisasContext *s, - TCGv_i64 dest, TCGv_i64 tcg_addr, - int size, bool is_signed, bool extend, +static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr, + MemOp memop, bool extend, bool iss_valid, unsigned int iss_srt, bool iss_sf, bool iss_ar) { - do_gpr_ld_memidx(s, dest, tcg_addr, size, is_signed, extend, - get_mem_index(s), + do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s), iss_valid, iss_srt, iss_sf, iss_ar); } @@ -975,25 +963,33 @@ static void do_gpr_ld(DisasContext *s, static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, int size) { /* This writes the bottom N bits of a 128 bit wide vector to memory */ - TCGv_i64 tmp = tcg_temp_new_i64(); - tcg_gen_ld_i64(tmp, cpu_env, fp_reg_offset(s, srcidx, MO_64)); + TCGv_i64 tmplo = tcg_temp_new_i64(); + MemOp mop; + + tcg_gen_ld_i64(tmplo, cpu_env, fp_reg_offset(s, srcidx, MO_64)); + if (size < 4) { - tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), - s->be_data + size); + mop = finalize_memop(s, size); + tcg_gen_qemu_st_i64(tmplo, tcg_addr, get_mem_index(s), mop); } else { bool be = s->be_data == MO_BE; TCGv_i64 tcg_hiaddr = tcg_temp_new_i64(); + TCGv_i64 tmphi = tcg_temp_new_i64(); + tcg_gen_ld_i64(tmphi, cpu_env, fp_reg_hi_offset(s, srcidx)); + + mop = s->be_data | MO_Q; + tcg_gen_qemu_st_i64(be ? tmphi : tmplo, tcg_addr, get_mem_index(s), + mop | (s->align_mem ? MO_ALIGN_16 : 0)); tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8); - tcg_gen_qemu_st_i64(tmp, be ? tcg_hiaddr : tcg_addr, get_mem_index(s), - s->be_data | MO_Q); - tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx)); - tcg_gen_qemu_st_i64(tmp, be ? tcg_addr : tcg_hiaddr, get_mem_index(s), - s->be_data | MO_Q); + tcg_gen_qemu_st_i64(be ? tmplo : tmphi, tcg_hiaddr, + get_mem_index(s), mop); + tcg_temp_free_i64(tcg_hiaddr); + tcg_temp_free_i64(tmphi); } - tcg_temp_free_i64(tmp); + tcg_temp_free_i64(tmplo); } /* @@ -1004,10 +1000,11 @@ static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, int size) /* This always zero-extends and writes to a full 128 bit wide vector */ TCGv_i64 tmplo = tcg_temp_new_i64(); TCGv_i64 tmphi = NULL; + MemOp mop; if (size < 4) { - MemOp memop = s->be_data + size; - tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), memop); + mop = finalize_memop(s, size); + tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), mop); } else { bool be = s->be_data == MO_BE; TCGv_i64 tcg_hiaddr; @@ -1015,11 +1012,12 @@ static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, int size) tmphi = tcg_temp_new_i64(); tcg_hiaddr = tcg_temp_new_i64(); + mop = s->be_data | MO_Q; + tcg_gen_qemu_ld_i64(be ? tmphi : tmplo, tcg_addr, get_mem_index(s), + mop | (s->align_mem ? MO_ALIGN_16 : 0)); tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8); - tcg_gen_qemu_ld_i64(tmplo, be ? tcg_hiaddr : tcg_addr, get_mem_index(s), - s->be_data | MO_Q); - tcg_gen_qemu_ld_i64(tmphi, be ? tcg_addr : tcg_hiaddr, get_mem_index(s), - s->be_data | MO_Q); + tcg_gen_qemu_ld_i64(be ? tmplo : tmphi, tcg_hiaddr, + get_mem_index(s), mop); tcg_temp_free_i64(tcg_hiaddr); } @@ -1148,24 +1146,24 @@ static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src, /* Store from vector register to memory */ static void do_vec_st(DisasContext *s, int srcidx, int element, - TCGv_i64 tcg_addr, int size, MemOp endian) + TCGv_i64 tcg_addr, MemOp mop) { TCGv_i64 tcg_tmp = tcg_temp_new_i64(); - read_vec_element(s, tcg_tmp, srcidx, element, size); - tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), endian | size); + read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE); + tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop); tcg_temp_free_i64(tcg_tmp); } /* Load from memory to vector register */ static void do_vec_ld(DisasContext *s, int destidx, int element, - TCGv_i64 tcg_addr, int size, MemOp endian) + TCGv_i64 tcg_addr, MemOp mop) { TCGv_i64 tcg_tmp = tcg_temp_new_i64(); - tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), endian | size); - write_vec_element(s, tcg_tmp, destidx, element, size); + tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop); + write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE); tcg_temp_free_i64(tcg_tmp); } @@ -2701,7 +2699,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn) tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, size); - do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt, + /* TODO: ARMv8.4-LSE SCTLR.nAA */ + do_gpr_st(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, true, rt, disas_ldst_compute_iss_sf(size, false, 0), is_lasr); return; @@ -2718,8 +2717,9 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn) } clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), false, rn != 31, size); - do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, false, true, rt, - disas_ldst_compute_iss_sf(size, false, 0), is_lasr); + /* TODO: ARMv8.4-LSE SCTLR.nAA */ + do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, false, true, + rt, disas_ldst_compute_iss_sf(size, false, 0), is_lasr); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); return; @@ -2831,8 +2831,8 @@ static void disas_ld_lit(DisasContext *s, uint32_t insn) /* Only unsigned 32bit loads target 32bit registers. */ bool iss_sf = opc != 0; - do_gpr_ld(s, tcg_rt, clean_addr, size, is_signed, false, - true, rt, iss_sf, false); + do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + false, true, rt, iss_sf, false); } tcg_temp_free_i64(clean_addr); } @@ -2966,8 +2966,7 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn) } clean_addr = gen_mte_checkN(s, dirty_addr, !is_load, - (wback || rn != 31) && !set_tag, - size, 2 << size); + (wback || rn != 31) && !set_tag, 2 << size); if (is_vector) { if (is_load) { @@ -2991,11 +2990,11 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn) /* Do not modify tcg_rt before recognizing any exception * from the second load. */ - do_gpr_ld(s, tmp, clean_addr, size, is_signed, false, - false, 0, false, false); + do_gpr_ld(s, tmp, clean_addr, size + is_signed * MO_SIGN, + false, false, 0, false, false); tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size); - do_gpr_ld(s, tcg_rt2, clean_addr, size, is_signed, false, - false, 0, false, false); + do_gpr_ld(s, tcg_rt2, clean_addr, size + is_signed * MO_SIGN, + false, false, 0, false, false); tcg_gen_mov_i64(tcg_rt, tmp); tcg_temp_free_i64(tmp); @@ -3126,8 +3125,8 @@ static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn, do_gpr_st_memidx(s, tcg_rt, clean_addr, size, memidx, iss_valid, rt, iss_sf, false); } else { - do_gpr_ld_memidx(s, tcg_rt, clean_addr, size, - is_signed, is_extended, memidx, + do_gpr_ld_memidx(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + is_extended, memidx, iss_valid, rt, iss_sf, false); } } @@ -3231,9 +3230,8 @@ static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn, do_gpr_st(s, tcg_rt, clean_addr, size, true, rt, iss_sf, false); } else { - do_gpr_ld(s, tcg_rt, clean_addr, size, - is_signed, is_extended, - true, rt, iss_sf, false); + do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + is_extended, true, rt, iss_sf, false); } } } @@ -3316,8 +3314,8 @@ static void disas_ldst_reg_unsigned_imm(DisasContext *s, uint32_t insn, do_gpr_st(s, tcg_rt, clean_addr, size, true, rt, iss_sf, false); } else { - do_gpr_ld(s, tcg_rt, clean_addr, size, is_signed, is_extended, - true, rt, iss_sf, false); + do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + is_extended, true, rt, iss_sf, false); } } } @@ -3404,7 +3402,7 @@ static void disas_ldst_atomic(DisasContext *s, uint32_t insn, * full load-acquire (we only need "load-acquire processor consistent"), * but we choose to implement them as full LDAQ. */ - do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, false, + do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, true, rt, disas_ldst_compute_iss_sf(size, false, 0), true); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); return; @@ -3477,7 +3475,7 @@ static void disas_ldst_pac(DisasContext *s, uint32_t insn, is_wback || rn != 31, size); tcg_rt = cpu_reg(s, rt); - do_gpr_ld(s, tcg_rt, clean_addr, size, /* is_signed */ false, + do_gpr_ld(s, tcg_rt, clean_addr, size, /* extend */ false, /* iss_valid */ !is_wback, /* iss_srt */ rt, /* iss_sf */ true, /* iss_ar */ false); @@ -3509,15 +3507,18 @@ static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn) int size = extract32(insn, 30, 2); TCGv_i64 clean_addr, dirty_addr; bool is_store = false; - bool is_signed = false; bool extend = false; bool iss_sf; + MemOp mop; if (!dc_isar_feature(aa64_rcpc_8_4, s)) { unallocated_encoding(s); return; } + /* TODO: ARMv8.4-LSE SCTLR.nAA */ + mop = size | MO_ALIGN; + switch (opc) { case 0: /* STLURB */ is_store = true; @@ -3529,21 +3530,21 @@ static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn) unallocated_encoding(s); return; } - is_signed = true; + mop |= MO_SIGN; break; case 3: /* LDAPURS* 32-bit variant */ if (size > 1) { unallocated_encoding(s); return; } - is_signed = true; + mop |= MO_SIGN; extend = true; /* zero-extend 32->64 after signed load */ break; default: g_assert_not_reached(); } - iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc); + iss_sf = disas_ldst_compute_iss_sf(size, (mop & MO_SIGN) != 0, opc); if (rn == 31) { gen_check_sp_alignment(s); @@ -3556,14 +3557,14 @@ static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn) if (is_store) { /* Store-Release semantics */ tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); - do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt, iss_sf, true); + do_gpr_st(s, cpu_reg(s, rt), clean_addr, mop, true, rt, iss_sf, true); } else { /* * Load-AcquirePC semantics; we implement as the slightly more * restrictive Load-Acquire. */ - do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, is_signed, extend, - true, rt, iss_sf, true); + do_gpr_ld(s, cpu_reg(s, rt), clean_addr, mop, + extend, true, rt, iss_sf, true); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); } } @@ -3634,7 +3635,7 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) bool is_postidx = extract32(insn, 23, 1); bool is_q = extract32(insn, 30, 1); TCGv_i64 clean_addr, tcg_rn, tcg_ebytes; - MemOp endian = s->be_data; + MemOp endian, align, mop; int total; /* total bytes */ int elements; /* elements per vector */ @@ -3702,6 +3703,7 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) } /* For our purposes, bytes are always little-endian. */ + endian = s->be_data; if (size == 0) { endian = MO_LE; } @@ -3714,17 +3716,23 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) * promote consecutive little-endian elements below. */ clean_addr = gen_mte_checkN(s, tcg_rn, is_store, is_postidx || rn != 31, - size, total); + total); /* * Consecutive little-endian elements from a single register * can be promoted to a larger little-endian operation. */ + align = MO_ALIGN; if (selem == 1 && endian == MO_LE) { + align = pow2_align(size); size = 3; } - elements = (is_q ? 16 : 8) >> size; + if (!s->align_mem) { + align = 0; + } + mop = endian | size | align; + elements = (is_q ? 16 : 8) >> size; tcg_ebytes = tcg_const_i64(1 << size); for (r = 0; r < rpt; r++) { int e; @@ -3733,9 +3741,9 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) for (xs = 0; xs < selem; xs++) { int tt = (rt + r + xs) % 32; if (is_store) { - do_vec_st(s, tt, e, clean_addr, size, endian); + do_vec_st(s, tt, e, clean_addr, mop); } else { - do_vec_ld(s, tt, e, clean_addr, size, endian); + do_vec_ld(s, tt, e, clean_addr, mop); } tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes); } @@ -3807,6 +3815,7 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) int index = is_q << 3 | S << 2 | size; int xs, total; TCGv_i64 clean_addr, tcg_rn, tcg_ebytes; + MemOp mop; if (extract32(insn, 31, 1)) { unallocated_encoding(s); @@ -3867,7 +3876,8 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) tcg_rn = cpu_reg_sp(s, rn); clean_addr = gen_mte_checkN(s, tcg_rn, !is_load, is_postidx || rn != 31, - scale, total); + total); + mop = finalize_memop(s, scale); tcg_ebytes = tcg_const_i64(1 << scale); for (xs = 0; xs < selem; xs++) { @@ -3875,8 +3885,7 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) /* Load and replicate to all elements */ TCGv_i64 tcg_tmp = tcg_temp_new_i64(); - tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, - get_mem_index(s), s->be_data + scale); + tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, get_mem_index(s), mop); tcg_gen_gvec_dup_i64(scale, vec_full_reg_offset(s, rt), (is_q + 1) * 8, vec_full_reg_size(s), tcg_tmp); @@ -3884,9 +3893,9 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) } else { /* Load/store one element per register */ if (is_load) { - do_vec_ld(s, rt, index, clean_addr, scale, s->be_data); + do_vec_ld(s, rt, index, clean_addr, mop); } else { - do_vec_st(s, rt, index, clean_addr, scale, s->be_data); + do_vec_st(s, rt, index, clean_addr, mop); } } tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes); @@ -14672,7 +14681,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, DisasContext *dc = container_of(dcbase, DisasContext, base); CPUARMState *env = cpu->env_ptr; ARMCPU *arm_cpu = env_archcpu(env); - uint32_t tb_flags = dc->base.tb->flags; + CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); int bound, core_mmu_idx; dc->isar = &arm_cpu->isar; @@ -14686,28 +14695,29 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, !arm_el_is_aa64(env, 3); dc->thumb = 0; dc->sctlr_b = 0; - dc->be_data = FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; + dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; dc->condexec_mask = 0; dc->condexec_cond = 0; - core_mmu_idx = FIELD_EX32(tb_flags, TBFLAG_ANY, MMUIDX); + core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX); dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx); - dc->tbii = FIELD_EX32(tb_flags, TBFLAG_A64, TBII); - dc->tbid = FIELD_EX32(tb_flags, TBFLAG_A64, TBID); - dc->tcma = FIELD_EX32(tb_flags, TBFLAG_A64, TCMA); + dc->tbii = EX_TBFLAG_A64(tb_flags, TBII); + dc->tbid = EX_TBFLAG_A64(tb_flags, TBID); + dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA); dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); #if !defined(CONFIG_USER_ONLY) dc->user = (dc->current_el == 0); #endif - dc->fp_excp_el = FIELD_EX32(tb_flags, TBFLAG_ANY, FPEXC_EL); - dc->sve_excp_el = FIELD_EX32(tb_flags, TBFLAG_A64, SVEEXC_EL); - dc->sve_len = (FIELD_EX32(tb_flags, TBFLAG_A64, ZCR_LEN) + 1) * 16; - dc->pauth_active = FIELD_EX32(tb_flags, TBFLAG_A64, PAUTH_ACTIVE); - dc->bt = FIELD_EX32(tb_flags, TBFLAG_A64, BT); - dc->btype = FIELD_EX32(tb_flags, TBFLAG_A64, BTYPE); - dc->unpriv = FIELD_EX32(tb_flags, TBFLAG_A64, UNPRIV); - dc->ata = FIELD_EX32(tb_flags, TBFLAG_A64, ATA); - dc->mte_active[0] = FIELD_EX32(tb_flags, TBFLAG_A64, MTE_ACTIVE); - dc->mte_active[1] = FIELD_EX32(tb_flags, TBFLAG_A64, MTE0_ACTIVE); + dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); + dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); + dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); + dc->sve_len = (EX_TBFLAG_A64(tb_flags, ZCR_LEN) + 1) * 16; + dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); + dc->bt = EX_TBFLAG_A64(tb_flags, BT); + dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); + dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); + dc->ata = EX_TBFLAG_A64(tb_flags, ATA); + dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); + dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); dc->vec_len = 0; dc->vec_stride = 0; dc->cp_regs = arm_cpu->cp_regs; @@ -14734,10 +14744,10 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, * emit code to generate a software step exception * end the TB */ - dc->ss_active = FIELD_EX32(tb_flags, TBFLAG_ANY, SS_ACTIVE); - dc->pstate_ss = FIELD_EX32(tb_flags, TBFLAG_ANY, PSTATE_SS); + dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); + dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); dc->is_ldex = false; - dc->debug_target_el = FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); + dc->debug_target_el = EX_TBFLAG_ANY(tb_flags, DEBUG_TARGET_EL); /* Bound the number of insns to execute to those left on the page. */ bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4; diff --git a/target/arm/translate-a64.h b/target/arm/translate-a64.h index 3668b671dd..868d355048 100644 --- a/target/arm/translate-a64.h +++ b/target/arm/translate-a64.h @@ -44,7 +44,7 @@ TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr); TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write, bool tag_checked, int log2_size); TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write, - bool tag_checked, int count, int log2_esize); + bool tag_checked, int size); /* We should have at some point before trying to access an FP register * done the necessary access check, so assert that diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc index f6c68e30ab..a02b8369a1 100644 --- a/target/arm/translate-neon.c.inc +++ b/target/arm/translate-neon.c.inc @@ -429,7 +429,7 @@ static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) { /* Neon load/store multiple structures */ int nregs, interleave, spacing, reg, n; - MemOp endian = s->be_data; + MemOp mop, align, endian; int mmu_idx = get_mem_index(s); int size = a->size; TCGv_i64 tmp64; @@ -473,20 +473,36 @@ static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) } /* For our purposes, bytes are always little-endian. */ + endian = s->be_data; if (size == 0) { endian = MO_LE; } + + /* Enforce alignment requested by the instruction */ + if (a->align) { + align = pow2_align(a->align + 2); /* 4 ** a->align */ + } else { + align = s->align_mem ? MO_ALIGN : 0; + } + /* * Consecutive little-endian elements from a single register * can be promoted to a larger little-endian operation. */ if (interleave == 1 && endian == MO_LE) { + /* Retain any natural alignment. */ + if (align == MO_ALIGN) { + align = pow2_align(size); + } size = 3; } + tmp64 = tcg_temp_new_i64(); addr = tcg_temp_new_i32(); tmp = tcg_const_i32(1 << size); load_reg_var(s, addr, a->rn); + + mop = endian | size | align; for (reg = 0; reg < nregs; reg++) { for (n = 0; n < 8 >> size; n++) { int xs; @@ -494,13 +510,16 @@ static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) int tt = a->vd + reg + spacing * xs; if (a->l) { - gen_aa32_ld_i64(s, tmp64, addr, mmu_idx, endian | size); + gen_aa32_ld_internal_i64(s, tmp64, addr, mmu_idx, mop); neon_store_element64(tt, n, size, tmp64); } else { neon_load_element64(tmp64, tt, n, size); - gen_aa32_st_i64(s, tmp64, addr, mmu_idx, endian | size); + gen_aa32_st_internal_i64(s, tmp64, addr, mmu_idx, mop); } tcg_gen_add_i32(addr, addr, tmp); + + /* Subsequent memory operations inherit alignment */ + mop &= ~MO_AMASK; } } } @@ -520,6 +539,7 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) int size = a->size; int nregs = a->n + 1; TCGv_i32 addr, tmp; + MemOp mop, align; if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { return false; @@ -530,18 +550,33 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) return false; } + align = 0; if (size == 3) { if (nregs != 4 || a->a == 0) { return false; } /* For VLD4 size == 3 a == 1 means 32 bits at 16 byte alignment */ - size = 2; - } - if (nregs == 1 && a->a == 1 && size == 0) { - return false; - } - if (nregs == 3 && a->a == 1) { - return false; + size = MO_32; + align = MO_ALIGN_16; + } else if (a->a) { + switch (nregs) { + case 1: + if (size == 0) { + return false; + } + align = MO_ALIGN; + break; + case 2: + align = pow2_align(size + 1); + break; + case 3: + return false; + case 4: + align = pow2_align(size + 2); + break; + default: + g_assert_not_reached(); + } } if (!vfp_access_check(s)) { @@ -554,13 +589,12 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) */ stride = a->t ? 2 : 1; vec_size = nregs == 1 ? stride * 8 : 8; - + mop = size | align; tmp = tcg_temp_new_i32(); addr = tcg_temp_new_i32(); load_reg_var(s, addr, a->rn); for (reg = 0; reg < nregs; reg++) { - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - s->be_data | size); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), mop); if ((vd & 1) && vec_size == 16) { /* * We cannot write 16 bytes at once because the @@ -576,6 +610,9 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) } tcg_gen_addi_i32(addr, addr, 1 << size); vd += stride; + + /* Subsequent memory operations inherit alignment */ + mop &= ~MO_AMASK; } tcg_temp_free_i32(tmp); tcg_temp_free_i32(addr); @@ -592,6 +629,7 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) int nregs = a->n + 1; int vd = a->vd; TCGv_i32 addr, tmp; + MemOp mop; if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { return false; @@ -606,7 +644,7 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) switch (nregs) { case 1: if (((a->align & (1 << a->size)) != 0) || - (a->size == 2 && ((a->align & 3) == 1 || (a->align & 3) == 2))) { + (a->size == 2 && (a->align == 1 || a->align == 2))) { return false; } break; @@ -621,7 +659,7 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) } break; case 4: - if ((a->size == 2) && ((a->align & 3) == 3)) { + if (a->size == 2 && a->align == 3) { return false; } break; @@ -641,25 +679,58 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) return true; } + /* Pick up SCTLR settings */ + mop = finalize_memop(s, a->size); + + if (a->align) { + MemOp align_op; + + switch (nregs) { + case 1: + /* For VLD1, use natural alignment. */ + align_op = MO_ALIGN; + break; + case 2: + /* For VLD2, use double alignment. */ + align_op = pow2_align(a->size + 1); + break; + case 4: + if (a->size == MO_32) { + /* + * For VLD4.32, align = 1 is double alignment, align = 2 is + * quad alignment; align = 3 is rejected above. + */ + align_op = pow2_align(a->size + a->align); + } else { + /* For VLD4.8 and VLD.16, we want quad alignment. */ + align_op = pow2_align(a->size + 2); + } + break; + default: + /* For VLD3, the alignment field is zero and rejected above. */ + g_assert_not_reached(); + } + + mop = (mop & ~MO_AMASK) | align_op; + } + tmp = tcg_temp_new_i32(); addr = tcg_temp_new_i32(); load_reg_var(s, addr, a->rn); - /* - * TODO: if we implemented alignment exceptions, we should check - * addr against the alignment encoded in a->align here. - */ + for (reg = 0; reg < nregs; reg++) { if (a->l) { - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - s->be_data | a->size); + gen_aa32_ld_internal_i32(s, tmp, addr, get_mem_index(s), mop); neon_store_element(vd, a->reg_idx, a->size, tmp); } else { /* Store */ neon_load_element(tmp, vd, a->reg_idx, a->size); - gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), - s->be_data | a->size); + gen_aa32_st_internal_i32(s, tmp, addr, get_mem_index(s), mop); } vd += a->stride; tcg_gen_addi_i32(addr, addr, 1 << a->size); + + /* Subsequent memory operations inherit alignment */ + mop &= ~MO_AMASK; } tcg_temp_free_i32(addr); tcg_temp_free_i32(tmp); diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index 0eefb61214..864ed669c4 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -4264,7 +4264,7 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm) dirty_addr = tcg_temp_new_i64(); tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); - clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8); + clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len); tcg_temp_free_i64(dirty_addr); /* @@ -4352,7 +4352,7 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm) dirty_addr = tcg_temp_new_i64(); tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); - clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8); + clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len); tcg_temp_free_i64(dirty_addr); /* Note that unpredicated load/store of vector/predicate registers @@ -4509,8 +4509,7 @@ static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz); - desc = FIELD_DP32(desc, MTEDESC, TSIZE, mte_n << msz); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (mte_n << msz) - 1); desc <<= SVE_MTEDESC_SHIFT; } else { addr = clean_data_tbi(s, addr); @@ -5002,7 +5001,7 @@ static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) clean_addr = gen_mte_check1(s, temp, false, true, msz); tcg_gen_qemu_ld_i64(temp, clean_addr, get_mem_index(s), - s->be_data | dtype_mop[a->dtype]); + finalize_memop(s, dtype_mop[a->dtype])); /* Broadcast to *all* elements. */ tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), @@ -5189,7 +5188,7 @@ static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << msz) - 1); desc <<= SVE_MTEDESC_SHIFT; } desc = simd_desc(vsz, vsz, desc | scale); diff --git a/target/arm/translate-vfp.c.inc b/target/arm/translate-vfp.c.inc index 10766f210c..e20d9c7ba6 100644 --- a/target/arm/translate-vfp.c.inc +++ b/target/arm/translate-vfp.c.inc @@ -1364,11 +1364,11 @@ static bool trans_VLDR_VSTR_hp(DisasContext *s, arg_VLDR_VSTR_sp *a) addr = add_reg_for_lit(s, a->rn, offset); tmp = tcg_temp_new_i32(); if (a->l) { - gen_aa32_ld16u(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), MO_UW | MO_ALIGN); vfp_store_reg32(tmp, a->vd); } else { vfp_load_reg32(tmp, a->vd); - gen_aa32_st16(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UW | MO_ALIGN); } tcg_temp_free_i32(tmp); tcg_temp_free_i32(addr); @@ -1398,11 +1398,11 @@ static bool trans_VLDR_VSTR_sp(DisasContext *s, arg_VLDR_VSTR_sp *a) addr = add_reg_for_lit(s, a->rn, offset); tmp = tcg_temp_new_i32(); if (a->l) { - gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); vfp_store_reg32(tmp, a->vd); } else { vfp_load_reg32(tmp, a->vd); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); } tcg_temp_free_i32(tmp); tcg_temp_free_i32(addr); @@ -1439,11 +1439,11 @@ static bool trans_VLDR_VSTR_dp(DisasContext *s, arg_VLDR_VSTR_dp *a) addr = add_reg_for_lit(s, a->rn, offset); tmp = tcg_temp_new_i64(); if (a->l) { - gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); vfp_store_reg64(tmp, a->vd); } else { vfp_load_reg64(tmp, a->vd); - gen_aa32_st64(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); } tcg_temp_free_i64(tmp); tcg_temp_free_i32(addr); @@ -1503,12 +1503,12 @@ static bool trans_VLDM_VSTM_sp(DisasContext *s, arg_VLDM_VSTM_sp *a) for (i = 0; i < n; i++) { if (a->l) { /* load */ - gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); vfp_store_reg32(tmp, a->vd + i); } else { /* store */ vfp_load_reg32(tmp, a->vd + i); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); } tcg_gen_addi_i32(addr, addr, offset); } @@ -1586,12 +1586,12 @@ static bool trans_VLDM_VSTM_dp(DisasContext *s, arg_VLDM_VSTM_dp *a) for (i = 0; i < n; i++) { if (a->l) { /* load */ - gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); vfp_store_reg64(tmp, a->vd + i); } else { /* store */ vfp_load_reg64(tmp, a->vd + i); - gen_aa32_st64(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); } tcg_gen_addi_i32(addr, addr, offset); } diff --git a/target/arm/translate.c b/target/arm/translate.c index 7103da2d7a..43ff0d4b8a 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -908,7 +908,23 @@ static inline void store_reg_from_load(DisasContext *s, int reg, TCGv_i32 var) #define IS_USER_ONLY 0 #endif -/* Abstractions of "generate code to do a guest load/store for +MemOp pow2_align(unsigned i) +{ + static const MemOp mop_align[] = { + 0, MO_ALIGN_2, MO_ALIGN_4, MO_ALIGN_8, MO_ALIGN_16, + /* + * FIXME: TARGET_PAGE_BITS_MIN affects TLB_FLAGS_MASK such + * that 256-bit alignment (MO_ALIGN_32) cannot be supported: + * see get_alignment_bits(). Enforce only 128-bit alignment for now. + */ + MO_ALIGN_16 + }; + g_assert(i < ARRAY_SIZE(mop_align)); + return mop_align[i]; +} + +/* + * Abstractions of "generate code to do a guest load/store for * AArch32", where a vaddr is always 32 bits (and is zero * extended if we're a 64 bit core) and data is also * 32 bits unless specifically doing a 64 bit access. @@ -916,7 +932,7 @@ static inline void store_reg_from_load(DisasContext *s, int reg, TCGv_i32 var) * that the address argument is TCGv_i32 rather than TCGv. */ -static inline TCGv gen_aa32_addr(DisasContext *s, TCGv_i32 a32, MemOp op) +static TCGv gen_aa32_addr(DisasContext *s, TCGv_i32 a32, MemOp op) { TCGv addr = tcg_temp_new(); tcg_gen_extu_i32_tl(addr, a32); @@ -928,80 +944,47 @@ static inline TCGv gen_aa32_addr(DisasContext *s, TCGv_i32 a32, MemOp op) return addr; } -static void gen_aa32_ld_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, - int index, MemOp opc) +/* + * Internal routines are used for NEON cases where the endianness + * and/or alignment has already been taken into account and manipulated. + */ +static void gen_aa32_ld_internal_i32(DisasContext *s, TCGv_i32 val, + TCGv_i32 a32, int index, MemOp opc) { - TCGv addr; - - if (arm_dc_feature(s, ARM_FEATURE_M) && - !arm_dc_feature(s, ARM_FEATURE_M_MAIN)) { - opc |= MO_ALIGN; - } - - addr = gen_aa32_addr(s, a32, opc); + TCGv addr = gen_aa32_addr(s, a32, opc); tcg_gen_qemu_ld_i32(val, addr, index, opc); tcg_temp_free(addr); } -static void gen_aa32_st_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, - int index, MemOp opc) +static void gen_aa32_st_internal_i32(DisasContext *s, TCGv_i32 val, + TCGv_i32 a32, int index, MemOp opc) { - TCGv addr; - - if (arm_dc_feature(s, ARM_FEATURE_M) && - !arm_dc_feature(s, ARM_FEATURE_M_MAIN)) { - opc |= MO_ALIGN; - } - - addr = gen_aa32_addr(s, a32, opc); + TCGv addr = gen_aa32_addr(s, a32, opc); tcg_gen_qemu_st_i32(val, addr, index, opc); tcg_temp_free(addr); } -#define DO_GEN_LD(SUFF, OPC) \ -static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val, \ - TCGv_i32 a32, int index) \ -{ \ - gen_aa32_ld_i32(s, val, a32, index, OPC | s->be_data); \ -} +static void gen_aa32_ld_internal_i64(DisasContext *s, TCGv_i64 val, + TCGv_i32 a32, int index, MemOp opc) +{ + TCGv addr = gen_aa32_addr(s, a32, opc); -#define DO_GEN_ST(SUFF, OPC) \ -static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val, \ - TCGv_i32 a32, int index) \ -{ \ - gen_aa32_st_i32(s, val, a32, index, OPC | s->be_data); \ -} + tcg_gen_qemu_ld_i64(val, addr, index, opc); -static inline void gen_aa32_frob64(DisasContext *s, TCGv_i64 val) -{ /* Not needed for user-mode BE32, where we use MO_BE instead. */ - if (!IS_USER_ONLY && s->sctlr_b) { + if (!IS_USER_ONLY && s->sctlr_b && (opc & MO_SIZE) == MO_64) { tcg_gen_rotri_i64(val, val, 32); } -} - -static void gen_aa32_ld_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, - int index, MemOp opc) -{ - TCGv addr = gen_aa32_addr(s, a32, opc); - tcg_gen_qemu_ld_i64(val, addr, index, opc); - gen_aa32_frob64(s, val); tcg_temp_free(addr); } -static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val, - TCGv_i32 a32, int index) -{ - gen_aa32_ld_i64(s, val, a32, index, MO_Q | s->be_data); -} - -static void gen_aa32_st_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, - int index, MemOp opc) +static void gen_aa32_st_internal_i64(DisasContext *s, TCGv_i64 val, + TCGv_i32 a32, int index, MemOp opc) { TCGv addr = gen_aa32_addr(s, a32, opc); /* Not needed for user-mode BE32, where we use MO_BE instead. */ - if (!IS_USER_ONLY && s->sctlr_b) { + if (!IS_USER_ONLY && s->sctlr_b && (opc & MO_SIZE) == MO_64) { TCGv_i64 tmp = tcg_temp_new_i64(); tcg_gen_rotri_i64(tmp, val, 32); tcg_gen_qemu_st_i64(tmp, addr, index, opc); @@ -1012,10 +995,54 @@ static void gen_aa32_st_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, tcg_temp_free(addr); } +static void gen_aa32_ld_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_ld_internal_i32(s, val, a32, index, finalize_memop(s, opc)); +} + +static void gen_aa32_st_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_st_internal_i32(s, val, a32, index, finalize_memop(s, opc)); +} + +static void gen_aa32_ld_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_ld_internal_i64(s, val, a32, index, finalize_memop(s, opc)); +} + +static void gen_aa32_st_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_st_internal_i64(s, val, a32, index, finalize_memop(s, opc)); +} + +#define DO_GEN_LD(SUFF, OPC) \ + static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val, \ + TCGv_i32 a32, int index) \ + { \ + gen_aa32_ld_i32(s, val, a32, index, OPC); \ + } + +#define DO_GEN_ST(SUFF, OPC) \ + static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val, \ + TCGv_i32 a32, int index) \ + { \ + gen_aa32_st_i32(s, val, a32, index, OPC); \ + } + +static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val, + TCGv_i32 a32, int index) +{ + gen_aa32_ld_i64(s, val, a32, index, MO_Q); +} + static inline void gen_aa32_st64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, int index) { - gen_aa32_st_i64(s, val, a32, index, MO_Q | s->be_data); + gen_aa32_st_i64(s, val, a32, index, MO_Q); } DO_GEN_LD(8u, MO_UB) @@ -4984,16 +5011,13 @@ static void gen_load_exclusive(DisasContext *s, int rt, int rt2, TCGv_i32 tmp2 = tcg_temp_new_i32(); TCGv_i64 t64 = tcg_temp_new_i64(); - /* For AArch32, architecturally the 32-bit word at the lowest + /* + * For AArch32, architecturally the 32-bit word at the lowest * address is always Rt and the one at addr+4 is Rt2, even if * the CPU is big-endian. That means we don't want to do a - * gen_aa32_ld_i64(), which invokes gen_aa32_frob64() as if - * for an architecturally 64-bit access, but instead do a - * 64-bit access using MO_BE if appropriate and then split - * the two halves. - * This only makes a difference for BE32 user-mode, where - * frob64() must not flip the two halves of the 64-bit data - * but this code must treat BE32 user-mode like BE32 system. + * gen_aa32_ld_i64(), which checks SCTLR_B as if for an + * architecturally 64-bit access, but instead do a 64-bit access + * using MO_BE if appropriate and then split the two halves. */ TCGv taddr = gen_aa32_addr(s, addr, opc); @@ -5053,14 +5077,15 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, TCGv_i64 n64 = tcg_temp_new_i64(); t2 = load_reg(s, rt2); - /* For AArch32, architecturally the 32-bit word at the lowest + + /* + * For AArch32, architecturally the 32-bit word at the lowest * address is always Rt and the one at addr+4 is Rt2, even if * the CPU is big-endian. Since we're going to treat this as a * single 64-bit BE store, we need to put the two halves in the * opposite order for BE to LE, so that they end up in the right - * places. - * We don't want gen_aa32_frob64() because that does the wrong - * thing for BE32 usermode. + * places. We don't want gen_aa32_st_i64, because that checks + * SCTLR_B as if for an architectural 64-bit access. */ if (s->be_data == MO_BE) { tcg_gen_concat_i32_i64(n64, t2, t1); @@ -5190,11 +5215,11 @@ static void gen_srs(DisasContext *s, } tcg_gen_addi_i32(addr, addr, offset); tmp = load_reg(s, 14); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); tmp = load_cpu_field(spsr); tcg_gen_addi_i32(addr, addr, 4); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); if (writeback) { switch (amode) { @@ -6458,7 +6483,7 @@ static bool op_load_rr(DisasContext *s, arg_ldst_rr *a, addr = op_addr_rr_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); /* @@ -6476,10 +6501,18 @@ static bool op_store_rr(DisasContext *s, arg_ldst_rr *a, ISSInfo issinfo = make_issinfo(s, a->rt, a->p, a->w) | ISSIsWrite; TCGv_i32 addr, tmp; + /* + * In Thumb encodings of stores Rn=1111 is UNDEF; for Arm it + * is either UNPREDICTABLE or has defined behaviour + */ + if (s->thumb && a->rn == 15) { + return false; + } + addr = op_addr_rr_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); tcg_temp_free_i32(tmp); @@ -6502,13 +6535,13 @@ static bool trans_LDRD_rr(DisasContext *s, arg_ldst_rr *a) addr = op_addr_rr_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, a->rt, tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, a->rt + 1, tmp); /* LDRD w/ base writeback is undefined if the registers overlap. */ @@ -6531,13 +6564,13 @@ static bool trans_STRD_rr(DisasContext *s, arg_ldst_rr *a) addr = op_addr_rr_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = load_reg(s, a->rt + 1); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); op_addr_rr_post(s, a, addr, -4); @@ -6602,7 +6635,7 @@ static bool op_load_ri(DisasContext *s, arg_ldst_ri *a, addr = op_addr_ri_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); /* @@ -6620,10 +6653,18 @@ static bool op_store_ri(DisasContext *s, arg_ldst_ri *a, ISSInfo issinfo = make_issinfo(s, a->rt, a->p, a->w) | ISSIsWrite; TCGv_i32 addr, tmp; + /* + * In Thumb encodings of stores Rn=1111 is UNDEF; for Arm it + * is either UNPREDICTABLE or has defined behaviour + */ + if (s->thumb && a->rn == 15) { + return false; + } + addr = op_addr_ri_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); tcg_temp_free_i32(tmp); @@ -6639,13 +6680,13 @@ static bool op_ldrd_ri(DisasContext *s, arg_ldst_ri *a, int rt2) addr = op_addr_ri_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, a->rt, tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, rt2, tmp); /* LDRD w/ base writeback is undefined if the registers overlap. */ @@ -6678,13 +6719,13 @@ static bool op_strd_ri(DisasContext *s, arg_ldst_ri *a, int rt2) addr = op_addr_ri_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = load_reg(s, rt2); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); op_addr_ri_post(s, a, addr, -4); @@ -6910,7 +6951,7 @@ static bool op_stl(DisasContext *s, arg_STL *a, MemOp mop) addr = load_reg(s, a->rn); tmp = load_reg(s, a->rt); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); - gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), mop | s->be_data); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), mop | MO_ALIGN); disas_set_da_iss(s, mop, a->rt | ISSIsAcqRel | ISSIsWrite); tcg_temp_free_i32(tmp); @@ -7066,7 +7107,7 @@ static bool op_lda(DisasContext *s, arg_LDA *a, MemOp mop) addr = load_reg(s, a->rn); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), mop | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), mop | MO_ALIGN); disas_set_da_iss(s, mop, a->rt | ISSIsAcqRel); tcg_temp_free_i32(addr); @@ -7858,7 +7899,7 @@ static bool op_stm(DisasContext *s, arg_ldst_block *a, int min_n) } else { tmp = load_reg(s, i); } - gen_aa32_st32(s, tmp, addr, mem_idx); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); /* No need to add after the last transfer. */ @@ -7933,7 +7974,7 @@ static bool do_ldm(DisasContext *s, arg_ldst_block *a, int min_n) } tmp = tcg_temp_new_i32(); - gen_aa32_ld32u(s, tmp, addr, mem_idx); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); if (user) { tmp2 = tcg_const_i32(i); gen_helper_set_user_reg(cpu_env, tmp2, tmp); @@ -8250,8 +8291,7 @@ static bool op_tbranch(DisasContext *s, arg_tbranch *a, bool half) addr = load_reg(s, a->rn); tcg_gen_add_i32(addr, addr, tmp); - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - half ? MO_UW | s->be_data : MO_UB); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), half ? MO_UW : MO_UB); tcg_temp_free_i32(addr); tcg_gen_add_i32(tmp, tmp, tmp); @@ -8332,10 +8372,10 @@ static bool trans_RFE(DisasContext *s, arg_RFE *a) /* Load PC into tmp and CPSR into tmp2. */ t1 = tcg_temp_new_i32(); - gen_aa32_ld32u(s, t1, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, t1, addr, get_mem_index(s), MO_UL | MO_ALIGN); tcg_gen_addi_i32(addr, addr, 4); t2 = tcg_temp_new_i32(); - gen_aa32_ld32u(s, t2, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, t2, addr, get_mem_index(s), MO_UL | MO_ALIGN); if (a->w) { /* Base writeback. */ @@ -8836,7 +8876,7 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) DisasContext *dc = container_of(dcbase, DisasContext, base); CPUARMState *env = cs->env_ptr; ARMCPU *cpu = env_archcpu(env); - uint32_t tb_flags = dc->base.tb->flags; + CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); uint32_t condexec, core_mmu_idx; dc->isar = &cpu->isar; @@ -8848,46 +8888,43 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) */ dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3); - dc->thumb = FIELD_EX32(tb_flags, TBFLAG_AM32, THUMB); - dc->be_data = FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; - condexec = FIELD_EX32(tb_flags, TBFLAG_AM32, CONDEXEC); + dc->thumb = EX_TBFLAG_AM32(tb_flags, THUMB); + dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; + condexec = EX_TBFLAG_AM32(tb_flags, CONDEXEC); dc->condexec_mask = (condexec & 0xf) << 1; dc->condexec_cond = condexec >> 4; - core_mmu_idx = FIELD_EX32(tb_flags, TBFLAG_ANY, MMUIDX); + core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX); dc->mmu_idx = core_to_arm_mmu_idx(env, core_mmu_idx); dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); #if !defined(CONFIG_USER_ONLY) dc->user = (dc->current_el == 0); #endif - dc->fp_excp_el = FIELD_EX32(tb_flags, TBFLAG_ANY, FPEXC_EL); + dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); + dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); if (arm_feature(env, ARM_FEATURE_M)) { dc->vfp_enabled = 1; dc->be_data = MO_TE; - dc->v7m_handler_mode = FIELD_EX32(tb_flags, TBFLAG_M32, HANDLER); + dc->v7m_handler_mode = EX_TBFLAG_M32(tb_flags, HANDLER); dc->v8m_secure = arm_feature(env, ARM_FEATURE_M_SECURITY) && regime_is_secure(env, dc->mmu_idx); - dc->v8m_stackcheck = FIELD_EX32(tb_flags, TBFLAG_M32, STACKCHECK); - dc->v8m_fpccr_s_wrong = - FIELD_EX32(tb_flags, TBFLAG_M32, FPCCR_S_WRONG); + dc->v8m_stackcheck = EX_TBFLAG_M32(tb_flags, STACKCHECK); + dc->v8m_fpccr_s_wrong = EX_TBFLAG_M32(tb_flags, FPCCR_S_WRONG); dc->v7m_new_fp_ctxt_needed = - FIELD_EX32(tb_flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED); - dc->v7m_lspact = FIELD_EX32(tb_flags, TBFLAG_M32, LSPACT); + EX_TBFLAG_M32(tb_flags, NEW_FP_CTXT_NEEDED); + dc->v7m_lspact = EX_TBFLAG_M32(tb_flags, LSPACT); } else { - dc->be_data = - FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; - dc->debug_target_el = - FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); - dc->sctlr_b = FIELD_EX32(tb_flags, TBFLAG_A32, SCTLR_B); - dc->hstr_active = FIELD_EX32(tb_flags, TBFLAG_A32, HSTR_ACTIVE); - dc->ns = FIELD_EX32(tb_flags, TBFLAG_A32, NS); - dc->vfp_enabled = FIELD_EX32(tb_flags, TBFLAG_A32, VFPEN); + dc->debug_target_el = EX_TBFLAG_ANY(tb_flags, DEBUG_TARGET_EL); + dc->sctlr_b = EX_TBFLAG_A32(tb_flags, SCTLR__B); + dc->hstr_active = EX_TBFLAG_A32(tb_flags, HSTR_ACTIVE); + dc->ns = EX_TBFLAG_A32(tb_flags, NS); + dc->vfp_enabled = EX_TBFLAG_A32(tb_flags, VFPEN); if (arm_feature(env, ARM_FEATURE_XSCALE)) { - dc->c15_cpar = FIELD_EX32(tb_flags, TBFLAG_A32, XSCALE_CPAR); + dc->c15_cpar = EX_TBFLAG_A32(tb_flags, XSCALE_CPAR); } else { - dc->vec_len = FIELD_EX32(tb_flags, TBFLAG_A32, VECLEN); - dc->vec_stride = FIELD_EX32(tb_flags, TBFLAG_A32, VECSTRIDE); + dc->vec_len = EX_TBFLAG_A32(tb_flags, VECLEN); + dc->vec_stride = EX_TBFLAG_A32(tb_flags, VECSTRIDE); } } dc->cp_regs = cpu->cp_regs; @@ -8908,8 +8945,8 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) * emit code to generate a software step exception * end the TB */ - dc->ss_active = FIELD_EX32(tb_flags, TBFLAG_ANY, SS_ACTIVE); - dc->pstate_ss = FIELD_EX32(tb_flags, TBFLAG_ANY, PSTATE_SS); + dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); + dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); dc->is_ldex = false; dc->page_start = dc->base.pc_first & TARGET_PAGE_MASK; @@ -9347,12 +9384,13 @@ void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) { DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; + CPUARMTBFlags tb_flags = arm_tbflags_from_tb(tb); - if (FIELD_EX32(tb->flags, TBFLAG_AM32, THUMB)) { + if (EX_TBFLAG_AM32(tb_flags, THUMB)) { ops = &thumb_translator_ops; } #ifdef TARGET_AARCH64 - if (FIELD_EX32(tb->flags, TBFLAG_ANY, AARCH64_STATE)) { + if (EX_TBFLAG_ANY(tb_flags, AARCH64_STATE)) { ops = &aarch64_translator_ops; } #endif diff --git a/target/arm/translate.h b/target/arm/translate.h index 423b0e08df..ccf60c96d8 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -87,6 +87,8 @@ typedef struct DisasContext { bool bt; /* True if any CP15 access is trapped by HSTR_EL2 */ bool hstr_active; + /* True if memory operations require alignment */ + bool align_mem; /* * >= 0, a copy of PSTATE.BTYPE, which will be 0 without v8.5-BTI. * < 0, set by the current instruction. @@ -202,6 +204,7 @@ void arm_test_cc(DisasCompare *cmp, int cc); void arm_free_cc(DisasCompare *cmp); void arm_jump_cc(DisasCompare *cmp, TCGLabel *label); void arm_gen_test_cc(int cc, TCGLabel *label); +MemOp pow2_align(unsigned i); /* Return state of Alternate Half-precision flag, caller frees result */ static inline TCGv_i32 get_ahp_flag(void) @@ -394,6 +397,17 @@ typedef void CryptoThreeOpIntFn(TCGv_ptr, TCGv_ptr, TCGv_i32); typedef void CryptoThreeOpFn(TCGv_ptr, TCGv_ptr, TCGv_ptr); typedef void AtomicThreeOpFn(TCGv_i64, TCGv_i64, TCGv_i64, TCGArg, MemOp); +/** + * arm_tbflags_from_tb: + * @tb: the TranslationBlock + * + * Extract the flag values from @tb. + */ +static inline CPUARMTBFlags arm_tbflags_from_tb(const TranslationBlock *tb) +{ + return (CPUARMTBFlags){ tb->flags, tb->cs_base }; +} + /* * Enum for argument to fpstatus_ptr(). */ @@ -446,4 +460,28 @@ static inline TCGv_ptr fpstatus_ptr(ARMFPStatusFlavour flavour) return statusptr; } +/** + * finalize_memop: + * @s: DisasContext + * @opc: size+sign+align of the memory operation + * + * Build the complete MemOp for a memory operation, including alignment + * and endianness. + * + * If (op & MO_AMASK) then the operation already contains the required + * alignment, e.g. for AccType_ATOMIC. Otherwise, this an optionally + * unaligned operation, e.g. for AccType_NORMAL. + * + * In the latter case, there are configuration bits that require alignment, + * and this is applied here. Note that there is no way to indicate that + * no alignment should ever be enforced; this must be handled manually. + */ +static inline MemOp finalize_memop(DisasContext *s, MemOp opc) +{ + if (s->align_mem && !(opc & MO_AMASK)) { + opc |= MO_ALIGN; + } + return opc | s->be_data; +} + #endif /* TARGET_ARM_TRANSLATE_H */ diff --git a/tests/qemu-iotests/245 b/tests/qemu-iotests/245 index 11104b9208..fc5297e268 100755 --- a/tests/qemu-iotests/245 +++ b/tests/qemu-iotests/245 @@ -905,7 +905,7 @@ class TestBlockdevReopen(iotests.QMPTestCase): # We can't reopen hd1 to read-only, as block-stream requires it to be # read-write self.reopen(opts['backing'], {'read-only': True}, - "Cannot make block node read-only, there is a writer on it") + "Read-only block node 'hd1' cannot support read-write users") # We can't remove hd2 while the stream job is ongoing opts['backing']['backing'] = None diff --git a/tests/qemu-iotests/283.out b/tests/qemu-iotests/283.out index 37c35058ae..97e62a4c94 100644 --- a/tests/qemu-iotests/283.out +++ b/tests/qemu-iotests/283.out @@ -5,7 +5,7 @@ {"execute": "blockdev-add", "arguments": {"driver": "blkdebug", "image": "base", "node-name": "other", "take-child-perms": ["write"]}} {"return": {}} {"execute": "blockdev-backup", "arguments": {"device": "source", "sync": "full", "target": "target"}} -{"error": {"class": "GenericError", "desc": "Cannot set permissions for backup-top filter: Conflicts with use by other as 'image', which uses 'write' on base"}} +{"error": {"class": "GenericError", "desc": "Cannot append backup-top filter: Conflicts with use by source as 'image', which does not allow 'write' on base"}} === backup-top should be gone after job-finalize === diff --git a/tests/qemu-iotests/tests/qsd-jobs.out b/tests/qemu-iotests/tests/qsd-jobs.out index 5f41491e05..9f52255da8 100644 --- a/tests/qemu-iotests/tests/qsd-jobs.out +++ b/tests/qemu-iotests/tests/qsd-jobs.out @@ -16,7 +16,7 @@ QMP_VERSION {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job0"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job0"}} -{"error": {"class": "GenericError", "desc": "Conflicts with use by a block device as 'root', which uses 'write' on fmt_base"}} +{"error": {"class": "GenericError", "desc": "Conflicts with use by stream job 'job0' as 'intermediate node', which does not allow 'write' on fmt_base"}} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_EXPORT_DELETED", "data": {"id": "export1"}} *** done diff --git a/tests/tcg/aarch64/Makefile.target b/tests/tcg/aarch64/Makefile.target index 05b2622bfc..928357b10a 100644 --- a/tests/tcg/aarch64/Makefile.target +++ b/tests/tcg/aarch64/Makefile.target @@ -37,7 +37,7 @@ AARCH64_TESTS += bti-2 # MTE Tests ifneq ($(DOCKER_IMAGE)$(CROSS_CC_HAS_ARMV8_MTE),) -AARCH64_TESTS += mte-1 mte-2 mte-3 mte-4 mte-6 +AARCH64_TESTS += mte-1 mte-2 mte-3 mte-4 mte-5 mte-6 mte-%: CFLAGS += -march=armv8.5-a+memtag endif diff --git a/tests/tcg/aarch64/mte-5.c b/tests/tcg/aarch64/mte-5.c new file mode 100644 index 0000000000..6dbd6ab3ea --- /dev/null +++ b/tests/tcg/aarch64/mte-5.c @@ -0,0 +1,44 @@ +/* + * Memory tagging, faulting unaligned access. + * + * Copyright (c) 2021 Linaro Ltd + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "mte.h" + +void pass(int sig, siginfo_t *info, void *uc) +{ + assert(info->si_code == SEGV_MTESERR); + exit(0); +} + +int main(int ac, char **av) +{ + struct sigaction sa; + void *p0, *p1, *p2; + long excl = 1; + + enable_mte(PR_MTE_TCF_SYNC); + p0 = alloc_mte_mem(sizeof(*p0)); + + /* Create two differently tagged pointers. */ + asm("irg %0,%1,%2" : "=r"(p1) : "r"(p0), "r"(excl)); + asm("gmi %0,%1,%0" : "+r"(excl) : "r" (p1)); + assert(excl != 1); + asm("irg %0,%1,%2" : "=r"(p2) : "r"(p0), "r"(excl)); + assert(p1 != p2); + + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = pass; + sa.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &sa, NULL); + + /* Store store two different tags in sequential granules. */ + asm("stg %0, [%0]" : : "r"(p1)); + asm("stg %0, [%0]" : : "r"(p2 + 16)); + + /* Perform an unaligned load crossing the granules. */ + asm volatile("ldr %0, [%1]" : "=r"(p0) : "r"(p1 + 12)); + abort(); +} diff --git a/tests/unit/test-bdrv-drain.c b/tests/unit/test-bdrv-drain.c index 8a29e33e00..892f7f47d8 100644 --- a/tests/unit/test-bdrv-drain.c +++ b/tests/unit/test-bdrv-drain.c @@ -1478,7 +1478,6 @@ static void test_append_to_drained(void) g_assert_cmpint(base_s->drain_count, ==, 1); g_assert_cmpint(base->in_flight, ==, 0); - /* Takes ownership of overlay, so we don't have to unref it later */ bdrv_append(overlay, base, &error_abort); g_assert_cmpint(base->in_flight, ==, 0); g_assert_cmpint(overlay->in_flight, ==, 0); @@ -1495,6 +1494,7 @@ static void test_append_to_drained(void) g_assert_cmpint(overlay->quiesce_counter, ==, 0); g_assert_cmpint(overlay_s->drain_count, ==, 0); + bdrv_unref(overlay); bdrv_unref(base); blk_unref(blk); } diff --git a/tests/unit/test-bdrv-graph-mod.c b/tests/unit/test-bdrv-graph-mod.c index c4f7d16039..88f25c0cdb 100644 --- a/tests/unit/test-bdrv-graph-mod.c +++ b/tests/unit/test-bdrv-graph-mod.c @@ -1,7 +1,7 @@ /* * Block node graph modifications tests * - * Copyright (c) 2019 Virtuozzo International GmbH. All rights reserved. + * Copyright (c) 2019-2021 Virtuozzo International GmbH. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -44,6 +44,21 @@ static BlockDriver bdrv_no_perm = { .bdrv_child_perm = no_perm_default_perms, }; +static void exclusive_write_perms(BlockDriverState *bs, BdrvChild *c, + BdrvChildRole role, + BlockReopenQueue *reopen_queue, + uint64_t perm, uint64_t shared, + uint64_t *nperm, uint64_t *nshared) +{ + *nperm = BLK_PERM_WRITE; + *nshared = BLK_PERM_ALL & ~BLK_PERM_WRITE; +} + +static BlockDriver bdrv_exclusive_writer = { + .format_name = "exclusive-writer", + .bdrv_child_perm = exclusive_write_perms, +}; + static BlockDriverState *no_perm_node(const char *name) { return bdrv_new_open_driver(&bdrv_no_perm, name, BDRV_O_RDWR, &error_abort); @@ -55,6 +70,12 @@ static BlockDriverState *pass_through_node(const char *name) BDRV_O_RDWR, &error_abort); } +static BlockDriverState *exclusive_writer_node(const char *name) +{ + return bdrv_new_open_driver(&bdrv_exclusive_writer, name, + BDRV_O_RDWR, &error_abort); +} + /* * test_update_perm_tree * @@ -117,6 +138,7 @@ static void test_update_perm_tree(void) ret = bdrv_append(filter, bs, NULL); g_assert_cmpint(ret, <, 0); + bdrv_unref(filter); blk_unref(root); } @@ -181,10 +203,189 @@ static void test_should_update_child(void) bdrv_append(filter, bs, &error_abort); g_assert(target->backing->bs == bs); + bdrv_unref(filter); bdrv_unref(bs); blk_unref(root); } +/* + * test_parallel_exclusive_write + * + * Check that when we replace node, old permissions of the node being removed + * doesn't break the replacement. + */ +static void test_parallel_exclusive_write(void) +{ + BlockDriverState *top = exclusive_writer_node("top"); + BlockDriverState *base = no_perm_node("base"); + BlockDriverState *fl1 = pass_through_node("fl1"); + BlockDriverState *fl2 = pass_through_node("fl2"); + + /* + * bdrv_attach_child() eats child bs reference, so we need two @base + * references for two filters: + */ + bdrv_ref(base); + + bdrv_attach_child(top, fl1, "backing", &child_of_bds, BDRV_CHILD_DATA, + &error_abort); + bdrv_attach_child(fl1, base, "backing", &child_of_bds, BDRV_CHILD_FILTERED, + &error_abort); + bdrv_attach_child(fl2, base, "backing", &child_of_bds, BDRV_CHILD_FILTERED, + &error_abort); + + bdrv_replace_node(fl1, fl2, &error_abort); + + bdrv_unref(fl2); + bdrv_unref(top); +} + +static void write_to_file_perms(BlockDriverState *bs, BdrvChild *c, + BdrvChildRole role, + BlockReopenQueue *reopen_queue, + uint64_t perm, uint64_t shared, + uint64_t *nperm, uint64_t *nshared) +{ + if (bs->file && c == bs->file) { + *nperm = BLK_PERM_WRITE; + *nshared = BLK_PERM_ALL & ~BLK_PERM_WRITE; + } else { + *nperm = 0; + *nshared = BLK_PERM_ALL; + } +} + +static BlockDriver bdrv_write_to_file = { + .format_name = "tricky-perm", + .bdrv_child_perm = write_to_file_perms, +}; + + +/* + * The following test shows that topological-sort order is required for + * permission update, simple DFS is not enough. + * + * Consider the block driver which has two filter children: one active + * with exclusive write access and one inactive with no specific + * permissions. + * + * And, these two children has a common base child, like this: + * + * ┌─────┐ ┌──────┐ + * │ fl2 │ ◀── │ top │ + * └─────┘ └──────┘ + * │ │ + * │ │ w + * │ ▼ + * │ ┌──────┐ + * │ │ fl1 │ + * │ └──────┘ + * │ │ + * │ │ w + * │ ▼ + * │ ┌──────┐ + * └───────▶ │ base │ + * └──────┘ + * + * So, exclusive write is propagated. + * + * Assume, we want to make fl2 active instead of fl1. + * So, we set some option for top driver and do permission update. + * + * With simple DFS, if permission update goes first through + * top->fl1->base branch it will succeed: it firstly drop exclusive write + * permissions and than apply them for another BdrvChildren. + * But if permission update goes first through top->fl2->base branch it + * will fail, as when we try to update fl2->base child, old not yet + * updated fl1->base child will be in conflict. + * + * With topological-sort order we always update parents before children, so fl1 + * and fl2 are both updated when we update base and there is no conflict. + */ +static void test_parallel_perm_update(void) +{ + BlockDriverState *top = no_perm_node("top"); + BlockDriverState *tricky = + bdrv_new_open_driver(&bdrv_write_to_file, "tricky", BDRV_O_RDWR, + &error_abort); + BlockDriverState *base = no_perm_node("base"); + BlockDriverState *fl1 = pass_through_node("fl1"); + BlockDriverState *fl2 = pass_through_node("fl2"); + BdrvChild *c_fl1, *c_fl2; + + /* + * bdrv_attach_child() eats child bs reference, so we need two @base + * references for two filters: + */ + bdrv_ref(base); + + bdrv_attach_child(top, tricky, "file", &child_of_bds, BDRV_CHILD_DATA, + &error_abort); + c_fl1 = bdrv_attach_child(tricky, fl1, "first", &child_of_bds, + BDRV_CHILD_FILTERED, &error_abort); + c_fl2 = bdrv_attach_child(tricky, fl2, "second", &child_of_bds, + BDRV_CHILD_FILTERED, &error_abort); + bdrv_attach_child(fl1, base, "backing", &child_of_bds, BDRV_CHILD_FILTERED, + &error_abort); + bdrv_attach_child(fl2, base, "backing", &child_of_bds, BDRV_CHILD_FILTERED, + &error_abort); + + /* Select fl1 as first child to be active */ + tricky->file = c_fl1; + bdrv_child_refresh_perms(top, top->children.lh_first, &error_abort); + + assert(c_fl1->perm & BLK_PERM_WRITE); + assert(!(c_fl2->perm & BLK_PERM_WRITE)); + + /* Now, try to switch active child and update permissions */ + tricky->file = c_fl2; + bdrv_child_refresh_perms(top, top->children.lh_first, &error_abort); + + assert(c_fl2->perm & BLK_PERM_WRITE); + assert(!(c_fl1->perm & BLK_PERM_WRITE)); + + /* Switch once more, to not care about real child order in the list */ + tricky->file = c_fl1; + bdrv_child_refresh_perms(top, top->children.lh_first, &error_abort); + + assert(c_fl1->perm & BLK_PERM_WRITE); + assert(!(c_fl2->perm & BLK_PERM_WRITE)); + + bdrv_unref(top); +} + +/* + * It's possible that filter required permissions allows to insert it to backing + * chain, like: + * + * 1. [top] -> [filter] -> [base] + * + * but doesn't allow to add it as a branch: + * + * 2. [filter] --\ + * v + * [top] -> [base] + * + * So, inserting such filter should do all graph modifications and only then + * update permissions. If we try to go through intermediate state [2] and update + * permissions on it we'll fail. + * + * Let's check that bdrv_append() can append such a filter. + */ +static void test_append_greedy_filter(void) +{ + BlockDriverState *top = exclusive_writer_node("top"); + BlockDriverState *base = no_perm_node("base"); + BlockDriverState *fl = exclusive_writer_node("fl1"); + + bdrv_attach_child(top, base, "backing", &child_of_bds, BDRV_CHILD_COW, + &error_abort); + + bdrv_append(fl, base, &error_abort); + bdrv_unref(fl); + bdrv_unref(top); +} + int main(int argc, char *argv[]) { bdrv_init(); @@ -195,6 +396,12 @@ int main(int argc, char *argv[]) g_test_add_func("/bdrv-graph-mod/update-perm-tree", test_update_perm_tree); g_test_add_func("/bdrv-graph-mod/should-update-child", test_should_update_child); + g_test_add_func("/bdrv-graph-mod/parallel-perm-update", + test_parallel_perm_update); + g_test_add_func("/bdrv-graph-mod/parallel-exclusive-write", + test_parallel_exclusive_write); + g_test_add_func("/bdrv-graph-mod/append-greedy-filter", + test_append_greedy_filter); return g_test_run(); } diff --git a/util/meson.build b/util/meson.build index 510765cde4..97fad44105 100644 --- a/util/meson.build +++ b/util/meson.build @@ -41,6 +41,7 @@ util_ss.add(files('qsp.c')) util_ss.add(files('range.c')) util_ss.add(files('stats64.c')) util_ss.add(files('systemd.c')) +util_ss.add(files('transactions.c')) util_ss.add(when: 'CONFIG_POSIX', if_true: files('drm.c')) util_ss.add(files('guest-random.c')) util_ss.add(files('yank.c')) diff --git a/util/transactions.c b/util/transactions.c new file mode 100644 index 0000000000..d0bc9a3e73 --- /dev/null +++ b/util/transactions.c @@ -0,0 +1,96 @@ +/* + * Simple transactions API + * + * Copyright (c) 2021 Virtuozzo International GmbH. + * + * Author: + * Sementsov-Ogievskiy Vladimir <vsementsov@virtuozzo.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" + +#include "qemu/transactions.h" +#include "qemu/queue.h" + +typedef struct TransactionAction { + TransactionActionDrv *drv; + void *opaque; + QSLIST_ENTRY(TransactionAction) entry; +} TransactionAction; + +struct Transaction { + QSLIST_HEAD(, TransactionAction) actions; +}; + +Transaction *tran_new(void) +{ + Transaction *tran = g_new(Transaction, 1); + + QSLIST_INIT(&tran->actions); + + return tran; +} + +void tran_add(Transaction *tran, TransactionActionDrv *drv, void *opaque) +{ + TransactionAction *act; + + act = g_new(TransactionAction, 1); + *act = (TransactionAction) { + .drv = drv, + .opaque = opaque + }; + + QSLIST_INSERT_HEAD(&tran->actions, act, entry); +} + +void tran_abort(Transaction *tran) +{ + TransactionAction *act, *next; + + QSLIST_FOREACH_SAFE(act, &tran->actions, entry, next) { + if (act->drv->abort) { + act->drv->abort(act->opaque); + } + + if (act->drv->clean) { + act->drv->clean(act->opaque); + } + + g_free(act); + } + + g_free(tran); +} + +void tran_commit(Transaction *tran) +{ + TransactionAction *act, *next; + + QSLIST_FOREACH_SAFE(act, &tran->actions, entry, next) { + if (act->drv->commit) { + act->drv->commit(act->opaque); + } + + if (act->drv->clean) { + act->drv->clean(act->opaque); + } + + g_free(act); + } + + g_free(tran); +} |