diff options
Diffstat (limited to '')
185 files changed, 13101 insertions, 3617 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index 3f24bc2787..95c957d587 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2236,14 +2236,27 @@ F: docs/specs/rocker.txt e1000x M: Dmitry Fleytman <dmitry.fleytman@gmail.com> +R: Akihiko Odaki <akihiko.odaki@daynix.com> S: Maintained F: hw/net/e1000x* e1000e M: Dmitry Fleytman <dmitry.fleytman@gmail.com> +R: Akihiko Odaki <akihiko.odaki@daynix.com> S: Maintained F: hw/net/e1000e* F: tests/qtest/fuzz-e1000e-test.c +F: tests/qtest/e1000e-test.c +F: tests/qtest/libqos/e1000e.* + +igb +M: Akihiko Odaki <akihiko.odaki@daynix.com> +S: Maintained +F: docs/system/devices/igb.rst +F: hw/net/igb* +F: tests/avocado/igb.py +F: tests/qtest/igb-test.c +F: tests/qtest/libqos/igb.c eepro100 M: Stefan Weil <sw@weilnetz.de> @@ -2889,9 +2902,11 @@ T: git https://gitlab.com/ehabkost/qemu.git machine-next Cryptodev Backends M: Gonglei <arei.gonglei@huawei.com> +M: zhenwei pi <pizhenwei@bytedance.com> S: Maintained F: include/sysemu/cryptodev*.h F: backends/cryptodev*.c +F: qapi/cryptodev.json Python library M: John Snow <jsnow@redhat.com> diff --git a/backends/cryptodev-builtin.c b/backends/cryptodev-builtin.c index cda6ca3b71..39d0455280 100644 --- a/backends/cryptodev-builtin.c +++ b/backends/cryptodev-builtin.c @@ -59,6 +59,19 @@ struct CryptoDevBackendBuiltin { CryptoDevBackendBuiltinSession *sessions[MAX_NUM_SESSIONS]; }; +static void cryptodev_builtin_init_akcipher(CryptoDevBackend *backend) +{ + QCryptoAkCipherOptions opts; + + opts.alg = QCRYPTO_AKCIPHER_ALG_RSA; + opts.u.rsa.padding_alg = QCRYPTO_RSA_PADDING_ALG_RAW; + if (qcrypto_akcipher_supports(&opts)) { + backend->conf.crypto_services |= + (1u << QCRYPTODEV_BACKEND_SERVICE_AKCIPHER); + backend->conf.akcipher_algo = 1u << VIRTIO_CRYPTO_AKCIPHER_RSA; + } +} + static void cryptodev_builtin_init( CryptoDevBackend *backend, Error **errp) { @@ -72,21 +85,18 @@ static void cryptodev_builtin_init( return; } - cc = cryptodev_backend_new_client( - "cryptodev-builtin", NULL); + cc = cryptodev_backend_new_client(); cc->info_str = g_strdup_printf("cryptodev-builtin0"); cc->queue_index = 0; - cc->type = CRYPTODEV_BACKEND_TYPE_BUILTIN; + cc->type = QCRYPTODEV_BACKEND_TYPE_BUILTIN; backend->conf.peers.ccs[0] = cc; backend->conf.crypto_services = - 1u << VIRTIO_CRYPTO_SERVICE_CIPHER | - 1u << VIRTIO_CRYPTO_SERVICE_HASH | - 1u << VIRTIO_CRYPTO_SERVICE_MAC | - 1u << VIRTIO_CRYPTO_SERVICE_AKCIPHER; + 1u << QCRYPTODEV_BACKEND_SERVICE_CIPHER | + 1u << QCRYPTODEV_BACKEND_SERVICE_HASH | + 1u << QCRYPTODEV_BACKEND_SERVICE_MAC; backend->conf.cipher_algo_l = 1u << VIRTIO_CRYPTO_CIPHER_AES_CBC; backend->conf.hash_algo = 1u << VIRTIO_CRYPTO_HASH_SHA1; - backend->conf.akcipher_algo = 1u << VIRTIO_CRYPTO_AKCIPHER_RSA; /* * Set the Maximum length of crypto request. * Why this value? Just avoid to overflow when @@ -95,6 +105,7 @@ static void cryptodev_builtin_init( backend->conf.max_size = LONG_MAX - sizeof(CryptoDevBackendOpInfo); backend->conf.max_cipher_key_len = CRYPTODEV_BUITLIN_MAX_CIPHER_KEY_LEN; backend->conf.max_auth_key_len = CRYPTODEV_BUITLIN_MAX_AUTH_KEY_LEN; + cryptodev_builtin_init_akcipher(backend); cryptodev_backend_set_ready(backend, true); } @@ -528,17 +539,14 @@ static int cryptodev_builtin_asym_operation( static int cryptodev_builtin_operation( CryptoDevBackend *backend, - CryptoDevBackendOpInfo *op_info, - uint32_t queue_index, - CryptoDevCompletionFunc cb, - void *opaque) + CryptoDevBackendOpInfo *op_info) { CryptoDevBackendBuiltin *builtin = CRYPTODEV_BACKEND_BUILTIN(backend); CryptoDevBackendBuiltinSession *sess; CryptoDevBackendSymOpInfo *sym_op_info; CryptoDevBackendAsymOpInfo *asym_op_info; - enum CryptoDevBackendAlgType algtype = op_info->algtype; + QCryptodevBackendAlgType algtype = op_info->algtype; int status = -VIRTIO_CRYPTO_ERR; Error *local_error = NULL; @@ -550,11 +558,11 @@ static int cryptodev_builtin_operation( } sess = builtin->sessions[op_info->session_id]; - if (algtype == CRYPTODEV_BACKEND_ALG_SYM) { + if (algtype == QCRYPTODEV_BACKEND_ALG_SYM) { sym_op_info = op_info->u.sym_op_info; status = cryptodev_builtin_sym_operation(sess, sym_op_info, &local_error); - } else if (algtype == CRYPTODEV_BACKEND_ALG_ASYM) { + } else if (algtype == QCRYPTODEV_BACKEND_ALG_ASYM) { asym_op_info = op_info->u.asym_op_info; status = cryptodev_builtin_asym_operation(sess, op_info->op_code, asym_op_info, &local_error); @@ -563,8 +571,8 @@ static int cryptodev_builtin_operation( if (local_error) { error_report_err(local_error); } - if (cb) { - cb(opaque, status); + if (op_info->cb) { + op_info->cb(op_info->opaque, status); } return 0; } diff --git a/backends/cryptodev-hmp-cmds.c b/backends/cryptodev-hmp-cmds.c new file mode 100644 index 0000000000..4f7220bb13 --- /dev/null +++ b/backends/cryptodev-hmp-cmds.c @@ -0,0 +1,54 @@ +/* + * HMP commands related to cryptodev + * + * Copyright (c) 2023 Bytedance.Inc + * + * Authors: + * zhenwei pi<pizhenwei@bytedance.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "monitor/hmp.h" +#include "monitor/monitor.h" +#include "qapi/qapi-commands-cryptodev.h" +#include "qapi/qmp/qdict.h" + + +void hmp_info_cryptodev(Monitor *mon, const QDict *qdict) +{ + QCryptodevInfoList *il; + QCryptodevBackendServiceTypeList *sl; + QCryptodevBackendClientList *cl; + + for (il = qmp_query_cryptodev(NULL); il; il = il->next) { + g_autofree char *services = NULL; + QCryptodevInfo *info = il->value; + char *tmp_services; + + /* build a string like 'service=[akcipher|mac|hash|cipher]' */ + for (sl = info->service; sl; sl = sl->next) { + const char *service = QCryptodevBackendServiceType_str(sl->value); + + if (!services) { + services = g_strdup(service); + } else { + tmp_services = g_strjoin("|", services, service, NULL); + g_free(services); + services = tmp_services; + } + } + monitor_printf(mon, "%s: service=[%s]\n", info->id, services); + + for (cl = info->client; cl; cl = cl->next) { + QCryptodevBackendClient *client = cl->value; + monitor_printf(mon, " queue %" PRIu32 ": type=%s\n", + client->queue, + QCryptodevBackendType_str(client->type)); + } + } + + qapi_free_QCryptodevInfoList(il); +} diff --git a/backends/cryptodev-lkcf.c b/backends/cryptodev-lkcf.c index 133bd706a4..45aba1ff67 100644 --- a/backends/cryptodev-lkcf.c +++ b/backends/cryptodev-lkcf.c @@ -223,14 +223,14 @@ static void cryptodev_lkcf_init(CryptoDevBackend *backend, Error **errp) return; } - cc = cryptodev_backend_new_client("cryptodev-lkcf", NULL); + cc = cryptodev_backend_new_client(); cc->info_str = g_strdup_printf("cryptodev-lkcf0"); cc->queue_index = 0; - cc->type = CRYPTODEV_BACKEND_TYPE_LKCF; + cc->type = QCRYPTODEV_BACKEND_TYPE_LKCF; backend->conf.peers.ccs[0] = cc; backend->conf.crypto_services = - 1u << VIRTIO_CRYPTO_SERVICE_AKCIPHER; + 1u << QCRYPTODEV_BACKEND_SERVICE_AKCIPHER; backend->conf.akcipher_algo = 1u << VIRTIO_CRYPTO_AKCIPHER_RSA; lkcf->running = true; @@ -469,15 +469,12 @@ static void *cryptodev_lkcf_worker(void *arg) static int cryptodev_lkcf_operation( CryptoDevBackend *backend, - CryptoDevBackendOpInfo *op_info, - uint32_t queue_index, - CryptoDevCompletionFunc cb, - void *opaque) + CryptoDevBackendOpInfo *op_info) { CryptoDevBackendLKCF *lkcf = CRYPTODEV_BACKEND_LKCF(backend); CryptoDevBackendLKCFSession *sess; - enum CryptoDevBackendAlgType algtype = op_info->algtype; + QCryptodevBackendAlgType algtype = op_info->algtype; CryptoDevLKCFTask *task; if (op_info->session_id >= MAX_SESSIONS || @@ -488,15 +485,15 @@ static int cryptodev_lkcf_operation( } sess = lkcf->sess[op_info->session_id]; - if (algtype != CRYPTODEV_BACKEND_ALG_ASYM) { + if (algtype != QCRYPTODEV_BACKEND_ALG_ASYM) { error_report("algtype not supported: %u", algtype); return -VIRTIO_CRYPTO_NOTSUPP; } task = g_new0(CryptoDevLKCFTask, 1); task->op_info = op_info; - task->cb = cb; - task->opaque = opaque; + task->cb = op_info->cb; + task->opaque = op_info->opaque; task->sess = sess; task->lkcf = lkcf; task->status = -VIRTIO_CRYPTO_ERR; diff --git a/backends/cryptodev-vhost-user.c b/backends/cryptodev-vhost-user.c index ab3028e045..b1d9eb735f 100644 --- a/backends/cryptodev-vhost-user.c +++ b/backends/cryptodev-vhost-user.c @@ -67,7 +67,7 @@ cryptodev_vhost_user_get_vhost( { CryptoDevBackendVhostUser *s = CRYPTODEV_BACKEND_VHOST_USER(b); - assert(cc->type == CRYPTODEV_BACKEND_TYPE_VHOST_USER); + assert(cc->type == QCRYPTODEV_BACKEND_TYPE_VHOST_USER); assert(queue < MAX_CRYPTO_QUEUE_NUM); return s->vhost_crypto[queue]; @@ -198,12 +198,11 @@ static void cryptodev_vhost_user_init( s->opened = true; for (i = 0; i < queues; i++) { - cc = cryptodev_backend_new_client( - "cryptodev-vhost-user", NULL); + cc = cryptodev_backend_new_client(); cc->info_str = g_strdup_printf("cryptodev-vhost-user%zu to %s ", i, chr->label); cc->queue_index = i; - cc->type = CRYPTODEV_BACKEND_TYPE_VHOST_USER; + cc->type = QCRYPTODEV_BACKEND_TYPE_VHOST_USER; backend->conf.peers.ccs[i] = cc; @@ -222,9 +221,9 @@ static void cryptodev_vhost_user_init( cryptodev_vhost_user_event, NULL, s, NULL, true); backend->conf.crypto_services = - 1u << VIRTIO_CRYPTO_SERVICE_CIPHER | - 1u << VIRTIO_CRYPTO_SERVICE_HASH | - 1u << VIRTIO_CRYPTO_SERVICE_MAC; + 1u << QCRYPTODEV_BACKEND_SERVICE_CIPHER | + 1u << QCRYPTODEV_BACKEND_SERVICE_HASH | + 1u << QCRYPTODEV_BACKEND_SERVICE_MAC; backend->conf.cipher_algo_l = 1u << VIRTIO_CRYPTO_CIPHER_AES_CBC; backend->conf.hash_algo = 1u << VIRTIO_CRYPTO_HASH_SHA1; diff --git a/backends/cryptodev-vhost.c b/backends/cryptodev-vhost.c index 74ea0ad63d..93523732f3 100644 --- a/backends/cryptodev-vhost.c +++ b/backends/cryptodev-vhost.c @@ -127,7 +127,7 @@ cryptodev_get_vhost(CryptoDevBackendClient *cc, switch (cc->type) { #if defined(CONFIG_VHOST_USER) && defined(CONFIG_LINUX) - case CRYPTODEV_BACKEND_TYPE_VHOST_USER: + case QCRYPTODEV_BACKEND_TYPE_VHOST_USER: vhost_crypto = cryptodev_vhost_user_get_vhost(cc, b, queue); break; #endif @@ -195,7 +195,7 @@ int cryptodev_vhost_start(VirtIODevice *dev, int total_queues) * because vhost user doesn't interrupt masking/unmasking * properly. */ - if (cc->type == CRYPTODEV_BACKEND_TYPE_VHOST_USER) { + if (cc->type == QCRYPTODEV_BACKEND_TYPE_VHOST_USER) { dev->use_guest_notifier_mask = false; } } diff --git a/backends/cryptodev.c b/backends/cryptodev.c index 54ee8c81f5..94ca393cee 100644 --- a/backends/cryptodev.c +++ b/backends/cryptodev.c @@ -23,29 +23,92 @@ #include "qemu/osdep.h" #include "sysemu/cryptodev.h" +#include "sysemu/stats.h" #include "qapi/error.h" +#include "qapi/qapi-commands-cryptodev.h" +#include "qapi/qapi-types-stats.h" #include "qapi/visitor.h" #include "qemu/config-file.h" #include "qemu/error-report.h" +#include "qemu/main-loop.h" #include "qom/object_interfaces.h" #include "hw/virtio/virtio-crypto.h" +#define SYM_ENCRYPT_OPS_STR "sym-encrypt-ops" +#define SYM_DECRYPT_OPS_STR "sym-decrypt-ops" +#define SYM_ENCRYPT_BYTES_STR "sym-encrypt-bytes" +#define SYM_DECRYPT_BYTES_STR "sym-decrypt-bytes" + +#define ASYM_ENCRYPT_OPS_STR "asym-encrypt-ops" +#define ASYM_DECRYPT_OPS_STR "asym-decrypt-ops" +#define ASYM_SIGN_OPS_STR "asym-sign-ops" +#define ASYM_VERIFY_OPS_STR "asym-verify-ops" +#define ASYM_ENCRYPT_BYTES_STR "asym-encrypt-bytes" +#define ASYM_DECRYPT_BYTES_STR "asym-decrypt-bytes" +#define ASYM_SIGN_BYTES_STR "asym-sign-bytes" +#define ASYM_VERIFY_BYTES_STR "asym-verify-bytes" + +typedef struct StatsArgs { + union StatsResultsType { + StatsResultList **stats; + StatsSchemaList **schema; + } result; + strList *names; + Error **errp; +} StatsArgs; static QTAILQ_HEAD(, CryptoDevBackendClient) crypto_clients; +static int qmp_query_cryptodev_foreach(Object *obj, void *data) +{ + CryptoDevBackend *backend; + QCryptodevInfoList **infolist = data; + uint32_t services, i; + + if (!object_dynamic_cast(obj, TYPE_CRYPTODEV_BACKEND)) { + return 0; + } + + QCryptodevInfo *info = g_new0(QCryptodevInfo, 1); + info->id = g_strdup(object_get_canonical_path_component(obj)); + + backend = CRYPTODEV_BACKEND(obj); + services = backend->conf.crypto_services; + for (i = 0; i < QCRYPTODEV_BACKEND_SERVICE__MAX; i++) { + if (services & (1 << i)) { + QAPI_LIST_PREPEND(info->service, i); + } + } + + for (i = 0; i < backend->conf.peers.queues; i++) { + CryptoDevBackendClient *cc = backend->conf.peers.ccs[i]; + QCryptodevBackendClient *client = g_new0(QCryptodevBackendClient, 1); + + client->queue = cc->queue_index; + client->type = cc->type; + QAPI_LIST_PREPEND(info->client, client); + } + + QAPI_LIST_PREPEND(*infolist, info); + + return 0; +} -CryptoDevBackendClient * -cryptodev_backend_new_client(const char *model, - const char *name) +QCryptodevInfoList *qmp_query_cryptodev(Error **errp) +{ + QCryptodevInfoList *list = NULL; + Object *objs = container_get(object_get_root(), "/objects"); + + object_child_foreach(objs, qmp_query_cryptodev_foreach, &list); + + return list; +} + +CryptoDevBackendClient *cryptodev_backend_new_client(void) { CryptoDevBackendClient *cc; cc = g_new0(CryptoDevBackendClient, 1); - cc->model = g_strdup(model); - if (name) { - cc->name = g_strdup(name); - } - QTAILQ_INSERT_TAIL(&crypto_clients, cc, next); return cc; @@ -55,8 +118,6 @@ void cryptodev_backend_free_client( CryptoDevBackendClient *cc) { QTAILQ_REMOVE(&crypto_clients, cc, next); - g_free(cc->name); - g_free(cc->model); g_free(cc->info_str); g_free(cc); } @@ -71,6 +132,9 @@ void cryptodev_backend_cleanup( if (bc->cleanup) { bc->cleanup(backend, errp); } + + g_free(backend->sym_stat); + g_free(backend->asym_stat); } int cryptodev_backend_create_session( @@ -107,38 +171,111 @@ int cryptodev_backend_close_session( static int cryptodev_backend_operation( CryptoDevBackend *backend, - CryptoDevBackendOpInfo *op_info, - uint32_t queue_index, - CryptoDevCompletionFunc cb, - void *opaque) + CryptoDevBackendOpInfo *op_info) { CryptoDevBackendClass *bc = CRYPTODEV_BACKEND_GET_CLASS(backend); if (bc->do_op) { - return bc->do_op(backend, op_info, queue_index, cb, opaque); + return bc->do_op(backend, op_info); } return -VIRTIO_CRYPTO_NOTSUPP; } +static int cryptodev_backend_account(CryptoDevBackend *backend, + CryptoDevBackendOpInfo *op_info) +{ + enum QCryptodevBackendAlgType algtype = op_info->algtype; + int len; + + if (algtype == QCRYPTODEV_BACKEND_ALG_ASYM) { + CryptoDevBackendAsymOpInfo *asym_op_info = op_info->u.asym_op_info; + len = asym_op_info->src_len; + switch (op_info->op_code) { + case VIRTIO_CRYPTO_AKCIPHER_ENCRYPT: + CryptodevAsymStatIncEncrypt(backend, len); + break; + case VIRTIO_CRYPTO_AKCIPHER_DECRYPT: + CryptodevAsymStatIncDecrypt(backend, len); + break; + case VIRTIO_CRYPTO_AKCIPHER_SIGN: + CryptodevAsymStatIncSign(backend, len); + break; + case VIRTIO_CRYPTO_AKCIPHER_VERIFY: + CryptodevAsymStatIncVerify(backend, len); + break; + default: + return -VIRTIO_CRYPTO_NOTSUPP; + } + } else if (algtype == QCRYPTODEV_BACKEND_ALG_SYM) { + CryptoDevBackendSymOpInfo *sym_op_info = op_info->u.sym_op_info; + len = sym_op_info->src_len; + switch (op_info->op_code) { + case VIRTIO_CRYPTO_CIPHER_ENCRYPT: + CryptodevSymStatIncEncrypt(backend, len); + break; + case VIRTIO_CRYPTO_CIPHER_DECRYPT: + CryptodevSymStatIncDecrypt(backend, len); + break; + default: + return -VIRTIO_CRYPTO_NOTSUPP; + } + } else { + error_report("Unsupported cryptodev alg type: %" PRIu32 "", algtype); + return -VIRTIO_CRYPTO_NOTSUPP; + } + + return len; +} + +static void cryptodev_backend_throttle_timer_cb(void *opaque) +{ + CryptoDevBackend *backend = (CryptoDevBackend *)opaque; + CryptoDevBackendOpInfo *op_info, *tmpop; + int ret; + + QTAILQ_FOREACH_SAFE(op_info, &backend->opinfos, next, tmpop) { + QTAILQ_REMOVE(&backend->opinfos, op_info, next); + ret = cryptodev_backend_account(backend, op_info); + if (ret < 0) { + op_info->cb(op_info->opaque, ret); + continue; + } + + throttle_account(&backend->ts, true, ret); + cryptodev_backend_operation(backend, op_info); + if (throttle_enabled(&backend->tc) && + throttle_schedule_timer(&backend->ts, &backend->tt, true)) { + break; + } + } +} + int cryptodev_backend_crypto_operation( CryptoDevBackend *backend, - void *opaque1, - uint32_t queue_index, - CryptoDevCompletionFunc cb, void *opaque2) + CryptoDevBackendOpInfo *op_info) { - VirtIOCryptoReq *req = opaque1; - CryptoDevBackendOpInfo *op_info = &req->op_info; - enum CryptoDevBackendAlgType algtype = req->flags; + int ret; - if ((algtype != CRYPTODEV_BACKEND_ALG_SYM) - && (algtype != CRYPTODEV_BACKEND_ALG_ASYM)) { - error_report("Unsupported cryptodev alg type: %" PRIu32 "", algtype); - return -VIRTIO_CRYPTO_NOTSUPP; + if (!throttle_enabled(&backend->tc)) { + goto do_account; + } + + if (throttle_schedule_timer(&backend->ts, &backend->tt, true) || + !QTAILQ_EMPTY(&backend->opinfos)) { + QTAILQ_INSERT_TAIL(&backend->opinfos, op_info, next); + return 0; } - return cryptodev_backend_operation(backend, op_info, queue_index, - cb, opaque2); +do_account: + ret = cryptodev_backend_account(backend, op_info); + if (ret < 0) { + return ret; + } + + throttle_account(&backend->ts, true, ret); + + return cryptodev_backend_operation(backend, op_info); } static void @@ -169,15 +306,111 @@ cryptodev_backend_set_queues(Object *obj, Visitor *v, const char *name, backend->conf.peers.queues = value; } +static void cryptodev_backend_set_throttle(CryptoDevBackend *backend, int field, + uint64_t value, Error **errp) +{ + uint64_t orig = backend->tc.buckets[field].avg; + bool enabled = throttle_enabled(&backend->tc); + + if (orig == value) { + return; + } + + backend->tc.buckets[field].avg = value; + if (!throttle_enabled(&backend->tc)) { + throttle_timers_destroy(&backend->tt); + cryptodev_backend_throttle_timer_cb(backend); /* drain opinfos */ + return; + } + + if (!throttle_is_valid(&backend->tc, errp)) { + backend->tc.buckets[field].avg = orig; /* revert change */ + return; + } + + if (!enabled) { + throttle_init(&backend->ts); + throttle_timers_init(&backend->tt, qemu_get_aio_context(), + QEMU_CLOCK_REALTIME, + cryptodev_backend_throttle_timer_cb, /* FIXME */ + cryptodev_backend_throttle_timer_cb, backend); + } + + throttle_config(&backend->ts, QEMU_CLOCK_REALTIME, &backend->tc); +} + +static void cryptodev_backend_get_bps(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + CryptoDevBackend *backend = CRYPTODEV_BACKEND(obj); + uint64_t value = backend->tc.buckets[THROTTLE_BPS_TOTAL].avg; + + visit_type_uint64(v, name, &value, errp); +} + +static void cryptodev_backend_set_bps(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + CryptoDevBackend *backend = CRYPTODEV_BACKEND(obj); + uint64_t value; + + if (!visit_type_uint64(v, name, &value, errp)) { + return; + } + + cryptodev_backend_set_throttle(backend, THROTTLE_BPS_TOTAL, value, errp); +} + +static void cryptodev_backend_get_ops(Object *obj, Visitor *v, const char *name, + void *opaque, Error **errp) +{ + CryptoDevBackend *backend = CRYPTODEV_BACKEND(obj); + uint64_t value = backend->tc.buckets[THROTTLE_OPS_TOTAL].avg; + + visit_type_uint64(v, name, &value, errp); +} + +static void cryptodev_backend_set_ops(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + CryptoDevBackend *backend = CRYPTODEV_BACKEND(obj); + uint64_t value; + + if (!visit_type_uint64(v, name, &value, errp)) { + return; + } + + cryptodev_backend_set_throttle(backend, THROTTLE_OPS_TOTAL, value, errp); +} + static void cryptodev_backend_complete(UserCreatable *uc, Error **errp) { CryptoDevBackend *backend = CRYPTODEV_BACKEND(uc); CryptoDevBackendClass *bc = CRYPTODEV_BACKEND_GET_CLASS(uc); + uint32_t services; + uint64_t value; + + QTAILQ_INIT(&backend->opinfos); + value = backend->tc.buckets[THROTTLE_OPS_TOTAL].avg; + cryptodev_backend_set_throttle(backend, THROTTLE_OPS_TOTAL, value, errp); + value = backend->tc.buckets[THROTTLE_BPS_TOTAL].avg; + cryptodev_backend_set_throttle(backend, THROTTLE_BPS_TOTAL, value, errp); if (bc->init) { bc->init(backend, errp); } + + services = backend->conf.crypto_services; + if (services & (1 << QCRYPTODEV_BACKEND_SERVICE_CIPHER)) { + backend->sym_stat = g_new0(CryptodevBackendSymStat, 1); + } + + if (services & (1 << QCRYPTODEV_BACKEND_SERVICE_AKCIPHER)) { + backend->asym_stat = g_new0(CryptodevBackendAsymStat, 1); + } } void cryptodev_backend_set_used(CryptoDevBackend *backend, bool used) @@ -208,8 +441,12 @@ cryptodev_backend_can_be_deleted(UserCreatable *uc) static void cryptodev_backend_instance_init(Object *obj) { + CryptoDevBackend *backend = CRYPTODEV_BACKEND(obj); + /* Initialize devices' queues property to 1 */ object_property_set_int(obj, "queues", 1, NULL); + + throttle_config_init(&backend->tc); } static void cryptodev_backend_finalize(Object *obj) @@ -217,6 +454,137 @@ static void cryptodev_backend_finalize(Object *obj) CryptoDevBackend *backend = CRYPTODEV_BACKEND(obj); cryptodev_backend_cleanup(backend, NULL); + if (throttle_enabled(&backend->tc)) { + throttle_timers_destroy(&backend->tt); + } +} + +static StatsList *cryptodev_backend_stats_add(const char *name, int64_t *val, + StatsList *stats_list) +{ + Stats *stats = g_new0(Stats, 1); + + stats->name = g_strdup(name); + stats->value = g_new0(StatsValue, 1); + stats->value->type = QTYPE_QNUM; + stats->value->u.scalar = *val; + + QAPI_LIST_PREPEND(stats_list, stats); + return stats_list; +} + +static int cryptodev_backend_stats_query(Object *obj, void *data) +{ + StatsArgs *stats_args = data; + StatsResultList **stats_results = stats_args->result.stats; + StatsList *stats_list = NULL; + StatsResult *entry; + CryptoDevBackend *backend; + CryptodevBackendSymStat *sym_stat; + CryptodevBackendAsymStat *asym_stat; + + if (!object_dynamic_cast(obj, TYPE_CRYPTODEV_BACKEND)) { + return 0; + } + + backend = CRYPTODEV_BACKEND(obj); + sym_stat = backend->sym_stat; + if (sym_stat) { + stats_list = cryptodev_backend_stats_add(SYM_ENCRYPT_OPS_STR, + &sym_stat->encrypt_ops, stats_list); + stats_list = cryptodev_backend_stats_add(SYM_DECRYPT_OPS_STR, + &sym_stat->decrypt_ops, stats_list); + stats_list = cryptodev_backend_stats_add(SYM_ENCRYPT_BYTES_STR, + &sym_stat->encrypt_bytes, stats_list); + stats_list = cryptodev_backend_stats_add(SYM_DECRYPT_BYTES_STR, + &sym_stat->decrypt_bytes, stats_list); + } + + asym_stat = backend->asym_stat; + if (asym_stat) { + stats_list = cryptodev_backend_stats_add(ASYM_ENCRYPT_OPS_STR, + &asym_stat->encrypt_ops, stats_list); + stats_list = cryptodev_backend_stats_add(ASYM_DECRYPT_OPS_STR, + &asym_stat->decrypt_ops, stats_list); + stats_list = cryptodev_backend_stats_add(ASYM_SIGN_OPS_STR, + &asym_stat->sign_ops, stats_list); + stats_list = cryptodev_backend_stats_add(ASYM_VERIFY_OPS_STR, + &asym_stat->verify_ops, stats_list); + stats_list = cryptodev_backend_stats_add(ASYM_ENCRYPT_BYTES_STR, + &asym_stat->encrypt_bytes, stats_list); + stats_list = cryptodev_backend_stats_add(ASYM_DECRYPT_BYTES_STR, + &asym_stat->decrypt_bytes, stats_list); + stats_list = cryptodev_backend_stats_add(ASYM_SIGN_BYTES_STR, + &asym_stat->sign_bytes, stats_list); + stats_list = cryptodev_backend_stats_add(ASYM_VERIFY_BYTES_STR, + &asym_stat->verify_bytes, stats_list); + } + + entry = g_new0(StatsResult, 1); + entry->provider = STATS_PROVIDER_CRYPTODEV; + entry->qom_path = g_strdup(object_get_canonical_path(obj)); + entry->stats = stats_list; + QAPI_LIST_PREPEND(*stats_results, entry); + + return 0; +} + +static void cryptodev_backend_stats_cb(StatsResultList **result, + StatsTarget target, + strList *names, strList *targets, + Error **errp) +{ + switch (target) { + case STATS_TARGET_CRYPTODEV: + { + Object *objs = container_get(object_get_root(), "/objects"); + StatsArgs stats_args; + stats_args.result.stats = result; + stats_args.names = names; + stats_args.errp = errp; + + object_child_foreach(objs, cryptodev_backend_stats_query, &stats_args); + break; + } + default: + break; + } +} + +static StatsSchemaValueList *cryptodev_backend_schemas_add(const char *name, + StatsSchemaValueList *list) +{ + StatsSchemaValueList *schema_entry = g_new0(StatsSchemaValueList, 1); + + schema_entry->value = g_new0(StatsSchemaValue, 1); + schema_entry->value->type = STATS_TYPE_CUMULATIVE; + schema_entry->value->name = g_strdup(name); + schema_entry->next = list; + + return schema_entry; +} + +static void cryptodev_backend_schemas_cb(StatsSchemaList **result, + Error **errp) +{ + StatsSchemaValueList *stats_list = NULL; + const char *sym_stats[] = { SYM_ENCRYPT_OPS_STR, SYM_DECRYPT_OPS_STR, + SYM_ENCRYPT_BYTES_STR, SYM_DECRYPT_BYTES_STR }; + const char *asym_stats[] = { ASYM_ENCRYPT_OPS_STR, ASYM_DECRYPT_OPS_STR, + ASYM_SIGN_OPS_STR, ASYM_VERIFY_OPS_STR, + ASYM_ENCRYPT_BYTES_STR, ASYM_DECRYPT_BYTES_STR, + ASYM_SIGN_BYTES_STR, ASYM_VERIFY_BYTES_STR }; + + for (int i = 0; i < ARRAY_SIZE(sym_stats); i++) { + stats_list = cryptodev_backend_schemas_add(sym_stats[i], stats_list); + } + + for (int i = 0; i < ARRAY_SIZE(asym_stats); i++) { + stats_list = cryptodev_backend_schemas_add(asym_stats[i], stats_list); + } + + add_stats_schema(result, STATS_PROVIDER_CRYPTODEV, STATS_TARGET_CRYPTODEV, + stats_list); } static void @@ -232,6 +600,17 @@ cryptodev_backend_class_init(ObjectClass *oc, void *data) cryptodev_backend_get_queues, cryptodev_backend_set_queues, NULL, NULL); + object_class_property_add(oc, "throttle-bps", "uint64", + cryptodev_backend_get_bps, + cryptodev_backend_set_bps, + NULL, NULL); + object_class_property_add(oc, "throttle-ops", "uint64", + cryptodev_backend_get_ops, + cryptodev_backend_set_ops, + NULL, NULL); + + add_stats_callbacks(STATS_PROVIDER_CRYPTODEV, cryptodev_backend_stats_cb, + cryptodev_backend_schemas_cb); } static const TypeInfo cryptodev_backend_info = { diff --git a/backends/meson.build b/backends/meson.build index 954e658b25..b369e0a9d0 100644 --- a/backends/meson.build +++ b/backends/meson.build @@ -1,5 +1,6 @@ softmmu_ss.add([files( 'cryptodev-builtin.c', + 'cryptodev-hmp-cmds.c', 'cryptodev.c', 'hostmem-ram.c', 'hostmem.c', diff --git a/contrib/gitdm/domain-map b/contrib/gitdm/domain-map index 3727918641..8913a886c9 100644 --- a/contrib/gitdm/domain-map +++ b/contrib/gitdm/domain-map @@ -12,6 +12,7 @@ citrix.com Citrix crudebyte.com Crudebyte chinatelecom.cn China Telecom eldorado.org.br Instituto de Pesquisas Eldorado +fb.com Facebook fujitsu.com Fujitsu google.com Google greensocs.com GreenSocs @@ -37,7 +38,9 @@ siemens.com Siemens sifive.com SiFive suse.com SUSE suse.de SUSE +ventanamicro.com Ventana Micro Systems virtuozzo.com Virtuozzo +vrull.eu VRULL wdc.com Western Digital windriver.com Wind River xilinx.com Xilinx diff --git a/contrib/gitdm/group-map-facebook b/contrib/gitdm/group-map-facebook new file mode 100644 index 0000000000..38589f8fb9 --- /dev/null +++ b/contrib/gitdm/group-map-facebook @@ -0,0 +1,5 @@ +# +# Some Facebook contributors also occasionally use personal email addresses. +# + +peter@pjd.dev diff --git a/contrib/gitdm/group-map-ibm b/contrib/gitdm/group-map-ibm index da62fa3f44..24d8dc1b86 100644 --- a/contrib/gitdm/group-map-ibm +++ b/contrib/gitdm/group-map-ibm @@ -12,3 +12,4 @@ jcfaracco@gmail.com joel@jms.id.au sjitindarsingh@gmail.com tommusta@gmail.com +idan.horowitz@gmail.com diff --git a/contrib/gitdm/group-map-individuals b/contrib/gitdm/group-map-individuals index 53883cc526..e2263a5ee3 100644 --- a/contrib/gitdm/group-map-individuals +++ b/contrib/gitdm/group-map-individuals @@ -37,3 +37,4 @@ akihiko.odaki@gmail.com paul@nowt.org git@xen0n.name simon@simonsafar.com +research_trasio@irq.a4lg.com diff --git a/docs/system/device-emulation.rst b/docs/system/device-emulation.rst index 0506006056..c1b1934e3d 100644 --- a/docs/system/device-emulation.rst +++ b/docs/system/device-emulation.rst @@ -93,3 +93,4 @@ Emulated Devices devices/virtio-pmem.rst devices/vhost-user-rng.rst devices/canokey.rst + devices/igb.rst diff --git a/docs/system/devices/igb.rst b/docs/system/devices/igb.rst new file mode 100644 index 0000000000..70edadd574 --- /dev/null +++ b/docs/system/devices/igb.rst @@ -0,0 +1,71 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later +.. _igb: + +igb +--- + +igb is a family of Intel's gigabit ethernet controllers. In QEMU, 82576 +emulation is implemented in particular. Its datasheet is available at [1]_. + +This implementation is expected to be useful to test SR-IOV networking without +requiring physical hardware. + +Limitations +=========== + +This igb implementation was tested with Linux Test Project [2]_ and Windows HLK +[3]_ during the initial development. The command used when testing with LTP is: + +.. code-block:: shell + + network.sh -6mta + +Be aware that this implementation lacks many functionalities available with the +actual hardware, and you may experience various failures if you try to use it +with a different operating system other than Linux and Windows or if you try +functionalities not covered by the tests. + +Using igb +========= + +Using igb should be nothing different from using another network device. See +:ref:`pcsys_005fnetwork` in general. + +However, you may also need to perform additional steps to activate SR-IOV +feature on your guest. For Linux, refer to [4]_. + +Developing igb +============== + +igb is the successor of e1000e, and e1000e is the successor of e1000 in turn. +As these devices are very similar, if you make a change for igb and the same +change can be applied to e1000e and e1000, please do so. + +Please do not forget to run tests before submitting a change. As tests included +in QEMU is very minimal, run some application which is likely to be affected by +the change to confirm it works in an integrated system. + +Testing igb +=========== + +A qtest of the basic functionality is available. Run the below at the build +directory: + +.. code-block:: shell + + meson test qtest-x86_64/qos-test + +ethtool can test register accesses, interrupts, etc. It is automated as an +Avocado test and can be ran with the following command: + +.. code:: shell + + make check-avocado AVOCADO_TESTS=tests/avocado/igb.py + +References +========== + +.. [1] https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eb-gigabit-ethernet-controller-datasheet.pdf +.. [2] https://github.com/linux-test-project/ltp +.. [3] https://learn.microsoft.com/en-us/windows-hardware/test/hlk/ +.. [4] https://docs.kernel.org/PCI/pci-iov-howto.html diff --git a/ebpf/rss.bpf.skeleton.h b/ebpf/rss.bpf.skeleton.h index 126683eb87..18eb2adb12 100644 --- a/ebpf/rss.bpf.skeleton.h +++ b/ebpf/rss.bpf.skeleton.h @@ -1,9 +1,10 @@ /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* THIS FILE IS AUTOGENERATED! */ +/* THIS FILE IS AUTOGENERATED BY BPFTOOL! */ #ifndef __RSS_BPF_SKEL_H__ #define __RSS_BPF_SKEL_H__ +#include <errno.h> #include <stdlib.h> #include <bpf/libbpf.h> @@ -12,8 +13,8 @@ struct rss_bpf { struct bpf_object *obj; struct { struct bpf_map *tap_rss_map_configurations; - struct bpf_map *tap_rss_map_indirection_table; struct bpf_map *tap_rss_map_toeplitz_key; + struct bpf_map *tap_rss_map_indirection_table; } maps; struct { struct bpf_program *tun_rss_steering_prog; @@ -21,6 +22,16 @@ struct rss_bpf { struct { struct bpf_link *tun_rss_steering_prog; } links; + +#ifdef __cplusplus + static inline struct rss_bpf *open(const struct bpf_object_open_opts *opts = nullptr); + static inline struct rss_bpf *open_and_load(); + static inline int load(struct rss_bpf *skel); + static inline int attach(struct rss_bpf *skel); + static inline void detach(struct rss_bpf *skel); + static inline void destroy(struct rss_bpf *skel); + static inline const void *elf_bytes(size_t *sz); +#endif /* __cplusplus */ }; static void @@ -40,18 +51,26 @@ static inline struct rss_bpf * rss_bpf__open_opts(const struct bpf_object_open_opts *opts) { struct rss_bpf *obj; + int err; obj = (struct rss_bpf *)calloc(1, sizeof(*obj)); - if (!obj) + if (!obj) { + errno = ENOMEM; return NULL; - if (rss_bpf__create_skeleton(obj)) - goto err; - if (bpf_object__open_skeleton(obj->skeleton, opts)) - goto err; + } + + err = rss_bpf__create_skeleton(obj); + if (err) + goto err_out; + + err = bpf_object__open_skeleton(obj->skeleton, opts); + if (err) + goto err_out; return obj; -err: +err_out: rss_bpf__destroy(obj); + errno = -err; return NULL; } @@ -71,12 +90,15 @@ static inline struct rss_bpf * rss_bpf__open_and_load(void) { struct rss_bpf *obj; + int err; obj = rss_bpf__open(); if (!obj) return NULL; - if (rss_bpf__load(obj)) { + err = rss_bpf__load(obj); + if (err) { rss_bpf__destroy(obj); + errno = -err; return NULL; } return obj; @@ -91,18 +113,22 @@ rss_bpf__attach(struct rss_bpf *obj) static inline void rss_bpf__detach(struct rss_bpf *obj) { - return bpf_object__detach_skeleton(obj->skeleton); + bpf_object__detach_skeleton(obj->skeleton); } +static inline const void *rss_bpf__elf_bytes(size_t *sz); + static inline int rss_bpf__create_skeleton(struct rss_bpf *obj) { struct bpf_object_skeleton *s; + int err; s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s)); - if (!s) - return -1; - obj->skeleton = s; + if (!s) { + err = -ENOMEM; + goto err; + } s->sz = sizeof(*s); s->name = "rss_bpf"; @@ -112,320 +138,855 @@ rss_bpf__create_skeleton(struct rss_bpf *obj) s->map_cnt = 3; s->map_skel_sz = sizeof(*s->maps); s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt, s->map_skel_sz); - if (!s->maps) + if (!s->maps) { + err = -ENOMEM; goto err; + } s->maps[0].name = "tap_rss_map_configurations"; s->maps[0].map = &obj->maps.tap_rss_map_configurations; - s->maps[1].name = "tap_rss_map_indirection_table"; - s->maps[1].map = &obj->maps.tap_rss_map_indirection_table; + s->maps[1].name = "tap_rss_map_toeplitz_key"; + s->maps[1].map = &obj->maps.tap_rss_map_toeplitz_key; - s->maps[2].name = "tap_rss_map_toeplitz_key"; - s->maps[2].map = &obj->maps.tap_rss_map_toeplitz_key; + s->maps[2].name = "tap_rss_map_indirection_table"; + s->maps[2].map = &obj->maps.tap_rss_map_indirection_table; /* programs */ s->prog_cnt = 1; s->prog_skel_sz = sizeof(*s->progs); s->progs = (struct bpf_prog_skeleton *)calloc(s->prog_cnt, s->prog_skel_sz); - if (!s->progs) + if (!s->progs) { + err = -ENOMEM; goto err; + } s->progs[0].name = "tun_rss_steering_prog"; s->progs[0].prog = &obj->progs.tun_rss_steering_prog; s->progs[0].link = &obj->links.tun_rss_steering_prog; - s->data_sz = 8088; - s->data = (void *)"\ + s->data = (void *)rss_bpf__elf_bytes(&s->data_sz); + + obj->skeleton = s; + return 0; +err: + bpf_object__destroy_skeleton(s); + return err; +} + +static inline const void *rss_bpf__elf_bytes(size_t *sz) +{ + *sz = 20440; + return (const void *)"\ \x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\x18\x1d\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0a\0\ -\x01\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x4c\xff\0\0\0\0\xbf\xa7\ -\0\0\0\0\0\0\x07\x07\0\0\x4c\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\x98\x4c\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\ +\x01\0\xbf\x19\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x54\xff\0\0\0\0\xbf\xa7\ +\0\0\0\0\0\0\x07\x07\0\0\x54\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ \xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x06\0\0\0\0\0\0\x18\x01\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\ -\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x06\x66\x02\0\0\0\0\xbf\x79\0\0\ -\0\0\0\0\x15\x09\x64\x02\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\ -\0\x5d\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc0\xff\0\0\0\0\x7b\x1a\xb8\xff\ -\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\ -\0\x63\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\x1a\x88\xff\0\0\0\0\x7b\ -\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\x70\xff\0\0\0\0\x7b\x1a\ -\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\xff\0\0\0\0\x7b\x1a\x50\ -\xff\0\0\0\0\x15\x08\x4c\x02\0\0\0\0\x6b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\ -\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\ +\0\0\0\0\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\ +\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x06\x67\x02\0\0\0\0\xbf\x87\0\0\ +\0\0\0\0\x15\x07\x65\x02\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\ +\0\x5e\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc8\xff\0\0\0\0\x7b\x1a\xc0\xff\ +\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\ +\0\x63\x1a\xa0\xff\0\0\0\0\x7b\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\ +\x1a\x88\xff\0\0\0\0\x7b\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\ +\x70\xff\0\0\0\0\x7b\x1a\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\ +\xff\0\0\0\0\x15\x09\x4d\x02\0\0\0\0\x6b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\ +\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\ \x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\ -\x77\0\0\0\x20\0\0\0\x55\0\x11\0\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\xff\ -\0\0\0\0\xbf\x13\0\0\0\0\0\0\xdc\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\x55\ -\x03\x0c\0\xa8\x88\0\0\xb7\x02\0\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\ -\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\ -\x85\0\0\0\x44\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\ -\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\x2f\x02\0\0\0\0\x15\x01\x2e\x02\0\0\0\0\x7b\ -\x9a\x30\xff\0\0\0\0\x15\x01\x57\0\x86\xdd\0\0\x55\x01\x3b\0\x08\0\0\0\x7b\x7a\ -\x20\xff\0\0\0\0\xb7\x07\0\0\x01\0\0\0\x73\x7a\x50\xff\0\0\0\0\xb7\x01\0\0\0\0\ -\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\ -\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\xb7\x02\0\ -\0\0\0\0\0\xb7\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\ -\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x1a\x02\0\0\0\0\x69\xa1\xd6\xff\0\0\ -\0\0\x55\x01\x01\0\0\0\0\0\xb7\x07\0\0\0\0\0\0\x61\xa1\xdc\xff\0\0\0\0\x63\x1a\ -\x5c\xff\0\0\0\0\x61\xa1\xe0\xff\0\0\0\0\x63\x1a\x60\xff\0\0\0\0\x73\x7a\x56\ -\xff\0\0\0\0\x71\xa9\xd9\xff\0\0\0\0\x71\xa1\xd0\xff\0\0\0\0\x67\x01\0\0\x02\0\ -\0\0\x57\x01\0\0\x3c\0\0\0\x7b\x1a\x40\xff\0\0\0\0\x79\xa7\x20\xff\0\0\0\0\xbf\ -\x91\0\0\0\0\0\0\x57\x01\0\0\xff\0\0\0\x15\x01\x19\0\0\0\0\0\x71\xa1\x56\xff\0\ -\0\0\0\x55\x01\x17\0\0\0\0\0\x57\x09\0\0\xff\0\0\0\x15\x09\x7a\x01\x11\0\0\0\ -\x55\x09\x14\0\x06\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x53\xff\0\0\0\0\xb7\x01\ -\0\0\0\0\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\ -\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\ -\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\ -\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\xf4\x01\0\0\0\0\x69\xa1\ -\xd0\xff\0\0\0\0\x6b\x1a\x58\xff\0\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x5a\ -\xff\0\0\0\0\x71\xa1\x50\xff\0\0\0\0\x15\x01\xd4\0\0\0\0\0\x71\x62\x03\0\0\0\0\ -\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\ -\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\ -\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\0\x71\xa2\x53\xff\0\0\0\0\x79\xa0\x30\xff\ -\0\0\0\0\x15\x02\x06\x01\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\ -\x02\x03\x01\0\0\0\0\x61\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\ -\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x69\xa1\x58\xff\0\0\0\0\x6b\x1a\xa8\ -\xff\0\0\0\0\x69\xa1\x5a\xff\0\0\0\0\x6b\x1a\xaa\xff\0\0\0\0\x05\0\x65\x01\0\0\ -\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x51\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\ -\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\ -\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\ -\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\x81\0\0\0\0\0\0\xb7\ -\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\ -\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x10\x01\0\0\0\0\x79\xa1\xe0\ -\xff\0\0\0\0\x63\x1a\x64\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x68\xff\0\0\ -\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\ -\x1a\x60\xff\0\0\0\0\x79\xa1\xe8\xff\0\0\0\0\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\ -\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\xf0\xff\0\0\0\0\x63\x1a\x74\xff\0\ -\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\0\0\0\0\x71\xa9\xd6\xff\0\0\0\0\ -\x25\x09\xff\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x18\x02\0\0\ -\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\ -\xf8\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\xfe\xff\0\0\0\0\xb7\x01\0\0\x28\0\0\ -\0\x7b\x1a\x40\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x8c\xff\xff\xff\x7b\ -\x1a\x18\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x7c\xff\xff\xff\x7b\x1a\ -\x10\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\x28\xff\0\0\0\0\x7b\x7a\x20\xff\0\ -\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xfe\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\ -\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\ -\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\x90\ -\x01\0\0\0\0\xbf\x91\0\0\0\0\0\0\x15\x01\x23\0\x3c\0\0\0\x15\x01\x59\0\x2c\0\0\ -\0\x55\x01\x5a\0\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf8\xff\0\0\0\0\xbf\xa3\ -\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\xa2\x40\xff\0\ -\0\0\0\xb7\x04\0\0\x04\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\ -\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x03\x01\0\0\0\ -\0\x71\xa1\xfa\xff\0\0\0\0\x55\x01\x4b\0\x02\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x55\ -\x01\x49\0\x02\0\0\0\x71\xa1\xfb\xff\0\0\0\0\x55\x01\x47\0\x01\0\0\0\x79\xa2\ -\x40\xff\0\0\0\0\x07\x02\0\0\x08\0\0\0\xbf\x81\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\ -\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\ -\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\xf2\0\0\0\0\0\ -\xb7\x01\0\0\x01\0\0\0\x73\x1a\x55\xff\0\0\0\0\x05\0\x39\0\0\0\0\0\xb7\x01\0\0\ -\0\0\0\0\x6b\x1a\xf8\xff\0\0\0\0\xb7\x09\0\0\x02\0\0\0\xb7\x07\0\0\x1e\0\0\0\ -\x05\0\x0e\0\0\0\0\0\x79\xa2\x38\xff\0\0\0\0\x0f\x29\0\0\0\0\0\0\xbf\x92\0\0\0\ -\0\0\0\x07\x02\0\0\x01\0\0\0\x71\xa3\xff\xff\0\0\0\0\x67\x03\0\0\x03\0\0\0\x2d\ -\x23\x02\0\0\0\0\0\x79\xa7\x20\xff\0\0\0\0\x05\0\x2b\0\0\0\0\0\x07\x07\0\0\xff\ -\xff\xff\xff\xbf\x72\0\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x77\x02\0\0\x20\0\0\0\ -\x15\x02\xf9\xff\0\0\0\0\x7b\x9a\x38\xff\0\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\ -\x19\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\xbf\x81\0\0\0\ -\0\0\0\xbf\x92\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\ -\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\ -\x55\x01\x94\0\0\0\0\0\x71\xa2\xf8\xff\0\0\0\0\x55\x02\x0f\0\xc9\0\0\0\x07\x09\ -\0\0\x02\0\0\0\xbf\x81\0\0\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa3\x10\xff\0\0\0\0\ -\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\ -\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x87\0\0\0\0\0\xb7\ -\x01\0\0\x01\0\0\0\x73\x1a\x54\xff\0\0\0\0\x79\xa7\x20\xff\0\0\0\0\x05\0\x07\0\ -\0\0\0\0\xb7\x09\0\0\x01\0\0\0\x15\x02\xd1\xff\0\0\0\0\x71\xa9\xf9\xff\0\0\0\0\ -\x07\x09\0\0\x02\0\0\0\x05\0\xce\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x56\ -\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\x67\x01\0\0\x03\0\0\0\x79\xa2\x40\xff\0\0\ -\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\0\0\x08\0\0\0\x7b\x2a\x40\xff\0\0\0\0\x71\xa9\ -\xfe\xff\0\0\0\0\x25\x09\x0e\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\ -\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\0\0\0\0\x55\x01\x01\ -\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x79\xa1\x28\xff\0\0\0\0\x07\x01\0\0\x01\0\0\0\ -\x7b\x1a\x28\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\ -\x82\xff\x0b\0\0\0\x05\0\x10\xff\0\0\0\0\x15\x09\xf8\xff\x87\0\0\0\x05\0\xfd\ -\xff\0\0\0\0\x71\xa1\x51\xff\0\0\0\0\x79\xa0\x30\xff\0\0\0\0\x15\x01\x17\x01\0\ -\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\0\0\0\0\0\x4f\ -\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\x01\0\0\x08\0\ -\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\0\x71\xa2\x53\ -\xff\0\0\0\0\x15\x02\x3d\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x10\0\0\0\ -\x15\x02\x3a\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\ -\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\ -\x07\x03\0\0\x7c\xff\xff\xff\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x38\0\0\0\x65\ -\x01\x01\0\xff\xff\xff\xff\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\ -\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\x05\x02\0\0\0\ -\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x65\x01\x01\0\xff\xff\xff\ -\xff\xbf\x43\0\0\0\0\0\0\x61\x21\x04\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\x24\0\ -\0\0\0\0\0\x4f\x41\0\0\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\0\x61\x21\x08\0\0\0\0\0\ -\x61\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\xa8\ -\xff\0\0\0\0\x61\x31\0\0\0\0\0\0\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\0\0\0\ -\x61\x33\x0c\0\0\0\0\0\x69\xa5\x5a\xff\0\0\0\0\x6b\x5a\xc2\xff\0\0\0\0\x69\xa5\ -\x58\xff\0\0\0\0\x6b\x5a\xc0\xff\0\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\0\0\0\0\ -\0\0\x7b\x3a\xb8\xff\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\ -\xb0\xff\0\0\0\0\x05\0\x6b\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x04\0\0\0\ -\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x04\0\0\0\x15\x02\x01\0\0\0\0\0\x05\0\xf7\ -\xfe\0\0\0\0\x57\x01\0\0\x01\0\0\0\x15\x01\xd3\0\0\0\0\0\x61\xa1\x5c\xff\0\0\0\ -\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x05\ -\0\x5e\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x1e\0\0\0\0\0\xbf\x12\0\0\0\0\ -\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x1b\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\ -\0\x5c\xff\xff\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\ -\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x57\x01\0\0\0\x01\0\0\ -\x15\x01\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x6c\ -\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\x05\x02\0\0\0\0\0\ -\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x15\x01\xc3\xff\0\0\0\0\x05\0\ -\xc1\xff\0\0\0\0\xb7\x09\0\0\x3c\0\0\0\x79\xa7\x20\xff\0\0\0\0\x67\0\0\0\x20\0\ -\0\0\x77\0\0\0\x20\0\0\0\x15\0\xa5\xfe\0\0\0\0\x05\0\xb0\0\0\0\0\0\x15\x09\x07\ -\xff\x87\0\0\0\x05\0\xa2\xfe\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\ -\x15\x02\xab\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\ -\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\ -\x07\x03\0\0\x7c\xff\xff\xff\x57\x01\0\0\x40\0\0\0\x15\x01\x01\0\0\0\0\0\xbf\ -\x32\0\0\0\0\0\0\x61\x23\x04\0\0\0\0\0\x67\x03\0\0\x20\0\0\0\x61\x24\0\0\0\0\0\ -\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xa0\xff\0\0\0\0\x61\x23\x08\0\0\0\0\0\x61\x22\ -\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x32\0\0\0\0\0\0\x7b\x2a\xa8\xff\0\0\0\ -\0\x15\x01\x1c\0\0\0\0\0\x71\xa1\x55\xff\0\0\0\0\x15\x01\x1a\0\0\0\0\0\x61\xa1\ -\x98\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\0\0\x4f\x21\0\0\0\0\ -\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x90\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\ -\xa2\x8c\xff\0\0\0\0\x05\0\x19\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x52\xff\ -\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\ -\x03\0\0\xd0\xff\xff\xff\xbf\x81\0\0\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\ -\0\x08\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\ -\0\0\0\x20\0\0\0\x55\0\x7d\0\0\0\0\0\x05\0\x88\xfe\0\0\0\0\xb7\x09\0\0\x2b\0\0\ -\0\x05\0\xc6\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\ -\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x70\xff\0\ -\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x6c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\ -\x1a\xb0\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x07\x07\0\0\x04\0\0\0\x61\x03\0\0\0\0\ -\0\0\xb7\x05\0\0\0\0\0\0\x05\0\x4e\0\0\0\0\0\xaf\x52\0\0\0\0\0\0\xbf\x75\0\0\0\ -\0\0\0\x0f\x15\0\0\0\0\0\0\x71\x55\0\0\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\ -\0\0\0\0\0\x77\0\0\0\x07\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\ -\0\x39\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\ -\x50\0\0\0\0\0\0\x77\0\0\0\x06\0\0\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\ -\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3a\0\0\0\xc7\0\0\0\x3f\0\0\ -\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\ -\0\0\0\x77\0\0\0\x05\0\0\0\x57\0\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\ -\0\0\0\0\x67\0\0\0\x3b\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\ -\0\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x04\0\0\0\x57\0\ -\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3c\0\0\0\xc7\ -\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\ -\x77\0\0\0\x03\0\0\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\ -\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3d\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\ -\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x02\0\0\0\x57\0\0\0\ -\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\ -\0\0\x3e\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x02\0\0\0\0\0\0\xbf\ -\x50\0\0\0\0\0\0\x77\0\0\0\x01\0\0\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\ -\x4f\x03\0\0\0\0\0\0\x57\x04\0\0\x01\0\0\0\x87\x04\0\0\0\0\0\0\x5f\x34\0\0\0\0\ -\0\0\xaf\x42\0\0\0\0\0\0\x57\x05\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x53\0\ -\0\0\0\0\0\x07\x01\0\0\x01\0\0\0\xbf\x25\0\0\0\0\0\0\x15\x01\x0b\0\x24\0\0\0\ -\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\xa0\xff\xff\xff\x0f\x12\0\0\0\0\0\0\x71\x24\0\ -\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x38\0\0\0\xc7\0\0\0\x38\0\0\0\xb7\x02\ -\0\0\0\0\0\0\x65\0\xa9\xff\xff\xff\xff\xff\xbf\x32\0\0\0\0\0\0\x05\0\xa7\xff\0\ -\0\0\0\xbf\x21\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x15\x01\ -\x0e\0\0\0\0\0\x71\x63\x06\0\0\0\0\0\x71\x64\x07\0\0\0\0\0\x67\x04\0\0\x08\0\0\ -\0\x4f\x34\0\0\0\0\0\0\x3f\x41\0\0\0\0\0\0\x2f\x41\0\0\0\0\0\0\x1f\x12\0\0\0\0\ -\0\0\x63\x2a\x50\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x50\xff\xff\xff\ -\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\x55\0\x05\0\0\0\0\0\ -\x71\x61\x08\0\0\0\0\0\x71\x60\x09\0\0\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\ -\0\0\0\x95\0\0\0\0\0\0\0\x69\0\0\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\x02\0\0\0\x04\ -\0\0\0\x0a\0\0\0\x01\0\0\0\0\0\0\0\x02\0\0\0\x04\0\0\0\x28\0\0\0\x01\0\0\0\0\0\ -\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\x20\x76\x32\0\ -\0\0\0\0\0\x10\0\0\0\0\0\0\0\x01\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\0\0\0\ -\x18\0\0\0\0\0\0\0\0\0\0\0\xd8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\xa0\0\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\x60\x02\0\0\0\0\x03\0\x20\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3f\x02\0\0\0\0\ -\x03\0\xd0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xed\x01\0\0\0\0\x03\0\x10\x10\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\xd4\x01\0\0\0\0\x03\0\x20\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\xa3\x01\0\0\0\0\x03\0\xb8\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x63\x01\0\0\0\0\ -\x03\0\x48\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x2a\x01\0\0\0\0\x03\0\x10\x13\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\xe1\0\0\0\0\0\x03\0\xa0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\x2e\x02\0\0\0\0\x03\0\x28\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x68\x02\0\0\0\0\x03\ -\0\xc0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x36\x02\0\0\0\0\x03\0\xc8\x13\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\x22\x01\0\0\0\0\x03\0\xe8\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\x02\x01\0\0\0\0\x03\0\x40\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd9\0\0\0\0\0\x03\0\ -\xf8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x26\x02\0\0\0\0\x03\0\x20\x0e\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\xcc\x01\0\0\0\0\x03\0\x60\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x9b\ -\x01\0\0\0\0\x03\0\xc8\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5b\x01\0\0\0\0\x03\0\ -\x20\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7c\x01\0\0\0\0\x03\0\x48\x08\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\x53\x01\0\0\0\0\x03\0\xb8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\ -\x01\0\0\0\0\x03\0\xe0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x84\x01\0\0\0\0\x03\0\ -\xb8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1e\x02\0\0\0\0\x03\0\xd8\x09\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\xc4\x01\0\0\0\0\x03\0\x70\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x93\ -\x01\0\0\0\0\x03\0\xa8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x74\x01\0\0\0\0\x03\0\ -\xf0\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4b\x01\0\0\0\0\x03\0\0\x0a\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\x12\x01\0\0\0\0\x03\0\x10\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xfa\0\ -\0\0\0\0\x03\0\xc0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x58\x02\0\0\0\0\x03\0\x88\ -\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x16\x02\0\0\0\0\x03\0\xb8\x0a\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\xe5\x01\0\0\0\0\x03\0\xc0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbc\x01\ -\0\0\0\0\x03\0\0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x8b\x01\0\0\0\0\x03\0\x18\x0e\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd1\0\0\0\0\0\x03\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\x50\x02\0\0\0\0\x03\0\x20\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0e\x02\0\0\0\0\ -\x03\0\x48\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6c\x01\0\0\0\0\x03\0\xb0\x04\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\x43\x01\0\0\0\0\x03\0\xc8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\xc9\0\0\0\0\0\x03\0\xf8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x06\x02\0\0\0\0\x03\ -\0\xd0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3b\x01\0\0\0\0\x03\0\x98\x0b\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\xf2\0\0\0\0\0\x03\0\xb8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\ -\x02\0\0\0\0\x03\0\xf0\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xfe\x01\0\0\0\0\x03\0\ -\xf8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdd\x01\0\0\0\0\x03\0\0\x0c\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\xb4\x01\0\0\0\0\x03\0\x30\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0a\ -\x01\0\0\0\0\x03\0\x90\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\xa8\ -\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xd0\x01\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\xf6\x01\0\0\0\0\x03\0\xe0\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xac\x01\0\ -\0\0\0\x03\0\x30\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x33\x01\0\0\0\0\x03\0\x80\x0e\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xea\0\0\0\0\0\x03\0\x98\x0e\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6b\0\0\0\x11\0\x06\ -\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x25\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x14\ -\0\0\0\0\0\0\0\x82\0\0\0\x11\0\x05\0\x28\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x01\0\ -\0\0\x11\0\x05\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x40\0\0\0\x12\0\x03\0\0\0\ -\0\0\0\0\0\0\xd8\x13\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x3a\0\0\0\x50\0\0\ -\0\0\0\0\0\x01\0\0\0\x3c\0\0\0\x80\x13\0\0\0\0\0\0\x01\0\0\0\x3b\0\0\0\x1c\0\0\ -\0\0\0\0\0\x01\0\0\0\x38\0\0\0\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\ +\x77\0\0\0\x20\0\0\0\x55\0\x42\x02\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\ +\xff\0\0\0\0\xbf\x13\0\0\0\0\0\0\xdc\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\ +\x55\x03\x0b\0\xa8\x88\0\0\xb7\x02\0\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\ +\0\xd0\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\ +\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x32\x02\0\ +\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x15\x01\x30\x02\0\0\0\0\x7b\x7a\x38\xff\0\0\0\0\ +\x7b\x9a\x40\xff\0\0\0\0\x15\x01\x55\0\x86\xdd\0\0\x55\x01\x39\0\x08\0\0\0\xb7\ +\x07\0\0\x01\0\0\0\x73\x7a\x58\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xe0\xff\ +\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\ +\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\ +\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\ +\0\x77\0\0\0\x20\0\0\0\x55\0\x1c\x02\0\0\0\0\x69\xa1\xd6\xff\0\0\0\0\x55\x01\ +\x01\0\0\0\0\0\xb7\x07\0\0\0\0\0\0\x61\xa1\xdc\xff\0\0\0\0\x63\x1a\x64\xff\0\0\ +\0\0\x61\xa1\xe0\xff\0\0\0\0\x63\x1a\x68\xff\0\0\0\0\x71\xa9\xd9\xff\0\0\0\0\ +\x73\x7a\x5e\xff\0\0\0\0\x71\xa1\xd0\xff\0\0\0\0\x67\x01\0\0\x02\0\0\0\x57\x01\ +\0\0\x3c\0\0\0\x7b\x1a\x48\xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\x57\x01\0\0\xff\0\0\ +\0\x15\x01\x19\0\0\0\0\0\x57\x07\0\0\xff\0\0\0\x55\x07\x17\0\0\0\0\0\x57\x09\0\ +\0\xff\0\0\0\x15\x09\x5a\x01\x11\0\0\0\x55\x09\x14\0\x06\0\0\0\xb7\x01\0\0\x01\ +\0\0\0\x73\x1a\x5b\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\ +\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\ +\xd0\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\ +\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\ +\0\0\x20\0\0\0\x55\0\xf7\x01\0\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x6b\x1a\x60\xff\0\ +\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x62\xff\0\0\0\0\x71\xa1\x58\xff\0\0\0\0\ +\x15\x01\xdb\0\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\ +\0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\ +\x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\ +\0\x71\xa2\x5b\xff\0\0\0\0\x79\xa0\x38\xff\0\0\0\0\x15\x02\x0c\x01\0\0\0\0\xbf\ +\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\x02\x09\x01\0\0\0\0\x61\xa1\x64\xff\ +\0\0\0\0\x63\x1a\xa8\xff\0\0\0\0\x61\xa1\x68\xff\0\0\0\0\x63\x1a\xac\xff\0\0\0\ +\0\x69\xa1\x60\xff\0\0\0\0\x6b\x1a\xb0\xff\0\0\0\0\x69\xa1\x62\xff\0\0\0\0\x6b\ +\x1a\xb2\xff\0\0\0\0\x05\0\x6b\x01\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x59\ +\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\ +\0\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\ +\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x48\ +\xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\ +\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\ +\x55\0\xfe\0\0\0\0\0\x79\xa1\xe0\xff\0\0\0\0\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\ +\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x64\xff\0\ +\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x68\xff\0\0\0\0\x79\xa1\xe8\xff\0\0\0\0\ +\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\0\0\0\0\x79\xa1\ +\xf0\xff\0\0\0\0\x63\x1a\x7c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x80\xff\ +\0\0\0\0\x71\xa9\xd6\xff\0\0\0\0\x25\x09\x13\x01\x3c\0\0\0\xb7\x01\0\0\x01\0\0\ +\0\x6f\x91\0\0\0\0\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\0\ +\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x0c\x01\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\ +\xfe\xff\0\0\0\0\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x48\xff\0\0\0\0\xbf\xa1\0\0\0\0\ +\0\0\x07\x01\0\0\x94\xff\xff\xff\x7b\x1a\x20\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\ +\x07\x01\0\0\x84\xff\xff\xff\x7b\x1a\x18\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\ +\0\0\0\x18\0\x1c\xb7\x02\0\0\0\0\0\0\x7b\x8a\x28\xff\0\0\0\0\x7b\x2a\x30\xff\0\ +\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xfe\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\ +\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\ +\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\ +\x91\x01\0\0\0\0\xbf\x91\0\0\0\0\0\0\x15\x01\x26\0\x3c\0\0\0\x15\x01\x5f\0\x2c\ +\0\0\0\x55\x01\x60\0\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf8\xff\0\0\0\0\xbf\ +\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa7\x40\xff\0\0\0\0\xbf\x71\0\ +\0\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x04\0\0\0\xb7\x05\0\0\x01\0\0\0\ +\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\ +\0\0\0\x55\x01\x06\x01\0\0\0\0\x71\xa1\xfa\xff\0\0\0\0\x55\x01\x11\0\x02\0\0\0\ +\x71\xa1\xf9\xff\0\0\0\0\x55\x01\x0f\0\x02\0\0\0\x71\xa1\xfb\xff\0\0\0\0\x55\ +\x01\x0d\0\x01\0\0\0\x79\xa2\x48\xff\0\0\0\0\x07\x02\0\0\x08\0\0\0\xbf\x71\0\0\ +\0\0\0\0\x79\xa3\x20\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\ +\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\ +\0\0\0\x55\x01\xf5\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5d\xff\0\0\0\0\x18\ +\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x3c\0\0\0\0\0\xb7\x08\0\0\x02\0\0\ +\0\xb7\x07\0\0\0\0\0\0\x6b\x7a\xf8\xff\0\0\0\0\x05\0\x13\0\0\0\0\0\x0f\x81\0\0\ +\0\0\0\0\xbf\x12\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\x71\xa3\xff\xff\0\0\0\0\x67\ +\x03\0\0\x03\0\0\0\x3d\x32\x09\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x07\x02\0\0\x01\0\ +\0\0\x67\x07\0\0\x20\0\0\0\xbf\x73\0\0\0\0\0\0\x77\x03\0\0\x20\0\0\0\xbf\x27\0\ +\0\0\0\0\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\x1d\0\0\0\x2d\x31\x04\0\0\0\0\0\x79\ +\xa8\x28\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x25\0\0\0\ +\0\0\xbf\x89\0\0\0\0\0\0\x79\xa1\x48\xff\0\0\0\0\x0f\x19\0\0\0\0\0\0\xbf\xa3\0\ +\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\xbf\x92\0\0\0\0\ +\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\ +\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x79\0\0\0\0\0\ +\x71\xa2\xf8\xff\0\0\0\0\x55\x02\x0e\0\xc9\0\0\0\x07\x09\0\0\x02\0\0\0\x79\xa1\ +\x40\xff\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\0\0\xb7\x04\0\0\x10\0\ +\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\ +\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x6c\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\ +\x73\x1a\x5c\xff\0\0\0\0\x05\0\xde\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x15\x02\ +\xcd\xff\0\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x07\x01\0\0\x02\0\0\0\x05\0\xca\xff\0\ +\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5e\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\ +\x67\x01\0\0\x03\0\0\0\x79\xa2\x48\xff\0\0\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\0\0\ +\x08\0\0\0\x7b\x2a\x48\xff\0\0\0\0\x71\xa9\xfe\xff\0\0\0\0\x79\xa2\x30\xff\0\0\ +\0\0\x25\x09\x0c\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x5f\x71\ +\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\ +\xbf\x21\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x7d\ +\xff\x0b\0\0\0\x71\xa7\x5e\xff\0\0\0\0\x05\0\x09\xff\0\0\0\0\x15\x09\xf8\xff\ +\x87\0\0\0\x05\0\xfc\xff\0\0\0\0\x71\xa1\x59\xff\0\0\0\0\x79\xa0\x38\xff\0\0\0\ +\0\x15\x01\x13\x01\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\ +\x02\0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\ +\x67\x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\ +\0\0\0\x71\xa2\x5b\xff\0\0\0\0\x15\x02\x42\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\ +\x02\0\0\x10\0\0\0\x15\x02\x3f\0\0\0\0\0\x57\x01\0\0\x80\0\0\0\xb7\x02\0\0\x10\ +\0\0\0\xb7\x03\0\0\x10\0\0\0\x15\x01\x01\0\0\0\0\0\xb7\x03\0\0\x30\0\0\0\x71\ +\xa4\x5d\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\ +\0\0\x07\x03\0\0\x64\xff\xff\xff\xbf\x34\0\0\0\0\0\0\x15\x01\x02\0\0\0\0\0\xbf\ +\xa4\0\0\0\0\0\0\x07\x04\0\0\x84\xff\xff\xff\x71\xa5\x5c\xff\0\0\0\0\xbf\x31\0\ +\0\0\0\0\0\x15\x05\x01\0\0\0\0\0\xbf\x41\0\0\0\0\0\0\x61\x14\x04\0\0\0\0\0\x67\ +\x04\0\0\x20\0\0\0\x61\x15\0\0\0\0\0\0\x4f\x54\0\0\0\0\0\0\x7b\x4a\xa8\xff\0\0\ +\0\0\x61\x14\x08\0\0\0\0\0\x61\x11\x0c\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x4f\x41\ +\0\0\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x0f\x23\0\0\0\0\0\0\x61\x31\0\0\0\0\0\0\ +\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x62\ +\xff\0\0\0\0\x6b\x5a\xca\xff\0\0\0\0\x69\xa5\x60\xff\0\0\0\0\x6b\x5a\xc8\xff\0\ +\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xc0\xff\0\0\0\0\x67\ +\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\xb8\xff\0\0\0\0\x05\0\x6b\0\0\0\ +\0\0\x71\xa2\x5a\xff\0\0\0\0\x15\x02\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\ +\0\0\x04\0\0\0\x15\x02\x01\0\0\0\0\0\x05\0\xf1\xfe\0\0\0\0\x57\x01\0\0\x01\0\0\ +\0\x15\x01\xd0\0\0\0\0\0\x61\xa1\x64\xff\0\0\0\0\x63\x1a\xa8\xff\0\0\0\0\x61\ +\xa1\x68\xff\0\0\0\0\x63\x1a\xac\xff\0\0\0\0\x05\0\x5e\0\0\0\0\0\xb7\x09\0\0\ +\x3c\0\0\0\x79\xa8\x28\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\ +\0\xac\xff\0\0\0\0\x05\0\xc5\0\0\0\0\0\x71\xa2\x5a\xff\0\0\0\0\x15\x02\x26\0\0\ +\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x23\0\0\0\0\0\x57\x01\ +\0\0\0\x01\0\0\xb7\x02\0\0\x10\0\0\0\xb7\x03\0\0\x10\0\0\0\x15\x01\x01\0\0\0\0\ +\0\xb7\x03\0\0\x30\0\0\0\x71\xa4\x5d\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\ +\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x64\xff\xff\xff\xbf\x34\0\0\0\0\0\ +\0\x15\x01\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x84\xff\xff\xff\x71\ +\xa5\x5c\xff\0\0\0\0\xbf\x31\0\0\0\0\0\0\x15\x05\xbd\xff\0\0\0\0\x05\0\xbb\xff\ +\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5a\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\ +\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\ +\x40\xff\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\ +\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\xa0\0\ +\0\0\0\0\x05\0\xa8\xfe\0\0\0\0\x15\x09\xf3\xfe\x87\0\0\0\x05\0\x83\xff\0\0\0\0\ +\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\x15\x02\x9a\0\0\0\0\0\x57\x01\0\0\ +\x40\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\x03\0\0\x0c\0\0\0\x15\x01\x01\0\0\0\0\0\ +\xb7\x03\0\0\x2c\0\0\0\x71\xa4\x5c\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\ +\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x58\xff\xff\xff\x0f\x23\0\0\0\0\0\0\ +\x61\x32\x04\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x61\x34\0\0\0\0\0\0\x4f\x42\0\0\0\ +\0\0\0\x7b\x2a\xa8\xff\0\0\0\0\x61\x32\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x67\ +\x03\0\0\x20\0\0\0\x4f\x23\0\0\0\0\0\0\x7b\x3a\xb0\xff\0\0\0\0\x71\xa2\x5d\xff\ +\0\0\0\0\x15\x02\x0c\0\0\0\0\0\x15\x01\x0b\0\0\0\0\0\x61\xa1\xa0\xff\0\0\0\0\ +\x67\x01\0\0\x20\0\0\0\x61\xa2\x9c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xc0\ +\xff\0\0\0\0\x61\xa1\x98\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\ +\0\0\x05\0\x0a\0\0\0\0\0\xb7\x09\0\0\x2b\0\0\0\x05\0\xae\xff\0\0\0\0\x61\xa1\ +\x80\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x7c\xff\0\0\0\0\x4f\x21\0\0\0\0\ +\0\0\x7b\x1a\xc0\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\ +\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\xb7\x02\0\0\0\ +\0\0\0\x07\x08\0\0\x04\0\0\0\x61\x03\0\0\0\0\0\0\xb7\x05\0\0\0\0\0\0\xbf\xa1\0\ +\0\0\0\0\0\x07\x01\0\0\xa8\xff\xff\xff\x0f\x21\0\0\0\0\0\0\x71\x14\0\0\0\0\0\0\ +\xbf\x41\0\0\0\0\0\0\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x3f\0\0\0\x5f\x31\0\0\0\ +\0\0\0\xaf\x51\0\0\0\0\0\0\xbf\x85\0\0\0\0\0\0\x0f\x25\0\0\0\0\0\0\x71\x55\0\0\ +\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x07\0\0\0\x4f\x03\ +\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x39\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\ +\x30\0\0\0\0\0\0\xaf\x01\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x06\0\0\0\ +\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\ +\0\0\x67\0\0\0\x3a\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x01\0\0\0\ +\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x05\0\0\0\x57\0\0\0\ +\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3b\0\0\0\xc7\0\0\ +\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x01\0\0\0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\ +\x50\0\0\0\0\0\0\x77\0\0\0\x04\0\0\0\x57\0\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\ +\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3c\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\ +\0\xaf\x01\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x03\0\0\0\x57\0\0\0\x01\0\ +\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\ +\x3d\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x30\0\0\0\0\0\0\xaf\x01\0\0\0\0\0\0\xbf\x50\ +\0\0\0\0\0\0\x77\0\0\0\x02\0\0\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\ +\x03\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3e\0\0\0\xc7\0\0\0\x3f\0\0\0\ +\x5f\x30\0\0\0\0\0\0\xaf\x01\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x01\0\0\ +\0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\x57\x04\0\0\ +\x01\0\0\0\x87\x04\0\0\0\0\0\0\x5f\x34\0\0\0\0\0\0\xaf\x41\0\0\0\0\0\0\x57\x05\ +\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x53\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\ +\xbf\x15\0\0\0\0\0\0\x15\x02\x01\0\x24\0\0\0\x05\0\xa9\xff\0\0\0\0\xbf\x12\0\0\ +\0\0\0\0\x67\x02\0\0\x20\0\0\0\x77\x02\0\0\x20\0\0\0\x15\x02\x0e\0\0\0\0\0\x71\ +\x63\x06\0\0\0\0\0\x71\x64\x07\0\0\0\0\0\x67\x04\0\0\x08\0\0\0\x4f\x34\0\0\0\0\ +\0\0\x3f\x42\0\0\0\0\0\0\x2f\x42\0\0\0\0\0\0\x1f\x21\0\0\0\0\0\0\x63\x1a\x58\ +\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x58\xff\xff\xff\x18\x01\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\x55\0\x05\0\0\0\0\0\x71\x61\x08\0\0\0\0\ +\0\x71\x60\x09\0\0\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\0\0\0\x95\0\0\0\0\0\ +\0\0\x69\0\0\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x47\x50\x4c\x20\x76\x32\0\0\x9f\xeb\x01\0\x18\0\0\0\0\0\0\0\x10\x05\0\0\x10\ +\x05\0\0\x65\x11\0\0\0\0\0\0\0\0\0\x02\x03\0\0\0\x01\0\0\0\0\0\0\x01\x04\0\0\0\ +\x20\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\0\x05\0\0\0\ +\0\0\0\x01\x04\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x06\0\0\0\0\0\0\0\0\0\0\x03\0\ +\0\0\0\x02\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\ +\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x0a\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\0\0\0\0\ +\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x01\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\ +\x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x07\0\0\0\ +\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\0\x3e\0\0\0\0\0\0\x0e\x0b\0\0\0\x01\0\0\ +\0\0\0\0\0\0\0\0\x02\x0e\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\ +\x28\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\ +\x05\0\0\0\x40\0\0\0\x27\0\0\0\x0d\0\0\0\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\ +\0\x59\0\0\0\0\0\0\x0e\x0f\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x12\0\0\0\0\0\0\0\ +\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\ +\x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x01\0\0\0\ +\x80\0\0\0\x32\0\0\0\x11\0\0\0\xc0\0\0\0\x72\0\0\0\0\0\0\x0e\x13\0\0\0\x01\0\0\ +\0\0\0\0\0\0\0\0\x02\x16\0\0\0\x90\0\0\0\x22\0\0\x04\xc0\0\0\0\x9a\0\0\0\x17\0\ +\0\0\0\0\0\0\x9e\0\0\0\x17\0\0\0\x20\0\0\0\xa7\0\0\0\x17\0\0\0\x40\0\0\0\xac\0\ +\0\0\x17\0\0\0\x60\0\0\0\xba\0\0\0\x17\0\0\0\x80\0\0\0\xc3\0\0\0\x17\0\0\0\xa0\ +\0\0\0\xd0\0\0\0\x17\0\0\0\xc0\0\0\0\xd9\0\0\0\x17\0\0\0\xe0\0\0\0\xe4\0\0\0\ +\x17\0\0\0\0\x01\0\0\xed\0\0\0\x17\0\0\0\x20\x01\0\0\xfd\0\0\0\x17\0\0\0\x40\ +\x01\0\0\x05\x01\0\0\x17\0\0\0\x60\x01\0\0\x0e\x01\0\0\x19\0\0\0\x80\x01\0\0\ +\x11\x01\0\0\x17\0\0\0\x20\x02\0\0\x16\x01\0\0\x17\0\0\0\x40\x02\0\0\x21\x01\0\ +\0\x17\0\0\0\x60\x02\0\0\x26\x01\0\0\x17\0\0\0\x80\x02\0\0\x2f\x01\0\0\x17\0\0\ +\0\xa0\x02\0\0\x37\x01\0\0\x17\0\0\0\xc0\x02\0\0\x3e\x01\0\0\x17\0\0\0\xe0\x02\ +\0\0\x49\x01\0\0\x17\0\0\0\0\x03\0\0\x53\x01\0\0\x1a\0\0\0\x20\x03\0\0\x5e\x01\ +\0\0\x1a\0\0\0\xa0\x03\0\0\x68\x01\0\0\x17\0\0\0\x20\x04\0\0\x74\x01\0\0\x17\0\ +\0\0\x40\x04\0\0\x7f\x01\0\0\x17\0\0\0\x60\x04\0\0\0\0\0\0\x1b\0\0\0\x80\x04\0\ +\0\x89\x01\0\0\x1d\0\0\0\xc0\x04\0\0\x90\x01\0\0\x17\0\0\0\0\x05\0\0\x99\x01\0\ +\0\x17\0\0\0\x20\x05\0\0\0\0\0\0\x1f\0\0\0\x40\x05\0\0\xa2\x01\0\0\x17\0\0\0\ +\x80\x05\0\0\xab\x01\0\0\x21\0\0\0\xa0\x05\0\0\xb7\x01\0\0\x1d\0\0\0\xc0\x05\0\ +\0\xc0\x01\0\0\0\0\0\x08\x18\0\0\0\xc6\x01\0\0\0\0\0\x01\x04\0\0\0\x20\0\0\0\0\ +\0\0\0\0\0\0\x03\0\0\0\0\x17\0\0\0\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\ +\0\x17\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xd3\x01\0\0\x1c\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2a\0\0\0\xdd\x01\0\0\0\0\0\x08\x1e\0\0\0\xe3\ +\x01\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xf6\x01\0\ +\0\x20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2b\0\0\0\xf9\x01\0\0\0\0\0\x08\x22\0\0\ +\0\xfe\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\0\0\0\0\0\x01\0\0\x0d\x02\0\0\0\x0c\ +\x02\0\0\x15\0\0\0\x10\x02\0\0\x01\0\0\x0c\x23\0\0\0\x32\x11\0\0\0\0\0\x01\x01\ +\0\0\0\x08\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x25\0\0\0\x04\0\0\0\x07\0\0\0\x37\ +\x11\0\0\0\0\0\x0e\x26\0\0\0\x01\0\0\0\x40\x11\0\0\x03\0\0\x0f\0\0\0\0\x0c\0\0\ +\0\0\0\0\0\x20\0\0\0\x10\0\0\0\0\0\0\0\x20\0\0\0\x14\0\0\0\0\0\0\0\x20\0\0\0\ +\x46\x11\0\0\x01\0\0\x0f\0\0\0\0\x27\0\0\0\0\0\0\0\x07\0\0\0\x4e\x11\0\0\0\0\0\ +\x07\0\0\0\0\x5c\x11\0\0\0\0\0\x07\0\0\0\0\0\x69\x6e\x74\0\x5f\x5f\x41\x52\x52\ +\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x74\x79\x70\x65\0\ +\x6b\x65\x79\x5f\x73\x69\x7a\x65\0\x76\x61\x6c\x75\x65\x5f\x73\x69\x7a\x65\0\ +\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\ +\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\ +\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\ +\x5f\x6b\x65\x79\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\ +\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x5f\x5f\x73\x6b\x5f\ +\x62\x75\x66\x66\0\x6c\x65\x6e\0\x70\x6b\x74\x5f\x74\x79\x70\x65\0\x6d\x61\x72\ +\x6b\0\x71\x75\x65\x75\x65\x5f\x6d\x61\x70\x70\x69\x6e\x67\0\x70\x72\x6f\x74\ +\x6f\x63\x6f\x6c\0\x76\x6c\x61\x6e\x5f\x70\x72\x65\x73\x65\x6e\x74\0\x76\x6c\ +\x61\x6e\x5f\x74\x63\x69\0\x76\x6c\x61\x6e\x5f\x70\x72\x6f\x74\x6f\0\x70\x72\ +\x69\x6f\x72\x69\x74\x79\0\x69\x6e\x67\x72\x65\x73\x73\x5f\x69\x66\x69\x6e\x64\ +\x65\x78\0\x69\x66\x69\x6e\x64\x65\x78\0\x74\x63\x5f\x69\x6e\x64\x65\x78\0\x63\ +\x62\0\x68\x61\x73\x68\0\x74\x63\x5f\x63\x6c\x61\x73\x73\x69\x64\0\x64\x61\x74\ +\x61\0\x64\x61\x74\x61\x5f\x65\x6e\x64\0\x6e\x61\x70\x69\x5f\x69\x64\0\x66\x61\ +\x6d\x69\x6c\x79\0\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x34\0\x6c\x6f\x63\x61\ +\x6c\x5f\x69\x70\x34\0\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x36\0\x6c\x6f\x63\ +\x61\x6c\x5f\x69\x70\x36\0\x72\x65\x6d\x6f\x74\x65\x5f\x70\x6f\x72\x74\0\x6c\ +\x6f\x63\x61\x6c\x5f\x70\x6f\x72\x74\0\x64\x61\x74\x61\x5f\x6d\x65\x74\x61\0\ +\x74\x73\x74\x61\x6d\x70\0\x77\x69\x72\x65\x5f\x6c\x65\x6e\0\x67\x73\x6f\x5f\ +\x73\x65\x67\x73\0\x67\x73\x6f\x5f\x73\x69\x7a\x65\0\x74\x73\x74\x61\x6d\x70\ +\x5f\x74\x79\x70\x65\0\x68\x77\x74\x73\x74\x61\x6d\x70\0\x5f\x5f\x75\x33\x32\0\ +\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x66\x6c\x6f\x77\x5f\x6b\x65\ +\x79\x73\0\x5f\x5f\x75\x36\x34\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x6c\x6f\ +\x6e\x67\x20\x6c\x6f\x6e\x67\0\x73\x6b\0\x5f\x5f\x75\x38\0\x75\x6e\x73\x69\x67\ +\x6e\x65\x64\x20\x63\x68\x61\x72\0\x73\x6b\x62\0\x74\x75\x6e\x5f\x72\x73\x73\ +\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x74\x75\x6e\x5f\x72\ +\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x2f\x68\x6f\x6d\x65\x2f\x73\x68\ +\x72\x65\x65\x73\x68\x2f\x63\x2f\x71\x65\x6d\x75\x2f\x74\x6f\x6f\x6c\x73\x2f\ +\x65\x62\x70\x66\x2f\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\0\x69\x6e\x74\x20\x74\ +\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\ +\x67\x28\x73\x74\x72\x75\x63\x74\x20\x5f\x5f\x73\x6b\x5f\x62\x75\x66\x66\x20\ +\x2a\x73\x6b\x62\x29\0\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x6b\x65\x79\x20\ +\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x63\x6f\x6e\x66\x69\x67\x20\x3d\x20\x62\x70\ +\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\ +\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\ +\x72\x61\x74\x69\x6f\x6e\x73\x2c\x20\x26\x6b\x65\x79\x29\x3b\0\x20\x20\x20\x20\ +\x74\x6f\x65\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\ +\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\ +\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\x2c\x20\x26\x6b\x65\x79\ +\x29\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x20\x26\x26\ +\x20\x74\x6f\x65\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\ +\x21\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x72\x65\x64\x69\x72\x65\x63\x74\x29\x20\ +\x7b\0\x20\x20\x20\x20\x5f\x5f\x75\x38\x20\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\ +\x5b\x48\x41\x53\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\ +\x55\x46\x46\x45\x52\x5f\x53\x49\x5a\x45\x5d\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\ +\x20\x20\x73\x74\x72\x75\x63\x74\x20\x70\x61\x63\x6b\x65\x74\x5f\x68\x61\x73\ +\x68\x5f\x69\x6e\x66\x6f\x5f\x74\x20\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\ +\x6f\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x21\x69\x6e\x66\ +\x6f\x20\x7c\x7c\x20\x21\x73\x6b\x62\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x62\ +\x65\x31\x36\x20\x72\x65\x74\x20\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x65\x72\x72\ +\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\ +\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6f\x66\ +\x66\x73\x65\x74\x2c\x20\x26\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\ +\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\ +\0\x20\x20\x20\x20\x73\x77\x69\x74\x63\x68\x20\x28\x62\x70\x66\x5f\x6e\x74\x6f\ +\x68\x73\x28\x72\x65\x74\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\ +\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\ +\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\ +\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\ +\x66\x28\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x72\ +\x65\x74\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x33\x5f\x70\x72\x6f\x74\x6f\ +\x63\x6f\x6c\x20\x3d\x3d\x20\x30\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\ +\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x34\x20\x3d\x20\x31\x3b\0\x20\ +\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x68\x64\x72\ +\x20\x69\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\ +\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\ +\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\ +\x2c\x20\x26\x69\x70\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x69\x70\x29\x2c\0\x20\ +\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\ +\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\ +\x6d\x65\x6e\x74\x65\x64\x20\x3d\x20\x21\x21\x69\x70\x2e\x66\x72\x61\x67\x5f\ +\x6f\x66\x66\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\ +\x6e\x5f\x73\x72\x63\x20\x3d\x20\x69\x70\x2e\x73\x61\x64\x64\x72\x3b\0\x20\x20\ +\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x5f\x64\x73\x74\x20\ +\x3d\x20\x69\x70\x2e\x64\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\ +\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x20\x69\x70\x2e\x70\x72\ +\x6f\x74\x6f\x63\x6f\x6c\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x6f\ +\x66\x66\x73\x65\x74\x20\x3d\x20\x69\x70\x2e\x69\x68\x6c\x20\x2a\x20\x34\x3b\0\ +\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\ +\x20\x21\x3d\x20\x30\x20\x26\x26\x20\x21\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\ +\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\x64\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\ +\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\ +\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\x5f\x54\x43\x50\x29\x20\x7b\0\x20\x20\x20\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x74\ +\x63\x70\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x73\x74\x72\x75\x63\x74\x20\x74\x63\x70\x68\x64\x72\x20\x74\x63\x70\x20\x3d\ +\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\ +\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\ +\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6c\x34\ +\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x74\x63\x70\x2c\x20\x73\x69\x7a\x65\ +\x6f\x66\x28\x74\x63\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x69\x66\x20\x28\ +\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x34\ +\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\ +\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x74\x63\x70\x20\x26\x26\0\x20\x20\ +\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\ +\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\ +\x20\x69\x70\x76\x36\x68\x64\x72\x20\x69\x70\x36\x20\x3d\x20\x7b\x7d\x3b\0\x20\ +\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\ +\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\ +\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\x2c\x20\x26\x69\x70\x36\x2c\x20\x73\x69\ +\x7a\x65\x6f\x66\x28\x69\x70\x36\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\ +\x6e\x66\x6f\x2d\x3e\x69\x6e\x36\x5f\x73\x72\x63\x20\x3d\x20\x69\x70\x36\x2e\ +\x73\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\ +\x3e\x69\x6e\x36\x5f\x64\x73\x74\x20\x3d\x20\x69\x70\x36\x2e\x64\x61\x64\x64\ +\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\ +\x6f\x6c\x20\x3d\x20\x69\x70\x36\x2e\x6e\x65\x78\x74\x68\x64\x72\x3b\0\x20\x20\ +\x20\x20\x73\x77\x69\x74\x63\x68\x20\x28\x68\x64\x72\x5f\x74\x79\x70\x65\x29\ +\x20\x7b\0\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x76\x36\x5f\x6f\ +\x70\x74\x5f\x68\x64\x72\x20\x65\x78\x74\x5f\x68\x64\x72\x20\x3d\x20\x7b\x7d\ +\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\ +\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\ +\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\ +\x74\x2c\x20\x26\x65\x78\x74\x5f\x68\x64\x72\x2c\0\x20\x20\x20\x20\x20\x20\x20\ +\x20\x69\x66\x20\x28\x2a\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\ +\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\x5f\x52\x4f\x55\x54\x49\x4e\x47\x29\x20\ +\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\ +\x20\x69\x70\x76\x36\x5f\x72\x74\x5f\x68\x64\x72\x20\x65\x78\x74\x5f\x72\x74\ +\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\ +\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\ +\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\ +\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x65\x78\x74\x5f\x72\x74\ +\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x28\x65\ +\x78\x74\x5f\x72\x74\x2e\x74\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\ +\x53\x52\x43\x52\x54\x5f\x54\x59\x50\x45\x5f\x32\x29\x20\x26\x26\0\x20\x20\x20\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\ +\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x66\x66\x73\x65\x74\x6f\x66\ +\x28\x73\x74\x72\x75\x63\x74\x20\x72\x74\x32\x5f\x68\x64\x72\x2c\x20\x61\x64\ +\x64\x72\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\ +\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\ +\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\ +\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\ +\x65\x78\x74\x5f\x64\x73\x74\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x7d\x20\x5f\x5f\x61\x74\x74\x72\x69\x62\x75\x74\x65\x5f\ +\x5f\x28\x28\x70\x61\x63\x6b\x65\x64\x29\x29\x20\x6f\x70\x74\x20\x3d\x20\x7b\ +\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6f\ +\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x3d\x20\x28\x6f\x70\x74\x2e\x74\ +\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\x4c\x56\x5f\x50\x41\x44\ +\x31\x29\x20\x3f\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x69\x66\x20\x28\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x31\ +\x20\x3e\x3d\x20\x65\x78\x74\x5f\x68\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\ +\x2a\x20\x38\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\ +\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\ +\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\ +\x74\x5f\x6f\x66\x66\x73\x65\x74\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x6f\x70\x74\x2e\x74\x79\x70\x65\x20\ +\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\x4c\x56\x5f\x48\x41\x4f\x29\x20\x7b\0\x20\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\ +\x74\x5f\x6f\x66\x66\x73\x65\x74\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\ +\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\ +\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\ +\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\ +\x5f\x73\x72\x63\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\ +\x65\x64\x20\x3d\x20\x74\x72\x75\x65\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x2a\ +\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x3d\x20\x28\x65\x78\x74\x5f\x68\ +\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\x2b\x20\x31\x29\x20\x2a\x20\x38\x3b\0\ +\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\ +\x6c\x20\x3d\x20\x65\x78\x74\x5f\x68\x64\x72\x2e\x6e\x65\x78\x74\x68\x64\x72\ +\x3b\0\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\ +\x69\x6e\x74\x20\x69\x20\x3d\x20\x30\x3b\x20\x69\x20\x3c\x20\x49\x50\x36\x5f\ +\x45\x58\x54\x45\x4e\x53\x49\x4f\x4e\x53\x5f\x43\x4f\x55\x4e\x54\x3b\x20\x2b\ +\x2b\x69\x29\x20\x7b\0\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\ +\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\ +\x36\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\ +\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\ +\x36\x5f\x65\x78\x74\x5f\x64\x73\x74\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\ +\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x73\x72\x63\x20\x26\ +\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\ +\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x75\x64\x70\ +\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\ +\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\x70\ +\x65\x73\x20\x26\x20\x56\x49\x52\x54\x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\x53\ +\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\x45\x5f\x49\x50\x76\x34\x29\x20\x7b\0\x20\ +\x20\x20\x20\x5f\x5f\x62\x75\x69\x6c\x74\x69\x6e\x5f\x6d\x65\x6d\x63\x70\x79\ +\x28\x26\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\x5b\x2a\x62\x79\x74\x65\x73\x5f\ +\x77\x72\x69\x74\x74\x65\x6e\x5d\x2c\x20\x70\x74\x72\x2c\x20\x73\x69\x7a\x65\ +\x29\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\ +\x3e\x69\x73\x5f\x75\x64\x70\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x75\x64\x70\x68\x64\x72\x20\ +\x75\x64\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\ +\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\ +\x62\x2c\x20\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x75\x64\x70\x2c\ +\x20\x73\x69\x7a\x65\x6f\x66\x28\x75\x64\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\ +\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\ +\x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\x70\x65\x73\x20\x26\x20\x56\x49\x52\x54\ +\x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\x53\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\ +\x45\x5f\x49\x50\x76\x36\x29\x20\x7b\0\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x62\ +\x79\x74\x65\x20\x3d\x20\x30\x3b\x20\x62\x79\x74\x65\x20\x3c\x20\x48\x41\x53\ +\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\x55\x46\x46\x45\ +\x52\x5f\x53\x49\x5a\x45\x3b\x20\x62\x79\x74\x65\x2b\x2b\x29\x20\x7b\0\x20\x20\ +\x20\x20\x5f\x5f\x75\x33\x32\x20\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\ +\x5f\x62\x69\x74\x73\x20\x3d\x20\x6b\x65\x79\x2d\x3e\x6c\x65\x66\x74\x6d\x6f\ +\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\ +\x5f\x5f\x75\x38\x20\x69\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\x3d\x20\x69\ +\x6e\x70\x75\x74\x5b\x62\x79\x74\x65\x5d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x69\x66\x20\x28\x69\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\ +\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\ +\x20\x20\x5f\x5f\x75\x38\x20\x6b\x65\x79\x5f\x62\x79\x74\x65\x20\x3d\x20\x6b\ +\x65\x79\x2d\x3e\x6e\x65\x78\x74\x5f\x62\x79\x74\x65\x5b\x62\x79\x74\x65\x5d\ +\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x28\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\ +\x20\x3c\x3c\x20\x31\x29\x20\x7c\x20\x28\x28\x6b\x65\x79\x5f\x62\x79\x74\x65\ +\x20\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\x29\x29\x20\x3e\x3e\x20\x37\x29\x3b\0\ +\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x68\x61\x73\x68\x29\x20\x7b\0\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x74\ +\x61\x62\x6c\x65\x5f\x69\x64\x78\x20\x3d\x20\x68\x61\x73\x68\x20\x25\x20\x63\ +\x6f\x6e\x66\x69\x67\x2d\x3e\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x73\ +\x5f\x6c\x65\x6e\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x71\x75\ +\x65\x75\x65\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\ +\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\ +\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\x2c\0\ +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x71\x75\x65\ +\x75\x65\x29\x20\x7b\0\x7d\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x2a\x71\x75\x65\x75\x65\x3b\0\x63\ +\x68\x61\x72\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x6d\x61\x70\x73\0\x6c\x69\ +\x63\x65\x6e\x73\x65\0\x62\x70\x66\x5f\x66\x6c\x6f\x77\x5f\x6b\x65\x79\x73\0\ +\x62\x70\x66\x5f\x73\x6f\x63\x6b\0\0\0\0\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x14\0\ +\0\0\x14\0\0\0\x6c\x0c\0\0\x80\x0c\0\0\0\0\0\0\x08\0\0\0\x26\x02\0\0\x01\0\0\0\ +\0\0\0\0\x24\0\0\0\x10\0\0\0\x26\x02\0\0\xc6\0\0\0\0\0\0\0\x37\x02\0\0\x61\x02\ +\0\0\0\x50\x08\0\x10\0\0\0\x37\x02\0\0\x92\x02\0\0\x0b\x68\x08\0\x20\0\0\0\x37\ +\x02\0\0\0\0\0\0\0\0\0\0\x28\0\0\0\x37\x02\0\0\xa5\x02\0\0\x0e\x74\x08\0\x50\0\ +\0\0\x37\x02\0\0\xea\x02\0\0\x0b\x78\x08\0\x88\0\0\0\x37\x02\0\0\x2a\x03\0\0\ +\x10\x80\x08\0\x90\0\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x98\0\0\0\x37\x02\0\0\x2a\ +\x03\0\0\x10\x80\x08\0\xa0\0\0\0\x37\x02\0\0\x43\x03\0\0\x16\x84\x08\0\xa8\0\0\ +\0\x37\x02\0\0\x43\x03\0\0\x0d\x84\x08\0\xc0\0\0\0\x37\x02\0\0\x64\x03\0\0\x0a\ +\xfc\x05\0\xe8\0\0\0\x37\x02\0\0\x9b\x03\0\0\x1f\x0c\x06\0\x38\x01\0\0\x37\x02\ +\0\0\xcb\x03\0\0\x0f\xa0\x04\0\x40\x01\0\0\x37\x02\0\0\xe4\x03\0\0\x0c\x20\x04\ +\0\x50\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x58\x01\0\0\x37\x02\0\0\xf8\x03\0\0\ +\x0b\x2c\x04\0\x90\x01\0\0\x37\x02\0\0\x3e\x04\0\0\x09\x34\x04\0\xa0\x01\0\0\ +\x37\x02\0\0\x4d\x04\0\0\x0d\x44\x04\0\xb8\x01\0\0\x37\x02\0\0\x4d\x04\0\0\x05\ +\x44\x04\0\xd8\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xe0\x01\0\0\x37\x02\0\0\x6b\ +\x04\0\0\x0f\x58\x04\0\x10\x02\0\0\x37\x02\0\0\x3e\x04\0\0\x09\x70\x04\0\x18\ +\x02\0\0\x37\x02\0\0\xb5\x04\0\0\x0c\x80\x04\0\x20\x02\0\0\x37\x02\0\0\xc5\x04\ +\0\0\x09\xbc\x04\0\x50\x02\0\0\x37\x02\0\0\xe1\x04\0\0\x17\xd4\x04\0\x60\x02\0\ +\0\x37\x02\0\0\xfc\x04\0\0\x16\xdc\x04\0\x80\x02\0\0\x37\x02\0\0\xe1\x04\0\0\ +\x17\xd4\x04\0\x88\x02\0\0\x37\x02\0\0\x1a\x05\0\0\x0f\xe0\x04\0\xc0\x02\0\0\ +\x37\x02\0\0\x5d\x05\0\0\x0d\xe8\x04\0\xc8\x02\0\0\x37\x02\0\0\x70\x05\0\0\x24\ +\0\x05\0\xd0\x02\0\0\x37\x02\0\0\x70\x05\0\0\x20\0\x05\0\xe0\x02\0\0\x37\x02\0\ +\0\x9d\x05\0\0\x1b\xf8\x04\0\xe8\x02\0\0\x37\x02\0\0\x9d\x05\0\0\x16\xf8\x04\0\ +\xf0\x02\0\0\x37\x02\0\0\xbe\x05\0\0\x1b\xfc\x04\0\xf8\x02\0\0\x37\x02\0\0\xbe\ +\x05\0\0\x16\xfc\x04\0\0\x03\0\0\x37\x02\0\0\xdf\x05\0\0\x1a\x08\x05\0\x08\x03\ +\0\0\x37\x02\0\0\x70\x05\0\0\x1d\0\x05\0\x10\x03\0\0\x37\x02\0\0\x02\x06\0\0\ +\x18\x0c\x05\0\x18\x03\0\0\x37\x02\0\0\x02\x06\0\0\x1c\x0c\x05\0\x30\x03\0\0\ +\x37\x02\0\0\x22\x06\0\0\x15\x68\x05\0\x40\x03\0\0\x37\x02\0\0\x22\x06\0\0\x1a\ +\x68\x05\0\x58\x03\0\0\x37\x02\0\0\x56\x06\0\0\x0d\x6c\x05\0\x78\x03\0\0\x37\ +\x02\0\0\x80\x06\0\0\x1a\x70\x05\0\x88\x03\0\0\x37\x02\0\0\x9e\x06\0\0\x1b\x78\ +\x05\0\xa8\x03\0\0\x37\x02\0\0\x80\x06\0\0\x1a\x70\x05\0\xb0\x03\0\0\x37\x02\0\ +\0\xc2\x06\0\0\x13\x7c\x05\0\xe8\x03\0\0\x37\x02\0\0\x13\x07\0\0\x11\x84\x05\0\ +\xf0\x03\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x10\x04\0\0\x37\x02\0\0\x2a\x07\0\0\ +\x15\x28\x06\0\x18\x04\0\0\x37\x02\0\0\x2a\x07\0\0\x09\x28\x06\0\x20\x04\0\0\ +\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x04\0\0\x37\x02\0\0\x49\x07\0\0\x19\x2c\x06\0\ +\x80\x04\0\0\x37\x02\0\0\x49\x07\0\0\x20\x2c\x06\0\xa0\x04\0\0\x37\x02\0\0\0\0\ +\0\0\0\0\0\0\xf0\x04\0\0\x37\x02\0\0\x6b\x07\0\0\x17\x14\x05\0\0\x05\0\0\x37\ +\x02\0\0\x86\x07\0\0\x18\x1c\x05\0\x30\x05\0\0\x37\x02\0\0\x6b\x07\0\0\x17\x14\ +\x05\0\x48\x05\0\0\x37\x02\0\0\xa7\x07\0\0\x0f\x20\x05\0\x80\x05\0\0\x37\x02\0\ +\0\x5d\x05\0\0\x0d\x28\x05\0\x88\x05\0\0\x37\x02\0\0\xec\x07\0\0\x1d\x38\x05\0\ +\xc8\x05\0\0\x37\x02\0\0\x0f\x08\0\0\x1d\x3c\x05\0\x08\x06\0\0\x37\x02\0\0\x32\ +\x08\0\0\x1b\x44\x05\0\x10\x06\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\x58\ +\x06\0\0\x37\x02\0\0\x6d\x08\0\0\x19\xb8\x02\0\xd0\x06\0\0\x37\x02\0\0\0\0\0\0\ +\0\0\0\0\xd8\x06\0\0\x37\x02\0\0\x93\x08\0\0\x0f\xc8\x02\0\x10\x07\0\0\x37\x02\ +\0\0\x5d\x05\0\0\x0d\xd0\x02\0\x20\x07\0\0\x37\x02\0\0\xd8\x08\0\0\x0d\xe0\x02\ +\0\x40\x07\0\0\x37\x02\0\0\x07\x09\0\0\x20\xe4\x02\0\x68\x07\0\0\x37\x02\0\0\ +\x33\x09\0\0\x13\xec\x02\0\xa8\x07\0\0\x37\x02\0\0\x13\x07\0\0\x11\xf4\x02\0\ +\xb0\x07\0\0\x37\x02\0\0\x7b\x09\0\0\x19\x04\x03\0\xb8\x07\0\0\x37\x02\0\0\x7b\ +\x09\0\0\x34\x04\x03\0\xe0\x07\0\0\x37\x02\0\0\xb1\x09\0\0\x15\x18\x03\0\xf0\ +\x07\0\0\x37\x02\0\0\xf2\x09\0\0\x17\x14\x03\0\x30\x08\0\0\x37\x02\0\0\x29\x0a\ +\0\0\x15\x24\x03\0\x38\x08\0\0\x37\x02\0\0\x44\x0a\0\0\x27\x34\x03\0\x70\x08\0\ +\0\x37\x02\0\0\x6f\x0a\0\0\x27\x50\x03\0\x80\x08\0\0\x37\x02\0\0\x9f\x0a\0\0\ +\x1c\xb4\x03\0\x88\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x20\xc0\x03\0\x98\x08\0\0\ +\x37\x02\0\0\xdb\x0a\0\0\x2f\xc0\x03\0\xa0\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x36\ +\xc0\x03\0\xa8\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x15\xc0\x03\0\x18\x09\0\0\x37\ +\x02\0\0\x17\x0b\0\0\x43\x64\x03\0\x38\x09\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x40\ +\x09\0\0\x37\x02\0\0\x17\x0b\0\0\x17\x64\x03\0\x80\x09\0\0\x37\x02\0\0\x29\x0a\ +\0\0\x15\x6c\x03\0\x88\x09\0\0\x37\x02\0\0\x67\x0b\0\0\x19\x7c\x03\0\x90\x09\0\ +\0\x37\x02\0\0\x67\x0b\0\0\x15\x7c\x03\0\x98\x09\0\0\x37\x02\0\0\x97\x0b\0\0\ +\x19\x84\x03\0\xa0\x09\0\0\x37\x02\0\0\xc7\x0b\0\0\x1b\x80\x03\0\xe8\x09\0\0\ +\x37\x02\0\0\x02\x0c\0\0\x19\x94\x03\0\xf0\x09\0\0\x37\x02\0\0\x21\x0c\0\0\x2b\ +\xa4\x03\0\x10\x0a\0\0\x37\x02\0\0\x9f\x0a\0\0\x1f\xb4\x03\0\x30\x0a\0\0\x37\ +\x02\0\0\x50\x0c\0\0\x21\xd4\x03\0\x40\x0a\0\0\x37\x02\0\0\x78\x0c\0\0\x20\xe4\ +\x03\0\x48\x0a\0\0\x37\x02\0\0\x78\x0c\0\0\x2c\xe4\x03\0\x60\x0a\0\0\x37\x02\0\ +\0\x78\x0c\0\0\x14\xe4\x03\0\x70\x0a\0\0\x37\x02\0\0\xa8\x0c\0\0\x20\xe0\x03\0\ +\x80\x0a\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xb0\x0a\0\0\x37\x02\0\0\xd0\ +\x0c\0\0\x38\xc0\x02\0\xd0\x0a\0\0\x37\x02\0\0\xd0\x0c\0\0\x05\xc0\x02\0\xe8\ +\x0a\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xf8\x0a\0\0\x37\x02\0\0\x0e\x0d\ +\0\0\x1c\xc4\x06\0\x08\x0b\0\0\x37\x02\0\0\x0e\x0d\0\0\x10\xc4\x06\0\x10\x0b\0\ +\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x60\x0b\0\0\x37\x02\0\0\x49\x07\0\0\x19\xc8\x06\ +\0\x68\x0b\0\0\x37\x02\0\0\x49\x07\0\0\x20\xc8\x06\0\xa0\x0b\0\0\x37\x02\0\0\ +\x34\x0d\0\0\x2d\0\x07\0\xb0\x0b\0\0\x37\x02\0\0\x34\x0d\0\0\x1d\0\x07\0\xb8\ +\x0b\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\0\x07\0\xc8\x0b\0\0\x37\x02\0\0\x63\x0d\0\ +\0\x2d\xd4\x06\0\xf8\x0b\0\0\x37\x02\0\0\x63\x0d\0\0\x1d\xd4\x06\0\x08\x0c\0\0\ +\x37\x02\0\0\x63\x0d\0\0\x2d\xd4\x06\0\x18\x0c\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\ +\xe8\x0c\0\0\x37\x02\0\0\x92\x0d\0\0\x20\x68\x06\0\xf0\x0c\0\0\x37\x02\0\0\x92\ +\x0d\0\0\x27\x68\x06\0\x18\x0d\0\0\x37\x02\0\0\xbb\x0d\0\0\x27\xa4\x06\0\x20\ +\x0d\0\0\x37\x02\0\0\xbb\x0d\0\0\x14\xa4\x06\0\x28\x0d\0\0\x37\x02\0\0\x04\x0e\ +\0\0\x05\x98\x01\0\x38\x0d\0\0\x37\x02\0\0\x04\x0e\0\0\x05\x98\x01\0\x60\x0d\0\ +\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x0d\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0d\ +\0\0\x37\x02\0\0\x92\x0d\0\0\x20\x44\x07\0\x88\x0d\0\0\x37\x02\0\0\x92\x0d\0\0\ +\x27\x44\x07\0\xc0\x0d\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\x7c\x07\0\xd0\x0d\0\0\ +\x37\x02\0\0\x34\x0d\0\0\x1d\x7c\x07\0\xd8\x0d\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\ +\x7c\x07\0\xe8\x0d\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\x50\x07\0\x18\x0e\0\0\x37\ +\x02\0\0\x63\x0d\0\0\x1d\x50\x07\0\x28\x0e\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\x50\ +\x07\0\x40\x0e\0\0\x37\x02\0\0\x41\x0e\0\0\x1a\xa0\x05\0\x50\x0e\0\0\x37\x02\0\ +\0\x5f\x0e\0\0\x1b\xa8\x05\0\x60\x0e\0\0\x37\x02\0\0\x41\x0e\0\0\x1a\xa0\x05\0\ +\x68\x0e\0\0\x37\x02\0\0\x83\x0e\0\0\x13\xac\x05\0\xa0\x0e\0\0\x37\x02\0\0\x13\ +\x07\0\0\x11\xb4\x05\0\xb0\x0e\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xc0\ +\x0e\0\0\x37\x02\0\0\xd4\x0e\0\0\x27\xc8\x07\0\xd0\x0e\0\0\x37\x02\0\0\xd4\x0e\ +\0\0\x14\xc8\x07\0\xf0\x0e\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\xcc\x07\0\0\x0f\0\0\ +\x37\x02\0\0\x63\x0d\0\0\x1d\xcc\x07\0\x08\x0f\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\ +\xcc\x07\0\x30\x0f\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0f\0\0\x37\x02\0\0\x34\ +\x0d\0\0\x1d\xf8\x07\0\x88\x0f\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\xf8\x07\0\x98\ +\x0f\0\0\x37\x02\0\0\x04\x0e\0\0\x05\x98\x01\0\xf0\x0f\0\0\x37\x02\0\0\x04\x0e\ +\0\0\x05\x98\x01\0\x30\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x48\x10\0\0\x37\x02\ +\0\0\x1d\x0f\0\0\x05\xd0\x01\0\x50\x10\0\0\x37\x02\0\0\x5f\x0f\0\0\x23\xc4\x01\ +\0\x68\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x10\0\0\x37\x02\0\0\x93\x0f\0\0\ +\x1b\xd4\x01\0\x90\x10\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xa8\x10\0\0\ +\x37\x02\0\0\xe3\x0f\0\0\x19\xd8\x01\0\xc0\x10\0\0\x37\x02\0\0\x11\x10\0\0\x27\ +\xfc\x01\0\xc8\x10\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xd8\x10\0\0\x37\ +\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xe0\x10\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\ +\x01\0\x08\x11\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x20\x11\0\0\x37\x02\0\ +\0\x11\x10\0\0\x27\xfc\x01\0\x28\x11\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\ +\x30\x11\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\x58\x11\0\0\x37\x02\0\0\x11\ +\x10\0\0\x27\xfc\x01\0\x60\x11\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x78\ +\x11\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\x80\x11\0\0\x37\x02\0\0\xba\x0f\ +\0\0\x11\xe8\x01\0\xa8\x11\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\xb0\x11\0\ +\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xc8\x11\0\0\x37\x02\0\0\x11\x10\0\0\ +\x2d\xfc\x01\0\xd0\x11\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xf8\x11\0\0\ +\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x10\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\ +\xfc\x01\0\x18\x12\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\x20\x12\0\0\x37\ +\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\x48\x12\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\ +\x01\0\x60\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\x68\x12\0\0\x37\x02\0\ +\0\x11\x10\0\0\x2d\xfc\x01\0\x70\x12\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\ +\x98\x12\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xb0\x12\0\0\x37\x02\0\0\x11\ +\x10\0\0\x27\xfc\x01\0\xb8\x12\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xc0\ +\x12\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xe0\x12\0\0\x37\x02\0\0\x11\x10\ +\0\0\x46\xfc\x01\0\xe8\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\xf0\x12\0\ +\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xf8\x12\0\0\x37\x02\0\0\x1d\x0f\0\0\ +\x3d\xd0\x01\0\x08\x13\0\0\x37\x02\0\0\x1d\x0f\0\0\x05\xd0\x01\0\x18\x13\0\0\ +\x37\x02\0\0\x5d\x10\0\0\x0d\x98\x08\0\x30\x13\0\0\x37\x02\0\0\x5d\x10\0\0\x0d\ +\x98\x08\0\x38\x13\0\0\x37\x02\0\0\x71\x10\0\0\x2e\x9c\x08\0\x58\x13\0\0\x37\ +\x02\0\0\x71\x10\0\0\x24\x9c\x08\0\x70\x13\0\0\x37\x02\0\0\x71\x10\0\0\x13\x9c\ +\x08\0\x80\x13\0\0\x37\x02\0\0\x71\x10\0\0\x2e\x9c\x08\0\x88\x13\0\0\x37\x02\0\ +\0\xb0\x10\0\0\x15\xa8\x08\0\xa0\x13\0\0\x37\x02\0\0\xf8\x10\0\0\x11\xb4\x08\0\ +\xa8\x13\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xc8\x13\0\0\x37\x02\0\0\x11\x11\0\0\ +\x01\xd8\x08\0\xd0\x13\0\0\x37\x02\0\0\x13\x11\0\0\x18\xb8\x08\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\xde\0\0\0\0\0\x03\0\xc8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7a\x01\0\0\0\ +\0\x03\0\xb8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xff\0\0\0\0\0\x03\0\xa8\x13\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\xc7\0\0\0\0\0\x03\0\xd0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\x2c\x02\0\0\0\0\x03\0\x20\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf7\0\0\0\0\0\x03\0\ +\xe8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1c\x02\0\0\0\0\x03\0\x10\x04\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x28\x01\0\0\0\0\x03\0\xe0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf3\ +\x01\0\0\0\0\x03\0\x30\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xeb\x01\0\0\0\0\x03\0\ +\x38\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x44\x02\0\0\0\0\x03\0\xf0\x03\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xe3\x01\0\0\0\0\x03\0\xf8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\ +\x01\0\0\0\0\x03\0\xe8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x30\x01\0\0\0\0\x03\0\ +\xa0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa9\x01\0\0\0\0\x03\0\x40\x10\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x51\x01\0\0\0\0\x03\0\x78\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5c\ +\x02\0\0\0\0\x03\0\xb0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\x02\0\0\0\0\x03\0\ +\x50\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\x01\0\0\0\0\x03\0\xc0\x06\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x69\x01\0\0\0\0\x03\0\x20\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\ +\x01\0\0\0\0\x03\0\x60\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x01\0\0\0\0\x03\0\ +\x30\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x01\0\0\0\0\x03\0\x40\x0a\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xba\x01\0\0\0\0\x03\0\xe0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\ +\x01\0\0\0\0\x03\0\x48\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\0\0\x03\0\ +\x18\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xfb\x01\0\0\0\0\x03\0\x80\x08\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x99\x01\0\0\0\0\x03\0\xf8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x59\ +\x01\0\0\0\0\x03\0\x50\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x54\x02\0\0\0\0\x03\0\ +\x08\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xef\0\0\0\0\0\x03\0\xe8\x0a\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\x4c\x02\0\0\0\0\x03\0\xb0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x24\ +\x02\0\0\0\0\x03\0\xd8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x89\x01\0\0\0\0\x03\0\ +\x80\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\x01\0\0\0\0\x03\0\xb0\x0b\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xd6\0\0\0\0\0\x03\0\xc8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x14\ +\x02\0\0\0\0\x03\0\xf8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb2\x01\0\0\0\0\x03\0\ +\x18\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\x01\0\0\0\0\x03\0\x10\x0c\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x3c\x02\0\0\0\0\x03\0\x18\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x91\ +\x01\0\0\0\0\x03\0\x60\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x81\x01\0\0\0\0\x03\0\ +\xc0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe7\0\0\0\0\0\x03\0\xd0\x0d\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\x34\x02\0\0\0\0\x03\0\xe8\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd3\ +\x01\0\0\0\0\x03\0\x18\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\x01\0\0\0\0\x03\0\0\ +\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xce\0\0\0\0\0\x03\0\x18\x0f\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\x0b\x02\0\0\0\0\x03\0\xf0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xca\x01\0\ +\0\0\0\x03\0\x30\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x71\x01\0\0\0\0\x03\0\x60\x10\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x01\0\0\0\0\x03\0\x18\x13\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\x64\x02\0\0\0\0\x03\0\xd0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4e\0\0\0\ +\x12\0\x03\0\0\0\0\0\0\0\0\0\xe0\x13\0\0\0\0\0\0\x33\0\0\0\x11\0\x05\0\0\0\0\0\ +\0\0\0\0\x20\0\0\0\0\0\0\0\x01\0\0\0\x11\0\x05\0\x20\0\0\0\0\0\0\0\x20\0\0\0\0\ +\0\0\0\x90\0\0\0\x11\0\x05\0\x40\0\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x87\0\0\0\x11\ +\0\x06\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x37\0\0\ +\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x38\0\0\0\x88\x13\0\0\0\0\0\0\x01\0\0\0\x39\0\0\ +\0\xd8\x04\0\0\0\0\0\0\x04\0\0\0\x37\0\0\0\xe4\x04\0\0\0\0\0\0\x04\0\0\0\x38\0\ +\0\0\xf0\x04\0\0\0\0\0\0\x04\0\0\0\x39\0\0\0\x08\x05\0\0\0\0\0\0\x04\0\0\0\x3a\ +\0\0\0\x2c\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\0\0\0\0\0\0\0\x04\0\0\0\x01\0\ +\0\0\x50\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\x70\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\x90\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\xb0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\xd0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\xf0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\x10\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\x30\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\ +\0\0\x50\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x01\0\0\0\0\0\0\x04\0\0\0\x01\ +\0\0\0\x70\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x01\0\0\0\0\0\0\x04\0\0\0\ +\x01\0\0\0\x90\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x01\0\0\0\0\0\0\x04\0\0\ +\0\x01\0\0\0\xb0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x01\0\0\0\0\0\0\x04\0\ +\0\0\x01\0\0\0\xd0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x01\0\0\0\0\0\0\x04\ +\0\0\0\x01\0\0\0\xf0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x02\0\0\0\0\0\0\x04\ +\0\0\0\x01\0\0\0\x10\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x02\0\0\0\0\0\0\ +\x04\0\0\0\x01\0\0\0\x30\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x02\0\0\0\0\0\ +\0\x04\0\0\0\x01\0\0\0\x50\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x02\0\0\0\0\ +\0\0\x04\0\0\0\x01\0\0\0\x70\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x02\0\0\0\ +\0\0\0\x04\0\0\0\x01\0\0\0\x90\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x02\0\0\ +\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x02\0\ +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x02\ +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x03\ +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\ +\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\x40\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\x60\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\ +\0\0\x80\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x03\0\0\0\0\0\0\x04\0\0\0\x01\ +\0\0\0\xa0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x03\0\0\0\0\0\0\x04\0\0\0\ +\x01\0\0\0\xc0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x03\0\0\0\0\0\0\x04\0\0\ +\0\x01\0\0\0\xe0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x03\0\0\0\0\0\0\x04\0\ +\0\0\x01\0\0\0\0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x04\0\0\0\0\0\0\x04\0\ +\0\0\x01\0\0\0\x20\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x04\0\0\0\0\0\0\x04\ +\0\0\0\x01\0\0\0\x40\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x04\0\0\0\0\0\0\ +\x04\0\0\0\x01\0\0\0\x60\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x04\0\0\0\0\0\ +\0\x04\0\0\0\x01\0\0\0\x80\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x04\0\0\0\0\ +\0\0\x04\0\0\0\x01\0\0\0\xa0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x04\0\0\0\ +\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x04\0\0\ +\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x04\0\ +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x05\0\ +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x05\ +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\ +\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\x70\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\x90\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\ +\0\0\xb0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x05\0\0\0\0\0\0\x04\0\0\0\x01\ +\0\0\0\xd0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x05\0\0\0\0\0\0\x04\0\0\0\ +\x01\0\0\0\xf0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x06\0\0\0\0\0\0\x04\0\0\0\ +\x01\0\0\0\x10\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x06\0\0\0\0\0\0\x04\0\0\ +\0\x01\0\0\0\x30\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x06\0\0\0\0\0\0\x04\0\ +\0\0\x01\0\0\0\x50\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x06\0\0\0\0\0\0\x04\ +\0\0\0\x01\0\0\0\x70\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x06\0\0\0\0\0\0\ +\x04\0\0\0\x01\0\0\0\x90\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x06\0\0\0\0\0\ +\0\x04\0\0\0\x01\0\0\0\xb0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x06\0\0\0\0\ +\0\0\x04\0\0\0\x01\0\0\0\xd0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x06\0\0\0\ +\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x07\0\0\0\ +\0\0\0\x04\0\0\0\x01\0\0\0\x10\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x07\0\0\ +\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x07\0\ +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x07\ +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\ +\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\xa0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\xc0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\ +\0\0\xe0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x07\0\0\0\0\0\0\x04\0\0\0\x01\ +\0\0\0\0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x08\0\0\0\0\0\0\x04\0\0\0\x01\ +\0\0\0\x20\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x08\0\0\0\0\0\0\x04\0\0\0\ +\x01\0\0\0\x40\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x08\0\0\0\0\0\0\x04\0\0\ +\0\x01\0\0\0\x60\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x08\0\0\0\0\0\0\x04\0\ +\0\0\x01\0\0\0\x80\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x08\0\0\0\0\0\0\x04\ +\0\0\0\x01\0\0\0\xa0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x08\0\0\0\0\0\0\ +\x04\0\0\0\x01\0\0\0\xc0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x08\0\0\0\0\0\ +\0\x04\0\0\0\x01\0\0\0\xe0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x08\0\0\0\0\ +\0\0\x04\0\0\0\x01\0\0\0\0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x09\0\0\0\0\ +\0\0\x04\0\0\0\x01\0\0\0\x20\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x09\0\0\0\ +\0\0\0\x04\0\0\0\x01\0\0\0\x40\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x09\0\0\ +\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x09\0\ +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x09\ +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\ +\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\xd0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\xf0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\x10\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\ +\0\0\x30\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0a\0\0\0\0\0\0\x04\0\0\0\x01\ +\0\0\0\x50\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0a\0\0\0\0\0\0\x04\0\0\0\ +\x01\0\0\0\x70\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0a\0\0\0\0\0\0\x04\0\0\ +\0\x01\0\0\0\x90\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0a\0\0\0\0\0\0\x04\0\ +\0\0\x01\0\0\0\xb0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0a\0\0\0\0\0\0\x04\ +\0\0\0\x01\0\0\0\xd0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x0a\0\0\0\0\0\0\ +\x04\0\0\0\x01\0\0\0\xf0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0b\0\0\0\0\0\0\ +\x04\0\0\0\x01\0\0\0\x10\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0b\0\0\0\0\0\ +\0\x04\0\0\0\x01\0\0\0\x30\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0b\0\0\0\0\ +\0\0\x04\0\0\0\x01\0\0\0\x50\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0b\0\0\0\ +\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0b\0\0\ +\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0b\0\ +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0b\ +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\ +\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\ +\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\ +\x20\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\ +\0\x40\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\ +\0\0\x60\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0c\0\0\0\0\0\0\x04\0\0\0\x01\ +\0\0\0\x80\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0c\0\0\0\0\0\0\x04\0\0\0\ +\x01\0\0\0\x40\x41\x42\x43\x44\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\ \x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\ -\x6d\x61\x70\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\ -\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\ -\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\ -\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x5f\x6c\x69\x63\x65\ -\x6e\x73\x65\0\x2e\x72\x65\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x74\x61\ -\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\ -\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\0\x2e\ -\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x4c\x42\x42\x30\x5f\ -\x39\0\x4c\x42\x42\x30\x5f\x38\x39\0\x4c\x42\x42\x30\x5f\x36\x39\0\x4c\x42\x42\ -\x30\x5f\x35\x39\0\x4c\x42\x42\x30\x5f\x31\x39\0\x4c\x42\x42\x30\x5f\x31\x30\ -\x39\0\x4c\x42\x42\x30\x5f\x39\x38\0\x4c\x42\x42\x30\x5f\x37\x38\0\x4c\x42\x42\ -\x30\x5f\x34\x38\0\x4c\x42\x42\x30\x5f\x31\x38\0\x4c\x42\x42\x30\x5f\x38\x37\0\ -\x4c\x42\x42\x30\x5f\x34\x37\0\x4c\x42\x42\x30\x5f\x33\x37\0\x4c\x42\x42\x30\ -\x5f\x31\x37\0\x4c\x42\x42\x30\x5f\x31\x30\x37\0\x4c\x42\x42\x30\x5f\x39\x36\0\ -\x4c\x42\x42\x30\x5f\x37\x36\0\x4c\x42\x42\x30\x5f\x36\x36\0\x4c\x42\x42\x30\ -\x5f\x34\x36\0\x4c\x42\x42\x30\x5f\x33\x36\0\x4c\x42\x42\x30\x5f\x32\x36\0\x4c\ -\x42\x42\x30\x5f\x31\x30\x36\0\x4c\x42\x42\x30\x5f\x36\x35\0\x4c\x42\x42\x30\ -\x5f\x34\x35\0\x4c\x42\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\ -\x42\x30\x5f\x35\x34\0\x4c\x42\x42\x30\x5f\x34\x34\0\x4c\x42\x42\x30\x5f\x32\ -\x34\0\x4c\x42\x42\x30\x5f\x31\x30\x34\0\x4c\x42\x42\x30\x5f\x39\x33\0\x4c\x42\ -\x42\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x35\x33\0\x4c\x42\x42\x30\x5f\x34\ -\x33\0\x4c\x42\x42\x30\x5f\x32\x33\0\x4c\x42\x42\x30\x5f\x31\x30\x33\0\x4c\x42\ -\x42\x30\x5f\x38\x32\0\x4c\x42\x42\x30\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x31\ -\x30\x32\0\x4c\x42\x42\x30\x5f\x39\x31\0\x4c\x42\x42\x30\x5f\x38\x31\0\x4c\x42\ -\x42\x30\x5f\x37\x31\0\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\x42\x42\x30\x5f\x35\ -\x31\0\x4c\x42\x42\x30\x5f\x34\x31\0\x4c\x42\x42\x30\x5f\x32\x31\0\x4c\x42\x42\ -\x30\x5f\x31\x31\0\x4c\x42\x42\x30\x5f\x31\x31\x31\0\x4c\x42\x42\x30\x5f\x31\ -\x30\x31\0\x4c\x42\x42\x30\x5f\x38\x30\0\x4c\x42\x42\x30\x5f\x36\x30\0\x4c\x42\ -\x42\x30\x5f\x35\x30\0\x4c\x42\x42\x30\x5f\x31\x30\0\x4c\x42\x42\x30\x5f\x31\ -\x31\x30\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\ -\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa0\x1a\0\0\0\0\0\0\x71\x02\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\ -\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\xd8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x56\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\x60\x1a\0\0\0\0\0\0\x30\0\0\0\0\0\0\0\x09\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\ -\x10\0\0\0\0\0\0\0\x20\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\ -\x14\0\0\0\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\x6c\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x54\x14\0\0\0\0\0\ -\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x78\0\0\ -\0\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x60\x14\0\0\0\0\0\0\x30\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x74\0\0\0\x09\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x1a\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x09\0\0\0\ -\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\xb2\0\0\0\x02\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\x90\x14\0\0\0\0\0\0\xd0\x05\0\0\0\0\0\0\x01\0\0\0\x39\0\0\ -\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0"; +\x2e\x72\x65\x6c\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x2e\x6d\x61\x70\x73\0\x74\ +\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\ +\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\ +\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\x6e\x5f\x72\x73\x73\ +\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\x64\x72\ +\x73\x69\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x74\x61\x70\x5f\x72\x73\x73\ +\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\ +\x62\x6c\x65\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\ +\x72\x65\x6c\x2e\x42\x54\x46\0\x4c\x42\x42\x30\x5f\x39\0\x4c\x42\x42\x30\x5f\ +\x39\x39\0\x4c\x42\x42\x30\x5f\x37\x39\0\x4c\x42\x42\x30\x5f\x31\x30\x39\0\x4c\ +\x42\x42\x30\x5f\x38\x38\0\x4c\x42\x42\x30\x5f\x34\x38\0\x4c\x42\x42\x30\x5f\ +\x31\x38\0\x4c\x42\x42\x30\x5f\x31\x30\x38\0\x4c\x42\x42\x30\x5f\x39\x37\0\x4c\ +\x42\x42\x30\x5f\x37\x37\0\x4c\x42\x42\x30\x5f\x36\x37\0\x4c\x42\x42\x30\x5f\ +\x34\x37\0\x4c\x42\x42\x30\x5f\x31\x37\0\x4c\x42\x42\x30\x5f\x36\x36\0\x4c\x42\ +\x42\x30\x5f\x34\x36\0\x4c\x42\x42\x30\x5f\x33\x36\0\x4c\x42\x42\x30\x5f\x31\ +\x30\x36\0\x4c\x42\x42\x30\x5f\x35\x35\0\x4c\x42\x42\x30\x5f\x34\x35\0\x4c\x42\ +\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x32\x35\0\x4c\x42\x42\x30\x5f\x31\ +\x30\x35\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x30\x5f\x39\x34\0\x4c\x42\x42\ +\x30\x5f\x38\x34\0\x4c\x42\x42\x30\x5f\x35\x34\0\x4c\x42\x42\x30\x5f\x34\x34\0\ +\x4c\x42\x42\x30\x5f\x33\x34\0\x4c\x42\x42\x30\x5f\x31\x30\x34\0\x4c\x42\x42\ +\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x35\x33\0\x4c\x42\x42\x30\x5f\x32\x33\0\ +\x4c\x42\x42\x30\x5f\x31\x30\x33\0\x4c\x42\x42\x30\x5f\x39\x32\0\x4c\x42\x42\ +\x30\x5f\x38\x32\0\x4c\x42\x42\x30\x5f\x37\x32\0\x4c\x42\x42\x30\x5f\x36\x32\0\ +\x4c\x42\x42\x30\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x34\x32\0\x4c\x42\x42\x30\ +\x5f\x32\x32\0\x4c\x42\x42\x30\x5f\x31\x30\x32\0\x4c\x42\x42\x30\x5f\x38\x31\0\ +\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\x42\x42\x30\x5f\x35\x31\0\x4c\x42\x42\x30\ +\x5f\x31\x31\0\x4c\x42\x42\x30\x5f\x39\x30\0\x4c\x42\x42\x30\x5f\x37\x30\0\x4c\ +\x42\x42\x30\x5f\x36\x30\0\x4c\x42\x42\x30\x5f\x35\x30\0\x4c\x42\x42\x30\x5f\ +\x34\x30\0\x4c\x42\x42\x30\x5f\x32\x30\0\x4c\x42\x42\x30\x5f\x31\x31\x30\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xae\0\0\0\x03\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x25\x4a\0\0\0\0\0\0\x6d\x02\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\x68\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x40\0\0\0\0\0\0\0\xe0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x64\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x3d\0\0\ +\0\0\0\0\x30\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\ +\0\0\x2d\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x14\0\0\0\0\0\0\ +\x60\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x88\0\0\0\ +\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x80\x14\0\0\0\0\0\0\x07\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\0\0\0\x01\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x88\x14\0\0\0\0\0\0\x8d\x16\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbe\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x70\x3d\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\x0c\0\0\0\x07\0\0\0\x08\ +\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x24\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x18\x2b\0\0\0\0\0\0\xa0\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x20\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\ +\x3d\0\0\0\0\0\0\x70\x0c\0\0\0\0\0\0\x0c\0\0\0\x09\0\0\0\x08\0\0\0\0\0\0\0\x10\ +\0\0\0\0\0\0\0\x79\0\0\0\x03\x4c\xff\x6f\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\x20\ +\x4a\0\0\0\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\xb6\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x37\0\0\0\0\0\0\ +\x88\x05\0\0\0\0\0\0\x01\0\0\0\x36\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0"; +} - return 0; -err: - bpf_object__destroy_skeleton(s); - return -1; +#ifdef __cplusplus +struct rss_bpf *rss_bpf::open(const struct bpf_object_open_opts *opts) { return rss_bpf__open_opts(opts); } +struct rss_bpf *rss_bpf::open_and_load() { return rss_bpf__open_and_load(); } +int rss_bpf::load(struct rss_bpf *skel) { return rss_bpf__load(skel); } +int rss_bpf::attach(struct rss_bpf *skel) { return rss_bpf__attach(skel); } +void rss_bpf::detach(struct rss_bpf *skel) { rss_bpf__detach(skel); } +void rss_bpf::destroy(struct rss_bpf *skel) { rss_bpf__destroy(skel); } +const void *rss_bpf::elf_bytes(size_t *sz) { return rss_bpf__elf_bytes(sz); } +#endif /* __cplusplus */ + +__attribute__((unused)) static void +rss_bpf__assert(struct rss_bpf *s __attribute__((unused))) +{ +#ifdef __cplusplus +#define _Static_assert static_assert +#endif +#ifdef __cplusplus +#undef _Static_assert +#endif } #endif /* __RSS_BPF_SKEL_H__ */ diff --git a/gitdm.config b/gitdm.config index 288b100d89..907ffde017 100644 --- a/gitdm.config +++ b/gitdm.config @@ -33,6 +33,7 @@ EmailMap contrib/gitdm/domain-map GroupMap contrib/gitdm/group-map-cadence Cadence Design Systems GroupMap contrib/gitdm/group-map-codeweavers CodeWeavers +GroupMap contrib/gitdm/group-map-facebook Facebook GroupMap contrib/gitdm/group-map-ibm IBM GroupMap contrib/gitdm/group-map-janustech Janus Technologies GroupMap contrib/gitdm/group-map-netflix Netflix diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx index 754b1e8408..47d63d26db 100644 --- a/hmp-commands-info.hx +++ b/hmp-commands-info.hx @@ -993,3 +993,17 @@ SRST ``info virtio-queue-element`` *path* *queue* [*index*] Display element of a given virtio queue ERST + + { + .name = "cryptodev", + .args_type = "", + .params = "", + .help = "show the crypto devices", + .cmd = hmp_info_cryptodev, + .flags = "p", + }, + +SRST + ``info cryptodev`` + Show the crypto devices. +ERST diff --git a/hw/acpi/acpi-pci-hotplug-stub.c b/hw/acpi/acpi-pci-hotplug-stub.c index a43f6dafc9..dcee3ad7a1 100644 --- a/hw/acpi/acpi-pci-hotplug-stub.c +++ b/hw/acpi/acpi-pci-hotplug-stub.c @@ -5,8 +5,7 @@ const VMStateDescription vmstate_acpi_pcihp_pci_status; void acpi_pcihp_init(Object *owner, AcpiPciHpState *s, PCIBus *root_bus, - MemoryRegion *address_space_io, bool bridges_enabled, - uint16_t io_base) + MemoryRegion *address_space_io, uint16_t io_base) { return; } @@ -36,8 +35,12 @@ void acpi_pcihp_device_unplug_request_cb(HotplugHandler *hotplug_dev, return; } -void acpi_pcihp_reset(AcpiPciHpState *s, bool acpihp_root_off) +void acpi_pcihp_reset(AcpiPciHpState *s) { return; } +bool acpi_pcihp_is_hotpluggbale_bus(AcpiPciHpState *s, BusState *bus) +{ + return true; +} diff --git a/hw/acpi/ich9.c b/hw/acpi/ich9.c index d23bfcaa6b..25e2c7243e 100644 --- a/hw/acpi/ich9.c +++ b/hw/acpi/ich9.c @@ -218,7 +218,7 @@ static bool vmstate_test_use_pcihp(void *opaque) { ICH9LPCPMRegs *s = opaque; - return s->use_acpi_hotplug_bridge; + return s->acpi_pci_hotplug.use_acpi_hotplug_bridge; } static const VMStateDescription vmstate_pcihp_state = { @@ -277,8 +277,8 @@ static void pm_reset(void *opaque) } pm->smi_en_wmask = ~0; - if (pm->use_acpi_hotplug_bridge) { - acpi_pcihp_reset(&pm->acpi_pci_hotplug, true); + if (pm->acpi_pci_hotplug.use_acpi_hotplug_bridge) { + acpi_pcihp_reset(&pm->acpi_pci_hotplug); } acpi_update_sci(&pm->acpi_regs, pm->irq); @@ -316,12 +316,11 @@ void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm, qemu_irq sci_irq) acpi_pm_tco_init(&pm->tco_regs, &pm->io); } - if (pm->use_acpi_hotplug_bridge) { + if (pm->acpi_pci_hotplug.use_acpi_hotplug_bridge) { acpi_pcihp_init(OBJECT(lpc_pci), &pm->acpi_pci_hotplug, pci_get_bus(lpc_pci), pci_address_space_io(lpc_pci), - true, ACPI_PCIHP_ADDR_ICH9); qbus_set_hotplug_handler(BUS(pci_get_bus(lpc_pci)), @@ -403,14 +402,14 @@ static bool ich9_pm_get_acpi_pci_hotplug(Object *obj, Error **errp) { ICH9LPCState *s = ICH9_LPC_DEVICE(obj); - return s->pm.use_acpi_hotplug_bridge; + return s->pm.acpi_pci_hotplug.use_acpi_hotplug_bridge; } static void ich9_pm_set_acpi_pci_hotplug(Object *obj, bool value, Error **errp) { ICH9LPCState *s = ICH9_LPC_DEVICE(obj); - s->pm.use_acpi_hotplug_bridge = value; + s->pm.acpi_pci_hotplug.use_acpi_hotplug_bridge = value; } static bool ich9_pm_get_keep_pci_slot_hpc(Object *obj, Error **errp) @@ -435,7 +434,7 @@ void ich9_pm_add_properties(Object *obj, ICH9LPCPMRegs *pm) pm->disable_s3 = 0; pm->disable_s4 = 0; pm->s4_val = 2; - pm->use_acpi_hotplug_bridge = true; + pm->acpi_pci_hotplug.use_acpi_hotplug_bridge = true; pm->keep_pci_slot_hpc = true; pm->enable_tco = true; @@ -579,6 +578,12 @@ void ich9_pm_device_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, } } +bool ich9_pm_is_hotpluggable_bus(HotplugHandler *hotplug_dev, BusState *bus) +{ + ICH9LPCState *lpc = ICH9_LPC_DEVICE(hotplug_dev); + return acpi_pcihp_is_hotpluggbale_bus(&lpc->pm.acpi_pci_hotplug, bus); +} + void ich9_pm_ospm_status(AcpiDeviceIf *adev, ACPIOSTInfoList ***list) { ICH9LPCState *s = ICH9_LPC_DEVICE(adev); diff --git a/hw/acpi/pci-bridge.c b/hw/acpi/pci-bridge.c index 5f3ee5157f..7baa7034a1 100644 --- a/hw/acpi/pci-bridge.c +++ b/hw/acpi/pci-bridge.c @@ -21,7 +21,17 @@ void build_pci_bridge_aml(AcpiDevAmlIf *adev, Aml *scope) { PCIBridge *br = PCI_BRIDGE(adev); - if (object_property_find(OBJECT(&br->sec_bus), ACPI_PCIHP_PROP_BSEL)) { - build_append_pci_bus_devices(scope, pci_bridge_get_sec_bus(br)); + if (!DEVICE(br)->hotplugged) { + PCIBus *sec_bus = pci_bridge_get_sec_bus(br); + + build_append_pci_bus_devices(scope, sec_bus); + + /* + * generate hotplug slots descriptors if + * bridge has ACPI PCI hotplug attached, + */ + if (object_property_find(OBJECT(sec_bus), ACPI_PCIHP_PROP_BSEL)) { + build_append_pcihp_slots(scope, sec_bus); + } } } diff --git a/hw/acpi/pcihp.c b/hw/acpi/pcihp.c index 5dc7377411..dcfb779a7a 100644 --- a/hw/acpi/pcihp.c +++ b/hw/acpi/pcihp.c @@ -54,21 +54,6 @@ typedef struct AcpiPciHpFind { PCIBus *bus; } AcpiPciHpFind; -static gint g_cmp_uint32(gconstpointer a, gconstpointer b, gpointer user_data) -{ - return a - b; -} - -static GSequence *pci_acpi_index_list(void) -{ - static GSequence *used_acpi_index_list; - - if (!used_acpi_index_list) { - used_acpi_index_list = g_sequence_new(NULL); - } - return used_acpi_index_list; -} - static int acpi_pcihp_get_bsel(PCIBus *bus) { Error *local_err = NULL; @@ -136,20 +121,6 @@ static void acpi_set_pci_info(bool has_bridge_hotplug) } } -static void acpi_pcihp_disable_root_bus(void) -{ - Object *host = acpi_get_i386_pci_host(); - PCIBus *bus; - - bus = PCI_HOST_BRIDGE(host)->bus; - if (bus && qbus_is_hotpluggable(BUS(bus))) { - /* setting the hotplug handler to NULL makes the bus non-hotpluggable */ - qbus_set_hotplug_handler(BUS(bus), NULL); - } - - return; -} - static void acpi_pcihp_test_hotplug_bus(PCIBus *bus, void *opaque) { AcpiPciHpFind *find = opaque; @@ -291,17 +262,12 @@ static void acpi_pcihp_update(AcpiPciHpState *s) } } -void acpi_pcihp_reset(AcpiPciHpState *s, bool acpihp_root_off) +void acpi_pcihp_reset(AcpiPciHpState *s) { - if (acpihp_root_off) { - acpi_pcihp_disable_root_bus(); - } - acpi_set_pci_info(!s->legacy_piix); + acpi_set_pci_info(s->use_acpi_hotplug_bridge); acpi_pcihp_update(s); } -#define ONBOARD_INDEX_MAX (16 * 1024 - 1) - void acpi_pcihp_device_pre_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { @@ -314,34 +280,6 @@ void acpi_pcihp_device_pre_plug_cb(HotplugHandler *hotplug_dev, ACPI_PCIHP_PROP_BSEL "' set"); return; } - - /* - * capped by systemd (see: udev-builtin-net_id.c) - * as it's the only known user honor it to avoid users - * misconfigure QEMU and then wonder why acpi-index doesn't work - */ - if (pdev->acpi_index > ONBOARD_INDEX_MAX) { - error_setg(errp, "acpi-index should be less or equal to %u", - ONBOARD_INDEX_MAX); - return; - } - - /* - * make sure that acpi-index is unique across all present PCI devices - */ - if (pdev->acpi_index) { - GSequence *used_indexes = pci_acpi_index_list(); - - if (g_sequence_lookup(used_indexes, GINT_TO_POINTER(pdev->acpi_index), - g_cmp_uint32, NULL)) { - error_setg(errp, "a PCI device with acpi-index = %" PRIu32 - " already exist", pdev->acpi_index); - return; - } - g_sequence_insert_sorted(used_indexes, - GINT_TO_POINTER(pdev->acpi_index), - g_cmp_uint32, NULL); - } } void acpi_pcihp_device_plug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s, @@ -361,17 +299,10 @@ void acpi_pcihp_device_plug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s, * Overwrite the default hotplug handler with the ACPI PCI one * for cold plugged bridges only. */ - if (!s->legacy_piix && + if (s->use_acpi_hotplug_bridge && object_dynamic_cast(OBJECT(dev), TYPE_PCI_BRIDGE)) { PCIBus *sec = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev)); - /* Remove all hot-plug handlers if hot-plug is disabled on slot */ - if (object_dynamic_cast(OBJECT(dev), TYPE_PCIE_SLOT) && - !PCIE_SLOT(pdev)->hotplug) { - qbus_set_hotplug_handler(BUS(sec), NULL); - return; - } - qbus_set_hotplug_handler(BUS(sec), OBJECT(hotplug_dev)); /* We don't have to overwrite any other hotplug handler yet */ assert(QLIST_EMPTY(&sec->child)); @@ -401,17 +332,6 @@ void acpi_pcihp_device_unplug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s, trace_acpi_pci_unplug(PCI_SLOT(pdev->devfn), acpi_pcihp_get_bsel(pci_get_bus(pdev))); - /* - * clean up acpi-index so it could reused by another device - */ - if (pdev->acpi_index) { - GSequence *used_indexes = pci_acpi_index_list(); - - g_sequence_remove(g_sequence_lookup(used_indexes, - GINT_TO_POINTER(pdev->acpi_index), - g_cmp_uint32, NULL)); - } - qdev_unrealize(dev); } @@ -441,6 +361,24 @@ void acpi_pcihp_device_unplug_request_cb(HotplugHandler *hotplug_dev, acpi_send_event(DEVICE(hotplug_dev), ACPI_PCI_HOTPLUG_STATUS); } +bool acpi_pcihp_is_hotpluggbale_bus(AcpiPciHpState *s, BusState *bus) +{ + Object *o = OBJECT(bus->parent); + + if (s->use_acpi_hotplug_bridge && + object_dynamic_cast(o, TYPE_PCI_BRIDGE)) { + if (object_dynamic_cast(o, TYPE_PCIE_SLOT) && !PCIE_SLOT(o)->hotplug) { + return false; + } + return true; + } + + if (s->use_acpi_root_pci_hotplug) { + return true; + } + return false; +} + static uint64_t pci_read(void *opaque, hwaddr addr, unsigned int size) { AcpiPciHpState *s = opaque; @@ -454,7 +392,7 @@ static uint64_t pci_read(void *opaque, hwaddr addr, unsigned int size) switch (addr) { case PCI_UP_BASE: val = s->acpi_pcihp_pci_status[bsel].up; - if (!s->legacy_piix) { + if (s->use_acpi_hotplug_bridge) { s->acpi_pcihp_pci_status[bsel].up = 0; } trace_acpi_pci_up_read(val); @@ -529,7 +467,8 @@ static void pci_write(void *opaque, hwaddr addr, uint64_t data, trace_acpi_pci_ej_write(addr, data); break; case PCI_SEL_BASE: - s->hotplug_select = s->legacy_piix ? ACPI_PCIHP_BSEL_DEFAULT : data; + s->hotplug_select = s->use_acpi_hotplug_bridge ? data : + ACPI_PCIHP_BSEL_DEFAULT; trace_acpi_pci_sel_write(addr, data); default: break; @@ -547,14 +486,13 @@ static const MemoryRegionOps acpi_pcihp_io_ops = { }; void acpi_pcihp_init(Object *owner, AcpiPciHpState *s, PCIBus *root_bus, - MemoryRegion *address_space_io, bool bridges_enabled, + MemoryRegion *address_space_io, uint16_t io_base) { s->io_len = ACPI_PCIHP_SIZE; s->io_base = io_base; s->root = root_bus; - s->legacy_piix = !bridges_enabled; memory_region_init_io(&s->io, owner, &acpi_pcihp_io_ops, s, "acpi-pci-hotplug", s->io_len); diff --git a/hw/acpi/piix4.c b/hw/acpi/piix4.c index eac2125abd..63d2113b86 100644 --- a/hw/acpi/piix4.c +++ b/hw/acpi/piix4.c @@ -170,14 +170,14 @@ static const VMStateDescription vmstate_pci_status = { static bool vmstate_test_use_acpi_hotplug_bridge(void *opaque, int version_id) { PIIX4PMState *s = opaque; - return s->use_acpi_hotplug_bridge; + return s->acpi_pci_hotplug.use_acpi_hotplug_bridge; } static bool vmstate_test_no_use_acpi_hotplug_bridge(void *opaque, int version_id) { PIIX4PMState *s = opaque; - return !s->use_acpi_hotplug_bridge; + return !s->acpi_pci_hotplug.use_acpi_hotplug_bridge; } static bool vmstate_test_use_memhp(void *opaque) @@ -234,7 +234,8 @@ static bool piix4_vmstate_need_smbus(void *opaque, int version_id) static bool vmstate_test_migrate_acpi_index(void *opaque, int version_id) { PIIX4PMState *s = PIIX4_PM(opaque); - return s->use_acpi_hotplug_bridge && !s->not_migrate_acpi_index; + return s->acpi_pci_hotplug.use_acpi_hotplug_bridge && + !s->not_migrate_acpi_index; } /* qemu-kvm 1.2 uses version 3 but advertised as 2 @@ -303,8 +304,9 @@ static void piix4_pm_reset(DeviceState *dev) acpi_update_sci(&s->ar, s->irq); pm_io_space_update(s); - if (s->use_acpi_hotplug_bridge || s->use_acpi_root_pci_hotplug) { - acpi_pcihp_reset(&s->acpi_pci_hotplug, !s->use_acpi_root_pci_hotplug); + if (s->acpi_pci_hotplug.use_acpi_hotplug_bridge || + s->acpi_pci_hotplug.use_acpi_root_pci_hotplug) { + acpi_pcihp_reset(&s->acpi_pci_hotplug); } } @@ -402,6 +404,13 @@ static void piix4_device_unplug_cb(HotplugHandler *hotplug_dev, } } +static bool piix4_is_hotpluggable_bus(HotplugHandler *hotplug_dev, + BusState *bus) +{ + PIIX4PMState *s = PIIX4_PM(hotplug_dev); + return acpi_pcihp_is_hotpluggbale_bus(&s->acpi_pci_hotplug, bus); +} + static void piix4_pm_machine_ready(Notifier *n, void *opaque) { PIIX4PMState *s = container_of(n, PIIX4PMState, machine_ready); @@ -487,12 +496,11 @@ static void piix4_pm_realize(PCIDevice *dev, Error **errp) qemu_add_machine_init_done_notifier(&s->machine_ready); if (xen_enabled()) { - s->use_acpi_hotplug_bridge = false; + s->acpi_pci_hotplug.use_acpi_hotplug_bridge = false; } piix4_acpi_system_hot_add_init(pci_address_space_io(dev), pci_get_bus(dev), s); - qbus_set_hotplug_handler(BUS(pci_get_bus(dev)), OBJECT(s)); piix4_pm_add_properties(s); } @@ -561,9 +569,11 @@ static void piix4_acpi_system_hot_add_init(MemoryRegion *parent, "acpi-gpe0", GPE_LEN); memory_region_add_subregion(parent, GPE_BASE, &s->io_gpe); - if (s->use_acpi_hotplug_bridge || s->use_acpi_root_pci_hotplug) { + if (s->acpi_pci_hotplug.use_acpi_hotplug_bridge || + s->acpi_pci_hotplug.use_acpi_root_pci_hotplug) { acpi_pcihp_init(OBJECT(s), &s->acpi_pci_hotplug, bus, parent, - s->use_acpi_hotplug_bridge, ACPI_PCIHP_ADDR_PIIX4); + ACPI_PCIHP_ADDR_PIIX4); + qbus_set_hotplug_handler(BUS(pci_get_bus(PCI_DEVICE(s))), OBJECT(s)); } s->cpu_hotplug_legacy = true; @@ -602,9 +612,9 @@ static Property piix4_pm_properties[] = { DEFINE_PROP_UINT8(ACPI_PM_PROP_S4_DISABLED, PIIX4PMState, disable_s4, 0), DEFINE_PROP_UINT8(ACPI_PM_PROP_S4_VAL, PIIX4PMState, s4_val, 2), DEFINE_PROP_BOOL(ACPI_PM_PROP_ACPI_PCIHP_BRIDGE, PIIX4PMState, - use_acpi_hotplug_bridge, true), + acpi_pci_hotplug.use_acpi_hotplug_bridge, true), DEFINE_PROP_BOOL(ACPI_PM_PROP_ACPI_PCI_ROOTHP, PIIX4PMState, - use_acpi_root_pci_hotplug, true), + acpi_pci_hotplug.use_acpi_root_pci_hotplug, true), DEFINE_PROP_BOOL("memory-hotplug-support", PIIX4PMState, acpi_memory_hotplug.is_enabled, true), DEFINE_PROP_BOOL("smm-compat", PIIX4PMState, smm_compat, false), @@ -641,6 +651,7 @@ static void piix4_pm_class_init(ObjectClass *klass, void *data) hc->plug = piix4_device_plug_cb; hc->unplug_request = piix4_device_unplug_request_cb; hc->unplug = piix4_device_unplug_cb; + hc->is_hotpluggable_bus = piix4_is_hotpluggable_bus; adevc->ospm_status = piix4_ospm_status; adevc->send_event = piix4_send_gpe; adevc->madt_cpu = pc_madt_cpu_entry; diff --git a/hw/core/machine.c b/hw/core/machine.c index 1cf6822e06..45e3d24fdc 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -40,6 +40,7 @@ #include "hw/virtio/virtio-pci.h" GlobalProperty hw_compat_7_2[] = { + { "e1000e", "migrate-timadj", "off" }, { "virtio-mem", "x-early-migration", "false" }, }; const size_t hw_compat_7_2_len = G_N_ELEMENTS(hw_compat_7_2); diff --git a/hw/cxl/cxl-component-utils.c b/hw/cxl/cxl-component-utils.c index 3edd303a33..b665d4f565 100644 --- a/hw/cxl/cxl-component-utils.c +++ b/hw/cxl/cxl-component-utils.c @@ -141,17 +141,19 @@ static void ras_init_common(uint32_t *reg_state, uint32_t *write_msk) * Error status is RW1C but given bits are not yet set, it can * be handled as RO. */ - reg_state[R_CXL_RAS_UNC_ERR_STATUS] = 0; + stl_le_p(reg_state + R_CXL_RAS_UNC_ERR_STATUS, 0); + stl_le_p(write_msk + R_CXL_RAS_UNC_ERR_STATUS, 0x1cfff); /* Bits 12-13 and 17-31 reserved in CXL 2.0 */ - reg_state[R_CXL_RAS_UNC_ERR_MASK] = 0x1cfff; - write_msk[R_CXL_RAS_UNC_ERR_MASK] = 0x1cfff; - reg_state[R_CXL_RAS_UNC_ERR_SEVERITY] = 0x1cfff; - write_msk[R_CXL_RAS_UNC_ERR_SEVERITY] = 0x1cfff; - reg_state[R_CXL_RAS_COR_ERR_STATUS] = 0; - reg_state[R_CXL_RAS_COR_ERR_MASK] = 0x7f; - write_msk[R_CXL_RAS_COR_ERR_MASK] = 0x7f; + stl_le_p(reg_state + R_CXL_RAS_UNC_ERR_MASK, 0x1cfff); + stl_le_p(write_msk + R_CXL_RAS_UNC_ERR_MASK, 0x1cfff); + stl_le_p(reg_state + R_CXL_RAS_UNC_ERR_SEVERITY, 0x1cfff); + stl_le_p(write_msk + R_CXL_RAS_UNC_ERR_SEVERITY, 0x1cfff); + stl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS, 0); + stl_le_p(write_msk + R_CXL_RAS_COR_ERR_STATUS, 0x7f); + stl_le_p(reg_state + R_CXL_RAS_COR_ERR_MASK, 0x7f); + stl_le_p(write_msk + R_CXL_RAS_COR_ERR_MASK, 0x7f); /* CXL switches and devices must set */ - reg_state[R_CXL_RAS_ERR_CAP_CTRL] = 0x00; + stl_le_p(reg_state + R_CXL_RAS_ERR_CAP_CTRL, 0x200); } static void hdm_init_common(uint32_t *reg_state, uint32_t *write_msk, diff --git a/hw/cxl/cxl-host.c b/hw/cxl/cxl-host.c index 3c1ec8732a..6e923ceeaf 100644 --- a/hw/cxl/cxl-host.c +++ b/hw/cxl/cxl-host.c @@ -146,21 +146,28 @@ static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr) return NULL; } - hb_cstate = cxl_get_hb_cstate(hb); - if (!hb_cstate) { - return NULL; - } + if (cxl_get_hb_passthrough(hb)) { + rp = pcie_find_port_first(hb->bus); + if (!rp) { + return NULL; + } + } else { + hb_cstate = cxl_get_hb_cstate(hb); + if (!hb_cstate) { + return NULL; + } - cache_mem = hb_cstate->crb.cache_mem_registers; + cache_mem = hb_cstate->crb.cache_mem_registers; - target_found = cxl_hdm_find_target(cache_mem, addr, &target); - if (!target_found) { - return NULL; - } + target_found = cxl_hdm_find_target(cache_mem, addr, &target); + if (!target_found) { + return NULL; + } - rp = pcie_find_port_by_pn(hb->bus, target); - if (!rp) { - return NULL; + rp = pcie_find_port_by_pn(hb->bus, target); + if (!rp) { + return NULL; + } } d = pci_bridge_get_sec_bus(PCI_BRIDGE(rp))->devices[0]; diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index b19fb4259e..ec857a117e 100644 --- a/hw/i386/acpi-build.c +++ b/hw/i386/acpi-build.c @@ -373,6 +373,104 @@ Aml *aml_pci_device_dsm(void) return method; } +static void build_append_pci_dsm_func0_common(Aml *ctx, Aml *retvar) +{ + Aml *UUID, *ifctx1; + uint8_t byte_list[1] = { 0 }; /* nothing supported yet */ + + aml_append(ctx, aml_store(aml_buffer(1, byte_list), retvar)); + /* + * PCI Firmware Specification 3.1 + * 4.6. _DSM Definitions for PCI + */ + UUID = aml_touuid("E5C937D0-3553-4D7A-9117-EA4D19C3434D"); + ifctx1 = aml_if(aml_lnot(aml_equal(aml_arg(0), UUID))); + { + /* call is for unsupported UUID, bail out */ + aml_append(ifctx1, aml_return(retvar)); + } + aml_append(ctx, ifctx1); + + ifctx1 = aml_if(aml_lless(aml_arg(1), aml_int(2))); + { + /* call is for unsupported REV, bail out */ + aml_append(ifctx1, aml_return(retvar)); + } + aml_append(ctx, ifctx1); +} + +static Aml *aml_pci_edsm(void) +{ + Aml *method, *ifctx; + Aml *zero = aml_int(0); + Aml *func = aml_arg(2); + Aml *ret = aml_local(0); + Aml *aidx = aml_local(1); + Aml *params = aml_arg(4); + + method = aml_method("EDSM", 5, AML_SERIALIZED); + + /* get supported functions */ + ifctx = aml_if(aml_equal(func, zero)); + { + /* 1: have supported functions */ + /* 7: support for function 7 */ + const uint8_t caps = 1 | BIT(7); + build_append_pci_dsm_func0_common(ifctx, ret); + aml_append(ifctx, aml_store(aml_int(caps), aml_index(ret, zero))); + aml_append(ifctx, aml_return(ret)); + } + aml_append(method, ifctx); + + /* handle specific functions requests */ + /* + * PCI Firmware Specification 3.1 + * 4.6.7. _DSM for Naming a PCI or PCI Express Device Under + * Operating Systems + */ + ifctx = aml_if(aml_equal(func, aml_int(7))); + { + Aml *pkg = aml_package(2); + aml_append(pkg, zero); + /* optional, if not impl. should return null string */ + aml_append(pkg, aml_string("%s", "")); + aml_append(ifctx, aml_store(pkg, ret)); + + /* + * IASL is fine when initializing Package with computational data, + * however it makes guest unhappy /it fails to process such AML/. + * So use runtime assignment to set acpi-index after initializer + * to make OSPM happy. + */ + aml_append(ifctx, + aml_store(aml_derefof(aml_index(params, aml_int(0))), aidx)); + aml_append(ifctx, aml_store(aidx, aml_index(ret, zero))); + aml_append(ifctx, aml_return(ret)); + } + aml_append(method, ifctx); + + return method; +} + +static Aml *aml_pci_static_endpoint_dsm(PCIDevice *pdev) +{ + Aml *method; + + g_assert(pdev->acpi_index != 0); + method = aml_method("_DSM", 4, AML_SERIALIZED); + { + Aml *params = aml_local(0); + Aml *pkg = aml_package(1); + aml_append(pkg, aml_int(pdev->acpi_index)); + aml_append(method, aml_store(pkg, params)); + aml_append(method, + aml_return(aml_call5("EDSM", aml_arg(0), aml_arg(1), + aml_arg(2), aml_arg(3), params)) + ); + } + return method; +} + static void build_append_pcihp_notify_entry(Aml *method, int slot) { Aml *if_ctx; @@ -396,12 +494,6 @@ static bool is_devfn_ignored_generic(const int devfn, const PCIBus *bus) if (DEVICE(pdev)->hotplugged) { return true; } - } else if (!get_dev_aml_func(DEVICE(pdev))) { - /* - * Ignore all other devices on !0 functions unless they - * have AML description (i.e have get_dev_aml_func() != 0) - */ - return true; } } return false; @@ -428,12 +520,14 @@ static bool is_devfn_ignored_hotplug(const int devfn, const PCIBus *bus) return false; } -static void build_append_pcihp_slots(Aml *parent_scope, PCIBus *bus, - QObject *bsel) +void build_append_pcihp_slots(Aml *parent_scope, PCIBus *bus) { int devfn; Aml *dev, *notify_method = NULL, *method; + QObject *bsel = object_property_get_qobject(OBJECT(bus), + ACPI_PCIHP_PROP_BSEL, NULL); uint64_t bsel_val = qnum_get_uint(qobject_to(QNum, bsel)); + qobject_unref(bsel); aml_append(parent_scope, aml_name_decl("BSEL", aml_int(bsel_val))); notify_method = aml_method("DVNT", 2, AML_NOTSERIALIZED); @@ -478,12 +572,9 @@ static void build_append_pcihp_slots(Aml *parent_scope, PCIBus *bus, void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus) { - QObject *bsel; int devfn; Aml *dev; - bsel = object_property_get_qobject(OBJECT(bus), ACPI_PCIHP_PROP_BSEL, NULL); - for (devfn = 0; devfn < ARRAY_SIZE(bus->devices); devfn++) { /* ACPI spec: 1.0b: Table 6-2 _ADR Object Bus Types, PCI type */ int adr = PCI_SLOT(devfn) << 16 | PCI_FUNC(devfn); @@ -498,16 +589,16 @@ void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus) aml_append(dev, aml_name_decl("_ADR", aml_int(adr))); call_dev_aml_func(DEVICE(bus->devices[devfn]), dev); + /* add _DSM if device has acpi-index set */ + if (pdev->acpi_index && + !object_property_get_bool(OBJECT(pdev), "hotpluggable", + &error_abort)) { + aml_append(dev, aml_pci_static_endpoint_dsm(pdev)); + } /* device descriptor has been composed, add it into parent context */ aml_append(parent_scope, dev); } - - if (bsel) { - build_append_pcihp_slots(parent_scope, bus, bsel); - } - - qobject_unref(bsel); } static bool build_append_notfication_callback(Aml *parent_scope, @@ -517,16 +608,24 @@ static bool build_append_notfication_callback(Aml *parent_scope, PCIBus *sec; QObject *bsel; int nr_notifiers = 0; + GQueue *pcnt_bus_list = g_queue_new(); QLIST_FOREACH(sec, &bus->child, sibling) { Aml *br_scope = aml_scope("S%.02X", sec->parent_dev->devfn); - if (pci_bus_is_root(sec) || - !object_property_find(OBJECT(sec), ACPI_PCIHP_PROP_BSEL)) { + if (pci_bus_is_root(sec)) { continue; } nr_notifiers = nr_notifiers + build_append_notfication_callback(br_scope, sec); - aml_append(parent_scope, br_scope); + /* + * add new child scope to parent + * and keep track of bus that have PCNT, + * bus list is used later to call children PCNTs from this level PCNT + */ + if (nr_notifiers) { + g_queue_push_tail(pcnt_bus_list, sec); + aml_append(parent_scope, br_scope); + } } /* @@ -550,30 +649,25 @@ static bool build_append_notfication_callback(Aml *parent_scope, } /* Notify about child bus events in any case */ - QLIST_FOREACH(sec, &bus->child, sibling) { - if (pci_bus_is_root(sec) || - !object_property_find(OBJECT(sec), ACPI_PCIHP_PROP_BSEL)) { - continue; - } - + while ((sec = g_queue_pop_head(pcnt_bus_list))) { aml_append(method, aml_name("^S%.02X.PCNT", sec->parent_dev->devfn)); } aml_append(parent_scope, method); qobject_unref(bsel); + g_queue_free(pcnt_bus_list); return !!nr_notifiers; } static Aml *aml_pci_pdsm(void) { - Aml *method, *UUID, *ifctx, *ifctx1; + Aml *method, *ifctx, *ifctx1; Aml *ret = aml_local(0); Aml *caps = aml_local(1); Aml *acpi_index = aml_local(2); Aml *zero = aml_int(0); Aml *one = aml_int(1); Aml *func = aml_arg(2); - Aml *rev = aml_arg(1); Aml *params = aml_arg(4); Aml *bnum = aml_derefof(aml_index(params, aml_int(0))); Aml *sunum = aml_derefof(aml_index(params, aml_int(1))); @@ -583,29 +677,9 @@ static Aml *aml_pci_pdsm(void) /* get supported functions */ ifctx = aml_if(aml_equal(func, zero)); { - uint8_t byte_list[1] = { 0 }; /* nothing supported yet */ - aml_append(ifctx, aml_store(aml_buffer(1, byte_list), ret)); - aml_append(ifctx, aml_store(zero, caps)); - - /* - * PCI Firmware Specification 3.1 - * 4.6. _DSM Definitions for PCI - */ - UUID = aml_touuid("E5C937D0-3553-4D7A-9117-EA4D19C3434D"); - ifctx1 = aml_if(aml_lnot(aml_equal(aml_arg(0), UUID))); - { - /* call is for unsupported UUID, bail out */ - aml_append(ifctx1, aml_return(ret)); - } - aml_append(ifctx, ifctx1); - - ifctx1 = aml_if(aml_lless(rev, aml_int(2))); - { - /* call is for unsupported REV, bail out */ - aml_append(ifctx1, aml_return(ret)); - } - aml_append(ifctx, ifctx1); + build_append_pci_dsm_func0_common(ifctx, ret); + aml_append(ifctx, aml_store(zero, caps)); aml_append(ifctx, aml_store(aml_call2("AIDX", bnum, sunum), acpi_index)); /* @@ -1388,6 +1462,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A03"))); aml_append(dev, aml_name_decl("_ADR", aml_int(0))); aml_append(dev, aml_name_decl("_UID", aml_int(pcmc->pci_root_uid))); + aml_append(dev, aml_pci_edsm()); aml_append(sb_scope, dev); aml_append(dsdt, sb_scope); @@ -1403,6 +1478,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, aml_append(dev, aml_name_decl("_ADR", aml_int(0))); aml_append(dev, aml_name_decl("_UID", aml_int(pcmc->pci_root_uid))); aml_append(dev, build_q35_osc_method(!pm->pcihp_bridge_en)); + aml_append(dev, aml_pci_edsm()); aml_append(sb_scope, dev); if (mcfg_valid) { aml_append(sb_scope, build_q35_dram_controller(&mcfg)); @@ -1710,6 +1786,9 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, Aml *scope = aml_scope("PCI0"); /* Scan all PCI buses. Generate tables to support hotplug. */ build_append_pci_bus_devices(scope, bus); + if (object_property_find(OBJECT(bus), ACPI_PCIHP_PROP_BSEL)) { + build_append_pcihp_slots(scope, bus); + } aml_append(sb_scope, scope); } } diff --git a/hw/isa/lpc_ich9.c b/hw/isa/lpc_ich9.c index d8303d0322..9714b0001e 100644 --- a/hw/isa/lpc_ich9.c +++ b/hw/isa/lpc_ich9.c @@ -865,6 +865,7 @@ static void ich9_lpc_class_init(ObjectClass *klass, void *data) hc->plug = ich9_pm_device_plug_cb; hc->unplug_request = ich9_pm_device_unplug_request_cb; hc->unplug = ich9_pm_device_unplug_cb; + hc->is_hotpluggable_bus = ich9_pm_is_hotpluggable_bus; adevc->ospm_status = ich9_pm_ospm_status; adevc->send_event = ich9_send_gpe; adevc->madt_cpu = pc_madt_cpu_entry; diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c index 217a5e639b..abe60b362c 100644 --- a/hw/mem/cxl_type3.c +++ b/hw/mem/cxl_type3.c @@ -1,6 +1,7 @@ #include "qemu/osdep.h" #include "qemu/units.h" #include "qemu/error-report.h" +#include "qapi/qapi-commands-cxl.h" #include "hw/mem/memory-device.h" #include "hw/mem/pc-dimm.h" #include "hw/pci/pci.h" @@ -250,6 +251,7 @@ static void ct3d_config_write(PCIDevice *pci_dev, uint32_t addr, uint32_t val, pcie_doe_write_config(&ct3d->doe_cdat, addr, val, size); pci_default_write_config(pci_dev, addr, val, size); + pcie_aer_write_config(pci_dev, addr, val, size); } /* @@ -322,6 +324,66 @@ static void hdm_decoder_commit(CXLType3Dev *ct3d, int which) ARRAY_FIELD_DP32(cache_mem, CXL_HDM_DECODER0_CTRL, COMMITTED, 1); } +static int ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err) +{ + switch (qmp_err) { + case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_PARITY: + return CXL_RAS_UNC_ERR_CACHE_DATA_PARITY; + case CXL_UNCOR_ERROR_TYPE_CACHE_ADDRESS_PARITY: + return CXL_RAS_UNC_ERR_CACHE_ADDRESS_PARITY; + case CXL_UNCOR_ERROR_TYPE_CACHE_BE_PARITY: + return CXL_RAS_UNC_ERR_CACHE_BE_PARITY; + case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_ECC: + return CXL_RAS_UNC_ERR_CACHE_DATA_ECC; + case CXL_UNCOR_ERROR_TYPE_MEM_DATA_PARITY: + return CXL_RAS_UNC_ERR_MEM_DATA_PARITY; + case CXL_UNCOR_ERROR_TYPE_MEM_ADDRESS_PARITY: + return CXL_RAS_UNC_ERR_MEM_ADDRESS_PARITY; + case CXL_UNCOR_ERROR_TYPE_MEM_BE_PARITY: + return CXL_RAS_UNC_ERR_MEM_BE_PARITY; + case CXL_UNCOR_ERROR_TYPE_MEM_DATA_ECC: + return CXL_RAS_UNC_ERR_MEM_DATA_ECC; + case CXL_UNCOR_ERROR_TYPE_REINIT_THRESHOLD: + return CXL_RAS_UNC_ERR_REINIT_THRESHOLD; + case CXL_UNCOR_ERROR_TYPE_RSVD_ENCODING: + return CXL_RAS_UNC_ERR_RSVD_ENCODING; + case CXL_UNCOR_ERROR_TYPE_POISON_RECEIVED: + return CXL_RAS_UNC_ERR_POISON_RECEIVED; + case CXL_UNCOR_ERROR_TYPE_RECEIVER_OVERFLOW: + return CXL_RAS_UNC_ERR_RECEIVER_OVERFLOW; + case CXL_UNCOR_ERROR_TYPE_INTERNAL: + return CXL_RAS_UNC_ERR_INTERNAL; + case CXL_UNCOR_ERROR_TYPE_CXL_IDE_TX: + return CXL_RAS_UNC_ERR_CXL_IDE_TX; + case CXL_UNCOR_ERROR_TYPE_CXL_IDE_RX: + return CXL_RAS_UNC_ERR_CXL_IDE_RX; + default: + return -EINVAL; + } +} + +static int ct3d_qmp_cor_err_to_cxl(CxlCorErrorType qmp_err) +{ + switch (qmp_err) { + case CXL_COR_ERROR_TYPE_CACHE_DATA_ECC: + return CXL_RAS_COR_ERR_CACHE_DATA_ECC; + case CXL_COR_ERROR_TYPE_MEM_DATA_ECC: + return CXL_RAS_COR_ERR_MEM_DATA_ECC; + case CXL_COR_ERROR_TYPE_CRC_THRESHOLD: + return CXL_RAS_COR_ERR_CRC_THRESHOLD; + case CXL_COR_ERROR_TYPE_RETRY_THRESHOLD: + return CXL_RAS_COR_ERR_RETRY_THRESHOLD; + case CXL_COR_ERROR_TYPE_CACHE_POISON_RECEIVED: + return CXL_RAS_COR_ERR_CACHE_POISON_RECEIVED; + case CXL_COR_ERROR_TYPE_MEM_POISON_RECEIVED: + return CXL_RAS_COR_ERR_MEM_POISON_RECEIVED; + case CXL_COR_ERROR_TYPE_PHYSICAL: + return CXL_RAS_COR_ERR_PHYSICAL; + default: + return -EINVAL; + } +} + static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value, unsigned size) { @@ -340,6 +402,83 @@ static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value, should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT); which_hdm = 0; break; + case A_CXL_RAS_UNC_ERR_STATUS: + { + uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL); + uint32_t fe = FIELD_EX32(capctrl, CXL_RAS_ERR_CAP_CTRL, FIRST_ERROR_POINTER); + CXLError *cxl_err; + uint32_t unc_err; + + /* + * If single bit written that corresponds to the first error + * pointer being cleared, update the status and header log. + */ + if (!QTAILQ_EMPTY(&ct3d->error_list)) { + if ((1 << fe) ^ value) { + CXLError *cxl_next; + /* + * Software is using wrong flow for multiple header recording + * Following behavior in PCIe r6.0 and assuming multiple + * header support. Implementation defined choice to clear all + * matching records if more than one bit set - which corresponds + * closest to behavior of hardware not capable of multiple + * header recording. + */ + QTAILQ_FOREACH_SAFE(cxl_err, &ct3d->error_list, node, cxl_next) { + if ((1 << cxl_err->type) & value) { + QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node); + g_free(cxl_err); + } + } + } else { + /* Done with previous FE, so drop from list */ + cxl_err = QTAILQ_FIRST(&ct3d->error_list); + QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node); + g_free(cxl_err); + } + + /* + * If there is another FE, then put that in place and update + * the header log + */ + if (!QTAILQ_EMPTY(&ct3d->error_list)) { + uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0]; + int i; + + cxl_err = QTAILQ_FIRST(&ct3d->error_list); + for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) { + stl_le_p(header_log + i, cxl_err->header[i]); + } + capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL, + FIRST_ERROR_POINTER, cxl_err->type); + } else { + /* + * If no more errors, then follow recomendation of PCI spec + * r6.0 6.2.4.2 to set the first error pointer to a status + * bit that will never be used. + */ + capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL, + FIRST_ERROR_POINTER, + CXL_RAS_UNC_ERR_CXL_UNUSED); + } + stl_le_p((uint8_t *)cache_mem + A_CXL_RAS_ERR_CAP_CTRL, capctrl); + } + unc_err = 0; + QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) { + unc_err |= 1 << cxl_err->type; + } + stl_le_p((uint8_t *)cache_mem + offset, unc_err); + + return; + } + case A_CXL_RAS_COR_ERR_STATUS: + { + uint32_t rw1c = value; + uint32_t temp = ldl_le_p((uint8_t *)cache_mem + offset); + temp &= ~rw1c; + stl_le_p((uint8_t *)cache_mem + offset, temp); + return; + } default: break; } @@ -403,6 +542,8 @@ static void ct3_realize(PCIDevice *pci_dev, Error **errp) unsigned short msix_num = 1; int i, rc; + QTAILQ_INIT(&ct3d->error_list); + if (!cxl_setup_memory(ct3d, errp)) { return; } @@ -452,8 +593,19 @@ static void ct3_realize(PCIDevice *pci_dev, Error **errp) cxl_cstate->cdat.free_cdat_table = ct3_free_cdat_table; cxl_cstate->cdat.private = ct3d; cxl_doe_cdat_init(cxl_cstate, errp); + + pcie_cap_deverr_init(pci_dev); + /* Leave a bit of room for expansion */ + rc = pcie_aer_init(pci_dev, PCI_ERR_VER, 0x200, PCI_ERR_SIZEOF, NULL); + if (rc) { + goto err_release_cdat; + } + return; +err_release_cdat: + cxl_doe_cdat_release(cxl_cstate); + g_free(regs->special_ops); err_address_space_free: address_space_destroy(&ct3d->hostmem_as); return; @@ -465,6 +617,7 @@ static void ct3_exit(PCIDevice *pci_dev) CXLComponentState *cxl_cstate = &ct3d->cxl_cstate; ComponentRegisters *regs = &cxl_cstate->crb; + pcie_aer_exit(pci_dev); cxl_doe_cdat_release(cxl_cstate); g_free(regs->special_ops); address_space_destroy(&ct3d->hostmem_as); @@ -618,6 +771,147 @@ static void set_lsa(CXLType3Dev *ct3d, const void *buf, uint64_t size, */ } +/* For uncorrectable errors include support for multiple header recording */ +void qmp_cxl_inject_uncorrectable_errors(const char *path, + CXLUncorErrorRecordList *errors, + Error **errp) +{ + Object *obj = object_resolve_path(path, NULL); + static PCIEAERErr err = {}; + CXLType3Dev *ct3d; + CXLError *cxl_err; + uint32_t *reg_state; + uint32_t unc_err; + bool first; + + if (!obj) { + error_setg(errp, "Unable to resolve path"); + return; + } + + if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) { + error_setg(errp, "Path does not point to a CXL type 3 device"); + return; + } + + err.status = PCI_ERR_UNC_INTN; + err.source_id = pci_requester_id(PCI_DEVICE(obj)); + err.flags = 0; + + ct3d = CXL_TYPE3(obj); + + first = QTAILQ_EMPTY(&ct3d->error_list); + reg_state = ct3d->cxl_cstate.crb.cache_mem_registers; + while (errors) { + uint32List *header = errors->value->header; + uint8_t header_count = 0; + int cxl_err_code; + + cxl_err_code = ct3d_qmp_uncor_err_to_cxl(errors->value->type); + if (cxl_err_code < 0) { + error_setg(errp, "Unknown error code"); + return; + } + + /* If the error is masked, nothing to do here */ + if (!((1 << cxl_err_code) & + ~ldl_le_p(reg_state + R_CXL_RAS_UNC_ERR_MASK))) { + errors = errors->next; + continue; + } + + cxl_err = g_malloc0(sizeof(*cxl_err)); + if (!cxl_err) { + return; + } + + cxl_err->type = cxl_err_code; + while (header && header_count < 32) { + cxl_err->header[header_count++] = header->value; + header = header->next; + } + if (header_count > 32) { + error_setg(errp, "Header must be 32 DWORD or less"); + return; + } + QTAILQ_INSERT_TAIL(&ct3d->error_list, cxl_err, node); + + errors = errors->next; + } + + if (first && !QTAILQ_EMPTY(&ct3d->error_list)) { + uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers; + uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL); + uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0]; + int i; + + cxl_err = QTAILQ_FIRST(&ct3d->error_list); + for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) { + stl_le_p(header_log + i, cxl_err->header[i]); + } + + capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL, + FIRST_ERROR_POINTER, cxl_err->type); + stl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL, capctrl); + } + + unc_err = 0; + QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) { + unc_err |= (1 << cxl_err->type); + } + if (!unc_err) { + return; + } + + stl_le_p(reg_state + R_CXL_RAS_UNC_ERR_STATUS, unc_err); + pcie_aer_inject_error(PCI_DEVICE(obj), &err); + + return; +} + +void qmp_cxl_inject_correctable_error(const char *path, CxlCorErrorType type, + Error **errp) +{ + static PCIEAERErr err = {}; + Object *obj = object_resolve_path(path, NULL); + CXLType3Dev *ct3d; + uint32_t *reg_state; + uint32_t cor_err; + int cxl_err_type; + + if (!obj) { + error_setg(errp, "Unable to resolve path"); + return; + } + if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) { + error_setg(errp, "Path does not point to a CXL type 3 device"); + return; + } + + err.status = PCI_ERR_COR_INTERNAL; + err.source_id = pci_requester_id(PCI_DEVICE(obj)); + err.flags = PCIE_AER_ERR_IS_CORRECTABLE; + + ct3d = CXL_TYPE3(obj); + reg_state = ct3d->cxl_cstate.crb.cache_mem_registers; + cor_err = ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS); + + cxl_err_type = ct3d_qmp_cor_err_to_cxl(type); + if (cxl_err_type < 0) { + error_setg(errp, "Invalid COR error"); + return; + } + /* If the error is masked, nothting to do here */ + if (!((1 << cxl_err_type) & ~ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_MASK))) { + return; + } + + cor_err |= (1 << cxl_err_type); + stl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS, cor_err); + + pcie_aer_inject_error(PCI_DEVICE(obj), &err); +} + static void ct3_class_init(ObjectClass *oc, void *data) { DeviceClass *dc = DEVICE_CLASS(oc); diff --git a/hw/mem/cxl_type3_stubs.c b/hw/mem/cxl_type3_stubs.c new file mode 100644 index 0000000000..d574c58f9a --- /dev/null +++ b/hw/mem/cxl_type3_stubs.c @@ -0,0 +1,17 @@ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "qapi/qapi-commands-cxl.h" + +void qmp_cxl_inject_uncorrectable_errors(const char *path, + CXLUncorErrorRecordList *errors, + Error **errp) +{ + error_setg(errp, "CXL Type 3 support is not compiled in"); +} + +void qmp_cxl_inject_correctable_error(const char *path, CxlCorErrorType type, + Error **errp) +{ + error_setg(errp, "CXL Type 3 support is not compiled in"); +} diff --git a/hw/mem/meson.build b/hw/mem/meson.build index 609b2b36fc..56c2618b84 100644 --- a/hw/mem/meson.build +++ b/hw/mem/meson.build @@ -4,6 +4,8 @@ mem_ss.add(when: 'CONFIG_DIMM', if_true: files('pc-dimm.c')) mem_ss.add(when: 'CONFIG_NPCM7XX', if_true: files('npcm7xx_mc.c')) mem_ss.add(when: 'CONFIG_NVDIMM', if_true: files('nvdimm.c')) mem_ss.add(when: 'CONFIG_CXL_MEM_DEVICE', if_true: files('cxl_type3.c')) +softmmu_ss.add(when: 'CONFIG_CXL_MEM_DEVICE', if_false: files('cxl_type3_stubs.c')) +softmmu_ss.add(when: 'CONFIG_ALL', if_true: files('cxl_type3_stubs.c')) softmmu_ss.add_all(when: 'CONFIG_MEM_DEVICE', if_true: mem_ss) diff --git a/hw/net/Kconfig b/hw/net/Kconfig index 1cc1c5775e..18c7851efe 100644 --- a/hw/net/Kconfig +++ b/hw/net/Kconfig @@ -44,6 +44,11 @@ config E1000E_PCI_EXPRESS default y if PCI_DEVICES depends on PCI_EXPRESS && MSI_NONBROKEN +config IGB_PCI_EXPRESS + bool + default y if PCI_DEVICES + depends on PCI_EXPRESS && MSI_NONBROKEN + config RTL8139_PCI bool default y if PCI_DEVICES diff --git a/hw/net/e1000.c b/hw/net/e1000.c index 7efb8a4c52..23d660619f 100644 --- a/hw/net/e1000.c +++ b/hw/net/e1000.c @@ -26,6 +26,7 @@ #include "qemu/osdep.h" +#include "hw/net/mii.h" #include "hw/pci/pci_device.h" #include "hw/qdev-properties.h" #include "migration/vmstate.h" @@ -38,12 +39,11 @@ #include "qemu/module.h" #include "qemu/range.h" +#include "e1000_common.h" #include "e1000x_common.h" #include "trace.h" #include "qom/object.h" -static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; - /* #define E1000_DEBUG */ #ifdef E1000_DEBUG @@ -66,9 +66,8 @@ static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL); #define IOPORT_SIZE 0x40 #define PNPMMIO_SIZE 0x20000 -#define MIN_BUF_SIZE 60 /* Min. octets in an ethernet frame sans FCS */ -#define MAXIMUM_ETHERNET_HDR_LEN (14+4) +#define MAXIMUM_ETHERNET_HDR_LEN (ETH_HLEN + 4) /* * HW models: @@ -181,67 +180,73 @@ e1000_autoneg_done(E1000State *s) static bool have_autoneg(E1000State *s) { - return chkflag(AUTONEG) && (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN); + return chkflag(AUTONEG) && (s->phy_reg[MII_BMCR] & MII_BMCR_AUTOEN); } static void set_phy_ctrl(E1000State *s, int index, uint16_t val) { - /* bits 0-5 reserved; MII_CR_[RESTART_AUTO_NEG,RESET] are self clearing */ - s->phy_reg[PHY_CTRL] = val & ~(0x3f | - MII_CR_RESET | - MII_CR_RESTART_AUTO_NEG); + /* bits 0-5 reserved; MII_BMCR_[ANRESTART,RESET] are self clearing */ + s->phy_reg[MII_BMCR] = val & ~(0x3f | + MII_BMCR_RESET | + MII_BMCR_ANRESTART); /* * QEMU 1.3 does not support link auto-negotiation emulation, so if we * migrate during auto negotiation, after migration the link will be * down. */ - if (have_autoneg(s) && (val & MII_CR_RESTART_AUTO_NEG)) { + if (have_autoneg(s) && (val & MII_BMCR_ANRESTART)) { e1000x_restart_autoneg(s->mac_reg, s->phy_reg, s->autoneg_timer); } } static void (*phyreg_writeops[])(E1000State *, int, uint16_t) = { - [PHY_CTRL] = set_phy_ctrl, + [MII_BMCR] = set_phy_ctrl, }; enum { NPHYWRITEOPS = ARRAY_SIZE(phyreg_writeops) }; enum { PHY_R = 1, PHY_W = 2, PHY_RW = PHY_R | PHY_W }; static const char phy_regcap[0x20] = { - [PHY_STATUS] = PHY_R, [M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW, - [PHY_ID1] = PHY_R, [M88E1000_PHY_SPEC_CTRL] = PHY_RW, - [PHY_CTRL] = PHY_RW, [PHY_1000T_CTRL] = PHY_RW, - [PHY_LP_ABILITY] = PHY_R, [PHY_1000T_STATUS] = PHY_R, - [PHY_AUTONEG_ADV] = PHY_RW, [M88E1000_RX_ERR_CNTR] = PHY_R, - [PHY_ID2] = PHY_R, [M88E1000_PHY_SPEC_STATUS] = PHY_R, - [PHY_AUTONEG_EXP] = PHY_R, + [MII_BMSR] = PHY_R, [M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW, + [MII_PHYID1] = PHY_R, [M88E1000_PHY_SPEC_CTRL] = PHY_RW, + [MII_BMCR] = PHY_RW, [MII_CTRL1000] = PHY_RW, + [MII_ANLPAR] = PHY_R, [MII_STAT1000] = PHY_R, + [MII_ANAR] = PHY_RW, [M88E1000_RX_ERR_CNTR] = PHY_R, + [MII_PHYID2] = PHY_R, [M88E1000_PHY_SPEC_STATUS] = PHY_R, + [MII_ANER] = PHY_R, }; -/* PHY_ID2 documented in 8254x_GBe_SDM.pdf, pp. 250 */ +/* MII_PHYID2 documented in 8254x_GBe_SDM.pdf, pp. 250 */ static const uint16_t phy_reg_init[] = { - [PHY_CTRL] = MII_CR_SPEED_SELECT_MSB | - MII_CR_FULL_DUPLEX | - MII_CR_AUTO_NEG_EN, - - [PHY_STATUS] = MII_SR_EXTENDED_CAPS | - MII_SR_LINK_STATUS | /* link initially up */ - MII_SR_AUTONEG_CAPS | - /* MII_SR_AUTONEG_COMPLETE: initially NOT completed */ - MII_SR_PREAMBLE_SUPPRESS | - MII_SR_EXTENDED_STATUS | - MII_SR_10T_HD_CAPS | - MII_SR_10T_FD_CAPS | - MII_SR_100X_HD_CAPS | - MII_SR_100X_FD_CAPS, - - [PHY_ID1] = 0x141, - /* [PHY_ID2] configured per DevId, from e1000_reset() */ - [PHY_AUTONEG_ADV] = 0xde1, - [PHY_LP_ABILITY] = 0x1e0, - [PHY_1000T_CTRL] = 0x0e00, - [PHY_1000T_STATUS] = 0x3c00, + [MII_BMCR] = MII_BMCR_SPEED1000 | + MII_BMCR_FD | + MII_BMCR_AUTOEN, + + [MII_BMSR] = MII_BMSR_EXTCAP | + MII_BMSR_LINK_ST | /* link initially up */ + MII_BMSR_AUTONEG | + /* MII_BMSR_AN_COMP: initially NOT completed */ + MII_BMSR_MFPS | + MII_BMSR_EXTSTAT | + MII_BMSR_10T_HD | + MII_BMSR_10T_FD | + MII_BMSR_100TX_HD | + MII_BMSR_100TX_FD, + + [MII_PHYID1] = 0x141, + /* [MII_PHYID2] configured per DevId, from e1000_reset() */ + [MII_ANAR] = MII_ANAR_CSMACD | MII_ANAR_10 | + MII_ANAR_10FD | MII_ANAR_TX | + MII_ANAR_TXFD | MII_ANAR_PAUSE | + MII_ANAR_PAUSE_ASYM, + [MII_ANLPAR] = MII_ANLPAR_10 | MII_ANLPAR_10FD | + MII_ANLPAR_TX | MII_ANLPAR_TXFD, + [MII_CTRL1000] = MII_CTRL1000_FULL | MII_CTRL1000_PORT | + MII_CTRL1000_MASTER, + [MII_STAT1000] = MII_STAT1000_HALF | MII_STAT1000_FULL | + MII_STAT1000_ROK | MII_STAT1000_LOK, [M88E1000_PHY_SPEC_CTRL] = 0x360, [M88E1000_PHY_SPEC_STATUS] = 0xac00, [M88E1000_EXT_PHY_SPEC_CTRL] = 0x0d60, @@ -373,9 +378,9 @@ static bool e1000_vet_init_need(void *opaque) return chkflag(VET); } -static void e1000_reset(void *opaque) +static void e1000_reset_hold(Object *obj) { - E1000State *d = opaque; + E1000State *d = E1000(obj); E1000BaseClass *edc = E1000_GET_CLASS(d); uint8_t *macaddr = d->conf.macaddr.a; @@ -386,10 +391,10 @@ static void e1000_reset(void *opaque) d->mit_irq_level = 0; d->mit_ide = 0; memset(d->phy_reg, 0, sizeof d->phy_reg); - memmove(d->phy_reg, phy_reg_init, sizeof phy_reg_init); - d->phy_reg[PHY_ID2] = edc->phy_id2; + memcpy(d->phy_reg, phy_reg_init, sizeof phy_reg_init); + d->phy_reg[MII_PHYID2] = edc->phy_id2; memset(d->mac_reg, 0, sizeof d->mac_reg); - memmove(d->mac_reg, mac_reg_init, sizeof mac_reg_init); + memcpy(d->mac_reg, mac_reg_init, sizeof mac_reg_init); d->rxbuf_min_shift = 1; memset(&d->tx, 0, sizeof d->tx); @@ -547,9 +552,9 @@ putsum(uint8_t *data, uint32_t n, uint32_t sloc, uint32_t css, uint32_t cse) static inline void inc_tx_bcast_or_mcast_count(E1000State *s, const unsigned char *arr) { - if (!memcmp(arr, bcast, sizeof bcast)) { + if (is_broadcast_ether_addr(arr)) { e1000x_inc_reg_if_not_full(s->mac_reg, BPTC); - } else if (arr[0] & 1) { + } else if (is_multicast_ether_addr(arr)) { e1000x_inc_reg_if_not_full(s->mac_reg, MPTC); } } @@ -561,13 +566,13 @@ e1000_send_packet(E1000State *s, const uint8_t *buf, int size) PTC1023, PTC1522 }; NetClientState *nc = qemu_get_queue(s->nic); - if (s->phy_reg[PHY_CTRL] & MII_CR_LOOPBACK) { + if (s->phy_reg[MII_BMCR] & MII_BMCR_LOOPBACK) { qemu_receive_packet(nc, buf, size); } else { qemu_send_packet(nc, buf, size); } inc_tx_bcast_or_mcast_count(s, buf); - e1000x_increase_size_stats(s->mac_reg, PTCregs, size); + e1000x_increase_size_stats(s->mac_reg, PTCregs, size + 4); } static void @@ -631,7 +636,7 @@ xmit_seg(E1000State *s) } e1000x_inc_reg_if_not_full(s->mac_reg, TPT); - e1000x_grow_8reg_if_not_full(s->mac_reg, TOTL, s->tx.size); + e1000x_grow_8reg_if_not_full(s->mac_reg, TOTL, s->tx.size + 4); s->mac_reg[GPTC] = s->mac_reg[TPT]; s->mac_reg[GOTCL] = s->mac_reg[TOTL]; s->mac_reg[GOTCH] = s->mac_reg[TOTH]; @@ -803,15 +808,18 @@ static int receive_filter(E1000State *s, const uint8_t *buf, int size) { uint32_t rctl = s->mac_reg[RCTL]; - int isbcast = !memcmp(buf, bcast, sizeof bcast), ismcast = (buf[0] & 1); + int isbcast = is_broadcast_ether_addr(buf); + int ismcast = is_multicast_ether_addr(buf); if (e1000x_is_vlan_packet(buf, le16_to_cpu(s->mac_reg[VET])) && e1000x_vlan_rx_filter_enabled(s->mac_reg)) { - uint16_t vid = lduw_be_p(buf + 14); - uint32_t vfta = ldl_le_p((uint32_t*)(s->mac_reg + VFTA) + - ((vid >> 5) & 0x7f)); - if ((vfta & (1 << (vid & 0x1f))) == 0) + uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(buf)->h_tci); + uint32_t vfta = + ldl_le_p((uint32_t *)(s->mac_reg + VFTA) + + ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK)); + if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) { return 0; + } } if (!isbcast && !ismcast && (rctl & E1000_RCTL_UPE)) { /* promiscuous ucast */ @@ -841,7 +849,7 @@ e1000_set_link_status(NetClientState *nc) e1000x_update_regs_on_link_down(s->mac_reg, s->phy_reg); } else { if (have_autoneg(s) && - !(s->phy_reg[PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) { + !(s->phy_reg[MII_BMSR] & MII_BMSR_AN_COMP)) { e1000x_restart_autoneg(s->mac_reg, s->phy_reg, s->autoneg_timer); } else { e1000_link_up(s); @@ -907,7 +915,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt) uint32_t rdh_start; uint16_t vlan_special = 0; uint8_t vlan_status = 0; - uint8_t min_buf[MIN_BUF_SIZE]; + uint8_t min_buf[ETH_ZLEN]; struct iovec min_iov; uint8_t *filter_buf = iov->iov_base; size_t size = iov_size(iov, iovcnt); @@ -1061,30 +1069,6 @@ mac_readreg(E1000State *s, int index) } static uint32_t -mac_low4_read(E1000State *s, int index) -{ - return s->mac_reg[index] & 0xf; -} - -static uint32_t -mac_low11_read(E1000State *s, int index) -{ - return s->mac_reg[index] & 0x7ff; -} - -static uint32_t -mac_low13_read(E1000State *s, int index) -{ - return s->mac_reg[index] & 0x1fff; -} - -static uint32_t -mac_low16_read(E1000State *s, int index) -{ - return s->mac_reg[index] & 0xffff; -} - -static uint32_t mac_icr_read(E1000State *s, int index) { uint32_t ret = s->mac_reg[ICR]; @@ -1136,11 +1120,17 @@ set_rdt(E1000State *s, int index, uint32_t val) } } -static void -set_16bit(E1000State *s, int index, uint32_t val) -{ - s->mac_reg[index] = val & 0xffff; -} +#define LOW_BITS_SET_FUNC(num) \ + static void \ + set_##num##bit(E1000State *s, int index, uint32_t val) \ + { \ + s->mac_reg[index] = val & (BIT(num) - 1); \ + } + +LOW_BITS_SET_FUNC(4) +LOW_BITS_SET_FUNC(11) +LOW_BITS_SET_FUNC(13) +LOW_BITS_SET_FUNC(16) static void set_dlen(E1000State *s, int index, uint32_t val) @@ -1194,7 +1184,9 @@ static const readops macreg_readops[] = { getreg(XONRXC), getreg(XONTXC), getreg(XOFFRXC), getreg(XOFFTXC), getreg(RFC), getreg(RJC), getreg(RNBC), getreg(TSCTFC), getreg(MGTPRC), getreg(MGTPDC), getreg(MGTPTC), getreg(GORCL), - getreg(GOTCL), + getreg(GOTCL), getreg(RDFH), getreg(RDFT), getreg(RDFHS), + getreg(RDFTS), getreg(RDFPC), getreg(TDFH), getreg(TDFT), + getreg(TDFHS), getreg(TDFTS), getreg(TDFPC), getreg(AIT), [TOTH] = mac_read_clr8, [TORH] = mac_read_clr8, [GOTCH] = mac_read_clr8, [GORCH] = mac_read_clr8, @@ -1212,24 +1204,17 @@ static const readops macreg_readops[] = { [MPTC] = mac_read_clr4, [ICR] = mac_icr_read, [EECD] = get_eecd, [EERD] = flash_eerd_read, - [RDFH] = mac_low13_read, [RDFT] = mac_low13_read, - [RDFHS] = mac_low13_read, [RDFTS] = mac_low13_read, - [RDFPC] = mac_low13_read, - [TDFH] = mac_low11_read, [TDFT] = mac_low11_read, - [TDFHS] = mac_low13_read, [TDFTS] = mac_low13_read, - [TDFPC] = mac_low13_read, - [AIT] = mac_low16_read, - - [CRCERRS ... MPC] = &mac_readreg, - [IP6AT ... IP6AT+3] = &mac_readreg, [IP4AT ... IP4AT+6] = &mac_readreg, - [FFLT ... FFLT+6] = &mac_low11_read, - [RA ... RA+31] = &mac_readreg, - [WUPM ... WUPM+31] = &mac_readreg, - [MTA ... MTA+127] = &mac_readreg, - [VFTA ... VFTA+127] = &mac_readreg, - [FFMT ... FFMT+254] = &mac_low4_read, - [FFVT ... FFVT+254] = &mac_readreg, - [PBM ... PBM+16383] = &mac_readreg, + + [CRCERRS ... MPC] = &mac_readreg, + [IP6AT ... IP6AT + 3] = &mac_readreg, [IP4AT ... IP4AT + 6] = &mac_readreg, + [FFLT ... FFLT + 6] = &mac_readreg, + [RA ... RA + 31] = &mac_readreg, + [WUPM ... WUPM + 31] = &mac_readreg, + [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = &mac_readreg, + [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = &mac_readreg, + [FFMT ... FFMT + 254] = &mac_readreg, + [FFVT ... FFVT + 254] = &mac_readreg, + [PBM ... PBM + 16383] = &mac_readreg, }; enum { NREADOPS = ARRAY_SIZE(macreg_readops) }; @@ -1239,27 +1224,28 @@ static const writeops macreg_writeops[] = { putreg(PBA), putreg(EERD), putreg(SWSM), putreg(WUFC), putreg(TDBAL), putreg(TDBAH), putreg(TXDCTL), putreg(RDBAH), putreg(RDBAL), putreg(LEDCTL), putreg(VET), putreg(FCRUC), - putreg(TDFH), putreg(TDFT), putreg(TDFHS), putreg(TDFTS), - putreg(TDFPC), putreg(RDFH), putreg(RDFT), putreg(RDFHS), - putreg(RDFTS), putreg(RDFPC), putreg(IPAV), putreg(WUC), - putreg(WUS), putreg(AIT), - - [TDLEN] = set_dlen, [RDLEN] = set_dlen, [TCTL] = set_tctl, - [TDT] = set_tctl, [MDIC] = set_mdic, [ICS] = set_ics, - [TDH] = set_16bit, [RDH] = set_16bit, [RDT] = set_rdt, - [IMC] = set_imc, [IMS] = set_ims, [ICR] = set_icr, - [EECD] = set_eecd, [RCTL] = set_rx_control, [CTRL] = set_ctrl, - [RDTR] = set_16bit, [RADV] = set_16bit, [TADV] = set_16bit, - [ITR] = set_16bit, - - [IP6AT ... IP6AT+3] = &mac_writereg, [IP4AT ... IP4AT+6] = &mac_writereg, - [FFLT ... FFLT+6] = &mac_writereg, - [RA ... RA+31] = &mac_writereg, - [WUPM ... WUPM+31] = &mac_writereg, - [MTA ... MTA+127] = &mac_writereg, - [VFTA ... VFTA+127] = &mac_writereg, - [FFMT ... FFMT+254] = &mac_writereg, [FFVT ... FFVT+254] = &mac_writereg, - [PBM ... PBM+16383] = &mac_writereg, + putreg(IPAV), putreg(WUC), + putreg(WUS), + + [TDLEN] = set_dlen, [RDLEN] = set_dlen, [TCTL] = set_tctl, + [TDT] = set_tctl, [MDIC] = set_mdic, [ICS] = set_ics, + [TDH] = set_16bit, [RDH] = set_16bit, [RDT] = set_rdt, + [IMC] = set_imc, [IMS] = set_ims, [ICR] = set_icr, + [EECD] = set_eecd, [RCTL] = set_rx_control, [CTRL] = set_ctrl, + [RDTR] = set_16bit, [RADV] = set_16bit, [TADV] = set_16bit, + [ITR] = set_16bit, [TDFH] = set_11bit, [TDFT] = set_11bit, + [TDFHS] = set_13bit, [TDFTS] = set_13bit, [TDFPC] = set_13bit, + [RDFH] = set_13bit, [RDFT] = set_13bit, [RDFHS] = set_13bit, + [RDFTS] = set_13bit, [RDFPC] = set_13bit, [AIT] = set_16bit, + + [IP6AT ... IP6AT + 3] = &mac_writereg, [IP4AT ... IP4AT + 6] = &mac_writereg, + [FFLT ... FFLT + 6] = &set_11bit, + [RA ... RA + 31] = &mac_writereg, + [WUPM ... WUPM + 31] = &mac_writereg, + [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = &mac_writereg, + [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = &mac_writereg, + [FFMT ... FFMT + 254] = &set_4bit, [FFVT ... FFVT + 254] = &mac_writereg, + [PBM ... PBM + 16383] = &mac_writereg, }; enum { NWRITEOPS = ARRAY_SIZE(macreg_writeops) }; @@ -1415,10 +1401,10 @@ static int e1000_pre_save(void *opaque) /* * If link is down and auto-negotiation is supported and ongoing, * complete auto-negotiation immediately. This allows us to look - * at MII_SR_AUTONEG_COMPLETE to infer link status on load. + * at MII_BMSR_AN_COMP to infer link status on load. */ if (nc->link_down && have_autoneg(s)) { - s->phy_reg[PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE; + s->phy_reg[MII_BMSR] |= MII_BMSR_AN_COMP; } /* Decide which set of props to migrate in the main structure */ @@ -1457,8 +1443,7 @@ static int e1000_post_load(void *opaque, int version_id) * Alternatively, restart link negotiation if it was in progress. */ nc->link_down = (s->mac_reg[STATUS] & E1000_STATUS_LU) == 0; - if (have_autoneg(s) && - !(s->phy_reg[PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) { + if (have_autoneg(s) && !(s->phy_reg[MII_BMSR] & MII_BMSR_AN_COMP)) { nc->link_down = false; timer_mod(s->autoneg_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500); @@ -1624,8 +1609,9 @@ static const VMStateDescription vmstate_e1000 = { VMSTATE_UINT32(mac_reg[WUFC], E1000State), VMSTATE_UINT32(mac_reg[VET], E1000State), VMSTATE_UINT32_SUB_ARRAY(mac_reg, E1000State, RA, 32), - VMSTATE_UINT32_SUB_ARRAY(mac_reg, E1000State, MTA, 128), - VMSTATE_UINT32_SUB_ARRAY(mac_reg, E1000State, VFTA, 128), + VMSTATE_UINT32_SUB_ARRAY(mac_reg, E1000State, MTA, E1000_MC_TBL_SIZE), + VMSTATE_UINT32_SUB_ARRAY(mac_reg, E1000State, VFTA, + E1000_VLAN_FILTER_TBL_SIZE), VMSTATE_END_OF_LIST() }, .subsections = (const VMStateDescription*[]) { @@ -1746,12 +1732,6 @@ static void pci_e1000_realize(PCIDevice *pci_dev, Error **errp) e1000_flush_queue_timer, d); } -static void qdev_e1000_reset(DeviceState *dev) -{ - E1000State *d = E1000(dev); - e1000_reset(d); -} - static Property e1000_properties[] = { DEFINE_NIC_PROPERTIES(E1000State, conf), DEFINE_PROP_BIT("autonegotiation", E1000State, @@ -1777,6 +1757,7 @@ typedef struct E1000Info { static void e1000_class_init(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); + ResettableClass *rc = RESETTABLE_CLASS(klass); PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); E1000BaseClass *e = E1000_CLASS(klass); const E1000Info *info = data; @@ -1789,9 +1770,9 @@ static void e1000_class_init(ObjectClass *klass, void *data) k->revision = info->revision; e->phy_id2 = info->phy_id2; k->class_id = PCI_CLASS_NETWORK_ETHERNET; + rc->phases.hold = e1000_reset_hold; set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); dc->desc = "Intel Gigabit Ethernet"; - dc->reset = qdev_e1000_reset; dc->vmsd = &vmstate_e1000; device_class_set_props(dc, e1000_properties); } diff --git a/hw/net/e1000_common.h b/hw/net/e1000_common.h new file mode 100644 index 0000000000..48feda7404 --- /dev/null +++ b/hw/net/e1000_common.h @@ -0,0 +1,102 @@ +/* + * QEMU e1000(e) emulation - shared definitions + * + * Copyright (c) 2008 Qumranet + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef HW_NET_E1000_COMMON_H +#define HW_NET_E1000_COMMON_H + +#include "e1000_regs.h" + +#define defreg(x) x = (E1000_##x >> 2) +enum { + defreg(CTRL), defreg(EECD), defreg(EERD), defreg(GPRC), + defreg(GPTC), defreg(ICR), defreg(ICS), defreg(IMC), + defreg(IMS), defreg(LEDCTL), defreg(MANC), defreg(MDIC), + defreg(MPC), defreg(PBA), defreg(RCTL), defreg(RDBAH0), + defreg(RDBAL0), defreg(RDH0), defreg(RDLEN0), defreg(RDT0), + defreg(STATUS), defreg(SWSM), defreg(TCTL), defreg(TDBAH), + defreg(TDBAL), defreg(TDH), defreg(TDLEN), defreg(TDT), + defreg(TDLEN1), defreg(TDBAL1), defreg(TDBAH1), defreg(TDH1), + defreg(TDT1), defreg(TORH), defreg(TORL), defreg(TOTH), + defreg(TOTL), defreg(TPR), defreg(TPT), defreg(TXDCTL), + defreg(WUFC), defreg(RA), defreg(MTA), defreg(CRCERRS), + defreg(VFTA), defreg(VET), defreg(RDTR), defreg(RADV), + defreg(TADV), defreg(ITR), defreg(SCC), defreg(ECOL), + defreg(MCC), defreg(LATECOL), defreg(COLC), defreg(DC), + defreg(TNCRS), defreg(SEQEC), defreg(CEXTERR), defreg(RLEC), + defreg(XONRXC), defreg(XONTXC), defreg(XOFFRXC), defreg(XOFFTXC), + defreg(FCRUC), defreg(AIT), defreg(TDFH), defreg(TDFT), + defreg(TDFHS), defreg(TDFTS), defreg(TDFPC), defreg(WUC), + defreg(WUS), defreg(POEMB), defreg(PBS), defreg(RDFH), + defreg(RDFT), defreg(RDFHS), defreg(RDFTS), defreg(RDFPC), + defreg(PBM), defreg(IPAV), defreg(IP4AT), defreg(IP6AT), + defreg(WUPM), defreg(FFLT), defreg(FFMT), defreg(FFVT), + defreg(TARC0), defreg(TARC1), defreg(IAM), defreg(EXTCNF_CTRL), + defreg(GCR), defreg(TIMINCA), defreg(EIAC), defreg(CTRL_EXT), + defreg(IVAR), defreg(MFUTP01), defreg(MFUTP23), defreg(MANC2H), + defreg(MFVAL), defreg(MDEF), defreg(FACTPS), defreg(FTFT), + defreg(RUC), defreg(ROC), defreg(RFC), defreg(RJC), + defreg(PRC64), defreg(PRC127), defreg(PRC255), defreg(PRC511), + defreg(PRC1023), defreg(PRC1522), defreg(PTC64), defreg(PTC127), + defreg(PTC255), defreg(PTC511), defreg(PTC1023), defreg(PTC1522), + defreg(GORCL), defreg(GORCH), defreg(GOTCL), defreg(GOTCH), + defreg(RNBC), defreg(BPRC), defreg(MPRC), defreg(RFCTL), + defreg(PSRCTL), defreg(MPTC), defreg(BPTC), defreg(TSCTFC), + defreg(IAC), defreg(MGTPRC), defreg(MGTPDC), defreg(MGTPTC), + defreg(TSCTC), defreg(RXCSUM), defreg(FUNCTAG), defreg(GSCL_1), + defreg(GSCL_2), defreg(GSCL_3), defreg(GSCL_4), defreg(GSCN_0), + defreg(GSCN_1), defreg(GSCN_2), defreg(GSCN_3), defreg(GCR2), + defreg(RAID), defreg(RSRPD), defreg(TIDV), defreg(EITR), + defreg(MRQC), defreg(RETA), defreg(RSSRK), defreg(RDBAH1), + defreg(RDBAL1), defreg(RDLEN1), defreg(RDH1), defreg(RDT1), + defreg(PBACLR), defreg(FCAL), defreg(FCAH), defreg(FCT), + defreg(FCRTH), defreg(FCRTL), defreg(FCTTV), defreg(FCRTV), + defreg(FLA), defreg(EEWR), defreg(FLOP), defreg(FLOL), + defreg(FLSWCTL), defreg(FLSWCNT), defreg(RXDCTL), defreg(RXDCTL1), + defreg(MAVTV0), defreg(MAVTV1), defreg(MAVTV2), defreg(MAVTV3), + defreg(TXSTMPL), defreg(TXSTMPH), defreg(SYSTIML), defreg(SYSTIMH), + defreg(RXCFGL), defreg(RXUDP), defreg(TIMADJL), defreg(TIMADJH), + defreg(RXSTMPH), defreg(RXSTMPL), defreg(RXSATRL), defreg(RXSATRH), + defreg(FLASHT), defreg(TIPG), defreg(RDH), defreg(RDT), + defreg(RDLEN), defreg(RDBAH), defreg(RDBAL), + defreg(TXDCTL1), + defreg(FLSWDATA), + defreg(CTRL_DUP), + defreg(EXTCNF_SIZE), + defreg(EEMNGCTL), + defreg(EEMNGDATA), + defreg(FLMNGCTL), + defreg(FLMNGDATA), + defreg(FLMNGCNT), + defreg(TSYNCRXCTL), + defreg(TSYNCTXCTL), + + /* Aliases */ + defreg(RDH0_A), defreg(RDT0_A), defreg(RDTR_A), defreg(RDFH_A), + defreg(RDFT_A), defreg(TDH_A), defreg(TDT_A), defreg(TIDV_A), + defreg(TDFH_A), defreg(TDFT_A), defreg(RA_A), defreg(RDBAL0_A), + defreg(TDBAL_A), defreg(TDLEN_A), defreg(VFTA_A), defreg(RDLEN0_A), + defreg(FCRTL_A), defreg(FCRTH_A) +}; + +#endif diff --git a/hw/net/e1000_regs.h b/hw/net/e1000_regs.h index 59e050742b..8a4ce82034 100644 --- a/hw/net/e1000_regs.h +++ b/hw/net/e1000_regs.h @@ -32,157 +32,35 @@ #ifndef HW_E1000_REGS_H #define HW_E1000_REGS_H -/* PCI Device IDs */ -#define E1000_DEV_ID_82542 0x1000 -#define E1000_DEV_ID_82543GC_FIBER 0x1001 -#define E1000_DEV_ID_82543GC_COPPER 0x1004 -#define E1000_DEV_ID_82544EI_COPPER 0x1008 -#define E1000_DEV_ID_82544EI_FIBER 0x1009 -#define E1000_DEV_ID_82544GC_COPPER 0x100C -#define E1000_DEV_ID_82544GC_LOM 0x100D -#define E1000_DEV_ID_82540EM 0x100E -#define E1000_DEV_ID_82540EM_LOM 0x1015 -#define E1000_DEV_ID_82540EP_LOM 0x1016 -#define E1000_DEV_ID_82540EP 0x1017 -#define E1000_DEV_ID_82540EP_LP 0x101E -#define E1000_DEV_ID_82545EM_COPPER 0x100F -#define E1000_DEV_ID_82545EM_FIBER 0x1011 -#define E1000_DEV_ID_82545GM_COPPER 0x1026 -#define E1000_DEV_ID_82545GM_FIBER 0x1027 -#define E1000_DEV_ID_82545GM_SERDES 0x1028 -#define E1000_DEV_ID_82546EB_COPPER 0x1010 -#define E1000_DEV_ID_82546EB_FIBER 0x1012 -#define E1000_DEV_ID_82546EB_QUAD_COPPER 0x101D -#define E1000_DEV_ID_82541EI 0x1013 -#define E1000_DEV_ID_82541EI_MOBILE 0x1018 -#define E1000_DEV_ID_82541ER_LOM 0x1014 -#define E1000_DEV_ID_82541ER 0x1078 -#define E1000_DEV_ID_82547GI 0x1075 -#define E1000_DEV_ID_82541GI 0x1076 -#define E1000_DEV_ID_82541GI_MOBILE 0x1077 -#define E1000_DEV_ID_82541GI_LF 0x107C -#define E1000_DEV_ID_82546GB_COPPER 0x1079 -#define E1000_DEV_ID_82546GB_FIBER 0x107A -#define E1000_DEV_ID_82546GB_SERDES 0x107B -#define E1000_DEV_ID_82546GB_PCIE 0x108A -#define E1000_DEV_ID_82546GB_QUAD_COPPER 0x1099 -#define E1000_DEV_ID_82547EI 0x1019 -#define E1000_DEV_ID_82547EI_MOBILE 0x101A -#define E1000_DEV_ID_82571EB_COPPER 0x105E -#define E1000_DEV_ID_82571EB_FIBER 0x105F -#define E1000_DEV_ID_82571EB_SERDES 0x1060 -#define E1000_DEV_ID_82571EB_QUAD_COPPER 0x10A4 -#define E1000_DEV_ID_82571PT_QUAD_COPPER 0x10D5 -#define E1000_DEV_ID_82571EB_QUAD_FIBER 0x10A5 -#define E1000_DEV_ID_82571EB_QUAD_COPPER_LOWPROFILE 0x10BC -#define E1000_DEV_ID_82571EB_SERDES_DUAL 0x10D9 -#define E1000_DEV_ID_82571EB_SERDES_QUAD 0x10DA -#define E1000_DEV_ID_82572EI_COPPER 0x107D -#define E1000_DEV_ID_82572EI_FIBER 0x107E -#define E1000_DEV_ID_82572EI_SERDES 0x107F -#define E1000_DEV_ID_82572EI 0x10B9 -#define E1000_DEV_ID_82573E 0x108B -#define E1000_DEV_ID_82573E_IAMT 0x108C -#define E1000_DEV_ID_82573L 0x109A -#define E1000_DEV_ID_82574L 0x10D3 -#define E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3 0x10B5 -#define E1000_DEV_ID_80003ES2LAN_COPPER_DPT 0x1096 -#define E1000_DEV_ID_80003ES2LAN_SERDES_DPT 0x1098 -#define E1000_DEV_ID_80003ES2LAN_COPPER_SPT 0x10BA -#define E1000_DEV_ID_80003ES2LAN_SERDES_SPT 0x10BB +#include "e1000x_regs.h" -#define E1000_DEV_ID_ICH8_IGP_M_AMT 0x1049 -#define E1000_DEV_ID_ICH8_IGP_AMT 0x104A -#define E1000_DEV_ID_ICH8_IGP_C 0x104B -#define E1000_DEV_ID_ICH8_IFE 0x104C -#define E1000_DEV_ID_ICH8_IFE_GT 0x10C4 -#define E1000_DEV_ID_ICH8_IFE_G 0x10C5 -#define E1000_DEV_ID_ICH8_IGP_M 0x104D - -/* Device Specific Register Defaults */ -#define E1000_PHY_ID2_82541x 0x380 -#define E1000_PHY_ID2_82544x 0xC30 -#define E1000_PHY_ID2_8254xx_DEFAULT 0xC20 /* 82540x, 82545x, and 82546x */ -#define E1000_PHY_ID2_82573x 0xCC0 -#define E1000_PHY_ID2_82574x 0xCB1 - -/* Register Set. (82543, 82544) - * - * Registers are defined to be 32 bits and should be accessed as 32 bit values. - * These registers are physically located on the NIC, but are mapped into the - * host memory address space. - * - * RW - register is both readable and writable - * RO - register is read only - * WO - register is write only - * R/clr - register is read only and is cleared when read - * A - register array - */ -#define E1000_CTRL 0x00000 /* Device Control - RW */ -#define E1000_CTRL_DUP 0x00004 /* Device Control Duplicate (Shadow) - RW */ -#define E1000_STATUS 0x00008 /* Device Status - RO */ -#define E1000_EECD 0x00010 /* EEPROM/Flash Control - RW */ -#define E1000_EERD 0x00014 /* EEPROM Read - RW */ -#define E1000_CTRL_EXT 0x00018 /* Extended Device Control - RW */ -#define E1000_FLA 0x0001C /* Flash Access - RW */ -#define E1000_MDIC 0x00020 /* MDI Control - RW */ -#define E1000_SCTL 0x00024 /* SerDes Control - RW */ -#define E1000_FEXTNVM 0x00028 /* Future Extended NVM register */ -#define E1000_FCAL 0x00028 /* Flow Control Address Low - RW */ -#define E1000_FCAH 0x0002C /* Flow Control Address High -RW */ -#define E1000_FCT 0x00030 /* Flow Control Type - RW */ -#define E1000_VET 0x00038 /* VLAN Ether Type - RW */ -#define E1000_ICR 0x000C0 /* Interrupt Cause Read - R/clr */ #define E1000_ITR 0x000C4 /* Interrupt Throttling Rate - RW */ -#define E1000_ICS 0x000C8 /* Interrupt Cause Set - WO */ -#define E1000_IMS 0x000D0 /* Interrupt Mask Set - RW */ #define E1000_EIAC 0x000DC /* Ext. Interrupt Auto Clear - RW */ -#define E1000_IMC 0x000D8 /* Interrupt Mask Clear - WO */ -#define E1000_IAM 0x000E0 /* Interrupt Acknowledge Auto Mask */ #define E1000_IVAR 0x000E4 /* Interrupt Vector Allocation Register - RW */ #define E1000_EITR 0x000E8 /* Extended Interrupt Throttling Rate - RW */ -#define E1000_RCTL 0x00100 /* RX Control - RW */ -#define E1000_RDTR1 0x02820 /* RX Delay Timer (1) - RW */ #define E1000_RDBAL1 0x02900 /* RX Descriptor Base Address Low (1) - RW */ #define E1000_RDBAH1 0x02904 /* RX Descriptor Base Address High (1) - RW */ #define E1000_RDLEN1 0x02908 /* RX Descriptor Length (1) - RW */ #define E1000_RDH1 0x02910 /* RX Descriptor Head (1) - RW */ #define E1000_RDT1 0x02918 /* RX Descriptor Tail (1) - RW */ -#define E1000_FCTTV 0x00170 /* Flow Control Transmit Timer Value - RW */ #define E1000_FCRTV 0x05F40 /* Flow Control Refresh Timer Value - RW */ #define E1000_TXCW 0x00178 /* TX Configuration Word - RW */ #define E1000_RXCW 0x00180 /* RX Configuration Word - RO */ -#define E1000_TCTL 0x00400 /* TX Control - RW */ -#define E1000_TCTL_EXT 0x00404 /* Extended TX Control - RW */ -#define E1000_TIPG 0x00410 /* TX Inter-packet gap -RW */ #define E1000_TBT 0x00448 /* TX Burst Timer - RW */ #define E1000_AIT 0x00458 /* Adaptive Interframe Spacing Throttle - RW */ -#define E1000_LEDCTL 0x00E00 /* LED Control - RW */ #define E1000_EXTCNF_CTRL 0x00F00 /* Extended Configuration Control */ #define E1000_EXTCNF_SIZE 0x00F08 /* Extended Configuration Size */ #define E1000_PHY_CTRL 0x00F10 /* PHY Control Register in CSR */ -#define FEXTNVM_SW_CONFIG 0x0001 #define E1000_PBA 0x01000 /* Packet Buffer Allocation - RW */ #define E1000_PBM 0x10000 /* Packet Buffer Memory - RW */ #define E1000_PBS 0x01008 /* Packet Buffer Size - RW */ -#define E1000_EEMNGCTL 0x01010 /* MNG EEprom Control */ -#define E1000_EEMNGDATA 0x01014 /* MNG EEPROM Read/Write data */ -#define E1000_FLMNGCTL 0x01018 /* MNG Flash Control */ -#define E1000_FLMNGDATA 0x0101C /* MNG FLASH Read data */ -#define E1000_FLMNGCNT 0x01020 /* MNG FLASH Read Counter */ -#define E1000_FLASH_UPDATES 1000 -#define E1000_EEARBC 0x01024 /* EEPROM Auto Read Bus Control */ #define E1000_FLASHT 0x01028 /* FLASH Timer Register */ #define E1000_EEWR 0x0102C /* EEPROM Write Register - RW */ #define E1000_FLSWCTL 0x01030 /* FLASH control register */ #define E1000_FLSWDATA 0x01034 /* FLASH data register */ #define E1000_FLSWCNT 0x01038 /* FLASH Access Counter */ -#define E1000_FLOP 0x0103C /* FLASH Opcode Register */ #define E1000_FLOL 0x01050 /* FEEP Auto Load */ #define E1000_ERT 0x02008 /* Early Rx Threshold - RW */ -#define E1000_FCRTL 0x02160 /* Flow Control Receive Threshold Low - RW */ -#define E1000_FCRTL_A 0x00168 /* Alias to FCRTL */ -#define E1000_FCRTH 0x02168 /* Flow Control Receive Threshold High - RW */ #define E1000_FCRTH_A 0x00160 /* Alias to FCRTH */ #define E1000_PSRCTL 0x02170 /* Packet Split Receive Control - RW */ #define E1000_RDBAL 0x02800 /* RX Descriptor Base Address Low - RW */ @@ -208,23 +86,7 @@ #define E1000_RADV 0x0282C /* RX Interrupt Absolute Delay Timer - RW */ #define E1000_RSRPD 0x02C00 /* RX Small Packet Detect - RW */ #define E1000_RAID 0x02C08 /* Receive Ack Interrupt Delay - RW */ -#define E1000_TXDMAC 0x03000 /* TX DMA Control - RW */ -#define E1000_KABGTXD 0x03004 /* AFE Band Gap Transmit Ref Data */ #define E1000_POEMB 0x00F10 /* PHY OEM Bits Register - RW */ -#define E1000_RDFH 0x02410 /* Receive Data FIFO Head Register - RW */ -#define E1000_RDFH_A 0x08000 /* Alias to RDFH */ -#define E1000_RDFT 0x02418 /* Receive Data FIFO Tail Register - RW */ -#define E1000_RDFT_A 0x08008 /* Alias to RDFT */ -#define E1000_RDFHS 0x02420 /* Receive Data FIFO Head Saved Register - RW */ -#define E1000_RDFTS 0x02428 /* Receive Data FIFO Tail Saved Register - RW */ -#define E1000_RDFPC 0x02430 /* Receive Data FIFO Packet Count - RW */ -#define E1000_TDFH 0x03410 /* TX Data FIFO Head - RW */ -#define E1000_TDFH_A 0x08010 /* Alias to TDFH */ -#define E1000_TDFT 0x03418 /* TX Data FIFO Tail - RW */ -#define E1000_TDFT_A 0x08018 /* Alias to TDFT */ -#define E1000_TDFHS 0x03420 /* TX Data FIFO Head Saved - RW */ -#define E1000_TDFTS 0x03428 /* TX Data FIFO Tail Saved - RW */ -#define E1000_TDFPC 0x03430 /* TX Data FIFO Packet Count - RW */ #define E1000_TDBAL 0x03800 /* TX Descriptor Base Address Low - RW */ #define E1000_TDBAL_A 0x00420 /* Alias to TDBAL */ #define E1000_TDBAH 0x03804 /* TX Descriptor Base Address High - RW */ @@ -248,174 +110,40 @@ #define E1000_TDT1 0x03918 /* TX Desc Tail (1) - RW */ #define E1000_TXDCTL1 0x03928 /* TX Descriptor Control (1) - RW */ #define E1000_TARC1 0x03940 /* TX Arbitration Count (1) */ -#define E1000_CRCERRS 0x04000 /* CRC Error Count - R/clr */ -#define E1000_ALGNERRC 0x04004 /* Alignment Error Count - R/clr */ -#define E1000_SYMERRS 0x04008 /* Symbol Error Count - R/clr */ -#define E1000_RXERRC 0x0400C /* Receive Error Count - R/clr */ -#define E1000_MPC 0x04010 /* Missed Packet Count - R/clr */ -#define E1000_SCC 0x04014 /* Single Collision Count - R/clr */ -#define E1000_ECOL 0x04018 /* Excessive Collision Count - R/clr */ -#define E1000_MCC 0x0401C /* Multiple Collision Count - R/clr */ -#define E1000_LATECOL 0x04020 /* Late Collision Count - R/clr */ -#define E1000_COLC 0x04028 /* Collision Count - R/clr */ -#define E1000_DC 0x04030 /* Defer Count - R/clr */ -#define E1000_TNCRS 0x04034 /* TX-No CRS - R/clr */ #define E1000_SEQEC 0x04038 /* Sequence Error Count - R/clr */ #define E1000_CEXTERR 0x0403C /* Carrier Extension Error Count - R/clr */ -#define E1000_RLEC 0x04040 /* Receive Length Error Count - R/clr */ -#define E1000_XONRXC 0x04048 /* XON RX Count - R/clr */ -#define E1000_XONTXC 0x0404C /* XON TX Count - R/clr */ -#define E1000_XOFFRXC 0x04050 /* XOFF RX Count - R/clr */ -#define E1000_XOFFTXC 0x04054 /* XOFF TX Count - R/clr */ -#define E1000_FCRUC 0x04058 /* Flow Control RX Unsupported Count- R/clr */ -#define E1000_PRC64 0x0405C /* Packets RX (64 bytes) - R/clr */ -#define E1000_PRC127 0x04060 /* Packets RX (65-127 bytes) - R/clr */ -#define E1000_PRC255 0x04064 /* Packets RX (128-255 bytes) - R/clr */ -#define E1000_PRC511 0x04068 /* Packets RX (255-511 bytes) - R/clr */ -#define E1000_PRC1023 0x0406C /* Packets RX (512-1023 bytes) - R/clr */ -#define E1000_PRC1522 0x04070 /* Packets RX (1024-1522 bytes) - R/clr */ -#define E1000_GPRC 0x04074 /* Good Packets RX Count - R/clr */ -#define E1000_BPRC 0x04078 /* Broadcast Packets RX Count - R/clr */ -#define E1000_MPRC 0x0407C /* Multicast Packets RX Count - R/clr */ -#define E1000_GPTC 0x04080 /* Good Packets TX Count - R/clr */ -#define E1000_GORCL 0x04088 /* Good Octets RX Count Low - R/clr */ -#define E1000_GORCH 0x0408C /* Good Octets RX Count High - R/clr */ -#define E1000_GOTCL 0x04090 /* Good Octets TX Count Low - R/clr */ -#define E1000_GOTCH 0x04094 /* Good Octets TX Count High - R/clr */ -#define E1000_RNBC 0x040A0 /* RX No Buffers Count - R/clr */ -#define E1000_RUC 0x040A4 /* RX Undersize Count - R/clr */ -#define E1000_RFC 0x040A8 /* RX Fragment Count - R/clr */ -#define E1000_ROC 0x040AC /* RX Oversize Count - R/clr */ -#define E1000_RJC 0x040B0 /* RX Jabber Count - R/clr */ -#define E1000_MGTPRC 0x040B4 /* Management Packets RX Count - R/clr */ -#define E1000_MGTPDC 0x040B8 /* Management Packets Dropped Count - R/clr */ -#define E1000_MGTPTC 0x040BC /* Management Packets TX Count - R/clr */ -#define E1000_TORL 0x040C0 /* Total Octets RX Low - R/clr */ -#define E1000_TORH 0x040C4 /* Total Octets RX High - R/clr */ -#define E1000_TOTL 0x040C8 /* Total Octets TX Low - R/clr */ -#define E1000_TOTH 0x040CC /* Total Octets TX High - R/clr */ -#define E1000_TPR 0x040D0 /* Total Packets RX - R/clr */ -#define E1000_TPT 0x040D4 /* Total Packets TX - R/clr */ -#define E1000_PTC64 0x040D8 /* Packets TX (64 bytes) - R/clr */ -#define E1000_PTC127 0x040DC /* Packets TX (65-127 bytes) - R/clr */ -#define E1000_PTC255 0x040E0 /* Packets TX (128-255 bytes) - R/clr */ -#define E1000_PTC511 0x040E4 /* Packets TX (256-511 bytes) - R/clr */ -#define E1000_PTC1023 0x040E8 /* Packets TX (512-1023 bytes) - R/clr */ -#define E1000_PTC1522 0x040EC /* Packets TX (1024-1522 Bytes) - R/clr */ -#define E1000_MPTC 0x040F0 /* Multicast Packets TX Count - R/clr */ -#define E1000_BPTC 0x040F4 /* Broadcast Packets TX Count - R/clr */ -#define E1000_TSCTC 0x040F8 /* TCP Segmentation Context TX - R/clr */ #define E1000_TSCTFC 0x040FC /* TCP Segmentation Context TX Fail - R/clr */ -#define E1000_IAC 0x04100 /* Interrupt Assertion Count */ -#define E1000_ICRXPTC 0x04104 /* Interrupt Cause Rx Packet Timer Expire Count */ #define E1000_ICRXATC 0x04108 /* Interrupt Cause Rx Absolute Timer Expire Count */ #define E1000_ICTXPTC 0x0410C /* Interrupt Cause Tx Packet Timer Expire Count */ #define E1000_ICTXATC 0x04110 /* Interrupt Cause Tx Absolute Timer Expire Count */ #define E1000_ICTXQEC 0x04118 /* Interrupt Cause Tx Queue Empty Count */ #define E1000_ICTXQMTC 0x0411C /* Interrupt Cause Tx Queue Minimum Threshold Count */ -#define E1000_ICRXDMTC 0x04120 /* Interrupt Cause Rx Descriptor Minimum Threshold Count */ #define E1000_ICRXOC 0x04124 /* Interrupt Cause Receiver Overrun Count */ -#define E1000_RXCSUM 0x05000 /* RX Checksum Control - RW */ -#define E1000_RFCTL 0x05008 /* Receive Filter Control*/ -#define E1000_MAVTV0 0x05010 /* Management VLAN TAG Value 0 */ -#define E1000_MAVTV1 0x05014 /* Management VLAN TAG Value 1 */ -#define E1000_MAVTV2 0x05018 /* Management VLAN TAG Value 2 */ -#define E1000_MAVTV3 0x0501c /* Management VLAN TAG Value 3 */ -#define E1000_MTA 0x05200 /* Multicast Table Array - RW Array */ -#define E1000_RA 0x05400 /* Receive Address - RW Array */ -#define E1000_RA_A 0x00040 /* Alias to RA */ -#define E1000_VFTA 0x05600 /* VLAN Filter Table Array - RW Array */ -#define E1000_VFTA_A 0x00600 /* Alias to VFTA */ -#define E1000_WUC 0x05800 /* Wakeup Control - RW */ -#define E1000_WUFC 0x05808 /* Wakeup Filter Control - RW */ -#define E1000_WUS 0x05810 /* Wakeup Status - RO */ -#define E1000_MANC 0x05820 /* Management Control - RW */ -#define E1000_IPAV 0x05838 /* IP Address Valid - RW */ -#define E1000_IP4AT 0x05840 /* IPv4 Address Table - RW Array */ -#define E1000_IP6AT 0x05880 /* IPv6 Address Table - RW Array */ -#define E1000_WUPL 0x05900 /* Wakeup Packet Length - RW */ -#define E1000_WUPM 0x05A00 /* Wakeup Packet Memory - RO A */ #define E1000_MFUTP01 0x05828 /* Management Flex UDP/TCP Ports 0/1 - RW */ #define E1000_MFUTP23 0x05830 /* Management Flex UDP/TCP Ports 2/3 - RW */ -#define E1000_MFVAL 0x05824 /* Manageability Filters Valid - RW */ -#define E1000_MDEF 0x05890 /* Manageability Decision Filters - RW Array */ #define E1000_FFLT 0x05F00 /* Flexible Filter Length Table - RW Array */ #define E1000_HOST_IF 0x08800 /* Host Interface */ -#define E1000_FFMT 0x09000 /* Flexible Filter Mask Table - RW Array */ -#define E1000_FTFT 0x09400 /* Flexible TCO Filter Table - RW Array */ #define E1000_FFVT 0x09800 /* Flexible Filter Value Table - RW Array */ #define E1000_KUMCTRLSTA 0x00034 /* MAC-PHY interface - RW */ #define E1000_MDPHYA 0x0003C /* PHY address - RW */ -#define E1000_MANC2H 0x05860 /* Management Control To Host - RW */ -#define E1000_SW_FW_SYNC 0x05B5C /* Software-Firmware Synchronization - RW */ -#define E1000_GCR 0x05B00 /* PCI-Ex Control */ -#define E1000_FUNCTAG 0x05B08 /* Function-Tag Register */ -#define E1000_GSCL_1 0x05B10 /* PCI-Ex Statistic Control #1 */ -#define E1000_GSCL_2 0x05B14 /* PCI-Ex Statistic Control #2 */ -#define E1000_GSCL_3 0x05B18 /* PCI-Ex Statistic Control #3 */ -#define E1000_GSCL_4 0x05B1C /* PCI-Ex Statistic Control #4 */ -#define E1000_GSCN_0 0x05B20 /* 3GIO Statistic Counter Register #0 */ -#define E1000_GSCN_1 0x05B24 /* 3GIO Statistic Counter Register #1 */ -#define E1000_GSCN_2 0x05B28 /* 3GIO Statistic Counter Register #2 */ -#define E1000_GSCN_3 0x05B2C /* 3GIO Statistic Counter Register #3 */ -#define E1000_FACTPS 0x05B30 /* Function Active and Power State to MNG */ -#define E1000_SWSM 0x05B50 /* SW Semaphore */ #define E1000_GCR2 0x05B64 /* 3GIO Control Register 2 */ -#define E1000_FWSM 0x05B54 /* FW Semaphore */ -#define E1000_PBACLR 0x05B68 /* MSI-X PBA Clear */ #define E1000_FFLT_DBG 0x05F04 /* Debug Register */ #define E1000_HICR 0x08F00 /* Host Inteface Control */ -#define E1000_TSYNCRXCTL 0x0B620 /* Rx Time Sync Control register - RW */ -#define E1000_TSYNCTXCTL 0x0B614 /* Tx Time Sync Control register - RW */ -#define E1000_TIMINCA 0x0B608 /* Increment attributes register - RW */ -#define E1000_RXSTMPL 0x0B624 /* Rx timestamp Low - RO */ -#define E1000_RXSTMPH 0x0B628 /* Rx timestamp High - RO */ -#define E1000_TXSTMPL 0x0B618 /* Tx timestamp value Low - RO */ -#define E1000_TXSTMPH 0x0B61C /* Tx timestamp value High - RO */ -#define E1000_SYSTIML 0x0B600 /* System time register Low - RO */ -#define E1000_SYSTIMH 0x0B604 /* System time register High - RO */ -#define E1000_TIMINCA 0x0B608 /* Increment attributes register - RW */ #define E1000_RXMTRL 0x0B634 /* Time sync Rx EtherType and Msg Type - RW */ #define E1000_RXUDP 0x0B638 /* Time Sync Rx UDP Port - RW */ -#define E1000_RXSATRL 0x0B62C /* Rx timestamp attribute low - RO */ -#define E1000_RXSATRH 0x0B630 /* Rx timestamp attribute high - RO */ -#define E1000_TIMADJL 0x0B60C /* Time Adjustment Offset register Low - RW */ -#define E1000_TIMADJH 0x0B610 /* Time Adjustment Offset register High - RW */ #define E1000_RXCFGL 0x0B634 /* RX Ethertype and Message Type - RW*/ -/* RSS registers */ +#define E1000_MRQC_ENABLED(mrqc) (((mrqc) & (BIT(0) | BIT(1))) == BIT(0)) + #define E1000_CPUVEC 0x02C10 /* CPU Vector Register - RW */ -#define E1000_MRQC 0x05818 /* Multiple Receive Control - RW */ -#define E1000_RETA 0x05C00 /* Redirection Table - RW Array */ -#define E1000_RSSRK 0x05C80 /* RSS Random Key - RW Array */ #define E1000_RSSIM 0x05864 /* RSS Interrupt Mask */ #define E1000_RSSIR 0x05868 /* RSS Interrupt Request */ -#define E1000_MRQC_ENABLED(mrqc) (((mrqc) & (BIT(0) | BIT(1))) == BIT(0)) - -#define E1000_RETA_IDX(hash) ((hash) & (BIT(7) - 1)) -#define E1000_RETA_VAL(reta, hash) (((uint8_t *)(reta))[E1000_RETA_IDX(hash)]) #define E1000_RSS_QUEUE(reta, hash) ((E1000_RETA_VAL(reta, hash) & BIT(7)) >> 7) -#define E1000_MRQC_EN_TCPIPV4(mrqc) ((mrqc) & BIT(16)) -#define E1000_MRQC_EN_IPV4(mrqc) ((mrqc) & BIT(17)) -#define E1000_MRQC_EN_TCPIPV6(mrqc) ((mrqc) & BIT(18)) -#define E1000_MRQC_EN_IPV6EX(mrqc) ((mrqc) & BIT(19)) -#define E1000_MRQC_EN_IPV6(mrqc) ((mrqc) & BIT(20)) - -#define E1000_MRQ_RSS_TYPE_NONE (0) -#define E1000_MRQ_RSS_TYPE_IPV4TCP (1) -#define E1000_MRQ_RSS_TYPE_IPV4 (2) -#define E1000_MRQ_RSS_TYPE_IPV6TCP (3) -#define E1000_MRQ_RSS_TYPE_IPV6EX (4) -#define E1000_MRQ_RSS_TYPE_IPV6 (5) - -#define E1000_ICR_ASSERTED BIT(31) -#define E1000_EIAC_MASK 0x01F00000 - /* [TR]DBAL and [TR]DLEN masks */ #define E1000_XDBAL_MASK (~(BIT(4) - 1)) #define E1000_XDLEN_MASK ((BIT(20) - 1) & (~(BIT(7) - 1))) @@ -444,18 +172,8 @@ #define E1000_IVAR_TX_INT_EVERY_WB BIT(31) -/* RFCTL register bits */ -#define E1000_RFCTL_ISCSI_DIS 0x00000001 -#define E1000_RFCTL_NFSW_DIS 0x00000040 -#define E1000_RFCTL_NFSR_DIS 0x00000080 -#define E1000_RFCTL_IPV6_DIS 0x00000400 -#define E1000_RFCTL_IPV6_XSUM_DIS 0x00000800 #define E1000_RFCTL_ACK_DIS 0x00001000 #define E1000_RFCTL_ACK_DATA_DIS 0x00002000 -#define E1000_RFCTL_IPFRSP_DIS 0x00004000 -#define E1000_RFCTL_EXTEN 0x00008000 -#define E1000_RFCTL_IPV6_EX_DIS 0x00010000 -#define E1000_RFCTL_NEW_IPV6_EXT_DIS 0x00020000 /* PSRCTL parsing */ #define E1000_PSRCTL_BSIZE0_MASK 0x0000007F @@ -470,24 +188,7 @@ #define E1000_PSRCTL_BUFFS_PER_DESC 4 -/* TARC* parsing */ -#define E1000_TARC_ENABLE BIT(10) - /* PHY 1000 MII Register/Bit Definitions */ -/* PHY Registers defined by IEEE */ -#define PHY_CTRL 0x00 /* Control Register */ -#define PHY_STATUS 0x01 /* Status Regiser */ -#define PHY_ID1 0x02 /* Phy Id Reg (word 1) */ -#define PHY_ID2 0x03 /* Phy Id Reg (word 2) */ -#define PHY_AUTONEG_ADV 0x04 /* Autoneg Advertisement */ -#define PHY_LP_ABILITY 0x05 /* Link Partner Ability (Base Page) */ -#define PHY_AUTONEG_EXP 0x06 /* Autoneg Expansion Reg */ -#define PHY_NEXT_PAGE_TX 0x07 /* Next Page TX */ -#define PHY_LP_NEXT_PAGE 0x08 /* Link Partner Next Page */ -#define PHY_1000T_CTRL 0x09 /* 1000Base-T Control Reg */ -#define PHY_1000T_STATUS 0x0A /* 1000Base-T Status Reg */ -#define PHY_EXT_STATUS 0x0F /* Extended Status Reg */ - /* 82574-specific registers */ #define PHY_COPPER_CTRL1 0x10 /* Copper Specific Control Register 1 */ #define PHY_COPPER_STAT1 0x11 /* Copper Specific Status Register 1 */ @@ -539,287 +240,6 @@ #define M88E1000_PHY_VCO_REG_BIT8 0x100 /* Bits 8 & 11 are adjusted for */ #define M88E1000_PHY_VCO_REG_BIT11 0x800 /* improved BER performance */ -/* PHY Control Register */ -#define MII_CR_SPEED_SELECT_MSB 0x0040 /* bits 6,13: 10=1000, 01=100, 00=10 */ -#define MII_CR_COLL_TEST_ENABLE 0x0080 /* Collision test enable */ -#define MII_CR_FULL_DUPLEX 0x0100 /* FDX =1, half duplex =0 */ -#define MII_CR_RESTART_AUTO_NEG 0x0200 /* Restart auto negotiation */ -#define MII_CR_ISOLATE 0x0400 /* Isolate PHY from MII */ -#define MII_CR_POWER_DOWN 0x0800 /* Power down */ -#define MII_CR_AUTO_NEG_EN 0x1000 /* Auto Neg Enable */ -#define MII_CR_SPEED_SELECT_LSB 0x2000 /* bits 6,13: 10=1000, 01=100, 00=10 */ -#define MII_CR_LOOPBACK 0x4000 /* 0 = normal, 1 = loopback */ -#define MII_CR_RESET 0x8000 /* 0 = normal, 1 = PHY reset */ - -/* PHY Status Register */ -#define MII_SR_EXTENDED_CAPS 0x0001 /* Extended register capabilities */ -#define MII_SR_JABBER_DETECT 0x0002 /* Jabber Detected */ -#define MII_SR_LINK_STATUS 0x0004 /* Link Status 1 = link */ -#define MII_SR_AUTONEG_CAPS 0x0008 /* Auto Neg Capable */ -#define MII_SR_REMOTE_FAULT 0x0010 /* Remote Fault Detect */ -#define MII_SR_AUTONEG_COMPLETE 0x0020 /* Auto Neg Complete */ -#define MII_SR_PREAMBLE_SUPPRESS 0x0040 /* Preamble may be suppressed */ -#define MII_SR_EXTENDED_STATUS 0x0100 /* Ext. status info in Reg 0x0F */ -#define MII_SR_100T2_HD_CAPS 0x0200 /* 100T2 Half Duplex Capable */ -#define MII_SR_100T2_FD_CAPS 0x0400 /* 100T2 Full Duplex Capable */ -#define MII_SR_10T_HD_CAPS 0x0800 /* 10T Half Duplex Capable */ -#define MII_SR_10T_FD_CAPS 0x1000 /* 10T Full Duplex Capable */ -#define MII_SR_100X_HD_CAPS 0x2000 /* 100X Half Duplex Capable */ -#define MII_SR_100X_FD_CAPS 0x4000 /* 100X Full Duplex Capable */ -#define MII_SR_100T4_CAPS 0x8000 /* 100T4 Capable */ - -/* PHY Link Partner Ability Register */ -#define MII_LPAR_LPACK 0x4000 /* Acked by link partner */ - -/* Interrupt Cause Read */ -#define E1000_ICR_TXDW 0x00000001 /* Transmit desc written back */ -#define E1000_ICR_TXQE 0x00000002 /* Transmit Queue empty */ -#define E1000_ICR_LSC 0x00000004 /* Link Status Change */ -#define E1000_ICR_RXSEQ 0x00000008 /* rx sequence error */ -#define E1000_ICR_RXDMT0 0x00000010 /* rx desc min. threshold (0) */ -#define E1000_ICR_RXO 0x00000040 /* rx overrun */ -#define E1000_ICR_RXT0 0x00000080 /* rx timer intr (ring 0) */ -#define E1000_ICR_MDAC 0x00000200 /* MDIO access complete */ -#define E1000_ICR_RXCFG 0x00000400 /* RX /c/ ordered set */ -#define E1000_ICR_GPI_EN0 0x00000800 /* GP Int 0 */ -#define E1000_ICR_GPI_EN1 0x00001000 /* GP Int 1 */ -#define E1000_ICR_GPI_EN2 0x00002000 /* GP Int 2 */ -#define E1000_ICR_GPI_EN3 0x00004000 /* GP Int 3 */ -#define E1000_ICR_TXD_LOW 0x00008000 -#define E1000_ICR_SRPD 0x00010000 -#define E1000_ICR_ACK 0x00020000 /* Receive Ack frame */ -#define E1000_ICR_MNG 0x00040000 /* Manageability event */ -#define E1000_ICR_DOCK 0x00080000 /* Dock/Undock */ -#define E1000_ICR_INT_ASSERTED 0x80000000 /* If this bit asserted, the driver should claim the interrupt */ -#define E1000_ICR_RXD_FIFO_PAR0 0x00100000 /* queue 0 Rx descriptor FIFO parity error */ -#define E1000_ICR_TXD_FIFO_PAR0 0x00200000 /* queue 0 Tx descriptor FIFO parity error */ -#define E1000_ICR_HOST_ARB_PAR 0x00400000 /* host arb read buffer parity error */ -#define E1000_ICR_PB_PAR 0x00800000 /* packet buffer parity error */ -#define E1000_ICR_RXD_FIFO_PAR1 0x01000000 /* queue 1 Rx descriptor FIFO parity error */ -#define E1000_ICR_TXD_FIFO_PAR1 0x02000000 /* queue 1 Tx descriptor FIFO parity error */ -#define E1000_ICR_ALL_PARITY 0x03F00000 /* all parity error bits */ -#define E1000_ICR_DSW 0x00000020 /* FW changed the status of DISSW bit in the FWSM */ -#define E1000_ICR_PHYINT 0x00001000 /* LAN connected device generates an interrupt */ -#define E1000_ICR_EPRST 0x00100000 /* ME handware reset occurs */ -#define E1000_ICR_RXQ0 0x00100000 /* Rx Queue 0 Interrupt */ -#define E1000_ICR_RXQ1 0x00200000 /* Rx Queue 1 Interrupt */ -#define E1000_ICR_TXQ0 0x00400000 /* Tx Queue 0 Interrupt */ -#define E1000_ICR_TXQ1 0x00800000 /* Tx Queue 1 Interrupt */ -#define E1000_ICR_OTHER 0x01000000 /* Other Interrupts */ - -#define E1000_ICR_OTHER_CAUSES (E1000_ICR_LSC | \ - E1000_ICR_RXO | \ - E1000_ICR_MDAC | \ - E1000_ICR_SRPD | \ - E1000_ICR_ACK | \ - E1000_ICR_MNG) - -/* Interrupt Cause Set */ -#define E1000_ICS_TXDW E1000_ICR_TXDW /* Transmit desc written back */ -#define E1000_ICS_TXQE E1000_ICR_TXQE /* Transmit Queue empty */ -#define E1000_ICS_LSC E1000_ICR_LSC /* Link Status Change */ -#define E1000_ICS_RXSEQ E1000_ICR_RXSEQ /* rx sequence error */ -#define E1000_ICS_RXDMT0 E1000_ICR_RXDMT0 /* rx desc min. threshold */ -#define E1000_ICS_RXO E1000_ICR_RXO /* rx overrun */ -#define E1000_ICS_RXT0 E1000_ICR_RXT0 /* rx timer intr */ -#define E1000_ICS_MDAC E1000_ICR_MDAC /* MDIO access complete */ -#define E1000_ICS_RXCFG E1000_ICR_RXCFG /* RX /c/ ordered set */ -#define E1000_ICS_GPI_EN0 E1000_ICR_GPI_EN0 /* GP Int 0 */ -#define E1000_ICS_GPI_EN1 E1000_ICR_GPI_EN1 /* GP Int 1 */ -#define E1000_ICS_GPI_EN2 E1000_ICR_GPI_EN2 /* GP Int 2 */ -#define E1000_ICS_GPI_EN3 E1000_ICR_GPI_EN3 /* GP Int 3 */ -#define E1000_ICS_TXD_LOW E1000_ICR_TXD_LOW -#define E1000_ICS_SRPD E1000_ICR_SRPD -#define E1000_ICS_ACK E1000_ICR_ACK /* Receive Ack frame */ -#define E1000_ICS_MNG E1000_ICR_MNG /* Manageability event */ -#define E1000_ICS_DOCK E1000_ICR_DOCK /* Dock/Undock */ -#define E1000_ICS_RXD_FIFO_PAR0 E1000_ICR_RXD_FIFO_PAR0 /* queue 0 Rx descriptor FIFO parity error */ -#define E1000_ICS_TXD_FIFO_PAR0 E1000_ICR_TXD_FIFO_PAR0 /* queue 0 Tx descriptor FIFO parity error */ -#define E1000_ICS_HOST_ARB_PAR E1000_ICR_HOST_ARB_PAR /* host arb read buffer parity error */ -#define E1000_ICS_PB_PAR E1000_ICR_PB_PAR /* packet buffer parity error */ -#define E1000_ICS_RXD_FIFO_PAR1 E1000_ICR_RXD_FIFO_PAR1 /* queue 1 Rx descriptor FIFO parity error */ -#define E1000_ICS_TXD_FIFO_PAR1 E1000_ICR_TXD_FIFO_PAR1 /* queue 1 Tx descriptor FIFO parity error */ -#define E1000_ICS_DSW E1000_ICR_DSW -#define E1000_ICS_PHYINT E1000_ICR_PHYINT -#define E1000_ICS_EPRST E1000_ICR_EPRST - -/* Interrupt Mask Set */ -#define E1000_IMS_TXDW E1000_ICR_TXDW /* Transmit desc written back */ -#define E1000_IMS_TXQE E1000_ICR_TXQE /* Transmit Queue empty */ -#define E1000_IMS_LSC E1000_ICR_LSC /* Link Status Change */ -#define E1000_IMS_RXSEQ E1000_ICR_RXSEQ /* rx sequence error */ -#define E1000_IMS_RXDMT0 E1000_ICR_RXDMT0 /* rx desc min. threshold */ -#define E1000_IMS_RXO E1000_ICR_RXO /* rx overrun */ -#define E1000_IMS_RXT0 E1000_ICR_RXT0 /* rx timer intr */ -#define E1000_IMS_MDAC E1000_ICR_MDAC /* MDIO access complete */ -#define E1000_IMS_RXCFG E1000_ICR_RXCFG /* RX /c/ ordered set */ -#define E1000_IMS_GPI_EN0 E1000_ICR_GPI_EN0 /* GP Int 0 */ -#define E1000_IMS_GPI_EN1 E1000_ICR_GPI_EN1 /* GP Int 1 */ -#define E1000_IMS_GPI_EN2 E1000_ICR_GPI_EN2 /* GP Int 2 */ -#define E1000_IMS_GPI_EN3 E1000_ICR_GPI_EN3 /* GP Int 3 */ -#define E1000_IMS_TXD_LOW E1000_ICR_TXD_LOW -#define E1000_IMS_SRPD E1000_ICR_SRPD -#define E1000_IMS_ACK E1000_ICR_ACK /* Receive Ack frame */ -#define E1000_IMS_MNG E1000_ICR_MNG /* Manageability event */ -#define E1000_IMS_RXQ0 E1000_ICR_RXQ0 -#define E1000_IMS_RXQ1 E1000_ICR_RXQ1 -#define E1000_IMS_TXQ0 E1000_ICR_TXQ0 -#define E1000_IMS_TXQ1 E1000_ICR_TXQ1 -#define E1000_IMS_OTHER E1000_ICR_OTHER -#define E1000_IMS_DOCK E1000_ICR_DOCK /* Dock/Undock */ -#define E1000_IMS_RXD_FIFO_PAR0 E1000_ICR_RXD_FIFO_PAR0 /* queue 0 Rx descriptor FIFO parity error */ -#define E1000_IMS_TXD_FIFO_PAR0 E1000_ICR_TXD_FIFO_PAR0 /* queue 0 Tx descriptor FIFO parity error */ -#define E1000_IMS_HOST_ARB_PAR E1000_ICR_HOST_ARB_PAR /* host arb read buffer parity error */ -#define E1000_IMS_PB_PAR E1000_ICR_PB_PAR /* packet buffer parity error */ -#define E1000_IMS_RXD_FIFO_PAR1 E1000_ICR_RXD_FIFO_PAR1 /* queue 1 Rx descriptor FIFO parity error */ -#define E1000_IMS_TXD_FIFO_PAR1 E1000_ICR_TXD_FIFO_PAR1 /* queue 1 Tx descriptor FIFO parity error */ -#define E1000_IMS_DSW E1000_ICR_DSW -#define E1000_IMS_PHYINT E1000_ICR_PHYINT -#define E1000_IMS_EPRST E1000_ICR_EPRST - -/* Interrupt Mask Clear */ -#define E1000_IMC_TXDW E1000_ICR_TXDW /* Transmit desc written back */ -#define E1000_IMC_TXQE E1000_ICR_TXQE /* Transmit Queue empty */ -#define E1000_IMC_LSC E1000_ICR_LSC /* Link Status Change */ -#define E1000_IMC_RXSEQ E1000_ICR_RXSEQ /* rx sequence error */ -#define E1000_IMC_RXDMT0 E1000_ICR_RXDMT0 /* rx desc min. threshold */ -#define E1000_IMC_RXO E1000_ICR_RXO /* rx overrun */ -#define E1000_IMC_RXT0 E1000_ICR_RXT0 /* rx timer intr */ -#define E1000_IMC_MDAC E1000_ICR_MDAC /* MDIO access complete */ -#define E1000_IMC_RXCFG E1000_ICR_RXCFG /* RX /c/ ordered set */ -#define E1000_IMC_GPI_EN0 E1000_ICR_GPI_EN0 /* GP Int 0 */ -#define E1000_IMC_GPI_EN1 E1000_ICR_GPI_EN1 /* GP Int 1 */ -#define E1000_IMC_GPI_EN2 E1000_ICR_GPI_EN2 /* GP Int 2 */ -#define E1000_IMC_GPI_EN3 E1000_ICR_GPI_EN3 /* GP Int 3 */ -#define E1000_IMC_TXD_LOW E1000_ICR_TXD_LOW -#define E1000_IMC_SRPD E1000_ICR_SRPD -#define E1000_IMC_ACK E1000_ICR_ACK /* Receive Ack frame */ -#define E1000_IMC_MNG E1000_ICR_MNG /* Manageability event */ -#define E1000_IMC_DOCK E1000_ICR_DOCK /* Dock/Undock */ -#define E1000_IMC_RXD_FIFO_PAR0 E1000_ICR_RXD_FIFO_PAR0 /* queue 0 Rx descriptor FIFO parity error */ -#define E1000_IMC_TXD_FIFO_PAR0 E1000_ICR_TXD_FIFO_PAR0 /* queue 0 Tx descriptor FIFO parity error */ -#define E1000_IMC_HOST_ARB_PAR E1000_ICR_HOST_ARB_PAR /* host arb read buffer parity error */ -#define E1000_IMC_PB_PAR E1000_ICR_PB_PAR /* packet buffer parity error */ -#define E1000_IMC_RXD_FIFO_PAR1 E1000_ICR_RXD_FIFO_PAR1 /* queue 1 Rx descriptor FIFO parity error */ -#define E1000_IMC_TXD_FIFO_PAR1 E1000_ICR_TXD_FIFO_PAR1 /* queue 1 Tx descriptor FIFO parity error */ -#define E1000_IMC_DSW E1000_ICR_DSW -#define E1000_IMC_PHYINT E1000_ICR_PHYINT -#define E1000_IMC_EPRST E1000_ICR_EPRST - -/* Receive Control */ -#define E1000_RCTL_RST 0x00000001 /* Software reset */ -#define E1000_RCTL_EN 0x00000002 /* enable */ -#define E1000_RCTL_SBP 0x00000004 /* store bad packet */ -#define E1000_RCTL_UPE 0x00000008 /* unicast promiscuous enable */ -#define E1000_RCTL_MPE 0x00000010 /* multicast promiscuous enab */ -#define E1000_RCTL_LPE 0x00000020 /* long packet enable */ -#define E1000_RCTL_LBM_NO 0x00000000 /* no loopback mode */ -#define E1000_RCTL_LBM_MAC 0x00000040 /* MAC loopback mode */ -#define E1000_RCTL_LBM_SLP 0x00000080 /* serial link loopback mode */ -#define E1000_RCTL_LBM_TCVR 0x000000C0 /* tcvr loopback mode */ -#define E1000_RCTL_DTYP_MASK 0x00000C00 /* Descriptor type mask */ -#define E1000_RCTL_DTYP_PS 0x00000400 /* Packet Split descriptor */ -#define E1000_RCTL_RDMTS_HALF 0x00000000 /* rx desc min threshold size */ -#define E1000_RCTL_RDMTS_QUAT 0x00000100 /* rx desc min threshold size */ -#define E1000_RCTL_RDMTS_EIGTH 0x00000200 /* rx desc min threshold size */ -#define E1000_RCTL_MO_SHIFT 12 /* multicast offset shift */ -#define E1000_RCTL_MO_0 0x00000000 /* multicast offset 11:0 */ -#define E1000_RCTL_MO_1 0x00001000 /* multicast offset 12:1 */ -#define E1000_RCTL_MO_2 0x00002000 /* multicast offset 13:2 */ -#define E1000_RCTL_MO_3 0x00003000 /* multicast offset 15:4 */ -#define E1000_RCTL_MDR 0x00004000 /* multicast desc ring 0 */ -#define E1000_RCTL_BAM 0x00008000 /* broadcast enable */ -/* these buffer sizes are valid if E1000_RCTL_BSEX is 0 */ -#define E1000_RCTL_SZ_2048 0x00000000 /* rx buffer size 2048 */ -#define E1000_RCTL_SZ_1024 0x00010000 /* rx buffer size 1024 */ -#define E1000_RCTL_SZ_512 0x00020000 /* rx buffer size 512 */ -#define E1000_RCTL_SZ_256 0x00030000 /* rx buffer size 256 */ -/* these buffer sizes are valid if E1000_RCTL_BSEX is 1 */ -#define E1000_RCTL_SZ_16384 0x00010000 /* rx buffer size 16384 */ -#define E1000_RCTL_SZ_8192 0x00020000 /* rx buffer size 8192 */ -#define E1000_RCTL_SZ_4096 0x00030000 /* rx buffer size 4096 */ -#define E1000_RCTL_VFE 0x00040000 /* vlan filter enable */ -#define E1000_RCTL_CFIEN 0x00080000 /* canonical form enable */ -#define E1000_RCTL_CFI 0x00100000 /* canonical form indicator */ -#define E1000_RCTL_DPF 0x00400000 /* discard pause frames */ -#define E1000_RCTL_PMCF 0x00800000 /* pass MAC control frames */ -#define E1000_RCTL_BSEX 0x02000000 /* Buffer size extension */ -#define E1000_RCTL_SECRC 0x04000000 /* Strip Ethernet CRC */ -#define E1000_RCTL_FLXBUF_MASK 0x78000000 /* Flexible buffer size */ -#define E1000_RCTL_FLXBUF_SHIFT 27 /* Flexible buffer shift */ - - -#define E1000_EEPROM_SWDPIN0 0x0001 /* SWDPIN 0 EEPROM Value */ -#define E1000_EEPROM_LED_LOGIC 0x0020 /* Led Logic Word */ -#define E1000_EEPROM_RW_REG_DATA 16 /* Offset to data in EEPROM read/write registers */ -#define E1000_EEPROM_RW_REG_DONE 0x10 /* Offset to READ/WRITE done bit */ -#define E1000_EEPROM_RW_REG_START 1 /* First bit for telling part to start operation */ -#define E1000_EEPROM_RW_ADDR_SHIFT 8 /* Shift to the address bits */ -#define E1000_EEPROM_POLL_WRITE 1 /* Flag for polling for write complete */ -#define E1000_EEPROM_POLL_READ 0 /* Flag for polling for read complete */ - -/* 82574 EERD/EEWR registers layout */ -#define E1000_EERW_START BIT(0) -#define E1000_EERW_DONE BIT(1) -#define E1000_EERW_ADDR_SHIFT 2 -#define E1000_EERW_ADDR_MASK ((1L << 14) - 1) -#define E1000_EERW_DATA_SHIFT 16 -#define E1000_EERW_DATA_MASK ((1L << 16) - 1) - -/* Register Bit Masks */ -/* Device Control */ -#define E1000_CTRL_FD 0x00000001 /* Full duplex.0=half; 1=full */ -#define E1000_CTRL_BEM 0x00000002 /* Endian Mode.0=little,1=big */ -#define E1000_CTRL_PRIOR 0x00000004 /* Priority on PCI. 0=rx,1=fair */ -#define E1000_CTRL_GIO_MASTER_DISABLE 0x00000004 /*Blocks new Master requests */ -#define E1000_CTRL_LRST 0x00000008 /* Link reset. 0=normal,1=reset */ -#define E1000_CTRL_TME 0x00000010 /* Test mode. 0=normal,1=test */ -#define E1000_CTRL_SLE 0x00000020 /* Serial Link on 0=dis,1=en */ -#define E1000_CTRL_ASDE 0x00000020 /* Auto-speed detect enable */ -#define E1000_CTRL_SLU 0x00000040 /* Set link up (Force Link) */ -#define E1000_CTRL_ILOS 0x00000080 /* Invert Loss-Of Signal */ -#define E1000_CTRL_SPD_SEL 0x00000300 /* Speed Select Mask */ -#define E1000_CTRL_SPD_10 0x00000000 /* Force 10Mb */ -#define E1000_CTRL_SPD_100 0x00000100 /* Force 100Mb */ -#define E1000_CTRL_SPD_1000 0x00000200 /* Force 1Gb */ -#define E1000_CTRL_BEM32 0x00000400 /* Big Endian 32 mode */ -#define E1000_CTRL_FRCSPD 0x00000800 /* Force Speed */ -#define E1000_CTRL_FRCDPX 0x00001000 /* Force Duplex */ -#define E1000_CTRL_D_UD_EN 0x00002000 /* Dock/Undock enable */ -#define E1000_CTRL_D_UD_POLARITY 0x00004000 /* Defined polarity of Dock/Undock indication in SDP[0] */ -#define E1000_CTRL_FORCE_PHY_RESET 0x00008000 /* Reset both PHY ports, through PHYRST_N pin */ -#define E1000_CTRL_SPD_SHIFT 8 /* Speed Select Shift */ - -#define E1000_CTRL_EXT_ASDCHK 0x00001000 /* auto speed detection check */ -#define E1000_CTRL_EXT_EE_RST 0x00002000 /* EEPROM reset */ -#define E1000_CTRL_EXT_LINK_EN 0x00010000 /* enable link status from external LINK_0 and LINK_1 pins */ -#define E1000_CTRL_EXT_DRV_LOAD 0x10000000 /* Driver loaded bit for FW */ -#define E1000_CTRL_EXT_EIAME 0x01000000 -#define E1000_CTRL_EXT_IAME 0x08000000 /* Int ACK Auto-mask */ -#define E1000_CTRL_EXT_PBA_CLR 0x80000000 /* PBA Clear */ -#define E1000_CTRL_EXT_INT_TIMERS_CLEAR_ENA 0x20000000 -#define E1000_CTRL_EXT_SPD_BYPS 0x00008000 /* Speed Select Bypass */ - -#define E1000_CTRL_SWDPIN0 0x00040000 /* SWDPIN 0 value */ -#define E1000_CTRL_SWDPIN1 0x00080000 /* SWDPIN 1 value */ -#define E1000_CTRL_SWDPIN2 0x00100000 /* SWDPIN 2 value */ -#define E1000_CTRL_SWDPIN3 0x00200000 /* SWDPIN 3 value */ -#define E1000_CTRL_SWDPIO0 0x00400000 /* SWDPIN 0 Input or output */ -#define E1000_CTRL_SWDPIO1 0x00800000 /* SWDPIN 1 input or output */ -#define E1000_CTRL_SWDPIO2 0x01000000 /* SWDPIN 2 input or output */ -#define E1000_CTRL_SWDPIO3 0x02000000 /* SWDPIN 3 input or output */ -#define E1000_CTRL_ADVD3WUC 0x00100000 /* D3 WUC */ -#define E1000_CTRL_RST 0x04000000 /* Global reset */ -#define E1000_CTRL_RFCE 0x08000000 /* Receive Flow Control enable */ -#define E1000_CTRL_TFCE 0x10000000 /* Transmit flow control enable */ -#define E1000_CTRL_RTE 0x20000000 /* Routing tag enable */ -#define E1000_CTRL_VME 0x40000000 /* IEEE VLAN mode enable */ -#define E1000_CTRL_PHY_RST 0x80000000 /* PHY Reset */ -#define E1000_CTRL_SW2FW_INT 0x02000000 /* Initiate an interrupt to manageability engine */ - -/* Device Status */ -#define E1000_STATUS_FD 0x00000001 /* Full duplex.0=half,1=full */ -#define E1000_STATUS_LU 0x00000002 /* Link up.0=no,1=link */ #define E1000_STATUS_FUNC_MASK 0x0000000C /* PCI Function Mask */ #define E1000_STATUS_FUNC_SHIFT 2 #define E1000_STATUS_FUNC_0 0x00000000 /* Function 0 */ @@ -827,9 +247,6 @@ #define E1000_STATUS_TXOFF 0x00000010 /* transmission paused */ #define E1000_STATUS_TBIMODE 0x00000020 /* TBI mode */ #define E1000_STATUS_SPEED_MASK 0x000000C0 -#define E1000_STATUS_SPEED_10 0x00000000 /* Speed 10Mb/s */ -#define E1000_STATUS_SPEED_100 0x00000040 /* Speed 100Mb/s */ -#define E1000_STATUS_SPEED_1000 0x00000080 /* Speed 1000Mb/s */ #define E1000_STATUS_LAN_INIT_DONE 0x00000200 /* Lan Init Completion by EEPROM/Flash */ #define E1000_STATUS_ASDV 0x00000300 /* Auto speed detect value */ @@ -837,9 +254,7 @@ #define E1000_STATUS_ASDV_100 0x00000100 /* ASDV 100Mb */ #define E1000_STATUS_ASDV_1000 0x00000200 /* ASDV 1Gb */ #define E1000_STATUS_DOCK_CI 0x00000800 /* Change in Dock/Undock state. Clear on write '0'. */ -#define E1000_STATUS_GIO_MASTER_ENABLE 0x00080000 /* Status of Master requests. */ #define E1000_STATUS_MTXCKOK 0x00000400 /* MTX clock running OK */ -#define E1000_STATUS_PHYRA 0x00000400 /* PHY Reset Asserted */ #define E1000_STATUS_PCI66 0x00000800 /* In 66Mhz slot */ #define E1000_STATUS_BUS64 0x00001000 /* In 64 bit slot */ #define E1000_STATUS_PCIX_MODE 0x00002000 /* PCI-X mode */ @@ -857,111 +272,6 @@ #define E1000_STATUS_SPEED_SHIFT 6 #define E1000_STATUS_ASDV_SHIFT 8 -/* EEPROM/Flash Control */ -#define E1000_EECD_SK 0x00000001 /* EEPROM Clock */ -#define E1000_EECD_CS 0x00000002 /* EEPROM Chip Select */ -#define E1000_EECD_DI 0x00000004 /* EEPROM Data In */ -#define E1000_EECD_DO 0x00000008 /* EEPROM Data Out */ -#define E1000_EECD_FWE_MASK 0x00000030 -#define E1000_EECD_FWE_DIS 0x00000010 /* Disable FLASH writes */ -#define E1000_EECD_FWE_EN 0x00000020 /* Enable FLASH writes */ -#define E1000_EECD_FWE_SHIFT 4 -#define E1000_EECD_REQ 0x00000040 /* EEPROM Access Request */ -#define E1000_EECD_GNT 0x00000080 /* EEPROM Access Grant */ -#define E1000_EECD_PRES 0x00000100 /* EEPROM Present */ -#define E1000_EECD_SIZE 0x00000200 /* EEPROM Size (0=64 word 1=256 word) */ -#define E1000_EECD_ADDR_BITS 0x00000400 /* EEPROM Addressing bits based on type - * (0-small, 1-large) */ -#define E1000_EECD_TYPE 0x00002000 /* EEPROM Type (1-SPI, 0-Microwire) */ -#ifndef E1000_EEPROM_GRANT_ATTEMPTS -#define E1000_EEPROM_GRANT_ATTEMPTS 1000 /* EEPROM # attempts to gain grant */ -#endif -#define E1000_EECD_AUTO_RD 0x00000200 /* EEPROM Auto Read done */ -#define E1000_EECD_SIZE_EX_MASK 0x00007800 /* EEprom Size */ -#define E1000_EECD_SIZE_EX_SHIFT 11 -#define E1000_EECD_NVADDS 0x00018000 /* NVM Address Size */ -#define E1000_EECD_SELSHAD 0x00020000 /* Select Shadow RAM */ -#define E1000_EECD_INITSRAM 0x00040000 /* Initialize Shadow RAM */ -#define E1000_EECD_FLUPD 0x00080000 /* Update FLASH */ -#define E1000_EECD_AUPDEN 0x00100000 /* Enable Autonomous FLASH update */ -#define E1000_EECD_SHADV 0x00200000 /* Shadow RAM Data Valid */ -#define E1000_EECD_SEC1VAL 0x00400000 /* Sector One Valid */ - - -#define E1000_EECD_SECVAL_SHIFT 22 -#define E1000_STM_OPCODE 0xDB00 -#define E1000_HICR_FW_RESET 0xC0 - -#define E1000_SHADOW_RAM_WORDS 2048 -#define E1000_ICH_NVM_SIG_WORD 0x13 -#define E1000_ICH_NVM_SIG_MASK 0xC0 - -/* MDI Control */ -#define E1000_MDIC_DATA_MASK 0x0000FFFF -#define E1000_MDIC_REG_MASK 0x001F0000 -#define E1000_MDIC_REG_SHIFT 16 -#define E1000_MDIC_PHY_MASK 0x03E00000 -#define E1000_MDIC_PHY_SHIFT 21 -#define E1000_MDIC_OP_WRITE 0x04000000 -#define E1000_MDIC_OP_READ 0x08000000 -#define E1000_MDIC_READY 0x10000000 -#define E1000_MDIC_INT_EN 0x20000000 -#define E1000_MDIC_ERROR 0x40000000 - -/* Rx Interrupt Delay Timer */ -#define E1000_RDTR_FPD BIT(31) - -/* Tx Interrupt Delay Timer */ -#define E1000_TIDV_FPD BIT(31) - -/* Delay increments in nanoseconds for delayed interrupts registers */ -#define E1000_INTR_DELAY_NS_RES (1024) - -/* Delay increments in nanoseconds for interrupt throttling registers */ -#define E1000_INTR_THROTTLING_NS_RES (256) - -/* EEPROM Commands - Microwire */ -#define EEPROM_READ_OPCODE_MICROWIRE 0x6 /* EEPROM read opcode */ -#define EEPROM_WRITE_OPCODE_MICROWIRE 0x5 /* EEPROM write opcode */ -#define EEPROM_ERASE_OPCODE_MICROWIRE 0x7 /* EEPROM erase opcode */ -#define EEPROM_EWEN_OPCODE_MICROWIRE 0x13 /* EEPROM erase/write enable */ -#define EEPROM_EWDS_OPCODE_MICROWIRE 0x10 /* EEPROM erast/write disable */ - -/* EEPROM Word Offsets */ -#define EEPROM_COMPAT 0x0003 -#define EEPROM_ID_LED_SETTINGS 0x0004 -#define EEPROM_VERSION 0x0005 -#define EEPROM_SERDES_AMPLITUDE 0x0006 /* For SERDES output amplitude adjustment. */ -#define EEPROM_PHY_CLASS_WORD 0x0007 -#define EEPROM_INIT_CONTROL1_REG 0x000A -#define EEPROM_INIT_CONTROL2_REG 0x000F -#define EEPROM_SWDEF_PINS_CTRL_PORT_1 0x0010 -#define EEPROM_INIT_CONTROL3_PORT_B 0x0014 -#define EEPROM_INIT_3GIO_3 0x001A -#define EEPROM_SWDEF_PINS_CTRL_PORT_0 0x0020 -#define EEPROM_INIT_CONTROL3_PORT_A 0x0024 -#define EEPROM_CFG 0x0012 -#define EEPROM_FLASH_VERSION 0x0032 -#define EEPROM_CHECKSUM_REG 0x003F - -#define E1000_EEPROM_CFG_DONE 0x00040000 /* MNG config cycle done */ -#define E1000_EEPROM_CFG_DONE_PORT_1 0x00080000 /* ...for second port */ - -/* PCI Express Control */ -/* 3GIO Control Register - GCR (0x05B00; RW) */ -#define E1000_L0S_ADJUST (1 << 9) -#define E1000_L1_ENTRY_LATENCY_MSB (1 << 23) -#define E1000_L1_ENTRY_LATENCY_LSB (1 << 25 | 1 << 26) - -#define E1000_L0S_ADJUST (1 << 9) -#define E1000_L1_ENTRY_LATENCY_MSB (1 << 23) -#define E1000_L1_ENTRY_LATENCY_LSB (1 << 25 | 1 << 26) - -#define E1000_GCR_RO_BITS (1 << 23 | 1 << 25 | 1 << 26) - -/* MSI-X PBA Clear register */ -#define E1000_PBACLR_VALID_MASK (BIT(5) - 1) - /* Transmit Descriptor */ struct e1000_tx_desc { uint64_t buffer_addr; /* Address of the descriptor's data buffer */ @@ -983,269 +293,7 @@ struct e1000_tx_desc { } upper; }; -/* Transmit Descriptor bit definitions */ -#define E1000_TXD_DTYP_D 0x00100000 /* Data Descriptor */ -#define E1000_TXD_DTYP_C 0x00000000 /* Context Descriptor */ #define E1000_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ #define E1000_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define E1000_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define E1000_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ -#define E1000_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define E1000_TXD_CMD_RS 0x08000000 /* Report Status */ -#define E1000_TXD_CMD_RPS 0x10000000 /* Report Packet Sent */ -#define E1000_TXD_CMD_DEXT 0x20000000 /* Descriptor extension (0 = legacy) */ -#define E1000_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define E1000_TXD_CMD_IDE 0x80000000 /* Enable Tidv register */ -#define E1000_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define E1000_TXD_STAT_EC 0x00000002 /* Excess Collisions */ -#define E1000_TXD_STAT_LC 0x00000004 /* Late Collisions */ -#define E1000_TXD_STAT_TU 0x00000008 /* Transmit underrun */ -#define E1000_TXD_CMD_TCP 0x01000000 /* TCP packet */ -#define E1000_TXD_CMD_IP 0x02000000 /* IP packet */ -#define E1000_TXD_CMD_TSE 0x04000000 /* TCP Seg enable */ -#define E1000_TXD_CMD_SNAP 0x40000000 /* Update SNAP header */ -#define E1000_TXD_STAT_TC 0x00000004 /* Tx Underrun */ -#define E1000_TXD_EXTCMD_TSTAMP 0x00000010 /* IEEE1588 Timestamp packet */ - -/* Transmit Control */ -#define E1000_TCTL_RST 0x00000001 /* software reset */ -#define E1000_TCTL_EN 0x00000002 /* enable tx */ -#define E1000_TCTL_BCE 0x00000004 /* busy check enable */ -#define E1000_TCTL_PSP 0x00000008 /* pad short packets */ -#define E1000_TCTL_CT 0x00000ff0 /* collision threshold */ -#define E1000_TCTL_COLD 0x003ff000 /* collision distance */ -#define E1000_TCTL_SWXOFF 0x00400000 /* SW Xoff transmission */ -#define E1000_TCTL_PBE 0x00800000 /* Packet Burst Enable */ -#define E1000_TCTL_RTLC 0x01000000 /* Re-transmit on late collision */ -#define E1000_TCTL_NRTU 0x02000000 /* No Re-transmit on underrun */ -#define E1000_TCTL_MULR 0x10000000 /* Multiple request support */ - -/* Legacy Receive Descriptor */ -struct e1000_rx_desc { - uint64_t buffer_addr; /* Address of the descriptor's data buffer */ - uint16_t length; /* Length of data DMAed into data buffer */ - uint16_t csum; /* Packet checksum */ - uint8_t status; /* Descriptor status */ - uint8_t errors; /* Descriptor Errors */ - uint16_t special; -}; - -/* Extended Receive Descriptor */ -union e1000_rx_desc_extended { - struct { - uint64_t buffer_addr; - uint64_t reserved; - } read; - struct { - struct { - uint32_t mrq; /* Multiple Rx Queues */ - union { - uint32_t rss; /* RSS Hash */ - struct { - uint16_t ip_id; /* IP id */ - uint16_t csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - uint32_t status_error; /* ext status/error */ - uint16_t length; - uint16_t vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; - -#define MAX_PS_BUFFERS 4 - -/* Number of packet split data buffers (not including the header buffer) */ -#define PS_PAGE_BUFFERS (MAX_PS_BUFFERS - 1) - -/* Receive Descriptor - Packet Split */ -union e1000_rx_desc_packet_split { - struct { - /* one buffer for protocol header(s), three data buffers */ - uint64_t buffer_addr[MAX_PS_BUFFERS]; - } read; - struct { - struct { - uint32_t mrq; /* Multiple Rx Queues */ - union { - uint32_t rss; /* RSS Hash */ - struct { - uint16_t ip_id; /* IP id */ - uint16_t csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - uint32_t status_error; /* ext status/error */ - uint16_t length0; /* length of buffer 0 */ - uint16_t vlan; /* VLAN tag */ - } middle; - struct { - uint16_t header_status; - /* length of buffers 1-3 */ - uint16_t length[PS_PAGE_BUFFERS]; - } upper; - uint64_t reserved; - } wb; /* writeback */ -}; - -/* Receive Checksum Control bits */ -#define E1000_RXCSUM_IPOFLD 0x100 /* IP Checksum Offload Enable */ -#define E1000_RXCSUM_TUOFLD 0x200 /* TCP/UDP Checksum Offload Enable */ -#define E1000_RXCSUM_PCSD 0x2000 /* Packet Checksum Disable */ - -#define E1000_RING_DESC_LEN (16) -#define E1000_RING_DESC_LEN_SHIFT (4) - -#define E1000_MIN_RX_DESC_LEN E1000_RING_DESC_LEN -#define E1000_MAX_RX_DESC_LEN (sizeof(union e1000_rx_desc_packet_split)) - -/* Receive Descriptor bit definitions */ -#define E1000_RXD_STAT_DD 0x01 /* Descriptor Done */ -#define E1000_RXD_STAT_EOP 0x02 /* End of Packet */ -#define E1000_RXD_STAT_IXSM 0x04 /* Ignore checksum */ -#define E1000_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ -#define E1000_RXD_STAT_UDPCS 0x10 /* UDP xsum caculated */ -#define E1000_RXD_STAT_TCPCS 0x20 /* TCP xsum calculated */ -#define E1000_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ -#define E1000_RXD_STAT_PIF 0x80 /* passed in-exact filter */ -#define E1000_RXD_STAT_IPIDV 0x200 /* IP identification valid */ -#define E1000_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ -#define E1000_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ -#define E1000_RXD_ERR_CE 0x01 /* CRC Error */ -#define E1000_RXD_ERR_SE 0x02 /* Symbol Error */ -#define E1000_RXD_ERR_SEQ 0x04 /* Sequence Error */ -#define E1000_RXD_ERR_CXE 0x10 /* Carrier Extension Error */ -#define E1000_RXD_ERR_TCPE 0x20 /* TCP/UDP Checksum Error */ -#define E1000_RXD_ERR_IPE 0x40 /* IP Checksum Error */ -#define E1000_RXD_ERR_RXE 0x80 /* Rx Data Error */ -#define E1000_RXD_SPC_VLAN_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ -#define E1000_RXD_SPC_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ -#define E1000_RXD_SPC_PRI_SHIFT 13 -#define E1000_RXD_SPC_CFI_MASK 0x1000 /* CFI is bit 12 */ -#define E1000_RXD_SPC_CFI_SHIFT 12 - -/* RX packet types */ -#define E1000_RXD_PKT_MAC (0) -#define E1000_RXD_PKT_IP4 (1) -#define E1000_RXD_PKT_IP4_XDP (2) -#define E1000_RXD_PKT_IP6 (5) -#define E1000_RXD_PKT_IP6_XDP (6) - -#define E1000_RXD_PKT_TYPE(t) ((t) << 16) - -#define E1000_RXDEXT_STATERR_CE 0x01000000 -#define E1000_RXDEXT_STATERR_SE 0x02000000 -#define E1000_RXDEXT_STATERR_SEQ 0x04000000 -#define E1000_RXDEXT_STATERR_CXE 0x10000000 -#define E1000_RXDEXT_STATERR_TCPE 0x20000000 -#define E1000_RXDEXT_STATERR_IPE 0x40000000 -#define E1000_RXDEXT_STATERR_RXE 0x80000000 - -#define E1000_RXDPS_HDRSTAT_HDRSP 0x00008000 -#define E1000_RXDPS_HDRSTAT_HDRLEN_MASK 0x000003FF - -/* Receive Address */ -#define E1000_RAH_AV 0x80000000 /* Receive descriptor valid */ - -/* Offload Context Descriptor */ -struct e1000_context_desc { - union { - uint32_t ip_config; - struct { - uint8_t ipcss; /* IP checksum start */ - uint8_t ipcso; /* IP checksum offset */ - uint16_t ipcse; /* IP checksum end */ - } ip_fields; - } lower_setup; - union { - uint32_t tcp_config; - struct { - uint8_t tucss; /* TCP checksum start */ - uint8_t tucso; /* TCP checksum offset */ - uint16_t tucse; /* TCP checksum end */ - } tcp_fields; - } upper_setup; - uint32_t cmd_and_length; /* */ - union { - uint32_t data; - struct { - uint8_t status; /* Descriptor status */ - uint8_t hdr_len; /* Header length */ - uint16_t mss; /* Maximum segment size */ - } fields; - } tcp_seg_setup; -}; - -/* Offload data descriptor */ -struct e1000_data_desc { - uint64_t buffer_addr; /* Address of the descriptor's buffer address */ - union { - uint32_t data; - struct { - uint16_t length; /* Data buffer length */ - uint8_t typ_len_ext; /* */ - uint8_t cmd; /* */ - } flags; - } lower; - union { - uint32_t data; - struct { - uint8_t status; /* Descriptor status */ - uint8_t popts; /* Packet Options */ - uint16_t special; /* */ - } fields; - } upper; -}; - -/* Management Control */ -#define E1000_MANC_SMBUS_EN 0x00000001 /* SMBus Enabled - RO */ -#define E1000_MANC_ASF_EN 0x00000002 /* ASF Enabled - RO */ -#define E1000_MANC_R_ON_FORCE 0x00000004 /* Reset on Force TCO - RO */ -#define E1000_MANC_RMCP_EN 0x00000100 /* Enable RCMP 026Fh Filtering */ -#define E1000_MANC_0298_EN 0x00000200 /* Enable RCMP 0298h Filtering */ -#define E1000_MANC_IPV4_EN 0x00000400 /* Enable IPv4 */ -#define E1000_MANC_IPV6_EN 0x00000800 /* Enable IPv6 */ -#define E1000_MANC_SNAP_EN 0x00001000 /* Accept LLC/SNAP */ -#define E1000_MANC_ARP_EN 0x00002000 /* Enable ARP Request Filtering */ -#define E1000_MANC_NEIGHBOR_EN 0x00004000 /* Enable Neighbor Discovery - * Filtering */ -#define E1000_MANC_ARP_RES_EN 0x00008000 /* Enable ARP response Filtering */ -#define E1000_MANC_DIS_IP_CHK_ARP 0x10000000 /* Disable IP address chacking */ - /*for ARP packets - in 82574 */ -#define E1000_MANC_TCO_RESET 0x00010000 /* TCO Reset Occurred */ -#define E1000_MANC_RCV_TCO_EN 0x00020000 /* Receive TCO Packets Enabled */ -#define E1000_MANC_REPORT_STATUS 0x00040000 /* Status Reporting Enabled */ -#define E1000_MANC_RCV_ALL 0x00080000 /* Receive All Enabled */ -#define E1000_MANC_BLK_PHY_RST_ON_IDE 0x00040000 /* Block phy resets */ -#define E1000_MANC_EN_MAC_ADDR_FILTER 0x00100000 /* Enable MAC address - * filtering */ -#define E1000_MANC_EN_MNG2HOST 0x00200000 /* Enable MNG packets to host - * memory */ -#define E1000_MANC_EN_IP_ADDR_FILTER 0x00400000 /* Enable IP address - * filtering */ -#define E1000_MANC_EN_XSUM_FILTER 0x00800000 /* Enable checksum filtering */ -#define E1000_MANC_BR_EN 0x01000000 /* Enable broadcast filtering */ -#define E1000_MANC_SMB_REQ 0x01000000 /* SMBus Request */ -#define E1000_MANC_SMB_GNT 0x02000000 /* SMBus Grant */ -#define E1000_MANC_SMB_CLK_IN 0x04000000 /* SMBus Clock In */ -#define E1000_MANC_SMB_DATA_IN 0x08000000 /* SMBus Data In */ -#define E1000_MANC_SMB_DATA_OUT 0x10000000 /* SMBus Data Out */ -#define E1000_MANC_SMB_CLK_OUT 0x20000000 /* SMBus Clock Out */ - -#define E1000_MANC_SMB_DATA_OUT_SHIFT 28 /* SMBus Data Out Shift */ -#define E1000_MANC_SMB_CLK_OUT_SHIFT 29 /* SMBus Clock Out Shift */ - -/* FACTPS Control */ -#define E1000_FACTPS_LAN0_ON 0x00000004 /* Lan 0 enable */ - -/* For checksumming, the sum of all words in the EEPROM should equal 0xBABA. */ -#define EEPROM_SUM 0xBABA - -/* I/O-Mapped Access to Internal Registers, Memories, and Flash */ -#define E1000_IOADDR 0x00 -#define E1000_IODATA 0x04 #endif /* HW_E1000_REGS_H */ diff --git a/hw/net/e1000e.c b/hw/net/e1000e.c index 7523e9f5d2..c3848797b8 100644 --- a/hw/net/e1000e.c +++ b/hw/net/e1000e.c @@ -1,37 +1,37 @@ /* -* QEMU INTEL 82574 GbE NIC emulation -* -* Software developer's manuals: -* http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf -* -* Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) -* Developed by Daynix Computing LTD (http://www.daynix.com) -* -* Authors: -* Dmitry Fleytman <dmitry@daynix.com> -* Leonid Bloch <leonid@daynix.com> -* Yan Vugenfirer <yan@daynix.com> -* -* Based on work done by: -* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. -* Copyright (c) 2008 Qumranet -* Based on work done by: -* Copyright (c) 2007 Dan Aloni -* Copyright (c) 2004 Antony T Curtis -* -* This library is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License as published by the Free Software Foundation; either -* version 2.1 of the License, or (at your option) any later version. -* -* This library is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* Lesser General Public License for more details. -* -* You should have received a copy of the GNU Lesser General Public -* License along with this library; if not, see <http://www.gnu.org/licenses/>. -*/ + * QEMU INTEL 82574 GbE NIC emulation + * + * Software developer's manuals: + * http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf + * + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2008 Qumranet + * Based on work done by: + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ #include "qemu/osdep.h" #include "qemu/units.h" @@ -42,13 +42,13 @@ #include "qemu/range.h" #include "sysemu/sysemu.h" #include "hw/hw.h" +#include "hw/net/mii.h" #include "hw/pci/msi.h" #include "hw/pci/msix.h" #include "hw/qdev-properties.h" #include "migration/vmstate.h" -#include "e1000_regs.h" - +#include "e1000_common.h" #include "e1000x_common.h" #include "e1000e_core.h" @@ -81,6 +81,7 @@ struct E1000EState { E1000ECore core; bool init_vet; + bool timadj; }; #define E1000E_MMIO_IDX 0 @@ -239,9 +240,9 @@ static NetClientInfo net_e1000e_info = { }; /* -* EEPROM (NVM) contents documented in Table 36, section 6.1 -* and generally 6.1.2 Software accessed words. -*/ + * EEPROM (NVM) contents documented in Table 36, section 6.1 + * and generally 6.1.2 Software accessed words. + */ static const uint16_t e1000e_eeprom_template[64] = { /* Address | Compat. | ImVer | Compat. */ 0x0000, 0x0000, 0x0000, 0x0420, 0xf746, 0x2010, 0xffff, 0xffff, @@ -512,11 +513,11 @@ static void e1000e_pci_uninit(PCIDevice *pci_dev) msi_uninit(pci_dev); } -static void e1000e_qdev_reset(DeviceState *dev) +static void e1000e_qdev_reset_hold(Object *obj) { - E1000EState *s = E1000E(dev); + E1000EState *s = E1000E(obj); - trace_e1000e_cb_qdev_reset(); + trace_e1000e_cb_qdev_reset_hold(); e1000e_core_reset(&s->core); @@ -553,6 +554,12 @@ static int e1000e_post_load(void *opaque, int version_id) return e1000e_core_post_load(&s->core); } +static bool e1000e_migrate_timadj(void *opaque, int version_id) +{ + E1000EState *s = opaque; + return s->timadj; +} + static const VMStateDescription e1000e_vmstate_tx = { .name = "e1000e-tx", .version_id = 1, @@ -630,12 +637,11 @@ static const VMStateDescription e1000e_vmstate = { VMSTATE_E1000E_INTR_DELAY_TIMER(core.tidv, E1000EState), VMSTATE_E1000E_INTR_DELAY_TIMER(core.itr, E1000EState), - VMSTATE_BOOL(core.itr_intr_pending, E1000EState), + VMSTATE_UNUSED(1), VMSTATE_E1000E_INTR_DELAY_TIMER_ARRAY(core.eitr, E1000EState, E1000E_MSIX_VEC_NUM), - VMSTATE_BOOL_ARRAY(core.eitr_intr_pending, E1000EState, - E1000E_MSIX_VEC_NUM), + VMSTATE_UNUSED(E1000E_MSIX_VEC_NUM), VMSTATE_UINT32(core.itr_guest_value, E1000EState), VMSTATE_UINT32_ARRAY(core.eitr_guest_value, E1000EState, @@ -645,6 +651,9 @@ static const VMStateDescription e1000e_vmstate = { VMSTATE_STRUCT_ARRAY(core.tx, E1000EState, E1000E_NUM_QUEUES, 0, e1000e_vmstate_tx, struct e1000e_tx), + + VMSTATE_INT64_TEST(core.timadj, E1000EState, e1000e_migrate_timadj), + VMSTATE_END_OF_LIST() } }; @@ -663,12 +672,14 @@ static Property e1000e_properties[] = { DEFINE_PROP_SIGNED("subsys", E1000EState, subsys, 0, e1000e_prop_subsys, uint16_t), DEFINE_PROP_BOOL("init-vet", E1000EState, init_vet, true), + DEFINE_PROP_BOOL("migrate-timadj", E1000EState, timadj, true), DEFINE_PROP_END_OF_LIST(), }; static void e1000e_class_init(ObjectClass *class, void *data) { DeviceClass *dc = DEVICE_CLASS(class); + ResettableClass *rc = RESETTABLE_CLASS(class); PCIDeviceClass *c = PCI_DEVICE_CLASS(class); c->realize = e1000e_pci_realize; @@ -679,8 +690,9 @@ static void e1000e_class_init(ObjectClass *class, void *data) c->romfile = "efi-e1000e.rom"; c->class_id = PCI_CLASS_NETWORK_ETHERNET; + rc->phases.hold = e1000e_qdev_reset_hold; + dc->desc = "Intel 82574L GbE Controller"; - dc->reset = e1000e_qdev_reset; dc->vmsd = &e1000e_vmstate; e1000e_prop_disable_vnet = qdev_prop_uint8; diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c index fc9cdb4528..4d9679ca0b 100644 --- a/hw/net/e1000e_core.c +++ b/hw/net/e1000e_core.c @@ -1,42 +1,43 @@ /* -* Core code for QEMU e1000e emulation -* -* Software developer's manuals: -* http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf -* -* Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) -* Developed by Daynix Computing LTD (http://www.daynix.com) -* -* Authors: -* Dmitry Fleytman <dmitry@daynix.com> -* Leonid Bloch <leonid@daynix.com> -* Yan Vugenfirer <yan@daynix.com> -* -* Based on work done by: -* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. -* Copyright (c) 2008 Qumranet -* Based on work done by: -* Copyright (c) 2007 Dan Aloni -* Copyright (c) 2004 Antony T Curtis -* -* This library is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License as published by the Free Software Foundation; either -* version 2.1 of the License, or (at your option) any later version. -* -* This library is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* Lesser General Public License for more details. -* -* You should have received a copy of the GNU Lesser General Public -* License along with this library; if not, see <http://www.gnu.org/licenses/>. -*/ + * Core code for QEMU e1000e emulation + * + * Software developer's manuals: + * http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf + * + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2008 Qumranet + * Based on work done by: + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ #include "qemu/osdep.h" #include "qemu/log.h" #include "net/net.h" #include "net/tap.h" +#include "hw/net/mii.h" #include "hw/pci/msi.h" #include "hw/pci/msix.h" #include "sysemu/runstate.h" @@ -44,18 +45,32 @@ #include "net_tx_pkt.h" #include "net_rx_pkt.h" +#include "e1000_common.h" #include "e1000x_common.h" #include "e1000e_core.h" #include "trace.h" -#define E1000E_MIN_XITR (500) /* No more then 7813 interrupts per - second according to spec 10.2.4.2 */ +/* No more then 7813 interrupts per second according to spec 10.2.4.2 */ +#define E1000E_MIN_XITR (500) + #define E1000E_MAX_TX_FRAGS (64) +union e1000_rx_desc_union { + struct e1000_rx_desc legacy; + union e1000_rx_desc_extended extended; + union e1000_rx_desc_packet_split packet_split; +}; + +static ssize_t +e1000e_receive_internal(E1000ECore *core, const struct iovec *iov, int iovcnt, + bool has_vnet); + static inline void e1000e_set_interrupt_cause(E1000ECore *core, uint32_t val); +static void e1000e_reset(E1000ECore *core, bool sw); + static inline void e1000e_process_ts_option(E1000ECore *core, struct e1000_tx_desc *dp) { @@ -148,15 +163,8 @@ e1000e_intrmgr_on_throttling_timer(void *opaque) { E1000IntrDelayTimer *timer = opaque; - assert(!msix_enabled(timer->core->owner)); - timer->running = false; - if (!timer->core->itr_intr_pending) { - trace_e1000e_irq_throttling_no_pending_interrupts(); - return; - } - if (msi_enabled(timer->core->owner)) { trace_e1000e_irq_msi_notify_postponed(); /* Clear msi_causes_pending to fire MSI eventually */ @@ -174,15 +182,8 @@ e1000e_intrmgr_on_msix_throttling_timer(void *opaque) E1000IntrDelayTimer *timer = opaque; int idx = timer - &timer->core->eitr[0]; - assert(msix_enabled(timer->core->owner)); - timer->running = false; - if (!timer->core->eitr_intr_pending[idx]) { - trace_e1000e_irq_throttling_no_pending_vec(idx); - return; - } - trace_e1000e_irq_msix_notify_postponed_vec(idx); msix_notify(timer->core->owner, idx); } @@ -282,14 +283,18 @@ e1000e_intrmgr_delay_rx_causes(E1000ECore *core, uint32_t *causes) core->delayed_causes |= *causes & delayable_causes; *causes &= ~delayable_causes; - /* Check if delayed RX interrupts disabled by client - or if there are causes that cannot be delayed */ + /* + * Check if delayed RX interrupts disabled by client + * or if there are causes that cannot be delayed + */ if ((rdtr == 0) || (*causes != 0)) { return false; } - /* Check if delayed RX ACK interrupts disabled by client - and there is an ACK packet received */ + /* + * Check if delayed RX ACK interrupts disabled by client + * and there is an ACK packet received + */ if ((raid == 0) && (core->delayed_causes & E1000_ICR_ACK)) { return false; } @@ -493,27 +498,27 @@ typedef struct E1000E_RSSInfo_st { static uint32_t e1000e_rss_get_hash_type(E1000ECore *core, struct NetRxPkt *pkt) { - bool isip4, isip6, isudp, istcp; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; assert(e1000e_rss_enabled(core)); - net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); - - if (isip4) { - bool fragment = net_rx_pkt_get_ip4_info(pkt)->fragment; + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); - trace_e1000e_rx_rss_ip4(fragment, istcp, core->mac[MRQC], + if (hasip4) { + trace_e1000e_rx_rss_ip4(l4hdr_proto, core->mac[MRQC], E1000_MRQC_EN_TCPIPV4(core->mac[MRQC]), E1000_MRQC_EN_IPV4(core->mac[MRQC])); - if (!fragment && istcp && E1000_MRQC_EN_TCPIPV4(core->mac[MRQC])) { + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && + E1000_MRQC_EN_TCPIPV4(core->mac[MRQC])) { return E1000_MRQ_RSS_TYPE_IPV4TCP; } if (E1000_MRQC_EN_IPV4(core->mac[MRQC])) { return E1000_MRQ_RSS_TYPE_IPV4; } - } else if (isip6) { + } else if (hasip6) { eth_ip6_hdr_info *ip6info = net_rx_pkt_get_ip6_info(pkt); bool ex_dis = core->mac[RFCTL] & E1000_RFCTL_IPV6_EX_DIS; @@ -527,7 +532,7 @@ e1000e_rss_get_hash_type(E1000ECore *core, struct NetRxPkt *pkt) * backends like these. */ trace_e1000e_rx_rss_ip6_rfctl(core->mac[RFCTL]); - trace_e1000e_rx_rss_ip6(ex_dis, new_ex_dis, istcp, + trace_e1000e_rx_rss_ip6(ex_dis, new_ex_dis, l4hdr_proto, ip6info->has_ext_hdrs, ip6info->rss_ex_dst_valid, ip6info->rss_ex_src_valid, @@ -540,7 +545,7 @@ e1000e_rss_get_hash_type(E1000ECore *core, struct NetRxPkt *pkt) (!new_ex_dis || !(ip6info->rss_ex_dst_valid || ip6info->rss_ex_src_valid))) { - if (istcp && !ip6info->fragment && + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && E1000_MRQC_EN_TCPIPV6(core->mac[MRQC])) { return E1000_MRQ_RSS_TYPE_IPV6TCP; } @@ -625,23 +630,39 @@ e1000e_rss_parse_packet(E1000ECore *core, info->queue = E1000_RSS_QUEUE(&core->mac[RETA], info->hash); } -static void +static bool e1000e_setup_tx_offloads(E1000ECore *core, struct e1000e_tx *tx) { if (tx->props.tse && tx->cptse) { - net_tx_pkt_build_vheader(tx->tx_pkt, true, true, tx->props.mss); + if (!net_tx_pkt_build_vheader(tx->tx_pkt, true, true, tx->props.mss)) { + return false; + } + net_tx_pkt_update_ip_checksums(tx->tx_pkt); e1000x_inc_reg_if_not_full(core->mac, TSCTC); - return; + return true; } if (tx->sum_needed & E1000_TXD_POPTS_TXSM) { - net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0); + if (!net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0)) { + return false; + } } if (tx->sum_needed & E1000_TXD_POPTS_IXSM) { net_tx_pkt_update_ip_hdr_checksum(tx->tx_pkt); } + + return true; +} + +static void e1000e_tx_pkt_callback(void *core, + const struct iovec *iov, + int iovcnt, + const struct iovec *virt_iov, + int virt_iovcnt) +{ + e1000e_receive_internal(core, virt_iov, virt_iovcnt, true); } static bool @@ -650,13 +671,16 @@ e1000e_tx_pkt_send(E1000ECore *core, struct e1000e_tx *tx, int queue_index) int target_queue = MIN(core->max_queue_num, queue_index); NetClientState *queue = qemu_get_subqueue(core->owner_nic, target_queue); - e1000e_setup_tx_offloads(core, tx); + if (!e1000e_setup_tx_offloads(core, tx)) { + return false; + } net_tx_pkt_dump(tx->tx_pkt); - if ((core->phy[0][PHY_CTRL] & MII_CR_LOOPBACK) || + if ((core->phy[0][MII_BMCR] & MII_BMCR_LOOPBACK) || ((core->mac[RCTL] & E1000_RCTL_LBM_MAC) == E1000_RCTL_LBM_MAC)) { - return net_tx_pkt_send_loopback(tx->tx_pkt, queue); + return net_tx_pkt_send_custom(tx->tx_pkt, false, + e1000e_tx_pkt_callback, core); } else { return net_tx_pkt_send(tx->tx_pkt, queue); } @@ -668,7 +692,7 @@ e1000e_on_tx_done_update_stats(E1000ECore *core, struct NetTxPkt *tx_pkt) static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511, PTC1023, PTC1522 }; - size_t tot_len = net_tx_pkt_get_total_len(tx_pkt); + size_t tot_len = net_tx_pkt_get_total_len(tx_pkt) + 4; e1000x_increase_size_stats(core->mac, PTCregs, tot_len); e1000x_inc_reg_if_not_full(core->mac, TPT); @@ -1016,10 +1040,11 @@ e1000e_receive_filter(E1000ECore *core, const uint8_t *buf, int size) if (e1000x_is_vlan_packet(buf, core->mac[VET]) && e1000x_vlan_rx_filter_enabled(core->mac)) { - uint16_t vid = lduw_be_p(buf + 14); - uint32_t vfta = ldl_le_p((uint32_t *)(core->mac + VFTA) + - ((vid >> 5) & 0x7f)); - if ((vfta & (1 << (vid & 0x1f))) == 0) { + uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(buf)->h_tci); + uint32_t vfta = + ldl_le_p((uint32_t *)(core->mac + VFTA) + + ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK)); + if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) { trace_e1000e_rx_flt_vlan_mismatch(vid); return false; } else { @@ -1054,48 +1079,47 @@ e1000e_receive_filter(E1000ECore *core, const uint8_t *buf, int size) } static inline void -e1000e_read_lgcy_rx_descr(E1000ECore *core, uint8_t *desc, hwaddr *buff_addr) +e1000e_read_lgcy_rx_descr(E1000ECore *core, struct e1000_rx_desc *desc, + hwaddr *buff_addr) { - struct e1000_rx_desc *d = (struct e1000_rx_desc *) desc; - *buff_addr = le64_to_cpu(d->buffer_addr); + *buff_addr = le64_to_cpu(desc->buffer_addr); } static inline void -e1000e_read_ext_rx_descr(E1000ECore *core, uint8_t *desc, hwaddr *buff_addr) +e1000e_read_ext_rx_descr(E1000ECore *core, union e1000_rx_desc_extended *desc, + hwaddr *buff_addr) { - union e1000_rx_desc_extended *d = (union e1000_rx_desc_extended *) desc; - *buff_addr = le64_to_cpu(d->read.buffer_addr); + *buff_addr = le64_to_cpu(desc->read.buffer_addr); } static inline void -e1000e_read_ps_rx_descr(E1000ECore *core, uint8_t *desc, - hwaddr (*buff_addr)[MAX_PS_BUFFERS]) +e1000e_read_ps_rx_descr(E1000ECore *core, + union e1000_rx_desc_packet_split *desc, + hwaddr buff_addr[MAX_PS_BUFFERS]) { int i; - union e1000_rx_desc_packet_split *d = - (union e1000_rx_desc_packet_split *) desc; for (i = 0; i < MAX_PS_BUFFERS; i++) { - (*buff_addr)[i] = le64_to_cpu(d->read.buffer_addr[i]); + buff_addr[i] = le64_to_cpu(desc->read.buffer_addr[i]); } - trace_e1000e_rx_desc_ps_read((*buff_addr)[0], (*buff_addr)[1], - (*buff_addr)[2], (*buff_addr)[3]); + trace_e1000e_rx_desc_ps_read(buff_addr[0], buff_addr[1], + buff_addr[2], buff_addr[3]); } static inline void -e1000e_read_rx_descr(E1000ECore *core, uint8_t *desc, - hwaddr (*buff_addr)[MAX_PS_BUFFERS]) +e1000e_read_rx_descr(E1000ECore *core, union e1000_rx_desc_union *desc, + hwaddr buff_addr[MAX_PS_BUFFERS]) { if (e1000e_rx_use_legacy_descriptor(core)) { - e1000e_read_lgcy_rx_descr(core, desc, &(*buff_addr)[0]); - (*buff_addr)[1] = (*buff_addr)[2] = (*buff_addr)[3] = 0; + e1000e_read_lgcy_rx_descr(core, &desc->legacy, &buff_addr[0]); + buff_addr[1] = buff_addr[2] = buff_addr[3] = 0; } else { if (core->mac[RCTL] & E1000_RCTL_DTYP_PS) { - e1000e_read_ps_rx_descr(core, desc, buff_addr); + e1000e_read_ps_rx_descr(core, &desc->packet_split, buff_addr); } else { - e1000e_read_ext_rx_descr(core, desc, &(*buff_addr)[0]); - (*buff_addr)[1] = (*buff_addr)[2] = (*buff_addr)[3] = 0; + e1000e_read_ext_rx_descr(core, &desc->extended, &buff_addr[0]); + buff_addr[1] = buff_addr[2] = buff_addr[3] = 0; } } } @@ -1104,7 +1128,7 @@ static void e1000e_verify_csum_in_sw(E1000ECore *core, struct NetRxPkt *pkt, uint32_t *status_flags, - bool istcp, bool isudp) + EthL4HdrProto l4hdr_proto) { bool csum_valid; uint32_t csum_error; @@ -1131,14 +1155,10 @@ e1000e_verify_csum_in_sw(E1000ECore *core, } csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_TCPE; + *status_flags |= E1000_RXD_STAT_TCPCS | csum_error; - if (istcp) { - *status_flags |= E1000_RXD_STAT_TCPCS | - csum_error; - } else if (isudp) { - *status_flags |= E1000_RXD_STAT_TCPCS | - E1000_RXD_STAT_UDPCS | - csum_error; + if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { + *status_flags |= E1000_RXD_STAT_UDPCS; } } @@ -1167,7 +1187,8 @@ e1000e_build_rx_metadata(E1000ECore *core, uint16_t *vlan_tag) { struct virtio_net_hdr *vhdr; - bool isip4, isip6, istcp, isudp; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; uint32_t pkt_type; *status_flags = E1000_RXD_STAT_DD; @@ -1179,8 +1200,8 @@ e1000e_build_rx_metadata(E1000ECore *core, *status_flags |= E1000_RXD_STAT_EOP; - net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); - trace_e1000e_rx_metadata_protocols(isip4, isip6, isudp, istcp); + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + trace_e1000e_rx_metadata_protocols(hasip4, hasip6, l4hdr_proto); /* VLAN state */ if (net_rx_pkt_is_vlan_stripped(pkt)) { @@ -1196,24 +1217,25 @@ e1000e_build_rx_metadata(E1000ECore *core, *mrq = cpu_to_le32(rss_info->type | (rss_info->queue << 8)); trace_e1000e_rx_metadata_rss(*rss, *mrq); } - } else if (isip4) { + } else if (hasip4) { *status_flags |= E1000_RXD_STAT_IPIDV; *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt)); trace_e1000e_rx_metadata_ip_id(*ip_id); } - if (istcp && e1000e_is_tcp_ack(core, pkt)) { + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && e1000e_is_tcp_ack(core, pkt)) { *status_flags |= E1000_RXD_STAT_ACK; trace_e1000e_rx_metadata_ack(); } - if (isip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) { + if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) { trace_e1000e_rx_metadata_ipv6_filtering_disabled(); pkt_type = E1000_RXD_PKT_MAC; - } else if (istcp || isudp) { - pkt_type = isip4 ? E1000_RXD_PKT_IP4_XDP : E1000_RXD_PKT_IP6_XDP; - } else if (isip4 || isip6) { - pkt_type = isip4 ? E1000_RXD_PKT_IP4 : E1000_RXD_PKT_IP6; + } else if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP || + l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { + pkt_type = hasip4 ? E1000_RXD_PKT_IP4_XDP : E1000_RXD_PKT_IP6_XDP; + } else if (hasip4 || hasip6) { + pkt_type = hasip4 ? E1000_RXD_PKT_IP4 : E1000_RXD_PKT_IP6; } else { pkt_type = E1000_RXD_PKT_MAC; } @@ -1222,37 +1244,38 @@ e1000e_build_rx_metadata(E1000ECore *core, trace_e1000e_rx_metadata_pkt_type(pkt_type); /* RX CSO information */ - if (isip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) { + if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) { trace_e1000e_rx_metadata_ipv6_sum_disabled(); goto func_exit; } - if (!net_rx_pkt_has_virt_hdr(pkt)) { - trace_e1000e_rx_metadata_no_virthdr(); - e1000e_verify_csum_in_sw(core, pkt, status_flags, istcp, isudp); - goto func_exit; - } - vhdr = net_rx_pkt_get_vhdr(pkt); if (!(vhdr->flags & VIRTIO_NET_HDR_F_DATA_VALID) && !(vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) { trace_e1000e_rx_metadata_virthdr_no_csum_info(); - e1000e_verify_csum_in_sw(core, pkt, status_flags, istcp, isudp); + e1000e_verify_csum_in_sw(core, pkt, status_flags, l4hdr_proto); goto func_exit; } if (e1000e_rx_l3_cso_enabled(core)) { - *status_flags |= isip4 ? E1000_RXD_STAT_IPCS : 0; + *status_flags |= hasip4 ? E1000_RXD_STAT_IPCS : 0; } else { trace_e1000e_rx_metadata_l3_cso_disabled(); } if (e1000e_rx_l4_cso_enabled(core)) { - if (istcp) { + switch (l4hdr_proto) { + case ETH_L4_HDR_PROTO_TCP: *status_flags |= E1000_RXD_STAT_TCPCS; - } else if (isudp) { + break; + + case ETH_L4_HDR_PROTO_UDP: *status_flags |= E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS; + break; + + default: + break; } } else { trace_e1000e_rx_metadata_l4_cso_disabled(); @@ -1265,7 +1288,7 @@ func_exit: } static inline void -e1000e_write_lgcy_rx_descr(E1000ECore *core, uint8_t *desc, +e1000e_write_lgcy_rx_descr(E1000ECore *core, struct e1000_rx_desc *desc, struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length) @@ -1273,71 +1296,66 @@ e1000e_write_lgcy_rx_descr(E1000ECore *core, uint8_t *desc, uint32_t status_flags, rss, mrq; uint16_t ip_id; - struct e1000_rx_desc *d = (struct e1000_rx_desc *) desc; - assert(!rss_info->enabled); - d->length = cpu_to_le16(length); - d->csum = 0; + desc->length = cpu_to_le16(length); + desc->csum = 0; e1000e_build_rx_metadata(core, pkt, pkt != NULL, rss_info, &rss, &mrq, &status_flags, &ip_id, - &d->special); - d->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24); - d->status = (uint8_t) le32_to_cpu(status_flags); + &desc->special); + desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24); + desc->status = (uint8_t) le32_to_cpu(status_flags); } static inline void -e1000e_write_ext_rx_descr(E1000ECore *core, uint8_t *desc, +e1000e_write_ext_rx_descr(E1000ECore *core, union e1000_rx_desc_extended *desc, struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length) { - union e1000_rx_desc_extended *d = (union e1000_rx_desc_extended *) desc; - - memset(&d->wb, 0, sizeof(d->wb)); + memset(&desc->wb, 0, sizeof(desc->wb)); - d->wb.upper.length = cpu_to_le16(length); + desc->wb.upper.length = cpu_to_le16(length); e1000e_build_rx_metadata(core, pkt, pkt != NULL, rss_info, - &d->wb.lower.hi_dword.rss, - &d->wb.lower.mrq, - &d->wb.upper.status_error, - &d->wb.lower.hi_dword.csum_ip.ip_id, - &d->wb.upper.vlan); + &desc->wb.lower.hi_dword.rss, + &desc->wb.lower.mrq, + &desc->wb.upper.status_error, + &desc->wb.lower.hi_dword.csum_ip.ip_id, + &desc->wb.upper.vlan); } static inline void -e1000e_write_ps_rx_descr(E1000ECore *core, uint8_t *desc, +e1000e_write_ps_rx_descr(E1000ECore *core, + union e1000_rx_desc_packet_split *desc, struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, size_t ps_hdr_len, uint16_t(*written)[MAX_PS_BUFFERS]) { int i; - union e1000_rx_desc_packet_split *d = - (union e1000_rx_desc_packet_split *) desc; - memset(&d->wb, 0, sizeof(d->wb)); + memset(&desc->wb, 0, sizeof(desc->wb)); - d->wb.middle.length0 = cpu_to_le16((*written)[0]); + desc->wb.middle.length0 = cpu_to_le16((*written)[0]); for (i = 0; i < PS_PAGE_BUFFERS; i++) { - d->wb.upper.length[i] = cpu_to_le16((*written)[i + 1]); + desc->wb.upper.length[i] = cpu_to_le16((*written)[i + 1]); } e1000e_build_rx_metadata(core, pkt, pkt != NULL, rss_info, - &d->wb.lower.hi_dword.rss, - &d->wb.lower.mrq, - &d->wb.middle.status_error, - &d->wb.lower.hi_dword.csum_ip.ip_id, - &d->wb.middle.vlan); + &desc->wb.lower.hi_dword.rss, + &desc->wb.lower.mrq, + &desc->wb.middle.status_error, + &desc->wb.lower.hi_dword.csum_ip.ip_id, + &desc->wb.middle.vlan); - d->wb.upper.header_status = + desc->wb.upper.header_status = cpu_to_le16(ps_hdr_len | (ps_hdr_len ? E1000_RXDPS_HDRSTAT_HDRSP : 0)); trace_e1000e_rx_desc_ps_write((*written)[0], (*written)[1], @@ -1345,20 +1363,21 @@ e1000e_write_ps_rx_descr(E1000ECore *core, uint8_t *desc, } static inline void -e1000e_write_rx_descr(E1000ECore *core, uint8_t *desc, +e1000e_write_rx_descr(E1000ECore *core, union e1000_rx_desc_union *desc, struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, size_t ps_hdr_len, uint16_t(*written)[MAX_PS_BUFFERS]) { if (e1000e_rx_use_legacy_descriptor(core)) { assert(ps_hdr_len == 0); - e1000e_write_lgcy_rx_descr(core, desc, pkt, rss_info, (*written)[0]); + e1000e_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, + (*written)[0]); } else { if (core->mac[RCTL] & E1000_RCTL_DTYP_PS) { - e1000e_write_ps_rx_descr(core, desc, pkt, rss_info, + e1000e_write_ps_rx_descr(core, &desc->packet_split, pkt, rss_info, ps_hdr_len, written); } else { assert(ps_hdr_len == 0); - e1000e_write_ext_rx_descr(core, desc, pkt, rss_info, + e1000e_write_ext_rx_descr(core, &desc->extended, pkt, rss_info, (*written)[0]); } } @@ -1366,12 +1385,12 @@ struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, static inline void e1000e_pci_dma_write_rx_desc(E1000ECore *core, dma_addr_t addr, - uint8_t *desc, dma_addr_t len) + union e1000_rx_desc_union *desc, dma_addr_t len) { PCIDevice *dev = core->owner; if (e1000e_rx_use_legacy_descriptor(core)) { - struct e1000_rx_desc *d = (struct e1000_rx_desc *) desc; + struct e1000_rx_desc *d = &desc->legacy; size_t offset = offsetof(struct e1000_rx_desc, status); uint8_t status = d->status; @@ -1384,8 +1403,7 @@ e1000e_pci_dma_write_rx_desc(E1000ECore *core, dma_addr_t addr, } } else { if (core->mac[RCTL] & E1000_RCTL_DTYP_PS) { - union e1000_rx_desc_packet_split *d = - (union e1000_rx_desc_packet_split *) desc; + union e1000_rx_desc_packet_split *d = &desc->packet_split; size_t offset = offsetof(union e1000_rx_desc_packet_split, wb.middle.status_error); uint32_t status = d->wb.middle.status_error; @@ -1398,8 +1416,7 @@ e1000e_pci_dma_write_rx_desc(E1000ECore *core, dma_addr_t addr, pci_dma_write(dev, addr + offset, &status, sizeof(status)); } } else { - union e1000_rx_desc_extended *d = - (union e1000_rx_desc_extended *) desc; + union e1000_rx_desc_extended *d = &desc->extended; size_t offset = offsetof(union e1000_rx_desc_extended, wb.upper.status_error); uint32_t status = d->wb.upper.status_error; @@ -1422,14 +1439,14 @@ typedef struct e1000e_ba_state_st { static inline void e1000e_write_hdr_to_rx_buffers(E1000ECore *core, - hwaddr (*ba)[MAX_PS_BUFFERS], + hwaddr ba[MAX_PS_BUFFERS], e1000e_ba_state *bastate, const char *data, dma_addr_t data_len) { assert(data_len <= core->rxbuf_sizes[0] - bastate->written[0]); - pci_dma_write(core->owner, (*ba)[0] + bastate->written[0], data, data_len); + pci_dma_write(core->owner, ba[0] + bastate->written[0], data, data_len); bastate->written[0] += data_len; bastate->cur_idx = 1; @@ -1437,7 +1454,7 @@ e1000e_write_hdr_to_rx_buffers(E1000ECore *core, static void e1000e_write_to_rx_buffers(E1000ECore *core, - hwaddr (*ba)[MAX_PS_BUFFERS], + hwaddr ba[MAX_PS_BUFFERS], e1000e_ba_state *bastate, const char *data, dma_addr_t data_len) @@ -1449,13 +1466,13 @@ e1000e_write_to_rx_buffers(E1000ECore *core, uint32_t bytes_to_write = MIN(data_len, cur_buf_bytes_left); trace_e1000e_rx_desc_buff_write(bastate->cur_idx, - (*ba)[bastate->cur_idx], + ba[bastate->cur_idx], bastate->written[bastate->cur_idx], data, bytes_to_write); pci_dma_write(core->owner, - (*ba)[bastate->cur_idx] + bastate->written[bastate->cur_idx], + ba[bastate->cur_idx] + bastate->written[bastate->cur_idx], data, bytes_to_write); bastate->written[bastate->cur_idx] += bytes_to_write; @@ -1501,18 +1518,19 @@ e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000E_RingInfo *rxi) static bool e1000e_do_ps(E1000ECore *core, struct NetRxPkt *pkt, size_t *hdr_len) { - bool isip4, isip6, isudp, istcp; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; bool fragment; if (!e1000e_rx_use_ps_descriptor(core)) { return false; } - net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); - if (isip4) { + if (hasip4) { fragment = net_rx_pkt_get_ip4_info(pkt)->fragment; - } else if (isip6) { + } else if (hasip6) { fragment = net_rx_pkt_get_ip6_info(pkt)->fragment; } else { return false; @@ -1522,7 +1540,8 @@ e1000e_do_ps(E1000ECore *core, struct NetRxPkt *pkt, size_t *hdr_len) return false; } - if (!fragment && (isudp || istcp)) { + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP || + l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { *hdr_len = net_rx_pkt_get_l5_hdr_offset(pkt); } else { *hdr_len = net_rx_pkt_get_l4_hdr_offset(pkt); @@ -1543,7 +1562,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, { PCIDevice *d = core->owner; dma_addr_t base; - uint8_t desc[E1000_MAX_RX_DESC_LEN]; + union e1000_rx_desc_union desc; size_t desc_size; size_t desc_offset = 0; size_t iov_ofs = 0; @@ -1579,7 +1598,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len); - e1000e_read_rx_descr(core, desc, &ba); + e1000e_read_rx_descr(core, &desc, ba); if (ba[0]) { if (desc_offset < size) { @@ -1598,7 +1617,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, iov_copy = MIN(ps_hdr_len - ps_hdr_copied, iov->iov_len - iov_ofs); - e1000e_write_hdr_to_rx_buffers(core, &ba, &bastate, + e1000e_write_hdr_to_rx_buffers(core, ba, &bastate, iov->iov_base, iov_copy); copy_size -= iov_copy; @@ -1615,7 +1634,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, } else { /* Leave buffer 0 of each descriptor except first */ /* empty as per spec 7.1.5.1 */ - e1000e_write_hdr_to_rx_buffers(core, &ba, &bastate, + e1000e_write_hdr_to_rx_buffers(core, ba, &bastate, NULL, 0); } } @@ -1624,7 +1643,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, while (copy_size) { iov_copy = MIN(copy_size, iov->iov_len - iov_ofs); - e1000e_write_to_rx_buffers(core, &ba, &bastate, + e1000e_write_to_rx_buffers(core, ba, &bastate, iov->iov_base + iov_ofs, iov_copy); copy_size -= iov_copy; @@ -1637,7 +1656,7 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, if (desc_offset + desc_size >= total_size) { /* Simulate FCS checksum presence in the last descriptor */ - e1000e_write_to_rx_buffers(core, &ba, &bastate, + e1000e_write_to_rx_buffers(core, ba, &bastate, (const char *) &fcs_pad, e1000x_fcs_len(core->mac)); } } @@ -1649,9 +1668,9 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, is_last = true; } - e1000e_write_rx_descr(core, desc, is_last ? core->rx_pkt : NULL, + e1000e_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL, rss_info, do_ps ? ps_hdr_len : 0, &bastate.written); - e1000e_pci_dma_write_rx_desc(core, base, desc, core->rx_desc_len); + e1000e_pci_dma_write_rx_desc(core, base, &desc, core->rx_desc_len); e1000e_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN); @@ -1664,25 +1683,27 @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt, static inline void e1000e_rx_fix_l4_csum(E1000ECore *core, struct NetRxPkt *pkt) { - if (net_rx_pkt_has_virt_hdr(pkt)) { - struct virtio_net_hdr *vhdr = net_rx_pkt_get_vhdr(pkt); + struct virtio_net_hdr *vhdr = net_rx_pkt_get_vhdr(pkt); - if (vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { - net_rx_pkt_fix_l4_csum(pkt); - } + if (vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { + net_rx_pkt_fix_l4_csum(pkt); } } -/* Min. octets in an ethernet frame sans FCS */ -#define MIN_BUF_SIZE 60 - ssize_t e1000e_receive_iov(E1000ECore *core, const struct iovec *iov, int iovcnt) { - static const int maximum_ethernet_hdr_len = (14 + 4); + return e1000e_receive_internal(core, iov, iovcnt, core->has_vnet); +} + +static ssize_t +e1000e_receive_internal(E1000ECore *core, const struct iovec *iov, int iovcnt, + bool has_vnet) +{ + static const int maximum_ethernet_hdr_len = (ETH_HLEN + 4); uint32_t n = 0; - uint8_t min_buf[MIN_BUF_SIZE]; + uint8_t min_buf[ETH_ZLEN]; struct iovec min_iov; uint8_t *filter_buf; size_t size, orig_size; @@ -1700,9 +1721,11 @@ e1000e_receive_iov(E1000ECore *core, const struct iovec *iov, int iovcnt) } /* Pull virtio header in */ - if (core->has_vnet) { + if (has_vnet) { net_rx_pkt_set_vhdr_iovec(core->rx_pkt, iov, iovcnt); iov_ofs = sizeof(struct virtio_net_hdr); + } else { + net_rx_pkt_unset_vhdr(core->rx_pkt); } filter_buf = iov->iov_base + iov_ofs; @@ -1744,8 +1767,6 @@ e1000e_receive_iov(E1000ECore *core, const struct iovec *iov, int iovcnt) e1000e_rss_parse_packet(core, core->rx_pkt, &rss_info); e1000e_rx_ring_init(core, &rxr, rss_info.queue); - trace_e1000e_rx_rss_dispatched_to_queue(rxr.i->idx); - total_size = net_rx_pkt_get_total_len(core->rx_pkt) + e1000x_fcs_len(core->mac); @@ -1771,12 +1792,12 @@ e1000e_receive_iov(E1000ECore *core, const struct iovec *iov, int iovcnt) rdmts_hit = e1000e_rx_descr_threshold_hit(core, rxr.i); n |= e1000e_rx_wb_interrupt_cause(core, rxr.i->idx, rdmts_hit); - trace_e1000e_rx_written_to_guest(n); + trace_e1000e_rx_written_to_guest(rxr.i->idx); } else { n |= E1000_ICS_RXO; retval = 0; - trace_e1000e_rx_not_written_to_guest(n); + trace_e1000e_rx_not_written_to_guest(rxr.i->idx); } if (!e1000e_intrmgr_delay_rx_causes(core, &n)) { @@ -1792,13 +1813,13 @@ e1000e_receive_iov(E1000ECore *core, const struct iovec *iov, int iovcnt) static inline bool e1000e_have_autoneg(E1000ECore *core) { - return core->phy[0][PHY_CTRL] & MII_CR_AUTO_NEG_EN; + return core->phy[0][MII_BMCR] & MII_BMCR_AUTOEN; } static void e1000e_update_flowctl_status(E1000ECore *core) { if (e1000e_have_autoneg(core) && - core->phy[0][PHY_STATUS] & MII_SR_AUTONEG_COMPLETE) { + core->phy[0][MII_BMSR] & MII_BMSR_AN_COMP) { trace_e1000e_link_autoneg_flowctl(true); core->mac[CTRL] |= E1000_CTRL_TFCE | E1000_CTRL_RFCE; } else { @@ -1816,12 +1837,12 @@ e1000e_link_down(E1000ECore *core) static inline void e1000e_set_phy_ctrl(E1000ECore *core, int index, uint16_t val) { - /* bits 0-5 reserved; MII_CR_[RESTART_AUTO_NEG,RESET] are self clearing */ - core->phy[0][PHY_CTRL] = val & ~(0x3f | - MII_CR_RESET | - MII_CR_RESTART_AUTO_NEG); + /* bits 0-5 reserved; MII_BMCR_[ANRESTART,RESET] are self clearing */ + core->phy[0][MII_BMCR] = val & ~(0x3f | + MII_BMCR_RESET | + MII_BMCR_ANRESTART); - if ((val & MII_CR_RESTART_AUTO_NEG) && + if ((val & MII_BMCR_ANRESTART) && e1000e_have_autoneg(core)) { e1000x_restart_autoneg(core->mac, core->phy[0], core->autoneg_timer); } @@ -1855,7 +1876,7 @@ e1000e_core_set_link_status(E1000ECore *core) e1000x_update_regs_on_link_down(core->mac, core->phy[0]); } else { if (e1000e_have_autoneg(core) && - !(core->phy[0][PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) { + !(core->phy[0][MII_BMSR] & MII_BMSR_AN_COMP)) { e1000x_restart_autoneg(core->mac, core->phy[0], core->autoneg_timer); } else { @@ -1888,7 +1909,7 @@ e1000e_set_ctrl(E1000ECore *core, int index, uint32_t val) if (val & E1000_CTRL_RST) { trace_e1000e_core_ctrl_sw_reset(); - e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac); + e1000e_reset(core, true); } if (val & E1000_CTRL_PHY_RST) { @@ -1997,7 +2018,7 @@ static void(*e1000e_phyreg_writeops[E1000E_PHY_PAGES][E1000E_PHY_PAGE_SIZE]) (E1000ECore *, int, uint16_t) = { [0] = { - [PHY_CTRL] = e1000e_set_phy_ctrl, + [MII_BMCR] = e1000e_set_phy_ctrl, [PHY_PAGE] = e1000e_set_phy_page, [PHY_OEM_BITS] = e1000e_set_phy_oem_bits } @@ -2011,13 +2032,11 @@ e1000e_clear_ims_bits(E1000ECore *core, uint32_t bits) } static inline bool -e1000e_postpone_interrupt(bool *interrupt_pending, - E1000IntrDelayTimer *timer) +e1000e_postpone_interrupt(E1000IntrDelayTimer *timer) { if (timer->running) { trace_e1000e_irq_postponed_by_xitr(timer->delay_reg << 2); - *interrupt_pending = true; return true; } @@ -2031,14 +2050,13 @@ e1000e_postpone_interrupt(bool *interrupt_pending, static inline bool e1000e_itr_should_postpone(E1000ECore *core) { - return e1000e_postpone_interrupt(&core->itr_intr_pending, &core->itr); + return e1000e_postpone_interrupt(&core->itr); } static inline bool e1000e_eitr_should_postpone(E1000ECore *core, int idx) { - return e1000e_postpone_interrupt(&core->eitr_intr_pending[idx], - &core->eitr[idx]); + return e1000e_postpone_interrupt(&core->eitr[idx]); } static void @@ -2269,19 +2287,19 @@ e1000e_get_reg_index_with_offset(const uint16_t *mac_reg_access, hwaddr addr) static const char e1000e_phy_regcap[E1000E_PHY_PAGES][0x20] = { [0] = { - [PHY_CTRL] = PHY_ANYPAGE | PHY_RW, - [PHY_STATUS] = PHY_ANYPAGE | PHY_R, - [PHY_ID1] = PHY_ANYPAGE | PHY_R, - [PHY_ID2] = PHY_ANYPAGE | PHY_R, - [PHY_AUTONEG_ADV] = PHY_ANYPAGE | PHY_RW, - [PHY_LP_ABILITY] = PHY_ANYPAGE | PHY_R, - [PHY_AUTONEG_EXP] = PHY_ANYPAGE | PHY_R, - [PHY_NEXT_PAGE_TX] = PHY_ANYPAGE | PHY_RW, - [PHY_LP_NEXT_PAGE] = PHY_ANYPAGE | PHY_R, - [PHY_1000T_CTRL] = PHY_ANYPAGE | PHY_RW, - [PHY_1000T_STATUS] = PHY_ANYPAGE | PHY_R, - [PHY_EXT_STATUS] = PHY_ANYPAGE | PHY_R, - [PHY_PAGE] = PHY_ANYPAGE | PHY_RW, + [MII_BMCR] = PHY_ANYPAGE | PHY_RW, + [MII_BMSR] = PHY_ANYPAGE | PHY_R, + [MII_PHYID1] = PHY_ANYPAGE | PHY_R, + [MII_PHYID2] = PHY_ANYPAGE | PHY_R, + [MII_ANAR] = PHY_ANYPAGE | PHY_RW, + [MII_ANLPAR] = PHY_ANYPAGE | PHY_R, + [MII_ANER] = PHY_ANYPAGE | PHY_R, + [MII_ANNP] = PHY_ANYPAGE | PHY_RW, + [MII_ANLPRNP] = PHY_ANYPAGE | PHY_R, + [MII_CTRL1000] = PHY_ANYPAGE | PHY_RW, + [MII_STAT1000] = PHY_ANYPAGE | PHY_R, + [MII_EXTSTAT] = PHY_ANYPAGE | PHY_R, + [PHY_PAGE] = PHY_ANYPAGE | PHY_RW, [PHY_COPPER_CTRL1] = PHY_RW, [PHY_COPPER_STAT1] = PHY_R, @@ -2434,17 +2452,19 @@ e1000e_set_fcrtl(E1000ECore *core, int index, uint32_t val) core->mac[FCRTL] = val & 0x8000FFF8; } -static inline void -e1000e_set_16bit(E1000ECore *core, int index, uint32_t val) -{ - core->mac[index] = val & 0xffff; -} +#define E1000E_LOW_BITS_SET_FUNC(num) \ + static void \ + e1000e_set_##num##bit(E1000ECore *core, int index, uint32_t val) \ + { \ + core->mac[index] = val & (BIT(num) - 1); \ + } -static void -e1000e_set_12bit(E1000ECore *core, int index, uint32_t val) -{ - core->mac[index] = val & 0xfff; -} +E1000E_LOW_BITS_SET_FUNC(4) +E1000E_LOW_BITS_SET_FUNC(6) +E1000E_LOW_BITS_SET_FUNC(11) +E1000E_LOW_BITS_SET_FUNC(12) +E1000E_LOW_BITS_SET_FUNC(13) +E1000E_LOW_BITS_SET_FUNC(16) static void e1000e_set_vet(E1000ECore *core, int index, uint32_t val) @@ -2515,7 +2535,8 @@ e1000e_set_icr(E1000ECore *core, int index, uint32_t val) } icr = core->mac[ICR] & ~val; - /* Windows driver expects that the "receive overrun" bit and other + /* + * Windows driver expects that the "receive overrun" bit and other * ones to be cleared when the "Other" bit (#24) is cleared. */ icr = (val & E1000_ICR_OTHER) ? (icr & ~E1000_ICR_OTHER_CAUSES) : icr; @@ -2614,27 +2635,11 @@ e1000e_mac_ims_read(E1000ECore *core, int index) return core->mac[IMS]; } -#define E1000E_LOW_BITS_READ_FUNC(num) \ - static uint32_t \ - e1000e_mac_low##num##_read(E1000ECore *core, int index) \ - { \ - return core->mac[index] & (BIT(num) - 1); \ - } \ - -#define E1000E_LOW_BITS_READ(num) \ - e1000e_mac_low##num##_read - -E1000E_LOW_BITS_READ_FUNC(4); -E1000E_LOW_BITS_READ_FUNC(6); -E1000E_LOW_BITS_READ_FUNC(11); -E1000E_LOW_BITS_READ_FUNC(13); -E1000E_LOW_BITS_READ_FUNC(16); - static uint32_t e1000e_mac_swsm_read(E1000ECore *core, int index) { uint32_t val = core->mac[SWSM]; - core->mac[SWSM] = val | 1; + core->mac[SWSM] = val | E1000_SWSM_SMBI; return val; } @@ -2908,6 +2913,35 @@ e1000e_set_gcr(E1000ECore *core, int index, uint32_t val) core->mac[GCR] = (val & ~E1000_GCR_RO_BITS) | ro_bits; } +static uint32_t e1000e_get_systiml(E1000ECore *core, int index) +{ + e1000x_timestamp(core->mac, core->timadj, SYSTIML, SYSTIMH); + return core->mac[SYSTIML]; +} + +static uint32_t e1000e_get_rxsatrh(E1000ECore *core, int index) +{ + core->mac[TSYNCRXCTL] &= ~E1000_TSYNCRXCTL_VALID; + return core->mac[RXSATRH]; +} + +static uint32_t e1000e_get_txstmph(E1000ECore *core, int index) +{ + core->mac[TSYNCTXCTL] &= ~E1000_TSYNCTXCTL_VALID; + return core->mac[TXSTMPH]; +} + +static void e1000e_set_timinca(E1000ECore *core, int index, uint32_t val) +{ + e1000x_set_timinca(core->mac, &core->timadj, val); +} + +static void e1000e_set_timadjh(E1000ECore *core, int index, uint32_t val) +{ + core->mac[TIMADJH] = val; + core->timadj += core->mac[TIMADJL] | ((int64_t)core->mac[TIMADJH] << 32); +} + #define e1000e_getreg(x) [x] = e1000e_mac_readreg typedef uint32_t (*readops)(E1000ECore *, int); static const readops e1000e_macreg_readops[] = { @@ -2923,7 +2957,19 @@ static const readops e1000e_macreg_readops[] = { e1000e_getreg(LATECOL), e1000e_getreg(SEQEC), e1000e_getreg(XONTXC), + e1000e_getreg(AIT), + e1000e_getreg(TDFH), + e1000e_getreg(TDFT), + e1000e_getreg(TDFHS), + e1000e_getreg(TDFTS), + e1000e_getreg(TDFPC), e1000e_getreg(WUS), + e1000e_getreg(PBS), + e1000e_getreg(RDFH), + e1000e_getreg(RDFT), + e1000e_getreg(RDFHS), + e1000e_getreg(RDFTS), + e1000e_getreg(RDFPC), e1000e_getreg(GORCL), e1000e_getreg(MGTPRC), e1000e_getreg(EERD), @@ -2951,7 +2997,6 @@ static const readops e1000e_macreg_readops[] = { e1000e_getreg(GSCL_2), e1000e_getreg(RDBAH1), e1000e_getreg(FLSWDATA), - e1000e_getreg(RXSATRH), e1000e_getreg(TIPG), e1000e_getreg(FLMNGCTL), e1000e_getreg(FLMNGCNT), @@ -2992,7 +3037,6 @@ static const readops e1000e_macreg_readops[] = { e1000e_getreg(FLSWCTL), e1000e_getreg(RXDCTL1), e1000e_getreg(RXSATRL), - e1000e_getreg(SYSTIML), e1000e_getreg(RXUDP), e1000e_getreg(TORL), e1000e_getreg(TDLEN1), @@ -3032,7 +3076,6 @@ static const readops e1000e_macreg_readops[] = { e1000e_getreg(FLOL), e1000e_getreg(RXDCTL), e1000e_getreg(RXSTMPL), - e1000e_getreg(TXSTMPH), e1000e_getreg(TIMADJH), e1000e_getreg(FCRTL), e1000e_getreg(TDBAH), @@ -3059,16 +3102,9 @@ static const readops e1000e_macreg_readops[] = { [MPTC] = e1000e_mac_read_clr4, [IAC] = e1000e_mac_read_clr4, [ICR] = e1000e_mac_icr_read, - [RDFH] = E1000E_LOW_BITS_READ(13), - [RDFHS] = E1000E_LOW_BITS_READ(13), - [RDFPC] = E1000E_LOW_BITS_READ(13), - [TDFH] = E1000E_LOW_BITS_READ(13), - [TDFHS] = E1000E_LOW_BITS_READ(13), [STATUS] = e1000e_get_status, [TARC0] = e1000e_get_tarc, - [PBS] = E1000E_LOW_BITS_READ(6), [ICS] = e1000e_mac_ics_read, - [AIT] = E1000E_LOW_BITS_READ(16), [TORH] = e1000e_mac_read_clr8, [GORCH] = e1000e_mac_read_clr8, [PRC127] = e1000e_mac_read_clr4, @@ -3084,27 +3120,25 @@ static const readops e1000e_macreg_readops[] = { [BPTC] = e1000e_mac_read_clr4, [TSCTC] = e1000e_mac_read_clr4, [ITR] = e1000e_mac_itr_read, - [RDFT] = E1000E_LOW_BITS_READ(13), - [RDFTS] = E1000E_LOW_BITS_READ(13), - [TDFPC] = E1000E_LOW_BITS_READ(13), - [TDFT] = E1000E_LOW_BITS_READ(13), - [TDFTS] = E1000E_LOW_BITS_READ(13), [CTRL] = e1000e_get_ctrl, [TARC1] = e1000e_get_tarc, [SWSM] = e1000e_mac_swsm_read, [IMS] = e1000e_mac_ims_read, + [SYSTIML] = e1000e_get_systiml, + [RXSATRH] = e1000e_get_rxsatrh, + [TXSTMPH] = e1000e_get_txstmph, [CRCERRS ... MPC] = e1000e_mac_readreg, [IP6AT ... IP6AT + 3] = e1000e_mac_readreg, [IP4AT ... IP4AT + 6] = e1000e_mac_readreg, [RA ... RA + 31] = e1000e_mac_readreg, [WUPM ... WUPM + 31] = e1000e_mac_readreg, - [MTA ... MTA + 127] = e1000e_mac_readreg, - [VFTA ... VFTA + 127] = e1000e_mac_readreg, - [FFMT ... FFMT + 254] = E1000E_LOW_BITS_READ(4), + [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = e1000e_mac_readreg, + [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = e1000e_mac_readreg, + [FFMT ... FFMT + 254] = e1000e_mac_readreg, [FFVT ... FFVT + 254] = e1000e_mac_readreg, [MDEF ... MDEF + 7] = e1000e_mac_readreg, - [FFLT ... FFLT + 10] = E1000E_LOW_BITS_READ(11), + [FFLT ... FFLT + 10] = e1000e_mac_readreg, [FTFT ... FTFT + 254] = e1000e_mac_readreg, [PBM ... PBM + 10239] = e1000e_mac_readreg, [RETA ... RETA + 31] = e1000e_mac_readreg, @@ -3127,22 +3161,10 @@ static const writeops e1000e_macreg_writeops[] = { e1000e_putreg(LEDCTL), e1000e_putreg(FCAL), e1000e_putreg(FCRUC), - e1000e_putreg(AIT), - e1000e_putreg(TDFH), - e1000e_putreg(TDFT), - e1000e_putreg(TDFHS), - e1000e_putreg(TDFTS), - e1000e_putreg(TDFPC), e1000e_putreg(WUC), e1000e_putreg(WUS), - e1000e_putreg(RDFH), - e1000e_putreg(RDFT), - e1000e_putreg(RDFHS), - e1000e_putreg(RDFTS), - e1000e_putreg(RDFPC), e1000e_putreg(IPAV), e1000e_putreg(TDBAH1), - e1000e_putreg(TIMINCA), e1000e_putreg(IAM), e1000e_putreg(EIAC), e1000e_putreg(IVAR), @@ -3150,7 +3172,6 @@ static const writeops e1000e_macreg_writeops[] = { e1000e_putreg(TARC1), e1000e_putreg(FLSWDATA), e1000e_putreg(POEMB), - e1000e_putreg(PBS), e1000e_putreg(MFUTP01), e1000e_putreg(MFUTP23), e1000e_putreg(MANC), @@ -3186,7 +3207,6 @@ static const writeops e1000e_macreg_writeops[] = { e1000e_putreg(SYSTIML), e1000e_putreg(SYSTIMH), e1000e_putreg(TIMADJL), - e1000e_putreg(TIMADJH), e1000e_putreg(RXUDP), e1000e_putreg(RXCFGL), e1000e_putreg(TSYNCRXCTL), @@ -3215,6 +3235,18 @@ static const writeops e1000e_macreg_writeops[] = { [TADV] = e1000e_set_16bit, [ITR] = e1000e_set_itr, [EERD] = e1000e_set_eerd, + [AIT] = e1000e_set_16bit, + [TDFH] = e1000e_set_13bit, + [TDFT] = e1000e_set_13bit, + [TDFHS] = e1000e_set_13bit, + [TDFTS] = e1000e_set_13bit, + [TDFPC] = e1000e_set_13bit, + [RDFH] = e1000e_set_13bit, + [RDFHS] = e1000e_set_13bit, + [RDFT] = e1000e_set_13bit, + [RDFTS] = e1000e_set_13bit, + [RDFPC] = e1000e_set_13bit, + [PBS] = e1000e_set_6bit, [GCR] = e1000e_set_gcr, [PSRCTL] = e1000e_set_psrctl, [RXCSUM] = e1000e_set_rxcsum, @@ -3247,18 +3279,20 @@ static const writeops e1000e_macreg_writeops[] = { [CTRL_DUP] = e1000e_set_ctrl, [RFCTL] = e1000e_set_rfctl, [RA + 1] = e1000e_mac_setmacaddr, + [TIMINCA] = e1000e_set_timinca, + [TIMADJH] = e1000e_set_timadjh, [IP6AT ... IP6AT + 3] = e1000e_mac_writereg, [IP4AT ... IP4AT + 6] = e1000e_mac_writereg, [RA + 2 ... RA + 31] = e1000e_mac_writereg, [WUPM ... WUPM + 31] = e1000e_mac_writereg, - [MTA ... MTA + 127] = e1000e_mac_writereg, - [VFTA ... VFTA + 127] = e1000e_mac_writereg, - [FFMT ... FFMT + 254] = e1000e_mac_writereg, + [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = e1000e_mac_writereg, + [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = e1000e_mac_writereg, + [FFMT ... FFMT + 254] = e1000e_set_4bit, [FFVT ... FFVT + 254] = e1000e_mac_writereg, [PBM ... PBM + 10239] = e1000e_mac_writereg, [MDEF ... MDEF + 7] = e1000e_mac_writereg, - [FFLT ... FFLT + 10] = e1000e_mac_writereg, + [FFLT ... FFLT + 10] = e1000e_set_11bit, [FTFT ... FTFT + 254] = e1000e_mac_writereg, [RETA ... RETA + 31] = e1000e_mac_writereg, [RSSRK ... RSSRK + 31] = e1000e_mac_writereg, @@ -3269,10 +3303,12 @@ enum { E1000E_NWRITEOPS = ARRAY_SIZE(e1000e_macreg_writeops) }; enum { MAC_ACCESS_PARTIAL = 1 }; -/* The array below combines alias offsets of the index values for the +/* + * The array below combines alias offsets of the index values for the * MAC registers that have aliases, with the indication of not fully * implemented registers (lowest bit). This combination is possible - * because all of the offsets are even. */ + * because all of the offsets are even. + */ static const uint16_t mac_reg_access[E1000E_MAC_SIZE] = { /* Alias index offsets */ [FCRTL_A] = 0x07fe, [FCRTH_A] = 0x0802, @@ -3281,7 +3317,7 @@ static const uint16_t mac_reg_access[E1000E_MAC_SIZE] = { [TDH_A] = 0x0cf8, [TDT_A] = 0x0cf8, [TIDV_A] = 0x0cf8, [TDFH_A] = 0xed00, [TDFT_A] = 0xed00, [RA_A ... RA_A + 31] = 0x14f0, - [VFTA_A ... VFTA_A + 127] = 0x1400, + [VFTA_A ... VFTA_A + E1000_VLAN_FILTER_TBL_SIZE - 1] = 0x1400, [RDBAL0_A ... RDLEN0_A] = 0x09bc, [TDBAL_A ... TDLEN_A] = 0x0cf8, /* Access options */ @@ -3347,7 +3383,7 @@ static void e1000e_autoneg_resume(E1000ECore *core) { if (e1000e_have_autoneg(core) && - !(core->phy[0][PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) { + !(core->phy[0][MII_BMSR] & MII_BMSR_AN_COMP)) { qemu_get_queue(core->owner_nic)->link_down = false; timer_mod(core->autoneg_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500); @@ -3386,11 +3422,10 @@ e1000e_core_pci_realize(E1000ECore *core, qemu_add_vm_change_state_handler(e1000e_vm_state_change, core); for (i = 0; i < E1000E_NUM_QUEUES; i++) { - net_tx_pkt_init(&core->tx[i].tx_pkt, core->owner, - E1000E_MAX_TX_FRAGS, core->has_vnet); + net_tx_pkt_init(&core->tx[i].tx_pkt, core->owner, E1000E_MAX_TX_FRAGS); } - net_rx_pkt_init(&core->rx_pkt, core->has_vnet); + net_rx_pkt_init(&core->rx_pkt); e1000x_core_prepare_eeprom(core->eeprom, eeprom_templ, @@ -3422,29 +3457,36 @@ e1000e_core_pci_uninit(E1000ECore *core) static const uint16_t e1000e_phy_reg_init[E1000E_PHY_PAGES][E1000E_PHY_PAGE_SIZE] = { [0] = { - [PHY_CTRL] = MII_CR_SPEED_SELECT_MSB | - MII_CR_FULL_DUPLEX | - MII_CR_AUTO_NEG_EN, - - [PHY_STATUS] = MII_SR_EXTENDED_CAPS | - MII_SR_LINK_STATUS | - MII_SR_AUTONEG_CAPS | - MII_SR_PREAMBLE_SUPPRESS | - MII_SR_EXTENDED_STATUS | - MII_SR_10T_HD_CAPS | - MII_SR_10T_FD_CAPS | - MII_SR_100X_HD_CAPS | - MII_SR_100X_FD_CAPS, - - [PHY_ID1] = 0x141, - [PHY_ID2] = E1000_PHY_ID2_82574x, - [PHY_AUTONEG_ADV] = 0xde1, - [PHY_LP_ABILITY] = 0x7e0, - [PHY_AUTONEG_EXP] = BIT(2), - [PHY_NEXT_PAGE_TX] = BIT(0) | BIT(13), - [PHY_1000T_CTRL] = BIT(8) | BIT(9) | BIT(10) | BIT(11), - [PHY_1000T_STATUS] = 0x3c00, - [PHY_EXT_STATUS] = BIT(12) | BIT(13), + [MII_BMCR] = MII_BMCR_SPEED1000 | + MII_BMCR_FD | + MII_BMCR_AUTOEN, + + [MII_BMSR] = MII_BMSR_EXTCAP | + MII_BMSR_LINK_ST | + MII_BMSR_AUTONEG | + MII_BMSR_MFPS | + MII_BMSR_EXTSTAT | + MII_BMSR_10T_HD | + MII_BMSR_10T_FD | + MII_BMSR_100TX_HD | + MII_BMSR_100TX_FD, + + [MII_PHYID1] = 0x141, + [MII_PHYID2] = E1000_PHY_ID2_82574x, + [MII_ANAR] = MII_ANAR_CSMACD | MII_ANAR_10 | + MII_ANAR_10FD | MII_ANAR_TX | + MII_ANAR_TXFD | MII_ANAR_PAUSE | + MII_ANAR_PAUSE_ASYM, + [MII_ANLPAR] = MII_ANLPAR_10 | MII_ANLPAR_10FD | + MII_ANLPAR_TX | MII_ANLPAR_TXFD | + MII_ANLPAR_T4 | MII_ANLPAR_PAUSE, + [MII_ANER] = MII_ANER_NP | MII_ANER_NWAY, + [MII_ANNP] = 1 | MII_ANNP_MP, + [MII_CTRL1000] = MII_CTRL1000_HALF | MII_CTRL1000_FULL | + MII_CTRL1000_PORT | MII_CTRL1000_MASTER, + [MII_STAT1000] = MII_STAT1000_HALF | MII_STAT1000_FULL | + MII_STAT1000_ROK | MII_STAT1000_LOK, + [MII_EXTSTAT] = MII_EXTSTAT_1000T_HD | MII_EXTSTAT_1000T_FD, [PHY_COPPER_CTRL1] = BIT(5) | BIT(6) | BIT(8) | BIT(9) | BIT(12) | BIT(13), @@ -3501,8 +3543,7 @@ static const uint32_t e1000e_mac_reg_init[] = { [EITR...EITR + E1000E_MSIX_VEC_NUM - 1] = E1000E_MIN_XITR, }; -void -e1000e_core_reset(E1000ECore *core) +static void e1000e_reset(E1000ECore *core, bool sw) { int i; @@ -3511,9 +3552,16 @@ e1000e_core_reset(E1000ECore *core) e1000e_intrmgr_reset(core); memset(core->phy, 0, sizeof core->phy); - memmove(core->phy, e1000e_phy_reg_init, sizeof e1000e_phy_reg_init); - memset(core->mac, 0, sizeof core->mac); - memmove(core->mac, e1000e_mac_reg_init, sizeof e1000e_mac_reg_init); + memcpy(core->phy, e1000e_phy_reg_init, sizeof e1000e_phy_reg_init); + + for (i = 0; i < E1000E_MAC_SIZE; i++) { + if (sw && (i == PBA || i == PBS || i == FLA)) { + continue; + } + + core->mac[i] = i < ARRAY_SIZE(e1000e_mac_reg_init) ? + e1000e_mac_reg_init[i] : 0; + } core->rxbuf_min_shift = 1 + E1000_RING_DESC_LEN_SHIFT; @@ -3530,18 +3578,24 @@ e1000e_core_reset(E1000ECore *core) } } +void +e1000e_core_reset(E1000ECore *core) +{ + e1000e_reset(core, false); +} + void e1000e_core_pre_save(E1000ECore *core) { int i; NetClientState *nc = qemu_get_queue(core->owner_nic); /* - * If link is down and auto-negotiation is supported and ongoing, - * complete auto-negotiation immediately. This allows us to look - * at MII_SR_AUTONEG_COMPLETE to infer link status on load. - */ + * If link is down and auto-negotiation is supported and ongoing, + * complete auto-negotiation immediately. This allows us to look + * at MII_BMSR_AN_COMP to infer link status on load. + */ if (nc->link_down && e1000e_have_autoneg(core)) { - core->phy[0][PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE; + core->phy[0][MII_BMSR] |= MII_BMSR_AN_COMP; e1000e_update_flowctl_status(core); } @@ -3557,7 +3611,8 @@ e1000e_core_post_load(E1000ECore *core) { NetClientState *nc = qemu_get_queue(core->owner_nic); - /* nc.link_down can't be migrated, so infer link_down according + /* + * nc.link_down can't be migrated, so infer link_down according * to link status bit in core.mac[STATUS]. */ nc->link_down = (core->mac[STATUS] & E1000_STATUS_LU) == 0; diff --git a/hw/net/e1000e_core.h b/hw/net/e1000e_core.h index 4ddb4d2c39..213a70530d 100644 --- a/hw/net/e1000e_core.h +++ b/hw/net/e1000e_core.h @@ -1,37 +1,37 @@ /* -* Core code for QEMU e1000e emulation -* -* Software developer's manuals: -* http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf -* -* Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) -* Developed by Daynix Computing LTD (http://www.daynix.com) -* -* Authors: -* Dmitry Fleytman <dmitry@daynix.com> -* Leonid Bloch <leonid@daynix.com> -* Yan Vugenfirer <yan@daynix.com> -* -* Based on work done by: -* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. -* Copyright (c) 2008 Qumranet -* Based on work done by: -* Copyright (c) 2007 Dan Aloni -* Copyright (c) 2004 Antony T Curtis -* -* This library is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License as published by the Free Software Foundation; either -* version 2.1 of the License, or (at your option) any later version. -* -* This library is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* Lesser General Public License for more details. -* -* You should have received a copy of the GNU Lesser General Public -* License along with this library; if not, see <http://www.gnu.org/licenses/>. -*/ + * Core code for QEMU e1000e emulation + * + * Software developer's manuals: + * http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf + * + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2008 Qumranet + * Based on work done by: + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ #ifndef HW_NET_E1000E_CORE_H #define HW_NET_E1000E_CORE_H @@ -95,10 +95,8 @@ struct E1000Core { E1000IntrDelayTimer tidv; E1000IntrDelayTimer itr; - bool itr_intr_pending; E1000IntrDelayTimer eitr[E1000E_MSIX_VEC_NUM]; - bool eitr_intr_pending[E1000E_MSIX_VEC_NUM]; VMChangeStateEntry *vmstate; @@ -114,6 +112,8 @@ struct E1000Core { void (*owner_start_recv)(PCIDevice *d); uint32_t msi_causes_pending; + + int64_t timadj; }; void diff --git a/hw/net/e1000x_common.c b/hw/net/e1000x_common.c index 2f43e8cd13..b844af590a 100644 --- a/hw/net/e1000x_common.c +++ b/hw/net/e1000x_common.c @@ -24,9 +24,12 @@ #include "qemu/osdep.h" #include "qemu/units.h" +#include "hw/net/mii.h" #include "hw/pci/pci_device.h" +#include "net/eth.h" #include "net/net.h" +#include "e1000_common.h" #include "e1000x_common.h" #include "trace.h" @@ -45,9 +48,9 @@ bool e1000x_rx_ready(PCIDevice *d, uint32_t *mac) return true; } -bool e1000x_is_vlan_packet(const uint8_t *buf, uint16_t vet) +bool e1000x_is_vlan_packet(const void *buf, uint16_t vet) { - uint16_t eth_proto = lduw_be_p(buf + 12); + uint16_t eth_proto = lduw_be_p(&PKT_GET_ETH_HDR(buf)->h_proto); bool res = (eth_proto == vet); trace_e1000x_vlan_is_vlan_pkt(res, eth_proto, vet); @@ -66,7 +69,7 @@ bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf) } ra[0] = cpu_to_le32(rp[0]); ra[1] = cpu_to_le32(rp[1]); - if (!memcmp(buf, (uint8_t *)ra, 6)) { + if (!memcmp(buf, (uint8_t *)ra, ETH_ALEN)) { trace_e1000x_rx_flt_ucast_match((int)(rp - mac - RA) / 2, MAC_ARG(buf)); return true; @@ -152,8 +155,8 @@ void e1000x_reset_mac_addr(NICState *nic, uint32_t *mac_regs, void e1000x_update_regs_on_autoneg_done(uint32_t *mac, uint16_t *phy) { e1000x_update_regs_on_link_up(mac, phy); - phy[PHY_LP_ABILITY] |= MII_LPAR_LPACK; - phy[PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE; + phy[MII_ANLPAR] |= MII_ANLPAR_ACK; + phy[MII_BMSR] |= MII_BMSR_AN_COMP; trace_e1000x_link_negotiation_done(); } @@ -265,3 +268,28 @@ e1000x_read_tx_ctx_descr(struct e1000_context_desc *d, props->tcp = (op & E1000_TXD_CMD_TCP) ? 1 : 0; props->tse = (op & E1000_TXD_CMD_TSE) ? 1 : 0; } + +void e1000x_timestamp(uint32_t *mac, int64_t timadj, size_t lo, size_t hi) +{ + int64_t ns = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + uint32_t timinca = mac[TIMINCA]; + uint32_t incvalue = timinca & E1000_TIMINCA_INCVALUE_MASK; + uint32_t incperiod = MAX(timinca >> E1000_TIMINCA_INCPERIOD_SHIFT, 1); + int64_t timestamp = timadj + muldiv64(ns, incvalue, incperiod * 16); + + mac[lo] = timestamp & 0xffffffff; + mac[hi] = timestamp >> 32; +} + +void e1000x_set_timinca(uint32_t *mac, int64_t *timadj, uint32_t val) +{ + int64_t ns = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + uint32_t old_val = mac[TIMINCA]; + uint32_t old_incvalue = old_val & E1000_TIMINCA_INCVALUE_MASK; + uint32_t old_incperiod = MAX(old_val >> E1000_TIMINCA_INCPERIOD_SHIFT, 1); + uint32_t incvalue = val & E1000_TIMINCA_INCVALUE_MASK; + uint32_t incperiod = MAX(val >> E1000_TIMINCA_INCPERIOD_SHIFT, 1); + + mac[TIMINCA] = val; + *timadj += (muldiv64(ns, incvalue, incperiod) - muldiv64(ns, old_incvalue, old_incperiod)) / 16; +} diff --git a/hw/net/e1000x_common.h b/hw/net/e1000x_common.h index b7742775c4..911abd8a90 100644 --- a/hw/net/e1000x_common.h +++ b/hw/net/e1000x_common.h @@ -1,108 +1,34 @@ /* -* QEMU e1000(e) emulation - shared code -* -* Copyright (c) 2008 Qumranet -* -* Based on work done by: -* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. -* Copyright (c) 2007 Dan Aloni -* Copyright (c) 2004 Antony T Curtis -* -* This library is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License as published by the Free Software Foundation; either -* version 2.1 of the License, or (at your option) any later version. -* -* This library is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* Lesser General Public License for more details. -* -* You should have received a copy of the GNU Lesser General Public -* License along with this library; if not, see <http://www.gnu.org/licenses/>. -*/ + * QEMU e1000(e) emulation - shared code + * + * Copyright (c) 2008 Qumranet + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ #ifndef HW_NET_E1000X_COMMON_H #define HW_NET_E1000X_COMMON_H -#include "e1000_regs.h" - -#define defreg(x) x = (E1000_##x >> 2) -enum { - defreg(CTRL), defreg(EECD), defreg(EERD), defreg(GPRC), - defreg(GPTC), defreg(ICR), defreg(ICS), defreg(IMC), - defreg(IMS), defreg(LEDCTL), defreg(MANC), defreg(MDIC), - defreg(MPC), defreg(PBA), defreg(RCTL), defreg(RDBAH0), - defreg(RDBAL0), defreg(RDH0), defreg(RDLEN0), defreg(RDT0), - defreg(STATUS), defreg(SWSM), defreg(TCTL), defreg(TDBAH), - defreg(TDBAL), defreg(TDH), defreg(TDLEN), defreg(TDT), - defreg(TDLEN1), defreg(TDBAL1), defreg(TDBAH1), defreg(TDH1), - defreg(TDT1), defreg(TORH), defreg(TORL), defreg(TOTH), - defreg(TOTL), defreg(TPR), defreg(TPT), defreg(TXDCTL), - defreg(WUFC), defreg(RA), defreg(MTA), defreg(CRCERRS), - defreg(VFTA), defreg(VET), defreg(RDTR), defreg(RADV), - defreg(TADV), defreg(ITR), defreg(SCC), defreg(ECOL), - defreg(MCC), defreg(LATECOL), defreg(COLC), defreg(DC), - defreg(TNCRS), defreg(SEQEC), defreg(CEXTERR), defreg(RLEC), - defreg(XONRXC), defreg(XONTXC), defreg(XOFFRXC), defreg(XOFFTXC), - defreg(FCRUC), defreg(AIT), defreg(TDFH), defreg(TDFT), - defreg(TDFHS), defreg(TDFTS), defreg(TDFPC), defreg(WUC), - defreg(WUS), defreg(POEMB), defreg(PBS), defreg(RDFH), - defreg(RDFT), defreg(RDFHS), defreg(RDFTS), defreg(RDFPC), - defreg(PBM), defreg(IPAV), defreg(IP4AT), defreg(IP6AT), - defreg(WUPM), defreg(FFLT), defreg(FFMT), defreg(FFVT), - defreg(TARC0), defreg(TARC1), defreg(IAM), defreg(EXTCNF_CTRL), - defreg(GCR), defreg(TIMINCA), defreg(EIAC), defreg(CTRL_EXT), - defreg(IVAR), defreg(MFUTP01), defreg(MFUTP23), defreg(MANC2H), - defreg(MFVAL), defreg(MDEF), defreg(FACTPS), defreg(FTFT), - defreg(RUC), defreg(ROC), defreg(RFC), defreg(RJC), - defreg(PRC64), defreg(PRC127), defreg(PRC255), defreg(PRC511), - defreg(PRC1023), defreg(PRC1522), defreg(PTC64), defreg(PTC127), - defreg(PTC255), defreg(PTC511), defreg(PTC1023), defreg(PTC1522), - defreg(GORCL), defreg(GORCH), defreg(GOTCL), defreg(GOTCH), - defreg(RNBC), defreg(BPRC), defreg(MPRC), defreg(RFCTL), - defreg(PSRCTL), defreg(MPTC), defreg(BPTC), defreg(TSCTFC), - defreg(IAC), defreg(MGTPRC), defreg(MGTPDC), defreg(MGTPTC), - defreg(TSCTC), defreg(RXCSUM), defreg(FUNCTAG), defreg(GSCL_1), - defreg(GSCL_2), defreg(GSCL_3), defreg(GSCL_4), defreg(GSCN_0), - defreg(GSCN_1), defreg(GSCN_2), defreg(GSCN_3), defreg(GCR2), - defreg(RAID), defreg(RSRPD), defreg(TIDV), defreg(EITR), - defreg(MRQC), defreg(RETA), defreg(RSSRK), defreg(RDBAH1), - defreg(RDBAL1), defreg(RDLEN1), defreg(RDH1), defreg(RDT1), - defreg(PBACLR), defreg(FCAL), defreg(FCAH), defreg(FCT), - defreg(FCRTH), defreg(FCRTL), defreg(FCTTV), defreg(FCRTV), - defreg(FLA), defreg(EEWR), defreg(FLOP), defreg(FLOL), - defreg(FLSWCTL), defreg(FLSWCNT), defreg(RXDCTL), defreg(RXDCTL1), - defreg(MAVTV0), defreg(MAVTV1), defreg(MAVTV2), defreg(MAVTV3), - defreg(TXSTMPL), defreg(TXSTMPH), defreg(SYSTIML), defreg(SYSTIMH), - defreg(RXCFGL), defreg(RXUDP), defreg(TIMADJL), defreg(TIMADJH), - defreg(RXSTMPH), defreg(RXSTMPL), defreg(RXSATRL), defreg(RXSATRH), - defreg(FLASHT), defreg(TIPG), defreg(RDH), defreg(RDT), - defreg(RDLEN), defreg(RDBAH), defreg(RDBAL), - defreg(TXDCTL1), - defreg(FLSWDATA), - defreg(CTRL_DUP), - defreg(EXTCNF_SIZE), - defreg(EEMNGCTL), - defreg(EEMNGDATA), - defreg(FLMNGCTL), - defreg(FLMNGDATA), - defreg(FLMNGCNT), - defreg(TSYNCRXCTL), - defreg(TSYNCTXCTL), - - /* Aliases */ - defreg(RDH0_A), defreg(RDT0_A), defreg(RDTR_A), defreg(RDFH_A), - defreg(RDFT_A), defreg(TDH_A), defreg(TDT_A), defreg(TIDV_A), - defreg(TDFH_A), defreg(TDFT_A), defreg(RA_A), defreg(RDBAL0_A), - defreg(TDBAL_A), defreg(TDLEN_A), defreg(VFTA_A), defreg(RDLEN0_A), - defreg(FCRTL_A), defreg(FCRTH_A) -}; - static inline void e1000x_inc_reg_if_not_full(uint32_t *mac, int index) { - if (mac[index] != 0xffffffff) { + if (mac[index] != UINT32_MAX) { mac[index]++; } } @@ -152,16 +78,16 @@ static inline void e1000x_update_regs_on_link_down(uint32_t *mac, uint16_t *phy) { mac[STATUS] &= ~E1000_STATUS_LU; - phy[PHY_STATUS] &= ~MII_SR_LINK_STATUS; - phy[PHY_STATUS] &= ~MII_SR_AUTONEG_COMPLETE; - phy[PHY_LP_ABILITY] &= ~MII_LPAR_LPACK; + phy[MII_BMSR] &= ~MII_BMSR_LINK_ST; + phy[MII_BMSR] &= ~MII_BMSR_AN_COMP; + phy[MII_ANLPAR] &= ~MII_ANLPAR_ACK; } static inline void e1000x_update_regs_on_link_up(uint32_t *mac, uint16_t *phy) { mac[STATUS] |= E1000_STATUS_LU; - phy[PHY_STATUS] |= MII_SR_LINK_STATUS; + phy[MII_BMSR] |= MII_BMSR_LINK_ST; } void e1000x_update_rx_total_stats(uint32_t *mac, @@ -178,7 +104,7 @@ uint32_t e1000x_rxbufsize(uint32_t rctl); bool e1000x_rx_ready(PCIDevice *d, uint32_t *mac); -bool e1000x_is_vlan_packet(const uint8_t *buf, uint16_t vet); +bool e1000x_is_vlan_packet(const void *buf, uint16_t vet); bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf); @@ -213,4 +139,7 @@ typedef struct e1000x_txd_props { void e1000x_read_tx_ctx_descr(struct e1000_context_desc *d, e1000x_txd_props *props); +void e1000x_timestamp(uint32_t *mac, int64_t timadj, size_t lo, size_t hi); +void e1000x_set_timinca(uint32_t *mac, int64_t *timadj, uint32_t val); + #endif diff --git a/hw/net/e1000x_regs.h b/hw/net/e1000x_regs.h new file mode 100644 index 0000000000..c0832fa23d --- /dev/null +++ b/hw/net/e1000x_regs.h @@ -0,0 +1,967 @@ +/******************************************************************************* + + Intel PRO/1000 Linux driver + Copyright(c) 1999 - 2006 Intel Corporation. + + This program is free software; you can redistribute it and/or modify it + under the terms and conditions of the GNU General Public License, + version 2, as published by the Free Software Foundation. + + This program is distributed in the hope it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program; if not, see <http://www.gnu.org/licenses/>. + + The full GNU General Public License is included in this distribution in + the file called "COPYING". + + Contact Information: + Linux NICS <linux.nics@intel.com> + e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> + Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 + +*******************************************************************************/ + +/* e1000_hw.h + * Structures, enums, and macros for the MAC + */ + +#ifndef HW_E1000X_REGS_H +#define HW_E1000X_REGS_H + +/* PCI Device IDs */ +#define E1000_DEV_ID_82542 0x1000 +#define E1000_DEV_ID_82543GC_FIBER 0x1001 +#define E1000_DEV_ID_82543GC_COPPER 0x1004 +#define E1000_DEV_ID_82544EI_COPPER 0x1008 +#define E1000_DEV_ID_82544EI_FIBER 0x1009 +#define E1000_DEV_ID_82544GC_COPPER 0x100C +#define E1000_DEV_ID_82544GC_LOM 0x100D +#define E1000_DEV_ID_82540EM 0x100E +#define E1000_DEV_ID_82540EM_LOM 0x1015 +#define E1000_DEV_ID_82540EP_LOM 0x1016 +#define E1000_DEV_ID_82540EP 0x1017 +#define E1000_DEV_ID_82540EP_LP 0x101E +#define E1000_DEV_ID_82545EM_COPPER 0x100F +#define E1000_DEV_ID_82545EM_FIBER 0x1011 +#define E1000_DEV_ID_82545GM_COPPER 0x1026 +#define E1000_DEV_ID_82545GM_FIBER 0x1027 +#define E1000_DEV_ID_82545GM_SERDES 0x1028 +#define E1000_DEV_ID_82546EB_COPPER 0x1010 +#define E1000_DEV_ID_82546EB_FIBER 0x1012 +#define E1000_DEV_ID_82546EB_QUAD_COPPER 0x101D +#define E1000_DEV_ID_82541EI 0x1013 +#define E1000_DEV_ID_82541EI_MOBILE 0x1018 +#define E1000_DEV_ID_82541ER_LOM 0x1014 +#define E1000_DEV_ID_82541ER 0x1078 +#define E1000_DEV_ID_82547GI 0x1075 +#define E1000_DEV_ID_82541GI 0x1076 +#define E1000_DEV_ID_82541GI_MOBILE 0x1077 +#define E1000_DEV_ID_82541GI_LF 0x107C +#define E1000_DEV_ID_82546GB_COPPER 0x1079 +#define E1000_DEV_ID_82546GB_FIBER 0x107A +#define E1000_DEV_ID_82546GB_SERDES 0x107B +#define E1000_DEV_ID_82546GB_PCIE 0x108A +#define E1000_DEV_ID_82546GB_QUAD_COPPER 0x1099 +#define E1000_DEV_ID_82547EI 0x1019 +#define E1000_DEV_ID_82547EI_MOBILE 0x101A +#define E1000_DEV_ID_82571EB_COPPER 0x105E +#define E1000_DEV_ID_82571EB_FIBER 0x105F +#define E1000_DEV_ID_82571EB_SERDES 0x1060 +#define E1000_DEV_ID_82571EB_QUAD_COPPER 0x10A4 +#define E1000_DEV_ID_82571PT_QUAD_COPPER 0x10D5 +#define E1000_DEV_ID_82571EB_QUAD_FIBER 0x10A5 +#define E1000_DEV_ID_82571EB_QUAD_COPPER_LOWPROFILE 0x10BC +#define E1000_DEV_ID_82571EB_SERDES_DUAL 0x10D9 +#define E1000_DEV_ID_82571EB_SERDES_QUAD 0x10DA +#define E1000_DEV_ID_82572EI_COPPER 0x107D +#define E1000_DEV_ID_82572EI_FIBER 0x107E +#define E1000_DEV_ID_82572EI_SERDES 0x107F +#define E1000_DEV_ID_82572EI 0x10B9 +#define E1000_DEV_ID_82573E 0x108B +#define E1000_DEV_ID_82573E_IAMT 0x108C +#define E1000_DEV_ID_82573L 0x109A +#define E1000_DEV_ID_82574L 0x10D3 +#define E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3 0x10B5 +#define E1000_DEV_ID_80003ES2LAN_COPPER_DPT 0x1096 +#define E1000_DEV_ID_80003ES2LAN_SERDES_DPT 0x1098 +#define E1000_DEV_ID_80003ES2LAN_COPPER_SPT 0x10BA +#define E1000_DEV_ID_80003ES2LAN_SERDES_SPT 0x10BB + +#define E1000_DEV_ID_ICH8_IGP_M_AMT 0x1049 +#define E1000_DEV_ID_ICH8_IGP_AMT 0x104A +#define E1000_DEV_ID_ICH8_IGP_C 0x104B +#define E1000_DEV_ID_ICH8_IFE 0x104C +#define E1000_DEV_ID_ICH8_IFE_GT 0x10C4 +#define E1000_DEV_ID_ICH8_IFE_G 0x10C5 +#define E1000_DEV_ID_ICH8_IGP_M 0x104D + +/* Device Specific Register Defaults */ +#define E1000_PHY_ID2_82541x 0x380 +#define E1000_PHY_ID2_82544x 0xC30 +#define E1000_PHY_ID2_8254xx_DEFAULT 0xC20 /* 82540x, 82545x, and 82546x */ +#define E1000_PHY_ID2_82573x 0xCC0 +#define E1000_PHY_ID2_82574x 0xCB1 + +/* Register Set. (82543, 82544) + * + * Registers are defined to be 32 bits and should be accessed as 32 bit values. + * These registers are physically located on the NIC, but are mapped into the + * host memory address space. + * + * RW - register is both readable and writable + * RO - register is read only + * WO - register is write only + * R/clr - register is read only and is cleared when read + * A - register array + */ +#define E1000_CTRL 0x00000 /* Device Control - RW */ +#define E1000_CTRL_DUP 0x00004 /* Device Control Duplicate (Shadow) - RW */ +#define E1000_STATUS 0x00008 /* Device Status - RO */ +#define E1000_EECD 0x00010 /* EEPROM/Flash Control - RW */ +#define E1000_EERD 0x00014 /* EEPROM Read - RW */ +#define E1000_CTRL_EXT 0x00018 /* Extended Device Control - RW */ +#define E1000_FLA 0x0001C /* Flash Access - RW */ +#define E1000_MDIC 0x00020 /* MDI Control - RW */ +#define E1000_SCTL 0x00024 /* SerDes Control - RW */ +#define E1000_FCAL 0x00028 /* Flow Control Address Low - RW */ +#define E1000_FCAH 0x0002C /* Flow Control Address High -RW */ +#define E1000_FCT 0x00030 /* Flow Control Type - RW */ +#define E1000_VET 0x00038 /* VLAN Ether Type - RW */ +#define E1000_ICR 0x000C0 /* Interrupt Cause Read - R/clr */ +#define E1000_ICS 0x000C8 /* Interrupt Cause Set - WO */ +#define E1000_IMS 0x000D0 /* Interrupt Mask Set - RW */ +#define E1000_IMC 0x000D8 /* Interrupt Mask Clear - WO */ +#define E1000_IAM 0x000E0 /* Interrupt Acknowledge Auto Mask */ +#define E1000_RCTL 0x00100 /* RX Control - RW */ +#define E1000_FCTTV 0x00170 /* Flow Control Transmit Timer Value - RW */ +#define E1000_TCTL 0x00400 /* TX Control - RW */ +#define E1000_TCTL_EXT 0x00404 /* Extended TX Control - RW */ +#define E1000_TIPG 0x00410 /* TX Inter-packet gap -RW */ +#define E1000_LEDCTL 0x00E00 /* LED Control - RW */ +#define E1000_EEMNGCTL 0x01010 /* MNG EEprom Control */ +#define E1000_EEMNGDATA 0x01014 /* MNG EEPROM Read/Write data */ +#define E1000_FLMNGCTL 0x01018 /* MNG Flash Control */ +#define E1000_FLMNGDATA 0x0101C /* MNG FLASH Read data */ +#define E1000_FLMNGCNT 0x01020 /* MNG FLASH Read Counter */ +#define E1000_EEARBC 0x01024 /* EEPROM Auto Read Bus Control */ +#define E1000_FLOP 0x0103C /* FLASH Opcode Register */ +#define E1000_FCRTL 0x02160 /* Flow Control Receive Threshold Low - RW */ +#define E1000_FCRTL_A 0x00168 /* Alias to FCRTL */ +#define E1000_FCRTH 0x02168 /* Flow Control Receive Threshold High - RW */ +#define E1000_RDFH 0x02410 /* Receive Data FIFO Head Register - RW */ +#define E1000_RDFH_A 0x08000 /* Alias to RDFH */ +#define E1000_RDFT 0x02418 /* Receive Data FIFO Tail Register - RW */ +#define E1000_RDFT_A 0x08008 /* Alias to RDFT */ +#define E1000_RDFHS 0x02420 /* Receive Data FIFO Head Saved Register - RW */ +#define E1000_RDFTS 0x02428 /* Receive Data FIFO Tail Saved Register - RW */ +#define E1000_RDFPC 0x02430 /* Receive Data FIFO Packet Count - RW */ +#define E1000_TDFH 0x03410 /* TX Data FIFO Head - RW */ +#define E1000_TDFH_A 0x08010 /* Alias to TDFH */ +#define E1000_TDFT 0x03418 /* TX Data FIFO Tail - RW */ +#define E1000_TDFT_A 0x08018 /* Alias to TDFT */ +#define E1000_TDFHS 0x03420 /* TX Data FIFO Head Saved - RW */ +#define E1000_TDFTS 0x03428 /* TX Data FIFO Tail Saved - RW */ +#define E1000_TDFPC 0x03430 /* TX Data FIFO Packet Count - RW */ +#define E1000_CRCERRS 0x04000 /* CRC Error Count - R/clr */ +#define E1000_ALGNERRC 0x04004 /* Alignment Error Count - R/clr */ +#define E1000_SYMERRS 0x04008 /* Symbol Error Count - R/clr */ +#define E1000_RXERRC 0x0400C /* Receive Error Count - R/clr */ +#define E1000_MPC 0x04010 /* Missed Packet Count - R/clr */ +#define E1000_SCC 0x04014 /* Single Collision Count - R/clr */ +#define E1000_ECOL 0x04018 /* Excessive Collision Count - R/clr */ +#define E1000_MCC 0x0401C /* Multiple Collision Count - R/clr */ +#define E1000_LATECOL 0x04020 /* Late Collision Count - R/clr */ +#define E1000_COLC 0x04028 /* Collision Count - R/clr */ +#define E1000_DC 0x04030 /* Defer Count - R/clr */ +#define E1000_TNCRS 0x04034 /* TX-No CRS - R/clr */ +#define E1000_RLEC 0x04040 /* Receive Length Error Count - R/clr */ +#define E1000_XONRXC 0x04048 /* XON RX Count - R/clr */ +#define E1000_XONTXC 0x0404C /* XON TX Count - R/clr */ +#define E1000_XOFFRXC 0x04050 /* XOFF RX Count - R/clr */ +#define E1000_XOFFTXC 0x04054 /* XOFF TX Count - R/clr */ +#define E1000_FCRUC 0x04058 /* Flow Control RX Unsupported Count- R/clr */ +#define E1000_PRC64 0x0405C /* Packets RX (64 bytes) - R/clr */ +#define E1000_PRC127 0x04060 /* Packets RX (65-127 bytes) - R/clr */ +#define E1000_PRC255 0x04064 /* Packets RX (128-255 bytes) - R/clr */ +#define E1000_PRC511 0x04068 /* Packets RX (255-511 bytes) - R/clr */ +#define E1000_PRC1023 0x0406C /* Packets RX (512-1023 bytes) - R/clr */ +#define E1000_PRC1522 0x04070 /* Packets RX (1024-1522 bytes) - R/clr */ +#define E1000_GPRC 0x04074 /* Good Packets RX Count - R/clr */ +#define E1000_BPRC 0x04078 /* Broadcast Packets RX Count - R/clr */ +#define E1000_MPRC 0x0407C /* Multicast Packets RX Count - R/clr */ +#define E1000_GPTC 0x04080 /* Good Packets TX Count - R/clr */ +#define E1000_GORCL 0x04088 /* Good Octets RX Count Low - R/clr */ +#define E1000_GORCH 0x0408C /* Good Octets RX Count High - R/clr */ +#define E1000_GOTCL 0x04090 /* Good Octets TX Count Low - R/clr */ +#define E1000_GOTCH 0x04094 /* Good Octets TX Count High - R/clr */ +#define E1000_RNBC 0x040A0 /* RX No Buffers Count - R/clr */ +#define E1000_RUC 0x040A4 /* RX Undersize Count - R/clr */ +#define E1000_RFC 0x040A8 /* RX Fragment Count - R/clr */ +#define E1000_ROC 0x040AC /* RX Oversize Count - R/clr */ +#define E1000_RJC 0x040B0 /* RX Jabber Count - R/clr */ +#define E1000_MGTPRC 0x040B4 /* Management Packets RX Count - R/clr */ +#define E1000_MGTPDC 0x040B8 /* Management Packets Dropped Count - R/clr */ +#define E1000_MGTPTC 0x040BC /* Management Packets TX Count - R/clr */ +#define E1000_TORL 0x040C0 /* Total Octets RX Low - R/clr */ +#define E1000_TORH 0x040C4 /* Total Octets RX High - R/clr */ +#define E1000_TOTL 0x040C8 /* Total Octets TX Low - R/clr */ +#define E1000_TOTH 0x040CC /* Total Octets TX High - R/clr */ +#define E1000_TPR 0x040D0 /* Total Packets RX - R/clr */ +#define E1000_TPT 0x040D4 /* Total Packets TX - R/clr */ +#define E1000_PTC64 0x040D8 /* Packets TX (64 bytes) - R/clr */ +#define E1000_PTC127 0x040DC /* Packets TX (65-127 bytes) - R/clr */ +#define E1000_PTC255 0x040E0 /* Packets TX (128-255 bytes) - R/clr */ +#define E1000_PTC511 0x040E4 /* Packets TX (256-511 bytes) - R/clr */ +#define E1000_PTC1023 0x040E8 /* Packets TX (512-1023 bytes) - R/clr */ +#define E1000_PTC1522 0x040EC /* Packets TX (1024-1522 Bytes) - R/clr */ +#define E1000_MPTC 0x040F0 /* Multicast Packets TX Count - R/clr */ +#define E1000_BPTC 0x040F4 /* Broadcast Packets TX Count - R/clr */ +#define E1000_TSCTC 0x040F8 /* TCP Segmentation Context TX - R/clr */ +#define E1000_IAC 0x04100 /* Interrupt Assertion Count */ +#define E1000_ICRXPTC 0x04104 /* Interrupt Cause Rx Packet Timer Expire Count */ +#define E1000_ICRXDMTC 0x04120 /* Interrupt Cause Rx Descriptor Minimum Threshold Count */ +#define E1000_RXCSUM 0x05000 /* RX Checksum Control - RW */ +#define E1000_RFCTL 0x05008 /* Receive Filter Control*/ +#define E1000_MAVTV0 0x05010 /* Management VLAN TAG Value 0 */ +#define E1000_MAVTV1 0x05014 /* Management VLAN TAG Value 1 */ +#define E1000_MAVTV2 0x05018 /* Management VLAN TAG Value 2 */ +#define E1000_MAVTV3 0x0501c /* Management VLAN TAG Value 3 */ +#define E1000_MTA 0x05200 /* Multicast Table Array - RW Array */ +#define E1000_RA 0x05400 /* Receive Address - RW Array */ +#define E1000_RA_A 0x00040 /* Alias to RA */ +#define E1000_VFTA 0x05600 /* VLAN Filter Table Array - RW Array */ +#define E1000_VFTA_A 0x00600 /* Alias to VFTA */ +#define E1000_WUC 0x05800 /* Wakeup Control - RW */ +#define E1000_WUFC 0x05808 /* Wakeup Filter Control - RW */ +#define E1000_WUS 0x05810 /* Wakeup Status - RO */ +#define E1000_MANC 0x05820 /* Management Control - RW */ +#define E1000_IPAV 0x05838 /* IP Address Valid - RW */ +#define E1000_IP4AT 0x05840 /* IPv4 Address Table - RW Array */ +#define E1000_IP6AT 0x05880 /* IPv6 Address Table - RW Array */ +#define E1000_WUPL 0x05900 /* Wakeup Packet Length - RW */ +#define E1000_WUPM 0x05A00 /* Wakeup Packet Memory - RO A */ +#define E1000_MFVAL 0x05824 /* Manageability Filters Valid - RW */ +#define E1000_MDEF 0x05890 /* Manageability Decision Filters - RW Array */ +#define E1000_FFMT 0x09000 /* Flexible Filter Mask Table - RW Array */ +#define E1000_FTFT 0x09400 /* Flexible TCO Filter Table - RW Array */ + +#define E1000_MANC2H 0x05860 /* Management Control To Host - RW */ +#define E1000_SW_FW_SYNC 0x05B5C /* Software-Firmware Synchronization - RW */ + +#define E1000_GCR 0x05B00 /* PCI-Ex Control */ +#define E1000_FUNCTAG 0x05B08 /* Function-Tag Register */ +#define E1000_GSCL_1 0x05B10 /* PCI-Ex Statistic Control #1 */ +#define E1000_GSCL_2 0x05B14 /* PCI-Ex Statistic Control #2 */ +#define E1000_GSCL_3 0x05B18 /* PCI-Ex Statistic Control #3 */ +#define E1000_GSCL_4 0x05B1C /* PCI-Ex Statistic Control #4 */ +#define E1000_GSCN_0 0x05B20 /* 3GIO Statistic Counter Register #0 */ +#define E1000_GSCN_1 0x05B24 /* 3GIO Statistic Counter Register #1 */ +#define E1000_GSCN_2 0x05B28 /* 3GIO Statistic Counter Register #2 */ +#define E1000_GSCN_3 0x05B2C /* 3GIO Statistic Counter Register #3 */ +#define E1000_FACTPS 0x05B30 /* Function Active and Power State to MNG */ +#define E1000_SWSM 0x05B50 /* SW Semaphore */ +#define E1000_FWSM 0x05B54 /* FW Semaphore */ +#define E1000_PBACLR 0x05B68 /* MSI-X PBA Clear */ + +#define E1000_TSYNCRXCTL 0x0B620 /* Rx Time Sync Control register - RW */ +#define E1000_TSYNCTXCTL 0x0B614 /* Tx Time Sync Control register - RW */ +#define E1000_TIMINCA 0x0B608 /* Increment attributes register - RW */ +#define E1000_RXSTMPL 0x0B624 /* Rx timestamp Low - RO */ +#define E1000_RXSTMPH 0x0B628 /* Rx timestamp High - RO */ +#define E1000_TXSTMPL 0x0B618 /* Tx timestamp value Low - RO */ +#define E1000_TXSTMPH 0x0B61C /* Tx timestamp value High - RO */ +#define E1000_SYSTIML 0x0B600 /* System time register Low - RO */ +#define E1000_SYSTIMH 0x0B604 /* System time register High - RO */ +#define E1000_TIMINCA 0x0B608 /* Increment attributes register - RW */ +#define E1000_RXSATRL 0x0B62C /* Rx timestamp attribute low - RO */ +#define E1000_RXSATRH 0x0B630 /* Rx timestamp attribute high - RO */ +#define E1000_TIMADJL 0x0B60C /* Time Adjustment Offset register Low - RW */ +#define E1000_TIMADJH 0x0B610 /* Time Adjustment Offset register High - RW */ + +/* RSS registers */ +#define E1000_MRQC 0x05818 /* Multiple Receive Control - RW */ +#define E1000_RETA 0x05C00 /* Redirection Table - RW Array */ +#define E1000_RSSRK 0x05C80 /* RSS Random Key - RW Array */ + +#define E1000_RETA_IDX(hash) ((hash) & (BIT(7) - 1)) +#define E1000_RETA_VAL(reta, hash) (((uint8_t *)(reta))[E1000_RETA_IDX(hash)]) + +#define E1000_MRQC_EN_TCPIPV4(mrqc) ((mrqc) & BIT(16)) +#define E1000_MRQC_EN_IPV4(mrqc) ((mrqc) & BIT(17)) +#define E1000_MRQC_EN_TCPIPV6(mrqc) ((mrqc) & BIT(18)) +#define E1000_MRQC_EN_IPV6EX(mrqc) ((mrqc) & BIT(19)) +#define E1000_MRQC_EN_IPV6(mrqc) ((mrqc) & BIT(20)) + +#define E1000_MRQ_RSS_TYPE_NONE (0) +#define E1000_MRQ_RSS_TYPE_IPV4TCP (1) +#define E1000_MRQ_RSS_TYPE_IPV4 (2) +#define E1000_MRQ_RSS_TYPE_IPV6TCP (3) +#define E1000_MRQ_RSS_TYPE_IPV6EX (4) +#define E1000_MRQ_RSS_TYPE_IPV6 (5) + +#define E1000_ICR_ASSERTED BIT(31) +#define E1000_EIAC_MASK 0x01F00000 + +/* RFCTL register bits */ +#define E1000_RFCTL_ISCSI_DIS 0x00000001 +#define E1000_RFCTL_NFSW_DIS 0x00000040 +#define E1000_RFCTL_NFSR_DIS 0x00000080 +#define E1000_RFCTL_IPV6_DIS 0x00000400 +#define E1000_RFCTL_IPV6_XSUM_DIS 0x00000800 +#define E1000_RFCTL_IPFRSP_DIS 0x00004000 +#define E1000_RFCTL_EXTEN 0x00008000 +#define E1000_RFCTL_IPV6_EX_DIS 0x00010000 +#define E1000_RFCTL_NEW_IPV6_EXT_DIS 0x00020000 + +/* TARC* parsing */ +#define E1000_TARC_ENABLE BIT(10) + +/* SW Semaphore Register */ +#define E1000_SWSM_SMBI 0x00000001 /* Driver Semaphore bit */ +#define E1000_SWSM_SWESMBI 0x00000002 /* FW Semaphore bit */ +#define E1000_SWSM_DRV_LOAD 0x00000008 /* Driver Loaded Bit */ + +#define E1000_SWSM2_LOCK 0x00000002 /* Secondary driver semaphore bit */ + +/* Interrupt Cause Read */ +#define E1000_ICR_TXDW 0x00000001 /* Transmit desc written back */ +#define E1000_ICR_TXQE 0x00000002 /* Transmit Queue empty */ +#define E1000_ICR_LSC 0x00000004 /* Link Status Change */ +#define E1000_ICR_RXSEQ 0x00000008 /* rx sequence error */ +#define E1000_ICR_RXDMT0 0x00000010 /* rx desc min. threshold (0) */ +#define E1000_ICR_RXO 0x00000040 /* rx overrun */ +#define E1000_ICR_RXT0 0x00000080 /* rx timer intr (ring 0) */ +#define E1000_ICR_MDAC 0x00000200 /* MDIO access complete */ +#define E1000_ICR_RXCFG 0x00000400 /* RX /c/ ordered set */ +#define E1000_ICR_GPI_EN0 0x00000800 /* GP Int 0 */ +#define E1000_ICR_GPI_EN1 0x00001000 /* GP Int 1 */ +#define E1000_ICR_GPI_EN2 0x00002000 /* GP Int 2 */ +#define E1000_ICR_GPI_EN3 0x00004000 /* GP Int 3 */ +#define E1000_ICR_TXD_LOW 0x00008000 +#define E1000_ICR_SRPD 0x00010000 +#define E1000_ICR_ACK 0x00020000 /* Receive Ack frame */ +#define E1000_ICR_MNG 0x00040000 /* Manageability event */ +#define E1000_ICR_DOCK 0x00080000 /* Dock/Undock */ +#define E1000_ICR_INT_ASSERTED 0x80000000 /* If this bit asserted, the driver should claim the interrupt */ +#define E1000_ICR_RXD_FIFO_PAR0 0x00100000 /* queue 0 Rx descriptor FIFO parity error */ +#define E1000_ICR_TXD_FIFO_PAR0 0x00200000 /* queue 0 Tx descriptor FIFO parity error */ +#define E1000_ICR_HOST_ARB_PAR 0x00400000 /* host arb read buffer parity error */ +#define E1000_ICR_PB_PAR 0x00800000 /* packet buffer parity error */ +#define E1000_ICR_RXD_FIFO_PAR1 0x01000000 /* queue 1 Rx descriptor FIFO parity error */ +#define E1000_ICR_TXD_FIFO_PAR1 0x02000000 /* queue 1 Tx descriptor FIFO parity error */ +#define E1000_ICR_ALL_PARITY 0x03F00000 /* all parity error bits */ +#define E1000_ICR_DSW 0x00000020 /* FW changed the status of DISSW bit in the FWSM */ +#define E1000_ICR_PHYINT 0x00001000 /* LAN connected device generates an interrupt */ +#define E1000_ICR_EPRST 0x00100000 /* ME handware reset occurs */ +#define E1000_ICR_RXQ0 0x00100000 /* Rx Queue 0 Interrupt */ +#define E1000_ICR_RXQ1 0x00200000 /* Rx Queue 1 Interrupt */ +#define E1000_ICR_TXQ0 0x00400000 /* Tx Queue 0 Interrupt */ +#define E1000_ICR_TXQ1 0x00800000 /* Tx Queue 1 Interrupt */ +#define E1000_ICR_OTHER 0x01000000 /* Other Interrupts */ + +#define E1000_ICR_OTHER_CAUSES (E1000_ICR_LSC | \ + E1000_ICR_RXO | \ + E1000_ICR_MDAC | \ + E1000_ICR_SRPD | \ + E1000_ICR_ACK | \ + E1000_ICR_MNG) + +/* Interrupt Cause Set */ +#define E1000_ICS_TXDW E1000_ICR_TXDW /* Transmit desc written back */ +#define E1000_ICS_TXQE E1000_ICR_TXQE /* Transmit Queue empty */ +#define E1000_ICS_LSC E1000_ICR_LSC /* Link Status Change */ +#define E1000_ICS_RXSEQ E1000_ICR_RXSEQ /* rx sequence error */ +#define E1000_ICS_RXDMT0 E1000_ICR_RXDMT0 /* rx desc min. threshold */ +#define E1000_ICS_RXO E1000_ICR_RXO /* rx overrun */ +#define E1000_ICS_RXT0 E1000_ICR_RXT0 /* rx timer intr */ +#define E1000_ICS_MDAC E1000_ICR_MDAC /* MDIO access complete */ +#define E1000_ICS_RXCFG E1000_ICR_RXCFG /* RX /c/ ordered set */ +#define E1000_ICS_GPI_EN0 E1000_ICR_GPI_EN0 /* GP Int 0 */ +#define E1000_ICS_GPI_EN1 E1000_ICR_GPI_EN1 /* GP Int 1 */ +#define E1000_ICS_GPI_EN2 E1000_ICR_GPI_EN2 /* GP Int 2 */ +#define E1000_ICS_GPI_EN3 E1000_ICR_GPI_EN3 /* GP Int 3 */ +#define E1000_ICS_TXD_LOW E1000_ICR_TXD_LOW +#define E1000_ICS_SRPD E1000_ICR_SRPD +#define E1000_ICS_ACK E1000_ICR_ACK /* Receive Ack frame */ +#define E1000_ICS_MNG E1000_ICR_MNG /* Manageability event */ +#define E1000_ICS_DOCK E1000_ICR_DOCK /* Dock/Undock */ +#define E1000_ICS_RXD_FIFO_PAR0 E1000_ICR_RXD_FIFO_PAR0 /* queue 0 Rx descriptor FIFO parity error */ +#define E1000_ICS_TXD_FIFO_PAR0 E1000_ICR_TXD_FIFO_PAR0 /* queue 0 Tx descriptor FIFO parity error */ +#define E1000_ICS_HOST_ARB_PAR E1000_ICR_HOST_ARB_PAR /* host arb read buffer parity error */ +#define E1000_ICS_PB_PAR E1000_ICR_PB_PAR /* packet buffer parity error */ +#define E1000_ICS_RXD_FIFO_PAR1 E1000_ICR_RXD_FIFO_PAR1 /* queue 1 Rx descriptor FIFO parity error */ +#define E1000_ICS_TXD_FIFO_PAR1 E1000_ICR_TXD_FIFO_PAR1 /* queue 1 Tx descriptor FIFO parity error */ +#define E1000_ICS_DSW E1000_ICR_DSW +#define E1000_ICS_PHYINT E1000_ICR_PHYINT +#define E1000_ICS_EPRST E1000_ICR_EPRST + +/* Interrupt Mask Set */ +#define E1000_IMS_TXDW E1000_ICR_TXDW /* Transmit desc written back */ +#define E1000_IMS_TXQE E1000_ICR_TXQE /* Transmit Queue empty */ +#define E1000_IMS_LSC E1000_ICR_LSC /* Link Status Change */ +#define E1000_IMS_RXSEQ E1000_ICR_RXSEQ /* rx sequence error */ +#define E1000_IMS_RXDMT0 E1000_ICR_RXDMT0 /* rx desc min. threshold */ +#define E1000_IMS_RXO E1000_ICR_RXO /* rx overrun */ +#define E1000_IMS_RXT0 E1000_ICR_RXT0 /* rx timer intr */ +#define E1000_IMS_MDAC E1000_ICR_MDAC /* MDIO access complete */ +#define E1000_IMS_RXCFG E1000_ICR_RXCFG /* RX /c/ ordered set */ +#define E1000_IMS_GPI_EN0 E1000_ICR_GPI_EN0 /* GP Int 0 */ +#define E1000_IMS_GPI_EN1 E1000_ICR_GPI_EN1 /* GP Int 1 */ +#define E1000_IMS_GPI_EN2 E1000_ICR_GPI_EN2 /* GP Int 2 */ +#define E1000_IMS_GPI_EN3 E1000_ICR_GPI_EN3 /* GP Int 3 */ +#define E1000_IMS_TXD_LOW E1000_ICR_TXD_LOW +#define E1000_IMS_SRPD E1000_ICR_SRPD +#define E1000_IMS_ACK E1000_ICR_ACK /* Receive Ack frame */ +#define E1000_IMS_MNG E1000_ICR_MNG /* Manageability event */ +#define E1000_IMS_RXQ0 E1000_ICR_RXQ0 +#define E1000_IMS_RXQ1 E1000_ICR_RXQ1 +#define E1000_IMS_TXQ0 E1000_ICR_TXQ0 +#define E1000_IMS_TXQ1 E1000_ICR_TXQ1 +#define E1000_IMS_OTHER E1000_ICR_OTHER +#define E1000_IMS_DOCK E1000_ICR_DOCK /* Dock/Undock */ +#define E1000_IMS_RXD_FIFO_PAR0 E1000_ICR_RXD_FIFO_PAR0 /* queue 0 Rx descriptor FIFO parity error */ +#define E1000_IMS_TXD_FIFO_PAR0 E1000_ICR_TXD_FIFO_PAR0 /* queue 0 Tx descriptor FIFO parity error */ +#define E1000_IMS_HOST_ARB_PAR E1000_ICR_HOST_ARB_PAR /* host arb read buffer parity error */ +#define E1000_IMS_PB_PAR E1000_ICR_PB_PAR /* packet buffer parity error */ +#define E1000_IMS_RXD_FIFO_PAR1 E1000_ICR_RXD_FIFO_PAR1 /* queue 1 Rx descriptor FIFO parity error */ +#define E1000_IMS_TXD_FIFO_PAR1 E1000_ICR_TXD_FIFO_PAR1 /* queue 1 Tx descriptor FIFO parity error */ +#define E1000_IMS_DSW E1000_ICR_DSW +#define E1000_IMS_PHYINT E1000_ICR_PHYINT +#define E1000_IMS_EPRST E1000_ICR_EPRST + +/* Interrupt Mask Clear */ +#define E1000_IMC_TXDW E1000_ICR_TXDW /* Transmit desc written back */ +#define E1000_IMC_TXQE E1000_ICR_TXQE /* Transmit Queue empty */ +#define E1000_IMC_LSC E1000_ICR_LSC /* Link Status Change */ +#define E1000_IMC_RXSEQ E1000_ICR_RXSEQ /* rx sequence error */ +#define E1000_IMC_RXDMT0 E1000_ICR_RXDMT0 /* rx desc min. threshold */ +#define E1000_IMC_RXO E1000_ICR_RXO /* rx overrun */ +#define E1000_IMC_RXT0 E1000_ICR_RXT0 /* rx timer intr */ +#define E1000_IMC_MDAC E1000_ICR_MDAC /* MDIO access complete */ +#define E1000_IMC_RXCFG E1000_ICR_RXCFG /* RX /c/ ordered set */ +#define E1000_IMC_GPI_EN0 E1000_ICR_GPI_EN0 /* GP Int 0 */ +#define E1000_IMC_GPI_EN1 E1000_ICR_GPI_EN1 /* GP Int 1 */ +#define E1000_IMC_GPI_EN2 E1000_ICR_GPI_EN2 /* GP Int 2 */ +#define E1000_IMC_GPI_EN3 E1000_ICR_GPI_EN3 /* GP Int 3 */ +#define E1000_IMC_TXD_LOW E1000_ICR_TXD_LOW +#define E1000_IMC_SRPD E1000_ICR_SRPD +#define E1000_IMC_ACK E1000_ICR_ACK /* Receive Ack frame */ +#define E1000_IMC_MNG E1000_ICR_MNG /* Manageability event */ +#define E1000_IMC_DOCK E1000_ICR_DOCK /* Dock/Undock */ +#define E1000_IMC_RXD_FIFO_PAR0 E1000_ICR_RXD_FIFO_PAR0 /* queue 0 Rx descriptor FIFO parity error */ +#define E1000_IMC_TXD_FIFO_PAR0 E1000_ICR_TXD_FIFO_PAR0 /* queue 0 Tx descriptor FIFO parity error */ +#define E1000_IMC_HOST_ARB_PAR E1000_ICR_HOST_ARB_PAR /* host arb read buffer parity error */ +#define E1000_IMC_PB_PAR E1000_ICR_PB_PAR /* packet buffer parity error */ +#define E1000_IMC_RXD_FIFO_PAR1 E1000_ICR_RXD_FIFO_PAR1 /* queue 1 Rx descriptor FIFO parity error */ +#define E1000_IMC_TXD_FIFO_PAR1 E1000_ICR_TXD_FIFO_PAR1 /* queue 1 Tx descriptor FIFO parity error */ +#define E1000_IMC_DSW E1000_ICR_DSW +#define E1000_IMC_PHYINT E1000_ICR_PHYINT +#define E1000_IMC_EPRST E1000_ICR_EPRST + +/* Receive Control */ +#define E1000_RCTL_RST 0x00000001 /* Software reset */ +#define E1000_RCTL_EN 0x00000002 /* enable */ +#define E1000_RCTL_SBP 0x00000004 /* store bad packet */ +#define E1000_RCTL_UPE 0x00000008 /* unicast promiscuous enable */ +#define E1000_RCTL_MPE 0x00000010 /* multicast promiscuous enab */ +#define E1000_RCTL_LPE 0x00000020 /* long packet enable */ +#define E1000_RCTL_LBM_NO 0x00000000 /* no loopback mode */ +#define E1000_RCTL_LBM_MAC 0x00000040 /* MAC loopback mode */ +#define E1000_RCTL_LBM_SLP 0x00000080 /* serial link loopback mode */ +#define E1000_RCTL_LBM_TCVR 0x000000C0 /* tcvr loopback mode */ +#define E1000_RCTL_DTYP_MASK 0x00000C00 /* Descriptor type mask */ +#define E1000_RCTL_DTYP_PS 0x00000400 /* Packet Split descriptor */ +#define E1000_RCTL_RDMTS_HALF 0x00000000 /* rx desc min threshold size */ +#define E1000_RCTL_RDMTS_QUAT 0x00000100 /* rx desc min threshold size */ +#define E1000_RCTL_RDMTS_EIGTH 0x00000200 /* rx desc min threshold size */ +#define E1000_RCTL_MO_SHIFT 12 /* multicast offset shift */ +#define E1000_RCTL_MO_0 0x00000000 /* multicast offset 11:0 */ +#define E1000_RCTL_MO_1 0x00001000 /* multicast offset 12:1 */ +#define E1000_RCTL_MO_2 0x00002000 /* multicast offset 13:2 */ +#define E1000_RCTL_MO_3 0x00003000 /* multicast offset 15:4 */ +#define E1000_RCTL_MDR 0x00004000 /* multicast desc ring 0 */ +#define E1000_RCTL_BAM 0x00008000 /* broadcast enable */ +/* these buffer sizes are valid if E1000_RCTL_BSEX is 0 */ +#define E1000_RCTL_SZ_2048 0x00000000 /* rx buffer size 2048 */ +#define E1000_RCTL_SZ_1024 0x00010000 /* rx buffer size 1024 */ +#define E1000_RCTL_SZ_512 0x00020000 /* rx buffer size 512 */ +#define E1000_RCTL_SZ_256 0x00030000 /* rx buffer size 256 */ +/* these buffer sizes are valid if E1000_RCTL_BSEX is 1 */ +#define E1000_RCTL_SZ_16384 0x00010000 /* rx buffer size 16384 */ +#define E1000_RCTL_SZ_8192 0x00020000 /* rx buffer size 8192 */ +#define E1000_RCTL_SZ_4096 0x00030000 /* rx buffer size 4096 */ +#define E1000_RCTL_VFE 0x00040000 /* vlan filter enable */ +#define E1000_RCTL_CFIEN 0x00080000 /* canonical form enable */ +#define E1000_RCTL_CFI 0x00100000 /* canonical form indicator */ +#define E1000_RCTL_DPF 0x00400000 /* discard pause frames */ +#define E1000_RCTL_PMCF 0x00800000 /* pass MAC control frames */ +#define E1000_RCTL_BSEX 0x02000000 /* Buffer size extension */ +#define E1000_RCTL_SECRC 0x04000000 /* Strip Ethernet CRC */ +#define E1000_RCTL_FLXBUF_MASK 0x78000000 /* Flexible buffer size */ +#define E1000_RCTL_FLXBUF_SHIFT 27 /* Flexible buffer shift */ + + +#define E1000_EEPROM_SWDPIN0 0x0001 /* SWDPIN 0 EEPROM Value */ +#define E1000_EEPROM_LED_LOGIC 0x0020 /* Led Logic Word */ +#define E1000_EEPROM_RW_REG_DATA 16 /* Offset to data in EEPROM read/write registers */ +#define E1000_EEPROM_RW_REG_DONE 0x10 /* Offset to READ/WRITE done bit */ +#define E1000_EEPROM_RW_REG_START 1 /* First bit for telling part to start operation */ +#define E1000_EEPROM_RW_ADDR_SHIFT 8 /* Shift to the address bits */ +#define E1000_EEPROM_POLL_WRITE 1 /* Flag for polling for write complete */ +#define E1000_EEPROM_POLL_READ 0 /* Flag for polling for read complete */ + +/* 82574 EERD/EEWR registers layout */ +#define E1000_EERW_START BIT(0) +#define E1000_EERW_DONE BIT(1) +#define E1000_EERW_ADDR_SHIFT 2 +#define E1000_EERW_ADDR_MASK ((1L << 14) - 1) +#define E1000_EERW_DATA_SHIFT 16 +#define E1000_EERW_DATA_MASK ((1L << 16) - 1) + +/* Register Bit Masks */ +/* Device Control */ +#define E1000_CTRL_FD 0x00000001 /* Full duplex.0=half; 1=full */ +#define E1000_CTRL_BEM 0x00000002 /* Endian Mode.0=little,1=big */ +#define E1000_CTRL_PRIOR 0x00000004 /* Priority on PCI. 0=rx,1=fair */ +#define E1000_CTRL_GIO_MASTER_DISABLE 0x00000004 /*Blocks new Master requests */ +#define E1000_CTRL_LRST 0x00000008 /* Link reset. 0=normal,1=reset */ +#define E1000_CTRL_TME 0x00000010 /* Test mode. 0=normal,1=test */ +#define E1000_CTRL_SLE 0x00000020 /* Serial Link on 0=dis,1=en */ +#define E1000_CTRL_ASDE 0x00000020 /* Auto-speed detect enable */ +#define E1000_CTRL_SLU 0x00000040 /* Set link up (Force Link) */ +#define E1000_CTRL_ILOS 0x00000080 /* Invert Loss-Of Signal */ +#define E1000_CTRL_SPD_SEL 0x00000300 /* Speed Select Mask */ +#define E1000_CTRL_SPD_10 0x00000000 /* Force 10Mb */ +#define E1000_CTRL_SPD_100 0x00000100 /* Force 100Mb */ +#define E1000_CTRL_SPD_1000 0x00000200 /* Force 1Gb */ +#define E1000_CTRL_BEM32 0x00000400 /* Big Endian 32 mode */ +#define E1000_CTRL_FRCSPD 0x00000800 /* Force Speed */ +#define E1000_CTRL_FRCDPX 0x00001000 /* Force Duplex */ +#define E1000_CTRL_D_UD_EN 0x00002000 /* Dock/Undock enable */ +#define E1000_CTRL_D_UD_POLARITY 0x00004000 /* Defined polarity of Dock/Undock indication in SDP[0] */ +#define E1000_CTRL_FORCE_PHY_RESET 0x00008000 /* Reset both PHY ports, through PHYRST_N pin */ +#define E1000_CTRL_SPD_SHIFT 8 /* Speed Select Shift */ + +#define E1000_CTRL_EXT_ASDCHK 0x00001000 /* auto speed detection check */ +#define E1000_CTRL_EXT_EE_RST 0x00002000 /* EEPROM reset */ +#define E1000_CTRL_EXT_LINK_EN 0x00010000 /* enable link status from external LINK_0 and LINK_1 pins */ +#define E1000_CTRL_EXT_DRV_LOAD 0x10000000 /* Driver loaded bit for FW */ +#define E1000_CTRL_EXT_EIAME 0x01000000 +#define E1000_CTRL_EXT_IAME 0x08000000 /* Int ACK Auto-mask */ +#define E1000_CTRL_EXT_PBA_CLR 0x80000000 /* PBA Clear */ +#define E1000_CTRL_EXT_INT_TIMERS_CLEAR_ENA 0x20000000 +#define E1000_CTRL_EXT_SPD_BYPS 0x00008000 /* Speed Select Bypass */ + +#define E1000_CTRL_SWDPIN0 0x00040000 /* SWDPIN 0 value */ +#define E1000_CTRL_SWDPIN1 0x00080000 /* SWDPIN 1 value */ +#define E1000_CTRL_SWDPIN2 0x00100000 /* SWDPIN 2 value */ +#define E1000_CTRL_SWDPIN3 0x00200000 /* SWDPIN 3 value */ +#define E1000_CTRL_SWDPIO0 0x00400000 /* SWDPIN 0 Input or output */ +#define E1000_CTRL_SWDPIO1 0x00800000 /* SWDPIN 1 input or output */ +#define E1000_CTRL_SWDPIO2 0x01000000 /* SWDPIN 2 input or output */ +#define E1000_CTRL_SWDPIO3 0x02000000 /* SWDPIN 3 input or output */ +#define E1000_CTRL_ADVD3WUC 0x00100000 /* D3 WUC */ +#define E1000_CTRL_RST 0x04000000 /* Global reset */ +#define E1000_CTRL_RFCE 0x08000000 /* Receive Flow Control enable */ +#define E1000_CTRL_TFCE 0x10000000 /* Transmit flow control enable */ +#define E1000_CTRL_RTE 0x20000000 /* Routing tag enable */ +#define E1000_CTRL_VME 0x40000000 /* IEEE VLAN mode enable */ +#define E1000_CTRL_PHY_RST 0x80000000 /* PHY Reset */ +#define E1000_CTRL_SW2FW_INT 0x02000000 /* Initiate an interrupt to manageability engine */ + +/* Device Status */ +#define E1000_STATUS_FD 0x00000001 /* Full duplex.0=half,1=full */ +#define E1000_STATUS_LU 0x00000002 /* Link up.0=no,1=link */ +#define E1000_STATUS_SPEED_10 0x00000000 /* Speed 10Mb/s */ +#define E1000_STATUS_SPEED_100 0x00000040 /* Speed 100Mb/s */ +#define E1000_STATUS_SPEED_1000 0x00000080 /* Speed 1000Mb/s */ +#define E1000_STATUS_PHYRA 0x00000400 /* PHY Reset Asserted */ +#define E1000_STATUS_GIO_MASTER_ENABLE 0x00080000 + +/* EEPROM/Flash Control */ +#define E1000_EECD_SK 0x00000001 /* EEPROM Clock */ +#define E1000_EECD_CS 0x00000002 /* EEPROM Chip Select */ +#define E1000_EECD_DI 0x00000004 /* EEPROM Data In */ +#define E1000_EECD_DO 0x00000008 /* EEPROM Data Out */ +#define E1000_EECD_FWE_MASK 0x00000030 +#define E1000_EECD_FWE_DIS 0x00000010 /* Disable FLASH writes */ +#define E1000_EECD_FWE_EN 0x00000020 /* Enable FLASH writes */ +#define E1000_EECD_FWE_SHIFT 4 +#define E1000_EECD_REQ 0x00000040 /* EEPROM Access Request */ +#define E1000_EECD_GNT 0x00000080 /* EEPROM Access Grant */ +#define E1000_EECD_PRES 0x00000100 /* EEPROM Present */ +#define E1000_EECD_SIZE 0x00000200 /* EEPROM Size (0=64 word 1=256 word) */ +#define E1000_EECD_ADDR_BITS 0x00000400 /* EEPROM Addressing bits based on type + * (0-small, 1-large) */ +#define E1000_EECD_TYPE 0x00002000 /* EEPROM Type (1-SPI, 0-Microwire) */ +#ifndef E1000_EEPROM_GRANT_ATTEMPTS +#define E1000_EEPROM_GRANT_ATTEMPTS 1000 /* EEPROM # attempts to gain grant */ +#endif +#define E1000_EECD_AUTO_RD 0x00000200 /* EEPROM Auto Read done */ +#define E1000_EECD_SIZE_EX_MASK 0x00007800 /* EEprom Size */ +#define E1000_EECD_SIZE_EX_SHIFT 11 +#define E1000_EECD_NVADDS 0x00018000 /* NVM Address Size */ +#define E1000_EECD_SELSHAD 0x00020000 /* Select Shadow RAM */ +#define E1000_EECD_INITSRAM 0x00040000 /* Initialize Shadow RAM */ +#define E1000_EECD_FLUPD 0x00080000 /* Update FLASH */ +#define E1000_EECD_AUPDEN 0x00100000 /* Enable Autonomous FLASH update */ +#define E1000_EECD_SHADV 0x00200000 /* Shadow RAM Data Valid */ +#define E1000_EECD_SEC1VAL 0x00400000 /* Sector One Valid */ + + +#define E1000_EECD_SECVAL_SHIFT 22 +#define E1000_STM_OPCODE 0xDB00 +#define E1000_HICR_FW_RESET 0xC0 + +#define E1000_SHADOW_RAM_WORDS 2048 +#define E1000_ICH_NVM_SIG_WORD 0x13 +#define E1000_ICH_NVM_SIG_MASK 0xC0 + +/* MDI Control */ +#define E1000_MDIC_DATA_MASK 0x0000FFFF +#define E1000_MDIC_REG_MASK 0x001F0000 +#define E1000_MDIC_REG_SHIFT 16 +#define E1000_MDIC_PHY_MASK 0x03E00000 +#define E1000_MDIC_PHY_SHIFT 21 +#define E1000_MDIC_OP_WRITE 0x04000000 +#define E1000_MDIC_OP_READ 0x08000000 +#define E1000_MDIC_READY 0x10000000 +#define E1000_MDIC_INT_EN 0x20000000 +#define E1000_MDIC_ERROR 0x40000000 + +/* Rx Interrupt Delay Timer */ +#define E1000_RDTR_FPD BIT(31) + +/* Tx Interrupt Delay Timer */ +#define E1000_TIDV_FPD BIT(31) + +/* Delay increments in nanoseconds for delayed interrupts registers */ +#define E1000_INTR_DELAY_NS_RES (1024) + +/* Delay increments in nanoseconds for interrupt throttling registers */ +#define E1000_INTR_THROTTLING_NS_RES (256) + +/* EEPROM Commands - Microwire */ +#define EEPROM_READ_OPCODE_MICROWIRE 0x6 /* EEPROM read opcode */ +#define EEPROM_WRITE_OPCODE_MICROWIRE 0x5 /* EEPROM write opcode */ +#define EEPROM_ERASE_OPCODE_MICROWIRE 0x7 /* EEPROM erase opcode */ +#define EEPROM_EWEN_OPCODE_MICROWIRE 0x13 /* EEPROM erase/write enable */ +#define EEPROM_EWDS_OPCODE_MICROWIRE 0x10 /* EEPROM erast/write disable */ + +/* EEPROM Word Offsets */ +#define EEPROM_COMPAT 0x0003 +#define EEPROM_ID_LED_SETTINGS 0x0004 +#define EEPROM_VERSION 0x0005 +#define EEPROM_SERDES_AMPLITUDE 0x0006 /* For SERDES output amplitude adjustment. */ +#define EEPROM_PHY_CLASS_WORD 0x0007 +#define EEPROM_INIT_CONTROL1_REG 0x000A +#define EEPROM_INIT_CONTROL2_REG 0x000F +#define EEPROM_SWDEF_PINS_CTRL_PORT_1 0x0010 +#define EEPROM_INIT_CONTROL3_PORT_B 0x0014 +#define EEPROM_INIT_3GIO_3 0x001A +#define EEPROM_SWDEF_PINS_CTRL_PORT_0 0x0020 +#define EEPROM_INIT_CONTROL3_PORT_A 0x0024 +#define EEPROM_CFG 0x0012 +#define EEPROM_FLASH_VERSION 0x0032 +#define EEPROM_CHECKSUM_REG 0x003F + +#define E1000_EEPROM_CFG_DONE 0x00040000 /* MNG config cycle done */ +#define E1000_EEPROM_CFG_DONE_PORT_1 0x00080000 /* ...for second port */ + +/* HH Time Sync */ +#define E1000_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */ +#define E1000_TSYNCTXCTL_SYNC_COMP 0x40000000 /* sync complete */ +#define E1000_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */ + +#define E1000_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */ +#define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */ + +#define E1000_TSYNCRXCTL_VALID 0x00000001 /* Rx timestamp valid */ +#define E1000_TSYNCRXCTL_TYPE_MASK 0x0000000E /* Rx type mask */ +#define E1000_TSYNCRXCTL_TYPE_L2_V2 0x00 +#define E1000_TSYNCRXCTL_TYPE_L4_V1 0x02 +#define E1000_TSYNCRXCTL_TYPE_L2_L4_V2 0x04 +#define E1000_TSYNCRXCTL_TYPE_ALL 0x08 +#define E1000_TSYNCRXCTL_TYPE_EVENT_V2 0x0A +#define E1000_TSYNCRXCTL_ENABLED 0x00000010 /* enable Rx timestamping */ +#define E1000_TSYNCRXCTL_SYSCFI 0x00000020 /* Sys clock frequency */ + +#define E1000_RXMTRL_PTP_V1_SYNC_MESSAGE 0x00000000 +#define E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE 0x00010000 + +#define E1000_RXMTRL_PTP_V2_SYNC_MESSAGE 0x00000000 +#define E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE 0x01000000 + +#define E1000_TIMINCA_INCPERIOD_SHIFT 24 +#define E1000_TIMINCA_INCVALUE_MASK 0x00FFFFFF + +/* PCI Express Control */ +/* 3GIO Control Register - GCR (0x05B00; RW) */ +#define E1000_L0S_ADJUST (1 << 9) +#define E1000_L1_ENTRY_LATENCY_MSB (1 << 23) +#define E1000_L1_ENTRY_LATENCY_LSB (1 << 25 | 1 << 26) + +#define E1000_L0S_ADJUST (1 << 9) +#define E1000_L1_ENTRY_LATENCY_MSB (1 << 23) +#define E1000_L1_ENTRY_LATENCY_LSB (1 << 25 | 1 << 26) + +#define E1000_GCR_RO_BITS (1 << 23 | 1 << 25 | 1 << 26) + +/* MSI-X PBA Clear register */ +#define E1000_PBACLR_VALID_MASK (BIT(5) - 1) + +/* Transmit Descriptor bit definitions */ +#define E1000_TXD_DTYP_D 0x00100000 /* Data Descriptor */ +#define E1000_TXD_DTYP_C 0x00000000 /* Context Descriptor */ +#define E1000_TXD_CMD_EOP 0x01000000 /* End of Packet */ +#define E1000_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define E1000_TXD_CMD_IC 0x04000000 /* Insert Checksum */ +#define E1000_TXD_CMD_RS 0x08000000 /* Report Status */ +#define E1000_TXD_CMD_RPS 0x10000000 /* Report Packet Sent */ +#define E1000_TXD_CMD_DEXT 0x20000000 /* Descriptor extension (0 = legacy) */ +#define E1000_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ +#define E1000_TXD_CMD_IDE 0x80000000 /* Enable Tidv register */ +#define E1000_TXD_STAT_DD 0x00000001 /* Descriptor Done */ +#define E1000_TXD_STAT_EC 0x00000002 /* Excess Collisions */ +#define E1000_TXD_STAT_LC 0x00000004 /* Late Collisions */ +#define E1000_TXD_STAT_TU 0x00000008 /* Transmit underrun */ +#define E1000_TXD_CMD_TCP 0x01000000 /* TCP packet */ +#define E1000_TXD_CMD_IP 0x02000000 /* IP packet */ +#define E1000_TXD_CMD_TSE 0x04000000 /* TCP Seg enable */ +#define E1000_TXD_CMD_SNAP 0x40000000 /* Update SNAP header */ +#define E1000_TXD_STAT_TC 0x00000004 /* Tx Underrun */ +#define E1000_TXD_EXTCMD_TSTAMP 0x00000010 /* IEEE1588 Timestamp packet */ + +/* Transmit Control */ +#define E1000_TCTL_RST 0x00000001 /* software reset */ +#define E1000_TCTL_EN 0x00000002 /* enable tx */ +#define E1000_TCTL_BCE 0x00000004 /* busy check enable */ +#define E1000_TCTL_PSP 0x00000008 /* pad short packets */ +#define E1000_TCTL_CT 0x00000ff0 /* collision threshold */ +#define E1000_TCTL_COLD 0x003ff000 /* collision distance */ +#define E1000_TCTL_SWXOFF 0x00400000 /* SW Xoff transmission */ +#define E1000_TCTL_PBE 0x00800000 /* Packet Burst Enable */ +#define E1000_TCTL_RTLC 0x01000000 /* Re-transmit on late collision */ +#define E1000_TCTL_NRTU 0x02000000 /* No Re-transmit on underrun */ +#define E1000_TCTL_MULR 0x10000000 /* Multiple request support */ + +/* Legacy Receive Descriptor */ +struct e1000_rx_desc { + uint64_t buffer_addr; /* Address of the descriptor's data buffer */ + uint16_t length; /* Length of data DMAed into data buffer */ + uint16_t csum; /* Packet checksum */ + uint8_t status; /* Descriptor status */ + uint8_t errors; /* Descriptor Errors */ + uint16_t special; +}; + +/* Extended Receive Descriptor */ +union e1000_rx_desc_extended { + struct { + uint64_t buffer_addr; + uint64_t reserved; + } read; + struct { + struct { + uint32_t mrq; /* Multiple Rx Queues */ + union { + uint32_t rss; /* RSS Hash */ + struct { + uint16_t ip_id; /* IP id */ + uint16_t csum; /* Packet Checksum */ + } csum_ip; + } hi_dword; + } lower; + struct { + uint32_t status_error; /* ext status/error */ + uint16_t length; + uint16_t vlan; /* VLAN tag */ + } upper; + } wb; /* writeback */ +}; + +#define MAX_PS_BUFFERS 4 + +/* Number of packet split data buffers (not including the header buffer) */ +#define PS_PAGE_BUFFERS (MAX_PS_BUFFERS - 1) + +/* Receive Descriptor - Packet Split */ +union e1000_rx_desc_packet_split { + struct { + /* one buffer for protocol header(s), three data buffers */ + uint64_t buffer_addr[MAX_PS_BUFFERS]; + } read; + struct { + struct { + uint32_t mrq; /* Multiple Rx Queues */ + union { + uint32_t rss; /* RSS Hash */ + struct { + uint16_t ip_id; /* IP id */ + uint16_t csum; /* Packet Checksum */ + } csum_ip; + } hi_dword; + } lower; + struct { + uint32_t status_error; /* ext status/error */ + uint16_t length0; /* length of buffer 0 */ + uint16_t vlan; /* VLAN tag */ + } middle; + struct { + uint16_t header_status; + /* length of buffers 1-3 */ + uint16_t length[PS_PAGE_BUFFERS]; + } upper; + uint64_t reserved; + } wb; /* writeback */ +}; + +/* Receive Checksum Control bits */ +#define E1000_RXCSUM_IPOFLD 0x100 /* IP Checksum Offload Enable */ +#define E1000_RXCSUM_TUOFLD 0x200 /* TCP/UDP Checksum Offload Enable */ +#define E1000_RXCSUM_PCSD 0x2000 /* Packet Checksum Disable */ + +#define E1000_RING_DESC_LEN (16) +#define E1000_RING_DESC_LEN_SHIFT (4) + +#define E1000_MIN_RX_DESC_LEN E1000_RING_DESC_LEN + +/* Receive Descriptor bit definitions */ +#define E1000_RXD_STAT_DD 0x01 /* Descriptor Done */ +#define E1000_RXD_STAT_EOP 0x02 /* End of Packet */ +#define E1000_RXD_STAT_IXSM 0x04 /* Ignore checksum */ +#define E1000_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ +#define E1000_RXD_STAT_UDPCS 0x10 /* UDP xsum caculated */ +#define E1000_RXD_STAT_TCPCS 0x20 /* TCP xsum calculated */ +#define E1000_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ +#define E1000_RXD_STAT_PIF 0x80 /* passed in-exact filter */ +#define E1000_RXD_STAT_IPIDV 0x200 /* IP identification valid */ +#define E1000_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ +#define E1000_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ +#define E1000_RXD_ERR_CE 0x01 /* CRC Error */ +#define E1000_RXD_ERR_SE 0x02 /* Symbol Error */ +#define E1000_RXD_ERR_SEQ 0x04 /* Sequence Error */ +#define E1000_RXD_ERR_CXE 0x10 /* Carrier Extension Error */ +#define E1000_RXD_ERR_TCPE 0x20 /* TCP/UDP Checksum Error */ +#define E1000_RXD_ERR_IPE 0x40 /* IP Checksum Error */ +#define E1000_RXD_ERR_RXE 0x80 /* Rx Data Error */ +#define E1000_RXD_SPC_VLAN_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ +#define E1000_RXD_SPC_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ +#define E1000_RXD_SPC_PRI_SHIFT 13 +#define E1000_RXD_SPC_CFI_MASK 0x1000 /* CFI is bit 12 */ +#define E1000_RXD_SPC_CFI_SHIFT 12 + +/* RX packet types */ +#define E1000_RXD_PKT_MAC (0) +#define E1000_RXD_PKT_IP4 (1) +#define E1000_RXD_PKT_IP4_XDP (2) +#define E1000_RXD_PKT_IP6 (5) +#define E1000_RXD_PKT_IP6_XDP (6) + +#define E1000_RXD_PKT_TYPE(t) ((t) << 16) + +#define E1000_RXDEXT_STATERR_CE 0x01000000 +#define E1000_RXDEXT_STATERR_SE 0x02000000 +#define E1000_RXDEXT_STATERR_SEQ 0x04000000 +#define E1000_RXDEXT_STATERR_CXE 0x10000000 +#define E1000_RXDEXT_STATERR_TCPE 0x20000000 +#define E1000_RXDEXT_STATERR_IPE 0x40000000 +#define E1000_RXDEXT_STATERR_RXE 0x80000000 + +#define E1000_RXDPS_HDRSTAT_HDRSP 0x00008000 +#define E1000_RXDPS_HDRSTAT_HDRLEN_MASK 0x000003FF + +/* Receive Address */ +#define E1000_RAH_AV 0x80000000 /* Receive descriptor valid */ + +/* Offload Context Descriptor */ +struct e1000_context_desc { + union { + uint32_t ip_config; + struct { + uint8_t ipcss; /* IP checksum start */ + uint8_t ipcso; /* IP checksum offset */ + uint16_t ipcse; /* IP checksum end */ + } ip_fields; + } lower_setup; + union { + uint32_t tcp_config; + struct { + uint8_t tucss; /* TCP checksum start */ + uint8_t tucso; /* TCP checksum offset */ + uint16_t tucse; /* TCP checksum end */ + } tcp_fields; + } upper_setup; + uint32_t cmd_and_length; /* */ + union { + uint32_t data; + struct { + uint8_t status; /* Descriptor status */ + uint8_t hdr_len; /* Header length */ + uint16_t mss; /* Maximum segment size */ + } fields; + } tcp_seg_setup; +}; + +/* Filters */ +#define E1000_NUM_UNICAST 16 /* Unicast filter entries */ +#define E1000_MC_TBL_SIZE 128 /* Multicast Filter Table (4096 bits) */ +#define E1000_VLAN_FILTER_TBL_SIZE 128 /* VLAN Filter Table (4096 bits) */ + +/* Management Control */ +#define E1000_MANC_SMBUS_EN 0x00000001 /* SMBus Enabled - RO */ +#define E1000_MANC_ASF_EN 0x00000002 /* ASF Enabled - RO */ +#define E1000_MANC_R_ON_FORCE 0x00000004 /* Reset on Force TCO - RO */ +#define E1000_MANC_RMCP_EN 0x00000100 /* Enable RCMP 026Fh Filtering */ +#define E1000_MANC_0298_EN 0x00000200 /* Enable RCMP 0298h Filtering */ +#define E1000_MANC_IPV4_EN 0x00000400 /* Enable IPv4 */ +#define E1000_MANC_IPV6_EN 0x00000800 /* Enable IPv6 */ +#define E1000_MANC_SNAP_EN 0x00001000 /* Accept LLC/SNAP */ +#define E1000_MANC_ARP_EN 0x00002000 /* Enable ARP Request Filtering */ +#define E1000_MANC_NEIGHBOR_EN 0x00004000 /* Enable Neighbor Discovery + * Filtering */ +#define E1000_MANC_ARP_RES_EN 0x00008000 /* Enable ARP response Filtering */ +#define E1000_MANC_DIS_IP_CHK_ARP 0x10000000 /* Disable IP address chacking */ + /*for ARP packets - in 82574 */ +#define E1000_MANC_TCO_RESET 0x00010000 /* TCO Reset Occurred */ +#define E1000_MANC_RCV_TCO_EN 0x00020000 /* Receive TCO Packets Enabled */ +#define E1000_MANC_REPORT_STATUS 0x00040000 /* Status Reporting Enabled */ +#define E1000_MANC_RCV_ALL 0x00080000 /* Receive All Enabled */ +#define E1000_MANC_BLK_PHY_RST_ON_IDE 0x00040000 /* Block phy resets */ +#define E1000_MANC_EN_MAC_ADDR_FILTER 0x00100000 /* Enable MAC address + * filtering */ +#define E1000_MANC_EN_MNG2HOST 0x00200000 /* Enable MNG packets to host + * memory */ +#define E1000_MANC_EN_IP_ADDR_FILTER 0x00400000 /* Enable IP address + * filtering */ +#define E1000_MANC_EN_XSUM_FILTER 0x00800000 /* Enable checksum filtering */ +#define E1000_MANC_BR_EN 0x01000000 /* Enable broadcast filtering */ +#define E1000_MANC_SMB_REQ 0x01000000 /* SMBus Request */ +#define E1000_MANC_SMB_GNT 0x02000000 /* SMBus Grant */ +#define E1000_MANC_SMB_CLK_IN 0x04000000 /* SMBus Clock In */ +#define E1000_MANC_SMB_DATA_IN 0x08000000 /* SMBus Data In */ +#define E1000_MANC_SMB_DATA_OUT 0x10000000 /* SMBus Data Out */ +#define E1000_MANC_SMB_CLK_OUT 0x20000000 /* SMBus Clock Out */ + +#define E1000_MANC_SMB_DATA_OUT_SHIFT 28 /* SMBus Data Out Shift */ +#define E1000_MANC_SMB_CLK_OUT_SHIFT 29 /* SMBus Clock Out Shift */ + +/* FACTPS Control */ +#define E1000_FACTPS_LAN0_ON 0x00000004 /* Lan 0 enable */ + +/* For checksumming, the sum of all words in the EEPROM should equal 0xBABA. */ +#define EEPROM_SUM 0xBABA + +/* I/O-Mapped Access to Internal Registers, Memories, and Flash */ +#define E1000_IOADDR 0x00 +#define E1000_IODATA 0x04 + +#define E1000_VFTA_ENTRY_SHIFT 5 +#define E1000_VFTA_ENTRY_MASK 0x7F +#define E1000_VFTA_ENTRY_BIT_SHIFT_MASK 0x1F + +#endif /* HW_E1000_REGS_H */ diff --git a/hw/net/fsl_etsec/etsec.c b/hw/net/fsl_etsec/etsec.c index c753bfb3a8..798ea33d08 100644 --- a/hw/net/fsl_etsec/etsec.c +++ b/hw/net/fsl_etsec/etsec.c @@ -29,6 +29,7 @@ #include "qemu/osdep.h" #include "hw/sysbus.h" #include "hw/irq.h" +#include "hw/net/mii.h" #include "hw/ptimer.h" #include "hw/qdev-properties.h" #include "etsec.h" @@ -339,11 +340,11 @@ static void etsec_reset(DeviceState *d) etsec->rx_buffer_len = 0; etsec->phy_status = - MII_SR_EXTENDED_CAPS | MII_SR_LINK_STATUS | MII_SR_AUTONEG_CAPS | - MII_SR_AUTONEG_COMPLETE | MII_SR_PREAMBLE_SUPPRESS | - MII_SR_EXTENDED_STATUS | MII_SR_100T2_HD_CAPS | MII_SR_100T2_FD_CAPS | - MII_SR_10T_HD_CAPS | MII_SR_10T_FD_CAPS | MII_SR_100X_HD_CAPS | - MII_SR_100X_FD_CAPS | MII_SR_100T4_CAPS; + MII_BMSR_EXTCAP | MII_BMSR_LINK_ST | MII_BMSR_AUTONEG | + MII_BMSR_AN_COMP | MII_BMSR_MFPS | MII_BMSR_EXTSTAT | + MII_BMSR_100T2_HD | MII_BMSR_100T2_FD | + MII_BMSR_10T_HD | MII_BMSR_10T_FD | + MII_BMSR_100TX_HD | MII_BMSR_100TX_FD | MII_BMSR_100T4; etsec_update_irq(etsec); } diff --git a/hw/net/fsl_etsec/etsec.h b/hw/net/fsl_etsec/etsec.h index 3c625c955c..3860864a3f 100644 --- a/hw/net/fsl_etsec/etsec.h +++ b/hw/net/fsl_etsec/etsec.h @@ -76,23 +76,6 @@ typedef struct eTSEC_rxtx_bd { #define FCB_TX_CTU (1 << 1) #define FCB_TX_NPH (1 << 0) -/* PHY Status Register */ -#define MII_SR_EXTENDED_CAPS 0x0001 /* Extended register capabilities */ -#define MII_SR_JABBER_DETECT 0x0002 /* Jabber Detected */ -#define MII_SR_LINK_STATUS 0x0004 /* Link Status 1 = link */ -#define MII_SR_AUTONEG_CAPS 0x0008 /* Auto Neg Capable */ -#define MII_SR_REMOTE_FAULT 0x0010 /* Remote Fault Detect */ -#define MII_SR_AUTONEG_COMPLETE 0x0020 /* Auto Neg Complete */ -#define MII_SR_PREAMBLE_SUPPRESS 0x0040 /* Preamble may be suppressed */ -#define MII_SR_EXTENDED_STATUS 0x0100 /* Ext. status info in Reg 0x0F */ -#define MII_SR_100T2_HD_CAPS 0x0200 /* 100T2 Half Duplex Capable */ -#define MII_SR_100T2_FD_CAPS 0x0400 /* 100T2 Full Duplex Capable */ -#define MII_SR_10T_HD_CAPS 0x0800 /* 10T Half Duplex Capable */ -#define MII_SR_10T_FD_CAPS 0x1000 /* 10T Full Duplex Capable */ -#define MII_SR_100X_HD_CAPS 0x2000 /* 100X Half Duplex Capable */ -#define MII_SR_100X_FD_CAPS 0x4000 /* 100X Full Duplex Capable */ -#define MII_SR_100T4_CAPS 0x8000 /* 100T4 Capable */ - /* eTSEC */ /* Number of register in the device */ diff --git a/hw/net/fsl_etsec/miim.c b/hw/net/fsl_etsec/miim.c index 6bba01c82a..b48d2cb57b 100644 --- a/hw/net/fsl_etsec/miim.c +++ b/hw/net/fsl_etsec/miim.c @@ -23,6 +23,7 @@ */ #include "qemu/osdep.h" +#include "hw/net/mii.h" #include "etsec.h" #include "registers.h" @@ -140,8 +141,8 @@ void etsec_miim_link_status(eTSEC *etsec, NetClientState *nc) { /* Set link status */ if (nc->link_down) { - etsec->phy_status &= ~MII_SR_LINK_STATUS; + etsec->phy_status &= ~MII_BMSR_LINK_ST; } else { - etsec->phy_status |= MII_SR_LINK_STATUS; + etsec->phy_status |= MII_BMSR_LINK_ST; } } diff --git a/hw/net/igb.c b/hw/net/igb.c new file mode 100644 index 0000000000..c6d753df87 --- /dev/null +++ b/hw/net/igb.c @@ -0,0 +1,623 @@ +/* + * QEMU Intel 82576 SR/IOV Ethernet Controller Emulation + * + * Datasheet: + * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eg-gbe-datasheet.pdf + * + * Copyright (c) 2020-2023 Red Hat, Inc. + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Akihiko Odaki <akihiko.odaki@daynix.com> + * Gal Hammmer <gal.hammer@sap.com> + * Marcel Apfelbaum <marcel.apfelbaum@gmail.com> + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2008 Qumranet + * Based on work done by: + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "qemu/units.h" +#include "net/eth.h" +#include "net/net.h" +#include "net/tap.h" +#include "qemu/module.h" +#include "qemu/range.h" +#include "sysemu/sysemu.h" +#include "hw/hw.h" +#include "hw/net/mii.h" +#include "hw/pci/pci.h" +#include "hw/pci/pcie.h" +#include "hw/pci/pcie_sriov.h" +#include "hw/pci/msi.h" +#include "hw/pci/msix.h" +#include "hw/qdev-properties.h" +#include "migration/vmstate.h" + +#include "igb_common.h" +#include "igb_core.h" + +#include "trace.h" +#include "qapi/error.h" +#include "qom/object.h" + +#define TYPE_IGB "igb" +OBJECT_DECLARE_SIMPLE_TYPE(IGBState, IGB) + +struct IGBState { + PCIDevice parent_obj; + NICState *nic; + NICConf conf; + + MemoryRegion mmio; + MemoryRegion flash; + MemoryRegion io; + MemoryRegion msix; + + uint32_t ioaddr; + + IGBCore core; +}; + +#define IGB_CAP_SRIOV_OFFSET (0x160) +#define IGB_VF_OFFSET (0x80) +#define IGB_VF_STRIDE (2) + +#define E1000E_MMIO_IDX 0 +#define E1000E_FLASH_IDX 1 +#define E1000E_IO_IDX 2 +#define E1000E_MSIX_IDX 3 + +#define E1000E_MMIO_SIZE (128 * KiB) +#define E1000E_FLASH_SIZE (128 * KiB) +#define E1000E_IO_SIZE (32) +#define E1000E_MSIX_SIZE (16 * KiB) + +static void igb_write_config(PCIDevice *dev, uint32_t addr, + uint32_t val, int len) +{ + IGBState *s = IGB(dev); + + trace_igb_write_config(addr, val, len); + pci_default_write_config(dev, addr, val, len); + + if (range_covers_byte(addr, len, PCI_COMMAND) && + (dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) { + igb_start_recv(&s->core); + } +} + +uint64_t +igb_mmio_read(void *opaque, hwaddr addr, unsigned size) +{ + IGBState *s = opaque; + return igb_core_read(&s->core, addr, size); +} + +void +igb_mmio_write(void *opaque, hwaddr addr, uint64_t val, unsigned size) +{ + IGBState *s = opaque; + igb_core_write(&s->core, addr, val, size); +} + +static bool +igb_io_get_reg_index(IGBState *s, uint32_t *idx) +{ + if (s->ioaddr < 0x1FFFF) { + *idx = s->ioaddr; + return true; + } + + if (s->ioaddr < 0x7FFFF) { + trace_e1000e_wrn_io_addr_undefined(s->ioaddr); + return false; + } + + if (s->ioaddr < 0xFFFFF) { + trace_e1000e_wrn_io_addr_flash(s->ioaddr); + return false; + } + + trace_e1000e_wrn_io_addr_unknown(s->ioaddr); + return false; +} + +static uint64_t +igb_io_read(void *opaque, hwaddr addr, unsigned size) +{ + IGBState *s = opaque; + uint32_t idx = 0; + uint64_t val; + + switch (addr) { + case E1000_IOADDR: + trace_e1000e_io_read_addr(s->ioaddr); + return s->ioaddr; + case E1000_IODATA: + if (igb_io_get_reg_index(s, &idx)) { + val = igb_core_read(&s->core, idx, sizeof(val)); + trace_e1000e_io_read_data(idx, val); + return val; + } + return 0; + default: + trace_e1000e_wrn_io_read_unknown(addr); + return 0; + } +} + +static void +igb_io_write(void *opaque, hwaddr addr, uint64_t val, unsigned size) +{ + IGBState *s = opaque; + uint32_t idx = 0; + + switch (addr) { + case E1000_IOADDR: + trace_e1000e_io_write_addr(val); + s->ioaddr = (uint32_t) val; + return; + case E1000_IODATA: + if (igb_io_get_reg_index(s, &idx)) { + trace_e1000e_io_write_data(idx, val); + igb_core_write(&s->core, idx, val, sizeof(val)); + } + return; + default: + trace_e1000e_wrn_io_write_unknown(addr); + return; + } +} + +static const MemoryRegionOps mmio_ops = { + .read = igb_mmio_read, + .write = igb_mmio_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .impl = { + .min_access_size = 4, + .max_access_size = 4, + }, +}; + +static const MemoryRegionOps io_ops = { + .read = igb_io_read, + .write = igb_io_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .impl = { + .min_access_size = 4, + .max_access_size = 4, + }, +}; + +static bool +igb_nc_can_receive(NetClientState *nc) +{ + IGBState *s = qemu_get_nic_opaque(nc); + return igb_can_receive(&s->core); +} + +static ssize_t +igb_nc_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt) +{ + IGBState *s = qemu_get_nic_opaque(nc); + return igb_receive_iov(&s->core, iov, iovcnt); +} + +static ssize_t +igb_nc_receive(NetClientState *nc, const uint8_t *buf, size_t size) +{ + IGBState *s = qemu_get_nic_opaque(nc); + return igb_receive(&s->core, buf, size); +} + +static void +igb_set_link_status(NetClientState *nc) +{ + IGBState *s = qemu_get_nic_opaque(nc); + igb_core_set_link_status(&s->core); +} + +static NetClientInfo net_igb_info = { + .type = NET_CLIENT_DRIVER_NIC, + .size = sizeof(NICState), + .can_receive = igb_nc_can_receive, + .receive = igb_nc_receive, + .receive_iov = igb_nc_receive_iov, + .link_status_changed = igb_set_link_status, +}; + +/* + * EEPROM (NVM) contents documented in section 6.1, table 6-1: + * and in 6.10 Software accessed words. + */ +static const uint16_t igb_eeprom_template[] = { + /* Address |Compat.|OEM sp.| ImRev | OEM sp. */ + 0x0000, 0x0000, 0x0000, 0x0d34, 0xffff, 0x2010, 0xffff, 0xffff, + /* PBA |ICtrl1 | SSID | SVID | DevID |-------|ICtrl2 */ + 0x1040, 0xffff, 0x002b, 0x0000, 0x8086, 0x10c9, 0x0000, 0x70c3, + /* SwPin0| DevID | EESZ |-------|ICtrl3 |PCI-tc | MSIX | APtr */ + 0x0004, 0x10c9, 0x5c00, 0x0000, 0x2880, 0x0014, 0x4a40, 0x0060, + /* PCIe Init. Conf 1,2,3 |PCICtrl| LD1,3 |DDevID |DevRev | LD0,2 */ + 0x6cfb, 0xc7b0, 0x0abe, 0x0403, 0x0783, 0x10a6, 0x0001, 0x0602, + /* SwPin1| FunC |LAN-PWR|ManHwC |ICtrl3 | IOVct |VDevID |-------*/ + 0x0004, 0x0020, 0x0000, 0x004a, 0x2080, 0x00f5, 0x10ca, 0x0000, + /*---------------| LD1,3 | LD0,2 | ROEnd | ROSta | Wdog | VPD */ + 0x0000, 0x0000, 0x4784, 0x4602, 0x0000, 0x0000, 0x1000, 0xffff, + /* PCSet0| Ccfg0 |PXEver |IBAcap |PCSet1 | Ccfg1 |iSCVer | ?? */ + 0x0100, 0x4000, 0x131f, 0x4013, 0x0100, 0x4000, 0xffff, 0xffff, + /* PCSet2| Ccfg2 |PCSet3 | Ccfg3 | ?? |AltMacP| ?? |CHKSUM */ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x00e0, 0xffff, 0x0000, + /* NC-SIC */ + 0x0003, +}; + +static void igb_core_realize(IGBState *s) +{ + s->core.owner = &s->parent_obj; + s->core.owner_nic = s->nic; +} + +static void +igb_init_msix(IGBState *s) +{ + int i, res; + + res = msix_init(PCI_DEVICE(s), IGB_MSIX_VEC_NUM, + &s->msix, + E1000E_MSIX_IDX, 0, + &s->msix, + E1000E_MSIX_IDX, 0x2000, + 0x70, NULL); + + if (res < 0) { + trace_e1000e_msix_init_fail(res); + } else { + for (i = 0; i < IGB_MSIX_VEC_NUM; i++) { + msix_vector_use(PCI_DEVICE(s), i); + } + } +} + +static void +igb_cleanup_msix(IGBState *s) +{ + msix_unuse_all_vectors(PCI_DEVICE(s)); + msix_uninit(PCI_DEVICE(s), &s->msix, &s->msix); +} + +static void +igb_init_net_peer(IGBState *s, PCIDevice *pci_dev, uint8_t *macaddr) +{ + DeviceState *dev = DEVICE(pci_dev); + NetClientState *nc; + int i; + + s->nic = qemu_new_nic(&net_igb_info, &s->conf, + object_get_typename(OBJECT(s)), dev->id, s); + + s->core.max_queue_num = s->conf.peers.queues ? s->conf.peers.queues - 1 : 0; + + trace_e1000e_mac_set_permanent(MAC_ARG(macaddr)); + memcpy(s->core.permanent_mac, macaddr, sizeof(s->core.permanent_mac)); + + qemu_format_nic_info_str(qemu_get_queue(s->nic), macaddr); + + /* Setup virtio headers */ + for (i = 0; i < s->conf.peers.queues; i++) { + nc = qemu_get_subqueue(s->nic, i); + if (!nc->peer || !qemu_has_vnet_hdr(nc->peer)) { + trace_e1000e_cfg_support_virtio(false); + return; + } + } + + trace_e1000e_cfg_support_virtio(true); + s->core.has_vnet = true; + + for (i = 0; i < s->conf.peers.queues; i++) { + nc = qemu_get_subqueue(s->nic, i); + qemu_set_vnet_hdr_len(nc->peer, sizeof(struct virtio_net_hdr)); + qemu_using_vnet_hdr(nc->peer, true); + } +} + +static int +igb_add_pm_capability(PCIDevice *pdev, uint8_t offset, uint16_t pmc) +{ + Error *local_err = NULL; + int ret = pci_add_capability(pdev, PCI_CAP_ID_PM, offset, + PCI_PM_SIZEOF, &local_err); + + if (local_err) { + error_report_err(local_err); + return ret; + } + + pci_set_word(pdev->config + offset + PCI_PM_PMC, + PCI_PM_CAP_VER_1_1 | + pmc); + + pci_set_word(pdev->wmask + offset + PCI_PM_CTRL, + PCI_PM_CTRL_STATE_MASK | + PCI_PM_CTRL_PME_ENABLE | + PCI_PM_CTRL_DATA_SEL_MASK); + + pci_set_word(pdev->w1cmask + offset + PCI_PM_CTRL, + PCI_PM_CTRL_PME_STATUS); + + return ret; +} + +static void igb_pci_realize(PCIDevice *pci_dev, Error **errp) +{ + IGBState *s = IGB(pci_dev); + uint8_t *macaddr; + int ret; + + trace_e1000e_cb_pci_realize(); + + pci_dev->config_write = igb_write_config; + + pci_dev->config[PCI_CACHE_LINE_SIZE] = 0x10; + pci_dev->config[PCI_INTERRUPT_PIN] = 1; + + /* Define IO/MMIO regions */ + memory_region_init_io(&s->mmio, OBJECT(s), &mmio_ops, s, + "igb-mmio", E1000E_MMIO_SIZE); + pci_register_bar(pci_dev, E1000E_MMIO_IDX, + PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mmio); + + /* + * We provide a dummy implementation for the flash BAR + * for drivers that may theoretically probe for its presence. + */ + memory_region_init(&s->flash, OBJECT(s), + "igb-flash", E1000E_FLASH_SIZE); + pci_register_bar(pci_dev, E1000E_FLASH_IDX, + PCI_BASE_ADDRESS_SPACE_MEMORY, &s->flash); + + memory_region_init_io(&s->io, OBJECT(s), &io_ops, s, + "igb-io", E1000E_IO_SIZE); + pci_register_bar(pci_dev, E1000E_IO_IDX, + PCI_BASE_ADDRESS_SPACE_IO, &s->io); + + memory_region_init(&s->msix, OBJECT(s), "igb-msix", + E1000E_MSIX_SIZE); + pci_register_bar(pci_dev, E1000E_MSIX_IDX, + PCI_BASE_ADDRESS_MEM_TYPE_64, &s->msix); + + /* Create networking backend */ + qemu_macaddr_default_if_unset(&s->conf.macaddr); + macaddr = s->conf.macaddr.a; + + /* Add PCI capabilities in reverse order */ + assert(pcie_endpoint_cap_init(pci_dev, 0xa0) > 0); + + igb_init_msix(s); + + ret = msi_init(pci_dev, 0x50, 1, true, true, NULL); + if (ret) { + trace_e1000e_msi_init_fail(ret); + } + + if (igb_add_pm_capability(pci_dev, 0x40, PCI_PM_CAP_DSI) < 0) { + hw_error("Failed to initialize PM capability"); + } + + /* PCIe extended capabilities (in order) */ + if (pcie_aer_init(pci_dev, 1, 0x100, 0x40, errp) < 0) { + hw_error("Failed to initialize AER capability"); + } + + pcie_ari_init(pci_dev, 0x150, 1); + + pcie_sriov_pf_init(pci_dev, IGB_CAP_SRIOV_OFFSET, "igbvf", + IGB_82576_VF_DEV_ID, IGB_MAX_VF_FUNCTIONS, IGB_MAX_VF_FUNCTIONS, + IGB_VF_OFFSET, IGB_VF_STRIDE); + + pcie_sriov_pf_init_vf_bar(pci_dev, 0, + PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH, + 16 * KiB); + pcie_sriov_pf_init_vf_bar(pci_dev, 3, + PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH, + 16 * KiB); + + igb_init_net_peer(s, pci_dev, macaddr); + + /* Initialize core */ + igb_core_realize(s); + + igb_core_pci_realize(&s->core, + igb_eeprom_template, + sizeof(igb_eeprom_template), + macaddr); +} + +static void igb_pci_uninit(PCIDevice *pci_dev) +{ + IGBState *s = IGB(pci_dev); + + trace_e1000e_cb_pci_uninit(); + + igb_core_pci_uninit(&s->core); + + pcie_sriov_pf_exit(pci_dev); + pcie_cap_exit(pci_dev); + + qemu_del_nic(s->nic); + + igb_cleanup_msix(s); + msi_uninit(pci_dev); +} + +static void igb_qdev_reset_hold(Object *obj) +{ + PCIDevice *d = PCI_DEVICE(obj); + IGBState *s = IGB(obj); + + trace_e1000e_cb_qdev_reset_hold(); + + pcie_sriov_pf_disable_vfs(d); + igb_core_reset(&s->core); +} + +static int igb_pre_save(void *opaque) +{ + IGBState *s = opaque; + + trace_e1000e_cb_pre_save(); + + igb_core_pre_save(&s->core); + + return 0; +} + +static int igb_post_load(void *opaque, int version_id) +{ + IGBState *s = opaque; + + trace_e1000e_cb_post_load(); + return igb_core_post_load(&s->core); +} + +static const VMStateDescription igb_vmstate_tx = { + .name = "igb-tx", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_UINT16(vlan, struct igb_tx), + VMSTATE_UINT16(mss, struct igb_tx), + VMSTATE_BOOL(tse, struct igb_tx), + VMSTATE_BOOL(ixsm, struct igb_tx), + VMSTATE_BOOL(txsm, struct igb_tx), + VMSTATE_BOOL(first, struct igb_tx), + VMSTATE_BOOL(skip_cp, struct igb_tx), + VMSTATE_END_OF_LIST() + } +}; + +static const VMStateDescription igb_vmstate_intr_timer = { + .name = "igb-intr-timer", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_TIMER_PTR(timer, IGBIntrDelayTimer), + VMSTATE_BOOL(running, IGBIntrDelayTimer), + VMSTATE_END_OF_LIST() + } +}; + +#define VMSTATE_IGB_INTR_DELAY_TIMER(_f, _s) \ + VMSTATE_STRUCT(_f, _s, 0, \ + igb_vmstate_intr_timer, IGBIntrDelayTimer) + +#define VMSTATE_IGB_INTR_DELAY_TIMER_ARRAY(_f, _s, _num) \ + VMSTATE_STRUCT_ARRAY(_f, _s, _num, 0, \ + igb_vmstate_intr_timer, IGBIntrDelayTimer) + +static const VMStateDescription igb_vmstate = { + .name = "igb", + .version_id = 1, + .minimum_version_id = 1, + .pre_save = igb_pre_save, + .post_load = igb_post_load, + .fields = (VMStateField[]) { + VMSTATE_PCI_DEVICE(parent_obj, IGBState), + VMSTATE_MSIX(parent_obj, IGBState), + + VMSTATE_UINT32(ioaddr, IGBState), + VMSTATE_UINT8(core.rx_desc_len, IGBState), + VMSTATE_UINT16_ARRAY(core.eeprom, IGBState, IGB_EEPROM_SIZE), + VMSTATE_UINT16_ARRAY(core.phy, IGBState, MAX_PHY_REG_ADDRESS + 1), + VMSTATE_UINT32_ARRAY(core.mac, IGBState, E1000E_MAC_SIZE), + VMSTATE_UINT8_ARRAY(core.permanent_mac, IGBState, ETH_ALEN), + + VMSTATE_IGB_INTR_DELAY_TIMER_ARRAY(core.eitr, IGBState, + IGB_INTR_NUM), + + VMSTATE_UINT32_ARRAY(core.eitr_guest_value, IGBState, IGB_INTR_NUM), + + VMSTATE_STRUCT_ARRAY(core.tx, IGBState, IGB_NUM_QUEUES, 0, + igb_vmstate_tx, struct igb_tx), + + VMSTATE_INT64(core.timadj, IGBState), + + VMSTATE_END_OF_LIST() + } +}; + +static Property igb_properties[] = { + DEFINE_NIC_PROPERTIES(IGBState, conf), + DEFINE_PROP_END_OF_LIST(), +}; + +static void igb_class_init(ObjectClass *class, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(class); + ResettableClass *rc = RESETTABLE_CLASS(class); + PCIDeviceClass *c = PCI_DEVICE_CLASS(class); + + c->realize = igb_pci_realize; + c->exit = igb_pci_uninit; + c->vendor_id = PCI_VENDOR_ID_INTEL; + c->device_id = E1000_DEV_ID_82576; + c->revision = 1; + c->class_id = PCI_CLASS_NETWORK_ETHERNET; + + rc->phases.hold = igb_qdev_reset_hold; + + dc->desc = "Intel 82576 Gigabit Ethernet Controller"; + dc->vmsd = &igb_vmstate; + + device_class_set_props(dc, igb_properties); + set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); +} + +static void igb_instance_init(Object *obj) +{ + IGBState *s = IGB(obj); + device_add_bootindex_property(obj, &s->conf.bootindex, + "bootindex", "/ethernet-phy@0", + DEVICE(obj)); +} + +static const TypeInfo igb_info = { + .name = TYPE_IGB, + .parent = TYPE_PCI_DEVICE, + .instance_size = sizeof(IGBState), + .class_init = igb_class_init, + .instance_init = igb_instance_init, + .interfaces = (InterfaceInfo[]) { + { INTERFACE_PCIE_DEVICE }, + { } + }, +}; + +static void igb_register_types(void) +{ + type_register_static(&igb_info); +} + +type_init(igb_register_types) diff --git a/hw/net/igb_common.h b/hw/net/igb_common.h new file mode 100644 index 0000000000..69ac490f75 --- /dev/null +++ b/hw/net/igb_common.h @@ -0,0 +1,146 @@ +/* + * QEMU igb emulation - shared definitions + * + * Copyright (c) 2020-2023 Red Hat, Inc. + * Copyright (c) 2008 Qumranet + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef HW_NET_IGB_COMMON_H +#define HW_NET_IGB_COMMON_H + +#include "igb_regs.h" + +#define defreg(x) x = (E1000_##x >> 2) +#define defreg_indexed(x, i) x##i = (E1000_##x(i) >> 2) +#define defreg_indexeda(x, i) x##i##_A = (E1000_##x##_A(i) >> 2) + +#define defregd(x) defreg_indexed(x, 0), defreg_indexed(x, 1), \ + defreg_indexed(x, 2), defreg_indexed(x, 3), \ + defreg_indexed(x, 4), defreg_indexed(x, 5), \ + defreg_indexed(x, 6), defreg_indexed(x, 7), \ + defreg_indexed(x, 8), defreg_indexed(x, 9), \ + defreg_indexed(x, 10), defreg_indexed(x, 11), \ + defreg_indexed(x, 12), defreg_indexed(x, 13), \ + defreg_indexed(x, 14), defreg_indexed(x, 15), \ + defreg_indexeda(x, 0), defreg_indexeda(x, 1), \ + defreg_indexeda(x, 2), defreg_indexeda(x, 3) + +#define defregv(x) defreg_indexed(x, 0), defreg_indexed(x, 1), \ + defreg_indexed(x, 2), defreg_indexed(x, 3), \ + defreg_indexed(x, 4), defreg_indexed(x, 5), \ + defreg_indexed(x, 6), defreg_indexed(x, 7) + +enum { + defreg(CTRL), defreg(EECD), defreg(EERD), defreg(GPRC), + defreg(GPTC), defreg(ICR), defreg(ICS), defreg(IMC), + defreg(IMS), defreg(LEDCTL), defreg(MANC), defreg(MDIC), + defreg(MPC), defreg(RCTL), + defreg(STATUS), defreg(SWSM), defreg(TCTL), + defreg(TORH), defreg(TORL), defreg(TOTH), + defreg(TOTL), defreg(TPR), defreg(TPT), + defreg(WUFC), defreg(RA), defreg(MTA), defreg(CRCERRS), + defreg(VFTA), defreg(VET), + defreg(SCC), defreg(ECOL), + defreg(MCC), defreg(LATECOL), defreg(COLC), defreg(DC), + defreg(TNCRS), defreg(RLEC), + defreg(XONRXC), defreg(XONTXC), defreg(XOFFRXC), defreg(XOFFTXC), + defreg(FCRUC), defreg(TDFH), defreg(TDFT), + defreg(TDFHS), defreg(TDFTS), defreg(TDFPC), defreg(WUC), + defreg(WUS), defreg(RDFH), + defreg(RDFT), defreg(RDFHS), defreg(RDFTS), defreg(RDFPC), + defreg(IPAV), defreg(IP4AT), defreg(IP6AT), + defreg(WUPM), defreg(FFMT), + defreg(IAM), + defreg(GCR), defreg(TIMINCA), defreg(EIAC), defreg(CTRL_EXT), + defreg(IVAR0), defreg(MANC2H), + defreg(MFVAL), defreg(MDEF), defreg(FACTPS), defreg(FTFT), + defreg(RUC), defreg(ROC), defreg(RFC), defreg(RJC), + defreg(PRC64), defreg(PRC127), defreg(PRC255), defreg(PRC511), + defreg(PRC1023), defreg(PRC1522), defreg(PTC64), defreg(PTC127), + defreg(PTC255), defreg(PTC511), defreg(PTC1023), defreg(PTC1522), + defreg(GORCL), defreg(GORCH), defreg(GOTCL), defreg(GOTCH), + defreg(RNBC), defreg(BPRC), defreg(MPRC), defreg(RFCTL), + defreg(MPTC), defreg(BPTC), + defreg(IAC), defreg(MGTPRC), defreg(MGTPDC), defreg(MGTPTC), + defreg(TSCTC), defreg(RXCSUM), defreg(FUNCTAG), defreg(GSCL_1), + defreg(GSCL_2), defreg(GSCL_3), defreg(GSCL_4), defreg(GSCN_0), + defreg(GSCN_1), defreg(GSCN_2), defreg(GSCN_3), + defreg_indexed(EITR, 0), + defreg(MRQC), defreg(RETA), defreg(RSSRK), + defreg(PBACLR), defreg(FCAL), defreg(FCAH), defreg(FCT), + defreg(FCRTH), defreg(FCRTL), defreg(FCTTV), defreg(FCRTV), + defreg(FLA), defreg(FLOP), + defreg(MAVTV0), defreg(MAVTV1), defreg(MAVTV2), defreg(MAVTV3), + defreg(TXSTMPL), defreg(TXSTMPH), defreg(SYSTIML), defreg(SYSTIMH), + defreg(TIMADJL), defreg(TIMADJH), + defreg(RXSTMPH), defreg(RXSTMPL), defreg(RXSATRL), defreg(RXSATRH), + defreg(TIPG), + defreg(CTRL_DUP), + defreg(EEMNGCTL), + defreg(EEMNGDATA), + defreg(FLMNGCTL), + defreg(FLMNGDATA), + defreg(FLMNGCNT), + defreg(TSYNCRXCTL), + defreg(TSYNCTXCTL), + defreg(RLPML), + defreg(UTA), + + /* Aliases */ + defreg(RDFH_A), defreg(RDFT_A), defreg(TDFH_A), defreg(TDFT_A), + defreg(RA_A), defreg(VFTA_A), defreg(FCRTL_A), + + /* Additional regs used by IGB */ + defreg(FWSM), defreg(SW_FW_SYNC), + + defreg(EICS), defreg(EIMS), defreg(EIMC), defreg(EIAM), + defreg(EICR), defreg(IVAR_MISC), defreg(GPIE), + + defreg(RXPBS), defregd(RDBAL), defregd(RDBAH), defregd(RDLEN), + defregd(SRRCTL), defregd(RDH), defregd(RDT), + defregd(RXDCTL), defregd(RXCTL), defregd(RQDPC), defreg(RA2), + + defreg(TXPBS), defreg(TCTL_EXT), defreg(DTXCTL), defreg(HTCBDPC), + defregd(TDBAL), defregd(TDBAH), defregd(TDLEN), defregd(TDH), + defregd(TDT), defregd(TXDCTL), defregd(TXCTL), + defregd(TDWBAL), defregd(TDWBAH), + + defreg(VT_CTL), + + defregv(P2VMAILBOX), defregv(V2PMAILBOX), defreg(MBVFICR), defreg(MBVFIMR), + defreg(VFLRE), defreg(VFRE), defreg(VFTE), defreg(WVBR), + defreg(QDE), defreg(DTXSWC), defreg_indexed(VLVF, 0), + defregv(VMOLR), defreg(RPLOLR), defregv(VMBMEM), defregv(VMVIR), + + defregv(PVTCTRL), defregv(PVTEICS), defregv(PVTEIMS), defregv(PVTEIMC), + defregv(PVTEIAC), defregv(PVTEIAM), defregv(PVTEICR), defregv(PVFGPRC), + defregv(PVFGPTC), defregv(PVFGORC), defregv(PVFGOTC), defregv(PVFMPRC), + defregv(PVFGPRLBC), defregv(PVFGPTLBC), defregv(PVFGORLBC), defregv(PVFGOTLBC), + + defreg(MTA_A), + + defreg(VTIVAR), defreg(VTIVAR_MISC), +}; + +uint64_t igb_mmio_read(void *opaque, hwaddr addr, unsigned size); +void igb_mmio_write(void *opaque, hwaddr addr, uint64_t val, unsigned size); + +#endif diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c new file mode 100644 index 0000000000..a7c7bfdc75 --- /dev/null +++ b/hw/net/igb_core.c @@ -0,0 +1,4077 @@ +/* + * Core code for QEMU igb emulation + * + * Datasheet: + * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eg-gbe-datasheet.pdf + * + * Copyright (c) 2020-2023 Red Hat, Inc. + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Akihiko Odaki <akihiko.odaki@daynix.com> + * Gal Hammmer <gal.hammer@sap.com> + * Marcel Apfelbaum <marcel.apfelbaum@gmail.com> + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2008 Qumranet + * Based on work done by: + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "net/net.h" +#include "net/tap.h" +#include "hw/net/mii.h" +#include "hw/pci/msi.h" +#include "hw/pci/msix.h" +#include "sysemu/runstate.h" + +#include "net_tx_pkt.h" +#include "net_rx_pkt.h" + +#include "igb_common.h" +#include "e1000x_common.h" +#include "igb_core.h" + +#include "trace.h" + +#define E1000E_MAX_TX_FRAGS (64) + +union e1000_rx_desc_union { + struct e1000_rx_desc legacy; + union e1000_adv_rx_desc adv; +}; + +typedef struct IGBTxPktVmdqCallbackContext { + IGBCore *core; + NetClientState *nc; +} IGBTxPktVmdqCallbackContext; + +static ssize_t +igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt, + bool has_vnet, bool *external_tx); + +static inline void +igb_set_interrupt_cause(IGBCore *core, uint32_t val); + +static void igb_update_interrupt_state(IGBCore *core); +static void igb_reset(IGBCore *core, bool sw); + +static inline void +igb_raise_legacy_irq(IGBCore *core) +{ + trace_e1000e_irq_legacy_notify(true); + e1000x_inc_reg_if_not_full(core->mac, IAC); + pci_set_irq(core->owner, 1); +} + +static inline void +igb_lower_legacy_irq(IGBCore *core) +{ + trace_e1000e_irq_legacy_notify(false); + pci_set_irq(core->owner, 0); +} + +static void igb_msix_notify(IGBCore *core, unsigned int vector) +{ + PCIDevice *dev = core->owner; + uint16_t vfn; + + vfn = 8 - (vector + 2) / IGBVF_MSIX_VEC_NUM; + if (vfn < pcie_sriov_num_vfs(core->owner)) { + dev = pcie_sriov_get_vf_at_index(core->owner, vfn); + assert(dev); + vector = (vector + 2) % IGBVF_MSIX_VEC_NUM; + } else if (vector >= IGB_MSIX_VEC_NUM) { + qemu_log_mask(LOG_GUEST_ERROR, + "igb: Tried to use vector unavailable for PF"); + return; + } + + msix_notify(dev, vector); +} + +static inline void +igb_intrmgr_rearm_timer(IGBIntrDelayTimer *timer) +{ + int64_t delay_ns = (int64_t) timer->core->mac[timer->delay_reg] * + timer->delay_resolution_ns; + + trace_e1000e_irq_rearm_timer(timer->delay_reg << 2, delay_ns); + + timer_mod(timer->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + delay_ns); + + timer->running = true; +} + +static void +igb_intmgr_timer_resume(IGBIntrDelayTimer *timer) +{ + if (timer->running) { + igb_intrmgr_rearm_timer(timer); + } +} + +static void +igb_intmgr_timer_pause(IGBIntrDelayTimer *timer) +{ + if (timer->running) { + timer_del(timer->timer); + } +} + +static void +igb_intrmgr_on_msix_throttling_timer(void *opaque) +{ + IGBIntrDelayTimer *timer = opaque; + int idx = timer - &timer->core->eitr[0]; + + timer->running = false; + + trace_e1000e_irq_msix_notify_postponed_vec(idx); + igb_msix_notify(timer->core, idx); +} + +static void +igb_intrmgr_initialize_all_timers(IGBCore *core, bool create) +{ + int i; + + for (i = 0; i < IGB_INTR_NUM; i++) { + core->eitr[i].core = core; + core->eitr[i].delay_reg = EITR0 + i; + core->eitr[i].delay_resolution_ns = E1000_INTR_DELAY_NS_RES; + } + + if (!create) { + return; + } + + for (i = 0; i < IGB_INTR_NUM; i++) { + core->eitr[i].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, + igb_intrmgr_on_msix_throttling_timer, + &core->eitr[i]); + } +} + +static void +igb_intrmgr_resume(IGBCore *core) +{ + int i; + + for (i = 0; i < IGB_INTR_NUM; i++) { + igb_intmgr_timer_resume(&core->eitr[i]); + } +} + +static void +igb_intrmgr_pause(IGBCore *core) +{ + int i; + + for (i = 0; i < IGB_INTR_NUM; i++) { + igb_intmgr_timer_pause(&core->eitr[i]); + } +} + +static void +igb_intrmgr_reset(IGBCore *core) +{ + int i; + + for (i = 0; i < IGB_INTR_NUM; i++) { + if (core->eitr[i].running) { + timer_del(core->eitr[i].timer); + igb_intrmgr_on_msix_throttling_timer(&core->eitr[i]); + } + } +} + +static void +igb_intrmgr_pci_unint(IGBCore *core) +{ + int i; + + for (i = 0; i < IGB_INTR_NUM; i++) { + timer_free(core->eitr[i].timer); + } +} + +static void +igb_intrmgr_pci_realize(IGBCore *core) +{ + igb_intrmgr_initialize_all_timers(core, true); +} + +static inline bool +igb_rx_csum_enabled(IGBCore *core) +{ + return (core->mac[RXCSUM] & E1000_RXCSUM_PCSD) ? false : true; +} + +static inline bool +igb_rx_use_legacy_descriptor(IGBCore *core) +{ + /* + * TODO: If SRRCTL[n],DESCTYPE = 000b, the 82576 uses the legacy Rx + * descriptor. + */ + return false; +} + +static inline bool +igb_rss_enabled(IGBCore *core) +{ + return (core->mac[MRQC] & 3) == E1000_MRQC_ENABLE_RSS_MQ && + !igb_rx_csum_enabled(core) && + !igb_rx_use_legacy_descriptor(core); +} + +typedef struct E1000E_RSSInfo_st { + bool enabled; + uint32_t hash; + uint32_t queue; + uint32_t type; +} E1000E_RSSInfo; + +static uint32_t +igb_rss_get_hash_type(IGBCore *core, struct NetRxPkt *pkt) +{ + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; + + assert(igb_rss_enabled(core)); + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + + if (hasip4) { + trace_e1000e_rx_rss_ip4(l4hdr_proto, core->mac[MRQC], + E1000_MRQC_EN_TCPIPV4(core->mac[MRQC]), + E1000_MRQC_EN_IPV4(core->mac[MRQC])); + + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && + E1000_MRQC_EN_TCPIPV4(core->mac[MRQC])) { + return E1000_MRQ_RSS_TYPE_IPV4TCP; + } + + if (E1000_MRQC_EN_IPV4(core->mac[MRQC])) { + return E1000_MRQ_RSS_TYPE_IPV4; + } + } else if (hasip6) { + eth_ip6_hdr_info *ip6info = net_rx_pkt_get_ip6_info(pkt); + + bool ex_dis = core->mac[RFCTL] & E1000_RFCTL_IPV6_EX_DIS; + bool new_ex_dis = core->mac[RFCTL] & E1000_RFCTL_NEW_IPV6_EXT_DIS; + + /* + * Following two traces must not be combined because resulting + * event will have 11 arguments totally and some trace backends + * (at least "ust") have limitation of maximum 10 arguments per + * event. Events with more arguments fail to compile for + * backends like these. + */ + trace_e1000e_rx_rss_ip6_rfctl(core->mac[RFCTL]); + trace_e1000e_rx_rss_ip6(ex_dis, new_ex_dis, l4hdr_proto, + ip6info->has_ext_hdrs, + ip6info->rss_ex_dst_valid, + ip6info->rss_ex_src_valid, + core->mac[MRQC], + E1000_MRQC_EN_TCPIPV6(core->mac[MRQC]), + E1000_MRQC_EN_IPV6EX(core->mac[MRQC]), + E1000_MRQC_EN_IPV6(core->mac[MRQC])); + + if ((!ex_dis || !ip6info->has_ext_hdrs) && + (!new_ex_dis || !(ip6info->rss_ex_dst_valid || + ip6info->rss_ex_src_valid))) { + + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && + E1000_MRQC_EN_TCPIPV6(core->mac[MRQC])) { + return E1000_MRQ_RSS_TYPE_IPV6TCP; + } + + if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) { + return E1000_MRQ_RSS_TYPE_IPV6EX; + } + + } + + if (E1000_MRQC_EN_IPV6(core->mac[MRQC])) { + return E1000_MRQ_RSS_TYPE_IPV6; + } + + } + + return E1000_MRQ_RSS_TYPE_NONE; +} + +static uint32_t +igb_rss_calc_hash(IGBCore *core, struct NetRxPkt *pkt, E1000E_RSSInfo *info) +{ + NetRxPktRssType type; + + assert(igb_rss_enabled(core)); + + switch (info->type) { + case E1000_MRQ_RSS_TYPE_IPV4: + type = NetPktRssIpV4; + break; + case E1000_MRQ_RSS_TYPE_IPV4TCP: + type = NetPktRssIpV4Tcp; + break; + case E1000_MRQ_RSS_TYPE_IPV6TCP: + type = NetPktRssIpV6TcpEx; + break; + case E1000_MRQ_RSS_TYPE_IPV6: + type = NetPktRssIpV6; + break; + case E1000_MRQ_RSS_TYPE_IPV6EX: + type = NetPktRssIpV6Ex; + break; + default: + assert(false); + return 0; + } + + return net_rx_pkt_calc_rss_hash(pkt, type, (uint8_t *) &core->mac[RSSRK]); +} + +static void +igb_rss_parse_packet(IGBCore *core, struct NetRxPkt *pkt, bool tx, + E1000E_RSSInfo *info) +{ + trace_e1000e_rx_rss_started(); + + if (tx || !igb_rss_enabled(core)) { + info->enabled = false; + info->hash = 0; + info->queue = 0; + info->type = 0; + trace_e1000e_rx_rss_disabled(); + return; + } + + info->enabled = true; + + info->type = igb_rss_get_hash_type(core, pkt); + + trace_e1000e_rx_rss_type(info->type); + + if (info->type == E1000_MRQ_RSS_TYPE_NONE) { + info->hash = 0; + info->queue = 0; + return; + } + + info->hash = igb_rss_calc_hash(core, pkt, info); + info->queue = E1000_RSS_QUEUE(&core->mac[RETA], info->hash); +} + +static bool +igb_setup_tx_offloads(IGBCore *core, struct igb_tx *tx) +{ + if (tx->tse) { + if (!net_tx_pkt_build_vheader(tx->tx_pkt, true, true, tx->mss)) { + return false; + } + + net_tx_pkt_update_ip_checksums(tx->tx_pkt); + e1000x_inc_reg_if_not_full(core->mac, TSCTC); + return true; + } + + if (tx->txsm) { + if (!net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0)) { + return false; + } + } + + if (tx->ixsm) { + net_tx_pkt_update_ip_hdr_checksum(tx->tx_pkt); + } + + return true; +} + +static void igb_tx_pkt_mac_callback(void *core, + const struct iovec *iov, + int iovcnt, + const struct iovec *virt_iov, + int virt_iovcnt) +{ + igb_receive_internal(core, virt_iov, virt_iovcnt, true, NULL); +} + +static void igb_tx_pkt_vmdq_callback(void *opaque, + const struct iovec *iov, + int iovcnt, + const struct iovec *virt_iov, + int virt_iovcnt) +{ + IGBTxPktVmdqCallbackContext *context = opaque; + bool external_tx; + + igb_receive_internal(context->core, virt_iov, virt_iovcnt, true, + &external_tx); + + if (external_tx) { + if (context->core->has_vnet) { + qemu_sendv_packet(context->nc, virt_iov, virt_iovcnt); + } else { + qemu_sendv_packet(context->nc, iov, iovcnt); + } + } +} + +/* TX Packets Switching (7.10.3.6) */ +static bool igb_tx_pkt_switch(IGBCore *core, struct igb_tx *tx, + NetClientState *nc) +{ + IGBTxPktVmdqCallbackContext context; + + /* TX switching is only used to serve VM to VM traffic. */ + if (!(core->mac[MRQC] & 1)) { + goto send_out; + } + + /* TX switching requires DTXSWC.Loopback_en bit enabled. */ + if (!(core->mac[DTXSWC] & E1000_DTXSWC_VMDQ_LOOPBACK_EN)) { + goto send_out; + } + + context.core = core; + context.nc = nc; + + return net_tx_pkt_send_custom(tx->tx_pkt, false, + igb_tx_pkt_vmdq_callback, &context); + +send_out: + return net_tx_pkt_send(tx->tx_pkt, nc); +} + +static bool +igb_tx_pkt_send(IGBCore *core, struct igb_tx *tx, int queue_index) +{ + int target_queue = MIN(core->max_queue_num, queue_index); + NetClientState *queue = qemu_get_subqueue(core->owner_nic, target_queue); + + if (!igb_setup_tx_offloads(core, tx)) { + return false; + } + + net_tx_pkt_dump(tx->tx_pkt); + + if ((core->phy[MII_BMCR] & MII_BMCR_LOOPBACK) || + ((core->mac[RCTL] & E1000_RCTL_LBM_MAC) == E1000_RCTL_LBM_MAC)) { + return net_tx_pkt_send_custom(tx->tx_pkt, false, + igb_tx_pkt_mac_callback, core); + } else { + return igb_tx_pkt_switch(core, tx, queue); + } +} + +static void +igb_on_tx_done_update_stats(IGBCore *core, struct NetTxPkt *tx_pkt) +{ + static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511, + PTC1023, PTC1522 }; + + size_t tot_len = net_tx_pkt_get_total_len(tx_pkt) + 4; + + e1000x_increase_size_stats(core->mac, PTCregs, tot_len); + e1000x_inc_reg_if_not_full(core->mac, TPT); + e1000x_grow_8reg_if_not_full(core->mac, TOTL, tot_len); + + switch (net_tx_pkt_get_packet_type(tx_pkt)) { + case ETH_PKT_BCAST: + e1000x_inc_reg_if_not_full(core->mac, BPTC); + break; + case ETH_PKT_MCAST: + e1000x_inc_reg_if_not_full(core->mac, MPTC); + break; + case ETH_PKT_UCAST: + break; + default: + g_assert_not_reached(); + } + + core->mac[GPTC] = core->mac[TPT]; + core->mac[GOTCL] = core->mac[TOTL]; + core->mac[GOTCH] = core->mac[TOTH]; +} + +static void +igb_process_tx_desc(IGBCore *core, + struct igb_tx *tx, + union e1000_adv_tx_desc *tx_desc, + int queue_index) +{ + struct e1000_adv_tx_context_desc *tx_ctx_desc; + uint32_t cmd_type_len; + uint32_t olinfo_status; + uint64_t buffer_addr; + uint16_t length; + + cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len); + + if (cmd_type_len & E1000_ADVTXD_DCMD_DEXT) { + if ((cmd_type_len & E1000_ADVTXD_DTYP_DATA) == + E1000_ADVTXD_DTYP_DATA) { + /* advanced transmit data descriptor */ + if (tx->first) { + olinfo_status = le32_to_cpu(tx_desc->read.olinfo_status); + + tx->tse = !!(cmd_type_len & E1000_ADVTXD_DCMD_TSE); + tx->ixsm = !!(olinfo_status & E1000_ADVTXD_POTS_IXSM); + tx->txsm = !!(olinfo_status & E1000_ADVTXD_POTS_TXSM); + + tx->first = false; + } + } else if ((cmd_type_len & E1000_ADVTXD_DTYP_CTXT) == + E1000_ADVTXD_DTYP_CTXT) { + /* advanced transmit context descriptor */ + tx_ctx_desc = (struct e1000_adv_tx_context_desc *)tx_desc; + tx->vlan = le32_to_cpu(tx_ctx_desc->vlan_macip_lens) >> 16; + tx->mss = le32_to_cpu(tx_ctx_desc->mss_l4len_idx) >> 16; + return; + } else { + /* unknown descriptor type */ + return; + } + } else { + /* legacy descriptor */ + + /* TODO: Implement a support for legacy descriptors (7.2.2.1). */ + } + + buffer_addr = le64_to_cpu(tx_desc->read.buffer_addr); + length = cmd_type_len & 0xFFFF; + + if (!tx->skip_cp) { + if (!net_tx_pkt_add_raw_fragment(tx->tx_pkt, buffer_addr, length)) { + tx->skip_cp = true; + } + } + + if (cmd_type_len & E1000_TXD_CMD_EOP) { + if (!tx->skip_cp && net_tx_pkt_parse(tx->tx_pkt)) { + if (cmd_type_len & E1000_TXD_CMD_VLE) { + net_tx_pkt_setup_vlan_header_ex(tx->tx_pkt, tx->vlan, + core->mac[VET] & 0xffff); + } + if (igb_tx_pkt_send(core, tx, queue_index)) { + igb_on_tx_done_update_stats(core, tx->tx_pkt); + } + } + + tx->first = true; + tx->skip_cp = false; + net_tx_pkt_reset(tx->tx_pkt); + } +} + +static uint32_t igb_tx_wb_eic(IGBCore *core, int queue_idx) +{ + uint32_t n, ent = 0; + + n = igb_ivar_entry_tx(queue_idx); + ent = (core->mac[IVAR0 + n / 4] >> (8 * (n % 4))) & 0xff; + + return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0; +} + +static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx) +{ + uint32_t n, ent = 0; + + n = igb_ivar_entry_rx(queue_idx); + ent = (core->mac[IVAR0 + n / 4] >> (8 * (n % 4))) & 0xff; + + return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0; +} + +typedef struct E1000E_RingInfo_st { + int dbah; + int dbal; + int dlen; + int dh; + int dt; + int idx; +} E1000E_RingInfo; + +static inline bool +igb_ring_empty(IGBCore *core, const E1000E_RingInfo *r) +{ + return core->mac[r->dh] == core->mac[r->dt] || + core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN; +} + +static inline uint64_t +igb_ring_base(IGBCore *core, const E1000E_RingInfo *r) +{ + uint64_t bah = core->mac[r->dbah]; + uint64_t bal = core->mac[r->dbal]; + + return (bah << 32) + bal; +} + +static inline uint64_t +igb_ring_head_descr(IGBCore *core, const E1000E_RingInfo *r) +{ + return igb_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh]; +} + +static inline void +igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count) +{ + core->mac[r->dh] += count; + + if (core->mac[r->dh] * E1000_RING_DESC_LEN >= core->mac[r->dlen]) { + core->mac[r->dh] = 0; + } +} + +static inline uint32_t +igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r) +{ + trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen], + core->mac[r->dh], core->mac[r->dt]); + + if (core->mac[r->dh] <= core->mac[r->dt]) { + return core->mac[r->dt] - core->mac[r->dh]; + } + + if (core->mac[r->dh] > core->mac[r->dt]) { + return core->mac[r->dlen] / E1000_RING_DESC_LEN + + core->mac[r->dt] - core->mac[r->dh]; + } + + g_assert_not_reached(); + return 0; +} + +static inline bool +igb_ring_enabled(IGBCore *core, const E1000E_RingInfo *r) +{ + return core->mac[r->dlen] > 0; +} + +typedef struct IGB_TxRing_st { + const E1000E_RingInfo *i; + struct igb_tx *tx; +} IGB_TxRing; + +static inline int +igb_mq_queue_idx(int base_reg_idx, int reg_idx) +{ + return (reg_idx - base_reg_idx) / 16; +} + +static inline void +igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx) +{ + static const E1000E_RingInfo i[IGB_NUM_QUEUES] = { + { TDBAH0, TDBAL0, TDLEN0, TDH0, TDT0, 0 }, + { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 }, + { TDBAH2, TDBAL2, TDLEN2, TDH2, TDT2, 2 }, + { TDBAH3, TDBAL3, TDLEN3, TDH3, TDT3, 3 }, + { TDBAH4, TDBAL4, TDLEN4, TDH4, TDT4, 4 }, + { TDBAH5, TDBAL5, TDLEN5, TDH5, TDT5, 5 }, + { TDBAH6, TDBAL6, TDLEN6, TDH6, TDT6, 6 }, + { TDBAH7, TDBAL7, TDLEN7, TDH7, TDT7, 7 }, + { TDBAH8, TDBAL8, TDLEN8, TDH8, TDT8, 8 }, + { TDBAH9, TDBAL9, TDLEN9, TDH9, TDT9, 9 }, + { TDBAH10, TDBAL10, TDLEN10, TDH10, TDT10, 10 }, + { TDBAH11, TDBAL11, TDLEN11, TDH11, TDT11, 11 }, + { TDBAH12, TDBAL12, TDLEN12, TDH12, TDT12, 12 }, + { TDBAH13, TDBAL13, TDLEN13, TDH13, TDT13, 13 }, + { TDBAH14, TDBAL14, TDLEN14, TDH14, TDT14, 14 }, + { TDBAH15, TDBAL15, TDLEN15, TDH15, TDT15, 15 } + }; + + assert(idx < ARRAY_SIZE(i)); + + txr->i = &i[idx]; + txr->tx = &core->tx[idx]; +} + +typedef struct E1000E_RxRing_st { + const E1000E_RingInfo *i; +} E1000E_RxRing; + +static inline void +igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx) +{ + static const E1000E_RingInfo i[IGB_NUM_QUEUES] = { + { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 }, + { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 }, + { RDBAH2, RDBAL2, RDLEN2, RDH2, RDT2, 2 }, + { RDBAH3, RDBAL3, RDLEN3, RDH3, RDT3, 3 }, + { RDBAH4, RDBAL4, RDLEN4, RDH4, RDT4, 4 }, + { RDBAH5, RDBAL5, RDLEN5, RDH5, RDT5, 5 }, + { RDBAH6, RDBAL6, RDLEN6, RDH6, RDT6, 6 }, + { RDBAH7, RDBAL7, RDLEN7, RDH7, RDT7, 7 }, + { RDBAH8, RDBAL8, RDLEN8, RDH8, RDT8, 8 }, + { RDBAH9, RDBAL9, RDLEN9, RDH9, RDT9, 9 }, + { RDBAH10, RDBAL10, RDLEN10, RDH10, RDT10, 10 }, + { RDBAH11, RDBAL11, RDLEN11, RDH11, RDT11, 11 }, + { RDBAH12, RDBAL12, RDLEN12, RDH12, RDT12, 12 }, + { RDBAH13, RDBAL13, RDLEN13, RDH13, RDT13, 13 }, + { RDBAH14, RDBAL14, RDLEN14, RDH14, RDT14, 14 }, + { RDBAH15, RDBAL15, RDLEN15, RDH15, RDT15, 15 } + }; + + assert(idx < ARRAY_SIZE(i)); + + rxr->i = &i[idx]; +} + +static uint32_t +igb_txdesc_writeback(IGBCore *core, dma_addr_t base, + union e1000_adv_tx_desc *tx_desc, + const E1000E_RingInfo *txi) +{ + PCIDevice *d; + uint32_t cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len); + uint64_t tdwba; + + tdwba = core->mac[E1000_TDWBAL(txi->idx) >> 2]; + tdwba |= (uint64_t)core->mac[E1000_TDWBAH(txi->idx) >> 2] << 32; + + if (!(cmd_type_len & E1000_TXD_CMD_RS)) { + return 0; + } + + d = pcie_sriov_get_vf_at_index(core->owner, txi->idx % 8); + if (!d) { + d = core->owner; + } + + if (tdwba & 1) { + uint32_t buffer = cpu_to_le32(core->mac[txi->dh]); + pci_dma_write(d, tdwba & ~3, &buffer, sizeof(buffer)); + } else { + uint32_t status = le32_to_cpu(tx_desc->wb.status) | E1000_TXD_STAT_DD; + + tx_desc->wb.status = cpu_to_le32(status); + pci_dma_write(d, base + offsetof(union e1000_adv_tx_desc, wb), + &tx_desc->wb, sizeof(tx_desc->wb)); + } + + return igb_tx_wb_eic(core, txi->idx); +} + +static void +igb_start_xmit(IGBCore *core, const IGB_TxRing *txr) +{ + PCIDevice *d; + dma_addr_t base; + union e1000_adv_tx_desc desc; + const E1000E_RingInfo *txi = txr->i; + uint32_t eic = 0; + + /* TODO: check if the queue itself is enabled too. */ + if (!(core->mac[TCTL] & E1000_TCTL_EN)) { + trace_e1000e_tx_disabled(); + return; + } + + d = pcie_sriov_get_vf_at_index(core->owner, txi->idx % 8); + if (!d) { + d = core->owner; + } + + while (!igb_ring_empty(core, txi)) { + base = igb_ring_head_descr(core, txi); + + pci_dma_read(d, base, &desc, sizeof(desc)); + + trace_e1000e_tx_descr((void *)(intptr_t)desc.read.buffer_addr, + desc.read.cmd_type_len, desc.wb.status); + + igb_process_tx_desc(core, txr->tx, &desc, txi->idx); + igb_ring_advance(core, txi, 1); + eic |= igb_txdesc_writeback(core, base, &desc, txi); + } + + if (eic) { + core->mac[EICR] |= eic; + igb_set_interrupt_cause(core, E1000_ICR_TXDW); + } +} + +static uint32_t +igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r) +{ + uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2]; + uint32_t bsizepkt = srrctl & E1000_SRRCTL_BSIZEPKT_MASK; + if (bsizepkt) { + return bsizepkt << E1000_SRRCTL_BSIZEPKT_SHIFT; + } + + return e1000x_rxbufsize(core->mac[RCTL]); +} + +static bool +igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size) +{ + uint32_t bufs = igb_ring_free_descr_num(core, r); + uint32_t bufsize = igb_rxbufsize(core, r); + + trace_e1000e_rx_has_buffers(r->idx, bufs, total_size, bufsize); + + return total_size <= bufs / (core->rx_desc_len / E1000_MIN_RX_DESC_LEN) * + bufsize; +} + +void +igb_start_recv(IGBCore *core) +{ + int i; + + trace_e1000e_rx_start_recv(); + + for (i = 0; i <= core->max_queue_num; i++) { + qemu_flush_queued_packets(qemu_get_subqueue(core->owner_nic, i)); + } +} + +bool +igb_can_receive(IGBCore *core) +{ + int i; + + if (!e1000x_rx_ready(core->owner, core->mac)) { + return false; + } + + for (i = 0; i < IGB_NUM_QUEUES; i++) { + E1000E_RxRing rxr; + + igb_rx_ring_init(core, &rxr, i); + if (igb_ring_enabled(core, rxr.i) && igb_has_rxbufs(core, rxr.i, 1)) { + trace_e1000e_rx_can_recv(); + return true; + } + } + + trace_e1000e_rx_can_recv_rings_full(); + return false; +} + +ssize_t +igb_receive(IGBCore *core, const uint8_t *buf, size_t size) +{ + const struct iovec iov = { + .iov_base = (uint8_t *)buf, + .iov_len = size + }; + + return igb_receive_iov(core, &iov, 1); +} + +static inline bool +igb_rx_l3_cso_enabled(IGBCore *core) +{ + return !!(core->mac[RXCSUM] & E1000_RXCSUM_IPOFLD); +} + +static inline bool +igb_rx_l4_cso_enabled(IGBCore *core) +{ + return !!(core->mac[RXCSUM] & E1000_RXCSUM_TUOFLD); +} + +static uint16_t igb_receive_assign(IGBCore *core, const struct eth_header *ehdr, + E1000E_RSSInfo *rss_info, bool *external_tx) +{ + static const int ta_shift[] = { 4, 3, 2, 0 }; + uint32_t f, ra[2], *macp, rctl = core->mac[RCTL]; + uint16_t queues = 0; + uint16_t vid = lduw_be_p(&PKT_GET_VLAN_HDR(ehdr)->h_tci) & VLAN_VID_MASK; + bool accepted = false; + int i; + + memset(rss_info, 0, sizeof(E1000E_RSSInfo)); + + if (external_tx) { + *external_tx = true; + } + + if (e1000x_is_vlan_packet(ehdr, core->mac[VET] & 0xffff) && + e1000x_vlan_rx_filter_enabled(core->mac)) { + uint32_t vfta = + ldl_le_p((uint32_t *)(core->mac + VFTA) + + ((vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK)); + if ((vfta & (1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK))) == 0) { + trace_e1000e_rx_flt_vlan_mismatch(vid); + return queues; + } else { + trace_e1000e_rx_flt_vlan_match(vid); + } + } + + if (core->mac[MRQC] & 1) { + if (is_broadcast_ether_addr(ehdr->h_dest)) { + for (i = 0; i < 8; i++) { + if (core->mac[VMOLR0 + i] & E1000_VMOLR_BAM) { + queues |= BIT(i); + } + } + } else { + for (macp = core->mac + RA; macp < core->mac + RA + 32; macp += 2) { + if (!(macp[1] & E1000_RAH_AV)) { + continue; + } + ra[0] = cpu_to_le32(macp[0]); + ra[1] = cpu_to_le32(macp[1]); + if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) { + queues |= (macp[1] & E1000_RAH_POOL_MASK) / E1000_RAH_POOL_1; + } + } + + for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) { + if (!(macp[1] & E1000_RAH_AV)) { + continue; + } + ra[0] = cpu_to_le32(macp[0]); + ra[1] = cpu_to_le32(macp[1]); + if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) { + queues |= (macp[1] & E1000_RAH_POOL_MASK) / E1000_RAH_POOL_1; + } + } + + if (!queues) { + macp = core->mac + (is_multicast_ether_addr(ehdr->h_dest) ? MTA : UTA); + + f = ta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3]; + f = (((ehdr->h_dest[5] << 8) | ehdr->h_dest[4]) >> f) & 0xfff; + if (macp[f >> 5] & (1 << (f & 0x1f))) { + for (i = 0; i < 8; i++) { + if (core->mac[VMOLR0 + i] & E1000_VMOLR_ROMPE) { + queues |= BIT(i); + } + } + } + } else if (is_unicast_ether_addr(ehdr->h_dest) && external_tx) { + *external_tx = false; + } + } + + if (e1000x_vlan_rx_filter_enabled(core->mac)) { + uint16_t mask = 0; + + if (e1000x_is_vlan_packet(ehdr, core->mac[VET] & 0xffff)) { + for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) { + if ((core->mac[VLVF0 + i] & E1000_VLVF_VLANID_MASK) == vid && + (core->mac[VLVF0 + i] & E1000_VLVF_VLANID_ENABLE)) { + uint32_t poolsel = core->mac[VLVF0 + i] & E1000_VLVF_POOLSEL_MASK; + mask |= poolsel >> E1000_VLVF_POOLSEL_SHIFT; + } + } + } else { + for (i = 0; i < 8; i++) { + if (core->mac[VMOLR0 + i] & E1000_VMOLR_AUPE) { + mask |= BIT(i); + } + } + } + + queues &= mask; + } + + if (is_unicast_ether_addr(ehdr->h_dest) && !queues && !external_tx && + !(core->mac[VT_CTL] & E1000_VT_CTL_DISABLE_DEF_POOL)) { + uint32_t def_pl = core->mac[VT_CTL] & E1000_VT_CTL_DEFAULT_POOL_MASK; + queues = BIT(def_pl >> E1000_VT_CTL_DEFAULT_POOL_SHIFT); + } + + igb_rss_parse_packet(core, core->rx_pkt, external_tx != NULL, rss_info); + if (rss_info->queue & 1) { + queues <<= 8; + } + } else { + switch (net_rx_pkt_get_packet_type(core->rx_pkt)) { + case ETH_PKT_UCAST: + if (rctl & E1000_RCTL_UPE) { + accepted = true; /* promiscuous ucast */ + } + break; + + case ETH_PKT_BCAST: + if (rctl & E1000_RCTL_BAM) { + accepted = true; /* broadcast enabled */ + } + break; + + case ETH_PKT_MCAST: + if (rctl & E1000_RCTL_MPE) { + accepted = true; /* promiscuous mcast */ + } + break; + + default: + g_assert_not_reached(); + } + + if (!accepted) { + accepted = e1000x_rx_group_filter(core->mac, ehdr->h_dest); + } + + if (!accepted) { + for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) { + if (!(macp[1] & E1000_RAH_AV)) { + continue; + } + ra[0] = cpu_to_le32(macp[0]); + ra[1] = cpu_to_le32(macp[1]); + if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) { + trace_e1000x_rx_flt_ucast_match((int)(macp - core->mac - RA2) / 2, + MAC_ARG(ehdr->h_dest)); + + accepted = true; + break; + } + } + } + + if (accepted) { + igb_rss_parse_packet(core, core->rx_pkt, false, rss_info); + queues = BIT(rss_info->queue); + } + } + + return queues; +} + +static inline void +igb_read_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, + hwaddr *buff_addr) +{ + *buff_addr = le64_to_cpu(desc->buffer_addr); +} + +static inline void +igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, + hwaddr *buff_addr) +{ + *buff_addr = le64_to_cpu(desc->read.pkt_addr); +} + +static inline void +igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc, + hwaddr *buff_addr) +{ + if (igb_rx_use_legacy_descriptor(core)) { + igb_read_lgcy_rx_descr(core, &desc->legacy, buff_addr); + } else { + igb_read_adv_rx_descr(core, &desc->adv, buff_addr); + } +} + +static void +igb_verify_csum_in_sw(IGBCore *core, + struct NetRxPkt *pkt, + uint32_t *status_flags, + EthL4HdrProto l4hdr_proto) +{ + bool csum_valid; + uint32_t csum_error; + + if (igb_rx_l3_cso_enabled(core)) { + if (!net_rx_pkt_validate_l3_csum(pkt, &csum_valid)) { + trace_e1000e_rx_metadata_l3_csum_validation_failed(); + } else { + csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_IPE; + *status_flags |= E1000_RXD_STAT_IPCS | csum_error; + } + } else { + trace_e1000e_rx_metadata_l3_cso_disabled(); + } + + if (!igb_rx_l4_cso_enabled(core)) { + trace_e1000e_rx_metadata_l4_cso_disabled(); + return; + } + + if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) { + trace_e1000e_rx_metadata_l4_csum_validation_failed(); + return; + } + + csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_TCPE; + *status_flags |= E1000_RXD_STAT_TCPCS | csum_error; + + if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { + *status_flags |= E1000_RXD_STAT_UDPCS; + } +} + +static void +igb_build_rx_metadata(IGBCore *core, + struct NetRxPkt *pkt, + bool is_eop, + const E1000E_RSSInfo *rss_info, + uint16_t *pkt_info, uint16_t *hdr_info, + uint32_t *rss, + uint32_t *status_flags, + uint16_t *ip_id, + uint16_t *vlan_tag) +{ + struct virtio_net_hdr *vhdr; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; + uint32_t pkt_type; + + *status_flags = E1000_RXD_STAT_DD; + + /* No additional metadata needed for non-EOP descriptors */ + /* TODO: EOP apply only to status so don't skip whole function. */ + if (!is_eop) { + goto func_exit; + } + + *status_flags |= E1000_RXD_STAT_EOP; + + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + trace_e1000e_rx_metadata_protocols(hasip4, hasip6, l4hdr_proto); + + /* VLAN state */ + if (net_rx_pkt_is_vlan_stripped(pkt)) { + *status_flags |= E1000_RXD_STAT_VP; + *vlan_tag = cpu_to_le16(net_rx_pkt_get_vlan_tag(pkt)); + trace_e1000e_rx_metadata_vlan(*vlan_tag); + } + + /* Packet parsing results */ + if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) { + if (rss_info->enabled) { + *rss = cpu_to_le32(rss_info->hash); + trace_igb_rx_metadata_rss(*rss); + } + } else if (hasip4) { + *status_flags |= E1000_RXD_STAT_IPIDV; + *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt)); + trace_e1000e_rx_metadata_ip_id(*ip_id); + } + + if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && net_rx_pkt_is_tcp_ack(pkt)) { + *status_flags |= E1000_RXD_STAT_ACK; + trace_e1000e_rx_metadata_ack(); + } + + if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) { + trace_e1000e_rx_metadata_ipv6_filtering_disabled(); + pkt_type = E1000_RXD_PKT_MAC; + } else if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP || + l4hdr_proto == ETH_L4_HDR_PROTO_UDP) { + pkt_type = hasip4 ? E1000_RXD_PKT_IP4_XDP : E1000_RXD_PKT_IP6_XDP; + } else if (hasip4 || hasip6) { + pkt_type = hasip4 ? E1000_RXD_PKT_IP4 : E1000_RXD_PKT_IP6; + } else { + pkt_type = E1000_RXD_PKT_MAC; + } + + trace_e1000e_rx_metadata_pkt_type(pkt_type); + + if (pkt_info) { + if (rss_info->enabled) { + *pkt_info = rss_info->type; + } + + *pkt_info |= (pkt_type << 4); + } else { + *status_flags |= E1000_RXD_PKT_TYPE(pkt_type); + } + + if (hdr_info) { + *hdr_info = 0; + } + + /* RX CSO information */ + if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) { + trace_e1000e_rx_metadata_ipv6_sum_disabled(); + goto func_exit; + } + + vhdr = net_rx_pkt_get_vhdr(pkt); + + if (!(vhdr->flags & VIRTIO_NET_HDR_F_DATA_VALID) && + !(vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) { + trace_e1000e_rx_metadata_virthdr_no_csum_info(); + igb_verify_csum_in_sw(core, pkt, status_flags, l4hdr_proto); + goto func_exit; + } + + if (igb_rx_l3_cso_enabled(core)) { + *status_flags |= hasip4 ? E1000_RXD_STAT_IPCS : 0; + } else { + trace_e1000e_rx_metadata_l3_cso_disabled(); + } + + if (igb_rx_l4_cso_enabled(core)) { + switch (l4hdr_proto) { + case ETH_L4_HDR_PROTO_TCP: + *status_flags |= E1000_RXD_STAT_TCPCS; + break; + + case ETH_L4_HDR_PROTO_UDP: + *status_flags |= E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS; + break; + + default: + goto func_exit; + } + } else { + trace_e1000e_rx_metadata_l4_cso_disabled(); + } + + trace_e1000e_rx_metadata_status_flags(*status_flags); + +func_exit: + *status_flags = cpu_to_le32(*status_flags); +} + +static inline void +igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc, + struct NetRxPkt *pkt, + const E1000E_RSSInfo *rss_info, + uint16_t length) +{ + uint32_t status_flags, rss; + uint16_t ip_id; + + assert(!rss_info->enabled); + desc->length = cpu_to_le16(length); + desc->csum = 0; + + igb_build_rx_metadata(core, pkt, pkt != NULL, + rss_info, + NULL, NULL, &rss, + &status_flags, &ip_id, + &desc->special); + desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24); + desc->status = (uint8_t) le32_to_cpu(status_flags); +} + +static inline void +igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc, + struct NetRxPkt *pkt, + const E1000E_RSSInfo *rss_info, + uint16_t length) +{ + memset(&desc->wb, 0, sizeof(desc->wb)); + + desc->wb.upper.length = cpu_to_le16(length); + + igb_build_rx_metadata(core, pkt, pkt != NULL, + rss_info, + &desc->wb.lower.lo_dword.pkt_info, + &desc->wb.lower.lo_dword.hdr_info, + &desc->wb.lower.hi_dword.rss, + &desc->wb.upper.status_error, + &desc->wb.lower.hi_dword.csum_ip.ip_id, + &desc->wb.upper.vlan); +} + +static inline void +igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc, +struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info, uint16_t length) +{ + if (igb_rx_use_legacy_descriptor(core)) { + igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length); + } else { + igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info, length); + } +} + +static inline void +igb_pci_dma_write_rx_desc(IGBCore *core, PCIDevice *dev, dma_addr_t addr, + union e1000_rx_desc_union *desc, dma_addr_t len) +{ + if (igb_rx_use_legacy_descriptor(core)) { + struct e1000_rx_desc *d = &desc->legacy; + size_t offset = offsetof(struct e1000_rx_desc, status); + uint8_t status = d->status; + + d->status &= ~E1000_RXD_STAT_DD; + pci_dma_write(dev, addr, desc, len); + + if (status & E1000_RXD_STAT_DD) { + d->status = status; + pci_dma_write(dev, addr + offset, &status, sizeof(status)); + } + } else { + union e1000_adv_rx_desc *d = &desc->adv; + size_t offset = + offsetof(union e1000_adv_rx_desc, wb.upper.status_error); + uint32_t status = d->wb.upper.status_error; + + d->wb.upper.status_error &= ~E1000_RXD_STAT_DD; + pci_dma_write(dev, addr, desc, len); + + if (status & E1000_RXD_STAT_DD) { + d->wb.upper.status_error = status; + pci_dma_write(dev, addr + offset, &status, sizeof(status)); + } + } +} + +static void +igb_write_to_rx_buffers(IGBCore *core, + PCIDevice *d, + hwaddr ba, + uint16_t *written, + const char *data, + dma_addr_t data_len) +{ + trace_igb_rx_desc_buff_write(ba, *written, data, data_len); + pci_dma_write(d, ba + *written, data, data_len); + *written += data_len; +} + +static void +igb_update_rx_stats(IGBCore *core, size_t data_size, size_t data_fcs_size) +{ + e1000x_update_rx_total_stats(core->mac, data_size, data_fcs_size); + + switch (net_rx_pkt_get_packet_type(core->rx_pkt)) { + case ETH_PKT_BCAST: + e1000x_inc_reg_if_not_full(core->mac, BPRC); + break; + + case ETH_PKT_MCAST: + e1000x_inc_reg_if_not_full(core->mac, MPRC); + break; + + default: + break; + } +} + +static inline bool +igb_rx_descr_threshold_hit(IGBCore *core, const E1000E_RingInfo *rxi) +{ + return igb_ring_free_descr_num(core, rxi) == + ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16; +} + +static void +igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt, + const E1000E_RxRing *rxr, + const E1000E_RSSInfo *rss_info) +{ + PCIDevice *d; + dma_addr_t base; + union e1000_rx_desc_union desc; + size_t desc_size; + size_t desc_offset = 0; + size_t iov_ofs = 0; + + struct iovec *iov = net_rx_pkt_get_iovec(pkt); + size_t size = net_rx_pkt_get_total_len(pkt); + size_t total_size = size + e1000x_fcs_len(core->mac); + const E1000E_RingInfo *rxi = rxr->i; + size_t bufsize = igb_rxbufsize(core, rxi); + + d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8); + if (!d) { + d = core->owner; + } + + do { + hwaddr ba; + uint16_t written = 0; + bool is_last = false; + + desc_size = total_size - desc_offset; + + if (desc_size > bufsize) { + desc_size = bufsize; + } + + if (igb_ring_empty(core, rxi)) { + return; + } + + base = igb_ring_head_descr(core, rxi); + + pci_dma_read(d, base, &desc, core->rx_desc_len); + + trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len); + + igb_read_rx_descr(core, &desc, &ba); + + if (ba) { + if (desc_offset < size) { + static const uint32_t fcs_pad; + size_t iov_copy; + size_t copy_size = size - desc_offset; + if (copy_size > bufsize) { + copy_size = bufsize; + } + + /* Copy packet payload */ + while (copy_size) { + iov_copy = MIN(copy_size, iov->iov_len - iov_ofs); + + igb_write_to_rx_buffers(core, d, ba, &written, + iov->iov_base + iov_ofs, iov_copy); + + copy_size -= iov_copy; + iov_ofs += iov_copy; + if (iov_ofs == iov->iov_len) { + iov++; + iov_ofs = 0; + } + } + + if (desc_offset + desc_size >= total_size) { + /* Simulate FCS checksum presence in the last descriptor */ + igb_write_to_rx_buffers(core, d, ba, &written, + (const char *) &fcs_pad, e1000x_fcs_len(core->mac)); + } + } + } else { /* as per intel docs; skip descriptors with null buf addr */ + trace_e1000e_rx_null_descriptor(); + } + desc_offset += desc_size; + if (desc_offset >= total_size) { + is_last = true; + } + + igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL, + rss_info, written); + igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len); + + igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN); + + } while (desc_offset < total_size); + + igb_update_rx_stats(core, size, total_size); +} + +static inline void +igb_rx_fix_l4_csum(IGBCore *core, struct NetRxPkt *pkt) +{ + struct virtio_net_hdr *vhdr = net_rx_pkt_get_vhdr(pkt); + + if (vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { + net_rx_pkt_fix_l4_csum(pkt); + } +} + +ssize_t +igb_receive_iov(IGBCore *core, const struct iovec *iov, int iovcnt) +{ + return igb_receive_internal(core, iov, iovcnt, core->has_vnet, NULL); +} + +static ssize_t +igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt, + bool has_vnet, bool *external_tx) +{ + static const int maximum_ethernet_hdr_len = (ETH_HLEN + 4); + + uint16_t queues = 0; + uint32_t n = 0; + uint8_t min_buf[ETH_ZLEN]; + struct iovec min_iov; + struct eth_header *ehdr; + uint8_t *filter_buf; + size_t size, orig_size; + size_t iov_ofs = 0; + E1000E_RxRing rxr; + E1000E_RSSInfo rss_info; + size_t total_size; + int i; + + trace_e1000e_rx_receive_iov(iovcnt); + + if (external_tx) { + *external_tx = true; + } + + if (!e1000x_hw_rx_enabled(core->mac)) { + return -1; + } + + /* Pull virtio header in */ + if (has_vnet) { + net_rx_pkt_set_vhdr_iovec(core->rx_pkt, iov, iovcnt); + iov_ofs = sizeof(struct virtio_net_hdr); + } else { + net_rx_pkt_unset_vhdr(core->rx_pkt); + } + + filter_buf = iov->iov_base + iov_ofs; + orig_size = iov_size(iov, iovcnt); + size = orig_size - iov_ofs; + + /* Pad to minimum Ethernet frame length */ + if (size < sizeof(min_buf)) { + iov_to_buf(iov, iovcnt, iov_ofs, min_buf, size); + memset(&min_buf[size], 0, sizeof(min_buf) - size); + e1000x_inc_reg_if_not_full(core->mac, RUC); + min_iov.iov_base = filter_buf = min_buf; + min_iov.iov_len = size = sizeof(min_buf); + iovcnt = 1; + iov = &min_iov; + iov_ofs = 0; + } else if (iov->iov_len < maximum_ethernet_hdr_len) { + /* This is very unlikely, but may happen. */ + iov_to_buf(iov, iovcnt, iov_ofs, min_buf, maximum_ethernet_hdr_len); + filter_buf = min_buf; + } + + /* Discard oversized packets if !LPE and !SBP. */ + if (e1000x_is_oversized(core->mac, size)) { + return orig_size; + } + + ehdr = PKT_GET_ETH_HDR(filter_buf); + net_rx_pkt_set_packet_type(core->rx_pkt, get_eth_packet_type(ehdr)); + + net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs, + e1000x_vlan_enabled(core->mac), + core->mac[VET] & 0xffff); + + queues = igb_receive_assign(core, ehdr, &rss_info, external_tx); + if (!queues) { + trace_e1000e_rx_flt_dropped(); + return orig_size; + } + + total_size = net_rx_pkt_get_total_len(core->rx_pkt) + + e1000x_fcs_len(core->mac); + + for (i = 0; i < IGB_NUM_QUEUES; i++) { + if (!(queues & BIT(i))) { + continue; + } + + igb_rx_ring_init(core, &rxr, i); + + if (!igb_has_rxbufs(core, rxr.i, total_size)) { + n |= E1000_ICS_RXO; + trace_e1000e_rx_not_written_to_guest(rxr.i->idx); + continue; + } + + n |= E1000_ICR_RXT0; + + igb_rx_fix_l4_csum(core, core->rx_pkt); + igb_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info); + + /* Check if receive descriptor minimum threshold hit */ + if (igb_rx_descr_threshold_hit(core, rxr.i)) { + n |= E1000_ICS_RXDMT0; + } + + core->mac[EICR] |= igb_rx_wb_eic(core, rxr.i->idx); + + trace_e1000e_rx_written_to_guest(rxr.i->idx); + } + + trace_e1000e_rx_interrupt_set(n); + igb_set_interrupt_cause(core, n); + + return orig_size; +} + +static inline bool +igb_have_autoneg(IGBCore *core) +{ + return core->phy[MII_BMCR] & MII_BMCR_AUTOEN; +} + +static void igb_update_flowctl_status(IGBCore *core) +{ + if (igb_have_autoneg(core) && core->phy[MII_BMSR] & MII_BMSR_AN_COMP) { + trace_e1000e_link_autoneg_flowctl(true); + core->mac[CTRL] |= E1000_CTRL_TFCE | E1000_CTRL_RFCE; + } else { + trace_e1000e_link_autoneg_flowctl(false); + } +} + +static inline void +igb_link_down(IGBCore *core) +{ + e1000x_update_regs_on_link_down(core->mac, core->phy); + igb_update_flowctl_status(core); +} + +static inline void +igb_set_phy_ctrl(IGBCore *core, uint16_t val) +{ + /* bits 0-5 reserved; MII_BMCR_[ANRESTART,RESET] are self clearing */ + core->phy[MII_BMCR] = val & ~(0x3f | MII_BMCR_RESET | MII_BMCR_ANRESTART); + + if ((val & MII_BMCR_ANRESTART) && igb_have_autoneg(core)) { + e1000x_restart_autoneg(core->mac, core->phy, core->autoneg_timer); + } +} + +void igb_core_set_link_status(IGBCore *core) +{ + NetClientState *nc = qemu_get_queue(core->owner_nic); + uint32_t old_status = core->mac[STATUS]; + + trace_e1000e_link_status_changed(nc->link_down ? false : true); + + if (nc->link_down) { + e1000x_update_regs_on_link_down(core->mac, core->phy); + } else { + if (igb_have_autoneg(core) && + !(core->phy[MII_BMSR] & MII_BMSR_AN_COMP)) { + e1000x_restart_autoneg(core->mac, core->phy, + core->autoneg_timer); + } else { + e1000x_update_regs_on_link_up(core->mac, core->phy); + igb_start_recv(core); + } + } + + if (core->mac[STATUS] != old_status) { + igb_set_interrupt_cause(core, E1000_ICR_LSC); + } +} + +static void +igb_set_ctrl(IGBCore *core, int index, uint32_t val) +{ + trace_e1000e_core_ctrl_write(index, val); + + /* RST is self clearing */ + core->mac[CTRL] = val & ~E1000_CTRL_RST; + core->mac[CTRL_DUP] = core->mac[CTRL]; + + trace_e1000e_link_set_params( + !!(val & E1000_CTRL_ASDE), + (val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT, + !!(val & E1000_CTRL_FRCSPD), + !!(val & E1000_CTRL_FRCDPX), + !!(val & E1000_CTRL_RFCE), + !!(val & E1000_CTRL_TFCE)); + + if (val & E1000_CTRL_RST) { + trace_e1000e_core_ctrl_sw_reset(); + igb_reset(core, true); + } + + if (val & E1000_CTRL_PHY_RST) { + trace_e1000e_core_ctrl_phy_reset(); + core->mac[STATUS] |= E1000_STATUS_PHYRA; + } +} + +static void +igb_set_rfctl(IGBCore *core, int index, uint32_t val) +{ + trace_e1000e_rx_set_rfctl(val); + + if (!(val & E1000_RFCTL_ISCSI_DIS)) { + trace_e1000e_wrn_iscsi_filtering_not_supported(); + } + + if (!(val & E1000_RFCTL_NFSW_DIS)) { + trace_e1000e_wrn_nfsw_filtering_not_supported(); + } + + if (!(val & E1000_RFCTL_NFSR_DIS)) { + trace_e1000e_wrn_nfsr_filtering_not_supported(); + } + + core->mac[RFCTL] = val; +} + +static void +igb_calc_rxdesclen(IGBCore *core) +{ + if (igb_rx_use_legacy_descriptor(core)) { + core->rx_desc_len = sizeof(struct e1000_rx_desc); + } else { + core->rx_desc_len = sizeof(union e1000_adv_rx_desc); + } + trace_e1000e_rx_desc_len(core->rx_desc_len); +} + +static void +igb_set_rx_control(IGBCore *core, int index, uint32_t val) +{ + core->mac[RCTL] = val; + trace_e1000e_rx_set_rctl(core->mac[RCTL]); + + if (val & E1000_RCTL_DTYP_MASK) { + qemu_log_mask(LOG_GUEST_ERROR, + "igb: RCTL.DTYP must be zero for compatibility"); + } + + if (val & E1000_RCTL_EN) { + igb_calc_rxdesclen(core); + igb_start_recv(core); + } +} + +static inline void +igb_clear_ims_bits(IGBCore *core, uint32_t bits) +{ + trace_e1000e_irq_clear_ims(bits, core->mac[IMS], core->mac[IMS] & ~bits); + core->mac[IMS] &= ~bits; +} + +static inline bool +igb_postpone_interrupt(IGBIntrDelayTimer *timer) +{ + if (timer->running) { + trace_e1000e_irq_postponed_by_xitr(timer->delay_reg << 2); + + return true; + } + + if (timer->core->mac[timer->delay_reg] != 0) { + igb_intrmgr_rearm_timer(timer); + } + + return false; +} + +static inline bool +igb_eitr_should_postpone(IGBCore *core, int idx) +{ + return igb_postpone_interrupt(&core->eitr[idx]); +} + +static void igb_send_msix(IGBCore *core) +{ + uint32_t causes = core->mac[EICR] & core->mac[EIMS]; + uint32_t effective_eiac; + int vector; + + for (vector = 0; vector < IGB_INTR_NUM; ++vector) { + if ((causes & BIT(vector)) && !igb_eitr_should_postpone(core, vector)) { + + trace_e1000e_irq_msix_notify_vec(vector); + igb_msix_notify(core, vector); + + trace_e1000e_irq_icr_clear_eiac(core->mac[EICR], core->mac[EIAC]); + effective_eiac = core->mac[EIAC] & BIT(vector); + core->mac[EICR] &= ~effective_eiac; + } + } +} + +static inline void +igb_fix_icr_asserted(IGBCore *core) +{ + core->mac[ICR] &= ~E1000_ICR_ASSERTED; + if (core->mac[ICR]) { + core->mac[ICR] |= E1000_ICR_ASSERTED; + } + + trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]); +} + +static void +igb_update_interrupt_state(IGBCore *core) +{ + uint32_t icr; + uint32_t causes; + uint32_t int_alloc; + + icr = core->mac[ICR] & core->mac[IMS]; + + if (msix_enabled(core->owner)) { + if (icr) { + causes = 0; + if (icr & E1000_ICR_DRSTA) { + int_alloc = core->mac[IVAR_MISC] & 0xff; + if (int_alloc & E1000_IVAR_VALID) { + causes |= BIT(int_alloc & 0x1f); + } + } + /* Check if other bits (excluding the TCP Timer) are enabled. */ + if (icr & ~E1000_ICR_DRSTA) { + int_alloc = (core->mac[IVAR_MISC] >> 8) & 0xff; + if (int_alloc & E1000_IVAR_VALID) { + causes |= BIT(int_alloc & 0x1f); + } + trace_e1000e_irq_add_msi_other(core->mac[EICR]); + } + core->mac[EICR] |= causes; + } + + if ((core->mac[EICR] & core->mac[EIMS])) { + igb_send_msix(core); + } + } else { + igb_fix_icr_asserted(core); + + if (icr) { + core->mac[EICR] |= (icr & E1000_ICR_DRSTA) | E1000_EICR_OTHER; + } else { + core->mac[EICR] &= ~E1000_EICR_OTHER; + } + + trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS], + core->mac[ICR], core->mac[IMS]); + + if (msi_enabled(core->owner)) { + if (icr) { + msi_notify(core->owner, 0); + } + } else { + if (icr) { + igb_raise_legacy_irq(core); + } else { + igb_lower_legacy_irq(core); + } + } + } +} + +static void +igb_set_interrupt_cause(IGBCore *core, uint32_t val) +{ + trace_e1000e_irq_set_cause_entry(val, core->mac[ICR]); + + core->mac[ICR] |= val; + + trace_e1000e_irq_set_cause_exit(val, core->mac[ICR]); + + igb_update_interrupt_state(core); +} + +static void igb_set_eics(IGBCore *core, int index, uint32_t val) +{ + bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + + trace_igb_irq_write_eics(val, msix); + + core->mac[EICS] |= + val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK); + + /* + * TODO: Move to igb_update_interrupt_state if EICS is modified in other + * places. + */ + core->mac[EICR] = core->mac[EICS]; + + igb_update_interrupt_state(core); +} + +static void igb_set_eims(IGBCore *core, int index, uint32_t val) +{ + bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + + trace_igb_irq_write_eims(val, msix); + + core->mac[EIMS] |= + val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK); + + igb_update_interrupt_state(core); +} + +static void igb_vf_reset(IGBCore *core, uint16_t vfn) +{ + /* TODO: Reset of the queue enable and the interrupt registers of the VF. */ + + core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_RSTI; + core->mac[V2PMAILBOX0 + vfn] = E1000_V2PMAILBOX_RSTD; +} + +static void mailbox_interrupt_to_vf(IGBCore *core, uint16_t vfn) +{ + uint32_t ent = core->mac[VTIVAR_MISC + vfn]; + + if ((ent & E1000_IVAR_VALID)) { + core->mac[EICR] |= (ent & 0x3) << (22 - vfn * IGBVF_MSIX_VEC_NUM); + igb_update_interrupt_state(core); + } +} + +static void mailbox_interrupt_to_pf(IGBCore *core) +{ + igb_set_interrupt_cause(core, E1000_ICR_VMMB); +} + +static void igb_set_pfmailbox(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = index - P2VMAILBOX0; + + trace_igb_set_pfmailbox(vfn, val); + + if (val & E1000_P2VMAILBOX_STS) { + core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFSTS; + mailbox_interrupt_to_vf(core, vfn); + } + + if (val & E1000_P2VMAILBOX_ACK) { + core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFACK; + mailbox_interrupt_to_vf(core, vfn); + } + + /* Buffer Taken by PF (can be set only if the VFU is cleared). */ + if (val & E1000_P2VMAILBOX_PFU) { + if (!(core->mac[index] & E1000_P2VMAILBOX_VFU)) { + core->mac[index] |= E1000_P2VMAILBOX_PFU; + core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFU; + } + } else { + core->mac[index] &= ~E1000_P2VMAILBOX_PFU; + core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_PFU; + } + + if (val & E1000_P2VMAILBOX_RVFU) { + core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_VFU; + core->mac[MBVFICR] &= ~((E1000_MBVFICR_VFACK_VF1 << vfn) | + (E1000_MBVFICR_VFREQ_VF1 << vfn)); + } +} + +static void igb_set_vfmailbox(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = index - V2PMAILBOX0; + + trace_igb_set_vfmailbox(vfn, val); + + if (val & E1000_V2PMAILBOX_REQ) { + core->mac[MBVFICR] |= E1000_MBVFICR_VFREQ_VF1 << vfn; + mailbox_interrupt_to_pf(core); + } + + if (val & E1000_V2PMAILBOX_ACK) { + core->mac[MBVFICR] |= E1000_MBVFICR_VFACK_VF1 << vfn; + mailbox_interrupt_to_pf(core); + } + + /* Buffer Taken by VF (can be set only if the PFU is cleared). */ + if (val & E1000_V2PMAILBOX_VFU) { + if (!(core->mac[index] & E1000_V2PMAILBOX_PFU)) { + core->mac[index] |= E1000_V2PMAILBOX_VFU; + core->mac[P2VMAILBOX0 + vfn] |= E1000_P2VMAILBOX_VFU; + } + } else { + core->mac[index] &= ~E1000_V2PMAILBOX_VFU; + core->mac[P2VMAILBOX0 + vfn] &= ~E1000_P2VMAILBOX_VFU; + } +} + +static void igb_w1c(IGBCore *core, int index, uint32_t val) +{ + core->mac[index] &= ~val; +} + +static void igb_set_eimc(IGBCore *core, int index, uint32_t val) +{ + bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + + /* Interrupts are disabled via a write to EIMC and reflected in EIMS. */ + core->mac[EIMS] &= + ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK)); + + trace_igb_irq_write_eimc(val, core->mac[EIMS], msix); + igb_update_interrupt_state(core); +} + +static void igb_set_eiac(IGBCore *core, int index, uint32_t val) +{ + bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + + if (msix) { + trace_igb_irq_write_eiac(val); + + /* + * TODO: When using IOV, the bits that correspond to MSI-X vectors + * that are assigned to a VF are read-only. + */ + core->mac[EIAC] |= (val & E1000_EICR_MSIX_MASK); + } +} + +static void igb_set_eiam(IGBCore *core, int index, uint32_t val) +{ + bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + + /* + * TODO: When using IOV, the bits that correspond to MSI-X vectors that + * are assigned to a VF are read-only. + */ + core->mac[EIAM] |= + ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK)); + + trace_igb_irq_write_eiam(val, msix); +} + +static void igb_set_eicr(IGBCore *core, int index, uint32_t val) +{ + bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE); + + /* + * TODO: In IOV mode, only bit zero of this vector is available for the PF + * function. + */ + core->mac[EICR] &= + ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK)); + + trace_igb_irq_write_eicr(val, msix); + igb_update_interrupt_state(core); +} + +static void igb_set_vtctrl(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn; + + if (val & E1000_CTRL_RST) { + vfn = (index - PVTCTRL0) / 0x40; + igb_vf_reset(core, vfn); + } +} + +static void igb_set_vteics(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = (index - PVTEICS0) / 0x40; + + core->mac[index] = val; + igb_set_eics(core, EICS, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM)); +} + +static void igb_set_vteims(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = (index - PVTEIMS0) / 0x40; + + core->mac[index] = val; + igb_set_eims(core, EIMS, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM)); +} + +static void igb_set_vteimc(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = (index - PVTEIMC0) / 0x40; + + core->mac[index] = val; + igb_set_eimc(core, EIMC, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM)); +} + +static void igb_set_vteiac(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = (index - PVTEIAC0) / 0x40; + + core->mac[index] = val; + igb_set_eiac(core, EIAC, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM)); +} + +static void igb_set_vteiam(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = (index - PVTEIAM0) / 0x40; + + core->mac[index] = val; + igb_set_eiam(core, EIAM, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM)); +} + +static void igb_set_vteicr(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = (index - PVTEICR0) / 0x40; + + core->mac[index] = val; + igb_set_eicr(core, EICR, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM)); +} + +static void igb_set_vtivar(IGBCore *core, int index, uint32_t val) +{ + uint16_t vfn = (index - VTIVAR); + uint16_t qn = vfn; + uint8_t ent; + int n; + + core->mac[index] = val; + + /* Get assigned vector associated with queue Rx#0. */ + if ((val & E1000_IVAR_VALID)) { + n = igb_ivar_entry_rx(qn); + ent = E1000_IVAR_VALID | (24 - vfn * IGBVF_MSIX_VEC_NUM - (2 - (val & 0x7))); + core->mac[IVAR0 + n / 4] |= ent << 8 * (n % 4); + } + + /* Get assigned vector associated with queue Tx#0 */ + ent = val >> 8; + if ((ent & E1000_IVAR_VALID)) { + n = igb_ivar_entry_tx(qn); + ent = E1000_IVAR_VALID | (24 - vfn * IGBVF_MSIX_VEC_NUM - (2 - (ent & 0x7))); + core->mac[IVAR0 + n / 4] |= ent << 8 * (n % 4); + } + + /* + * Ignoring assigned vectors associated with queues Rx#1 and Tx#1 for now. + */ +} + +static inline void +igb_autoneg_timer(void *opaque) +{ + IGBCore *core = opaque; + if (!qemu_get_queue(core->owner_nic)->link_down) { + e1000x_update_regs_on_autoneg_done(core->mac, core->phy); + igb_start_recv(core); + + igb_update_flowctl_status(core); + /* signal link status change to the guest */ + igb_set_interrupt_cause(core, E1000_ICR_LSC); + } +} + +static inline uint16_t +igb_get_reg_index_with_offset(const uint16_t *mac_reg_access, hwaddr addr) +{ + uint16_t index = (addr & 0x1ffff) >> 2; + return index + (mac_reg_access[index] & 0xfffe); +} + +static const char igb_phy_regcap[MAX_PHY_REG_ADDRESS + 1] = { + [MII_BMCR] = PHY_RW, + [MII_BMSR] = PHY_R, + [MII_PHYID1] = PHY_R, + [MII_PHYID2] = PHY_R, + [MII_ANAR] = PHY_RW, + [MII_ANLPAR] = PHY_R, + [MII_ANER] = PHY_R, + [MII_ANNP] = PHY_RW, + [MII_ANLPRNP] = PHY_R, + [MII_CTRL1000] = PHY_RW, + [MII_STAT1000] = PHY_R, + [MII_EXTSTAT] = PHY_R, + + [IGP01E1000_PHY_PORT_CONFIG] = PHY_RW, + [IGP01E1000_PHY_PORT_STATUS] = PHY_R, + [IGP01E1000_PHY_PORT_CTRL] = PHY_RW, + [IGP01E1000_PHY_LINK_HEALTH] = PHY_R, + [IGP02E1000_PHY_POWER_MGMT] = PHY_RW, + [IGP01E1000_PHY_PAGE_SELECT] = PHY_W +}; + +static void +igb_phy_reg_write(IGBCore *core, uint32_t addr, uint16_t data) +{ + assert(addr <= MAX_PHY_REG_ADDRESS); + + if (addr == MII_BMCR) { + igb_set_phy_ctrl(core, data); + } else { + core->phy[addr] = data; + } +} + +static void +igb_set_mdic(IGBCore *core, int index, uint32_t val) +{ + uint32_t data = val & E1000_MDIC_DATA_MASK; + uint32_t addr = ((val & E1000_MDIC_REG_MASK) >> E1000_MDIC_REG_SHIFT); + + if ((val & E1000_MDIC_PHY_MASK) >> E1000_MDIC_PHY_SHIFT != 1) { /* phy # */ + val = core->mac[MDIC] | E1000_MDIC_ERROR; + } else if (val & E1000_MDIC_OP_READ) { + if (!(igb_phy_regcap[addr] & PHY_R)) { + trace_igb_core_mdic_read_unhandled(addr); + val |= E1000_MDIC_ERROR; + } else { + val = (val ^ data) | core->phy[addr]; + trace_igb_core_mdic_read(addr, val); + } + } else if (val & E1000_MDIC_OP_WRITE) { + if (!(igb_phy_regcap[addr] & PHY_W)) { + trace_igb_core_mdic_write_unhandled(addr); + val |= E1000_MDIC_ERROR; + } else { + trace_igb_core_mdic_write(addr, data); + igb_phy_reg_write(core, addr, data); + } + } + core->mac[MDIC] = val | E1000_MDIC_READY; + + if (val & E1000_MDIC_INT_EN) { + igb_set_interrupt_cause(core, E1000_ICR_MDAC); + } +} + +static void +igb_set_rdt(IGBCore *core, int index, uint32_t val) +{ + core->mac[index] = val & 0xffff; + trace_e1000e_rx_set_rdt(igb_mq_queue_idx(RDT0, index), val); + igb_start_recv(core); +} + +static void +igb_set_status(IGBCore *core, int index, uint32_t val) +{ + if ((val & E1000_STATUS_PHYRA) == 0) { + core->mac[index] &= ~E1000_STATUS_PHYRA; + } +} + +static void +igb_set_ctrlext(IGBCore *core, int index, uint32_t val) +{ + trace_e1000e_link_set_ext_params(!!(val & E1000_CTRL_EXT_ASDCHK), + !!(val & E1000_CTRL_EXT_SPD_BYPS)); + + /* TODO: PFRSTD */ + + /* Zero self-clearing bits */ + val &= ~(E1000_CTRL_EXT_ASDCHK | E1000_CTRL_EXT_EE_RST); + core->mac[CTRL_EXT] = val; +} + +static void +igb_set_pbaclr(IGBCore *core, int index, uint32_t val) +{ + int i; + + core->mac[PBACLR] = val & E1000_PBACLR_VALID_MASK; + + if (!msix_enabled(core->owner)) { + return; + } + + for (i = 0; i < IGB_INTR_NUM; i++) { + if (core->mac[PBACLR] & BIT(i)) { + msix_clr_pending(core->owner, i); + } + } +} + +static void +igb_set_fcrth(IGBCore *core, int index, uint32_t val) +{ + core->mac[FCRTH] = val & 0xFFF8; +} + +static void +igb_set_fcrtl(IGBCore *core, int index, uint32_t val) +{ + core->mac[FCRTL] = val & 0x8000FFF8; +} + +#define IGB_LOW_BITS_SET_FUNC(num) \ + static void \ + igb_set_##num##bit(IGBCore *core, int index, uint32_t val) \ + { \ + core->mac[index] = val & (BIT(num) - 1); \ + } + +IGB_LOW_BITS_SET_FUNC(4) +IGB_LOW_BITS_SET_FUNC(13) +IGB_LOW_BITS_SET_FUNC(16) + +static void +igb_set_dlen(IGBCore *core, int index, uint32_t val) +{ + core->mac[index] = val & 0xffff0; +} + +static void +igb_set_dbal(IGBCore *core, int index, uint32_t val) +{ + core->mac[index] = val & E1000_XDBAL_MASK; +} + +static void +igb_set_tdt(IGBCore *core, int index, uint32_t val) +{ + IGB_TxRing txr; + int qn = igb_mq_queue_idx(TDT0, index); + + core->mac[index] = val & 0xffff; + + igb_tx_ring_init(core, &txr, qn); + igb_start_xmit(core, &txr); +} + +static void +igb_set_ics(IGBCore *core, int index, uint32_t val) +{ + trace_e1000e_irq_write_ics(val); + igb_set_interrupt_cause(core, val); +} + +static void +igb_set_imc(IGBCore *core, int index, uint32_t val) +{ + trace_e1000e_irq_ims_clear_set_imc(val); + igb_clear_ims_bits(core, val); + igb_update_interrupt_state(core); +} + +static void +igb_set_ims(IGBCore *core, int index, uint32_t val) +{ + uint32_t valid_val = val & 0x77D4FBFD; + + trace_e1000e_irq_set_ims(val, core->mac[IMS], core->mac[IMS] | valid_val); + core->mac[IMS] |= valid_val; + igb_update_interrupt_state(core); +} + +static void igb_commit_icr(IGBCore *core) +{ + /* + * If GPIE.NSICR = 0, then the copy of IAM to IMS will occur only if at + * least one bit is set in the IMS and there is a true interrupt as + * reflected in ICR.INTA. + */ + if ((core->mac[GPIE] & E1000_GPIE_NSICR) || + (core->mac[IMS] && (core->mac[ICR] & E1000_ICR_INT_ASSERTED))) { + igb_set_ims(core, IMS, core->mac[IAM]); + } else { + igb_update_interrupt_state(core); + } +} + +static void igb_set_icr(IGBCore *core, int index, uint32_t val) +{ + uint32_t icr = core->mac[ICR] & ~val; + + trace_igb_irq_icr_write(val, core->mac[ICR], icr); + core->mac[ICR] = icr; + igb_commit_icr(core); +} + +static uint32_t +igb_mac_readreg(IGBCore *core, int index) +{ + return core->mac[index]; +} + +static uint32_t +igb_mac_ics_read(IGBCore *core, int index) +{ + trace_e1000e_irq_read_ics(core->mac[ICS]); + return core->mac[ICS]; +} + +static uint32_t +igb_mac_ims_read(IGBCore *core, int index) +{ + trace_e1000e_irq_read_ims(core->mac[IMS]); + return core->mac[IMS]; +} + +static uint32_t +igb_mac_swsm_read(IGBCore *core, int index) +{ + uint32_t val = core->mac[SWSM]; + core->mac[SWSM] = val | E1000_SWSM_SMBI; + return val; +} + +static uint32_t +igb_mac_eitr_read(IGBCore *core, int index) +{ + return core->eitr_guest_value[index - EITR0]; +} + +static uint32_t igb_mac_vfmailbox_read(IGBCore *core, int index) +{ + uint32_t val = core->mac[index]; + + core->mac[index] &= ~(E1000_V2PMAILBOX_PFSTS | E1000_V2PMAILBOX_PFACK | + E1000_V2PMAILBOX_RSTD); + + return val; +} + +static uint32_t +igb_mac_icr_read(IGBCore *core, int index) +{ + uint32_t ret = core->mac[ICR]; + trace_e1000e_irq_icr_read_entry(ret); + + if (core->mac[GPIE] & E1000_GPIE_NSICR) { + trace_igb_irq_icr_clear_gpie_nsicr(); + core->mac[ICR] = 0; + } else if (core->mac[IMS] == 0) { + trace_e1000e_irq_icr_clear_zero_ims(); + core->mac[ICR] = 0; + } else if (!msix_enabled(core->owner)) { + trace_e1000e_irq_icr_clear_nonmsix_icr_read(); + core->mac[ICR] = 0; + } + + trace_e1000e_irq_icr_read_exit(core->mac[ICR]); + igb_commit_icr(core); + return ret; +} + +static uint32_t +igb_mac_read_clr4(IGBCore *core, int index) +{ + uint32_t ret = core->mac[index]; + + core->mac[index] = 0; + return ret; +} + +static uint32_t +igb_mac_read_clr8(IGBCore *core, int index) +{ + uint32_t ret = core->mac[index]; + + core->mac[index] = 0; + core->mac[index - 1] = 0; + return ret; +} + +static uint32_t +igb_get_ctrl(IGBCore *core, int index) +{ + uint32_t val = core->mac[CTRL]; + + trace_e1000e_link_read_params( + !!(val & E1000_CTRL_ASDE), + (val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT, + !!(val & E1000_CTRL_FRCSPD), + !!(val & E1000_CTRL_FRCDPX), + !!(val & E1000_CTRL_RFCE), + !!(val & E1000_CTRL_TFCE)); + + return val; +} + +static uint32_t igb_get_status(IGBCore *core, int index) +{ + uint32_t res = core->mac[STATUS]; + uint16_t num_vfs = pcie_sriov_num_vfs(core->owner); + + if (core->mac[CTRL] & E1000_CTRL_FRCDPX) { + res |= (core->mac[CTRL] & E1000_CTRL_FD) ? E1000_STATUS_FD : 0; + } else { + res |= E1000_STATUS_FD; + } + + if ((core->mac[CTRL] & E1000_CTRL_FRCSPD) || + (core->mac[CTRL_EXT] & E1000_CTRL_EXT_SPD_BYPS)) { + switch (core->mac[CTRL] & E1000_CTRL_SPD_SEL) { + case E1000_CTRL_SPD_10: + res |= E1000_STATUS_SPEED_10; + break; + case E1000_CTRL_SPD_100: + res |= E1000_STATUS_SPEED_100; + break; + case E1000_CTRL_SPD_1000: + default: + res |= E1000_STATUS_SPEED_1000; + break; + } + } else { + res |= E1000_STATUS_SPEED_1000; + } + + if (num_vfs) { + res |= num_vfs << E1000_STATUS_NUM_VFS_SHIFT; + res |= E1000_STATUS_IOV_MODE; + } + + /* + * Windows driver 12.18.9.23 resets if E1000_STATUS_GIO_MASTER_ENABLE is + * left set after E1000_CTRL_LRST is set. + */ + if (!(core->mac[CTRL] & E1000_CTRL_GIO_MASTER_DISABLE) && + !(core->mac[CTRL] & E1000_CTRL_LRST)) { + res |= E1000_STATUS_GIO_MASTER_ENABLE; + } + + return res; +} + +static void +igb_mac_writereg(IGBCore *core, int index, uint32_t val) +{ + core->mac[index] = val; +} + +static void +igb_mac_setmacaddr(IGBCore *core, int index, uint32_t val) +{ + uint32_t macaddr[2]; + + core->mac[index] = val; + + macaddr[0] = cpu_to_le32(core->mac[RA]); + macaddr[1] = cpu_to_le32(core->mac[RA + 1]); + qemu_format_nic_info_str(qemu_get_queue(core->owner_nic), + (uint8_t *) macaddr); + + trace_e1000e_mac_set_sw(MAC_ARG(macaddr)); +} + +static void +igb_set_eecd(IGBCore *core, int index, uint32_t val) +{ + static const uint32_t ro_bits = E1000_EECD_PRES | + E1000_EECD_AUTO_RD | + E1000_EECD_SIZE_EX_MASK; + + core->mac[EECD] = (core->mac[EECD] & ro_bits) | (val & ~ro_bits); +} + +static void +igb_set_eerd(IGBCore *core, int index, uint32_t val) +{ + uint32_t addr = (val >> E1000_EERW_ADDR_SHIFT) & E1000_EERW_ADDR_MASK; + uint32_t flags = 0; + uint32_t data = 0; + + if ((addr < IGB_EEPROM_SIZE) && (val & E1000_EERW_START)) { + data = core->eeprom[addr]; + flags = E1000_EERW_DONE; + } + + core->mac[EERD] = flags | + (addr << E1000_EERW_ADDR_SHIFT) | + (data << E1000_EERW_DATA_SHIFT); +} + +static void +igb_set_eitr(IGBCore *core, int index, uint32_t val) +{ + uint32_t eitr_num = index - EITR0; + + trace_igb_irq_eitr_set(eitr_num, val); + + core->eitr_guest_value[eitr_num] = val & ~E1000_EITR_CNT_IGNR; + core->mac[index] = val & 0x7FFE; +} + +static void +igb_update_rx_offloads(IGBCore *core) +{ + int cso_state = igb_rx_l4_cso_enabled(core); + + trace_e1000e_rx_set_cso(cso_state); + + if (core->has_vnet) { + qemu_set_offload(qemu_get_queue(core->owner_nic)->peer, + cso_state, 0, 0, 0, 0); + } +} + +static void +igb_set_rxcsum(IGBCore *core, int index, uint32_t val) +{ + core->mac[RXCSUM] = val; + igb_update_rx_offloads(core); +} + +static void +igb_set_gcr(IGBCore *core, int index, uint32_t val) +{ + uint32_t ro_bits = core->mac[GCR] & E1000_GCR_RO_BITS; + core->mac[GCR] = (val & ~E1000_GCR_RO_BITS) | ro_bits; +} + +static uint32_t igb_get_systiml(IGBCore *core, int index) +{ + e1000x_timestamp(core->mac, core->timadj, SYSTIML, SYSTIMH); + return core->mac[SYSTIML]; +} + +static uint32_t igb_get_rxsatrh(IGBCore *core, int index) +{ + core->mac[TSYNCRXCTL] &= ~E1000_TSYNCRXCTL_VALID; + return core->mac[RXSATRH]; +} + +static uint32_t igb_get_txstmph(IGBCore *core, int index) +{ + core->mac[TSYNCTXCTL] &= ~E1000_TSYNCTXCTL_VALID; + return core->mac[TXSTMPH]; +} + +static void igb_set_timinca(IGBCore *core, int index, uint32_t val) +{ + e1000x_set_timinca(core->mac, &core->timadj, val); +} + +static void igb_set_timadjh(IGBCore *core, int index, uint32_t val) +{ + core->mac[TIMADJH] = val; + core->timadj += core->mac[TIMADJL] | ((int64_t)core->mac[TIMADJH] << 32); +} + +#define igb_getreg(x) [x] = igb_mac_readreg +typedef uint32_t (*readops)(IGBCore *, int); +static const readops igb_macreg_readops[] = { + igb_getreg(WUFC), + igb_getreg(MANC), + igb_getreg(TOTL), + igb_getreg(RDT0), + igb_getreg(RDT1), + igb_getreg(RDT2), + igb_getreg(RDT3), + igb_getreg(RDT4), + igb_getreg(RDT5), + igb_getreg(RDT6), + igb_getreg(RDT7), + igb_getreg(RDT8), + igb_getreg(RDT9), + igb_getreg(RDT10), + igb_getreg(RDT11), + igb_getreg(RDT12), + igb_getreg(RDT13), + igb_getreg(RDT14), + igb_getreg(RDT15), + igb_getreg(RDBAH0), + igb_getreg(RDBAH1), + igb_getreg(RDBAH2), + igb_getreg(RDBAH3), + igb_getreg(RDBAH4), + igb_getreg(RDBAH5), + igb_getreg(RDBAH6), + igb_getreg(RDBAH7), + igb_getreg(RDBAH8), + igb_getreg(RDBAH9), + igb_getreg(RDBAH10), + igb_getreg(RDBAH11), + igb_getreg(RDBAH12), + igb_getreg(RDBAH13), + igb_getreg(RDBAH14), + igb_getreg(RDBAH15), + igb_getreg(TDBAL0), + igb_getreg(TDBAL1), + igb_getreg(TDBAL2), + igb_getreg(TDBAL3), + igb_getreg(TDBAL4), + igb_getreg(TDBAL5), + igb_getreg(TDBAL6), + igb_getreg(TDBAL7), + igb_getreg(TDBAL8), + igb_getreg(TDBAL9), + igb_getreg(TDBAL10), + igb_getreg(TDBAL11), + igb_getreg(TDBAL12), + igb_getreg(TDBAL13), + igb_getreg(TDBAL14), + igb_getreg(TDBAL15), + igb_getreg(RDLEN0), + igb_getreg(RDLEN1), + igb_getreg(RDLEN2), + igb_getreg(RDLEN3), + igb_getreg(RDLEN4), + igb_getreg(RDLEN5), + igb_getreg(RDLEN6), + igb_getreg(RDLEN7), + igb_getreg(RDLEN8), + igb_getreg(RDLEN9), + igb_getreg(RDLEN10), + igb_getreg(RDLEN11), + igb_getreg(RDLEN12), + igb_getreg(RDLEN13), + igb_getreg(RDLEN14), + igb_getreg(RDLEN15), + igb_getreg(SRRCTL0), + igb_getreg(SRRCTL1), + igb_getreg(SRRCTL2), + igb_getreg(SRRCTL3), + igb_getreg(SRRCTL4), + igb_getreg(SRRCTL5), + igb_getreg(SRRCTL6), + igb_getreg(SRRCTL7), + igb_getreg(SRRCTL8), + igb_getreg(SRRCTL9), + igb_getreg(SRRCTL10), + igb_getreg(SRRCTL11), + igb_getreg(SRRCTL12), + igb_getreg(SRRCTL13), + igb_getreg(SRRCTL14), + igb_getreg(SRRCTL15), + igb_getreg(LATECOL), + igb_getreg(XONTXC), + igb_getreg(TDFH), + igb_getreg(TDFT), + igb_getreg(TDFHS), + igb_getreg(TDFTS), + igb_getreg(TDFPC), + igb_getreg(WUS), + igb_getreg(RDFH), + igb_getreg(RDFT), + igb_getreg(RDFHS), + igb_getreg(RDFTS), + igb_getreg(RDFPC), + igb_getreg(GORCL), + igb_getreg(MGTPRC), + igb_getreg(EERD), + igb_getreg(EIAC), + igb_getreg(MANC2H), + igb_getreg(RXCSUM), + igb_getreg(GSCL_3), + igb_getreg(GSCN_2), + igb_getreg(FCAH), + igb_getreg(FCRTH), + igb_getreg(FLOP), + igb_getreg(RXSTMPH), + igb_getreg(TXSTMPL), + igb_getreg(TIMADJL), + igb_getreg(RDH0), + igb_getreg(RDH1), + igb_getreg(RDH2), + igb_getreg(RDH3), + igb_getreg(RDH4), + igb_getreg(RDH5), + igb_getreg(RDH6), + igb_getreg(RDH7), + igb_getreg(RDH8), + igb_getreg(RDH9), + igb_getreg(RDH10), + igb_getreg(RDH11), + igb_getreg(RDH12), + igb_getreg(RDH13), + igb_getreg(RDH14), + igb_getreg(RDH15), + igb_getreg(TDT0), + igb_getreg(TDT1), + igb_getreg(TDT2), + igb_getreg(TDT3), + igb_getreg(TDT4), + igb_getreg(TDT5), + igb_getreg(TDT6), + igb_getreg(TDT7), + igb_getreg(TDT8), + igb_getreg(TDT9), + igb_getreg(TDT10), + igb_getreg(TDT11), + igb_getreg(TDT12), + igb_getreg(TDT13), + igb_getreg(TDT14), + igb_getreg(TDT15), + igb_getreg(TNCRS), + igb_getreg(RJC), + igb_getreg(IAM), + igb_getreg(GSCL_2), + igb_getreg(TIPG), + igb_getreg(FLMNGCTL), + igb_getreg(FLMNGCNT), + igb_getreg(TSYNCTXCTL), + igb_getreg(EEMNGDATA), + igb_getreg(CTRL_EXT), + igb_getreg(SYSTIMH), + igb_getreg(EEMNGCTL), + igb_getreg(FLMNGDATA), + igb_getreg(TSYNCRXCTL), + igb_getreg(LEDCTL), + igb_getreg(TCTL), + igb_getreg(TCTL_EXT), + igb_getreg(DTXCTL), + igb_getreg(RXPBS), + igb_getreg(TDH0), + igb_getreg(TDH1), + igb_getreg(TDH2), + igb_getreg(TDH3), + igb_getreg(TDH4), + igb_getreg(TDH5), + igb_getreg(TDH6), + igb_getreg(TDH7), + igb_getreg(TDH8), + igb_getreg(TDH9), + igb_getreg(TDH10), + igb_getreg(TDH11), + igb_getreg(TDH12), + igb_getreg(TDH13), + igb_getreg(TDH14), + igb_getreg(TDH15), + igb_getreg(ECOL), + igb_getreg(DC), + igb_getreg(RLEC), + igb_getreg(XOFFTXC), + igb_getreg(RFC), + igb_getreg(RNBC), + igb_getreg(MGTPTC), + igb_getreg(TIMINCA), + igb_getreg(FACTPS), + igb_getreg(GSCL_1), + igb_getreg(GSCN_0), + igb_getreg(PBACLR), + igb_getreg(FCTTV), + igb_getreg(RXSATRL), + igb_getreg(TORL), + igb_getreg(TDLEN0), + igb_getreg(TDLEN1), + igb_getreg(TDLEN2), + igb_getreg(TDLEN3), + igb_getreg(TDLEN4), + igb_getreg(TDLEN5), + igb_getreg(TDLEN6), + igb_getreg(TDLEN7), + igb_getreg(TDLEN8), + igb_getreg(TDLEN9), + igb_getreg(TDLEN10), + igb_getreg(TDLEN11), + igb_getreg(TDLEN12), + igb_getreg(TDLEN13), + igb_getreg(TDLEN14), + igb_getreg(TDLEN15), + igb_getreg(MCC), + igb_getreg(WUC), + igb_getreg(EECD), + igb_getreg(FCRTV), + igb_getreg(TXDCTL0), + igb_getreg(TXDCTL1), + igb_getreg(TXDCTL2), + igb_getreg(TXDCTL3), + igb_getreg(TXDCTL4), + igb_getreg(TXDCTL5), + igb_getreg(TXDCTL6), + igb_getreg(TXDCTL7), + igb_getreg(TXDCTL8), + igb_getreg(TXDCTL9), + igb_getreg(TXDCTL10), + igb_getreg(TXDCTL11), + igb_getreg(TXDCTL12), + igb_getreg(TXDCTL13), + igb_getreg(TXDCTL14), + igb_getreg(TXDCTL15), + igb_getreg(TXCTL0), + igb_getreg(TXCTL1), + igb_getreg(TXCTL2), + igb_getreg(TXCTL3), + igb_getreg(TXCTL4), + igb_getreg(TXCTL5), + igb_getreg(TXCTL6), + igb_getreg(TXCTL7), + igb_getreg(TXCTL8), + igb_getreg(TXCTL9), + igb_getreg(TXCTL10), + igb_getreg(TXCTL11), + igb_getreg(TXCTL12), + igb_getreg(TXCTL13), + igb_getreg(TXCTL14), + igb_getreg(TXCTL15), + igb_getreg(TDWBAL0), + igb_getreg(TDWBAL1), + igb_getreg(TDWBAL2), + igb_getreg(TDWBAL3), + igb_getreg(TDWBAL4), + igb_getreg(TDWBAL5), + igb_getreg(TDWBAL6), + igb_getreg(TDWBAL7), + igb_getreg(TDWBAL8), + igb_getreg(TDWBAL9), + igb_getreg(TDWBAL10), + igb_getreg(TDWBAL11), + igb_getreg(TDWBAL12), + igb_getreg(TDWBAL13), + igb_getreg(TDWBAL14), + igb_getreg(TDWBAL15), + igb_getreg(TDWBAH0), + igb_getreg(TDWBAH1), + igb_getreg(TDWBAH2), + igb_getreg(TDWBAH3), + igb_getreg(TDWBAH4), + igb_getreg(TDWBAH5), + igb_getreg(TDWBAH6), + igb_getreg(TDWBAH7), + igb_getreg(TDWBAH8), + igb_getreg(TDWBAH9), + igb_getreg(TDWBAH10), + igb_getreg(TDWBAH11), + igb_getreg(TDWBAH12), + igb_getreg(TDWBAH13), + igb_getreg(TDWBAH14), + igb_getreg(TDWBAH15), + igb_getreg(PVTCTRL0), + igb_getreg(PVTCTRL1), + igb_getreg(PVTCTRL2), + igb_getreg(PVTCTRL3), + igb_getreg(PVTCTRL4), + igb_getreg(PVTCTRL5), + igb_getreg(PVTCTRL6), + igb_getreg(PVTCTRL7), + igb_getreg(PVTEIMS0), + igb_getreg(PVTEIMS1), + igb_getreg(PVTEIMS2), + igb_getreg(PVTEIMS3), + igb_getreg(PVTEIMS4), + igb_getreg(PVTEIMS5), + igb_getreg(PVTEIMS6), + igb_getreg(PVTEIMS7), + igb_getreg(PVTEIAC0), + igb_getreg(PVTEIAC1), + igb_getreg(PVTEIAC2), + igb_getreg(PVTEIAC3), + igb_getreg(PVTEIAC4), + igb_getreg(PVTEIAC5), + igb_getreg(PVTEIAC6), + igb_getreg(PVTEIAC7), + igb_getreg(PVTEIAM0), + igb_getreg(PVTEIAM1), + igb_getreg(PVTEIAM2), + igb_getreg(PVTEIAM3), + igb_getreg(PVTEIAM4), + igb_getreg(PVTEIAM5), + igb_getreg(PVTEIAM6), + igb_getreg(PVTEIAM7), + igb_getreg(PVFGPRC0), + igb_getreg(PVFGPRC1), + igb_getreg(PVFGPRC2), + igb_getreg(PVFGPRC3), + igb_getreg(PVFGPRC4), + igb_getreg(PVFGPRC5), + igb_getreg(PVFGPRC6), + igb_getreg(PVFGPRC7), + igb_getreg(PVFGPTC0), + igb_getreg(PVFGPTC1), + igb_getreg(PVFGPTC2), + igb_getreg(PVFGPTC3), + igb_getreg(PVFGPTC4), + igb_getreg(PVFGPTC5), + igb_getreg(PVFGPTC6), + igb_getreg(PVFGPTC7), + igb_getreg(PVFGORC0), + igb_getreg(PVFGORC1), + igb_getreg(PVFGORC2), + igb_getreg(PVFGORC3), + igb_getreg(PVFGORC4), + igb_getreg(PVFGORC5), + igb_getreg(PVFGORC6), + igb_getreg(PVFGORC7), + igb_getreg(PVFGOTC0), + igb_getreg(PVFGOTC1), + igb_getreg(PVFGOTC2), + igb_getreg(PVFGOTC3), + igb_getreg(PVFGOTC4), + igb_getreg(PVFGOTC5), + igb_getreg(PVFGOTC6), + igb_getreg(PVFGOTC7), + igb_getreg(PVFMPRC0), + igb_getreg(PVFMPRC1), + igb_getreg(PVFMPRC2), + igb_getreg(PVFMPRC3), + igb_getreg(PVFMPRC4), + igb_getreg(PVFMPRC5), + igb_getreg(PVFMPRC6), + igb_getreg(PVFMPRC7), + igb_getreg(PVFGPRLBC0), + igb_getreg(PVFGPRLBC1), + igb_getreg(PVFGPRLBC2), + igb_getreg(PVFGPRLBC3), + igb_getreg(PVFGPRLBC4), + igb_getreg(PVFGPRLBC5), + igb_getreg(PVFGPRLBC6), + igb_getreg(PVFGPRLBC7), + igb_getreg(PVFGPTLBC0), + igb_getreg(PVFGPTLBC1), + igb_getreg(PVFGPTLBC2), + igb_getreg(PVFGPTLBC3), + igb_getreg(PVFGPTLBC4), + igb_getreg(PVFGPTLBC5), + igb_getreg(PVFGPTLBC6), + igb_getreg(PVFGPTLBC7), + igb_getreg(PVFGORLBC0), + igb_getreg(PVFGORLBC1), + igb_getreg(PVFGORLBC2), + igb_getreg(PVFGORLBC3), + igb_getreg(PVFGORLBC4), + igb_getreg(PVFGORLBC5), + igb_getreg(PVFGORLBC6), + igb_getreg(PVFGORLBC7), + igb_getreg(PVFGOTLBC0), + igb_getreg(PVFGOTLBC1), + igb_getreg(PVFGOTLBC2), + igb_getreg(PVFGOTLBC3), + igb_getreg(PVFGOTLBC4), + igb_getreg(PVFGOTLBC5), + igb_getreg(PVFGOTLBC6), + igb_getreg(PVFGOTLBC7), + igb_getreg(RCTL), + igb_getreg(MDIC), + igb_getreg(FCRUC), + igb_getreg(VET), + igb_getreg(RDBAL0), + igb_getreg(RDBAL1), + igb_getreg(RDBAL2), + igb_getreg(RDBAL3), + igb_getreg(RDBAL4), + igb_getreg(RDBAL5), + igb_getreg(RDBAL6), + igb_getreg(RDBAL7), + igb_getreg(RDBAL8), + igb_getreg(RDBAL9), + igb_getreg(RDBAL10), + igb_getreg(RDBAL11), + igb_getreg(RDBAL12), + igb_getreg(RDBAL13), + igb_getreg(RDBAL14), + igb_getreg(RDBAL15), + igb_getreg(TDBAH0), + igb_getreg(TDBAH1), + igb_getreg(TDBAH2), + igb_getreg(TDBAH3), + igb_getreg(TDBAH4), + igb_getreg(TDBAH5), + igb_getreg(TDBAH6), + igb_getreg(TDBAH7), + igb_getreg(TDBAH8), + igb_getreg(TDBAH9), + igb_getreg(TDBAH10), + igb_getreg(TDBAH11), + igb_getreg(TDBAH12), + igb_getreg(TDBAH13), + igb_getreg(TDBAH14), + igb_getreg(TDBAH15), + igb_getreg(SCC), + igb_getreg(COLC), + igb_getreg(XOFFRXC), + igb_getreg(IPAV), + igb_getreg(GOTCL), + igb_getreg(MGTPDC), + igb_getreg(GCR), + igb_getreg(MFVAL), + igb_getreg(FUNCTAG), + igb_getreg(GSCL_4), + igb_getreg(GSCN_3), + igb_getreg(MRQC), + igb_getreg(FCT), + igb_getreg(FLA), + igb_getreg(RXDCTL0), + igb_getreg(RXDCTL1), + igb_getreg(RXDCTL2), + igb_getreg(RXDCTL3), + igb_getreg(RXDCTL4), + igb_getreg(RXDCTL5), + igb_getreg(RXDCTL6), + igb_getreg(RXDCTL7), + igb_getreg(RXDCTL8), + igb_getreg(RXDCTL9), + igb_getreg(RXDCTL10), + igb_getreg(RXDCTL11), + igb_getreg(RXDCTL12), + igb_getreg(RXDCTL13), + igb_getreg(RXDCTL14), + igb_getreg(RXDCTL15), + igb_getreg(RXSTMPL), + igb_getreg(TIMADJH), + igb_getreg(FCRTL), + igb_getreg(XONRXC), + igb_getreg(RFCTL), + igb_getreg(GSCN_1), + igb_getreg(FCAL), + igb_getreg(GPIE), + igb_getreg(TXPBS), + igb_getreg(RLPML), + + [TOTH] = igb_mac_read_clr8, + [GOTCH] = igb_mac_read_clr8, + [PRC64] = igb_mac_read_clr4, + [PRC255] = igb_mac_read_clr4, + [PRC1023] = igb_mac_read_clr4, + [PTC64] = igb_mac_read_clr4, + [PTC255] = igb_mac_read_clr4, + [PTC1023] = igb_mac_read_clr4, + [GPRC] = igb_mac_read_clr4, + [TPT] = igb_mac_read_clr4, + [RUC] = igb_mac_read_clr4, + [BPRC] = igb_mac_read_clr4, + [MPTC] = igb_mac_read_clr4, + [IAC] = igb_mac_read_clr4, + [ICR] = igb_mac_icr_read, + [STATUS] = igb_get_status, + [ICS] = igb_mac_ics_read, + /* + * 8.8.10: Reading the IMC register returns the value of the IMS register. + */ + [IMC] = igb_mac_ims_read, + [TORH] = igb_mac_read_clr8, + [GORCH] = igb_mac_read_clr8, + [PRC127] = igb_mac_read_clr4, + [PRC511] = igb_mac_read_clr4, + [PRC1522] = igb_mac_read_clr4, + [PTC127] = igb_mac_read_clr4, + [PTC511] = igb_mac_read_clr4, + [PTC1522] = igb_mac_read_clr4, + [GPTC] = igb_mac_read_clr4, + [TPR] = igb_mac_read_clr4, + [ROC] = igb_mac_read_clr4, + [MPRC] = igb_mac_read_clr4, + [BPTC] = igb_mac_read_clr4, + [TSCTC] = igb_mac_read_clr4, + [CTRL] = igb_get_ctrl, + [SWSM] = igb_mac_swsm_read, + [IMS] = igb_mac_ims_read, + [SYSTIML] = igb_get_systiml, + [RXSATRH] = igb_get_rxsatrh, + [TXSTMPH] = igb_get_txstmph, + + [CRCERRS ... MPC] = igb_mac_readreg, + [IP6AT ... IP6AT + 3] = igb_mac_readreg, + [IP4AT ... IP4AT + 6] = igb_mac_readreg, + [RA ... RA + 31] = igb_mac_readreg, + [RA2 ... RA2 + 31] = igb_mac_readreg, + [WUPM ... WUPM + 31] = igb_mac_readreg, + [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = igb_mac_readreg, + [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = igb_mac_readreg, + [FFMT ... FFMT + 254] = igb_mac_readreg, + [MDEF ... MDEF + 7] = igb_mac_readreg, + [FTFT ... FTFT + 254] = igb_mac_readreg, + [RETA ... RETA + 31] = igb_mac_readreg, + [RSSRK ... RSSRK + 9] = igb_mac_readreg, + [MAVTV0 ... MAVTV3] = igb_mac_readreg, + [EITR0 ... EITR0 + IGB_INTR_NUM - 1] = igb_mac_eitr_read, + [PVTEICR0] = igb_mac_read_clr4, + [PVTEICR1] = igb_mac_read_clr4, + [PVTEICR2] = igb_mac_read_clr4, + [PVTEICR3] = igb_mac_read_clr4, + [PVTEICR4] = igb_mac_read_clr4, + [PVTEICR5] = igb_mac_read_clr4, + [PVTEICR6] = igb_mac_read_clr4, + [PVTEICR7] = igb_mac_read_clr4, + + /* IGB specific: */ + [FWSM] = igb_mac_readreg, + [SW_FW_SYNC] = igb_mac_readreg, + [HTCBDPC] = igb_mac_read_clr4, + [EICR] = igb_mac_read_clr4, + [EIMS] = igb_mac_readreg, + [EIAM] = igb_mac_readreg, + [IVAR0 ... IVAR0 + 7] = igb_mac_readreg, + igb_getreg(IVAR_MISC), + igb_getreg(VT_CTL), + [P2VMAILBOX0 ... P2VMAILBOX7] = igb_mac_readreg, + [V2PMAILBOX0 ... V2PMAILBOX7] = igb_mac_vfmailbox_read, + igb_getreg(MBVFICR), + [VMBMEM0 ... VMBMEM0 + 127] = igb_mac_readreg, + igb_getreg(MBVFIMR), + igb_getreg(VFLRE), + igb_getreg(VFRE), + igb_getreg(VFTE), + igb_getreg(QDE), + igb_getreg(DTXSWC), + igb_getreg(RPLOLR), + [VLVF0 ... VLVF0 + E1000_VLVF_ARRAY_SIZE - 1] = igb_mac_readreg, + [VMVIR0 ... VMVIR7] = igb_mac_readreg, + [VMOLR0 ... VMOLR7] = igb_mac_readreg, + [WVBR] = igb_mac_read_clr4, + [RQDPC0] = igb_mac_read_clr4, + [RQDPC1] = igb_mac_read_clr4, + [RQDPC2] = igb_mac_read_clr4, + [RQDPC3] = igb_mac_read_clr4, + [RQDPC4] = igb_mac_read_clr4, + [RQDPC5] = igb_mac_read_clr4, + [RQDPC6] = igb_mac_read_clr4, + [RQDPC7] = igb_mac_read_clr4, + [RQDPC8] = igb_mac_read_clr4, + [RQDPC9] = igb_mac_read_clr4, + [RQDPC10] = igb_mac_read_clr4, + [RQDPC11] = igb_mac_read_clr4, + [RQDPC12] = igb_mac_read_clr4, + [RQDPC13] = igb_mac_read_clr4, + [RQDPC14] = igb_mac_read_clr4, + [RQDPC15] = igb_mac_read_clr4, + [VTIVAR ... VTIVAR + 7] = igb_mac_readreg, + [VTIVAR_MISC ... VTIVAR_MISC + 7] = igb_mac_readreg, +}; +enum { IGB_NREADOPS = ARRAY_SIZE(igb_macreg_readops) }; + +#define igb_putreg(x) [x] = igb_mac_writereg +typedef void (*writeops)(IGBCore *, int, uint32_t); +static const writeops igb_macreg_writeops[] = { + igb_putreg(SWSM), + igb_putreg(WUFC), + igb_putreg(RDBAH0), + igb_putreg(RDBAH1), + igb_putreg(RDBAH2), + igb_putreg(RDBAH3), + igb_putreg(RDBAH4), + igb_putreg(RDBAH5), + igb_putreg(RDBAH6), + igb_putreg(RDBAH7), + igb_putreg(RDBAH8), + igb_putreg(RDBAH9), + igb_putreg(RDBAH10), + igb_putreg(RDBAH11), + igb_putreg(RDBAH12), + igb_putreg(RDBAH13), + igb_putreg(RDBAH14), + igb_putreg(RDBAH15), + igb_putreg(SRRCTL0), + igb_putreg(SRRCTL1), + igb_putreg(SRRCTL2), + igb_putreg(SRRCTL3), + igb_putreg(SRRCTL4), + igb_putreg(SRRCTL5), + igb_putreg(SRRCTL6), + igb_putreg(SRRCTL7), + igb_putreg(SRRCTL8), + igb_putreg(SRRCTL9), + igb_putreg(SRRCTL10), + igb_putreg(SRRCTL11), + igb_putreg(SRRCTL12), + igb_putreg(SRRCTL13), + igb_putreg(SRRCTL14), + igb_putreg(SRRCTL15), + igb_putreg(RXDCTL0), + igb_putreg(RXDCTL1), + igb_putreg(RXDCTL2), + igb_putreg(RXDCTL3), + igb_putreg(RXDCTL4), + igb_putreg(RXDCTL5), + igb_putreg(RXDCTL6), + igb_putreg(RXDCTL7), + igb_putreg(RXDCTL8), + igb_putreg(RXDCTL9), + igb_putreg(RXDCTL10), + igb_putreg(RXDCTL11), + igb_putreg(RXDCTL12), + igb_putreg(RXDCTL13), + igb_putreg(RXDCTL14), + igb_putreg(RXDCTL15), + igb_putreg(LEDCTL), + igb_putreg(TCTL), + igb_putreg(TCTL_EXT), + igb_putreg(DTXCTL), + igb_putreg(RXPBS), + igb_putreg(RQDPC0), + igb_putreg(FCAL), + igb_putreg(FCRUC), + igb_putreg(WUC), + igb_putreg(WUS), + igb_putreg(IPAV), + igb_putreg(TDBAH0), + igb_putreg(TDBAH1), + igb_putreg(TDBAH2), + igb_putreg(TDBAH3), + igb_putreg(TDBAH4), + igb_putreg(TDBAH5), + igb_putreg(TDBAH6), + igb_putreg(TDBAH7), + igb_putreg(TDBAH8), + igb_putreg(TDBAH9), + igb_putreg(TDBAH10), + igb_putreg(TDBAH11), + igb_putreg(TDBAH12), + igb_putreg(TDBAH13), + igb_putreg(TDBAH14), + igb_putreg(TDBAH15), + igb_putreg(IAM), + igb_putreg(MANC), + igb_putreg(MANC2H), + igb_putreg(MFVAL), + igb_putreg(FACTPS), + igb_putreg(FUNCTAG), + igb_putreg(GSCL_1), + igb_putreg(GSCL_2), + igb_putreg(GSCL_3), + igb_putreg(GSCL_4), + igb_putreg(GSCN_0), + igb_putreg(GSCN_1), + igb_putreg(GSCN_2), + igb_putreg(GSCN_3), + igb_putreg(MRQC), + igb_putreg(FLOP), + igb_putreg(FLA), + igb_putreg(TXDCTL0), + igb_putreg(TXDCTL1), + igb_putreg(TXDCTL2), + igb_putreg(TXDCTL3), + igb_putreg(TXDCTL4), + igb_putreg(TXDCTL5), + igb_putreg(TXDCTL6), + igb_putreg(TXDCTL7), + igb_putreg(TXDCTL8), + igb_putreg(TXDCTL9), + igb_putreg(TXDCTL10), + igb_putreg(TXDCTL11), + igb_putreg(TXDCTL12), + igb_putreg(TXDCTL13), + igb_putreg(TXDCTL14), + igb_putreg(TXDCTL15), + igb_putreg(TXCTL0), + igb_putreg(TXCTL1), + igb_putreg(TXCTL2), + igb_putreg(TXCTL3), + igb_putreg(TXCTL4), + igb_putreg(TXCTL5), + igb_putreg(TXCTL6), + igb_putreg(TXCTL7), + igb_putreg(TXCTL8), + igb_putreg(TXCTL9), + igb_putreg(TXCTL10), + igb_putreg(TXCTL11), + igb_putreg(TXCTL12), + igb_putreg(TXCTL13), + igb_putreg(TXCTL14), + igb_putreg(TXCTL15), + igb_putreg(TDWBAL0), + igb_putreg(TDWBAL1), + igb_putreg(TDWBAL2), + igb_putreg(TDWBAL3), + igb_putreg(TDWBAL4), + igb_putreg(TDWBAL5), + igb_putreg(TDWBAL6), + igb_putreg(TDWBAL7), + igb_putreg(TDWBAL8), + igb_putreg(TDWBAL9), + igb_putreg(TDWBAL10), + igb_putreg(TDWBAL11), + igb_putreg(TDWBAL12), + igb_putreg(TDWBAL13), + igb_putreg(TDWBAL14), + igb_putreg(TDWBAL15), + igb_putreg(TDWBAH0), + igb_putreg(TDWBAH1), + igb_putreg(TDWBAH2), + igb_putreg(TDWBAH3), + igb_putreg(TDWBAH4), + igb_putreg(TDWBAH5), + igb_putreg(TDWBAH6), + igb_putreg(TDWBAH7), + igb_putreg(TDWBAH8), + igb_putreg(TDWBAH9), + igb_putreg(TDWBAH10), + igb_putreg(TDWBAH11), + igb_putreg(TDWBAH12), + igb_putreg(TDWBAH13), + igb_putreg(TDWBAH14), + igb_putreg(TDWBAH15), + igb_putreg(TIPG), + igb_putreg(RXSTMPH), + igb_putreg(RXSTMPL), + igb_putreg(RXSATRL), + igb_putreg(RXSATRH), + igb_putreg(TXSTMPL), + igb_putreg(TXSTMPH), + igb_putreg(SYSTIML), + igb_putreg(SYSTIMH), + igb_putreg(TIMADJL), + igb_putreg(TSYNCRXCTL), + igb_putreg(TSYNCTXCTL), + igb_putreg(EEMNGCTL), + igb_putreg(GPIE), + igb_putreg(TXPBS), + igb_putreg(RLPML), + igb_putreg(VET), + + [TDH0] = igb_set_16bit, + [TDH1] = igb_set_16bit, + [TDH2] = igb_set_16bit, + [TDH3] = igb_set_16bit, + [TDH4] = igb_set_16bit, + [TDH5] = igb_set_16bit, + [TDH6] = igb_set_16bit, + [TDH7] = igb_set_16bit, + [TDH8] = igb_set_16bit, + [TDH9] = igb_set_16bit, + [TDH10] = igb_set_16bit, + [TDH11] = igb_set_16bit, + [TDH12] = igb_set_16bit, + [TDH13] = igb_set_16bit, + [TDH14] = igb_set_16bit, + [TDH15] = igb_set_16bit, + [TDT0] = igb_set_tdt, + [TDT1] = igb_set_tdt, + [TDT2] = igb_set_tdt, + [TDT3] = igb_set_tdt, + [TDT4] = igb_set_tdt, + [TDT5] = igb_set_tdt, + [TDT6] = igb_set_tdt, + [TDT7] = igb_set_tdt, + [TDT8] = igb_set_tdt, + [TDT9] = igb_set_tdt, + [TDT10] = igb_set_tdt, + [TDT11] = igb_set_tdt, + [TDT12] = igb_set_tdt, + [TDT13] = igb_set_tdt, + [TDT14] = igb_set_tdt, + [TDT15] = igb_set_tdt, + [MDIC] = igb_set_mdic, + [ICS] = igb_set_ics, + [RDH0] = igb_set_16bit, + [RDH1] = igb_set_16bit, + [RDH2] = igb_set_16bit, + [RDH3] = igb_set_16bit, + [RDH4] = igb_set_16bit, + [RDH5] = igb_set_16bit, + [RDH6] = igb_set_16bit, + [RDH7] = igb_set_16bit, + [RDH8] = igb_set_16bit, + [RDH9] = igb_set_16bit, + [RDH10] = igb_set_16bit, + [RDH11] = igb_set_16bit, + [RDH12] = igb_set_16bit, + [RDH13] = igb_set_16bit, + [RDH14] = igb_set_16bit, + [RDH15] = igb_set_16bit, + [RDT0] = igb_set_rdt, + [RDT1] = igb_set_rdt, + [RDT2] = igb_set_rdt, + [RDT3] = igb_set_rdt, + [RDT4] = igb_set_rdt, + [RDT5] = igb_set_rdt, + [RDT6] = igb_set_rdt, + [RDT7] = igb_set_rdt, + [RDT8] = igb_set_rdt, + [RDT9] = igb_set_rdt, + [RDT10] = igb_set_rdt, + [RDT11] = igb_set_rdt, + [RDT12] = igb_set_rdt, + [RDT13] = igb_set_rdt, + [RDT14] = igb_set_rdt, + [RDT15] = igb_set_rdt, + [IMC] = igb_set_imc, + [IMS] = igb_set_ims, + [ICR] = igb_set_icr, + [EECD] = igb_set_eecd, + [RCTL] = igb_set_rx_control, + [CTRL] = igb_set_ctrl, + [EERD] = igb_set_eerd, + [TDFH] = igb_set_13bit, + [TDFT] = igb_set_13bit, + [TDFHS] = igb_set_13bit, + [TDFTS] = igb_set_13bit, + [TDFPC] = igb_set_13bit, + [RDFH] = igb_set_13bit, + [RDFT] = igb_set_13bit, + [RDFHS] = igb_set_13bit, + [RDFTS] = igb_set_13bit, + [RDFPC] = igb_set_13bit, + [GCR] = igb_set_gcr, + [RXCSUM] = igb_set_rxcsum, + [TDLEN0] = igb_set_dlen, + [TDLEN1] = igb_set_dlen, + [TDLEN2] = igb_set_dlen, + [TDLEN3] = igb_set_dlen, + [TDLEN4] = igb_set_dlen, + [TDLEN5] = igb_set_dlen, + [TDLEN6] = igb_set_dlen, + [TDLEN7] = igb_set_dlen, + [TDLEN8] = igb_set_dlen, + [TDLEN9] = igb_set_dlen, + [TDLEN10] = igb_set_dlen, + [TDLEN11] = igb_set_dlen, + [TDLEN12] = igb_set_dlen, + [TDLEN13] = igb_set_dlen, + [TDLEN14] = igb_set_dlen, + [TDLEN15] = igb_set_dlen, + [RDLEN0] = igb_set_dlen, + [RDLEN1] = igb_set_dlen, + [RDLEN2] = igb_set_dlen, + [RDLEN3] = igb_set_dlen, + [RDLEN4] = igb_set_dlen, + [RDLEN5] = igb_set_dlen, + [RDLEN6] = igb_set_dlen, + [RDLEN7] = igb_set_dlen, + [RDLEN8] = igb_set_dlen, + [RDLEN9] = igb_set_dlen, + [RDLEN10] = igb_set_dlen, + [RDLEN11] = igb_set_dlen, + [RDLEN12] = igb_set_dlen, + [RDLEN13] = igb_set_dlen, + [RDLEN14] = igb_set_dlen, + [RDLEN15] = igb_set_dlen, + [TDBAL0] = igb_set_dbal, + [TDBAL1] = igb_set_dbal, + [TDBAL2] = igb_set_dbal, + [TDBAL3] = igb_set_dbal, + [TDBAL4] = igb_set_dbal, + [TDBAL5] = igb_set_dbal, + [TDBAL6] = igb_set_dbal, + [TDBAL7] = igb_set_dbal, + [TDBAL8] = igb_set_dbal, + [TDBAL9] = igb_set_dbal, + [TDBAL10] = igb_set_dbal, + [TDBAL11] = igb_set_dbal, + [TDBAL12] = igb_set_dbal, + [TDBAL13] = igb_set_dbal, + [TDBAL14] = igb_set_dbal, + [TDBAL15] = igb_set_dbal, + [RDBAL0] = igb_set_dbal, + [RDBAL1] = igb_set_dbal, + [RDBAL2] = igb_set_dbal, + [RDBAL3] = igb_set_dbal, + [RDBAL4] = igb_set_dbal, + [RDBAL5] = igb_set_dbal, + [RDBAL6] = igb_set_dbal, + [RDBAL7] = igb_set_dbal, + [RDBAL8] = igb_set_dbal, + [RDBAL9] = igb_set_dbal, + [RDBAL10] = igb_set_dbal, + [RDBAL11] = igb_set_dbal, + [RDBAL12] = igb_set_dbal, + [RDBAL13] = igb_set_dbal, + [RDBAL14] = igb_set_dbal, + [RDBAL15] = igb_set_dbal, + [STATUS] = igb_set_status, + [PBACLR] = igb_set_pbaclr, + [CTRL_EXT] = igb_set_ctrlext, + [FCAH] = igb_set_16bit, + [FCT] = igb_set_16bit, + [FCTTV] = igb_set_16bit, + [FCRTV] = igb_set_16bit, + [FCRTH] = igb_set_fcrth, + [FCRTL] = igb_set_fcrtl, + [CTRL_DUP] = igb_set_ctrl, + [RFCTL] = igb_set_rfctl, + [TIMINCA] = igb_set_timinca, + [TIMADJH] = igb_set_timadjh, + + [IP6AT ... IP6AT + 3] = igb_mac_writereg, + [IP4AT ... IP4AT + 6] = igb_mac_writereg, + [RA] = igb_mac_writereg, + [RA + 1] = igb_mac_setmacaddr, + [RA + 2 ... RA + 31] = igb_mac_writereg, + [RA2 ... RA2 + 31] = igb_mac_writereg, + [WUPM ... WUPM + 31] = igb_mac_writereg, + [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = igb_mac_writereg, + [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = igb_mac_writereg, + [FFMT ... FFMT + 254] = igb_set_4bit, + [MDEF ... MDEF + 7] = igb_mac_writereg, + [FTFT ... FTFT + 254] = igb_mac_writereg, + [RETA ... RETA + 31] = igb_mac_writereg, + [RSSRK ... RSSRK + 9] = igb_mac_writereg, + [MAVTV0 ... MAVTV3] = igb_mac_writereg, + [EITR0 ... EITR0 + IGB_INTR_NUM - 1] = igb_set_eitr, + + /* IGB specific: */ + [FWSM] = igb_mac_writereg, + [SW_FW_SYNC] = igb_mac_writereg, + [EICR] = igb_set_eicr, + [EICS] = igb_set_eics, + [EIAC] = igb_set_eiac, + [EIAM] = igb_set_eiam, + [EIMC] = igb_set_eimc, + [EIMS] = igb_set_eims, + [IVAR0 ... IVAR0 + 7] = igb_mac_writereg, + igb_putreg(IVAR_MISC), + igb_putreg(VT_CTL), + [P2VMAILBOX0 ... P2VMAILBOX7] = igb_set_pfmailbox, + [V2PMAILBOX0 ... V2PMAILBOX7] = igb_set_vfmailbox, + [MBVFICR] = igb_w1c, + [VMBMEM0 ... VMBMEM0 + 127] = igb_mac_writereg, + igb_putreg(MBVFIMR), + [VFLRE] = igb_w1c, + igb_putreg(VFRE), + igb_putreg(VFTE), + igb_putreg(QDE), + igb_putreg(DTXSWC), + igb_putreg(RPLOLR), + [VLVF0 ... VLVF0 + E1000_VLVF_ARRAY_SIZE - 1] = igb_mac_writereg, + [VMVIR0 ... VMVIR7] = igb_mac_writereg, + [VMOLR0 ... VMOLR7] = igb_mac_writereg, + [UTA ... UTA + E1000_MC_TBL_SIZE - 1] = igb_mac_writereg, + [PVTCTRL0] = igb_set_vtctrl, + [PVTCTRL1] = igb_set_vtctrl, + [PVTCTRL2] = igb_set_vtctrl, + [PVTCTRL3] = igb_set_vtctrl, + [PVTCTRL4] = igb_set_vtctrl, + [PVTCTRL5] = igb_set_vtctrl, + [PVTCTRL6] = igb_set_vtctrl, + [PVTCTRL7] = igb_set_vtctrl, + [PVTEICS0] = igb_set_vteics, + [PVTEICS1] = igb_set_vteics, + [PVTEICS2] = igb_set_vteics, + [PVTEICS3] = igb_set_vteics, + [PVTEICS4] = igb_set_vteics, + [PVTEICS5] = igb_set_vteics, + [PVTEICS6] = igb_set_vteics, + [PVTEICS7] = igb_set_vteics, + [PVTEIMS0] = igb_set_vteims, + [PVTEIMS1] = igb_set_vteims, + [PVTEIMS2] = igb_set_vteims, + [PVTEIMS3] = igb_set_vteims, + [PVTEIMS4] = igb_set_vteims, + [PVTEIMS5] = igb_set_vteims, + [PVTEIMS6] = igb_set_vteims, + [PVTEIMS7] = igb_set_vteims, + [PVTEIMC0] = igb_set_vteimc, + [PVTEIMC1] = igb_set_vteimc, + [PVTEIMC2] = igb_set_vteimc, + [PVTEIMC3] = igb_set_vteimc, + [PVTEIMC4] = igb_set_vteimc, + [PVTEIMC5] = igb_set_vteimc, + [PVTEIMC6] = igb_set_vteimc, + [PVTEIMC7] = igb_set_vteimc, + [PVTEIAC0] = igb_set_vteiac, + [PVTEIAC1] = igb_set_vteiac, + [PVTEIAC2] = igb_set_vteiac, + [PVTEIAC3] = igb_set_vteiac, + [PVTEIAC4] = igb_set_vteiac, + [PVTEIAC5] = igb_set_vteiac, + [PVTEIAC6] = igb_set_vteiac, + [PVTEIAC7] = igb_set_vteiac, + [PVTEIAM0] = igb_set_vteiam, + [PVTEIAM1] = igb_set_vteiam, + [PVTEIAM2] = igb_set_vteiam, + [PVTEIAM3] = igb_set_vteiam, + [PVTEIAM4] = igb_set_vteiam, + [PVTEIAM5] = igb_set_vteiam, + [PVTEIAM6] = igb_set_vteiam, + [PVTEIAM7] = igb_set_vteiam, + [PVTEICR0] = igb_set_vteicr, + [PVTEICR1] = igb_set_vteicr, + [PVTEICR2] = igb_set_vteicr, + [PVTEICR3] = igb_set_vteicr, + [PVTEICR4] = igb_set_vteicr, + [PVTEICR5] = igb_set_vteicr, + [PVTEICR6] = igb_set_vteicr, + [PVTEICR7] = igb_set_vteicr, + [VTIVAR ... VTIVAR + 7] = igb_set_vtivar, + [VTIVAR_MISC ... VTIVAR_MISC + 7] = igb_mac_writereg +}; +enum { IGB_NWRITEOPS = ARRAY_SIZE(igb_macreg_writeops) }; + +enum { MAC_ACCESS_PARTIAL = 1 }; + +/* + * The array below combines alias offsets of the index values for the + * MAC registers that have aliases, with the indication of not fully + * implemented registers (lowest bit). This combination is possible + * because all of the offsets are even. + */ +static const uint16_t mac_reg_access[E1000E_MAC_SIZE] = { + /* Alias index offsets */ + [FCRTL_A] = 0x07fe, + [RDFH_A] = 0xe904, [RDFT_A] = 0xe904, + [TDFH_A] = 0xed00, [TDFT_A] = 0xed00, + [RA_A ... RA_A + 31] = 0x14f0, + [VFTA_A ... VFTA_A + E1000_VLAN_FILTER_TBL_SIZE - 1] = 0x1400, + + [RDBAL0_A] = 0x2600, + [RDBAH0_A] = 0x2600, + [RDLEN0_A] = 0x2600, + [SRRCTL0_A] = 0x2600, + [RDH0_A] = 0x2600, + [RDT0_A] = 0x2600, + [RXDCTL0_A] = 0x2600, + [RXCTL0_A] = 0x2600, + [RQDPC0_A] = 0x2600, + [RDBAL1_A] = 0x25D0, + [RDBAL2_A] = 0x25A0, + [RDBAL3_A] = 0x2570, + [RDBAH1_A] = 0x25D0, + [RDBAH2_A] = 0x25A0, + [RDBAH3_A] = 0x2570, + [RDLEN1_A] = 0x25D0, + [RDLEN2_A] = 0x25A0, + [RDLEN3_A] = 0x2570, + [SRRCTL1_A] = 0x25D0, + [SRRCTL2_A] = 0x25A0, + [SRRCTL3_A] = 0x2570, + [RDH1_A] = 0x25D0, + [RDH2_A] = 0x25A0, + [RDH3_A] = 0x2570, + [RDT1_A] = 0x25D0, + [RDT2_A] = 0x25A0, + [RDT3_A] = 0x2570, + [RXDCTL1_A] = 0x25D0, + [RXDCTL2_A] = 0x25A0, + [RXDCTL3_A] = 0x2570, + [RXCTL1_A] = 0x25D0, + [RXCTL2_A] = 0x25A0, + [RXCTL3_A] = 0x2570, + [RQDPC1_A] = 0x25D0, + [RQDPC2_A] = 0x25A0, + [RQDPC3_A] = 0x2570, + [TDBAL0_A] = 0x2A00, + [TDBAH0_A] = 0x2A00, + [TDLEN0_A] = 0x2A00, + [TDH0_A] = 0x2A00, + [TDT0_A] = 0x2A00, + [TXCTL0_A] = 0x2A00, + [TDWBAL0_A] = 0x2A00, + [TDWBAH0_A] = 0x2A00, + [TDBAL1_A] = 0x29D0, + [TDBAL2_A] = 0x29A0, + [TDBAL3_A] = 0x2970, + [TDBAH1_A] = 0x29D0, + [TDBAH2_A] = 0x29A0, + [TDBAH3_A] = 0x2970, + [TDLEN1_A] = 0x29D0, + [TDLEN2_A] = 0x29A0, + [TDLEN3_A] = 0x2970, + [TDH1_A] = 0x29D0, + [TDH2_A] = 0x29A0, + [TDH3_A] = 0x2970, + [TDT1_A] = 0x29D0, + [TDT2_A] = 0x29A0, + [TDT3_A] = 0x2970, + [TXDCTL0_A] = 0x2A00, + [TXDCTL1_A] = 0x29D0, + [TXDCTL2_A] = 0x29A0, + [TXDCTL3_A] = 0x2970, + [TXCTL1_A] = 0x29D0, + [TXCTL2_A] = 0x29A0, + [TXCTL3_A] = 0x29D0, + [TDWBAL1_A] = 0x29D0, + [TDWBAL2_A] = 0x29A0, + [TDWBAL3_A] = 0x2970, + [TDWBAH1_A] = 0x29D0, + [TDWBAH2_A] = 0x29A0, + [TDWBAH3_A] = 0x2970, + + /* Access options */ + [RDFH] = MAC_ACCESS_PARTIAL, [RDFT] = MAC_ACCESS_PARTIAL, + [RDFHS] = MAC_ACCESS_PARTIAL, [RDFTS] = MAC_ACCESS_PARTIAL, + [RDFPC] = MAC_ACCESS_PARTIAL, + [TDFH] = MAC_ACCESS_PARTIAL, [TDFT] = MAC_ACCESS_PARTIAL, + [TDFHS] = MAC_ACCESS_PARTIAL, [TDFTS] = MAC_ACCESS_PARTIAL, + [TDFPC] = MAC_ACCESS_PARTIAL, [EECD] = MAC_ACCESS_PARTIAL, + [FLA] = MAC_ACCESS_PARTIAL, + [FCAL] = MAC_ACCESS_PARTIAL, [FCAH] = MAC_ACCESS_PARTIAL, + [FCT] = MAC_ACCESS_PARTIAL, [FCTTV] = MAC_ACCESS_PARTIAL, + [FCRTV] = MAC_ACCESS_PARTIAL, [FCRTL] = MAC_ACCESS_PARTIAL, + [FCRTH] = MAC_ACCESS_PARTIAL, + [MAVTV0 ... MAVTV3] = MAC_ACCESS_PARTIAL +}; + +void +igb_core_write(IGBCore *core, hwaddr addr, uint64_t val, unsigned size) +{ + uint16_t index = igb_get_reg_index_with_offset(mac_reg_access, addr); + + if (index < IGB_NWRITEOPS && igb_macreg_writeops[index]) { + if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) { + trace_e1000e_wrn_regs_write_trivial(index << 2); + } + trace_e1000e_core_write(index << 2, size, val); + igb_macreg_writeops[index](core, index, val); + } else if (index < IGB_NREADOPS && igb_macreg_readops[index]) { + trace_e1000e_wrn_regs_write_ro(index << 2, size, val); + } else { + trace_e1000e_wrn_regs_write_unknown(index << 2, size, val); + } +} + +uint64_t +igb_core_read(IGBCore *core, hwaddr addr, unsigned size) +{ + uint64_t val; + uint16_t index = igb_get_reg_index_with_offset(mac_reg_access, addr); + + if (index < IGB_NREADOPS && igb_macreg_readops[index]) { + if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) { + trace_e1000e_wrn_regs_read_trivial(index << 2); + } + val = igb_macreg_readops[index](core, index); + trace_e1000e_core_read(index << 2, size, val); + return val; + } else { + trace_e1000e_wrn_regs_read_unknown(index << 2, size); + } + return 0; +} + +static inline void +igb_autoneg_pause(IGBCore *core) +{ + timer_del(core->autoneg_timer); +} + +static void +igb_autoneg_resume(IGBCore *core) +{ + if (igb_have_autoneg(core) && + !(core->phy[MII_BMSR] & MII_BMSR_AN_COMP)) { + qemu_get_queue(core->owner_nic)->link_down = false; + timer_mod(core->autoneg_timer, + qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500); + } +} + +static void +igb_vm_state_change(void *opaque, bool running, RunState state) +{ + IGBCore *core = opaque; + + if (running) { + trace_e1000e_vm_state_running(); + igb_intrmgr_resume(core); + igb_autoneg_resume(core); + } else { + trace_e1000e_vm_state_stopped(); + igb_autoneg_pause(core); + igb_intrmgr_pause(core); + } +} + +void +igb_core_pci_realize(IGBCore *core, + const uint16_t *eeprom_templ, + uint32_t eeprom_size, + const uint8_t *macaddr) +{ + int i; + + core->autoneg_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, + igb_autoneg_timer, core); + igb_intrmgr_pci_realize(core); + + core->vmstate = qemu_add_vm_change_state_handler(igb_vm_state_change, core); + + for (i = 0; i < IGB_NUM_QUEUES; i++) { + net_tx_pkt_init(&core->tx[i].tx_pkt, core->owner, E1000E_MAX_TX_FRAGS); + } + + net_rx_pkt_init(&core->rx_pkt); + + e1000x_core_prepare_eeprom(core->eeprom, + eeprom_templ, + eeprom_size, + PCI_DEVICE_GET_CLASS(core->owner)->device_id, + macaddr); + igb_update_rx_offloads(core); +} + +void +igb_core_pci_uninit(IGBCore *core) +{ + int i; + + timer_free(core->autoneg_timer); + + igb_intrmgr_pci_unint(core); + + qemu_del_vm_change_state_handler(core->vmstate); + + for (i = 0; i < IGB_NUM_QUEUES; i++) { + net_tx_pkt_reset(core->tx[i].tx_pkt); + net_tx_pkt_uninit(core->tx[i].tx_pkt); + } + + net_rx_pkt_uninit(core->rx_pkt); +} + +static const uint16_t +igb_phy_reg_init[] = { + [MII_BMCR] = MII_BMCR_SPEED1000 | + MII_BMCR_FD | + MII_BMCR_AUTOEN, + + [MII_BMSR] = MII_BMSR_EXTCAP | + MII_BMSR_LINK_ST | + MII_BMSR_AUTONEG | + MII_BMSR_MFPS | + MII_BMSR_EXTSTAT | + MII_BMSR_10T_HD | + MII_BMSR_10T_FD | + MII_BMSR_100TX_HD | + MII_BMSR_100TX_FD, + + [MII_PHYID1] = IGP03E1000_E_PHY_ID >> 16, + [MII_PHYID2] = (IGP03E1000_E_PHY_ID & 0xfff0) | 1, + [MII_ANAR] = MII_ANAR_CSMACD | MII_ANAR_10 | + MII_ANAR_10FD | MII_ANAR_TX | + MII_ANAR_TXFD | MII_ANAR_PAUSE | + MII_ANAR_PAUSE_ASYM, + [MII_ANLPAR] = MII_ANLPAR_10 | MII_ANLPAR_10FD | + MII_ANLPAR_TX | MII_ANLPAR_TXFD | + MII_ANLPAR_T4 | MII_ANLPAR_PAUSE, + [MII_ANER] = MII_ANER_NP | MII_ANER_NWAY, + [MII_ANNP] = 0x1 | MII_ANNP_MP, + [MII_CTRL1000] = MII_CTRL1000_HALF | MII_CTRL1000_FULL | + MII_CTRL1000_PORT | MII_CTRL1000_MASTER, + [MII_STAT1000] = MII_STAT1000_HALF | MII_STAT1000_FULL | + MII_STAT1000_ROK | MII_STAT1000_LOK, + [MII_EXTSTAT] = MII_EXTSTAT_1000T_HD | MII_EXTSTAT_1000T_FD, + + [IGP01E1000_PHY_PORT_CONFIG] = BIT(5) | BIT(8), + [IGP01E1000_PHY_PORT_STATUS] = IGP01E1000_PSSR_SPEED_1000MBPS, + [IGP02E1000_PHY_POWER_MGMT] = BIT(0) | BIT(3) | IGP02E1000_PM_D3_LPLU | + IGP01E1000_PSCFR_SMART_SPEED +}; + +static const uint32_t igb_mac_reg_init[] = { + [LEDCTL] = 2 | (3 << 8) | BIT(15) | (6 << 16) | (7 << 24), + [EEMNGCTL] = BIT(31), + [RXDCTL0] = E1000_RXDCTL_QUEUE_ENABLE | (1 << 16), + [RXDCTL1] = 1 << 16, + [RXDCTL2] = 1 << 16, + [RXDCTL3] = 1 << 16, + [RXDCTL4] = 1 << 16, + [RXDCTL5] = 1 << 16, + [RXDCTL6] = 1 << 16, + [RXDCTL7] = 1 << 16, + [RXDCTL8] = 1 << 16, + [RXDCTL9] = 1 << 16, + [RXDCTL10] = 1 << 16, + [RXDCTL11] = 1 << 16, + [RXDCTL12] = 1 << 16, + [RXDCTL13] = 1 << 16, + [RXDCTL14] = 1 << 16, + [RXDCTL15] = 1 << 16, + [TIPG] = 0x08 | (0x04 << 10) | (0x06 << 20), + [CTRL] = E1000_CTRL_FD | E1000_CTRL_LRST | E1000_CTRL_SPD_1000 | + E1000_CTRL_ADVD3WUC, + [STATUS] = E1000_STATUS_PHYRA | BIT(31), + [EECD] = E1000_EECD_FWE_DIS | E1000_EECD_PRES | + (2 << E1000_EECD_SIZE_EX_SHIFT), + [GCR] = E1000_L0S_ADJUST | + E1000_GCR_CMPL_TMOUT_RESEND | + E1000_GCR_CAP_VER2 | + E1000_L1_ENTRY_LATENCY_MSB | + E1000_L1_ENTRY_LATENCY_LSB, + [RXCSUM] = E1000_RXCSUM_IPOFLD | E1000_RXCSUM_TUOFLD, + [TXPBS] = 0x28, + [RXPBS] = 0x40, + [TCTL] = E1000_TCTL_PSP | (0xF << E1000_CT_SHIFT) | + (0x40 << E1000_COLD_SHIFT) | (0x1 << 26) | (0xA << 28), + [TCTL_EXT] = 0x40 | (0x42 << 10), + [DTXCTL] = E1000_DTXCTL_8023LL | E1000_DTXCTL_SPOOF_INT, + [VET] = ETH_P_VLAN | (ETH_P_VLAN << 16), + + [V2PMAILBOX0 ... V2PMAILBOX0 + IGB_MAX_VF_FUNCTIONS - 1] = E1000_V2PMAILBOX_RSTI, + [MBVFIMR] = 0xFF, + [VFRE] = 0xFF, + [VFTE] = 0xFF, + [VMOLR0 ... VMOLR0 + 7] = 0x2600 | E1000_VMOLR_STRCRC, + [RPLOLR] = E1000_RPLOLR_STRCRC, + [RLPML] = 0x2600, + [TXCTL0] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL1] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL2] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL3] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL4] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL5] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL6] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL7] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL8] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL9] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL10] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL11] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL12] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL13] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL14] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, + [TXCTL15] = E1000_DCA_TXCTRL_DATA_RRO_EN | + E1000_DCA_TXCTRL_TX_WB_RO_EN | + E1000_DCA_TXCTRL_DESC_RRO_EN, +}; + +static void igb_reset(IGBCore *core, bool sw) +{ + struct igb_tx *tx; + int i; + + timer_del(core->autoneg_timer); + + igb_intrmgr_reset(core); + + memset(core->phy, 0, sizeof core->phy); + memcpy(core->phy, igb_phy_reg_init, sizeof igb_phy_reg_init); + + for (i = 0; i < E1000E_MAC_SIZE; i++) { + if (sw && + (i == RXPBS || i == TXPBS || + (i >= EITR0 && i < EITR0 + IGB_INTR_NUM))) { + continue; + } + + core->mac[i] = i < ARRAY_SIZE(igb_mac_reg_init) ? + igb_mac_reg_init[i] : 0; + } + + if (qemu_get_queue(core->owner_nic)->link_down) { + igb_link_down(core); + } + + e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac); + + for (i = 0; i < ARRAY_SIZE(core->tx); i++) { + tx = &core->tx[i]; + net_tx_pkt_reset(tx->tx_pkt); + tx->vlan = 0; + tx->mss = 0; + tx->tse = false; + tx->ixsm = false; + tx->txsm = false; + tx->first = true; + tx->skip_cp = false; + } +} + +void +igb_core_reset(IGBCore *core) +{ + igb_reset(core, false); +} + +void igb_core_pre_save(IGBCore *core) +{ + int i; + NetClientState *nc = qemu_get_queue(core->owner_nic); + + /* + * If link is down and auto-negotiation is supported and ongoing, + * complete auto-negotiation immediately. This allows us to look + * at MII_BMSR_AN_COMP to infer link status on load. + */ + if (nc->link_down && igb_have_autoneg(core)) { + core->phy[MII_BMSR] |= MII_BMSR_AN_COMP; + igb_update_flowctl_status(core); + } + + for (i = 0; i < ARRAY_SIZE(core->tx); i++) { + if (net_tx_pkt_has_fragments(core->tx[i].tx_pkt)) { + core->tx[i].skip_cp = true; + } + } +} + +int +igb_core_post_load(IGBCore *core) +{ + NetClientState *nc = qemu_get_queue(core->owner_nic); + + /* + * nc.link_down can't be migrated, so infer link_down according + * to link status bit in core.mac[STATUS]. + */ + nc->link_down = (core->mac[STATUS] & E1000_STATUS_LU) == 0; + + return 0; +} diff --git a/hw/net/igb_core.h b/hw/net/igb_core.h new file mode 100644 index 0000000000..814c1e264b --- /dev/null +++ b/hw/net/igb_core.h @@ -0,0 +1,146 @@ +/* + * Core code for QEMU igb emulation + * + * Datasheet: + * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eg-gbe-datasheet.pdf + * + * Copyright (c) 2020-2023 Red Hat, Inc. + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Akihiko Odaki <akihiko.odaki@daynix.com> + * Gal Hammmer <gal.hammer@sap.com> + * Marcel Apfelbaum <marcel.apfelbaum@gmail.com> + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2008 Qumranet + * Based on work done by: + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef HW_NET_IGB_CORE_H +#define HW_NET_IGB_CORE_H + +#define E1000E_MAC_SIZE (0x8000) +#define IGB_EEPROM_SIZE (1024) + +#define IGB_INTR_NUM (25) +#define IGB_MSIX_VEC_NUM (10) +#define IGBVF_MSIX_VEC_NUM (3) +#define IGB_NUM_QUEUES (16) + +typedef struct IGBCore IGBCore; + +enum { PHY_R = BIT(0), + PHY_W = BIT(1), + PHY_RW = PHY_R | PHY_W }; + +typedef struct IGBIntrDelayTimer_st { + QEMUTimer *timer; + bool running; + uint32_t delay_reg; + uint32_t delay_resolution_ns; + IGBCore *core; +} IGBIntrDelayTimer; + +struct IGBCore { + uint32_t mac[E1000E_MAC_SIZE]; + uint16_t phy[MAX_PHY_REG_ADDRESS + 1]; + uint16_t eeprom[IGB_EEPROM_SIZE]; + + uint8_t rx_desc_len; + + QEMUTimer *autoneg_timer; + + struct igb_tx { + uint16_t vlan; /* VLAN Tag */ + uint16_t mss; /* Maximum Segment Size */ + bool tse; /* TCP/UDP Segmentation Enable */ + bool ixsm; /* Insert IP Checksum */ + bool txsm; /* Insert TCP/UDP Checksum */ + + bool first; + bool skip_cp; + + struct NetTxPkt *tx_pkt; + } tx[IGB_NUM_QUEUES]; + + struct NetRxPkt *rx_pkt; + + bool has_vnet; + int max_queue_num; + + IGBIntrDelayTimer eitr[IGB_INTR_NUM]; + + VMChangeStateEntry *vmstate; + + uint32_t eitr_guest_value[IGB_INTR_NUM]; + + uint8_t permanent_mac[ETH_ALEN]; + + NICState *owner_nic; + PCIDevice *owner; + void (*owner_start_recv)(PCIDevice *d); + + int64_t timadj; +}; + +void +igb_core_write(IGBCore *core, hwaddr addr, uint64_t val, unsigned size); + +uint64_t +igb_core_read(IGBCore *core, hwaddr addr, unsigned size); + +void +igb_core_pci_realize(IGBCore *regs, + const uint16_t *eeprom_templ, + uint32_t eeprom_size, + const uint8_t *macaddr); + +void +igb_core_reset(IGBCore *core); + +void +igb_core_pre_save(IGBCore *core); + +int +igb_core_post_load(IGBCore *core); + +void +igb_core_set_link_status(IGBCore *core); + +void +igb_core_pci_uninit(IGBCore *core); + +bool +igb_can_receive(IGBCore *core); + +ssize_t +igb_receive(IGBCore *core, const uint8_t *buf, size_t size); + +ssize_t +igb_receive_iov(IGBCore *core, const struct iovec *iov, int iovcnt); + +void +igb_start_recv(IGBCore *core); + +#endif diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h new file mode 100644 index 0000000000..00934d4f20 --- /dev/null +++ b/hw/net/igb_regs.h @@ -0,0 +1,648 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This is copied + edited from kernel header files in + * drivers/net/ethernet/intel/igb + */ + +#ifndef HW_IGB_REGS_H_ +#define HW_IGB_REGS_H_ + +#include "e1000x_regs.h" + +/* from igb/e1000_hw.h */ + +#define E1000_DEV_ID_82576 0x10C9 +#define E1000_DEV_ID_82576_FIBER 0x10E6 +#define E1000_DEV_ID_82576_SERDES 0x10E7 +#define E1000_DEV_ID_82576_QUAD_COPPER 0x10E8 +#define E1000_DEV_ID_82576_QUAD_COPPER_ET2 0x1526 +#define E1000_DEV_ID_82576_NS 0x150A +#define E1000_DEV_ID_82576_NS_SERDES 0x1518 +#define E1000_DEV_ID_82576_SERDES_QUAD 0x150D + +/* Context Descriptor */ +struct e1000_adv_tx_context_desc { + uint32_t vlan_macip_lens; + uint32_t seqnum_seed; + uint32_t type_tucmd_mlhl; + uint32_t mss_l4len_idx; +}; + +/* Advanced Transmit Descriptor */ +union e1000_adv_tx_desc { + struct { + uint64_t buffer_addr; /* Address of descriptor's data buffer */ + uint32_t cmd_type_len; + uint32_t olinfo_status; + } read; + struct { + uint64_t rsvd; /* Reserved */ + uint32_t nxtseq_seed; + uint32_t status; + } wb; +}; + +#define E1000_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Descriptor */ +#define E1000_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define E1000_ADVTXD_DCMD_DEXT 0x20000000 /* Descriptor Extension (1=Adv) */ +#define E1000_ADVTXD_DCMD_TSE 0x80000000 /* TCP/UDP Segmentation Enable */ + +#define E1000_ADVTXD_POTS_IXSM 0x00000100 /* Insert TCP/UDP Checksum */ +#define E1000_ADVTXD_POTS_TXSM 0x00000200 /* Insert TCP/UDP Checksum */ + +#define E1000_TXD_POPTS_IXSM 0x00000001 /* Insert IP checksum */ +#define E1000_TXD_POPTS_TXSM 0x00000002 /* Insert TCP/UDP checksum */ + +/* Receive Descriptor - Advanced */ +union e1000_adv_rx_desc { + struct { + uint64_t pkt_addr; /* Packet Buffer Address */ + uint64_t hdr_addr; /* Header Buffer Address */ + } read; + struct { + struct { + struct { + uint16_t pkt_info; /* RSS Type, Packet Type */ + uint16_t hdr_info; /* Split Head, Buffer Length */ + } lo_dword; + union { + uint32_t rss; /* RSS Hash */ + struct { + uint16_t ip_id; /* IP Id */ + uint16_t csum; /* Packet Checksum */ + } csum_ip; + } hi_dword; + } lower; + struct { + uint32_t status_error; /* Ext Status/Error */ + uint16_t length; /* Packet Length */ + uint16_t vlan; /* VLAN tag */ + } upper; + } wb; /* writeback */ +}; + +/* from igb/e1000_phy.h */ + +/* IGP01E1000 Specific Registers */ +#define IGP01E1000_PHY_PORT_CONFIG 0x10 /* Port Config */ +#define IGP01E1000_PHY_PORT_STATUS 0x11 /* Status */ +#define IGP01E1000_PHY_PORT_CTRL 0x12 /* Control */ +#define IGP01E1000_PHY_LINK_HEALTH 0x13 /* PHY Link Health */ +#define IGP02E1000_PHY_POWER_MGMT 0x19 /* Power Management */ +#define IGP01E1000_PHY_PAGE_SELECT 0x1F /* Page Select */ +#define IGP01E1000_PHY_PCS_INIT_REG 0x00B4 +#define IGP01E1000_PHY_POLARITY_MASK 0x0078 +#define IGP01E1000_PSCR_AUTO_MDIX 0x1000 +#define IGP01E1000_PSCR_FORCE_MDI_MDIX 0x2000 /* 0=MDI, 1=MDIX */ +#define IGP01E1000_PSCFR_SMART_SPEED 0x0080 + +/* Enable flexible speed on link-up */ +#define IGP02E1000_PM_D0_LPLU 0x0002 /* For D0a states */ +#define IGP02E1000_PM_D3_LPLU 0x0004 /* For all other states */ +#define IGP01E1000_PLHR_SS_DOWNGRADE 0x8000 +#define IGP01E1000_PSSR_POLARITY_REVERSED 0x0002 +#define IGP01E1000_PSSR_MDIX 0x0800 +#define IGP01E1000_PSSR_SPEED_MASK 0xC000 +#define IGP01E1000_PSSR_SPEED_1000MBPS 0xC000 +#define IGP02E1000_PHY_CHANNEL_NUM 4 +#define IGP02E1000_PHY_AGC_A 0x11B1 +#define IGP02E1000_PHY_AGC_B 0x12B1 +#define IGP02E1000_PHY_AGC_C 0x14B1 +#define IGP02E1000_PHY_AGC_D 0x18B1 +#define IGP02E1000_AGC_LENGTH_SHIFT 9 /* Course - 15:13, Fine - 12:9 */ +#define IGP02E1000_AGC_LENGTH_MASK 0x7F +#define IGP02E1000_AGC_RANGE 15 + +/* from igb/igb.h */ + +#define E1000_PCS_CFG_IGN_SD 1 + +/* Interrupt defines */ +#define IGB_START_ITR 648 /* ~6000 ints/sec */ +#define IGB_4K_ITR 980 +#define IGB_20K_ITR 196 +#define IGB_70K_ITR 56 + +/* TX/RX descriptor defines */ +#define IGB_DEFAULT_TXD 256 +#define IGB_DEFAULT_TX_WORK 128 +#define IGB_MIN_TXD 80 +#define IGB_MAX_TXD 4096 + +#define IGB_DEFAULT_RXD 256 +#define IGB_MIN_RXD 80 +#define IGB_MAX_RXD 4096 + +#define IGB_DEFAULT_ITR 3 /* dynamic */ +#define IGB_MAX_ITR_USECS 10000 +#define IGB_MIN_ITR_USECS 10 +#define NON_Q_VECTORS 1 +#define MAX_Q_VECTORS 8 +#define MAX_MSIX_ENTRIES 10 + +/* Transmit and receive queues */ +#define IGB_MAX_RX_QUEUES 8 +#define IGB_MAX_RX_QUEUES_82575 4 +#define IGB_MAX_RX_QUEUES_I211 2 +#define IGB_MAX_TX_QUEUES 8 +#define IGB_MAX_VF_MC_ENTRIES 30 +#define IGB_MAX_VF_FUNCTIONS 8 +#define IGB_MAX_VFTA_ENTRIES 128 +#define IGB_82576_VF_DEV_ID 0x10CA +#define IGB_I350_VF_DEV_ID 0x1520 + +/* from igb/e1000_82575.h */ + +#define E1000_MRQC_ENABLE_RSS_MQ 0x00000002 +#define E1000_MRQC_ENABLE_VMDQ 0x00000003 +#define E1000_MRQC_RSS_FIELD_IPV4_UDP 0x00400000 +#define E1000_MRQC_ENABLE_VMDQ_RSS_MQ 0x00000005 +#define E1000_MRQC_RSS_FIELD_IPV6_UDP 0x00800000 +#define E1000_MRQC_RSS_FIELD_IPV6_UDP_EX 0x01000000 + +/* Additional Receive Descriptor Control definitions */ +#define E1000_RXDCTL_QUEUE_ENABLE 0x02000000 /* Enable specific Rx Queue */ + +/* Direct Cache Access (DCA) definitions */ +#define E1000_DCA_CTRL_DCA_MODE_DISABLE 0x01 /* DCA Disable */ +#define E1000_DCA_CTRL_DCA_MODE_CB2 0x02 /* DCA Mode CB2 */ + +#define E1000_DCA_RXCTRL_CPUID_MASK 0x0000001F /* Rx CPUID Mask */ +#define E1000_DCA_RXCTRL_DESC_DCA_EN BIT(5) /* DCA Rx Desc enable */ +#define E1000_DCA_RXCTRL_HEAD_DCA_EN BIT(6) /* DCA Rx Desc header enable */ +#define E1000_DCA_RXCTRL_DATA_DCA_EN BIT(7) /* DCA Rx Desc payload enable */ +#define E1000_DCA_RXCTRL_DESC_RRO_EN BIT(9) /* DCA Rx rd Desc Relax Order */ + +#define E1000_DCA_TXCTRL_CPUID_MASK 0x0000001F /* Tx CPUID Mask */ +#define E1000_DCA_TXCTRL_DESC_DCA_EN BIT(5) /* DCA Tx Desc enable */ +#define E1000_DCA_TXCTRL_DESC_RRO_EN BIT(9) /* Tx rd Desc Relax Order */ +#define E1000_DCA_TXCTRL_TX_WB_RO_EN BIT(11) /* Tx Desc writeback RO bit */ +#define E1000_DCA_TXCTRL_DATA_RRO_EN BIT(13) /* Tx rd data Relax Order */ + +/* Additional DCA related definitions, note change in position of CPUID */ +#define E1000_DCA_TXCTRL_CPUID_MASK_82576 0xFF000000 /* Tx CPUID Mask */ +#define E1000_DCA_RXCTRL_CPUID_MASK_82576 0xFF000000 /* Rx CPUID Mask */ +#define E1000_DCA_TXCTRL_CPUID_SHIFT 24 /* Tx CPUID now in the last byte */ +#define E1000_DCA_RXCTRL_CPUID_SHIFT 24 /* Rx CPUID now in the last byte */ + +#define E1000_DTXSWC_MAC_SPOOF_MASK 0x000000FF /* Per VF MAC spoof control */ +#define E1000_DTXSWC_VLAN_SPOOF_MASK 0x0000FF00 /* Per VF VLAN spoof control */ +#define E1000_DTXSWC_LLE_MASK 0x00FF0000 /* Per VF Local LB enables */ +#define E1000_DTXSWC_VLAN_SPOOF_SHIFT 8 +#define E1000_DTXSWC_VMDQ_LOOPBACK_EN BIT(31) /* global VF LB enable */ + +/* Easy defines for setting default pool, would normally be left a zero */ +#define E1000_VT_CTL_DEFAULT_POOL_SHIFT 7 +#define E1000_VT_CTL_DEFAULT_POOL_MASK (0x7 << E1000_VT_CTL_DEFAULT_POOL_SHIFT) + +/* Other useful VMD_CTL register defines */ +#define E1000_VT_CTL_IGNORE_MAC BIT(28) +#define E1000_VT_CTL_DISABLE_DEF_POOL BIT(29) +#define E1000_VT_CTL_VM_REPL_EN BIT(30) + +/* Per VM Offload register setup */ +#define E1000_VMOLR_RLPML_MASK 0x00003FFF /* Long Packet Maximum Length mask */ +#define E1000_VMOLR_LPE 0x00010000 /* Accept Long packet */ +#define E1000_VMOLR_RSSE 0x00020000 /* Enable RSS */ +#define E1000_VMOLR_AUPE 0x01000000 /* Accept untagged packets */ +#define E1000_VMOLR_ROMPE 0x02000000 /* Accept overflow multicast */ +#define E1000_VMOLR_ROPE 0x04000000 /* Accept overflow unicast */ +#define E1000_VMOLR_BAM 0x08000000 /* Accept Broadcast packets */ +#define E1000_VMOLR_MPME 0x10000000 /* Multicast promiscuous mode */ +#define E1000_VMOLR_STRVLAN 0x40000000 /* Vlan stripping enable */ +#define E1000_VMOLR_STRCRC 0x80000000 /* CRC stripping enable */ + +#define E1000_DVMOLR_HIDEVLAN 0x20000000 /* Hide vlan enable */ +#define E1000_DVMOLR_STRVLAN 0x40000000 /* Vlan stripping enable */ +#define E1000_DVMOLR_STRCRC 0x80000000 /* CRC stripping enable */ + +#define E1000_VLVF_ARRAY_SIZE 32 +#define E1000_VLVF_VLANID_MASK 0x00000FFF +#define E1000_VLVF_POOLSEL_SHIFT 12 +#define E1000_VLVF_POOLSEL_MASK (0xFF << E1000_VLVF_POOLSEL_SHIFT) +#define E1000_VLVF_LVLAN 0x00100000 +#define E1000_VLVF_VLANID_ENABLE 0x80000000 + +#define E1000_VMVIR_VLANA_DEFAULT 0x40000000 /* Always use default VLAN */ +#define E1000_VMVIR_VLANA_NEVER 0x80000000 /* Never insert VLAN tag */ + +#define E1000_IOVCTL 0x05BBC +#define E1000_IOVCTL_REUSE_VFQ 0x00000001 + +#define E1000_RPLOLR_STRVLAN 0x40000000 +#define E1000_RPLOLR_STRCRC 0x80000000 + +#define E1000_DTXCTL_8023LL 0x0004 +#define E1000_DTXCTL_VLAN_ADDED 0x0008 +#define E1000_DTXCTL_OOS_ENABLE 0x0010 +#define E1000_DTXCTL_MDP_EN 0x0020 +#define E1000_DTXCTL_SPOOF_INT 0x0040 + +/* from igb/e1000_defines.h */ + +#define E1000_IVAR_VALID 0x80 +#define E1000_GPIE_NSICR 0x00000001 +#define E1000_GPIE_MSIX_MODE 0x00000010 +#define E1000_GPIE_EIAME 0x40000000 +#define E1000_GPIE_PBA 0x80000000 + +/* Transmit Control */ +#define E1000_TCTL_EN 0x00000002 /* enable tx */ +#define E1000_TCTL_PSP 0x00000008 /* pad short packets */ +#define E1000_TCTL_CT 0x00000ff0 /* collision threshold */ +#define E1000_TCTL_COLD 0x003ff000 /* collision distance */ +#define E1000_TCTL_RTLC 0x01000000 /* Re-transmit on late collision */ + +/* Collision related configuration parameters */ +#define E1000_COLLISION_THRESHOLD 15 +#define E1000_CT_SHIFT 4 +#define E1000_COLLISION_DISTANCE 63 +#define E1000_COLD_SHIFT 12 + +#define E1000_RAH_POOL_MASK 0x03FC0000 +#define E1000_RAH_POOL_1 0x00040000 + +#define E1000_ICR_VMMB 0x00000100 /* VM MB event */ +#define E1000_ICR_TS 0x00080000 /* Time Sync Interrupt */ +#define E1000_ICR_DRSTA 0x40000000 /* Device Reset Asserted */ +/* If this bit asserted, the driver should claim the interrupt */ +#define E1000_ICR_INT_ASSERTED 0x80000000 +/* LAN connected device generates an interrupt */ +#define E1000_ICR_DOUTSYNC 0x10000000 /* NIC DMA out of sync */ + +/* Extended Interrupt Cause Read */ +#define E1000_EICR_RX_QUEUE0 0x00000001 /* Rx Queue 0 Interrupt */ +#define E1000_EICR_RX_QUEUE1 0x00000002 /* Rx Queue 1 Interrupt */ +#define E1000_EICR_RX_QUEUE2 0x00000004 /* Rx Queue 2 Interrupt */ +#define E1000_EICR_RX_QUEUE3 0x00000008 /* Rx Queue 3 Interrupt */ +#define E1000_EICR_TX_QUEUE0 0x00000100 /* Tx Queue 0 Interrupt */ +#define E1000_EICR_TX_QUEUE1 0x00000200 /* Tx Queue 1 Interrupt */ +#define E1000_EICR_TX_QUEUE2 0x00000400 /* Tx Queue 2 Interrupt */ +#define E1000_EICR_TX_QUEUE3 0x00000800 /* Tx Queue 3 Interrupt */ +#define E1000_EICR_OTHER 0x80000000 /* Interrupt Cause Active */ + +/* Extended Interrupt Cause Set */ +/* E1000_EITR_CNT_IGNR is only for 82576 and newer */ +#define E1000_EITR_CNT_IGNR 0x80000000 /* Don't reset counters on write */ + +/* PCI Express Control */ +#define E1000_GCR_CMPL_TMOUT_MASK 0x0000F000 +#define E1000_GCR_CMPL_TMOUT_10ms 0x00001000 +#define E1000_GCR_CMPL_TMOUT_RESEND 0x00010000 +#define E1000_GCR_CAP_VER2 0x00040000 + +#define PHY_REVISION_MASK 0xFFFFFFF0 +#define MAX_PHY_REG_ADDRESS 0x1F /* 5 bit address bus (0-0x1F) */ +#define MAX_PHY_MULTI_PAGE_REG 0xF + +#define IGP03E1000_E_PHY_ID 0x02A80390 + +/* from igb/e1000_mbox.h */ + +#define E1000_P2VMAILBOX_STS 0x00000001 /* Initiate message send to VF */ +#define E1000_P2VMAILBOX_ACK 0x00000002 /* Ack message recv'd from VF */ +#define E1000_P2VMAILBOX_VFU 0x00000004 /* VF owns the mailbox buffer */ +#define E1000_P2VMAILBOX_PFU 0x00000008 /* PF owns the mailbox buffer */ +#define E1000_P2VMAILBOX_RVFU 0x00000010 /* Reset VFU - used when VF stuck */ + +#define E1000_MBVFICR_VFREQ_MASK 0x000000FF /* bits for VF messages */ +#define E1000_MBVFICR_VFREQ_VF1 0x00000001 /* bit for VF 1 message */ +#define E1000_MBVFICR_VFACK_MASK 0x00FF0000 /* bits for VF acks */ +#define E1000_MBVFICR_VFACK_VF1 0x00010000 /* bit for VF 1 ack */ + +#define E1000_V2PMAILBOX_SIZE 16 /* 16 32 bit words - 64 bytes */ + +/* + * If it's a E1000_VF_* msg then it originates in the VF and is sent to the + * PF. The reverse is true if it is E1000_PF_*. + * Message ACK's are the value or'd with 0xF0000000 + */ +/* Messages below or'd with this are the ACK */ +#define E1000_VT_MSGTYPE_ACK 0x80000000 +/* Messages below or'd with this are the NACK */ +#define E1000_VT_MSGTYPE_NACK 0x40000000 +/* Indicates that VF is still clear to send requests */ +#define E1000_VT_MSGTYPE_CTS 0x20000000 +#define E1000_VT_MSGINFO_SHIFT 16 +/* bits 23:16 are used for exra info for certain messages */ +#define E1000_VT_MSGINFO_MASK (0xFF << E1000_VT_MSGINFO_SHIFT) + +#define E1000_VF_RESET 0x01 /* VF requests reset */ +#define E1000_VF_SET_MAC_ADDR 0x02 /* VF requests to set MAC addr */ +/* VF requests to clear all unicast MAC filters */ +#define E1000_VF_MAC_FILTER_CLR (0x01 << E1000_VT_MSGINFO_SHIFT) +/* VF requests to add unicast MAC filter */ +#define E1000_VF_MAC_FILTER_ADD (0x02 << E1000_VT_MSGINFO_SHIFT) +#define E1000_VF_SET_MULTICAST 0x03 /* VF requests to set MC addr */ +#define E1000_VF_SET_VLAN 0x04 /* VF requests to set VLAN */ +#define E1000_VF_SET_LPE 0x05 /* VF requests to set VMOLR.LPE */ +#define E1000_VF_SET_PROMISC 0x06 /*VF requests to clear VMOLR.ROPE/MPME*/ +#define E1000_VF_SET_PROMISC_MULTICAST (0x02 << E1000_VT_MSGINFO_SHIFT) + +#define E1000_PF_CONTROL_MSG 0x0100 /* PF control message */ + +/* from igb/e1000_regs.h */ + +#define E1000_EICR 0x01580 /* Ext. Interrupt Cause Read - R/clr */ +#define E1000_EITR(_n) (0x01680 + (0x4 * (_n))) +#define E1000_EICS 0x01520 /* Ext. Interrupt Cause Set - W0 */ +#define E1000_EIMS 0x01524 /* Ext. Interrupt Mask Set/Read - RW */ +#define E1000_EIMC 0x01528 /* Ext. Interrupt Mask Clear - WO */ +#define E1000_EIAC 0x0152C /* Ext. Interrupt Auto Clear - RW */ +#define E1000_EIAM 0x01530 /* Ext. Interrupt Ack Auto Clear Mask - RW */ +#define E1000_GPIE 0x01514 /* General Purpose Interrupt Enable; RW */ +#define E1000_IVAR0 0x01700 /* Interrupt Vector Allocation Register - RW */ +#define E1000_IVAR_MISC 0x01740 /* Interrupt Vector Allocation Register (last) - RW */ +#define E1000_FRTIMER 0x01048 /* Free Running Timer - RW */ +#define E1000_FCRTV 0x02460 /* Flow Control Refresh Timer Value - RW */ + +#define E1000_RQDPC(_n) (0x0C030 + ((_n) * 0x40)) + +#define E1000_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */ +#define E1000_TXPBS 0x03404 /* Tx Packet Buffer Size - RW */ + +#define E1000_DTXCTL 0x03590 /* DMA TX Control - RW */ + +#define E1000_HTCBDPC 0x04124 /* Host TX Circuit Breaker Dropped Count */ +#define E1000_RLPML 0x05004 /* RX Long Packet Max Length */ +#define E1000_RA2 0x054E0 /* 2nd half of Rx address array - RW Array */ +#define E1000_PSRTYPE(_i) (0x05480 + ((_i) * 4)) +#define E1000_VT_CTL 0x0581C /* VMDq Control - RW */ + +/* VT Registers */ +#define E1000_MBVFICR 0x00C80 /* Mailbox VF Cause - RWC */ +#define E1000_MBVFIMR 0x00C84 /* Mailbox VF int Mask - RW */ +#define E1000_VFLRE 0x00C88 /* VF Register Events - RWC */ +#define E1000_VFRE 0x00C8C /* VF Receive Enables */ +#define E1000_VFTE 0x00C90 /* VF Transmit Enables */ +#define E1000_QDE 0x02408 /* Queue Drop Enable - RW */ +#define E1000_DTXSWC 0x03500 /* DMA Tx Switch Control - RW */ +#define E1000_WVBR 0x03554 /* VM Wrong Behavior - RWS */ +#define E1000_RPLOLR 0x05AF0 /* Replication Offload - RW */ +#define E1000_UTA 0x0A000 /* Unicast Table Array - RW */ +#define E1000_IOVTCL 0x05BBC /* IOV Control Register */ +#define E1000_TXSWC 0x05ACC /* Tx Switch Control */ +#define E1000_LVMMC 0x03548 /* Last VM Misbehavior cause */ +/* These act per VF so an array friendly macro is used */ +#define E1000_P2VMAILBOX(_n) (0x00C00 + (4 * (_n))) +#define E1000_VMBMEM(_n) (0x00800 + (64 * (_n))) +#define E1000_VMOLR(_n) (0x05AD0 + (4 * (_n))) +#define E1000_DVMOLR(_n) (0x0C038 + (64 * (_n))) +#define E1000_VLVF(_n) (0x05D00 + (4 * (_n))) /* VLAN VM Filter */ +#define E1000_VMVIR(_n) (0x03700 + (4 * (_n))) + +/* from igbvf/defines.h */ + +/* SRRCTL bit definitions */ +#define E1000_SRRCTL_BSIZEPKT_SHIFT 10 /* Shift _right_ */ +#define E1000_SRRCTL_BSIZEHDRSIZE_MASK 0x00000F00 +#define E1000_SRRCTL_BSIZEHDRSIZE_SHIFT 2 /* Shift _left_ */ +#define E1000_SRRCTL_DESCTYPE_ADV_ONEBUF 0x02000000 +#define E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000 +#define E1000_SRRCTL_DESCTYPE_MASK 0x0E000000 +#define E1000_SRRCTL_DROP_EN 0x80000000 + +#define E1000_SRRCTL_BSIZEPKT_MASK 0x0000007F +#define E1000_SRRCTL_BSIZEHDR_MASK 0x00003F00 + +/* from igbvf/mbox.h */ + +#define E1000_V2PMAILBOX_REQ 0x00000001 /* Request for PF Ready bit */ +#define E1000_V2PMAILBOX_ACK 0x00000002 /* Ack PF message received */ +#define E1000_V2PMAILBOX_VFU 0x00000004 /* VF owns the mailbox buffer */ +#define E1000_V2PMAILBOX_PFU 0x00000008 /* PF owns the mailbox buffer */ +#define E1000_V2PMAILBOX_PFSTS 0x00000010 /* PF wrote a message in the MB */ +#define E1000_V2PMAILBOX_PFACK 0x00000020 /* PF ack the previous VF msg */ +#define E1000_V2PMAILBOX_RSTI 0x00000040 /* PF has reset indication */ +#define E1000_V2PMAILBOX_RSTD 0x00000080 /* PF has indicated reset done */ +#define E1000_V2PMAILBOX_R2C_BITS 0x000000B0 /* All read to clear bits */ + +#define E1000_VFMAILBOX_SIZE 16 /* 16 32 bit words - 64 bytes */ + +/* + * If it's a E1000_VF_* msg then it originates in the VF and is sent to the + * PF. The reverse is true if it is E1000_PF_*. + * Message ACK's are the value or'd with 0xF0000000 + */ +/* Messages below or'd with this are the ACK */ +#define E1000_VT_MSGTYPE_ACK 0x80000000 +/* Messages below or'd with this are the NACK */ +#define E1000_VT_MSGTYPE_NACK 0x40000000 +/* Indicates that VF is still clear to send requests */ +#define E1000_VT_MSGTYPE_CTS 0x20000000 + +/* We have a total wait time of 1s for vf mailbox posted messages */ +#define E1000_VF_MBX_INIT_TIMEOUT 2000 /* retry count for mbx timeout */ +#define E1000_VF_MBX_INIT_DELAY 500 /* usec delay between retries */ + +#define E1000_VT_MSGINFO_SHIFT 16 +/* bits 23:16 are used for exra info for certain messages */ +#define E1000_VT_MSGINFO_MASK (0xFF << E1000_VT_MSGINFO_SHIFT) + +#define E1000_VF_RESET 0x01 /* VF requests reset */ +#define E1000_VF_SET_MAC_ADDR 0x02 /* VF requests PF to set MAC addr */ +/* VF requests PF to clear all unicast MAC filters */ +#define E1000_VF_MAC_FILTER_CLR (0x01 << E1000_VT_MSGINFO_SHIFT) +/* VF requests PF to add unicast MAC filter */ +#define E1000_VF_MAC_FILTER_ADD (0x02 << E1000_VT_MSGINFO_SHIFT) +#define E1000_VF_SET_MULTICAST 0x03 /* VF requests PF to set MC addr */ +#define E1000_VF_SET_VLAN 0x04 /* VF requests PF to set VLAN */ +#define E1000_VF_SET_LPE 0x05 /* VF requests PF to set VMOLR.LPE */ + +#define E1000_PF_CONTROL_MSG 0x0100 /* PF control message */ + +/* from igbvf/regs.h */ + +/* Statistics registers */ +#define E1000_VFGPRC 0x00F10 +#define E1000_VFGORC 0x00F18 +#define E1000_VFMPRC 0x00F3C +#define E1000_VFGPTC 0x00F14 +#define E1000_VFGOTC 0x00F34 +#define E1000_VFGOTLBC 0x00F50 +#define E1000_VFGPTLBC 0x00F44 +#define E1000_VFGORLBC 0x00F48 +#define E1000_VFGPRLBC 0x00F40 + +/* These act per VF so an array friendly macro is used */ +#define E1000_V2PMAILBOX(_n) (0x00C40 + (4 * (_n))) +#define E1000_VMBMEM(_n) (0x00800 + (64 * (_n))) + +/* from igbvf/vf.h */ + +#define E1000_DEV_ID_82576_VF 0x10CA + +/* new */ + +/* Receive Registers */ + +/* RX Descriptor Base Low; RW */ +#define E1000_RDBAL(_n) (0x0C000 + (0x40 * (_n))) +#define E1000_RDBAL_A(_n) (0x02800 + (0x100 * (_n))) + +/* RX Descriptor Base High; RW */ +#define E1000_RDBAH(_n) (0x0C004 + (0x40 * (_n))) +#define E1000_RDBAH_A(_n) (0x02804 + (0x100 * (_n))) + +/* RX Descriptor Ring Length; RW */ +#define E1000_RDLEN(_n) (0x0C008 + (0x40 * (_n))) +#define E1000_RDLEN_A(_n) (0x02808 + (0x100 * (_n))) + +/* Split and Replication Receive Control; RW */ +#define E1000_SRRCTL(_n) (0x0C00C + (0x40 * (_n))) +#define E1000_SRRCTL_A(_n) (0x0280C + (0x100 * (_n))) + +/* RX Descriptor Head; RW */ +#define E1000_RDH(_n) (0x0C010 + (0x40 * (_n))) +#define E1000_RDH_A(_n) (0x02810 + (0x100 * (_n))) + +/* RX DCA Control; RW */ +#define E1000_RXCTL(_n) (0x0C014 + (0x40 * (_n))) +#define E1000_RXCTL_A(_n) (0x02814 + (0x100 * (_n))) + +/* RX Descriptor Tail; RW */ +#define E1000_RDT(_n) (0x0C018 + (0x40 * (_n))) +#define E1000_RDT_A(_n) (0x02818 + (0x100 * (_n))) + +/* RX Descriptor Control; RW */ +#define E1000_RXDCTL(_n) (0x0C028 + (0x40 * (_n))) +#define E1000_RXDCTL_A(_n) (0x02828 + (0x100 * (_n))) + +/* RX Queue Drop Packet Count; RC */ +#define E1000_RQDPC_A(_n) (0x02830 + (0x100 * (_n))) + +/* Transmit Registers */ + +/* TX Descriptor Base Low; RW */ +#define E1000_TDBAL(_n) (0x0E000 + (0x40 * (_n))) +#define E1000_TDBAL_A(_n) (0x03800 + (0x100 * (_n))) + +/* TX Descriptor Base High; RW */ +#define E1000_TDBAH(_n) (0x0E004 + (0x40 * (_n))) +#define E1000_TDBAH_A(_n) (0x03804 + (0x100 * (_n))) + +/* TX Descriptor Ring Length; RW */ +#define E1000_TDLEN(_n) (0x0E008 + (0x40 * (_n))) +#define E1000_TDLEN_A(_n) (0x03808 + (0x100 * (_n))) + +/* TX Descriptor Head; RW */ +#define E1000_TDH(_n) (0x0E010 + (0x40 * (_n))) +#define E1000_TDH_A(_n) (0x03810 + (0x100 * (_n))) + +/* TX DCA Control; RW */ +#define E1000_TXCTL(_n) (0x0E014 + (0x40 * (_n))) +#define E1000_TXCTL_A(_n) (0x03814 + (0x100 * (_n))) + +/* TX Descriptor Tail; RW */ +#define E1000_TDT(_n) (0x0E018 + (0x40 * (_n))) +#define E1000_TDT_A(_n) (0x03818 + (0x100 * (_n))) + +/* TX Descriptor Control; RW */ +#define E1000_TXDCTL(_n) (0x0E028 + (0x40 * (_n))) +#define E1000_TXDCTL_A(_n) (0x03828 + (0x100 * (_n))) + +/* TX Descriptor Completion Write–Back Address Low; RW */ +#define E1000_TDWBAL(_n) (0x0E038 + (0x40 * (_n))) +#define E1000_TDWBAL_A(_n) (0x03838 + (0x100 * (_n))) + +/* TX Descriptor Completion Write–Back Address High; RW */ +#define E1000_TDWBAH(_n) (0x0E03C + (0x40 * (_n))) +#define E1000_TDWBAH_A(_n) (0x0383C + (0x100 * (_n))) + +#define E1000_MTA_A 0x0200 + +#define E1000_XDBAL_MASK (~(BIT(5) - 1)) /* TDBAL and RDBAL Registers Mask */ + +#define E1000_ICR_MACSEC 0x00000020 /* MACSec */ +#define E1000_ICR_RX0 0x00000040 /* Receiver Overrun */ +#define E1000_ICR_GPI_SDP0 0x00000800 /* General Purpose, SDP0 pin */ +#define E1000_ICR_GPI_SDP1 0x00001000 /* General Purpose, SDP1 pin */ +#define E1000_ICR_GPI_SDP2 0x00002000 /* General Purpose, SDP2 pin */ +#define E1000_ICR_GPI_SDP3 0x00004000 /* General Purpose, SDP3 pin */ +#define E1000_ICR_PTRAP 0x00008000 /* Probe Trap */ +#define E1000_ICR_MNG 0x00040000 /* Management Event */ +#define E1000_ICR_OMED 0x00100000 /* Other Media Energy Detected */ +#define E1000_ICR_FER 0x00400000 /* Fatal Error */ +#define E1000_ICR_NFER 0x00800000 /* Non Fatal Error */ +#define E1000_ICR_CSRTO 0x01000000 /* CSR access Time Out Indication */ +#define E1000_ICR_SCE 0x02000000 /* Storm Control Event */ +#define E1000_ICR_SW_WD 0x04000000 /* Software Watchdog */ + +/* Extended Interrupts */ + +#define E1000_EICR_MSIX_MASK 0x01FFFFFF /* Bits used in MSI-X mode */ +#define E1000_EICR_LEGACY_MASK 0x4000FFFF /* Bits used in non MSI-X mode */ + +/* Mirror VF Control (only RST bit); RW */ +#define E1000_PVTCTRL(_n) (0x10000 + (_n) * 0x100) + +/* Mirror Good Packets Received Count; RO */ +#define E1000_PVFGPRC(_n) (0x10010 + (_n) * 0x100) + +/* Mirror Good Packets Transmitted Count; RO */ +#define E1000_PVFGPTC(_n) (0x10014 + (_n) * 0x100) + +/* Mirror Good Octets Received Count; RO */ +#define E1000_PVFGORC(_n) (0x10018 + (_n) * 0x100) + +/* Mirror Extended Interrupt Cause Set; WO */ +#define E1000_PVTEICS(_n) (0x10020 + (_n) * 0x100) + +/* Mirror Extended Interrupt Mask Set/Read; RW */ +#define E1000_PVTEIMS(_n) (0x10024 + (_n) * 0x100) + +/* Mirror Extended Interrupt Mask Clear; WO */ +#define E1000_PVTEIMC(_n) (0x10028 + (_n) * 0x100) + +/* Mirror Extended Interrupt Auto Clear; RW */ +#define E1000_PVTEIAC(_n) (0x1002C + (_n) * 0x100) + +/* Mirror Extended Interrupt Auto Mask Enable; RW */ +#define E1000_PVTEIAM(_n) (0x10030 + (_n) * 0x100) + +/* Mirror Good Octets Transmitted Count; RO */ +#define E1000_PVFGOTC(_n) (0x10034 + (_n) * 0x100) + +/* Mirror Multicast Packets Received Count; RO */ +#define E1000_PVFMPRC(_n) (0x1003C + (_n) * 0x100) + +/* Mirror Good RX Packets loopback Count; RO */ +#define E1000_PVFGPRLBC(_n) (0x10040 + (_n) * 0x100) + +/* Mirror Good TX packets loopback Count; RO */ +#define E1000_PVFGPTLBC(_n) (0x10044 + (_n) * 0x100) + +/* Mirror Good RX Octets loopback Count; RO */ +#define E1000_PVFGORLBC(_n) (0x10048 + (_n) * 0x100) + +/* Mirror Good TX Octets loopback Count; RO */ +#define E1000_PVFGOTLBC(_n) (0x10050 + (_n) * 0x100) + +/* Mirror Extended Interrupt Cause Set; RC/W1C */ +#define E1000_PVTEICR(_n) (0x10080 + (_n) * 0x100) + +/* + * These are fake addresses that, according to the specification, the device + * is not using. They are used to distinguish between the PF and the VFs + * accessing their VTIVAR register (which is the same address, 0x1700) + */ +#define E1000_VTIVAR 0x11700 +#define E1000_VTIVAR_MISC 0x11720 + +#define E1000_RSS_QUEUE(reta, hash) (E1000_RETA_VAL(reta, hash) & 0x0F) + +#define E1000_STATUS_IOV_MODE 0x00040000 + +#define E1000_STATUS_NUM_VFS_SHIFT 14 + +static inline uint8_t igb_ivar_entry_rx(uint8_t i) +{ + return i < 8 ? i * 4 : (i - 8) * 4 + 2; +} + +static inline uint8_t igb_ivar_entry_tx(uint8_t i) +{ + return i < 8 ? i * 4 + 1 : (i - 8) * 4 + 3; +} + +#endif diff --git a/hw/net/igbvf.c b/hw/net/igbvf.c new file mode 100644 index 0000000000..70beb7af50 --- /dev/null +++ b/hw/net/igbvf.c @@ -0,0 +1,327 @@ +/* + * QEMU Intel 82576 SR/IOV Ethernet Controller Emulation + * + * Datasheet: + * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eg-gbe-datasheet.pdf + * + * Copyright (c) 2020-2023 Red Hat, Inc. + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Akihiko Odaki <akihiko.odaki@daynix.com> + * Gal Hammmer <gal.hammer@sap.com> + * Marcel Apfelbaum <marcel.apfelbaum@gmail.com> + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * Based on work done by: + * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc. + * Copyright (c) 2008 Qumranet + * Based on work done by: + * Copyright (c) 2007 Dan Aloni + * Copyright (c) 2004 Antony T Curtis + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "hw/hw.h" +#include "hw/net/mii.h" +#include "hw/pci/pci_device.h" +#include "hw/pci/pcie.h" +#include "hw/pci/msix.h" +#include "net/eth.h" +#include "net/net.h" +#include "igb_common.h" +#include "igb_core.h" +#include "trace.h" +#include "qapi/error.h" + +#define TYPE_IGBVF "igbvf" +OBJECT_DECLARE_SIMPLE_TYPE(IgbVfState, IGBVF) + +#define IGBVF_MMIO_BAR_IDX (0) +#define IGBVF_MSIX_BAR_IDX (3) + +#define IGBVF_MMIO_SIZE (16 * 1024) +#define IGBVF_MSIX_SIZE (16 * 1024) + +struct IgbVfState { + PCIDevice parent_obj; + + MemoryRegion mmio; + MemoryRegion msix; +}; + +static hwaddr vf_to_pf_addr(hwaddr addr, uint16_t vfn, bool write) +{ + switch (addr) { + case E1000_CTRL: + case E1000_CTRL_DUP: + return E1000_PVTCTRL(vfn); + case E1000_EICS: + return E1000_PVTEICS(vfn); + case E1000_EIMS: + return E1000_PVTEIMS(vfn); + case E1000_EIMC: + return E1000_PVTEIMC(vfn); + case E1000_EIAC: + return E1000_PVTEIAC(vfn); + case E1000_EIAM: + return E1000_PVTEIAM(vfn); + case E1000_EICR: + return E1000_PVTEICR(vfn); + case E1000_EITR(0): + case E1000_EITR(1): + case E1000_EITR(2): + return E1000_EITR(22) + (addr - E1000_EITR(0)) - vfn * 0xC; + case E1000_IVAR0: + return E1000_VTIVAR + vfn * 4; + case E1000_IVAR_MISC: + return E1000_VTIVAR_MISC + vfn * 4; + case 0x0F04: /* PBACL */ + return E1000_PBACLR; + case 0x0F0C: /* PSRTYPE */ + return E1000_PSRTYPE(vfn); + case E1000_V2PMAILBOX(0): + return E1000_V2PMAILBOX(vfn); + case E1000_VMBMEM(0) ... E1000_VMBMEM(0) + 0x3F: + return addr + vfn * 0x40; + case E1000_RDBAL_A(0): + return E1000_RDBAL(vfn); + case E1000_RDBAL_A(1): + return E1000_RDBAL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_RDBAH_A(0): + return E1000_RDBAH(vfn); + case E1000_RDBAH_A(1): + return E1000_RDBAH(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_RDLEN_A(0): + return E1000_RDLEN(vfn); + case E1000_RDLEN_A(1): + return E1000_RDLEN(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_SRRCTL_A(0): + return E1000_SRRCTL(vfn); + case E1000_SRRCTL_A(1): + return E1000_SRRCTL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_RDH_A(0): + return E1000_RDH(vfn); + case E1000_RDH_A(1): + return E1000_RDH(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_RXCTL_A(0): + return E1000_RXCTL(vfn); + case E1000_RXCTL_A(1): + return E1000_RXCTL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_RDT_A(0): + return E1000_RDT(vfn); + case E1000_RDT_A(1): + return E1000_RDT(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_RXDCTL_A(0): + return E1000_RXDCTL(vfn); + case E1000_RXDCTL_A(1): + return E1000_RXDCTL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_RQDPC_A(0): + return E1000_RQDPC(vfn); + case E1000_RQDPC_A(1): + return E1000_RQDPC(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TDBAL_A(0): + return E1000_TDBAL(vfn); + case E1000_TDBAL_A(1): + return E1000_TDBAL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TDBAH_A(0): + return E1000_TDBAH(vfn); + case E1000_TDBAH_A(1): + return E1000_TDBAH(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TDLEN_A(0): + return E1000_TDLEN(vfn); + case E1000_TDLEN_A(1): + return E1000_TDLEN(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TDH_A(0): + return E1000_TDH(vfn); + case E1000_TDH_A(1): + return E1000_TDH(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TXCTL_A(0): + return E1000_TXCTL(vfn); + case E1000_TXCTL_A(1): + return E1000_TXCTL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TDT_A(0): + return E1000_TDT(vfn); + case E1000_TDT_A(1): + return E1000_TDT(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TXDCTL_A(0): + return E1000_TXDCTL(vfn); + case E1000_TXDCTL_A(1): + return E1000_TXDCTL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TDWBAL_A(0): + return E1000_TDWBAL(vfn); + case E1000_TDWBAL_A(1): + return E1000_TDWBAL(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_TDWBAH_A(0): + return E1000_TDWBAH(vfn); + case E1000_TDWBAH_A(1): + return E1000_TDWBAH(vfn + IGB_MAX_VF_FUNCTIONS); + case E1000_VFGPRC: + return E1000_PVFGPRC(vfn); + case E1000_VFGPTC: + return E1000_PVFGPTC(vfn); + case E1000_VFGORC: + return E1000_PVFGORC(vfn); + case E1000_VFGOTC: + return E1000_PVFGOTC(vfn); + case E1000_VFMPRC: + return E1000_PVFMPRC(vfn); + case E1000_VFGPRLBC: + return E1000_PVFGPRLBC(vfn); + case E1000_VFGPTLBC: + return E1000_PVFGPTLBC(vfn); + case E1000_VFGORLBC: + return E1000_PVFGORLBC(vfn); + case E1000_VFGOTLBC: + return E1000_PVFGOTLBC(vfn); + case E1000_STATUS: + case E1000_FRTIMER: + if (write) { + return HWADDR_MAX; + } + /* fallthrough */ + case 0x34E8: /* PBTWAC */ + case 0x24E8: /* PBRWAC */ + return addr; + } + + trace_igbvf_wrn_io_addr_unknown(addr); + + return HWADDR_MAX; +} + +static void igbvf_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, + int len) +{ + trace_igbvf_write_config(addr, val, len); + pci_default_write_config(dev, addr, val, len); +} + +static uint64_t igbvf_mmio_read(void *opaque, hwaddr addr, unsigned size) +{ + PCIDevice *vf = PCI_DEVICE(opaque); + PCIDevice *pf = pcie_sriov_get_pf(vf); + + addr = vf_to_pf_addr(addr, pcie_sriov_vf_number(vf), false); + return addr == HWADDR_MAX ? 0 : igb_mmio_read(pf, addr, size); +} + +static void igbvf_mmio_write(void *opaque, hwaddr addr, uint64_t val, + unsigned size) +{ + PCIDevice *vf = PCI_DEVICE(opaque); + PCIDevice *pf = pcie_sriov_get_pf(vf); + + addr = vf_to_pf_addr(addr, pcie_sriov_vf_number(vf), true); + if (addr != HWADDR_MAX) { + igb_mmio_write(pf, addr, val, size); + } +} + +static const MemoryRegionOps mmio_ops = { + .read = igbvf_mmio_read, + .write = igbvf_mmio_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .impl = { + .min_access_size = 4, + .max_access_size = 4, + }, +}; + +static void igbvf_pci_realize(PCIDevice *dev, Error **errp) +{ + IgbVfState *s = IGBVF(dev); + int ret; + int i; + + dev->config_write = igbvf_write_config; + + memory_region_init_io(&s->mmio, OBJECT(dev), &mmio_ops, s, "igbvf-mmio", + IGBVF_MMIO_SIZE); + pcie_sriov_vf_register_bar(dev, IGBVF_MMIO_BAR_IDX, &s->mmio); + + memory_region_init(&s->msix, OBJECT(dev), "igbvf-msix", IGBVF_MSIX_SIZE); + pcie_sriov_vf_register_bar(dev, IGBVF_MSIX_BAR_IDX, &s->msix); + + ret = msix_init(dev, IGBVF_MSIX_VEC_NUM, &s->msix, IGBVF_MSIX_BAR_IDX, 0, + &s->msix, IGBVF_MSIX_BAR_IDX, 0x2000, 0x70, errp); + if (ret) { + return; + } + + for (i = 0; i < IGBVF_MSIX_VEC_NUM; i++) { + msix_vector_use(dev, i); + } + + if (pcie_endpoint_cap_init(dev, 0xa0) < 0) { + hw_error("Failed to initialize PCIe capability"); + } + + if (pcie_aer_init(dev, 1, 0x100, 0x40, errp) < 0) { + hw_error("Failed to initialize AER capability"); + } + + pcie_ari_init(dev, 0x150, 1); +} + +static void igbvf_pci_uninit(PCIDevice *dev) +{ + IgbVfState *s = IGBVF(dev); + + pcie_aer_exit(dev); + pcie_cap_exit(dev); + msix_unuse_all_vectors(dev); + msix_uninit(dev, &s->msix, &s->msix); +} + +static void igbvf_class_init(ObjectClass *class, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(class); + PCIDeviceClass *c = PCI_DEVICE_CLASS(class); + + c->realize = igbvf_pci_realize; + c->exit = igbvf_pci_uninit; + c->vendor_id = PCI_VENDOR_ID_INTEL; + c->device_id = E1000_DEV_ID_82576_VF; + c->revision = 1; + c->class_id = PCI_CLASS_NETWORK_ETHERNET; + + dc->desc = "Intel 82576 Virtual Function"; + dc->user_creatable = false; + + set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); +} + +static const TypeInfo igbvf_info = { + .name = TYPE_IGBVF, + .parent = TYPE_PCI_DEVICE, + .instance_size = sizeof(IgbVfState), + .class_init = igbvf_class_init, + .interfaces = (InterfaceInfo[]) { + { INTERFACE_PCIE_DEVICE }, + { } + }, +}; + +static void igb_register_types(void) +{ + type_register_static(&igbvf_info); +} + +type_init(igb_register_types) diff --git a/hw/net/meson.build b/hw/net/meson.build index 4285145715..e2be0654a1 100644 --- a/hw/net/meson.build +++ b/hw/net/meson.build @@ -10,6 +10,8 @@ softmmu_ss.add(when: 'CONFIG_PCNET_COMMON', if_true: files('pcnet.c')) softmmu_ss.add(when: 'CONFIG_E1000_PCI', if_true: files('e1000.c', 'e1000x_common.c')) softmmu_ss.add(when: 'CONFIG_E1000E_PCI_EXPRESS', if_true: files('net_tx_pkt.c', 'net_rx_pkt.c')) softmmu_ss.add(when: 'CONFIG_E1000E_PCI_EXPRESS', if_true: files('e1000e.c', 'e1000e_core.c', 'e1000x_common.c')) +softmmu_ss.add(when: 'CONFIG_IGB_PCI_EXPRESS', if_true: files('net_tx_pkt.c', 'net_rx_pkt.c')) +softmmu_ss.add(when: 'CONFIG_IGB_PCI_EXPRESS', if_true: files('igb.c', 'igbvf.c', 'igb_core.c')) softmmu_ss.add(when: 'CONFIG_RTL8139_PCI', if_true: files('rtl8139.c')) softmmu_ss.add(when: 'CONFIG_TULIP', if_true: files('tulip.c')) softmmu_ss.add(when: 'CONFIG_VMXNET3_PCI', if_true: files('net_tx_pkt.c', 'net_rx_pkt.c')) diff --git a/hw/net/net_rx_pkt.c b/hw/net/net_rx_pkt.c index 1e1c504e42..39cdea06de 100644 --- a/hw/net/net_rx_pkt.c +++ b/hw/net/net_rx_pkt.c @@ -30,14 +30,11 @@ struct NetRxPkt { uint32_t tot_len; uint16_t tci; size_t ehdr_buf_len; - bool has_virt_hdr; eth_pkt_types_e packet_type; /* Analysis results */ - bool isip4; - bool isip6; - bool isudp; - bool istcp; + bool hasip4; + bool hasip6; size_t l3hdr_off; size_t l4hdr_off; @@ -48,10 +45,9 @@ struct NetRxPkt { eth_l4_hdr_info l4hdr_info; }; -void net_rx_pkt_init(struct NetRxPkt **pkt, bool has_virt_hdr) +void net_rx_pkt_init(struct NetRxPkt **pkt) { struct NetRxPkt *p = g_malloc0(sizeof *p); - p->has_virt_hdr = has_virt_hdr; p->vec = NULL; p->vec_len_total = 0; *pkt = p; @@ -107,12 +103,11 @@ net_rx_pkt_pull_data(struct NetRxPkt *pkt, iov, iovcnt, ploff, pkt->tot_len); } - eth_get_protocols(pkt->vec, pkt->vec_len, &pkt->isip4, &pkt->isip6, - &pkt->isudp, &pkt->istcp, + eth_get_protocols(pkt->vec, pkt->vec_len, &pkt->hasip4, &pkt->hasip6, &pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off, &pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info); - trace_net_rx_pkt_parsed(pkt->isip4, pkt->isip6, pkt->isudp, pkt->istcp, + trace_net_rx_pkt_parsed(pkt->hasip4, pkt->hasip6, pkt->l4hdr_info.proto, pkt->l3hdr_off, pkt->l4hdr_off, pkt->l5hdr_off); } @@ -201,22 +196,20 @@ void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data, assert(pkt); - eth_get_protocols(&iov, 1, &pkt->isip4, &pkt->isip6, - &pkt->isudp, &pkt->istcp, + eth_get_protocols(&iov, 1, &pkt->hasip4, &pkt->hasip6, &pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off, &pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info); } void net_rx_pkt_get_protocols(struct NetRxPkt *pkt, - bool *isip4, bool *isip6, - bool *isudp, bool *istcp) + bool *hasip4, bool *hasip6, + EthL4HdrProto *l4hdr_proto) { assert(pkt); - *isip4 = pkt->isip4; - *isip6 = pkt->isip6; - *isudp = pkt->isudp; - *istcp = pkt->istcp; + *hasip4 = pkt->hasip4; + *hasip6 = pkt->hasip6; + *l4hdr_proto = pkt->l4hdr_info.proto; } size_t net_rx_pkt_get_l3_hdr_offset(struct NetRxPkt *pkt) @@ -333,58 +326,58 @@ net_rx_pkt_calc_rss_hash(struct NetRxPkt *pkt, switch (type) { case NetPktRssIpV4: - assert(pkt->isip4); + assert(pkt->hasip4); trace_net_rx_pkt_rss_ip4(); _net_rx_rss_prepare_ip4(&rss_input[0], pkt, &rss_length); break; case NetPktRssIpV4Tcp: - assert(pkt->isip4); - assert(pkt->istcp); + assert(pkt->hasip4); + assert(pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_TCP); trace_net_rx_pkt_rss_ip4_tcp(); _net_rx_rss_prepare_ip4(&rss_input[0], pkt, &rss_length); _net_rx_rss_prepare_tcp(&rss_input[0], pkt, &rss_length); break; case NetPktRssIpV6Tcp: - assert(pkt->isip6); - assert(pkt->istcp); + assert(pkt->hasip6); + assert(pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_TCP); trace_net_rx_pkt_rss_ip6_tcp(); _net_rx_rss_prepare_ip6(&rss_input[0], pkt, false, &rss_length); _net_rx_rss_prepare_tcp(&rss_input[0], pkt, &rss_length); break; case NetPktRssIpV6: - assert(pkt->isip6); + assert(pkt->hasip6); trace_net_rx_pkt_rss_ip6(); _net_rx_rss_prepare_ip6(&rss_input[0], pkt, false, &rss_length); break; case NetPktRssIpV6Ex: - assert(pkt->isip6); + assert(pkt->hasip6); trace_net_rx_pkt_rss_ip6_ex(); _net_rx_rss_prepare_ip6(&rss_input[0], pkt, true, &rss_length); break; case NetPktRssIpV6TcpEx: - assert(pkt->isip6); - assert(pkt->istcp); + assert(pkt->hasip6); + assert(pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_TCP); trace_net_rx_pkt_rss_ip6_ex_tcp(); _net_rx_rss_prepare_ip6(&rss_input[0], pkt, true, &rss_length); _net_rx_rss_prepare_tcp(&rss_input[0], pkt, &rss_length); break; case NetPktRssIpV4Udp: - assert(pkt->isip4); - assert(pkt->isudp); + assert(pkt->hasip4); + assert(pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP); trace_net_rx_pkt_rss_ip4_udp(); _net_rx_rss_prepare_ip4(&rss_input[0], pkt, &rss_length); _net_rx_rss_prepare_udp(&rss_input[0], pkt, &rss_length); break; case NetPktRssIpV6Udp: - assert(pkt->isip6); - assert(pkt->isudp); + assert(pkt->hasip6); + assert(pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP); trace_net_rx_pkt_rss_ip6_udp(); _net_rx_rss_prepare_ip6(&rss_input[0], pkt, false, &rss_length); _net_rx_rss_prepare_udp(&rss_input[0], pkt, &rss_length); break; case NetPktRssIpV6UdpEx: - assert(pkt->isip6); - assert(pkt->isudp); + assert(pkt->hasip6); + assert(pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP); trace_net_rx_pkt_rss_ip6_ex_udp(); _net_rx_rss_prepare_ip6(&rss_input[0], pkt, true, &rss_length); _net_rx_rss_prepare_udp(&rss_input[0], pkt, &rss_length); @@ -406,7 +399,7 @@ uint16_t net_rx_pkt_get_ip_id(struct NetRxPkt *pkt) { assert(pkt); - if (pkt->isip4) { + if (pkt->hasip4) { return be16_to_cpu(pkt->ip4hdr_info.ip4_hdr.ip_id); } @@ -417,7 +410,7 @@ bool net_rx_pkt_is_tcp_ack(struct NetRxPkt *pkt) { assert(pkt); - if (pkt->istcp) { + if (pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_TCP) { return TCP_HEADER_FLAGS(&pkt->l4hdr_info.hdr.tcp) & TCP_FLAG_ACK; } @@ -428,7 +421,7 @@ bool net_rx_pkt_has_tcp_data(struct NetRxPkt *pkt) { assert(pkt); - if (pkt->istcp) { + if (pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_TCP) { return pkt->l4hdr_info.has_tcp_data; } @@ -465,18 +458,18 @@ void net_rx_pkt_set_vhdr_iovec(struct NetRxPkt *pkt, iov_to_buf(iov, iovcnt, 0, &pkt->virt_hdr, sizeof pkt->virt_hdr); } -bool net_rx_pkt_is_vlan_stripped(struct NetRxPkt *pkt) +void net_rx_pkt_unset_vhdr(struct NetRxPkt *pkt) { assert(pkt); - return pkt->ehdr_buf_len ? true : false; + memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr)); } -bool net_rx_pkt_has_virt_hdr(struct NetRxPkt *pkt) +bool net_rx_pkt_is_vlan_stripped(struct NetRxPkt *pkt) { assert(pkt); - return pkt->has_virt_hdr; + return pkt->ehdr_buf_len ? true : false; } uint16_t net_rx_pkt_get_vlan_tag(struct NetRxPkt *pkt) @@ -494,7 +487,7 @@ bool net_rx_pkt_validate_l3_csum(struct NetRxPkt *pkt, bool *csum_valid) trace_net_rx_pkt_l3_csum_validate_entry(); - if (!pkt->isip4) { + if (!pkt->hasip4) { trace_net_rx_pkt_l3_csum_validate_not_ip4(); return false; } @@ -525,8 +518,8 @@ _net_rx_pkt_calc_l4_csum(struct NetRxPkt *pkt) trace_net_rx_pkt_l4_csum_calc_entry(); - if (pkt->isip4) { - if (pkt->isudp) { + if (pkt->hasip4) { + if (pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP) { csl = be16_to_cpu(pkt->l4hdr_info.hdr.udp.uh_ulen); trace_net_rx_pkt_l4_csum_calc_ip4_udp(); } else { @@ -539,7 +532,7 @@ _net_rx_pkt_calc_l4_csum(struct NetRxPkt *pkt) csl, &cso); trace_net_rx_pkt_l4_csum_calc_ph_csum(cntr, csl); } else { - if (pkt->isudp) { + if (pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP) { csl = be16_to_cpu(pkt->l4hdr_info.hdr.udp.uh_ulen); trace_net_rx_pkt_l4_csum_calc_ip6_udp(); } else { @@ -573,17 +566,19 @@ bool net_rx_pkt_validate_l4_csum(struct NetRxPkt *pkt, bool *csum_valid) trace_net_rx_pkt_l4_csum_validate_entry(); - if (!pkt->istcp && !pkt->isudp) { + if (pkt->l4hdr_info.proto != ETH_L4_HDR_PROTO_TCP && + pkt->l4hdr_info.proto != ETH_L4_HDR_PROTO_UDP) { trace_net_rx_pkt_l4_csum_validate_not_xxp(); return false; } - if (pkt->isudp && (pkt->l4hdr_info.hdr.udp.uh_sum == 0)) { + if (pkt->l4hdr_info.proto == ETH_L4_HDR_PROTO_UDP && + pkt->l4hdr_info.hdr.udp.uh_sum == 0) { trace_net_rx_pkt_l4_csum_validate_udp_with_no_checksum(); return false; } - if (pkt->isip4 && pkt->ip4hdr_info.fragment) { + if (pkt->hasip4 && pkt->ip4hdr_info.fragment) { trace_net_rx_pkt_l4_csum_validate_ip4_fragment(); return false; } @@ -604,22 +599,27 @@ bool net_rx_pkt_fix_l4_csum(struct NetRxPkt *pkt) trace_net_rx_pkt_l4_csum_fix_entry(); - if (pkt->istcp) { + switch (pkt->l4hdr_info.proto) { + case ETH_L4_HDR_PROTO_TCP: l4_cso = offsetof(struct tcp_header, th_sum); trace_net_rx_pkt_l4_csum_fix_tcp(l4_cso); - } else if (pkt->isudp) { + break; + + case ETH_L4_HDR_PROTO_UDP: if (pkt->l4hdr_info.hdr.udp.uh_sum == 0) { trace_net_rx_pkt_l4_csum_fix_udp_with_no_checksum(); return false; } l4_cso = offsetof(struct udp_header, uh_sum); trace_net_rx_pkt_l4_csum_fix_udp(l4_cso); - } else { + break; + + default: trace_net_rx_pkt_l4_csum_fix_not_xxp(); return false; } - if (pkt->isip4 && pkt->ip4hdr_info.fragment) { + if (pkt->hasip4 && pkt->ip4hdr_info.fragment) { trace_net_rx_pkt_l4_csum_fix_ip4_fragment(); return false; } diff --git a/hw/net/net_rx_pkt.h b/hw/net/net_rx_pkt.h index 048e3461f0..d00b484900 100644 --- a/hw/net/net_rx_pkt.h +++ b/hw/net/net_rx_pkt.h @@ -37,10 +37,9 @@ void net_rx_pkt_uninit(struct NetRxPkt *pkt); * Init function for rx packet functionality * * @pkt: packet pointer - * @has_virt_hdr: device uses virtio header * */ -void net_rx_pkt_init(struct NetRxPkt **pkt, bool has_virt_hdr); +void net_rx_pkt_init(struct NetRxPkt **pkt); /** * returns total length of data attached to rx context @@ -67,15 +66,14 @@ void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data, * fetches packet analysis results * * @pkt: packet - * @isip4: whether the packet given is IPv4 - * @isip6: whether the packet given is IPv6 - * @isudp: whether the packet given is UDP - * @istcp: whether the packet given is TCP + * @hasip4: whether the packet has an IPv4 header + * @hasip6: whether the packet has an IPv6 header + * @l4hdr_proto: protocol of L4 header * */ void net_rx_pkt_get_protocols(struct NetRxPkt *pkt, - bool *isip4, bool *isip6, - bool *isudp, bool *istcp); + bool *hasip4, bool *hasip6, + EthL4HdrProto *l4hdr_proto); /** * fetches L3 header offset @@ -215,15 +213,6 @@ uint16_t net_rx_pkt_get_vlan_tag(struct NetRxPkt *pkt); bool net_rx_pkt_is_vlan_stripped(struct NetRxPkt *pkt); /** - * notifies caller if the packet has virtio header - * - * @pkt: packet - * @ret: true if packet has virtio header, false otherwize - * - */ -bool net_rx_pkt_has_virt_hdr(struct NetRxPkt *pkt); - -/** * attach scatter-gather data to rx packet * * @pkt: packet @@ -323,6 +312,14 @@ void net_rx_pkt_set_vhdr_iovec(struct NetRxPkt *pkt, const struct iovec *iov, int iovcnt); /** + * unset vhdr data from packet context + * + * @pkt: packet + * + */ +void net_rx_pkt_unset_vhdr(struct NetRxPkt *pkt); + +/** * save packet type in packet context * * @pkt: packet diff --git a/hw/net/net_tx_pkt.c b/hw/net/net_tx_pkt.c index 2533ea2700..986a3adfe9 100644 --- a/hw/net/net_tx_pkt.c +++ b/hw/net/net_tx_pkt.c @@ -35,7 +35,6 @@ struct NetTxPkt { PCIDevice *pci_dev; struct virtio_net_hdr virt_hdr; - bool has_virt_hdr; struct iovec *raw; uint32_t raw_frags; @@ -54,12 +53,10 @@ struct NetTxPkt { uint16_t hdr_len; eth_pkt_types_e packet_type; uint8_t l4proto; - - bool is_loopback; }; void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev, - uint32_t max_frags, bool has_virt_hdr) + uint32_t max_frags) { struct NetTxPkt *p = g_malloc0(sizeof *p); @@ -71,10 +68,8 @@ void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev, p->max_payload_frags = max_frags; p->max_raw_frags = max_frags; - p->has_virt_hdr = has_virt_hdr; p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr; - p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = - p->has_virt_hdr ? sizeof p->virt_hdr : 0; + p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof p->virt_hdr; p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr; p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr; @@ -304,10 +299,11 @@ func_exit: return rc; } -void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, +bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, bool csum_enable, uint32_t gso_size) { struct tcp_hdr l4hdr; + size_t bytes_read; assert(pkt); /* csum has to be enabled if tso is. */ @@ -328,8 +324,13 @@ void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, case VIRTIO_NET_HDR_GSO_TCPV4: case VIRTIO_NET_HDR_GSO_TCPV6: - iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags, - 0, &l4hdr, sizeof(l4hdr)); + bytes_read = iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], + pkt->payload_frags, 0, &l4hdr, sizeof(l4hdr)); + if (bytes_read < sizeof(l4hdr) || + l4hdr.th_off * sizeof(uint32_t) < sizeof(l4hdr)) { + return false; + } + pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t); pkt->virt_hdr.gso_size = gso_size; break; @@ -341,11 +342,17 @@ void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, if (csum_enable) { switch (pkt->l4proto) { case IP_PROTO_TCP: + if (pkt->payload_len < sizeof(struct tcp_hdr)) { + return false; + } pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; pkt->virt_hdr.csum_start = pkt->hdr_len; pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum); break; case IP_PROTO_UDP: + if (pkt->payload_len < sizeof(struct udp_hdr)) { + return false; + } pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; pkt->virt_hdr.csum_start = pkt->hdr_len; pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum); @@ -354,6 +361,8 @@ void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, break; } } + + return true; } void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt, @@ -464,15 +473,14 @@ void net_tx_pkt_reset(struct NetTxPkt *pkt) pkt->l4proto = 0; } -static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt) +static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt, + struct iovec *iov, uint32_t iov_len, + uint16_t csl) { - struct iovec *iov = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; uint32_t csum_cntr; uint16_t csum = 0; uint32_t cso; /* num of iovec without vhdr */ - uint32_t iov_len = pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1; - uint16_t csl; size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset; uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len); @@ -480,8 +488,6 @@ static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt) iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); /* Calculate L4 TCP/UDP checksum */ - csl = pkt->payload_len; - csum_cntr = 0; cso = 0; /* add pseudo header to csum */ @@ -504,23 +510,16 @@ static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt) iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); } -enum { - NET_TX_PKT_FRAGMENT_L2_HDR_POS = 0, - NET_TX_PKT_FRAGMENT_L3_HDR_POS, - NET_TX_PKT_FRAGMENT_HEADER_NUM -}; - #define NET_MAX_FRAG_SG_LIST (64) static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt, - int *src_idx, size_t *src_offset, struct iovec *dst, int *dst_idx) + int *src_idx, size_t *src_offset, size_t src_len, + struct iovec *dst, int *dst_idx) { size_t fetched = 0; struct iovec *src = pkt->vec; - *dst_idx = NET_TX_PKT_FRAGMENT_HEADER_NUM; - - while (fetched < IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size)) { + while (fetched < src_len) { /* no more place in fragment iov */ if (*dst_idx == NET_MAX_FRAG_SG_LIST) { @@ -535,7 +534,7 @@ static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt, dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset; dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset, - IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size) - fetched); + src_len - fetched); *src_offset += dst[*dst_idx].iov_len; fetched += dst[*dst_idx].iov_len; @@ -551,71 +550,250 @@ static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt, return fetched; } -static inline void net_tx_pkt_sendv(struct NetTxPkt *pkt, - NetClientState *nc, const struct iovec *iov, int iov_cnt) +static void net_tx_pkt_sendv( + void *opaque, const struct iovec *iov, int iov_cnt, + const struct iovec *virt_iov, int virt_iov_cnt) { - if (pkt->is_loopback) { - qemu_receive_packet_iov(nc, iov, iov_cnt); + NetClientState *nc = opaque; + + if (qemu_get_using_vnet_hdr(nc->peer)) { + qemu_sendv_packet(nc, virt_iov, virt_iov_cnt); } else { qemu_sendv_packet(nc, iov, iov_cnt); } } +static bool net_tx_pkt_tcp_fragment_init(struct NetTxPkt *pkt, + struct iovec *fragment, + int *pl_idx, + size_t *l4hdr_len, + int *src_idx, + size_t *src_offset, + size_t *src_len) +{ + struct iovec *l4 = fragment + NET_TX_PKT_PL_START_FRAG; + size_t bytes_read = 0; + struct tcp_hdr *th; + + if (!pkt->payload_frags) { + return false; + } + + l4->iov_len = pkt->virt_hdr.hdr_len - pkt->hdr_len; + l4->iov_base = g_malloc(l4->iov_len); + + *src_idx = NET_TX_PKT_PL_START_FRAG; + while (pkt->vec[*src_idx].iov_len < l4->iov_len - bytes_read) { + memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, + pkt->vec[*src_idx].iov_len); + + bytes_read += pkt->vec[*src_idx].iov_len; + + (*src_idx)++; + if (*src_idx >= pkt->payload_frags + NET_TX_PKT_PL_START_FRAG) { + g_free(l4->iov_base); + return false; + } + } + + *src_offset = l4->iov_len - bytes_read; + memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, + *src_offset); + + th = l4->iov_base; + th->th_flags &= ~(TH_FIN | TH_PUSH); + + *pl_idx = NET_TX_PKT_PL_START_FRAG + 1; + *l4hdr_len = l4->iov_len; + *src_len = pkt->virt_hdr.gso_size; + + return true; +} + +static void net_tx_pkt_tcp_fragment_deinit(struct iovec *fragment) +{ + g_free(fragment[NET_TX_PKT_PL_START_FRAG].iov_base); +} + +static void net_tx_pkt_tcp_fragment_fix(struct NetTxPkt *pkt, + struct iovec *fragment, + size_t fragment_len, + uint8_t gso_type) +{ + struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; + struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; + struct ip_header *ip = l3hdr->iov_base; + struct ip6_header *ip6 = l3hdr->iov_base; + size_t len = l3hdr->iov_len + l4hdr->iov_len + fragment_len; + + switch (gso_type) { + case VIRTIO_NET_HDR_GSO_TCPV4: + ip->ip_len = cpu_to_be16(len); + eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); + break; + + case VIRTIO_NET_HDR_GSO_TCPV6: + len -= sizeof(struct ip6_header); + ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = cpu_to_be16(len); + break; + } +} + +static void net_tx_pkt_tcp_fragment_advance(struct NetTxPkt *pkt, + struct iovec *fragment, + size_t fragment_len, + uint8_t gso_type) +{ + struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; + struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; + struct ip_header *ip = l3hdr->iov_base; + struct tcp_hdr *th = l4hdr->iov_base; + + if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4) { + ip->ip_id = cpu_to_be16(be16_to_cpu(ip->ip_id) + 1); + } + + th->th_seq = cpu_to_be32(be32_to_cpu(th->th_seq) + fragment_len); + th->th_flags &= ~TH_CWR; +} + +static void net_tx_pkt_udp_fragment_init(struct NetTxPkt *pkt, + int *pl_idx, + size_t *l4hdr_len, + int *src_idx, size_t *src_offset, + size_t *src_len) +{ + *pl_idx = NET_TX_PKT_PL_START_FRAG; + *l4hdr_len = 0; + *src_idx = NET_TX_PKT_PL_START_FRAG; + *src_offset = 0; + *src_len = IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size); +} + +static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt, + struct iovec *fragment, + size_t fragment_offset, + size_t fragment_len) +{ + bool more_frags = fragment_offset + fragment_len < pkt->payload_len; + uint16_t orig_flags; + struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; + struct ip_header *ip = l3hdr->iov_base; + uint16_t frag_off_units = fragment_offset / IP_FRAG_UNIT_SIZE; + uint16_t new_ip_off; + + assert(fragment_offset % IP_FRAG_UNIT_SIZE == 0); + assert((frag_off_units & ~IP_OFFMASK) == 0); + + orig_flags = be16_to_cpu(ip->ip_off) & ~(IP_OFFMASK | IP_MF); + new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0); + ip->ip_off = cpu_to_be16(new_ip_off); + ip->ip_len = cpu_to_be16(l3hdr->iov_len + fragment_len); + + eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); +} + static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt, - NetClientState *nc) + NetTxPktCallback callback, + void *context) { + uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; + struct iovec fragment[NET_MAX_FRAG_SG_LIST]; - size_t fragment_len = 0; - bool more_frags = false; - - /* some pointers for shorter code */ - void *l2_iov_base, *l3_iov_base; - size_t l2_iov_len, l3_iov_len; - int src_idx = NET_TX_PKT_PL_START_FRAG, dst_idx; - size_t src_offset = 0; - size_t fragment_offset = 0; + size_t fragment_len; + size_t l4hdr_len; + size_t src_len; - l2_iov_base = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base; - l2_iov_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len; - l3_iov_base = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base; - l3_iov_len = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len; + int src_idx, dst_idx, pl_idx; + size_t src_offset; + size_t fragment_offset = 0; + struct virtio_net_hdr virt_hdr = { + .flags = pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM ? + VIRTIO_NET_HDR_F_DATA_VALID : 0 + }; /* Copy headers */ - fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_base = l2_iov_base; - fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_len = l2_iov_len; - fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_base = l3_iov_base; - fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_len = l3_iov_len; + fragment[NET_TX_PKT_VHDR_FRAG].iov_base = &virt_hdr; + fragment[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof(virt_hdr); + fragment[NET_TX_PKT_L2HDR_FRAG] = pkt->vec[NET_TX_PKT_L2HDR_FRAG]; + fragment[NET_TX_PKT_L3HDR_FRAG] = pkt->vec[NET_TX_PKT_L3HDR_FRAG]; + + switch (gso_type) { + case VIRTIO_NET_HDR_GSO_TCPV4: + case VIRTIO_NET_HDR_GSO_TCPV6: + if (!net_tx_pkt_tcp_fragment_init(pkt, fragment, &pl_idx, &l4hdr_len, + &src_idx, &src_offset, &src_len)) { + return false; + } + break; + + case VIRTIO_NET_HDR_GSO_UDP: + net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], + pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, + pkt->payload_len); + net_tx_pkt_udp_fragment_init(pkt, &pl_idx, &l4hdr_len, + &src_idx, &src_offset, &src_len); + break; + default: + abort(); + } /* Put as much data as possible and send */ - do { - fragment_len = net_tx_pkt_fetch_fragment(pkt, &src_idx, &src_offset, - fragment, &dst_idx); + while (true) { + dst_idx = pl_idx; + fragment_len = net_tx_pkt_fetch_fragment(pkt, + &src_idx, &src_offset, src_len, fragment, &dst_idx); + if (!fragment_len) { + break; + } - more_frags = (fragment_offset + fragment_len < pkt->payload_len); + switch (gso_type) { + case VIRTIO_NET_HDR_GSO_TCPV4: + case VIRTIO_NET_HDR_GSO_TCPV6: + net_tx_pkt_tcp_fragment_fix(pkt, fragment, fragment_len, gso_type); + net_tx_pkt_do_sw_csum(pkt, fragment + NET_TX_PKT_L2HDR_FRAG, + dst_idx - NET_TX_PKT_L2HDR_FRAG, + l4hdr_len + fragment_len); + break; - eth_setup_ip4_fragmentation(l2_iov_base, l2_iov_len, l3_iov_base, - l3_iov_len, fragment_len, fragment_offset, more_frags); + case VIRTIO_NET_HDR_GSO_UDP: + net_tx_pkt_udp_fragment_fix(pkt, fragment, fragment_offset, + fragment_len); + break; + } - eth_fix_ip4_checksum(l3_iov_base, l3_iov_len); + callback(context, + fragment + NET_TX_PKT_L2HDR_FRAG, dst_idx - NET_TX_PKT_L2HDR_FRAG, + fragment + NET_TX_PKT_VHDR_FRAG, dst_idx - NET_TX_PKT_VHDR_FRAG); - net_tx_pkt_sendv(pkt, nc, fragment, dst_idx); + if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || + gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { + net_tx_pkt_tcp_fragment_advance(pkt, fragment, fragment_len, + gso_type); + } fragment_offset += fragment_len; + } - } while (fragment_len && more_frags); + if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || + gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { + net_tx_pkt_tcp_fragment_deinit(fragment); + } return true; } bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc) { - assert(pkt); + bool offload = qemu_get_using_vnet_hdr(nc->peer); + return net_tx_pkt_send_custom(pkt, offload, net_tx_pkt_sendv, nc); +} - if (!pkt->has_virt_hdr && - pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { - net_tx_pkt_do_sw_csum(pkt); - } +bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, + NetTxPktCallback callback, void *context) +{ + assert(pkt); /* * Since underlying infrastructure does not support IP datagrams longer @@ -629,26 +807,22 @@ bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc) } } - if (pkt->has_virt_hdr || - pkt->virt_hdr.gso_type == VIRTIO_NET_HDR_GSO_NONE) { + if (offload || pkt->virt_hdr.gso_type == VIRTIO_NET_HDR_GSO_NONE) { + if (!offload && pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { + net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], + pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, + pkt->payload_len); + } + net_tx_pkt_fix_ip6_payload_len(pkt); - net_tx_pkt_sendv(pkt, nc, pkt->vec, - pkt->payload_frags + NET_TX_PKT_PL_START_FRAG); + callback(context, pkt->vec + NET_TX_PKT_L2HDR_FRAG, + pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_L2HDR_FRAG, + pkt->vec + NET_TX_PKT_VHDR_FRAG, + pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_VHDR_FRAG); return true; } - return net_tx_pkt_do_sw_fragmentation(pkt, nc); -} - -bool net_tx_pkt_send_loopback(struct NetTxPkt *pkt, NetClientState *nc) -{ - bool res; - - pkt->is_loopback = true; - res = net_tx_pkt_send(pkt, nc); - pkt->is_loopback = false; - - return res; + return net_tx_pkt_do_sw_fragmentation(pkt, callback, context); } void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt) diff --git a/hw/net/net_tx_pkt.h b/hw/net/net_tx_pkt.h index 4ec8bbe9bd..f57b4e034b 100644 --- a/hw/net/net_tx_pkt.h +++ b/hw/net/net_tx_pkt.h @@ -26,16 +26,17 @@ struct NetTxPkt; +typedef void (* NetTxPktCallback)(void *, const struct iovec *, int, const struct iovec *, int); + /** * Init function for tx packet functionality * * @pkt: packet pointer * @pci_dev: PCI device processing this packet * @max_frags: max tx ip fragments - * @has_virt_hdr: device uses virtio header. */ void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev, - uint32_t max_frags, bool has_virt_hdr); + uint32_t max_frags); /** * Clean all tx packet resources. @@ -59,9 +60,10 @@ struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt); * @tso_enable: TSO enabled * @csum_enable: CSO enabled * @gso_size: MSS size for TSO + * @ret: operation result * */ -void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, +bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, bool csum_enable, uint32_t gso_size); /** @@ -161,15 +163,16 @@ void net_tx_pkt_reset(struct NetTxPkt *pkt); bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc); /** -* Redirect packet directly to receive path (emulate loopback phy). -* Handles sw offloads if vhdr is not supported. -* -* @pkt: packet -* @nc: NetClientState -* @ret: operation result -* -*/ -bool net_tx_pkt_send_loopback(struct NetTxPkt *pkt, NetClientState *nc); + * Send packet with a custom function. + * + * @pkt: packet + * @offload: whether the callback implements offloading + * @callback: a function to be called back for each transformed packet + * @context: a pointer to be passed to the callback. + * @ret: operation result + */ +bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, + NetTxPktCallback callback, void *context); /** * parse raw packet data and analyze offload requirements. diff --git a/hw/net/trace-events b/hw/net/trace-events index 4c0ec3fda1..65753411fc 100644 --- a/hw/net/trace-events +++ b/hw/net/trace-events @@ -61,7 +61,7 @@ pcnet_ioport_read(void *opaque, uint64_t addr, unsigned size) "opaque=%p addr=0x pcnet_ioport_write(void *opaque, uint64_t addr, uint64_t data, unsigned size) "opaque=%p addr=0x%"PRIx64" data=0x%"PRIx64" size=%d" # net_rx_pkt.c -net_rx_pkt_parsed(bool ip4, bool ip6, bool udp, bool tcp, size_t l3o, size_t l4o, size_t l5o) "RX packet parsed: ip4: %d, ip6: %d, udp: %d, tcp: %d, l3 offset: %zu, l4 offset: %zu, l5 offset: %zu" +net_rx_pkt_parsed(bool ip4, bool ip6, int l4proto, size_t l3o, size_t l4o, size_t l5o) "RX packet parsed: ip4: %d, ip6: %d, l4 protocol: %d, l3 offset: %zu, l4 offset: %zu, l5 offset: %zu" net_rx_pkt_l4_csum_validate_entry(void) "Starting L4 checksum validation" net_rx_pkt_l4_csum_validate_not_xxp(void) "Not a TCP/UDP packet" net_rx_pkt_l4_csum_validate_udp_with_no_checksum(void) "UDP packet without checksum" @@ -165,8 +165,8 @@ e1000e_rx_descr(int ridx, uint64_t base, uint8_t len) "Next RX descriptor: ring e1000e_rx_set_rctl(uint32_t rctl) "RCTL = 0x%x" e1000e_rx_receive_iov(int iovcnt) "Received vector of %d fragments" e1000e_rx_flt_dropped(void) "Received packet dropped by RX filter" -e1000e_rx_written_to_guest(uint32_t causes) "Received packet written to guest (ICR causes %u)" -e1000e_rx_not_written_to_guest(uint32_t causes) "Received packet NOT written to guest (ICR causes %u)" +e1000e_rx_written_to_guest(int queue_idx) "Received packet written to guest (queue %d)" +e1000e_rx_not_written_to_guest(int queue_idx) "Received packet NOT written to guest (queue %d)" e1000e_rx_interrupt_set(uint32_t causes) "Receive interrupt set (ICR causes %u)" e1000e_rx_interrupt_delayed(uint32_t causes) "Receive interrupt delayed (ICR causes %u)" e1000e_rx_set_cso(int cso_state) "RX CSO state set to %d" @@ -177,18 +177,16 @@ e1000e_rx_start_recv(void) e1000e_rx_rss_started(void) "Starting RSS processing" e1000e_rx_rss_disabled(void) "RSS is disabled" e1000e_rx_rss_type(uint32_t type) "RSS type is %u" -e1000e_rx_rss_ip4(bool isfragment, bool istcp, uint32_t mrqc, bool tcpipv4_enabled, bool ipv4_enabled) "RSS IPv4: fragment %d, tcp %d, mrqc 0x%X, tcpipv4 enabled %d, ipv4 enabled %d" +e1000e_rx_rss_ip4(int l4hdr_proto, uint32_t mrqc, bool tcpipv4_enabled, bool ipv4_enabled) "RSS IPv4: L4 header protocol %d, mrqc 0x%X, tcpipv4 enabled %d, ipv4 enabled %d" e1000e_rx_rss_ip6_rfctl(uint32_t rfctl) "RSS IPv6: rfctl 0x%X" -e1000e_rx_rss_ip6(bool ex_dis, bool new_ex_dis, bool istcp, bool has_ext_headers, bool ex_dst_valid, bool ex_src_valid, uint32_t mrqc, bool tcpipv6_enabled, bool ipv6ex_enabled, bool ipv6_enabled) "RSS IPv6: ex_dis: %d, new_ex_dis: %d, tcp %d, has_ext_headers %d, ex_dst_valid %d, ex_src_valid %d, mrqc 0x%X, tcpipv6 enabled %d, ipv6ex enabled %d, ipv6 enabled %d" -e1000e_rx_rss_dispatched_to_queue(int queue_idx) "Packet being dispatched to queue %d" +e1000e_rx_rss_ip6(bool ex_dis, bool new_ex_dis, int l4hdr_proto, bool has_ext_headers, bool ex_dst_valid, bool ex_src_valid, uint32_t mrqc, bool tcpipv6_enabled, bool ipv6ex_enabled, bool ipv6_enabled) "RSS IPv6: ex_dis: %d, new_ex_dis: %d, L4 header protocol %d, has_ext_headers %d, ex_dst_valid %d, ex_src_valid %d, mrqc 0x%X, tcpipv6 enabled %d, ipv6ex enabled %d, ipv6 enabled %d" -e1000e_rx_metadata_protocols(bool isip4, bool isip6, bool isudp, bool istcp) "protocols: ip4: %d, ip6: %d, udp: %d, tcp: %d" +e1000e_rx_metadata_protocols(bool hasip4, bool hasip6, int l4hdr_protocol) "protocols: ip4: %d, ip6: %d, l4hdr: %d" e1000e_rx_metadata_vlan(uint16_t vlan_tag) "VLAN tag is 0x%X" e1000e_rx_metadata_rss(uint32_t rss, uint32_t mrq) "RSS data: rss: 0x%X, mrq: 0x%X" e1000e_rx_metadata_ip_id(uint16_t ip_id) "the IPv4 ID is 0x%X" e1000e_rx_metadata_ack(void) "the packet is TCP ACK" e1000e_rx_metadata_pkt_type(uint32_t pkt_type) "the packet type is %u" -e1000e_rx_metadata_no_virthdr(void) "the packet has no virt-header" e1000e_rx_metadata_virthdr_no_csum_info(void) "virt-header does not contain checksum info" e1000e_rx_metadata_l3_cso_disabled(void) "IP4 CSO is disabled" e1000e_rx_metadata_l4_cso_disabled(void) "TCP/UDP CSO is disabled" @@ -201,10 +199,8 @@ e1000e_rx_metadata_ipv6_filtering_disabled(void) "IPv6 RX filtering disabled by e1000e_vlan_vet(uint16_t vet) "Setting VLAN ethernet type 0x%X" e1000e_irq_msi_notify(uint32_t cause) "MSI notify 0x%x" -e1000e_irq_throttling_no_pending_interrupts(void) "No pending interrupts to notify" e1000e_irq_msi_notify_postponed(void) "Sending MSI postponed by ITR" e1000e_irq_legacy_notify_postponed(void) "Raising legacy IRQ postponed by ITR" -e1000e_irq_throttling_no_pending_vec(int idx) "No pending interrupts for vector %d" e1000e_irq_msix_notify_postponed_vec(int idx) "Sending MSI-X postponed by EITR[%d]" e1000e_irq_legacy_notify(bool level) "IRQ line state: %d" e1000e_irq_msix_notify_vec(uint32_t vector) "MSI-X notify vector 0x%x" @@ -253,7 +249,7 @@ e1000e_vm_state_stopped(void) "VM state is stopped" # e1000e.c e1000e_cb_pci_realize(void) "E1000E PCI realize entry" e1000e_cb_pci_uninit(void) "E1000E PCI unit entry" -e1000e_cb_qdev_reset(void) "E1000E qdev reset entry" +e1000e_cb_qdev_reset_hold(void) "E1000E qdev reset hold" e1000e_cb_pre_save(void) "E1000E pre save entry" e1000e_cb_post_load(void) "E1000E post load entry" @@ -274,6 +270,38 @@ e1000e_msix_use_vector_fail(uint32_t vec, int32_t res) "Failed to use MSI-X vect e1000e_mac_set_permanent(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "Set permanent MAC: %02x:%02x:%02x:%02x:%02x:%02x" e1000e_cfg_support_virtio(bool support) "Virtio header supported: %d" +# igb.c +igb_write_config(uint32_t address, uint32_t val, int len) "CONFIG write 0x%"PRIx32", value: 0x%"PRIx32", len: %"PRId32 +igbvf_write_config(uint32_t address, uint32_t val, int len) "CONFIG write 0x%"PRIx32", value: 0x%"PRIx32", len: %"PRId32 + +# igb_core.c +igb_core_mdic_read(uint32_t addr, uint32_t data) "MDIC READ: PHY[%u] = 0x%x" +igb_core_mdic_read_unhandled(uint32_t addr) "MDIC READ: PHY[%u] UNHANDLED" +igb_core_mdic_write(uint32_t addr, uint32_t data) "MDIC WRITE: PHY[%u] = 0x%x" +igb_core_mdic_write_unhandled(uint32_t addr) "MDIC WRITE: PHY[%u] UNHANDLED" + +igb_rx_desc_buff_size(uint32_t b) "buffer size: %u" +igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint32_t len) "addr: 0x%"PRIx64", offset: %u, from: %p, length: %u" + +igb_rx_metadata_rss(uint32_t rss) "RSS data: 0x%X" + +igb_irq_icr_clear_gpie_nsicr(void) "Clearing ICR on read due to GPIE.NSICR enabled" +igb_irq_icr_write(uint32_t bits, uint32_t old_icr, uint32_t new_icr) "Clearing ICR bits 0x%x: 0x%x --> 0x%x" +igb_irq_set_iam(uint32_t icr) "Update IAM: 0x%x" +igb_irq_read_iam(uint32_t icr) "Current IAM: 0x%x" +igb_irq_write_eics(uint32_t val, bool msix) "Update EICS: 0x%x MSI-X: %d" +igb_irq_write_eims(uint32_t val, bool msix) "Update EIMS: 0x%x MSI-X: %d" +igb_irq_write_eimc(uint32_t val, uint32_t eims, bool msix) "Update EIMC: 0x%x EIMS: 0x%x MSI-X: %d" +igb_irq_write_eiac(uint32_t val) "Update EIAC: 0x%x" +igb_irq_write_eiam(uint32_t val, bool msix) "Update EIAM: 0x%x MSI-X: %d" +igb_irq_write_eicr(uint32_t val, bool msix) "Update EICR: 0x%x MSI-X: %d" +igb_irq_eitr_set(uint32_t eitr_num, uint32_t val) "EITR[%u] = 0x%x" +igb_set_pfmailbox(uint32_t vf_num, uint32_t val) "PFMailbox[%d]: 0x%x" +igb_set_vfmailbox(uint32_t vf_num, uint32_t val) "VFMailbox[%d]: 0x%x" + +# igbvf.c +igbvf_wrn_io_addr_unknown(uint64_t addr) "IO unknown register 0x%"PRIx64 + # spapr_llan.c spapr_vlan_get_rx_bd_from_pool_found(int pool, int32_t count, uint32_t rx_bufs) "pool=%d count=%"PRId32" rxbufs=%"PRIu32 spapr_vlan_get_rx_bd_from_page(int buf_ptr, uint64_t bd) "use_buf_ptr=%d bd=0x%016"PRIx64 diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c index 09d5c7a664..53e1c32643 100644 --- a/hw/net/virtio-net.c +++ b/hw/net/virtio-net.c @@ -1746,39 +1746,61 @@ static int receive_filter(VirtIONet *n, const uint8_t *buf, int size) return 0; } -static uint8_t virtio_net_get_hash_type(bool isip4, - bool isip6, - bool isudp, - bool istcp, +static uint8_t virtio_net_get_hash_type(bool hasip4, + bool hasip6, + EthL4HdrProto l4hdr_proto, uint32_t types) { - if (isip4) { - if (istcp && (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4)) { - return NetPktRssIpV4Tcp; - } - if (isudp && (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4)) { - return NetPktRssIpV4Udp; + if (hasip4) { + switch (l4hdr_proto) { + case ETH_L4_HDR_PROTO_TCP: + if (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { + return NetPktRssIpV4Tcp; + } + break; + + case ETH_L4_HDR_PROTO_UDP: + if (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { + return NetPktRssIpV4Udp; + } + break; + + default: + break; } + if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { return NetPktRssIpV4; } - } else if (isip6) { - uint32_t mask = VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | - VIRTIO_NET_RSS_HASH_TYPE_TCPv6; + } else if (hasip6) { + switch (l4hdr_proto) { + case ETH_L4_HDR_PROTO_TCP: + if (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) { + return NetPktRssIpV6TcpEx; + } + if (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { + return NetPktRssIpV6Tcp; + } + break; + + case ETH_L4_HDR_PROTO_UDP: + if (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) { + return NetPktRssIpV6UdpEx; + } + if (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { + return NetPktRssIpV6Udp; + } + break; - if (istcp && (types & mask)) { - return (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) ? - NetPktRssIpV6TcpEx : NetPktRssIpV6Tcp; + default: + break; } - mask = VIRTIO_NET_RSS_HASH_TYPE_UDP_EX | VIRTIO_NET_RSS_HASH_TYPE_UDPv6; - if (isudp && (types & mask)) { - return (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) ? - NetPktRssIpV6UdpEx : NetPktRssIpV6Udp; + + if (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) { + return NetPktRssIpV6Ex; } - mask = VIRTIO_NET_RSS_HASH_TYPE_IP_EX | VIRTIO_NET_RSS_HASH_TYPE_IPv6; - if (types & mask) { - return (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) ? - NetPktRssIpV6Ex : NetPktRssIpV6; + if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { + return NetPktRssIpV6; } } return 0xff; @@ -1800,7 +1822,8 @@ static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, struct NetRxPkt *pkt = n->rx_pkt; uint8_t net_hash_type; uint32_t hash; - bool isip4, isip6, isudp, istcp; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = { VIRTIO_NET_HASH_REPORT_IPv4, VIRTIO_NET_HASH_REPORT_TCPv4, @@ -1815,14 +1838,8 @@ static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len, size - n->host_hdr_len); - net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); - if (isip4 && (net_rx_pkt_get_ip4_info(pkt)->fragment)) { - istcp = isudp = false; - } - if (isip6 && (net_rx_pkt_get_ip6_info(pkt)->fragment)) { - istcp = isudp = false; - } - net_hash_type = virtio_net_get_hash_type(isip4, isip6, isudp, istcp, + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + net_hash_type = virtio_net_get_hash_type(hasip4, hasip6, l4hdr_proto, n->rss_data.hash_types); if (net_hash_type > NetPktRssIpV6UdpEx) { if (n->rss_data.populate_hash) { @@ -3718,7 +3735,7 @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp) QTAILQ_INIT(&n->rsc_chains); n->qdev = dev; - net_rx_pkt_init(&n->rx_pkt, false); + net_rx_pkt_init(&n->rx_pkt); if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { virtio_net_load_ebpf(n); diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c index 56559cda24..1068b80868 100644 --- a/hw/net/vmxnet3.c +++ b/hw/net/vmxnet3.c @@ -440,19 +440,19 @@ vmxnet3_setup_tx_offloads(VMXNET3State *s) { switch (s->offload_mode) { case VMXNET3_OM_NONE: - net_tx_pkt_build_vheader(s->tx_pkt, false, false, 0); - break; + return net_tx_pkt_build_vheader(s->tx_pkt, false, false, 0); case VMXNET3_OM_CSUM: - net_tx_pkt_build_vheader(s->tx_pkt, false, true, 0); VMW_PKPRN("L4 CSO requested\n"); - break; + return net_tx_pkt_build_vheader(s->tx_pkt, false, true, 0); case VMXNET3_OM_TSO: - net_tx_pkt_build_vheader(s->tx_pkt, true, true, - s->cso_or_gso_size); - net_tx_pkt_update_ip_checksums(s->tx_pkt); VMW_PKPRN("GSO offload requested."); + if (!net_tx_pkt_build_vheader(s->tx_pkt, true, true, + s->cso_or_gso_size)) { + return false; + } + net_tx_pkt_update_ip_checksums(s->tx_pkt); break; default: @@ -847,21 +847,20 @@ static void vmxnet3_rx_need_csum_calculate(struct NetRxPkt *pkt, size_t pkt_len) { struct virtio_net_hdr *vhdr; - bool isip4, isip6, istcp, isudp; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; uint8_t *data; int len; - if (!net_rx_pkt_has_virt_hdr(pkt)) { - return; - } - vhdr = net_rx_pkt_get_vhdr(pkt); if (!VMXNET_FLAG_IS_SET(vhdr->flags, VIRTIO_NET_HDR_F_NEEDS_CSUM)) { return; } - net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); - if (!(isip4 || isip6) || !(istcp || isudp)) { + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + if (!(hasip4 || hasip6) || + (l4hdr_proto != ETH_L4_HDR_PROTO_TCP && + l4hdr_proto != ETH_L4_HDR_PROTO_UDP)) { return; } @@ -889,7 +888,8 @@ static void vmxnet3_rx_update_descr(struct NetRxPkt *pkt, struct Vmxnet3_RxCompDesc *rxcd) { int csum_ok, is_gso; - bool isip4, isip6, istcp, isudp; + bool hasip4, hasip6; + EthL4HdrProto l4hdr_proto; struct virtio_net_hdr *vhdr; uint8_t offload_type; @@ -898,10 +898,6 @@ static void vmxnet3_rx_update_descr(struct NetRxPkt *pkt, rxcd->tci = net_rx_pkt_get_vlan_tag(pkt); } - if (!net_rx_pkt_has_virt_hdr(pkt)) { - goto nocsum; - } - vhdr = net_rx_pkt_get_vhdr(pkt); /* * Checksum is valid when lower level tell so or when lower level @@ -919,16 +915,18 @@ static void vmxnet3_rx_update_descr(struct NetRxPkt *pkt, goto nocsum; } - net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); - if ((!istcp && !isudp) || (!isip4 && !isip6)) { + net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto); + if ((l4hdr_proto != ETH_L4_HDR_PROTO_TCP && + l4hdr_proto != ETH_L4_HDR_PROTO_UDP) || + (!hasip4 && !hasip6)) { goto nocsum; } rxcd->cnc = 0; - rxcd->v4 = isip4 ? 1 : 0; - rxcd->v6 = isip6 ? 1 : 0; - rxcd->tcp = istcp ? 1 : 0; - rxcd->udp = isudp ? 1 : 0; + rxcd->v4 = hasip4 ? 1 : 0; + rxcd->v6 = hasip6 ? 1 : 0; + rxcd->tcp = l4hdr_proto == ETH_L4_HDR_PROTO_TCP; + rxcd->udp = l4hdr_proto == ETH_L4_HDR_PROTO_UDP; rxcd->fcs = rxcd->tuc = rxcd->ipc = 1; return; @@ -1521,9 +1519,8 @@ static void vmxnet3_activate_device(VMXNET3State *s) /* Preallocate TX packet wrapper */ VMW_CFPRN("Max TX fragments is %u", s->max_tx_frags); - net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), - s->max_tx_frags, s->peer_has_vhdr); - net_rx_pkt_init(&s->rx_pkt, s->peer_has_vhdr); + net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), s->max_tx_frags); + net_rx_pkt_init(&s->rx_pkt); /* Read rings memory locations for RX queues */ for (i = 0; i < s->rxq_num; i++) { @@ -2402,9 +2399,8 @@ static int vmxnet3_post_load(void *opaque, int version_id) { VMXNET3State *s = opaque; - net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), - s->max_tx_frags, s->peer_has_vhdr); - net_rx_pkt_init(&s->rx_pkt, s->peer_has_vhdr); + net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), s->max_tx_frags); + net_rx_pkt_init(&s->rx_pkt); if (s->msix_used) { vmxnet3_use_msix_vectors(s, VMXNET3_MAX_INTRS); diff --git a/hw/pci-bridge/cxl_root_port.c b/hw/pci-bridge/cxl_root_port.c index 6664783974..7dfd20aa67 100644 --- a/hw/pci-bridge/cxl_root_port.c +++ b/hw/pci-bridge/cxl_root_port.c @@ -22,6 +22,7 @@ #include "qemu/range.h" #include "hw/pci/pci_bridge.h" #include "hw/pci/pcie_port.h" +#include "hw/pci/msi.h" #include "hw/qdev-properties.h" #include "hw/sysbus.h" #include "qapi/error.h" @@ -29,6 +30,10 @@ #define CXL_ROOT_PORT_DID 0x7075 +#define CXL_RP_MSI_OFFSET 0x60 +#define CXL_RP_MSI_SUPPORTED_FLAGS PCI_MSI_FLAGS_MASKBIT +#define CXL_RP_MSI_NR_VECTOR 2 + /* Copied from the gen root port which we derive */ #define GEN_PCIE_ROOT_PORT_AER_OFFSET 0x100 #define GEN_PCIE_ROOT_PORT_ACS_OFFSET \ @@ -47,6 +52,49 @@ typedef struct CXLRootPort { #define TYPE_CXL_ROOT_PORT "cxl-rp" DECLARE_INSTANCE_CHECKER(CXLRootPort, CXL_ROOT_PORT, TYPE_CXL_ROOT_PORT) +/* + * If two MSI vector are allocated, Advanced Error Interrupt Message Number + * is 1. otherwise 0. + * 17.12.5.10 RPERRSTS, 32:27 bit Advanced Error Interrupt Message Number. + */ +static uint8_t cxl_rp_aer_vector(const PCIDevice *d) +{ + switch (msi_nr_vectors_allocated(d)) { + case 1: + return 0; + case 2: + return 1; + case 4: + case 8: + case 16: + case 32: + default: + break; + } + abort(); + return 0; +} + +static int cxl_rp_interrupts_init(PCIDevice *d, Error **errp) +{ + int rc; + + rc = msi_init(d, CXL_RP_MSI_OFFSET, CXL_RP_MSI_NR_VECTOR, + CXL_RP_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_64BIT, + CXL_RP_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_MASKBIT, + errp); + if (rc < 0) { + assert(rc == -ENOTSUP); + } + + return rc; +} + +static void cxl_rp_interrupts_uninit(PCIDevice *d) +{ + msi_uninit(d); +} + static void latch_registers(CXLRootPort *crp) { uint32_t *reg_state = crp->cxl_cstate.crb.cache_mem_registers; @@ -183,16 +231,29 @@ static void cxl_rp_dvsec_write_config(PCIDevice *dev, uint32_t addr, } } +static void cxl_rp_aer_vector_update(PCIDevice *d) +{ + PCIERootPortClass *rpc = PCIE_ROOT_PORT_GET_CLASS(d); + + if (rpc->aer_vector) { + pcie_aer_root_set_vector(d, rpc->aer_vector(d)); + } +} + static void cxl_rp_write_config(PCIDevice *d, uint32_t address, uint32_t val, int len) { uint16_t slt_ctl, slt_sta; + uint32_t root_cmd = + pci_get_long(d->config + d->exp.aer_cap + PCI_ERR_ROOT_COMMAND); pcie_cap_slot_get(d, &slt_ctl, &slt_sta); pci_bridge_write_config(d, address, val, len); + cxl_rp_aer_vector_update(d); pcie_cap_flr_write_config(d, address, val, len); pcie_cap_slot_write_config(d, slt_ctl, slt_sta, address, val, len); pcie_aer_write_config(d, address, val, len); + pcie_aer_root_write_config(d, address, val, len, root_cmd); cxl_rp_dvsec_write_config(d, address, val, len); } @@ -217,6 +278,9 @@ static void cxl_root_port_class_init(ObjectClass *oc, void *data) rpc->aer_offset = GEN_PCIE_ROOT_PORT_AER_OFFSET; rpc->acs_offset = GEN_PCIE_ROOT_PORT_ACS_OFFSET; + rpc->aer_vector = cxl_rp_aer_vector; + rpc->interrupts_init = cxl_rp_interrupts_init; + rpc->interrupts_uninit = cxl_rp_interrupts_uninit; dc->hotpluggable = false; } diff --git a/hw/pci-bridge/pci_expander_bridge.c b/hw/pci-bridge/pci_expander_bridge.c index e752a21292..ead33f0c05 100644 --- a/hw/pci-bridge/pci_expander_bridge.c +++ b/hw/pci-bridge/pci_expander_bridge.c @@ -15,6 +15,7 @@ #include "hw/pci/pci.h" #include "hw/pci/pci_bus.h" #include "hw/pci/pci_host.h" +#include "hw/pci/pcie_port.h" #include "hw/qdev-properties.h" #include "hw/pci/pci_bridge.h" #include "hw/pci-bridge/pci_expander_bridge.h" @@ -79,6 +80,13 @@ CXLComponentState *cxl_get_hb_cstate(PCIHostState *hb) return &host->cxl_cstate; } +bool cxl_get_hb_passthrough(PCIHostState *hb) +{ + CXLHost *host = PXB_CXL_HOST(hb); + + return host->passthrough; +} + static int pxb_bus_num(PCIBus *bus) { PXBDev *pxb = convert_to_pxb(bus->parent_dev); @@ -289,15 +297,32 @@ static int pxb_map_irq_fn(PCIDevice *pci_dev, int pin) return pin - PCI_SLOT(pxb->devfn); } -static void pxb_dev_reset(DeviceState *dev) +static void pxb_cxl_dev_reset(DeviceState *dev) { CXLHost *cxl = PXB_CXL_DEV(dev)->cxl.cxl_host_bridge; CXLComponentState *cxl_cstate = &cxl->cxl_cstate; + PCIHostState *hb = PCI_HOST_BRIDGE(cxl); uint32_t *reg_state = cxl_cstate->crb.cache_mem_registers; uint32_t *write_msk = cxl_cstate->crb.cache_mem_regs_write_mask; + int dsp_count = 0; cxl_component_register_init_common(reg_state, write_msk, CXL2_ROOT_PORT); - ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, TARGET_COUNT, 8); + /* + * The CXL specification allows for host bridges with no HDM decoders + * if they only have a single root port. + */ + if (!PXB_DEV(dev)->hdm_for_passthrough) { + dsp_count = pcie_count_ds_ports(hb->bus); + } + /* Initial reset will have 0 dsp so wait until > 0 */ + if (dsp_count == 1) { + cxl->passthrough = true; + /* Set Capability ID in header to NONE */ + ARRAY_FIELD_DP32(reg_state, CXL_HDM_CAPABILITY_HEADER, ID, 0); + } else { + ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, TARGET_COUNT, + 8); + } } static gint pxb_compare(gconstpointer a, gconstpointer b) @@ -481,9 +506,18 @@ static void pxb_cxl_dev_realize(PCIDevice *dev, Error **errp) } pxb_dev_realize_common(dev, CXL, errp); - pxb_dev_reset(DEVICE(dev)); + pxb_cxl_dev_reset(DEVICE(dev)); } +static Property pxb_cxl_dev_properties[] = { + /* Note: 0 is not a legal PXB bus number. */ + DEFINE_PROP_UINT8("bus_nr", PXBDev, bus_nr, 0), + DEFINE_PROP_UINT16("numa_node", PXBDev, numa_node, NUMA_NODE_UNASSIGNED), + DEFINE_PROP_BOOL("bypass_iommu", PXBDev, bypass_iommu, false), + DEFINE_PROP_BOOL("hdm_for_passthrough", PXBDev, hdm_for_passthrough, false), + DEFINE_PROP_END_OF_LIST(), +}; + static void pxb_cxl_dev_class_init(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); @@ -497,12 +531,12 @@ static void pxb_cxl_dev_class_init(ObjectClass *klass, void *data) */ dc->desc = "CXL Host Bridge"; - device_class_set_props(dc, pxb_dev_properties); + device_class_set_props(dc, pxb_cxl_dev_properties); set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); /* Host bridges aren't hotpluggable. FIXME: spec reference */ dc->hotpluggable = false; - dc->reset = pxb_dev_reset; + dc->reset = pxb_cxl_dev_reset; } static const TypeInfo pxb_cxl_dev_info = { diff --git a/hw/pci/pci-internal.h b/hw/pci/pci-internal.h index 2ea356bdf5..a7d6d8a732 100644 --- a/hw/pci/pci-internal.h +++ b/hw/pci/pci-internal.h @@ -20,6 +20,5 @@ void pcibus_dev_print(Monitor *mon, DeviceState *dev, int indent); int pcie_aer_parse_error_string(const char *error_name, uint32_t *status, bool *correctable); -int pcie_aer_inject_error(PCIDevice *dev, const PCIEAERErr *err); #endif diff --git a/hw/pci/pci.c b/hw/pci/pci.c index 034fe49e9a..def5000e7b 100644 --- a/hw/pci/pci.c +++ b/hw/pci/pci.c @@ -95,6 +95,21 @@ static const VMStateDescription vmstate_pcibus = { } }; +static gint g_cmp_uint32(gconstpointer a, gconstpointer b, gpointer user_data) +{ + return a - b; +} + +static GSequence *pci_acpi_index_list(void) +{ + static GSequence *used_acpi_index_list; + + if (!used_acpi_index_list) { + used_acpi_index_list = g_sequence_new(NULL); + } + return used_acpi_index_list; +} + static void pci_init_bus_master(PCIDevice *pci_dev) { AddressSpace *dma_as = pci_device_iommu_address_space(pci_dev); @@ -1246,6 +1261,17 @@ static void pci_qdev_unrealize(DeviceState *dev) do_pci_unregister_device(pci_dev); pci_dev->msi_trigger = NULL; + + /* + * clean up acpi-index so it could reused by another device + */ + if (pci_dev->acpi_index) { + GSequence *used_indexes = pci_acpi_index_list(); + + g_sequence_remove(g_sequence_lookup(used_indexes, + GINT_TO_POINTER(pci_dev->acpi_index), + g_cmp_uint32, NULL)); + } } void pci_register_bar(PCIDevice *pci_dev, int region_num, @@ -2005,6 +2031,8 @@ PCIDevice *pci_find_device(PCIBus *bus, int bus_num, uint8_t devfn) return bus->devices[devfn]; } +#define ONBOARD_INDEX_MAX (16 * 1024 - 1) + static void pci_qdev_realize(DeviceState *qdev, Error **errp) { PCIDevice *pci_dev = (PCIDevice *)qdev; @@ -2014,6 +2042,35 @@ static void pci_qdev_realize(DeviceState *qdev, Error **errp) bool is_default_rom; uint16_t class_id; + /* + * capped by systemd (see: udev-builtin-net_id.c) + * as it's the only known user honor it to avoid users + * misconfigure QEMU and then wonder why acpi-index doesn't work + */ + if (pci_dev->acpi_index > ONBOARD_INDEX_MAX) { + error_setg(errp, "acpi-index should be less or equal to %u", + ONBOARD_INDEX_MAX); + return; + } + + /* + * make sure that acpi-index is unique across all present PCI devices + */ + if (pci_dev->acpi_index) { + GSequence *used_indexes = pci_acpi_index_list(); + + if (g_sequence_lookup(used_indexes, + GINT_TO_POINTER(pci_dev->acpi_index), + g_cmp_uint32, NULL)) { + error_setg(errp, "a PCI device with acpi-index = %" PRIu32 + " already exist", pci_dev->acpi_index); + return; + } + g_sequence_insert_sorted(used_indexes, + GINT_TO_POINTER(pci_dev->acpi_index), + g_cmp_uint32, NULL); + } + if (pci_dev->romsize != -1 && !is_power_of_2(pci_dev->romsize)) { error_setg(errp, "ROM size %u is not a power of two", pci_dev->romsize); return; diff --git a/hw/pci/pcie_aer.c b/hw/pci/pcie_aer.c index 9a19be44ae..103667c368 100644 --- a/hw/pci/pcie_aer.c +++ b/hw/pci/pcie_aer.c @@ -112,6 +112,10 @@ int pcie_aer_init(PCIDevice *dev, uint8_t cap_ver, uint16_t offset, pci_set_long(dev->w1cmask + offset + PCI_ERR_UNCOR_STATUS, PCI_ERR_UNC_SUPPORTED); + pci_set_long(dev->config + offset + PCI_ERR_UNCOR_MASK, + PCI_ERR_UNC_MASK_DEFAULT); + pci_set_long(dev->wmask + offset + PCI_ERR_UNCOR_MASK, + PCI_ERR_UNC_SUPPORTED); pci_set_long(dev->config + offset + PCI_ERR_UNCOR_SEVER, PCI_ERR_UNC_SEVERITY_DEFAULT); @@ -188,8 +192,16 @@ static void pcie_aer_update_uncor_status(PCIDevice *dev) static bool pcie_aer_msg_alldev(PCIDevice *dev, const PCIEAERMsg *msg) { + uint16_t devctl = pci_get_word(dev->config + dev->exp.exp_cap + + PCI_EXP_DEVCTL); if (!(pcie_aer_msg_is_uncor(msg) && - (pci_get_word(dev->config + PCI_COMMAND) & PCI_COMMAND_SERR))) { + (pci_get_word(dev->config + PCI_COMMAND) & PCI_COMMAND_SERR)) && + !((msg->severity == PCI_ERR_ROOT_CMD_NONFATAL_EN) && + (devctl & PCI_EXP_DEVCTL_NFERE)) && + !((msg->severity == PCI_ERR_ROOT_CMD_COR_EN) && + (devctl & PCI_EXP_DEVCTL_CERE)) && + !((msg->severity == PCI_ERR_ROOT_CMD_FATAL_EN) && + (devctl & PCI_EXP_DEVCTL_FERE))) { return false; } diff --git a/hw/pci/pcie_port.c b/hw/pci/pcie_port.c index 65a397ad23..20ff2b39e8 100644 --- a/hw/pci/pcie_port.c +++ b/hw/pci/pcie_port.c @@ -161,6 +161,51 @@ PCIDevice *pcie_find_port_by_pn(PCIBus *bus, uint8_t pn) return NULL; } +/* Find first port in devfn number order */ +PCIDevice *pcie_find_port_first(PCIBus *bus) +{ + int devfn; + + for (devfn = 0; devfn < ARRAY_SIZE(bus->devices); devfn++) { + PCIDevice *d = bus->devices[devfn]; + + if (!d || !pci_is_express(d) || !d->exp.exp_cap) { + continue; + } + + if (object_dynamic_cast(OBJECT(d), TYPE_PCIE_PORT)) { + return d; + } + } + + return NULL; +} + +int pcie_count_ds_ports(PCIBus *bus) +{ + int dsp_count = 0; + int devfn; + + for (devfn = 0; devfn < ARRAY_SIZE(bus->devices); devfn++) { + PCIDevice *d = bus->devices[devfn]; + + if (!d || !pci_is_express(d) || !d->exp.exp_cap) { + continue; + } + if (object_dynamic_cast(OBJECT(d), TYPE_PCIE_PORT)) { + dsp_count++; + } + } + return dsp_count; +} + +static bool pcie_slot_is_hotpluggbale_bus(HotplugHandler *plug_handler, + BusState *bus) +{ + PCIESlot *s = PCIE_SLOT(bus->parent); + return s->hotplug; +} + static const TypeInfo pcie_port_type_info = { .name = TYPE_PCIE_PORT, .parent = TYPE_PCI_BRIDGE, @@ -188,6 +233,7 @@ static void pcie_slot_class_init(ObjectClass *oc, void *data) hc->plug = pcie_cap_slot_plug_cb; hc->unplug = pcie_cap_slot_unplug_cb; hc->unplug_request = pcie_cap_slot_unplug_request_cb; + hc->is_hotpluggable_bus = pcie_slot_is_hotpluggbale_bus; } static const TypeInfo pcie_slot_type_info = { diff --git a/hw/pci/pcie_sriov.c b/hw/pci/pcie_sriov.c index f0bd72e069..aa5a757b11 100644 --- a/hw/pci/pcie_sriov.c +++ b/hw/pci/pcie_sriov.c @@ -300,3 +300,8 @@ PCIDevice *pcie_sriov_get_vf_at_index(PCIDevice *dev, int n) } return NULL; } + +uint16_t pcie_sriov_num_vfs(PCIDevice *dev) +{ + return dev->exp.sriov_pf.num_vfs; +} diff --git a/hw/virtio/trace-events b/hw/virtio/trace-events index a87c5f39a2..8f8d05cf9b 100644 --- a/hw/virtio/trace-events +++ b/hw/virtio/trace-events @@ -50,6 +50,7 @@ vhost_vdpa_set_vring_ready(void *dev) "dev: %p" vhost_vdpa_dump_config(void *dev, const char *line) "dev: %p %s" vhost_vdpa_set_config(void *dev, uint32_t offset, uint32_t size, uint32_t flags) "dev: %p offset: %"PRIu32" size: %"PRIu32" flags: 0x%"PRIx32 vhost_vdpa_get_config(void *dev, void *config, uint32_t config_len) "dev: %p config: %p config_len: %"PRIu32 +vhost_vdpa_suspend(void *dev) "dev: %p" vhost_vdpa_dev_start(void *dev, bool started) "dev: %p started: %d" vhost_vdpa_set_log_base(void *dev, uint64_t base, unsigned long long size, int refcnt, int fd, void *log) "dev: %p base: 0x%"PRIx64" size: %llu refcnt: %d fd: %d log: %p" vhost_vdpa_set_vring_addr(void *dev, unsigned int index, unsigned int flags, uint64_t desc_user_addr, uint64_t used_user_addr, uint64_t avail_user_addr, uint64_t log_guest_addr) "dev: %p index: %u flags: 0x%x desc_user_addr: 0x%"PRIx64" used_user_addr: 0x%"PRIx64" avail_user_addr: 0x%"PRIx64" log_guest_addr: 0x%"PRIx64 diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c index 515ccf870d..8361e70d1b 100644 --- a/hw/virtio/vhost-shadow-virtqueue.c +++ b/hw/virtio/vhost-shadow-virtqueue.c @@ -694,13 +694,17 @@ void vhost_svq_stop(VhostShadowVirtqueue *svq) g_autofree VirtQueueElement *elem = NULL; elem = g_steal_pointer(&svq->desc_state[i].elem); if (elem) { - virtqueue_detach_element(svq->vq, elem, 0); + /* + * TODO: This is ok for networking, but other kinds of devices + * might have problems with just unpop these. + */ + virtqueue_unpop(svq->vq, elem, 0); } } next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem); if (next_avail_elem) { - virtqueue_detach_element(svq->vq, next_avail_elem, 0); + virtqueue_unpop(svq->vq, next_avail_elem, 0); } svq->vq = NULL; g_free(svq->desc_next); diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index 8968541514..e5285df4ba 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -2031,8 +2031,8 @@ static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque, } else { if (virtio_has_feature(protocol_features, VHOST_USER_PROTOCOL_F_CONFIG)) { - warn_reportf_err(*errp, "vhost-user backend supports " - "VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not."); + warn_report("vhost-user backend supports " + "VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not."); protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG); } } diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index df3a1e92ac..bc6bad23d5 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -431,16 +431,6 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) trace_vhost_vdpa_init(dev, opaque); int ret; - /* - * Similar to VFIO, we end up pinning all guest memory and have to - * disable discarding of RAM. - */ - ret = ram_block_discard_disable(true); - if (ret) { - error_report("Cannot set discarding of RAM broken"); - return ret; - } - v = opaque; v->dev = dev; dev->opaque = opaque ; @@ -448,10 +438,36 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp) v->msg_type = VHOST_IOTLB_MSG_V2; vhost_vdpa_init_svq(dev, v); + error_propagate(&dev->migration_blocker, v->migration_blocker); if (!vhost_vdpa_first_dev(dev)) { return 0; } + /* + * If dev->shadow_vqs_enabled at initialization that means the device has + * been started with x-svq=on, so don't block migration + */ + if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) { + /* We don't have dev->features yet */ + uint64_t features; + ret = vhost_vdpa_get_dev_features(dev, &features); + if (unlikely(ret)) { + error_setg_errno(errp, -ret, "Could not get device features"); + return ret; + } + vhost_svq_valid_features(features, &dev->migration_blocker); + } + + /* + * Similar to VFIO, we end up pinning all guest memory and have to + * disable discarding of RAM. + */ + ret = ram_block_discard_disable(true); + if (ret) { + error_report("Cannot set discarding of RAM broken"); + return ret; + } + vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER); @@ -577,12 +593,15 @@ static int vhost_vdpa_cleanup(struct vhost_dev *dev) assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); v = dev->opaque; trace_vhost_vdpa_cleanup(dev, v); + if (vhost_vdpa_first_dev(dev)) { + ram_block_discard_disable(false); + } + vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); memory_listener_unregister(&v->listener); vhost_vdpa_svq_cleanup(dev); dev->opaque = NULL; - ram_block_discard_disable(false); return 0; } @@ -659,7 +678,8 @@ static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev) uint64_t features; uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 | 0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH | - 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID; + 0x1ULL << VHOST_BACKEND_F_IOTLB_ASID | + 0x1ULL << VHOST_BACKEND_F_SUSPEND; int r; if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) { @@ -691,11 +711,13 @@ static int vhost_vdpa_get_device_id(struct vhost_dev *dev, static int vhost_vdpa_reset_device(struct vhost_dev *dev) { + struct vhost_vdpa *v = dev->opaque; int ret; uint8_t status = 0; ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status); trace_vhost_vdpa_reset_device(dev, status); + v->suspended = false; return ret; } @@ -1094,6 +1116,29 @@ static void vhost_vdpa_svqs_stop(struct vhost_dev *dev) } } +static void vhost_vdpa_suspend(struct vhost_dev *dev) +{ + struct vhost_vdpa *v = dev->opaque; + int r; + + if (!vhost_vdpa_first_dev(dev)) { + return; + } + + if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) { + trace_vhost_vdpa_suspend(dev); + r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND); + if (unlikely(r)) { + error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno); + } else { + v->suspended = true; + return; + } + } + + vhost_vdpa_reset_device(dev); +} + static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) { struct vhost_vdpa *v = dev->opaque; @@ -1108,6 +1153,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) } vhost_vdpa_set_vring_ready(dev); } else { + vhost_vdpa_suspend(dev); vhost_vdpa_svqs_stop(dev); vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); } @@ -1119,14 +1165,23 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) if (started) { memory_listener_register(&v->listener, &address_space_memory); return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); - } else { - vhost_vdpa_reset_device(dev); - vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | - VIRTIO_CONFIG_S_DRIVER); - memory_listener_unregister(&v->listener); + } - return 0; + return 0; +} + +static void vhost_vdpa_reset_status(struct vhost_dev *dev) +{ + struct vhost_vdpa *v = dev->opaque; + + if (dev->vq_index + dev->nvqs != dev->vq_index_end) { + return; } + + vhost_vdpa_reset_device(dev); + vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | + VIRTIO_CONFIG_S_DRIVER); + memory_listener_unregister(&v->listener); } static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base, @@ -1169,18 +1224,7 @@ static int vhost_vdpa_set_vring_base(struct vhost_dev *dev, struct vhost_vring_state *ring) { struct vhost_vdpa *v = dev->opaque; - VirtQueue *vq = virtio_get_queue(dev->vdev, ring->index); - /* - * vhost-vdpa devices does not support in-flight requests. Set all of them - * as available. - * - * TODO: This is ok for networking, but other kinds of devices might - * have problems with these retransmissions. - */ - while (virtqueue_rewind(vq, 1)) { - continue; - } if (v->shadow_vqs_enabled) { /* * Device vring base was set at device start. SVQ base is handled by @@ -1203,6 +1247,14 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, return 0; } + if (!v->suspended) { + /* + * Cannot trust in value returned by device, let vhost recover used + * idx from guest. + */ + return -1; + } + ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring); trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num); return ret; @@ -1227,25 +1279,24 @@ static int vhost_vdpa_set_vring_call(struct vhost_dev *dev, struct vhost_vring_file *file) { struct vhost_vdpa *v = dev->opaque; + int vdpa_idx = file->index - dev->vq_index; + VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); + /* Remember last call fd because we can switch to SVQ anytime. */ + vhost_svq_set_svq_call_fd(svq, file->fd); if (v->shadow_vqs_enabled) { - int vdpa_idx = file->index - dev->vq_index; - VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx); - - vhost_svq_set_svq_call_fd(svq, file->fd); return 0; - } else { - return vhost_vdpa_set_vring_dev_call(dev, file); } + + return vhost_vdpa_set_vring_dev_call(dev, file); } static int vhost_vdpa_get_features(struct vhost_dev *dev, uint64_t *features) { - struct vhost_vdpa *v = dev->opaque; int ret = vhost_vdpa_get_dev_features(dev, features); - if (ret == 0 && v->shadow_vqs_enabled) { + if (ret == 0) { /* Add SVQ logging capabilities */ *features |= BIT_ULL(VHOST_F_LOG_ALL); } @@ -1313,4 +1364,5 @@ const VhostOps vdpa_ops = { .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, .vhost_force_iommu = vhost_vdpa_force_iommu, .vhost_set_config_call = vhost_vdpa_set_config_call, + .vhost_reset_status = vhost_vdpa_reset_status, }; diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index eb8c4c378c..a266396576 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -2049,6 +2049,9 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings) hdev->vqs + i, hdev->vq_index + i); } + if (hdev->vhost_ops->vhost_reset_status) { + hdev->vhost_ops->vhost_reset_status(hdev); + } if (vhost_dev_has_iommu(hdev)) { if (hdev->vhost_ops->vhost_set_iotlb_callback) { diff --git a/hw/virtio/virtio-crypto.c b/hw/virtio/virtio-crypto.c index 516425e26a..802e1b9659 100644 --- a/hw/virtio/virtio-crypto.c +++ b/hw/virtio/virtio-crypto.c @@ -462,7 +462,7 @@ static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq, req->in_iov = NULL; req->in_num = 0; req->in_len = 0; - req->flags = CRYPTODEV_BACKEND_ALG__MAX; + req->flags = QCRYPTODEV_BACKEND_ALG__MAX; memset(&req->op_info, 0x00, sizeof(req->op_info)); } @@ -472,7 +472,7 @@ static void virtio_crypto_free_request(VirtIOCryptoReq *req) return; } - if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { + if (req->flags == QCRYPTODEV_BACKEND_ALG_SYM) { size_t max_len; CryptoDevBackendSymOpInfo *op_info = req->op_info.u.sym_op_info; @@ -485,7 +485,7 @@ static void virtio_crypto_free_request(VirtIOCryptoReq *req) /* Zeroize and free request data structure */ memset(op_info, 0, sizeof(*op_info) + max_len); g_free(op_info); - } else if (req->flags == CRYPTODEV_BACKEND_ALG_ASYM) { + } else if (req->flags == QCRYPTODEV_BACKEND_ALG_ASYM) { CryptoDevBackendAsymOpInfo *op_info = req->op_info.u.asym_op_info; if (op_info) { g_free(op_info->src); @@ -570,10 +570,10 @@ static void virtio_crypto_req_complete(void *opaque, int ret) VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); uint8_t status = -ret; - if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { + if (req->flags == QCRYPTODEV_BACKEND_ALG_SYM) { virtio_crypto_sym_input_data_helper(vdev, req, status, req->op_info.u.sym_op_info); - } else if (req->flags == CRYPTODEV_BACKEND_ALG_ASYM) { + } else if (req->flags == QCRYPTODEV_BACKEND_ALG_ASYM) { virtio_crypto_akcipher_input_data_helper(vdev, req, status, req->op_info.u.asym_op_info); } @@ -871,11 +871,14 @@ virtio_crypto_handle_request(VirtIOCryptoReq *request) opcode = ldl_le_p(&req.header.opcode); op_info->session_id = ldq_le_p(&req.header.session_id); op_info->op_code = opcode; + op_info->queue_index = queue_index; + op_info->cb = virtio_crypto_req_complete; + op_info->opaque = request; switch (opcode) { case VIRTIO_CRYPTO_CIPHER_ENCRYPT: case VIRTIO_CRYPTO_CIPHER_DECRYPT: - op_info->algtype = request->flags = CRYPTODEV_BACKEND_ALG_SYM; + op_info->algtype = request->flags = QCRYPTODEV_BACKEND_ALG_SYM; ret = virtio_crypto_handle_sym_req(vcrypto, &req.u.sym_req, op_info, out_iov, out_num); @@ -885,7 +888,7 @@ virtio_crypto_handle_request(VirtIOCryptoReq *request) case VIRTIO_CRYPTO_AKCIPHER_DECRYPT: case VIRTIO_CRYPTO_AKCIPHER_SIGN: case VIRTIO_CRYPTO_AKCIPHER_VERIFY: - op_info->algtype = request->flags = CRYPTODEV_BACKEND_ALG_ASYM; + op_info->algtype = request->flags = QCRYPTODEV_BACKEND_ALG_ASYM; ret = virtio_crypto_handle_asym_req(vcrypto, &req.u.akcipher_req, op_info, out_iov, out_num); @@ -898,9 +901,7 @@ check_result: virtio_crypto_req_complete(request, -VIRTIO_CRYPTO_NOTSUPP); } else { ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev, - request, queue_index, - virtio_crypto_req_complete, - request); + op_info); if (ret < 0) { virtio_crypto_req_complete(request, ret); } @@ -997,12 +998,35 @@ static void virtio_crypto_reset(VirtIODevice *vdev) } } +static uint32_t virtio_crypto_init_services(uint32_t qservices) +{ + uint32_t vservices = 0; + + if (qservices & (1 << QCRYPTODEV_BACKEND_SERVICE_CIPHER)) { + vservices |= (1 << VIRTIO_CRYPTO_SERVICE_CIPHER); + } + if (qservices & (1 << QCRYPTODEV_BACKEND_SERVICE_HASH)) { + vservices |= (1 << VIRTIO_CRYPTO_SERVICE_HASH); + } + if (qservices & (1 << QCRYPTODEV_BACKEND_SERVICE_MAC)) { + vservices |= (1 << VIRTIO_CRYPTO_SERVICE_MAC); + } + if (qservices & (1 << QCRYPTODEV_BACKEND_SERVICE_AEAD)) { + vservices |= (1 << VIRTIO_CRYPTO_SERVICE_AEAD); + } + if (qservices & (1 << QCRYPTODEV_BACKEND_SERVICE_AKCIPHER)) { + vservices |= (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER); + } + + return vservices; +} + static void virtio_crypto_init_config(VirtIODevice *vdev) { VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); - vcrypto->conf.crypto_services = - vcrypto->conf.cryptodev->conf.crypto_services; + vcrypto->conf.crypto_services = virtio_crypto_init_services( + vcrypto->conf.cryptodev->conf.crypto_services); vcrypto->conf.cipher_algo_l = vcrypto->conf.cryptodev->conf.cipher_algo_l; vcrypto->conf.cipher_algo_h = diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index f35178f5fc..98c4819fcc 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -1069,7 +1069,7 @@ static void virtqueue_split_get_avail_bytes(VirtQueue *vq, VRingMemoryRegionCaches *caches) { VirtIODevice *vdev = vq->vdev; - unsigned int max, idx; + unsigned int idx; unsigned int total_bufs, in_total, out_total; MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; int64_t len = 0; @@ -1078,13 +1078,12 @@ static void virtqueue_split_get_avail_bytes(VirtQueue *vq, idx = vq->last_avail_idx; total_bufs = in_total = out_total = 0; - max = vq->vring.num; - while ((rc = virtqueue_num_heads(vq, idx)) > 0) { MemoryRegionCache *desc_cache = &caches->desc; unsigned int num_bufs; VRingDesc desc; unsigned int i; + unsigned int max = vq->vring.num; num_bufs = total_bufs; @@ -1206,7 +1205,7 @@ static void virtqueue_packed_get_avail_bytes(VirtQueue *vq, VRingMemoryRegionCaches *caches) { VirtIODevice *vdev = vq->vdev; - unsigned int max, idx; + unsigned int idx; unsigned int total_bufs, in_total, out_total; MemoryRegionCache *desc_cache; MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID; @@ -1218,14 +1217,14 @@ static void virtqueue_packed_get_avail_bytes(VirtQueue *vq, wrap_counter = vq->last_avail_wrap_counter; total_bufs = in_total = out_total = 0; - max = vq->vring.num; - for (;;) { unsigned int num_bufs = total_bufs; unsigned int i = idx; int rc; + unsigned int max = vq->vring.num; desc_cache = &caches->desc; + vring_packed_desc_read(vdev, &desc, desc_cache, idx, true); if (!is_desc_avail(desc.flags, wrap_counter)) { break; diff --git a/include/hw/acpi/ich9.h b/include/hw/acpi/ich9.h index 57a542c4b8..2faf7f0cae 100644 --- a/include/hw/acpi/ich9.h +++ b/include/hw/acpi/ich9.h @@ -87,6 +87,7 @@ void ich9_pm_device_unplug_request_cb(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp); void ich9_pm_device_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp); +bool ich9_pm_is_hotpluggable_bus(HotplugHandler *hotplug_dev, BusState *bus); void ich9_pm_ospm_status(AcpiDeviceIf *adev, ACPIOSTInfoList ***list); #endif /* HW_ACPI_ICH9_H */ diff --git a/include/hw/acpi/pcihp.h b/include/hw/acpi/pcihp.h index 7e268c2c9c..ef59810c17 100644 --- a/include/hw/acpi/pcihp.h +++ b/include/hw/acpi/pcihp.h @@ -49,15 +49,16 @@ typedef struct AcpiPciHpState { uint32_t acpi_index; PCIBus *root; MemoryRegion io; - bool legacy_piix; uint16_t io_base; uint16_t io_len; + bool use_acpi_hotplug_bridge; + bool use_acpi_root_pci_hotplug; } AcpiPciHpState; void acpi_pcihp_init(Object *owner, AcpiPciHpState *, PCIBus *root, - MemoryRegion *address_space_io, bool bridges_enabled, - uint16_t io_base); + MemoryRegion *address_space_io, uint16_t io_base); +bool acpi_pcihp_is_hotpluggbale_bus(AcpiPciHpState *s, BusState *bus); void acpi_pcihp_device_pre_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp); void acpi_pcihp_device_plug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s, @@ -69,7 +70,9 @@ void acpi_pcihp_device_unplug_request_cb(HotplugHandler *hotplug_dev, Error **errp); /* Called on reset */ -void acpi_pcihp_reset(AcpiPciHpState *s, bool acpihp_root_off); +void acpi_pcihp_reset(AcpiPciHpState *s); + +void build_append_pcihp_slots(Aml *parent_scope, PCIBus *bus); extern const VMStateDescription vmstate_acpi_pcihp_pci_status; diff --git a/include/hw/acpi/piix4.h b/include/hw/acpi/piix4.h index be1f8ea80e..eb1c122d80 100644 --- a/include/hw/acpi/piix4.h +++ b/include/hw/acpi/piix4.h @@ -57,8 +57,6 @@ struct PIIX4PMState { Notifier powerdown_notifier; AcpiPciHpState acpi_pci_hotplug; - bool use_acpi_hotplug_bridge; - bool use_acpi_root_pci_hotplug; bool not_migrate_acpi_index; uint8_t disable_s3; diff --git a/include/hw/cxl/cxl.h b/include/hw/cxl/cxl.h index b161be59b7..b2cffbb364 100644 --- a/include/hw/cxl/cxl.h +++ b/include/hw/cxl/cxl.h @@ -49,6 +49,7 @@ struct CXLHost { PCIHostState parent_obj; CXLComponentState cxl_cstate; + bool passthrough; }; #define TYPE_PXB_CXL_HOST "pxb-cxl-host" diff --git a/include/hw/cxl/cxl_component.h b/include/hw/cxl/cxl_component.h index 692d7a5507..42c7e581a7 100644 --- a/include/hw/cxl/cxl_component.h +++ b/include/hw/cxl/cxl_component.h @@ -65,11 +65,37 @@ CXLx_CAPABILITY_HEADER(SNOOP, 0x14) #define CXL_RAS_REGISTERS_OFFSET 0x80 #define CXL_RAS_REGISTERS_SIZE 0x58 REG32(CXL_RAS_UNC_ERR_STATUS, CXL_RAS_REGISTERS_OFFSET) +#define CXL_RAS_UNC_ERR_CACHE_DATA_PARITY 0 +#define CXL_RAS_UNC_ERR_CACHE_ADDRESS_PARITY 1 +#define CXL_RAS_UNC_ERR_CACHE_BE_PARITY 2 +#define CXL_RAS_UNC_ERR_CACHE_DATA_ECC 3 +#define CXL_RAS_UNC_ERR_MEM_DATA_PARITY 4 +#define CXL_RAS_UNC_ERR_MEM_ADDRESS_PARITY 5 +#define CXL_RAS_UNC_ERR_MEM_BE_PARITY 6 +#define CXL_RAS_UNC_ERR_MEM_DATA_ECC 7 +#define CXL_RAS_UNC_ERR_REINIT_THRESHOLD 8 +#define CXL_RAS_UNC_ERR_RSVD_ENCODING 9 +#define CXL_RAS_UNC_ERR_POISON_RECEIVED 10 +#define CXL_RAS_UNC_ERR_RECEIVER_OVERFLOW 11 +#define CXL_RAS_UNC_ERR_INTERNAL 14 +#define CXL_RAS_UNC_ERR_CXL_IDE_TX 15 +#define CXL_RAS_UNC_ERR_CXL_IDE_RX 16 +#define CXL_RAS_UNC_ERR_CXL_UNUSED 63 /* Magic value */ REG32(CXL_RAS_UNC_ERR_MASK, CXL_RAS_REGISTERS_OFFSET + 0x4) REG32(CXL_RAS_UNC_ERR_SEVERITY, CXL_RAS_REGISTERS_OFFSET + 0x8) REG32(CXL_RAS_COR_ERR_STATUS, CXL_RAS_REGISTERS_OFFSET + 0xc) +#define CXL_RAS_COR_ERR_CACHE_DATA_ECC 0 +#define CXL_RAS_COR_ERR_MEM_DATA_ECC 1 +#define CXL_RAS_COR_ERR_CRC_THRESHOLD 2 +#define CXL_RAS_COR_ERR_RETRY_THRESHOLD 3 +#define CXL_RAS_COR_ERR_CACHE_POISON_RECEIVED 4 +#define CXL_RAS_COR_ERR_MEM_POISON_RECEIVED 5 +#define CXL_RAS_COR_ERR_PHYSICAL 6 REG32(CXL_RAS_COR_ERR_MASK, CXL_RAS_REGISTERS_OFFSET + 0x10) REG32(CXL_RAS_ERR_CAP_CTRL, CXL_RAS_REGISTERS_OFFSET + 0x14) + FIELD(CXL_RAS_ERR_CAP_CTRL, FIRST_ERROR_POINTER, 0, 6) +REG32(CXL_RAS_ERR_HEADER0, CXL_RAS_REGISTERS_OFFSET + 0x18) +#define CXL_RAS_ERR_HEADER_NUM 32 /* Offset 0x18 - 0x58 reserved for RAS logs */ /* 8.2.5.10 - CXL Security Capability Structure */ @@ -221,6 +247,7 @@ static inline hwaddr cxl_decode_ig(int ig) } CXLComponentState *cxl_get_hb_cstate(PCIHostState *hb); +bool cxl_get_hb_passthrough(PCIHostState *hb); void cxl_doe_cdat_init(CXLComponentState *cxl_cstate, Error **errp); void cxl_doe_cdat_release(CXLComponentState *cxl_cstate); diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h index 7e5ad65c1d..d589f78202 100644 --- a/include/hw/cxl/cxl_device.h +++ b/include/hw/cxl/cxl_device.h @@ -232,6 +232,14 @@ REG64(CXL_MEM_DEV_STS, 0) FIELD(CXL_MEM_DEV_STS, MBOX_READY, 4, 1) FIELD(CXL_MEM_DEV_STS, RESET_NEEDED, 5, 3) +typedef struct CXLError { + QTAILQ_ENTRY(CXLError) node; + int type; /* Error code as per FE definition */ + uint32_t header[32]; +} CXLError; + +typedef QTAILQ_HEAD(, CXLError) CXLErrorList; + struct CXLType3Dev { /* Private */ PCIDevice parent_obj; @@ -248,6 +256,9 @@ struct CXLType3Dev { /* DOE */ DOECap doe_cdat; + + /* Error injection */ + CXLErrorList error_list; }; #define TYPE_CXL_TYPE3 "cxl-type3" diff --git a/include/hw/hotplug.h b/include/hw/hotplug.h index e15f59c8b3..a9840ed485 100644 --- a/include/hw/hotplug.h +++ b/include/hw/hotplug.h @@ -48,6 +48,7 @@ typedef void (*hotplug_fn)(HotplugHandler *plug_handler, * @unplug: unplug callback. * Used for device removal with devices that implement * asynchronous and synchronous (surprise) removal. + * @is_hotpluggable_bus: called to check if bus/its parent allow hotplug on bus */ struct HotplugHandlerClass { /* <private> */ @@ -58,6 +59,7 @@ struct HotplugHandlerClass { hotplug_fn plug; hotplug_fn unplug_request; hotplug_fn unplug; + bool (*is_hotpluggable_bus)(HotplugHandler *plug_handler, BusState *bus); }; /** diff --git a/include/hw/net/mii.h b/include/hw/net/mii.h index 4ae4dcce7e..ed1bb52b0f 100644 --- a/include/hw/net/mii.h +++ b/include/hw/net/mii.h @@ -55,6 +55,7 @@ #define MII_BMCR_CTST (1 << 7) /* Collision test */ #define MII_BMCR_SPEED1000 (1 << 6) /* MSB of Speed (1000) */ +#define MII_BMSR_100T4 (1 << 15) /* Can do 100mbps T4 */ #define MII_BMSR_100TX_FD (1 << 14) /* Can do 100mbps, full-duplex */ #define MII_BMSR_100TX_HD (1 << 13) /* Can do 100mbps, half-duplex */ #define MII_BMSR_10T_FD (1 << 12) /* Can do 10mbps, full-duplex */ @@ -81,20 +82,31 @@ #define MII_ANLPAR_ACK (1 << 14) #define MII_ANLPAR_PAUSEASY (1 << 11) /* can pause asymmetrically */ #define MII_ANLPAR_PAUSE (1 << 10) /* can pause */ +#define MII_ANLPAR_T4 (1 << 9) #define MII_ANLPAR_TXFD (1 << 8) #define MII_ANLPAR_TX (1 << 7) #define MII_ANLPAR_10FD (1 << 6) #define MII_ANLPAR_10 (1 << 5) #define MII_ANLPAR_CSMACD (1 << 0) -#define MII_ANER_NWAY (1 << 0) /* Can do N-way auto-nego */ +#define MII_ANER_NP (1 << 2) /* Next Page Able */ +#define MII_ANER_NWAY (1 << 0) /* Can do N-way auto-nego */ +#define MII_ANNP_MP (1 << 13) /* Message Page */ + +#define MII_CTRL1000_MASTER (1 << 11) /* MASTER-SLAVE Manual Configuration Value */ +#define MII_CTRL1000_PORT (1 << 10) /* T2_Repeater/DTE bit */ #define MII_CTRL1000_FULL (1 << 9) /* 1000BASE-T full duplex */ #define MII_CTRL1000_HALF (1 << 8) /* 1000BASE-T half duplex */ +#define MII_STAT1000_LOK (1 << 13) /* Local Receiver Status */ +#define MII_STAT1000_ROK (1 << 12) /* Remote Receiver Status */ #define MII_STAT1000_FULL (1 << 11) /* 1000BASE-T full duplex */ #define MII_STAT1000_HALF (1 << 10) /* 1000BASE-T half duplex */ +#define MII_EXTSTAT_1000T_FD (1 << 13) /* 1000BASE-T Full Duplex */ +#define MII_EXTSTAT_1000T_HD (1 << 12) /* 1000BASE-T Half Duplex */ + /* List of vendor identifiers */ /* RealTek 8201 */ #define RTL8201CP_PHYID1 0x0000 diff --git a/include/hw/pci/pci_bridge.h b/include/hw/pci/pci_bridge.h index 63a7521567..81a058bb2c 100644 --- a/include/hw/pci/pci_bridge.h +++ b/include/hw/pci/pci_bridge.h @@ -92,6 +92,7 @@ struct PXBDev { uint8_t bus_nr; uint16_t numa_node; bool bypass_iommu; + bool hdm_for_passthrough; struct cxl_dev { CXLHost *cxl_host_bridge; /* Pointer to a CXLHost */ } cxl; diff --git a/include/hw/pci/pcie_aer.h b/include/hw/pci/pcie_aer.h index 65e71d98fe..1234fdc4e2 100644 --- a/include/hw/pci/pcie_aer.h +++ b/include/hw/pci/pcie_aer.h @@ -100,4 +100,5 @@ void pcie_aer_root_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len, uint32_t root_cmd_prev); +int pcie_aer_inject_error(PCIDevice *dev, const PCIEAERErr *err); #endif /* QEMU_PCIE_AER_H */ diff --git a/include/hw/pci/pcie_port.h b/include/hw/pci/pcie_port.h index 6c40e3733f..90e6cf45b8 100644 --- a/include/hw/pci/pcie_port.h +++ b/include/hw/pci/pcie_port.h @@ -41,6 +41,8 @@ struct PCIEPort { void pcie_port_init_reg(PCIDevice *d); PCIDevice *pcie_find_port_by_pn(PCIBus *bus, uint8_t pn); +PCIDevice *pcie_find_port_first(PCIBus *bus); +int pcie_count_ds_ports(PCIBus *bus); #define TYPE_PCIE_SLOT "pcie-slot" OBJECT_DECLARE_SIMPLE_TYPE(PCIESlot, PCIE_SLOT) diff --git a/include/hw/pci/pcie_regs.h b/include/hw/pci/pcie_regs.h index 1fe0bdd25b..4972106c42 100644 --- a/include/hw/pci/pcie_regs.h +++ b/include/hw/pci/pcie_regs.h @@ -141,6 +141,9 @@ typedef enum PCIExpLinkWidth { PCI_ERR_UNC_ATOP_EBLOCKED | \ PCI_ERR_UNC_TLP_PRF_BLOCKED) +#define PCI_ERR_UNC_MASK_DEFAULT (PCI_ERR_UNC_INTN | \ + PCI_ERR_UNC_TLP_PRF_BLOCKED) + #define PCI_ERR_UNC_SEVERITY_DEFAULT (PCI_ERR_UNC_DLP | \ PCI_ERR_UNC_SDN | \ PCI_ERR_UNC_FCP | \ diff --git a/include/hw/pci/pcie_sriov.h b/include/hw/pci/pcie_sriov.h index 96cc743309..095fb0c9ed 100644 --- a/include/hw/pci/pcie_sriov.h +++ b/include/hw/pci/pcie_sriov.h @@ -76,4 +76,7 @@ PCIDevice *pcie_sriov_get_pf(PCIDevice *dev); */ PCIDevice *pcie_sriov_get_vf_at_index(PCIDevice *dev, int n); +/* Returns the current number of virtual functions. */ +uint16_t pcie_sriov_num_vfs(PCIDevice *dev); + #endif /* QEMU_PCIE_SRIOV_H */ diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h index f5b3b2f89a..bd50ad5ee1 100644 --- a/include/hw/qdev-core.h +++ b/include/hw/qdev-core.h @@ -812,7 +812,18 @@ void qbus_set_bus_hotplug_handler(BusState *bus); static inline bool qbus_is_hotpluggable(BusState *bus) { - return bus->hotplug_handler; + HotplugHandler *plug_handler = bus->hotplug_handler; + bool ret = !!plug_handler; + + if (plug_handler) { + HotplugHandlerClass *hdc; + + hdc = HOTPLUG_HANDLER_GET_CLASS(plug_handler); + if (hdc->is_hotpluggable_bus) { + ret = hdc->is_hotpluggable_bus(plug_handler, bus); + } + } + return ret; } /** diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h index c5ab49051e..ec3fbae58d 100644 --- a/include/hw/virtio/vhost-backend.h +++ b/include/hw/virtio/vhost-backend.h @@ -130,6 +130,9 @@ typedef bool (*vhost_force_iommu_op)(struct vhost_dev *dev); typedef int (*vhost_set_config_call_op)(struct vhost_dev *dev, int fd); + +typedef void (*vhost_reset_status_op)(struct vhost_dev *dev); + typedef struct VhostOps { VhostBackendType backend_type; vhost_backend_init vhost_backend_init; @@ -177,6 +180,7 @@ typedef struct VhostOps { vhost_get_device_id_op vhost_get_device_id; vhost_force_iommu_op vhost_force_iommu; vhost_set_config_call_op vhost_set_config_call; + vhost_reset_status_op vhost_reset_status; } VhostOps; int vhost_backend_update_device_iotlb(struct vhost_dev *dev, diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h index 7997f09a8d..c278a2a8de 100644 --- a/include/hw/virtio/vhost-vdpa.h +++ b/include/hw/virtio/vhost-vdpa.h @@ -42,12 +42,15 @@ typedef struct vhost_vdpa { bool shadow_vqs_enabled; /* Vdpa must send shadow addresses as IOTLB key for data queues, not GPA */ bool shadow_data; + /* Device suspended successfully */ + bool suspended; /* IOVA mapping used by the Shadow Virtqueue */ VhostIOVATree *iova_tree; GPtrArray *shadow_vqs; const VhostShadowVirtqueueOps *shadow_vq_ops; void *shadow_vq_ops_opaque; struct vhost_dev *dev; + Error *migration_blocker; VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX]; } VhostVDPA; diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h index efae6b06bc..fdb69b7f9c 100644 --- a/include/monitor/hmp.h +++ b/include/monitor/hmp.h @@ -180,5 +180,6 @@ void hmp_ioport_read(Monitor *mon, const QDict *qdict); void hmp_ioport_write(Monitor *mon, const QDict *qdict); void hmp_boot_set(Monitor *mon, const QDict *qdict); void hmp_info_mtree(Monitor *mon, const QDict *qdict); +void hmp_info_cryptodev(Monitor *mon, const QDict *qdict); #endif diff --git a/include/net/eth.h b/include/net/eth.h index 6e699b0d7a..c5ae4493b4 100644 --- a/include/net/eth.h +++ b/include/net/eth.h @@ -381,18 +381,24 @@ typedef struct eth_ip4_hdr_info_st { bool fragment; } eth_ip4_hdr_info; +typedef enum EthL4HdrProto { + ETH_L4_HDR_PROTO_INVALID, + ETH_L4_HDR_PROTO_TCP, + ETH_L4_HDR_PROTO_UDP +} EthL4HdrProto; + typedef struct eth_l4_hdr_info_st { union { struct tcp_header tcp; struct udp_header udp; } hdr; + EthL4HdrProto proto; bool has_tcp_data; } eth_l4_hdr_info; void eth_get_protocols(const struct iovec *iov, int iovcnt, - bool *isip4, bool *isip6, - bool *isudp, bool *istcp, + bool *hasip4, bool *hasip6, size_t *l3hdr_off, size_t *l4hdr_off, size_t *l5hdr_off, @@ -400,11 +406,6 @@ void eth_get_protocols(const struct iovec *iov, int iovcnt, eth_ip4_hdr_info *ip4hdr_info, eth_l4_hdr_info *l4hdr_info); -void eth_setup_ip4_fragmentation(const void *l2hdr, size_t l2hdr_len, - void *l3hdr, size_t l3hdr_len, - size_t l3payload_len, - size_t frag_offset, bool more_frags); - void eth_fix_ip4_checksum(void *l3hdr, size_t l3hdr_len); diff --git a/include/net/net.h b/include/net/net.h index 1d88621c12..1448d00afb 100644 --- a/include/net/net.h +++ b/include/net/net.h @@ -56,8 +56,10 @@ typedef RxFilterInfo *(QueryRxFilter)(NetClientState *); typedef bool (HasUfo)(NetClientState *); typedef bool (HasVnetHdr)(NetClientState *); typedef bool (HasVnetHdrLen)(NetClientState *, int); +typedef bool (GetUsingVnetHdr)(NetClientState *); typedef void (UsingVnetHdr)(NetClientState *, bool); typedef void (SetOffload)(NetClientState *, int, int, int, int, int); +typedef int (GetVnetHdrLen)(NetClientState *); typedef void (SetVnetHdrLen)(NetClientState *, int); typedef int (SetVnetLE)(NetClientState *, bool); typedef int (SetVnetBE)(NetClientState *, bool); @@ -84,8 +86,10 @@ typedef struct NetClientInfo { HasUfo *has_ufo; HasVnetHdr *has_vnet_hdr; HasVnetHdrLen *has_vnet_hdr_len; + GetUsingVnetHdr *get_using_vnet_hdr; UsingVnetHdr *using_vnet_hdr; SetOffload *set_offload; + GetVnetHdrLen *get_vnet_hdr_len; SetVnetHdrLen *set_vnet_hdr_len; SetVnetLE *set_vnet_le; SetVnetBE *set_vnet_be; @@ -185,9 +189,11 @@ void qemu_format_nic_info_str(NetClientState *nc, uint8_t macaddr[6]); bool qemu_has_ufo(NetClientState *nc); bool qemu_has_vnet_hdr(NetClientState *nc); bool qemu_has_vnet_hdr_len(NetClientState *nc, int len); +bool qemu_get_using_vnet_hdr(NetClientState *nc); void qemu_using_vnet_hdr(NetClientState *nc, bool enable); void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6, int ecn, int ufo); +int qemu_get_vnet_hdr_len(NetClientState *nc); void qemu_set_vnet_hdr_len(NetClientState *nc, int len); int qemu_set_vnet_le(NetClientState *nc, bool is_le); int qemu_set_vnet_be(NetClientState *nc, bool is_be); diff --git a/include/sysemu/cryptodev.h b/include/sysemu/cryptodev.h index cf9b3f07fe..bc021ce847 100644 --- a/include/sysemu/cryptodev.h +++ b/include/sysemu/cryptodev.h @@ -24,7 +24,9 @@ #define CRYPTODEV_H #include "qemu/queue.h" +#include "qemu/throttle.h" #include "qom/object.h" +#include "qapi/qapi-types-cryptodev.h" /** * CryptoDevBackend: @@ -48,12 +50,6 @@ typedef struct CryptoDevBackendPeers CryptoDevBackendPeers; typedef struct CryptoDevBackendClient CryptoDevBackendClient; -enum CryptoDevBackendAlgType { - CRYPTODEV_BACKEND_ALG_SYM, - CRYPTODEV_BACKEND_ALG_ASYM, - CRYPTODEV_BACKEND_ALG__MAX, -}; - /** * CryptoDevBackendSymSessionInfo: * @@ -179,17 +175,22 @@ typedef struct CryptoDevBackendAsymOpInfo { uint8_t *dst; } CryptoDevBackendAsymOpInfo; +typedef void (*CryptoDevCompletionFunc) (void *opaque, int ret); + typedef struct CryptoDevBackendOpInfo { - enum CryptoDevBackendAlgType algtype; + QCryptodevBackendAlgType algtype; uint32_t op_code; + uint32_t queue_index; + CryptoDevCompletionFunc cb; + void *opaque; /* argument for cb */ uint64_t session_id; union { CryptoDevBackendSymOpInfo *sym_op_info; CryptoDevBackendAsymOpInfo *asym_op_info; } u; + QTAILQ_ENTRY(CryptoDevBackendOpInfo) next; } CryptoDevBackendOpInfo; -typedef void (*CryptoDevCompletionFunc) (void *opaque, int ret); struct CryptoDevBackendClass { ObjectClass parent_class; @@ -209,24 +210,11 @@ struct CryptoDevBackendClass { void *opaque); int (*do_op)(CryptoDevBackend *backend, - CryptoDevBackendOpInfo *op_info, - uint32_t queue_index, - CryptoDevCompletionFunc cb, - void *opaque); + CryptoDevBackendOpInfo *op_info); }; -typedef enum CryptoDevBackendOptionsType { - CRYPTODEV_BACKEND_TYPE_NONE = 0, - CRYPTODEV_BACKEND_TYPE_BUILTIN = 1, - CRYPTODEV_BACKEND_TYPE_VHOST_USER = 2, - CRYPTODEV_BACKEND_TYPE_LKCF = 3, - CRYPTODEV_BACKEND_TYPE__MAX, -} CryptoDevBackendOptionsType; - struct CryptoDevBackendClient { - CryptoDevBackendOptionsType type; - char *model; - char *name; + QCryptodevBackendType type; char *info_str; unsigned int queue_index; int vring_enable; @@ -260,6 +248,24 @@ struct CryptoDevBackendConf { uint64_t max_size; }; +typedef struct CryptodevBackendSymStat { + int64_t encrypt_ops; + int64_t decrypt_ops; + int64_t encrypt_bytes; + int64_t decrypt_bytes; +} CryptodevBackendSymStat; + +typedef struct CryptodevBackendAsymStat { + int64_t encrypt_ops; + int64_t decrypt_ops; + int64_t sign_ops; + int64_t verify_ops; + int64_t encrypt_bytes; + int64_t decrypt_bytes; + int64_t sign_bytes; + int64_t verify_bytes; +} CryptodevBackendAsymStat; + struct CryptoDevBackend { Object parent_obj; @@ -267,15 +273,48 @@ struct CryptoDevBackend { /* Tag the cryptodev backend is used by virtio-crypto or not */ bool is_used; CryptoDevBackendConf conf; + CryptodevBackendSymStat *sym_stat; + CryptodevBackendAsymStat *asym_stat; + + ThrottleState ts; + ThrottleTimers tt; + ThrottleConfig tc; + QTAILQ_HEAD(, CryptoDevBackendOpInfo) opinfos; }; +#define CryptodevSymStatInc(be, op, bytes) do { \ + be->sym_stat->op##_bytes += (bytes); \ + be->sym_stat->op##_ops += 1; \ +} while (/*CONSTCOND*/0) + +#define CryptodevSymStatIncEncrypt(be, bytes) \ + CryptodevSymStatInc(be, encrypt, bytes) + +#define CryptodevSymStatIncDecrypt(be, bytes) \ + CryptodevSymStatInc(be, decrypt, bytes) + +#define CryptodevAsymStatInc(be, op, bytes) do { \ + be->asym_stat->op##_bytes += (bytes); \ + be->asym_stat->op##_ops += 1; \ +} while (/*CONSTCOND*/0) + +#define CryptodevAsymStatIncEncrypt(be, bytes) \ + CryptodevAsymStatInc(be, encrypt, bytes) + +#define CryptodevAsymStatIncDecrypt(be, bytes) \ + CryptodevAsymStatInc(be, decrypt, bytes) + +#define CryptodevAsymStatIncSign(be, bytes) \ + CryptodevAsymStatInc(be, sign, bytes) + +#define CryptodevAsymStatIncVerify(be, bytes) \ + CryptodevAsymStatInc(be, verify, bytes) + + /** * cryptodev_backend_new_client: - * @model: the cryptodev backend model - * @name: the cryptodev backend name, can be NULL * - * Creates a new cryptodev backend client object - * with the @name in the model @model. + * Creates a new cryptodev backend client object. * * The returned object must be released with * cryptodev_backend_free_client() when no @@ -283,9 +322,8 @@ struct CryptoDevBackend { * * Returns: a new cryptodev backend client object */ -CryptoDevBackendClient * -cryptodev_backend_new_client(const char *model, - const char *name); +CryptoDevBackendClient *cryptodev_backend_new_client(void); + /** * cryptodev_backend_free_client: * @cc: the cryptodev backend client object @@ -354,24 +392,17 @@ int cryptodev_backend_close_session( /** * cryptodev_backend_crypto_operation: * @backend: the cryptodev backend object - * @opaque1: pointer to a VirtIOCryptoReq object - * @queue_index: queue index of cryptodev backend client - * @errp: pointer to a NULL-initialized error object - * @cb: callbacks when operation is completed - * @opaque2: parameter passed to cb + * @op_info: pointer to a CryptoDevBackendOpInfo object * - * Do crypto operation, such as encryption and - * decryption + * Do crypto operation, such as encryption, decryption, signature and + * verification * * Returns: 0 for success and cb will be called when creation is completed, * negative value for error, and cb will not be called. */ int cryptodev_backend_crypto_operation( CryptoDevBackend *backend, - void *opaque1, - uint32_t queue_index, - CryptoDevCompletionFunc cb, - void *opaque2); + CryptoDevBackendOpInfo *op_info); /** * cryptodev_backend_set_used: diff --git a/net/dump.c b/net/dump.c index 6a63b15359..7d05f16ca7 100644 --- a/net/dump.c +++ b/net/dump.c @@ -61,12 +61,13 @@ struct pcap_sf_pkthdr { uint32_t len; }; -static ssize_t dump_receive_iov(DumpState *s, const struct iovec *iov, int cnt) +static ssize_t dump_receive_iov(DumpState *s, const struct iovec *iov, int cnt, + int offset) { struct pcap_sf_pkthdr hdr; int64_t ts; int caplen; - size_t size = iov_size(iov, cnt); + size_t size = iov_size(iov, cnt) - offset; struct iovec dumpiov[cnt + 1]; /* Early return in case of previous error. */ @@ -84,7 +85,7 @@ static ssize_t dump_receive_iov(DumpState *s, const struct iovec *iov, int cnt) dumpiov[0].iov_base = &hdr; dumpiov[0].iov_len = sizeof(hdr); - cnt = iov_copy(&dumpiov[1], cnt, iov, cnt, 0, caplen); + cnt = iov_copy(&dumpiov[1], cnt, iov, cnt, offset, caplen); if (writev(s->fd, dumpiov, cnt + 1) != sizeof(hdr) + caplen) { error_report("network dump write error - stopping dump"); @@ -153,8 +154,10 @@ static ssize_t filter_dump_receive_iov(NetFilterState *nf, NetClientState *sndr, int iovcnt, NetPacketSent *sent_cb) { NetFilterDumpState *nfds = FILTER_DUMP(nf); + int offset = qemu_get_using_vnet_hdr(nf->netdev) ? + qemu_get_vnet_hdr_len(nf->netdev) : 0; - dump_receive_iov(&nfds->ds, iov, iovcnt); + dump_receive_iov(&nfds->ds, iov, iovcnt, offset); return 0; } diff --git a/net/eth.c b/net/eth.c index f074b2f9f3..70bcd8e355 100644 --- a/net/eth.c +++ b/net/eth.c @@ -137,8 +137,7 @@ _eth_tcp_has_data(bool is_ip4, } void eth_get_protocols(const struct iovec *iov, int iovcnt, - bool *isip4, bool *isip6, - bool *isudp, bool *istcp, + bool *hasip4, bool *hasip6, size_t *l3hdr_off, size_t *l4hdr_off, size_t *l5hdr_off, @@ -151,8 +150,10 @@ void eth_get_protocols(const struct iovec *iov, int iovcnt, size_t l2hdr_len = eth_get_l2_hdr_length_iov(iov, iovcnt); size_t input_size = iov_size(iov, iovcnt); size_t copied; + uint8_t ip_p; - *isip4 = *isip6 = *isudp = *istcp = false; + *hasip4 = *hasip6 = false; + l4hdr_info->proto = ETH_L4_HDR_PROTO_INVALID; proto = eth_get_l3_proto(iov, iovcnt, l2hdr_len); @@ -166,68 +167,62 @@ void eth_get_protocols(const struct iovec *iov, int iovcnt, } copied = iov_to_buf(iov, iovcnt, l2hdr_len, iphdr, sizeof(*iphdr)); - - *isip4 = true; - - if (copied < sizeof(*iphdr)) { + if (copied < sizeof(*iphdr) || + IP_HEADER_VERSION(iphdr) != IP_HEADER_VERSION_4) { return; } - if (IP_HEADER_VERSION(iphdr) == IP_HEADER_VERSION_4) { - if (iphdr->ip_p == IP_PROTO_TCP) { - *istcp = true; - } else if (iphdr->ip_p == IP_PROTO_UDP) { - *isudp = true; - } - } - + *hasip4 = true; + ip_p = iphdr->ip_p; ip4hdr_info->fragment = IP4_IS_FRAGMENT(iphdr); *l4hdr_off = l2hdr_len + IP_HDR_GET_LEN(iphdr); fragment = ip4hdr_info->fragment; } else if (proto == ETH_P_IPV6) { - - *isip6 = true; - if (eth_parse_ipv6_hdr(iov, iovcnt, l2hdr_len, - ip6hdr_info)) { - if (ip6hdr_info->l4proto == IP_PROTO_TCP) { - *istcp = true; - } else if (ip6hdr_info->l4proto == IP_PROTO_UDP) { - *isudp = true; - } - } else { + if (!eth_parse_ipv6_hdr(iov, iovcnt, l2hdr_len, ip6hdr_info)) { return; } + *hasip6 = true; + ip_p = ip6hdr_info->l4proto; *l4hdr_off = l2hdr_len + ip6hdr_info->full_hdr_len; fragment = ip6hdr_info->fragment; + } else { + return; } - if (!fragment) { - if (*istcp) { - *istcp = _eth_copy_chunk(input_size, - iov, iovcnt, - *l4hdr_off, sizeof(l4hdr_info->hdr.tcp), - &l4hdr_info->hdr.tcp); - - if (*istcp) { - *l5hdr_off = *l4hdr_off + - TCP_HEADER_DATA_OFFSET(&l4hdr_info->hdr.tcp); - - l4hdr_info->has_tcp_data = - _eth_tcp_has_data(proto == ETH_P_IP, - &ip4hdr_info->ip4_hdr, - &ip6hdr_info->ip6_hdr, - *l4hdr_off - *l3hdr_off, - &l4hdr_info->hdr.tcp); - } - } else if (*isudp) { - *isudp = _eth_copy_chunk(input_size, - iov, iovcnt, - *l4hdr_off, sizeof(l4hdr_info->hdr.udp), - &l4hdr_info->hdr.udp); + if (fragment) { + return; + } + + switch (ip_p) { + case IP_PROTO_TCP: + if (_eth_copy_chunk(input_size, + iov, iovcnt, + *l4hdr_off, sizeof(l4hdr_info->hdr.tcp), + &l4hdr_info->hdr.tcp)) { + l4hdr_info->proto = ETH_L4_HDR_PROTO_TCP; + *l5hdr_off = *l4hdr_off + + TCP_HEADER_DATA_OFFSET(&l4hdr_info->hdr.tcp); + + l4hdr_info->has_tcp_data = + _eth_tcp_has_data(proto == ETH_P_IP, + &ip4hdr_info->ip4_hdr, + &ip6hdr_info->ip6_hdr, + *l4hdr_off - *l3hdr_off, + &l4hdr_info->hdr.tcp); + } + break; + + case IP_PROTO_UDP: + if (_eth_copy_chunk(input_size, + iov, iovcnt, + *l4hdr_off, sizeof(l4hdr_info->hdr.udp), + &l4hdr_info->hdr.udp)) { + l4hdr_info->proto = ETH_L4_HDR_PROTO_UDP; *l5hdr_off = *l4hdr_off + sizeof(l4hdr_info->hdr.udp); } + break; } } @@ -315,33 +310,6 @@ eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff, } void -eth_setup_ip4_fragmentation(const void *l2hdr, size_t l2hdr_len, - void *l3hdr, size_t l3hdr_len, - size_t l3payload_len, - size_t frag_offset, bool more_frags) -{ - const struct iovec l2vec = { - .iov_base = (void *) l2hdr, - .iov_len = l2hdr_len - }; - - if (eth_get_l3_proto(&l2vec, 1, l2hdr_len) == ETH_P_IP) { - uint16_t orig_flags; - struct ip_header *iphdr = (struct ip_header *) l3hdr; - uint16_t frag_off_units = frag_offset / IP_FRAG_UNIT_SIZE; - uint16_t new_ip_off; - - assert(frag_offset % IP_FRAG_UNIT_SIZE == 0); - assert((frag_off_units & ~IP_OFFMASK) == 0); - - orig_flags = be16_to_cpu(iphdr->ip_off) & ~(IP_OFFMASK|IP_MF); - new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0); - iphdr->ip_off = cpu_to_be16(new_ip_off); - iphdr->ip_len = cpu_to_be16(l3payload_len + l3hdr_len); - } -} - -void eth_fix_ip4_checksum(void *l3hdr, size_t l3hdr_len) { struct ip_header *iphdr = (struct ip_header *) l3hdr; diff --git a/net/net.c b/net/net.c index ebc7ce0231..6492ad530e 100644 --- a/net/net.c +++ b/net/net.c @@ -513,6 +513,15 @@ bool qemu_has_vnet_hdr_len(NetClientState *nc, int len) return nc->info->has_vnet_hdr_len(nc, len); } +bool qemu_get_using_vnet_hdr(NetClientState *nc) +{ + if (!nc || !nc->info->get_using_vnet_hdr) { + return false; + } + + return nc->info->get_using_vnet_hdr(nc); +} + void qemu_using_vnet_hdr(NetClientState *nc, bool enable) { if (!nc || !nc->info->using_vnet_hdr) { @@ -532,6 +541,15 @@ void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6, nc->info->set_offload(nc, csum, tso4, tso6, ecn, ufo); } +int qemu_get_vnet_hdr_len(NetClientState *nc) +{ + if (!nc || !nc->info->get_vnet_hdr_len) { + return 0; + } + + return nc->info->get_vnet_hdr_len(nc); +} + void qemu_set_vnet_hdr_len(NetClientState *nc, int len) { if (!nc || !nc->info->set_vnet_hdr_len) { diff --git a/net/tap.c b/net/tap.c index 7d7bc1dc5f..1bf085d422 100644 --- a/net/tap.c +++ b/net/tap.c @@ -255,6 +255,13 @@ static bool tap_has_vnet_hdr_len(NetClientState *nc, int len) return !!tap_probe_vnet_hdr_len(s->fd, len); } +static int tap_get_vnet_hdr_len(NetClientState *nc) +{ + TAPState *s = DO_UPCAST(TAPState, nc, nc); + + return s->host_vnet_hdr_len; +} + static void tap_set_vnet_hdr_len(NetClientState *nc, int len) { TAPState *s = DO_UPCAST(TAPState, nc, nc); @@ -268,6 +275,13 @@ static void tap_set_vnet_hdr_len(NetClientState *nc, int len) s->host_vnet_hdr_len = len; } +static bool tap_get_using_vnet_hdr(NetClientState *nc) +{ + TAPState *s = DO_UPCAST(TAPState, nc, nc); + + return s->using_vnet_hdr; +} + static void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr) { TAPState *s = DO_UPCAST(TAPState, nc, nc); @@ -372,8 +386,10 @@ static NetClientInfo net_tap_info = { .has_ufo = tap_has_ufo, .has_vnet_hdr = tap_has_vnet_hdr, .has_vnet_hdr_len = tap_has_vnet_hdr_len, + .get_using_vnet_hdr = tap_get_using_vnet_hdr, .using_vnet_hdr = tap_using_vnet_hdr, .set_offload = tap_set_offload, + .get_vnet_hdr_len = tap_get_vnet_hdr_len, .set_vnet_hdr_len = tap_set_vnet_hdr_len, .set_vnet_le = tap_set_vnet_le, .set_vnet_be = tap_set_vnet_be, diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c index de5ed8ff22..99904a0da7 100644 --- a/net/vhost-vdpa.c +++ b/net/vhost-vdpa.c @@ -26,12 +26,15 @@ #include <err.h> #include "standard-headers/linux/virtio_net.h" #include "monitor/monitor.h" +#include "migration/migration.h" +#include "migration/misc.h" #include "hw/virtio/vhost.h" /* Todo:need to add the multiqueue support here */ typedef struct VhostVDPAState { NetClientState nc; struct vhost_vdpa vhost_vdpa; + Notifier migration_state; VHostNetState *vhost_net; /* Control commands shadow buffers */ @@ -98,6 +101,8 @@ static const uint64_t vdpa_svq_device_features = BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | + /* VHOST_F_LOG_ALL is exposed by SVQ */ + BIT_ULL(VHOST_F_LOG_ALL) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY); @@ -178,13 +183,9 @@ err_init: static void vhost_vdpa_cleanup(NetClientState *nc) { VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); - struct vhost_dev *dev = &s->vhost_net->dev; qemu_vfree(s->cvq_cmd_out_buffer); qemu_vfree(s->status); - if (dev->vq_index + dev->nvqs == dev->vq_index_end) { - g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete); - } if (s->vhost_net) { vhost_net_cleanup(s->vhost_net); g_free(s->vhost_net); @@ -234,10 +235,126 @@ static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf, return size; } +/** From any vdpa net client, get the netclient of the first queue pair */ +static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s) +{ + NICState *nic = qemu_get_nic(s->nc.peer); + NetClientState *nc0 = qemu_get_peer(nic->ncs, 0); + + return DO_UPCAST(VhostVDPAState, nc, nc0); +} + +static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable) +{ + struct vhost_vdpa *v = &s->vhost_vdpa; + VirtIONet *n; + VirtIODevice *vdev; + int data_queue_pairs, cvq, r; + + /* We are only called on the first data vqs and only if x-svq is not set */ + if (s->vhost_vdpa.shadow_vqs_enabled == enable) { + return; + } + + vdev = v->dev->vdev; + n = VIRTIO_NET(vdev); + if (!n->vhost_started) { + return; + } + + data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; + cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ? + n->max_ncs - n->max_queue_pairs : 0; + /* + * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter + * in the future and resume the device if read-only operations between + * suspend and reset goes wrong. + */ + vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq); + + /* Start will check migration setup_or_active to configure or not SVQ */ + r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq); + if (unlikely(r < 0)) { + error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r); + } +} + +static void vdpa_net_migration_state_notifier(Notifier *notifier, void *data) +{ + MigrationState *migration = data; + VhostVDPAState *s = container_of(notifier, VhostVDPAState, + migration_state); + + if (migration_in_setup(migration)) { + vhost_vdpa_net_log_global_enable(s, true); + } else if (migration_has_failed(migration)) { + vhost_vdpa_net_log_global_enable(s, false); + } +} + +static void vhost_vdpa_net_data_start_first(VhostVDPAState *s) +{ + struct vhost_vdpa *v = &s->vhost_vdpa; + + add_migration_state_change_notifier(&s->migration_state); + if (v->shadow_vqs_enabled) { + v->iova_tree = vhost_iova_tree_new(v->iova_range.first, + v->iova_range.last); + } +} + +static int vhost_vdpa_net_data_start(NetClientState *nc) +{ + VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); + struct vhost_vdpa *v = &s->vhost_vdpa; + + assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); + + if (s->always_svq || + migration_is_setup_or_active(migrate_get_current()->state)) { + v->shadow_vqs_enabled = true; + v->shadow_data = true; + } else { + v->shadow_vqs_enabled = false; + v->shadow_data = false; + } + + if (v->index == 0) { + vhost_vdpa_net_data_start_first(s); + return 0; + } + + if (v->shadow_vqs_enabled) { + VhostVDPAState *s0 = vhost_vdpa_net_first_nc_vdpa(s); + v->iova_tree = s0->vhost_vdpa.iova_tree; + } + + return 0; +} + +static void vhost_vdpa_net_client_stop(NetClientState *nc) +{ + VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); + struct vhost_dev *dev; + + assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); + + if (s->vhost_vdpa.index == 0) { + remove_migration_state_change_notifier(&s->migration_state); + } + + dev = s->vhost_vdpa.dev; + if (dev->vq_index + dev->nvqs == dev->vq_index_end) { + g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete); + } +} + static NetClientInfo net_vhost_vdpa_info = { .type = NET_CLIENT_DRIVER_VHOST_VDPA, .size = sizeof(VhostVDPAState), .receive = vhost_vdpa_receive, + .start = vhost_vdpa_net_data_start, + .stop = vhost_vdpa_net_client_stop, .cleanup = vhost_vdpa_cleanup, .has_vnet_hdr = vhost_vdpa_has_vnet_hdr, .has_ufo = vhost_vdpa_has_ufo, @@ -351,7 +468,7 @@ dma_map_err: static int vhost_vdpa_net_cvq_start(NetClientState *nc) { - VhostVDPAState *s; + VhostVDPAState *s, *s0; struct vhost_vdpa *v; uint64_t backend_features; int64_t cvq_group; @@ -362,11 +479,12 @@ static int vhost_vdpa_net_cvq_start(NetClientState *nc) s = DO_UPCAST(VhostVDPAState, nc, nc); v = &s->vhost_vdpa; - v->shadow_data = s->always_svq; + s0 = vhost_vdpa_net_first_nc_vdpa(s); + v->shadow_data = s0->vhost_vdpa.shadow_vqs_enabled; v->shadow_vqs_enabled = s->always_svq; s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID; - if (s->always_svq) { + if (s->vhost_vdpa.shadow_data) { /* SVQ is already configured for all virtqueues */ goto out; } @@ -415,8 +533,6 @@ static int vhost_vdpa_net_cvq_start(NetClientState *nc) return r; } - v->iova_tree = vhost_iova_tree_new(v->iova_range.first, - v->iova_range.last); v->shadow_vqs_enabled = true; s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID; @@ -425,6 +541,26 @@ out: return 0; } + if (s0->vhost_vdpa.iova_tree) { + /* + * SVQ is already configured for all virtqueues. Reuse IOVA tree for + * simplicity, whether CVQ shares ASID with guest or not, because: + * - Memory listener need access to guest's memory addresses allocated + * in the IOVA tree. + * - There should be plenty of IOVA address space for both ASID not to + * worry about collisions between them. Guest's translations are + * still validated with virtio virtqueue_pop so there is no risk for + * the guest to access memory that it shouldn't. + * + * To allocate a iova tree per ASID is doable but it complicates the + * code and it is not worth it for the moment. + */ + v->iova_tree = s0->vhost_vdpa.iova_tree; + } else { + v->iova_tree = vhost_iova_tree_new(v->iova_range.first, + v->iova_range.last); + } + r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer, vhost_vdpa_net_cvq_cmd_page_len(), false); if (unlikely(r < 0)) { @@ -449,15 +585,9 @@ static void vhost_vdpa_net_cvq_stop(NetClientState *nc) if (s->vhost_vdpa.shadow_vqs_enabled) { vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer); vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status); - if (!s->always_svq) { - /* - * If only the CVQ is shadowed we can delete this safely. - * If all the VQs are shadows this will be needed by the time the - * device is started again to register SVQ vrings and similar. - */ - g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete); - } } + + vhost_vdpa_net_client_stop(nc); } static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s, size_t out_len, @@ -668,7 +798,7 @@ static NetClientState *net_vhost_vdpa_init(NetClientState *peer, bool is_datapath, bool svq, struct vhost_vdpa_iova_range iova_range, - VhostIOVATree *iova_tree) + uint64_t features) { NetClientState *nc = NULL; VhostVDPAState *s; @@ -687,11 +817,14 @@ static NetClientState *net_vhost_vdpa_init(NetClientState *peer, s->vhost_vdpa.device_fd = vdpa_device_fd; s->vhost_vdpa.index = queue_pair_index; s->always_svq = svq; + s->migration_state.notify = vdpa_net_migration_state_notifier; s->vhost_vdpa.shadow_vqs_enabled = svq; s->vhost_vdpa.iova_range = iova_range; s->vhost_vdpa.shadow_data = svq; - s->vhost_vdpa.iova_tree = iova_tree; - if (!is_datapath) { + if (queue_pair_index == 0) { + vhost_vdpa_net_valid_svq_features(features, + &s->vhost_vdpa.migration_blocker); + } else if (!is_datapath) { s->cvq_cmd_out_buffer = qemu_memalign(qemu_real_host_page_size(), vhost_vdpa_net_cvq_cmd_page_len()); memset(s->cvq_cmd_out_buffer, 0, vhost_vdpa_net_cvq_cmd_page_len()); @@ -701,6 +834,15 @@ static NetClientState *net_vhost_vdpa_init(NetClientState *peer, s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops; s->vhost_vdpa.shadow_vq_ops_opaque = s; + + /* + * TODO: We cannot migrate devices with CVQ as there is no way to set + * the device state (MAC, MQ, etc) before starting the datapath. + * + * Migration blocker ownership now belongs to s->vhost_vdpa. + */ + error_setg(&s->vhost_vdpa.migration_blocker, + "net vdpa cannot migrate with CVQ feature"); } ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs); if (ret) { @@ -760,7 +902,6 @@ int net_init_vhost_vdpa(const Netdev *netdev, const char *name, uint64_t features; int vdpa_device_fd; g_autofree NetClientState **ncs = NULL; - g_autoptr(VhostIOVATree) iova_tree = NULL; struct vhost_vdpa_iova_range iova_range; NetClientState *nc; int queue_pairs, r, i = 0, has_cvq = 0; @@ -812,12 +953,8 @@ int net_init_vhost_vdpa(const Netdev *netdev, const char *name, goto err; } - if (opts->x_svq) { - if (!vhost_vdpa_net_valid_svq_features(features, errp)) { - goto err_svq; - } - - iova_tree = vhost_iova_tree_new(iova_range.first, iova_range.last); + if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) { + goto err; } ncs = g_malloc0(sizeof(*ncs) * queue_pairs); @@ -825,7 +962,7 @@ int net_init_vhost_vdpa(const Netdev *netdev, const char *name, for (i = 0; i < queue_pairs; i++) { ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name, vdpa_device_fd, i, 2, true, opts->x_svq, - iova_range, iova_tree); + iova_range, features); if (!ncs[i]) goto err; } @@ -833,13 +970,11 @@ int net_init_vhost_vdpa(const Netdev *netdev, const char *name, if (has_cvq) { nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name, vdpa_device_fd, i, 1, false, - opts->x_svq, iova_range, iova_tree); + opts->x_svq, iova_range, features); if (!nc) goto err; } - /* iova_tree ownership belongs to last NetClientState */ - g_steal_pointer(&iova_tree); return 0; err: @@ -849,7 +984,6 @@ err: } } -err_svq: qemu_close(vdpa_device_fd); return -1; diff --git a/qapi/cryptodev.json b/qapi/cryptodev.json new file mode 100644 index 0000000000..f33f96a692 --- /dev/null +++ b/qapi/cryptodev.json @@ -0,0 +1,89 @@ +# -*- Mode: Python -*- +# vim: filetype=python +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. + +## +# @QCryptodevBackendAlgType: +# +# The supported algorithm types of a crypto device. +# +# @sym: symmetric encryption +# @asym: asymmetric Encryption +# +# Since: 8.0 +## +{ 'enum': 'QCryptodevBackendAlgType', + 'prefix': 'QCRYPTODEV_BACKEND_ALG', + 'data': ['sym', 'asym']} + +## +# @QCryptodevBackendServiceType: +# +# The supported service types of a crypto device. +# +# Since: 8.0 +## +{ 'enum': 'QCryptodevBackendServiceType', + 'prefix': 'QCRYPTODEV_BACKEND_SERVICE', + 'data': ['cipher', 'hash', 'mac', 'aead', 'akcipher']} + +## +# @QCryptodevBackendType: +# +# The crypto device backend type +# +# @builtin: the QEMU builtin support +# @vhost-user: vhost-user +# @lkcf: Linux kernel cryptographic framework +# +# Since: 8.0 +## +{ 'enum': 'QCryptodevBackendType', + 'prefix': 'QCRYPTODEV_BACKEND_TYPE', + 'data': ['builtin', 'vhost-user', 'lkcf']} + +## +# @QCryptodevBackendClient: +# +# Information about a queue of crypto device. +# +# @queue: the queue index of the crypto device +# +# @type: the type of the crypto device +# +# Since: 8.0 +## +{ 'struct': 'QCryptodevBackendClient', + 'data': { 'queue': 'uint32', + 'type': 'QCryptodevBackendType' } } + +## +# @QCryptodevInfo: +# +# Information about a crypto device. +# +# @id: the id of the crypto device +# +# @service: supported service types of a crypto device +# +# @client: the additional infomation of the crypto device +# +# Since: 8.0 +## +{ 'struct': 'QCryptodevInfo', + 'data': { 'id': 'str', + 'service': ['QCryptodevBackendServiceType'], + 'client': ['QCryptodevBackendClient'] } } + +## +# @query-cryptodev: +# +# Returns information about current crypto devices. +# +# Returns: a list of @QCryptodevInfo +# +# Since: 8.0 +## +{ 'command': 'query-cryptodev', 'returns': ['QCryptodevInfo']} diff --git a/qapi/cxl.json b/qapi/cxl.json new file mode 100644 index 0000000000..4be7d46041 --- /dev/null +++ b/qapi/cxl.json @@ -0,0 +1,128 @@ +# -*- Mode: Python -*- +# vim: filetype=python + +## +# = CXL devices +## + +## +# @CxlUncorErrorType: +# +# Type of uncorrectable CXL error to inject. These errors are reported via +# an AER uncorrectable internal error with additional information logged at +# the CXL device. +# +# @cache-data-parity: Data error such as data parity or data ECC error CXL.cache +# @cache-address-parity: Address parity or other errors associated with the +# address field on CXL.cache +# @cache-be-parity: Byte enable parity or other byte enable errors on CXL.cache +# @cache-data-ecc: ECC error on CXL.cache +# @mem-data-parity: Data error such as data parity or data ECC error on CXL.mem +# @mem-address-parity: Address parity or other errors associated with the +# address field on CXL.mem +# @mem-be-parity: Byte enable parity or other byte enable errors on CXL.mem. +# @mem-data-ecc: Data ECC error on CXL.mem. +# @reinit-threshold: REINIT threshold hit. +# @rsvd-encoding: Received unrecognized encoding. +# @poison-received: Received poison from the peer. +# @receiver-overflow: Buffer overflows (first 3 bits of header log indicate which) +# @internal: Component specific error +# @cxl-ide-tx: Integrity and data encryption tx error. +# @cxl-ide-rx: Integrity and data encryption rx error. +# +# Since: 8.0 +## + +{ 'enum': 'CxlUncorErrorType', + 'data': ['cache-data-parity', + 'cache-address-parity', + 'cache-be-parity', + 'cache-data-ecc', + 'mem-data-parity', + 'mem-address-parity', + 'mem-be-parity', + 'mem-data-ecc', + 'reinit-threshold', + 'rsvd-encoding', + 'poison-received', + 'receiver-overflow', + 'internal', + 'cxl-ide-tx', + 'cxl-ide-rx' + ] + } + +## +# @CXLUncorErrorRecord: +# +# Record of a single error including header log. +# +# @type: Type of error +# @header: 16 DWORD of header. +# +# Since: 8.0 +## +{ 'struct': 'CXLUncorErrorRecord', + 'data': { + 'type': 'CxlUncorErrorType', + 'header': [ 'uint32' ] + } +} + +## +# @cxl-inject-uncorrectable-errors: +# +# Command to allow injection of multiple errors in one go. This allows testing +# of multiple header log handling in the OS. +# +# @path: CXL Type 3 device canonical QOM path +# @errors: Errors to inject +# +# Since: 8.0 +## +{ 'command': 'cxl-inject-uncorrectable-errors', + 'data': { 'path': 'str', + 'errors': [ 'CXLUncorErrorRecord' ] }} + +## +# @CxlCorErrorType: +# +# Type of CXL correctable error to inject +# +# @cache-data-ecc: Data ECC error on CXL.cache +# @mem-data-ecc: Data ECC error on CXL.mem +# @crc-threshold: Component specific and applicable to 68 byte Flit mode only. +# @cache-poison-received: Received poison from a peer on CXL.cache. +# @mem-poison-received: Received poison from a peer on CXL.mem +# @physical: Received error indication from the physical layer. +# +# Since: 8.0 +## +{ 'enum': 'CxlCorErrorType', + 'data': ['cache-data-ecc', + 'mem-data-ecc', + 'crc-threshold', + 'retry-threshold', + 'cache-poison-received', + 'mem-poison-received', + 'physical'] +} + +## +# @cxl-inject-correctable-error: +# +# Command to inject a single correctable error. Multiple error injection +# of this error type is not interesting as there is no associated header log. +# These errors are reported via AER as a correctable internal error, with +# additional detail available from the CXL device. +# +# @path: CXL Type 3 device canonical QOM path +# @type: Type of error. +# +# Since: 8.0 +## +{ 'command': 'cxl-inject-correctable-error', + 'data': { 'path': 'str', + 'type': 'CxlCorErrorType' + } +} diff --git a/qapi/meson.build b/qapi/meson.build index fbdb442fdf..9fd480c4d8 100644 --- a/qapi/meson.build +++ b/qapi/meson.build @@ -31,6 +31,7 @@ qapi_all_modules = [ 'compat', 'control', 'crypto', + 'cxl', 'dump', 'error', 'introspect', @@ -56,6 +57,7 @@ if have_system qapi_all_modules += [ 'acpi', 'audio', + 'cryptodev', 'qdev', 'pci', 'rdma', diff --git a/qapi/qapi-schema.json b/qapi/qapi-schema.json index f000b90744..7c09af5cc8 100644 --- a/qapi/qapi-schema.json +++ b/qapi/qapi-schema.json @@ -95,3 +95,5 @@ { 'include': 'pci.json' } { 'include': 'stats.json' } { 'include': 'virtio.json' } +{ 'include': 'cryptodev.json' } +{ 'include': 'cxl.json' } diff --git a/qapi/qom.json b/qapi/qom.json index 30e76653ad..a877b879b9 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -278,10 +278,16 @@ # cryptodev-backend and must be 1 for cryptodev-backend-builtin. # (default: 1) # +# @throttle-bps: limit total bytes per second (Since 8.0) +# +# @throttle-ops: limit total operations per second (Since 8.0) +# # Since: 2.8 ## { 'struct': 'CryptodevBackendProperties', - 'data': { '*queues': 'uint32' } } + 'data': { '*queues': 'uint32', + '*throttle-bps': 'uint64', + '*throttle-ops': 'uint64' } } ## # @CryptodevVhostUserProperties: diff --git a/qapi/stats.json b/qapi/stats.json index 57db5b1c74..1f5d3c59ab 100644 --- a/qapi/stats.json +++ b/qapi/stats.json @@ -50,10 +50,14 @@ # # Enumeration of statistics providers. # +# @kvm: since 7.1 +# +# @cryptodev: since 8.0 +# # Since: 7.1 ## { 'enum': 'StatsProvider', - 'data': [ 'kvm' ] } + 'data': [ 'kvm', 'cryptodev' ] } ## # @StatsTarget: @@ -65,10 +69,12 @@ # # @vcpu: statistics that apply to a single virtual CPU. # +# @cryptodev: statistics that apply to a crypto device. since 8.0 +# # Since: 7.1 ## { 'enum': 'StatsTarget', - 'data': [ 'vm', 'vcpu' ] } + 'data': [ 'vm', 'vcpu', 'cryptodev' ] } ## # @StatsRequest: diff --git a/qga/installer/qemu-ga.wxs b/qga/installer/qemu-ga.wxs index 51340f7ecc..df572adb4a 100644 --- a/qga/installer/qemu-ga.wxs +++ b/qga/installer/qemu-ga.wxs @@ -31,6 +31,7 @@ /> <Media Id="1" Cabinet="qemu_ga.$(var.QEMU_GA_VERSION).cab" EmbedCab="yes" /> <Property Id="WHSLogo">1</Property> + <Property Id="ARPNOMODIFY" Value="yes" Secure="yes" /> <MajorUpgrade DowngradeErrorMessage="Error: A newer version of QEMU guest agent is already installed." /> @@ -121,27 +122,31 @@ <RegistryValue Type="integer" Name="TypesSupported" Value="7" /> <RegistryValue Type="string" Name="EventMessageFile" Value="[qemu_ga_directory]qemu-ga.exe" /> </RegistryKey> + <RegistryKey Root="HKLM" + Key="System\CurrentControlSet\Services\QEMU Guest Agent VSS Provider"> + <RegistryValue Type="integer" Name="VssOption" Value="1" /> + </RegistryKey> </Component> </Directory> </Directory> </Directory> - <Property Id="cmd" Value="cmd.exe"/> + <Property Id="rundll" Value="rundll32.exe"/> <Property Id="REINSTALLMODE" Value="amus"/> <?ifdef var.InstallVss?> <CustomAction Id="RegisterCom" - ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-install' + ExeCommand='"[qemu_ga_directory]qga-vss.dll",DLLCOMRegister' Execute="deferred" - Property="cmd" + Property="rundll" Impersonate="no" Return="check" > </CustomAction> <CustomAction Id="UnRegisterCom" - ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-uninstall' + ExeCommand='"[qemu_ga_directory]qga-vss.dll",DLLCOMUnregister' Execute="deferred" - Property="cmd" + Property="rundll" Impersonate="no" Return="check" > diff --git a/qga/vss-win32/install.cpp b/qga/vss-win32/install.cpp index b57508fbe0..68662a6dfc 100644 --- a/qga/vss-win32/install.cpp +++ b/qga/vss-win32/install.cpp @@ -357,6 +357,15 @@ out: return hr; } +STDAPI_(void) CALLBACK DLLCOMRegister(HWND, HINSTANCE, LPSTR, int) +{ + COMRegister(); +} + +STDAPI_(void) CALLBACK DLLCOMUnregister(HWND, HINSTANCE, LPSTR, int) +{ + COMUnregister(); +} static BOOL CreateRegistryKey(LPCTSTR key, LPCTSTR value, LPCTSTR data) { diff --git a/qga/vss-win32/qga-vss.def b/qga/vss-win32/qga-vss.def index 927782c31b..ee97a81427 100644 --- a/qga/vss-win32/qga-vss.def +++ b/qga/vss-win32/qga-vss.def @@ -1,6 +1,8 @@ LIBRARY "QGA-PROVIDER.DLL" EXPORTS + DLLCOMRegister + DLLCOMUnregister COMRegister PRIVATE COMUnregister PRIVATE DllCanUnloadNow PRIVATE diff --git a/qga/vss-win32/requester.cpp b/qga/vss-win32/requester.cpp index b371affeab..3e998af4a8 100644 --- a/qga/vss-win32/requester.cpp +++ b/qga/vss-win32/requester.cpp @@ -23,6 +23,8 @@ /* Call QueryStatus every 10 ms while waiting for frozen event */ #define VSS_TIMEOUT_EVENT_MSEC 10 +#define DEFAULT_VSS_BACKUP_TYPE VSS_BT_FULL + #define err_set(e, err, fmt, ...) \ ((e)->error_setg_win32_wrapper((e)->errp, __FILE__, __LINE__, __func__, \ err, fmt, ## __VA_ARGS__)) @@ -234,6 +236,42 @@ out: } } +DWORD get_reg_dword_value(HKEY baseKey, LPCSTR subKey, LPCSTR valueName, + DWORD defaultData) +{ + DWORD regGetValueError; + DWORD dwordData; + DWORD dataSize = sizeof(DWORD); + + regGetValueError = RegGetValue(baseKey, subKey, valueName, RRF_RT_DWORD, + NULL, &dwordData, &dataSize); + if (regGetValueError != ERROR_SUCCESS) { + return defaultData; + } + return dwordData; +} + +bool is_valid_vss_backup_type(VSS_BACKUP_TYPE vssBT) +{ + return (vssBT > VSS_BT_UNDEFINED && vssBT < VSS_BT_OTHER); +} + +VSS_BACKUP_TYPE get_vss_backup_type( + VSS_BACKUP_TYPE defaultVssBT = DEFAULT_VSS_BACKUP_TYPE) +{ + VSS_BACKUP_TYPE vssBackupType; + + vssBackupType = static_cast<VSS_BACKUP_TYPE>( + get_reg_dword_value(HKEY_LOCAL_MACHINE, + QGA_PROVIDER_REGISTRY_ADDRESS, + "VssOption", + defaultVssBT)); + if (!is_valid_vss_backup_type(vssBackupType)) { + return defaultVssBT; + } + return vssBackupType; +} + void requester_freeze(int *num_vols, void *mountpoints, ErrorSet *errset) { COMPointer<IVssAsync> pAsync; @@ -247,6 +285,7 @@ void requester_freeze(int *num_vols, void *mountpoints, ErrorSet *errset) DWORD wait_status; int num_fixed_drives = 0, i; int num_mount_points = 0; + VSS_BACKUP_TYPE vss_bt = get_vss_backup_type(); if (vss_ctx.pVssbc) { /* already frozen */ *num_vols = 0; @@ -294,7 +333,7 @@ void requester_freeze(int *num_vols, void *mountpoints, ErrorSet *errset) goto out; } - hr = vss_ctx.pVssbc->SetBackupState(true, true, VSS_BT_FULL, false); + hr = vss_ctx.pVssbc->SetBackupState(true, true, vss_bt, false); if (FAILED(hr)) { err_set(errset, hr, "failed to set backup state"); goto out; diff --git a/qga/vss-win32/vss-handles.h b/qga/vss-win32/vss-handles.h index 0f8a741ad2..1a7d842129 100644 --- a/qga/vss-win32/vss-handles.h +++ b/qga/vss-win32/vss-handles.h @@ -6,6 +6,9 @@ #define QGA_PROVIDER_NAME "QEMU Guest Agent VSS Provider" #define QGA_PROVIDER_LNAME L(QGA_PROVIDER_NAME) #define QGA_PROVIDER_VERSION L(QEMU_VERSION) +#define QGA_PROVIDER_REGISTRY_ADDRESS "SYSTEM\\CurrentControlSet"\ + "\\Services"\ + "\\" QGA_PROVIDER_NAME #define EVENT_NAME_FROZEN "Global\\QGAVSSEvent-frozen" #define EVENT_NAME_THAW "Global\\QGAVSSEvent-thaw" diff --git a/scripts/ci/org.centos/stream/8/x86_64/test-avocado b/scripts/ci/org.centos/stream/8/x86_64/test-avocado index f403e4e7ec..d2c0e5fb4c 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/test-avocado +++ b/scripts/ci/org.centos/stream/8/x86_64/test-avocado @@ -30,6 +30,7 @@ make get-vm-images tests/avocado/cpu_queries.py:QueryCPUModelExpansion.test \ tests/avocado/empty_cpu_model.py:EmptyCPUModel.test \ tests/avocado/hotplug_cpu.py:HotPlugCPU.test \ + tests/avocado/igb.py:IGB.test \ tests/avocado/info_usernet.py:InfoUsernet.test_hostfwd \ tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu \ tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu_pt \ diff --git a/stats/stats-hmp-cmds.c b/stats/stats-hmp-cmds.c index 531e35d128..1f91bf8bd5 100644 --- a/stats/stats-hmp-cmds.c +++ b/stats/stats-hmp-cmds.c @@ -155,6 +155,8 @@ static StatsFilter *stats_filter(StatsTarget target, const char *names, filter->u.vcpu.vcpus = vcpu_list; break; } + case STATS_TARGET_CRYPTODEV: + break; default: break; } @@ -226,6 +228,9 @@ void hmp_info_stats(Monitor *mon, const QDict *qdict) int cpu_index = monitor_get_cpu_index(mon); filter = stats_filter(target, names, cpu_index, provider); break; + case STATS_TARGET_CRYPTODEV: + filter = stats_filter(target, names, -1, provider); + break; default: abort(); } diff --git a/stats/stats-qmp-cmds.c b/stats/stats-qmp-cmds.c index bc973747fb..e214b964fd 100644 --- a/stats/stats-qmp-cmds.c +++ b/stats/stats-qmp-cmds.c @@ -64,6 +64,8 @@ static bool invoke_stats_cb(StatsCallbacks *entry, targets = filter->u.vcpu.vcpus; } break; + case STATS_TARGET_CRYPTODEV: + break; default: abort(); } diff --git a/target/hexagon/README b/target/hexagon/README index 251960b862..ebafc78b1c 100644 --- a/target/hexagon/README +++ b/target/hexagon/README @@ -52,6 +52,7 @@ header files in <BUILD_DIR>/target/hexagon gen_tcg_func_table.py -> tcg_func_table_generated.c.inc gen_helper_funcs.py -> helper_funcs_generated.c.inc gen_idef_parser_funcs.py -> idef_parser_input.h + gen_analyze_funcs.py -> analyze_funcs_generated.c.inc Qemu helper functions have 3 parts DEF_HELPER declaration indicates the signature of the helper @@ -87,7 +88,6 @@ tcg_funcs_generated.c.inc TCGv RtV = hex_gpr[insn->regno[2]]; gen_helper_A2_add(RdV, cpu_env, RsV, RtV); gen_log_reg_write(RdN, RdV); - ctx_log_reg_write(ctx, RdN); } helper_funcs_generated.c.inc @@ -136,12 +136,9 @@ For HVX vectors, the generator behaves slightly differently. The wide vectors won't fit in a TCGv or TCGv_i64, so we pass TCGv_ptr variables to pass the address to helper functions. Here's an example for an HVX vector-add-word istruction. - static void generate_V6_vaddw( - CPUHexagonState *env, - DisasContext *ctx, - Insn *insn, - Packet *pkt) + static void generate_V6_vaddw(DisasContext *ctx) { + Insn *insn __attribute__((unused)) = ctx->insn; const int VdN = insn->regno[0]; const intptr_t VdV_off = ctx_future_vreg_off(ctx, VdN, 1, true); @@ -157,10 +154,7 @@ istruction. TCGv_ptr VvV = tcg_temp_new_ptr(); tcg_gen_addi_ptr(VuV, cpu_env, VuV_off); tcg_gen_addi_ptr(VvV, cpu_env, VvV_off); - TCGv slot = tcg_constant_tl(insn->slot); - gen_helper_V6_vaddw(cpu_env, VdV, VuV, VvV, slot); - gen_log_vreg_write(ctx, VdV_off, VdN, EXT_DFL, insn->slot, false); - ctx_log_vreg_write(ctx, VdN, EXT_DFL, false); + gen_helper_V6_vaddw(cpu_env, VdV, VuV, VvV); } Notice that we also generate a variable named <operand>_off for each operand of @@ -173,12 +167,9 @@ functions from tcg-op-gvec.h. Here's the override for this instruction. Finally, we notice that the override doesn't use the TCGv_ptr variables, so we don't generate them when an override is present. Here is what we generate when the override is present. - static void generate_V6_vaddw( - CPUHexagonState *env, - DisasContext *ctx, - Insn *insn, - Packet *pkt) + static void generate_V6_vaddw(DisasContext *ctx) { + Insn *insn __attribute__((unused)) = ctx->insn; const int VdN = insn->regno[0]; const intptr_t VdV_off = ctx_future_vreg_off(ctx, VdN, 1, true); @@ -189,10 +180,14 @@ when the override is present. const intptr_t VvV_off = vreg_src_off(ctx, VvN); fGEN_TCG_V6_vaddw({ fHIDE(int i;) fVFOREACH(32, i) { VdV.w[i] = VuV.w[i] + VvV.w[i] ; } }); - gen_log_vreg_write(ctx, VdV_off, VdN, EXT_DFL, insn->slot, false); - ctx_log_vreg_write(ctx, VdN, EXT_DFL, false); } +We also generate an analyze_<tag> function for each instruction. Currently, +these functions record the writes to registers by calling ctx_log_*. During +gen_start_packet, we invoke the analyze_<tag> function for each instruction in +the packet, and we mark the implicit writes. After the analysis is performed, +we initialize hex_new_value for each of the predicated assignments. + In addition to instruction semantics, we use a generator to create the decode tree. This generation is also a two step process. The first step is to run target/hexagon/gen_dectree_import.c to produce @@ -277,10 +272,8 @@ For Hexagon Vector eXtensions (HVX), the following fields are used VRegs Vector registers future_VRegs Registers to be stored during packet commit tmp_VRegs Temporary registers *not* stored during commit - VRegs_updated Mask of predicated vector writes QRegs Q (vector predicate) registers future_QRegs Registers to be stored during packet commit - QRegs_updated Mask of predicated vector writes *** Debugging *** diff --git a/target/hexagon/attribs_def.h.inc b/target/hexagon/attribs_def.h.inc index 5d2a102c18..9874d1658f 100644 --- a/target/hexagon/attribs_def.h.inc +++ b/target/hexagon/attribs_def.h.inc @@ -44,6 +44,7 @@ DEF_ATTRIB(MEMSIZE_1B, "Memory width is 1 byte", "", "") DEF_ATTRIB(MEMSIZE_2B, "Memory width is 2 bytes", "", "") DEF_ATTRIB(MEMSIZE_4B, "Memory width is 4 bytes", "", "") DEF_ATTRIB(MEMSIZE_8B, "Memory width is 8 bytes", "", "") +DEF_ATTRIB(SCALAR_LOAD, "Load is scalar", "", "") DEF_ATTRIB(SCALAR_STORE, "Store is scalar", "", "") DEF_ATTRIB(REGWRSIZE_1B, "Memory width is 1 byte", "", "") DEF_ATTRIB(REGWRSIZE_2B, "Memory width is 2 bytes", "", "") diff --git a/target/hexagon/cpu.h b/target/hexagon/cpu.h index 34c0ae0a67..81b663ecfb 100644 --- a/target/hexagon/cpu.h +++ b/target/hexagon/cpu.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -111,11 +111,8 @@ typedef struct CPUArchState { MMVector future_VRegs[VECTOR_TEMPS_MAX] QEMU_ALIGNED(16); MMVector tmp_VRegs[VECTOR_TEMPS_MAX] QEMU_ALIGNED(16); - VRegMask VRegs_updated; - MMQReg QRegs[NUM_QREGS] QEMU_ALIGNED(16); MMQReg future_QRegs[NUM_QREGS] QEMU_ALIGNED(16); - QRegMask QRegs_updated; /* Temporaries used within instructions */ MMVectorPair VuuV QEMU_ALIGNED(16); diff --git a/target/hexagon/gen_analyze_funcs.py b/target/hexagon/gen_analyze_funcs.py new file mode 100755 index 0000000000..ebd3e7afb9 --- /dev/null +++ b/target/hexagon/gen_analyze_funcs.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 + +## +## Copyright(c) 2022-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 2 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, see <http://www.gnu.org/licenses/>. +## + +import sys +import re +import string +import hex_common + +## +## Helpers for gen_analyze_func +## +def is_predicated(tag): + return 'A_CONDEXEC' in hex_common.attribdict[tag] + +def analyze_opn_old(f, tag, regtype, regid, regno): + regN = "%s%sN" % (regtype, regid) + predicated = "true" if is_predicated(tag) else "false" + if (regtype == "R"): + if (regid in {"ss", "tt"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"dd", "ee", "xx", "yy"}): + f.write(" const int %s = insn->regno[%d];\n" % (regN, regno)) + f.write(" ctx_log_reg_write_pair(ctx, %s, %s);\n" % \ + (regN, predicated)) + elif (regid in {"s", "t", "u", "v"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"d", "e", "x", "y"}): + f.write(" const int %s = insn->regno[%d];\n" % (regN, regno)) + f.write(" ctx_log_reg_write(ctx, %s, %s);\n" % \ + (regN, predicated)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "P"): + if (regid in {"s", "t", "u", "v"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"d", "e", "x"}): + f.write(" const int %s = insn->regno[%d];\n" % (regN, regno)) + f.write(" ctx_log_pred_write(ctx, %s);\n" % (regN)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "C"): + if (regid == "ss"): + f.write("// const int %s = insn->regno[%d] + HEX_REG_SA0;\n" % \ + (regN, regno)) + elif (regid == "dd"): + f.write(" const int %s = insn->regno[%d] + HEX_REG_SA0;\n" % \ + (regN, regno)) + f.write(" ctx_log_reg_write_pair(ctx, %s, %s);\n" % \ + (regN, predicated)) + elif (regid == "s"): + f.write("// const int %s = insn->regno[%d] + HEX_REG_SA0;\n" % \ + (regN, regno)) + elif (regid == "d"): + f.write(" const int %s = insn->regno[%d] + HEX_REG_SA0;\n" % \ + (regN, regno)) + f.write(" ctx_log_reg_write(ctx, %s, %s);\n" % \ + (regN, predicated)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "M"): + if (regid == "u"): + f.write("// const int %s = insn->regno[%d];\n"% \ + (regN, regno)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "V"): + newv = "EXT_DFL" + if (hex_common.is_new_result(tag)): + newv = "EXT_NEW" + elif (hex_common.is_tmp_result(tag)): + newv = "EXT_TMP" + if (regid in {"dd", "xx"}): + f.write(" const int %s = insn->regno[%d];\n" %\ + (regN, regno)) + f.write(" ctx_log_vreg_write_pair(ctx, %s, %s, %s);\n" % \ + (regN, newv, predicated)) + elif (regid in {"uu", "vv"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"s", "u", "v", "w"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"d", "x", "y"}): + f.write(" const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + f.write(" ctx_log_vreg_write(ctx, %s, %s, %s);\n" % \ + (regN, newv, predicated)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "Q"): + if (regid in {"d", "e", "x"}): + f.write(" const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + f.write(" ctx_log_qreg_write(ctx, %s);\n" % (regN)) + elif (regid in {"s", "t", "u", "v"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "G"): + if (regid in {"dd"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"d"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"ss"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"s"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "S"): + if (regid in {"dd"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"d"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"ss"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + elif (regid in {"s"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + else: + print("Bad register parse: ", regtype, regid) + else: + print("Bad register parse: ", regtype, regid) + +def analyze_opn_new(f, tag, regtype, regid, regno): + regN = "%s%sN" % (regtype, regid) + if (regtype == "N"): + if (regid in {"s", "t"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "P"): + if (regid in {"t", "u", "v"}): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + else: + print("Bad register parse: ", regtype, regid) + elif (regtype == "O"): + if (regid == "s"): + f.write("// const int %s = insn->regno[%d];\n" % \ + (regN, regno)) + else: + print("Bad register parse: ", regtype, regid) + else: + print("Bad register parse: ", regtype, regid) + +def analyze_opn(f, tag, regtype, regid, toss, numregs, i): + if (hex_common.is_pair(regid)): + analyze_opn_old(f, tag, regtype, regid, i) + elif (hex_common.is_single(regid)): + if hex_common.is_old_val(regtype, regid, tag): + analyze_opn_old(f,tag, regtype, regid, i) + elif hex_common.is_new_val(regtype, regid, tag): + analyze_opn_new(f, tag, regtype, regid, i) + else: + print("Bad register parse: ", regtype, regid, toss, numregs) + else: + print("Bad register parse: ", regtype, regid, toss, numregs) + +## +## Generate the code to analyze the instruction +## For A2_add: Rd32=add(Rs32,Rt32), { RdV=RsV+RtV;} +## We produce: +## static void analyze_A2_add(DisasContext *ctx) +## { +## Insn *insn G_GNUC_UNUSED = ctx->insn; +## const int RdN = insn->regno[0]; +## ctx_log_reg_write(ctx, RdN, false); +## // const int RsN = insn->regno[1]; +## // const int RtN = insn->regno[2]; +## } +## +def gen_analyze_func(f, tag, regs, imms): + f.write("static void analyze_%s(DisasContext *ctx)\n" %tag) + f.write('{\n') + + f.write(" Insn *insn G_GNUC_UNUSED = ctx->insn;\n") + + i=0 + ## Analyze all the registers + for regtype, regid, toss, numregs in regs: + analyze_opn(f, tag, regtype, regid, toss, numregs, i) + i += 1 + + has_generated_helper = (not hex_common.skip_qemu_helper(tag) and + not hex_common.is_idef_parser_enabled(tag)) + if (has_generated_helper and + 'A_SCALAR_LOAD' in hex_common.attribdict[tag]): + f.write(" ctx->need_pkt_has_store_s1 = true;\n") + + f.write("}\n\n") + +def main(): + hex_common.read_semantics_file(sys.argv[1]) + hex_common.read_attribs_file(sys.argv[2]) + hex_common.read_overrides_file(sys.argv[3]) + hex_common.read_overrides_file(sys.argv[4]) + ## Whether or not idef-parser is enabled is + ## determined by the number of arguments to + ## this script: + ## + ## 5 args. -> not enabled, + ## 6 args. -> idef-parser enabled. + ## + ## The 6:th arg. then holds a list of the successfully + ## parsed instructions. + is_idef_parser_enabled = len(sys.argv) > 6 + if is_idef_parser_enabled: + hex_common.read_idef_parser_enabled_file(sys.argv[5]) + hex_common.calculate_attribs() + tagregs = hex_common.get_tagregs() + tagimms = hex_common.get_tagimms() + + with open(sys.argv[-1], 'w') as f: + f.write("#ifndef HEXAGON_TCG_FUNCS_H\n") + f.write("#define HEXAGON_TCG_FUNCS_H\n\n") + + for tag in hex_common.tags: + gen_analyze_func(f, tag, tagregs[tag], tagimms[tag]) + + f.write("#endif /* HEXAGON_TCG_FUNCS_H */\n") + +if __name__ == "__main__": + main() diff --git a/target/hexagon/gen_helper_funcs.py b/target/hexagon/gen_helper_funcs.py index 19e9883f4c..7a224b66e6 100755 --- a/target/hexagon/gen_helper_funcs.py +++ b/target/hexagon/gen_helper_funcs.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ## -## Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. +## Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. ## ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by @@ -226,6 +226,14 @@ def gen_helper_function(f, tag, tagregs, tagimms): print("Bad register parse: ",regtype,regid,toss,numregs) i += 1 + ## For conditional instructions, we pass in the destination register + if 'A_CONDEXEC' in hex_common.attribdict[tag]: + for regtype, regid, toss, numregs in regs: + if (hex_common.is_writeonly(regid) and + not hex_common.is_hvx_reg(regtype)): + gen_helper_arg_opn(f, regtype, regid, i, tag) + i += 1 + ## Arguments to the helper function are the source regs and immediates for regtype,regid,toss,numregs in regs: if (hex_common.is_read(regid)): @@ -262,10 +270,11 @@ def gen_helper_function(f, tag, tagregs, tagimms): if hex_common.need_ea(tag): gen_decl_ea(f) ## Declare the return variable i=0 - for regtype,regid,toss,numregs in regs: - if (hex_common.is_writeonly(regid)): - gen_helper_dest_decl_opn(f,regtype,regid,i) - i += 1 + if 'A_CONDEXEC' not in hex_common.attribdict[tag]: + for regtype,regid,toss,numregs in regs: + if (hex_common.is_writeonly(regid)): + gen_helper_dest_decl_opn(f,regtype,regid,i) + i += 1 for regtype,regid,toss,numregs in regs: if (hex_common.is_read(regid)): diff --git a/target/hexagon/gen_helper_protos.py b/target/hexagon/gen_helper_protos.py index 674bf370fa..ddddc9e4f0 100755 --- a/target/hexagon/gen_helper_protos.py +++ b/target/hexagon/gen_helper_protos.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ## -## Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. +## Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. ## ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by @@ -87,6 +87,7 @@ def gen_helper_prototype(f, tag, tagregs, tagimms): if hex_common.need_slot(tag): def_helper_size += 1 if hex_common.need_PC(tag): def_helper_size += 1 if hex_common.helper_needs_next_PC(tag): def_helper_size += 1 + if hex_common.need_condexec_reg(tag, regs): def_helper_size += 1 f.write('DEF_HELPER_%s(%s' % (def_helper_size, tag)) ## The return type is void f.write(', void' ) @@ -96,6 +97,7 @@ def gen_helper_prototype(f, tag, tagregs, tagimms): if hex_common.need_part1(tag): def_helper_size += 1 if hex_common.need_slot(tag): def_helper_size += 1 if hex_common.need_PC(tag): def_helper_size += 1 + if hex_common.need_condexec_reg(tag, regs): def_helper_size += 1 if hex_common.helper_needs_next_PC(tag): def_helper_size += 1 f.write('DEF_HELPER_%s(%s' % (def_helper_size, tag)) @@ -121,6 +123,14 @@ def gen_helper_prototype(f, tag, tagregs, tagimms): gen_def_helper_opn(f, tag, regtype, regid, toss, numregs, i) i += 1 + ## For conditional instructions, we pass in the destination register + if 'A_CONDEXEC' in hex_common.attribdict[tag]: + for regtype, regid, toss, numregs in regs: + if (hex_common.is_writeonly(regid) and + not hex_common.is_hvx_reg(regtype)): + gen_def_helper_opn(f, tag, regtype, regid, toss, numregs, i) + i += 1 + ## Generate the qemu type for each input operand (regs and immediates) for regtype,regid,toss,numregs in regs: if (hex_common.is_read(regid)): diff --git a/target/hexagon/gen_tcg.h b/target/hexagon/gen_tcg.h index b2e7880b5c..bcf0cf466a 100644 --- a/target/hexagon/gen_tcg.h +++ b/target/hexagon/gen_tcg.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -332,8 +332,6 @@ tcg_gen_movi_tl(EA, 0); \ PRED; \ CHECK_NOSHUF_PRED(GET_EA, SIZE, LSB); \ - PRED_LOAD_CANCEL(LSB, EA); \ - tcg_gen_movi_tl(RdV, 0); \ tcg_gen_brcondi_tl(TCG_COND_EQ, LSB, 0, label); \ fLOAD(1, SIZE, SIGN, EA, RdV); \ gen_set_label(label); \ @@ -391,8 +389,6 @@ tcg_gen_movi_tl(EA, 0); \ PRED; \ CHECK_NOSHUF_PRED(GET_EA, 8, LSB); \ - PRED_LOAD_CANCEL(LSB, EA); \ - tcg_gen_movi_i64(RddV, 0); \ tcg_gen_brcondi_tl(TCG_COND_EQ, LSB, 0, label); \ fLOAD(1, 8, u, EA, RddV); \ gen_set_label(label); \ @@ -419,16 +415,16 @@ #define fGEN_TCG_STORE(SHORTCODE) \ do { \ - TCGv HALF = tcg_temp_new(); \ - TCGv BYTE = tcg_temp_new(); \ + TCGv HALF G_GNUC_UNUSED = tcg_temp_new(); \ + TCGv BYTE G_GNUC_UNUSED = tcg_temp_new(); \ SHORTCODE; \ } while (0) #define fGEN_TCG_STORE_pcr(SHIFT, STORE) \ do { \ TCGv ireg = tcg_temp_new(); \ - TCGv HALF = tcg_temp_new(); \ - TCGv BYTE = tcg_temp_new(); \ + TCGv HALF G_GNUC_UNUSED = tcg_temp_new(); \ + TCGv BYTE G_GNUC_UNUSED = tcg_temp_new(); \ tcg_gen_mov_tl(EA, RxV); \ gen_read_ireg(ireg, MuV, SHIFT); \ gen_helper_fcircadd(RxV, RxV, ireg, MuV, hex_gpr[HEX_REG_CS0 + MuN]); \ @@ -492,6 +488,59 @@ fGEN_TCG_STORE_pcr(2, fSTORE(1, 4, EA, NtN)) /* + * dealloc_return + * Assembler mapped to + * r31:30 = dealloc_return(r30):raw + */ +#define fGEN_TCG_L4_return(SHORTCODE) \ + gen_return(ctx, RddV, RsV) + +/* + * sub-instruction version (no RddV, so handle it manually) + */ +#define fGEN_TCG_SL2_return(SHORTCODE) \ + do { \ + TCGv_i64 RddV = get_result_gpr_pair(ctx, HEX_REG_FP); \ + gen_return(ctx, RddV, hex_gpr[HEX_REG_FP]); \ + gen_log_reg_write_pair(HEX_REG_FP, RddV); \ + } while (0) + +/* + * Conditional returns follow this naming convention + * _t predicate true + * _f predicate false + * _tnew_pt predicate.new true predict taken + * _fnew_pt predicate.new false predict taken + * _tnew_pnt predicate.new true predict not taken + * _fnew_pnt predicate.new false predict not taken + * Predictions are not modelled in QEMU + * + * Example: + * if (p1) r31:30 = dealloc_return(r30):raw + */ +#define fGEN_TCG_L4_return_t(SHORTCODE) \ + gen_cond_return(ctx, RddV, RsV, PvV, TCG_COND_EQ); +#define fGEN_TCG_L4_return_f(SHORTCODE) \ + gen_cond_return(ctx, RddV, RsV, PvV, TCG_COND_NE) +#define fGEN_TCG_L4_return_tnew_pt(SHORTCODE) \ + gen_cond_return(ctx, RddV, RsV, PvN, TCG_COND_EQ) +#define fGEN_TCG_L4_return_fnew_pt(SHORTCODE) \ + gen_cond_return(ctx, RddV, RsV, PvN, TCG_COND_NE) +#define fGEN_TCG_L4_return_tnew_pnt(SHORTCODE) \ + gen_cond_return(ctx, RddV, RsV, PvN, TCG_COND_EQ) +#define fGEN_TCG_L4_return_fnew_pnt(SHORTCODE) \ + gen_cond_return(ctx, RddV, RsV, PvN, TCG_COND_NE) + +#define fGEN_TCG_SL2_return_t(SHORTCODE) \ + gen_cond_return_subinsn(ctx, TCG_COND_EQ, hex_pred[0]) +#define fGEN_TCG_SL2_return_f(SHORTCODE) \ + gen_cond_return_subinsn(ctx, TCG_COND_NE, hex_pred[0]) +#define fGEN_TCG_SL2_return_tnew(SHORTCODE) \ + gen_cond_return_subinsn(ctx, TCG_COND_EQ, hex_new_pred_value[0]) +#define fGEN_TCG_SL2_return_fnew(SHORTCODE) \ + gen_cond_return_subinsn(ctx, TCG_COND_NE, hex_new_pred_value[0]) + +/* * Mathematical operations with more than one definition require * special handling */ @@ -589,14 +638,24 @@ #define fGEN_TCG_J2_call(SHORTCODE) \ gen_call(ctx, riV) +#define fGEN_TCG_J2_callr(SHORTCODE) \ + gen_callr(ctx, RsV) #define fGEN_TCG_J2_callt(SHORTCODE) \ gen_cond_call(ctx, PuV, TCG_COND_EQ, riV) #define fGEN_TCG_J2_callf(SHORTCODE) \ gen_cond_call(ctx, PuV, TCG_COND_NE, riV) +#define fGEN_TCG_J2_callrt(SHORTCODE) \ + gen_cond_callr(ctx, TCG_COND_EQ, PuV, RsV) +#define fGEN_TCG_J2_callrf(SHORTCODE) \ + gen_cond_callr(ctx, TCG_COND_NE, PuV, RsV) #define fGEN_TCG_J2_endloop0(SHORTCODE) \ gen_endloop0(ctx) +#define fGEN_TCG_J2_endloop1(SHORTCODE) \ + gen_endloop1(ctx) +#define fGEN_TCG_J2_endloop01(SHORTCODE) \ + gen_endloop01(ctx) /* * Compound compare and jump instructions @@ -986,6 +1045,19 @@ #define fGEN_TCG_S2_asl_r_r_sat(SHORTCODE) \ gen_asl_r_r_sat(RdV, RsV, RtV) +#define fGEN_TCG_SL2_jumpr31(SHORTCODE) \ + gen_jumpr(ctx, hex_gpr[HEX_REG_LR]) + +#define fGEN_TCG_SL2_jumpr31_t(SHORTCODE) \ + gen_cond_jumpr31(ctx, TCG_COND_EQ, hex_pred[0]) +#define fGEN_TCG_SL2_jumpr31_f(SHORTCODE) \ + gen_cond_jumpr31(ctx, TCG_COND_NE, hex_pred[0]) + +#define fGEN_TCG_SL2_jumpr31_tnew(SHORTCODE) \ + gen_cond_jumpr31(ctx, TCG_COND_EQ, hex_new_pred_value[0]) +#define fGEN_TCG_SL2_jumpr31_fnew(SHORTCODE) \ + gen_cond_jumpr31(ctx, TCG_COND_NE, hex_new_pred_value[0]) + /* Floating point */ #define fGEN_TCG_F2_conv_sf2df(SHORTCODE) \ gen_helper_conv_sf2df(RddV, cpu_env, RsV) diff --git a/target/hexagon/gen_tcg_funcs.py b/target/hexagon/gen_tcg_funcs.py index 02cb52c21e..fa93e185ce 100755 --- a/target/hexagon/gen_tcg_funcs.py +++ b/target/hexagon/gen_tcg_funcs.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ## -## Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. +## Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. ## ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by @@ -30,37 +30,33 @@ def gen_decl_ea_tcg(f, tag): def genptr_decl_pair_writable(f, tag, regtype, regid, regno): regN="%s%sN" % (regtype,regid) - f.write(" TCGv_i64 %s%sV = tcg_temp_new_i64();\n" % \ - (regtype, regid)) - if (regtype == "C"): + if (regtype == "R"): + f.write(" const int %s = insn->regno[%d];\n" % (regN, regno)) + elif (regtype == "C"): f.write(" const int %s = insn->regno[%d] + HEX_REG_SA0;\n" % \ (regN, regno)) else: - f.write(" const int %s = insn->regno[%d];\n" % (regN, regno)) - if ('A_CONDEXEC' in hex_common.attribdict[tag]): - f.write(" if (!is_preloaded(ctx, %s)) {\n" % regN) - f.write(" tcg_gen_mov_tl(hex_new_value[%s], hex_gpr[%s]);\n" % \ - (regN, regN)) - f.write(" }\n") - f.write(" if (!is_preloaded(ctx, %s + 1)) {\n" % regN) - f.write(" tcg_gen_mov_tl(hex_new_value[%s + 1], hex_gpr[%s + 1]);\n" % \ - (regN, regN)) - f.write(" }\n") + print("Bad register parse: ", regtype, regid) + f.write(" TCGv_i64 %s%sV = get_result_gpr_pair(ctx, %s);\n" % \ + (regtype, regid, regN)) def genptr_decl_writable(f, tag, regtype, regid, regno): regN="%s%sN" % (regtype,regid) - f.write(" TCGv %s%sV = tcg_temp_new();\n" % \ - (regtype, regid)) - if (regtype == "C"): + if (regtype == "R"): + f.write(" const int %s = insn->regno[%d];\n" % (regN, regno)) + f.write(" TCGv %s%sV = get_result_gpr(ctx, %s);\n" % \ + (regtype, regid, regN)) + elif (regtype == "C"): f.write(" const int %s = insn->regno[%d] + HEX_REG_SA0;\n" % \ (regN, regno)) - else: + f.write(" TCGv %s%sV = get_result_gpr(ctx, %s);\n" % \ + (regtype, regid, regN)) + elif (regtype == "P"): f.write(" const int %s = insn->regno[%d];\n" % (regN, regno)) - if ('A_CONDEXEC' in hex_common.attribdict[tag]): - f.write(" if (!is_preloaded(ctx, %s)) {\n" % regN) - f.write(" tcg_gen_mov_tl(hex_new_value[%s], hex_gpr[%s]);\n" % \ - (regN, regN)) - f.write(" }\n") + f.write(" TCGv %s%sV = tcg_temp_new();\n" % \ + (regtype, regid)) + else: + print("Bad register parse: ", regtype, regid) def genptr_decl(f, tag, regtype, regid, regno): regN="%s%sN" % (regtype,regid) @@ -166,17 +162,6 @@ def genptr_decl(f, tag, regtype, regid, regno): f.write(" ctx_future_vreg_off(ctx, %s%sN," % \ (regtype, regid)) f.write(" 1, true);\n"); - if 'A_CONDEXEC' in hex_common.attribdict[tag]: - f.write(" if (!is_vreg_preloaded(ctx, %s)) {\n" % (regN)) - f.write(" intptr_t src_off =") - f.write(" offsetof(CPUHexagonState, VRegs[%s%sN]);\n"% \ - (regtype, regid)) - f.write(" tcg_gen_gvec_mov(MO_64, %s%sV_off,\n" % \ - (regtype, regid)) - f.write(" src_off,\n") - f.write(" sizeof(MMVector),\n") - f.write(" sizeof(MMVector));\n") - f.write(" }\n") if (not hex_common.skip_qemu_helper(tag)): f.write(" TCGv_ptr %s%sV = tcg_temp_new_ptr();\n" % \ @@ -191,8 +176,7 @@ def genptr_decl(f, tag, regtype, regid, regno): (regtype, regid, regno)) f.write(" const intptr_t %s%sV_off =\n" % \ (regtype, regid)) - f.write(" offsetof(CPUHexagonState,\n") - f.write(" future_QRegs[%s%sN]);\n" % \ + f.write(" get_result_qreg(ctx, %s%sN);\n" % \ (regtype, regid)) if (not hex_common.skip_qemu_helper(tag)): f.write(" TCGv_ptr %s%sV = tcg_temp_new_ptr();\n" % \ @@ -274,8 +258,12 @@ def genptr_src_read(f, tag, regtype, regid): f.write(" hex_gpr[%s%sN + 1]);\n" % \ (regtype, regid)) elif (regid in {"x", "y"}): - f.write(" tcg_gen_mov_tl(%s%sV, hex_gpr[%s%sN]);\n" % \ - (regtype,regid,regtype,regid)) + ## For read/write registers, we need to get the original value into + ## the result TCGv. For conditional instructions, this is done in + ## gen_start_packet. For unconditional instructions, we do it here. + if ('A_CONDEXEC' not in hex_common.attribdict[tag]): + f.write(" tcg_gen_mov_tl(%s%sV, hex_gpr[%s%sN]);\n" % \ + (regtype, regid, regtype, regid)) elif (regid not in {"s", "t", "u", "v"}): print("Bad register parse: ", regtype, regid) elif (regtype == "P"): @@ -385,37 +373,22 @@ def gen_helper_call_imm(f,immlett): f.write(", tcgv_%s" % hex_common.imm_name(immlett)) def genptr_dst_write_pair(f, tag, regtype, regid): - if ('A_CONDEXEC' in hex_common.attribdict[tag]): - f.write(" gen_log_predicated_reg_write_pair(%s%sN, %s%sV, insn->slot);\n" % \ - (regtype, regid, regtype, regid)) - else: - f.write(" gen_log_reg_write_pair(%s%sN, %s%sV);\n" % \ - (regtype, regid, regtype, regid)) - f.write(" ctx_log_reg_write_pair(ctx, %s%sN);\n" % \ - (regtype, regid)) + f.write(" gen_log_reg_write_pair(%s%sN, %s%sV);\n" % \ + (regtype, regid, regtype, regid)) def genptr_dst_write(f, tag, regtype, regid): if (regtype == "R"): if (regid in {"dd", "xx", "yy"}): genptr_dst_write_pair(f, tag, regtype, regid) elif (regid in {"d", "e", "x", "y"}): - if ('A_CONDEXEC' in hex_common.attribdict[tag]): - f.write(" gen_log_predicated_reg_write(%s%sN, %s%sV,\n" % \ - (regtype, regid, regtype, regid)) - f.write(" insn->slot);\n") - else: - f.write(" gen_log_reg_write(%s%sN, %s%sV);\n" % \ - (regtype, regid, regtype, regid)) - f.write(" ctx_log_reg_write(ctx, %s%sN);\n" % \ - (regtype, regid)) + f.write(" gen_log_reg_write(%s%sN, %s%sV);\n" % \ + (regtype, regid, regtype, regid)) else: print("Bad register parse: ", regtype, regid) elif (regtype == "P"): if (regid in {"d", "e", "x"}): f.write(" gen_log_pred_write(ctx, %s%sN, %s%sV);\n" % \ (regtype, regid, regtype, regid)) - f.write(" ctx_log_pred_write(ctx, %s%sN);\n" % \ - (regtype, regid)) else: print("Bad register parse: ", regtype, regid) elif (regtype == "C"): @@ -432,43 +405,18 @@ def genptr_dst_write(f, tag, regtype, regid): def genptr_dst_write_ext(f, tag, regtype, regid, newv="EXT_DFL"): if (regtype == "V"): - if (regid in {"dd", "xx", "yy"}): - if ('A_CONDEXEC' in hex_common.attribdict[tag]): - is_predicated = "true" - else: - is_predicated = "false" + if (regid in {"xx"}): f.write(" gen_log_vreg_write_pair(ctx, %s%sV_off, %s%sN, " % \ (regtype, regid, regtype, regid)) - f.write("%s, insn->slot, %s);\n" % \ - (newv, is_predicated)) - f.write(" ctx_log_vreg_write_pair(ctx, %s%sN, %s,\n" % \ - (regtype, regid, newv)) - f.write(" %s);\n" % (is_predicated)) - elif (regid in {"d", "x", "y"}): - if ('A_CONDEXEC' in hex_common.attribdict[tag]): - is_predicated = "true" - else: - is_predicated = "false" - f.write(" gen_log_vreg_write(ctx, %s%sV_off, %s%sN, %s, " % \ + f.write("%s);\n" % \ + (newv)) + elif (regid in {"y"}): + f.write(" gen_log_vreg_write(ctx, %s%sV_off, %s%sN, %s);\n" % \ (regtype, regid, regtype, regid, newv)) - f.write("insn->slot, %s);\n" % \ - (is_predicated)) - f.write(" ctx_log_vreg_write(ctx, %s%sN, %s, %s);\n" % \ - (regtype, regid, newv, is_predicated)) - else: + elif (regid not in {"dd", "d", "x"}): print("Bad register parse: ", regtype, regid) elif (regtype == "Q"): - if (regid in {"d", "e", "x"}): - if ('A_CONDEXEC' in hex_common.attribdict[tag]): - is_predicated = "true" - else: - is_predicated = "false" - f.write(" gen_log_qreg_write(%s%sV_off, %s%sN, %s, " % \ - (regtype, regid, regtype, regid, newv)) - f.write("insn->slot, %s);\n" % (is_predicated)) - f.write(" ctx_log_qreg_write(ctx, %s%sN, %s);\n" % \ - (regtype, regid, is_predicated)) - else: + if (regid not in {"d", "e", "x"}): print("Bad register parse: ", regtype, regid) else: print("Bad register parse: ", regtype, regid) @@ -500,15 +448,15 @@ def genptr_dst_write_opn(f,regtype, regid, tag): ## For A2_add: Rd32=add(Rs32,Rt32), { RdV=RsV+RtV;} ## We produce: ## static void generate_A2_add(DisasContext *ctx) -## { -## TCGv RdV = tcg_temp_new(); -## const int RdN = insn->regno[0]; -## TCGv RsV = hex_gpr[insn->regno[1]]; -## TCGv RtV = hex_gpr[insn->regno[2]]; -## <GEN> -## gen_log_reg_write(RdN, RdV); -## ctx_log_reg_write(ctx, RdN); -## } +## { +## Insn *insn __attribute__((unused)) = ctx->insn; +## const int RdN = insn->regno[0]; +## TCGv RdV = get_result_gpr(ctx, RdN); +## TCGv RsV = hex_gpr[insn->regno[1]]; +## TCGv RtV = hex_gpr[insn->regno[2]]; +## <GEN> +## gen_log_reg_write(RdN, RdV); +## } ## ## where <GEN> depends on hex_common.skip_qemu_helper(tag) ## if hex_common.skip_qemu_helper(tag) is True @@ -592,6 +540,14 @@ def gen_tcg_func(f, tag, regs, imms): if (i > 0): f.write(", ") f.write("cpu_env") i=1 + ## For conditional instructions, we pass in the destination register + if 'A_CONDEXEC' in hex_common.attribdict[tag]: + for regtype, regid, toss, numregs in regs: + if (hex_common.is_writeonly(regid) and + not hex_common.is_hvx_reg(regtype)): + gen_helper_call_opn(f, tag, regtype, regid, toss, \ + numregs, i) + i += 1 for regtype,regid,toss,numregs in regs: if (hex_common.is_written(regid)): if (not hex_common.is_hvx_reg(regtype)): diff --git a/target/hexagon/gen_tcg_hvx.h b/target/hexagon/gen_tcg_hvx.h index 94f272e286..d4aefe8e3f 100644 --- a/target/hexagon/gen_tcg_hvx.h +++ b/target/hexagon/gen_tcg_hvx.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -133,16 +133,11 @@ static inline void assert_vhist_tmp(DisasContext *ctx) do { \ TCGv lsb = tcg_temp_new(); \ TCGLabel *false_label = gen_new_label(); \ - TCGLabel *end_label = gen_new_label(); \ tcg_gen_andi_tl(lsb, PsV, 1); \ tcg_gen_brcondi_tl(TCG_COND_NE, lsb, PRED, false_label); \ tcg_gen_gvec_mov(MO_64, VdV_off, VuV_off, \ sizeof(MMVector), sizeof(MMVector)); \ - tcg_gen_br(end_label); \ gen_set_label(false_label); \ - tcg_gen_ori_tl(hex_slot_cancelled, hex_slot_cancelled, \ - 1 << insn->slot); \ - gen_set_label(end_label); \ } while (0) @@ -547,17 +542,12 @@ static inline void assert_vhist_tmp(DisasContext *ctx) do { \ TCGv LSB = tcg_temp_new(); \ TCGLabel *false_label = gen_new_label(); \ - TCGLabel *end_label = gen_new_label(); \ GET_EA; \ PRED; \ tcg_gen_brcondi_tl(TCG_COND_EQ, LSB, 0, false_label); \ gen_vreg_load(ctx, DSTOFF, EA, true); \ INC; \ - tcg_gen_br(end_label); \ gen_set_label(false_label); \ - tcg_gen_ori_tl(hex_slot_cancelled, hex_slot_cancelled, \ - 1 << insn->slot); \ - gen_set_label(end_label); \ } while (0) #define fGEN_TCG_PRED_VEC_LOAD_pred_pi \ @@ -717,17 +707,12 @@ static inline void assert_vhist_tmp(DisasContext *ctx) do { \ TCGv LSB = tcg_temp_new(); \ TCGLabel *false_label = gen_new_label(); \ - TCGLabel *end_label = gen_new_label(); \ GET_EA; \ PRED; \ tcg_gen_brcondi_tl(TCG_COND_EQ, LSB, 0, false_label); \ gen_vreg_store(ctx, EA, SRCOFF, insn->slot, ALIGN); \ INC; \ - tcg_gen_br(end_label); \ gen_set_label(false_label); \ - tcg_gen_ori_tl(hex_slot_cancelled, hex_slot_cancelled, \ - 1 << insn->slot); \ - gen_set_label(end_label); \ } while (0) #define fGEN_TCG_PRED_VEC_STORE_pred_pi(ALIGN) \ diff --git a/target/hexagon/genptr.c b/target/hexagon/genptr.c index 86bd093ce8..bb274d4a71 100644 --- a/target/hexagon/genptr.c +++ b/target/hexagon/genptr.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -68,26 +68,17 @@ static inline void gen_masked_reg_write(TCGv new_val, TCGv cur_val, } } -static inline void gen_log_predicated_reg_write(int rnum, TCGv val, - uint32_t slot) +static TCGv get_result_gpr(DisasContext *ctx, int rnum) { - TCGv zero = tcg_constant_tl(0); - TCGv slot_mask = tcg_temp_new(); + return hex_new_value[rnum]; +} - tcg_gen_andi_tl(slot_mask, hex_slot_cancelled, 1 << slot); - tcg_gen_movcond_tl(TCG_COND_EQ, hex_new_value[rnum], slot_mask, zero, - val, hex_new_value[rnum]); - if (HEX_DEBUG) { - /* - * Do this so HELPER(debug_commit_end) will know - * - * Note that slot_mask indicates the value is not written - * (i.e., slot was cancelled), so we create a true/false value before - * or'ing with hex_reg_written[rnum]. - */ - tcg_gen_setcond_tl(TCG_COND_EQ, slot_mask, slot_mask, zero); - tcg_gen_or_tl(hex_reg_written[rnum], hex_reg_written[rnum], slot_mask); - } +static TCGv_i64 get_result_gpr_pair(DisasContext *ctx, int rnum) +{ + TCGv_i64 result = tcg_temp_new_i64(); + tcg_gen_concat_i32_i64(result, hex_new_value[rnum], + hex_new_value[rnum + 1]); + return result; } void gen_log_reg_write(int rnum, TCGv val) @@ -102,39 +93,6 @@ void gen_log_reg_write(int rnum, TCGv val) } } -static void gen_log_predicated_reg_write_pair(int rnum, TCGv_i64 val, - uint32_t slot) -{ - TCGv val32 = tcg_temp_new(); - TCGv zero = tcg_constant_tl(0); - TCGv slot_mask = tcg_temp_new(); - - tcg_gen_andi_tl(slot_mask, hex_slot_cancelled, 1 << slot); - /* Low word */ - tcg_gen_extrl_i64_i32(val32, val); - tcg_gen_movcond_tl(TCG_COND_EQ, hex_new_value[rnum], - slot_mask, zero, - val32, hex_new_value[rnum]); - /* High word */ - tcg_gen_extrh_i64_i32(val32, val); - tcg_gen_movcond_tl(TCG_COND_EQ, hex_new_value[rnum + 1], - slot_mask, zero, - val32, hex_new_value[rnum + 1]); - if (HEX_DEBUG) { - /* - * Do this so HELPER(debug_commit_end) will know - * - * Note that slot_mask indicates the value is not written - * (i.e., slot was cancelled), so we create a true/false value before - * or'ing with hex_reg_written[rnum]. - */ - tcg_gen_setcond_tl(TCG_COND_EQ, slot_mask, slot_mask, zero); - tcg_gen_or_tl(hex_reg_written[rnum], hex_reg_written[rnum], slot_mask); - tcg_gen_or_tl(hex_reg_written[rnum + 1], hex_reg_written[rnum + 1], - slot_mask); - } -} - static void gen_log_reg_write_pair(int rnum, TCGv_i64 val) { const target_ulong reg_mask_low = reg_immut_masks[rnum]; @@ -180,6 +138,7 @@ void gen_log_pred_write(DisasContext *ctx, int pnum, TCGv val) hex_new_pred_value[pnum], base_val); } tcg_gen_ori_tl(hex_pred_written, hex_pred_written, 1 << pnum); + set_bit(pnum, ctx->pregs_written); } static inline void gen_read_p3_0(TCGv control_reg) @@ -256,7 +215,6 @@ static void gen_write_p3_0(DisasContext *ctx, TCGv control_reg) for (int i = 0; i < NUM_PREGS; i++) { tcg_gen_extract_tl(hex_p8, control_reg, i * 8, 8); gen_log_pred_write(ctx, i, hex_p8); - ctx_log_pred_write(ctx, i); } } @@ -274,7 +232,6 @@ static inline void gen_write_ctrl_reg(DisasContext *ctx, int reg_num, gen_write_p3_0(ctx, val); } else { gen_log_reg_write(reg_num, val); - ctx_log_reg_write(ctx, reg_num); if (reg_num == HEX_REG_QEMU_PKT_CNT) { ctx->num_packets = 0; } @@ -291,15 +248,14 @@ static inline void gen_write_ctrl_reg_pair(DisasContext *ctx, int reg_num, TCGv_i64 val) { if (reg_num == HEX_REG_P3_0_ALIASED) { + TCGv result = get_result_gpr(ctx, reg_num + 1); TCGv val32 = tcg_temp_new(); tcg_gen_extrl_i64_i32(val32, val); gen_write_p3_0(ctx, val32); tcg_gen_extrh_i64_i32(val32, val); - gen_log_reg_write(reg_num + 1, val32); - ctx_log_reg_write(ctx, reg_num + 1); + tcg_gen_mov_tl(result, val32); } else { gen_log_reg_write_pair(reg_num, val); - ctx_log_reg_write_pair(ctx, reg_num); if (reg_num == HEX_REG_QEMU_PKT_CNT) { ctx->num_packets = 0; ctx->num_insns = 0; @@ -571,6 +527,13 @@ static void gen_cond_jumpr(DisasContext *ctx, TCGv dst_pc, gen_write_new_pc_addr(ctx, dst_pc, cond, pred); } +static void gen_cond_jumpr31(DisasContext *ctx, TCGCond cond, TCGv pred) +{ + TCGv LSB = tcg_temp_new(); + tcg_gen_andi_tl(LSB, pred, 1); + gen_cond_jumpr(ctx, hex_gpr[HEX_REG_LR], cond, LSB); +} + static void gen_cond_jump(DisasContext *ctx, TCGCond cond, TCGv pred, int pc_off) { @@ -669,27 +632,99 @@ static void gen_jumpr(DisasContext *ctx, TCGv new_pc) static void gen_call(DisasContext *ctx, int pc_off) { - TCGv next_PC = - tcg_constant_tl(ctx->pkt->pc + ctx->pkt->encod_pkt_size_in_bytes); - gen_log_reg_write(HEX_REG_LR, next_PC); + TCGv lr = get_result_gpr(ctx, HEX_REG_LR); + tcg_gen_movi_tl(lr, ctx->next_PC); gen_write_new_pc_pcrel(ctx, pc_off, TCG_COND_ALWAYS, NULL); } +static void gen_callr(DisasContext *ctx, TCGv new_pc) +{ + TCGv lr = get_result_gpr(ctx, HEX_REG_LR); + tcg_gen_movi_tl(lr, ctx->next_PC); + gen_write_new_pc_addr(ctx, new_pc, TCG_COND_ALWAYS, NULL); +} + static void gen_cond_call(DisasContext *ctx, TCGv pred, TCGCond cond, int pc_off) { - TCGv next_PC; + TCGv lr = get_result_gpr(ctx, HEX_REG_LR); TCGv lsb = tcg_temp_new(); TCGLabel *skip = gen_new_label(); tcg_gen_andi_tl(lsb, pred, 1); gen_write_new_pc_pcrel(ctx, pc_off, cond, lsb); tcg_gen_brcondi_tl(cond, lsb, 0, skip); - next_PC = - tcg_constant_tl(ctx->pkt->pc + ctx->pkt->encod_pkt_size_in_bytes); - gen_log_reg_write(HEX_REG_LR, next_PC); + tcg_gen_movi_tl(lr, ctx->next_PC); gen_set_label(skip); } +static void gen_cond_callr(DisasContext *ctx, + TCGCond cond, TCGv pred, TCGv new_pc) +{ + TCGv lsb = tcg_temp_new(); + TCGLabel *skip = gen_new_label(); + tcg_gen_andi_tl(lsb, pred, 1); + tcg_gen_brcondi_tl(cond, lsb, 0, skip); + gen_callr(ctx, new_pc); + gen_set_label(skip); +} + +/* frame ^= (int64_t)FRAMEKEY << 32 */ +static void gen_frame_unscramble(TCGv_i64 frame) +{ + TCGv_i64 framekey = tcg_temp_new_i64(); + tcg_gen_extu_i32_i64(framekey, hex_gpr[HEX_REG_FRAMEKEY]); + tcg_gen_shli_i64(framekey, framekey, 32); + tcg_gen_xor_i64(frame, frame, framekey); +} + +static void gen_load_frame(DisasContext *ctx, TCGv_i64 frame, TCGv EA) +{ + Insn *insn = ctx->insn; /* Needed for CHECK_NOSHUF */ + CHECK_NOSHUF(EA, 8); + tcg_gen_qemu_ld64(frame, EA, ctx->mem_idx); +} + +static void gen_return(DisasContext *ctx, TCGv_i64 dst, TCGv src) +{ + /* + * frame = *src + * dst = frame_unscramble(frame) + * SP = src + 8 + * PC = dst.w[1] + */ + TCGv_i64 frame = tcg_temp_new_i64(); + TCGv r31 = tcg_temp_new(); + TCGv r29 = get_result_gpr(ctx, HEX_REG_SP); + + gen_load_frame(ctx, frame, src); + gen_frame_unscramble(frame); + tcg_gen_mov_i64(dst, frame); + tcg_gen_addi_tl(r29, src, 8); + tcg_gen_extrh_i64_i32(r31, dst); + gen_jumpr(ctx, r31); +} + +/* if (pred) dst = dealloc_return(src):raw */ +static void gen_cond_return(DisasContext *ctx, TCGv_i64 dst, TCGv src, + TCGv pred, TCGCond cond) +{ + TCGv LSB = tcg_temp_new(); + TCGLabel *skip = gen_new_label(); + tcg_gen_andi_tl(LSB, pred, 1); + + tcg_gen_brcondi_tl(cond, LSB, 0, skip); + gen_return(ctx, dst, src); + gen_set_label(skip); +} + +/* sub-instruction version (no RddV, so handle it manually) */ +static void gen_cond_return_subinsn(DisasContext *ctx, TCGCond cond, TCGv pred) +{ + TCGv_i64 RddV = get_result_gpr_pair(ctx, HEX_REG_FP); + gen_cond_return(ctx, RddV, hex_gpr[HEX_REG_FP], pred, cond); + gen_log_reg_write_pair(HEX_REG_FP, RddV); +} + static void gen_endloop0(DisasContext *ctx) { TCGv lpcfg = tcg_temp_new(); @@ -737,14 +772,95 @@ static void gen_endloop0(DisasContext *ctx) TCGLabel *label3 = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_LEU, hex_gpr[HEX_REG_LC0], 1, label3); { + TCGv lc0 = get_result_gpr(ctx, HEX_REG_LC0); gen_jumpr(ctx, hex_gpr[HEX_REG_SA0]); - tcg_gen_subi_tl(hex_new_value[HEX_REG_LC0], - hex_gpr[HEX_REG_LC0], 1); + tcg_gen_subi_tl(lc0, hex_gpr[HEX_REG_LC0], 1); } gen_set_label(label3); } } +static void gen_endloop1(DisasContext *ctx) +{ + /* + * if (hex_gpr[HEX_REG_LC1] > 1) { + * PC = hex_gpr[HEX_REG_SA1]; + * hex_new_value[HEX_REG_LC1] = hex_gpr[HEX_REG_LC1] - 1; + * } + */ + TCGLabel *label = gen_new_label(); + tcg_gen_brcondi_tl(TCG_COND_LEU, hex_gpr[HEX_REG_LC1], 1, label); + { + TCGv lc1 = get_result_gpr(ctx, HEX_REG_LC1); + gen_jumpr(ctx, hex_gpr[HEX_REG_SA1]); + tcg_gen_subi_tl(lc1, hex_gpr[HEX_REG_LC1], 1); + } + gen_set_label(label); +} + +static void gen_endloop01(DisasContext *ctx) +{ + TCGv lpcfg = tcg_temp_new(); + TCGLabel *label1 = gen_new_label(); + TCGLabel *label2 = gen_new_label(); + TCGLabel *label3 = gen_new_label(); + TCGLabel *done = gen_new_label(); + + GET_USR_FIELD(USR_LPCFG, lpcfg); + + /* + * if (lpcfg == 1) { + * hex_new_pred_value[3] = 0xff; + * hex_pred_written |= 1 << 3; + * } + */ + tcg_gen_brcondi_tl(TCG_COND_NE, lpcfg, 1, label1); + { + tcg_gen_movi_tl(hex_new_pred_value[3], 0xff); + tcg_gen_ori_tl(hex_pred_written, hex_pred_written, 1 << 3); + } + gen_set_label(label1); + + /* + * if (lpcfg) { + * SET_USR_FIELD(USR_LPCFG, lpcfg - 1); + * } + */ + tcg_gen_brcondi_tl(TCG_COND_EQ, lpcfg, 0, label2); + { + tcg_gen_subi_tl(lpcfg, lpcfg, 1); + SET_USR_FIELD(USR_LPCFG, lpcfg); + } + gen_set_label(label2); + + /* + * if (hex_gpr[HEX_REG_LC0] > 1) { + * PC = hex_gpr[HEX_REG_SA0]; + * hex_new_value[HEX_REG_LC0] = hex_gpr[HEX_REG_LC0] - 1; + * } else { + * if (hex_gpr[HEX_REG_LC1] > 1) { + * hex_next_pc = hex_gpr[HEX_REG_SA1]; + * hex_new_value[HEX_REG_LC1] = hex_gpr[HEX_REG_LC1] - 1; + * } + * } + */ + tcg_gen_brcondi_tl(TCG_COND_LEU, hex_gpr[HEX_REG_LC0], 1, label3); + { + TCGv lc0 = get_result_gpr(ctx, HEX_REG_LC0); + gen_jumpr(ctx, hex_gpr[HEX_REG_SA0]); + tcg_gen_subi_tl(lc0, hex_gpr[HEX_REG_LC0], 1); + tcg_gen_br(done); + } + gen_set_label(label3); + tcg_gen_brcondi_tl(TCG_COND_LEU, hex_gpr[HEX_REG_LC1], 1, done); + { + TCGv lc1 = get_result_gpr(ctx, HEX_REG_LC1); + gen_jumpr(ctx, hex_gpr[HEX_REG_SA1]); + tcg_gen_subi_tl(lc1, hex_gpr[HEX_REG_LC1], 1); + } + gen_set_label(done); +} + static void gen_cmp_jumpnv(DisasContext *ctx, TCGCond cond, TCGv val, TCGv src, int pc_off) { @@ -869,68 +985,32 @@ static intptr_t vreg_src_off(DisasContext *ctx, int num) } static void gen_log_vreg_write(DisasContext *ctx, intptr_t srcoff, int num, - VRegWriteType type, int slot_num, - bool is_predicated) + VRegWriteType type) { - TCGLabel *label_end = NULL; intptr_t dstoff; - if (is_predicated) { - TCGv cancelled = tcg_temp_new(); - label_end = gen_new_label(); - - /* Don't do anything if the slot was cancelled */ - tcg_gen_extract_tl(cancelled, hex_slot_cancelled, slot_num, 1); - tcg_gen_brcondi_tl(TCG_COND_NE, cancelled, 0, label_end); - } - if (type != EXT_TMP) { dstoff = ctx_future_vreg_off(ctx, num, 1, true); tcg_gen_gvec_mov(MO_64, dstoff, srcoff, sizeof(MMVector), sizeof(MMVector)); - tcg_gen_ori_tl(hex_VRegs_updated, hex_VRegs_updated, 1 << num); } else { dstoff = ctx_tmp_vreg_off(ctx, num, 1, false); tcg_gen_gvec_mov(MO_64, dstoff, srcoff, sizeof(MMVector), sizeof(MMVector)); } - - if (is_predicated) { - gen_set_label(label_end); - } } static void gen_log_vreg_write_pair(DisasContext *ctx, intptr_t srcoff, int num, - VRegWriteType type, int slot_num, - bool is_predicated) + VRegWriteType type) { - gen_log_vreg_write(ctx, srcoff, num ^ 0, type, slot_num, is_predicated); + gen_log_vreg_write(ctx, srcoff, num ^ 0, type); srcoff += sizeof(MMVector); - gen_log_vreg_write(ctx, srcoff, num ^ 1, type, slot_num, is_predicated); + gen_log_vreg_write(ctx, srcoff, num ^ 1, type); } -static void gen_log_qreg_write(intptr_t srcoff, int num, int vnew, - int slot_num, bool is_predicated) +static intptr_t get_result_qreg(DisasContext *ctx, int qnum) { - TCGLabel *label_end = NULL; - intptr_t dstoff; - - if (is_predicated) { - TCGv cancelled = tcg_temp_new(); - label_end = gen_new_label(); - - /* Don't do anything if the slot was cancelled */ - tcg_gen_extract_tl(cancelled, hex_slot_cancelled, slot_num, 1); - tcg_gen_brcondi_tl(TCG_COND_NE, cancelled, 0, label_end); - } - - dstoff = offsetof(CPUHexagonState, future_QRegs[num]); - tcg_gen_gvec_mov(MO_64, dstoff, srcoff, sizeof(MMQReg), sizeof(MMQReg)); - - if (is_predicated) { - tcg_gen_ori_tl(hex_QRegs_updated, hex_QRegs_updated, 1 << num); - gen_set_label(label_end); - } + return offsetof(CPUHexagonState, future_QRegs[qnum]); } static void gen_vreg_load(DisasContext *ctx, intptr_t dstoff, TCGv src, diff --git a/target/hexagon/hex_common.py b/target/hexagon/hex_common.py index a29f61bb4f..0200a66cb6 100755 --- a/target/hexagon/hex_common.py +++ b/target/hexagon/hex_common.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ## -## Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. +## Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. ## ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by @@ -89,6 +89,7 @@ def calculate_attribs(): add_qemu_macro_attrib('fWRITE_P3', 'A_WRITES_PRED_REG') add_qemu_macro_attrib('fSET_OVERFLOW', 'A_IMPLICIT_WRITES_USR') add_qemu_macro_attrib('fSET_LPCFG', 'A_IMPLICIT_WRITES_USR') + add_qemu_macro_attrib('fLOAD', 'A_SCALAR_LOAD') add_qemu_macro_attrib('fSTORE', 'A_SCALAR_STORE') # Recurse down macros, find attributes from sub-macros @@ -236,6 +237,13 @@ def helper_needs_next_PC(tag): def need_pkt_has_multi_cof(tag): return 'A_COF' in attribdict[tag] +def need_condexec_reg(tag, regs): + if 'A_CONDEXEC' in attribdict[tag]: + for regtype, regid, toss, numregs in regs: + if is_writeonly(regid) and not is_hvx_reg(regtype): + return True + return False + def skip_qemu_helper(tag): return tag in overrides.keys() diff --git a/target/hexagon/idef-parser/idef-parser.h b/target/hexagon/idef-parser/idef-parser.h index 17d2ebfaf6..d23e71f13b 100644 --- a/target/hexagon/idef-parser/idef-parser.h +++ b/target/hexagon/idef-parser/idef-parser.h @@ -82,7 +82,6 @@ enum ImmUnionTag { VALUE, QEMU_TMP, IMM_PC, - IMM_NPC, IMM_CONSTEXT, }; diff --git a/target/hexagon/idef-parser/idef-parser.lex b/target/hexagon/idef-parser/idef-parser.lex index ff87a02c3a..5eb8ac5a80 100644 --- a/target/hexagon/idef-parser/idef-parser.lex +++ b/target/hexagon/idef-parser/idef-parser.lex @@ -5,7 +5,7 @@ %{ /* - * Copyright(c) 2019-2022 rev.ng Labs Srl. All Rights Reserved. + * Copyright(c) 2019-2023 rev.ng Labs Srl. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -140,8 +140,6 @@ STRING_LIT \"(\\.|[^"\\])*\" yylval->rvalue.is_dotnew = true; yylval->rvalue.signedness = SIGNED; return PRED; } -"IV1DEAD()" | -"fPAUSE(uiV);" { return ';'; } "+=" { return INC; } "-=" { return DEC; } "++" { return PLUSPLUS; } @@ -159,9 +157,8 @@ STRING_LIT \"(\\.|[^"\\])*\" "else" { return ELSE; } "for" { return FOR; } "fREAD_IREG" { return ICIRC; } -"fPART1" { return PART1; } "if" { return IF; } -"fFRAME_SCRAMBLE" { return FSCR; } +"fFRAME_SCRAMBLE" | "fFRAME_UNSCRAMBLE" { return FSCR; } "fFRAMECHECK" { return FCHK; } "Constant_extended" { return CONSTEXT; } @@ -312,14 +309,10 @@ STRING_LIT \"(\\.|[^"\\])*\" "(unsigned int)" { yylval->cast.bit_width = 32; yylval->cast.signedness = UNSIGNED; return CAST; } -"fREAD_PC()" | -"PC" { return PC; } -"fREAD_NPC()" | -"NPC" { return NPC; } -"fGET_LPCFG" | +"fREAD_PC()" { return PC; } "USR.LPCFG" { return LPCFG; } "LOAD_CANCEL(EA)" { return LOAD_CANCEL; } -"STORE_CANCEL(EA)" | +"STORE_CANCEL(EA)" { return STORE_CANCEL; } "CANCEL" { return CANCEL; } "N"{LOWER_ID}"N" { yylval->rvalue.type = REGISTER_ARG; yylval->rvalue.reg.type = DOTNEW; @@ -360,14 +353,6 @@ STRING_LIT \"(\\.|[^"\\])*\" yylval->rvalue.bit_width = 32; yylval->rvalue.signedness = UNSIGNED; return REG; } -"fREAD_LC"[01] { yylval->rvalue.type = REGISTER; - yylval->rvalue.reg.type = CONTROL; - yylval->rvalue.reg.id = HEX_REG_LC0 - + (yytext[8] - '0') * 2; - yylval->rvalue.reg.bit_width = 32; - yylval->rvalue.bit_width = 32; - yylval->rvalue.signedness = UNSIGNED; - return REG; } "LC"[01] { yylval->rvalue.type = REGISTER; yylval->rvalue.reg.type = CONTROL; yylval->rvalue.reg.id = HEX_REG_LC0 @@ -376,14 +361,6 @@ STRING_LIT \"(\\.|[^"\\])*\" yylval->rvalue.bit_width = 32; yylval->rvalue.signedness = UNSIGNED; return REG; } -"fREAD_SA"[01] { yylval->rvalue.type = REGISTER; - yylval->rvalue.reg.type = CONTROL; - yylval->rvalue.reg.id = HEX_REG_SA0 - + (yytext[8] - '0') * 2; - yylval->rvalue.reg.bit_width = 32; - yylval->rvalue.bit_width = 32; - yylval->rvalue.signedness = UNSIGNED; - return REG; } "SA"[01] { yylval->rvalue.type = REGISTER; yylval->rvalue.reg.type = CONTROL; yylval->rvalue.reg.id = HEX_REG_SA0 diff --git a/target/hexagon/idef-parser/idef-parser.y b/target/hexagon/idef-parser/idef-parser.y index c784726d41..7d05773b67 100644 --- a/target/hexagon/idef-parser/idef-parser.y +++ b/target/hexagon/idef-parser/idef-parser.y @@ -1,6 +1,6 @@ %{ /* - * Copyright(c) 2019-2022 rev.ng Labs Srl. All Rights Reserved. + * Copyright(c) 2019-2023 rev.ng Labs Srl. All Rights Reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -52,8 +52,8 @@ %token IN INAME VAR %token ABS CROUND ROUND CIRCADD COUNTONES INC DEC ANDA ORA XORA PLUSPLUS ASL %token ASR LSR EQ NEQ LTE GTE MIN MAX ANDL FOR ICIRC IF MUN FSCR FCHK SXT -%token ZXT CONSTEXT LOCNT BREV SIGN LOAD STORE PC NPC LPCFG -%token LOAD_CANCEL CANCEL IDENTITY PART1 ROTL INSBITS SETBITS EXTRANGE +%token ZXT CONSTEXT LOCNT BREV SIGN LOAD STORE PC LPCFG +%token LOAD_CANCEL STORE_CANCEL CANCEL IDENTITY ROTL INSBITS SETBITS EXTRANGE %token CAST4_8U FAIL CARRY_FROM_ADD ADDSAT64 LSBNEW %token TYPE_SIZE_T TYPE_INT TYPE_SIGNED TYPE_UNSIGNED TYPE_LONG @@ -336,15 +336,6 @@ assign_statement : lvalue '=' rvalue OUT(c, &@1, &$1, " = ", &$3, ";\n"); $$ = $1; } - | PC '=' rvalue - { - @1.last_column = @3.last_column; - yyassert(c, &@1, !is_inside_ternary(c), - "Assignment side-effect not modeled!"); - $3 = gen_rvalue_truncate(c, &@1, &$3); - $3 = rvalue_materialize(c, &@1, &$3); - OUT(c, &@1, "gen_write_new_pc(", &$3, ");\n"); - } | LOAD '(' IMM ',' IMM ',' SIGN ',' var ',' lvalue ')' { @1.last_column = @12.last_column; @@ -412,7 +403,6 @@ control_statement : frame_check | cancel_statement | if_statement | for_statement - | fpart1_statement ; frame_check : FCHK '(' rvalue ',' rvalue ')' ';' @@ -422,10 +412,11 @@ cancel_statement : LOAD_CANCEL { gen_load_cancel(c, &@1); } - | CANCEL + | STORE_CANCEL { gen_cancel(c, &@1); } + | CANCEL ; if_statement : if_stmt @@ -462,17 +453,6 @@ for_statement : FOR '(' IMM '=' IMM ';' IMM '<' IMM ';' IMM PLUSPLUS ')' } ; -fpart1_statement : PART1 - { - OUT(c, &@1, "if (insn->part1) {\n"); - } - '(' statements ')' - { - @1.last_column = @3.last_column; - OUT(c, &@1, "return; }\n"); - } - ; - if_stmt : IF '(' rvalue ')' { @1.last_column = @3.last_column; @@ -512,20 +492,6 @@ rvalue : FAIL rvalue.signedness = UNSIGNED; $$ = rvalue; } - | NPC - { - /* - * NPC is only read from CALLs, so we can hardcode it - * at translation time - */ - HexValue rvalue; - memset(&rvalue, 0, sizeof(HexValue)); - rvalue.type = IMMEDIATE; - rvalue.imm.type = IMM_NPC; - rvalue.bit_width = 32; - rvalue.signedness = UNSIGNED; - $$ = rvalue; - } | CONSTEXT { HexValue rvalue; @@ -781,11 +747,6 @@ rvalue : FAIL /* Ones count */ $$ = gen_ctpop_op(c, &@1, &$3); } - | LPCFG - { - $$ = gen_tmp(c, &@1, 32, UNSIGNED); - OUT(c, &@1, "GET_USR_FIELD(USR_LPCFG, ", &$$, ");\n"); - } | EXTRACT '(' rvalue ',' rvalue ')' { @1.last_column = @6.last_column; diff --git a/target/hexagon/idef-parser/macros.inc b/target/hexagon/idef-parser/macros.inc index 6b697da87a..7478d4db17 100644 --- a/target/hexagon/idef-parser/macros.inc +++ b/target/hexagon/idef-parser/macros.inc @@ -97,16 +97,8 @@ #define fWRITE_LR(A) (LR = A) #define fWRITE_FP(A) (FP = A) #define fWRITE_SP(A) (SP = A) -/* - * Note: There is a rule in the parser that matches `PC = ...` and emits - * a call to `gen_write_new_pc`. We need to call `gen_write_new_pc` to - * get the correct semantics when there are multiple stores in a packet. - */ -#define fBRANCH(LOC, TYPE) (PC = LOC) -#define fJUMPR(REGNO, TARGET, TYPE) (PC = TARGET) #define fWRITE_LOOP_REGS0(START, COUNT) SA0 = START; (LC0 = COUNT) #define fWRITE_LOOP_REGS1(START, COUNT) SA1 = START; (LC1 = COUNT) -#define fWRITE_LC0(VAL) (LC0 = VAL) #define fWRITE_LC1(VAL) (LC1 = VAL) #define fSET_LPCFG(VAL) (USR.LPCFG = VAL) #define fWRITE_P0(VAL) P0 = VAL; @@ -121,7 +113,6 @@ #define fEA_GPI(IMM) (EA = fREAD_GP() + IMM) #define fPM_I(REG, IMM) (REG = REG + IMM) #define fPM_M(REG, MVAL) (REG = REG + MVAL) -#define fWRITE_NPC(VAL) (PC = VAL) /* Unary operators */ #define fROUND(A) (A + 0x8000) diff --git a/target/hexagon/idef-parser/parser-helpers.c b/target/hexagon/idef-parser/parser-helpers.c index e1a55412c8..18cde6a1be 100644 --- a/target/hexagon/idef-parser/parser-helpers.c +++ b/target/hexagon/idef-parser/parser-helpers.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 rev.ng Labs Srl. All Rights Reserved. + * Copyright(c) 2019-2023 rev.ng Labs Srl. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -185,9 +185,6 @@ void imm_print(Context *c, YYLTYPE *locp, HexImm *imm) case IMM_PC: EMIT(c, "ctx->base.pc_next"); break; - case IMM_NPC: - EMIT(c, "ctx->npc"); - break; case IMM_CONSTEXT: EMIT(c, "insn->extension_valid"); break; @@ -1323,10 +1320,6 @@ void gen_write_reg(Context *c, YYLTYPE *locp, HexValue *reg, HexValue *value) locp, "gen_log_reg_write(", ®->reg.id, ", ", &value_m, ");\n"); - OUT(c, - locp, - "ctx_log_reg_write(ctx, ", ®->reg.id, - ");\n"); } void gen_assign(Context *c, @@ -1675,9 +1668,7 @@ void gen_inst_init_args(Context *c, YYLTYPE *locp) for (unsigned i = 0; i < c->inst.init_list->len; i++) { HexValue *val = &g_array_index(c->inst.init_list, HexValue, i); if (val->type == REGISTER_ARG) { - char reg_id[5]; - reg_compose(c, locp, &val->reg, reg_id); - EMIT_HEAD(c, "tcg_gen_movi_i%u(%s, 0);\n", val->bit_width, reg_id); + /* Nothing to do here */ } else if (val->type == PREDICATE) { char suffix = val->is_dotnew ? 'N' : 'V'; EMIT_HEAD(c, "tcg_gen_movi_i%u(P%c%c, 0);\n", val->bit_width, @@ -1722,13 +1713,10 @@ void gen_pred_assign(Context *c, YYLTYPE *locp, HexValue *left_pred, *left_pred = gen_tmp(c, locp, 32, UNSIGNED); } /* Extract first 8 bits, and store new predicate value */ - OUT(c, locp, "tcg_gen_mov_i32(", left_pred, ", ", &r, ");\n"); - OUT(c, locp, "tcg_gen_andi_i32(", left_pred, ", ", left_pred, - ", 0xff);\n"); + OUT(c, locp, "tcg_gen_andi_i32(", left_pred, ", ", &r, ", 0xff);\n"); if (is_direct) { OUT(c, locp, "gen_log_pred_write(ctx, ", pred_id, ", ", left_pred, ");\n"); - OUT(c, locp, "ctx_log_pred_write(ctx, ", pred_id, ");\n"); } } @@ -1739,7 +1727,6 @@ void gen_cancel(Context *c, YYLTYPE *locp) void gen_load_cancel(Context *c, YYLTYPE *locp) { - gen_cancel(c, locp); OUT(c, locp, "if (insn->slot == 0 && pkt->pkt_has_store_s1) {\n"); OUT(c, locp, "ctx->s1_store_processed = false;\n"); OUT(c, locp, "process_store(ctx, 1);\n"); diff --git a/target/hexagon/macros.h b/target/hexagon/macros.h index 17facadaad..482a9c787f 100644 --- a/target/hexagon/macros.h +++ b/target/hexagon/macros.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -205,26 +205,11 @@ static inline void gen_cancel(uint32_t slot) #define CANCEL gen_cancel(slot); #else -#define CANCEL cancel_slot(env, slot) +#define CANCEL do { } while (0) #endif #define LOAD_CANCEL(EA) do { CANCEL; } while (0) -#ifdef QEMU_GENERATE -static inline void gen_pred_cancel(TCGv pred, uint32_t slot_num) - { - TCGv slot_mask = tcg_temp_new(); - TCGv tmp = tcg_temp_new(); - TCGv zero = tcg_constant_tl(0); - tcg_gen_ori_tl(slot_mask, hex_slot_cancelled, 1 << slot_num); - tcg_gen_andi_tl(tmp, pred, 1); - tcg_gen_movcond_tl(TCG_COND_EQ, hex_slot_cancelled, tmp, zero, - slot_mask, hex_slot_cancelled); -} -#define PRED_LOAD_CANCEL(PRED, EA) \ - gen_pred_cancel(PRED, insn->is_endloop ? 4 : insn->slot) -#endif - #define STORE_CANCEL(EA) { env->slot_cancelled |= (1 << slot); } #define fMAX(A, B) (((A) > (B)) ? (A) : (B)) @@ -415,16 +400,6 @@ static inline TCGv gen_read_ireg(TCGv result, TCGv val, int shift) #define fBRANCH(LOC, TYPE) fWRITE_NPC(LOC) #define fJUMPR(REGNO, TARGET, TYPE) fBRANCH(TARGET, COF_TYPE_JUMPR) #define fHINTJR(TARGET) { /* Not modelled in qemu */} -#define fCALL(A) \ - do { \ - fWRITE_LR(fREAD_NPC()); \ - fBRANCH(A, COF_TYPE_CALL); \ - } while (0) -#define fCALLR(A) \ - do { \ - fWRITE_LR(fREAD_NPC()); \ - fBRANCH(A, COF_TYPE_CALLR); \ - } while (0) #define fWRITE_LOOP_REGS0(START, COUNT) \ do { \ WRITE_RREG(HEX_REG_LC0, COUNT); \ diff --git a/target/hexagon/meson.build b/target/hexagon/meson.build index 42b03c81e6..da8e608d00 100644 --- a/target/hexagon/meson.build +++ b/target/hexagon/meson.build @@ -1,5 +1,5 @@ ## -## Copyright(c) 2020-2021 Qualcomm Innovation Center, Inc. All Rights Reserved. +## Copyright(c) 2020-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. ## ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by @@ -276,4 +276,13 @@ tcg_funcs_generated = custom_target( ) hexagon_ss.add(tcg_funcs_generated) +analyze_funcs_generated = custom_target( + 'analyze_funcs_generated.c.inc', + output: 'analyze_funcs_generated.c.inc', + depends: helper_dep, + depend_files: [hex_common_py, attribs_def, gen_tcg_h, gen_tcg_hvx_h], + command: [python, files('gen_analyze_funcs.py'), helper_in, '@OUTPUT@'], +) +hexagon_ss.add(analyze_funcs_generated) + target_arch += {'hexagon': hexagon_ss} diff --git a/target/hexagon/op_helper.c b/target/hexagon/op_helper.c index 35449ef524..c9a156030e 100644 --- a/target/hexagon/op_helper.c +++ b/target/hexagon/op_helper.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -30,6 +30,7 @@ #include "mmvec/mmvec.h" #include "mmvec/macros.h" #include "op_helper.h" +#include "translate.h" #define SF_BIAS 127 #define SF_MANTBITS 23 @@ -105,30 +106,6 @@ void log_store64(CPUHexagonState *env, target_ulong addr, env->mem_log_stores[slot].data64 = val; } -void write_new_pc(CPUHexagonState *env, bool pkt_has_multi_cof, - target_ulong addr) -{ - HEX_DEBUG_LOG("write_new_pc(0x" TARGET_FMT_lx ")\n", addr); - - if (pkt_has_multi_cof) { - /* - * If more than one branch is taken in a packet, only the first one - * is actually done. - */ - if (env->branch_taken) { - HEX_DEBUG_LOG("INFO: multiple branches taken in same packet, " - "ignoring the second one\n"); - } else { - fCHECK_PCALIGN(addr); - env->gpr[HEX_REG_PC] = addr; - env->branch_taken = 1; - } - } else { - fCHECK_PCALIGN(addr); - env->gpr[HEX_REG_PC] = addr; - } -} - /* Handy place to set a breakpoint */ void HELPER(debug_start_packet)(CPUHexagonState *env) { @@ -439,9 +416,10 @@ int32_t HELPER(vacsh_pred)(CPUHexagonState *env, return PeV; } -static void probe_store(CPUHexagonState *env, int slot, int mmu_idx) +static void probe_store(CPUHexagonState *env, int slot, int mmu_idx, + bool is_predicated) { - if (!(env->slot_cancelled & (1 << slot))) { + if (!is_predicated || !(env->slot_cancelled & (1 << slot))) { size1u_t width = env->mem_log_stores[slot].width; target_ulong va = env->mem_log_stores[slot].va; uintptr_t ra = GETPC(); @@ -461,9 +439,12 @@ void HELPER(probe_noshuf_load)(CPUHexagonState *env, target_ulong va, } /* Called during packet commit when there are two scalar stores */ -void HELPER(probe_pkt_scalar_store_s0)(CPUHexagonState *env, int mmu_idx) +void HELPER(probe_pkt_scalar_store_s0)(CPUHexagonState *env, int args) { - probe_store(env, 0, mmu_idx); + int mmu_idx = FIELD_EX32(args, PROBE_PKT_SCALAR_STORE_S0, MMU_IDX); + bool is_predicated = + FIELD_EX32(args, PROBE_PKT_SCALAR_STORE_S0, IS_PREDICATED); + probe_store(env, 0, mmu_idx, is_predicated); } void HELPER(probe_hvx_stores)(CPUHexagonState *env, int mmu_idx) @@ -510,15 +491,18 @@ void HELPER(probe_hvx_stores)(CPUHexagonState *env, int mmu_idx) void HELPER(probe_pkt_scalar_hvx_stores)(CPUHexagonState *env, int mask, int mmu_idx) { - bool has_st0 = (mask >> 0) & 1; - bool has_st1 = (mask >> 1) & 1; - bool has_hvx_stores = (mask >> 2) & 1; + bool has_st0 = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_ST0); + bool has_st1 = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_ST1); + bool has_hvx_stores = + FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_HVX_STORES); + bool s0_is_pred = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, S0_IS_PRED); + bool s1_is_pred = FIELD_EX32(mask, PROBE_PKT_SCALAR_HVX_STORES, S1_IS_PRED); if (has_st0) { - probe_store(env, 0, mmu_idx); + probe_store(env, 0, mmu_idx, s0_is_pred); } if (has_st1) { - probe_store(env, 1, mmu_idx); + probe_store(env, 1, mmu_idx, s1_is_pred); } if (has_hvx_stores) { HELPER(probe_hvx_stores)(env, mmu_idx); @@ -1193,7 +1177,7 @@ float32 HELPER(sffms)(CPUHexagonState *env, float32 RxV, { float32 neg_RsV; arch_fpop_start(env); - neg_RsV = float32_sub(float32_zero, RsV, &env->fp_status); + neg_RsV = float32_set_sign(RsV, float32_is_neg(RsV) ? 0 : 1); RxV = internal_fmafx(neg_RsV, RtV, RxV, 0, &env->fp_status); arch_fpop_end(env); return RxV; @@ -1468,12 +1452,6 @@ void HELPER(vwhist128qm)(CPUHexagonState *env, int32_t uiV) } } -void cancel_slot(CPUHexagonState *env, uint32_t slot) -{ - HEX_DEBUG_LOG("Slot %d cancelled\n", slot); - env->slot_cancelled |= (1 << slot); -} - /* These macros can be referenced in the generated helper functions */ #define warn(...) /* Nothing */ #define fatal(...) g_assert_not_reached(); diff --git a/target/hexagon/op_helper.h b/target/hexagon/op_helper.h index 02347edee8..34b3a53975 100644 --- a/target/hexagon/op_helper.h +++ b/target/hexagon/op_helper.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2021 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,7 +19,6 @@ #define HEXAGON_OP_HELPER_H /* Misc functions */ -void cancel_slot(CPUHexagonState *env, uint32_t slot); void write_new_pc(CPUHexagonState *env, bool pkt_has_multi_cof, target_ulong addr); uint8_t mem_load1(CPUHexagonState *env, uint32_t slot, target_ulong vaddr); diff --git a/target/hexagon/translate.c b/target/hexagon/translate.c index 93fd1b55e3..665476ab48 100644 --- a/target/hexagon/translate.c +++ b/target/hexagon/translate.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,6 +29,15 @@ #include "translate.h" #include "printinsn.h" +#include "analyze_funcs_generated.c.inc" + +typedef void (*AnalyzeInsn)(DisasContext *ctx); +static const AnalyzeInsn opcode_analyze[XX_LAST_OPCODE] = { +#define OPCODE(X) [X] = analyze_##X +#include "opcodes_def_generated.h.inc" +#undef OPCODE +}; + TCGv hex_gpr[TOTAL_PER_THREAD_REGS]; TCGv hex_pred[NUM_PREGS]; TCGv hex_this_PC; @@ -47,8 +56,6 @@ TCGv hex_dczero_addr; TCGv hex_llsc_addr; TCGv hex_llsc_val; TCGv_i64 hex_llsc_val_i64; -TCGv hex_VRegs_updated; -TCGv hex_QRegs_updated; TCGv hex_vstore_addr[VSTORES_MAX]; TCGv hex_vstore_size[VSTORES_MAX]; TCGv hex_vstore_pending[VSTORES_MAX]; @@ -239,7 +246,15 @@ static bool check_for_attrib(Packet *pkt, int attrib) static bool need_slot_cancelled(Packet *pkt) { - return check_for_attrib(pkt, A_CONDEXEC); + /* We only need slot_cancelled for conditional store instructions */ + for (int i = 0; i < pkt->num_insns; i++) { + uint16_t opcode = pkt->insn[i].opcode; + if (GET_ATTRIB(opcode, A_CONDEXEC) && + GET_ATTRIB(opcode, A_SCALAR_STORE)) { + return true; + } + } + return false; } static bool need_pred_written(Packet *pkt) @@ -265,6 +280,77 @@ static bool need_next_PC(DisasContext *ctx) return false; } +/* + * The opcode_analyze functions mark most of the writes in a packet + * However, there are some implicit writes marked as attributes + * of the applicable instructions. + */ +static void mark_implicit_reg_write(DisasContext *ctx, int attrib, int rnum) +{ + uint16_t opcode = ctx->insn->opcode; + if (GET_ATTRIB(opcode, attrib)) { + /* + * USR is used to set overflow and FP exceptions, + * so treat it as conditional + */ + bool is_predicated = GET_ATTRIB(opcode, A_CONDEXEC) || + rnum == HEX_REG_USR; + + /* LC0/LC1 is conditionally written by endloop instructions */ + if ((rnum == HEX_REG_LC0 || rnum == HEX_REG_LC1) && + (opcode == J2_endloop0 || + opcode == J2_endloop1 || + opcode == J2_endloop01)) { + is_predicated = true; + } + + ctx_log_reg_write(ctx, rnum, is_predicated); + } +} + +static void mark_implicit_reg_writes(DisasContext *ctx) +{ + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_FP, HEX_REG_FP); + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_SP, HEX_REG_SP); + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_LR, HEX_REG_LR); + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_LC0, HEX_REG_LC0); + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_SA0, HEX_REG_SA0); + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_LC1, HEX_REG_LC1); + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_SA1, HEX_REG_SA1); + mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_USR, HEX_REG_USR); + mark_implicit_reg_write(ctx, A_FPOP, HEX_REG_USR); +} + +static void mark_implicit_pred_write(DisasContext *ctx, int attrib, int pnum) +{ + if (GET_ATTRIB(ctx->insn->opcode, attrib)) { + ctx_log_pred_write(ctx, pnum); + } +} + +static void mark_implicit_pred_writes(DisasContext *ctx) +{ + mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P0, 0); + mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P1, 1); + mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P2, 2); + mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P3, 3); +} + +static void analyze_packet(DisasContext *ctx) +{ + Packet *pkt = ctx->pkt; + ctx->need_pkt_has_store_s1 = false; + for (int i = 0; i < pkt->num_insns; i++) { + Insn *insn = &pkt->insn[i]; + ctx->insn = insn; + if (opcode_analyze[insn->opcode]) { + opcode_analyze[insn->opcode](ctx); + } + mark_implicit_reg_writes(ctx); + mark_implicit_pred_writes(ctx); + } +} + static void gen_start_packet(DisasContext *ctx) { Packet *pkt = ctx->pkt; @@ -275,6 +361,7 @@ static void gen_start_packet(DisasContext *ctx) ctx->next_PC = next_PC; ctx->reg_log_idx = 0; bitmap_zero(ctx->regs_written, TOTAL_PER_THREAD_REGS); + bitmap_zero(ctx->predicated_regs, TOTAL_PER_THREAD_REGS); ctx->preg_log_idx = 0; bitmap_zero(ctx->pregs_written, NUM_PREGS); ctx->future_vregs_idx = 0; @@ -283,14 +370,27 @@ static void gen_start_packet(DisasContext *ctx) bitmap_zero(ctx->vregs_updated_tmp, NUM_VREGS); bitmap_zero(ctx->vregs_updated, NUM_VREGS); bitmap_zero(ctx->vregs_select, NUM_VREGS); + bitmap_zero(ctx->predicated_future_vregs, NUM_VREGS); + bitmap_zero(ctx->predicated_tmp_vregs, NUM_VREGS); ctx->qreg_log_idx = 0; for (i = 0; i < STORES_MAX; i++) { ctx->store_width[i] = 0; } - tcg_gen_movi_tl(hex_pkt_has_store_s1, pkt->pkt_has_store_s1); ctx->s1_store_processed = false; ctx->pre_commit = true; + analyze_packet(ctx); + + if (ctx->need_pkt_has_store_s1) { + tcg_gen_movi_tl(hex_pkt_has_store_s1, pkt->pkt_has_store_s1); + } + + /* + * pregs_written is used both in the analyze phase as well as the code + * gen phase, so clear it again. + */ + bitmap_zero(ctx->pregs_written, NUM_PREGS); + if (HEX_DEBUG) { /* Handy place to set a breakpoint before the packet executes */ gen_helper_debug_start_packet(cpu_env); @@ -313,9 +413,42 @@ static void gen_start_packet(DisasContext *ctx) tcg_gen_movi_tl(hex_pred_written, 0); } - if (pkt->pkt_has_hvx) { - tcg_gen_movi_tl(hex_VRegs_updated, 0); - tcg_gen_movi_tl(hex_QRegs_updated, 0); + /* Preload the predicated registers into hex_new_value[i] */ + if (!bitmap_empty(ctx->predicated_regs, TOTAL_PER_THREAD_REGS)) { + int i = find_first_bit(ctx->predicated_regs, TOTAL_PER_THREAD_REGS); + while (i < TOTAL_PER_THREAD_REGS) { + tcg_gen_mov_tl(hex_new_value[i], hex_gpr[i]); + i = find_next_bit(ctx->predicated_regs, TOTAL_PER_THREAD_REGS, + i + 1); + } + } + + /* Preload the predicated HVX registers into future_VRegs and tmp_VRegs */ + if (!bitmap_empty(ctx->predicated_future_vregs, NUM_VREGS)) { + int i = find_first_bit(ctx->predicated_future_vregs, NUM_VREGS); + while (i < NUM_VREGS) { + const intptr_t VdV_off = + ctx_future_vreg_off(ctx, i, 1, true); + intptr_t src_off = offsetof(CPUHexagonState, VRegs[i]); + tcg_gen_gvec_mov(MO_64, VdV_off, + src_off, + sizeof(MMVector), + sizeof(MMVector)); + i = find_next_bit(ctx->predicated_future_vregs, NUM_VREGS, i + 1); + } + } + if (!bitmap_empty(ctx->predicated_tmp_vregs, NUM_VREGS)) { + int i = find_first_bit(ctx->predicated_tmp_vregs, NUM_VREGS); + while (i < NUM_VREGS) { + const intptr_t VdV_off = + ctx_tmp_vreg_off(ctx, i, 1, true); + intptr_t src_off = offsetof(CPUHexagonState, VRegs[i]); + tcg_gen_gvec_mov(MO_64, VdV_off, + src_off, + sizeof(MMVector), + sizeof(MMVector)); + i = find_next_bit(ctx->predicated_tmp_vregs, NUM_VREGS, i + 1); + } } } @@ -336,66 +469,6 @@ bool is_gather_store_insn(DisasContext *ctx) return false; } -/* - * The LOG_*_WRITE macros mark most of the writes in a packet - * However, there are some implicit writes marked as attributes - * of the applicable instructions. - */ -static void mark_implicit_reg_write(DisasContext *ctx, int attrib, int rnum) -{ - uint16_t opcode = ctx->insn->opcode; - if (GET_ATTRIB(opcode, attrib)) { - /* - * USR is used to set overflow and FP exceptions, - * so treat it as conditional - */ - bool is_predicated = GET_ATTRIB(opcode, A_CONDEXEC) || - rnum == HEX_REG_USR; - - /* LC0/LC1 is conditionally written by endloop instructions */ - if ((rnum == HEX_REG_LC0 || rnum == HEX_REG_LC1) && - (opcode == J2_endloop0 || - opcode == J2_endloop1 || - opcode == J2_endloop01)) { - is_predicated = true; - } - - if (is_predicated && !is_preloaded(ctx, rnum)) { - tcg_gen_mov_tl(hex_new_value[rnum], hex_gpr[rnum]); - } - - ctx_log_reg_write(ctx, rnum); - } -} - -static void mark_implicit_pred_write(DisasContext *ctx, int attrib, int pnum) -{ - if (GET_ATTRIB(ctx->insn->opcode, attrib)) { - ctx_log_pred_write(ctx, pnum); - } -} - -static void mark_implicit_reg_writes(DisasContext *ctx) -{ - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_FP, HEX_REG_FP); - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_SP, HEX_REG_SP); - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_LR, HEX_REG_LR); - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_LC0, HEX_REG_LC0); - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_SA0, HEX_REG_SA0); - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_LC1, HEX_REG_LC1); - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_SA1, HEX_REG_SA1); - mark_implicit_reg_write(ctx, A_IMPLICIT_WRITES_USR, HEX_REG_USR); - mark_implicit_reg_write(ctx, A_FPOP, HEX_REG_USR); -} - -static void mark_implicit_pred_writes(DisasContext *ctx) -{ - mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P0, 0); - mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P1, 1); - mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P2, 2); - mark_implicit_pred_write(ctx, A_IMPLICIT_WRITES_P3, 3); -} - static void mark_store_width(DisasContext *ctx) { uint16_t opcode = ctx->insn->opcode; @@ -423,9 +496,7 @@ static void mark_store_width(DisasContext *ctx) static void gen_insn(DisasContext *ctx) { if (ctx->insn->generate) { - mark_implicit_reg_writes(ctx); ctx->insn->generate(ctx); - mark_implicit_pred_writes(ctx); mark_store_width(ctx); } else { gen_exception_end_tb(ctx, HEX_EXCP_INVALID_OPCODE); @@ -646,65 +717,31 @@ static void gen_commit_hvx(DisasContext *ctx) /* * for (i = 0; i < ctx->vreg_log_idx; i++) { * int rnum = ctx->vreg_log[i]; - * if (ctx->vreg_is_predicated[i]) { - * if (env->VRegs_updated & (1 << rnum)) { - * env->VRegs[rnum] = env->future_VRegs[rnum]; - * } - * } else { - * env->VRegs[rnum] = env->future_VRegs[rnum]; - * } + * env->VRegs[rnum] = env->future_VRegs[rnum]; * } */ for (i = 0; i < ctx->vreg_log_idx; i++) { int rnum = ctx->vreg_log[i]; - bool is_predicated = ctx->vreg_is_predicated[i]; intptr_t dstoff = offsetof(CPUHexagonState, VRegs[rnum]); intptr_t srcoff = ctx_future_vreg_off(ctx, rnum, 1, false); size_t size = sizeof(MMVector); - if (is_predicated) { - TCGv cmp = tcg_temp_new(); - TCGLabel *label_skip = gen_new_label(); - - tcg_gen_andi_tl(cmp, hex_VRegs_updated, 1 << rnum); - tcg_gen_brcondi_tl(TCG_COND_EQ, cmp, 0, label_skip); - tcg_gen_gvec_mov(MO_64, dstoff, srcoff, size, size); - gen_set_label(label_skip); - } else { - tcg_gen_gvec_mov(MO_64, dstoff, srcoff, size, size); - } + tcg_gen_gvec_mov(MO_64, dstoff, srcoff, size, size); } /* * for (i = 0; i < ctx->qreg_log_idx; i++) { * int rnum = ctx->qreg_log[i]; - * if (ctx->qreg_is_predicated[i]) { - * if (env->QRegs_updated) & (1 << rnum)) { - * env->QRegs[rnum] = env->future_QRegs[rnum]; - * } - * } else { - * env->QRegs[rnum] = env->future_QRegs[rnum]; - * } + * env->QRegs[rnum] = env->future_QRegs[rnum]; * } */ for (i = 0; i < ctx->qreg_log_idx; i++) { int rnum = ctx->qreg_log[i]; - bool is_predicated = ctx->qreg_is_predicated[i]; intptr_t dstoff = offsetof(CPUHexagonState, QRegs[rnum]); intptr_t srcoff = offsetof(CPUHexagonState, future_QRegs[rnum]); size_t size = sizeof(MMQReg); - if (is_predicated) { - TCGv cmp = tcg_temp_new(); - TCGLabel *label_skip = gen_new_label(); - - tcg_gen_andi_tl(cmp, hex_QRegs_updated, 1 << rnum); - tcg_gen_brcondi_tl(TCG_COND_EQ, cmp, 0, label_skip); - tcg_gen_gvec_mov(MO_64, dstoff, srcoff, size, size); - gen_set_label(label_skip); - } else { - tcg_gen_gvec_mov(MO_64, dstoff, srcoff, size, size); - } + tcg_gen_gvec_mov(MO_64, dstoff, srcoff, size, size); } if (pkt_has_hvx_store(ctx->pkt)) { @@ -775,13 +812,27 @@ static void gen_commit_packet(DisasContext *ctx) TCGv mask_tcgv; if (has_store_s0) { - mask |= (1 << 0); + mask = + FIELD_DP32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_ST0, 1); } if (has_store_s1) { - mask |= (1 << 1); + mask = + FIELD_DP32(mask, PROBE_PKT_SCALAR_HVX_STORES, HAS_ST1, 1); } if (has_hvx_store) { - mask |= (1 << 2); + mask = + FIELD_DP32(mask, PROBE_PKT_SCALAR_HVX_STORES, + HAS_HVX_STORES, 1); + } + if (has_store_s0 && slot_is_predicated(pkt, 0)) { + mask = + FIELD_DP32(mask, PROBE_PKT_SCALAR_HVX_STORES, + S0_IS_PRED, 1); + } + if (has_store_s1 && slot_is_predicated(pkt, 1)) { + mask = + FIELD_DP32(mask, PROBE_PKT_SCALAR_HVX_STORES, + S1_IS_PRED, 1); } mask_tcgv = tcg_constant_tl(mask); gen_helper_probe_pkt_scalar_hvx_stores(cpu_env, mask_tcgv, mem_idx); @@ -791,8 +842,15 @@ static void gen_commit_packet(DisasContext *ctx) * process_store_log will execute the slot 1 store first, * so we only have to probe the store in slot 0 */ - TCGv mem_idx = tcg_constant_tl(ctx->mem_idx); - gen_helper_probe_pkt_scalar_store_s0(cpu_env, mem_idx); + int args = 0; + args = + FIELD_DP32(args, PROBE_PKT_SCALAR_STORE_S0, MMU_IDX, ctx->mem_idx); + if (slot_is_predicated(pkt, 0)) { + args = + FIELD_DP32(args, PROBE_PKT_SCALAR_STORE_S0, IS_PREDICATED, 1); + } + TCGv args_tcgv = tcg_constant_tl(args); + gen_helper_probe_pkt_scalar_store_s0(cpu_env, args_tcgv); } process_store_log(ctx); @@ -1029,10 +1087,6 @@ void hexagon_translate_init(void) offsetof(CPUHexagonState, llsc_val), "llsc_val"); hex_llsc_val_i64 = tcg_global_mem_new_i64(cpu_env, offsetof(CPUHexagonState, llsc_val_i64), "llsc_val_i64"); - hex_VRegs_updated = tcg_global_mem_new(cpu_env, - offsetof(CPUHexagonState, VRegs_updated), "VRegs_updated"); - hex_QRegs_updated = tcg_global_mem_new(cpu_env, - offsetof(CPUHexagonState, QRegs_updated), "QRegs_updated"); for (i = 0; i < STORES_MAX; i++) { snprintf(store_addr_names[i], NAME_LEN, "store_addr_%d", i); hex_store_addr[i] = tcg_global_mem_new(cpu_env, diff --git a/target/hexagon/translate.h b/target/hexagon/translate.h index d971f4f095..db832b0f88 100644 --- a/target/hexagon/translate.h +++ b/target/hexagon/translate.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -38,6 +38,7 @@ typedef struct DisasContext { int reg_log[REG_WRITES_MAX]; int reg_log_idx; DECLARE_BITMAP(regs_written, TOTAL_PER_THREAD_REGS); + DECLARE_BITMAP(predicated_regs, TOTAL_PER_THREAD_REGS); int preg_log[PRED_WRITES_MAX]; int preg_log_idx; DECLARE_BITMAP(pregs_written, NUM_PREGS); @@ -48,52 +49,54 @@ typedef struct DisasContext { int tmp_vregs_idx; int tmp_vregs_num[VECTOR_TEMPS_MAX]; int vreg_log[NUM_VREGS]; - bool vreg_is_predicated[NUM_VREGS]; int vreg_log_idx; DECLARE_BITMAP(vregs_updated_tmp, NUM_VREGS); DECLARE_BITMAP(vregs_updated, NUM_VREGS); DECLARE_BITMAP(vregs_select, NUM_VREGS); + DECLARE_BITMAP(predicated_future_vregs, NUM_VREGS); + DECLARE_BITMAP(predicated_tmp_vregs, NUM_VREGS); int qreg_log[NUM_QREGS]; - bool qreg_is_predicated[NUM_QREGS]; int qreg_log_idx; bool pre_commit; TCGCond branch_cond; target_ulong branch_dest; bool is_tight_loop; + bool need_pkt_has_store_s1; } DisasContext; -static inline void ctx_log_reg_write(DisasContext *ctx, int rnum) -{ - if (test_bit(rnum, ctx->regs_written)) { - HEX_DEBUG_LOG("WARNING: Multiple writes to r%d\n", rnum); - } - ctx->reg_log[ctx->reg_log_idx] = rnum; - ctx->reg_log_idx++; - set_bit(rnum, ctx->regs_written); -} - -static inline void ctx_log_reg_write_pair(DisasContext *ctx, int rnum) -{ - ctx_log_reg_write(ctx, rnum); - ctx_log_reg_write(ctx, rnum + 1); -} - static inline void ctx_log_pred_write(DisasContext *ctx, int pnum) { - ctx->preg_log[ctx->preg_log_idx] = pnum; - ctx->preg_log_idx++; - set_bit(pnum, ctx->pregs_written); + if (!test_bit(pnum, ctx->pregs_written)) { + ctx->preg_log[ctx->preg_log_idx] = pnum; + ctx->preg_log_idx++; + set_bit(pnum, ctx->pregs_written); + } } -static inline bool is_preloaded(DisasContext *ctx, int num) +static inline void ctx_log_reg_write(DisasContext *ctx, int rnum, + bool is_predicated) { - return test_bit(num, ctx->regs_written); + if (rnum == HEX_REG_P3_0_ALIASED) { + for (int i = 0; i < NUM_PREGS; i++) { + ctx_log_pred_write(ctx, i); + } + } else { + if (!test_bit(rnum, ctx->regs_written)) { + ctx->reg_log[ctx->reg_log_idx] = rnum; + ctx->reg_log_idx++; + set_bit(rnum, ctx->regs_written); + } + if (is_predicated) { + set_bit(rnum, ctx->predicated_regs); + } + } } -static inline bool is_vreg_preloaded(DisasContext *ctx, int num) +static inline void ctx_log_reg_write_pair(DisasContext *ctx, int rnum, + bool is_predicated) { - return test_bit(num, ctx->vregs_updated) || - test_bit(num, ctx->vregs_updated_tmp); + ctx_log_reg_write(ctx, rnum, is_predicated); + ctx_log_reg_write(ctx, rnum + 1, is_predicated); } intptr_t ctx_future_vreg_off(DisasContext *ctx, int regnum, @@ -106,17 +109,25 @@ static inline void ctx_log_vreg_write(DisasContext *ctx, bool is_predicated) { if (type != EXT_TMP) { - ctx->vreg_log[ctx->vreg_log_idx] = rnum; - ctx->vreg_is_predicated[ctx->vreg_log_idx] = is_predicated; - ctx->vreg_log_idx++; + if (!test_bit(rnum, ctx->vregs_updated)) { + ctx->vreg_log[ctx->vreg_log_idx] = rnum; + ctx->vreg_log_idx++; + set_bit(rnum, ctx->vregs_updated); + } set_bit(rnum, ctx->vregs_updated); + if (is_predicated) { + set_bit(rnum, ctx->predicated_future_vregs); + } } if (type == EXT_NEW) { set_bit(rnum, ctx->vregs_select); } if (type == EXT_TMP) { set_bit(rnum, ctx->vregs_updated_tmp); + if (is_predicated) { + set_bit(rnum, ctx->predicated_tmp_vregs); + } } } @@ -129,10 +140,9 @@ static inline void ctx_log_vreg_write_pair(DisasContext *ctx, } static inline void ctx_log_qreg_write(DisasContext *ctx, - int rnum, bool is_predicated) + int rnum) { ctx->qreg_log[ctx->qreg_log_idx] = rnum; - ctx->qreg_is_predicated[ctx->qreg_log_idx] = is_predicated; ctx->qreg_log_idx++; } @@ -153,12 +163,20 @@ extern TCGv hex_dczero_addr; extern TCGv hex_llsc_addr; extern TCGv hex_llsc_val; extern TCGv_i64 hex_llsc_val_i64; -extern TCGv hex_VRegs_updated; -extern TCGv hex_QRegs_updated; extern TCGv hex_vstore_addr[VSTORES_MAX]; extern TCGv hex_vstore_size[VSTORES_MAX]; extern TCGv hex_vstore_pending[VSTORES_MAX]; bool is_gather_store_insn(DisasContext *ctx); void process_store(DisasContext *ctx, int slot_num); + +FIELD(PROBE_PKT_SCALAR_STORE_S0, MMU_IDX, 0, 2) +FIELD(PROBE_PKT_SCALAR_STORE_S0, IS_PREDICATED, 2, 1) + +FIELD(PROBE_PKT_SCALAR_HVX_STORES, HAS_ST0, 0, 1) +FIELD(PROBE_PKT_SCALAR_HVX_STORES, HAS_ST1, 1, 1) +FIELD(PROBE_PKT_SCALAR_HVX_STORES, HAS_HVX_STORES, 2, 1) +FIELD(PROBE_PKT_SCALAR_HVX_STORES, S0_IS_PRED, 3, 1) +FIELD(PROBE_PKT_SCALAR_HVX_STORES, S1_IS_PRED, 4, 1) + #endif diff --git a/tests/avocado/igb.py b/tests/avocado/igb.py new file mode 100644 index 0000000000..abf5dfa07f --- /dev/null +++ b/tests/avocado/igb.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# ethtool tests for igb registers, interrupts, etc + +from avocado_qemu import LinuxTest + +class IGB(LinuxTest): + """ + :avocado: tags=accel:kvm + :avocado: tags=arch:x86_64 + :avocado: tags=distro:fedora + :avocado: tags=distro_version:31 + :avocado: tags=machine:q35 + """ + + timeout = 180 + + def test(self): + self.require_accelerator('kvm') + kernel_url = self.distro.pxeboot_url + 'vmlinuz' + kernel_hash = '5b6f6876e1b5bda314f93893271da0d5777b1f3c' + kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash) + initrd_url = self.distro.pxeboot_url + 'initrd.img' + initrd_hash = 'dd0340a1b39bd28f88532babd4581c67649ec5b1' + initrd_path = self.fetch_asset(initrd_url, asset_hash=initrd_hash) + + # Ideally we want to test MSI as well, but it is blocked by a bug + # fixed with: + # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=28e96556baca7056d11d9fb3cdd0aba4483e00d8 + kernel_params = self.distro.default_kernel_params + ' pci=nomsi' + + self.vm.add_args('-kernel', kernel_path, + '-initrd', initrd_path, + '-append', kernel_params, + '-accel', 'kvm', + '-device', 'igb') + self.launch_and_wait() + self.ssh_command('dnf -y install ethtool') + self.ssh_command('ethtool -t eth1 offline') diff --git a/tests/data/acpi/pc/DSDT b/tests/data/acpi/pc/DSDT index 0b475fb5a9..32d255cfc0 100644 --- a/tests/data/acpi/pc/DSDT +++ b/tests/data/acpi/pc/DSDT Binary files differdiff --git a/tests/data/acpi/pc/DSDT.acpierst b/tests/data/acpi/pc/DSDT.acpierst index 17ef7caeb6..33e872b2fa 100644 --- a/tests/data/acpi/pc/DSDT.acpierst +++ b/tests/data/acpi/pc/DSDT.acpierst Binary files differdiff --git a/tests/data/acpi/pc/DSDT.acpihmat b/tests/data/acpi/pc/DSDT.acpihmat index 675b674eaa..cd84abc1b1 100644 --- a/tests/data/acpi/pc/DSDT.acpihmat +++ b/tests/data/acpi/pc/DSDT.acpihmat Binary files differdiff --git a/tests/data/acpi/pc/DSDT.bridge b/tests/data/acpi/pc/DSDT.bridge index c1ce061366..69a73ea2a6 100644 --- a/tests/data/acpi/pc/DSDT.bridge +++ b/tests/data/acpi/pc/DSDT.bridge Binary files differdiff --git a/tests/data/acpi/pc/DSDT.cphp b/tests/data/acpi/pc/DSDT.cphp index 754ab854dc..20379056b3 100644 --- a/tests/data/acpi/pc/DSDT.cphp +++ b/tests/data/acpi/pc/DSDT.cphp Binary files differdiff --git a/tests/data/acpi/pc/DSDT.dimmpxm b/tests/data/acpi/pc/DSDT.dimmpxm index 170503336b..435496e836 100644 --- a/tests/data/acpi/pc/DSDT.dimmpxm +++ b/tests/data/acpi/pc/DSDT.dimmpxm Binary files differdiff --git a/tests/data/acpi/pc/DSDT.hpbridge b/tests/data/acpi/pc/DSDT.hpbridge index 834c27002e..b6eafab250 100644 --- a/tests/data/acpi/pc/DSDT.hpbridge +++ b/tests/data/acpi/pc/DSDT.hpbridge Binary files differdiff --git a/tests/data/acpi/pc/DSDT.hpbrroot b/tests/data/acpi/pc/DSDT.hpbrroot index a71ed4fbaa..a4073f36d6 100644 --- a/tests/data/acpi/pc/DSDT.hpbrroot +++ b/tests/data/acpi/pc/DSDT.hpbrroot Binary files differdiff --git a/tests/data/acpi/pc/DSDT.ipmikcs b/tests/data/acpi/pc/DSDT.ipmikcs index dd71356027..06aa7bfdec 100644 --- a/tests/data/acpi/pc/DSDT.ipmikcs +++ b/tests/data/acpi/pc/DSDT.ipmikcs Binary files differdiff --git a/tests/data/acpi/pc/DSDT.memhp b/tests/data/acpi/pc/DSDT.memhp index 2f895e9b38..10a0e44d61 100644 --- a/tests/data/acpi/pc/DSDT.memhp +++ b/tests/data/acpi/pc/DSDT.memhp Binary files differdiff --git a/tests/data/acpi/pc/DSDT.nohpet b/tests/data/acpi/pc/DSDT.nohpet index c012b63ace..6905312d82 100644 --- a/tests/data/acpi/pc/DSDT.nohpet +++ b/tests/data/acpi/pc/DSDT.nohpet Binary files differdiff --git a/tests/data/acpi/pc/DSDT.numamem b/tests/data/acpi/pc/DSDT.numamem index f2ef4b9729..59e31338ee 100644 --- a/tests/data/acpi/pc/DSDT.numamem +++ b/tests/data/acpi/pc/DSDT.numamem Binary files differdiff --git a/tests/data/acpi/pc/DSDT.roothp b/tests/data/acpi/pc/DSDT.roothp index 657c8263f0..448d596cf4 100644 --- a/tests/data/acpi/pc/DSDT.roothp +++ b/tests/data/acpi/pc/DSDT.roothp Binary files differdiff --git a/tests/data/acpi/q35/DSDT b/tests/data/acpi/q35/DSDT index d68c472b46..720e8cbbbb 100644 --- a/tests/data/acpi/q35/DSDT +++ b/tests/data/acpi/q35/DSDT Binary files differdiff --git a/tests/data/acpi/q35/DSDT.acpierst b/tests/data/acpi/q35/DSDT.acpierst index de7ae27125..f26b1f2a19 100644 --- a/tests/data/acpi/q35/DSDT.acpierst +++ b/tests/data/acpi/q35/DSDT.acpierst Binary files differdiff --git a/tests/data/acpi/q35/DSDT.acpihmat b/tests/data/acpi/q35/DSDT.acpihmat index 48e2862257..86771f1746 100644 --- a/tests/data/acpi/q35/DSDT.acpihmat +++ b/tests/data/acpi/q35/DSDT.acpihmat Binary files differdiff --git a/tests/data/acpi/q35/DSDT.acpihmat-noinitiator b/tests/data/acpi/q35/DSDT.acpihmat-noinitiator index 30a4aa2ec8..a894a2d16c 100644 --- a/tests/data/acpi/q35/DSDT.acpihmat-noinitiator +++ b/tests/data/acpi/q35/DSDT.acpihmat-noinitiator Binary files differdiff --git a/tests/data/acpi/q35/DSDT.applesmc b/tests/data/acpi/q35/DSDT.applesmc index 84e2b5cbc4..276ae1df51 100644 --- a/tests/data/acpi/q35/DSDT.applesmc +++ b/tests/data/acpi/q35/DSDT.applesmc Binary files differdiff --git a/tests/data/acpi/q35/DSDT.bridge b/tests/data/acpi/q35/DSDT.bridge index e411d40fd1..9f8a208aaa 100644 --- a/tests/data/acpi/q35/DSDT.bridge +++ b/tests/data/acpi/q35/DSDT.bridge Binary files differdiff --git a/tests/data/acpi/q35/DSDT.core-count2 b/tests/data/acpi/q35/DSDT.core-count2 index 0603db8cc6..2ec11fe3c3 100644 --- a/tests/data/acpi/q35/DSDT.core-count2 +++ b/tests/data/acpi/q35/DSDT.core-count2 Binary files differdiff --git a/tests/data/acpi/q35/DSDT.cphp b/tests/data/acpi/q35/DSDT.cphp index beeb83c33b..612c85b1b4 100644 --- a/tests/data/acpi/q35/DSDT.cphp +++ b/tests/data/acpi/q35/DSDT.cphp Binary files differdiff --git a/tests/data/acpi/q35/DSDT.cxl b/tests/data/acpi/q35/DSDT.cxl index 4586b9a18b..f049f414f0 100644 --- a/tests/data/acpi/q35/DSDT.cxl +++ b/tests/data/acpi/q35/DSDT.cxl Binary files differdiff --git a/tests/data/acpi/q35/DSDT.dimmpxm b/tests/data/acpi/q35/DSDT.dimmpxm index 99a93e12a7..23dabeacb0 100644 --- a/tests/data/acpi/q35/DSDT.dimmpxm +++ b/tests/data/acpi/q35/DSDT.dimmpxm Binary files differdiff --git a/tests/data/acpi/q35/DSDT.ipmibt b/tests/data/acpi/q35/DSDT.ipmibt index 7f7601dbff..541bb70522 100644 --- a/tests/data/acpi/q35/DSDT.ipmibt +++ b/tests/data/acpi/q35/DSDT.ipmibt Binary files differdiff --git a/tests/data/acpi/q35/DSDT.ipmismbus b/tests/data/acpi/q35/DSDT.ipmismbus index 6c5d1afe44..e2d57a3318 100644 --- a/tests/data/acpi/q35/DSDT.ipmismbus +++ b/tests/data/acpi/q35/DSDT.ipmismbus Binary files differdiff --git a/tests/data/acpi/q35/DSDT.ivrs b/tests/data/acpi/q35/DSDT.ivrs index de7ae27125..f26b1f2a19 100644 --- a/tests/data/acpi/q35/DSDT.ivrs +++ b/tests/data/acpi/q35/DSDT.ivrs Binary files differdiff --git a/tests/data/acpi/q35/DSDT.memhp b/tests/data/acpi/q35/DSDT.memhp index 79bce5c8f0..809d7e2f0f 100644 --- a/tests/data/acpi/q35/DSDT.memhp +++ b/tests/data/acpi/q35/DSDT.memhp Binary files differdiff --git a/tests/data/acpi/q35/DSDT.mmio64 b/tests/data/acpi/q35/DSDT.mmio64 index c249929add..ab3fe3c1b5 100644 --- a/tests/data/acpi/q35/DSDT.mmio64 +++ b/tests/data/acpi/q35/DSDT.mmio64 Binary files differdiff --git a/tests/data/acpi/q35/DSDT.multi-bridge b/tests/data/acpi/q35/DSDT.multi-bridge index 66b39be294..9ae8ee0b41 100644 --- a/tests/data/acpi/q35/DSDT.multi-bridge +++ b/tests/data/acpi/q35/DSDT.multi-bridge Binary files differdiff --git a/tests/data/acpi/q35/DSDT.noacpihp b/tests/data/acpi/q35/DSDT.noacpihp new file mode 100644 index 0000000000..6ab1f0e525 --- /dev/null +++ b/tests/data/acpi/q35/DSDT.noacpihp Binary files differdiff --git a/tests/data/acpi/q35/DSDT.nohpet b/tests/data/acpi/q35/DSDT.nohpet index 9ff9983a80..becb5f7cad 100644 --- a/tests/data/acpi/q35/DSDT.nohpet +++ b/tests/data/acpi/q35/DSDT.nohpet Binary files differdiff --git a/tests/data/acpi/q35/DSDT.numamem b/tests/data/acpi/q35/DSDT.numamem index 1e7c45ef3c..0cdec0b4c5 100644 --- a/tests/data/acpi/q35/DSDT.numamem +++ b/tests/data/acpi/q35/DSDT.numamem Binary files differdiff --git a/tests/data/acpi/q35/DSDT.pvpanic-isa b/tests/data/acpi/q35/DSDT.pvpanic-isa index ed47451c44..6a9904ec94 100644 --- a/tests/data/acpi/q35/DSDT.pvpanic-isa +++ b/tests/data/acpi/q35/DSDT.pvpanic-isa Binary files differdiff --git a/tests/data/acpi/q35/DSDT.tis.tpm12 b/tests/data/acpi/q35/DSDT.tis.tpm12 index efc2efc19f..628bf628f6 100644 --- a/tests/data/acpi/q35/DSDT.tis.tpm12 +++ b/tests/data/acpi/q35/DSDT.tis.tpm12 Binary files differdiff --git a/tests/data/acpi/q35/DSDT.tis.tpm2 b/tests/data/acpi/q35/DSDT.tis.tpm2 index 675339715f..35c6b08068 100644 --- a/tests/data/acpi/q35/DSDT.tis.tpm2 +++ b/tests/data/acpi/q35/DSDT.tis.tpm2 Binary files differdiff --git a/tests/data/acpi/q35/DSDT.viot b/tests/data/acpi/q35/DSDT.viot index eeb40b360f..3ad4d26b7f 100644 --- a/tests/data/acpi/q35/DSDT.viot +++ b/tests/data/acpi/q35/DSDT.viot Binary files differdiff --git a/tests/data/acpi/q35/DSDT.xapic b/tests/data/acpi/q35/DSDT.xapic index 3aa86f0724..d4a34e2351 100644 --- a/tests/data/acpi/q35/DSDT.xapic +++ b/tests/data/acpi/q35/DSDT.xapic Binary files differdiff --git a/tests/qtest/bios-tables-test.c b/tests/qtest/bios-tables-test.c index d29a4e47af..76d5100911 100644 --- a/tests/qtest/bios-tables-test.c +++ b/tests/qtest/bios-tables-test.c @@ -949,9 +949,14 @@ static void test_acpi_piix4_no_acpi_pci_hotplug(void) data.required_struct_types_len = ARRAY_SIZE(base_required_struct_types); test_acpi_one("-global PIIX4_PM.acpi-root-pci-hotplug=off " "-global PIIX4_PM.acpi-pci-hotplug-with-bridge-support=off " - "-device pci-bridge,chassis_nr=1 " - "-device pci-testdev,bus=pci.0 " - "-device pci-testdev,bus=pci.1", &data); + "-device pci-bridge,chassis_nr=1,addr=4.0 " + "-device pci-testdev,bus=pci.0,addr=5.0 " + "-device pci-testdev,bus=pci.0,addr=6.0,acpi-index=101 " + "-device pci-testdev,bus=pci.1,addr=1.0 " + "-device pci-testdev,bus=pci.1,addr=2.0,acpi-index=201 " + "-device pci-bridge,id=nhpbr,chassis_nr=2,shpc=off,addr=7.0 " + "-device pci-testdev,bus=nhpbr,addr=1.0,acpi-index=301 " + , &data); free_test_data(&data); } @@ -1002,18 +1007,42 @@ static void test_acpi_q35_tcg_bridge(void) free_test_data(&data); } +static void test_acpi_q35_tcg_no_acpi_hotplug(void) +{ + test_data data; + + memset(&data, 0, sizeof(data)); + data.machine = MACHINE_Q35; + data.variant = ".noacpihp"; + data.required_struct_types = base_required_struct_types; + data.required_struct_types_len = ARRAY_SIZE(base_required_struct_types); + test_acpi_one("-global ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off" + " -device pci-testdev,bus=pcie.0,acpi-index=101,addr=3.0" + " -device pci-bridge,chassis_nr=1,id=shpcbr,addr=4.0" + " -device pci-testdev,bus=shpcbr,addr=1.0,acpi-index=201" + " -device pci-bridge,chassis_nr=2,shpc=off,id=noshpcbr,addr=5.0" + " -device pci-testdev,bus=noshpcbr,addr=1.0,acpi-index=301" + " -device pcie-root-port,id=hprp,port=0x0,chassis=1,addr=6.0" + " -device pci-testdev,bus=hprp,acpi-index=401" + " -device pcie-root-port,id=nohprp,port=0x0,chassis=2,hotplug=off," + "addr=7.0" + " -device pci-testdev,bus=nohprp,acpi-index=501" + " -device pcie-root-port,id=nohprpint,port=0x0,chassis=3,hotplug=off," + "multifunction=on,addr=8.0" + " -device pci-testdev,bus=nohprpint,acpi-index=601,addr=8.1" + " -device pcie-root-port,id=hprp2,port=0x0,chassis=4,bus=nohprpint," + "addr=9.0" + " -device pci-testdev,bus=hprp2,acpi-index=602" + , &data); + free_test_data(&data); +} + static void test_acpi_q35_multif_bridge(void) { test_data data = { .machine = MACHINE_Q35, .variant = ".multi-bridge", }; - - if (!qtest_has_device("pcie-root-port")) { - g_test_skip("Device pcie-root-port is not available"); - goto out; - } - test_vm_prepare("-S" " -device virtio-balloon,id=balloon0,addr=0x4.0x2" " -device pcie-root-port,id=rp0,multifunction=on," @@ -1025,9 +1054,14 @@ static void test_acpi_q35_multif_bridge(void) " -device pcie-root-port,id=rphptgt2,port=0x0,chassis=6,addr=2.2" " -device pcie-root-port,id=rphptgt3,port=0x0,chassis=7,addr=2.3" " -device pci-testdev,bus=pcie.0,addr=2.4" + " -device pci-testdev,bus=pcie.0,addr=2.5,acpi-index=102" " -device pci-testdev,bus=pcie.0,addr=5.0" + " -device pci-testdev,bus=pcie.0,addr=0xf.0,acpi-index=101" " -device pci-testdev,bus=rp0,addr=0.0" - " -device pci-testdev,bus=br1", &data); + " -device pci-testdev,bus=br1" + " -device pcie-root-port,id=rpnohp,chassis=8,addr=0xA.0,hotplug=off" + " -device pcie-root-port,id=rp3,chassis=9,bus=rpnohp" + , &data); /* hotplugged bridges section */ qtest_qmp_device_add(data.qts, "pci-bridge", "hpbr1", @@ -1049,7 +1083,6 @@ static void test_acpi_q35_multif_bridge(void) /* check that reboot/reset doesn't change any ACPI tables */ qtest_qmp_send(data.qts, "{'execute':'system_reset' }"); process_acpi_tables(&data); -out: free_test_data(&data); } @@ -1403,11 +1436,6 @@ static void test_acpi_tcg_dimm_pxm(const char *machine) { test_data data; - if (!qtest_has_device("nvdimm")) { - g_test_skip("Device nvdimm is not available"); - return; - } - memset(&data, 0, sizeof(data)); data.machine = machine; data.variant = ".dimmpxm"; @@ -1456,11 +1484,6 @@ static void test_acpi_virt_tcg_memhp(void) .scan_len = 256ULL * 1024 * 1024, }; - if (!qtest_has_device("nvdimm")) { - g_test_skip("Device nvdimm is not available"); - goto out; - } - data.variant = ".memhp"; test_acpi_one(" -machine nvdimm=on" " -cpu cortex-a57" @@ -1474,7 +1497,7 @@ static void test_acpi_virt_tcg_memhp(void) " -device pc-dimm,id=dimm0,memdev=ram2,node=0" " -device nvdimm,id=dimm1,memdev=nvm0,node=1", &data); -out: + free_test_data(&data); } @@ -1492,11 +1515,6 @@ static void test_acpi_microvm_tcg(void) { test_data data; - if (!qtest_has_device("virtio-blk-device")) { - g_test_skip("Device virtio-blk-device is not available"); - return; - } - test_acpi_microvm_prepare(&data); test_acpi_one(" -machine microvm,acpi=on,ioapic2=off,rtc=off", &data); @@ -1507,11 +1525,6 @@ static void test_acpi_microvm_usb_tcg(void) { test_data data; - if (!qtest_has_device("virtio-blk-device")) { - g_test_skip("Device virtio-blk-device is not available"); - return; - } - test_acpi_microvm_prepare(&data); data.variant = ".usb"; test_acpi_one(" -machine microvm,acpi=on,ioapic2=off,usb=on,rtc=off", @@ -1523,11 +1536,6 @@ static void test_acpi_microvm_rtc_tcg(void) { test_data data; - if (!qtest_has_device("virtio-blk-device")) { - g_test_skip("Device virtio-blk-device is not available"); - return; - } - test_acpi_microvm_prepare(&data); data.variant = ".rtc"; test_acpi_one(" -machine microvm,acpi=on,ioapic2=off,rtc=on", @@ -1539,11 +1547,6 @@ static void test_acpi_microvm_pcie_tcg(void) { test_data data; - if (!qtest_has_device("virtio-blk-device")) { - g_test_skip("Device virtio-blk-device is not available"); - return; - } - test_acpi_microvm_prepare(&data); data.variant = ".pcie"; data.tcg_only = true; /* need constant host-phys-bits */ @@ -1556,11 +1559,6 @@ static void test_acpi_microvm_ioapic2_tcg(void) { test_data data; - if (!qtest_has_device("virtio-blk-device")) { - g_test_skip("Device virtio-blk-device is not available"); - return; - } - test_acpi_microvm_prepare(&data); data.variant = ".ioapic2"; test_acpi_one(" -machine microvm,acpi=on,ioapic2=on,rtc=off", @@ -1600,12 +1598,6 @@ static void test_acpi_virt_tcg_pxb(void) .ram_start = 0x40000000ULL, .scan_len = 128ULL * 1024 * 1024, }; - - if (!qtest_has_device("pcie-root-port")) { - g_test_skip("Device pcie-root-port is not available"); - goto out; - } - /* * While using -cdrom, the cdrom would auto plugged into pxb-pcie, * the reason is the bus of pxb-pcie is also root bus, it would lead @@ -1624,7 +1616,7 @@ static void test_acpi_virt_tcg_pxb(void) " -cpu cortex-a57" " -device pxb-pcie,bus_nr=128", &data); -out: + free_test_data(&data); } @@ -1812,12 +1804,6 @@ static void test_acpi_microvm_acpi_erst(void) gchar *params; test_data data; - if (!qtest_has_device("virtio-blk-device")) { - g_test_skip("Device virtio-blk-device is not available"); - g_free(tmp_path); - return; - } - test_acpi_microvm_prepare(&data); data.variant = ".pcie"; data.tcg_only = true; /* need constant host-phys-bits */ @@ -1878,11 +1864,6 @@ static void test_acpi_q35_viot(void) .variant = ".viot", }; - if (!qtest_has_device("virtio-iommu")) { - g_test_skip("Device virtio-iommu is not available"); - goto out; - } - /* * To keep things interesting, two buses bypass the IOMMU. * VIOT should only describes the other two buses. @@ -1893,7 +1874,6 @@ static void test_acpi_q35_viot(void) "-device pxb-pcie,bus_nr=0x20,id=pcie.200,bus=pcie.0,bypass_iommu=on " "-device pxb-pcie,bus_nr=0x30,id=pcie.300,bus=pcie.0", &data); -out: free_test_data(&data); } @@ -1954,10 +1934,8 @@ static void test_acpi_virt_viot(void) .scan_len = 128ULL * 1024 * 1024, }; - if (qtest_has_device("virtio-iommu")) { - test_acpi_one("-cpu cortex-a57 " - "-device virtio-iommu-pci", &data); - } + test_acpi_one("-cpu cortex-a57 " + "-device virtio-iommu-pci", &data); free_test_data(&data); } @@ -2066,11 +2044,6 @@ static void test_acpi_microvm_oem_fields(void) test_data data; char *args; - if (!qtest_has_device("virtio-blk-device")) { - g_test_skip("Device virtio-blk-device is not available"); - return; - } - test_acpi_microvm_prepare(&data); args = test_acpi_create_args(&data, @@ -2161,6 +2134,8 @@ int main(int argc, char *argv[]) test_acpi_q35_tcg_tpm12_tis); } qtest_add_func("acpi/q35/bridge", test_acpi_q35_tcg_bridge); + qtest_add_func("acpi/q35/no-acpi-hotplug", + test_acpi_q35_tcg_no_acpi_hotplug); qtest_add_func("acpi/q35/multif-bridge", test_acpi_q35_multif_bridge); qtest_add_func("acpi/q35/mmio64", test_acpi_q35_tcg_mmio64); diff --git a/tests/qtest/e1000e-test.c b/tests/qtest/e1000e-test.c index b63a4d3c91..de9738fdb7 100644 --- a/tests/qtest/e1000e-test.c +++ b/tests/qtest/e1000e-test.c @@ -27,6 +27,7 @@ #include "qemu/osdep.h" #include "libqtest-single.h" #include "libqos/pci-pc.h" +#include "net/eth.h" #include "qemu/sockets.h" #include "qemu/iov.h" #include "qemu/module.h" @@ -35,9 +36,13 @@ #include "libqos/e1000e.h" #include "hw/net/e1000_regs.h" +static const struct eth_header packet = { + .h_dest = E1000E_ADDRESS, + .h_source = E1000E_ADDRESS, +}; + static void e1000e_send_verify(QE1000E *d, int *test_sockets, QGuestAllocator *alloc) { - static const char test[] = "TEST"; struct e1000_tx_desc descr; char buffer[64]; int ret; @@ -45,7 +50,7 @@ static void e1000e_send_verify(QE1000E *d, int *test_sockets, QGuestAllocator *a /* Prepare test data buffer */ uint64_t data = guest_alloc(alloc, sizeof(buffer)); - memwrite(data, test, sizeof(test)); + memwrite(data, &packet, sizeof(packet)); /* Prepare TX descriptor */ memset(&descr, 0, sizeof(descr)); @@ -71,7 +76,7 @@ static void e1000e_send_verify(QE1000E *d, int *test_sockets, QGuestAllocator *a g_assert_cmpint(ret, == , sizeof(recv_len)); ret = recv(test_sockets[0], buffer, sizeof(buffer), 0); g_assert_cmpint(ret, ==, sizeof(buffer)); - g_assert_cmpstr(buffer, == , test); + g_assert_false(memcmp(buffer, &packet, sizeof(packet))); /* Free test data buffer */ guest_free(alloc, data); @@ -81,15 +86,15 @@ static void e1000e_receive_verify(QE1000E *d, int *test_sockets, QGuestAllocator { union e1000_rx_desc_extended descr; - char test[] = "TEST"; - int len = htonl(sizeof(test)); + struct eth_header test_iov = packet; + int len = htonl(sizeof(packet)); struct iovec iov[] = { { .iov_base = &len, .iov_len = sizeof(len), },{ - .iov_base = test, - .iov_len = sizeof(test), + .iov_base = &test_iov, + .iov_len = sizeof(packet), }, }; @@ -97,8 +102,8 @@ static void e1000e_receive_verify(QE1000E *d, int *test_sockets, QGuestAllocator int ret; /* Send a dummy packet to device's socket*/ - ret = iov_send(test_sockets[0], iov, 2, 0, sizeof(len) + sizeof(test)); - g_assert_cmpint(ret, == , sizeof(test) + sizeof(len)); + ret = iov_send(test_sockets[0], iov, 2, 0, sizeof(len) + sizeof(packet)); + g_assert_cmpint(ret, == , sizeof(packet) + sizeof(len)); /* Prepare test data buffer */ uint64_t data = guest_alloc(alloc, sizeof(buffer)); @@ -119,7 +124,7 @@ static void e1000e_receive_verify(QE1000E *d, int *test_sockets, QGuestAllocator /* Check data sent to the backend */ memread(data, buffer, sizeof(buffer)); - g_assert_cmpstr(buffer, == , test); + g_assert_false(memcmp(buffer, &packet, sizeof(packet))); /* Free test data buffer */ guest_free(alloc, data); diff --git a/tests/qtest/fuzz/generic_fuzz_configs.h b/tests/qtest/fuzz/generic_fuzz_configs.h index a825b78c14..50689da653 100644 --- a/tests/qtest/fuzz/generic_fuzz_configs.h +++ b/tests/qtest/fuzz/generic_fuzz_configs.h @@ -91,6 +91,11 @@ const generic_fuzz_config predefined_configs[] = { "-device e1000e,netdev=net0 -netdev user,id=net0", .objects = "e1000e", },{ + .name = "igb", + .args = "-M q35 -nodefaults " + "-device igb,netdev=net0 -netdev user,id=net0", + .objects = "igb", + },{ .name = "cirrus-vga", .args = "-machine q35 -nodefaults -device cirrus-vga", .objects = "cirrus*", diff --git a/tests/qtest/igb-test.c b/tests/qtest/igb-test.c new file mode 100644 index 0000000000..3d397ea697 --- /dev/null +++ b/tests/qtest/igb-test.c @@ -0,0 +1,256 @@ +/* + * QTest testcase for igb NIC + * + * Copyright (c) 2022-2023 Red Hat, Inc. + * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com) + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Akihiko Odaki <akihiko.odaki@daynix.com> + * Dmitry Fleytman <dmitry@daynix.com> + * Leonid Bloch <leonid@daynix.com> + * Yan Vugenfirer <yan@daynix.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + + +#include "qemu/osdep.h" +#include "libqtest-single.h" +#include "libqos/pci-pc.h" +#include "net/eth.h" +#include "qemu/sockets.h" +#include "qemu/iov.h" +#include "qemu/module.h" +#include "qemu/bitops.h" +#include "libqos/libqos-malloc.h" +#include "libqos/e1000e.h" +#include "hw/net/igb_regs.h" + +#ifndef _WIN32 + +static const struct eth_header packet = { + .h_dest = E1000E_ADDRESS, + .h_source = E1000E_ADDRESS, +}; + +static void igb_send_verify(QE1000E *d, int *test_sockets, QGuestAllocator *alloc) +{ + union e1000_adv_tx_desc descr; + char buffer[64]; + int ret; + uint32_t recv_len; + + /* Prepare test data buffer */ + uint64_t data = guest_alloc(alloc, sizeof(buffer)); + memwrite(data, &packet, sizeof(packet)); + + /* Prepare TX descriptor */ + memset(&descr, 0, sizeof(descr)); + descr.read.buffer_addr = cpu_to_le64(data); + descr.read.cmd_type_len = cpu_to_le32(E1000_TXD_CMD_RS | + E1000_TXD_CMD_EOP | + E1000_TXD_DTYP_D | + sizeof(buffer)); + + /* Put descriptor to the ring */ + e1000e_tx_ring_push(d, &descr); + + /* Wait for TX WB interrupt */ + e1000e_wait_isr(d, E1000E_TX0_MSG_ID); + + /* Check DD bit */ + g_assert_cmphex(le32_to_cpu(descr.wb.status) & E1000_TXD_STAT_DD, ==, + E1000_TXD_STAT_DD); + + /* Check data sent to the backend */ + ret = recv(test_sockets[0], &recv_len, sizeof(recv_len), 0); + g_assert_cmpint(ret, == , sizeof(recv_len)); + ret = recv(test_sockets[0], buffer, sizeof(buffer), 0); + g_assert_cmpint(ret, ==, sizeof(buffer)); + g_assert_false(memcmp(buffer, &packet, sizeof(packet))); + + /* Free test data buffer */ + guest_free(alloc, data); +} + +static void igb_receive_verify(QE1000E *d, int *test_sockets, QGuestAllocator *alloc) +{ + union e1000_adv_rx_desc descr; + + struct eth_header test_iov = packet; + int len = htonl(sizeof(packet)); + struct iovec iov[] = { + { + .iov_base = &len, + .iov_len = sizeof(len), + },{ + .iov_base = &test_iov, + .iov_len = sizeof(packet), + }, + }; + + char buffer[64]; + int ret; + + /* Send a dummy packet to device's socket*/ + ret = iov_send(test_sockets[0], iov, 2, 0, sizeof(len) + sizeof(packet)); + g_assert_cmpint(ret, == , sizeof(packet) + sizeof(len)); + + /* Prepare test data buffer */ + uint64_t data = guest_alloc(alloc, sizeof(buffer)); + + /* Prepare RX descriptor */ + memset(&descr, 0, sizeof(descr)); + descr.read.pkt_addr = cpu_to_le64(data); + + /* Put descriptor to the ring */ + e1000e_rx_ring_push(d, &descr); + + /* Wait for TX WB interrupt */ + e1000e_wait_isr(d, E1000E_RX0_MSG_ID); + + /* Check DD bit */ + g_assert_cmphex(le32_to_cpu(descr.wb.upper.status_error) & + E1000_RXD_STAT_DD, ==, E1000_RXD_STAT_DD); + + /* Check data sent to the backend */ + memread(data, buffer, sizeof(buffer)); + g_assert_false(memcmp(buffer, &packet, sizeof(packet))); + + /* Free test data buffer */ + guest_free(alloc, data); +} + +static void test_e1000e_init(void *obj, void *data, QGuestAllocator * alloc) +{ + /* init does nothing */ +} + +static void test_igb_tx(void *obj, void *data, QGuestAllocator * alloc) +{ + QE1000E_PCI *e1000e = obj; + QE1000E *d = &e1000e->e1000e; + QOSGraphObject *e_object = obj; + QPCIDevice *dev = e_object->get_driver(e_object, "pci-device"); + + /* FIXME: add spapr support */ + if (qpci_check_buggy_msi(dev)) { + return; + } + + igb_send_verify(d, data, alloc); +} + +static void test_igb_rx(void *obj, void *data, QGuestAllocator * alloc) +{ + QE1000E_PCI *e1000e = obj; + QE1000E *d = &e1000e->e1000e; + QOSGraphObject *e_object = obj; + QPCIDevice *dev = e_object->get_driver(e_object, "pci-device"); + + /* FIXME: add spapr support */ + if (qpci_check_buggy_msi(dev)) { + return; + } + + igb_receive_verify(d, data, alloc); +} + +static void test_igb_multiple_transfers(void *obj, void *data, + QGuestAllocator *alloc) +{ + static const long iterations = 4 * 1024; + long i; + + QE1000E_PCI *e1000e = obj; + QE1000E *d = &e1000e->e1000e; + QOSGraphObject *e_object = obj; + QPCIDevice *dev = e_object->get_driver(e_object, "pci-device"); + + /* FIXME: add spapr support */ + if (qpci_check_buggy_msi(dev)) { + return; + } + + for (i = 0; i < iterations; i++) { + igb_send_verify(d, data, alloc); + igb_receive_verify(d, data, alloc); + } + +} + +static void data_test_clear(void *sockets) +{ + int *test_sockets = sockets; + + close(test_sockets[0]); + qos_invalidate_command_line(); + close(test_sockets[1]); + g_free(test_sockets); +} + +static void *data_test_init(GString *cmd_line, void *arg) +{ + int *test_sockets = g_new(int, 2); + int ret = socketpair(PF_UNIX, SOCK_STREAM, 0, test_sockets); + g_assert_cmpint(ret, != , -1); + + g_string_append_printf(cmd_line, " -netdev socket,fd=%d,id=hs0 ", + test_sockets[1]); + + g_test_queue_destroy(data_test_clear, test_sockets); + return test_sockets; +} + +#endif + +static void *data_test_init_no_socket(GString *cmd_line, void *arg) +{ + g_string_append(cmd_line, " -netdev hubport,hubid=0,id=hs0 "); + return arg; +} + +static void test_igb_hotplug(void *obj, void *data, QGuestAllocator * alloc) +{ + QTestState *qts = global_qtest; /* TODO: get rid of global_qtest here */ + QE1000E_PCI *dev = obj; + + if (dev->pci_dev.bus->not_hotpluggable) { + g_test_skip("pci bus does not support hotplug"); + return; + } + + qtest_qmp_device_add(qts, "igb", "igb_net", "{'addr': '0x06'}"); + qpci_unplug_acpi_device_test(qts, "igb_net", 0x06); +} + +static void register_igb_test(void) +{ + QOSGraphTestOptions opts = { 0 }; + +#ifndef _WIN32 + opts.before = data_test_init, + qos_add_test("init", "igb", test_e1000e_init, &opts); + qos_add_test("tx", "igb", test_igb_tx, &opts); + qos_add_test("rx", "igb", test_igb_rx, &opts); + qos_add_test("multiple_transfers", "igb", + test_igb_multiple_transfers, &opts); +#endif + + opts.before = data_test_init_no_socket; + qos_add_test("hotplug", "igb", test_igb_hotplug, &opts); +} + +libqos_init(register_igb_test); diff --git a/tests/qtest/libqos/e1000e.c b/tests/qtest/libqos/e1000e.c index 28fb3052aa..925654c7fd 100644 --- a/tests/qtest/libqos/e1000e.c +++ b/tests/qtest/libqos/e1000e.c @@ -36,18 +36,6 @@ #define E1000E_RING_LEN (0x1000) -static void e1000e_macreg_write(QE1000E *d, uint32_t reg, uint32_t val) -{ - QE1000E_PCI *d_pci = container_of(d, QE1000E_PCI, e1000e); - qpci_io_writel(&d_pci->pci_dev, d_pci->mac_regs, reg, val); -} - -static uint32_t e1000e_macreg_read(QE1000E *d, uint32_t reg) -{ - QE1000E_PCI *d_pci = container_of(d, QE1000E_PCI, e1000e); - return qpci_io_readl(&d_pci->pci_dev, d_pci->mac_regs, reg); -} - void e1000e_tx_ring_push(QE1000E *d, void *descr) { QE1000E_PCI *d_pci = container_of(d, QE1000E_PCI, e1000e); diff --git a/tests/qtest/libqos/e1000e.h b/tests/qtest/libqos/e1000e.h index 091ce139da..30643c8094 100644 --- a/tests/qtest/libqos/e1000e.h +++ b/tests/qtest/libqos/e1000e.h @@ -25,6 +25,8 @@ #define E1000E_RX0_MSG_ID (0) #define E1000E_TX0_MSG_ID (1) +#define E1000E_ADDRESS { 0x52, 0x54, 0x00, 0x12, 0x34, 0x56 } + typedef struct QE1000E QE1000E; typedef struct QE1000E_PCI QE1000E_PCI; @@ -40,6 +42,18 @@ struct QE1000E_PCI { QE1000E e1000e; }; +static inline void e1000e_macreg_write(QE1000E *d, uint32_t reg, uint32_t val) +{ + QE1000E_PCI *d_pci = container_of(d, QE1000E_PCI, e1000e); + qpci_io_writel(&d_pci->pci_dev, d_pci->mac_regs, reg, val); +} + +static inline uint32_t e1000e_macreg_read(QE1000E *d, uint32_t reg) +{ + QE1000E_PCI *d_pci = container_of(d, QE1000E_PCI, e1000e); + return qpci_io_readl(&d_pci->pci_dev, d_pci->mac_regs, reg); +} + void e1000e_wait_isr(QE1000E *d, uint16_t msg_id); void e1000e_tx_ring_push(QE1000E *d, void *descr); void e1000e_rx_ring_push(QE1000E *d, void *descr); diff --git a/tests/qtest/libqos/igb.c b/tests/qtest/libqos/igb.c new file mode 100644 index 0000000000..12fb531bf0 --- /dev/null +++ b/tests/qtest/libqos/igb.c @@ -0,0 +1,185 @@ +/* + * libqos driver framework + * + * Copyright (c) 2022-2023 Red Hat, Inc. + * Copyright (c) 2018 Emanuele Giuseppe Esposito <e.emanuelegiuseppe@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/> + */ + +#include "qemu/osdep.h" +#include "hw/net/igb_regs.h" +#include "hw/net/mii.h" +#include "hw/pci/pci_ids.h" +#include "../libqtest.h" +#include "pci-pc.h" +#include "qemu/sockets.h" +#include "qemu/iov.h" +#include "qemu/module.h" +#include "qemu/bitops.h" +#include "libqos-malloc.h" +#include "qgraph.h" +#include "e1000e.h" + +#define IGB_IVAR_TEST_CFG \ + ((E1000E_RX0_MSG_ID | E1000_IVAR_VALID) << (igb_ivar_entry_rx(0) * 8) | \ + ((E1000E_TX0_MSG_ID | E1000_IVAR_VALID) << (igb_ivar_entry_tx(0) * 8))) + +#define E1000E_RING_LEN (0x1000) + +static void e1000e_foreach_callback(QPCIDevice *dev, int devfn, void *data) +{ + QPCIDevice *res = data; + memcpy(res, dev, sizeof(QPCIDevice)); + g_free(dev); +} + +static void e1000e_pci_destructor(QOSGraphObject *obj) +{ + QE1000E_PCI *epci = (QE1000E_PCI *) obj; + qpci_iounmap(&epci->pci_dev, epci->mac_regs); + qpci_msix_disable(&epci->pci_dev); +} + +static void igb_pci_start_hw(QOSGraphObject *obj) +{ + static const uint8_t address[] = E1000E_ADDRESS; + QE1000E_PCI *d = (QE1000E_PCI *) obj; + uint32_t val; + + /* Enable the device */ + qpci_device_enable(&d->pci_dev); + + /* Reset the device */ + val = e1000e_macreg_read(&d->e1000e, E1000_CTRL); + e1000e_macreg_write(&d->e1000e, E1000_CTRL, val | E1000_CTRL_RST | E1000_CTRL_SLU); + + /* Setup link */ + e1000e_macreg_write(&d->e1000e, E1000_MDIC, + MII_BMCR_AUTOEN | MII_BMCR_ANRESTART | + (MII_BMCR << E1000_MDIC_REG_SHIFT) | + (1 << E1000_MDIC_PHY_SHIFT) | + E1000_MDIC_OP_WRITE); + + qtest_clock_step(d->pci_dev.bus->qts, 900000000); + + /* Enable and configure MSI-X */ + qpci_msix_enable(&d->pci_dev); + e1000e_macreg_write(&d->e1000e, E1000_IVAR0, IGB_IVAR_TEST_CFG); + + /* Check the device link status */ + val = e1000e_macreg_read(&d->e1000e, E1000_STATUS); + g_assert_cmphex(val & E1000_STATUS_LU, ==, E1000_STATUS_LU); + + /* Initialize TX/RX logic */ + e1000e_macreg_write(&d->e1000e, E1000_RCTL, 0); + e1000e_macreg_write(&d->e1000e, E1000_TCTL, 0); + + e1000e_macreg_write(&d->e1000e, E1000_TDBAL(0), + (uint32_t) d->e1000e.tx_ring); + e1000e_macreg_write(&d->e1000e, E1000_TDBAH(0), + (uint32_t) (d->e1000e.tx_ring >> 32)); + e1000e_macreg_write(&d->e1000e, E1000_TDLEN(0), E1000E_RING_LEN); + e1000e_macreg_write(&d->e1000e, E1000_TDT(0), 0); + e1000e_macreg_write(&d->e1000e, E1000_TDH(0), 0); + + /* Enable transmit */ + e1000e_macreg_write(&d->e1000e, E1000_TCTL, E1000_TCTL_EN); + + e1000e_macreg_write(&d->e1000e, E1000_RDBAL(0), + (uint32_t)d->e1000e.rx_ring); + e1000e_macreg_write(&d->e1000e, E1000_RDBAH(0), + (uint32_t)(d->e1000e.rx_ring >> 32)); + e1000e_macreg_write(&d->e1000e, E1000_RDLEN(0), E1000E_RING_LEN); + e1000e_macreg_write(&d->e1000e, E1000_RDT(0), 0); + e1000e_macreg_write(&d->e1000e, E1000_RDH(0), 0); + e1000e_macreg_write(&d->e1000e, E1000_RA, + le32_to_cpu(*(uint32_t *)address)); + e1000e_macreg_write(&d->e1000e, E1000_RA + 4, + E1000_RAH_AV | E1000_RAH_POOL_1 | + le16_to_cpu(*(uint16_t *)(address + 4))); + + /* Enable receive */ + e1000e_macreg_write(&d->e1000e, E1000_RFCTL, E1000_RFCTL_EXTEN); + e1000e_macreg_write(&d->e1000e, E1000_RCTL, E1000_RCTL_EN); + + /* Enable all interrupts */ + e1000e_macreg_write(&d->e1000e, E1000_IMS, 0xFFFFFFFF); + e1000e_macreg_write(&d->e1000e, E1000_EIMS, 0xFFFFFFFF); + +} + +static void *igb_pci_get_driver(void *obj, const char *interface) +{ + QE1000E_PCI *epci = obj; + if (!g_strcmp0(interface, "igb-if")) { + return &epci->e1000e; + } + + /* implicit contains */ + if (!g_strcmp0(interface, "pci-device")) { + return &epci->pci_dev; + } + + fprintf(stderr, "%s not present in igb\n", interface); + g_assert_not_reached(); +} + +static void *igb_pci_create(void *pci_bus, QGuestAllocator *alloc, void *addr) +{ + QE1000E_PCI *d = g_new0(QE1000E_PCI, 1); + QPCIBus *bus = pci_bus; + QPCIAddress *address = addr; + + qpci_device_foreach(bus, address->vendor_id, address->device_id, + e1000e_foreach_callback, &d->pci_dev); + + /* Map BAR0 (mac registers) */ + d->mac_regs = qpci_iomap(&d->pci_dev, 0, NULL); + + /* Allocate and setup TX ring */ + d->e1000e.tx_ring = guest_alloc(alloc, E1000E_RING_LEN); + g_assert(d->e1000e.tx_ring != 0); + + /* Allocate and setup RX ring */ + d->e1000e.rx_ring = guest_alloc(alloc, E1000E_RING_LEN); + g_assert(d->e1000e.rx_ring != 0); + + d->obj.get_driver = igb_pci_get_driver; + d->obj.start_hw = igb_pci_start_hw; + d->obj.destructor = e1000e_pci_destructor; + + return &d->obj; +} + +static void igb_register_nodes(void) +{ + QPCIAddress addr = { + .vendor_id = PCI_VENDOR_ID_INTEL, + .device_id = E1000_DEV_ID_82576, + }; + + /* + * FIXME: every test using this node needs to setup a -netdev socket,id=hs0 + * otherwise QEMU is not going to start + */ + QOSGraphEdgeOptions opts = { + .extra_device_opts = "netdev=hs0", + }; + add_qpci_address(&opts, &addr); + + qos_node_create_driver("igb", igb_pci_create); + qos_node_consumes("igb", "pci-bus", &opts); +} + +libqos_init(igb_register_nodes); diff --git a/tests/qtest/libqos/meson.build b/tests/qtest/libqos/meson.build index 32f028872c..cc209a8de5 100644 --- a/tests/qtest/libqos/meson.build +++ b/tests/qtest/libqos/meson.build @@ -30,6 +30,7 @@ libqos_srcs = files( 'i2c.c', 'i2c-imx.c', 'i2c-omap.c', + 'igb.c', 'sdhci.c', 'tpci200.c', 'virtio.c', diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index 62eecf2edf..c9292b64fb 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -259,6 +259,7 @@ qos_test_ss.add( 'virtio-scsi-test.c', 'virtio-iommu-test.c', 'vmxnet3-test.c', + 'igb-test.c', ) if config_all_devices.has_key('CONFIG_VIRTIO_SERIAL') diff --git a/tests/tcg/hexagon/Makefile.target b/tests/tcg/hexagon/Makefile.target index 18e6a5969e..0d82dfa76e 100644 --- a/tests/tcg/hexagon/Makefile.target +++ b/tests/tcg/hexagon/Makefile.target @@ -1,5 +1,5 @@ ## -## Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. +## Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. ## ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by @@ -45,6 +45,10 @@ HEX_TESTS += fpstuff HEX_TESTS += overflow HEX_TESTS += signal_context HEX_TESTS += reg_mut +HEX_TESTS += vector_add_int +HEX_TESTS += scatter_gather +HEX_TESTS += hvx_misc +HEX_TESTS += hvx_histogram HEX_TESTS += test_abs HEX_TESTS += test_bitcnt @@ -78,3 +82,10 @@ TESTS += $(HEX_TESTS) usr: usr.c $(CC) $(CFLAGS) -mv67t -O2 -Wno-inline-asm -Wno-expansion-to-defined $< -o $@ $(LDFLAGS) +scatter_gather: CFLAGS += -mhvx +vector_add_int: CFLAGS += -mhvx -fvectorize +hvx_misc: CFLAGS += -mhvx +hvx_histogram: CFLAGS += -mhvx -Wno-gnu-folding-constant + +hvx_histogram: hvx_histogram.c hvx_histogram_row.S + $(CC) $(CFLAGS) $(CROSS_CC_GUEST_CFLAGS) $^ -o $@ $(LDFLAGS) diff --git a/tests/tcg/hexagon/fpstuff.c b/tests/tcg/hexagon/fpstuff.c index 56bf562a40..90ce9a6ef3 100644 --- a/tests/tcg/hexagon/fpstuff.c +++ b/tests/tcg/hexagon/fpstuff.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2020-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2020-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -40,6 +40,7 @@ const int SF_HEX_NAN = 0xffffffff; const int SF_small_neg = 0xab98fba8; const int SF_denorm = 0x00000001; const int SF_random = 0x346001d6; +const int SF_neg_zero = 0x80000000; const long long DF_QNaN = 0x7ff8000000000000ULL; const long long DF_SNaN = 0x7ff7000000000000ULL; @@ -536,6 +537,33 @@ static void check_sffixupd(void) check32(result, 0x146001d6); } +static void check_sffms(void) +{ + int result; + + /* Check that sffms properly deals with -0 */ + result = SF_neg_zero; + asm ("%0 -= sfmpy(%1 , %2)\n\t" + : "+r"(result) + : "r"(SF_ZERO), "r"(SF_ZERO) + : "r12", "r8"); + check32(result, SF_neg_zero); + + result = SF_ZERO; + asm ("%0 -= sfmpy(%1 , %2)\n\t" + : "+r"(result) + : "r"(SF_neg_zero), "r"(SF_ZERO) + : "r12", "r8"); + check32(result, SF_ZERO); + + result = SF_ZERO; + asm ("%0 -= sfmpy(%1 , %2)\n\t" + : "+r"(result) + : "r"(SF_ZERO), "r"(SF_neg_zero) + : "r12", "r8"); + check32(result, SF_ZERO); +} + static void check_float2int_convs() { int res32; @@ -688,6 +716,7 @@ int main() check_invsqrta(); check_sffixupn(); check_sffixupd(); + check_sffms(); check_float2int_convs(); puts(err ? "FAIL" : "PASS"); diff --git a/tests/tcg/hexagon/preg_alias.c b/tests/tcg/hexagon/preg_alias.c index b44a8112b4..8798fbcaf3 100644 --- a/tests/tcg/hexagon/preg_alias.c +++ b/tests/tcg/hexagon/preg_alias.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2022 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -65,7 +65,7 @@ static inline void creg_alias(int cval, PRegs *pregs) : "=r"(pregs->pregs.p0), "=r"(pregs->pregs.p1), "=r"(pregs->pregs.p2), "=r"(pregs->pregs.p3) : "r"(cval) - : "p0", "p1", "p2", "p3"); + : "c4", "p0", "p1", "p2", "p3"); } int err; @@ -92,7 +92,7 @@ static inline void creg_alias_pair(unsigned int cval, PRegs *pregs) : "=r"(pregs->pregs.p0), "=r"(pregs->pregs.p1), "=r"(pregs->pregs.p2), "=r"(pregs->pregs.p3), "=r"(c5) : "r"(cval_pair) - : "p0", "p1", "p2", "p3"); + : "c4", "c5", "p0", "p1", "p2", "p3"); check(c5, 0xdeadbeef); } @@ -117,7 +117,7 @@ static void test_packet(void) "}\n\t" : "+r"(result) : "r"(0xffffffff), "r"(0xff00ffff), "r"(0x837ed653) - : "p0", "p1", "p2", "p3"); + : "c4", "p0", "p1", "p2", "p3"); check(result, old_val); /* Test a predicated store */ @@ -129,7 +129,7 @@ static void test_packet(void) "}\n\t" : : "r"(0), "r"(0xffffffff), "r"(&result) - : "p0", "p1", "p2", "p3", "memory"); + : "c4", "p0", "p1", "p2", "p3", "memory"); check(result, 0x0); } diff --git a/tests/tcg/hexagon/scatter_gather.c b/tests/tcg/hexagon/scatter_gather.c index b93eb18133..bf8b5e0317 100644 --- a/tests/tcg/hexagon/scatter_gather.c +++ b/tests/tcg/hexagon/scatter_gather.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2021 Qualcomm Innovation Center, Inc. All Rights Reserved. + * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -40,47 +40,6 @@ typedef long HVX_VectorPair __attribute__((__vector_size__(256))) typedef long HVX_VectorPred __attribute__((__vector_size__(128))) __attribute__((aligned(128))); -#define VSCATTER_16(BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermh_128B((int)BASE, RGN, OFF, VALS) -#define VSCATTER_16_MASKED(MASK, BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermhq_128B(MASK, (int)BASE, RGN, OFF, VALS) -#define VSCATTER_32(BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermw_128B((int)BASE, RGN, OFF, VALS) -#define VSCATTER_32_MASKED(MASK, BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermwq_128B(MASK, (int)BASE, RGN, OFF, VALS) -#define VSCATTER_16_32(BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermhw_128B((int)BASE, RGN, OFF, VALS) -#define VSCATTER_16_32_MASKED(MASK, BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermhwq_128B(MASK, (int)BASE, RGN, OFF, VALS) -#define VSCATTER_16_ACC(BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermh_add_128B((int)BASE, RGN, OFF, VALS) -#define VSCATTER_32_ACC(BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermw_add_128B((int)BASE, RGN, OFF, VALS) -#define VSCATTER_16_32_ACC(BASE, RGN, OFF, VALS) \ - __builtin_HEXAGON_V6_vscattermhw_add_128B((int)BASE, RGN, OFF, VALS) - -#define VGATHER_16(DSTADDR, BASE, RGN, OFF) \ - __builtin_HEXAGON_V6_vgathermh_128B(DSTADDR, (int)BASE, RGN, OFF) -#define VGATHER_16_MASKED(DSTADDR, MASK, BASE, RGN, OFF) \ - __builtin_HEXAGON_V6_vgathermhq_128B(DSTADDR, MASK, (int)BASE, RGN, OFF) -#define VGATHER_32(DSTADDR, BASE, RGN, OFF) \ - __builtin_HEXAGON_V6_vgathermw_128B(DSTADDR, (int)BASE, RGN, OFF) -#define VGATHER_32_MASKED(DSTADDR, MASK, BASE, RGN, OFF) \ - __builtin_HEXAGON_V6_vgathermwq_128B(DSTADDR, MASK, (int)BASE, RGN, OFF) -#define VGATHER_16_32(DSTADDR, BASE, RGN, OFF) \ - __builtin_HEXAGON_V6_vgathermhw_128B(DSTADDR, (int)BASE, RGN, OFF) -#define VGATHER_16_32_MASKED(DSTADDR, MASK, BASE, RGN, OFF) \ - __builtin_HEXAGON_V6_vgathermhwq_128B(DSTADDR, MASK, (int)BASE, RGN, OFF) - -#define VSHUFF_H(V) \ - __builtin_HEXAGON_V6_vshuffh_128B(V) -#define VSPLAT_H(X) \ - __builtin_HEXAGON_V6_lvsplath_128B(X) -#define VAND_VAL(PRED, VAL) \ - __builtin_HEXAGON_V6_vandvrt_128B(PRED, VAL) -#define VDEAL_H(V) \ - __builtin_HEXAGON_V6_vdealh_128B(V) - int err; /* define the number of rows/cols in a square matrix */ @@ -108,22 +67,22 @@ unsigned short vscatter16_32_ref[SCATTER_BUFFER_SIZE]; unsigned short vgather16_32_ref[MATRIX_SIZE]; /* declare the arrays of offsets */ -unsigned short half_offsets[MATRIX_SIZE]; -unsigned int word_offsets[MATRIX_SIZE]; +unsigned short half_offsets[MATRIX_SIZE] __attribute__((aligned(128))); +unsigned int word_offsets[MATRIX_SIZE] __attribute__((aligned(128))); /* declare the arrays of values */ -unsigned short half_values[MATRIX_SIZE]; -unsigned short half_values_acc[MATRIX_SIZE]; -unsigned short half_values_masked[MATRIX_SIZE]; -unsigned int word_values[MATRIX_SIZE]; -unsigned int word_values_acc[MATRIX_SIZE]; -unsigned int word_values_masked[MATRIX_SIZE]; +unsigned short half_values[MATRIX_SIZE] __attribute__((aligned(128))); +unsigned short half_values_acc[MATRIX_SIZE] __attribute__((aligned(128))); +unsigned short half_values_masked[MATRIX_SIZE] __attribute__((aligned(128))); +unsigned int word_values[MATRIX_SIZE] __attribute__((aligned(128))); +unsigned int word_values_acc[MATRIX_SIZE] __attribute__((aligned(128))); +unsigned int word_values_masked[MATRIX_SIZE] __attribute__((aligned(128))); /* declare the arrays of predicates */ -unsigned short half_predicates[MATRIX_SIZE]; -unsigned int word_predicates[MATRIX_SIZE]; +unsigned short half_predicates[MATRIX_SIZE] __attribute__((aligned(128))); +unsigned int word_predicates[MATRIX_SIZE] __attribute__((aligned(128))); -/* make this big enough for all the intrinsics */ +/* make this big enough for all the operations */ const size_t region_len = sizeof(vtcm); /* optionally add sync instructions */ @@ -261,164 +220,201 @@ void create_offsets_values_preds_16_32(void) } } -/* scatter the 16 bit elements using intrinsics */ +/* scatter the 16 bit elements using HVX */ void vector_scatter_16(void) { - /* copy the offsets and values to vectors */ - HVX_Vector offsets = *(HVX_Vector *)half_offsets; - HVX_Vector values = *(HVX_Vector *)half_values; - - VSCATTER_16(&vtcm.vscatter16, region_len, offsets, values); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%3 + #0)\n\t" + "vscatter(%0, m0, v0.h).h = v1\n\t" + : : "r"(vtcm.vscatter16), "r"(region_len), + "r"(half_offsets), "r"(half_values) + : "m0", "v0", "v1", "memory"); sync_scatter(vtcm.vscatter16); } -/* scatter-accumulate the 16 bit elements using intrinsics */ +/* scatter-accumulate the 16 bit elements using HVX */ void vector_scatter_16_acc(void) { - /* copy the offsets and values to vectors */ - HVX_Vector offsets = *(HVX_Vector *)half_offsets; - HVX_Vector values = *(HVX_Vector *)half_values_acc; - - VSCATTER_16_ACC(&vtcm.vscatter16, region_len, offsets, values); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%3 + #0)\n\t" + "vscatter(%0, m0, v0.h).h += v1\n\t" + : : "r"(vtcm.vscatter16), "r"(region_len), + "r"(half_offsets), "r"(half_values_acc) + : "m0", "v0", "v1", "memory"); sync_scatter(vtcm.vscatter16); } -/* scatter the 16 bit elements using intrinsics */ +/* masked scatter the 16 bit elements using HVX */ void vector_scatter_16_masked(void) { - /* copy the offsets and values to vectors */ - HVX_Vector offsets = *(HVX_Vector *)half_offsets; - HVX_Vector values = *(HVX_Vector *)half_values_masked; - HVX_Vector pred_reg = *(HVX_Vector *)half_predicates; - HVX_VectorPred preds = VAND_VAL(pred_reg, ~0); - - VSCATTER_16_MASKED(preds, &vtcm.vscatter16, region_len, offsets, values); + asm ("r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "v1 = vmem(%4 + #0)\n\t" + "if (q0) vscatter(%1, m0, v0.h).h = v1\n\t" + : : "r"(half_predicates), "r"(vtcm.vscatter16), "r"(region_len), + "r"(half_offsets), "r"(half_values_masked) + : "r1", "q0", "m0", "q0", "v0", "v1", "memory"); sync_scatter(vtcm.vscatter16); } -/* scatter the 32 bit elements using intrinsics */ +/* scatter the 32 bit elements using HVX */ void vector_scatter_32(void) { - /* copy the offsets and values to vectors */ - HVX_Vector offsetslo = *(HVX_Vector *)word_offsets; - HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; - HVX_Vector valueslo = *(HVX_Vector *)word_values; - HVX_Vector valueshi = *(HVX_Vector *)&word_values[MATRIX_SIZE / 2]; - - VSCATTER_32(&vtcm.vscatter32, region_len, offsetslo, valueslo); - VSCATTER_32(&vtcm.vscatter32, region_len, offsetshi, valueshi); + HVX_Vector *offsetslo = (HVX_Vector *)word_offsets; + HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; + HVX_Vector *valueslo = (HVX_Vector *)word_values; + HVX_Vector *valueshi = (HVX_Vector *)&word_values[MATRIX_SIZE / 2]; + + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%3 + #0)\n\t" + "vscatter(%0, m0, v0.w).w = v1\n\t" + : : "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetslo), "r"(valueslo) + : "m0", "v0", "v1", "memory"); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%3 + #0)\n\t" + "vscatter(%0, m0, v0.w).w = v1\n\t" + : : "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetshi), "r"(valueshi) + : "m0", "v0", "v1", "memory"); sync_scatter(vtcm.vscatter32); } -/* scatter-acc the 32 bit elements using intrinsics */ +/* scatter-accumulate the 32 bit elements using HVX */ void vector_scatter_32_acc(void) { - /* copy the offsets and values to vectors */ - HVX_Vector offsetslo = *(HVX_Vector *)word_offsets; - HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; - HVX_Vector valueslo = *(HVX_Vector *)word_values_acc; - HVX_Vector valueshi = *(HVX_Vector *)&word_values_acc[MATRIX_SIZE / 2]; - - VSCATTER_32_ACC(&vtcm.vscatter32, region_len, offsetslo, valueslo); - VSCATTER_32_ACC(&vtcm.vscatter32, region_len, offsetshi, valueshi); + HVX_Vector *offsetslo = (HVX_Vector *)word_offsets; + HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; + HVX_Vector *valueslo = (HVX_Vector *)word_values_acc; + HVX_Vector *valueshi = (HVX_Vector *)&word_values_acc[MATRIX_SIZE / 2]; + + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%3 + #0)\n\t" + "vscatter(%0, m0, v0.w).w += v1\n\t" + : : "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetslo), "r"(valueslo) + : "m0", "v0", "v1", "memory"); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%3 + #0)\n\t" + "vscatter(%0, m0, v0.w).w += v1\n\t" + : : "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetshi), "r"(valueshi) + : "m0", "v0", "v1", "memory"); sync_scatter(vtcm.vscatter32); } -/* scatter the 32 bit elements using intrinsics */ +/* masked scatter the 32 bit elements using HVX */ void vector_scatter_32_masked(void) { - /* copy the offsets and values to vectors */ - HVX_Vector offsetslo = *(HVX_Vector *)word_offsets; - HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; - HVX_Vector valueslo = *(HVX_Vector *)word_values_masked; - HVX_Vector valueshi = *(HVX_Vector *)&word_values_masked[MATRIX_SIZE / 2]; - HVX_Vector pred_reglo = *(HVX_Vector *)word_predicates; - HVX_Vector pred_reghi = *(HVX_Vector *)&word_predicates[MATRIX_SIZE / 2]; - HVX_VectorPred predslo = VAND_VAL(pred_reglo, ~0); - HVX_VectorPred predshi = VAND_VAL(pred_reghi, ~0); - - VSCATTER_32_MASKED(predslo, &vtcm.vscatter32, region_len, offsetslo, - valueslo); - VSCATTER_32_MASKED(predshi, &vtcm.vscatter32, region_len, offsetshi, - valueshi); + HVX_Vector *offsetslo = (HVX_Vector *)word_offsets; + HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; + HVX_Vector *valueslo = (HVX_Vector *)word_values_masked; + HVX_Vector *valueshi = (HVX_Vector *)&word_values_masked[MATRIX_SIZE / 2]; + HVX_Vector *predslo = (HVX_Vector *)word_predicates; + HVX_Vector *predshi = (HVX_Vector *)&word_predicates[MATRIX_SIZE / 2]; + + asm ("r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "v1 = vmem(%4 + #0)\n\t" + "if (q0) vscatter(%1, m0, v0.w).w = v1\n\t" + : : "r"(predslo), "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetslo), "r"(valueslo) + : "r1", "q0", "m0", "q0", "v0", "v1", "memory"); + asm ("r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "v1 = vmem(%4 + #0)\n\t" + "if (q0) vscatter(%1, m0, v0.w).w = v1\n\t" + : : "r"(predshi), "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetshi), "r"(valueshi) + : "r1", "q0", "m0", "q0", "v0", "v1", "memory"); - sync_scatter(vtcm.vscatter16); + sync_scatter(vtcm.vscatter32); } -/* scatter the 16 bit elements with 32 bit offsets using intrinsics */ +/* scatter the 16 bit elements with 32 bit offsets using HVX */ void vector_scatter_16_32(void) { - HVX_VectorPair offsets; - HVX_Vector values; - - /* get the word offsets in a vector pair */ - offsets = *(HVX_VectorPair *)word_offsets; - - /* these values need to be shuffled for the scatter */ - values = *(HVX_Vector *)half_values; - values = VSHUFF_H(values); - - VSCATTER_16_32(&vtcm.vscatter16_32, region_len, offsets, values); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%2 + #1)\n\t" + "v2 = vmem(%3 + #0)\n\t" + "v2.h = vshuff(v2.h)\n\t" /* shuffle the values for the scatter */ + "vscatter(%0, m0, v1:0.w).h = v2\n\t" + : : "r"(vtcm.vscatter16_32), "r"(region_len), + "r"(word_offsets), "r"(half_values) + : "m0", "v0", "v1", "v2", "memory"); sync_scatter(vtcm.vscatter16_32); } -/* scatter-acc the 16 bit elements with 32 bit offsets using intrinsics */ +/* scatter-accumulate the 16 bit elements with 32 bit offsets using HVX */ void vector_scatter_16_32_acc(void) { - HVX_VectorPair offsets; - HVX_Vector values; - - /* get the word offsets in a vector pair */ - offsets = *(HVX_VectorPair *)word_offsets; - - /* these values need to be shuffled for the scatter */ - values = *(HVX_Vector *)half_values_acc; - values = VSHUFF_H(values); - - VSCATTER_16_32_ACC(&vtcm.vscatter16_32, region_len, offsets, values); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%2 + #1)\n\t" + "v2 = vmem(%3 + #0)\n\t" \ + "v2.h = vshuff(v2.h)\n\t" /* shuffle the values for the scatter */ + "vscatter(%0, m0, v1:0.w).h += v2\n\t" + : : "r"(vtcm.vscatter16_32), "r"(region_len), + "r"(word_offsets), "r"(half_values_acc) + : "m0", "v0", "v1", "v2", "memory"); sync_scatter(vtcm.vscatter16_32); } -/* masked scatter the 16 bit elements with 32 bit offsets using intrinsics */ +/* masked scatter the 16 bit elements with 32 bit offsets using HVX */ void vector_scatter_16_32_masked(void) { - HVX_VectorPair offsets; - HVX_Vector values; - HVX_Vector pred_reg; - - /* get the word offsets in a vector pair */ - offsets = *(HVX_VectorPair *)word_offsets; - - /* these values need to be shuffled for the scatter */ - values = *(HVX_Vector *)half_values_masked; - values = VSHUFF_H(values); - - pred_reg = *(HVX_Vector *)half_predicates; - pred_reg = VSHUFF_H(pred_reg); - HVX_VectorPred preds = VAND_VAL(pred_reg, ~0); - - VSCATTER_16_32_MASKED(preds, &vtcm.vscatter16_32, region_len, offsets, - values); + asm ("r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "v0.h = vshuff(v0.h)\n\t" /* shuffle the predicates */ + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "v1 = vmem(%3 + #1)\n\t" + "v2 = vmem(%4 + #0)\n\t" \ + "v2.h = vshuff(v2.h)\n\t" /* shuffle the values for the scatter */ + "if (q0) vscatter(%1, m0, v1:0.w).h = v2\n\t" + : : "r"(half_predicates), "r"(vtcm.vscatter16_32), "r"(region_len), + "r"(word_offsets), "r"(half_values_masked) + : "r1", "q0", "m0", "v0", "v1", "v2", "memory"); sync_scatter(vtcm.vscatter16_32); } -/* gather the elements from the scatter16 buffer */ +/* gather the elements from the scatter16 buffer using HVX */ void vector_gather_16(void) { - HVX_Vector *vgather = (HVX_Vector *)&vtcm.vgather16; - HVX_Vector offsets = *(HVX_Vector *)half_offsets; - - VGATHER_16(vgather, &vtcm.vscatter16, region_len, offsets); - - sync_gather(vgather); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "{ vtmp.h = vgather(%0, m0, v0.h).h\n\t" + " vmem(%3 + #0) = vtmp.new }\n\t" + : : "r"(vtcm.vscatter16), "r"(region_len), + "r"(half_offsets), "r"(vtcm.vgather16) + : "m0", "v0", "memory"); + + sync_gather(vtcm.vgather16); } static unsigned short gather_16_masked_init(void) @@ -427,31 +423,51 @@ static unsigned short gather_16_masked_init(void) return letter | (letter << 8); } +/* masked gather the elements from the scatter16 buffer using HVX */ void vector_gather_16_masked(void) { - HVX_Vector *vgather = (HVX_Vector *)&vtcm.vgather16; - HVX_Vector offsets = *(HVX_Vector *)half_offsets; - HVX_Vector pred_reg = *(HVX_Vector *)half_predicates; - HVX_VectorPred preds = VAND_VAL(pred_reg, ~0); - - *vgather = VSPLAT_H(gather_16_masked_init()); - VGATHER_16_MASKED(vgather, preds, &vtcm.vscatter16, region_len, offsets); - - sync_gather(vgather); + unsigned short init = gather_16_masked_init(); + + asm ("v0.h = vsplat(%5)\n\t" + "vmem(%4 + #0) = v0\n\t" /* initialize the write area */ + "r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "{ if (q0) vtmp.h = vgather(%1, m0, v0.h).h\n\t" + " vmem(%4 + #0) = vtmp.new }\n\t" + : : "r"(half_predicates), "r"(vtcm.vscatter16), "r"(region_len), + "r"(half_offsets), "r"(vtcm.vgather16), "r"(init) + : "r1", "q0", "m0", "v0", "memory"); + + sync_gather(vtcm.vgather16); } -/* gather the elements from the scatter32 buffer */ +/* gather the elements from the scatter32 buffer using HVX */ void vector_gather_32(void) { - HVX_Vector *vgatherlo = (HVX_Vector *)&vtcm.vgather32; - HVX_Vector *vgatherhi = - (HVX_Vector *)((int)&vtcm.vgather32 + (MATRIX_SIZE * 2)); - HVX_Vector offsetslo = *(HVX_Vector *)word_offsets; - HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; - - VGATHER_32(vgatherlo, &vtcm.vscatter32, region_len, offsetslo); - VGATHER_32(vgatherhi, &vtcm.vscatter32, region_len, offsetshi); + HVX_Vector *vgatherlo = (HVX_Vector *)vtcm.vgather32; + HVX_Vector *vgatherhi = (HVX_Vector *)&vtcm.vgather32[MATRIX_SIZE / 2]; + HVX_Vector *offsetslo = (HVX_Vector *)word_offsets; + HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; + + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "{ vtmp.w = vgather(%0, m0, v0.w).w\n\t" + " vmem(%3 + #0) = vtmp.new }\n\t" + : : "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetslo), "r"(vgatherlo) + : "m0", "v0", "memory"); + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "{ vtmp.w = vgather(%0, m0, v0.w).w\n\t" + " vmem(%3 + #0) = vtmp.new }\n\t" + : : "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetshi), "r"(vgatherhi) + : "m0", "v0", "memory"); + sync_gather(vgatherlo); sync_gather(vgatherhi); } @@ -461,79 +477,88 @@ static unsigned int gather_32_masked_init(void) return letter | (letter << 8) | (letter << 16) | (letter << 24); } +/* masked gather the elements from the scatter32 buffer using HVX */ void vector_gather_32_masked(void) { - HVX_Vector *vgatherlo = (HVX_Vector *)&vtcm.vgather32; - HVX_Vector *vgatherhi = - (HVX_Vector *)((int)&vtcm.vgather32 + (MATRIX_SIZE * 2)); - HVX_Vector offsetslo = *(HVX_Vector *)word_offsets; - HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; - HVX_Vector pred_reglo = *(HVX_Vector *)word_predicates; - HVX_VectorPred predslo = VAND_VAL(pred_reglo, ~0); - HVX_Vector pred_reghi = *(HVX_Vector *)&word_predicates[MATRIX_SIZE / 2]; - HVX_VectorPred predshi = VAND_VAL(pred_reghi, ~0); - - *vgatherlo = VSPLAT_H(gather_32_masked_init()); - *vgatherhi = VSPLAT_H(gather_32_masked_init()); - VGATHER_32_MASKED(vgatherlo, predslo, &vtcm.vscatter32, region_len, - offsetslo); - VGATHER_32_MASKED(vgatherhi, predshi, &vtcm.vscatter32, region_len, - offsetshi); + unsigned int init = gather_32_masked_init(); + HVX_Vector *vgatherlo = (HVX_Vector *)vtcm.vgather32; + HVX_Vector *vgatherhi = (HVX_Vector *)&vtcm.vgather32[MATRIX_SIZE / 2]; + HVX_Vector *offsetslo = (HVX_Vector *)word_offsets; + HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2]; + HVX_Vector *predslo = (HVX_Vector *)word_predicates; + HVX_Vector *predshi = (HVX_Vector *)&word_predicates[MATRIX_SIZE / 2]; + + asm ("v0.h = vsplat(%5)\n\t" + "vmem(%4 + #0) = v0\n\t" /* initialize the write area */ + "r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "{ if (q0) vtmp.w = vgather(%1, m0, v0.w).w\n\t" + " vmem(%4 + #0) = vtmp.new }\n\t" + : : "r"(predslo), "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetslo), "r"(vgatherlo), "r"(init) + : "r1", "q0", "m0", "v0", "memory"); + asm ("v0.h = vsplat(%5)\n\t" + "vmem(%4 + #0) = v0\n\t" /* initialize the write area */ + "r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "{ if (q0) vtmp.w = vgather(%1, m0, v0.w).w\n\t" + " vmem(%4 + #0) = vtmp.new }\n\t" + : : "r"(predshi), "r"(vtcm.vscatter32), "r"(region_len), + "r"(offsetshi), "r"(vgatherhi), "r"(init) + : "r1", "q0", "m0", "v0", "memory"); sync_gather(vgatherlo); sync_gather(vgatherhi); } -/* gather the elements from the scatter16_32 buffer */ +/* gather the elements from the scatter16_32 buffer using HVX */ void vector_gather_16_32(void) { - HVX_Vector *vgather; - HVX_VectorPair offsets; - HVX_Vector values; - - /* get the vtcm address to gather from */ - vgather = (HVX_Vector *)&vtcm.vgather16_32; - - /* get the word offsets in a vector pair */ - offsets = *(HVX_VectorPair *)word_offsets; - - VGATHER_16_32(vgather, &vtcm.vscatter16_32, region_len, offsets); - - /* deal the elements to get the order back */ - values = *(HVX_Vector *)vgather; - values = VDEAL_H(values); - - /* write it back to vtcm address */ - *(HVX_Vector *)vgather = values; + asm ("m0 = %1\n\t" + "v0 = vmem(%2 + #0)\n\t" + "v1 = vmem(%2 + #1)\n\t" + "{ vtmp.h = vgather(%0, m0, v1:0.w).h\n\t" + " vmem(%3 + #0) = vtmp.new }\n\t" + "v0 = vmem(%3 + #0)\n\t" + "v0.h = vdeal(v0.h)\n\t" /* deal the elements to get the order back */ + "vmem(%3 + #0) = v0\n\t" + : : "r"(vtcm.vscatter16_32), "r"(region_len), + "r"(word_offsets), "r"(vtcm.vgather16_32) + : "m0", "v0", "v1", "memory"); + + sync_gather(vtcm.vgather16_32); } +/* masked gather the elements from the scatter16_32 buffer using HVX */ void vector_gather_16_32_masked(void) { - HVX_Vector *vgather; - HVX_VectorPair offsets; - HVX_Vector pred_reg; - HVX_VectorPred preds; - HVX_Vector values; - - /* get the vtcm address to gather from */ - vgather = (HVX_Vector *)&vtcm.vgather16_32; - - /* get the word offsets in a vector pair */ - offsets = *(HVX_VectorPair *)word_offsets; - pred_reg = *(HVX_Vector *)half_predicates; - pred_reg = VSHUFF_H(pred_reg); - preds = VAND_VAL(pred_reg, ~0); - - *vgather = VSPLAT_H(gather_16_masked_init()); - VGATHER_16_32_MASKED(vgather, preds, &vtcm.vscatter16_32, region_len, - offsets); - - /* deal the elements to get the order back */ - values = *(HVX_Vector *)vgather; - values = VDEAL_H(values); - - /* write it back to vtcm address */ - *(HVX_Vector *)vgather = values; + unsigned short init = gather_16_masked_init(); + + asm ("v0.h = vsplat(%5)\n\t" + "vmem(%4 + #0) = v0\n\t" /* initialize the write area */ + "r1 = #-1\n\t" + "v0 = vmem(%0 + #0)\n\t" + "v0.h = vshuff(v0.h)\n\t" /* shuffle the predicates */ + "q0 = vand(v0, r1)\n\t" + "m0 = %2\n\t" + "v0 = vmem(%3 + #0)\n\t" + "v1 = vmem(%3 + #1)\n\t" + "{ if (q0) vtmp.h = vgather(%1, m0, v1:0.w).h\n\t" + " vmem(%4 + #0) = vtmp.new }\n\t" + "v0 = vmem(%4 + #0)\n\t" + "v0.h = vdeal(v0.h)\n\t" /* deal the elements to get the order back */ + "vmem(%4 + #0) = v0\n\t" + : : "r"(half_predicates), "r"(vtcm.vscatter16_32), "r"(region_len), + "r"(word_offsets), "r"(vtcm.vgather16_32), "r"(init) + : "r1", "q0", "m0", "v0", "v1", "memory"); + + sync_gather(vtcm.vgather16_32); } static void check_buffer(const char *name, void *c, void *r, size_t size) @@ -579,6 +604,7 @@ void scalar_scatter_16_acc(unsigned short *vscatter16) } } +/* scatter-accumulate the 16 bit elements using C */ void check_scatter_16_acc() { memset(vscatter16_ref, FILL_CHAR, @@ -589,7 +615,7 @@ void check_scatter_16_acc() SCATTER_BUFFER_SIZE * sizeof(unsigned short)); } -/* scatter the 16 bit elements using C */ +/* masked scatter the 16 bit elements using C */ void scalar_scatter_16_masked(unsigned short *vscatter16) { for (int i = 0; i < MATRIX_SIZE; i++) { @@ -628,7 +654,7 @@ void check_scatter_32() SCATTER_BUFFER_SIZE * sizeof(unsigned int)); } -/* scatter the 32 bit elements using C */ +/* scatter-accumulate the 32 bit elements using C */ void scalar_scatter_32_acc(unsigned int *vscatter32) { for (int i = 0; i < MATRIX_SIZE; ++i) { @@ -646,7 +672,7 @@ void check_scatter_32_acc() SCATTER_BUFFER_SIZE * sizeof(unsigned int)); } -/* scatter the 32 bit elements using C */ +/* masked scatter the 32 bit elements using C */ void scalar_scatter_32_masked(unsigned int *vscatter32) { for (int i = 0; i < MATRIX_SIZE; i++) { @@ -667,7 +693,7 @@ void check_scatter_32_masked() SCATTER_BUFFER_SIZE * sizeof(unsigned int)); } -/* scatter the 32 bit elements using C */ +/* scatter the 16 bit elements with 32 bit offsets using C */ void scalar_scatter_16_32(unsigned short *vscatter16_32) { for (int i = 0; i < MATRIX_SIZE; ++i) { @@ -684,7 +710,7 @@ void check_scatter_16_32() SCATTER_BUFFER_SIZE * sizeof(unsigned short)); } -/* scatter the 32 bit elements using C */ +/* scatter-accumulate the 16 bit elements with 32 bit offsets using C */ void scalar_scatter_16_32_acc(unsigned short *vscatter16_32) { for (int i = 0; i < MATRIX_SIZE; ++i) { @@ -702,6 +728,7 @@ void check_scatter_16_32_acc() SCATTER_BUFFER_SIZE * sizeof(unsigned short)); } +/* masked scatter the 16 bit elements with 32 bit offsets using C */ void scalar_scatter_16_32_masked(unsigned short *vscatter16_32) { for (int i = 0; i < MATRIX_SIZE; i++) { @@ -738,6 +765,7 @@ void check_gather_16() MATRIX_SIZE * sizeof(unsigned short)); } +/* masked gather the elements from the scatter buffer using C */ void scalar_gather_16_masked(unsigned short *vgather16) { for (int i = 0; i < MATRIX_SIZE; ++i) { @@ -756,7 +784,7 @@ void check_gather_16_masked() MATRIX_SIZE * sizeof(unsigned short)); } -/* gather the elements from the scatter buffer using C */ +/* gather the elements from the scatter32 buffer using C */ void scalar_gather_32(unsigned int *vgather32) { for (int i = 0; i < MATRIX_SIZE; ++i) { @@ -772,6 +800,7 @@ void check_gather_32(void) MATRIX_SIZE * sizeof(unsigned int)); } +/* masked gather the elements from the scatter32 buffer using C */ void scalar_gather_32_masked(unsigned int *vgather32) { for (int i = 0; i < MATRIX_SIZE; ++i) { @@ -781,7 +810,6 @@ void scalar_gather_32_masked(unsigned int *vgather32) } } - void check_gather_32_masked(void) { memset(vgather32_ref, gather_32_masked_init(), @@ -791,7 +819,7 @@ void check_gather_32_masked(void) vgather32_ref, MATRIX_SIZE * sizeof(unsigned int)); } -/* gather the elements from the scatter buffer using C */ +/* gather the elements from the scatter16_32 buffer using C */ void scalar_gather_16_32(unsigned short *vgather16_32) { for (int i = 0; i < MATRIX_SIZE; ++i) { @@ -807,6 +835,7 @@ void check_gather_16_32(void) MATRIX_SIZE * sizeof(unsigned short)); } +/* masked gather the elements from the scatter16_32 buffer using C */ void scalar_gather_16_32_masked(unsigned short *vgather16_32) { for (int i = 0; i < MATRIX_SIZE; ++i) { diff --git a/tools/ebpf/Makefile.ebpf b/tools/ebpf/Makefile.ebpf index 8f327ae3b8..3391e7ce08 100755 --- a/tools/ebpf/Makefile.ebpf +++ b/tools/ebpf/Makefile.ebpf @@ -1,9 +1,9 @@ OBJS = rss.bpf.o -LLC ?= llc +LLVM_STRIP ?= llvm-strip CLANG ?= clang INC_FLAGS = `$(CLANG) -print-file-name=include` -EXTRA_CFLAGS ?= -O2 -emit-llvm -fno-stack-protector +EXTRA_CFLAGS ?= -O2 -g -target bpf all: $(OBJS) @@ -11,11 +11,13 @@ all: $(OBJS) clean: rm -f $(OBJS) + rm -f rss.bpf.skeleton.h $(OBJS): %.o:%.c $(CLANG) $(INC_FLAGS) \ -D__KERNEL__ -D__ASM_SYSREG_H \ -I../include $(LINUXINCLUDE) \ - $(EXTRA_CFLAGS) -c $< -o -| $(LLC) -march=bpf -filetype=obj -o $@ + $(EXTRA_CFLAGS) -c $< -o $@ + $(LLVM_STRIP) -g $@ bpftool gen skeleton rss.bpf.o > rss.bpf.skeleton.h cp rss.bpf.skeleton.h ../../ebpf/ diff --git a/tools/ebpf/rss.bpf.c b/tools/ebpf/rss.bpf.c index e85ec55f9b..20f227e2ac 100644 --- a/tools/ebpf/rss.bpf.c +++ b/tools/ebpf/rss.bpf.c @@ -76,29 +76,26 @@ struct packet_hash_info_t { }; }; -struct bpf_map_def SEC("maps") -tap_rss_map_configurations = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(struct rss_config_t), - .max_entries = 1, -}; - -struct bpf_map_def SEC("maps") -tap_rss_map_toeplitz_key = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(struct toeplitz_key_data_t), - .max_entries = 1, -}; - -struct bpf_map_def SEC("maps") -tap_rss_map_indirection_table = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(__u16), - .max_entries = INDIRECTION_TABLE_SIZE, -}; +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct rss_config_t)); + __uint(max_entries, 1); +} tap_rss_map_configurations SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct toeplitz_key_data_t)); + __uint(max_entries, 1); +} tap_rss_map_toeplitz_key SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u16)); + __uint(max_entries, INDIRECTION_TABLE_SIZE); +} tap_rss_map_indirection_table SEC(".maps"); static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written, const void *ptr, size_t size) { |