diff options
50 files changed, 2431 insertions, 1625 deletions
diff --git a/audio/alsaaudio.c b/audio/alsaaudio.c index f37ce1ce85..a23a5a0b60 100644 --- a/audio/alsaaudio.c +++ b/audio/alsaaudio.c @@ -307,6 +307,13 @@ static snd_pcm_format_t aud_to_alsafmt (AudioFormat fmt, int endianness) return SND_PCM_FORMAT_U32_LE; } + case AUDIO_FORMAT_F32: + if (endianness) { + return SND_PCM_FORMAT_FLOAT_BE; + } else { + return SND_PCM_FORMAT_FLOAT_LE; + } + default: dolog ("Internal logic error: Bad audio format %d\n", fmt); #ifdef DEBUG_AUDIO @@ -370,6 +377,16 @@ static int alsa_to_audfmt (snd_pcm_format_t alsafmt, AudioFormat *fmt, *fmt = AUDIO_FORMAT_U32; break; + case SND_PCM_FORMAT_FLOAT_LE: + *endianness = 0; + *fmt = AUDIO_FORMAT_F32; + break; + + case SND_PCM_FORMAT_FLOAT_BE: + *endianness = 1; + *fmt = AUDIO_FORMAT_F32; + break; + default: dolog ("Unrecognized audio format %d\n", alsafmt); return -1; @@ -906,6 +923,7 @@ static struct audio_pcm_ops alsa_pcm_ops = { .init_out = alsa_init_out, .fini_out = alsa_fini_out, .write = alsa_write, + .run_buffer_out = audio_generic_run_buffer_out, .enable_out = alsa_enable_out, .init_in = alsa_init_in, diff --git a/audio/audio.c b/audio/audio.c index f63f39769a..9ac9a20c41 100644 --- a/audio/audio.c +++ b/audio/audio.c @@ -218,6 +218,9 @@ static void audio_print_settings (struct audsettings *as) case AUDIO_FORMAT_U32: AUD_log (NULL, "U32"); break; + case AUDIO_FORMAT_F32: + AUD_log (NULL, "F32"); + break; default: AUD_log (NULL, "invalid(%d)", as->fmt); break; @@ -252,6 +255,7 @@ static int audio_validate_settings (struct audsettings *as) case AUDIO_FORMAT_U16: case AUDIO_FORMAT_S32: case AUDIO_FORMAT_U32: + case AUDIO_FORMAT_F32: break; default: invalid = 1; @@ -264,24 +268,28 @@ static int audio_validate_settings (struct audsettings *as) static int audio_pcm_info_eq (struct audio_pcm_info *info, struct audsettings *as) { - int bits = 8, sign = 0; + int bits = 8; + bool is_signed = false, is_float = false; switch (as->fmt) { case AUDIO_FORMAT_S8: - sign = 1; + is_signed = true; /* fall through */ case AUDIO_FORMAT_U8: break; case AUDIO_FORMAT_S16: - sign = 1; + is_signed = true; /* fall through */ case AUDIO_FORMAT_U16: bits = 16; break; + case AUDIO_FORMAT_F32: + is_float = true; + /* fall through */ case AUDIO_FORMAT_S32: - sign = 1; + is_signed = true; /* fall through */ case AUDIO_FORMAT_U32: bits = 32; @@ -292,33 +300,38 @@ static int audio_pcm_info_eq (struct audio_pcm_info *info, struct audsettings *a } return info->freq == as->freq && info->nchannels == as->nchannels - && info->sign == sign + && info->is_signed == is_signed + && info->is_float == is_float && info->bits == bits && info->swap_endianness == (as->endianness != AUDIO_HOST_ENDIANNESS); } void audio_pcm_init_info (struct audio_pcm_info *info, struct audsettings *as) { - int bits = 8, sign = 0, mul; + int bits = 8, mul; + bool is_signed = false, is_float = false; switch (as->fmt) { case AUDIO_FORMAT_S8: - sign = 1; + is_signed = true; /* fall through */ case AUDIO_FORMAT_U8: mul = 1; break; case AUDIO_FORMAT_S16: - sign = 1; + is_signed = true; /* fall through */ case AUDIO_FORMAT_U16: bits = 16; mul = 2; break; + case AUDIO_FORMAT_F32: + is_float = true; + /* fall through */ case AUDIO_FORMAT_S32: - sign = 1; + is_signed = true; /* fall through */ case AUDIO_FORMAT_U32: bits = 32; @@ -331,7 +344,8 @@ void audio_pcm_init_info (struct audio_pcm_info *info, struct audsettings *as) info->freq = as->freq; info->bits = bits; - info->sign = sign; + info->is_signed = is_signed; + info->is_float = is_float; info->nchannels = as->nchannels; info->bytes_per_frame = as->nchannels * mul; info->bytes_per_second = info->freq * info->bytes_per_frame; @@ -344,7 +358,7 @@ void audio_pcm_info_clear_buf (struct audio_pcm_info *info, void *buf, int len) return; } - if (info->sign) { + if (info->is_signed || info->is_float) { memset(buf, 0x00, len * info->bytes_per_frame); } else { @@ -770,8 +784,9 @@ static size_t audio_pcm_sw_write(SWVoiceOut *sw, void *buf, size_t size) #ifdef DEBUG_AUDIO static void audio_pcm_print_info (const char *cap, struct audio_pcm_info *info) { - dolog ("%s: bits %d, sign %d, freq %d, nchan %d\n", - cap, info->bits, info->sign, info->freq, info->nchannels); + dolog("%s: bits %d, sign %d, float %d, freq %d, nchan %d\n", + cap, info->bits, info->is_signed, info->is_float, info->freq, + info->nchannels); } #endif @@ -879,9 +894,9 @@ size_t AUD_read(SWVoiceIn *sw, void *buf, size_t size) } } -int AUD_get_buffer_size_out (SWVoiceOut *sw) +int AUD_get_buffer_size_out(SWVoiceOut *sw) { - return sw->hw->mix_buf->size * sw->hw->info.bytes_per_frame; + return sw->hw->samples * sw->hw->info.bytes_per_frame; } void AUD_set_active_out (SWVoiceOut *sw, int on) @@ -1076,10 +1091,8 @@ static size_t audio_pcm_hw_run_out(HWVoiceOut *hw, size_t live) while (live) { size_t size, decr, proc; void *buf = hw->pcm_ops->get_buffer_out(hw, &size); - if (!buf) { - /* retrying will likely won't help, drop everything. */ - hw->mix_buf->pos = (hw->mix_buf->pos + live) % hw->mix_buf->size; - return clipped + live; + if (!buf || size == 0) { + break; } decr = MIN(size / hw->info.bytes_per_frame, live); @@ -1097,6 +1110,10 @@ static size_t audio_pcm_hw_run_out(HWVoiceOut *hw, size_t live) } } + if (hw->pcm_ops->run_buffer_out) { + hw->pcm_ops->run_buffer_out(hw); + } + return clipped; } @@ -1403,7 +1420,8 @@ void *audio_generic_get_buffer_in(HWVoiceIn *hw, size_t *size) } assert(start >= 0 && start < hw->size_emul); - *size = MIN(hw->pending_emul, hw->size_emul - start); + *size = MIN(*size, hw->pending_emul); + *size = MIN(*size, hw->size_emul - start); return hw->buf_emul + start; } @@ -1413,6 +1431,28 @@ void audio_generic_put_buffer_in(HWVoiceIn *hw, void *buf, size_t size) hw->pending_emul -= size; } +void audio_generic_run_buffer_out(HWVoiceOut *hw) +{ + while (hw->pending_emul) { + size_t write_len, written; + ssize_t start = ((ssize_t) hw->pos_emul) - hw->pending_emul; + + if (start < 0) { + start += hw->size_emul; + } + assert(start >= 0 && start < hw->size_emul); + + write_len = MIN(hw->pending_emul, hw->size_emul - start); + + written = hw->pcm_ops->write(hw, hw->buf_emul + start, write_len); + hw->pending_emul -= written; + + if (written < write_len) { + break; + } + } +} + void *audio_generic_get_buffer_out(HWVoiceOut *hw, size_t *size) { if (unlikely(!hw->buf_emul)) { @@ -1428,8 +1468,7 @@ void *audio_generic_get_buffer_out(HWVoiceOut *hw, size_t *size) return hw->buf_emul + hw->pos_emul; } -size_t audio_generic_put_buffer_out_nowrite(HWVoiceOut *hw, void *buf, - size_t size) +size_t audio_generic_put_buffer_out(HWVoiceOut *hw, void *buf, size_t size) { assert(buf == hw->buf_emul + hw->pos_emul && size + hw->pending_emul <= hw->size_emul); @@ -1440,35 +1479,6 @@ size_t audio_generic_put_buffer_out_nowrite(HWVoiceOut *hw, void *buf, return size; } -size_t audio_generic_put_buffer_out(HWVoiceOut *hw, void *buf, size_t size) -{ - audio_generic_put_buffer_out_nowrite(hw, buf, size); - - while (hw->pending_emul) { - size_t write_len, written; - ssize_t start = ((ssize_t) hw->pos_emul) - hw->pending_emul; - if (start < 0) { - start += hw->size_emul; - } - assert(start >= 0 && start < hw->size_emul); - - write_len = MIN(hw->pending_emul, hw->size_emul - start); - - written = hw->pcm_ops->write(hw, hw->buf_emul + start, write_len); - hw->pending_emul -= written; - - if (written < write_len) { - break; - } - } - - /* - * fake we have written everything. non-written data remain in pending_emul, - * so we do not have to clip them multiple times - */ - return size; -} - size_t audio_generic_write(HWVoiceOut *hw, void *buf, size_t size) { size_t dst_size, copy_size; @@ -1476,17 +1486,17 @@ size_t audio_generic_write(HWVoiceOut *hw, void *buf, size_t size) copy_size = MIN(size, dst_size); memcpy(dst, buf, copy_size); - return hw->pcm_ops->put_buffer_out(hw, buf, copy_size); + return hw->pcm_ops->put_buffer_out(hw, dst, copy_size); } size_t audio_generic_read(HWVoiceIn *hw, void *buf, size_t size) { - size_t dst_size, copy_size; - void *dst = hw->pcm_ops->get_buffer_in(hw, &dst_size); - copy_size = MIN(size, dst_size); + size_t src_size, copy_size; + void *src = hw->pcm_ops->get_buffer_in(hw, &src_size); + copy_size = MIN(size, src_size); - memcpy(dst, buf, copy_size); - hw->pcm_ops->put_buffer_in(hw, buf, copy_size); + memcpy(buf, src, copy_size); + hw->pcm_ops->put_buffer_in(hw, src, copy_size); return copy_size; } @@ -1837,11 +1847,15 @@ CaptureVoiceOut *AUD_add_capture( cap->buf = g_malloc0_n(hw->mix_buf->size, hw->info.bytes_per_frame); - hw->clip = mixeng_clip - [hw->info.nchannels == 2] - [hw->info.sign] - [hw->info.swap_endianness] - [audio_bits_to_index (hw->info.bits)]; + if (hw->info.is_float) { + hw->clip = mixeng_clip_float[hw->info.nchannels == 2]; + } else { + hw->clip = mixeng_clip + [hw->info.nchannels == 2] + [hw->info.is_signed] + [hw->info.swap_endianness] + [audio_bits_to_index(hw->info.bits)]; + } QLIST_INSERT_HEAD (&s->cap_head, cap, entries); QLIST_INSERT_HEAD (&cap->cb_head, cb, entries); @@ -2080,6 +2094,7 @@ int audioformat_bytes_per_sample(AudioFormat fmt) case AUDIO_FORMAT_U32: case AUDIO_FORMAT_S32: + case AUDIO_FORMAT_F32: return 4; case AUDIO_FORMAT__MAX: diff --git a/audio/audio_int.h b/audio/audio_int.h index 5ba2078346..4775857bf2 100644 --- a/audio/audio_int.h +++ b/audio/audio_int.h @@ -40,7 +40,8 @@ struct audio_callback { struct audio_pcm_info { int bits; - int sign; + bool is_signed; + bool is_float; int freq; int nchannels; int bytes_per_frame; @@ -152,6 +153,7 @@ struct audio_pcm_ops { int (*init_out)(HWVoiceOut *hw, audsettings *as, void *drv_opaque); void (*fini_out)(HWVoiceOut *hw); size_t (*write) (HWVoiceOut *hw, void *buf, size_t size); + void (*run_buffer_out)(HWVoiceOut *hw); /* * get a buffer that after later can be passed to put_buffer_out; optional * returns the buffer, and writes it's size to size (in bytes) @@ -178,10 +180,9 @@ struct audio_pcm_ops { void *audio_generic_get_buffer_in(HWVoiceIn *hw, size_t *size); void audio_generic_put_buffer_in(HWVoiceIn *hw, void *buf, size_t size); +void audio_generic_run_buffer_out(HWVoiceOut *hw); void *audio_generic_get_buffer_out(HWVoiceOut *hw, size_t *size); size_t audio_generic_put_buffer_out(HWVoiceOut *hw, void *buf, size_t size); -size_t audio_generic_put_buffer_out_nowrite(HWVoiceOut *hw, void *buf, - size_t size); size_t audio_generic_write(HWVoiceOut *hw, void *buf, size_t size); size_t audio_generic_read(HWVoiceIn *hw, void *buf, size_t size); diff --git a/audio/audio_template.h b/audio/audio_template.h index 3287d7075e..7013d3041f 100644 --- a/audio/audio_template.h +++ b/audio/audio_template.h @@ -153,15 +153,23 @@ static int glue (audio_pcm_sw_init_, TYPE) ( sw->ratio = ((int64_t) sw->info.freq << 32) / sw->hw->info.freq; #endif + if (sw->info.is_float) { #ifdef DAC - sw->conv = mixeng_conv + sw->conv = mixeng_conv_float[sw->info.nchannels == 2]; #else - sw->clip = mixeng_clip + sw->clip = mixeng_clip_float[sw->info.nchannels == 2]; #endif - [sw->info.nchannels == 2] - [sw->info.sign] - [sw->info.swap_endianness] - [audio_bits_to_index (sw->info.bits)]; + } else { +#ifdef DAC + sw->conv = mixeng_conv +#else + sw->clip = mixeng_clip +#endif + [sw->info.nchannels == 2] + [sw->info.is_signed] + [sw->info.swap_endianness] + [audio_bits_to_index(sw->info.bits)]; + } sw->name = g_strdup (name); err = glue (audio_pcm_sw_alloc_resources_, TYPE) (sw); @@ -276,15 +284,23 @@ static HW *glue(audio_pcm_hw_add_new_, TYPE)(AudioState *s, goto err1; } + if (hw->info.is_float) { #ifdef DAC - hw->clip = mixeng_clip + hw->clip = mixeng_clip_float[hw->info.nchannels == 2]; #else - hw->conv = mixeng_conv + hw->conv = mixeng_conv_float[hw->info.nchannels == 2]; #endif - [hw->info.nchannels == 2] - [hw->info.sign] - [hw->info.swap_endianness] - [audio_bits_to_index (hw->info.bits)]; + } else { +#ifdef DAC + hw->clip = mixeng_clip +#else + hw->conv = mixeng_conv +#endif + [hw->info.nchannels == 2] + [hw->info.is_signed] + [hw->info.swap_endianness] + [audio_bits_to_index(hw->info.bits)]; + } glue(audio_pcm_hw_alloc_resources_, TYPE)(hw); diff --git a/audio/coreaudio.c b/audio/coreaudio.c index 66f0f459cf..4b4365660f 100644 --- a/audio/coreaudio.c +++ b/audio/coreaudio.c @@ -411,7 +411,7 @@ static int coreaudio_unlock (coreaudioVoiceOut *core, const char *fn_name) } COREAUDIO_WRAPPER_FUNC(get_buffer_out, void *, (HWVoiceOut *hw, size_t *size), (hw, size)) -COREAUDIO_WRAPPER_FUNC(put_buffer_out_nowrite, size_t, +COREAUDIO_WRAPPER_FUNC(put_buffer_out, size_t, (HWVoiceOut *hw, void *buf, size_t size), (hw, buf, size)) COREAUDIO_WRAPPER_FUNC(write, size_t, (HWVoiceOut *hw, void *buf, size_t size), @@ -471,20 +471,6 @@ static OSStatus audioDeviceIOProc( return 0; } -static UInt32 coreaudio_get_flags(struct audio_pcm_info *info, - struct audsettings *as) -{ - UInt32 flags = info->sign ? kAudioFormatFlagIsSignedInteger : 0; - if (as->endianness) { /* 0 = little, 1 = big */ - flags |= kAudioFormatFlagIsBigEndian; - } - - if (flags == 0) { /* must not be 0 */ - flags = kAudioFormatFlagsAreAllClear; - } - return flags; -} - static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as, void *drv_opaque) { @@ -496,6 +482,7 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as, Audiodev *dev = drv_opaque; AudiodevCoreaudioPerDirectionOptions *cpdo = dev->u.coreaudio.out; int frames; + struct audsettings fake_as; /* create mutex */ err = pthread_mutex_init(&core->mutex, NULL); @@ -504,6 +491,9 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as, return -1; } + fake_as = *as; + as = &fake_as; + as->fmt = AUDIO_FORMAT_F32; audio_pcm_init_info (&hw->info, as); status = coreaudio_get_voice(&core->outputDeviceID); @@ -572,15 +562,6 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as, /* set Samplerate */ core->outputStreamBasicDescription.mSampleRate = (Float64) as->freq; - core->outputStreamBasicDescription.mFormatID = kAudioFormatLinearPCM; - core->outputStreamBasicDescription.mFormatFlags = - coreaudio_get_flags(&hw->info, as); - core->outputStreamBasicDescription.mBytesPerPacket = - core->outputStreamBasicDescription.mBytesPerFrame = - hw->info.nchannels * hw->info.bits / 8; - core->outputStreamBasicDescription.mFramesPerPacket = 1; - core->outputStreamBasicDescription.mChannelsPerFrame = hw->info.nchannels; - core->outputStreamBasicDescription.mBitsPerChannel = hw->info.bits; status = coreaudio_set_streamformat(core->outputDeviceID, &core->outputStreamBasicDescription); @@ -687,9 +668,12 @@ static void coreaudio_audio_fini (void *opaque) static struct audio_pcm_ops coreaudio_pcm_ops = { .init_out = coreaudio_init_out, .fini_out = coreaudio_fini_out, + /* wrapper for audio_generic_write */ .write = coreaudio_write, + /* wrapper for audio_generic_get_buffer_out */ .get_buffer_out = coreaudio_get_buffer_out, - .put_buffer_out = coreaudio_put_buffer_out_nowrite, + /* wrapper for audio_generic_put_buffer_out */ + .put_buffer_out = coreaudio_put_buffer_out, .enable_out = coreaudio_enable_out }; diff --git a/audio/dsound_template.h b/audio/dsound_template.h index 7a15f91ce5..9c5ce625ab 100644 --- a/audio/dsound_template.h +++ b/audio/dsound_template.h @@ -244,6 +244,7 @@ static int dsound_init_out(HWVoiceOut *hw, struct audsettings *as, goto fail0; } + ds->first_time = true; obt_as.endianness = 0; audio_pcm_init_info (&hw->info, &obt_as); diff --git a/audio/dsoundaudio.c b/audio/dsoundaudio.c index c265c0094b..bd57082a8d 100644 --- a/audio/dsoundaudio.c +++ b/audio/dsoundaudio.c @@ -53,12 +53,14 @@ typedef struct { typedef struct { HWVoiceOut hw; LPDIRECTSOUNDBUFFER dsound_buffer; + bool first_time; dsound *s; } DSoundVoiceOut; typedef struct { HWVoiceIn hw; LPDIRECTSOUNDCAPTUREBUFFER dsound_capture_buffer; + bool first_time; dsound *s; } DSoundVoiceIn; @@ -414,21 +416,32 @@ static void *dsound_get_buffer_out(HWVoiceOut *hw, size_t *size) DSoundVoiceOut *ds = (DSoundVoiceOut *) hw; LPDIRECTSOUNDBUFFER dsb = ds->dsound_buffer; HRESULT hr; - DWORD ppos, act_size; + DWORD ppos, wpos, act_size; size_t req_size; int err; void *ret; - hr = IDirectSoundBuffer_GetCurrentPosition(dsb, &ppos, NULL); + hr = IDirectSoundBuffer_GetCurrentPosition( + dsb, &ppos, ds->first_time ? &wpos : NULL); if (FAILED(hr)) { dsound_logerr(hr, "Could not get playback buffer position\n"); *size = 0; return NULL; } + if (ds->first_time) { + hw->pos_emul = wpos; + ds->first_time = false; + } + req_size = audio_ring_dist(ppos, hw->pos_emul, hw->size_emul); req_size = MIN(req_size, hw->size_emul - hw->pos_emul); + if (req_size == 0) { + *size = 0; + return NULL; + } + err = dsound_lock_out(dsb, &hw->info, hw->pos_emul, req_size, &ret, NULL, &act_size, NULL, false, ds->s); if (err) { @@ -508,18 +521,24 @@ static void *dsound_get_buffer_in(HWVoiceIn *hw, size_t *size) DSoundVoiceIn *ds = (DSoundVoiceIn *) hw; LPDIRECTSOUNDCAPTUREBUFFER dscb = ds->dsound_capture_buffer; HRESULT hr; - DWORD cpos, act_size; + DWORD cpos, rpos, act_size; size_t req_size; int err; void *ret; - hr = IDirectSoundCaptureBuffer_GetCurrentPosition(dscb, &cpos, NULL); + hr = IDirectSoundCaptureBuffer_GetCurrentPosition( + dscb, &cpos, ds->first_time ? &rpos : NULL); if (FAILED(hr)) { dsound_logerr(hr, "Could not get capture buffer position\n"); *size = 0; return NULL; } + if (ds->first_time) { + hw->pos_emul = rpos; + ds->first_time = false; + } + req_size = audio_ring_dist(cpos, hw->pos_emul, hw->size_emul); req_size = MIN(req_size, hw->size_emul - hw->pos_emul); diff --git a/audio/mixeng.c b/audio/mixeng.c index 2f5ba71381..c14b0d874c 100644 --- a/audio/mixeng.c +++ b/audio/mixeng.c @@ -267,6 +267,76 @@ f_sample *mixeng_clip[2][2][2][3] = { } }; +#ifdef FLOAT_MIXENG +#define FLOAT_CONV_TO(x) (x) +#define FLOAT_CONV_FROM(x) (x) +#else +static const float float_scale = UINT_MAX; +#define FLOAT_CONV_TO(x) ((x) * float_scale) + +#ifdef RECIPROCAL +static const float float_scale_reciprocal = 1.f / UINT_MAX; +#define FLOAT_CONV_FROM(x) ((x) * float_scale_reciprocal) +#else +#define FLOAT_CONV_FROM(x) ((x) / float_scale) +#endif +#endif + +static void conv_natural_float_to_mono(struct st_sample *dst, const void *src, + int samples) +{ + float *in = (float *)src; + + while (samples--) { + dst->r = dst->l = FLOAT_CONV_TO(*in++); + dst++; + } +} + +static void conv_natural_float_to_stereo(struct st_sample *dst, const void *src, + int samples) +{ + float *in = (float *)src; + + while (samples--) { + dst->l = FLOAT_CONV_TO(*in++); + dst->r = FLOAT_CONV_TO(*in++); + dst++; + } +} + +t_sample *mixeng_conv_float[2] = { + conv_natural_float_to_mono, + conv_natural_float_to_stereo, +}; + +static void clip_natural_float_from_mono(void *dst, const struct st_sample *src, + int samples) +{ + float *out = (float *)dst; + + while (samples--) { + *out++ = FLOAT_CONV_FROM(src->l) + FLOAT_CONV_FROM(src->r); + src++; + } +} + +static void clip_natural_float_from_stereo( + void *dst, const struct st_sample *src, int samples) +{ + float *out = (float *)dst; + + while (samples--) { + *out++ = FLOAT_CONV_FROM(src->l); + *out++ = FLOAT_CONV_FROM(src->r); + src++; + } +} + +f_sample *mixeng_clip_float[2] = { + clip_natural_float_from_mono, + clip_natural_float_from_stereo, +}; void audio_sample_to_uint64(void *samples, int pos, uint64_t *left, uint64_t *right) diff --git a/audio/mixeng.h b/audio/mixeng.h index 18e62c7c49..2dcd6df245 100644 --- a/audio/mixeng.h +++ b/audio/mixeng.h @@ -38,9 +38,14 @@ typedef struct st_sample st_sample; typedef void (t_sample) (struct st_sample *dst, const void *src, int samples); typedef void (f_sample) (void *dst, const struct st_sample *src, int samples); +/* indices: [stereo][signed][swap endiannes][8, 16 or 32-bits] */ extern t_sample *mixeng_conv[2][2][2][3]; extern f_sample *mixeng_clip[2][2][2][3]; +/* indices: [stereo] */ +extern t_sample *mixeng_conv_float[2]; +extern f_sample *mixeng_clip_float[2]; + void *st_rate_start (int inrate, int outrate); void st_rate_flow(void *opaque, st_sample *ibuf, st_sample *obuf, size_t *isamp, size_t *osamp); diff --git a/audio/noaudio.c b/audio/noaudio.c index ff99b253ff..05798ea210 100644 --- a/audio/noaudio.c +++ b/audio/noaudio.c @@ -118,6 +118,7 @@ static struct audio_pcm_ops no_pcm_ops = { .init_out = no_init_out, .fini_out = no_fini_out, .write = no_write, + .run_buffer_out = audio_generic_run_buffer_out, .enable_out = no_enable_out, .init_in = no_init_in, diff --git a/audio/ossaudio.c b/audio/ossaudio.c index c43faeeea4..f88d076ec2 100644 --- a/audio/ossaudio.c +++ b/audio/ossaudio.c @@ -382,6 +382,15 @@ static size_t oss_get_available_bytes(OSSVoiceOut *oss) return audio_ring_dist(cntinfo.ptr, oss->hw.pos_emul, oss->hw.size_emul); } +static void oss_run_buffer_out(HWVoiceOut *hw) +{ + OSSVoiceOut *oss = (OSSVoiceOut *)hw; + + if (!oss->mmapped) { + audio_generic_run_buffer_out(hw); + } +} + static void *oss_get_buffer_out(HWVoiceOut *hw, size_t *size) { OSSVoiceOut *oss = (OSSVoiceOut *) hw; @@ -420,7 +429,7 @@ static size_t oss_write(HWVoiceOut *hw, void *buf, size_t len) size_t to_copy = MIN(len, hw->size_emul - hw->pos_emul); memcpy(hw->buf_emul + hw->pos_emul, buf, to_copy); - hw->pos_emul = (hw->pos_emul + to_copy) % hw->pos_emul; + hw->pos_emul = (hw->pos_emul + to_copy) % hw->size_emul; buf += to_copy; len -= to_copy; } @@ -570,20 +579,18 @@ static void oss_enable_out(HWVoiceOut *hw, bool enable) AudiodevOssPerDirectionOptions *opdo = oss->dev->u.oss.out; if (enable) { - bool poll_mode = opdo->try_poll; + hw->poll_mode = opdo->try_poll; ldebug("enabling voice\n"); - if (poll_mode) { + if (hw->poll_mode) { oss_poll_out(hw); - poll_mode = 0; } - hw->poll_mode = poll_mode; if (!oss->mmapped) { return; } - audio_pcm_info_clear_buf(&hw->info, hw->buf_emul, hw->mix_buf->size); + audio_pcm_info_clear_buf(&hw->info, hw->buf_emul, hw->samples); trig = PCM_ENABLE_OUTPUT; if (ioctl(oss->fd, SNDCTL_DSP_SETTRIGGER, &trig) < 0) { oss_logerr(errno, @@ -699,17 +706,15 @@ static void oss_enable_in(HWVoiceIn *hw, bool enable) AudiodevOssPerDirectionOptions *opdo = oss->dev->u.oss.out; if (enable) { - bool poll_mode = opdo->try_poll; + hw->poll_mode = opdo->try_poll; - if (poll_mode) { + if (hw->poll_mode) { oss_poll_in(hw); - poll_mode = 0; } - hw->poll_mode = poll_mode; } else { if (hw->poll_mode) { - hw->poll_mode = 0; qemu_set_fd_handler (oss->fd, NULL, NULL, NULL); + hw->poll_mode = 0; } } } @@ -748,6 +753,7 @@ static struct audio_pcm_ops oss_pcm_ops = { .init_out = oss_init_out, .fini_out = oss_fini_out, .write = oss_write, + .run_buffer_out = oss_run_buffer_out, .get_buffer_out = oss_get_buffer_out, .put_buffer_out = oss_put_buffer_out, .enable_out = oss_enable_out, diff --git a/audio/paaudio.c b/audio/paaudio.c index dbfe48c03a..b052084698 100644 --- a/audio/paaudio.c +++ b/audio/paaudio.c @@ -32,7 +32,6 @@ typedef struct { HWVoiceOut hw; pa_stream *stream; paaudio *g; - size_t samples; } PAVoiceOut; typedef struct { @@ -41,7 +40,6 @@ typedef struct { const void *read_data; size_t read_length; paaudio *g; - size_t samples; } PAVoiceIn; static void qpa_conn_fini(PAConnection *c); @@ -279,6 +277,9 @@ static pa_sample_format_t audfmt_to_pa (AudioFormat afmt, int endianness) case AUDIO_FORMAT_U32: format = endianness ? PA_SAMPLE_S32BE : PA_SAMPLE_S32LE; break; + case AUDIO_FORMAT_F32: + format = endianness ? PA_SAMPLE_FLOAT32BE : PA_SAMPLE_FLOAT32LE; + break; default: dolog ("Internal logic error: Bad audio format %d\n", afmt); format = PA_SAMPLE_U8; @@ -304,6 +305,12 @@ static AudioFormat pa_to_audfmt (pa_sample_format_t fmt, int *endianness) case PA_SAMPLE_S32LE: *endianness = 0; return AUDIO_FORMAT_S32; + case PA_SAMPLE_FLOAT32BE: + *endianness = 1; + return AUDIO_FORMAT_F32; + case PA_SAMPLE_FLOAT32LE: + *endianness = 0; + return AUDIO_FORMAT_F32; default: dolog ("Internal logic error: Bad pa_sample_format %d\n", fmt); return AUDIO_FORMAT_U8; @@ -488,7 +495,7 @@ static int qpa_init_out(HWVoiceOut *hw, struct audsettings *as, } audio_pcm_init_info (&hw->info, &obt_as); - hw->samples = pa->samples = audio_buffer_samples( + hw->samples = audio_buffer_samples( qapi_AudiodevPaPerDirectionOptions_base(ppdo), &obt_as, ppdo->buffer_length); @@ -536,7 +543,7 @@ static int qpa_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque) } audio_pcm_init_info (&hw->info, &obt_as); - hw->samples = pa->samples = audio_buffer_samples( + hw->samples = audio_buffer_samples( qapi_AudiodevPaPerDirectionOptions_base(ppdo), &obt_as, ppdo->buffer_length); diff --git a/audio/sdlaudio.c b/audio/sdlaudio.c index 5c6bcfcb3e..21b7a0484b 100644 --- a/audio/sdlaudio.c +++ b/audio/sdlaudio.c @@ -77,6 +77,14 @@ static int aud_to_sdlfmt (AudioFormat fmt) case AUDIO_FORMAT_U16: return AUDIO_U16LSB; + case AUDIO_FORMAT_S32: + return AUDIO_S32LSB; + + /* no unsigned 32-bit support in SDL */ + + case AUDIO_FORMAT_F32: + return AUDIO_F32LSB; + default: dolog ("Internal logic error: Bad audio format %d\n", fmt); #ifdef DEBUG_AUDIO @@ -119,6 +127,26 @@ static int sdl_to_audfmt(int sdlfmt, AudioFormat *fmt, int *endianness) *fmt = AUDIO_FORMAT_U16; break; + case AUDIO_S32LSB: + *endianness = 0; + *fmt = AUDIO_FORMAT_S32; + break; + + case AUDIO_S32MSB: + *endianness = 1; + *fmt = AUDIO_FORMAT_S32; + break; + + case AUDIO_F32LSB: + *endianness = 0; + *fmt = AUDIO_FORMAT_F32; + break; + + case AUDIO_F32MSB: + *endianness = 1; + *fmt = AUDIO_FORMAT_F32; + break; + default: dolog ("Unrecognized SDL audio format %d\n", sdlfmt); return -1; @@ -227,7 +255,7 @@ static void sdl_callback (void *opaque, Uint8 *buf, int len) SDL_WRAPPER_FUNC(get_buffer_out, void *, (HWVoiceOut *hw, size_t *size), (hw, size), *size = 0, sdl_unlock) -SDL_WRAPPER_FUNC(put_buffer_out_nowrite, size_t, +SDL_WRAPPER_FUNC(put_buffer_out, size_t, (HWVoiceOut *hw, void *buf, size_t size), (hw, buf, size), /*nothing*/, sdl_unlock_and_post) SDL_WRAPPER_FUNC(write, size_t, @@ -320,9 +348,12 @@ static void sdl_audio_fini (void *opaque) static struct audio_pcm_ops sdl_pcm_ops = { .init_out = sdl_init_out, .fini_out = sdl_fini_out, + /* wrapper for audio_generic_write */ .write = sdl_write, + /* wrapper for audio_generic_get_buffer_out */ .get_buffer_out = sdl_get_buffer_out, - .put_buffer_out = sdl_put_buffer_out_nowrite, + /* wrapper for audio_generic_put_buffer_out */ + .put_buffer_out = sdl_put_buffer_out, .enable_out = sdl_enable_out, }; diff --git a/audio/wavaudio.c b/audio/wavaudio.c index e46d834bd3..20e6853f85 100644 --- a/audio/wavaudio.c +++ b/audio/wavaudio.c @@ -197,6 +197,7 @@ static struct audio_pcm_ops wav_pcm_ops = { .init_out = wav_init_out, .fini_out = wav_fini_out, .write = wav_write_out, + .run_buffer_out = audio_generic_run_buffer_out, .enable_out = wav_enable_out, }; diff --git a/authz/listfile.c b/authz/listfile.c index e7a8c19bcb..b71f57d30a 100644 --- a/authz/listfile.c +++ b/authz/listfile.c @@ -239,7 +239,7 @@ qauthz_list_file_init(Object *obj) authz->file_watch = -1; #ifdef CONFIG_INOTIFY1 - authz->refresh = TRUE; + authz->refresh = true; #endif } diff --git a/blockdev-nbd.c b/blockdev-nbd.c index 8c20baa4a4..de2f2ff713 100644 --- a/blockdev-nbd.c +++ b/blockdev-nbd.c @@ -144,6 +144,7 @@ void qmp_nbd_server_start(SocketAddressLegacy *addr, } void qmp_nbd_server_add(const char *device, bool has_name, const char *name, + bool has_description, const char *description, bool has_writable, bool writable, bool has_bitmap, const char *bitmap, Error **errp) { @@ -167,6 +168,11 @@ void qmp_nbd_server_add(const char *device, bool has_name, const char *name, return; } + if (has_description && strlen(description) > NBD_MAX_STRING_SIZE) { + error_setg(errp, "description '%s' too long", description); + return; + } + if (nbd_export_find(name)) { error_setg(errp, "NBD server already has export named '%s'", name); return; @@ -195,7 +201,8 @@ void qmp_nbd_server_add(const char *device, bool has_name, const char *name, writable = false; } - exp = nbd_export_new(bs, 0, len, name, NULL, bitmap, !writable, !writable, + exp = nbd_export_new(bs, 0, len, name, description, bitmap, + !writable, !writable, NULL, false, on_eject_blk, errp); if (!exp) { goto out; diff --git a/docs/arm-cpu-features.rst b/docs/arm-cpu-features.rst index dbf3b7cf42..fc1623aeca 100644 --- a/docs/arm-cpu-features.rst +++ b/docs/arm-cpu-features.rst @@ -185,7 +185,7 @@ the list of KVM VCPU features and their descriptions. kvm-no-adjvtime By default kvm-no-adjvtime is disabled. This means that by default the virtual time - adjustment is enabled (vtime is *not not* + adjustment is enabled (vtime is not *not* adjusted). When virtual time adjustment is enabled each diff --git a/docs/interop/qemu-nbd.rst b/docs/interop/qemu-nbd.rst index df7b6b9d0d..e548403100 100644 --- a/docs/interop/qemu-nbd.rst +++ b/docs/interop/qemu-nbd.rst @@ -72,13 +72,6 @@ driver options if ``--image-opts`` is specified. Export the disk as read-only. -.. option:: -P, --partition=NUM - - Deprecated: Only expose MBR partition *NUM*. Understands physical - partitions 1-4 and logical partition 5. New code should instead use - :option:`--image-opts` with the raw driver wrapping a subset of the - original image. - .. option:: -B, --bitmap=NAME If *filename* has a qcow2 persistent bitmap *NAME*, expose @@ -224,14 +217,14 @@ a 1 megabyte subset of a raw file, using the export name 'subset': -t -x subset -p 10810 \ --image-opts driver=raw,offset=1M,size=1M,file.driver=file,file.filename=file.raw -Serve a read-only copy of just the first MBR partition of a guest -image over a Unix socket with as many as 5 simultaneous readers, with -a persistent process forked as a daemon: +Serve a read-only copy of a guest image over a Unix socket with as +many as 5 simultaneous readers, with a persistent process forked as a +daemon: :: qemu-nbd --fork --persistent --shared=5 --socket=/path/to/sock \ - --partition=1 --read-only --format=qcow2 file.qcow2 + --read-only --format=qcow2 file.qcow2 Expose the guest-visible contents of a qcow2 file via a block device /dev/nbd0 (and possibly creating /dev/nbd0p1 and friends for diff --git a/hw/arm/stellaris.c b/hw/arm/stellaris.c index bb025e0bd0..221a78674e 100644 --- a/hw/arm/stellaris.c +++ b/hw/arm/stellaris.c @@ -347,11 +347,15 @@ static void stellaris_gptm_init(Object *obj) sysbus_init_mmio(sbd, &s->iomem); s->opaque[0] = s->opaque[1] = s; +} + +static void stellaris_gptm_realize(DeviceState *dev, Error **errp) +{ + gptm_state *s = STELLARIS_GPTM(dev); s->timer[0] = timer_new_ns(QEMU_CLOCK_VIRTUAL, gptm_tick, &s->opaque[0]); s->timer[1] = timer_new_ns(QEMU_CLOCK_VIRTUAL, gptm_tick, &s->opaque[1]); } - /* System controller. */ typedef struct { @@ -1536,6 +1540,7 @@ static void stellaris_gptm_class_init(ObjectClass *klass, void *data) DeviceClass *dc = DEVICE_CLASS(klass); dc->vmsd = &vmstate_stellaris_gptm; + dc->realize = stellaris_gptm_realize; } static const TypeInfo stellaris_gptm_info = { diff --git a/hw/dma/bcm2835_dma.c b/hw/dma/bcm2835_dma.c index 1e458d7fba..ccff5ed55b 100644 --- a/hw/dma/bcm2835_dma.c +++ b/hw/dma/bcm2835_dma.c @@ -54,7 +54,7 @@ static void bcm2835_dma_update(BCM2835DMAState *s, unsigned c) { BCM2835DMAChan *ch = &s->chan[c]; - uint32_t data, xlen, ylen; + uint32_t data, xlen, xlen_td, ylen; int16_t dst_stride, src_stride; if (!(s->enable & (1 << c))) { @@ -70,18 +70,19 @@ static void bcm2835_dma_update(BCM2835DMAState *s, unsigned c) ch->stride = ldl_le_phys(&s->dma_as, ch->conblk_ad + 16); ch->nextconbk = ldl_le_phys(&s->dma_as, ch->conblk_ad + 20); + ylen = 1; if (ch->ti & BCM2708_DMA_TDMODE) { /* 2D transfer mode */ - ylen = (ch->txfr_len >> 16) & 0x3fff; + ylen += (ch->txfr_len >> 16) & 0x3fff; xlen = ch->txfr_len & 0xffff; dst_stride = ch->stride >> 16; src_stride = ch->stride & 0xffff; } else { - ylen = 1; xlen = ch->txfr_len; dst_stride = 0; src_stride = 0; } + xlen_td = xlen; while (ylen != 0) { /* Normal transfer mode */ @@ -117,6 +118,7 @@ static void bcm2835_dma_update(BCM2835DMAState *s, unsigned c) if (--ylen != 0) { ch->source_ad += src_stride; ch->dest_ad += dst_stride; + xlen = xlen_td; } } ch->cs |= BCM2708_DMA_END; diff --git a/hw/timer/armv7m_systick.c b/hw/timer/armv7m_systick.c index 85d122dbcb..74c58bcf24 100644 --- a/hw/timer/armv7m_systick.c +++ b/hw/timer/armv7m_systick.c @@ -216,6 +216,11 @@ static void systick_instance_init(Object *obj) memory_region_init_io(&s->iomem, obj, &systick_ops, s, "systick", 0xe0); sysbus_init_mmio(sbd, &s->iomem); sysbus_init_irq(sbd, &s->irq); +} + +static void systick_realize(DeviceState *dev, Error **errp) +{ + SysTickState *s = SYSTICK(dev); s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, systick_timer_tick, s); } @@ -238,6 +243,7 @@ static void systick_class_init(ObjectClass *klass, void *data) dc->vmsd = &vmstate_systick; dc->reset = systick_reset; + dc->realize = systick_realize; } static const TypeInfo armv7m_systick_info = { diff --git a/hw/timer/stm32f2xx_timer.c b/hw/timer/stm32f2xx_timer.c index fb370ce0f0..06ec8a02c2 100644 --- a/hw/timer/stm32f2xx_timer.c +++ b/hw/timer/stm32f2xx_timer.c @@ -314,7 +314,11 @@ static void stm32f2xx_timer_init(Object *obj) memory_region_init_io(&s->iomem, obj, &stm32f2xx_timer_ops, s, "stm32f2xx_timer", 0x400); sysbus_init_mmio(SYS_BUS_DEVICE(obj), &s->iomem); +} +static void stm32f2xx_timer_realize(DeviceState *dev, Error **errp) +{ + STM32F2XXTimerState *s = STM32F2XXTIMER(dev); s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, stm32f2xx_timer_interrupt, s); } @@ -325,6 +329,7 @@ static void stm32f2xx_timer_class_init(ObjectClass *klass, void *data) dc->reset = stm32f2xx_timer_reset; device_class_set_props(dc, stm32f2xx_timer_properties); dc->vmsd = &vmstate_stm32f2xx_timer; + dc->realize = stm32f2xx_timer_realize; } static const TypeInfo stm32f2xx_timer_info = { diff --git a/hw/vfio/Kconfig b/hw/vfio/Kconfig index f0eaa75ce7..7cdba0560a 100644 --- a/hw/vfio/Kconfig +++ b/hw/vfio/Kconfig @@ -36,3 +36,8 @@ config VFIO_AP default y select VFIO depends on LINUX && S390_CCW_VIRTIO + +config VFIO_IGD + bool + default y if PC_PCI + depends on VFIO_PCI diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs index abad8b818c..9bb1c09e84 100644 --- a/hw/vfio/Makefile.objs +++ b/hw/vfio/Makefile.objs @@ -5,3 +5,4 @@ obj-$(CONFIG_VFIO_PLATFORM) += platform.o obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o obj-$(CONFIG_VFIO_AP) += ap.o +obj-$(CONFIG_VFIO_IGD) += igd.o diff --git a/hw/vfio/igd.c b/hw/vfio/igd.c new file mode 100644 index 0000000000..64e332746b --- /dev/null +++ b/hw/vfio/igd.c @@ -0,0 +1,616 @@ +/* + * IGD device quirks + * + * Copyright Red Hat, Inc. 2016 + * + * Authors: + * Alex Williamson <alex.williamson@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qemu/units.h" +#include "qapi/error.h" +#include "hw/hw.h" +#include "hw/nvram/fw_cfg.h" +#include "pci.h" +#include "trace.h" + +/* + * Intel IGD support + * + * Obviously IGD is not a discrete device, this is evidenced not only by it + * being integrated into the CPU, but by the various chipset and BIOS + * dependencies that it brings along with it. Intel is trying to move away + * from this and Broadwell and newer devices can run in what Intel calls + * "Universal Pass-Through" mode, or UPT. Theoretically in UPT mode, nothing + * more is required beyond assigning the IGD device to a VM. There are + * however support limitations to this mode. It only supports IGD as a + * secondary graphics device in the VM and it doesn't officially support any + * physical outputs. + * + * The code here attempts to enable what we'll call legacy mode assignment, + * IGD retains most of the capabilities we expect for it to have on bare + * metal. To enable this mode, the IGD device must be assigned to the VM + * at PCI address 00:02.0, it must have a ROM, it very likely needs VGA + * support, we must have VM BIOS support for reserving and populating some + * of the required tables, and we need to tweak the chipset with revisions + * and IDs and an LPC/ISA bridge device. The intention is to make all of + * this happen automatically by installing the device at the correct VM PCI + * bus address. If any of the conditions are not met, we cross our fingers + * and hope the user knows better. + * + * NB - It is possible to enable physical outputs in UPT mode by supplying + * an OpRegion table. We don't do this by default because the guest driver + * behaves differently if an OpRegion is provided and no monitor is attached + * vs no OpRegion and a monitor being attached or not. Effectively, if a + * headless setup is desired, the OpRegion gets in the way of that. + */ + +/* + * This presumes the device is already known to be an Intel VGA device, so we + * take liberties in which device ID bits match which generation. This should + * not be taken as an indication that all the devices are supported, or even + * supportable, some of them don't even support VT-d. + * See linux:include/drm/i915_pciids.h for IDs. + */ +static int igd_gen(VFIOPCIDevice *vdev) +{ + if ((vdev->device_id & 0xfff) == 0xa84) { + return 8; /* Broxton */ + } + + switch (vdev->device_id & 0xff00) { + /* Old, untested, unavailable, unknown */ + case 0x0000: + case 0x2500: + case 0x2700: + case 0x2900: + case 0x2a00: + case 0x2e00: + case 0x3500: + case 0xa000: + return -1; + /* SandyBridge, IvyBridge, ValleyView, Haswell */ + case 0x0100: + case 0x0400: + case 0x0a00: + case 0x0c00: + case 0x0d00: + case 0x0f00: + return 6; + /* BroadWell, CherryView, SkyLake, KabyLake */ + case 0x1600: + case 0x1900: + case 0x2200: + case 0x5900: + return 8; + } + + return 8; /* Assume newer is compatible */ +} + +typedef struct VFIOIGDQuirk { + struct VFIOPCIDevice *vdev; + uint32_t index; + uint32_t bdsm; +} VFIOIGDQuirk; + +#define IGD_GMCH 0x50 /* Graphics Control Register */ +#define IGD_BDSM 0x5c /* Base Data of Stolen Memory */ + + +/* + * The rather short list of registers that we copy from the host devices. + * The LPC/ISA bridge values are definitely needed to support the vBIOS, the + * host bridge values may or may not be needed depending on the guest OS. + * Since we're only munging revision and subsystem values on the host bridge, + * we don't require our own device. The LPC/ISA bridge needs to be our very + * own though. + */ +typedef struct { + uint8_t offset; + uint8_t len; +} IGDHostInfo; + +static const IGDHostInfo igd_host_bridge_infos[] = { + {PCI_REVISION_ID, 2}, + {PCI_SUBSYSTEM_VENDOR_ID, 2}, + {PCI_SUBSYSTEM_ID, 2}, +}; + +static const IGDHostInfo igd_lpc_bridge_infos[] = { + {PCI_VENDOR_ID, 2}, + {PCI_DEVICE_ID, 2}, + {PCI_REVISION_ID, 2}, + {PCI_SUBSYSTEM_VENDOR_ID, 2}, + {PCI_SUBSYSTEM_ID, 2}, +}; + +static int vfio_pci_igd_copy(VFIOPCIDevice *vdev, PCIDevice *pdev, + struct vfio_region_info *info, + const IGDHostInfo *list, int len) +{ + int i, ret; + + for (i = 0; i < len; i++) { + ret = pread(vdev->vbasedev.fd, pdev->config + list[i].offset, + list[i].len, info->offset + list[i].offset); + if (ret != list[i].len) { + error_report("IGD copy failed: %m"); + return -errno; + } + } + + return 0; +} + +/* + * Stuff a few values into the host bridge. + */ +static int vfio_pci_igd_host_init(VFIOPCIDevice *vdev, + struct vfio_region_info *info) +{ + PCIBus *bus; + PCIDevice *host_bridge; + int ret; + + bus = pci_device_root_bus(&vdev->pdev); + host_bridge = pci_find_device(bus, 0, PCI_DEVFN(0, 0)); + + if (!host_bridge) { + error_report("Can't find host bridge"); + return -ENODEV; + } + + ret = vfio_pci_igd_copy(vdev, host_bridge, info, igd_host_bridge_infos, + ARRAY_SIZE(igd_host_bridge_infos)); + if (!ret) { + trace_vfio_pci_igd_host_bridge_enabled(vdev->vbasedev.name); + } + + return ret; +} + +/* + * IGD LPC/ISA bridge support code. The vBIOS needs this, but we can't write + * arbitrary values into just any bridge, so we must create our own. We try + * to handle if the user has created it for us, which they might want to do + * to enable multifunction so we don't occupy the whole PCI slot. + */ +static void vfio_pci_igd_lpc_bridge_realize(PCIDevice *pdev, Error **errp) +{ + if (pdev->devfn != PCI_DEVFN(0x1f, 0)) { + error_setg(errp, "VFIO dummy ISA/LPC bridge must have address 1f.0"); + } +} + +static void vfio_pci_igd_lpc_bridge_class_init(ObjectClass *klass, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); + + set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); + dc->desc = "VFIO dummy ISA/LPC bridge for IGD assignment"; + dc->hotpluggable = false; + k->realize = vfio_pci_igd_lpc_bridge_realize; + k->class_id = PCI_CLASS_BRIDGE_ISA; +} + +static TypeInfo vfio_pci_igd_lpc_bridge_info = { + .name = "vfio-pci-igd-lpc-bridge", + .parent = TYPE_PCI_DEVICE, + .class_init = vfio_pci_igd_lpc_bridge_class_init, + .interfaces = (InterfaceInfo[]) { + { INTERFACE_CONVENTIONAL_PCI_DEVICE }, + { }, + }, +}; + +static void vfio_pci_igd_register_types(void) +{ + type_register_static(&vfio_pci_igd_lpc_bridge_info); +} + +type_init(vfio_pci_igd_register_types) + +static int vfio_pci_igd_lpc_init(VFIOPCIDevice *vdev, + struct vfio_region_info *info) +{ + PCIDevice *lpc_bridge; + int ret; + + lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), + 0, PCI_DEVFN(0x1f, 0)); + if (!lpc_bridge) { + lpc_bridge = pci_create_simple(pci_device_root_bus(&vdev->pdev), + PCI_DEVFN(0x1f, 0), "vfio-pci-igd-lpc-bridge"); + } + + ret = vfio_pci_igd_copy(vdev, lpc_bridge, info, igd_lpc_bridge_infos, + ARRAY_SIZE(igd_lpc_bridge_infos)); + if (!ret) { + trace_vfio_pci_igd_lpc_bridge_enabled(vdev->vbasedev.name); + } + + return ret; +} + +/* + * IGD Gen8 and newer support up to 8MB for the GTT and use a 64bit PTE + * entry, older IGDs use 2MB and 32bit. Each PTE maps a 4k page. Therefore + * we either have 2M/4k * 4 = 2k or 8M/4k * 8 = 16k as the maximum iobar index + * for programming the GTT. + * + * See linux:include/drm/i915_drm.h for shift and mask values. + */ +static int vfio_igd_gtt_max(VFIOPCIDevice *vdev) +{ + uint32_t gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); + int ggms, gen = igd_gen(vdev); + + gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); + ggms = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; + if (gen > 6) { + ggms = 1 << ggms; + } + + ggms *= MiB; + + return (ggms / (4 * KiB)) * (gen < 8 ? 4 : 8); +} + +/* + * The IGD ROM will make use of stolen memory (GGMS) for support of VESA modes. + * Somehow the host stolen memory range is used for this, but how the ROM gets + * it is a mystery, perhaps it's hardcoded into the ROM. Thankfully though, it + * reprograms the GTT through the IOBAR where we can trap it and transpose the + * programming to the VM allocated buffer. That buffer gets reserved by the VM + * firmware via the fw_cfg entry added below. Here we're just monitoring the + * IOBAR address and data registers to detect a write sequence targeting the + * GTTADR. This code is developed by observed behavior and doesn't have a + * direct spec reference, unfortunately. + */ +static uint64_t vfio_igd_quirk_data_read(void *opaque, + hwaddr addr, unsigned size) +{ + VFIOIGDQuirk *igd = opaque; + VFIOPCIDevice *vdev = igd->vdev; + + igd->index = ~0; + + return vfio_region_read(&vdev->bars[4].region, addr + 4, size); +} + +static void vfio_igd_quirk_data_write(void *opaque, hwaddr addr, + uint64_t data, unsigned size) +{ + VFIOIGDQuirk *igd = opaque; + VFIOPCIDevice *vdev = igd->vdev; + uint64_t val = data; + int gen = igd_gen(vdev); + + /* + * Programming the GGMS starts at index 0x1 and uses every 4th index (ie. + * 0x1, 0x5, 0x9, 0xd,...). For pre-Gen8 each 4-byte write is a whole PTE + * entry, with 0th bit enable set. For Gen8 and up, PTEs are 64bit, so + * entries 0x5 & 0xd are the high dword, in our case zero. Each PTE points + * to a 4k page, which we translate to a page from the VM allocated region, + * pointed to by the BDSM register. If this is not set, we fail. + * + * We trap writes to the full configured GTT size, but we typically only + * see the vBIOS writing up to (nearly) the 1MB barrier. In fact it often + * seems to miss the last entry for an even 1MB GTT. Doing a gratuitous + * write of that last entry does work, but is hopefully unnecessary since + * we clear the previous GTT on initialization. + */ + if ((igd->index % 4 == 1) && igd->index < vfio_igd_gtt_max(vdev)) { + if (gen < 8 || (igd->index % 8 == 1)) { + uint32_t base; + + base = pci_get_long(vdev->pdev.config + IGD_BDSM); + if (!base) { + hw_error("vfio-igd: Guest attempted to program IGD GTT before " + "BIOS reserved stolen memory. Unsupported BIOS?"); + } + + val = data - igd->bdsm + base; + } else { + val = 0; /* upper 32bits of pte, we only enable below 4G PTEs */ + } + + trace_vfio_pci_igd_bar4_write(vdev->vbasedev.name, + igd->index, data, val); + } + + vfio_region_write(&vdev->bars[4].region, addr + 4, val, size); + + igd->index = ~0; +} + +static const MemoryRegionOps vfio_igd_data_quirk = { + .read = vfio_igd_quirk_data_read, + .write = vfio_igd_quirk_data_write, + .endianness = DEVICE_LITTLE_ENDIAN, +}; + +static uint64_t vfio_igd_quirk_index_read(void *opaque, + hwaddr addr, unsigned size) +{ + VFIOIGDQuirk *igd = opaque; + VFIOPCIDevice *vdev = igd->vdev; + + igd->index = ~0; + + return vfio_region_read(&vdev->bars[4].region, addr, size); +} + +static void vfio_igd_quirk_index_write(void *opaque, hwaddr addr, + uint64_t data, unsigned size) +{ + VFIOIGDQuirk *igd = opaque; + VFIOPCIDevice *vdev = igd->vdev; + + igd->index = data; + + vfio_region_write(&vdev->bars[4].region, addr, data, size); +} + +static const MemoryRegionOps vfio_igd_index_quirk = { + .read = vfio_igd_quirk_index_read, + .write = vfio_igd_quirk_index_write, + .endianness = DEVICE_LITTLE_ENDIAN, +}; + +void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr) +{ + struct vfio_region_info *rom = NULL, *opregion = NULL, + *host = NULL, *lpc = NULL; + VFIOQuirk *quirk; + VFIOIGDQuirk *igd; + PCIDevice *lpc_bridge; + int i, ret, ggms_mb, gms_mb = 0, gen; + uint64_t *bdsm_size; + uint32_t gmch; + uint16_t cmd_orig, cmd; + Error *err = NULL; + + /* + * This must be an Intel VGA device at address 00:02.0 for us to even + * consider enabling legacy mode. The vBIOS has dependencies on the + * PCI bus address. + */ + if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) || + !vfio_is_vga(vdev) || nr != 4 || + &vdev->pdev != pci_find_device(pci_device_root_bus(&vdev->pdev), + 0, PCI_DEVFN(0x2, 0))) { + return; + } + + /* + * We need to create an LPC/ISA bridge at PCI bus address 00:1f.0 that we + * can stuff host values into, so if there's already one there and it's not + * one we can hack on, legacy mode is no-go. Sorry Q35. + */ + lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), + 0, PCI_DEVFN(0x1f, 0)); + if (lpc_bridge && !object_dynamic_cast(OBJECT(lpc_bridge), + "vfio-pci-igd-lpc-bridge")) { + error_report("IGD device %s cannot support legacy mode due to existing " + "devices at address 1f.0", vdev->vbasedev.name); + return; + } + + /* + * IGD is not a standard, they like to change their specs often. We + * only attempt to support back to SandBridge and we hope that newer + * devices maintain compatibility with generation 8. + */ + gen = igd_gen(vdev); + if (gen != 6 && gen != 8) { + error_report("IGD device %s is unsupported in legacy mode, " + "try SandyBridge or newer", vdev->vbasedev.name); + return; + } + + /* + * Most of what we're doing here is to enable the ROM to run, so if + * there's no ROM, there's no point in setting up this quirk. + * NB. We only seem to get BIOS ROMs, so a UEFI VM would need CSM support. + */ + ret = vfio_get_region_info(&vdev->vbasedev, + VFIO_PCI_ROM_REGION_INDEX, &rom); + if ((ret || !rom->size) && !vdev->pdev.romfile) { + error_report("IGD device %s has no ROM, legacy mode disabled", + vdev->vbasedev.name); + goto out; + } + + /* + * Ignore the hotplug corner case, mark the ROM failed, we can't + * create the devices we need for legacy mode in the hotplug scenario. + */ + if (vdev->pdev.qdev.hotplugged) { + error_report("IGD device %s hotplugged, ROM disabled, " + "legacy mode disabled", vdev->vbasedev.name); + vdev->rom_read_failed = true; + goto out; + } + + /* + * Check whether we have all the vfio device specific regions to + * support legacy mode (added in Linux v4.6). If not, bail. + */ + ret = vfio_get_dev_region_info(&vdev->vbasedev, + VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, + VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion); + if (ret) { + error_report("IGD device %s does not support OpRegion access," + "legacy mode disabled", vdev->vbasedev.name); + goto out; + } + + ret = vfio_get_dev_region_info(&vdev->vbasedev, + VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, + VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG, &host); + if (ret) { + error_report("IGD device %s does not support host bridge access," + "legacy mode disabled", vdev->vbasedev.name); + goto out; + } + + ret = vfio_get_dev_region_info(&vdev->vbasedev, + VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, + VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG, &lpc); + if (ret) { + error_report("IGD device %s does not support LPC bridge access," + "legacy mode disabled", vdev->vbasedev.name); + goto out; + } + + gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, 4); + + /* + * If IGD VGA Disable is clear (expected) and VGA is not already enabled, + * try to enable it. Probably shouldn't be using legacy mode without VGA, + * but also no point in us enabling VGA if disabled in hardware. + */ + if (!(gmch & 0x2) && !vdev->vga && vfio_populate_vga(vdev, &err)) { + error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + error_report("IGD device %s failed to enable VGA access, " + "legacy mode disabled", vdev->vbasedev.name); + goto out; + } + + /* Create our LPC/ISA bridge */ + ret = vfio_pci_igd_lpc_init(vdev, lpc); + if (ret) { + error_report("IGD device %s failed to create LPC bridge, " + "legacy mode disabled", vdev->vbasedev.name); + goto out; + } + + /* Stuff some host values into the VM PCI host bridge */ + ret = vfio_pci_igd_host_init(vdev, host); + if (ret) { + error_report("IGD device %s failed to modify host bridge, " + "legacy mode disabled", vdev->vbasedev.name); + goto out; + } + + /* Setup OpRegion access */ + ret = vfio_pci_igd_opregion_init(vdev, opregion, &err); + if (ret) { + error_append_hint(&err, "IGD legacy mode disabled\n"); + error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + goto out; + } + + /* Setup our quirk to munge GTT addresses to the VM allocated buffer */ + quirk = vfio_quirk_alloc(2); + igd = quirk->data = g_malloc0(sizeof(*igd)); + igd->vdev = vdev; + igd->index = ~0; + igd->bdsm = vfio_pci_read_config(&vdev->pdev, IGD_BDSM, 4); + igd->bdsm &= ~((1 * MiB) - 1); /* 1MB aligned */ + + memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_igd_index_quirk, + igd, "vfio-igd-index-quirk", 4); + memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, + 0, &quirk->mem[0], 1); + + memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_igd_data_quirk, + igd, "vfio-igd-data-quirk", 4); + memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, + 4, &quirk->mem[1], 1); + + QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); + + /* Determine the size of stolen memory needed for GTT */ + ggms_mb = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; + if (gen > 6) { + ggms_mb = 1 << ggms_mb; + } + + /* + * Assume we have no GMS memory, but allow it to be overrided by device + * option (experimental). The spec doesn't actually allow zero GMS when + * when IVD (IGD VGA Disable) is clear, but the claim is that it's unused, + * so let's not waste VM memory for it. + */ + gmch &= ~((gen < 8 ? 0x1f : 0xff) << (gen < 8 ? 3 : 8)); + + if (vdev->igd_gms) { + if (vdev->igd_gms <= 0x10) { + gms_mb = vdev->igd_gms * 32; + gmch |= vdev->igd_gms << (gen < 8 ? 3 : 8); + } else { + error_report("Unsupported IGD GMS value 0x%x", vdev->igd_gms); + vdev->igd_gms = 0; + } + } + + /* + * Request reserved memory for stolen memory via fw_cfg. VM firmware + * must allocate a 1MB aligned reserved memory region below 4GB with + * the requested size (in bytes) for use by the Intel PCI class VGA + * device at VM address 00:02.0. The base address of this reserved + * memory region must be written to the device BDSM regsiter at PCI + * config offset 0x5C. + */ + bdsm_size = g_malloc(sizeof(*bdsm_size)); + *bdsm_size = cpu_to_le64((ggms_mb + gms_mb) * MiB); + fw_cfg_add_file(fw_cfg_find(), "etc/igd-bdsm-size", + bdsm_size, sizeof(*bdsm_size)); + + /* GMCH is read-only, emulated */ + pci_set_long(vdev->pdev.config + IGD_GMCH, gmch); + pci_set_long(vdev->pdev.wmask + IGD_GMCH, 0); + pci_set_long(vdev->emulated_config_bits + IGD_GMCH, ~0); + + /* BDSM is read-write, emulated. The BIOS needs to be able to write it */ + pci_set_long(vdev->pdev.config + IGD_BDSM, 0); + pci_set_long(vdev->pdev.wmask + IGD_BDSM, ~0); + pci_set_long(vdev->emulated_config_bits + IGD_BDSM, ~0); + + /* + * This IOBAR gives us access to GTTADR, which allows us to write to + * the GTT itself. So let's go ahead and write zero to all the GTT + * entries to avoid spurious DMA faults. Be sure I/O access is enabled + * before talking to the device. + */ + if (pread(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), + vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { + error_report("IGD device %s - failed to read PCI command register", + vdev->vbasedev.name); + } + + cmd = cmd_orig | PCI_COMMAND_IO; + + if (pwrite(vdev->vbasedev.fd, &cmd, sizeof(cmd), + vdev->config_offset + PCI_COMMAND) != sizeof(cmd)) { + error_report("IGD device %s - failed to write PCI command register", + vdev->vbasedev.name); + } + + for (i = 1; i < vfio_igd_gtt_max(vdev); i += 4) { + vfio_region_write(&vdev->bars[4].region, 0, i, 4); + vfio_region_write(&vdev->bars[4].region, 4, 0, 4); + } + + if (pwrite(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), + vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { + error_report("IGD device %s - failed to restore PCI command register", + vdev->vbasedev.name); + } + + trace_vfio_pci_igd_bdsm_enabled(vdev->vbasedev.name, ggms_mb + gms_mb); + +out: + g_free(rom); + g_free(opregion); + g_free(host); + g_free(lpc); +} diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c index 136f3a9ad6..2d348f8237 100644 --- a/hw/vfio/pci-quirks.c +++ b/hw/vfio/pci-quirks.c @@ -26,21 +26,6 @@ #include "pci.h" #include "trace.h" -/* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */ -static bool vfio_pci_is(VFIOPCIDevice *vdev, uint32_t vendor, uint32_t device) -{ - return (vendor == PCI_ANY_ID || vendor == vdev->vendor_id) && - (device == PCI_ANY_ID || device == vdev->device_id); -} - -static bool vfio_is_vga(VFIOPCIDevice *vdev) -{ - PCIDevice *pdev = &vdev->pdev; - uint16_t class = pci_get_word(pdev->config + PCI_CLASS_DEVICE); - - return class == PCI_CLASS_DISPLAY_VGA; -} - /* * List of device ids/vendor ids for which to disable * option rom loading. This avoids the guest hangs during rom @@ -283,7 +268,7 @@ static const MemoryRegionOps vfio_ati_3c3_quirk = { .endianness = DEVICE_LITTLE_ENDIAN, }; -static VFIOQuirk *vfio_quirk_alloc(int nr_mem) +VFIOQuirk *vfio_quirk_alloc(int nr_mem) { VFIOQuirk *quirk = g_new0(VFIOQuirk, 1); QLIST_INIT(&quirk->ioeventfds); @@ -1166,88 +1151,6 @@ static void vfio_probe_rtl8168_bar2_quirk(VFIOPCIDevice *vdev, int nr) trace_vfio_quirk_rtl8168_probe(vdev->vbasedev.name); } -/* - * Intel IGD support - * - * Obviously IGD is not a discrete device, this is evidenced not only by it - * being integrated into the CPU, but by the various chipset and BIOS - * dependencies that it brings along with it. Intel is trying to move away - * from this and Broadwell and newer devices can run in what Intel calls - * "Universal Pass-Through" mode, or UPT. Theoretically in UPT mode, nothing - * more is required beyond assigning the IGD device to a VM. There are - * however support limitations to this mode. It only supports IGD as a - * secondary graphics device in the VM and it doesn't officially support any - * physical outputs. - * - * The code here attempts to enable what we'll call legacy mode assignment, - * IGD retains most of the capabilities we expect for it to have on bare - * metal. To enable this mode, the IGD device must be assigned to the VM - * at PCI address 00:02.0, it must have a ROM, it very likely needs VGA - * support, we must have VM BIOS support for reserving and populating some - * of the required tables, and we need to tweak the chipset with revisions - * and IDs and an LPC/ISA bridge device. The intention is to make all of - * this happen automatically by installing the device at the correct VM PCI - * bus address. If any of the conditions are not met, we cross our fingers - * and hope the user knows better. - * - * NB - It is possible to enable physical outputs in UPT mode by supplying - * an OpRegion table. We don't do this by default because the guest driver - * behaves differently if an OpRegion is provided and no monitor is attached - * vs no OpRegion and a monitor being attached or not. Effectively, if a - * headless setup is desired, the OpRegion gets in the way of that. - */ - -/* - * This presumes the device is already known to be an Intel VGA device, so we - * take liberties in which device ID bits match which generation. This should - * not be taken as an indication that all the devices are supported, or even - * supportable, some of them don't even support VT-d. - * See linux:include/drm/i915_pciids.h for IDs. - */ -static int igd_gen(VFIOPCIDevice *vdev) -{ - if ((vdev->device_id & 0xfff) == 0xa84) { - return 8; /* Broxton */ - } - - switch (vdev->device_id & 0xff00) { - /* Old, untested, unavailable, unknown */ - case 0x0000: - case 0x2500: - case 0x2700: - case 0x2900: - case 0x2a00: - case 0x2e00: - case 0x3500: - case 0xa000: - return -1; - /* SandyBridge, IvyBridge, ValleyView, Haswell */ - case 0x0100: - case 0x0400: - case 0x0a00: - case 0x0c00: - case 0x0d00: - case 0x0f00: - return 6; - /* BroadWell, CherryView, SkyLake, KabyLake */ - case 0x1600: - case 0x1900: - case 0x2200: - case 0x5900: - return 8; - } - - return 8; /* Assume newer is compatible */ -} - -typedef struct VFIOIGDQuirk { - struct VFIOPCIDevice *vdev; - uint32_t index; - uint32_t bdsm; -} VFIOIGDQuirk; - -#define IGD_GMCH 0x50 /* Graphics Control Register */ -#define IGD_BDSM 0x5c /* Base Data of Stolen Memory */ #define IGD_ASLS 0xfc /* ASL Storage Register */ /* @@ -1299,519 +1202,6 @@ int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev, } /* - * The rather short list of registers that we copy from the host devices. - * The LPC/ISA bridge values are definitely needed to support the vBIOS, the - * host bridge values may or may not be needed depending on the guest OS. - * Since we're only munging revision and subsystem values on the host bridge, - * we don't require our own device. The LPC/ISA bridge needs to be our very - * own though. - */ -typedef struct { - uint8_t offset; - uint8_t len; -} IGDHostInfo; - -static const IGDHostInfo igd_host_bridge_infos[] = { - {PCI_REVISION_ID, 2}, - {PCI_SUBSYSTEM_VENDOR_ID, 2}, - {PCI_SUBSYSTEM_ID, 2}, -}; - -static const IGDHostInfo igd_lpc_bridge_infos[] = { - {PCI_VENDOR_ID, 2}, - {PCI_DEVICE_ID, 2}, - {PCI_REVISION_ID, 2}, - {PCI_SUBSYSTEM_VENDOR_ID, 2}, - {PCI_SUBSYSTEM_ID, 2}, -}; - -static int vfio_pci_igd_copy(VFIOPCIDevice *vdev, PCIDevice *pdev, - struct vfio_region_info *info, - const IGDHostInfo *list, int len) -{ - int i, ret; - - for (i = 0; i < len; i++) { - ret = pread(vdev->vbasedev.fd, pdev->config + list[i].offset, - list[i].len, info->offset + list[i].offset); - if (ret != list[i].len) { - error_report("IGD copy failed: %m"); - return -errno; - } - } - - return 0; -} - -/* - * Stuff a few values into the host bridge. - */ -static int vfio_pci_igd_host_init(VFIOPCIDevice *vdev, - struct vfio_region_info *info) -{ - PCIBus *bus; - PCIDevice *host_bridge; - int ret; - - bus = pci_device_root_bus(&vdev->pdev); - host_bridge = pci_find_device(bus, 0, PCI_DEVFN(0, 0)); - - if (!host_bridge) { - error_report("Can't find host bridge"); - return -ENODEV; - } - - ret = vfio_pci_igd_copy(vdev, host_bridge, info, igd_host_bridge_infos, - ARRAY_SIZE(igd_host_bridge_infos)); - if (!ret) { - trace_vfio_pci_igd_host_bridge_enabled(vdev->vbasedev.name); - } - - return ret; -} - -/* - * IGD LPC/ISA bridge support code. The vBIOS needs this, but we can't write - * arbitrary values into just any bridge, so we must create our own. We try - * to handle if the user has created it for us, which they might want to do - * to enable multifunction so we don't occupy the whole PCI slot. - */ -static void vfio_pci_igd_lpc_bridge_realize(PCIDevice *pdev, Error **errp) -{ - if (pdev->devfn != PCI_DEVFN(0x1f, 0)) { - error_setg(errp, "VFIO dummy ISA/LPC bridge must have address 1f.0"); - } -} - -static void vfio_pci_igd_lpc_bridge_class_init(ObjectClass *klass, void *data) -{ - DeviceClass *dc = DEVICE_CLASS(klass); - PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); - - set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); - dc->desc = "VFIO dummy ISA/LPC bridge for IGD assignment"; - dc->hotpluggable = false; - k->realize = vfio_pci_igd_lpc_bridge_realize; - k->class_id = PCI_CLASS_BRIDGE_ISA; -} - -static TypeInfo vfio_pci_igd_lpc_bridge_info = { - .name = "vfio-pci-igd-lpc-bridge", - .parent = TYPE_PCI_DEVICE, - .class_init = vfio_pci_igd_lpc_bridge_class_init, - .interfaces = (InterfaceInfo[]) { - { INTERFACE_CONVENTIONAL_PCI_DEVICE }, - { }, - }, -}; - -static void vfio_pci_igd_register_types(void) -{ - type_register_static(&vfio_pci_igd_lpc_bridge_info); -} - -type_init(vfio_pci_igd_register_types) - -static int vfio_pci_igd_lpc_init(VFIOPCIDevice *vdev, - struct vfio_region_info *info) -{ - PCIDevice *lpc_bridge; - int ret; - - lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), - 0, PCI_DEVFN(0x1f, 0)); - if (!lpc_bridge) { - lpc_bridge = pci_create_simple(pci_device_root_bus(&vdev->pdev), - PCI_DEVFN(0x1f, 0), "vfio-pci-igd-lpc-bridge"); - } - - ret = vfio_pci_igd_copy(vdev, lpc_bridge, info, igd_lpc_bridge_infos, - ARRAY_SIZE(igd_lpc_bridge_infos)); - if (!ret) { - trace_vfio_pci_igd_lpc_bridge_enabled(vdev->vbasedev.name); - } - - return ret; -} - -/* - * IGD Gen8 and newer support up to 8MB for the GTT and use a 64bit PTE - * entry, older IGDs use 2MB and 32bit. Each PTE maps a 4k page. Therefore - * we either have 2M/4k * 4 = 2k or 8M/4k * 8 = 16k as the maximum iobar index - * for programming the GTT. - * - * See linux:include/drm/i915_drm.h for shift and mask values. - */ -static int vfio_igd_gtt_max(VFIOPCIDevice *vdev) -{ - uint32_t gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); - int ggms, gen = igd_gen(vdev); - - gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); - ggms = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; - if (gen > 6) { - ggms = 1 << ggms; - } - - ggms *= MiB; - - return (ggms / (4 * KiB)) * (gen < 8 ? 4 : 8); -} - -/* - * The IGD ROM will make use of stolen memory (GGMS) for support of VESA modes. - * Somehow the host stolen memory range is used for this, but how the ROM gets - * it is a mystery, perhaps it's hardcoded into the ROM. Thankfully though, it - * reprograms the GTT through the IOBAR where we can trap it and transpose the - * programming to the VM allocated buffer. That buffer gets reserved by the VM - * firmware via the fw_cfg entry added below. Here we're just monitoring the - * IOBAR address and data registers to detect a write sequence targeting the - * GTTADR. This code is developed by observed behavior and doesn't have a - * direct spec reference, unfortunately. - */ -static uint64_t vfio_igd_quirk_data_read(void *opaque, - hwaddr addr, unsigned size) -{ - VFIOIGDQuirk *igd = opaque; - VFIOPCIDevice *vdev = igd->vdev; - - igd->index = ~0; - - return vfio_region_read(&vdev->bars[4].region, addr + 4, size); -} - -static void vfio_igd_quirk_data_write(void *opaque, hwaddr addr, - uint64_t data, unsigned size) -{ - VFIOIGDQuirk *igd = opaque; - VFIOPCIDevice *vdev = igd->vdev; - uint64_t val = data; - int gen = igd_gen(vdev); - - /* - * Programming the GGMS starts at index 0x1 and uses every 4th index (ie. - * 0x1, 0x5, 0x9, 0xd,...). For pre-Gen8 each 4-byte write is a whole PTE - * entry, with 0th bit enable set. For Gen8 and up, PTEs are 64bit, so - * entries 0x5 & 0xd are the high dword, in our case zero. Each PTE points - * to a 4k page, which we translate to a page from the VM allocated region, - * pointed to by the BDSM register. If this is not set, we fail. - * - * We trap writes to the full configured GTT size, but we typically only - * see the vBIOS writing up to (nearly) the 1MB barrier. In fact it often - * seems to miss the last entry for an even 1MB GTT. Doing a gratuitous - * write of that last entry does work, but is hopefully unnecessary since - * we clear the previous GTT on initialization. - */ - if ((igd->index % 4 == 1) && igd->index < vfio_igd_gtt_max(vdev)) { - if (gen < 8 || (igd->index % 8 == 1)) { - uint32_t base; - - base = pci_get_long(vdev->pdev.config + IGD_BDSM); - if (!base) { - hw_error("vfio-igd: Guest attempted to program IGD GTT before " - "BIOS reserved stolen memory. Unsupported BIOS?"); - } - - val = data - igd->bdsm + base; - } else { - val = 0; /* upper 32bits of pte, we only enable below 4G PTEs */ - } - - trace_vfio_pci_igd_bar4_write(vdev->vbasedev.name, - igd->index, data, val); - } - - vfio_region_write(&vdev->bars[4].region, addr + 4, val, size); - - igd->index = ~0; -} - -static const MemoryRegionOps vfio_igd_data_quirk = { - .read = vfio_igd_quirk_data_read, - .write = vfio_igd_quirk_data_write, - .endianness = DEVICE_LITTLE_ENDIAN, -}; - -static uint64_t vfio_igd_quirk_index_read(void *opaque, - hwaddr addr, unsigned size) -{ - VFIOIGDQuirk *igd = opaque; - VFIOPCIDevice *vdev = igd->vdev; - - igd->index = ~0; - - return vfio_region_read(&vdev->bars[4].region, addr, size); -} - -static void vfio_igd_quirk_index_write(void *opaque, hwaddr addr, - uint64_t data, unsigned size) -{ - VFIOIGDQuirk *igd = opaque; - VFIOPCIDevice *vdev = igd->vdev; - - igd->index = data; - - vfio_region_write(&vdev->bars[4].region, addr, data, size); -} - -static const MemoryRegionOps vfio_igd_index_quirk = { - .read = vfio_igd_quirk_index_read, - .write = vfio_igd_quirk_index_write, - .endianness = DEVICE_LITTLE_ENDIAN, -}; - -static void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr) -{ - struct vfio_region_info *rom = NULL, *opregion = NULL, - *host = NULL, *lpc = NULL; - VFIOQuirk *quirk; - VFIOIGDQuirk *igd; - PCIDevice *lpc_bridge; - int i, ret, ggms_mb, gms_mb = 0, gen; - uint64_t *bdsm_size; - uint32_t gmch; - uint16_t cmd_orig, cmd; - Error *err = NULL; - - /* - * This must be an Intel VGA device at address 00:02.0 for us to even - * consider enabling legacy mode. The vBIOS has dependencies on the - * PCI bus address. - */ - if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) || - !vfio_is_vga(vdev) || nr != 4 || - &vdev->pdev != pci_find_device(pci_device_root_bus(&vdev->pdev), - 0, PCI_DEVFN(0x2, 0))) { - return; - } - - /* - * We need to create an LPC/ISA bridge at PCI bus address 00:1f.0 that we - * can stuff host values into, so if there's already one there and it's not - * one we can hack on, legacy mode is no-go. Sorry Q35. - */ - lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), - 0, PCI_DEVFN(0x1f, 0)); - if (lpc_bridge && !object_dynamic_cast(OBJECT(lpc_bridge), - "vfio-pci-igd-lpc-bridge")) { - error_report("IGD device %s cannot support legacy mode due to existing " - "devices at address 1f.0", vdev->vbasedev.name); - return; - } - - /* - * IGD is not a standard, they like to change their specs often. We - * only attempt to support back to SandBridge and we hope that newer - * devices maintain compatibility with generation 8. - */ - gen = igd_gen(vdev); - if (gen != 6 && gen != 8) { - error_report("IGD device %s is unsupported in legacy mode, " - "try SandyBridge or newer", vdev->vbasedev.name); - return; - } - - /* - * Most of what we're doing here is to enable the ROM to run, so if - * there's no ROM, there's no point in setting up this quirk. - * NB. We only seem to get BIOS ROMs, so a UEFI VM would need CSM support. - */ - ret = vfio_get_region_info(&vdev->vbasedev, - VFIO_PCI_ROM_REGION_INDEX, &rom); - if ((ret || !rom->size) && !vdev->pdev.romfile) { - error_report("IGD device %s has no ROM, legacy mode disabled", - vdev->vbasedev.name); - goto out; - } - - /* - * Ignore the hotplug corner case, mark the ROM failed, we can't - * create the devices we need for legacy mode in the hotplug scenario. - */ - if (vdev->pdev.qdev.hotplugged) { - error_report("IGD device %s hotplugged, ROM disabled, " - "legacy mode disabled", vdev->vbasedev.name); - vdev->rom_read_failed = true; - goto out; - } - - /* - * Check whether we have all the vfio device specific regions to - * support legacy mode (added in Linux v4.6). If not, bail. - */ - ret = vfio_get_dev_region_info(&vdev->vbasedev, - VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, - VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion); - if (ret) { - error_report("IGD device %s does not support OpRegion access," - "legacy mode disabled", vdev->vbasedev.name); - goto out; - } - - ret = vfio_get_dev_region_info(&vdev->vbasedev, - VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, - VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG, &host); - if (ret) { - error_report("IGD device %s does not support host bridge access," - "legacy mode disabled", vdev->vbasedev.name); - goto out; - } - - ret = vfio_get_dev_region_info(&vdev->vbasedev, - VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, - VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG, &lpc); - if (ret) { - error_report("IGD device %s does not support LPC bridge access," - "legacy mode disabled", vdev->vbasedev.name); - goto out; - } - - gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, 4); - - /* - * If IGD VGA Disable is clear (expected) and VGA is not already enabled, - * try to enable it. Probably shouldn't be using legacy mode without VGA, - * but also no point in us enabling VGA if disabled in hardware. - */ - if (!(gmch & 0x2) && !vdev->vga && vfio_populate_vga(vdev, &err)) { - error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); - error_report("IGD device %s failed to enable VGA access, " - "legacy mode disabled", vdev->vbasedev.name); - goto out; - } - - /* Create our LPC/ISA bridge */ - ret = vfio_pci_igd_lpc_init(vdev, lpc); - if (ret) { - error_report("IGD device %s failed to create LPC bridge, " - "legacy mode disabled", vdev->vbasedev.name); - goto out; - } - - /* Stuff some host values into the VM PCI host bridge */ - ret = vfio_pci_igd_host_init(vdev, host); - if (ret) { - error_report("IGD device %s failed to modify host bridge, " - "legacy mode disabled", vdev->vbasedev.name); - goto out; - } - - /* Setup OpRegion access */ - ret = vfio_pci_igd_opregion_init(vdev, opregion, &err); - if (ret) { - error_append_hint(&err, "IGD legacy mode disabled\n"); - error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); - goto out; - } - - /* Setup our quirk to munge GTT addresses to the VM allocated buffer */ - quirk = vfio_quirk_alloc(2); - igd = quirk->data = g_malloc0(sizeof(*igd)); - igd->vdev = vdev; - igd->index = ~0; - igd->bdsm = vfio_pci_read_config(&vdev->pdev, IGD_BDSM, 4); - igd->bdsm &= ~((1 * MiB) - 1); /* 1MB aligned */ - - memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_igd_index_quirk, - igd, "vfio-igd-index-quirk", 4); - memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, - 0, &quirk->mem[0], 1); - - memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_igd_data_quirk, - igd, "vfio-igd-data-quirk", 4); - memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, - 4, &quirk->mem[1], 1); - - QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); - - /* Determine the size of stolen memory needed for GTT */ - ggms_mb = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; - if (gen > 6) { - ggms_mb = 1 << ggms_mb; - } - - /* - * Assume we have no GMS memory, but allow it to be overrided by device - * option (experimental). The spec doesn't actually allow zero GMS when - * when IVD (IGD VGA Disable) is clear, but the claim is that it's unused, - * so let's not waste VM memory for it. - */ - gmch &= ~((gen < 8 ? 0x1f : 0xff) << (gen < 8 ? 3 : 8)); - - if (vdev->igd_gms) { - if (vdev->igd_gms <= 0x10) { - gms_mb = vdev->igd_gms * 32; - gmch |= vdev->igd_gms << (gen < 8 ? 3 : 8); - } else { - error_report("Unsupported IGD GMS value 0x%x", vdev->igd_gms); - vdev->igd_gms = 0; - } - } - - /* - * Request reserved memory for stolen memory via fw_cfg. VM firmware - * must allocate a 1MB aligned reserved memory region below 4GB with - * the requested size (in bytes) for use by the Intel PCI class VGA - * device at VM address 00:02.0. The base address of this reserved - * memory region must be written to the device BDSM regsiter at PCI - * config offset 0x5C. - */ - bdsm_size = g_malloc(sizeof(*bdsm_size)); - *bdsm_size = cpu_to_le64((ggms_mb + gms_mb) * MiB); - fw_cfg_add_file(fw_cfg_find(), "etc/igd-bdsm-size", - bdsm_size, sizeof(*bdsm_size)); - - /* GMCH is read-only, emulated */ - pci_set_long(vdev->pdev.config + IGD_GMCH, gmch); - pci_set_long(vdev->pdev.wmask + IGD_GMCH, 0); - pci_set_long(vdev->emulated_config_bits + IGD_GMCH, ~0); - - /* BDSM is read-write, emulated. The BIOS needs to be able to write it */ - pci_set_long(vdev->pdev.config + IGD_BDSM, 0); - pci_set_long(vdev->pdev.wmask + IGD_BDSM, ~0); - pci_set_long(vdev->emulated_config_bits + IGD_BDSM, ~0); - - /* - * This IOBAR gives us access to GTTADR, which allows us to write to - * the GTT itself. So let's go ahead and write zero to all the GTT - * entries to avoid spurious DMA faults. Be sure I/O access is enabled - * before talking to the device. - */ - if (pread(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), - vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { - error_report("IGD device %s - failed to read PCI command register", - vdev->vbasedev.name); - } - - cmd = cmd_orig | PCI_COMMAND_IO; - - if (pwrite(vdev->vbasedev.fd, &cmd, sizeof(cmd), - vdev->config_offset + PCI_COMMAND) != sizeof(cmd)) { - error_report("IGD device %s - failed to write PCI command register", - vdev->vbasedev.name); - } - - for (i = 1; i < vfio_igd_gtt_max(vdev); i += 4) { - vfio_region_write(&vdev->bars[4].region, 0, i, 4); - vfio_region_write(&vdev->bars[4].region, 4, 0, 4); - } - - if (pwrite(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), - vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { - error_report("IGD device %s - failed to restore PCI command register", - vdev->vbasedev.name); - } - - trace_vfio_pci_igd_bdsm_enabled(vdev->vbasedev.name, ggms_mb + gms_mb); - -out: - g_free(rom); - g_free(opregion); - g_free(host); - g_free(lpc); -} - -/* * Common quirk probe entry points. */ void vfio_vga_quirk_setup(VFIOPCIDevice *vdev) @@ -1860,7 +1250,9 @@ void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr) vfio_probe_nvidia_bar5_quirk(vdev, nr); vfio_probe_nvidia_bar0_quirk(vdev, nr); vfio_probe_rtl8168_bar2_quirk(vdev, nr); +#ifdef CONFIG_VFIO_IGD vfio_probe_igd_bar4_quirk(vdev, nr); +#endif } void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr) diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index 35626cd63e..0da7a20a7e 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -172,6 +172,21 @@ typedef struct VFIOPCIDevice { Notifier irqchip_change_notifier; } VFIOPCIDevice; +/* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */ +static inline bool vfio_pci_is(VFIOPCIDevice *vdev, uint32_t vendor, uint32_t device) +{ + return (vendor == PCI_ANY_ID || vendor == vdev->vendor_id) && + (device == PCI_ANY_ID || device == vdev->device_id); +} + +static inline bool vfio_is_vga(VFIOPCIDevice *vdev) +{ + PCIDevice *pdev = &vdev->pdev; + uint16_t class = pci_get_word(pdev->config + PCI_CLASS_DEVICE); + + return class == PCI_CLASS_DISPLAY_VGA; +} + uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len); void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr, uint32_t val, int len); @@ -189,6 +204,8 @@ void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr); void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev); int vfio_add_virt_caps(VFIOPCIDevice *vdev, Error **errp); void vfio_quirk_reset(VFIOPCIDevice *vdev); +VFIOQuirk *vfio_quirk_alloc(int nr_mem); +void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr); extern const PropertyInfo qdev_prop_nv_gpudirect_clique; diff --git a/io/channel-websock.c b/io/channel-websock.c index fc36d44eba..47a0e941d9 100644 --- a/io/channel-websock.c +++ b/io/channel-websock.c @@ -49,13 +49,20 @@ "Server: QEMU VNC\r\n" \ "Date: %s\r\n" +#define QIO_CHANNEL_WEBSOCK_HANDSHAKE_WITH_PROTO_RES_OK \ + "HTTP/1.1 101 Switching Protocols\r\n" \ + QIO_CHANNEL_WEBSOCK_HANDSHAKE_RES_COMMON \ + "Upgrade: websocket\r\n" \ + "Connection: Upgrade\r\n" \ + "Sec-WebSocket-Accept: %s\r\n" \ + "Sec-WebSocket-Protocol: binary\r\n" \ + "\r\n" #define QIO_CHANNEL_WEBSOCK_HANDSHAKE_RES_OK \ "HTTP/1.1 101 Switching Protocols\r\n" \ QIO_CHANNEL_WEBSOCK_HANDSHAKE_RES_COMMON \ "Upgrade: websocket\r\n" \ "Connection: Upgrade\r\n" \ "Sec-WebSocket-Accept: %s\r\n" \ - "Sec-WebSocket-Protocol: binary\r\n" \ "\r\n" #define QIO_CHANNEL_WEBSOCK_HANDSHAKE_RES_NOT_FOUND \ "HTTP/1.1 404 Not Found\r\n" \ @@ -336,6 +343,7 @@ qio_channel_websock_find_header(QIOChannelWebsockHTTPHeader *hdrs, static void qio_channel_websock_handshake_send_res_ok(QIOChannelWebsock *ioc, const char *key, + const bool use_protocols, Error **errp) { char combined_key[QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN + @@ -361,8 +369,14 @@ static void qio_channel_websock_handshake_send_res_ok(QIOChannelWebsock *ioc, } date = qio_channel_websock_date_str(); - qio_channel_websock_handshake_send_res( - ioc, QIO_CHANNEL_WEBSOCK_HANDSHAKE_RES_OK, date, accept); + if (use_protocols) { + qio_channel_websock_handshake_send_res( + ioc, QIO_CHANNEL_WEBSOCK_HANDSHAKE_WITH_PROTO_RES_OK, + date, accept); + } else { + qio_channel_websock_handshake_send_res( + ioc, QIO_CHANNEL_WEBSOCK_HANDSHAKE_RES_OK, date, accept); + } g_free(date); g_free(accept); @@ -387,10 +401,6 @@ static void qio_channel_websock_handshake_process(QIOChannelWebsock *ioc, protocols = qio_channel_websock_find_header( hdrs, nhdrs, QIO_CHANNEL_WEBSOCK_HEADER_PROTOCOL); - if (!protocols) { - error_setg(errp, "Missing websocket protocol header data"); - goto bad_request; - } version = qio_channel_websock_find_header( hdrs, nhdrs, QIO_CHANNEL_WEBSOCK_HEADER_VERSION); @@ -430,10 +440,12 @@ static void qio_channel_websock_handshake_process(QIOChannelWebsock *ioc, trace_qio_channel_websock_http_request(ioc, protocols, version, host, connection, upgrade, key); - if (!g_strrstr(protocols, QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY)) { - error_setg(errp, "No '%s' protocol is supported by client '%s'", - QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY, protocols); - goto bad_request; + if (protocols) { + if (!g_strrstr(protocols, QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY)) { + error_setg(errp, "No '%s' protocol is supported by client '%s'", + QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY, protocols); + goto bad_request; + } } if (!g_str_equal(version, QIO_CHANNEL_WEBSOCK_SUPPORTED_VERSION)) { @@ -467,7 +479,7 @@ static void qio_channel_websock_handshake_process(QIOChannelWebsock *ioc, goto bad_request; } - qio_channel_websock_handshake_send_res_ok(ioc, key, errp); + qio_channel_websock_handshake_send_res_ok(ioc, key, !!protocols, errp); return; bad_request: diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index d0e0af893a..558fe06b8f 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -2351,7 +2351,7 @@ void hmp_nbd_server_start(Monitor *mon, const QDict *qdict) continue; } - qmp_nbd_server_add(info->value->device, false, NULL, + qmp_nbd_server_add(info->value->device, false, NULL, false, NULL, true, writable, false, NULL, &local_err); if (local_err != NULL) { @@ -2373,7 +2373,7 @@ void hmp_nbd_server_add(Monitor *mon, const QDict *qdict) bool writable = qdict_get_try_bool(qdict, "writable", false); Error *local_err = NULL; - qmp_nbd_server_add(device, !!name, name, true, writable, + qmp_nbd_server_add(device, !!name, name, false, NULL, true, writable, false, NULL, &local_err); hmp_handle_error(mon, local_err); } diff --git a/qapi/audio.json b/qapi/audio.json index 83312b2339..d8c507cced 100644 --- a/qapi/audio.json +++ b/qapi/audio.json @@ -276,7 +276,7 @@ # Since: 4.0 ## { 'enum': 'AudioFormat', - 'data': [ 'u8', 's8', 'u16', 's16', 'u32', 's32' ] } + 'data': [ 'u8', 's8', 'u16', 's16', 'u32', 's32', 'f32' ] } ## # @AudiodevDriver: diff --git a/qapi/block.json b/qapi/block.json index 145c268bb6..7898104dae 100644 --- a/qapi/block.json +++ b/qapi/block.json @@ -250,9 +250,12 @@ # @name: Export name. If unspecified, the @device parameter is used as the # export name. (Since 2.12) # +# @description: Free-form description of the export, up to 4096 bytes. +# (Since 5.0) +# # @writable: Whether clients should be able to write to the device via the # NBD connection (default false). - +# # @bitmap: Also export the dirty bitmap reachable from @device, so the # NBD client can use NBD_OPT_SET_META_CONTEXT with # "qemu:dirty-bitmap:NAME" to inspect the bitmap. (since 4.0) @@ -263,8 +266,8 @@ # Since: 1.3.0 ## { 'command': 'nbd-server-add', - 'data': {'device': 'str', '*name': 'str', '*writable': 'bool', - '*bitmap': 'str' } } + 'data': {'device': 'str', '*name': 'str', '*description': 'str', + '*writable': 'bool', '*bitmap': 'str' } } ## # @NbdServerRemoveMode: diff --git a/qemu-deprecated.texi b/qemu-deprecated.texi index 97668edf92..017b750ca8 100644 --- a/qemu-deprecated.texi +++ b/qemu-deprecated.texi @@ -313,37 +313,6 @@ The above, converted to the current supported format: @section Related binaries -@subsection qemu-nbd --partition (since 4.0.0) - -The ``qemu-nbd --partition $digit'' code (also spelled @option{-P}) -can only handle MBR partitions, and has never correctly handled -logical partitions beyond partition 5. If you know the offset and -length of the partition (perhaps by using @code{sfdisk} within the -guest), you can achieve the effect of exporting just that subset of -the disk by use of the @option{--image-opts} option with a raw -blockdev using the @code{offset} and @code{size} parameters layered on -top of any other existing blockdev. For example, if partition 1 is -100MiB long starting at 1MiB, the old command: - -@code{qemu-nbd -t -P 1 -f qcow2 file.qcow2} - -can be rewritten as: - -@code{qemu-nbd -t --image-opts driver=raw,offset=1M,size=100M,file.driver=qcow2,file.backing.driver=file,file.backing.filename=file.qcow2} - -Alternatively, the @code{nbdkit} project provides a more powerful -partition filter on top of its nbd plugin, which can be used to select -an arbitrary MBR or GPT partition on top of any other full-image NBD -export. Using this to rewrite the above example results in: - -@code{qemu-nbd -t -k /tmp/sock -f qcow2 file.qcow2 &} -@code{nbdkit -f --filter=partition nbd socket=/tmp/sock partition=1} - -Note that if you are exposing the export via /dev/nbd0, it is easier -to just export the entire image and then mount only /dev/nbd0p1 than -it is to reinvoke @command{qemu-nbd -c /dev/nbd0} limited to just a -subset of the image. - @subsection qemu-img convert -n -o (since 4.2.0) All options specified in @option{-o} are image creation options, so @@ -392,3 +361,21 @@ trouble after a recent upgrade. The "autoload" parameter has been ignored since 2.12.0. All bitmaps are automatically loaded from qcow2 images. + +@section Related binaries + +@subsection qemu-nbd --partition (removed in 5.0.0) + +The ``qemu-nbd --partition $digit'' code (also spelled @option{-P}) +could only handle MBR partitions, and never correctly handled logical +partitions beyond partition 5. Exporting a partition can still be +done by utilizing the @option{--image-opts} option with a raw blockdev +using the @code{offset} and @code{size} parameters layered on top of +any other existing blockdev. For example, if partition 1 is 100MiB +long starting at 1MiB, the old command: + +@code{qemu-nbd -t -P 1 -f qcow2 file.qcow2} + +can be rewritten as: + +@code{qemu-nbd -t --image-opts driver=raw,offset=1M,size=100M,file.driver=qcow2,file.file.driver=file,file.file.filename=file.qcow2} diff --git a/qemu-nbd.c b/qemu-nbd.c index db29a0d0ed..4aa005004e 100644 --- a/qemu-nbd.c +++ b/qemu-nbd.c @@ -100,7 +100,6 @@ static void usage(const char *name) "\n" "Exposing part of the image:\n" " -o, --offset=OFFSET offset into the image\n" -" -P, --partition=NUM only expose partition NUM\n" " -B, --bitmap=NAME expose a persistent dirty bitmap\n" "\n" "General purpose options:\n" @@ -156,96 +155,6 @@ QEMU_COPYRIGHT "\n" , name); } -struct partition_record -{ - uint8_t bootable; - uint8_t start_head; - uint32_t start_cylinder; - uint8_t start_sector; - uint8_t system; - uint8_t end_head; - uint8_t end_cylinder; - uint8_t end_sector; - uint32_t start_sector_abs; - uint32_t nb_sectors_abs; -}; - -static void read_partition(uint8_t *p, struct partition_record *r) -{ - r->bootable = p[0]; - r->start_head = p[1]; - r->start_cylinder = p[3] | ((p[2] << 2) & 0x0300); - r->start_sector = p[2] & 0x3f; - r->system = p[4]; - r->end_head = p[5]; - r->end_cylinder = p[7] | ((p[6] << 2) & 0x300); - r->end_sector = p[6] & 0x3f; - - r->start_sector_abs = ldl_le_p(p + 8); - r->nb_sectors_abs = ldl_le_p(p + 12); -} - -static int find_partition(BlockBackend *blk, int partition, - uint64_t *offset, uint64_t *size) -{ - struct partition_record mbr[4]; - uint8_t data[MBR_SIZE]; - int i; - int ext_partnum = 4; - int ret; - - ret = blk_pread(blk, 0, data, sizeof(data)); - if (ret < 0) { - error_report("error while reading: %s", strerror(-ret)); - exit(EXIT_FAILURE); - } - - if (data[510] != 0x55 || data[511] != 0xaa) { - return -EINVAL; - } - - for (i = 0; i < 4; i++) { - read_partition(&data[446 + 16 * i], &mbr[i]); - - if (!mbr[i].system || !mbr[i].nb_sectors_abs) { - continue; - } - - if (mbr[i].system == 0xF || mbr[i].system == 0x5) { - struct partition_record ext[4]; - uint8_t data1[MBR_SIZE]; - int j; - - ret = blk_pread(blk, mbr[i].start_sector_abs * MBR_SIZE, - data1, sizeof(data1)); - if (ret < 0) { - error_report("error while reading: %s", strerror(-ret)); - exit(EXIT_FAILURE); - } - - for (j = 0; j < 4; j++) { - read_partition(&data1[446 + 16 * j], &ext[j]); - if (!ext[j].system || !ext[j].nb_sectors_abs) { - continue; - } - - if ((ext_partnum + j + 1) == partition) { - *offset = (uint64_t)ext[j].start_sector_abs << 9; - *size = (uint64_t)ext[j].nb_sectors_abs << 9; - return 0; - } - } - ext_partnum += 4; - } else if ((i + 1) == partition) { - *offset = (uint64_t)mbr[i].start_sector_abs << 9; - *size = (uint64_t)mbr[i].nb_sectors_abs << 9; - return 0; - } - } - - return -ENOENT; -} - static void termsig_handler(int signum) { atomic_cmpxchg(&state, RUNNING, TERMINATE); @@ -617,7 +526,7 @@ int main(int argc, char **argv) int64_t fd_size; QemuOpts *sn_opts = NULL; const char *sn_id_or_name = NULL; - const char *sopt = "hVb:o:p:rsnP:c:dvk:e:f:tl:x:T:D:B:L"; + const char *sopt = "hVb:o:p:rsnc:dvk:e:f:tl:x:T:D:B:L"; struct option lopt[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, 'V' }, @@ -626,7 +535,6 @@ int main(int argc, char **argv) { "socket", required_argument, NULL, 'k' }, { "offset", required_argument, NULL, 'o' }, { "read-only", no_argument, NULL, 'r' }, - { "partition", required_argument, NULL, 'P' }, { "bitmap", required_argument, NULL, 'B' }, { "connect", required_argument, NULL, 'c' }, { "disconnect", no_argument, NULL, 'd' }, @@ -657,7 +565,6 @@ int main(int argc, char **argv) int ch; int opt_ind = 0; int flags = BDRV_O_RDWR; - int partition = 0; int ret = 0; bool seen_cache = false; bool seen_discard = false; @@ -789,15 +696,6 @@ int main(int argc, char **argv) readonly = true; flags &= ~BDRV_O_RDWR; break; - case 'P': - warn_report("The '-P' option is deprecated; use --image-opts with " - "a raw device wrapper for subset exports instead"); - if (qemu_strtoi(optarg, NULL, 0, &partition) < 0 || - partition < 1 || partition > 8) { - error_report("Invalid partition '%s'", optarg); - exit(EXIT_FAILURE); - } - break; case 'B': bitmap = optarg; break; @@ -894,7 +792,7 @@ int main(int argc, char **argv) error_report("List mode is incompatible with a file name"); exit(EXIT_FAILURE); } - if (export_name || export_description || dev_offset || partition || + if (export_name || export_description || dev_offset || device || disconnect || fmt || sn_id_or_name || bitmap || seen_aio || seen_discard || seen_cache) { error_report("List mode is incompatible with per-device settings"); @@ -1158,33 +1056,6 @@ int main(int argc, char **argv) } fd_size -= dev_offset; - if (partition) { - uint64_t limit; - - if (dev_offset) { - error_report("Cannot request partition and offset together"); - exit(EXIT_FAILURE); - } - ret = find_partition(blk, partition, &dev_offset, &limit); - if (ret < 0) { - error_report("Could not find partition %d: %s", partition, - strerror(-ret)); - exit(EXIT_FAILURE); - } - /* - * MBR partition limits are (32-bit << 9); this assert lets - * the compiler know that we can't overflow 64 bits. - */ - assert(dev_offset + limit >= dev_offset); - if (dev_offset + limit > fd_size) { - error_report("Discovered partition %d at offset %" PRIu64 - " size %" PRIu64 ", but size exceeds file length %" - PRId64, partition, dev_offset, limit, fd_size); - exit(EXIT_FAILURE); - } - fd_size = limit; - } - export = nbd_export_new(bs, dev_offset, fd_size, export_name, export_description, bitmap, readonly, shared > 1, nbd_export_closed, writethrough, NULL, diff --git a/qemu-options.hx b/qemu-options.hx index ff3e806977..ac315c1ac4 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -2428,8 +2428,7 @@ Use @option{model=help} to list the available device types. The hardware MAC address can be set with @option{mac=@var{macaddr}}. The following two example do exactly the same, to show how @option{-nic} can -be used to shorten the command line length (note that the e1000 is the default -on i386, so the @option{model=e1000} parameter could even be omitted here, too): +be used to shorten the command line length: @example @value{qemu_system} -netdev user,id=n1,ipv6=off -device e1000,netdev=n1,mac=52:54:98:76:54:32 @value{qemu_system} -nic user,ipv6=off,model=e1000,mac=52:54:98:76:54:32 @@ -2843,9 +2842,12 @@ netdev with ID @var{nd} by using the @option{netdev=@var{nd}} option. Legacy option to configure or create an on-board (or machine default) Network Interface Card(NIC) and connect it either to the emulated hub with ID 0 (i.e. the default hub), or to the netdev @var{nd}. -The NIC is an e1000 by default on the PC target. Optionally, the MAC address -can be changed to @var{mac}, the device address set to @var{addr} (PCI cards -only), and a @var{name} can be assigned for use in monitor commands. +If @var{model} is omitted, then the default NIC model associated with +the machine type is used. Note that the default NIC model may change in +future QEMU releases, so it is highly recommended to always specify a model. +Optionally, the MAC address can be changed to @var{mac}, the device +address set to @var{addr} (PCI cards only), and a @var{name} can be +assigned for use in monitor commands. Optionally, for PCI cards, you can specify the number @var{v} of MSI-X vectors that the card should have; this option currently only affects virtio cards; set @var{v} = 0 to disable MSI-X. If no @option{-net} option is specified, a single diff --git a/target/arm/cpu-param.h b/target/arm/cpu-param.h index 6e6948e960..18ac562346 100644 --- a/target/arm/cpu-param.h +++ b/target/arm/cpu-param.h @@ -29,6 +29,6 @@ # define TARGET_PAGE_BITS_MIN 10 #endif -#define NB_MMU_MODES 8 +#define NB_MMU_MODES 9 #endif diff --git a/target/arm/cpu-qom.h b/target/arm/cpu-qom.h index 7f5b244bde..3a9d31ea9d 100644 --- a/target/arm/cpu-qom.h +++ b/target/arm/cpu-qom.h @@ -76,6 +76,7 @@ void arm_gt_ptimer_cb(void *opaque); void arm_gt_vtimer_cb(void *opaque); void arm_gt_htimer_cb(void *opaque); void arm_gt_stimer_cb(void *opaque); +void arm_gt_hvtimer_cb(void *opaque); #define ARM_AFF0_SHIFT 0 #define ARM_AFF0_MASK (0xFFULL << ARM_AFF0_SHIFT) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index f86e71a260..b0762a76c4 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -410,58 +410,173 @@ static void arm_cpu_reset(CPUState *s) arm_rebuild_hflags(env); } +static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, + unsigned int target_el, + unsigned int cur_el, bool secure, + uint64_t hcr_el2) +{ + CPUARMState *env = cs->env_ptr; + bool pstate_unmasked; + bool unmasked = false; + + /* + * Don't take exceptions if they target a lower EL. + * This check should catch any exceptions that would not be taken + * but left pending. + */ + if (cur_el > target_el) { + return false; + } + + switch (excp_idx) { + case EXCP_FIQ: + pstate_unmasked = !(env->daif & PSTATE_F); + break; + + case EXCP_IRQ: + pstate_unmasked = !(env->daif & PSTATE_I); + break; + + case EXCP_VFIQ: + if (secure || !(hcr_el2 & HCR_FMO) || (hcr_el2 & HCR_TGE)) { + /* VFIQs are only taken when hypervized and non-secure. */ + return false; + } + return !(env->daif & PSTATE_F); + case EXCP_VIRQ: + if (secure || !(hcr_el2 & HCR_IMO) || (hcr_el2 & HCR_TGE)) { + /* VIRQs are only taken when hypervized and non-secure. */ + return false; + } + return !(env->daif & PSTATE_I); + default: + g_assert_not_reached(); + } + + /* + * Use the target EL, current execution state and SCR/HCR settings to + * determine whether the corresponding CPSR bit is used to mask the + * interrupt. + */ + if ((target_el > cur_el) && (target_el != 1)) { + /* Exceptions targeting a higher EL may not be maskable */ + if (arm_feature(env, ARM_FEATURE_AARCH64)) { + /* + * 64-bit masking rules are simple: exceptions to EL3 + * can't be masked, and exceptions to EL2 can only be + * masked from Secure state. The HCR and SCR settings + * don't affect the masking logic, only the interrupt routing. + */ + if (target_el == 3 || !secure) { + unmasked = true; + } + } else { + /* + * The old 32-bit-only environment has a more complicated + * masking setup. HCR and SCR bits not only affect interrupt + * routing but also change the behaviour of masking. + */ + bool hcr, scr; + + switch (excp_idx) { + case EXCP_FIQ: + /* + * If FIQs are routed to EL3 or EL2 then there are cases where + * we override the CPSR.F in determining if the exception is + * masked or not. If neither of these are set then we fall back + * to the CPSR.F setting otherwise we further assess the state + * below. + */ + hcr = hcr_el2 & HCR_FMO; + scr = (env->cp15.scr_el3 & SCR_FIQ); + + /* + * When EL3 is 32-bit, the SCR.FW bit controls whether the + * CPSR.F bit masks FIQ interrupts when taken in non-secure + * state. If SCR.FW is set then FIQs can be masked by CPSR.F + * when non-secure but only when FIQs are only routed to EL3. + */ + scr = scr && !((env->cp15.scr_el3 & SCR_FW) && !hcr); + break; + case EXCP_IRQ: + /* + * When EL3 execution state is 32-bit, if HCR.IMO is set then + * we may override the CPSR.I masking when in non-secure state. + * The SCR.IRQ setting has already been taken into consideration + * when setting the target EL, so it does not have a further + * affect here. + */ + hcr = hcr_el2 & HCR_IMO; + scr = false; + break; + default: + g_assert_not_reached(); + } + + if ((scr || hcr) && !secure) { + unmasked = true; + } + } + } + + /* + * The PSTATE bits only mask the interrupt if we have not overriden the + * ability above. + */ + return unmasked || pstate_unmasked; +} + bool arm_cpu_exec_interrupt(CPUState *cs, int interrupt_request) { CPUClass *cc = CPU_GET_CLASS(cs); CPUARMState *env = cs->env_ptr; uint32_t cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr_el2 = arm_hcr_el2_eff(env); uint32_t target_el; uint32_t excp_idx; - bool ret = false; + + /* The prioritization of interrupts is IMPLEMENTATION DEFINED. */ if (interrupt_request & CPU_INTERRUPT_FIQ) { excp_idx = EXCP_FIQ; target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure); - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } if (interrupt_request & CPU_INTERRUPT_HARD) { excp_idx = EXCP_IRQ; target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure); - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } if (interrupt_request & CPU_INTERRUPT_VIRQ) { excp_idx = EXCP_VIRQ; target_el = 1; - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } if (interrupt_request & CPU_INTERRUPT_VFIQ) { excp_idx = EXCP_VFIQ; target_el = 1; - if (arm_excp_unmasked(cs, excp_idx, target_el)) { - cs->exception_index = excp_idx; - env->exception.target_el = target_el; - cc->do_interrupt(cs); - ret = true; + if (arm_excp_unmasked(cs, excp_idx, target_el, + cur_el, secure, hcr_el2)) { + goto found; } } + return false; - return ret; + found: + cs->exception_index = excp_idx; + env->exception.target_el = target_el; + cc->do_interrupt(cs); + return true; } #if !defined(CONFIG_USER_ONLY) || !defined(TARGET_AARCH64) @@ -1272,7 +1387,6 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) } } - { uint64_t scale; @@ -1295,6 +1409,8 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) arm_gt_htimer_cb, cpu); cpu->gt_timer[GTIMER_SEC] = timer_new(QEMU_CLOCK_VIRTUAL, scale, arm_gt_stimer_cb, cpu); + cpu->gt_timer[GTIMER_HYPVIRT] = timer_new(QEMU_CLOCK_VIRTUAL, scale, + arm_gt_hvtimer_cb, cpu); } #endif diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 608fcbd0b7..0b3036c484 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -144,11 +144,12 @@ typedef struct ARMGenericTimer { uint64_t ctl; /* Timer Control register */ } ARMGenericTimer; -#define GTIMER_PHYS 0 -#define GTIMER_VIRT 1 -#define GTIMER_HYP 2 -#define GTIMER_SEC 3 -#define NUM_GTIMERS 4 +#define GTIMER_PHYS 0 +#define GTIMER_VIRT 1 +#define GTIMER_HYP 2 +#define GTIMER_SEC 3 +#define GTIMER_HYPVIRT 4 +#define NUM_GTIMERS 5 typedef struct { uint64_t raw_tcr; @@ -1424,13 +1425,6 @@ static inline void xpsr_write(CPUARMState *env, uint32_t val, uint32_t mask) #define HCR_ATA (1ULL << 56) #define HCR_DCT (1ULL << 57) -/* - * When we actually implement ARMv8.1-VHE we should add HCR_E2H to - * HCR_MASK and then clear it again if the feature bit is not set in - * hcr_write(). - */ -#define HCR_MASK ((1ULL << 34) - 1) - #define SCR_NS (1U << 0) #define SCR_IRQ (1U << 1) #define SCR_FIQ (1U << 2) @@ -2582,6 +2576,19 @@ struct ARMCPRegInfo { * fieldoffset is 0 then no reset will be done. */ CPResetFn *resetfn; + + /* + * "Original" writefn and readfn. + * For ARMv8.1-VHE register aliases, we overwrite the read/write + * accessor functions of various EL1/EL0 to perform the runtime + * check for which sysreg should actually be modified, and then + * forwards the operation. Before overwriting the accessors, + * the original function is copied here, so that accesses that + * really do go to the EL1/EL0 version proceed normally. + * (The corresponding EL2 register is linked via opaque.) + */ + CPReadFn *orig_readfn; + CPWriteFn *orig_writefn; }; /* Macros which are lvalues for the field in CPUARMState for the @@ -2702,117 +2709,6 @@ bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync); #define ARM_CPUID_TI915T 0x54029152 #define ARM_CPUID_TI925T 0x54029252 -static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, - unsigned int target_el) -{ - CPUARMState *env = cs->env_ptr; - unsigned int cur_el = arm_current_el(env); - bool secure = arm_is_secure(env); - bool pstate_unmasked; - int8_t unmasked = 0; - uint64_t hcr_el2; - - /* Don't take exceptions if they target a lower EL. - * This check should catch any exceptions that would not be taken but left - * pending. - */ - if (cur_el > target_el) { - return false; - } - - hcr_el2 = arm_hcr_el2_eff(env); - - switch (excp_idx) { - case EXCP_FIQ: - pstate_unmasked = !(env->daif & PSTATE_F); - break; - - case EXCP_IRQ: - pstate_unmasked = !(env->daif & PSTATE_I); - break; - - case EXCP_VFIQ: - if (secure || !(hcr_el2 & HCR_FMO) || (hcr_el2 & HCR_TGE)) { - /* VFIQs are only taken when hypervized and non-secure. */ - return false; - } - return !(env->daif & PSTATE_F); - case EXCP_VIRQ: - if (secure || !(hcr_el2 & HCR_IMO) || (hcr_el2 & HCR_TGE)) { - /* VIRQs are only taken when hypervized and non-secure. */ - return false; - } - return !(env->daif & PSTATE_I); - default: - g_assert_not_reached(); - } - - /* Use the target EL, current execution state and SCR/HCR settings to - * determine whether the corresponding CPSR bit is used to mask the - * interrupt. - */ - if ((target_el > cur_el) && (target_el != 1)) { - /* Exceptions targeting a higher EL may not be maskable */ - if (arm_feature(env, ARM_FEATURE_AARCH64)) { - /* 64-bit masking rules are simple: exceptions to EL3 - * can't be masked, and exceptions to EL2 can only be - * masked from Secure state. The HCR and SCR settings - * don't affect the masking logic, only the interrupt routing. - */ - if (target_el == 3 || !secure) { - unmasked = 1; - } - } else { - /* The old 32-bit-only environment has a more complicated - * masking setup. HCR and SCR bits not only affect interrupt - * routing but also change the behaviour of masking. - */ - bool hcr, scr; - - switch (excp_idx) { - case EXCP_FIQ: - /* If FIQs are routed to EL3 or EL2 then there are cases where - * we override the CPSR.F in determining if the exception is - * masked or not. If neither of these are set then we fall back - * to the CPSR.F setting otherwise we further assess the state - * below. - */ - hcr = hcr_el2 & HCR_FMO; - scr = (env->cp15.scr_el3 & SCR_FIQ); - - /* When EL3 is 32-bit, the SCR.FW bit controls whether the - * CPSR.F bit masks FIQ interrupts when taken in non-secure - * state. If SCR.FW is set then FIQs can be masked by CPSR.F - * when non-secure but only when FIQs are only routed to EL3. - */ - scr = scr && !((env->cp15.scr_el3 & SCR_FW) && !hcr); - break; - case EXCP_IRQ: - /* When EL3 execution state is 32-bit, if HCR.IMO is set then - * we may override the CPSR.I masking when in non-secure state. - * The SCR.IRQ setting has already been taken into consideration - * when setting the target EL, so it does not have a further - * affect here. - */ - hcr = hcr_el2 & HCR_IMO; - scr = false; - break; - default: - g_assert_not_reached(); - } - - if ((scr || hcr) && !secure) { - unmasked = 1; - } - } - } - - /* The PSTATE bits only mask the interrupt if we have not overriden the - * ability above. - */ - return unmasked || pstate_unmasked; -} - #define ARM_CPU_TYPE_SUFFIX "-" TYPE_ARM_CPU #define ARM_CPU_TYPE_NAME(name) (name ARM_CPU_TYPE_SUFFIX) #define CPU_RESOLVING_TYPE TYPE_ARM_CPU @@ -2826,18 +2722,21 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, * + NonSecure EL1 & 0 stage 1 * + NonSecure EL1 & 0 stage 2 * + NonSecure EL2 - * + Secure EL1 & EL0 + * + NonSecure EL2 & 0 (ARMv8.1-VHE) + * + Secure EL1 & 0 * + Secure EL3 * If EL3 is 32-bit: * + NonSecure PL1 & 0 stage 1 * + NonSecure PL1 & 0 stage 2 * + NonSecure PL2 - * + Secure PL0 & PL1 + * + Secure PL0 + * + Secure PL1 * (reminder: for 32 bit EL3, Secure PL1 is *EL3*, not EL1.) * * For QEMU, an mmu_idx is not quite the same as a translation regime because: - * 1. we need to split the "EL1 & 0" regimes into two mmu_idxes, because they - * may differ in access permissions even if the VA->PA map is the same + * 1. we need to split the "EL1 & 0" and "EL2 & 0" regimes into two mmu_idxes, + * because they may differ in access permissions even if the VA->PA map is + * the same * 2. we want to cache in our TLB the full VA->IPA->PA lookup for a stage 1+2 * translation, which means that we have one mmu_idx that deals with two * concatenated translation regimes [this sort of combined s1+2 TLB is @@ -2849,19 +2748,23 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, * 4. we can also safely fold together the "32 bit EL3" and "64 bit EL3" * translation regimes, because they map reasonably well to each other * and they can't both be active at the same time. - * This gives us the following list of mmu_idx values: + * 5. we want to be able to use the TLB for accesses done as part of a + * stage1 page table walk, rather than having to walk the stage2 page + * table over and over. + * + * This gives us the following list of cases: * - * NS EL0 (aka NS PL0) stage 1+2 - * NS EL1 (aka NS PL1) stage 1+2 + * NS EL0 EL1&0 stage 1+2 (aka NS PL0) + * NS EL1 EL1&0 stage 1+2 (aka NS PL1) + * NS EL0 EL2&0 + * NS EL2 EL2&0 * NS EL2 (aka NS PL2) + * S EL0 EL1&0 (aka S PL0) + * S EL1 EL1&0 (not used if EL3 is 32 bit) * S EL3 (aka S PL1) - * S EL0 (aka S PL0) - * S EL1 (not used if EL3 is 32 bit) - * NS EL0+1 stage 2 + * NS EL1&0 stage 2 * - * (The last of these is an mmu_idx because we want to be able to use the TLB - * for the accesses done as part of a stage 1 page table walk, rather than - * having to walk the stage 2 page table over and over.) + * for a total of 9 different mmu_idx. * * R profile CPUs have an MPU, but can use the same set of MMU indexes * as A profile. They only need to distinguish NS EL0 and NS EL1 (and @@ -2899,106 +2802,88 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx, * For M profile we arrange them to have a bit for priv, a bit for negpri * and a bit for secure. */ -#define ARM_MMU_IDX_A 0x10 /* A profile */ -#define ARM_MMU_IDX_NOTLB 0x20 /* does not have a TLB */ -#define ARM_MMU_IDX_M 0x40 /* M profile */ +#define ARM_MMU_IDX_A 0x10 /* A profile */ +#define ARM_MMU_IDX_NOTLB 0x20 /* does not have a TLB */ +#define ARM_MMU_IDX_M 0x40 /* M profile */ -/* meanings of the bits for M profile mmu idx values */ -#define ARM_MMU_IDX_M_PRIV 0x1 +/* Meanings of the bits for M profile mmu idx values */ +#define ARM_MMU_IDX_M_PRIV 0x1 #define ARM_MMU_IDX_M_NEGPRI 0x2 -#define ARM_MMU_IDX_M_S 0x4 +#define ARM_MMU_IDX_M_S 0x4 /* Secure */ -#define ARM_MMU_IDX_TYPE_MASK (~0x7) -#define ARM_MMU_IDX_COREIDX_MASK 0x7 +#define ARM_MMU_IDX_TYPE_MASK \ + (ARM_MMU_IDX_A | ARM_MMU_IDX_M | ARM_MMU_IDX_NOTLB) +#define ARM_MMU_IDX_COREIDX_MASK 0xf typedef enum ARMMMUIdx { - ARMMMUIdx_S12NSE0 = 0 | ARM_MMU_IDX_A, - ARMMMUIdx_S12NSE1 = 1 | ARM_MMU_IDX_A, - ARMMMUIdx_S1E2 = 2 | ARM_MMU_IDX_A, - ARMMMUIdx_S1E3 = 3 | ARM_MMU_IDX_A, - ARMMMUIdx_S1SE0 = 4 | ARM_MMU_IDX_A, - ARMMMUIdx_S1SE1 = 5 | ARM_MMU_IDX_A, - ARMMMUIdx_S2NS = 6 | ARM_MMU_IDX_A, - ARMMMUIdx_MUser = 0 | ARM_MMU_IDX_M, - ARMMMUIdx_MPriv = 1 | ARM_MMU_IDX_M, - ARMMMUIdx_MUserNegPri = 2 | ARM_MMU_IDX_M, - ARMMMUIdx_MPrivNegPri = 3 | ARM_MMU_IDX_M, - ARMMMUIdx_MSUser = 4 | ARM_MMU_IDX_M, - ARMMMUIdx_MSPriv = 5 | ARM_MMU_IDX_M, - ARMMMUIdx_MSUserNegPri = 6 | ARM_MMU_IDX_M, - ARMMMUIdx_MSPrivNegPri = 7 | ARM_MMU_IDX_M, - /* Indexes below here don't have TLBs and are used only for AT system - * instructions or for the first stage of an S12 page table walk. + /* + * A-profile. */ - ARMMMUIdx_S1NSE0 = 0 | ARM_MMU_IDX_NOTLB, - ARMMMUIdx_S1NSE1 = 1 | ARM_MMU_IDX_NOTLB, -} ARMMMUIdx; + ARMMMUIdx_E10_0 = 0 | ARM_MMU_IDX_A, + ARMMMUIdx_E20_0 = 1 | ARM_MMU_IDX_A, -/* Bit macros for the core-mmu-index values for each index, - * for use when calling tlb_flush_by_mmuidx() and friends. - */ -typedef enum ARMMMUIdxBit { - ARMMMUIdxBit_S12NSE0 = 1 << 0, - ARMMMUIdxBit_S12NSE1 = 1 << 1, - ARMMMUIdxBit_S1E2 = 1 << 2, - ARMMMUIdxBit_S1E3 = 1 << 3, - ARMMMUIdxBit_S1SE0 = 1 << 4, - ARMMMUIdxBit_S1SE1 = 1 << 5, - ARMMMUIdxBit_S2NS = 1 << 6, - ARMMMUIdxBit_MUser = 1 << 0, - ARMMMUIdxBit_MPriv = 1 << 1, - ARMMMUIdxBit_MUserNegPri = 1 << 2, - ARMMMUIdxBit_MPrivNegPri = 1 << 3, - ARMMMUIdxBit_MSUser = 1 << 4, - ARMMMUIdxBit_MSPriv = 1 << 5, - ARMMMUIdxBit_MSUserNegPri = 1 << 6, - ARMMMUIdxBit_MSPrivNegPri = 1 << 7, -} ARMMMUIdxBit; + ARMMMUIdx_E10_1 = 2 | ARM_MMU_IDX_A, -#define MMU_USER_IDX 0 + ARMMMUIdx_E2 = 3 | ARM_MMU_IDX_A, + ARMMMUIdx_E20_2 = 4 | ARM_MMU_IDX_A, -static inline int arm_to_core_mmu_idx(ARMMMUIdx mmu_idx) -{ - return mmu_idx & ARM_MMU_IDX_COREIDX_MASK; -} + ARMMMUIdx_SE10_0 = 5 | ARM_MMU_IDX_A, + ARMMMUIdx_SE10_1 = 6 | ARM_MMU_IDX_A, + ARMMMUIdx_SE3 = 7 | ARM_MMU_IDX_A, -static inline ARMMMUIdx core_to_arm_mmu_idx(CPUARMState *env, int mmu_idx) -{ - if (arm_feature(env, ARM_FEATURE_M)) { - return mmu_idx | ARM_MMU_IDX_M; - } else { - return mmu_idx | ARM_MMU_IDX_A; - } -} + ARMMMUIdx_Stage2 = 8 | ARM_MMU_IDX_A, -/* Return the exception level we're running at if this is our mmu_idx */ -static inline int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx) -{ - switch (mmu_idx & ARM_MMU_IDX_TYPE_MASK) { - case ARM_MMU_IDX_A: - return mmu_idx & 3; - case ARM_MMU_IDX_M: - return mmu_idx & ARM_MMU_IDX_M_PRIV; - default: - g_assert_not_reached(); - } -} + /* + * These are not allocated TLBs and are used only for AT system + * instructions or for the first stage of an S12 page table walk. + */ + ARMMMUIdx_Stage1_E0 = 0 | ARM_MMU_IDX_NOTLB, + ARMMMUIdx_Stage1_E1 = 1 | ARM_MMU_IDX_NOTLB, + + /* + * M-profile. + */ + ARMMMUIdx_MUser = ARM_MMU_IDX_M, + ARMMMUIdx_MPriv = ARM_MMU_IDX_M | ARM_MMU_IDX_M_PRIV, + ARMMMUIdx_MUserNegPri = ARMMMUIdx_MUser | ARM_MMU_IDX_M_NEGPRI, + ARMMMUIdx_MPrivNegPri = ARMMMUIdx_MPriv | ARM_MMU_IDX_M_NEGPRI, + ARMMMUIdx_MSUser = ARMMMUIdx_MUser | ARM_MMU_IDX_M_S, + ARMMMUIdx_MSPriv = ARMMMUIdx_MPriv | ARM_MMU_IDX_M_S, + ARMMMUIdx_MSUserNegPri = ARMMMUIdx_MUserNegPri | ARM_MMU_IDX_M_S, + ARMMMUIdx_MSPrivNegPri = ARMMMUIdx_MPrivNegPri | ARM_MMU_IDX_M_S, +} ARMMMUIdx; /* - * Return the MMU index for a v7M CPU with all relevant information - * manually specified. + * Bit macros for the core-mmu-index values for each index, + * for use when calling tlb_flush_by_mmuidx() and friends. */ -ARMMMUIdx arm_v7m_mmu_idx_all(CPUARMState *env, - bool secstate, bool priv, bool negpri); +#define TO_CORE_BIT(NAME) \ + ARMMMUIdxBit_##NAME = 1 << (ARMMMUIdx_##NAME & ARM_MMU_IDX_COREIDX_MASK) -/* Return the MMU index for a v7M CPU in the specified security and - * privilege state. - */ -ARMMMUIdx arm_v7m_mmu_idx_for_secstate_and_priv(CPUARMState *env, - bool secstate, bool priv); +typedef enum ARMMMUIdxBit { + TO_CORE_BIT(E10_0), + TO_CORE_BIT(E20_0), + TO_CORE_BIT(E10_1), + TO_CORE_BIT(E2), + TO_CORE_BIT(E20_2), + TO_CORE_BIT(SE10_0), + TO_CORE_BIT(SE10_1), + TO_CORE_BIT(SE3), + TO_CORE_BIT(Stage2), + + TO_CORE_BIT(MUser), + TO_CORE_BIT(MPriv), + TO_CORE_BIT(MUserNegPri), + TO_CORE_BIT(MPrivNegPri), + TO_CORE_BIT(MSUser), + TO_CORE_BIT(MSPriv), + TO_CORE_BIT(MSUserNegPri), + TO_CORE_BIT(MSPrivNegPri), +} ARMMMUIdxBit; -/* Return the MMU index for a v7M CPU in the specified security state */ -ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate); +#undef TO_CORE_BIT + +#define MMU_USER_IDX 0 /** * cpu_mmu_index: @@ -3159,15 +3044,7 @@ static inline bool arm_sctlr_b(CPUARMState *env) (env->cp15.sctlr_el[1] & SCTLR_B) != 0; } -static inline uint64_t arm_sctlr(CPUARMState *env, int el) -{ - if (el == 0) { - /* FIXME: ARMv8.1-VHE S2 translation regime. */ - return env->cp15.sctlr_el[1]; - } else { - return env->cp15.sctlr_el[el]; - } -} +uint64_t arm_sctlr(CPUARMState *env, int el); static inline bool arm_cpu_data_is_big_endian_a32(CPUARMState *env, bool sctlr_b) @@ -3221,55 +3098,73 @@ typedef ARMCPU ArchCPU; * We put flags which are shared between 32 and 64 bit mode at the top * of the word, and flags which apply to only one mode at the bottom. * + * 31 20 18 14 9 0 + * +--------------+-----+-----+----------+--------------+ + * | | | TBFLAG_A32 | | + * | | +-----+----------+ TBFLAG_AM32 | + * | TBFLAG_ANY | |TBFLAG_M32| | + * | | +-+----------+--------------| + * | | | TBFLAG_A64 | + * +--------------+---------+---------------------------+ + * 31 20 15 0 + * * Unless otherwise noted, these bits are cached in env->hflags. */ FIELD(TBFLAG_ANY, AARCH64_STATE, 31, 1) -FIELD(TBFLAG_ANY, MMUIDX, 28, 3) -FIELD(TBFLAG_ANY, SS_ACTIVE, 27, 1) -FIELD(TBFLAG_ANY, PSTATE_SS, 26, 1) /* Not cached. */ +FIELD(TBFLAG_ANY, SS_ACTIVE, 30, 1) +FIELD(TBFLAG_ANY, PSTATE_SS, 29, 1) /* Not cached. */ +FIELD(TBFLAG_ANY, BE_DATA, 28, 1) +FIELD(TBFLAG_ANY, MMUIDX, 24, 4) /* Target EL if we take a floating-point-disabled exception */ -FIELD(TBFLAG_ANY, FPEXC_EL, 24, 2) -FIELD(TBFLAG_ANY, BE_DATA, 23, 1) +FIELD(TBFLAG_ANY, FPEXC_EL, 22, 2) +/* For A-profile only, target EL for debug exceptions. */ +FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 20, 2) + /* - * For A-profile only, target EL for debug exceptions. - * Note that this overlaps with the M-profile-only HANDLER and STACKCHECK bits. + * Bit usage when in AArch32 state, both A- and M-profile. */ -FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 21, 2) +FIELD(TBFLAG_AM32, CONDEXEC, 0, 8) /* Not cached. */ +FIELD(TBFLAG_AM32, THUMB, 8, 1) /* Not cached. */ -/* Bit usage when in AArch32 state: */ -FIELD(TBFLAG_A32, THUMB, 0, 1) /* Not cached. */ -FIELD(TBFLAG_A32, VECLEN, 1, 3) /* Not cached. */ -FIELD(TBFLAG_A32, VECSTRIDE, 4, 2) /* Not cached. */ +/* + * Bit usage when in AArch32 state, for A-profile only. + */ +FIELD(TBFLAG_A32, VECLEN, 9, 3) /* Not cached. */ +FIELD(TBFLAG_A32, VECSTRIDE, 12, 2) /* Not cached. */ /* * We store the bottom two bits of the CPAR as TB flags and handle * checks on the other bits at runtime. This shares the same bits as * VECSTRIDE, which is OK as no XScale CPU has VFP. * Not cached, because VECLEN+VECSTRIDE are not cached. */ -FIELD(TBFLAG_A32, XSCALE_CPAR, 4, 2) +FIELD(TBFLAG_A32, XSCALE_CPAR, 12, 2) +FIELD(TBFLAG_A32, VFPEN, 14, 1) /* Partially cached, minus FPEXC. */ +FIELD(TBFLAG_A32, SCTLR_B, 15, 1) +FIELD(TBFLAG_A32, HSTR_ACTIVE, 16, 1) /* * Indicates whether cp register reads and writes by guest code should access * the secure or nonsecure bank of banked registers; note that this is not * the same thing as the current security state of the processor! */ -FIELD(TBFLAG_A32, NS, 6, 1) -FIELD(TBFLAG_A32, VFPEN, 7, 1) /* Partially cached, minus FPEXC. */ -FIELD(TBFLAG_A32, CONDEXEC, 8, 8) /* Not cached. */ -FIELD(TBFLAG_A32, SCTLR_B, 16, 1) -FIELD(TBFLAG_A32, HSTR_ACTIVE, 17, 1) - -/* For M profile only, set if FPCCR.LSPACT is set */ -FIELD(TBFLAG_A32, LSPACT, 18, 1) /* Not cached. */ -/* For M profile only, set if we must create a new FP context */ -FIELD(TBFLAG_A32, NEW_FP_CTXT_NEEDED, 19, 1) /* Not cached. */ -/* For M profile only, set if FPCCR.S does not match current security state */ -FIELD(TBFLAG_A32, FPCCR_S_WRONG, 20, 1) /* Not cached. */ -/* For M profile only, Handler (ie not Thread) mode */ -FIELD(TBFLAG_A32, HANDLER, 21, 1) -/* For M profile only, whether we should generate stack-limit checks */ -FIELD(TBFLAG_A32, STACKCHECK, 22, 1) - -/* Bit usage when in AArch64 state */ +FIELD(TBFLAG_A32, NS, 17, 1) + +/* + * Bit usage when in AArch32 state, for M-profile only. + */ +/* Handler (ie not Thread) mode */ +FIELD(TBFLAG_M32, HANDLER, 9, 1) +/* Whether we should generate stack-limit checks */ +FIELD(TBFLAG_M32, STACKCHECK, 10, 1) +/* Set if FPCCR.LSPACT is set */ +FIELD(TBFLAG_M32, LSPACT, 11, 1) /* Not cached. */ +/* Set if we must create a new FP context */ +FIELD(TBFLAG_M32, NEW_FP_CTXT_NEEDED, 12, 1) /* Not cached. */ +/* Set if FPCCR.S does not match current security state */ +FIELD(TBFLAG_M32, FPCCR_S_WRONG, 13, 1) /* Not cached. */ + +/* + * Bit usage when in AArch64 state + */ FIELD(TBFLAG_A64, TBII, 0, 2) FIELD(TBFLAG_A64, SVEEXC_EL, 2, 2) FIELD(TBFLAG_A64, ZCR_LEN, 4, 4) @@ -3277,6 +3172,7 @@ FIELD(TBFLAG_A64, PAUTH_ACTIVE, 8, 1) FIELD(TBFLAG_A64, BT, 9, 1) FIELD(TBFLAG_A64, BTYPE, 10, 2) /* Not cached. */ FIELD(TBFLAG_A64, TBID, 12, 2) +FIELD(TBFLAG_A64, UNPRIV, 14, 1) static inline bool bswap_code(bool sctlr_b) { @@ -3685,6 +3581,11 @@ static inline bool isar_feature_aa64_sve(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, SVE) != 0; } +static inline bool isar_feature_aa64_vh(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64mmfr1, ID_AA64MMFR1, VH) != 0; +} + static inline bool isar_feature_aa64_lor(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64mmfr1, ID_AA64MMFR1, LO) != 0; diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index 2d97bf45e1..c80fb5fd43 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -672,6 +672,7 @@ static void aarch64_max_initfn(Object *obj) t = cpu->isar.id_aa64mmfr1; t = FIELD_DP64(t, ID_AA64MMFR1, HPDS, 1); /* HPD */ t = FIELD_DP64(t, ID_AA64MMFR1, LO, 1); + t = FIELD_DP64(t, ID_AA64MMFR1, VH, 1); cpu->isar.id_aa64mmfr1 = t; /* Replicate the same data to the 32-bit id registers. */ diff --git a/target/arm/debug_helper.c b/target/arm/debug_helper.c index dde80273ff..2e3e90c6a5 100644 --- a/target/arm/debug_helper.c +++ b/target/arm/debug_helper.c @@ -20,6 +20,7 @@ static bool linked_bp_matches(ARMCPU *cpu, int lbn) int ctx_cmps = extract32(cpu->dbgdidr, 20, 4); int bt; uint32_t contextidr; + uint64_t hcr_el2; /* * Links to unimplemented or non-context aware breakpoints are @@ -40,24 +41,44 @@ static bool linked_bp_matches(ARMCPU *cpu, int lbn) } bt = extract64(bcr, 20, 4); - - /* - * We match the whole register even if this is AArch32 using the - * short descriptor format (in which case it holds both PROCID and ASID), - * since we don't implement the optional v7 context ID masking. - */ - contextidr = extract64(env->cp15.contextidr_el[1], 0, 32); + hcr_el2 = arm_hcr_el2_eff(env); switch (bt) { case 3: /* linked context ID match */ - if (arm_current_el(env) > 1) { - /* Context matches never fire in EL2 or (AArch64) EL3 */ + switch (arm_current_el(env)) { + default: + /* Context matches never fire in AArch64 EL3 */ return false; + case 2: + if (!(hcr_el2 & HCR_E2H)) { + /* Context matches never fire in EL2 without E2H enabled. */ + return false; + } + contextidr = env->cp15.contextidr_el[2]; + break; + case 1: + contextidr = env->cp15.contextidr_el[1]; + break; + case 0: + if ((hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + contextidr = env->cp15.contextidr_el[2]; + } else { + contextidr = env->cp15.contextidr_el[1]; + } + break; } - return (contextidr == extract64(env->cp15.dbgbvr[lbn], 0, 32)); - case 5: /* linked address mismatch (reserved in AArch64) */ + break; + + case 7: /* linked contextidr_el1 match */ + contextidr = env->cp15.contextidr_el[1]; + break; + case 13: /* linked contextidr_el2 match */ + contextidr = env->cp15.contextidr_el[2]; + break; + case 9: /* linked VMID match (reserved if no EL2) */ case 11: /* linked context ID and VMID match (reserved if no EL2) */ + case 15: /* linked full context ID match */ default: /* * Links to Unlinked context breakpoints must generate no @@ -66,7 +87,12 @@ static bool linked_bp_matches(ARMCPU *cpu, int lbn) return false; } - return false; + /* + * We match the whole register even if this is AArch32 using the + * short descriptor format (in which case it holds both PROCID and ASID), + * since we don't implement the optional v7 context ID masking. + */ + return contextidr == (uint32_t)env->cp15.dbgbvr[lbn]; } static bool bp_wp_matches(ARMCPU *cpu, int n, bool is_wp) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index 36aa6badfd..bf45f8a785 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -70,7 +70,7 @@ static void daif_check(CPUARMState *env, uint32_t op, uint32_t imm, uintptr_t ra) { /* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { raise_exception_ra(env, EXCP_UDEF, syn_aa64_sysregtrap(0, extract32(op, 0, 3), extract32(op, 3, 3), 4, diff --git a/target/arm/helper.c b/target/arm/helper.c index 19a57a17da..7d15d5c933 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -614,56 +614,54 @@ static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate all (TLBIALL) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); if (tlb_force_broadcast(env)) { - tlbiall_is_write(env, NULL, value); - return; + tlb_flush_all_cpus_synced(cs); + } else { + tlb_flush(cs); } - - tlb_flush(CPU(cpu)); } static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); + value &= TARGET_PAGE_MASK; if (tlb_force_broadcast(env)) { - tlbimva_is_write(env, NULL, value); - return; + tlb_flush_page_all_cpus_synced(cs, value); + } else { + tlb_flush_page(cs, value); } - - tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK); } static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate by ASID (TLBIASID) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); if (tlb_force_broadcast(env)) { - tlbiasid_is_write(env, NULL, value); - return; + tlb_flush_all_cpus_synced(cs); + } else { + tlb_flush(cs); } - - tlb_flush(CPU(cpu)); } static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */ - ARMCPU *cpu = env_archcpu(env); + CPUState *cs = env_cpu(env); + value &= TARGET_PAGE_MASK; if (tlb_force_broadcast(env)) { - tlbimvaa_is_write(env, NULL, value); - return; + tlb_flush_page_all_cpus_synced(cs, value); + } else { + tlb_flush_page(cs, value); } - - tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK); } static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -672,9 +670,9 @@ static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); } static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -683,9 +681,9 @@ static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); } static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -706,7 +704,7 @@ static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 40); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S2NS); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_Stage2); } static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -722,7 +720,7 @@ static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 40); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_Stage2); } static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -730,7 +728,7 @@ static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_E2); } static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -738,7 +736,7 @@ static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_E2); } static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -747,7 +745,7 @@ static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = env_cpu(env); uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E2); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_E2); } static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -757,7 +755,7 @@ static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E2); + ARMMMUIdxBit_E2); } static const ARMCPRegInfo cp_reginfo[] = { @@ -2326,10 +2324,18 @@ static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri, * Writable only at the highest implemented exception level. */ int el = arm_current_el(env); + uint64_t hcr; + uint32_t cntkctl; switch (el) { case 0: - if (!extract32(env->cp15.c14_cntkctl, 0, 2)) { + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + cntkctl = env->cp15.cnthctl_el2; + } else { + cntkctl = env->cp15.c14_cntkctl; + } + if (!extract32(cntkctl, 0, 2)) { return CP_ACCESS_TRAP; } break; @@ -2357,17 +2363,47 @@ static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx, { unsigned int cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr = arm_hcr_el2_eff(env); - /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */ - if (cur_el == 0 && - !extract32(env->cp15.c14_cntkctl, timeridx, 1)) { - return CP_ACCESS_TRAP; - } + switch (cur_el) { + case 0: + /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]CTEN. */ + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + return (extract32(env->cp15.cnthctl_el2, timeridx, 1) + ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2); + } - if (arm_feature(env, ARM_FEATURE_EL2) && - timeridx == GTIMER_PHYS && !secure && cur_el < 2 && - !extract32(env->cp15.cnthctl_el2, 0, 1)) { - return CP_ACCESS_TRAP_EL2; + /* CNT[PV]CT: not visible from PL0 if EL0[PV]CTEN is zero */ + if (!extract32(env->cp15.c14_cntkctl, timeridx, 1)) { + return CP_ACCESS_TRAP; + } + + /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PCTEN. */ + if (hcr & HCR_E2H) { + if (timeridx == GTIMER_PHYS && + !extract32(env->cp15.cnthctl_el2, 10, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */ + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure && + !extract32(env->cp15.cnthctl_el2, 1, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } + break; + + case 1: + /* Check CNTHCTL_EL2.EL1PCTEN, which changes location based on E2H. */ + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure && + (hcr & HCR_E2H + ? !extract32(env->cp15.cnthctl_el2, 10, 1) + : !extract32(env->cp15.cnthctl_el2, 0, 1))) { + return CP_ACCESS_TRAP_EL2; + } + break; } return CP_ACCESS_OK; } @@ -2377,19 +2413,41 @@ static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx, { unsigned int cur_el = arm_current_el(env); bool secure = arm_is_secure(env); + uint64_t hcr = arm_hcr_el2_eff(env); - /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if - * EL0[PV]TEN is zero. - */ - if (cur_el == 0 && - !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) { - return CP_ACCESS_TRAP; - } + switch (cur_el) { + case 0: + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]TEN. */ + return (extract32(env->cp15.cnthctl_el2, 9 - timeridx, 1) + ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2); + } - if (arm_feature(env, ARM_FEATURE_EL2) && - timeridx == GTIMER_PHYS && !secure && cur_el < 2 && - !extract32(env->cp15.cnthctl_el2, 1, 1)) { - return CP_ACCESS_TRAP_EL2; + /* + * CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from + * EL0 if EL0[PV]TEN is zero. + */ + if (!extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) { + return CP_ACCESS_TRAP; + } + /* fall through */ + + case 1: + if (arm_feature(env, ARM_FEATURE_EL2) && + timeridx == GTIMER_PHYS && !secure) { + if (hcr & HCR_E2H) { + /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PTEN. */ + if (!extract32(env->cp15.cnthctl_el2, 11, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */ + if (!extract32(env->cp15.cnthctl_el2, 1, 1)) { + return CP_ACCESS_TRAP_EL2; + } + } + } + break; } return CP_ACCESS_OK; } @@ -2515,9 +2573,31 @@ static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri) return gt_get_countervalue(env); } +static uint64_t gt_virt_cnt_offset(CPUARMState *env) +{ + uint64_t hcr; + + switch (arm_current_el(env)) { + case 2: + hcr = arm_hcr_el2_eff(env); + if (hcr & HCR_E2H) { + return 0; + } + break; + case 0: + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + return 0; + } + break; + } + + return env->cp15.cntvoff_el2; +} + static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri) { - return gt_get_countervalue(env) - env->cp15.cntvoff_el2; + return gt_get_countervalue(env) - gt_virt_cnt_offset(env); } static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -2532,7 +2612,14 @@ static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri, int timeridx) { - uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0; + uint64_t offset = 0; + + switch (timeridx) { + case GTIMER_VIRT: + case GTIMER_HYPVIRT: + offset = gt_virt_cnt_offset(env); + break; + } return (uint32_t)(env->cp15.c14_timer[timeridx].cval - (gt_get_countervalue(env) - offset)); @@ -2542,7 +2629,14 @@ static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, int timeridx, uint64_t value) { - uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0; + uint64_t offset = 0; + + switch (timeridx) { + case GTIMER_VIRT: + case GTIMER_HYPVIRT: + offset = gt_virt_cnt_offset(env); + break; + } trace_arm_gt_tval_write(timeridx, value); env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset + @@ -2601,6 +2695,70 @@ static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_ctl_write(env, ri, GTIMER_PHYS, value); } +static int gt_phys_redir_timeridx(CPUARMState *env) +{ + switch (arm_mmu_idx(env)) { + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + return GTIMER_HYP; + default: + return GTIMER_PHYS; + } +} + +static int gt_virt_redir_timeridx(CPUARMState *env) +{ + switch (arm_mmu_idx(env)) { + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + return GTIMER_HYPVIRT; + default: + return GTIMER_VIRT; + } +} + +static uint64_t gt_phys_redir_cval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].cval; +} + +static void gt_phys_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_cval_write(env, ri, timeridx, value); +} + +static uint64_t gt_phys_redir_tval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return gt_tval_read(env, ri, timeridx); +} + +static void gt_phys_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_tval_write(env, ri, timeridx, value); +} + +static uint64_t gt_phys_redir_ctl_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_phys_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].ctl; +} + +static void gt_phys_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_phys_redir_timeridx(env); + gt_ctl_write(env, ri, timeridx, value); +} + static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) { gt_timer_reset(env, ri, GTIMER_VIRT); @@ -2639,6 +2797,48 @@ static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_recalc_timer(cpu, GTIMER_VIRT); } +static uint64_t gt_virt_redir_cval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].cval; +} + +static void gt_virt_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_cval_write(env, ri, timeridx, value); +} + +static uint64_t gt_virt_redir_tval_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return gt_tval_read(env, ri, timeridx); +} + +static void gt_virt_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_tval_write(env, ri, timeridx, value); +} + +static uint64_t gt_virt_redir_ctl_read(CPUARMState *env, + const ARMCPRegInfo *ri) +{ + int timeridx = gt_virt_redir_timeridx(env); + return env->cp15.c14_timer[timeridx].ctl; +} + +static void gt_virt_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + int timeridx = gt_virt_redir_timeridx(env); + gt_ctl_write(env, ri, timeridx, value); +} + static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) { gt_timer_reset(env, ri, GTIMER_HYP); @@ -2695,6 +2895,34 @@ static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, gt_ctl_write(env, ri, GTIMER_SEC, value); } +static void gt_hv_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri) +{ + gt_timer_reset(env, ri, GTIMER_HYPVIRT); +} + +static void gt_hv_cval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_cval_write(env, ri, GTIMER_HYPVIRT, value); +} + +static uint64_t gt_hv_tval_read(CPUARMState *env, const ARMCPRegInfo *ri) +{ + return gt_tval_read(env, ri, GTIMER_HYPVIRT); +} + +static void gt_hv_tval_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_tval_write(env, ri, GTIMER_HYPVIRT, value); +} + +static void gt_hv_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + gt_ctl_write(env, ri, GTIMER_HYPVIRT, value); +} + void arm_gt_ptimer_cb(void *opaque) { ARMCPU *cpu = opaque; @@ -2723,6 +2951,13 @@ void arm_gt_stimer_cb(void *opaque) gt_recalc_timer(cpu, GTIMER_SEC); } +void arm_gt_hvtimer_cb(void *opaque) +{ + ARMCPU *cpu = opaque; + + gt_recalc_timer(cpu, GTIMER_HYPVIRT); +} + static void arm_gt_cntfrq_reset(CPUARMState *env, const ARMCPRegInfo *opaque) { ARMCPU *cpu = env_archcpu(env); @@ -2760,7 +2995,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_ptimer_access, .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), - .writefn = gt_phys_ctl_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTP_CTL_S", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1, @@ -2777,14 +3013,16 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_ptimer_access, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), .resetvalue = 0, - .writefn = gt_phys_ctl_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1, .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW, .accessfn = gt_vtimer_access, .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), - .writefn = gt_virt_ctl_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write, }, { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1, @@ -2792,14 +3030,15 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .accessfn = gt_vtimer_access, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), .resetvalue = 0, - .writefn = gt_virt_ctl_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write, }, /* TimerValue views: a 32 bit downcounting view of the underlying state */ { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0, .secure = ARM_CP_SECSTATE_NS, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_ptimer_access, - .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write, + .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write, }, { .name = "CNTP_TVAL_S", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0, @@ -2812,18 +3051,18 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset, - .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write, + .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write, }, { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_vtimer_access, - .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write, + .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write, }, { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0, .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW, .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset, - .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write, + .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write, }, /* The counter itself */ { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0, @@ -2853,7 +3092,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), .accessfn = gt_ptimer_access, - .writefn = gt_phys_cval_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTP_CVAL_S", .cp = 15, .crm = 14, .opc1 = 2, .secure = ARM_CP_SECSTATE_S, @@ -2869,14 +3109,16 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), .resetvalue = 0, .accessfn = gt_ptimer_access, - .writefn = gt_phys_cval_write, .raw_writefn = raw_write, + .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3, .access = PL0_RW, .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), .accessfn = gt_vtimer_access, - .writefn = gt_virt_cval_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write, }, { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2, @@ -2884,7 +3126,8 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { .type = ARM_CP_IO, .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), .resetvalue = 0, .accessfn = gt_vtimer_access, - .writefn = gt_virt_cval_write, .raw_writefn = raw_write, + .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read, + .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write, }, /* Secure timer -- this is actually restricted to only EL3 * and configurably Secure-EL1 via the accessfn. @@ -2915,6 +3158,15 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = { REGINFO_SENTINEL }; +static CPAccessResult e2h_access(CPUARMState *env, const ARMCPRegInfo *ri, + bool isread) +{ + if (!(arm_hcr_el2_eff(env) & HCR_E2H)) { + return CP_ACCESS_TRAP; + } + return CP_ACCESS_OK; +} + #else /* In user-mode most of the generic timer registers are inaccessible @@ -3009,7 +3261,8 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value, bool take_exc = false; if (fi.s1ptw && current_el == 1 && !arm_is_secure(env) - && (mmu_idx == ARMMMUIdx_S1NSE1 || mmu_idx == ARMMMUIdx_S1NSE0)) { + && (mmu_idx == ARMMMUIdx_Stage1_E1 || + mmu_idx == ARMMMUIdx_Stage1_E0)) { /* * Synchronous stage 2 fault on an access made as part of the * translation table walk for AT S1E0* or AT S1E1* insn @@ -3085,7 +3338,7 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value, format64 = arm_s1_regime_using_lpae_format(env, mmu_idx); if (arm_feature(env, ARM_FEATURE_EL2)) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { + if (mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_E10_1) { format64 |= env->cp15.hcr_el2 & (HCR_VM | HCR_DC); } else { format64 |= arm_current_el(env) == 2; @@ -3154,13 +3407,13 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) /* stage 1 current state PL1: ATS1CPR, ATS1CPW */ switch (el) { case 3: - mmu_idx = ARMMMUIdx_S1E3; + mmu_idx = ARMMMUIdx_SE3; break; case 2: - mmu_idx = ARMMMUIdx_S1NSE1; + mmu_idx = ARMMMUIdx_Stage1_E1; break; case 1: - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1; break; default: g_assert_not_reached(); @@ -3170,13 +3423,13 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) /* stage 1 current state PL0: ATS1CUR, ATS1CUW */ switch (el) { case 3: - mmu_idx = ARMMMUIdx_S1SE0; + mmu_idx = ARMMMUIdx_SE10_0; break; case 2: - mmu_idx = ARMMMUIdx_S1NSE0; + mmu_idx = ARMMMUIdx_Stage1_E0; break; case 1: - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0; break; default: g_assert_not_reached(); @@ -3184,11 +3437,11 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) break; case 4: /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */ - mmu_idx = ARMMMUIdx_S12NSE1; + mmu_idx = ARMMMUIdx_E10_1; break; case 6: /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */ - mmu_idx = ARMMMUIdx_S12NSE0; + mmu_idx = ARMMMUIdx_E10_0; break; default: g_assert_not_reached(); @@ -3205,7 +3458,7 @@ static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri, MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD; uint64_t par64; - par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S1E2); + par64 = do_ats_write(env, value, access_type, ARMMMUIdx_E2); A32_BANKED_CURRENT_REG_SET(env, par, par64); } @@ -3230,26 +3483,26 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri, case 0: switch (ri->opc1) { case 0: /* AT S1E1R, AT S1E1W */ - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1; break; case 4: /* AT S1E2R, AT S1E2W */ - mmu_idx = ARMMMUIdx_S1E2; + mmu_idx = ARMMMUIdx_E2; break; case 6: /* AT S1E3R, AT S1E3W */ - mmu_idx = ARMMMUIdx_S1E3; + mmu_idx = ARMMMUIdx_SE3; break; default: g_assert_not_reached(); } break; case 2: /* AT S1E0R, AT S1E0W */ - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0; break; case 4: /* AT S12E1R, AT S12E1W */ - mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1; + mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_E10_1; break; case 6: /* AT S12E0R, AT S12E0W */ - mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0; + mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_E10_0; break; default: g_assert_not_reached(); @@ -3510,7 +3763,7 @@ static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri) tcr->base_mask = 0xffffc000u; } -static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri, +static void vmsa_tcr_el12_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); @@ -3533,18 +3786,38 @@ static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri, raw_write(env, ri, value); } +static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + /* + * If we are running with E2&0 regime, then an ASID is active. + * Flush if that might be changing. Note we're not checking + * TCR_EL2.A1 to know if this is really the TTBRx_EL2 that + * holds the active ASID, only checking the field that might. + */ + if (extract64(raw_read(env, ri) ^ value, 48, 16) && + (arm_hcr_el2_eff(env) & HCR_E2H)) { + tlb_flush_by_mmuidx(env_cpu(env), + ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E20_0); + } + raw_write(env, ri, value); +} + static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); - /* Accesses to VTTBR may change the VMID so we must flush the TLB. */ + /* + * A change in VMID to the stage2 page table (Stage2) invalidates + * the combined stage 1&2 tlbs (EL10_1 and EL10_0). + */ if (raw_read(env, ri) != value) { tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_E10_1 | + ARMMMUIdxBit_E10_0 | + ARMMMUIdxBit_Stage2); raw_write(env, ri, value); } } @@ -3586,7 +3859,7 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = { offsetof(CPUARMState, cp15.ttbr1_ns) } }, { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64, .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, - .access = PL1_RW, .writefn = vmsa_tcr_el1_write, + .access = PL1_RW, .writefn = vmsa_tcr_el12_write, .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write, .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) }, { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2, @@ -3870,7 +4143,7 @@ static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri, static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { return CP_ACCESS_TRAP; } return CP_ACCESS_OK; @@ -3889,7 +4162,7 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env, /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless * SCTLR_EL1.UCI is set. */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) { + if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UCI)) { return CP_ACCESS_TRAP; } return CP_ACCESS_OK; @@ -3899,79 +4172,79 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env, * Page D4-1736 (DDI0487A.b) */ +static int vae1_tlbmask(CPUARMState *env) +{ + /* Since we exclude secure first, we may read HCR_EL2 directly. */ + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdxBit_SE10_1 | ARMMMUIdxBit_SE10_0; + } else if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE)) + == (HCR_E2H | HCR_TGE)) { + return ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E20_0; + } else { + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0; + } +} + static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); - bool sec = arm_is_secure_below_el3(env); + int mask = vae1_tlbmask(env); - if (sec) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); if (tlb_force_broadcast(env)) { - tlbi_aa64_vmalle1is_write(env, NULL, value); - return; - } - - if (arm_is_secure_below_el3(env)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } else { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); + tlb_flush_by_mmuidx(cs, mask); } } -static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri, - uint64_t value) +static int alle1_tlbmask(CPUARMState *env) { - /* Note that the 'ALL' scope must invalidate both stage 1 and + /* + * Note that the 'ALL' scope must invalidate both stage 1 and * stage 2 translations, whereas most other scopes only invalidate * stage 1 translations. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - if (arm_is_secure_below_el3(env)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + return ARMMMUIdxBit_SE10_1 | ARMMMUIdxBit_SE10_0; + } else if (arm_feature(env, ARM_FEATURE_EL2)) { + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0 | ARMMMUIdxBit_Stage2; } else { - if (arm_feature(env, ARM_FEATURE_EL2)) { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); - } else { - tlb_flush_by_mmuidx(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + return ARMMMUIdxBit_E10_1 | ARMMMUIdxBit_E10_0; } } +static int e2_tlbmask(CPUARMState *env) +{ + /* TODO: ARMv8.4-SecEL2 */ + return ARMMMUIdxBit_E20_0 | ARMMMUIdxBit_E20_2 | ARMMMUIdxBit_E2; +} + +static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + CPUState *cs = env_cpu(env); + int mask = alle1_tlbmask(env); + + tlb_flush_by_mmuidx(cs, mask); +} + static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx(cs, mask); } static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -3980,42 +4253,25 @@ static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri, ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); - tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_S1E3); + tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_SE3); } static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - /* Note that the 'ALL' scope must invalidate both stage 1 and - * stage 2 translations, whereas most other scopes only invalidate - * stage 1 translations. - */ CPUState *cs = env_cpu(env); - bool sec = arm_is_secure_below_el3(env); - bool has_el2 = arm_feature(env, ARM_FEATURE_EL2); - - if (sec) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else if (has_el2) { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0 | - ARMMMUIdxBit_S2NS); - } else { - tlb_flush_by_mmuidx_all_cpus_synced(cs, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + int mask = alle1_tlbmask(env); + + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E2); + tlb_flush_by_mmuidx_all_cpus_synced(cs, mask); } static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4023,7 +4279,7 @@ static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri, { CPUState *cs = env_cpu(env); - tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_S1E3); + tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_SE3); } static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4033,11 +4289,11 @@ static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri, * Currently handles both VAE2 and VALE2, since we don't support * flush-last-level-only. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = e2_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E2); + tlb_flush_page_by_mmuidx(cs, pageaddr, mask); } static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4051,26 +4307,17 @@ static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri, CPUState *cs = CPU(cpu); uint64_t pageaddr = sextract64(value << 12, 0, 56); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S1E3); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_SE3); } static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - bool sec = arm_is_secure_below_el3(env); + CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); - if (sec) { - tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); - } else { - tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); - } + tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask); } static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4081,23 +4328,14 @@ static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri, * since we don't support flush-for-specific-ASID-only or * flush-last-level-only. */ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); + CPUState *cs = env_cpu(env); + int mask = vae1_tlbmask(env); uint64_t pageaddr = sextract64(value << 12, 0, 56); if (tlb_force_broadcast(env)) { - tlbi_aa64_vae1is_write(env, NULL, value); - return; - } - - if (arm_is_secure_below_el3(env)) { - tlb_flush_page_by_mmuidx(cs, pageaddr, - ARMMMUIdxBit_S1SE1 | - ARMMMUIdxBit_S1SE0); + tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask); } else { - tlb_flush_page_by_mmuidx(cs, pageaddr, - ARMMMUIdxBit_S12NSE1 | - ARMMMUIdxBit_S12NSE0); + tlb_flush_page_by_mmuidx(cs, pageaddr, mask); } } @@ -4108,7 +4346,7 @@ static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = sextract64(value << 12, 0, 56); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E2); + ARMMMUIdxBit_E2); } static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4118,7 +4356,7 @@ static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t pageaddr = sextract64(value << 12, 0, 56); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S1E3); + ARMMMUIdxBit_SE3); } static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4140,7 +4378,7 @@ static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 48); - tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_S2NS); + tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_Stage2); } static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri, @@ -4156,17 +4394,33 @@ static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri, pageaddr = sextract64(value << 12, 0, 48); tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, - ARMMMUIdxBit_S2NS); + ARMMMUIdxBit_Stage2); } static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - /* We don't implement EL2, so the only control on DC ZVA is the - * bit in the SCTLR which can prohibit access for EL0. - */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) { - return CP_ACCESS_TRAP; + int cur_el = arm_current_el(env); + + if (cur_el < 2) { + uint64_t hcr = arm_hcr_el2_eff(env); + + if (cur_el == 0) { + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + if (!(env->cp15.sctlr_el[2] & SCTLR_DZE)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + if (!(env->cp15.sctlr_el[1] & SCTLR_DZE)) { + return CP_ACCESS_TRAP; + } + if (hcr & HCR_TDZ) { + return CP_ACCESS_TRAP_EL2; + } + } + } else if (hcr & HCR_TDZ) { + return CP_ACCESS_TRAP_EL2; + } } return CP_ACCESS_OK; } @@ -4721,7 +4975,8 @@ static const ARMCPRegInfo el3_no_el2_v8_cp_reginfo[] = { static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { ARMCPU *cpu = env_archcpu(env); - uint64_t valid_mask = HCR_MASK; + /* Begin with bits defined in base ARMv8.0. */ + uint64_t valid_mask = MAKE_64BIT_MASK(0, 34); if (arm_feature(env, ARM_FEATURE_EL3)) { valid_mask &= ~HCR_HCD; @@ -4735,6 +4990,9 @@ static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) */ valid_mask &= ~HCR_TSC; } + if (cpu_isar_feature(aa64_vh, cpu)) { + valid_mask |= HCR_E2H; + } if (cpu_isar_feature(aa64_lor, cpu)) { valid_mask |= HCR_TLOR; } @@ -4938,10 +5196,8 @@ static const ARMCPRegInfo el2_cp_reginfo[] = { .resetvalue = 0 }, { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH, .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2, - .access = PL2_RW, - /* no .writefn needed as this can't cause an ASID change; - * no .raw_writefn or .resetfn needed as we never use mask/base_mask - */ + .access = PL2_RW, .writefn = vmsa_tcr_el12_write, + /* no .raw_writefn or .resetfn needed as we never use mask/base_mask */ .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) }, { .name = "VTCR", .state = ARM_CP_STATE_AA32, .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2, @@ -4975,7 +5231,7 @@ static const ARMCPRegInfo el2_cp_reginfo[] = { .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) }, { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0, - .access = PL2_RW, .resetvalue = 0, + .access = PL2_RW, .resetvalue = 0, .writefn = vmsa_tcr_ttbr_el2_write, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) }, { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2, .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS, @@ -5244,14 +5500,182 @@ static const ARMCPRegInfo el3_cp_reginfo[] = { REGINFO_SENTINEL }; +#ifndef CONFIG_USER_ONLY +/* Test if system register redirection is to occur in the current state. */ +static bool redirect_for_e2h(CPUARMState *env) +{ + return arm_current_el(env) == 2 && (arm_hcr_el2_eff(env) & HCR_E2H); +} + +static uint64_t el2_e2h_read(CPUARMState *env, const ARMCPRegInfo *ri) +{ + CPReadFn *readfn; + + if (redirect_for_e2h(env)) { + /* Switch to the saved EL2 version of the register. */ + ri = ri->opaque; + readfn = ri->readfn; + } else { + readfn = ri->orig_readfn; + } + if (readfn == NULL) { + readfn = raw_read; + } + return readfn(env, ri); +} + +static void el2_e2h_write(CPUARMState *env, const ARMCPRegInfo *ri, + uint64_t value) +{ + CPWriteFn *writefn; + + if (redirect_for_e2h(env)) { + /* Switch to the saved EL2 version of the register. */ + ri = ri->opaque; + writefn = ri->writefn; + } else { + writefn = ri->orig_writefn; + } + if (writefn == NULL) { + writefn = raw_write; + } + writefn(env, ri, value); +} + +static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu) +{ + struct E2HAlias { + uint32_t src_key, dst_key, new_key; + const char *src_name, *dst_name, *new_name; + bool (*feature)(const ARMISARegisters *id); + }; + +#define K(op0, op1, crn, crm, op2) \ + ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, crn, crm, op0, op1, op2) + + static const struct E2HAlias aliases[] = { + { K(3, 0, 1, 0, 0), K(3, 4, 1, 0, 0), K(3, 5, 1, 0, 0), + "SCTLR", "SCTLR_EL2", "SCTLR_EL12" }, + { K(3, 0, 1, 0, 2), K(3, 4, 1, 1, 2), K(3, 5, 1, 0, 2), + "CPACR", "CPTR_EL2", "CPACR_EL12" }, + { K(3, 0, 2, 0, 0), K(3, 4, 2, 0, 0), K(3, 5, 2, 0, 0), + "TTBR0_EL1", "TTBR0_EL2", "TTBR0_EL12" }, + { K(3, 0, 2, 0, 1), K(3, 4, 2, 0, 1), K(3, 5, 2, 0, 1), + "TTBR1_EL1", "TTBR1_EL2", "TTBR1_EL12" }, + { K(3, 0, 2, 0, 2), K(3, 4, 2, 0, 2), K(3, 5, 2, 0, 2), + "TCR_EL1", "TCR_EL2", "TCR_EL12" }, + { K(3, 0, 4, 0, 0), K(3, 4, 4, 0, 0), K(3, 5, 4, 0, 0), + "SPSR_EL1", "SPSR_EL2", "SPSR_EL12" }, + { K(3, 0, 4, 0, 1), K(3, 4, 4, 0, 1), K(3, 5, 4, 0, 1), + "ELR_EL1", "ELR_EL2", "ELR_EL12" }, + { K(3, 0, 5, 1, 0), K(3, 4, 5, 1, 0), K(3, 5, 5, 1, 0), + "AFSR0_EL1", "AFSR0_EL2", "AFSR0_EL12" }, + { K(3, 0, 5, 1, 1), K(3, 4, 5, 1, 1), K(3, 5, 5, 1, 1), + "AFSR1_EL1", "AFSR1_EL2", "AFSR1_EL12" }, + { K(3, 0, 5, 2, 0), K(3, 4, 5, 2, 0), K(3, 5, 5, 2, 0), + "ESR_EL1", "ESR_EL2", "ESR_EL12" }, + { K(3, 0, 6, 0, 0), K(3, 4, 6, 0, 0), K(3, 5, 6, 0, 0), + "FAR_EL1", "FAR_EL2", "FAR_EL12" }, + { K(3, 0, 10, 2, 0), K(3, 4, 10, 2, 0), K(3, 5, 10, 2, 0), + "MAIR_EL1", "MAIR_EL2", "MAIR_EL12" }, + { K(3, 0, 10, 3, 0), K(3, 4, 10, 3, 0), K(3, 5, 10, 3, 0), + "AMAIR0", "AMAIR_EL2", "AMAIR_EL12" }, + { K(3, 0, 12, 0, 0), K(3, 4, 12, 0, 0), K(3, 5, 12, 0, 0), + "VBAR", "VBAR_EL2", "VBAR_EL12" }, + { K(3, 0, 13, 0, 1), K(3, 4, 13, 0, 1), K(3, 5, 13, 0, 1), + "CONTEXTIDR_EL1", "CONTEXTIDR_EL2", "CONTEXTIDR_EL12" }, + { K(3, 0, 14, 1, 0), K(3, 4, 14, 1, 0), K(3, 5, 14, 1, 0), + "CNTKCTL", "CNTHCTL_EL2", "CNTKCTL_EL12" }, + + /* + * Note that redirection of ZCR is mentioned in the description + * of ZCR_EL2, and aliasing in the description of ZCR_EL1, but + * not in the summary table. + */ + { K(3, 0, 1, 2, 0), K(3, 4, 1, 2, 0), K(3, 5, 1, 2, 0), + "ZCR_EL1", "ZCR_EL2", "ZCR_EL12", isar_feature_aa64_sve }, + + /* TODO: ARMv8.2-SPE -- PMSCR_EL2 */ + /* TODO: ARMv8.4-Trace -- TRFCR_EL2 */ + }; +#undef K + + size_t i; + + for (i = 0; i < ARRAY_SIZE(aliases); i++) { + const struct E2HAlias *a = &aliases[i]; + ARMCPRegInfo *src_reg, *dst_reg; + + if (a->feature && !a->feature(&cpu->isar)) { + continue; + } + + src_reg = g_hash_table_lookup(cpu->cp_regs, &a->src_key); + dst_reg = g_hash_table_lookup(cpu->cp_regs, &a->dst_key); + g_assert(src_reg != NULL); + g_assert(dst_reg != NULL); + + /* Cross-compare names to detect typos in the keys. */ + g_assert(strcmp(src_reg->name, a->src_name) == 0); + g_assert(strcmp(dst_reg->name, a->dst_name) == 0); + + /* None of the core system registers use opaque; we will. */ + g_assert(src_reg->opaque == NULL); + + /* Create alias before redirection so we dup the right data. */ + if (a->new_key) { + ARMCPRegInfo *new_reg = g_memdup(src_reg, sizeof(ARMCPRegInfo)); + uint32_t *new_key = g_memdup(&a->new_key, sizeof(uint32_t)); + bool ok; + + new_reg->name = a->new_name; + new_reg->type |= ARM_CP_ALIAS; + /* Remove PL1/PL0 access, leaving PL2/PL3 R/W in place. */ + new_reg->access &= PL2_RW | PL3_RW; + + ok = g_hash_table_insert(cpu->cp_regs, new_key, new_reg); + g_assert(ok); + } + + src_reg->opaque = dst_reg; + src_reg->orig_readfn = src_reg->readfn ?: raw_read; + src_reg->orig_writefn = src_reg->writefn ?: raw_write; + if (!src_reg->raw_readfn) { + src_reg->raw_readfn = raw_read; + } + if (!src_reg->raw_writefn) { + src_reg->raw_writefn = raw_write; + } + src_reg->readfn = el2_e2h_read; + src_reg->writefn = el2_e2h_write; + } +} +#endif + static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri, bool isread) { - /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64, - * but the AArch32 CTR has its own reginfo struct) - */ - if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) { - return CP_ACCESS_TRAP; + int cur_el = arm_current_el(env); + + if (cur_el < 2) { + uint64_t hcr = arm_hcr_el2_eff(env); + + if (cur_el == 0) { + if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) { + if (!(env->cp15.sctlr_el[2] & SCTLR_UCT)) { + return CP_ACCESS_TRAP_EL2; + } + } else { + if (!(env->cp15.sctlr_el[1] & SCTLR_UCT)) { + return CP_ACCESS_TRAP; + } + if (hcr & HCR_TID2) { + return CP_ACCESS_TRAP_EL2; + } + } + } else if (hcr & HCR_TID2) { + return CP_ACCESS_TRAP_EL2; + } } if (arm_current_el(env) < 2 && arm_hcr_el2_eff(env) & HCR_TID2) { @@ -5367,7 +5791,9 @@ static const ARMCPRegInfo debug_lpae_cp_reginfo[] = { int sve_exception_el(CPUARMState *env, int el) { #ifndef CONFIG_USER_ONLY - if (el <= 1) { + uint64_t hcr_el2 = arm_hcr_el2_eff(env); + + if (el <= 1 && (hcr_el2 & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { bool disabled = false; /* The CPACR.ZEN controls traps to EL1: @@ -5382,8 +5808,7 @@ int sve_exception_el(CPUARMState *env, int el) } if (disabled) { /* route_to_el2 */ - return (arm_feature(env, ARM_FEATURE_EL2) - && (arm_hcr_el2_eff(env) & HCR_TGE) ? 2 : 1); + return hcr_el2 & HCR_TGE ? 2 : 1; } /* Check CPACR.FPEN. */ @@ -6122,6 +6547,71 @@ static const ARMCPRegInfo jazelle_regs[] = { REGINFO_SENTINEL }; +static const ARMCPRegInfo vhe_reginfo[] = { + { .name = "CONTEXTIDR_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 1, + .access = PL2_RW, + .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[2]) }, + { .name = "TTBR1_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 1, + .access = PL2_RW, .writefn = vmsa_tcr_ttbr_el2_write, + .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[2]) }, +#ifndef CONFIG_USER_ONLY + { .name = "CNTHV_CVAL_EL2", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 2, + .fieldoffset = + offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].cval), + .type = ARM_CP_IO, .access = PL2_RW, + .writefn = gt_hv_cval_write, .raw_writefn = raw_write }, + { .name = "CNTHV_TVAL_EL2", .state = ARM_CP_STATE_BOTH, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW, + .resetfn = gt_hv_timer_reset, + .readfn = gt_hv_tval_read, .writefn = gt_hv_tval_write }, + { .name = "CNTHV_CTL_EL2", .state = ARM_CP_STATE_BOTH, + .type = ARM_CP_IO, + .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 1, + .access = PL2_RW, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].ctl), + .writefn = gt_hv_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTP_CTL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 1, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl), + .writefn = gt_phys_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTV_CTL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 1, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl), + .writefn = gt_virt_ctl_write, .raw_writefn = raw_write }, + { .name = "CNTP_TVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write }, + { .name = "CNTV_TVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 0, + .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS, + .access = PL2_RW, .accessfn = e2h_access, + .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write }, + { .name = "CNTP_CVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 2, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval), + .access = PL2_RW, .accessfn = e2h_access, + .writefn = gt_phys_cval_write, .raw_writefn = raw_write }, + { .name = "CNTV_CVAL_EL02", .state = ARM_CP_STATE_AA64, + .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 2, + .type = ARM_CP_IO | ARM_CP_ALIAS, + .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval), + .access = PL2_RW, .accessfn = e2h_access, + .writefn = gt_virt_cval_write, .raw_writefn = raw_write }, +#endif + REGINFO_SENTINEL +}; + void register_cp_regs_for_features(ARMCPU *cpu) { /* Register all the coprocessor registers based on feature bits */ @@ -7085,6 +7575,10 @@ void register_cp_regs_for_features(ARMCPU *cpu) define_arm_cp_regs(cpu, lor_reginfo); } + if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) { + define_arm_cp_regs(cpu, vhe_reginfo); + } + if (cpu_isar_feature(aa64_sve, cpu)) { define_one_arm_cp_reg(cpu, &zcr_el1_reginfo); if (arm_feature(env, ARM_FEATURE_EL2)) { @@ -7126,6 +7620,16 @@ void register_cp_regs_for_features(ARMCPU *cpu) : cpu_isar_feature(aa32_predinv, cpu)) { define_arm_cp_regs(cpu, predinv_reginfo); } + +#ifndef CONFIG_USER_ONLY + /* + * Register redirections and aliases must be done last, + * after the registers from the other extensions have been defined. + */ + if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) { + define_arm_vh_e2h_redirects_aliases(cpu); + } +#endif } void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu) @@ -7421,13 +7925,10 @@ void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu, mask = PL0_RW; break; case 4: + case 5: /* min_EL EL2 */ mask = PL2_RW; break; - case 5: - /* unallocated encoding, so not possible */ - assert(false); - break; case 6: /* min_EL EL3 */ mask = PL3_RW; @@ -7946,6 +8447,12 @@ uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx, break; }; + /* + * For these purposes, TGE and AMO/IMO/FMO both force the + * interrupt to EL2. Fold TGE into the bit extracted above. + */ + hcr |= (hcr_el2 & HCR_TGE) != 0; + /* Perform a table-lookup for the target EL given the current state */ target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el]; @@ -8510,14 +9017,19 @@ static void arm_cpu_do_interrupt_aarch64(CPUState *cs) * immediately lower than the target level is using AArch32 or AArch64 */ bool is_aa64; + uint64_t hcr; switch (new_el) { case 3: is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0; break; case 2: - is_aa64 = (env->cp15.hcr_el2 & HCR_RW) != 0; - break; + hcr = arm_hcr_el2_eff(env); + if ((hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + is_aa64 = (hcr & HCR_RW) != 0; + break; + } + /* fall through */ case 1: is_aa64 = is_a64(env); break; @@ -8690,19 +9202,23 @@ void arm_cpu_do_interrupt(CPUState *cs) #endif /* !CONFIG_USER_ONLY */ /* Return the exception level which controls this address translation regime */ -static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) +static uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S2NS: - case ARMMMUIdx_S1E2: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + case ARMMMUIdx_Stage2: + case ARMMMUIdx_E2: return 2; - case ARMMMUIdx_S1E3: + case ARMMMUIdx_SE3: return 3; - case ARMMMUIdx_S1SE0: + case ARMMMUIdx_SE10_0: return arm_el_is_aa64(env, 3) ? 1 : 3; - case ARMMMUIdx_S1SE1: - case ARMMMUIdx_S1NSE0: - case ARMMMUIdx_S1NSE1: + case ARMMMUIdx_SE10_1: + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_E1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: case ARMMMUIdx_MPrivNegPri: case ARMMMUIdx_MUserNegPri: case ARMMMUIdx_MPriv: @@ -8717,14 +9233,24 @@ static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) } } -#ifndef CONFIG_USER_ONLY +uint64_t arm_sctlr(CPUARMState *env, int el) +{ + /* Only EL0 needs to be adjusted for EL1&0 or EL2&0. */ + if (el == 0) { + ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, 0); + el = (mmu_idx == ARMMMUIdx_E20_0 ? 2 : 1); + } + return env->cp15.sctlr_el[el]; +} /* Return the SCTLR value which controls this address translation regime */ -static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) +static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) { return env->cp15.sctlr_el[regime_el(env, mmu_idx)]; } +#ifndef CONFIG_USER_ONLY + /* Return true if the specified stage of address translation is disabled */ static inline bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) @@ -8747,7 +9273,7 @@ static inline bool regime_translation_disabled(CPUARMState *env, } } - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* HCR.DC means HCR.VM behaves as 1 */ return (env->cp15.hcr_el2 & (HCR_DC | HCR_VM)) == 0; } @@ -8760,7 +9286,7 @@ static inline bool regime_translation_disabled(CPUARMState *env, } if ((env->cp15.hcr_el2 & HCR_DC) && - (mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1)) { + (mmu_idx == ARMMMUIdx_Stage1_E0 || mmu_idx == ARMMMUIdx_Stage1_E1)) { /* HCR.DC means SCTLR_EL1.M behaves as 0 */ return true; } @@ -8778,7 +9304,7 @@ static inline bool regime_translation_big_endian(CPUARMState *env, static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { return env->cp15.vttbr_el2; } if (ttbrn == 0) { @@ -8793,7 +9319,7 @@ static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, /* Return the TCR controlling this translation regime */ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { return &env->cp15.vtcr_el2; } return &env->cp15.tcr_el[regime_el(env, mmu_idx)]; @@ -8804,10 +9330,14 @@ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) */ static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { - mmu_idx += (ARMMMUIdx_S1NSE0 - ARMMMUIdx_S12NSE0); + switch (mmu_idx) { + case ARMMMUIdx_E10_0: + return ARMMMUIdx_Stage1_E0; + case ARMMMUIdx_E10_1: + return ARMMMUIdx_Stage1_E1; + default: + return mmu_idx; } - return mmu_idx; } /* Return true if the translation regime is using LPAE format page tables */ @@ -8839,8 +9369,9 @@ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S1SE0: - case ARMMMUIdx_S1NSE0: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_Stage1_E0: case ARMMMUIdx_MUser: case ARMMMUIdx_MSUser: case ARMMMUIdx_MUserNegPri: @@ -8848,8 +9379,8 @@ static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) return true; default: return false; - case ARMMMUIdx_S12NSE0: - case ARMMMUIdx_S12NSE1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: g_assert_not_reached(); } } @@ -8980,7 +9511,7 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, bool have_wxn; int wxn = 0; - assert(mmu_idx != ARMMMUIdx_S2NS); + assert(mmu_idx != ARMMMUIdx_Stage2); user_rw = simple_ap_to_rw_prot_is_user(ap, true); if (is_user) { @@ -9005,15 +9536,8 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, } if (is_aa64) { - switch (regime_el(env, mmu_idx)) { - case 1: - if (!is_user) { - xn = pxn || (user_rw & PAGE_WRITE); - } - break; - case 2: - case 3: - break; + if (regime_has_2_ranges(mmu_idx) && !is_user) { + xn = pxn || (user_rw & PAGE_WRITE); } } else if (arm_feature(env, ARM_FEATURE_V7)) { switch (regime_el(env, mmu_idx)) { @@ -9071,8 +9595,8 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, hwaddr addr, MemTxAttrs txattrs, ARMMMUFaultInfo *fi) { - if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) && - !regime_translation_disabled(env, ARMMMUIdx_S2NS)) { + if ((mmu_idx == ARMMMUIdx_Stage1_E0 || mmu_idx == ARMMMUIdx_Stage1_E1) && + !regime_translation_disabled(env, ARMMMUIdx_Stage2)) { target_ulong s2size; hwaddr s2pa; int s2prot; @@ -9089,7 +9613,7 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, pcacheattrs = &cacheattrs; } - ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa, + ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_Stage2, &s2pa, &txattrs, &s2prot, &s2size, fi, pcacheattrs); if (ret) { assert(fi->type != ARMFault_None); @@ -9547,7 +10071,6 @@ ARMVAParameters aa64_va_parameters_both(CPUARMState *env, uint64_t va, ARMMMUIdx mmu_idx) { uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; - uint32_t el = regime_el(env, mmu_idx); bool tbi, tbid, epd, hpd, using16k, using64k; int select, tsz; @@ -9557,11 +10080,11 @@ ARMVAParameters aa64_va_parameters_both(CPUARMState *env, uint64_t va, */ select = extract64(va, 55, 1); - if (el > 1) { + if (!regime_has_2_ranges(mmu_idx)) { tsz = extract32(tcr, 0, 6); using64k = extract32(tcr, 14, 1); using16k = extract32(tcr, 15, 1); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* VTCR_EL2 */ tbi = tbid = hpd = false; } else { @@ -9622,7 +10145,7 @@ static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, int select, tsz; bool epd, hpd; - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* VTCR */ bool sext = extract32(tcr, 4, 1); bool sign = extract32(tcr, 3, 1); @@ -9713,10 +10236,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, param = aa64_va_parameters(env, address, mmu_idx, access_type != MMU_INST_FETCH); level = 0; - /* If we are in 64-bit EL2 or EL3 then there is no TTBR1, so mark it - * invalid. - */ - ttbr1_valid = (el < 2); + ttbr1_valid = regime_has_2_ranges(mmu_idx); addrsize = 64 - 8 * param.tbi; inputsize = 64 - param.tsz; } else { @@ -9724,7 +10244,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, level = 1; /* There is no TTBR1 for EL2 */ ttbr1_valid = (el != 2); - addrsize = (mmu_idx == ARMMMUIdx_S2NS ? 40 : 32); + addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32); inputsize = addrsize - param.tsz; } @@ -9775,7 +10295,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, goto do_fault; } - if (mmu_idx != ARMMMUIdx_S2NS) { + if (mmu_idx != ARMMMUIdx_Stage2) { /* The starting level depends on the virtual address size (which can * be up to 48 bits) and the translation granule size. It indicates * the number of strides (stride bits at a time) needed to @@ -9875,7 +10395,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, attrs = extract64(descriptor, 2, 10) | (extract64(descriptor, 52, 12) << 10); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { /* Stage 2 table descriptors do not include any attribute fields */ break; } @@ -9906,7 +10426,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, ap = extract32(attrs, 4, 2); xn = extract32(attrs, 12, 1); - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { ns = true; *prot = get_S2prot(env, ap, xn); } else { @@ -9933,7 +10453,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address, } if (cacheattrs != NULL) { - if (mmu_idx == ARMMMUIdx_S2NS) { + if (mmu_idx == ARMMMUIdx_Stage2) { cacheattrs->attrs = convert_stage2_attrs(env, extract32(attrs, 0, 4)); } else { @@ -9954,7 +10474,7 @@ do_fault: fi->type = fault_type; fi->level = level; /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */ - fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS); + fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2); return true; } @@ -10753,7 +11273,7 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, target_ulong *page_size, ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { - if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) { + if (mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_E10_1) { /* Call ourselves recursively to do the stage 1 and then stage 2 * translations. */ @@ -10768,13 +11288,13 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, prot, page_size, fi, cacheattrs); /* If S1 fails or S2 is disabled, return early. */ - if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) { + if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) { *phys_ptr = ipa; return ret; } /* S1 is done. Now do S2 translation. */ - ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS, + ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_Stage2, phys_ptr, attrs, &s2_prot, page_size, fi, cacheattrs != NULL ? &cacheattrs2 : NULL); @@ -10816,7 +11336,7 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, /* Fast Context Switch Extension. This doesn't exist at all in v8. * In v7 and earlier it affects all stage 1 translations. */ - if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS + if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2 && !arm_feature(env, ARM_FEATURE_V8)) { if (regime_el(env, mmu_idx) == 3) { address += env->cp15.fcseidr_s; @@ -11177,8 +11697,6 @@ uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes) int fp_exception_el(CPUARMState *env, int cur_el) { #ifndef CONFIG_USER_ONLY - int fpen; - /* CPACR and the CPTR registers don't exist before v6, so FP is * always accessible */ @@ -11206,30 +11724,34 @@ int fp_exception_el(CPUARMState *env, int cur_el) * 0, 2 : trap EL0 and EL1/PL1 accesses * 1 : trap only EL0 accesses * 3 : trap no accesses + * This register is ignored if E2H+TGE are both set. */ - fpen = extract32(env->cp15.cpacr_el1, 20, 2); - switch (fpen) { - case 0: - case 2: - if (cur_el == 0 || cur_el == 1) { - /* Trap to PL1, which might be EL1 or EL3 */ - if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + if ((arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + int fpen = extract32(env->cp15.cpacr_el1, 20, 2); + + switch (fpen) { + case 0: + case 2: + if (cur_el == 0 || cur_el == 1) { + /* Trap to PL1, which might be EL1 or EL3 */ + if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) { + return 3; + } + return 1; + } + if (cur_el == 3 && !is_a64(env)) { + /* Secure PL1 running at EL3 */ return 3; } - return 1; - } - if (cur_el == 3 && !is_a64(env)) { - /* Secure PL1 running at EL3 */ - return 3; - } - break; - case 1: - if (cur_el == 0) { - return 1; + break; + case 1: + if (cur_el == 0) { + return 1; + } + break; + case 3: + break; } - break; - case 3: - break; } /* @@ -11265,6 +11787,31 @@ int fp_exception_el(CPUARMState *env, int cur_el) return 0; } +/* Return the exception level we're running at if this is our mmu_idx */ +int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx) +{ + if (mmu_idx & ARM_MMU_IDX_M) { + return mmu_idx & ARM_MMU_IDX_M_PRIV; + } + + switch (mmu_idx) { + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_SE10_0: + return 0; + case ARMMMUIdx_E10_1: + case ARMMMUIdx_SE10_1: + return 1; + case ARMMMUIdx_E2: + case ARMMMUIdx_E20_2: + return 2; + case ARMMMUIdx_SE3: + return 3; + default: + g_assert_not_reached(); + } +} + #ifndef CONFIG_TCG ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate) { @@ -11278,10 +11825,33 @@ ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el) return arm_v7m_mmu_idx_for_secstate(env, env->v7m.secure); } - if (el < 2 && arm_is_secure_below_el3(env)) { - return ARMMMUIdx_S1SE0 + el; - } else { - return ARMMMUIdx_S12NSE0 + el; + /* See ARM pseudo-function ELIsInHost. */ + switch (el) { + case 0: + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdx_SE10_0; + } + if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE) + && arm_el_is_aa64(env, 2)) { + return ARMMMUIdx_E20_0; + } + return ARMMMUIdx_E10_0; + case 1: + if (arm_is_secure_below_el3(env)) { + return ARMMMUIdx_SE10_1; + } + return ARMMMUIdx_E10_1; + case 2: + /* TODO: ARMv8.4-SecEL2 */ + /* Note that TGE does not apply at EL2. */ + if ((env->cp15.hcr_el2 & HCR_E2H) && arm_el_is_aa64(env, 2)) { + return ARMMMUIdx_E20_2; + } + return ARMMMUIdx_E2; + case 3: + return ARMMMUIdx_SE3; + default: + g_assert_not_reached(); } } @@ -11336,11 +11906,8 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, { uint32_t flags = 0; - /* v8M always enables the fpu. */ - flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1); - if (arm_v7m_is_handler_mode(env)) { - flags = FIELD_DP32(flags, TBFLAG_A32, HANDLER, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, HANDLER, 1); } /* @@ -11351,7 +11918,7 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, if (arm_feature(env, ARM_FEATURE_V8) && !((mmu_idx & ARM_MMU_IDX_M_NEGPRI) && (env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKOFHFNMIGN_MASK))) { - flags = FIELD_DP32(flags, TBFLAG_A32, STACKCHECK, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, STACKCHECK, 1); } return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags); @@ -11394,8 +11961,8 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, flags = FIELD_DP32(flags, TBFLAG_ANY, AARCH64_STATE, 1); - /* FIXME: ARMv8.1-VHE S2 translation regime. */ - if (regime_el(env, stage1) < 2) { + /* Get control bits for tagged addresses. */ + if (regime_has_2_ranges(mmu_idx)) { ARMVAParameters p1 = aa64_va_parameters_both(env, -1, stage1); tbid = (p1.tbi << 1) | p0.tbi; tbii = tbid & ~((p1.tbid << 1) | p0.tbid); @@ -11424,7 +11991,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, flags = FIELD_DP32(flags, TBFLAG_A64, ZCR_LEN, zcr_len); } - sctlr = arm_sctlr(env, el); + sctlr = regime_sctlr(env, stage1); if (arm_cpu_data_is_big_endian_a64(el, sctlr)) { flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1); @@ -11449,6 +12016,28 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, } } + /* Compute the condition for using AccType_UNPRIV for LDTR et al. */ + /* TODO: ARMv8.2-UAO */ + switch (mmu_idx) { + case ARMMMUIdx_E10_1: + case ARMMMUIdx_SE10_1: + /* TODO: ARMv8.3-NV */ + flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + break; + case ARMMMUIdx_E20_2: + /* TODO: ARMv8.4-SecEL2 */ + /* + * Note that E20_2 is gated by HCR_EL2.E2H == 1, but E20_0 is + * gated by HCR_EL2.<E2H,TGE> == '11', and so is LDTR. + */ + if (env->cp15.hcr_el2 & HCR_TGE) { + flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + } + break; + default: + break; + } + return rebuild_hflags_common(env, fp_el, mmu_idx, flags); } @@ -11544,7 +12133,7 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, if (arm_feature(env, ARM_FEATURE_M_SECURITY) && FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S) != env->v7m.secure) { - flags = FIELD_DP32(flags, TBFLAG_A32, FPCCR_S_WRONG, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, FPCCR_S_WRONG, 1); } if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) && @@ -11556,12 +12145,12 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * active FP context; we must create a new FP context before * executing any FP insn. */ - flags = FIELD_DP32(flags, TBFLAG_A32, NEW_FP_CTXT_NEEDED, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 1); } bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK; if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) { - flags = FIELD_DP32(flags, TBFLAG_A32, LSPACT, 1); + flags = FIELD_DP32(flags, TBFLAG_M32, LSPACT, 1); } } else { /* @@ -11582,8 +12171,8 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, } } - flags = FIELD_DP32(flags, TBFLAG_A32, THUMB, env->thumb); - flags = FIELD_DP32(flags, TBFLAG_A32, CONDEXEC, env->condexec_bits); + flags = FIELD_DP32(flags, TBFLAG_AM32, THUMB, env->thumb); + flags = FIELD_DP32(flags, TBFLAG_AM32, CONDEXEC, env->condexec_bits); pstate_for_ss = env->uncached_cpsr; } diff --git a/target/arm/internals.h b/target/arm/internals.h index f5313dd3d4..6d4a942bde 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -769,6 +769,39 @@ bool arm_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr); +static inline int arm_to_core_mmu_idx(ARMMMUIdx mmu_idx) +{ + return mmu_idx & ARM_MMU_IDX_COREIDX_MASK; +} + +static inline ARMMMUIdx core_to_arm_mmu_idx(CPUARMState *env, int mmu_idx) +{ + if (arm_feature(env, ARM_FEATURE_M)) { + return mmu_idx | ARM_MMU_IDX_M; + } else { + return mmu_idx | ARM_MMU_IDX_A; + } +} + +int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx); + +/* + * Return the MMU index for a v7M CPU with all relevant information + * manually specified. + */ +ARMMMUIdx arm_v7m_mmu_idx_all(CPUARMState *env, + bool secstate, bool priv, bool negpri); + +/* + * Return the MMU index for a v7M CPU in the specified security and + * privilege state. + */ +ARMMMUIdx arm_v7m_mmu_idx_for_secstate_and_priv(CPUARMState *env, + bool secstate, bool priv); + +/* Return the MMU index for a v7M CPU in the specified security state */ +ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate); + /* Return true if the stage 1 translation regime is using LPAE format page * tables */ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx); @@ -804,24 +837,44 @@ static inline void arm_call_el_change_hook(ARMCPU *cpu) } } +/* Return true if this address translation regime has two ranges. */ +static inline bool regime_has_2_ranges(ARMMMUIdx mmu_idx) +{ + switch (mmu_idx) { + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_E1: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_SE10_1: + return true; + default: + return false; + } +} + /* Return true if this address translation regime is secure */ static inline bool regime_is_secure(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { - case ARMMMUIdx_S12NSE0: - case ARMMMUIdx_S12NSE1: - case ARMMMUIdx_S1NSE0: - case ARMMMUIdx_S1NSE1: - case ARMMMUIdx_S1E2: - case ARMMMUIdx_S2NS: + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_E20_2: + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_E1: + case ARMMMUIdx_E2: + case ARMMMUIdx_Stage2: case ARMMMUIdx_MPrivNegPri: case ARMMMUIdx_MUserNegPri: case ARMMMUIdx_MPriv: case ARMMMUIdx_MUser: return false; - case ARMMMUIdx_S1E3: - case ARMMMUIdx_S1SE0: - case ARMMMUIdx_S1SE1: + case ARMMMUIdx_SE3: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_SE10_1: case ARMMMUIdx_MSPrivNegPri: case ARMMMUIdx_MSUserNegPri: case ARMMMUIdx_MSPriv: @@ -975,7 +1028,7 @@ ARMMMUIdx arm_mmu_idx(CPUARMState *env); #ifdef CONFIG_USER_ONLY static inline ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) { - return ARMMMUIdx_S1NSE0; + return ARMMMUIdx_Stage1_E0; } #else ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env); diff --git a/target/arm/monitor.c b/target/arm/monitor.c index 9725dfff16..c2dc7908de 100644 --- a/target/arm/monitor.c +++ b/target/arm/monitor.c @@ -137,17 +137,20 @@ CpuModelExpansionInfo *qmp_query_cpu_model_expansion(CpuModelExpansionType type, } if (kvm_enabled()) { - const char *cpu_type = current_machine->cpu_type; - int len = strlen(cpu_type) - strlen(ARM_CPU_TYPE_SUFFIX); bool supported = false; if (!strcmp(model->name, "host") || !strcmp(model->name, "max")) { /* These are kvmarm's recommended cpu types */ supported = true; - } else if (strlen(model->name) == len && - !strncmp(model->name, cpu_type, len)) { - /* KVM is enabled and we're using this type, so it works. */ - supported = true; + } else if (current_machine->cpu_type) { + const char *cpu_type = current_machine->cpu_type; + int len = strlen(cpu_type) - strlen(ARM_CPU_TYPE_SUFFIX); + + if (strlen(model->name) == len && + !strncmp(model->name, cpu_type, len)) { + /* KVM is enabled and we're using this type, so it works. */ + supported = true; + } } if (!supported) { error_setg(errp, "We cannot guarantee the CPU type '%s' works " diff --git a/target/arm/pauth_helper.c b/target/arm/pauth_helper.c index 0a5f41e10c..9746e32bf8 100644 --- a/target/arm/pauth_helper.c +++ b/target/arm/pauth_helper.c @@ -371,7 +371,10 @@ static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra) if (el < 2 && arm_feature(env, ARM_FEATURE_EL2)) { uint64_t hcr = arm_hcr_el2_eff(env); bool trap = !(hcr & HCR_API); - /* FIXME: ARMv8.1-VHE: trap only applies to EL1&0 regime. */ + if (el == 0) { + /* Trap only applies to EL1&0 regime. */ + trap &= (hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE); + } /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */ if (trap) { pauth_trap(env, 2, ra); @@ -386,14 +389,7 @@ static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra) static bool pauth_key_enabled(CPUARMState *env, int el, uint32_t bit) { - uint32_t sctlr; - if (el == 0) { - /* FIXME: ARMv8.1-VHE S2 translation regime. */ - sctlr = env->cp15.sctlr_el[1]; - } else { - sctlr = env->cp15.sctlr_el[el]; - } - return (sctlr & bit) != 0; + return (arm_sctlr(env, el) & bit) != 0; } uint64_t HELPER(pacia)(CPUARMState *env, uint64_t x, uint64_t y) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index 766a03335b..6e82486884 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -105,25 +105,36 @@ void a64_translate_init(void) offsetof(CPUARMState, exclusive_high), "exclusive_high"); } -static inline int get_a64_user_mem_index(DisasContext *s) +/* + * Return the core mmu_idx to use for A64 "unprivileged load/store" insns + */ +static int get_a64_user_mem_index(DisasContext *s) { - /* Return the core mmu_idx to use for A64 "unprivileged load/store" insns: - * if EL1, access as if EL0; otherwise access at current EL + /* + * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL, + * which is the usual mmu_idx for this cpu state. */ - ARMMMUIdx useridx; + ARMMMUIdx useridx = s->mmu_idx; - switch (s->mmu_idx) { - case ARMMMUIdx_S12NSE1: - useridx = ARMMMUIdx_S12NSE0; - break; - case ARMMMUIdx_S1SE1: - useridx = ARMMMUIdx_S1SE0; - break; - case ARMMMUIdx_S2NS: - g_assert_not_reached(); - default: - useridx = s->mmu_idx; - break; + if (s->unpriv) { + /* + * We have pre-computed the condition for AccType_UNPRIV. + * Therefore we should never get here with a mmu_idx for + * which we do not know the corresponding user mmu_idx. + */ + switch (useridx) { + case ARMMMUIdx_E10_1: + useridx = ARMMMUIdx_E10_0; + break; + case ARMMMUIdx_E20_2: + useridx = ARMMMUIdx_E20_0; + break; + case ARMMMUIdx_SE10_1: + useridx = ARMMMUIdx_SE10_0; + break; + default: + g_assert_not_reached(); + } } return arm_to_core_mmu_idx(useridx); } @@ -175,8 +186,7 @@ static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst, if (tbi == 0) { /* Load unmodified address */ tcg_gen_mov_i64(dst, src); - } else if (s->current_el >= 2) { - /* FIXME: ARMv8.1-VHE S2 translation regime. */ + } else if (!regime_has_2_ranges(s->mmu_idx)) { /* Force tag byte to all zero */ tcg_gen_extract_i64(dst, src, 0, 56); } else { @@ -14172,6 +14182,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->pauth_active = FIELD_EX32(tb_flags, TBFLAG_A64, PAUTH_ACTIVE); dc->bt = FIELD_EX32(tb_flags, TBFLAG_A64, BT); dc->btype = FIELD_EX32(tb_flags, TBFLAG_A64, BTYPE); + dc->unpriv = FIELD_EX32(tb_flags, TBFLAG_A64, UNPRIV); dc->vec_len = 0; dc->vec_stride = 0; dc->cp_regs = arm_cpu->cp_regs; diff --git a/target/arm/translate.c b/target/arm/translate.c index 2f4aea927f..e11a5871d0 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -152,14 +152,14 @@ static inline int get_a32_user_mem_index(DisasContext *s) * otherwise, access as if at PL0. */ switch (s->mmu_idx) { - case ARMMMUIdx_S1E2: /* this one is UNPREDICTABLE */ - case ARMMMUIdx_S12NSE0: - case ARMMMUIdx_S12NSE1: - return arm_to_core_mmu_idx(ARMMMUIdx_S12NSE0); - case ARMMMUIdx_S1E3: - case ARMMMUIdx_S1SE0: - case ARMMMUIdx_S1SE1: - return arm_to_core_mmu_idx(ARMMMUIdx_S1SE0); + case ARMMMUIdx_E2: /* this one is UNPREDICTABLE */ + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + return arm_to_core_mmu_idx(ARMMMUIdx_E10_0); + case ARMMMUIdx_SE3: + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_SE10_1: + return arm_to_core_mmu_idx(ARMMMUIdx_SE10_0); case ARMMMUIdx_MUser: case ARMMMUIdx_MPriv: return arm_to_core_mmu_idx(ARMMMUIdx_MUser); @@ -172,7 +172,6 @@ static inline int get_a32_user_mem_index(DisasContext *s) case ARMMMUIdx_MSUserNegPri: case ARMMMUIdx_MSPrivNegPri: return arm_to_core_mmu_idx(ARMMMUIdx_MSUserNegPri); - case ARMMMUIdx_S2NS: default: g_assert_not_reached(); } @@ -10848,38 +10847,48 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) */ dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3); - dc->thumb = FIELD_EX32(tb_flags, TBFLAG_A32, THUMB); - dc->sctlr_b = FIELD_EX32(tb_flags, TBFLAG_A32, SCTLR_B); - dc->hstr_active = FIELD_EX32(tb_flags, TBFLAG_A32, HSTR_ACTIVE); + dc->thumb = FIELD_EX32(tb_flags, TBFLAG_AM32, THUMB); dc->be_data = FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; - condexec = FIELD_EX32(tb_flags, TBFLAG_A32, CONDEXEC); + condexec = FIELD_EX32(tb_flags, TBFLAG_AM32, CONDEXEC); dc->condexec_mask = (condexec & 0xf) << 1; dc->condexec_cond = condexec >> 4; + core_mmu_idx = FIELD_EX32(tb_flags, TBFLAG_ANY, MMUIDX); dc->mmu_idx = core_to_arm_mmu_idx(env, core_mmu_idx); dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); #if !defined(CONFIG_USER_ONLY) dc->user = (dc->current_el == 0); #endif - dc->ns = FIELD_EX32(tb_flags, TBFLAG_A32, NS); dc->fp_excp_el = FIELD_EX32(tb_flags, TBFLAG_ANY, FPEXC_EL); - dc->vfp_enabled = FIELD_EX32(tb_flags, TBFLAG_A32, VFPEN); - dc->vec_len = FIELD_EX32(tb_flags, TBFLAG_A32, VECLEN); - if (arm_feature(env, ARM_FEATURE_XSCALE)) { - dc->c15_cpar = FIELD_EX32(tb_flags, TBFLAG_A32, XSCALE_CPAR); - dc->vec_stride = 0; + + if (arm_feature(env, ARM_FEATURE_M)) { + dc->vfp_enabled = 1; + dc->be_data = MO_TE; + dc->v7m_handler_mode = FIELD_EX32(tb_flags, TBFLAG_M32, HANDLER); + dc->v8m_secure = arm_feature(env, ARM_FEATURE_M_SECURITY) && + regime_is_secure(env, dc->mmu_idx); + dc->v8m_stackcheck = FIELD_EX32(tb_flags, TBFLAG_M32, STACKCHECK); + dc->v8m_fpccr_s_wrong = + FIELD_EX32(tb_flags, TBFLAG_M32, FPCCR_S_WRONG); + dc->v7m_new_fp_ctxt_needed = + FIELD_EX32(tb_flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED); + dc->v7m_lspact = FIELD_EX32(tb_flags, TBFLAG_M32, LSPACT); } else { - dc->vec_stride = FIELD_EX32(tb_flags, TBFLAG_A32, VECSTRIDE); - dc->c15_cpar = 0; - } - dc->v7m_handler_mode = FIELD_EX32(tb_flags, TBFLAG_A32, HANDLER); - dc->v8m_secure = arm_feature(env, ARM_FEATURE_M_SECURITY) && - regime_is_secure(env, dc->mmu_idx); - dc->v8m_stackcheck = FIELD_EX32(tb_flags, TBFLAG_A32, STACKCHECK); - dc->v8m_fpccr_s_wrong = FIELD_EX32(tb_flags, TBFLAG_A32, FPCCR_S_WRONG); - dc->v7m_new_fp_ctxt_needed = - FIELD_EX32(tb_flags, TBFLAG_A32, NEW_FP_CTXT_NEEDED); - dc->v7m_lspact = FIELD_EX32(tb_flags, TBFLAG_A32, LSPACT); + dc->be_data = + FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; + dc->debug_target_el = + FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); + dc->sctlr_b = FIELD_EX32(tb_flags, TBFLAG_A32, SCTLR_B); + dc->hstr_active = FIELD_EX32(tb_flags, TBFLAG_A32, HSTR_ACTIVE); + dc->ns = FIELD_EX32(tb_flags, TBFLAG_A32, NS); + dc->vfp_enabled = FIELD_EX32(tb_flags, TBFLAG_A32, VFPEN); + if (arm_feature(env, ARM_FEATURE_XSCALE)) { + dc->c15_cpar = FIELD_EX32(tb_flags, TBFLAG_A32, XSCALE_CPAR); + } else { + dc->vec_len = FIELD_EX32(tb_flags, TBFLAG_A32, VECLEN); + dc->vec_stride = FIELD_EX32(tb_flags, TBFLAG_A32, VECSTRIDE); + } + } dc->cp_regs = cpu->cp_regs; dc->features = env->features; @@ -10901,9 +10910,6 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) dc->ss_active = FIELD_EX32(tb_flags, TBFLAG_ANY, SS_ACTIVE); dc->pstate_ss = FIELD_EX32(tb_flags, TBFLAG_ANY, PSTATE_SS); dc->is_ldex = false; - if (!arm_feature(env, ARM_FEATURE_M)) { - dc->debug_target_el = FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); - } dc->page_start = dc->base.pc_first & TARGET_PAGE_MASK; @@ -11340,10 +11346,10 @@ static const TranslatorOps thumb_translator_ops = { /* generate intermediate code for basic block 'tb'. */ void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) { - DisasContext dc; + DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; - if (FIELD_EX32(tb->flags, TBFLAG_A32, THUMB)) { + if (FIELD_EX32(tb->flags, TBFLAG_AM32, THUMB)) { ops = &thumb_translator_ops; } #ifdef TARGET_AARCH64 diff --git a/target/arm/translate.h b/target/arm/translate.h index b837b7fcbf..5b167c416a 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -73,6 +73,8 @@ typedef struct DisasContext { * ie A64 LDX*, LDAX*, A32/T32 LDREX*, LDAEX*. */ bool is_ldex; + /* True if AccType_UNPRIV should be used for LDTR et al */ + bool unpriv; /* True if v8.3-PAuth is active. */ bool pauth_active; /* True with v8.5-BTI and SCTLR_ELx.BT* set. */ @@ -126,7 +128,7 @@ static inline int default_exception_el(DisasContext *s) * exceptions can only be routed to ELs above 1, so we target the higher of * 1 or the current EL. */ - return (s->mmu_idx == ARMMMUIdx_S1SE0 && s->secure_routed_to_el3) + return (s->mmu_idx == ARMMMUIdx_SE10_0 && s->secure_routed_to_el3) ? 3 : MAX(1, s->current_el); } diff --git a/tests/qemu-iotests/223 b/tests/qemu-iotests/223 index ea69cd4b8b..56fbc5fb09 100755 --- a/tests/qemu-iotests/223 +++ b/tests/qemu-iotests/223 @@ -153,7 +153,7 @@ _send_qemu_cmd $QEMU_HANDLE '{"execute":"nbd-server-add", "bitmap":"b3"}}' "error" # Missing bitmap _send_qemu_cmd $QEMU_HANDLE '{"execute":"nbd-server-add", "arguments":{"device":"n", "name":"n2", "writable":true, - "bitmap":"b2"}}' "return" + "description":"some text", "bitmap":"b2"}}' "return" $QEMU_NBD_PROG -L -k "$SOCK_DIR/nbd" echo diff --git a/tests/qemu-iotests/223.out b/tests/qemu-iotests/223.out index f175598802..80c0cf6509 100644 --- a/tests/qemu-iotests/223.out +++ b/tests/qemu-iotests/223.out @@ -50,7 +50,7 @@ exports available: 0 {"error": {"class": "GenericError", "desc": "Enabled bitmap 'b2' incompatible with readonly export"}} {"execute":"nbd-server-add", "arguments":{"device":"n", "name":"n2", "bitmap":"b3"}} {"error": {"class": "GenericError", "desc": "Bitmap 'b3' is not found"}} -{"execute":"nbd-server-add", "arguments":{"device":"n", "name":"n2", "writable":true, "bitmap":"b2"}} +{"execute":"nbd-server-add", "arguments":{"device":"n", "name":"n2", "writable":true, "description":"some text", "bitmap":"b2"}} {"return": {}} exports available: 2 export: 'n' @@ -63,6 +63,7 @@ exports available: 2 base:allocation qemu:dirty-bitmap:b export: 'n2' + description: some text size: 4194304 flags: 0xced ( flush fua trim zeroes df cache fast-zero ) min block: 1 @@ -130,7 +131,7 @@ exports available: 0 {"error": {"class": "GenericError", "desc": "Enabled bitmap 'b2' incompatible with readonly export"}} {"execute":"nbd-server-add", "arguments":{"device":"n", "name":"n2", "bitmap":"b3"}} {"error": {"class": "GenericError", "desc": "Bitmap 'b3' is not found"}} -{"execute":"nbd-server-add", "arguments":{"device":"n", "name":"n2", "writable":true, "bitmap":"b2"}} +{"execute":"nbd-server-add", "arguments":{"device":"n", "name":"n2", "writable":true, "description":"some text", "bitmap":"b2"}} {"return": {}} exports available: 2 export: 'n' @@ -143,6 +144,7 @@ exports available: 2 base:allocation qemu:dirty-bitmap:b export: 'n2' + description: some text size: 4194304 flags: 0xced ( flush fua trim zeroes df cache fast-zero ) min block: 1 |