ati-vga: Add limited support for big endian frame buffer aperture

Set frame buffer endianness according to requested endianness for
frame buffer apertures. We set frame buffer to big endian if any of
the two apertures are set to big endian. Using different endianness
for the two apertures is not implemented. This fixes inverted colors
with MacOS and Xorg frame buffer driver but some Linux drivers may
have endianness issues even on real hardware so this may not fix all
cases. MorphOS uses aper0 in LE, Linux uses aper0 in BE and MacOS uses
aper1 in BE but not sure about others or if MacOS also may need aper0
in which case we'll need a more complex fix but MacOS has other
problems yet so for now this might work.

Signed-off-by: BALATON Zoltan <balaton@eik.bme.hu>
Message-id: e2a7ec7af3fc30523213bcd27832ccad34323f2c.1565558093.git.balaton@eik.bme.hu
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>

1 files changed, 9 insertions, 1 deletions

diff --git a/hw/display/ati.c b/hw/display/ati.c
index f24c23fa89..cceb45b34a 100644
--- a/hw/display/ati.c
+++ b/hw/display/ati.c
@@ -90,7 +90,9 @@ static void ati_vga_switch_mode(ATIVGAState *s)
             DPRINTF("Switching to %dx%d %d %d @ %x\n", h, v, stride, bpp, offs);
             vbe_ioport_write_index(&s->vga, 0, VBE_DISPI_INDEX_ENABLE);
             vbe_ioport_write_data(&s->vga, 0, VBE_DISPI_DISABLED);
-            s->vga.big_endian_fb = false;
+            s->vga.big_endian_fb = (s->regs.config_cntl & APER_0_ENDIAN ||
+                                    s->regs.config_cntl & APER_1_ENDIAN ?
+                                    true : false);
             /* reset VBE regs then set up mode */
             s->vga.vbe_regs[VBE_DISPI_INDEX_XRES] = h;
             s->vga.vbe_regs[VBE_DISPI_INDEX_YRES] = v;
@@ -310,6 +312,9 @@ static uint64_t ati_mm_read(void *opaque, hwaddr addr, unsigned int size)
     case PALETTE_DATA:
         val = vga_ioport_read(&s->vga, VGA_PEL_D);
         break;
+    case CNFG_CNTL:
+        val = s->regs.config_cntl;
+        break;
     case CNFG_MEMSIZE:
         val = s->vga.vram_size;
         break;
@@ -604,6 +609,9 @@ static void ati_mm_write(void *opaque, hwaddr addr,
         data >>= 8;
         vga_ioport_write(&s->vga, VGA_PEL_D, data & 0xff);
         break;
+    case CNFG_CNTL:
+        s->regs.config_cntl = data;
+        break;
     case CRTC_H_TOTAL_DISP:
         s->regs.crtc_h_total_disp = data & 0x07ff07ff;
         break;