-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--
-- Copyright 2025 VLSI Solution Oy
-- SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
--
-- Licensed under the Solderpad Hardware License v 2.1 (the "License");
-- you may not use this file except in compliance with the License, or,
-- at your option, the Apache License version 2.0.
-- You may obtain a copy of the License at
--
--    https://solderpad.org/licenses/SHL-2.1/
--
-- Unless required by applicable law or agreed to in writing, any work
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--                               RISCV core
--                                 V1.1
--
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

-- - - - - - - -
--  libraries
-- - - - - - - -
library ieee;
use ieee.std_logic_1164.all;
use ieee.numeric_std.all;
use work.rv_pkg.all;


entity rv_amo is
  port ( clk_i          : in  std_logic;
         xrst_i         : in  std_logic;   -- sync reset
         -- cpu port
         mmu_rd_i       : in  std_logic;   -- page adress fetch
         amofunc_i      : in  slv_03_00;   -- AMO func  (no LR/SC)
         amo_pending_o  : out std_logic;
         --
         rd_i           : in  std_logic;
         wr_i           : in  slv_03_00;   -- byte access
         addr_i         : in  slv_31_00;   -- access address
         wdata_i        : in  slv_31_00;   -- write data
         --
         accept_o       : out std_logic;   -- read/write accept to cpu
         ack_o          : out std_logic;   -- rd/wr ack to cpu
         rdata_o        : out slv_31_00;   -- read data, valid if ack=1
         err_o          : out std_logic;   -- set if mem access error
         -- system port
         sys_accept_i   : in  std_logic;   -- read/write accept from mem
         sys_ack_i      : in  std_logic;   -- read/write ack from mem
         sys_rdata_i    : in  slv_31_00;   -- rdata from dmem, valid if ack=1
         sys_err_i      : in  std_logic;   -- memory access error
         --
         sys_rd_o       : out std_logic;   -- request strobe, single-shot
         sys_wr_o       : out slv_03_00;   -- byte enable
         sys_addr_o     : out slv_31_00;   -- access address
         sys_wdata_o    : out slv_31_00    -- write data
         );
end rv_amo;

architecture rtl of rv_amo is

  constant WB_ACK    : slv_02_00 := "101";
  constant WB_WAIT   : slv_02_00 := "100";
  constant WRITEBACK : slv_02_00 := "011";
  constant MODIFY    : slv_02_00 := "010";
  constant READ      : slv_02_00 := "001";
  constant IDLE      : slv_02_00 := "000";

  signal state_w     : slv_02_00;
  signal cmd_w       : slv_03_00;
  signal rdata_w     : slv_31_00;
  signal wdata_w     : slv_31_00;
  signal waddr_w     : slv_31_00;

  signal state_q     : slv_02_00;
  signal cmd_q       : slv_03_00;
  signal rdata_q     : slv_31_00;
  signal wdata_q     : slv_31_00;
  signal waddr_q     : slv_31_00;

  signal sys_err_q   : std_logic;

  -- internal data ALU
  signal alu_wb_w    : slv_31_00;

  -- comparator
  signal cmp_opa_w   : slv_32_00;
  signal cmp_opb_w   : slv_32_00;
  signal cmp_less_w  : std_logic;
  signal cmp_res_w   : slv_31_00;

  signal amo_w       : std_logic;          -- AMO op   (no LR/SC)

begin

  p_st: process(clk_i)
  begin
    if rising_edge(clk_i) then
      if (xrst_i = '0') then
        state_q      <= IDLE;
        cmd_q        <= (others => '0');
        rdata_q      <= (others => '0');
        wdata_q      <= (others => '0');
        waddr_q      <= (others => '0');
        sys_err_q    <= '0';
      else
        state_q      <= state_w;
        cmd_q        <= cmd_w;
        rdata_q      <= rdata_w;
        wdata_q      <= wdata_w;
        waddr_q      <= waddr_w;
        sys_err_q    <= '0';
        if (state_q = READ or state_q = WB_WAIT) then
          sys_err_q  <= sys_err_i;
        end if;
      end if;
    end if;
  end process p_st;


  p_c_st : process( state_q, cmd_q, rdata_q, wdata_q, waddr_q, amo_w,
                    sys_accept_i, amofunc_i, wdata_i, addr_i, sys_rdata_i,
                    sys_err_i, sys_ack_i, alu_wb_w )
  begin
    state_w        <= state_q;
    cmd_w          <= cmd_q;
    rdata_w        <= rdata_q;
    wdata_w        <= wdata_q;
    waddr_w        <= waddr_q;
    case state_q is
      when IDLE =>       -- wait for RMW request, pass through regular ld/st
        if (amo_w = '1' and sys_accept_i = '1') then  -- NO lr/sc
          cmd_w    <= amofunc_i;
          wdata_w  <= wdata_i;
          waddr_w  <= addr_i;
          state_w  <= READ;
        end if;
      when READ =>                         -- wait for read-access to complete
        rdata_w    <= sys_rdata_i;
        if (sys_err_i = '1'and sys_ack_i = '1') then  -- abort if error
          state_w  <= WB_ACK;
        elsif (sys_ack_i = '1') then
          state_w  <= MODIFY;
        end if;
      when MODIFY =>                       -- execute atomic data operation
        state_w    <= WRITEBACK;
        wdata_w    <= alu_wb_w;
      when WRITEBACK =>                   -- write operation result
        if (sys_accept_i = '1') then
          state_w  <= WB_WAIT;
        end if;
      when WB_WAIT =>                     -- wait for write-access to complete
        if (sys_err_i = '1' and sys_ack_i = '1') then
          state_w  <= WB_ACK;
        elsif (sys_ack_i = '1') then
          state_w  <= WB_ACK;
        end if;
      when WB_ACK =>
        state_w    <= IDLE;
      when others =>                      -- undefined
        state_w    <= IDLE;
    end case;
  end process p_c_st;



  amo_w <= not_zero(amofunc_i) and (not mmu_rd_i)     and    rd_i;

  -- request to dmem
  sys_rd_o      <= rd_i        when ( state_q = IDLE )      else '0';
  sys_wr_o      <= "1111"      when ( state_q = WRITEBACK ) else wr_i;
  sys_addr_o    <= waddr_q     when ( state_q = WRITEBACK ) else addr_i;
  sys_wdata_o   <= wdata_q     when ( state_q = WRITEBACK ) else wdata_i;

  -- response from dmem
  accept_o      <= '0'         when ( state_q /= IDLE)      else sys_accept_i;
  rdata_o       <= sys_rdata_i when ( state_q = IDLE)       else rdata_q;
  ack_o         <= sys_ack_i   when ( state_q = IDLE)       else
                   '1'         when ( state_q = WB_ACK )    else
                   '0';
  err_o         <= sys_err_q   when ( state_q = WB_ACK )    else sys_err_i ;


  amo_pending_o <= '1' when (state_q /= IDLE) else '0';


  -- AMO ALU
  p_c_amo_alu : process(cmd_q, wdata_q, rdata_q, cmp_res_w)
  begin
    alu_wb_w <= X"00000000";
    case cmd_q is
      when "0001"   => alu_wb_w <= wdata_q;                   -- AMOSWAP.w
      when "0010"   => alu_wb_w <= addu(rdata_q, wdata_q);    -- AMOADD.w
      when "0011"   => alu_wb_w <= rdata_q xor wdata_q;       -- AMOXOR.w
      when "0100"   => alu_wb_w <= rdata_q and wdata_q;       -- AMOAND.w
      when "0101"   => alu_wb_w <= rdata_q or  wdata_q;       -- AMOOR.w
      when "0110" |                                           -- AMOMIN.w
           "0111" |                                           -- AMOMAX.w
           "1000" |                                           -- AMOMINU.w
           "1001"   => alu_wb_w <= cmp_res_w;                 -- AMOMAXU.w
      when others   => alu_wb_w <= X"00000000";               -- NULL
    end case;
  end process p_c_amo_alu;


  -- comparator logic (min/max and signed/unsigned)
  -- sign-extend if signed operation
  cmp_opa_w  <= (rdata_q(rdata_q'left) and (not cmd_q(3))) & rdata_q;
  cmp_opb_w  <= (wdata_q(wdata_q'left) and (not cmd_q(3))) & wdata_q;
  cmp_less_w <= lt_signed(cmp_opa_w, cmp_opb_w);
  cmp_res_w  <= cmp_opa_w(31 downto 0) when ((cmp_less_w xor cmd_q(0)) = '1')
                else cmp_opb_w(31 downto 0);


end rtl;




-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--                               RISCV core
--                                 V1.1
--
-- Changes:
--
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

-- - - - - - - -
--  libraries
-- - - - - - - -
library IEEE;
use IEEE.std_logic_1164.all;
use IEEE.numeric_std.all;
use work.rv_pkg.all;


entity rv_lrsc is
  port ( clk_i        : in  std_logic;
         xrst_i       : in  std_logic;
         -- cpu port
         rd_i         : in  std_logic;   -- memory read     (cpu)
         wr_i         : in  slv_03_00;   -- memory write    (cpu)
         addr_i       : in  slv_31_00;   -- physical memory address
         wdata_i      : in  slv_31_00;   -- NC                ! LOOPBACK ONLY !
         --
         accept_o     : out std_logic;   -- accept  rd / wr (cpu)
         ack_o        : out std_logic;   -- ack for rd / wr (cpu)
         rdata_o      : out slv_31_00;   -- NC                ! LOOPBACK ONLY !
         err_o        : out std_logic;   -- mem err         (cpu)
         -- system port
         sys_accept_i : in  std_logic;   -- accept  rd / wr (dmem)
         sys_ack_i    : in  std_logic;   -- ack for rd / wr (dmem)
         sys_rdata_i  : in  slv_31_00;   -- NC                ! LOOPBACK ONLY !
         sys_err_i    : in  std_logic;   -- mem err         (dmem)
         --
         sys_rd_o     : out std_logic;
         sys_wr_o     : out slv_03_00;
         sys_addr_o   : out slv_31_00;   -- access address
         sys_wdata_o  : out slv_31_00;   -- NC write data      ! LOOPBACK ONLY !
         --
         mmu_rd_i     : in  std_logic;   -- page adress fetch
         atomic_i     : in  std_logic;   -- atomic memory operaiton lr/sc
         lr_cancel_i  : in  std_logic    -- irq taken, context change
         );
end rv_lrsc;

architecture rtl of rv_lrsc is

  -- handle atomic load reserved / store conditional
  signal atomic_w       : std_logic;
  signal lr_op_w        : std_logic;
  signal sc_op_w        : std_logic;

  signal lr_vld_q       : std_logic;
  signal lr_paddr_q     : slv_31_02;    -- physical address

  signal waddr_match_w  : std_logic;
  signal squash_sc_w    : std_logic;
  signal sc_vld_q       : std_logic;
  signal ack_q          : std_logic;

begin

  sys_rd_o       <= rd_i;
  sys_wr_o       <= "0000" when (squash_sc_w = '1') else wr_i;

  rdata_o        <= (0 => ack_q, others => '0') when (sc_vld_q = '1') else
                    sys_rdata_i;

  sys_addr_o     <= addr_i;       -- ! LOOPBACK ONLY !
  sys_wdata_o    <= wdata_i;      -- ! LOOPBACK ONLY !

  accept_o       <= sys_accept_i;
  ack_o          <= sys_ack_i or ack_q;
  err_o          <= sys_err_i;

  atomic_w       <= atomic_i  and (not mmu_rd_i);
  lr_op_w        <= atomic_w and rd_i;
  sc_op_w        <= atomic_w and not_zero(wr_i);

  waddr_match_w  <= lr_vld_q and eq(addr_i(31 downto 2), lr_paddr_q);

  squash_sc_w    <= sc_op_w and ((not lr_vld_q) or (not waddr_match_w));


  -- reservation set-clear op
  rvso_ctrl : process(clk_i)
  begin
    if rising_edge(clk_i) then
      if (xrst_i = '0') then
        lr_vld_q    <= '0';
        lr_paddr_q  <= (others => '0');
        sc_vld_q    <= '0';
        ack_q       <= '0';
      else
        sc_vld_q    <= '0';
        ack_q       <= '0';
        --
        -- execute SC
        if (sc_op_w = '1' and sys_accept_i = '1') then
          lr_vld_q    <= '0';
          lr_paddr_q  <= (others => '0');
          sc_vld_q    <= '1';
          ack_q       <= squash_sc_w;  -- generate local ack when mem op skipped
          --
          -- non-atomic load/store, invalidate reservation
        elsif ( (rd_i = '1' or not_zero(wr_i) ) and sys_accept_i = '1') then
          lr_vld_q    <= '0';
          lr_paddr_q  <= (others => '0');
        end if;
        --
        -- active reservation: wait for condition to invalidate reservation
        if (lr_op_w = '1' and sys_accept_i = '1') then
          lr_vld_q    <= '1';
          lr_paddr_q  <= addr_i(31 downto 2);
        end if;
        --
        --
        if ( (sys_err_i = '1' and sys_ack_i = '1') or lr_cancel_i = '1' ) then
          lr_vld_q    <= '0';          -- mem error / irq taken
          sc_vld_q    <= '0';
          lr_paddr_q  <= (others => '0');
        end if;
      end if;
    end if;
  end process rvso_ctrl;



end rtl;




-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--                               RISCV core
--                                 V1.1
--
-- Changes:
--
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

-- - - - - - - -
--  libraries
-- - - - - - - -
library IEEE;
use IEEE.std_logic_1164.all;
use IEEE.numeric_std.all;
use work.rv_pkg.all;


entity rv_mmu is
  generic ( CACHE_ADDR_L   : slv_31_00 := X"80000000";
            CACHE_ADDR_H   : slv_31_00 := X"8FFFFFFF";
            ITLB_ENTRIES   : natural range 1 to 32 := 2;
            DTLB_ENTRIES   : natural range 1 to 32 := 4;
            SUPPORT_MMU    : std_logic := '1' );
  port ( clk_i                  : in  std_logic;
         xrst_i                 : in  std_logic;  -- syncr reset
         --  inputs
         priv_d_i               : in  slv_01_00;
         sum_i                  : in  std_logic;  -- supervisor user mem access
         mxr_i                  : in  std_logic;  -- read-only mem is exscutable
         flush_i                : in  std_logic;
         satp_i                 : in  slv_31_00;  -- NC 30:20
         --
         -- atomic_lrsc_i       : in  std_logic;
         ifence_p1_i            : in  std_logic;  -- ifence instr
         imem_pc_o              : out slv_31_00;
         lr_cancel_i            : in  std_logic;  -- irq taken, context change
         amofunc_i              : in  slv_03_00;
         amo_pending_o          : out std_logic;
         --
         fetch_in_priv_i        : in  slv_01_00;
         fetch_in_rd_i          : in  std_logic;
         fetch_in_pc_i          : in  slv_31_00;
         fetch_in_flush_i       : in  std_logic;
         fetch_in_invalidate_i  : in  std_logic;
         fetch_in_accept_o      : out std_logic;
         fetch_in_valid_o       : out std_logic;
         fetch_in_error_o       : out std_logic;
         fetch_in_fault_o       : out std_logic;
         fetch_out_accept_i     : in  std_logic;
         fetch_out_valid_i      : in  std_logic;
         fetch_out_error_i      : in  std_logic;
         fetch_out_rd_o         : out std_logic;
         fetch_out_flush_o      : out std_logic;
         fetch_out_invalidate_o : out std_logic;
         --
         lsu_atomic_i           : in  std_logic;  -- atomic op
         lsu_in_cacheable_i     : in  std_logic;  -- NC in MMU ver
         lsu_in_req_tag_i       : in  slv_10_00;
         --
         lsu_in_rd_i            : in  std_logic;
         lsu_in_wr_i            : in  slv_03_00;
         lsu_in_addr_i          : in  slv_31_00;
         lsu_in_wdata_i         : in  slv_31_00;
         lsu_in_invalidate_i    : in  std_logic;
         lsu_in_flush_i         : in  std_logic;
         lsu_in_wb_i            : in  std_logic;
         lsu_in_accept_o        : out std_logic;
         lsu_in_ack_o           : out std_logic;
         lsu_in_error_o         : out std_logic;
         lsu_in_load_fault_o    : out std_logic;
         lsu_in_store_fault_o   : out std_logic;
         --
         dmem_cacheable_o       : out std_logic;
         dmem_resp_tag_i        : in  slv_10_00;
         dmem_req_tag_o         : out slv_10_00;
         --
         dmem_accept_i          : in  std_logic;  -- dcache accept
         dmem_ack_i             : in  std_logic;
         dmem_rdata_i           : in  slv_31_00;  -- dcache / perip
         dmem_error_i           : in  std_logic;
         dmem_rd_o              : out std_logic;
         dmem_wr_o              : out slv_03_00;
         dmem_addr_o            : out slv_31_00;
         dmem_wdata_o           : out slv_31_00;
         dmem_flush_o           : out std_logic;
         dmem_invalidate_o      : out std_logic;
         dmem_wb_o              : out std_logic;
         --
         dmem_accept_o          : out std_logic;
         dmem_rdata_o           : out slv_31_00;
         --
         dmem_accept_d1_o       : out std_logic
         --
         );
end entity rv_mmu;


architecture rtl of rv_mmu is

  component rv_lrsc
    port ( clk_i        : in  std_logic;
           xrst_i       : in  std_logic;
           --
           rd_i         : in  std_logic;     -- memory read     (cpu)
           wr_i         : in  slv_03_00;     -- memory write    (cpu)
           addr_i       : in  slv_31_00;     -- physical memory address
           wdata_i      : in  slv_31_00;     -- NC            ! LOOPBACK ONLY !
           --
           accept_o     : out std_logic;     -- accept  rd / wr (cpu)
           ack_o        : out std_logic;     -- ack for rd / wr (cpu)
           rdata_o      : out slv_31_00;     -- NC            ! LOOPBACK ONLY !
           err_o        : out std_logic;     -- mem err         (cpu)
           --
           sys_accept_i : in  std_logic;     -- accept  rd / wr (dmem)
           sys_ack_i    : in  std_logic;     -- ack for rd / wr (dmem)
           sys_rdata_i  : in  slv_31_00;     -- NC            ! LOOPBACK ONLY !
           sys_err_i    : in  std_logic;     -- mem err         (dmem)
           --
           sys_rd_o     : out std_logic;
           sys_wr_o     : out slv_03_00;
           sys_addr_o   : out slv_31_00;     -- access address
           sys_wdata_o  : out slv_31_00;     -- NC write data  ! LOOPBACK ONLY !
           --
           mmu_rd_i     : in  std_logic;     -- page adress fetch
           atomic_i     : in  std_logic;     -- atomic memory operaiton lr/sc
           lr_cancel_i  : in  std_logic );   -- irq taken, context change
  end component;


  component rv_amo
    port ( clk_i          : in  std_logic;
           xrst_i         : in  std_logic;   -- sync reset
           -- cpu port
           mmu_rd_i       : in  std_logic;   -- page adress fetch
           amofunc_i      : in  slv_03_00;   -- AMO func  (no LR/SC)
           amo_pending_o  : out std_logic;
           --
           rd_i           : in  std_logic;
           wr_i           : in  slv_03_00;   -- byte access
           addr_i         : in  slv_31_00;   -- access address
           wdata_i        : in  slv_31_00;   -- write data
           --
           accept_o       : out std_logic;   -- read/write accept to cpu
           ack_o          : out std_logic;   -- rd/wr ack to cpu
           rdata_o        : out slv_31_00;   -- read data, valid if ack=1
           err_o          : out std_logic;   -- set if mem access error
           -- system port
           sys_accept_i   : in  std_logic;   -- read/write accept from mem
           sys_ack_i      : in  std_logic;   -- read/write ack from mem
           sys_rdata_i    : in  slv_31_00;   -- rdata from dmem, valid if ack=1
           sys_err_i      : in  std_logic;   -- memory access error
           --
           sys_rd_o       : out std_logic;   -- request strobe, single-shot
           sys_wr_o       : out slv_03_00;   -- byte enable
           sys_addr_o     : out slv_31_00;   -- access address
           sys_wdata_o    : out slv_31_00 ); -- write data
      end component;                         -- by not allowing rf write


  signal dmem_accept_iw   : std_logic;       -- dcache accept
  signal dmem_ack_iw      : std_logic;
  signal dmem_rdata_iw    : slv_31_00;       -- dcache / perip
  signal dmem_error_iw    : std_logic;

  signal dmem_rd_ow       : std_logic;
  signal dmem_wr_ow       : slv_03_00;
  signal dmem_addr_ow     : slv_31_00;
  signal dmem_wdata_ow    : slv_31_00;

  signal dmem_accept_i2w  : std_logic;       -- dcache accept
  signal dmem_ack_i2w     : std_logic;
  signal dmem_rdata_i2w   : slv_31_00;       -- dcache / perip
  signal dmem_error_i2w   : std_logic;

  signal dmem_rd_o2w      : std_logic;
  signal dmem_wr_o2w      : slv_03_00;
  signal dmem_addr_o2w    : slv_31_00;
  signal dmem_wdata_o2w   : slv_31_00;

  signal lr_cancel_w      : std_logic;       -- irq taken, context change

  constant ASID_R : slv_t(25 downto 22) := "0000"; -- 30:22 max

  -- - - - - - - -
  --  local defs
  -- - - - - - - -
  constant  STATE_W              : natural  := 2;
  constant  STATE_IDLE           : natural  := 0;
  constant  STATE_LEVEL_FIRST    : natural  := 1;
  constant  STATE_LEVEL_SECOND   : natural  := 2;
  constant  STATE_UPDATE         : natural  := 3;

  constant  STATE_IDLE_B         : slv_t(STATE_W-1 downto 0) := "00";
  constant  STATE_LEVEL_FIRST_B  : slv_t(STATE_W-1 downto 0) := "01";
  constant  STATE_LEVEL_SECOND_B : slv_t(STATE_W-1 downto 0) := "10";
  constant  STATE_UPDATE_B       : slv_t(STATE_W-1 downto 0) := "11";

  signal dmem_accept_q           : std_logic;

begin

  p_accept : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        dmem_accept_q    <= '0';
      else
        if (ifence_p1_i = '1') then
          dmem_accept_q  <=  '0';
        else
          dmem_accept_q  <=  dmem_accept_iw;
        end if;
      end if;
    end if;
  end process p_accept;

  dmem_accept_d1_o       <= dmem_accept_q;


  -- - - - - - - - - - -
  --  basic MMU support
  -- - - - - - - - - - -
  g_mmu : if ( SUPPORT_MMU /= '0' ) generate
    -- - - - - - - -
    --  registers
    -- - - - - - - -
    signal state_q         : slv_t(STATE_W-1 downto 0);
    signal idle_w          : std_logic;

    --  magic combo used only by MMU
    signal resp_mmu_w      : std_logic;
    signal resp_valid_w    : std_logic;
    signal resp_error_w    : std_logic;
    signal resp_data_w     : slv_31_00;
    signal cpu_accept_w    : std_logic;
    -- - - - - - - - -
    --  load / store
    -- - - - - - - - -
    signal load_q          : std_logic;
    signal store_q         : slv_03_00;

    signal lsu_in_accept_w : std_logic;

    signal load_w          : std_logic;
    signal store_w         : slv_03_00;
    signal lsu_in_addr_q   : slv_31_00;

    signal lsu_addr_w      : slv_31_00;

    -- - - - - - - - - - -
    --  page table walker
    -- - - - - - - - - - -
    signal itlb_hit_w      : std_logic;
    signal dtlb_hit_w      : std_logic;
    signal dtlb_req_q      : std_logic;

    --  global enable
    signal vm_enable_w     : std_logic;

    signal ifetch_vm_w     : std_logic;
    signal dfetch_vm_w     : std_logic;

    signal supervisor_i_w  : std_logic;
    signal supervisor_d_w  : std_logic;

    signal vm_i_enable_w   : std_logic;
    signal vm_d_enable_w   : std_logic;

    --  TLB entry does not match request address
    signal itlb_miss_w     : std_logic;
    signal dtlb_miss_w     : std_logic;

    --  data miss is higher priority than instruction...
    signal request_addr_w  : slv_31_00;

    signal pte_addr_q      : slv_31_00;
    signal pte_addr_w      : slv_31_00;
    signal pte_entry_q     : slv_31_00;
    signal virt_addr_q     : slv_31_00;

    signal pte_ppn_w       : slv_31_00;
    signal pte_flags_w     : slv_09_00;

    signal amo_pending_w   : std_logic;


    -- - - - - - - - - - -
    --  page table walker
    -- - - - - - - - - - -
    --
    -- A 2-level page table mechanism that maps 4GB virtual address space
    -- to 16 GB Physical address space.
    --
    --
    --  sv32 virtual address
    --  31             22 21          12 11              0
    --  ----------------------------------------------------
    --  |     VPN[1]     |   VPN[0]     |    page offset   |
    --  ----------------------------------------------------
    --          10             10                12
    --
    --  sv32 physical address
    --  33             22 21          12 11              0
    --  ----------------------------------------------------
    --  |     PPN[1]     |   PPN[0]     |    page offset   |
    --  ----------------------------------------------------
    --          12             10                12
    --
    --  sv32 page table entry
    --  33             22 21          12 11              0
    --  ---------------------------------------------------------------------
    --  |     PPN[1]     |   PPN[0]     | RSW | D | A | G | U X | W | R | V |
    --  ---------------------------------------------------------------------
    --          12             10                12
    --
    --
    -- sv32 page table entry bits :
    --   [31:20]      Physical page number, 1st level PTE
    --   [19:10]      Physical page number, 2nd level PTE
    --   [9:8]        RSW             :  PAGE_SOFT   (reserved, ignore)
    --   [7]     D    DIRTY           :  PAGE_USER
    --   [6]     A    ACCESSES        :  PAGE_USER
    --   [5]     G    GLOBAL (shared) :  PAGE_USER
    --   [4]     U    USER            :  PAGE_USER
    --   [3]     X    EXECUTABLE      :  PAGE_EXEC
    --   [2]     W    WRITEABLE       :  PAGE_WRITE
    --   [1]     R    READABLE        :  PAGE_READ
    --   [0]     V    VALID           :  PAGE_PRESENT


    type tlb_sv32_t is
    record
      valid : std_logic;                       -- valid flag
      vpn   : slv_t(20-1 downto 0);            -- virtual page nro (+12b offset)
      asid  : slv_t(ASID_R'length-1 downto 0); -- ASID length (max 9 for sv32)
      pte   : slv_t(31 downto 0);              -- page table entry
    end record;


    -- - - - - - -
    --  IMMU TLB
    -- - - - - - -
    type itlb_t is array (0 to ITLB_ENTRIES-1) of tlb_sv32_t;

    signal itlb_q               : itlb_t;
    signal itlb_hits_w          : slv_t(ITLB_ENTRIES-1 downto 0);
    -- current fetch page index
    signal ipage_w              : natural range 0 to ITLB_ENTRIES-1;
    signal itlb_replace_q       : slv_t(log2(ITLB_ENTRIES)-1 downto 0);

    signal pc_fault_w           : std_logic;
    signal pc_fault_q           : std_logic;

    type dtlb_t is array (0 to DTLB_ENTRIES-1) of tlb_sv32_t;
    signal dtlb_q               : dtlb_t;
    signal dtlb_hits_w          : slv_t(DTLB_ENTRIES-1 downto 0);
    -- current fetch page index
    signal dpage_w              : natural range 0 to DTLB_ENTRIES-1;
    signal dtlb_replace_q       : slv_t(log2(DTLB_ENTRIES)-1 downto 0);


    signal load_fault_w         : std_logic;

    signal store_fault_w        : std_logic;

    signal store_fault_q        : std_logic;
    signal load_fault_q         : std_logic;

    signal lsu_out_rd_w         : std_logic;
    signal lsu_out_wr_w         : slv_03_00;
    signal lsu_out_addr_w       : slv_31_00;

    signal lsu_out_invalidate_w : std_logic;

    signal lsu_out_cacheable_w  : std_logic;
    signal lsu_out_flush_w      : std_logic;

    -- - - - - - - - - -
    --  PTE Fetch Port
    -- - - - - - - - - -
    signal mem_req_q            : std_logic;
    signal mmu_accept_w         : std_logic;

    -- - - - - - - - - -
    --  request muxing
    -- - - - - - - - - -
    signal read_hold_q          : std_logic;
    signal src_mmu_q            : std_logic;
    signal src_mmu_w            : std_logic;

    signal lsu_out_rd_o_w       : std_logic;
    signal lsu_out_wr_o_w       : slv_03_00;

    signal resp_data_29_10_q    : slv_19_00;
    signal request_addr_21_12_q : slv_09_00;
    signal sel_entry_q          : std_logic;

  begin

    idle_w       <= eq(state_q, STATE_IDLE_B);
    resp_mmu_w   <= eq(dmem_resp_tag_i(9 downto 7), "111");
    resp_valid_w <= resp_mmu_w and dmem_ack_iw;
    resp_error_w <= resp_mmu_w and dmem_error_iw;
    resp_data_w  <= dmem_rdata_iw;                -- [31:0], from dcache


    p_ld : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          load_q <= '0';
        else
          if (lsu_in_rd_i = '1') then
            load_q <= not lsu_in_accept_w;
          end if;
        end if;
      end if;
    end process p_ld;


    p_st : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          store_q <= (others => '0');           -- 4b
        else
          if (not_zero(lsu_in_wr_i)) then       -- 3:0
            if (lsu_in_accept_w = '1') then
              store_q <= (others => '0');       -- 4b
            else
              store_q <= lsu_in_wr_i;
            end if;
          end if;
        end if;
      end if;
    end process p_st;


    load_w  <= lsu_in_rd_i or load_q;
    store_w <= lsu_in_wr_i or store_q;


    p_lsuiaddr : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          lsu_in_addr_q <= (others => '0');     -- 32b
        else
          if ( load_w = '1' or not_zero(store_w) ) then
            lsu_in_addr_q <= lsu_in_addr_i;
          end if;
        end if;
      end if;
    end process p_lsuiaddr;

    lsu_addr_w <= lsu_in_addr_i when (load_w = '1' or not_zero(store_w)) else
                  lsu_in_addr_q;                                -- [31:0]


    --  global enable
    vm_enable_w    <= satp_i(SATP_MODE_R);                      -- 31
    -- page table base register

    ifetch_vm_w    <= ne(fetch_in_priv_i, PRIV_MACHINE);
    dfetch_vm_w    <= ne(priv_d_i,        PRIV_MACHINE);

    supervisor_i_w <= eq(fetch_in_priv_i, PRIV_SUPER);
    supervisor_d_w <= eq(priv_d_i,        PRIV_SUPER);

    vm_i_enable_w  <= ifetch_vm_w;
    vm_d_enable_w  <= vm_enable_w and dfetch_vm_w;

    --  TLB address matched (even on page fault)
    p_c_ihits : process (fetch_in_rd_i, itlb_q, fetch_in_pc_i, satp_i)
    begin
      for ii in 0 to ITLB_ENTRIES-1 loop
        itlb_hits_w(ii)  <= fetch_in_rd_i and itlb_q(ii).valid and
                            eq(itlb_q(ii).vpn, fetch_in_pc_i(31 downto 12)) and
                            eq(itlb_q(ii).asid, satp_i(ASID_R'range));
      end loop;
    end process p_c_ihits;

    --  TLB vaddr/ASID does not match request address
    itlb_hit_w   <= not_zero(itlb_hits_w);
    itlb_miss_w  <= fetch_in_rd_i and vm_i_enable_w and (not itlb_hit_w);

    p_c_ipage : process (itlb_hits_w)
    begin
      ipage_w <= 0;
      --
      for  ii in 0 to ITLB_ENTRIES-1 loop
        if (itlb_hits_w(ii) = '1') then
          ipage_w <= ii;
        end if;
      end loop;
      --
    end process p_c_ipage;


    -- - - - - - -
    --  IMMU TLB
    -- - - - - - -
    p_itlb : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          itlb_q <= (others => ( valid => '0',   -- valid flag
                                 vpn   => (others => '0'),  -- VPN 20+12b offset
                                 asid  => (others => '0'),  -- ASID 9b for Sv32
                                 pte   => (others => '0')) );
          --
          itlb_replace_q <= (others => '0');
        else
          if (flush_i = '1') then
            for ii in 0 to ITLB_ENTRIES-1 loop
              itlb_q(ii).valid <= '0';
            end loop;
          elsif ( state_q = STATE_UPDATE_B and dtlb_req_q = '0' ) then
            --  fetch TLB still matches incoming request
            itlb_q(to_uint(itlb_replace_q)).valid <= '1';
            --  virtual address
            itlb_q(to_uint(itlb_replace_q)).vpn   <= virt_addr_q(31 downto 12);
            --  physical address
            itlb_q(to_uint(itlb_replace_q)).pte   <= pte_entry_q;
            itlb_q(to_uint(itlb_replace_q)).asid  <= satp_i(ASID_R'range);
            --
            -- if (unsigned(itlb_replace_q) >= ITLB_ENTRIES) then
            if (ITLB_ENTRIES = 0 or ITLB_ENTRIES = 1) then
              itlb_replace_q <= (others => '0');
            else
              itlb_replace_q <= inc1u(itlb_replace_q);
            end if;
          elsif ( state_q /= STATE_IDLE_B and dtlb_req_q = '0' ) then
            itlb_q(to_uint(itlb_replace_q)).valid <= '0';
          end if;
        end if;
      end if;
    end process p_itlb;


    request_addr_w <= lsu_addr_w when (idle_w = '1' and dtlb_miss_w = '1')  else
                      fetch_in_pc_i when (idle_w ='1' and dtlb_miss_w ='0') else
                      lsu_addr_w when (idle_w ='0' and dtlb_req_q = '1')    else
                      fetch_in_pc_i;

    -- pte_ppn : page table physical page number
    -- pfn     : page frame number
    pte_ppn_w   <= (PAGE_PFN_SHIFT-1 downto 0 => '0') &     -- 9:0
                   resp_data_w(31 downto PAGE_PFN_SHIFT);   -- 31:10
    pte_flags_w <= resp_data_w(9 downto 0);


    p_state : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          pte_addr_q  <= X"00000000";   -- 31:0
          pte_entry_q <= X"00000000";   -- 31:0
          virt_addr_q <= X"00000000";   -- 31:0
          dtlb_req_q  <= '0';
          state_q     <= STATE_IDLE_B;
          -- retiming adder
          resp_data_29_10_q    <= (others => '0');
          request_addr_21_12_q <= (others => '0');
          sel_entry_q          <= '0';
        else
          case state_q is                        --  TLB miss, walk page table
            when STATE_IDLE_B =>
              if (itlb_miss_w = '1' or dtlb_miss_w = '1') then
                -- page table base  &  request_addr(31:22) & "00"
                -- 19:00    &  X"000"
                -- X"00000" &  9:0    &  "00"
                pte_addr_q  <= satp_i(SATP_PPN_R'range) &
                               request_addr_w(31 downto 22) & "00";
                virt_addr_q <= request_addr_w;    -- intr / data addr
                dtlb_req_q  <= dtlb_miss_w;
                state_q     <= STATE_LEVEL_FIRST_B;
                sel_entry_q <= '0';
              end if;
            when STATE_LEVEL_FIRST_B =>           -- 1st level (4MB superpage)
              if (resp_valid_w = '1') then
                --  error or page not present
                if (resp_error_w = '1' or resp_data_w(PAGE_PRESENT) = '0') then
                  pte_entry_q <= X"00000000";
                  state_q     <= STATE_UPDATE_B;
                  --  Valid entry, but another level to fetch
                elsif (resp_data_w(PAGE_READ) = '0' and
                       resp_data_w(PAGE_WRITE) = '0' and
                       resp_data_w(PAGE_EXEC) = '0') then
                  resp_data_29_10_q    <= resp_data_w(29 downto 10);
                  request_addr_21_12_q <= request_addr_w(21 downto 12);
                  sel_entry_q          <= '1';
                  state_q              <= STATE_LEVEL_SECOND_B;
                else   --  valid entry, actual valid PTE
                  --
                  -- pte_ppn_w(31:10)  & ( pte_ppn_w(9:0) or
                  --                       request_addr_w(21:12) ) << 12
                  -- ==>
                  -- pte_ppn_w(31-12:10) & ( pte_ppn_w(9:0) or
                  --                         request_addr_w(21:12) ) & 0x000
                  --
                  -- PGSIZE   4096  - bytes mapped by a page
                  -- PGSHIFT  12    - log2(PGSIZE)
                  -- PTE PPN        - page table physical page number
                  pte_entry_q <= pte_ppn_w(31-MMU_PGSHIFT downto 10) & -- 10b
                                 ( pte_ppn_w(9 downto 0) or            -- 10b |
                                   request_addr_w(21 downto 12) ) &    -- 10b
                                 (MMU_PGSHIFT-1 downto 0 => '0');      -- << 12
                  pte_entry_q(9 downto 0) <= pte_flags_w;  -- 9:0
                  state_q     <= STATE_UPDATE_B;
                end if;
              end if;
            when STATE_LEVEL_SECOND_B =>      --  2nd level (4KB page)
              if (resp_valid_w = '1') then
                --  valid entry, final level
                if (resp_data_w(PAGE_PRESENT) = '1') then             -- [0]
                  --                       << 12
                  -- PGSIZE   4096  - bytes mapped by a page
                  -- PGSHIFT  12    - log2(PGSIZE)
                  pte_entry_q <= pte_ppn_w(31-MMU_PGSHIFT downto 0) & -- 19:0
                                 (MMU_PGSHIFT-1 downto 0 => '0');     -- 11:0
                  pte_entry_q(9 downto 0)  <=  pte_flags_w;           -- 9:0
                  state_q     <= STATE_UPDATE_B;
                  --  page fault
                else
                  pte_entry_q <= (others => '0');                     -- 31:0
                  state_q     <= STATE_UPDATE_B;
                end if;
              end if;
              --
            when STATE_UPDATE_B =>
              state_q    <= STATE_IDLE_B;
            when others =>
              null;
          end case;
          --
        end if;
      end if;
    end process p_state;


    pte_addr_w <= (resp_data_29_10_q &  request_addr_21_12_q & "00")
                  when (sel_entry_q = '1')
                  else pte_addr_q;


    p_c_pc_fault : process (vm_i_enable_w, itlb_hit_w, ipage_w,
                            supervisor_i_w, itlb_q)
    begin
      pc_fault_w <= '0';
      if (vm_i_enable_w = '1' and itlb_hit_w = '1') then
        --  supervisor mode
        if (supervisor_i_w = '1') then
          --  user page, supervisor cannot execute
          if (itlb_q(ipage_w).pte(PAGE_USER) = '1') then
            pc_fault_w <= '1';
            --  check exec permissions
          else
            pc_fault_w <= not itlb_q(ipage_w).pte(PAGE_EXEC);
          end if;
          --  user mode
        else
          pc_fault_w <= (not itlb_q(ipage_w).pte(PAGE_EXEC)) or
                        (not itlb_q(ipage_w).pte(PAGE_USER));
        end if;
      end if;
    end process p_c_pc_fault;


    p_pc_fault : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        pc_fault_q <= pc_fault_w  and xrst_i;
      end if;
    end process p_pc_fault;


    fetch_out_rd_o         <= ((not vm_i_enable_w) and fetch_in_rd_i) or
                              (itlb_hit_w and (not pc_fault_w));
    imem_pc_o              <= itlb_q(ipage_w).pte(31 downto 12) &
                              fetch_in_pc_i(11 downto 0) when
                              vm_i_enable_w = '1' else fetch_in_pc_i;
    fetch_out_flush_o      <= fetch_in_flush_i;
    fetch_out_invalidate_o <= fetch_in_invalidate_i;

    fetch_in_accept_o      <= ((not vm_i_enable_w) and fetch_out_accept_i) or
                              (vm_i_enable_w and itlb_hit_w and
                               fetch_out_accept_i) or  pc_fault_w;
    fetch_in_valid_o       <= fetch_out_valid_i or pc_fault_q;
    fetch_in_error_o       <= fetch_out_valid_i and fetch_out_error_i;
    fetch_in_fault_o       <= pc_fault_q;


    -- - - - - - -
    --  DMMU TLB
    -- - - - - - -
    p_c_dhits : process (dtlb_q,  lsu_addr_w, satp_i)
      -- load_w, store_w, dpage_w,
    begin
      for ii in 0 to DTLB_ENTRIES-1 loop
        dtlb_hits_w(ii)  <= eq(dtlb_q(ii).vpn, lsu_addr_w(31 downto 12)) and
                            dtlb_q(ii).valid and
                            eq(dtlb_q(ii).asid, satp_i(ASID_R'range));
      end loop;
    end process p_c_dhits;


    --  TLB address matched (even on page fault)
    dtlb_hit_w  <= not_zero(dtlb_hits_w);

    dtlb_miss_w <= (load_w or not_zero(store_w)) and vm_d_enable_w and
                   (not dtlb_hit_w);

    p_c_dpage : process (dtlb_hits_w)
    begin
      dpage_w <= 0;
      --
      for  ii in 0 to DTLB_ENTRIES-1 loop
        if (dtlb_hits_w(ii) = '1') then
          dpage_w <= ii;
        end if;
      end loop;
      --
    end process p_c_dpage;


    p_dtlb : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          dtlb_q <= (others => ( valid => '0',   -- valid flag
                                 vpn   => (others => '0'),  -- VPN 20+12b offset
                                 asid  => (others => '0'),  -- ASID 9b for Sv32
                                 pte   => (others => '0')) );
          --
          dtlb_replace_q <= (others => '0');
        else
          if (flush_i = '1') then
            for ii in 0 to DTLB_ENTRIES-1 loop
              dtlb_q(ii).valid <= '0';
            end loop;
          elsif ( state_q = STATE_UPDATE_B and dtlb_req_q = '1' ) then
            --  mark TLB entry as valid
            dtlb_q(to_uint(dtlb_replace_q)).valid <= '1';
            --  virtual address
            dtlb_q(to_uint(dtlb_replace_q)).vpn   <= virt_addr_q(31 downto 12);
            --  physical address (ppn)
            dtlb_q(to_uint(dtlb_replace_q)).pte   <= pte_entry_q;
            dtlb_q(to_uint(dtlb_replace_q)).asid  <= satp_i(ASID_R'range);
            --
            -- if (unsigned(dtlb_replace_q) >= DTLB_ENTRIES) then
            if (DTLB_ENTRIES = 0 or DTLB_ENTRIES = 1) then
              dtlb_replace_q <= (others => '0');
            else
              dtlb_replace_q <= inc1u(dtlb_replace_q);
            end if;
          elsif ( state_q /= STATE_IDLE_B and dtlb_req_q = '0' ) then
            dtlb_q(to_uint(dtlb_replace_q)).valid <= '0';
          end if;
        end if;
      end if;
    end process p_dtlb;


    p_c_load_fault : process (vm_d_enable_w, load_w, dtlb_hit_w,
                              supervisor_d_w, sum_i, dtlb_q, mxr_i, dpage_w)
    begin
      load_fault_w <= '0';
      if (vm_d_enable_w = '1' and load_w = '1' and dtlb_hit_w = '1') then
        --  supervisor mode
        if (supervisor_d_w = '1') then
          --  user page, supervisor user mode not enabled
          if (dtlb_q(dpage_w).pte(PAGE_USER) = '1' and sum_i = '0') then
            load_fault_w <= '1';
            --  check exec permissions
          else
            load_fault_w <= not (dtlb_q(dpage_w).pte(PAGE_READ) or
                                 (mxr_i and dtlb_q(dpage_w).pte(PAGE_EXEC)));
          end if;
          --  user mode
        else
          load_fault_w <= ( not dtlb_q(dpage_w).pte(PAGE_READ)) or
                          ( not dtlb_q(dpage_w).pte(PAGE_USER));
        end if;
      end if;
    end process p_c_load_fault;


    p_c_store_fault : process (vm_d_enable_w, store_w, dtlb_hit_w,
                               supervisor_d_w, dtlb_q, sum_i, dpage_w)
    begin
      store_fault_w <= '0';
      if (vm_d_enable_w = '1' and not_zero(store_w) and dtlb_hit_w = '1') then
        --  supervisor mode
        if (supervisor_d_w = '1') then
          --  user page, supervisor user mode not enabled
          if (dtlb_q(dpage_w).pte(PAGE_USER) = '1' and sum_i = '0') then
            store_fault_w <= '1';
            --  check exec permissions
          else
            store_fault_w <= (not dtlb_q(dpage_w).pte(PAGE_READ)) or
                             (not dtlb_q(dpage_w).pte(PAGE_WRITE));
          end if;
          --   user mode
        else
          store_fault_w <= (not dtlb_q(dpage_w).pte(PAGE_READ)) or
                           (not dtlb_q(dpage_w).pte(PAGE_WRITE)) or
                           (not dtlb_q(dpage_w).pte(PAGE_USER));
        end if;
      end if;
    end process p_c_store_fault;


    p_ld_fault : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        load_fault_q <= load_fault_w  and xrst_i ;
      end if;
    end process p_ld_fault;


    p_st_fault : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        store_fault_q <= store_fault_w  and xrst_i;
      end if;
    end process p_st_fault;


    lsu_out_rd_w         <= (load_w and dtlb_hit_w and (not load_fault_w))
                            when (vm_d_enable_w = '1') else lsu_in_rd_i;
    lsu_out_wr_w         <= (store_w and
                             ( (dtlb_hit_w and (not store_fault_w)) &
                               (dtlb_hit_w and (not store_fault_w)) &
                               (dtlb_hit_w and (not store_fault_w)) &
                               (dtlb_hit_w and (not store_fault_w)) ) )
                            when (vm_d_enable_w = '1') else
                            lsu_in_wr_i;                           -- [3:0]

    lsu_out_addr_w(31 downto 12) <= dtlb_q(dpage_w).pte(31 downto 12)
                                    when (vm_d_enable_w = '1') else
                                    lsu_addr_w(31 downto 12);      -- [31:0]
    lsu_out_addr_w(11 downto 0)  <= lsu_addr_w(11 downto 0);


    lsu_out_invalidate_w         <= lsu_in_invalidate_i;


    p_c_lsu_o_c : process (lsu_in_invalidate_i, lsu_in_wb_i,
                           lsu_in_flush_i, lsu_out_addr_w)
    begin
      lsu_out_cacheable_w     <= '0';
      if (lsu_in_invalidate_i = '1' or lsu_in_wb_i = '1' or
          lsu_in_flush_i = '1') then
        lsu_out_cacheable_w   <= '1';
      else
        if ( (lsu_out_addr_w >= CACHE_ADDR_L) and
             (lsu_out_addr_w  <= CACHE_ADDR_H) ) then
          lsu_out_cacheable_w <= '1';
        end if;
      end if;
    end process p_c_lsu_o_c;


    lsu_out_flush_w      <= lsu_in_flush_i;

    lsu_in_ack_o         <= (dmem_ack_iw and (not resp_mmu_w)) or
                            store_fault_q or load_fault_q;
    lsu_in_error_o       <= (dmem_error_iw and (not resp_mmu_w)) or
                            store_fault_q or load_fault_q;
    lsu_in_store_fault_o <= store_fault_q;
    lsu_in_load_fault_o  <= load_fault_q;

    lsu_in_accept_w      <= ((not vm_d_enable_w) and cpu_accept_w) or
                            (vm_d_enable_w and dtlb_hit_w and cpu_accept_w) or
                            store_fault_w or load_fault_w;

    -- - - - - - - - - -
    --  PTE fetch port
    -- - - - - - - - - -
    p_mem_req : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          mem_req_q <= '0';
        else
          if (state_q = STATE_IDLE_B and
              (itlb_miss_w = '1' or dtlb_miss_w = '1')) then
            mem_req_q <= '1';
          elsif ( state_q = STATE_LEVEL_FIRST_B and resp_valid_w = '1' and
                  resp_error_w = '0' and
                  resp_data_w(PAGE_PRESENT) = '1' and       -- [0]
                  ( resp_data_w(PAGE_READ)  = '0' and       -- [1]
                    resp_data_w(PAGE_WRITE) = '0' and       -- [2]
                    resp_data_w(PAGE_EXEC)  = '0' ) ) then  -- [3]
            mem_req_q <= '1';
          elsif (mmu_accept_w = '1') then
            mem_req_q <= '0';
          end if;
        end if;
      end if;
    end process p_mem_req;


    src_mmu_w <= src_mmu_q when (read_hold_q = '1') else mem_req_q;

    p_rd_hold : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          read_hold_q  <= '0';
          src_mmu_q    <= '0';
        else
          if ( (lsu_out_rd_o_w = '1' or not_zero(lsu_out_wr_o_w) ) and
               dmem_accept_iw = '0' ) then
            read_hold_q  <= '1';
            src_mmu_q    <= src_mmu_w;
          elsif (dmem_accept_iw = '1') then
            read_hold_q  <= '0';
          end if;
        end if;
      end if;
    end process p_rd_hold;


    mmu_accept_w      <= src_mmu_w  and dmem_accept_iw;
    cpu_accept_w      <= (not src_mmu_w) and dmem_accept_iw;

    lsu_out_rd_o_w    <= mem_req_q when (src_mmu_w = '1') else
                         lsu_out_rd_w;
    lsu_out_wr_o_w    <= X"0" when (src_mmu_w = '1') else lsu_out_wr_w;

    dmem_invalidate_o <= '0' when (src_mmu_w = '1') else
                         lsu_out_invalidate_w;
    dmem_wb_o         <= '0' when (src_mmu_w = '1') else
                         lsu_in_wb_i;
    dmem_cacheable_o  <= '1' when (src_mmu_w = '1') else
                         lsu_out_cacheable_w;

    dmem_req_tag_o    <= "01110000000" when (src_mmu_w = '1')
                         else lsu_in_req_tag_i;   -- [10:0]
    dmem_flush_o      <= '0' when (src_mmu_w = '1') else lsu_out_flush_w;

    lsu_in_accept_o   <= lsu_in_accept_w;


    dmem_rd_ow         <= lsu_out_rd_o_w;
    dmem_wr_ow         <= lsu_out_wr_o_w;
    dmem_addr_ow       <= pte_addr_w when (src_mmu_w = '1') else lsu_out_addr_w;
    dmem_wdata_ow      <= lsu_in_wdata_i;

    dmem_accept_o      <= dmem_accept_iw;
    dmem_rdata_o       <= dmem_rdata_iw;


  u_rv_lrsc : rv_lrsc
      port map ( clk_i       => clk_i,
                 xrst_i      => xrst_i,
                 --
                 rd_i         => dmem_rd_ow,       -- memory read     (cpu)
                 wr_i         => dmem_wr_ow,       -- memory write    (cpu)
                 addr_i       => dmem_addr_ow,     -- physical memory address
                 wdata_i      => dmem_wdata_ow,    -- NC      ! LOOPBACK ONLY !
                 --
                 accept_o     => dmem_accept_iw,   -- accept  rd / wr (cpu)
                 ack_o        => dmem_ack_iw,      -- ack for rd / wr (cpu)
                 rdata_o      => dmem_rdata_iw,    -- NC      ! LOOPBACK ONLY !
                 err_o        => dmem_error_iw,    -- mem err         (cpu)
                 --
                 sys_accept_i => dmem_accept_i2w,  -- accept  rd / wr (dmem)
                 sys_ack_i    => dmem_ack_i2w,     -- ack for rd / wr (dmem)
                 sys_rdata_i  => dmem_rdata_i2w,   -- NC      ! LOOPBACK ONLY !
                 sys_err_i    => dmem_error_i2w,   -- mem err         (dmem)
                 --
                 sys_rd_o     => dmem_rd_o2w,
                 sys_wr_o     => dmem_wr_o2w,
                 sys_addr_o   => dmem_addr_o2w,    -- access address
                 sys_wdata_o  => dmem_wdata_o2w,   -- NC      ! LOOPBACK ONLY !
                 --
                 mmu_rd_i     => src_mmu_w,        -- page adress fetch
                 atomic_i     => lsu_atomic_i,
                 lr_cancel_i  => lr_cancel_w );


    lr_cancel_w   <= lr_cancel_i and flush_i;

    amo_pending_o <= amo_pending_w;

    u_rv_amo : rv_amo
      port map ( clk_i         => clk_i,
                 xrst_i        => xrst_i,
                 -- cpu port
                 mmu_rd_i      => src_mmu_w,       -- page adress fetch
                 amofunc_i     => amofunc_i,
                 amo_pending_o => amo_pending_w,
                 --
                 rd_i          => dmem_rd_o2w,
                 wr_i          => dmem_wr_o2w,     -- byte access
                 addr_i        => dmem_addr_o2w,   -- access address
                 wdata_i       => dmem_wdata_o2w,  -- write data
                 --
                 accept_o      => dmem_accept_i2w, -- read/write ack from mem
                 ack_o         => dmem_ack_i2w,    -- rd/wr ack to cpu
                 rdata_o       => dmem_rdata_i2w,  -- read data, valid if ack=1
                 err_o         => dmem_error_i2w,  -- set if mem access error
                 -- system port
                 sys_accept_i  => dmem_accept_i,   -- read/write ack from mem
                 sys_ack_i     => dmem_ack_i,      -- read/write ack from mem
                 sys_rdata_i   => dmem_rdata_i,    -- from dmem, valid if ack=1
                 sys_err_i     => dmem_error_i,    -- memory access error
                 --
                 sys_rd_o      => dmem_rd_o,       -- request strobe
                 sys_wr_o      => dmem_wr_o,       -- byte enable
                 sys_addr_o    => dmem_addr_o,     -- access address
                 sys_wdata_o   => dmem_wdata_o );  -- write data

  end generate g_mmu;




  -- -----------------------------------------------------------------
  --  No MMU support
  -- -----------------------------------------------------------------
  g_nommu : if ( SUPPORT_MMU = '0' ) generate
  begin
    fetch_out_rd_o         <= fetch_in_rd_i;
    imem_pc_o              <= fetch_in_pc_i;          -- iaddr
    fetch_out_flush_o      <= fetch_in_flush_i;
    fetch_out_invalidate_o <= fetch_in_invalidate_i;
    fetch_in_accept_o      <= fetch_out_accept_i;
    fetch_in_valid_o       <= fetch_out_valid_i;
    fetch_in_error_o       <= fetch_out_error_i;
    fetch_in_fault_o       <= '0';

    dmem_cacheable_o       <= lsu_in_cacheable_i;
    dmem_req_tag_o         <= lsu_in_req_tag_i;

    dmem_rd_o              <= lsu_in_rd_i;
    dmem_wr_o              <= lsu_in_wr_i;
    dmem_addr_o            <= lsu_in_addr_i;          -- daddr
    dmem_wdata_o           <= lsu_in_wdata_i;
    dmem_invalidate_o      <= lsu_in_invalidate_i;
    dmem_wb_o              <= lsu_in_wb_i;
    dmem_flush_o           <= lsu_in_flush_i;

    lsu_in_ack_o           <= dmem_ack_i;
    lsu_in_error_o         <= dmem_error_i;
    lsu_in_store_fault_o   <= '0';
    lsu_in_load_fault_o    <= '0';

    lsu_in_accept_o        <= dmem_accept_i;

    amo_pending_o          <= '0';

  end generate g_nommu;


end rtl;

