-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--
-- Copyright 2025 VLSI Solution Oy
-- SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
--
-- Licensed under the Solderpad Hardware License v 2.1 (the "License");
-- you may not use this file except in compliance with the License, or,
-- at your option, the Apache License version 2.0.
-- You may obtain a copy of the License at
--
--    https://solderpad.org/licenses/SHL-2.1/
--
-- Unless required by applicable law or agreed to in writing, any work
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--                               RISCV core
--                                 V1.1
--
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

-- - - - - - - -
--  libraries
-- - - - - - - -
library IEEE;
use IEEE.std_logic_1164.all;
use IEEE.numeric_std.all;
use work.rv_pkg.all;

entity rv_mmu is
  generic ( CACHE_ADDR_L   : slv_31_00 := X"80000000";
            CACHE_ADDR_H   : slv_31_00 := X"8FFFFFFF";
            SUPPORT_MMU    : std_logic := '1' );
  port ( clk_i                  : in  std_logic;
         xrst_i                 : in  std_logic;  -- syncr reset
         --
         priv_d_i               : in  slv_01_00;
         sum_i                  : in  std_logic;
         mxr_i                  : in  std_logic;
         flush_i                : in  std_logic;
         satp_i                 : in  slv_31_00;  -- NC 30:20
         ifence_p1_i            : in  std_logic;  -- ifence instr
         imem_pc_o              : out slv_31_00;
         --  atomic sc: [1] vld, [0] 0 okay, 1 fail
         lr_cancel_i            : in  std_logic;  -- irq taken, context change
         amofunc_i              : in  slv_03_00;
         amo_pending_o          : out std_logic;
         --
         fetch_in_priv_i        : in  slv_01_00;
         fetch_in_rd_i          : in  std_logic;
         fetch_in_pc_i          : in  slv_31_00;
         fetch_in_flush_i       : in  std_logic;
         fetch_in_invalidate_i  : in  std_logic;
         fetch_in_accept_o      : out std_logic;
         fetch_in_valid_o       : out std_logic;
         fetch_in_error_o       : out std_logic;
         fetch_in_fault_o       : out std_logic;
         fetch_out_accept_i     : in  std_logic;
         fetch_out_valid_i      : in  std_logic;
         fetch_out_error_i      : in  std_logic;
         fetch_out_rd_o         : out std_logic;
         fetch_out_flush_o      : out std_logic;
         fetch_out_invalidate_o : out std_logic;
         --
         lsu_atomic_i           : in  std_logic;  -- atomic op
         lsu_in_cacheable_i     : in  std_logic;  -- NC in MMU ver
         lsu_in_req_tag_i       : in  slv_10_00;
         --
         lsu_in_rd_i            : in  std_logic;
         lsu_in_wr_i            : in  slv_03_00;
         lsu_in_addr_i          : in  slv_31_00;
         lsu_in_wdata_i         : in  slv_31_00;
         lsu_in_invalidate_i    : in  std_logic;
         lsu_in_flush_i         : in  std_logic;
         lsu_in_wb_i            : in  std_logic;
         lsu_in_accept_o        : out std_logic;
         lsu_in_ack_o           : out std_logic;
         lsu_in_error_o         : out std_logic;
         lsu_in_load_fault_o    : out std_logic;
         lsu_in_store_fault_o   : out std_logic;
         --
         dmem_cacheable_o       : out std_logic;
         dmem_resp_tag_i        : in  slv_10_00;
         dmem_req_tag_o         : out slv_10_00;
         --
         dmem_accept_i          : in  std_logic;
         dmem_ack_i             : in  std_logic;
         dmem_rdata_i           : in  slv_31_00;  -- dcache / perip
         dmem_error_i           : in  std_logic;
         dmem_rd_o              : out std_logic;
         dmem_wr_o              : out slv_03_00;
         dmem_addr_o            : out slv_31_00;
         dmem_wdata_o           : out slv_31_00;
         dmem_flush_o           : out std_logic;
         dmem_invalidate_o      : out std_logic;
         dmem_wb_o              : out std_logic;
         dmem_accept_o          : out std_logic;
         dmem_rdata_o           : out slv_31_00;
         dmem_accept_d1_o       : out std_logic
         );
end entity rv_mmu;


architecture rtl of rv_mmu is

  -- - - - - - - -
  --  local defs
  -- - - - - - - -
  constant  STATE_W              : natural  := 2;
  constant  STATE_IDLE           : natural  := 0;
  constant  STATE_LEVEL_FIRST    : natural  := 1;
  constant  STATE_LEVEL_SECOND   : natural  := 2;
  constant  STATE_UPDATE         : natural  := 3;

  constant  STATE_IDLE_B         : slv_t(STATE_W-1 downto 0) := "00";
  constant  STATE_LEVEL_FIRST_B  : slv_t(STATE_W-1 downto 0) := "01";
  constant  STATE_LEVEL_SECOND_B : slv_t(STATE_W-1 downto 0) := "10";
  constant  STATE_UPDATE_B       : slv_t(STATE_W-1 downto 0) := "11";

  signal dmem_accept_q          : std_logic;

begin

  p_accept : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        dmem_accept_q <= '0';
      else
        if (ifence_p1_i = '1') then
          dmem_accept_q <=  '0';
        else
          dmem_accept_q <=  dmem_accept_i;
        end if;
      end if;
    end if;
  end process p_accept;

  dmem_accept_d1_o      <= dmem_accept_q;


  -- - - - - - - - - - -
  --  basic MMU support
  -- - - - - - - - - - -
  g_mmu : if ( SUPPORT_MMU /= '0' ) generate
    -- - - - - - - -
    --  registers
    -- - - - - - - -
    signal state_q              : slv_t(STATE_W-1 downto 0);
    signal idle_w               : std_logic;
    --  magic combo used only by MMU
    signal resp_mmu_w           : std_logic;
    signal resp_valid_w         : std_logic;
    signal resp_error_w         : std_logic;
    signal resp_data_w          : slv_31_00;
    signal cpu_accept_w         : std_logic;
    -- - - - - - - - -
    --  load / store
    -- - - - - - - - -
    signal load_q               : std_logic;
    signal store_q              : slv_03_00;

    signal lsu_in_accept_w      : std_logic;

    signal load_w               : std_logic;
    signal store_w              : slv_03_00;
    signal lsu_in_addr_q        : slv_31_00;

    signal lsu_addr_w           : slv_31_00;

    -- - - - - - - - - - -
    --  page table walker
    -- - - - - - - - - - -
    signal itlb_hit_w           : std_logic;
    signal dtlb_hit_w           : std_logic;
    signal dtlb_req_q           : std_logic;

    --  global enable
    signal vm_enable_w          : std_logic;
    --  page table base register
    signal ptbr_w               : slv_31_00;

    signal ifetch_vm_w          : std_logic;
    signal dfetch_vm_w          : std_logic;

    signal supervisor_i_w       : std_logic;
    signal supervisor_d_w       : std_logic;

    signal vm_i_enable_w        : std_logic;
    signal vm_d_enable_w        : std_logic;

    --  TLB entry does not match request address
    signal itlb_miss_w          : std_logic;
    signal dtlb_miss_w          : std_logic;

    --  data miss is higher priority than instruction...
    signal request_addr_w       : slv_31_00;

    signal pte_addr_q           : slv_31_00;
    signal pte_addr_w           : slv_31_00;
    signal pte_entry_q          : slv_31_00;
    signal virt_addr_q          : slv_31_00;

    signal pte_ppn_w            : slv_31_00;
    signal pte_flags_w          : slv_09_00;

    -- - - - - - -
    --  IMMU TLB
    -- - - - - - -
    signal itlb_valid_w         : std_logic;
    signal itlb_valid_q         : std_logic;
    signal itlb_va_addr_q       : slv_31_12;
    signal itlb_entry_q         : slv_31_00;

    signal pc_fault_w           : std_logic;

    signal pc_fault_q           : std_logic;

    signal dtlb_valid_q         : std_logic;
    signal dtlb_va_addr_q       : slv_31_12;
    signal dtlb_entry_q         : slv_31_00;

    signal load_fault_w         : std_logic;

    signal store_fault_w        : std_logic;

    signal store_fault_q        : std_logic;
    signal load_fault_q         : std_logic;

    signal lsu_out_rd_w         : std_logic;
    signal lsu_out_wr_w         : slv_03_00;
    signal lsu_out_addr_w       : slv_31_00;

    signal lsu_out_invalidate_w : std_logic;

    signal lsu_out_cacheable_w  : std_logic;
    signal lsu_out_flush_w      : std_logic;

    -- - - - - - - - - -
    --  PTE fetch port
    -- - - - - - - - - -
    signal mem_req_q            : std_logic;
    signal mmu_accept_w         : std_logic;

    -- - - - - - - - - -
    --  request muxing
    -- - - - - - - - - -
    signal read_hold_q          : std_logic;
    signal src_mmu_q            : std_logic;
    signal src_mmu_w            : std_logic;

    signal lsu_out_rd_o_w       : std_logic;
    signal lsu_out_wr_o_w       : slv_t(dmem_wr_o'range);

    -- pre-adder to post addition  -- adds 20 + 10 + 1 regs
    signal resp_data_29_10_q    : slv_19_00;
    signal request_addr_21_12_q : slv_09_00;
    signal sel_entry_q          : std_logic;

  begin

    idle_w       <= eq(state_q, STATE_IDLE_B);
    resp_mmu_w   <= eq(dmem_resp_tag_i(9 downto 7),"111");
    resp_valid_w <= resp_mmu_w and dmem_ack_i;
    resp_error_w <= resp_mmu_w and dmem_error_i;
    resp_data_w  <= dmem_rdata_i;                     -- [31:0], from dcache


    p_ld : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          load_q <= '0';
        elsif (lsu_in_rd_i = '1') then
          load_q <= not lsu_in_accept_w;
        end if;
      end if;
    end process p_ld;


    p_st : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          store_q <= (others => '0');                 --  4b
        else
          if (not_zero(lsu_in_wr_i)) then             --  3:0
            if (lsu_in_accept_w = '1') then
              store_q <= (others => '0');             --  4b
            else
              store_q <= lsu_in_wr_i;
            end if;
          end if;
        end if;
      end if;
    end process p_st;


    load_w  <= lsu_in_rd_i or load_q;
    store_w <= lsu_in_wr_i or store_q;


    p_lsuiaddr : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          lsu_in_addr_q <= (others => '0');           -- 32b
        elsif ( load_w = '1' or not_zero(store_w) ) then
          lsu_in_addr_q <= lsu_in_addr_i;
        end if;
      end if;
    end process p_lsuiaddr;


    lsu_addr_w <= lsu_in_addr_i when (load_w = '1' or not_zero(store_w)) else
                  lsu_in_addr_q;                      -- [31:0]


    -- - - - - - - - - - -
    --  page table walker
    -- - - - - - - - - - -
    --
    -- satp bits (sv32) :
    --   [0]     V    VALID           :  PAGE_PRESENT
    --   [1]     R    READABLE        :  PAGE_READ
    --   [2]     W    WRITEABLE       :  PAGE_WRITE
    --   [3]     X    EXECUTABLE      :  PAGE_EXEC
    --   [4]     U    USER            :  PAGE_USER
    --   [5]     G    GLOBAL (shared) :  PAGE_USER
    --   [6]     A    ACCESSES        :  PAGE_USER
    --   [7]     D    DIRTY           :  PAGE_USER
    --   [9:8]        RSW             :  PAGE_SOFT
    --   [31:10]      Physical page number (22 bits)

    --  global enable
    vm_enable_w    <= satp_i(SATP_MODE_R);  -- 31;
    -- page table base register [31:0] :  19:0 & 11:0
    ptbr_w         <= satp_i(SATP_PPN_R'range) &  (11 downto 0 => '0');

    ifetch_vm_w    <= ne(fetch_in_priv_i, PRIV_MACHINE);
    dfetch_vm_w    <= ne(priv_d_i,        PRIV_MACHINE);

    supervisor_i_w <= eq(fetch_in_priv_i, PRIV_SUPER);
    supervisor_d_w <= eq(priv_d_i,        PRIV_SUPER);

    vm_i_enable_w  <= ifetch_vm_w;
    vm_d_enable_w  <= vm_enable_w and dfetch_vm_w;


    --  TLB entry does not match request address
    itlb_miss_w    <= fetch_in_rd_i and vm_i_enable_w and (not itlb_hit_w);

    dtlb_miss_w    <= (load_w or (not eq(store_w, (store_w'range => '0'))))
                      and vm_d_enable_w and (not dtlb_hit_w);


    request_addr_w <= lsu_addr_w when (idle_w = '1' and dtlb_miss_w = '1') else
                      fetch_in_pc_i when (idle_w ='1' and dtlb_miss_w ='0') else
                      lsu_addr_w when (idle_w ='0' and dtlb_req_q = '1') else
                      fetch_in_pc_i;

    -- pte_ppn : page table physical page number
    -- pfn     : page frame number
    pte_ppn_w   <= (PAGE_PFN_SHIFT-1 downto 0 => '0') &     -- 9:0
                   resp_data_w(31 downto PAGE_PFN_SHIFT);   -- 31:10
    pte_flags_w <= resp_data_w(9 downto 0);


    p_state : process (clk_i)
      variable pte_entry_v  : slv_t(pte_entry_q'range);
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          pte_addr_q  <= X"00000000";   -- 31:0
          pte_entry_q <= X"00000000";   -- 31:0
          virt_addr_q <= X"00000000";   -- 31:0
          dtlb_req_q  <= '0';
          state_q     <= STATE_IDLE_B;
          -- retiming adder
          resp_data_29_10_q    <= (others => '0');
          request_addr_21_12_q <= (others => '0');
          sel_entry_q          <= '0';
        else
          case state_q is                     --  TLB miss, walk page table
            when STATE_IDLE_B =>
              if (itlb_miss_w = '1' or dtlb_miss_w = '1') then
                pte_addr_q  <= addu((ptbr_w), -- page table base register
                                    (X"00000" &
                                     request_addr_w(31 downto 22)&"00"));
                virt_addr_q <= request_addr_w;
                dtlb_req_q  <= dtlb_miss_w;
                state_q     <= STATE_LEVEL_FIRST_B;
                sel_entry_q <= '0';
              end if;
            when STATE_LEVEL_FIRST_B =>      --  first level (4MB superpage)
              if (resp_valid_w = '1') then
                --  error or page not present
                if (resp_error_w = '1' or
                    resp_data_w(PAGE_PRESENT) = '0') then
                  pte_entry_q <= X"00000000";
                  state_q     <= STATE_UPDATE_B;
                  --  valid entry, but another level to fetch
                elsif (resp_data_w(PAGE_READ) = '0' and
                       resp_data_w(PAGE_WRITE) = '0' and
                       resp_data_w(PAGE_EXEC) = '0') then
                  resp_data_29_10_q    <= resp_data_w(29 downto 10);
                  request_addr_21_12_q <= request_addr_w(21 downto 12);
                  sel_entry_q          <= '1';

                  state_q <= STATE_LEVEL_SECOND_B;
                  --  valid entry, actual valid PTE
                else
                  pte_entry_v := pte_ppn_w(31-MMU_PGSHIFT downto 10) &
                                 ( pte_ppn_w(9 downto 0) or
                                   request_addr_w(21 downto 12) )  &
                                 (MMU_PGSHIFT-1 downto 0 => '0');  -- << 12
                  --
                  pte_entry_q             <= pte_entry_v;  -- 31:13 & X"000"

                  pte_entry_q(9 downto 0) <= pte_flags_w;  -- 9:0
                  state_q     <= STATE_UPDATE_B;
                end if;
              end if;
            when STATE_LEVEL_SECOND_B =>      --  second level (4KB page)
              if (resp_valid_w = '1') then
                --  valid entry, final level
                if (resp_data_w(PAGE_PRESENT) = '1') then             -- [0]
                  --                       << 12
                  -- PGSIZE   4096  - bytes mapped by a page
                  -- PGSHIFT  12    - log2(PGSIZE)
                  pte_entry_q <= pte_ppn_w(31-MMU_PGSHIFT downto 0) & -- 19:0
                                 (MMU_PGSHIFT-1 downto 0 => '0');     -- 11:0
                  pte_entry_q(9 downto 0)  <=  pte_flags_w;           -- 9:0
                  state_q     <= STATE_UPDATE_B;
                  --  page fault
                else
                  pte_entry_q <= (others => '0');                     -- 31:0
                  state_q     <= STATE_UPDATE_B;
                end if;
              end if;
              --
            when STATE_UPDATE_B =>
              state_q    <= STATE_IDLE_B;
            when others =>
              null;
          end case;
          --
        end if;
      end if;
    end process p_state;

    pte_addr_w <= resp_data_29_10_q & request_addr_21_12_q & "00"
                  when (sel_entry_q = '1')
                  else pte_addr_q;


    -- - - - - - -
    --  IMMU TLB
    -- - - - - - -
    p_itlb_valid : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          itlb_valid_q <= '0';
        else
          if (flush_i = '1') then
            itlb_valid_q <= '0';
          elsif ( state_q = STATE_UPDATE_B and dtlb_req_q = '0' ) then
            --  fetch TLB still matches incoming request
            itlb_valid_q <= itlb_valid_w;
          elsif ( state_q /= STATE_IDLE_B and dtlb_req_q = '0' ) then
            itlb_valid_q <= '0';
          end if;
        end if;
      end if;
    end process p_itlb_valid;


    p_itlb_entry : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          itlb_va_addr_q <= (others => '0');  -- 20b
          itlb_entry_q   <= (others => '0');  -- 32b
        else
          if (state_q = STATE_UPDATE_B and  dtlb_req_q = '0') then
            itlb_va_addr_q <= virt_addr_q(31 downto 12);
            itlb_entry_q   <= pte_entry_q;
          end if;
        end if;
      end if;
    end process p_itlb_entry;

    itlb_valid_w <= eq(itlb_va_addr_q, fetch_in_pc_i(31 downto 12));

    --  TLB address matched (even on page fault)
    itlb_hit_w <= fetch_in_rd_i and itlb_valid_q  and  itlb_valid_w;


    p_c_pc_fault : process (vm_i_enable_w, itlb_hit_w,
                            supervisor_i_w, itlb_entry_q)
    begin
      pc_fault_w <= '0';
      if (vm_i_enable_w = '1' and itlb_hit_w = '1') then
        if (supervisor_i_w = '1') then        --  supervisor mode
          --  user page, supervisor cannot execute
          if (itlb_entry_q(PAGE_USER) = '1') then
            pc_fault_w <= '1';
          else                                --  check exec permissions
            pc_fault_w <= not itlb_entry_q(PAGE_EXEC);
          end if;
        else                                  --  user mode
          pc_fault_w <= (not itlb_entry_q(PAGE_EXEC)) or
                        (not itlb_entry_q(PAGE_USER));
        end if;
      end if;
    end process p_c_pc_fault;


    p_pc_fault : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          pc_fault_q <= '0';
        else
          pc_fault_q <= pc_fault_w;
        end if;
      end if;
    end process p_pc_fault;


    fetch_out_rd_o         <= ((not vm_i_enable_w) and fetch_in_rd_i) or
                              (itlb_hit_w and (not pc_fault_w));
    imem_pc_o              <= itlb_entry_q(31 downto 12) &
                              fetch_in_pc_i(11 downto 0) when
                              vm_i_enable_w = '1' else fetch_in_pc_i;
    fetch_out_flush_o      <= fetch_in_flush_i;
    fetch_out_invalidate_o <= fetch_in_invalidate_i;

    fetch_in_accept_o      <= ((not vm_i_enable_w) and fetch_out_accept_i) or
                              (vm_i_enable_w and itlb_hit_w and
                               fetch_out_accept_i) or  pc_fault_w;
    fetch_in_valid_o       <= fetch_out_valid_i or pc_fault_q;
    fetch_in_error_o       <= fetch_out_valid_i and fetch_out_error_i;
    fetch_in_fault_o       <= pc_fault_q;


    -- - - - - - -
    --  DMMU TLB
    -- - - - - - -
    p_dtlb_valid : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          dtlb_valid_q <= '0';
        else
          if (flush_i = '1') then
            dtlb_valid_q <= '0';
          elsif (state_q = STATE_UPDATE_B and dtlb_req_q = '1') then
            dtlb_valid_q <= '1';
          end if;
        end if;
      end if;
    end process p_dtlb_valid;


    p_dtlb_entry : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          dtlb_va_addr_q <= (others => '0');  -- 20b
          dtlb_entry_q   <= (others => '0');  -- 32b
        else
          if (state_q = STATE_UPDATE_B and dtlb_req_q = '1') then
            dtlb_va_addr_q <= virt_addr_q(31 downto 12);
            dtlb_entry_q   <= pte_entry_q;
          end if;
        end if;
      end if;
    end process p_dtlb_entry;


    --  TLB address matched (even on page fault)
    dtlb_hit_w <= dtlb_valid_q and
                  eq(dtlb_va_addr_q, lsu_addr_w(31 downto 12));


    p_c_load_fault : process (vm_d_enable_w, load_w, dtlb_hit_w,
                              supervisor_d_w, sum_i, dtlb_entry_q, mxr_i)
    begin
      load_fault_w <= '0';
      if (vm_d_enable_w = '1' and load_w = '1' and dtlb_hit_w = '1') then
        --  supervisor mode
        if (supervisor_d_w = '1') then
          --  user page, supervisor user mode not enabled
          if (dtlb_entry_q(PAGE_USER) = '1' and sum_i = '0') then
            load_fault_w <= '1';
            --  check exec permissions
          else
            load_fault_w <= not (dtlb_entry_q(PAGE_READ) or
                                 (mxr_i and dtlb_entry_q(PAGE_EXEC)));
          end if;
          --  user mode
        else
          load_fault_w <= ( not dtlb_entry_q(PAGE_READ)) or
                          ( not dtlb_entry_q(PAGE_USER));
        end if;
      end if;
    end process p_c_load_fault;


    p_c_store_fault : process (vm_d_enable_w, store_w, dtlb_hit_w,
                               supervisor_d_w, dtlb_entry_q, sum_i)
    begin
      store_fault_w <= '0';
      if (vm_d_enable_w = '1' and not_zero(store_w) and dtlb_hit_w = '1') then
        --  supervisor mode
        if (supervisor_d_w = '1') then
          --  user page, supervisor user mode not enabled
          if (dtlb_entry_q(PAGE_USER) = '1' and sum_i = '0') then
            store_fault_w <= '1';
            --  check exec permissions
          else
            store_fault_w <= (not dtlb_entry_q(PAGE_READ)) or
                             (not dtlb_entry_q(PAGE_WRITE));
          end if;
          --   user mode
        else
          store_fault_w <= (not dtlb_entry_q(PAGE_READ)) or
                           (not dtlb_entry_q(PAGE_WRITE)) or
                           (not dtlb_entry_q(PAGE_USER));
        end if;
      end if;
    end process p_c_store_fault;


    p_ld_fault : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          load_fault_q <= '0';
        else
          load_fault_q <= load_fault_w;
        end if;
      end if;
    end process p_ld_fault;


    p_st_fault : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          store_fault_q <= '0';
        else
          store_fault_q <= store_fault_w;
        end if;
      end if;
    end process p_st_fault;



    lsu_out_rd_w         <= (load_w and dtlb_hit_w and
                             (not load_fault_w))
                            when (vm_d_enable_w = '1') else lsu_in_rd_i;
    lsu_out_wr_w         <= (store_w and
                             ( (dtlb_hit_w and (not store_fault_w)) &
                               (dtlb_hit_w and (not store_fault_w)) &
                               (dtlb_hit_w and (not store_fault_w)) &
                               (dtlb_hit_w and (not store_fault_w)) ) )
                            when (vm_d_enable_w = '1') else
                            lsu_in_wr_i;                           -- [3:0]

    lsu_out_addr_w(31 downto 12) <= dtlb_entry_q(31 downto 12)
                                    when (vm_d_enable_w = '1') else
                                    lsu_addr_w(31 downto 12);      -- [31:0]
    lsu_out_addr_w(11 downto 0)  <= lsu_addr_w(11 downto 0);


    lsu_out_invalidate_w         <= lsu_in_invalidate_i;


    p_c_lsu_o_c : process (lsu_in_invalidate_i, lsu_in_wb_i,
                           lsu_in_flush_i, lsu_out_addr_w)
    begin
      lsu_out_cacheable_w <= '0';
      if (lsu_in_invalidate_i = '1' or lsu_in_wb_i = '1' or
          lsu_in_flush_i = '1') then
        lsu_out_cacheable_w <= '1';
      else
        if ( (lsu_out_addr_w >= CACHE_ADDR_L) and
             (lsu_out_addr_w <= CACHE_ADDR_H) ) then
          lsu_out_cacheable_w <= '1';
        end if;
      end if;
    end process p_c_lsu_o_c;


    lsu_out_flush_w      <= lsu_in_flush_i;

    lsu_in_ack_o         <= (dmem_ack_i and (not resp_mmu_w)) or
                            store_fault_q or load_fault_q;
    lsu_in_error_o       <= (dmem_error_i and (not resp_mmu_w)) or
                            store_fault_q or load_fault_q;
    lsu_in_store_fault_o <= store_fault_q;
    lsu_in_load_fault_o  <= load_fault_q;

    lsu_in_accept_w      <= ((not vm_d_enable_w) and cpu_accept_w) or
                            (vm_d_enable_w and dtlb_hit_w and cpu_accept_w) or
                            store_fault_w or load_fault_w;

    -- - - - - - - - - -
    --  PTE fetch port
    -- - - - - - - - - -
    p_mem_req : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          mem_req_q <= '0';
        else
          if (state_q = STATE_IDLE_B and
              (itlb_miss_w = '1' or dtlb_miss_w = '1')) then
            mem_req_q <= '1';
          elsif ( state_q = STATE_LEVEL_FIRST_B and resp_valid_w = '1' and
                  resp_error_w = '0' and
                  resp_data_w(PAGE_PRESENT) = '1' and       -- [0]
                  ( resp_data_w(PAGE_READ)  = '0' and       -- [1]
                    resp_data_w(PAGE_WRITE) = '0' and       -- [2]
                    resp_data_w(PAGE_EXEC)  = '0' ) ) then  -- [3]
            mem_req_q <= '1';
          elsif (mmu_accept_w = '1') then
            mem_req_q <= '0';
          end if;
        end if;
      end if;
    end process p_mem_req;


    -- - - - - - - - - -
    --  request muxing
    -- - - - - - - - - -
    src_mmu_w <= src_mmu_q when (read_hold_q = '1') else mem_req_q;

    p_rd_hold : process (clk_i)
    begin
      if (clk_i'event and clk_i = '1') then
        if (xrst_i = '0') then
          read_hold_q    <= '0';
          src_mmu_q      <= '0';
        else
          if ( (lsu_out_rd_o_w = '1' or not_zero(lsu_out_wr_o_w) ) and
               dmem_accept_i = '0' ) then
            read_hold_q  <= '1';
            src_mmu_q    <= src_mmu_w;
          elsif (dmem_accept_i = '1') then
            read_hold_q  <= '0';
          end if;
        end if;
      end if;
    end process p_rd_hold;


    mmu_accept_w      <= src_mmu_w  and dmem_accept_i;
    cpu_accept_w      <= (not src_mmu_w) and dmem_accept_i;

    lsu_out_rd_o_w    <= mem_req_q when (src_mmu_w = '1') else
                         lsu_out_rd_w;
    lsu_out_wr_o_w    <= X"0" when (src_mmu_w = '1') else lsu_out_wr_w;

    dmem_invalidate_o <= '0' when (src_mmu_w = '1') else
                         lsu_out_invalidate_w;
    dmem_wb_o         <= '0' when (src_mmu_w = '1') else
                         lsu_in_wb_i;
    dmem_cacheable_o  <= '1' when (src_mmu_w = '1') else
                         lsu_out_cacheable_w;

    dmem_req_tag_o    <= "01110000000" when (src_mmu_w = '1')
                         else lsu_in_req_tag_i;   -- [10:0]
    dmem_flush_o      <= '0' when (src_mmu_w = '1') else lsu_out_flush_w;

    lsu_in_accept_o      <= lsu_in_accept_w;

    dmem_rd_o         <= lsu_out_rd_o_w;
    dmem_wr_o         <= lsu_out_wr_o_w;
    dmem_addr_o       <= pte_addr_w when (src_mmu_w = '1') else lsu_out_addr_w;
    dmem_wdata_o      <= lsu_in_wdata_i;


    dmem_accept_o     <= dmem_accept_i;
    dmem_rdata_o      <= dmem_rdata_i;


    -- NC in mmu version:  lsu_in_cacheable_i

    --  rvo_cnl <= lr_cancel_i and flush_i

    amo_pending_o     <= '0';

  end generate g_mmu;




  -- -----------------------------------------------------------------
  --  No MMU support
  -- -----------------------------------------------------------------
  g_nommu : if ( SUPPORT_MMU = '0' ) generate
  begin
    fetch_out_rd_o         <= fetch_in_rd_i;
    imem_pc_o              <= fetch_in_pc_i;          -- iaddr
    fetch_out_flush_o      <= fetch_in_flush_i;
    fetch_out_invalidate_o <= fetch_in_invalidate_i;
    fetch_in_accept_o      <= fetch_out_accept_i;
    fetch_in_valid_o       <= fetch_out_valid_i;
    fetch_in_error_o       <= fetch_out_error_i;
    fetch_in_fault_o       <= '0';

    dmem_cacheable_o       <= lsu_in_cacheable_i;
    dmem_req_tag_o         <= lsu_in_req_tag_i;
    dmem_rd_o              <= lsu_in_rd_i;
    dmem_wr_o              <= lsu_in_wr_i;
    dmem_addr_o            <= lsu_in_addr_i;          -- daddr
    dmem_wdata_o           <= lsu_in_wdata_i;
    dmem_invalidate_o      <= lsu_in_invalidate_i;
    dmem_wb_o              <= lsu_in_wb_i;
    dmem_flush_o           <= lsu_in_flush_i;

    dmem_accept_o          <= dmem_accept_i;
    dmem_rdata_o           <= dmem_rdata_i;

    lsu_in_ack_o           <= dmem_ack_i;
    lsu_in_error_o         <= dmem_error_i;
    lsu_in_store_fault_o   <= '0';
    lsu_in_load_fault_o    <= '0';

    lsu_in_accept_o        <= dmem_accept_i;

    amo_pending_o          <= '0';

  end generate g_nommu;

end rtl;
