-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--
-- Copyright 2025 VLSI Solution Oy
-- SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
--
-- Licensed under the Solderpad Hardware License v 2.1 (the "License");
-- you may not use this file except in compliance with the License, or,
-- at your option, the Apache License version 2.0.
-- You may obtain a copy of the License at
--
--    https://solderpad.org/licenses/SHL-2.1/
--
-- Unless required by applicable law or agreed to in writing, any work
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
--                               RISCV core
--                                 V1.1
--
--
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

-- - - - - - - -
--  libraries
-- - - - - - - -
library ieee;
use ieee.std_logic_1164.all;
use ieee.numeric_std.all;
use work.rv_pkg.all;


entity icachew is
  generic ( AXIDATA_W   : natural range 32 to 64 := 64;  -- Data width
            WAYS        : natural range 1 to 8   := 2;   -- Nro of ways 1,2,4,8
            LINE_ADDR_W : natural range 6 to 12  := 8;   -- Line address
            LINE_SIZE_W : natural range 2 to 8   := 5 ); -- Line size (bytes !)
  port ( clk_i            : in  std_logic;
         xrst_i           : in  std_logic;
         --  mem bist if
         --  (NOTE: mem count is 2x WAYS as there are tag/data rams)
         --  rd/wr select is 1-hot style in order like ...t1,d1,t0,d0
         --  NOTE: mbist uses xmem_rdata_i as its input data also
         mbist_ena_i      : in  std_logic;
         mbist_rd_i       : in  slv_t(2*WAYS-1 downto 0); -- 1-hot  t1,d1,t0,d0
         mbist_wr_i       : in  slv_t(2*WAYS-1 downto 0); -- 1-hot  t1,d1,t0,d0
         mbist_addr_i     : in  slv_t(LINE_ADDR_W+LINE_SIZE_W-1-2 downto 0);
         mbist_data_o     : out slv_t(AXIDATA_W-1 downto 0);
         -- mem inputs (from cpu)
         cpu_rd_i         : in  std_logic;
         cpu_flush_i      : in  std_logic;
         cpu_invalidate_i : in  std_logic;
         cpu_pc_i         : in  slv_31_00;
         -- mem outputs (to cpu)
         cpu_accept_o     : out std_logic;
         cpu_valid_o      : out std_logic;
         cpu_error_o      : out std_logic;
         cpu_inst_o       : out slv_31_00;
         --
         xmem_accept_i    : in     std_logic;
         xmem_ack_i       : in     std_logic;
         xmem_error_i     : in     std_logic;
         xmem_rdata_i     : in     slv_t(AXIDATA_W-1 downto 0); -- ddr / mbist
         xmem_rd_o        : buffer std_logic;
         xmem_wr_o        : out    slv_t(AXIDATA_W/8-1 downto 0);
         xmem_len_o       : out    slv_07_00;     -- burst len
         xmem_addr_o      : buffer slv_31_00;
         xmem_wdata_o     : out    slv_t(AXIDATA_W-1 downto 0)
         );
end entity icachew;


architecture rtl of icachew is

  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- Default cache instance is 2 way set associative.
  -- The total size is 16KB.
  -- The replacement policy is a limited pseudo random scheme
  -- (between lines, toggling on line thrashing).
  -- The cache is a write back cache, with allocate on read and write.
  --
  --   _________    _________________________________________________________
  --  |31|30|29|...|18|17|16|15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
  --   ---------    ---------------------------------------------------------
  --  |                              |                       | LINE   |     |
  --  |           TAG ADDR           |    LINE ADDR          | OFFSET | byte|
  --  |------------------------------|-----------------------|--------|-----|
  --        31:13, 2^19 = 512k         12:5, 2^8 = 256         2^3=8   ignore
  --
  --   -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
  --  | WAY-0 default implementation                                    |
  --
  --  |  addr[12:5] (line)      addr[12:5] & addr[4:2] (line+offset)    |
  --      |                      |
  --  |   |         TAG RAM      |         DATA RAM                     |
  --      |         256 x 21bit  |         256*8 x 32bit
  --  |   | 0x00   ___________   | 0x00   ____________                  |
  --      |------>|_ line 0 __|  |------>|_ data 0 ___|
  --  |   |         \      /     |       |_________+1_|                 |
  --      |                      |       |_________+2_|
  --  |   |                      |       |_________+3_|                 |
  --      |                      |       |_________+4_|
  --  |   |                      |       |_________+5_|                 |
  --      |                      |       |_________+6_|
  --  |   |                      |       |_________+6_|                 |
  --      | 0x01   ___________   |       |_________+7_|
  --  |   |------>|_ line 1 __|  |------>|_ data 7 ___|                 |
  --      |         \      /     |       |_________+1_|
  --  |   |                      |       |_________+2_|                 |
  --      |                      |       |_________+3_|
  --  |   |                      |       |_________+4_|                 |
  --      |                      |       |_________+5_|
  --  |   |                      |       |_________+6_|                 |
  --      | 0x02   ___________   |       |_________+7_|
  --  |   |------>|_ line 2 __|  |------>|_ data 16___|                 |
  --      |         \      /     |       |_________+1_|
  --  |   |                      |       |_________+2_|                 |
  --      |                      |       |_________+3_|
  --  |   |                      |       |_________+4_|                 |
  --      |                      |       |_________+5_|
  --  |   |                      |       |_________+6_|                 |
  --      |                      |       |_________+7_|
  --  |   |       . . . . . . . .| . . . . . . . . . . .                |
  --      | 0xFF   ___________   |       |____________|
  --  |   |------>|_ line 255_|  |------>|_ data 2040_|                 |
  --                \      /             |_________+1_|
  --  |                                  |_________+2_|                 |
  --                                     |_________+3_|
  --  |                                  |_________+4_|                 |
  --                                     |_________+5_|
  --  |                                  |_________+6_|                 |
  --                                     |_________+7_|
  --  |__ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __|
  --
  --
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -


  -- Address mapping example:                                  NOTE: OBSOLETE
  --                                                           =====
  --  31          16 15 14 13 12 11 10 09 08 07 06 05 04 03 02 01 00
  -- |--------------|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
  --  +--------------------+  +--------------------+   +------------+
  --  |  Tag address       |  |   Line address     |      Address
  --  |                    |  |                    |      within line
  --  |                    |  |                    |
  --  |                    |  |                    |- LINE_SIZE_W
  --  |                    |  |- LINE_ADDR_H
  --  |                    |- TAG_ADDR_L
  --  |- 31

  function f_is_axi64b (dwidth : natural ) return natural is
  begin
    if (dwidth = 64) then
      return 1;
    else
      return 0;
    end if;
  end function;

  -- TAG address / data widths
  constant TAG_ADDR_W   : natural range 15 to 20:= 32-LINE_ADDR_W-LINE_SIZE_W;
  constant TAG_DATA_W   : natural := TAG_ADDR_W + 1;  -- +1 for valid bit
  -- data address width
  constant DATA_ADDR_W  : natural := LINE_ADDR_W + LINE_SIZE_W-2-
                                     f_is_axi64b(AXIDATA_W);  -- words

  constant DATA_ADDR_R  : slv_t(DATA_ADDR_W+2+f_is_axi64b(AXIDATA_W)-1 downto
                                2+f_is_axi64b(AXIDATA_W)) := (others => '0');

  -- Number of cache lines (words in tag ram) : 2**LINE_ADDR_W

  constant TAG_ADDR_H   : natural := 31;
  constant TAG_ADDR_L   : natural := LINE_ADDR_W + LINE_SIZE_W;

  constant LINE_ADDR_H  : natural := LINE_ADDR_W + LINE_SIZE_W - 1;
  constant LINE_ADDR_L  : natural := LINE_SIZE_W;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  --  states
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  --  one hot decoding
  constant STATE_W         : natural := 3;
  constant STATE_FLUSH     : slv_t(STATE_W-1 downto 0) := "001"; -- 0
  constant STATE_LOOKUP    : slv_t(STATE_W-1 downto 0) := "010"; -- 1
  constant STATE_REFILL    : slv_t(STATE_W-1 downto 0) := "100"; -- 2

  constant STATE_FLUSH_B   : natural := 0;
  constant STATE_LOOKUP_B  : natural := 1;
  constant STATE_REFILL_B  : natural := 2;


  component tag_ram
    -- ambit translate off
    generic ( DATA_W             : natural range 8 to 32 := 32;
              ADDR_W             : natural range 4 to 32 := 8;
              ODELAY             : time := 3.5 ns;  -- clk to dout delay
              INITIALIZE_TO_ZERO : std_logic := '0' );
    -- ambit translate on
    port ( clk      : in  std_logic;
           xreset   : in  std_logic;  -- NC now in synthesis
           --  inputs
           rd       : in  std_logic;
           wr       : in  std_logic;
           addr     : in  slv_t(LINE_ADDR_W-1 downto 0);
           din      : in  slv_t(23 downto 0);   -- TAG_DATA_W-1 downto 0);
           --  outputs
           dout     : out slv_t(23 downto 0) ); -- TAG_DATA_W-1 downto 0);
  end component;


  component data_ram
    -- ambit translate off
    generic ( DATA_W             : natural range 8 to 64 := 32;
              ADDR_W             : natural range 4 to 32 := 8;
              ODELAY             : time := 4.5 ns;  -- clk to dout delay
              INITIALIZE_TO_ZERO : std_logic := '0' );
    -- ambit translate on
    port ( clk      : in  std_logic;
           xreset   : in  std_logic;  -- NC now in synthesis
           --  inputs
           rd       : in  std_logic;
           wr       : in  slv_t(AXIDATA_W/8-1 downto 0);
           addr     : in  slv_t(DATA_ADDR_W-1 downto 0);
           din      : in  slv_t(AXIDATA_W-1 downto 0);
           --  outputs
           dout     : out slv_t(AXIDATA_W-1 downto 0) );
  end component;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- registers / wires
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  type tag_data_t  is array (0 to WAYS-1) of slv_23_00;    -- TAG_DATA_W-1:0
  type tag_addr_t  is array (0 to WAYS-1) of slv_t(TAG_ADDR_W-1 downto 0);
  type data_data_t is array (0 to WAYS-1) of slv_t(AXIDATA_W-1 downto 0);

  --  states
  signal next_state_w       : slv_t(STATE_W-1 downto 0);
  signal state_q            : slv_t(STATE_W-1 downto 0);
  signal old_state_lookup_q : std_logic;

  signal invalidate_q       : std_logic;

  --  lookup validation
  signal lu_valid_q        : std_logic;
  signal lu_addr_q         : slv_31_00;                      -- 1:0 NC
  signal lu_line_addr_w    : slv_t(LINE_ADDR_W-1 downto 0);  -- lookup line addr
  signal lu_data_addr_w    : slv_t(DATA_ADDR_W-1 downto 0);  -- lookup data addr


  signal tag_addr_m_w      : slv_t(LINE_ADDR_W-1 downto 0);

  signal flush_addr_q      : slv_t(LINE_ADDR_W-1 downto 0);

  signal tag_din_m_w       : slv_t(TAG_DATA_W-1 downto 0);

  signal tag_rd_w          : std_logic;
  signal tag_wr_w          : slv_t(WAYS-1 downto 0);
  signal tag_mdout_w       : tag_data_t;  -- fixed 24 bit ram, extra bits zero
  signal tag_valid_w       : slv_t(WAYS-1 downto 0);
  signal tag_addr_bits_w   : tag_addr_t;

  signal tag_hit_w         : slv_t(WAYS-1 downto 0);
  signal tag_hit_any_w     : std_logic;

  signal data_addr_m_w     : slv_t(DATA_ADDR_W-1 downto 0);
  signal data_waddr_q      : slv_t(DATA_ADDR_W-1 downto 0);
  signal data_rd_w         : std_logic;
  type data_wr_t is array (0 to WAYS-1) of slv_t(AXIDATA_W/8-1 downto 0);
  signal data_wr_w         : data_wr_t;

  signal data_mdout_w      : data_data_t;  -- from mem
  signal data_masked_w     : data_data_t;  -- mem data or FFFF to andlogic

  --  data output mux
  signal instr_w           : slv_31_00;

  signal random_num_q      : slv_03_00;
  signal next_way_w        : slv_t(WAYS-1 downto 0);
  signal next_way_q        : slv_t(WAYS-1 downto 0);

  -- bypass and ack request during REFILLL
  signal refill_bypass_w   : std_logic;

  signal xmem_error_q      : std_logic;
  signal xmem_rlast_w      : std_logic;
  signal xmem_rdpending_q  : std_logic;

  --  cannot read output fixes
  signal cpu_accept_o_w    : std_logic;
  signal cpu_valid_o_w     : std_logic;


  -- signals for mbist
  -- NOTE: TAG ram is fixed to 24 bits, extra msb bits are zero in cache mode
  signal tag_maddr_w       : slv_t(tag_addr_m_w'range);
  signal data_maddr_w      : slv_t(data_addr_m_w'range);
  signal tag_mdin_w        : slv_t(23 downto 0);  -- tag_din_m_w'range);

  signal tag_mrd_w         : slv_t(WAYS-1 downto 0);
  signal tag_mwr_w         : slv_t(WAYS-1 downto 0);
  signal data_mrd_w        : slv_t(WAYS-1 downto 0);
  type data_mwr_t is array (0 to WAYS-1) of slv_t(AXIDATA_W/8-1 downto 0);
  signal data_mwr_w        : data_mwr_t;
  signal mbist_rd_i_q      : slv_t(mbist_rd_i'range); -- 1-hot  t1,d1,t0,d0

  -- pragma translate_off
  type state_dbg_t is (FLUSH, LOOKUP, REFILL, xx);
  signal state_dbg         : state_dbg_t;
  -- pragma translate_on

begin

  -- pragma translate_off
  --
  state_dbg <= FLUSH    when (state_q = STATE_FLUSH) else
               LOOKUP   when (state_q = STATE_LOOKUP) else
               REFILL   when (state_q = STATE_REFILL) else
               xx;
  -- pragma translate_on


  cpu_accept_o    <= cpu_accept_o_w;
  cpu_valid_o     <= cpu_valid_o_w;

  lu_line_addr_w  <= cpu_pc_i(LINE_ADDR_H downto LINE_ADDR_L);
  lu_data_addr_w  <= cpu_pc_i(DATA_ADDR_R'range);


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  --  lookup address and valid
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  p_lu_addr : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        lu_valid_q    <= '0';
        lu_addr_q     <= (others => '0');                         -- 32 bit
      else
        if (cpu_rd_i = '1' and cpu_accept_o_w = '1') then
          lu_valid_q  <= '1';
          lu_addr_q   <= cpu_pc_i;
        elsif (cpu_valid_o_w = '1') then
          lu_valid_q  <= '0';
        end if;
      end if;
    end if;
  end process p_lu_addr;



  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- TAG RAMS
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  -- tag RAM address
  p_c_tag_addr : process (flush_addr_q, state_q, lu_addr_q, lu_line_addr_w)
  begin
    --
    tag_addr_m_w    <= flush_addr_q;
    --
    if (state_q(STATE_FLUSH_B) = '1') then                       -- Cache flush
      tag_addr_m_w  <= flush_addr_q;
    elsif (state_q(STATE_REFILL_B) = '1') then                   -- Line refill
      tag_addr_m_w  <= lu_addr_q(LINE_ADDR_H downto LINE_ADDR_L);
    else                                                         -- Lookup
      tag_addr_m_w  <= lu_line_addr_w;
    end if;
  end process p_c_tag_addr;


  -- tag RAM write data
  p_c_tag_data : process (state_q, lu_addr_q)
  begin
    tag_din_m_w    <= (others => '0');                           -- TAG_DATA_W
    --
    if (state_q(STATE_FLUSH_B) = '1') then                       -- Cache flush
      tag_din_m_w  <= (others => '0');
    elsif (state_q(STATE_REFILL_B) = '1') then                   -- Line refill
      tag_din_m_w(TAG_ADDR_W)            <= '1';                 -- valid bit
      tag_din_m_w(TAG_ADDR_W-1 downto 0) <=
        lu_addr_q(TAG_ADDR_H downto TAG_ADDR_L);
    end if;
  end process p_c_tag_data;


  -- tag read signal, common for all LINEs
  tag_rd_w <= state_q(STATE_LOOKUP_B) or
              (state_q(STATE_REFILL_B) and xmem_rlast_w );


  -- data read signal, common for all LINEs
  data_rd_w <= state_q(STATE_LOOKUP_B);


  -- ram read is pipelined
  p_mbist : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        mbist_rd_i_q <= (others => '0');
      else
        mbist_rd_i_q <= mbist_rd_i;
        if (mbist_ena_i = '0') then
          mbist_rd_i_q <= (others => '0');
        end if;
      end if;
    end if;
  end process p_mbist;


  -- data RAM address
  p_c_data_addr : process (lu_data_addr_w, state_q, data_waddr_q)
  begin
    data_addr_m_w    <= lu_data_addr_w;                  -- lookup
    --
    if (state_q(STATE_REFILL_B) = '1') then              -- line refill
      data_addr_m_w  <= data_waddr_q;
    end if;
  end process p_c_data_addr;


  -- NOTE: tad ram data width is fixed to 24-bits in asic
  p_c_mbisti : process (tag_addr_m_w, data_addr_m_w,  tag_din_m_w,
                        mbist_ena_i, mbist_addr_i, xmem_rdata_i)
  begin
    tag_maddr_w <= (others => '0');
    tag_maddr_w(tag_addr_m_w'range)  <=  tag_addr_m_w;   -- common for all WAYs
    --
    data_maddr_w <= (others => '0');
    data_maddr_w <= data_addr_m_w;                       -- common for all WAYs
    --
    tag_mdin_w   <= (others => '0');
    tag_mdin_w(tag_din_m_w'range)   <= tag_din_m_w;      --  common for all WAYs
    if (mbist_ena_i = '1') then
      tag_maddr_w  <= mbist_addr_i(tag_maddr_w'range);
      data_maddr_w <= mbist_addr_i(data_maddr_w'range);
      -- NOTE: mbist din data muxed in higher level to xmem_rdata_i
      tag_mdin_w   <= xmem_rdata_i(tag_mdin_w'range);
    end if;
  end process p_c_mbisti;


  g_rams : for ii in 0 to WAYS-1 generate
    --
    -- tag RAMs  (semi synchronous ram for tag addr)
    --
    u_tag : tag_ram
      -- ambit translate off
      generic map ( DATA_W             => 24,        -- TAG_DATA_W
                    ADDR_W             => LINE_ADDR_W,
                    ODELAY             => 3.5 ns,
                    INITIALIZE_TO_ZERO => '0' )
      -- ambit translate on
      port map ( clk      => clk_i,
                 xreset   => xrst_i,
                 --  inputs
                 rd       => tag_mrd_w(ii),          -- common for all WAYs
                 wr       => tag_mwr_w(ii),
                 addr     => tag_maddr_w,
                 din      => tag_mdin_w,             -- 23:0  cache 17 & 16:0
                 --  outputs
                 dout     => tag_mdout_w(ii) );      -- 23:0, cache 17 & 16:0


    --
    -- data RAMs  (semi synchronous ram for data)
    --
    u_data : data_ram
      -- ambit translate off
      generic map ( DATA_W             => AXIDATA_W,
                    ADDR_W             => DATA_ADDR_W,
                    ODELAY             => 4.5 ns,
                    INITIALIZE_TO_ZERO => '0' )
      -- ambit translate on
      port map ( clk      => clk_i,
                 xreset   => xrst_i,
                 --  inputs
                 rd       => data_mrd_w(ii),         -- common read enable
                 wr       => data_mwr_w(ii),
                 addr     => data_maddr_w,
                 din      => xmem_rdata_i,
                 --  outputs
                 dout     => data_mdout_w(ii) );



    -- tag RAM write enable  (Cache flush or Line refill)
    tag_wr_w(ii)       <= state_q(STATE_FLUSH_B) or
                          ( state_q(STATE_REFILL_B) and next_way_q(ii) and
                            old_state_lookup_q and state_q(STATE_REFILL_B));

    -- data RAM write enable (way 0 to WAYS-1)
    -- ! ! ! AXI bus always writes all bytes
    data_wr_w(ii)    <= ( AXIDATA_W/8-1 downto 0 => (state_q(STATE_REFILL_B) and
                                                     xmem_ack_i and
                                                     next_way_q(ii)) );


    tag_valid_w(ii)    <= tag_mdout_w(ii)(TAG_ADDR_W);            -- valid bit
    tag_addr_bits_w(ii)<= tag_mdout_w(ii)(TAG_ADDR_W-1 downto 0);
    tag_hit_w(ii)      <= tag_valid_w(ii) and
                          eq(tag_addr_bits_w(ii),
                             lu_addr_q(TAG_ADDR_H downto TAG_ADDR_L));


    data_masked_w(ii)  <= data_mdout_w(ii) when (tag_hit_w(ii) = '1')
                          else (others => '1' );

    --
    -- mbist signal order :  tn, dn, . . . t1,d1, t0,d0
    --
    -- in cache mode the data in and rd enable is common for all WAYs
    tag_mrd_w(ii)      <= mbist_rd_i(2*ii+1) when (mbist_ena_i = '1') else
                          tag_rd_w;
    tag_mwr_w(ii)      <= mbist_wr_i(2*ii+1) when (mbist_ena_i = '1') else
                          tag_wr_w(ii);
    data_mrd_w(ii)     <= mbist_rd_i(2*ii)   when (mbist_ena_i = '1') else
                          data_rd_w;
    data_mwr_w(ii)     <= (AXIDATA_W/8-1 downto 0 => mbist_wr_i(2*ii))
                          when (mbist_ena_i = '1')
                          else data_wr_w(ii);
  end generate g_rams;


  tag_hit_any_w <= '1' when (unsigned(tag_hit_w) /= 0) else '0';


  --
  -- MBIST out mux
  --
  p_c_mbisto : process (mbist_ena_i, mbist_rd_i_q, tag_mdout_w, data_mdout_w)
  begin
    mbist_data_o <= (others => '0');
    if (mbist_ena_i = '1') then
      -- i.e. when 2-way cache => 4 mems (3, 2, 1, 0) => t1, d1, t0, d0
      for ii in WAYS-1 downto 0 loop   -- i.e. 2*WAYS-1 downto 0,  1-hot
        --
        -- ! ! ! NOTE: THIS RETURNS THE MEMORY DATA OF THE LAST BIT THAT IS '1'
        --
        if (mbist_rd_i_q(2*ii+1) = '1') then
          mbist_data_o(31 downto TAG_DATA_W)  <= (others => '0');
          -- mbist_data_o(TAG_DATA_W-1 downto 0) <= tag_mdout_w(ii);
          mbist_data_o(23 downto 0) <= tag_mdout_w(ii);
        end if;
        if (mbist_rd_i_q(2*ii) = '1') then
          mbist_data_o <= data_mdout_w(ii);
        end if;
      end loop;
    end if;
  end process p_c_mbisto;



  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- DATA RAMS
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  -- data RAM refill write address
  p_data_wr_addr : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        data_waddr_q    <= (others => '0');  --  DATA_ADDR_W
      else
        if (state_q(STATE_LOOKUP_B) = '1' and
            next_state_w(STATE_REFILL_B) = '1') then
          data_waddr_q  <= xmem_addr_o(DATA_ADDR_R'range);
        elsif (state_q(STATE_REFILL_B) = '1' and xmem_ack_i = '1') then
          data_waddr_q  <= inc1u(data_waddr_q);
        end if;
      end if;
    end if;
  end process p_data_wr_addr;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- dlush counter
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  p_flush_addr : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0' or mbist_ena_i = '1') then
        flush_addr_q    <= (others => '0');   -- LINE_ADDR_W
      else
        if (state_q(STATE_FLUSH_B) = '1') then
          flush_addr_q  <=  slv_t(unsigned(flush_addr_q) + 1);
        elsif (cpu_invalidate_i = '1' and cpu_accept_o_w = '1') then
          -- Invalidate specified line
          flush_addr_q  <= lu_line_addr_w;
        else
          flush_addr_q  <= (others => '0');
        end if;
      end if;
    end if;
  end process p_flush_addr;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- replacement policy
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- Using random replacement policy - this way we cycle through the ways
  -- when needing to replace a line.
  p_replace_way : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        next_way_q     <= (others => '0');
        next_way_q(0)  <= '1';             -- ...001
      else
        if (state_q(STATE_LOOKUP_B) = '1' and
            next_state_w(STATE_REFILL_B) = '1') then
          next_way_q   <= next_way_w;
        end if;
      end if;
    end if;
  end process p_replace_way;


  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  -- function to select the way to use for fills when WAYS = 1
  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  g_pick_1w : if ( WAYS = 1 ) generate
    next_way_w <= "1";
  end generate g_pick_1w;


  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  -- function to select the way to use for fills when WAYS = 2
  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  g_pick_2w : if ( WAYS = 2 ) generate
    -- pick_way_2ways
    b_pick_2w : block
      --
      function pick_way ( valid_bits : slv_t(WAYS-1 downto 0);
                          random_num : slv_03_00;
                          curr_way   : slv_t(1 downto 0) )
        return slv_t is  -- [WAYS-1:0]
        variable pick_way_v :  slv_t(WAYS-1 downto 0);
      begin
        if ( valid_bits(0) = '0' ) then
          -- way 0 not occupied so use it
          pick_way_v   := "01";           -- ONE HOT encoding in pick_way
        elsif ( valid_bits(1) = '0' ) then
          -- way 1 not occupied so use it
          pick_way_v   := "10";           -- ONE HOT encoding in pick_way
        else
          -- all ways occupied
          pick_way_v := not curr_way;
        --
        end if;
        return pick_way_v;
      end function pick_way;
    --
    begin
      next_way_w <= pick_way( tag_valid_w, random_num_q, next_way_q );
    end block b_pick_2w;
  end generate g_pick_2w;


  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  -- function to select the way to use for fills when WAYS = 4
  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  g_pick_4w : if ( WAYS = 4 ) generate
    -- pick_way_4ways
    b_pick_4w : block
      --
      function pick_way ( valid_bits : slv_t(WAYS-1 downto 0);
                          random_num : slv_03_00 )
        return slv_t is  -- [WAYS-1:0)
        variable pick_way_v :  slv_t(WAYS-1 downto 0);
      begin
        if ( valid_bits(0) = '0' ) then
          -- way 0 not occupied so use it
          pick_way_v     := "0001";
        elsif ( valid_bits(1) = '0' ) then
          -- way 1 not occupied so use it
          pick_way_v     := "0010";
        elsif ( valid_bits(2) = '0' )  then
          -- way 2 not occupied so use it
          pick_way_v     := "0100";
        elsif ( valid_bits(3) = '0' ) then
          -- way 3 not occupied so use it
          pick_way_v     := "1000";
        else
          -- all ways occupied so pick one randomly
          case random_num(3 downto 1) is
            when "000" | "001" =>
              pick_way_v := "0100";
            when "010" | "011" =>
              pick_way_v := "1000";
            when "100" | "101" =>
              pick_way_v := "0001";
            when others =>
              pick_way_v := "0010";
          end case;
        end if;
        return pick_way_v;
      end function pick_way;
    --
    begin
      next_way_w <= pick_way( tag_valid_w, random_num_q );
    end block b_pick_4w;
  end generate g_pick_4w;


  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  -- function to select the way to use for fills when WAYS = 8
  -- = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
  g_pick_8w : if (  WAYS /= 1 and  WAYS /= 2 and  WAYS /= 3 and
                    WAYS /= 4 and  WAYS /= 5 and  WAYS /= 6 and
                    WAYS /= 7) generate
    --  pick_way_8ways
    b_pick_8w : block
      --
      function pick_way ( valid_bits : slv_t(WAYS-1 downto 0);
                          random_num : slv_03_00 )
        return slv_t is  -- [WAYS-1:0)
        variable pick_way_v :  slv_t(WAYS-1 downto 0);
      begin
        if ( valid_bits(0) = '0' ) then
          -- way 0 not occupied so use it
          pick_way_v     := "00000001";
        elsif ( valid_bits(1) = '0' ) then
          -- way 1 not occupied so use it
          pick_way_v     := "00000010";
        elsif ( valid_bits(2) = '0' ) then
          -- way 2 not occupied so use it
          pick_way_v     := "00000100";
        elsif ( valid_bits(3) = '0' ) then
          -- way 3 not occupied so use it
          pick_way_v     := "00001000";
        elsif ( valid_bits(4) = '0' ) then
          -- way 3 not occupied so use it
          pick_way_v     := "00010000";
        elsif ( valid_bits(5) = '0' ) then
          -- way 3 not occupied so use it
          pick_way_v     := "00100000";
        elsif ( valid_bits(6) = '0' ) then
          -- way 3 not occupied so use it
          pick_way_v     := "01000000";
        elsif ( valid_bits(7) = '0' ) then
          -- way 3 not occupied so use it
          pick_way_v     := "10000000";
        else
          -- all ways occupied so pick one randomly
          case random_num(3 downto 1) is
            when "000"  =>  pick_way_v := "00010000";
            when "001"  =>  pick_way_v := "00100000";
            when "010"  =>  pick_way_v := "01000000";
            when "011"  =>  pick_way_v := "10000000";
            when "100"  =>  pick_way_v := "00000001";
            when "101"  =>  pick_way_v := "00000010";
            when "110"  =>  pick_way_v := "00000100";
            when others =>  pick_way_v := "00001000";
          end case;
        end if;
        return pick_way_v;
      end function pick_way;
    --
    begin
      next_way_w <= pick_way( tag_valid_w, random_num_q );
    end block b_pick_8w;
  end generate g_pick_8w;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- instruction output
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  g_axi32o : if ( AXIDATA_W /= 64 ) generate
  begin
    -- data output mux
    p_c_inst_mux : process (data_masked_w)
      variable instr_v  : slv_t(AXIDATA_W-1 downto 0);
    begin
      instr_v := data_masked_w(0);
      for ii in 0 to WAYS-1 loop
        instr_v := instr_v and data_masked_w(ii);
      end loop;
      instr_w <= instr_v;
    end process p_c_inst_mux;

    cpu_inst_o <= xmem_rdata_i when (refill_bypass_w = '1') else instr_w;
  end generate g_axi32o;


  g_axi64o : if ( AXIDATA_W = 64 ) generate
  begin
    -- data output mux
    p_c_inst_mux : process (data_masked_w, lu_addr_q)
      variable instr_v  : slv_t(AXIDATA_W-1 downto 0);
    begin
      instr_v := data_masked_w(0);
      for ii in 0 to WAYS-1 loop
        instr_v := instr_v and data_masked_w(ii);
      end loop;
      instr_w <= instr_v(31 downto 0);
      if (lu_addr_q(2) = '1') then
        instr_w <= instr_v(63 downto 32);
      end if;
    end process p_c_inst_mux;

    cpu_inst_o <= xmem_rdata_i(63 downto 32) when (refill_bypass_w = '1' and
                                                   lu_addr_q(2) = '1')   else
                  xmem_rdata_i(31 downto 0) when (refill_bypass_w = '1') else
                  instr_w;
  end generate g_axi64o;




  -- bypass data and ack as data available during REFILL
  cpu_valid_o_w <= (lu_valid_q and state_q(STATE_LOOKUP_B) and
                    tag_hit_any_w) or  (refill_bypass_w);

  -- bypass condition (64bit / 32bit)
  refill_bypass_w <= state_q(STATE_REFILL_B) and
                     eq(lu_addr_q(LINE_SIZE_W-1 downto 2),
                        data_waddr_q(LINE_SIZE_W-2-1 downto 0)) and
                     xmem_ack_i  when (AXIDATA_W /= 64) else
                     --
                     state_q(STATE_REFILL_B) and
                     eq(lu_addr_q(LINE_SIZE_W-1 downto 3),          -- 4:3
                        data_waddr_q(LINE_SIZE_W-3-1 downto 0)) and -- 1:0
                     xmem_ack_i ;



  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- next state logic
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  --
  --                          xrst_i == '0'
  --                              |
  --                              |
  --                             \/
  --                         STATE_FLUSH <--------------
  --                              |                     |
  --  invalidate_q == '1'  OR     |                     |
  --  flush_addr_q == X"FF"       |                     |
  --                             \/                     |
  --                         STATE_LOOKUP --------------
  --                              |    /\         cpu_invalidate_i == '1' OR
  --  lu_valid_q == '1' AND       |     \         cpu_flush_i      == '1'
  --  tag_hit_any_w == '0'        |      \        (Invalidate a line /
  --  (Tried a lookup but         |       \        flush cache)
  --   no match found)           \/        \
  --                         STATE_REFILL   \
  --                              |          |
  --  xmem_rvalid_i == '1' AND    |- - - - ->|
  --  xmem__last_w  == '1
  --
  --

  p_next_state : process (state_q, invalidate_q, flush_addr_q, lu_valid_q,
                          tag_hit_any_w, cpu_invalidate_i, cpu_flush_i,
                          xmem_ack_i, xmem_rlast_w)
  begin
    --
    next_state_w <= state_q;
    --
    case state_q is
      -- - - - - - - - - - - - - - - - - - - - -
      -- STATE_FLUSH
      -- - - - - - - - - - - - - - - - - - - - -
      when STATE_FLUSH =>
        if (invalidate_q = '1') then
          next_state_w <= STATE_LOOKUP;
        elsif (unsigned(not flush_addr_q) = 0) then   -- all ONES => last addr
          next_state_w <= STATE_LOOKUP;
        end if;
      -- - - - - - - - - - - - - - - - - - - - -
      -- STATE_LOOKUP
      -- - - - - - - - - - - - - - - - - - - - -
      when STATE_LOOKUP =>
        -- tried a lookup but no match found
        if (lu_valid_q = '1' and  tag_hit_any_w = '0') then
          next_state_w <= STATE_REFILL;
        -- invalidate a line / flush cache
        elsif (cpu_invalidate_i = '1' or cpu_flush_i = '1') then
          next_state_w <= STATE_FLUSH;
        end if;
      -- - - - - - - - - - - - - - - - - - - - -
      -- STATE_REFILL
      -- - - - - - - - - - - - - - - - - - - - -
      when STATE_REFILL =>
        -- end of refill
        if (xmem_ack_i = '1' and  xmem_rlast_w = '1') then
          next_state_w <= STATE_LOOKUP;
        end if;
      when others =>
        next_state_w <= STATE_FLUSH;
    end case;
  end process p_next_state;


  -- update state
  p_state : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if ( xrst_i = '0' ) then
        state_q            <= STATE_FLUSH;
        old_state_lookup_q <= '0';
        random_num_q       <= X"F";
      else
        state_q            <= next_state_w;
        old_state_lookup_q <= state_q(STATE_LOOKUP_B);
        random_num_q       <= random_num_q(2) & random_num_q(1) &
                              random_num_q(0) &
                              ( random_num_q(3) xor random_num_q(2) );
      end if;
    end if;
  end process p_state;


  -- state_q == STATE_LOOKUP and  next_state_w /= STATE_REFILL
  cpu_accept_o_w <= state_q(STATE_LOOKUP_B) and
                    ( not next_state_w(STATE_REFILL_B) );


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- invalidate
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  p_invalidate : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        invalidate_q   <= '0';
      else
        if (cpu_invalidate_i = '1' and cpu_accept_o_w = '1') then
          invalidate_q   <= '1';
        else
          invalidate_q   <= '0';
        end if;
      end if;
    end if;
  end process p_invalidate;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- XMEM error handling
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  p_perror : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        xmem_error_q   <= '0';
      else
        if (xmem_ack_i = '1' and xmem_error_i = '1') then
          xmem_error_q   <= '1';
        elsif (cpu_valid_o_w = '1') then
          xmem_error_q   <= '0';
        end if;
      end if;
    end if;
  end process p_perror;


  cpu_error_o <= xmem_error_q;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- XMEM request hold
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  p_axi_arvalid : process (clk_i)
  begin
    if (clk_i'event and clk_i = '1') then
      if (xrst_i = '0') then
        xmem_rdpending_q   <= '0';
      else
        if (xmem_rd_o = '1' and xmem_accept_i = '1') then
          xmem_rdpending_q  <= '0';           -- read was accepted
        elsif (xmem_rd_o = '1' and xmem_accept_i = '0') then
          xmem_rdpending_q  <= '1';           -- in queue, down strem busy
        end if;
        --
        if (xmem_rdpending_q = '1' and xmem_accept_i = '1') then
          xmem_rdpending_q  <= '0';
        end if;
      end if;
    end if;
  end process p_axi_arvalid;


  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  -- XMEM  (read only)
  -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  xmem_rlast_w <= '1' when (to_uint(data_waddr_q(LINE_SIZE_W-2-1 downto 0)) =
                            2**LINE_SIZE_W/4-1 and xmem_ack_i = '1' and
                            AXIDATA_W /= 64) else
                  '1' when (to_uint(data_waddr_q(LINE_SIZE_W-3-1 downto 0)) =
                            2**LINE_SIZE_W/8-1 and xmem_ack_i = '1' and
                            AXIDATA_W = 64) else
                  '0';

  xmem_rd_o    <= (state_q(STATE_LOOKUP_B) and next_state_w(STATE_REFILL_B)) or
                  xmem_rdpending_q;


  xmem_addr_o  <= lu_addr_q(31 downto LINE_SIZE_W) &
                  (LINE_SIZE_W-1 downto 0 => '0') when (xmem_rd_o = '1')
                  else X"00000000";

  xmem_len_o <= slv_t(to_unsigned((2**(LINE_SIZE_W-2-f_is_axi64b(AXIDATA_W))-1),
                                  xmem_len_o'length));

  xmem_wr_o    <= (others => '0');    -- no write channel
  xmem_wdata_o <= (others => '0');    -- no write channel


end rtl;
