/*****************************************************************************\ | Copyright (C) 2021-2022 Luke Wren | | SPDX-License-Identifier: Apache-2.0 | \*****************************************************************************/ `default_nettype none module hazard3_frontend #( `include "hazard3_config.vh" ) ( input wire clk, input wire rst_n, // Fetch interface // addr_vld may be asserted at any time, but after assertion, // neither addr nor addr_vld may change until the cycle after addr_rdy. // There is no backpressure on the data interface; the front end // must ensure it does not request data it cannot receive. // addr_rdy and dat_vld may be functions of hready, and // may not be used to compute combinational outputs. output wire mem_size, // 1'b1 -> 32 bit access output wire [W_ADDR-1:0] mem_addr, output wire mem_priv, output wire mem_addr_vld, input wire mem_addr_rdy, input wire [W_DATA-1:0] mem_data, input wire mem_data_err, input wire mem_data_vld, // Jump/flush interface // Processor may assert vld at any time. The request will not go through // unless rdy is high. Processor *may* alter request during this time. // Inputs must not be a function of hready. input wire [W_ADDR-1:0] jump_target, input wire jump_priv, input wire jump_target_vld, output wire jump_target_rdy, // Interface to the branch target buffer. `src_addr` is the address of the // last halfword of a taken backward branch. The frontend redirects fetch // such that `src_addr` appears to be sequentially followed by `target`. input wire btb_set, input wire [W_ADDR-1:0] btb_set_src_addr, input wire btb_set_src_size, input wire [W_ADDR-1:0] btb_set_target_addr, input wire btb_clear, output wire [W_ADDR-1:0] btb_target_addr_out, // Interface to Decode output reg [31:0] cir, // Current instruction register; pre-expanded to 32-bit output wire [31:0] cir_raw, // Unexpanded instruction data output reg [1:0] cir_vld, // number of valid halfwords in CIR input wire [1:0] cir_use, // number of halfwords D intends to consume // *may* be a function of hready output wire [1:0] cir_err, // Bus error on upper/lower halfword of CIR. output wire [1:0] cir_predbranch, // Set for last halfword of a predicted-taken branch output wire cir_break_any, // Set for exact match of a breakpoint address on CIR LSB output wire cir_break_d_mode, // As above but specifically break to debug mode output reg cir_is_32bit, // Can't be decoded from CIR due to pre-expansion output reg cir_invalid_16bit, // Expanded an invalid 32-bit instruction output reg cir_is_uop, // Current instruction is part of a micro-op sequence output reg cir_uop_nonfinal, // ...and there are more to follow in this instruction output reg cir_uop_no_pc_update, // Suppress PC increment or jump (note the jump in cm.popret is not the final uop!) output reg cir_uop_atomic, // Prevent IRQ entry, so intermediate states are not observed input wire uop_stall, input wire uop_clear, // "flush_behind": do not flush the oldest instruction when accepting a // jump request (but still flush younger instructions). Sometimes a // stalled instruction may assert a jump request, because e.g. the stall // is dependent on a bus stall signal so can't gate the request. input wire cir_flush_behind, // Required for regnum predecode when Zilsd is enabled: input wire df_lspair_phase_next, // Signal to power controller that power down is safe. (When going to // sleep, first the pipeline is stalled, and then the power controller // waits for the frontend to naturally come to a halt before releasing // its power request. This avoids manually halting the frontend.) output wire pwrdown_ok, // Signal to delay the first instruction fetch following reset, because // powerup has not yet been negotiated. input wire delay_first_fetch, // Provide the rs1/rs2 register numbers which will be in CIR next cycle. // Coarse: valid if this instruction has a nonzero register operand. // (Suitable for regfile read) output reg [4:0] predecode_rs1_coarse, output reg [4:0] predecode_rs2_coarse, // Fine: like coarse, but accurate zeroing when the operand is implicit. // (Suitable for bypass. Still not precise enough for stall logic.) output reg [4:0] predecode_rs1_fine, output reg [4:0] predecode_rs2_fine, // Debugger instruction injection: instruction fetch is suppressed when in // debug halt state, and the DM can then inject instructions into the last // entry of the prefetch queue using the vld/rdy handshake. input wire debug_mode, input wire [W_DATA-1:0] dbg_instr_data, input wire dbg_instr_data_vld, output wire dbg_instr_data_rdy, // PMP query->kill interface for X permission checks output wire [W_ADDR-1:0] pmp_i_addr, output wire pmp_i_m_mode, input wire pmp_i_kill, // Trigger unit query->break interface for breakpoints output wire [W_ADDR-1:0] trigger_addr, output wire trigger_m_mode, input wire [1:0] trigger_break_any, input wire [1:0] trigger_break_d_mode ); `include "rv_opcodes.vh" localparam W_BUNDLE = 16; // This is the minimum for full throughput (enough to avoid dropping data when // decode stalls) and there is no significant advantage to going larger. localparam FIFO_DEPTH = 2; // ---------------------------------------------------------------------------- // Fetch queue wire jump_now = jump_target_vld && jump_target_rdy; reg [1:0] mem_data_hwvld; // PMP X faults are checked in parallel with the fetch (fine if executable // memory is read-idempotent) and failures are promoted to bus errors: wire pmp_kill_fetch_dph; wire mem_or_pmp_err = mem_data_err || pmp_kill_fetch_dph; // Similarly, breakpoint matches are checked during fetch data phase. These // are called mem_xxx because they are the breakpoint metadata for the data // coming back from memory in this dphase. wire [1:0] mem_break_any; wire [1:0] mem_break_d_mode; // Mark data as containing a predicted-taken branch instruction so that // mispredicts can be recovered -- need to track both halfwords so that we // can mark the entire instruction, and nothing but the instruction: reg [1:0] mem_data_predbranch; // Bus errors (and other metadata) travel alongside data. They cause an // exception if the core decodes the instruction, but until then can be // flushed harmlessly. reg [W_DATA-1:0] fifo_mem [0:FIFO_DEPTH]; reg fifo_err [0:FIFO_DEPTH]; reg [1:0] fifo_break_any [0:FIFO_DEPTH]; reg [1:0] fifo_break_d_mode [0:FIFO_DEPTH]; reg [1:0] fifo_predbranch [0:FIFO_DEPTH]; reg [1:0] fifo_valid_hw [0:FIFO_DEPTH]; reg fifo_valid [0:FIFO_DEPTH]; reg fifo_valid_m1 [0:FIFO_DEPTH]; wire [W_DATA-1:0] fifo_rdata = fifo_mem[0]; wire fifo_full = fifo_valid[FIFO_DEPTH - 1]; wire fifo_empty = !fifo_valid[0]; wire fifo_almost_full = fifo_valid[FIFO_DEPTH - 2]; wire fifo_push; wire fifo_pop; wire fifo_dbg_inject = DEBUG_SUPPORT && dbg_instr_data_vld && dbg_instr_data_rdy; always @ (*) begin: boundary_conditions integer i; fifo_mem[FIFO_DEPTH] = mem_data; fifo_predbranch[FIFO_DEPTH] = 2'b00; fifo_err[FIFO_DEPTH] = 1'b0; fifo_break_any[FIFO_DEPTH] = 2'b00; fifo_break_d_mode[FIFO_DEPTH] = 2'b00; for (i = 0; i <= FIFO_DEPTH; i = i + 1) begin fifo_valid[i] = |EXTENSION_C ? |fifo_valid_hw[i] : fifo_valid_hw[i][0]; // valid-to-right condition: i == 0 || fifo_valid[i - 1], but without // using negative array bound (seems broken in Yosys?) or OOB in the // short circuit case (gives lint although result is well-defined) if (i == 0) begin fifo_valid_m1[i] = 1'b1; end else begin fifo_valid_m1[i] = fifo_valid[i - 1]; end end end always @ (posedge clk or negedge rst_n) begin: fifo_update integer i; if (!rst_n) begin for (i = 0; i < FIFO_DEPTH; i = i + 1) begin fifo_valid_hw[i] <= 2'b00; fifo_mem[i] <= 32'd0; fifo_err[i] <= 1'b0; fifo_break_any[i] <= 2'b00; fifo_break_d_mode[i] <= 2'b00; fifo_predbranch[i] <= 2'b00; end // This exists only for loop boundary conditions, but is tied off in // this synchronous process to work around a Verilator scheduling // issue (see issue #21) fifo_valid_hw[FIFO_DEPTH] <= 2'b00; end else begin for (i = 0; i < FIFO_DEPTH; i = i + 1) begin if (fifo_pop || (fifo_push && !fifo_valid[i])) begin fifo_mem[i] <= fifo_valid[i + 1] ? fifo_mem[i + 1] : mem_data; fifo_err[i] <= fifo_valid[i + 1] ? fifo_err[i + 1] : mem_or_pmp_err; fifo_break_any[i] <= fifo_valid[i + 1] ? fifo_break_any[i + 1] : mem_break_any; fifo_break_d_mode[i] <= fifo_valid[i + 1] ? fifo_break_d_mode[i + 1] : mem_break_d_mode; fifo_predbranch[i] <= fifo_valid[i + 1] ? fifo_predbranch[i + 1] : mem_data_predbranch; end fifo_valid_hw[i] <= jump_now ? 2'h0 : fifo_valid[i + 1] && fifo_pop ? fifo_valid_hw[i + 1] : fifo_valid[i] && fifo_pop ? mem_data_hwvld & {2{fifo_push}} : fifo_valid[i] ? fifo_valid_hw[i] : fifo_push && !fifo_pop && fifo_valid_m1[i] ? mem_data_hwvld : 2'h0; end // Allow DM to inject instructions directly into the lowest-numbered // queue entry. This mux should not extend critical path since it is // balanced with the instruction-assembly muxes on the queue bypass // path. Note that flush takes precedence over debug injection // (and the debug module design must account for this) if (fifo_dbg_inject) begin fifo_mem[0] <= dbg_instr_data; fifo_err[0] <= 1'b0; fifo_predbranch[0] <= 2'b00; fifo_break_any[0] <= 2'b00; fifo_break_d_mode[0] <= 2'b00; fifo_valid_hw[0] <= jump_now ? 2'b00 : 2'b11; end fifo_valid_hw[FIFO_DEPTH] <= 2'b00; end end `ifdef HAZARD3_ASSERTIONS always @ (posedge clk) if (rst_n) begin // FIFO validity must be compact, so we can always consume from the end if (!fifo_valid[0]) begin assert(!fifo_valid[1]); end end `endif assign pwrdown_ok = (fifo_full && !jump_target_vld) || debug_mode; // ---------------------------------------------------------------------------- // Branch target buffer wire [W_ADDR-1:0] btb_src_addr; wire btb_src_size; wire [W_ADDR-1:0] btb_target_addr; wire btb_valid; generate if (BRANCH_PREDICTOR) begin: have_btb reg [W_ADDR-1:0] btb_src_addr_r; reg btb_src_size_r; reg [W_ADDR-1:0] btb_target_addr_r; reg btb_valid_r; assign btb_src_addr = btb_src_addr_r; assign btb_src_size = btb_src_size_r; assign btb_target_addr = btb_target_addr_r; assign btb_valid = btb_valid_r; always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin btb_src_addr_r <= {W_ADDR{1'b0}}; btb_src_size_r <= 1'b0; btb_target_addr_r <= {W_ADDR{1'b0}}; btb_valid_r <= 1'b0; end else if (btb_clear) begin // Clear takes precedences over set. E.g. if a taken branch is in // stage 2 and an exception is in stage 3, we must clear the BTB. btb_valid_r <= 1'b0; end else if (btb_set) begin btb_src_addr_r <= btb_set_src_addr; btb_src_size_r <= btb_set_src_size; btb_target_addr_r <= btb_set_target_addr; btb_valid_r <= 1'b1; end end end else begin: no_btb assign btb_src_addr = {W_ADDR{1'b0}}; assign btb_src_size = 1'b0; assign btb_target_addr = {W_ADDR{1'b0}}; assign btb_valid = 1'b0; end endgenerate // Decode uses the target address to set the PC to the correct branch target // value following a predicted-taken branch (as normally it would update PC // by following an X jump request, and in this case there is none). // // Note this assumes the BTB target has not changed by the time the predicted // branch arrives at decode! This is always true because the only way for the // target address to change is when an older branch is taken, which would // flush the younger predicted-taken branch before it reaches decode. assign btb_target_addr_out = btb_target_addr; // ---------------------------------------------------------------------------- // Fetch request generation // Fetch addr runs ahead of the PC, in word increments. reg [W_ADDR-1:0] fetch_addr; reg fetch_priv; reg btb_prev_start_of_overhanging; reg [1:0] mem_aph_hwvld; reg mem_addr_hold; wire btb_match_word = |BRANCH_PREDICTOR && btb_valid && ( fetch_addr[W_ADDR-1:2] == btb_src_addr[W_ADDR-1:2] ); // Catch case where predicted-taken branch instruction extends into next word: wire btb_src_overhanging = btb_src_size && btb_src_addr[1]; // Suppress case where we have jumped immediately after a word-aligned halfword-sized // branch, and the jump target went into fetch_addr due to an address-phase hold: wire btb_jumped_beyond = !btb_src_size && !btb_src_addr[1] && !mem_aph_hwvld[0]; wire btb_match_current_addr = btb_match_word && !btb_src_overhanging && !btb_jumped_beyond; wire btb_match_next_addr = btb_match_word && btb_src_overhanging; wire btb_match_now = btb_match_current_addr || btb_prev_start_of_overhanging; // Post-increment if jump request is going straight through wire [W_ADDR-1:0] jump_target_post_increment = {jump_target[W_ADDR-1:2], 2'b00} + {{W_ADDR-3{1'b0}}, mem_addr_rdy && !mem_addr_hold, 2'b00}; always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin fetch_addr <= RESET_VECTOR; // M-mode at reset: fetch_priv <= 1'b1; btb_prev_start_of_overhanging <= 1'b0; end else begin if (jump_now) begin fetch_addr <= jump_target_post_increment; fetch_priv <= jump_priv || !U_MODE; btb_prev_start_of_overhanging <= 1'b0; end else if (mem_addr_vld && mem_addr_rdy) begin if (btb_match_now && |BRANCH_PREDICTOR) begin fetch_addr <= {btb_target_addr[W_ADDR-1:2], 2'b00}; end else begin fetch_addr <= fetch_addr + 32'd4; end btb_prev_start_of_overhanging <= btb_match_next_addr; end end end // Combinatorially generate the address-phase request reg reset_holdoff; always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin reset_holdoff <= 1'b1; end else begin reset_holdoff <= (|EXTENSION_XH3POWER && delay_first_fetch) ? reset_holdoff : 1'b0; // This should be impossible, but assert to be sure, because it *will* // change the fetch address (and we shouldn't check it in hardware if // we can prove it doesn't happen) end end `ifdef HAZARD3_ASSERTIONS always @ (posedge clk) if (rst_n) begin assert(!(jump_target_vld && reset_holdoff)); end `endif reg [W_ADDR-1:0] mem_addr_r; reg mem_priv_r; reg mem_addr_vld_r; // Downstream accesses are always word-sized word-aligned. assign mem_addr = mem_addr_r; assign mem_priv = mem_priv_r; assign mem_addr_vld = mem_addr_vld_r && !reset_holdoff; assign mem_size = 1'b1; wire fetch_stall; always @ (*) begin mem_addr_r = fetch_addr; mem_priv_r = fetch_priv; mem_addr_vld_r = 1'b1; case (1'b1) mem_addr_hold : begin mem_addr_r = fetch_addr; end jump_target_vld || reset_holdoff : begin mem_addr_r = {jump_target[W_ADDR-1:2], 2'b00}; mem_priv_r = jump_priv || !U_MODE; end DEBUG_SUPPORT && debug_mode : begin mem_addr_vld_r = 1'b0; end !fetch_stall : begin mem_addr_r = fetch_addr; end default : begin mem_addr_vld_r = 1'b0; end endcase end assign jump_target_rdy = !mem_addr_hold; // ---------------------------------------------------------------------------- // Bus Pipeline Tracking // Keep track of some useful state of the memory interface reg [1:0] pending_fetches; reg [1:0] ctr_flush_pending; wire [1:0] pending_fetches_next = pending_fetches + (mem_addr_vld && !mem_addr_hold) - mem_data_vld; // Using the non-registered version of pending_fetches would improve FIFO // utilisation, but create a combinatorial path from hready to address phase! // This means at least a 2-word FIFO is required for full fetch throughput. assign fetch_stall = fifo_full || fifo_almost_full && |pending_fetches || pending_fetches > 2'h1; // Debugger only injects instructions when the frontend is at rest and empty. assign dbg_instr_data_rdy = DEBUG_SUPPORT && !fifo_valid[0] && ~|ctr_flush_pending; wire cir_room_for_fetch; // If fetch data is forwarded past the FIFO, ensure it is not also written to it. assign fifo_push = mem_data_vld && ~|ctr_flush_pending && !(cir_room_for_fetch && fifo_empty) && !(DEBUG_SUPPORT && debug_mode); always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin mem_addr_hold <= 1'b0; pending_fetches <= 2'h0; ctr_flush_pending <= 2'h0; end else begin mem_addr_hold <= mem_addr_vld && !mem_addr_rdy; pending_fetches <= pending_fetches_next; if (jump_now) begin ctr_flush_pending <= pending_fetches - mem_data_vld; end else if (|ctr_flush_pending && mem_data_vld) begin ctr_flush_pending <= ctr_flush_pending - 1'b1; end end end `ifdef HAZARD3_ASSERTIONS always @ (posedge clk) if (rst_n) begin assert(ctr_flush_pending <= pending_fetches); assert(pending_fetches < 2'd3); assert(!(mem_data_vld && !pending_fetches)); end `endif always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin mem_data_hwvld <= 2'b11; mem_aph_hwvld <= 2'b11; mem_data_predbranch <= 2'b00; end else begin if (jump_now) begin if (|EXTENSION_C) begin if (mem_addr_rdy) begin mem_aph_hwvld <= 2'b11; mem_data_hwvld <= {1'b1, !jump_target[1]}; end else begin mem_aph_hwvld <= {1'b1, !jump_target[1]}; end end mem_data_predbranch <= 2'b00; end else if (mem_addr_vld && mem_addr_rdy) begin if (|EXTENSION_C) begin // If a predicted-taken branch instruction only spans the first // half of a word, need to flag the second half as invalid. mem_data_hwvld <= mem_aph_hwvld & { !(|BRANCH_PREDICTOR && btb_match_now && (btb_src_addr[1] == btb_src_size)), 1'b1 }; // Also need to take the alignment of the destination into account. mem_aph_hwvld <= { 1'b1, !(|BRANCH_PREDICTOR && btb_match_now && btb_target_addr[1]) }; end mem_data_predbranch <= |BRANCH_PREDICTOR && btb_match_word ? ( btb_src_addr[1] ? 2'b10 : btb_src_size ? 2'b11 : 2'b01 ) : |BRANCH_PREDICTOR && btb_prev_start_of_overhanging ? ( 2'b01 ) : 2'b00; end end end // ---------------------------------------------------------------------------- // PMP and trigger unit interfacing: query -> kill/break wire [W_ADDR-1:0] pmp_trigger_check_dph_addr; wire pmp_trigger_check_dph_m_mode; // Register the fetch address into stage F so that the PMP can check it in // parallel with the bus data phase. Feels wasteful to have a separate // register, but using the fetch_addr counter is fraught due to the way that // new addresses go into it or past it (depending on aphase hold). generate if (PMP_REGIONS > 0 || DEBUG_SUPPORT != 0) begin: have_check_reg reg [W_ADDR-1:0] check_addr_dph; reg check_m_mode_dph; always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin check_addr_dph <= {W_ADDR{1'b0}}; check_m_mode_dph <= 1'b0; end else if (mem_addr_vld && mem_addr_rdy) begin check_addr_dph <= mem_addr; check_m_mode_dph <= mem_priv; end end assign pmp_trigger_check_dph_addr = check_addr_dph; assign pmp_trigger_check_dph_m_mode = check_m_mode_dph; end else begin: no_check_reg assign pmp_trigger_check_dph_addr = {W_ADDR{1'b0}}; assign pmp_trigger_check_dph_m_mode = 1'b0; end endgenerate generate if (PMP_REGIONS == 0) begin: no_pmp assign pmp_i_addr = {W_ADDR{1'b0}}; assign pmp_i_m_mode = 1'b0; assign pmp_kill_fetch_dph = 1'b0; end else begin: have_pmp assign pmp_i_addr = pmp_trigger_check_dph_addr; assign pmp_i_m_mode = pmp_trigger_check_dph_m_mode; assign pmp_kill_fetch_dph = pmp_i_kill && !debug_mode; end endgenerate generate if (DEBUG_SUPPORT == 0) begin: no_triggers assign trigger_addr = {W_ADDR{1'b0}}; assign trigger_m_mode = 1'b0; assign mem_break_any = 2'b00; assign mem_break_d_mode = 2'b00; end else begin: have_triggers assign trigger_addr = pmp_trigger_check_dph_addr; assign trigger_m_mode = pmp_trigger_check_dph_m_mode; assign mem_break_any = trigger_break_any & {|EXTENSION_C, 1'b1}; assign mem_break_d_mode = trigger_break_d_mode & {|EXTENSION_C, 1'b1}; end endgenerate // ---------------------------------------------------------------------------- // Instruction buffer // The instruction buffer is a 3 x ~16-bit shift register: // // * 2 x 16-bit entries form the 32-bit current instruction register (CIR) // which is the processor's decode window // // * 1 x 16-bit entry allows the decode window to be non-32-bit-aligned with // respect to the 2 x 32-bit prefetch queue entries, which are always // naturally aligned in memory (if fully populated). // // The third entry should be trimmed for non-RVC configurations due to // constant-folding on EXTENSION_C; it is unnecessary here because the // instructions are always 32-bit-aligned. // The entries ("slots") are slightly larger than 16 bits because they also // contain metadata like bus errors: localparam W_SLOT = 4 + W_BUNDLE; localparam SLOT_BREAK_ANY_BIT = 3 + W_BUNDLE; localparam SLOT_BREAK_D_MODE_BIT = 2 + W_BUNDLE; localparam SLOT_ERR_BIT = 1 + W_BUNDLE; localparam SLOT_PREDBRANCH_BIT = 0 + W_BUNDLE; reg [3*W_SLOT-1:0] buf_contents; reg [1:0] buf_level; wire fetch_data_vld = !fifo_empty || (mem_data_vld && ~|ctr_flush_pending && !debug_mode); wire [W_DATA-1:0] fetch_data = fifo_empty ? mem_data : fifo_rdata; wire [1:0] fetch_data_hwvld = fifo_empty ? mem_data_hwvld : fifo_valid_hw[0]; wire fetch_bus_err = fifo_empty ? mem_or_pmp_err : fifo_err[0]; wire [1:0] fetch_break_any = fifo_empty ? mem_break_any : fifo_break_any[0]; wire [1:0] fetch_break_d_mode = fifo_empty ? mem_break_d_mode : fifo_break_d_mode[0]; wire [1:0] fetch_predbranch = fifo_empty ? mem_data_predbranch : fifo_predbranch[0]; wire [W_SLOT-1:0] fetch_contents_hw1 = { fetch_break_any[1], fetch_break_d_mode[1], fetch_bus_err, fetch_predbranch[1], fetch_data[W_BUNDLE +: W_BUNDLE] }; wire [W_SLOT-1:0] fetch_contents_hw0 = { fetch_break_any[0], fetch_break_d_mode[0], fetch_bus_err, fetch_predbranch[0], fetch_data[0 +: W_BUNDLE] }; wire [2*W_SLOT-1:0] fetch_contents_aligned = { fetch_contents_hw1, fetch_data_hwvld[0] || ~|EXTENSION_C ? fetch_contents_hw0 : fetch_contents_hw1 }; // Shift not-yet-used contents down to backfill D's consumption. We don't care // about anything which is invalid or will be overlaid with fresh data, so // choose these values in a way that minimises muxes. wire [3*W_SLOT-1:0] buf_shifted = cir_use[1] ? {buf_contents[W_SLOT +: 2 * W_SLOT], buf_contents[2 * W_SLOT +: W_SLOT]} : cir_use[0] && EXTENSION_C ? {buf_contents[2 * W_SLOT +: W_SLOT], buf_contents[W_SLOT +: 2 * W_SLOT]} : buf_contents; wire [1:0] level_next_no_fetch = buf_level - cir_use; // Overlay fresh fetch data onto the shifted/recycled buffer contents. Again, // if something won't be looked at, generate the cheapest possible garbage. assign cir_room_for_fetch = level_next_no_fetch <= (|EXTENSION_C && ~&fetch_data_hwvld ? 2'h2 : 2'h1); assign fifo_pop = cir_room_for_fetch && !fifo_empty; wire [3*W_SLOT-1:0] buf_shifted_plus_fetch = !cir_room_for_fetch ? buf_shifted : level_next_no_fetch[1] && |EXTENSION_C ? {fetch_contents_aligned[0 +: W_SLOT], buf_shifted[0 +: 2 * W_SLOT]} : level_next_no_fetch[0] && |EXTENSION_C ? {fetch_contents_aligned, buf_shifted[0 +: W_SLOT]} : {buf_shifted[2 * W_SLOT +: W_SLOT], fetch_contents_aligned}; wire [1:0] fetch_fill_amount = cir_room_for_fetch && fetch_data_vld ? ( &fetch_data_hwvld || ~|EXTENSION_C ? 2'h2 : 2'h1 ) : 2'h0; wire [1:0] buf_level_next = {1'b1, |EXTENSION_C} & ( jump_now && cir_flush_behind ? (cir_is_32bit ? 2'h2 : 2'h1) : jump_now ? 2'h0 : level_next_no_fetch + fetch_fill_amount ); always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin buf_level <= 2'h0; cir_vld <= 2'h0; // Mysterious reset value ensures address buses are zero in reset // (see definition of d_addr_offs in hazard3_decode) buf_contents <= {{3 * W_SLOT - 2{1'b0}}, 2'b11}; end else begin buf_level <= buf_level_next; cir_vld <= buf_level_next & ~(buf_level_next >> 1'b1); buf_contents <= buf_shifted_plus_fetch; end end `ifdef HAZARD3_ASSERTIONS reg [1:0] prop_past_buf_level; // Workaround for weird non-constant $past reset issue always @ (posedge clk) begin if (!rst_n) begin prop_past_buf_level <= 2'h0; end else begin prop_past_buf_level <= buf_level; assert(cir_vld <= 2); assert(cir_use <= cir_vld); if (!jump_now) assert(buf_level_next >= level_next_no_fetch); // We fetch 32 bits per cycle, max. If this happens it's due to negative overflow. if (prop_past_buf_level == 2'h0) assert(buf_level != 2'h3); end end `endif assign cir_err = { buf_contents[1 * W_SLOT + SLOT_ERR_BIT], buf_contents[0 * W_SLOT + SLOT_ERR_BIT] }; assign cir_predbranch = { buf_contents[1 * W_SLOT + SLOT_PREDBRANCH_BIT], buf_contents[0 * W_SLOT + SLOT_PREDBRANCH_BIT] }; assign cir_break_any = buf_contents[0 * W_SLOT + SLOT_BREAK_ANY_BIT] && |cir_vld; assign cir_break_d_mode = buf_contents[0 * W_SLOT + SLOT_BREAK_D_MODE_BIT] && |cir_vld; // ---------------------------------------------------------------------------- // Register number predecode wire [31:0] next_instr = { buf_shifted_plus_fetch[1 * W_SLOT +: W_BUNDLE], buf_shifted_plus_fetch[0 * W_SLOT +: W_BUNDLE] }; wire next_instr_is_32bit = next_instr[1:0] == 2'b11 || ~|EXTENSION_C; wire [3:0] decomp_uop_step; wire [3:0] uop_ctr = decomp_uop_step & {4{|EXTENSION_ZCMP}}; wire [4:0] zcmp_pushpop_rs2 = uop_ctr == 4'h0 ? 5'd01 : // ra uop_ctr == 4'h1 ? 5'd08 : // s0 uop_ctr == 4'h2 ? 5'd09 : // s1 5'd15 + {1'b0, uop_ctr} ; // s2-s11 wire [4:0] zcmp_pushpop_rs1 = uop_ctr < 4'hd ? 5'd02 : // sp (addr base reg) uop_ctr == 4'hd ? 5'd00 : // zero (clear a0) uop_ctr == 4'he ? 5'd01 : // ra (ret) 5'd02 ; // sp (stack adj) wire [4:0] zcmp_sa01_r1s = {|next_instr[9:8], ~|next_instr[9:8], next_instr[9:7]}; wire [4:0] zcmp_sa01_r2s = {|next_instr[4:3], ~|next_instr[4:3], next_instr[4:2]}; wire [4:0] zcmp_mvsa01_rs1 = {4'h5, uop_ctr[0]}; wire [4:0] zcmp_mva01s_rs1 = uop_ctr[0] ? zcmp_sa01_r2s : zcmp_sa01_r1s; // "coarse" because the mapping of pair (x0, x1) -> (x0, x0) is not yet applied wire [4:0] zilsd_rs2_coarse = { next_instr[24:21], df_lspair_phase_next ^ ~next_instr[15]}; wire [4:0] zclsd_sd_rs2_coarse = {2'b01, next_instr[4:3], df_lspair_phase_next ^ ~next_instr[7] }; wire [4:0] zclsd_sdsp_rs2_coarse = { next_instr[6:3], df_lspair_phase_next ^ 1'b1 }; always @ (*) begin casez ({next_instr_is_32bit, |EXTENSION_ZCMP, next_instr[15:0]}) {1'b1, 1'bz, 16'bzzzzzzzzzzzzzzzz}: predecode_rs1_coarse = next_instr[19:15]; // 32-bit R, S, B formats {1'b0, 1'bz, 16'b00zzzzzzzzzzzz00}: predecode_rs1_coarse = 5'd2; // c.addi4spn + don't care {1'b0, 1'bz, 16'b0zzzzzzzzzzzzz01}: predecode_rs1_coarse = next_instr[11:7]; // c.addi, c.addi16sp + don't care (jal, li) {1'b0, 1'bz, 16'bz1zzzzzzzzzzzz10}: predecode_rs1_coarse = 5'd2; // c.lwsp, c.swsp, c.ldsp, c.sdsp {1'b0, 1'bz, 16'bz00zzzzzzzzzzz10}: predecode_rs1_coarse = next_instr[11:7]; // c.slli, c.mv, c.add {1'b0, 1'b1, 16'b1011zzzzzzzzzz10}: predecode_rs1_coarse = zcmp_pushpop_rs1; // cm.push, cm.pop* {1'b0, 1'b1, 16'b1010zzzzz0zzzz10}: predecode_rs1_coarse = zcmp_mvsa01_rs1; // cm.mvsa01 {1'b0, 1'b1, 16'b1010zzzzz1zzzz10}: predecode_rs1_coarse = zcmp_mva01s_rs1; // cm.mva01s default: predecode_rs1_coarse = {2'b01, next_instr[9:7]}; endcase casez ({next_instr_is_32bit, |EXTENSION_ZCMP, |EXTENSION_ZILSD, |EXTENSION_ZCLSD, next_instr[15:0]}) {1'b1, 1'bz, 1'b1, 1'bz, 16'bzz11zzzzz0z0zzzz}: predecode_rs2_coarse = zilsd_rs2_coarse; // ld, sd (Zilsd) {1'b1, 1'bz, 1'b0, 1'bz, 16'bzz11zzzzz0z0zzzz}: predecode_rs2_coarse = next_instr[24:20]; // ld, sd (no Zilsd) {1'b1, 1'bz, 1'bz, 1'bz, 16'bzz0zzzzzzzzzzzzz}: predecode_rs2_coarse = next_instr[24:20]; // (cover remaining 32-bit {1'b1, 1'bz, 1'bz, 1'bz, 16'bzz10zzzzzzzzzzzz}: predecode_rs2_coarse = next_instr[24:20]; // patterns, without overlap) {1'b1, 1'bz, 1'bz, 1'bz, 16'bzz11zzzzz1zzzzzz}: predecode_rs2_coarse = next_instr[24:20]; {1'b1, 1'bz, 1'bz, 1'bz, 16'bzz11zzzzz0z1zzzz}: predecode_rs2_coarse = next_instr[24:20]; {1'b0, 1'bz, 1'b1, 1'b1, 16'bzz1zzzzzzzzzzz00}: predecode_rs2_coarse = zclsd_sd_rs2_coarse; {1'b0, 1'bz, 1'bz, 1'bz, 16'bzz0zzzzzzzzzzz10}: predecode_rs2_coarse = next_instr[6:2]; // c.add, c.swsp {1'b0, 1'b1, 1'bz, 1'bz, 16'bz01zzzzzzzzzzz10}: predecode_rs2_coarse = zcmp_pushpop_rs2; // cm.push {1'b0, 1'bz, 1'b1, 1'b1, 16'bz11zzzzzzzzzzz10}: predecode_rs2_coarse = zclsd_sdsp_rs2_coarse; default: predecode_rs2_coarse = {2'b01, next_instr[4:2]}; endcase // The "fine" predecode targets those instructions which either: // - Have an implicit zero-register operand in their expanded form (e.g. c.beqz) // - Do not have a register operand on that port, but rely on the port being 0 // We don't care about instructions which ignore the reg ports, e.g. ebreak casez ({|EXTENSION_C, next_instr}) // -> addi rd, x0, imm: {1'b1, 16'hzzzz, `RVOPC_C_LI}: predecode_rs1_fine = 5'd0; {1'b1, 16'hzzzz, `RVOPC_C_MV}: begin if (next_instr[6:2] == 5'd0) begin // c.jr has rs1 as normal predecode_rs1_fine = predecode_rs1_coarse; end else begin // -> add rd, x0, rs2: predecode_rs1_fine = 5'd0; end end default: predecode_rs1_fine = predecode_rs1_coarse; endcase casez ({|EXTENSION_C, |EXTENSION_ZILSD, |EXTENSION_ZCLSD, next_instr}) {1'b1, 1'bz, 1'bz, 16'hzzzz, `RVOPC_C_BEQZ}: predecode_rs2_fine = 5'd0; // -> beq rs1, x0, label {1'b1, 1'bz, 1'bz, 16'hzzzz, `RVOPC_C_BNEZ}: predecode_rs2_fine = 5'd0; // -> bne rs1, x0, label {1'b1, 1'b1, 1'bz, `RVOPC_SD }: predecode_rs2_fine = predecode_rs2_coarse & {5{|next_instr[24:21]}}; {1'b1, 1'b1, 1'b1, 16'hzzzz, `RVOPC_C_SDSP}: predecode_rs2_fine = predecode_rs2_coarse & {5{|next_instr[ 6: 3]}}; default: predecode_rs2_fine = predecode_rs2_coarse; endcase if (|EXTENSION_E) begin predecode_rs1_coarse[4] = 1'b0; predecode_rs2_coarse[4] = 1'b0; predecode_rs1_fine[4] = 1'b0; predecode_rs2_fine[4] = 1'b0; end end // ---------------------------------------------------------------------------- // Instruction decompression // Instructions are decompressed at the end of stage 1 (fetch data phase). On // ASIC, where the register file is synthesised with muxes, this puts // decompression somewhat in parallel with register file read, which uses // approximately decoded regnums. generate if (~|EXTENSION_C) begin: no_decompress // No decompression; instructions decoded directly from prefetch buffer always @ (*) begin cir = { buf_contents[1 * W_SLOT +: W_BUNDLE], buf_contents[0 * W_SLOT +: W_BUNDLE] } | 32'd3; cir_is_32bit = 1'b1; cir_invalid_16bit = ~&buf_contents[1:0]; cir_is_uop = 1'b0; cir_uop_nonfinal = 1'b0; cir_uop_no_pc_update = 1'b0; cir_uop_atomic = 1'b0; end assign decomp_uop_step = 4'h0; end else begin: have_decompress wire decomp_instr_is_32bit; wire [31:0] decomp_instr_out; wire decomp_is_uop; wire decomp_is_final_uop; wire decomp_uop_no_pc_update; wire decomp_uop_atomic; wire decomp_invalid; wire first_uop = ~|decomp_uop_step; // Ensure the first uop goes straight through, as it is registered into CIR: wire uop_stall_non_first = first_uop ? ~|buf_level_next : uop_stall; // Ensure the uop counter stops at 0 after rolling over once: wire uop_stall_on_repeat = cir_is_uop && !cir_uop_nonfinal && ~|cir_use; hazard3_instr_decompress #( `include "hazard3_config_inst.vh" ) decomp ( .clk (clk), .rst_n (rst_n), .instr_in (next_instr), .instr_is_32bit (decomp_instr_is_32bit), .instr_out (decomp_instr_out), .instr_out_is_uop (decomp_is_uop), .instr_out_is_final_uop (decomp_is_final_uop), .instr_out_uop_no_pc_update (decomp_uop_no_pc_update), .instr_out_uop_atomic (decomp_uop_atomic), .instr_out_uop_stall (uop_stall_non_first || uop_stall_on_repeat), .instr_out_uop_clear (uop_clear), .df_uop_step (decomp_uop_step), .invalid (decomp_invalid) ); wire cir_clken = ~|cir_vld || (!cir_vld[1] && &buf_contents[1:0]) || |cir_use || (|EXTENSION_ZCMP && cir_is_uop && !uop_stall) || (|EXTENSION_ZCMP && uop_clear); wire cir_is_uop_next = decomp_is_uop && |buf_level_next; wire cir_uop_nonfinal_next = cir_is_uop_next && !decomp_is_final_uop; always @ (posedge clk or negedge rst_n) begin if (!rst_n) begin cir <= 32'd3; cir_is_32bit <= 1'b0; cir_invalid_16bit <= 1'b0; cir_is_uop <= 1'b0; cir_uop_nonfinal <= 1'b0; cir_uop_no_pc_update <= 1'b0; cir_uop_atomic <= 1'b0; end else if (cir_clken) begin cir <= decomp_instr_out | 32'd3; cir_is_32bit <= decomp_instr_is_32bit; cir_invalid_16bit <= decomp_invalid; cir_is_uop <= |EXTENSION_ZCMP && !uop_clear && cir_is_uop_next; cir_uop_nonfinal <= |EXTENSION_ZCMP && !uop_clear && cir_uop_nonfinal_next; cir_uop_no_pc_update <= |EXTENSION_ZCMP && !uop_clear && decomp_uop_no_pc_update; cir_uop_atomic <= |EXTENSION_ZCMP && !uop_clear && decomp_uop_atomic; end end end endgenerate assign cir_raw = { buf_contents[1 * W_SLOT +: W_BUNDLE], buf_contents[0 * W_SLOT +: W_BUNDLE] }; endmodule `ifndef YOSYS `default_nettype wire `endif