/* * Copyright (C) 2021 Valve Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef _IR3_RA_H #define _IR3_RA_H #include "util/rb_tree.h" #include "ir3.h" #include "ir3_compiler.h" #ifdef DEBUG #define RA_DEBUG (ir3_shader_debug & IR3_DBG_RAMSGS) #else #define RA_DEBUG 0 #endif #define d(fmt, ...) \ do { \ if (RA_DEBUG) { \ mesa_logi("RA: " fmt, ##__VA_ARGS__); \ } \ } while (0) #define di(instr, fmt, ...) \ do { \ if (RA_DEBUG) { \ struct log_stream *stream = mesa_log_streami(); \ mesa_log_stream_printf(stream, "RA: " fmt ": ", ##__VA_ARGS__); \ ir3_print_instr_stream(stream, instr); \ mesa_log_stream_destroy(stream); \ } \ } while (0) typedef uint16_t physreg_t; static inline unsigned ra_physreg_to_num(physreg_t physreg, unsigned flags) { if (!(flags & IR3_REG_HALF)) physreg /= 2; if (flags & IR3_REG_SHARED) physreg += 48 * 4; return physreg; } static inline physreg_t ra_num_to_physreg(unsigned num, unsigned flags) { if (flags & IR3_REG_SHARED) num -= 48 * 4; if (!(flags & IR3_REG_HALF)) num *= 2; return num; } static inline unsigned ra_reg_get_num(const struct ir3_register *reg) { return (reg->flags & IR3_REG_ARRAY) ? reg->array.base : reg->num; } static inline physreg_t ra_reg_get_physreg(const struct ir3_register *reg) { return ra_num_to_physreg(ra_reg_get_num(reg), reg->flags); } static inline bool def_is_gpr(const struct ir3_register *reg) { return reg_num(reg) != REG_A0 && reg_num(reg) != REG_P0; } /* Note: don't count undef as a source. */ static inline bool ra_reg_is_src(const struct ir3_register *reg) { return (reg->flags & IR3_REG_SSA) && reg->def && def_is_gpr(reg->def); } static inline bool ra_reg_is_dst(const struct ir3_register *reg) { return (reg->flags & IR3_REG_SSA) && def_is_gpr(reg) && ((reg->flags & IR3_REG_ARRAY) || reg->wrmask); } /* Iterators for sources and destinations which: * - Don't include fake sources (irrelevant for RA) * - Don't include non-SSA sources (immediates and constants, also irrelevant) */ #define ra_foreach_src_n(__srcreg, __n, __instr) \ foreach_src_n(__srcreg, __n, __instr) \ if (ra_reg_is_src(__srcreg)) #define ra_foreach_src(__srcreg, __instr) \ ra_foreach_src_n(__srcreg, __i, __instr) #define ra_foreach_src_rev(__srcreg, __instr) \ for (struct ir3_register *__srcreg = (void *)~0; __srcreg; __srcreg = NULL) \ for (int __cnt = (__instr)->srcs_count, __i = __cnt - 1; __i >= 0; \ __i--) \ if (ra_reg_is_src((__srcreg = (__instr)->srcs[__i]))) #define ra_foreach_dst_n(__dstreg, __n, __instr) \ foreach_dst_n(__dstreg, __n, instr) \ if (ra_reg_is_dst(__dstreg)) #define ra_foreach_dst(__dstreg, __instr) \ ra_foreach_dst_n(__dstreg, __i, __instr) #define RA_HALF_SIZE (4 * 48) #define RA_FULL_SIZE (4 * 48 * 2) #define RA_SHARED_SIZE (2 * 4 * 8) #define RA_MAX_FILE_SIZE RA_FULL_SIZE struct ir3_liveness { unsigned block_count; unsigned interval_offset; DECLARE_ARRAY(struct ir3_register *, definitions); DECLARE_ARRAY(BITSET_WORD *, live_out); DECLARE_ARRAY(BITSET_WORD *, live_in); }; struct ir3_liveness *ir3_calc_liveness(void *mem_ctx, struct ir3 *ir); bool ir3_def_live_after(struct ir3_liveness *live, struct ir3_register *def, struct ir3_instruction *instr); void ir3_create_parallel_copies(struct ir3 *ir); void ir3_merge_regs(struct ir3_liveness *live, struct ir3 *ir); void ir3_force_merge(struct ir3_register *a, struct ir3_register *b, int b_offset); struct ir3_pressure { unsigned full, half, shared; }; void ir3_calc_pressure(struct ir3_shader_variant *v, struct ir3_liveness *live, struct ir3_pressure *max_pressure); bool ir3_spill(struct ir3 *ir, struct ir3_shader_variant *v, struct ir3_liveness **live, const struct ir3_pressure *limit_pressure); bool ir3_lower_spill(struct ir3 *ir); void ir3_ra_validate(struct ir3_shader_variant *v, unsigned full_size, unsigned half_size, unsigned block_count); void ir3_lower_copies(struct ir3_shader_variant *v); /* Register interval datastructure * * ir3_reg_ctx is used to track which registers are live. The tricky part is * that some registers may overlap each other, when registers with overlapping * live ranges get coalesced. For example, splits will overlap with their * parent vector and sometimes collect sources will also overlap with the * collect'ed vector. ir3_merge_regs guarantees for us that none of the * registers in a merge set that are live at any given point partially * overlap, which means that we can organize them into a forest. While each * register has a per-merge-set offset, ir3_merge_regs also computes a * "global" offset which allows us to throw away the original merge sets and * think of registers as just intervals in a forest of live intervals. When a * register becomes live, we insert it into the forest, and when it dies we * remove it from the forest (and then its children get moved up a level). We * use red-black trees to keep track of each level of the forest, so insertion * and deletion should be fast operations. ir3_reg_ctx handles all the * internal bookkeeping for this, so that it can be shared between RA, * spilling, and register pressure tracking. */ struct ir3_reg_interval { struct rb_node node; struct rb_tree children; struct ir3_reg_interval *parent; struct ir3_register *reg; bool inserted; }; struct ir3_reg_ctx { /* The tree of top-level intervals in the forest. */ struct rb_tree intervals; /* Users of ir3_reg_ctx need to keep around additional state that is * modified when top-level intervals are added or removed. For register * pressure tracking, this is just the register pressure, but for RA we * need to keep track of the physreg of each top-level interval. These * callbacks provide a place to let users deriving from ir3_reg_ctx update * their state when top-level intervals are inserted/removed. */ /* Called when an interval is added and it turns out to be at the top * level. */ void (*interval_add)(struct ir3_reg_ctx *ctx, struct ir3_reg_interval *interval); /* Called when an interval is deleted from the top level. */ void (*interval_delete)(struct ir3_reg_ctx *ctx, struct ir3_reg_interval *interval); /* Called when an interval is deleted and its child becomes top-level. */ void (*interval_readd)(struct ir3_reg_ctx *ctx, struct ir3_reg_interval *parent, struct ir3_reg_interval *child); }; static inline struct ir3_reg_interval * ir3_rb_node_to_interval(struct rb_node *node) { return rb_node_data(struct ir3_reg_interval, node, node); } static inline const struct ir3_reg_interval * ir3_rb_node_to_interval_const(const struct rb_node *node) { return rb_node_data(struct ir3_reg_interval, node, node); } static inline struct ir3_reg_interval * ir3_reg_interval_next(struct ir3_reg_interval *interval) { struct rb_node *next = rb_node_next(&interval->node); return next ? ir3_rb_node_to_interval(next) : NULL; } static inline struct ir3_reg_interval * ir3_reg_interval_next_or_null(struct ir3_reg_interval *interval) { return interval ? ir3_reg_interval_next(interval) : NULL; } static inline void ir3_reg_interval_init(struct ir3_reg_interval *interval, struct ir3_register *reg) { rb_tree_init(&interval->children); interval->reg = reg; interval->parent = NULL; interval->inserted = false; } void ir3_reg_interval_dump(struct log_stream *stream, struct ir3_reg_interval *interval); void ir3_reg_interval_insert(struct ir3_reg_ctx *ctx, struct ir3_reg_interval *interval); void ir3_reg_interval_remove(struct ir3_reg_ctx *ctx, struct ir3_reg_interval *interval); void ir3_reg_interval_remove_all(struct ir3_reg_ctx *ctx, struct ir3_reg_interval *interval); #endif