mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 09:39:58 -04:00
Pool marks: a fragment shader stores its marks where it ends, not between a sample's coordinates and the sample.
On AMD's driver for Polaris (RX 580, 23.10) a store between a pool texture's access chain and its sample left the quad's helper lanes out of the code in between: derivatives at a triangle's edge were garbage and edge pixels came out dark, the dotted lines along the paths' triangles since 0.7.0 (frame dump f1714: the colour pass covers every pixel, the depth test is not what drops them). Each access now notes its slot in a Private variable; before each return of the entry function and before a kill, terminate or demote every noted slot is marked, an unset one going to a spare slot past the pool. Accesses in loops or outside the entry function note theirs in a ring of 8, marked in a loop at the way out. Checked with spirv-val on the 45 marked shaders of the dump. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2068,13 +2068,15 @@ static VkDescriptorSetLayout pool_layout_make(void) {
|
||||
|
||||
static void pool_marks_make(void) {
|
||||
VkCtx *c = vk_ctx();
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)VK_POOL_TEX * 4,
|
||||
// a uint past the slots: the spare that a fragment shader's marks of accesses it did not reach go to
|
||||
// (vk_pipeline.c spirv_pool_mark); it is never read back
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)(VK_POOL_TEX + 1) * 4,
|
||||
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
|
||||
VK_BUFFER_USAGE_TRANSFER_DST_BIT};
|
||||
VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
|
||||
if (vmaCreateBuffer(c->vma, &bi, &ai, &PL.marks, &PL.marks_alloc, NULL) != VK_SUCCESS)
|
||||
rt_fatal("vk: the texture pool marks buffer failed");
|
||||
bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
||||
bi.size = (VkDeviceSize)VK_POOL_TEX * 4, bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
||||
VmaAllocationCreateInfo hi = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST,
|
||||
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT};
|
||||
for (int i = 0; i < MARKS_RB; i++) {
|
||||
|
||||
@@ -2209,6 +2209,14 @@ int vk_pool_marks_enabled(void) {
|
||||
// its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it
|
||||
// does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the
|
||||
// shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows).
|
||||
// A fragment shader's marks are stored where it ends, not where it samples. A store between a sample's coordinates and
|
||||
// the sample takes the quad's helper lanes out of the code in between on AMD's driver for Polaris: their coordinates
|
||||
// stayed stale, the derivatives at a triangle's edge were garbage, and edge pixels came out dark (the lines along the
|
||||
// paths' triangles since 0.7.0). Each access only notes its slot in a Private variable (a register); before each
|
||||
// return of the entry function, and before a kill, terminate or demote anywhere, every variable is stored to its slot,
|
||||
// a variable no access set going to the spare slot past the pool (vk_exec_draw.c pool_marks_make). An access in a loop
|
||||
// or outside the entry function can run more than once with other slots: it notes them in a ring of RING Private
|
||||
// variables, so a loop's last RING slots are all marked.
|
||||
// *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked (*why says what stopped it).
|
||||
// NULL unless marked.
|
||||
static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status,
|
||||
@@ -2219,7 +2227,10 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui
|
||||
OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227,
|
||||
OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380,
|
||||
EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014,
|
||||
SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 };
|
||||
SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35, OP_TYPE_BOOL = 20,
|
||||
OP_LOAD_ = 61, OP_BITCAST = 124, OP_SELECT_ = 169, OP_INOT_EQUAL = 171, OP_LOOP_MERGE = 246, OP_LABEL_ = 248,
|
||||
OP_RETURN = 253, OP_RETURN_VALUE = 254, SC_PRIVATE = 6, LOOP_DEPTH = 64, OP_TYPE_ARRAY_ = 28, OP_IADD = 128,
|
||||
OP_BITWISE_AND = 199, OP_ULESS_THAN = 176, OP_BRANCH = 249, OP_BRANCH_CONDITIONAL = 250, OP_PHI = 245 };
|
||||
*status = 0, *why = NULL;
|
||||
if (n < 5) return NULL;
|
||||
uint32_t bound = w[3];
|
||||
@@ -2228,11 +2239,19 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui
|
||||
uint8_t *set2 = calloc(bound, 1);
|
||||
if (!bind || !set2) return free(bind), free(set2), *status = -1, *why = "no memory", NULL;
|
||||
uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes
|
||||
uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0;
|
||||
uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0, int_t = 0, bool_t = 0;
|
||||
int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok;
|
||||
uint32_t *rtype = calloc(bound, sizeof *rtype); // per id: its result type, where the defer path needs it
|
||||
uint8_t *is_type = calloc(bound, 1);
|
||||
if (!rtype || !is_type)
|
||||
return free(bind), free(set2), free(rtype), free(is_type), *status = -1, *why = "no memory", NULL;
|
||||
#define POOL_MARK_FREE() (free(bind), free(set2), free(rtype), free(is_type), free(site_def))
|
||||
// per site: 0 stored in place; else deferred, bit 1 the index is signed (a bitcast), bit 2 a ring (in a loop or
|
||||
// outside the entry function)
|
||||
uint8_t *site_def = NULL;
|
||||
for (uint64_t i = 5; i < n;) {
|
||||
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
||||
if (!l || i + l > n) return free(bind), free(set2), *status = -1, *why = "an instruction past the module's end", NULL;
|
||||
if (!l || i + l > n) return POOL_MARK_FREE(), *status = -1, *why = "an instruction past the module's end", NULL;
|
||||
if (op == SPV_OP_FUNCTION) { fn_start = i; break; }
|
||||
if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l;
|
||||
if (op == SPV_OP_ENTRY_POINT && l >= 3) {
|
||||
@@ -2251,6 +2270,10 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui
|
||||
early = 0; // the guest's own storage buffers: its stores would change under early tests
|
||||
} else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) {
|
||||
uint_t = w[i + 1];
|
||||
} else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 1) {
|
||||
int_t = w[i + 1];
|
||||
} else if (op == OP_TYPE_BOOL && l == 2) {
|
||||
bool_t = w[i + 1];
|
||||
} else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) {
|
||||
if (w[i + 3] == 0 && !c0) c0 = w[i + 2];
|
||||
if (w[i + 3] == 1 && !c1) c1 = w[i + 2];
|
||||
@@ -2260,50 +2283,95 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui
|
||||
if (op == SPV_OP_DECORATE || op == OP_MEMBER_DECORATE_ || op == OP_DECORATION_GROUP || op == OP_GROUP_DECORATE ||
|
||||
op == OP_GROUP_MEMBER_DECORATE || op == OP_DECORATE_ID || op == OP_DECORATE_STRING || op == OP_MEMBER_DECORATE_STRING)
|
||||
ann_end = i + l;
|
||||
if (op >= 19 && op <= 39 && l >= 2 && w[i + 1] < bound) is_type[w[i + 1]] = 1; // OpTypeVoid .. OpTypeForwardPointer
|
||||
else if (op == SPV_OP_CONSTANT && l >= 4 && w[i + 2] < bound) rtype[w[i + 2]] = w[i + 1];
|
||||
i += l;
|
||||
}
|
||||
// pool texture variables: set 2, a binding below VK_POOL_KINDS (spirv_pool_rebind); one already at the marks
|
||||
// binding means the module was marked before
|
||||
uint64_t sites = 0;
|
||||
enum { RING = 8 };
|
||||
uint64_t sites = 0, deferred = 0, rings = 0, exits = 0;
|
||||
int marked = 0;
|
||||
for (uint32_t id = 0; id < bound; id++)
|
||||
if (set2[id] && bind[id] == VK_POOL_MARK_BINDING + 1) marked = 1;
|
||||
site_def = fn_start && !marked ? calloc((n - fn_start) / 5 + 1, 1) : NULL; // an access chain is 5 words or more
|
||||
if (fn_start && !marked && !site_def) return POOL_MARK_FREE(), *status = -1, *why = "no memory", NULL;
|
||||
uint32_t loops[LOOP_DEPTH], fn = 0;
|
||||
int depth = 0;
|
||||
for (uint64_t i = fn_start; fn_start && !marked && i < n;) {
|
||||
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
||||
if (!l || i + l > n) break;
|
||||
if (l >= 3 && w[i + 1] < bound && is_type[w[i + 1]] && w[i + 2] < bound) rtype[w[i + 2]] = w[i + 1];
|
||||
if (op == SPV_OP_FUNCTION && l >= 3) fn = w[i + 2], depth = 0;
|
||||
else if (op == OP_LOOP_MERGE && l >= 3) depth < LOOP_DEPTH ? (void)(loops[depth++] = w[i + 1]) : (void)depth++;
|
||||
else if (op == OP_LABEL_ && l >= 2 && depth > 0 && depth <= LOOP_DEPTH && loops[depth - 1] == w[i + 1]) depth--;
|
||||
if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] &&
|
||||
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS)
|
||||
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) {
|
||||
uint32_t ix = w[i + 4], t = ix < bound ? rtype[ix] : 0;
|
||||
if (fragment_stage && entries == 1 && fs_entry && t && (t == uint_t || t == int_t)) {
|
||||
int ring = fn != fs_entry || depth;
|
||||
site_def[sites] = (uint8_t)(1 | (t == int_t ? 2 : 0) | (ring ? 4 : 0));
|
||||
ring ? rings++ : deferred++;
|
||||
}
|
||||
sites++;
|
||||
else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE ||
|
||||
(op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST))
|
||||
} else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE ||
|
||||
(op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST))
|
||||
early = 0;
|
||||
if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER ||
|
||||
(fn == fs_entry && (op == OP_RETURN || op == OP_RETURN_VALUE)))
|
||||
exits++;
|
||||
i += l;
|
||||
}
|
||||
if (marked || !sites) return free(bind), free(set2), NULL;
|
||||
if (marked || !sites) return POOL_MARK_FREE(), NULL;
|
||||
uint64_t stores = deferred + rings; // what each way out stores: a variable, or a ring's entries in a loop
|
||||
if (stores && (!exits || exits * (deferred + 3 * rings) > 8192)) // no way out to store at, or code that would grow
|
||||
memset(site_def, 0, sites), deferred = rings = stores = 0; // too much
|
||||
*why = w[1] < 0x00010300 || w[1] > 0x00010600 ? "a SPIR-V version before 1.3 or after 1.6"
|
||||
: !ann_end || !fn_start ? "no decorations or no function"
|
||||
: vertex_stage && !marks_vs_ok ? "the GPU allows no stores in vertex-pipeline shaders"
|
||||
: fragment_stage && !marks_fs_ok ? "the GPU allows no stores in fragment shaders"
|
||||
: NULL;
|
||||
if (*why) return free(bind), free(set2), *status = -1, NULL;
|
||||
if (*why) return POOL_MARK_FREE(), *status = -1, NULL;
|
||||
early = early && fragment_stage && entries == 1 && em_end && em_end < ann_end;
|
||||
// From SPIR-V 1.4 on an entry point lists every global variable its functions use, not only the inputs and outputs
|
||||
// (the translator's mesh shaders are 1.4): each entry point gets the marks variable, a word more.
|
||||
int iface = w[1] >= 0x00010400;
|
||||
// the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site
|
||||
uint64_t on = n + 3 + (uint64_t)entries + 20 + 30 + 9 * sites;
|
||||
// the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site; deferred:
|
||||
// a word per entry point and variable, 80 words of types and constants, then per site its variables and the store of
|
||||
// its slot (ring: 10 + 29), and per way out 29 words a variable and 8 * 29 a ring (its loop is 83)
|
||||
uint64_t on = n + 3 + (uint64_t)entries * (1 + deferred + 2 * rings) + 20 + 30 + 9 * sites + 80 + deferred * 12 +
|
||||
rings * 40 + (deferred * 29 + rings * RING * 29) * exits;
|
||||
uint32_t *o = malloc(on * 4);
|
||||
if (!o) return free(bind), free(set2), *status = -1, *why = "no memory", NULL;
|
||||
if (!o) return POOL_MARK_FREE(), *status = -1, *why = "no memory", NULL;
|
||||
uint32_t next = bound;
|
||||
uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++);
|
||||
uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++);
|
||||
// deferred: the types, constants and variables (pv0: one per site; ra0, rc0: a ring and its count per ring site)
|
||||
uint32_t new_bool = 0, ptr_pv = 0, c_none = 0, c_spare = 0, pv0 = 0, ring_t = 0, ptr_ring = 0, ring_none = 0,
|
||||
cr[RING] = {0}, c_ring = 0, ra0 = 0, rc0 = 0;
|
||||
if (stores) {
|
||||
new_bool = bool_t ? 0 : (bool_t = next++);
|
||||
ptr_pv = next++, c_none = next++, c_spare = next++, pv0 = next, next += (uint32_t)deferred;
|
||||
if (rings) {
|
||||
ring_t = next++, ptr_ring = next++, ring_none = next++, c_ring = next++;
|
||||
for (int j = 0; j < RING; j++) cr[j] = next++;
|
||||
ra0 = next, next += (uint32_t)rings, rc0 = next, next += (uint32_t)rings;
|
||||
}
|
||||
}
|
||||
uint64_t p = 5;
|
||||
memcpy(o, w, 5 * 4);
|
||||
for (uint64_t i = 5; i < ann_end;) { // the header up to the annotations' end: entry points widened, the mode added
|
||||
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
||||
if (early && i == em_end) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS;
|
||||
memcpy(o + p, w + i, l * 4);
|
||||
if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) o[p] = (l + 1) << 16 | op, o[p + l] = var, p++;
|
||||
if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) {
|
||||
uint32_t nv = 0, *at = o + p + l + 1; // the marks buffer, then pv0.., ra0.. and rc0..
|
||||
o[p] = (uint32_t)(l + 1 + deferred + 2 * rings) << 16 | op, o[p + l] = var, p++;
|
||||
for (uint32_t k = 0; k < deferred; k++) at[nv++] = pv0 + k;
|
||||
for (uint32_t k = 0; k < rings; k++) at[nv++] = ra0 + k;
|
||||
for (uint32_t k = 0; k < rings; k++) at[nv++] = rc0 + k;
|
||||
p += nv;
|
||||
}
|
||||
p += l;
|
||||
i += l;
|
||||
}
|
||||
@@ -2321,19 +2389,109 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui
|
||||
o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_st, o[p++] = SC_STORAGE_BUFFER, o[p++] = st;
|
||||
if (new_ptr_u) o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_u, o[p++] = SC_STORAGE_BUFFER, o[p++] = uint_t;
|
||||
o[p++] = 4u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_st, o[p++] = var, o[p++] = SC_STORAGE_BUFFER;
|
||||
if (stores) {
|
||||
if (new_bool) o[p++] = 2u << 16 | OP_TYPE_BOOL, o[p++] = bool_t;
|
||||
o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_pv, o[p++] = SC_PRIVATE, o[p++] = uint_t;
|
||||
o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_none, o[p++] = 0xffffffffu;
|
||||
o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_spare, o[p++] = VK_POOL_TEX;
|
||||
if (rings) {
|
||||
for (int j = 0; j < RING; j++) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = cr[j], o[p++] = (uint32_t)j;
|
||||
o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_ring, o[p++] = RING;
|
||||
o[p++] = 4u << 16 | OP_TYPE_ARRAY_, o[p++] = ring_t, o[p++] = uint_t, o[p++] = c_ring;
|
||||
o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_ring, o[p++] = SC_PRIVATE, o[p++] = ring_t;
|
||||
o[p++] = (3u + RING) << 16 | SPV_OP_CONSTANT_COMPOSITE, o[p++] = ring_t, o[p++] = ring_none;
|
||||
for (int j = 0; j < RING; j++) o[p++] = c_none;
|
||||
}
|
||||
for (uint32_t k = 0; k < deferred; k++)
|
||||
o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_pv, o[p++] = pv0 + k, o[p++] = SC_PRIVATE, o[p++] = c_none;
|
||||
for (uint32_t k = 0; k < rings; k++)
|
||||
o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_ring, o[p++] = ra0 + k, o[p++] = SC_PRIVATE, o[p++] = ring_none;
|
||||
for (uint32_t k = 0; k < rings; k++)
|
||||
o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_pv, o[p++] = rc0 + k, o[p++] = SC_PRIVATE, o[p++] = c0;
|
||||
}
|
||||
uint64_t site = 0;
|
||||
uint32_t k_def = 0, k_ring = 0, cur = 0; // cur: the label of the block being copied
|
||||
fn = 0;
|
||||
for (uint64_t i = fn_start; i < n;) {
|
||||
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
||||
if (op == SPV_OP_FUNCTION && l >= 3) fn = w[i + 2];
|
||||
if (op == OP_LABEL_ && l >= 2) cur = w[i + 1];
|
||||
if (stores && (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER ||
|
||||
(fn == fs_entry && (op == OP_RETURN || op == OP_RETURN_VALUE)))) {
|
||||
// marks[the slot, or the spare when none was noted] = 1
|
||||
#define POOL_MARK_STORE(from) \
|
||||
do { \
|
||||
uint32_t v_ = next++, ok_ = next++, ix_ = next++, at_ = next++; \
|
||||
o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = v_, o[p++] = (from); \
|
||||
o[p++] = 5u << 16 | OP_INOT_EQUAL, o[p++] = bool_t, o[p++] = ok_, o[p++] = v_, o[p++] = c_none; \
|
||||
o[p++] = 6u << 16 | OP_SELECT_, o[p++] = uint_t, o[p++] = ix_, o[p++] = ok_, o[p++] = v_, o[p++] = c_spare; \
|
||||
o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at_, o[p++] = var, o[p++] = c0, o[p++] = ix_; \
|
||||
o[p++] = 3u << 16 | OP_STORE_, o[p++] = at_, o[p++] = c1; \
|
||||
} while (0)
|
||||
for (uint32_t k = 0; k < deferred; k++) POOL_MARK_STORE(pv0 + k);
|
||||
for (uint32_t r = 0; r < rings; r++) {
|
||||
if (op == OP_DEMOTE_TO_HELPER) { // not the block's end: every entry, without a loop
|
||||
for (int j = 0; j < RING; j++) {
|
||||
uint32_t from = next++;
|
||||
o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = from, o[p++] = ra0 + r, o[p++] = cr[j];
|
||||
POOL_MARK_STORE(from);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// the block ends here (a return, kill or terminate), so it can end in a loop over the entries noted:
|
||||
// for (j = 0; j < min(count, RING); j++) mark ring[j]
|
||||
uint32_t cnt = next++, lt = next++, lim = next++, hd = next++, j = next++, mg = next++, ct = next++,
|
||||
cond = next++, body = next++, from = next++, jn = next++;
|
||||
o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = cnt, o[p++] = rc0 + r;
|
||||
o[p++] = 5u << 16 | OP_ULESS_THAN, o[p++] = bool_t, o[p++] = lt, o[p++] = cnt, o[p++] = c_ring;
|
||||
o[p++] = 6u << 16 | OP_SELECT_, o[p++] = uint_t, o[p++] = lim, o[p++] = lt, o[p++] = cnt, o[p++] = c_ring;
|
||||
o[p++] = 2u << 16 | OP_BRANCH, o[p++] = hd;
|
||||
o[p++] = 2u << 16 | OP_LABEL_, o[p++] = hd;
|
||||
o[p++] = 7u << 16 | OP_PHI, o[p++] = uint_t, o[p++] = j, o[p++] = c0, o[p++] = cur, o[p++] = jn, o[p++] = ct;
|
||||
o[p++] = 5u << 16 | OP_ULESS_THAN, o[p++] = bool_t, o[p++] = cond, o[p++] = j, o[p++] = lim;
|
||||
o[p++] = 4u << 16 | OP_LOOP_MERGE, o[p++] = mg, o[p++] = ct, o[p++] = 0;
|
||||
o[p++] = 4u << 16 | OP_BRANCH_CONDITIONAL, o[p++] = cond, o[p++] = body, o[p++] = mg;
|
||||
o[p++] = 2u << 16 | OP_LABEL_, o[p++] = body;
|
||||
o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = from, o[p++] = ra0 + r, o[p++] = j;
|
||||
POOL_MARK_STORE(from);
|
||||
o[p++] = 2u << 16 | OP_BRANCH, o[p++] = ct;
|
||||
o[p++] = 2u << 16 | OP_LABEL_, o[p++] = ct;
|
||||
o[p++] = 5u << 16 | OP_IADD, o[p++] = uint_t, o[p++] = jn, o[p++] = j, o[p++] = c1;
|
||||
o[p++] = 2u << 16 | OP_BRANCH, o[p++] = hd;
|
||||
o[p++] = 2u << 16 | OP_LABEL_, o[p++] = mg;
|
||||
cur = mg;
|
||||
}
|
||||
#undef POOL_MARK_STORE
|
||||
}
|
||||
memcpy(o + p, w + i, l * 4), p += l;
|
||||
if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] &&
|
||||
bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) {
|
||||
uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access
|
||||
o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4];
|
||||
o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1;
|
||||
uint8_t d = site_def[site++];
|
||||
if (d) { // the slot noted for the stores at the end
|
||||
uint32_t ix = w[i + 4];
|
||||
if (d & 2) o[p++] = 4u << 16 | OP_BITCAST, o[p++] = uint_t, o[p++] = next, o[p++] = ix, ix = next++;
|
||||
if (d & 4) { // ring[count % RING] = slot, count + 1
|
||||
uint32_t ra = ra0 + k_ring, rc = rc0 + k_ring++, cnt = next++, j = next++, at = next++, inc = next++;
|
||||
o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = cnt, o[p++] = rc;
|
||||
o[p++] = 5u << 16 | OP_BITWISE_AND, o[p++] = uint_t, o[p++] = j, o[p++] = cnt, o[p++] = cr[RING - 1];
|
||||
o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = at, o[p++] = ra, o[p++] = j;
|
||||
o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = ix;
|
||||
o[p++] = 5u << 16 | OP_IADD, o[p++] = uint_t, o[p++] = inc, o[p++] = cnt, o[p++] = c1;
|
||||
o[p++] = 3u << 16 | OP_STORE_, o[p++] = rc, o[p++] = inc;
|
||||
} else {
|
||||
o[p++] = 3u << 16 | OP_STORE_, o[p++] = pv0 + k_def++, o[p++] = ix;
|
||||
}
|
||||
} else {
|
||||
uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access
|
||||
o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4];
|
||||
o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1;
|
||||
}
|
||||
}
|
||||
i += l;
|
||||
}
|
||||
o[3] = next;
|
||||
free(bind), free(set2);
|
||||
POOL_MARK_FREE();
|
||||
#undef POOL_MARK_FREE
|
||||
*out_n = p;
|
||||
*status = 1;
|
||||
return o;
|
||||
|
||||
Reference in New Issue
Block a user