From b5d4afcfab4a0705a9aad55364267bdc9541d304 Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Tue, 26 May 2026 16:53:55 +0200 Subject: [PATCH 01/12] fix --- packages/typegpu/src/core/slot/accessor.ts | 6 +-- packages/typegpu/tests/tgsl/comptime.test.ts | 42 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/typegpu/src/core/slot/accessor.ts b/packages/typegpu/src/core/slot/accessor.ts index 109e1aa782..b5e30cfb97 100644 --- a/packages/typegpu/src/core/slot/accessor.ts +++ b/packages/typegpu/src/core/slot/accessor.ts @@ -2,7 +2,7 @@ import { type AnyData, isData } from '../../data/dataTypes.ts'; import { schemaCallWrapper } from '../../data/schemaCallWrapper.ts'; import { isSnippet, type ResolvedSnippet, snip } from '../../data/snippet.ts'; import type { BaseData } from '../../data/wgslTypes.ts'; -import { getResolutionCtx, inCodegenMode } from '../../execMode.ts'; +import { getResolutionCtx } from '../../execMode.ts'; import { getName, hasTinyestMetadata, setName } from '../../shared/meta.ts'; import type { InferGPU } from '../../shared/repr.ts'; import { @@ -193,7 +193,7 @@ export class TgpuAccessorImpl } get $(): InferGPU { - if (inCodegenMode()) { + if (getResolutionCtx()) { return this[$gpuValueOf]; } @@ -217,7 +217,7 @@ export class TgpuMutableAccessorImpl } get $(): InferGPU { - if (inCodegenMode()) { + if (getResolutionCtx()) { return this[$gpuValueOf]; } diff --git a/packages/typegpu/tests/tgsl/comptime.test.ts b/packages/typegpu/tests/tgsl/comptime.test.ts index dae9f759e4..03ec6345de 100644 --- a/packages/typegpu/tests/tgsl/comptime.test.ts +++ b/packages/typegpu/tests/tgsl/comptime.test.ts @@ -75,4 +75,46 @@ describe('comptime', () => { }" `); }); + + it('can read accessors during shader resolution', () => { + const value = tgpu.accessor(d.f32, 1); + const readValue = tgpu.comptime(() => value.$); + + const myFn = tgpu.fn( + [], + d.f32, + )(() => { + return readValue(); + }); + + expect(tgpu.resolve([myFn])).toMatchInlineSnapshot(` + "fn myFn() -> f32 { + return 1f; + }" + `); + + expect(tgpu.resolve([myFn.with(value, 2)])).toMatchInlineSnapshot(` + "fn myFn() -> f32 { + return 2f; + }" + `); + }); + + it('still throws when a comptime-read accessor has no value', () => { + const value = tgpu.accessor(d.f32); + const readValue = tgpu.comptime(() => value.$); + const myFn = tgpu.fn( + [], + d.f32, + )(() => { + return readValue(); + }); + + expect(() => tgpu.resolve([myFn])).toThrowErrorMatchingInlineSnapshot(` + [Error: Resolution of the following tree failed: + - + - fn:myFn + - fn:readValue: Missing value for 'slot:value'] + `); + }); }); From c8db87b5132fbe0d5f9bc83d5209e00a93a031b0 Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Tue, 26 May 2026 16:56:27 +0200 Subject: [PATCH 02/12] nit --- packages/typegpu/tests/tgsl/comptime.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/typegpu/tests/tgsl/comptime.test.ts b/packages/typegpu/tests/tgsl/comptime.test.ts index 03ec6345de..54a343ad98 100644 --- a/packages/typegpu/tests/tgsl/comptime.test.ts +++ b/packages/typegpu/tests/tgsl/comptime.test.ts @@ -100,7 +100,7 @@ describe('comptime', () => { `); }); - it('still throws when a comptime-read accessor has no value', () => { + it('throws when a comptime-read accessor has no value', () => { const value = tgpu.accessor(d.f32); const readValue = tgpu.comptime(() => value.$); const myFn = tgpu.fn( From 7997382fc1c0dc8ef42276b4c462b512e7585f19 Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Tue, 26 May 2026 16:57:48 +0200 Subject: [PATCH 03/12] ban turtles --- packages/typegpu/tests/tgsl/comptime.test.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/typegpu/tests/tgsl/comptime.test.ts b/packages/typegpu/tests/tgsl/comptime.test.ts index 54a343ad98..ad1b96f8eb 100644 --- a/packages/typegpu/tests/tgsl/comptime.test.ts +++ b/packages/typegpu/tests/tgsl/comptime.test.ts @@ -103,17 +103,16 @@ describe('comptime', () => { it('throws when a comptime-read accessor has no value', () => { const value = tgpu.accessor(d.f32); const readValue = tgpu.comptime(() => value.$); - const myFn = tgpu.fn( - [], - d.f32, - )(() => { + const myFn = () => { + 'use gpu'; return readValue(); - }); + }; expect(() => tgpu.resolve([myFn])).toThrowErrorMatchingInlineSnapshot(` [Error: Resolution of the following tree failed: - - - fn:myFn + - fn*:myFn + - fn*:myFn() - fn:readValue: Missing value for 'slot:value'] `); }); From e2faa8bf0d926ac29df8a3522b00d21d8069adce Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Fri, 20 Mar 2026 13:50:24 +0100 Subject: [PATCH 04/12] use unroll, extract elements per thread, add benchmark, failed test logging --- .../src/examples/tests/prefix-scan/index.ts | 132 ++++++++++++++---- .../src/scan/compute/applySums.ts | 13 +- .../typegpu-sort/src/scan/compute/scan.ts | 18 +-- packages/typegpu-sort/src/scan/prefixScan.ts | 3 +- packages/typegpu-sort/src/scan/schemas.ts | 2 + 5 files changed, 126 insertions(+), 42 deletions(-) diff --git a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts index 9049fd9982..6bfab5c866 100644 --- a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts +++ b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts @@ -8,6 +8,30 @@ const root = await tgpu.init({ device: { requiredFeatures: ['timestamp-query'] }, }); +function compareAndLog(actual: number[], expected: number[]): boolean { + if (isArrayEqual(actual, expected)) { + return true; + } + + if (actual.length !== expected.length) { + console.error(` Mismatch: length ${actual.length} !== ${expected.length}`); + } else if (actual.length <= 32) { + console.error(' actual: ', actual); + console.error(' expected:', expected); + } else { + const idx = actual.findIndex((v, i) => v !== expected[i]); + const lo = Math.max(0, idx - 2); + const hi = Math.min(actual.length, idx + 3); + console.error( + ` first mismatch at index ${idx} (showing [${lo}..${hi - 1}] of ${actual.length}):`, + ); + console.error(' actual: ', actual.slice(lo, hi)); + console.error(' expected:', expected.slice(lo, hi)); + } + + return false; +} + async function runAndCompare(arr: number[], op: BinaryOp, scanOnly: boolean) { const input = root.createBuffer(d.arrayOf(d.f32, arr.length), arr).$usage('storage'); @@ -24,7 +48,9 @@ async function runAndCompare(arr: number[], op: BinaryOp, scanOnly: boolean) { identityElement: op.identityElement, }); - return isArrayEqual(await output.read(), scanOnly ? scanJS(arr, op) : prefixScanJS(arr, op)); + const actual = await output.read(); + const expected = scanOnly ? scanJS(arr, op) : prefixScanJS(arr, op); + return compareAndLog(actual, expected); } // single element f32 tests @@ -88,7 +114,7 @@ async function testDoesNotDestroyBuffer(): Promise { identityElement: 0, }); - return isArrayEqual(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); + return compareAndLog(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); } async function testDoesNotCacheBuffers(): Promise { @@ -115,7 +141,7 @@ async function testDoesNotCacheBuffers(): Promise { identityElement: op.identityElement, }); - return isArrayEqual(await output1.read(), [36]) && isArrayEqual(await output2.read(), [10]); + return compareAndLog(await output1.read(), [36]) && compareAndLog(await output2.read(), [10]); } // prefix f32 tests @@ -179,7 +205,7 @@ async function testPrefixDoesNotDestroyBuffer(): Promise { operation: addFn, identityElement: 0, }); - return isArrayEqual(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); + return compareAndLog(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); } async function testPrefixDoesNotCacheBuffers(): Promise { @@ -206,37 +232,83 @@ async function testPrefixDoesNotCacheBuffers(): Promise { }); return ( - isArrayEqual(await output1.read(), prefixScanJS(arr1, op)) && - isArrayEqual(await output2.read(), prefixScanJS(arr2, op)) + compareAndLog(await output1.read(), prefixScanJS(arr1, op)) && + compareAndLog(await output2.read(), prefixScanJS(arr2, op)) ); } +// benchmark + +const BENCH_SIZES = [2_048, 65_536, 1_048_576, 16_777_216]; +const BENCH_WARMUP = 3; +const BENCH_RUNS = 10; + +async function benchmarkSize(size: number): Promise { + const buf = root.createBuffer(d.arrayOf(d.f32, size)).$usage('storage'); + + for (let i = 0; i < BENCH_WARMUP; i++) { + prefixScan(root, { inputBuffer: buf, operation: addFn, identityElement: 0 }); + await root.device.queue.onSubmittedWorkDone(); + } + + let total = 0; + for (let i = 0; i < BENCH_RUNS; i++) { + const t0 = performance.now(); + prefixScan(root, { inputBuffer: buf, operation: addFn, identityElement: 0 }); + await root.device.queue.onSubmittedWorkDone(); + total += performance.now() - t0; + } + + return total / BENCH_RUNS; +} + +async function runBenchmarks(): Promise { + console.log('=== Prefix Scan Benchmark ==='); + for (const size of BENCH_SIZES) { + const avgMs = await benchmarkSize(size); + console.log( + ` size ${size.toLocaleString().padStart(12)}: ${avgMs.toFixed(2)} ms avg (${BENCH_RUNS} runs)`, + ); + } + console.log('=============================='); +} + // running the tests +async function runTest(name: string, fn: () => Promise): Promise { + const passed = await fn(); + if (!passed) { + console.error(`FAILED: ${name}`); + } + return passed; +} + async function runTests(): Promise { let result = true; - result = (await testAdd8()) && result; - result = (await testAdd123()) && result; - result = (await testMul()) && result; - result = (await testStdMax()) && result; - result = (await testConcat()) && result; - result = (await testLength1()) && result; - result = (await testLength65537()) && result; - result = (await testLength16777217()) && result; - result = (await testDoesNotDestroyBuffer()) && result; - result = (await testDoesNotCacheBuffers()) && result; - - result = (await testPrefixAdd8()) && result; - result = (await testPrefixAdd123()) && result; - result = (await testPrefixMul()) && result; - result = (await testPrefixStdMax()) && result; - result = (await testPrefixConcat()) && result; - result = (await testPrefixLength1()) && result; - result = (await testPrefixLength65537()) && result; - result = (await testPrefixLength16777217()) && result; - result = (await testPrefixDoesNotDestroyBuffer()) && result; - result = (await testPrefixDoesNotCacheBuffers()) && result; + result = (await runTest('testAdd8', testAdd8)) && result; + result = (await runTest('testAdd123', testAdd123)) && result; + result = (await runTest('testMul', testMul)) && result; + result = (await runTest('testStdMax', testStdMax)) && result; + result = (await runTest('testConcat', testConcat)) && result; + result = (await runTest('testLength1', testLength1)) && result; + result = (await runTest('testLength65537', testLength65537)) && result; + result = (await runTest('testLength16777217', testLength16777217)) && result; + result = (await runTest('testDoesNotDestroyBuffer', testDoesNotDestroyBuffer)) && result; + result = (await runTest('testDoesNotCacheBuffers', testDoesNotCacheBuffers)) && result; + + result = (await runTest('testPrefixAdd8', testPrefixAdd8)) && result; + result = (await runTest('testPrefixAdd123', testPrefixAdd123)) && result; + result = (await runTest('testPrefixMul', testPrefixMul)) && result; + result = (await runTest('testPrefixStdMax', testPrefixStdMax)) && result; + result = (await runTest('testPrefixConcat', testPrefixConcat)) && result; + result = (await runTest('testPrefixLength1', testPrefixLength1)) && result; + result = (await runTest('testPrefixLength65537', testPrefixLength65537)) && result; + result = (await runTest('testPrefixLength16777217', testPrefixLength16777217)) && result; + result = + (await runTest('testPrefixDoesNotDestroyBuffer', testPrefixDoesNotDestroyBuffer)) && result; + result = + (await runTest('testPrefixDoesNotCacheBuffers', testPrefixDoesNotCacheBuffers)) && result; return result; } @@ -245,8 +317,10 @@ const table = document.querySelector('.result'); if (!table) { throw new Error('Nowhere to display the results'); } -void runTests().then((result) => { - table.innerText = `Tests ${result ? 'succeeded' : 'failed'}.`; +void runTests().then(async (result) => { + table.innerText = `Tests ${result ? 'succeeded' : 'failed'}. Running benchmarks...`; + await runBenchmarks(); + table.innerText = `Tests ${result ? 'succeeded' : 'failed'}. Benchmark complete (see console).`; }); // #region Example controls and cleanup diff --git a/packages/typegpu-sort/src/scan/compute/applySums.ts b/packages/typegpu-sort/src/scan/compute/applySums.ts index 64b7db6578..ddc2eb64a1 100644 --- a/packages/typegpu-sort/src/scan/compute/applySums.ts +++ b/packages/typegpu-sort/src/scan/compute/applySums.ts @@ -1,5 +1,11 @@ import { tgpu, d } from 'typegpu'; -import { operatorSlot, uniformOpLayout, WORKGROUP_SIZE } from '../schemas.ts'; +import { + ELEMENTS_PER_THREAD, + ELEMENTS_RANGE, + operatorSlot, + uniformOpLayout, + WORKGROUP_SIZE, +} from '../schemas.ts'; export const uniformOp = tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], @@ -10,11 +16,10 @@ export const uniformOp = tgpu.computeFn({ })(({ gid, wid }) => { const globalIdx = gid.x; const workgroupId = wid.x; - const baseIdx = globalIdx * 8; + const baseIdx = globalIdx * ELEMENTS_PER_THREAD; const opValue = uniformOpLayout.$.sums[workgroupId]; - // TODO: use `tgpu.unroll(8)` - for (let i = d.u32(0); i < 8; i++) { + for (const i of tgpu.unroll(ELEMENTS_RANGE)) { if (baseIdx + i < uniformOpLayout.$.input.length) { (uniformOpLayout.$.input[baseIdx + i] as number) = operatorSlot.$( opValue as number, diff --git a/packages/typegpu-sort/src/scan/compute/scan.ts b/packages/typegpu-sort/src/scan/compute/scan.ts index fca23d6c0c..e1f45cbbe8 100644 --- a/packages/typegpu-sort/src/scan/compute/scan.ts +++ b/packages/typegpu-sort/src/scan/compute/scan.ts @@ -1,5 +1,7 @@ import { tgpu, d, std } from 'typegpu'; import { + ELEMENTS_PER_THREAD, + ELEMENTS_RANGE, identitySlot, onlyGreatestElementSlot, operatorSlot, @@ -8,7 +10,9 @@ import { } from '../schemas.ts'; import { downsweep, upsweep, workgroupMemory } from './shared.ts'; -const fillIdentityArray = tgpu.comptime(() => Array.from({ length: 8 }, () => identitySlot.$)); +const fillIdentityArray = tgpu.comptime(() => + Array.from({ length: ELEMENTS_PER_THREAD }, () => identitySlot.$), +); export const computeBlock = tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], @@ -22,19 +26,17 @@ export const computeBlock = tgpu.computeFn({ const workgroupId = wid.x; const localIdx = lid.x; - // 8 elements per thread - const baseIdx = globalIdx * 8; + const baseIdx = globalIdx * ELEMENTS_PER_THREAD; - const partialSums = d.arrayOf(d.f32, 8)(fillIdentityArray()); + const partialSums = d.arrayOf(d.f32, ELEMENTS_PER_THREAD)(fillIdentityArray()); let prev = d.f32(identitySlot.$); let lastIdx = d.u32(0); - // TODO: use `tgpu.unroll(8)` - for (let i = d.u32(); i < 8; i++) { + for (const i of tgpu.unroll(ELEMENTS_RANGE)) { if (baseIdx + i < scanLayout.$.input.length) { partialSums[i] = operatorSlot.$(prev, scanLayout.$.input[baseIdx + i] as number); - prev = partialSums[i] as number; + prev = partialSums[i]; lastIdx = i; } } @@ -56,7 +58,7 @@ export const computeBlock = tgpu.computeFn({ const scannedSum = workgroupMemory.$[localIdx]; - for (let i = d.u32(0); i < 8; i++) { + for (const i of tgpu.unroll(ELEMENTS_RANGE)) { if (baseIdx + i < scanLayout.$.input.length) { if (i === 0) { scanLayout.$.input[baseIdx + i] = scannedSum; diff --git a/packages/typegpu-sort/src/scan/prefixScan.ts b/packages/typegpu-sort/src/scan/prefixScan.ts index b5797de946..5db2e17163 100644 --- a/packages/typegpu-sort/src/scan/prefixScan.ts +++ b/packages/typegpu-sort/src/scan/prefixScan.ts @@ -8,6 +8,7 @@ import { } from 'typegpu'; import type { BinaryOp } from './types.ts'; import { + ELEMENTS_PER_THREAD, identitySlot, onlyGreatestElementSlot, operatorSlot, @@ -83,7 +84,7 @@ export class PrefixScanComputer { querySet: TgpuQuerySet<'timestamp'> | null, isFirstPass: boolean, ): TgpuBuffer> & StorageFlag { - const numWorkgroups = Math.ceil(actualLength / (WORKGROUP_SIZE * 8)); + const numWorkgroups = Math.ceil(actualLength / (WORKGROUP_SIZE * ELEMENTS_PER_THREAD)); const scanPipeline = this.getScanPipeline(onlyGreatestElement); // Base case: single workgroup diff --git a/packages/typegpu-sort/src/scan/schemas.ts b/packages/typegpu-sort/src/scan/schemas.ts index f8709b4e4c..ace5da9db2 100644 --- a/packages/typegpu-sort/src/scan/schemas.ts +++ b/packages/typegpu-sort/src/scan/schemas.ts @@ -1,6 +1,8 @@ import { tgpu, d } from 'typegpu'; export const WORKGROUP_SIZE = 256; +export const ELEMENTS_PER_THREAD = 8; +export const ELEMENTS_RANGE = Array.from({ length: ELEMENTS_PER_THREAD }, (_, i) => i); export const scanLayout = tgpu.bindGroupLayout({ input: { storage: d.arrayOf(d.f32), access: 'mutable' }, From 6b5a307ac3acd4133e598bcbe86ca2724f935f9a Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Tue, 26 May 2026 16:36:40 +0200 Subject: [PATCH 05/12] review changes --- .../src/examples/tests/prefix-scan/index.ts | 28 +++++++++++++++---- .../src/scan/compute/applySums.ts | 5 ++-- .../typegpu-sort/src/scan/compute/scan.ts | 5 ++-- packages/typegpu-sort/src/scan/schemas.ts | 1 - 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts index 6bfab5c866..65f1d3b6de 100644 --- a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts +++ b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts @@ -244,22 +244,27 @@ const BENCH_WARMUP = 3; const BENCH_RUNS = 10; async function benchmarkSize(size: number): Promise { - const buf = root.createBuffer(d.arrayOf(d.f32, size)).$usage('storage'); + const inputData = Array.from({ length: size }, () => 1); + const inputBuffer = root.createBuffer(d.arrayOf(d.f32, size), inputData).$usage('storage'); + const outputBuffer = root.createBuffer(d.arrayOf(d.f32, size)).$usage('storage'); for (let i = 0; i < BENCH_WARMUP; i++) { - prefixScan(root, { inputBuffer: buf, operation: addFn, identityElement: 0 }); + prefixScan(root, { inputBuffer, outputBuffer, operation: addFn, identityElement: 0 }); await root.device.queue.onSubmittedWorkDone(); } let total = 0; for (let i = 0; i < BENCH_RUNS; i++) { const t0 = performance.now(); - prefixScan(root, { inputBuffer: buf, operation: addFn, identityElement: 0 }); + prefixScan(root, { inputBuffer, outputBuffer, operation: addFn, identityElement: 0 }); await root.device.queue.onSubmittedWorkDone(); total += performance.now() - t0; } - return total / BENCH_RUNS; + const avgMs = total / BENCH_RUNS; + inputBuffer.destroy(); + outputBuffer.destroy(); + return avgMs; } async function runBenchmarks(): Promise { @@ -318,9 +323,20 @@ if (!table) { throw new Error('Nowhere to display the results'); } void runTests().then(async (result) => { - table.innerText = `Tests ${result ? 'succeeded' : 'failed'}. Running benchmarks...`; + if (!result) { + table.innerText = 'Tests failed. Benchmarks skipped.'; + return; + } + + const shouldRunBenchmarks = new URLSearchParams(window.location.search).get('bench') === '1'; + if (!shouldRunBenchmarks) { + table.innerText = "Tests succeeded. Benchmarks skipped (enable with '?bench=1' in the URL)."; + return; + } + + table.innerText = 'Tests succeeded. Running benchmarks...'; await runBenchmarks(); - table.innerText = `Tests ${result ? 'succeeded' : 'failed'}. Benchmark complete (see console).`; + table.innerText = 'Tests succeeded. Benchmark complete (see console).'; }); // #region Example controls and cleanup diff --git a/packages/typegpu-sort/src/scan/compute/applySums.ts b/packages/typegpu-sort/src/scan/compute/applySums.ts index ddc2eb64a1..8c3e87ca63 100644 --- a/packages/typegpu-sort/src/scan/compute/applySums.ts +++ b/packages/typegpu-sort/src/scan/compute/applySums.ts @@ -1,7 +1,6 @@ -import { tgpu, d } from 'typegpu'; +import { tgpu, d, std } from 'typegpu'; import { ELEMENTS_PER_THREAD, - ELEMENTS_RANGE, operatorSlot, uniformOpLayout, WORKGROUP_SIZE, @@ -19,7 +18,7 @@ export const uniformOp = tgpu.computeFn({ const baseIdx = globalIdx * ELEMENTS_PER_THREAD; const opValue = uniformOpLayout.$.sums[workgroupId]; - for (const i of tgpu.unroll(ELEMENTS_RANGE)) { + for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { if (baseIdx + i < uniformOpLayout.$.input.length) { (uniformOpLayout.$.input[baseIdx + i] as number) = operatorSlot.$( opValue as number, diff --git a/packages/typegpu-sort/src/scan/compute/scan.ts b/packages/typegpu-sort/src/scan/compute/scan.ts index e1f45cbbe8..daa5665a6b 100644 --- a/packages/typegpu-sort/src/scan/compute/scan.ts +++ b/packages/typegpu-sort/src/scan/compute/scan.ts @@ -1,7 +1,6 @@ import { tgpu, d, std } from 'typegpu'; import { ELEMENTS_PER_THREAD, - ELEMENTS_RANGE, identitySlot, onlyGreatestElementSlot, operatorSlot, @@ -33,7 +32,7 @@ export const computeBlock = tgpu.computeFn({ let prev = d.f32(identitySlot.$); let lastIdx = d.u32(0); - for (const i of tgpu.unroll(ELEMENTS_RANGE)) { + for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { if (baseIdx + i < scanLayout.$.input.length) { partialSums[i] = operatorSlot.$(prev, scanLayout.$.input[baseIdx + i] as number); prev = partialSums[i]; @@ -58,7 +57,7 @@ export const computeBlock = tgpu.computeFn({ const scannedSum = workgroupMemory.$[localIdx]; - for (const i of tgpu.unroll(ELEMENTS_RANGE)) { + for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { if (baseIdx + i < scanLayout.$.input.length) { if (i === 0) { scanLayout.$.input[baseIdx + i] = scannedSum; diff --git a/packages/typegpu-sort/src/scan/schemas.ts b/packages/typegpu-sort/src/scan/schemas.ts index ace5da9db2..89ce4933d6 100644 --- a/packages/typegpu-sort/src/scan/schemas.ts +++ b/packages/typegpu-sort/src/scan/schemas.ts @@ -2,7 +2,6 @@ import { tgpu, d } from 'typegpu'; export const WORKGROUP_SIZE = 256; export const ELEMENTS_PER_THREAD = 8; -export const ELEMENTS_RANGE = Array.from({ length: ELEMENTS_PER_THREAD }, (_, i) => i); export const scanLayout = tgpu.bindGroupLayout({ input: { storage: d.arrayOf(d.f32), access: 'mutable' }, From 2bd540f08cccb06ea843ea966434143fc877ceae Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Tue, 26 May 2026 16:47:46 +0200 Subject: [PATCH 06/12] better --- .../src/examples/tests/prefix-scan/index.ts | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts index 65f1d3b6de..f802498dcd 100644 --- a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts +++ b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts @@ -2,6 +2,7 @@ import { tgpu } from 'typegpu'; import * as d from 'typegpu/data'; import { type BinaryOp, prefixScan, scan } from '@typegpu/sort'; import * as std from 'typegpu/std'; +import { defineControls } from '../../common/defineControls.ts'; import { addFn, concat10, isArrayEqual, mulFn, prefixScanJS, scanJS } from './functions.ts'; const root = await tgpu.init({ @@ -322,25 +323,50 @@ const table = document.querySelector('.result'); if (!table) { throw new Error('Nowhere to display the results'); } -void runTests().then(async (result) => { - if (!result) { - table.innerText = 'Tests failed. Benchmarks skipped.'; + +let testsPassed: boolean | null = null; +let benchmarkPromise: Promise | null = null; + +void runTests().then((result) => { + testsPassed = result; + table.innerText = `Tests ${result ? 'succeeded' : 'failed'}.`; +}); + +async function startBenchmarks(): Promise { + if (testsPassed === null) { + table.innerText = 'Tests are still running.'; return; } - const shouldRunBenchmarks = new URLSearchParams(window.location.search).get('bench') === '1'; - if (!shouldRunBenchmarks) { - table.innerText = "Tests succeeded. Benchmarks skipped (enable with '?bench=1' in the URL)."; + if (!testsPassed) { + table.innerText = 'Tests failed. Benchmarks skipped.'; return; } + if (benchmarkPromise) { + return benchmarkPromise; + } + table.innerText = 'Tests succeeded. Running benchmarks...'; - await runBenchmarks(); - table.innerText = 'Tests succeeded. Benchmark complete (see console).'; -}); + benchmarkPromise = runBenchmarks() + .then(() => { + table.innerText = 'Tests succeeded. Benchmark complete (see console).'; + }) + .finally(() => { + benchmarkPromise = null; + }); + + return benchmarkPromise; +} // #region Example controls and cleanup +export const controls = defineControls({ + 'Run benchmarks': { + onButtonClick: startBenchmarks, + }, +}); + export function onCleanup() { root.destroy(); } From a0b05d6989c80328b7609501f2ad0d7de54c85cd Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Wed, 15 Jul 2026 20:21:42 +0200 Subject: [PATCH 07/12] work --- .../algorithms/concurrent-chart/calculator.ts | 138 ++-- .../algorithms/concurrent-chart/index.html | 156 ++++- .../algorithms/concurrent-chart/index.ts | 49 +- .../{bitonic-sort => sort}/index.html | 0 .../{bitonic-sort => sort}/index.ts | 189 +++-- .../{bitonic-sort => sort}/meta.json | 2 +- .../{bitonic-sort => sort}/thumbnail.png | Bin .../src/examples/tests/prefix-scan/index.ts | 120 +++- .../src/examples/tests/sort/index.html | 1 + .../src/examples/tests/sort/index.ts | 532 ++++++++++++++ .../src/examples/tests/sort/meta.json | 7 + .../bitonic-sort.test.ts | 144 ---- .../individual-example-tests/sort.test.ts | 282 ++++++++ packages/typegpu-sort/README.md | 132 +++- packages/typegpu-sort/package.json | 1 + .../typegpu-sort/src/bitonic/bitonicSort.ts | 658 ++++++++++++------ packages/typegpu-sort/src/bitonic/index.ts | 4 +- packages/typegpu-sort/src/bitonic/slots.ts | 15 + packages/typegpu-sort/src/bitonic/types.ts | 22 +- packages/typegpu-sort/src/index.ts | 7 +- packages/typegpu-sort/src/radix/count.ts | 45 ++ packages/typegpu-sort/src/radix/index.ts | 2 + packages/typegpu-sort/src/radix/radixSort.ts | 181 +++++ packages/typegpu-sort/src/radix/scatter.ts | 144 ++++ .../typegpu-sort/src/radix/scatterFallback.ts | 80 +++ packages/typegpu-sort/src/radix/schemas.ts | 115 +++ packages/typegpu-sort/src/radix/types.ts | 31 + packages/typegpu-sort/src/runPass.ts | 73 ++ .../src/scan/compute/applySums.ts | 50 +- .../typegpu-sort/src/scan/compute/scan.ts | 115 +-- .../typegpu-sort/src/scan/compute/shared.ts | 62 +- packages/typegpu-sort/src/scan/index.ts | 10 +- packages/typegpu-sort/src/scan/prefixScan.ts | 328 +++++---- packages/typegpu-sort/src/scan/schemas.ts | 30 +- packages/typegpu-sort/src/scan/types.ts | 7 +- packages/typegpu-sort/src/wgslUtils.ts | 13 + packages/typegpu-sort/tests/bitonic.test.ts | 54 ++ packages/typegpu-sort/tests/radix.test.ts | 90 +++ packages/typegpu-sort/tests/scan.test.ts | 65 ++ packages/typegpu-sort/tests/utils.ts | 11 + packages/typegpu-sort/vitest.config.mts | 14 + pnpm-lock.yaml | 3 + 42 files changed, 3215 insertions(+), 767 deletions(-) rename apps/typegpu-docs/src/examples/algorithms/{bitonic-sort => sort}/index.html (100%) rename apps/typegpu-docs/src/examples/algorithms/{bitonic-sort => sort}/index.ts (58%) rename apps/typegpu-docs/src/examples/algorithms/{bitonic-sort => sort}/meta.json (76%) rename apps/typegpu-docs/src/examples/algorithms/{bitonic-sort => sort}/thumbnail.png (100%) create mode 100644 apps/typegpu-docs/src/examples/tests/sort/index.html create mode 100644 apps/typegpu-docs/src/examples/tests/sort/index.ts create mode 100644 apps/typegpu-docs/src/examples/tests/sort/meta.json delete mode 100644 apps/typegpu-docs/tests/individual-example-tests/bitonic-sort.test.ts create mode 100644 apps/typegpu-docs/tests/individual-example-tests/sort.test.ts create mode 100644 packages/typegpu-sort/src/radix/count.ts create mode 100644 packages/typegpu-sort/src/radix/index.ts create mode 100644 packages/typegpu-sort/src/radix/radixSort.ts create mode 100644 packages/typegpu-sort/src/radix/scatter.ts create mode 100644 packages/typegpu-sort/src/radix/scatterFallback.ts create mode 100644 packages/typegpu-sort/src/radix/schemas.ts create mode 100644 packages/typegpu-sort/src/radix/types.ts create mode 100644 packages/typegpu-sort/src/runPass.ts create mode 100644 packages/typegpu-sort/src/wgslUtils.ts create mode 100644 packages/typegpu-sort/tests/bitonic.test.ts create mode 100644 packages/typegpu-sort/tests/radix.test.ts create mode 100644 packages/typegpu-sort/tests/scan.test.ts create mode 100644 packages/typegpu-sort/tests/utils.ts create mode 100644 packages/typegpu-sort/vitest.config.mts diff --git a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/calculator.ts b/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/calculator.ts index 737c1e69d7..3483ec61bc 100644 --- a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/calculator.ts +++ b/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/calculator.ts @@ -1,25 +1,30 @@ -import { createPrefixScanComputer, prefixScan } from '@typegpu/sort'; +import { createPrefixScanComputer } from '@typegpu/sort'; import type { TgpuRoot } from 'typegpu'; import { d, std } from 'typegpu'; type SumResult = { success: boolean; jsTime: number; - gpuTime: number; - gpuShaderTime: number; + uploadTime: number; + computeTime: number; + syncTime: number; + readbackTime: number; }; -function prefixSumOnJS(arr: number[]) { - for (let i = 1; i < arr.length; i++) { - arr[i] += arr[i - 1]; +const ITERATIONS = 3; + +// Exclusive (Blelloch) scan — the result starts with the identity element +function prefixSumOnJS(arr: Float32Array) { + let acc = 0; + for (let i = 0; i < arr.length; i++) { + const value = arr[i]; + arr[i] = acc; + acc += value; } - // In Blelloch scan, the result starts with identity element - arr.unshift(0); - arr.pop(); return arr; } -function arraysEqual(a: number[], b: number[]): boolean { +function arraysEqual(a: Float32Array, b: Float32Array): boolean { if (a.length !== b.length) { return false; } @@ -31,45 +36,92 @@ function arraysEqual(a: number[], b: number[]): boolean { return true; } +function median(values: number[]): number { + return values.toSorted((a, b) => a - b)[values.length >> 1]; +} + export async function performCalculationsWithTime( root: TgpuRoot, - inputArray: number[], + input: Float32Array, ): Promise { - const arraySize = inputArray.length; - const inputBuffer = root.createBuffer(d.arrayOf(d.f32, arraySize)).$usage('storage'); - inputBuffer.write(inputArray); + const device = root.device; + const inputBuffer = root.createBuffer(d.arrayOf(d.f32, input.length)).$usage('storage'); + const computer = createPrefixScanComputer(root, { operation: std.add, identityElement: 0 }); + const plan = computer.prepare(inputBuffer); + const querySet = root.createQuerySet('timestamp', 2); + const readbackBuffer = device.createBuffer({ + size: input.byteLength, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); - // JS version - const jsStartTime = performance.now(); - const jsResult = prefixSumOnJS(inputArray); - const jsTime = performance.now() - jsStartTime; + const upload = () => device.queue.writeBuffer(root.unwrap(inputBuffer), 0, input); - // GPU version - createPrefixScanComputer(root, { operation: std.add, identityElement: 0 }); - const querySet = root.createQuerySet('timestamp', 2); - const gpuStartTime = performance.now(); - const calcResult = prefixScan( - root, - { - inputBuffer: inputBuffer, - outputBuffer: inputBuffer, - operation: std.add, - identityElement: 0, - }, - querySet, - ); - querySet.resolve(); - await root.device.queue.onSubmittedWorkDone(); - const gpuTime = performance.now() - gpuStartTime; - - const gpuResult = await calcResult.read(); - const timestamps = await querySet.read(); - const gpuShaderTime = Number(timestamps[1] - timestamps[0]) / 1_000_000; + const compute = async () => { + const encoder = device.createCommandEncoder(); + plan.run({ encoder, querySet }); + device.queue.submit([encoder.finish()]); + await device.queue.onSubmittedWorkDone(); + }; + + const readback = async () => { + const encoder = device.createCommandEncoder(); + encoder.copyBufferToBuffer(root.unwrap(inputBuffer), 0, readbackBuffer, 0, input.byteLength); + device.queue.submit([encoder.finish()]); + await readbackBuffer.mapAsync(GPUMapMode.READ); + return new Float32Array(readbackBuffer.getMappedRange()); + }; + + const jsTimes: number[] = []; + let jsResult: Float32Array = new Float32Array(0); + for (let i = 0; i < ITERATIONS; i++) { + const copy = input.slice(); + const start = performance.now(); + jsResult = prefixSumOnJS(copy); + jsTimes.push(performance.now() - start); + } + + // Untimed warmup — compiles the pipelines and doubles as the correctness check + upload(); + await compute(); + const success = arraysEqual(jsResult, await readback()); + readbackBuffer.unmap(); + + const uploadTimes: number[] = []; + const computeTimes: number[] = []; + const syncTimes: number[] = []; + const readbackTimes: number[] = []; + + for (let i = 0; i < ITERATIONS; i++) { + const t0 = performance.now(); + upload(); + const t1 = performance.now(); + await compute(); + const t2 = performance.now(); + await readback(); + const t3 = performance.now(); + readbackBuffer.unmap(); + + querySet.resolve(); + const timestamps = await querySet.read(); + const passMs = Number(timestamps[1] - timestamps[0]) / 1_000_000; + + uploadTimes.push(t1 - t0); + computeTimes.push(passMs); + syncTimes.push(Math.max(0, t2 - t1 - passMs)); + readbackTimes.push(t3 - t2); + } + + plan.destroy(); + querySet.destroy(); + inputBuffer.destroy(); + readbackBuffer.destroy(); return { - success: arraysEqual(jsResult, gpuResult), - jsTime, - gpuTime, - gpuShaderTime, + success, + jsTime: median(jsTimes), + uploadTime: median(uploadTimes), + computeTime: median(computeTimes), + syncTime: median(syncTimes), + readbackTime: median(readbackTimes), }; } diff --git a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/index.html b/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/index.html index 74f59eb01c..9d98c29f30 100644 --- a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/index.html +++ b/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/index.html @@ -20,11 +20,11 @@
-
-
-
-
-
+
+
+
+
+
@@ -37,11 +37,11 @@
-
-
-
-
-
+
+
+
+
+
@@ -54,11 +54,11 @@
-
-
-
-
-
+
+
+
+
+
@@ -71,11 +71,11 @@
-
-
-
-
-
+
+
+
+
+
@@ -88,11 +88,11 @@
-
-
-
-
-
+
+
+
+
+
@@ -100,14 +100,25 @@ + +
+ JS (single thread) + Upload + GPU compute + Submit & sync + Readback + Nx = JS / GPU compute (GPU-resident data) +
diff --git a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/index.ts b/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/index.ts deleted file mode 100644 index 1c48be86ec..0000000000 --- a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/index.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { tgpu } from 'typegpu'; -import { defineControls } from '../../common/defineControls.ts'; -import { performCalculationsWithTime } from './calculator.ts'; - -const SIZES = [21037, 131072, 1048576, 4194304, 8388608] as const; - -const root = await tgpu.init({ - device: { - requiredFeatures: ['timestamp-query'], - }, -}); - -const dataGroups = Array.from(document.querySelectorAll('.data-group')); -const yAxisLabels = Array.from(document.querySelectorAll('.y-axis-labels span')); - -const results = SIZES.map(() => ({ - jsTime: 0, - uploadTime: 0, - computeTime: 0, - syncTime: 0, - readbackTime: 0, -})); - -const SEGMENTS = [ - { cls: '.seg-upload', key: 'uploadTime', label: 'Upload' }, - { cls: '.seg-compute', key: 'computeTime', label: 'GPU compute' }, - { cls: '.seg-sync', key: 'syncTime', label: 'Submit & sync' }, - { cls: '.seg-readback', key: 'readbackTime', label: 'Readback' }, -] as const; - -function drawCharts() { - const overallMax = Math.max( - ...results.map((r) => - Math.max(r.jsTime, r.uploadTime + r.computeTime + r.syncTime + r.readbackTime), - ), - ); - - // Update y-axis - const ticks = - overallMax <= 0 ? [0, 0, 0, 0, 0] : Array.from({ length: 5 }, (_, i) => (i / 4) * overallMax); - for (const [i, label] of yAxisLabels.toReversed().entries()) { - label.textContent = ticks[i].toFixed(1); - } - - for (const [i, group] of dataGroups.entries()) { - const r = results[i]; - - // Update speedup label - const speedup = r.computeTime > 0 ? (r.jsTime / r.computeTime).toFixed(1) : '-'; - (group.querySelector('.speedup-label') as HTMLDivElement).textContent = `${speedup}x`; - - // Update bars and tooltips - const jsBar = group.querySelector('.bar-js') as HTMLDivElement; - jsBar.style.setProperty('--bar-height', `${overallMax > 0 ? r.jsTime / overallMax : 0}`); - (jsBar.querySelector('.bar-tooltip') as HTMLDivElement).textContent = - `JS: ${r.jsTime.toFixed(2)}ms`; - - for (const s of SEGMENTS) { - const segment = group.querySelector(s.cls) as HTMLDivElement; - const value = r[s.key]; - segment.style.setProperty('--seg-height', `${overallMax > 0 ? value / overallMax : 0}`); - segment.classList.toggle('nonzero', value > 0); - (segment.querySelector('.bar-tooltip') as HTMLDivElement).textContent = - `${s.label}: ${value.toFixed(2)}ms`; - } - } -} - -async function runBenchmarks() { - for (const [i, size] of SIZES.entries()) { - const input = new Float32Array(size).fill(1); - const result = await performCalculationsWithTime(root, input); - if (result.success) { - results[i] = result; - } - } - drawCharts(); -} - -void runBenchmarks(); - -// #region Example controls & Cleanup - -export const controls = defineControls({ - Recalculate: { - onButtonClick: runBenchmarks, - }, -}); - -export function onCleanup() { - root.destroy(); -} - -// #endregion diff --git a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/meta.json b/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/meta.json deleted file mode 100644 index eb745e2c29..0000000000 --- a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/meta.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "title": "Concurrent Chart", - "category": "algorithms", - "tags": ["concurrent"], - "dev": true, - "coolFactor": 4 -} diff --git a/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/thumbnail.png b/apps/typegpu-docs/src/examples/algorithms/concurrent-chart/thumbnail.png deleted file mode 100644 index de4e7cb32957d493a200bc68350f145c51499be3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 124327 zcmeEucQ~8v`?oz?+G>lM9jbQGB1UL+&{{2uT5Z)9dy`l#RYhB?wNkaIRU*XfK+V`e z5K63w6=FnqZ_o35$MO4ppZCA_ulJkd$dTK9UnAFbo!51K&d=wRXal{w$Jx%Y(a_Kw z*VelAkcNg)iH3&m1q%~!=i(jXAQ~FBQb$cq18q%BVFOP$J4Y8=8XB$Wq!eZ&!_l*m z=!V7fVHyh$Z@Pth(`m7)ZBDwpxTJAKmgy$qLd3DEHoSTxwDOqY#lFJA5C7FJL{m;i6lIoQ?Pt0oX|@I#Y?Id{WR8jUcPHqVEn zVl436!LOxGln`47+?P7gxHDcY2G;aOd)SvhV3<5h0E7Q&OC+CtX(>L zpNEsC{dAw>;@K`cn&FY!NFPGq+0$5ZcT?lj@woSA=K z@w-J8nQZo>Zv7!+`S2wUi@fH{vY>kB75W&ht!^DzATPtA+@qv z@{+nvNe>F&W8)&QTI4B(_}+B*shUsvPHjPmyxfueS@YT_>xyapS!vvW?Nckxb#PA3 zfh|!rZ)c+Fg@MdExcuzW{01*O$natd!lTA@gS9IvW^JT=Fpt~ zMdSSK@>@=}Zye>!iD3~_ECqC;-yC~cq(dd9&ebrkgbS`QY=+%fV|o*&|KwD-F#oN{ zg14t~I2JYTyanBg;y>^Ai<>R-^CzAwg1&j&i!=^Jyh(&mg-%#z8XmbzW9% zAK5|1n?97=9Qj{l?T$BS|%woCz9`3FcNCGTT;s4g9JzdTaFNXu{Ww zuQr}?p2?omo*zB&p4|Ao*#{e_4V~TRyTpxrd=S1#yg>Zw=^8fkGfSsYXD*-S7E_k+ zJi~sPa3<(AI~nxg+vkB8!Qy7| zNbzstAo06KuElSPiwXma*o?Ldt&9bX6HL4w-8aIP_-aFLWfZLx^cdnwU}n_ox^bqFpZARd%oV&|LeVT zK=Ve{nV}A}Q;c!(kfJl6bCyrW)~ncR-8Z@~yXjGqYHzJ>HMz!A_*6_h3HnBk$v{}9 zPB_-76Jv=9%g)PDDN9m#X*HC0^NoQpy!%KN`rRDQjQ_6+wDyl^ywT$3ceBoqGr?T} zubwt*Jx_m9?<4DT5|!@t(H-GsIY0J0(v##Z;ZE>Q@!<55@d{bVT0FV>fFL{Pi2pb< z`upYj#`@UeR%l_zbNS+rHj@gIgnDeS7Wh2a7AzBd9Qv}c8&(cS;-s7M50npr4_j$Y zG#?u|vnFXEr|HlhbD(@b_`oUh=e@ExCffVr67_OO<-Wu}3t;@2?AXzA1SnU{m*X`zy{p9aIOpXu<3%>vyH+ ze5sPF(&TxV$-Y*&;Z5!DSI|i&twAXZN&Z(oE{F*)`uBT#-tvU9fkiIkRjni8Ki$kr zs+GNbQku2i$}E`MAnnkpvRUzKrs9dH3kWgYvyO?bF`Kv5U+oPD40H~t+rO|+Z%^!* zwV&*f>l#bFo=AL0_2N*D${0XBvLWUns01DYM_JTxTdCW`C&YQTA5(YGvBqhy(%vgj zlbut|(%{Ug8&`OH_^vlgH(4I9bF5?c4H>U= zLdTx!d>;`M9dtX0au0exjqJFJt#B)n%=(f)o?G8|GxwQu%THL(!3up3^Abmqprq=~ z)``{LA3rjq_{4oA?#Dfe&k+jnxwh}nhTeo1$BPL`sFI0FhyK>a?#AW_T_2y=d)?&6 zHIIx`JfH1G*NIN(DGc#q1WHrNWKDX&sCq;;)(8dWDRIPf-{3bFm< zV@yaWerK|ZWJLH>Q>v_+<~)c_%-J&=K2Zd2*O3XOkOWxedh%9DB=!*!O|{#?*6I;@ zB2^UpQdh<%AuEr^dOiFANzdyf~we*xU z&3;kvqMx-)klN0jH9OC1hn$P*;*(2FYLt8u9NBiEjS0t!(`}O3`?NW|bGI|TWF~9| zt)3g4nto;H%VzBoJN14W=2e+y0_( zjaK+BtwsYojmM}(h?;Zncu2Sz-C^{6;F?(_)6{czny?<48bntG$A$YKv}0BeO?CMR zO0bM77K!G;Gtfwj*?A9?hsjYir-T$R1BbaQA#ob%TW&K=G?K#Xxf(Q0Hvs_q+SXLt zPFI&k9JpqoVW2%n!w6i_0&g%a-#@SK&|aaT|KmO#4Na6I4Z~k$^nlN!*IVFyROio6 z`uCAE%)no#fVclAx_^{rRQg2!k88RYz%!Z~hML;iz^9>&r>(84mxG&k4cRUcxWVeK zW#&agb5`W&O{@Lz(i+e{#?i>s+f-La*~Seb{q(upGh1nYi2G4HG^+l}z$L`i`>C)$ z#KqN1*!aH;7li*P;_a+ViKR;*Ak{ z{G**)wq7=#j_%%$Zmz;d?LK|x=Hsn);lfcz|9t)VJ#GCR|J9SL*I&Z|1}Jm%L`GKn zs?0yz2CAwa-BmVl^tW{}yX6Q0<_zcqtZ?m`>L2C**OPzs_-{2$|5Z~~US8qftNz=g z|5?@8%hprV4FdG(4gS}9{Z;wjAO2NQRpw~u|27qW&hsC4fr$pQsmlCw(ZFnN1%W!i zI`TW-(l-J=0V_Lt(H>dol|Mg^t~XA>aG4P_G#WJ8w{95u(=Io#j!T*tmsz%hL_wm* z!hT(;WMg6JyUH5)KtqEe3LdKd#$?)X?#4$WmIwxx2s*m(W1iOw;GaBYl)_;;nBuMr z{6lfiaqPDtN8YnJmiSQ#fI1g1G24RYZhPDK~dO+M( zljRyTbSFY-7=&qPS^mf6hz4ETiK+2ss~^w*=c9jAXl6Yzb&KV1>W4PBo)T`Zt+0%m z{<{)1w0hV6cgKIV?jlJ`m(GXv;o(mR?jcz>Na)h1re*L-G__CKuV zAIH|4wB|8JC7H&Z_j6x6IfBZTv3@RI;~vbd6PoF2|LI^mIz7sz!VqlkOZkf_R)LEz zDk$5lZaam045v>0-FMSCU#>vgWA?3kStLANJN)|JHQXywewMapB}lQZ)dv|MWS_!> z{ktYZS-#v3t)xtL6Hza>JzVcb2LH`D`zEFN0jK-Q=D_Bn>vaCoz%4D!|9omc8E1%d z3iWz7W#`Lvqvz7MbN^|vkB)gIC)ytH>cn_ykiXEp@AF?#|Iq`4HrKH5BWUd5+peyd zGxhOaDsr=b*Rusf+`Uk*kMKNwKX!Fv_~n~_*Z0iD@_E{x2^Lp7#gnr-r@oy3&ko?K z9$nD6DGPi<8rFw%b!I&7*zVuVBM&%C?YMjC#?d^6|6k|v7`!=?QOS>#zc?`QU4JM) z`fcRjj9ZCi*(|hDriWd+#^LiqmS&rNj`sg>kN@z2z-(&KF(@T4IRDM+X*`)>0sLr< z^p7WhS3(%@qklgYG-0t9HA1~+P{$(vX0m9)E@_0yYnT7caA+75;%I1lVhM7`{ zpuqpdU`CXHF)EgJUgh~allt!@{MQoxJfi<=3IDZ(|60O70m6T`!~cKVVI;a4pW?iV z@5Wc3JiLpl-RJ3qq}s$iKsm;#kxeO0mo7N#gDc$huX1h(CydD^-Hj}n@y?yI(HUs_ z@GHKty29=41ywrHq5GITN5>$DPb~DV25;KLC6rak;cy1d$sRe1kWml5QMvj#1QOS< zGt*s>5}XH`TQeaZlBhWxLO}(!`y<8IqqeP)px|-cv>$D_B9I*}OHFNBo48=55rvG8 zri9jH@>88V_C}HU87<$0-5CI!qDp7rcnScB$_%I1K*36%)k-vH@~*6!Qmi@R4#Y~_ zd+~w` zMnw*n6-y`6cd=G56HgxM=lG_BI++(1G8@1ur(_omm(U zc~KP$o$34Snz@Kq5EVDz2{hL5DlSzE#PwCYt}ERVNyZDcgZL=EmB_Y%v!rP!Xv50> zBbB=D(uP&OGB=L&!{H>o-|x9r#w>jCl+wXg-=uu1OOk}ihu_#pay6Fe8g+}Vn}dVNlNTD!m7HfZEE zjwE*VzEw^0c8deu>FHjfdaFTRA%tpto_oSQZE49M7BE}bVAQZ@t8?>L3Pb*xYEkS% z6>&cN=aXgjX^oF)4p|)eDB+5566^98=S7U2O~{VjV@&Eip}P>5>}0WASX$u*m5E$AWV=~J8@z_c~?kjZ0$R~YEHU(A+a$j*?&R?)6vPE{1t;=d> ze==X|4kIkz3o69&-#z8Y`%plaVBoWSn>L9g(S8d=nMSECyEClP4|g?ME;5`8Yfe20 z&`5Ry;h+7m01;Nj#~A-V6yWfbqw>Jj7E%w*W?YPAHE%F!W{e1*2^Lv|2u zN-|!lH86`$?VSOA^1FF`RFddOPMN8l>NiJ~;3gbTk%iI*GsiMc1VYGf88J}~HI|{P z^cGOgfQ8Gare!%V)@FxDOndFU8@Nv1S{i%6otjG+CHZuHm-mM34?k^eZG6BU(Ah2& zv@V)TZygdqfptqUCU@2Bk8q{N=mnb&@Q2KZnE!G*XD>UPm;m!7oC~Nx;owu7K`(x* z?D`^0#;bP-C+j&)g(yqAUm3&B1H%O6OwL*eU%f!bJ06W#3H*bn=m z&SudaKXCq@_oDxorST^?SBB_HlU77&*pJ@As%sQhJ**5bH{^cZgr9rh7#Vf8By4cD zXsi`n)zI~7xi~FgwPthO{WNfvbZ#EF6E=r@x_D}l{+WJN0EA{jP zSK3Ic)qz%uWy*!3Ey}&ViELZJy0Ydz7lVfE2(kxP*A>dG2+%-~V^Hqs+s!f_9& zqr`(xGe3jKU^Db@q}*&7T^vs@cw?o|h=b*QUFcWyb+Kn3Nx5W-z91~1Hz91U4CjgA zn;)^Jzg08*?h<+3pU#R1PVuNLbg z20UYt8~NT4eyHjQ2PQz9z}X^oz>FwImqiZKcCPc~7UU{X)Hg-xvNY$MD;AJnRGMJ+ zvOgbEns(LGsPhfB`g+9Q5q^#w)_54ZkVLSvV}S*`i)8YxRz>Km4y=rd3xdj(4rbPi zb+k;3yKb$m?3WK#uUXJC zmwx>{ugbKE&$Vr8D-haFiK^WJKHT0g63~_Enw$=(IQ6mgpcM=hNoOaT;fVH#jw z9b!m`t7pP$W@)tu^=nE)If$~1c4>#FtjnvDaA~X5NprMBwFhOu@fQLfx+~Mt{Z75# zOzu;GdmcYul7xtTGKntZA(_A%^x81wC2IA(FMfKqDh!*BZ@cM$;$ZBOej-kE+6A-&t%EH z_jjCWjAm|vY=gbMEqny#B+HzK^aa}1$F7kXEx_h<#u$){VBzjnj@tUt3VjA{JjQqP zmqOf-{y@{l7d15kyPk3h$Sw?C)FyIa&Z)bvR%Z@bTY@9%yFXqJEGe|D8nOzkZW z_+BQOs#7+;0A^TlwBHc+pt2;E;3dPu(f!HQ2JZ{ypovf6N%@(3&2vGG^RtVV-4xtm z=3EH13_~=vPFe|`uQUI&Y30k6kF6f5+}YJD4ZCXaOx`bd z=IwBs?wxtD%&#_I)A`@*bwg^TV8s^vsU;j*J3432BAQeiej$+wZAm|Jsw-qDFOO!=kateG25355wqvZu~q{;X5as&S&y= z8JE{*R__atmW^n9dZoD~dPidE+|>t7n=!fZYGj_YN=lOz1LgOgXxzr_vv|du+7z)U zX-YRHHOex7%=E|ZIZFNg-K%tM(3$u&8}wnmnyp9sHx-2xOZEm(?;PvQu#im1wj9WwkrjhKEYCgF!JI7{H^N=)D2CEI&t~)HO-T56;@5l%W*)oBT zS!@-FBBJ&-^VLV|_wh1L1I(O)8yYi6D0V27U0SJWcccwGp6)Xof}pth)u~+Du0^A( z7Z+pw>SqoK0#m{0pvlokDV8R6V?)@@s|sD`+x_fO%>AhIL+>iSq^8BatUdL*YouRU za8z$LIauF6tNJ)r>0ru3&|f^r&-AJFl#b}YL-utnEpf_E*Qw0{*E{=$rstMM+ex&4 ze>Zf|pn_St|D!9e<*d#0TgSR_H}v^8P>=YBd$-4ryM@5K);N5(6J2#eB${TvLc>8h zb&s2#;GFiEB*sCa3b}l91jhLvKJ$wC6lRoph{3{+40xp=ec~m?KdVKEvs=xVPqIr^ zRJ2kOoFZoV^_fV??XWor2{9F@}2 z;LfuLliIrj=AoU=ON$+K<9I=?k}Ea4%pq$Xa|!VM7xTE;%Zkph#T3EGz^g+a-|h~bcN7p%m#CEiPH36SQJ4Ao0u=7GJcKie_le-VrkX1b zz&#Hg+Rl*ABy-$+ifYZwTSB|tt)a(>J%9ZpD$h$G^~M=#`GmID2GQ4l*m#iW=T@CE!4Ril2mi*{<Mz zhd~zd>!CZ{Gb+ney{TK<@`Rmeg$90LvgqSv<6BR5}l$ zKDVe$t}7MYEIQfi*SNxT$Il!yf`nk={M%?L0eg*V2LXY<8+WDzmgA38Z(c@PZm(C8 z9-oLEtXR@_AM^R{>1pC>hDwgFa#BA+zdiwT)8W$nZlASf&O|5c2J~$9F71!oE>hAMNo3 zMUbZZnLie%4nRbP~|X zZ;MM?Y|78KqlE{i233?U_JVjd9zN09jbVUSpl=h$lc$mIl@w|`JFa;bfFAK{hMd$| z&VlGNNg@|et6b@44IZ0fd@Ys@HM;j~Z@~B@yOVTqh~2B!DcQSCL(^||$LiA#8nt%h z>O9qW3!xIxHkuMbhlu%WZ5F}94=N76QZdfbwPTKIaT&1c*Tc@9ylM*Z_KmFuuJfo# z=7Q2jmo*<`IwYvS*r9W*;&3m%!3U`?+DaGplncPq%Ln-7vzOMYk<<(-p47Ddm@}~V zVj3TACl6b_24ICx(3CvbGuTvUL#JyPzDwR3AChHhEJ!Z5c69a5UbXn-D;2nyo3e*q zlBr*#qloQ?%{l6;WP=WNA`s0Q)!rmt#UJwgr&QWXFLGf$etXD6q+g5sb>f z)>kY|6}Z-i*Eyf>Xqy}p6iPXT)Kn#`5~1YsMpEhbm#kr18b5gLt?^}i>SRR^cDvR`RO+l{{aA=9BJc>2 z{UY}>991gXz;K1Uxt9m~zRy%u`GlN_vfYlq!#L{*>nae z`?-9oGz@vMpzF3Lfv%lb3;Db|Ep+g%XOh-6_3jD18@oE3-}J8wypFs0V1wlNNvk7^ zmHAez*qQqcysY{^`*RO&P-T!DQvN0>EMU4-Q1~SjRk~AOIurLH-nhozufmVjY6JT{- z@V`NgcNf+aLrAT2LI=~3$$g?*NZVX~$ld`~aOrc5bd~6Jb(fF;>a#=Vg*|#r>;2#J z4fR&0)KF&CoZ3ZQw`vGA!`H(78zXsh&XHKBVZxfY;+^g@G7pOk5*a$LJ%4A^Q*%5i zI`-LD{@7pEc!6}!2o3?|#wt)yQrkOuBjT0T5%KE&OqYuTo|UI_Hc~63jn$qavY|4) zVrLf5k8sa}deG+ujkaX%nuLAc_ ztcmpv3~aMNy~KH$y(W0CUZIdMNPL%CW-@f0y!%VxSgP%u(AHh+> zwX+B0Q@>S;4wpGgOaZLy=zOQ$r2}JDWL5kK#aVulA41u10nqZo5n67D0ySBR_Cj&F znH#|y3Glvj_wQ$_)q0rKcMj$dWz-*16n*7ItYppZ)ZAmt9$AXAXIH--8!}l>usm#w zDcci7H@2yw@UsOkY!ZBR7~CW)xAb2}oWdu*${#lp9#75?$Z;r`^xh6S$3p23Gmo<- zFDYT3<@3#hSEsyhTHP@3_f$f!Goh~e+_o6j9RTm$X4X)-BU4#H8V?N{B_xKN7 zIq+ltPGAn|P_v&lGVn~nN__?L0cD>qmf4nnFH>-RYz*^F+h8zZ^svnXLkq{%= zjx^kUG9+-g)t6Q;siDWG6r;Kp#_3oZjOVg9o{(Kl_i7wIxRDLSd9M$3CYv(GD~6d? zz16doIad6P6fff4^nD>1g1k`e^dwgq zyYjFu&yK^Yb{~YWamb(iR!B3TnlC`vUqpFuo@rnqK>+)}&F`Y-?D`>q*_JJ{|9#*Tr+Y&RXE6>-~8BxC>! zS!l=v5)7iLZ}#0uv2MW)yW{gDgVB|6|tLjs@$`V%_Mr zi~aXc8v_rtIrEgcl)IPSR>j4^8`3{+beC{fdIA^}Bn_a?2<_>`D_yf^=9t3n(F-sa z4ti_O-g5lbIMqBaN%#I9^mF&%!>VkbZq8VgxpcHy^A&_8;V!`ntA@8t^GFC?7iFPku84K%v=dW_cPNb!l}?YLkJ~cciZaaf-HUSKUyk=eHJ)$QD9RJOIDAjHYeUf8R(w`Dy6)Mg(8&+7*H1t7pSsgu>CK?2Avp8ZSn=~)We#Xi9P5A&QCYXa34P4+K3afE z$j{#rttsLnE+u(hm&`}l*P+FV32jxwwxxW)Exd+g@kCm?U%kI*?1q- z+PE&}cfeZ~6e=%$(nGEK-84A%=&6B{+tJBVjW#JqtvP7F&E&BoHI0!PCkz%&+w zJGE%(2i=n|X}H_jSQT-dA@XPs_6B_*nMa2nJlrZ3L#`jh-+|7M`HNIEAyU!&hP40#~ zetNeRyjld?uU~LbzjS67_ZMvZer7CgFK^^WHIhNj88qLE5yU`eosSL40qNKD ziNi_bmb>Mt4Z2i){5{3t(PF#apbEYX-6_d-Lg)?ah)T7ymG1JU>&LI)q_ zicHiTv5&T3dJXx8oj(N065X+`n;F>Tp|}CPAQuXEuf94pKqa|5L+D_wNg-q>s*F7A z2gT^70>0xxtuq6`o4;y!vD)?5EoCxeT+PSiY?-@xI}$}HdO@?vdzouDA)R<7QYO>u z)Ddo_Sj`)}d?a{Y$8Q^ppysaGkKck8F_um@jGD7plmQ$k@WGh8cVmK}pCd1vPxf5? zY1&)%4W!}uGzLjWY z6bv{522>`QO^Nk)q<*)R)BRUF=^~F_04>Z2U~6< zstY&bKTtAtsJFeSwq8=m2I98jUgsQEY6AXo8yaq?o)SW?gbxRk(r{~bLWes8sm>NN zkFf&JKJ3obyjXn*wlVHOC|{cXFsoP6sfZMJR3W9AU(Yj=yzYNGGd33*#kjz+5?pC< z&VA-<+{R<(!m0$vW#{OM2P|4#8%+bd!;ke>X1O4XNBrR5Db!zaK#Vocb*Gj)b<~aHuyDtuqb2^5~ ze|VjQiK_rm&-DYR37;R~r`wy)h3VWBEbBGx_GWRgM>ZRA1mhQ-mMmm$^QJl;>FD8 zz`I)l>2~~N0eHsuz}KP74r8IY!Aqadn%{0VJoB2%m3Nn`Uou}-mX}8=JI9~+KsufL zSkEyj?a*6_T&SKruQ)W3t?fD#tEc88nc>%8dJFBa)l^eTPuxJQj10uJJ(D>iJ1yZp zR+BNifyS2eHD6C&FLCZPr3T7LQvLzkG%=C&$qpI!SK@ur%uBELH!+1`HZu?w0v9VZ z;R@S9->ZXXHtdj-E!OeoafTe;IYf#4A8d3Hq5w^_X4%CmeRE1vXitMA8sceZetUJi zejsC|KM#qmnINw89(Y@ZI9i5ysHGHJQ&Y*{9_%%MLB1YzVe|VO5NtoEw7oRC6LOmx ztsjiCRN@@9HaooI)yk{CUi8mk#U-zgq5*97Ylkt*9je4C`yKQgmM(jwx7$uhAppho zjjuD^SL<`SXD3&BHLq)D568o6Icxe=BddWtpZ?C^F-}DjY z1I#TQ_H2M&pKT;@k6M*Mw84GduTQwMbJ8n()5iMR$7_~;w?oYPTSeifJ%E0?6sVVh zhW)-eRz{l!AU@Y`;4Sp;{X172)$wxB>cEVAXWZ&3Jpt!NMflUDjP^QZ>$F@@X1PB1 z$btTcxCa=5;M5!N=p?#YyWW3!1B;(2hzv>eY2Y+~)xOtBuKaCw(brT(Bp-L(c7n*o zb(ZB=8ep+cw`J^+!n9Px=61vC1pBblzS@UCp}e!8cV$qb^Et-*^HPfS(|*@D{hK4w zxb$kmxv5jkI5(fe@)Gi_sCs4XMw0&KX4>RtO1H}7`dOUSW?3onxa&rwY8?5p3C9b) zJReP`v9C_9n^E_M0Z}Nhj>@4YaGJtk((5ebZ{HPi@2Yql$N#P;i58uQHTk}Y|mcsypr?uK_jg`2mjgL|{Jw{tzU-%ZDe_x_-M6`^>*pCI>N@S!gC3DEA=JpZ&nRw3_M?@_Ru34*4WUxE`uTxHzH1gw|DXNle9qjMG>7|>^t z`b>1X>xgmtS>g63kQmfsoYKFfhkADaQCO*uFeEt)Vo;Omkq0FWp9wA)xP}p5 zWd6ynI~~<0gOpuVvKS1Z7mqwnjy0PE(Bm8Dn-=WPRJH5+A?=mK8anUmY9R9G(2m`t zoNVd>m506Ba;*C-#nNyI;iLxe!n0g-nn%3wuZGUrlH;_DSI}zPgoA{QnK8Tm@yB^To7GsDT5R>^=j~jYt;WYKsVLbxj>SNO3R1h35}5_Dd^(OI zr@t68oc0mS+FjAts!Zpnv^U$->^0)B#zAcEMGgEaS=ui@QPdaDs*-9(=8fhz*qvr5 ziF!pa0>4sWT5piz)sK6pCZtfwnOIRJpP(`7MTJ^U_P9(?Z@X6=0A~0^th2c&m`B1#K>5V6AN8xK@4Cn^SvFOnBO?Uo{FTd?Z zR@qdG32%)sw`=i$E_@7WYtMX1m9O?%-|uI*@w_v>lG|0~WkOBMLms6Q(F%o|0nGWu ziuEgxun%yjRkrkyH6995b!Rmb*pA&k_^pq6YRwP{C@gx$_ucTmWiBr`G()b3n)Ne@ z;dnoI`4%66wd7D;9Eh3nJ1`fv2n*{^!4>#4@R58PrR|@kcYP@1-niXS^8?(}v%BSb z17@BmqJcb)NL>K$ zmQzwHBbu98a%Mgu^h$g*FZWl-=&8Bpb>HsGsB(JVP+$0J{QWAc%wV-e2Rf)j%ITT) z)HM!9TbHBLBPQNMAdIoS^eA{_|GcPheVj@7b+a0=`Aa6X7+?QbG=mm|NU;%Z4UcS5 z%~y+yfyO~ufv;NCV^;q=dOEH&7T2*l<+8QcI~RU)7?Ub9mu=1bp-W;8T)46J+GzHu+6#=C!TI^JRICIUd1r(8yHz zor9CK+}US-Zy_>O+eOFvJ%*JtgSpC!h>z7Zs`2~L)#r9byZzjs;r40Pf7x2nWh9^; zIg||L6D$``ogf}5!McYXfaXz}yMMvo&0f6ZJ#;?3U#BXC_{7@nPd4j+L7ZTr%4)<8 zLIbyP=ku&nR7zYhdg6e!=iJg8XTxedrbufuoeMM37#?wkfk_8gu27W}!-(d$EI+$T z(m(!=tg+_m(N&UF#-5vcaMlN0U3CfxK z(Xz)#>D~I8?FkSfsaRsivqN+ZXK&_L^5t-zTMhRHCF&|C zl2+))$i-@X!MZ|@l#a#n@aCzbBR-hhEBpk{DOwQS^vJAP`$spoewX4c{orif(M&)o zRZ>(@(dsj3SX|k@c7mLHv4;PTP1vG$;0O1!P(EMwNy-tFPTG+eOqN#99nM@7RiHA} zFen%y;MJATW%FdXUq6fbGrhMkIfK%GT0T=kj(G!{QP-%fw5O{Sal{Hvfi&-+a_X#U zBtJU~9H$Jbw|af^`|x^#CnX(*4EYXwh{-Z}H}+$9EG`nd7hc!J#cr|MTCfN(XLsPU z2#uNI*`3?1%9!DIX*T+l%Jktau*UAg$g&14G5g}LwtQo9#Kd&iA7EB5nvstF2z^Zn zbS>U^a+P5mT@V*R5g0nos(x#62b1YLxxc5Fos1ps+!hGbMqnSU--Prl&Z!@t&GR>K zjQnxU!$Pq_jfaf5a;NSAy@avK%ES9Xm2^tCzgVGP?-0`{;b{;t=cIr$lx2h=Jr|H>per~@scBiFY)VV?*xzb4j5*ad5x|zXE%4!%PlFx#c zg#v&zA6XLW70)*p2Gjwu&2}A0pkb~q@zF{1XU5RXBO$VbM>;nfpzVFr$iGQtVnwwp zWc+AFWb@mjh4ozZbUnc%95y_yg=vwW&2cD<(@B)jdy^uC8xNF~FlnqElFohh;6#B- zitk0+I9;@#mfhAcLY#PuGGd}pnR-Lx;YSXgj(25YJcSiG5Q&`J`jrG%5!^O+Qc7fl zKqnmb+3O_42Dh<)jCv*I0F`_IIih0*?CpiPQ&ns-^#v#vknl~!yRCh_z-=InC;Rl8 zg6M=bhWCWArn9m&ws(WTN(NY`nO{IaVsF7!gkH_=XffY#G0&FVpvs>TfZU}`G8C{h zN8vps9a4Npu3r}4tmiZcBDlbNlKw3cK3d8m`mYI^&w8}>CFU&!QnL3r^eXcZLKLR# z_6g;8EL}Nws*()_7Fn(Yc3dB%vN|Rw(1UsXjr1Jb6h5cZ-4M~>>b^V1q;n`b@%(Oe z6tt(3x#^QaBf13TyLDP15~kXwfhskh;128QM{JIRzZGH!R)nV6TA>t1%A*-Y@a_D@ z984+9vc(X^fwKv0BnJc}mzJ=?^F`+fC2bE1Y}WyCKRvWGL~W;V*)#>P-ByvCm?$3Y z>jG-Yi>CH6bI{Ozq2pJn5*QxbzHJT3${Hz=(L5J@=(b*s)X_%m{fZl(^XL}bu9`KvZMC9(SF%U5c7$89l_IA5 z>u?271Wd|IPL!>1rRS{xEXV%ohGfIgN{8Oz&iaI7M?-mJo3Df9oOMl#$K=}Q zgA-Na4vxA+_VI>r88E%2fN52nEYqHtWR}I~%NPFOCHbto?vXo7yD=9LZf{Ul+cfr@ z+JI*?o|m@NOiW(6Xl-)^>19cEk*UkoGm3qLD3Nlu&#FKIZj$kTxyfFiXTML^#p$@% z!)nA2x=U;k_V^^}u zY`q~wQ!X(7W6QfL9qRrIH3Ef4DBJgi%2Y;IRB?9{klfkapAqDAL>$fh1qw`po`hnGuWV}#+EuW?0@0(~fhs(TG6>S@YbLypaW$NYA@Z{ zLyp$`B?1vGmQV^Gz=AH#UH4*q{vroL-AmcmxCpY+mEJ#sYsPBKMFIG*d6_{8PuhgG z<&I}B;Y_TxcPsK4l%y;*Gdy(H4=}Bo>ZNZUZt?7|nCHDHLgS9ggmRlB zv9Ugav zKsBjnq#21p^?3N?l!|&h>Wr`3`s&to${H|YDEB!52tf)>_9FA9JX`T&%es{6q80)P?=dFT1lREt@d zlBQy9^;9lB(EAs{6+8FU_kIjDlj(b-_f+ysN`rC_Vx%3|rFx2@vuCPAq@qat`pssj z{pTh@dGL|V<^(#0T$nnaMS8dK%F8g zjpWPJa5qf`h{i749EmLg{}5Z&WA3kycr81IBo6>G_7&@nH3YX9-o?1uMbkVF;3rp8 zzpju+#VRzOOdSFI`UsF3%dyB%3h$7Ids8;*{B+b6fy1$ZxHQKJt--X{SB&`iWD0$; z;wyty+PH=F6@esDS|Tq;0?~SxjZi0(-VF?eT-2M-&6lM z&g(bAO}feX2Sg6YD~&Z-c)J@oP$p_Kgy;$4qD!Bi9|U3VJjdFCnUbW zj&0@z_|M<;#$wP-c?FXwEcbk?=)c4JDIl#pdqBMO{=?jN8VPQuw32?IF5Pdm_88;= zze(}uZvoJyYpz)K9H*ndV36SklJyk!_FQZ|eT1h>dg_x^idJh9a~);KNsUjF81`GM z8@?qzvXCtO$iiRr++h?x{rpqqZr5R> zXd834p;(rjIL-V_o15t{Ign|0FW`pznqYyCL&EtG$ryyW{PmURqGU!hu;uyEs8X0c zAZWOfR$f?@Sgr1&_;_Mx#0L8h+h-SKVwnYo+HMDBv;zr^BH-Ik04p!F^>cZhmrJzp z4Ech!z%)KQ0CLO~x7zgna>dVMd)T~Ia|!Cy_XPw9&-FdECpM{@Rc98kKw8jDpcxEER39HS77!Qk z`s`IDz~vZ`NZToeS^8jwW$-(> z6`dvtYyCS;xMU8Y8?8q%E&EYS8>l;Z&S;RQ$o8`fz^XmAR@gzA_f%&a6Z?}9hMs$t zM+(2})3AS$LMM25!j5!;Lc||BK}%sIq%#1aHwNcri`8!`?p^Bk8mj=- z{*PdmRDL+CIzRT_3gE+R!?($u9&(?`!lnWd`+zQxzd|P6d#2M@Jd!J>b7DBn^V4iB}@FM+;nOp+BAs^$az*0uQen9qUOD^p8dH;u|w__#|2;J+c>(7 z@~1_p;3R`&-S{YUAAd=GD`NyT9(VNJ$e;7L_))(qcgDi)M2l8>oxVG&Ucl;4GCyV_ zI3GK!|KdIz8~roj3Ec1JFLk@cvH8eHqJ{rRr@J0;dbe5;Ug~VKG?NufJB2L7Yh=0s zKgBmXVp1A;e}k{M-hMF>7_J9W4+tO$x-9M;^X=QRQHO@5jC;u@F~@Z*9|Tv#raoLD z>$?vd*zeqX&}5Nz&xTqm1_Kfpm>$2>>;{7&G4^d3##nx@(Wm>qpZj^f z$MO58f1;!J^180;Jg@US-$c!Fi)PzXHjiICZCP~ph81)PlOY33-o<)UKg!}LNPvcx zzw_>;_i~}J9&~=hpS-GvIxtg-9|xO6a$~T1twnpN04gi9S1pf#_0 zN4LDx@=~%Zd(px~k_#vVrAZgJ`dpKNT;byRqs^dzByL$F&PL;?4{ybzkJf@;FH{i( z{#gs+!K1h>=hJIox*z%Qx9_&Qd7Q|`$JnQfI#Hl2AcvOl1R$p*|VT~A`Z}Ak863~hv+Dyxitco(O<3J`i*y=$rmJsVN!;*L= z?e=bOs*Lt$CdWiuPvHdSTE2g}JKcuJvT@Bc(sAhXe;FCC#&JV(UR$o?YR^?R^HKEx zz){;k+$-G4G)WGJb7HxUNtqn|ByQS(bBEM*N7RU;up#4X(0*a6EI<*yB&rjCBq4@eT@&WUrS+bwosbx{I?X-__r{io;n(w(TmW0-<&J$#3dv4&|N+ZW!!u%ph-(j_L$F2 z$l)k-P5ak333yb9J4c@5qdM=LDX$Rk{&A^x(hNNv_eH=j8tU2x`H-{F+wzbF6xT3* zi9ss^B@P#jt1a`c8ogq$#N|`>yzV@0C6%2DS4jx<>G$%^4}VMi)PO@F&@Puc0`Gjh zQwP@|pvFu@R>zeV8ViL^whnMSwUh)ja2I`7KAsMFM*bSbPjq>2-=S*x+=V4|I{QVD zFrLc1Y6s1{O1c_X3RtFaZ-|4=bh7trk))hlImWz}UcvclpAqQFL%EjNrFOiiENt>D z@pT6f<%8b&qB6E8`>ePE>VYws2@dFMG(sbMaI|>KNuh3dPUVn0Ne(dkdk&M3U5ez} z#*T3j-0PlZt&z=$IX)_Pk-{pcgEQIEv*f6u{bs8)KY>nw`g9W5$~3s%aN@cn z($?9V64MX}YSB1Q8X-@m43*ND&k&#{F{b_MBw1%=fWiNXV)NU~HX{vQtyPJgD7r^- z4)yI8HIP>G%c}WVf*e7B@*OsCB|1I7&Tvi~6r9Et6C@ou1@FgD{~>7U5N)hVORdD# zcXx4CCCpgX-68c183%R1+1y7vT$Nk3{PO?ojtL$P*1M`j_e;G;fqO`lTkAkk*p2AW zR|5$Cn=B8H2Hy45$)f*Qq-%9O@}bIbRkW<*%&0wegIs}~m5QE=U!=^apgSWZ(>+

*ZV zM`9nJZ#gN_@bqGl{zBxmRfxA>dbq6$Q!#l2LD2w;-#VfFCCkNX|JZOe4xuts-((B$ zuh#45804#x@b(Is<*bg+t7Qw134Ixx@M3!`S3)#wx81(Cx75Z%O8luIR>v|bf%W?% zHbGKPJg&H@ zw#%Za&hNFesonHKc9{}@?D=?}5@tRfuP)zK0)nqzEU6odhq=j_L2bQf*Ey0?_an;} z?vC8#hxe9lS{%9ua#BY>a&OsL8yTd!8yazDGU1%uKH!2Mv}D3sR&gIaD2%AW<^gS` zIu1-a9)&&)vQ9F&Bef*9^iL1v`yV~j=t}j=CClvQy9V76?n*@)d@`#>8&5PJotIO~ zEEmG3aphdN;RO66?*(7Ee7%G%EXMlMHdxQoRQV} zhb1l@8!t47MC@8s2mJ$QGycF?CT;*{^|gLv$^iE8c|Y9GCM=9gphpWmB8wO7=$5h! z;JL}KBFCoE|C?bqo2>Jb?bfyXH_fX;jk+^&w@@viJ%v%B_}s*ef1%o93AsX!>0cYg zOqc^2d?+$EVUtf9A|SPWpT6WoCJlU{n$A*z?wcfHK&g!BFpRY z91mQgMwPeQUd)S!-ckpa-t8B)WfX6?Zk-^0Rc_v{pdmwkzijcsBDOkRQEzK5JH7Z% zk7bJ}e1?j20L1@UfpHvHvT}=T#!o8B6Qrfwm#UZu|9U&>hsyN|Qe5WJ$ii`WsOs+J zDQe5QAF&SB()>fE_kX_}Ma?63gj{W17RRM-3DLjto?uQnd|w}&nM~bC0r`}a##?Yz z(IGBM+`P*BwP2L!G>v9{{z)t4nX^0E`xY$*o8vfyTKzu{E4PyI&4B$J+VS4!JQ&Jy zLD%!;oW94(97fH_=VA`DH_uj;DA(|0@yT+gJuCjH$&FtkTMSU9u#d|nwzp*dQRYpA zYPBZ*L9Lx8F|4W^FJ>+)1#oNREjOK&TR(&g6X)j? zB_%okJtL26e$VK~(P%EgDx}Jk2uHxx<)j6kw24Ityw}|h-NJt!B>}0tWEOSaI4RyG~_sSVDS(5OzqAWqKSC2xjil7ORVe|Kv!M+W@=J}5*?Ra*{ zKixX+Rgf3Iv)~Na7$s(w+}}nb^Zvx&J#n`E0hQ8@@qeB zm}=43EsW|oY>fWi&%d1q)BV4!2Aq^{73%j-yNYBfT%>AZHAtg4GqX`WV_{(cu{%lr zMtyqm!JvqZuR(q!+!g(nGc^kMgKQ#G%4uEuhgtV;{4oM-*N51sMU2b8Ozj#0wAu}; zVVh*ypg<2}6WSNGn z`^@;4A>m2A3y&8J@1eFND*22IQCyzsVDm*5(~Xpl^H3j;d!FHJ%9h=Ya!+th%ztNw z`OH@fPH^>0i%Y*t6&H^Ps4<7!-kpJCTSrv0sf5d%PLl8SkrlOVGOJ#Wr7^f0PR#qi zrDS)Ws!;x>22y@==7#7q)rFmw?=AtGajhr+A0@v^d0X%Ex{Ox^!d%~H{*X>C8+Amm zt)p>2Zll^UCEdbg7Y#0*1aIl3uRhgT*zR{J3-j*Sz-h8iDRpwv+ab#$u$+R05Y!yJyLo&8HRebuUPioA(F2f`>{}Bn5DDXd4 zMt0P#(-v@t8Tq*ZAQ4O#%$^-}IU5XyQQpO6yp+VC5I$3f(*aHOX1h8g^Xo4OIoQ*t zM)5pt64))%73BotsUJ2F_`%aV2AUA0F44FfzLrw}6P3vR-cF*Rx@lU*3MoO|f^M=x zm*z%ya`)(vc$#PZ`J1cLm-1=ChjwBi*vn>X;&YV%`>f_4}u097a4&q~pb7QO2i*$WG~#XCQV`EbqQRQCzQ`~QYPxz8|j;0+;H$+RFNS2)2^%70u4;z zcQYR^bg8&qtkl4tbo7@`do>jP<6nBIxJ0^pPzAI$P9bHyUegL-Vvl87=aKOFdH#Ue z;}XTfRz$OhR}{RhVN2?2ITgxEcYhP79Y{{|5>uilG zeR9hHrVN)HgL{BytX1J+ofx}kq>lBxWnyaL^rKY{3?^(S%8=87&MTJlV)ExC#hyk$dfMp)U48Gw&HVP$ny`46y-M6~*8Qb(US)aaGqpy$Gh#cH z1)tk2?BVQ%FmUEQkvW~fRI2Cab?ZJ`38@*|A|@YxDJClCeT`o9x@u&KVCs?X^*8C^ z#+b_)WgtU-2N;WyznmMrk!6HthHd$Uol5gxUs(XNO2!z$yc!(ibEoWOhE%fVp?9r( zlHk!1hbd&Ud}_zFqF~}i@2f3w??4yq~BmZkp)JjIva-q8&e)3mdeQ|qD zedKh?*lt&D>GSzSu8($hzx)1y#Giwvxx*5&T03M089$A*H_3*d!(IH3)o$Sm)9v)( zOxMb$5XZ3_*(Df>7!C_sf1go08q%u%r%$jtFg~8!6Fmf;QmcWzNB`A=SKJ(=`ni`l zQeJo_;=oRqAo8T^(B=0z{yzk=Q9{15m5DIp>6{)w9#go2RZtpCxhDNeu00!pU7;dz zalzbkm~aPU)XtXk zp@X0fR$2Z`Ggz4!7O}!^$?IMDYdj-aCp9>t zhgT>X(2s&9j0Y?Gu0gsjTWZ}N*gf%?P~zRA_SQfKdc1a<0XZYeXA?P4Z9d`1L6h`F zg_dO%E~3}$<2X#H9aVM2U*`C6rK6S&Sd8>(#a29D$(r!c$bK(qHiM!}TlAJ^>500y zzrVndnS(8-RjlmxuCGL6Gn&C{`PXFRUq4Cxxsg#;nQ)Klr&bB{$_j<2{i5gEMp|)VfcRyBA*h;ElY^m2$&Ke!i=_BMiq&F7(Mf`Q>68Vv;4j zecSJ4frbrgw?S*!P15&k?g<|kT%zqTL{45>Qw&v{SD90ZtSCG2D^05-5m+%_e!OUc zS^W@`XVcgcC^iX2sAU_Hq;6#!h5?B7Q2SV5d~Y0V9*3@Tq@;thdDT}M(~@H&I3+Tl zH5vFR+eUbKMB9bPF9G`|l6Tcu)!*wGN%$oo0qF615>NHrQBsVg-XpTDl%0kA+>f)f z{SnwZ(q1^1MB#ART z`SPaNcI^5%?Tfpk39Cq{@z%r^*Igsp;TBc1Z*E-Wf9)Wx)x zuDJssifZMat+9s66*A2bG9v|1g_+VzeeLDBKr$8nsKe6B%FjbHqZ*ZP&s-`k(AqVP zcnl_hmej{b^*=lQssYH|D&84+9T2c$a9^UqGWN+XQ}SZr*c6>ykKwY_BNKYl0&;Rk znz>mC&iL!>kVd|pr?D$B7Dk+rS27g$XZ6U+-AU?dbhSkxreA&-<=|=^v=KDHzL;@@ zkm)qd0YXl@$ru;LKY88g;PXfPJ-Nbp$lS?i|65?KlXL|ie9Jsu-sqiDm~K8fvtwDu zz*bI8T4K?VU*PUYZCZ>Ql{tE)M5fbVHPV8^{9$Mgl#Mgp8Oycmqu-8;aoWb~lCzdspk}_1B%$;*tBaf4s zZJ_dkn9UP0R9VVXFUM^5C-8DiL#9iiR^qRZXxS^(SQ!mw<`%-$kmd-v%5_?sD)r^K zPZ8Xrq%{$&)2%PJrXlp^7sP?PNM7$`PD!W~IO($ECj`6U30Xpwj$AHIHvjB?J1gI8 z&8K(Pa2cT`;{Nn#g7h;hLQ=y5chqe6CSy*|7Ju^9`FwALHK&h6~ zg3SLe#8`S2YryqssCQlFngJNMM_pzi+S=9|%+139%w5uU85&vyfBc|&C6OrXt z3}?&t=!X}3Br_A)-)s9=42(NyEu;`r4F!mCSCLiMh9Oq&@H>7wrdRTh z-%Lt13d#O1tJ11wnHh(V^xkxPU7Q=kQAz)vq+SO5zQU(}_I+cG2Yle|FOVFV1dzE2 zZj-?o6UuAqqXLDDwjVh$hBz$W1vl0p zHqh(eLOw2EUDF{KP19dWI($fury1VZ8}KwWEL*sH_CZUZ+5JEVu^aQ1TSg-P#2gfr z9VkkmaT9Sw%8U4>h_;T6Q=`+wrHuH2ig}Ma`RrssqH)e>o7(6tSKY^$( zul7XWW)e1p-3_J>NsSA2>~JzRP_y;SjRo|mShBBK8~*xyY&24)toKNpyzGT=C|UZq zJA=N;4O&mgdQ_8Mv7lRLLoa9%=ZGX&;Xb~T{vvZ(P8+UWP{>I6+b>}H<7{(R0fyO4 z4|O3kFUVmLQa)9-wB#3H&Kq-C-*71&UGn3)oz@BB?s|tY7gqm)wtR87l!Q@@@$DQS z;sJ1AVx9$pOrQCs3(Kv)|GWDz&;x%0-}K!$8tB{6P_sA>oO7><_eF`#D@Nw^HbHqr z*@h?NYTj5*U^LU$`YF^3xAs8d#!CqWJE9TLgqtS9zk^!0hmQ(cb+c~ z6RXNjFpIslioIe0{AJPnzZY#K(V=>!>IOraKodfuJOAY`qbIt88({8>kLgn7U$*hX z0WYH!4#X2NEk9wJwAQvBczbYvu)KCFceede?Z(}an##g?35(nzeaMT8T*O}%_9BAH zCZYG%A3YRK#J37=$xOW&jqb3Yt?B+xjq}1KpmAO%6NxZCv6^P$nt5EC?A^t=E08Fc zr+OocRL=Qw;Yh#q(T+kU)7y>D|0p~_iE>6JH?~P?%iT61IV9ZdMq0+k7og4W+D4O{ z(1g;9B!Tn#!ssjo!X&NevsP@rV>SF>>ji0T0bYNRy<{j}^n_!exiLdNuOzV*VBgqerxGU*P_bI{o$B< zkNL?gFWM$3T;hIQ(rPB#8y#AVCDN8zWm#|2XvUO7Mxt^2MOKsiquKha*)~f-PfWo? zbd`7Sa~j8+A${-N7vBI*cYN`ORqwWET&qs9u}U;Kwqs+(pZh6$j-9xR3o>TIn1>t8 z^=QgDwRzRFe~QquFNRq~#*aghJL*{|^U}#6{_JF)4t@GR2 z>Q%AUv<8*UYLc(kR7VUebr;~6?E(~y-O7a}8R&rTJ}N`7o^VHQH0*Fk3nI7UFE>^_ zMe93MO&mY`UT~Zr;icxdkV&8LWOW&S2)rfmgT+|lEu97=mGopomj>Je*U3IG`iB5G zgUm}zm%0fxl4Ke0N0yK?Ief@27O6^Los0ZX!kMb@F0}k7{uG{B9BJ~?s*=9$j~3zQ zi05Rpcv>d4+cmk8>QN~PTpRCJXu^)&I94Vx*t-oT>9bIC69kj-p3I!?agE2vtJv%L z%6`jzMD=V-e+tv z?$0_el9XkJ?5XB;XWS6_-DO76MAO6(`z~#(U9JR@bkGpKIzM{R|=WQ98D||FIFcpTo5^ z2pd-aUxhHuey}os?h`~BSfD-;$spYfB>erPgJt~Q#6=AxHM^ldq?OvZ!=C+HK}=8F z#vu+3fqAa3*U)9e%x3^AyImg7q$O>CRQS7P%H4l4@u0bJg<<=pvzH-+74fdSzU06t zZ;1mXvZd>t>?T=$BpI?+kd#`{C4z8|Tb%mIl;0;kubRMI;-17M7YCcscHQzjgz&7@ z2}T;`lT?@R>Ha!bN;Or^>Jr?wDhRjtW$e<7;1c@Vye~(Gz))3~FVC~8#{OPjYcRBG z;7ohYG0om!RPjZJKG+wPxDBh^vag2rc(g0?pH+k752SOf>*rA2k-wTbKEHWr%z<~p zfTZ*WwtHk4TRx_aQ$Tg1Iy&Epv?LleJ?C76GIKuw%Zt^9E!)2(byK?~N&1W79cZ-B z6D8GTL4k*)%t6Z*j&G=sM>UnCP6vwL{|X|At%n)oJiGX1*X`k3MIAym{)6q0Me7O8 zkOOyIp6fmjTwP8O1Pa0D(c@ttawTC_vYSBL-PfP!(e79^L+F-EA#B9d4KHDQzqk(- zCt-|R%LVE(2~FE#xS+)|#`cnlN5X+s?Gd2d=_?bdtI!nzv}fY{+WYIvQoD6PlX0@6 zPdI2pfa_*^)JKgI*OY`spWxI-@s!V_XaMAT)eUE;F%(Sz1PK&-Tq31Z!njlc9P#+t zy-(A_L%>be@h>-7zgiVirVd01<#inVRSBHh;D}Gcc@Y;c*VQ|0p<6)u6$>`dHio4@ znCNCrlI%2#-F~GCn2}HUXF$zk&*$*BJJ5A1pko>#Lp-^u_um!H?a=b=2+c%+k5;Gz5B`sZx;l<=O{xQx+usRGKm5qt-K_ zGn3M|&PQUT1W9)(dfy$>IF5-msh8|KA@BrPzL;CqYSr#w{)qI7y_Qv|j5xXT_(^P{ zt8ogV$i!ow?|>4^W?ucJ%2OWgVECi3E#gD`lm}~%v&s*!hMoBkpSzFf10De5ruNjJ z;$eU}ALu({s#4D)fza!?R9=4+X5KZL&smL8!DO!%ur;$kE3fg2+6t(^CY*&niP}r3Y)WG@$_CkQSHxV&1$)k|Nlo> zobU2i4-lA_zyZ`^By|dGsP5!vnO}@`B~4kjemF}l&*wO#YjYJx`Pu7iAQG}1lJ7AH zi&J=~Ih;pLsFHsgdkm{?ykXEaV(!}pO#JX)nD}1iW2?1qV`4Ef z`F&s!V_KXuGu~OjgFP-STZ_sFPMp|WweljFk0Vz zYRnwd$?iIgLXdEl2PqwaxcCyJU}rs*M1>U_EByMJlKnBdmgMpHw|Wvb zRFjd2516f4^+pBgp43ZnpIwaeCgR0-cIt`oCd5*c*kRvQ&%=j~{qYky?k!RieS0{$ zy8Q8N9XWT*taezQ@6DR=hUPp*Qs)Z~+r!-T{if2<#*gSr@K~Vm_MB)@Vl^5D;Z6DF z2fnLiN~;bro}#W*Xc_Z^0B^J!M~MQmfb;aY+gtyL`BqJ;OYN2j7D@I(*?*LqZmW#4 zWoIlX!Y_Ohm)C(GQ)mna8*3cUZQAs@R?f>i{MvV1slpcsyU#=z#ICf!VPRps=GnGR z6F2#l>x(0zxgDc}Rnaa&S+G)b<;VIcW2psK^37guyriM5mrveq)PyL{*>R1-?lQkQ}2Nb(tlWy^S)n~Z`F107kQLOk4t`r|>DgG0(I zx|a0?Ox8bOER7F$QKj=>h_LJm+iR7P_HN^g5e3g=XV&Ck1|gH1v1d#ig_M|$411FY zQA*IGS2s43I=#+-D5Ip_1h-$hv+T|d$5};vH~7uPL8gZ?Ho9h-YxTEpm-mn-6stG! zp4@@*hrNp9^z5C6IDG6qOw|8Wb$g|I=%7iV39yPP-tsH|mq8NiMvm;8;jbDM+5h=6 zIO?28*64q$Ui!&?v4IFJ@l?myN@o*>-ch^&Abq1QLN zt+?rGK14oHQ?E^5{6pl~v^2;P-UgYhf;a|zA*ld|R%)TH2LE?)o@3L-BuLyin;KD@OZdfF*8|}5VQ=+TE@-%G zD$C=)Sjw$@rdYW}^XI$Jt3G#~IWKd?{Can`TT#hzJ^rLR*c(`cMGY^Ly zh?~{dA0w5z6as@Iq|gOm97)w@s-Zf6XD?i@R&o+7`{IRbl-;?y7TZ+eVp)bW$`$UK zQurC#qT4Xc)-VFm$UW6^Z~R)T)6oKAC=hW>%_px@xwp zF7Wt|SK%{F&X$RZtsVP?LdKzgT%(SLzlMh6wXa2cIG#C*vyl|1BG#Zp!@|5TmP^$2 zF{T)*O7*`|HMfug@sK4P!)+O7toG|O7H>xey%mw!59ql#n&zxZ zE^UP?4O3WRdUaD6kPidr7Kn7n?`JJKEi+ApIf))TH=g8$s@Azx@Nn1w;5Rg0`=tSR zc1Rv?_N|o^?ybpLQy$`bgUz729R^RiX^P7^{rw(t+m73oIPQ&)%Hnc3k&LqWu^j@U z&ICv3_C-d7xp(G^0jzxg@T*JGdauDYtzGTBhVT5UN>rMM)-0*=c(UySHtc6o&t5drqVKv+n`~ zzFPlNx+D3T=-A$QFEB-4x=$e!&P0ajxkqUn$)mq=5FpxdRYH{Qi+1QjeBWNP?gt(D zHrWh>e2ELle7G#7R#mI#`HUYg$u0|}EL+f*(rL-$$YnnE!7n4`eOsTnIoNf=z^+sN z8&UcDL#ffyq3E7cl+ZLKU{_Ajc?R&125JbO=G>r@m^vyCJx|b|{+g|TXmV9WfyD56 zq|D48HQ^o`iCfj_aPp*CU%Jlkt3=Tbi$0lj-Ox%hYCiGRGa+F7^0NfMK3CJ!aKs+ZHLwe{d1clw=oo{M(AyvvH6?^=g(z%grg?b$7no(r1!t zWgy+r)L80C=+*+|E~P}>dI(&U)B$$1TEnDPc8=mrx4~O4C3TdnL6x4gzl53SofP(- zKh<^0c+FZECxdZxz~ST$)uo~_orwH$3LDu_hGLI;nY}5 zog>%|f0xIznHOofu>m++O1tNw4{MzgS5J?9g&~Nhnz_5e&@vz?ftvgRR8I}mT-j5{ za*3lvk5#h5C_W!&UKDp26zY88z5OST^36N7?-!Q)wqB&-1;4&YjwI0>c1SQgJ1u{6 zEa~ebC56qnnBMf#oNv~zt zVF3Z;;;gppW+Ez09N$R|piXy+O!5g6a!L6w(@j46DBng*RfSO^z4si&LCgD+x>YIJ zav01T*Ka=uz4^=r6&|?UN&hIV)>E0V@pMn`;&{rhNkuNcqZO~WBgcQPcpv;-@%BS1 z!56p9ww6q4F5s;aolg87Fb+*z8_%t%$TuO$>O$@zm*=gxL;2rIRf_MEgc-LQJc%x% zT4j|(8a9+_^xUI>3pU0qJ~K&V-NnJAJYI18HT1r4NRk)_C0k8yW$km!O8lTfz9n;E zTFzeF7pEHfR&uiy6tYZ4>t1@z&K|!+O_W%5s*Lvw-kD;v?}dxr8=dOe-3j>jPs^oe5Q=I=&Vn6stiEV2*q-`wmw1SoRiE=TQ zGL`u_XXC4|t>0R9X*p;7)*P%9+>@8}O$ceV?F1$pF}uR})ybc=1hQXqA?rSRM$xgo z{7reGIw;a){VpUbHw`9WrsXvIo%${J`9cjIe!j_7l0`|Y8>-FX#6SVqqVi}<9J&hZJm>pFr&wjceNy;;cq zemoZ8o0I`3F(4q`ZMEF2G})WrMQ>fNwH$ZD4*HbSTR`lNSy**h3@Od zvk1{0#33OrQcC8K*;(Qa;$Rc|8xjSZuuC+nKbL+}=kytf?NjgXWOpQeCC;&5FW0GS zVB1X-G#C{&t$uScfL8EjS6qgcOkgh?C7yAeu44+x6ugWE&6!nQ+hShZYXUVwa zDqD_!HNvCGV?lDANn#1Oka;n;oz=2=VbRZjF-t znkIV9F+z80%Vnfc2=Z{BGn|^@l`c>=B~VyR;kxC7Wp;b(V#I!!jQH$LBd)`P#WMi> zuOVO;8S1-H>mn|4NSW;fr@i%-SveK|O|T5OFb~4r-v%ju=VJIgJ1M=G15bz4u8Vm* zlHfwD2V*q{C*KHr+3v1~3G2j4jZGEoDW^~V-m9wf^oC*+iZh~>XN3=barJ`;MJgs& zcNZ38R`SQE`wQ%y2Lq>(RH~d}Dp%eOY|KW#>~Nw8WS;60ge{eGT-oe6k@Q1cz-J=M z`E5NNU7{@J`@?{eV|n#vj>qQdL;`sgy+6rTn(DC54VjXc-9x|3yL@XJa!(+&MApI6i_bTGoqey5ES5jC8|_)mqERC^ zzn`&L)swGxIR?vJ=^qi7XJekC^UNiD;>w|E=sJwP1A6RMg1v=6;pw0vIzs~)J(!$< z<=ie=kOcXw!MmjU3&WIG&uGjG!X?vmvOy%I$tWzpMt5Z{g(ns%OGKNRY07LFS=R02 zdDO!KW-jzN*~!k`rtgQFcr16Vn#+Q~8a40_bh{7zTmv)I{oS5-2fBw}dO+0kbYCvq zJN`OVBS_po{b`YwDZPs0&G9|L z%`m=6&Wx?e$ro~0O~pMTd6qoRlwVp5gr?t4&!I4^U+6tK)*B1!n0ahcchi1um(Z_t zjymTS`P3xm<3(fP=tB^IhMfn0bjHQW%&TgtgGQ|2cmO^*5O5$Z!{6EzuFM@3D!Xgn zmHOS7in`vu!hYH|QR>EeZkM`n&BMdFp0~6dLk$svT1o~|DsNs^+Bw4@S5ByjJBd;i z;b&Rw+X85EoK;w!OSeA{h^CEpm-tyQBE3(qS5U2I6Qg%)GoD7SrLdXdp{1}_!v%+; zR0Ga$R^pn!U9M_vIwfa2G%DdO!kJaZ6%%(Ziun0COb3}we|AX zNsYqB*dZjM6nf#l4knEqoei{Sla zFHiE6k&uw&ulH%uZWbC=yLXL6CWE~F{?HIv9HnIB6z8;aGF|+gCGy2%L%r}kWp9Nc zE$t6F=L`C%1N*kbN(m=#(vFYeIE@@k{+Xs(xjzO4*LM0Gu0U4{!|oSMCVkxQjon zp+l{Ye&3rE+g0^H9DZWp^E&8O(3-xwfgH_#W%KdlRX=p(jHSBG0&LcG-NXV>?FOfN zCKJOT8ZC20pdRX~j3br7fz_#cyLo>3ig?TyWx^EUv}0E2{QGA~uf7)`Vm)MpT9&Ri zN!~tF)vQ=|cz+AWA991?u`}h52FXd5=57S|Mo6dAdJ{&=bG0Ua?@+pyl<8 zCu7_Nt#RnrWwhepcF?Gp#PjTY0hGG_3M)lZ2$Nx^QgSxA_dSU7hvZi;QllZ|q6Uw( z4CuCEtKZ^kE1U)I$Uzv1HG~uRE$&I~rBzwX=OC)NG#uM&(N(}JU)F~>vAya<#y-K#U7CBYa9?D*f*~8Ze{W}lZtv?g~A#4k3VR< zTk~bEFCJ^1H1mI}Az3l>jiq5QXN;}V$F1ORCCeVKU3Yp_|KY;qTZR7jQ@rA|Y{s5Y z4Q6jg0o5l+R_1H7TZN|sKD_cTS%H*aXWP9>=Qyo=7ADnLct=_5X4zn>9-93wbVQ{h z{QGOZncpVi5f%b;BHS~IZcbDgtqcm3M7+e<0lq?6RT3x#_9r(^Ea`;1{~EM)Y)=qmt*Q#%(ja1F(|==k3)uIHMc9 z0vww3)cS|WDtylNJV*plP~(k*^w4HT>KrI27=~Lcq1aQDdbF>st9qOYOY>yXIHYM6 zlpW8*;I@o0cW(3!TJ`QeW{vi=r5?!|SxP;?Oe&HMy)@5f-;Ux$K{%dAbMIbd&_&b- zs)nj!dS~Ct+-WBhV#ulR*Le`^_(aq71#?UHt-6MhgTNcM*4yDLx)R*G$!S(Hm)9%1 zn6R`v!lo}kBl>Z9q<4P*HTAXpnFmCFrvO!Z6&J~sCmin3(bYS32Ro9XrEenKxTc@^ zPMqed8{G*$x?Z+Iv(Ai^Gbjlsq5JX1ywM+EQE%UKel+r=%BZ-c^w#mDno3i!NS~u7 zc!;|<>`zkavA?pj%X7jtHy$U?_V*iQv-jQs?`IViu~4h_Puq^B)D;G;j4D z@^(4s!I9CYGWl#qiUPmF5)|YpUTW?Q`IUZn;oS4$-IukH7Z1>t^R0?SKG{7!cNQFO zw4&+QQyi0?T11r?!+C=h!!_mpLXAC&-SP) zi`VsoiZLoi;}~@u{`$-vxnPb4*sbRqq}j%xK`?9@={U9ZW7>I&-=l2_Oh0~4*F0_^ z;xiRo?HsI*AqIc+iZ^{;euow5%)I>WAdv2ZZ<^M9t4n#>F&AC$TO~XDjais4V&5x* z20?n>oOY*7yw>Ua<}-XRf<{T$J#@z=@?xf|zn3@1i`(1-!>j827k|1Hqsx>Oub%Px zDN)dJpdvK69Vb*Sc11o{xlfll(1U1Rv#_8RCbwLI z+B59UX3s*lyS1^^1MB+i%|xBKcQj>(bp7m-GEP+d=odCp1}E$dD0Vpg&wOuwRggo$ zP##O!bRF?3=IPtuK>add*SC2*Nn)Y$PfHo8sj$b;uIMtGd6`v0%VsmA*5%ox z9(&Van&yc8;Yc>aw4!ayh9bRTI;Q}B zXO1p8?|5dpQYOzfB}$i!RzXhr>JM$oUEGPinBMQ}w3=LrQMHe)Uw8c;zmo4*$KT-; zB;nEU!xH=H&AQVIZy?sg9|4<#g|TS`flioDeU?Hori|7!K`Rp1WS89L?{tp+wCe&y zA~|rL44l>yRnAa9C^wFr$?$>#LytYRszU{V=|PM?Q;Ed6HnC+s{Z}ZemM}?+V870l zi719$vI4?7&LvNf6HCir#~qOocTw@7vH!=91$Gznd$l&`G$q#HJv%gcEd1Rwviq)f z(GJNxXrOazQn0;Z}mVmx%}>2ix2JCYy)_k1fAlbm{x_2e2G~>p~)QxMT^_GD;1U# zGii;_=%J_3KD193mnGtaY^1Hd<9=GdRh{0tS4OSzHLa6BY60{{9SKqa_<B|@LJey(}#U|S+G*bY;Cq;eCnKapd%26JA(>q;6<&I@=|!K{wQCTGsMcl^vZ5~7T0+NAmqGgBDBouP zGgI3=tI4%g^Wb&6!gag39Stj-e&3M4Khs`ec#!F%LUHwkbS>mFwqd1$!VkHk&Rply zw)FJ3v}V>uW6)z}Zn1-31@c^6a1RT#cBvXQZYif9-cpsKjW= z`t9k$t0!n^nYtA%;I$KyF5_BFXZ57qllCC>LizP$;X&R~A z`^9aqhaILk&BkKxhk#EvBDBD!nS(0T=-V0UB}2*qAMN6%PKHtm3zzdFv<&SPc{fDu zLXLBu=72oCRPr_M3PI9#L()L#>8ACc`CwmGw5VD(TWL}Lw$`?qH21FLMX|RFU)tzy z+eK90?s0l?OQ_j(I^OoDdh{FpT5Zi+B}oakpfUMrtXdvk6SDj0`)(;8TiKwK$3V#PZh$+%1yji{)ml-1&j)dtf+t zjoROx@8L?AqbNwZdFR4)`&3h8Z}FSBfB{aP{S*8%PwsXvU8_Fc{)E|h_0Hlki=v3F z;+DYk43FJO6aSaXvb%H2vvo;6VV-*)Q2g-ppL=ndjO^SsQENjE=U-1HFGnkgw9wMD zH)LIb+gg_0gM73Yc^a$Cl!3AR(jz>bSJ3qVQEruZp`ekxq8lQ?d_mApz%I5~#I;^^ z%ttK*#Jj~W2`q_R8U1f!;?~_$7>3uyOw+OX$3B>r^%)XQiyO$We7!S--BKUpR4ZPC zqBwTDf_VzE*Y6v*`{&mH`L|=ZPSUkEi7n?=n_iwHyxe#IfMD2ka0sNh(^bsDV~$R< ztZme#OtkFD(!}+h!Cy)xXFlW?xUkB;0b^_fz70Rd2=1K!+qN zUA@^N&Fc;Hw%Q?dvO6E0+OKBNsZ`zPV;F9u{+6hH`d#$xi&-;n{Sv)Sao1+hQf_qA z&%(8XFGMRRah`d0)ev@P+a&Uhp>cqMR&*nqGRVYxdGkphTeh)WV@mTE36J22J2dwF zA4MWKC7Rg;scZwC>^Z|>%iq-=dn3CpqRdL(kt){MX zxY{K-BaHf~eU?W0$0)aZzcqqU(7LA+PgZfNAqjYMV}JrSrSj%H1@4yX4%hk^hgfw~mUk?b}8v2|*C)E+wQvIu#|9FbQc8knV;7q>)fM21cY+ z7zL5;7(zl2hHeKXK!Xabgio}BAh;{_2o6N z`P(s1;M=jIae(fXOqkd^8qbm$l=vVuR`C-=cBR$Ltn9cfHAJ*?RXm4R<#q5Ovpo$! zD7oXi@+#NxK0F)zr0mZvy6YUq3iX%woAAF7q7|!cSTe25U3tW{opNds`UKMH+pe%p zYQ+OXW=;kz6p9we!F^A0pHmB`NyRnj-dmh4+4fY5LH1?lD=i`h^LMHl>7k1XzI6>c z;!*oNW4XqLgP67w8-$#1goN_G%J)c^{SQ{kY7!3F<6i1HU)|DS^9cpqtT>nL*w9W) z+FUF+{Jt<^wP5_pv2CJ8l`QytQ!TRSuoqQRpQM%B1L%w z`{RILk>%8*o_J}(zKz7{@x53EO!RUKAHJFVcknB$40uHD;Btz`G(7rvPhzSQha^Da zXk%##9&f#Ac#~F+bpQB!sigaOC}b=}4>U~ELE?+R)EDdC+4oReF~fpiQJk-lr1h<~ zlvErx%>Hp}OLF&fOvSi{G#Gbz`1>SFq@tk?kotbl+8$mXlcc880OH(RGK5d0BIwC( ztxGulf^TGEJp7hdC9Bz`j_lbB`)A$%KpEn=6U0Fem>Q$DQA> zzh8UQJSc)?Cw?c2TJ~}OyXn%r{^wki6L%$fsP!Kv56=J!ATrft>T&Igx7}Y3z~WGg zl3Xdq@o8ILr{KHm#CU9X<`rim5&#Z(p#D~5X8oOlZ}3V!C2=kv%6GAh8j|o!LXPif zfRSwL*-mg>L&`~DA7hhiAaa#yQ6;TR^HZcz7IZ^pTr( zokmzw4Fz{+66Yj%u)Xo-FX#We6(vmGfeP=+-!gjtJ^_AgrdUAOWHvl|9!_4IA5adi zXUD0lhcbGN2Csx)#R_O(erHRS%J*YXWN&OL05kArhroBSvQ!=?zy-2~ZE<$9>5Aw^ zpYqu3?6wrY_)unI>$d;t1@msFcuNa4mP*N{7(V}9jBKoVS#;pZnbqx!d zuc!*P&*G58qify!8bNp6sx%hG25U>%RcBn&IJ)V0&02erP^Hk{=AAUu3jU=`K z(ie@rQfunAXi<`h+2EKf2WhZY-<~s$M84gvV(H=#+4cgvfmwEvpdmYs%)`khXk)g? zuO@$f!M38BK(ZXJPKlP4zKV$V8O|NOd;Ou||KR)wd#Xj{}#SPvt_0Gn!4iA z$IyY#d!djwCUAWQ5g~7=n0$lhs!y5tA?HN_PFMlrOSO5yk!RF)rlH|!n42%k(Ll<+ z4C$d9dGZ6j8^7*$FPYY2%IYK|8EFR8Jh;EIaA) zEcsHIa7vPzx@jl(KnZw*)bue%v93QjaU5MCIa_Gapetx3L*Q&880M+<(ul(prFSUg zMcmm>hOG9Ki=xl^jPHE+*tt^AQ{6fa_N|yHn#m~~21ybo2y()BazW%M!peQhgV&w# zZ1c@%YdFn9l{(g!5A^^A1&yJwcKILuQcI=;8WEmtj_?zxn(RL%V(he^-X zwsO02<+;a7$TdrN7ah2CP%KY(;Hw4(EKFSA~fYdww0*-hVhdNw$yDwaN%0$A-`M% zKV+{fKbulQnEv^Vy#?0W@mXHKOp_?*eiz(nV746Bdht_?@Jxt&ziN>$ zh;ykpx--7r+45fOI$DV+i;OnwhPTp@TV6Fi9HrZ|JvCsyes}Ij=9Li=8$Oa7PCTEj zl6wfNT`YzUqGmRI?c&dS)a>K;C;Rja#Rji z!#_K5)@)CRtF z#t%0?Ne4P*MXR(t7J4Wt@yVQ@lbx6b1*C{W7Z$#x4$Fdiv>0*nL#?2$%eGL9yzfMt z>FU`Kt|Ko0@iwiKkSKs9;ok!1N#DGfqd97cXqS2F<9)V-m%RfM}#7 zBotyZ?9~G}nRbZ)XUr2a#QA~G$;|G1LVlh6t@YB$o@_iO7qWEc@2oCPLw82(Pr3m) zVYM9URjvLzLg8~+CD*7ao&_I@s$@A-3h~c)dVErQI?s^{LSF|KQPGmii(n=9i$||# zza!-Zaa=5=OsMY=UHL{o=24%Jga#<_A+3E)8IKN=zT;BFe&t;$BRUCsX zxj6>t*dGOg6K(@D#>&woP{P$Kr4q#^V9RR4L2@0Cbd)qQ~ydX8!V-<=w=R+NAE9 z(@&*LHX+b7+wpit$F_;eG%^`dr3;A%IrGnV12Z7y=Rm_4j7bK^nxm(X`SM;leM!l4 z`?KoYf9bh-a%bU7ordQht%6`VbtsvO#7*HHu;$CxlltFPXky9B&-w5fOU>z7BiLnF zN~>vdbOZp1!3>zk`75%PbK3wwTlmrfe|73`Y3G`{8JrSBb569A5D%WD=lqIN!je!l zI;6PSTo32iaWvBBJ;~XkwsYux!_(zgrToOCZ>A_|FCt7cd~giMjU`%FYNf<2PD8TY zvw%2JaVbtD2{NXt^)E~ZZF#Zlz6zgp%{d-uTLBXGFuR>ap39(_RyOjYj z)vWbphvVuVc7PdRI~4krr({w?u;4Q33eZ}bglry^VpW8vFAsL6w7~L%gCc^eH1c$# zcXu|@;Dte4bjPGIv^@flrA4l>Cvmi6bi)#&m2pJ3j5ueRqFy^yi$^D`hqB)%MBbGF zbVm!C?m#t>@IIft3kk3{;_?h-XeRUv>~B>qWD>EKb2R7|43e^&D0aNPnh7JObDkBW z7^CR^hwms9aFt@R1;gGFHO8EN5%;-|yIWlDV(UY9XxbB_x2MyY9OLG4!|hx= z$R3n5_ZN56?^_9$z7lOeban8`{kzh6y1q%uh4y&H3IswbJOIbTeMu7jE=Mr9t4f~j zYUC@{ZF9S;aX>fwFfFM*;b39kWgf-7gYjCR+|I1ACnU)H?%+fuD_;tbN^bXFN+oyH zIkOzw-nt}_DYtN1fdMT>1)*s>=Bpq6tC0xb>!6u_DJ7mTan&HPpFyW(tK_G>L!%?@ zv6_4K0Ag;ROk<`rrZjkCiUa5)&Hx3+-ag%iG;G8K zqW*FfoKcGmeY;o1g5DgFfCbtYH%L7jNa|5R@hn*^S(okwRM|h&s9VDv&fO5pdn`A& zw1N?&?cBzKjrYv4P$R3IcFQ&5s(d#+^hO!?`*~i)PID?+vTtA=@VMqJ+6{kF4J~UNk_I9EBte4{-%v>dcp?Nn+kjw@ zmntL_dfH#+_fFQU!Nl)zzP;4sPOe-WqeqF%1D^(046EME#x)4?L$hlazfkPQlz7Cv z^4Q9^+tm|At`Xr%|Kd>3Rt(iNAltLWiakCH&VAqt`V0D)xAM%_{aHQ~r4c>o^kgfz zgB`(g_ilEg{b;X;k&GR+N3cW@FnMrVqxWoUOc@8;?pqbxP#>lm!KVsx!5?SH!~?E4 z!R+;7MLGSsWm1}+P)luJ>js3%@{pkV$JcUpB^tqy)&3#NY+rBOQ>z?Ngk+Q2rd3pU zmLq1nJI+aR8)VwFZq|MepG60THTUs+i2fZ}e1;V=iGF6?n<%wxe!Ge}3GAz^(Wyg0U^& zZpo+a#Qby+Gfd3gK?r72VO}V|{HyrPR$`3zU+@UnKh%J--rq-6cNqL-0G!jQE}>Fm zf{0nx>-m(JrFaq3y`n@$@)$G{TZSRwA;HNvHh;>qcCxISV=fQaWZaC+VxQbgdQ|EU zne1KOJJBwkul;0FtFxGtnZJ)kk0CSeNm7*GewA3wY3iLS>&C)H5CiljEVg&CxW3tQ`qWt-6=tGVhU{hqx9K4UrJ6^ES(Pq!6=v)G&G`UV(S&G0lpc zu(TfWar8Dr@N{jMck~OZTbc4u@>d?O6YZXCU|58Ek0%cmO5SJ>4H)Z34uM}qL&LKJmJMLH z+u&-(LAAT&cAHWTaAKK_m{Fbo6t!r5!2Fm7%SvidxF3VF73aoVOnC|68!jL}lXaWj zNssoku|GA5xX$WGBze`NykB+{$(UJUzp{H{v|V_;=<8Ay9v7|QyO!Gwu?D-}e|%&uy-%e(bwo@9SCA9EH;)1%4PL!} zy!-t^(mL;ZU%0z8=eWl~m&|&UE4~+Bw9t>nMAN4_j`4yGX5On)_Y1!a^1b@`<0Odp zUKjv~z?&gYFOGIs!F}c*EtP=y-7&~e$vr8-CXwb`E1Nu@SsW7(^t-dAaXzO)U^at~ zGbn}Ca^3$u1~!()TuHUY;SH3W_ZTZMs5ug-=>vDb2s@UNe>4pW|K6>)P9f21%CUbe z8k~4d72@(b;H3G@M)idxnvkchMx$KsK=-1TM7$>kR_1IwEpG4czmt60PtIcA|83zE z)@Hp~(cotcndNA_pZpybiUfj1@MqMz>Ef%e)el?l$Xj^|-!4`5sl>eZ*{70= z95A;lsRm_GbkGr{W(RY__XaF)9Eb#c_tWh%Cw&eF+O0Ob6m_eZN!sgXlFQ2dzYEtk z46kZrGSTt4VXJL?qqk2IFi2DDU*{lewRU#hwN2{Ph_7IYmKwn)phDabImE#o(xYV(agb>tf z0q*f&Mw*d_0%izGB4_*Ys>YnLw`w!-%*s5+;vC0amO4cS$+nf5(uWSKpj=Pp=~Fjm zySZzF;%F|n4EGB+jQ+1%L)Q~a!?Pa!cc1oEEI~Wr&~2Pi{uOgwu^iOc(o{E{KR7!^ zD&M$KU$vr)G(o#K-ak#Ie$jp9y1RGfklpQ|UA=(B2>k;b_U-`rt3bz4{whNc1)}Dl z3vi^G$ZOC5ltReLOeTqPkvhSF4|a zY?3nn;!e`9ScB55K@gg}a%u$(XwED`n}c8tHpMzlFK%)`8NE~J$JO&`=iXC4Rvmh| z&eL4oMdTVYqxE9PjjQ~G@KI?ugTMfN6C zTDKF)%vGXxGj^U!s|VCyy(F_m_;BULc#<+HCvc8NKbZZda>c=tBEw8axYCwc8d$if zuPE;mWH=RZ9g`P*^ADVU92wsJhr_{bOXN+vS-d}e5b!P2{o9FEcrTo+uJpbB8GYo5 zi_Nx$+1H%my74^r7*F=C@-oXGYIGvC1e-D_v~cF+WvnI0pJyQ`1wrWe=j-pA^FNNLSN z+&~}PJOYq7M~D4&70=R8uI_bvzfVq}Ar+k{$smY_Y!lSVndO)R_d0Ju%7da^WAmxV z1OgIxnRU9!3@L!GF+k(istupT1m@Oz7;n{RiO-+AEGO^Wy`jY6^WRm+_`y^7z>vMz z<3&`+pwFS_=Rq1Qsi898TE_x>OH93h&PHEeetP;NTohnzT%{X~P3QjyWAi#h!IA9K z?Ttu37xKFwBB#-A(o9%c%=f^vC(zniETGbCP9u*-TCBT}nO=C&$UcEgCO?AdLp(Tg zy$~3W=;-ELB*b;R)?6_n@2H2$qB4Wj-q9wyUhT8Bmx-W|sMx>pGubu$O(i(aHR-@N zzFxmyGZ*`)ukL)syQ^Gd$(ZECcX;5tE1s2q@Y3EA2t#Tew`=erPbM}XB+9_B)PpGZ@3%<*b{Q5>J5n@uQ4|M|!ajKj}StXRr9Av6J_N4w^5vtB}RnR&kAJqe%Y^ zk2Z=1eHu9Es%g%XGsp!|II%g_^LA&>aIzvs9TS)jcFL~4)kD7FM|(CTb*@1I;P`aa zl;^ys*Sb%&3YJEYg1+&{KJLm^Xs^BC=^NxV1H#F+ruSQ52NX{=%u~} zqr=HLc`|DnI}DRY-G1s~73~HRwQPKl!=|_Wpe_b`r*H=T#e#FjB%K-XCLO7k@1h8D zJt3kucPoIiOYwy&5w_!1ONQ8B>X>5j%z0+VuEmRM>UCiFqwO5DA3HF{Xk1lqhr~YV zYky(}zZXeXq2EC!tFPRVL)-bISFz$3L%Ke(`e2CHy}AdR&h*xVC&0`R`;VE!UHd1d zFHmtfSbPQZeg1d8q%U(*I+IBt{{SGM-5(pXQ}>y+DBOH)r$b+^(m@wKv>#yVR!tsU}MY4AG^t~iFMayxzHH`=s}py9T zQCJKO$iFCmECRoh7mQH2pt)^a=IGhzArr@I0vN?-(yp~>q;<5O44|T=aAJPym8Bsb zbL(DU4_8<}k%Hv0$NNkhoxFS=+!;+8&2}_zfm4S0iatOdZ6BwJSo0$#?p}*=95Wsu zb!f`8yt$c@cF9TP${+o>)_1Gxuk$i8#6g0rrLLgx)tl3sJK6HA1>tZ@p5U{(CpU0!F>l5BsLVV~hx1VjWGMQ7%o zuC_x(=T!NwgGs(`R>S(4h@0v|L*MmGbLQ%#CU z_}%DBPmzXinnI zO)AWkHcRrMlmHp~l+6mG-F<=`p|S`_s*co&^bYoafHh^Z?2H9fcc~T=0vlc0nQ$%N#49$Cj6=%&; zTt_cG3QZLjzU}|5-uP3M>l}j_AL$K9WxV$6S#x1g7#0wEeR1?Hv@qf{`p8JpB7USw#=pS$UZ&*niXsNp37Cu%$f)e^C{u&PceyPnt>xtX+Q0seSG=cXq-kQ=97iCB~ zjvX&1ON&)2(y_y&p%bup#DiXN%&PLd}@DkJ6Hn zA(RNLUxpYz8?DD|Kg#yWY0GUqJ0^P=1d-1iYzj5JFt>TjLFSQl#M~QA51+ToWHN9V z6=yj-V4$*^a}^iQEAvVUjLY#Yddj99=VSA(y;+l77LRpw9(XJMJ-n~m={|Iz6K^5K zZ{zW9zoj(~5yFu#s@`5sr2V`r%1RF{%#BiJ7NEnTF8FO3NP$I!PtM<@75(U`N`kfd zW*6r}j|$7lAEcRjq3!N#xP4feOtcT~X3)qbe%U69UViJ1%J$D==az@g)eEXyOVmfr z>l!?cn_VmC-+T-^NC!3w0M>zx;#NUJr^?^)TpdVr79`C$3$%_RK%as6J>~lCfl0MN z`cbx~!E6QCA!_!(NEU<-wY?mq)8thXYYz>hCyG#`oSeGJ=SYl0`2hT!Z&}3^jtKe0 z6#1DRT`5LF~Ml*h7Ww+dgDK$IDm!xYJds;V&4zr z$Q?oF+99$Z2JeWdY1=}j$U*{h1$geM?v=NDlP5HIERLwH9|Sbzc^cNxJpE!YnDorq zYwgl2Tp>3Xa${^AhM?K!6%j&eS{ZVEikl_!)|SN@nOjQ1iR)ttFkesevo&18DF z(F#{o=Qa&y4z8OsoBoDq*O(WlH}eSZIRFDkgCd<5@JA^t}MD zyD8QY$`lJos*?!fT<0(KrD*j*Ws9dS*5kdYN#e{9TezS(*$-ecot`2h#=#1xm%Ily z?j-!8JCWMdC6Y~@3IFj)a&}=2_9yOG-8bNFbb9k=Ae%{}8e3~RK|2h1 zF}|lty$K3LJ}NpRuW55^z_Zjp9LBn?pT|@-ECyT?TiE*|jOx!AJ_p;&nsbvV&0@Wi z;<@xm=1Ihx&yHWJ1u)gj2DU3_H0NOVUI8 zM{7D8+WTsWXnus<=;Al?VA@^8Yx^rc>EB~gMgyHl2WF$i9`fXgC zzT;XNumGW|4{6R5uC)Am7O5kmGvF3Xsw(;v#H+MbDpmo|u5d1TLag|8P^&yze6EuI zZYFHKB9*SLV-d=fbZQk7epqmXwnd%YwUq1Q`(ui*`B|x2i4W$L{f%6a>=uvmArEmE zv_tY`4>fUj={sXDCOnQqTQ+V~ZgDE7W2(7CsF_iW@xq&HD6Ft58GRcuihGse-#*r^ zkS9hv|HYRHx?&l&p=_HVE6d}%8F;}O_nU7CRM@zo6Kn_<@r`QdK2W%XOg--nWlRC) z5k6C)xEr-+YZq!D2p!rF8h{j_X-Kp8Y2&LOcCG8&;Jc6NT(s47mk3jOCGtjxIXrw9 zg{GZn4lZZCQ!D|QzI7~5vj;hflov1DI)s`yEssC35DWhFqh{adA6{e31ZP4fj>h=@ zWO4LywOL))EP~<{?*9J?Tx2*s@B$kFnNS2n~k$aX{Nq%1K7-mZC=ZB770(` zeX*9locJxr_hkeE`jy zL1!$e(HRSe+19Kz4;ez_8wO$`%Gj2*Pt-IHu zl7GH#1pf%tZ<$x)dZ&V-ZghP8J>y}0=#M^4u8%wA>a(I55nxlpzmx3S^>z z4>Qz6RWuEs~f}NVIa4vD9)?s=yRmq_@~LVZRNls- z^y#MIBNE7FQ_1SGe*N_i(^EZ)91#-sH{YAfr)+Y8FsXUGd4mgf5JkcRcQm?y(K+l) z{}DNjCijnU^H{U5I9%b2+Zn^;*lHCMS8}&m#$q78#Xb3d+BLI?_X!*u5biuD$%S`b z$V&BW-3!~xY~qaJopAZu^Ea|_T^hV;9~4mN&<3}svg{|=y>NTS0=1ol#%|Nd0~6s$ zA7Rge+0rrOf1(>`^9Fbtd16**kbyM7YhyODCjX)$Ao?LFgNJ8@JbP3XnMmfy-#}D= zcnS#6a%P{zb z_s>tAo*&dD8SEEREucJpnldH<_wVSuIG;luD7DneVLIP5QM)*s^m|?jN_oa&b;9w@ zcdOfiF$3~C`hCJL5z5u&Mzu#C6=q;b*b9O9Q-4Sj6{xOqBPcdXw{VCjG;tl}Od5Fg zLftVyRl5t*7&_?WuS+yv{6X_jY36o=>__@r-TAolm60=Oj$rwf2x3CefbfD6q`iO! zaf|h6o!Hl?usYG;m&k(Z9I(;T0jLs*G(EGsvJm&uGYUlvmgYgQG&fh92$R_snOsXM zA88TNT3Yjw!S*IHy^2yd7@}?pOF*LYAKc3sJJqj({yIKi*iU=Zu7zN@sWG~bl{lRp zHpz80F+jlwXdk~}4tZ8mRJA@*J$i#cpSaua9Od^AcKn;;`zoo@nB4fuL|p+gEAK5| zr9jOL1&7r(Ks?Zl!{)r0h%c&RK2qD#=CrR&v0}8@eL7MoGK=@kpSMi;XHEN~k2H%L zym;j)-{tej+HP~=h9(i6;iS^k8ZXOhQop%vt8{BfNj=f!E%uA9*6=aDc>)7B|Cb?` zmQlbH(QRn$%1iHJi~4S%*@pi+ELy~$qivwHfeWuO7!Sy=UY+F2G_*|e-0W8@-24p_ zGqJ@tR_O#bjZDK}B1XSC2Zw0WK2`8l%|U}{t}4;!8g~j7d=c+Ro+_4cbNHyKX-%b_ zC%|GudXFB3-qIQZzB;g#E-!_;mpantV^b>A8=M-_(bt_^#krt)WPiD+}69nN0VD#)fe>%zXtd+VLk3Uf6Qc|fh9(QOP@L$ z!Yh=_iH~41HO)F{2pXYl+l&b5u$fqsE81yFdeOr=5zdJ>-J-7jSB&^P(YWO^dM#jO z-ra4(@50;Zj84o>6%;|H@77&-=jK)fUIw-=eCXq-=(Au)Km9(bg2(tzUj9+ldZn)9 zbWjq?TdmS#baVphcO1RpFyjsV?EdFmYi$vT#G=J9r_{cKxL^Gr|AycSdin2SF6Z9; z$G!3AU@O)l#P|$0;_0b2vGMOQk>c;B{`yblBnQn~;Od&rukE%65H>-T8U1$yeS)BV z!VZy0Uw4P@!)q!ViW3=`Ot>GNAT-d~>$5v}Yezr;rGVw9tjxmh{r^SA=#mwkDssg66HL;oaGf1?6UVwZhHt3B zPUjH2wd+&#FJ{svPhr4<4j-7~!{dOsMZ;jT;=rVltMAq@Cw3&hl%4c8aL=Zz8X>cu zzV3H{r$(uVEU*)2ZVsHYtfjDEWQxDcscxO@8bSJBGp`KR`)*OeIJvK$K^U$yr9STwKe=-!qoyCL<^*;T;x>|vGaic($p_2EzWLhLF~Gnj)@9xB;v`VwG}x|C+-O6}Tw~ci)6nnK-!%$xbE4T}2qvb002|EZ z>vG`e(8xa9Y4!Vo=}%j+Lj5p%_EhsE>Zizwh5~13Vs_NL>ylo}|FC1|O}9oVjc(!d zj5Zt%z#(9FEzDaCT&+F1)(;ASzxZ@IJX&ekj5`pkS#muhpL(0RSq*h%cZl!VN1z%M6S>E_H4yE-e)Ld+z4eF%?Ak;d;b6+lXg39 zJ~@9DE-l{EVa;2M0pDAXJJDZMa+u{g-E^!!TBsg3m@S>moAe+%K01^hQ9s-KoHBz( zo5z<3Y7W+8i+{FxD&1mL33Q<15+0WwxV< z-EtRWyEPm7*{uU;&m}s-;rh{6#)@yWA$-H9cE8jQnB03?qtg29FZLDsSVG6{L5gbc zN^Av?RP&!LDkX8YrHR>tnlRV-hmzv$tiZzTAohJ}D+L_PUV)d#5bE}>A27l`=8*D>;-Hzjxvft;4YE-s&@fK| zO=e&co0K4Nj{<=(v&-oJVDPh@XRzZ}$0$FDyX0&nf7zWHP+%$ys$&qj=!Fk70GRft zZ=7;3O&rLwUN+{nKX`>;{c3~9@x5*#)w&O~%zirc?JCW{ms7boM96*dQ#lhYtI%)M z7|~j;k1SHB;~_SZpIX|!V>lE0{ONq^8z13_8wPeGWw!S@k;dElRb$`f7ef`Fyt8PH zx%F9^3OHPUq<+C5CB~Z-0N#0jx=yju^l(Xe@v}D=7=o25_wPP$9(6RFLZ4Hsgzr{_ zA}+J^vA?HsuD_SI*?f6e27+{5n0eC36V<7|3{c#(5$0+3>jcF83H(WyWXeW1COq(; zE5~ek`GA-&WW*yO1Fh3c15#-Cq7xd*xPcdn8?GB0MNThuU)kN^;i75V?>dSpU)%)i z2jk9+_kkj9g~kimw)1E!hva&>=_}NP1epj3Xw=pmf?6xF&aoAPIuT&RDw1Zj3QIHgC7 zH2%j1@=VEMjf#3y(7c4HR}euw`}XdBoKOfqNaq;Zj-~HcpLz1mhFquIp4wpsF)aI+ z&$QyGz%j2jjsS+(x6TwNQpd8#H9762#8+aUw&IMHMP4KKr|&RY-|-1D=;r^bZXD%{ zk}kaYA2C;S{o z-ms}NBie(>rPe&+GkUzKv8X@Y9atJy{n97r45b*H-rU5$9U4mJc}-UAFb0;92%EaW zFz*=ytRG)}GMMBpg&}(=YHcg^FZ9}_u?N|IHhPVbS=C6SevJs=WJs5t9GtGQUA)ST ztsvS1Zb3a)|JLxd`8<-d(3 z_UUiD(ft^I!$Gnri5^vOuSQ;prh|>_h9^h1;bE{;c3I8Z#ys4t?(Pmi1#KW5^l5U* zXD3agd)~ouCY(RX-uc{$o0oNzD!@!i9Q3H9Ek)V;MBR=WPj%2c7JHeh@g_YT!E~W_ z#=gSnJ;o_>P{D6w4(}}m?v*WV&s6)C6a-?ygf>K$=t>FR6_QoUlf^~-Oh4` z=*@rIR--k<#fsrSy{3Mvn7Ci>+){2!A|RS&%yB0%N+(H$sz~GWB=0O;X2PNvK|gID zV93{wvT=#wdy#RvUm94T!Qd!s-;v-*<|xLz!l74|5%j^=?`*w)mHHoO$8JjG%60+-{e2MzDi z3tbs=L~`a#3O?{x=FgCo3;xY4^AXzkzbJp!LDtNZCshp8vk80U#N&3|LPI^(%t;re zmo4#wO0Z@+@fu+{mJg1bhKjR*ED^+_^WB+=t_Jj2OWaJkq}&g|0|K*tUGXI=-k%d~Lg*oBR`Oy;wxQuG3)YSf4}{x#soRrK{EO zdanJ#A=9j@DWgp^nZF&P3&dK{bggbpOduKk-o}GCxTx0BiumL5FXHm38{A`>sn+U_ z@swH|CqY&bck!J$*lJMH6fq#=)Ocy6i}D_^ykM@{Ge{HhnkF=ih3si`@zQp^+J0|dMuKk4Gb!%x;sR8q!7L(Dsqx2?8}j^@lKy9MR#>wR_M~$uJ(2bD%4lY@hB&WH+=cG&L}V zy*1V)QvUdl3^6KthcK6p`Aa@e<;fcE)ia^>k?a59*RS&HAn@>~#wGWCCxgQcBn(&B z!Ud7b*K3Oz_3^BH4TRNpX7Zw*Pluag9nMDIAtF}8Q)lA?CX^wB%|0QMZXx6Q{%?k8 zwb8y7BRLrPRC>36pY$&wm|0EEh2r)oAB~x0>+XtiNla&V*jq5v(6aYcPDAcQBiWAC zR@_%LdTsE!Ij{#wlz+RSpYhwjU{>UfNVs$SE;kKZ^0G5yu|Bb}P9C$qxoQ_eJ8ph% z1mwEc`hh{PTfz391a4dso;>$#{Sk^snELrey-bT8lV$F|=V&L907iYfh zn1~G1Y<-%Vlh-}Y!}&oyhX9wrs`sj+;TZ>)w2z}f4TM-Zm_@uk-7Z*anO0L>&0&gK zrxZGMg)MnL;Z;Yeo8GOqyUspX=FhP!__D;&i}_Wq!WilIVvThmH)=^NJ`dj-VmK}r z?5}cX4uQCsEjb8D^0npnNuVsbk?yI@Y5GMna<<1?x_#G*B;2|D^~$ErmpTe#;$iTO z9>-BnhS@zg*hXTX|H}7R2M2Zga#>>aND|TFmyv9d;Ojy5x^J~&ZdM3=UQ)6$A zLyk>mGk~OhtDIUC>(KPBmSGL=+BXE_Gm4tk;?6!1q)l88$=J=cVnRPJnbP{JW(x%A z!j-C<$-~psjZfH5ZIrdWAV6GYwE(UzkWP;f4o}1$Q+@fNpWqb*d0^9{5C~QjPokK; zOBkyjHs)iq-+}~oNmoRMUOQH>mcDB(JI|4}|1keN`hQxX6_}RZ6qYY^s?c*}!sH)s z{eqz@Dq2BBg~_UsXAnIq9nm4SBrwNtln25bN#oJp_OZS2U6kjgBPv}?to-=r_pLr; z9-Us#hBY5yC^+M9oxNHy?JupGxOeVPLKt)epOJFsCf>WdV({z<0Ngiv4}4VZl($R+bHxG7Z_~1vk5@C#y#%M zQ4;%^ND|_WV zzKN-*dqeI|iEhbs!f&qcE!~=pUzQO~_P`0K@A%^ybp(|pC#xm!(}ooHoNIKVudacG zPj0V3BNvRh;fkLs9O+PEncW`}jNqvqD`JpUQ)6^-8S)03n5F3mm}oPj6J%TUUSm@Z zdQN)WH)bc-2R6E1qrq%>OEjQ5<`Nue5X6JH z7LRzak!<+%_iknd;so*@;n!9+*LI{j*$l7q_2Onk=V-R{1luPElxOaBY-4p@d7 znz)kwEfteu2K` z5EnpPtM}(hhzp8m_-wEAS>~N7@LlNE?bYx8&O3h)jd;NP^#*?1U!L+;N05ehgD~kx z+7so6e=#dMTEC~$O*HxL_wg?K%ba0eO}|A8`TGKxm=3|w>f2AK`erJ28Eoyutt*FW zf@ixJpc&D<^m&z8h2ngFWA&M3;~cwby^XHyYJXz&n})4z0ImkBHTrE(HOGKEM-@X; zK%y*Zc0!l!GE;_;Q394PO^ObUelf$-Muy`yxrtf+-Mr!>KmsFL*X zxc@4htzLwR!(?-W-|o-tXJ;W-O`)RPEQQx zoM;oAG`}I~+;byO$==%HQ^6v!KB`6 z#3`TT)?9YxZ|j5QGHX_V9{y!C6Q@8M%^Xt3=846_K?qu+(lpLD*cbQzO72_yzAMeb zcX$|W##A!Q?_*Qy6~D16kWYoT!cUy0|71eGx^5-?-l7^n$!u!%gW~=NmPz^o zkI|bw{-3BlfMjg{f@JmB_XK9fdE{%9%M@58K2gck+G0z%-qxwnEP9fo_mK6%vv^GF z%FzPl4TB=lGa=SJeShpRooD4)?A+!445F>19Z}5n^ z$dO^L(8h#UlU~}yfe-DG6rHe=CvVNii|#o^P+=m@w_6XSfY}uHitQ51*&O=`t{A_T z^_YiMtB%NmgN&h#Y#CSD@T2Sl!kPkR_-k|YIc@`B!ZEB%b=A4=;Plcyepkm2y@U^q zWmAw#3?k8I6cpw4pZ3SsgfQps;iYD>3!!w+1u^r!O|a?_EKQTh2|?3@GbtNa2ry~! zdxy}rHG!@jkg_MiyL7FYP&-(DL9>eDF*YJo?n}1hCua(BMpwlcqJe_@Y1=iUJ{|P_ zm~hhEN~gD#bc2rONN@EA^YxYy#l);Ht~xn*S;rq2zIg91nkcaA9M4Ms;Delfy&p{! z-Q-mBP{THUgV$f%nzvU|K#31}%wZ(J+^UG=G%kLt-^G-BRtM?H@*1!5tJ~S48!%DL zD}3Iu<0k9(+OGr>0x4HmAgdtV9e#hjRj?w7R#dzJN%1C2R_i@{iqG~|p?*<9#|OI{-pyilf z9-S*77DFM2Pbz_s0uKn-DerQhFLlL0QT5)sP|=>}K2||{&TEd8%EK<3zsLhJF0F0V zOQ9UObT(2JdD4fjRTDh=B?r#A6IF(FJCLCajn_4*i|$TJXyBA7_fTT#wHd_3FT-B8 zVYXcMOJ1{kHKaFpkPS^Q)Pir0EAp&KBJ$5NTe1ca?gKq9Y~ejGwBY-&^I*x_;MqTx z0@uDqD#O!27l1Tx{y%JeWmuF8*EQWGD4l|UqI4rAAkre;-O}9+N{4g{g3{exLw9!% z-8JNQkDl{9=e*zZ$DdvoGxzL!?X_0~2;PjoFsoVQ%mqk7sfT60#k;MV55PfpkJmNk zBO8t?0xV%0ds`DeKpO5&(Qv(3h{`1WWQm+djPa!+y*iZ;U)>k&S6B+pZ{jd?6*QH9 zfw5nzyZ@I5Uwx2_tG#XS#k5zs#zrk=U?roprEiE8kaWIJlC^wy4SP$BowqZGg&P6d z_;G{+|B1Tsu)%+vG?VQ_{Sg1AO;3qb(i6zc9ZzYw!k~KD65NuVai7OE6I0y0!{D~) zWOnw|n-Td;0dTBs;o0|{Zw`)J>>-|pp#DCm09*d=tFE3^MV-;-L0^HKx>rEJkgPCi z$uaxqzFrQC$&gJN=tZU3gewdp6Am*%}lH{WK8YR|TlOZ1ZwIcN=cPte0H4Dup?v@pKkd z9t=ZiR_cS>HE3dgC3}>x@`6Dx{u>);@BMMkUV6CE_z_3Fe`w=3wFjL=GVPT?m05hg z`i&7MZcAR=4pxE>=hj2Frcy6~eL=zZq8=QUsI16*7Ka|~}%jDbUm;O6GQZokDZnY1>ESsPeB7tXi8L*YWjk>1tQL^4KV1={`IB zc8Vb1FCyz)`8z^53IE2C@5?OlMLHnA!lV+b+O~K2v7Us}gs9eU42=|N$ z)qsKkZIIK?vsAL^ZcOJ9pI8SM{9<-*Xr{2ue1v=siggtnx*md5V-FQz zG}{BLc%aLZ+aNFCS{ggKp|#4iCPN%^aqVC-5kvgdTeV8b*!4Rp}{)>Iz+1b-PB1Ytg zYW}2u8n@CmbXp!2i6im`5894nah|2U7iSb z)4@AO(N!T`Zj|3*4c^gN{UH(w2lv+BVH+OvLH`n=0TOo-{eK?^{`46T*8ws#+uL47 zyZ2BX8X*?R|JI?M45+|#v1pj(dpM=92jc9B1k00aG(zBX@wL|h`TCs5aq0<4ZfkL_ z!L}@=H9EVimx(kr!wFlLv(grQ(SNFNkPf&S@!Ng zLW!NxfuoN!YxR3y->~LjT(@#S6dPs5rlld<*@VEBLoT($2VVV;j01V5{RySi8<6@|D5Ts%V< z+2Z=m%U|Wy!n`Ql@?nO`+}{M*sg=IEqqdFHrI3%nHBt1p`)dhuEKfFIa-CiI6Cex? z_CbFDm({dc>^##l7!n0usIGTA?Ip_E*o&y_(bHoTGjjR@JIP1pn;hxx+Y5h^1G$;> zl6HOscx}|aV6owuF`x(4#S$oEQ!8h@r&L-~mj}?q3u+7mWiseOSQyv>2;hIv2|~)G z#sOo4lt1e$ViY)boXoH!q}fE>WUz_IyNjU0CL$#k7K{NHVGbZHth=QM>2=;C7x(=G z;Qk6s!4mhrQ*0c=s~vYzB>v+!<8bF7$=j0U5+;$b+kQ9wPI2XAl#JVxz$1%7JRE9o z_ytvT{ga6?IuUm(U?PUgM=JoTEZ5BEc0f^VeT9?!|hhbyQrIu_0O zf`$%&&`<<%&*JAgZCX(YX9Qmg7306Pn=u!i^P>sLn$$nxVFEvE9WsXAB>x#+@H-;f zHqzYzkbBvys~@yH!k)ev)RBXdg_TFmx3nG_6~krF8{}opW7qFoVVb$T;wV%ECr-eW zp2)e{Pc!VD%lI2E_~4O|b9qYpHDv;$k=`28Wv}{x5 zaALd-B=u~cc8Y*6fgFg`c2lMvjPLfSP$7ISuapzdj7j23CCwR-FI;^?Q>-M(QzFcG zbIxD#vRz1b@=qAn^*Y7l2SCdOu3>ir>TF|yT+Q2DfMcl~qsB#<_zHsoo4oRHr-{)) zzoNLDdOV1RBcC&eC0myREGP>r^cSm=fiwZVt;DxYfNq>M9c{kw+LdAAFSlyjul-55 z)_fpso}2r0kR1q_WVsC~)#&TRjko>vlcGkR@f?2cY%LhLzsM?_MqTgBppm0J_DG#R zX>j1QZPu0eav|$EWeqmvqyG&(vyT++P1Ec@x|KT{(WY zb9~%`@&My}OD9u>qZ#+J8F|YO2gsYy*co4_ZQb|%Z=U4I8^(0K4;>$w-C;Ht=#BIL zR%~fC;QodNR#p<9Y40!|``~!-g&On+0KL{e5Wbyl+a=l8D+uPj#;#AOB3sfj`e#zQ_61t zP=lr^v=!ramJTkODuMXX3YBn_L&K91IoPiX@US-eF$QDKm36C{_@(q1?>GPvhU8Xa zDQSf4Vzd&e;F*^Q>-qXm7ACLo+~Qq3T_CM6`R-e4S#~K_O=3JH7 z{?mtkXh3dn`0%i`8obO;=6x6XpIBiB@@cb$T-UTwo}%#&K3-kEA1wK?KnPGZd#QkA zRUfc!vXy_dPzQ`|>cNZrDhpx3UQ4(fK{h`}%6jvdBJjCA0=)Z(?9^J6Org;@gtgzO z>nL?d1?VHsIY>F%4~qX}dHpV#;kIU1g1qgvI97N#vvXr%D^Ux%0}?9Su14fQkygKx z1c4FReVcYwaKMrAdhRIbhmL0$BM>KQ1$fYRs*}}pc@uOISO4x8K=sFdl+4JS_@J51 z7+8n@X>spqi^1PZ0BqH=e7muEajGt_=5RGXh0@0CedNCbjOm3+qF!^yc9nJCb_Gk(TSzOk~-s#vh}+JO0;E zs94L&+%~`qftkdXZ3PsglpbpE18PiW^sQGxEjgmm>(Q~Vih1b z6(PCDq9zGI1gZZ<1Sv_T_B(4ng=4Qfsh%VI>&M3Z<`>MyUY9(1leYef+@P8oPY()^ zVQp7{pd)9uivpc=4!5z`^my{~%;2rx(+wt#wX+S%Do|DtqRmEq)awCg&S!W$jNy~* zLSe|Mgie+hb6RdBUD|I`7?rVsWJru{6k1A{{MOZNQU?(tz5m$ZtUrtJ)J?d3)(gGf zju=vNs(;3>DK}2HM|LA)I+q`d#$?GZ=EoZ*?uz-)PdE#F(cSX#;wYCf&e@GaJKgr3 z34l!lyLQS87;zg1<1Ndsf}#@VgIhyp=UhO9Fb++je&kUUL=;T0UABI{t}yLL&RYYG z(;~g73-PED$etpke+_xha9KneJ3-FJt%hhZg4VfJ4y6&CZlQ)x=tcx@J_j@@u>+U# zD{Hg!n1nl-`tdZcx;Xr3q0ILPAyk=C8f^88%pX&Pb)&f-i9~^9aa3_^3}nqRjB$>y zzoe&5#Uv{#2i^f?D;&AVR!22JDj{H!bCm*WN8hn9t+YOaqJfaV(ZI@Ub3XED{R`8k zpV$%N6JSp`Nb+2r0t1xLtJDq$0a{GIKan-xPl*Ep0s?ixge7g@Mt{6@x3kDx(-TCO zBIg<}+#A4^d}?<7)`9M0ix>F*<5r-uH|`RkT7~nw?4R=R^P1nMI~$^ly;&R@x2|lm z2*LjS)3Dq9-sF?6xMmm)0``VpM&w%zGD#_Q*l=RRP*OwUPk;iUfZk6?ck_4HQSx== z_ybiFP%dq07AVN=un`?8iA?5j=Fma1R2oMBd_s%}%x$5tt+QGbX`!q7MthCesBAD_ zu%B-Z{+xJj3wY-&zKx$&Ky75K>Gpr|1OHWd7i~q5tgc+dSVRmwM@G)$3<3% zn^g`!13?xQb#wQI?H>A0YC208T0p^7uFduWv=B}6n`t0?Q0Y$@+Tz~kx+0TnFX|381!P5Z@Yr(5^KzEUa;8*I-;3=DD~R0P zgLlOz!;%){NLd;IINo8Y$f@Pdin!0N6E&qnQGMdrC0%&9=90{Jh~=|1D|&y=%_p*4 zH89+t*5-vm`iTL##LMo&?hL@bbZSCsr{?SNkWY4My5LIqQto}VH zu8@E5gANBQF0p?rf)rSwHgM}#wG62|z@xEza?@u~B4?s$=gNCF#xU-x@>xo-c6ZKU z({b9Y-!kr^Lpy~l2Z(P>wQ|gs*}3Fe;>ob|n@pL}?8Ryz7awjJwUdH_23ZfhsqYu+tivPCLDMeJx&K4;90+IDk2@Ftt zDvZQQOeI0dN2c~2YU5ppZ%C~VSI!+6uz40om7#Gr#_G3p?UW6`DTTyhSsm|xw-=S` zL@jSK$F4=~^BD&N+ygNVsR?j&B7u(A)62w8i?C_%{l@P3*P{n|&)x6+nw2Up*8*IU zS2IT@V-G3LlUO_VR!ZPl?aac&kw1`wm8~}w0Yrl}V9BYn`i33oIKqfCuFw4)Wn8Dc z3Z(Rn^y=BOjJXdfoB&M4pY+0+TEN1D4Cwki?*1F$>*F)D&Ko|@Uc$zk=STz1=2Mv= z3jvahZEWC$-!Vu^B>eHxy#iJclkMrR6!Td_6^u^1?RPFEj$ft#hBS zs8yT0?jh0#P z&NV7Jy zt*-w|A!NlZ9sn{~X)vC{P;1@Ke`fZ54{c5%Klu_|&F&|1cIEx}1nS&0nR8zDG`fYv zD|?B;`I-X+NAdgp092jw^8W*Aa)U&x3F)Ev#q0B&QRw$Dxt!et*JXxr;5F5k3(B!d zJREJlBIW4d#k;i9ahd$@@`E^nG86jMIF-hqg3S};fd~gM(2o%_aXJE&BHU?Qlcc&L zS76k+QSgMHs^fdyJJWLAXl>`MBXnml-kEP+jHC%=-(xM$+|57MF>BerX(_2IbegZ< z3vxJrb0twTV^E>|6@tjL4OA(WzxJ3i;0IP*HyfL=!&mkp0|7~B?2EuW!Z<#Scy5%o z2B#cpOu!?_1h*8l3$S|6feG3?B}rMpCTH#$l_0{0xW7MpLyivO z!BN%v-6K-%xpbBuCvd;>I6$1{ajG#sXzdjW&A-evMhLM*r}hoi1{fv}3G*D|Y!^U$ zjYnMYNs;mVOgR(>Wu$C{R4;Ys@OUE8?kCWsr1P;odco=K^RC@!bgtqyVMj_LSPsUf z9~ODa+sLge*GbraBOEx;;aVxBVB9n*O?XLETC2q|&(X@p(A+7W$~R%pIdywzbs}E{ zG_qLA_D<%&`PmNgXY;o~hY04E4J|jPKo;AJ``4aD)dDtGGCRNl!@Ku>2!y)hT=c+r zZ#Py;*=N@BdC*w~Wcp~UR|GDk^8EAqS0Fqn!l z^GQ}K^LnZ_`a<7RV@305?me@9i9uuTWBlK!2ft&OxMU3Gul)#`ORPFb_U1n~kvS~l zhYez`R8Nr1wm2x6>F4dCYW}yW>g)Cy$z}lbL`BGA)NjyT?_%|{XkRV1@vL*vG?4%X zlgPD?XxegzWx%|tn>xCgx*8W}Qr*W!bJfyXZL+#y+wny{{H5>dntdn~q8v!NoI2{Yd_RBNL@T8NPanNiZN-Qanj@(qu3gseISs*bT zp$4-LC0~5y5g7*N@v%&oF6Zw?$NeVMMp`jPPVK9uq`E6CjB-roo&l8dZW_P$2;t9@ z6KfG(d=-R{)O-*SdqFHFNT4b7QD0F%?92D-x)m%_+;A3R8O3ihVlVodG1!>1>F1f9 zFK-2hc^vBjoukI0v%7VRRvuA(eZH)cN}5~#?!=k$G+8}3eK59IY`U{E{+(GjNF4ov z(H-cdeXQLpzC~0ysa;RsU%Hv9sMpjgNHNnzB2ILOjTQNZ#TXO3%5aom((5(m9uyQ6 z@yV2TQ2d=Ix4SBR|7?FG2XSbj$9LH=f0Z9^rR$Av#zoEQTIpAkK$lVasn4HwJXblb z$fsw?BPfDjkIA@s>3tJfCYa+Xegn7B-5oFb%xW_w?csahflbazDK{EucYZv-I>CC~)g$kJ%wxiA3Y^)b!3n+-m9QZku5wV~W3avgs^& zKHJWo405wPyL`~%(AzppXJ3>c+*fii;|000R{iLh{umrQ)l6|eJS@%1g=oPI1Vr}h z(_n5<4``MSD#CbG<@q0UzcZ({gR4tQdy`z&V5ZxHGIZ&)a(5=U!dSO~t_hR|^3DL2 zf^_o=+zemMGT#tqy6WV5d0#pCJY^z;(Wfw_SfDeGL0L`rp}a;4L$4Nbi{zB4QD8-?_I2xJ z(rtQKE%MN;V&f$uAovY=Rk=AmNak%O8kblqy_JtQG4)p%#|`~~Zj}Aiy%R?6_0f5g z!dl}ydGRNtQ+QW%uayM^Nw&^W7um@O6eEAtTGIKBU=b`2r9vBmE7bAtOe3qYo7=HAaPjj5osKE6+34Ppx+A;>Lv2%UD zalja;h6OUO7M%FZYSFnwE6B8sL^+5^+q@f+3Ry=~8ftkSiz_%baX>qL_2o%61d#}V_r9|5yXy{rHM^X}beH^b75Fg0TjN~WK1TX3+iAT~^^ z7>crCYm~@>DScH97DXN-X9E`r&vS{&5K4V?x@y8)sTOXtbjV(qgjtN957-X=i1&)V z5NFWIzfQ|v0zM;7%lTE=hPAC#yF%aW6`d$A!QgFd{y`QVMh5ow}U*zMK4p z5`-lBWY^ioxZ8h2;Tw98pW##d#X+YERS2u6ortuGw`qX`(>B)Sf$VeUKo~z<{%XCe za=Wf!^)&<+xOl{da`=e5yPcxWZ_a?eaU2|)Ggw*#(SR7|3tHtMShLcic((GX| zl2!cB*`C=w(`%Fpj`qreC_Eku!bl@eo8nukyz<#lSv)HA7*U*4nU~09@N#Uq?Zn(8 z$caw(9K${y^dZ< zVl~kKr^wExgBNiXLYA6D`qd1r*XQ(vN(TJ4aguRdD1k#4!K7dP%6<9P+IcG}&*TU1 z&9=d!l+F#>RcSEUx4>C{GfD|e-Su*JeAj5T2!ge<*c~@rh0G6HGAcrF#EyV zcJ4Ak)ZQ^4ewFLRzW<)SQh7Qk>;1Jo8MH*MiT7aaiDXg@WGjgt7YBF+S?oS@1V<~a z&GmtxXpHy%N621vtPsIs?UGjY>I0$~xkv89_YP&zWv$sG6zk9aqdvYan!qrm_DBcw zdAWYCsWx!OL6L2>_wiF zVX;>+YEfpHRm8{DOmV|0NczuiPm$m$*=V?t${!NsuV3Q6X9Ww(ANM{BGb`gR9q14X z5*eh>k3LVW@;-lA58~D3%(^NkH*c>A8Y9y?L8x<#fF^A4;9K?jd`K~^K zIV0Me_4hK{Q%<8 zN4}~;kBV`ff;O(mz`ZU^ZQ5(}TKq=vSB@_47sb9{(uYv*Z=$a{xHFy<7kL!IhT&nJ z)6K~qEdhUjWxuskO7)RyR1i_#AOV=%-R698up5z#F61mt)msMK@RnuHqd646c|F<^ zzW^tj=!b)sVGkon{|##3XE+ z{f_VTc)JW7gC|P{%)W-Lq$)V78>1J)5@P6co5~xqgejE}A}qb8C&EX;YzYfd!{HL@ z^l4?4v-^D6|1*w>ZCllHd@`fjJekn}c`dJ(C^bx57Dj8+xe^D-`BNbst3PQ~;N=rL zVT`g2gk(L9`ouVFx5T7}G0`dzrjqsvE%(zqME#{(3EAO|y*7@>`+V=gk^qaokEkPN zt8;yHwduYOpE|0dg--AVAF;8+(XUn0n@t~*EEaErTqzkSUqe1h-R>TN3q9kl*`<^#nPh^`m_cwfpwoPV7>EqJ#6PS%MkmxX+W+ zq*wS4Z74&5*6<9%^BkE$B1<2lSizWxu2VKFh}Q}7Xnh+K^u{cMa$`dDgg7V`H^AR5 zH^HND?|M(S)q4HxdvJnDn7npG*@3$HPhX#77_dVjgQWQ5J8oBWuNNC1YJqQHWA_Us z!}9)l0%2;!E?n;FxE+zfYQ_aoD$xVc<$?}e@egmDLNhL@J8X7!h;d#^VHTb{GcNto ziX+X(d2p=DK|PtpOhk<}S#`AfKJ$POE4Xb^Dwpx}mNHy^{tXaP)e7z)GWAlkV!dkX zUz3@BnY^SqF60cIm!Wf)Aq-wCR&|@XJPf19$6wrU#^r4K0h4>h+c(tXt$r3hw0221 z_gbVng}N|?K?oqBo<~+RK{3Dbl~j5(fjT(X4VA&!hTRX88NS9ZUDu1%DtzLY#$)-< zO<`1jJ*RaPPaz)C^ZoJ~rt}VL$fr&^LU0GOb67LIP3=yU(CFkS>+so4W*qjLxLu{d zz2~w|xmpFoqwCnXQDrK#6>bAfqikg&7bxDY4rkYSKM7Xw7-yO91DkXZ?J{v_lAu1d zKB1+-vHQ8}Y?%zyLhKtC>hUcV50+x`QLy1M4p@2>uW=x|d(fw!$Qg<{?UJRI>-+AL z7AnvaG)Tj1O#xr9mK7AXW{ME|u)Id{$=G6RK?wbdgChE?{sx+Xj1z|13%tM;S)+iq zc`Obg0RJl0j#ZtyR$V*EJ919y{P={2I`HK~3HuAvb~v06oY!Y+fpu&sVq)X`u^3E# z(F0d=_9=xP`Ao#(rq|JzX2pVL+2@B%wZ2$SK_do3*l+bqeF8*B#vN=2drf>Tyc%-A zPD79)kMEideAK>cls~C`-9U&3R}<^~_gI3u18&W+r<`d=YgKzr(7(0$V&=VD8G%DF zUdu*G##wucLmJ#&HoT2w#87VgXRGsHwf);+5~oW#HL|Ejm2~4mHPElUW`5e-gm^m< zNf#vwc7@JekueH$HeI^7LiT!J4|3f9GZ8UHPM(nv?KGrx+p3zRr!QeA7PEL~S5BQ_ zlFtFiTI@|SB)n5?|*}X1dZuk0B^=1F=wAad@1d9g);4C*(vUxK7F^NhOn6+pEKfM?}VB?)RO)1>fU7)-0(5>JRKNOoFyHcpWv6h3^DgGGv0{n1;Rb4?;HhWek=o~$;jj` zL3_~#RA_N`l)+IqJaS{Ri|iww&|wYY?9RNZ8}{tGpKtN(I;a?KK7Yx^De(`AkuU3gB$E;YvPkg*TEAV16zq4& zl#-2wUUHN5OLXrt9c`y#RTp{GcQA)NlewTPido`c!Bs`HBC35>i}YZ|K9mNuN2!c; zV1^2kH+go)WNh^Faq~R7t!Znto2>U6Z7O&Y7eAvCoBojbU7)WS_@$4rA2}Nq$pJo2{3MH-tCSrI!8N6d( z4wdk=Q6ufp8e!v>$7THXbf}Nc(%5+vCTzLX2VNP0%I00IF3%%sai8z`Yureb6UcEp z4#YP~#A}WYA@(V6I8X-A=u&=KrfhLRr!IB06l(Ea29?Wi{4EPn7sqkx!AfVXz_;5@ zC5M69)PKL@t$<&_;NFK!^_($m}T}8Gy7FU7Bx5>kR5h0 z3A}V)-N`lYW+0lXqv(x>N;b$vGNvS+BLGqs5JP!x(6LSfjjux`a0A8biy;>Y+ z-kXvv~dagaW;Azn;FQVB2E2e`60Gu!D zJOSxv!6H`+#Ia^t+r0s38$+mhe83)*gn>J6zC9egtMDWwCUy!Yb8@*vz59GtaehI3 z4H+)6CD1xQaT8`L)9(^lT)tleI-^3-}!EUC`#vT6;0GIOp=iqj`60!8}@0P+@5oCpSV~0 zpPVayt&-w@`L5CB^nt0qf8#oKNl0^GlPI-IE?`vS&b^0`V4wPRONU~|s+&Gn^r=Ka zjx%;G0)Eb7D7X9q%p*H7cxa^)piiR1e}s_?g76+1G0vIZy)Dk|tcV*P4q}_2VBpJ2 z0dQ%z)jK$<;C!#?TnLSJzg0JJ_4>ga#=L4Uio66p7{f^RR1d)e^#{Ehia>yDlYcCj zTlOVq{BR=aKum4&V-XB~4S-ELpbuw9xP{VQ3lfkK9DEofB>-X)I3J-V9a0O|>Q6Z8 zZ(uTD1u}W+8m`xdjh`b^c>Ltsi0W8~d69iM#5c_wl~JF4He-;nl77DrHZ0e21HewV z-*2$9hrnc~Zcj)YVlE9saUW1lPR=Rq z%4g6aR35ptTQyQ{LxSL%Q(+>fXrwTNkdDjvR6~bz_qPIvh46@oi#mkMddNhpbIYyeRaJ>+pw3CYN>lofiWW}+PzgTWVt3?u zG^P0kOeX6&_x&J7?-QylU}q` z*6iI8Cd`unx;s|Mx+tAf`3|8CL)(Em8#mTs&xk=Mcr}1(8T#Nohh|nkLWb`mGgNDc zX$$ujw%<={7FOKr@g;|TA0MZyFKT;to^_Dr;Bp_A!e5{X_WC7W=d%Zll8=gB;F5?U z0g8}kcW$Ka-Vew=J1fN26{lJJa9I7ZEXhMw6G7NCMxDLe?U7LB%&j=+Uz$I=miNCBJ7v;qM7>G z9IrgBi=iNV4o}c&8ntf7BPfN^&3}R_My-)kz!p*jY}FJFX=_SfN!phShgSxinDM+M z$`<(hYIC(mJxuJ1m(?JKnx>g8Pg%lwp{oRh7x$-~CS{|9AmgC*CL~gufB8(S(P=S) z<1=W!25bJq8z14=pm3Q92I+K=Y?OAaaKvXm5fwh3{ZBr zGwstlk5X5CDi1Q3@2R=jlpMBJ8FU~qx2xDKpbMV8!mDGGjf!6j$b#1b&LFZHd;B98 zkf5mZUGwApk(2XcRfN&&Iot|yRdx@u9G}|Wsm=vrx~RO z!{`)2nwnNu1A&rm@7wp(+;I$D6R8L&c{lhsc}Jl!4S=&(?!MX=P#CLaKR9`J3H)r~ z_t)@Ft1PCuJ*=ANt*}=GUk3^@PHgbr?$^{Jlh-c4QLmh7I!ikWrG@O92(EhFj8jx# z46Sy$rfSufhtrzP48Lg*m|Z4|*c%tX^L^fZkcUoCLRW`SUUJ(2X%VKAtOBb+St zPW=$b9z3Fh+yPe*ss<7G_i$adAI0G~NC+w^#%eXTkG8X9zbWqhk{2dG(>38AG{9ZVz;Fww_c+a-kJ;l6XJ0K(56p2Q^oUc zox%xtDA8$YPNqi%FU!uZ!v^eZFPGfw+)qMvw&IzV^GKDB*DLHjydw5e?U#;%f&+6H zP+W&y!A-+OYW94$5qyU)4|n9emE&f7g4=L#{1h0Thpy^&xm9Eus!~oItr+^ldi9#T zf)~a3vh%=dzS)1?i^-yOZ&r}I^Uhf{ii5{L_CGLiMKptp<}Ldiy3`d4-I!LELN>0UrjPXl|f|Tghdx9fY>}$U*S-;Jx#@ zC!WJ<5)$E~u{+88h*yOHgPL(A zAD0kEC>Wmqc}qj5=SAuu-4gA_9liYy=!aME$@{SS?te{CN0?JIPLZQzctT9p=Oj}v zkNpQsxR}I;V;H_48_opb0|;mXx?F3MoZWp!017p05oa@;m+C}Wm%3jNWd%k)IG*Ky zUoFY>M_2>`vJ1ZR?8j!@{k#Pt&4z}Rymdd|SuNC#LNkm9m#%um1^W1pT$!e}vFgmZ zTg9r%&6-ydxPMi@e&|7Hl_Y-2o<`h-yW26_@lI_>yLpx=`(ksx(O~TUoQq^p=1Q(f zRlos^hdV0 za766pT_ad*uF6B1>PG>!`tN=9lkSibe(9#%kYnM+gIY#g(gpIL8{0^1cXz^u5H$ngDx7)HIHG36-5~M@$CRyoEK?^!w z#sOLHC(j&s`vB;9Hub-tCo|k$VkD{%BT+@~kw5!;%bHSIdDskWy6L;}1;a!Cgf74s zXWp#|<>@>-q5mje!ZqyzWk+boN}@-0N-~%y;*Mp^5i4 zIOwBra*xsOm9wqKq(5vF5iTh_)nh;X-P2z!VUk|S5|ojhc9bo0I^u9;gkUYLW5r=Rjg>Y?BmRl72O>7M|J`LicBkK45=-GSk7OF~CyH z-wb>FN^oMqVW9Zca)R}J_$_|;bVXgn@)6bHY{6x6!fBO+cg5%^WiAIm9h%F0pxpR4 zc{Inq++{I42%RljDS9MM>|Z1_+u}PC-w3OEUx$+WVP`2Et%#aO$MHnk3(@(>(5H9X zlZVa^vXAKfrX0FC{+ca2Nl7mEieTNtoCAjIbzz{`yX}X^SnNX&|A85uVmx6v#m$JI zv9~Eog8&d{i?;~m@Q7v{ zetJJ#fJ&s!eWM86XK*v6D=yLf2G zKrv^qbtCn3Dy+pvXXQ!qN7975#t#Y$q3CIH*wf$58j+=%ch=|X__!=>GwOF#4kN-- zz&W#6s=T}D&MVsI@RPVY-oLJ-`=@E|2fz9uG2;RBUAVG1+=aJ5^eHD9!GHEDC}lUG z^9(R(A5Y{uaTwYn{x!o1ZB>I_Ob!=I272sGy-fYJC=HKl3MDK!%m043NtUmZ8y+9r zAth-UHtjmnYH0hXomO(1*B0RG0{LI}XKU%a><6!y-4xV#C2e^$HYf_60Rk+&(`3!pQN z2BPZFJ(!K3Gr7nu-&^b9>?uKO?^SHm;95M`{O;JbJ7@ZZUA}v8Tb`BRM@{?}aMVcK zH18^vWD6pknVc!Z5sq;qj}<$auM}B(HqF;NqSuT9sgseT@2r-a7qvW2zzcF-c1n?? zPL_VsK;U1b69Cy#7X7iU9DrcmNXKE$jo&Q|^?t)U(gX9Zve0&b53!5sRb%uJyy zyYJp|c^=!FV_c+d|Gp)G@EP$^?_Dm$1#e#7q(WBj zU6f9bYLA-T7v)MIEPJc1{#d!goho9EF^{SDi$@Paiptc&Uy3zqrWT!-`9;GR?c5I~ zGMW#(VdO%tMMO-}E{yR#rz#%C_z!L6CJfW;x`c${8qK1J@NjcalBn9Mf(w`1di4zn zd&UwgeC3LT$iHs-7}3+2dQ@XBg{Gx|A3P2k8RK=jLN`Z=;^cOBEv8uSC1%GOvtRag z(qL?sQi`Uy3i3;gQ%OTP&4>STnul1jEGs7unyx)qOxHyoV<=80shk}g$8y=WhZwRL zBYqaOdMBWM0qpCy*WwD{4bC9a?`aO_UCFhiZac13G&W!NBc*&z2|+?#`iKx z%f&&^4P+L2%s5`;+QSZnWWUWQ%}^LE68tx$Mmr;5?5e)H{@dvU)2|lh{g8>RL?qsV-&;r}y0A^U^Ac!5FPLl;q@H`54^YUhX`Yumg+3zK z{0o2vjUiXDJ68s~o^99z4`hJ6m1NS~@aZy$q}U$dZ!kgpTP%^GF}aVBjK{-=y_N@m zqsOC1PmS}E3YF%Tj2z(?no-X%_1O_ZBm=ZNkSC%4& zJ7jz!%`*E47Iw0v>(nArwQW-@EmuLHIAdBW7xp5yAit-7u;!tX^kqG>S72+@9CABS z0Ctt1HA$`XiXq=j<&RfRHlLJJ<@B<)QE4*5&a$Vhnqvq0=)9k4Uid6&mk34!H_=dw zZK3WBAB#4#^hCuwkp?k+?CTd0(g%s>w=^0zsHYZFjOxvy){v^(627AYDR};`5A`>w z$or!xT;Rd&GdzVOf0|~k{+Pu*);dKg{ePhwPYhJhM_CckaMA(X}M}!{vSZj zT#E9=s~AtP-B-TZJ;5I*83KgL_qOJOLo?hrX?SDT&W%HVrVnsne#CABv{cOmRU%|~ zcp7MkzceZd0yGX;{3`S z@M|@*L!!%0z!Rc_kiD7c=6gBiT~4xr(myBOgUFJD-flb-&1k&oPi~ysVwLT@JNHuh znYEutn+_#&Q%3ne@Otu!+d+5HVcPTpr{gM^YU`ph6wlk?rv&8qhI7xFd}93ObvQx@ zmpmD2&M%^AdzR%Llg32Bm-a*^ueSK)h&c|N)rY=m0sy5dMNi}4^TrDX+_bY5Kmbij zIBDY8!5e6VI4Z^?NnQ7xZiKKEXrYG z##|rVj-*uLuGhH`SFz}i%#=7&pA)9$!Zl3GJQ{{`1hJUnK?Ku>?XhE(8s7J#s<%sU z>ndN!wbRQ6`)C>Fi+=7ep_K`P8<5-J{|uZF5j7DQ2( z5Olg&koj~?fXfwYZs^&SRcqBbwjW3MS1C%)&A{`FI#>UO&-b1ReFNg1f3#b31@aI> z5VJY;eYMO@3icw#yL4dv)c6&FV1Da?TcTBCfH{A1Sp=M+Ldo#EKZh;4^-hR+QRJ|g zHv~F{pj~@#`|CDHc<~5^a%tq5ssT^7gBq0jEpzNg09$CEAzg);q>g}aX9~77;PLSA z4)4yEk2pWqGPpl`SQ&;zF0@PGlH=a*>3nn}bDS_(08FnQ&xwK-hP}PQ8@9hk+E0v% zQ|vvgF+B)+HSgy4?_gtiD#zARM_g-N0ph1-4N>o;i|TnR+v$W?h|c@fyboDH^r??#w7zfGCfOcY`%uj#C_iX9jKjT&R{?$k zubUIaA=fdl#^JxPAYw*_>zH>w3Dksg_% z|MPxL)$mrg2jD|uwu{iF{}R~w+nLzlCYy@qU82iVt|yb%34RRzbt+PBC7n9cJl=G9 znnE)!usqCT9yjkv@EE*izO<-(Kl!wwVifoqQmRo=14#327XM8dmu3q~ueC31fnVLB zy&ticWJwHp%2Q8X9kOVSFZckP6awV#B1#{5|-4fGS!1^MPfUi$`*2Sc~;dP$PFx%44*ia8-_G+musI=S|0b!4xbZ093g_?D9 zP5VYAcA*4J#UbR^r9$!7X6cr2)`U^ia=w|dXay7wv>3s<(jUB@cV!7M^wKAi9(SlX zYj)>a@-3RFydKbpXo(j)xbO28$Nh4cFK+33M~9pEB%0h`dg0?@$M19bIwI%<1WQRE2%RAJ6>MH|61)U1S_i?3_A}vIM)Eoy@j=8IdOj|;$&+;J&~}J zF)@v&g!s4+FLyrceJ?@s!}+L-!y%I!-^0XfCHlbtm8fvrw!U_2Biz=rE9iES&PtQ% z>y3x^gReV4uT@?kLk_P}T%6r9}v52zwjIysJtn zZpXsAD)N4#gDG1d9}olA>C*Qww101~;u?9@^1bU66TMPZz_iZ&?m;G^!L;NdCvB`5 zK2&uD%Fl3gp&lc=Mt3qoCWYMRZGU-rG_!$7i}E6)8EZb(HQw7A0_bMD?uT#O7mY*- zWx%{n5UWFOUU8?E6S22W5<2k$;c0K}l>~~JUcNiMf_%^PoLY8p7=NMmlSoC04yUayNiGLD3?{Jg*AmYh?L1o{(^y7B~6N>z;qf z0d$gkNmn*|MxTD}&^OaAWXBkkLyE@le<{Ijp^Upis3e)vcMN2eXi*3vs4ydagny?9V5!X z>8-Ayq!{((juD^?$kG>jT4d+=-nP&~r}=>EP{TQp!q#{!0CZH7;(poJl!NWAj}I9T zgJhzR!RL^_5L^;L5BT1^rR>c%DnT%&xE3_$3}_GB;9^f!ZNCQyoYSsefTPpw2Z?Kp z*S8SmFJPs_7k47n$vmCsF9_av1XDv_z*c0aieD{=14peb8zc6dnt^wKE#|Zy$uD1P zIM#H8@r~Us`QuwkTScs6ao$0AYyk??3rHDqblWh|-^zA`(A6#F~=NZEDii1AMbV70Xi-wM!5wp3Ezjv8h|bdiPWFPhZvzI zYx!J4I(#mv#DI8^JZ4499+NeCiA6P8V%}rTR`6I*2loc_(w=+LS1r)*EKmBwUS4~aUgG!+FlZZ+=v)nt-Q_~+3l5tKWLCq zXzhd|>F~ibX;kb{ANUNZSN_@KF`BjT2ObzG_8%1Vn6PYZHe#o&=QSRg#mvm}SkBbY zb#VxDZgvYD&r0oBWnWR;tnm{ERhqIGQKdgEeER;7rk4Na)IO~84sCmnrenD+@~Qf-)TmokUI}21nyKD}*u0mlg5Y{Ij!M{e zY0>upqYMb+8)+}2KESI-SffE*zj~D|WX%OoGpW6w`5*b=%>|U3!L9EaJYt|yH+l;7 zwmiuGK^Cn%{f0Gf^Hd6$!*nw?NII>N*st0Xpgk;2XD`?I1fn=Cvflz!;ez;49^Xa$ zF`md4$U-Fc@tL96L6zB{xHsYeP$3|LFXg`lR@omvT+f zP8HH^AUyz_^ZolJcoXlF#$BMz=Ze*SR9FgY)N%>zb26Sprsvrr?`O)*LfmJ1hF^LK zn&xWa!6vSbcj2bcm2V%MKP@$$r{tW+n``4g@x&R(#KxLc-o>uEr=Mx)^kyR+#z&b{ z&wtC1r%R__zVIr=p60|*S-pPdF&rP~YPli-9F~VK9;2AOS{t!v*Kv=}A5{dvmSdfw z=YN20Ksjq@-!5tf5kcsBVTFETL%}mi=C6@jk@!h>X=(chqZ=nS!IJ$C(t0<$`K4g% z8TF7i1%c~rv8+$LB#Ro2%_LYR6LeIhDAE0|NKHN8V@OG`(@EA?#Z2h2L@O!kTIcFL z@9<46hMXUKQtZc;d}K<=qtPeWG>YwmtVs$}20ZYszdZ10eW*hln;x|%#23_}8V|4Q zfisa(o23U|wu>KspWRm|^_a|1VmsW5US*Scs`3>bfkTr&^_=4=Sg9x1dMP4s=wTX% zz&4#z@#{`-Y7Hv3>=F>$%ADIi`v<}AMr^f@VZ-^m7gLM#=IPH&d;$h^pV#=)`u8FU z3az)XdisXgD9%DW9^vWxZbj&Ph-GDTeXL36`O=MPthw}Z96pj*c;P25AAjOt#8dC+uRsSzeNMUa{>2zY@N)pknrIroaqErl-dfz zxDyl?<;EYwBhqw3lEoALXyQNbX_cDQWany^N+94k5U!5)#irVzYoggFp#|K;{Om_q z0NB|@{_q20O@)K`a#g`%k>`c6e_2B`im@zHidA0>J-m<~vl(ZdZ1tM~nRJ9TKi4sp z&>HD)5f9n1L;&XmUWDzr5fAuAKPH{v%?=YsWuZZbqsE`#+2wzh}E&hW&8y9uG~LS+~< zYa-E*x`73R#F28L0uHWk_~Pu-PCD=ZY`k>W zw4k2tYUw=MF{_onwH_EIJU3t3e^G4ImNMyNCmYOBIehnb@me|E@LXg-G=m#SLKI6O zhcwif-n>94j_T_h|28qEOU`>QeSRAvMv3+Nrru47ZVe?Dsf;`8-SC%+;$qZLNkCGp zE71E7vB?C8&7jjmpoW?LsIBtH+ITbTv92J3}6^ zV!<~PQ3{Nn3p0+KR1Ydv-d%Xc~hMFAD9CKsmD&77k}BN$`r}Pxnzk9DfDgyl36qz4Z9I zf;4!64gQKlotKO&T}ZiAx4Mi_y!r37)zgOJ%k(fLXhdU*3?^F+98ZaUCt<4gsD54T z@s3Q(>qQueww|iftAG;ySJal0H{R+;-anH;lgdvc8${`NBqHmgdF!=w(RTnv_5h-v zG*g?bqLa7pR?42-u58nNmYjuTIWnWVwC(0%T-n+Dhj4T$OAjQ<%~TuQlC!#qopORZ zQrD^s^+=~=a^Ic|Frol(Z=LIhcCeoChoSgU3+{qwha}9`4QjqsTt1{ znbDIGkQ3zep&ueMRA*;x!e~J6I1TVxM!>`-XG$G9G@b6tl8H|Mc87i4LcVPo!b z?tOFjKktCT{W)OA#>l!dr2s6pc>^I3@)6IW^wU=LXNU}R#~3P+$4}@v zs|wh{Uy?+1QuBCm3YmN5^W=&j!o9-v;|KL)%)Ghv&@$@lxb-M*(cyoai{K?nJ9U*1 zOM0j!=VMcX-@q*Q&VY2_k`X`jjh<+b#}wNaKPj8#qvnT;II=lp;$AujG6FuU*exva z-%SY&yqoymuufDC;4|yZ&g&wyGPRoy3re zhj9ZCSmwwW0eU-})d(ombKwFOw;%3qnJ~r0H@spvZ654KcqbqISRJI*&UpID^JcB8 zmE#79^q4V)v=}(h>AgRnKL&O#ZqYYBqZn7)_~@AivAf3eBhqESdW5CSc$c<27@>BF zkTKx*e%UaT2z0shIm9D!+1neL7%H`3@uzr+k%+supc#bnHhGp|l9*4EwXA9z#%a#9lFt{uXsTm~iq$Y5YH1r4F*sTt3>8lUy z&2+Y5C~o_iu)@WJS;&}s?uZjI*2Nv+4VVGBl*pX7={^rkky`R^;d*}Rxg`2dT-9dw zPTy2o0OqH1h-4Vll@`6GsLAq(_cjB1lia9{d896u;_(?eJ$WI*YoPnbYrp~)nR!>> zUmE}8GT1_)Zi}7tGuKIz!Hj68Gs*}tPfO)>ARW(}iRn?$St{)wq^k-AC7(8d0A*Rv zPfGBFR<`0lqh_4wZOxnVsM`IaJgM1;o!K|z*j+H8-zPMz2OOyZBUIL~gdK6o9wQ`W z4(o!vnc2KKFFuRRAj!Z?-*R$bOYpN`^}U={RP1-aBDM%T{4Ax=YWM!fc?V1Y6PR4b z-wKa^6JgcOvZ}k2|JLc}Teg3p8@#_)ppNA6)JmyS2i+iXL>&0t`2Rcbb%Yv z1c8-i{L$XqBDoA*8*Xr@Z6w_P4onH*&O>WNZzgYn7%uUd{Y2C|m zf1G<2d8Gd@C|Dqel(N`pcfE6`3uULvgOSFQ{AQu(zLMYV#Xx8Yd)u9b7c9YHo3j>q zSE|>{m30iHM)KXWMu&DNrYyBz0)7w8eSTVzq9};X50L%>d+xoL<5`IBo72 zDXND<;p{{QQX zfGK(0;1q}Pz8466WVj2T^>g>${L>k&{y#dS7)r6)zdNEuWi;ulCahqvTzO9L!rLdr zi$0+Wh%j7{zcVv{ffr%rK^P{FVi><=_WS0|$NDLDA@(b%V=Fr|#5SyOKRx+MjcCbd zGoD&syh_x(l$Mr)hPQm% zA{}NHW-3u=-(OMN%dtviur97NOSwR(ye}**9&+P5$f9!ZbPVq7OcO*r$r7Uv(fvp= z0wnHYFX(U|GRD#L1M>Q7AY#BH^mRK*d%;`wmPU{bq+dj9C zAFmyd-?+1C7JmgQ#K`V5RFcMCfuBmqn$2e%&DXv126k#a=(6J}Rvh^`fc00XlVhdY1vvFpPO% z$r2Ul&0NRLE=}Q0Pxr+>*Oot7Vh=jkOEPQwo}bDs4L8` zFetWUJx~3og9qT0CMv?v`UeGGn|ZCAwe`r(1>(nLjFt?bS*)$ODDud!b*_zd|98)$ zj7t4}*MXqvf>9DkzkPJo9&%#3@Y`_*$mMp8Fz=0Pd^byhAr2wT((EAz!p#xeC%Y24 zl4{)fmk@Q5~s1qc8@26l>qV2_OK;f8cLH1jneU-&|jy)|= z%9(3cmkkTIuyJBqX$c@*BeYDIN4NopK;!#q>vNucxX^$3Jn4FMi0(BAZh+UWngp-g zrq=(bJ^b2RPTg#=pnJ~N!gnbwouC&rMxKzxY%h;mh|mfa4kUl&!H>l_|BlQ7*;Tft z2zs_Ddjl^3u`_HNCr^0#Nb{(*>%%F&rHalsp9XAP@!8RfV%Ff@r-(Ku{kP3&VIbNZ zPRYc7wYggxb&14QE*(9SZ6%abZO6skTc__RlO`Dp8fB-t=c*^WA7PeJhY;9KL|@Xp zyEIH1#VZ^3IK#s1_Sy}0e~vV@en@s0yyv%EH%XrUVLT%c*Er>ysho1WP|K%4o(tGc*PFs7p=fF$a|U zO8N|^hJQfIw;C_{OnE-M?%%F_w>TrCMRr8CuE}Uaa(!#V`10;MC=lUwhH#C{6j9_z zG$6?N!Z5TfN;0&OA>e~&yIGdbz_BeOx|@8h{}dqGe^Czr*>X&`Y*|oOE?HJ6_`JqI zT1ciad#|aX@R=*K#Xf1#(0M@g1hLgPnHYTq$g~LHm<^gs`KW9dZQ_D+_|oU(rSMy9 zgaZy(-x~k4Hw{&24 z94qT}z%-G7cV+Ei@clp-j^_N<=ufBWBgfKJXMNT1TJsDoRydOpdW#{`~+j}sR=s(e>!DmT+N z-&GDZ-6zak6hE*klI~eICC>i$InJg%wl5?f+W#@O@v!eV&hFS9jHhJn5z(#E14Ucf zFj%eYhB79Ks`{7cN8Riat6dGT#oIjI9u&vX8Saz#HJB28g5aw^h8Z@BCMr!jF?`!P zUtoH&%XEq|jglMa{M|le=bGeom}hH(sZnbivX#Nw!P>#m%|jX^W;nVv6{RlG>&F+i z@2)fkX)g2!Ui zCQCI_NS9PlztKO5s` zMA`sHMnszQQ{e>SM~FTh1r)&|k0a*Tc}7Ewb;WWxU0-(vCfe03Ynd6@xl9U`9e$OxqiLd+K5w5TKFOjTN*O<>=8^}jP;mkEwxr|EDN&gzRCrMhxAuPxu-SZLOrtOfiPfESiQ0QF5&-;|uqE=t2a2L~>2)}%`;yt6=#CTPBB|XrtBgYQ; zeltp3F9-~{-=y*dE4lbGX?NX`Ja4solydROkcqMB?DUr1mDU;_={yV^mficg;AT1@ z!0Xl{(vLA;euwv2UX;Xw)k1Eb9Zc$YhdozGlNu8(BaZeo?OI(N^S#-^rzqaxWa$iN zUO-hW147bMOH|=NWJbm{zT0_k+!3*mu%2?}*gh_M0X7zY^BLTVXf}uc(tT5-`Mo_r0|L(ct zTSdz-vy=mheCmWmYlG|*vlLm1x_Je_PJg`Bx*zgVSPJS9*|@pmOxOP<7M(#G5!Z7I zqTi=}(qRfxJBB1Qk21F63UFC`P0j7zFm41B&tj{SZvG?y84rqmH&lKh2gOl(vp>k8 z%$BGIIsHqmP%FN*CRg5z(Q?zT?s{VzW|+4HluK>!o6!w|)>kg9>6RO*zQJl7yQ6F$ zttcdi_Yk5g(8Y?*M%AWXJx%{ja{QvL6KhEF&^5!(t?LzoN5ir46XSF7M!s@on80Ri zmxHH)Y5ff_uu+?HD0F;PuW^Q_T`Vm{ZCB84IZWJjgNeDqSb0P(3S_S=7iS_5ZeJ3H zQ8+2`xi#gC1~S~$j^+|TUPhr+HtC4W*G~6g1k9HSQJ2L>fP+O*1;a5LeO8Xxy9$3~ zb9wQcsT4Qo(}8F5#QsF~Ae}Kx>MpidT|V_Hk>9{p82sXpM7e>`&q_;Qoju)mYwj$) zDj;JzMWV)4n&XhMtYf-gs)3F?H{m~EoBeE4H0^uQmL$PhL_V0tU!Lww3w$Zt8U>VR z$_5FWWlZD+n_~!X*ES)}!=ahwzJ;l#(ZcfU(0AcdQTU9xnb+N5*`=swcMZjV7KY1x zV{nZvYm(qG1(0%10=050I{e2on^jA8$R?h2Ty3mP&xMcL`V^n zg4t)_U$2@&%nH7ZdL76=$~=#cWKL>hqSV%v&W zNqlX3VnXE{X<~If9zp>YToDj1TkVzzRV{c zV5$0|hEe;R`}HQT$w~1#A&WbzS5^{DE{A8YMKs9lG2x|AW9R*RV(76-O;)f=Rj-;M z@-MW4J)O=mbHjn72DRrvXl9NzQwKDp4)EXKGWQv|SR+29)#hB=u2BiYM+ zyODVKVyeE~s-5t&J?0|w)YS!kIawn6{IRZrImI@K5plm)nMLQzhofBxzEuAgUy6s1 zwm=%feG788RBrt;&C||4H7+5sKJ)z5pE;lEn+Do|4SKze|ET{1PCtrgWlr zXswL9MGsIp*0CWoVvH46YS;Auq+lsm7d`S~UT|wiB`c4V7g^25DG8(5;CUxh>V!Ez zJ4f+q&{p)EKM>N}0h^MX1AFh5I15Z@Q1>M*N0h33D>_i7{&n(CpAnpWw9WDSREpyg zV1%9<>4!AZlYA9e*HHB~uE6uZp^3iU5A@~ja>ssc+emw2hnjSvPwV$0+dYZcA`*e0 zj9$#y$dg{YwI$L0O!t9Z;P-|We7%|cuDAK|W_9Lu!b-e%V=?4VQPYrbRfDy8YCtE} z^Y1HGE)8T7sPD0kV0>$lcJ#H2sEu@OeNpFQ0sNhW#lQCFp)c&ZG_ypQb;q4~l(D0K zQ2}qC$vz>rkaZKcSd-(_IBoYOq*z90igpuz4H!p1AIr5$I=6}RS|cSP{<#*ODkrfl zuNjHyEvZ)LkQ(YTF~5L0t#TmxIq`rf3I*^!1PQ3BwLSJ!HXur+A7#S`?nXO^Y>xB> zW~vv`sEW}6J*C@%1@-DIrD|T2c)`;sO30y~-z|=~v~bKl$HJejaL$NM`Lun=)`q;} zTDv*;{*uw_+ou@`yWb;CaKkvhqSjpW+wY#CwA^oi zkDjq;AWVT;jqx!J88_`UDo)Rvt4Hs>;DogAHGdOkziIZe5&+mH&JbSQhm&{{QQPTW z&f6tBhNbBd(d+wT@o1NEAXA7@z^qjG-Hw->mo_{t1JW+j<0$gH)!gJqy3aPkNwZz_ zb*PEE*9?F6Q&>9{J(FK@mOL&W%brb)&GF)83YS>#5C5|F-pROYd{k(rqybt}r*+ql zbOtetLGh?6iE`j%==W2=#?aFM@Z?dx5P&B?P(RqsE1-NPH2o$L^?gh7=^xtYSNpZp zE~Fcp`LT*rYPmD;Wrs#k>x+N@K??24x&&j=-=LH3J8In=}3f{2A)s^L_i8S1TvrNcj-Cq@{Z`b}(O?BL znMfC;^C2hoLIr*5VP$QEWM@h)uC2con~ItgoLK&C_}OZg19i(8JDj(sJx% za5RG;~B=l z7vDU+RX;C#g{Sb%M~3aG&ej0!3$~&A5VBOupqI1gT6d7=IJ(xP%~Fn0`;a>g=g9Ke z?C3b9M~Rbri+|z^)Q>sA#YUKQO4P$j-KGr6iAK~1`?H>(moBo|-TD0fiz(!8D5`KG zl)d~qN^7Z1^vjc}@&N!od53MNttZImDjxlp^Bl%QSWcT%t|Ba!;ZwXGbvVt%_bB%~ z?;9@ir`Mkpno;RrOxI&y2VPqZZk1n{R-urZQ%Fl9Rx$m(&*cY1)wt_IIC4WFG3t|h z_qqeks>&E7Wcl7E&TuheTJlD$qJ>O2t%k?5?NNUEEGLID;j`6fS&hYjgegVjoar z&KlI|t2NqRRoMD)$?m=^3*$YXToCZ`Z@BFF56~8{n0fuqe&jXB=FdUu`1R*El9s3w zo_TKeFZK+XVT~vb+<4P7F!t5<+SsdPRA{MhVGl?ByBXaLAPtL?me71CkLs6;{Jl9} z9d+(Y2+!Vast}$Udx*#r5I+fa+wdY)uSqRX#*v{oSpKL7gye4m!s)~rCVak1=Z)6O zXkh!yX)`?$LY|MZ^&HGtcY0NvSY=&#MHEn6*lY*3Eu2+Nzr8W>KxM}nGaY4RJ1m`CJslur|ZgOU_+x&z7>wlc; z_1+d&GH9hNrrWw)mosdNKl^s>gaKs^9%Ph7Yer?md66Q*stDF-A=C+=VJ|*kMC*;! z)9hEFTyVd$xhU>Qav!&r0U55zXltOIxLg07)`M!EIOKzVW$nwAjuP-<;8 z*?{^MVN;+#ZmLW}(y1ip$862r;>1~t%B!+qqN*_i{^?c65M%J_H3wyK3PPzChxYaQ z!j>)+*XJ%Z;12RcKmT~M;nFy=@FMu^)T7j~(7sf+bRehNF=HZsx_oAmYmFw29U>gk z5;$q)zUgrB{gzX<8qE1L^|qCT>UW+vG9SZxD|r7}2ve7MISfa{k8)3O@rgah5dRST zbiac@q23*QJ;%7vS+qSF&j|44-9y7>E(hbCLe%OE#lZ>Csp)*tH*{|(<2FAOgq`M4w8{52ahZyHlC62?qr-p$OU&1Nu6kRqM zN)4=$tL8l_{;P(mRq>z1^fPBCX(w>^T75wlx^8!SV?3KMpM6+I1!kD zKixdbXY8j(H%XauH6XOxJ(vP;M7uNNs!Zvh!?HAOi+r}m=k1wz@I~a{G#5^7JJ9^F zU%I9(XFK~Bv`5_hL)$LVmx9-qB1}`)IvCINire6_(m@4y3f(bdB{<==Imt%M2{Jya*#KQe2bPQCa<)_;<#{0$&)CQS# zH&Ln=Lj#-c>j4lWiRf7j_?ANe&`F@GPe$IECA0e$6(NvR)!w{1Psqt&ZDJ!%2=olU zByFU)1xzOIfBf#%y<`zd<7I$Ia>c|Yh5{4mc}3z2OXyDdz6N|E1FZ7?RSH(q%TVTm zkwjLMM%a3MDi$hNEOe6obm#Xv2hiH<+bEt?F>#Wu+L;LVIqcipsU3k|XXFh3HSy@fYW)gL zPM8cwh~xZ6EV)C~wnmLSqQ1}5S^>;l1WR6olt%3rQYWNgf$&Bh(B3FW5KLr!FH+tb ziWoFRxOV>wg^?!?PHKUv1bUQ+U*Xa@B_5<(kr~z2XNiWr`!GPS;;lC%{~~}=S#@*p z^LLykbhh2z8;o9q{s6)|N=i*&cb9AsRw(QhfQvE&s{hBb7V+rk1RJv{bm5;%bw`5(as^mPkfNO^LJj04}neMG%?MU&1S^R**o6x z{BZZD*H4(h@?=MHVHknC_$fH*(`FTF!+* zx7ziW0$r{$?DQj7$L=JfvL3h0d)cIJF-EBQtIB?uedbfaR&@pF;7U2Zrz(rVGDY%q z0%=+lYAw>?Zk4<9oS(^wQKD%kTz-{&axfaIq)aQ=WxqV$` zR*lb-l5COfzl0tEp2Wi7f&BQcu}Q19jOzK`{W*y$Da z-&FrxEW5_F_TbA7?Qc9!(i*Z)ppbb=0;>eav+88x zdN#6$uuu|S26<)`cpH?@7KGw?;ppQ^pYCH_Oophd3|F&wsDtP+7C%W)@;z~^CZ-10 z49ca{rgWkty4TP9bzgW*=(4?}Yz24_*65@q^80|eX-lmR*h@Km7KxxW0{PCmMs7|v z?^%RZO8;DoG*C0~G17g6P4x^68dx<7nn1E)XZr-gbW~z7wi*3k3WSbPCdlQN5&J51 zuoyuCn3)%EH*LgGaS}PiP|`4y?aK;3;_rGBpe|*!;dNwvVW2VCuLGAk>#yOOlg_vJ z;|eu7tNpBCsU_eSbs^mY{qC7TYOGrN#h5*x^);El&BS5My)4TgRMuw|%?0VhRuAB} z_+rj*tww*az{wYU|AQ^eRkUUfccQ=~=k6O=po*fR4c-*mb?2vNN2OO4_Ty4?@MUyc z@4LO7mccUbJJR)U;JD&!bG$j7_T5+winuAEgNzBU_=u0E!(EP0Pe88jena6F=p-@C z=?E+IfTer(nV|VB!q*JqtJJ2nopbEz7;U!-I&!zaeYvU=T4MI%w6wDuNXMK4(N@XZ zts*r)g-&Osx(WT)-rgj11ff=32pe_1{!|URZ`gqH67NYr-y(c$9&VNpgxWJvPoz{Gf;Q)Wm!quQ>@jr3d{+&%{57C_-yDXAjGsdql37Cy=hF9*iiapoBegZ-~;a7{$yI`>yO;dz8j z69iK9kSl!(A6WTi;cOtgdPhkUjXLzO%Kia4s(QFFASGU{Gsu%#-@Bc=X&05o2tF+X z*XiQ>=ER>^U1++_-LoIgpCq2A!7zY9RugoqnLZ&|2c~U@E0u$)3!)+t24JmO!0bvpdi)X>o@ zE6pcqQ=Dw!b-PSxSiAt0iYw>1Wc*kRl}yK_rgH9$?n`R{knT4L;@Iuuc`WTlv<|*B zY~mk_jxQZ6RLezgdoe(`TG2`}aF+iDbw=@AB{p3un=Pw7PIEPaW-5h8?M8 z4oaeMF(}V%)-(H~2Vve7Z(9Mv3yrI^k7hRj%X=;vseWATPl2!G{)fNP=L^eVxdc6=eiFVpfhO2X7tRoEsfMsHsi$Yu~H< zUX3EdcW;z78&uW0XGsjO(!M6uW6u*eg%PEYm&f7u5GPdl5!-=9f8W~VV;!#b<4GZ_ zj{lf2^M$sk6AlGh9?rqS&Br2HMlIbipiVwI240VZroBnyLlFwT^vs;k@ZLH_uXDqb zu^vwb_o2*;n(iE}@_PphDvnLM?puod7AI3DHr5^sCQ(~ z$*sHV8-;7)nyX#e0zVL`1wq_ZE7hY2YxvL0DL16{+h~cy*`mTv>FmX9&gc)6f@XZA z#R?!FJl@)sJ0x(kP;59A7X4WZBTPwk3~zk@QYc235r=~;Z!JfP9DfTp&GNpiPZA!#; z`dipu?YVPP6eNlCQ2)0lM-x5KRI1KLJy86PTh6_iz}g00QOpq%PU$p2dUl ze;g&e)xDsNeR0``8-%%Cs5l7Em7f5&ohw9M1##QQwx4gf_9Br|nmAZ1l1p)b* zbY*eUw2(A;ke;JiS_Z;qEaN7Vug_$*e}uo*3a3KJEt|*O?w1wmu{AVY*zU3x9(Wga zOkpx}G3^Yj#!qH0d(q3ex=$~dWmHHhnkY$Kxl`}gGyg(z-#Qh==vB%5=)&4K@3z&m zsX{B8CFuzX@@BgF-F%6UOn+SU@?`4xlRy{@X`Zj=Gkk4=6YS9CMI21 z5<9Hw7AM;p4@@dcLb+z~k59SC({F6I)S^R}h z>rePMmzx4EBdp2T=sA_RzW@BtRL_Y|hV}HiDfcA*R|X0lPvo2Vv*e%X`)zsWe0TiU z5$6$&DCensQPvAC0(BR-*_pCaAIa?8K@;QMAW?^uy{hD-v^5BDHm^MjNeEkFG(3uN zc5uR;upCS-l5%tY^Amp0l&F=`9ob8LrB!z-W=|PQWf!u5vMfcu;z;Ekf*h!w%tq@| zk17k;+z*WH9RSN53ux)OPe)W*QwxiY*yG&S@?HM`%<>juR-9wlF_&kB*dMWQlG8i7 zc_CuiCzMV*{Kg|L=tKzGm$H^%0*toe{kR}K)CxmfcQ5(I@hV|m^$yvln~0McN>31= z7&d>>k`_=W?C}(iawCKrpSG-ulQ8i;w#;>HXW-kV{42zY9=WjP*w)80`@^L*)B3#7 zTo9rWu33=g@&GbV=Z9+>5v(n8Id?LtRwF(G?*%G(;W42F@}96@Lal`F<@u&x;u4w8 zXJfLAJc)ch5OytY;whM>Ri7W=N2fLVqt=ff;P5y2oCV@F1|?6CCrIbLemwp7P-;sd zrzGkG=;Yt0q(7MeRK&hx$dZoMZIQKF@x{OF(mb9*C>L1nOkX6v?x$&??JPK2@U6OC zT?w%CvSV4zIm-%<;wrSFeuz3zeAgQZeQW`nw!!iyeVU2=KD3& zIP4^ocUwx)>6uwwzV_FTkmrG0Wdif>ZQaWv#oP=8ySQ z^s~zC7c0}(SI+jMKR;%>J~kCgSu8&qj-Lt?xm2WbYoiY=uRV)bi)O5t9n!wbj}xV4 ztX?b=+Y-ckptJ-hP{8@D)uw2++r)4$V=N!i5o7_kiUtp(sFGKVXa8D7>D_306a|8e zs}qm}5K4ZYH%A-6opfr9qCw{nhL30!C4PmL=dw445S**Wl!1ndIWJ~puvv)qj8|ir z@mkO7juS@4D=SZ}Mn|nY$1)6T#>xpJvUH*tV>Rb|voc7sI2N`Kvk~F`Pb|81q=*jU z#EPgWk)=ChAwBHA_F1Id7K&fVv8{bAr*br$0_yPmUy_fwR-x9es|gm$o-QNUEny_N z`#yx6Gx@ArM$QPS*&F16Iw~`ej@n~O$urqpUJc(`>DA>APd>2UYfg3qk-hvK$smDe zGBuD!=WaVbuhz7PA1QM+qwM*_4y+9KoC-$|5j2RKrPaeED1L%{_XJ2oV|!5)dll4s z@oMJ%b)zUW@esv%P$`@_?d(sov@L=AGMFY5T^jZRPVe=sWh5_`SkEj+vrcqXFR_}a z1>V3XJ{)R+Uz69!WZe<>v0>_t!Ne64lZ7L z=4(Hq?fmrsHuLPhh%|~%XDWFz)9I`UTfl5T0!-4q6-D`HWtOnN$Km2_?NQ|-l_ixu zaZH^crn@zWcnSy@SA~cw*Z=4!&byV4HEvDoG-Lm83l$OL<;Y}t0Fa~ zkw@DA{kE-SWNU4-E#&Q`?Fg~FvmqF+OF*MuvL<3Mp{$2gvJZU(l@ZUU z>4vhjlY?*X+E~2|Cq)vJ??4h#b|vaSRI>KkNnvhq%Vad^1#eJhrxp;t=Q(w9^Jjl2bR;F1_#Q z?7vq5b0J(o?EHMgYfkuTkI;V9koJNPJa6HtOkcAZ7aMB>P!k4$PK}4|NySK_4(XaV zZ3v8hwTQDI_8^2C=fl?omVP_ow|g)?86^L&y6Z}wZ89oL{kD*3*Md_@>UUnb&f`m^ zW%3jHzK7QowB$~R*M9uSz?K>xXGY`3G9}jVIuF0Z`;Yb5yZ*wdYnkt4O>Cl_EjUwC zn=l+h#9B+%<)=yz_L<)sIZZhlt%y&G1n(+YFOn!h5Jam*GnbZ@PmrW&Qf%iLvy?N- zMgsI~b#&)o@62cH1HbA4RK)U(>J|Hgr=&7EZ#FXHtAknhzZAY%49APKL=TZcAWqoW zy&(BEpHaydQ(MD_c-?y(E5c@8My5Y8K=I6^7$KP;KEy0S5zCXTHkdNxxqXcK=Dy*yB;@4jAA82j&gu4Pj%QW{sMDdXmRN796K?a=F0iR-p{La$T$ zZ7`!w6i8!UR!IAWte<|QO&V0R!ut4#lF`H-V`aAu0>5+7-1hS4MuPcp*+WWRMRJkk z8~2^b9j%35Ir^y;{rcH$BMm0liwWq|Za|XuAfJg`@ z=@u9%sh0W$ZSnlp28CT6Q}Gr#cKHt4w0{0*H8Jf#_hJzkrwo;3v&ZMdWG}JQ819u% zb7PSJn(+aErc|untR{fx zm9o|F5Sx{a!JE>8E%`zZktp;ps_30}#uG0y%jX5`+m$;`Y>BKjGUatF2r0qrDw9J{;j64MHZ*jaRm-Uacp_%h?dAW(fL4r3>Ei9=+owg+x!SVOiDqZj$qc z!(?R0%upyE@nHhc2zyZ)4J&+BpAY&P&>tcb@#w@)}g?rPEPe z!Jcd5V->~MiJIkhLmyl{5gGNRlXk?dwTpMyG8Cu7W0jqr-gXgN4wj)`_s0@{%_)Mu zBwn}+h=+2QcC0N2%HXW6x4J^J!nTdtN^xgzJ?82!DhyBDr7e#whjlIIU~EVz;`f-h z7-LBr2~v?^xUzoMnw@J_>E==v_sICk zrC6}q*YA9;HFKrCGJNIG4|jUK-6@`>?eREb^HzTV7imj{O}DV)A?|YEHf)pTAV~it zqcwDwZ_r392qDb{RfV!NnAPq?F`;r>7P+;Vi|->>?bjFD?j*{BDFitLpD7+g?Jn&P^7F)oYW^-8( z>u@~Zh)txfpEds3BxG}C*T`d5|6sryEZY<$e9iR2xbqEL3Ch?L7M9J z)Q<3g&~6=^Ny~Y-av7Wg)=AGoN*?UOPLQCMbK-rTv^}IUyiLUDs;nbZN zkFX`tCh(S}yE%6GekDd@Ugc9k35kGiY1o~I9*>%`?QFmea7zP-H<}|!-QPNa^S~bB zwK~3<*RHS`>yQUerBbRRKiYe9z-ue~6h5Q_ABw5IAd;Bau?wNOjHaC4=i)80U#L=b zv2#%?wd~ue$T3u8`r@v8jw-QNXsi@eU&qseLeCwaP~TpiOq}wDlSkTZ{kFPLFN(w$ z*HGpihcA1mbMl9QylF1@@(N{r_<7n9Sqqo)3612H7|0{>UK^9e$#D)R{njQG><`M& zhNcs(o*3b*eMw_ti*=j}b>O~U!lvKEcp;O_Uo)=}GMNQ`hxQ2v2EHKo!YRwHfTmIF zGQIN|47kD+GVDIaj=0?Mnk7Twf{2qNtNNJ3Z%fpjK&Oj3S*h>PSKfQ5H=3)8i^=n& z&}WO5sKn>lRB>i*Xrm_;6f4>`u@%;03|7cq$BiV-3;eZy1NlOZ6B=@E2;!`or<9`S z3&zz&ZS7Lc)sfc{ueYg3m-+M}2s4Me-zIIMW5}ymw_uQjvJ&5_h;s3=iLy2u80WNJy%UC(bCGDt77nOfwOVb`aqJCuR91cmi_`; z^0kS$FE009G@`MP@zWZ4!oZp84#T+|yg=A1eR{DlPR^WlBsrSvk+adkkkwu_*>UlD zd`$Idxi-zjYNBN;&pmo=?O+)ZsSm;J1HSS*fLdlQTzLhmL)3mLUg>#AowX!hM4*#A^(yo92&i0RRI#I{RlukrzzWmpj zUxW>zy;(OUwrMw6O7D8&ja0A4g|XTZzfP|?eiClpkTFlm7%{J=ea8*Q^ZfvIJcTaE zd&(0d@WFDMR|55G^Wbp$&HaE|C9*j5J4N}gne{JTHE_>;a<KYMw$GB@F5bK7K=tF>ktj;b;S2@L{o>asG`Ew4zEFsT%w%7tCn!L!9D^n zMpFZWsEfYvT$U!G>oYdn+7tK31*TVcJu5;x&D)XT9wdR+Aidufq(fg1Y=VG^ow+dK zb9E+h#GKi!avN}Q<*==yfvH`as40(6uj;foG5!yGZy6S47;Xy-qJWfwfOJSnBi#+s z9Yc3_N(%~zNJuvV5(*3r(n?4SLw5}bGsHL`3^~;Iy3e)Gb@un|^XL1$|8ZR~yze}@ z?zPr^-%_X_57<|jf?pg_4HGB4ONISRxTih~SboBb)(wl1BA&j^cNak+j>QMwk=yER zexUG5{SpqIdFzq_{E)cdRe07lhK@jgs}=%71SQv0MVvF*(AdiZYI>BB8J^}+$P2%! zpl37+p?bJv`OTkySZAvIQV5u~%f&f*JMw!FIuvL2AlT66^_4{1_CeE?S!aSY>`z=) zjmOEj$QEY&P$C!^bwVnmeo~(p%eSc~6P+>0fB5v6z`;r>oh?9ssIju%hAk?xq^4Nr zc8(#X!&!Pg$XELTbAMo+Q-yt^huPt3CIbhljKP-KHm5Jq^~^IFaxnP>J!x_t58Zbr=1>{? zfJOd*>`_b%j&jU<5I-{_d_UKJ+?m*UUeJ~p=zFMUedAtL(NP)Z6cik^R2Z_4yRtSn zS5gE%_=;ID_onlhH}~%67_7IwUuKI0YTx(lt+s14AD%8)3$(3=Ml=_;GFum^GOW#v zi8~VwfSbvLG2aW`zYt(%*WRY^nlXOrcO*j=Y3}EweUjPyHMK}va@DFWl^m?}G4GwE z%+-|C*(sl}_?LOtwuVU2^M+edcM59J-aL5uOZ~RI*WHC5;jv3;xvu`6ZWUv}K57eI z5{_Z>lJt*qLwAsfe#5?s9y-CAQzr;@6pea}MRbDMZBS=m_utoVw>80(6@2wNN#HqC z$Z3=+jo_v^2zyDTkvp6eV)6WS*1dgYP1u{Dm8^FuCij;_3(8881!*S7O-d}!73s4b zTlY#G-}^}u=y~ol7fbQ?ho7z1_!Z_GMHo{-6BOEyYmfIIOx9amy-tw&pu@8u5*e>$ zmdi`W)$JvP44Dl~YLrg2sfC<1wQ^UlE@KqH_c={k)8S(KRgZcm)a$%k?wnM4pLrCF zhH$JN+T5@O?+p)tnu?yeKdukFv5vgtS|D{{5n|ICB^o?!6Wl2{VGN_-&L_sY+??eP zxSRy)UoGtZV$G@T*r!&B6s5Ejl0mkdw|y0gr(a&H={n||@-@5Mj6W=p9Y2Rfu~R35 zF94cy!lM&KJpavq=Fw^ASBqbY+67|~9V{Y?Iv@v2xU7q9UV)qVOw0`1a6&=r!xG`( zH;J>Zr6?z8p7{b+3lha}gvdrfV0Ry20l96<(79zoQ}0}}LgCRSJRyb1mQh18nW|F~ zcJt!=UT7BWn2StZZr3h|)w;7G+tcyOQ-%WGe$Fcl;r8wKU|CUumZavGG#H9U#Q*Ta znxoP9awN@aVD;Y#TG&jl=e6B z0b_ag#+I_yf=zOq41#)%G_K1rl}yw#lt*|I<5>wc;mVo|&~-Nl=j(ypEh$6xm?eGZ zMSVAivi(hxJ;gtIuJjqzTXY^5HSM0jJJx0yo07*XKMXqYGS~17rs53B6 zjjKREoMKyPXXq}2_B^yqiNu4k`2sqKva2%5MK$R1WYASEM^J`zsJM5OBfGPnaQU^o z_O*_M|BD%~qGfXyA*V<<`~3HbMMP(rGm_}G<)WO`y4N)WwWr+}aM(fqyqo`PdDF&N z1dc`c0hf)m$8siU@1z^A%@GTXHq{g|Yw5X;5Yz3q3rTDRIe2b7A z()bGYy0vGJQw`(l%$zMAQGW`r_BfxesRFY1eGmOu+?J2c1Q`n2e>FdpV)S9tP`ed@ zcX$E4$9nT^t!M@NH5+e!mhjgd(~yaHiYI%n+>~Ezc&z=(!HRDaKvC9>hx$b(@JPos zT2?elzgWoKhSXWBz}(Fn{!--M(&;V*(dl#UoQX0n=m~4=W;}34iU@iv6{Q0Wu|Bam zl0T8#s40!tGG@{R;rNw0XlK;-p7D;6a{5!pWlOXaxc>Ya5LC%R(D@QwtdpJSzSMZo zb1IYTVoBb0o^L+Wt7yZ{K38iOSvz4#{OH@WCpj`sb-$p(l~y;RXt%% z_v!LV+`BuG-TY5vBAF12&~s(Hpyq)?Ct_`$TamjfGRaT#<@08j6K1r95yKx5{<$M) z21oO7E=P7wTOE6@g~LVsIJR&>4P_+W*Z-^quy{nYwKz>N8O~7fg2?I0yi%pPjqnl8 z*!nb2^M-{}0tMpFfoXT9t_R_N4Chg?c~rrIj%buRUgHnT%j8U?MHd;0^?LEJnexr* zw3jLzG^F;pai>A#LB$YOEtC_&?3JaTJTonHcNKM(AEEaA*6Gd>xnwQ69NId1rqSwQ zKH&(q*rzF=ZKO}g6YQOGUE!R-Xz%oF0t4)&vZT#bKOk?zhJ~a30UL*L(g9!0cP1f| z^mzu+75lljQ&HE(3vmPN%An47f-@7E6{a*iNN~Y-S9c~yJZp1}ad%nF`p2s>ss6d_ zqRy7R#^c{kFFLtZUH)8;ej2LqNv`<)R?uPT>1YrbZ9m`kvv;dq?d{XzD~e%mL9d3d zIk{J;57Q%#%Hq95Ob|D-Y$M%j>pEd*WippW>stNEWv6M=a`~uV&o4u8Z-`Ue%?!NUOVa?NPMV`G9>v)96=wtj-kuC9Y zRRA_AV}XmU;67_&LC_P=C22t(F;7G7?Pg*7+IE&ap)Vobm;BvZO$>hq6q~{u<`E$t zo!i(>hj7YTXo+%)>!0fnPM;IlT^==6IWu_?I$kw(CS4-RK?*6}axB^4BmO3NT9J%Q zUS}+!>gXrJ93G)`J2c}~&!!&>fVw<;H)#>Wg($IC3Q2ew_N0j~{5{<(pl!iPU~{+M zzjVQYb>3!Nt=03dkHyue*kC0G(helBJuFlIA2KgDrh_3!XFcx^4WtDVc=0WdKNYTB^8Re8Hmd7 z$0U&bIh7{aXISoq-n9r_{V}D#A8&UKiNq6f1Wr+{%RS%2$5qVOwNVmtLiw02kCCH~ zS&2C#1SDT)T6!@NjUmS?)=&bNPSsS12q-|NsfAoLyQtEO{H$$U7|~YjB=U}o=y!c7 z`8leh#CkoAd13eON)L7xS)9`&x#$P6zvpbS9ygn=I5`ER^xZnUJOJj|(+n$BcqpsCXW8OeSl z)!!ACF3>yH?16?k_ZnXEn1bizS`XjeX~z>edH?)+{AXds9MV>FHSS7b%^~_*{#>q# z32QWvD0(pc4pYm6=F=8E?3L*LJehlKJ`q!!t%f)Aq%KG6o`phlBH2?EbSFg?rdZ*>VPy zV$w@$29^13Lw8;>dHf?En*?j`q(a=bWnZP^@~l>5x{99DI>9dyBJM4PLt}H_q==wx z7tzue0#bgUV=WP65kxkw9iCV^}88UofA1& zBW?hWAgAB2_T}0*aoF_^H4l_GANs^9>lEPKEEZymE>}X&F0XI)oK~F$$AV$nqFYh9 zwzn5t=S1GNS7AHmlR3&Lokc%L3x9R(^z_(VeQ~S=ymo|P^$R5EX0Ayi0^H*fx{>zUp z)WF>3|MLIM%k=;K<$ru6`YEubUg*A$`ENfD{7K$JES&L*qk``m|L=ePpO0jWy@&1m z=xOPH^Zftyhzj|8(M_&W><6U(<>mhK^S~>=VtVr<7dw{Y|MrN_u8-w5Jg6rUI^X@@ z-t&Kk;(yQn{|v?duc3%udS$cR?22gLDQdkd7O;*uEx*Jd>&C9p6W+^h*7G$Dzs#Jn zErUeSTv3g2D9hR*KT5nCy;{9C^wT(hhrmC-3tHFm}pzK%`P>UY2v zZ&j(S`x*WC73u54?qI0kgw*CJdZ#$bjZHJhCd=F_q^a)_*Ao0uP*44?|C>{jLE<&LM9(pRYoSVnEI$+8w?Ge47CWMNseAV-^7FgTmg`(3Naa4W;q04% z18bd&BSu#P7f{0OH5x0eDCP>FBCuOHEv_76(hs1Pz1KnS9X%QlRu?^&T6EZ`VX z`TO|$13bCUN2IA@{%1@5P~f{6{AKF|GV}j*dhf)zP6yo{L--D^`FgMImJduj*9X-w6@Wmk@3fAv0u%v|9SD z&`wcUMacfwAp-ikxgsuF@kx|A=OX3wID$>JojJDjUj) zrL_8@`|?blG4l;hk=GhAST8sIUdWWPiD@F-QGX)$V}^2L%^EP>;B9csN<70lG~i&h z+=eLLw0z6Q7TpZgu>65Qq%n$G0 z8>feay9Q^O zI+*)ITMCqS0Qx7b?XIdUG^hJ~U6xOVO2GaXp}TlM70m~-gY;pb2S)xP(_exdhN$Z) zsiTYM2yo|?;*dnZu8X!zh|TL~$j2NBa);|fxd3zYJj>YlNxo-m?37Rc(NObBiJs_S}|DqI;< z8i=DMo=n!C05A9QlvenO7|_^T1!=c=GmHMBEQ20PZ`j4U|1iI{ZC!?9=pl}q#%Bw- z*+Q?&$z?-moJ!)D^tqI>x7KL}nf9)M5FI{asFEkB8ZLSy}4ypFAh4rq2wkOc7_YQtX+4vovq_HiW!EdsSWnt{6J&V_rFb} z|LgVo8S7s~fZqv2d3U$xwY%RM?PvskTbS$}f7%bpdVm0E>jl;J@;HyL*v1@HCE zR0q;N(uH+{f|lU+zRRh16F=Wx4gEgsx!O#lW^8!7SmlO&Ot(_5v5ISxLeHLO8N3GM zc5oAn0c(Lx=V{0Ow|w=+-?aQ2Ea+(sKBXGTpw+>XgFc<yw57gua z>lEKQ4bcTB56(H3Mcv7;1&{~sf9)TmF!!*`Ht^%dVdT#TAkR44UkeIgCw1Xig}`MI z83v(4UVj0zcTjri05j^Bb(B^kd2><2rf^{pY=w5Atp2cgsMZnj{z;{3C9uWs*A4=i zl$5Auu5YWcG4As>?}qA$qf>uKpRc;cCl`|`d$z452km-w===~O*eX#}&C?QD;+Bwc zhrhXh_8z1FIQrh^ru(?Ye3Xd@RjN#b|4L zDDRP+N@~E4Cd59~1J6Z&OEzRU=t7so%=^mFY2eMq%1zh#mXz!O{@~>H^L5az)!pTU zR&B~*6VeaR=pCe$$ilxRYsDMBdryuh^x|Lqv+wKM^!SaO;^XfdF-`Wj_k}e=CyLme ztt9r4s@;>pYp1$DN2e4?*1X5XcRubGkczVUImMPJ;tsOKeqzvu`qWaf3zM~y0Jcuw z(#!)Us@vNu?=a|-pw>UurIzt@^U#{cQMV}(u1t-MX${p}$%F60qZ@CqDp#1AR+Lvp zG6l!Qq(y{kOhcotWv>2^0xp8#=Mzpn1ZpOa{U&^VQSB@)8AJt>J=Mw=0SeT3^hT&Q ztp!Oct!#dX`yShLk8jN@x3rLO^QgvzB3kI~-3_MDjlwcOz`5|y-<$!vvL@Sc@Xya|pE^qQD>AygeX) z5BpRJz_dj_{LFys%#~Dk@QXnu9ke*=_KBm-SdM#AXh2g;^3){Er{YwvjJHit8vH5n zJIgNBvQ^=q?T9)~qY5C>yCO{+!z9p3zgF)svUZ?S&$)5&Tb4n4b(u)a@m^ zd6sN&?|px$aJGMU)Bd+=c%~8Et0nOGOVEmi?B%w7D;c_Z_b+;)cL397z2d1;be(-; zdG~YPeTZhxV4ETQ_Hrj>I?Epppb`F;jDO{OWcj-OzU<94Fzt$20vEsvY2 zeTZ+o$bdT=r{4YKwLD89g0F?_tkvlHYkIikC>el4i7;A`NCVN)0JLK=L4MlKlu&N7G6`j6>lOas&G9*Y(%3vlBnp$}Kl z+|GD>vRg(nSd$a@V~{c>>Dv42PwobxW^R4ew@v})=Ot*D_hGv|wkgtQ4Y!_QDhtl# z9e6A^WTSU**6fdgrJ4~01)MX_RmU3@%z!;#tJAL6fXfJB4SXM4=%UufMw&$|S>w+Q zb*&^41t5&k9}5pdl4uO~0xmY{3=+;7jo^JOlfnm73%dB1vJPWouT-mXk#nq#E)Rh!q4L^gx#l{;xL z5E6i%_+>HZ95SNzY7jE()2}PJYS1Z0YvaP_Vu@1d(b=;unxM@PW-nZxG0G zC?^Z^>$ev|UxgiZlcCJRP=QHts>IZ@q9QEQBnux&8};XK9tnF1ho4<+T|EPp&an)` zkU0=Ok>It6u&I6)1jFl%5+@*~Lfx~OC4PVZ9p{4QaQ(Y=gro= zRt?%E2HN6Kk512c4)o~@pY@;>C<1`?AHYT)eKlX^dZ{mgQ!uf`BXV;5d!$-G(ImF; z2ZwSuJRzm_b+69811Uoh#YB4&>-m80u;R2G``ktpe#02^=P9c7%%uH$1iXv$;g~&k z(1`s}7^<(cqxjnnPJLB|i14Wb7MgK7i2&{qEKD#g0vWfp`g%)P zxo)b(J-7)84wEQWa!n8}uvCQhF-E(se&;LW@PuSDYfn69G3wQSwPCmfbFH*=x3F@E zJp*FL9AsT@hfA4b6SOrzabg}YX^uHFA$NU9nS{Taw-r5i;#sKFiE7$PmUJ$9OP8s| zDE^V()9?FsaEC=Yp{ol#8ux(QotqN9R-=9I()O%}xQ7tPNGLEJ&fcl)P#4JoZgE zc_n@x##8q8V%>`r030;uv9+oeY7Kr^S{oQt;P_(i?X~OKzOehs zP&{N?V6&$t@Ny_kwx8VC^o6k6v~SqUPi#%Yc@3huS*U$R-B%k=u`b{4$KbhrO|l4-p$l5Xn(`gVwCgEI-&+E6V-x9tS;r3>RKxo+fIuAP zQU6orD?i-cxW_3X^&V%i*ofScHmR*7%VhO}fBf_Y_rG?(JerTI-wFai}eOj10894 z)Vfc9)N-!Igm#NnzL;-JAvxxKPO&!BhO;-;l?meaI~uphL1jqDW&r*qb#bXPIAzg6 zv7B({=1~s1dJ~Z~oT$== ze9ZsMK{28-_gD;BV~hBtO8@bgOCS=6UsGE9GwXWkDQWgOkWIo17Avf-OQq%FVAVSQ z5H;jZktDMd$nYCb0>$srbk#m3xnUEv%x=P5Zl-nR-%vw~4G031%rBzZ18P&@ovDqM z@9hE)HRf}bw?3&BLD4(Z`VyzJk%(BzqNLJs$wRxQc{8NaN_ou}D#}%#epCoqFondz zzqm#vNrF8~De&US^YQ<6lZ?>=u|!k6q{%;W<^>kMq~sxOvE}u=`JssC3a`cAqc7N1##e5R%;K7K;sGo#gO%9=+$w^j zyL|+Fzr)>Cp1k17yQd+89bp=ADvbJAmVF7Gh5u(*;$mH?9cG@m+gJLDyTVa;zkhei zzTl3h7nQAslw{zO*NA*X3B&{pgN+~X2%q_+eRBWg_il(jtVgBt@*1rJeWFTBD~ZT5 zKX@71t4<`k{yIs|UY1HoqmKG7(A-b+z^?NVo{(Yt?LR^aIMbg!v2?F(DTF~em>pR` zuo2QcudYvxg)u1{8ggY)Ld*TQMB>@-h?3Oiu8dALP{#rbOA>+lY13g{Nrz#Z7w*vh zVI=JDrn7l(Oa4Fd=gkif<~KjY^MCk*{)5hXCBiLa@S7=~wkV|kA01c`cCKiD1q9-@ zg6FH~f#270;ZH{oOGv2wB3C@m{|&^&k!*g%`5Mo1rER+?OiH8vSW>9@^`AwJ8>e-@ zf%pDUVD}bK>L@vqU2LVmY87wkN;TD=8dC8HT?@NHYwR5?K^!a*Pl^Jj?>(1ojqOt| zL*Xd=^gy}_rfTQ#|9qr|TOq<3;FVD4=FsLcRXSO|XcCQ+219l@G*n<$Uh~HHIJqw; zoQz9)=NCz*jCaBB6N?SB^NeWzNdCY&sSC#dN0nfs!kKNhrNq+9kTlqsxf^y#VbHP! zMKOXjrjbr@Kqck}5Xn0@E68$Fn2*H>UTS}&4r~wHYgYL~8!N(sa2GOs_JNA>ovWi9 zzt6@90-t+WaNVNjMd!jp zKm`K!arwHeC#*S+mU|>c0H`lSiLWEW54%&GFtfA1hu!DNg`mbx+1RG~R}yDY0Fv^| z`wpA<7q8FXGqD2qU@A%}zD~uOetm}h5BQEjKWy^K|k;X;Ly$u{qaoI&4d4CpW?t|C?;E$(gi)q!M#Z8t7mK#usy_U2rc zK7TT?aqR?Rt+>r79$J?AU>jgSUCv;=27g-J+<2m<)rN*FGBWgK@%dh_c#5v5z2p5* z>JYg6ZH~5KX#;+|UEr@WvqY_D_W+yd9}M5j^;ZT4{1e#)!1dH}G)2{HzVO|tY2tm4 z|8^gmdH199phG&VY*JmnAH?#s#dBr;)kT4QvbcqrR#vH+K{&_uwuRuU^F9AAGVutE zXX~B~=3Ex)O)YM7IFCwQ9N=#(O6QXh?(1jzkte2>Csi+p<0>c%d&1nUOtM2*RxE_ znDH?wM~g-E_z}R2T)OB`Rp+^eDXa)`4!4>)yx2|rcX)JH3(fqe`!Ayrmy;M~9l1lyDk5Rnfr4NyAZp67wjc?dCE~0U#8hqIZ%rJPESne8884 zwfb*b^tuY=7LJ|*nPQGK=%|(P*yr}f@&Mp2$^TIrf<{aU)X4oH<0!Xahj{FU?i>v< zG{BJoQ4(#2xiWu$@PcKS+awxId(h+8_>`(i)b{cCb@X%^OUb(R`tROIsKfg&0J<|wvlN%e?}@u&f9YlW4fdn6rg@@&gb=9bhy~>_hq%p zyYi7u8~Ryy5;9#AR^1r0s2lM%y}S$Ez`y(sm*qq=^}r0ylKcxmHb+7K0uXIEW4xWh zU?-2DRXb3cV{Y6{s0IyabC{Z)aNQ)6y5auAlLVq{{28^+pUCyh7CqafQMxDVQ((Y4 zHuR{dDbDy?L->-T$q@C*NRc0qv8Yx?)~Mfr!@`+NA7Pb*4!+o2(*&UAs;_GnArMm8 zImFhaHfQ*J61=NAD*gP>|7F!!3T=oyizlw4SeRnP2lysTmoSXzA=egbd zBz2EW_=WfbXZs}WAL>R=IF#`?Pu3{71EuIfwtqG}JO>CLZy}KK3#QsKc^L8^TPJZ1 zP$NG53uSnn(=A9n#GCuS$*5zmKy_Bw85f3U{f=4x(){Q>mq=ZAA zyq>GBYidjscCAOtm8m7?XhxA@VigkdZ8&GFdvy4-eE*o4do3AC9UV(}2}L3CQYVu)2Xv2uX9M(;{+iBnWy`&QONi&@ z*K9Zn|LJdbovGRNR~(CPgk`t>SwcHBL3o=~s&rdtIG&h1!fH3wkB{0tevT9)x-)jJ z4sQ4&krYl9-;)VQVNveBOZ|lkzi=!&{!seEX3K^WTFV0JYBUCZr^9TC$`}L@9bF}; zs2kytBoX{WMZs^?Uhm)$gf9SrE*LNbh|kf z9jzXyvI|^xy{S=N(IAeFTXQj9b(q|itLwp}(a1bUeh1VcCUxfO$xLIjLGHHIV<%*7 z8w8!56EMBredVhHa4s=52%(iQmnSB}=g!OG@2d&1-E^;!)}kPS-zG^lgJ~3*T(u`8 z$d|T)O#rQxd$}_~`GYm}pEODw_f8cgc=0fv{ma8cK(F1uwi{_=_vm_6W9?d=SvZ&H zB%kRX@1t`5g}h3M@6ny8~L#Ai2hXiqVr-kK|p+49Ks% z7<_)vH+Tn#88$o{1H?%|AZ;fiL$j|ZoG#8$cG=Y=Ez*DPd(Y7?$>T`(K6}$TnSW~^ zkT_Q{5p=uU@Du6G-UTAjRWXvRND}&xZTTxyq?qw|rtaw{&o-2eM8M(*cgL<>gZ+yY z0a}mQ)HgVN@eGkas_v_u#`0J7eD2sM26|X0{lz<%z4~L|G+X6w5&vzUcOTC({D{>e zZF`qBLeWU{FMx&7EDKS9IBebv6Q?K)7MvVQp2?sFqP;G;W3vj#u@KddGH$B$Jmz?UPxc|Ah=-+5(# zKjh2@u&AS~wIA;PHW2(5Q%m#*HsJc5O35AruHyghA^-CV|7=G8GYbDR3jew7{?D}_ zM})5l$T_)Em)?ShUFbX@abWK6?sWRy&ACJN>tu1W>_}Z^c*cvUd@4bwoQ{I$zUm^? zAAbNBf-NG?9URJoR4_|Ew#ru(jKBn-0c4gdzT?p!rcd)GX**}hC^BtnAwo0vN-=g6uMx2)>h@(W3r_4 z)%Ebj(786e*{IsM!V!|y5+~sMTM6I@(K!fc{=z=(H{9#nt3xM17p$H;l6Vz}lD)fb z(mpNsD^NVwH=&on#lp_i1fr|7V&0Iq_i)si|DxKg^8wmo7|{PERmeRV5XDhg|5UGh zfR!w_8N32?5}|AhzaB}Cx^7~A0JwgVRra_9BK~|JT3ragK8hFk={!hoA_5S1|5VZ= zQ%rLbpanvtF1O46DR(L30szC1Ge~LO*X-R}u;i4FvL`A^#2a>0ED#2g!H-Le)}Wk{fL>cowwA26Vg{1A-A!W(7N8+ z-(_oI4ZEd@xu~{77h{m)DMwLe9ILLn_#E9_^L0jEB8klF1&iRdMUKm(p<;*LVht5Q zRW`A@ObIz(YfCz~0#wnCwS7Y~zmf7WKS-@jZ#$hG%p&R<2tw?BC3i1+Q&`?^F;wUB zS|$s@LrDVUg{xqZS48NGEt34=NDLDuV#YfzGB{}2HMj}5LMXaLNoNb>v>;2L<=K~Z z1m|kH&oEBCH^l1Q8-OdsqbTA`txO`31QS`;b^Og~#WPq)gKuqJb-^X%?j|@FuxTyR z%~U`O5VHmo9IwN*kW8-Z>5I1mWGOiSVLNzzG<3+DaNIaRs*y`f7bJ3bG4#zfo;+#h za2cTG9B(o1X&Rjn-mn}k(Y4ZDUzXt~RjBiEbUr|4aFqcuY#Np9?app~8RkS^g9o)w z379iLKegwqBdNTLLiYs(y;dm!>qk$*c*CTErze&5FMzQArHp)mx$QZbCOGT7iM=_y z+4a(aYn=W$LuCt?doeKfdJib*5o~~9NLR&^V@&_yTzoTO;XA0T z_kfe9gj2atlEH1>)E?Emh5(jY8}?3!7Yy8a;3~4>QQZ~bma!TPD-1cZeaHD5kRAh{ zk=H^xHi1AI=*a_+-lJXUap#UeNnTP7)nqAYC@j{Ti|>1~9v45Dsc-Uq!eLAd z*zZ`;hfy&7`Wh}(bMLjEA25>6B69QqIz$gYC;2|6v0>rSz!68C$_ znIx(EK6`;rBRSW~gRg$oZma77GcDjPE^p_@CBMAR4~BN|Z|-gm^ASLW!AJTOR+;rP zP0rQ^$>(w=X^v}-*-3tV9wl6d?K`)K;=9#mUDC6;R5`8DT-iWa&rqjPLMFf(pvu;ZC8K;iS-aNsmRykqHwzATip>U~M{)pt zWh4+WoF_#kgW0|Bk7_Z=`D#$f5)DS)Ec4=~G^0szqt*Yx zp%IK=1t;UaZx78|v&-AVgvv_8Jl#^7$;{g2{ zms0mQ?i@2~kdnpi^Q~NIdts3)l{&G)C>8-tZtHfH$uh{?`%> zEOMnQ(PfwP93zM3e(-SLpS?T*YTH1IFH9v3Df0J^l0zHKhf(*;FbDTjKKVlo&_xYkCEhBQkTWuT8m86u7sOxWH@X)& z!(%kmeSNH_X8{;Q|M{j?=QAKE9?2uuj>o^jwSpWR&}qKqX zK8#|Jd}6dGv~8j|i?0&1*p6nrFn&Wt5dn!<=8kTn zJ$>9Q=v*W&)=^O-} zEWng2bxD5<0p=wT8jVX&q2WKHwV)J;30zp4Z2yQCaf}SsyiBDbe-K#ztxTBE)=H@H zoAX4Z06pnDzAhh!fV#nPJ#+d^X}~e+2IM1vcy#i@0o#fIw%cQipru$|?H=5)lU=fv zL(y$^x5fJChS7|;iPu~Lmt&7^g4iBt3cnFH)RUK)0Jt2nsQtxbz+UUCsQ0uTk%!EA zODrL^{O@!Vb7xWx*l$ijP9+CD{x7%Ay)pFi4k>aq<2_x!gZvbDb5z+VVXeB$73q<~p}jq6ZpW z&4_oeLBjw?AC#dK=-RGuB#$C!5lmxxfnaZ+%TzrM^-%n5_AgNnlFn8PsQDk|OsCc( z&btXDN=Uf8-r}YRHO8T(+ZxiBlXN!>9m6I+%YdhA)ib!EY5k_xNBi>G<(1SDZZ?E; zq1r48UtjLTrxXWF|B$K{iX!H@IzNPiE8{#H60om*ZDR#Jhf{Qh8hxMqB|mBId;}?Z z=M92#)Bv>%Rc?)%Vw4z!9mpL%eO8M%tY}nN{LWz#GSQLs8@@8WqfFL6j6D-Sqm4u= z(RI7`#}^~Z&{_KhsW8h2m-HFKYE`n>O?e2QyPQSt4UPZ{SfjGlkDD47gE~H!@7u>R|Z1xi!BrXNAJlEFm@P40pA>XIBhn^{)EuV9OMNGL=G0 z-3@M4X%GBzNn)&hiu6>V`j_%d-KcqxVj2FAWJz!e845nHiWM!G zWF`fsg-Kq@WY$;ajuK=VfNa!18o$C!$iOp-c&*6r>4GhSa3fsk7%+u^tsexmI!d_q z<0?OSq!?E~j*r^UI4&enBQVGoyC`e6ju0N7n>$~s^gl@=sRhAjEa>n+@;wU0%`%GQ z3caQK7gby-HSVM&(dz8LVX&u@a}A)JuQ+|`AK7R?uH{i*Dh##(KP1g1`)s{sLQ&yK`g5{Znk4Z-+px+oEFiU^ij4kV4C0dfw0&Is8gYo)*{-XCT2HfPM3yg zF1sb=U5SAX9lg;>ICWN1`Wx!u@{Inx)o!-kf6>P((=mkIy~M|_MjVmTPZXG~aUU3C zkN3y)>(=_OkJAHKVs(PceirlfT#RWy%F0(#B#YTAWSwP*)`z%8WH4xbcpGsU+J@uQy3JwvVGhP%`oTOUndQ)Y|!`rhA{5M*QWDi8h5wdQWwQEhvt1XMz(UxFn~!tZM5N0FjB;L74IVNjNdKGxgeQ3-O-dBS9zcSl-+;<}ZA```d>r z!Sy%V(Bfd`h+dWq_qf1VnTUs`*+hx$ftTa7GU2J_CI$QK(0tM0l}!dBk_Fu$s;4ZH zN)Zv(Fh~4?N8i{}81B+J7iY^%P%F7aLRP-=ATYFR8HyU4##AK=At6PU85l zuBID+a)9cbdU@z`Nq(Z(r5N4R0||4Yb;Z9`2FBoC;MNTSq?pzvaR-jh~*Y8JJ7Md_G093?pwue`!wuIZKUN1hm z0x3+;f&{UwoTL~WH}mmpHAVMYV6)f%W@mWxfHKPBJ7KLfIbX1tUn1GVOV7DUkk(@9 zV+QOtilLOr4#F?bARd05$S;-l22wUvj@N#k<@O z_(+mvVjcz~m<$m0*!3i`Ucme(P@kg4n#ac`)lCbv!$7d%M$XesASU(&>#_`Rd9=e@ zoq!CQ^G?*&{7S^kTtf>$`?o*g0q7silfrX+N4?t6j#0~(ud5>m0Z(LXE{Xs}0$ceK z_TKOBq%hupnOfcF*d;mX!Ue+BKk4bshwDD|GHDqGqz8I|b%*WbuN-45sF>Kbb6k3^ zZu1vEe$0DeB7-**3J$Cx*<>M)#o;L~a@=A|H6orlFeAzXN^2|-!>?9A(W8E*kz-E{ zM-HaT^9w-%bIpqOGTi+S)g!Wk*WP=-ahL7_w`veYe(&KHefn3_oU{G79Ku-%=anr< z=U9-vcaQb2nv%R>W{*=H3aGD=kzj{iNg4@U)j`v>0$_iQl~6u_9P(xNb895zBXxx# z^1VHUXr%~k70rjfrE_z=^2oq58=f9wF#jVxiNn)4ozC$Ee`}h}>S$+L-*vl{xjLOs z6A~&D-H|0&D4?-1D+7}LR%YZ_6?IpV>*XcX-#p;Sux+Oli5HQYT1u76O3Ud|DlQ$y z5Z*o=IPIi&zSrldwa>H}W<4fb3TC?7sVpvPa?&tsb-nwS<;L56f?eA&?hHC^aTAVT zQP~K1=5|sEhIMWa8m=4Gl;e>1JiZcW@I+gobVyzi=6Pe4phOZ)sRi{a7*o6;vMnSm*;N2K_(@6WSn%1w*^sS+YnGzY2%w?in6@H;44(HpNz;{JCh74 zB0P1nJWis-31`%3TtSbed1H3XB`EN6%y5!FP%AX;IwJeRe?7!`K`66>0_np6aFxGYgHag`q+5=rbHR2{y*(q_g7QPx4$APMFc}r z5JBnP5~R0KB@%iGQba+TbO=J|1`rek>C%y26hjjbP`Fa1NC`+sh)9tdI)OL2>wVXI zcP)AA{Q>6ZtaHwuGc$Yk{**I_w4n*dOwwyDt(c1J%rwn4wJ~Vr?qEg0YyD=x6L#)b z#>Yh$=$;!XTGkk&V?U$Uo?T{ybkD8Pr70c>Dw-1m^YV{f zoQ!LQrif4ZG%r-GwWLU1H<%m%&5d!Bmha>jcjc=eu#2RJu*u?aMmdRIe7oRwjsPn>}BYbeZK3 zR?{T6$8L@_kjjH4Kx-(RXGk6_i{1=jp=B}GZOz}3~{|?8k{FrxA&uR#R&~UqjW-$cmu_4wy=TEh*?zor@4R@Y$Jn+(|IPv z9-1Pv!q1d~Zu-u7`1Yg|*E^IXPjy%Ls1IA37E!Z_x@feqK>3WsA%cFL*107vy^3Ke zfXmogx-U+hZ~ z1qoI1@y}Fp3Vl|Lr}>&GPCNj@Y!k-;3h8pF6@NVo2XQ|mU20lC;Um%lEa$k)UW};Su70YJ2(&O7 zLv|`0H2Fsb#hgV>1T4gZd@QWCt5btJThC%r5WVRr$m6n*B4821a5V-6j zJwk$ifot<>b_4pAwdo`}S8{h&v5-x<^1OtDE-htPuT9N98bu?DMtY<* zQ_xK=q;LJD$-0VS+W}0l)Jd>Ct4Uv+bN|W}8Zq>I8+6Fci$wr&s*A+yBc!MrCaiPT zfgX76aTk zz7l&?yu5#HDh+tjfA;+x+LUutN=f{|w3n`yTp>8IVD1VE8D>wpfq1pYG`K6;nxt;d zyj`pjz#2JD7(_WW67!5-8zhJp{l!Ryq}l2{(ZEyKySfHbyg6x4#t?F!%zDIo;N)A? zXy>>Lp9Knsuh+fI`&?c|+^1)rWxJ#Nnj0)`2e;-dMl-`1kp4T;e7JjWA0(zhr9XXU2q1SFg=sU zyLjg3QgIAi3TcGWyrwA95*5MFLJi9>YUBDfDQ7c zD6D{9bCdp*Ul^&?GBT}`dVnr^pz%pdAV0i*K(aVBAAoo{v*(Q$d zOO*8Tca;1Ff&Rzg88@ZTP2nNsyLO)U;&le8KU9|c;v6p228RxI?&)9YT%Y(=5zzDj z!T<<%xOI8W~WfOsr}xy8{nvB zVr5SeAF9hpwXT(Wx*U`n`p&BWNLdIi6K>dnsI2f3iUDO_Eg!+){^=BFDv^V645xtdFvY)G*GbfDMMp{ANmi7@^& z^69m8qXMrE>O=b_IJtIn(zSh%{`=kH*mtiP&Qw-?+SlEn(T|VMQtIJ1eh3o;dTl^s zQZ;mUM#?)F4^sg=`EnE-ZJDGA1vL!4J1KeuqBz3-*DIQVMW3 zRbS=8b@)v(C<)!n(P>+g99vhJls&unlVah0VC6B)IPS~f-vL+`CUlO zn|4KoW*6YTU6gS?0SH;o$B(M|fhLCtNU3g07yUkj7TUdMwcG;pCLhly@5z`x8)b!$ zH!43Ia4Q!Vty@}OuFj;ZbV;f{`Vs0s9D!5~^`GpUsCfmFH9n=kfj>{5P(wsIyy#lem~vv5I{CI zKe#1<>&*9zrxVBl0$V1U;Xc zBz2fy7;F|;cG%aVWTk4udwr^-S2Sv7qBoO+c|N=p!-7+w44e{D@L@1n^hvE+Jv|k^ zAWUZM@8)jt((Xh;Mu7zTY?#jB8Ah0(JyI2I(IMwJqANs@&ID5Yqy0wVu|2?tO5Fus zz6LwILg-53gk2NXy}P~3P4wM*PVDL2S^}N9r3SZx01s&?Zp~~G% z>(hSBU@bGEGM%aEb}(#mLcpSkSI{=8j>iI?5~#P5U@DRm8A`RNBc)|%Ce4717hynk zeo~s_QpjW>E!G)5-X6NZK+eq2EpQ^dOk+8I?@2)IHy7gbh}`=E1NUfI*`zXrFtARV z&mr}bBe-jdlT$d9#Fc24_VK!mDPKps4BDbcg>!%k!0x7F&%j0e6q29{V0n^}=rj3L z>r-@KcW&44Fq~0592!tq^)GC?Vss^4h#Ha0ujW1E&Bf6+qNYeQed1)O^{ph+tNUv7 z=73zlh}s&UHH|BgtMgj1hSejvbRz&-Zw3pn#t|npIB4t~lb-`;2NP1lIb;6QEO&0tplqBY#edoq> z1Nw(01h=T3X)L*sPpgub^YP-|cfCEp_pP}&KzwJiP_Ls_$^+5;IT@h7pbT@eX<-*V zXN_!(e|`C?tHaOUps(tTl<)SJ-0$5{=Ep?TP(BnaR?TfX%@~wdS_wX*~GCNX*8mq8WqJ=&dCOQV+nfC3x&syZ9oI zvb(Q=6qxL3QTFyOt@KHKBSSN(-Eln_4Yd)mY0|lG$X`Myr9{*^h9x4k<1J6j#v29b z0(`~v0!+ReV#ysBx}EI+=h#%*jO8rq_{8~v{of}qj;nNA69T$&;(%tJVjF<#P-S&| zIixN;aBs*PicwKQ`bIbV0@i29{1e&6!N8!P{rQs)iXH;BkU*sm(JKGQ( zXCCIqH5T<^vmUgXbpcWT+Z>UC8;o18hVo9#_I$>=LIAp} zh#Y|D5ERpFA!A~!Rlr%J+9La-|2`XogZdCbc6I3Mz6(A+Q5hCIVWY{yBi{`q?)i1j zy9Jw{_AhUa3Uw-x=7teJmy19bII%s@w^_4{)x0I{mXYaP{V*iq$YjOgebNyuAZiwy z^br~5^Yl$&i~!5K?@uO#2W_Ca&v z0KAbBW_~e~1(F18@IkDId(%o<_myY(K_iP%oF4dK(%XO}(Z-!I8K=^~O`?7@U6D_R z?*?A#;q3B**|sZR{RbDT$E%)rc9v%^KHdXodDFCg`6F);9}6x_(en_pyoEjpyPwTl z)$&JWhw@!8$I(}7fc0sPCgR&1F>hSQTzRD&w7m3g9<{NU7?vK^>(>IJ%h>K`Hu*zd z@z)=ijpJ#NyTx*l(Ass1p3hR`Md5&CDJJWIxio`MG&>DYUSutkPRA3u5m6{D3sPua|e=*z&xl4Sd7QJF8jqC%TKb zqzgUuVo;aHAyS)e1*RDL#`c0})&aQ&zfG%^_Doa@Ilwgl5%SV%nN<0G zti>rEe6|Q6GP*C84()dh_rWao=q;D1pm z4d0_>iwv`?I0VMPqn1y$=frfOcWlDgP`A_8WMslZr{su)2damKFbB=>R}0)jcj^Jx z$T9t`_$Fk(Ju1$yna`o$mSQDjvqdd4&aL1ymW2+4Z2*;lOX@sPV){f9N5yz*+jd2N zvEy^G&of6p$qN}t@S5z!NfJv2?B-}^n!Zgo@7!O6i`GA{Z1`y=baB5^9bBuZbFk_l z&aHTQbxr8VR_aB`HL__R*Lz>S6UR!y{v*vx zjwCHXQs;_2Xzn8(j(j+CXxylO64`hWa@i_~ITZhEX&I}bi5d>EGk(hoFcT!73)8C$ z8ckEcdCR8BEl2v%r2ob8eneUXg2RYUoY}(n4SfjL`?Ku+P7nJ=a7r(E;}#icHsVuG zJLB`9KB86Qs51_OHHGE9@XJOWD2cB)r@Ce9L+5f~xMayff7}e3*HW8pf!&YbU*fLb zTyA}$@ym_Qnz_+h&y+h8SP>&+qKdOI4$tWk&Yj>fDKzN0$%dd_;OZCG+UA(Pb+FZb z-`LYBJY}(@+@Dmp@jOE>y*snS6)D=PtYPNWB{n36O>i1TP4oGUM34{Vr~3#!d{^i( zk*)V8(5l{)4^+@u?&Y*(@#^5v2FVcc2EKnRVkZn5xU}sHI}EQ}=NIIZLVf?6@~b4a z?>udg5E-NaoRsAS2sH8W4OMG+zLm=QTdiV#WRZmj06{4JGU8v&ZAAnYGN`2eOEeSh zap+0jVK2XP;yLBF8l^QGp$4kw2#B^9nHVOTo;0%_tSKM_7{&4s=f{pPv7!bo{r0sr zd?ebXe)Hg5dR^JmM3vit=#&m)k=_0K5$83myJ`Dcy&nG1jB!k@YDzmkc-vyXB7 z{dbBDg#`rl#M)@cY-sMqFdUPRA|pTM+>X_J#?+C4HMmdYaKTC67dg(* zcaM`Js6PzIrVP`>c**hu_Joc}SH0wv7_xk)#t~%%c{h)-ZH@IYne#E(?I69iL8uj5 zP=6vha6kBMezXlH?;}TKMB11AP@W=ar2xl{-#+3ijvyFWK2~r3DleH0y->y2jbr{O z69iE1K7;jWPjZ9>#27Sye|P6hN(i6I6yaNuvIBw~apXaAnmXox1>qkHmkeIDefxO| z#GCj2xOk!VWbve6%XOH=oX-9BYQW7(EfZ2Ad0t}it*^_&V+Iaf2&m~+AKNe7%)IOr zq<7q-Wm8^LuFqx{O$IHr-wm2V{}pY$C!aLRM$Tp+UbALjp18F5yBJB4D_D{NXGTHI zF@}re-A(u>^~e0tb^vIrB?Rr%eby}j3nRRY<`~Tzk8&ocOv4RRre_oZ511re{?C&B z-|shwM>R7V@T9)yU1kLDRj_PKMGhkaOfzoU+q4^RJjuJc&Q~2bKQRcf|KP;_UJaikUQjMs=c#DN^zH z_HVy?F{IZObQ);#zXg`nFdP#vhqgJG27fN`&qVw)7XPz+{*1++vH0Dd2Ij>PEk?Co Ud9ETU`7iLht%6W~1G^XeUpWX4 { - if (querySet?.available) { - querySet.resolve(); - const timestamps = await querySet.read(); - return Number(timestamps[1] - timestamps[0]) / 1_000_000; - } - return null; -} - -function formatMs(gpuTimeMs: number): string { - return gpuTimeMs >= 1000 ? `${(gpuTimeMs / 1000).toFixed(2)}s` : `${gpuTimeMs.toFixed(2)}ms`; -} - async function sort() { const { sorter, note } = pickSorter(); showOverlay('Sorting...'); - sorter.run({ querySet: querySet ?? undefined }); - - const gpuTimeMs = await measureGpuTime(); + sorter.run(); render(); - const timeStr = gpuTimeMs !== null ? ` in ${formatMs(gpuTimeMs)}` : ''; - showOverlay(`✔ Sorted${timeStr}${note}`, false); + showOverlay(`✔ Sorted${note}`, false); hideOverlay(); } @@ -279,7 +264,16 @@ async function sort() { const BENCH_WARMUP = 3; const BENCH_RUNS = 10; -async function benchmarkSorter(sorter: BitonicSorter | RadixSorter): Promise { +function formatMs(milliseconds: number): string { + return milliseconds >= 1000 + ? `${(milliseconds / 1000).toFixed(2)}s` + : `${milliseconds.toFixed(2)}ms`; +} + +async function benchmarkSorter( + sorter: BitonicSorter | RadixSorter, + timestamps: TgpuQuerySet<'timestamp'>, +): Promise { for (let i = 0; i < BENCH_WARMUP; i++) { sorter.run(); } @@ -287,13 +281,21 @@ async function benchmarkSorter(sorter: BitonicSorter | RadixSorter): Promise v !== expected[i]); - const lo = Math.max(0, idx - 2); - const hi = Math.min(actual.length, idx + 3); - console.error( - ` first mismatch at index ${idx} (showing [${lo}..${hi - 1}] of ${actual.length}):`, - ); - console.error(' actual: ', actual.slice(lo, hi)); - console.error(' expected:', expected.slice(lo, hi)); - } - - return false; -} - async function runAndCompare(arr: number[], op: BinaryOp, scanOnly: boolean) { const input = root.createBuffer(d.arrayOf(d.f32, arr.length), arr).$usage('storage'); @@ -55,37 +24,7 @@ async function runAndCompare(arr: number[], op: BinaryOp, scanOnly: boolean) { identityElement: op.identityElement, }); - const actual = await output.read(); - const expected = scanOnly ? scanJS(arr, op) : prefixScanJS(arr, op); - return compareAndLog(actual, expected); -} - -async function runAndCompareTyped( - arr: number[], - op: BinaryOp, - dataType: ScanElementType, - scanOnly: boolean, -) { - const input = root.createBuffer(d.arrayOf(dataType, arr.length), arr).$usage('storage'); - - const output = scanOnly - ? scan(root, { - inputBuffer: input, - operation: op.operation, - identityElement: op.identityElement, - dataType, - }) - : prefixScan(root, { - inputBuffer: input, - outputBuffer: input, - operation: op.operation, - identityElement: op.identityElement, - dataType, - }); - - const actual = await output.read(); - const expected = scanOnly ? scanJS(arr, op) : prefixScanJS(arr, op); - return compareAndLog(actual, expected); + return isArrayEqual(await output.read(), scanOnly ? scanJS(arr, op) : prefixScanJS(arr, op)); } // single element f32 tests @@ -149,7 +88,7 @@ async function testDoesNotDestroyBuffer(): Promise { identityElement: 0, }); - return compareAndLog(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); + return isArrayEqual(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); } async function testDoesNotCacheBuffers(): Promise { @@ -176,7 +115,7 @@ async function testDoesNotCacheBuffers(): Promise { identityElement: op.identityElement, }); - return compareAndLog(await output1.read(), [36]) && compareAndLog(await output2.read(), [10]); + return isArrayEqual(await output1.read(), [36]) && isArrayEqual(await output2.read(), [10]); } // prefix f32 tests @@ -240,7 +179,7 @@ async function testPrefixDoesNotDestroyBuffer(): Promise { operation: addFn, identityElement: 0, }); - return compareAndLog(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); + return isArrayEqual(await input.read(), [1, 2, 3, 4, 5, 6, 7, 8]); } async function testPrefixDoesNotCacheBuffers(): Promise { @@ -267,220 +206,51 @@ async function testPrefixDoesNotCacheBuffers(): Promise { }); return ( - compareAndLog(await output1.read(), prefixScanJS(arr1, op)) && - compareAndLog(await output2.read(), prefixScanJS(arr2, op)) + isArrayEqual(await output1.read(), prefixScanJS(arr1, op)) && + isArrayEqual(await output2.read(), prefixScanJS(arr2, op)) ); } -// integer element type tests - -async function testU32Prefix65537(): Promise { - const arr = Array.from({ length: 65537 }, () => 1); - const op = { operation: std.add, identityElement: 0 }; - return runAndCompareTyped(arr, op, d.u32, false); -} - -async function testU32Reduce(): Promise { - const arr = Array.from({ length: 123 }, (_, i) => i % 7); - const op = { operation: std.add, identityElement: 0 }; - return runAndCompareTyped(arr, op, d.u32, true); -} - -async function testI32PrefixNegatives(): Promise { - const arr = Array.from({ length: 4099 }, (_, i) => (i % 2 === 0 ? -3 : 2)); - const op = { operation: std.add, identityElement: 0 }; - return runAndCompareTyped(arr, op, d.i32, false); -} - -async function testI32Max(): Promise { - const arr = Array.from({ length: 257 }, (_, i) => ((i * 37) % 513) - 256); - const op = { operation: std.max, identityElement: -2147483647 }; - return runAndCompareTyped(arr, op, d.i32, true); -} - -// composition tests - -async function testRecordsIntoUserEncoder(): Promise { - const arr = Array.from({ length: 4099 }, () => 1); - const input = root.createBuffer(d.arrayOf(d.u32, arr.length), arr).$usage('storage'); - - const computer = createPrefixScanComputer(root, { - operation: std.add, - identityElement: 0, - dataType: d.u32, - }); - const plan = computer.prepare(input); - - const encoder = root.device.createCommandEncoder(); - plan.run({ encoder }); - root.device.queue.submit([encoder.finish()]); - - const actual = await input.read(); - const expected = prefixScanJS(arr, { operation: std.add, identityElement: 0 }); - plan.destroy(); - return compareAndLog(actual, expected); -} - -async function testRecordsIntoUserPass(): Promise { - const arr = Array.from({ length: 4099 }, () => 2); - const input = root.createBuffer(d.arrayOf(d.u32, arr.length), arr).$usage('storage'); - - const computer = createPrefixScanComputer(root, { - operation: std.add, - identityElement: 0, - dataType: d.u32, - }); - const plan = computer.prepare(input); - - const encoder = root.device.createCommandEncoder(); - const pass = encoder.beginComputePass(); - plan.run({ pass }); - pass.end(); - root.device.queue.submit([encoder.finish()]); - - const actual = await input.read(); - const expected = prefixScanJS(arr, { operation: std.add, identityElement: 0 }); - plan.destroy(); - return compareAndLog(actual, expected); -} - -// benchmark - -const BENCH_SIZES = [2_048, 65_536, 1_048_576, 16_777_216]; -const BENCH_WARMUP = 3; -const BENCH_RUNS = 10; - -async function benchmarkSize(size: number): Promise { - const inputData = Array.from({ length: size }, () => 1); - const inputBuffer = root.createBuffer(d.arrayOf(d.f32, size), inputData).$usage('storage'); - const outputBuffer = root.createBuffer(d.arrayOf(d.f32, size)).$usage('storage'); - - for (let i = 0; i < BENCH_WARMUP; i++) { - prefixScan(root, { inputBuffer, outputBuffer, operation: addFn, identityElement: 0 }); - await root.device.queue.onSubmittedWorkDone(); - } - - let total = 0; - for (let i = 0; i < BENCH_RUNS; i++) { - const t0 = performance.now(); - prefixScan(root, { inputBuffer, outputBuffer, operation: addFn, identityElement: 0 }); - await root.device.queue.onSubmittedWorkDone(); - total += performance.now() - t0; - } - - const avgMs = total / BENCH_RUNS; - inputBuffer.destroy(); - outputBuffer.destroy(); - return avgMs; -} - -async function runBenchmarks(): Promise { - console.log('=== Prefix Scan Benchmark ==='); - for (const size of BENCH_SIZES) { - const avgMs = await benchmarkSize(size); - console.log( - ` size ${size.toLocaleString().padStart(12)}: ${avgMs.toFixed(2)} ms avg (${BENCH_RUNS} runs)`, - ); - } - console.log('=============================='); -} - // running the tests -async function runTest(name: string, fn: () => Promise): Promise { - const passed = await fn(); - if (!passed) { - console.error(`FAILED: ${name}`); - } - return passed; -} - async function runTests(): Promise { let result = true; - result = (await runTest('testAdd8', testAdd8)) && result; - result = (await runTest('testAdd123', testAdd123)) && result; - result = (await runTest('testMul', testMul)) && result; - result = (await runTest('testStdMax', testStdMax)) && result; - result = (await runTest('testConcat', testConcat)) && result; - result = (await runTest('testLength1', testLength1)) && result; - result = (await runTest('testLength65537', testLength65537)) && result; - result = (await runTest('testLength16777217', testLength16777217)) && result; - result = (await runTest('testDoesNotDestroyBuffer', testDoesNotDestroyBuffer)) && result; - result = (await runTest('testDoesNotCacheBuffers', testDoesNotCacheBuffers)) && result; - - result = (await runTest('testPrefixAdd8', testPrefixAdd8)) && result; - result = (await runTest('testPrefixAdd123', testPrefixAdd123)) && result; - result = (await runTest('testPrefixMul', testPrefixMul)) && result; - result = (await runTest('testPrefixStdMax', testPrefixStdMax)) && result; - result = (await runTest('testPrefixConcat', testPrefixConcat)) && result; - result = (await runTest('testPrefixLength1', testPrefixLength1)) && result; - result = (await runTest('testPrefixLength65537', testPrefixLength65537)) && result; - result = (await runTest('testPrefixLength16777217', testPrefixLength16777217)) && result; - result = - (await runTest('testPrefixDoesNotDestroyBuffer', testPrefixDoesNotDestroyBuffer)) && result; - result = - (await runTest('testPrefixDoesNotCacheBuffers', testPrefixDoesNotCacheBuffers)) && result; - - result = (await runTest('testU32Prefix65537', testU32Prefix65537)) && result; - result = (await runTest('testU32Reduce', testU32Reduce)) && result; - result = (await runTest('testI32PrefixNegatives', testI32PrefixNegatives)) && result; - result = (await runTest('testI32Max', testI32Max)) && result; - result = (await runTest('testRecordsIntoUserEncoder', testRecordsIntoUserEncoder)) && result; - result = (await runTest('testRecordsIntoUserPass', testRecordsIntoUserPass)) && result; + result = (await testAdd8()) && result; + result = (await testAdd123()) && result; + result = (await testMul()) && result; + result = (await testStdMax()) && result; + result = (await testConcat()) && result; + result = (await testLength1()) && result; + result = (await testLength65537()) && result; + result = (await testLength16777217()) && result; + result = (await testDoesNotDestroyBuffer()) && result; + result = (await testDoesNotCacheBuffers()) && result; + + result = (await testPrefixAdd8()) && result; + result = (await testPrefixAdd123()) && result; + result = (await testPrefixMul()) && result; + result = (await testPrefixStdMax()) && result; + result = (await testPrefixConcat()) && result; + result = (await testPrefixLength1()) && result; + result = (await testPrefixLength65537()) && result; + result = (await testPrefixLength16777217()) && result; + result = (await testPrefixDoesNotDestroyBuffer()) && result; + result = (await testPrefixDoesNotCacheBuffers()) && result; return result; } -const maybeTable = document.querySelector('.result'); -if (!maybeTable) { +const table = document.querySelector('.result'); +if (!table) { throw new Error('Nowhere to display the results'); } -const table: HTMLDivElement = maybeTable; - -let testsPassed: boolean | null = null; -let benchmarkPromise: Promise | null = null; - void runTests().then((result) => { - testsPassed = result; table.innerText = `Tests ${result ? 'succeeded' : 'failed'}.`; }); -async function startBenchmarks(): Promise { - if (testsPassed === null) { - table.innerText = 'Tests are still running.'; - return; - } - - if (!testsPassed) { - table.innerText = 'Tests failed. Benchmarks skipped.'; - return; - } - - if (benchmarkPromise) { - return benchmarkPromise; - } - - table.innerText = 'Tests succeeded. Running benchmarks...'; - benchmarkPromise = runBenchmarks() - .then(() => { - table.innerText = 'Tests succeeded. Benchmark complete (see console).'; - }) - .finally(() => { - benchmarkPromise = null; - }); - - return benchmarkPromise; -} - // #region Example controls and cleanup -export const controls = defineControls({ - 'Run benchmarks': { - onButtonClick: startBenchmarks, - }, -}); - export function onCleanup() { root.destroy(); } diff --git a/apps/typegpu-docs/src/examples/tests/sort/index.html b/apps/typegpu-docs/src/examples/tests/sort/index.html deleted file mode 100644 index ebebada879..0000000000 --- a/apps/typegpu-docs/src/examples/tests/sort/index.html +++ /dev/null @@ -1 +0,0 @@ -

Wait for the tests to finish running...
diff --git a/apps/typegpu-docs/src/examples/tests/sort/index.ts b/apps/typegpu-docs/src/examples/tests/sort/index.ts deleted file mode 100644 index a9e17909c0..0000000000 --- a/apps/typegpu-docs/src/examples/tests/sort/index.ts +++ /dev/null @@ -1,511 +0,0 @@ -import { tgpu } from 'typegpu'; -import * as d from 'typegpu/data'; -import * as std from 'typegpu/std'; -import { - createBitonicSorter, - createRadixSorter, - decomposeWorkgroups, - type RadixSorter, - scan, - type ScanBuffer, -} from '@typegpu/sort'; -import { randf } from '@typegpu/noise'; -import { defineControls } from '../../common/defineControls.ts'; - -const maxBufferSize = await navigator.gpu.requestAdapter().then((adapter) => { - if (!adapter) { - throw new Error('No GPU adapter found'); - } - return Math.min(adapter.limits.maxStorageBufferBindingSize, adapter.limits.maxBufferSize); -}); - -const root = await tgpu.init({ - device: { - requiredLimits: { - maxStorageBufferBindingSize: maxBufferSize, - maxBufferSize: maxBufferSize, - }, - }, -}); - -// reference implementations & comparison - -function refSortNumeric(arr: number[], direction: 'ascending' | 'descending'): number[] { - const out = arr.toSorted((a, b) => (a < b ? -1 : a > b ? 1 : 0)); - if (direction === 'descending') { - out.reverse(); - } - return out; -} - -function refSortF32WithNaNs(arr: number[]): number[] { - const nanCount = arr.filter((v) => Number.isNaN(v)).length; - const rest = arr.filter((v) => !Number.isNaN(v)).toSorted((a, b) => (a < b ? -1 : a > b ? 1 : 0)); - return rest.concat(Array.from({ length: nanCount }, () => Number.NaN)); -} - -function compareAndLog(actual: number[], expected: number[]): boolean { - const equal = - actual.length === expected.length && - actual.every((v, i) => v === expected[i] || (Number.isNaN(v) && Number.isNaN(expected[i]))); - if (equal) { - return true; - } - - if (actual.length !== expected.length) { - console.error(` Mismatch: length ${actual.length} !== ${expected.length}`); - } else { - const idx = actual.findIndex( - (v, i) => v !== expected[i] && !(Number.isNaN(v) && Number.isNaN(expected[i])), - ); - const lo = Math.max(0, idx - 2); - const hi = Math.min(actual.length, idx + 3); - console.error( - ` first mismatch at index ${idx} (showing [${lo}..${hi - 1}] of ${actual.length}):`, - ); - console.error(' actual: ', actual.slice(lo, hi)); - console.error(' expected:', expected.slice(lo, hi)); - } - return false; -} - -// input generators - -function randomU32Array(length: number): number[] { - return Array.from({ length }, () => Math.floor(Math.random() * 0x100000000) >>> 0); -} - -function randomI32Array(length: number): number[] { - const arr = Array.from({ length }, () => Math.floor(Math.random() * 0x100000000) | 0); - if (length >= 4) { - arr[0] = -2147483648; - arr[1] = 2147483647; - arr[2] = 0; - arr[3] = -1; - } - return arr; -} - -function randomF32Array(length: number): number[] { - const arr = Array.from({ length }, () => - Math.fround((Math.random() - 0.5) * 2 ** (Math.random() * 80 - 40)), - ); - if (length >= 6) { - arr[0] = Number.POSITIVE_INFINITY; - arr[1] = Number.NEGATIVE_INFINITY; - arr[2] = 0; - arr[3] = -0; - arr[4] = Math.fround(1e-45); - arr[5] = Math.fround(-1e-45); - } - return arr; -} - -// radix test runners - -type Root = typeof root; -type KeySchema = d.U32 | d.I32 | d.F32; - -async function runRadixAndRead( - root: Root, - keySchema: KeySchema, - arr: number[], - direction: 'ascending' | 'descending', -): Promise { - const buffer = root - .createBuffer(d.arrayOf(keySchema as d.U32, arr.length), arr) - .$usage('storage'); - const sorter = createRadixSorter(root, buffer, { direction }); - sorter.run(); - const actual = await buffer.read(); - sorter.destroy(); - buffer.destroy(); - return actual; -} - -async function testRadixU32Sizes(root: Root): Promise { - const sizes = [1, 2, 3, 255, 256, 257, 2047, 2048, 2049, 65537]; - let ok = true; - for (const size of sizes) { - const arr = randomU32Array(size); - const actual = await runRadixAndRead(root, d.u32, arr, 'ascending'); - if (!compareAndLog(actual, refSortNumeric(arr, 'ascending'))) { - console.error(` (size ${size})`); - ok = false; - } - } - return ok; -} - -async function testRadixU32Large(root: Root): Promise { - const arr = randomU32Array(1048577); - const actual = await runRadixAndRead(root, d.u32, arr, 'ascending'); - return compareAndLog(actual, refSortNumeric(arr, 'ascending')); -} - -async function testRadixU32Descending(root: Root): Promise { - const arr = randomU32Array(65537); - const actual = await runRadixAndRead(root, d.u32, arr, 'descending'); - return compareAndLog(actual, refSortNumeric(arr, 'descending')); -} - -async function testRadixI32(root: Root): Promise { - const arr = randomI32Array(65537); - const actual = await runRadixAndRead(root, d.i32, arr, 'ascending'); - return compareAndLog(actual, refSortNumeric(arr, 'ascending')); -} - -async function testRadixI32Descending(root: Root): Promise { - const arr = randomI32Array(4099); - const actual = await runRadixAndRead(root, d.i32, arr, 'descending'); - return compareAndLog(actual, refSortNumeric(arr, 'descending')); -} - -async function testRadixF32(root: Root): Promise { - const arr = randomF32Array(65537); - const actual = await runRadixAndRead(root, d.f32, arr, 'ascending'); - return compareAndLog(actual, refSortNumeric(arr, 'ascending')); -} - -async function testRadixF32Descending(root: Root): Promise { - const arr = randomF32Array(4099); - const actual = await runRadixAndRead(root, d.f32, arr, 'descending'); - return compareAndLog(actual, refSortNumeric(arr, 'descending')); -} - -async function testRadixF32NaNs(root: Root): Promise { - const arr = randomF32Array(4099); - for (let i = 0; i < 7; i++) { - arr[100 + i * 500] = Number.NaN; - } - const actual = await runRadixAndRead(root, d.f32, arr, 'ascending'); - return compareAndLog(actual, refSortF32WithNaNs(arr)); -} - -async function testRadixRunTwice(root: Root): Promise { - const arr = randomU32Array(65537); - const buffer = root.createBuffer(d.arrayOf(d.u32, arr.length), arr).$usage('storage'); - const sorter = createRadixSorter(root, buffer); - sorter.run(); - sorter.run(); - const actual = await buffer.read(); - sorter.destroy(); - buffer.destroy(); - return compareAndLog(actual, refSortNumeric(arr, 'ascending')); -} - -async function testRadixStability(root: Root): Promise { - const length = 100003; - const keys = Array.from({ length }, () => Math.floor(Math.random() * 64)); - const values = Array.from({ length }, (_, i) => i); - - const keyBuffer = root.createBuffer(d.arrayOf(d.u32, length), keys).$usage('storage'); - const valueBuffer = root.createBuffer(d.arrayOf(d.u32, length), values).$usage('storage'); - const sorter = createRadixSorter(root, keyBuffer, { values: valueBuffer }); - sorter.run(); - const actualKeys = await keyBuffer.read(); - const actualValues = await valueBuffer.read(); - sorter.destroy(); - keyBuffer.destroy(); - valueBuffer.destroy(); - - if (!compareAndLog(actualKeys, refSortNumeric(keys, 'ascending'))) { - return false; - } - - for (let i = 1; i < length; i++) { - const [key, value] = [actualKeys[i] as number, actualValues[i] as number]; - const [prevKey, prevValue] = [actualKeys[i - 1] as number, actualValues[i - 1] as number]; - if ((keys[value] as number) !== key) { - console.error(` value ${value} at index ${i} does not map back to key ${key}`); - return false; - } - if (key === prevKey && value <= prevValue) { - console.error(` stability violated at index ${i}: values ${prevValue} -> ${value}`); - return false; - } - } - return true; -} - -async function testRadixVec4fPayload(root: Root): Promise { - const length = 10000; - const keys = Array.from({ length }, () => Math.floor(Math.random() * 256)); - const values = Array.from({ length }, (_, i) => d.vec4f(i, 0, 0, 0)); - - const keyBuffer = root.createBuffer(d.arrayOf(d.u32, length), keys).$usage('storage'); - const valueBuffer = root.createBuffer(d.arrayOf(d.vec4f, length), values).$usage('storage'); - const sorter = createRadixSorter(root, keyBuffer, { values: valueBuffer }); - sorter.run(); - const actualKeys = await keyBuffer.read(); - const actualValues = await valueBuffer.read(); - sorter.destroy(); - keyBuffer.destroy(); - valueBuffer.destroy(); - - const expectedOrder = keys - .map((key, i) => ({ key, i })) - .toSorted((a, b) => a.key - b.key) - .map((entry) => entry.i); - - return ( - compareAndLog(actualKeys, refSortNumeric(keys, 'ascending')) && - compareAndLog( - actualValues.map((v) => v.x), - expectedOrder, - ) - ); -} - -// composition tests - -async function testRadixIntoUserEncoder(root: Root): Promise { - const arr = randomU32Array(4099); - const buffer = root.createBuffer(d.arrayOf(d.u32, arr.length), arr).$usage('storage'); - const sorter = createRadixSorter(root, buffer); - - const encoder = root.device.createCommandEncoder(); - sorter.run({ encoder }); - root.device.queue.submit([encoder.finish()]); - - const actual = await buffer.read(); - sorter.destroy(); - buffer.destroy(); - return compareAndLog(actual, refSortNumeric(arr, 'ascending')); -} - -async function testTwoSortersIntoUserPass(root: Root): Promise { - const arrA = randomU32Array(4099); - const arrB = randomF32Array(2048); - const bufferA = root.createBuffer(d.arrayOf(d.u32, arrA.length), arrA).$usage('storage'); - const bufferB = root.createBuffer(d.arrayOf(d.f32, arrB.length), arrB).$usage('storage'); - const sorterA = createRadixSorter(root, bufferA); - const sorterB = createRadixSorter(root, bufferB, { direction: 'descending' }); - - const encoder = root.device.createCommandEncoder(); - const pass = encoder.beginComputePass(); - sorterA.run({ pass }); - sorterB.run({ pass }); - pass.end(); - root.device.queue.submit([encoder.finish()]); - - const actualA = await bufferA.read(); - const actualB = await bufferB.read(); - sorterA.destroy(); - sorterB.destroy(); - bufferA.destroy(); - bufferB.destroy(); - return ( - compareAndLog(actualA, refSortNumeric(arrA, 'ascending')) && - compareAndLog(actualB, refSortNumeric(arrB, 'descending')) - ); -} - -// bitonic sanity tests - -async function testBitonicU32(root: Root): Promise { - const arr = randomU32Array(1000); - const buffer = root.createBuffer(d.arrayOf(d.u32, arr.length), arr).$usage('storage'); - const sorter = createBitonicSorter(root, buffer); - sorter.run(); - const actual = await buffer.read(); - sorter.destroy(); - buffer.destroy(); - return compareAndLog(actual, refSortNumeric(arr, 'ascending')); -} - -async function testBitonicDescending(root: Root): Promise { - const arr = randomU32Array(4096); - const buffer = root.createBuffer(d.arrayOf(d.u32, arr.length), arr).$usage('storage'); - const sorter = createBitonicSorter(root, buffer, { - compare: (a, b) => { - 'use gpu'; - return a > b; - }, - paddingValue: 0, - }); - sorter.run(); - const actual = await buffer.read(); - sorter.destroy(); - buffer.destroy(); - return compareAndLog(actual, refSortNumeric(arr, 'descending')); -} - -// huge buffer test (past the 65535 workgroups-per-dimension limit), GPU-verified - -const WORKGROUP_SIZE = 256; - -const fillLayout = tgpu.bindGroupLayout({ - data: { storage: d.arrayOf(d.u32), access: 'mutable' }, -}); - -const fillKernel = tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - gid: d.builtin.globalInvocationId, - numWorkgroups: d.builtin.numWorkgroups, - }, -})((input) => { - const spanX = input.numWorkgroups.x * WORKGROUP_SIZE; - const spanY = input.numWorkgroups.y * spanX; - const idx = input.gid.x + input.gid.y * spanX + input.gid.z * spanY; - if (idx >= fillLayout.$.data.length) { - return; - } - randf.seed3(d.vec3f(d.f32(idx & 0xffff), d.f32(idx >> 16), 0.5)); - const hi = d.u32(std.floor(randf.sample() * 65536)); - const lo = d.u32(std.floor(randf.sample() * 65536)); - fillLayout.$.data[idx] = (hi << 16) | lo; -}); - -const checkLayout = tgpu.bindGroupLayout({ - data: { storage: d.arrayOf(d.u32), access: 'readonly' }, - descents: { storage: d.arrayOf(d.atomic(d.u32)), access: 'mutable' }, -}); - -const checkSortedKernel = tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - gid: d.builtin.globalInvocationId, - numWorkgroups: d.builtin.numWorkgroups, - }, -})((input) => { - const spanX = input.numWorkgroups.x * WORKGROUP_SIZE; - const spanY = input.numWorkgroups.y * spanX; - const idx = input.gid.x + input.gid.y * spanX + input.gid.z * spanY; - if (idx + 1 >= checkLayout.$.data.length) { - return; - } - if ((checkLayout.$.data[idx] as number) > (checkLayout.$.data[idx + 1] as number)) { - std.atomicAdd(checkLayout.$.descents[0] as d.atomicU32, 1); - } -}); - -const fillPipeline = root.createComputePipeline({ compute: fillKernel }); -const checkPipeline = root.createComputePipeline({ compute: checkSortedKernel }); - -async function wrappingSum(root: Root, buffer: ScanBuffer) { - const result = scan(root, { - inputBuffer: buffer, - operation: std.add, - identityElement: 0, - dataType: d.u32, - }); - return (await result.read())[0]; -} - -async function testHugeBuffer(): Promise { - const length = 2 ** 27 + 5; - if (length * 4 > maxBufferSize) { - console.warn(`Skipping huge buffer test: needs ${length * 4} bytes, limit ${maxBufferSize}`); - return true; - } - - const buffer = root.createBuffer(d.arrayOf(d.u32, length)).$usage('storage'); - const descents = root.createBuffer(d.arrayOf(d.atomic(d.u32), 1), [0]).$usage('storage'); - let sorter: RadixSorter | null = null; - - try { - fillPipeline - .with(root.createBindGroup(fillLayout, { data: buffer })) - .dispatchWorkgroups(...decomposeWorkgroups(Math.ceil(length / WORKGROUP_SIZE))); - - const sumBefore = await wrappingSum(root, buffer); - - sorter = createRadixSorter(root, buffer); - sorter.run(); - - const sumAfter = await wrappingSum(root, buffer); - - checkPipeline - .with(root.createBindGroup(checkLayout, { data: buffer, descents })) - .dispatchWorkgroups(...decomposeWorkgroups(Math.ceil(length / WORKGROUP_SIZE))); - const descentCount = (await descents.read())[0] as unknown as number; - - if (descentCount !== 0) { - console.error(` huge buffer not sorted: ${descentCount} descending adjacent pairs`); - return false; - } - if (sumBefore !== sumAfter) { - console.error(` huge buffer checksum mismatch: ${sumBefore} !== ${sumAfter}`); - return false; - } - return true; - } finally { - sorter?.destroy(); - buffer.destroy(); - descents.destroy(); - } -} - -// running the tests - -async function runTest(name: string, fn: () => Promise): Promise { - const passed = await fn(); - if (!passed) { - console.error(`FAILED: ${name}`); - } - return passed; -} - -const radixTests = { - testRadixU32Sizes, - testRadixU32Large, - testRadixU32Descending, - testRadixI32, - testRadixI32Descending, - testRadixF32, - testRadixF32Descending, - testRadixF32NaNs, - testRadixRunTwice, - testRadixStability, - testRadixVec4fPayload, - testRadixIntoUserEncoder, - testTwoSortersIntoUserPass, -}; - -async function runTests(): Promise { - let result = true; - - for (const [name, fn] of Object.entries(radixTests)) { - result = (await runTest(name, () => fn(root))) && result; - } - - result = (await runTest('testBitonicU32', () => testBitonicU32(root))) && result; - result = (await runTest('testBitonicDescending', () => testBitonicDescending(root))) && result; - - return result; -} - -const maybeTable = document.querySelector('.result'); -if (!maybeTable) { - throw new Error('Nowhere to display the results'); -} -const table: HTMLDivElement = maybeTable; - -let testsPassed: boolean | null = null; - -void runTests().then((result) => { - testsPassed = result; - table.innerText = `Tests ${result ? 'succeeded' : 'failed'}.`; - console.log(`Sort tests ${result ? 'succeeded' : 'FAILED'}.`); -}); - -async function runHugeTest(): Promise { - if (testsPassed === null) { - table.innerText = 'Tests are still running.'; - return; - } - table.innerText = 'Running huge buffer test (537MB, >65535 tiles)...'; - const passed = await runTest('testHugeBuffer', testHugeBuffer); - table.innerText = `Huge buffer test ${passed ? 'succeeded' : 'failed'}.`; - console.log(`Huge buffer test ${passed ? 'succeeded' : 'FAILED'}.`); -} - -export const controls = defineControls({ - 'Test Huge Buffer': { onButtonClick: runHugeTest }, -}); - -export function onCleanup() { - root.destroy(); -} diff --git a/apps/typegpu-docs/src/examples/tests/sort/meta.json b/apps/typegpu-docs/src/examples/tests/sort/meta.json deleted file mode 100644 index d441e67059..0000000000 --- a/apps/typegpu-docs/src/examples/tests/sort/meta.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "title": "Sort Tests", - "category": "tests", - "tags": ["ecosystem", "test"], - "dev": true, - "coolFactor": 1 -} diff --git a/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts b/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts index fb5f6b1ac0..fa69064415 100644 --- a/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts +++ b/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts @@ -9,7 +9,7 @@ import { runExampleTest, setupCommonMocks } from './utils/baseTest.ts'; describe('sort example', () => { setupCommonMocks(); - it('should produce valid code', async ({ device }) => { + it('resolves the sort and render pipelines', async ({ device }) => { const shaderCodes = await runExampleTest( { category: 'algorithms', @@ -20,269 +20,18 @@ describe('sort example', () => { device, ); - expect(shaderCodes).toMatchInlineSnapshot(` - "fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { - return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); - } - - struct copyParamsType { - srcLength: u32, - dstLength: u32, - paddingValue: u32, - } - - @group(0) @binding(2) var params: copyParamsType; - - @group(0) @binding(1) var dst: array; - - @group(0) @binding(0) var src: array; - - @compute @workgroup_size(256) fn copyPadKernel(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { - let idx = ((flatWorkgroupIndex(wid, numWorkgroups) * 256u) + lid.x); - if ((idx >= params.dstLength)) { - return; - } - dst[idx] = select(params.paddingValue, src[idx], (idx < params.srcLength)); - } - - fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { - return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); - } - - @group(0) @binding(0) var data: array; - - var localKeys: array; - - fn loadShared(base: u32, tid: u32) { - localKeys[tid] = data[(base + tid)]; - localKeys[(tid + 256u)] = data[((base + tid) + 256u)]; - } - - fn defaultCompare(a: u32, b: u32) -> bool { - return (a < b); - } - - fn swapLocalValues(_arg_0: u32, _arg_1: u32) { - - } - - fn exchangeLocal(base: u32, iLocal: u32, stride: u32, k: u32) { - let ixjLocal = (iLocal + stride); - let left = localKeys[iLocal]; - let right = localKeys[ixjLocal]; - let ascending = (((base + iLocal) & k) == 0u); - let leftFirst = defaultCompare(left, right); - let shouldSwap = select(leftFirst, !leftFirst, ascending); - if (shouldSwap) { - localKeys[iLocal] = right; - localKeys[ixjLocal] = left; - swapLocalValues(iLocal, ixjLocal); - } - } - - fn mergeDown(base: u32, tid: u32, startShift: u32, k: u32) { - for (var jShift = startShift; (jShift > 0u); jShift--) { - workgroupBarrier(); - let stride = (1u << (jShift - 1u)); - let below = (tid & (stride - 1u)); - let above = (tid >> (jShift - 1u)); - let iLocal = (below + (above * (stride << 1u))); - exchangeLocal(base, iLocal, stride, k); - } - } - - fn storeShared(base: u32, tid: u32) { - data[(base + tid)] = localKeys[tid]; - data[((base + tid) + 256u)] = localKeys[(tid + 256u)]; - } - - @compute @workgroup_size(256) fn localSortKernel(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { - let tid = lid.x; - let base = (flatWorkgroupIndex(wid, numWorkgroups) * 512u); - if ((base >= arrayLength(&data))) { - return; - } - loadShared(base, tid); - for (var kShift = 1u; (kShift <= 9u); kShift++) { - mergeDown(base, tid, kShift, (1u << kShift)); - } - workgroupBarrier(); - storeShared(base, tid); - } - - fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { - return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); - } - - struct sortUniformsType { - k: u32, - jShift: u32, - } - - @group(0) @binding(1) var uniforms: sortUniformsType; - - @group(0) @binding(0) var data: array; - - fn defaultCompare(a: u32, b: u32) -> bool { - return (a < b); - } - - fn swapValues(_arg_0: u32, _arg_1: u32) { - - } - - @compute @workgroup_size(256) fn bitonicStepKernel(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { - let tid = ((flatWorkgroupIndex(wid, numWorkgroups) * 256u) + lid.x); - let k = uniforms.k; - let shift = uniforms.jShift; - let dataLength = arrayLength(&data); - let stride = (1u << shift); - let maskBelow = (stride - 1u); - let below = (tid & maskBelow); - let above = (tid >> shift); - let i = (below + (above * (stride << 1u))); - let ixj = (i + stride); - if ((ixj >= dataLength)) { - return; - } - let ascending = ((i & k) == 0u); - let left = data[i]; - let right = data[ixj]; - let leftFirst = defaultCompare(left, right); - let shouldSwap = select(leftFirst, !leftFirst, ascending); - if (shouldSwap) { - data[i] = right; - data[ixj] = left; - swapValues(i, ixj); - } - } - - fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { - return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); - } - - @group(0) @binding(0) var data: array; - - var localKeys: array; - - fn loadShared(base: u32, tid: u32) { - localKeys[tid] = data[(base + tid)]; - localKeys[(tid + 256u)] = data[((base + tid) + 256u)]; - } - - struct sortUniformsType { - k: u32, - jShift: u32, - } - - @group(0) @binding(1) var uniforms: sortUniformsType; - - fn defaultCompare(a: u32, b: u32) -> bool { - return (a < b); - } - - fn swapLocalValues(_arg_0: u32, _arg_1: u32) { - - } - - fn exchangeLocal(base: u32, iLocal: u32, stride: u32, k: u32) { - let ixjLocal = (iLocal + stride); - let left = localKeys[iLocal]; - let right = localKeys[ixjLocal]; - let ascending = (((base + iLocal) & k) == 0u); - let leftFirst = defaultCompare(left, right); - let shouldSwap = select(leftFirst, !leftFirst, ascending); - if (shouldSwap) { - localKeys[iLocal] = right; - localKeys[ixjLocal] = left; - swapLocalValues(iLocal, ixjLocal); - } - } - - fn mergeDown(base: u32, tid: u32, startShift: u32, k: u32) { - for (var jShift = startShift; (jShift > 0u); jShift--) { - workgroupBarrier(); - let stride = (1u << (jShift - 1u)); - let below = (tid & (stride - 1u)); - let above = (tid >> (jShift - 1u)); - let iLocal = (below + (above * (stride << 1u))); - exchangeLocal(base, iLocal, stride, k); - } - } - - fn storeShared(base: u32, tid: u32) { - data[(base + tid)] = localKeys[tid]; - data[((base + tid) + 256u)] = localKeys[(tid + 256u)]; - } - - @compute @workgroup_size(256) fn localMergeKernel(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { - let tid = lid.x; - let base = (flatWorkgroupIndex(wid, numWorkgroups) * 512u); - if ((base >= arrayLength(&data))) { - return; - } - loadShared(base, tid); - mergeDown(base, tid, 9u, uniforms.k); - workgroupBarrier(); - storeShared(base, tid); - } - - fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { - return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); - } - - struct copyParamsType { - srcLength: u32, - dstLength: u32, - paddingValue: u32, - } - - @group(0) @binding(2) var params: copyParamsType; - - @group(0) @binding(1) var dst: array; - - @group(0) @binding(0) var src: array; - - @compute @workgroup_size(256) fn copyBack(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { - let idx = ((flatWorkgroupIndex(wid, numWorkgroups) * 256u) + lid.x); - if ((idx < params.srcLength)) { - dst[idx] = src[idx]; - } - } - - struct fullScreenTriangle_Output { - @builtin(position) pos: vec4f, - @location(0) uv: vec2f, - } - - @vertex fn fullScreenTriangle(@builtin(vertex_index) vertexIndex: u32) -> fullScreenTriangle_Output { - const pos = array(vec2f(-1, -1), vec2f(3, -1), vec2f(-1, 3)); - const uv = array(vec2f(0, 1), vec2f(2, 1), vec2f(0, -1)); - - return fullScreenTriangle_Output(vec4f(pos[vertexIndex], 0, 1), uv[vertexIndex]); - } - - @group(0) @binding(0) var data_1: array; - - struct fragmentFn_Input { - @location(0) uv: vec2f, + expect({ + copyPad: shaderCodes.includes('@compute @workgroup_size(256) fn copyPadKernel'), + localSort: shaderCodes.includes('@compute @workgroup_size(256) fn localSortKernel'), + globalStep: shaderCodes.includes('@compute @workgroup_size(256) fn bitonicStepKernel'), + render: shaderCodes.includes('@fragment fn fragmentFn'), + }).toMatchInlineSnapshot(` + { + "copyPad": true, + "globalStep": true, + "localSort": true, + "render": true, } - - @fragment fn fragmentFn(_arg_0: fragmentFn_Input) -> @location(0) vec4f { - let data = (&data_1); - let arrayLength_1 = arrayLength(&(*data)); - let cols = u32(round(sqrt(f32(arrayLength_1)))); - let rows = u32(round((f32(arrayLength_1) / f32(cols)))); - let col = u32(floor((_arg_0.uv.x * f32(cols)))); - let row = u32(floor((_arg_0.uv.y * f32(rows)))); - let idx = ((row * cols) + col); - if ((idx >= arrayLength_1)) { - return vec4f(0.10000000149011612, 0.10000000149011612, 0.10000000149011612, 1); - } - let value = (*data)[idx]; - let normalized = (f32(value) / 255f); - return vec4f(normalized, normalized, normalized, 1f); - }" `); }); }); diff --git a/packages/typegpu-sort/README.md b/packages/typegpu-sort/README.md index 05d80b4eaf..3fa0dfe1cd 100644 --- a/packages/typegpu-sort/README.md +++ b/packages/typegpu-sort/README.md @@ -66,9 +66,9 @@ const sorter = createBitonicSorter(root, keys, { }); ``` -The bitonic sorter also accepts a `values` payload buffer, swapped alongside the -keys. With a payload, keys equal to the padding value may read back payloads -from padding slots, so the padding value should not occur among the keys. +The bitonic sorter also accepts a `values` payload buffer for power-of-two input +sizes, swapped alongside the keys. Use radix sort for arbitrary-length numeric +key/payload pairs. ## Prefix Scan @@ -90,7 +90,6 @@ const sums = prefixScan(root, { inputBuffer: u32Buffer, operation: std.add, identityElement: 0, - dataType: d.u32, }); ``` @@ -134,18 +133,6 @@ sorter.run({ pass }); pass.end(); ``` -## GPU timing - -With the `timestamp-query` feature enabled (not combinable with `pass`): - -```ts -const querySet = root.createQuerySet('timestamp', 2); -sorter.run({ querySet }); -querySet.resolve(); -const [start, end] = await querySet.read(); -const gpuTimeMs = Number(end - start) / 1_000_000; -``` - ## TypeGPU is created by Software Mansion [![swm](https://logo.swmansion.com/logo?color=white&variant=desktop&width=150&tag=typegpu-github 'Software Mansion')](https://swmansion.com) diff --git a/packages/typegpu-sort/src/bitonic/bitonicSort.ts b/packages/typegpu-sort/src/bitonic/bitonicSort.ts index 95aae31e5e..9f418817cf 100644 --- a/packages/typegpu-sort/src/bitonic/bitonicSort.ts +++ b/packages/typegpu-sort/src/bitonic/bitonicSort.ts @@ -101,11 +101,11 @@ function makeKernels(schemas: BitonicSchemas) { return; } - copyLayout.$.dst[idx] = std.select( - copyLayout.$.params.paddingValue as number, - copyLayout.$.src[idx] as number, - idx < copyLayout.$.params.srcLength, - ); + if (idx < copyLayout.$.params.srcLength) { + copyLayout.$.dst[idx] = copyLayout.$.src[idx] as number; + } else { + copyLayout.$.dst[idx] = copyLayout.$.params.paddingValue; + } }); function makeCopyBackKernel( @@ -162,7 +162,8 @@ function makeKernels(schemas: BitonicSchemas) { const right = sortLayout.$.data[ixj] as number; const leftFirst = compareSlot.$(left, right); - const shouldSwap = std.select(leftFirst, !leftFirst, ascending); + const rightFirst = compareSlot.$(right, left); + const shouldSwap = std.select(leftFirst, rightFirst, ascending); if (shouldSwap) { sortLayout.$.data[i] = right; @@ -246,7 +247,8 @@ function makeLocalKernels(schemas: BitonicSchemas, valueType?: d.AnyWgslData) { const right = localKeys.$[ixjLocal] as number; const ascending = ((base + iLocal) & k) === 0; const leftFirst = compareSlot.$(left, right); - const shouldSwap = std.select(leftFirst, !leftFirst, ascending); + const rightFirst = compareSlot.$(right, left); + const shouldSwap = std.select(leftFirst, rightFirst, ascending); if (shouldSwap) { localKeys.$[iLocal] = right; localKeys.$[ixjLocal] = left; @@ -339,6 +341,9 @@ export function createBitonicSorter< const valueBuffer = options?.values as ValueBuffer | undefined; const originalSize = keyBuffer.dataType.elementCount; + if (originalSize === 0) { + throw new Error('Cannot create a bitonic sorter for an empty buffer.'); + } const paddedSize = nextPowerOf2(originalSize); const wasPadded = paddedSize !== originalSize; @@ -347,6 +352,9 @@ export function createBitonicSorter< `The values buffer (${valueBuffer.dataType.elementCount} elements) must match the key buffer (${originalSize} elements).`, ); } + if (valueBuffer && wasPadded) { + throw new Error('Bitonic sorting with a values buffer requires a power-of-two element count.'); + } const keyType = keyBuffer.dataType.elementType; const paddingValue = options?.paddingValue ?? defaultPaddingValues[keyType.type]; diff --git a/packages/typegpu-sort/src/bitonic/types.ts b/packages/typegpu-sort/src/bitonic/types.ts index 22b0d4b0bb..31f3c95969 100644 --- a/packages/typegpu-sort/src/bitonic/types.ts +++ b/packages/typegpu-sort/src/bitonic/types.ts @@ -8,14 +8,13 @@ export interface BitonicSorterOptions { - const bits = std.bitcastF32toU32(v); + // Canonicalize signed zero so a stable numeric sort preserves its input order. + const bits = std.select(std.bitcastF32toU32(v), d.u32(0), v === 0); const mask = std.select(d.u32(0x80000000), d.u32(0xffffffff), bits >> 31 === 1); return ((bits ^ mask) >> shift) & (RADIX_SIZE - 1); }), diff --git a/packages/typegpu-sort/src/runPass.ts b/packages/typegpu-sort/src/runPass.ts index c52a206bd1..e28c610cf9 100644 --- a/packages/typegpu-sort/src/runPass.ts +++ b/packages/typegpu-sort/src/runPass.ts @@ -1,28 +1,26 @@ -import type { TgpuQuerySet } from 'typegpu'; - /** * Controls where a `run` call records its dispatches. */ -export interface RunPassOptions { - /** - * Timestamp query set (size >= 2) for GPU timing. The whole run is recorded - * into a single compute pass — timestamp 0 is written at its beginning and - * timestamp 1 at its end. Cannot be combined with `pass` (timestamps are a - * per-pass setting). - */ - querySet?: TgpuQuerySet<'timestamp'>; +interface EncoderOptions { /** * Record the dispatches into an existing command encoder (as a single compute * pass). Nothing is submitted — the caller owns the encoder. */ - encoder?: GPUCommandEncoder; + encoder: GPUCommandEncoder; + pass?: never; +} + +interface ExternalPassOptions { + encoder?: never; /** * Record the dispatches into an already-begun compute pass. Nothing is * submitted and the pass is not ended — the caller owns the pass. */ - pass?: GPUComputePassEncoder; + pass: GPUComputePassEncoder; } +export type RunPassOptions = EncoderOptions | ExternalPassOptions; + export interface RunPassRecording { pass: GPUComputePassEncoder; finish(): void; @@ -32,15 +30,11 @@ export interface RunPassRecording { * Resolves the compute pass that a `run` call should record into. All dispatches * of a run land in a single compute pass — standalone runs get their own encoder * and a single queue submit, encoder runs get one pass on the caller's encoder, - * and pass runs record straight into the caller's pass. Timestamps (when a - * querySet is given) wrap the whole pass via pass-level `timestampWrites`. + * and pass runs record straight into the caller's pass. */ export function beginRunPass(device: GPUDevice, options?: RunPassOptions): RunPassRecording { - const querySet = options?.querySet; - if (querySet && options?.pass) { - throw new Error( - 'A timestamp querySet cannot be used when recording into an existing compute pass.', - ); + if (options?.pass && options.encoder) { + throw new Error('A run cannot record into both an encoder and an existing compute pass.'); } const externalPass = options?.pass; @@ -49,17 +43,7 @@ export function beginRunPass(device: GPUDevice, options?: RunPassOptions): RunPa } const encoder = options?.encoder ?? device.createCommandEncoder(); - const pass = encoder.beginComputePass( - querySet - ? { - timestampWrites: { - querySet: querySet.querySet, - beginningOfPassWriteIndex: 0, - endOfPassWriteIndex: 1, - }, - } - : {}, - ); + const pass = encoder.beginComputePass(); return { pass, diff --git a/packages/typegpu-sort/src/scan/compute/scan.ts b/packages/typegpu-sort/src/scan/compute/scan.ts index 10582c0a04..5a69ec32bc 100644 --- a/packages/typegpu-sort/src/scan/compute/scan.ts +++ b/packages/typegpu-sort/src/scan/compute/scan.ts @@ -38,7 +38,7 @@ export function makeComputeBlock(schemas: ScanSchemas) { for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { if (baseIdx + i < scanLayout.$.input.length) { - partialSums[i] = operatorSlot.$(prev as number, scanLayout.$.input[baseIdx + i] as number); + partialSums[i] = operatorSlot.$(prev, scanLayout.$.input[baseIdx + i] as number); prev = partialSums[i]; lastIdx = i; } @@ -67,7 +67,7 @@ export function makeComputeBlock(schemas: ScanSchemas) { scanLayout.$.input[baseIdx + i] = scannedSum; } else { scanLayout.$.input[baseIdx + i] = operatorSlot.$( - scannedSum as number, + scannedSum, partialSums[i - 1] as number, ); } diff --git a/packages/typegpu-sort/src/scan/prefixScan.ts b/packages/typegpu-sort/src/scan/prefixScan.ts index 52235c3c6b..5f3c04160e 100644 --- a/packages/typegpu-sort/src/scan/prefixScan.ts +++ b/packages/typegpu-sort/src/scan/prefixScan.ts @@ -3,7 +3,6 @@ import { type StorageFlag, type TgpuBuffer, type TgpuComputePipeline, - type TgpuQuerySet, type TgpuRoot, } from 'typegpu'; import { decomposeWorkgroups } from '../bitonic/utils.ts'; @@ -148,6 +147,10 @@ export class PrefixScanComputer { let currentLength = buffer.dataType.elementCount; let resultBuffer = buffer; + if (currentLength === 0) { + throw new Error('Cannot scan an empty buffer.'); + } + for (;;) { const numWorkgroups = Math.ceil(currentLength / BLOCK_SIZE); const sumsBuffer = this.createScratchBuffer(numWorkgroups === 1 ? 1 : numWorkgroups); @@ -249,9 +252,6 @@ export class PrefixScanComputer { * Defaults to in-place (overwrites `inputBuffer`). * - operation: The binary operation to use for the scan (e.g., std.add) * - identityElement: The identity element for the operation (e.g., 0 for addition) - * - dataType: (optional) Element type of the buffers — `d.f32` (default), `d.u32` or `d.i32` - * @param querySet - Optional timestamp query set (size >= 2) for GPU timing. - * Index 0 gets the begin timestamp, index 1 gets the end timestamp. * @returns The output buffer instance which contains the scanned values. * * @example @@ -294,11 +294,9 @@ export function prefixScan( outputBuffer?: ScanBuffer; operation: BinaryOp['operation']; identityElement: BinaryOp['identityElement']; - dataType?: TElement; }, - querySet?: TgpuQuerySet<'timestamp'>, ): ScanBuffer { - return runScan(root, options, false, querySet); + return runScan(root, options, false); } /** @@ -311,9 +309,6 @@ export function prefixScan( * - inputBuffer: A storage buffer with the input values to reduce * - operation: The binary operation to use for the reduction (e.g., std.add) * - identityElement: The identity element for the operation (e.g., 0 for addition) - * - dataType: (optional) Element type of the buffers — `d.f32` (default), `d.u32` or `d.i32` - * @param querySet - Optional timestamp query set (size >= 2) for GPU timing. - * Index 0 gets the begin timestamp, index 1 gets the end timestamp. * @returns A buffer containing the aggregated reduction result (single-element buffer). * The buffer is owned by the internally cached scan plan and is reused by * subsequent `scan` calls on the same input buffer. @@ -354,11 +349,9 @@ export function scan( inputBuffer: ScanBuffer; operation: BinaryOp['operation']; identityElement: BinaryOp['identityElement']; - dataType?: TElement; }, - querySet?: TgpuQuerySet<'timestamp'>, ): ScanBuffer { - return runScan(root, options, true, querySet); + return runScan(root, options, true); } function runScan( @@ -368,29 +361,32 @@ function runScan( outputBuffer?: ScanBuffer; operation: BinaryOp['operation']; identityElement: BinaryOp['identityElement']; - dataType?: TElement; }, onlyGreatestElement: boolean, - querySet?: TgpuQuerySet<'timestamp'>, ): ScanBuffer { + const elementType = options.inputBuffer.dataType.elementType; const computer = createPrefixScanComputer(root, { operation: options.operation, identityElement: options.identityElement, - ...(options.dataType !== undefined && { dataType: options.dataType }), + dataType: elementType, }); - const runOptions = querySet ? { querySet } : undefined; - if (onlyGreatestElement) { - return computer.compute(options.inputBuffer, true, runOptions); + return computer.compute(options.inputBuffer, true); } const outputBuffer = options.outputBuffer ?? options.inputBuffer; if (options.inputBuffer !== outputBuffer) { - (outputBuffer as ScanBuffer).copyFrom(options.inputBuffer as ScanBuffer); + if ( + outputBuffer.dataType.elementType.type !== elementType.type || + outputBuffer.dataType.elementCount !== options.inputBuffer.dataType.elementCount + ) { + throw new Error('The input and output scan buffers must have the same type and length.'); + } + (outputBuffer as ScanBuffer).copyFrom(options.inputBuffer as ScanBuffer); } - return computer.compute(outputBuffer, false, runOptions); + return computer.compute(outputBuffer, false); } /** diff --git a/packages/typegpu-sort/tests/bitonic.test.ts b/packages/typegpu-sort/tests/bitonic.test.ts index 74b5f49dc5..c8f5c938ec 100644 --- a/packages/typegpu-sort/tests/bitonic.test.ts +++ b/packages/typegpu-sort/tests/bitonic.test.ts @@ -51,4 +51,20 @@ describe('bitonic sort', () => { createBitonicSorter(root, keys, { values }).run(); expect(getResolvedWgsl(device)).toContain('vals'); }); + + it('rejects payload sorting when padding would be required', ({ root }) => { + const keys = root.createBuffer(d.arrayOf(d.u32, 3)).$usage('storage'); + const values = root.createBuffer(d.arrayOf(d.u32, 3)).$usage('storage'); + + expect(() => createBitonicSorter(root, keys, { values })).toThrowErrorMatchingInlineSnapshot( + `[Error: Bitonic sorting with a values buffer requires a power-of-two element count.]`, + ); + }); + + it('rejects empty buffers', ({ root }) => { + const keys = root.createBuffer(d.arrayOf(d.u32, 0)).$usage('storage'); + expect(() => createBitonicSorter(root, keys)).toThrowErrorMatchingInlineSnapshot( + `[Error: Cannot create a bitonic sorter for an empty buffer.]`, + ); + }); }); diff --git a/packages/typegpu-sort/tests/radix.test.ts b/packages/typegpu-sort/tests/radix.test.ts index f75b1bf837..4054229ce0 100644 --- a/packages/typegpu-sort/tests/radix.test.ts +++ b/packages/typegpu-sort/tests/radix.test.ts @@ -26,6 +26,18 @@ describe('radix sort', () => { expect(wgsl).not.toContain('-2147483648i'); }); + it('canonicalizes signed zero before extracting f32 digits', ({ root, device }) => { + const data = root.createBuffer(d.arrayOf(d.f32, 512)).$usage('storage'); + createRadixSorter(root, data).run(); + + const canonicalization = getResolvedWgsl(device) + .split('\n') + .find((line) => line.includes('bitcast(v)')); + expect(canonicalization).toMatchInlineSnapshot( + `" let bits = select(bitcast(v), 0u, (v == 0f));"`, + ); + }); + it('allocates no new GPU resources on repeated runs', ({ root, device }) => { const data = root.createBuffer(d.arrayOf(d.u32, 4096)).$usage('storage'); const sorter = createRadixSorter(root, data); diff --git a/packages/typegpu-sort/tests/scan.test.ts b/packages/typegpu-sort/tests/scan.test.ts index 3f114e3d86..665999b909 100644 --- a/packages/typegpu-sort/tests/scan.test.ts +++ b/packages/typegpu-sort/tests/scan.test.ts @@ -10,7 +10,7 @@ describe('prefix scan', () => { for (const dataType of [d.u32, d.i32] as const) { const buffer = root.createBuffer(d.arrayOf(dataType, 4096)).$usage('storage'); - prefixScan(root, { inputBuffer: buffer, operation: std.add, identityElement: 0, dataType }); + prefixScan(root, { inputBuffer: buffer, operation: std.add, identityElement: 0 }); } const f32Buffer = root.createBuffer(d.arrayOf(d.f32, 4096)).$usage('storage'); prefixScan(root, { inputBuffer: f32Buffer, operation: std.add, identityElement: 0 }); @@ -45,7 +45,7 @@ describe('prefix scan', () => { expect(device.mock.createShaderModule.mock.calls.length).toBe(modulesAfterFirst); }); - it('throws when combining querySet with an external pass', ({ root }) => { + it('rejects ambiguous recording destinations', ({ root, device }) => { const computer = createPrefixScanComputer(root, { operation: std.add, identityElement: 0, @@ -56,10 +56,36 @@ describe('prefix scan', () => { const externalPass = { __brand: 'GPUComputePassEncoder', } as unknown as GPUComputePassEncoder; - const querySet = root.createQuerySet('timestamp', 2); + const encoder = device.createCommandEncoder(); - expect(() => plan.run({ pass: externalPass, querySet })).toThrowError( - /cannot be used when recording/, + expect(() => + plan.run({ pass: externalPass, encoder } as never), + ).toThrowErrorMatchingInlineSnapshot( + `[Error: A run cannot record into both an encoder and an existing compute pass.]`, + ); + }); + + it('rejects empty and mismatched buffers', ({ root }) => { + const computer = createPrefixScanComputer(root, { + operation: std.add, + identityElement: 0, + }); + const empty = root.createBuffer(d.arrayOf(d.f32, 0)).$usage('storage'); + expect(() => computer.prepare(empty)).toThrowErrorMatchingInlineSnapshot( + `[Error: Cannot scan an empty buffer.]`, + ); + + const input = root.createBuffer(d.arrayOf(d.u32, 4)).$usage('storage'); + const output = root.createBuffer(d.arrayOf(d.u32, 3)).$usage('storage'); + expect(() => + prefixScan(root, { + inputBuffer: input, + outputBuffer: output, + operation: std.add, + identityElement: 0, + }), + ).toThrowErrorMatchingInlineSnapshot( + `[Error: The input and output scan buffers must have the same type and length.]`, ); }); }); diff --git a/packages/typegpu/src/core/slot/accessor.ts b/packages/typegpu/src/core/slot/accessor.ts index b5e30cfb97..4ec091d9d5 100644 --- a/packages/typegpu/src/core/slot/accessor.ts +++ b/packages/typegpu/src/core/slot/accessor.ts @@ -193,7 +193,8 @@ export class TgpuAccessorImpl } get $(): InferGPU { - if (getResolutionCtx()) { + const ctx = getResolutionCtx(); + if (ctx && ctx.mode.type !== 'simulate') { return this[$gpuValueOf]; } @@ -217,7 +218,8 @@ export class TgpuMutableAccessorImpl } get $(): InferGPU { - if (getResolutionCtx()) { + const ctx = getResolutionCtx(); + if (ctx && ctx.mode.type !== 'simulate') { return this[$gpuValueOf]; } diff --git a/packages/typegpu/tests/accessor.test.ts b/packages/typegpu/tests/accessor.test.ts index 284d7a07c5..c0400af72b 100644 --- a/packages/typegpu/tests/accessor.test.ts +++ b/packages/typegpu/tests/accessor.test.ts @@ -540,6 +540,15 @@ describe('tgpu.accessor', () => { ); }); + it('throws when $ is accessed in simulation mode', () => { + const valueAccess = tgpu.accessor(d.f32, 1); + expect(() => + tgpu['~unstable'].simulate(() => valueAccess.$), + ).toThrowErrorMatchingInlineSnapshot( + `[Error: \`tgpu.accessor\` relies on GPU resources and cannot be accessed outside of a compute dispatch or draw call. Use \`tgpu.slot\` for non-WGSL values instead.]`, + ); + }); + it('allows for arbitrarily nested access functions', ({ root }) => { const counterMutable = root.createMutable(d.u32); From dfcc6710e842e24a19998da6295f6109321ea137 Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Sat, 25 Jul 2026 02:35:39 +0200 Subject: [PATCH 10/12] optimize and bring back timing --- .../src/examples/algorithms/sort/index.ts | 32 +++++++- packages/typegpu-sort/src/radix/count.ts | 19 ++++- packages/typegpu-sort/src/radix/radixSort.ts | 4 +- packages/typegpu-sort/src/radix/scatter.ts | 80 ++++++++++++------- packages/typegpu-sort/src/radix/schemas.ts | 8 +- 5 files changed, 104 insertions(+), 39 deletions(-) diff --git a/apps/typegpu-docs/src/examples/algorithms/sort/index.ts b/apps/typegpu-docs/src/examples/algorithms/sort/index.ts index 2d79988955..93750d4fbd 100644 --- a/apps/typegpu-docs/src/examples/algorithms/sort/index.ts +++ b/apps/typegpu-docs/src/examples/algorithms/sort/index.ts @@ -247,15 +247,43 @@ function pickSorter(): { sorter: BitonicSorter | RadixSorter; note: string } { return { sorter: bitonicSorters[state.sortOrder], note: '' }; } +/** + * Runs the sorter, timing it on the GPU when timestamp queries are available. + * Returns the elapsed GPU time in milliseconds, or null when unsupported. + */ +async function runSorterTimed(sorter: BitonicSorter | RadixSorter): Promise { + if (!querySet?.available) { + sorter.run(); + return null; + } + + const encoder = root.device.createCommandEncoder(); + const pass = encoder.beginComputePass({ + timestampWrites: { + querySet: querySet.querySet, + beginningOfPassWriteIndex: 0, + endOfPassWriteIndex: 1, + }, + }); + sorter.run({ pass }); + pass.end(); + root.device.queue.submit([encoder.finish()]); + + querySet.resolve(); + const [start, end] = await querySet.read(); + return Number(end - start) / 1_000_000; +} + async function sort() { const { sorter, note } = pickSorter(); showOverlay('Sorting...'); - sorter.run(); + const gpuTimeMs = await runSorterTimed(sorter); render(); - showOverlay(`✔ Sorted${note}`, false); + const timeStr = gpuTimeMs !== null ? ` in ${formatMs(gpuTimeMs)}` : ''; + showOverlay(`✔ Sorted${timeStr}${note}`, false); hideOverlay(); } diff --git a/packages/typegpu-sort/src/radix/count.ts b/packages/typegpu-sort/src/radix/count.ts index 889c7f7c29..56871f9c7a 100644 --- a/packages/typegpu-sort/src/radix/count.ts +++ b/packages/typegpu-sort/src/radix/count.ts @@ -10,8 +10,15 @@ import { wgHist, } from './schemas.ts'; -export function makeCountKernel(schemas: RadixSchemas) { +/** + * Counts digit occurrences per tile into a digit-major histogram. `elementCount` + * is baked into the kernel, so keys are bounds-checked only when the buffer does + * not divide evenly into tiles. + */ +export function makeCountKernel(schemas: RadixSchemas, elementCount: number) { const { ioLayout, digitFn } = schemas; + const needsBoundsCheck = elementCount % TILE_SIZE !== 0; + const lastIndex = elementCount - 1; return tgpu.computeFn({ workgroupSize: [TILE_THREADS], @@ -24,14 +31,20 @@ export function makeCountKernel(schemas: RadixSchemas) { const local_i = lid.x; const tile_id = flatWorkgroupIndex(wid, numWorkgroups); const tile_base = tile_id * TILE_SIZE; + const shift = paramsLayout.$.params.shift; std.atomicStore(wgHist.$[local_i] as d.atomicU32, 0); std.workgroupBarrier(); for (const k of tgpu.unroll(std.range(KEYS_PER_THREAD))) { const global_i = tile_base + k * TILE_THREADS + local_i; - if (global_i < ioLayout.$.src.length) { - const digit = digitFn(ioLayout.$.src[global_i] as number, paramsLayout.$.params.shift); + let load_i = global_i; + if (needsBoundsCheck) { + load_i = std.min(global_i, lastIndex); + } + const digit = digitFn(ioLayout.$.src[load_i] as number, shift); + const inBounds = needsBoundsCheck ? global_i < elementCount : true; + if (inBounds) { std.atomicAdd(wgHist.$[digit] as d.atomicU32, 1); } } diff --git a/packages/typegpu-sort/src/radix/radixSort.ts b/packages/typegpu-sort/src/radix/radixSort.ts index ca8d4257ce..f53185b838 100644 --- a/packages/typegpu-sort/src/radix/radixSort.ts +++ b/packages/typegpu-sort/src/radix/radixSort.ts @@ -84,9 +84,9 @@ export function createRadixSorter< }); const scanPlan = scanComputer.prepare(histBuffer); - const scatterPipeline = root.createComputePipeline({ compute: makeScatterKernel(schemas) }); + const scatterPipeline = root.createComputePipeline({ compute: makeScatterKernel(schemas, n) }); - const countPipeline = root.createComputePipeline({ compute: makeCountKernel(schemas) }); + const countPipeline = root.createComputePipeline({ compute: makeCountKernel(schemas, n) }); const histBg = root.createBindGroup(histLayout, { hist: histBuffer }); const paramBgs = paramBuffers.map((buffer) => diff --git a/packages/typegpu-sort/src/radix/scatter.ts b/packages/typegpu-sort/src/radix/scatter.ts index 1a949e0e8b..46aa5aa110 100644 --- a/packages/typegpu-sort/src/radix/scatter.ts +++ b/packages/typegpu-sort/src/radix/scatter.ts @@ -10,22 +10,33 @@ import { TILE_THREADS, } from './schemas.ts'; -const tileDigits = tgpu.workgroupVar(d.arrayOf(d.u32, TILE_THREADS)); const runningTotal = tgpu.workgroupVar(d.arrayOf(d.u32, RADIX_SIZE)); +const BITSET_WORD_BITS = 32; +const BITSET_WORDS = TILE_THREADS / BITSET_WORD_BITS; +const digitBits = tgpu.workgroupVar(d.arrayOf(d.atomic(d.u32), BITSET_WORDS * RADIX_SIZE)); /** * Stable scatter. Each workgroup owns a TILE_SIZE-element tile and processes it * in KEYS_PER_THREAD sequential rounds of TILE_THREADS elements, carrying a * per-digit running offset in workgroup memory so ranks stay stable across - * rounds. + * rounds. The offsets start at the tile's scanned histogram bases, so a round's + * output position is the offset plus the rank, with no per-key histogram read. * - * Within a round, each thread derives its rank by counting matching digits among - * the preceding threads, and doubles as the accountant for one digit value - * (digit == its local index), tallying that digit's total for the round in the - * same loop — O(TILE_THREADS) work per thread. + * Within a round, each invocation sets its bit in a per-digit workgroup bitset, + * then reads that digit's whole row once. The population count of the earlier + * bits is its stable rank, and the count over the full row is the digit's total + * for the round; the invocation that ranks first for a digit advances that + * digit's offset by the total. The bitset is word-major + * (`word * RADIX_SIZE + digit`), so the lanes of a SIMD group address + * consecutive words rather than a fixed stride. + * + * `elementCount` is baked into the kernel, so keys are bounds-checked only when + * the buffer does not divide evenly into tiles. */ -export function makeScatterKernel(schemas: RadixSchemas): TgpuComputeFn { +export function makeScatterKernel(schemas: RadixSchemas, elementCount: number): TgpuComputeFn { const { ioLayout, digitFn, writeOutput } = schemas; + const needsBoundsCheck = elementCount % TILE_SIZE !== 0; + const lastIndex = elementCount - 1; return tgpu.computeFn({ workgroupSize: [TILE_THREADS], @@ -38,42 +49,55 @@ export function makeScatterKernel(schemas: RadixSchemas): TgpuComputeFn { const local_i = lid.x; const tile_id = flatWorkgroupIndex(wid, numWorkgroups); const tile_base = tile_id * TILE_SIZE; + const bitset_word = local_i >> 5; + const bitset_mask = d.u32(1) << (local_i & (BITSET_WORD_BITS - 1)); + const earlierBits = bitset_mask - 1; + const shift = paramsLayout.$.params.shift; - runningTotal.$[local_i] = 0; + runningTotal.$[local_i] = histLayout.$.hist[ + local_i * paramsLayout.$.params.numTiles + tile_id + ] as number; for (const k of tgpu.unroll(std.range(KEYS_PER_THREAD))) { const global_i = tile_base + k * TILE_THREADS + local_i; - const inBounds = global_i < ioLayout.$.src.length; - let my_digit = d.u32(RADIX_SIZE); - if (inBounds) { - my_digit = digitFn(ioLayout.$.src[global_i] as number, paramsLayout.$.params.shift); + let load_i = global_i; + if (needsBoundsCheck) { + load_i = std.min(global_i, lastIndex); } + const key = ioLayout.$.src[load_i] as number; + const my_digit = digitFn(key, shift); + const inBounds = needsBoundsCheck ? global_i < elementCount : true; - tileDigits.$[local_i] = my_digit; + if (inBounds) { + std.atomicOr(digitBits.$[bitset_word * RADIX_SIZE + my_digit] as d.atomicU32, bitset_mask); + } std.workgroupBarrier(); let rank = d.u32(0); - let round_total = d.u32(0); - for (let j = d.u32(0); j < TILE_THREADS; j++) { - const digit_j = tileDigits.$[j] as number; - if (j < local_i && digit_j === my_digit) { - rank = rank + 1; - } - if (digit_j === local_i) { - round_total = round_total + 1; + let digit_total = d.u32(0); + if (inBounds) { + for (const word of tgpu.unroll(std.range(BITSET_WORDS))) { + const bits = std.atomicLoad(digitBits.$[word * RADIX_SIZE + my_digit] as d.atomicU32); + const mask = std.select( + std.select(d.u32(0), earlierBits, word === bitset_word), + d.u32(0xffffffff), + word < bitset_word, + ); + rank = rank + std.countOneBits(bits & mask); + digit_total = digit_total + std.countOneBits(bits); } + const output_pos = (runningTotal.$[my_digit] as number) + rank; + writeOutput(key, global_i, output_pos); } + std.workgroupBarrier(); if (inBounds) { - const output_pos = - (histLayout.$.hist[my_digit * paramsLayout.$.params.numTiles + tile_id] as number) + - (runningTotal.$[my_digit] as number) + - rank; - writeOutput(global_i, output_pos); + std.atomicStore(digitBits.$[bitset_word * RADIX_SIZE + my_digit] as d.atomicU32, 0); + if (rank === 0) { + runningTotal.$[my_digit] = (runningTotal.$[my_digit] as number) + digit_total; + } } std.workgroupBarrier(); - - runningTotal.$[local_i] = runningTotal.$[local_i] + round_total; } }); } diff --git a/packages/typegpu-sort/src/radix/schemas.ts b/packages/typegpu-sort/src/radix/schemas.ts index 055e10329c..1f78d92e06 100644 --- a/packages/typegpu-sort/src/radix/schemas.ts +++ b/packages/typegpu-sort/src/radix/schemas.ts @@ -96,12 +96,12 @@ export function makeRadixSchemas( const copyValue = valueType as unknown as (value: unknown) => number; const writeOutput = valuesLayout && valueType - ? tgpu.fn([d.u32, d.u32])((srcIdx, dstIdx) => { - (ioLayout.$.dst[dstIdx] as number) = ioLayout.$.src[srcIdx] as number; + ? tgpu.fn([keyType, d.u32, d.u32])((key, srcIdx, dstIdx) => { + (ioLayout.$.dst[dstIdx] as number) = key; (valuesLayout.$.dstVals[dstIdx] as number) = copyValue(valuesLayout.$.srcVals[srcIdx]); }) - : tgpu.fn([d.u32, d.u32])((srcIdx, dstIdx) => { - (ioLayout.$.dst[dstIdx] as number) = ioLayout.$.src[srcIdx] as number; + : tgpu.fn([keyType, d.u32, d.u32])((key, _srcIdx, dstIdx) => { + (ioLayout.$.dst[dstIdx] as number) = key; }); return { From eab131f73b84914713aa463749130d8482e6eb00 Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Sat, 25 Jul 2026 02:41:13 +0200 Subject: [PATCH 11/12] strip --- .../src/examples/algorithms/sort/index.ts | 4 --- packages/typegpu-sort/README.md | 31 +++++++--------- .../typegpu-sort/src/bitonic/bitonicSort.ts | 23 ++---------- packages/typegpu-sort/src/bitonic/types.ts | 9 ++--- packages/typegpu-sort/src/radix/count.ts | 5 --- packages/typegpu-sort/src/radix/radixSort.ts | 16 +++------ packages/typegpu-sort/src/radix/scatter.ts | 18 ---------- packages/typegpu-sort/src/radix/schemas.ts | 22 +----------- packages/typegpu-sort/src/radix/types.ts | 17 ++++----- packages/typegpu-sort/src/runPass.ts | 20 ++--------- packages/typegpu-sort/src/scan/prefixScan.ts | 36 ++++++++----------- packages/typegpu-sort/src/scan/types.ts | 2 +- packages/typegpu-sort/src/wgslUtils.ts | 4 --- 13 files changed, 50 insertions(+), 157 deletions(-) diff --git a/apps/typegpu-docs/src/examples/algorithms/sort/index.ts b/apps/typegpu-docs/src/examples/algorithms/sort/index.ts index 93750d4fbd..2fb432cb2d 100644 --- a/apps/typegpu-docs/src/examples/algorithms/sort/index.ts +++ b/apps/typegpu-docs/src/examples/algorithms/sort/index.ts @@ -247,10 +247,6 @@ function pickSorter(): { sorter: BitonicSorter | RadixSorter; note: string } { return { sorter: bitonicSorters[state.sortOrder], note: '' }; } -/** - * Runs the sorter, timing it on the GPU when timestamp queries are available. - * Returns the elapsed GPU time in milliseconds, or null when unsupported. - */ async function runSorterTimed(sorter: BitonicSorter | RadixSorter): Promise { if (!querySet?.available) { sorter.run(); diff --git a/packages/typegpu-sort/README.md b/packages/typegpu-sort/README.md index 3fa0dfe1cd..027bbc3c30 100644 --- a/packages/typegpu-sort/README.md +++ b/packages/typegpu-sort/README.md @@ -10,9 +10,8 @@ and composes with your own command encoders and compute passes. ## Radix Sort -The fast path — a stable 4-pass LSD radix sort. Keys are ordered by the natural -order of their type (i32 and f32 keys are handled via order-preserving bit -transforms; the buffers themselves are never transformed). +A stable LSD radix sort, and the fastest option here. Keys are ordered by the +natural order of their type. ```ts import { tgpu, d } from 'typegpu'; @@ -37,18 +36,16 @@ const sorter = createRadixSorter(root, keys, { }); ``` -All internal buffers, bind groups and pipelines are created once in -`createRadixSorter` — `run()` only records dispatches, so it is safe (and cheap) -to call every frame. +All GPU resources are created once in `createRadixSorter`, so `run()` only +records dispatches and is cheap to call every frame. -For f32 keys sorted ascending, NaNs with a cleared sign bit sort after +For `f32` keys sorted ascending, NaNs with a cleared sign bit sort after +Infinity and NaNs with a set sign bit sort before -Infinity. ## Bitonic Sort -Sorts with an arbitrary comparator — the bitonic sorter's advantage over radix -(which is limited to the natural order of the key type, but is considerably -faster). Arrays with non-power-of-2 lengths are padded automatically. +Sorts with an arbitrary comparator, which radix sort cannot do. Slower than +radix sort. Arrays with non-power-of-2 lengths are padded automatically. ```ts import { createBitonicSorter } from '@typegpu/sort'; @@ -62,7 +59,7 @@ Custom comparator (descending): ```ts const sorter = createBitonicSorter(root, keys, { compare: (a, b) => { 'use gpu'; return a > b; }, - paddingValue: 0, // must sort to the end — use the minimum value for descending + paddingValue: 0, // must sort to the end, so the minimum value for descending }); ``` @@ -93,9 +90,8 @@ const sums = prefixScan(root, { }); ``` -For repeated scans of the same buffer (e.g. per frame), prepare a plan once — -all scratch buffers and bind groups are allocated up front and `run()` only -records dispatches: +For repeated scans of the same buffer, prepare a plan once. All scratch buffers +and bind groups are allocated up front and `run()` only records dispatches: ```ts import { createPrefixScanComputer } from '@typegpu/sort'; @@ -112,13 +108,12 @@ plan.destroy(); ``` Note: passing `-2147483648` (i32 minimum) as `identityElement` currently -generates WGSL that does not compile — use `-2147483647` instead. +generates WGSL that does not compile. Use `-2147483647` instead. ## Composing with your own passes -Sorting is rarely standalone. Every `run()` (sorters and scan plans alike) -accepts an `encoder` or `pass` to record the work into your frame instead of -submitting on its own: +Every `run()`, on sorters and scan plans alike, accepts an `encoder` or `pass` +to record the work into instead of submitting on its own: ```ts const encoder = root.device.createCommandEncoder(); diff --git a/packages/typegpu-sort/src/bitonic/bitonicSort.ts b/packages/typegpu-sort/src/bitonic/bitonicSort.ts index 9f418817cf..a581ee5300 100644 --- a/packages/typegpu-sort/src/bitonic/bitonicSort.ts +++ b/packages/typegpu-sort/src/bitonic/bitonicSort.ts @@ -175,18 +175,6 @@ function makeKernels(schemas: BitonicSchemas) { return { copyPadKernel, copyBackKernel, valsCopyKernel, bitonicStepKernel }; } -/** - * Shared-memory kernels that run many compare-exchange substeps in one dispatch. - * Each workgroup loads a LOCAL_BLOCK-sized slice of the data into workgroup - * memory and performs every substep with stride < LOCAL_BLOCK there — those pairs - * never cross the block boundary. `localSortKernel` covers all phases with - * k <= LOCAL_BLOCK (a full bitonic sort of each block); `localMergeKernel` - * finishes a k-phase (k read from the uniforms) once the stride drops below the - * block size. Only used when the padded size is a multiple of LOCAL_BLOCK, so - * per-element bounds checks are not needed. The 3D dispatch grid may contain - * more workgroups than blocks, and whole workgroups past the last block exit - * early. - */ function makeLocalKernels(schemas: BitonicSchemas, valueType?: d.AnyWgslData) { const { keyType, sortLayout, valsLayout } = schemas; const copyValue = valueType as unknown as (value: unknown) => number; @@ -320,14 +308,9 @@ interface SortStep { } /** - * Create a bitonic sorter for the given key buffer (u32, i32 or f32 elements), - * optionally reordering a payload buffer alongside the keys. The order is defined - * by the comparator (any comparison function works — this is the bitonic sorter's - * advantage over the radix sorter, which is limited to the natural order of the - * key type but is considerably faster). - * - * All internal buffers, bind groups, per-step uniforms and pipelines are created - * up front; `run` only records dispatches. + * Creates a bitonic sorter for a `u32`, `i32` or `f32` key buffer, optionally reordering + * a payload buffer alongside the keys. The order is defined by an arbitrary comparator. + * All GPU resources are created up front, so `run` only records dispatches. */ export function createBitonicSorter< TKey extends BitonicKeyType, diff --git a/packages/typegpu-sort/src/bitonic/types.ts b/packages/typegpu-sort/src/bitonic/types.ts index 31f3c95969..ab75632858 100644 --- a/packages/typegpu-sort/src/bitonic/types.ts +++ b/packages/typegpu-sort/src/bitonic/types.ts @@ -7,16 +7,13 @@ export interface BitonicSorterOptions boolean; /** * Value used to pad arrays to power-of-2 length. Must sort to the end with your comparator. - * Defaults to the maximum value of the key type (works for ascending). For descending + * Defaults to the maximum value of the key type, which works for ascending. For descending * order, use the minimum value of the key type. */ paddingValue?: number; /** - * Optional payload buffer reordered alongside the keys (e.g. indices into - * another data structure). Must have the same power-of-two element count as - * the key buffer. - * When omitted, the payload machinery is not built at all — a key-only sorter - * carries zero overhead for it. + * Payload buffer reordered alongside the keys, e.g. indices into another data structure. + * Must have the same power-of-two element count as the key buffer. */ values?: TgpuBuffer> & StorageFlag; } diff --git a/packages/typegpu-sort/src/radix/count.ts b/packages/typegpu-sort/src/radix/count.ts index 56871f9c7a..65d5e4ecea 100644 --- a/packages/typegpu-sort/src/radix/count.ts +++ b/packages/typegpu-sort/src/radix/count.ts @@ -10,11 +10,6 @@ import { wgHist, } from './schemas.ts'; -/** - * Counts digit occurrences per tile into a digit-major histogram. `elementCount` - * is baked into the kernel, so keys are bounds-checked only when the buffer does - * not divide evenly into tiles. - */ export function makeCountKernel(schemas: RadixSchemas, elementCount: number) { const { ioLayout, digitFn } = schemas; const needsBoundsCheck = elementCount % TILE_SIZE !== 0; diff --git a/packages/typegpu-sort/src/radix/radixSort.ts b/packages/typegpu-sort/src/radix/radixSort.ts index f53185b838..5d96a4a223 100644 --- a/packages/typegpu-sort/src/radix/radixSort.ts +++ b/packages/typegpu-sort/src/radix/radixSort.ts @@ -21,18 +21,12 @@ type KeyBuffer = TgpuBuffer> & StorageFlag; type ValueBuffer = TgpuBuffer> & StorageFlag; /** - * Create a radix sorter for the given key buffer (u32, i32 or f32 elements). The - * sorter is a 4-pass LSD radix sort (8 bits per pass): each pass counts digit - * occurrences per tile (TILE_SIZE elements, several keys per thread) into a - * digit-major histogram, turns the histogram into scatter offsets with a single - * exclusive prefix scan, and scatters elements to their stable positions. i32 and f32 keys are ordered via - * order-preserving bit transforms applied at digit extraction only — the data - * buffers are never transformed. For f32 keys sorted ascending, NaNs with a - * cleared sign bit sort after +Infinity and NaNs with a set sign bit sort - * before -Infinity; the sort is oblivious to the distinction between -0 and +0. + * Creates a stable LSD radix sorter for a `u32`, `i32` or `f32` key buffer, optionally + * reordering a payload buffer alongside the keys. Keys are ordered by the natural order + * of their type. All GPU resources are created up front, so `run` only records dispatches. * - * All internal buffers, bind groups and pipelines are created up front; `run` - * only records dispatches. + * For `f32` keys sorted ascending, NaNs with a cleared sign bit sort after +Infinity and + * NaNs with a set sign bit sort before -Infinity. -0 and +0 compare equal. */ export function createRadixSorter< TKey extends RadixKeyType, diff --git a/packages/typegpu-sort/src/radix/scatter.ts b/packages/typegpu-sort/src/radix/scatter.ts index 46aa5aa110..2f86b164b9 100644 --- a/packages/typegpu-sort/src/radix/scatter.ts +++ b/packages/typegpu-sort/src/radix/scatter.ts @@ -15,24 +15,6 @@ const BITSET_WORD_BITS = 32; const BITSET_WORDS = TILE_THREADS / BITSET_WORD_BITS; const digitBits = tgpu.workgroupVar(d.arrayOf(d.atomic(d.u32), BITSET_WORDS * RADIX_SIZE)); -/** - * Stable scatter. Each workgroup owns a TILE_SIZE-element tile and processes it - * in KEYS_PER_THREAD sequential rounds of TILE_THREADS elements, carrying a - * per-digit running offset in workgroup memory so ranks stay stable across - * rounds. The offsets start at the tile's scanned histogram bases, so a round's - * output position is the offset plus the rank, with no per-key histogram read. - * - * Within a round, each invocation sets its bit in a per-digit workgroup bitset, - * then reads that digit's whole row once. The population count of the earlier - * bits is its stable rank, and the count over the full row is the digit's total - * for the round; the invocation that ranks first for a digit advances that - * digit's offset by the total. The bitset is word-major - * (`word * RADIX_SIZE + digit`), so the lanes of a SIMD group address - * consecutive words rather than a fixed stride. - * - * `elementCount` is baked into the kernel, so keys are bounds-checked only when - * the buffer does not divide evenly into tiles. - */ export function makeScatterKernel(schemas: RadixSchemas, elementCount: number): TgpuComputeFn { const { ioLayout, digitFn, writeOutput } = schemas; const needsBoundsCheck = elementCount % TILE_SIZE !== 0; diff --git a/packages/typegpu-sort/src/radix/schemas.ts b/packages/typegpu-sort/src/radix/schemas.ts index 1f78d92e06..a563a42b5d 100644 --- a/packages/typegpu-sort/src/radix/schemas.ts +++ b/packages/typegpu-sort/src/radix/schemas.ts @@ -12,12 +12,6 @@ export type SortDirection = 'ascending' | 'descending'; export const paramsType = d.struct({ shift: d.u32, numTiles: d.u32 }); -/** - * Digit-major histogram over all tiles: `hist[digit * numTiles + tile]`. - * After counting, a single exclusive prefix scan over the whole buffer turns each - * entry into the final base offset for that (digit, tile) pair — the global offset - * of the digit plus the offset of the tile within the digit. - */ export const histLayout = tgpu.bindGroupLayout({ hist: { storage: d.arrayOf(d.u32), access: 'mutable' }, }); @@ -28,20 +22,6 @@ export const paramsLayout = tgpu.bindGroupLayout({ export const wgHist = tgpu.workgroupVar(d.arrayOf(d.atomic(d.u32), RADIX_SIZE)); -/** - * Builds the digit extraction function for a key type and direction. Each function - * maps a key and a bit shift to the key's current radix digit, via an - * order-preserving u32 reinterpretation of the key: - * - u32: the key itself - * - i32: equivalent to flipping the sign bit — the digit is extracted from the raw - * two's complement bits (masking makes arithmetic and logical shifts agree) and - * bit 7 is flipped for the top byte only, avoiding any INT_MIN literal that - * WGSL cannot represent - * - f32: sign bit flipped for non-negative values, all bits flipped for negative - * ones (NaNs sort after +Infinity when ascending) - * - * Descending order complements the digit instead of transforming the key. - */ function makeDigitFn(keyType: RadixKeyType, direction: SortDirection) { const ascendingImpls = { u32: tgpu.fn([d.u32, d.u32], d.u32)((v, shift) => (v >> shift) & (RADIX_SIZE - 1)), @@ -57,7 +37,7 @@ function makeDigitFn(keyType: RadixKeyType, direction: SortDirection) { [d.f32, d.u32], d.u32, )((v, shift) => { - // Canonicalize signed zero so a stable numeric sort preserves its input order. + // -0 and +0 must map to the same bits, otherwise they sort apart const bits = std.select(std.bitcastF32toU32(v), d.u32(0), v === 0); const mask = std.select(d.u32(0x80000000), d.u32(0xffffffff), bits >> 31 === 1); return ((bits ^ mask) >> shift) & (RADIX_SIZE - 1); diff --git a/packages/typegpu-sort/src/radix/types.ts b/packages/typegpu-sort/src/radix/types.ts index f7a768a2dc..cd0ec069fd 100644 --- a/packages/typegpu-sort/src/radix/types.ts +++ b/packages/typegpu-sort/src/radix/types.ts @@ -3,13 +3,11 @@ import type { RunPassOptions } from '../runPass.ts'; import type { SortDirection } from './schemas.ts'; export interface RadixSorterOptions { - /** Sort order. Defaults to 'ascending'. */ + /** Sort order. Defaults to `'ascending'` */ direction?: SortDirection; /** - * Optional payload buffer reordered alongside the keys (e.g. indices into - * another data structure). Must have the same element count as the key buffer. - * When omitted, the payload machinery is not built at all — a key-only sorter - * carries zero overhead for it. + * Payload buffer reordered alongside the keys, e.g. indices into another data structure. + * Must have the same element count as the key buffer. */ values?: TgpuBuffer> & StorageFlag; } @@ -17,13 +15,10 @@ export interface RadixSorterOptions { /** - * The buffer holding the result after `run`. For a full prefix scan this is - * the scanned buffer itself, for a reduction it is a single-element buffer - * owned by the plan (reused across runs). + * The buffer holding the result after `run`. For a full prefix scan this is the scanned + * buffer itself, for a reduction it is a single-element buffer owned by the plan and + * reused across runs. */ readonly resultBuffer: ScanBuffer; - /** - * Dispatch the scan — standalone by default, or into the encoder/pass provided - * in `options` to compose the scan with other GPU work. - */ + /** Dispatches the scan. Can be called repeatedly */ run(options?: ScanRunOptions): void; - /** Destroy the scratch buffers owned by this plan. */ + /** Destroys the scratch buffers owned by this plan */ destroy(): void; } @@ -124,9 +120,8 @@ export class PrefixScanComputer { } /** - * Create a reusable execution plan for scanning `buffer`. All scratch buffers, - * bind groups and pipelines are allocated up front — call `plan.run()` to - * dispatch, any number of times. + * Creates a reusable execution plan for scanning `buffer`. All scratch buffers, bind + * groups and pipelines are allocated up front, so `plan.run()` only records dispatches. */ prepare( buffer: ScanBuffer, @@ -211,11 +206,10 @@ export class PrefixScanComputer { } /** - * Scan `buffer` in place (or reduce it, when `onlyGreatestElement` is true). - * Plans are cached per buffer, so repeated calls on the same buffer reuse - * all scratch buffers and bind groups. Note that for reductions this means - * the returned single-element buffer is shared between calls on the same - * input buffer. + * Scans `buffer` in place, or reduces it when `onlyGreatestElement` is true. Plans are + * cached per buffer, so repeated calls on the same buffer reuse all scratch buffers and + * bind groups. For reductions this means the returned single-element buffer is shared + * between calls on the same input buffer. */ compute( buffer: ScanBuffer, @@ -310,8 +304,8 @@ export function prefixScan( * - operation: The binary operation to use for the reduction (e.g., std.add) * - identityElement: The identity element for the operation (e.g., 0 for addition) * @returns A buffer containing the aggregated reduction result (single-element buffer). - * The buffer is owned by the internally cached scan plan and is reused by - * subsequent `scan` calls on the same input buffer. + * It is owned by the internally cached scan plan and reused by subsequent + * `scan` calls on the same input buffer. * * @example * ```typescript diff --git a/packages/typegpu-sort/src/scan/types.ts b/packages/typegpu-sort/src/scan/types.ts index 846e42ce9e..8e753ea197 100644 --- a/packages/typegpu-sort/src/scan/types.ts +++ b/packages/typegpu-sort/src/scan/types.ts @@ -4,6 +4,6 @@ import type { ScanElementType } from './schemas.ts'; export interface BinaryOp { operation: (a: number, b: number) => number; identityElement: number; - /** Element type of the buffers to scan. Defaults to `d.f32`. */ + /** Element type of the buffers to scan. Defaults to `d.f32` */ dataType?: TElement; } diff --git a/packages/typegpu-sort/src/wgslUtils.ts b/packages/typegpu-sort/src/wgslUtils.ts index cd3e5ac523..f2a797034c 100644 --- a/packages/typegpu-sort/src/wgslUtils.ts +++ b/packages/typegpu-sort/src/wgslUtils.ts @@ -1,9 +1,5 @@ import { tgpu, d } from 'typegpu'; -/** - * Flattens a 3D workgroup id (as produced by {@link decomposeWorkgroups}) back - * into a linear workgroup index. - */ export const flatWorkgroupIndex = tgpu.fn( [d.vec3u, d.vec3u], d.u32, From d6b4ea312a4f7105248823891c4b23b34bf2b88b Mon Sep 17 00:00:00 2001 From: Konrad Reczko Date: Sat, 25 Jul 2026 03:33:23 +0200 Subject: [PATCH 12/12] better and simpler --- .../src/examples/algorithms/sort/index.ts | 8 +- .../src/examples/tests/prefix-scan/index.ts | 10 +- .../individual-example-tests/sort.test.ts | 251 +++++++- packages/typegpu-sort/README.md | 7 +- .../typegpu-sort/src/bitonic/bitonicSort.ts | 608 +++++++----------- packages/typegpu-sort/src/bitonic/index.ts | 4 +- packages/typegpu-sort/src/bitonic/slots.ts | 21 +- packages/typegpu-sort/src/bitonic/types.ts | 18 +- packages/typegpu-sort/src/bitonic/utils.ts | 39 -- packages/typegpu-sort/src/dispatch.ts | 31 + packages/typegpu-sort/src/index.ts | 35 +- packages/typegpu-sort/src/radix/count.ts | 60 +- packages/typegpu-sort/src/radix/index.ts | 2 +- packages/typegpu-sort/src/radix/radixSort.ts | 157 ++--- packages/typegpu-sort/src/radix/scatter.ts | 116 ++-- packages/typegpu-sort/src/radix/schemas.ts | 107 +-- packages/typegpu-sort/src/radix/types.ts | 12 - packages/typegpu-sort/src/runPass.ts | 33 +- .../src/scan/compute/applySums.ts | 29 - .../typegpu-sort/src/scan/compute/scan.ts | 78 --- .../typegpu-sort/src/scan/compute/shared.ts | 39 -- packages/typegpu-sort/src/scan/index.ts | 7 +- packages/typegpu-sort/src/scan/kernels.ts | 118 ++++ packages/typegpu-sort/src/scan/prefixScan.ts | 466 +++++--------- packages/typegpu-sort/src/scan/schemas.ts | 5 +- packages/typegpu-sort/src/types.ts | 23 + packages/typegpu-sort/src/wgslUtils.ts | 9 - packages/typegpu-sort/tests/bitonic.test.ts | 260 +++++++- packages/typegpu-sort/tests/radix.test.ts | 92 ++- packages/typegpu-sort/tests/scan.test.ts | 158 ++++- .../typegpu-testing-utility/src/extendedIt.ts | 1 + 31 files changed, 1524 insertions(+), 1280 deletions(-) delete mode 100644 packages/typegpu-sort/src/bitonic/utils.ts create mode 100644 packages/typegpu-sort/src/dispatch.ts delete mode 100644 packages/typegpu-sort/src/scan/compute/applySums.ts delete mode 100644 packages/typegpu-sort/src/scan/compute/scan.ts delete mode 100644 packages/typegpu-sort/src/scan/compute/shared.ts create mode 100644 packages/typegpu-sort/src/scan/kernels.ts create mode 100644 packages/typegpu-sort/src/types.ts delete mode 100644 packages/typegpu-sort/src/wgslUtils.ts diff --git a/apps/typegpu-docs/src/examples/algorithms/sort/index.ts b/apps/typegpu-docs/src/examples/algorithms/sort/index.ts index 2fb432cb2d..dc0ec5bacc 100644 --- a/apps/typegpu-docs/src/examples/algorithms/sort/index.ts +++ b/apps/typegpu-docs/src/examples/algorithms/sort/index.ts @@ -5,7 +5,7 @@ import { createBitonicSorter, createRadixSorter, decomposeWorkgroups, - type RadixSorter, + type Sorter, } from '@typegpu/sort'; import { randf } from '@typegpu/noise'; import { fullScreenTriangle } from 'typegpu/common'; @@ -234,7 +234,7 @@ function hideOverlay(delayMs = 1500) { }, delayMs); } -function pickSorter(): { sorter: BitonicSorter | RadixSorter; note: string } { +function pickSorter(): { sorter: Sorter; note: string } { if (state.algorithm === 'radix') { if (state.sortOrder === 'ascending' || state.sortOrder === 'descending') { return { sorter: radixSorters[state.sortOrder], note: '' }; @@ -247,7 +247,7 @@ function pickSorter(): { sorter: BitonicSorter | RadixSorter; note: string } { return { sorter: bitonicSorters[state.sortOrder], note: '' }; } -async function runSorterTimed(sorter: BitonicSorter | RadixSorter): Promise { +async function runSorterTimed(sorter: Sorter): Promise { if (!querySet?.available) { sorter.run(); return null; @@ -295,7 +295,7 @@ function formatMs(milliseconds: number): string { } async function benchmarkSorter( - sorter: BitonicSorter | RadixSorter, + sorter: Sorter, timestamps: TgpuQuerySet<'timestamp'>, ): Promise { for (let i = 0; i < BENCH_WARMUP; i++) { diff --git a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts index 9049fd9982..86364d08ea 100644 --- a/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts +++ b/apps/typegpu-docs/src/examples/tests/prefix-scan/index.ts @@ -1,6 +1,6 @@ import { tgpu } from 'typegpu'; import * as d from 'typegpu/data'; -import { type BinaryOp, prefixScan, scan } from '@typegpu/sort'; +import { type BinaryOp, prefixScan, reduce } from '@typegpu/sort'; import * as std from 'typegpu/std'; import { addFn, concat10, isArrayEqual, mulFn, prefixScanJS, scanJS } from './functions.ts'; @@ -12,7 +12,7 @@ async function runAndCompare(arr: number[], op: BinaryOp, scanOnly: boolean) { const input = root.createBuffer(d.arrayOf(d.f32, arr.length), arr).$usage('storage'); const output = scanOnly - ? scan(root, { + ? reduce(root, { inputBuffer: input, operation: op.operation, identityElement: op.identityElement, @@ -82,7 +82,7 @@ async function testLength16777217(): Promise { async function testDoesNotDestroyBuffer(): Promise { const input = root.createBuffer(d.arrayOf(d.f32, 8), [1, 2, 3, 4, 5, 6, 7, 8]).$usage('storage'); - scan(root, { + reduce(root, { inputBuffer: input, operation: addFn, identityElement: 0, @@ -96,7 +96,7 @@ async function testDoesNotCacheBuffers(): Promise { const input1 = root.createBuffer(d.arrayOf(d.f32, 8), [1, 2, 3, 4, 5, 6, 7, 8]).$usage('storage'); - const output1 = scan(root, { + const output1 = reduce(root, { inputBuffer: input1, operation: op.operation, identityElement: op.identityElement, @@ -109,7 +109,7 @@ async function testDoesNotCacheBuffers(): Promise { ) .$usage('storage'); - const output2 = scan(root, { + const output2 = reduce(root, { inputBuffer: input2, operation: op.operation, identityElement: op.identityElement, diff --git a/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts b/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts index fa69064415..3c9988ea30 100644 --- a/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts +++ b/apps/typegpu-docs/tests/individual-example-tests/sort.test.ts @@ -9,7 +9,7 @@ import { runExampleTest, setupCommonMocks } from './utils/baseTest.ts'; describe('sort example', () => { setupCommonMocks(); - it('resolves the sort and render pipelines', async ({ device }) => { + it('should produce valid code', async ({ device }) => { const shaderCodes = await runExampleTest( { category: 'algorithms', @@ -20,18 +20,245 @@ describe('sort example', () => { device, ); - expect({ - copyPad: shaderCodes.includes('@compute @workgroup_size(256) fn copyPadKernel'), - localSort: shaderCodes.includes('@compute @workgroup_size(256) fn localSortKernel'), - globalStep: shaderCodes.includes('@compute @workgroup_size(256) fn bitonicStepKernel'), - render: shaderCodes.includes('@fragment fn fragmentFn'), - }).toMatchInlineSnapshot(` - { - "copyPad": true, - "globalStep": true, - "localSort": true, - "render": true, + expect(shaderCodes).toMatchInlineSnapshot(` + "fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + @group(0) @binding(1) var dst: array; + + @group(0) @binding(0) var src: array; + + @group(0) @binding(2) var padding: u32; + + @compute @workgroup_size(256) fn pad(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let idx = ((flatWorkgroupIndex(wid, numWorkgroups) * 256u) + lid.x); + if ((idx >= 1024u)) { + return; + } + if ((idx < 841u)) { + dst[idx] = src[idx]; + } + else { + dst[idx] = padding; + } + } + + fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + @group(0) @binding(0) var data: array; + + var localKeys: array; + + fn loadShared(base: u32, tid: u32) { + localKeys[tid] = data[(base + tid)]; + localKeys[(tid + 256u)] = data[((base + tid) + 256u)]; + } + + fn defaultCompare(a: u32, b: u32) -> bool { + return (a < b); + } + + fn swapLocalAt(a: u32, b: u32, left: u32, right: u32) { + localKeys[a] = right; + localKeys[b] = left; + } + + fn exchangeLocal(base: u32, iLocal: u32, stride: u32, k: u32) { + let jLocal = (iLocal + stride); + let left = localKeys[iLocal]; + let right = localKeys[jLocal]; + let ascending = (((base + iLocal) & k) == 0u); + if (select(defaultCompare(left, right), defaultCompare(right, left), ascending)) { + swapLocalAt(iLocal, jLocal, left, right); + } + } + + fn mergeDown(base: u32, tid: u32, startShift: u32, k: u32) { + for (var jShift = startShift; (jShift > 0u); jShift--) { + workgroupBarrier(); + let stride = (1u << (jShift - 1u)); + let below = (tid & (stride - 1u)); + let above = (tid >> (jShift - 1u)); + exchangeLocal(base, (below + (above * (stride << 1u))), stride, k); + } + } + + fn storeShared(base: u32, tid: u32) { + data[(base + tid)] = localKeys[tid]; + data[((base + tid) + 256u)] = localKeys[(tid + 256u)]; + } + + @compute @workgroup_size(256) fn localSort(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let base = (flatWorkgroupIndex(wid, numWorkgroups) * 512u); + if ((base >= arrayLength(&data))) { + return; + } + loadShared(base, lid.x); + for (var kShift = 1u; (kShift <= 9u); kShift++) { + mergeDown(base, lid.x, kShift, (1u << kShift)); + } + workgroupBarrier(); + storeShared(base, lid.x); + } + + fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + struct sortUniformsType { + k: u32, + jShift: u32, + } + + @group(0) @binding(1) var uniforms: sortUniformsType; + + @group(0) @binding(0) var data: array; + + fn defaultCompare(a: u32, b: u32) -> bool { + return (a < b); + } + + fn swapAt(i: u32, j: u32, left: u32, right: u32) { + data[i] = right; + data[j] = left; + } + + @compute @workgroup_size(256) fn item(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let tid = ((flatWorkgroupIndex(wid, numWorkgroups) * 256u) + lid.x); + let k = uniforms.k; + let shift = uniforms.jShift; + let stride = (1u << shift); + let below = (tid & (stride - 1u)); + let above = (tid >> shift); + let i = (below + (above * (stride << 1u))); + let ixj = (i + stride); + if ((ixj >= arrayLength(&data))) { + return; + } + let left = data[i]; + let right = data[ixj]; + let ascending = ((i & k) == 0u); + if (select(defaultCompare(left, right), defaultCompare(right, left), ascending)) { + swapAt(i, ixj, left, right); + } } + + fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + @group(0) @binding(0) var data: array; + + var localKeys: array; + + fn loadShared(base: u32, tid: u32) { + localKeys[tid] = data[(base + tid)]; + localKeys[(tid + 256u)] = data[((base + tid) + 256u)]; + } + + struct sortUniformsType { + k: u32, + jShift: u32, + } + + @group(0) @binding(1) var uniforms: sortUniformsType; + + fn defaultCompare(a: u32, b: u32) -> bool { + return (a < b); + } + + fn swapLocalAt(a: u32, b: u32, left: u32, right: u32) { + localKeys[a] = right; + localKeys[b] = left; + } + + fn exchangeLocal(base: u32, iLocal: u32, stride: u32, k: u32) { + let jLocal = (iLocal + stride); + let left = localKeys[iLocal]; + let right = localKeys[jLocal]; + let ascending = (((base + iLocal) & k) == 0u); + if (select(defaultCompare(left, right), defaultCompare(right, left), ascending)) { + swapLocalAt(iLocal, jLocal, left, right); + } + } + + fn mergeDown(base: u32, tid: u32, startShift: u32, k: u32) { + for (var jShift = startShift; (jShift > 0u); jShift--) { + workgroupBarrier(); + let stride = (1u << (jShift - 1u)); + let below = (tid & (stride - 1u)); + let above = (tid >> (jShift - 1u)); + exchangeLocal(base, (below + (above * (stride << 1u))), stride, k); + } + } + + fn storeShared(base: u32, tid: u32) { + data[(base + tid)] = localKeys[tid]; + data[((base + tid) + 256u)] = localKeys[(tid + 256u)]; + } + + @compute @workgroup_size(256) fn localMerge(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let base = (flatWorkgroupIndex(wid, numWorkgroups) * 512u); + if ((base >= arrayLength(&data))) { + return; + } + loadShared(base, lid.x); + mergeDown(base, lid.x, 9u, uniforms.k); + workgroupBarrier(); + storeShared(base, lid.x); + } + + fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + @group(0) @binding(1) var dst: array; + + @group(0) @binding(0) var src: array; + + @compute @workgroup_size(256) fn unpad(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let idx = ((flatWorkgroupIndex(wid, numWorkgroups) * 256u) + lid.x); + if ((idx < 841u)) { + dst[idx] = src[idx]; + } + } + + struct fullScreenTriangle_Output { + @builtin(position) pos: vec4f, + @location(0) uv: vec2f, + } + + @vertex fn fullScreenTriangle(@builtin(vertex_index) vertexIndex: u32) -> fullScreenTriangle_Output { + const pos = array(vec2f(-1, -1), vec2f(3, -1), vec2f(-1, 3)); + const uv = array(vec2f(0, 1), vec2f(2, 1), vec2f(0, -1)); + + return fullScreenTriangle_Output(vec4f(pos[vertexIndex], 0, 1), uv[vertexIndex]); + } + + @group(0) @binding(0) var data_1: array; + + struct fragmentFn_Input { + @location(0) uv: vec2f, + } + + @fragment fn fragmentFn(_arg_0: fragmentFn_Input) -> @location(0) vec4f { + let data = (&data_1); + let arrayLength_1 = arrayLength(&(*data)); + let cols = u32(round(sqrt(f32(arrayLength_1)))); + let rows = u32(round((f32(arrayLength_1) / f32(cols)))); + let col = u32(floor((_arg_0.uv.x * f32(cols)))); + let row = u32(floor((_arg_0.uv.y * f32(rows)))); + let idx = ((row * cols) + col); + if ((idx >= arrayLength_1)) { + return vec4f(0.10000000149011612, 0.10000000149011612, 0.10000000149011612, 1); + } + let value = (*data)[idx]; + let normalized = (f32(value) / 255f); + return vec4f(normalized, normalized, normalized, 1f); + }" `); }); }); diff --git a/packages/typegpu-sort/README.md b/packages/typegpu-sort/README.md index 027bbc3c30..acee954c20 100644 --- a/packages/typegpu-sort/README.md +++ b/packages/typegpu-sort/README.md @@ -73,14 +73,14 @@ An exclusive work-efficient prefix scan over `f32` (default), `u32` or `i32` buffers, with any associative operation. ```ts -import { prefixScan, scan } from '@typegpu/sort'; +import { prefixScan, reduce } from '@typegpu/sort'; import * as std from 'typegpu/std'; // Full prefix scan (in place) const result = prefixScan(root, { inputBuffer, operation: std.add, identityElement: 0 }); // Reduction only (returns a single-element buffer with the aggregate) -const total = scan(root, { inputBuffer, operation: std.add, identityElement: 0 }); +const total = reduce(root, { inputBuffer, operation: std.add, identityElement: 0 }); // Integer scan const sums = prefixScan(root, { @@ -107,6 +107,9 @@ plan.run(); // any number of times plan.destroy(); ``` +`computer.scan(buffer)` and `computer.reduce(buffer)` do the same through a plan +cached per buffer, which is what `prefixScan` and `reduce` use internally. + Note: passing `-2147483648` (i32 minimum) as `identityElement` currently generates WGSL that does not compile. Use `-2147483647` instead. diff --git a/packages/typegpu-sort/src/bitonic/bitonicSort.ts b/packages/typegpu-sort/src/bitonic/bitonicSort.ts index a581ee5300..e6536723a1 100644 --- a/packages/typegpu-sort/src/bitonic/bitonicSort.ts +++ b/packages/typegpu-sort/src/bitonic/bitonicSort.ts @@ -4,15 +4,15 @@ import { std, type StorageFlag, type TgpuBuffer, + type TgpuComputeFn, type TgpuComputePipeline, type TgpuRoot, - type UniformFlag, } from 'typegpu'; +import { decomposeWorkgroups, dispatchIn, flatWorkgroupIndex } from '../dispatch.ts'; import { beginRunPass } from '../runPass.ts'; -import { flatWorkgroupIndex } from '../wgslUtils.ts'; -import { compareSlot, defaultCompares, defaultPaddingValues } from './slots.ts'; -import type { BitonicSorter, BitonicSorterOptions, BitonicSorterRunOptions } from './types.ts'; -import { decomposeWorkgroups, nextPowerOf2 } from './utils.ts'; +import type { RunOptions } from '../types.ts'; +import { compareSlot, defaultCompare, defaultPaddingValues } from './slots.ts'; +import type { BitonicSorter, BitonicSorterOptions } from './types.ts'; const WORKGROUP_SIZE = 256; const LOCAL_BLOCK = WORKGROUP_SIZE * 2; @@ -28,278 +28,215 @@ const sortUniformsType = d.struct({ jShift: d.u32, }); -function makeBitonicSchemas(keyType: BitonicKeyType, valueType?: d.AnyWgslData) { - const copyParamsType = d.struct({ - srcLength: d.u32, - dstLength: d.u32, - paddingValue: keyType, - }); +function nextPowerOf2(n: number): number { + let p = 1; + while (p < n) { + p <<= 1; + } + return p; +} +function makeBitonicSchemas(keyType: BitonicKeyType, valueType: d.AnyWgslData | undefined) { const sortLayout = tgpu.bindGroupLayout({ data: { storage: d.arrayOf(keyType), access: 'mutable' }, uniforms: { uniform: sortUniformsType }, }); - const copyLayout = tgpu.bindGroupLayout({ - src: { storage: d.arrayOf(keyType), access: 'readonly' }, - dst: { storage: d.arrayOf(keyType), access: 'mutable' }, - params: { uniform: copyParamsType }, - }); + const hasPayload = valueType !== undefined; + const payloadType = valueType ?? d.u32; - const valsLayout = valueType - ? tgpu.bindGroupLayout({ - vals: { storage: d.arrayOf(valueType), access: 'mutable' }, - }) - : undefined; - - const valsCopyLayout = valueType - ? tgpu.bindGroupLayout({ - src: { storage: d.arrayOf(valueType), access: 'readonly' }, - dst: { storage: d.arrayOf(valueType), access: 'mutable' }, - params: { uniform: copyParamsType }, - }) - : undefined; - - const copyValue = valueType as unknown as (value: unknown) => number; + const valsLayout = tgpu.bindGroupLayout({ + vals: { storage: d.arrayOf(payloadType), access: 'mutable' }, + }); - const swapValues = - valsLayout && valueType - ? tgpu.fn([d.u32, d.u32])((i, j) => { - const tmp = copyValue(valsLayout.$.vals[i]); - (valsLayout.$.vals[i] as number) = copyValue(valsLayout.$.vals[j]); - (valsLayout.$.vals[j] as number) = tmp; - }) - : tgpu.fn([d.u32, d.u32])(() => {}); + function swapAt(i: number, j: number, left: number, right: number) { + 'use gpu'; + sortLayout.$.data[i] = right; + sortLayout.$.data[j] = left; + if (hasPayload) { + const tmp = std.copy(valsLayout.$.vals[i] as number); + (valsLayout.$.vals[i] as number) = std.copy(valsLayout.$.vals[j] as number); + (valsLayout.$.vals[j] as number) = std.copy(tmp); + } + } - return { - keyType, - copyParamsType, - sortLayout, - copyLayout, - valsLayout, - valsCopyLayout, - swapValues, - }; + return { keyType, valueType, hasPayload, payloadType, sortLayout, valsLayout, swapAt }; } type BitonicSchemas = ReturnType; -function makeKernels(schemas: BitonicSchemas) { - const { sortLayout, copyLayout, valsCopyLayout, swapValues } = schemas; +function makePaddingKernels(keyType: BitonicKeyType, size: number, paddedSize: number) { + const copyLayout = tgpu.bindGroupLayout({ + src: { storage: d.arrayOf(keyType), access: 'readonly' }, + dst: { storage: d.arrayOf(keyType), access: 'mutable' }, + padding: { uniform: keyType }, + }); - const copyPadKernel = tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { + const pad = tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], in: dispatchIn })(({ + lid, + wid, + numWorkgroups, + }) => { const idx = flatWorkgroupIndex(wid, numWorkgroups) * WORKGROUP_SIZE + lid.x; - - if (idx >= copyLayout.$.params.dstLength) { + if (idx >= paddedSize) { return; } - if (idx < copyLayout.$.params.srcLength) { + if (idx < size) { copyLayout.$.dst[idx] = copyLayout.$.src[idx] as number; } else { - copyLayout.$.dst[idx] = copyLayout.$.params.paddingValue; + copyLayout.$.dst[idx] = copyLayout.$.padding; } }); - function makeCopyBackKernel( - layout: BitonicSchemas['copyLayout'] | NonNullable, - ) { - const copyBack = tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const idx = flatWorkgroupIndex(wid, numWorkgroups) * WORKGROUP_SIZE + lid.x; - - if (idx < layout.$.params.srcLength) { - (layout.$.dst[idx] as number) = layout.$.src[idx] as number; - } - }); - return copyBack; - } + const unpad = tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], in: dispatchIn })(({ + lid, + wid, + numWorkgroups, + }) => { + const idx = flatWorkgroupIndex(wid, numWorkgroups) * WORKGROUP_SIZE + lid.x; + if (idx < size) { + (copyLayout.$.dst[idx] as number) = copyLayout.$.src[idx] as number; + } + }); - const copyBackKernel = makeCopyBackKernel(copyLayout); - const valsCopyKernel = valsCopyLayout ? makeCopyBackKernel(valsCopyLayout) : undefined; + return { copyLayout, pad, unpad }; +} - const bitonicStepKernel = tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const tid = flatWorkgroupIndex(wid, numWorkgroups) * WORKGROUP_SIZE + lid.x; +function makeGlobalStepKernel(schemas: BitonicSchemas) { + const { sortLayout, swapAt } = schemas; - const k = sortLayout.$.uniforms.k; - const shift = sortLayout.$.uniforms.jShift; - const dataLength = d.u32(sortLayout.$.data.length); - const stride = d.u32(1) << shift; + return tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], in: dispatchIn })( + ({ lid, wid, numWorkgroups }) => { + const tid = flatWorkgroupIndex(wid, numWorkgroups) * WORKGROUP_SIZE + lid.x; - const maskBelow = stride - 1; - const below = tid & maskBelow; - const above = tid >> shift; + const k = sortLayout.$.uniforms.k; + const shift = sortLayout.$.uniforms.jShift; + const stride = d.u32(1) << shift; - const i = below + above * (stride << 1); - const ixj = i + stride; + const below = tid & (stride - 1); + const above = tid >> shift; + const i = below + above * (stride << 1); + const ixj = i + stride; - if (ixj >= dataLength) { - return; - } + if (ixj >= d.u32(sortLayout.$.data.length)) { + return; + } + + const left = sortLayout.$.data[i] as number; + const right = sortLayout.$.data[ixj] as number; + const ascending = (i & k) === 0; - const ascending = (i & k) === 0; - const left = sortLayout.$.data[i] as number; - const right = sortLayout.$.data[ixj] as number; + if (std.select(compareSlot.$(left, right), compareSlot.$(right, left), ascending)) { + swapAt(i, ixj, left, right); + } + }, + ); +} - const leftFirst = compareSlot.$(left, right); - const rightFirst = compareSlot.$(right, left); - const shouldSwap = std.select(leftFirst, rightFirst, ascending); +function makeLocalKernels(schemas: BitonicSchemas) { + const { keyType, hasPayload, payloadType, sortLayout, valsLayout } = schemas; - if (shouldSwap) { - sortLayout.$.data[i] = right; - sortLayout.$.data[ixj] = left; - swapValues(i, ixj); + const localKeys = tgpu.workgroupVar(d.arrayOf(keyType, LOCAL_BLOCK)); + const localVals = tgpu.workgroupVar(d.arrayOf(payloadType, LOCAL_BLOCK)); + + function loadShared(base: number, tid: number) { + 'use gpu'; + (localKeys.$[tid] as number) = sortLayout.$.data[base + tid] as number; + (localKeys.$[tid + WORKGROUP_SIZE] as number) = sortLayout.$.data[ + base + tid + WORKGROUP_SIZE + ] as number; + if (hasPayload) { + (localVals.$[tid] as number) = std.copy(valsLayout.$.vals[base + tid] as number); + (localVals.$[tid + WORKGROUP_SIZE] as number) = std.copy( + valsLayout.$.vals[base + tid + WORKGROUP_SIZE] as number, + ); } - }); + } - return { copyPadKernel, copyBackKernel, valsCopyKernel, bitonicStepKernel }; -} + function storeShared(base: number, tid: number) { + 'use gpu'; + (sortLayout.$.data[base + tid] as number) = localKeys.$[tid] as number; + (sortLayout.$.data[base + tid + WORKGROUP_SIZE] as number) = localKeys.$[ + tid + WORKGROUP_SIZE + ] as number; + if (hasPayload) { + (valsLayout.$.vals[base + tid] as number) = std.copy(localVals.$[tid] as number); + (valsLayout.$.vals[base + tid + WORKGROUP_SIZE] as number) = std.copy( + localVals.$[tid + WORKGROUP_SIZE] as number, + ); + } + } -function makeLocalKernels(schemas: BitonicSchemas, valueType?: d.AnyWgslData) { - const { keyType, sortLayout, valsLayout } = schemas; - const copyValue = valueType as unknown as (value: unknown) => number; + function swapLocalAt(a: number, b: number, left: number, right: number) { + 'use gpu'; + localKeys.$[a] = right; + localKeys.$[b] = left; + if (hasPayload) { + const tmp = std.copy(localVals.$[a] as number); + (localVals.$[a] as number) = std.copy(localVals.$[b] as number); + (localVals.$[b] as number) = std.copy(tmp); + } + } - const localKeys = tgpu.workgroupVar(d.arrayOf(keyType, LOCAL_BLOCK)); - const localVals = - valueType && valsLayout ? tgpu.workgroupVar(d.arrayOf(valueType, LOCAL_BLOCK)) : undefined; - - const loadShared = - localVals && valsLayout - ? tgpu.fn([d.u32, d.u32])((base, tid) => { - (localKeys.$[tid] as number) = sortLayout.$.data[base + tid] as number; - (localKeys.$[tid + WORKGROUP_SIZE] as number) = sortLayout.$.data[ - base + tid + WORKGROUP_SIZE - ] as number; - (localVals.$[tid] as number) = copyValue(valsLayout.$.vals[base + tid]); - (localVals.$[tid + WORKGROUP_SIZE] as number) = copyValue( - valsLayout.$.vals[base + tid + WORKGROUP_SIZE], - ); - }) - : tgpu.fn([d.u32, d.u32])((base, tid) => { - (localKeys.$[tid] as number) = sortLayout.$.data[base + tid] as number; - (localKeys.$[tid + WORKGROUP_SIZE] as number) = sortLayout.$.data[ - base + tid + WORKGROUP_SIZE - ] as number; - }); - - const storeShared = - localVals && valsLayout - ? tgpu.fn([d.u32, d.u32])((base, tid) => { - (sortLayout.$.data[base + tid] as number) = localKeys.$[tid] as number; - (sortLayout.$.data[base + tid + WORKGROUP_SIZE] as number) = localKeys.$[ - tid + WORKGROUP_SIZE - ] as number; - (valsLayout.$.vals[base + tid] as number) = copyValue(localVals.$[tid]); - (valsLayout.$.vals[base + tid + WORKGROUP_SIZE] as number) = copyValue( - localVals.$[tid + WORKGROUP_SIZE], - ); - }) - : tgpu.fn([d.u32, d.u32])((base, tid) => { - (sortLayout.$.data[base + tid] as number) = localKeys.$[tid] as number; - (sortLayout.$.data[base + tid + WORKGROUP_SIZE] as number) = localKeys.$[ - tid + WORKGROUP_SIZE - ] as number; - }); - - const swapLocalValues = localVals - ? tgpu.fn([d.u32, d.u32])((a, b) => { - const tmp = copyValue(localVals.$[a]); - (localVals.$[a] as number) = copyValue(localVals.$[b]); - (localVals.$[b] as number) = tmp; - }) - : tgpu.fn([d.u32, d.u32])(() => {}); - - const exchangeLocal = tgpu.fn([d.u32, d.u32, d.u32, d.u32])((base, iLocal, stride, k) => { - const ixjLocal = iLocal + stride; + function exchangeLocal(base: number, iLocal: number, stride: number, k: number) { + 'use gpu'; + const jLocal = iLocal + stride; const left = localKeys.$[iLocal] as number; - const right = localKeys.$[ixjLocal] as number; + const right = localKeys.$[jLocal] as number; const ascending = ((base + iLocal) & k) === 0; - const leftFirst = compareSlot.$(left, right); - const rightFirst = compareSlot.$(right, left); - const shouldSwap = std.select(leftFirst, rightFirst, ascending); - if (shouldSwap) { - localKeys.$[iLocal] = right; - localKeys.$[ixjLocal] = left; - swapLocalValues(iLocal, ixjLocal); + + if (std.select(compareSlot.$(left, right), compareSlot.$(right, left), ascending)) { + swapLocalAt(iLocal, jLocal, left, right); } - }); + } - const mergeDown = tgpu.fn([d.u32, d.u32, d.u32, d.u32])((base, tid, startShift, k) => { + function mergeDown(base: number, tid: number, startShift: number, k: number) { + 'use gpu'; for (let jShift = d.u32(startShift); jShift > 0; jShift--) { std.workgroupBarrier(); const stride = d.u32(1) << (jShift - 1); const below = tid & (stride - 1); const above = tid >> (jShift - 1); - const iLocal = below + above * (stride << 1); - exchangeLocal(base, iLocal, stride, k); + exchangeLocal(base, below + above * (stride << 1), stride, k); } - }); + } - const localSortKernel = tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const tid = lid.x; + const localSort = tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], in: dispatchIn })(({ + lid, + wid, + numWorkgroups, + }) => { const base = flatWorkgroupIndex(wid, numWorkgroups) * LOCAL_BLOCK; if (base >= sortLayout.$.data.length) { return; } - loadShared(base, tid); - + loadShared(base, lid.x); for (let kShift = d.u32(1); kShift <= LOCAL_BLOCK_LOG2; kShift++) { - mergeDown(base, tid, kShift, d.u32(1) << kShift); + mergeDown(base, lid.x, kShift, d.u32(1) << kShift); } - std.workgroupBarrier(); - storeShared(base, tid); + storeShared(base, lid.x); }); - const localMergeKernel = tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const tid = lid.x; + const localMerge = tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], in: dispatchIn })(({ + lid, + wid, + numWorkgroups, + }) => { const base = flatWorkgroupIndex(wid, numWorkgroups) * LOCAL_BLOCK; if (base >= sortLayout.$.data.length) { return; } - loadShared(base, tid); - mergeDown(base, tid, LOCAL_BLOCK_LOG2, sortLayout.$.uniforms.k); + loadShared(base, lid.x); + mergeDown(base, lid.x, d.u32(LOCAL_BLOCK_LOG2), sortLayout.$.uniforms.k); std.workgroupBarrier(); - storeShared(base, tid); + storeShared(base, lid.x); }); - return { localSortKernel, localMergeKernel }; + return { localSort, localMerge }; } interface SortStep { @@ -323,220 +260,129 @@ export function createBitonicSorter< const keyBuffer = data as KeyBuffer; const valueBuffer = options?.values as ValueBuffer | undefined; - const originalSize = keyBuffer.dataType.elementCount; - if (originalSize === 0) { + const keyType = keyBuffer.dataType.elementType; + const size = keyBuffer.dataType.elementCount; + const paddedSize = nextPowerOf2(size); + + if (size === 0) { throw new Error('Cannot create a bitonic sorter for an empty buffer.'); } - const paddedSize = nextPowerOf2(originalSize); - const wasPadded = paddedSize !== originalSize; - - if (valueBuffer && valueBuffer.dataType.elementCount !== originalSize) { + if (valueBuffer && valueBuffer.dataType.elementCount !== size) { throw new Error( - `The values buffer (${valueBuffer.dataType.elementCount} elements) must match the key buffer (${originalSize} elements).`, + `The values buffer (${valueBuffer.dataType.elementCount} elements) must match the key buffer (${size} elements).`, ); } - if (valueBuffer && wasPadded) { + if (valueBuffer && paddedSize !== size) { throw new Error('Bitonic sorting with a values buffer requires a power-of-two element count.'); } - const keyType = keyBuffer.dataType.elementType; - const paddingValue = options?.paddingValue ?? defaultPaddingValues[keyType.type]; - const compareFunc = options?.compare ?? defaultCompares[keyType.type]; - const schemas = makeBitonicSchemas(keyType, valueBuffer?.dataType.elementType); - const kernels = makeKernels(schemas); - - const ownedBuffers: { destroy(): void }[] = []; + const owned: { destroy(): void }[] = []; const steps: SortStep[] = []; - const sortWorkgroups = decomposeWorkgroups(Math.ceil(paddedSize / 2 / WORKGROUP_SIZE)); - const padWorkgroups = decomposeWorkgroups(Math.ceil(paddedSize / WORKGROUP_SIZE)); - const copyBackWorkgroups = decomposeWorkgroups(Math.ceil(originalSize / WORKGROUP_SIZE)); - let workBuffer = keyBuffer; - let workValuesBuffer = valueBuffer; - let copyBackParamsBuffer: - | (TgpuBuffer & UniformFlag) - | undefined; - - const copyPadPipeline = wasPadded - ? root.createComputePipeline({ compute: kernels.copyPadKernel }) - : undefined; - const copyBackPipeline = wasPadded - ? root.createComputePipeline({ compute: kernels.copyBackKernel }) - : undefined; - const valsCopyPipeline = - wasPadded && kernels.valsCopyKernel - ? root.createComputePipeline({ compute: kernels.valsCopyKernel }) - : undefined; + let unpadStep: SortStep | undefined; - if (wasPadded && copyPadPipeline && copyBackPipeline) { - workBuffer = root.createBuffer(d.arrayOf(keyType, paddedSize)).$usage('storage') as KeyBuffer; - ownedBuffers.push(workBuffer); - - const copyPadParams = root - .createBuffer(schemas.copyParamsType, { - srcLength: originalSize, - dstLength: paddedSize, - paddingValue, - }) - .$usage('uniform'); - copyBackParamsBuffer = root - .createBuffer(schemas.copyParamsType, { - srcLength: originalSize, - dstLength: originalSize, - paddingValue: 0, - }) + if (paddedSize !== size) { + const { copyLayout, pad, unpad } = makePaddingKernels(keyType, size, paddedSize); + const padding = root + .createBuffer(keyType, options?.paddingValue ?? defaultPaddingValues[keyType.type]) .$usage('uniform'); - ownedBuffers.push(copyPadParams, copyBackParamsBuffer); + workBuffer = root.createBuffer(d.arrayOf(keyType, paddedSize)).$usage('storage') as KeyBuffer; + owned.push(padding, workBuffer); steps.push({ - pipeline: copyPadPipeline.with( - root.createBindGroup(schemas.copyLayout, { - src: keyBuffer, - dst: workBuffer, - params: copyPadParams, - }), - ), - workgroups: padWorkgroups, + pipeline: root + .createComputePipeline({ compute: pad }) + .with(root.createBindGroup(copyLayout, { src: keyBuffer, dst: workBuffer, padding })), + workgroups: decomposeWorkgroups(Math.ceil(paddedSize / WORKGROUP_SIZE)), }); - - if (valueBuffer && schemas.valsCopyLayout && valsCopyPipeline) { - workValuesBuffer = root - .createBuffer(d.arrayOf(valueBuffer.dataType.elementType, paddedSize)) - .$usage('storage') as ValueBuffer; - ownedBuffers.push(workValuesBuffer); - - steps.push({ - pipeline: valsCopyPipeline.with( - root.createBindGroup(schemas.valsCopyLayout, { - src: valueBuffer, - dst: workValuesBuffer, - params: copyPadParams, - }), - ), - workgroups: copyBackWorkgroups, - }); - } + unpadStep = { + pipeline: root + .createComputePipeline({ compute: unpad }) + .with(root.createBindGroup(copyLayout, { src: workBuffer, dst: keyBuffer, padding })), + workgroups: decomposeWorkgroups(Math.ceil(size / WORKGROUP_SIZE)), + }; } - const valsBindGroup = - schemas.valsLayout && workValuesBuffer - ? root.createBindGroup(schemas.valsLayout, { vals: workValuesBuffer }) - : undefined; + const valsBindGroup = valueBuffer + ? root.createBindGroup(schemas.valsLayout, { vals: valueBuffer }) + : undefined; - let sortPipeline = root.with(compareSlot, compareFunc).createComputePipeline({ - compute: kernels.bitonicStepKernel, - }); - if (valsBindGroup) { - sortPipeline = sortPipeline.with(valsBindGroup); - } + const compare = options?.compare ?? defaultCompare; - const valueTypeSize = valueBuffer ? d.sizeOf(valueBuffer.dataType.elementType) : 0; - const sharedMemoryBytes = LOCAL_BLOCK * (d.sizeOf(keyType) + valueTypeSize); - const workgroupStorageLimit = root.device.limits.maxComputeWorkgroupStorageSize || 16384; - const useLocalKernels = paddedSize >= LOCAL_BLOCK && sharedMemoryBytes <= workgroupStorageLimit; + function createSortPipeline(compute: TgpuComputeFn): TgpuComputePipeline { + const pipeline = root.with(compareSlot, compare).createComputePipeline({ compute }); + return valsBindGroup ? pipeline.with(valsBindGroup) : pipeline; + } - function pushGlobalStep(k: number, j: number): void { - const jShift = 31 - Math.clz32(j); - const uniformBuffer = root.createBuffer(sortUniformsType, { k, jShift }).$usage('uniform'); - ownedBuffers.push(uniformBuffer); + function pushStep( + pipeline: TgpuComputePipeline, + k: number, + jShift: number, + workgroups: [number, number, number], + ): void { + const uniforms = root.createBuffer(sortUniformsType, { k, jShift }).$usage('uniform'); + owned.push(uniforms); steps.push({ - pipeline: sortPipeline.with( - root.createBindGroup(schemas.sortLayout, { data: workBuffer, uniforms: uniformBuffer }), + pipeline: pipeline.with( + root.createBindGroup(schemas.sortLayout, { data: workBuffer, uniforms }), ), - workgroups: sortWorkgroups, + workgroups, }); } - if (useLocalKernels) { - const localKernels = makeLocalKernels(schemas, valueBuffer?.dataType.elementType); - let localSortPipeline = root.with(compareSlot, compareFunc).createComputePipeline({ - compute: localKernels.localSortKernel, - }); - let localMergePipeline = root.with(compareSlot, compareFunc).createComputePipeline({ - compute: localKernels.localMergeKernel, - }); - if (valsBindGroup) { - localSortPipeline = localSortPipeline.with(valsBindGroup); - localMergePipeline = localMergePipeline.with(valsBindGroup); - } + const payloadSize = schemas.valueType ? d.sizeOf(schemas.valueType) : 0; + const sharedMemoryBytes = LOCAL_BLOCK * (d.sizeOf(keyType) + payloadSize); + const useLocalKernels = + paddedSize >= LOCAL_BLOCK && + sharedMemoryBytes <= root.device.limits.maxComputeWorkgroupStorageSize; - const blockWorkgroups = decomposeWorkgroups(paddedSize / LOCAL_BLOCK); + const globalWorkgroups = decomposeWorkgroups(Math.ceil(paddedSize / 2 / WORKGROUP_SIZE)); + const globalPipeline = createSortPipeline(makeGlobalStepKernel(schemas)); - function pushLocalStep(pipeline: TgpuComputePipeline, k: number): void { - const uniformBuffer = root.createBuffer(sortUniformsType, { k, jShift: 0 }).$usage('uniform'); - ownedBuffers.push(uniformBuffer); - - steps.push({ - pipeline: pipeline.with( - root.createBindGroup(schemas.sortLayout, { data: workBuffer, uniforms: uniformBuffer }), - ), - workgroups: blockWorkgroups, - }); - } + if (useLocalKernels) { + const { localSort, localMerge } = makeLocalKernels(schemas); + const localSortPipeline = createSortPipeline(localSort); + const localMergePipeline = createSortPipeline(localMerge); + const blockWorkgroups = decomposeWorkgroups(paddedSize / LOCAL_BLOCK); - pushLocalStep(localSortPipeline, 0); + pushStep(localSortPipeline, 0, 0, blockWorkgroups); for (let k = LOCAL_BLOCK * 2; k <= paddedSize; k <<= 1) { for (let j = k >> 1; j >= LOCAL_BLOCK; j >>= 1) { - pushGlobalStep(k, j); + pushStep(globalPipeline, k, Math.log2(j), globalWorkgroups); } - pushLocalStep(localMergePipeline, k); + pushStep(localMergePipeline, k, 0, blockWorkgroups); } } else { for (let k = 2; k <= paddedSize; k <<= 1) { for (let j = k >> 1; j > 0; j >>= 1) { - pushGlobalStep(k, j); + pushStep(globalPipeline, k, Math.log2(j), globalWorkgroups); } } } - if (wasPadded && copyBackPipeline && copyBackParamsBuffer) { - steps.push({ - pipeline: copyBackPipeline.with( - root.createBindGroup(schemas.copyLayout, { - src: workBuffer, - dst: keyBuffer, - params: copyBackParamsBuffer, - }), - ), - workgroups: copyBackWorkgroups, - }); - - if (valueBuffer && workValuesBuffer && schemas.valsCopyLayout && valsCopyPipeline) { - steps.push({ - pipeline: valsCopyPipeline.with( - root.createBindGroup(schemas.valsCopyLayout, { - src: workValuesBuffer, - dst: valueBuffer, - params: copyBackParamsBuffer, - }), - ), - workgroups: copyBackWorkgroups, - }); - } - } - - function run(runOptions?: BitonicSorterRunOptions): void { - const recording = beginRunPass(root.device, runOptions); - for (const step of steps) { - step.pipeline.with(recording.pass).dispatchWorkgroups(...step.workgroups); - } - recording.finish(); - } - - function destroy(): void { - for (const buffer of ownedBuffers) { - buffer.destroy(); - } + if (unpadStep) { + steps.push(unpadStep); } return { - originalSize, + size, paddedSize, - wasPadded, - run, - destroy, + + run(runOptions?: RunOptions): void { + const recording = beginRunPass(root.device, runOptions); + for (const step of steps) { + step.pipeline.with(recording.pass).dispatchWorkgroups(...step.workgroups); + } + recording.finish(); + }, + + destroy(): void { + for (const buffer of owned) { + buffer.destroy(); + } + }, }; } diff --git a/packages/typegpu-sort/src/bitonic/index.ts b/packages/typegpu-sort/src/bitonic/index.ts index 3c4a2d1929..0c463d9f40 100644 --- a/packages/typegpu-sort/src/bitonic/index.ts +++ b/packages/typegpu-sort/src/bitonic/index.ts @@ -1,4 +1,2 @@ export { type BitonicKeyType, createBitonicSorter } from './bitonicSort.ts'; -export { compareSlot, defaultCompare, defaultCompares } from './slots.ts'; -export type { BitonicSorter, BitonicSorterOptions, BitonicSorterRunOptions } from './types.ts'; -export { decomposeWorkgroups } from './utils.ts'; +export type { BitonicSorter, BitonicSorterOptions } from './types.ts'; diff --git a/packages/typegpu-sort/src/bitonic/slots.ts b/packages/typegpu-sort/src/bitonic/slots.ts index 05a0dca35b..ef8f228a48 100644 --- a/packages/typegpu-sort/src/bitonic/slots.ts +++ b/packages/typegpu-sort/src/bitonic/slots.ts @@ -1,16 +1,9 @@ -import { tgpu, d } from 'typegpu'; +import { tgpu } from 'typegpu'; -/** Default comparison function: ascending order (a < b means a comes before b) */ -export const defaultCompare = tgpu.fn([d.u32, d.u32], d.bool)((a, b) => a < b); - -const defaultCompareI32 = tgpu.fn([d.i32, d.i32], d.bool)((a, b) => a < b); -const defaultCompareF32 = tgpu.fn([d.f32, d.f32], d.bool)((a, b) => a < b); - -export const defaultCompares = { - u32: defaultCompare, - i32: defaultCompareI32, - f32: defaultCompareF32, -} as const; +export function defaultCompare(a: number, b: number): boolean { + 'use gpu'; + return a < b; +} export const defaultPaddingValues = { u32: 0xffffffff, @@ -18,6 +11,4 @@ export const defaultPaddingValues = { f32: Number.POSITIVE_INFINITY, } as const; -/** Slot for customizing the comparison function in bitonic sort. - * The function should return true if the first argument should come before the second. */ -export const compareSlot = tgpu.slot<(a: number, b: number) => boolean>(defaultCompare); +export const compareSlot = tgpu.slot<(a: number, b: number) => boolean>(); diff --git a/packages/typegpu-sort/src/bitonic/types.ts b/packages/typegpu-sort/src/bitonic/types.ts index ab75632858..63e09e1073 100644 --- a/packages/typegpu-sort/src/bitonic/types.ts +++ b/packages/typegpu-sort/src/bitonic/types.ts @@ -1,5 +1,5 @@ import type { d, StorageFlag, TgpuBuffer } from 'typegpu'; -import type { RunPassOptions } from '../runPass.ts'; +import type { Sorter } from '../types.ts'; export interface BitonicSorterOptions { /** Custom comparison function. Returns true if first argument should come before second. @@ -18,19 +18,7 @@ export interface BitonicSorterOptions> & StorageFlag; } -export type BitonicSorterRunOptions = RunPassOptions; - -export interface BitonicSorter { - /** Original size of the input array */ - readonly originalSize: number; - /** Size after padding to power of 2 */ +export interface BitonicSorter extends Sorter { + /** Size the keys are padded to, a power of two */ readonly paddedSize: number; - /** Whether the array was padded */ - readonly wasPadded: boolean; - - /** Execute the sort. Can be called repeatedly. */ - run(options?: BitonicSorterRunOptions): void; - - /** Clean up all GPU resources. */ - destroy(): void; } diff --git a/packages/typegpu-sort/src/bitonic/utils.ts b/packages/typegpu-sort/src/bitonic/utils.ts deleted file mode 100644 index 8961c7507c..0000000000 --- a/packages/typegpu-sort/src/bitonic/utils.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Returns the next power of 2 greater than or equal to n. - * If n is already a power of 2, returns n. - */ -export function nextPowerOf2(n: number): number { - if (n <= 0) return 1; - if ((n & (n - 1)) === 0) return n; - let p = 1; - while (p < n) p <<= 1; - return p; -} - -const MAX_WORKGROUPS_PER_DIMENSION = 65535; - -/** - * Decomposes a total workgroup count into a 3D dispatch grid (x, y, z), - * respecting the WebGPU limit of 65535 workgroups per dimension. - */ -export function decomposeWorkgroups(total: number): [number, number, number] { - if (total <= 0) { - return [1, 1, 1]; - } - - const x = Math.min(total, MAX_WORKGROUPS_PER_DIMENSION); - const remainingAfterX = Math.ceil(total / x); - - const y = Math.min(remainingAfterX, MAX_WORKGROUPS_PER_DIMENSION); - const remainingAfterY = Math.ceil(remainingAfterX / y); - - const z = Math.min(remainingAfterY, MAX_WORKGROUPS_PER_DIMENSION); - - if (Math.ceil(total / (x * y * z)) > 1) { - throw new Error( - `Required workgroups (${total}) exceed device dispatch limits (${MAX_WORKGROUPS_PER_DIMENSION} per dimension)`, - ); - } - - return [x, y, z]; -} diff --git a/packages/typegpu-sort/src/dispatch.ts b/packages/typegpu-sort/src/dispatch.ts new file mode 100644 index 0000000000..d4c5d767ee --- /dev/null +++ b/packages/typegpu-sort/src/dispatch.ts @@ -0,0 +1,31 @@ +import { d } from 'typegpu'; + +const MAX_WORKGROUPS_PER_DIMENSION = 65535; + +/** + * Decomposes a total workgroup count into a 3D dispatch grid (x, y, z), + * respecting the WebGPU limit of 65535 workgroups per dimension. The grid can cover more + * workgroups than requested, so kernels have to guard against running past their data. + */ +export function decomposeWorkgroups(total: number): [number, number, number] { + if (total <= 1) { + return [1, 1, 1]; + } + + const x = Math.min(total, MAX_WORKGROUPS_PER_DIMENSION); + const rows = Math.ceil(total / x); + const y = Math.min(rows, MAX_WORKGROUPS_PER_DIMENSION); + + return [x, y, Math.ceil(rows / y)]; +} + +export const dispatchIn = { + lid: d.builtin.localInvocationId, + wid: d.builtin.workgroupId, + numWorkgroups: d.builtin.numWorkgroups, +} as const; + +export function flatWorkgroupIndex(wid: d.v3u, numWorkgroups: d.v3u): number { + 'use gpu'; + return wid.x + wid.y * numWorkgroups.x + wid.z * numWorkgroups.x * numWorkgroups.y; +} diff --git a/packages/typegpu-sort/src/index.ts b/packages/typegpu-sort/src/index.ts index 5b2789dd8c..f9b86189a0 100644 --- a/packages/typegpu-sort/src/index.ts +++ b/packages/typegpu-sort/src/index.ts @@ -1,25 +1,18 @@ -export { - compareSlot, - createBitonicSorter, - defaultCompare, - defaultCompares, - decomposeWorkgroups, -} from './bitonic/index.ts'; -export type { - BitonicKeyType, - BitonicSorter, - BitonicSorterOptions, - BitonicSorterRunOptions, -} from './bitonic/index.ts'; +export { decomposeWorkgroups } from './dispatch.ts'; +export type { RunOptions, Sorter } from './types.ts'; + +export { type BitonicKeyType, createBitonicSorter } from './bitonic/index.ts'; +export type { BitonicSorter, BitonicSorterOptions } from './bitonic/index.ts'; export { createRadixSorter } from './radix/index.ts'; -export type { RadixSorter, RadixSorterOptions, RadixSorterRunOptions } from './radix/index.ts'; +export type { RadixSorterOptions } from './radix/index.ts'; -export { prefixScan, scan, createPrefixScanComputer, PrefixScanComputer } from './scan/index.ts'; -export type { - BinaryOp, - PrefixScanPlan, - ScanBuffer, - ScanElementType, - ScanRunOptions, +export { + createPrefixScanComputer, + type PrefixScanComputer, + type PrefixScanPlan, + prefixScan, + reduce, + type ScanBuffer, } from './scan/index.ts'; +export type { BinaryOp, ScanElementType } from './scan/index.ts'; diff --git a/packages/typegpu-sort/src/radix/count.ts b/packages/typegpu-sort/src/radix/count.ts index 65d5e4ecea..c8115e6a56 100644 --- a/packages/typegpu-sort/src/radix/count.ts +++ b/packages/typegpu-sort/src/radix/count.ts @@ -1,53 +1,45 @@ import { tgpu, d, std } from 'typegpu'; -import { flatWorkgroupIndex } from '../wgslUtils.ts'; +import { dispatchIn, flatWorkgroupIndex } from '../dispatch.ts'; import { histLayout, KEYS_PER_THREAD, - paramsLayout, type RadixSchemas, + shiftLayout, TILE_SIZE, TILE_THREADS, wgHist, } from './schemas.ts'; -export function makeCountKernel(schemas: RadixSchemas, elementCount: number) { +export function makeCountKernel(schemas: RadixSchemas, elementCount: number, numTiles: number) { const { ioLayout, digitFn } = schemas; const needsBoundsCheck = elementCount % TILE_SIZE !== 0; const lastIndex = elementCount - 1; - return tgpu.computeFn({ - workgroupSize: [TILE_THREADS], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const local_i = lid.x; - const tile_id = flatWorkgroupIndex(wid, numWorkgroups); - const tile_base = tile_id * TILE_SIZE; - const shift = paramsLayout.$.params.shift; + return tgpu.computeFn({ workgroupSize: [TILE_THREADS], in: dispatchIn })( + ({ lid, wid, numWorkgroups }) => { + const localIdx = lid.x; + const tileId = flatWorkgroupIndex(wid, numWorkgroups); + if (tileId >= numTiles) { + return; + } - std.atomicStore(wgHist.$[local_i] as d.atomicU32, 0); - std.workgroupBarrier(); + const tileBase = tileId * TILE_SIZE; + const shift = shiftLayout.$.shift; - for (const k of tgpu.unroll(std.range(KEYS_PER_THREAD))) { - const global_i = tile_base + k * TILE_THREADS + local_i; - let load_i = global_i; - if (needsBoundsCheck) { - load_i = std.min(global_i, lastIndex); - } - const digit = digitFn(ioLayout.$.src[load_i] as number, shift); - const inBounds = needsBoundsCheck ? global_i < elementCount : true; - if (inBounds) { - std.atomicAdd(wgHist.$[digit] as d.atomicU32, 1); + for (const k of tgpu.unroll(std.range(KEYS_PER_THREAD))) { + const globalIdx = tileBase + k * TILE_THREADS + localIdx; + const loadIdx = needsBoundsCheck ? std.min(globalIdx, lastIndex) : globalIdx; + const digit = digitFn(ioLayout.$.src[loadIdx] as number, shift); + + if (needsBoundsCheck ? globalIdx < elementCount : true) { + std.atomicAdd(wgHist.$[digit] as d.atomicU32, 1); + } } - } - std.workgroupBarrier(); + std.workgroupBarrier(); - const count = std.atomicLoad(wgHist.$[local_i] as d.atomicU32); - if (tile_id < paramsLayout.$.params.numTiles) { - histLayout.$.hist[local_i * paramsLayout.$.params.numTiles + tile_id] = count; - } - }); + histLayout.$.hist[localIdx * numTiles + tileId] = std.atomicLoad( + wgHist.$[localIdx] as d.atomicU32, + ); + }, + ); } diff --git a/packages/typegpu-sort/src/radix/index.ts b/packages/typegpu-sort/src/radix/index.ts index b73050c136..18660852cf 100644 --- a/packages/typegpu-sort/src/radix/index.ts +++ b/packages/typegpu-sort/src/radix/index.ts @@ -1,2 +1,2 @@ export { createRadixSorter } from './radixSort.ts'; -export type { RadixSorter, RadixSorterOptions, RadixSorterRunOptions } from './types.ts'; +export type { RadixSorterOptions } from './types.ts'; diff --git a/packages/typegpu-sort/src/radix/radixSort.ts b/packages/typegpu-sort/src/radix/radixSort.ts index 5d96a4a223..f6ca5d7f69 100644 --- a/packages/typegpu-sort/src/radix/radixSort.ts +++ b/packages/typegpu-sort/src/radix/radixSort.ts @@ -1,21 +1,21 @@ import { d, std, type StorageFlag, type TgpuBuffer, type TgpuRoot } from 'typegpu'; -import { decomposeWorkgroups } from '../bitonic/utils.ts'; +import { decomposeWorkgroups } from '../dispatch.ts'; import { beginRunPass } from '../runPass.ts'; import { createPrefixScanComputer } from '../scan/index.ts'; +import type { RunOptions, Sorter } from '../types.ts'; import { makeCountKernel } from './count.ts'; import { makeScatterKernel } from './scatter.ts'; import { histLayout, makeRadixSchemas, NUM_PASSES, - paramsLayout, - paramsType, RADIX_BITS, RADIX_SIZE, type RadixKeyType, + shiftLayout, TILE_SIZE, } from './schemas.ts'; -import type { RadixSorter, RadixSorterOptions, RadixSorterRunOptions } from './types.ts'; +import type { RadixSorterOptions } from './types.ts'; type KeyBuffer = TgpuBuffer> & StorageFlag; type ValueBuffer = TgpuBuffer> & StorageFlag; @@ -35,111 +35,114 @@ export function createRadixSorter< root: TgpuRoot, keys: TgpuBuffer> & StorageFlag, options?: RadixSorterOptions, -): RadixSorter { +): Sorter { const keyBuffer = keys as KeyBuffer; const valueBuffer = options?.values as ValueBuffer | undefined; + const keyType = keyBuffer.dataType.elementType; + const size = keyBuffer.dataType.elementCount; - const n = keyBuffer.dataType.elementCount; - if (n === 0) { + if (size === 0) { throw new Error('Cannot create a radix sorter for an empty buffer.'); } - - if (valueBuffer && valueBuffer.dataType.elementCount !== n) { + if (valueBuffer && valueBuffer.dataType.elementCount !== size) { throw new Error( - `The values buffer (${valueBuffer.dataType.elementCount} elements) must match the key buffer (${n} elements).`, + `The values buffer (${valueBuffer.dataType.elementCount} elements) must match the key buffer (${size} elements).`, ); } const schemas = makeRadixSchemas( - keyBuffer.dataType.elementType, + keyType, options?.direction ?? 'ascending', valueBuffer?.dataType.elementType, ); - const numTiles = Math.ceil(n / TILE_SIZE); + const numTiles = Math.ceil(size / TILE_SIZE); + const dispatch = decomposeWorkgroups(numTiles); const histBuffer = root.createBuffer(d.arrayOf(d.u32, numTiles * RADIX_SIZE)).$usage('storage'); - const tempBuffer = root - .createBuffer(d.arrayOf(keyBuffer.dataType.elementType, n)) - .$usage('storage') as KeyBuffer; - const tempValuesBuffer = valueBuffer - ? (root - .createBuffer(d.arrayOf(valueBuffer.dataType.elementType, n)) - .$usage('storage') as ValueBuffer) - : undefined; - const paramBuffers = Array.from({ length: NUM_PASSES }, (_, pass) => - root.createBuffer(paramsType, { shift: pass * RADIX_BITS, numTiles }).$usage('uniform'), - ); + const tempBuffer = root.createBuffer(d.arrayOf(keyType, size)).$usage('storage') as KeyBuffer; + const owned: { destroy(): void }[] = [histBuffer, tempBuffer]; - const scanComputer = createPrefixScanComputer(root, { + const scanPlan = createPrefixScanComputer(root, { operation: std.add, identityElement: 0, dataType: d.u32, - }); - const scanPlan = scanComputer.prepare(histBuffer); - - const scatterPipeline = root.createComputePipeline({ compute: makeScatterKernel(schemas, n) }); - - const countPipeline = root.createComputePipeline({ compute: makeCountKernel(schemas, n) }); + }).prepare(histBuffer); + + const tempValues = + valueBuffer && + (root + .createBuffer(d.arrayOf(valueBuffer.dataType.elementType, size)) + .$usage('storage') as ValueBuffer); + if (tempValues) { + owned.push(tempValues); + } const histBg = root.createBindGroup(histLayout, { hist: histBuffer }); - const paramBgs = paramBuffers.map((buffer) => - root.createBindGroup(paramsLayout, { params: buffer }), - ); - const ioBgAtoB = root.createBindGroup(schemas.ioLayout, { src: keyBuffer, dst: tempBuffer }); - const ioBgBtoA = root.createBindGroup(schemas.ioLayout, { src: tempBuffer, dst: keyBuffer }); + const ioBgKeysToTemp = root.createBindGroup(schemas.ioLayout, { + src: keyBuffer, + dst: tempBuffer, + }); + const ioBgTempToKeys = root.createBindGroup(schemas.ioLayout, { + src: tempBuffer, + dst: keyBuffer, + }); const valuesBgs = - schemas.valuesLayout && valueBuffer && tempValuesBuffer - ? [ - root.createBindGroup(schemas.valuesLayout, { + valueBuffer && tempValues + ? { + keysToTemp: root.createBindGroup(schemas.valuesLayout, { srcVals: valueBuffer, - dstVals: tempValuesBuffer, + dstVals: tempValues, }), - root.createBindGroup(schemas.valuesLayout, { - srcVals: tempValuesBuffer, + tempToKeys: root.createBindGroup(schemas.valuesLayout, { + srcVals: tempValues, dstVals: valueBuffer, }), - ] + } : undefined; - const [wgX, wgY, wgZ] = decomposeWorkgroups(numTiles); - - function run(runOptions?: RadixSorterRunOptions): void { - const recording = beginRunPass(root.device, runOptions); - const computePass = recording.pass; + const countPipeline = root + .createComputePipeline({ compute: makeCountKernel(schemas, size, numTiles) }) + .with(histBg); + const scatterPipeline = root + .createComputePipeline({ compute: makeScatterKernel(schemas, size, numTiles) }) + .with(histBg); + + const passes = Array.from({ length: NUM_PASSES }, (_, pass) => { + const shift = root.createBuffer(d.u32, pass * RADIX_BITS).$usage('uniform'); + owned.push(shift); + + const shiftBg = root.createBindGroup(shiftLayout, { shift }); + const forward = pass % 2 === 0; + const ioBg = forward ? ioBgKeysToTemp : ioBgTempToKeys; + const valuesBg = forward ? valuesBgs?.keysToTemp : valuesBgs?.tempToKeys; + + const scatter = scatterPipeline.with(ioBg).with(shiftBg); + return { + count: countPipeline.with(ioBg).with(shiftBg), + scatter: valuesBg ? scatter.with(valuesBg) : scatter, + }; + }); - for (let pass = 0; pass < NUM_PASSES; pass++) { - const paramsBg = paramBgs[pass] as (typeof paramBgs)[number]; - const ioBg = pass % 2 === 0 ? ioBgAtoB : ioBgBtoA; + return { + size, - let scatterPipe = scatterPipeline.with(histBg).with(ioBg).with(paramsBg); - if (valuesBgs) { - scatterPipe = scatterPipe.with(valuesBgs[pass % 2] as (typeof valuesBgs)[number]); + run(runOptions?: RunOptions): void { + const recording = beginRunPass(root.device, runOptions); + for (const { count, scatter } of passes) { + count.with(recording.pass).dispatchWorkgroups(...dispatch); + scanPlan.run({ pass: recording.pass }); + scatter.with(recording.pass).dispatchWorkgroups(...dispatch); } + recording.finish(); + }, - countPipeline - .with(histBg) - .with(ioBg) - .with(paramsBg) - .with(computePass) - .dispatchWorkgroups(wgX, wgY, wgZ); - scanPlan.run({ pass: computePass }); - scatterPipe.with(computePass).dispatchWorkgroups(wgX, wgY, wgZ); - } - - recording.finish(); - } - - function destroy(): void { - scanPlan.destroy(); - histBuffer.destroy(); - tempBuffer.destroy(); - tempValuesBuffer?.destroy(); - for (const buffer of paramBuffers) { - buffer.destroy(); - } - } - - return { size: n, run, destroy }; + destroy(): void { + scanPlan.destroy(); + for (const buffer of owned) { + buffer.destroy(); + } + }, + }; } diff --git a/packages/typegpu-sort/src/radix/scatter.ts b/packages/typegpu-sort/src/radix/scatter.ts index 2f86b164b9..0eb5c61f44 100644 --- a/packages/typegpu-sort/src/radix/scatter.ts +++ b/packages/typegpu-sort/src/radix/scatter.ts @@ -1,85 +1,85 @@ import { tgpu, d, std, type TgpuComputeFn } from 'typegpu'; -import { flatWorkgroupIndex } from '../wgslUtils.ts'; +import { dispatchIn, flatWorkgroupIndex } from '../dispatch.ts'; import { histLayout, KEYS_PER_THREAD, - paramsLayout, RADIX_SIZE, type RadixSchemas, + shiftLayout, TILE_SIZE, TILE_THREADS, } from './schemas.ts'; -const runningTotal = tgpu.workgroupVar(d.arrayOf(d.u32, RADIX_SIZE)); const BITSET_WORD_BITS = 32; +const BITSET_WORD_SHIFT = Math.log2(BITSET_WORD_BITS); const BITSET_WORDS = TILE_THREADS / BITSET_WORD_BITS; + +const runningTotal = tgpu.workgroupVar(d.arrayOf(d.u32, RADIX_SIZE)); const digitBits = tgpu.workgroupVar(d.arrayOf(d.atomic(d.u32), BITSET_WORDS * RADIX_SIZE)); -export function makeScatterKernel(schemas: RadixSchemas, elementCount: number): TgpuComputeFn { +export function makeScatterKernel( + schemas: RadixSchemas, + elementCount: number, + numTiles: number, +): TgpuComputeFn { const { ioLayout, digitFn, writeOutput } = schemas; const needsBoundsCheck = elementCount % TILE_SIZE !== 0; const lastIndex = elementCount - 1; - return tgpu.computeFn({ - workgroupSize: [TILE_THREADS], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const local_i = lid.x; - const tile_id = flatWorkgroupIndex(wid, numWorkgroups); - const tile_base = tile_id * TILE_SIZE; - const bitset_word = local_i >> 5; - const bitset_mask = d.u32(1) << (local_i & (BITSET_WORD_BITS - 1)); - const earlierBits = bitset_mask - 1; - const shift = paramsLayout.$.params.shift; + return tgpu.computeFn({ workgroupSize: [TILE_THREADS], in: dispatchIn })( + ({ lid, wid, numWorkgroups }) => { + const localIdx = lid.x; + const tileId = flatWorkgroupIndex(wid, numWorkgroups); + if (tileId >= numTiles) { + return; + } - runningTotal.$[local_i] = histLayout.$.hist[ - local_i * paramsLayout.$.params.numTiles + tile_id - ] as number; + const tileBase = tileId * TILE_SIZE; + const shift = shiftLayout.$.shift; + const bitsetWord = localIdx >> BITSET_WORD_SHIFT; + const bitsetMask = d.u32(1) << (localIdx & (BITSET_WORD_BITS - 1)); + const earlierBits = bitsetMask - 1; - for (const k of tgpu.unroll(std.range(KEYS_PER_THREAD))) { - const global_i = tile_base + k * TILE_THREADS + local_i; - let load_i = global_i; - if (needsBoundsCheck) { - load_i = std.min(global_i, lastIndex); - } - const key = ioLayout.$.src[load_i] as number; - const my_digit = digitFn(key, shift); - const inBounds = needsBoundsCheck ? global_i < elementCount : true; + runningTotal.$[localIdx] = histLayout.$.hist[localIdx * numTiles + tileId] as number; - if (inBounds) { - std.atomicOr(digitBits.$[bitset_word * RADIX_SIZE + my_digit] as d.atomicU32, bitset_mask); - } - std.workgroupBarrier(); + for (const k of tgpu.unroll(std.range(KEYS_PER_THREAD))) { + const globalIdx = tileBase + k * TILE_THREADS + localIdx; + const loadIdx = needsBoundsCheck ? std.min(globalIdx, lastIndex) : globalIdx; + const key = ioLayout.$.src[loadIdx] as number; + const digit = digitFn(key, shift); + const inBounds = needsBoundsCheck ? globalIdx < elementCount : true; - let rank = d.u32(0); - let digit_total = d.u32(0); - if (inBounds) { - for (const word of tgpu.unroll(std.range(BITSET_WORDS))) { - const bits = std.atomicLoad(digitBits.$[word * RADIX_SIZE + my_digit] as d.atomicU32); - const mask = std.select( - std.select(d.u32(0), earlierBits, word === bitset_word), - d.u32(0xffffffff), - word < bitset_word, - ); - rank = rank + std.countOneBits(bits & mask); - digit_total = digit_total + std.countOneBits(bits); + if (inBounds) { + std.atomicOr(digitBits.$[bitsetWord * RADIX_SIZE + digit] as d.atomicU32, bitsetMask); } - const output_pos = (runningTotal.$[my_digit] as number) + rank; - writeOutput(key, global_i, output_pos); - } - std.workgroupBarrier(); + std.workgroupBarrier(); + + let rank = d.u32(0); + let digitTotal = d.u32(0); + if (inBounds) { + for (const word of tgpu.unroll(std.range(BITSET_WORDS))) { + const bits = std.atomicLoad(digitBits.$[word * RADIX_SIZE + digit] as d.atomicU32); + const mask = std.select( + std.select(d.u32(0), earlierBits, word === bitsetWord), + d.u32(0xffffffff), + word < bitsetWord, + ); + rank = rank + std.countOneBits(bits & mask); + digitTotal = digitTotal + std.countOneBits(bits); + } - if (inBounds) { - std.atomicStore(digitBits.$[bitset_word * RADIX_SIZE + my_digit] as d.atomicU32, 0); - if (rank === 0) { - runningTotal.$[my_digit] = (runningTotal.$[my_digit] as number) + digit_total; + writeOutput(key, globalIdx, (runningTotal.$[digit] as number) + rank); } + std.workgroupBarrier(); + + if (inBounds) { + std.atomicStore(digitBits.$[bitsetWord * RADIX_SIZE + digit] as d.atomicU32, 0); + if (rank === 0) { + runningTotal.$[digit] = (runningTotal.$[digit] as number) + digitTotal; + } + } + std.workgroupBarrier(); } - std.workgroupBarrier(); - } - }); + }, + ); } diff --git a/packages/typegpu-sort/src/radix/schemas.ts b/packages/typegpu-sort/src/radix/schemas.ts index a563a42b5d..bb8d5d1c77 100644 --- a/packages/typegpu-sort/src/radix/schemas.ts +++ b/packages/typegpu-sort/src/radix/schemas.ts @@ -1,59 +1,62 @@ -import { tgpu, d, std, type TgpuFn } from 'typegpu'; +import { tgpu, d, std } from 'typegpu'; export const RADIX_BITS = 8; export const RADIX_SIZE = 1 << RADIX_BITS; -export const TILE_THREADS = 256; +export const NUM_PASSES = 32 / RADIX_BITS; +export const TILE_THREADS = RADIX_SIZE; export const KEYS_PER_THREAD = 8; export const TILE_SIZE = TILE_THREADS * KEYS_PER_THREAD; -export const NUM_PASSES = 32 / RADIX_BITS; export type RadixKeyType = d.U32 | d.I32 | d.F32; export type SortDirection = 'ascending' | 'descending'; -export const paramsType = d.struct({ shift: d.u32, numTiles: d.u32 }); - export const histLayout = tgpu.bindGroupLayout({ hist: { storage: d.arrayOf(d.u32), access: 'mutable' }, }); -export const paramsLayout = tgpu.bindGroupLayout({ - params: { uniform: paramsType }, +export const shiftLayout = tgpu.bindGroupLayout({ + shift: { uniform: d.u32 }, }); export const wgHist = tgpu.workgroupVar(d.arrayOf(d.atomic(d.u32), RADIX_SIZE)); -function makeDigitFn(keyType: RadixKeyType, direction: SortDirection) { - const ascendingImpls = { - u32: tgpu.fn([d.u32, d.u32], d.u32)((v, shift) => (v >> shift) & (RADIX_SIZE - 1)), - i32: tgpu.fn( - [d.i32, d.u32], - d.u32, - )((v, shift) => { - const raw = d.u32((v >> d.i32(shift)) & (RADIX_SIZE - 1)); - const signFix = std.select(d.u32(0), d.u32(RADIX_SIZE / 2), shift === 24); - return raw ^ signFix; - }), - f32: tgpu.fn( - [d.f32, d.u32], - d.u32, - )((v, shift) => { - // -0 and +0 must map to the same bits, otherwise they sort apart - const bits = std.select(std.bitcastF32toU32(v), d.u32(0), v === 0); - const mask = std.select(d.u32(0x80000000), d.u32(0xffffffff), bits >> 31 === 1); - return ((bits ^ mask) >> shift) & (RADIX_SIZE - 1); - }), - }; +function digitOfU32(v: number, shift: number): number { + 'use gpu'; + return (v >> shift) & (RADIX_SIZE - 1); +} + +function digitOfI32(v: number, shift: number): number { + 'use gpu'; + const raw = d.u32((v >> shift) & (RADIX_SIZE - 1)); + return raw ^ std.select(d.u32(0), d.u32(RADIX_SIZE / 2), shift === 24); +} + +function digitOfF32(v: number, shift: number): number { + 'use gpu'; + // -0 and +0 must map to the same bits, otherwise they sort apart + const bits = std.select(std.bitcastF32toU32(v), d.u32(0), v === 0); + const mask = std.select(d.u32(0x80000000), d.u32(0xffffffff), bits >> 31 === 1); + return ((bits ^ mask) >> shift) & (RADIX_SIZE - 1); +} + +const ascendingDigits = { + u32: digitOfU32, + i32: digitOfI32, + f32: digitOfF32, +} as const; - const ascending = ascendingImpls[keyType.type]; +export function makeDigitFn(keyType: RadixKeyType, direction: SortDirection) { + const ascending = ascendingDigits[keyType.type]; if (direction === 'ascending') { return ascending; } - const paramType = keyType as d.U32; - return tgpu.fn( - [paramType, d.u32], - d.u32, - )((v, shift) => RADIX_SIZE - 1 - ascending(v as never, shift)); + function descendingDigit(v: number, shift: number) { + 'use gpu'; + return RADIX_SIZE - 1 - ascending(v, shift); + } + + return descendingDigit; } export function makeRadixSchemas( @@ -66,30 +69,30 @@ export function makeRadixSchemas( dst: { storage: d.arrayOf(keyType), access: 'mutable' }, }); - const valuesLayout = valueType - ? tgpu.bindGroupLayout({ - srcVals: { storage: d.arrayOf(valueType), access: 'readonly' }, - dstVals: { storage: d.arrayOf(valueType), access: 'mutable' }, - }) - : undefined; - - const copyValue = valueType as unknown as (value: unknown) => number; - const writeOutput = - valuesLayout && valueType - ? tgpu.fn([keyType, d.u32, d.u32])((key, srcIdx, dstIdx) => { - (ioLayout.$.dst[dstIdx] as number) = key; - (valuesLayout.$.dstVals[dstIdx] as number) = copyValue(valuesLayout.$.srcVals[srcIdx]); - }) - : tgpu.fn([keyType, d.u32, d.u32])((key, _srcIdx, dstIdx) => { - (ioLayout.$.dst[dstIdx] as number) = key; - }); + const hasPayload = valueType !== undefined; + const payloadType = valueType ?? d.u32; + + const valuesLayout = tgpu.bindGroupLayout({ + srcVals: { storage: d.arrayOf(payloadType), access: 'readonly' }, + dstVals: { storage: d.arrayOf(payloadType), access: 'mutable' }, + }); + + function writeOutput(key: number, srcIdx: number, dstIdx: number) { + 'use gpu'; + (ioLayout.$.dst[dstIdx] as number) = key; + if (hasPayload) { + (valuesLayout.$.dstVals[dstIdx] as number) = std.copy( + valuesLayout.$.srcVals[srcIdx] as number, + ); + } + } return { keyType, ioLayout, valuesLayout, writeOutput, - digitFn: makeDigitFn(keyType, direction) as TgpuFn<(v: d.U32, shift: d.U32) => d.U32>, + digitFn: makeDigitFn(keyType, direction), }; } diff --git a/packages/typegpu-sort/src/radix/types.ts b/packages/typegpu-sort/src/radix/types.ts index cd0ec069fd..b8adaae1ef 100644 --- a/packages/typegpu-sort/src/radix/types.ts +++ b/packages/typegpu-sort/src/radix/types.ts @@ -1,5 +1,4 @@ import type { d, StorageFlag, TgpuBuffer } from 'typegpu'; -import type { RunPassOptions } from '../runPass.ts'; import type { SortDirection } from './schemas.ts'; export interface RadixSorterOptions { @@ -11,14 +10,3 @@ export interface RadixSorterOptions> & StorageFlag; } - -export type RadixSorterRunOptions = RunPassOptions; - -export interface RadixSorter { - /** Number of elements in the sorted buffer */ - readonly size: number; - /** Sorts the buffer in place. Can be called repeatedly */ - run(options?: RadixSorterRunOptions): void; - /** Destroys the internal buffers owned by this sorter */ - destroy(): void; -} diff --git a/packages/typegpu-sort/src/runPass.ts b/packages/typegpu-sort/src/runPass.ts index 1b21afd1f7..735c9c305d 100644 --- a/packages/typegpu-sort/src/runPass.ts +++ b/packages/typegpu-sort/src/runPass.ts @@ -1,41 +1,26 @@ -interface EncoderOptions { - /** Records the dispatches as a single compute pass on this encoder. Nothing is submitted */ - encoder: GPUCommandEncoder; - pass?: never; -} - -interface ExternalPassOptions { - encoder?: never; - /** Records the dispatches into this pass. Nothing is submitted and the pass is not ended */ - pass: GPUComputePassEncoder; -} - -/** Controls where a `run` call records its dispatches. Defaults to a standalone submit */ -export type RunPassOptions = EncoderOptions | ExternalPassOptions; +import type { RunOptions } from './types.ts'; -export interface RunPassRecording { +export interface RunRecording { pass: GPUComputePassEncoder; finish(): void; } -export function beginRunPass(device: GPUDevice, options?: RunPassOptions): RunPassRecording { - if (options?.pass && options.encoder) { - throw new Error('A run cannot record into both an encoder and an existing compute pass.'); - } +const noop = () => {}; - const externalPass = options?.pass; - if (externalPass) { - return { pass: externalPass, finish() {} }; +export function beginRunPass(device: GPUDevice, options?: RunOptions): RunRecording { + if (options?.pass) { + return { pass: options.pass, finish: noop }; } - const encoder = options?.encoder ?? device.createCommandEncoder(); + const externalEncoder = options?.encoder; + const encoder = externalEncoder ?? device.createCommandEncoder(); const pass = encoder.beginComputePass(); return { pass, finish() { pass.end(); - if (!options?.encoder) { + if (!externalEncoder) { device.queue.submit([encoder.finish()]); } }, diff --git a/packages/typegpu-sort/src/scan/compute/applySums.ts b/packages/typegpu-sort/src/scan/compute/applySums.ts deleted file mode 100644 index 1d46d54629..0000000000 --- a/packages/typegpu-sort/src/scan/compute/applySums.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { tgpu, d, std } from 'typegpu'; -import { flatWorkgroupIndex } from '../../wgslUtils.ts'; -import { ELEMENTS_PER_THREAD, type ScanSchemas, WORKGROUP_SIZE } from '../schemas.ts'; - -export function makeUniformOp(schemas: ScanSchemas) { - const { operatorSlot, uniformOpLayout } = schemas; - - return tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const workgroupId = flatWorkgroupIndex(wid, numWorkgroups); - const baseIdx = (workgroupId * WORKGROUP_SIZE + lid.x) * ELEMENTS_PER_THREAD; - const opValue = uniformOpLayout.$.sums[workgroupId]; - - for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { - if (baseIdx + i < uniformOpLayout.$.input.length) { - (uniformOpLayout.$.input[baseIdx + i] as number) = operatorSlot.$( - opValue as number, - uniformOpLayout.$.input[baseIdx + i] as number, - ); - } - } - }); -} diff --git a/packages/typegpu-sort/src/scan/compute/scan.ts b/packages/typegpu-sort/src/scan/compute/scan.ts deleted file mode 100644 index 5a69ec32bc..0000000000 --- a/packages/typegpu-sort/src/scan/compute/scan.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { tgpu, d, std } from 'typegpu'; -import { flatWorkgroupIndex } from '../../wgslUtils.ts'; -import { ELEMENTS_PER_THREAD, type ScanSchemas, WORKGROUP_SIZE } from '../schemas.ts'; -import { makeShared } from './shared.ts'; - -export function makeComputeBlock(schemas: ScanSchemas) { - const { - elementType, - scanLayout, - identitySlot, - onlyGreatestElementSlot, - operatorSlot, - workgroupMemory, - } = schemas; - const { upsweep, downsweep } = makeShared(schemas); - - const fillIdentityArray = tgpu.comptime(() => - Array.from({ length: ELEMENTS_PER_THREAD }, () => identitySlot.$), - ); - - return tgpu.computeFn({ - workgroupSize: [WORKGROUP_SIZE], - in: { - lid: d.builtin.localInvocationId, - wid: d.builtin.workgroupId, - numWorkgroups: d.builtin.numWorkgroups, - }, - })(({ lid, wid, numWorkgroups }) => { - const workgroupId = flatWorkgroupIndex(wid, numWorkgroups); - const localIdx = lid.x; - - const baseIdx = (workgroupId * WORKGROUP_SIZE + localIdx) * ELEMENTS_PER_THREAD; - - const partialSums = d.arrayOf(elementType, ELEMENTS_PER_THREAD)(fillIdentityArray()); - - let prev = identitySlot.$; - let lastIdx = d.u32(0); - - for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { - if (baseIdx + i < scanLayout.$.input.length) { - partialSums[i] = operatorSlot.$(prev, scanLayout.$.input[baseIdx + i] as number); - prev = partialSums[i]; - lastIdx = i; - } - } - workgroupMemory.$[localIdx] = partialSums[lastIdx] as number; - - upsweep(localIdx); - - if (localIdx === 0 && workgroupId < scanLayout.$.sums.length) { - scanLayout.$.sums[workgroupId] = workgroupMemory.$[WORKGROUP_SIZE - 1] as number; - if (!onlyGreatestElementSlot.$) { - workgroupMemory.$[WORKGROUP_SIZE - 1] = identitySlot.$; - } - } - - if (!onlyGreatestElementSlot.$) { - downsweep(localIdx); - - std.workgroupBarrier(); - - const scannedSum = workgroupMemory.$[localIdx]; - - for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { - if (baseIdx + i < scanLayout.$.input.length) { - if (i === 0) { - scanLayout.$.input[baseIdx + i] = scannedSum; - } else { - scanLayout.$.input[baseIdx + i] = operatorSlot.$( - scannedSum, - partialSums[i - 1] as number, - ); - } - } - } - } - }); -} diff --git a/packages/typegpu-sort/src/scan/compute/shared.ts b/packages/typegpu-sort/src/scan/compute/shared.ts deleted file mode 100644 index acbe128e10..0000000000 --- a/packages/typegpu-sort/src/scan/compute/shared.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { tgpu, d, std } from 'typegpu'; -import { type ScanSchemas, WORKGROUP_SIZE } from '../schemas.ts'; - -export function makeShared(schemas: ScanSchemas) { - const { operatorSlot, workgroupMemory } = schemas; - - const upsweep = tgpu.fn([d.u32])((localIdx) => { - let offset = d.u32(1); - for (let d_val = d.u32(WORKGROUP_SIZE / 2); d_val > 0; d_val >>= 1) { - std.workgroupBarrier(); - if (localIdx < d_val) { - const ai = offset * (2 * localIdx + 1) - 1; - const bi = offset * (2 * localIdx + 2) - 1; - workgroupMemory.$[bi] = operatorSlot.$( - workgroupMemory.$[ai] as number, - workgroupMemory.$[bi] as number, - ); - } - offset <<= 1; - } - }); - - const downsweep = tgpu.fn([d.u32])((localIdx) => { - let offset = d.u32(WORKGROUP_SIZE); - for (let d_val = d.u32(1); d_val < WORKGROUP_SIZE; d_val <<= 1) { - offset >>= 1; - std.workgroupBarrier(); - if (localIdx < d_val) { - const ai = offset * (2 * localIdx + 1) - 1; - const bi = offset * (2 * localIdx + 2) - 1; - const t = workgroupMemory.$[ai] as number; - workgroupMemory.$[ai] = workgroupMemory.$[bi] as number; - workgroupMemory.$[bi] = operatorSlot.$(workgroupMemory.$[bi] as number, t); - } - } - }); - - return { upsweep, downsweep }; -} diff --git a/packages/typegpu-sort/src/scan/index.ts b/packages/typegpu-sort/src/scan/index.ts index bb569d4863..259afd4e9e 100644 --- a/packages/typegpu-sort/src/scan/index.ts +++ b/packages/typegpu-sort/src/scan/index.ts @@ -1,11 +1,10 @@ export { createPrefixScanComputer, - prefixScan, - PrefixScanComputer, + type PrefixScanComputer, type PrefixScanPlan, - scan, + prefixScan, + reduce, type ScanBuffer, - type ScanRunOptions, } from './prefixScan.ts'; export type { ScanElementType } from './schemas.ts'; export type { BinaryOp } from './types.ts'; diff --git a/packages/typegpu-sort/src/scan/kernels.ts b/packages/typegpu-sort/src/scan/kernels.ts new file mode 100644 index 0000000000..6707fdcdf9 --- /dev/null +++ b/packages/typegpu-sort/src/scan/kernels.ts @@ -0,0 +1,118 @@ +import { tgpu, d, std } from 'typegpu'; +import { dispatchIn, flatWorkgroupIndex } from '../dispatch.ts'; +import { ELEMENTS_PER_THREAD, type ScanSchemas, WORKGROUP_SIZE } from './schemas.ts'; + +export function makeScanKernel(schemas: ScanSchemas) { + const { elementType, scanLayout, identitySlot, reduceOnlySlot, operatorSlot, workgroupMemory } = + schemas; + + function upsweep(localIdx: number) { + 'use gpu'; + let offset = d.u32(1); + for (let span = d.u32(WORKGROUP_SIZE / 2); span > 0; span >>= 1) { + std.workgroupBarrier(); + if (localIdx < span) { + const ai = offset * (2 * localIdx + 1) - 1; + const bi = offset * (2 * localIdx + 2) - 1; + workgroupMemory.$[bi] = operatorSlot.$( + workgroupMemory.$[ai] as number, + workgroupMemory.$[bi] as number, + ); + } + offset <<= 1; + } + } + + function downsweep(localIdx: number) { + 'use gpu'; + let offset = d.u32(WORKGROUP_SIZE); + for (let span = d.u32(1); span < WORKGROUP_SIZE; span <<= 1) { + offset >>= 1; + std.workgroupBarrier(); + if (localIdx < span) { + const ai = offset * (2 * localIdx + 1) - 1; + const bi = offset * (2 * localIdx + 2) - 1; + const t = workgroupMemory.$[ai] as number; + workgroupMemory.$[ai] = workgroupMemory.$[bi] as number; + workgroupMemory.$[bi] = operatorSlot.$(workgroupMemory.$[bi] as number, t); + } + } + } + + const fillIdentityArray = tgpu.comptime(() => + Array.from({ length: ELEMENTS_PER_THREAD }, () => identitySlot.$), + ); + + return tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], in: dispatchIn })( + ({ lid, wid, numWorkgroups }) => { + const workgroupId = flatWorkgroupIndex(wid, numWorkgroups); + const localIdx = lid.x; + const baseIdx = (workgroupId * WORKGROUP_SIZE + localIdx) * ELEMENTS_PER_THREAD; + + const partialSums = d.arrayOf(elementType, ELEMENTS_PER_THREAD)(fillIdentityArray()); + + let prev = identitySlot.$; + let lastIdx = d.u32(0); + + for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { + if (baseIdx + i < scanLayout.$.input.length) { + partialSums[i] = operatorSlot.$(prev, scanLayout.$.input[baseIdx + i] as number); + prev = partialSums[i]; + lastIdx = i; + } + } + workgroupMemory.$[localIdx] = partialSums[lastIdx] as number; + + upsweep(localIdx); + + if (localIdx === 0 && workgroupId < scanLayout.$.sums.length) { + scanLayout.$.sums[workgroupId] = workgroupMemory.$[WORKGROUP_SIZE - 1] as number; + if (!reduceOnlySlot.$) { + workgroupMemory.$[WORKGROUP_SIZE - 1] = identitySlot.$; + } + } + + if (!reduceOnlySlot.$) { + downsweep(localIdx); + + std.workgroupBarrier(); + + const scannedSum = workgroupMemory.$[localIdx]; + + for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { + if (baseIdx + i < scanLayout.$.input.length) { + if (i === 0) { + scanLayout.$.input[baseIdx + i] = scannedSum; + } else { + scanLayout.$.input[baseIdx + i] = operatorSlot.$( + scannedSum, + partialSums[i - 1] as number, + ); + } + } + } + } + }, + ); +} + +export function makeApplySumsKernel(schemas: ScanSchemas) { + const { operatorSlot, applySumsLayout } = schemas; + + return tgpu.computeFn({ workgroupSize: [WORKGROUP_SIZE], in: dispatchIn })( + ({ lid, wid, numWorkgroups }) => { + const workgroupId = flatWorkgroupIndex(wid, numWorkgroups); + const baseIdx = (workgroupId * WORKGROUP_SIZE + lid.x) * ELEMENTS_PER_THREAD; + const blockSum = applySumsLayout.$.sums[workgroupId]; + + for (const i of tgpu.unroll(std.range(ELEMENTS_PER_THREAD))) { + if (baseIdx + i < applySumsLayout.$.input.length) { + (applySumsLayout.$.input[baseIdx + i] as number) = operatorSlot.$( + blockSum as number, + applySumsLayout.$.input[baseIdx + i] as number, + ); + } + } + }, + ); +} diff --git a/packages/typegpu-sort/src/scan/prefixScan.ts b/packages/typegpu-sort/src/scan/prefixScan.ts index 745ff180ad..bf46b3e415 100644 --- a/packages/typegpu-sort/src/scan/prefixScan.ts +++ b/packages/typegpu-sort/src/scan/prefixScan.ts @@ -5,17 +5,11 @@ import { type TgpuComputePipeline, type TgpuRoot, } from 'typegpu'; -import { decomposeWorkgroups } from '../bitonic/utils.ts'; -import { beginRunPass, type RunPassOptions } from '../runPass.ts'; -import { makeUniformOp } from './compute/applySums.ts'; -import { makeComputeBlock } from './compute/scan.ts'; -import { - ELEMENTS_PER_THREAD, - makeScanSchemas, - type ScanElementType, - type ScanSchemas, - WORKGROUP_SIZE, -} from './schemas.ts'; +import { decomposeWorkgroups } from '../dispatch.ts'; +import { beginRunPass } from '../runPass.ts'; +import type { RunOptions } from '../types.ts'; +import { makeApplySumsKernel, makeScanKernel } from './kernels.ts'; +import { BLOCK_SIZE, makeScanSchemas, type ScanElementType } from './schemas.ts'; import type { BinaryOp } from './types.ts'; export type ScanBuffer = TgpuBuffer< @@ -25,20 +19,11 @@ export type ScanBuffer = TgpuBuffer< type AnyScanBuffer = TgpuBuffer> & StorageFlag; -const BLOCK_SIZE = WORKGROUP_SIZE * ELEMENTS_PER_THREAD; - -const cache = new WeakMap< - TgpuRoot, - WeakMap>> ->(); - interface PlanStep { pipeline: TgpuComputePipeline; workgroups: [number, number, number]; } -export type ScanRunOptions = RunPassOptions; - /** * A reusable execution plan for scanning a specific buffer. All scratch buffers and * bind groups are created once at `prepare` time, so `run` only records dispatches. @@ -51,84 +36,77 @@ export interface PrefixScanPlan { */ readonly resultBuffer: ScanBuffer; /** Dispatches the scan. Can be called repeatedly */ - run(options?: ScanRunOptions): void; + run(options?: RunOptions): void; /** Destroys the scratch buffers owned by this plan */ destroy(): void; } -export class PrefixScanComputer { - readonly #root: TgpuRoot; - readonly #operation: BinaryOp['operation']; - readonly #identityElement: BinaryOp['identityElement']; - readonly #schemas: ScanSchemas; - readonly #computeBlock: ReturnType; - readonly #uniformOp: ReturnType; - readonly #plans = new WeakMap< +export interface PrefixScanComputer { + /** Creates a reusable execution plan for scanning `buffer` */ + prepare( + buffer: ScanBuffer, + options?: { reduceOnly?: boolean }, + ): PrefixScanPlan; + /** Scans `buffer` in place through a plan cached per buffer */ + scan(buffer: ScanBuffer, options?: RunOptions): ScanBuffer; + /** + * Reduces `buffer` through a plan cached per buffer. The returned single-element buffer + * belongs to that plan, so it is shared between calls on the same input buffer. + */ + reduce(buffer: ScanBuffer, options?: RunOptions): ScanBuffer; +} + +function makeComputer( + root: TgpuRoot, + operation: BinaryOp['operation'], + identityElement: BinaryOp['identityElement'], + elementType: TElement, +): PrefixScanComputer { + const schemas = makeScanSchemas(elementType); + const scanKernel = makeScanKernel(schemas); + const applySumsKernel = makeApplySumsKernel(schemas); + const withOperation = root.with(schemas.operatorSlot, operation); + + const plans = new WeakMap< ScanBuffer, { scan?: PrefixScanPlan; reduce?: PrefixScanPlan } >(); - #scanPipeline?: TgpuComputePipeline; - #reducePipeline?: TgpuComputePipeline; - #opPipeline?: TgpuComputePipeline; + let scanPipeline: TgpuComputePipeline | undefined; + let reducePipeline: TgpuComputePipeline | undefined; + let applySumsPipeline: TgpuComputePipeline | undefined; - constructor( - root: TgpuRoot, - operation: BinaryOp['operation'], - identityElement: BinaryOp['identityElement'], - elementType: TElement, - ) { - this.#root = root; - this.#operation = operation; - this.#identityElement = identityElement; - this.#schemas = makeScanSchemas(elementType); - this.#computeBlock = makeComputeBlock(this.#schemas); - this.#uniformOp = makeUniformOp(this.#schemas); + function createScanPipeline(reduceOnly: boolean): TgpuComputePipeline { + return withOperation + .with(schemas.identitySlot, identityElement) + .with(schemas.reduceOnlySlot, reduceOnly) + .createComputePipeline({ compute: scanKernel }); } - private getScanPipeline(onlyGreatestElement: boolean): TgpuComputePipeline { - const cached = onlyGreatestElement ? this.#reducePipeline : this.#scanPipeline; - if (cached) { - return cached; + function scanPipelineFor(reduceOnly: boolean): TgpuComputePipeline { + if (reduceOnly) { + reducePipeline ??= createScanPipeline(true); + return reducePipeline; } - - const pipeline = this.#root - .with(this.#schemas.operatorSlot, this.#operation) - .with(this.#schemas.identitySlot, this.#identityElement) - .with(this.#schemas.onlyGreatestElementSlot, onlyGreatestElement) - .createComputePipeline({ compute: this.#computeBlock }); - - if (onlyGreatestElement) { - this.#reducePipeline = pipeline; - } else { - this.#scanPipeline = pipeline; - } - return pipeline; - } - - private get opPipeline(): TgpuComputePipeline { - this.#opPipeline ??= this.#root - .with(this.#schemas.operatorSlot, this.#operation) - .createComputePipeline({ compute: this.#uniformOp }); - return this.#opPipeline; + scanPipeline ??= createScanPipeline(false); + return scanPipeline; } - private createScratchBuffer(size: number): ScanBuffer { - return this.#root - .createBuffer(d.arrayOf(this.#schemas.elementType, size)) - .$usage('storage') as ScanBuffer; + function applySums(): TgpuComputePipeline { + applySumsPipeline ??= withOperation.createComputePipeline({ compute: applySumsKernel }); + return applySumsPipeline; } - /** - * Creates a reusable execution plan for scanning `buffer`. All scratch buffers, bind - * groups and pipelines are allocated up front, so `plan.run()` only records dispatches. - */ - prepare( + function prepare( buffer: ScanBuffer, - options?: { onlyGreatestElement?: boolean }, + options?: { reduceOnly?: boolean }, ): PrefixScanPlan { - const onlyGreatestElement = options?.onlyGreatestElement ?? false; - const scanPipeline = this.getScanPipeline(onlyGreatestElement); + if (buffer.dataType.elementCount === 0) { + throw new Error('Cannot scan an empty buffer.'); + } + + const reduceOnly = options?.reduceOnly ?? false; + const pipeline = scanPipelineFor(reduceOnly); const steps: PlanStep[] = []; const scratchBuffers: ScanBuffer[] = []; @@ -142,62 +120,62 @@ export class PrefixScanComputer { let currentLength = buffer.dataType.elementCount; let resultBuffer = buffer; - if (currentLength === 0) { - throw new Error('Cannot scan an empty buffer.'); - } - for (;;) { const numWorkgroups = Math.ceil(currentLength / BLOCK_SIZE); - const sumsBuffer = this.createScratchBuffer(numWorkgroups === 1 ? 1 : numWorkgroups); - scratchBuffers.push(sumsBuffer); + const sums = root + .createBuffer(d.arrayOf(schemas.elementType, numWorkgroups)) + .$usage('storage') as ScanBuffer; + scratchBuffers.push(sums); - const bindGroup = this.#root.createBindGroup(this.#schemas.scanLayout, { - input: currentBuffer as AnyScanBuffer, - sums: sumsBuffer as AnyScanBuffer, - }); steps.push({ - pipeline: scanPipeline.with(bindGroup), + pipeline: pipeline.with( + root.createBindGroup(schemas.scanLayout, { + input: currentBuffer as AnyScanBuffer, + sums: sums as AnyScanBuffer, + }), + ), workgroups: decomposeWorkgroups(numWorkgroups), }); if (numWorkgroups === 1) { - if (onlyGreatestElement) { - resultBuffer = sumsBuffer; + if (reduceOnly) { + resultBuffer = sums; } break; } - applyLevels.push({ target: currentBuffer, sums: sumsBuffer, numWorkgroups }); - currentBuffer = sumsBuffer; + applyLevels.push({ target: currentBuffer, sums, numWorkgroups }); + currentBuffer = sums; currentLength = numWorkgroups; } - if (!onlyGreatestElement) { - for (let i = applyLevels.length - 1; i >= 0; i--) { - const level = applyLevels[i] as (typeof applyLevels)[number]; - const bindGroup = this.#root.createBindGroup(this.#schemas.uniformOpLayout, { - input: level.target as AnyScanBuffer, - sums: level.sums as AnyScanBuffer, - }); + if (!reduceOnly) { + applyLevels.reverse(); + for (const level of applyLevels) { steps.push({ - pipeline: this.opPipeline.with(bindGroup), + pipeline: applySums().with( + root.createBindGroup(schemas.applySumsLayout, { + input: level.target as AnyScanBuffer, + sums: level.sums as AnyScanBuffer, + }), + ), workgroups: decomposeWorkgroups(level.numWorkgroups), }); } } - const device = this.#root.device; - return { resultBuffer, - run(options?: ScanRunOptions) { - const recording = beginRunPass(device, options); + + run(options?: RunOptions): void { + const recording = beginRunPass(root.device, options); for (const step of steps) { step.pipeline.with(recording.pass).dispatchWorkgroups(...step.workgroups); } recording.finish(); }, - destroy() { + + destroy(): void { for (const scratch of scratchBuffers) { scratch.destroy(); } @@ -205,220 +183,122 @@ export class PrefixScanComputer { }; } - /** - * Scans `buffer` in place, or reduces it when `onlyGreatestElement` is true. Plans are - * cached per buffer, so repeated calls on the same buffer reuse all scratch buffers and - * bind groups. For reductions this means the returned single-element buffer is shared - * between calls on the same input buffer. - */ - compute( - buffer: ScanBuffer, - onlyGreatestElement: boolean, - options?: ScanRunOptions, - ): ScanBuffer { - let plans = this.#plans.get(buffer); - if (!plans) { - plans = {}; - this.#plans.set(buffer, plans); + function cachedPlan(buffer: ScanBuffer, reduceOnly: boolean): PrefixScanPlan { + let forBuffer = plans.get(buffer); + if (!forBuffer) { + forBuffer = {}; + plans.set(buffer, forBuffer); } - const key = onlyGreatestElement ? 'reduce' : 'scan'; - let plan = plans[key]; + const key = reduceOnly ? 'reduce' : 'scan'; + let plan = forBuffer[key]; if (!plan) { - plan = this.prepare(buffer, { onlyGreatestElement }); - plans[key] = plan; + plan = prepare(buffer, { reduceOnly }); + forBuffer[key] = plan; } + return plan; + } + + return { + prepare, + + scan(buffer, options) { + const plan = cachedPlan(buffer, false); + plan.run(options); + return plan.resultBuffer; + }, + + reduce(buffer, options) { + const plan = cachedPlan(buffer, true); + plan.run(options); + return plan.resultBuffer; + }, + }; +} + +interface CacheLike { + get(key: K): V | undefined; + set(key: K, value: V): unknown; +} - plan.run(options); - return plan.resultBuffer; +function getOrCreate(cache: CacheLike, key: K, create: () => V): V { + const cached = cache.get(key); + if (cached !== undefined) { + return cached; } + + const created = create(); + cache.set(key, created); + return created; } +const computerCache = new WeakMap>>(); + /** - * Perform a GPU prefix-scan (parallel prefix scan depending on the - * provided operation) over the values in `inputBuffer`. For instance, this can be used to - * compute a prefix sum over an array of numbers. - * - * @param root - The TypeGPU root/context used to create pipelines, bind groups and buffers. - * @param options - Configuration object containing: - * - inputBuffer: A storage buffer with the input values to scan - * - outputBuffer: (optional) A storage buffer where the scanned values will be written. - * Defaults to in-place (overwrites `inputBuffer`). - * - operation: The binary operation to use for the scan (e.g., std.add) - * - identityElement: The identity element for the operation (e.g., 0 for addition) - * @returns The output buffer instance which contains the scanned values. - * - * @example - * ```typescript - * const root = await tgpu.init(); - * const inputBuffer = root - * .createBuffer(d.arrayOf(d.f32, 4), [1, 2, 3, 4]) - * .$usage('storage'); - * - * // in-place (inputBuffer is modified) - * const result = prefixScan( - * root, - * { - * inputBuffer, - * operation: std.add, - * identityElement: 0, - * }, - * ); - * - * // with separate output buffer - * const outputBuffer = root - * .createBuffer(d.arrayOf(d.f32, 4)) - * .$usage('storage'); - * - * const result = prefixScan( - * root, - * { - * inputBuffer, - * outputBuffer, - * operation: std.add, - * identityElement: 0, - * }, - * ); - * ``` + * Creates a computer for the given operation, reusing the one cached for the same `root` and + * `binaryOp` so that repeated calls share pipelines. Set `dataType` to `d.u32` or `d.i32` to + * scan integer buffers (defaults to `d.f32`). */ -export function prefixScan( +export function createPrefixScanComputer( root: TgpuRoot, - options: { - inputBuffer: ScanBuffer; - outputBuffer?: ScanBuffer; - operation: BinaryOp['operation']; - identityElement: BinaryOp['identityElement']; - }, -): ScanBuffer { - return runScan(root, options, false); + binaryOp: BinaryOp, +): PrefixScanComputer { + const elementType = (binaryOp.dataType ?? d.f32) as TElement; + const byOperation = getOrCreate(computerCache, root, () => new WeakMap()); + const byElement = getOrCreate(byOperation, binaryOp.operation, () => new Map()); + + return getOrCreate(byElement, `${binaryOp.identityElement}_${elementType.type}`, () => + makeComputer(root, binaryOp.operation, binaryOp.identityElement, elementType), + ) as PrefixScanComputer; } -/** - * Compute only the aggregated reduction result for `inputBuffer` using the provided operation. - * Returns only the top-level sums/reductions instead of the full scan. This is useful when - * you only need the final reduction - for instance, the sum of the whole array. - * - * @param root - The TypeGPU root/context used to create pipelines, bind groups and buffers. - * @param options - Configuration object containing: - * - inputBuffer: A storage buffer with the input values to reduce - * - operation: The binary operation to use for the reduction (e.g., std.add) - * - identityElement: The identity element for the operation (e.g., 0 for addition) - * @returns A buffer containing the aggregated reduction result (single-element buffer). - * It is owned by the internally cached scan plan and reused by subsequent - * `scan` calls on the same input buffer. - * - * @example - * ```typescript - * const root = await tgpu.init(); - * const inputBuffer = root - * .createBuffer(d.arrayOf(d.f32, 4), [1, 2, 3, 4]) - * .$usage('storage'); - * - * // using an std function - * const result = scan( - * root, - * { - * inputBuffer, - * operation: std.add, - * identityElement: 0, - * }, - * ); - * - * // using a custom tgpu.fn - * const multiply = tgpu.fn([d.f32, d.f32], d.f32)((a, b) => a * b); - * - * const result = scan( - * root, - * { - * inputBuffer, - * operation: multiply, - * identityElement: 1, - * }, - * ); - * ``` - */ -export function scan( - root: TgpuRoot, - options: { - inputBuffer: ScanBuffer; - operation: BinaryOp['operation']; - identityElement: BinaryOp['identityElement']; - }, -): ScanBuffer { - return runScan(root, options, true); +interface ScanOptions { + inputBuffer: ScanBuffer; + operation: BinaryOp['operation']; + identityElement: BinaryOp['identityElement']; } -function runScan( +/** + * Performs an exclusive prefix scan over `inputBuffer` with the given associative operation, + * writing to `outputBuffer` if provided and in place otherwise. + */ +export function prefixScan( root: TgpuRoot, - options: { - inputBuffer: ScanBuffer; - outputBuffer?: ScanBuffer; - operation: BinaryOp['operation']; - identityElement: BinaryOp['identityElement']; - }, - onlyGreatestElement: boolean, + options: ScanOptions & { outputBuffer?: ScanBuffer }, ): ScanBuffer { - const elementType = options.inputBuffer.dataType.elementType; + const { inputBuffer, outputBuffer } = options; const computer = createPrefixScanComputer(root, { operation: options.operation, identityElement: options.identityElement, - dataType: elementType, + dataType: inputBuffer.dataType.elementType, }); - if (onlyGreatestElement) { - return computer.compute(options.inputBuffer, true); + if (!outputBuffer || outputBuffer === inputBuffer) { + return computer.scan(inputBuffer); } - const outputBuffer = options.outputBuffer ?? options.inputBuffer; - if (options.inputBuffer !== outputBuffer) { - if ( - outputBuffer.dataType.elementType.type !== elementType.type || - outputBuffer.dataType.elementCount !== options.inputBuffer.dataType.elementCount - ) { - throw new Error('The input and output scan buffers must have the same type and length.'); - } - (outputBuffer as ScanBuffer).copyFrom(options.inputBuffer as ScanBuffer); + if ( + outputBuffer.dataType.elementType.type !== inputBuffer.dataType.elementType.type || + outputBuffer.dataType.elementCount !== inputBuffer.dataType.elementCount + ) { + throw new Error('The input and output scan buffers must have the same type and length.'); } - return computer.compute(outputBuffer, false); + (outputBuffer as ScanBuffer).copyFrom(inputBuffer as ScanBuffer); + return computer.scan(outputBuffer); } /** - * Create or retrieve a cached `PrefixScanComputer` for the given `root` and `binaryOp`. - * - * @param root - The TypeGPU root/context to associate with the cached computer. - * @param binaryOp - The binary operation used by the computer. Set `dataType` to `d.u32` - * or `d.i32` to scan integer buffers (defaults to `d.f32`). - * @returns A `PrefixScanComputer` instance associated with the provided `root` and `binaryOp`. + * Reduces `inputBuffer` with the given associative operation, returning a single-element + * buffer with the aggregate. The input is left untouched. */ -export function createPrefixScanComputer( +export function reduce( root: TgpuRoot, - binaryOp: BinaryOp, -): PrefixScanComputer { - const elementType = (binaryOp.dataType ?? d.f32) as TElement; - const cacheKey = `${binaryOp.identityElement}_${elementType.type}`; - - let rootCache = cache.get(root); - if (!rootCache) { - rootCache = new WeakMap(); - cache.set(root, rootCache); - } - - let opCache = rootCache.get(binaryOp.operation); - if (!opCache) { - opCache = new Map(); - rootCache.set(binaryOp.operation, opCache); - } - - let computer = opCache.get(cacheKey); - if (!computer) { - computer = new PrefixScanComputer( - root, - binaryOp.operation, - binaryOp.identityElement, - elementType, - ); - opCache.set(cacheKey, computer); - } - return computer as PrefixScanComputer; + options: ScanOptions, +): ScanBuffer { + return createPrefixScanComputer(root, { + operation: options.operation, + identityElement: options.identityElement, + dataType: options.inputBuffer.dataType.elementType, + }).reduce(options.inputBuffer); } diff --git a/packages/typegpu-sort/src/scan/schemas.ts b/packages/typegpu-sort/src/scan/schemas.ts index 6ead48818d..8507fe6405 100644 --- a/packages/typegpu-sort/src/scan/schemas.ts +++ b/packages/typegpu-sort/src/scan/schemas.ts @@ -2,6 +2,7 @@ import { tgpu, d } from 'typegpu'; export const WORKGROUP_SIZE = 256; export const ELEMENTS_PER_THREAD = 8; +export const BLOCK_SIZE = WORKGROUP_SIZE * ELEMENTS_PER_THREAD; export type ScanElementType = d.F32 | d.U32 | d.I32; @@ -12,13 +13,13 @@ export function makeScanSchemas(elementType: ScanElementType) { input: { storage: d.arrayOf(elementType), access: 'mutable' }, sums: { storage: d.arrayOf(elementType), access: 'mutable' }, }), - uniformOpLayout: tgpu.bindGroupLayout({ + applySumsLayout: tgpu.bindGroupLayout({ input: { storage: d.arrayOf(elementType), access: 'mutable' }, sums: { storage: d.arrayOf(elementType), access: 'readonly' }, }), operatorSlot: tgpu.slot<(a: number, b: number) => number>(), identitySlot: tgpu.accessor(elementType), - onlyGreatestElementSlot: tgpu.slot(), + reduceOnlySlot: tgpu.slot(), workgroupMemory: tgpu.workgroupVar(d.arrayOf(elementType, WORKGROUP_SIZE)), }; } diff --git a/packages/typegpu-sort/src/types.ts b/packages/typegpu-sort/src/types.ts new file mode 100644 index 0000000000..ed277376c3 --- /dev/null +++ b/packages/typegpu-sort/src/types.ts @@ -0,0 +1,23 @@ +interface EncoderOptions { + /** Records the dispatches as a single compute pass on this encoder. Nothing is submitted */ + encoder: GPUCommandEncoder; + pass?: never; +} + +interface PassOptions { + encoder?: never; + /** Records the dispatches into this pass. Nothing is submitted and the pass is not ended */ + pass: GPUComputePassEncoder; +} + +/** Controls where a `run` call records its dispatches. Defaults to a standalone submit */ +export type RunOptions = EncoderOptions | PassOptions; + +export interface Sorter { + /** Number of elements this sorter was created for */ + readonly size: number; + /** Sorts the buffer in place. Can be called repeatedly */ + run(options?: RunOptions): void; + /** Destroys the internal buffers owned by this sorter */ + destroy(): void; +} diff --git a/packages/typegpu-sort/src/wgslUtils.ts b/packages/typegpu-sort/src/wgslUtils.ts deleted file mode 100644 index f2a797034c..0000000000 --- a/packages/typegpu-sort/src/wgslUtils.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { tgpu, d } from 'typegpu'; - -export const flatWorkgroupIndex = tgpu.fn( - [d.vec3u, d.vec3u], - d.u32, -)( - (wid, numWorkgroups) => - wid.x + wid.y * numWorkgroups.x + wid.z * numWorkgroups.x * numWorkgroups.y, -); diff --git a/packages/typegpu-sort/tests/bitonic.test.ts b/packages/typegpu-sort/tests/bitonic.test.ts index c8f5c938ec..bb4865330b 100644 --- a/packages/typegpu-sort/tests/bitonic.test.ts +++ b/packages/typegpu-sort/tests/bitonic.test.ts @@ -1,11 +1,12 @@ -import { d } from 'typegpu'; +import { tgpu, d } from 'typegpu'; import { it } from 'typegpu-testing-utility'; import { describe, expect, vi } from 'vitest'; import { createBitonicSorter } from '../src/index.ts'; +import { defaultCompare } from '../src/bitonic/slots.ts'; import { getConversionWarnings, getResolvedWgsl } from './utils.ts'; describe('bitonic sort', () => { - it('creates type-matched kernels without conversion warnings', ({ root, device }) => { + it('emits no implicit conversion warnings for any key type', ({ root }) => { const warnSpy = vi.spyOn(console, 'warn'); for (const keyType of [d.u32, d.i32, d.f32] as const) { @@ -13,15 +14,248 @@ describe('bitonic sort', () => { createBitonicSorter(root, data).run(); } - const wgsl = getResolvedWgsl(device); - expect(wgsl).toContain('array'); - expect(wgsl).toContain('array'); - expect(wgsl).toContain('array'); - - expect(getConversionWarnings(warnSpy)).toEqual([]); + expect(getConversionWarnings(warnSpy)).toMatchInlineSnapshot(`[]`); warnSpy.mockRestore(); }); + it('specializes the comparator per key type', () => { + expect( + tgpu.resolve([ + tgpu.fn([d.u32, d.u32], d.bool)(defaultCompare), + tgpu.fn([d.i32, d.i32], d.bool)(defaultCompare), + tgpu.fn([d.f32, d.f32], d.bool)(defaultCompare), + ]), + ).toMatchInlineSnapshot(` + "fn defaultCompare(a: u32, b: u32) -> bool { + return (a < b); + } + + fn defaultCompare_1(a: i32, b: i32) -> bool { + return (a < b); + } + + fn defaultCompare_2(a: f32, b: f32) -> bool { + return (a < b); + }" + `); + }); + + it('should produce valid code for a composite payload', ({ root, device }) => { + const keys = root.createBuffer(d.arrayOf(d.u32, 1024)).$usage('storage'); + const values = root.createBuffer(d.arrayOf(d.vec2f, 1024)).$usage('storage'); + createBitonicSorter(root, keys, { values }).run(); + + expect(getResolvedWgsl(device)).toMatchInlineSnapshot(` + "fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + @group(0) @binding(0) var data: array; + + var localKeys: array; + + var localVals: array; + + @group(1) @binding(0) var vals: array; + + fn loadShared(base: u32, tid: u32) { + localKeys[tid] = data[(base + tid)]; + localKeys[(tid + 256u)] = data[((base + tid) + 256u)]; + { + localVals[tid] = vals[(base + tid)]; + localVals[(tid + 256u)] = vals[((base + tid) + 256u)]; + } + } + + fn defaultCompare(a: u32, b: u32) -> bool { + return (a < b); + } + + fn swapLocalAt(a: u32, b: u32, left: u32, right: u32) { + localKeys[a] = right; + localKeys[b] = left; + { + let tmp = localVals[a]; + localVals[a] = localVals[b]; + localVals[b] = tmp; + } + } + + fn exchangeLocal(base: u32, iLocal: u32, stride: u32, k: u32) { + let jLocal = (iLocal + stride); + let left = localKeys[iLocal]; + let right = localKeys[jLocal]; + let ascending = (((base + iLocal) & k) == 0u); + if (select(defaultCompare(left, right), defaultCompare(right, left), ascending)) { + swapLocalAt(iLocal, jLocal, left, right); + } + } + + fn mergeDown(base: u32, tid: u32, startShift: u32, k: u32) { + for (var jShift = startShift; (jShift > 0u); jShift--) { + workgroupBarrier(); + let stride = (1u << (jShift - 1u)); + let below = (tid & (stride - 1u)); + let above = (tid >> (jShift - 1u)); + exchangeLocal(base, (below + (above * (stride << 1u))), stride, k); + } + } + + fn storeShared(base: u32, tid: u32) { + data[(base + tid)] = localKeys[tid]; + data[((base + tid) + 256u)] = localKeys[(tid + 256u)]; + { + vals[(base + tid)] = localVals[tid]; + vals[((base + tid) + 256u)] = localVals[(tid + 256u)]; + } + } + + @compute @workgroup_size(256) fn localSort(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let base = (flatWorkgroupIndex(wid, numWorkgroups) * 512u); + if ((base >= arrayLength(&data))) { + return; + } + loadShared(base, lid.x); + for (var kShift = 1u; (kShift <= 9u); kShift++) { + mergeDown(base, lid.x, kShift, (1u << kShift)); + } + workgroupBarrier(); + storeShared(base, lid.x); + } + + fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + struct sortUniformsType { + k: u32, + jShift: u32, + } + + @group(0) @binding(1) var uniforms: sortUniformsType; + + @group(0) @binding(0) var data: array; + + fn defaultCompare(a: u32, b: u32) -> bool { + return (a < b); + } + + @group(1) @binding(0) var vals: array; + + fn swapAt(i: u32, j: u32, left: u32, right: u32) { + data[i] = right; + data[j] = left; + { + let tmp = vals[i]; + vals[i] = vals[j]; + vals[j] = tmp; + } + } + + @compute @workgroup_size(256) fn item(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let tid = ((flatWorkgroupIndex(wid, numWorkgroups) * 256u) + lid.x); + let k = uniforms.k; + let shift = uniforms.jShift; + let stride = (1u << shift); + let below = (tid & (stride - 1u)); + let above = (tid >> shift); + let i = (below + (above * (stride << 1u))); + let ixj = (i + stride); + if ((ixj >= arrayLength(&data))) { + return; + } + let left = data[i]; + let right = data[ixj]; + let ascending = ((i & k) == 0u); + if (select(defaultCompare(left, right), defaultCompare(right, left), ascending)) { + swapAt(i, ixj, left, right); + } + } + + fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + @group(0) @binding(0) var data: array; + + var localKeys: array; + + var localVals: array; + + @group(1) @binding(0) var vals: array; + + fn loadShared(base: u32, tid: u32) { + localKeys[tid] = data[(base + tid)]; + localKeys[(tid + 256u)] = data[((base + tid) + 256u)]; + { + localVals[tid] = vals[(base + tid)]; + localVals[(tid + 256u)] = vals[((base + tid) + 256u)]; + } + } + + struct sortUniformsType { + k: u32, + jShift: u32, + } + + @group(0) @binding(1) var uniforms: sortUniformsType; + + fn defaultCompare(a: u32, b: u32) -> bool { + return (a < b); + } + + fn swapLocalAt(a: u32, b: u32, left: u32, right: u32) { + localKeys[a] = right; + localKeys[b] = left; + { + let tmp = localVals[a]; + localVals[a] = localVals[b]; + localVals[b] = tmp; + } + } + + fn exchangeLocal(base: u32, iLocal: u32, stride: u32, k: u32) { + let jLocal = (iLocal + stride); + let left = localKeys[iLocal]; + let right = localKeys[jLocal]; + let ascending = (((base + iLocal) & k) == 0u); + if (select(defaultCompare(left, right), defaultCompare(right, left), ascending)) { + swapLocalAt(iLocal, jLocal, left, right); + } + } + + fn mergeDown(base: u32, tid: u32, startShift: u32, k: u32) { + for (var jShift = startShift; (jShift > 0u); jShift--) { + workgroupBarrier(); + let stride = (1u << (jShift - 1u)); + let below = (tid & (stride - 1u)); + let above = (tid >> (jShift - 1u)); + exchangeLocal(base, (below + (above * (stride << 1u))), stride, k); + } + } + + fn storeShared(base: u32, tid: u32) { + data[(base + tid)] = localKeys[tid]; + data[((base + tid) + 256u)] = localKeys[(tid + 256u)]; + { + vals[(base + tid)] = localVals[tid]; + vals[((base + tid) + 256u)] = localVals[(tid + 256u)]; + } + } + + @compute @workgroup_size(256) fn localMerge(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let base = (flatWorkgroupIndex(wid, numWorkgroups) * 512u); + if ((base >= arrayLength(&data))) { + return; + } + loadShared(base, lid.x); + mergeDown(base, lid.x, 9u, uniforms.k); + workgroupBarrier(); + storeShared(base, lid.x); + }" + `); + }); + it('performs no buffer writes or allocations during run (uniforms precreated)', ({ root, device, @@ -42,16 +276,6 @@ describe('bitonic sort', () => { expect(device.mock.createBuffer.mock.calls.length).toBe(buffersAfterFirst); }); - it('includes payload machinery only when a values buffer is provided', ({ root, device }) => { - const keys = root.createBuffer(d.arrayOf(d.u32, 256)).$usage('storage'); - createBitonicSorter(root, keys).run(); - expect(getResolvedWgsl(device)).not.toContain('vals'); - - const values = root.createBuffer(d.arrayOf(d.u32, 256)).$usage('storage'); - createBitonicSorter(root, keys, { values }).run(); - expect(getResolvedWgsl(device)).toContain('vals'); - }); - it('rejects payload sorting when padding would be required', ({ root }) => { const keys = root.createBuffer(d.arrayOf(d.u32, 3)).$usage('storage'); const values = root.createBuffer(d.arrayOf(d.u32, 3)).$usage('storage'); diff --git a/packages/typegpu-sort/tests/radix.test.ts b/packages/typegpu-sort/tests/radix.test.ts index 4054229ce0..2d88154f14 100644 --- a/packages/typegpu-sort/tests/radix.test.ts +++ b/packages/typegpu-sort/tests/radix.test.ts @@ -1,8 +1,9 @@ -import { d } from 'typegpu'; +import { tgpu, d } from 'typegpu'; import { it } from 'typegpu-testing-utility'; import { describe, expect, vi } from 'vitest'; import { createRadixSorter } from '../src/index.ts'; -import { getConversionWarnings, getResolvedWgsl } from './utils.ts'; +import { makeDigitFn, makeRadixSchemas } from '../src/radix/schemas.ts'; +import { getConversionWarnings } from './utils.ts'; describe('radix sort', () => { it('emits no implicit conversion warnings for any key type', ({ root }) => { @@ -13,29 +14,71 @@ describe('radix sort', () => { createRadixSorter(root, data).run(); } - expect(getConversionWarnings(warnSpy)).toEqual([]); + expect(getConversionWarnings(warnSpy)).toMatchInlineSnapshot(`[]`); warnSpy.mockRestore(); }); - it('extracts i32 digits without an INT_MIN literal', ({ root, device }) => { - const data = root.createBuffer(d.arrayOf(d.i32, 512)).$usage('storage'); - createRadixSorter(root, data).run(); + it('extracts i32 digits without an INT_MIN literal', () => { + const digit = tgpu.fn([d.i32, d.u32], d.u32)(makeDigitFn(d.i32, 'ascending')); + expect(tgpu.resolve([digit])).toMatchInlineSnapshot(` + "fn digitOfI32(v: i32, shift: u32) -> u32 { + let raw = u32(((v >> shift) & 255i)); + return (raw ^ select(0u, 128u, (shift == 24u))); + }" + `); + }); - const wgsl = getResolvedWgsl(device); - expect(wgsl).toContain('array'); - expect(wgsl).not.toContain('-2147483648i'); + it('canonicalizes signed zero before extracting f32 digits', () => { + const digit = tgpu.fn([d.f32, d.u32], d.u32)(makeDigitFn(d.f32, 'ascending')); + expect(tgpu.resolve([digit])).toMatchInlineSnapshot(` + "fn digitOfF32(v: f32, shift: u32) -> u32 { + let bits = select(bitcast(v), 0u, (v == 0f)); + let mask = select(2147483648u, 4294967295u, ((bits >> 31u) == 1u)); + return (((bits ^ mask) >> shift) & 255u); + }" + `); }); - it('canonicalizes signed zero before extracting f32 digits', ({ root, device }) => { - const data = root.createBuffer(d.arrayOf(d.f32, 512)).$usage('storage'); - createRadixSorter(root, data).run(); + it('inverts digits for a descending sort', () => { + const digit = tgpu.fn([d.u32, d.u32], d.u32)(makeDigitFn(d.u32, 'descending')); + expect(tgpu.resolve([digit])).toMatchInlineSnapshot(` + "fn digitOfU32(v: u32, shift: u32) -> u32 { + return ((v >> shift) & 255u); + } - const canonicalization = getResolvedWgsl(device) - .split('\n') - .find((line) => line.includes('bitcast(v)')); - expect(canonicalization).toMatchInlineSnapshot( - `" let bits = select(bitcast(v), 0u, (v == 0f));"`, - ); + fn descendingDigit(v: u32, shift: u32) -> u32 { + return (255u - digitOfU32(v, shift)); + }" + `); + }); + + it('writes keys only when no values buffer is provided', () => { + const { writeOutput } = makeRadixSchemas(d.u32, 'ascending'); + expect(tgpu.resolve([tgpu.fn([d.u32, d.u32, d.u32])(writeOutput)])).toMatchInlineSnapshot(` + "@group(0) @binding(1) var dst: array; + + fn writeOutput(key: u32, srcIdx: u32, dstIdx: u32) { + dst[dstIdx] = key; + }" + `); + }); + + it('reorders the payload alongside the keys', () => { + const { writeOutput } = makeRadixSchemas(d.u32, 'ascending', d.vec4f); + expect(tgpu.resolve([tgpu.fn([d.u32, d.u32, d.u32])(writeOutput)])).toMatchInlineSnapshot(` + "@group(0) @binding(1) var dst: array; + + @group(1) @binding(1) var dstVals: array; + + @group(1) @binding(0) var srcVals: array; + + fn writeOutput(key: u32, srcIdx: u32, dstIdx: u32) { + dst[dstIdx] = key; + { + dstVals[dstIdx] = srcVals[srcIdx]; + } + }" + `); }); it('allocates no new GPU resources on repeated runs', ({ root, device }) => { @@ -55,13 +98,10 @@ describe('radix sort', () => { expect(device.mock.createComputePipeline.mock.calls.length).toBe(pipelinesAfterFirst); }); - it('includes payload machinery only when a values buffer is provided', ({ root, device }) => { - const keys = root.createBuffer(d.arrayOf(d.u32, 512)).$usage('storage'); - createRadixSorter(root, keys).run(); - expect(getResolvedWgsl(device)).not.toContain('dstVals'); - - const values = root.createBuffer(d.arrayOf(d.vec4f, 512)).$usage('storage'); - createRadixSorter(root, keys, { values }).run(); - expect(getResolvedWgsl(device)).toContain('dstVals'); + it('rejects empty buffers', ({ root }) => { + const keys = root.createBuffer(d.arrayOf(d.u32, 0)).$usage('storage'); + expect(() => createRadixSorter(root, keys)).toThrowErrorMatchingInlineSnapshot( + `[Error: Cannot create a radix sorter for an empty buffer.]`, + ); }); }); diff --git a/packages/typegpu-sort/tests/scan.test.ts b/packages/typegpu-sort/tests/scan.test.ts index 665999b909..0605ac437f 100644 --- a/packages/typegpu-sort/tests/scan.test.ts +++ b/packages/typegpu-sort/tests/scan.test.ts @@ -1,30 +1,39 @@ -import { d, std } from 'typegpu'; +import { tgpu, d, std } from 'typegpu'; import { it } from 'typegpu-testing-utility'; import { describe, expect, vi } from 'vitest'; import { createPrefixScanComputer, prefixScan } from '../src/index.ts'; +import { makeScanSchemas } from '../src/scan/schemas.ts'; import { getConversionWarnings, getResolvedWgsl } from './utils.ts'; describe('prefix scan', () => { - it('generates type-matched kernels without conversion warnings', ({ root, device }) => { + it('emits no implicit conversion warnings for any element type', ({ root }) => { const warnSpy = vi.spyOn(console, 'warn'); - for (const dataType of [d.u32, d.i32] as const) { + for (const dataType of [d.u32, d.i32, d.f32] as const) { const buffer = root.createBuffer(d.arrayOf(dataType, 4096)).$usage('storage'); prefixScan(root, { inputBuffer: buffer, operation: std.add, identityElement: 0 }); } - const f32Buffer = root.createBuffer(d.arrayOf(d.f32, 4096)).$usage('storage'); - prefixScan(root, { inputBuffer: f32Buffer, operation: std.add, identityElement: 0 }); - const wgsl = getResolvedWgsl(device); - expect(wgsl).toContain('array'); - expect(wgsl).toContain('array'); - expect(wgsl).toContain('array'); - - expect(getConversionWarnings(warnSpy)).toEqual([]); + expect(getConversionWarnings(warnSpy)).toMatchInlineSnapshot(`[]`); warnSpy.mockRestore(); }); - it('reuses scratch buffers and bind groups across repeated computes', ({ root, device }) => { + it('types the workgroup memory and layouts after the element type', () => { + const { workgroupMemory, scanLayout, applySumsLayout } = makeScanSchemas(d.i32); + expect(tgpu.resolve([workgroupMemory, scanLayout, applySumsLayout])).toMatchInlineSnapshot(` + "var workgroupMemory: array; + + @group(0) @binding(0) var input: array; + + @group(0) @binding(1) var sums: array; + + @group(1) @binding(0) var input_1: array; + + @group(1) @binding(1) var sums_1: array;" + `); + }); + + it('reuses scratch buffers and bind groups across repeated scans', ({ root, device }) => { const computer = createPrefixScanComputer(root, { operation: std.add, identityElement: 0, @@ -32,37 +41,132 @@ describe('prefix scan', () => { }); const buffer = root.createBuffer(d.arrayOf(d.u32, 4096)).$usage('storage'); - computer.compute(buffer, false); + computer.scan(buffer); const buffersAfterFirst = device.mock.createBuffer.mock.calls.length; const bindGroupsAfterFirst = device.mock.createBindGroup.mock.calls.length; const modulesAfterFirst = device.mock.createShaderModule.mock.calls.length; - computer.compute(buffer, false); - computer.compute(buffer, false); + computer.scan(buffer); + computer.scan(buffer); expect(device.mock.createBuffer.mock.calls.length).toBe(buffersAfterFirst); expect(device.mock.createBindGroup.mock.calls.length).toBe(bindGroupsAfterFirst); expect(device.mock.createShaderModule.mock.calls.length).toBe(modulesAfterFirst); }); - it('rejects ambiguous recording destinations', ({ root, device }) => { + it('should produce valid code for a reduction', ({ root, device }) => { const computer = createPrefixScanComputer(root, { operation: std.add, identityElement: 0, + dataType: d.u32, }); - const buffer = root.createBuffer(d.arrayOf(d.f32, 64)).$usage('storage'); - const plan = computer.prepare(buffer); + const buffer = root.createBuffer(d.arrayOf(d.u32, 4096)).$usage('storage'); - const externalPass = { - __brand: 'GPUComputePassEncoder', - } as unknown as GPUComputePassEncoder; - const encoder = device.createCommandEncoder(); + computer.reduce(buffer); - expect(() => - plan.run({ pass: externalPass, encoder } as never), - ).toThrowErrorMatchingInlineSnapshot( - `[Error: A run cannot record into both an encoder and an existing compute pass.]`, - ); + expect(getResolvedWgsl(device)).toMatchInlineSnapshot(` + "fn flatWorkgroupIndex(wid: vec3u, numWorkgroups: vec3u) -> u32 { + return ((wid.x + (wid.y * numWorkgroups.x)) + ((wid.z * numWorkgroups.x) * numWorkgroups.y)); + } + + @group(0) @binding(0) var input: array; + + var workgroupMemory: array; + + fn upsweep(localIdx: u32) { + var offset = 1u; + for (var span = 128u; (span > 0u); span >>= 1u) { + workgroupBarrier(); + if ((localIdx < span)) { + let ai = ((offset * ((2u * localIdx) + 1u)) - 1u); + let bi = ((offset * ((2u * localIdx) + 2u)) - 1u); + workgroupMemory[bi] = (workgroupMemory[ai] + workgroupMemory[bi]); + } + offset <<= 1u; + } + } + + @group(0) @binding(1) var sums: array; + + @compute @workgroup_size(256) fn item(@builtin(local_invocation_id) lid: vec3u, @builtin(workgroup_id) wid: vec3u, @builtin(num_workgroups) numWorkgroups: vec3u) { + let workgroupId = flatWorkgroupIndex(wid, numWorkgroups); + let localIdx = lid.x; + let baseIdx = (((workgroupId * 256u) + localIdx) * 8u); + var partialSums = array(0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u); + var prev = 0u; + var lastIdx = 0u; + // unrolled iteration #0 + { + if (((baseIdx + 0u) < arrayLength(&input))) { + partialSums[0i] = (prev + input[(baseIdx + 0u)]); + prev = partialSums[0i]; + lastIdx = 0u; + } + } + // unrolled iteration #1 + { + if (((baseIdx + 1u) < arrayLength(&input))) { + partialSums[1i] = (prev + input[(baseIdx + 1u)]); + prev = partialSums[1i]; + lastIdx = 1u; + } + } + // unrolled iteration #2 + { + if (((baseIdx + 2u) < arrayLength(&input))) { + partialSums[2i] = (prev + input[(baseIdx + 2u)]); + prev = partialSums[2i]; + lastIdx = 2u; + } + } + // unrolled iteration #3 + { + if (((baseIdx + 3u) < arrayLength(&input))) { + partialSums[3i] = (prev + input[(baseIdx + 3u)]); + prev = partialSums[3i]; + lastIdx = 3u; + } + } + // unrolled iteration #4 + { + if (((baseIdx + 4u) < arrayLength(&input))) { + partialSums[4i] = (prev + input[(baseIdx + 4u)]); + prev = partialSums[4i]; + lastIdx = 4u; + } + } + // unrolled iteration #5 + { + if (((baseIdx + 5u) < arrayLength(&input))) { + partialSums[5i] = (prev + input[(baseIdx + 5u)]); + prev = partialSums[5i]; + lastIdx = 5u; + } + } + // unrolled iteration #6 + { + if (((baseIdx + 6u) < arrayLength(&input))) { + partialSums[6i] = (prev + input[(baseIdx + 6u)]); + prev = partialSums[6i]; + lastIdx = 6u; + } + } + // unrolled iteration #7 + { + if (((baseIdx + 7u) < arrayLength(&input))) { + partialSums[7i] = (prev + input[(baseIdx + 7u)]); + prev = partialSums[7i]; + lastIdx = 7u; + } + } + workgroupMemory[localIdx] = partialSums[lastIdx]; + upsweep(localIdx); + if (((localIdx == 0u) && (workgroupId < arrayLength(&sums)))) { + sums[workgroupId] = workgroupMemory[255i]; + } + }" + `); + expect(device.mock.createComputePipeline.mock.calls.length).toMatchInlineSnapshot(`1`); }); it('rejects empty and mismatched buffers', ({ root }) => { diff --git a/packages/typegpu-testing-utility/src/extendedIt.ts b/packages/typegpu-testing-utility/src/extendedIt.ts index 6e6158930c..1f4353a451 100644 --- a/packages/typegpu-testing-utility/src/extendedIt.ts +++ b/packages/typegpu-testing-utility/src/extendedIt.ts @@ -137,6 +137,7 @@ export const it = base limits: { maxUniformBuffersPerShaderStage: 12, maxStorageBuffersPerShaderStage: 8, + maxComputeWorkgroupStorageSize: 16384, }, destroy: vi.fn(), };