From 1bbf1b1d94b5cb1c3e11deb8260c76ca1cfd2097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=AA=E3=82=AD=E8=90=8C?= Date: Mon, 8 Sep 2025 22:10:55 +0200 Subject: [PATCH] implementing more chunk ops based on GPU composing, toEdits --- static/canvas-renderer.js | 28 +++++- static/chunk-allocator.js | 197 ++++++++++++++++++++++++++++++++++++-- static/index.js | 4 +- static/online-users.js | 4 +- static/wall.js | 64 ++++++++----- 5 files changed, 259 insertions(+), 38 deletions(-) diff --git a/static/canvas-renderer.js b/static/canvas-renderer.js index 26fcb48..dfb00ec 100644 --- a/static/canvas-renderer.js +++ b/static/canvas-renderer.js @@ -81,9 +81,6 @@ class CanvasRenderer extends HTMLElement { console.info("vendor", this.gl.getParameter(this.gl.VENDOR)); console.info("renderer", this.gl.getParameter(this.gl.RENDERER)); - this.gl.enable(this.gl.BLEND); - this.gl.blendFunc(this.gl.SRC_ALPHA, this.gl.ONE_MINUS_SRC_ALPHA); - // Due to an ANGLE bug on Windows, we can only render around 64 rectangles in a batch. // // It seems that for DirectX it generates a horribly inefficient shader that the DirectX @@ -135,12 +132,25 @@ class CanvasRenderer extends HTMLElement { precision highp float; uniform sampler2D u_texture; + uniform int u_visAtlasIndex; in vec2 vf_uv; out vec4 f_color; + float goldNoise(vec2 xy, float seed) { + return fract(tan(distance(xy * 1.6180339, xy) * seed) * xy.x); + } + void main() { vec4 color = texture(u_texture, vf_uv); + if (u_visAtlasIndex != 0) { + color = vec4( + goldNoise(vec2(float(u_visAtlasIndex), 0.0), 0.1), + goldNoise(vec2(float(u_visAtlasIndex), 0.0), 0.2), + goldNoise(vec2(float(u_visAtlasIndex), 0.0), 0.3), + 1.0 + ); + } f_color = color; } `, @@ -152,6 +162,7 @@ class CanvasRenderer extends HTMLElement { u_projection: this.gl.getUniformLocation(renderChunksProgramId, "u_projection"), u_view: this.gl.getUniformLocation(renderChunksProgramId, "u_view"), u_texture: this.gl.getUniformLocation(renderChunksProgramId, "u_texture"), + u_visAtlasIndex: this.gl.getUniformLocation(renderChunksProgramId, "u_visAtlasIndex"), ub_rects: this.gl.getUniformBlockIndex(renderChunksProgramId, "ub_rects"), }; @@ -209,13 +220,20 @@ class CanvasRenderer extends HTMLElement { console.debug("GL error state", this.gl.getError()); console.groupEnd(); + + // Flag that prevents the renderer from exploding in case any part of + // initialisation throws an exception. + this.ok = true; } // Renderer #render() { + if (!this.ok) return; + // NOTE: We should probably render on-demand only when it's needed. requestAnimationFrame(() => this.#render()); + this.atlasAllocator.tickDownloads(); this.#renderWall(); } @@ -228,6 +246,9 @@ class CanvasRenderer extends HTMLElement { this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); this.gl.scissor(0, 0, this.canvas.width, this.canvas.height); + this.gl.enable(this.gl.BLEND); + this.gl.blendFunc(this.gl.SRC_ALPHA, this.gl.ONE_MINUS_SRC_ALPHA); + this.gl.clearColor(1, 1, 1, 1); this.gl.clear(this.gl.COLOR_BUFFER_BIT); @@ -271,6 +292,7 @@ class CanvasRenderer extends HTMLElement { for (let [i, chunks] of batch) { let atlas = this.atlasAllocator.atlases[i]; this.gl.bindTexture(this.gl.TEXTURE_2D, atlas.texture); + // this.gl.uniform1i(this.renderChunksProgram.u_visAtlasIndex, i + 1); this.#resetRectBuffer(); for (let chunk of chunks) { diff --git a/static/chunk-allocator.js b/static/chunk-allocator.js index 48a64b7..b3e937f 100644 --- a/static/chunk-allocator.js +++ b/static/chunk-allocator.js @@ -1,3 +1,5 @@ +import { compileProgram } from "rkgk/webgl.js"; + class Atlas { static getInitBuffer(chunkSize, nChunks) { let imageSize = chunkSize * nChunks; @@ -81,7 +83,7 @@ class Atlas { this.chunkSize, gl.RGBA, gl.UNSIGNED_BYTE, - null, + 0, ); gl.bindFramebuffer(gl.FRAMEBUFFER, null); } @@ -96,6 +98,35 @@ class Atlas { } } +const compositeVertexShader = `#version 300 es + precision highp float; + + layout (location = 0) in vec2 a_position; + layout (location = 1) in vec2 a_uv; + + out vec2 vf_uv; + + void main() { + gl_Position = vec4(a_position, 0.0, 1.0); + vf_uv = a_uv; + } +`; + +const compositeFragmentShader = `#version 300 es + precision highp float; + + uniform sampler2D u_chunk; + + in vec2 vf_uv; + + out vec4 f_color; + + void main() { + f_color = texture(u_chunk, vf_uv); + // f_color = vec4(vec3(0.0), 1.0); + } +`; + export class AtlasAllocator { atlases = []; @@ -113,6 +144,74 @@ export class AtlasAllocator { this.chunkSize = chunkSize; this.nChunks = nChunks; this.initBuffer = Atlas.getInitBuffer(chunkSize, nChunks); + + // Compositing pipeline + + let compositeProgramId = compileProgram(gl, compositeVertexShader, compositeFragmentShader); + this.compositeProgram = { + id: compositeProgramId, + u_chunk: gl.getUniformLocation(compositeProgramId, "u_chunk"), + }; + + // prettier-ignore + this.compositeRectData = new Float32Array([ + // a_position + -1, 1, // 0: top left + 1, 1, // 1: top right + 1, -1, // 2: bottom right + -1, -1, // 3: bottom left + + // a_uv - filled out later when compositing + 0, 0, + 0, 0, + 0, 0, + 0, 0, + ]); + let compositeRectIndices = new Uint16Array([0, 1, 2, 2, 3, 0]); + this.compositeRectUv = this.compositeRectData.subarray(8); + + this.compositeRectVao = gl.createVertexArray(); + this.compositeRectVbo = gl.createBuffer(); + this.compositeRectIbo = gl.createBuffer(); + gl.bindVertexArray(this.compositeRectVao); + + gl.bindBuffer(gl.ARRAY_BUFFER, this.compositeRectVbo); + gl.bufferData(gl.ARRAY_BUFFER, this.compositeRectData, gl.DYNAMIC_DRAW); + console.log(this.compositeRectData.byteLength); + + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.compositeRectIbo); + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, compositeRectIndices, gl.DYNAMIC_DRAW); + + gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 2 * 4, 0); + gl.vertexAttribPointer(1, 2, gl.FLOAT, false, 2 * 4, this.compositeRectUv.byteOffset); + for (let i = 0; i < 2; ++i) gl.enableVertexAttribArray(i); + + this.compositeTexture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, this.compositeTexture); + gl.texImage2D( + gl.TEXTURE_2D, + 0, + gl.RGBA8, + chunkSize, + chunkSize, + 0, + gl.RGBA, + gl.UNSIGNED_BYTE, + null, + ); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + + this.compositeFramebuffer = gl.createFramebuffer(); + gl.bindFramebuffer(gl.FRAMEBUFFER, this.compositeFramebuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + gl.COLOR_ATTACHMENT0, + gl.TEXTURE_2D, + this.compositeTexture, + 0, + ); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); } #obtainId(allocInfo) { @@ -190,7 +289,7 @@ export class AtlasAllocator { this.atlases[allocInfo.i].download(gl, allocInfo.allocation); let fence = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0); - gl.bindBuffer(gl.PIXEL_PACK_BUFFER, 0); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); // Add for ticking @@ -207,6 +306,8 @@ export class AtlasAllocator { // Call every frame to poll for download completion. tickDownloads() { + if (this.#pendingDownloads.length == 0) return; + let gl = this.gl; for (let i = 0; i < this.#pendingDownloads.length; ++i) { @@ -218,16 +319,22 @@ export class AtlasAllocator { let arrayBuffer = this.#downloadBufferPool.pop() ?? new ArrayBuffer(dataSize); gl.bindBuffer(gl.PIXEL_PACK_BUFFER, pending.pbo); - gl.getBufferSubData(gl.PIXEL_PACK_BUFFER, 0, arrayBuffer); - gl.bindBuffer(gl.PIXEL_PACK_BUFFER, 0); + gl.getBufferSubData(gl.PIXEL_PACK_BUFFER, 0, new Uint8Array(arrayBuffer)); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); gl.deleteSync(pending.fence); - pending.resolve(arrayBuffer); + pending.resolve({ + width: this.chunkSize, + height: this.chunkSize, + data: arrayBuffer, + }); let last = this.#pendingDownloads.pop(); - if (last != null) { + if (this.#pendingDownloads.length > 0) { this.#pendingDownloads[i] = last; --i; + } else { + break; // now empty } } } @@ -258,4 +365,82 @@ export class AtlasAllocator { resetCanvas, }; } + + // NOTE: I was thinking a bit about whether the chunk allocator is the right place to put + // compositing operations like this. After much consideration, I've decided that it's pretty + // much the only sensible place, because it's the only place concerned with the layout of + // chunks in memory, and rendering of laid out chunks is quite implementation dependent. + // + // This does break the purity of the "allocator" role a bit though, but I don't really know if + // there's a good way around that. + // + // Maybe. But I don't feel like breaking this apart to 10 smaller classes, either. + + #drawComposite(u, v, uvScale) { + // Assumes bound source texture, destination framebuffer, and viewport set. + + let gl = this.gl; + + gl.bindVertexArray(this.compositeRectVao); + gl.bindBuffer(gl.ARRAY_BUFFER, this.compositeRectVbo); + gl.useProgram(this.compositeProgram.id); + + gl.uniform1i(this.compositeProgram.u_chunk, 0); + + let uv = this.compositeRectUv; + uv[0] = u * uvScale; + uv[1] = v * uvScale; + uv[2] = (u + 1) * uvScale; + uv[3] = v * uvScale; + uv[4] = (u + 1) * uvScale; + uv[5] = (v + 1) * uvScale; + uv[6] = u * uvScale; + uv[7] = (v + 1) * uvScale; + gl.bufferSubData(gl.ARRAY_BUFFER, uv.byteOffset, uv); + + gl.drawElements(gl.TRIANGLES, 6, gl.UNSIGNED_SHORT, 0); + } + + composite(dstId, srcId, op) { + // NOTE: This has to go through an intermediate buffer in case the source and destination + // atlas are the same. + + console.assert(op == "alphaBlend", "composite operation must be alphaBlend"); + + let gl = this.gl; + + let dstAtlas = this.atlases[this.getAtlasIndex(dstId)]; + let srcAtlas = this.atlases[this.getAtlasIndex(srcId)]; + let dstAllocation = this.getAllocation(dstId); + let srcAllocation = this.getAllocation(srcId); + + // Source -> intermediate buffer + + gl.disable(gl.BLEND); + gl.disable(gl.SCISSOR_TEST); + + gl.bindFramebuffer(gl.FRAMEBUFFER, this.compositeFramebuffer); + gl.bindTexture(gl.TEXTURE_2D, srcAtlas.texture); + gl.viewport(0, 0, this.chunkSize, this.chunkSize); + this.#drawComposite(srcAllocation.x, srcAllocation.y, 1 / this.nChunks); + + // Intermediate buffer -> destination + + gl.enable(gl.BLEND); + gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); + + gl.bindFramebuffer(gl.FRAMEBUFFER, dstAtlas.framebuffer); + gl.bindTexture(gl.TEXTURE_2D, this.compositeTexture); + gl.viewport( + dstAllocation.x * this.chunkSize, + dstAllocation.y * this.chunkSize, + this.chunkSize, + this.chunkSize, + ); + this.#drawComposite(0, 0, 1); + + // Cleanup + + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + } } diff --git a/static/index.js b/static/index.js index 4b9fe52..b31f30d 100644 --- a/static/index.js +++ b/static/index.js @@ -261,10 +261,10 @@ function readUrl(urlString) { }); canvasRenderer.addEventListener(".commitInteraction", async () => { - let scratchLayer = currentUser.commitScratchLayer(wall); + let scratchLayer = currentUser.commitScratchLayer(chunkAllocator, wall); if (scratchLayer == null) return; - let edits = await scratchLayer.toEdits(); + let edits = await scratchLayer.toEdits(chunkAllocator); scratchLayer.destroy(chunkAllocator); let editRecords = []; diff --git a/static/online-users.js b/static/online-users.js index ceaf896..a23ef53 100644 --- a/static/online-users.js +++ b/static/online-users.js @@ -122,9 +122,9 @@ export class User { // Returns the scratch layer committed to the wall, so that the caller may do additional // processing with the completed layer (i.e. send to the server.) // The layer has to be .destroy()ed once you're done working with it. - commitScratchLayer(wall) { + commitScratchLayer(chunkAllocator, wall) { if (this.scratchLayer != null) { - wall.mainLayer.compositeAlpha(this.scratchLayer); + wall.mainLayer.composite(chunkAllocator, this.scratchLayer, "alphaBlend"); wall.removeLayer(this.scratchLayer); let scratchLayer = this.scratchLayer; this.scratchLayer = null; diff --git a/static/wall.js b/static/wall.js index eba4119..e6915f8 100644 --- a/static/wall.js +++ b/static/wall.js @@ -55,39 +55,53 @@ export class Layer { } } - compositeAlpha(src) { - // TODO - // for (let { x, y, chunk: srcChunk } of src.chunks.values()) { - // srcChunk.syncFromPixmap(); - // let dstChunk = this.getOrCreateChunk(x, y); - // if (dstChunk == null) continue; - // dstChunk.ctx.globalCompositeOperation = "source-over"; - // dstChunk.ctx.drawImage(srcChunk.canvas, 0, 0); - // dstChunk.syncToPixmap(); - // dstChunk.markModified(); - // } + composite(chunkAllocator, src, op) { + for (let { x, y, chunk: srcChunk } of src.chunks.values()) { + let dstChunk = this.getOrCreateChunk(chunkAllocator, x, y); + if (dstChunk == null) continue; + + chunkAllocator.composite(dstChunk.id, srcChunk.id, op); + } } - async toEdits() { + async toEdits(chunkAllocator) { + console.time("toEdits"); + let edits = []; + let encodeTime = 0; + for (let { x, y, chunk } of this.chunks.values()) { + edits.push({ + chunk: { x, y }, + data: chunk.download(chunkAllocator).then(async (downloaded) => { + let start = performance.now(); - // TODO + let imageBitmap = await createImageBitmap( + new ImageData( + new Uint8ClampedArray(downloaded.data), + downloaded.width, + downloaded.height, + ), + ); + chunkAllocator.freeDownloaded(downloaded.data); + let canvas = new OffscreenCanvas(downloaded.width, downloaded.height); + let ctx = canvas.getContext("bitmaprenderer"); + ctx.transferFromImageBitmap(imageBitmap); + let blob = canvas.convertToBlob({ type: "image/png" }); - // let start = performance.now(); + let end = performance.now(); + console.log("encoding image took", end - start, "ms"); + encodeTime += end - start; - // for (let { x, y, chunk } of this.chunks.values()) { - // edits.push({ - // chunk: { x, y }, - // data: chunk.canvas.convertToBlob({ type: "image/png" }), - // }); - // } + return blob; + }), + }); + } - // for (let edit of edits) { - // edit.data = await edit.data; - // } + for (let edit of edits) { + edit.data = await edit.data; + } - // let end = performance.now(); - // console.debug("toEdits done", end - start); + console.timeEnd("toEdits"); return edits; }