implementing more chunk ops based on GPU

composing, toEdits

static/canvas-renderer.js

@@ -81,9 +81,6 @@ class CanvasRenderer extends HTMLElement {
         console.info("vendor", this.gl.getParameter(this.gl.VENDOR));
         console.info("renderer", this.gl.getParameter(this.gl.RENDERER));
 
-        this.gl.enable(this.gl.BLEND);
-        this.gl.blendFunc(this.gl.SRC_ALPHA, this.gl.ONE_MINUS_SRC_ALPHA);
-
         // Due to an ANGLE bug on Windows, we can only render around 64 rectangles in a batch.
         //
         // It seems that for DirectX it generates a horribly inefficient shader that the DirectX
@@ -135,12 +132,25 @@ class CanvasRenderer extends HTMLElement {
             precision highp float;
 
             uniform sampler2D u_texture;
+            uniform int u_visAtlasIndex;
 
             in vec2 vf_uv;
             out vec4 f_color;
 
+            float goldNoise(vec2 xy, float seed) {
+                return fract(tan(distance(xy * 1.6180339, xy) * seed) * xy.x);
+            }
+
             void main() {
                 vec4 color = texture(u_texture, vf_uv);
+                if (u_visAtlasIndex != 0) {
+                    color = vec4(
+                        goldNoise(vec2(float(u_visAtlasIndex), 0.0), 0.1),
+                        goldNoise(vec2(float(u_visAtlasIndex), 0.0), 0.2),
+                        goldNoise(vec2(float(u_visAtlasIndex), 0.0), 0.3),
+                        1.0
+                    );
+                }
                 f_color = color;
             }
             `,
@@ -152,6 +162,7 @@ class CanvasRenderer extends HTMLElement {
             u_projection: this.gl.getUniformLocation(renderChunksProgramId, "u_projection"),
             u_view: this.gl.getUniformLocation(renderChunksProgramId, "u_view"),
             u_texture: this.gl.getUniformLocation(renderChunksProgramId, "u_texture"),
+            u_visAtlasIndex: this.gl.getUniformLocation(renderChunksProgramId, "u_visAtlasIndex"),
             ub_rects: this.gl.getUniformBlockIndex(renderChunksProgramId, "ub_rects"),
         };
 
@@ -209,13 +220,20 @@ class CanvasRenderer extends HTMLElement {
         console.debug("GL error state", this.gl.getError());
 
         console.groupEnd();
+
+        // Flag that prevents the renderer from exploding in case any part of
+        // initialisation throws an exception.
+        this.ok = true;
     }
 
     // Renderer
 
     #render() {
+        if (!this.ok) return;
+
         // NOTE: We should probably render on-demand only when it's needed.
         requestAnimationFrame(() => this.#render());
+        this.atlasAllocator.tickDownloads();
         this.#renderWall();
     }
 
@@ -228,6 +246,9 @@ class CanvasRenderer extends HTMLElement {
         this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
         this.gl.scissor(0, 0, this.canvas.width, this.canvas.height);
 
+        this.gl.enable(this.gl.BLEND);
+        this.gl.blendFunc(this.gl.SRC_ALPHA, this.gl.ONE_MINUS_SRC_ALPHA);
+
         this.gl.clearColor(1, 1, 1, 1);
         this.gl.clear(this.gl.COLOR_BUFFER_BIT);
 
@@ -271,6 +292,7 @@ class CanvasRenderer extends HTMLElement {
             for (let [i, chunks] of batch) {
                 let atlas = this.atlasAllocator.atlases[i];
                 this.gl.bindTexture(this.gl.TEXTURE_2D, atlas.texture);
+                // this.gl.uniform1i(this.renderChunksProgram.u_visAtlasIndex, i + 1);
 
                 this.#resetRectBuffer();
                 for (let chunk of chunks) {

static/chunk-allocator.js

@@ -1,3 +1,5 @@
+import { compileProgram } from "rkgk/webgl.js";
+
 class Atlas {
     static getInitBuffer(chunkSize, nChunks) {
         let imageSize = chunkSize * nChunks;
@@ -81,7 +83,7 @@ class Atlas {
             this.chunkSize,
             gl.RGBA,
             gl.UNSIGNED_BYTE,
-            null,
+            0,
         );
         gl.bindFramebuffer(gl.FRAMEBUFFER, null);
     }
@@ -96,6 +98,35 @@ class Atlas {
     }
 }
 
+const compositeVertexShader = `#version 300 es
+    precision highp float;
+
+    layout (location = 0) in vec2 a_position;
+    layout (location = 1) in vec2 a_uv;
+
+    out vec2 vf_uv;
+
+    void main() {
+        gl_Position = vec4(a_position, 0.0, 1.0);
+        vf_uv = a_uv;
+    }
+`;
+
+const compositeFragmentShader = `#version 300 es
+    precision highp float;
+
+    uniform sampler2D u_chunk;
+
+    in vec2 vf_uv;
+
+    out vec4 f_color;
+
+    void main() {
+        f_color = texture(u_chunk, vf_uv);
+        // f_color = vec4(vec3(0.0), 1.0);
+    }
+`;
+
 export class AtlasAllocator {
     atlases = [];
 
@@ -113,6 +144,74 @@ export class AtlasAllocator {
         this.chunkSize = chunkSize;
         this.nChunks = nChunks;
         this.initBuffer = Atlas.getInitBuffer(chunkSize, nChunks);
+
+        // Compositing pipeline
+
+        let compositeProgramId = compileProgram(gl, compositeVertexShader, compositeFragmentShader);
+        this.compositeProgram = {
+            id: compositeProgramId,
+            u_chunk: gl.getUniformLocation(compositeProgramId, "u_chunk"),
+        };
+
+        // prettier-ignore
+        this.compositeRectData = new Float32Array([
+            // a_position
+            -1, 1, // 0: top left
+            1, 1, // 1: top right
+            1, -1, // 2: bottom right
+            -1, -1, // 3: bottom left
+
+            // a_uv - filled out later when compositing
+            0, 0,
+            0, 0,
+            0, 0,
+            0, 0,
+        ]);
+        let compositeRectIndices = new Uint16Array([0, 1, 2, 2, 3, 0]);
+        this.compositeRectUv = this.compositeRectData.subarray(8);
+
+        this.compositeRectVao = gl.createVertexArray();
+        this.compositeRectVbo = gl.createBuffer();
+        this.compositeRectIbo = gl.createBuffer();
+        gl.bindVertexArray(this.compositeRectVao);
+
+        gl.bindBuffer(gl.ARRAY_BUFFER, this.compositeRectVbo);
+        gl.bufferData(gl.ARRAY_BUFFER, this.compositeRectData, gl.DYNAMIC_DRAW);
+        console.log(this.compositeRectData.byteLength);
+
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.compositeRectIbo);
+        gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, compositeRectIndices, gl.DYNAMIC_DRAW);
+
+        gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 2 * 4, 0);
+        gl.vertexAttribPointer(1, 2, gl.FLOAT, false, 2 * 4, this.compositeRectUv.byteOffset);
+        for (let i = 0; i < 2; ++i) gl.enableVertexAttribArray(i);
+
+        this.compositeTexture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, this.compositeTexture);
+        gl.texImage2D(
+            gl.TEXTURE_2D,
+            0,
+            gl.RGBA8,
+            chunkSize,
+            chunkSize,
+            0,
+            gl.RGBA,
+            gl.UNSIGNED_BYTE,
+            null,
+        );
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+
+        this.compositeFramebuffer = gl.createFramebuffer();
+        gl.bindFramebuffer(gl.FRAMEBUFFER, this.compositeFramebuffer);
+        gl.framebufferTexture2D(
+            gl.FRAMEBUFFER,
+            gl.COLOR_ATTACHMENT0,
+            gl.TEXTURE_2D,
+            this.compositeTexture,
+            0,
+        );
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
     }
 
     #obtainId(allocInfo) {
@@ -190,7 +289,7 @@ export class AtlasAllocator {
         this.atlases[allocInfo.i].download(gl, allocInfo.allocation);
         let fence = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0);
 
-        gl.bindBuffer(gl.PIXEL_PACK_BUFFER, 0);
+        gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null);
 
         // Add for ticking
 
@@ -207,6 +306,8 @@ export class AtlasAllocator {
 
     // Call every frame to poll for download completion.
     tickDownloads() {
+        if (this.#pendingDownloads.length == 0) return;
+
         let gl = this.gl;
 
         for (let i = 0; i < this.#pendingDownloads.length; ++i) {
@@ -218,16 +319,22 @@ export class AtlasAllocator {
                 let arrayBuffer = this.#downloadBufferPool.pop() ?? new ArrayBuffer(dataSize);
 
                 gl.bindBuffer(gl.PIXEL_PACK_BUFFER, pending.pbo);
-                gl.getBufferSubData(gl.PIXEL_PACK_BUFFER, 0, arrayBuffer);
+                gl.getBufferSubData(gl.PIXEL_PACK_BUFFER, 0, new Uint8Array(arrayBuffer));
-                gl.bindBuffer(gl.PIXEL_PACK_BUFFER, 0);
+                gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null);
                 gl.deleteSync(pending.fence);
 
-                pending.resolve(arrayBuffer);
+                pending.resolve({
+                    width: this.chunkSize,
+                    height: this.chunkSize,
+                    data: arrayBuffer,
+                });
 
                 let last = this.#pendingDownloads.pop();
-                if (last != null) {
+                if (this.#pendingDownloads.length > 0) {
                     this.#pendingDownloads[i] = last;
                     --i;
+                } else {
+                    break; // now empty
                 }
             }
         }
@@ -258,4 +365,82 @@ export class AtlasAllocator {
             resetCanvas,
         };
     }
+
+    // NOTE: I was thinking a bit about whether the chunk allocator is the right place to put
+    // compositing operations like this. After much consideration, I've decided that it's pretty
+    // much the only sensible place, because it's the only place concerned with the layout of
+    // chunks in memory, and rendering of laid out chunks is quite implementation dependent.
+    //
+    // This does break the purity of the "allocator" role a bit though, but I don't really know if
+    // there's a good way around that.
+    //
+    // Maybe. But I don't feel like breaking this apart to 10 smaller classes, either.
+
+    #drawComposite(u, v, uvScale) {
+        // Assumes bound source texture, destination framebuffer, and viewport set.
+
+        let gl = this.gl;
+
+        gl.bindVertexArray(this.compositeRectVao);
+        gl.bindBuffer(gl.ARRAY_BUFFER, this.compositeRectVbo);
+        gl.useProgram(this.compositeProgram.id);
+
+        gl.uniform1i(this.compositeProgram.u_chunk, 0);
+
+        let uv = this.compositeRectUv;
+        uv[0] = u * uvScale;
+        uv[1] = v * uvScale;
+        uv[2] = (u + 1) * uvScale;
+        uv[3] = v * uvScale;
+        uv[4] = (u + 1) * uvScale;
+        uv[5] = (v + 1) * uvScale;
+        uv[6] = u * uvScale;
+        uv[7] = (v + 1) * uvScale;
+        gl.bufferSubData(gl.ARRAY_BUFFER, uv.byteOffset, uv);
+
+        gl.drawElements(gl.TRIANGLES, 6, gl.UNSIGNED_SHORT, 0);
+    }
+
+    composite(dstId, srcId, op) {
+        // NOTE: This has to go through an intermediate buffer in case the source and destination
+        // atlas are the same.
+
+        console.assert(op == "alphaBlend", "composite operation must be alphaBlend");
+
+        let gl = this.gl;
+
+        let dstAtlas = this.atlases[this.getAtlasIndex(dstId)];
+        let srcAtlas = this.atlases[this.getAtlasIndex(srcId)];
+        let dstAllocation = this.getAllocation(dstId);
+        let srcAllocation = this.getAllocation(srcId);
+
+        // Source -> intermediate buffer
+
+        gl.disable(gl.BLEND);
+        gl.disable(gl.SCISSOR_TEST);
+
+        gl.bindFramebuffer(gl.FRAMEBUFFER, this.compositeFramebuffer);
+        gl.bindTexture(gl.TEXTURE_2D, srcAtlas.texture);
+        gl.viewport(0, 0, this.chunkSize, this.chunkSize);
+        this.#drawComposite(srcAllocation.x, srcAllocation.y, 1 / this.nChunks);
+
+        // Intermediate buffer -> destination
+
+        gl.enable(gl.BLEND);
+        gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+
+        gl.bindFramebuffer(gl.FRAMEBUFFER, dstAtlas.framebuffer);
+        gl.bindTexture(gl.TEXTURE_2D, this.compositeTexture);
+        gl.viewport(
+            dstAllocation.x * this.chunkSize,
+            dstAllocation.y * this.chunkSize,
+            this.chunkSize,
+            this.chunkSize,
+        );
+        this.#drawComposite(0, 0, 1);
+
+        // Cleanup
+
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    }
 }

static/index.js

@@ -261,10 +261,10 @@ function readUrl(urlString) {
     });
 
     canvasRenderer.addEventListener(".commitInteraction", async () => {
-        let scratchLayer = currentUser.commitScratchLayer(wall);
+        let scratchLayer = currentUser.commitScratchLayer(chunkAllocator, wall);
         if (scratchLayer == null) return;
 
-        let edits = await scratchLayer.toEdits();
+        let edits = await scratchLayer.toEdits(chunkAllocator);
         scratchLayer.destroy(chunkAllocator);
 
         let editRecords = [];

static/online-users.js

@@ -122,9 +122,9 @@ export class User {
     // Returns the scratch layer committed to the wall, so that the caller may do additional
     // processing with the completed layer (i.e. send to the server.)
     // The layer has to be .destroy()ed once you're done working with it.
-    commitScratchLayer(wall) {
+    commitScratchLayer(chunkAllocator, wall) {
         if (this.scratchLayer != null) {
-            wall.mainLayer.compositeAlpha(this.scratchLayer);
+            wall.mainLayer.composite(chunkAllocator, this.scratchLayer, "alphaBlend");
             wall.removeLayer(this.scratchLayer);
             let scratchLayer = this.scratchLayer;
             this.scratchLayer = null;

static/wall.js

@@ -55,39 +55,53 @@ export class Layer {
         }
     }
 
-    compositeAlpha(src) {
+    composite(chunkAllocator, src, op) {
-        // TODO
+        for (let { x, y, chunk: srcChunk } of src.chunks.values()) {
-        // for (let { x, y, chunk: srcChunk } of src.chunks.values()) {
+            let dstChunk = this.getOrCreateChunk(chunkAllocator, x, y);
-        //     srcChunk.syncFromPixmap();
+            if (dstChunk == null) continue;
-        //     let dstChunk = this.getOrCreateChunk(x, y);
+
-        //     if (dstChunk == null) continue;
+            chunkAllocator.composite(dstChunk.id, srcChunk.id, op);
-        //     dstChunk.ctx.globalCompositeOperation = "source-over";
+        }
-        //     dstChunk.ctx.drawImage(srcChunk.canvas, 0, 0);
-        //     dstChunk.syncToPixmap();
-        //     dstChunk.markModified();
-        // }
     }
 
-    async toEdits() {
+    async toEdits(chunkAllocator) {
+        console.time("toEdits");
+
         let edits = [];
+        let encodeTime = 0;
+        for (let { x, y, chunk } of this.chunks.values()) {
+            edits.push({
+                chunk: { x, y },
+                data: chunk.download(chunkAllocator).then(async (downloaded) => {
+                    let start = performance.now();
 
-        // TODO
+                    let imageBitmap = await createImageBitmap(
+                        new ImageData(
+                            new Uint8ClampedArray(downloaded.data),
+                            downloaded.width,
+                            downloaded.height,
+                        ),
+                    );
+                    chunkAllocator.freeDownloaded(downloaded.data);
+                    let canvas = new OffscreenCanvas(downloaded.width, downloaded.height);
+                    let ctx = canvas.getContext("bitmaprenderer");
+                    ctx.transferFromImageBitmap(imageBitmap);
+                    let blob = canvas.convertToBlob({ type: "image/png" });
 
-        // let start = performance.now();
+                    let end = performance.now();
+                    console.log("encoding image took", end - start, "ms");
+                    encodeTime += end - start;
 
-        // for (let { x, y, chunk } of this.chunks.values()) {
+                    return blob;
-        //     edits.push({
+                }),
-        //         chunk: { x, y },
+            });
-        //         data: chunk.canvas.convertToBlob({ type: "image/png" }),
+        }
-        //     });
-        // }
 
-        // for (let edit of edits) {
+        for (let edit of edits) {
-        //     edit.data = await edit.data;
+            edit.data = await edit.data;
-        // }
+        }
 
-        // let end = performance.now();
+        console.timeEnd("toEdits");
-        // console.debug("toEdits done", end - start);
 
         return edits;
     }