Fix multiplayer, add mouse wheel panning

README.md

@@ -3,18 +3,20 @@ Release:
         + Benchmark harness
         + Reuse points, pack "nodraw" in high bit of stroke id (probably have at least one more bit, so up to 4 flag configurations)
         + Draw dynamic data (strokes in progress)
+        - Z-prepass fringe bug (also, when do we enable the prepass?)
         - Textured quads (pictures, code already written in older version)
         - Resize and move pictures (draw handles)
-        - Z-prepass fringe bug (also, when do we enable the prepass?)
+    - Debug
         - Restore ability to limit event range
-        - Only upload stroke data to texture as it arrives (texSubImage2D)
-    - Listeners/events
-        - Investigate skipped inputs on mobile (panning, zooming)
+    * Listeners/events/multiplayer
+        + Fix multiplayer LUL
+        + Fix blinking own stroke inbetween SYN->server and SYN->client
+        + Drag with mouse button 3
+        + Investigate skipped inputs on mobile (panning, zooming) [Events were not actually getting skipped. The stroke previews were just not being drawn]
         - Save events to indexeddb (as some kind of a blob), restore on reconnect and page reload
-        - Separate events and other data clearly (events are self-contained, other data is temporal/non-vital)
         - Do NOT use session id as player id LUL
         - Local prediction for tools!
-        - Drag with mouse button 3
+        - Be able to have multiple "current" strokes per player. In case of bad internet this can happen!
     - Missing features I do not consider bonus
         - Eraser
         - Line drawing
@@ -24,9 +26,9 @@ Release:
         - Undo/redo
         - Dynamic svg cursor to represent the brush
     - Polish
+        * Use typedvector where appropriate
         - Show what's happening while the desk is loading (downloading, processing, uploading to gpu)
         - Settings panel (including the setting for "offline mode")
-        - Use typedvector where appropriate
         - Set up VAOs
     - Presentation / "marketing"
         - Title
@@ -45,6 +47,10 @@ Bonus:
         - Move multiple points
     - Customizable background
         - Color, textures, procedural
+    - Further optimization
+        - Draw LOD size histogram for various cases (maybe we see that in our worst case 90% of strokes are down to 3-4 points)
+        - If we see lots of very low detail strokes, precompute zoom level for 3,4,... points left
+        - Further investigate GC pauses on Mobile Firefox
 
 Bonus-bonus:
     - Actually infinite canvas (replace floats with something, some kind of fixed point scheme? chunks? multilevel scheme?)

client/aux.js

@@ -131,13 +131,24 @@ function tv_ensure(tv, capacity) {
 }
 
 function tv_ensure_by(tv, by) {
-    return tv_ensure(tv, tv.capacity + by);
+    return tv_ensure(tv, round_to_pow2(tv.size + by, 4096));
 }
 
 function tv_add(tv, item) {
     tv.data[tv.size++] = item;
 }
 
+function tv_pop(tv) {
+    const result = tv.data[tv.size - 1];
+    tv.size--;
+    return result;
+}
+
+function tv_append(tv, typedarray) {
+    tv.data.set(typedarray, tv.size);
+    tv.size += typedarray.length;
+}
+
 function tv_clear(tv) {
     tv.size = 0;
 }

client/bvh.js

@@ -159,12 +159,12 @@ function bvh_intersect_quad(bvh, quad, result_buffer) {
     if (bvh.root === null) {
         return;
     }
+    
+    tv_clear(bvh.traverse_stack);
+    tv_add(bvh.traverse_stack, bvh.root);
 
-    const stack = [bvh.root];
-    const result = [];
-
-    while (stack.length > 0) {
-        const node_index = stack.pop();
+    while (bvh.traverse_stack.size > 0) {
+        const node_index = tv_pop(bvh.traverse_stack);
         const node = bvh.nodes[node_index];
 
         if (!quads_intersect(node.bbox, quad)) {
@@ -175,7 +175,8 @@ function bvh_intersect_quad(bvh, quad, result_buffer) {
             result_buffer.data[result_buffer.count] = node.stroke_index;
             result_buffer.count += 1;
         } else {
-            stack.push(node.child1, node.child2);
+            tv_add(bvh.traverse_stack, node.child1);
+            tv_add(bvh.traverse_stack, node.child2);
         }
     }
 }
@@ -190,6 +191,8 @@ function bvh_clip(state, context) {
         context.clipped_indices.data = new Uint32Array(context.clipped_indices.cap);
     }
 
+    state.bvh.traverse_stack = tv_ensure(state.bvh.traverse_stack, round_to_pow2(state.stroke_count, 4096));
+
     context.clipped_indices.count = 0;
 
     const screen_topleft     = screen_to_canvas(state, {'x': 0, 'y': 0});

client/client_recv.js

@@ -80,11 +80,12 @@ function des_event(d, state = null) {
 
             const coords = des_f32array(d, point_count * 2);
 
-            event.coords_from = state.coordinates.count;
-            event.coords_to = state.coordinates.count + point_count * 2;
+            state.coordinates = tv_ensure_by(state.coordinates, coords.length);
 
-            state.coordinates.data.set(coords, state.coordinates.count);
-            state.coordinates.count += point_count * 2;
+            event.coords_from = state.coordinates.size;
+            event.coords_to = state.coordinates.size + point_count * 2;
+            
+            tv_append(state.coordinates, coords);
 
             event.stroke_id = stroke_id;
 
@@ -171,13 +172,10 @@ function handle_event(state, context, event, options = {}) {
         }
 
         case EVENT.STROKE: {
-            // TODO: @speed do proper local prediction, it's not that hard
-            
-            //if (event.user_id != state.me) {
-                geometry_clear_player(state, context, event.user_id);
-                need_draw = true;
-            //}
-          
+            // TODO: do not do this for my own strokes when we bake locally
+            geometry_clear_player(state, context, event.user_id);
+            need_draw = true;
+
             event.index = state.events.length;
 
             geometry_add_stroke(state, context, event, state.events.length, options.skip_bvh === true);
@@ -346,7 +344,7 @@ async function handle_message(state, context, d) {
             const user_count = des_u32(d);
             const total_points = des_u32(d);
 
-            state.coordinates.data = new Float32Array(round_to_pow2(total_points * 2, 4096));
+            state.coordinates = tv_create(Float32Array, round_to_pow2(total_points * 2, 4096));
 
             if (config.debug_print) console.debug(`${event_count} events in init`);
             

client/default.css

@@ -47,6 +47,10 @@ canvas.movemode.moving {
     cursor: grabbing;
 }
 
+canvas.mousemoving {
+    cursor: move;
+}
+
 .tools-wrapper {
     position: fixed;
     bottom: 0;

client/index.html

@@ -77,7 +77,7 @@
 
         <div class="tools-wrapper">
             <div class="tools">
-                <div class="tool active" data-tool="pencil"><img draggable="false" src="icons/draw.svg"></div>
+                <div class="tool active" data-tool="pencil"><img draggable="false" src="icons/pen.svg"></div>
                 <div class="tool" data-tool="ruler"><img draggable="false" src="icons/ruler.svg"></div>
                 <div class="tool" data-tool="eraser"><img draggable="false" src="icons/erase.svg"></div>
                 <div class="tool" data-tool="undo"><img draggable="false" src="icons/undo.svg"></div>

client/index.js

@@ -25,8 +25,8 @@ const config = {
     stroke_texture_size: 1024, // means no more than 1024^2 = 1M strokes in total (this is a LOT. HMH blackboard has like 80K)
     dynamic_stroke_texture_size: 128, // means no more than 128^2 = 16K dynamic strokes at once
     benchmark: {
-        zoom: 0.035,
-        offset: { x: 900, y: 400 },
+        zoom: 0.00003,
+        offset: { x: 1400, y: 400 },
         frames: 500,
     },
 };
@@ -170,10 +170,7 @@ function main() {
         'starting_index': 0,
         'total_points': 0,
 
-        'coordinates': {
-            'data': null,
-            'count': 0,
-        },
+        'coordinates': tv_create(Float32Array, 4096),
 
         'segments_from': {
             'data': null,
@@ -191,6 +188,7 @@ function main() {
             'nodes': [],
             'root': null,
             'pqueue': new MinQueue(1024),
+            'traverse_stack': tv_create(Uint32Array, 1024),
         },
 
         'tools': {
@@ -209,7 +207,6 @@ function main() {
         },
 
         'players': {},
-        'onscreen_segments': new Uint32Array(1024),
 
         'debug': {
             'red': false,

client/math.js

@@ -126,7 +126,7 @@ function process_stroke(state, zoom, stroke) {
 }
 
 function rdp_find_max2(points, start, end) {
-    const EPS = 0.5;
+    const EPS = 0.25;
 
     let result = -1;
     let max_dist = 0;
@@ -334,68 +334,3 @@ function quad_union(a, b) {
 function box_area(box) {
     return (box.x2 - box.x1) * (box.y2 - box.y1);
 }
-
-function segments_onscreen(state, context, do_clip) {
-    // TODO: handle stroke width
-    
-    if (state.onscreen_segments === null) {       
-        let total_points = 0;
-
-        for (const event of state.events) {
-            if (event.type === EVENT.STROKE && !event.deleted && event.points.length > 0) {
-                total_points += event.points.length - 1;
-            }
-        }
-    
-        if (total_points > 0) {
-            state.onscreen_segments = new Uint32Array(total_points * 6);
-        }
-    }
-
-    let at = 0;
-
-    const screen_topleft = screen_to_canvas(state, {'x': 0, 'y': 0});
-    const screen_bottomright = screen_to_canvas(state, {'x': context.canvas.width, 'y': context.canvas.height});
-
-    /*
-    screen_topleft.x += 300;
-    screen_topleft.y += 300;
-    screen_bottomright.x -= 300;
-    screen_bottomright.y -= 300;
-    */
-
-    const screen_topright = { 'x': screen_bottomright.x, 'y': screen_topleft.y };
-    const screen_bottomleft = { 'x': screen_topleft.x, 'y': screen_bottomright.y };
-    const screen = {'x1': screen_topleft.x, 'y1': screen_topleft.y, 'x2': screen_bottomright.x, 'y2': screen_bottomright.y};
-
-    let head = 0;
-
-    for (let i = 0; i < state.events.length; ++i) {
-        if (state.debug.limit_to && i >= state.debug.render_to) break;
-       
-        const event = state.events[i];
-
-        if (!(state.debug.limit_from && i < state.debug.render_from)) {
-            if (event.type === EVENT.STROKE && !event.deleted && event.points.length > 0) {
-                if (!do_clip || quads_intersect(screen, event.bbox)) {
-                    for (let j = 0; j < event.points.length - 1; ++j) {
-                        let base = head + j * 4;
-                        // We draw quads as [1, 2, 3, 4, 3, 2]
-                        state.onscreen_segments[at + 0] = base + 0;
-                        state.onscreen_segments[at + 1] = base + 1;
-                        state.onscreen_segments[at + 2] = base + 2;
-                        state.onscreen_segments[at + 3] = base + 3;
-                        state.onscreen_segments[at + 4] = base + 2;
-                        state.onscreen_segments[at + 5] = base + 1;
-
-                        at += 6;
-                    }
-                }
-            }
-        }
-
-        head += (event.points.length - 1) * 4;
-    }
-
-    return at;
-}

client/webgl_draw.js

@@ -72,74 +72,78 @@ function draw(state, context) {
     gl.useProgram(context.programs['sdf'].main);
 
     bvh_clip(state, context); 
+    
     const segment_count = geometry_write_instances(state, context);
     const dynamic_segment_count = context.dynamic_segment_count;
     const dynamic_stroke_count = context.dynamic_stroke_count;
 
     // "Static" data upload
-    gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_instance']);
-    gl.bufferData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4 + context.instance_data_ids.size * 4, gl.STREAM_DRAW);
-    gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.instance_data_points)); 
-    gl.bufferSubData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4, tv_data(context.instance_data_ids));
-    gl.bindTexture(gl.TEXTURE_2D, context.textures['stroke_data']);
-    // TODO: this is stable data, only upload new strokes as they arrive
-    upload_square_rgba16ui_texture(gl, context.stroke_data, config.stroke_texture_size);
-
-    gl.uniform2f(locations['u_res'], context.canvas.width, context.canvas.height);
-    gl.uniform2f(locations['u_scale'], state.canvas.zoom, state.canvas.zoom);
-    gl.uniform2f(locations['u_translation'], state.canvas.offset.x, state.canvas.offset.y);
-    gl.uniform1i(locations['u_stroke_count'], state.events.length);
-    gl.uniform1i(locations['u_debug_mode'], state.debug.red);
-    gl.uniform1i(locations['u_stroke_data'], 0);
-    gl.uniform1i(locations['u_stroke_texture_size'], config.stroke_texture_size);
-
-    gl.enableVertexAttribArray(locations['a_a']);
-    gl.enableVertexAttribArray(locations['a_b']);
-    gl.enableVertexAttribArray(locations['a_stroke_id']);
-    
-    // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values)
-    gl.vertexAttribPointer(locations['a_a'],          2, gl.FLOAT,         false, 2 * 4, 0);
-    gl.vertexAttribPointer(locations['a_b'],          2, gl.FLOAT,         false, 2 * 4, 2 * 4);
-    gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT,                  4, context.instance_data_points.size * 4);
-
-    gl.vertexAttribDivisor(locations['a_a'], 1);
-    gl.vertexAttribDivisor(locations['a_b'], 1);
-    gl.vertexAttribDivisor(locations['a_stroke_id'], 1);
-    
-    // Static draw (everything already bound)
-    gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, segment_count); 
-    
+    if (segment_count > 0) {
+        gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_instance']);
+        gl.bufferData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4 + context.instance_data_ids.size * 4, gl.STREAM_DRAW);
+        gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.instance_data_points)); 
+        gl.bufferSubData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4, tv_data(context.instance_data_ids));
+        gl.bindTexture(gl.TEXTURE_2D, context.textures['stroke_data']);
+        upload_square_rgba16ui_texture(gl, context.stroke_data, config.stroke_texture_size);
+
+        gl.uniform2f(locations['u_res'], context.canvas.width, context.canvas.height);
+        gl.uniform2f(locations['u_scale'], state.canvas.zoom, state.canvas.zoom);
+        gl.uniform2f(locations['u_translation'], state.canvas.offset.x, state.canvas.offset.y);
+        gl.uniform1i(locations['u_stroke_count'], state.events.length);
+        gl.uniform1i(locations['u_debug_mode'], state.debug.red);
+        gl.uniform1i(locations['u_stroke_data'], 0);
+        gl.uniform1i(locations['u_stroke_texture_size'], config.stroke_texture_size);
+
+        gl.enableVertexAttribArray(locations['a_a']);
+        gl.enableVertexAttribArray(locations['a_b']);
+        gl.enableVertexAttribArray(locations['a_stroke_id']);
+
+        // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values)
+        gl.vertexAttribPointer(locations['a_a'],          2, gl.FLOAT,         false, 2 * 4, 0);
+        gl.vertexAttribPointer(locations['a_b'],          2, gl.FLOAT,         false, 2 * 4, 2 * 4);
+        gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT,                  4, context.instance_data_points.size * 4);
+
+        gl.vertexAttribDivisor(locations['a_a'], 1);
+        gl.vertexAttribDivisor(locations['a_b'], 1);
+        gl.vertexAttribDivisor(locations['a_stroke_id'], 1);
+
+        // Static draw (everything already bound)
+        gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, segment_count); 
+    }
+
     // Dynamic strokes should be drawn above static strokes
     gl.clear(gl.DEPTH_BUFFER_BIT);
 
     // Dynamic draw (strokes currently being drawn)
-    gl.uniform1i(locations['u_stroke_count'], dynamic_stroke_count);
-    gl.uniform1i(locations['u_stroke_data'], 0);
-    gl.uniform1i(locations['u_stroke_texture_size'], config.dynamic_stroke_texture_size);
-    
-    gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_dynamic_instance']);
-
-    // Dynamic data upload
-    gl.bufferData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4 + context.dynamic_instance_ids.size * 4, gl.STREAM_DRAW);
-    gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.dynamic_instance_points)); 
-    gl.bufferSubData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4, tv_data(context.dynamic_instance_ids));
-    gl.bindTexture(gl.TEXTURE_2D, context.textures['dynamic_stroke_data']);
-    upload_square_rgba16ui_texture(gl, context.dynamic_stroke_data, config.dynamic_stroke_texture_size);
-
-    gl.enableVertexAttribArray(locations['a_a']);
-    gl.enableVertexAttribArray(locations['a_b']);
-    gl.enableVertexAttribArray(locations['a_stroke_id']);
-    
-    // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values)
-    gl.vertexAttribPointer(locations['a_a'],          2, gl.FLOAT,         false, 2 * 4, 0);
-    gl.vertexAttribPointer(locations['a_b'],          2, gl.FLOAT,         false, 2 * 4, 2 * 4);
-    gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT,                  4, context.dynamic_instance_points.size * 4);
-
-    gl.vertexAttribDivisor(locations['a_a'], 1);
-    gl.vertexAttribDivisor(locations['a_b'], 1);
-    gl.vertexAttribDivisor(locations['a_stroke_id'], 1);
-
-    gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, dynamic_segment_count);
+    if (dynamic_segment_count > 0) {
+        gl.uniform1i(locations['u_stroke_count'], dynamic_stroke_count);
+        gl.uniform1i(locations['u_stroke_data'], 0);
+        gl.uniform1i(locations['u_stroke_texture_size'], config.dynamic_stroke_texture_size);
+
+        gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_dynamic_instance']);
+
+        // Dynamic data upload
+        gl.bufferData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4 + context.dynamic_instance_ids.size * 4, gl.STREAM_DRAW);
+        gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.dynamic_instance_points)); 
+        gl.bufferSubData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4, tv_data(context.dynamic_instance_ids));
+        gl.bindTexture(gl.TEXTURE_2D, context.textures['dynamic_stroke_data']);
+        upload_square_rgba16ui_texture(gl, context.dynamic_stroke_data, config.dynamic_stroke_texture_size);
+
+        gl.enableVertexAttribArray(locations['a_a']);
+        gl.enableVertexAttribArray(locations['a_b']);
+        gl.enableVertexAttribArray(locations['a_stroke_id']);
+
+        // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values)
+        gl.vertexAttribPointer(locations['a_a'],          2, gl.FLOAT,         false, 2 * 4, 0);
+        gl.vertexAttribPointer(locations['a_b'],          2, gl.FLOAT,         false, 2 * 4, 2 * 4);
+        gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT,                  4, context.dynamic_instance_points.size * 4);
+
+        gl.vertexAttribDivisor(locations['a_a'], 1);
+        gl.vertexAttribDivisor(locations['a_b'], 1);
+        gl.vertexAttribDivisor(locations['a_stroke_id'], 1);
+
+        gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, dynamic_segment_count);
+    }
 
     document.getElementById('debug-stats').innerHTML = `
     <span>Segments onscreen: ${segment_count}</span>

client/webgl_geometry.js

@@ -82,8 +82,8 @@ function geometry_write_instances(state, context) {
         state.segments_from.data = new Uint32Array(state.segments_from.cap);
     }
 
-    if (state.segments.cap < state.coordinates.count / 2) {
-        state.segments.cap = round_to_pow2(state.coordinates.count, 4096); 
+    if (state.segments.cap < state.coordinates.size / 2) {
+        state.segments.cap = round_to_pow2(state.coordinates.size, 4096); 
         state.segments.data = new Uint32Array(state.segments.cap);
     }
     
@@ -295,8 +295,6 @@ function geometry_clear_player(state, context, player_id) {
 }
 
 function add_image(context, image_id, bitmap, p) {
-    return; // TODO
-
     const x = p.x;
     const y = p.y;
     const gl = context.gl;

client/webgl_listeners.js

@@ -173,7 +173,7 @@ function mousedown(e, state, context) {
         return;
     }
 
-    if (e.button !== 0) {
+    if (e.button !== 0 && e.button !== 1) {
         return;
     }
 
@@ -186,9 +186,14 @@ function mousedown(e, state, context) {
         }
     }
 
-    if (state.spacedown) {
+    if (state.spacedown || e.button === 1) {
         state.moving = true;
         context.canvas.classList.add('moving');
+
+        if (e.button === 1) {
+            context.canvas.classList.add('mousemoving');
+        }
+
         return;
     }
 
@@ -258,7 +263,7 @@ function mousemove(e, state, context) {
 }
 
 function mouseup(e, state, context) {
-    if (e.button !== 0) {
+    if (e.button !== 0 && e.button !== 1) {
         return;
     }
 
@@ -269,9 +274,14 @@ function mouseup(e, state, context) {
         return;
     }
 
-    if (state.moving) {
+    if (state.moving || e.button === 1) {
         state.moving = false;
         context.canvas.classList.remove('moving');
+        
+        if (e.button === 1) {
+            context.canvas.classList.remove('mousemoving');
+        }
+
         return;
     }
 
@@ -279,9 +289,12 @@ function mouseup(e, state, context) {
         const stroke = geometry_prepare_stroke(state);
 
         if (stroke) {
+            // TODO: be able to add a baked stroke locally
+
+
             //geometry_add_stroke(state, context, stroke, 0);
             queue_event(state, stroke_event(state));
-            geometry_clear_player(state, context, state.me);
+            //geometry_clear_player(state, context, state.me);
             schedule_draw(state, context);
         }
 
@@ -479,10 +492,8 @@ function touchend(e, state, context) {
                 
                 const stroke = geometry_prepare_stroke(state);
 
-                if (false && stroke) { // TODO: FIX!
-                    geometry_add_stroke(state, context, stroke, 0); // TODO: stroke index
+                if (stroke) {                    
                     queue_event(state, stroke_event(state));
-                    geometry_clear_player(state, context, state.me);
                     schedule_draw(state, context);
                 }
 

server/recv.js

@@ -39,7 +39,7 @@ async function recv_syn(d, session) {
             events.push(event);
         }
     }
-
+    
     desks[session.desk_id].sn += we_expect;
     desks[session.desk_id].events.push(...events);
     session.lsn = lsn;

server/send.js

@@ -231,7 +231,7 @@ async function sync_session(session_id) {
         const event = desk.events[desk.events.length - 1 - i];
         ser.event(s, event);
     }
-
+    
     if (config.DEBUG_PRINT) console.log(`syn ${desk.sn} out`);
 
     await session.ws.send(s.buffer);

server/storage.js

@@ -124,6 +124,10 @@ export function startup() {
         desks[event.desk_id].events.push(event);
     }
 
+    for (const desk of stored_desks) {
+        desk.sn = desk.events.length;
+    }
+
     for (const session of stored_sessions) {
         session.state = SESSION.CLOSED;
         session.ws = null;