diff --git a/README.md b/README.md index c518925..c21a7de 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,20 @@ Release: + Benchmark harness + Reuse points, pack "nodraw" in high bit of stroke id (probably have at least one more bit, so up to 4 flag configurations) + Draw dynamic data (strokes in progress) + - Z-prepass fringe bug (also, when do we enable the prepass?) - Textured quads (pictures, code already written in older version) - Resize and move pictures (draw handles) - - Z-prepass fringe bug (also, when do we enable the prepass?) + - Debug - Restore ability to limit event range - - Only upload stroke data to texture as it arrives (texSubImage2D) - - Listeners/events - - Investigate skipped inputs on mobile (panning, zooming) + * Listeners/events/multiplayer + + Fix multiplayer LUL + + Fix blinking own stroke inbetween SYN->server and SYN->client + + Drag with mouse button 3 + + Investigate skipped inputs on mobile (panning, zooming) [Events were not actually getting skipped. The stroke previews were just not being drawn] - Save events to indexeddb (as some kind of a blob), restore on reconnect and page reload - - Separate events and other data clearly (events are self-contained, other data is temporal/non-vital) - Do NOT use session id as player id LUL - Local prediction for tools! - - Drag with mouse button 3 + - Be able to have multiple "current" strokes per player. In case of bad internet this can happen! - Missing features I do not consider bonus - Eraser - Line drawing @@ -24,9 +26,9 @@ Release: - Undo/redo - Dynamic svg cursor to represent the brush - Polish + * Use typedvector where appropriate - Show what's happening while the desk is loading (downloading, processing, uploading to gpu) - Settings panel (including the setting for "offline mode") - - Use typedvector where appropriate - Set up VAOs - Presentation / "marketing" - Title @@ -45,6 +47,10 @@ Bonus: - Move multiple points - Customizable background - Color, textures, procedural + - Further optimization + - Draw LOD size histogram for various cases (maybe we see that in our worst case 90% of strokes are down to 3-4 points) + - If we see lots of very low detail strokes, precompute zoom level for 3,4,... points left + - Further investigate GC pauses on Mobile Firefox Bonus-bonus: - Actually infinite canvas (replace floats with something, some kind of fixed point scheme? chunks? multilevel scheme?) diff --git a/client/aux.js b/client/aux.js index 2d533fb..e6718db 100644 --- a/client/aux.js +++ b/client/aux.js @@ -131,13 +131,24 @@ function tv_ensure(tv, capacity) { } function tv_ensure_by(tv, by) { - return tv_ensure(tv, tv.capacity + by); + return tv_ensure(tv, round_to_pow2(tv.size + by, 4096)); } function tv_add(tv, item) { tv.data[tv.size++] = item; } +function tv_pop(tv) { + const result = tv.data[tv.size - 1]; + tv.size--; + return result; +} + +function tv_append(tv, typedarray) { + tv.data.set(typedarray, tv.size); + tv.size += typedarray.length; +} + function tv_clear(tv) { tv.size = 0; } diff --git a/client/bvh.js b/client/bvh.js index f1f9b61..e16f7de 100644 --- a/client/bvh.js +++ b/client/bvh.js @@ -159,12 +159,12 @@ function bvh_intersect_quad(bvh, quad, result_buffer) { if (bvh.root === null) { return; } + + tv_clear(bvh.traverse_stack); + tv_add(bvh.traverse_stack, bvh.root); - const stack = [bvh.root]; - const result = []; - - while (stack.length > 0) { - const node_index = stack.pop(); + while (bvh.traverse_stack.size > 0) { + const node_index = tv_pop(bvh.traverse_stack); const node = bvh.nodes[node_index]; if (!quads_intersect(node.bbox, quad)) { @@ -175,7 +175,8 @@ function bvh_intersect_quad(bvh, quad, result_buffer) { result_buffer.data[result_buffer.count] = node.stroke_index; result_buffer.count += 1; } else { - stack.push(node.child1, node.child2); + tv_add(bvh.traverse_stack, node.child1); + tv_add(bvh.traverse_stack, node.child2); } } } @@ -190,6 +191,8 @@ function bvh_clip(state, context) { context.clipped_indices.data = new Uint32Array(context.clipped_indices.cap); } + state.bvh.traverse_stack = tv_ensure(state.bvh.traverse_stack, round_to_pow2(state.stroke_count, 4096)); + context.clipped_indices.count = 0; const screen_topleft = screen_to_canvas(state, {'x': 0, 'y': 0}); diff --git a/client/client_recv.js b/client/client_recv.js index 2a178ae..1987e8b 100644 --- a/client/client_recv.js +++ b/client/client_recv.js @@ -80,11 +80,12 @@ function des_event(d, state = null) { const coords = des_f32array(d, point_count * 2); - event.coords_from = state.coordinates.count; - event.coords_to = state.coordinates.count + point_count * 2; + state.coordinates = tv_ensure_by(state.coordinates, coords.length); - state.coordinates.data.set(coords, state.coordinates.count); - state.coordinates.count += point_count * 2; + event.coords_from = state.coordinates.size; + event.coords_to = state.coordinates.size + point_count * 2; + + tv_append(state.coordinates, coords); event.stroke_id = stroke_id; @@ -171,13 +172,10 @@ function handle_event(state, context, event, options = {}) { } case EVENT.STROKE: { - // TODO: @speed do proper local prediction, it's not that hard - - //if (event.user_id != state.me) { - geometry_clear_player(state, context, event.user_id); - need_draw = true; - //} - + // TODO: do not do this for my own strokes when we bake locally + geometry_clear_player(state, context, event.user_id); + need_draw = true; + event.index = state.events.length; geometry_add_stroke(state, context, event, state.events.length, options.skip_bvh === true); @@ -346,7 +344,7 @@ async function handle_message(state, context, d) { const user_count = des_u32(d); const total_points = des_u32(d); - state.coordinates.data = new Float32Array(round_to_pow2(total_points * 2, 4096)); + state.coordinates = tv_create(Float32Array, round_to_pow2(total_points * 2, 4096)); if (config.debug_print) console.debug(`${event_count} events in init`); diff --git a/client/default.css b/client/default.css index a6927fc..147c49a 100644 --- a/client/default.css +++ b/client/default.css @@ -47,6 +47,10 @@ canvas.movemode.moving { cursor: grabbing; } +canvas.mousemoving { + cursor: move; +} + .tools-wrapper { position: fixed; bottom: 0; diff --git a/client/icons/draw.svg b/client/icons/pen.svg similarity index 100% rename from client/icons/draw.svg rename to client/icons/pen.svg diff --git a/client/index.html b/client/index.html index 341d140..3ceb5c8 100644 --- a/client/index.html +++ b/client/index.html @@ -77,7 +77,7 @@
-
+
diff --git a/client/index.js b/client/index.js index 22de0e0..335573e 100644 --- a/client/index.js +++ b/client/index.js @@ -25,8 +25,8 @@ const config = { stroke_texture_size: 1024, // means no more than 1024^2 = 1M strokes in total (this is a LOT. HMH blackboard has like 80K) dynamic_stroke_texture_size: 128, // means no more than 128^2 = 16K dynamic strokes at once benchmark: { - zoom: 0.035, - offset: { x: 900, y: 400 }, + zoom: 0.00003, + offset: { x: 1400, y: 400 }, frames: 500, }, }; @@ -170,10 +170,7 @@ function main() { 'starting_index': 0, 'total_points': 0, - 'coordinates': { - 'data': null, - 'count': 0, - }, + 'coordinates': tv_create(Float32Array, 4096), 'segments_from': { 'data': null, @@ -191,6 +188,7 @@ function main() { 'nodes': [], 'root': null, 'pqueue': new MinQueue(1024), + 'traverse_stack': tv_create(Uint32Array, 1024), }, 'tools': { @@ -209,7 +207,6 @@ function main() { }, 'players': {}, - 'onscreen_segments': new Uint32Array(1024), 'debug': { 'red': false, diff --git a/client/math.js b/client/math.js index 62fd7b5..f01247a 100644 --- a/client/math.js +++ b/client/math.js @@ -126,7 +126,7 @@ function process_stroke(state, zoom, stroke) { } function rdp_find_max2(points, start, end) { - const EPS = 0.5; + const EPS = 0.25; let result = -1; let max_dist = 0; @@ -334,68 +334,3 @@ function quad_union(a, b) { function box_area(box) { return (box.x2 - box.x1) * (box.y2 - box.y1); } - -function segments_onscreen(state, context, do_clip) { - // TODO: handle stroke width - - if (state.onscreen_segments === null) { - let total_points = 0; - - for (const event of state.events) { - if (event.type === EVENT.STROKE && !event.deleted && event.points.length > 0) { - total_points += event.points.length - 1; - } - } - - if (total_points > 0) { - state.onscreen_segments = new Uint32Array(total_points * 6); - } - } - - let at = 0; - - const screen_topleft = screen_to_canvas(state, {'x': 0, 'y': 0}); - const screen_bottomright = screen_to_canvas(state, {'x': context.canvas.width, 'y': context.canvas.height}); - - /* - screen_topleft.x += 300; - screen_topleft.y += 300; - screen_bottomright.x -= 300; - screen_bottomright.y -= 300; - */ - - const screen_topright = { 'x': screen_bottomright.x, 'y': screen_topleft.y }; - const screen_bottomleft = { 'x': screen_topleft.x, 'y': screen_bottomright.y }; - const screen = {'x1': screen_topleft.x, 'y1': screen_topleft.y, 'x2': screen_bottomright.x, 'y2': screen_bottomright.y}; - - let head = 0; - - for (let i = 0; i < state.events.length; ++i) { - if (state.debug.limit_to && i >= state.debug.render_to) break; - - const event = state.events[i]; - - if (!(state.debug.limit_from && i < state.debug.render_from)) { - if (event.type === EVENT.STROKE && !event.deleted && event.points.length > 0) { - if (!do_clip || quads_intersect(screen, event.bbox)) { - for (let j = 0; j < event.points.length - 1; ++j) { - let base = head + j * 4; - // We draw quads as [1, 2, 3, 4, 3, 2] - state.onscreen_segments[at + 0] = base + 0; - state.onscreen_segments[at + 1] = base + 1; - state.onscreen_segments[at + 2] = base + 2; - state.onscreen_segments[at + 3] = base + 3; - state.onscreen_segments[at + 4] = base + 2; - state.onscreen_segments[at + 5] = base + 1; - - at += 6; - } - } - } - } - - head += (event.points.length - 1) * 4; - } - - return at; -} diff --git a/client/webgl_draw.js b/client/webgl_draw.js index 80ff898..892a60b 100644 --- a/client/webgl_draw.js +++ b/client/webgl_draw.js @@ -72,74 +72,78 @@ function draw(state, context) { gl.useProgram(context.programs['sdf'].main); bvh_clip(state, context); + const segment_count = geometry_write_instances(state, context); const dynamic_segment_count = context.dynamic_segment_count; const dynamic_stroke_count = context.dynamic_stroke_count; // "Static" data upload - gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_instance']); - gl.bufferData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4 + context.instance_data_ids.size * 4, gl.STREAM_DRAW); - gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.instance_data_points)); - gl.bufferSubData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4, tv_data(context.instance_data_ids)); - gl.bindTexture(gl.TEXTURE_2D, context.textures['stroke_data']); - // TODO: this is stable data, only upload new strokes as they arrive - upload_square_rgba16ui_texture(gl, context.stroke_data, config.stroke_texture_size); - - gl.uniform2f(locations['u_res'], context.canvas.width, context.canvas.height); - gl.uniform2f(locations['u_scale'], state.canvas.zoom, state.canvas.zoom); - gl.uniform2f(locations['u_translation'], state.canvas.offset.x, state.canvas.offset.y); - gl.uniform1i(locations['u_stroke_count'], state.events.length); - gl.uniform1i(locations['u_debug_mode'], state.debug.red); - gl.uniform1i(locations['u_stroke_data'], 0); - gl.uniform1i(locations['u_stroke_texture_size'], config.stroke_texture_size); - - gl.enableVertexAttribArray(locations['a_a']); - gl.enableVertexAttribArray(locations['a_b']); - gl.enableVertexAttribArray(locations['a_stroke_id']); - - // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values) - gl.vertexAttribPointer(locations['a_a'], 2, gl.FLOAT, false, 2 * 4, 0); - gl.vertexAttribPointer(locations['a_b'], 2, gl.FLOAT, false, 2 * 4, 2 * 4); - gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT, 4, context.instance_data_points.size * 4); - - gl.vertexAttribDivisor(locations['a_a'], 1); - gl.vertexAttribDivisor(locations['a_b'], 1); - gl.vertexAttribDivisor(locations['a_stroke_id'], 1); - - // Static draw (everything already bound) - gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, segment_count); - + if (segment_count > 0) { + gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_instance']); + gl.bufferData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4 + context.instance_data_ids.size * 4, gl.STREAM_DRAW); + gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.instance_data_points)); + gl.bufferSubData(gl.ARRAY_BUFFER, context.instance_data_points.size * 4, tv_data(context.instance_data_ids)); + gl.bindTexture(gl.TEXTURE_2D, context.textures['stroke_data']); + upload_square_rgba16ui_texture(gl, context.stroke_data, config.stroke_texture_size); + + gl.uniform2f(locations['u_res'], context.canvas.width, context.canvas.height); + gl.uniform2f(locations['u_scale'], state.canvas.zoom, state.canvas.zoom); + gl.uniform2f(locations['u_translation'], state.canvas.offset.x, state.canvas.offset.y); + gl.uniform1i(locations['u_stroke_count'], state.events.length); + gl.uniform1i(locations['u_debug_mode'], state.debug.red); + gl.uniform1i(locations['u_stroke_data'], 0); + gl.uniform1i(locations['u_stroke_texture_size'], config.stroke_texture_size); + + gl.enableVertexAttribArray(locations['a_a']); + gl.enableVertexAttribArray(locations['a_b']); + gl.enableVertexAttribArray(locations['a_stroke_id']); + + // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values) + gl.vertexAttribPointer(locations['a_a'], 2, gl.FLOAT, false, 2 * 4, 0); + gl.vertexAttribPointer(locations['a_b'], 2, gl.FLOAT, false, 2 * 4, 2 * 4); + gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT, 4, context.instance_data_points.size * 4); + + gl.vertexAttribDivisor(locations['a_a'], 1); + gl.vertexAttribDivisor(locations['a_b'], 1); + gl.vertexAttribDivisor(locations['a_stroke_id'], 1); + + // Static draw (everything already bound) + gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, segment_count); + } + // Dynamic strokes should be drawn above static strokes gl.clear(gl.DEPTH_BUFFER_BIT); // Dynamic draw (strokes currently being drawn) - gl.uniform1i(locations['u_stroke_count'], dynamic_stroke_count); - gl.uniform1i(locations['u_stroke_data'], 0); - gl.uniform1i(locations['u_stroke_texture_size'], config.dynamic_stroke_texture_size); - - gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_dynamic_instance']); - - // Dynamic data upload - gl.bufferData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4 + context.dynamic_instance_ids.size * 4, gl.STREAM_DRAW); - gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.dynamic_instance_points)); - gl.bufferSubData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4, tv_data(context.dynamic_instance_ids)); - gl.bindTexture(gl.TEXTURE_2D, context.textures['dynamic_stroke_data']); - upload_square_rgba16ui_texture(gl, context.dynamic_stroke_data, config.dynamic_stroke_texture_size); - - gl.enableVertexAttribArray(locations['a_a']); - gl.enableVertexAttribArray(locations['a_b']); - gl.enableVertexAttribArray(locations['a_stroke_id']); - - // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values) - gl.vertexAttribPointer(locations['a_a'], 2, gl.FLOAT, false, 2 * 4, 0); - gl.vertexAttribPointer(locations['a_b'], 2, gl.FLOAT, false, 2 * 4, 2 * 4); - gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT, 4, context.dynamic_instance_points.size * 4); - - gl.vertexAttribDivisor(locations['a_a'], 1); - gl.vertexAttribDivisor(locations['a_b'], 1); - gl.vertexAttribDivisor(locations['a_stroke_id'], 1); - - gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, dynamic_segment_count); + if (dynamic_segment_count > 0) { + gl.uniform1i(locations['u_stroke_count'], dynamic_stroke_count); + gl.uniform1i(locations['u_stroke_data'], 0); + gl.uniform1i(locations['u_stroke_texture_size'], config.dynamic_stroke_texture_size); + + gl.bindBuffer(gl.ARRAY_BUFFER, buffers['b_dynamic_instance']); + + // Dynamic data upload + gl.bufferData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4 + context.dynamic_instance_ids.size * 4, gl.STREAM_DRAW); + gl.bufferSubData(gl.ARRAY_BUFFER, 0, tv_data(context.dynamic_instance_points)); + gl.bufferSubData(gl.ARRAY_BUFFER, context.dynamic_instance_points.size * 4, tv_data(context.dynamic_instance_ids)); + gl.bindTexture(gl.TEXTURE_2D, context.textures['dynamic_stroke_data']); + upload_square_rgba16ui_texture(gl, context.dynamic_stroke_data, config.dynamic_stroke_texture_size); + + gl.enableVertexAttribArray(locations['a_a']); + gl.enableVertexAttribArray(locations['a_b']); + gl.enableVertexAttribArray(locations['a_stroke_id']); + + // Points (a, b) and stroke ids are stored in separate cpu buffers so that points can be reused (look at stride and offset values) + gl.vertexAttribPointer(locations['a_a'], 2, gl.FLOAT, false, 2 * 4, 0); + gl.vertexAttribPointer(locations['a_b'], 2, gl.FLOAT, false, 2 * 4, 2 * 4); + gl.vertexAttribIPointer(locations['a_stroke_id'], 1, gl.INT, 4, context.dynamic_instance_points.size * 4); + + gl.vertexAttribDivisor(locations['a_a'], 1); + gl.vertexAttribDivisor(locations['a_b'], 1); + gl.vertexAttribDivisor(locations['a_stroke_id'], 1); + + gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, dynamic_segment_count); + } document.getElementById('debug-stats').innerHTML = ` Segments onscreen: ${segment_count} diff --git a/client/webgl_geometry.js b/client/webgl_geometry.js index 20649cc..af33102 100644 --- a/client/webgl_geometry.js +++ b/client/webgl_geometry.js @@ -82,8 +82,8 @@ function geometry_write_instances(state, context) { state.segments_from.data = new Uint32Array(state.segments_from.cap); } - if (state.segments.cap < state.coordinates.count / 2) { - state.segments.cap = round_to_pow2(state.coordinates.count, 4096); + if (state.segments.cap < state.coordinates.size / 2) { + state.segments.cap = round_to_pow2(state.coordinates.size, 4096); state.segments.data = new Uint32Array(state.segments.cap); } @@ -295,8 +295,6 @@ function geometry_clear_player(state, context, player_id) { } function add_image(context, image_id, bitmap, p) { - return; // TODO - const x = p.x; const y = p.y; const gl = context.gl; diff --git a/client/webgl_listeners.js b/client/webgl_listeners.js index 50bfe89..fac2db9 100644 --- a/client/webgl_listeners.js +++ b/client/webgl_listeners.js @@ -173,7 +173,7 @@ function mousedown(e, state, context) { return; } - if (e.button !== 0) { + if (e.button !== 0 && e.button !== 1) { return; } @@ -186,9 +186,14 @@ function mousedown(e, state, context) { } } - if (state.spacedown) { + if (state.spacedown || e.button === 1) { state.moving = true; context.canvas.classList.add('moving'); + + if (e.button === 1) { + context.canvas.classList.add('mousemoving'); + } + return; } @@ -258,7 +263,7 @@ function mousemove(e, state, context) { } function mouseup(e, state, context) { - if (e.button !== 0) { + if (e.button !== 0 && e.button !== 1) { return; } @@ -269,9 +274,14 @@ function mouseup(e, state, context) { return; } - if (state.moving) { + if (state.moving || e.button === 1) { state.moving = false; context.canvas.classList.remove('moving'); + + if (e.button === 1) { + context.canvas.classList.remove('mousemoving'); + } + return; } @@ -279,9 +289,12 @@ function mouseup(e, state, context) { const stroke = geometry_prepare_stroke(state); if (stroke) { + // TODO: be able to add a baked stroke locally + + //geometry_add_stroke(state, context, stroke, 0); queue_event(state, stroke_event(state)); - geometry_clear_player(state, context, state.me); + //geometry_clear_player(state, context, state.me); schedule_draw(state, context); } @@ -479,10 +492,8 @@ function touchend(e, state, context) { const stroke = geometry_prepare_stroke(state); - if (false && stroke) { // TODO: FIX! - geometry_add_stroke(state, context, stroke, 0); // TODO: stroke index + if (stroke) { queue_event(state, stroke_event(state)); - geometry_clear_player(state, context, state.me); schedule_draw(state, context); } diff --git a/server/recv.js b/server/recv.js index 2281fde..268bc87 100644 --- a/server/recv.js +++ b/server/recv.js @@ -39,7 +39,7 @@ async function recv_syn(d, session) { events.push(event); } } - + desks[session.desk_id].sn += we_expect; desks[session.desk_id].events.push(...events); session.lsn = lsn; diff --git a/server/send.js b/server/send.js index 1ff6157..296e71e 100644 --- a/server/send.js +++ b/server/send.js @@ -231,7 +231,7 @@ async function sync_session(session_id) { const event = desk.events[desk.events.length - 1 - i]; ser.event(s, event); } - + if (config.DEBUG_PRINT) console.log(`syn ${desk.sn} out`); await session.ws.send(s.buffer); diff --git a/server/storage.js b/server/storage.js index de99b2b..03d17a0 100644 --- a/server/storage.js +++ b/server/storage.js @@ -124,6 +124,10 @@ export function startup() { desks[event.desk_id].events.push(event); } + for (const desk of stored_desks) { + desk.sn = desk.events.length; + } + for (const session of stored_sessions) { session.state = SESSION.CLOSED; session.ws = null;