From 21aecb7d080836dacdb19c1e6a6fd106046df2c7 Mon Sep 17 00:00:00 2001 From: "A.Olokhtonov" Date: Mon, 17 Jun 2024 01:07:37 +0300 Subject: [PATCH] Make image move and image scale work in multiplayer. Add width and height to image event and fix late-arriving bitmaps breaking things --- client/aux.js | 9 ++++-- client/client_recv.js | 68 ++++++++++++++++++++++++++++++--------- client/client_send.js | 26 ++++++++++++++- client/index.js | 1 + client/webgl_geometry.js | 48 ++++++++++++++++----------- client/webgl_listeners.js | 4 +++ server/deserializer.js | 18 ++++++++++- server/enums.js | 1 + server/milton.js | 17 +++++++++- server/recv.js | 7 ++++ server/send.js | 11 ++++++- server/serializer.js | 18 ++++++++++- server/storage.js | 5 ++- 13 files changed, 191 insertions(+), 42 deletions(-) diff --git a/client/aux.js b/client/aux.js index 399af9a..86d22fb 100644 --- a/client/aux.js +++ b/client/aux.js @@ -27,7 +27,7 @@ async function insert_image(state, context, file) { if (resp.ok) { const image_id = await resp.text(); - const event = image_event(image_id, canvasp.x, canvasp.y); + const event = image_event(image_id, canvasp.x, canvasp.y, bitmap.width. bitmap.height); await queue_event(state, event); } } @@ -81,7 +81,12 @@ function event_size(event) { case EVENT.IMAGE: case EVENT.IMAGE_MOVE: { - size += 4 + 4 + 4; // file id + x + y + size += 4 + 4 + 4 + 4 + 4; // file id + x + y + width + height + break; + } + + case EVENT.IMAGE_SCALE: { + size += 4 + 4 + 4 + 4; // file_id + corner + x + y break; } diff --git a/client/client_recv.js b/client/client_recv.js index c2a9c61..a22328e 100644 --- a/client/client_recv.js +++ b/client/client_recv.js @@ -127,7 +127,15 @@ function des_event(d, state = null) { break; } - case EVENT.IMAGE: + case EVENT.IMAGE: { + event.image_id = des_u32(d); + event.x = des_f32(d); + event.y = des_f32(d); + event.width = des_u32(d); + event.height = des_u32(d); + break; + } + case EVENT.IMAGE_MOVE: { event.image_id = des_u32(d); event.x = des_f32(d); @@ -135,6 +143,14 @@ function des_event(d, state = null) { break; } + case EVENT.IMAGE_SCALE: { + event.image_id = des_u32(d); + event.corner = des_u32(d); + event.x = des_f32(d); + event.y = des_f32(d); + break; + } + case EVENT.UNDO: case EVENT.REDO: { break; @@ -350,6 +366,14 @@ function handle_event(state, context, event, options = {}) { break; } else if (other_event.type === EVENT.UNDO) { // do not undo an undo, we are not Notepad + } else if (other_event.type === EVENT.IMAGE_MOVE) { + // TODO + console.log('TODO: undo image scale'); + break; + } else if (other_event.type === EVENT.IMAGE_SCALE) { + // TODO + console.log('TODO: undo image scale'); + break; } else { console.error('cant undo event type', other_event.type); break; @@ -361,21 +385,24 @@ function handle_event(state, context, event, options = {}) { } case EVENT.IMAGE: { + const p = {'x': event.x, 'y': event.y}; + geometry_add_dummy_stroke(context); + add_image(context, event.image_id, null, p, event.width, event.height); try { (async () => { const url = config.image_url + event.image_id; const r = await fetch(config.image_url + event.image_id); const blob = await r.blob(); + + // NOTE: this will resolve when bitmap is ready, which will be much later const bitmap = await createImageBitmap(blob); - const p = {'x': event.x, 'y': event.y}; event.width = bitmap.width; event.height = bitmap.height; - // TODO: preserve image order - add_image(context, event.image_id, bitmap, p); + add_image(context, event.image_id, bitmap, p, bitmap.width, bitmap.height); // God knows when this will actually complete (it loads the image from the server) // so do not set need_draw. Instead just schedule the draw ourselves when done @@ -389,17 +416,28 @@ function handle_event(state, context, event, options = {}) { } case EVENT.IMAGE_MOVE: { - // Already moved due to local prediction - if (event.user_id !== state.me) { - const image_id = event.image_id; - const image = get_image(context, image_id); - - if (image) { - // if (config.debug_print) console.debug('move image', image_id, 'to', image_event.x, image_event.y); - image.at.x = event.x; - image.at.y = event.y; - need_draw = true; - } + geometry_add_dummy_stroke(context); + const image_id = event.image_id; + const image = get_image(context, image_id); + + if (image) { + // if (config.debug_print) console.debug('move image', image_id, 'to', image_event.x, image_event.y); + image.at.x = event.x; + image.at.y = event.y; + need_draw = true; + } + + break; + } + + case EVENT.IMAGE_SCALE: { + geometry_add_dummy_stroke(context); + const image_id = event.image_id; + const image = get_image(context, image_id); + + if (image !== null) { + scale_image(context, image, event.corner, {'x': event.x, 'y': event.y}); + need_draw = true; } break; diff --git a/client/client_send.js b/client/client_send.js index 33d9c61..ba8f0f5 100644 --- a/client/client_send.js +++ b/client/client_send.js @@ -146,6 +146,17 @@ function ser_event(s, event) { ser_u32(s, image_id); ser_f32(s, event.x); ser_f32(s, event.y); + ser_u32(s, event.width); + ser_u32(s, event.height); + break; + } + + case EVENT.IMAGE_SCALE: { + const image_id = parseInt(event.image_id); + ser_u32(s, image_id); + ser_u32(s, event.corner); // which corner was moved + ser_f32(s, event.x); // where corner was moved to (canvas coordinates) + ser_f32(s, event.y); break; } @@ -263,6 +274,7 @@ function push_event(state, event) { case EVENT.ERASER: case EVENT.IMAGE: case EVENT.IMAGE_MOVE: + case EVENT.IMAGE_SCALE: case EVENT.UNDO: case EVENT.REDO: { state.queue.push(event); @@ -330,12 +342,14 @@ function width_event(width) { }; } -function image_event(image_id, x, y) { +function image_event(image_id, x, y, width, height) { return { 'type': EVENT.IMAGE, 'image_id': image_id, 'x': x, 'y': y, + 'width': width, + 'height': height, }; } @@ -348,6 +362,16 @@ function image_move_event(image_id, x, y) { }; } +function image_scale_event(image_id, corner, x, y) { + return { + 'type': EVENT.IMAGE_SCALE, + 'image_id': image_id, + 'corner': corner, + 'x': x, + 'y': y, + }; +} + function stroke_event(state) { const stroke = geometry_prepare_stroke(state); diff --git a/client/index.js b/client/index.js index f3f0bcf..242fc51 100644 --- a/client/index.js +++ b/client/index.js @@ -58,6 +58,7 @@ const EVENT = Object.freeze({ IMAGE: 40, IMAGE_MOVE: 41, + IMAGE_SCALE: 42, ERASER: 50, }); diff --git a/client/webgl_geometry.js b/client/webgl_geometry.js index ee6598c..398c0b5 100644 --- a/client/webgl_geometry.js +++ b/client/webgl_geometry.js @@ -199,28 +199,38 @@ function geometry_clear_player(state, context, player_id) { recompute_dynamic_data(state, context); } -function add_image(context, image_id, bitmap, p) { - const x = p.x; - const y = p.y; +function add_image(context, image_id, bitmap, p, width, height) { const gl = context.gl; - const id = Object.keys(context.images).length; - const entry = { - 'texture': gl.createTexture(), - 'key': image_id, - 'at': p, - 'width': bitmap.width, - 'height': bitmap.height, - }; - - context.images.push(entry); + let entry = null; + + // If bitmap not available yet - create placeholder + // Otherwise - upload actual bitmap + if (bitmap === null) { + entry = { + 'texture': gl.createTexture(), + 'key': image_id, + 'at': p, + 'width': width, + 'height': height, + }; + + context.images.push(entry); + } else { + entry = get_image(context, image_id); + } gl.bindTexture(gl.TEXTURE_2D, entry.texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, bitmap); - gl.generateMipmap(gl.TEXTURE_2D); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR_MIPMAP_LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + + if (bitmap !== null) { + gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, gl.RGBA, gl.UNSIGNED_BYTE, bitmap); + gl.generateMipmap(gl.TEXTURE_2D); + } else { + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array(4 * width * height)); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR_MIPMAP_LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + } } function move_image(context, image, dx, dy) { diff --git a/client/webgl_listeners.js b/client/webgl_listeners.js index 1386755..535907a 100644 --- a/client/webgl_listeners.js +++ b/client/webgl_listeners.js @@ -404,6 +404,9 @@ function mousemove(e, state, context) { } function mouseup(e, state, context) { + const screenp = {'x': window.devicePixelRatio * e.clientX, 'y': window.devicePixelRatio * e.clientY}; + const canvasp = screen_to_canvas(state, screenp); + if (e.button !== 0 && e.button !== 1) { return; } @@ -422,6 +425,7 @@ function mouseup(e, state, context) { } if (state.imagescaling) { + queue_event(state, image_scale_event(state.active_image, state.scaling_corner, canvasp.x, canvasp.y)); state.imagescaling = false; state.scaling_corner = null; return; diff --git a/server/deserializer.js b/server/deserializer.js index e7e2eac..2f234a1 100644 --- a/server/deserializer.js +++ b/server/deserializer.js @@ -115,13 +115,29 @@ export function event(d) { break; } - case EVENT.IMAGE: + case EVENT.IMAGE: { + event.image_id = u32(d); + event.x = f32(d); + event.y = f32(d); + event.width = u32(d); + event.height = u32(d); + break; + } + case EVENT.IMAGE_MOVE: { event.image_id = u32(d); event.x = f32(d); event.y = f32(d); break; } + + case EVENT.IMAGE_SCALE: { + event.image_id = u32(d); + event.corner = u32(d); + event.x = f32(d); + event.y = f32(d); + break; + } case EVENT.UNDO: case EVENT.REDO: { diff --git a/server/enums.js b/server/enums.js index 8c61987..51bc768 100644 --- a/server/enums.js +++ b/server/enums.js @@ -21,6 +21,7 @@ export const EVENT = Object.freeze({ REDO: 31, IMAGE: 40, IMAGE_MOVE: 41, + IMAGE_SCALE: 42, ERASER: 50, }); diff --git a/server/milton.js b/server/milton.js index e437ee1..7cafcb5 100644 --- a/server/milton.js +++ b/server/milton.js @@ -38,8 +38,11 @@ function parse_and_insert_stroke(desk_id, line) { '$session_id': 0, '$stroke_id': stroke_res.id, '$image_id': 0, + '$corner': 0, '$x': 0, '$y': 0, + '$width': 0, + '$height': 0, }); } @@ -67,4 +70,16 @@ async function import_milton_file_to_sqlite(fullpath) { console.log(`Finished importing desk ${desk_id}`); } -import_milton_file_to_sqlite("/home/aolo2/Documents/bin/milton/build/points_pressure.txt"); +async function set_dimentions_to_images(fullpath) { + const images = [ + // + ]; + + storage.startup(); + + for (const image of images) { + storage.db.run(`UPDATE events SET width = ${image.w}, height = ${image.h} WHERE image_id = ${image.t};`); + } +} + +set_dimentions_to_images(); diff --git a/server/recv.js b/server/recv.js index 01b9606..969874a 100644 --- a/server/recv.js +++ b/server/recv.js @@ -122,8 +122,11 @@ function handle_event(session, event) { '$session_id': session.id, '$stroke_id': event.stroke_id, '$image_id': 0, + '$corner': 0, '$x': 0, '$y': 0, + '$width': 0, + '$height': 0, }); desks[session.desk_id].total_points += event.points.length; @@ -134,6 +137,7 @@ function handle_event(session, event) { case EVENT.ERASER: case EVENT.IMAGE: case EVENT.IMAGE_MOVE: + case EVENT.IMAGE_SCALE: case EVENT.UNDO: { storage.queries.insert_event.run({ '$type': event.type, @@ -141,8 +145,11 @@ function handle_event(session, event) { '$session_id': session.id, '$stroke_id': event.stroke_id || 0, '$image_id': event.image_id || 0, + '$corner': event.corner || 0, '$x': event.x || 0, '$y': event.y || 0, + '$width': event.width || 0, + '$height': event.height || 0, }); break; diff --git a/server/send.js b/server/send.js index bab4b46..a0324bd 100644 --- a/server/send.js +++ b/server/send.js @@ -49,12 +49,21 @@ function event_size(event) { break; } - case EVENT.IMAGE: + case EVENT.IMAGE: { + size += 4 + 4 + 4 + 4 + 4; // file_id + x + y + width + height + break; + } + case EVENT.IMAGE_MOVE: { size += 4 + 4 + 4; // file id + x + y break; } + case EVENT.IMAGE_SCALE: { + size += 4 + 4 + 4 + 4; // file_id + corner + x + y + break; + } + case EVENT.UNDO: case EVENT.REDO: { break; diff --git a/server/serializer.js b/server/serializer.js index f43db6a..ed59b0d 100644 --- a/server/serializer.js +++ b/server/serializer.js @@ -108,7 +108,15 @@ export function event(s, event) { break; } - case EVENT.IMAGE: + case EVENT.IMAGE: { + u32(s, event.image_id); + f32(s, event.x); + f32(s, event.y); + u32(s, event.width); + u32(s, event.height); + break; + } + case EVENT.IMAGE_MOVE: { u32(s, event.image_id); f32(s, event.x); @@ -116,6 +124,14 @@ export function event(s, event) { break; } + case EVENT.IMAGE_SCALE: { + u32(s, event.image_id); + u32(s, event.corner); + f32(s, event.x); + f32(s, event.y); + break; + } + case EVENT.UNDO: case EVENT.REDO: { break; diff --git a/server/storage.js b/server/storage.js index b88cd6d..f049679 100644 --- a/server/storage.js +++ b/server/storage.js @@ -49,8 +49,11 @@ export function startup() { session_id INTEGER, stroke_id INTEGER, image_id INTEGER, + corner INTEGER, x INTEGER, y INTEGER, + width INTEGER, + height INTEGER, FOREIGN KEY (desk_id) REFERENCES desks (id) @@ -72,7 +75,7 @@ export function startup() { queries.insert_desk = db.query('INSERT INTO desks (id, title, sn) VALUES ($id, $title, 0) RETURNING id'); queries.insert_stroke = db.query('INSERT INTO strokes (width, color, points, pressures) VALUES ($width, $color, $points, $pressures) RETURNING id'); queries.insert_session = db.query('INSERT INTO sessions (id, desk_id, lsn) VALUES ($id, $desk_id, 0) RETURNING id'); - queries.insert_event = db.query('INSERT INTO events (type, desk_id, session_id, stroke_id, image_id, x, y) VALUES ($type, $desk_id, $session_id, $stroke_id, $image_id, $x, $y) RETURNING id'); + queries.insert_event = db.query('INSERT INTO events (type, desk_id, session_id, stroke_id, image_id, corner, x, y, width, height) VALUES ($type, $desk_id, $session_id, $stroke_id, $image_id, $corner, $x, $y, $width, $height) RETURNING id'); // UPDATE queries.update_desk_sn = db.query('UPDATE desks SET sn = $sn WHERE id = $id');