From d3fba7fd5ff11a62353da7ce83d3215df9421150 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 18 Jan 2023 10:52:12 -0500 Subject: [PATCH 01/40] WIP minus unfinished split grid layouts --- src/room/InCallView.module.css | 1 + src/room/InCallView.tsx | 5 +- src/video-grid/NewVideoGrid.module.css | 18 ++++ src/video-grid/NewVideoGrid.tsx | 132 +++++++++++++++++++++++++ src/video-grid/VideoGrid.tsx | 2 +- src/video-grid/VideoTile.module.css | 1 + 6 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 src/video-grid/NewVideoGrid.module.css create mode 100644 src/video-grid/NewVideoGrid.tsx diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 8c37d465..ef565731 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -17,6 +17,7 @@ limitations under the License. .inRoom { position: relative; display: flex; + gap: 8px; flex-direction: column; overflow: hidden; min-height: 100%; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 22145012..88330dcc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -64,6 +64,7 @@ import { ParticipantInfo } from "./useGroupCall"; import { TileDescriptor } from "../video-grid/TileDescriptor"; import { AudioSink } from "../video-grid/AudioSink"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; +import { NewVideoGrid } from "../video-grid/NewVideoGrid"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -305,7 +306,7 @@ export function InCallView({ } return ( - )} - + ); }; diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css new file mode 100644 index 00000000..b035e655 --- /dev/null +++ b/src/video-grid/NewVideoGrid.module.css @@ -0,0 +1,18 @@ +.grid { + position: relative; + flex-grow: 1; + padding: 0 22px; + overflow-y: scroll; +} + +.slotGrid { + position: relative; + display: grid; + grid-auto-rows: 183px; + column-gap: 18px; + row-gap: 21px; +} + +.slot { + background-color: red; +} diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx new file mode 100644 index 00000000..6bc78e15 --- /dev/null +++ b/src/video-grid/NewVideoGrid.tsx @@ -0,0 +1,132 @@ +import { useTransition } from "@react-spring/web"; +import React, { FC, memo, ReactNode, useMemo, useRef } from "react"; +import useMeasure from "react-use-measure"; +import styles from "./NewVideoGrid.module.css"; +import { TileDescriptor } from "./TileDescriptor"; +import { VideoGridProps as Props } from "./VideoGrid"; + +interface Cell { + /** + * The item held by the slot containing this cell. + */ + item: TileDescriptor + /** + * Whether this cell is the first cell of the containing slot. + */ + slot: boolean + /** + * The width, in columns, of the containing slot. + */ + columns: number + /** + * The height, in rows, of the containing slot. + */ + rows: number +} + +interface Rect { + x: number + y: number + width: number + height: number +} + +interface Tile extends Rect { + item: TileDescriptor + dragging: boolean +} + +interface SlotsProps { + count: number +} + +/** + * Generates a number of empty slot divs. + */ +const Slots: FC = memo(({ count }) => { + const slots = new Array(count) + for (let i = 0; i < count; i++) slots[i] =
+ return <>{slots} +}) + +export const NewVideoGrid: FC = ({ items, children }) => { + const slotGridRef = useRef(null); + const [gridRef, gridBounds] = useMeasure(); + + const slotRects = useMemo(() => { + if (slotGridRef.current === null) return []; + + const slots = slotGridRef.current.getElementsByClassName(styles.slot) + const rects = new Array(slots.length) + for (let i = 0; i < slots.length; i++) { + const slot = slots[i] as HTMLElement + rects[i] = { + x: slot.offsetLeft, + y: slot.offsetTop, + width: slot.offsetWidth, + height: slot.offsetHeight, + } + } + + return rects; + }, [items, gridBounds]); + + const cells: Cell[] = useMemo(() => items.map(item => ({ + item, + slot: true, + columns: 1, + rows: 1, + })), [items]) + + const slotCells = useMemo(() => cells.filter(cell => cell.slot), [cells]) + + const tiles: Tile[] = useMemo(() => slotRects.map((slot, i) => { + const cell = slotCells[i] + return { + item: cell.item, + x: slot.x, + y: slot.y, + width: slot.width, + height: slot.height, + dragging: false, + } + }), [slotRects, cells]) + + const [tileTransitions] = useTransition(tiles, () => ({ + key: ({ item }: Tile) => item.id, + from: { opacity: 0 }, + enter: ({ x, y, width, height }: Tile) => ({ opacity: 1, x, y, width, height }), + update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), + leave: { opacity: 0 }, + }), [tiles]) + + const slotGridStyle = useMemo(() => { + const columnCount = gridBounds.width >= 800 ? 6 : 3; + return { + gridTemplateColumns: `repeat(${columnCount}, 1fr)`, + }; + }, [gridBounds]); + + // Render nothing if the bounds are not yet known + if (gridBounds.width === 0) { + return
+ } + + return ( +
+
+ +
+ {tileTransitions((style, tile) => children({ + key: tile.item.id, + style: style as any, + width: tile.width, + height: tile.height, + item: tile.item, + }))} +
+ ); +}; diff --git a/src/video-grid/VideoGrid.tsx b/src/video-grid/VideoGrid.tsx index 70633e9c..fc6e44d8 100644 --- a/src/video-grid/VideoGrid.tsx +++ b/src/video-grid/VideoGrid.tsx @@ -705,7 +705,7 @@ interface ChildrenProperties extends ReactDOMAttributes { [index: string]: unknown; } -interface VideoGridProps { +export interface VideoGridProps { items: TileDescriptor[]; layout: Layout; disableAnimations?: boolean; diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css index d6f6e066..c13976d1 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/video-grid/VideoTile.module.css @@ -16,6 +16,7 @@ limitations under the License. .videoTile { position: absolute; + top: 0; will-change: transform, width, height, opacity, box-shadow; border-radius: 20px; overflow: hidden; From 486674c442e1ee9d9930326df769239a64f2777f Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 18 Jan 2023 11:32:51 -0500 Subject: [PATCH 02/40] fixes --- src/video-grid/NewVideoGrid.tsx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 6bc78e15..003c0709 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -80,16 +80,18 @@ export const NewVideoGrid: FC = ({ items, children }) => { const slotCells = useMemo(() => cells.filter(cell => cell.slot), [cells]) - const tiles: Tile[] = useMemo(() => slotRects.map((slot, i) => { + const tiles: Tile[] = useMemo(() => slotRects.flatMap((slot, i) => { const cell = slotCells[i] - return { + if (cell === undefined) return [] + + return [{ item: cell.item, x: slot.x, y: slot.y, width: slot.width, height: slot.height, dragging: false, - } + }] }), [slotRects, cells]) const [tileTransitions] = useTransition(tiles, () => ({ @@ -109,7 +111,11 @@ export const NewVideoGrid: FC = ({ items, children }) => { // Render nothing if the bounds are not yet known if (gridBounds.width === 0) { - return
+ return
+ {/* It's important that we always attach slotGridRef to something, + or else we may not receive the initial slot rects. */} +
+
} return ( From 2318d75bc78b5ce5e1f46a902567d481db222bd9 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 18 Jan 2023 11:33:40 -0500 Subject: [PATCH 03/40] prettier --- src/video-grid/NewVideoGrid.tsx | 142 ++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 60 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 003c0709..82c4b32e 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -9,45 +9,46 @@ interface Cell { /** * The item held by the slot containing this cell. */ - item: TileDescriptor + item: TileDescriptor; /** * Whether this cell is the first cell of the containing slot. */ - slot: boolean + slot: boolean; /** * The width, in columns, of the containing slot. */ - columns: number + columns: number; /** * The height, in rows, of the containing slot. */ - rows: number + rows: number; } interface Rect { - x: number - y: number - width: number - height: number + x: number; + y: number; + width: number; + height: number; } interface Tile extends Rect { - item: TileDescriptor - dragging: boolean + item: TileDescriptor; + dragging: boolean; } interface SlotsProps { - count: number + count: number; } /** * Generates a number of empty slot divs. */ const Slots: FC = memo(({ count }) => { - const slots = new Array(count) - for (let i = 0; i < count; i++) slots[i] =
- return <>{slots} -}) + const slots = new Array(count); + for (let i = 0; i < count; i++) + slots[i] =
; + return <>{slots}; +}); export const NewVideoGrid: FC = ({ items, children }) => { const slotGridRef = useRef(null); @@ -56,51 +57,71 @@ export const NewVideoGrid: FC = ({ items, children }) => { const slotRects = useMemo(() => { if (slotGridRef.current === null) return []; - const slots = slotGridRef.current.getElementsByClassName(styles.slot) - const rects = new Array(slots.length) + const slots = slotGridRef.current.getElementsByClassName(styles.slot); + const rects = new Array(slots.length); for (let i = 0; i < slots.length; i++) { - const slot = slots[i] as HTMLElement + const slot = slots[i] as HTMLElement; rects[i] = { x: slot.offsetLeft, y: slot.offsetTop, width: slot.offsetWidth, height: slot.offsetHeight, - } + }; } return rects; }, [items, gridBounds]); - const cells: Cell[] = useMemo(() => items.map(item => ({ - item, - slot: true, - columns: 1, - rows: 1, - })), [items]) + const cells: Cell[] = useMemo( + () => + items.map((item) => ({ + item, + slot: true, + columns: 1, + rows: 1, + })), + [items] + ); - const slotCells = useMemo(() => cells.filter(cell => cell.slot), [cells]) + const slotCells = useMemo(() => cells.filter((cell) => cell.slot), [cells]); - const tiles: Tile[] = useMemo(() => slotRects.flatMap((slot, i) => { - const cell = slotCells[i] - if (cell === undefined) return [] + const tiles: Tile[] = useMemo( + () => + slotRects.flatMap((slot, i) => { + const cell = slotCells[i]; + if (cell === undefined) return []; - return [{ - item: cell.item, - x: slot.x, - y: slot.y, - width: slot.width, - height: slot.height, - dragging: false, - }] - }), [slotRects, cells]) + return [ + { + item: cell.item, + x: slot.x, + y: slot.y, + width: slot.width, + height: slot.height, + dragging: false, + }, + ]; + }), + [slotRects, cells] + ); - const [tileTransitions] = useTransition(tiles, () => ({ - key: ({ item }: Tile) => item.id, - from: { opacity: 0 }, - enter: ({ x, y, width, height }: Tile) => ({ opacity: 1, x, y, width, height }), - update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), - leave: { opacity: 0 }, - }), [tiles]) + const [tileTransitions] = useTransition( + tiles, + () => ({ + key: ({ item }: Tile) => item.id, + from: { opacity: 0 }, + enter: ({ x, y, width, height }: Tile) => ({ + opacity: 1, + x, + y, + width, + height, + }), + update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), + leave: { opacity: 0 }, + }), + [tiles] + ); const slotGridStyle = useMemo(() => { const columnCount = gridBounds.width >= 800 ? 6 : 3; @@ -111,28 +132,29 @@ export const NewVideoGrid: FC = ({ items, children }) => { // Render nothing if the bounds are not yet known if (gridBounds.width === 0) { - return
- {/* It's important that we always attach slotGridRef to something, + return ( +
+ {/* It's important that we always attach slotGridRef to something, or else we may not receive the initial slot rects. */} -
-
+
+
+ ); } return ( -
+
- {tileTransitions((style, tile) => children({ - key: tile.item.id, - style: style as any, - width: tile.width, - height: tile.height, - item: tile.item, - }))} + {tileTransitions((style, tile) => + children({ + key: tile.item.id, + style: style as any, + width: tile.width, + height: tile.height, + item: tile.item, + }) + )}
); }; From 46d1351d8358158cc9d1369bc5087dfb5bac89ad Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 18 Jan 2023 13:38:29 -0500 Subject: [PATCH 04/40] More fixes --- src/video-grid/NewVideoGrid.tsx | 75 ++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 82c4b32e..2c334447 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -1,5 +1,13 @@ import { useTransition } from "@react-spring/web"; -import React, { FC, memo, ReactNode, useMemo, useRef } from "react"; +import React, { + FC, + memo, + ReactNode, + useCallback, + useMemo, + useRef, + useState, +} from "react"; import useMeasure from "react-use-measure"; import styles from "./NewVideoGrid.module.css"; import { TileDescriptor } from "./TileDescriptor"; @@ -33,7 +41,6 @@ interface Rect { interface Tile extends Rect { item: TileDescriptor; - dragging: boolean; } interface SlotsProps { @@ -50,14 +57,18 @@ const Slots: FC = memo(({ count }) => { return <>{slots}; }); -export const NewVideoGrid: FC = ({ items, children }) => { - const slotGridRef = useRef(null); +export const NewVideoGrid: FC = ({ + items, + disableAnimations, + children, +}) => { + const [slotGrid, setSlotGrid] = useState(null); const [gridRef, gridBounds] = useMeasure(); const slotRects = useMemo(() => { - if (slotGridRef.current === null) return []; + if (slotGrid === null) return []; - const slots = slotGridRef.current.getElementsByClassName(styles.slot); + const slots = slotGrid.getElementsByClassName(styles.slot); const rects = new Array(slots.length); for (let i = 0; i < slots.length; i++) { const slot = slots[i] as HTMLElement; @@ -70,7 +81,7 @@ export const NewVideoGrid: FC = ({ items, children }) => { } return rects; - }, [items, gridBounds]); + }, [items, gridBounds, slotGrid]); const cells: Cell[] = useMemo( () => @@ -98,7 +109,6 @@ export const NewVideoGrid: FC = ({ items, children }) => { y: slot.y, width: slot.width, height: slot.height, - dragging: false, }, ]; }), @@ -109,18 +119,36 @@ export const NewVideoGrid: FC = ({ items, children }) => { tiles, () => ({ key: ({ item }: Tile) => item.id, - from: { opacity: 0 }, - enter: ({ x, y, width, height }: Tile) => ({ - opacity: 1, + from: (({ x, y, width, height }: Tile) => ({ + opacity: 0, + scale: 0, + shadow: 1, x, y, width, height, - }), + // react-spring's types are bugged and need this to be a function with no + // parameters to infer the spring type + })) as unknown as () => { + opacity: number; + scale: number; + shadow: number; + x: number; + y: number; + width: number; + height: number; + }, + enter: { opacity: 1, scale: 1 }, update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), - leave: { opacity: 0 }, + leave: { opacity: 0, scale: 0 }, + immediate: (key: string) => + disableAnimations || key === "zIndex" || key === "shadow", + // If we just stopped dragging a tile, give it time for the + // animation to settle before pushing its z-index back down + delay: (key: string) => (key === "zIndex" ? 500 : 0), + trail: 20, }), - [tiles] + [tiles, disableAnimations] ); const slotGridStyle = useMemo(() => { @@ -132,24 +160,23 @@ export const NewVideoGrid: FC = ({ items, children }) => { // Render nothing if the bounds are not yet known if (gridBounds.width === 0) { - return ( -
- {/* It's important that we always attach slotGridRef to something, - or else we may not receive the initial slot rects. */} -
-
- ); + return
; } return (
-
+
- {tileTransitions((style, tile) => + {tileTransitions(({ shadow, ...style }, tile) => children({ key: tile.item.id, - style: style as any, + style: { + boxShadow: shadow.to( + (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px` + ), + ...style, + }, width: tile.width, height: tile.height, item: tile.item, From 045103dbc97abb418a33c8b58dde8434752c88dc Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 25 Jan 2023 02:30:52 -0500 Subject: [PATCH 05/40] Backfill the grid as people leave by moving tiles along paths --- package.json | 1 + src/useReactiveState.ts | 46 ++++ src/video-grid/NewVideoGrid.module.css | 4 +- src/video-grid/NewVideoGrid.tsx | 301 +++++++++++++++++++++---- yarn.lock | 5 + 5 files changed, 315 insertions(+), 42 deletions(-) create mode 100644 src/useReactiveState.ts diff --git a/package.json b/package.json index c3a511f7..4242b19c 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "react-use-clipboard": "^1.0.7", "react-use-measure": "^2.1.1", "sdp-transform": "^2.14.1", + "tinyqueue": "^2.0.3", "unique-names-generator": "^4.6.0" }, "devDependencies": { diff --git a/src/useReactiveState.ts b/src/useReactiveState.ts new file mode 100644 index 00000000..fe995724 --- /dev/null +++ b/src/useReactiveState.ts @@ -0,0 +1,46 @@ +import { + DependencyList, + Dispatch, + SetStateAction, + useCallback, + useRef, + useState, +} from "react"; + +export const useReactiveState = ( + updateFn: (prevState?: T) => T, + deps: DependencyList +): [T, Dispatch>] => { + const state = useRef(); + if (state.current === undefined) state.current = updateFn(); + const prevDeps = useRef(); + + // Since we store the state in a ref, we use this counter to force an update + // when someone calls setState + const [, setNumUpdates] = useState(0); + + // If this is the first render or the deps have changed, recalculate the state + if ( + prevDeps.current === undefined || + deps.length !== prevDeps.current.length || + deps.some((d, i) => d !== prevDeps.current![i]) + ) { + state.current = updateFn(state.current); + } + prevDeps.current = deps; + + return [ + state.current, + useCallback( + (action) => { + if (typeof action === "function") { + state.current = (action as (prevValue: T) => T)(state.current!); + } else { + state.current = action; + } + setNumUpdates((n) => n + 1); // Force an update + }, + [setNumUpdates] + ), + ]; +}; diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index b035e655..d654beb8 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -13,6 +13,4 @@ row-gap: 21px; } -.slot { - background-color: red; -} +.slot {} diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 2c334447..5cda71ec 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -1,17 +1,21 @@ import { useTransition } from "@react-spring/web"; +import { useDrag } from "@use-gesture/react"; import React, { FC, memo, ReactNode, useCallback, + useEffect, useMemo, - useRef, useState, } from "react"; import useMeasure from "react-use-measure"; import styles from "./NewVideoGrid.module.css"; import { TileDescriptor } from "./TileDescriptor"; import { VideoGridProps as Props } from "./VideoGrid"; +import { useReactiveState } from "../useReactiveState"; +import TinyQueue from "tinyqueue"; +import { zipWith } from "lodash"; interface Cell { /** @@ -32,6 +36,12 @@ interface Cell { rows: number; } +interface Grid { + generation: number; + columns: number; + cells: (Cell | undefined)[]; +} + interface Rect { x: number; y: number; @@ -43,6 +53,162 @@ interface Tile extends Rect { item: TileDescriptor; } +interface TileSpring { + opacity: number; + scale: number; + shadow: number; + x: number; + y: number; + width: number; + height: number; +} + +const dijkstra = (g: Grid): number[] => { + const end = findLast1By1Index(g) ?? 0; + const endRow = row(end, g); + const endColumn = column(end, g); + + const distances = new Array(end + 1).fill(Infinity); + distances[end] = 0; + const edges = new Array(end).fill(undefined); + const heap = new TinyQueue([end], (i) => distances[i]); + + const visit = (curr: number, via: number) => { + const viaCell = g.cells[via]; + const viaLargeSlot = + viaCell !== undefined && (viaCell.rows > 1 || viaCell.columns > 1); + const distanceVia = distances[via] + (viaLargeSlot ? 4 : 1); + + if (distanceVia < distances[curr]) { + distances[curr] = distanceVia; + edges[curr] = via; + heap.push(curr); + } + }; + + while (heap.length > 0) { + const via = heap.pop()!; + const viaRow = row(via, g); + const viaColumn = column(via, g); + + if (viaRow > 0) visit(via - g.columns, via); + if (viaColumn > 0) visit(via - 1, via); + if (viaColumn < (viaRow === endRow ? endColumn : g.columns - 1)) + visit(via + 1, via); + if ( + viaRow < endRow - 1 || + (viaRow === endRow - 1 && viaColumn <= endColumn) + ) + visit(via + g.columns, via); + } + + return edges as number[]; +}; + +const findLastIndex = ( + array: T[], + predicate: (item: T) => boolean +): number | null => { + for (let i = array.length - 1; i > 0; i--) { + if (predicate(array[i])) return i; + } + + return null; +}; + +const findLast1By1Index = (g: Grid): number | null => + findLastIndex(g.cells, (c) => c?.rows === 1 && c?.columns === 1); + +const row = (index: number, g: Grid): number => Math.floor(index / g.columns); +const column = (index: number, g: Grid): number => index % g.columns; + +/** + * Gets the index of the next gap in the grid that should be backfilled by 1×1 + * tiles. + */ +const getNextGap = (g: Grid): number | null => { + const last1By1Index = findLast1By1Index(g); + if (last1By1Index === null) return null; + + for (let i = 0; i < last1By1Index; i++) { + // To make the backfilling process look natural when there are multiple + // gaps, we actually scan each row from right to left + const j = + (row(i, g) === row(last1By1Index, g) + ? last1By1Index + : (row(i, g) + 1) * g.columns) - + 1 - + column(i, g); + + if (g.cells[j] === undefined) return j; + } + + return null; +}; + +const fillGaps = (g: Grid): Grid => { + const result: Grid = { ...g, cells: [...g.cells] }; + let gap = getNextGap(result); + + if (gap !== null) { + const pathToEnd = dijkstra(result); + + do { + let filled = false; + let to = gap; + let from: number | undefined = pathToEnd[gap]; + + // First, attempt to fill the gap by moving 1×1 tiles backwards from the + // end of the grid along a set path + while (from !== undefined) { + const toCell = result.cells[to]; + const fromCell = result.cells[from]; + + // Skip over large tiles + if (toCell !== undefined) { + to = pathToEnd[to]; + // Skip over large tiles. Also, we might run into gaps along the path + // created during the filling of previous gaps. Skip over those too; + // they'll be picked up on the next iteration of the outer loop. + } else if ( + fromCell === undefined || + fromCell.rows > 1 || + fromCell.columns > 1 + ) { + from = pathToEnd[from]; + } else { + result.cells[to] = result.cells[from]; + result.cells[from] = undefined; + filled = true; + to = pathToEnd[to]; + from = pathToEnd[from]; + } + } + + // In case the path approach failed, fall back to taking the very last 1×1 + // tile, and just dropping it into place + if (!filled) { + const last1By1Index = findLast1By1Index(result)!; + result.cells[gap] = result.cells[last1By1Index]; + result.cells[last1By1Index] = undefined; + } + + gap = getNextGap(result); + } while (gap !== null); + } + + // TODO: If there are any large tiles on the last row, shuffle them back + // upwards into a full row + + // Shrink the array to remove trailing gaps + const finalLength = + (findLastIndex(result.cells, (c) => c !== undefined) ?? -1) + 1; + if (finalLength < result.cells.length) + result.cells = result.cells.slice(0, finalLength); + + return result; +}; + interface SlotsProps { count: number; } @@ -63,8 +229,24 @@ export const NewVideoGrid: FC = ({ children, }) => { const [slotGrid, setSlotGrid] = useState(null); + const [slotGridGeneration, setSlotGridGeneration] = useState(0) const [gridRef, gridBounds] = useMeasure(); + useEffect(() => { + if (slotGrid !== null) { + setSlotGridGeneration(parseInt(slotGrid.getAttribute("data-generation")!)) + + const observer = new MutationObserver(mutations => { + if (mutations.some(m => m.type === "attributes")) { + setSlotGridGeneration(parseInt(slotGrid.getAttribute("data-generation")!)) + } + }) + + observer.observe(slotGrid, { attributes: true }) + return () => observer.disconnect() + } + }, [slotGrid, setSlotGridGeneration]) + const slotRects = useMemo(() => { if (slotGrid === null) return []; @@ -81,38 +263,66 @@ export const NewVideoGrid: FC = ({ } return rects; - }, [items, gridBounds, slotGrid]); + }, [items, slotGridGeneration, slotGrid]); - const cells: Cell[] = useMemo( - () => - items.map((item) => ({ - item, - slot: true, - columns: 1, - rows: 1, - })), + const [grid, setGrid] = useReactiveState( + (prevGrid = { generation: 0, columns: 6, cells: [] }) => { + // Step 1: Update tiles that still exist, and remove tiles that have left + // the grid + const itemsById = new Map(items.map((i) => [i.id, i])); + const grid1: Grid = { + ...prevGrid, + generation: prevGrid.generation + 1, + cells: prevGrid.cells.map((c) => { + if (c === undefined) return undefined; + const item = itemsById.get(c.item.id); + return item === undefined ? undefined : { ...c, item }; + }), + }; + + // Step 2: Backfill gaps left behind by removed tiles + const grid2 = fillGaps(grid1); + + // Step 3: Add new tiles to the end of the grid + const existingItemIds = new Set( + grid2.cells.filter((c) => c !== undefined).map((c) => c!.item.id) + ); + const newItems = items.filter((i) => !existingItemIds.has(i.id)); + const grid3: Grid = { + ...grid2, + cells: [ + ...grid2.cells, + ...newItems.map((i) => ({ + item: i, + slot: true, + columns: 1, + rows: 1, + })), + ], + }; + + return grid3; + }, [items] ); - const slotCells = useMemo(() => cells.filter((cell) => cell.slot), [cells]); + const [tiles] = useReactiveState( + (prevTiles) => { + // If React hasn't yet rendered the current generation of the layout, skip + // the update, because grid and slotRects will be out of sync + if (slotGridGeneration !== grid.generation) return prevTiles ?? []; - const tiles: Tile[] = useMemo( - () => - slotRects.flatMap((slot, i) => { - const cell = slotCells[i]; - if (cell === undefined) return []; - - return [ - { - item: cell.item, - x: slot.x, - y: slot.y, - width: slot.width, - height: slot.height, - }, - ]; - }), - [slotRects, cells] + const slotCells = grid.cells.filter((c) => c?.slot) as Cell[]; + console.log(slotGridGeneration, grid.generation, slotCells.length, slotRects.length, slotGrid?.getElementsByClassName(styles.slot).length) + return zipWith(slotCells, slotRects, (cell, rect) => ({ + item: cell.item, + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + })); + }, + [slotRects, grid, slotGridGeneration] ); const [tileTransitions] = useTransition( @@ -129,15 +339,7 @@ export const NewVideoGrid: FC = ({ height, // react-spring's types are bugged and need this to be a function with no // parameters to infer the spring type - })) as unknown as () => { - opacity: number; - scale: number; - shadow: number; - x: number; - y: number; - width: number; - height: number; - }, + })) as unknown as () => TileSpring, enter: { opacity: 1, scale: 1 }, update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), leave: { opacity: 0, scale: 0 }, @@ -146,7 +348,6 @@ export const NewVideoGrid: FC = ({ // If we just stopped dragging a tile, give it time for the // animation to settle before pushing its z-index back down delay: (key: string) => (key === "zIndex" ? 500 : 0), - trail: 20, }), [tiles, disableAnimations] ); @@ -158,6 +359,22 @@ export const NewVideoGrid: FC = ({ }; }, [gridBounds]); + const bindTile = useDrag( + useCallback(({ event, tap }) => { + event.preventDefault(); + + if (tap) { + // TODO: When enlarging tiles, add the minimum number of rows required + // to not need to force any tiles towards the end, find the right number + // of consecutive spots for a tile of size w * (h - added rows), + // displace overlapping tiles, and then backfill. + // When unenlarging tiles, consider doing that in reverse (deleting + // rows and displacing tiles. pushing tiles outwards might be necessary) + } + }, []), + { filterTaps: true, pointer: { buttons: [1] } } + ); + // Render nothing if the bounds are not yet known if (gridBounds.width === 0) { return
; @@ -165,11 +382,17 @@ export const NewVideoGrid: FC = ({ return (
-
+
{tileTransitions(({ shadow, ...style }, tile) => children({ + ...bindTile(tile.item.id), key: tile.item.id, style: { boxShadow: shadow.to( diff --git a/yarn.lock b/yarn.lock index f5719527..7980fa2c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13717,6 +13717,11 @@ tiny-warning@^1.0.0, tiny-warning@^1.0.3: resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.3.tgz#94a30db453df4c643d0fd566060d60a875d84754" integrity sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA== +tinyqueue@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/tinyqueue/-/tinyqueue-2.0.3.tgz#64d8492ebf39e7801d7bd34062e29b45b2035f08" + integrity sha512-ppJZNDuKGgxzkHihX8v9v9G5f+18gzaTfrukGrq6ueg0lmH4nqVnA2IPG0AEH3jKEk2GRJCUhDoqpoiw3PHLBA== + tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" From 8912daa9221a9c4316e3aaa3801090c03007e124 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 25 Jan 2023 23:51:36 -0500 Subject: [PATCH 06/40] Make tiles resizable and fix some miscellaneous bugs --- src/video-grid/NewVideoGrid.module.css | 2 +- src/video-grid/NewVideoGrid.tsx | 170 ++++++++++++++++++++----- 2 files changed, 142 insertions(+), 30 deletions(-) diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index d654beb8..5be19d3f 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -2,7 +2,7 @@ position: relative; flex-grow: 1; padding: 0 22px; - overflow-y: scroll; + overflow-y: auto; } .slotGrid { diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 5cda71ec..d9313d18 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -1,10 +1,8 @@ -import { useTransition } from "@react-spring/web"; +import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; import { useDrag } from "@use-gesture/react"; import React, { FC, - memo, ReactNode, - useCallback, useEffect, useMemo, useState, @@ -25,6 +23,7 @@ interface Cell { /** * Whether this cell is the first cell of the containing slot. */ + // TODO: Rename to 'start'? slot: boolean; /** * The width, in columns, of the containing slot. @@ -122,6 +121,25 @@ const findLast1By1Index = (g: Grid): number | null => const row = (index: number, g: Grid): number => Math.floor(index / g.columns); const column = (index: number, g: Grid): number => index % g.columns; +function* cellsInArea(start: number, end: number, g: Grid): Generator{ + const startColumn = column(start, g) + const endColumn = column(end, g) + for (let i = start; i <= end; i = column(i, g) === endColumn ? i + g.columns + startColumn - endColumn : i + 1) + yield i +} + +const forEachCellInArea = (start: number, end: number, g: Grid, fn: (c: Cell | undefined, i: number) => void) => { + for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i) +} + +const allCellsInArea = (start: number, end: number, g: Grid, fn: (c: Cell | undefined, i: number) => boolean) => { + for (const i of cellsInArea(start, end, g)) { + if (!fn(g.cells[i], i)) return false + } + + return true +} + /** * Gets the index of the next gap in the grid that should be backfilled by 1×1 * tiles. @@ -209,20 +227,6 @@ const fillGaps = (g: Grid): Grid => { return result; }; -interface SlotsProps { - count: number; -} - -/** - * Generates a number of empty slot divs. - */ -const Slots: FC = memo(({ count }) => { - const slots = new Array(count); - for (let i = 0; i < count; i++) - slots[i] =
; - return <>{slots}; -}); - export const NewVideoGrid: FC = ({ items, disableAnimations, @@ -263,7 +267,7 @@ export const NewVideoGrid: FC = ({ } return rects; - }, [items, slotGridGeneration, slotGrid]); + }, [items, slotGridGeneration, slotGrid, gridBounds]); const [grid, setGrid] = useReactiveState( (prevGrid = { generation: 0, columns: 6, cells: [] }) => { @@ -313,7 +317,6 @@ export const NewVideoGrid: FC = ({ if (slotGridGeneration !== grid.generation) return prevTiles ?? []; const slotCells = grid.cells.filter((c) => c?.slot) as Cell[]; - console.log(slotGridGeneration, grid.generation, slotCells.length, slotRects.length, slotGrid?.getElementsByClassName(styles.slot).length) return zipWith(slotCells, slotRects, (cell, rect) => ({ item: cell.item, x: rect.x, @@ -329,7 +332,7 @@ export const NewVideoGrid: FC = ({ tiles, () => ({ key: ({ item }: Tile) => item.id, - from: (({ x, y, width, height }: Tile) => ({ + from: ({ x, y, width, height }: Tile) => ({ opacity: 0, scale: 0, shadow: 1, @@ -337,9 +340,7 @@ export const NewVideoGrid: FC = ({ y, width, height, - // react-spring's types are bugged and need this to be a function with no - // parameters to infer the spring type - })) as unknown as () => TileSpring, + }), enter: { opacity: 1, scale: 1 }, update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), leave: { opacity: 0, scale: 0 }, @@ -350,18 +351,35 @@ export const NewVideoGrid: FC = ({ delay: (key: string) => (key === "zIndex" ? 500 : 0), }), [tiles, disableAnimations] - ); + // react-spring's types are bugged and can't infer the spring type + ) as unknown as [TransitionFn, SpringRef]; const slotGridStyle = useMemo(() => { - const columnCount = gridBounds.width >= 800 ? 6 : 3; + const columnCount = 6 + + const areas = new Array<(number | null)[]>(Math.ceil(grid.cells.length / grid.columns)) + for (let i = 0; i < areas.length; i++) areas[i] = new Array(grid.columns).fill(null) + + let slotId = 0 + for (let i = 0; i < grid.cells.length; i++) { + const cell = grid.cells[i] + if (cell?.slot) { + const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1) + forEachCellInArea(i, slotEnd, grid, (_c, j) => areas[row(j, grid)][column(j, grid)] = slotId) + slotId++ + } + } + return { + gridTemplateAreas: areas.map(row => `'${row.map(slotId => slotId === null ? "." : `s${slotId}`).join(" ")}'`).join(" "), gridTemplateColumns: `repeat(${columnCount}, 1fr)`, }; - }, [gridBounds]); + }, [grid]); const bindTile = useDrag( - useCallback(({ event, tap }) => { + ({ event, tap, args }) => { event.preventDefault(); + const tileId = args[0] as string if (tap) { // TODO: When enlarging tiles, add the minimum number of rows required @@ -370,11 +388,105 @@ export const NewVideoGrid: FC = ({ // displace overlapping tiles, and then backfill. // When unenlarging tiles, consider doing that in reverse (deleting // rows and displacing tiles. pushing tiles outwards might be necessary) + setGrid(g => { + const from = g.cells.findIndex(c => c?.item.id === tileId) + if (from === -1) return g // Tile removed, no change + const fromWidth = g.cells[from]!.columns + const fromHeight = g.cells[from]!.rows + + const [toWidth, toHeight] = fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1] + const newRows = Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) + + const candidateWidth = toWidth + const candidateHeight = toHeight - newRows + + const slotStarts = new Array(g.cells.length) + g.cells.forEach((c, start) => { + if (c === undefined || c.item.id === tileId) { + slotStarts[start] = start + } else if (c.slot) { + const end = start + c.columns - 1 + g.columns * (c.rows - 1) + forEachCellInArea(start, end, g, (_c, i) => slotStarts[i] = start) + } else if (slotStarts[start] === undefined) { + slotStarts[start] = start + } + }) + + const nextScanLocations = new Set([from]) + const scanColumnOffset = Math.floor((toWidth - 1) / 2) + const scanRowOffset = Math.floor((toHeight - 1) / 2) + let to: number | null = null + + const displaceable = (c: Cell | undefined, i: number): boolean => c === undefined || (c.columns === 1 && c.rows === 1) || g.cells[slotStarts[i]]?.item.id === tileId + + for (const scanLocation of nextScanLocations) { + const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset + const end = start + candidateWidth - 1 + g.columns * (candidateHeight - 1) + const startColumn = column(start, g); + const endColumn = column(end, g); + + if (start >= 0 && end < g.cells.length && endColumn - startColumn + 1 === candidateWidth) { + if (allCellsInArea(start, end, g, displaceable)) { + to = start + break + } + } + + if (startColumn > 0) nextScanLocations.add(scanLocation - 1) + if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1) + nextScanLocations.add(scanLocation - g.columns) + nextScanLocations.add(scanLocation + g.columns) + } + + if (to === null) return g + + const gappyGrid: Grid = { + ...g, + generation: g.generation + 1, + cells: new Array(g.cells.length + newRows * g.columns), + } + + const toRow = row(to, g) + + for (let src = 0; src < g.cells.length; src++) { + if (g.cells[src]?.item.id !== tileId) { + const dest = row(src, g) > toRow + toHeight - 1 ? src + g.columns * newRows : src + gappyGrid.cells[dest] = g.cells[src] + } + } + + const displacedTiles: Cell[] = [] + const toEnd = to + toWidth - 1 + g.columns * (toHeight - 1) + forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { + if (c !== undefined) displacedTiles.push(c) + gappyGrid.cells[i] = { + item: g.cells[from]!.item, + slot: i === to, + columns: toWidth, + rows: toHeight, + } + }) + + for (let i = 0; displacedTiles.length > 0; i++) { + if (gappyGrid.cells[i] === undefined) gappyGrid.cells[i] = displacedTiles.shift() + } + + const nonGappy = fillGaps(gappyGrid) + console.log(`${g.cells.length} => ${nonGappy.cells.length}, ${g.generation} => ${nonGappy.generation}`) + return nonGappy + }) } - }, []), + }, { filterTaps: true, pointer: { buttons: [1] } } ); + const slots = useMemo(() => { + const slots = new Array(items.length); + for (let i = 0; i < items.length; i++) + slots[i] =
; + return slots + }, [items.length]) + // Render nothing if the bounds are not yet known if (gridBounds.width === 0) { return
; @@ -388,7 +500,7 @@ export const NewVideoGrid: FC = ({ className={styles.slotGrid} data-generation={grid.generation} > - + {slots}
{tileTransitions(({ shadow, ...style }, tile) => children({ From e99294c3f12f50ab97be2f9b15201fd73b360262 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Sun, 29 Jan 2023 21:45:10 -0500 Subject: [PATCH 07/40] Simplify some code --- src/video-grid/NewVideoGrid.tsx | 39 ++++++++++++++------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index d9313d18..ecae8174 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -121,6 +121,12 @@ const findLast1By1Index = (g: Grid): number | null => const row = (index: number, g: Grid): number => Math.floor(index / g.columns); const column = (index: number, g: Grid): number => index % g.columns; +const inArea = (index: number, start: number, end: number, g: Grid): boolean => { + const indexColumn = column(index, g) + const indexRow = column(index, g) + return indexRow >= row(start, g) && indexRow <= row(end, g) && indexColumn >= column(start, g) && indexColumn <= column(end, g) +} + function* cellsInArea(start: number, end: number, g: Grid): Generator{ const startColumn = column(start, g) const endColumn = column(end, g) @@ -140,6 +146,9 @@ const allCellsInArea = (start: number, end: number, g: Grid, fn: (c: Cell | unde return true } +const areaEnd = (start: number, columns: number, rows: number, g: Grid): number => + start + columns - 1 + g.columns * (rows - 1) + /** * Gets the index of the next gap in the grid that should be backfilled by 1×1 * tiles. @@ -382,17 +391,14 @@ export const NewVideoGrid: FC = ({ const tileId = args[0] as string if (tap) { - // TODO: When enlarging tiles, add the minimum number of rows required - // to not need to force any tiles towards the end, find the right number - // of consecutive spots for a tile of size w * (h - added rows), - // displace overlapping tiles, and then backfill. - // When unenlarging tiles, consider doing that in reverse (deleting + // TODO: When unenlarging tiles, do this in reverse somehow (deleting // rows and displacing tiles. pushing tiles outwards might be necessary) setGrid(g => { const from = g.cells.findIndex(c => c?.item.id === tileId) if (from === -1) return g // Tile removed, no change const fromWidth = g.cells[from]!.columns const fromHeight = g.cells[from]!.rows + const fromEnd = areaEnd(from, fromWidth, fromHeight, g) const [toWidth, toHeight] = fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1] const newRows = Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) @@ -400,28 +406,16 @@ export const NewVideoGrid: FC = ({ const candidateWidth = toWidth const candidateHeight = toHeight - newRows - const slotStarts = new Array(g.cells.length) - g.cells.forEach((c, start) => { - if (c === undefined || c.item.id === tileId) { - slotStarts[start] = start - } else if (c.slot) { - const end = start + c.columns - 1 + g.columns * (c.rows - 1) - forEachCellInArea(start, end, g, (_c, i) => slotStarts[i] = start) - } else if (slotStarts[start] === undefined) { - slotStarts[start] = start - } - }) - const nextScanLocations = new Set([from]) const scanColumnOffset = Math.floor((toWidth - 1) / 2) const scanRowOffset = Math.floor((toHeight - 1) / 2) let to: number | null = null - const displaceable = (c: Cell | undefined, i: number): boolean => c === undefined || (c.columns === 1 && c.rows === 1) || g.cells[slotStarts[i]]?.item.id === tileId + const displaceable = (c: Cell | undefined, i: number): boolean => c === undefined || (c.columns === 1 && c.rows === 1) || inArea(i, from, fromEnd, g) for (const scanLocation of nextScanLocations) { const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset - const end = start + candidateWidth - 1 + g.columns * (candidateHeight - 1) + const end = areaEnd(start, candidateWidth, candidateHeight, g) const startColumn = column(start, g); const endColumn = column(end, g); @@ -438,6 +432,7 @@ export const NewVideoGrid: FC = ({ nextScanLocations.add(scanLocation + g.columns) } + // TODO: Don't give up on placing the tile yet if (to === null) return g const gappyGrid: Grid = { @@ -456,7 +451,7 @@ export const NewVideoGrid: FC = ({ } const displacedTiles: Cell[] = [] - const toEnd = to + toWidth - 1 + g.columns * (toHeight - 1) + const toEnd = areaEnd(to, toWidth, toHeight, g) forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { if (c !== undefined) displacedTiles.push(c) gappyGrid.cells[i] = { @@ -471,9 +466,7 @@ export const NewVideoGrid: FC = ({ if (gappyGrid.cells[i] === undefined) gappyGrid.cells[i] = displacedTiles.shift() } - const nonGappy = fillGaps(gappyGrid) - console.log(`${g.cells.length} => ${nonGappy.cells.length}, ${g.generation} => ${nonGappy.generation}`) - return nonGappy + return fillGaps(gappyGrid) }) } }, From 4e3598490016f9dc6aa301211bf042422d5dba66 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Sun, 29 Jan 2023 21:54:53 -0500 Subject: [PATCH 08/40] Extract tile size change logic into a function --- src/video-grid/NewVideoGrid.tsx | 159 ++++++++++++++++---------------- 1 file changed, 82 insertions(+), 77 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index ecae8174..df14183d 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -236,6 +236,85 @@ const fillGaps = (g: Grid): Grid => { return result; }; +const cycleTileSize = (tileId: string, g: Grid): Grid => { + // TODO: When unenlarging tiles, do all this in reverse somehow (deleting + // rows and displacing tiles. pushing tiles outwards might be necessary) + + const from = g.cells.findIndex(c => c?.item.id === tileId) + if (from === -1) return g // Tile removed, no change + const fromWidth = g.cells[from]!.columns + const fromHeight = g.cells[from]!.rows + const fromEnd = areaEnd(from, fromWidth, fromHeight, g) + + const [toWidth, toHeight] = fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1] + const newRows = Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) + + const candidateWidth = toWidth + const candidateHeight = toHeight - newRows + + const nextScanLocations = new Set([from]) + const scanColumnOffset = Math.floor((toWidth - 1) / 2) + const scanRowOffset = Math.floor((toHeight - 1) / 2) + let to: number | null = null + + const displaceable = (c: Cell | undefined, i: number): boolean => c === undefined || (c.columns === 1 && c.rows === 1) || inArea(i, from, fromEnd, g) + + for (const scanLocation of nextScanLocations) { + const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset + const end = areaEnd(start, candidateWidth, candidateHeight, g) + const startColumn = column(start, g); + const endColumn = column(end, g); + + if (start >= 0 && end < g.cells.length && endColumn - startColumn + 1 === candidateWidth) { + if (allCellsInArea(start, end, g, displaceable)) { + to = start + break + } + } + + if (startColumn > 0) nextScanLocations.add(scanLocation - 1) + if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1) + nextScanLocations.add(scanLocation - g.columns) + nextScanLocations.add(scanLocation + g.columns) + } + + // TODO: Don't give up on placing the tile yet + if (to === null) return g + + const gappyGrid: Grid = { + ...g, + generation: g.generation + 1, + cells: new Array(g.cells.length + newRows * g.columns), + } + + const toRow = row(to, g) + + for (let src = 0; src < g.cells.length; src++) { + if (g.cells[src]?.item.id !== tileId) { + const dest = row(src, g) > toRow + toHeight - 1 ? src + g.columns * newRows : src + gappyGrid.cells[dest] = g.cells[src] + } + } + + const displacedTiles: Cell[] = [] + const toEnd = areaEnd(to, toWidth, toHeight, g) + forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { + if (c !== undefined) displacedTiles.push(c) + gappyGrid.cells[i] = { + item: g.cells[from]!.item, + slot: i === to, + columns: toWidth, + rows: toHeight, + } + }) + + for (let i = 0; displacedTiles.length > 0; i++) { + if (gappyGrid.cells[i] === undefined) gappyGrid.cells[i] = displacedTiles.shift() + } + + return fillGaps(gappyGrid) +} + export const NewVideoGrid: FC = ({ items, disableAnimations, @@ -391,83 +470,9 @@ export const NewVideoGrid: FC = ({ const tileId = args[0] as string if (tap) { - // TODO: When unenlarging tiles, do this in reverse somehow (deleting - // rows and displacing tiles. pushing tiles outwards might be necessary) - setGrid(g => { - const from = g.cells.findIndex(c => c?.item.id === tileId) - if (from === -1) return g // Tile removed, no change - const fromWidth = g.cells[from]!.columns - const fromHeight = g.cells[from]!.rows - const fromEnd = areaEnd(from, fromWidth, fromHeight, g) - - const [toWidth, toHeight] = fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1] - const newRows = Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) - - const candidateWidth = toWidth - const candidateHeight = toHeight - newRows - - const nextScanLocations = new Set([from]) - const scanColumnOffset = Math.floor((toWidth - 1) / 2) - const scanRowOffset = Math.floor((toHeight - 1) / 2) - let to: number | null = null - - const displaceable = (c: Cell | undefined, i: number): boolean => c === undefined || (c.columns === 1 && c.rows === 1) || inArea(i, from, fromEnd, g) - - for (const scanLocation of nextScanLocations) { - const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset - const end = areaEnd(start, candidateWidth, candidateHeight, g) - const startColumn = column(start, g); - const endColumn = column(end, g); - - if (start >= 0 && end < g.cells.length && endColumn - startColumn + 1 === candidateWidth) { - if (allCellsInArea(start, end, g, displaceable)) { - to = start - break - } - } - - if (startColumn > 0) nextScanLocations.add(scanLocation - 1) - if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1) - nextScanLocations.add(scanLocation - g.columns) - nextScanLocations.add(scanLocation + g.columns) - } - - // TODO: Don't give up on placing the tile yet - if (to === null) return g - - const gappyGrid: Grid = { - ...g, - generation: g.generation + 1, - cells: new Array(g.cells.length + newRows * g.columns), - } - - const toRow = row(to, g) - - for (let src = 0; src < g.cells.length; src++) { - if (g.cells[src]?.item.id !== tileId) { - const dest = row(src, g) > toRow + toHeight - 1 ? src + g.columns * newRows : src - gappyGrid.cells[dest] = g.cells[src] - } - } - - const displacedTiles: Cell[] = [] - const toEnd = areaEnd(to, toWidth, toHeight, g) - forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { - if (c !== undefined) displacedTiles.push(c) - gappyGrid.cells[i] = { - item: g.cells[from]!.item, - slot: i === to, - columns: toWidth, - rows: toHeight, - } - }) - - for (let i = 0; displacedTiles.length > 0; i++) { - if (gappyGrid.cells[i] === undefined) gappyGrid.cells[i] = displacedTiles.shift() - } - - return fillGaps(gappyGrid) - }) + setGrid(g => cycleTileSize(tileId, g)) + } else { + // TODO } }, { filterTaps: true, pointer: { buttons: [1] } } From 3805a2f20e5df29a53330accb3dec244fe905db3 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Sun, 29 Jan 2023 21:56:07 -0500 Subject: [PATCH 09/40] Format with Prettier --- src/video-grid/NewVideoGrid.module.css | 3 +- src/video-grid/NewVideoGrid.tsx | 234 ++++++++++++++++--------- 2 files changed, 150 insertions(+), 87 deletions(-) diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index 5be19d3f..dff0e871 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -13,4 +13,5 @@ row-gap: 21px; } -.slot {} +.slot { +} diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index df14183d..e5c15df5 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -1,12 +1,6 @@ import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; import { useDrag } from "@use-gesture/react"; -import React, { - FC, - ReactNode, - useEffect, - useMemo, - useState, -} from "react"; +import React, { FC, ReactNode, useEffect, useMemo, useState } from "react"; import useMeasure from "react-use-measure"; import styles from "./NewVideoGrid.module.css"; import { TileDescriptor } from "./TileDescriptor"; @@ -121,33 +115,68 @@ const findLast1By1Index = (g: Grid): number | null => const row = (index: number, g: Grid): number => Math.floor(index / g.columns); const column = (index: number, g: Grid): number => index % g.columns; -const inArea = (index: number, start: number, end: number, g: Grid): boolean => { - const indexColumn = column(index, g) - const indexRow = column(index, g) - return indexRow >= row(start, g) && indexRow <= row(end, g) && indexColumn >= column(start, g) && indexColumn <= column(end, g) +const inArea = ( + index: number, + start: number, + end: number, + g: Grid +): boolean => { + const indexColumn = column(index, g); + const indexRow = column(index, g); + return ( + indexRow >= row(start, g) && + indexRow <= row(end, g) && + indexColumn >= column(start, g) && + indexColumn <= column(end, g) + ); +}; + +function* cellsInArea( + start: number, + end: number, + g: Grid +): Generator { + const startColumn = column(start, g); + const endColumn = column(end, g); + for ( + let i = start; + i <= end; + i = + column(i, g) === endColumn + ? i + g.columns + startColumn - endColumn + : i + 1 + ) + yield i; } -function* cellsInArea(start: number, end: number, g: Grid): Generator{ - const startColumn = column(start, g) - const endColumn = column(end, g) - for (let i = start; i <= end; i = column(i, g) === endColumn ? i + g.columns + startColumn - endColumn : i + 1) - yield i -} +const forEachCellInArea = ( + start: number, + end: number, + g: Grid, + fn: (c: Cell | undefined, i: number) => void +) => { + for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i); +}; -const forEachCellInArea = (start: number, end: number, g: Grid, fn: (c: Cell | undefined, i: number) => void) => { - for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i) -} - -const allCellsInArea = (start: number, end: number, g: Grid, fn: (c: Cell | undefined, i: number) => boolean) => { +const allCellsInArea = ( + start: number, + end: number, + g: Grid, + fn: (c: Cell | undefined, i: number) => boolean +) => { for (const i of cellsInArea(start, end, g)) { - if (!fn(g.cells[i], i)) return false + if (!fn(g.cells[i], i)) return false; } - return true -} + return true; +}; -const areaEnd = (start: number, columns: number, rows: number, g: Grid): number => - start + columns - 1 + g.columns * (rows - 1) +const areaEnd = ( + start: number, + columns: number, + rows: number, + g: Grid +): number => start + columns - 1 + g.columns * (rows - 1); /** * Gets the index of the next gap in the grid that should be backfilled by 1×1 @@ -240,80 +269,92 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { // TODO: When unenlarging tiles, do all this in reverse somehow (deleting // rows and displacing tiles. pushing tiles outwards might be necessary) - const from = g.cells.findIndex(c => c?.item.id === tileId) - if (from === -1) return g // Tile removed, no change - const fromWidth = g.cells[from]!.columns - const fromHeight = g.cells[from]!.rows - const fromEnd = areaEnd(from, fromWidth, fromHeight, g) + const from = g.cells.findIndex((c) => c?.item.id === tileId); + if (from === -1) return g; // Tile removed, no change + const fromWidth = g.cells[from]!.columns; + const fromHeight = g.cells[from]!.rows; + const fromEnd = areaEnd(from, fromWidth, fromHeight, g); - const [toWidth, toHeight] = fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1] - const newRows = Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) + const [toWidth, toHeight] = + fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1]; + const newRows = Math.ceil( + (toWidth * toHeight - fromWidth * fromHeight) / g.columns + ); - const candidateWidth = toWidth - const candidateHeight = toHeight - newRows + const candidateWidth = toWidth; + const candidateHeight = toHeight - newRows; - const nextScanLocations = new Set([from]) - const scanColumnOffset = Math.floor((toWidth - 1) / 2) - const scanRowOffset = Math.floor((toHeight - 1) / 2) - let to: number | null = null + const nextScanLocations = new Set([from]); + const scanColumnOffset = Math.floor((toWidth - 1) / 2); + const scanRowOffset = Math.floor((toHeight - 1) / 2); + let to: number | null = null; - const displaceable = (c: Cell | undefined, i: number): boolean => c === undefined || (c.columns === 1 && c.rows === 1) || inArea(i, from, fromEnd, g) + const displaceable = (c: Cell | undefined, i: number): boolean => + c === undefined || + (c.columns === 1 && c.rows === 1) || + inArea(i, from, fromEnd, g); for (const scanLocation of nextScanLocations) { - const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset - const end = areaEnd(start, candidateWidth, candidateHeight, g) + const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset; + const end = areaEnd(start, candidateWidth, candidateHeight, g); const startColumn = column(start, g); const endColumn = column(end, g); - if (start >= 0 && end < g.cells.length && endColumn - startColumn + 1 === candidateWidth) { + if ( + start >= 0 && + end < g.cells.length && + endColumn - startColumn + 1 === candidateWidth + ) { if (allCellsInArea(start, end, g, displaceable)) { - to = start - break + to = start; + break; } } - if (startColumn > 0) nextScanLocations.add(scanLocation - 1) - if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1) - nextScanLocations.add(scanLocation - g.columns) - nextScanLocations.add(scanLocation + g.columns) + if (startColumn > 0) nextScanLocations.add(scanLocation - 1); + if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1); + nextScanLocations.add(scanLocation - g.columns); + nextScanLocations.add(scanLocation + g.columns); } // TODO: Don't give up on placing the tile yet - if (to === null) return g + if (to === null) return g; const gappyGrid: Grid = { ...g, generation: g.generation + 1, cells: new Array(g.cells.length + newRows * g.columns), - } + }; - const toRow = row(to, g) + const toRow = row(to, g); for (let src = 0; src < g.cells.length; src++) { if (g.cells[src]?.item.id !== tileId) { - const dest = row(src, g) > toRow + toHeight - 1 ? src + g.columns * newRows : src - gappyGrid.cells[dest] = g.cells[src] + const dest = + row(src, g) > toRow + toHeight - 1 ? src + g.columns * newRows : src; + gappyGrid.cells[dest] = g.cells[src]; } } - const displacedTiles: Cell[] = [] - const toEnd = areaEnd(to, toWidth, toHeight, g) + const displacedTiles: Cell[] = []; + const toEnd = areaEnd(to, toWidth, toHeight, g); forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { - if (c !== undefined) displacedTiles.push(c) + if (c !== undefined) displacedTiles.push(c); gappyGrid.cells[i] = { item: g.cells[from]!.item, slot: i === to, columns: toWidth, rows: toHeight, - } - }) + }; + }); for (let i = 0; displacedTiles.length > 0; i++) { - if (gappyGrid.cells[i] === undefined) gappyGrid.cells[i] = displacedTiles.shift() + if (gappyGrid.cells[i] === undefined) + gappyGrid.cells[i] = displacedTiles.shift(); } - return fillGaps(gappyGrid) -} + return fillGaps(gappyGrid); +}; export const NewVideoGrid: FC = ({ items, @@ -321,23 +362,27 @@ export const NewVideoGrid: FC = ({ children, }) => { const [slotGrid, setSlotGrid] = useState(null); - const [slotGridGeneration, setSlotGridGeneration] = useState(0) + const [slotGridGeneration, setSlotGridGeneration] = useState(0); const [gridRef, gridBounds] = useMeasure(); useEffect(() => { if (slotGrid !== null) { - setSlotGridGeneration(parseInt(slotGrid.getAttribute("data-generation")!)) + setSlotGridGeneration( + parseInt(slotGrid.getAttribute("data-generation")!) + ); - const observer = new MutationObserver(mutations => { - if (mutations.some(m => m.type === "attributes")) { - setSlotGridGeneration(parseInt(slotGrid.getAttribute("data-generation")!)) + const observer = new MutationObserver((mutations) => { + if (mutations.some((m) => m.type === "attributes")) { + setSlotGridGeneration( + parseInt(slotGrid.getAttribute("data-generation")!) + ); } - }) + }); - observer.observe(slotGrid, { attributes: true }) - return () => observer.disconnect() + observer.observe(slotGrid, { attributes: true }); + return () => observer.disconnect(); } - }, [slotGrid, setSlotGridGeneration]) + }, [slotGrid, setSlotGridGeneration]); const slotRects = useMemo(() => { if (slotGrid === null) return []; @@ -443,23 +488,38 @@ export const NewVideoGrid: FC = ({ ) as unknown as [TransitionFn, SpringRef]; const slotGridStyle = useMemo(() => { - const columnCount = 6 + const columnCount = 6; - const areas = new Array<(number | null)[]>(Math.ceil(grid.cells.length / grid.columns)) - for (let i = 0; i < areas.length; i++) areas[i] = new Array(grid.columns).fill(null) + const areas = new Array<(number | null)[]>( + Math.ceil(grid.cells.length / grid.columns) + ); + for (let i = 0; i < areas.length; i++) + areas[i] = new Array(grid.columns).fill(null); - let slotId = 0 + let slotId = 0; for (let i = 0; i < grid.cells.length; i++) { - const cell = grid.cells[i] + const cell = grid.cells[i]; if (cell?.slot) { - const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1) - forEachCellInArea(i, slotEnd, grid, (_c, j) => areas[row(j, grid)][column(j, grid)] = slotId) - slotId++ + const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1); + forEachCellInArea( + i, + slotEnd, + grid, + (_c, j) => (areas[row(j, grid)][column(j, grid)] = slotId) + ); + slotId++; } } return { - gridTemplateAreas: areas.map(row => `'${row.map(slotId => slotId === null ? "." : `s${slotId}`).join(" ")}'`).join(" "), + gridTemplateAreas: areas + .map( + (row) => + `'${row + .map((slotId) => (slotId === null ? "." : `s${slotId}`)) + .join(" ")}'` + ) + .join(" "), gridTemplateColumns: `repeat(${columnCount}, 1fr)`, }; }, [grid]); @@ -467,10 +527,10 @@ export const NewVideoGrid: FC = ({ const bindTile = useDrag( ({ event, tap, args }) => { event.preventDefault(); - const tileId = args[0] as string + const tileId = args[0] as string; if (tap) { - setGrid(g => cycleTileSize(tileId, g)) + setGrid((g) => cycleTileSize(tileId, g)); } else { // TODO } @@ -481,9 +541,11 @@ export const NewVideoGrid: FC = ({ const slots = useMemo(() => { const slots = new Array(items.length); for (let i = 0; i < items.length; i++) - slots[i] =
; - return slots - }, [items.length]) + slots[i] = ( +
+ ); + return slots; + }, [items.length]); // Render nothing if the bounds are not yet known if (gridBounds.width === 0) { From 4e73c07cb23d98050898f955bb73ba6fa752e0ce Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 30 Jan 2023 09:35:40 -0500 Subject: [PATCH 10/40] Try out scroll snapping --- src/room/InCallView.module.css | 2 +- src/video-grid/NewVideoGrid.module.css | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index ef565731..896e1f99 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -17,7 +17,6 @@ limitations under the License. .inRoom { position: relative; display: flex; - gap: 8px; flex-direction: column; overflow: hidden; min-height: 100%; @@ -41,6 +40,7 @@ limitations under the License. .footer { position: relative; + flex-shrink: 0; display: flex; justify-content: center; align-items: center; diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index dff0e871..c8e0d806 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -3,6 +3,7 @@ flex-grow: 1; padding: 0 22px; overflow-y: auto; + scroll-snap-type: both mandatory; } .slotGrid { @@ -14,4 +15,12 @@ } .slot { + scroll-snap-align: start; +} + +.slot:last-child { + /* This causes the grid to scroll up smoothly to the last item when its height + shrinks, and ensures that the user can always scroll the last row fully in + view */ + scroll-snap-align: end; } From 55dece274f71c0bcec7d050c605ba05acafbe99c Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 30 Jan 2023 17:04:43 -0500 Subject: [PATCH 11/40] Fix some tile resizing bugs --- src/video-grid/NewVideoGrid.tsx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index e5c15df5..2ad5e3ba 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -328,13 +328,15 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const toRow = row(to, g); - for (let src = 0; src < g.cells.length; src++) { - if (g.cells[src]?.item.id !== tileId) { - const dest = - row(src, g) > toRow + toHeight - 1 ? src + g.columns * newRows : src; - gappyGrid.cells[dest] = g.cells[src]; + g.cells.forEach((c, src) => { + if (c?.slot && c.item.id !== tileId) { + const offset = + row(src, g) > toRow + candidateHeight - 1 ? g.columns * newRows : 0; + forEachCellInArea(src, areaEnd(src, c.columns, c.rows, g), g, (c, i) => { + gappyGrid.cells[i + offset] = c + }) } - } + }) const displacedTiles: Cell[] = []; const toEnd = areaEnd(to, toWidth, toHeight, g); From f540f48461c13b05af2fe7fbeb053275ae3ffba0 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 30 Jan 2023 23:32:00 -0500 Subject: [PATCH 12/40] Fix some layout bugs --- src/video-grid/NewVideoGrid.tsx | 30 ++++++++++++++--------------- src/video-grid/VideoTile.module.css | 7 ------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 2ad5e3ba..28b9e0aa 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -122,7 +122,7 @@ const inArea = ( g: Grid ): boolean => { const indexColumn = column(index, g); - const indexRow = column(index, g); + const indexRow = row(index, g); return ( indexRow >= row(start, g) && indexRow <= row(end, g) && @@ -266,9 +266,6 @@ const fillGaps = (g: Grid): Grid => { }; const cycleTileSize = (tileId: string, g: Grid): Grid => { - // TODO: When unenlarging tiles, do all this in reverse somehow (deleting - // rows and displacing tiles. pushing tiles outwards might be necessary) - const from = g.cells.findIndex((c) => c?.item.id === tileId); if (from === -1) return g; // Tile removed, no change const fromWidth = g.cells[from]!.columns; @@ -277,13 +274,20 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const [toWidth, toHeight] = fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1]; - const newRows = Math.ceil( - (toWidth * toHeight - fromWidth * fromHeight) / g.columns + const newRows = Math.max( + 0, + Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) ); const candidateWidth = toWidth; const candidateHeight = toHeight - newRows; + const gappyGrid: Grid = { + ...g, + generation: g.generation + 1, + cells: new Array(g.cells.length + newRows * g.columns), + }; + const nextScanLocations = new Set([from]); const scanColumnOffset = Math.floor((toWidth - 1) / 2); const scanRowOffset = Math.floor((toHeight - 1) / 2); @@ -302,7 +306,7 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { if ( start >= 0 && - end < g.cells.length && + end < gappyGrid.cells.length && endColumn - startColumn + 1 === candidateWidth ) { if (allCellsInArea(start, end, g, displaceable)) { @@ -320,12 +324,6 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { // TODO: Don't give up on placing the tile yet if (to === null) return g; - const gappyGrid: Grid = { - ...g, - generation: g.generation + 1, - cells: new Array(g.cells.length + newRows * g.columns), - }; - const toRow = row(to, g); g.cells.forEach((c, src) => { @@ -333,10 +331,10 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const offset = row(src, g) > toRow + candidateHeight - 1 ? g.columns * newRows : 0; forEachCellInArea(src, areaEnd(src, c.columns, c.rows, g), g, (c, i) => { - gappyGrid.cells[i + offset] = c - }) + gappyGrid.cells[i + offset] = c; + }); } - }) + }); const displacedTiles: Cell[] = []; const toEnd = areaEnd(to, toWidth, toHeight, g); diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css index c13976d1..00429867 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/video-grid/VideoTile.module.css @@ -144,13 +144,6 @@ limitations under the License. white-space: nowrap; } -.videoMutedAvatar { - position: absolute; - left: 50%; - top: 50%; - transform: translate(-50%, -50%); -} - .videoMutedOverlay { width: 100%; height: 100%; From e3081c1c069f3d43c0879c6dfd45d818c2ec0e18 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 30 Jan 2023 23:32:26 -0500 Subject: [PATCH 13/40] Try out a snappier spring --- src/video-grid/NewVideoGrid.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 28b9e0aa..a7db7dda 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -477,6 +477,7 @@ export const NewVideoGrid: FC = ({ enter: { opacity: 1, scale: 1 }, update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), leave: { opacity: 0, scale: 0 }, + config: { mass: 0.7, tension: 252, friction: 25 }, immediate: (key: string) => disableAnimations || key === "zIndex" || key === "shadow", // If we just stopped dragging a tile, give it time for the From 0166eb67fb35bb0d22cf7bd650f1c67f691608bb Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 30 Jan 2023 23:43:45 -0500 Subject: [PATCH 14/40] Make avatars scale smoothly during animations --- src/room/InCallView.module.css | 6 ++++++ src/video-grid/NewVideoGrid.tsx | 4 +++- src/video-grid/VideoTile.module.css | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 896e1f99..310cf47b 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -66,6 +66,12 @@ limitations under the License. top: 50%; left: 50%; transform: translate(-50%, -50%); + /* To make avatars scale smoothly with their tiles during animations, we + override the styles set on the element */ + --avatarSize: calc(min(var(--tileWidth), var(--tileHeight)) / 2); + width: var(--avatarSize) !important; + height: var(--avatarSize) !important; + border-radius: 10000px !important; } @media (min-height: 300px) { diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index a7db7dda..04300dda 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -563,7 +563,7 @@ export const NewVideoGrid: FC = ({ > {slots}
- {tileTransitions(({ shadow, ...style }, tile) => + {tileTransitions(({ shadow, width, height, ...style }, tile) => children({ ...bindTile(tile.item.id), key: tile.item.id, @@ -571,6 +571,8 @@ export const NewVideoGrid: FC = ({ boxShadow: shadow.to( (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px` ), + "--tileWidth": width.to((w) => `${w}px`), + "--tileHeight": height.to((h) => `${h}px`), ...style, }, width: tile.width, diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css index 00429867..f475d0c5 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/video-grid/VideoTile.module.css @@ -18,6 +18,8 @@ limitations under the License. position: absolute; top: 0; will-change: transform, width, height, opacity, box-shadow; + width: var(--tileWidth); + height: var(--tileHeight); border-radius: 20px; overflow: hidden; cursor: pointer; From 82c7293308418161136bc01d6a383a1ef75fb8fd Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 30 Jan 2023 23:44:19 -0500 Subject: [PATCH 15/40] Replace premature animation optimization with a potentially wiser one --- src/video-grid/NewVideoGrid.module.css | 3 +++ src/video-grid/VideoTile.module.css | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index c8e0d806..f30f12c8 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -1,4 +1,5 @@ .grid { + contain: strict; position: relative; flex-grow: 1; padding: 0 22px; @@ -7,6 +8,7 @@ } .slotGrid { + contain: strict; position: relative; display: grid; grid-auto-rows: 183px; @@ -15,6 +17,7 @@ } .slot { + contain: strict; scroll-snap-align: start; } diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css index f475d0c5..05f3a0ce 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/video-grid/VideoTile.module.css @@ -16,8 +16,8 @@ limitations under the License. .videoTile { position: absolute; + contain: strict; top: 0; - will-change: transform, width, height, opacity, box-shadow; width: var(--tileWidth); height: var(--tileHeight); border-radius: 20px; From d7db845f3b243e05387f9035ed0746a991500bc3 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 30 Jan 2023 23:52:46 -0500 Subject: [PATCH 16/40] Scroll snap was a bad idea --- src/video-grid/NewVideoGrid.module.css | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index f30f12c8..27c453ff 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -4,7 +4,6 @@ flex-grow: 1; padding: 0 22px; overflow-y: auto; - scroll-snap-type: both mandatory; } .slotGrid { @@ -18,12 +17,4 @@ .slot { contain: strict; - scroll-snap-align: start; -} - -.slot:last-child { - /* This causes the grid to scroll up smoothly to the last item when its height - shrinks, and ensures that the user can always scroll the last row fully in - view */ - scroll-snap-align: end; } From eedf8a6d1b7369a60b5e93d7d7db69ed8baf95ef Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 1 Feb 2023 00:17:22 -0500 Subject: [PATCH 17/40] Make tiles draggable (but not yet droppable) --- src/video-grid/NewVideoGrid.tsx | 115 +++++++++++++++++++++++++++----- 1 file changed, 100 insertions(+), 15 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 04300dda..13d1860f 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -1,6 +1,13 @@ import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; -import { useDrag } from "@use-gesture/react"; -import React, { FC, ReactNode, useEffect, useMemo, useState } from "react"; +import { useDrag, useScroll } from "@use-gesture/react"; +import React, { + FC, + ReactNode, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import useMeasure from "react-use-measure"; import styles from "./NewVideoGrid.module.css"; import { TileDescriptor } from "./TileDescriptor"; @@ -50,12 +57,19 @@ interface TileSpring { opacity: number; scale: number; shadow: number; + zIndex: number; x: number; y: number; width: number; height: number; } +interface DragState { + tileId: string; + x: number; + y: number; +} + const dijkstra = (g: Grid): number[] => { const end = findLast1By1Index(g) ?? 0; const endRow = row(end, g); @@ -461,7 +475,11 @@ export const NewVideoGrid: FC = ({ [slotRects, grid, slotGridGeneration] ); - const [tileTransitions] = useTransition( + // Drag state is stored in a ref rather than component state, because we use + // react-spring's imperative API during gestures to improve responsiveness + const dragState = useRef(null); + + const [tileTransitions, springRef] = useTransition( tiles, () => ({ key: ({ item }: Tile) => item.id, @@ -469,20 +487,26 @@ export const NewVideoGrid: FC = ({ opacity: 0, scale: 0, shadow: 1, + zIndex: 1, x, y, width, height, + immediate: disableAnimations, }), - enter: { opacity: 1, scale: 1 }, - update: ({ x, y, width, height }: Tile) => ({ x, y, width, height }), - leave: { opacity: 0, scale: 0 }, + enter: { opacity: 1, scale: 1, immediate: disableAnimations }, + update: ({ item, x, y, width, height }: Tile) => + item.id === dragState.current?.tileId + ? {} + : { + x, + y, + width, + height, + immediate: disableAnimations, + }, + leave: { opacity: 0, scale: 0, immediate: disableAnimations }, config: { mass: 0.7, tension: 252, friction: 25 }, - immediate: (key: string) => - disableAnimations || key === "zIndex" || key === "shadow", - // If we just stopped dragging a tile, give it time for the - // animation to settle before pushing its z-index back down - delay: (key: string) => (key === "zIndex" ? 500 : 0), }), [tiles, disableAnimations] // react-spring's types are bugged and can't infer the spring type @@ -525,20 +549,81 @@ export const NewVideoGrid: FC = ({ }; }, [grid]); + const animateDraggedTile = (endOfGesture: boolean) => + springRef.start((_i, controller) => { + const { tileId, x, y } = dragState.current!; + + // react-spring appears to not update a controller's item as long as the + // key remains stable, so we can use it to look up the tile's ID but not + // its position + if ((controller.item as Tile).item.id === tileId) { + if (endOfGesture) { + const tile = tiles.find((t) => t.item.id === tileId)!; + + return { + scale: 1, + zIndex: 1, + shadow: 1, + x: tile.x, + y: tile.y, + immediate: disableAnimations || ((key) => key === "zIndex"), + // Allow the tile's position to settle before pushing its + // z-index back down + delay: (key) => (key === "zIndex" ? 500 : 0), + }; + } else { + return { + scale: 1.1, + zIndex: 2, + shadow: 15, + x, + y, + immediate: + disableAnimations || + ((key) => key === "zIndex" || key === "x" || key === "y"), + }; + } + } else { + return {}; + } + }); + const bindTile = useDrag( - ({ event, tap, args }) => { - event.preventDefault(); + ({ tap, args, delta: [dx, dy], last }) => { const tileId = args[0] as string; if (tap) { setGrid((g) => cycleTileSize(tileId, g)); } else { - // TODO + const tileSpring = springRef.current + .find((c) => (c.item as Tile).item.id === tileId)! + .get(); + + if (dragState.current === null) { + dragState.current = { tileId, x: tileSpring.x, y: tileSpring.y }; + } + dragState.current.x += dx; + dragState.current.y += dy; + + animateDraggedTile(last); + + if (last) dragState.current = null; } }, { filterTaps: true, pointer: { buttons: [1] } } ); + const scrollOffset = useRef(0); + + const bindGrid = useScroll(({ xy: [, y], delta: [, dy] }) => { + scrollOffset.current = y; + + if (dragState.current !== null) { + dragState.current.y += dy; + animateDraggedTile(false); + } + }); + const slots = useMemo(() => { const slots = new Array(items.length); for (let i = 0; i < items.length; i++) @@ -554,7 +639,7 @@ export const NewVideoGrid: FC = ({ } return ( -
+
Date: Wed, 1 Feb 2023 11:32:10 -0500 Subject: [PATCH 18/40] Implement somewhat working drag & drop and improve render memoization --- src/useMergedRefs.ts | 16 +++ src/video-grid/NewVideoGrid.tsx | 144 ++++++++++++-------- src/video-grid/VideoGrid.tsx | 17 ++- src/video-grid/VideoTile.tsx | 36 ++++- src/video-grid/VideoTileContainer.tsx | 185 +++++++++++++++----------- src/video-grid/useMediaStream.ts | 4 +- 6 files changed, 256 insertions(+), 146 deletions(-) create mode 100644 src/useMergedRefs.ts diff --git a/src/useMergedRefs.ts b/src/useMergedRefs.ts new file mode 100644 index 00000000..9931505d --- /dev/null +++ b/src/useMergedRefs.ts @@ -0,0 +1,16 @@ +import { MutableRefObject, RefCallback, useCallback } from "react"; + +export const useMergedRefs = ( + ...refs: (MutableRefObject | RefCallback)[] +): RefCallback => + useCallback( + (value) => + refs.forEach((ref) => { + if (typeof ref === "function") { + ref(value); + } else { + ref.current = value; + } + }), + refs + ); diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 13d1860f..a916f681 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -1,5 +1,5 @@ import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; -import { useDrag, useScroll } from "@use-gesture/react"; +import { EventTypes, Handler, useScroll } from "@use-gesture/react"; import React, { FC, ReactNode, @@ -15,6 +15,7 @@ import { VideoGridProps as Props } from "./VideoGrid"; import { useReactiveState } from "../useReactiveState"; import TinyQueue from "tinyqueue"; import { zipWith } from "lodash"; +import { useMergedRefs } from "../useMergedRefs"; interface Cell { /** @@ -66,8 +67,10 @@ interface TileSpring { interface DragState { tileId: string; - x: number; - y: number; + tileX: number; + tileY: number; + cursorX: number; + cursorY: number; } const dijkstra = (g: Grid): number[] => { @@ -377,7 +380,10 @@ export const NewVideoGrid: FC = ({ }) => { const [slotGrid, setSlotGrid] = useState(null); const [slotGridGeneration, setSlotGridGeneration] = useState(0); - const [gridRef, gridBounds] = useMeasure(); + + const [gridRef1, gridBounds] = useMeasure(); + const gridRef2 = useRef(null); + const gridRef = useMergedRefs(gridRef1, gridRef2); useEffect(() => { if (slotGrid !== null) { @@ -549,23 +555,21 @@ export const NewVideoGrid: FC = ({ }; }, [grid]); - const animateDraggedTile = (endOfGesture: boolean) => - springRef.start((_i, controller) => { - const { tileId, x, y } = dragState.current!; + const animateDraggedTile = (endOfGesture: boolean) => { + const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!; + const tile = tiles.find((t) => t.item.id === tileId)!; - // react-spring appears to not update a controller's item as long as the - // key remains stable, so we can use it to look up the tile's ID but not - // its position + springRef.start((_i, controller) => { if ((controller.item as Tile).item.id === tileId) { if (endOfGesture) { - const tile = tiles.find((t) => t.item.id === tileId)!; - return { scale: 1, zIndex: 1, shadow: 1, x: tile.x, y: tile.y, + width: tile.width, + height: tile.height, immediate: disableAnimations || ((key) => key === "zIndex"), // Allow the tile's position to settle before pushing its // z-index back down @@ -576,8 +580,8 @@ export const NewVideoGrid: FC = ({ scale: 1.1, zIndex: 2, shadow: 15, - x, - y, + x: tileX, + y: tileY, immediate: disableAnimations || ((key) => key === "zIndex" || key === "x" || key === "y"), @@ -588,41 +592,78 @@ export const NewVideoGrid: FC = ({ } }); - const bindTile = useDrag( - ({ tap, args, delta: [dx, dy], last }) => { - const tileId = args[0] as string; + const overTile = tiles.find( + (t) => + cursorX >= t.x && + cursorX < t.x + t.width && + cursorY >= t.y && + cursorY < t.y + t.height + ); + if (overTile !== undefined && overTile.item.id !== tileId) { + setGrid((g) => ({ + ...g, + cells: g.cells.map((c) => { + if (c?.item === overTile.item) return { ...c, item: tile.item }; + if (c?.item === tile.item) return { ...c, item: overTile.item }; + return c; + }), + })); + } + }; - if (tap) { - setGrid((g) => cycleTileSize(tileId, g)); - } else { - const tileSpring = springRef.current - .find((c) => (c.item as Tile).item.id === tileId)! - .get(); + const onTileDrag = ( + tileId: string, + { + tap, + initial: [initialX, initialY], + delta: [dx, dy], + last, + }: Parameters>[0] + ) => { + if (tap) { + setGrid((g) => cycleTileSize(tileId, g)); + } else { + const tileSpring = springRef.current + .find((c) => (c.item as Tile).item.id === tileId)! + .get(); - if (dragState.current === null) { - dragState.current = { tileId, x: tileSpring.x, y: tileSpring.y }; - } - dragState.current.x += dx; - dragState.current.y += dy; - - animateDraggedTile(last); - - if (last) dragState.current = null; + if (dragState.current === null) { + dragState.current = { + tileId, + tileX: tileSpring.x, + tileY: tileSpring.y, + cursorX: initialX - gridBounds.x, + cursorY: initialY - gridBounds.y + scrollOffset.current, + }; } - }, - { filterTaps: true, pointer: { buttons: [1] } } - ); + dragState.current.tileX += dx; + dragState.current.tileY += dy; + dragState.current.cursorX += dx; + dragState.current.cursorY += dy; + + animateDraggedTile(last); + + if (last) dragState.current = null; + } + }; + + const onTileDragRef = useRef(onTileDrag); + onTileDragRef.current = onTileDrag; const scrollOffset = useRef(0); - const bindGrid = useScroll(({ xy: [, y], delta: [, dy] }) => { - scrollOffset.current = y; + useScroll( + ({ xy: [, y], delta: [, dy] }) => { + scrollOffset.current = y; - if (dragState.current !== null) { - dragState.current.y += dy; - animateDraggedTile(false); - } - }); + if (dragState.current !== null) { + dragState.current.tileY += dy; + dragState.current.cursorY += dy; + animateDraggedTile(false); + } + }, + { target: gridRef2 } + ); const slots = useMemo(() => { const slots = new Array(items.length); @@ -639,7 +680,7 @@ export const NewVideoGrid: FC = ({ } return ( -
+
= ({ > {slots}
- {tileTransitions(({ shadow, width, height, ...style }, tile) => + {tileTransitions((style, tile) => children({ - ...bindTile(tile.item.id), + ...style, key: tile.item.id, - style: { - boxShadow: shadow.to( - (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px` - ), - "--tileWidth": width.to((w) => `${w}px`), - "--tileHeight": height.to((h) => `${h}px`), - ...style, - }, - width: tile.width, - height: tile.height, + targetWidth: tile.width, + targetHeight: tile.height, item: tile.item, + onDragRef: onTileDragRef, }) )}
diff --git a/src/video-grid/VideoGrid.tsx b/src/video-grid/VideoGrid.tsx index fc6e44d8..2c6cb5b9 100644 --- a/src/video-grid/VideoGrid.tsx +++ b/src/video-grid/VideoGrid.tsx @@ -694,14 +694,17 @@ interface DragTileData { interface ChildrenProperties extends ReactDOMAttributes { key: Key; - style: { - scale: SpringValue; - opacity: SpringValue; - boxShadow: Interpolation; - }; - width: number; - height: number; + targetWidth: number; + targetHeight: number; item: TileDescriptor; + opacity: SpringValue; + scale: SpringValue; + shadow: SpringValue; + zIndex: SpringValue; + x: SpringValue; + y: SpringValue; + width: SpringValue; + height: SpringValue; [index: string]: unknown; } diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 4e04199f..ca9ebbea 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { forwardRef } from "react"; -import { animated } from "@react-spring/web"; +import React, { ForwardedRef, forwardRef } from "react"; +import { animated, SpringValue } from "@react-spring/web"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; @@ -44,9 +44,17 @@ interface Props { showOptions?: boolean; isLocal?: boolean; disableSpeakingIndicator?: boolean; + opacity: SpringValue; + scale: SpringValue; + shadow: SpringValue; + zIndex: SpringValue; + x: SpringValue; + y: SpringValue; + width: SpringValue; + height: SpringValue; } -export const VideoTile = forwardRef( +export const VideoTile = forwardRef( ( { name, @@ -68,6 +76,14 @@ export const VideoTile = forwardRef( isLocal, // TODO: disableSpeakingIndicator is not used atm. disableSpeakingIndicator, + opacity, + scale, + shadow, + zIndex, + x, + y, + width, + height, ...rest }, ref @@ -122,7 +138,19 @@ export const VideoTile = forwardRef( [styles.screenshare]: screenshare, [styles.maximised]: maximised, })} - ref={ref} + style={{ + opacity, + scale, + boxShadow: shadow.to( + (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px` + ), + zIndex, + x, + y, + "--tileWidth": width.to((w) => `${w}px`), + "--tileHeight": height.to((h) => `${h}px`), + }} + ref={ref as ForwardedRef} {...rest} > {toolbarButtons.length > 0 && !maximised && ( diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index 1741ca85..28aec61d 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -15,7 +15,7 @@ limitations under the License. */ import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes"; -import React from "react"; +import React, { FC, memo, RefObject } from "react"; import { useCallback } from "react"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; @@ -26,11 +26,13 @@ import { VideoTile } from "./VideoTile"; import { VideoTileSettingsModal } from "./VideoTileSettingsModal"; import { useModalTriggerState } from "../Modal"; import { TileDescriptor } from "./TileDescriptor"; +import { SpringValue } from "@react-spring/web"; +import { EventTypes, Handler, useDrag } from "@use-gesture/react"; interface Props { item: TileDescriptor; - width?: number; - height?: number; + targetWidth: number; + targetHeight: number; getAvatar: ( roomMember: RoomMember, width: number, @@ -42,86 +44,113 @@ interface Props { maximised: boolean; fullscreen: boolean; onFullscreen: (item: TileDescriptor) => void; + opacity: SpringValue; + scale: SpringValue; + shadow: SpringValue; + zIndex: SpringValue; + x: SpringValue; + y: SpringValue; + width: SpringValue; + height: SpringValue; + onDragRef: RefObject< + ( + tileId: string, + state: Parameters>[0] + ) => void + >; } -export function VideoTileContainer({ - item, - width, - height, - getAvatar, - audioContext, - audioDestination, - disableSpeakingIndicator, - maximised, - fullscreen, - onFullscreen, - ...rest -}: Props) { - const { - isLocal, - audioMuted, - videoMuted, - localVolume, - hasAudio, - speaking, - stream, - purpose, - } = useCallFeed(item.callFeed); - const { rawDisplayName } = useRoomMemberName(item.member); - const [tileRef, mediaRef] = useSpatialMediaStream( - stream ?? null, +export const VideoTileContainer: FC = memo( + ({ + item, + targetWidth, + targetHeight, + getAvatar, audioContext, audioDestination, - localVolume, - // The feed is muted if it's local audio (because we don't want our own audio, - // but it's a hook and we can't call it conditionally so we're stuck with it) - // or if there's a maximised feed in which case we always render audio via audio - // elements because we wire it up at the video tile container level and only one - // video tile container is displayed. - isLocal || maximised - ); - const { - modalState: videoTileSettingsModalState, - modalProps: videoTileSettingsModalProps, - } = useModalTriggerState(); - const onOptionsPress = () => { - videoTileSettingsModalState.open(); - }; + disableSpeakingIndicator, + maximised, + fullscreen, + onFullscreen, + onDragRef, + ...rest + }) => { + const { + isLocal, + audioMuted, + videoMuted, + localVolume, + hasAudio, + speaking, + stream, + purpose, + } = useCallFeed(item.callFeed); + const { rawDisplayName } = useRoomMemberName(item.member); - const onFullscreenCallback = useCallback(() => { - onFullscreen(item); - }, [onFullscreen, item]); + const [tileRef, mediaRef] = useSpatialMediaStream( + stream ?? null, + audioContext, + audioDestination, + localVolume, + // The feed is muted if it's local audio (because we don't want our own audio, + // but it's a hook and we can't call it conditionally so we're stuck with it) + // or if there's a maximised feed in which case we always render audio via audio + // elements because we wire it up at the video tile container level and only one + // video tile container is displayed. + isLocal || maximised + ); - // Firefox doesn't respect the disablePictureInPicture attribute - // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831 + useDrag((state) => onDragRef.current!(item.id, state), { + target: tileRef, + filterTaps: true, + pointer: { buttons: [1] }, + }); - return ( - <> - - {videoTileSettingsModalState.isOpen && !maximised && item.callFeed && ( - { + videoTileSettingsModalState.open(); + }; + + const onFullscreenCallback = useCallback(() => { + onFullscreen(item); + }, [onFullscreen, item]); + + // Firefox doesn't respect the disablePictureInPicture attribute + // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831 + + return ( + <> + - )} - - ); -} + {videoTileSettingsModalState.isOpen && !maximised && item.callFeed && ( + + )} + + ); + } +); diff --git a/src/video-grid/useMediaStream.ts b/src/video-grid/useMediaStream.ts index 259ede7b..9eb8e5aa 100644 --- a/src/video-grid/useMediaStream.ts +++ b/src/video-grid/useMediaStream.ts @@ -158,8 +158,8 @@ export const useSpatialMediaStream = ( audioDestination: AudioNode, localVolume: number, mute = false -): [RefObject, RefObject] => { - const tileRef = useRef(null); +): [RefObject, RefObject] => { + const tileRef = useRef(null); const [spatialAudio] = useSpatialAudio(); // This media stream is only used for the video - the audio goes via the audio From 6784d2ba97e7aecc032796beca8cbf09c749adb3 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 1 Feb 2023 11:50:25 -0500 Subject: [PATCH 19/40] Remove redundant key prop --- src/room/InCallView.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 88330dcc..a0f7917f 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -319,7 +319,6 @@ export function InCallView({ [x: string]: unknown; }) => ( Date: Wed, 1 Feb 2023 11:50:52 -0500 Subject: [PATCH 20/40] Keep tile elements in a stable order --- src/video-grid/NewVideoGrid.tsx | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index a916f681..4a9d3f73 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -470,13 +470,8 @@ export const NewVideoGrid: FC = ({ if (slotGridGeneration !== grid.generation) return prevTiles ?? []; const slotCells = grid.cells.filter((c) => c?.slot) as Cell[]; - return zipWith(slotCells, slotRects, (cell, rect) => ({ - item: cell.item, - x: rect.x, - y: rect.y, - width: rect.width, - height: rect.height, - })); + const tileRects = new Map(zipWith(slotCells, slotRects, (cell, rect) => [cell.item, rect])) + return items.map(item => ({ ...tileRects.get(item)!, item })) }, [slotRects, grid, slotGridGeneration] ); From 42e4f6ce83f0be8589e325df21eedc9a83a5982d Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Fri, 3 Feb 2023 08:44:35 -0500 Subject: [PATCH 21/40] Don't allow the grid to overflow horizontally --- src/video-grid/NewVideoGrid.module.css | 1 + 1 file changed, 1 insertion(+) diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index 27c453ff..a7179f4d 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -4,6 +4,7 @@ flex-grow: 1; padding: 0 22px; overflow-y: auto; + overflow-x: none; } .slotGrid { From 6cd939db0c2bbdce69913b838d558b684e7dcda3 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Fri, 3 Feb 2023 09:11:25 -0500 Subject: [PATCH 22/40] Fix a crash when there's only 1 tile and it gets shrunk --- src/video-grid/NewVideoGrid.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 4a9d3f73..bf77c918 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -119,7 +119,7 @@ const findLastIndex = ( array: T[], predicate: (item: T) => boolean ): number | null => { - for (let i = array.length - 1; i > 0; i--) { + for (let i = array.length - 1; i >= 0; i--) { if (predicate(array[i])) return i; } From 22382413dcb0227c3863440fc747fe6aee66ea02 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Fri, 3 Feb 2023 15:42:47 -0500 Subject: [PATCH 23/40] Make drag and drop mobile-friendly --- src/video-grid/VideoTile.module.css | 1 - src/video-grid/VideoTileContainer.tsx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css index 05f3a0ce..543e4fa3 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/video-grid/VideoTile.module.css @@ -23,7 +23,6 @@ limitations under the License. border-radius: 20px; overflow: hidden; cursor: pointer; - touch-action: none; /* HACK: This has no visual effect due to the short duration, but allows the JS to detect movement via the transform property for audio spatialization */ diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index 28aec61d..5bffc6c6 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -103,7 +103,7 @@ export const VideoTileContainer: FC = memo( useDrag((state) => onDragRef.current!(item.id, state), { target: tileRef, filterTaps: true, - pointer: { buttons: [1] }, + preventScroll: true, }); const { From 1e858f6ba3037d69f03bb8e31316e0073e87cf98 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Fri, 3 Feb 2023 16:27:49 -0500 Subject: [PATCH 24/40] Fix a typo --- src/video-grid/NewVideoGrid.module.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index a7179f4d..7d0a8964 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -4,7 +4,7 @@ flex-grow: 1; padding: 0 22px; overflow-y: auto; - overflow-x: none; + overflow-x: hidden; } .slotGrid { From 206730ffc0b28c4413356b5127a272f0fb762ec7 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Fri, 3 Feb 2023 16:52:42 -0500 Subject: [PATCH 25/40] Fix infinite loop when a tile can't be enlarged --- src/video-grid/NewVideoGrid.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index bf77c918..f4b5811d 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -308,6 +308,7 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const nextScanLocations = new Set([from]); const scanColumnOffset = Math.floor((toWidth - 1) / 2); const scanRowOffset = Math.floor((toHeight - 1) / 2); + const rows = row(g.cells.length - 1, g) + 1 let to: number | null = null; const displaceable = (c: Cell | undefined, i: number): boolean => @@ -319,6 +320,7 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset; const end = areaEnd(start, candidateWidth, candidateHeight, g); const startColumn = column(start, g); + const startRow = row(start, g) const endColumn = column(end, g); if ( @@ -334,8 +336,8 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { if (startColumn > 0) nextScanLocations.add(scanLocation - 1); if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1); - nextScanLocations.add(scanLocation - g.columns); - nextScanLocations.add(scanLocation + g.columns); + if (startRow > 0) nextScanLocations.add(scanLocation - g.columns); + if (startRow <= rows) nextScanLocations.add(scanLocation + g.columns); } // TODO: Don't give up on placing the tile yet From 6adcf95aaa08b2148c5dbd7aaa99da09ffa89ac7 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Sat, 4 Feb 2023 00:43:53 -0500 Subject: [PATCH 26/40] Implement different column counts and mobile layout --- src/room/InCallView.module.css | 32 ++++++++++---- src/video-grid/NewVideoGrid.module.css | 21 ++++++--- src/video-grid/NewVideoGrid.tsx | 60 ++++++++++++++++++-------- src/video-grid/VideoTile.module.css | 8 +++- 4 files changed, 90 insertions(+), 31 deletions(-) diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 310cf47b..04228aa9 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -20,9 +20,10 @@ limitations under the License. flex-direction: column; overflow: hidden; min-height: 100%; - position: fixed; height: 100%; width: 100%; + --footerPadding: 8px; + --footerHeight: calc(50px + 2 * var(--footerPadding)); } .centerMessage { @@ -39,12 +40,27 @@ limitations under the License. } .footer { - position: relative; - flex-shrink: 0; + position: absolute; + left: 0; + bottom: 0; + width: 100%; display: flex; justify-content: center; align-items: center; - height: calc(50px + 2 * 8px); + padding: var(--footerPadding) 0; + /* TODO: Un-hardcode these colors */ + background: linear-gradient( + 360deg, + #15191e 0%, + rgba(21, 25, 30, 0.9) 37%, + rgba(21, 25, 30, 0.8) 49.68%, + rgba(21, 25, 30, 0.7) 56.68%, + rgba(21, 25, 30, 0.427397) 72.92%, + rgba(21, 25, 30, 0.257534) 81.06%, + rgba(21, 25, 30, 0.136986) 87.29%, + rgba(21, 25, 30, 0.0658079) 92.4%, + rgba(21, 25, 30, 0) 100% + ); } .footer > * { @@ -75,13 +91,13 @@ limitations under the License. } @media (min-height: 300px) { - .footer { - height: calc(50px + 2 * 24px); + .inRoom { + --footerPadding: 24px; } } @media (min-width: 800px) { - .footer { - height: calc(50px + 2 * 32px); + .inRoom { + --footerPadding: 32px; } } diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index 7d0a8964..e6f64ba9 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -2,20 +2,31 @@ contain: strict; position: relative; flex-grow: 1; - padding: 0 22px; + padding: 0 20px; overflow-y: auto; overflow-x: hidden; } .slotGrid { - contain: strict; position: relative; display: grid; - grid-auto-rows: 183px; - column-gap: 18px; - row-gap: 21px; + grid-auto-rows: 163px; + gap: 8px; + padding-bottom: var(--footerHeight); } .slot { contain: strict; } + +@media (min-width: 800px) { + .grid { + padding: 0 22px; + } + + .slotGrid { + grid-auto-rows: 183px; + column-gap: 18px; + row-gap: 21px; + } +} diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index f4b5811d..b4d13fc0 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -290,7 +290,9 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const fromEnd = areaEnd(from, fromWidth, fromHeight, g); const [toWidth, toHeight] = - fromWidth === 1 && fromHeight === 1 ? [3, 2] : [1, 1]; + fromWidth === 1 && fromHeight === 1 + ? [Math.min(3, Math.max(2, g.columns - 1)), 2] + : [1, 1]; const newRows = Math.max( 0, Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) @@ -308,7 +310,7 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const nextScanLocations = new Set([from]); const scanColumnOffset = Math.floor((toWidth - 1) / 2); const scanRowOffset = Math.floor((toHeight - 1) / 2); - const rows = row(g.cells.length - 1, g) + 1 + const rows = row(g.cells.length - 1, g) + 1; let to: number | null = null; const displaceable = (c: Cell | undefined, i: number): boolean => @@ -320,7 +322,7 @@ const cycleTileSize = (tileId: string, g: Grid): Grid => { const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset; const end = areaEnd(start, candidateWidth, candidateHeight, g); const startColumn = column(start, g); - const startRow = row(start, g) + const startRow = row(start, g); const endColumn = column(end, g); if ( @@ -424,8 +426,30 @@ export const NewVideoGrid: FC = ({ return rects; }, [items, slotGridGeneration, slotGrid, gridBounds]); - const [grid, setGrid] = useReactiveState( - (prevGrid = { generation: 0, columns: 6, cells: [] }) => { + const [columns] = useReactiveState( + // Since grid resizing isn't implemented yet, pick a column count on mount + // and stick to it + (prevColumns) => + prevColumns !== undefined && prevColumns !== null + ? prevColumns + : // The grid bounds might not be known yet + gridBounds.width === 0 + ? null + : Math.max(2, Math.floor(gridBounds.width * 0.0045)), + [gridBounds] + ); + + const [grid, setGrid] = useReactiveState( + (prevGrid = null) => { + if (prevGrid === null) { + // We can't do anything if the column count isn't known yet + if (columns === null) { + return null; + } else { + prevGrid = { generation: slotGridGeneration, columns, cells: [] }; + } + } + // Step 1: Update tiles that still exist, and remove tiles that have left // the grid const itemsById = new Map(items.map((i) => [i.id, i])); @@ -462,18 +486,20 @@ export const NewVideoGrid: FC = ({ return grid3; }, - [items] + [items, columns] ); const [tiles] = useReactiveState( (prevTiles) => { // If React hasn't yet rendered the current generation of the layout, skip // the update, because grid and slotRects will be out of sync - if (slotGridGeneration !== grid.generation) return prevTiles ?? []; + if (slotGridGeneration !== grid?.generation) return prevTiles ?? []; const slotCells = grid.cells.filter((c) => c?.slot) as Cell[]; - const tileRects = new Map(zipWith(slotCells, slotRects, (cell, rect) => [cell.item, rect])) - return items.map(item => ({ ...tileRects.get(item)!, item })) + const tileRects = new Map( + zipWith(slotCells, slotRects, (cell, rect) => [cell.item, rect]) + ); + return items.map((item) => ({ ...tileRects.get(item)!, item })); }, [slotRects, grid, slotGridGeneration] ); @@ -516,7 +542,7 @@ export const NewVideoGrid: FC = ({ ) as unknown as [TransitionFn, SpringRef]; const slotGridStyle = useMemo(() => { - const columnCount = 6; + if (grid === null) return {}; const areas = new Array<(number | null)[]>( Math.ceil(grid.cells.length / grid.columns) @@ -548,9 +574,9 @@ export const NewVideoGrid: FC = ({ .join(" ")}'` ) .join(" "), - gridTemplateColumns: `repeat(${columnCount}, 1fr)`, + gridTemplateColumns: `repeat(${columns}, 1fr)`, }; - }, [grid]); + }, [grid, columns]); const animateDraggedTile = (endOfGesture: boolean) => { const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!; @@ -598,8 +624,8 @@ export const NewVideoGrid: FC = ({ ); if (overTile !== undefined && overTile.item.id !== tileId) { setGrid((g) => ({ - ...g, - cells: g.cells.map((c) => { + ...g!, + cells: g!.cells.map((c) => { if (c?.item === overTile.item) return { ...c, item: tile.item }; if (c?.item === tile.item) return { ...c, item: overTile.item }; return c; @@ -618,7 +644,7 @@ export const NewVideoGrid: FC = ({ }: Parameters>[0] ) => { if (tap) { - setGrid((g) => cycleTileSize(tileId, g)); + setGrid((g) => cycleTileSize(tileId, g!)); } else { const tileSpring = springRef.current .find((c) => (c.item as Tile).item.id === tileId)! @@ -671,8 +697,8 @@ export const NewVideoGrid: FC = ({ return slots; }, [items.length]); - // Render nothing if the bounds are not yet known - if (gridBounds.width === 0) { + // Render nothing if the grid has yet to be generated + if (grid === null) { return
; } diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css index 543e4fa3..ce44d225 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/video-grid/VideoTile.module.css @@ -20,7 +20,7 @@ limitations under the License. top: 0; width: var(--tileWidth); height: var(--tileHeight); - border-radius: 20px; + border-radius: 8px; overflow: hidden; cursor: pointer; @@ -174,3 +174,9 @@ limitations under the License. max-width: 360px; border-radius: 20px; } + +@media (min-width: 800px) { + .videoTile { + border-radius: 20px; + } +} From 82ac775124e927bf0e04481706ab3335d4ab7fb5 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Sun, 5 Feb 2023 00:55:12 -0500 Subject: [PATCH 27/40] Fix scrolling on mobile --- src/video-grid/VideoTile.module.css | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css index ce44d225..8da1f760 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/video-grid/VideoTile.module.css @@ -30,9 +30,6 @@ limitations under the License. } .videoTile * { - touch-action: none; - -moz-user-select: none; - -webkit-user-drag: none; user-select: none; } From 374c68e3c053cd103e849d6db3cc57b990de31be Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Sun, 5 Feb 2023 01:17:28 -0500 Subject: [PATCH 28/40] Fix tiles enlarging to the wrong place on mobile --- src/video-grid/NewVideoGrid.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index b4d13fc0..d22ad6e2 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -130,7 +130,8 @@ const findLast1By1Index = (g: Grid): number | null => findLastIndex(g.cells, (c) => c?.rows === 1 && c?.columns === 1); const row = (index: number, g: Grid): number => Math.floor(index / g.columns); -const column = (index: number, g: Grid): number => index % g.columns; +const column = (index: number, g: Grid): number => + ((index % g.columns) + g.columns) % g.columns; const inArea = ( index: number, From 978b0f08e8f2ccd7d7eb7d599afc7b77fb4873e8 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Tue, 7 Feb 2023 22:13:50 -0500 Subject: [PATCH 29/40] Move grid algorithms into a separate file --- src/video-grid/NewVideoGrid.tsx | 357 +++----------------------------- src/video-grid/model.ts | 348 +++++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+), 332 deletions(-) create mode 100644 src/video-grid/model.ts diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index d22ad6e2..2c25f62d 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -1,3 +1,19 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; import { EventTypes, Handler, useScroll } from "@use-gesture/react"; import React, { @@ -13,35 +29,17 @@ import styles from "./NewVideoGrid.module.css"; import { TileDescriptor } from "./TileDescriptor"; import { VideoGridProps as Props } from "./VideoGrid"; import { useReactiveState } from "../useReactiveState"; -import TinyQueue from "tinyqueue"; import { zipWith } from "lodash"; import { useMergedRefs } from "../useMergedRefs"; - -interface Cell { - /** - * The item held by the slot containing this cell. - */ - item: TileDescriptor; - /** - * Whether this cell is the first cell of the containing slot. - */ - // TODO: Rename to 'start'? - slot: boolean; - /** - * The width, in columns, of the containing slot. - */ - columns: number; - /** - * The height, in rows, of the containing slot. - */ - rows: number; -} - -interface Grid { - generation: number; - columns: number; - cells: (Cell | undefined)[]; -} +import { + Grid, + Cell, + row, + column, + fillGaps, + forEachCellInArea, + cycleTileSize, +} from "./model"; interface Rect { x: number; @@ -73,311 +71,6 @@ interface DragState { cursorY: number; } -const dijkstra = (g: Grid): number[] => { - const end = findLast1By1Index(g) ?? 0; - const endRow = row(end, g); - const endColumn = column(end, g); - - const distances = new Array(end + 1).fill(Infinity); - distances[end] = 0; - const edges = new Array(end).fill(undefined); - const heap = new TinyQueue([end], (i) => distances[i]); - - const visit = (curr: number, via: number) => { - const viaCell = g.cells[via]; - const viaLargeSlot = - viaCell !== undefined && (viaCell.rows > 1 || viaCell.columns > 1); - const distanceVia = distances[via] + (viaLargeSlot ? 4 : 1); - - if (distanceVia < distances[curr]) { - distances[curr] = distanceVia; - edges[curr] = via; - heap.push(curr); - } - }; - - while (heap.length > 0) { - const via = heap.pop()!; - const viaRow = row(via, g); - const viaColumn = column(via, g); - - if (viaRow > 0) visit(via - g.columns, via); - if (viaColumn > 0) visit(via - 1, via); - if (viaColumn < (viaRow === endRow ? endColumn : g.columns - 1)) - visit(via + 1, via); - if ( - viaRow < endRow - 1 || - (viaRow === endRow - 1 && viaColumn <= endColumn) - ) - visit(via + g.columns, via); - } - - return edges as number[]; -}; - -const findLastIndex = ( - array: T[], - predicate: (item: T) => boolean -): number | null => { - for (let i = array.length - 1; i >= 0; i--) { - if (predicate(array[i])) return i; - } - - return null; -}; - -const findLast1By1Index = (g: Grid): number | null => - findLastIndex(g.cells, (c) => c?.rows === 1 && c?.columns === 1); - -const row = (index: number, g: Grid): number => Math.floor(index / g.columns); -const column = (index: number, g: Grid): number => - ((index % g.columns) + g.columns) % g.columns; - -const inArea = ( - index: number, - start: number, - end: number, - g: Grid -): boolean => { - const indexColumn = column(index, g); - const indexRow = row(index, g); - return ( - indexRow >= row(start, g) && - indexRow <= row(end, g) && - indexColumn >= column(start, g) && - indexColumn <= column(end, g) - ); -}; - -function* cellsInArea( - start: number, - end: number, - g: Grid -): Generator { - const startColumn = column(start, g); - const endColumn = column(end, g); - for ( - let i = start; - i <= end; - i = - column(i, g) === endColumn - ? i + g.columns + startColumn - endColumn - : i + 1 - ) - yield i; -} - -const forEachCellInArea = ( - start: number, - end: number, - g: Grid, - fn: (c: Cell | undefined, i: number) => void -) => { - for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i); -}; - -const allCellsInArea = ( - start: number, - end: number, - g: Grid, - fn: (c: Cell | undefined, i: number) => boolean -) => { - for (const i of cellsInArea(start, end, g)) { - if (!fn(g.cells[i], i)) return false; - } - - return true; -}; - -const areaEnd = ( - start: number, - columns: number, - rows: number, - g: Grid -): number => start + columns - 1 + g.columns * (rows - 1); - -/** - * Gets the index of the next gap in the grid that should be backfilled by 1×1 - * tiles. - */ -const getNextGap = (g: Grid): number | null => { - const last1By1Index = findLast1By1Index(g); - if (last1By1Index === null) return null; - - for (let i = 0; i < last1By1Index; i++) { - // To make the backfilling process look natural when there are multiple - // gaps, we actually scan each row from right to left - const j = - (row(i, g) === row(last1By1Index, g) - ? last1By1Index - : (row(i, g) + 1) * g.columns) - - 1 - - column(i, g); - - if (g.cells[j] === undefined) return j; - } - - return null; -}; - -const fillGaps = (g: Grid): Grid => { - const result: Grid = { ...g, cells: [...g.cells] }; - let gap = getNextGap(result); - - if (gap !== null) { - const pathToEnd = dijkstra(result); - - do { - let filled = false; - let to = gap; - let from: number | undefined = pathToEnd[gap]; - - // First, attempt to fill the gap by moving 1×1 tiles backwards from the - // end of the grid along a set path - while (from !== undefined) { - const toCell = result.cells[to]; - const fromCell = result.cells[from]; - - // Skip over large tiles - if (toCell !== undefined) { - to = pathToEnd[to]; - // Skip over large tiles. Also, we might run into gaps along the path - // created during the filling of previous gaps. Skip over those too; - // they'll be picked up on the next iteration of the outer loop. - } else if ( - fromCell === undefined || - fromCell.rows > 1 || - fromCell.columns > 1 - ) { - from = pathToEnd[from]; - } else { - result.cells[to] = result.cells[from]; - result.cells[from] = undefined; - filled = true; - to = pathToEnd[to]; - from = pathToEnd[from]; - } - } - - // In case the path approach failed, fall back to taking the very last 1×1 - // tile, and just dropping it into place - if (!filled) { - const last1By1Index = findLast1By1Index(result)!; - result.cells[gap] = result.cells[last1By1Index]; - result.cells[last1By1Index] = undefined; - } - - gap = getNextGap(result); - } while (gap !== null); - } - - // TODO: If there are any large tiles on the last row, shuffle them back - // upwards into a full row - - // Shrink the array to remove trailing gaps - const finalLength = - (findLastIndex(result.cells, (c) => c !== undefined) ?? -1) + 1; - if (finalLength < result.cells.length) - result.cells = result.cells.slice(0, finalLength); - - return result; -}; - -const cycleTileSize = (tileId: string, g: Grid): Grid => { - const from = g.cells.findIndex((c) => c?.item.id === tileId); - if (from === -1) return g; // Tile removed, no change - const fromWidth = g.cells[from]!.columns; - const fromHeight = g.cells[from]!.rows; - const fromEnd = areaEnd(from, fromWidth, fromHeight, g); - - const [toWidth, toHeight] = - fromWidth === 1 && fromHeight === 1 - ? [Math.min(3, Math.max(2, g.columns - 1)), 2] - : [1, 1]; - const newRows = Math.max( - 0, - Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) - ); - - const candidateWidth = toWidth; - const candidateHeight = toHeight - newRows; - - const gappyGrid: Grid = { - ...g, - generation: g.generation + 1, - cells: new Array(g.cells.length + newRows * g.columns), - }; - - const nextScanLocations = new Set([from]); - const scanColumnOffset = Math.floor((toWidth - 1) / 2); - const scanRowOffset = Math.floor((toHeight - 1) / 2); - const rows = row(g.cells.length - 1, g) + 1; - let to: number | null = null; - - const displaceable = (c: Cell | undefined, i: number): boolean => - c === undefined || - (c.columns === 1 && c.rows === 1) || - inArea(i, from, fromEnd, g); - - for (const scanLocation of nextScanLocations) { - const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset; - const end = areaEnd(start, candidateWidth, candidateHeight, g); - const startColumn = column(start, g); - const startRow = row(start, g); - const endColumn = column(end, g); - - if ( - start >= 0 && - end < gappyGrid.cells.length && - endColumn - startColumn + 1 === candidateWidth - ) { - if (allCellsInArea(start, end, g, displaceable)) { - to = start; - break; - } - } - - if (startColumn > 0) nextScanLocations.add(scanLocation - 1); - if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1); - if (startRow > 0) nextScanLocations.add(scanLocation - g.columns); - if (startRow <= rows) nextScanLocations.add(scanLocation + g.columns); - } - - // TODO: Don't give up on placing the tile yet - if (to === null) return g; - - const toRow = row(to, g); - - g.cells.forEach((c, src) => { - if (c?.slot && c.item.id !== tileId) { - const offset = - row(src, g) > toRow + candidateHeight - 1 ? g.columns * newRows : 0; - forEachCellInArea(src, areaEnd(src, c.columns, c.rows, g), g, (c, i) => { - gappyGrid.cells[i + offset] = c; - }); - } - }); - - const displacedTiles: Cell[] = []; - const toEnd = areaEnd(to, toWidth, toHeight, g); - forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { - if (c !== undefined) displacedTiles.push(c); - gappyGrid.cells[i] = { - item: g.cells[from]!.item, - slot: i === to, - columns: toWidth, - rows: toHeight, - }; - }); - - for (let i = 0; displacedTiles.length > 0; i++) { - if (gappyGrid.cells[i] === undefined) - gappyGrid.cells[i] = displacedTiles.shift(); - } - - return fillGaps(gappyGrid); -}; - export const NewVideoGrid: FC = ({ items, disableAnimations, diff --git a/src/video-grid/model.ts b/src/video-grid/model.ts new file mode 100644 index 00000000..0a4136c6 --- /dev/null +++ b/src/video-grid/model.ts @@ -0,0 +1,348 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import TinyQueue from "tinyqueue"; +import { TileDescriptor } from "./TileDescriptor"; + +export interface Cell { + /** + * The item held by the slot containing this cell. + */ + item: TileDescriptor; + /** + * Whether this cell is the first cell of the containing slot. + */ + // TODO: Rename to 'start'? + slot: boolean; + /** + * The width, in columns, of the containing slot. + */ + columns: number; + /** + * The height, in rows, of the containing slot. + */ + rows: number; +} + +export interface Grid { + generation: number; + columns: number; + cells: (Cell | undefined)[]; +} + +export function dijkstra(g: Grid): number[] { + const end = findLast1By1Index(g) ?? 0; + const endRow = row(end, g); + const endColumn = column(end, g); + + const distances = new Array(end + 1).fill(Infinity); + distances[end] = 0; + const edges = new Array(end).fill(undefined); + const heap = new TinyQueue([end], (i) => distances[i]); + + const visit = (curr: number, via: number) => { + const viaCell = g.cells[via]; + const viaLargeSlot = + viaCell !== undefined && (viaCell.rows > 1 || viaCell.columns > 1); + const distanceVia = distances[via] + (viaLargeSlot ? 4 : 1); + + if (distanceVia < distances[curr]) { + distances[curr] = distanceVia; + edges[curr] = via; + heap.push(curr); + } + }; + + while (heap.length > 0) { + const via = heap.pop()!; + const viaRow = row(via, g); + const viaColumn = column(via, g); + + if (viaRow > 0) visit(via - g.columns, via); + if (viaColumn > 0) visit(via - 1, via); + if (viaColumn < (viaRow === endRow ? endColumn : g.columns - 1)) + visit(via + 1, via); + if ( + viaRow < endRow - 1 || + (viaRow === endRow - 1 && viaColumn <= endColumn) + ) + visit(via + g.columns, via); + } + + return edges as number[]; +} + +function findLastIndex( + array: T[], + predicate: (item: T) => boolean +): number | null { + for (let i = array.length - 1; i >= 0; i--) { + if (predicate(array[i])) return i; + } + + return null; +} + +const findLast1By1Index = (g: Grid): number | null => + findLastIndex(g.cells, (c) => c?.rows === 1 && c?.columns === 1); + +export function row(index: number, g: Grid): number { + return Math.floor(index / g.columns); +} + +export function column(index: number, g: Grid): number { + return ((index % g.columns) + g.columns) % g.columns; +} + +function inArea(index: number, start: number, end: number, g: Grid): boolean { + const indexColumn = column(index, g); + const indexRow = row(index, g); + return ( + indexRow >= row(start, g) && + indexRow <= row(end, g) && + indexColumn >= column(start, g) && + indexColumn <= column(end, g) + ); +} + +function* cellsInArea( + start: number, + end: number, + g: Grid +): Generator { + const startColumn = column(start, g); + const endColumn = column(end, g); + for ( + let i = start; + i <= end; + i = + column(i, g) === endColumn + ? i + g.columns + startColumn - endColumn + : i + 1 + ) + yield i; +} + +export function forEachCellInArea( + start: number, + end: number, + g: Grid, + fn: (c: Cell | undefined, i: number) => void +): void { + for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i); +} + +function allCellsInArea( + start: number, + end: number, + g: Grid, + fn: (c: Cell | undefined, i: number) => boolean +): boolean { + for (const i of cellsInArea(start, end, g)) { + if (!fn(g.cells[i], i)) return false; + } + + return true; +} + +const areaEnd = ( + start: number, + columns: number, + rows: number, + g: Grid +): number => start + columns - 1 + g.columns * (rows - 1); + +/** + * Gets the index of the next gap in the grid that should be backfilled by 1×1 + * tiles. + */ +function getNextGap(g: Grid): number | null { + const last1By1Index = findLast1By1Index(g); + if (last1By1Index === null) return null; + + for (let i = 0; i < last1By1Index; i++) { + // To make the backfilling process look natural when there are multiple + // gaps, we actually scan each row from right to left + const j = + (row(i, g) === row(last1By1Index, g) + ? last1By1Index + : (row(i, g) + 1) * g.columns) - + 1 - + column(i, g); + + if (g.cells[j] === undefined) return j; + } + + return null; +} + +export function fillGaps(g: Grid): Grid { + const result: Grid = { ...g, cells: [...g.cells] }; + let gap = getNextGap(result); + + if (gap !== null) { + const pathToEnd = dijkstra(result); + + do { + let filled = false; + let to = gap; + let from: number | undefined = pathToEnd[gap]; + + // First, attempt to fill the gap by moving 1×1 tiles backwards from the + // end of the grid along a set path + while (from !== undefined) { + const toCell = result.cells[to]; + const fromCell = result.cells[from]; + + // Skip over large tiles + if (toCell !== undefined) { + to = pathToEnd[to]; + // Skip over large tiles. Also, we might run into gaps along the path + // created during the filling of previous gaps. Skip over those too; + // they'll be picked up on the next iteration of the outer loop. + } else if ( + fromCell === undefined || + fromCell.rows > 1 || + fromCell.columns > 1 + ) { + from = pathToEnd[from]; + } else { + result.cells[to] = result.cells[from]; + result.cells[from] = undefined; + filled = true; + to = pathToEnd[to]; + from = pathToEnd[from]; + } + } + + // In case the path approach failed, fall back to taking the very last 1×1 + // tile, and just dropping it into place + if (!filled) { + const last1By1Index = findLast1By1Index(result)!; + result.cells[gap] = result.cells[last1By1Index]; + result.cells[last1By1Index] = undefined; + } + + gap = getNextGap(result); + } while (gap !== null); + } + + // TODO: If there are any large tiles on the last row, shuffle them back + // upwards into a full row + + // Shrink the array to remove trailing gaps + const finalLength = + (findLastIndex(result.cells, (c) => c !== undefined) ?? -1) + 1; + if (finalLength < result.cells.length) + result.cells = result.cells.slice(0, finalLength); + + return result; +} + +export function cycleTileSize(tileId: string, g: Grid): Grid { + const from = g.cells.findIndex((c) => c?.item.id === tileId); + if (from === -1) return g; // Tile removed, no change + const fromWidth = g.cells[from]!.columns; + const fromHeight = g.cells[from]!.rows; + const fromEnd = areaEnd(from, fromWidth, fromHeight, g); + + const [toWidth, toHeight] = + fromWidth === 1 && fromHeight === 1 + ? [Math.min(3, Math.max(2, g.columns - 1)), 2] + : [1, 1]; + const newRows = Math.max( + 0, + Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) + ); + + const candidateWidth = toWidth; + const candidateHeight = toHeight - newRows; + + const gappyGrid: Grid = { + ...g, + generation: g.generation + 1, + cells: new Array(g.cells.length + newRows * g.columns), + }; + + const nextScanLocations = new Set([from]); + const scanColumnOffset = Math.floor((toWidth - 1) / 2); + const scanRowOffset = Math.floor((toHeight - 1) / 2); + const rows = row(g.cells.length - 1, g) + 1; + let to: number | null = null; + + const displaceable = (c: Cell | undefined, i: number): boolean => + c === undefined || + (c.columns === 1 && c.rows === 1) || + inArea(i, from, fromEnd, g); + + for (const scanLocation of nextScanLocations) { + const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset; + const end = areaEnd(start, candidateWidth, candidateHeight, g); + const startColumn = column(start, g); + const startRow = row(start, g); + const endColumn = column(end, g); + + if ( + start >= 0 && + end < gappyGrid.cells.length && + endColumn - startColumn + 1 === candidateWidth + ) { + if (allCellsInArea(start, end, g, displaceable)) { + to = start; + break; + } + } + + if (startColumn > 0) nextScanLocations.add(scanLocation - 1); + if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1); + if (startRow > 0) nextScanLocations.add(scanLocation - g.columns); + if (startRow <= rows) nextScanLocations.add(scanLocation + g.columns); + } + + // TODO: Don't give up on placing the tile yet + if (to === null) return g; + + const toRow = row(to, g); + + g.cells.forEach((c, src) => { + if (c?.slot && c.item.id !== tileId) { + const offset = + row(src, g) > toRow + candidateHeight - 1 ? g.columns * newRows : 0; + forEachCellInArea(src, areaEnd(src, c.columns, c.rows, g), g, (c, i) => { + gappyGrid.cells[i + offset] = c; + }); + } + }); + + const displacedTiles: Cell[] = []; + const toEnd = areaEnd(to, toWidth, toHeight, g); + forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { + if (c !== undefined) displacedTiles.push(c); + gappyGrid.cells[i] = { + item: g.cells[from]!.item, + slot: i === to, + columns: toWidth, + rows: toHeight, + }; + }); + + for (let i = 0; displacedTiles.length > 0; i++) { + if (gappyGrid.cells[i] === undefined) + gappyGrid.cells[i] = displacedTiles.shift(); + } + + return fillGaps(gappyGrid); +} From 8d46687a54b7fafb10cfd224e191188bce62ca53 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Tue, 7 Feb 2023 23:27:49 -0500 Subject: [PATCH 30/40] Refactor grid state tracking --- src/video-grid/NewVideoGrid.tsx | 133 +++++++++++++++++++------------- src/video-grid/model.ts | 43 ++++++++--- 2 files changed, 112 insertions(+), 64 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index 2c25f62d..aba499ea 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -17,8 +17,11 @@ limitations under the License. import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; import { EventTypes, Handler, useScroll } from "@use-gesture/react"; import React, { + Dispatch, FC, ReactNode, + SetStateAction, + useCallback, useEffect, useMemo, useRef, @@ -39,8 +42,82 @@ import { fillGaps, forEachCellInArea, cycleTileSize, + appendItems, } from "./model"; +interface GridState extends Grid { + /** + * The ID of the current state of the grid. + */ + generation: number; +} + +const useGridState = ( + columns: number | null, + items: TileDescriptor[] +): [GridState | null, Dispatch>] => { + const [grid, setGrid_] = useReactiveState( + (prevGrid = null) => { + if (prevGrid === null) { + // We can't do anything if the column count isn't known yet + if (columns === null) { + return null; + } else { + prevGrid = { generation: 0, columns, cells: [] }; + } + } + + // Step 1: Update tiles that still exist, and remove tiles that have left + // the grid + const itemsById = new Map(items.map((i) => [i.id, i])); + const grid1: Grid = { + ...prevGrid, + cells: prevGrid.cells.map((c) => { + if (c === undefined) return undefined; + const item = itemsById.get(c.item.id); + return item === undefined ? undefined : { ...c, item }; + }), + }; + + // Step 2: Backfill gaps left behind by removed tiles + const grid2 = fillGaps(grid1); + + // Step 3: Add new tiles to the end of the grid + const existingItemIds = new Set( + grid2.cells.filter((c) => c !== undefined).map((c) => c!.item.id) + ); + const newItems = items.filter((i) => !existingItemIds.has(i.id)); + const grid3 = appendItems(newItems, grid2); + + return { ...grid3, generation: prevGrid.generation + 1 }; + }, + [columns, items] + ); + + const setGrid: Dispatch> = useCallback( + (action) => { + if (typeof action === "function") { + setGrid_((prevGrid) => + prevGrid === null + ? null + : { + ...(action as (prev: Grid) => Grid)(prevGrid), + generation: prevGrid.generation + 1, + } + ); + } else { + setGrid_((prevGrid) => ({ + ...action, + generation: prevGrid?.generation ?? 1, + })); + } + }, + [setGrid_] + ); + + return [grid, setGrid]; +}; + interface Rect { x: number; y: number; @@ -133,55 +210,7 @@ export const NewVideoGrid: FC = ({ [gridBounds] ); - const [grid, setGrid] = useReactiveState( - (prevGrid = null) => { - if (prevGrid === null) { - // We can't do anything if the column count isn't known yet - if (columns === null) { - return null; - } else { - prevGrid = { generation: slotGridGeneration, columns, cells: [] }; - } - } - - // Step 1: Update tiles that still exist, and remove tiles that have left - // the grid - const itemsById = new Map(items.map((i) => [i.id, i])); - const grid1: Grid = { - ...prevGrid, - generation: prevGrid.generation + 1, - cells: prevGrid.cells.map((c) => { - if (c === undefined) return undefined; - const item = itemsById.get(c.item.id); - return item === undefined ? undefined : { ...c, item }; - }), - }; - - // Step 2: Backfill gaps left behind by removed tiles - const grid2 = fillGaps(grid1); - - // Step 3: Add new tiles to the end of the grid - const existingItemIds = new Set( - grid2.cells.filter((c) => c !== undefined).map((c) => c!.item.id) - ); - const newItems = items.filter((i) => !existingItemIds.has(i.id)); - const grid3: Grid = { - ...grid2, - cells: [ - ...grid2.cells, - ...newItems.map((i) => ({ - item: i, - slot: true, - columns: 1, - rows: 1, - })), - ], - }; - - return grid3; - }, - [items, columns] - ); + const [grid, setGrid] = useGridState(columns, items); const [tiles] = useReactiveState( (prevTiles) => { @@ -189,9 +218,9 @@ export const NewVideoGrid: FC = ({ // the update, because grid and slotRects will be out of sync if (slotGridGeneration !== grid?.generation) return prevTiles ?? []; - const slotCells = grid.cells.filter((c) => c?.slot) as Cell[]; + const tileCells = grid.cells.filter((c) => c?.origin) as Cell[]; const tileRects = new Map( - zipWith(slotCells, slotRects, (cell, rect) => [cell.item, rect]) + zipWith(tileCells, slotRects, (cell, rect) => [cell.item, rect]) ); return items.map((item) => ({ ...tileRects.get(item)!, item })); }, @@ -247,7 +276,7 @@ export const NewVideoGrid: FC = ({ let slotId = 0; for (let i = 0; i < grid.cells.length; i++) { const cell = grid.cells[i]; - if (cell?.slot) { + if (cell?.origin) { const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1); forEachCellInArea( i, diff --git a/src/video-grid/model.ts b/src/video-grid/model.ts index 0a4136c6..54f3c817 100644 --- a/src/video-grid/model.ts +++ b/src/video-grid/model.ts @@ -17,29 +17,34 @@ limitations under the License. import TinyQueue from "tinyqueue"; import { TileDescriptor } from "./TileDescriptor"; +/** + * A 1×1 cell in a grid which belongs to a tile. + */ export interface Cell { /** - * The item held by the slot containing this cell. + * The item displayed on the tile. */ item: TileDescriptor; /** - * Whether this cell is the first cell of the containing slot. + * Whether this cell is the origin (top left corner) of the tile. */ - // TODO: Rename to 'start'? - slot: boolean; + origin: boolean; /** - * The width, in columns, of the containing slot. + * The width, in columns, of the tile. */ columns: number; /** - * The height, in rows, of the containing slot. + * The height, in rows, of the tile. */ rows: number; } export interface Grid { - generation: number; columns: number; + /** + * The cells of the grid, in left-to-right top-to-bottom order. + * undefined = empty. + */ cells: (Cell | undefined)[]; } @@ -55,9 +60,9 @@ export function dijkstra(g: Grid): number[] { const visit = (curr: number, via: number) => { const viaCell = g.cells[via]; - const viaLargeSlot = + const viaLargeTile = viaCell !== undefined && (viaCell.rows > 1 || viaCell.columns > 1); - const distanceVia = distances[via] + (viaLargeSlot ? 4 : 1); + const distanceVia = distances[via] + (viaLargeTile ? 4 : 1); if (distanceVia < distances[curr]) { distances[curr] = distanceVia; @@ -252,6 +257,21 @@ export function fillGaps(g: Grid): Grid { return result; } +export function appendItems(items: TileDescriptor[], g: Grid): Grid { + return { + ...g, + cells: [ + ...g.cells, + ...items.map((i) => ({ + item: i, + origin: true, + columns: 1, + rows: 1, + })), + ], + }; +} + export function cycleTileSize(tileId: string, g: Grid): Grid { const from = g.cells.findIndex((c) => c?.item.id === tileId); if (from === -1) return g; // Tile removed, no change @@ -273,7 +293,6 @@ export function cycleTileSize(tileId: string, g: Grid): Grid { const gappyGrid: Grid = { ...g, - generation: g.generation + 1, cells: new Array(g.cells.length + newRows * g.columns), }; @@ -318,7 +337,7 @@ export function cycleTileSize(tileId: string, g: Grid): Grid { const toRow = row(to, g); g.cells.forEach((c, src) => { - if (c?.slot && c.item.id !== tileId) { + if (c?.origin && c.item.id !== tileId) { const offset = row(src, g) > toRow + candidateHeight - 1 ? g.columns * newRows : 0; forEachCellInArea(src, areaEnd(src, c.columns, c.rows, g), g, (c, i) => { @@ -333,7 +352,7 @@ export function cycleTileSize(tileId: string, g: Grid): Grid { if (c !== undefined) displacedTiles.push(c); gappyGrid.cells[i] = { item: g.cells[from]!.item, - slot: i === to, + origin: i === to, columns: toWidth, rows: toHeight, }; From d852e334130c0e6d0cd53bac185118a6ce21dcca Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 8 Feb 2023 00:32:08 -0500 Subject: [PATCH 31/40] Document the component --- src/video-grid/NewVideoGrid.tsx | 102 +++++++++++++++++++------------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx index aba499ea..810946bc 100644 --- a/src/video-grid/NewVideoGrid.tsx +++ b/src/video-grid/NewVideoGrid.tsx @@ -148,18 +148,27 @@ interface DragState { cursorY: number; } +/** + * An interactive, animated grid of video tiles. + */ export const NewVideoGrid: FC = ({ items, disableAnimations, children, }) => { + // Overview: This component lays out tiles by rendering an invisible template + // grid of "slots" for tiles to go in. Once rendered, it uses the DOM API to + // get the dimensions of each slot, feeding these numbers back into + // react-spring to let the actual tiles move freely atop the template. + + // To know when the rendered grid becomes consistent with the layout we've + // requested, we give it a data-generation attribute which holds the ID of the + // most recently rendered generation of the grid, and watch it with a + // MutationObserver. + const [slotGrid, setSlotGrid] = useState(null); const [slotGridGeneration, setSlotGridGeneration] = useState(0); - const [gridRef1, gridBounds] = useMeasure(); - const gridRef2 = useRef(null); - const gridRef = useMergedRefs(gridRef1, gridRef2); - useEffect(() => { if (slotGrid !== null) { setSlotGridGeneration( @@ -179,6 +188,10 @@ export const NewVideoGrid: FC = ({ } }, [slotGrid, setSlotGridGeneration]); + const [gridRef1, gridBounds] = useMeasure(); + const gridRef2 = useRef(null); + const gridRef = useMergedRefs(gridRef1, gridRef2); + const slotRects = useMemo(() => { if (slotGrid === null) return []; @@ -214,7 +227,7 @@ export const NewVideoGrid: FC = ({ const [tiles] = useReactiveState( (prevTiles) => { - // If React hasn't yet rendered the current generation of the layout, skip + // If React hasn't yet rendered the current generation of the grid, skip // the update, because grid and slotRects will be out of sync if (slotGridGeneration !== grid?.generation) return prevTiles ?? []; @@ -264,43 +277,6 @@ export const NewVideoGrid: FC = ({ // react-spring's types are bugged and can't infer the spring type ) as unknown as [TransitionFn, SpringRef]; - const slotGridStyle = useMemo(() => { - if (grid === null) return {}; - - const areas = new Array<(number | null)[]>( - Math.ceil(grid.cells.length / grid.columns) - ); - for (let i = 0; i < areas.length; i++) - areas[i] = new Array(grid.columns).fill(null); - - let slotId = 0; - for (let i = 0; i < grid.cells.length; i++) { - const cell = grid.cells[i]; - if (cell?.origin) { - const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1); - forEachCellInArea( - i, - slotEnd, - grid, - (_c, j) => (areas[row(j, grid)][column(j, grid)] = slotId) - ); - slotId++; - } - } - - return { - gridTemplateAreas: areas - .map( - (row) => - `'${row - .map((slotId) => (slotId === null ? "." : `s${slotId}`)) - .join(" ")}'` - ) - .join(" "), - gridTemplateColumns: `repeat(${columns}, 1fr)`, - }; - }, [grid, columns]); - const animateDraggedTile = (endOfGesture: boolean) => { const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!; const tile = tiles.find((t) => t.item.id === tileId)!; @@ -357,6 +333,11 @@ export const NewVideoGrid: FC = ({ } }; + // Callback for useDrag. We could call useDrag here, but the default + // pattern of spreading {...bind()} across the children to bind the gesture + // ends up breaking memoization and ruining this component's performance. + // Instead, we pass this callback to each tile via a ref, to let them bind the + // gesture using the much more sensible ref-based method. const onTileDrag = ( tileId: string, { @@ -411,6 +392,43 @@ export const NewVideoGrid: FC = ({ { target: gridRef2 } ); + const slotGridStyle = useMemo(() => { + if (grid === null) return {}; + + const areas = new Array<(number | null)[]>( + Math.ceil(grid.cells.length / grid.columns) + ); + for (let i = 0; i < areas.length; i++) + areas[i] = new Array(grid.columns).fill(null); + + let slotId = 0; + for (let i = 0; i < grid.cells.length; i++) { + const cell = grid.cells[i]; + if (cell?.origin) { + const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1); + forEachCellInArea( + i, + slotEnd, + grid, + (_c, j) => (areas[row(j, grid)][column(j, grid)] = slotId) + ); + slotId++; + } + } + + return { + gridTemplateAreas: areas + .map( + (row) => + `'${row + .map((slotId) => (slotId === null ? "." : `s${slotId}`)) + .join(" ")}'` + ) + .join(" "), + gridTemplateColumns: `repeat(${columns}, 1fr)`, + }; + }, [grid, columns]); + const slots = useMemo(() => { const slots = new Array(items.length); for (let i = 0; i < items.length; i++) From 544874487101f7f95b90f6e0e044bc6c9fe3ef96 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 13 Feb 2023 12:17:09 -0500 Subject: [PATCH 32/40] Document grid operations --- src/video-grid/model.ts | 114 ++++++++++++++++++++++++++++------------ 1 file changed, 81 insertions(+), 33 deletions(-) diff --git a/src/video-grid/model.ts b/src/video-grid/model.ts index 54f3c817..7d9729ca 100644 --- a/src/video-grid/model.ts +++ b/src/video-grid/model.ts @@ -48,21 +48,33 @@ export interface Grid { cells: (Cell | undefined)[]; } -export function dijkstra(g: Grid): number[] { - const end = findLast1By1Index(g) ?? 0; - const endRow = row(end, g); - const endColumn = column(end, g); +/** + * Gets the paths that tiles should travel along in the grid to reach a + * particular destination. + * @param dest The destination index. + * @param g The grid. + * @returns An array in which each cell holds the index of the next cell to move + * to to reach the destination, or null if it is the destination. + */ +export function getPaths(dest: number, g: Grid): (number | null)[] { + const destRow = row(dest, g); + const destColumn = column(dest, g); - const distances = new Array(end + 1).fill(Infinity); - distances[end] = 0; - const edges = new Array(end).fill(undefined); - const heap = new TinyQueue([end], (i) => distances[i]); + // This is Dijkstra's algorithm + + const distances = new Array(dest + 1).fill(Infinity); + distances[dest] = 0; + const edges = new Array(dest).fill(undefined); + edges[dest] = null; + const heap = new TinyQueue([dest], (i) => distances[i]); const visit = (curr: number, via: number) => { const viaCell = g.cells[via]; const viaLargeTile = viaCell !== undefined && (viaCell.rows > 1 || viaCell.columns > 1); - const distanceVia = distances[via] + (viaLargeTile ? 4 : 1); + // Since it looks nicer to have paths go around large tiles, we impose an + // increased cost for moving through them + const distanceVia = distances[via] + (viaLargeTile ? 8 : 1); if (distanceVia < distances[curr]) { distances[curr] = distanceVia; @@ -76,18 +88,20 @@ export function dijkstra(g: Grid): number[] { const viaRow = row(via, g); const viaColumn = column(via, g); + // Visit each neighbor if (viaRow > 0) visit(via - g.columns, via); if (viaColumn > 0) visit(via - 1, via); - if (viaColumn < (viaRow === endRow ? endColumn : g.columns - 1)) + if (viaColumn < (viaRow === destRow ? destColumn : g.columns - 1)) visit(via + 1, via); if ( - viaRow < endRow - 1 || - (viaRow === endRow - 1 && viaColumn <= endColumn) + viaRow < destRow - 1 || + (viaRow === destRow - 1 && viaColumn <= destColumn) ) visit(via + g.columns, via); } - return edges as number[]; + // The heap is empty, so we've generated all paths + return edges as (number | null)[]; } function findLastIndex( @@ -194,27 +208,30 @@ function getNextGap(g: Grid): number | null { return null; } +/** + * Backfill any gaps in the grid. + */ export function fillGaps(g: Grid): Grid { const result: Grid = { ...g, cells: [...g.cells] }; let gap = getNextGap(result); if (gap !== null) { - const pathToEnd = dijkstra(result); + const pathsToEnd = getPaths(findLast1By1Index(result)!, result); do { let filled = false; let to = gap; - let from: number | undefined = pathToEnd[gap]; + let from = pathsToEnd[gap]; // First, attempt to fill the gap by moving 1×1 tiles backwards from the // end of the grid along a set path - while (from !== undefined) { + while (from !== null) { const toCell = result.cells[to]; const fromCell = result.cells[from]; - // Skip over large tiles + // Skip over slots that are already full if (toCell !== undefined) { - to = pathToEnd[to]; + to = pathsToEnd[to]!; // Skip over large tiles. Also, we might run into gaps along the path // created during the filling of previous gaps. Skip over those too; // they'll be picked up on the next iteration of the outer loop. @@ -223,13 +240,13 @@ export function fillGaps(g: Grid): Grid { fromCell.rows > 1 || fromCell.columns > 1 ) { - from = pathToEnd[from]; + from = pathsToEnd[from]; } else { result.cells[to] = result.cells[from]; result.cells[from] = undefined; filled = true; - to = pathToEnd[to]; - from = pathToEnd[from]; + to = pathsToEnd[to]!; + from = pathsToEnd[from]; } } @@ -272,6 +289,12 @@ export function appendItems(items: TileDescriptor[], g: Grid): Grid { }; } +/** + * Changes the size of a tile, rearranging the grid to make space. + * @param tileId The ID of the tile to modify. + * @param g The grid. + * @returns The updated grid. + */ export function cycleTileSize(tileId: string, g: Grid): Grid { const from = g.cells.findIndex((c) => c?.item.id === tileId); if (from === -1) return g; // Tile removed, no change @@ -279,63 +302,84 @@ export function cycleTileSize(tileId: string, g: Grid): Grid { const fromHeight = g.cells[from]!.rows; const fromEnd = areaEnd(from, fromWidth, fromHeight, g); + // The target dimensions, which toggle between 1×1 and larger than 1×1 const [toWidth, toHeight] = fromWidth === 1 && fromHeight === 1 ? [Math.min(3, Math.max(2, g.columns - 1)), 2] : [1, 1]; + + // If we're expanding the tile, we want to create enough new rows at the + // tile's target position such that every new unit of grid area created during + // the expansion can fit within the new rows. + // We do it this way, since it's easier to backfill gaps in the grid than it + // is to push colliding tiles outwards. const newRows = Math.max( 0, Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns) ); - const candidateWidth = toWidth; - const candidateHeight = toHeight - newRows; - + // This is the grid with the new rows added const gappyGrid: Grid = { ...g, cells: new Array(g.cells.length + newRows * g.columns), }; - const nextScanLocations = new Set([from]); + // The next task is to scan for a spot to place the modified tile. Since we + // might be creating new rows at the target position, this spot can be shorter + // than the target height. + const candidateWidth = toWidth; + const candidateHeight = toHeight - newRows; + + // To make the tile appear to expand outwards from its center, we're actually + // scanning for locations to put the *center* of the tile. These numbers are + // the offsets between the tile's origin and its center. const scanColumnOffset = Math.floor((toWidth - 1) / 2); const scanRowOffset = Math.floor((toHeight - 1) / 2); + + const nextScanLocations = new Set([from]); const rows = row(g.cells.length - 1, g) + 1; let to: number | null = null; + // The contents of a given cell are 'displaceable' if it's empty, holds a 1×1 + // tile, or is part of the original tile we're trying to reposition const displaceable = (c: Cell | undefined, i: number): boolean => c === undefined || (c.columns === 1 && c.rows === 1) || inArea(i, from, fromEnd, g); + // Do the scanning for (const scanLocation of nextScanLocations) { const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset; const end = areaEnd(start, candidateWidth, candidateHeight, g); const startColumn = column(start, g); const startRow = row(start, g); const endColumn = column(end, g); + const endRow = row(end, g); if ( start >= 0 && - end < gappyGrid.cells.length && - endColumn - startColumn + 1 === candidateWidth + endColumn - startColumn + 1 === candidateWidth && + allCellsInArea(start, end, g, displaceable) ) { - if (allCellsInArea(start, end, g, displaceable)) { - to = start; - break; - } + // This location works! + to = start; + break; } + // Scan outwards in all directions if (startColumn > 0) nextScanLocations.add(scanLocation - 1); if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1); if (startRow > 0) nextScanLocations.add(scanLocation - g.columns); - if (startRow <= rows) nextScanLocations.add(scanLocation + g.columns); + if (endRow < rows - 1) nextScanLocations.add(scanLocation + g.columns); } - // TODO: Don't give up on placing the tile yet + // If there is no space in the grid, give up if (to === null) return g; const toRow = row(to, g); + // Copy tiles from the original grid to the new one, with the new rows + // inserted at the target location g.cells.forEach((c, src) => { if (c?.origin && c.item.id !== tileId) { const offset = @@ -346,6 +390,8 @@ export function cycleTileSize(tileId: string, g: Grid): Grid { } }); + // Place the tile in its target position, making a note of the tiles being + // overwritten const displacedTiles: Cell[] = []; const toEnd = areaEnd(to, toWidth, toHeight, g); forEachCellInArea(to, toEnd, gappyGrid, (c, i) => { @@ -358,10 +404,12 @@ export function cycleTileSize(tileId: string, g: Grid): Grid { }; }); + // Place the displaced tiles in the remaining space for (let i = 0; displacedTiles.length > 0; i++) { if (gappyGrid.cells[i] === undefined) gappyGrid.cells[i] = displacedTiles.shift(); } + // Fill any gaps that remain return fillGaps(gappyGrid); } From 8d0bf4caccf0a4b10b9da9145a7794cefb95e3b2 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 13 Feb 2023 18:35:50 -0500 Subject: [PATCH 33/40] Test grid operations --- test/video-grid/model-test.ts | 247 ++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 test/video-grid/model-test.ts diff --git a/test/video-grid/model-test.ts b/test/video-grid/model-test.ts new file mode 100644 index 00000000..d71fd2f5 --- /dev/null +++ b/test/video-grid/model-test.ts @@ -0,0 +1,247 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + column, + cycleTileSize, + fillGaps, + forEachCellInArea, + Grid, + row, +} from "../../src/video-grid/model"; +import { TileDescriptor } from "../../src/video-grid/TileDescriptor"; + +/** + * Builds a grid from a string specifying the contents of each cell as a letter. + */ +function mkGrid(spec: string): Grid { + const secondNewline = spec.indexOf("\n", 1); + const columns = secondNewline === -1 ? spec.length : secondNewline - 1; + const cells = spec.match(/[a-z ]/g) ?? []; + const areas = new Set(cells); + areas.delete(" "); // Space represents an empty cell, not an area + const grid: Grid = { columns, cells: new Array(cells.length) }; + + for (const area of areas) { + const start = cells.indexOf(area); + const end = cells.lastIndexOf(area); + const rows = row(end, grid) - row(start, grid) + 1; + const columns = column(end, grid) - column(start, grid) + 1; + + forEachCellInArea(start, end, grid, (_c, i) => { + grid.cells[i] = { + item: { id: area } as unknown as TileDescriptor, + origin: i === start, + rows, + columns, + }; + }); + } + + return grid; +} + +/** + * Turns a grid into a string showing the contents of each cell as a letter. + */ +function showGrid(g: Grid): string { + let result = "\n"; + g.cells.forEach((c, i) => { + if (i > 0 && i % g.columns == 0) result += "\n"; + result += c?.item.id ?? " "; + }); + return result; +} + +function testFillGaps(title: string, input: string, output: string): void { + test(`fillGaps ${title}`, () => { + expect(showGrid(fillGaps(mkGrid(input)))).toBe(output); + }); +} + +testFillGaps( + "fills a gap", + ` +a b +cde +f`, + ` +cab +fde` +); + +testFillGaps( + "fills multiple gaps", + ` +a bc +defgh + ijkl +mno`, + ` +aebch +difgl +monjk` +); + +testFillGaps( + "fills a big gap", + ` +abcd +e f +g h +ijkl`, + ` +abcd +elhf +gkji` +); + +testFillGaps( + "only moves 1×1 tiles", + ` + +aa +bc`, + ` +bc +aa` +); + +testFillGaps( + "prefers moving around large tiles", + ` +a bc +ddde +dddf +ghij +k`, + ` +abce +dddf +dddj +kghi` +); + +testFillGaps( + "moves through large tiles if necessary", + ` +a bc +dddd +efgh +i`, + ` +afbc +dddd +iegh` +); + +function testCycleTileSize( + title: string, + tileId: string, + input: string, + output: string +): void { + test(`cycleTileSize ${title}`, () => { + expect(showGrid(cycleTileSize(tileId, mkGrid(input)))).toBe(output); + }); +} + +testCycleTileSize( + "does nothing if the tile is not present", + "z", + ` +abcd +efgh`, + ` +abcd +efgh` +); + +testCycleTileSize( + "expands a tile to 2×2 in a 3 column layout", + "c", + ` +abc +def +ghi`, + ` +acc +bcc +def +ghi` +); + +testCycleTileSize( + "expands a tile to 3×3 in a 4 column layout", + "g", + ` +abcd +efgh`, + ` +abcd +eggg +fggg +h` +); + +testCycleTileSize( + "restores a tile to 1×1", + "b", + ` +abbc +dbbe +fghi +jk`, + ` +abhc +djge +fik` +); + +testCycleTileSize( + "expands a tile even in a crowded grid", + "c", + ` +abb +cbb +dde +ddf +ghi +klm`, + ` +abb +gbb +dde +ddf +cci +cch +klm` +); + +testCycleTileSize( + "does nothing if the tile has no room to expand", + "c", + ` +abb +cbb +dde +ddf`, + ` +abb +cbb +dde +ddf` +); From 69e6ba93c18585ea190eee6cbb1c68e5f75abf9a Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 13 Feb 2023 20:36:42 -0500 Subject: [PATCH 34/40] Add a switch to toggle between the new and old grids --- src/room/InCallView.tsx | 12 +++++++++--- src/settings/SettingsModal.tsx | 13 +++++++++++++ src/settings/useSetting.ts | 1 + src/video-grid/VideoGrid.module.css | 1 + src/video-grid/VideoGrid.tsx | 18 ++++++++---------- src/video-grid/VideoTileContainer.tsx | 4 ++-- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index a0f7917f..d4717c1b 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -51,7 +51,11 @@ import { UserMenuContainer } from "../UserMenuContainer"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { useMediaHandler } from "../settings/useMediaHandler"; -import { useShowInspector, useSpatialAudio } from "../settings/useSetting"; +import { + useNewGrid, + useShowInspector, + useSpatialAudio, +} from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; import { useAudioContext } from "../video-grid/useMediaStream"; import { useFullscreen } from "../video-grid/useFullscreen"; @@ -277,6 +281,8 @@ export function InCallView({ [] ); + const [newGrid] = useNewGrid(); + const Grid = newGrid ? NewVideoGrid : VideoGrid; const prefersReducedMotion = usePrefersReducedMotion(); const renderContent = (): JSX.Element => { @@ -306,7 +312,7 @@ export function InCallView({ } return ( - )} - + ); }; diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 106ef9e0..244d1d15 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -33,6 +33,7 @@ import { useShowInspector, useOptInAnalytics, canEnableSpatialAudio, + useNewGrid, } from "./useSetting"; import { FieldRow, InputField } from "../input/Input"; import { Button } from "../button"; @@ -62,6 +63,7 @@ export const SettingsModal = (props: Props) => { const [showInspector, setShowInspector] = useShowInspector(); const [optInAnalytics, setOptInAnalytics] = useOptInAnalytics(); const [keyboardShortcuts, setKeyboardShortcuts] = useKeyboardShortcuts(); + const [newGrid, setNewGrid] = useNewGrid(); const downloadDebugLog = useDownloadDebugLog(); @@ -216,6 +218,17 @@ export const SettingsModal = (props: Props) => { } /> + + ) => + setNewGrid(e.target.checked) + } + /> +
diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index ca9ebbea..eea617f2 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -44,14 +44,14 @@ interface Props { showOptions?: boolean; isLocal?: boolean; disableSpeakingIndicator?: boolean; - opacity: SpringValue; - scale: SpringValue; - shadow: SpringValue; - zIndex: SpringValue; - x: SpringValue; - y: SpringValue; - width: SpringValue; - height: SpringValue; + opacity?: SpringValue; + scale?: SpringValue; + shadow?: SpringValue; + zIndex?: SpringValue; + x?: SpringValue; + y?: SpringValue; + width?: SpringValue; + height?: SpringValue; } export const VideoTile = forwardRef( @@ -141,14 +141,17 @@ export const VideoTile = forwardRef( style={{ opacity, scale, - boxShadow: shadow.to( + boxShadow: shadow?.to( (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px` ), zIndex, x, y, - "--tileWidth": width.to((w) => `${w}px`), - "--tileHeight": height.to((h) => `${h}px`), + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore React does in fact support assigning custom properties, + // but React's types say no + "--tileWidth": width?.to((w) => `${w}px`), + "--tileHeight": height?.to((h) => `${h}px`), }} ref={ref as ForwardedRef} {...rest} diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index 86140339..c3a4c611 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -18,6 +18,8 @@ import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventType import React, { FC, memo, RefObject } from "react"; import { useCallback } from "react"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; +import { SpringValue } from "@react-spring/web"; +import { EventTypes, Handler, useDrag } from "@use-gesture/react"; import { useCallFeed } from "./useCallFeed"; import { useSpatialMediaStream } from "./useMediaStream"; @@ -26,8 +28,6 @@ import { VideoTile } from "./VideoTile"; import { VideoTileSettingsModal } from "./VideoTileSettingsModal"; import { useModalTriggerState } from "../Modal"; import { TileDescriptor } from "./TileDescriptor"; -import { SpringValue } from "@react-spring/web"; -import { EventTypes, Handler, useDrag } from "@use-gesture/react"; interface Props { item: TileDescriptor; @@ -44,14 +44,14 @@ interface Props { maximised: boolean; fullscreen: boolean; onFullscreen: (item: TileDescriptor) => void; - opacity: SpringValue; - scale: SpringValue; - shadow: SpringValue; - zIndex: SpringValue; - x: SpringValue; - y: SpringValue; - width: SpringValue; - height: SpringValue; + opacity?: SpringValue; + scale?: SpringValue; + shadow?: SpringValue; + zIndex?: SpringValue; + x?: SpringValue; + y?: SpringValue; + width?: SpringValue; + height?: SpringValue; onDragRef?: RefObject< ( tileId: string, diff --git a/src/video-grid/model.ts b/src/video-grid/model.ts index 7d9729ca..48f19b34 100644 --- a/src/video-grid/model.ts +++ b/src/video-grid/model.ts @@ -15,6 +15,7 @@ limitations under the License. */ import TinyQueue from "tinyqueue"; + import { TileDescriptor } from "./TileDescriptor"; /** From 8c818b9ce16dd2d51525d05f98c103e26ea0cc9d Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 13 Feb 2023 22:24:04 -0500 Subject: [PATCH 36/40] Get 100% test coverage on grid operations --- test/video-grid/model-test.ts | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/video-grid/model-test.ts b/test/video-grid/model-test.ts index d71fd2f5..cc7741d9 100644 --- a/test/video-grid/model-test.ts +++ b/test/video-grid/model-test.ts @@ -15,6 +15,7 @@ limitations under the License. */ import { + appendItems, column, cycleTileSize, fillGaps, @@ -72,6 +73,26 @@ function testFillGaps(title: string, input: string, output: string): void { }); } +testFillGaps( + "does nothing on an empty grid", + ` +`, + ` +` +); + +testFillGaps( + "does nothing if there are no gaps", + ` +ab +cd +ef`, + ` +ab +cd +ef` +); + testFillGaps( "fills a gap", ` @@ -245,3 +266,18 @@ cbb dde ddf` ); + +test("appendItems appends 1×1 tiles", () => { + const grid1 = ` +aab +aac +d`; + const grid2 = ` +aab +aac +def`; + const newItems = ["e", "f"].map( + (i) => ({ id: i } as unknown as TileDescriptor) + ); + expect(showGrid(appendItems(newItems, mkGrid(grid1)))).toBe(grid2); +}); From ef4a62ca6243787c2eaf5c9d78c6ffd182b3f0d5 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 13 Feb 2023 22:38:27 -0500 Subject: [PATCH 37/40] Document useMergedRefs --- src/useMergedRefs.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/useMergedRefs.ts b/src/useMergedRefs.ts index f8712f53..7c6da658 100644 --- a/src/useMergedRefs.ts +++ b/src/useMergedRefs.ts @@ -1,5 +1,9 @@ import { MutableRefObject, RefCallback, useCallback } from "react"; +/** + * Combines multiple refs into one, useful for attaching multiple refs to the + * same DOM node. + */ export const useMergedRefs = ( ...refs: (MutableRefObject | RefCallback)[] ): RefCallback => From efbf319fa1048087662938639f104a7c1b212678 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 13 Feb 2023 22:40:26 -0500 Subject: [PATCH 38/40] Explain why we cast the tile springs --- src/video-grid/VideoGrid.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/video-grid/VideoGrid.tsx b/src/video-grid/VideoGrid.tsx index 6623eee4..03d95837 100644 --- a/src/video-grid/VideoGrid.tsx +++ b/src/video-grid/VideoGrid.tsx @@ -986,6 +986,7 @@ export function VideoGrid({ tilePositions, tiles, scrollPosition, + // react-spring's types are bugged and can't infer the spring type ]) as unknown as [SpringValues[], SpringRef]; const onTap = useCallback( From fbc72283d4855f1164042cf51a5a84221fe88225 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 15 Feb 2023 16:20:58 -0500 Subject: [PATCH 39/40] Add missing copyright headers --- src/useMergedRefs.ts | 16 ++++++++++++++++ src/useReactiveState.ts | 16 ++++++++++++++++ src/video-grid/NewVideoGrid.module.css | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/useMergedRefs.ts b/src/useMergedRefs.ts index 7c6da658..3fab929d 100644 --- a/src/useMergedRefs.ts +++ b/src/useMergedRefs.ts @@ -1,3 +1,19 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import { MutableRefObject, RefCallback, useCallback } from "react"; /** diff --git a/src/useReactiveState.ts b/src/useReactiveState.ts index fe995724..f20399a4 100644 --- a/src/useReactiveState.ts +++ b/src/useReactiveState.ts @@ -1,3 +1,19 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import { DependencyList, Dispatch, diff --git a/src/video-grid/NewVideoGrid.module.css b/src/video-grid/NewVideoGrid.module.css index e6f64ba9..75ee7076 100644 --- a/src/video-grid/NewVideoGrid.module.css +++ b/src/video-grid/NewVideoGrid.module.css @@ -1,3 +1,19 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + .grid { contain: strict; position: relative; From 1fc181dc280bf6341dccf7dc781e281bb27359d3 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 15 Feb 2023 16:38:49 -0500 Subject: [PATCH 40/40] Document useReactiveState --- src/useReactiveState.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/useReactiveState.ts b/src/useReactiveState.ts index f20399a4..5693be54 100644 --- a/src/useReactiveState.ts +++ b/src/useReactiveState.ts @@ -23,6 +23,11 @@ import { useState, } from "react"; +/** + * Hook creating a stateful value that updates automatically whenever the + * dependencies change. Or equivalently, a version of useMemo that takes its own + * previous value as an input, and can be updated manually. + */ export const useReactiveState = ( updateFn: (prevState?: T) => T, deps: DependencyList