Skip to content

Commit dedf3b1

Browse files
anthonykim1anthonykim1
authored
Merge pull request #5722 from xtermjs/anthonykim1/placementActions
Implment kitty graphics placement action
2 parents 6e35b44 + 2158827 commit dedf3b1

File tree

3 files changed

+445
-111
lines changed

3 files changed

+445
-111
lines changed

‎addons/addon-image/src/kitty/KittyGraphicsHandler.ts‎

Lines changed: 148 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -323,9 +323,10 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
323323
case KittyAction.QUERY:
324324
this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0);
325325
return true;
326+
case KittyAction.PLACEMENT:
327+
return this._handlePlacement(cmd);
326328
default:
327329
// TODO: Implement remaining actions when needed:
328-
// - a=p (placement): place a previously transmitted image
329330
// - a=f (frame): animation frame operations
330331
// - a=a (animation): animation control
331332
// - a=c (compose): compose images
@@ -357,9 +358,11 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
357358
return this._handleTransmitDisplay(cmd, bytes, decodeError);
358359
case KittyAction.QUERY:
359360
return this._handleQuery(cmd, bytes, decodeError);
361+
case KittyAction.PLACEMENT:
362+
// a=p ignores any payload — image data was already transmitted
363+
return this._handlePlacement(cmd);
360364
default:
361365
// TODO: Implement remaining actions when needed:
362-
// - a=p (placement): place a previously transmitted image
363366
// - a=f (frame): animation frame operations
364367
// - a=a (animation): animation control
365368
// - a=c (compose): compose images
@@ -370,6 +373,23 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
370373
}
371374
}
372375

376+
private _handlePlacement(cmd: IKittyCommand): boolean | Promise<boolean> {
377+
if (cmd.id === undefined) {
378+
return true;
379+
}
380+
const id = cmd.id;
381+
const image = this._kittyStorage.getImage(id);
382+
if (!image) {
383+
this._sendResponse(id, 'ENOENT:image not found', cmd.quiet ?? 0, cmd.placementId);
384+
return true;
385+
}
386+
const result = this._displayImage(image, cmd);
387+
return result.then(success => {
388+
this._sendResponse(id, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0, cmd.placementId);
389+
return true;
390+
});
391+
}
392+
373393
private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
374394
// TODO: Support file-based transmission modes (t=f, t=t, t=s)
375395
// Currently only supports direct transmission (t=d, the default).
@@ -492,6 +512,10 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
492512
break;
493513
case 'i':
494514
case 'I':
515+
// TODO: When placement id tracking is implemented (see TODO in
516+
// KittyImageStorage), d=i with p=<pid> should delete only that
517+
// specific placement, while d=i without p should delete all
518+
// placements for the image.
495519
if (cmd.id !== undefined) {
496520
const pending = this._pendingTransmissions.get(cmd.id);
497521
if (pending) {
@@ -508,12 +532,13 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
508532
return true;
509533
}
510534

511-
private _sendResponse(id: number, message: string, quiet: number): void {
535+
private _sendResponse(id: number, message: string, quiet: number, placementId?: number): void {
512536
const isOk = message === 'OK';
513537
if (isOk && quiet === 1) return;
514538
if (!isOk && quiet === 2) return;
515539

516-
const response = `\x1b_Gi=${id};${message}\x1b\\`;
540+
const pPart = placementId ? `,p=${placementId}` : '';
541+
const response = `\x1b_Gi=${id}${pPart};${message}\x1b\\`;
517542
this._coreTerminal._core.coreService.triggerDataEvent(response);
518543
}
519544

@@ -526,115 +551,137 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
526551
}
527552

528553
private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise<void> {
529-
let bitmap = await this._createBitmap(image);
530-
531-
const cropX = Math.max(0, cmd.x ?? 0);
532-
const cropY = Math.max(0, cmd.y ?? 0);
533-
const cropW = cmd.sourceWidth || (bitmap.width - cropX);
534-
const cropH = cmd.sourceHeight || (bitmap.height - cropY);
554+
let bitmap: ImageBitmap | undefined = await this._createBitmap(image);
535555

536-
const maxCropW = Math.max(0, bitmap.width - cropX);
537-
const maxCropH = Math.max(0, bitmap.height - cropY);
538-
const finalCropW = Math.max(0, Math.min(cropW, maxCropW));
539-
const finalCropH = Math.max(0, Math.min(cropH, maxCropH));
540-
541-
if (finalCropW === 0 || finalCropH === 0) {
542-
bitmap.close();
543-
throw new Error('invalid source rectangle');
544-
}
545-
546-
if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) {
547-
const cropped = await createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH);
548-
bitmap.close();
549-
bitmap = cropped;
550-
}
551-
552-
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
553-
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
554-
555-
// Per spec: c/r default to image's natural cell dimensions
556-
let imgCols = cmd.columns ?? Math.ceil(bitmap.width / cw);
557-
let imgRows = cmd.rows ?? Math.ceil(bitmap.height / ch);
558-
559-
let w = bitmap.width;
560-
let h = bitmap.height;
561-
562-
// Scale bitmap to fit placement rectangle when c/r are specified
563-
if (cmd.columns !== undefined || cmd.rows !== undefined) {
564-
w = Math.round(imgCols * cw);
565-
h = Math.round(imgRows * ch);
566-
}
567-
568-
if (w * h > this._opts.pixelLimit) {
569-
bitmap.close();
570-
throw new Error('image exceeds pixel limit');
571-
}
556+
try {
557+
const cropX = Math.max(0, cmd.x ?? 0);
558+
const cropY = Math.max(0, cmd.y ?? 0);
559+
const cropW = cmd.sourceWidth || (bitmap.width - cropX);
560+
const cropH = cmd.sourceHeight || (bitmap.height - cropY);
561+
562+
const maxCropW = Math.max(0, bitmap.width - cropX);
563+
const maxCropH = Math.max(0, bitmap.height - cropY);
564+
const finalCropW = Math.max(0, Math.min(cropW, maxCropW));
565+
const finalCropH = Math.max(0, Math.min(cropH, maxCropH));
566+
567+
if (finalCropW === 0 || finalCropH === 0) {
568+
throw new Error('invalid source rectangle');
569+
}
572570

573-
// Save cursor position before addImage modifies it
574-
const buffer = this._coreTerminal._core.buffer;
575-
const savedX = buffer.x;
576-
const savedY = buffer.y;
577-
const savedYbase = buffer.ybase;
571+
if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) {
572+
const cropped = await createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH);
573+
bitmap.close();
574+
bitmap = cropped;
575+
}
578576

579-
// Determine layer based on z-index: negative = behind text, 0+ = on top.
580-
// When z<0 we always use the bottom layer even without allowTransparency —
581-
// the image will simply be hidden behind the opaque text background, which
582-
// is the correct behavior (client asked for "behind text").
583-
const wantsBottom = cmd.zIndex !== undefined && cmd.zIndex < 0;
584-
const layer: ImageLayer = wantsBottom ? 'bottom' : 'top';
577+
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
578+
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
579+
580+
// Per spec: c/r default to image's natural cell dimensions.
581+
// If only one of c/r is specified, compute the other from image aspect ratio.
582+
let imgCols: number;
583+
let imgRows: number;
584+
if (cmd.columns !== undefined && cmd.rows !== undefined) {
585+
imgCols = cmd.columns;
586+
imgRows = cmd.rows;
587+
} else if (cmd.columns !== undefined) {
588+
imgCols = cmd.columns;
589+
imgRows = Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (imgCols * cw) / ch));
590+
} else if (cmd.rows !== undefined) {
591+
imgRows = cmd.rows;
592+
imgCols = Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (imgRows * ch) / cw));
593+
} else {
594+
imgCols = Math.ceil(bitmap.width / cw);
595+
imgRows = Math.ceil(bitmap.height / ch);
596+
}
585597

586-
let finalBitmap = bitmap;
587-
if (w !== bitmap.width || h !== bitmap.height) {
588-
finalBitmap = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h });
589-
bitmap.close();
590-
}
598+
let w = bitmap.width;
599+
let h = bitmap.height;
591600

592-
// Per spec: X/Y are pixel offsets within the first cell, so clamp to cell dimensions
593-
const xOffset = Math.min(Math.max(0, cmd.xOffset ?? 0), cw - 1);
594-
const yOffset = Math.min(Math.max(0, cmd.yOffset ?? 0), ch - 1);
595-
if (xOffset !== 0 || yOffset !== 0) {
596-
const offsetCanvas = ImageRenderer.createCanvas(window.document, finalBitmap.width + xOffset, finalBitmap.height + yOffset);
597-
const offsetCtx = offsetCanvas.getContext('2d');
598-
if (!offsetCtx) {
599-
finalBitmap.close();
600-
throw new Error('Failed to create offset canvas context');
601+
// Scale bitmap to fit placement rectangle when c/r are specified
602+
if (cmd.columns !== undefined || cmd.rows !== undefined) {
603+
w = Math.round(imgCols * cw);
604+
h = Math.round(imgRows * ch);
601605
}
602-
offsetCtx.drawImage(finalBitmap, xOffset, yOffset);
603-
604-
const offsetBitmap = await createImageBitmap(offsetCanvas);
605-
offsetCanvas.width = offsetCanvas.height = 0;
606-
finalBitmap.close();
607-
finalBitmap = offsetBitmap;
608-
w = finalBitmap.width;
609-
h = finalBitmap.height;
606+
610607
if (w * h > this._opts.pixelLimit) {
611-
finalBitmap.close();
612608
throw new Error('image exceeds pixel limit');
613609
}
614-
if (cmd.columns === undefined) {
615-
imgCols = Math.ceil(finalBitmap.width / cw);
616-
}
617-
if (cmd.rows === undefined) {
618-
imgRows = Math.ceil(finalBitmap.height / ch);
610+
611+
// Save cursor position before addImage modifies it
612+
const buffer = this._coreTerminal._core.buffer;
613+
const savedX = buffer.x;
614+
const savedY = buffer.y;
615+
const savedYbase = buffer.ybase;
616+
617+
// Determine layer based on z-index: negative = behind text, 0+ = on top.
618+
// When z<0 we always use the bottom layer even without allowTransparency —
619+
// the image will simply be hidden behind the opaque text background, which
620+
// is the correct behavior (client asked for "behind text").
621+
const wantsBottom = cmd.zIndex !== undefined && cmd.zIndex < 0;
622+
const layer: ImageLayer = wantsBottom ? 'bottom' : 'top';
623+
624+
if (w !== bitmap.width || h !== bitmap.height) {
625+
const scaled = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h });
626+
bitmap.close();
627+
bitmap = scaled;
619628
}
620-
}
621629

622-
const zIndex = cmd.zIndex ?? 0;
623-
this._kittyStorage.addImage(image.id, finalBitmap, true, layer, zIndex);
630+
// Per spec: X/Y are pixel offsets within the first cell, so clamp to cell dimensions
631+
const xOffset = Math.min(Math.max(0, cmd.xOffset ?? 0), cw - 1);
632+
const yOffset = Math.min(Math.max(0, cmd.yOffset ?? 0), ch - 1);
633+
if (xOffset !== 0 || yOffset !== 0) {
634+
// Per spec: X/Y is not added to c/r area. When c/r are explicit, the
635+
// total placement area remains c*cw × r*ch pixels and the offset image
636+
// is clipped to fit. When c/r are unset, the padded canvas determines
637+
// the natural cell dimensions.
638+
const canvasW = (cmd.columns !== undefined) ? Math.round(imgCols * cw) : bitmap.width + xOffset;
639+
const canvasH = (cmd.rows !== undefined) ? Math.round(imgRows * ch) : bitmap.height + yOffset;
640+
const offsetCanvas = ImageRenderer.createCanvas(window.document, canvasW, canvasH);
641+
const offsetCtx = offsetCanvas.getContext('2d');
642+
if (!offsetCtx) {
643+
throw new Error('Failed to create offset canvas context');
644+
}
645+
offsetCtx.drawImage(bitmap, xOffset, yOffset);
646+
647+
const offsetBitmap = await createImageBitmap(offsetCanvas);
648+
offsetCanvas.width = offsetCanvas.height = 0;
649+
bitmap.close();
650+
bitmap = offsetBitmap;
651+
w = bitmap.width;
652+
h = bitmap.height;
653+
if (w * h > this._opts.pixelLimit) {
654+
throw new Error('image exceeds pixel limit');
655+
}
656+
if (cmd.columns === undefined) {
657+
imgCols = Math.ceil(bitmap.width / cw);
658+
}
659+
if (cmd.rows === undefined) {
660+
imgRows = Math.ceil(bitmap.height / ch);
661+
}
662+
}
624663

625-
// Kitty cursor movement
626-
// Per spec: cursor placed at first column after last image column,
627-
// on the last row of the image. C=1 means don't move cursor.
628-
if (cmd.cursorMovement === 1) {
629-
// C=1: restore cursor to position before image was placed
630-
const scrolled = buffer.ybase - savedYbase;
631-
buffer.x = savedX;
632-
// Can't restore cursor to scrollback?
633-
buffer.y = Math.max(savedY - scrolled, 0);
634-
} else {
635-
// Default (C=0): advance cursor horizontally past the image
636-
// addImage already positioned cursor on the last row via lineFeeds
637-
buffer.x = Math.min(savedX + imgCols, this._coreTerminal.cols);
664+
const zIndex = cmd.zIndex ?? 0;
665+
this._kittyStorage.addImage(image.id, bitmap, true, layer, zIndex);
666+
bitmap = undefined; // ownership transferred to storage
667+
668+
// Kitty cursor movement
669+
// Per spec: cursor placed at first column after last image column,
670+
// on the last row of the image. C=1 means don't move cursor.
671+
if (cmd.cursorMovement === 1) {
672+
// C=1: restore cursor to position before image was placed
673+
const scrolled = buffer.ybase - savedYbase;
674+
buffer.x = savedX;
675+
// Can't restore cursor to scrollback?
676+
buffer.y = Math.max(savedY - scrolled, 0);
677+
} else {
678+
// Default (C=0): advance cursor horizontally past the image
679+
// addImage already positioned cursor on the last row via lineFeeds
680+
buffer.x = Math.min(savedX + imgCols, this._coreTerminal.cols);
681+
}
682+
} catch (e) {
683+
bitmap?.close();
684+
throw e;
638685
}
639686
}
640687

‎addons/addon-image/src/kitty/KittyImageStorage.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,15 @@ export class KittyImageStorage implements IDisposable {
2020

2121
private _nextImageId = 1;
2222
private readonly _images: Map<number, IKittyImageData> = new Map();
23+
// TODO: Support multiple placements per image. The kitty spec identifies
24+
// placements by an (image id, placement id) pair — same i + different p
25+
// values should coexist, and same i + same p should replace the prior
26+
// placement. Currently we track only one storage entry per kitty image id,
27+
// so multiple placements of the same image overwrite each other. Fixing
28+
// this requires changing these maps to Map<number, Map<number, number>>
29+
// (kittyId → placementId → storageId) and updating addImage/deleteById
30+
// accordingly. The underlying shared ImageStorage would also need to
31+
// support multiple entries per logical image.
2332
private readonly _kittyIdToStorageId: Map<number, number> = new Map();
2433
private readonly _storageIdToKittyId: Map<number, number> = new Map();
2534

@@ -81,6 +90,14 @@ export class KittyImageStorage implements IDisposable {
8190
}
8291

8392
public addImage(kittyId: number, image: HTMLCanvasElement | ImageBitmap, scrolling: boolean, layer: ImageLayer, zIndex: number): void {
93+
// Clean up stale reverse-mapping from a previous placement of the same
94+
// kitty image. The old shared-storage entry is kept (it may still be
95+
// visible on screen) but its reverse mapping is removed so that eviction
96+
// of the old entry won't incorrectly delete the kitty image data.
97+
const oldStorageId = this._kittyIdToStorageId.get(kittyId);
98+
if (oldStorageId !== undefined) {
99+
this._storageIdToKittyId.delete(oldStorageId);
100+
}
84101
const storageId = this._storage.addImage(image, scrolling, layer, zIndex);
85102
this._kittyIdToStorageId.set(kittyId, storageId);
86103
this._storageIdToKittyId.set(storageId, kittyId);

0 commit comments

Comments
 (0)