Skip to content

Fix 15bpp handling across encoders #683

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jknockel wants to merge 3 commits into
base: master
Choose a base branch
from
+29 −29
Open

Fix 15bpp handling across encoders #683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2beeb1
zrle: use 2 bytes for 15bpp format
jknockel Dec 6, 2025
ff9a7ba
trle: use 2 bytes for 15bpp format
jknockel Dec 6, 2025
015d586
libvncclient: remove unused CARDREALBPP defines
jknockel Dec 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions src/libvncclient/trle.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
#define REALBPP BPP
#endif

#define REALBytesPP ((REALBPP + 7) / 8)

#if !defined(UNCOMP) || UNCOMP == 0
#define HandleTRLE CONCAT2E(HandleTRLE, REALBPP)
#elif UNCOMP > 0
Expand All @@ -38,7 +40,6 @@
#define HandleTRLE CONCAT3E(HandleTRLE, REALBPP, Up)
#endif
#define CARDBPP CONCAT3E(uint, BPP, _t)
#define CARDREALBPP CONCAT3E(uint, REALBPP, _t)

#if REALBPP != BPP && defined(UNCOMP) && UNCOMP != 0
#if UNCOMP > 0
Expand All @@ -53,7 +54,7 @@
static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) {
int x, y, w, h;
uint8_t type, last_type = 0;
int min_buffer_size = 16 * 16 * (REALBPP / 8) * 2;
int min_buffer_size = 16 * 16 * REALBytesPP * 2;
uint8_t *buffer;
CARDBPP palette[128];
int bpp = 0, mask = 0, divider = 0;
Expand Down Expand Up @@ -93,14 +94,14 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) {

switch (type) {
case 0: {
if (!ReadFromRFBServer(client, (char *)buffer, w * h * REALBPP / 8))
if (!ReadFromRFBServer(client, (char *)buffer, w * h * REALBytesPP))
return FALSE;
#if REALBPP != BPP
int i, j;

for (j = y * client->width; j < (y + h) * client->width;
j += client->width)
for (i = x; i < x + w; i++, buffer += REALBPP / 8)
for (i = x; i < x + w; i++, buffer += REALBytesPP)
((CARDBPP *)client->frameBuffer)[j + i] = UncompressCPixel(buffer);
#else
client->GotBitmap(client, buffer, x, y, w, h);
Expand All @@ -109,7 +110,7 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) {
break;
}
case 1: {
if (!ReadFromRFBServer(client, (char *)buffer, REALBPP / 8))
if (!ReadFromRFBServer(client, (char *)buffer, REALBytesPP))
return FALSE;

color = UncompressCPixel(buffer);
Expand Down Expand Up @@ -171,11 +172,11 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) {
while (j < h) {
int color, length, buffer_pos = 0;
/* read color */
if (!ReadFromRFBServer(client, (char*)buffer, REALBPP / 8 + 1))
if (!ReadFromRFBServer(client, (char*)buffer, REALBytesPP + 1))
return FALSE;
color = UncompressCPixel(buffer);
buffer += REALBPP / 8;
buffer_pos += REALBPP / 8;
buffer += REALBytesPP;
buffer_pos += REALBytesPP;
/* read run length */
length = 1;
while (*buffer == 0xff && buffer_pos < client->raw_buffer_size-1) {
Expand Down Expand Up @@ -260,23 +261,23 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) {
bpp = (type > 4 ? 4 : (type > 2 ? 2 : 1)),
mask = (1 << bpp) - 1, divider = (8 / bpp);

if (!ReadFromRFBServer(client, (char *)buffer, type * REALBPP / 8))
if (!ReadFromRFBServer(client, (char *)buffer, type * REALBytesPP))
return FALSE;

/* read palette */
for (i = 0; i < type; i++, buffer += REALBPP / 8)
for (i = 0; i < type; i++, buffer += REALBytesPP)
palette[i] = UncompressCPixel(buffer);

last_type = type;
goto case_127;
} else if (type >= 130) {
int i;

if (!ReadFromRFBServer(client, (char *)buffer, (type - 128) * REALBPP / 8))
if (!ReadFromRFBServer(client, (char *)buffer, (type - 128) * REALBytesPP))
return FALSE;

/* read palette */
for (i = 0; i < type - 128; i++, buffer += REALBPP / 8)
for (i = 0; i < type - 128; i++, buffer += REALBytesPP)
palette[i] = UncompressCPixel(buffer);

last_type = type;
Expand All @@ -292,7 +293,6 @@ static rfbBool HandleTRLE(rfbClient *client, int rx, int ry, int rw, int rh) {
}

#undef CARDBPP
#undef CARDREALBPP
#undef HandleTRLE
#undef UncompressCPixel
#undef REALBPP
Expand Down
30 changes: 15 additions & 15 deletions src/libvncclient/zrle.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
#define REALBPP BPP
#endif

#define REALBytesPP ((REALBPP + 7) / 8)

#if !defined(UNCOMP) || UNCOMP==0
#define HandleZRLE CONCAT2E(HandleZRLE,REALBPP)
#define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP)
Expand All @@ -43,7 +45,6 @@
#define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up)
#endif
#define CARDBPP CONCAT3E(uint,BPP,_t)
#define CARDREALBPP CONCAT3E(uint,REALBPP,_t)

#define ENDIAN_LITTLE 0
#define ENDIAN_BIG 1
Expand Down Expand Up @@ -87,7 +88,7 @@ HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh)
int remaining;
int inflateResult;
int toRead;
int min_buffer_size = rw * rh * (REALBPP / 8) * 2;
int min_buffer_size = rw * rh * REALBytesPP * 2;

/* First make sure we have a large enough raw buffer to hold the
* decompressed data. In practice, with a fixed REALBPP, fixed frame
Expand Down Expand Up @@ -269,29 +270,29 @@ static int HandleZRLETile(rfbClient* client,
#if REALBPP!=BPP
int i,j;

if(1+w*h*REALBPP/8>buffer_length) {
rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h);
if(1+w*h*REALBytesPP>buffer_length) {
rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBytesPP,buffer_length,w,h);
return -3;
}

for(j=y*client->width; j<(y+h)*client->width; j+=client->width)
for(i=x; i<x+w; i++,buffer+=REALBPP/8)
for(i=x; i<x+w; i++,buffer+=REALBytesPP)
((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer);
#else
client->GotBitmap(client, buffer, x, y, w, h);
buffer+=w*h*REALBPP/8;
buffer+=w*h*REALBytesPP;
#endif
}
else if( type == 1 ) /* solid */
{
CARDBPP color = UncompressCPixel(buffer);

if(1+REALBPP/8>buffer_length)
if(1+REALBytesPP>buffer_length)
return -4;

client->GotFillRect(client, x, y, w, h, color);

buffer+=REALBPP/8;
buffer+=REALBytesPP;

}
else if( type <= 127 ) /* packed Palette */
Expand All @@ -302,11 +303,11 @@ static int HandleZRLETile(rfbClient* client,
mask=(1<<bpp)-1,
divider=(8/bpp);

if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length)
if(1+type*REALBytesPP+((w+divider-1)/divider)*h>buffer_length)
return -5;

/* read palette */
for(i=0; i<type; i++,buffer+=REALBPP/8)
for(i=0; i<type; i++,buffer+=REALBytesPP)
palette[i] = UncompressCPixel(buffer);

/* read palettized pixels */
Expand All @@ -331,10 +332,10 @@ static int HandleZRLETile(rfbClient* client,
while(j<h) {
int color,length;
/* read color */
if(buffer+REALBPP/8+1>buffer_end)
if(buffer+REALBytesPP+1>buffer_end)
return -7;
color = UncompressCPixel(buffer);
buffer+=REALBPP/8;
buffer+=REALBytesPP;
/* read run length */
length=1;
while(*buffer==0xff) {
Expand Down Expand Up @@ -368,11 +369,11 @@ static int HandleZRLETile(rfbClient* client,
CARDBPP palette[128];
int i,j;

if(2+(type-128)*REALBPP/8>buffer_length)
if(2+(type-128)*REALBytesPP>buffer_length)
return -9;

/* read palette */
for(i=0; i<type-128; i++,buffer+=REALBPP/8)
for(i=0; i<type-128; i++,buffer+=REALBytesPP)
palette[i] = UncompressCPixel(buffer);
/* read palettized pixels */
i=j=0;
Expand Down Expand Up @@ -416,7 +417,6 @@ static int HandleZRLETile(rfbClient* client,
}

#undef CARDBPP
#undef CARDREALBPP
#undef HandleZRLE
#undef HandleZRLETile
#undef UncompressCPixel
Expand Down
2 changes: 1 addition & 1 deletion src/libvncserver/zrleencodetemplate.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ void ZRLE_ENCODE_TILE(PIXEL_T* data, int w, int h, zrleOutStream* os,
for (ptr = data; ptr < data+w*h; ptr++)
zrleOutStreamWRITE_PIXEL(os, *ptr);
#else
zrleOutStreamWriteBytes(os, (zrle_U8 *)data, w*h*(BPP/8));
zrleOutStreamWriteBytes(os, (zrle_U8 *)data, w*h*(BPPOUT/8));
#endif
}
}
Expand Down
Loading