diff --git a/include/rfb/rfbproto.h b/include/rfb/rfbproto.h index ebcd303c..edfba4ac 100644 --- a/include/rfb/rfbproto.h +++ b/include/rfb/rfbproto.h @@ -1557,7 +1557,7 @@ typedef union { extern int rfbEncryptAndStorePasswd(char *passwd, char *fname); extern char *rfbDecryptPasswdFromFile(char *fname); extern void rfbRandomBytes(unsigned char *bytes); -extern void rfbEncryptBytes(unsigned char *bytes, char *passwd); +extern rfbBool rfbEncryptBytes(unsigned char *bytes, char *passwd); #endif diff --git a/src/common/vncauth.c b/src/common/vncauth.c index c3f318c3..1ec6d4ca 100644 --- a/src/common/vncauth.c +++ b/src/common/vncauth.c @@ -102,7 +102,10 @@ rfbEncryptAndStorePasswd(char *passwd, char *fname) /* Do encryption in-place - this way we overwrite our copy of the plaintext password */ - encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd)); + if (encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd)) == 0) { + fclose(fp); + return 1; + } for (i = 0; i < 8; i++) { putc(encryptedPasswd[i], fp); @@ -177,10 +180,11 @@ rfbRandomBytes(unsigned char *bytes) #endif /* - * Encrypt CHALLENGESIZE bytes in memory using a password. + * Encrypt CHALLENGESIZE bytes in memory using a password. Returns TRUE + * if successful, FALSE if the encryption failed. */ -void +rfbBool rfbEncryptBytes(unsigned char *bytes, char *passwd) { unsigned char key[8]; @@ -197,19 +201,24 @@ rfbEncryptBytes(unsigned char *bytes, char *passwd) } } - encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE); + return encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE) != 0; } -void +rfbBool rfbEncryptBytes2(unsigned char *where, const int length, unsigned char *key) { int i, j, out_len; for (i = 0; i< 8; i++) where[i] ^= key[i]; - encrypt_rfbdes(where, &out_len, key, where, 8); + if (encrypt_rfbdes(where, &out_len, key, where, 8) == 0) { + return FALSE; + } for (i = 8; i < length; i += 8) { for (j = 0; j < 8; j++) { where[i + j] ^= where[i + j - 8]; } - encrypt_rfbdes(where + i, &out_len, key, where + i, 8); + if (encrypt_rfbdes(where + i, &out_len, key, where + i, 8) == 0) { + return FALSE; + } } + return TRUE; } diff --git a/src/libvncclient/rfbclient.c b/src/libvncclient/rfbclient.c index 94a37149..6544badc 100644 --- a/src/libvncclient/rfbclient.c +++ b/src/libvncclient/rfbclient.c @@ -404,8 +404,8 @@ rfbBool ConnectToRFBRepeater(rfbClient* client,const char *repeaterHost, int rep return TRUE; } -extern void rfbClientEncryptBytes(unsigned char* bytes, char* passwd); -extern void rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key); +extern rfbBool rfbClientEncryptBytes(unsigned char* bytes, char* passwd); +extern rfbBool rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key); static void ReadReason(rfbClient* client) @@ -585,7 +585,10 @@ HandleVncAuth(rfbClient *client) passwd[8] = '\0'; } - rfbClientEncryptBytes(challenge, passwd); + if (!rfbClientEncryptBytes(challenge, passwd)) { + rfbClientLog("Encryption failed\n"); + return FALSE; + } /* Lose the password from memory */ for (i = strlen(passwd); i >= 0; i--) { @@ -733,8 +736,14 @@ HandleUltraMSLogonIIAuth(rfbClient *client) strncpy((char *)password, cred->userCredential.password, sizeof(password)-1); FreeUserCredential(cred); - rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key); - rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key); + if (!rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key)) { + rfbClientLog("Encrypting username failed\n"); + return FALSE; + } + if (!rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key)) { + rfbClientLog("Encrypting password failed\n"); + return FALSE; + } if (!WriteToRFBServer(client, (char *)pub, sizeof(pub))) return FALSE; if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE; @@ -789,8 +798,14 @@ HandleMSLogonAuth(rfbClient *client) pub = rfbClientSwap64IfLE(pub); key = rfbClientSwap64IfLE(key); - rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)&key); - rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)&key); + if (!rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key)) { + rfbClientLog("Encrypting username failed\n"); + return FALSE; + } + if (!rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key)) { + rfbClientLog("Encrypting password failed\n"); + return FALSE; + } if (!WriteToRFBServer(client, (char *)&pub, 8)) return FALSE; if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE; diff --git a/src/libvncserver/main.c b/src/libvncserver/main.c index 3387d166..0c7390d1 100644 --- a/src/libvncserver/main.c +++ b/src/libvncserver/main.c @@ -798,7 +798,11 @@ static rfbBool rfbDefaultPasswordCheck(rfbClientPtr cl,const char* response,int return(FALSE); } - rfbEncryptBytes(cl->authChallenge, passwd); + if (!rfbEncryptBytes(cl->authChallenge, passwd)) { + rfbErr("Encryption failed\n"); + free(passwd); + return(FALSE); + } /* Lose the password from memory */ for (i = strlen(passwd); i >= 0; i--) { @@ -826,7 +830,10 @@ rfbBool rfbCheckPasswordByList(rfbClientPtr cl,const char* response,int len) for(passwds=(char**)cl->screen->authPasswdData;*passwds;passwds++,i++) { uint8_t auth_tmp[CHALLENGESIZE]; memcpy((char *)auth_tmp, (char *)cl->authChallenge, CHALLENGESIZE); - rfbEncryptBytes(auth_tmp, *passwds); + if (!rfbEncryptBytes(auth_tmp, *passwds)) { + rfbErr("Encryption failed\n"); + return(FALSE); + } if (memcmp(auth_tmp, response, len) == 0) { if(i>=cl->screen->authPasswdFirstViewOnly)