fixed errors in message encryption/decryption
[public/netxms.git] / src / libnetxms / crypto.cpp
CommitLineData
5039dede
AK
1/*
2** NetXMS - Network Management System
3** NetXMS Foundation Library
7aad6641 4** Copyright (C) 2003-2013 Victor Kirhenshtein
5039dede
AK
5**
6** This program is free software; you can redistribute it and/or modify
68f384ea
VK
7** it under the terms of the GNU Lesser General Public License as published
8** by the Free Software Foundation; either version 3 of the License, or
5039dede
AK
9** (at your option) any later version.
10**
11** This program is distributed in the hope that it will be useful,
12** but WITHOUT ANY WARRANTY; without even the implied warranty of
13** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14** GNU General Public License for more details.
15**
68f384ea 16** You should have received a copy of the GNU Lesser General Public License
5039dede
AK
17** along with this program; if not, write to the Free Software
18** Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19**
20** File: crypto.cpp
21**
22**/
23
24#include "libnetxms.h"
4e2debcc 25#include "ice.h"
5039dede 26
b1e9b6b3
VK
27/**
28 * Constants
29 */
5039dede
AK
30#define KEY_BUFFER_SIZE 4096
31
b1e9b6b3
VK
32/**
33 * Supported ciphers. By default, we support all ciphers compiled
34 * into OpenSSL library.
35 */
967893bb 36static UINT32 m_dwSupportedCiphers =
5039dede
AK
37#ifdef _WITH_ENCRYPTION
38#ifndef OPENSSL_NO_AES
39 CSCP_SUPPORT_AES_256 |
9e9d631e 40 CSCP_SUPPORT_AES_128 |
5039dede
AK
41#endif
42#ifndef OPENSSL_NO_BF
d772d0dd
VK
43 CSCP_SUPPORT_BLOWFISH_256 |
44 CSCP_SUPPORT_BLOWFISH_128 |
5039dede
AK
45#endif
46#ifndef OPENSSL_NO_IDEA
47 CSCP_SUPPORT_IDEA |
48#endif
49#ifndef OPENSSL_NO_DES
50 CSCP_SUPPORT_3DES |
51#endif
52#endif /* _WITH_ENCRYPTION */
53 0;
54
b1e9b6b3
VK
55/**
56 * Static data
57 */
5039dede
AK
58#ifdef _WITH_ENCRYPTION
59
9e9d631e 60typedef OPENSSL_CONST EVP_CIPHER * (*CIPHER_FUNC)();
5039dede
AK
61static CIPHER_FUNC m_pfCipherList[NETXMS_MAX_CIPHERS] =
62{
63#ifndef OPENSSL_NO_AES
64 EVP_aes_256_cbc,
65#else
66 NULL,
67#endif
68#ifndef OPENSSL_NO_BF
69 EVP_bf_cbc,
70#else
71 NULL,
72#endif
73#ifndef OPENSSL_NO_IDEA
74 EVP_idea_cbc,
75#else
76 NULL,
77#endif
78#ifndef OPENSSL_NO_DES
9e9d631e
VK
79 EVP_des_ede3_cbc,
80#else
81 NULL,
82#endif
83#ifndef OPENSSL_NO_AES
d772d0dd
VK
84 EVP_aes_128_cbc,
85#else
86 NULL,
87#endif
88#ifndef OPENSSL_NO_BF
89 EVP_bf_cbc
5039dede
AK
90#else
91 NULL
92#endif
93};
94static WORD m_wNoEncryptionFlag = 0;
95static MUTEX *m_pCryptoMutexList = NULL;
96
b1e9b6b3
VK
97/**
98 * Locking callback for CRYPTO library
99 */
5039dede
AK
100static void CryptoLockingCallback(int nMode, int nLock, const char *pszFile, int nLine)
101{
102 if (nMode & CRYPTO_LOCK)
c17f6cbc 103 MutexLock(m_pCryptoMutexList[nLock]);
5039dede
AK
104 else
105 MutexUnlock(m_pCryptoMutexList[nLock]);
106}
107
b1e9b6b3
VK
108/**
109 * ID callback for CRYPTO library
110 */
5039dede
AK
111#ifndef _WIN32
112
9e9d631e 113static unsigned long CryptoIdCallback()
5039dede
AK
114{
115 return (unsigned long)GetCurrentThreadId();
116}
117
118#endif
119
120#endif /* _WITH_ENCRYPTION */
121
b1e9b6b3
VK
122/**
123 * Initialize OpenSSL library
124 */
967893bb 125BOOL LIBNETXMS_EXPORTABLE InitCryptoLib(UINT32 dwEnabledCiphers)
5039dede
AK
126{
127#ifdef _WITH_ENCRYPTION
128 BYTE random[8192];
129 int i;
130
131 CRYPTO_malloc_init();
132 ERR_load_CRYPTO_strings();
133 OpenSSL_add_all_algorithms();
134 RAND_seed(random, 8192);
135 m_dwSupportedCiphers &= dwEnabledCiphers;
136 m_wNoEncryptionFlag = htons(MF_DONT_ENCRYPT);
137 m_pCryptoMutexList = (MUTEX *)malloc(sizeof(MUTEX) * CRYPTO_num_locks());
138 for(i = 0; i < CRYPTO_num_locks(); i++)
139 m_pCryptoMutexList[i] = MutexCreate();
140 CRYPTO_set_locking_callback(CryptoLockingCallback);
141#ifndef _WIN32
142 CRYPTO_set_id_callback(CryptoIdCallback);
143#endif /* _WIN32 */
144#endif /* _WITH_ENCRYPTION */
145 return TRUE;
146}
147
b1e9b6b3
VK
148/**
149 * Get supported ciphers
150 */
967893bb 151UINT32 LIBNETXMS_EXPORTABLE CSCPGetSupportedCiphers()
5039dede
AK
152{
153 return m_dwSupportedCiphers;
154}
155
b1e9b6b3
VK
156/**
157 * Encrypt message
158 */
98abc9f1 159CSCP_ENCRYPTED_MESSAGE LIBNETXMS_EXPORTABLE *CSCPEncryptMessage(NXCPEncryptionContext *pCtx, CSCP_MESSAGE *pMsg)
5039dede 160{
e6336a90 161 return (pCtx != NULL) ? pCtx->encryptMessage(pMsg) : NULL;
5039dede
AK
162}
163
7aad6641
VK
164/**
165 * Decrypt message
166 */
e6336a90 167BOOL LIBNETXMS_EXPORTABLE CSCPDecryptMessage(NXCPEncryptionContext *pCtx, CSCP_ENCRYPTED_MESSAGE *pMsg, BYTE *pDecryptionBuffer)
5039dede 168{
e6336a90 169 return (pCtx != NULL) ? pCtx->decryptMessage(pMsg, pDecryptionBuffer) : NULL;
5039dede
AK
170}
171
b1e9b6b3
VK
172/**
173 * Setup encryption context
174 * Function will determine is it called on initiator or responder side
175 * by message code. Initiator should provide it's private key,
176 * and responder should provide pointer to response message.
177 */
967893bb 178UINT32 LIBNETXMS_EXPORTABLE SetupEncryptionContext(CSCPMessage *pMsg,
98abc9f1 179 NXCPEncryptionContext **ppCtx,
5039dede
AK
180 CSCPMessage **ppResponse,
181 RSA *pPrivateKey, int nNXCPVersion)
182{
967893bb 183 UINT32 dwResult = RCC_NOT_IMPLEMENTED;
5039dede 184
98abc9f1 185 *ppCtx = NULL;
5039dede
AK
186#ifdef _WITH_ENCRYPTION
187 if (pMsg->GetCode() == CMD_REQUEST_SESSION_KEY)
188 {
967893bb 189 UINT32 dwCiphers;
5039dede
AK
190
191 *ppResponse = new CSCPMessage(nNXCPVersion);
192 (*ppResponse)->SetCode(CMD_SESSION_KEY);
193 (*ppResponse)->SetId(pMsg->GetId());
4af351c7 194 (*ppResponse)->disableEncryption();
5039dede
AK
195
196 dwCiphers = pMsg->GetVariableLong(VID_SUPPORTED_ENCRYPTION) & m_dwSupportedCiphers;
197 if (dwCiphers == 0)
198 {
199 (*ppResponse)->SetVariable(VID_RCC, RCC_NO_CIPHERS);
200 dwResult = RCC_NO_CIPHERS;
201 }
202 else
203 {
98abc9f1
VK
204 BYTE *pBufPos, ucKeyBuffer[KEY_BUFFER_SIZE];
205 RSA *pServerKey;
5039dede 206
98abc9f1 207 *ppCtx = NXCPEncryptionContext::create(dwCiphers);
5039dede
AK
208
209 // Encrypt key
eae8c74c 210 int size = pMsg->GetVariableBinary(VID_PUBLIC_KEY, ucKeyBuffer, KEY_BUFFER_SIZE);
5039dede 211 pBufPos = ucKeyBuffer;
eae8c74c 212 pServerKey = d2i_RSAPublicKey(NULL, (OPENSSL_CONST BYTE **)&pBufPos, size);
5039dede
AK
213 if (pServerKey != NULL)
214 {
215 (*ppResponse)->SetVariable(VID_RCC, RCC_SUCCESS);
eae8c74c
VK
216
217 size = RSA_public_encrypt((*ppCtx)->getKeyLength(), (*ppCtx)->getSessionKey(), ucKeyBuffer, pServerKey, RSA_PKCS1_OAEP_PADDING);
218 (*ppResponse)->SetVariable(VID_SESSION_KEY, ucKeyBuffer, (UINT32)size);
98abc9f1 219 (*ppResponse)->SetVariable(VID_KEY_LENGTH, (WORD)(*ppCtx)->getKeyLength());
eae8c74c
VK
220
221 int ivLength = EVP_CIPHER_iv_length(m_pfCipherList[(*ppCtx)->getCipher()]());
222 if ((ivLength <= 0) || (ivLength > EVP_MAX_IV_LENGTH))
223 ivLength = EVP_MAX_IV_LENGTH;
224 size = RSA_public_encrypt(ivLength, (*ppCtx)->getIV(), ucKeyBuffer, pServerKey, RSA_PKCS1_OAEP_PADDING);
225 (*ppResponse)->SetVariable(VID_SESSION_IV, ucKeyBuffer, (UINT32)size);
226 (*ppResponse)->SetVariable(VID_IV_LENGTH, (WORD)ivLength);
227
228 (*ppResponse)->SetVariable(VID_CIPHER, (WORD)(*ppCtx)->getCipher());
5039dede
AK
229 RSA_free(pServerKey);
230 dwResult = RCC_SUCCESS;
231 }
232 else
233 {
234 (*ppResponse)->SetVariable(VID_RCC, RCC_INVALID_PUBLIC_KEY);
235 dwResult = RCC_INVALID_PUBLIC_KEY;
236 }
237 }
238 }
239 else if (pMsg->GetCode() == CMD_SESSION_KEY)
240 {
241 dwResult = pMsg->GetVariableLong(VID_RCC);
242 if (dwResult == RCC_SUCCESS)
243 {
98abc9f1
VK
244 *ppCtx = NXCPEncryptionContext::create(pMsg, pPrivateKey);
245 if (*ppCtx == NULL)
246 {
247 dwResult = RCC_INVALID_SESSION_KEY;
248 }
5039dede
AK
249 }
250 }
251
252 if ((dwResult != RCC_SUCCESS) && (*ppCtx != NULL))
253 {
98abc9f1 254 delete *ppCtx;
5039dede
AK
255 *ppCtx = NULL;
256 }
257#else
258 if (pMsg->GetCode() == CMD_REQUEST_SESSION_KEY)
259 {
260 *ppResponse = new CSCPMessage(nNXCPVersion);
261 (*ppResponse)->SetCode(CMD_SESSION_KEY);
262 (*ppResponse)->SetId(pMsg->GetId());
26de9040 263 (*ppResponse)->disableEncryption();
5039dede
AK
264 (*ppResponse)->SetVariable(VID_RCC, dwResult);
265 }
266#endif
267
268 return dwResult;
269}
270
b1e9b6b3
VK
271/**
272 * Prepare session key request message
273 */
9e9d631e 274void LIBNETXMS_EXPORTABLE PrepareKeyRequestMsg(CSCPMessage *pMsg, RSA *pServerKey, bool useX509Format)
5039dede
AK
275{
276#ifdef _WITH_ENCRYPTION
277 int iLen;
278 BYTE *pKeyBuffer, *pBufPos;
279
280 pMsg->SetCode(CMD_REQUEST_SESSION_KEY);
281 pMsg->SetVariable(VID_SUPPORTED_ENCRYPTION, m_dwSupportedCiphers);
282
9e9d631e
VK
283 if (useX509Format)
284 {
285 iLen = i2d_RSA_PUBKEY(pServerKey, NULL);
286 pKeyBuffer = (BYTE *)malloc(iLen);
287 pBufPos = pKeyBuffer;
288 i2d_RSA_PUBKEY(pServerKey, &pBufPos);
289 }
290 else
291 {
292 iLen = i2d_RSAPublicKey(pServerKey, NULL);
293 pKeyBuffer = (BYTE *)malloc(iLen);
294 pBufPos = pKeyBuffer;
295 i2d_RSAPublicKey(pServerKey, &pBufPos);
296 }
297 pMsg->SetVariable(VID_PUBLIC_KEY, pKeyBuffer, iLen);
5039dede
AK
298 free(pKeyBuffer);
299#endif
300}
301
7aad6641
VK
302/**
303 * Load RSA key(s) from file
304 */
5039dede
AK
305RSA LIBNETXMS_EXPORTABLE *LoadRSAKeys(const TCHAR *pszKeyFile)
306{
307#ifdef _WITH_ENCRYPTION
308 FILE *fp;
309 BYTE *pKeyBuffer, *pBufPos, hash[SHA1_DIGEST_SIZE];
967893bb 310 UINT32 dwLen;
5039dede
AK
311 RSA *pKey = NULL;
312
313 fp = _tfopen(pszKeyFile, _T("rb"));
314 if (fp != NULL)
315 {
967893bb 316 if (fread(&dwLen, 1, sizeof(UINT32), fp) == sizeof(UINT32) && dwLen < 10 * 1024)
5039dede
AK
317 {
318 pKeyBuffer = (BYTE *)malloc(dwLen);
319 pBufPos = pKeyBuffer;
320 if (fread(pKeyBuffer, 1, dwLen, fp) == dwLen)
321 {
322 BYTE hash2[SHA1_DIGEST_SIZE];
323
7aad6641 324 if (fread(hash, 1, SHA1_DIGEST_SIZE, fp) == SHA1_DIGEST_SIZE)
5039dede 325 {
7aad6641
VK
326 CalculateSHA1Hash(pKeyBuffer, dwLen, hash2);
327 if (!memcmp(hash, hash2, SHA1_DIGEST_SIZE))
5039dede 328 {
7aad6641
VK
329 pKey = d2i_RSAPublicKey(NULL, (OPENSSL_CONST BYTE **)&pBufPos, dwLen);
330 if (pKey != NULL)
5039dede 331 {
7aad6641 332 if (d2i_RSAPrivateKey(&pKey, (OPENSSL_CONST BYTE **)&pBufPos,
967893bb 333 dwLen - CAST_FROM_POINTER((pBufPos - pKeyBuffer), UINT32)) == NULL)
7aad6641
VK
334 {
335 RSA_free(pKey);
336 pKey = NULL;
337 }
5039dede
AK
338 }
339 }
340 }
341 }
342 free(pKeyBuffer);
343 }
344 fclose(fp);
345 }
346
347 return pKey;
348#else
349 return NULL;
350#endif
351}
352
7aad6641
VK
353/**
354 * Create signature for message using certificate (MS CAPI version)
355 * Paraeters:
356 * pMsg and dwMsgLen - message to sign and it's length
357 * pCert - certificate
358 * pBuffer - output buffer
359 * dwBufSize - buffer size
360 * pdwSigLen - actual signature size
361 */
5039dede
AK
362#ifdef _WIN32
363
967893bb
VK
364BOOL LIBNETXMS_EXPORTABLE SignMessageWithCAPI(BYTE *pMsg, UINT32 dwMsgLen, const CERT_CONTEXT *pCert,
365 BYTE *pBuffer, UINT32 dwBufSize, UINT32 *pdwSigLen)
5039dede
AK
366{
367 BOOL bFreeProv, bRet = FALSE;
368 DWORD i, j, dwLen, dwKeySpec;
369 HCRYPTPROV hProv;
370 HCRYPTHASH hHash;
371 BYTE *pTemp;
372
373 if (CryptAcquireCertificatePrivateKey(pCert, CRYPT_ACQUIRE_COMPARE_KEY_FLAG,
374 NULL, &hProv, &dwKeySpec, &bFreeProv))
375 {
376 if (CryptCreateHash(hProv, CALG_SHA1, NULL, 0, &hHash))
377 {
378 if (CryptHashData(hHash, pMsg, dwMsgLen, 0))
379 {
380 dwLen = dwBufSize;
381 pTemp = (BYTE *)malloc(dwBufSize);;
382 bRet = CryptSignHash(hHash, dwKeySpec, NULL, 0, pTemp, &dwLen);
383 *pdwSigLen = dwLen;
384 // we have to reverse the byte-order in the result from CryptSignHash()
385 for(i = 0, j = dwLen - 1; i < dwLen; i++, j--)
386 pBuffer[i] = pTemp[j];
387 }
388 CryptDestroyHash(hHash);
389 }
390
391 if (bFreeProv)
392 CryptReleaseContext(hProv, 0);
393 }
394 return bRet;
395}
396
397#endif
4e2debcc 398
7aad6641
VK
399/**
400 * Encrypt data block with ICE
401 */
4e2debcc
VK
402void LIBNETXMS_EXPORTABLE ICEEncryptData(const BYTE *in, int inLen, BYTE *out, const BYTE *key)
403{
404 ICE_KEY *ice = ice_key_create(1);
405 ice_key_set(ice, key);
406
407 int stopPos = inLen - (inLen % 8);
4e2debcc
VK
408 for(int pos = 0; pos < stopPos; pos += 8)
409 ice_key_encrypt(ice, &in[pos], &out[pos]);
410
411 if (stopPos < inLen)
412 {
413 BYTE plainText[8], encrypted[8];
414
415 memcpy(plainText, &in[stopPos], inLen - stopPos);
416 ice_key_encrypt(ice, plainText, encrypted);
417 memcpy(&out[stopPos], encrypted, inLen - stopPos);
418 }
419
420 ice_key_destroy(ice);
421}
422
7aad6641
VK
423/**
424 * Decrypt data block with ICE
425 */
4e2debcc
VK
426void LIBNETXMS_EXPORTABLE ICEDecryptData(const BYTE *in, int inLen, BYTE *out, const BYTE *key)
427{
428 ICE_KEY *ice = ice_key_create(1);
429 ice_key_set(ice, key);
430
431 int stopPos = inLen - (inLen % 8);
4e2debcc
VK
432 for(int pos = 0; pos < stopPos; pos += 8)
433 ice_key_decrypt(ice, &in[pos], &out[pos]);
434
435 if (stopPos < inLen)
436 {
437 BYTE plainText[8], encrypted[8];
438
439 memcpy(encrypted, &in[stopPos], inLen - stopPos);
440 ice_key_decrypt(ice, encrypted, plainText);
441 memcpy(&out[stopPos], plainText, inLen - stopPos);
442 }
443
444 ice_key_destroy(ice);
445}
98abc9f1 446
7aad6641
VK
447/**
448 * Encryption context constructor
449 */
98abc9f1
VK
450NXCPEncryptionContext::NXCPEncryptionContext()
451{
452 m_sessionKey = NULL;
e6336a90
VK
453 EVP_CIPHER_CTX_init(&m_encryptor);
454 EVP_CIPHER_CTX_init(&m_decryptor);
98abc9f1
VK
455}
456
7aad6641
VK
457/**
458 * Encryption context destructor
459 */
98abc9f1
VK
460NXCPEncryptionContext::~NXCPEncryptionContext()
461{
462 safe_free(m_sessionKey);
e6336a90
VK
463 EVP_CIPHER_CTX_cleanup(&m_encryptor);
464 EVP_CIPHER_CTX_cleanup(&m_decryptor);
98abc9f1
VK
465}
466
7aad6641
VK
467/**
468 * Create encryption context from CMD_SESSION_KEY NXCP message
469 */
98abc9f1
VK
470NXCPEncryptionContext *NXCPEncryptionContext::create(CSCPMessage *msg, RSA *privateKey)
471{
e4cba530 472#ifdef _WITH_ENCRYPTION
98abc9f1 473 BYTE ucKeyBuffer[KEY_BUFFER_SIZE], ucSessionKey[KEY_BUFFER_SIZE];
967893bb 474 UINT32 dwKeySize;
98abc9f1
VK
475 int nSize, nIVLen;
476 NXCPEncryptionContext *ctx = new NXCPEncryptionContext;
477
e6336a90
VK
478 int cipher = (int)msg->GetVariableShort(VID_CIPHER);
479 if (ctx->initCipher(cipher))
98abc9f1 480 {
e6336a90 481 if (ctx->m_keyLength == (int)msg->GetVariableShort(VID_KEY_LENGTH))
98abc9f1 482 {
e6336a90
VK
483 ctx->m_sessionKey = (BYTE *)malloc(ctx->m_keyLength);
484
485 // Decrypt session key
486 dwKeySize = msg->GetVariableBinary(VID_SESSION_KEY, ucKeyBuffer, KEY_BUFFER_SIZE);
487 nSize = RSA_private_decrypt(dwKeySize, ucKeyBuffer, ucSessionKey, privateKey, RSA_PKCS1_OAEP_PADDING);
488 if (nSize == ctx->m_keyLength)
489 {
490 memcpy(ctx->m_sessionKey, ucSessionKey, nSize);
491
492 // Decrypt session IV
493 nIVLen = msg->GetVariableShort(VID_IV_LENGTH);
494 if (nIVLen == 0) // Versions prior to 0.2.13 don't send IV length, assume 16
495 nIVLen = 16;
496 dwKeySize = msg->GetVariableBinary(VID_SESSION_IV, ucKeyBuffer, KEY_BUFFER_SIZE);
497 nSize = RSA_private_decrypt(dwKeySize, ucKeyBuffer, ucSessionKey, privateKey, RSA_PKCS1_OAEP_PADDING);
498 if ((nSize == nIVLen) &&
499 (nIVLen <= EVP_CIPHER_iv_length(m_pfCipherList[ctx->m_cipher]())))
500 {
501 memcpy(ctx->m_iv, ucSessionKey, min(EVP_MAX_IV_LENGTH, nIVLen));
502 }
503 else
504 {
505 delete_and_null(ctx);
506 }
507 }
508 else
509 {
510 delete_and_null(ctx);
511 }
98abc9f1
VK
512 }
513 else
514 {
515 delete_and_null(ctx);
516 }
517 }
518 else
519 {
520 delete_and_null(ctx);
521 }
522 return ctx;
e4cba530
VK
523#else
524 return new NXCPEncryptionContext;
525#endif
98abc9f1
VK
526}
527
e6336a90
VK
528/**
529 * Initialize cipher
530 */
531bool NXCPEncryptionContext::initCipher(int cipher)
532{
533#ifdef _WITH_ENCRYPTION
534 if (m_pfCipherList[cipher] == NULL)
535 return false; // Unsupported cipher
536
537 if (!EVP_EncryptInit_ex(&m_encryptor, m_pfCipherList[cipher](), NULL, NULL, NULL))
538 return false;
539 if (!EVP_DecryptInit_ex(&m_decryptor, m_pfCipherList[cipher](), NULL, NULL, NULL))
540 return false;
541
542 switch(cipher)
543 {
544 case CSCP_CIPHER_AES_256:
545 m_keyLength = 32;
546 break;
547 case CSCP_CIPHER_AES_128:
548 m_keyLength = 16;
549 break;
550 case CSCP_CIPHER_BLOWFISH_256:
551 m_keyLength = 32;
552 break;
553 case CSCP_CIPHER_BLOWFISH_128:
554 m_keyLength = 16;
555 break;
556 case CSCP_CIPHER_IDEA:
557 m_keyLength = 16;
558 break;
559 case CSCP_CIPHER_3DES:
560 m_keyLength = 24;
561 break;
562 default:
563 return false;
564 }
565
566 if (!EVP_CIPHER_CTX_set_key_length(&m_encryptor, m_keyLength) || !EVP_CIPHER_CTX_set_key_length(&m_decryptor, m_keyLength))
567 return false;
568
569 m_cipher = cipher;
570 return true;
571#else
572 return false;
573#endif
574}
575
7aad6641
VK
576/**
577 * Create encryption context from CMD_REQUEST_SESSION_KEY NXCP message
578 */
967893bb 579NXCPEncryptionContext *NXCPEncryptionContext::create(UINT32 ciphers)
98abc9f1
VK
580{
581 NXCPEncryptionContext *ctx = new NXCPEncryptionContext;
582
e4cba530 583#ifdef _WITH_ENCRYPTION
98abc9f1 584 // Select cipher
e6336a90
VK
585 bool selected = false;
586
98abc9f1
VK
587 if (ciphers & CSCP_SUPPORT_AES_256)
588 {
e6336a90 589 selected = ctx->initCipher(CSCP_CIPHER_AES_256);
98abc9f1 590 }
e6336a90
VK
591
592 if (!selected && (ciphers & CSCP_SUPPORT_BLOWFISH_256))
98abc9f1 593 {
e6336a90 594 selected = ctx->initCipher(CSCP_CIPHER_BLOWFISH_256);
98abc9f1 595 }
e6336a90
VK
596
597 if (!selected && (ciphers & CSCP_SUPPORT_AES_128))
9e9d631e 598 {
e6336a90 599 selected = ctx->initCipher(CSCP_CIPHER_AES_128);
9e9d631e 600 }
e6336a90
VK
601
602 if (!selected && (ciphers & CSCP_SUPPORT_BLOWFISH_128))
d772d0dd 603 {
e6336a90 604 selected = ctx->initCipher(CSCP_CIPHER_BLOWFISH_128);
d772d0dd 605 }
e6336a90
VK
606
607 if (!selected && (ciphers & CSCP_SUPPORT_IDEA))
98abc9f1 608 {
e6336a90 609 selected = ctx->initCipher(CSCP_CIPHER_IDEA);
98abc9f1 610 }
e6336a90
VK
611
612 if (!selected && (ciphers & CSCP_SUPPORT_3DES))
98abc9f1 613 {
e6336a90
VK
614 selected = ctx->initCipher(CSCP_CIPHER_3DES);
615 }
616
617 if (!selected)
618 {
619 delete ctx;
620 return NULL;
98abc9f1
VK
621 }
622
623 // Generate key
624 ctx->m_sessionKey = (BYTE *)malloc(ctx->m_keyLength);
625 RAND_bytes(ctx->m_sessionKey, ctx->m_keyLength);
626 RAND_bytes(ctx->m_iv, EVP_MAX_IV_LENGTH);
e4cba530 627#endif
98abc9f1
VK
628
629 return ctx;
630}
e6336a90
VK
631
632/**
633 * Encrypt message
634 */
635CSCP_ENCRYPTED_MESSAGE *NXCPEncryptionContext::encryptMessage(CSCP_MESSAGE *msg)
636{
637 if (msg->wFlags & m_wNoEncryptionFlag)
638 return (CSCP_ENCRYPTED_MESSAGE *)nx_memdup(msg, ntohl(msg->dwSize));
639
640#ifdef _WITH_ENCRYPTION
641 if (!EVP_EncryptInit_ex(&m_encryptor, NULL, NULL, m_sessionKey, m_iv))
642 return NULL;
643
644 UINT32 msgSize = ntohl(msg->dwSize);
645 CSCP_ENCRYPTED_MESSAGE *emsg =
646 (CSCP_ENCRYPTED_MESSAGE *)malloc(msgSize + CSCP_ENCRYPTION_HEADER_SIZE + EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(&m_encryptor)) + 8);
647 emsg->wCode = htons(CMD_ENCRYPTED_MESSAGE);
648 emsg->nReserved = 0;
649
650 CSCP_ENCRYPTED_PAYLOAD_HEADER header;
651 header.dwChecksum = htonl(CalculateCRC32((BYTE *)msg, msgSize, 0));
652 header.dwReserved = 0;
653
654 int dataSize;
655 EVP_EncryptUpdate(&m_encryptor, emsg->data, &dataSize, (BYTE *)&header, CSCP_EH_ENCRYPTED_BYTES);
656 msgSize = dataSize;
657 EVP_EncryptUpdate(&m_encryptor, emsg->data + msgSize, &dataSize, (BYTE *)msg, ntohl(msg->dwSize));
658 msgSize += dataSize;
659 EVP_EncryptFinal_ex(&m_encryptor, emsg->data + msgSize, &dataSize);
660 msgSize += dataSize + CSCP_EH_UNENCRYPTED_BYTES;
661
662 if (msgSize % 8 != 0)
663 {
664 emsg->nPadding = (BYTE)(8 - (msgSize % 8));
665 msgSize += emsg->nPadding;
666 }
667 else
668 {
669 emsg->nPadding = 0;
670 }
671 emsg->dwSize = htonl(msgSize);
672
673 return emsg;
674#else /* _WITH_ENCRYPTION */
675 return NULL;
676#endif
677}
678
679/**
680 * Decrypt message
681 */
682bool NXCPEncryptionContext::decryptMessage(CSCP_ENCRYPTED_MESSAGE *msg, BYTE *decryptionBuffer)
683{
684#ifdef _WITH_ENCRYPTION
685 if (!EVP_DecryptInit_ex(&m_decryptor, NULL, NULL, m_sessionKey, m_iv))
686 return false;
687
688 msg->dwSize = ntohl(msg->dwSize);
689 int dataSize;
690 EVP_DecryptUpdate(&m_decryptor, decryptionBuffer, &dataSize, msg->data,
691 msg->dwSize - CSCP_EH_UNENCRYPTED_BYTES - msg->nPadding);
692 EVP_DecryptFinal(&m_decryptor, decryptionBuffer + dataSize, &dataSize);
693
694 CSCP_MESSAGE *clearMsg = (CSCP_MESSAGE *)(decryptionBuffer + CSCP_EH_ENCRYPTED_BYTES);
695 UINT32 msgSize = ntohl(clearMsg->dwSize);
696 if (msgSize > msg->dwSize)
697 return false; // Message decrypted incorrectly, because it can't be larger than encrypted
698 UINT32 crc32 = CalculateCRC32((BYTE *)clearMsg, msgSize, 0);
699 if (crc32 != ntohl(((CSCP_ENCRYPTED_PAYLOAD_HEADER *)decryptionBuffer)->dwChecksum))
700 return false; // Bad checksum
701
702 memcpy(msg, clearMsg, msgSize);
703 return true;
704#else /* _WITH_ENCRYPTION */
705 return false;
706#endif
707}