fixed build error; fixed compilation warnings
[public/netxms.git] / src / libnetxms / crypto.cpp
CommitLineData
5039dede
AK
1/*
2** NetXMS - Network Management System
3** NetXMS Foundation Library
7aad6641 4** Copyright (C) 2003-2013 Victor Kirhenshtein
5039dede
AK
5**
6** This program is free software; you can redistribute it and/or modify
68f384ea
VK
7** it under the terms of the GNU Lesser General Public License as published
8** by the Free Software Foundation; either version 3 of the License, or
5039dede
AK
9** (at your option) any later version.
10**
11** This program is distributed in the hope that it will be useful,
12** but WITHOUT ANY WARRANTY; without even the implied warranty of
13** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14** GNU General Public License for more details.
15**
68f384ea 16** You should have received a copy of the GNU Lesser General Public License
5039dede
AK
17** along with this program; if not, write to the Free Software
18** Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19**
20** File: crypto.cpp
21**
22**/
23
24#include "libnetxms.h"
4e2debcc 25#include "ice.h"
5039dede 26
b1e9b6b3
VK
27/**
28 * Constants
29 */
5039dede
AK
30#define KEY_BUFFER_SIZE 4096
31
b1e9b6b3
VK
32/**
33 * Supported ciphers. By default, we support all ciphers compiled
34 * into OpenSSL library.
35 */
967893bb 36static UINT32 m_dwSupportedCiphers =
5039dede
AK
37#ifdef _WITH_ENCRYPTION
38#ifndef OPENSSL_NO_AES
39 CSCP_SUPPORT_AES_256 |
9e9d631e 40 CSCP_SUPPORT_AES_128 |
5039dede
AK
41#endif
42#ifndef OPENSSL_NO_BF
d772d0dd
VK
43 CSCP_SUPPORT_BLOWFISH_256 |
44 CSCP_SUPPORT_BLOWFISH_128 |
5039dede
AK
45#endif
46#ifndef OPENSSL_NO_IDEA
47 CSCP_SUPPORT_IDEA |
48#endif
49#ifndef OPENSSL_NO_DES
50 CSCP_SUPPORT_3DES |
51#endif
52#endif /* _WITH_ENCRYPTION */
53 0;
54
b1e9b6b3
VK
55/**
56 * Static data
57 */
e4e091f0 58static void (*s_debugCallback)(int, const TCHAR *, va_list args) = NULL;
c1099678 59static WORD s_noEncryptionFlag = 0;
e4e091f0 60
5039dede
AK
61#ifdef _WITH_ENCRYPTION
62
bb586165 63extern "C" typedef OPENSSL_CONST EVP_CIPHER * (*CIPHER_FUNC)();
6468147c 64static CIPHER_FUNC s_ciphers[NETXMS_MAX_CIPHERS] =
5039dede
AK
65{
66#ifndef OPENSSL_NO_AES
67 EVP_aes_256_cbc,
68#else
69 NULL,
70#endif
71#ifndef OPENSSL_NO_BF
72 EVP_bf_cbc,
73#else
74 NULL,
75#endif
76#ifndef OPENSSL_NO_IDEA
77 EVP_idea_cbc,
78#else
79 NULL,
80#endif
81#ifndef OPENSSL_NO_DES
9e9d631e
VK
82 EVP_des_ede3_cbc,
83#else
84 NULL,
85#endif
86#ifndef OPENSSL_NO_AES
d772d0dd
VK
87 EVP_aes_128_cbc,
88#else
89 NULL,
90#endif
91#ifndef OPENSSL_NO_BF
92 EVP_bf_cbc
5039dede
AK
93#else
94 NULL
95#endif
96};
6468147c 97static const TCHAR *s_cipherNames[NETXMS_MAX_CIPHERS] = { _T("AES-256"), _T("Blowfish-256"), _T("IDEA"), _T("3DES"), _T("AES-128"), _T("Blowfish-128") };
6468147c 98static MUTEX *s_cryptoMutexList = NULL;
5039dede 99
b1e9b6b3
VK
100/**
101 * Locking callback for CRYPTO library
102 */
bb586165
VK
103#if defined(__SUNPRO_CC)
104extern "C"
105#endif
5039dede
AK
106static void CryptoLockingCallback(int nMode, int nLock, const char *pszFile, int nLine)
107{
108 if (nMode & CRYPTO_LOCK)
6468147c 109 MutexLock(s_cryptoMutexList[nLock]);
5039dede 110 else
6468147c 111 MutexUnlock(s_cryptoMutexList[nLock]);
5039dede
AK
112}
113
b1e9b6b3
VK
114/**
115 * ID callback for CRYPTO library
116 */
5039dede
AK
117#ifndef _WIN32
118
bb586165
VK
119#if defined(__SUNPRO_CC)
120extern "C"
121#endif
9e9d631e 122static unsigned long CryptoIdCallback()
5039dede
AK
123{
124 return (unsigned long)GetCurrentThreadId();
125}
126
127#endif
128
129#endif /* _WITH_ENCRYPTION */
130
b1e9b6b3 131/**
6468147c
VK
132 * Debug output
133 */
134static void CryptoDbgPrintf(int level, const TCHAR *format, ...)
135{
136 if (s_debugCallback == NULL)
137 return;
138
139 va_list args;
140 va_start(args, format);
141 s_debugCallback(level, format, args);
142 va_end(args);
143}
144
145/**
b1e9b6b3
VK
146 * Initialize OpenSSL library
147 */
6468147c 148BOOL LIBNETXMS_EXPORTABLE InitCryptoLib(UINT32 dwEnabledCiphers, void (*debugCallback)(int, const TCHAR *, va_list args))
5039dede 149{
6468147c 150 s_debugCallback = debugCallback;
c1099678 151 s_noEncryptionFlag = htons(MF_DONT_ENCRYPT);
6468147c 152
5039dede
AK
153#ifdef _WITH_ENCRYPTION
154 BYTE random[8192];
155 int i;
156
157 CRYPTO_malloc_init();
158 ERR_load_CRYPTO_strings();
159 OpenSSL_add_all_algorithms();
160 RAND_seed(random, 8192);
6468147c 161 s_cryptoMutexList = (MUTEX *)malloc(sizeof(MUTEX) * CRYPTO_num_locks());
5039dede 162 for(i = 0; i < CRYPTO_num_locks(); i++)
6468147c 163 s_cryptoMutexList[i] = MutexCreate();
5039dede
AK
164 CRYPTO_set_locking_callback(CryptoLockingCallback);
165#ifndef _WIN32
166 CRYPTO_set_id_callback(CryptoIdCallback);
167#endif /* _WIN32 */
6468147c
VK
168
169 // validate supported ciphers
ae8e3292 170 CryptoDbgPrintf(1, _T("Validating ciphers"));
6468147c
VK
171 m_dwSupportedCiphers &= dwEnabledCiphers;
172 UINT32 cipherBit = 1;
173 for(i = 0; i < NETXMS_MAX_CIPHERS; i++, cipherBit = cipherBit << 1)
174 {
175 if ((m_dwSupportedCiphers & cipherBit) == 0)
ae8e3292
VK
176 {
177 CryptoDbgPrintf(1, _T(" %s disabled (config)"), s_cipherNames[i]);
6468147c 178 continue;
ae8e3292 179 }
6468147c
VK
180 NXCPEncryptionContext *ctx = NXCPEncryptionContext::create(cipherBit);
181 if (ctx != NULL)
182 {
183 delete ctx;
ae8e3292 184 CryptoDbgPrintf(1, _T(" %s enabled"), s_cipherNames[i]);
6468147c
VK
185 }
186 else
187 {
188 m_dwSupportedCiphers &= ~cipherBit;
ae8e3292 189 CryptoDbgPrintf(1, _T(" %s disabled (validation failed)"), s_cipherNames[i]);
6468147c
VK
190 }
191 }
192
193 CryptoDbgPrintf(1, _T("Crypto library initialized"));
194
195#else
196 CryptoDbgPrintf(1, _T("Crypto library will not be initialized because libnetxms was built without encryption support"));
5039dede
AK
197#endif /* _WITH_ENCRYPTION */
198 return TRUE;
199}
200
b1e9b6b3
VK
201/**
202 * Get supported ciphers
203 */
967893bb 204UINT32 LIBNETXMS_EXPORTABLE CSCPGetSupportedCiphers()
5039dede
AK
205{
206 return m_dwSupportedCiphers;
207}
208
b1e9b6b3
VK
209/**
210 * Encrypt message
211 */
6be0a20b 212NXCP_ENCRYPTED_MESSAGE LIBNETXMS_EXPORTABLE *CSCPEncryptMessage(NXCPEncryptionContext *pCtx, CSCP_MESSAGE *pMsg)
5039dede 213{
e6336a90 214 return (pCtx != NULL) ? pCtx->encryptMessage(pMsg) : NULL;
5039dede
AK
215}
216
7aad6641
VK
217/**
218 * Decrypt message
219 */
6be0a20b 220BOOL LIBNETXMS_EXPORTABLE CSCPDecryptMessage(NXCPEncryptionContext *pCtx, NXCP_ENCRYPTED_MESSAGE *pMsg, BYTE *pDecryptionBuffer)
5039dede 221{
79bef515 222 return (pCtx != NULL) ? pCtx->decryptMessage(pMsg, pDecryptionBuffer) : FALSE;
5039dede
AK
223}
224
b1e9b6b3
VK
225/**
226 * Setup encryption context
227 * Function will determine is it called on initiator or responder side
228 * by message code. Initiator should provide it's private key,
229 * and responder should provide pointer to response message.
230 */
967893bb 231UINT32 LIBNETXMS_EXPORTABLE SetupEncryptionContext(CSCPMessage *pMsg,
98abc9f1 232 NXCPEncryptionContext **ppCtx,
5039dede
AK
233 CSCPMessage **ppResponse,
234 RSA *pPrivateKey, int nNXCPVersion)
235{
967893bb 236 UINT32 dwResult = RCC_NOT_IMPLEMENTED;
5039dede 237
98abc9f1 238 *ppCtx = NULL;
5039dede
AK
239#ifdef _WITH_ENCRYPTION
240 if (pMsg->GetCode() == CMD_REQUEST_SESSION_KEY)
241 {
967893bb 242 UINT32 dwCiphers;
5039dede
AK
243
244 *ppResponse = new CSCPMessage(nNXCPVersion);
245 (*ppResponse)->SetCode(CMD_SESSION_KEY);
246 (*ppResponse)->SetId(pMsg->GetId());
4af351c7 247 (*ppResponse)->disableEncryption();
5039dede
AK
248
249 dwCiphers = pMsg->GetVariableLong(VID_SUPPORTED_ENCRYPTION) & m_dwSupportedCiphers;
250 if (dwCiphers == 0)
251 {
252 (*ppResponse)->SetVariable(VID_RCC, RCC_NO_CIPHERS);
253 dwResult = RCC_NO_CIPHERS;
254 }
255 else
256 {
98abc9f1
VK
257 BYTE *pBufPos, ucKeyBuffer[KEY_BUFFER_SIZE];
258 RSA *pServerKey;
5039dede 259
98abc9f1 260 *ppCtx = NXCPEncryptionContext::create(dwCiphers);
5039dede
AK
261
262 // Encrypt key
eae8c74c 263 int size = pMsg->GetVariableBinary(VID_PUBLIC_KEY, ucKeyBuffer, KEY_BUFFER_SIZE);
5039dede 264 pBufPos = ucKeyBuffer;
eae8c74c 265 pServerKey = d2i_RSAPublicKey(NULL, (OPENSSL_CONST BYTE **)&pBufPos, size);
5039dede
AK
266 if (pServerKey != NULL)
267 {
268 (*ppResponse)->SetVariable(VID_RCC, RCC_SUCCESS);
eae8c74c
VK
269
270 size = RSA_public_encrypt((*ppCtx)->getKeyLength(), (*ppCtx)->getSessionKey(), ucKeyBuffer, pServerKey, RSA_PKCS1_OAEP_PADDING);
271 (*ppResponse)->SetVariable(VID_SESSION_KEY, ucKeyBuffer, (UINT32)size);
98abc9f1 272 (*ppResponse)->SetVariable(VID_KEY_LENGTH, (WORD)(*ppCtx)->getKeyLength());
eae8c74c 273
6468147c 274 int ivLength = EVP_CIPHER_iv_length(s_ciphers[(*ppCtx)->getCipher()]());
eae8c74c
VK
275 if ((ivLength <= 0) || (ivLength > EVP_MAX_IV_LENGTH))
276 ivLength = EVP_MAX_IV_LENGTH;
277 size = RSA_public_encrypt(ivLength, (*ppCtx)->getIV(), ucKeyBuffer, pServerKey, RSA_PKCS1_OAEP_PADDING);
278 (*ppResponse)->SetVariable(VID_SESSION_IV, ucKeyBuffer, (UINT32)size);
279 (*ppResponse)->SetVariable(VID_IV_LENGTH, (WORD)ivLength);
280
281 (*ppResponse)->SetVariable(VID_CIPHER, (WORD)(*ppCtx)->getCipher());
5039dede
AK
282 RSA_free(pServerKey);
283 dwResult = RCC_SUCCESS;
284 }
285 else
286 {
287 (*ppResponse)->SetVariable(VID_RCC, RCC_INVALID_PUBLIC_KEY);
288 dwResult = RCC_INVALID_PUBLIC_KEY;
289 }
290 }
291 }
292 else if (pMsg->GetCode() == CMD_SESSION_KEY)
293 {
294 dwResult = pMsg->GetVariableLong(VID_RCC);
295 if (dwResult == RCC_SUCCESS)
296 {
98abc9f1
VK
297 *ppCtx = NXCPEncryptionContext::create(pMsg, pPrivateKey);
298 if (*ppCtx == NULL)
299 {
300 dwResult = RCC_INVALID_SESSION_KEY;
301 }
5039dede
AK
302 }
303 }
304
305 if ((dwResult != RCC_SUCCESS) && (*ppCtx != NULL))
306 {
98abc9f1 307 delete *ppCtx;
5039dede
AK
308 *ppCtx = NULL;
309 }
310#else
311 if (pMsg->GetCode() == CMD_REQUEST_SESSION_KEY)
312 {
313 *ppResponse = new CSCPMessage(nNXCPVersion);
314 (*ppResponse)->SetCode(CMD_SESSION_KEY);
315 (*ppResponse)->SetId(pMsg->GetId());
26de9040 316 (*ppResponse)->disableEncryption();
5039dede
AK
317 (*ppResponse)->SetVariable(VID_RCC, dwResult);
318 }
319#endif
320
321 return dwResult;
322}
323
b1e9b6b3
VK
324/**
325 * Prepare session key request message
326 */
9e9d631e 327void LIBNETXMS_EXPORTABLE PrepareKeyRequestMsg(CSCPMessage *pMsg, RSA *pServerKey, bool useX509Format)
5039dede
AK
328{
329#ifdef _WITH_ENCRYPTION
330 int iLen;
331 BYTE *pKeyBuffer, *pBufPos;
332
333 pMsg->SetCode(CMD_REQUEST_SESSION_KEY);
334 pMsg->SetVariable(VID_SUPPORTED_ENCRYPTION, m_dwSupportedCiphers);
335
9e9d631e
VK
336 if (useX509Format)
337 {
338 iLen = i2d_RSA_PUBKEY(pServerKey, NULL);
339 pKeyBuffer = (BYTE *)malloc(iLen);
340 pBufPos = pKeyBuffer;
341 i2d_RSA_PUBKEY(pServerKey, &pBufPos);
342 }
343 else
344 {
345 iLen = i2d_RSAPublicKey(pServerKey, NULL);
346 pKeyBuffer = (BYTE *)malloc(iLen);
347 pBufPos = pKeyBuffer;
348 i2d_RSAPublicKey(pServerKey, &pBufPos);
349 }
350 pMsg->SetVariable(VID_PUBLIC_KEY, pKeyBuffer, iLen);
5039dede
AK
351 free(pKeyBuffer);
352#endif
353}
354
7aad6641
VK
355/**
356 * Load RSA key(s) from file
357 */
5039dede
AK
358RSA LIBNETXMS_EXPORTABLE *LoadRSAKeys(const TCHAR *pszKeyFile)
359{
360#ifdef _WITH_ENCRYPTION
361 FILE *fp;
362 BYTE *pKeyBuffer, *pBufPos, hash[SHA1_DIGEST_SIZE];
967893bb 363 UINT32 dwLen;
5039dede
AK
364 RSA *pKey = NULL;
365
366 fp = _tfopen(pszKeyFile, _T("rb"));
367 if (fp != NULL)
368 {
967893bb 369 if (fread(&dwLen, 1, sizeof(UINT32), fp) == sizeof(UINT32) && dwLen < 10 * 1024)
5039dede
AK
370 {
371 pKeyBuffer = (BYTE *)malloc(dwLen);
372 pBufPos = pKeyBuffer;
373 if (fread(pKeyBuffer, 1, dwLen, fp) == dwLen)
374 {
375 BYTE hash2[SHA1_DIGEST_SIZE];
376
7aad6641 377 if (fread(hash, 1, SHA1_DIGEST_SIZE, fp) == SHA1_DIGEST_SIZE)
5039dede 378 {
7aad6641
VK
379 CalculateSHA1Hash(pKeyBuffer, dwLen, hash2);
380 if (!memcmp(hash, hash2, SHA1_DIGEST_SIZE))
5039dede 381 {
7aad6641
VK
382 pKey = d2i_RSAPublicKey(NULL, (OPENSSL_CONST BYTE **)&pBufPos, dwLen);
383 if (pKey != NULL)
5039dede 384 {
7aad6641 385 if (d2i_RSAPrivateKey(&pKey, (OPENSSL_CONST BYTE **)&pBufPos,
967893bb 386 dwLen - CAST_FROM_POINTER((pBufPos - pKeyBuffer), UINT32)) == NULL)
7aad6641
VK
387 {
388 RSA_free(pKey);
389 pKey = NULL;
390 }
5039dede
AK
391 }
392 }
393 }
394 }
395 free(pKeyBuffer);
396 }
397 fclose(fp);
398 }
399
400 return pKey;
401#else
402 return NULL;
403#endif
404}
405
7aad6641
VK
406/**
407 * Create signature for message using certificate (MS CAPI version)
408 * Paraeters:
409 * pMsg and dwMsgLen - message to sign and it's length
410 * pCert - certificate
411 * pBuffer - output buffer
412 * dwBufSize - buffer size
413 * pdwSigLen - actual signature size
414 */
5039dede
AK
415#ifdef _WIN32
416
967893bb
VK
417BOOL LIBNETXMS_EXPORTABLE SignMessageWithCAPI(BYTE *pMsg, UINT32 dwMsgLen, const CERT_CONTEXT *pCert,
418 BYTE *pBuffer, UINT32 dwBufSize, UINT32 *pdwSigLen)
5039dede
AK
419{
420 BOOL bFreeProv, bRet = FALSE;
421 DWORD i, j, dwLen, dwKeySpec;
422 HCRYPTPROV hProv;
423 HCRYPTHASH hHash;
424 BYTE *pTemp;
425
6468147c 426 if (CryptAcquireCertificatePrivateKey(pCert, CRYPT_ACQUIRE_COMPARE_KEY_FLAG, NULL, &hProv, &dwKeySpec, &bFreeProv))
5039dede
AK
427 {
428 if (CryptCreateHash(hProv, CALG_SHA1, NULL, 0, &hHash))
429 {
430 if (CryptHashData(hHash, pMsg, dwMsgLen, 0))
431 {
432 dwLen = dwBufSize;
433 pTemp = (BYTE *)malloc(dwBufSize);;
434 bRet = CryptSignHash(hHash, dwKeySpec, NULL, 0, pTemp, &dwLen);
435 *pdwSigLen = dwLen;
436 // we have to reverse the byte-order in the result from CryptSignHash()
437 for(i = 0, j = dwLen - 1; i < dwLen; i++, j--)
438 pBuffer[i] = pTemp[j];
439 }
440 CryptDestroyHash(hHash);
441 }
442
443 if (bFreeProv)
444 CryptReleaseContext(hProv, 0);
445 }
446 return bRet;
447}
448
449#endif
4e2debcc 450
7aad6641
VK
451/**
452 * Encrypt data block with ICE
453 */
4e2debcc
VK
454void LIBNETXMS_EXPORTABLE ICEEncryptData(const BYTE *in, int inLen, BYTE *out, const BYTE *key)
455{
456 ICE_KEY *ice = ice_key_create(1);
457 ice_key_set(ice, key);
458
459 int stopPos = inLen - (inLen % 8);
4e2debcc
VK
460 for(int pos = 0; pos < stopPos; pos += 8)
461 ice_key_encrypt(ice, &in[pos], &out[pos]);
462
463 if (stopPos < inLen)
464 {
465 BYTE plainText[8], encrypted[8];
466
467 memcpy(plainText, &in[stopPos], inLen - stopPos);
468 ice_key_encrypt(ice, plainText, encrypted);
469 memcpy(&out[stopPos], encrypted, inLen - stopPos);
470 }
471
472 ice_key_destroy(ice);
473}
474
7aad6641
VK
475/**
476 * Decrypt data block with ICE
477 */
4e2debcc
VK
478void LIBNETXMS_EXPORTABLE ICEDecryptData(const BYTE *in, int inLen, BYTE *out, const BYTE *key)
479{
480 ICE_KEY *ice = ice_key_create(1);
481 ice_key_set(ice, key);
482
483 int stopPos = inLen - (inLen % 8);
4e2debcc
VK
484 for(int pos = 0; pos < stopPos; pos += 8)
485 ice_key_decrypt(ice, &in[pos], &out[pos]);
486
487 if (stopPos < inLen)
488 {
489 BYTE plainText[8], encrypted[8];
490
491 memcpy(encrypted, &in[stopPos], inLen - stopPos);
492 ice_key_decrypt(ice, encrypted, plainText);
493 memcpy(&out[stopPos], plainText, inLen - stopPos);
494 }
495
496 ice_key_destroy(ice);
497}
98abc9f1 498
7aad6641
VK
499/**
500 * Encryption context constructor
501 */
98abc9f1
VK
502NXCPEncryptionContext::NXCPEncryptionContext()
503{
e4e091f0
VK
504 m_sessionKey = NULL;
505#ifdef _WITH_ENCRYPTION
e6336a90
VK
506 EVP_CIPHER_CTX_init(&m_encryptor);
507 EVP_CIPHER_CTX_init(&m_decryptor);
db05c2af 508 m_encryptorLock = MutexCreate();
e4e091f0 509#endif
98abc9f1
VK
510}
511
7aad6641
VK
512/**
513 * Encryption context destructor
514 */
98abc9f1
VK
515NXCPEncryptionContext::~NXCPEncryptionContext()
516{
e4e091f0
VK
517 safe_free(m_sessionKey);
518#ifdef _WITH_ENCRYPTION
e6336a90
VK
519 EVP_CIPHER_CTX_cleanup(&m_encryptor);
520 EVP_CIPHER_CTX_cleanup(&m_decryptor);
db05c2af 521 MutexDestroy(m_encryptorLock);
e4e091f0 522#endif
98abc9f1
VK
523}
524
7aad6641
VK
525/**
526 * Create encryption context from CMD_SESSION_KEY NXCP message
527 */
98abc9f1
VK
528NXCPEncryptionContext *NXCPEncryptionContext::create(CSCPMessage *msg, RSA *privateKey)
529{
e4cba530 530#ifdef _WITH_ENCRYPTION
98abc9f1 531 BYTE ucKeyBuffer[KEY_BUFFER_SIZE], ucSessionKey[KEY_BUFFER_SIZE];
967893bb 532 UINT32 dwKeySize;
98abc9f1
VK
533 int nSize, nIVLen;
534 NXCPEncryptionContext *ctx = new NXCPEncryptionContext;
535
e6336a90
VK
536 int cipher = (int)msg->GetVariableShort(VID_CIPHER);
537 if (ctx->initCipher(cipher))
98abc9f1 538 {
e6336a90 539 if (ctx->m_keyLength == (int)msg->GetVariableShort(VID_KEY_LENGTH))
98abc9f1 540 {
e6336a90
VK
541 ctx->m_sessionKey = (BYTE *)malloc(ctx->m_keyLength);
542
543 // Decrypt session key
544 dwKeySize = msg->GetVariableBinary(VID_SESSION_KEY, ucKeyBuffer, KEY_BUFFER_SIZE);
545 nSize = RSA_private_decrypt(dwKeySize, ucKeyBuffer, ucSessionKey, privateKey, RSA_PKCS1_OAEP_PADDING);
546 if (nSize == ctx->m_keyLength)
547 {
548 memcpy(ctx->m_sessionKey, ucSessionKey, nSize);
549
550 // Decrypt session IV
551 nIVLen = msg->GetVariableShort(VID_IV_LENGTH);
552 if (nIVLen == 0) // Versions prior to 0.2.13 don't send IV length, assume 16
553 nIVLen = 16;
554 dwKeySize = msg->GetVariableBinary(VID_SESSION_IV, ucKeyBuffer, KEY_BUFFER_SIZE);
555 nSize = RSA_private_decrypt(dwKeySize, ucKeyBuffer, ucSessionKey, privateKey, RSA_PKCS1_OAEP_PADDING);
556 if ((nSize == nIVLen) &&
6468147c 557 (nIVLen <= EVP_CIPHER_iv_length(s_ciphers[ctx->m_cipher]())))
e6336a90
VK
558 {
559 memcpy(ctx->m_iv, ucSessionKey, min(EVP_MAX_IV_LENGTH, nIVLen));
560 }
561 else
562 {
6468147c 563 CryptoDbgPrintf(6, _T("NXCPEncryptionContext::create: IV decryption failed"));
e6336a90
VK
564 delete_and_null(ctx);
565 }
566 }
567 else
568 {
6468147c 569 CryptoDbgPrintf(6, _T("NXCPEncryptionContext::create: session key decryption failed"));
e6336a90
VK
570 delete_and_null(ctx);
571 }
98abc9f1
VK
572 }
573 else
574 {
6468147c 575 CryptoDbgPrintf(6, _T("NXCPEncryptionContext::create: key length mismatch (remote: %d local: %d)"), (int)msg->GetVariableShort(VID_KEY_LENGTH), ctx->m_keyLength);
98abc9f1
VK
576 delete_and_null(ctx);
577 }
578 }
579 else
580 {
6468147c 581 CryptoDbgPrintf(6, _T("NXCPEncryptionContext::create: initCipher(%d) call failed"), cipher);
98abc9f1
VK
582 delete_and_null(ctx);
583 }
584 return ctx;
e4cba530
VK
585#else
586 return new NXCPEncryptionContext;
587#endif
98abc9f1
VK
588}
589
7aad6641 590/**
e6336a90
VK
591 * Initialize cipher
592 */
593bool NXCPEncryptionContext::initCipher(int cipher)
594{
595#ifdef _WITH_ENCRYPTION
6468147c 596 if (s_ciphers[cipher] == NULL)
e6336a90
VK
597 return false; // Unsupported cipher
598
6468147c 599 if (!EVP_EncryptInit_ex(&m_encryptor, s_ciphers[cipher](), NULL, NULL, NULL))
e6336a90 600 return false;
6468147c 601 if (!EVP_DecryptInit_ex(&m_decryptor, s_ciphers[cipher](), NULL, NULL, NULL))
e6336a90
VK
602 return false;
603
604 switch(cipher)
605 {
606 case CSCP_CIPHER_AES_256:
607 m_keyLength = 32;
608 break;
609 case CSCP_CIPHER_AES_128:
610 m_keyLength = 16;
611 break;
612 case CSCP_CIPHER_BLOWFISH_256:
613 m_keyLength = 32;
614 break;
615 case CSCP_CIPHER_BLOWFISH_128:
616 m_keyLength = 16;
617 break;
618 case CSCP_CIPHER_IDEA:
619 m_keyLength = 16;
620 break;
621 case CSCP_CIPHER_3DES:
622 m_keyLength = 24;
623 break;
624 default:
625 return false;
626 }
627
628 if (!EVP_CIPHER_CTX_set_key_length(&m_encryptor, m_keyLength) || !EVP_CIPHER_CTX_set_key_length(&m_decryptor, m_keyLength))
629 return false;
630
6468147c
VK
631 // This check is needed because at least some OpenSSL versions return no error
632 // from EVP_CIPHER_CTX_set_key_length but still not change key length
633 if ((EVP_CIPHER_CTX_key_length(&m_encryptor) != m_keyLength) || (EVP_CIPHER_CTX_key_length(&m_decryptor) != m_keyLength))
634 return false;
635
e6336a90
VK
636 m_cipher = cipher;
637 return true;
638#else
639 return false;
640#endif
641}
642
643/**
7aad6641
VK
644 * Create encryption context from CMD_REQUEST_SESSION_KEY NXCP message
645 */
967893bb 646NXCPEncryptionContext *NXCPEncryptionContext::create(UINT32 ciphers)
98abc9f1
VK
647{
648 NXCPEncryptionContext *ctx = new NXCPEncryptionContext;
649
e4cba530 650#ifdef _WITH_ENCRYPTION
98abc9f1 651 // Select cipher
e6336a90
VK
652 bool selected = false;
653
98abc9f1
VK
654 if (ciphers & CSCP_SUPPORT_AES_256)
655 {
e6336a90 656 selected = ctx->initCipher(CSCP_CIPHER_AES_256);
98abc9f1 657 }
e6336a90
VK
658
659 if (!selected && (ciphers & CSCP_SUPPORT_BLOWFISH_256))
98abc9f1 660 {
e6336a90 661 selected = ctx->initCipher(CSCP_CIPHER_BLOWFISH_256);
98abc9f1 662 }
e6336a90
VK
663
664 if (!selected && (ciphers & CSCP_SUPPORT_AES_128))
9e9d631e 665 {
e6336a90 666 selected = ctx->initCipher(CSCP_CIPHER_AES_128);
9e9d631e 667 }
e6336a90
VK
668
669 if (!selected && (ciphers & CSCP_SUPPORT_BLOWFISH_128))
d772d0dd 670 {
e6336a90 671 selected = ctx->initCipher(CSCP_CIPHER_BLOWFISH_128);
d772d0dd 672 }
e6336a90
VK
673
674 if (!selected && (ciphers & CSCP_SUPPORT_IDEA))
98abc9f1 675 {
e6336a90 676 selected = ctx->initCipher(CSCP_CIPHER_IDEA);
98abc9f1 677 }
e6336a90
VK
678
679 if (!selected && (ciphers & CSCP_SUPPORT_3DES))
98abc9f1 680 {
e6336a90
VK
681 selected = ctx->initCipher(CSCP_CIPHER_3DES);
682 }
683
684 if (!selected)
685 {
686 delete ctx;
687 return NULL;
98abc9f1
VK
688 }
689
690 // Generate key
691 ctx->m_sessionKey = (BYTE *)malloc(ctx->m_keyLength);
692 RAND_bytes(ctx->m_sessionKey, ctx->m_keyLength);
693 RAND_bytes(ctx->m_iv, EVP_MAX_IV_LENGTH);
e4cba530 694#endif
98abc9f1
VK
695
696 return ctx;
697}
e6336a90
VK
698
699/**
700 * Encrypt message
701 */
6be0a20b 702NXCP_ENCRYPTED_MESSAGE *NXCPEncryptionContext::encryptMessage(CSCP_MESSAGE *msg)
e6336a90 703{
6468147c 704 if (msg->wFlags & s_noEncryptionFlag)
6be0a20b 705 return (NXCP_ENCRYPTED_MESSAGE *)nx_memdup(msg, ntohl(msg->dwSize));
e6336a90
VK
706
707#ifdef _WITH_ENCRYPTION
db05c2af
VK
708 MutexLock(m_encryptorLock);
709
e6336a90 710 if (!EVP_EncryptInit_ex(&m_encryptor, NULL, NULL, m_sessionKey, m_iv))
db05c2af
VK
711 {
712 MutexUnlock(m_encryptorLock);
e6336a90 713 return NULL;
db05c2af 714 }
e6336a90
VK
715
716 UINT32 msgSize = ntohl(msg->dwSize);
6be0a20b
VK
717 NXCP_ENCRYPTED_MESSAGE *emsg =
718 (NXCP_ENCRYPTED_MESSAGE *)malloc(msgSize + NXCP_ENCRYPTION_HEADER_SIZE + EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(&m_encryptor)) + 8);
e6336a90
VK
719 emsg->wCode = htons(CMD_ENCRYPTED_MESSAGE);
720 emsg->nReserved = 0;
721
6be0a20b 722 NXCP_ENCRYPTED_PAYLOAD_HEADER header;
e6336a90
VK
723 header.dwChecksum = htonl(CalculateCRC32((BYTE *)msg, msgSize, 0));
724 header.dwReserved = 0;
725
726 int dataSize;
6be0a20b 727 EVP_EncryptUpdate(&m_encryptor, emsg->data, &dataSize, (BYTE *)&header, NXCP_EH_ENCRYPTED_BYTES);
e6336a90
VK
728 msgSize = dataSize;
729 EVP_EncryptUpdate(&m_encryptor, emsg->data + msgSize, &dataSize, (BYTE *)msg, ntohl(msg->dwSize));
730 msgSize += dataSize;
731 EVP_EncryptFinal_ex(&m_encryptor, emsg->data + msgSize, &dataSize);
6be0a20b 732 msgSize += dataSize + NXCP_EH_UNENCRYPTED_BYTES;
e6336a90 733
db05c2af
VK
734 MutexUnlock(m_encryptorLock);
735
e6336a90
VK
736 if (msgSize % 8 != 0)
737 {
738 emsg->nPadding = (BYTE)(8 - (msgSize % 8));
739 msgSize += emsg->nPadding;
740 }
741 else
742 {
743 emsg->nPadding = 0;
744 }
745 emsg->dwSize = htonl(msgSize);
746
747 return emsg;
748#else /* _WITH_ENCRYPTION */
749 return NULL;
750#endif
751}
752
753/**
754 * Decrypt message
755 */
6be0a20b 756bool NXCPEncryptionContext::decryptMessage(NXCP_ENCRYPTED_MESSAGE *msg, BYTE *decryptionBuffer)
e6336a90
VK
757{
758#ifdef _WITH_ENCRYPTION
759 if (!EVP_DecryptInit_ex(&m_decryptor, NULL, NULL, m_sessionKey, m_iv))
760 return false;
761
762 msg->dwSize = ntohl(msg->dwSize);
763 int dataSize;
764 EVP_DecryptUpdate(&m_decryptor, decryptionBuffer, &dataSize, msg->data,
6be0a20b 765 msg->dwSize - NXCP_EH_UNENCRYPTED_BYTES - msg->nPadding);
e6336a90
VK
766 EVP_DecryptFinal(&m_decryptor, decryptionBuffer + dataSize, &dataSize);
767
6be0a20b 768 CSCP_MESSAGE *clearMsg = (CSCP_MESSAGE *)(decryptionBuffer + NXCP_EH_ENCRYPTED_BYTES);
e6336a90
VK
769 UINT32 msgSize = ntohl(clearMsg->dwSize);
770 if (msgSize > msg->dwSize)
771 return false; // Message decrypted incorrectly, because it can't be larger than encrypted
772 UINT32 crc32 = CalculateCRC32((BYTE *)clearMsg, msgSize, 0);
6be0a20b 773 if (crc32 != ntohl(((NXCP_ENCRYPTED_PAYLOAD_HEADER *)decryptionBuffer)->dwChecksum))
e6336a90
VK
774 return false; // Bad checksum
775
776 memcpy(msg, clearMsg, msgSize);
777 return true;
778#else /* _WITH_ENCRYPTION */
779 return false;
780#endif
781}