Rollback from r3608 to r3606
[public/netxms.git] / src / libnetxms / crypto.cpp
CommitLineData
5039dede
AK
1/*
2** NetXMS - Network Management System
3** NetXMS Foundation Library
4** Copyright (C) 2003, 2004, 2005, 2006, 2007 Victor Kirhenshtein
5**
6** This program is free software; you can redistribute it and/or modify
7** it under the terms of the GNU General Public License as published by
8** the Free Software Foundation; either version 2 of the License, or
9** (at your option) any later version.
10**
11** This program is distributed in the hope that it will be useful,
12** but WITHOUT ANY WARRANTY; without even the implied warranty of
13** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14** GNU General Public License for more details.
15**
16** You should have received a copy of the GNU General Public License
17** along with this program; if not, write to the Free Software
18** Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19**
20** File: crypto.cpp
21**
22**/
23
24#include "libnetxms.h"
25
26
27//
28// Constants
29//
30
31#define KEY_BUFFER_SIZE 4096
32
33
34//
35// Supported ciphers. By default, we support all ciphers compiled
36// into OpenSSL library.
37//
38
39static DWORD m_dwSupportedCiphers =
40#ifdef _WITH_ENCRYPTION
41#ifndef OPENSSL_NO_AES
42 CSCP_SUPPORT_AES_256 |
43#endif
44#ifndef OPENSSL_NO_BF
45 CSCP_SUPPORT_BLOWFISH |
46#endif
47#ifndef OPENSSL_NO_IDEA
48 CSCP_SUPPORT_IDEA |
49#endif
50#ifndef OPENSSL_NO_DES
51 CSCP_SUPPORT_3DES |
52#endif
53#endif /* _WITH_ENCRYPTION */
54 0;
55
56
57//
58// Static data
59//
60
61#ifdef _WITH_ENCRYPTION
62
63typedef OPENSSL_CONST EVP_CIPHER * (*CIPHER_FUNC)(void);
64static CIPHER_FUNC m_pfCipherList[NETXMS_MAX_CIPHERS] =
65{
66#ifndef OPENSSL_NO_AES
67 EVP_aes_256_cbc,
68#else
69 NULL,
70#endif
71#ifndef OPENSSL_NO_BF
72 EVP_bf_cbc,
73#else
74 NULL,
75#endif
76#ifndef OPENSSL_NO_IDEA
77 EVP_idea_cbc,
78#else
79 NULL,
80#endif
81#ifndef OPENSSL_NO_DES
82 EVP_des_ede3_cbc
83#else
84 NULL
85#endif
86};
87static WORD m_wNoEncryptionFlag = 0;
88static MUTEX *m_pCryptoMutexList = NULL;
89
90
91//
92// Locking callback for CRYPTO library
93//
94
95static void CryptoLockingCallback(int nMode, int nLock, const char *pszFile, int nLine)
96{
97 if (nMode & CRYPTO_LOCK)
98 MutexLock(m_pCryptoMutexList[nLock], INFINITE);
99 else
100 MutexUnlock(m_pCryptoMutexList[nLock]);
101}
102
103
104//
105// ID callback for CRYPTO library
106//
107
108#ifndef _WIN32
109
110static unsigned long CryptoIdCallback(void)
111{
112 return (unsigned long)GetCurrentThreadId();
113}
114
115#endif
116
117#endif /* _WITH_ENCRYPTION */
118
119
120//
121// Initialize OpenSSL library
122//
123
124BOOL LIBNETXMS_EXPORTABLE InitCryptoLib(DWORD dwEnabledCiphers)
125{
126#ifdef _WITH_ENCRYPTION
127 BYTE random[8192];
128 int i;
129
130 CRYPTO_malloc_init();
131 ERR_load_CRYPTO_strings();
132 OpenSSL_add_all_algorithms();
133 RAND_seed(random, 8192);
134 m_dwSupportedCiphers &= dwEnabledCiphers;
135 m_wNoEncryptionFlag = htons(MF_DONT_ENCRYPT);
136 m_pCryptoMutexList = (MUTEX *)malloc(sizeof(MUTEX) * CRYPTO_num_locks());
137 for(i = 0; i < CRYPTO_num_locks(); i++)
138 m_pCryptoMutexList[i] = MutexCreate();
139 CRYPTO_set_locking_callback(CryptoLockingCallback);
140#ifndef _WIN32
141 CRYPTO_set_id_callback(CryptoIdCallback);
142#endif /* _WIN32 */
143#endif /* _WITH_ENCRYPTION */
144 return TRUE;
145}
146
147
148//
149// Get supported ciphers
150//
151
152DWORD LIBNETXMS_EXPORTABLE CSCPGetSupportedCiphers(void)
153{
154 return m_dwSupportedCiphers;
155}
156
157
158//
159// Encrypt message
160//
161
162CSCP_ENCRYPTED_MESSAGE LIBNETXMS_EXPORTABLE
163 *CSCPEncryptMessage(CSCP_ENCRYPTION_CONTEXT *pCtx, CSCP_MESSAGE *pMsg)
164{
165#ifdef _WITH_ENCRYPTION
166 CSCP_ENCRYPTED_MESSAGE *pEnMsg;
167 CSCP_ENCRYPTED_PAYLOAD_HEADER header;
168 int nSize;
169 EVP_CIPHER_CTX cipher;
170 DWORD dwMsgSize;
171
172 if (pMsg->wFlags & m_wNoEncryptionFlag)
173 return (CSCP_ENCRYPTED_MESSAGE *)nx_memdup(pMsg, ntohl(pMsg->dwSize));
174
175 if (m_pfCipherList[pCtx->nCipher] == NULL)
176 return NULL; // Unsupported cipher
177
178 EVP_EncryptInit(&cipher, m_pfCipherList[pCtx->nCipher](), pCtx->pSessionKey, pCtx->iv);
179 EVP_CIPHER_CTX_set_key_length(&cipher, pCtx->nKeyLen);
180
181 dwMsgSize = ntohl(pMsg->dwSize);
182 pEnMsg = (CSCP_ENCRYPTED_MESSAGE *)malloc(dwMsgSize + CSCP_ENCRYPTION_HEADER_SIZE +
183 EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(&cipher)) + 8);
184 pEnMsg->wCode = htons(CMD_ENCRYPTED_MESSAGE);
185 pEnMsg->nReserved = 0;
186
187 header.dwChecksum = htonl(CalculateCRC32((BYTE *)pMsg, dwMsgSize, 0));
188 header.dwReserved = 0;
189 EVP_EncryptUpdate(&cipher, pEnMsg->data, &nSize, (BYTE *)&header, CSCP_EH_ENCRYPTED_BYTES);
190 dwMsgSize = nSize;
191 EVP_EncryptUpdate(&cipher, pEnMsg->data + dwMsgSize, &nSize, (BYTE *)pMsg, ntohl(pMsg->dwSize));
192 dwMsgSize += nSize;
193 EVP_EncryptFinal(&cipher, pEnMsg->data + dwMsgSize, &nSize);
194 dwMsgSize += nSize + CSCP_EH_UNENCRYPTED_BYTES;
195 EVP_CIPHER_CTX_cleanup(&cipher);
196
197 if (dwMsgSize % 8 != 0)
198 {
199 pEnMsg->nPadding = (BYTE)(8 - (dwMsgSize % 8));
200 dwMsgSize += pEnMsg->nPadding;
201 }
202 else
203 {
204 pEnMsg->nPadding = 0;
205 }
206 pEnMsg->dwSize = htonl(dwMsgSize);
207
208 return pEnMsg;
209#else /* _WITH_ENCRYPTION */
210 return NULL;
211#endif
212}
213
214
215//
216// Decrypt message
217//
218
219BOOL LIBNETXMS_EXPORTABLE CSCPDecryptMessage(CSCP_ENCRYPTION_CONTEXT *pCtx,
220 CSCP_ENCRYPTED_MESSAGE *pMsg,
221 BYTE *pDecryptionBuffer)
222{
223#ifdef _WITH_ENCRYPTION
224 int nSize;
225 EVP_CIPHER_CTX cipher;
226 DWORD dwChecksum, dwMsgSize;
227 CSCP_MESSAGE *pClearMsg;
228
229 if (m_pfCipherList[pCtx->nCipher] == NULL)
230 return FALSE; // Unsupported cipher
231
232 pMsg->dwSize = ntohl(pMsg->dwSize);
233 EVP_DecryptInit(&cipher, m_pfCipherList[pCtx->nCipher](), pCtx->pSessionKey, pCtx->iv);
234 EVP_CIPHER_CTX_set_key_length(&cipher, pCtx->nKeyLen);
235 EVP_DecryptUpdate(&cipher, pDecryptionBuffer, &nSize, pMsg->data,
236 pMsg->dwSize - CSCP_EH_UNENCRYPTED_BYTES - pMsg->nPadding);
237 EVP_DecryptFinal(&cipher, pDecryptionBuffer + nSize, &nSize);
238 EVP_CIPHER_CTX_cleanup(&cipher);
239
240 pClearMsg = (CSCP_MESSAGE *)(pDecryptionBuffer + CSCP_EH_ENCRYPTED_BYTES);
241 dwMsgSize = ntohl(pClearMsg->dwSize);
242 if (dwMsgSize > pMsg->dwSize)
243 return FALSE; // Message decrypted incorrectly, because it can't be larger than encrypted
244 dwChecksum = CalculateCRC32((BYTE *)pClearMsg, dwMsgSize, 0);
245 if (dwChecksum != ntohl(((CSCP_ENCRYPTED_PAYLOAD_HEADER *)pDecryptionBuffer)->dwChecksum))
246 return FALSE; // Bad checksum
247
248 memcpy(pMsg, pClearMsg, dwMsgSize);
249 return TRUE;
250#else /* _WITH_ENCRYPTION */
251 return FALSE;
252#endif
253}
254
255
256//
257// Destroy encryption context
258//
259
260void LIBNETXMS_EXPORTABLE DestroyEncryptionContext(CSCP_ENCRYPTION_CONTEXT *pCtx)
261{
262 if ((pCtx != NULL) && (pCtx != PROXY_ENCRYPTION_CTX))
263 {
264 safe_free(pCtx->pSessionKey);
265 free(pCtx);
266 }
267}
268
269
270//
271// Setup encryption context
272// Function will determine is it called on initiator or responder side
273// by message code. Initiator should provide it's private key,
274// and responder should provide pointer to response message.
275//
276
277DWORD LIBNETXMS_EXPORTABLE SetupEncryptionContext(CSCPMessage *pMsg,
278 CSCP_ENCRYPTION_CONTEXT **ppCtx,
279 CSCPMessage **ppResponse,
280 RSA *pPrivateKey, int nNXCPVersion)
281{
282 DWORD dwResult = RCC_NOT_IMPLEMENTED;
283
284#ifdef _WITH_ENCRYPTION
285 if (pMsg->GetCode() == CMD_REQUEST_SESSION_KEY)
286 {
287 DWORD dwCiphers;
288
289 *ppResponse = new CSCPMessage(nNXCPVersion);
290 (*ppResponse)->SetCode(CMD_SESSION_KEY);
291 (*ppResponse)->SetId(pMsg->GetId());
292 (*ppResponse)->DisableEncryption();
293
294 dwCiphers = pMsg->GetVariableLong(VID_SUPPORTED_ENCRYPTION) & m_dwSupportedCiphers;
295 if (dwCiphers == 0)
296 {
297 (*ppResponse)->SetVariable(VID_RCC, RCC_NO_CIPHERS);
298 dwResult = RCC_NO_CIPHERS;
299 }
300 else
301 {
302 BYTE *pBufPos, ucKeyBuffer[KEY_BUFFER_SIZE];
303 RSA *pServerKey;
304 DWORD dwKeySize;
305
306 // Create new context
307 *ppCtx = (CSCP_ENCRYPTION_CONTEXT *)malloc(sizeof(CSCP_ENCRYPTION_CONTEXT));
308
309 // Select cipher
310 if (dwCiphers & CSCP_SUPPORT_AES_256)
311 {
312 (*ppCtx)->nCipher = CSCP_CIPHER_AES_256;
313 (*ppCtx)->nKeyLen = 32;
314 }
315 else if (dwCiphers & CSCP_SUPPORT_BLOWFISH)
316 {
317 (*ppCtx)->nCipher = CSCP_CIPHER_BLOWFISH;
318 (*ppCtx)->nKeyLen = 32;
319 }
320 else if (dwCiphers & CSCP_SUPPORT_IDEA)
321 {
322 (*ppCtx)->nCipher = CSCP_CIPHER_IDEA;
323 (*ppCtx)->nKeyLen = 16;
324 }
325 else if (dwCiphers & CSCP_SUPPORT_3DES)
326 {
327 (*ppCtx)->nCipher = CSCP_CIPHER_3DES;
328 (*ppCtx)->nKeyLen = 24;
329 }
330
331 // Generate key
332 (*ppCtx)->pSessionKey = (BYTE *)malloc((*ppCtx)->nKeyLen);
333 RAND_bytes((*ppCtx)->pSessionKey, (*ppCtx)->nKeyLen);
334 RAND_bytes((*ppCtx)->iv, EVP_MAX_IV_LENGTH);
335
336 // Encrypt key
337 dwKeySize = pMsg->GetVariableBinary(VID_PUBLIC_KEY, ucKeyBuffer, KEY_BUFFER_SIZE);
338 pBufPos = ucKeyBuffer;
339 pServerKey = d2i_RSAPublicKey(NULL, (OPENSSL_CONST BYTE **)&pBufPos, dwKeySize);
340 if (pServerKey != NULL)
341 {
342 (*ppResponse)->SetVariable(VID_RCC, RCC_SUCCESS);
343 dwKeySize = RSA_public_encrypt((*ppCtx)->nKeyLen, (*ppCtx)->pSessionKey,
344 ucKeyBuffer, pServerKey, RSA_PKCS1_OAEP_PADDING);
345 (*ppResponse)->SetVariable(VID_SESSION_KEY, ucKeyBuffer, dwKeySize);
346 dwKeySize = RSA_public_encrypt(EVP_MAX_IV_LENGTH, (*ppCtx)->iv,
347 ucKeyBuffer, pServerKey, RSA_PKCS1_OAEP_PADDING);
348 (*ppResponse)->SetVariable(VID_SESSION_IV, ucKeyBuffer, dwKeySize);
349 (*ppResponse)->SetVariable(VID_CIPHER, (WORD)(*ppCtx)->nCipher);
350 (*ppResponse)->SetVariable(VID_KEY_LENGTH, (WORD)(*ppCtx)->nKeyLen);
351 (*ppResponse)->SetVariable(VID_IV_LENGTH, (WORD)EVP_MAX_IV_LENGTH);
352 RSA_free(pServerKey);
353 dwResult = RCC_SUCCESS;
354 }
355 else
356 {
357 (*ppResponse)->SetVariable(VID_RCC, RCC_INVALID_PUBLIC_KEY);
358 dwResult = RCC_INVALID_PUBLIC_KEY;
359 }
360 }
361 }
362 else if (pMsg->GetCode() == CMD_SESSION_KEY)
363 {
364 dwResult = pMsg->GetVariableLong(VID_RCC);
365 if (dwResult == RCC_SUCCESS)
366 {
367 BYTE ucKeyBuffer[KEY_BUFFER_SIZE], ucSessionKey[KEY_BUFFER_SIZE];
368 DWORD dwKeySize;
369 int nSize, nIVLen;
370
371 // Create new context
372 *ppCtx = (CSCP_ENCRYPTION_CONTEXT *)malloc(sizeof(CSCP_ENCRYPTION_CONTEXT));
373 (*ppCtx)->nCipher = pMsg->GetVariableShort(VID_CIPHER);
374 (*ppCtx)->nKeyLen = pMsg->GetVariableShort(VID_KEY_LENGTH);
375 (*ppCtx)->pSessionKey = (BYTE *)malloc((*ppCtx)->nKeyLen);
376
377 // Decrypt session key
378 dwKeySize = pMsg->GetVariableBinary(VID_SESSION_KEY, ucKeyBuffer, KEY_BUFFER_SIZE);
379 nSize = RSA_private_decrypt(dwKeySize, ucKeyBuffer, ucSessionKey,
380 pPrivateKey, RSA_PKCS1_OAEP_PADDING);
381 if (nSize == (*ppCtx)->nKeyLen)
382 {
383 memcpy((*ppCtx)->pSessionKey, ucSessionKey, nSize);
384
385 // Decrypt session IV
386 nIVLen = pMsg->GetVariableShort(VID_IV_LENGTH);
387 if (nIVLen == 0) // Versions prior to 0.2.13 don't send IV length, assume 16
388 nIVLen = 16;
389 dwKeySize = pMsg->GetVariableBinary(VID_SESSION_IV, ucKeyBuffer, KEY_BUFFER_SIZE);
390 nSize = RSA_private_decrypt(dwKeySize, ucKeyBuffer, ucSessionKey,
391 pPrivateKey, RSA_PKCS1_OAEP_PADDING);
392 if ((nSize == nIVLen) &&
393 (nIVLen <= EVP_CIPHER_iv_length(m_pfCipherList[(*ppCtx)->nCipher]())))
394 {
395 memcpy((*ppCtx)->iv, ucSessionKey, min(EVP_MAX_IV_LENGTH, nIVLen));
396 }
397 else
398 {
399 dwResult = RCC_INVALID_SESSION_KEY;
400 }
401 }
402 else
403 {
404 dwResult = RCC_INVALID_SESSION_KEY;
405 }
406 }
407 }
408
409 if ((dwResult != RCC_SUCCESS) && (*ppCtx != NULL))
410 {
411 DestroyEncryptionContext(*ppCtx);
412 *ppCtx = NULL;
413 }
414#else
415 if (pMsg->GetCode() == CMD_REQUEST_SESSION_KEY)
416 {
417 *ppResponse = new CSCPMessage(nNXCPVersion);
418 (*ppResponse)->SetCode(CMD_SESSION_KEY);
419 (*ppResponse)->SetId(pMsg->GetId());
420 (*ppResponse)->DisableEncryption();
421 (*ppResponse)->SetVariable(VID_RCC, dwResult);
422 }
423#endif
424
425 return dwResult;
426}
427
428
429//
430// Prepare session key request message
431//
432
433void LIBNETXMS_EXPORTABLE PrepareKeyRequestMsg(CSCPMessage *pMsg, RSA *pServerKey)
434{
435#ifdef _WITH_ENCRYPTION
436 int iLen;
437 BYTE *pKeyBuffer, *pBufPos;
438
439 pMsg->SetCode(CMD_REQUEST_SESSION_KEY);
440 pMsg->SetVariable(VID_SUPPORTED_ENCRYPTION, m_dwSupportedCiphers);
441
442 iLen = i2d_RSAPublicKey(pServerKey, NULL);
443 pKeyBuffer = (BYTE *)malloc(iLen);
444 pBufPos = pKeyBuffer;
445 i2d_RSAPublicKey(pServerKey, &pBufPos);
446 pMsg->SetVariable(VID_PUBLIC_KEY, pKeyBuffer, iLen);
447 free(pKeyBuffer);
448#endif
449}
450
451
452//
453// Load RSA key(s) from file
454//
455
456RSA LIBNETXMS_EXPORTABLE *LoadRSAKeys(const TCHAR *pszKeyFile)
457{
458#ifdef _WITH_ENCRYPTION
459 FILE *fp;
460 BYTE *pKeyBuffer, *pBufPos, hash[SHA1_DIGEST_SIZE];
461 DWORD dwLen;
462 RSA *pKey = NULL;
463
464 fp = _tfopen(pszKeyFile, _T("rb"));
465 if (fp != NULL)
466 {
467 if (fread(&dwLen, 1, sizeof(DWORD), fp) == sizeof(DWORD))
468 {
469 pKeyBuffer = (BYTE *)malloc(dwLen);
470 pBufPos = pKeyBuffer;
471 if (fread(pKeyBuffer, 1, dwLen, fp) == dwLen)
472 {
473 BYTE hash2[SHA1_DIGEST_SIZE];
474
475 fread(hash, 1, SHA1_DIGEST_SIZE, fp);
476 CalculateSHA1Hash(pKeyBuffer, dwLen, hash2);
477 if (!memcmp(hash, hash2, SHA1_DIGEST_SIZE))
478 {
479 pKey = d2i_RSAPublicKey(NULL, (OPENSSL_CONST BYTE **)&pBufPos, dwLen);
480 if (pKey != NULL)
481 {
482 if (d2i_RSAPrivateKey(&pKey, (OPENSSL_CONST BYTE **)&pBufPos,
483 dwLen - CAST_FROM_POINTER((pBufPos - pKeyBuffer), DWORD)) == NULL)
484 {
485 RSA_free(pKey);
486 pKey = NULL;
487 }
488 }
489 }
490 }
491 free(pKeyBuffer);
492 }
493 fclose(fp);
494 }
495
496 return pKey;
497#else
498 return NULL;
499#endif
500}
501
502
503//
504// Create signature for message using certificate (MS CAPI version)
505// Paraeters:
506// pMsg and dwMsgLen - message to sign and it's length
507// pCert - certificate
508// pBuffer - output buffer
509// dwBufSize - buffer size
510// pdwSigLen - actual signature size
511//
512
513#ifdef _WIN32
514
515BOOL LIBNETXMS_EXPORTABLE SignMessageWithCAPI(BYTE *pMsg, DWORD dwMsgLen, const CERT_CONTEXT *pCert,
516 BYTE *pBuffer, DWORD dwBufSize, DWORD *pdwSigLen)
517{
518 BOOL bFreeProv, bRet = FALSE;
519 DWORD i, j, dwLen, dwKeySpec;
520 HCRYPTPROV hProv;
521 HCRYPTHASH hHash;
522 BYTE *pTemp;
523
524 if (CryptAcquireCertificatePrivateKey(pCert, CRYPT_ACQUIRE_COMPARE_KEY_FLAG,
525 NULL, &hProv, &dwKeySpec, &bFreeProv))
526 {
527 if (CryptCreateHash(hProv, CALG_SHA1, NULL, 0, &hHash))
528 {
529 if (CryptHashData(hHash, pMsg, dwMsgLen, 0))
530 {
531 dwLen = dwBufSize;
532 pTemp = (BYTE *)malloc(dwBufSize);;
533 bRet = CryptSignHash(hHash, dwKeySpec, NULL, 0, pTemp, &dwLen);
534 *pdwSigLen = dwLen;
535 // we have to reverse the byte-order in the result from CryptSignHash()
536 for(i = 0, j = dwLen - 1; i < dwLen; i++, j--)
537 pBuffer[i] = pTemp[j];
538 }
539 CryptDestroyHash(hHash);
540 }
541
542 if (bFreeProv)
543 CryptReleaseContext(hProv, 0);
544 }
545 return bRet;
546}
547
548#endif