From 6ea1b5818e5f014bf432bfc23ec6e009e5d452ea Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Sun, 25 Feb 2024 17:50:05 +0100 Subject: [PATCH 01/13] Add ALPN extension decoding to ssl.c --- src/include/stream.h | 2 + src/lib/decode/ssl.c | 115 +++++++++++++------------------------------ src/lib/decode/ssl.h | 3 ++ 3 files changed, 39 insertions(+), 81 deletions(-) diff --git a/src/include/stream.h b/src/include/stream.h index da690e01..d7338269 100644 --- a/src/include/stream.h +++ b/src/include/stream.h @@ -60,6 +60,8 @@ typedef struct ByteStream_s { #define ByteStream_IS_ERROR(b) ((b).last == NULL) +#define ByteStream_NO_ERROR(b) ((b).last != NULL) + #define ByteStream_AVAILABLE(b) ((b).size) #define ByteStream_SIZE(b) ((b).last != NULL ? (b).last - (b).stream : 0) diff --git a/src/lib/decode/ssl.c b/src/lib/decode/ssl.c index ee0ec9af..e82454b5 100644 --- a/src/lib/decode/ssl.c +++ b/src/lib/decode/ssl.c @@ -153,6 +153,33 @@ static int sslParseExtensions(ssl_t *ssl, BytesStream_t sslStream, uint16_t leng dbg_printf("Found curvePF: 0x%x\n", curvePF); } } break; + case 16: { // application_layer_protocol_negotiation (16) + uint16_t alpnLength; + ByteStream_GET_u16(sslStream, alpnLength); + + if (alpnLength > ByteStream_AVAILABLE(sslStream) || alpnLength >= ALPNmaxLen) { + LogError("%s():%d ALPN extension length error", __FUNCTION__, __LINE__); + return 0; + } + + uint32_t alpnCnt = 0; + do { + uint8_t alpnStrLen; + ByteStream_GET_u8(sslStream, alpnStrLen); + if (alpnCnt == 0) { // add first ALPN + ByteStream_GET_X(sslStream, ssl->alpnName, alpnStrLen); + ssl->alpnName[alpnStrLen] = '\0'; + printf("Found first ALPN: %s\n", ssl->alpnName); + } else { + ByteStream_SKIP(sslStream, alpnStrLen); + } + alpnCnt += (alpnStrLen + 1); + } while ((alpnCnt < alpnLength) && ByteStream_NO_ERROR(sslStream)); + if (ByteStream_IS_ERROR(sslStream)) { + LogError("%s():%d ALPN decoding error", __FUNCTION__, __LINE__); + return 0; + } + } break; default: if (exLength) ByteStream_SKIP(sslStream, exLength); } @@ -318,6 +345,10 @@ void sslPrint(ssl_t *ssl) { printf("SNI name : %s\n", ssl->sniName); } + if (ssl->alpnName[0]) { + printf("ALPN name : %s\n", ssl->alpnName); + } + if (ssl->type == CLIENTssl) { printf("curves :"); for (int i = 0; i < LenArray(ssl->ellipticCurves); i++) { @@ -471,87 +502,9 @@ void sslTest(void) { 0x0e, 0x00, 0x0c, 0x02, 0x68, 0x32, 0x08, 0x68, 0x74, 0x74, 0x70, 0x2f, 0x31, 0x2e, 0x31, 0x75, 0x50, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, 0x00, 0x0a, 0x00, 0x0a, 0x00, 0x08, 0x1a, 0x1a, 0x00, 0x1d, 0x00, 0x17, 0x00, 0x18, 0x1a, 0x1a, 0x00, 0x01, 0x00}; - const uint8_t tls12[] = { - 0x16, 0x03, // ..X..... - 0x01, 0x02, 0x00, 0x01, 0x00, 0x01, 0xfc, 0x03, // ........ - 0x03, 0xec, 0xb2, 0x69, 0x1a, 0xdd, 0xb2, 0xbf, // ...i.... - 0x6c, 0x59, 0x9c, 0x7a, 0xaa, 0xe2, 0x3d, 0xe5, // lY.z..=. - 0xf4, 0x25, 0x61, 0xcc, 0x04, 0xeb, 0x41, 0x02, // .%a...A. - 0x9a, 0xcc, 0x6f, 0xc0, 0x50, 0xa1, 0x6a, 0xc1, // ..o.P.j. - 0xd2, 0x20, 0x46, 0xf8, 0x61, 0x7b, 0x58, 0x0a, // . F.a{X. - 0xc9, 0x35, 0x8e, 0x2a, 0xa4, 0x4e, 0x30, 0x6d, // .5.*.N0m - 0x52, 0x46, 0x6b, 0xcc, 0x98, 0x9c, 0x87, 0xc8, // RFk..... - 0xca, 0x64, 0x30, 0x9f, 0x5f, 0xaf, 0x50, 0xba, // .d0._.P. - 0x7b, 0x4d, 0x00, 0x22, 0x13, 0x01, 0x13, 0x03, // {M.".... - 0x13, 0x02, 0xc0, 0x2b, 0xc0, 0x2f, 0xcc, 0xa9, // ...+./.. - 0xcc, 0xa8, 0xc0, 0x2c, 0xc0, 0x30, 0xc0, 0x0a, // ...,.0.. - 0xc0, 0x09, 0xc0, 0x13, 0xc0, 0x14, 0x00, 0x9c, // ........ - 0x00, 0x9d, 0x00, 0x2f, 0x00, 0x35, 0x01, 0x00, // .../.5.. - 0x01, 0x91, 0x00, 0x00, 0x00, 0x21, 0x00, 0x1f, // .....!.. - 0x00, 0x00, 0x1c, 0x63, 0x6f, 0x6e, 0x74, 0x69, // ...conti - 0x6c, 0x65, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, // le.servi - 0x63, 0x65, 0x73, 0x2e, 0x6d, 0x6f, 0x7a, 0x69, // ces.mozi - 0x6c, 0x6c, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x00, // lla.com. - 0x17, 0x00, 0x00, 0xff, 0x01, 0x00, 0x01, 0x00, // ........ - 0x00, 0x0a, 0x00, 0x0e, 0x00, 0x0c, 0x00, 0x1d, // ........ - 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, 0x01, 0x00, // ........ - 0x01, 0x01, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, // ........ - 0x00, 0x23, 0x00, 0x00, 0x00, 0x10, 0x00, 0x0e, // .#...... - 0x00, 0x0c, 0x02, 0x68, 0x32, 0x08, 0x68, 0x74, // ...h2.ht - 0x74, 0x70, 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x05, // tp/1.1.. - 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x22, 0x00, 0x0a, 0x00, 0x08, 0x04, 0x03, 0x05, // "....... - 0x03, 0x06, 0x03, 0x02, 0x03, 0x00, 0x33, 0x00, // ......3. - 0x6b, 0x00, 0x69, 0x00, 0x1d, 0x00, 0x20, 0x89, // k.i... . - 0x09, 0x85, 0x8f, 0xbe, 0xb6, 0xed, 0x2f, 0x12, // ....../. - 0x48, 0xba, 0x5b, 0x9e, 0x29, 0x78, 0xbe, 0xad, // H.[.)x.. - 0x0e, 0x84, 0x01, 0x10, 0x19, 0x2c, 0x61, 0xda, // .....,a. - 0xed, 0x00, 0x96, 0x79, 0x8b, 0x18, 0x44, 0x00, // ...y..D. - 0x17, 0x00, 0x41, 0x04, 0x4d, 0x18, 0x3d, 0x91, // ..A.M.=. - 0xf5, 0xee, 0xd3, 0x57, 0x91, 0xfa, 0x98, 0x24, // ...W...$ - 0x64, 0xe3, 0xb0, 0x21, 0x4a, 0xaa, 0x5f, 0x5d, // d..!J._] - 0x1b, 0x78, 0x61, 0x6d, 0x9b, 0x9f, 0xbe, 0xbc, // .xam.... - 0x22, 0xd1, 0x1f, 0x53, 0x5b, 0x2f, 0x94, 0xc6, // "..S[/.. - 0x86, 0x14, 0x31, 0x36, 0xaa, 0x79, 0x5e, 0x6e, // ..16.y^n - 0x5a, 0x87, 0x5d, 0x6c, 0x08, 0x06, 0x4a, 0xd5, // Z.]l..J. - 0xb7, 0x6d, 0x44, 0xca, 0xad, 0x76, 0x6e, 0x24, // .mD..vn$ - 0x83, 0x01, 0x27, 0x48, 0x00, 0x2b, 0x00, 0x05, // ..'H.+.. - 0x04, 0x03, 0x04, 0x03, 0x03, 0x00, 0x0d, 0x00, // ........ - 0x18, 0x00, 0x16, 0x04, 0x03, 0x05, 0x03, 0x06, // ........ - 0x03, 0x08, 0x04, 0x08, 0x05, 0x08, 0x06, 0x04, // ........ - 0x01, 0x05, 0x01, 0x06, 0x01, 0x02, 0x03, 0x02, // ........ - 0x01, 0x00, 0x2d, 0x00, 0x02, 0x01, 0x01, 0x00, // ..-..... - 0x1c, 0x00, 0x02, 0x40, 0x01, 0x00, 0x15, 0x00, // ...@.... - 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // z....... - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00 // ... - }; - /* - [JA4: t13d1715h2_5b57614c22b0_3d5424432f57] - JA4_r: - t13d1715h2_002f,0035,009c,009d,1301,1302,1303,c009,c00a,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0015,0017,001c,0022,0023,002b,002d,0033,ff01_0403,0503,0603,0804,0805,0806,0401,0501,0601,0203,0201] - - JA3 Fullstring: - 771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28-21,29-23-24-25-256-257,0] - [JA3: 579ccef312d18482fc42e2b822ca2430] - */ - // size_t len = sizeof(clientHello); - size_t len = sizeof(tls12); - - ssl_t *ssl = sslProcess(tls12, len); + size_t len = sizeof(clientHello); + + ssl_t *ssl = sslProcess(clientHello, len); if (ssl) sslPrint(ssl); else diff --git a/src/lib/decode/ssl.h b/src/lib/decode/ssl.h index 83d12e9e..4a04933d 100644 --- a/src/lib/decode/ssl.h +++ b/src/lib/decode/ssl.h @@ -76,6 +76,9 @@ typedef struct ssl_s { uint16Array_t extensions; uint16Array_t ellipticCurves; uint16Array_t ellipticCurvesPF; +#define ALPNmaxLen 16 + // ALPN are currently defined up to 8 bytes + char alpnName[ALPNmaxLen]; char sniName[256]; } ssl_t; From a4f6a53a1f1bd5d996ec53112000cdc3554e2dca Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Mon, 26 Feb 2024 20:12:24 +0100 Subject: [PATCH 02/13] Refrag ssl extension code --- src/include/stream.h | 2 +- src/lib/decode/ssl.c | 192 ++++++++++++++++++++++++------------------- 2 files changed, 108 insertions(+), 86 deletions(-) diff --git a/src/include/stream.h b/src/include/stream.h index d7338269..8b0dfcb6 100644 --- a/src/include/stream.h +++ b/src/include/stream.h @@ -135,7 +135,7 @@ typedef struct ByteStream_s { #define ByteStream_GET_X(b, x, len) \ do { \ if ((b).size >= len) { \ - memcpy(x, b.ptr, len); \ + memcpy(x, (b).ptr, len); \ (b).ptr += len; \ (b).size -= len; \ } else { \ diff --git a/src/lib/decode/ssl.c b/src/lib/decode/ssl.c index e82454b5..8f8fc751 100644 --- a/src/lib/decode/ssl.c +++ b/src/lib/decode/ssl.c @@ -67,6 +67,98 @@ static int checkGREASE(uint16_t val) { } // End of checkGrease +static int ProcessExtSNI(ssl_t *ssl, BytesStream_t *sslStream) { + uint16_t sniListLength; + ByteStream_GET_u16(*sslStream, sniListLength); + + // skip server name type 1 + ByteStream_SKIP(*sslStream, 1); + + uint16_t sniLen; + ByteStream_GET_u16(*sslStream, sniLen); + if (sniLen > ByteStream_AVAILABLE(*sslStream) || sniLen > 255) { + LogError("%s():%d sni extension length error", __FUNCTION__, __LINE__); + return 0; + } + + ByteStream_GET_X(*sslStream, ssl->sniName, sniLen); + ssl->sniName[sniLen] = '\0'; + dbg_printf("Found sni name: %s\n", ssl->sniName); + + if ((sniLen + 3) < sniListLength) { + // should not happen as only one host_type suported + size_t skipBytes = sniListLength - sniLen - 3; + ByteStream_SKIP(*sslStream, skipBytes); + } + + return 1; +} // End of ProcessExtSNI + +static int ProcessExtElCurves(ssl_t *ssl, BytesStream_t *sslStream) { + uint16_t ecsLen; + ByteStream_GET_u16(*sslStream, ecsLen); + + if (ecsLen > ByteStream_AVAILABLE(*sslStream)) { + LogError("%s():%d ecs extension length error", __FUNCTION__, __LINE__); + return 0; + } + + for (int i = 0; i < (ecsLen >> 1); i++) { + uint16_t curve; + ByteStream_GET_u16(*sslStream, curve); + AppendArray(ssl->ellipticCurves, curve); + dbg_printf("Found curve: 0x%x\n", curve); + } + return 1; +} // End of ProcessExtElCurves + +static int ProcessExtElCurvesPoints(ssl_t *ssl, BytesStream_t *sslStream) { + uint8_t ecspLen; + ByteStream_GET_u8(*sslStream, ecspLen); + + if (ecspLen > ByteStream_AVAILABLE(*sslStream)) { + LogError("%s():%d ecsp extension length error", __FUNCTION__, __LINE__); + return 0; + } + + for (int i = 0; i < ecspLen; i++) { + uint8_t curvePF; + ByteStream_GET_u8(*sslStream, curvePF); + AppendArray(ssl->ellipticCurvesPF, curvePF); + dbg_printf("Found curvePF: 0x%x\n", curvePF); + } + return 1; +} // End of ProcessExtElCurvesPoints + +static int ProcessExtALPN(ssl_t *ssl, BytesStream_t *sslStream) { + uint16_t alpnLength; + ByteStream_GET_u16(*sslStream, alpnLength); + + if (alpnLength > ByteStream_AVAILABLE(*sslStream) || alpnLength >= ALPNmaxLen) { + LogError("%s():%d ALPN extension length error", __FUNCTION__, __LINE__); + return 0; + } + + uint32_t alpnCnt = 0; + do { + uint8_t alpnStrLen; + ByteStream_GET_u8(*sslStream, alpnStrLen); + if (alpnCnt == 0) { // add first ALPN + ByteStream_GET_X(*sslStream, ssl->alpnName, alpnStrLen); + ssl->alpnName[alpnStrLen] = '\0'; + printf("Found first ALPN: %s\n", ssl->alpnName); + } else { + ByteStream_SKIP(*sslStream, alpnStrLen); + } + alpnCnt += (alpnStrLen + 1); + } while ((alpnCnt < alpnLength) && ByteStream_NO_ERROR(*sslStream)); + if (ByteStream_IS_ERROR(*sslStream)) { + LogError("%s():%d ALPN decoding error", __FUNCTION__, __LINE__); + return 0; + } + return 1; +} // End of ProcessExtALPN + static int sslParseExtensions(ssl_t *ssl, BytesStream_t sslStream, uint16_t length) { dbg_printf("Parse extensions: %x\n", length); if (length == 0) { @@ -94,95 +186,25 @@ static int sslParseExtensions(ssl_t *ssl, BytesStream_t sslStream, uint16_t leng } AppendArray(ssl->extensions, exType); - + int ret = 0; switch (exType) { - case 0: { // sni name - uint16_t sniListLength; - ByteStream_GET_u16(sslStream, sniListLength); - - // skip server name type 1 - ByteStream_SKIP(sslStream, 1); - - uint16_t sniLen; - ByteStream_GET_u16(sslStream, sniLen); - - if (sniLen > ByteStream_AVAILABLE(sslStream) || sniLen > 255) { - LogError("%s():%d sni extension length error", __FUNCTION__, __LINE__); - return 0; - } - - ByteStream_GET_X(sslStream, ssl->sniName, sniLen); - ssl->sniName[sniLen] = '\0'; - dbg_printf("Found sni name: %s\n", ssl->sniName); - - if ((sniLen + 3) < sniListLength) { - // should not happen as only one host_type suported - size_t skipBytes = sniListLength - sniLen - 3; - ByteStream_SKIP(sslStream, skipBytes); - } - } break; - case 10: { // Elliptic curves - uint16_t ecsLen; - ByteStream_GET_u16(sslStream, ecsLen); - - if (ecsLen > ByteStream_AVAILABLE(sslStream)) { - LogError("%s():%d ecs extension length error", __FUNCTION__, __LINE__); - return 0; - } - - for (int i = 0; i < (ecsLen >> 1); i++) { - uint16_t curve; - ByteStream_GET_u16(sslStream, curve); - AppendArray(ssl->ellipticCurves, curve); - dbg_printf("Found curve: 0x%x\n", curve); - } - } break; - case 11: { // Elliptic curve point formats uncompressed - uint8_t ecspLen; - ByteStream_GET_u8(sslStream, ecspLen); - - if (ecspLen > ByteStream_AVAILABLE(sslStream)) { - LogError("%s():%d ecsp extension length error", __FUNCTION__, __LINE__); - return 0; - } - - for (int i = 0; i < ecspLen; i++) { - uint8_t curvePF; - ByteStream_GET_u8(sslStream, curvePF); - AppendArray(ssl->ellipticCurvesPF, curvePF); - dbg_printf("Found curvePF: 0x%x\n", curvePF); - } - } break; - case 16: { // application_layer_protocol_negotiation (16) - uint16_t alpnLength; - ByteStream_GET_u16(sslStream, alpnLength); - - if (alpnLength > ByteStream_AVAILABLE(sslStream) || alpnLength >= ALPNmaxLen) { - LogError("%s():%d ALPN extension length error", __FUNCTION__, __LINE__); - return 0; - } - - uint32_t alpnCnt = 0; - do { - uint8_t alpnStrLen; - ByteStream_GET_u8(sslStream, alpnStrLen); - if (alpnCnt == 0) { // add first ALPN - ByteStream_GET_X(sslStream, ssl->alpnName, alpnStrLen); - ssl->alpnName[alpnStrLen] = '\0'; - printf("Found first ALPN: %s\n", ssl->alpnName); - } else { - ByteStream_SKIP(sslStream, alpnStrLen); - } - alpnCnt += (alpnStrLen + 1); - } while ((alpnCnt < alpnLength) && ByteStream_NO_ERROR(sslStream)); - if (ByteStream_IS_ERROR(sslStream)) { - LogError("%s():%d ALPN decoding error", __FUNCTION__, __LINE__); - return 0; - } - } break; + case 0: // sni name + ret = ProcessExtSNI(ssl, &sslStream); + break; + case 10: // Elliptic curves + ret = ProcessExtElCurves(ssl, &sslStream); + break; + case 11: // Elliptic curve point formats uncompressed + ret = ProcessExtElCurvesPoints(ssl, &sslStream); + break; + case 16: // application_layer_protocol_negotiation (16) + ret = ProcessExtALPN(ssl, &sslStream); + break; default: + ret = 1; if (exLength) ByteStream_SKIP(sslStream, exLength); } + if (ret == 0) return 0; extensionLength -= (4 + exLength); } dbg_printf("End extension. size: %d\n", extensionLength); From 16148ed96a3b414f9b1f706c3dbf38b7b0a98fef Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Thu, 29 Feb 2024 13:44:54 +0100 Subject: [PATCH 03/13] Add sha256 --- src/lib/Makefile.am | 2 +- src/lib/decode/sha256.c | 222 ++++++++++++++++++++++++++++++++++++++++ src/lib/decode/sha256.h | 42 ++++++++ src/lib/decode/ssl.c | 5 +- 4 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 src/lib/decode/sha256.c create mode 100644 src/lib/decode/sha256.h diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index 2321eac4..d41378d9 100755 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -27,7 +27,7 @@ output = userio.c userio.h output_short.c output_short.h regex = sgregex/sgregex.c sgregex/sgregex.h daemon = daemon.c daemon.h version = version.c version.h -decode = decode/content_dns.c decode/content_dns.h decode/ssl.c decode/ssl.h decode/ja3.c decode/ja3.h decode/md5.c decode/md5.h +decode = decode/content_dns.c decode/content_dns.h decode/ssl.c decode/ssl.h decode/ja3.c decode/ja3.h decode/md5.c decode/md5.h decode/sha256.c decode/sha256.h vcs_track.h: Makefile ./gen_version.sh diff --git a/src/lib/decode/sha256.c b/src/lib/decode/sha256.c new file mode 100644 index 00000000..59aaae5d --- /dev/null +++ b/src/lib/decode/sha256.c @@ -0,0 +1,222 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2013, Con Kolivas + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "sha256.h" + +#include + +#define SHA256_DIGEST_SIZE (256 / 8) +#define SHA256_BLOCK_SIZE (512 / 8) + +#define SHFR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) +#define CH(x, y, z) ((x & y) ^ (~x & z)) +#define MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) + +#define SHA256_F1(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define SHA256_F2(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define SHA256_F3(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHFR(x, 3)) +#define SHA256_F4(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHFR(x, 10)) + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA256_BLOCK_SIZE]; + uint32_t h[8]; +} sha256_ctx; + +#define UNPACK32(x, str) \ + { \ + *((str) + 3) = (uint8_t)((x)); \ + *((str) + 2) = (uint8_t)((x) >> 8); \ + *((str) + 1) = (uint8_t)((x) >> 16); \ + *((str) + 0) = (uint8_t)((x) >> 24); \ + } + +#define PACK32(str, x) \ + { *(x) = ((uint32_t) * ((str) + 3)) | ((uint32_t) * ((str) + 2) << 8) | ((uint32_t) * ((str) + 1) << 16) | ((uint32_t) * ((str) + 0) << 24); } + +#define SHA256_SCR(i) \ + { w[i] = SHA256_F4(w[i - 2]) + w[i - 7] + SHA256_F3(w[i - 15]) + w[i - 16]; } + +static uint32_t sha256_h0[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + +static uint32_t sha256_k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, + 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, + 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, + 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, + 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +/* SHA-256 functions */ + +static void sha256_init(sha256_ctx *ctx); + +static void sha256_update(sha256_ctx *ctx, const unsigned char *message, unsigned int len); + +static void sha256_final(sha256_ctx *ctx, unsigned char *digest); + +static void sha256_transf(sha256_ctx *ctx, const unsigned char *message, unsigned int block_nb) { + uint32_t w[64]; + uint32_t wv[8]; + uint32_t t1, t2; + const unsigned char *sub_block; + int i; + + int j; + + for (i = 0; i < (int)block_nb; i++) { + sub_block = message + (i << 6); + + for (j = 0; j < 16; j++) { + PACK32(&sub_block[j << 2], &w[j]); + } + + for (j = 16; j < 64; j++) { + SHA256_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 64; j++) { + t1 = wv[7] + SHA256_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + sha256_k[j] + w[j]; + t2 = SHA256_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } + } +} + +static void sha256_init(sha256_ctx *ctx) { + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha256_h0[i]; + } + + ctx->len = 0; + ctx->tot_len = 0; +} + +static void sha256_update(sha256_ctx *ctx, const unsigned char *message, unsigned int len) { + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA256_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA256_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA256_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA256_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +static void sha256_final(sha256_ctx *ctx, unsigned char *digest) { + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + + int i; + + block_nb = (1 + ((SHA256_BLOCK_SIZE - 9) < (ctx->len % SHA256_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + + for (i = 0; i < 8; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +} + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest) { + sha256_ctx ctx = {0}; + + sha256_init(&ctx); + sha256_update(&ctx, message, len); + sha256_final(&ctx, digest); +} + +#ifdef MAIN +#include + +int main(int argc, char **argv) { + char *str = "b08abd37"; + + uint8_t shasum[32] = {0}; + + sha256((const unsigned char *)str, strlen(str), (unsigned char *)shasum); + for (int i = 0; i < 32; i++) { + printf("%x ", shasum[i]); + } + printf("\n"); + return 0; +} + +#endif \ No newline at end of file diff --git a/src/lib/decode/sha256.h b/src/lib/decode/sha256.h new file mode 100644 index 00000000..704f23fc --- /dev/null +++ b/src/lib/decode/sha256.h @@ -0,0 +1,42 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2013, Con Kolivas + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef SHA2_H +#define SHA2_H + +#include + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest); + +#endif /* !SHA2_H */ \ No newline at end of file diff --git a/src/lib/decode/ssl.c b/src/lib/decode/ssl.c index 8f8fc751..f4b0fce1 100644 --- a/src/lib/decode/ssl.c +++ b/src/lib/decode/ssl.c @@ -146,7 +146,7 @@ static int ProcessExtALPN(ssl_t *ssl, BytesStream_t *sslStream) { if (alpnCnt == 0) { // add first ALPN ByteStream_GET_X(*sslStream, ssl->alpnName, alpnStrLen); ssl->alpnName[alpnStrLen] = '\0'; - printf("Found first ALPN: %s\n", ssl->alpnName); + dbg_printf("Found first ALPN: %s\n", ssl->alpnName); } else { ByteStream_SKIP(*sslStream, alpnStrLen); } @@ -186,7 +186,7 @@ static int sslParseExtensions(ssl_t *ssl, BytesStream_t sslStream, uint16_t leng } AppendArray(ssl->extensions, exType); - int ret = 0; + int ret = 1; switch (exType) { case 0: // sni name ret = ProcessExtSNI(ssl, &sslStream); @@ -201,7 +201,6 @@ static int sslParseExtensions(ssl_t *ssl, BytesStream_t sslStream, uint16_t leng ret = ProcessExtALPN(ssl, &sslStream); break; default: - ret = 1; if (exLength) ByteStream_SKIP(sslStream, exLength); } if (ret == 0) return 0; From ae92ef6f4f1d7ac35742ab640ae71cabd6146957 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Thu, 29 Feb 2024 15:26:25 +0100 Subject: [PATCH 04/13] Add ja4 files to start implementation --- src/lib/decode/ja4.c | 189 +++++++++++++++++++++++++++++++++++++++++++ src/lib/decode/ja4.h | 54 +++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 src/lib/decode/ja4.c create mode 100644 src/lib/decode/ja4.h diff --git a/src/lib/decode/ja4.c b/src/lib/decode/ja4.c new file mode 100644 index 00000000..bd3aef3f --- /dev/null +++ b/src/lib/decode/ja4.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2023-2024, Peter Haag + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of SWITCH nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "ja4.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "md5.h" +#include "ssl.h" +#include "util.h" + +static int ja4Hash(ja4_t *ja4); + +#define CheckStringSize(s, l) \ + { \ + if ((s) < (l)) { \ + LogError("sLen error in %s line %d: %s\n", __FILE__, __LINE__, ""); \ + abort(); \ + return 0; \ + } else { \ + (s) -= (l); \ + } \ + } + +static int DecodeJA4(ja4_t *ja4, const uint8_t *data, size_t len, uint8_t proto) { + ja4->ja4.a[0] = proto == IPPROTO_TCP ? 't' : 'q'; + return 1; + +} // End of DecodeJA4 + +void ja4Print(ja4_t *ja4) { + printf("SSL/TLS info:\n"); + sslPrint(ja4->ssl); + + printf("ja4 : %s-%s-%s\n", ja4->ja4.a, ja4->ja4.b, ja4->ja4.c); + +} // End of ja4Print + +void ja4Free(ja4_t *ja4) { + if (ja4) free(ja4); +} // End of ja4Free + +ja4_t *ja4Process(const uint8_t *data, size_t len, uint8_t proto) { + ssl_t *ssl = sslProcess(data, len); + if (!ssl) return NULL; + + ja4_t *ja4 = calloc(1, sizeof(ja4_t)); + if (!ja4) { + LogError("calloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); + return NULL; + } + ja4->ssl = ssl; + + if (DecodeJA4(ja4, data, len, proto) == 0) { + free(ja4); + return NULL; + } + + return ja4; + +} // End of ja4Process + +#ifdef MAIN + +int main(int argc, char **argv) { + const uint8_t tls12[] = { + 0x16, 0x03, // ..X..... + 0x01, 0x02, 0x00, 0x01, 0x00, 0x01, 0xfc, 0x03, // ........ + 0x03, 0xec, 0xb2, 0x69, 0x1a, 0xdd, 0xb2, 0xbf, // ...i.... + 0x6c, 0x59, 0x9c, 0x7a, 0xaa, 0xe2, 0x3d, 0xe5, // lY.z..=. + 0xf4, 0x25, 0x61, 0xcc, 0x04, 0xeb, 0x41, 0x02, // .%a...A. + 0x9a, 0xcc, 0x6f, 0xc0, 0x50, 0xa1, 0x6a, 0xc1, // ..o.P.j. + 0xd2, 0x20, 0x46, 0xf8, 0x61, 0x7b, 0x58, 0x0a, // . F.a{X. + 0xc9, 0x35, 0x8e, 0x2a, 0xa4, 0x4e, 0x30, 0x6d, // .5.*.N0m + 0x52, 0x46, 0x6b, 0xcc, 0x98, 0x9c, 0x87, 0xc8, // RFk..... + 0xca, 0x64, 0x30, 0x9f, 0x5f, 0xaf, 0x50, 0xba, // .d0._.P. + 0x7b, 0x4d, 0x00, 0x22, 0x13, 0x01, 0x13, 0x03, // {M.".... + 0x13, 0x02, 0xc0, 0x2b, 0xc0, 0x2f, 0xcc, 0xa9, // ...+./.. + 0xcc, 0xa8, 0xc0, 0x2c, 0xc0, 0x30, 0xc0, 0x0a, // ...,.0.. + 0xc0, 0x09, 0xc0, 0x13, 0xc0, 0x14, 0x00, 0x9c, // ........ + 0x00, 0x9d, 0x00, 0x2f, 0x00, 0x35, 0x01, 0x00, // .../.5.. + 0x01, 0x91, 0x00, 0x00, 0x00, 0x21, 0x00, 0x1f, // .....!.. + 0x00, 0x00, 0x1c, 0x63, 0x6f, 0x6e, 0x74, 0x69, // ...conti + 0x6c, 0x65, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, // le.servi + 0x63, 0x65, 0x73, 0x2e, 0x6d, 0x6f, 0x7a, 0x69, // ces.mozi + 0x6c, 0x6c, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x00, // lla.com. + 0x17, 0x00, 0x00, 0xff, 0x01, 0x00, 0x01, 0x00, // ........ + 0x00, 0x0a, 0x00, 0x0e, 0x00, 0x0c, 0x00, 0x1d, // ........ + 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, 0x01, 0x00, // ........ + 0x01, 0x01, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, // ........ + 0x00, 0x23, 0x00, 0x00, 0x00, 0x10, 0x00, 0x0e, // .#...... + 0x00, 0x0c, 0x02, 0x68, 0x32, 0x08, 0x68, 0x74, // ...h2.ht + 0x74, 0x70, 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x05, // tp/1.1.. + 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x22, 0x00, 0x0a, 0x00, 0x08, 0x04, 0x03, 0x05, // "....... + 0x03, 0x06, 0x03, 0x02, 0x03, 0x00, 0x33, 0x00, // ......3. + 0x6b, 0x00, 0x69, 0x00, 0x1d, 0x00, 0x20, 0x89, // k.i... . + 0x09, 0x85, 0x8f, 0xbe, 0xb6, 0xed, 0x2f, 0x12, // ....../. + 0x48, 0xba, 0x5b, 0x9e, 0x29, 0x78, 0xbe, 0xad, // H.[.)x.. + 0x0e, 0x84, 0x01, 0x10, 0x19, 0x2c, 0x61, 0xda, // .....,a. + 0xed, 0x00, 0x96, 0x79, 0x8b, 0x18, 0x44, 0x00, // ...y..D. + 0x17, 0x00, 0x41, 0x04, 0x4d, 0x18, 0x3d, 0x91, // ..A.M.=. + 0xf5, 0xee, 0xd3, 0x57, 0x91, 0xfa, 0x98, 0x24, // ...W...$ + 0x64, 0xe3, 0xb0, 0x21, 0x4a, 0xaa, 0x5f, 0x5d, // d..!J._] + 0x1b, 0x78, 0x61, 0x6d, 0x9b, 0x9f, 0xbe, 0xbc, // .xam.... + 0x22, 0xd1, 0x1f, 0x53, 0x5b, 0x2f, 0x94, 0xc6, // "..S[/.. + 0x86, 0x14, 0x31, 0x36, 0xaa, 0x79, 0x5e, 0x6e, // ..16.y^n + 0x5a, 0x87, 0x5d, 0x6c, 0x08, 0x06, 0x4a, 0xd5, // Z.]l..J. + 0xb7, 0x6d, 0x44, 0xca, 0xad, 0x76, 0x6e, 0x24, // .mD..vn$ + 0x83, 0x01, 0x27, 0x48, 0x00, 0x2b, 0x00, 0x05, // ..'H.+.. + 0x04, 0x03, 0x04, 0x03, 0x03, 0x00, 0x0d, 0x00, // ........ + 0x18, 0x00, 0x16, 0x04, 0x03, 0x05, 0x03, 0x06, // ........ + 0x03, 0x08, 0x04, 0x08, 0x05, 0x08, 0x06, 0x04, // ........ + 0x01, 0x05, 0x01, 0x06, 0x01, 0x02, 0x03, 0x02, // ........ + 0x01, 0x00, 0x2d, 0x00, 0x02, 0x01, 0x01, 0x00, // ..-..... + 0x1c, 0x00, 0x02, 0x40, 0x01, 0x00, 0x15, 0x00, // ...@.... + 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // z....... + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00 // ... + }; + /* + [JA4: t13d1715h2_5b57614c22b0_3d5424432f57] + JA4_r: + t13d1715h2_002f,0035,009c,009d,1301,1302,1303,c009,c00a,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0015,0017,001c,0022,0023,002b,002d,0033,ff01_0403,0503,0603,0804,0805,0806,0401,0501,0601,0203,0201] + + ja4 Fullstring: + 771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28-21,29-23-24-25-256-257,0] + [ja4: 579ccef312d18482fc42e2b822ca2430] + */ + // size_t len = sizeof(clientHello); + size_t len = sizeof(tls12); + + ja4_t *ja4 = ja4Process(tls12, len, IPPROTO_TCP); + if (ja4) + ja4Print(ja4); + else + printf("Failed to parse ssl\n"); + + return 0; +} + +#endif \ No newline at end of file diff --git a/src/lib/decode/ja4.h b/src/lib/decode/ja4.h new file mode 100644 index 00000000..48df9185 --- /dev/null +++ b/src/lib/decode/ja4.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023-2024, Peter Haag + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of SWITCH nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef _jA4_H +#define _jA4_H 1 + +#include +#include + +#include "ssl.h" + +typedef struct ja4_s { + ssl_t *ssl; + struct { + char a[16]; + char b[16]; + char c[16]; + } ja4; +} ja4_t; + +void ja4Print(ja4_t *ja4); + +void ja4Free(ja4_t *ja4); + +ja4_t *ja4Process(const uint8_t *data, size_t len, uint8_t proto); + +#endif From 2ded6b5859af87a24e4a50d8d69cacb322495426 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Thu, 29 Feb 2024 18:16:07 +0100 Subject: [PATCH 05/13] Implement ja_a and ja_b of ja --- src/inline/itoa.c | 163 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 src/inline/itoa.c diff --git a/src/inline/itoa.c b/src/inline/itoa.c new file mode 100644 index 00000000..60698a80 --- /dev/null +++ b/src/inline/itoa.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2024, Peter Haag + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of SWITCH nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +// https://johnnylee-sde.github.io/Fast-unsigned-integer-to-string/ + +#include + +static uint32_t SmallDecimalToString(uint64_t x, char *s) { + if (x <= 9) { + *s = (char)(x | 0x30); + return 1; + } else if (x <= 99) { + uint64_t low = x; + uint64_t ll = ((low * 103) >> 9) & 0x1E; + low += ll * 3; + ll = ((low & 0xF0) >> 4) | ((low & 0x0F) << 8); + *(uint16_t *)s = (uint16_t)(ll | 0x3030); + return 2; + } + return 0; +} + +static uint32_t SmallUlongToString(uint64_t x, char *s) { + uint64_t low; + uint64_t ll; + uint32_t digits; + + if (x <= 99) return SmallDecimalToString(x, s); + + low = x; + digits = (low > 999) ? 4 : 3; + + // division and remainder by 100 + // Simply dividing by 100 instead of multiply-and-shift + // is about 50% more expensive timewise on my box + ll = ((low * 5243) >> 19) & 0xFF; + low -= ll * 100; + + low = (low << 16) | ll; + + // Two divisions by 10 (14 bits needed) + ll = ((low * 103) >> 9) & 0x1E001E; + low += ll * 3; + + // move digits into correct spot + ll = ((low & 0x00F000F0) << 28) | (low & 0x000F000F) << 40; + + // convert from decimal digits to ASCII number digit range + ll |= 0x3030303000000000; + + uint8_t *p = (uint8_t *)≪ + if (digits == 4) { + *(uint32_t *)s = *(uint32_t *)(&p[4]); + } else { + *(uint16_t *)s = *(uint16_t *)(&p[5]); + *(((uint8_t *)s) + 2) = *(uint8_t *)(&p[7]); + } + + return digits; +} + +static uint32_t itoa(uint64_t x, char *s) { + uint64_t low; + uint64_t ll; + uint32_t digits; + + // 8 digits or less? + // fits into single 64-bit CPU register + if (x <= 9999) { + return SmallUlongToString(x, s); + } else if (x < 100000000) { + low = x; + + // more than 6 digits? + if (low > 999999) { + digits = (low > 9999999) ? 8 : 7; + } else { + digits = (low > 99999) ? 6 : 5; + } + } else { + uint64_t high = (((uint64_t)x) * 0x55E63B89) >> 57; + low = x - (high * 100000000); + // h will be at most 42 + // calc num digits + digits = SmallDecimalToString(high, s); + digits += 8; + } + + ll = (low * 109951163) >> 40; + low -= ll * 10000; + low |= ll << 32; + + // Four divisions and remainders by 100 + ll = ((low * 5243) >> 19) & 0x000000FF000000FF; + low -= ll * 100; + low = (low << 16) | ll; + + // Eight divisions by 10 (14 bits needed) + ll = ((low * 103) >> 9) & 0x001E001E001E001E; + low += ll * 3; + + // move digits into correct spot + ll = ((low & 0x00F000F000F000F0) >> 4) | (low & 0x000F000F000F000F) << 8; + ll = (ll >> 32) | (ll << 32); + + // convert from decimal digits to ASCII number digit range + ll |= 0x3030303030303030; + + if (digits >= 8) { + *(uint64_t *)(s + digits - 8) = ll; + } else { + uint32_t d = digits; + char *s1 = s; + char *pll = (char *)&(((char *)&ll)[8 - digits]); + + if (d >= 4) { + *(uint32_t *)s1 = *(uint32_t *)pll; + + s1 += 4; + pll += 4; + d -= 4; + } + if (d >= 2) { + *(uint16_t *)s1 = *(uint16_t *)pll; + + s1 += 2; + pll += 2; + d -= 2; + } + if (d > 0) { + *(uint8_t *)s1 = *(uint8_t *)pll; + } + } + + return digits; +} From 8ed9f34129b22d9d790bf47d505fe12dc4a699da Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Fri, 1 Mar 2024 08:43:57 +0100 Subject: [PATCH 06/13] Add ja4_c --- src/lib/Makefile.am | 4 +- src/lib/decode/ja3.c | 9 +-- src/lib/decode/ja4.c | 147 +++++++++++++++++++++++++++++++++++++++++-- src/lib/decode/ja4.h | 2 +- src/lib/decode/ssl.c | 73 ++++++++++++++++++++- src/lib/decode/ssl.h | 46 +++++++------- src/lib/util.c | 23 ++++++- src/lib/util.h | 4 +- 8 files changed, 269 insertions(+), 39 deletions(-) diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index d41378d9..7a2b660c 100755 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -27,7 +27,9 @@ output = userio.c userio.h output_short.c output_short.h regex = sgregex/sgregex.c sgregex/sgregex.h daemon = daemon.c daemon.h version = version.c version.h -decode = decode/content_dns.c decode/content_dns.h decode/ssl.c decode/ssl.h decode/ja3.c decode/ja3.h decode/md5.c decode/md5.h decode/sha256.c decode/sha256.h +decode = decode/content_dns.c decode/content_dns.h +decode += decode/ssl.c decode/ssl.h decode/ja3.c decode/ja3.h decode/ja4.c decode/ja4.h +decode += decode/md5.c decode/md5.h decode/sha256.c decode/sha256.h vcs_track.h: Makefile ./gen_version.sh diff --git a/src/lib/decode/ja3.c b/src/lib/decode/ja3.c index 4e205ad2..89f13e8e 100644 --- a/src/lib/decode/ja3.c +++ b/src/lib/decode/ja3.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -126,13 +127,13 @@ char *ja3HashString(ja3_t *ja3) { uint8_t *u8 = (uint8_t *)ja3->md5Hash; int i, j; - for (i = 0, j = 0; i < 16; i++, j += 2) { + for (i = 0, j = 0; i < 16; i++) { uint8_t ln = u8[i] & 0xF; uint8_t hn = (u8[i] >> 4) & 0xF; - out[j + 1] = ln <= 9 ? ln + '0' : ln + 'a' - 10; - out[j] = hn <= 9 ? hn + '0' : hn + 'a' - 10; + out[j++] = hn <= 9 ? hn + '0' : hn + 'a' - 10; + out[j++] = ln <= 9 ? ln + '0' : ln + 'a' - 10; } - out[32] = '\0'; + out[j] = '\0'; return out; } // End of ja3HashString diff --git a/src/lib/decode/ja4.c b/src/lib/decode/ja4.c index bd3aef3f..a05d86d7 100644 --- a/src/lib/decode/ja4.c +++ b/src/lib/decode/ja4.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, Peter Haag + * Copyright (c) 2024, Peter Haag * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,12 +39,10 @@ #include #include -#include "md5.h" +#include "sha256.h" #include "ssl.h" #include "util.h" -static int ja4Hash(ja4_t *ja4); - #define CheckStringSize(s, l) \ { \ if ((s) < (l)) { \ @@ -56,8 +54,144 @@ static int ja4Hash(ja4_t *ja4); } \ } +#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) + +static void sort(uint16Array_t array) { + uint16_t *elements = array.array; + uint32_t numElements = LenArray(array); + + for (int i = 0; i < numElements - 1; i++) { + for (int j = 0; j < numElements - i - 1; j++) { + if (elements[j] > elements[j + 1]) { + uint16_t swap = elements[j]; + elements[j] = elements[j + 1]; + elements[j + 1] = swap; + } + } + } +} // End of sort + static int DecodeJA4(ja4_t *ja4, const uint8_t *data, size_t len, uint8_t proto) { + // create ja4_a ja4->ja4.a[0] = proto == IPPROTO_TCP ? 't' : 'q'; + + ja4->ja4.a[1] = ja4->ssl->tlsCharVersion[0]; + ja4->ja4.a[2] = ja4->ssl->tlsCharVersion[1]; + + ja4->ja4.a[3] = ja4->ssl->sniName[0] ? 'd' : 'i'; + + uint32_t num = LenArray(ja4->ssl->cipherSuites); + if (num > 99) return 0; + uint32_t ones = num % 10; + uint32_t tens = num / 10; + ja4->ja4.a[4] = tens + '0'; + ja4->ja4.a[5] = ones + '0'; + + num = LenArray(ja4->ssl->extensions); + if (num > 99) return 0; + ones = num % 10; + tens = num / 10; + ja4->ja4.a[6] = tens + '0'; + ja4->ja4.a[7] = ones + '0'; + + if (ja4->ssl->alpnName[0]) { + // first and last char + ja4->ja4.a[8] = ja4->ssl->alpnName[0]; + ja4->ja4.a[9] = ja4->ssl->alpnName[strlen(ja4->ssl->alpnName) - 1]; + } else { + ja4->ja4.a[8] = '0'; + ja4->ja4.a[9] = '0'; + } + + // create ja4_b + sort(ja4->ssl->cipherSuites); + + // generate string to sha256 + // create a string big enough for ciphersuites and extensions + // uint16_t = max 5 digits + ',' = 6 digits per cipher + '\0' + size_t maxStrLen = MAX(LenArray(ja4->ssl->cipherSuites), (LenArray(ja4->ssl->extensions) + LenArray(ja4->ssl->signatures))) * 6 + 1; + char *hashString = (char *)malloc(maxStrLen); + hashString[0] = '0'; + + char hexChars[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + uint32_t index = 0; + for (int i = 0; i < LenArray(ja4->ssl->cipherSuites); i++) { + uint16_t val = ArrayElement(ja4->ssl->cipherSuites, i); + uint8_t n1 = val >> 12; + uint8_t n2 = (val >> 8) & 0xF; + uint8_t n3 = (val >> 4) & 0xF; + uint8_t n4 = val & 0xF; + hashString[index++] = hexChars[n1]; + hashString[index++] = hexChars[n2]; + hashString[index++] = hexChars[n3]; + hashString[index++] = hexChars[n4]; + hashString[index++] = ','; + } + // overwrite last ',' with end of string + hashString[index - 1] = '\0'; + + uint8_t sha256Digest[32]; + char sha256String[65]; + sha256((const unsigned char *)hashString, strlen(hashString), (unsigned char *)sha256Digest); + +#ifdef DEVEL + HexString(sha256Digest, 32, sha256String); + printf("CipherString: %s\n", hashString); + printf(" . Digest: %s\n", sha256String); +#else + HexString(sha256Digest, 6, sha256String); +#endif + + memcpy((void *)ja4->ja4.b, (void *)sha256String, 12); + ja4->ja4.b[12] = '\0'; + + // create ja4_c + sort(ja4->ssl->extensions); + + hashString[0] = '0'; + index = 0; + for (int i = 0; i < LenArray(ja4->ssl->extensions); i++) { + uint16_t val = ArrayElement(ja4->ssl->extensions, i); + // skip extensions 0000 and 0010 + if (val == 0 || val == 0x10) continue; + uint8_t n1 = val >> 12; + uint8_t n2 = (val >> 8) & 0xF; + uint8_t n3 = (val >> 4) & 0xF; + uint8_t n4 = val & 0xF; + hashString[index++] = hexChars[n1]; + hashString[index++] = hexChars[n2]; + hashString[index++] = hexChars[n3]; + hashString[index++] = hexChars[n4]; + hashString[index++] = ','; + } + hashString[index - 1] = '_'; + for (int i = 0; i < LenArray(ja4->ssl->signatures); i++) { + uint16_t val = ArrayElement(ja4->ssl->signatures, i); + uint8_t n1 = val >> 12; + uint8_t n2 = (val >> 8) & 0xF; + uint8_t n3 = (val >> 4) & 0xF; + uint8_t n4 = val & 0xF; + hashString[index++] = hexChars[n1]; + hashString[index++] = hexChars[n2]; + hashString[index++] = hexChars[n3]; + hashString[index++] = hexChars[n4]; + hashString[index++] = ','; + } + // overwrite last ',' with end of string + hashString[index - 1] = '\0'; + + sha256((const unsigned char *)hashString, strlen(hashString), (unsigned char *)sha256Digest); + +#ifdef DEVEL + HexString(sha256Digest, 32, sha256String); + printf("ExtSigString: %s\n", hashString); + printf(" . Digest: %s\n", sha256String); +#else + HexString(sha256Digest, 6, sha256String); +#endif + memcpy((void *)ja4->ja4.c, (void *)sha256String, 12); + + free(hashString); return 1; } // End of DecodeJA4 @@ -66,7 +200,8 @@ void ja4Print(ja4_t *ja4) { printf("SSL/TLS info:\n"); sslPrint(ja4->ssl); - printf("ja4 : %s-%s-%s\n", ja4->ja4.a, ja4->ja4.b, ja4->ja4.c); + printf("ja4:\n"); + printf("ja4 : %s_%s_%s\n", ja4->ja4.a, ja4->ja4.b, ja4->ja4.c); } // End of ja4Print @@ -181,7 +316,7 @@ int main(int argc, char **argv) { if (ja4) ja4Print(ja4); else - printf("Failed to parse ssl\n"); + printf("Failed to parse ja4\n"); return 0; } diff --git a/src/lib/decode/ja4.h b/src/lib/decode/ja4.h index 48df9185..95794836 100644 --- a/src/lib/decode/ja4.h +++ b/src/lib/decode/ja4.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, Peter Haag + * Copyright (c) 2024, Peter Haag * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/lib/decode/ssl.c b/src/lib/decode/ssl.c index f4b0fce1..976aa0c8 100644 --- a/src/lib/decode/ssl.c +++ b/src/lib/decode/ssl.c @@ -112,6 +112,24 @@ static int ProcessExtElCurves(ssl_t *ssl, BytesStream_t *sslStream) { return 1; } // End of ProcessExtElCurves +static int ProcessSignatures(ssl_t *ssl, BytesStream_t *sslStream) { + uint16_t sigLen; + ByteStream_GET_u16(*sslStream, sigLen); + + if (sigLen > ByteStream_AVAILABLE(*sslStream)) { + LogError("%s():%d ecsp extension length error", __FUNCTION__, __LINE__); + return 0; + } + + for (int i = 0; i < sigLen >> 1; i++) { + uint16_t signature; + ByteStream_GET_u16(*sslStream, signature); + AppendArray(ssl->signatures, signature); + dbg_printf("Found signature: 0x%x\n", signature); + } + return 1; +} // End of ProcessSignatures + static int ProcessExtElCurvesPoints(ssl_t *ssl, BytesStream_t *sslStream) { uint8_t ecspLen; ByteStream_GET_u8(*sslStream, ecspLen); @@ -197,6 +215,9 @@ static int sslParseExtensions(ssl_t *ssl, BytesStream_t sslStream, uint16_t leng case 11: // Elliptic curve point formats uncompressed ret = ProcessExtElCurvesPoints(ssl, &sslStream); break; + case 13: // signatures + ret = ProcessSignatures(ssl, &sslStream); + break; case 16: // application_layer_protocol_negotiation (16) ret = ProcessExtALPN(ssl, &sslStream); break; @@ -220,13 +241,41 @@ static int sslParseClientHandshake(ssl_t *ssl, BytesStream_t sslStream, uint32_t ByteStream_GET_u16(sslStream, version); // client hello protocol version ByteStream_SKIP(sslStream, 32); // random init bytes + /* + 0x0304 = TLS 1.3 = “13” + 0x0303 = TLS 1.2 = “12” + 0x0302 = TLS 1.1 = “11” + 0x0301 = TLS 1.0 = “10” + 0x0300 = SSL 3.0 = “s3” + 0x0200 = SSL 2.0 = “s2” + 0x0100 = SSL 1.0 = “s1” + + Unknown = “00” + */ ssl->protocolVersion = version; switch (version) { + case 0x0100: + // SSL 1.0 was never really release! + break; case 0x0200: // SSL 2.0 + ssl->tlsCharVersion[0] = 's'; + ssl->tlsCharVersion[1] = '2'; + break; case 0x0300: // SSL 3.0 + ssl->tlsCharVersion[0] = 's'; + ssl->tlsCharVersion[1] = '3'; + break; case 0x0301: // TLS 1.1 + ssl->tlsCharVersion[0] = '1'; + ssl->tlsCharVersion[1] = '0'; + break; case 0x0302: // TLS 1.2 + ssl->tlsCharVersion[0] = '1'; + ssl->tlsCharVersion[1] = '2'; + break; case 0x0303: // TLS 1.3 + ssl->tlsCharVersion[0] = '1'; + ssl->tlsCharVersion[1] = '3'; break; default: LogError("%s():%d Not an SSL 3.0 - TLS 1.3 protocol", __FUNCTION__, __LINE__); @@ -289,11 +338,28 @@ static int sslParseServerHandshake(ssl_t *ssl, BytesStream_t sslStream, uint32_t ssl->protocolVersion = version; switch (version) { + case 0x0100: + // SSL 1.0 was never really release! + break; case 0x0200: // SSL 2.0 + ssl->tlsCharVersion[0] = 's'; + ssl->tlsCharVersion[1] = '2'; + break; case 0x0300: // SSL 3.0 + ssl->tlsCharVersion[0] = 's'; + ssl->tlsCharVersion[1] = '3'; + break; case 0x0301: // TLS 1.1 + ssl->tlsCharVersion[0] = '1'; + ssl->tlsCharVersion[1] = '0'; + break; case 0x0302: // TLS 1.2 + ssl->tlsCharVersion[0] = '1'; + ssl->tlsCharVersion[1] = '2'; + break; case 0x0303: // TLS 1.3 + ssl->tlsCharVersion[0] = '1'; + ssl->tlsCharVersion[1] = '3'; break; default: LogError("%s():%d Not an SSL 3.0 - TLS 1.3 protocol", __FUNCTION__, __LINE__); @@ -360,10 +426,13 @@ void sslPrint(ssl_t *ssl) { for (int i = 0; i < LenArray(ssl->extensions); i++) { printf(" 0x%x", ssl->extensions.array[i]); } - printf("\n"); + printf("\nsignatures :"); + for (int i = 0; i < LenArray(ssl->signatures); i++) { + printf(" 0x%x", ssl->signatures.array[i]); + } if (ssl->sniName[0]) { - printf("SNI name : %s\n", ssl->sniName); + printf("\nSNI name : %s\n", ssl->sniName); } if (ssl->alpnName[0]) { diff --git a/src/lib/decode/ssl.h b/src/lib/decode/ssl.h index 4a04933d..e3335ee4 100644 --- a/src/lib/decode/ssl.h +++ b/src/lib/decode/ssl.h @@ -41,41 +41,45 @@ typedef struct uint16Array_s { #define arrayMask 0x1F -#define NewArray(a) \ - { \ - a.numElements = 0; \ - a.array = NULL; \ +#define NewArray(a) \ + { \ + (a).numElements = 0; \ + (a).array = NULL; \ } -#define AppendArray(a, v) \ - if ((a.numElements & arrayMask) == 0) { \ - a.array = (uint16_t *)realloc(a.array, sizeof(uint16_t) * (a.numElements + (arrayMask + 1))); \ - if (!a.array) { \ - fprintf(stderr, "malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); \ - exit(255); \ - } \ - } \ - a.array[a.numElements++] = (v); - -#define FreeArray(a) \ - if (a.numElements && a.array) { \ - free(a.array); \ - a.numElements = 0; \ - a.array = NULL; \ +#define AppendArray(a, v) \ + if (((a).numElements & arrayMask) == 0) { \ + (a).array = (uint16_t *)realloc((a).array, sizeof(uint16_t) * ((a).numElements + (arrayMask + 1))); \ + if (!(a).array) { \ + fprintf(stderr, "malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); \ + exit(255); \ + } \ + } \ + (a).array[a.numElements++] = (v); + +#define FreeArray(a) \ + if ((a).numElements && (a).array) { \ + free((a).array); \ + (a).numElements = 0; \ + (a).array = NULL; \ } -#define LenArray(a) a.numElements +#define LenArray(a) (a).numElements + +#define ArrayElement(a, n) (a).array[n] typedef struct ssl_s { uint16_t tlsVersion; #define CLIENTssl 0 #define SERVERssls 1 - uint16_t type; + char tlsCharVersion[2]; uint16_t protocolVersion; + uint16_t type; uint16Array_t cipherSuites; uint16Array_t extensions; uint16Array_t ellipticCurves; uint16Array_t ellipticCurvesPF; + uint16Array_t signatures; #define ALPNmaxLen 16 // ALPN are currently defined up to 8 bytes char alpnName[ALPNmaxLen]; diff --git a/src/lib/util.c b/src/lib/util.c index 1f23cdb7..09f53262 100755 --- a/src/lib/util.c +++ b/src/lib/util.c @@ -735,7 +735,8 @@ static const uint8_t utf8d[] = { 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // s7..s8 }; -uint32_t decode(uint32_t *state, uint32_t *codep, uint32_t byte) { +/* +uint32_t utfDecode(uint32_t *state, uint32_t *codep, uint32_t byte) { uint32_t type = utf8d[byte]; *codep = (*state != UTF8_ACCEPT) ? (byte & 0x3fu) | (*codep << 6) : (0xff >> type) & (byte); @@ -743,6 +744,7 @@ uint32_t decode(uint32_t *state, uint32_t *codep, uint32_t byte) { *state = utf8d[256 + *state * 16 + type]; return *state; } +*/ uint32_t validate_utf8(uint32_t *state, char *str, size_t len) { size_t i; @@ -750,7 +752,7 @@ uint32_t validate_utf8(uint32_t *state, char *str, size_t len) { for (i = 0; i < len; i++) { // We don't care about the codepoint, so this is - // a simplified version of the decode function. + // a simplified version of the utfDecode function. type = utf8d[(uint8_t)str[i]]; *state = utf8d[256 + (*state) * 16 + type]; @@ -760,6 +762,23 @@ uint32_t validate_utf8(uint32_t *state, char *str, size_t len) { return *state; } +/* + * converts a uint8_t array ( e.g. md5 or sha256 ) to a readable string + * hexstring mus be big enough (2 * len) to hold the final string + */ +char *HexString(uint8_t *hex, size_t len, char *hexString) { + int i, j = 0; + for (i = 0, j = 0; i < len; i++) { + uint8_t ln = hex[i] & 0xF; + uint8_t hn = (hex[i] >> 4) & 0xF; + hexString[j++] = hn <= 9 ? hn + '0' : hn + 'a' - 10; + hexString[j++] = ln <= 9 ? ln + '0' : ln + 'a' - 10; + } + hexString[j] = '\0'; + + return hexString; +} // End of ja3HashString + void DumpHex(FILE *stream, const void *data, size_t size) { char ascii[17]; size_t i, j; diff --git a/src/lib/util.h b/src/lib/util.h index b0ddd6cb..9a594e53 100755 --- a/src/lib/util.h +++ b/src/lib/util.h @@ -157,10 +157,10 @@ void inet6_ntop_mask(uint64_t ipv6[2], int mask, char *s, socklen_t sSize); #define UTF8_ACCEPT 0 #define UTF8_REJECT 1 -uint32_t decode(uint32_t *state, uint32_t *codep, uint32_t byte); - uint32_t validate_utf8(uint32_t *state, char *str, size_t len); +char *HexString(uint8_t *hex, size_t len, char *hexString); + void DumpHex(FILE *stream, const void *data, size_t size); #endif //_UTIL_H From 5c72cc8943388675ca491d240bc3c1444621f45e Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Fri, 1 Mar 2024 13:25:08 +0100 Subject: [PATCH 07/13] Add ja3 filter in nfdump-ja4 --- Makefile.am | 2 +- configure.ac | 8 +++---- src/decode/Makefile.am | 14 ++++++++++++ src/{lib/decode => decode/digest}/md5.c | 0 src/{lib/decode => decode/digest}/md5.h | 0 src/{lib/decode => decode/digest}/sha256.c | 0 src/{lib/decode => decode/digest}/sha256.h | 0 .../decode/content_dns.c => decode/dns/dns.c} | 2 +- .../decode/content_dns.h => decode/dns/dns.h} | 0 src/{lib/decode => decode/ja3}/ja3.c | 4 ++-- src/{lib/decode => decode/ja3}/ja3.h | 2 +- src/{lib/decode => decode/ja4}/ja4.c | 4 ++-- src/{lib/decode => decode/ja4}/ja4.h | 0 src/decode/sha256 | Bin 0 -> 51160 bytes src/{lib/decode => decode/ssl}/ssl.c | 2 +- src/{lib/decode => decode/ssl}/ssl.h | 0 src/lib/Makefile.am | 7 ++---- src/lib/filter/filter.c | 21 ++++++++++++++++++ src/lib/filter/filter.h | 1 + src/lib/filter/grammar.y | 4 ++-- src/lib/nffile.c | 10 ++++++++- src/netflow/netflow_v1.c | 12 +++------- src/nfdump/Makefile.am | 4 ++-- src/nfdump/nflowcache.c | 2 +- src/nfdump/nfstat.c | 2 +- src/nfreplay/Makefile.am | 2 +- src/nfsen/Makefile.am | 6 ++--- src/output/Makefile.am | 2 +- src/output/output_fmt.c | 4 ++-- src/output/output_json.c | 2 +- src/output/output_raw.c | 6 ++--- src/test/Makefile.am | 4 ++-- 32 files changed, 80 insertions(+), 47 deletions(-) create mode 100755 src/decode/Makefile.am rename src/{lib/decode => decode/digest}/md5.c (100%) rename src/{lib/decode => decode/digest}/md5.h (100%) rename src/{lib/decode => decode/digest}/sha256.c (100%) rename src/{lib/decode => decode/digest}/sha256.h (100%) rename src/{lib/decode/content_dns.c => decode/dns/dns.c} (99%) rename src/{lib/decode/content_dns.h => decode/dns/dns.h} (100%) rename src/{lib/decode => decode/ja3}/ja3.c (99%) rename src/{lib/decode => decode/ja3}/ja3.h (98%) rename src/{lib/decode => decode/ja4}/ja4.c (99%) rename src/{lib/decode => decode/ja4}/ja4.h (100%) create mode 100755 src/decode/sha256 rename src/{lib/decode => decode/ssl}/ssl.c (99%) rename src/{lib/decode => decode/ssl}/ssl.h (100%) diff --git a/Makefile.am b/Makefile.am index 375db472..b069d85d 100755 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,6 @@ ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = src/lib src/output src/netflow src/collector src/maxmind src/nfdump src/nfcapd +SUBDIRS = src/lib src/output src/netflow src/collector src/maxmind src/decode src/nfdump src/nfcapd SUBDIRS += src/nfanon src/nfexpire src/nfreplay . src src/test src/nfreader src/inline src/include if SFLOW diff --git a/configure.ac b/configure.ac index 0b1df15e..af26c17d 100644 --- a/configure.ac +++ b/configure.ac @@ -616,13 +616,13 @@ AM_COND_IF([HAVE_DOXYGEN], AC_CONFIG_FILES([doc/Doxyfile])) AC_CONFIG_FILES([doc/Makefile]) AC_OUTPUT -AC_CONFIG_FILES([Makefile src/lib/Makefile src/Makefile src/test/Makefile - src/output/Makefile src/netflow/Makefile src/collector/Makefile - src/maxmind/Makefile src/nfdump/Makefile src/nfcapd/Makefile src/nfexpire/Makefile +AC_CONFIG_FILES([Makefile src/lib/Makefile src/decode/Makefile + src/Makefile src/test/Makefile src/output/Makefile + src/netflow/Makefile src/collector/Makefile src/maxmind/Makefile + src/nfdump/Makefile src/nfcapd/Makefile src/nfexpire/Makefile src/nfanon/Makefile src/nfreplay/Makefile src/nfreader/Makefile src/inline/Makefile src/include/Makefile man/Makefile ]) - if test "x$enable_ftconv" = "xyes"; then AC_CONFIG_FILES([src/ft2nfdump/Makefile]) fi diff --git a/src/decode/Makefile.am b/src/decode/Makefile.am new file mode 100755 index 00000000..d6e53082 --- /dev/null +++ b/src/decode/Makefile.am @@ -0,0 +1,14 @@ + +AM_CPPFLAGS = -I.. -I../include -I../inline -I../lib -Issl $(DEPS_CFLAGS) +AM_CFLAGS = -ggdb + +LDADD = $(DEPS_LIBS) + +# libnfdecode sources +decode = dns/dns.c dns/dns.h +decode += ssl/ssl.c ssl/ssl.h ja3/ja3.c ja3/ja3.h ja4/ja4.c ja4/ja4.h +decode += digest/md5.c digest/md5.h digest/sha256.c digest/sha256.h + +noinst_LIBRARIES = libnfdecode.a + +libnfdecode_a_SOURCES = $(decode) \ No newline at end of file diff --git a/src/lib/decode/md5.c b/src/decode/digest/md5.c similarity index 100% rename from src/lib/decode/md5.c rename to src/decode/digest/md5.c diff --git a/src/lib/decode/md5.h b/src/decode/digest/md5.h similarity index 100% rename from src/lib/decode/md5.h rename to src/decode/digest/md5.h diff --git a/src/lib/decode/sha256.c b/src/decode/digest/sha256.c similarity index 100% rename from src/lib/decode/sha256.c rename to src/decode/digest/sha256.c diff --git a/src/lib/decode/sha256.h b/src/decode/digest/sha256.h similarity index 100% rename from src/lib/decode/sha256.h rename to src/decode/digest/sha256.h diff --git a/src/lib/decode/content_dns.c b/src/decode/dns/dns.c similarity index 99% rename from src/lib/decode/content_dns.c rename to src/decode/dns/dns.c index 6b829649..0151a697 100644 --- a/src/lib/decode/content_dns.c +++ b/src/decode/dns/dns.c @@ -73,7 +73,7 @@ #include "nfdump.h" #include "util.h" // #include "nffile.h" -#include "content_dns.h" +#include "dns.h" #include "inline.c" /* diff --git a/src/lib/decode/content_dns.h b/src/decode/dns/dns.h similarity index 100% rename from src/lib/decode/content_dns.h rename to src/decode/dns/dns.h diff --git a/src/lib/decode/ja3.c b/src/decode/ja3/ja3.c similarity index 99% rename from src/lib/decode/ja3.c rename to src/decode/ja3/ja3.c index 89f13e8e..29755a1b 100644 --- a/src/lib/decode/ja3.c +++ b/src/decode/ja3/ja3.c @@ -38,8 +38,8 @@ #include #include -#include "md5.h" -#include "ssl.h" +#include "digest/md5.h" +#include "ssl/ssl.h" #include "util.h" static int ja3Hash(ja3_t *ja3); diff --git a/src/lib/decode/ja3.h b/src/decode/ja3/ja3.h similarity index 98% rename from src/lib/decode/ja3.h rename to src/decode/ja3/ja3.h index e00be898..99471fd9 100644 --- a/src/lib/decode/ja3.h +++ b/src/decode/ja3/ja3.h @@ -34,7 +34,7 @@ #include #include -#include "ssl.h" +#include "ssl/ssl.h" typedef struct ja3_s { ssl_t *ssl; diff --git a/src/lib/decode/ja4.c b/src/decode/ja4/ja4.c similarity index 99% rename from src/lib/decode/ja4.c rename to src/decode/ja4/ja4.c index a05d86d7..46e202b4 100644 --- a/src/lib/decode/ja4.c +++ b/src/decode/ja4/ja4.c @@ -39,8 +39,8 @@ #include #include -#include "sha256.h" -#include "ssl.h" +#include "digest/sha256.h" +#include "ssl/ssl.h" #include "util.h" #define CheckStringSize(s, l) \ diff --git a/src/lib/decode/ja4.h b/src/decode/ja4/ja4.h similarity index 100% rename from src/lib/decode/ja4.h rename to src/decode/ja4/ja4.h diff --git a/src/decode/sha256 b/src/decode/sha256 new file mode 100755 index 0000000000000000000000000000000000000000..23528c1f3455cbf9ad3eb6562d3fbd7cace19f0d GIT binary patch literal 51160 zcmeI3dvw&r702hdo5ycA2?-C0;gwKPvLR%7mIM(O14TB-Ba5%v^6QdplFe&Fc7tG` zu12ag+9uYv^w?HVtO?j2+WN?zQcOe<)N0e7swl?>?9p8~Xwj;y8q)9mJ(kVJmY$>i z@6I_lGxyHSojad9Gn;>YuY7v?T)GgFKrunaKzfpd*e>$W5+Vn3HAGR?ELpzno6E{q zlXZAeCl5D`(8$kS7^o=gmff&!c(*#=G2A9}JsD;agrpWl33xUKbOlwSzeV%4%_Ld1 zF{IPJVkJogV{J4P#UE(d;E$OS{W-4G{1F2xq{sV{b$?)}7DcJ>2b#T&)iHhee(%!# zW$OK;$Cr5DWLK1y#+$v3m5R5q%BSkX{q4~G)#y%0PxPU8sTJj62SibpE?KukDP6tF zrDemR9|F{BNvO>*l2ER)Ac|7$i%Y|Q^jD0rYH?`ubxow8L1L8QaT!+B!GyS=tHq&J z@o|wwQ7YX5x7MfWw9oUp;V(};bg&06%u74}D_5*4TeftCadvwX;(|`)^;E4U_r?-r znTM5#o9Gqd=`k$dYpi|B+hUE1LVMl%SvqCh%K^od?AXF~Zh4!2e(U(GNWXZz( zjFDzB3bs55omYE{zu8{z-C(cWQeO$3%0vjoA!g25wZ^ybH$N&Zf8(!DuS}i;Qyv5% zp}>o2l-j0PgmlhafJ)sT=_rUVNPdo$sA}vFtxIQu`e@J66~(yT()m~rk86uR;Atqj zst9p;H((ITWf1b8)u+`qT_;{tCPS#d_c{0`$6S~I6JP>NfC(@GCcp%k025#WOn?b6 z0Vco%m;e)C0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#WOn?b6 z0VV(fLv7~%!PboK!BTUlEW|!J!x5DwN3^Rg_v}EXh#t=tp%bYhI?yIM2h1WWiD0?u zqTpY)n#BDBW=C|K(CXuvj_8SON2t)yQytNJ(e7?j&c^$dCCjOB_Jb2@gjj&@FveBG z&tU5{-H4;J*Q~rcR4V!hu-;Ir2vL7H)e(}XijCo?9-I)Cv^kN~UK-&pojbATUFO`g z4;`G+(}lP#$yi$smLul6aIOe-9bDfN$%T#d1rJW>()U32aHa?)QM~AY5g=A^pWme9HFL8uUnB_fY5xg%K9Mo~>?}k6DktR#hMp_5^R9WqToqXg| zT#_{{AN!e&HPyJ_i{k1kMeL>Pdn|}Mk|sj3<;aAtHrTLUL>to{P7~3c@I&iKi0f;} z={v|3*^Z|PbeWSS$S`m($*!M4phQQfz`AAxycjP1r)n`eSc!k*&yb4n8T zldi=}4A%Epkq=du5Ni(Nmo!d;txLOyN+m5H2LK?>-qkJ4ktOge=_GHmxu}}_D z^xRBW8I6%C-(atf#~KE2mF33CE;qJ;pCL*T z$570X#Br7{8kaS4T=ExOG-&DR+p!A9r=I^{VcngZlk6n{Xady9WV%buToR_5xq;VJ~X`y2YJy(NFnDoUu0+=W^IvI1OX)$yxbe>+0Gn}cY!*j}Jk?S#!k!HKk_7va+G5#l zDL9!H3`HuW&e&T`YZEiX@)nrMreHa>V{y?th86|$VIf^XiX2^BGs0U zw!CDtOJ3&0I*1eFW*n{UG1>)aJ6DW#Ij<=zz#Hm}x63xCrcV|dX&k+=YX22+KG0Tu zPA6o`~OTy4PSz_gZE9w)jG}|4j6ybCuI{uGc$4C(9h62*h?#ip!ScEI5TW z*_^gCmvd%W0p2uc%;wHNWuP&>ZRx4G#e z34XL$9pnxUcP7J-B}|AH|8`Ny3eq}MYa_FQ>UX`+wE4O=s>*+u025#W zOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#W zOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(^xuY^GLJC@q* zzp4I1^M{R#7Jjt)kz310t-Yrv`GLJh+j4XMG;q@n``uS{E?uzkmCt&gc;w9|A~!vA zUCEizXKvhm`<3_oZT+OZJNBf$F5T2N=D_yZ(++M+zc%pg1I_1t`&RJLlw%JB&OFf9 z=Y8NG-&=m*{X@U#%Y129`ZQl*Wageb0?uRI`@Z$YUC~YDYZvcmI5fNK@O1OO=Wo67 z=?~Ke4lVC|^lxh`UP!67*~V?RJ~h2~>cpbY8w#%(n^Ejq=eK(PV6EL$bmW=awihNfC(@GCcp%k025#WOn?b60Vco%m;e)C z0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#WOn?b60Vco%m;e)C z0!-k4AdrVKYamUKREPvIO0v;FeQ{#jks^fSypiTfCdd@XLdbf^9gzDWyY+F3qBM9K zDw?(^6*YB2)BK)*q4@*uiaJu1Dz~?u`p;XcTinf+f~I4FP@0;(je#nG0nPQEMk5SK z5!uv69KtM)`SE9eEt*P0jpEttsb~p!lp2q_($g#~4Q_9v$ne*==gj}Q7^8aFU)v!y zb(lMng5*GmZIDse*h}-V2ej|j^+jLT^q(313pD*vT|Y+qNBc?Afk^~4y-(Nkbe+aj z(jUFNh`Q2HP}AD+dN%Ybcy8S`;GtnOwFrL}>G4g1J}pkC{%Md72o=&-CG^vLV}IX4 zo8}ujn(BNy8&pWA`SJbg`Z^Edqpq*(#{3RypcwNFex$4G>->nPy1uR(^Ru*c#OA*g z*H810m1w@l>+=oWn6FNZTR*;Eov-JM@<;0%`7`EMCZPj`^i^^Fl;6t|@=JPNP#a6* zE9sA3Xd9VQma37N;>{#~di<0>(v5sl?x508wamfac-@hpmnZ0SNaDvgBn4;b=MZaPHx2vx&pm-a-co($Ql$NGScfb=e=L}_cb+m8UyxK-w618^?rL} zRb@*dJUf1Sb--TJ zSl3ipJ-2qAQaiiZU)xZ09Z& zcAHS_%Br7yA!#v*r*#8Nj-Gv+}+Rq`qsyOcZ2Uxq^dXRwf_D()9M?NhqgYx z^_J~#{IUJ&?-xFLq5dDZCztOX%z3$R%ysYYt@>pBXRVnFmao|8D>*%5|6R40{&?(> zEs<$=Z*`cz``F(zHm&>A-tk-OH9t^BzxkrKHGTRMeHZfoW&2m$dupBQ&K*yt-ukii zwUl=s-?2aWV8zR0uYdN #include "filter.h" +#include "ja3/ja3.h" #include "maxmind.h" #include "sgregex.h" #include "util.h" @@ -103,6 +104,7 @@ static uint64_t mpls_exp_function(void *dataPtr, uint32_t length, data_t data, r static uint64_t mpls_any_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); static uint64_t pblock_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); static uint64_t mmASLookup_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); +static uint64_t ja3_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *handle); /* * flow processing function table: @@ -123,6 +125,7 @@ static struct flow_procs_map_s { {FUNC_MPLS_ANY, "mpls any", mpls_any_function}, {FUNC_PBLOCK, "pblock", pblock_function}, {FUNC_MMAS_LOOKUP, "AS Lockup", mmASLookup_function}, + {FUNC_JA3, "ja3", ja3_function}, {0, NULL, NULL}}; // 128bit compare for IPv6 @@ -277,6 +280,24 @@ static uint64_t mmASLookup_function(void *dataPtr, uint32_t length, data_t data, return as; } // End of mmASLookup_function +static uint64_t ja3_function(void *dataPtr, uint32_t length, data_t data, recordHandle_t *recordHandle) { + const uint8_t *payload = (const uint8_t *)recordHandle->extensionList[EXinPayloadID]; + + // check if ja3 already exists or no payload exists + if (recordHandle->ja3[0] != '\0' || payload == NULL) return 1; + + uint32_t len = ExtensionLength(payload); + ja3_t *ja3 = ja3Process(payload, len); + if (ja3) { + recordHandle->ja3Info = (void *)ja3; + memcpy((void *)recordHandle->ja3, ja3->md5Hash, sizeof(recordHandle->ja3)); + return 1; + } + // else - not a valid ssl handshare for ja3 + return 0; + +} // End of ja3_function + static int geoLookup(char *geoChar, uint64_t direction, recordHandle_t *recordHandle) { geoChar[0] = geoChar[1] = '.'; switch (direction) { diff --git a/src/lib/filter/filter.h b/src/lib/filter/filter.h index dedbe1bd..144293da 100755 --- a/src/lib/filter/filter.h +++ b/src/lib/filter/filter.h @@ -105,6 +105,7 @@ typedef enum { FUNC_MPLS_ANY, // function code for matching any MPLS label FUNC_PBLOCK, // function code for matching ports against pblock start FUNC_MMAS_LOOKUP, // function code for optional maxmind AS lookup + FUNC_JA3, // function code for ja3 calc } filterFunction_t; #define FULLMASK FFFFFFFFFFFFFFFFLL diff --git a/src/lib/filter/grammar.y b/src/lib/filter/grammar.y index 695c36f7..4b01f529 100644 --- a/src/lib/filter/grammar.y +++ b/src/lib/filter/grammar.y @@ -1226,7 +1226,7 @@ static int AddPayload(char *type, char *arg, char *opt) { } data_t data = {.dataPtr=md5}; if (strcasecmp(arg, "defined") == 0) { - return Invert(NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_NONE, data)); + return Invert(NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_JA3, data)); } else { if (IsMD5(arg) == 0) { yyerror("ja3 string %s is not an MD5 sum", arg); @@ -1237,7 +1237,7 @@ static int AddPayload(char *type, char *arg, char *opt) { sscanf(arg, "%2hhx", &md5[count]); arg += 2; } - return NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_NONE, data); + return NewElement(EXlocal, OFFja3, SIZEja3, 0, CMP_BINARY, FUNC_JA3, data); } } else { yyerror("Unknown PAYLOAD argument: %s\n", type); diff --git a/src/lib/nffile.c b/src/lib/nffile.c index 14503698..1e059144 100644 --- a/src/lib/nffile.c +++ b/src/lib/nffile.c @@ -27,21 +27,29 @@ * POSSIBILITY OF SUCH DAMAGE. * */ -#include "nffile.h" + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif #include #include +#include "nffile.h" + #ifdef HAVE_BZIP2 #include #endif + #ifdef HAVE_ZSTD #include #endif + #ifdef HAVE_LZ4 #include #include #endif + #include #include #include diff --git a/src/netflow/netflow_v1.c b/src/netflow/netflow_v1.c index 924b71fc..e3c6a7ea 100644 --- a/src/netflow/netflow_v1.c +++ b/src/netflow/netflow_v1.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2023, Peter Haag + * Copyright (c) 2009-2024, Peter Haag * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung * All rights reserved. * @@ -29,13 +29,12 @@ * */ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif +#include "netflow_v1.h" #include #include #include +#include #include #include #include @@ -43,13 +42,8 @@ #include #include -#ifdef HAVE_STDINT_H -#include -#endif - #include "collector.h" #include "metric.h" -#include "netflow_v1.h" #include "nfdump.h" #include "nffile.h" #include "nfnet.h" diff --git a/src/nfdump/Makefile.am b/src/nfdump/Makefile.am index 5ab15770..1a69919b 100755 --- a/src/nfdump/Makefile.am +++ b/src/nfdump/Makefile.am @@ -1,7 +1,7 @@ bin_PROGRAMS = nfdump -AM_CPPFLAGS = -I.. -Icompat_1_6_x -I../include -I../lib -I../output -I../maxmind -I../netflow -I../collector -I../lib/conf -I../lib/filter -I../lib/decode -I../inline $(DEPS_CFLAGS) +AM_CPPFLAGS = -I.. -Icompat_1_6_x -I../include -I../lib -I../output -I../maxmind -I../netflow -I../collector -I../lib/conf -I../lib/filter -I../decode -I../inline $(DEPS_CFLAGS) AM_LDFLAGS = -L../lib EXTRA_DIST = nffile_compat.c memhandle.c heapsort_inline.c @@ -19,6 +19,6 @@ compat = compat_1_6_x/nfx.h compat_1_6_x/nfx.c nfdump_SOURCES = nfdump.c spin_lock.h \ $(exporter) $(nbar) $(ifvrf) $(nfstat) $(nflowcache) $(nfprof) $(sort) $(compat) -nfdump_LDADD = ../output/liboutput.a ../lib/libnfdump.la ../maxmind/libmaxmind.a +nfdump_LDADD = ../output/liboutput.a ../lib/libnfdump.la ../maxmind/libmaxmind.a ../decode/libnfdecode.a CLEANFILES = *.gch diff --git a/src/nfdump/nflowcache.c b/src/nfdump/nflowcache.c index 97c0574e..84bf0f70 100755 --- a/src/nfdump/nflowcache.c +++ b/src/nfdump/nflowcache.c @@ -49,7 +49,7 @@ #include "blocksort.h" #include "config.h" #include "exporter.h" -#include "ja3.h" +#include "ja3/ja3.h" #include "khash.h" #include "klist.h" #include "maxmind.h" diff --git a/src/nfdump/nfstat.c b/src/nfdump/nfstat.c index 0344e156..3b2eae68 100644 --- a/src/nfdump/nfstat.c +++ b/src/nfdump/nfstat.c @@ -48,7 +48,7 @@ #include "blocksort.h" #include "config.h" -#include "ja3.h" +#include "ja3/ja3.h" #include "khash.h" #include "maxmind.h" #include "nfdump.h" diff --git a/src/nfreplay/Makefile.am b/src/nfreplay/Makefile.am index d13a1ec5..fb8d2c94 100755 --- a/src/nfreplay/Makefile.am +++ b/src/nfreplay/Makefile.am @@ -9,6 +9,6 @@ LDADD = $(DEPS_LIBS) replay = send_v5.c send_v5.h send_v9.c send_v9.h send_net.h send_net.c nfreplay_SOURCES = nfreplay.c $(replay) -nfreplay_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a +nfreplay_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a ../decode/libnfdecode.a CLEANFILES = *.gch diff --git a/src/nfsen/Makefile.am b/src/nfsen/Makefile.am index 406fc348..df780d7f 100755 --- a/src/nfsen/Makefile.am +++ b/src/nfsen/Makefile.am @@ -16,10 +16,8 @@ nfstatfile = ../collector/nfstatfile.c ../collector/nfstatfile.h exporter = ../nfdump/exporter.c nfprofile_SOURCES = nfprofile.c profile.c profile.h $(nfstatfile) $(exporter) -nfprofile_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a -lrrd +nfprofile_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a ../decode/libnfdecode.a -lrrd nftrack_SOURCES = nftrack.c nftrack_rrd.c nftrack_rrd.h \ nftrack_stat.c nftrack_stat.h -nftrack_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a -lrrd - - +nftrack_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a ../decode/libnfdecode.a -lrrd \ No newline at end of file diff --git a/src/output/Makefile.am b/src/output/Makefile.am index 97d8504b..1d4cae1c 100755 --- a/src/output/Makefile.am +++ b/src/output/Makefile.am @@ -1,5 +1,5 @@ -AM_CPPFLAGS = -I.. -I../include -I../lib -I../lib/conf -I../lib/decode -I../maxmind -I../inline $(DEPS_CFLAGS) +AM_CPPFLAGS = -I.. -I../include -I../lib -I../decode -I../lib/conf -I../maxmind -I../inline $(DEPS_CFLAGS) noinst_LIBRARIES = liboutput.a diff --git a/src/output/output_fmt.c b/src/output/output_fmt.c index fc3ad3bd..446d0121 100644 --- a/src/output/output_fmt.c +++ b/src/output/output_fmt.c @@ -44,9 +44,9 @@ #include #include -#include "content_dns.h" +#include "dns/dns.h" #include "ifvrf.h" -#include "ja3.h" +#include "ja3/ja3.h" #include "maxmind.h" #include "nbar.h" #include "nfdump.h" diff --git a/src/output/output_json.c b/src/output/output_json.c index 5f373577..58c95361 100644 --- a/src/output/output_json.c +++ b/src/output/output_json.c @@ -44,7 +44,7 @@ #include #include -#include "ja3.h" +#include "ja3/ja3.h" #include "maxmind.h" #include "nfdump.h" #include "nffile.h" diff --git a/src/output/output_raw.c b/src/output/output_raw.c index e55a49b8..f629b912 100644 --- a/src/output/output_raw.c +++ b/src/output/output_raw.c @@ -42,17 +42,17 @@ #include #include "config.h" -#include "content_dns.h" +#include "dns/dns.h" #include "exporter.h" #include "ifvrf.h" -#include "ja3.h" +#include "ja3/ja3.h" #include "maxmind.h" #include "nbar.h" #include "nfdump.h" #include "nffile.h" #include "nfxV3.h" #include "output_util.h" -#include "ssl.h" +#include "ssl/ssl.h" #include "userio.h" #include "util.h" diff --git a/src/test/Makefile.am b/src/test/Makefile.am index beea73a6..b422a263 100755 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -27,10 +27,10 @@ AM_LDFLAGS = -L../lib LDADD = $(DEPS_LIBS) nfgen_SOURCES = nfgen.c -nfgen_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a +nfgen_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a ../decode/libnfdecode.a nftest_SOURCES = nftest.c -nftest_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a +nftest_LDADD = ../lib/libnfdump.la ../maxmind/libmaxmind.a ../decode/libnfdecode.a nftest_DEPENDENCIES = nfgen EXTRA_DIST = runtest.sh nftest.1.out nftest.2.out From c129b789da5fce792040b449ba0a592348bd22f0 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Fri, 1 Mar 2024 14:14:50 +0100 Subject: [PATCH 08/13] Add ja4 Readme for license issue and add --enable-ja4, default no for building all ja4 modules --- configure.ac | 15 +++- src/decode/Makefile.am | 5 ++ src/decode/ja4/Readme.md | 16 ++++ src/decode/ja4/ja4.c | 11 --- src/decode/ja4/ja4s.c | 168 +++++++++++++++++++++++++++++++++++++++ src/decode/ja4/ja4s.h | 54 +++++++++++++ 6 files changed, 256 insertions(+), 13 deletions(-) create mode 100644 src/decode/ja4/Readme.md create mode 100644 src/decode/ja4/ja4s.c create mode 100644 src/decode/ja4/ja4s.h diff --git a/configure.ac b/configure.ac index af26c17d..0d0ed808 100644 --- a/configure.ac +++ b/configure.ac @@ -170,10 +170,17 @@ AM_CONDITIONAL([FT2NFDUMP], [test "x$build_ftconv" = "xyes"]) AC_ARG_ENABLE(maxmind, [ --enable-maxmind Build geolookup for MaxMind GeoDB; default is NO]) -build_maxmind="$enable_maxmind" # No dependencies +AS_IF([test "x$enable_maxmind" = xyes], +build_maxmind="$enable_maxmind", build_maxmind="no") AM_CONDITIONAL([MAXMIND], [test "x$build_maxmind" = "xyes"]) -#Needs tidy +AC_ARG_ENABLE(ja4, +[ --enable-ja4 Build with ja4 fingerprinting code; May require a license; default is NO]) + +AS_IF([test "x$enable_ja4" = xyes], + build_ja4="$enable_ja4", build_ja4="no") +AM_CONDITIONAL([JA4], [test "x$build_ja4" = "xyes"]) + AC_ARG_ENABLE(nfprofile, [ --enable-nfprofile Build nfprofile used by NfSen; default is NO]) @@ -629,6 +636,8 @@ fi if test "x$build_sflow" = "xyes"; then AC_CONFIG_FILES([src/sflow/Makefile]) +else + build_sflow="no" fi if test "x$build_nfpcapd" = "xyes"; then @@ -655,11 +664,13 @@ echo " LIBS = $LIBS" echo " Enable liblz4 = $use_lz4" echo " Enable libbz2 = $use_bzip2" echo " Enable libzstd = $use_zstd" +echo " Enable ja4 = $build_ja4" echo " Build geolookup = $build_maxmind" echo " Build sflow = $build_sflow" echo " Build nfpcapd = $build_nfpcapd" echo " Build flowtools conv = $build_ftconv" echo " Build nfprofile = $build_nfprofile" +echo " Build ft2nfdump = $build_ftconv" echo "----------------------------------" echo "" echo " You can run ./make now." diff --git a/src/decode/Makefile.am b/src/decode/Makefile.am index d6e53082..4fbb9cd5 100755 --- a/src/decode/Makefile.am +++ b/src/decode/Makefile.am @@ -4,11 +4,16 @@ AM_CFLAGS = -ggdb LDADD = $(DEPS_LIBS) + # libnfdecode sources decode = dns/dns.c dns/dns.h decode += ssl/ssl.c ssl/ssl.h ja3/ja3.c ja3/ja3.h ja4/ja4.c ja4/ja4.h decode += digest/md5.c digest/md5.h digest/sha256.c digest/sha256.h +if JA4 +decode += ja4/ja4s.c ja4/ja4s.h +endif + noinst_LIBRARIES = libnfdecode.a libnfdecode_a_SOURCES = $(decode) \ No newline at end of file diff --git a/src/decode/ja4/Readme.md b/src/decode/ja4/Readme.md new file mode 100644 index 00000000..ebc7f7d3 --- /dev/null +++ b/src/decode/ja4/Readme.md @@ -0,0 +1,16 @@ +# JA4 + +This directory contains the source files for ja4 fingerprint processing. + +See https://github.com/FoxIO-LLC/ja4 for details. + +The usage of ja4 fingerprinting may require a license to use, please see +https://github.com/FoxIO-LLC/ja4#licensing and +https://github.com/FoxIO-LLC/ja4/blob/main/License%20FAQ.md + +In order to use ja4 in nfdump, make sure to run `./configure --enable-ja4 ..` otherwise only the free ja4 client fingerprinting module will be enabled. The fingerprinting applies only on flows with collected payload data. These can be collected with nfpcapd: `nfpcapd -o fat,payload` or any other exporter capable to export payload data, such as **yaf**. + + + + + diff --git a/src/decode/ja4/ja4.c b/src/decode/ja4/ja4.c index 46e202b4..e7cfad08 100644 --- a/src/decode/ja4/ja4.c +++ b/src/decode/ja4/ja4.c @@ -43,17 +43,6 @@ #include "ssl/ssl.h" #include "util.h" -#define CheckStringSize(s, l) \ - { \ - if ((s) < (l)) { \ - LogError("sLen error in %s line %d: %s\n", __FILE__, __LINE__, ""); \ - abort(); \ - return 0; \ - } else { \ - (s) -= (l); \ - } \ - } - #define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) static void sort(uint16Array_t array) { diff --git a/src/decode/ja4/ja4s.c b/src/decode/ja4/ja4s.c new file mode 100644 index 00000000..b50ab8dc --- /dev/null +++ b/src/decode/ja4/ja4s.c @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2024, Peter Haag + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of SWITCH nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "ja4s.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "digest/sha256.h" +#include "ssl/ssl.h" +#include "util.h" + +#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) + +void ja4sPrint(ja4s_t *ja4s) { + printf("SSL/TLS info:\n"); + sslPrint(ja4s->ssl); + + printf("ja4:\n"); + // printf("ja4 : %s_%s_%s\n", ja4s->ja4.a, ja4s->ja4.b, ja4s->ja4.c); + +} // End of ja4Print + +void ja4sFree(ja4s_t *ja4s) { + if (ja4s) free(ja4s); +} // End of ja4Free + +ja4s_t *ja4sProcess(const uint8_t *data, size_t len, uint8_t proto) { + ssl_t *ssl = sslProcess(data, len); + if (!ssl) return NULL; + + ja4s_t *ja4s = calloc(1, sizeof(ja4s_t)); + if (!ja4s) { + LogError("calloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno)); + return NULL; + } + ja4s->ssl = ssl; + + return ja4s; + +} // End of ja4Process + +#ifdef MAIN + +int main(int argc, char **argv) { + const uint8_t tls12[] = { + 0x16, 0x03, // ..X..... + 0x01, 0x02, 0x00, 0x01, 0x00, 0x01, 0xfc, 0x03, // ........ + 0x03, 0xec, 0xb2, 0x69, 0x1a, 0xdd, 0xb2, 0xbf, // ...i.... + 0x6c, 0x59, 0x9c, 0x7a, 0xaa, 0xe2, 0x3d, 0xe5, // lY.z..=. + 0xf4, 0x25, 0x61, 0xcc, 0x04, 0xeb, 0x41, 0x02, // .%a...A. + 0x9a, 0xcc, 0x6f, 0xc0, 0x50, 0xa1, 0x6a, 0xc1, // ..o.P.j. + 0xd2, 0x20, 0x46, 0xf8, 0x61, 0x7b, 0x58, 0x0a, // . F.a{X. + 0xc9, 0x35, 0x8e, 0x2a, 0xa4, 0x4e, 0x30, 0x6d, // .5.*.N0m + 0x52, 0x46, 0x6b, 0xcc, 0x98, 0x9c, 0x87, 0xc8, // RFk..... + 0xca, 0x64, 0x30, 0x9f, 0x5f, 0xaf, 0x50, 0xba, // .d0._.P. + 0x7b, 0x4d, 0x00, 0x22, 0x13, 0x01, 0x13, 0x03, // {M.".... + 0x13, 0x02, 0xc0, 0x2b, 0xc0, 0x2f, 0xcc, 0xa9, // ...+./.. + 0xcc, 0xa8, 0xc0, 0x2c, 0xc0, 0x30, 0xc0, 0x0a, // ...,.0.. + 0xc0, 0x09, 0xc0, 0x13, 0xc0, 0x14, 0x00, 0x9c, // ........ + 0x00, 0x9d, 0x00, 0x2f, 0x00, 0x35, 0x01, 0x00, // .../.5.. + 0x01, 0x91, 0x00, 0x00, 0x00, 0x21, 0x00, 0x1f, // .....!.. + 0x00, 0x00, 0x1c, 0x63, 0x6f, 0x6e, 0x74, 0x69, // ...conti + 0x6c, 0x65, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, // le.servi + 0x63, 0x65, 0x73, 0x2e, 0x6d, 0x6f, 0x7a, 0x69, // ces.mozi + 0x6c, 0x6c, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x00, // lla.com. + 0x17, 0x00, 0x00, 0xff, 0x01, 0x00, 0x01, 0x00, // ........ + 0x00, 0x0a, 0x00, 0x0e, 0x00, 0x0c, 0x00, 0x1d, // ........ + 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, 0x01, 0x00, // ........ + 0x01, 0x01, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, // ........ + 0x00, 0x23, 0x00, 0x00, 0x00, 0x10, 0x00, 0x0e, // .#...... + 0x00, 0x0c, 0x02, 0x68, 0x32, 0x08, 0x68, 0x74, // ...h2.ht + 0x74, 0x70, 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x05, // tp/1.1.. + 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x22, 0x00, 0x0a, 0x00, 0x08, 0x04, 0x03, 0x05, // "....... + 0x03, 0x06, 0x03, 0x02, 0x03, 0x00, 0x33, 0x00, // ......3. + 0x6b, 0x00, 0x69, 0x00, 0x1d, 0x00, 0x20, 0x89, // k.i... . + 0x09, 0x85, 0x8f, 0xbe, 0xb6, 0xed, 0x2f, 0x12, // ....../. + 0x48, 0xba, 0x5b, 0x9e, 0x29, 0x78, 0xbe, 0xad, // H.[.)x.. + 0x0e, 0x84, 0x01, 0x10, 0x19, 0x2c, 0x61, 0xda, // .....,a. + 0xed, 0x00, 0x96, 0x79, 0x8b, 0x18, 0x44, 0x00, // ...y..D. + 0x17, 0x00, 0x41, 0x04, 0x4d, 0x18, 0x3d, 0x91, // ..A.M.=. + 0xf5, 0xee, 0xd3, 0x57, 0x91, 0xfa, 0x98, 0x24, // ...W...$ + 0x64, 0xe3, 0xb0, 0x21, 0x4a, 0xaa, 0x5f, 0x5d, // d..!J._] + 0x1b, 0x78, 0x61, 0x6d, 0x9b, 0x9f, 0xbe, 0xbc, // .xam.... + 0x22, 0xd1, 0x1f, 0x53, 0x5b, 0x2f, 0x94, 0xc6, // "..S[/.. + 0x86, 0x14, 0x31, 0x36, 0xaa, 0x79, 0x5e, 0x6e, // ..16.y^n + 0x5a, 0x87, 0x5d, 0x6c, 0x08, 0x06, 0x4a, 0xd5, // Z.]l..J. + 0xb7, 0x6d, 0x44, 0xca, 0xad, 0x76, 0x6e, 0x24, // .mD..vn$ + 0x83, 0x01, 0x27, 0x48, 0x00, 0x2b, 0x00, 0x05, // ..'H.+.. + 0x04, 0x03, 0x04, 0x03, 0x03, 0x00, 0x0d, 0x00, // ........ + 0x18, 0x00, 0x16, 0x04, 0x03, 0x05, 0x03, 0x06, // ........ + 0x03, 0x08, 0x04, 0x08, 0x05, 0x08, 0x06, 0x04, // ........ + 0x01, 0x05, 0x01, 0x06, 0x01, 0x02, 0x03, 0x02, // ........ + 0x01, 0x00, 0x2d, 0x00, 0x02, 0x01, 0x01, 0x00, // ..-..... + 0x1c, 0x00, 0x02, 0x40, 0x01, 0x00, 0x15, 0x00, // ...@.... + 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // z....... + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ + 0x00, 0x00, 0x00 // ... + }; + /* + [JA4: t13d1715h2_5b57614c22b0_3d5424432f57] + JA4_r: + t13d1715h2_002f,0035,009c,009d,1301,1302,1303,c009,c00a,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0015,0017,001c,0022,0023,002b,002d,0033,ff01_0403,0503,0603,0804,0805,0806,0401,0501,0601,0203,0201] + + ja4 Fullstring: + 771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28-21,29-23-24-25-256-257,0] + [ja4: 579ccef312d18482fc42e2b822ca2430] + */ + // size_t len = sizeof(clientHello); + size_t len = sizeof(tls12); + + ja4_t *ja4 = ja4Process(tls12, len, IPPROTO_TCP); + if (ja4) + ja4Print(ja4); + else + printf("Failed to parse ja4\n"); + + return 0; +} + +#endif \ No newline at end of file diff --git a/src/decode/ja4/ja4s.h b/src/decode/ja4/ja4s.h new file mode 100644 index 00000000..346030a2 --- /dev/null +++ b/src/decode/ja4/ja4s.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2024, Peter Haag + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of SWITCH nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef _jA4S_H +#define _jA4S_H 1 + +#include +#include + +#include "ssl.h" + +typedef struct ja4s_s { + ssl_t *ssl; + struct { + char a[16]; + char b[16]; + char c[16]; + } ja4; +} ja4s_t; + +void ja4sPrint(ja4s_t *ja4s); + +void ja4sFree(ja4s_t *ja4s); + +ja4s_t *ja4sProcess(const uint8_t *data, size_t len, uint8_t proto); + +#endif From 4a606725b785583b9fa19192c02641b4c8022942 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Fri, 1 Mar 2024 14:31:20 +0100 Subject: [PATCH 09/13] Update Readme --- README.md | 3 +++ src/decode/ja4/Readme.md | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 57d07083..d4a2c468 100755 --- a/README.md +++ b/README.md @@ -166,6 +166,9 @@ Build the flow-tools to nfdump converter; default is __NO__ Build nfprofile used by NfSen; default is __NO__ * __--enable-nftrack__ Build nftrack used by PortTracker; default is __NO__ +* **--enable-ja4** + Enable all ja4 module; default is **NO** + See JA4-Fingerprinting [JA4 Fingerprinting](https://github.com/phaag/nfdump/blob/nfdump-ja4/src/decode/ja4/Readme.md) module. Development and beta options diff --git a/src/decode/ja4/Readme.md b/src/decode/ja4/Readme.md index ebc7f7d3..921fd845 100644 --- a/src/decode/ja4/Readme.md +++ b/src/decode/ja4/Readme.md @@ -8,7 +8,9 @@ The usage of ja4 fingerprinting may require a license to use, please see https://github.com/FoxIO-LLC/ja4#licensing and https://github.com/FoxIO-LLC/ja4/blob/main/License%20FAQ.md -In order to use ja4 in nfdump, make sure to run `./configure --enable-ja4 ..` otherwise only the free ja4 client fingerprinting module will be enabled. The fingerprinting applies only on flows with collected payload data. These can be collected with nfpcapd: `nfpcapd -o fat,payload` or any other exporter capable to export payload data, such as **yaf**. +By default only the free **JA4: TLS Client Fingerprinting** module is build into nfdump. If you fulfill the license requirements, you can build all ja4 modules with `./configure --enable-ja4 ..`. + +In general, the fingerprinting applies only on flows with collected payload data. These can be collected with nfpcapd: `nfpcapd -o fat,payload` or any other exporter, which is capable to export payload data, such as **yaf**. From 1e88803a4a547c636c407ff0766a76bc835b1c2b Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Fri, 1 Mar 2024 16:01:07 +0100 Subject: [PATCH 10/13] Implement ja4s --- src/decode/ja4/Readme.md | 2 +- src/decode/ja4/ja4.h | 2 +- src/decode/ja4/ja4s.c | 361 +++++++++++++++++++++++++++++---------- src/decode/ja4/ja4s.h | 24 ++- src/decode/ssl/ssl.c | 4 +- 5 files changed, 298 insertions(+), 95 deletions(-) diff --git a/src/decode/ja4/Readme.md b/src/decode/ja4/Readme.md index 921fd845..7e0ba091 100644 --- a/src/decode/ja4/Readme.md +++ b/src/decode/ja4/Readme.md @@ -10,7 +10,7 @@ https://github.com/FoxIO-LLC/ja4/blob/main/License%20FAQ.md By default only the free **JA4: TLS Client Fingerprinting** module is build into nfdump. If you fulfill the license requirements, you can build all ja4 modules with `./configure --enable-ja4 ..`. -In general, the fingerprinting applies only on flows with collected payload data. These can be collected with nfpcapd: `nfpcapd -o fat,payload` or any other exporter, which is capable to export payload data, such as **yaf**. +In general, the fingerprinting applies only on flows with payload data. These can be collected with nfpcapd: `nfpcapd -o fat,payload` or any other exporter, which is capable to export payload data, such as **yaf**. diff --git a/src/decode/ja4/ja4.h b/src/decode/ja4/ja4.h index 95794836..ee47e340 100644 --- a/src/decode/ja4/ja4.h +++ b/src/decode/ja4/ja4.h @@ -34,7 +34,7 @@ #include #include -#include "ssl.h" +#include "ssl/ssl.h" typedef struct ja4_s { ssl_t *ssl; diff --git a/src/decode/ja4/ja4s.c b/src/decode/ja4/ja4s.c index b50ab8dc..c5d87d95 100644 --- a/src/decode/ja4/ja4s.c +++ b/src/decode/ja4/ja4s.c @@ -45,12 +45,93 @@ #define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) +static int DecodeJA4s(ja4s_t *ja4s, const uint8_t *data, size_t len, uint8_t proto) { + // create ja4s_a + ja4s->a[0] = proto == IPPROTO_TCP ? 't' : 'q'; + ja4s->a[1] = ja4s->ssl->tlsCharVersion[0]; + ja4s->a[2] = ja4s->ssl->tlsCharVersion[1]; + + uint32_t num = LenArray(ja4s->ssl->extensions); + if (num > 99) return 0; + uint32_t ones = num % 10; + uint32_t tens = num / 10; + ja4s->a[3] = tens + '0'; + ja4s->a[4] = ones + '0'; + + if (ja4s->ssl->alpnName[0]) { + // first and last char + ja4s->a[5] = ja4s->ssl->alpnName[0]; + ja4s->a[6] = ja4s->ssl->alpnName[strlen(ja4s->ssl->alpnName) - 1]; + } else { + ja4s->a[5] = '0'; + ja4s->a[6] = '0'; + } + ja4s->a[7] = '\0'; + + // create ja4s_b + char hexChars[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + uint16_t cipher = 0; + uint32_t numCipher = LenArray(ja4s->ssl->cipherSuites); + if (numCipher == 1) { + cipher = ArrayElement(ja4s->ssl->cipherSuites, 0); + } + uint8_t n1 = cipher >> 12; + uint8_t n2 = (cipher >> 8) & 0xF; + uint8_t n3 = (cipher >> 4) & 0xF; + uint8_t n4 = cipher & 0xF; + ja4s->b[0] = hexChars[n1]; + ja4s->b[1] = hexChars[n2]; + ja4s->b[2] = hexChars[n3]; + ja4s->b[3] = hexChars[n4]; + ja4s->b[4] = '\0'; + + // create ja4s_c + // generate string to sha256 + // create a string big enough for ciphersuites and extensions + // uint16_t = max 5 digits + ',' = 6 digits per cipher + '\0' + size_t maxStrLen = LenArray(ja4s->ssl->extensions) * 6 + 1; + char *hashString = (char *)malloc(maxStrLen); + hashString[0] = '0'; + + uint32_t index = 0; + for (int i = 0; i < LenArray(ja4s->ssl->extensions); i++) { + uint16_t val = ArrayElement(ja4s->ssl->extensions, i); + uint8_t n1 = val >> 12; + uint8_t n2 = (val >> 8) & 0xF; + uint8_t n3 = (val >> 4) & 0xF; + uint8_t n4 = val & 0xF; + hashString[index++] = hexChars[n1]; + hashString[index++] = hexChars[n2]; + hashString[index++] = hexChars[n3]; + hashString[index++] = hexChars[n4]; + hashString[index++] = ','; + } + // overwrite last ',' with end of string + hashString[index - 1] = '\0'; + + uint8_t sha256Digest[32]; + char sha256String[65]; + sha256((const unsigned char *)hashString, strlen(hashString), (unsigned char *)sha256Digest); + +#ifdef DEVEL + HexString(sha256Digest, 32, sha256String); + printf("CipherString: %s\n", hashString); + printf(" . Digest: %s\n", sha256String); +#else + HexString(sha256Digest, 6, sha256String); +#endif + + memcpy((void *)ja4s->c, (void *)sha256String, 12); + + return 1; + +} // End of DecodeJA4s + void ja4sPrint(ja4s_t *ja4s) { printf("SSL/TLS info:\n"); sslPrint(ja4s->ssl); - printf("ja4:\n"); - // printf("ja4 : %s_%s_%s\n", ja4s->ja4.a, ja4s->ja4.b, ja4s->ja4.c); + printf("ja4s: %s_%s_%s\n", ja4s->a, ja4s->b, ja4s->c); } // End of ja4Print @@ -69,6 +150,11 @@ ja4s_t *ja4sProcess(const uint8_t *data, size_t len, uint8_t proto) { } ja4s->ssl = ssl; + if (DecodeJA4s(ja4s, data, len, proto) == 0) { + free(ja4s); + return NULL; + } + return ja4s; } // End of ja4Process @@ -76,93 +162,196 @@ ja4s_t *ja4sProcess(const uint8_t *data, size_t len, uint8_t proto) { #ifdef MAIN int main(int argc, char **argv) { - const uint8_t tls12[] = { - 0x16, 0x03, // ..X..... - 0x01, 0x02, 0x00, 0x01, 0x00, 0x01, 0xfc, 0x03, // ........ - 0x03, 0xec, 0xb2, 0x69, 0x1a, 0xdd, 0xb2, 0xbf, // ...i.... - 0x6c, 0x59, 0x9c, 0x7a, 0xaa, 0xe2, 0x3d, 0xe5, // lY.z..=. - 0xf4, 0x25, 0x61, 0xcc, 0x04, 0xeb, 0x41, 0x02, // .%a...A. - 0x9a, 0xcc, 0x6f, 0xc0, 0x50, 0xa1, 0x6a, 0xc1, // ..o.P.j. - 0xd2, 0x20, 0x46, 0xf8, 0x61, 0x7b, 0x58, 0x0a, // . F.a{X. - 0xc9, 0x35, 0x8e, 0x2a, 0xa4, 0x4e, 0x30, 0x6d, // .5.*.N0m - 0x52, 0x46, 0x6b, 0xcc, 0x98, 0x9c, 0x87, 0xc8, // RFk..... - 0xca, 0x64, 0x30, 0x9f, 0x5f, 0xaf, 0x50, 0xba, // .d0._.P. - 0x7b, 0x4d, 0x00, 0x22, 0x13, 0x01, 0x13, 0x03, // {M.".... - 0x13, 0x02, 0xc0, 0x2b, 0xc0, 0x2f, 0xcc, 0xa9, // ...+./.. - 0xcc, 0xa8, 0xc0, 0x2c, 0xc0, 0x30, 0xc0, 0x0a, // ...,.0.. - 0xc0, 0x09, 0xc0, 0x13, 0xc0, 0x14, 0x00, 0x9c, // ........ - 0x00, 0x9d, 0x00, 0x2f, 0x00, 0x35, 0x01, 0x00, // .../.5.. - 0x01, 0x91, 0x00, 0x00, 0x00, 0x21, 0x00, 0x1f, // .....!.. - 0x00, 0x00, 0x1c, 0x63, 0x6f, 0x6e, 0x74, 0x69, // ...conti - 0x6c, 0x65, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, // le.servi - 0x63, 0x65, 0x73, 0x2e, 0x6d, 0x6f, 0x7a, 0x69, // ces.mozi - 0x6c, 0x6c, 0x61, 0x2e, 0x63, 0x6f, 0x6d, 0x00, // lla.com. - 0x17, 0x00, 0x00, 0xff, 0x01, 0x00, 0x01, 0x00, // ........ - 0x00, 0x0a, 0x00, 0x0e, 0x00, 0x0c, 0x00, 0x1d, // ........ - 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, 0x01, 0x00, // ........ - 0x01, 0x01, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, // ........ - 0x00, 0x23, 0x00, 0x00, 0x00, 0x10, 0x00, 0x0e, // .#...... - 0x00, 0x0c, 0x02, 0x68, 0x32, 0x08, 0x68, 0x74, // ...h2.ht - 0x74, 0x70, 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x05, // tp/1.1.. - 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x22, 0x00, 0x0a, 0x00, 0x08, 0x04, 0x03, 0x05, // "....... - 0x03, 0x06, 0x03, 0x02, 0x03, 0x00, 0x33, 0x00, // ......3. - 0x6b, 0x00, 0x69, 0x00, 0x1d, 0x00, 0x20, 0x89, // k.i... . - 0x09, 0x85, 0x8f, 0xbe, 0xb6, 0xed, 0x2f, 0x12, // ....../. - 0x48, 0xba, 0x5b, 0x9e, 0x29, 0x78, 0xbe, 0xad, // H.[.)x.. - 0x0e, 0x84, 0x01, 0x10, 0x19, 0x2c, 0x61, 0xda, // .....,a. - 0xed, 0x00, 0x96, 0x79, 0x8b, 0x18, 0x44, 0x00, // ...y..D. - 0x17, 0x00, 0x41, 0x04, 0x4d, 0x18, 0x3d, 0x91, // ..A.M.=. - 0xf5, 0xee, 0xd3, 0x57, 0x91, 0xfa, 0x98, 0x24, // ...W...$ - 0x64, 0xe3, 0xb0, 0x21, 0x4a, 0xaa, 0x5f, 0x5d, // d..!J._] - 0x1b, 0x78, 0x61, 0x6d, 0x9b, 0x9f, 0xbe, 0xbc, // .xam.... - 0x22, 0xd1, 0x1f, 0x53, 0x5b, 0x2f, 0x94, 0xc6, // "..S[/.. - 0x86, 0x14, 0x31, 0x36, 0xaa, 0x79, 0x5e, 0x6e, // ..16.y^n - 0x5a, 0x87, 0x5d, 0x6c, 0x08, 0x06, 0x4a, 0xd5, // Z.]l..J. - 0xb7, 0x6d, 0x44, 0xca, 0xad, 0x76, 0x6e, 0x24, // .mD..vn$ - 0x83, 0x01, 0x27, 0x48, 0x00, 0x2b, 0x00, 0x05, // ..'H.+.. - 0x04, 0x03, 0x04, 0x03, 0x03, 0x00, 0x0d, 0x00, // ........ - 0x18, 0x00, 0x16, 0x04, 0x03, 0x05, 0x03, 0x06, // ........ - 0x03, 0x08, 0x04, 0x08, 0x05, 0x08, 0x06, 0x04, // ........ - 0x01, 0x05, 0x01, 0x06, 0x01, 0x02, 0x03, 0x02, // ........ - 0x01, 0x00, 0x2d, 0x00, 0x02, 0x01, 0x01, 0x00, // ..-..... - 0x1c, 0x00, 0x02, 0x40, 0x01, 0x00, 0x15, 0x00, // ...@.... - 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // z....... - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ - 0x00, 0x00, 0x00 // ... + static const uint8_t srvHello[] = { + 0x16, 0x03, // ..H..... + 0x03, 0x00, 0x7a, 0x02, 0x00, 0x00, 0x76, 0x03, // ..z...v. + 0x03, 0x89, 0x7c, 0x23, 0x2e, 0x3e, 0xe3, 0x13, // ..|#.>.. + 0x31, 0x4f, 0x2b, 0x66, 0x23, 0x07, 0xff, 0x4f, // 1O+f#..O + 0x7e, 0x2c, 0xf1, 0xca, 0xee, 0xc1, 0xb2, 0x77, // ~,.....w + 0x11, 0xbc, 0xa7, 0x7f, 0x46, 0x95, 0x19, 0x16, // ....F... + 0x85, 0x20, 0xbc, 0x58, 0xb9, 0x2f, 0x86, 0x5e, // . .X./.^ + 0x6b, 0x9a, 0xa4, 0xa6, 0x37, 0x1c, 0xad, 0xcb, // k...7... + 0x0a, 0xfe, 0x1d, 0xa1, 0xc0, 0xf7, 0x05, 0x20, // ....... + 0x9a, 0x11, 0xd5, 0x23, 0x57, 0xf5, 0x6d, 0x5d, // ...#W.m] + 0xd9, 0x62, 0x13, 0x01, 0x00, 0x00, 0x2e, 0x00, // .b...... + 0x33, 0x00, 0x24, 0x00, 0x1d, 0x00, 0x20, 0x76, // 3.$... v + 0xb8, 0xb7, 0xed, 0x0f, 0x96, 0xb6, 0x3a, 0x77, // ......:w + 0x3d, 0x85, 0xab, 0x6f, 0x3a, 0x87, 0xa1, 0x51, // =..o:..Q + 0xc1, 0x30, 0x52, 0x97, 0x85, 0xb4, 0x1a, 0x4d, // .0R....M + 0xef, 0xb5, 0x31, 0x84, 0x05, 0x59, 0x57, 0x00, // ..1..YW. + 0x2b, 0x00, 0x02, 0x03, 0x04, 0x14, 0x03, 0x03, // +....... + 0x00, 0x01, 0x01, 0x17, 0x03, 0x03, 0x12, 0x72, // .......r + 0xde, 0xbd, 0x1a, 0x5c, 0x2b, 0x07, 0xc5, 0xf6, // ...\+... + 0xc5, 0xdd, 0xca, 0xa2, 0x43, 0xa2, 0x8f, 0x00, // ....C... + 0x80, 0xa8, 0xf7, 0x0d, 0xe8, 0x0a, 0xb5, 0x4a, // .......J + 0x4e, 0x89, 0x03, 0xd7, 0x1b, 0xd8, 0x64, 0xcc, // N.....d. + 0x11, 0xac, 0x5e, 0x80, 0xed, 0xf7, 0x2c, 0x5f, // ..^...,_ + 0xa1, 0xb7, 0xb4, 0x5f, 0xee, 0x57, 0x28, 0x7b, // ..._.W({ + 0x32, 0xf3, 0xca, 0x12, 0x5a, 0xb8, 0x87, 0x81, // 2...Z... + 0xe0, 0xfe, 0x86, 0x1a, 0x58, 0x23, 0x35, 0x1b, // ....X#5. + 0xac, 0xbf, 0xe6, 0x96, 0x19, 0x23, 0x71, 0xaa, // .....#q. + 0xf2, 0x0b, 0x04, 0x9f, 0x29, 0x9c, 0x7c, 0x5b, // ....).|[ + 0xc9, 0x58, 0x62, 0xee, 0x8f, 0xa3, 0xcb, 0x6d, // .Xb....m + 0x2a, 0x6c, 0x41, 0x3f, 0x61, 0xda, 0x99, 0x9b, // *lA?a... + 0x90, 0x8a, 0x32, 0x44, 0x25, 0x98, 0x34, 0x03, // ..2D%.4. + 0x29, 0x4a, 0xe0, 0xf6, 0x73, 0x4c, 0x61, 0x1d, // )J..sLa. + 0xce, 0xd7, 0x98, 0x73, 0xd6, 0x66, 0x7d, 0xbd, // ...s.f}. + 0xb7, 0x38, 0x3b, 0x65, 0xa9, 0xa9, 0x77, 0xbd, // .8;e..w. + 0x85, 0x68, 0x11, 0x26, 0x81, 0x6f, 0x85, 0xf0, // .h.&.o.. + 0x93, 0x26, 0xdd, 0xfd, 0xad, 0xf6, 0xce, 0xfe, // .&...... + 0xf8, 0x63, 0x37, 0xfb, 0xf4, 0xab, 0xb4, 0xef, // .c7..... + 0xd4, 0xc5, 0x2c, 0x24, 0x01, 0x69, 0x94, 0xd9, // ..,$.i.. + 0xe9, 0x2d, 0x9b, 0x0d, 0x8d, 0x45, 0x99, 0x55, // .-...E.U + 0xad, 0xcf, 0x7e, 0xf8, 0xc6, 0xb5, 0xb6, 0x32, // ..~....2 + 0x49, 0x31, 0x13, 0x10, 0x11, 0x0f, 0xdf, 0xe8, // I1...... + 0x8d, 0x24, 0x31, 0x11, 0x19, 0xa5, 0x93, 0xe1, // .$1..... + 0xf8, 0x7f, 0x08, 0x11, 0xbc, 0x6d, 0x14, 0x67, // .....m.g + 0x77, 0x74, 0xdf, 0xf9, 0xcb, 0x77, 0x99, 0x04, // wt...w.. + 0xe4, 0xef, 0x18, 0x07, 0x64, 0x23, 0x54, 0x1f, // ....d#T. + 0x75, 0x92, 0xa0, 0xce, 0x60, 0x8f, 0x60, 0x94, // u...`.`. + 0x4e, 0x6a, 0xde, 0xac, 0xf9, 0x08, 0x22, 0x48, // Nj...."H + 0xdb, 0x83, 0xb1, 0x9c, 0x3e, 0x80, 0x44, 0xc4, // ....>.D. + 0x84, 0x68, 0xe6, 0x65, 0xec, 0x94, 0x3f, 0x09, // .h.e..?. + 0x43, 0x59, 0xd2, 0x65, 0x5a, 0x98, 0x93, 0xad, // CY.eZ... + 0x56, 0xdc, 0x31, 0x41, 0xb5, 0x7c, 0x11, 0xed, // V.1A.|.. + 0x4f, 0x32, 0xaa, 0x39, 0x04, 0xcd, 0xb5, 0xe7, // O2.9.... + 0xea, 0xd0, 0x4d, 0x82, 0xf8, 0x39, 0x58, 0x48, // ..M..9XH + 0xc7, 0xb4, 0xdd, 0x16, 0xb5, 0x38, 0x0f, 0x42, // .....8.B + 0x02, 0x70, 0xe5, 0xae, 0x0a, 0xe1, 0xbd, 0xf3, // .p...... + 0x5b, 0xcc, 0xae, 0x33, 0xce, 0x98, 0x85, 0xc8, // [..3.... + 0x9e, 0x15, 0xac, 0x06, 0xe4, 0xbe, 0x1d, 0xea, // ........ + 0x3a, 0xac, 0xa3, 0xc9, 0x9a, 0x41, 0x44, 0xf9, // :....AD. + 0x27, 0xea, 0x05, 0x1c, 0x43, 0xeb, 0xdb, 0x9c, // '...C... + 0x74, 0x8e, 0xe4, 0x99, 0x75, 0x02, 0xd4, 0xec, // t...u... + 0x8c, 0x0e, 0x20, 0x81, 0xdc, 0x2d, 0x04, 0x0d, // .. ..-.. + 0xb0, 0xa2, 0x90, 0xec, 0x26, 0xc9, 0xc3, 0x38, // ....&..8 + 0x66, 0x82, 0xc8, 0xea, 0x35, 0x99, 0xae, 0x78, // f...5..x + 0x42, 0x06, 0xbf, 0xc0, 0xf6, 0x46, 0x3a, 0x5b, // B....F:[ + 0x1d, 0x0f, 0x39, 0xf3, 0x1d, 0x17, 0x97, 0xed, // ..9..... + 0x8b, 0xd4, 0x90, 0x34, 0xcf, 0xa2, 0xe6, 0x58, // ...4...X + 0x4d, 0xbc, 0xb6, 0xc4, 0x0c, 0x4a, 0x67, 0x64, // M....Jgd + 0x5c, 0x33, 0x80, 0x19, 0x37, 0x39, 0xbb, 0x6b, // \3..79.k + 0x95, 0x32, 0xab, 0xfb, 0x9e, 0xcf, 0x7b, 0x1d, // .2....{. + 0xe2, 0x3a, 0x46, 0xcb, 0x2c, 0xbe, 0x06, 0xff, // .:F.,... + 0x6e, 0x71, 0x63, 0xb2, 0x4b, 0xb2, 0xf1, 0xcc, // nqc.K... + 0xea, 0xb3, 0xa4, 0x9d, 0xd9, 0x09, 0x07, 0xa0, // ........ + 0xf0, 0x5a, 0x1f, 0xc6, 0x20, 0xcc, 0xb8, 0x34, // .Z.. ..4 + 0x7d, 0x5e, 0xf6, 0xf9, 0x0d, 0x3d, 0xc4, 0x0c, // }^...=.. + 0x12, 0x88, 0x30, 0x5a, 0xee, 0x31, 0x96, 0x4c, // ..0Z.1.L + 0x2d, 0xde, 0xe6, 0x6c, 0xe9, 0x87, 0xd8, 0xc4, // -..l.... + 0x2f, 0xc1, 0xf6, 0xaa, 0x3c, 0x42, 0x6b, 0xf5, // /...B.z. + 0x1c, 0x13, 0xd6, 0x2b, 0x71, 0xce, 0xbb, 0x91, // ...+q... + 0x75, 0x26, 0x7e, 0x7a, 0xc0, 0x83, 0x1f, 0xfe, // u&~z.... + 0xa7, 0x57, 0x3e, 0x65, 0x5b, 0xcd, 0x27, 0x75, // .W>e[.'u + 0x99, 0xf1, 0x28, 0x2e, 0x48, 0x9b, 0x98, 0xdf, // ..(.H... + 0xb1, 0xe2, 0xca, 0x0a, 0x3f, 0x9e, 0x2c, 0x59, // ....?.,Y + 0xd3, 0xfb, 0x40, 0x3f, 0xf8, 0x80, 0x73, 0x90, // ..@?..s. + 0x68, 0x92, 0xb7, 0x47, 0xc7, 0xf6, 0x04, 0xcb, // h..G.... + 0xef, 0x24, 0x1a, 0x62, 0x86, 0x25, 0x0e, 0xa3, // .$.b.%.. + 0x59, 0x59, 0x9c, 0xf8, 0xa5, 0x1b, 0x07, 0x66, // YY.....f + 0x2c, 0x59, 0x37, 0x15, 0xd1, 0x04, 0x14, 0xbe, // ,Y7..... + 0x65, 0x17, 0x06, 0xa8, 0x9f, 0x77, 0x1e, 0x13, // e....w.. + 0xf5, 0xb5, 0x63, 0x29, 0x88, 0x0a, 0x8e, 0x0d, // ..c).... + 0x24, 0xb4, 0x1b, 0xde, 0xcc, 0x17, 0x44, 0x40, // $.....D@ + 0x46, 0x2b, 0xa5, 0x23, 0x79, 0xf8, 0x1b, 0x2a, // F+.#y..* + 0x0a, 0xce, 0xe1, 0xf5, 0x06, 0xdb, 0xe8, 0x91, // ........ + 0x86, 0x08, 0x3e, 0xa4, 0x1d, 0xc2, 0xfd, 0xc9, // ..>..... + 0xd8, 0x7f, 0xe0, 0x82, 0xcf, 0x24, 0xd5, 0xb2, // .....$.. + 0xd9, 0x27, 0xa2, 0xe5, 0x88, 0xc2, 0xa0, 0x47, // .'.....G + 0xe4, 0xb9, 0xc6, 0x7b, 0xcd, 0x34, 0xfd, 0x8a, // ...{.4.. + 0x3c, 0xed, 0xdd, 0x11, 0x9e, 0xa3, 0xe4, 0x23, // <......# + 0xfc, 0xb6, 0x77, 0x1f, 0x99, 0xab, 0x31, 0x67, // ..w...1g + 0x6e, 0x47, 0x7f, 0xd3, 0xea, 0xf7, 0x81, 0xda, // nG...... + 0x71, 0x12, 0x40, 0x7f, 0x84, 0x6e, 0x23, 0xc0, // q.@..n#. + 0x1d, 0x78, 0xa7, 0xdc, 0x7b, 0x13, 0x58, 0x0c, // .x..{.X. + 0x02, 0x1c, 0xfc, 0x69, 0xe5, 0x96, 0x93, 0xdf, // ...i.... + 0x47, 0x5b, 0x96, 0x85, 0xe1, 0x58, 0xa6, 0xe1, // G[...X.. + 0x4d, 0xdb, 0x85, 0x9d, 0x5d, 0xa1, 0xe5, 0x4c, // M...]..L + 0x8b, 0x9b, 0x70, 0xce, 0x32, 0x27, 0x8c, 0xee, // ..p.2'.. + 0x99, 0xed, 0x86, 0xda, 0x63, 0xad, 0x31, 0x6a, // ....c.1j + 0xc5, 0xc3, 0xef, 0xf6, 0x75, 0x35, 0x30, 0x2e, // ....u50. + 0x67, 0x2d, 0xef, 0xd6, 0xa0, 0xee, 0x36, 0x24, // g-....6$ + 0xfc, 0x16, 0xd2, 0xf6, 0xaf, 0xdd, 0xd2, 0xa3, // ........ + 0xa8, 0xbb, 0x58, 0x40, 0x1e, 0x91, 0x7f, 0xc0, // ..X@.... + 0x49, 0x4d, 0x8b, 0x00, 0xcf, 0xe8, 0xfb, 0x90, // IM...... + 0x90, 0x5b, 0x60, 0xba, 0x0c, 0xc2, 0x97, 0xd3, // .[`..... + 0xb5, 0x7d // .} }; - /* - [JA4: t13d1715h2_5b57614c22b0_3d5424432f57] - JA4_r: - t13d1715h2_002f,0035,009c,009d,1301,1302,1303,c009,c00a,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0015,0017,001c,0022,0023,002b,002d,0033,ff01_0403,0503,0603,0804,0805,0806,0401,0501,0601,0203,0201] - - ja4 Fullstring: - 771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28-21,29-23-24-25-256-257,0] - [ja4: 579ccef312d18482fc42e2b822ca2430] - */ - // size_t len = sizeof(clientHello); - size_t len = sizeof(tls12); - - ja4_t *ja4 = ja4Process(tls12, len, IPPROTO_TCP); - if (ja4) - ja4Print(ja4); + + size_t len = sizeof(srvHello); + + ja4s_t *ja4s = ja4sProcess(srvHello, len, IPPROTO_TCP); + if (ja4s) + ja4sPrint(ja4s); else - printf("Failed to parse ja4\n"); + printf("Failed to parse ja4s\n"); return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/decode/ja4/ja4s.h b/src/decode/ja4/ja4s.h index 346030a2..3f2ee8d1 100644 --- a/src/decode/ja4/ja4s.h +++ b/src/decode/ja4/ja4s.h @@ -34,15 +34,27 @@ #include #include -#include "ssl.h" +#include "ssl/ssl.h" +/* + + ------ Protocol, TCP = "I" QUIC= "q" + | + ---- TLS version, 1.2 = "12", 1.3 = "13" + | | + -- Number of Extensions + | | | +- ALPN Chosen (00 if no ALPN) + | | | | +- Cipher Suite Chosen + | | | | | + | | | | | +- Truncated SHA256 hash of the Extensions, in the order they appear + | | | | | | + JA45=t120400_C030_4e8089608790 + ja4s_a ja4s_b ja4s_c + + +*/ typedef struct ja4s_s { ssl_t *ssl; - struct { - char a[16]; - char b[16]; - char c[16]; - } ja4; + char a[8]; // max 7 chars + char b[8]; // max 4 chars + char c[12]; // max 12 chars } ja4s_t; void ja4sPrint(ja4s_t *ja4s); diff --git a/src/decode/ssl/ssl.c b/src/decode/ssl/ssl.c index d6127e00..7bc29632 100644 --- a/src/decode/ssl/ssl.c +++ b/src/decode/ssl/ssl.c @@ -403,6 +403,7 @@ static int sslParseServerHandshake(ssl_t *ssl, BytesStream_t sslStream, uint32_t dbg_printf("Found extension type: %u, len: %u\n", exType, exLength); AppendArray(ssl->extensions, exType); + if (exLength) ByteStream_SKIP(sslStream, exLength); } dbg_printf("End extension. size: %d\n", sizeLeft); @@ -430,9 +431,10 @@ void sslPrint(ssl_t *ssl) { for (int i = 0; i < LenArray(ssl->signatures); i++) { printf(" 0x%x", ssl->signatures.array[i]); } + printf("\n"); if (ssl->sniName[0]) { - printf("\nSNI name : %s\n", ssl->sniName); + printf("SNI name : %s\n", ssl->sniName); } if (ssl->alpnName[0]) { From 2a55c3bf99b11a60a0d4a9ffa9bec0d621c1ee92 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Sun, 3 Mar 2024 14:38:17 +0100 Subject: [PATCH 11/13] Commit --- src/collector/launch.c | 2 -- src/decode/ssl/ssl.c | 33 +++++++++++++++++++++++++++++++-- src/nfdump/memhandle.c | 20 ++++++++++---------- src/nfdump/nfstat.c | 25 +++++++++++++------------ 4 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/collector/launch.c b/src/collector/launch.c index 7846107d..7efa7589 100644 --- a/src/collector/launch.c +++ b/src/collector/launch.c @@ -65,8 +65,6 @@ #include "nfdump.h" #include "nffile.h" #include "privsep.h" - -#define DEVEL 1 #include "util.h" typedef struct launcher_message_s { diff --git a/src/decode/ssl/ssl.c b/src/decode/ssl/ssl.c index 7bc29632..8e5f5aed 100644 --- a/src/decode/ssl/ssl.c +++ b/src/decode/ssl/ssl.c @@ -193,8 +193,10 @@ static int sslParseExtensions(ssl_t *ssl, BytesStream_t sslStream, uint16_t leng ByteStream_GET_u16(sslStream, exType); ByteStream_GET_u16(sslStream, exLength); dbg_printf("Ex Type: %x, Length: %x\n", exType, exLength); + if (checkGREASE(exType)) { extensionLength -= (4 + exLength); + if (exLength) ByteStream_SKIP(sslStream, exLength); continue; } @@ -398,6 +400,7 @@ static int sslParseServerHandshake(ssl_t *ssl, BytesStream_t sslStream, uint32_t sizeLeft -= (4 + exLength); if (checkGREASE(exType)) { + if (exLength) ByteStream_SKIP(sslStream, exLength); continue; } @@ -583,7 +586,7 @@ ssl_t *sslProcess(const uint8_t *data, size_t len) { #ifdef MAIN void sslTest(void) { - const uint8_t clientHello[] = { + const uint8_t clientHello2[] = { 0x16, 0x03, 0x01, 0x00, 0xc8, 0x01, 0x00, 0x00, 0xc4, 0x03, 0x03, 0xec, 0x12, 0xdd, 0x17, 0x64, 0xa4, 0x39, 0xfd, 0x7e, 0x8c, 0x85, 0x46, 0xb8, 0x4d, 0x1e, 0xa0, 0x6e, 0xb3, 0xd7, 0xa0, 0x51, 0xf0, 0x3c, 0xb8, 0x17, 0x47, 0x0d, 0x4c, 0x54, 0xc5, 0xdf, 0x72, 0x00, 0x00, 0x1c, 0xea, 0xea, 0xc0, 0x2b, 0xc0, 0x2f, 0xc0, 0x2c, 0xc0, 0x30, 0xcc, 0xa9, 0xcc, 0xa8, 0xc0, 0x13, 0xc0, 0x14, 0x00, 0x9c, 0x00, 0x9d, 0x00, @@ -594,6 +597,32 @@ void sslTest(void) { 0x0e, 0x00, 0x0c, 0x02, 0x68, 0x32, 0x08, 0x68, 0x74, 0x74, 0x70, 0x2f, 0x31, 0x2e, 0x31, 0x75, 0x50, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, 0x00, 0x0a, 0x00, 0x0a, 0x00, 0x08, 0x1a, 0x1a, 0x00, 0x1d, 0x00, 0x17, 0x00, 0x18, 0x1a, 0x1a, 0x00, 0x01, 0x00}; + uint8_t clientHello[] = { + 0x16, 0x03, 0x01, 0x02, 0x00, 0x01, 0x00, 0x01, 0xFC, 0x03, 0x03, 0x3C, 0xCA, 0xDD, 0xA8, 0xB0, 0x3F, 0x00, 0xBB, 0xCB, 0x0E, 0x41, 0x8B, + 0xEF, 0x0E, 0xEC, 0x8E, 0xDC, 0x44, 0xDF, 0x52, 0x3A, 0x31, 0x86, 0x8F, 0x72, 0xD1, 0xD1, 0xCC, 0x6F, 0xC1, 0x79, 0x46, 0x20, 0x41, 0xB3, + 0x5E, 0x05, 0x64, 0x48, 0x95, 0x04, 0x84, 0xF5, 0x5B, 0x62, 0xDD, 0xD6, 0x1F, 0xB8, 0xE6, 0x4E, 0x2D, 0xAD, 0xC5, 0xBF, 0x67, 0x16, 0x66, + 0x61, 0x17, 0xDB, 0x27, 0x4F, 0xDC, 0x86, 0x00, 0x2A, 0xDA, 0xDA, 0x13, 0x01, 0x13, 0x02, 0x13, 0x03, 0xC0, 0x2C, 0xC0, 0x2B, 0xCC, 0xA9, + 0xC0, 0x30, 0xC0, 0x2F, 0xCC, 0xA8, 0xC0, 0x0A, 0xC0, 0x09, 0xC0, 0x14, 0xC0, 0x13, 0x00, 0x9D, 0x00, 0x9C, 0x00, 0x35, 0x00, 0x2F, 0xC0, + 0x08, 0xC0, 0x12, 0x00, 0x0A, 0x01, 0x00, 0x01, 0x89, 0x0A, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x18, 0x00, 0x00, 0x15, 0x77, + 0x77, 0x77, 0x2E, 0x6D, 0x61, 0x72, 0x6B, 0x64, 0x6F, 0x77, 0x6E, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2E, 0x6F, 0x72, 0x67, 0x00, 0x17, 0x00, + 0x00, 0xFF, 0x01, 0x00, 0x01, 0x00, 0x00, 0x0A, 0x00, 0x0C, 0x00, 0x0A, 0xCA, 0xCA, 0x00, 0x1D, 0x00, 0x17, 0x00, 0x18, 0x00, 0x19, 0x00, + 0x0B, 0x00, 0x02, 0x01, 0x00, 0x00, 0x10, 0x00, 0x0E, 0x00, 0x0C, 0x02, 0x68, 0x32, 0x08, 0x68, 0x74, 0x74, 0x70, 0x2F, 0x31, 0x2E, 0x31, + 0x00, 0x05, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x18, 0x00, 0x16, 0x04, 0x03, 0x08, 0x04, 0x04, 0x01, 0x05, 0x03, + 0x02, 0x03, 0x08, 0x05, 0x08, 0x05, 0x05, 0x01, 0x08, 0x06, 0x06, 0x01, 0x02, 0x01, 0x00, 0x12, 0x00, 0x00, 0x00, 0x33, 0x00, 0x2B, 0x00, + 0x29, 0xCA, 0xCA, 0x00, 0x01, 0x00, 0x00, 0x1D, 0x00, 0x20, 0x55, 0x8B, 0xA5, 0x3F, 0x92, 0x92, 0xF8, 0x1B, 0xB5, 0xA8, 0xE2, 0xA9, 0xD2, + 0xEF, 0xAF, 0x90, 0x41, 0x69, 0x4E, 0x93, 0xFE, 0x77, 0x62, 0x17, 0x2F, 0xB8, 0x9E, 0x9C, 0xF7, 0x29, 0x1C, 0x4B, 0x00, 0x2D, 0x00, 0x02, + 0x01, 0x01, 0x00, 0x2B, 0x00, 0x0B, 0x0A, 0xDA, 0xDA, 0x03, 0x04, 0x03, 0x03, 0x03, 0x02, 0x03, 0x01, 0x00, 0x1B, 0x00, 0x03, 0x02, 0x00, + 0x01, 0x3A, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x15, 0x00, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x08, 0x00, + }; + size_t len = sizeof(clientHello); ssl_t *ssl = sslProcess(clientHello, len); @@ -608,4 +637,4 @@ int main(int argc, char **argv) { return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/nfdump/memhandle.c b/src/nfdump/memhandle.c index 17f7f4b0..2e6c4953 100755 --- a/src/nfdump/memhandle.c +++ b/src/nfdump/memhandle.c @@ -55,11 +55,11 @@ typedef struct MemHandler_s { size_t BlockSize; /* max size of each pre-allocated memblock */ /* memory blocks - containing the flow records and keys */ - void **memblock; /* array holding all NumBlocks allocated memory blocks */ - size_t MaxBlocks; /* Size of memblock array */ - size_t NumBlocks; /* number of allocated flow blocks in memblock array */ - size_t CurrentBlock; /* Index of current memblock to allocate memory from */ - size_t Allocted; /* Number of bytes already allocated in memblock */ + void **memblock; /* array holding all NumBlocks allocated memory blocks */ + uint32_t MaxBlocks; /* Size of memblock array */ + uint32_t NumBlocks; /* number of allocated flow blocks in memblock array */ + int32_t CurrentBlock; /* Index of current memblock to allocate memory from */ + uint32_t Allocted; /* Number of bytes already allocated in memblock */ atomic_int lock; @@ -93,9 +93,9 @@ static int nfalloc_Init(uint32_t memBlockSize) { } MemHandler->MaxBlocks = MaxMemBlocks; - MemHandler->NumBlocks = 1; - MemHandler->CurrentBlock = 0; - MemHandler->Allocted = 0; + MemHandler->NumBlocks = 0; + MemHandler->CurrentBlock = -1; // non allocated + MemHandler->Allocted = memBlockSize; // force new allocation with next nfmalloc MemHandler->lock = 0; return 1; @@ -109,8 +109,8 @@ static void nfalloc_free(void) { free(MemHandler->memblock[i]); } MemHandler->NumBlocks = 0; - MemHandler->CurrentBlock = 0; - MemHandler->Allocted = 0; + MemHandler->CurrentBlock = -1; + MemHandler->Allocted = MemHandler->BlockSize; free((void *)MemHandler->memblock); MemHandler->memblock = NULL; diff --git a/src/nfdump/nfstat.c b/src/nfdump/nfstat.c index 3b2eae68..728ac8c6 100644 --- a/src/nfdump/nfstat.c +++ b/src/nfdump/nfstat.c @@ -211,6 +211,7 @@ struct StatParameter_s { typedef struct hashkey_s { khint64_t v0; khint64_t v1; + void *ptr; uint8_t proto; } hashkey_t; @@ -567,6 +568,7 @@ void AddElementStat(recordHandle_t *recordHandle) { case 16: { hashkey.v0 = ((uint64_t *)inPtr)[0]; hashkey.v1 = ((uint64_t *)inPtr)[1]; + hashkey.ptr = nfmalloc(64); } break; default: LogError("Invalid stat element size: %d", length); @@ -628,7 +630,17 @@ static void PrintStatLine(stat_record_t *stat, outputParams_t *outputParams, Sta break; case IS_IPADDR: tag_string[0] = outputParams->doTag ? TAG_CHAR : '\0'; - if (StatData->hashkey.v0 != 0) { // IPv6 + if (StatData->hashkey.v0 == 0) { // IPv4 + uint32_t ipv4 = htonl(StatData->hashkey.v1); + if (LoadedGeoDB) { + char ipstr[16], country[4] = {0}; + inet_ntop(AF_INET, &ipv4, ipstr, sizeof(ipstr)); + LookupV4Country(StatData->hashkey.v1, country); + snprintf(valstr, 40, "%s(%s)", ipstr, country); + } else { + inet_ntop(AF_INET, &ipv4, valstr, sizeof(valstr)); + } + } else { // IPv6 uint64_t _key[2] = {htonll(StatData->hashkey.v0), htonll(StatData->hashkey.v1)}; if (LoadedGeoDB) { char ipstr[40], country[4] = {0}; @@ -641,17 +653,6 @@ static void PrintStatLine(stat_record_t *stat, outputParams_t *outputParams, Sta inet_ntop(AF_INET6, _key, valstr, sizeof(valstr)); if (!Getv6Mode()) CondenseV6(valstr); } - - } else { // IPv4 - uint32_t ipv4 = htonl(StatData->hashkey.v1); - if (LoadedGeoDB) { - char ipstr[16], country[4] = {0}; - inet_ntop(AF_INET, &ipv4, ipstr, sizeof(ipstr)); - LookupV4Country(StatData->hashkey.v1, country); - snprintf(valstr, 40, "%s(%s)", ipstr, country); - } else { - inet_ntop(AF_INET, &ipv4, valstr, sizeof(valstr)); - } } break; case IS_MACADDR: { From c9fe29c82925c1ca6357cdef52b2c7e77f6363cf Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Sun, 3 Mar 2024 15:12:39 +0100 Subject: [PATCH 12/13] Prepare nfstat for var length fields such as ja4 --- src/nfdump/nfstat.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/nfdump/nfstat.c b/src/nfdump/nfstat.c index 728ac8c6..53ee731b 100644 --- a/src/nfdump/nfstat.c +++ b/src/nfdump/nfstat.c @@ -209,10 +209,13 @@ struct StatParameter_s { // key for element stat typedef struct hashkey_s { - khint64_t v0; + union { + void *ptr; + khint64_t v0; + }; khint64_t v1; - void *ptr; uint8_t proto; + uint8_t ptrSize; } hashkey_t; // khash record for element stat @@ -279,7 +282,13 @@ static uint32_t NumStats = 0; // number of stats in StatRequest // definitions for khash element stat #define kh_key_hash_func(key) (khint32_t)((key.v1) >> 33 ^ (key.v1) ^ (key.v1) << 11) -#define kh_key_hash_equal(a, b) (((a).v1 == (b).v1) && ((a).v0 == (b).v0) && (a).proto == (b).proto) + +// up to 16 bytes (hashkey.v0, hashkey.v1) use faster compare. +// if > 16 bytes ( ptrSize != 0 ) use memcmp for var length +#define kh_key_hash_equal(a, b) \ + ((a).ptrSize == 0 ? (((a).v1 == (b).v1) && ((a).v0 == (b).v0) && (a).proto == (b).proto) \ + : ((a).ptrSize == (b).ptrSize && memcmp((a).ptr, (b).ptr, (a).ptrSize) == 0)) + KHASH_INIT(ElementHash, hashkey_t, StatRecord_t, 1, kh_key_hash_func, kh_key_hash_equal) static khash_t(ElementHash) * ElementKHash[MaxStats]; @@ -514,7 +523,7 @@ static inline void PreProcess(void *inPtr, preprocess_t process, recordHandle_t } break; case JA3: { EXinPayload_t *payload = (EXinPayload_t *)recordHandle->extensionList[EXinPayloadID]; - if (payload == NULL) return; + if (payload == NULL || recordHandle->ja3[0]) return; uint32_t payloadLength = ExtensionLength(payload); ja3_t *ja3 = ja3Process(payload, payloadLength); if (ja3) { @@ -568,10 +577,13 @@ void AddElementStat(recordHandle_t *recordHandle) { case 16: { hashkey.v0 = ((uint64_t *)inPtr)[0]; hashkey.v1 = ((uint64_t *)inPtr)[1]; - hashkey.ptr = nfmalloc(64); } break; - default: - LogError("Invalid stat element size: %d", length); + default: { + void *p = nfmalloc(length); + hashkey.ptr = p; + memcpy((void *)p, inPtr, length); + hashkey.ptrSize = length; + } } EXcntFlow_t *cntFlow = (EXcntFlow_t *)recordHandle->extensionList[EXcntFlowID]; From 17ead4c6cdd31c489ecd679a36ce1be8344cb4e9 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Sun, 3 Mar 2024 15:21:43 +0100 Subject: [PATCH 13/13] Updeate ChangeLog --- ChangeLog | 27 ++++++++++++++++++++++++--- src/lib/nffile.c | 2 +- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index d5fb0725..209560ac 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,27 @@ -- 466c27b 2024-02-17 (HEAD -> master, origin/master, origin/HEAD) Merge remote-tracking branch 'origin/nfdump-filter-ng' +- c9fe29c 2024-03-03 (HEAD -> nfdump-ja4, origin/nfdump-ja4) Prepare nfstat for var length fields such as ja4 +- 2a55c3b 2024-03-03 Commit +- 1e88803 2024-03-01 Implement ja4s +- 4a60672 2024-03-01 Update Readme +- c129b78 2024-03-01 Add ja4 Readme for license issue and add --enable-ja4, default no for building all ja4 modules +- 5c72cc8 2024-03-01 Add ja3 filter in nfdump-ja4 +- 8ed9f34 2024-03-01 Add ja4_c +- 2ded6b5 2024-02-29 Implement ja_a and ja_b of ja +- ae92ef6 2024-02-29 Add ja4 files to start implementation +- 16148ed 2024-02-29 Add sha256 +- a4f6a53 2024-02-26 Refrag ssl extension code +- 6ea1b58 2024-02-25 Add ALPN extension decoding to ssl.c +- 4115a8b 2024-02-25 Fix missing malloc() type casting +- 07136b3 2024-02-25 Use new ssl module for ja3 +- 17d127e 2024-02-24 Debug ssl code - Client Hello +- 79e9b8a 2024-02-24 Defrag ssl and ja3 - Add plain ssl code +- 00e0e45 2024-02-24 (origin/master, origin/HEAD) Fix nfreader code +- eb5bef5 2024-02-22 Cleanup code to handle legacy sampler record +- a988a40 2024-02-18 Rework maxmind code +- 9a0dddf 2024-02-17 Fix library dependencies on some *nix +- 2c987ee 2024-02-17 The previous commit 466c27b is a merge with a working branch to remove the clumsy master_record_t in nfdump. This removes lot of legacy code and updates the code base. Furthermore, the filter engin has been replaced by a more flexible and faster one. The speed gain depends on the filter and the number of filter elements. The overall speed gain with this merge is 10% in average. Further improvements are planned with more worker threats. Please note that the filter syntax had some small changes such as 'icmp-type' is now 'icmp type' etc. +- 466c27b 2024-02-17 Merge remote-tracking branch 'origin/nfdump-filter-ng' - bc08a0a 2024-02-17 Sync lz4 code -- 4f409e9 2024-02-17 Release v1.7.4 +- 4f409e9 2024-02-17 (tag: v1.7.4) Release v1.7.4 - 4a7de41 2024-02-17 Fix compiler warnings for lz4 - 4e98a35 2024-02-17 Update lz4 code - 519a052 2024-02-16 Fix compile issues @@ -35,7 +56,7 @@ - 627565a 2024-01-29 Remove local m4 files - f0d2750 2024-01-29 Fix err var in nfprofile - dc1b1b2 2024-01-29 Fix #503 - Append records in nfprofile -- 1dcb7ae 2024-01-28 (origin/nfdump-filter-ng) Fix fmt time 1970 +- 1dcb7ae 2024-01-28 Fix fmt time 1970 - c5ddbf0 2024-01-28 Fix fmt cores - 427b08e 2024-01-28 Removed master record from nfgen. Update make check - bbaef64 2024-01-26 master record removed from 1.7.x code diff --git a/src/lib/nffile.c b/src/lib/nffile.c index 1e059144..1b3998cc 100644 --- a/src/lib/nffile.c +++ b/src/lib/nffile.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2023, Peter Haag + * Copyright (c) 2004-2024, Peter Haag * All rights reserved. * * Redistribution and use in source and binary forms, with or without