Added new hash fields (elastic#1678)

* Added new hash fields * Generated new artifacts * Added changelog entry * Moved pehash to pe schema * Generated new artifact after pehash * Updated changelog
kgeller · Dec 1, 2021 · 889cbd4 · 889cbd4
1 parent c84e5a7
commit 889cbd4
Show file tree

Hide file tree

Showing 23 changed files with 1,772 additions and 0 deletions.
diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md
@@ -57,6 +57,7 @@ Thanks, you're awesome :-) -->
 #### Added
 
 * Added `faas.*` field set as beta. #1628
+* Added two new fields (sha384,tlsh) to hash schema and one field to pe schema (pehash). #1678
 
 #### Improvements
 

diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc
@@ -4057,6 +4057,22 @@ type: keyword
 
 
 
+| extended
+
+// ===============================================================
+
+|
+[[field-hash-sha384]]
+<<field-hash-sha384, hash.sha384>>
+
+| SHA384 hash.
+
+type: keyword
+
+
+
+
+
 | extended
 
 // ===============================================================
@@ -4089,6 +4105,22 @@ type: keyword
 
 
 
+| extended
+
+// ===============================================================
+
+|
+[[field-hash-tlsh]]
+<<field-hash-tlsh, hash.tlsh>>
+
+| TLSH hash.
+
+type: keyword
+
+
+
+
+
 | extended
 
 // ===============================================================
@@ -6287,6 +6319,24 @@ example: `MSPAINT.EXE`
 
 // ===============================================================
 
+|
+[[field-pe-pehash]]
+<<field-pe-pehash, pe.pehash>>
+
+| A hash of the PE header and data from one or more PE sections. An pehash can be used to cluster files by transforming structural information about a file into a hash value.
+
+Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.
+
+type: keyword
+
+
+
+example: `73ff189b63cd6be375a7ff25179a38d347651975`
+
+| extended
+
+// ===============================================================
+
 |
 [[field-pe-product]]
 <<field-pe-product, pe.product>>

diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml
@@ -1387,6 +1387,12 @@
       ignore_above: 1024
       description: SHA256 hash.
       default_field: false
+    - name: hash.sha384
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA384 hash.
+      default_field: false
     - name: hash.sha512
       level: extended
       type: keyword
@@ -1399,6 +1405,12 @@
       ignore_above: 1024
       description: SSDEEP hash.
       default_field: false
+    - name: hash.tlsh
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: TLSH hash.
+      default_field: false
     - name: name
       level: core
       type: keyword
@@ -1461,6 +1473,17 @@
       description: Internal name of the file, provided at compile-time.
       example: MSPAINT.EXE
       default_field: false
+    - name: pe.pehash
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'A hash of the PE header and data from one or more PE sections.
+        An pehash can be used to cluster files by transforming structural information
+        about a file into a hash value.
+
+        Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.'
+      example: 73ff189b63cd6be375a7ff25179a38d347651975
+      default_field: false
     - name: pe.product
       level: extended
       type: keyword
@@ -2798,6 +2821,12 @@
       type: keyword
       ignore_above: 1024
       description: SHA256 hash.
+    - name: hash.sha384
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA384 hash.
+      default_field: false
     - name: hash.sha512
       level: extended
       type: keyword
@@ -2809,6 +2838,12 @@
       ignore_above: 1024
       description: SSDEEP hash.
       default_field: false
+    - name: hash.tlsh
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: TLSH hash.
+      default_field: false
     - name: inode
       level: extended
       type: keyword
@@ -2903,6 +2938,17 @@
       description: Internal name of the file, provided at compile-time.
       example: MSPAINT.EXE
       default_field: false
+    - name: pe.pehash
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'A hash of the PE header and data from one or more PE sections.
+        An pehash can be used to cluster files by transforming structural information
+        about a file into a hash value.
+
+        Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.'
+      example: 73ff189b63cd6be375a7ff25179a38d347651975
+      default_field: false
     - name: pe.product
       level: extended
       type: keyword
@@ -3254,6 +3300,12 @@
       type: keyword
       ignore_above: 1024
       description: SHA256 hash.
+    - name: sha384
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA384 hash.
+      default_field: false
     - name: sha512
       level: extended
       type: keyword
@@ -3265,6 +3317,12 @@
       ignore_above: 1024
       description: SSDEEP hash.
       default_field: false
+    - name: tlsh
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: TLSH hash.
+      default_field: false
   - name: host
     title: Host
     group: 2
@@ -4580,6 +4638,17 @@
       description: Internal name of the file, provided at compile-time.
       example: MSPAINT.EXE
       default_field: false
+    - name: pehash
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'A hash of the PE header and data from one or more PE sections.
+        An pehash can be used to cluster files by transforming structural information
+        about a file into a hash value.
+
+        Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.'
+      example: 73ff189b63cd6be375a7ff25179a38d347651975
+      default_field: false
     - name: product
       level: extended
       type: keyword
@@ -4941,6 +5010,12 @@
       type: keyword
       ignore_above: 1024
       description: SHA256 hash.
+    - name: hash.sha384
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA384 hash.
+      default_field: false
     - name: hash.sha512
       level: extended
       type: keyword
@@ -4952,6 +5027,12 @@
       ignore_above: 1024
       description: SSDEEP hash.
       default_field: false
+    - name: hash.tlsh
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: TLSH hash.
+      default_field: false
     - name: name
       level: extended
       type: keyword
@@ -5311,6 +5392,12 @@
       ignore_above: 1024
       description: SHA256 hash.
       default_field: false
+    - name: parent.hash.sha384
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA384 hash.
+      default_field: false
     - name: parent.hash.sha512
       level: extended
       type: keyword
@@ -5323,6 +5410,12 @@
       ignore_above: 1024
       description: SSDEEP hash.
       default_field: false
+    - name: parent.hash.tlsh
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: TLSH hash.
+      default_field: false
     - name: parent.name
       level: extended
       type: keyword
@@ -5381,6 +5474,17 @@
       description: Internal name of the file, provided at compile-time.
       example: MSPAINT.EXE
       default_field: false
+    - name: parent.pe.pehash
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'A hash of the PE header and data from one or more PE sections.
+        An pehash can be used to cluster files by transforming structural information
+        about a file into a hash value.
+
+        Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.'
+      example: 73ff189b63cd6be375a7ff25179a38d347651975
+      default_field: false
     - name: parent.pe.product
       level: extended
       type: keyword
@@ -5495,6 +5599,17 @@
       description: Internal name of the file, provided at compile-time.
       example: MSPAINT.EXE
       default_field: false
+    - name: pe.pehash
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'A hash of the PE header and data from one or more PE sections.
+        An pehash can be used to cluster files by transforming structural information
+        about a file into a hash value.
+
+        Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.'
+      example: 73ff189b63cd6be375a7ff25179a38d347651975
+      default_field: false
     - name: pe.product
       level: extended
       type: keyword
@@ -7104,6 +7219,12 @@
       ignore_above: 1024
       description: SHA256 hash.
       default_field: false
+    - name: enrichments.indicator.file.hash.sha384
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA384 hash.
+      default_field: false
     - name: enrichments.indicator.file.hash.sha512
       level: extended
       type: keyword
@@ -7116,6 +7237,12 @@
       ignore_above: 1024
       description: SSDEEP hash.
       default_field: false
+    - name: enrichments.indicator.file.hash.tlsh
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: TLSH hash.
+      default_field: false
     - name: enrichments.indicator.file.inode
       level: extended
       type: keyword
@@ -7215,6 +7342,17 @@
       description: Internal name of the file, provided at compile-time.
       example: MSPAINT.EXE
       default_field: false
+    - name: enrichments.indicator.file.pe.pehash
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'A hash of the PE header and data from one or more PE sections.
+        An pehash can be used to cluster files by transforming structural information
+        about a file into a hash value.
+
+        Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.'
+      example: 73ff189b63cd6be375a7ff25179a38d347651975
+      default_field: false
     - name: enrichments.indicator.file.pe.product
       level: extended
       type: keyword
@@ -8477,6 +8615,12 @@
       ignore_above: 1024
       description: SHA256 hash.
       default_field: false
+    - name: indicator.file.hash.sha384
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA384 hash.
+      default_field: false
     - name: indicator.file.hash.sha512
       level: extended
       type: keyword
@@ -8489,6 +8633,12 @@
       ignore_above: 1024
       description: SSDEEP hash.
       default_field: false
+    - name: indicator.file.hash.tlsh
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: TLSH hash.
+      default_field: false
     - name: indicator.file.inode
       level: extended
       type: keyword
@@ -8588,6 +8738,17 @@
       description: Internal name of the file, provided at compile-time.
       example: MSPAINT.EXE
       default_field: false
+    - name: indicator.file.pe.pehash
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'A hash of the PE header and data from one or more PE sections.
+        An pehash can be used to cluster files by transforming structural information
+        about a file into a hash value.
+
+        Learn more at https://www.usenix.org/legacy/events/leet09/tech/full_papers/wicherski/wicherski_html/index.html.'
+      example: 73ff189b63cd6be375a7ff25179a38d347651975
+      default_field: false
     - name: indicator.file.pe.product
       level: extended
       type: keyword