feat: Frontend support for file_format in S3 batch export (#21076)

* refactor: Support for multiple file formats in batch exports * refactor: Prefer composition over inheritance * refactor: More clearly separate writer from temporary file We now should be more explicit about what is the context in which the batch export temporary file is alive. The writer outlives this context, so it can be used by callers to, for example, check how many records were written. * test: More parquet testing * refactor: Move temporary file to new module * feat: Frontend support for file_format * fix: Remove redefinition caused by rebase
PostHog · Mar 22, 2024 · fcc192e · fcc192e
1 parent 7c0258c
commit fcc192e
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 0 deletions.
diff --git a/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx b/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx
@@ -242,6 +242,15 @@ export function BatchExportsEditFields({
                                     ]}
                                 />
                             </LemonField>
+
+                            <LemonField name="file_format" label="Format" className="flex-1">
+                                <LemonSelect
+                                    options={[
+                                        { value: 'JSONLines', label: 'JSON lines' },
+                                        { value: 'Parquet', label: 'Apache Parquet' },
+                                    ]}
+                                />
+                            </LemonField>
                         </div>
 
                         <div className="flex gap-4">

diff --git a/frontend/src/scenes/batch_exports/BatchExports.stories.tsx b/frontend/src/scenes/batch_exports/BatchExports.stories.tsx
@@ -42,6 +42,7 @@ export default {
                             include_events: [],
                             encryption: null,
                             kms_key_id: null,
+                            file_format: 'JSONLines',
                         },
                     },
                     start_at: null,

diff --git a/frontend/src/scenes/batch_exports/batchExportEditLogic.ts b/frontend/src/scenes/batch_exports/batchExportEditLogic.ts
@@ -90,6 +90,7 @@ export const batchExportFormFields = (
                   aws_secret_access_key: isNew ? (!config.aws_secret_access_key ? 'This field is required' : '') : '',
                   compression: '',
                   encryption: '',
+                  file_format: isNew ? (!config.file_format ? 'This field is required' : '') : '',
                   kms_key_id: !config.kms_key_id && config.encryption == 'aws:kms' ? 'This field is required' : '',
                   exclude_events: '',
                   include_events: '',

diff --git a/frontend/src/types.ts b/frontend/src/types.ts
@@ -3597,6 +3597,7 @@ export type BatchExportDestinationS3 = {
         encryption: string | null
         kms_key_id: string | null
         endpoint_url: string | null
+        file_format: string
     }
 }