diff --git a/common/src/components/Scraper.jsx b/common/src/components/Scraper.jsx
index 52f89370..dace74a4 100644
--- a/common/src/components/Scraper.jsx
+++ b/common/src/components/Scraper.jsx
@@ -8,86 +8,93 @@ import Fields from './Fields'
const customScraperFields = [
{
- "field": "id",
- "description": "A deterministic or natural id for the resource",
- "scheme": "JSONPathOrString",
- "required": true
+ field: 'id',
+ description: 'A deterministic or natural id for the resource',
+ scheme: 'JSONPathOrString',
+ required: true
},
{
- "field": "name",
- default: "`id`",
- "scheme": "JSONPathOrString"
+ field: 'name',
+ default: '`id`',
+ scheme: 'JSONPathOrString'
},
{
- "field": "items",
- "description": "Extract multiple config items from this array",
- "scheme": "JSONPath"
+ field: 'items',
+ description: 'Extract multiple config items from this array',
+ scheme: 'JSONPath'
},
{
- "field": "type",
- "description": "e.g. `File::Host`, `File::Tomcat`, `File::Pom`",
- "scheme": "JSONPathOrString",
- "required": "true"
+ field: 'type',
+ description: 'e.g. `File::Host`, `File::Tomcat`, `File::Pom`',
+ scheme: 'JSONPathOrString',
+ required: 'true'
},
{
- "field": "class",
- "scheme": "JSONPathOrString"
+ field: 'class',
+ scheme: 'JSONPathOrString'
},
{
- "field": "format",
- "description": "Format of config item e.g. `xml`, `properties`",
- "default": "`JSON`",
- "scheme": "string"
+ field: 'format',
+ description: 'Format of config item e.g. `xml`, `properties`',
+ default: '`JSON`',
+ scheme: 'string'
},
{
- "field": "timestampFormat",
- "description": "Format to parse timestamps in `createFields` and `deletedFields`",
- "default": "RFC3339",
- "scheme": "Go time format"
+ field: 'timestampFormat',
+ description:
+ 'Format to parse timestamps in `createFields` and `deletedFields`',
+ default: 'RFC3339',
+ scheme: 'Go time format'
},
{
- "field": "createFields",
- "description": "Identify the created time for a resource (if different to scrape time). If multiple fields are specified, the first non-empty value will be used",
- "scheme": "[]JSONPathOrString"
+ field: 'createFields',
+ description:
+ 'Identify the created time for a resource (if different to scrape time). If multiple fields are specified, the first non-empty value will be used',
+ scheme: '[]JSONPathOrString'
},
{
- "field": "deleteFields",
- "description": "Identify when a config item was deleted. If multiple fields are specified, the first non-empty value will be used",
- "scheme": "[]JSONPathOrString"
- },
+ field: 'deleteFields',
+ description:
+ 'Identify when a config item was deleted. If multiple fields are specified, the first non-empty value will be used',
+ scheme: '[]JSONPathOrString'
+ }
]
const commonsRows = [
-
{
- "field": "ignore",
- "description": "Fields to ignore and strip out of the config",
- "scheme": "[]JSONPath"
+ field: 'transform',
+ description: "Transform configs after they've been scraped",
+ scheme: '[`Transform`](/config-db/concepts/transform)'
},
{
- "field": "properties",
- "description": "Custom templatable properties for the scraped config items.",
- "scheme": "[`[]ConfigProperty`](/reference/config-db/properties)"
+ field: 'properties',
+ description: 'Custom templatable properties for the scraped config items.',
+ scheme: '[`[]ConfigProperty`](/reference/config-db/properties)'
},
{
- "field": "tags",
- "description": "Additional tags ",
- "scheme": "map[string]string"
+ field: 'labels',
+ description: 'Labels for each config item.',
+ scheme: '`map[string]string`'
},
+ {
+ field: 'tags',
+ description: 'Tags for each config item. Max allowed: 5',
+ scheme: '[`[]ConfigTag`](/config-db/concepts/tags)'
+ }
]
export function Scraper({ name, edition, rows, ...props }) {
-
const { siteConfig } = useDocusaurusContext()
-
-
return
-
}
export function CustomScraper({ name, edition, rows, ...props }) {
-
const { siteConfig } = useDocusaurusContext()
- return
-
+ return (
+
+ )
}
diff --git a/mission-control/docs/config-db/scrapers/kubernetes.mdx b/mission-control/docs/config-db/scrapers/kubernetes.mdx
index 32277cb8..8c30551c 100644
--- a/mission-control/docs/config-db/scrapers/kubernetes.mdx
+++ b/mission-control/docs/config-db/scrapers/kubernetes.mdx
@@ -17,32 +17,102 @@ The `kubernetes` scraper collects all of the resources and events in a Kubernete
```
-| Field | Description | Scheme | Required |
-| ------------ | ---------------------------------------------------------------------------- | -------------------------------------------- | -------- |
-| `logLevel` | Specify the level of logging. | `string` | |
-| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | |
-| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | |
-| `kubernetes` | Specifies the list of Kubernetes configurations to scrape. | [`[]Kubernetes`](#kubernetes) | |
-
-### Kubernetes
-
-| Field | Description | Scheme | Required |
-| ---------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------- | -------- |
-| `clusterName` | Specify cluster name | `string` | |
-| `event` | Specify configuration to handle Kubernetes events. | [`Event`](#events) | |
-| `exclusions` | Resources to be excluded from scraping | `[]string` | |
-| `fieldSelector` | Resources to be included e.g `status.Phase=Running` | `string` | |
-| **`kubeconfig`** | Kubeconfig to connect to the cluster | []_EnvVar_ | |
-| `namespace` | Include resources only from this namespace | `string` | |
-| `relationships` | Create relationships between kubernetes objects. | [`[]Relationship`](#relationships) | |
-| `scope` | Specify scope for scrape. e.g `cluster` for scraping at Cluster level | `string` | |
-| `selector` | Include resources matching this selector only e.g `matchLabels` | `string` | |
-| `since` | Set time constraint for scraping resources within the set period | `string` | |
-| `properties` | Custom properties to be added for each item | [`[]ConfigProperty`](/reference/config-db/properties) | |
-| `transform` | Custom transformations to apply | [`Transform`](/config-db/concepts/transform) | |
-| `tags` | Tags to set on each config item. `cluster` and `namespace` are set by default | `map[string]string` | |
-
-## Events
+| Field | Description | Scheme |
+| ------------ | ---------------------------------------------------------------------------- | -------------------------------------------- |
+| `logLevel` | Specify the level of logging. | `string` |
+| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` |
+| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) |
+| `kubernetes` | Specifies the list of Kubernetes configurations to scrape. | [`[]Kubernetes`](#kubernetes) |
+
+## Kubernetes
+
+
+
+### Watch Events & Resources
+
+Use property `watch.disable` to disable watching events & resources.
+
+#### Events
`Kubernetes::Event` resources are mapped to config changes. Events can be very verbose so they can be excluded or their severity level changed:
@@ -51,16 +121,35 @@ The `kubernetes` scraper collects all of the resources and events in a Kubernete
| `exclusions` | A list of keywords used to exclude event objects based on the reason | `[]string` | |
| `severityKeywords` | Specify keywords used to identify the severity of the Kubernetes Event based on the reason | [`SeverityKeywords`](#severitykeywords) | |
-### SeverityKeywords
+##### SeverityKeywords
| Field | Description | Scheme | Required |
| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------- | -------- |
| `warn` | A list of keywords used to identify a warning severity from the reason. It could also be a match pattern: e.g. `*` to match all or `!badword` to exclude `badword` | `[]string` | |
| `error` | Same as `warn` but used to map to error severity. | `[]string` | |
-## Relationships
+#### Watch Selector
+
+
-You can create relationships between kubernetes objects on the basis of
+### Relationships
+
+You can create relationships between kubernetes objects on the basis of kind, name & namespace.
:::info
[Relationships](../concepts/relationships) can also be defined under `transform.relationships`, however defining them under `kubernetes.relationships` is simpler with specific support for `kind`, `name` and `namespace` fields.
@@ -101,7 +190,7 @@ kubernetes:
| `name` | `name` of Kubernetes Object | [`Lookup`](#lookup) | `true` |
| `namespace` | `namespace` of Kubernetes Object | [`Lookup`](#lookup) | `true` |
-##### Lookup
+###### Lookup
There are 3 different ways to specify which value to use when finding related configs:
@@ -111,6 +200,35 @@ There are 3 different ways to specify which value to use when finding related co
| `value` | Specify a static value | `string` | |
| `label` | Get the value from a label | `string` | |
+### Exclusion
+
+excludes certain kubernetes objects from being scraped
+
+
+
## Annotations
Kubernetes resources can be annotated with annotations that can direct the scraper to certain behaviors.
@@ -171,3 +289,78 @@ type: Opaque
data:
token: ...
```
+
+## Scraping remote clusters
+
+A single config-db instance can scrape multiple clusters when provided with a kubeconfig. Either the kubeconfig itself or the path to the kubeconfig can be provided.
+
+### Local path to kubeconfig
+
+```yaml title="remote-cluster.yaml"
+apiVersion: configs.flanksource.com/v1
+kind: ScrapeConfig
+metadata:
+ name: azure-scraper
+spec:
+ schedule: '@every 5h'
+ kubernetes:
+ - clusterName: 'azure production cluster'
+ kubeconfig:
+ value: /home/flanksource/.kube/azure_config
+```
+
+### Kubeconfig
+
+```yaml title="remote-cluster.yaml"
+apiVersion: configs.flanksource.com/v1
+kind: ScrapeConfig
+metadata:
+ name: aws-scraper
+spec:
+spec:
+ schedule: '@every 5h'
+ kubernetes:
+ - clusterName: 'aws cluster'
+ kubeconfig:
+ value: |
+ apiVersion: v1
+ clusters:
+ - cluster:
+ certificate-authority-data: xxxxx
+ server: https://xxxxx.sk1.eu-west-1.eks.amazonaws.com
+ name: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster
+ contexts:
+ - context:
+ cluster: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster
+ namespace: mission-control
+ user: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster
+ name: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster
+ current-context: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster
+ kind: Config
+ preferences: {}
+ users:
+ - name: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster
+ user:
+ exec:
+ ....
+
+```
+
+or, a kubeconfig inside a secret can be referenced as follows:
+
+```yaml title="remote-cluster.yaml"
+apiVersion: configs.flanksource.com/v1
+kind: ScrapeConfig
+metadata:
+ name: aws-scraper
+spec:
+spec:
+ schedule: '@every 5h'
+ kubernetes:
+ - clusterName: 'aws cluster'
+ kubeconfig:
+ valueFrom:
+ secretKeyRef:
+ name: aws-kubeconfig
+ key: kubeconfig
+```