From 68117a6390609538a0ff6dc96b45166d88706ec2 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 20 Jul 2023 13:09:33 -0400 Subject: [PATCH] Disable unused input types for policy_template (#11) Streams that are enabled by default need to be disabled. Previously we disabled streams for the input type that was being used. But streams under different input types that existed in the policy template were omitted so they were enabled. This adds an explicit enabled=false for all unused inputs. Fixes #7 --- .../internal/module/generate.go | 6 ++++++ fleet_input/cel/module.tf.json | 1 + fleet_input/log/module.tf.json | 1 + fleet_input/sql/module.tf.json | 1 + .../aws.cloudtrail.aws-s3/module.tf.json | 5 +++++ .../module.tf.json | 3 +++ .../github.issues.httpjson/module.tf.json | 3 +++ .../module.tf.json | 3 +++ .../google_workspace.admin.httpjson/module.tf.json | 3 +++ .../google_workspace.alert.httpjson/module.tf.json | 3 +++ .../module.tf.json | 3 +++ .../module.tf.json | 3 +++ .../google_workspace.drive.httpjson/module.tf.json | 3 +++ .../google_workspace.gcp.httpjson/module.tf.json | 3 +++ .../module.tf.json | 3 +++ .../module.tf.json | 3 +++ .../google_workspace.login.httpjson/module.tf.json | 3 +++ .../google_workspace.rules.httpjson/module.tf.json | 3 +++ .../google_workspace.saml.httpjson/module.tf.json | 3 +++ .../google_workspace.token.httpjson/module.tf.json | 3 +++ .../module.tf.json | 3 +++ .../system.application.winlog/module.tf.json | 6 ++++++ .../system.security.winlog/module.tf.json | 6 ++++++ .../ti_abusech.malware.httpjson/module.tf.json | 3 +++ .../module.tf.json | 3 +++ .../ti_abusech.threatfox.httpjson/module.tf.json | 3 +++ .../ti_abusech.url.httpjson/module.tf.json | 3 +++ .../module.tf.json | 4 ++++ .../windows.powershell.winlog/module.tf.json | 5 +++++ .../module.tf.json | 5 +++++ .../module.tf.json | 5 +++++ .../winlog.winlogs.winlog.winlog/module.tf.json | 4 ++++ fleet_package_policy/README.md | 3 ++- fleet_package_policy/main.tf | 14 +++++++++++--- fleet_package_policy/variables.tf | 8 +++++++- 35 files changed, 129 insertions(+), 5 deletions(-) diff --git a/fleet-terraform-generator/internal/module/generate.go b/fleet-terraform-generator/internal/module/generate.go index 5d0662c..2fcd2a8 100644 --- a/fleet-terraform-generator/internal/module/generate.go +++ b/fleet-terraform-generator/internal/module/generate.go @@ -206,6 +206,10 @@ func Generate(path, policyTemplateName, dataStreamName, inputName string, ignore } sort.Strings(dataStreamsForInput) } + inputTypesForPolicyTemplate := []string{} // Declare empty slice. + for _, input := range policyTemplate.Inputs { + inputTypesForPolicyTemplate = append(inputTypesForPolicyTemplate, input.Type) + } packagePolicyName := manifest.Name + "-" + dataStreamName + "-${var.fleet_data_stream_namespace}${var.fleet_package_policy_name_suffix}" if dataStreamName == "" { @@ -232,6 +236,7 @@ func Generate(path, policyTemplateName, dataStreamName, inputName string, ignore InputVariablesJSON: inputLevelVarExpression, DataStreamVariablesJSON: dataStreamVarExpression, AllDataStreams: dataStreamsForInput, + AllInputTypes: inputTypesForPolicyTemplate, }), }, }, @@ -387,6 +392,7 @@ type FleetPackagePolicyModule struct { InputVariablesJSON string `json:"input_variables_json,omitempty"` DataStreamVariablesJSON string `json:"data_stream_variables_json,omitempty"` AllDataStreams []string `json:"all_data_streams"` + AllInputTypes []string `json:"all_input_types"` // All input types in the policy template. } func toMap(v any) map[string]any { diff --git a/fleet_input/cel/module.tf.json b/fleet_input/cel/module.tf.json index cec44cb..7b8b86e 100644 --- a/fleet_input/cel/module.tf.json +++ b/fleet_input/cel/module.tf.json @@ -235,6 +235,7 @@ "fleet_package_policy": { "agent_policy_id": "${var.fleet_agent_policy_id}", "all_data_streams": [], + "all_input_types": [], "data_stream": "", "data_stream_variables_json": "${jsonencode({\n \"data_stream.dataset\" = var.data_stream_dataset\n delete_redacted_fields = var.delete_redacted_fields\n enable_request_tracer = var.enable_request_tracer\n oauth_azure_resource = var.oauth_azure_resource\n oauth_azure_tenant_id = var.oauth_azure_tenant_id\n oauth_endpoint_params = var.oauth_endpoint_params_yaml\n oauth_google_credentials_file = var.oauth_google_credentials_file\n oauth_google_credentials_json = var.oauth_google_credentials_json\n oauth_google_delegated_account = var.oauth_google_delegated_account\n oauth_google_jwt_file = var.oauth_google_jwt_file\n oauth_google_jwt_json = var.oauth_google_jwt_json\n oauth_id = var.oauth_id\n oauth_provider = var.oauth_provider\n oauth_scopes = var.oauth_scopes\n oauth_secret = var.oauth_secret\n oauth_token_url = var.oauth_token_url\n password = var.password\n pipeline = var.pipeline\n processors = var.processors_yaml\n program = var.program\n redact_fields = var.redact_fields\n regexp = var.regexp_yaml\n resource_interval = var.resource_interval\n resource_proxy_url = var.resource_proxy_url\n resource_rate_limit_burst = var.resource_rate_limit_burst\n resource_rate_limit_limit = var.resource_rate_limit_limit\n resource_redirect_forward_headers = var.resource_redirect_forward_headers\n resource_redirect_headers_ban_list = var.resource_redirect_headers_ban_list\n resource_redirect_max_redirects = var.resource_redirect_max_redirects\n resource_retry_max_attempts = var.resource_retry_max_attempts\n resource_retry_wait_max = var.resource_retry_wait_max\n resource_retry_wait_min = var.resource_retry_wait_min\n resource_ssl = var.resource_ssl_yaml\n resource_timeout = var.resource_timeout\n resource_url = var.resource_url\n state = var.state_yaml\n tags = var.tags\n username = var.username\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_input/log/module.tf.json b/fleet_input/log/module.tf.json index f3de4ca..15c3732 100644 --- a/fleet_input/log/module.tf.json +++ b/fleet_input/log/module.tf.json @@ -60,6 +60,7 @@ "fleet_package_policy": { "agent_policy_id": "${var.fleet_agent_policy_id}", "all_data_streams": [], + "all_input_types": [], "data_stream": "", "data_stream_variables_json": "${jsonencode({\n custom = var.custom_yaml\n \"data_stream.dataset\" = var.data_stream_dataset\n paths = var.paths\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_input/sql/module.tf.json b/fleet_input/sql/module.tf.json index 7adc7d6..3f6ef42 100644 --- a/fleet_input/sql/module.tf.json +++ b/fleet_input/sql/module.tf.json @@ -62,6 +62,7 @@ "fleet_package_policy": { "agent_policy_id": "${var.fleet_agent_policy_id}", "all_data_streams": [], + "all_input_types": [], "data_stream": "", "data_stream_variables_json": "${jsonencode({\n driver = var.driver\n hosts = var.hosts\n merge_results = var.merge_results\n period = var.period\n sql_queries = var.sql_queries_yaml\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/aws.cloudtrail.aws-s3/module.tf.json b/fleet_integration/aws.cloudtrail.aws-s3/module.tf.json index 2467ce7..aa4dfcc 100644 --- a/fleet_integration/aws.cloudtrail.aws-s3/module.tf.json +++ b/fleet_integration/aws.cloudtrail.aws-s3/module.tf.json @@ -159,6 +159,11 @@ "all_data_streams": [ "cloudtrail" ], + "all_input_types": [ + "aws-s3", + "aws-cloudwatch", + "httpjson" + ], "data_stream": "cloudtrail", "data_stream_variables_json": "${jsonencode({\n api_timeout = var.api_timeout\n bucket_arn = var.bucket_arn\n bucket_list_prefix = var.bucket_list_prefix\n cloudtrail_digest_regex = var.cloudtrail_digest_regex\n cloudtrail_insight_regex = var.cloudtrail_insight_regex\n cloudtrail_regex = var.cloudtrail_regex\n collect_s3_logs = var.collect_s3_logs\n fips_enabled = var.fips_enabled\n interval = var.interval\n max_number_of_messages = var.max_number_of_messages\n number_of_workers = var.number_of_workers\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n queue_url = var.queue_url\n tags = var.tags\n visibility_timeout = var.visibility_timeout\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/barracuda_cloudgen_firewall.log.lumberjack/module.tf.json b/fleet_integration/barracuda_cloudgen_firewall.log.lumberjack/module.tf.json index 3f7a936..28ff7f1 100644 --- a/fleet_integration/barracuda_cloudgen_firewall.log.lumberjack/module.tf.json +++ b/fleet_integration/barracuda_cloudgen_firewall.log.lumberjack/module.tf.json @@ -81,6 +81,9 @@ "all_data_streams": [ "log" ], + "all_input_types": [ + "lumberjack" + ], "data_stream": "log", "data_stream_variables_json": "${jsonencode({\n listen_address = var.listen_address\n listen_port = var.listen_port\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n ssl = var.ssl_yaml\n tags = var.tags\n versions = var.versions\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/github.issues.httpjson/module.tf.json b/fleet_integration/github.issues.httpjson/module.tf.json index 78cb16a..faf1376 100644 --- a/fleet_integration/github.issues.httpjson/module.tf.json +++ b/fleet_integration/github.issues.httpjson/module.tf.json @@ -127,6 +127,9 @@ "issues", "secret_scanning" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "issues", "data_stream_variables_json": "${jsonencode({\n access_token = var.access_token\n api_url = var.api_url\n filter = var.filter\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n labels = var.labels\n owner = var.owner\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n repo = var.repo\n since = var.since\n ssl = var.ssl_yaml\n state = var.state\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.access_transparency.httpjson/module.tf.json b/fleet_integration/google_workspace.access_transparency.httpjson/module.tf.json index 05c2590..b212aa7 100644 --- a/fleet_integration/google_workspace.access_transparency.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.access_transparency.httpjson/module.tf.json @@ -136,6 +136,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "access_transparency", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.admin.httpjson/module.tf.json b/fleet_integration/google_workspace.admin.httpjson/module.tf.json index 6061e2e..de733c5 100644 --- a/fleet_integration/google_workspace.admin.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.admin.httpjson/module.tf.json @@ -120,6 +120,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "admin", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.alert.httpjson/module.tf.json b/fleet_integration/google_workspace.alert.httpjson/module.tf.json index 76ea4de..27e3146 100644 --- a/fleet_integration/google_workspace.alert.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.alert.httpjson/module.tf.json @@ -148,6 +148,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "alert", "data_stream_variables_json": "${jsonencode({\n alert_center_api_host = var.alert_center_api_host\n interval = var.interval\n page_size = var.page_size\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.context_aware_access.httpjson/module.tf.json b/fleet_integration/google_workspace.context_aware_access.httpjson/module.tf.json index 3f3bb3e..0a4d087 100644 --- a/fleet_integration/google_workspace.context_aware_access.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.context_aware_access.httpjson/module.tf.json @@ -136,6 +136,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "context_aware_access", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.device.httpjson/module.tf.json b/fleet_integration/google_workspace.device.httpjson/module.tf.json index 513f495..a358904 100644 --- a/fleet_integration/google_workspace.device.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.device.httpjson/module.tf.json @@ -136,6 +136,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "device", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.drive.httpjson/module.tf.json b/fleet_integration/google_workspace.drive.httpjson/module.tf.json index b9f060c..f3dd017 100644 --- a/fleet_integration/google_workspace.drive.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.drive.httpjson/module.tf.json @@ -120,6 +120,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "drive", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.gcp.httpjson/module.tf.json b/fleet_integration/google_workspace.gcp.httpjson/module.tf.json index 49f6af0..7d1958d 100644 --- a/fleet_integration/google_workspace.gcp.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.gcp.httpjson/module.tf.json @@ -136,6 +136,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "gcp", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.group_enterprise.httpjson/module.tf.json b/fleet_integration/google_workspace.group_enterprise.httpjson/module.tf.json index 4a92c13..175b576 100644 --- a/fleet_integration/google_workspace.group_enterprise.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.group_enterprise.httpjson/module.tf.json @@ -136,6 +136,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "group_enterprise", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.groups.httpjson/module.tf.json b/fleet_integration/google_workspace.groups.httpjson/module.tf.json index b0bd66b..9feab17 100644 --- a/fleet_integration/google_workspace.groups.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.groups.httpjson/module.tf.json @@ -120,6 +120,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "groups", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.login.httpjson/module.tf.json b/fleet_integration/google_workspace.login.httpjson/module.tf.json index 2c51639..27f8a85 100644 --- a/fleet_integration/google_workspace.login.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.login.httpjson/module.tf.json @@ -120,6 +120,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "login", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.rules.httpjson/module.tf.json b/fleet_integration/google_workspace.rules.httpjson/module.tf.json index 7433856..edb6f49 100644 --- a/fleet_integration/google_workspace.rules.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.rules.httpjson/module.tf.json @@ -136,6 +136,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "rules", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.saml.httpjson/module.tf.json b/fleet_integration/google_workspace.saml.httpjson/module.tf.json index 38a2b54..1563e32 100644 --- a/fleet_integration/google_workspace.saml.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.saml.httpjson/module.tf.json @@ -120,6 +120,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "saml", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.token.httpjson/module.tf.json b/fleet_integration/google_workspace.token.httpjson/module.tf.json index b407c9d..22468d1 100644 --- a/fleet_integration/google_workspace.token.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.token.httpjson/module.tf.json @@ -136,6 +136,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "token", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/google_workspace.user_accounts.httpjson/module.tf.json b/fleet_integration/google_workspace.user_accounts.httpjson/module.tf.json index c441418..1969995 100644 --- a/fleet_integration/google_workspace.user_accounts.httpjson/module.tf.json +++ b/fleet_integration/google_workspace.user_accounts.httpjson/module.tf.json @@ -120,6 +120,9 @@ "token", "user_accounts" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "user_accounts", "data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/system.application.winlog/module.tf.json b/fleet_integration/system.application.winlog/module.tf.json index 1c52f19..381f1f3 100644 --- a/fleet_integration/system.application.winlog/module.tf.json +++ b/fleet_integration/system.application.winlog/module.tf.json @@ -69,6 +69,12 @@ "security", "system" ], + "all_input_types": [ + "logfile", + "winlog", + "system/metrics", + "httpjson" + ], "data_stream": "application", "data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/system.security.winlog/module.tf.json b/fleet_integration/system.security.winlog/module.tf.json index 072eed2..5c6b8d8 100644 --- a/fleet_integration/system.security.winlog/module.tf.json +++ b/fleet_integration/system.security.winlog/module.tf.json @@ -69,6 +69,12 @@ "security", "system" ], + "all_input_types": [ + "logfile", + "winlog", + "system/metrics", + "httpjson" + ], "data_stream": "security", "data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/ti_abusech.malware.httpjson/module.tf.json b/fleet_integration/ti_abusech.malware.httpjson/module.tf.json index 367139f..16c067b 100644 --- a/fleet_integration/ti_abusech.malware.httpjson/module.tf.json +++ b/fleet_integration/ti_abusech.malware.httpjson/module.tf.json @@ -89,6 +89,9 @@ "threatfox", "url" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "malware", "data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/ti_abusech.malwarebazaar.httpjson/module.tf.json b/fleet_integration/ti_abusech.malwarebazaar.httpjson/module.tf.json index df76f28..f9dd4a6 100644 --- a/fleet_integration/ti_abusech.malwarebazaar.httpjson/module.tf.json +++ b/fleet_integration/ti_abusech.malwarebazaar.httpjson/module.tf.json @@ -89,6 +89,9 @@ "threatfox", "url" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "malwarebazaar", "data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/ti_abusech.threatfox.httpjson/module.tf.json b/fleet_integration/ti_abusech.threatfox.httpjson/module.tf.json index e76beb1..f52b54a 100644 --- a/fleet_integration/ti_abusech.threatfox.httpjson/module.tf.json +++ b/fleet_integration/ti_abusech.threatfox.httpjson/module.tf.json @@ -95,6 +95,9 @@ "threatfox", "url" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "threatfox", "data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n initial_interval = var.initial_interval\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/ti_abusech.url.httpjson/module.tf.json b/fleet_integration/ti_abusech.url.httpjson/module.tf.json index 89b5c6c..881b572 100644 --- a/fleet_integration/ti_abusech.url.httpjson/module.tf.json +++ b/fleet_integration/ti_abusech.url.httpjson/module.tf.json @@ -88,6 +88,9 @@ "threatfox", "url" ], + "all_input_types": [ + "httpjson" + ], "data_stream": "url", "data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/ti_recordedfuture.threat.httpjson/module.tf.json b/fleet_integration/ti_recordedfuture.threat.httpjson/module.tf.json index 31900b3..64392e3 100644 --- a/fleet_integration/ti_recordedfuture.threat.httpjson/module.tf.json +++ b/fleet_integration/ti_recordedfuture.threat.httpjson/module.tf.json @@ -108,6 +108,10 @@ "all_data_streams": [ "threat" ], + "all_input_types": [ + "httpjson", + "logfile" + ], "data_stream": "threat", "data_stream_variables_json": "${jsonencode({\n api_token = var.api_token\n custom_url = var.custom_url\n enable_request_tracer = var.enable_request_tracer\n endpoint = var.endpoint\n entity = var.entity\n interval = var.interval\n list = var.list\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n tags = var.tags\n timeout = var.timeout\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/windows.powershell.winlog/module.tf.json b/fleet_integration/windows.powershell.winlog/module.tf.json index 746ac4b..fdef672 100644 --- a/fleet_integration/windows.powershell.winlog/module.tf.json +++ b/fleet_integration/windows.powershell.winlog/module.tf.json @@ -71,6 +71,11 @@ "powershell_operational", "sysmon_operational" ], + "all_input_types": [ + "winlog", + "windows/metrics", + "httpjson" + ], "data_stream": "powershell", "data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/windows.powershell_operational.winlog/module.tf.json b/fleet_integration/windows.powershell_operational.winlog/module.tf.json index 51535f6..1697056 100644 --- a/fleet_integration/windows.powershell_operational.winlog/module.tf.json +++ b/fleet_integration/windows.powershell_operational.winlog/module.tf.json @@ -71,6 +71,11 @@ "powershell_operational", "sysmon_operational" ], + "all_input_types": [ + "winlog", + "windows/metrics", + "httpjson" + ], "data_stream": "powershell_operational", "data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/windows.sysmon_operational.winlog/module.tf.json b/fleet_integration/windows.sysmon_operational.winlog/module.tf.json index 08f3c38..f4c8847 100644 --- a/fleet_integration/windows.sysmon_operational.winlog/module.tf.json +++ b/fleet_integration/windows.sysmon_operational.winlog/module.tf.json @@ -70,6 +70,11 @@ "powershell_operational", "sysmon_operational" ], + "all_input_types": [ + "winlog", + "windows/metrics", + "httpjson" + ], "data_stream": "sysmon_operational", "data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_integration/winlog.winlogs.winlog.winlog/module.tf.json b/fleet_integration/winlog.winlogs.winlog.winlog/module.tf.json index a3f1d3d..c4a0ae8 100644 --- a/fleet_integration/winlog.winlogs.winlog.winlog/module.tf.json +++ b/fleet_integration/winlog.winlogs.winlog.winlog/module.tf.json @@ -85,6 +85,10 @@ "all_data_streams": [ "winlog" ], + "all_input_types": [ + "winlog", + "httpjson" + ], "data_stream": "winlog", "data_stream_variables_json": "${jsonencode({\n channel = var.channel\n custom = var.custom_yaml\n \"data_stream.dataset\" = var.data_stream_dataset\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n providers = var.providers_names\n tags = var.tags\n})}", "description": "${var.fleet_package_policy_description}", diff --git a/fleet_package_policy/README.md b/fleet_package_policy/README.md index 37e52bc..236cd92 100644 --- a/fleet_package_policy/README.md +++ b/fleet_package_policy/README.md @@ -26,7 +26,8 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [agent\_policy\_id](#input\_agent\_policy\_id) | ID of the agent policy to add the package policy to. | `string` | n/a | yes | -| [all\_data\_streams](#input\_all\_data\_streams) | List of all data streams in the package policy. This is necessary to disable all data streams except the one being used. | `list(string)` | `[]` | no | +| [all\_data\_streams](#input\_all\_data\_streams) | List of all data streams associated to the input type in the policy template. This is necessary to disable all data streams except the one being used. | `list(string)` | `[]` | no | +| [all\_input\_types](#input\_all\_input\_types) | List of all input types in the package policy template. This is necessary to disable all inputs except the one being used. | `list(string)` | `[]` | no | | [data\_stream](#input\_data\_stream) | Name of the data\_stream within the integration (e.g. "log"). | `any` | n/a | yes | | [data\_stream\_variables\_json](#input\_data\_stream\_variables\_json) | JSON encoded data stream specific variables. | `string` | `"{}"` | no | | [description](#input\_description) | Description to apply to the package policy. | `string` | `""` | no | diff --git a/fleet_package_policy/main.tf b/fleet_package_policy/main.tf index 80f2b12..87d4a90 100644 --- a/fleet_package_policy/main.tf +++ b/fleet_package_policy/main.tf @@ -4,8 +4,11 @@ locals { unused_data_streams = [for data_stream in var.all_data_streams : data_stream if data_stream != var.data_stream ] + unused_input_types = [for input_type in var.all_input_types : + input_type if input_type != var.input_type + ] - # NOTE: This is a really painful part of Fleet. If package contains a stream that is enabled by default + # NOTE: This is a really painful part of Fleet. If package contains an input or stream that is enabled by default # and it contains required variables, then you must explicitly disable the streams in order to pass # validation. Beats had the same issue originally that made doing IaC hard (see # https://github.com/elastic/beats/issues/17256). Any time a package adds a new data stream that is @@ -15,6 +18,11 @@ locals { enabled = false } } + disabled_inputs_config = { for input_type in local.unused_input_types : + "${var.package_name}-${input_type}" => { + enabled = false + } + } } resource "restapi_object" "package_policy" { @@ -31,7 +39,7 @@ resource "restapi_object" "package_policy" { namespace = var.namespace description = var.description vars = var.package_variables_json - inputs = { + inputs = merge({ "${var.policy_template}-${var.input_type}" = { enabled = true vars = var.input_variables_json == null ? null : jsondecode(var.input_variables_json) @@ -43,6 +51,6 @@ resource "restapi_object" "package_policy" { } }, local.disabled_stream_config) } - } + }, local.disabled_inputs_config) }) } diff --git a/fleet_package_policy/variables.tf b/fleet_package_policy/variables.tf index d7e1976..354c69b 100644 --- a/fleet_package_policy/variables.tf +++ b/fleet_package_policy/variables.tf @@ -88,7 +88,13 @@ variable "input_variables_json" { } variable "all_data_streams" { - description = "List of all data streams in the package policy. This is necessary to disable all data streams except the one being used." + description = "List of all data streams associated to the input type in the policy template. This is necessary to disable all data streams except the one being used." + type = list(string) + default = [] +} + +variable "all_input_types" { + description = "List of all input types in the package policy template. This is necessary to disable all inputs except the one being used." type = list(string) default = [] }