Skip to content

Commit

Permalink
Disable unused input types for policy_template (#11)
Browse files Browse the repository at this point in the history
Streams that are enabled by default need to be disabled. Previously we
disabled streams for the input type that was being used. But streams under
different input types that existed in the policy template were omitted
so they were enabled. This adds an explicit enabled=false for all unused
inputs.

Fixes #7
  • Loading branch information
andrewkroh authored Jul 20, 2023
1 parent 8cbdd62 commit 68117a6
Show file tree
Hide file tree
Showing 35 changed files with 129 additions and 5 deletions.
6 changes: 6 additions & 0 deletions fleet-terraform-generator/internal/module/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,10 @@ func Generate(path, policyTemplateName, dataStreamName, inputName string, ignore
}
sort.Strings(dataStreamsForInput)
}
inputTypesForPolicyTemplate := []string{} // Declare empty slice.
for _, input := range policyTemplate.Inputs {
inputTypesForPolicyTemplate = append(inputTypesForPolicyTemplate, input.Type)
}

packagePolicyName := manifest.Name + "-" + dataStreamName + "-${var.fleet_data_stream_namespace}${var.fleet_package_policy_name_suffix}"
if dataStreamName == "" {
Expand All @@ -232,6 +236,7 @@ func Generate(path, policyTemplateName, dataStreamName, inputName string, ignore
InputVariablesJSON: inputLevelVarExpression,
DataStreamVariablesJSON: dataStreamVarExpression,
AllDataStreams: dataStreamsForInput,
AllInputTypes: inputTypesForPolicyTemplate,
}),
},
},
Expand Down Expand Up @@ -387,6 +392,7 @@ type FleetPackagePolicyModule struct {
InputVariablesJSON string `json:"input_variables_json,omitempty"`
DataStreamVariablesJSON string `json:"data_stream_variables_json,omitempty"`
AllDataStreams []string `json:"all_data_streams"`
AllInputTypes []string `json:"all_input_types"` // All input types in the policy template.
}

func toMap(v any) map[string]any {
Expand Down
1 change: 1 addition & 0 deletions fleet_input/cel/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@
"fleet_package_policy": {
"agent_policy_id": "${var.fleet_agent_policy_id}",
"all_data_streams": [],
"all_input_types": [],
"data_stream": "",
"data_stream_variables_json": "${jsonencode({\n \"data_stream.dataset\" = var.data_stream_dataset\n delete_redacted_fields = var.delete_redacted_fields\n enable_request_tracer = var.enable_request_tracer\n oauth_azure_resource = var.oauth_azure_resource\n oauth_azure_tenant_id = var.oauth_azure_tenant_id\n oauth_endpoint_params = var.oauth_endpoint_params_yaml\n oauth_google_credentials_file = var.oauth_google_credentials_file\n oauth_google_credentials_json = var.oauth_google_credentials_json\n oauth_google_delegated_account = var.oauth_google_delegated_account\n oauth_google_jwt_file = var.oauth_google_jwt_file\n oauth_google_jwt_json = var.oauth_google_jwt_json\n oauth_id = var.oauth_id\n oauth_provider = var.oauth_provider\n oauth_scopes = var.oauth_scopes\n oauth_secret = var.oauth_secret\n oauth_token_url = var.oauth_token_url\n password = var.password\n pipeline = var.pipeline\n processors = var.processors_yaml\n program = var.program\n redact_fields = var.redact_fields\n regexp = var.regexp_yaml\n resource_interval = var.resource_interval\n resource_proxy_url = var.resource_proxy_url\n resource_rate_limit_burst = var.resource_rate_limit_burst\n resource_rate_limit_limit = var.resource_rate_limit_limit\n resource_redirect_forward_headers = var.resource_redirect_forward_headers\n resource_redirect_headers_ban_list = var.resource_redirect_headers_ban_list\n resource_redirect_max_redirects = var.resource_redirect_max_redirects\n resource_retry_max_attempts = var.resource_retry_max_attempts\n resource_retry_wait_max = var.resource_retry_wait_max\n resource_retry_wait_min = var.resource_retry_wait_min\n resource_ssl = var.resource_ssl_yaml\n resource_timeout = var.resource_timeout\n resource_url = var.resource_url\n state = var.state_yaml\n tags = var.tags\n username = var.username\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
1 change: 1 addition & 0 deletions fleet_input/log/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
"fleet_package_policy": {
"agent_policy_id": "${var.fleet_agent_policy_id}",
"all_data_streams": [],
"all_input_types": [],
"data_stream": "",
"data_stream_variables_json": "${jsonencode({\n custom = var.custom_yaml\n \"data_stream.dataset\" = var.data_stream_dataset\n paths = var.paths\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
1 change: 1 addition & 0 deletions fleet_input/sql/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
"fleet_package_policy": {
"agent_policy_id": "${var.fleet_agent_policy_id}",
"all_data_streams": [],
"all_input_types": [],
"data_stream": "",
"data_stream_variables_json": "${jsonencode({\n driver = var.driver\n hosts = var.hosts\n merge_results = var.merge_results\n period = var.period\n sql_queries = var.sql_queries_yaml\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
5 changes: 5 additions & 0 deletions fleet_integration/aws.cloudtrail.aws-s3/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,11 @@
"all_data_streams": [
"cloudtrail"
],
"all_input_types": [
"aws-s3",
"aws-cloudwatch",
"httpjson"
],
"data_stream": "cloudtrail",
"data_stream_variables_json": "${jsonencode({\n api_timeout = var.api_timeout\n bucket_arn = var.bucket_arn\n bucket_list_prefix = var.bucket_list_prefix\n cloudtrail_digest_regex = var.cloudtrail_digest_regex\n cloudtrail_insight_regex = var.cloudtrail_insight_regex\n cloudtrail_regex = var.cloudtrail_regex\n collect_s3_logs = var.collect_s3_logs\n fips_enabled = var.fips_enabled\n interval = var.interval\n max_number_of_messages = var.max_number_of_messages\n number_of_workers = var.number_of_workers\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n queue_url = var.queue_url\n tags = var.tags\n visibility_timeout = var.visibility_timeout\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@
"all_data_streams": [
"log"
],
"all_input_types": [
"lumberjack"
],
"data_stream": "log",
"data_stream_variables_json": "${jsonencode({\n listen_address = var.listen_address\n listen_port = var.listen_port\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n ssl = var.ssl_yaml\n tags = var.tags\n versions = var.versions\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
3 changes: 3 additions & 0 deletions fleet_integration/github.issues.httpjson/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@
"issues",
"secret_scanning"
],
"all_input_types": [
"httpjson"
],
"data_stream": "issues",
"data_stream_variables_json": "${jsonencode({\n access_token = var.access_token\n api_url = var.api_url\n filter = var.filter\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n labels = var.labels\n owner = var.owner\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n repo = var.repo\n since = var.since\n ssl = var.ssl_yaml\n state = var.state\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "access_transparency",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "admin",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "alert",
"data_stream_variables_json": "${jsonencode({\n alert_center_api_host = var.alert_center_api_host\n interval = var.interval\n page_size = var.page_size\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "context_aware_access",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "device",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "drive",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "gcp",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "group_enterprise",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "groups",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "login",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "rules",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "saml",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "token",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_duplicate_custom_fields = var.preserve_duplicate_custom_fields\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@
"token",
"user_accounts"
],
"all_input_types": [
"httpjson"
],
"data_stream": "user_accounts",
"data_stream_variables_json": "${jsonencode({\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
6 changes: 6 additions & 0 deletions fleet_integration/system.application.winlog/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@
"security",
"system"
],
"all_input_types": [
"logfile",
"winlog",
"system/metrics",
"httpjson"
],
"data_stream": "application",
"data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
6 changes: 6 additions & 0 deletions fleet_integration/system.security.winlog/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@
"security",
"system"
],
"all_input_types": [
"logfile",
"winlog",
"system/metrics",
"httpjson"
],
"data_stream": "security",
"data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
3 changes: 3 additions & 0 deletions fleet_integration/ti_abusech.malware.httpjson/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@
"threatfox",
"url"
],
"all_input_types": [
"httpjson"
],
"data_stream": "malware",
"data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@
"threatfox",
"url"
],
"all_input_types": [
"httpjson"
],
"data_stream": "malwarebazaar",
"data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@
"threatfox",
"url"
],
"all_input_types": [
"httpjson"
],
"data_stream": "threatfox",
"data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n initial_interval = var.initial_interval\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
3 changes: 3 additions & 0 deletions fleet_integration/ti_abusech.url.httpjson/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@
"threatfox",
"url"
],
"all_input_types": [
"httpjson"
],
"data_stream": "url",
"data_stream_variables_json": "${jsonencode({\n http_client_timeout = var.http_client_timeout\n interval = var.interval\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n ssl = var.ssl_yaml\n tags = var.tags\n url = var.url\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,10 @@
"all_data_streams": [
"threat"
],
"all_input_types": [
"httpjson",
"logfile"
],
"data_stream": "threat",
"data_stream_variables_json": "${jsonencode({\n api_token = var.api_token\n custom_url = var.custom_url\n enable_request_tracer = var.enable_request_tracer\n endpoint = var.endpoint\n entity = var.entity\n interval = var.interval\n list = var.list\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n proxy_url = var.proxy_url\n tags = var.tags\n timeout = var.timeout\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
5 changes: 5 additions & 0 deletions fleet_integration/windows.powershell.winlog/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@
"powershell_operational",
"sysmon_operational"
],
"all_input_types": [
"winlog",
"windows/metrics",
"httpjson"
],
"data_stream": "powershell",
"data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@
"powershell_operational",
"sysmon_operational"
],
"all_input_types": [
"winlog",
"windows/metrics",
"httpjson"
],
"data_stream": "powershell_operational",
"data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@
"powershell_operational",
"sysmon_operational"
],
"all_input_types": [
"winlog",
"windows/metrics",
"httpjson"
],
"data_stream": "sysmon_operational",
"data_stream_variables_json": "${jsonencode({\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n processors = var.processors_yaml\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
4 changes: 4 additions & 0 deletions fleet_integration/winlog.winlogs.winlog.winlog/module.tf.json
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@
"all_data_streams": [
"winlog"
],
"all_input_types": [
"winlog",
"httpjson"
],
"data_stream": "winlog",
"data_stream_variables_json": "${jsonencode({\n channel = var.channel\n custom = var.custom_yaml\n \"data_stream.dataset\" = var.data_stream_dataset\n event_id = var.event_id\n ignore_older = var.ignore_older\n language = var.language\n preserve_original_event = var.preserve_original_event\n providers = var.providers_names\n tags = var.tags\n})}",
"description": "${var.fleet_package_policy_description}",
Expand Down
Loading

0 comments on commit 68117a6

Please sign in to comment.