generator - separate inputs and integrations (#10)

Create separate directories to hold Fleet input packages and integration packages.

Rename integration module directories to use dots as separators.

Simplify directory naming for input modules to be just the package name. This assumes
that input packages will only ever contain a single policy template with one input.

Makefile

@@ -4,9 +4,14 @@ all: fmt docs modules validate
 .PHONY: modules
 modules: fleet-modules terraform-docs
 
+# Generate fleet module. The modules are specified using selectors that are formatted as:
+#    {package_name}/{policy_template}/{data_stream}/{input_type} - for integrations
+#    {package_name/{policy_template}/{input_type} - for inputs
+# Any slashes (/) contained in the attribute values are replaced with underscores. For
+# example, if matching the input_type of "prometheus/metrics" use "prometheus_metrics".
 .PHONY: fleet-modules
 fleet-modules: install
-	fleet-terraform-generator generate batch --packages-dir ../integrations/packages --out fleet_integrations \
+	fleet-terraform-generator generate batch --packages-dir ../integrations/packages --out . \
 		"aws/cloudtrail/*/aws-s3" \
 		"barracuda_cloudgen_firewall/*/*/lumberjack" \
 		"github/*/issues/httpjson" \
@@ -45,7 +50,7 @@ terraform-docs:
 	terraform-docs markdown table --output-file="README.md" fleet_output
 	terraform-docs markdown table --output-file="README.md" fleet_package_policy
 	terraform-docs markdown table --output-file="README.md" fleet_server_host
-	@for i in $(shell find fleet_integrations/ -name module.tf.json); do \
+	@for i in $(shell find fleet_integration/ fleet_input/ -name module.tf.json); do \
 	  module=$$(dirname $$i); \
 	  terraform-docs markdown table --output-file="README.md" "$$module" || exit 1; \
 	done

examples/github/main.tf

@@ -19,7 +19,7 @@ module "agent_policy_github" {
 
 // Add GitHub Issues data stream to the agent policy.
 module "integration_github_issues" {
-  source                = "../../fleet_integrations/github_issues_httpjson"
+  source                = "../../fleet_integration/github.issues.httpjson"
   fleet_agent_policy_id = module.agent_policy_github.id
   access_token          = var.github_token
   owner                 = "elastic"

fleet-terraform-generator/docs/README.md

@@ -10,7 +10,7 @@ Tool for generating Terraform modules for managing Fleet integrations.
 
 ### SEE ALSO
 
-* [fleet-terraform-generator generate](fleet-terraform-generator_generate.md)	 - List all Fleet packages, policy templates, data streams, and inputs.
+* [fleet-terraform-generator generate](fleet-terraform-generator_generate.md)	 - Generate a Terraform module for managing a Fleet package policy.
 * [fleet-terraform-generator list](fleet-terraform-generator_list.md)	 - List all Fleet packages, policy templates, data streams, and inputs.
 
 

fleet-terraform-generator/docs/fleet-terraform-generator_generate.md

@@ -1,6 +1,6 @@
 ## fleet-terraform-generator generate
 
-List all Fleet packages, policy templates, data streams, and inputs.
+Generate a Terraform module for managing a Fleet package policy.
 
 ```
 fleet-terraform-generator generate [flags]
@@ -13,7 +13,7 @@ fleet-terraform-generator generate [flags]
   -h, --help                     help for generate
       --ignore-var-shadow        Ignore variable shadowing errors in Fleet packages.
   -i, --input string             Input name.
-      --out string               Output path. It creates a new sub-directory named based on the package, policy template, data stream, and input.
+      --out string               Output path. It creates a new sub-directory named based on the package type, package name, policy template, data stream, and input.
   -p, --package string           Package name.
       --packages-dir string      Directory containing Fleet packages.
   -t, --policy-template string   Policy template name.

fleet-terraform-generator/docs/fleet-terraform-generator_generate_batch.md

@@ -16,12 +16,12 @@ fleet-terraform-generator generate batch [flags]
 
 ```
       --ignore-var-shadow     Ignore variable shadowing errors in Fleet packages.
-      --out string            Output path. It creates a new sub-directory named based on the package, policy template, data stream, and input.
+      --out string            Output path. It creates a new sub-directory named based on the package type, package name, policy template, data stream, and input.
       --packages-dir string   Directory containing Fleet packages.
 ```
 
 ### SEE ALSO
 
-* [fleet-terraform-generator generate](fleet-terraform-generator_generate.md)	 - List all Fleet packages, policy templates, data streams, and inputs.
+* [fleet-terraform-generator generate](fleet-terraform-generator_generate.md)	 - Generate a Terraform module for managing a Fleet package policy.
 
 

fleet-terraform-generator/internal/cmd/generate.go

@@ -42,7 +42,7 @@ const (
 func GenerateCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "generate",
-		Short: "List all Fleet packages, policy templates, data streams, and inputs.",
+		Short: "Generate a Terraform module for managing a Fleet package policy.",
 		RunE:  generateModuleRunE,
 	}
 	cmd.PersistentFlags().String(packagesDirectoryFlagName, "", "Directory containing Fleet packages.")
@@ -61,7 +61,7 @@ func GenerateCmd() *cobra.Command {
 
 	cmd.PersistentFlags().Bool(generateIgnoreVariableShadowingFlagName, false, "Ignore variable shadowing errors in Fleet packages.")
 
-	cmd.PersistentFlags().String(generateOutputPathFlagName, "", "Output path. It creates a new sub-directory named based on the package, policy template, data stream, and input.")
+	cmd.PersistentFlags().String(generateOutputPathFlagName, "", "Output path. It creates a new sub-directory named based on the package type, package name, policy template, data stream, and input.")
 	must(cmd.MarkPersistentFlagRequired(generateOutputPathFlagName))
 
 	batchCmd := &cobra.Command{
@@ -111,7 +111,7 @@ func generateModuleRunE(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("output dir %q is not a directory", outputDir)
 	}
 
-	err = generateTerraformModuleToDisk(pkgsDir, packageName, policyTemplateName, dataStreamName, inputName, outputDir, allowVariableShadowing)
+	err = generateTerraformModuleToDisk(pkgsDir, policyTemplateName, dataStreamName, inputName, outputDir, allowVariableShadowing)
 	if err != nil {
 		s := module.Specifier{
 			Integration:    packageName,
@@ -156,7 +156,7 @@ func generateBatchRunE(cmd *cobra.Command, args []string) error {
 
 	var errs []error
 	for _, s := range specs {
-		if err = generateTerraformModuleToDisk(s.Path, s.Integration, s.PolicyTemplate, s.DataStream, s.Input, outputDir, allowVariableShadowing); err != nil {
+		if err = generateTerraformModuleToDisk(s.Path, s.PolicyTemplate, s.DataStream, s.Input, outputDir, allowVariableShadowing); err != nil {
 			errs = append(errs, fmt.Errorf("error generating module for %s: %w", s.String(), err))
 		}
 	}
@@ -167,14 +167,14 @@ func generateBatchRunE(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func generateTerraformModuleToDisk(path, packageName, policyTemplateName, dataStreamName, inputName, outputDir string, allowVariableShadowing bool) error {
+func generateTerraformModuleToDisk(path, policyTemplateName, dataStreamName, inputName, outputDir string, allowVariableShadowing bool) error {
 	tf, err := module.Generate(path, policyTemplateName, dataStreamName, inputName, allowVariableShadowing)
 	if err != nil {
 		return err
 	}
 
-	outputPath := filepath.Join(outputDir, tf.Name, "module.tf.json")
-	if err = os.Mkdir(filepath.Dir(outputPath), 0o700); err != nil {
+	outputPath := filepath.Join(outputDir, "fleet_"+tf.PackageType, tf.Name, "module.tf.json")
+	if err = os.MkdirAll(filepath.Dir(outputPath), 0o700); err != nil {
 		if !errors.Is(err, os.ErrExist) {
 			return fmt.Errorf("failed to create directory for module: %w", err)
 		}

fleet-terraform-generator/internal/module/generate.go

@@ -36,8 +36,9 @@ import (
 )
 
 type Terraform struct {
-	Name string
-	File *terraform.File
+	Name        string
+	PackageType string // Package type (input or integration).
+	File        *terraform.File
 }
 
 // Generate generates a Terraform module.
@@ -242,9 +243,19 @@ func Generate(path, policyTemplateName, dataStreamName, inputName string, ignore
 		},
 	}
 
+	pkgType := manifest.Type
+	if pkgType == "" {
+		pkgType = "integration"
+	}
+
+	tfModuleName := manifest.Name
+	if pkgType == "integration" {
+		tfModuleName = moduleName(manifest.Name, policyTemplateName, dataStreamName, inputName)
+	}
 	return &Terraform{
-		Name: moduleName(manifest.Name, policyTemplateName, dataStreamName, inputName),
-		File: tf,
+		Name:        tfModuleName,
+		PackageType: pkgType,
+		File:        tf,
 	}, nil
 }
 
@@ -454,9 +465,11 @@ func moduleName(integration, policyTemplate, dataStream, input string) string {
 		name = append(name, dataStream)
 	}
 	name = append(name, strings.ReplaceAll(input, "/", "_"))
-	return strings.Join(name, "_")
+	return strings.Join(name, ".")
 }
 
+// quoteIfNeeded surrounds a variable name with quotes if it contains dots.
+// HCL used in Terraform does not allow names with dots.
 func quoteIfNeeded(name string) string {
 	if strings.Contains(name, ".") {
 		return strconv.Quote(name)

fleet_agent_policy/providers.tf

@@ -5,4 +5,4 @@ terraform {
       version = "~> 0.6.2"
     }
   }
-}
+}

fleet_integrations/aws_cloudtrail_aws-s3/README.md → fleet_integration/aws.cloudtrail.aws-s3/README.md

@@ -36,7 +36,7 @@ No resources.
 | <a name="input_fleet_data_stream_namespace"></a> [fleet\_data\_stream\_namespace](#input\_fleet\_data\_stream\_namespace) | Namespace to use for the data stream. | `string` | `"default"` | no |
 | <a name="input_fleet_package_policy_description"></a> [fleet\_package\_policy\_description](#input\_fleet\_package\_policy\_description) | Description to use for the package policy. | `string` | `""` | no |
 | <a name="input_fleet_package_policy_name_suffix"></a> [fleet\_package\_policy\_name\_suffix](#input\_fleet\_package\_policy\_name\_suffix) | Suffix to append to the end of the package policy name. | `string` | `""` | no |
-| <a name="input_fleet_package_version"></a> [fleet\_package\_version](#input\_fleet\_package\_version) | Version of the aws package to use. | `string` | `"1.48.0"` | no |
+| <a name="input_fleet_package_version"></a> [fleet\_package\_version](#input\_fleet\_package\_version) | Version of the aws package to use. | `string` | `"1.49.0"` | no |
 | <a name="input_interval"></a> [interval](#input\_interval) | Time interval for polling listing of the S3 bucket. NOTE: Supported units for this parameter are h/m/s. | `string` | `"1m"` | no |
 | <a name="input_max_number_of_messages"></a> [max\_number\_of\_messages](#input\_max\_number\_of\_messages) | The maximum number of SQS messages that can be inflight at any time. | `number` | `5` | no |
 | <a name="input_number_of_workers"></a> [number\_of\_workers](#input\_number\_of\_workers) | Number of workers that will process the S3 objects listed. | `number` | `5` | no |

fleet_integrations/aws_cloudtrail_aws-s3/module.tf.json → fleet_integration/aws.cloudtrail.aws-s3/module.tf.json

@@ -77,7 +77,7 @@
     "fleet_package_version": {
       "type": "string",
       "description": "Version of the aws package to use.",
-      "default": "1.48.0"
+      "default": "1.49.0"
     },
     "interval": {
       "type": "string",