-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #39 from axiomhq/create-logs-subscriber
add logs subscriber
- Loading branch information
Showing
6 changed files
with
255 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
163 changes: 163 additions & 0 deletions
163
axiom-cloudwatch-logs-subscriber-cloudformation-stack.template.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| Description: A lambda function to auto subscribe Axiom Forwarder to new CloudWatch Log Groups. | ||
| Parameters: | ||
| LambdaFunctionName: | ||
| Type: String | ||
| Description: Name of the AWS Lambda Function. | ||
| Default: axiom-cloudwatch-logs-subscriber-lambda | ||
| AllowedPattern: ".+" # required | ||
| AxiomCloudWatchLambdaIngesterARN: | ||
| Type: String | ||
| Description: The ARN of the AWS Lambda Function that is used to ingest data to axiom. | ||
| AllowedPattern: ".+" # required | ||
| CloudWatchLogGroupsPrefix: | ||
| Type: String | ||
| Description: The Prefix of cloudwatch log groups to subscribe to the AWS Lambda ingester. | ||
| Default: "" # all | ||
| AxiomLambdaLogRetention: | ||
| Type: "Number" | ||
| Description: "The number of days to retain CloudWatch logs for the created lambda functions." | ||
| Default: 1 | ||
| Resources: | ||
| AxiomCloudWatchLogsSubscriberS3Bucket: | ||
| Type: AWS::S3::Bucket | ||
| Properties: | ||
| AccessControl: BucketOwnerFullControl | ||
| BucketName: !Join ["-", [!Ref AWS::StackName, "axiom", "cloudtrail"]] | ||
| AxiomCloudWatchLogsSubscriberS3BucketPolicy: | ||
| Type: AWS::S3::BucketPolicy | ||
| DependsOn: AxiomCloudWatchLogsSubscriberS3Bucket | ||
| Properties: | ||
| Bucket: !Ref AxiomCloudWatchLogsSubscriberS3Bucket | ||
| PolicyDocument: | ||
| { | ||
| "Version": "2012-10-17", | ||
| "Statement": | ||
| [ | ||
| { | ||
| "Sid": "AWSCloudTrailAclCheck20150319", | ||
| "Effect": "Allow", | ||
| "Principal": { "Service": "cloudtrail.amazonaws.com" }, | ||
| "Action": "s3:GetBucketAcl", | ||
| "Resource": | ||
| !GetAtt ["AxiomCloudWatchLogsSubscriberS3Bucket", "Arn"], | ||
| }, | ||
| { | ||
| "Sid": "AWSCloudTrailWrite20150319", | ||
| "Effect": "Allow", | ||
| "Principal": { "Service": "cloudtrail.amazonaws.com" }, | ||
| "Action": "s3:PutObject", | ||
| "Resource": | ||
| !Join [ | ||
| "", | ||
| [ | ||
| !GetAtt ["AxiomCloudWatchLogsSubscriberS3Bucket", "Arn"], | ||
| "/AWSLogs/", | ||
| { "Ref": "AWS::AccountId" }, | ||
| "/*", | ||
| ], | ||
| ], | ||
| "Condition": | ||
| { | ||
| "StringEquals": | ||
| { "s3:x-amz-acl": "bucket-owner-full-control" }, | ||
| }, | ||
| }, | ||
| ], | ||
| } | ||
| AxiomLogsSubscriberCloudTrail: | ||
| Type: AWS::CloudTrail::Trail | ||
| DependsOn: AxiomCloudWatchLogsSubscriberS3BucketPolicy | ||
| Properties: | ||
| EnableLogFileValidation: false | ||
| IncludeGlobalServiceEvents: true | ||
| IsMultiRegionTrail: true | ||
| IsLogging: true | ||
| S3BucketName: !Join ["-", [!Ref AWS::StackName, "axiom", "cloudtrail"]] | ||
| TrailName: | ||
| !Join ["-", [!Ref AWS::StackName, "axiom", { "Ref": "AWS::AccountId" }]] | ||
| AxiomLogsSubscriberEventRule: | ||
| DependsOn: AxiomCloudWatchLogsSubscriber | ||
| Type: AWS::Events::Rule | ||
| Properties: | ||
| Description: Axiom log group auto subscription event rule., | ||
| EventPattern: | ||
| source: ["aws.logs"] | ||
| detail-type: ["AWS API Call via CloudTrail"] | ||
| detail: | ||
| eventSource: ["logs.amazonaws.com"] | ||
| eventName: ["CreateLogGroup"] | ||
| Name: | ||
| "Fn::Join": | ||
| ["-", [{ "Ref": "AWS::StackName" }, "axiom-auto-subscription-rule"]] | ||
| Targets: | ||
| - Id: | ||
| !Join ["-", [!Ref "AWS::StackName", "axiom-auto-subscription-rule"]] | ||
| Arn: !GetAtt ["AxiomCloudWatchLogsSubscriber", "Arn"] | ||
| AxiomCloudWatchLogsSubscriberPolicy: | ||
| Type: AWS::IAM::Policy | ||
| Properties: | ||
| PolicyDocument: | ||
| Statement: | ||
| - Action: | ||
| - logs:DeleteSubscriptionFilter | ||
| - logs:PutSubscriptionFilter | ||
| - logs:DescribeLogGroups | ||
| - lambda:AddPermission | ||
| - lambda:RemovePermission | ||
| Effect: Allow | ||
| Resource: "*" | ||
| PolicyName: axiom-cloudwatch-logs-subscriber-lambda-policy | ||
| Roles: | ||
| - !Ref "AxiomCloudWatchLogsSubscriberRole" | ||
| AxiomCloudWatchLogsSubscriberRole: | ||
| Type: AWS::IAM::Role | ||
| Properties: | ||
| AssumeRolePolicyDocument: | ||
| Statement: | ||
| - Action: | ||
| - "sts:AssumeRole" | ||
| Effect: Allow | ||
| Principal: | ||
| Service: | ||
| - lambda.amazonaws.com | ||
| ManagedPolicyArns: | ||
| - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" | ||
| AxiomCloudWatchLogsSubscriber: | ||
| Type: AWS::Lambda::Function | ||
| DependsOn: | ||
| - AxiomCloudWatchLogsSubscriberRole | ||
| Properties: | ||
| Runtime: python3.9 | ||
| Handler: index.lambda_handler | ||
| Code: | ||
| ZipFile: | | ||
| # DO NOT EDIT | ||
| # CI will replace these comments with the code from ./logs_subscriber.py | ||
| Role: !GetAtt | ||
| - AxiomCloudWatchLogsSubscriberRole | ||
| - Arn | ||
| Description: Axiom CloudWatch Automatic Logs Subscriber Lambda | ||
| Environment: | ||
| Variables: | ||
| AXIOM_CLOUDWATCH_LAMBDA_INGESTER_ARN: !Ref "AxiomCloudWatchLambdaIngesterARN" | ||
| LOG_GROUP_PREFIX: !Ref "CloudWatchLogGroupsPrefix" | ||
| AxiomCloudWatchLogsSubscriberPermission: | ||
| Type: AWS::Lambda::Permission | ||
| Properties: | ||
| Action: "lambda:InvokeFunction" | ||
| FunctionName: { "Fn::GetAtt": ["AxiomCloudWatchLogsSubscriber", "Arn"] } | ||
| Principal: "events.amazonaws.com" | ||
| SourceAccount: | ||
| Ref: AWS::AccountId | ||
| SourceArn: !GetAtt ["AxiomLogsSubscriberEventRule", "Arn"] | ||
| AxiomCloudWatchLogsSubscriberLogGroup: | ||
| DependsOn: ["AxiomCloudWatchLogsSubscriberRole"] | ||
| Type: AWS::Logs::LogGroup | ||
| Properties: | ||
| LogGroupName: | ||
| !Join [ | ||
| "", | ||
| ["/aws/lambda/", { "Ref": "AxiomCloudWatchLogsSubscriber" }], | ||
| ] | ||
| RetentionInDays: | ||
| Ref: "AxiomLambdaLogRetention" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| # subscribe the Axiom ingester to newly created log groups | ||
| import boto3 | ||
| import os | ||
| import logging | ||
|
|
||
| # Set environment variables. | ||
| axiom_cloudwatch_lambda_ingester_arn = os.getenv("AXIOM_CLOUDWATCH_LAMBDA_INGESTER_ARN") | ||
| log_group_prefix = os.getenv("LOG_GROUP_PREFIX", "") | ||
|
|
||
| # set logger | ||
| level = os.getenv("log_level", "INFO") | ||
| logging.basicConfig(level=level) | ||
| logger = logging.getLogger() | ||
| logger.setLevel(level) | ||
|
|
||
| # Set up CloudWatch Logs client. | ||
| log_client = boto3.client("logs") | ||
| lambda_client = boto3.client("lambda") | ||
|
|
||
|
|
||
| def lambda_handler(event, context): | ||
| """ | ||
| Subscribes log ingester to log group from event. | ||
| :param event: Event data from CloudWatch Logs. | ||
| :type event: dict | ||
| :param context: Lambda context object. | ||
| :type context: obj | ||
| :return: None | ||
| """ | ||
| if not "detail" in event: | ||
| return | ||
| # Grab the log group name from incoming event. | ||
| aws_account_id = event["account"] | ||
| aws_region = event["detail"]["awsRegion"] | ||
| log_group_name = event["detail"]["requestParameters"]["logGroupName"] | ||
| log_group_arn = ( | ||
| f"arn:aws:logs:{aws_region}:{aws_account_id}:log-group:{log_group_name}:*" | ||
| ) | ||
|
|
||
| # Check whether the prefix is set - the prefix is used to determine which logs we want. | ||
| # or whether the log group's name starts with the set prefix. | ||
| if not log_group_prefix or log_group_name.startswith(log_group_prefix): | ||
| create_subscription_filter( | ||
| log_group_name, log_group_arn, axiom_cloudwatch_lambda_ingester_arn | ||
| ) | ||
|
|
||
| else: | ||
| print( | ||
| f"log group ({log_group_name}) did not match the prefix ({log_group_prefix})" | ||
| ) | ||
|
|
||
|
|
||
| def create_subscription_filter(log_group_name, log_group_arn, lambda_arn): | ||
| try: | ||
| logger.info(f"Creating subscription filter for {log_group_name}...") | ||
| lambda_client.add_permission( | ||
| FunctionName=lambda_arn, | ||
| StatementId="%s-axiom" % log_group_name.replace("/", "-"), | ||
| Action="lambda:InvokeFunction", | ||
| Principal=f"logs.amazonaws.com", | ||
| SourceArn=log_group_arn, | ||
| ) | ||
|
|
||
| log_client.put_subscription_filter( | ||
| logGroupName=log_group_name, | ||
| filterName="%s-axiom" % log_group_name, | ||
| filterPattern="", | ||
| destinationArn=lambda_arn, | ||
| distribution="ByLogStream", | ||
| ) | ||
| logger.info( | ||
| f"{log_group_name} subscription filter has been created successfully." | ||
| ) | ||
| except Exception as e: | ||
| logger.error(f"Error create Subscription filter: {e}") | ||
| raise e |