-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
8 changed files
with
325 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| Description: A lambda that creates subscription filters for existing log groups | ||
| Metadata: | ||
| AWS::CloudFormation::Interface: | ||
| ParameterGroups: | ||
| - Label: | ||
| default: "Subscriber lambda" | ||
| Parameters: | ||
| - LambdaFunctionName | ||
| - Label: | ||
| default: "Forwarder lambda" | ||
| Parameters: | ||
| - AxiomCloudWatchForwarderLambdaARN | ||
| - Label: | ||
| default: "Which log groups to subscribe to?" | ||
| Parameters: | ||
| - CloudWatchLogGroupsNames | ||
| - CloudWatchLogGroupsPrefix | ||
| - CloudWatchLogGroupsPattern | ||
|
|
||
| Parameters: | ||
| LambdaFunctionName: | ||
| Type: String | ||
| Description: Name of the AWS Lambda function. | ||
| Default: axiom-cloudwatch-subscriber | ||
| AllowedPattern: ".+" # required | ||
| AxiomCloudWatchForwarderLambdaARN: | ||
| Type: String | ||
| Description: The ARN of the Axiom CloudWatch Forwarder Lambda function used to ship logs to Axiom. | ||
| AllowedPattern: ".+" # required | ||
| CloudWatchLogGroupsNames: | ||
| Type: String | ||
| Description: A comma separated list of CloudWatch log groups to subscribe to. | ||
| Default: "" # all | ||
| CloudWatchLogGroupsPrefix: | ||
| Type: String | ||
| Description: The Prefix of CloudWatch log groups to subscribe to. | ||
| Default: "" # all | ||
| CloudWatchLogGroupsPattern: | ||
| Type: String | ||
| Description: A regular expression pattern of CloudWatch log groups to subscribe to. | ||
| Default: "" # all | ||
| Resources: | ||
| SubscriberPolicy: | ||
| Type: AWS::IAM::Policy | ||
| Properties: | ||
| PolicyDocument: | ||
| Statement: | ||
| - Action: | ||
| - logs:DeleteSubscriptionFilter | ||
| - logs:PutSubscriptionFilter | ||
| - logs:DescribeLogGroups | ||
| - lambda:AddPermission | ||
| - lambda:RemovePermission | ||
| Effect: Allow | ||
| Resource: '*' | ||
| PolicyName: axiom-cloudwatch-subscriber-lambda-policy | ||
| Roles: | ||
| - !Ref 'SubscriberRole' | ||
| SubscriberRole: | ||
| Type: AWS::IAM::Role | ||
| Properties: | ||
| AssumeRolePolicyDocument: | ||
| Statement: | ||
| - Action: | ||
| - 'sts:AssumeRole' | ||
| Effect: Allow | ||
| Principal: | ||
| Service: | ||
| - lambda.amazonaws.com | ||
| ManagedPolicyArns: | ||
| - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' | ||
| SubscriberLambda: | ||
| Type: AWS::Lambda::Function | ||
| Properties: | ||
| FunctionName: !Ref LambdaFunctionName | ||
| Runtime: python3.9 | ||
| Handler: index.lambda_handler | ||
| Timeout: 300 | ||
| Code: | ||
| ZipFile: | | ||
| # DO NOT EDIT | ||
| # CI will replace these comments with the code from ./subscriber.py | ||
| Role: !GetAtt | ||
| - SubscriberRole | ||
| - Arn | ||
| Environment: | ||
| Variables: | ||
| AXIOM_CLOUDWATCH_FORWARDER_LAMBDA_ARN: !Ref 'AxiomCloudWatchForwarderLambdaARN' | ||
| LOG_GROUP_NAMES: !Ref 'CloudWatchLogGroupsNames' | ||
| LOG_GROUP_PREFIX: !Ref 'CloudWatchLogGroupsPrefix' | ||
| LOG_GROUP_PATTERN: !Ref 'CloudWatchLogGroupsPattern' | ||
| SubscriberInvoker: | ||
| Type: AWS::CloudFormation::CustomResource | ||
| DependsOn: SubscriberLambda | ||
| Version: "1.0" | ||
| Properties: | ||
| ServiceToken: !GetAtt SubscriberLambda.Arn | ||
| StackName: !Ref AWS::StackName |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| Description: A lambda that removes subscription filters for provided log groups | ||
| Metadata: | ||
| AWS::CloudFormation::Interface: | ||
| ParameterGroups: | ||
| - Label: | ||
| default: "Unsubscriber lambda" | ||
| Parameters: | ||
| - LambdaFunctionName | ||
| - Label: | ||
| default: "Forwarder lambda" | ||
| Parameters: | ||
| - AxiomCloudWatchForwarderLambdaARN | ||
| - Label: | ||
| default: "Which log groups to unsubscribe from?" | ||
| Parameters: | ||
| - CloudWatchLogGroupsNames | ||
| - CloudWatchLogGroupsPrefix | ||
| - CloudWatchLogGroupsPattern | ||
|
|
||
| Parameters: | ||
| LambdaFunctionName: | ||
| Type: String | ||
| Description: Name of the AWS Lambda function. | ||
| Default: axiom-cloudwatch-unsubscriber | ||
| AllowedPattern: ".+" # required | ||
| AxiomCloudWatchForwarderLambdaARN: | ||
| Type: String | ||
| Description: The ARN of the Axiom CloudWatch Forwarder Lambda function used to ship logs to Axiom. | ||
| AllowedPattern: ".+" # required | ||
| CloudWatchLogGroupsNames: | ||
| Type: String | ||
| Description: A comma separated list of CloudWatch log groups to unsubscribe from. | ||
| Default: "" # all | ||
| CloudWatchLogGroupsPrefix: | ||
| Type: String | ||
| Description: The Prefix of CloudWatch log groups to unsubscribe from. | ||
| Default: "" # all | ||
| CloudWatchLogGroupsPattern: | ||
| Type: String | ||
| Description: A regular expression pattern of CloudWatch log groups to unsubscribe from. | ||
| Default: "" # all | ||
| Resources: | ||
| UnsubscriberPolicy: | ||
| Type: AWS::IAM::Policy | ||
| Properties: | ||
| PolicyDocument: | ||
| Statement: | ||
| - Action: | ||
| - logs:DeleteSubscriptionFilter | ||
| - logs:PutSubscriptionFilter | ||
| - logs:DescribeLogGroups | ||
| - lambda:AddPermission | ||
| - lambda:RemovePermission | ||
| Effect: Allow | ||
| Resource: '*' | ||
| PolicyName: axiom-cloudwatch-unsubscriber-lambda-policy | ||
| Roles: | ||
| - !Ref 'UnsubscriberRole' | ||
| UnsubscriberRole: | ||
| Type: AWS::IAM::Role | ||
| Properties: | ||
| AssumeRolePolicyDocument: | ||
| Statement: | ||
| - Action: | ||
| - 'sts:AssumeRole' | ||
| Effect: Allow | ||
| Principal: | ||
| Service: | ||
| - lambda.amazonaws.com | ||
| ManagedPolicyArns: | ||
| - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' | ||
| SubscriberLambda: | ||
| Type: AWS::Lambda::Function | ||
| Properties: | ||
| FunctionName: !Ref LambdaFunctionName | ||
| Runtime: python3.9 | ||
| Handler: index.lambda_handler | ||
| Timeout: 300 | ||
| Code: | ||
| ZipFile: | | ||
| # DO NOT EDIT | ||
| # CI will replace these comments with the code from ./unsubscriber.py | ||
| Role: !GetAtt | ||
| - UnsubscriberRole | ||
| - Arn | ||
| Environment: | ||
| Variables: | ||
| AXIOM_CLOUDWATCH_FORWARDER_LAMBDA_ARN: !Ref 'AxiomCloudWatchForwarderLambdaARN' | ||
| LOG_GROUP_NAMES: !Ref 'CloudWatchLogGroupsNames' | ||
| LOG_GROUP_PREFIX: !Ref 'CloudWatchLogGroupsPrefix' | ||
| LOG_GROUP_PATTERN: !Ref 'CloudWatchLogGroupsPattern' | ||
| SubscriberInvoker: | ||
| Type: AWS::CloudFormation::CustomResource | ||
| DependsOn: SubscriberLambda | ||
| Version: "1.0" | ||
| Properties: | ||
| ServiceToken: !GetAtt UnsubscriberLambda.Arn | ||
| StackName: !Ref AWS::StackName |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| import re | ||
| import boto3 | ||
| import os | ||
| import logging | ||
| import cfnresponse | ||
|
|
||
| level = os.getenv("log_level", "INFO") | ||
| logging.basicConfig(level=level) | ||
| logger = logging.getLogger() | ||
| logger.setLevel(level) | ||
|
|
||
|
|
||
| cloudwatch_logs_client = boto3.client("logs") | ||
| lambda_client = boto3.client("lambda") | ||
|
|
||
| axiom_cloudwatch_forwarder_lambda_arn = os.getenv( | ||
| "AXIOM_CLOUDWATCH_FORWARDER_LAMBDA_ARN" | ||
| ) | ||
| log_group_names = os.getenv("LOG_GROUP_NAMES", "") | ||
| log_group_prefix = os.getenv("LOG_GROUP_PREFIX", "") | ||
| log_group_pattern = os.getenv("LOG_GROUP_PATTERN", "") | ||
|
|
||
|
|
||
| def build_groups_list(all_groups, names, pattern, prefix): | ||
| # filter out the log groups based on the names, pattern, and prefix provided in the environment variables | ||
| groups = [] | ||
| for g in all_groups: | ||
| group = {"name": g["logGroupName"], "arn": g["arn"]} | ||
| if names is not None and group["name"] in names: | ||
| groups.append(group) | ||
| continue | ||
| elif prefix is not None and group["name"].startswith(prefix): | ||
| groups.append(group) | ||
| continue | ||
| elif pattern is not None and re.match(pattern, group["name"]): | ||
| groups.append(group) | ||
|
|
||
| return groups | ||
|
|
||
|
|
||
| def get_log_groups(nextToken=None): | ||
| # check docs: | ||
| # 1. boto3 https://boto3.amazonaws.com/v1/documentation/api/1.9.42/reference/services/logs.html#CloudWatchLogs.Client.describe_log_groups | ||
| # 2. AWS API https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_DescribeLogGroups.html#API_DescribeLogGroups_RequestSyntax | ||
| resp = cloudwatch_logs_client.describe_log_groups(limit=log_groups_return_limit) | ||
| all_groups = resp["logGroups"] | ||
| nextToken = resp["nextToken"] | ||
| # continue fetching log groups until nextToken is None | ||
| while nextToken is not None: | ||
| resp = cloudwatch_logs_client.describe_log_groups( | ||
| limit=log_groups_return_limit, nextToken=nextToken | ||
| ) | ||
| all_groups.extend(resp["logGroups"]) | ||
| # print(f'got ${len(all_groups)} groups') | ||
| nextToken = resp["nextToken"] if "nextToken" in resp else None | ||
|
|
||
| return all_groups | ||
|
|
||
|
|
||
| def delete_subscription_filter(log_group_arn): | ||
| try: | ||
| log_group_name = log_group_arn.split(":")[-2] | ||
|
|
||
| logger.info(f"Deleting subscription filter for {log_group_name}...") | ||
|
|
||
| cloudwatch_logs_client.delete_subscription_filter( | ||
| logGroupName=log_group_name, filterName="%s-axiom" % log_group_name | ||
| ) | ||
|
|
||
| logger.info( | ||
| f"{log_group_name} subscription filter has been deleted successfully." | ||
| ) | ||
|
|
||
| except Exception as e: | ||
| logger.error(f"Error deleting Subscription filter: {e}") | ||
| raise e | ||
|
|
||
|
|
||
| def lambda_handler(event: dict, context=None): | ||
| if axiom_cloudwatch_forwarder_lambda_arn is None: | ||
| raise Exception("AXIOM_CLOUDWATCH_LAMBDA_FORWARDER_ARN is not set") | ||
|
|
||
| aws_account_id = context.invoked_function_arn.split(":")[4] | ||
| region = os.getenv("AWS_REGION") | ||
|
|
||
| forwarder_lambda_group_name = ( | ||
| "/aws/lambda/" + axiom_cloudwatch_forwarder_lambda_arn.split(":")[-1] | ||
| ) | ||
|
|
||
| log_groups = build_groups_list( | ||
| get_log_groups(), log_group_names, log_group_pattern, log_group_prefix | ||
| ) | ||
|
|
||
| responseData = {} | ||
| try: | ||
| for group in log_groups: | ||
| # skip the Forwarder lambda log group to avoid circular logging | ||
| if group["logGroupName"] == forwarder_lambda_group_name: | ||
| continue | ||
|
|
||
| try: | ||
| delete_subscription_filter( | ||
| region, aws_account_id, axiom_cloudwatch_forwarder_lambda_arn | ||
| ) | ||
| except Exception: | ||
| pass | ||
| except Exception as e: | ||
| responseData["success"] = "False" | ||
| if "ResponseURL" in event: | ||
| cfnresponse.send(event, context, cfnresponse.FAILED, responseData) | ||
| else: | ||
| raise e | ||
|
|
||
| responseData["success"] = "True" | ||
| if "ResponseURL" in event: | ||
| cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) | ||
| else: | ||
| return "ok" |