From bc023bc618eb71f2b6ac024f7bc276d5a3816557 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 3 Apr 2024 15:54:29 +0200 Subject: [PATCH 1/4] remove extra slash --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3d0f86f..36adc80 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -34,5 +34,5 @@ jobs: env: SOURCE_DIR: build AWS_S3_BUCKET: "axiom-cloudformation" - DEST_DIR: "/stacks" + DEST_DIR: "stacks" run: aws s3 sync --acl public-read ./$SOURCE_DIR s3://$AWS_S3_BUCKET/$DEST_DIR From 020ff4cd56a9b584242ff5396eb71fec61965798 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 3 Apr 2024 16:00:45 +0200 Subject: [PATCH 2/4] fix bucket name in cloudtrail --- ...subscriber-axiom-cloudformation-stack.template.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudwatch-subscriber-axiom-cloudformation-stack.template.yaml b/cloudwatch-subscriber-axiom-cloudformation-stack.template.yaml index 840de97..c8f7491 100644 --- a/cloudwatch-subscriber-axiom-cloudformation-stack.template.yaml +++ b/cloudwatch-subscriber-axiom-cloudformation-stack.template.yaml @@ -72,9 +72,9 @@ Resources: IncludeGlobalServiceEvents: true IsMultiRegionTrail: true IsLogging: true - S3BucketName: !Join ["-", [!Ref AWS::StackName, "axiom", "cloudtrail"]] + S3BucketName: !Ref AxiomCloudWatchLogsSubscriberS3Bucket TrailName: - !Join ["-", [!Ref AWS::StackName, "axiom", { "Ref": "AWS::AccountId" }]] + !Join ["-", [!Ref AWS::StackName, { "Ref": "AWS::AccountId" }]] AxiomLogsSubscriberEventRule: DependsOn: AxiomCloudWatchLogsSubscriber Type: AWS::Events::Rule @@ -88,10 +88,10 @@ Resources: eventName: ["CreateLogGroup"] Name: "Fn::Join": - ["-", [{ "Ref": "AWS::StackName" }, "axiom-auto-subscription-rule"]] + ["-", [{ "Ref": "AWS::StackName" }, "auto-subscription-rule"]] Targets: - Id: - !Join ["-", [!Ref "AWS::StackName", "axiom-auto-subscription-rule"]] + !Join ["-", [!Ref "AWS::StackName", "auto-subscription-rule"]] Arn: !GetAtt ["AxiomCloudWatchLogsSubscriber", "Arn"] AxiomCloudWatchLogsSubscriberPolicy: Type: AWS::IAM::Policy @@ -106,7 +106,7 @@ Resources: - lambda:RemovePermission Effect: Allow Resource: "*" - PolicyName: axiom-cloudwatch-logs-subscriber-lambda-policy + PolicyName: cloudwatch-subscriber-axiom-policy Roles: - !Ref "AxiomCloudWatchLogsSubscriberRole" AxiomCloudWatchLogsSubscriberRole: From c0b9188f7553714f323cfa5c08a182e77ae9db01 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 3 Apr 2024 16:51:35 +0200 Subject: [PATCH 3/4] fix backfill --- backfill.py | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/backfill.py b/backfill.py index bd18223..aca6ace 100644 --- a/backfill.py +++ b/backfill.py @@ -19,16 +19,26 @@ def get_log_groups(token=None): if token is None: - return cloudwatch_logs_client.describe_log_groups( - logGroupNamePrefix=log_group_prefix, limit=log_groups_return_limit - ) + if log_group_prefix != "": + return cloudwatch_logs_client.describe_log_groups( + logGroupNamePrefix=log_group_prefix, limit=log_groups_return_limit + ) + else: + return cloudwatch_logs_client.describe_log_groups( + limit=log_groups_return_limit + ) else: - return cloudwatch_logs_client.describe_log_groups( - logGroupNamePrefix=log_group_prefix, - nextToken=token, - limit=log_groups_return_limit, - ) - + if log_group_prefix != "": + return cloudwatch_logs_client.describe_log_groups( + logGroupNamePrefix=log_group_prefix, + nextToken=token, + limit=log_groups_return_limit, + ) + else: + return cloudwatch_logs_client.describe_log_groups( + nextToken=token, + limit=log_groups_return_limit, + ) def delete_subscription_filter(log_group_arn, lambda_arn): try: @@ -101,9 +111,12 @@ def log_groups(token=None): except Exception: pass - create_subscription_filter( - group["arn"], axiom_cloudwatch_lambda_ingester_arn - ) + try: + create_subscription_filter( + group["arn"], axiom_cloudwatch_lambda_ingester_arn + ) + except cloudwatch_logs_client.exceptions.LimitExceededException as error: + print(error) if token is None: return @@ -118,7 +131,13 @@ def log_groups(token=None): log_groups() except Exception as e: responseData["success"] = "False" - cfnresponse.send(event, context, cfnresponse.FAILED, responseData) + if event["ResponseURL"]: + cfnresponse.send(event, context, cfnresponse.FAILED, responseData) + else: + raise e responseData["success"] = "True" - cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) + if event["ResponseURL"]: + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) + else: + return 'ok' From 473ea02370210012e231e6e2f39af5cf8fbe8f22 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Thu, 4 Apr 2024 12:38:09 +0200 Subject: [PATCH 4/4] checkout backfill --- backfill.py | 47 ++++++++++++++--------------------------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/backfill.py b/backfill.py index aca6ace..bd18223 100644 --- a/backfill.py +++ b/backfill.py @@ -19,26 +19,16 @@ def get_log_groups(token=None): if token is None: - if log_group_prefix != "": - return cloudwatch_logs_client.describe_log_groups( - logGroupNamePrefix=log_group_prefix, limit=log_groups_return_limit - ) - else: - return cloudwatch_logs_client.describe_log_groups( - limit=log_groups_return_limit - ) + return cloudwatch_logs_client.describe_log_groups( + logGroupNamePrefix=log_group_prefix, limit=log_groups_return_limit + ) else: - if log_group_prefix != "": - return cloudwatch_logs_client.describe_log_groups( - logGroupNamePrefix=log_group_prefix, - nextToken=token, - limit=log_groups_return_limit, - ) - else: - return cloudwatch_logs_client.describe_log_groups( - nextToken=token, - limit=log_groups_return_limit, - ) + return cloudwatch_logs_client.describe_log_groups( + logGroupNamePrefix=log_group_prefix, + nextToken=token, + limit=log_groups_return_limit, + ) + def delete_subscription_filter(log_group_arn, lambda_arn): try: @@ -111,12 +101,9 @@ def log_groups(token=None): except Exception: pass - try: - create_subscription_filter( - group["arn"], axiom_cloudwatch_lambda_ingester_arn - ) - except cloudwatch_logs_client.exceptions.LimitExceededException as error: - print(error) + create_subscription_filter( + group["arn"], axiom_cloudwatch_lambda_ingester_arn + ) if token is None: return @@ -131,13 +118,7 @@ def log_groups(token=None): log_groups() except Exception as e: responseData["success"] = "False" - if event["ResponseURL"]: - cfnresponse.send(event, context, cfnresponse.FAILED, responseData) - else: - raise e + cfnresponse.send(event, context, cfnresponse.FAILED, responseData) responseData["success"] = "True" - if event["ResponseURL"]: - cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) - else: - return 'ok' + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)