From e1866f2674a01845cb35b3cfac315e313f45a35c Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Sun, 16 Jul 2023 22:37:35 -0700 Subject: [PATCH 1/9] refactor: #1218, fix: #1152 --- cartography/data/indexes.cypher | 2 - ...aws_ingest_network_interfaces_cleanup.json | 30 -- .../data/jobs/scoped_analysis/__init__.py | 0 .../aws_ec2_subnet_membership.json | 38 ++ cartography/intel/aws/ec2/instances.py | 12 +- .../intel/aws/ec2/network_interfaces.py | 401 ++++++++---------- cartography/intel/aws/ec2/security_groups.py | 4 +- cartography/intel/aws/ec2/subnets.py | 4 +- .../aws/ec2/networkinterface_instance.py | 109 +++++ .../models/aws/ec2/networkinterfaces.py | 77 ++-- .../aws/ec2/privateip_networkinterface.py | 73 ++++ ...itygroups.py => securitygroup_instance.py} | 6 +- .../aws/ec2/securitygroup_networkinterface.py | 49 +++ .../ec2/{subnets.py => subnet_instance.py} | 6 +- .../models/aws/ec2/subnet_networkinterface.py | 48 +++ cartography/util.py | 13 + .../aws/ec2/test_ec2_network_interfaces.py | 67 ++- 17 files changed, 589 insertions(+), 350 deletions(-) delete mode 100644 cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json create mode 100644 cartography/data/jobs/scoped_analysis/__init__.py create mode 100644 cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json create mode 100644 cartography/models/aws/ec2/networkinterface_instance.py create mode 100644 cartography/models/aws/ec2/privateip_networkinterface.py rename cartography/models/aws/ec2/{securitygroups.py => securitygroup_instance.py} (91%) create mode 100644 cartography/models/aws/ec2/securitygroup_networkinterface.py rename cartography/models/aws/ec2/{subnets.py => subnet_instance.py} (92%) create mode 100644 cartography/models/aws/ec2/subnet_networkinterface.py diff --git a/cartography/data/indexes.cypher b/cartography/data/indexes.cypher index 085f84e27..b47279182 100644 --- a/cartography/data/indexes.cypher +++ b/cartography/data/indexes.cypher @@ -93,8 +93,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:DOProject) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.id); CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.keyfingerprint); -CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.id); -CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.id); CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:ECRImage) ON (n.id); diff --git a/cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json b/cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json deleted file mode 100644 index 9eed8cab1..000000000 --- a/cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "statements": [ - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[:PART_OF_SUBNET]-(:NetworkInterface)-[:PRIVATE_IP_ADDRESS]->(n:EC2PrivateIp) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)", - "iterative": true, - "iterationsize": 100 - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[:PART_OF_SUBNET]-(:NetworkInterface)-[r:PRIVATE_IP_ADDRESS]->(:EC2PrivateIp) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)", - "iterative": true, - "iterationsize": 100 - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[:PART_OF_SUBNET]-(n:NetworkInterface) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)", - "iterationsize": 100, - "iterative": true - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[r:PART_OF_SUBNET]-(:NetworkInterface) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)", - "iterationsize": 100, - "iterative": true - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:LoadBalancer)-[r:PART_OF_SUBNET]->(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)", - "iterationsize": 100, - "iterative": true - } - ], - "name": "cleanup NetworkInterface" -} diff --git a/cartography/data/jobs/scoped_analysis/__init__.py b/cartography/data/jobs/scoped_analysis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json b/cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json new file mode 100644 index 000000000..c732b5493 --- /dev/null +++ b/cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json @@ -0,0 +1,38 @@ +{ + "statements": [ + { + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(i:EC2Instance)-[:NETWORK_INTERFACE]-(:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) MERGE (i)-[r:PART_OF_SUBNET]->(s) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = $UPDATE_TAG", + "iterative": true, + "iterationsize": 100, + "__comment__": "Create (:EC2Instance)-[:PART_OF_SUBNET]->(:EC2Subnet) if (:EC2Instance)--(:NetworkInterface)--(:EC2Subnet)" + }, + { + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:EC2Instance)-[r:PART_OF_SUBNET]-(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(i:LoadBalancerV2)-[:NETWORK_INTERFACE]-(:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) MERGE (i)-[r:PART_OF_SUBNET]->(s) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = $UPDATE_TAG", + "iterative": true, + "iterationsize": 100, + "__comment__": "Creates (:LoadBalancerV2)-[:PART_OF_SUBNET]->(:EC2Subnet) if (:LoadBalancerV2)--(:NetworkInterface)--(:EC2Subnet)." + }, + { + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:LoadBalancerV2)-[r:PART_OF_SUBNET]-(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(i:LoadBalancer)-[:NETWORK_INTERFACE]-(:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) MERGE (i)-[r:PART_OF_SUBNET]->(s) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = $UPDATE_TAG", + "iterative": true, + "iterationsize": 100, + "__comment__": "Creates (:LoadBalancer)-[:PART_OF_SUBNET]->(:EC2Subnet) if (:LoadBalancer)--(:NetworkInterface)--(:EC2Subnet)." + }, + { + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:LoadBalancer)-[r:PART_OF_SUBNET]-(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r", + "iterative": true, + "iterationsize": 100 + } + ], + "name": "subnet membership" +} diff --git a/cartography/intel/aws/ec2/instances.py b/cartography/intel/aws/ec2/instances.py index 87c7e3202..d288c27e6 100644 --- a/cartography/intel/aws/ec2/instances.py +++ b/cartography/intel/aws/ec2/instances.py @@ -13,10 +13,10 @@ from cartography.intel.aws.ec2.util import get_botocore_config from cartography.models.aws.ec2.instances import EC2InstanceSchema from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema -from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema from cartography.models.aws.ec2.reservations import EC2ReservationSchema -from cartography.models.aws.ec2.securitygroups import EC2SecurityGroupSchema -from cartography.models.aws.ec2.subnets import EC2SubnetSchema +from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema +from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.models.aws.ec2.volumes import EBSVolumeInstanceSchema from cartography.util import aws_handle_regions from cartography.util import timeit @@ -183,7 +183,7 @@ def load_ec2_subnets( ) -> None: load( neo4j_session, - EC2SubnetSchema(), + EC2SubnetInstanceSchema(), subnet_list, Region=region, AWS_ID=current_aws_account_id, @@ -219,7 +219,7 @@ def load_ec2_security_groups( ) -> None: load( neo4j_session, - EC2SecurityGroupSchema(), + EC2SecurityGroupInstanceSchema(), sg_list, Region=region, AWS_ID=current_aws_account_id, @@ -237,7 +237,7 @@ def load_ec2_network_interfaces( ) -> None: load( neo4j_session, - EC2NetworkInterfaceSchema(), + EC2NetworkInterfaceInstanceSchema(), network_interface_list, Region=region, AWS_ID=current_aws_account_id, diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 612d93586..a911e56a6 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -1,5 +1,7 @@ import logging import re +from collections import namedtuple +from typing import Any from typing import Dict from typing import List @@ -7,18 +9,33 @@ import neo4j from .util import get_botocore_config +from cartography.client.core.tx import load from cartography.graph.job import GraphJob +from cartography.intel.aws.ec2.instances import load_ec2_security_groups +from cartography.intel.aws.ec2.instances import load_ec2_subnets from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema +from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema +from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema +from cartography.models.aws.ec2.subnet_networkinterface import EC2SubnetNetworkInterfaceSchema from cartography.util import aws_handle_regions -from cartography.util import run_cleanup_job +from cartography.util import run_scoped_analysis_job from cartography.util import timeit logger = logging.getLogger(__name__) +Ec2NetworkData = namedtuple( + "Ec2NetworkData", [ + "network_interface_list", + "private_ip_list", + "sg_list", + "subnet_list", + ], +) + @timeit @aws_handle_regions -def get_network_interface_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]: +def get_network_interface_data(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_network_interfaces') subnets: List[Dict] = [] @@ -27,256 +44,210 @@ def get_network_interface_data(boto3_session: boto3.session.Session, region: str return subnets -@timeit -def load_network_interfaces( - neo4j_session: neo4j.Session, data: Dict, region: str, aws_account_id: str, - update_tag: int, -) -> None: - """ - Creates (:NetworkInterface), - (:NetworkInterface)-[:RESOURCE]->(:AWSAccount), - (:NetworkInterface)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(:EC2SecurityGroup), - (:NetworkInterface)-[:PART_OF_SUBNET]->(:EC2Subnet), - (:PrivateIpAddress), - (:NetworkInterface)-[:PRIVATE_IP_ADDRESS]->(:PrivateIpAddress) - """ - logger.debug("Loading %d network interfaces in %s.", len(data), region) - ingest_network_interfaces = """ - UNWIND $network_interfaces AS network_interface - MERGE (netinf:NetworkInterface{id: network_interface.NetworkInterfaceId}) - ON CREATE SET netinf.firstseen = timestamp() - SET netinf.lastupdated = $update_tag, - netinf.mac_address = network_interface.MacAddress, - netinf.description = network_interface.Description, - netinf.private_ip_address = network_interface.PrivateIpAddress, - netinf.id = network_interface.NetworkInterfaceId, - netinf.private_dns_name = network_interface.PrivateDnsName, - netinf.status = network_interface.Status, - netinf.subnetid = network_interface.SubnetId, - netinf.interface_type = network_interface.InterfaceType, - netinf.requester_managed = network_interface.RequesterManaged, - netinf.source_dest_check = network_interface.SourceDestCheck, - netinf.requester_id = network_interface.RequesterId, - netinf.public_ip = network_interface.Association.PublicIp - WITH network_interface, netinf - - UNWIND network_interface.PrivateIpAddresses AS private_ip_address - MERGE (private_ip:EC2PrivateIp{id: network_interface.NetworkInterfaceId + ':' - + private_ip_address.PrivateIpAddress}) - ON CREATE SET private_ip.firstseen = timestamp() - SET private_ip.lastupdated = $update_tag, - private_ip.network_interface_id = network_interface.NetworkInterfaceId, - private_ip.primary = private_ip_address.Primary, - private_ip.private_ip_address = private_ip_address.PrivateIpAddress, - private_ip.public_ip = private_ip_address.Association.PublicIp, - private_ip.ip_owner_id = private_ip_address.Association.IpOwnerId - - MERGE (netinf)-[r:PRIVATE_IP_ADDRESS]->(private_ip) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - WITH network_interface, netinf +def transform_network_interface_data(data_list: List[Dict[str, Any]], region: str) -> Ec2NetworkData: + network_interface_list = [] + private_ip_list = [] + sg_list = [] + subnet_list = [] - UNWIND network_interface.Groups AS security_group - MERGE (sg:EC2SecurityGroup{id: security_group.GroupId}) - ON CREATE SET sg.firstseen = timestamp() - SET sg.lastupdated = $update_tag - MERGE (netinf)-[r:MEMBER_OF_EC2_SECURITY_GROUP]->(sg) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - WITH network_interface, netinf - MERGE (acc:AWSAccount{id: $aws_account_id}) - ON CREATE SET acc.firstseen = timestamp(), acc.inscope=true - SET acc.lastupdated = $update_tag - MERGE (acc)-[r:RESOURCE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - WITH network_interface, netinf - - MERGE (snet:EC2Subnet{subnetid: network_interface.SubnetId}) - ON CREATE SET snet.firstseen = timestamp() - SET snet.lastupdated = $update_tag - MERGE (netinf)-[r:PART_OF_SUBNET]->(snet) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - neo4j_session.run( - ingest_network_interfaces, network_interfaces=data, update_tag=update_tag, - region=region, aws_account_id=aws_account_id, + for network_interface in data_list: + # Parse network interface description for ELB association + # https://aws.amazon.com/premiumsupport/knowledge-center/elb-find-load-balancer-IP/ + elb_v1_id = None + elb_v2_id = None + elb_match = re.match(r'^ELB (?:net|app)/([^\/]+)\/(.*)', network_interface.get('Description', '')) + if elb_match: + elb_v1_id = f'{elb_match[1]}-{elb_match[2]}.elb.{region}.amazonaws.com', + else: + elb_match = re.match(r'^ELB (.*)', network_interface.get('Description', '')) + if elb_match: + elb_v2_id = elb_match[1] + # TODO change this to arn when ready + network_interface_id = network_interface['NetworkInterfaceId'] + network_interface_list.append( + { + 'Id': network_interface_id, + 'NetworkInterfaceId': network_interface['NetworkInterfaceId'], + 'Description': network_interface['Description'], + 'InstanceId': network_interface.get('Attachment', {}).get('InstanceId'), + 'InterfaceType': network_interface['InterfaceType'], + 'MacAddress': network_interface['MacAddress'], + 'PrivateDnsName': network_interface['PrivateDnsName'], + 'PrivateIpAddress': network_interface['PrivateIpAddress'], + 'PublicIp': network_interface.get('Association', {}).get('PublicIp'), + 'RequesterId': network_interface.get('RequesterId'), + 'RequesterManaged': network_interface['RequesterManaged'], + 'SourceDestCheck': network_interface['SourceDestCheck'], + 'Status': network_interface['Status'], + 'SubnetId': network_interface['SubnetId'], + 'ElbV1Id': elb_v1_id, + 'ElbV2Id': elb_v2_id, + }, + ) + if network_interface.get('PrivateIpAddresses'): + for private_ip_address in network_interface['PrivateIpAddresses']: + private_ip_list.append( + { + 'Id': f"{network_interface['NetworkInterfaceId']}:{private_ip_address['PrivateIpAddress']}", + 'NetworkInterfaceId': network_interface['NetworkInterfaceId'], + 'IpOwnerId': private_ip_address.get('Association', {}).get('IpOwnerId'), + 'Primary': private_ip_address['Primary'], + 'PrivateIpAddress': private_ip_address['PrivateIpAddress'], + 'PublicIp': private_ip_address.get('Association', {}).get('PublicIp'), + }, + ) + + if network_interface.get("Groups"): + for group in network_interface["Groups"]: + sg_list.append( + { + 'GroupId': group['GroupId'], + 'Id': network_interface_id, + }, + ) + + subnet_id = network_interface.get('SubnetId') + if subnet_id: + subnet_list.append( + { + 'Id': network_interface_id, + 'SubnetId': subnet_id, + }, + ) + + return Ec2NetworkData( + network_interface_list=network_interface_list, + private_ip_list=private_ip_list, + sg_list=sg_list, + subnet_list=subnet_list, ) @timeit -def load_network_interface_instance_relations( - neo4j_session: neo4j.Session, instance_associations: List[Dict], region: str, aws_account_id: str, update_tag: int, +def load_network_interfaces( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, ) -> None: - """ - Creates (:EC2Instance)-[:NETWORK_INTERFACE]->(:NetworkInterface) - """ - ingest_network_interface_instance_relations = """ - UNWIND $instance_associations AS instance_association - MATCH (netinf:NetworkInterface{id: instance_association.netinf_id}), - (instance:EC2Instance{id: instance_association.instance_id}) - MERGE (instance)-[r:NETWORK_INTERFACE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("Attaching %d EC2 instances to network interfaces in %s.", len(instance_associations), region) - neo4j_session.run( - ingest_network_interface_instance_relations, instance_associations=instance_associations, - update_tag=update_tag, region=region, aws_account_id=aws_account_id, + logger.info("Loading %d network interfaces in %s.", len(data), region) + load( + neo4j_session, + EC2NetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) @timeit -def load_network_interface_elb_relations( - neo4j_session: neo4j.Session, elb_associations: List[Dict], region: str, - aws_account_id: str, update_tag: int, +def load_private_ips( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, ) -> None: - """ - Creates (:LoadBalancer)-[:NETWORK_INTERFACE]->(:NetworkInterface) - """ - ingest_network_interface_elb_relations = """ - UNWIND $elb_associations AS elb_association - MATCH (netinf:NetworkInterface{id: elb_association.netinf_id}), - (elb:LoadBalancer{name: elb_association.elb_name}) - MERGE (elb)-[r:NETWORK_INTERFACE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("Attaching %d ELBs to network interfaces in %s.", len(elb_associations), region) - neo4j_session.run( - ingest_network_interface_elb_relations, elb_associations=elb_associations, - update_tag=update_tag, region=region, aws_account_id=aws_account_id, + logger.info("Loading %d private IPs in %s.", len(data), region) + load( + neo4j_session, + EC2PrivateIpNetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) @timeit -def load_network_interface_elbv2_relations( - neo4j_session: neo4j.Session, elb_associations_v2: List[Dict], region: str, - aws_account_id: str, update_tag: int, +def load_security_group_network_interface( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, ) -> None: - """ - Creates (:LoadBalancerV2)-[:NETWORK_INTERFACE]->(:NetworkInterface) - """ - ingest_network_interface_elb2_relations = """ - UNWIND $elb_associations AS elb_association - MATCH (netinf:NetworkInterface{id: elb_association.netinf_id}), - (elb:LoadBalancerV2{id: elb_association.elb_id}) - MERGE (elb)-[r:NETWORK_INTERFACE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("Attaching %d ELB V2s to network interfaces in %s.", len(elb_associations_v2), region) - neo4j_session.run( - ingest_network_interface_elb2_relations, elb_associations=elb_associations_v2, - update_tag=update_tag, region=region, aws_account_id=aws_account_id, + logger.info("Loading %d security groups in %s.", len(data), region) + load( + neo4j_session, + EC2SecurityGroupNetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) @timeit -def load_network_interface_instance_to_subnet_relations(neo4j_session: neo4j.Session, update_tag: int) -> None: - """ - Creates (:EC2Instance)-[:PART_OF_SUBNET]->(:EC2Subnet) if - (:EC2Instance)--(:NetworkInterface)--(:EC2Subnet). - """ - ingest_network_interface_instance_relations = """ - MATCH (i:EC2Instance)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) - MERGE (i)-[r:PART_OF_SUBNET]->(s) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("-> Instance to subnet") - neo4j_session.run( - ingest_network_interface_instance_relations, update_tag=update_tag, - ) - - -@timeit -def load_network_interface_load_balancer_relations(neo4j_session: neo4j.Session, update_tag: int) -> None: - """ - Creates (:LoadBalancer)-[:PART_OF_SUBNET]->(:EC2Subnet) if - (:LoadBalancer)--(:NetworkInterface)--(:EC2Subnet). - """ - ingest_network_interface_loadbalancer_relations = """ - MATCH (i:LoadBalancer)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) - MERGE (i)-[r:PART_OF_SUBNET]->(s) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("-> ELB to subnet") - neo4j_session.run( - ingest_network_interface_loadbalancer_relations, update_tag=update_tag, +def load_subnet_network_interface( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, +) -> None: + logger.info("Loading %d subnets in %s.", len(data), region) + load( + neo4j_session, + EC2SubnetNetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) -@timeit -def load_network_interface_load_balancer_v2_relations(neo4j_session: neo4j.Session, update_tag: int) -> None: - """ - Creates (:LoadBalancerV2)-[:PART_OF_SUBNET]->(:EC2Subnet) if - (:LoadBalancerV2)--(:NetworkInterface)--(:EC2Subnet). - """ - ingest_network_interface_loadbalancerv2_relations = """ - MATCH (i:LoadBalancerV2)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) - MERGE (i)-[r:PART_OF_SUBNET]->(s) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("-> ELBv2 to subnet") - neo4j_session.run( - ingest_network_interface_loadbalancerv2_relations, update_tag=update_tag, - ) +def load_network_data( + neo4j_session: neo4j.Session, + region: str, + current_aws_account_id: str, + update_tag: int, + network_interface_list: List[Dict[str, Any]], + private_ip_list: List[Dict[str, Any]], + subnet_list: List[Dict[str, Any]], + sg_list: List[Dict[str, Any]], +) -> None: + load_network_interfaces(neo4j_session, network_interface_list, region, current_aws_account_id, update_tag) + load_private_ips(neo4j_session, private_ip_list, region, current_aws_account_id, update_tag) + load_ec2_subnets(neo4j_session, subnet_list, region, current_aws_account_id, update_tag) + load_ec2_security_groups(neo4j_session, sg_list, region, current_aws_account_id, update_tag) @timeit -def load(neo4j_session: neo4j.Session, data: List[Dict], region: str, aws_account_id: str, update_tag: int) -> None: - elb_associations = [] - elb_associations_v2 = [] - instance_associations = [] - - for network_interface in data: - # https://aws.amazon.com/premiumsupport/knowledge-center/elb-find-load-balancer-IP/ - matchObj = re.match(r'^ELB (?:net|app)/([^\/]+)\/(.*)', network_interface.get('Description', '')) - if matchObj: - elb_associations_v2.append({ - 'netinf_id': network_interface['NetworkInterfaceId'], - 'elb_id': f'{matchObj[1]}-{matchObj[2]}.elb.{region}.amazonaws.com', - }) - else: - matchObj = re.match(r'^ELB (.*)', network_interface.get('Description', '')) - if matchObj: - elb_associations.append({ - 'netinf_id': network_interface['NetworkInterfaceId'], - 'elb_name': matchObj[1], - }) - - if 'Attachment' in network_interface and 'InstanceId' in network_interface['Attachment']: - instance_associations.append({ - 'netinf_id': network_interface['NetworkInterfaceId'], - 'instance_id': network_interface['Attachment']['InstanceId'], - }) - load_network_interfaces(neo4j_session, data, region, aws_account_id, update_tag) # type: ignore - load_network_interface_instance_relations( - neo4j_session, instance_associations, region, aws_account_id, update_tag, - ) - load_network_interface_elb_relations(neo4j_session, elb_associations, region, aws_account_id, update_tag) - load_network_interface_elbv2_relations(neo4j_session, elb_associations_v2, region, aws_account_id, update_tag) - load_network_interface_instance_to_subnet_relations(neo4j_session, update_tag) - load_network_interface_load_balancer_relations(neo4j_session, update_tag) +def load_subnet_membership_relations(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None: + run_scoped_analysis_job('aws_ec2_subnet_membership.json', neo4j_session, common_job_parameters) @timeit def cleanup_network_interfaces(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: - run_cleanup_job('aws_ingest_network_interfaces_cleanup.json', neo4j_session, common_job_parameters) GraphJob.from_node_schema(EC2NetworkInterfaceSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2PrivateIpNetworkInterfaceSchema(), common_job_parameters).run(neo4j_session) @timeit def sync_network_interfaces( - neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str, - update_tag: int, common_job_parameters: Dict, + neo4j_session: neo4j.Session, + boto3_session: boto3.session.Session, + regions: List[str], + current_aws_account_id: str, + update_tag: int, + common_job_parameters: Dict, ) -> None: for region in regions: logger.info("Syncing EC2 network interfaces for region '%s' in account '%s'.", region, current_aws_account_id) data = get_network_interface_data(boto3_session, region) - load(neo4j_session, data, region, current_aws_account_id, update_tag) + ec2_network_data = transform_network_interface_data(data, region) + load_network_data( + neo4j_session, + region, + current_aws_account_id, + update_tag, + ec2_network_data.network_interface_list, + ec2_network_data.private_ip_list, + ec2_network_data.subnet_list, + ec2_network_data.sg_list, + ) + load_subnet_membership_relations(neo4j_session, common_job_parameters) cleanup_network_interfaces(neo4j_session, common_job_parameters) diff --git a/cartography/intel/aws/ec2/security_groups.py b/cartography/intel/aws/ec2/security_groups.py index 9819f6490..c0a66c8d2 100644 --- a/cartography/intel/aws/ec2/security_groups.py +++ b/cartography/intel/aws/ec2/security_groups.py @@ -8,7 +8,7 @@ from .util import get_botocore_config from cartography.graph.job import GraphJob -from cartography.models.aws.ec2.securitygroups import EC2SecurityGroupSchema +from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -148,7 +148,7 @@ def cleanup_ec2_security_groupinfo(neo4j_session: neo4j.Session, common_job_para neo4j_session, common_job_parameters, ) - GraphJob.from_node_schema(EC2SecurityGroupSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2SecurityGroupInstanceSchema(), common_job_parameters).run(neo4j_session) @timeit diff --git a/cartography/intel/aws/ec2/subnets.py b/cartography/intel/aws/ec2/subnets.py index a46b39c14..d30604983 100644 --- a/cartography/intel/aws/ec2/subnets.py +++ b/cartography/intel/aws/ec2/subnets.py @@ -7,7 +7,7 @@ from .util import get_botocore_config from cartography.graph.job import GraphJob -from cartography.models.aws.ec2.subnets import EC2SubnetSchema +from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -78,7 +78,7 @@ def load_subnets( @timeit def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: run_cleanup_job('aws_ingest_subnets_cleanup.json', neo4j_session, common_job_parameters) - GraphJob.from_node_schema(EC2SubnetSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(neo4j_session) @timeit diff --git a/cartography/models/aws/ec2/networkinterface_instance.py b/cartography/models/aws/ec2/networkinterface_instance.py new file mode 100644 index 000000000..3156dd973 --- /dev/null +++ b/cartography/models/aws/ec2/networkinterface_instance.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2NetworkInterfaceInstanceNodeProperties(CartographyNodeProperties): + """ + Selection of properties of a network interface as known by an EC2 instance + """ + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO decide this + id: PropertyRef = PropertyRef('NetworkInterfaceId') + status: PropertyRef = PropertyRef('Status') + mac_address: PropertyRef = PropertyRef('MacAddress') + description: PropertyRef = PropertyRef('Description') + private_dns_name: PropertyRef = PropertyRef('PrivateDnsName') + private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress') + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToAWSAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: EC2NetworkInterfaceToAwsAccountRelProperties = EC2NetworkInterfaceToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2InstanceRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2Instance(CartographyRelSchema): + target_node_label: str = 'EC2Instance' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('InstanceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "NETWORK_INTERFACE" + properties: EC2NetworkInterfaceToEC2InstanceRelProperties = EC2NetworkInterfaceToEC2InstanceRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2SubnetRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2Subnet(CartographyRelSchema): + target_node_label: str = 'EC2Subnet' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('SubnetId')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2SecurityGroupRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2SecurityGroup(CartographyRelSchema): + target_node_label: str = 'EC2SecurityGroup' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('GroupId')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" + properties: EC2NetworkInterfaceToEC2SecurityGroupRelProperties = \ + EC2NetworkInterfaceToEC2SecurityGroupRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceInstanceSchema(CartographyNodeSchema): + """ + Network interface as known by an EC2 instance + """ + label: str = 'NetworkInterface' + properties: EC2NetworkInterfaceInstanceNodeProperties = EC2NetworkInterfaceInstanceNodeProperties() + sub_resource_relationship: EC2NetworkInterfaceToAWSAccount = EC2NetworkInterfaceToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2NetworkInterfaceToEC2Instance(), + EC2NetworkInterfaceToEC2Subnet(), + EC2NetworkInterfaceToEC2SecurityGroup(), + ], + ) diff --git a/cartography/models/aws/ec2/networkinterfaces.py b/cartography/models/aws/ec2/networkinterfaces.py index 3c536407c..6c0c4fd7f 100644 --- a/cartography/models/aws/ec2/networkinterfaces.py +++ b/cartography/models/aws/ec2/networkinterfaces.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToAWSAccount, \ + EC2NetworkInterfaceToEC2Subnet, EC2NetworkInterfaceToEC2SecurityGroup, EC2NetworkInterfaceToEC2Instance from cartography.models.core.common import PropertyRef from cartography.models.core.nodes import CartographyNodeProperties from cartography.models.core.nodes import CartographyNodeSchema @@ -13,7 +15,9 @@ @dataclass(frozen=True) class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties): - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO decide this + """ + Network interface properties + """ id: PropertyRef = PropertyRef('NetworkInterfaceId') status: PropertyRef = PropertyRef('Status') mac_address: PropertyRef = PropertyRef('MacAddress') @@ -23,80 +27,61 @@ class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties): region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) - -@dataclass(frozen=True) -class EC2NetworkInterfaceToAwsAccountRelProperties(CartographyRelProperties): - lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) - - -@dataclass(frozen=True) -class EC2NetworkInterfaceToAWSAccount(CartographyRelSchema): - target_node_label: str = 'AWSAccount' - target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, - ) - direction: LinkDirection = LinkDirection.INWARD - rel_label: str = "RESOURCE" - properties: EC2NetworkInterfaceToAwsAccountRelProperties = EC2NetworkInterfaceToAwsAccountRelProperties() + # Properties only returned by describe network interfaces + subnetid: PropertyRef = PropertyRef('SubnetId') + interface_type: PropertyRef = PropertyRef('InterfaceType') + requester_managed: PropertyRef = PropertyRef('RequesterManaged') + requester_id: PropertyRef = PropertyRef('RequesterId') + source_dest_check: PropertyRef = PropertyRef('SourceDestCheck') + public_ip: PropertyRef = PropertyRef('PublicIp') @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2InstanceRelProperties(CartographyRelProperties): +class EC2NetworkInterfaceToElbRelProperties(CartographyRelProperties): lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2Instance(CartographyRelSchema): - target_node_label: str = 'EC2Instance' +class EC2NetworkInterfaceToElb(CartographyRelSchema): + target_node_label: str = 'LoadBalancer' target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('InstanceId')}, + {'name': PropertyRef('ElbV1Id')}, ) direction: LinkDirection = LinkDirection.INWARD rel_label: str = "NETWORK_INTERFACE" - properties: EC2NetworkInterfaceToEC2InstanceRelProperties = EC2NetworkInterfaceToEC2InstanceRelProperties() + properties: EC2NetworkInterfaceToElbRelProperties = EC2NetworkInterfaceToElbRelProperties() @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2SubnetRelProperties(CartographyRelProperties): +class EC2NetworkInterfaceToElbV2RelProperties(CartographyRelProperties): lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2Subnet(CartographyRelSchema): - target_node_label: str = 'EC2Subnet' +class EC2NetworkInterfaceToElbV2(CartographyRelSchema): + target_node_label: str = 'LoadBalancerV2' target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('SubnetId')}, + {'id': PropertyRef('ElbV2Id')}, ) - direction: LinkDirection = LinkDirection.OUTWARD - rel_label: str = "PART_OF_SUBNET" - properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties() - - -@dataclass(frozen=True) -class EC2NetworkInterfaceToEC2SecurityGroupRelProperties(CartographyRelProperties): - lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) - - -@dataclass(frozen=True) -class EC2NetworkInterfaceToEC2SecurityGroup(CartographyRelSchema): - target_node_label: str = 'EC2SecurityGroup' - target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('GroupId')}, - ) - direction: LinkDirection = LinkDirection.OUTWARD - rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" - properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties() + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "NETWORK_INTERFACE" + properties: EC2NetworkInterfaceToElbV2RelProperties = EC2NetworkInterfaceToElbV2RelProperties() @dataclass(frozen=True) class EC2NetworkInterfaceSchema(CartographyNodeSchema): + """ + Network interface as known from describe network interfaces. + """ label: str = 'NetworkInterface' properties: EC2NetworkInterfaceNodeProperties = EC2NetworkInterfaceNodeProperties() sub_resource_relationship: EC2NetworkInterfaceToAWSAccount = EC2NetworkInterfaceToAWSAccount() other_relationships: OtherRelationships = OtherRelationships( [ - EC2NetworkInterfaceToEC2Instance(), EC2NetworkInterfaceToEC2Subnet(), EC2NetworkInterfaceToEC2SecurityGroup(), + EC2NetworkInterfaceToElb(), + EC2NetworkInterfaceToElbV2(), + EC2NetworkInterfaceToEC2Instance(), ], - ) + ) \ No newline at end of file diff --git a/cartography/models/aws/ec2/privateip_networkinterface.py b/cartography/models/aws/ec2/privateip_networkinterface.py new file mode 100644 index 000000000..1ed0e6de0 --- /dev/null +++ b/cartography/models/aws/ec2/privateip_networkinterface.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2NetworkInterfacePrivateIpNodeProperties(CartographyNodeProperties): + """ + Selection of properties of a private IP as known by an EC2 network interface + """ + id: PropertyRef = PropertyRef('Id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + network_interface_id: PropertyRef = PropertyRef('NetworkInterfaceId') + primary: PropertyRef = PropertyRef('Primary') + private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress') + public_ip: PropertyRef = PropertyRef('PublicIp') + ip_owner_id: PropertyRef = PropertyRef('IpOwnerId') + + +@dataclass(frozen=True) +class EC2PrivateIpToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2PrivateIpToAWSAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: EC2PrivateIpToAwsAccountRelProperties = EC2PrivateIpToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToPrivateIpRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToPrivateIp(CartographyRelSchema): + target_node_label: str = 'NetworkInterface' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('NetworkInterfaceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PRIVATE_IP_ADDRESS" + properties: EC2NetworkInterfaceToPrivateIpRelProperties = EC2NetworkInterfaceToPrivateIpRelProperties() + + +@dataclass(frozen=True) +class EC2PrivateIpNetworkInterfaceSchema(CartographyNodeSchema): + """ + PrivateIp as known by a Network Interface + """ + label: str = 'EC2PrivateIp' + properties: EC2NetworkInterfacePrivateIpNodeProperties = EC2NetworkInterfacePrivateIpNodeProperties() + sub_resource_relationship: EC2PrivateIpToAWSAccount = EC2PrivateIpToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2NetworkInterfaceToPrivateIp(), + ], + ) + diff --git a/cartography/models/aws/ec2/securitygroups.py b/cartography/models/aws/ec2/securitygroup_instance.py similarity index 91% rename from cartography/models/aws/ec2/securitygroups.py rename to cartography/models/aws/ec2/securitygroup_instance.py index 34813340c..82d585e22 100644 --- a/cartography/models/aws/ec2/securitygroups.py +++ b/cartography/models/aws/ec2/securitygroup_instance.py @@ -12,7 +12,7 @@ @dataclass(frozen=True) -class EC2SecurityGroupNodeProperties(CartographyNodeProperties): +class EC2SecurityGroupInstanceNodeProperties(CartographyNodeProperties): # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO decide on this id: PropertyRef = PropertyRef('GroupId') groupid: PropertyRef = PropertyRef('GroupId', extra_index=True) @@ -53,9 +53,9 @@ class EC2SecurityGroupToEC2Instance(CartographyRelSchema): @dataclass(frozen=True) -class EC2SecurityGroupSchema(CartographyNodeSchema): +class EC2SecurityGroupInstanceSchema(CartographyNodeSchema): label: str = 'EC2SecurityGroup' - properties: EC2SecurityGroupNodeProperties = EC2SecurityGroupNodeProperties() + properties: EC2SecurityGroupInstanceNodeProperties = EC2SecurityGroupInstanceNodeProperties() sub_resource_relationship: EC2SecurityGroupToAWSAccount = EC2SecurityGroupToAWSAccount() other_relationships: OtherRelationships = OtherRelationships( [ diff --git a/cartography/models/aws/ec2/securitygroup_networkinterface.py b/cartography/models/aws/ec2/securitygroup_networkinterface.py new file mode 100644 index 000000000..9144391ed --- /dev/null +++ b/cartography/models/aws/ec2/securitygroup_networkinterface.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass + +from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupToAWSAccount +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2SecurityGroupNetworkInterfaceNodeProperties(CartographyNodeProperties): + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO decide on this + id: PropertyRef = PropertyRef('GroupId') + groupid: PropertyRef = PropertyRef('GroupId', extra_index=True) + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToNetworkInterfaceRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SecurityGroupToNetworkInterface(CartographyRelSchema): + target_node_label: str = 'NetworkInterface' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('NetworkInterfaceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" + properties: EC2SubnetToNetworkInterfaceRelProperties = EC2SubnetToNetworkInterfaceRelProperties() + + +@dataclass(frozen=True) +class EC2SecurityGroupNetworkInterfaceSchema(CartographyNodeSchema): + label: str = 'EC2SecurityGroup' + properties: EC2SecurityGroupNetworkInterfaceNodeProperties = EC2SecurityGroupNetworkInterfaceNodeProperties() + sub_resource_relationship: EC2SecurityGroupToAWSAccount = EC2SecurityGroupToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2SecurityGroupToNetworkInterface(), + ], + ) diff --git a/cartography/models/aws/ec2/subnets.py b/cartography/models/aws/ec2/subnet_instance.py similarity index 92% rename from cartography/models/aws/ec2/subnets.py rename to cartography/models/aws/ec2/subnet_instance.py index 62db69909..407aea210 100644 --- a/cartography/models/aws/ec2/subnets.py +++ b/cartography/models/aws/ec2/subnet_instance.py @@ -12,7 +12,7 @@ @dataclass(frozen=True) -class EC2SubnetNodeProperties(CartographyNodeProperties): +class EC2SubnetInstanceNodeProperties(CartographyNodeProperties): # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO decide this id: PropertyRef = PropertyRef('SubnetId') subnet_id: PropertyRef = PropertyRef('SubnetId', extra_index=True) @@ -53,9 +53,9 @@ class EC2SubnetToEC2Instance(CartographyRelSchema): @dataclass(frozen=True) -class EC2SubnetSchema(CartographyNodeSchema): +class EC2SubnetInstanceSchema(CartographyNodeSchema): label: str = 'EC2Subnet' - properties: EC2SubnetNodeProperties = EC2SubnetNodeProperties() + properties: EC2SubnetInstanceNodeProperties = EC2SubnetInstanceNodeProperties() sub_resource_relationship: EC2SubnetToAWSAccount = EC2SubnetToAWSAccount() other_relationships: OtherRelationships = OtherRelationships( [ diff --git a/cartography/models/aws/ec2/subnet_networkinterface.py b/cartography/models/aws/ec2/subnet_networkinterface.py new file mode 100644 index 000000000..a20dd1f32 --- /dev/null +++ b/cartography/models/aws/ec2/subnet_networkinterface.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass + +from cartography.models.aws.ec2.subnet_instance import EC2SubnetToAWSAccount +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2SubnetNetworkInterfaceNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('SubnetId') + subnet_id: PropertyRef = PropertyRef('SubnetId', extra_index=True) + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToNetworkInterfaceRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToNetworkInterface(CartographyRelSchema): + target_node_label: str = 'NetworkInterface' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('NetworkInterfaceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2SubnetToNetworkInterfaceRelProperties = EC2SubnetToNetworkInterfaceRelProperties() + + +@dataclass(frozen=True) +class EC2SubnetNetworkInterfaceSchema(CartographyNodeSchema): + label: str = 'EC2Subnet' + properties: EC2SubnetNetworkInterfaceNodeProperties = EC2SubnetNetworkInterfaceNodeProperties() + sub_resource_relationship: EC2SubnetToAWSAccount = EC2SubnetToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2SubnetToNetworkInterface(), + ], + ) diff --git a/cartography/util.py b/cartography/util.py index 30b9bccd8..588c519f3 100644 --- a/cartography/util.py +++ b/cartography/util.py @@ -60,6 +60,19 @@ def run_analysis_job( ) +def run_scoped_analysis_job( + filename: str, + neo4j_session: neo4j.Session, + common_job_parameters: Dict, +) -> None: + GraphJob.run_from_json( + neo4j_session, + read_text('cartography.data.jobs.scoped_analysis', filename), + common_job_parameters, + get_job_shortname(filename), + ) + + def run_analysis_and_ensure_deps( analysis_job_name: str, resource_dependencies: Set[str], diff --git a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py index 1c5d271ad..527287165 100644 --- a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py +++ b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py @@ -1,22 +1,37 @@ -import cartography.intel.aws.ec2 -import tests.data.aws.ec2.network_interfaces +from unittest.mock import MagicMock +from unittest.mock import patch +import cartography.intel.aws.ec2.network_interfaces +from cartography.intel.aws.ec2.network_interfaces import sync_network_interfaces +from tests.data.aws.ec2.network_interfaces import DESCRIBE_NETWORK_INTERFACES +from tests.integration.cartography.intel.aws.common import create_test_account TEST_ACCOUNT_ID = '000000000000' TEST_REGION = 'eu-north-1' TEST_UPDATE_TAG = 123456789 -def test_load_network_interfaces(neo4j_session): - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( +@patch.object( + cartography.intel.aws.ec2.network_interfaces, + 'get_network_interface_data', + return_value=DESCRIBE_NETWORK_INTERFACES, +) +def test_load_network_interfaces(mock_get_network_interfaces, neo4j_session): + # Arrange + boto3_session = MagicMock() + create_test_account(neo4j_session, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) + + # Act + sync_network_interfaces( neo4j_session, - data, - TEST_REGION, + boto3_session, + [TEST_REGION], TEST_ACCOUNT_ID, TEST_UPDATE_TAG, + {'UPDATE_TAG': TEST_UPDATE_TAG, 'AWS_ID': TEST_ACCOUNT_ID}, ) + # Assert 1 expected_nodes = { "eni-0e106a07c15ff7d14", "eni-0d9877f559c240362", @@ -32,17 +47,9 @@ def test_load_network_interfaces(neo4j_session): assert actual_nodes == expected_nodes + # TODO use test helpers -def test_ec2_private_ips(neo4j_session): - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( - neo4j_session, - data, - TEST_REGION, - TEST_ACCOUNT_ID, - TEST_UPDATE_TAG, - ) - + # Assert 2 expected_nodes = { "eni-0e106a07c15ff7d14:10.0.4.180", "eni-0d9877f559c240362:10.0.4.96", @@ -59,17 +66,7 @@ def test_ec2_private_ips(neo4j_session): assert actual_nodes == expected_nodes - -def test_network_interface_relationships(neo4j_session): - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( - neo4j_session, - data, - TEST_REGION, - TEST_ACCOUNT_ID, - TEST_UPDATE_TAG, - ) - + # Assert 3 expected_nodes = { ('eni-0e106a07c15ff7d14', 'eni-0e106a07c15ff7d14:10.0.4.180'), ('eni-0d9877f559c240362', 'eni-0d9877f559c240362:10.0.4.96'), @@ -89,19 +86,7 @@ def test_network_interface_relationships(neo4j_session): assert actual == expected_nodes - -def test_network_interface_to_account(neo4j_session): - neo4j_session.run('MERGE (:AWSAccount{id:$ACC_ID})', ACC_ID=TEST_ACCOUNT_ID) - - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( - neo4j_session, - data, - TEST_REGION, - TEST_ACCOUNT_ID, - TEST_UPDATE_TAG, - ) - + # Assert 4 expected_nodes = { ('eni-0e106a07c15ff7d14', TEST_ACCOUNT_ID), ('eni-0d9877f559c240362', TEST_ACCOUNT_ID), From 9229cad48f7d614bf7b567f9527613c5e94b30a6 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Sun, 16 Jul 2023 22:41:48 -0700 Subject: [PATCH 2/9] lint --- cartography/models/aws/ec2/networkinterfaces.py | 8 +++++--- cartography/models/aws/ec2/privateip_networkinterface.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cartography/models/aws/ec2/networkinterfaces.py b/cartography/models/aws/ec2/networkinterfaces.py index 6c0c4fd7f..fa505eec8 100644 --- a/cartography/models/aws/ec2/networkinterfaces.py +++ b/cartography/models/aws/ec2/networkinterfaces.py @@ -1,7 +1,9 @@ from dataclasses import dataclass -from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToAWSAccount, \ - EC2NetworkInterfaceToEC2Subnet, EC2NetworkInterfaceToEC2SecurityGroup, EC2NetworkInterfaceToEC2Instance +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToAWSAccount +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2Instance +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2SecurityGroup +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2Subnet from cartography.models.core.common import PropertyRef from cartography.models.core.nodes import CartographyNodeProperties from cartography.models.core.nodes import CartographyNodeSchema @@ -84,4 +86,4 @@ class EC2NetworkInterfaceSchema(CartographyNodeSchema): EC2NetworkInterfaceToElbV2(), EC2NetworkInterfaceToEC2Instance(), ], - ) \ No newline at end of file + ) diff --git a/cartography/models/aws/ec2/privateip_networkinterface.py b/cartography/models/aws/ec2/privateip_networkinterface.py index 1ed0e6de0..988511b82 100644 --- a/cartography/models/aws/ec2/privateip_networkinterface.py +++ b/cartography/models/aws/ec2/privateip_networkinterface.py @@ -70,4 +70,3 @@ class EC2PrivateIpNetworkInterfaceSchema(CartographyNodeSchema): EC2NetworkInterfaceToPrivateIp(), ], ) - From 985107c82ca474b395816bba6f03147307ce858f Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 17 Jul 2023 10:05:14 -0700 Subject: [PATCH 3/9] names, tests --- .../intel/aws/ec2/network_interfaces.py | 33 +++--- .../aws/ec2/networkinterface_instance.py | 2 +- .../models/aws/ec2/networkinterfaces.py | 16 +-- .../aws/ec2/privateip_networkinterface.py | 8 +- .../models/aws/ec2/securitygroup_instance.py | 9 +- .../aws/ec2/securitygroup_networkinterface.py | 5 +- cartography/models/aws/ec2/subnet_instance.py | 5 +- .../models/aws/ec2/subnet_networkinterface.py | 3 + .../aws/ec2/test_ec2_network_interfaces.py | 104 ++++++++---------- 9 files changed, 97 insertions(+), 88 deletions(-) diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index a911e56a6..211bca3ac 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -11,8 +11,6 @@ from .util import get_botocore_config from cartography.client.core.tx import load from cartography.graph.job import GraphJob -from cartography.intel.aws.ec2.instances import load_ec2_security_groups -from cartography.intel.aws.ec2.instances import load_ec2_subnets from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema @@ -62,7 +60,7 @@ def transform_network_interface_data(data_list: List[Dict[str, Any]], region: st elb_match = re.match(r'^ELB (.*)', network_interface.get('Description', '')) if elb_match: elb_v2_id = elb_match[1] - # TODO change this to arn when ready + # TODO issue #1024 change this to arn when ready network_interface_id = network_interface['NetworkInterfaceId'] network_interface_list.append( { @@ -110,7 +108,7 @@ def transform_network_interface_data(data_list: List[Dict[str, Any]], region: st if subnet_id: subnet_list.append( { - 'Id': network_interface_id, + 'NetworkInterfaceId': network_interface_id, 'SubnetId': subnet_id, }, ) @@ -131,7 +129,7 @@ def load_network_interfaces( aws_account_id: str, update_tag: int, ) -> None: - logger.info("Loading %d network interfaces in %s.", len(data), region) + logger.info(f"Loading {len(data)} network interfaces in {region}.") load( neo4j_session, EC2NetworkInterfaceSchema(), @@ -143,14 +141,17 @@ def load_network_interfaces( @timeit -def load_private_ips( +def load_private_ip_network_interface( neo4j_session: neo4j.Session, data: List[Dict[str, Any]], region: str, aws_account_id: str, update_tag: int, ) -> None: - logger.info("Loading %d private IPs in %s.", len(data), region) + """ + Private IPs as known by describe-network-interfaces. + """ + logger.info(f"Loading {len(data)} private IPs in {region}.") load( neo4j_session, EC2PrivateIpNetworkInterfaceSchema(), @@ -169,7 +170,10 @@ def load_security_group_network_interface( aws_account_id: str, update_tag: int, ) -> None: - logger.info("Loading %d security groups in %s.", len(data), region) + """ + Security groups as known by describe-network-interfaces. + """ + logger.info(f"Loading {len(data)} security groups in {region}.") load( neo4j_session, EC2SecurityGroupNetworkInterfaceSchema(), @@ -188,7 +192,10 @@ def load_subnet_network_interface( aws_account_id: str, update_tag: int, ) -> None: - logger.info("Loading %d subnets in %s.", len(data), region) + """ + Subnets as known by describe-network-interfaces. + """ + logger.info(f"Loading {len(data)} subnets in {region}.") load( neo4j_session, EC2SubnetNetworkInterfaceSchema(), @@ -210,9 +217,9 @@ def load_network_data( sg_list: List[Dict[str, Any]], ) -> None: load_network_interfaces(neo4j_session, network_interface_list, region, current_aws_account_id, update_tag) - load_private_ips(neo4j_session, private_ip_list, region, current_aws_account_id, update_tag) - load_ec2_subnets(neo4j_session, subnet_list, region, current_aws_account_id, update_tag) - load_ec2_security_groups(neo4j_session, sg_list, region, current_aws_account_id, update_tag) + load_private_ip_network_interface(neo4j_session, private_ip_list, region, current_aws_account_id, update_tag) + load_subnet_network_interface(neo4j_session, subnet_list, region, current_aws_account_id, update_tag) + load_security_group_network_interface(neo4j_session, sg_list, region, current_aws_account_id, update_tag) @timeit @@ -236,7 +243,7 @@ def sync_network_interfaces( common_job_parameters: Dict, ) -> None: for region in regions: - logger.info("Syncing EC2 network interfaces for region '%s' in account '%s'.", region, current_aws_account_id) + logger.info(f"Syncing EC2 network interfaces for region '{region}' in account '{current_aws_account_id}'.") data = get_network_interface_data(boto3_session, region) ec2_network_data = transform_network_interface_data(data, region) load_network_data( diff --git a/cartography/models/aws/ec2/networkinterface_instance.py b/cartography/models/aws/ec2/networkinterface_instance.py index 3156dd973..8323cf5b4 100644 --- a/cartography/models/aws/ec2/networkinterface_instance.py +++ b/cartography/models/aws/ec2/networkinterface_instance.py @@ -16,7 +16,7 @@ class EC2NetworkInterfaceInstanceNodeProperties(CartographyNodeProperties): """ Selection of properties of a network interface as known by an EC2 instance """ - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO decide this + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024 id: PropertyRef = PropertyRef('NetworkInterfaceId') status: PropertyRef = PropertyRef('Status') mac_address: PropertyRef = PropertyRef('MacAddress') diff --git a/cartography/models/aws/ec2/networkinterfaces.py b/cartography/models/aws/ec2/networkinterfaces.py index fa505eec8..f5e96adf7 100644 --- a/cartography/models/aws/ec2/networkinterfaces.py +++ b/cartography/models/aws/ec2/networkinterfaces.py @@ -21,21 +21,21 @@ class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties): Network interface properties """ id: PropertyRef = PropertyRef('NetworkInterfaceId') - status: PropertyRef = PropertyRef('Status') - mac_address: PropertyRef = PropertyRef('MacAddress') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) description: PropertyRef = PropertyRef('Description') + mac_address: PropertyRef = PropertyRef('MacAddress') private_dns_name: PropertyRef = PropertyRef('PrivateDnsName') private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress') region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) - lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + status: PropertyRef = PropertyRef('Status') - # Properties only returned by describe network interfaces - subnetid: PropertyRef = PropertyRef('SubnetId') + # Properties only returned by describe-network-interfaces interface_type: PropertyRef = PropertyRef('InterfaceType') - requester_managed: PropertyRef = PropertyRef('RequesterManaged') + public_ip: PropertyRef = PropertyRef('PublicIp') requester_id: PropertyRef = PropertyRef('RequesterId') + requester_managed: PropertyRef = PropertyRef('RequesterManaged') source_dest_check: PropertyRef = PropertyRef('SourceDestCheck') - public_ip: PropertyRef = PropertyRef('PublicIp') + subnetid: PropertyRef = PropertyRef('SubnetId') @dataclass(frozen=True) @@ -73,7 +73,7 @@ class EC2NetworkInterfaceToElbV2(CartographyRelSchema): @dataclass(frozen=True) class EC2NetworkInterfaceSchema(CartographyNodeSchema): """ - Network interface as known from describe network interfaces. + Network interface as known by describe-network-interfaces. """ label: str = 'NetworkInterface' properties: EC2NetworkInterfaceNodeProperties = EC2NetworkInterfaceNodeProperties() diff --git a/cartography/models/aws/ec2/privateip_networkinterface.py b/cartography/models/aws/ec2/privateip_networkinterface.py index 988511b82..1200520fb 100644 --- a/cartography/models/aws/ec2/privateip_networkinterface.py +++ b/cartography/models/aws/ec2/privateip_networkinterface.py @@ -12,7 +12,7 @@ @dataclass(frozen=True) -class EC2NetworkInterfacePrivateIpNodeProperties(CartographyNodeProperties): +class EC2PrivateIpNetworkInterfaceNodeProperties(CartographyNodeProperties): """ Selection of properties of a private IP as known by an EC2 network interface """ @@ -47,7 +47,7 @@ class EC2NetworkInterfaceToPrivateIpRelProperties(CartographyRelProperties): @dataclass(frozen=True) -class EC2NetworkInterfaceToPrivateIp(CartographyRelSchema): +class EC2PrivateIpToNetworkInterface(CartographyRelSchema): target_node_label: str = 'NetworkInterface' target_node_matcher: TargetNodeMatcher = make_target_node_matcher( {'id': PropertyRef('NetworkInterfaceId')}, @@ -63,10 +63,10 @@ class EC2PrivateIpNetworkInterfaceSchema(CartographyNodeSchema): PrivateIp as known by a Network Interface """ label: str = 'EC2PrivateIp' - properties: EC2NetworkInterfacePrivateIpNodeProperties = EC2NetworkInterfacePrivateIpNodeProperties() + properties: EC2PrivateIpNetworkInterfaceNodeProperties = EC2PrivateIpNetworkInterfaceNodeProperties() sub_resource_relationship: EC2PrivateIpToAWSAccount = EC2PrivateIpToAWSAccount() other_relationships: OtherRelationships = OtherRelationships( [ - EC2NetworkInterfaceToPrivateIp(), + EC2PrivateIpToNetworkInterface(), ], ) diff --git a/cartography/models/aws/ec2/securitygroup_instance.py b/cartography/models/aws/ec2/securitygroup_instance.py index 82d585e22..041085d61 100644 --- a/cartography/models/aws/ec2/securitygroup_instance.py +++ b/cartography/models/aws/ec2/securitygroup_instance.py @@ -13,7 +13,7 @@ @dataclass(frozen=True) class EC2SecurityGroupInstanceNodeProperties(CartographyNodeProperties): - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO decide on this + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO use arn; #1024 id: PropertyRef = PropertyRef('GroupId') groupid: PropertyRef = PropertyRef('GroupId', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) @@ -37,7 +37,7 @@ class EC2SecurityGroupToAWSAccount(CartographyRelSchema): @dataclass(frozen=True) -class EC2SubnetToEC2InstanceRelProperties(CartographyRelProperties): +class EC2SecurityGroupToEC2InstanceRelProperties(CartographyRelProperties): lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) @@ -49,11 +49,14 @@ class EC2SecurityGroupToEC2Instance(CartographyRelSchema): ) direction: LinkDirection = LinkDirection.INWARD rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" - properties: EC2SubnetToEC2InstanceRelProperties = EC2SubnetToEC2InstanceRelProperties() + properties: EC2SecurityGroupToEC2InstanceRelProperties = EC2SecurityGroupToEC2InstanceRelProperties() @dataclass(frozen=True) class EC2SecurityGroupInstanceSchema(CartographyNodeSchema): + """ + Security groups as known by describe-ec2-instances + """ label: str = 'EC2SecurityGroup' properties: EC2SecurityGroupInstanceNodeProperties = EC2SecurityGroupInstanceNodeProperties() sub_resource_relationship: EC2SecurityGroupToAWSAccount = EC2SecurityGroupToAWSAccount() diff --git a/cartography/models/aws/ec2/securitygroup_networkinterface.py b/cartography/models/aws/ec2/securitygroup_networkinterface.py index 9144391ed..01178f9ee 100644 --- a/cartography/models/aws/ec2/securitygroup_networkinterface.py +++ b/cartography/models/aws/ec2/securitygroup_networkinterface.py @@ -14,7 +14,7 @@ @dataclass(frozen=True) class EC2SecurityGroupNetworkInterfaceNodeProperties(CartographyNodeProperties): - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO decide on this + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO use arn; issue #1024 id: PropertyRef = PropertyRef('GroupId') groupid: PropertyRef = PropertyRef('GroupId', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) @@ -39,6 +39,9 @@ class EC2SecurityGroupToNetworkInterface(CartographyRelSchema): @dataclass(frozen=True) class EC2SecurityGroupNetworkInterfaceSchema(CartographyNodeSchema): + """ + Security groups as known by describe-network-interfaces. + """ label: str = 'EC2SecurityGroup' properties: EC2SecurityGroupNetworkInterfaceNodeProperties = EC2SecurityGroupNetworkInterfaceNodeProperties() sub_resource_relationship: EC2SecurityGroupToAWSAccount = EC2SecurityGroupToAWSAccount() diff --git a/cartography/models/aws/ec2/subnet_instance.py b/cartography/models/aws/ec2/subnet_instance.py index 407aea210..9fe804087 100644 --- a/cartography/models/aws/ec2/subnet_instance.py +++ b/cartography/models/aws/ec2/subnet_instance.py @@ -13,7 +13,7 @@ @dataclass(frozen=True) class EC2SubnetInstanceNodeProperties(CartographyNodeProperties): - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO decide this + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024 id: PropertyRef = PropertyRef('SubnetId') subnet_id: PropertyRef = PropertyRef('SubnetId', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) @@ -54,6 +54,9 @@ class EC2SubnetToEC2Instance(CartographyRelSchema): @dataclass(frozen=True) class EC2SubnetInstanceSchema(CartographyNodeSchema): + """ + EC2 Subnet as known by describe-ec2-instances + """ label: str = 'EC2Subnet' properties: EC2SubnetInstanceNodeProperties = EC2SubnetInstanceNodeProperties() sub_resource_relationship: EC2SubnetToAWSAccount = EC2SubnetToAWSAccount() diff --git a/cartography/models/aws/ec2/subnet_networkinterface.py b/cartography/models/aws/ec2/subnet_networkinterface.py index a20dd1f32..cd9005ba9 100644 --- a/cartography/models/aws/ec2/subnet_networkinterface.py +++ b/cartography/models/aws/ec2/subnet_networkinterface.py @@ -38,6 +38,9 @@ class EC2SubnetToNetworkInterface(CartographyRelSchema): @dataclass(frozen=True) class EC2SubnetNetworkInterfaceSchema(CartographyNodeSchema): + """ + Subnet as known by describe-network-interfaces + """ label: str = 'EC2Subnet' properties: EC2SubnetNetworkInterfaceNodeProperties = EC2SubnetNetworkInterfaceNodeProperties() sub_resource_relationship: EC2SubnetToAWSAccount = EC2SubnetToAWSAccount() diff --git a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py index 527287165..4e8ec01de 100644 --- a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py +++ b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py @@ -5,6 +5,8 @@ from cartography.intel.aws.ec2.network_interfaces import sync_network_interfaces from tests.data.aws.ec2.network_interfaces import DESCRIBE_NETWORK_INTERFACES from tests.integration.cartography.intel.aws.common import create_test_account +from tests.integration.util import check_nodes +from tests.integration.util import check_rels TEST_ACCOUNT_ID = '000000000000' TEST_REGION = 'eu-north-1' @@ -31,77 +33,65 @@ def test_load_network_interfaces(mock_get_network_interfaces, neo4j_session): {'UPDATE_TAG': TEST_UPDATE_TAG, 'AWS_ID': TEST_ACCOUNT_ID}, ) - # Assert 1 - expected_nodes = { - "eni-0e106a07c15ff7d14", - "eni-0d9877f559c240362", - "eni-04b4289e1be7634e4", + # Assert NetworkInterfaces were created + assert check_nodes(neo4j_session, 'NetworkInterface', ['id']) == { + ("eni-0e106a07c15ff7d14",), + ("eni-0d9877f559c240362",), + ("eni-04b4289e1be7634e4",), } - nodes = neo4j_session.run( - """ - MATCH (ni:NetworkInterface) RETURN ni.id; - """, - ) - actual_nodes = {n['ni.id'] for n in nodes} - - assert actual_nodes == expected_nodes - - # TODO use test helpers - - # Assert 2 - expected_nodes = { - "eni-0e106a07c15ff7d14:10.0.4.180", - "eni-0d9877f559c240362:10.0.4.96", - "eni-04b4289e1be7634e4:10.0.4.5", - "eni-04b4289e1be7634e4:10.0.4.12", + # Assert EC2PrivateIps were created + assert check_nodes(neo4j_session, 'EC2PrivateIp', ['id']) == { + ("eni-0e106a07c15ff7d14:10.0.4.180",), + ("eni-0d9877f559c240362:10.0.4.96",), + ("eni-04b4289e1be7634e4:10.0.4.5",), + ("eni-04b4289e1be7634e4:10.0.4.12",), } - nodes = neo4j_session.run( - """ - MATCH (ni:EC2PrivateIp) RETURN ni.id; - """, - ) - actual_nodes = {n['ni.id'] for n in nodes} - - assert actual_nodes == expected_nodes - - # Assert 3 - expected_nodes = { + # Assert NetworkInterface to PrivateIp rels exist + assert check_rels( + neo4j_session, + 'NetworkInterface', + 'id', + 'EC2PrivateIp', + 'id', + 'PRIVATE_IP_ADDRESS', + rel_direction_right=True, + ) == { ('eni-0e106a07c15ff7d14', 'eni-0e106a07c15ff7d14:10.0.4.180'), ('eni-0d9877f559c240362', 'eni-0d9877f559c240362:10.0.4.96'), ('eni-04b4289e1be7634e4', 'eni-04b4289e1be7634e4:10.0.4.5'), ('eni-04b4289e1be7634e4', 'eni-04b4289e1be7634e4:10.0.4.12'), } - # Fetch relationships - result = neo4j_session.run( - """ - MATCH (n1:NetworkInterface)-[:PRIVATE_IP_ADDRESS]->(n2:EC2PrivateIp) RETURN n1.id, n2.id; - """, - ) - actual = { - (r['n1.id'], r['n2.id']) for r in result - } - - assert actual == expected_nodes - - # Assert 4 - expected_nodes = { + # Assert NetworkInterface to AWSAccount rels exist + assert check_rels( + neo4j_session, + 'NetworkInterface', + 'id', + 'AWSAccount', + 'id', + 'RESOURCE', + rel_direction_right=False, + ) == { ('eni-0e106a07c15ff7d14', TEST_ACCOUNT_ID), ('eni-0d9877f559c240362', TEST_ACCOUNT_ID), ('eni-04b4289e1be7634e4', TEST_ACCOUNT_ID), ('eni-04b4289e1be7634e4', TEST_ACCOUNT_ID), } - # Fetch relationships - result = neo4j_session.run( - """ - MATCH (n1:NetworkInterface)<-[:RESOURCE]-(n2:AWSAccount) RETURN n1.id, n2.id; - """, - ) - actual = { - (r['n1.id'], r['n2.id']) for r in result + # Assert NetworkInterface to Subnet rels exist + assert check_rels( + neo4j_session, + 'NetworkInterface', + 'id', + 'EC2Subnet', + 'id', + 'PART_OF_SUBNET', + rel_direction_right=True, + ) == { + ('eni-04b4289e1be7634e4', 'subnet-0fa10e76eeb24dbe7'), + ('eni-0d9877f559c240362', 'subnet-0fa10e76eeb24dbe7'), + ('eni-0e106a07c15ff7d14', 'subnet-0fa10e76eeb24dbe7'), } - - assert actual == expected_nodes + # TODO add test for security group to nic From 23879dcda32812b001639f8cdbc2c755b246d236 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 17 Jul 2023 10:11:12 -0700 Subject: [PATCH 4/9] Add test for sec group --- cartography/intel/aws/ec2/network_interfaces.py | 2 +- .../intel/aws/ec2/test_ec2_network_interfaces.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 211bca3ac..8080e9201 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -100,7 +100,7 @@ def transform_network_interface_data(data_list: List[Dict[str, Any]], region: st sg_list.append( { 'GroupId': group['GroupId'], - 'Id': network_interface_id, + 'NetworkInterfaceId': network_interface_id, }, ) diff --git a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py index 4e8ec01de..2780f3f78 100644 --- a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py +++ b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py @@ -94,4 +94,18 @@ def test_load_network_interfaces(mock_get_network_interfaces, neo4j_session): ('eni-0d9877f559c240362', 'subnet-0fa10e76eeb24dbe7'), ('eni-0e106a07c15ff7d14', 'subnet-0fa10e76eeb24dbe7'), } - # TODO add test for security group to nic + + # Assert NetworkInterface to security group rels exist + assert check_rels( + neo4j_session, + 'NetworkInterface', + 'id', + 'EC2SecurityGroup', + 'id', + 'MEMBER_OF_EC2_SECURITY_GROUP', + rel_direction_right=True, + ) == { + ('eni-04b4289e1be7634e4', 'sg-0e866e64db0c84705'), + ('eni-0d9877f559c240362', 'sg-0e866e64db0c84705'), + ('eni-0e106a07c15ff7d14', 'sg-0e866e64db0c84705'), + } From fa41d133c060231e7922afc8a354073043eb7751 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 17 Jul 2023 10:31:38 -0700 Subject: [PATCH 5/9] scoped analysis docs --- cartography/util.py | 13 +++++++++++++ docs/root/usage/tutorial.md | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cartography/util.py b/cartography/util.py index 588c519f3..2a002a42e 100644 --- a/cartography/util.py +++ b/cartography/util.py @@ -49,6 +49,14 @@ def run_analysis_job( common_job_parameters: Dict, package: str = 'cartography.data.jobs.analysis', ) -> None: + """ + Enriches existing graph data with analysis jobs. This is designed for use with the sync stage + cartography.intel.analysis. + Runs the queries in the given Python `package` directory (cartography.data.jobs.analysis by default) for the given + `filename`. All queries in this directory are intended to be run at the end of a full graph sync. As such, they are + not scoped to a single sub resource. That is they will apply to _all_ AWS accounts/_all_ GCP projects/_all_ Okta + organizations/etc. + """ GraphJob.run_from_json( neo4j_session, read_text( @@ -65,6 +73,11 @@ def run_scoped_analysis_job( neo4j_session: neo4j.Session, common_job_parameters: Dict, ) -> None: + """ + Enriches existing graph data scoped to a given sub resource - e.g. the current AWS account. + Runs the queries in the cartography.data.jobs.scoped_analysis directory for the given `filename`. View the queries + in cartography.data.jobs.scoped_analysis for specifics. + """ GraphJob.run_from_json( neo4j_session, read_text('cartography.data.jobs.scoped_analysis', filename), diff --git a/docs/root/usage/tutorial.md b/docs/root/usage/tutorial.md index f0cb52fa9..9e3d95a2f 100644 --- a/docs/root/usage/tutorial.md +++ b/docs/root/usage/tutorial.md @@ -150,7 +150,7 @@ If you want to learn more in depth about Neo4j and Cypher queries you can look a .. _data-augmentation: -Cartography adds custom attributes to nodes and relationships to point out security-related items of interest. Unless mentioned otherwise these data augmentation jobs are stored in `cartography/data/jobs/analysis`. Here is a summary of all of Cartography's custom attributes. +Cartography adds custom attributes to nodes and relationships to point out security-related items of interest. Data augmentation jobs meant to apply to the whole graph and run at the end of a sync are stored in `cartography/data/jobs/analysis`. Jobs that are meant to apply to a subset of the graph - say, the current AWS account - are stored in `cartography/data/jobs/scoped_analysis`. Here is a summary of all of Cartography's custom attributes. - `exposed_internet` indicates whether the asset is accessible to the public internet. From 3c7b0bd25b0798b71c00d48be0f970dd8c118031 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Fri, 21 Jul 2023 22:25:15 -0700 Subject: [PATCH 6/9] Remove unnecessary scoped analysis job --- .../data/jobs/scoped_analysis/__init__.py | 0 .../aws_ec2_subnet_membership.json | 38 ------------------- .../intel/aws/ec2/network_interfaces.py | 9 +---- .../models/aws/ec2/subnet_networkinterface.py | 36 ++++++++++++++++++ cartography/util.py | 18 --------- 5 files changed, 38 insertions(+), 63 deletions(-) delete mode 100644 cartography/data/jobs/scoped_analysis/__init__.py delete mode 100644 cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json diff --git a/cartography/data/jobs/scoped_analysis/__init__.py b/cartography/data/jobs/scoped_analysis/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json b/cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json deleted file mode 100644 index c732b5493..000000000 --- a/cartography/data/jobs/scoped_analysis/aws_ec2_subnet_membership.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "statements": [ - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(i:EC2Instance)-[:NETWORK_INTERFACE]-(:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) MERGE (i)-[r:PART_OF_SUBNET]->(s) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = $UPDATE_TAG", - "iterative": true, - "iterationsize": 100, - "__comment__": "Create (:EC2Instance)-[:PART_OF_SUBNET]->(:EC2Subnet) if (:EC2Instance)--(:NetworkInterface)--(:EC2Subnet)" - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:EC2Instance)-[r:PART_OF_SUBNET]-(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r", - "iterative": true, - "iterationsize": 100 - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(i:LoadBalancerV2)-[:NETWORK_INTERFACE]-(:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) MERGE (i)-[r:PART_OF_SUBNET]->(s) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = $UPDATE_TAG", - "iterative": true, - "iterationsize": 100, - "__comment__": "Creates (:LoadBalancerV2)-[:PART_OF_SUBNET]->(:EC2Subnet) if (:LoadBalancerV2)--(:NetworkInterface)--(:EC2Subnet)." - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:LoadBalancerV2)-[r:PART_OF_SUBNET]-(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r", - "iterative": true, - "iterationsize": 100 - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(i:LoadBalancer)-[:NETWORK_INTERFACE]-(:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) MERGE (i)-[r:PART_OF_SUBNET]->(s) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = $UPDATE_TAG", - "iterative": true, - "iterationsize": 100, - "__comment__": "Creates (:LoadBalancer)-[:PART_OF_SUBNET]->(:EC2Subnet) if (:LoadBalancer)--(:NetworkInterface)--(:EC2Subnet)." - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:LoadBalancer)-[r:PART_OF_SUBNET]-(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE r", - "iterative": true, - "iterationsize": 100 - } - ], - "name": "subnet membership" -} diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 8080e9201..f62496c3b 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -16,7 +16,6 @@ from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema from cartography.models.aws.ec2.subnet_networkinterface import EC2SubnetNetworkInterfaceSchema from cartography.util import aws_handle_regions -from cartography.util import run_scoped_analysis_job from cartography.util import timeit logger = logging.getLogger(__name__) @@ -110,6 +109,8 @@ def transform_network_interface_data(data_list: List[Dict[str, Any]], region: st { 'NetworkInterfaceId': network_interface_id, 'SubnetId': subnet_id, + 'ElbV1Id': elb_v1_id, + 'ElbV2Id': elb_v2_id, }, ) @@ -222,11 +223,6 @@ def load_network_data( load_security_group_network_interface(neo4j_session, sg_list, region, current_aws_account_id, update_tag) -@timeit -def load_subnet_membership_relations(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None: - run_scoped_analysis_job('aws_ec2_subnet_membership.json', neo4j_session, common_job_parameters) - - @timeit def cleanup_network_interfaces(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: GraphJob.from_node_schema(EC2NetworkInterfaceSchema(), common_job_parameters).run(neo4j_session) @@ -256,5 +252,4 @@ def sync_network_interfaces( ec2_network_data.subnet_list, ec2_network_data.sg_list, ) - load_subnet_membership_relations(neo4j_session, common_job_parameters) cleanup_network_interfaces(neo4j_session, common_job_parameters) diff --git a/cartography/models/aws/ec2/subnet_networkinterface.py b/cartography/models/aws/ec2/subnet_networkinterface.py index cd9005ba9..8a1adc89e 100644 --- a/cartography/models/aws/ec2/subnet_networkinterface.py +++ b/cartography/models/aws/ec2/subnet_networkinterface.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from cartography.models.aws.ec2.subnet_instance import EC2SubnetToAWSAccount +from cartography.models.aws.ec2.subnet_instance import EC2SubnetToEC2Instance from cartography.models.core.common import PropertyRef from cartography.models.core.nodes import CartographyNodeProperties from cartography.models.core.nodes import CartographyNodeSchema @@ -36,6 +37,38 @@ class EC2SubnetToNetworkInterface(CartographyRelSchema): properties: EC2SubnetToNetworkInterfaceRelProperties = EC2SubnetToNetworkInterfaceRelProperties() +@dataclass(frozen=True) +class EC2SubnetToLoadBalancerRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToLoadBalancer(CartographyRelSchema): + target_node_label: str = 'LoadBalancer' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('ElbV1Id')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2SubnetToLoadBalancerRelProperties = EC2SubnetToLoadBalancerRelProperties() + + +@dataclass(frozen=True) +class EC2SubnetToLoadBalancerV2RelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToLoadBalancerV2(CartographyRelSchema): + target_node_label: str = 'LoadBalancerV2' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('ElbV2Id')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2SubnetToLoadBalancerV2RelProperties = EC2SubnetToLoadBalancerV2RelProperties() + + @dataclass(frozen=True) class EC2SubnetNetworkInterfaceSchema(CartographyNodeSchema): """ @@ -47,5 +80,8 @@ class EC2SubnetNetworkInterfaceSchema(CartographyNodeSchema): other_relationships: OtherRelationships = OtherRelationships( [ EC2SubnetToNetworkInterface(), + EC2SubnetToEC2Instance(), + EC2SubnetToLoadBalancer(), + EC2SubnetToLoadBalancerV2(), ], ) diff --git a/cartography/util.py b/cartography/util.py index 2a002a42e..144f17fb3 100644 --- a/cartography/util.py +++ b/cartography/util.py @@ -68,24 +68,6 @@ def run_analysis_job( ) -def run_scoped_analysis_job( - filename: str, - neo4j_session: neo4j.Session, - common_job_parameters: Dict, -) -> None: - """ - Enriches existing graph data scoped to a given sub resource - e.g. the current AWS account. - Runs the queries in the cartography.data.jobs.scoped_analysis directory for the given `filename`. View the queries - in cartography.data.jobs.scoped_analysis for specifics. - """ - GraphJob.run_from_json( - neo4j_session, - read_text('cartography.data.jobs.scoped_analysis', filename), - common_job_parameters, - get_job_shortname(filename), - ) - - def run_analysis_and_ensure_deps( analysis_job_name: str, resource_dependencies: Set[str], From 1426cffa946f4ae3929cb8ed0259764cf3ca8718 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Mon, 11 Sep 2023 10:34:25 -0700 Subject: [PATCH 7/9] Add missing indexes --- cartography/models/aws/ec2/loadbalancerv2.py | 0 cartography/models/aws/ec2/networkinterface_instance.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 cartography/models/aws/ec2/loadbalancerv2.py diff --git a/cartography/models/aws/ec2/loadbalancerv2.py b/cartography/models/aws/ec2/loadbalancerv2.py new file mode 100644 index 000000000..e69de29bb diff --git a/cartography/models/aws/ec2/networkinterface_instance.py b/cartography/models/aws/ec2/networkinterface_instance.py index 8323cf5b4..cb1e2476d 100644 --- a/cartography/models/aws/ec2/networkinterface_instance.py +++ b/cartography/models/aws/ec2/networkinterface_instance.py @@ -19,10 +19,10 @@ class EC2NetworkInterfaceInstanceNodeProperties(CartographyNodeProperties): # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024 id: PropertyRef = PropertyRef('NetworkInterfaceId') status: PropertyRef = PropertyRef('Status') - mac_address: PropertyRef = PropertyRef('MacAddress') + mac_address: PropertyRef = PropertyRef('MacAddress', extra_index=True) description: PropertyRef = PropertyRef('Description') - private_dns_name: PropertyRef = PropertyRef('PrivateDnsName') - private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress') + private_dns_name: PropertyRef = PropertyRef('PrivateDnsName', extra_index=True) + private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) From 8bbff72d9c65432dc33c49bd834e4562fab9ea47 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Tue, 19 Sep 2023 09:13:15 -0700 Subject: [PATCH 8/9] Remove scoped analysis doc --- docs/root/usage/tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/usage/tutorial.md b/docs/root/usage/tutorial.md index 9e3d95a2f..731285a39 100644 --- a/docs/root/usage/tutorial.md +++ b/docs/root/usage/tutorial.md @@ -150,7 +150,7 @@ If you want to learn more in depth about Neo4j and Cypher queries you can look a .. _data-augmentation: -Cartography adds custom attributes to nodes and relationships to point out security-related items of interest. Data augmentation jobs meant to apply to the whole graph and run at the end of a sync are stored in `cartography/data/jobs/analysis`. Jobs that are meant to apply to a subset of the graph - say, the current AWS account - are stored in `cartography/data/jobs/scoped_analysis`. Here is a summary of all of Cartography's custom attributes. +Cartography adds custom attributes to nodes and relationships to point out security-related items of interest. Data augmentation jobs meant to apply to the whole graph and run at the end of a sync are stored in `cartography/data/jobs/analysis`. Here is a summary of all of Cartography's custom attributes. - `exposed_internet` indicates whether the asset is accessible to the public internet. From 01a368d2bfdfcc11b43fd039af9614076cf213f5 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Tue, 19 Sep 2023 09:28:12 -0700 Subject: [PATCH 9/9] Add missing indexes --- cartography/models/aws/ec2/networkinterfaces.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cartography/models/aws/ec2/networkinterfaces.py b/cartography/models/aws/ec2/networkinterfaces.py index f5e96adf7..a21d4ba85 100644 --- a/cartography/models/aws/ec2/networkinterfaces.py +++ b/cartography/models/aws/ec2/networkinterfaces.py @@ -23,19 +23,19 @@ class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties): id: PropertyRef = PropertyRef('NetworkInterfaceId') lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) description: PropertyRef = PropertyRef('Description') - mac_address: PropertyRef = PropertyRef('MacAddress') + mac_address: PropertyRef = PropertyRef('MacAddress', extra_index=True) private_dns_name: PropertyRef = PropertyRef('PrivateDnsName') - private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress') + private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) status: PropertyRef = PropertyRef('Status') # Properties only returned by describe-network-interfaces interface_type: PropertyRef = PropertyRef('InterfaceType') - public_ip: PropertyRef = PropertyRef('PublicIp') - requester_id: PropertyRef = PropertyRef('RequesterId') + public_ip: PropertyRef = PropertyRef('PublicIp', extra_index=True) + requester_id: PropertyRef = PropertyRef('RequesterId', extra_index=True) requester_managed: PropertyRef = PropertyRef('RequesterManaged') source_dest_check: PropertyRef = PropertyRef('SourceDestCheck') - subnetid: PropertyRef = PropertyRef('SubnetId') + subnetid: PropertyRef = PropertyRef('SubnetId', extra_index=True) @dataclass(frozen=True)